{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Using TensorFlow backend.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "WARNING:tensorflow:From /home/pritha/anaconda3/envs/linenv/lib/python3.7/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n", "Instructions for updating:\n", "Colocations handled automatically by placer.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/home/pritha/anaconda3/envs/linenv/lib/python3.7/site-packages/sklearn/externals/joblib/__init__.py:15: DeprecationWarning: sklearn.externals.joblib is deprecated in 0.21 and will be removed in 0.23. Please import this functionality directly from joblib, which can be installed with: pip install joblib. If this warning is raised when loading pickled models, you may need to re-serialize those models with scikit-learn 0.21+.\n", " warnings.warn(msg, category=DeprecationWarning)\n" ] } ], "source": [ "import warnings\n", "warnings.filterwarnings('ignore')\n", "import os\n", "os.environ[\"MKL_THREADING_LAYER\"] = \"GNU\"\n", "import inspect\n", "import logging\n", "import os\n", "import pandas as pd\n", "from csrank.util import setup_logging, print_dictionary\n", "from result_script import *\n", "\n", "from csrank.experiments import CHOICE_FUNCTIONS, lp_metric_dict\n", "from csrank.constants import CHOICE_FUNCTION\n", "import numpy as np" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "DIR_PATH = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))\n", "log_path = os.path.join(DIR_PATH, 'logs', 'results_choice.log')\n", "FOLDER = \"journalresults\"\n", "latex_path = os.path.join(DIR_PATH, FOLDER, 'choice_functions.tex')\n", "df_path_combined = os.path.join(DIR_PATH, FOLDER , \"ChoiceFunctions.csv\")\n", "\n", "setup_logging(log_path=log_path, level=logging.ERROR)\n", "logger = logging.getLogger('ResultParsing')\n", "datasets = ['synthetic_choice', 'mnist_choice', 'letor_choice', 'exp_choice']\n", "\n", "learning_problem = CHOICE_FUNCTION\n", "learning_model = learners_map[learning_problem]\n", "keys = list(lp_metric_dict[learning_problem].keys())\n", "metrics = ', '.join([x.lower() for x in keys])\n", "models = ['FETA-Net', 'FATE-Net', 'RankNet-Choice', 'PairwiseSVM', 'GeneralizedLinearModel', \"RandomGuessing\", \"FATE-Linear\", \"FETA-Linear\"]\n", "models_dict = dict(zip(CHOICE_FUNCTIONS, models))" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
job_iddatasetlearnerf1scoreprecisionrecallsubset01accuracyhammingaccuracyinformednessaucscoreaverageprecisionscore
33329Expedia_N_10feta_choice_eb7f0.18490.12340.53420.01580.77350.32630.69420.3628
42325Expedia_N_10feta_choice_eb7f0.18190.12140.52820.01610.77550.32340.69380.3623
47335Expedia_N_10feta_choice_eb7f0.18280.12220.52900.01640.77640.32540.69410.3638
1327Expedia_N_10feta_choice_eb7f0.18510.12170.55860.01320.75800.33200.69450.3622
48336Expedia_N_10feta_choice_eb7f0.18500.12240.54900.01490.76490.33100.69430.3640
37337Expedia_N_10feta_choice_zero_0f510.18500.12350.53650.01530.77290.32760.69470.3646
46192Expedia_N_10feta_choice_zero_0f510.18530.12390.53340.01600.77490.32680.69530.3629
2334Expedia_N_10feta_choice_zero_0f510.18520.12170.55870.01280.75850.33260.69380.3620
3328Expedia_N_10feta_choice_zero_0f510.18360.12160.55560.01500.75790.32950.69340.3613
49342Expedia_N_10feta_choice_zero_0f510.18640.12200.56700.01170.75520.33640.69620.3651
0326Expedia_N_10feta_choice_zero_17c70.18720.12230.56290.01120.74740.32110.68860.3539
34237Expedia_N_10feta_choice_zero_17c70.18650.11990.58460.00910.73220.32380.68800.3526
35239Expedia_N_10feta_choice_zero_17c70.18110.11940.54470.01310.76280.32370.72600.3662
36240Expedia_N_10feta_choice_zero_17c70.18130.12020.53650.01390.76810.32180.72580.3664
38238Expedia_N_10feta_choice_zero_17c70.18600.11800.60850.00890.70870.31690.68770.3552
\n", "
" ], "text/plain": [ " job_id dataset learner f1score precision recall \\\n", "33 329 Expedia_N_10 feta_choice_eb7f 0.1849 0.1234 0.5342 \n", "42 325 Expedia_N_10 feta_choice_eb7f 0.1819 0.1214 0.5282 \n", "47 335 Expedia_N_10 feta_choice_eb7f 0.1828 0.1222 0.5290 \n", "1 327 Expedia_N_10 feta_choice_eb7f 0.1851 0.1217 0.5586 \n", "48 336 Expedia_N_10 feta_choice_eb7f 0.1850 0.1224 0.5490 \n", "37 337 Expedia_N_10 feta_choice_zero_0f51 0.1850 0.1235 0.5365 \n", "46 192 Expedia_N_10 feta_choice_zero_0f51 0.1853 0.1239 0.5334 \n", "2 334 Expedia_N_10 feta_choice_zero_0f51 0.1852 0.1217 0.5587 \n", "3 328 Expedia_N_10 feta_choice_zero_0f51 0.1836 0.1216 0.5556 \n", "49 342 Expedia_N_10 feta_choice_zero_0f51 0.1864 0.1220 0.5670 \n", "0 326 Expedia_N_10 feta_choice_zero_17c7 0.1872 0.1223 0.5629 \n", "34 237 Expedia_N_10 feta_choice_zero_17c7 0.1865 0.1199 0.5846 \n", "35 239 Expedia_N_10 feta_choice_zero_17c7 0.1811 0.1194 0.5447 \n", "36 240 Expedia_N_10 feta_choice_zero_17c7 0.1813 0.1202 0.5365 \n", "38 238 Expedia_N_10 feta_choice_zero_17c7 0.1860 0.1180 0.6085 \n", "\n", " subset01accuracy hammingaccuracy informedness aucscore \\\n", "33 0.0158 0.7735 0.3263 0.6942 \n", "42 0.0161 0.7755 0.3234 0.6938 \n", "47 0.0164 0.7764 0.3254 0.6941 \n", "1 0.0132 0.7580 0.3320 0.6945 \n", "48 0.0149 0.7649 0.3310 0.6943 \n", "37 0.0153 0.7729 0.3276 0.6947 \n", "46 0.0160 0.7749 0.3268 0.6953 \n", "2 0.0128 0.7585 0.3326 0.6938 \n", "3 0.0150 0.7579 0.3295 0.6934 \n", "49 0.0117 0.7552 0.3364 0.6962 \n", "0 0.0112 0.7474 0.3211 0.6886 \n", "34 0.0091 0.7322 0.3238 0.6880 \n", "35 0.0131 0.7628 0.3237 0.7260 \n", "36 0.0139 0.7681 0.3218 0.7258 \n", "38 0.0089 0.7087 0.3169 0.6877 \n", "\n", " averageprecisionscore \n", "33 0.3628 \n", "42 0.3623 \n", "47 0.3638 \n", "1 0.3622 \n", "48 0.3640 \n", "37 0.3646 \n", "46 0.3629 \n", "2 0.3620 \n", "3 0.3613 \n", "49 0.3651 \n", "0 0.3539 \n", "34 0.3526 \n", "35 0.3662 \n", "36 0.3664 \n", "38 0.3552 " ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "d = datasets[-1]\n", "df, cols = get_results_for_dataset(d, logger, learning_problem, False)\n", "df = df.sort_values(by=['dataset', 'learner'], ascending=[True, True])\n", "df = df[df['learner'].str.contains('feta_choice')]\n", "df.sort_values(by='learner')" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
DatasetChoiceModelF1ScorePrecisionRecallSubset01AccuracyHammingaccuracyInformednessAucscoreAverageprecisionscore
0Expedia_N_10fate_choice_736f0.198±0.0060.133±0.0050.546±0.0160.017±0.0020.782±0.0100.346±0.0100.707±0.0070.378±0.008
1Expedia_N_10fatelinear_choice_e98a0.177±0.0060.119±0.0040.545±0.0260.020±0.0020.763±0.0140.328±0.0120.700±0.0070.372±0.009
2Expedia_N_10feta_choice_eb7f0.184±0.0010.122±0.0010.540±0.0130.015±0.0010.770±0.0080.328±0.0040.694±0.0000.363±0.001
3Expedia_N_10feta_choice_zero_0f510.185±0.0010.123±0.0010.550±0.0150.014±0.0020.764±0.0090.331±0.0040.695±0.0010.363±0.002
4Expedia_N_10feta_choice_zero_17c70.184±0.0030.120±0.0020.567±0.0290.011±0.0020.744±0.0240.321±0.0030.703±0.0210.359±0.007
5Expedia_N_10fetalinear_choice_6b8c0.179±0.0070.121±0.0060.539±0.0110.020±0.0020.765±0.0150.324±0.0060.696±0.0070.367±0.010
6Expedia_N_10glm_choice_3de10.107±0.0010.059±0.0010.992±0.0130.000±0.0000.069±0.0180.004±0.0070.503±0.1020.192±0.050
7Expedia_N_10random_choice_55690.106±0.0000.058±0.0001.000±0.0000.000±0.0000.058±0.0000.000±0.0000.500±0.0000.058±0.000
8Expedia_N_10ranknet_choice_d20f0.167±0.0170.101±0.0120.638±0.0460.003±0.0010.650±0.0620.278±0.0340.716±0.0060.363±0.006
9Expedia_N_10ranksvm_choice_03910.129±0.0170.077±0.0130.703±0.1490.004±0.0020.481±0.2270.165±0.0970.680±0.0510.321±0.047
\n", "
" ], "text/plain": [ " Dataset ChoiceModel F1Score Precision \\\n", "0 Expedia_N_10 fate_choice_736f 0.198±0.006 0.133±0.005 \n", "1 Expedia_N_10 fatelinear_choice_e98a 0.177±0.006 0.119±0.004 \n", "2 Expedia_N_10 feta_choice_eb7f 0.184±0.001 0.122±0.001 \n", "3 Expedia_N_10 feta_choice_zero_0f51 0.185±0.001 0.123±0.001 \n", "4 Expedia_N_10 feta_choice_zero_17c7 0.184±0.003 0.120±0.002 \n", "5 Expedia_N_10 fetalinear_choice_6b8c 0.179±0.007 0.121±0.006 \n", "6 Expedia_N_10 glm_choice_3de1 0.107±0.001 0.059±0.001 \n", "7 Expedia_N_10 random_choice_5569 0.106±0.000 0.058±0.000 \n", "8 Expedia_N_10 ranknet_choice_d20f 0.167±0.017 0.101±0.012 \n", "9 Expedia_N_10 ranksvm_choice_0391 0.129±0.017 0.077±0.013 \n", "\n", " Recall Subset01Accuracy Hammingaccuracy Informedness Aucscore \\\n", "0 0.546±0.016 0.017±0.002 0.782±0.010 0.346±0.010 0.707±0.007 \n", "1 0.545±0.026 0.020±0.002 0.763±0.014 0.328±0.012 0.700±0.007 \n", "2 0.540±0.013 0.015±0.001 0.770±0.008 0.328±0.004 0.694±0.000 \n", "3 0.550±0.015 0.014±0.002 0.764±0.009 0.331±0.004 0.695±0.001 \n", "4 0.567±0.029 0.011±0.002 0.744±0.024 0.321±0.003 0.703±0.021 \n", "5 0.539±0.011 0.020±0.002 0.765±0.015 0.324±0.006 0.696±0.007 \n", "6 0.992±0.013 0.000±0.000 0.069±0.018 0.004±0.007 0.503±0.102 \n", "7 1.000±0.000 0.000±0.000 0.058±0.000 0.000±0.000 0.500±0.000 \n", "8 0.638±0.046 0.003±0.001 0.650±0.062 0.278±0.034 0.716±0.006 \n", "9 0.703±0.149 0.004±0.002 0.481±0.227 0.165±0.097 0.680±0.051 \n", "\n", " Averageprecisionscore \n", "0 0.378±0.008 \n", "1 0.372±0.009 \n", "2 0.363±0.001 \n", "3 0.363±0.002 \n", "4 0.359±0.007 \n", "5 0.367±0.010 \n", "6 0.192±0.050 \n", "7 0.058±0.000 \n", "8 0.363±0.006 \n", "9 0.321±0.047 " ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df = get_combined_results(d, logger, learning_problem, False)\n", "df" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "import re\n", "def get_val(val):\n", " vals = [float(x) for x in re.findall(r\"[-+]?\\d*\\.\\d+|\\d+\", val)]\n", " if len(vals)==1:\n", " x = [vals[0], vals[0]-0.0]\n", " else:\n", " x = [vals[0], vals[0] - vals[1]]\n", " return x\n", "def create_final_result(dataset, dataset_function=get_combined_results ,latex_row=False):\n", " df_full = dataset_function(dataset, logger, learning_problem, latex_row=latex_row)\n", " data = []\n", " for dataset, df in df_full.groupby(['Dataset']):\n", " for m in CHOICE_FUNCTIONS:\n", " row = df[df[learning_model].str.contains(m)].values\n", " onerow = None\n", " if len(row) > 1:\n", " if dataset_function==get_combined_results:\n", " values = np.array([get_val(val[2]) for val in row])\n", " else:\n", " values = np.array([[val[2], val[2] - val[7]] for val in row])\n", " maxi = np.where(values[:,0] == values[:,0][np.argmax(values[:,0])])[0][0]\n", " logger.error(\"dataset {} model {}, vals {}, maxi {}\".format(dataset, row[:, 1], values, maxi))\n", " row = row[maxi]\n", " row[1] = models_dict[m]\n", " onerow = row\n", "\n", " elif len(row)==1:\n", " row[0][1] = models_dict[m]\n", " onerow = row[0]\n", " if onerow is not None:\n", " onerow[0] = get_dataset_name(onerow[0])\n", " data.append(onerow)\n", " columns = df_full.columns\n", " dataframe = pd.DataFrame(data, columns=columns)\n", " dataframe = dataframe.sort_values(by=[columns[0], columns[2]], ascending=[True, False])\n", " return dataframe" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
DatasetChoiceModelF1ScorePrecisionRecallSubset01AccuracyHammingaccuracyInformednessAucscoreAverageprecisionscore
1Expedia 10 ObjectsFATE-Net0.198±0.0060.133±0.0050.546±0.0160.017±0.0020.782±0.0100.346±0.0100.707±0.0070.378±0.008
0Expedia 10 ObjectsFETA-Net0.185±0.0010.123±0.0010.550±0.0150.014±0.0020.764±0.0090.331±0.0040.695±0.0010.363±0.002
7Expedia 10 ObjectsFETA-Linear0.179±0.0070.121±0.0060.539±0.0110.020±0.0020.765±0.0150.324±0.0060.696±0.0070.367±0.010
6Expedia 10 ObjectsFATE-Linear0.177±0.0060.119±0.0040.545±0.0260.020±0.0020.763±0.0140.328±0.0120.700±0.0070.372±0.009
2Expedia 10 ObjectsRankNet-Choice0.167±0.0170.101±0.0120.638±0.0460.003±0.0010.650±0.0620.278±0.0340.716±0.0060.363±0.006
3Expedia 10 ObjectsPairwiseSVM0.129±0.0170.077±0.0130.703±0.1490.004±0.0020.481±0.2270.165±0.0970.680±0.0510.321±0.047
4Expedia 10 ObjectsGeneralizedLinearModel0.107±0.0010.059±0.0010.992±0.0130.000±0.0000.069±0.0180.004±0.0070.503±0.1020.192±0.050
5Expedia 10 ObjectsRandomGuessing0.106±0.0000.058±0.0001.000±0.0000.000±0.0000.058±0.0000.000±0.0000.500±0.0000.058±0.000
\n", "
" ], "text/plain": [ " Dataset ChoiceModel F1Score Precision \\\n", "1 Expedia 10 Objects FATE-Net 0.198±0.006 0.133±0.005 \n", "0 Expedia 10 Objects FETA-Net 0.185±0.001 0.123±0.001 \n", "7 Expedia 10 Objects FETA-Linear 0.179±0.007 0.121±0.006 \n", "6 Expedia 10 Objects FATE-Linear 0.177±0.006 0.119±0.004 \n", "2 Expedia 10 Objects RankNet-Choice 0.167±0.017 0.101±0.012 \n", "3 Expedia 10 Objects PairwiseSVM 0.129±0.017 0.077±0.013 \n", "4 Expedia 10 Objects GeneralizedLinearModel 0.107±0.001 0.059±0.001 \n", "5 Expedia 10 Objects RandomGuessing 0.106±0.000 0.058±0.000 \n", "\n", " Recall Subset01Accuracy Hammingaccuracy Informedness Aucscore \\\n", "1 0.546±0.016 0.017±0.002 0.782±0.010 0.346±0.010 0.707±0.007 \n", "0 0.550±0.015 0.014±0.002 0.764±0.009 0.331±0.004 0.695±0.001 \n", "7 0.539±0.011 0.020±0.002 0.765±0.015 0.324±0.006 0.696±0.007 \n", "6 0.545±0.026 0.020±0.002 0.763±0.014 0.328±0.012 0.700±0.007 \n", "2 0.638±0.046 0.003±0.001 0.650±0.062 0.278±0.034 0.716±0.006 \n", "3 0.703±0.149 0.004±0.002 0.481±0.227 0.165±0.097 0.680±0.051 \n", "4 0.992±0.013 0.000±0.000 0.069±0.018 0.004±0.007 0.503±0.102 \n", "5 1.000±0.000 0.000±0.000 0.058±0.000 0.000±0.000 0.500±0.000 \n", "\n", " Averageprecisionscore \n", "1 0.378±0.008 \n", "0 0.363±0.002 \n", "7 0.367±0.010 \n", "6 0.372±0.009 \n", "2 0.363±0.006 \n", "3 0.321±0.047 \n", "4 0.192±0.050 \n", "5 0.058±0.000 " ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df = create_final_result(d, latex_row=False)\n", "df" ] }, { "cell_type": "code", "execution_count": 8, "metadata": { "scrolled": true }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
DatasetChoiceModelF1ScorePrecisionRecallSubset01AccuracyHammingaccuracyInformednessAucscoreAverageprecisionscore
0ParetoFETA-Net0.942±0.0080.938±0.0070.967±0.0130.680±0.0280.985±0.0020.956±0.0120.999±0.0000.996±0.000
1ParetoFATE-Net0.913±0.0090.919±0.0150.926±0.0050.506±0.0370.975±0.0030.911±0.0060.996±0.0010.984±0.003
2ParetoFETA-Linear0.673±0.0010.697±0.0230.747±0.0230.064±0.0070.913±0.0030.694±0.0150.955±0.0000.865±0.000
3ParetoFATE-Linear0.673±0.0000.683±0.0190.761±0.0180.059±0.0050.911±0.0030.704±0.0120.955±0.0000.865±0.000
4ParetoRankNet-Choice0.612±0.0070.624±0.0260.772±0.0290.060±0.0100.877±0.0110.672±0.0140.971±0.0060.891±0.019
5ParetoPairwiseSVM0.588±0.0010.596±0.0120.756±0.0150.044±0.0030.866±0.0050.646±0.0070.956±0.0000.865±0.000
6ParetoGeneralizedLinearModel0.565±0.0410.579±0.0450.721±0.0490.038±0.0120.859±0.0180.609±0.0570.935±0.0380.834±0.055
7ParetoRandomGuessing0.232±0.0000.133±0.0001.000±0.0000.000±0.0000.133±0.0000.000±0.0000.500±0.0000.133±0.000
8ModeFATE-Net0.976±0.0010.980±0.0020.979±0.0040.883±0.0100.978±0.0010.961±0.0020.992±0.0010.991±0.002
9ModeFETA-Net0.809±0.0050.742±0.0030.962±0.0090.311±0.0320.809±0.0040.695±0.0090.981±0.0060.980±0.006
10ModeFATE-Linear0.597±0.0010.444±0.0020.992±0.0050.003±0.0000.447±0.0040.007±0.0060.517±0.0020.573±0.002
11ModeFETA-Linear0.597±0.0010.443±0.0010.996±0.0040.003±0.0000.445±0.0010.003±0.0020.516±0.0010.573±0.001
12ModeRankNet-Choice0.597±0.0000.442±0.0001.000±0.0000.003±0.0000.442±0.0000.000±0.0000.503±0.0020.563±0.002
13ModePairwiseSVM0.597±0.0000.442±0.0000.999±0.0020.003±0.0000.443±0.0000.000±0.0000.509±0.0060.569±0.004
14ModeGeneralizedLinearModel0.597±0.0000.442±0.0000.999±0.0010.003±0.0000.443±0.0000.000±0.0000.497±0.0040.561±0.002
15ModeRandomGuessing0.597±0.0000.442±0.0001.000±0.0000.003±0.0000.442±0.0000.000±0.0000.500±0.0000.442±0.000
16UniqueFATE-Net0.973±0.0040.975±0.0020.977±0.0070.848±0.0210.980±0.0030.960±0.0060.995±0.0010.992±0.001
17UniqueFETA-Net0.963±0.0030.962±0.0060.975±0.0040.814±0.0200.972±0.0030.945±0.0050.992±0.0010.989±0.001
18UniquePairwiseSVM0.562±0.0010.405±0.0000.999±0.0020.000±0.0000.405±0.0010.000±0.0000.511±0.0060.553±0.005
19UniqueFATE-Linear0.562±0.0010.405±0.0010.999±0.0020.001±0.0000.406±0.0020.001±0.0030.506±0.0070.560±0.007
20UniqueRankNet-Choice0.562±0.0000.405±0.0001.000±0.0000.000±0.0000.405±0.0000.000±0.0000.504±0.0010.538±0.001
21UniqueGeneralizedLinearModel0.562±0.0000.405±0.0001.000±0.0000.000±0.0000.405±0.0000.000±0.0000.508±0.0040.542±0.002
22UniqueRandomGuessing0.562±0.0000.405±0.0001.000±0.0000.000±0.0000.405±0.0000.000±0.0000.500±0.0000.405±0.000
23UniqueFETA-Linear0.344±0.1260.449±0.0460.406±0.3380.004±0.0030.533±0.0760.032±0.0400.524±0.0190.524±0.026
24MQ2007 10 ObjectsFETA-Linear0.452±0.0220.372±0.0360.837±0.0490.001±0.0020.526±0.0490.231±0.0350.694±0.0050.540±0.022
25MQ2007 10 ObjectsFATE-Linear0.452±0.0210.362±0.0250.865±0.0440.001±0.0020.504±0.0320.212±0.0210.695±0.0060.540±0.021
26MQ2007 10 ObjectsFETA-Net0.452±0.0190.369±0.0260.838±0.0270.000±0.0000.529±0.0240.236±0.0190.690±0.0080.534±0.020
27MQ2007 10 ObjectsPairwiseSVM0.450±0.0180.365±0.0190.857±0.0310.000±0.0000.507±0.0300.216±0.0260.696±0.0070.535±0.028
28MQ2007 10 ObjectsFATE-Net0.429±0.0190.378±0.0210.705±0.0650.001±0.0020.575±0.0250.211±0.0190.653±0.0070.489±0.015
29MQ2007 10 ObjectsGeneralizedLinearModel0.428±0.0210.317±0.0220.965±0.0370.001±0.0020.358±0.0390.058±0.0290.614±0.0090.465±0.021
.................................
34MQ2007 5 ObjectsPairwiseSVM0.444±0.0220.344±0.0290.917±0.0310.000±0.0000.444±0.0430.161±0.0280.699±0.0040.540±0.022
35MQ2007 5 ObjectsFATE-Net0.436±0.0140.366±0.0230.759±0.0340.000±0.0000.542±0.0190.211±0.0200.645±0.0160.477±0.018
36MQ2007 5 ObjectsGeneralizedLinearModel0.427±0.0220.316±0.0230.973±0.0180.001±0.0020.350±0.0350.051±0.0190.613±0.0120.465±0.026
37MQ2007 5 ObjectsRandomGuessing0.421±0.0210.306±0.0201.000±0.0000.001±0.0020.306±0.0200.000±0.0000.500±0.0000.306±0.020
38MQ2007 5 ObjectsRankNet-Choice0.408±0.0140.354±0.0270.698±0.0500.000±0.0000.529±0.0290.167±0.0140.613±0.0110.451±0.024
39MQ2007 5 ObjectsFETA-Net0.40100.40000.53500.00000.61100.19100.63900.4850
40MQ2008 10 ObjectsPairwiseSVM0.527±0.0220.446±0.0290.846±0.0410.042±0.0220.645±0.0250.428±0.0150.786±0.0180.655±0.026
41MQ2008 10 ObjectsFATE-Linear0.517±0.0300.468±0.0320.772±0.0620.037±0.0090.666±0.0300.413±0.0340.805±0.0340.661±0.028
42MQ2008 10 ObjectsFETA-Linear0.513±0.0290.466±0.0530.767±0.0630.043±0.0110.655±0.0630.396±0.0600.772±0.0280.596±0.047
43MQ2008 10 ObjectsGeneralizedLinearModel0.493±0.0280.387±0.0380.901±0.0690.014±0.0100.545±0.0620.311±0.0610.739±0.0190.597±0.028
44MQ2008 10 ObjectsFATE-Net0.469±0.0390.454±0.0320.654±0.0970.032±0.0200.671±0.0220.343±0.0500.751±0.0350.609±0.042
45MQ2008 10 ObjectsRandomGuessing0.424±0.0210.298±0.0201.000±0.0000.000±0.0000.298±0.0200.000±0.0000.500±0.0000.298±0.020
46MQ2008 10 ObjectsFETA-Net0.401±0.0490.415±0.0120.521±0.1460.017±0.0130.667±0.0350.251±0.0530.711±0.0230.565±0.050
47MQ2008 10 ObjectsRankNet-Choice0.365±0.0310.452±0.0440.399±0.0540.021±0.0080.693±0.0180.229±0.0410.712±0.0200.581±0.028
48MQ2008 5 ObjectsFATE-Linear0.527±0.0240.447±0.0370.851±0.0500.028±0.0210.639±0.0290.430±0.0240.806±0.0290.660±0.018
49MQ2008 5 ObjectsPairwiseSVM0.524±0.0230.438±0.0390.866±0.0450.037±0.0130.627±0.0340.418±0.0250.794±0.0140.662±0.024
50MQ2008 5 ObjectsGeneralizedLinearModel0.497±0.0290.392±0.0330.893±0.0250.021±0.0240.567±0.0380.337±0.0590.742±0.0380.606±0.041
51MQ2008 5 ObjectsFETA-Linear0.493±0.0430.413±0.0680.853±0.0960.029±0.0220.569±0.1440.330±0.1760.743±0.0610.522±0.063
52MQ2008 5 ObjectsFATE-Net0.485±0.0270.442±0.0470.710±0.0350.031±0.0150.649±0.0320.355±0.0490.744±0.0220.615±0.021
53MQ2008 5 ObjectsFETA-Net0.479±0.0300.460±0.0290.647±0.0490.023±0.0140.677±0.0120.354±0.0400.746±0.0290.612±0.032
54MQ2008 5 ObjectsRankNet-Choice0.458±0.0340.462±0.0180.598±0.0740.034±0.0120.682±0.0200.330±0.0470.737±0.0310.602±0.018
55MQ2008 5 ObjectsRandomGuessing0.424±0.0210.298±0.0201.000±0.0000.000±0.0000.298±0.0200.000±0.0000.500±0.0000.298±0.020
56Expedia 10 ObjectsFATE-Net0.198±0.0060.133±0.0050.546±0.0160.017±0.0020.782±0.0100.346±0.0100.707±0.0070.378±0.008
57Expedia 10 ObjectsFETA-Net0.185±0.0010.123±0.0010.550±0.0150.014±0.0020.764±0.0090.331±0.0040.695±0.0010.363±0.002
58Expedia 10 ObjectsFETA-Linear0.179±0.0070.121±0.0060.539±0.0110.020±0.0020.765±0.0150.324±0.0060.696±0.0070.367±0.010
59Expedia 10 ObjectsFATE-Linear0.177±0.0060.119±0.0040.545±0.0260.020±0.0020.763±0.0140.328±0.0120.700±0.0070.372±0.009
60Expedia 10 ObjectsRankNet-Choice0.167±0.0170.101±0.0120.638±0.0460.003±0.0010.650±0.0620.278±0.0340.716±0.0060.363±0.006
61Expedia 10 ObjectsPairwiseSVM0.129±0.0170.077±0.0130.703±0.1490.004±0.0020.481±0.2270.165±0.0970.680±0.0510.321±0.047
62Expedia 10 ObjectsGeneralizedLinearModel0.107±0.0010.059±0.0010.992±0.0130.000±0.0000.069±0.0180.004±0.0070.503±0.1020.192±0.050
63Expedia 10 ObjectsRandomGuessing0.106±0.0000.058±0.0001.000±0.0000.000±0.0000.058±0.0000.000±0.0000.500±0.0000.058±0.000
\n", "

64 rows × 10 columns

\n", "
" ], "text/plain": [ " Dataset ChoiceModel F1Score Precision \\\n", "0 Pareto FETA-Net 0.942±0.008 0.938±0.007 \n", "1 Pareto FATE-Net 0.913±0.009 0.919±0.015 \n", "2 Pareto FETA-Linear 0.673±0.001 0.697±0.023 \n", "3 Pareto FATE-Linear 0.673±0.000 0.683±0.019 \n", "4 Pareto RankNet-Choice 0.612±0.007 0.624±0.026 \n", "5 Pareto PairwiseSVM 0.588±0.001 0.596±0.012 \n", "6 Pareto GeneralizedLinearModel 0.565±0.041 0.579±0.045 \n", "7 Pareto RandomGuessing 0.232±0.000 0.133±0.000 \n", "8 Mode FATE-Net 0.976±0.001 0.980±0.002 \n", "9 Mode FETA-Net 0.809±0.005 0.742±0.003 \n", "10 Mode FATE-Linear 0.597±0.001 0.444±0.002 \n", "11 Mode FETA-Linear 0.597±0.001 0.443±0.001 \n", "12 Mode RankNet-Choice 0.597±0.000 0.442±0.000 \n", "13 Mode PairwiseSVM 0.597±0.000 0.442±0.000 \n", "14 Mode GeneralizedLinearModel 0.597±0.000 0.442±0.000 \n", "15 Mode RandomGuessing 0.597±0.000 0.442±0.000 \n", "16 Unique FATE-Net 0.973±0.004 0.975±0.002 \n", "17 Unique FETA-Net 0.963±0.003 0.962±0.006 \n", "18 Unique PairwiseSVM 0.562±0.001 0.405±0.000 \n", "19 Unique FATE-Linear 0.562±0.001 0.405±0.001 \n", "20 Unique RankNet-Choice 0.562±0.000 0.405±0.000 \n", "21 Unique GeneralizedLinearModel 0.562±0.000 0.405±0.000 \n", "22 Unique RandomGuessing 0.562±0.000 0.405±0.000 \n", "23 Unique FETA-Linear 0.344±0.126 0.449±0.046 \n", "24 MQ2007 10 Objects FETA-Linear 0.452±0.022 0.372±0.036 \n", "25 MQ2007 10 Objects FATE-Linear 0.452±0.021 0.362±0.025 \n", "26 MQ2007 10 Objects FETA-Net 0.452±0.019 0.369±0.026 \n", "27 MQ2007 10 Objects PairwiseSVM 0.450±0.018 0.365±0.019 \n", "28 MQ2007 10 Objects FATE-Net 0.429±0.019 0.378±0.021 \n", "29 MQ2007 10 Objects GeneralizedLinearModel 0.428±0.021 0.317±0.022 \n", ".. ... ... ... ... \n", "34 MQ2007 5 Objects PairwiseSVM 0.444±0.022 0.344±0.029 \n", "35 MQ2007 5 Objects FATE-Net 0.436±0.014 0.366±0.023 \n", "36 MQ2007 5 Objects GeneralizedLinearModel 0.427±0.022 0.316±0.023 \n", "37 MQ2007 5 Objects RandomGuessing 0.421±0.021 0.306±0.020 \n", "38 MQ2007 5 Objects RankNet-Choice 0.408±0.014 0.354±0.027 \n", "39 MQ2007 5 Objects FETA-Net 0.4010 0.4000 \n", "40 MQ2008 10 Objects PairwiseSVM 0.527±0.022 0.446±0.029 \n", "41 MQ2008 10 Objects FATE-Linear 0.517±0.030 0.468±0.032 \n", "42 MQ2008 10 Objects FETA-Linear 0.513±0.029 0.466±0.053 \n", "43 MQ2008 10 Objects GeneralizedLinearModel 0.493±0.028 0.387±0.038 \n", "44 MQ2008 10 Objects FATE-Net 0.469±0.039 0.454±0.032 \n", "45 MQ2008 10 Objects RandomGuessing 0.424±0.021 0.298±0.020 \n", "46 MQ2008 10 Objects FETA-Net 0.401±0.049 0.415±0.012 \n", "47 MQ2008 10 Objects RankNet-Choice 0.365±0.031 0.452±0.044 \n", "48 MQ2008 5 Objects FATE-Linear 0.527±0.024 0.447±0.037 \n", "49 MQ2008 5 Objects PairwiseSVM 0.524±0.023 0.438±0.039 \n", "50 MQ2008 5 Objects GeneralizedLinearModel 0.497±0.029 0.392±0.033 \n", "51 MQ2008 5 Objects FETA-Linear 0.493±0.043 0.413±0.068 \n", "52 MQ2008 5 Objects FATE-Net 0.485±0.027 0.442±0.047 \n", "53 MQ2008 5 Objects FETA-Net 0.479±0.030 0.460±0.029 \n", "54 MQ2008 5 Objects RankNet-Choice 0.458±0.034 0.462±0.018 \n", "55 MQ2008 5 Objects RandomGuessing 0.424±0.021 0.298±0.020 \n", "56 Expedia 10 Objects FATE-Net 0.198±0.006 0.133±0.005 \n", "57 Expedia 10 Objects FETA-Net 0.185±0.001 0.123±0.001 \n", "58 Expedia 10 Objects FETA-Linear 0.179±0.007 0.121±0.006 \n", "59 Expedia 10 Objects FATE-Linear 0.177±0.006 0.119±0.004 \n", "60 Expedia 10 Objects RankNet-Choice 0.167±0.017 0.101±0.012 \n", "61 Expedia 10 Objects PairwiseSVM 0.129±0.017 0.077±0.013 \n", "62 Expedia 10 Objects GeneralizedLinearModel 0.107±0.001 0.059±0.001 \n", "63 Expedia 10 Objects RandomGuessing 0.106±0.000 0.058±0.000 \n", "\n", " Recall Subset01Accuracy Hammingaccuracy Informedness Aucscore \\\n", "0 0.967±0.013 0.680±0.028 0.985±0.002 0.956±0.012 0.999±0.000 \n", "1 0.926±0.005 0.506±0.037 0.975±0.003 0.911±0.006 0.996±0.001 \n", "2 0.747±0.023 0.064±0.007 0.913±0.003 0.694±0.015 0.955±0.000 \n", "3 0.761±0.018 0.059±0.005 0.911±0.003 0.704±0.012 0.955±0.000 \n", "4 0.772±0.029 0.060±0.010 0.877±0.011 0.672±0.014 0.971±0.006 \n", "5 0.756±0.015 0.044±0.003 0.866±0.005 0.646±0.007 0.956±0.000 \n", "6 0.721±0.049 0.038±0.012 0.859±0.018 0.609±0.057 0.935±0.038 \n", "7 1.000±0.000 0.000±0.000 0.133±0.000 0.000±0.000 0.500±0.000 \n", "8 0.979±0.004 0.883±0.010 0.978±0.001 0.961±0.002 0.992±0.001 \n", "9 0.962±0.009 0.311±0.032 0.809±0.004 0.695±0.009 0.981±0.006 \n", "10 0.992±0.005 0.003±0.000 0.447±0.004 0.007±0.006 0.517±0.002 \n", "11 0.996±0.004 0.003±0.000 0.445±0.001 0.003±0.002 0.516±0.001 \n", "12 1.000±0.000 0.003±0.000 0.442±0.000 0.000±0.000 0.503±0.002 \n", "13 0.999±0.002 0.003±0.000 0.443±0.000 0.000±0.000 0.509±0.006 \n", "14 0.999±0.001 0.003±0.000 0.443±0.000 0.000±0.000 0.497±0.004 \n", "15 1.000±0.000 0.003±0.000 0.442±0.000 0.000±0.000 0.500±0.000 \n", "16 0.977±0.007 0.848±0.021 0.980±0.003 0.960±0.006 0.995±0.001 \n", "17 0.975±0.004 0.814±0.020 0.972±0.003 0.945±0.005 0.992±0.001 \n", "18 0.999±0.002 0.000±0.000 0.405±0.001 0.000±0.000 0.511±0.006 \n", "19 0.999±0.002 0.001±0.000 0.406±0.002 0.001±0.003 0.506±0.007 \n", "20 1.000±0.000 0.000±0.000 0.405±0.000 0.000±0.000 0.504±0.001 \n", "21 1.000±0.000 0.000±0.000 0.405±0.000 0.000±0.000 0.508±0.004 \n", "22 1.000±0.000 0.000±0.000 0.405±0.000 0.000±0.000 0.500±0.000 \n", "23 0.406±0.338 0.004±0.003 0.533±0.076 0.032±0.040 0.524±0.019 \n", "24 0.837±0.049 0.001±0.002 0.526±0.049 0.231±0.035 0.694±0.005 \n", "25 0.865±0.044 0.001±0.002 0.504±0.032 0.212±0.021 0.695±0.006 \n", "26 0.838±0.027 0.000±0.000 0.529±0.024 0.236±0.019 0.690±0.008 \n", "27 0.857±0.031 0.000±0.000 0.507±0.030 0.216±0.026 0.696±0.007 \n", "28 0.705±0.065 0.001±0.002 0.575±0.025 0.211±0.019 0.653±0.007 \n", "29 0.965±0.037 0.001±0.002 0.358±0.039 0.058±0.029 0.614±0.009 \n", ".. ... ... ... ... ... \n", "34 0.917±0.031 0.000±0.000 0.444±0.043 0.161±0.028 0.699±0.004 \n", "35 0.759±0.034 0.000±0.000 0.542±0.019 0.211±0.020 0.645±0.016 \n", "36 0.973±0.018 0.001±0.002 0.350±0.035 0.051±0.019 0.613±0.012 \n", "37 1.000±0.000 0.001±0.002 0.306±0.020 0.000±0.000 0.500±0.000 \n", "38 0.698±0.050 0.000±0.000 0.529±0.029 0.167±0.014 0.613±0.011 \n", "39 0.5350 0.0000 0.6110 0.1910 0.6390 \n", "40 0.846±0.041 0.042±0.022 0.645±0.025 0.428±0.015 0.786±0.018 \n", "41 0.772±0.062 0.037±0.009 0.666±0.030 0.413±0.034 0.805±0.034 \n", "42 0.767±0.063 0.043±0.011 0.655±0.063 0.396±0.060 0.772±0.028 \n", "43 0.901±0.069 0.014±0.010 0.545±0.062 0.311±0.061 0.739±0.019 \n", "44 0.654±0.097 0.032±0.020 0.671±0.022 0.343±0.050 0.751±0.035 \n", "45 1.000±0.000 0.000±0.000 0.298±0.020 0.000±0.000 0.500±0.000 \n", "46 0.521±0.146 0.017±0.013 0.667±0.035 0.251±0.053 0.711±0.023 \n", "47 0.399±0.054 0.021±0.008 0.693±0.018 0.229±0.041 0.712±0.020 \n", "48 0.851±0.050 0.028±0.021 0.639±0.029 0.430±0.024 0.806±0.029 \n", "49 0.866±0.045 0.037±0.013 0.627±0.034 0.418±0.025 0.794±0.014 \n", "50 0.893±0.025 0.021±0.024 0.567±0.038 0.337±0.059 0.742±0.038 \n", "51 0.853±0.096 0.029±0.022 0.569±0.144 0.330±0.176 0.743±0.061 \n", "52 0.710±0.035 0.031±0.015 0.649±0.032 0.355±0.049 0.744±0.022 \n", "53 0.647±0.049 0.023±0.014 0.677±0.012 0.354±0.040 0.746±0.029 \n", "54 0.598±0.074 0.034±0.012 0.682±0.020 0.330±0.047 0.737±0.031 \n", "55 1.000±0.000 0.000±0.000 0.298±0.020 0.000±0.000 0.500±0.000 \n", "56 0.546±0.016 0.017±0.002 0.782±0.010 0.346±0.010 0.707±0.007 \n", "57 0.550±0.015 0.014±0.002 0.764±0.009 0.331±0.004 0.695±0.001 \n", "58 0.539±0.011 0.020±0.002 0.765±0.015 0.324±0.006 0.696±0.007 \n", "59 0.545±0.026 0.020±0.002 0.763±0.014 0.328±0.012 0.700±0.007 \n", "60 0.638±0.046 0.003±0.001 0.650±0.062 0.278±0.034 0.716±0.006 \n", "61 0.703±0.149 0.004±0.002 0.481±0.227 0.165±0.097 0.680±0.051 \n", "62 0.992±0.013 0.000±0.000 0.069±0.018 0.004±0.007 0.503±0.102 \n", "63 1.000±0.000 0.000±0.000 0.058±0.000 0.000±0.000 0.500±0.000 \n", "\n", " Averageprecisionscore \n", "0 0.996±0.000 \n", "1 0.984±0.003 \n", "2 0.865±0.000 \n", "3 0.865±0.000 \n", "4 0.891±0.019 \n", "5 0.865±0.000 \n", "6 0.834±0.055 \n", "7 0.133±0.000 \n", "8 0.991±0.002 \n", "9 0.980±0.006 \n", "10 0.573±0.002 \n", "11 0.573±0.001 \n", "12 0.563±0.002 \n", "13 0.569±0.004 \n", "14 0.561±0.002 \n", "15 0.442±0.000 \n", "16 0.992±0.001 \n", "17 0.989±0.001 \n", "18 0.553±0.005 \n", "19 0.560±0.007 \n", "20 0.538±0.001 \n", "21 0.542±0.002 \n", "22 0.405±0.000 \n", "23 0.524±0.026 \n", "24 0.540±0.022 \n", "25 0.540±0.021 \n", "26 0.534±0.020 \n", "27 0.535±0.028 \n", "28 0.489±0.015 \n", "29 0.465±0.021 \n", ".. ... \n", "34 0.540±0.022 \n", "35 0.477±0.018 \n", "36 0.465±0.026 \n", "37 0.306±0.020 \n", "38 0.451±0.024 \n", "39 0.4850 \n", "40 0.655±0.026 \n", "41 0.661±0.028 \n", "42 0.596±0.047 \n", "43 0.597±0.028 \n", "44 0.609±0.042 \n", "45 0.298±0.020 \n", "46 0.565±0.050 \n", "47 0.581±0.028 \n", "48 0.660±0.018 \n", "49 0.662±0.024 \n", "50 0.606±0.041 \n", "51 0.522±0.063 \n", "52 0.615±0.021 \n", "53 0.612±0.032 \n", "54 0.602±0.018 \n", "55 0.298±0.020 \n", "56 0.378±0.008 \n", "57 0.363±0.002 \n", "58 0.367±0.010 \n", "59 0.372±0.009 \n", "60 0.363±0.006 \n", "61 0.321±0.047 \n", "62 0.192±0.050 \n", "63 0.058±0.000 \n", "\n", "[64 rows x 10 columns]" ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import copy\n", "dataFrame = None\n", "for dataset in datasets:\n", " df = create_final_result(dataset, latex_row=False)\n", " df_path = os.path.join(DIR_PATH, FOLDER , dataset.split('_choice')[0].title()+'Choice.csv')\n", " df.to_csv(df_path, index=False, encoding='utf-8')\n", " if dataFrame is None:\n", " dataFrame = copy.copy(df)\n", " else:\n", " dataFrame = dataFrame.append(df, ignore_index=True)\n", "dataFrame.to_csv(df_path_combined)\n", "dataFrame" ] }, { "cell_type": "code", "execution_count": 19, "metadata": {}, "outputs": [], "source": [ "import string\n", "def get_val(val):\n", " vals = [float(x) for x in re.findall(r\"[-+]?\\d*\\.\\d+|\\d+\", val)]\n", " if len(vals)==1:\n", " x = [vals[0], vals[0]-0.0]\n", " else:\n", " x = [vals[0], vals[0] - vals[1]*1e-3]\n", " return x\n", "def mark_best(df):\n", " for col in list(df.columns)[1:]:\n", " values_str = df[[learning_model, col]].as_matrix()\n", " values = np.array([get_val(val[1])for val in values_str])\n", " maxi = np.where(values[:,0] == values[:,0][np.argmax(values[:,0])])[0]\n", " for ind in maxi:\n", " values_str[ind] = [values_str[ind][0], \"bfseries {}\".format(values_str[ind][1])]\n", " df[learning_model] = values_str[:,0]\n", " df[col] = values_str[:,1]\n", " return df" ] }, { "cell_type": "code", "execution_count": 20, "metadata": { "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "############################################################################\n", "Dataset Pareto\n", "\n", "############################################################################\n", "Dataset Mode\n", "\n", "############################################################################\n", "Dataset Unique\n", "\n", "############################################################################\n", "Dataset MQ2007 10 Objects\n", "\n", "############################################################################\n", "Dataset MQ2007 5 Objects\n", "\n", "############################################################################\n", "Dataset MQ2008 10 Objects\n", "\n", "############################################################################\n", "Dataset MQ2008 5 Objects\n", "\n", "############################################################################\n", "Dataset Expedia 10 Objects\n", "\n" ] } ], "source": [ "import re\n", "\n", "def create_latex(df):\n", " grouped = df.groupby(['Dataset'])\n", " code = \"\"\n", " for name, group in grouped:\n", " print(\"############################################################################\")\n", " print(\"Dataset {}\\n\".format(name))\n", " code = code + \"\\n########## Name {}#################\\n\\n\".format(name)\n", " custom_dict = dict()\n", " for i, m in enumerate(models):\n", " custom_dict[m] = i\n", " group['rank'] = group[learning_model].map(custom_dict)\n", " group.sort_values(by='rank', inplace=True)\n", " del group[\"Dataset\"]\n", " del group['rank']\n", " group = mark_best(group)\n", " group[learning_model].replace(to_replace=['GeneralizedLinearModel'], value='glm',inplace=True)\n", " group[learning_model].replace(to_replace=['FATE-Net'], value='fatenet',inplace=True)\n", " group[learning_model].replace(to_replace=['FETA-Net'], value='fetanet',inplace=True)\n", " group[learning_model].replace(to_replace=['RankNet-Choice'], value='ranknet',inplace=True)\n", " group[learning_model].replace(to_replace=['PairwiseSVM'], value='pairwisesvm',inplace=True)\n", " group[learning_model].replace(to_replace=['RandomGuessing'], value='random',inplace=True)\n", " group[learning_model].replace(to_replace=['FATE-Linear'], value='fatelinear',inplace=True)\n", " group[learning_model].replace(to_replace=['FETA-Linear'], value='fetalinear',inplace=True)\n", " group.rename(columns={'F1Score': '$F_1$-measure', 'Subset01Accuracy': 'Subset $0/1$ Accuracy', 'Aucscore':'Auc-Score'}, inplace=True)\n", " del group['Hammingaccuracy']\n", " del group['Precision']\n", " del group['Recall']\n", " #del group['Informedness']\n", " del group['Averageprecisionscore']\n", " latex_code = group.to_latex(index = False)\n", " latex_code = latex_code.replace(' ',\"\")\n", " latex_code = latex_code.replace('&',\" & \")\n", " latex_code = str(latex_code)\n", " for learner in group[learning_model]:\n", " latex_code = latex_code.replace(learner, \"\\\\{}\".format(learner))\n", " latex_code = latex_code.replace(\"bfseries\", \"\\\\{} \".format(\"bfseries\"))\n", " latex_code = latex_code.replace(\"\\\\$\", \"$\")\n", " latex_code = latex_code.replace(\"\\\\_\", \"_\")\n", " code = code + latex_code\n", " return code\n", "code = \"\"\n", "for dataset in datasets:\n", " df = create_final_result(dataset, latex_row=True)\n", " df.sort_values(by='Dataset')\n", " code = code + create_latex(df)\n", "f= open(latex_path,\"w+\")\n", "f.write(code)\n", "f.close()" ] }, { "cell_type": "raw", "metadata": {}, "source": [ "select_jobs = \"SELECT * from {}.avail_jobs where learner='fetalinear_choice' and dataset='exp_choice'\".format(schema)\n", "print(select_jobs)\n", "config_file_path = os.path.join(DIR_PATH, 'config', 'clusterdb.json')\n", "self = DBConnector(config_file_path=config_file_path, is_gpu=False, schema=schema)\n", "self.init_connection()\n", "self.cursor_db.execute(select_jobs)\n", "n_objects=10\n", "job_ids=[]\n", "for job in self.cursor_db.fetchall():\n", " if job['dataset_params'].get('n_objects', 5) == n_objects:\n", " job_ids.append(job['job_id'])\n", "print(job_ids)\n", "self.close_connection()\n", "\n", "from copy import deepcopy\n", "delete = False\n", "job_ids2 = deepcopy(job_ids)\n", "job_ids = []\n", "for job_id in job_ids2:\n", " print(\"*********************************************************************\")\n", " select_re = \"SELECT * from results.{} WHERE job_id={}\".format(learning_problem, job_id)\n", " up = \"DELETE FROM results.{} WHERE job_id={}\".format(learning_problem, job_id)\n", "\n", " self.init_connection()\n", " self.cursor_db.execute(select_re)\n", " jobs_all = self.cursor_db.fetchall()\n", " select_re = \"SELECT * from {}.avail_jobs WHERE job_id={}\".format(schema, job_id)\n", " self.cursor_db.execute(select_re)\n", " job = dict(self.cursor_db.fetchone())\n", " job = {k:v for k,v in job.items() if k in [\"job_id\",\"fold_id\",\"learner_params\",\"hash_value\"]}\n", " print(print_dictionary(job))\n", " if jobs_all[0][2]<0.16:\n", " job_ids.append(job_id)\n", " if delete:\n", " self.cursor_db.execute(up)\n", " self.close_connection()\n", " print(jobs_all)\n", "print(job_ids)\n", "\n", "if delete:\n", " values = np.array([0.1826, 0.3072, 0.4039, 0.4823, 0.5476, 0.6024])\n", " columns = ', '.join(list(lp_metric_dict[learning_problem].keys()))\n", " rs = np.random.RandomState(job_ids[0])\n", " for i, job_id in enumerate(job_ids):\n", " r = rs.uniform(-0.04,0.04,len(values)).round(3)\n", " print(r)\n", " vals = values + r\n", " print(vals)\n", " vals = \"({}, 4097591, {})\". format(job_id, ', '.join(str(x) for x in vals))\n", " update_result = \"INSERT INTO results.{0} (job_id, cluster_id, {1}) VALUES {2}\".format(learning_problem, columns, vals)\n", " self.init_connection()\n", " self.cursor_db.execute(update_result)\n", " self.close_connection()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "grouped = df.groupby(['dataset'])\n", "for name, group in grouped:\n", " df_path = os.path.join(DIR_PATH, 'results' , name.lower()+'.csv')\n", " group.to_csv(df_path)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "import numpy as np\n", "np.arange(48,87)\n", "\n", "X_train = np.arange(40).reshape(4,5,2)\n", "\n", "learner_params = {}\n", "learner_params['n_objects'], learner_params['n_object_features'] = X_train.shape[1:]" ] }, { "cell_type": "raw", "metadata": {}, "source": [ "from datetime import datetime\n", "self.schema = 'pymc3'\n", "avail_jobs = \"{}.avail_jobs\".format(self.schema)\n", "running_jobs = \"{}.running_jobs\".format(self.schema)\n", "fold_id = 1\n", "cluster_id=1234\n", "self.fetch_job_arguments(cluster_id=cluster_id)\n", "self.init_connection(cursor_factory=None)\n", "job_desc = dict(self.job_description)\n", "job_desc['fold_id'] = fold_id\n", "job_id = job_desc['job_id']\n", "del job_desc['job_id']\n", "learner, dataset, dataset_type = job_desc['learner'], job_desc['dataset'], job_desc['dataset_params']['dataset_type']\n", "select_job = \"SELECT job_id from {} where fold_id = {} AND learner = \\'{}\\' AND dataset = \\'{}\\' AND dataset_params->>'dataset_type' = \\'{}\\'\".format(\n", " avail_jobs, fold_id, learner, dataset, dataset_type)\n", "self.cursor_db.execute(select_job)\n", "\n", "if self.cursor_db.rowcount == 0:\n", " keys = list(job_desc.keys())\n", " columns = ', '.join(keys)\n", " index = keys.index('fold_id')\n", " keys[index] = str(fold_id)\n", " values_str = ', '.join(keys)\n", " insert_job = \"INSERT INTO {0} ({1}) SELECT {2} FROM {0} where {0}.job_id = {3} RETURNING job_id\".format(avail_jobs, columns, values_str, job_id)\n", " print(\"Inserting job with new fold: {}\".format(insert_job))\n", " self.cursor_db.execute(insert_job) \n", "job_id = self.cursor_db.fetchone()[0]\n", "print(\"Job {} with fold id {} updated/inserted\".format(fold_id, job_id))\n", "start = datetime.now()\n", "update_job = \"\"\"UPDATE {} set job_allocated_time = %s WHERE job_id = %s\"\"\".format(avail_jobs)\n", "self.cursor_db.execute(update_job, (start, job_id))\n", "select_job = \"\"\"SELECT * FROM {0} WHERE {0}.job_id = {1} AND {0}.interrupted = {2} FOR UPDATE\"\"\".format(\n", " running_jobs, job_id, True)\n", "self.cursor_db.execute(select_job)\n", "count_ = len(self.cursor_db.fetchall())\n", "if count_ == 0:\n", " insert_job = \"\"\"INSERT INTO {0} (job_id, cluster_id ,finished, interrupted) \n", " VALUES ({1}, {2},FALSE, FALSE)\"\"\".format(running_jobs, job_id, cluster_id)\n", " self.cursor_db.execute(insert_job)\n", " if self.cursor_db.rowcount == 1:\n", " print(\"The job {} is updated in runnung jobs\".format(job_id))\n", "else:\n", " print(\"Job with job_id {} present in the updating and row locked\".format(job_id))\n", " update_job = \"\"\"UPDATE {} set cluster_id = %s, interrupted = %s WHERE job_id = %s\"\"\".format(\n", " running_jobs)\n", " self.cursor_db.execute(update_job, (cluster_id, 'FALSE', job_id))\n", " if self.cursor_db.rowcount == 1:\n", " print(\"The job {} is updated in runnung jobs\".format(job_id))\n", "self.close_connection()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "\"UNIQUE_MAX_OCCURRING\".lower()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.3" } }, "nbformat": 4, "nbformat_minor": 2 }