{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "#### Data source:\n", "Lee M, Teber ET, Holmes O, Nones K, Patch AM, Dagg RA, Lau LMS, Lee JH, Napier CE, Arthur JW, Grimmond SM, Hayward NK, Johansson PA, Mann GJ, Scolyer RA, Wilmott JS, Reddel RR, Pearson JV, Waddell N, Pickett HA. \n", "**Telomere sequence content can be used to determine ALT activity in tumours.**\n", "_Nucleic Acids Res._ 2018 Jun 1;46(10):4903-4918." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# STEP 1: Preprocessing" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Load data" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
TTAGGGATAGGGCTAGGGGTAGGGTAAGGGTCAGGGTGAGGGTTCGGGTTGGGGTTTGGG...TTACGGTTATGGTTAGAGTTAGCGTTAGTGTTAGGATTAGGCTTAGGTrel_TLTMM
094.8460.0190.4300.4220.2160.5441.7620.5350.3380.068...0.0280.1180.1530.0000.0490.0600.0330.089-0.89-
194.9510.0110.2410.4910.2230.3171.3510.8180.7020.090...0.0240.1250.0800.0240.0350.1550.0300.093-0.39-
294.8890.0430.4390.4780.3550.3161.1510.6250.3130.079...0.0410.2530.1950.0320.0430.1610.0470.185-1.66-
394.2020.0170.2520.5090.3960.5481.8770.8560.4400.097...0.0530.1100.1250.0000.0430.0690.0290.110-1.73-
496.3680.0110.0780.1310.0150.3061.5251.1650.1260.000...0.0140.0990.0220.0000.0190.0260.0090.0140.21-
598.8430.0010.1120.1790.0000.0730.2850.2800.0940.000...0.0090.0110.0450.0000.0020.0140.0100.0030.56-
697.0410.0020.2090.3240.2000.3910.6400.3530.2570.041...0.0140.0660.0890.0000.0170.0860.0160.0410.01-
793.6870.0340.4440.6510.4630.6551.3470.8360.5500.133...0.0380.1360.1910.0460.0340.1600.0330.099-0.79-
897.5000.0150.1490.2640.0780.2380.9170.1840.2060.038...0.0190.0860.0600.0000.0290.0450.0240.031-0.83-
997.1100.0010.2230.3030.1660.2930.7290.3820.2530.026...0.0260.0360.1480.0000.0170.0590.0190.029-0.02-
\n", "

10 rows × 21 columns

\n", "
" ], "text/plain": [ " TTAGGG ATAGGG CTAGGG GTAGGG TAAGGG TCAGGG TGAGGG TTCGGG TTGGGG \\\n", "0 94.846 0.019 0.430 0.422 0.216 0.544 1.762 0.535 0.338 \n", "1 94.951 0.011 0.241 0.491 0.223 0.317 1.351 0.818 0.702 \n", "2 94.889 0.043 0.439 0.478 0.355 0.316 1.151 0.625 0.313 \n", "3 94.202 0.017 0.252 0.509 0.396 0.548 1.877 0.856 0.440 \n", "4 96.368 0.011 0.078 0.131 0.015 0.306 1.525 1.165 0.126 \n", "5 98.843 0.001 0.112 0.179 0.000 0.073 0.285 0.280 0.094 \n", "6 97.041 0.002 0.209 0.324 0.200 0.391 0.640 0.353 0.257 \n", "7 93.687 0.034 0.444 0.651 0.463 0.655 1.347 0.836 0.550 \n", "8 97.500 0.015 0.149 0.264 0.078 0.238 0.917 0.184 0.206 \n", "9 97.110 0.001 0.223 0.303 0.166 0.293 0.729 0.382 0.253 \n", "\n", " TTTGGG ... TTACGG TTATGG TTAGAG TTAGCG TTAGTG TTAGGA TTAGGC \\\n", "0 0.068 ... 0.028 0.118 0.153 0.000 0.049 0.060 0.033 \n", "1 0.090 ... 0.024 0.125 0.080 0.024 0.035 0.155 0.030 \n", "2 0.079 ... 0.041 0.253 0.195 0.032 0.043 0.161 0.047 \n", "3 0.097 ... 0.053 0.110 0.125 0.000 0.043 0.069 0.029 \n", "4 0.000 ... 0.014 0.099 0.022 0.000 0.019 0.026 0.009 \n", "5 0.000 ... 0.009 0.011 0.045 0.000 0.002 0.014 0.010 \n", "6 0.041 ... 0.014 0.066 0.089 0.000 0.017 0.086 0.016 \n", "7 0.133 ... 0.038 0.136 0.191 0.046 0.034 0.160 0.033 \n", "8 0.038 ... 0.019 0.086 0.060 0.000 0.029 0.045 0.024 \n", "9 0.026 ... 0.026 0.036 0.148 0.000 0.017 0.059 0.019 \n", "\n", " TTAGGT rel_TL TMM \n", "0 0.089 -0.89 - \n", "1 0.093 -0.39 - \n", "2 0.185 -1.66 - \n", "3 0.110 -1.73 - \n", "4 0.014 0.21 - \n", "5 0.003 0.56 - \n", "6 0.041 0.01 - \n", "7 0.099 -0.79 - \n", "8 0.031 -0.83 - \n", "9 0.029 -0.02 - \n", "\n", "[10 rows x 21 columns]" ] }, "execution_count": 2, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import pandas as pd\n", "\n", "# Load data\n", "data = pd.read_csv(\"data/telomere_ALT/telomere.csv\", sep='\\t')\n", "data.head(10) # Show first ten samples" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Split data into training and test sets" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "from sklearn import model_selection\n", "\n", "num_row = data.shape[0] # number of samples in the dataset\n", "num_col = data.shape[1] # number of features in the dataset (plus the label column)\n", "\n", "X = data.iloc[:, 0: num_col-1] # feature columns\n", "y = data['TMM'] # label column\n", "\n", "X_train, X_test, y_train, y_test = \\\n", " model_selection.train_test_split(X, y, \n", " test_size=0.2, # reserve 20 percent data for testing\n", " stratify=y, # stratified sampling\n", " random_state=0)" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(128, 20)" ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "X_train.shape" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(33, 20)" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "X_test.shape" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "- 96\n", "+ 32\n", "Name: TMM, dtype: int64" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "y_train.value_counts()" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "- 25\n", "+ 8\n", "Name: TMM, dtype: int64" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "y_test.value_counts()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# STEP 2: Learning" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Train a Random Forest classifier" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", "from sklearn import ensemble\n", "\n", "rf = ensemble.RandomForestClassifier(\n", " n_estimators = 10, # 10 random trees in the forest\n", " criterion = 'entropy', # use entropy as the measure of uncertainty\n", " max_depth = 3, # maximum depth of each tree is 3\n", " min_samples_split = 5, # generate a split only when there are at least 5 samples at current node\n", " class_weight = 'balanced', # class weight is inversely proportional to class frequencies\n", " random_state = 0)\n", "\n", "k = 3 # number of folds\n", "\n", "# split data into k folds\n", "kfold = model_selection.StratifiedKFold(\n", " n_splits = k,\n", " shuffle = True,\n", " random_state = 0)\n", "\n", "cv = list(kfold.split(X_train, y_train)) # indices of samples in each fold" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "validation fold 1:\n", "[ 3 6 7 10 12 13 20 24 31 34 36 37 39 42 45 56 57 59\n", " 62 63 69 70 71 75 77 81 83 88 90 91 95 97 100 104 105 106\n", " 107 108 111 113 116 117 127]\n", "validation fold 2:\n", "[ 0 2 4 8 9 11 14 16 17 18 22 25 26 29 33 38 40 43\n", " 46 48 51 53 54 55 65 66 68 72 73 76 78 80 82 84 93 98\n", " 103 112 114 119 120 121 123]\n", "validation fold 3:\n", "[ 1 5 15 19 21 23 27 28 30 32 35 41 44 47 49 50 52 58\n", " 60 61 64 67 74 79 85 86 87 89 92 94 96 99 101 102 109 110\n", " 115 118 122 124 125 126]\n" ] } ], "source": [ "for j, (train_idx, val_idx) in enumerate(cv):\n", " print('validation fold %d:' % (j+1))\n", " print(val_idx)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Compute classifier accuracy" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Fold 1 accuracy: 0.860\n", "Fold 2 accuracy: 0.860\n", "Fold 3 accuracy: 0.857\n", "Average accuracy: 0.859\n" ] } ], "source": [ "accuracy = []\n", "\n", "# Compute classifier's accuracy on each fold\n", "for j, (train_idx, val_idx) in enumerate(cv):\n", " rf.fit(X_train.iloc[train_idx], y_train.iloc[train_idx])\n", " accuracy_per_fold = rf.score(X_train.iloc[val_idx], y_train.iloc[val_idx])\n", " accuracy.append(accuracy_per_fold)\n", " print('Fold %d accuracy: %.3f' % (j+1, accuracy_per_fold))\n", "\n", "print('Average accuracy: %.3f' % np.mean(accuracy))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Plot receiver operating characteristic (ROC) curve" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [], "source": [ "from matplotlib import pyplot as plt\n", "from sklearn import metrics\n", "\n", "fpr = dict() # false positive rate\n", "tpr = dict() # true positive rate\n", "auroc = dict() # area under the ROC curve (AUROC)\n", "\n", "# Compute an ROC curve (with respect to the positive class) for each fold\n", "# Compute AUROC for each fold\n", "for j, (train_idx, val_idx) in enumerate(cv):\n", " rf.fit(X_train.iloc[train_idx], y_train.iloc[train_idx])\n", " y_prob = rf.predict_proba(X_train.iloc[val_idx])\n", " fpr[j], tpr[j], _ = metrics.roc_curve(y_train.iloc[val_idx], y_prob[:, 0], pos_label='+')\n", " auroc[j] = metrics.auc(fpr[j], tpr[j])\n", "\n", "# Compute an average ROC curve for all folds\n", "fpr['avg'] = np.unique(np.concatenate([fpr[j] for j in range(k)]))\n", "tpr['avg'] = np.zeros_like(fpr['avg'])\n", "\n", "for j in range(k):\n", " tpr['avg'] += np.interp(fpr['avg'], fpr[j], tpr[j])\n", "\n", "tpr['avg'] /= k\n", "\n", "# Compute AUROC of the average ROC curve\n", "auroc['avg'] = metrics.auc(fpr['avg'], tpr['avg'])" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xlc1VX++PHXuSwii4AiCCKuyL6IiLjgjlu5ZmVaaTo5Tj+rqabNlunrTNO+Z81ki5NjZWWmmZmWJIsSiwsiiiBuIG4o+849vz+QG8giIpcL3PN8PHzEvZ/z+Xzely73fT+fc877CCkliqIoigKgMXQAiqIoSvuhkoKiKIqio5KCoiiKoqOSgqIoiqKjkoKiKIqio5KCoiiKoqOSgqIoiqKjkoKiKIqio5KCoiiKomNq6ABulIODg+zXr5+hw1AURelQEhMTL0kpe16vXYdLCv369SMhIcHQYSiKonQoQohTzWmnbh8piqIoOiopKIqiKDoqKSiKoig6KikoiqIoOiopKIqiKDp6SwpCiE+FEBeEEMmNbBdCiHeFEOlCiCQhRJC+YlEURVGaR59XCmuBqU1snwa4X/23DPhQj7EoiqIozaC3eQpSykghRL8mmswCPpfV64HGCiHshBDOUspsfcV06dIl9u7dW+/5YcOG0atXL86dO0d8fHy97SNGjMDBwYHMzEz2799fb3tYWBh2dnacOnWKpKSketvHjx+PtbU1x48fJyUlpd728PBwLCwsSE1N5dixY/W2T5s2DdP0HRw+coSMgi71ts9wzQPg4JWunC4yr7PNVAPTXKq377tsSVaxWZ3tXUwkk53zAYi7ZMX50uq3RJWUFJVVYUYFbprq/yVntY6UYFF3f8px1ZwDIFPbizLqnr8rpbhoLgBwWutMBXXPb0kJzpqLAJzU9qYKkzrbrUURTiIHgBNaVyqAXMqQwIXMPI6lHyU2IQqAe+68X7efjZ0FljZdKNKcJSklnt+3n+Su2xbX+90dTE7Ee6o55qYWOFYFc+VCIeVlVbrtiQdiSUk9RDcbW+bPW0i3HpYAVFZWkZNdSGx8FGkZR+lh78D0yXN0+3V3tMKsiym5mjSiI+K5clIyecKt9c4fl/Qbw2Y50kXaY1/lyYXMPGqvkLtj11bOX8ymv9tApk2bjqVN9f//0uJy8nJK2LZjEzlXLuE+wJPQYWG6/RxduyGE4KLJfn76Ih4Hy74MDQytd/741O0ET3XDWuuKRbkLOdmFdbZ/uXEtlZUVDA0MZXRYKGZdqt8fBbklFBeUs27DGgBCg8NwH+gJgKmphh7ONkhRxXmTOL56Yy8jho6jX9+BdY5dUlLMiaI9eA1zwb7KE1lkRV5OiW57QUEe32/7GoDJ42/FL8gDIQQAVy4Uci77PD/u3ATALeFz6N7dAYCulmZ062FJucgj40oCWz/ez+zpd2BjY1vn/FlnT2PiloVTH1scq4ZSdKWK4oJy3faTp44TFbsLgHvmL8XJ1V63rc3fe7ffjW13KwZ5OzN99L31jtWaDDl5rTdwptbjzKvP1UsKQohlVF9N4Obm1ibBtTvb/gaVbtA1oP62jP9U/9dqDFh41d0mKyD90+qfrSdCl0F1t2uLIW1d9c82U8G8L1B9CWkDWFXlEXLlKwD22s4gx8ylzu62lZcIyd0IQJndbeSZOtTZ3qPiLCF5PwBQZD+fIpO6f5hO5acYlr8dgPzu91CqsayzvXdZOkMKfiW3VHLA6j60Pa24ZGqCAMpKKsm7VExm2uXq8xdX/LGjtaBUaEkTl4gpzOR8+uW626+6cr6In8jCUmPJNK0PJWUVVBX/8Yd5+VwRmWmXsbeXFJaXU3n1Q0krtZQVV5CTXUhm2mUqnMzqHD9XlmIiTIjXZLP/0lm6nrZs8PxZ53K5oCmkl7YXI0Rfykoqkdo/ssL503lkZl7G2sSRgsoySoUWgIqqSsqKKzh3Ko/z5y/TvWthneNfphQhYJfmNEfPXMDb1qHB86dcuMB5TSWesgsesnu9Nlnpl6moqGCgS5HuNQGUVZRTUVyh+93n9SumzLl634ouGi6LEiqp5EdNBqfSc/DuV0xZz7rHLikqJy4vkwxNKSO0vehRZVrn/MUF5brjFw4p1b0mgJKyCoryy3Tbi4aXYWVRvW+VqaRSCC6KPH6tPEVm2mWKC8oxN6l7/vzLJSQ5ncZKY8W0Kk9EhYaKWufPy/njvVVcUs5l8UfCasv33tCh/enezQmJ5NCpSL0nBSFrfy1p7YNXXylslVL6NrBtK/CylDL66uNfgSellE1OVw4ODpYtmdGcmZkJgKur6w3v2y687gGDp8DMd/V6mvQLhdz98e+UVFSx9r5hDHGzv/5Oevbrr7+yePFiMjMzGbB8AHPnz+W1sa8RHR2NVqttcJ8BAwbo/l+fP3+e1NTURo8/atQoTEyqP+wOHDhAfn5+g+0cHR3x9Kz+NlxcXNzkzPqAgABsbasT4PHjx8nKymqwXdeuXRk2bJjusXpN6jXVqKqq4sKFC+Tl5WFhYcGkSZNwcXFp7FDXJYRIlFIGX7edAZPCf4DfpJRfXn2cCoy73u2jliaFH36o/rY6Y8aMG963XWiDpJCclceiT+MQQrBuaQhezt30dq7mKCkp4emnn+add94BoJ9vP8zuNuOH+3/Ao7uHQWNTFH3SarV8++235OXl4e/vz9ChQzE1vbkbO81NCoa8fbQFWCGE+AoYDuTpsz9BaVriqcss/iwemy6m/O9PwxnQ09qw8SQmcs8993DkyBFMTU15+pmn+dX9V4J6BamEoHRapaWldOnSBY1Gw7Bhw7C2tqZnz+vWsGtV+hyS+iWwF/AQQmQKIZYKIZYLIZZfbbINyADSgTXAA/qKRWladNol7v44DgfrLnzzl5EGTwg7duwgNDSUI0eO4Onpyd69exkwbwD5lfnc73//9Q+gKB2MlJK0tDQ2bNjA0aNHAejfv3+bJwTQ7+iju66zXQL/T1/nV5pnx+FzrPhiPwN6WrFu6XB6Xh3dUllZSXp6OlVVVQ3uN3jwYMzMqkcSnTp1isLCwgbbWVtb07dvded1RUVFg6Orari5uWFjY0NYWBgeHh5MnDiRl19+GWEm+NvGvzHceTj+Pf1v5uUqSrtTWFhIVFQUZ86cwdHRkV69ehk0ng5XOltpPd/vz+Kxbw7i19uWtfcNw87yj+GkH3zwAQ8//HCj+549exZnZ2cAVqxYwdatWxtsd+utt+r6cy5evIivb73uJZ0tW7YwY8YMunbtSlxcHJaW1SORvjz6JTmlObzm/9oNv0ZFac/S09OJiopCSsmIESPw8fFBozFsoQmjSQphYWHXb2RE1v9+ime/T2aIoynTuqax5O43CAoK4tlnnwWqO+Rff/11bGxsGty/dqeXm5sb3t7eDbarPYTY1NS00XZQfVVRoyYhVGgr+Cz5MwJ7BhLsdN0+MkXpULp06YKjoyNhYWF062bYgR019Dr6SB9aOvqow2vF0UcvfxPNa2vW0yUrkQup+3S3iLy8vBqcXGdIm9I28fye51k9cTVjXMcYOhxFuSlarZZDhw5RVVVFUFB1ZR8ppW5Snj51hNFHberUqepFh2rubxsjKSXTFv+Vnz//I7GYmpoyadIkZs+ezaxZswwYXX1V2io+Sf4Er+5ehPVWV3pKx5aTk8Pu3bu5dOkSAwYM0CWDtkgIN8JokkJN+QljSQparZb4+Hg2bdrEmDFjmDp1Gqu2prCv2B6zLhbcMm0ac+fO4dZbb8Xe3vAT1Bqy49QOTuWf4o2xb7S7PxxFaa6qqir27dvHgQMHdJPQ+vfv327f00aTFDq6i4WVnDt5CQ4d0j03cOBA3b33rKwsLl++zNmzZ9m8eTObN2/m7NmzAKSnHyeiqDffJGbywMI5PL72caysrAzyOppLK7WsObSGAbYDmNR3kqHDUZQWy8vL4+DBgwwaNIgRI0ZgYWFx/Z0MSCWFdq60tJR//vOfvPpyBhVVGcCXum179+4lNLS6yNlLL73E6tWr6+zr5ubGjJmzOGsfwDeJmfx1kjsPT3Rvt99Qatt9ZjdpV9L41+h/oRFq2Q+lY6moqODkyZO4u7vTvXt37rjjjnbTkXw9Kim0Y5cvXyY0NJS0tDQAvPt0R2P7R+2Trl276n52cXHB19cXS0tLJk+ezJw5c/D08eeBL/axL/Uiz97ixZ/CBrT5a2gJKSVrDq2ht3VvpvWfZuhwFOWGZGZmEhUVRUFBAQ4ODtjb23eYhAAqKbRr3bt3x8/PDzMzM/4zroDRU2Y3Ovpo5cqVrFy5Uve4oLSCxWvjiT95mZfm+nFXSMepLhubHcuhS4d4LvQ5TDXqLap0DGVlZcTGxpKamoqtrS0zZsxot/11TTGav7jx48cbOoTrklLyv//9Dz8/PwIDAwH4+OOPsbKywvxdv2Yf50pROYs+iyPlbD7vzB/CzICWV1Y0hDWH1uDY1ZHZg2YbOhRFaRatVsvmzZvJy8sjMDCQoKCgmy5gZygdM+oWqD0xqj1KT09n+fLl/PrrrwQHBxMbG4uJickNf9O4kF/K3Z/8zsmcYv5zz1AmejnpKWL92H9hP/Hn4nli2BOYm5hffwdFMaDaBexCQkKwtrbGwcHh+ju2Y0aTFI4fPw5Uj9hpT86ePs6qh+/hsx/jKK+owtbKgqk+3fh9zYN1OoSHFudx+Ewu23860uTxtief42JBGWvvG8bIgR3vzflR0kfYd7HnNvfbDB2KojSqpoDd3r17CQkJwcvLi379+hk6rFZhNEmhZqZue0sKD/95Ed9ur14idIG/OS+Hd6Wn1UHIPlinXRWCH87Zsz77ZJPHc7Duwv/+NJygdrA4zo1KyUkhOiuah4Y8hKWZ5fV3UBQDKCgoICoqiszMTJycnHQ1wDoLo0kKrSn1cipvJb5Fpbbypo5TXlTOpl3VCWHC38fRdcRA/q/JPYoIY+N1j/vBkS+g6QuKdimzMBMbMxvme843dCiK0qC0tDSio6ORUjJy5Eh8fHw6xBDvG6GSQgvEn4sn5mwM/g7+NzU6JvtINlIr6eXRFachTlRo66/jakycLJ34s/+fsTFvuAifohiahYUFTk5OhIWFNVossqNTSeEmfDDpA2y72F6/YWOmwc9Of8dt/xv0HP4aDr36tF5wiqLcNK1WS1JSElqtlqCgIPr06YOrq2unuzqoTSUFA7PrZoVXTxMuGToQRVHquHTpErt37yYnJ4eBAwe22wJ2rc1okkJ4eLihQ6gjKSmpybUFFEUxjMrKSvbt28fBgwexsLAgPDyc/v37GzqsNmM0RWUsLCzaTSGqgoICRo0aRf/+/SkoKjF0OIqi1JKfn09SUhLu7u7ccccdRpUQwIiuFFJTUwHw8PAwcCTw1VdfUVhYSGBgIDZWXa+/g6IoelVRUcGJEycYPHhwhytg19qMJinULBjfHpLCmjVrAFi2bBmQbdhgFMXInTlzhqioKAoLC+nZs2eHK2DX2ozm9lF7sX//fuLj47Gzs2PevHmGDkdRjFZpaSkRERH89NNPmJqaMnPmzA5ZwK61Gc2VQntRc5Vwzz331Cl9rShK26kpYJefn8+QIUMYMmRIhy1g19rUb6ENFRUVsX79egDuv/9+A0ejKManpKQECwsLNBoNw4cP7xQF7FqbSgptKCMjAwcHB3x8fPDza34pbEVRbo6UkmPHjukK2Hl7e3eaAnatzWiSwrRphl/By8/Pj7S0NC5evGjoUBTFaBQUFBAZGUlWVha9evXCxaVjrS/S1owmKbSX+4UajQYnp461xoGidFTHjh0jOjoaIQSjR4/Gy8ur089Ivlnt45OyDRw+fBgAHx8fg5z/559/xs/PT31LUZQ2ZGlpibOzM2FhYe1+oa32wmiSQkZGBmCYpFBSUsL8+fMpKCggPT1d3ctUFD3RarUcOHAAKSVDhw7F1dUVV1dXQ4fVoRhNUjCkb7/9ltzcXIYNG6YSgqLoSe0CdoMGDdIVsFNujF6TghBiKvAOYAJ8LKV8+ZrtbsB/AburbZ6SUm7TZ0yGUHcGs6IoramyspLExESSkpKwsLBg8uTJ6svXTdBbUhBCmACrgXAgE4gXQmyRUqbUavYs8LWU8kMhhDewDeinr5gM4ciRI0RFRWFtbc38+WpFMUVpbfn5+Rw6dIjBgwcTGhpKly5dDB1Sh6bPK4UQIF1KmQEghPgKmAXUTgoSqCkyYguc1WM8BlFzlbBgwQLV0aUoraS8vJwTJ07g4eFB9+7dufPOOzvtSmhtTZ9JoTdwptbjTGD4NW1eAHYIIR4ErIBJ+gpmxowZ+jp0o0pLS/n8888BNYNZUVrL6dOniYqKori4GEdHR+zt7VVCaEWG7mi+C1grpXxDCDECWCeE8JVSams3EkIsA5YBuLm5GSDMlrl8+TJhYWFkZWUxdOhQQ4ejKB1aaWkpe/bsIT09HXt7eyZNmqQK2OmBPpNCFlB70WHXq8/VthSYCiCl3CuEsAAcgAu1G0kpPwI+AggODpYtCebgwYMABAQEtGT3FnFxcWHTpk1UVFSoURCKchNqF7ALCgpiyJAhmJiYGDqsTkmfSSEecBdC9Kc6GcwHFlzT5jQwEVgrhPACLAC91IA4ffo00LZJoYaZmVmbn1NROoPi4mK6du2KRqMhNDQUa2trevToYeiwOjW9racgpawEVgA/A0eoHmV0WAixSggx82qzx4D7hRAHgS+BxVLKFl0JtDc7d+5kx44dlJeXGzoURelwpJQcPXqUr7/+miNHjgDQt29flRDagF77FK7OOdh2zXPP1/o5BRilzxgM5fnnnyc2NpZvv/2W2267zdDhKEqHkZ+fT2RkJGfPnsXZ2ZnevXsbOiSjYuiO5k7p7NmzxMbGYmFhwdSpUw0djqJ0GKqAneEZTVJoyyqpmzdvBmDy5MlYWVm12XkVpaOztLTExcWF0aNHq3k9BmI0SaEt11PYtGkTAHPmzGmzcypKR1RVVaUrYBccHKwK2LUDRpMU2sqVK1eIiIjAxMTEIBPmFKWjuHDhArt37+bKlSu4u7urAnbthNEkhX379gEQFBSk1/Ns3bqVyspKJkyYoEZKKEoDKisrSUhI4NChQ1haWjJlyhT69u1r6LCUq4wmKWRlVc+b03dSABg4cKC6daQojcjPzyc5ORlPT0+GDx+Oubm5oUNSajGapNBW7rnnHu6++26qqqoMHYqitBvXFrCbP3++6khup1RS0AMhRLtZE1pRDK12ATsnJyfs7OxUQmjH1CdXK9q5cydeXl660RNxG99Cnt3f5D7d8tPbIjRFaXMlJSXs3btXV8AuPDwcOzs7Q4elXIfRJAV9L7xRUVHBHXfcQW5uLidOnKBfv364H3oTC1lKkbBsct90k4H0sVOd0krnodVq2bJlCwUFBQwdOpTAwEBVwK6DMJqkMHnyZL0ef/fu3eTm5uLt7V1rKUBJUs9bGb7isyb3ddBrZIrSdq4tYGdjY0P37t0NHZZyA/RWEM/YfPfdd4CasKYYJyklKSkpbNiwoU4BO5UQOh6juVKIi4sDICQkpNWPrdVq+f777wGVFBTjk5eXR2RkJNnZ2bi4uKgZyR2c0SSF8+fPN6vdpZJLnC9qum12UXadx3FxcWRnZ+Pm5tYm8yAUpb1ITU0lOjoajUbDmDFj8PDwULOSOzijSQrNNX/rfM4XXz+BmAgTzDTVi+fU1DqaPXu2+oNQjIq1tTWurq6MHj1aFX/sJFRSuEZhRSHj+4znNvem10DoadkTS7PqUUX5+fmYmZkxd+7ctghRUQymqqqK/furh1kHBwfTu3dvtd5BJ6OSQgNcbVwZ22dss9t/+OGHvPLKK+qbktKp1S5gN3jwYFXArpMymqSg7w/sbt266fX4imIoFRUVugJ2VlZWTJ06FTc3N0OHpeiJ0SSFCRMm6OW4v//+O0OHDlVlLZROq7CwkJSUFLy9vQkJCVEF7Do5NU/hJpw4cYLQ0FAGDx6MVqs1dDiK0mrKyso4evQoAPb29tx5552MHj1aJQQjYDRfb/fs2QPAyJEjW+2YNXMThg0bhkaj8qvSOZw8eZLo6GhKSkro1auXKmBnZIwmKeTk5LT6MWtmMatRR0pnUFJSQkxMDBkZGXTv3p0pU6aoAnZGyGiSQms7f/48MTExmJubt+n6z4qiD1qtls2bN1NYWEhwcDCBgYHq6tdIqaTQQlu2bEFKSXh4uBp5pHRYRUVFWFpaotFoGDlyJDY2Ntjb2xs6LMWA1FeBFlIF8JSOrKaA3ddff01KSgoAbm5uKiEoxnOlYGtr22rHqqys5Pjx42g0GmbOnNlqx1WUtpCbm0tUVBTZ2dn07t2bPn36GDokpR0xmqQwZsyYVjuWqakpqampHDt2jJ49e7bacRVF344ePUpMTAwmJiaMHTuWwYMHq1nJSh1GkxRamxACDw8PQ4ehKDfExsaGPn36MHr0aCwtm14RUDFORpMUIiMjgZu/Yjh//jxFRUUMGDCgNcJSFL2qqqpi3759QPV8GlXATrkeo+lozsvLIy8v76aP8+abbzJo0CDeeuutVohKUfTn3LlzbNy4kf3791NcXIyU0tAhKR2A0VwptIbi4mLWrFmDlJJRo0YZOhxFaVBFRQXx8fEkJydjbW3NtGnTVGey0mx6vVIQQkwVQqQKIdKFEE810uYOIUSKEOKwEOILfcZzs9avX8+VK1cYPny4Xpb1VJTWUFhYyJEjR/Dx8WHevHkqISg3RG9XCkIIE2A1EA5kAvFCiC1SypRabdyBp4FRUsorQghHfcVzs6SUvPvuuwA89NBDBo5GUeoqKysjIyMDLy8v7O3tmT9/vlrfQ2kRfd4+CgHSpZQZAEKIr4BZQEqtNvcDq6WUVwCklBf0FUyPHj1uav/ffvuN5ORkevXqxbx581opKkW5eSdOnCA6OprS0lKcnZ2xs7NTCUFpMX0mhd7AmVqPM4Hh17QZDCCEiAFMgBeklNuvPZAQYhmwDGjx4h43Wx31vffeA+Avf/mLKh+stAvFxcXExMRw4sQJevTowbRp01QBO+WmGbqj2RRwB8YBrkCkEMJPSplbu5GU8iPgI4Dg4OA2H0Kh1WqxtrbGysqKZcuWtfXpFaUerVbLli1bKCoqYtiwYQQEBKgCdkqr0GdSyAJq93C5Xn2utkzgdyllBXBCCHGM6iQR39rB7Nq1C2jZCmwajYbPP/+c999/XxW/UwyqsLAQKysrXQG7bt26qasDpVXp86tFPOAuhOgvhDAH5gNbrmnzPdVXCQghHKi+nZShj2CKioooKiq6qWOohKAYipSS5OTkegXsVEJQWpverhSklJVCiBXAz1T3F3wqpTwshFgFJEgpt1zdNlkIkQJUAY9LKVt/NZybsG3bNqqqqpg+fTomJiaGDkcxQrm5uezevZvz58/j6ura4n41RWkOvfYpSCm3Aduuee75Wj9L4NGr/9odKSVPPPEEhw8f5ptvvlGjjpQ2V1PAztTUlHHjxuHu7q4K2Cl6ZeiO5nYtIiKCw4cP4+zsrEpkKwbRrVs33NzcGDVqlCpgp7QJo0kKTk5ON7xPzWS15cuXq2GoSpuorKzUFbALCQnBxcUFFxcXA0elGBOjSQo3WpbixIkTbNmyBTMzM/785z/rKSpF+cO5c+fYvXs3eXl5eHp6IqVUt4qUNmc0SeFGrV69Gikl8+fPb9FVhqI0V3l5OfHx8Rw+fBgbGxumT5+Oq6urocNSjJTRJIUdO3YAMHny5CbblVdq2XEgk6gPPwOgxD2cFV/sa9E5V6lSxUozFBUVcfToUXx9fRk2bBhmZmaGDkkxYkaTFMrKyprVrrxKy9ncUlzHLaAgM5Ucyz7kZOe36JwmGoFTN4sW7at0bqWlpWRkZODt7Y29vT133XWX6khW2oUbTgpCCA1wl5RyvR7iaRe83Lqz8ceXb/5Ar5hh20MVJlP+IKXkxIkTxMTEUFpaiouLC3Z2diohKO1Go0lBCNEN+H9UF7bbAuwEVgCPAQeBTpsUFEUfiouLiY6O5uTJkzg4ODB9+nQ1I1lpd5q6UlgHXAH2An8CVgICmC2lPNAGsRnE2c9PIR33kDslV/3BKq2mdgG74cOH4+fnpwrYKe1SU0lhgJTSD0AI8TGQDbhJKUvbJLJW1pzFyjMzM7my+xJ5Jnsoe7d5fRCK0pTaBexGjRqFjY2N+rKhtGtNJYWKmh+klFVCiMyOmhAAgoKCrtsmJiYGJPT066uGoSo3RavVkpKSQlxcHMOHD8fHx0cti6l0CE0lhQAhRD7Vt4wAutZ6LKWUna5kaEJCAgD2A5wNHInSkV25coXIyEjOnz9Pnz596Nu3r6FDUpRmazQpSCk7VUnQn376CYBp06Y12iYxMREA+wG92iQmpfM5cuQIMTExmJmZMX78eAYNGqRmJSsdSlOjjyyA5cAgIInq0teVbRVYa6usbDp0rVarSwp2/dWtI6VlbG1t6devH6NGjaJr166GDkdRblhTt4/+S3W/QhQwHfABHm6LoAzh+PHj5OfnY2prRtfuNoYOR+kgKisrdV8mhg8frgrYKR1eU0nBu9boo0+AuLYJyTCEECxdupRNp38wdChKB5GdnU1kZCR5eXl4eXmpAnZKp9Dc0UeVnf3NPmjQID7++GNiPws2dChKO1deXk5cXBwpKSnY2Nhwyy23NGvIs6J0BE0lhcCro42gesRRhx59pJYwVFpLcXExqamp+Pn5ERwcrArYKZ1KU0nhoJRySJtFomcBAQGNbtNqtWzatKlZcxkU41RaWsrx48fx8fHBzs5OFbBTOq2mkoLR1H1OT09n3rx5uLi4YP9PNUdB+YOUkoyMDGJiYigvL6d3796qgJ3SqTWVFByFEI82tlFK+aYe4tGbH36o7kCeMWNGvW01k9aCg4M5TlabxqW0X0VFRURHR3Pq1Cl69uzJ2LFjVYkKpdNrKimYANb8MaO506oZUqiSglJDq9Xyww8/UFRURGhoKL6+vqqAnWIUmkoK2VLKVW0WiQHVXCkMHTqUDec3X3+Hi8cg4kXQXmcuX1lBK0SntKWCggJdAbvRo0djY2ODra2tocNSlDbTVFLo9FcIUP2NcN++6uU2hw4dCtuasdPxXyHle+hJRn4yAAAgAElEQVTpCZomfoU9PWHAuNYIU9EzrVZLcnIy8fHxDB8+HF9fX7VOsmKUmkoKE9ssCgM6duwYhYWFuLq63nhl1CXboau9fgJT2szly5fZvXs3Fy9exM3NjX79+hk6JEUxmKYK4l1uy0D0bcCAAQ0+n5aWhomJCcHBatKaMUpJSWHPnj2Ym5szYcIEBg4cqGYlK0bthtdo7qh8fHwafH7GjBnk5+eTm5vbxhEphlRTksLOzo4BAwYwYsQIVcBOUTCipFBTJdXUtP5LtrS0VOPOjURlZSUJCQkIIVQBO0VpgNEkhZr1FGrPU5Cyen6eul1gHM6ePUtkZCT5+fl4e3urAnaK0gCjHnh99OhRHB0due+++wwdiqJH5eXlREZGsnXrVgBuvfVWRo8erRKCojRAr0lBCDFVCJEqhEgXQjzVRLvbhBBSCNGmvb2JiYlcunSJvLy8tjyt0saKi4tJT0/H399fV85EUZSG6e32kRDCBFgNhAOZQLwQYouUMuWadjZUL97zu75iaUztSWtK51JSUsLx48fx9fXVFbBTHcmKcn367FMIAdKllBkAQoivgFlAyjXt/gG8Ajyux1gaVLu8hdI5SCk5fvw4MTExVFRU4Orqip2dnUoIitJM+kwKvYEztR5nAsNrNxBCBAF9pJQ/CiH0mhQGDx5c53FVVVXdmcxKh1dYWEh0dDSnT5/G0dGRMWPGqAJ2inKDDDb6SAihAd4EFjej7TJgGbR8sRwPD486j1NTUykuLqZv3744ODi06JhK+1FTwK6kpIQRI0bg4+OjCtgpSgvoMylkAX1qPXa9+lwNG8AX+O3qKJBewBYhxEwpZULtA0kpPwI+AggODm7ROg+lpaUAWFhYAH/cOlJXCR1b7QJ2YWFhdOvWjW7dOtSigIrSrugzKcQD7kKI/lQng/nAgpqNUso8QPcVXQjxG/C3axNCa9m5cyfwxzyFkSNH8s477zRa/kJp37RaLYcOHSIhIUEVsFOUVqS3pCClrBRCrAB+pnpthk+llIeFEKuABCnlFn2duzkGDhzIQw89ZMgQlBbKyckhMjKSixcv0rdvX/r372/okBSl09Brn4KUchvXFKOWUj7fSNtx+oxF6RwOHz7Mnj176NKlCxMnTmTAgAFqEpqitCKjKXNR24kTJ/j0008ZN24cEycaRYXwDq+mJEX37t0ZOHAgI0eO1PUPKYrSeowyKURGRvLPf/6TI0eOtCwplBe2flBKgyoqKoiPj0ej0RAaGoqzszPOzs6GDktROi2jSQre3t66n2tmMrdo0trRbbD7VejlD13UMo36lJWVRWRkJAUFBfj4+KgCdorSBowmKQwcOFD3c4uHoyZ9A5v+DM4BcPdGUOPg9aKsrIzY2FhSU1OxtbVlxowZ6upAUdqI0SSFwsLqWz4WFhYcOHAAuMGkkPAZbH0E+o6CBV9BFxt9hKnwR92igIAAhg4d2uAaGIqi6IfR/LVFREQA0K9fP0pKSujfvz/du3dv3s4x78LO58B9MtzxOZipOjqtrbi4mOPHj+Pn54ednR0LFixQHcmKYgBGkxRq3NCtIykh4l8Q+Sp4z4a5a8DUXM8RGhcpJenp6ezZs4eKigrc3NywtbVVCUFRDMTokoIQAk9PT0JCQppuKCVsfxp+/xCG3A0z3gWNSdsEaSQKCwuJiorizJkzODk5MWbMGGxtVee9ohiS0SWFRYsWsWjRIt1SnA2SWtjyIOxfB8P/AlP+pTqVW1ntAnYjR47E29tbFbBTlHbA6JJCjcaGNgpgUN5eOJoEY5+EcU+DGgbZavLz87G2tkaj0TBmzBi6deuGjY3qtFeU9sJokoK/vz9FRUVkZ2c3PryxogQLyulRdhHC/wGjVG2k1qLVaklKSiIxMVFXwK53796GDktRlGsYzfV63759iY2NxcXFhWXLltVvUFYA/5uHCVVkdBumEkIrunTpEt9//z1xcXH06dNHVaZVlHbMaK4UcnNzSU5OBsDd3b3uxuLLsH4enD1AqVtfLnYdZIAIO6fk5GT27t2LhYUFkyZNUglBUdo5o0kKUVFRusVX+nj24Uz+1ZVCiy7B98sh9zTMeofK/W8bMMrOo6YkRY8ePXB3dyc0NFQNM1WUDsBoksLF4ou6Wc2rzqzCZFOt4aWWgKUDHHwVoQGN8fxaWp0qYKcoHZvRfPoVFRUhpaRX3168MuUVKDgP0W9BVSmMeBB6VNdGevq7ZDw8xhs42o4pMzOTyMhICgsL8fX1VQXsFKUDMpqkUFpcvUbz6JDRzLDqB989Uj1B7Z5N4Oyva/d0oTldNGqN3xtRVlbG3r17OXbsGLa2tsycOZNevXoZOqw2U1FRQWZmpm4dcEUxJAsLC1xdXTEzM2vR/kaTFMqKygAYMsgR1t4C5tZw72ZwcL/Onsr1lJSUcOLECQIDAwkKCjK6AnaZmZnY2NjQr18/dWWkGJSUkpycHDIzM1u8TK3RDEntHtSdfd32MbfiG+jaHZZsVwnhJhQXF5OUlASAnZ0dd911FyEhIUaXEABKS0vp0aOHSgiKwdUM7riZq1aj+QsuLU/nUr9LaDJ7sdz8n+R+nQVk1WtXXF7Z9sF1IFJK0tLS2Lt3L5WVlfTt21cVsKPxGfKK0tZu9r1oNEnhyhWBg9aBf9g9QpmmBzRS+iikf3cmeDq2bXAdREFBAVFRUWRmZuLk5MTYsWNVATtF6WSMJinYVQwkrNKKh+8bTw9LVWvnRmm1WrZu3UppaSmjRo3C29tbfTtuR959910+/PBDgoKCWL9+fYNt1q5dS0JCAu+//369bdbW1roh27UtWbKErVu34ujoqJv82ZC3336b7t27c++997b8RehRWVkZ9957L4mJifTo0YMNGzbQr1+/eu3eeecd1qxZg5SS+++/n7/+9a8APPfcc2zevBmNRoOjoyNr167FxcWF9evX88orryClxMbGhg8//JCAgADKy8uZNGkSu3bt6nC3VI2mT0Fpmby8PLRaLRqNhrFjxzJv3jx8fHxUQmhnPvjgA3bu3NloQmipxYsXs3379ibbVFZW8umnn7JgwYJmH7eysm1v037yySfY29uTnp7OI488wpNPPlmvTXJyMmvWrCEuLo6DBw+ydetW0tPTAXj88cdJSkriwIED3HrrraxatQqA/v37s3v3bg4dOsRzzz2nK6Fjbm7OxIkT2bBhQ9u9yFbSsVKY0ma0Wi0HDx4kMTGR0NBQfH19cXFxMXRY7d7//XCYlLP5rXpMb5du/H2GT6Pbly9fTkZGBtOmTWPJkiUsWrSIJUuWkJGRgaWlJR999BH+/v519jlx4gQLFiygsLCQWbNmNXrsMWPGcPLkySbj27VrV51RZ2vWrOGjjz6ivLycQYMGsW7dOiwtLVm8eDEWFhbs37+fUaNG8Y9//IMHH3yQ5ORkKioqeOGFF5g1axYnT57knnvuoaioCID333+fkSNHNvO31bDNmzfzwgsvADBv3jxWrFhRbx7NkSNHGD58OJaWlgCMHTuW7777jieeeEJXDQGq5zzV7Fc7rtDQUDIzM3WPZ8+ezdNPP83ChQtvKva2ppKCUs+lS5fYvXs3OTk5DBgwQNUrauf+/e9/s337diIiInBwcODBBx9kyJAhfP/99+zatYt7771Xty55jYcffpi//OUv3Hvvvaxevfqmzh8TE1NnJcO5c+dy//33A/Dss8/yySef8OCDDwLVw3f37NmDiYkJK1euZMKECXz66afk5uYSEhLCpEmTcHR0ZOfOnVhYWJCWlsZdd91FQkJCvfOGhYVRUFBQ7/nXX3+dSZMm1XkuKyuLPn36AGBqaoqtrS05OTk4ODjo2vj6+vLMM8+Qk5ND165d2bZtG8HBwbrtzzzzDJ9//jm2tra65X1r++STT5g2bVqd48XHxzfrd9ieqKSg1FG7gF14eHiLxzobq6a+0beV6OhoNm7cCMCECRPIyckhP7/u1UtMTIyuzT333NPg7ZTmys7OxsvLS/c4OTmZZ599ltzcXAoLC5kyZYpu2+23346JSXWJmR07drBlyxZef/11oHpo7+nTp3FxcWHFihUcOHAAExMTjh071uB5o6KiWhxzQ7y8vHjyySeZPHkyVlZWBAYG6mIFePHFF3nxxRd56aWXeP/99/m///s/3baIiAg++eQToqOjdc+ZmJhgbm5OQUFBh1ozxGiSQpZNGYcK9vKwoQNpp64tYDdixAi6dOli6LAUPWqtfqGuXbvWGRe/ePFivv/+ewICAli7di2//fabbpuVlZXuZyklGzduxMPDo87xXnjhBZycnDh48CBarbbR4c43cqXQu3dvzpw5g6urK5WVleTl5dGjR496+y5dupSlS5cCsHLlSlxdXeu1WbhwIdOnT9clhaSkJP70pz/x008/1TtmWVlZhxuubTQdzUXmVZzTnDN0GO1OeXk50dHRxMbGAuDs7My4ceNUQujAwsLCdB3Ov/32Gw4ODnXuiQOMGjWKr776CuCmO6e9vLx0HbJQPXTZ2dmZioqKJo89ZcoU3nvvPd3SuPv37weqBzc4Ozuj0WhYt24dVVVVDe4fFRXFgQMH6v27NiEAzJw5k//+978AfPvtt0yYMKHBpHjhwgUATp8+zXfffafrPE9LS9O12bx5M56enrp2c+fOZd26dQwePLjOsWpuT7W03IShGE1SsCo3oZfWeOrxNMeZM2f49ttvSUlJAWh63Wqlw3jhhRdITEzE39+fp556SvdhWNs777zD6tWr8fPzIyur/iTOGnfddRcjRowgNTUVV1dXPvnkk3ptpk2bRmRkpO7xP/7xD4YPH86oUaN0H54Nee6556ioqMDf3x8fHx+ee+45AB544AH++9//EhAQwNGjR+tcXbTU0qVLycnJYdCgQbz55pu8/PLLAJw9e5bp06fr2t122214e3szY8YMVq9ejZ2dHQBPPfUUvr6++Pv7s2PHDt555x0AVq1aRU5ODg888ACBgYF1+iAiIiK45ZZbbjr2tiY62gdBcHCwbKjT6Xre+OwD8irP8fDCx41+nkJpaSl79+4lLS0NOzs7xo4di5OTk6HD6rCOHDlS5566MZozZw6vvvpq/QWsjNjcuXN5+eWX611BtIWG3pNCiEQpZXAju+gYTZ+C8ofS0lJOnjxJUFAQQ4YMqdOZpigt8fLLL5Odna2SwlXl5eXMnj3bIAnhZun19pEQYqoQIlUIkS6EeKqB7Y8KIVKEEElCiF+FEH31GY8xKy4u5uDBg0gpsbOzY8GCBQQHB6uEoLQKDw8PxowZY+gw2g1zc/N2O7v7evR2pSCEMAFWA+FAJhAvhNgipUyp1Ww/ECylLBZC/AV4FbhTXzEZIyklqampxMbGUlVVRb9+/bC1tVUdyYqiNEift49CgHQpZQaAEOIrYBagSwpSytozQGKBu/UYj9HJz88nKiqKrKwsnJ2dCQsLUwXsFEVpkj5vH/UGztR6nHn1ucYsBX5qaIMQYpkQIkEIkXDx4sUWBXOmWylRpq072aU902q1/Pjjj1y4cIHRo0dz66236kZSKIqiNKZddDQLIe4GgoGxDW2XUn4EfATVo49aco4SMy2XNJdaHGNHkZeXh42Nja6AXbdu3bC2tjZ0WIqidBD6vFLIAvrUeuxKA6vaCCEmAc8AM6WUZfoKxqbMhD7aPtdv2EFptVr27dvHN998w+HDhwFwcXFRCcFIvPvuu3h5eTVZfG3t2rWsWLGiwW0NvU/OnDnD+PHj8fb2xsfHRzc2vyFvv/02n3/++Y0H3kbKysq48847GTRoEMOHD2+0yN8777yDr68vPj4+vP3227rnH3/8cTw9PfH392fOnDnk5uYC1aOM7rvvPvz8/AgICKgze3vSpElcuXJFny9LL/SZFOIBdyFEfyGEOTAf2FK7gRBiCPAfqhPCBT3GgnNhF4IrrztEt0O6ePEi3333HQkJCfTv359BgwYZOiSljemjdLapqSlvvPEGKSkpxMbGsnr1at1Ex9qMoXR2eHg4ycnJJCUlMXjwYF566SWguiIswKFDh9i5cyePPfYYWq0WqK4p9cEHH7TRK2w9ert9JKWsFEKsAH4GTIBPpZSHhRCrgAQp5RbgNcAa+ObqlPPTUsqZ+oqpMzp06BCxsbF07dqVyZMnN7hwiNKGfnoKzh1q3WP28oNpLze6WV+ls52dnXF2dgbAxsYGLy8vsrKy8Pb2rtPOGEpnT548WdcuNDSUb7/9FoCUlBQmTJgAgKOjI3Z2diQkJBASEsLMmTMJCwvjmWeeuanY25pe5ylIKbdJKQdLKQdKKV+8+tzzVxMCUspJUkonKWXg1X8qITRTzUz0nj174uHhwe23364SgpH697//jYuLCxERETzyyCP8/e9/Z8iQISQlJfGvf/2rwfHyNaWzDx06pPvgb8rJkyfZv38/w4cPr7etodLZ8fHxHDx4EC8vrzqlMWpKZ7/55pu8+OKLTJgwgbi4OCIiInj88ccpKirSlc7et28fGzZs4KGHHmowprCwMAIDA+v9++WXX+q1bax0dm2+vr5ERUWRk5NDcXEx27Zt48yZM/WO9emnn+pKZAcEBLBlyxYqKys5ceIEiYmJun3s7e0pKyurd572rl10NCvNV15ezu+//46JiQkjR46kV69e9Oqlajq1G018o28rrV06u7CwkNtuu4233367XmE9MJ7S2VBdPtvU1FTXd7NkyRKOHDlCcHAwffv2ZeTIkXX2cXR05OzZsw1WZG2vVFLoQE6fPk1UVBTFxcX4+fnVu/xVlBvRnPdORUUFt912GwsXLmTu3LkNtjGW0tlr165l69at/Prrr7rfnampKW+99ZauzciRI+uUtigtLaVr164Nxt9eGU2V1FO2pUSY1l8tqSMoLS1l165dbN++HXNzc2bNmkVoaKhKCEqDWqt0tpSSpUuX4uXlxaOPPtro+YyhdPb27dt59dVX2bJli67PAarLx9T0fezcuRNTU1Ndn4uUknPnznW427pGkxTKTLXkanINHUaLlJWVcerUKYKCgpg7dy6Ojo6GDklpx1qrdHZMTAzr1q1j165duvv127Ztq9fOGEpnr1ixgoKCAsLDwwkMDGT58uVAdRIJCgrCy8uLV155hXXr1umOVbO+eU0HfEdhNKWzn/j6bY4U/sra+V90iNLZRUVFpKWlERAQgBCCsrIyVa+onVKls1Xp7IY8/PDDzJw5k4kTJ7b5uVXp7GZwKjLHomqIocO4LiklR48eJTY2Fq1WS//+/VUBO6XdU6Wz6/P19TVIQrhZRpMUOoL8/HwiIyM5e/Yszs7OjBkzRhWwUzoEDw+Peh3Gxu7+++83dAgtopJCO6HVatm6dStlZWWEhYXh6empOpIVRWlzKikYWG5uLt26dUOj0TBu3DhVwE5RFIMymtFH7U1VVRWJiYl8++23qoCdoijthtEkhRN2Jew022noMIDqYWzfffcdiYmJDBgwQHXOKYrSbhhNUqgwkRSKQkOHwaFDh9i8eTPl5eVMmTKFCRMmNDpjU1GaSx+ls0tLSwkJCSEgIAAfHx/+/ve/N3rsv/71r3XmKrQ3ly9fJjw8HHd3d8LDwxstaf3kk0/i6+uLr68vGzZs0D2/dOlSAgIC8Pf3Z968eRQWVn+WnDp1iokTJ+Lv78+4cePIzMwEqisXT506Vf8vTA+MJinYl5jiXmW4b+S1C9h5enpy++2307dvX4PFo3Qu+iid3aVLF3bt2sXBgwc5cOAA27dvJzY2tl67nJwcYmNjGTNmTLOP3dals19++WUmTpxIWloaEydO1E1eq+3HH39k3759HDhwgN9//53XX39dVzPqrbfe4uDBgyQlJeHm5sb7778PwN/+9jfuvfdekpKSeP7553n66aeB6r9zZ2dnYmJi2u5FthKj6WjuWWyOeZVvm5+3vLyc2NhYTE1NVQE7I/BK3CscvXy0VY/p2d2TJ0MaL1inr9LZQgjdFURFRQUVFRUNjojbuHFjnW/Fq1at4ocffqCkpISRI0fyn//8ByEE48aNIzAwkOjoaO666y7uvfdeli9fzunTp4HqhXpGjRpFXFwcDz/8sK5u0GeffXbTw103b96sq8G0aNEixo0bxyuvvFKnTUpKCmPGjMHU1BRTU1P8/f3Zvn07d9xxh65MiJSSkpIS3e8hJSWFN998E4Dx48cze/Zs3fFmz57N+vXrGTVq1E3F3taM5kpBCA1IU9pykOepU6f4+uuvSU1NxcTEhI42e1zpGPRZOruqqorAwEAcHR0JDw9vVunsFStWEB8fT3JyMiUlJWzdulW3rby8nISEBB577DEefvhhHnnkEeLj49m4cSN/+tOfAPD09CQqKor9+/ezatUqVq5cWe+cBQUFDZbNDgwMbHAhoPPnz+teZ69evTh//ny9NgEBAWzfvp3i4mIuXbpEREREndLZ9913H7169eLo0aM8+OCDun2+++47ADZt2kRBQYGuVHZwcHCrV3JtC0ZzpWBl0p2cEgssTPVfsbCkpIQ9e/Zw/PhxunfvzuTJk1W9IiPR1Df6ttKapbNNTEw4cOAAubm5zJkzh+TkZHx9615xZ2dn07NnT93jiIgIXn31VYqLi7l8+TI+Pj7MmDEDgDvvvFPX7pdffqnzAZ6fn09hYSF5eXksWrSItLQ0hBBUVFTUi8vGxoYDBw7cyK9FRwjR4BXP5MmTiY+PZ+TIkfTs2ZMRI0bUKYP92WefUVVVxYMPPsiGDRu47777eP3111mxYgVr165lzJgx9O7dW7dPTdnsjsZokoKFmQn2luZo2mBCWHl5OWfOnGHo0KEN1mRXlPbgRiZH2tnZMX78eLZv314vKdQunV1aWsoDDzxAQkICffr04YUXXqhTVrt2cTutVktsbGy9gRYrVqxg/PjxbNq0iZMnTzJu3Lh68RQUFBAWFtZgrF988UW91eGcnJzIzs7G2dmZ7OzsRr+kPfPMM7qV0hYsWFCnDDZUJ8n58+fz6quvct999+Hi4qK7UigsLGTjxo26InodsWw2GNHto+5W5nj0ssHCTD8f0IWFhezfvx8pJba2tixYsIChQ4eqhKC0udYqnX3x4kXdAvUlJSXs3LmzwaqntUtn1yQABwcHCgsLdctWNmTy5Mm89957usc13/zz8vLo3bs3UD1iqiE1VwoN/bs2IUDd0tn//e9/G+xHqaqq0t36SUpKIikpicmTJyOl1L0+KSVbtmzR/R4uXbqkW5P5pZdeYsmSJbrjHTt2rF4C7QiMJimEh4cTHh7e6seVUpKSksI333zD/v37dZfp5ubmrX4uRWmO1iqdnZ2dzfjx4/H392fYsGGEh4dz66231mt3yy236Dpx7ezsuP/++/H19WXKlCkMGzas0TjfffddEhIS8Pf3x9vbm3//+98APPHEEzz99NMMGTKk1UYpPfXUU+zcuRN3d3d++eUXnnrqKQASEhJ0fRkVFRWEhYXh7e3NsmXL+N///oepqSlSShYtWoSfnx9+fn5kZ2fz/PPPA9VJ18PDg8GDB3P+/Pk66zFHRERwyy23tEr8bcloSmfrQ15eHpGRkWRnZ9O7d2/CwsIaXK5Q6dxU6WwYPXo0W7du1d06UWDMmDFs3rwZe3v7Nj+3Kp3dDKmpqQCtVslRq9Xy448/Ul5ezpgxY/Dw8FAF7BSj9cYbb3D69GmVFK66ePEijz76qEESws0ymqRQs/j3zSaFK1euYGtri0ajYfz48XTr1q1VVoZSlI6soaGqxqxnz5515ix0JEbTp3CzqqqqSEhIqFPAztnZWSUERVE6FaO5UrgZ58+fJzIykitXruDu7q4K2CmK0mmppHAdSUlJxMbGYmVlxdSpU3FzczN0SIqiKHqjkkIjpJQIIXB0dMTb25uQkBA1zFRRlE7PaPoUpk2bxrRp067brqysjN27d7Nnzx6guk7K6NGjVUJQlEYsXryY/v37ExgYSEBAAL/++qtuW3l5OX/9618ZNGgQ7u7uzJo1S1deGuDcuXPMnz+fgQMHMnToUKZPn64bFFJbSUkJY8eOpaqqqk1eU0ts374dDw8PBg0a1GAVVmi81HaN/Px8XF1d65Q4nzp1qq58+fLly3W/g7/97W/s2rWr1V+H0SSFmsqHTTl58iTffPMNx44dw8zMTBWwU5Rmeu211zhw4ABvv/02y5cv1z2/cuVKCgoKSE1NJS0tjdmzZzN37lyklEgpmTNnDuPGjeP48eMkJiby0ksvNVis7tNPP2Xu3LnNrhAgpdTNNG4LVVVV/L//9//46aefSElJ4csvv2ywMF9jpbZrPPfcc/VKkH/99dccPHiQ5ORkLl68yDfffAPAgw8+2GjyuRlGkxQOHz6sGzV0rZKSEn755Rd27NhB165dmTNnDiEhIWregdIiNQXXGvr30Ucf6dp99NFHTbZtrpMnT+Lp6cnixYsZPHgwCxcu5JdffmHUqFG4u7sTFxcHQFFREUuWLCEkJIQhQ4awefNm3f5hYWEEBQURFBSku0r+7bffGDduHPPmzcPT05OFCxde94vSiBEjdDOki4uL+eyzz3jrrbd0H+b33Xefbp2GiIgIzMzM6iSRgICABmsarV+/XleaorCwkIkTJxIUFISfn1+d1+Hh4cG9996Lr68vZ86cYceOHYwYMYKgoCBuv/123eI4q1atYtiwYfj6+rJs2bKb/gIYFxfHoEGDGDBgAObm5syfP18XV20pKSlMmDABqC61XbtNYmIi58+fZ/LkyXX2qZkQW1lZSXl5ue690bdvX3Jycjh37txNxX4to0kKGRkZZGRkNLitvLyczMxMhg0bxpw5c3BwcGjj6BTl5qSnp/PYY49x9OhRjh49yhdffEF0dDSvv/46//rXvwB48cUXmTBhAnFxcURERPD4449TVFSEo6MjO3fuZN++fWzYsIGHHnpId9z9+/fz9ttvk5KSQkZGxnUXjdm+fbtufH56etIpK2EAAAtHSURBVDpubm71ZvkHBwdz+PBhkpOT65Tcbkx5eTkZGRn069cPAAsLCzZt2sS+ffuIiIjgscce032op6Wl8cADD3D48GGsrKz45z//yS+//MK+ffsIDg7WrX3QVHnvGuvXr2+wNPe8efPqtc3KyqJPnz66x66urg2WD2ms1LZWq+Wxxx7j9ddfb/B3MGXKFBwdHbGxsalz/qCgoFZfyEevHc1CiKnAO4AJ8LGU8uVrtncBPgeGAjnAnVLKk/qMqUZhYSHHjh1jyJAhugJ2qt9AaQ3N/da5bNkyli1b1irn7N+/P35+fgD4+PgwceJEhBD4+flx8uRJAHbs2MGWLVt0HzylpaWcPn0aFxcXVqxYwYEDBzAxMalzTz8kJARXV1cAAgMDOXnyJKNHj653/scff5yVK1eSmZnJ3r17W+U11bh06VKdmdJSSlauXElkZCQajYasrCzdLae+ffsSGhoKQGxsLCkpKbpFbsrLyxkxYgTQdHnvGgsXLmxyedOWaKzU9gcffMD06dN1v+tr/fzzz5SWlrJw4UJ27dqlq+Omj/LceksKQggTYDUQDmQC8UKILVLK2jfalgJXpJSDhBDzgVeAO+sfrfXUFLCLi4tDSsnAgQOxtbVVCUHp0Lp06aL7WaPR6B5rNBpdUTkpJRs3bqw3q/+FF17AycmJgwcPotVq65Syrn1cExOTRgvUvfbaa8ybN4/33nuPJUuWkJiYyMCBAzl9+jQFBQXY2Njo2iYmJuoK6zVVRbVG7dLcUP0N/uLFiyQmJmJmZka/fv1022tPJpVSEh4ezpdfflnneNcr7137PK+99lq95wcNGlQv7t69e9dZkCczM1NX6bW2xkpt7927l6ioKD744AMKCwspLy/H2tq6Tp+BhYUFs2bNYvPmzbqkoI/y3Pq8fRQCpEspM6SU5cBXwLX1amcBNSUcvwUmCj3eyK+srOSHH34gJiYGR0dHbr/9dmxtbfV1OkVpV6ZMmcJ7772nu5LZv38/UF3Y0dnZGY1Gw7p1625qhM+KFSvQarX8/PPPWFlZsWjRIh599FHdMT///HOKi4uZMGECEyZMoKysrE4/S1JSUr3Vyuzt7amqqtJ9cOfl5eHo6IiZmRkRERGcOnWqwVhCQ0OJiYnRlb0uKiri2LFjzS7vvXDhwgZLczfUftiwYf+/vfuPreou4zj+/hiYHTrmUmIwYwycJWEdnSLR8YcDMgOELOUPB4FksrnpkhH8Q4lR4mBm/IOIYsxMEDPCj6BsmNTUMLMEN8NYZFkFtoxZpRuEdULGipLANmm3xz/O4eZSWu4pvede7u3nlTQ5P75tn6e37XPP+Z7zHI4ePcqxY8e4cOECu3btorW19bJxg7Xa3rlzJydOnOD48eNs2LCBZcuWsW7dOs6dO8fJkyeB5P/Xnj17Lmlfnkd77jyLws3A20Xr3em2AcdERB9wFmjMI5iIoKenhzNnzjBr1iwWLFhwybsXs3q3evVqent7aWlpobm5mdWrVwOwfPlytm3bxp133klnZ+ewWrdI4rHHHmP9+vVA8o+voaGBKVOm0NTUxO7du2lraytMpre1tbF3715uu+02mpubWbVq1YDPMJ87dy779+8Hkn/WHR0dTJs2je3btw/4jAdI+g9t3bqVpUuX0tLSwsyZM+ns7BxSe++sRo0axZNPPsm8efOYOnUqixcvprm5GYA1a9bQ3t4OXLnV9kDOnz9Pa2srLS0thceiXpyY7+3tpaurixkzSjY+HZLcWmdLug+YHxHfTte/CXw1IlYUjXk9HdOdrr+Zjnmv39d6BHgEYOLEiV8e7J1BKadOnWLs2LGMGTPmqj7fbCBunZ2/gwcPsnHjRnbs2FHtUK4ZFyfb165de9m+4bTOzvNI4R3glqL1Cem2AcdIGgXcSDLhfImI2BwRMyJiRvGzYIdq/PjxLghmNWj69OnMmTPnmr55rdL6+vpYuXJl2b9unkXhFaBJ0mRJ1wFLgPZ+Y9qBB9Ll+4Dnw3eMmdkAHnroIT/etsiiRYtyeX5FblcfRUSfpBXAcySXpG6JiCOSngA6IqIdeArYIakLOENSOMxqzsVeWWbVNtz31bnepxARzwLP9tu2pmj5Q2BRnjGY5a2hoYGenh4aGxtdGKyqLl5QU3xZ8VC5S6rZME2YMIHu7m5Onz5d7VDMaGhoGPQmuCxcFMyGafTo0UyePLnaYZiVxYjpfWRmZqW5KJiZWYGLgpmZFeR2R3NeJJ0Gru6WZhgHvFdyVH1xziODcx4ZhpPzrRFR8u7fmisKwyGpI8tt3vXEOY8MznlkqETOPn1kZmYFLgpmZlYw0orC5tJD6o5zHhmc88iQe84jak7BzMyubKQdKZiZ2RXUZVGQNF/SPyV1SfrRAPs/KenpdP/LkiZVPsryypDz9yW9Iek1SX+RdGs14iynUjkXjfuGpJBU81eqZMlZ0uL0tT4i6XeVjrHcMvxuT5T0gqRD6e/3gmrEWS6Stkh6N30I2UD7JelX6c/jNUnTyxpARNTVB0mb7jeBzwPXAa8Ct/cbsxzYlC4vAZ6udtwVyHkOMCZdfnQk5JyOuwHYBxwAZlQ77gq8zk3AIeCmdP2z1Y67AjlvBh5Nl28Hjlc77mHmfDcwHXh9kP0LgD8DAu4CXi7n96/HI4WvAF0R8VZEXAB2AQv7jVkIbEuX/wDco9rueVwy54h4ISLeT1cPkDwJr5ZleZ0B1gI/BT6sZHA5yZLzd4BfR8R/ACLi3QrHWG5Zcg5gbLp8I/DvCsZXdhGxj+T5MoNZCGyPxAHgM5I+V67vX49F4Wbg7aL17nTbgGMiog84CzRWJLp8ZMm52MMk7zRqWcmc08PqWyJiTyUDy1GW13kKMEXSS5IOSJpfsejykSXnnwD3S+omeX7LdysTWtUM9e99SNw6e4SRdD8wA5hV7VjyJOkTwC+AB6scSqWNIjmFNJvkaHCfpGkR8d+qRpWvpcDWiPi5pJkkT3O8IyI+rnZgtagejxTeAW4pWp+QbhtwjKRRJIecPRWJLh9ZckbS14EfA60R8b8KxZaXUjnfANwB/FXScZJzr+01Ptmc5XXuBtojojcijgH/IikStSpLzg8DzwBExN+ABpIeQfUq09/71arHovAK0CRpsqTrSCaS2/uNaQceSJfvA56PdAanRpXMWdKXgN+QFIRaP88MJXKOiLMRMS4iJkXEJJJ5lNaI6KhOuGWR5Xf7jyRHCUgaR3I66a1KBllmWXI+AdwDIGkqSVGo58fgtQPL0quQ7gLORsTJcn3xujt9FBF9klYAz5FcubAlIo5IegLoiIh24CmSQ8wukgmdJdWLePgy5vwz4NPA7nRO/UREtFYt6GHKmHNdyZjzc8BcSW8AHwE/iIiaPQrOmPNK4LeSvkcy6fxgLb/Jk/R7ksI+Lp0neRwYDRARm0jmTRYAXcD7wLfK+v1r+GdnZmZlVo+nj8zM7Cq5KJiZWYGLgpmZFbgomJlZgYuCmZkVuCiYZSTpI0mHiz4mSZot6Wy6/g9Jj6dji7d3StpQ7fjNsqi7+xTMcvRBRHyxeEPadv3FiLhX0qeAw5L+lO6+uP164JCktoh4qbIhmw2NjxTMyiQizgN/B77Qb/sHwGHK2LTMLC8uCmbZXV906qit/05JjSQ9lo70234TSf+hfZUJ0+zq+fSRWXaXnT5KfU3SIeBjYF3ahmF2uv1VkoLwy4g4VcFYza6Ki4LZ8L0YEfcOtl3SZOCApGci4nClgzMbCp8+MstZ2sJ6HfDDasdiVoqLglllbALuTq9WMrtmuUuqmZkV+EjBzMwKXBTMzKzARcHMzApcFMzMrMBFwczMClwUzMyswEXBzMwKXBTMzKzg/36KJdvLZZMlAAAAAElFTkSuQmCC\n", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# Plot the ROC curves\n", "for j in range(k):\n", " plt.plot(fpr[j], tpr[j], label='fold %d (area = %.3f)' % (j+1, auroc[j]))\n", "\n", "plt.plot(fpr['avg'], tpr['avg'], 'k--', label='mean ROC (area = %.3f)' % auroc['avg'], lw=2)\n", "plt.plot([0,1], [0,1], linestyle='--', color=(0.6, 0.6, 0.6)) # ROC curve for a random classifier\n", "plt.plot([0,0,1], [0,1,1], linestyle='--', color=(0.6, 0.6, 0.6)) # ROC curve for a perfect classifier\n", "\n", "plt.xlim([-0.05, 1.05])\n", "plt.ylim([-0.05, 1.05])\n", "plt.xlabel('FPR')\n", "plt.ylabel('TPR')\n", "plt.legend(loc='lower right')\n", "\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Plot precision-recall (PR) curve" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [], "source": [ "precision_plus = dict()\n", "recall_plus = dict()\n", "auprc_plus = dict()\n", "\n", "precision_minus = dict()\n", "recall_minus = dict()\n", "auprc_minus = dict()\n", "\n", "# Compute PR curves (with respect to the positive and negative classes) for each fold\n", "# Compute AUPRC for each fold\n", "for j, (train_idx, val_idx) in enumerate(cv):\n", " rf.fit(X_train.iloc[train_idx], y_train.iloc[train_idx])\n", " y_prob = rf.predict_proba(X_train.iloc[val_idx])\n", " precision_plus[j], recall_plus[j], _ = metrics.precision_recall_curve(y_train.iloc[val_idx], y_prob[:, 0], pos_label='+')\n", " precision_minus[j], recall_minus[j], _ = metrics.precision_recall_curve(y_train.iloc[val_idx], y_prob[:, 1], pos_label='-')\n", " auprc_plus[j] = metrics.auc(recall_plus[j], precision_plus[j])\n", " auprc_minus[j] = metrics.auc(recall_minus[j], precision_minus[j])\n", "\n", "# Compute average PR curves for all folds\n", "recall_plus['avg'] = np.unique(np.concatenate([recall_plus[j] for j in range(k)]))\n", "recall_minus['avg'] = np.unique(np.concatenate([recall_minus[j] for j in range(k)]))\n", "precision_plus['avg'] = np.zeros_like(recall_plus['avg'])\n", "precision_minus['avg'] = np.zeros_like(recall_minus['avg'])\n", "\n", "for j in range(k):\n", " precision_plus['avg'] += np.interp(recall_plus['avg'], np.flip(recall_plus[j], 0), np.flip(precision_plus[j], 0))\n", " precision_minus['avg'] += np.interp(recall_minus['avg'], np.flip(recall_minus[j], 0), np.flip(precision_minus[j], 0))\n", "\n", "precision_plus['avg'] /= k\n", "precision_minus['avg'] /= k\n", "\n", "# Compute AUPRCs for the average curves\n", "auprc_plus['avg'] = metrics.auc(recall_plus['avg'], precision_plus['avg'])\n", "auprc_minus['avg'] = metrics.auc(recall_minus['avg'], precision_minus['avg'])" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### PR curve w.r.t the positive class" ] }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xd4FlXa+PHvSS+kkN4LJKSSUEJogqJUVwT7YoPFutZVV11dXHn1dV/d3+racF1dFewNFZCiKKg0TUILCZ2QHtJ7L+f3x5M8JCRAgDxJCPfnup6LzMyZOeeJce6ZOefco7TWCCGEEABmfd0AIYQQ/YcEBSGEEEYSFIQQQhhJUBBCCGEkQUEIIYSRBAUhhBBGEhSEEEIYSVAQQghhJEFBCCGEkUVfN+BMubm56aCgoL5uhhBCnFe2b99epLV2P1258y4oBAUFkZSU1NfNEEKI84pSKqM75eTxkRBCCCMJCkIIIYwkKAghhDCSoCCEEMJIgoIQQggjkwUFpdS7SqkCpVTKSbYrpdSrSqnDSqlkpdQoU7VFCCFE95jyTmEpMPMU22cBoa2fO4F/m7AtQgghusFk8xS01r8opYJOUWQO8L42vA/0V6WUs1LKW2udZ4r2fL3xTZK3ZlNf19Rh/d4De9i+61cCwz247opbAWhpbqEwtxKA3SnbSU7dga2tHddeeRMAg93tsLKxBCDl4HbWrdyIo4MTcy6/vsOxzc0Vln6F1JoVYKnt0ZlBNLd0fP3p5m0bOJp5hIumj2RczGUA1Nc2UlZUA8DGTd+RnZuJn08AUybNAMDd1wEzM0M8/27DKvbsOEBwwFAuGn9ph2Nb21rQ7HmERlWNbYsHTi1DO/1eCs130qzqsG/xxqElqNP2AvMkWlQjg1r8GNTib1yvzMyxsnekKbQJzMGswAyzkuPXGGaY4WHnwdw5cw2/x927yczM7HBsCwsLZs2aBcCOHTvIycnp2H5ra6ZPnw5AQkIC+fn5Hbbb29tz6aWG77x161aKi4s7bHdycmLy5MkA/PLLL5SXl3fY7urqyoQJEwDYsGED1dXVHbZ7enoSHx8PwPfff099fX2H7b6+vowaZbjBXbt2LU1NHf+2AgICiI2NBWDVqlWcaMiQIURFRdHU1MTatWs7bR82bBhhYWHU1dWxfv36TtsjIyMZOnQoVVVVbNy4sdP2mJgYAgMDKSsrY9OmTZ22jxw5Ej8/P4qKiti2bVun7WPGjMHLy4tjx46RmJjYafv48eNxc3MjOzubnTt3dto+adIknJ2dycjIIDk5udP2KVOmMGjQII4cOcLevXs7bZ82bRo2NjYcOHCAgwcPdto+a9YsLCwsSE1NJS0trdP22bNnAwPrby8kJISIiIhO37Un9eXkNV8gq91yduu6TkFBKXUnhrsJAgICzqqy5KyfqGkIoKmmucP6soJqsg+VUOMEJaoWgBY09TWNAJTmG7bb29cb15U112OhDCeAtKoSsg+VMHjw8X2M7bZQ7DbLJ93sKM7amfg6X3RTx6BQmFNJ9qES9o4rYFhr/U0tzcZjFWRVkJ1WglWzs3Fdia7DTClD/YXFZB8qYZC5R6f6m80028yyKFNlBKEZqX06/V42mGVSpaoI0eYMV56dtq83y6BO1RGurYlQbsc3aKCqmlUpq2hSTUQ3RxPaHNq6yfAdrS2sOx1PCHF22gKPqYOCMlyom+jghjuFb7XW0V1s+xZ4Xmu9uXX5R+BxrfUppyvHxcXps53RnJKSQklJSZfbXF1diYqKAqC+vp7ffvvtpMeJiorC1dUVgPT09E5XIW2srKwYN26ccXnbtm00NjZ2WTYwMJDAwEAAioqKurxyajNu3DisrKzO6DstW7aMAwcO8Mwzz2BhcW7XAu9tSOGan6diG/07LK97p9P29PJ0Zn8zm+cnPc/vhvzunOoSQhi03W223QGdKaXUdq113OnK9eWdQg7g327Zr3WdyURHd4pNXbK2tjbe+p1OUFAQ3c3FNH78+G6Vc3Nz63b93flOZWVlPPDAA1RUVLBp0yY++eQT/Pz8unX8rjRZ2LG8eTIL9q2Aqv+DQR5nfSwhRPf4+vr2Sj19OSR1JXBr6yikcUC5qfoTLnTOzs6sXLkSHx8fNm/ezIgRI1i9evU5HfOD5mmolkbYsayHWimEOJVRo0YZ+7BMyZRDUj8BtgFhSqlspdRtSqm7lVJ3txZZA6QBh4G3gXtM1RYBF198Mbt27WLmzJkUFxdzxRVX8Oijj9LQ0HBWx0vTPjQFXQJJ70Fz02nLCyHODyYLClrreVprb621pdbaT2v9jtb6Ta31m63btdb6Xq31UK318NP1JYhz5+7uzurVq3nhhRcwNzfnn//8J3fcccdZH68x7naoyIEDa3qwlUKIrqxdu7bLUWo9TWY0X2DMzMx47LHH+OWXXwgLC+PRRx8962M1D50OTgGQ8FYPtlAI0ZWmpqZOw55NQYLCBWrChAmkpqZ26Kj+6KOPOo3FPyUzcxizENI3QcF+E7RSCNHbJChcwMzNzY0/L1u2jJtvvpkJEyZw+PDh7h9k5K1gbg2Jb5ughUKI3iZBQQCG2bHBwcHs2LGDUaNG8fnnn3dvR3tXiL4Gdn8KdRWmbaQQwuQkKAjAkNJgx44dXHPNNVRWVnLDDTdw9913U1tbe/qd42+HhipDYBBCmERAQMBZZ3Q4ExIUhJGzszNffPEFS5YswcrKiv/85z+MGzfupDO2jXxHGz6Jb4MJZ8gLcSGLjY015tIyJQkKogOlFPfccw+//vorISEhNDQ04OLicvodx9wBRQfh6M+mb6QQwmQkKIgujRw5ku3bt/Ptt98yaNAgAGpqajplczSKugrsXCFBOpyFMIVVq1Z1mW23p0lQECfl6OjI0KHH023fd999xMfHk3u0cxpjLG1g1K2GiWyVkq1EiPOVBAXRLeXl5Wzbto29e/fy3F1XUZX8fedCcQsN/6Z81buNE0L0GAkKolucnJxISkpi/vz5NNbXUbz2VTb++GPHQs4BMGwWpH7dN40UQpwzCQqi2+zt7Vm6dCkzb7wLgPfe/W/nQvF3QG1pL7dMCNFTJCiIMzblqlsAxepVKzu/4GfIJTA4sA9aJcTANmTIEIYMGWLyeiQoiDM22MMbm6ARNDQ08PXXJzwqUgpiWt9VXZre620TYqCKiooyvknRlCQoiLPifNGNrF63nj/84Q+dN4a3voLzSOeXyQshzo5kSRX9mrVvBJMvuQQzsy7+hKwcDP9mJUBN1++PFkKcGXmfgjhvnDQ/Uksj7Hi/dxsjhDgnEhTEWSsvL+eKK64gNDSUxsbGzgXcwyHpHWhp7v3GCSHOikVfN0CcvxwdHTl8+DA5OTl89913XHHFFR0LDLkENrwEh76HsFl90UTRAxqbG8mvyedY9THyqvM4Vn3M+LOlmSUvXvIiFmZyKhko5L+kOGtKKRYsWMATTzzB0qVLOwcFn5Hg4GPIhyRBoV/SWlNaX2o42Vcd41jNMfKq8jqc/AtrC9F0zH472HowFmYWFNYWUlpXirudex99A9HTJCiIc3LLLbfw17/+lZUrV1JUVISbm9vxjWbmEPcH2PgcFB0Gt5C+a+gFqqaxhmM1x46f8KvzyKvKM6xrPenXN3d8Bau1uTXe9t542Xsx0XciXvZexuW2j62FLZ8f+Jxnf322j77ZhWfYsGG9Uo8EBXFOfH19mT59OuvWreOTTz7h/vvv71hg1Hz4+R+GvoWZ/9c3jeynmls0mSU1NLdoQjwGndUxKhoqSC9PJ6cqx3h13/7f8vryDuUVCnc7d7zsvQh3CWeK/xTjid7b3htve2+crZ1RSvXEVxQ9KCwsrFfqkaAgztmCBQtYt24dS5cu7RwUHDwhcg7s/AguXQRW9n3TyD7U0qLJKavlUEElB45VcSi/kgP5lRwuqKK+qQUrczOSF0/HxtK8y/2bW5rJrc7laPlR0svTOVpx1PhzcV1xh7IOlg54DTKc4GPcYvAe5N3hSt/DzgNLM8ve+Nqih9XV1QFgY2Nj0nokKIhzNmfOHJydndm9ezfp6ekw+IQC8XdCypeQ/LnhcdIApbWmoLKeA8cqOZhv+BzIr+JwfiXVDcdHYHk52jDMy4HxQ1zJLq1lXeoxGppbaNK1pFekc7S89aTf+nNmRSYNLQ3G/Z2snQh2DGay32SCnIIIdgzGz8EPb3tvBlmd3R2H6P/Wr18PwOzZs01ajwQFcc5sbGz44osviI6OxsvLi/Ty9I4F/OPBa7ihw3n0AkMqjPNccVU9B/IrOZRf1fpvJQeOVVJRd3zGqdsgK0I9HLguzp9Qz0GEeTow1MOe2pbi1pP+Tvbu341twAHmrHyRotpC477myhw/Bz+CHYO5yPcigp2CCXYKJsgxiME2J0ZdIXqOBAXRI6ZOnXryjUoZ7hZW3g+Z2yBwQu817ByV1zYaH/ccPFbJwfwqDuZXUlx9/Mrd0caCMC8HZsf6MMzTgWGeDvi5mlHZnNf6uGc7u8rT+Tr5KBkVGdQ11xn3tTazRylXxniOY5jLEMPJ3zEYfwd/LM3lMY/ofRIURI9qaWmhrLSs84boa+H7pyDhrX4bFIqr6klML2VHZin78io4lF/FsYrjJ3B7K3NCPR24LMLDePIP9bSnVh8jpTiF1KKf2Vyexoe56RyrPmbcz0yZ4TvIlyDHIMZ6jzVe8Qc7BfNVYhnPrdnPoj9Mx9FGgoDoexIURI/ZsmULN954IxExEXDtCRut7GDkzfDbm1CRB47efdLGNlprsktrSUwvITG9hISjJRwpNLx/2srCjFCPQUwY6kqopwNhXoMY5umAj5MtFQ3l7CnaQ3LRZj7O2MOe7XuoaKgAwM7CjiFOQxjjOcZw4m993h/gGICVuVWX7VCqvMv1QvQVCQqixwwZMoScnBxyc3MJmd7FnIQxt8G2JbB9KUx5olfb1tKiOVRQRUJ6CYlHDYEgr9xwF+BgY0Fc4GCuGe3HmCAXYvycsLYwp7G5kYOlB9ld+BvrU/awp2gPGRUZgOHqP8Q5hOlB04lxiyHGPYZgp2DMlGSOEaYRGRnZK/VIUBA9xtvbm5kzZ7J69WrKfi2D351QwGUIhE6D7e/BpEfAouur557Q0NRCSm65MQAkZZRSVmPIz+ThYM2YYBfig1wYE+RCmJcDZgryqvNILkzglZ3J7Cncw97ivcZRP+627sS4x3BVyFXEuMcQ5RqFnaWdydov+oeG5gbyq/MNk/5aP552nlwVelWvt2Xo0KG9Uo9Jg4JSaibwCmAO/Fdr/fwJ2wOAZYBza5m/aK3XmLJNwrQWLFhgCAqbu+hXABhzB3x8HexfBdHX9Fi91fVN7MgsJTG9lMSjJezMKqWusQWAYDd7pkd6MibIhfhgFwJc7KhpqiGlKIWtRT/w5v5kkguTjWP+rc2tiXSNZF74PIa7DyfWPRZPO0+Z0DXAaK0pry/vcMJvn+IjrzqPotqiTik+bMxt+iQoVFVVATBokGmHHZssKCilzIElwDQgG0hUSq3UWu9tV2wR8LnW+t9KqUhgDRBkqjYJ05s9ezbOg50pyywjbW8anPj2wJCpMDgIEv57TkGhrVO4rU8gNbeC5haNmYIIb0d+PyaA+GAX4oIG42pvSVp5GsmFCbx3YA+7C3dzpOyI8X/2IMcgJvpOZLjbcGLcYwgdHCoTvAaAxpZG41V+20k+tyrX+HNedR61TR3TvrdP8XGR70WGWd6DvI2zvT/Z/wmfH/i8T77Pxo2Gl1adz/MU4oHDWus0AKXUp8AcoH1Q0IBj689OQK4J2yN6gbW1NVdeeyXvv/0+/3jwH9wQfwMeHh7HC5iZwZjb4ftFcCwFvKJPe8zTdQqP8HPm7ouHMCbIhdGBg6nX5SQXJrOn6AeWb00mpSiFmqYaABytHIlxj2F64HSGuw9nuNtwnKydTPK7EKajtaaysdJ4ZX/ilX5edR6FNZ0T+bnYuOBt780QpyFM8JmAt703PoN8jIHAxcbllHeEJxswMJCYMij4AlntlrOBsSeUWQx8r5S6H7AHTjHYXZwv7nzgTj5d+SnO7s64u3eRPXPETbDhOUh8G2a/ctLjpBdV882uHFbsyuVokSEIOFhbMDpoMFeP8iM+2NApbGVuxoHSA6zP+Ix/rfuBtPI0ACyUBcNchnHl0CuJcTd0Bgc4BMhjoPNIWV0ZGZUZZFZkklHR+m9lBlkVWVQ2VnYoa2lmiZe9Fz72Poz3Ht/hCr/tpG9jYdoUEQNBX3c0zwOWaq1fVEqNBz5QSkVrrVvaF1JK3QncCRAQENAHzRRnwtffl5D/DeHRqEeNJ+B9+/bx+uuvs3jxYkOgGH6tIe3F1P8BW2fjvkVV9Xy7O5evd+WyO6sMpWBcsCu3jg8kPtiFcC9HzM0UWmv2FO1hya6lrM9YT3ZVNmbKjDjPOK4OvZoY9xgiXCLkJHAeKK8vN57s25/8MyszjcN9wTDiy9vem0DHQGKGxODn4GcMAt6DvHGxcZHRXz3AlEEhB/Bvt+zXuq6924CZAFrrbUopG8ANKGhfSGv9FvAWQFxcnEb0e2aWZrh6uhqXH3/8cVatWsVHH33EokWLuP/a+Vjv/AB2fUzN6Dv5PjWfb3blsOlQEc0tmghvR56YFc6VI3zwdrIFDInhdhZs54fMH/gh4wfya/KxMLNgrPdYbh9+O1MCpuBi49JXX1mcQlVDVZdX/JkVmZTVHx+UoFB42XsR4BjAzKCZBDgGEOgYSIBjAH6D/C6Ixzd9zZRBIREIVUoFYwgGvwduPKFMJnAZsFQpFQHYAIWIAef555+nsbGRdevW8eijj/Lmm0N5+hJ/Zvz4GlPWBlHdoPFxsuHOyUOYO8KXMC8HwNBZuC13G+sz1rMhcwPFdcVYmVkx0XciD456kIv9L8bRyvE0tYveUNNYQ0ZFRpdX/CV1JR3Ketp5EugYyNTAqQQ6BBpP/n4OflibW/fRN+jfYmJieqUekwUFrXWTUuo+4DsMw03f1VqnKqWeAZK01iuBR4C3lVIPYeh0XqC1ljuBASgyMpI1a9bwxofL+Z+//oUjR45w6xG4ONCcG+/aztQF9zAmyAUzM0VDcwO/ZP/C+oz1bMzaSHl9ObYWtkzyncS0wGlM8puEveWFl4K7v3pw44PG4Zvtudu6E+AYwCX+lxDgcPyK39/BH1sL2z5q7fkrMDCwV+oxaZ9C65yDNSes+1u7n/cCE03ZBtH3OnYY2+J088uE5W4mecVbbMsu5V3bX/EO+DMbsn5kfcZ6fsn+harGKhwsHbjY/2KmBk5los/EftU/0NTUxN69e0lISCAhIYHExETy8/P57bff8Pf3P/0BBoDQwaH4DfLD0sySi3wvMpz0W0/+/g7+Mrmvh5WVGR6zOTs7n6bkuenrjmYxgP1ysJC3125hV7sO47svHsLMaG+cbK8gp/B+/vv6bbxutZ/Nn06iprGOui11XDfvOi4Pu5xx3uO6nSlUa82BAwfw8fHB0dHwOCkjI4ODBw/i6emJp6cnbm5umJt3/SKb7vroo49488032bFjBzU1NR223XvvvRdMQAAY6TGStdes7etmXDA2bdoEnN/zFMQFpqahifV78/lkZzKYwfIdOYTaD+3QYVxeX85PWWv4IeMHtuZupWFIA25N1lxp7UNz+WgWv7OYD7/7kKj/i2Lijae+iWxsbGTz5s2sWrWKVatWcfjwYd5//31uueUWAFauXMkDDzxgLG9mZoabmxuenp54e3uzbt064+io1atXY25ujoeHB4MHD2bfvn3Gu4BHHnmEyy67DICCggI2b94MGHI9xcfHM2bMGOLj4xk5cqSxrg0bNlBUVMT111/fo79jIUxNgoI4J03NLWw+XMSKXbl8l3qMmoZmvFyqwRP+MjOc20dNori2mA1Z3/JDwg8k5CXQpJvwsvfi+rDrmRY4jdif/oV52ia2jPsTK0aOZOfOndxyyy289tpr/Otf/2LChI6ptj/77DO++eYb1q1bZ7ylBnB1daWi4vgQRg8PDy655BIKCgrIz8+nuLiYgoICCgoKyMvL6zBf4d577yUjI6PL7zhmzBhjULj66qsJDw9nzJgxuLm5dVm+rKyMm2++mby8PNatW8err75q8tQEQvQUCQrirD23eh/r9x6jqKoBRxsL5ozwZe4IH9xdKpizAvZVbGXhd6vZnr+dFt2Cv4M/t0TdwrSAaUS7RR8/KcffCftWMtExl8TERN5//32efPJJEhISmDhxIldddRVLly41PhZ67bXX2LJlCwARERHMnj2b2bNnM378+A6Ph2644QZuuOEG43JjYyOFhYXk5+cb88i0mTZtGhkZGcbgMXToUOLj44mPj2fixON3LIGBgaft8HNycuKpp57i4Ycf5r333mPz5s18/PHHxMXFndPv+2xorcnKysLe3h5XV8MQ4R07dvDuu+8SGhpKaGgow4YNIygoCAsLOR0IDH8059Nn9OjRWvStpVuO6sDHv9Whf12j7/4gSa9LydN1jU3G7TmVOTp6abSOXhqt53w9R7+24zW9v3i/bmlp6fqALS1avz5W6zcnGX7WWldWVupFixZpGxsbDejt27cbi3/22Wf6pZde0ocOHTLp9zxXKSkpevjw4RrQFhYW+oUXXtDNzc0dyrz9yxEd+Pi3ury24Zzra2lp0UePHtVffvmlfuKJJ/T06dO1m5ubBvQrr7xiLLdkyRKNYbSf8WNhYaFDQ0P15Zdfruvr641lCwsLO7X5QvZS0kt61Puj+qTulStX6pUrV571/hhGfZ72HCuXBuKMXTXKFy8nG8YNccXJtnNHsM8gH16/9HX8HfwZ4nxiRrwuKAXxt8PqRyA7CfzHMGjQIJ599lnuuOMOXnjhBTIzMxk1ahTAefOcPioqioSEBB5//HFeffVVHn/8cXbv3s1HH33UI8cvLi42Xv231bdv375O5VxcXKirO/4GucmTJ/Piiy9y6NAhDh48yKFDh8jKyuLQoUOUlpZiZXV8gtjEiRPJzMxk6NChxruKtjuM6OjoDvUL02rfZ2VKEhTEGXO0sWRGlNcpy1zsf/GZHTTm9/DD/xjyIfmPMa4OCAhgyZIlZ9PMfsHGxoZXXnmFGTNmsHDhQubPn39Wx6mqqmL79u389ttvxg7w3NxcKisrsbU1jPkPCAigsLCQ0aNHd/gEBHTM9xQdHU10dMdEhDU1NRw5coSiouNzDVpaWqitraWuro7U1FRSU1M77PPss8+yaNGis/o+4sz5+fn1Sj0SFET/YD0IYucZXsAz/TkY1EUivfPY5ZdfzpEjR7C3Pz7pbuXKldTbnvrFKYmJiSxcuJC9e/fS0tIhJRgODg4cOXLEeIL/8ssvsbe3P6uEf3Z2dgwfPrzDOjMzMzIzM6moqODw4cMd7iwOHjzYYYZteno6Pj4+He4yelNLSwvp6elorXvtZTS9rS1gn2yAQ0+RoCD6jzG3Q8J/YMcymPznvm5Nj2sfEH7++Wfmzp2Lp/8QWsb/geVflJGyawcJCQmMHj2al19+GTCcAFJSUrCwsGDEiBGMHTvW2AEeHh6OmdnxBHCmGuHk6OjIqFGjjI/vTlReXs7UqVNxc3Pjiy++wMPbh6NHj1Cc+hOqIJXQKx7G3afnZ+NmZWXx+OOPs2/fPg4cOEBtreHdCA888AAvvvjigOs437ZtGyDzFMSFxH0YDLkEkt6FiX8C84H75zl48GDCw8MNfQCZi1j42fFt7YfVBgUFsW3bNmJjY42PifqT2oZm1ifupbyyynDXEh7Cq3PdmR96PK11wtZA3K99+IyO23bln5qaSkpKivHfyMhIPv74YwBsbW355JNPjPt4e3tTVFTEq6++yr59+/jqq69kKPBZGLj/14nzU/yd8OmNcHAtRJj2iqgvxcTEkJSUxOxb/8gvP37PpeNGMmHc8buANkopxo0b14ctPa6irpG9uRWkZJdRkL4Xm9xfCa7exTiz/eyfX83NX5uz7nADf/gkh4Rrp3DvH+8k8ue70KdIZ6a1Jjs7Gzc3N2PQe/LJJ3nllVc6zRgHjHcDYLiLev/99wkNDSU8PBxnZ2c2b97M1VdfjZ2dHXZ2pk2zoVtaUGYDL1W3BAXRvwybCU7+kPBW7waF+io4uA5SvoL0zfD7DyF4skmrtLOzY96DT3Mk5Hq+WDwdR5v+8wrQkuoGUnPLScmpIDWnlOrsPfhV7GKs2X6uNNuPhzJMGqy2caHSMx7zkEl8+6dJ/P2/K3h68WLe+GIjewua+CLe0A+itaagoKDDVX/bvxUVFaxfv56pUw3v2LK1taWmpgZvb2+ioqKIjo42/hsZGdmhnW2z19tcdNFFJCYm4urqany01tzcfE7pTepqq8k9kkJpZgq5+T/Roho58uxIfJqySXa7nLH3LzvrY/dHEhRE/2JmDnEL4cf/gcID4B5muroaa+Hgd5D6FRz8HppqwdYF6suh5KjJg0J/oLWmoLKelBxDAEjJLWd/dglOlQcYa7afsWb7uNH8AE5UgSXU2XlB4GUQMhkCJ2LvGoJ9u47tp/4Wy7jx47nxxhv56edN/ORhi/8YKC0txcur6xFrrq6ulJQcT619zz33cM8995z1cNf2kwvr6uqYOnUq8+bN49577z3lfqWFeRxLS6YyK5WWwoPYVhzBrS4D75YChijD3c7Pg51RTg5UWXpQ01yMfVX6WbWxP5OgIPqfUbfCT/8HCW/D7/7Zs8duqofDPxoCwYG10FAF9u4w8iaIuhqcA+Dl0783+nyVW1ZLcnaZMQCk5FRQXlXNcJXGWPP93G59iFi9Dxtrw6ObZudgzIPnQuBECJyAjXOgYV7JKUybNo0dO3bw/nv/5dqWl/kNw1yJ8PBwXFxcOl39e3h4dBgx1ZNzH1atWsWWLVvYsmULqampvPTiixTnpVGUvofa3P2YFR/EoeooXo1ZDKaSwa371WlLciz8ODYoiszBc7DyDMM5IJrm6p9Rhz4n9i8/sO+5Caesu6eNGTPm9IV6gAQF0f/Yu0H0NbD7E7jsb2Bzji/RaW6EtJ8Mj4b2rzbcCdgONtQRfTUEXnS8U7v8xJcDnr+01hwuqCIhvYTEoyUkppeSU1aLNQ2MNj/CLIcj/NV6P8Ete7EEgtokAAAgAElEQVRoaZ3c5hIOgfMgcAIETsDc0ees6vb39+fO2/8Ab71sXNfVxDpTqakqJ/fIHoJtSlh820z+vux7/v3vf7Nv7dssv96WWFtDECrGiXyrAA66TEG7DcPOOwK34OF4+YcwtItHThbbt/XadzjRye60epoEBdE/jbnDEBSSP4P4O858/+YmSN9kuCPYtwpqS8HaCSKuMNwRDLkYupmW+3zR2NxCam4FiUdLSEgvISm9hNKaRuypZYrdUZ5yOspIm1TcK1Ixa2mEOgVewyH6D8YggL1px8D3tIb6OrIO7qLkyHaa8/ZgV34Qj7pMvCgkpLXMSF/FiPn+3PZZHj+lNxD7vjVvvvI8F112Ba6unpwvc7KPHTsGmD44SFAQ/ZPfaPAZZXiENOb20z6yAKClBTK3Gu4I9q2E6kKwGgRhswx3BUMvBYuB86rH2oZmdmaWGu4E0kvYkVFGbWMzjlTxO6d0/jX4CMMdUnGp2IdqaYYyc/AZCVH3GB4H+Y8FW9O+sKUnFednk3sgkerM3ZgXpuJaeRC/5iyGqmaGYnjkk20RQLbjCI66hGDtFY5LYDTewZHMsbFj1NNZzJkzh507d3LjHQ+TlDQZJ1fPvv5aXWtphvJsKD4MxUeg+DCJuS7g4M3sG85uVnx3SVAQ/Vf8HfDNH+HoL4Yr+65oDdmJhkCw9xuozAMLWxg2w/BoKHQ6WPa/8f1no7S6gaSMUhLTS0g4WkJKTjlNLRp3Vc5clwwe8DpMRMMeHMoPouo1NFmDXxzEPmwIAn5jDDPH+7mmxgayDydTdHg7jbnJ2Jfux6fuMG6UGa/qC3Ahz2Yo210mY+kbg/vQ0fgOjSbE8uQzqv39/dm0aRPz58/H0tKSkJCQk5btFVpDTbHhxF90qDUAtAaBkjRorj9e1soBXO8C3XLy4/UQCQqi/4q6Gr77qyEfUvugoDXk7jQ8Gkr9BsqzwNwaQqdB1FWGYa3nwcnvdHLKajs8CjqYb0j37W9eyrVu6Sz2O0RobTL2lWlQDTTYGa7+R11neBTkOxos+88rTLtSXpxP1v4EqjJ2YV6QyuDKg/g3ZRKkGgkCGrQFWRYBHHUax2GPKAYFjsA3bAwe7t54nEV99vb2fP755zQ2Nho7twsKCnB2djZdio76Kig50uGq3/ipKz9ezswSXIaAa4jhb9k15PhnkAd8+61p2ncCCQqi/7K0MYxE2vqq4Va6ttRwR5D6NZQeNfxPNPRSuHQRhF1+7h3SfUhrOJRf2alTGDQR1sVc75rBxMADBFfvxqYqC8ox9JEEjINxCwx3At6x/bafpLmpiewjKRQd2U5DTjJ2Jfvwqj2MJ8U4tZYpxolc66HsdL8eC5/huA4dhV/oCIZaWdOT2YzMzMywtjY8RqyurmbGjBk4OjqyfPnys88r1NwIpRkdT/htQaAyt2NZJ39wHQrDr2t34h8KTgH9YhZ/37dAiFOJW2gICv+eYLiqUuaGu4ZJjxg6jW0Hn/4Y54HJ/9hIeW0joBljX8h9LunEO+8noHIXltXHoASwc23tEL7XEAQ8owzzOvq54NTXaUh5nkDVQCDQqM3JNvcjy3EkR92jsA8YgXfYGNy8/Hu90zcrK4uCggJ27dpFfHw8q1atIioqqtv7BzccgldHQWk66ObjG2xdDCf7oVMMJ/y2k//gYLAy7UzrcyVBQfRvgwNh7B8hfw9EzoXIOefdCJlTifFxYLZHEZfZHSKOfXiX7cC8rgQKAQdvCL6oNRBMNEzkO4sMqH3F0cWDfZZRNJtZkuYUhpn3cFyGjMI/bCTBNnYE93UDgfDwcBISEpg7dy5JSUmMHz+ejz/+mCuuuOK0++6xGoEttdh7DTc8tmx/1W/n0uNtHT9+fI8fsysSFET/N/Pvfd2CntPcCHm7IWMLZGwlPmMb8fXlUAE4B0LYTAgyTBRjcPB5FQROZG1jR8Rft/Z1M07L19eXX375hYULF/Lpp59y5ZVX8o9//INHHnnklGnIv3C4mW/Mb+Xj63snN5WpU2a3kaAghCk11kHuDkjfYggEWQnQWG3Y5jYMoq8yzhbGqXdeoiI6s7W15eOPPyYyMpK//e1vPProo0RGRnL55Zf3ddOMsrOzAdO/bEeCghA9qaHacOLP2GoIAtlJrUMLlaEPYOTNxyeKDTqb8TPCVJRSPPXUU0RGRvLjjz8ya9asvm5SBzt37gQkKAjRv9WVQ+ZvkLHZEAhyd0JLk6FD3DvWMNci6CLDUFETPGcWPe+aa67hmmuuMS6npaVRWVlJbGxsH7aq90hQEOJMVBcbZk1nbDWk2M5PMUwoMrM0zAuY+KDhLsB/LFg79HVrxTmqqKjgyiuvJD09nQ8//BD8+7pFpidBQYhTqcgzdgqTsRUKW5O6WdiC/xi4+PHW2cJxA2bmtDjOysqKESNGkJqaylVXXcXl916Ojj/5S4MGAgkKQnRl+3uw5WVDugEwpBkIGAcx1xuCgM9IsOibl9SL3mNjY8MHH3xAdHQ0Tz75JGuWrMFlpwstNxvSTdQ1NtPcojE3O39HiZ1InepVef1RXFycTkpK6utmiIGqrgL+OcwwmzpgwvHhoZ7D+8VsU9F3VqxYwfXzrqehtoHVq1fzTZEH3+/Nx22QFdOjvLg82ptxQ1ywMDfNKzrLygxvu3N2Prskhkqp7VrruNOWk6AgxAkaasDCBgbg+3fFuZl590y++8933Hbbbbz+7/+wcX8Bq/fksWF/ATUNzQy2s2RGlBezhnszYagrliYKEGeju0HBpJc+SqmZwCuAOfBfrfXzXZS5HlgMaGC31vpGU7ZJiNPq52kIRN+JmRLDz9/+THh4ODaW5swa7s2s4d7UNTbz88FC1u7J49vkPD5NzMLJ1pJpkZ5cPtyLiSFuWFucW0qSjIwMoOPrRk3BZEFBKWUOLAGmAdlAolJqpdZ6b7syocATwEStdalSSgZuCyH6Le8QbyL/L5I/3/LnDuttLM2ZEeXFjCgv6hqb2XyoiDUpeXyXeowvt2fjYGPBtAhPZg33ZlKoGzaWZx4gkpOTgfM4KADxwGGtdRqAUupTYA6wt12ZO4AlWutSAK11gQnbI4QQJmdjac7USE+mRnrS0NTClsNFrNmTx/d78/lqZw6DrC24LMKDWdHeXBLmflYBwpS6FRSUUtbANUBQ+3201s+cYjdfIKvdcjYw9oQyw1qPvwXDI6bFWut1XdR/J3AnQEBAQHeaLIQQJpOWlsbatWu55557TpkfycrCjCnhHkwJ9+DvzS1sPVLM2j2GO4gVu3KxszJnSrgHl0d7MyXcHTurvh/M0N0WrMCQwX07UH+asmdafyhwCeAH/KKUGq61LmtfSGv9FvAWGDqae7B+IYQ4I1prJk2aRG5uLuPGjWP06NHd2s/S3IyLh7lz8TB3/nduNL8dLWH1njy+SznG6uQ8bCzNmBLmwazh3lwa7sEg674JEN2t1U9rPfMMj51Dx/l/fq3r2ssGftNaNwJHlVIHMQSJxDOsSwgheoVSirlz5/LGG2/w1VdfdTsotGdhbsbEEDcmhrjx7JxoEo6WsDYlj7Upx1ibcgxrC0MAuXy4N5dGeOBo03svT+puUNjaegW/5wyOnQiEKqWCMQSD3wMnjiz6BpgHvKeUcsPwOCntDOoQQohed9VVV/HGG2/wyiuvsG7dOhwcHAgNDeXtt982lnnhhRewsLDAwcGh0ycwMBBXV8MrhczNFOOHujJ+qCtPz45ie0Ypa/bksS7lGN/vzcfK3IxJoW5MDwtncojp02d3NyhcBCxQSh3F8PhIAVprHXOyHbTWTUqp+4DvMPQXvKu1TlVKPQMkaa1Xtm6brpTaCzQDj2qti8/h+wghhMldfPHFDB06lCNHjrBjxw4ASkpKjNu11ixatIimpqYu93/55Zd58MEHAfjwww+55557jAFj0KBBODg44D7IgcHm1sTP/yvr9xfz4/4CnrrCgtvcTfu2we4GhbPKIau1XgOsOWHd39r9rIGHWz9CCHFesLS0JCUlhfT0dCorK6msrMTS8vgjHq01TzzxBJWVlVRVVRnLtH3ap7+uqKgwrj+RUoo1X3/OYmBD4h5c7XuyS7dr3QoKWusMpVQsMKl11Sat9W7TNUsIIfo3GxsbwsPDu9xmZmbGM8+canDmcXfddRc33XRTh6DRFkhqa2uNo5tq8jOoAUZEdV1nT+nukNQHMcwp+Kp11YdKqbe01q+ZrGVCCHEBMDc3x8nJCScnp75uCtD9x0e3AWO11tUASqkXgG2ABAUhhBhAuputSWHoCG7T3LpOCCHEANLdO4X3gN+UUl+3Ls8F3jFNk4QQQvSV7nY0v6SU+gnD0FSAP2itd5qsVUIIITqYNm1ar9RzyqCglHLUWlcopVyA9NZP2zYXrXXJyfYVQgjRc2xsbHqlntPdKXwMXIEh51H7nEOqdXmIidolhBCinQMHDgAQFhZm0npOGRS01le0/hts0lYIIYQ4pYMHDwKmDwrdGn2klJqolLJv/flmpdRLSinJYS2EEANMd4ek/huoaZ3V/AhwBPjAZK0SQgjRJ7obFJpa8xTNAV7XWi8BHEzXLCGEEH2hu/MUKpVSTwA3A5OVUmZA7yX4FkII0Su6GxRuwPAuhNu01sda+xP+n+maJYQQor1Zs84qWfUZ6+7ktWPAS+2WM4H3TdUoIYQQHVlY9M7rOU83eW2z1voipVQlXcxT0Fo7mrR1QgghAEhNTQUgKirKpPWcbp7CRa3/SqeyEEL0obQ0w5uKTR0UujtPYZxSyqHdsoNSaqzpmiWEEKIvnMk8hap2y9Wt64QQQgwg3X6fQus8BQC01i10f+SSEEKI80R3g0KaUuoBpZRl6+dBIM2UDRNCCNH7unu1fzfwKrAIwyikH4E7TdUoIYQQHc2ePbtX6unuPIUC4PcmbosQQpzXWnQLDc0NNLQ0GP5tbqC+ub7jzydsa2xpPH2ZZkOZ68OuZ6LvRJN+h24FBaXUMAwdy55a62ilVAxwpdb6f03aOiGE6GcaWhq4asVVnU7q9c31NLU09UgdVmZWWJtbY2VuhZW54efQulByzHLAt0eqOKnuPj56G3gU+A+A1jpZKfUxIEFBCHHBmOI/hbSyNCzMLLA0t8Ta3Bprc2sszSw7nMS7OqmfuO1k+1uaWaKU6lT3qlWroNL037G7QcFOa51wQkN7JiQKIcR5YoTHCF677LW+boZJdXf0UZFSaiitqS6UUtcCeSZrlRBCiD7R3TuFe4G3gHClVA5wFLjJZK0SQgjRJ04bFFrfnRCntZ7a+kpOM611LzzZEkII0aZfZEkFw+xlpdRjwOda6+peaJMQQogT9Nb7FLrbp/CDUurPSil/pZRL2+d0OymlZiqlDiilDiul/nKKctcopbRSKq7bLRdCCNHjzuTNaxq454T1Q062g1LKHFgCTAOygUSl1Eqt9d4TyjkADwK/dbfRQghxodmxYwcAo0aNMmk93b1TiMRwgt8N7AJeA06X1DseOKy1TtNaNwCfAnO6KPcs8AJQ1822CCHEBScnJ4ecnByT19PdoLAMiMCQ/+g1DEFi2Wn28QWy2i1nc8JcPKXUKMBfa726m+0QQghhQt19fBSttY5st7xRKbX3pKW7oXVU00vAgm6UvZPWBHwBAQHnUq0QQohT6O6dwg6l1Li2hda3riWdZp8cwL/dsl/rujYOQDTwk1IqHRgHrOyqs1lr/ZbWOk5rHefu7t7NJgshhDhT3b1TGA1sVUplti4HAAeUUnsArbWO6WKfRCBUKRWMIRj8HrixbaPWuhxwa1tWSv0E/FlrfbpgI4QQFxxra+teqae7QWHmmR5Ya92klLoP+A4wB97VWqcqpZ4BkrTWK8/0mEIIcaGaPn16r9TT3fcpZJzNwbXWa4A1J6z720nKXnI2dQghhOg53e1TEEII0YcSEhJISEgweT29k0xDCCHEOcnPz++VeuROQQghhJEEBSGEEEYSFIQQQhhJn4IQQpwH7O3te6UeCQpCCHEeuPTSS3ulHnl8JIQQwkiCghBCnAe2bt3K1q1bTV6PPD4SQojzQHFxca/UI3cKQgghjCQoCCGEMJKgIIQQwkj6FIQQ4jzg5OTUK/VIUBBCiPPA5MmTe6UeeXwkhBDCaEDcKTQ2NpKdnU1dXV1fN0VcAGxsbPDz88PS0rKvmyIuIL/88gtg+juGAREUsrOzcXBwICgoCKVUXzdHDGBaa4qLi8nOziY4OLivmyMuIOXl5b1Sz4B4fFRXV4erq6sEBGFySilcXV3lrlQMWAMiKAASEESvkb81MZANmKAghBDi3ElQ6CGvvvoqERER3HTTTScts3TpUu67774utw0aNKjL9QsXLsTDw4Po6OhT1v/yyy/z/vvvd7/Bvay+vp4bbriBkJAQxo4dS3p6epfl/vWvfxEVFUV0dDTz5s0zPqY5evQoY8eOJSQkhBtuuIGGhgYAXnrpJSIjI4mJieGyyy4jIyMDgMLCQmbOnNkr302I3uDq6oqrq6vJ65Gg0EPeeOMN1q9fz0cffdSjx12wYAHr1q07ZZmmpibeffddbrzxxm4ft6mp6VybdkbeeecdBg8ezOHDh3nooYd4/PHHO5XJycnh1VdfJSkpiZSUFJqbm/n0008BePzxx3nooYc4fPgwgwcP5p133gFg5MiRJCUlkZyczLXXXstjjz0GgLu7O97e3mzZsqX3vqQQJjRhwgQmTJhg8noGxOij9v5nVSp7cyt69JiRPo48PTvqpNvvvvtu0tLSmDVrFgsXLmT+/PksXLiQtLQ07OzseOutt4iJiemwz9GjR7nxxhupqqpizpw5Jz325MmTT3pV3WbDhg2MGjUKCwvDf863336bt956i4aGBkJCQvjggw+ws7NjwYIF2NjYsHPnTiZOnMizzz7L/fffT0pKCo2NjSxevJg5c+aQnp7OLbfcQnV1NQCvv/76Of8xrlixgsWLFwNw7bXXct9996G17vR8vqmpidraWiwtLampqcHHxwetNRs2bODjjz8GYP78+SxevJg//vGPTJkyxbjvuHHj+PDDD43Lc+fO5aOPPmLixInn1HYhLiRyp9AD3nzzTXx8fNi4cSMPPfQQTz/9NCNHjiQ5OZm///3v3HrrrZ32efDBB/njH//Inj178Pb2Pqf6t2zZwujRo43LV199NYmJiezevZuIiAjjVTUYhu9u3bqVl156ieeee45LL72UhIQENm7cyKOPPkp1dTUeHh6sX7+eHTt28Nlnn/HAAw90We+kSZMYMWJEp88PP/zQqWxOTg7+/v4AWFhY4OTk1CkVsK+vL3/+858JCAjA29sbJycnpk+fTnFxMc7Ozsag5+fnR05OTqc63nnnHWbNmmVcjouLY9OmTWfwmxSi/9qwYQMbNmwweT0D7k7hVFf0vWXz5s0sX74cMLxCr7i4mIqKjncvW7ZsMZa55ZZbunyc0l15eXlEREQYl1NSUli0aBFlZWVUVVUxY8YM47brrrsOc3NzAL7//ntWrlzJP//5T8AwtDczMxMfHx/uu+8+du3ahbm5OQcPHuyy3p4+4ZaWlrJixQqOHj2Ks7Mz1113HR9++GG3+gY+/PBDkpKS+Pnnn43rPDw8yM3N7dE2CtFX2u7cTW3ABYXzSU8NbbS1te0wbn7BggV88803xMbGsnTpUn766SfjtvYv/9Zas3z5csLCwjocb/HixXh6erJ7925aWlqwsbHpst5JkyZRWVnZaf0///lPpk6d2mGdr68vWVlZ+Pn50dTURHl5eadOsx9++IHg4GDc3d0Bwx3P1q1buemmmygrK6OpqQkLCwuys7Px9fXtsN9zzz3Hzz//jLW1tXF9XV0dtra2J/u1CSG6II+PTGDSpEnGDueffvoJNzc3HB0dO5SZOHGisRP1XDunIyIiOHz4sHG5srISb29vGhsbT3nsGTNm8Nprr6G1BmDnzp2AYeakt7c3ZmZmfPDBBzQ3N3e5/6ZNm9i1a1enz4kBAeDKK69k2bJlAHz55ZdceumlnYJiQEAAv/76KzU1NWit+fHHH4mIiEApxZQpU/jyyy8BWLZsmbEfZufOndx1112sXLkSDw+PDsc7ePDgaUdtCSE6kqBgAosXL2b79u3ExMTwl7/8xXgybO+VV15hyZIlDB8+vMvn423mzZvH+PHjOXDgAH5+fh36B9rMmjXLmBcF4Nlnn2Xs2LFMnDiR8PDwkx77qaeeorGxkZiYGKKionjqqacAuOeee1i2bBmxsbHs37+/w93F2brtttsoLi4mJCSEl156ieeffx6A3NxcLr/8cgDGjh3Ltddey6hRoxg+fDgtLS3ceeedALzwwgu89NJLhISEUFxczG233QbAo48+SlVVFddddx0jRozgyiuvNNa5ceNGfve7351z24W4kKi2q8TzRVxcnE5KSuqwbt++fR2eqV+IrrrqKv7xj38QGhra103pNyZPnsyKFSsYPHhwjx9b/uZEb0tISAAgPj7+rPZXSm3XWsedrpz0KQwQzz//PHl5eRIUWhUWFvLwww+bJCAI0RfONhicKZM+PlJKzVRKHVBKHVZK/aWL7Q8rpfYqpZKVUj8qpQJN2Z6BLCwsrNdewnE+cHd3Z+7cuX3dDCHOOyYLCkopc2AJMAuIBOYppSJPKLYTiNNaxwBfAv8wVXuEEOJ89v333/P999+bvB5T3inEA4e11mla6wbgU6DD1F2t9UatdU3r4q+AnwnbI4QQ5636+nrq6+tNXo8pg4IvkNVuObt13cncBqztaoNS6k6lVJJSKqmwsLAHmyiEEKK9fjEkVSl1MxAH/L+utmut39Jax2mt49omNgkhhOh5pgwKOYB/u2W/1nUdKKWmAn8FrtRam/7eyERMkTo7KyuLKVOmEBkZSVRUFK+88spJjz0QUmcfOHCgQw4lR0dHXn75ZQB27drFuHHjGDFiBHFxccbheStWrCAmJsa4fvPmzYCkzhbirGmtTfLBMNw1DQgGrIDdQNQJZUYCR4DQ7h539OjR+kR79+7ttK63hYWF6aysrFOWee+99/S9997b5TZ7e/tO63Jzc/X27du11lpXVFTo0NBQnZqa2qlcY2OjHj58uG5sbOx2e8+kbE9YsmSJvuuuu7TWWn/yySf6+uuvP2X5pqYm7enpqdPT07XWWk+bNk2vWbNGa6316tWr9cUXX6y11rqyslK3tLRorbXevXu3DgsLMx5jwYIFevPmzT39VbTW/eNvTlxYtm/fbjwfnA0gSXfjHGuyeQpa6yal1H3Ad4A58K7WOlUp9Uxr41ZieFw0CPiiNeVBptb6ypMetDvW/gWO7Tm3xp/IazjMev6km02VOtvb29uYQdXBwYGIiAhycnKIjOw4iGsgpc5u8+OPPzJ06FACAw2jlJVSxqSC5eXl+Pj4AB3vsKqrqzscT1Jni4Fk1KhRvVKPSSevaa3XAGtOWPe3dj93TpJzHnrzzTdZt24dGzduxM3Njfvvv5+RI0fyzTffsGHDBm699VZ27drVYZ+21Nm33norS5YsOW0d6enp7Ny5k7Fjx3ba1lXq7DvuuAOARYsW8c4773D//fcDx1Nnm5ub8+STT3LppZfy7rvvUlZWRnx8PFOnTjWmzraxseHQoUPMmzePE2eRw5klxDtZ6mw3N7cuv++nn37KvHnzjMsvv/wyM2bM4M9//jMtLS1s3brVuO3rr7/miSeeoKCggNWrVxvXx8XFsWjRoi6PL4Q4ie7cTvSnT399fBQYGKgLCwu11lqPGDFCHzlyxLjNz89Pl5eXd3h85OLiohsaGrTWWpeXl3f5+KhNZWWlHjVqlF6+fHmX2++44w79ySefGJd/+uknfdFFF+no6GgdFBRkfGwzf/58vXTpUmO50aNH66ioKB0bG6tjY2O1v7+/3rt3ry4rK9M333yzjo6O1rGxsdrW1vYsfyvHRUVFdXi8NmTIEOPv60T19fXa1dVVHzt2zLju/vvv119++aXWWuvPPvtMX3bZZZ32+/nnnzusb2ho0C4uLufc9q70h785cWFZs2aN8RHq2aCvHx+J0+tO6uzGxkauueYabrrpJq6++uouywyU1Nlt1q5dy6hRo/D09DSuW7ZsmbGj/brrruP222/vtN/kyZNJS0ujqKgINzc3SZ0tBpTeeoVuvxiSOtD0VOpsrTW33XYbERERPPzwwyetb6Ckzm7zySefdHh0BODj42N8gc6GDRuMOZ4OHz5sbP+OHTuor683BhtJnS3EmZOgYAI9lTp7y5YtfPDBB2zYsME4THPNmjWdyg2U1Nlg6Cxev359p7uit99+m0ceeYTY2FiefPJJ3nrrLQCWL19OdHQ0I0aM4N577+Wzzz4zBhtJnS3EmZPU2QOEpM7uTFJni4Fk1apVAMyePfus9u9u6my5Uxgg2lJnCwNJnS0GmoCAAAICAkxej3Q0DxBhYWGdOowvZJI6Www0sbGxvVKP3CkIIYQwkqAghBDngVWrVhn7FUxJgoIQQggjCQpCCCGMJCj0EFOkzq6rqyM+Pp7Y2FiioqJ4+umnT3rsP/3pTx3mKvQ3JSUlTJs2jdDQUKZNm0ZpaWmX5R577DGioqKIiIjggQceME5MmzlzpvH3cPfddxsn1C1evBhfX99O8zj27NnDggULeuW7CTGQSFDoIW+88Qbr168/5QziM2Vtbc2GDRvYvXs3u3btYt26dfz666+dyhUXF/Prr78yefLkbh+7t6bMt3n++ee57LLLOHToEJdddplx8lp7W7duZcuWLSQnJ5OSkkJiYqJxFvPnn3/O7t27SUlJobCwkC+++MK430MPPWScTd02EW748OFkZ2eTmZnZO19QiAFiwA1JfSHhBfaX7O/RY4a7hPN4/OMn3W6q1NlKKeMdRGNjI42NjV2mhli+fHmHF8o888wzrFq1itraWiZMmMB//vMflFJccskljBgxgs2bNzNv3jxuvRlHhn4AABFRSURBVPVW7r77buOJ8+WXX2bixIkkJCTw4IMPGnMHvffee+c83HXFihXGHEzz58/nkksu4YUXXuj0fevq6mhoaEBrTWNjozH/UVuakKamJhoaGrqVN2r27Nl8+umnPPbYY+fUdiH6gyFDhvRKPXKn0APefPNNfHx82LhxIw899BBPP/00I0eOJDk5mb///e/ceuutnfZpS529Z88e4zsTutLc3MyIESPw8PBg2rRp3Uqdfd9995GYmEhKSgq1tbV8++23xm0NDQ0kJSXxyCOP8OCDD/LQQw+RmJjI8uXLjUnmwsPD2bRpEzt37uSZZ57hySef7FRnZWVlh7ektf/s3bu3U/n8/Hzj9/Ty8iI/P79TmfHjxzNlyhTjeyRmzJjRYdbwjBkz8PDwwMHBgWuvvda4/vXXXycmJoaFCxd2eCwVFxfHpk2bTvq7FeJ8EhUVRVRUlOkr6k4q1f70uRBTZ2utdWlpqb7kkv/f3r0HR1VnCRz/HiKYYgcNBYECAWWYIEgekCFZUCCJTEBQnrKLLGoAF1TMWuqWtcDKLjVr1SDo+mBnS9iVWqRKHoIhUUchkQgiYSAqzymBDAbIGJcYR0giMAk5+8ft3E2nE9JAdyfpnE9Vir73/rrv+aU7nL73d+/5peqRI0d8tqWnp2tBQYG7vGXLFk1OTtbY2Fjt3bu3/uY3v1FV1ZSUFP3000/ddtHR0W7Z7ISEBO3du7dWVFTomTNndOrUqTpkyBCNjY31ms3set16661ey1FRUT5tTp48qRMnTtSKigqtqKjQESNG6O7du73aXLx4UadPn647duxQVdXvvvtOa2pq9MqVK7pkyRKdO3eu2/bEiROamJh4w7E3pjV85kz7Ul1dfUMzJmKls1s/f06B1ImKiiItLY2PP/7Yp/Jn/dLZly5dYuHChRQWFtK3b1+WLVvmVVa7fnG72tpa9u3b51MaOzMzk7S0NLKysiguLiY1NdUnnoqKCkaPHt1orO+8847P7HA9e/aktLSUXr16UVpaSo8ePXyel5WVxYgRI9xTZhMmTKCgoMBrP5GRkUyZMoXs7GzS09O9ymvPnz+fBx54wF220tkmnHz00UfA9dc+8pedPgqCQJXOLisr48cffwTg4sWL5ObmNlr1tH7p7LoE0L17dyorK9myZUuTcY4bN45Vq1a5y3Wzw50/f57bbrsNcK6YakyXLl0aLZt98OBBn4QA3qWz161b1+g4Sr9+/di1axc1NTVUV1eza9cuBg8eTGVlpVvXqaamhg8//ND9PdSv95SVleWVMK10tjHXzpJCEASqdHZpaSlpaWnEx8eTlJREenq61zfhOvfff787iBsVFcX8+fOJjY1l/PjxJCUlNRnnG2+8QWFhIfHx8dx11128+eabgHNZ6OLFixk2bFjArlJatGgRubm5xMTEkJeXx6JFiwAoLCx0xzJmzJjBgAEDiIuLIyEhgYSEBCZNmkRVVRWTJ08mPj7eHV954okn3Fjj4uKIj48nPz+fV1991d2nlc425tpZ6ewwMWrUKD744AOioqJaOpRW4fLly6SkpLBnzx5uuinwZ0ntM2dCzUpnm2vyyiuv2DX59Zw5c4bly5cHJSEYE87sLyZMNHapansWExNjEw6ZsDJw4MCQ7MeSgjHGtAGhmi/FTh8ZY0wbcOnSJa/Ly4PFkoIxxrQBubm55ObmBn0/lhSMMca4LCmEgTlz5tC/f3+GDh1KYmIiBQUFPusTEhL45JNPmnyNcCi9nZ+f71WDKTIykm3btgGwc+dOEhMTiY2NJSMjw73/Ijs7273/Yfjw4ezZswdwbhysX2TQmPbCkkKYWLlyJQcPHmT58uU8/vjjPutfe+0194avhsKl9HZaWpp7V/XOnTvp3Lkz48aNo7a2loyMDDZu3MjRo0e5/fbb3RsKx44d65YmX7t2rXsjXXR0NL169eLzzz8PaT+NaWlhmRREpMmfNWvWuO3WrFlz1bb+Ki4uZtCgQcyZM4eBAwcye/Zs8vLyuOeee4iJiWH//v0AVFVVMW/ePJKTkxk2bBjZ2dnu80ePHk1iYiKJiYns3bsXcEpkpKamMmPGDAYNGsTs2bNp7mbDMWPGuCUv6hs5cmSTd043Vno7KSmJ2NhYFixY4O4zNTWVZ555huHDh/P6669TVlbGgw8+SFJSEklJSe5/oPv372fkyJEMGzaMu+++m+PHj/v9u2xKdnY2GRkZgFN6u+4IoClbtmxhwoQJdO7cmfLycjp16uRe0peens7WrVsBZ3Kjuve6qqrK632fOnVqQOfHMKZN8KdqXmv68adKKtDkz+rVq912q1evvmpbf33zzTcaERGhhw8f1itXrmhiYqLOnTtXa2trddu2bTplyhRVVV28eLGuX79eVZ2qpzExMVpZWalVVVV68eJFVXUqe9b1MT8/X2+55RY9e/asXrlyRUeMGKGfffaZz/4zMjL03XffVVXVzZs3a3Jyss/6rKwsnTVrVqPxP/roo5qTk+Mul5eXu48ffvhhd1tKSoo++eST7rZZs2a58Zw+fVoHDRqkqk7V17pqjrm5uTp9+nSffV64cMGrQmv9n2PHjvm0r19ltba21qfqakNpaWn6/vvvu+379eunBw4cUFXVp59+WmNjY9227733nt55553atWtX3bt3r7u+pKTEq119ViXVhFpRUZEWFRVd9/NpDVVSReQ+4HUgAvhvVV3eYPvNwNvAL4FyYKaqFt/oftXP0h0LFixgwYIFN7o7APr3709cXBzg1D0fO3YsIkJcXBzFxcUA7Nixg5ycHF5++WXAucTszJkz9O7dm8zMTA4ePEhERAQnTpxwXzc5OZk+ffoAMHToUIqLixk1apTP/p9//nlefPFFoqOjeeutt7zWL1myhJKSEnesoaHS0lKio6Pd5fz8fFasWMFPP/3EDz/8wJAhQ9xb62fOnOm2y8vL85o74cKFC1RWVnL+/HkyMjI4efIkIkJ1dbXPPusK6l2P5o7kSktLOXLkCOPHj3fbb9y4kWeffZbLly8zbtw4IiIi3PbTpk1j2rRp7N69m6VLl5KXlwdAjx49+Pbbb68rRmMCbcCAASHZT9CSgohEAL8F0oES4ICI5Khq/RlYHgP+rKq/EJGHgJeAmb6v1vrdfPPN7uMOHTq4yx06dHDPv6sqW7du9bkJZdmyZfTs2ZNDhw5RW1vrVcq6/utGREQ0eS5/5cqVXhPPNFy/atUq5s2bxxdffOHTJlxKb9fZvHkz06ZNo2PHju66kSNHuhPu7Nixwyvx1hkzZgynTp3i+++/p3v37lZ627QqlZWVQOPzuQdSMMcUkoEiVT2lqn8BNgIN6yVPAepKiG4Bxsq1nMxvY8aPH8+qVavcI5mvvvoKcEpV9+rViw4dOrB+/Xp3UvpAyszMpLa2lu3bt/tsC5fS23U2bNjArFmzvNadO3cOcArlvfTSS+6ge1FRkft+fPnll1y+fJlu3boBVnrbtC75+fnk5+cHfT/BTAq3AWfrLZd41jXaRlVrgPNAtyDG1KKWLl1KdXU18fHxDBkyhKVLlwKwcOFC1q1bR0JCAl9//bXXt/FAERFeeOEFVqxY4bMtXEpvgzNof/bsWVJSUryev3LlSgYPHkx8fDyTJk3i3nvvBZxB9tjYWIYOHcpTTz3Fpk2b3FNTVnrbtEdBK50tIjOA+1T17z3LjwB/raqZ9doc9bQp8Sz/0dPm+wavtQBYANCvX79fnj592mtfVsb4xlnpbV9jxowhOzubrl27+myzz5xpa1pD6ew/AX3rLffxrGu0jYjcBNyKM+DsRVXXqOpwVR1ef0DUBI6V3vZWVlbGc88912hCMCacBfPqowNAjIj0x/nP/yHg7xq0yQEygAJgBrBTg3XoYq7KSm97i46OZurUqS0dhjEhF7SkoKo1IpIJbMe5JHWtqh4TkV/jXC+bA7wFrBeRIuAHnMRxvfu7phvOjLle9r3FhLOg3qegqr8Dftdg3b/Ue3wJ+Jsb3U9kZCTl5eV069bNEoMJKlWlvLzc5zJcY8JFWEyy06dPH0pKSigrK2vpUEw7EBkZ6d5QaEy4CYuk0LFjR/r379/SYRhjTJsXlgXxjDHGXB9LCsYYY1yWFIwxxriCdkdzsIhIGXC62YaN6w5832yr8GJ9bh+sz+3DjfT5dlVt9u7fNpcUboSIFPpzm3c4sT63D9bn9iEUfbbTR8YYY1yWFIwxxrjaW1JY03yTsGN9bh+sz+1D0PvcrsYUjDHGXF17O1IwxhhzFWGZFETkPhE5LiJFIrKoke03i8gmz/bfi8gdoY8ysPzo83Mi8gcROSwin4jI7S0RZyA11+d67R4UERWRNn+lij99FpG/9bzXx0TknVDHGGh+fLb7iUi+iHzl+XxPbIk4A0VE1orIOc8kZI1tFxF5w/P7OCwiiQENQFXD6genTPcfgZ8DnYBDwF0N2iwE3vQ8fgjY1NJxh6DPaUBnz+Mn20OfPe26ALuBfcDwlo47BO9zDPAV0NWz3KOl4w5Bn9cAT3oe3wUUt3TcN9jnMUAicLSJ7ROBjwABRgC/D+T+w/FIIRkoUtVTqvoXYCPQcJb3KcA6z+MtwFhp2zW3m+2zquar6k+exX04M+G1Zf68zwD/BrwEXAplcEHiT5/nA79V1T8DqOq5EMcYaP70WYFbPI9vBb4NYXwBp6q7ceaXacoU4G117AOiRKRXoPYfjknhNuBsveUSz7pG26hqDXAe6BaS6ILDnz7X9xjON422rNk+ew6r+6rqh6EMLIj8eZ8HAgNF5HMR2Sci94UsuuDwp8/LgIdFpARn/pZ/CE1oLeZa/96vSViUzjb+E5GHgeFASkvHEkwi0gH4d2BOC4cSajfhnEJKxTka3C0icar6Y4tGFVyzgP9R1VdEZCTObI6xqlrb0oG1ReF4pPAnoG+95T6edY22EZGbcA45y0MSXXD402dE5FfAPwOTVfVyiGILlub63AWIBT4VkWKcc685bXyw2Z/3uQTIUdVqVf0GOIGTJNoqf/r8GLAZQFULgEicGkHhyq+/9+sVjknhABAjIv1FpBPOQHJOgzY5QIbn8Qxgp3pGcNqoZvssIsOA1TgJoa2fZ4Zm+qyq51W1u6reoap34IyjTFbVwpYJNyD8+WxvwzlKQES645xOOhXKIAPMnz6fAcYCiMhgnKQQztMw5gCPeq5CGgGcV9XSQL142J0+UtUaEckEtuNcubBWVY+JyK+BQlXNAd7COcQswhnQeajlIr5xfvZ5JfAz4F3PmPoZVZ3cYkHfID/7HFb87PN2YJyI/AG4Ajyvqm32KNjPPv8j8F8i8izOoPOctvwlT0Q24CT27p5xkn8FOgKo6ps44yYTgSLgJ2BuQPffhn93xhhjAiwcTx8ZY4y5TpYUjDHGuCwpGGOMcVlSMMYY47KkYIwxxmVJwZgQEpE76qpfikiqiHzQ0jEZU58lBWP84LlRyP5eTNizD7kxTfB8qz8uIm8DR4FHRKRARL4UkXdF5GeedkkisldEDonIfhHp4nnuZ562X4rI3S3bG2P8E3Z3NBsTYDE4JVGKgPeAX6lqlYj8E/CciCwHNgEzVfWAiNwCXATOAemqeklEYoANOIUIjWnVLCkYc3WnVXWfiDyAM4HL554yIZ2AAuBOoFRVDwCo6gUAEfkr4D9EZChOuYmBLRG8MdfKkoIxV1fl+VeAXFWdVX+jiMQ18bxngf8FEnBO04bDJD+mHbAxBWP8sw+4R0R+Ac6RgIgMBI4DvUQkybO+S71y7KWemv6P4BRzM6bVs6RgjB9UtQxnwp4NInIY59TRIM8UkTOBVSJyCMjFKd38n0CGZ90g/v+Iw5hWzaqkGmOMcdmRgjHGGJclBWOMMS5LCsYYY1yWFIwxxrgsKRhjjHFZUjDGGOOypGCMMcZlScEYY4zr/wDRjHAOWBKAYAAAAABJRU5ErkJggg==\n", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# Plot the PR curves for the positive class\n", "for j in range(k):\n", " plt.plot(recall_plus[j], precision_plus[j], label='fold %d (area = %.3f)' % (j+1, auprc_plus[j]))\n", "\n", "plt.plot(recall_plus['avg'], precision_plus['avg'], 'k--', label='mean PR (area = %.3f)' % auprc_plus['avg'], lw=2)\n", "plt.plot([0,1,1], [1,1,0], linestyle='--', color=(0.6, 0.6, 0.6)) # PR curve for a perfect classifier\n", "\n", "plt.xlim([-0.05, 1.05])\n", "plt.ylim([-0.05, 1.05])\n", "plt.xlabel('recall')\n", "plt.ylabel('precision')\n", "plt.legend(loc='lower left')\n", "\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### PR curve w.r.t the negative class" ] }, { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl4VOX1wPHvyWQPATRhDwEsWwgkgGEXZBEEK2LB1qIVqCh1wQWXYl1RW4uICqJWsVgVF2zVIigiIPEnqwkEwlZZRAgJe4Ds2yTv74+ZXDMkIQNkJiY5n+eZJ3Pvfe99zw1hzrx3OVeMMSillFIAPjUdgFJKqV8OTQpKKaUsmhSUUkpZNCkopZSyaFJQSill0aSglFLKoklBKaWURZOCUkopiyYFpZRSFt+aDuB8hYeHm7Zt29Z0GEopVats3rz5pDGmSVXtal1SaNu2LZs2barpMJRSqlYRkYPutNPDR0oppSyaFJRSSlk0KSillLJoUlBKKWXRpKCUUsrisaQgIm+LyHER2VHJchGRV0Rkn4hsE5GenopFKaWUezw5UngHGHmO5aOADs7XFOAfHoxFKaWUGzx2n4Ix5jsRaXuOJmOA94zjeaAbRaSxiLQwxhzxRDz/jX+DbetTKci3u8zftXs7m7dupE3npvz22gkAlBSXcOJwFgDJOzazbWcSQUHB3HDdzQBc0iQY/0A/AHbs2czyJfE0DG3EmGt+57Jtm03wizhBns9x/EwIJqUtxSWujz9du2E1P6X8yBUjetA3ZhgABXlFnDmZC0D8mq9JPZxCRMtIhgy8GoAmrULx8XHk869XL2V70m7aRf6KK/oNddl2QJAvxc1+pEhy8Mm5FHO8ebnfy+fL/k1mVga/nTCKNk2jAcjJyCc7swCAT5Z8QF5eLjHRPYntejk2mxDesqG1/ptvvU5WRi6Xd+9Ll07dXLbdoFEAWZduAcAca45P7qUuy+32Ij769B0A/nTfLYT6NQPgzIkcCvLt5OXl8smSDwAYOvBqWrWMJCDIl8bhIY7fkz2HeXMc3yVGDLmWZk1buGw/sLGdvEv2ACBp7aAwyGX5seNHWBH/BcGh/vzpjtuxGcfyk2mZFJcY0g6nsHrN1wDccN3NBAUF06BRACENAx3rn05h4YJ/AzB+3CR8ff1cth/UPIfcIMel4YFHYmrsby/YNKTxmSiC/P0ICvADBICmcpJQySXPBHDYNONszeUEIZJHjgniaAX3PLWUYwRJAVkmmOMmvNzyVnKUQCkk0zTghLm03PJInyN0bOJPFg35X0ZQueXDW2QSaDPszgxgT2ZgueWjWmbg6wM7MwLZnxVQbvnoiAwAkk8HkZLj75gZ2BCCLsHX15dRo0YBkJSURFpamsu6AQEBjBgxAoCEhASOHTvmsjwkJIShQx3/39avX096errL8kaNGjFo0CAAvvvuOzIyMlyWh4WF0b9/fwBWr15NTk6Oy/JmzZrRu3dvAFasWEFBgeP/Y/v27YmKiiq3r9WpJm9eawUcKjOd6pxXLimIyBQcowkiIyMvqLNth74ltzASe26xy/wzx3NI3XuK3EZwSvIAKMFQkFsEwOljjuUhIQXWvDPFBfiK4z/4/uxTpO49xSWX/LyOFbevkOxzjAM+P9HYNKZ3fiuM3TUpnEjLInXvKXb1PU5HZ//2kmJrW8cPZZK6/xT+xY2teadMPj7i+I+9/0Q6qXtP0cDWtFz/xT6GDT6HOCNnaG2K6JwbVu73cuSnM5w+fZr/FZ4k1Nl/ob2IQue2Dv94mpycHFqH51BwWRHiK9bvCeDAT+nkpefxq5Y5FLR27d8EC1/67AcgqjCQiNxQl+VFRUWk7j0FwD5O00YcySa/sBB7bjF5OYXW8szoPMIbF1HsYyhx9p9JtrU8u0c+jRu49n+6QS7rnP33KWhGwzzXP/fsM/mk7j2Fb2Nf0iSbBs75eflFGLsh81Setf287EJ8jB8mWCgQx7/hwcIMa3l+ThF+rjmBFE6xzdn/qMLONfa3F2pvRL+jzTElhiBfISxYCA8WOhaspnXRPtL9WpDV6DrO1iVzOc0KD3LMvw05DcsP+rtmLCGs6AhpAe3JCx1WbnnMmc9obD/BwcAo8hsMKrc89vQiGhzKIDMoFkL6llvOgYVQkgtBl0NwXPnlP70NpgiC+0FQTPnl+990/AwZBIHOD9LcIGh1efm2tUBp4vF0UhDHF3UPbdwxUvjCGNO1gmVfADONMWud098A040x57xdOS4uzlzoHc07duzg1KlTFS4LCwsjOtrxTbmgoIDvv/++0u1ER0cTFub4gD1w4AApKSkVtvP396dv35//2Dds2EBRUVGFbdu0aUObNm0AOHnyJLt27aq0/759++Lv76/7pPvk1j59ueNLJt0zicwtmRRmFFptmjRpwnXXXcef//xnOnbsWGmf56ukpAQRQZxfXPLy8sjPz6e4uNjllZVfxPdH7Czedpzk1Az8bT4M79KM3/VqzRXtw7H5SLXFBMAnk+HwFrg3qXq36yVLly4FYPTo0Re0vohsNsZUkF3PaleDSeFN4FtjzEfO6d3A4KoOH11MUlCqvtp0dBN3r7obDkL3Y91Z9eUq9u93jGJ27NhhJaXbbruNrVu3YrfbKS4udvk5duxYZs+eDcCuXbsYMGCA9QFf2q642DEaWrdunXV4ZOrUqbz22msVxhUZGcnBgwf54Wgm/05M5fUXn6PQN5gmzVtxTb+uTBrRi5j2ra0Ec1FqeVJISnLE3bPnhV2T425SqMnDR0uAqSKyCOgDZHjqfIJS9V1c8zjeHvk2f1r1J/Z22MvXT39NXmoe33zzDV26dLHa/fDDD2zevLnCbZQ9rm6M4cyZM5X2V1JSYr0PCQmhUaNG2Gw2l5cxhssuuwyAzs0b8sjV7Zkx5kOMMZwG9nwAcwCbXwDNW0Xwwsy/M/7G3wKOUdKBAwdo06YNrVq1skZkddmFJoPz5bGRgoh8BAwGwoFjwFOAH4Ax5g1xpP5XcVyhlAv8sapDR6AjBaUuxt7Te5mycgr2EjtvDH+D6LBol+W7du0iNzcXm82Gr6+vy8+GDRvSpInjhLPdbiczM9Nleemr9CIIdxhjrFFAbm4u8+bNIyUlhZSUFH7cf4CfDhwgPzcbgMjfPc6E8Tfyu7jWfPXhm0yfPh0AEaFVq1b06NGDAQMGWK9yo4taPlK4WL+Iw0eeoElBqYuTkpnC7StuJ7Mwk1eHvcrlzX7ZJ15Pnz7DFxu2sf4IrP4phwJ7CaEp68jfsYK808c5cjjNZWTSokUL0tLSrKSwYsUKYmNjabbmUY8lBWMMdmPHXmKnqKSIouIi7CV2fMSHJsFVVqt2y1dffQVgXTV1vmrD4SOlVA2IbBjJu6Pe5fYVt3PHyjuYM2QOA1oNqOmwKnXJJY255ZpB3AKczi1g8dYUPknyY1e7GIJ9S/jDZQ2IaVRAyg872bF5C76B/ry6cSn59gJOnTnFzDF3gjGEhQcR28WfOP8xiL/BboocH+TGTrGxY6eYYoqxA74tOuHj7+/4gHe+CosLf54u/nl+aSKozKtDX+XK1lde9O/BbrdX3agaaFJQqh5qHtKcd0a+wx2r7mDq6qnMGjSL4W2Gu72+MYaikiLyi/PJtztfZ70vKC4g3+74ab3sBdayguICCosLybfnU1hc6Dqv+Od5hcWFFJY43ttLnB+ModDAeYVzEpCUAbQArnXMm7/nMQAKjhcQEhVM3o95pJ/MY/V3eWw4tJI297QmpLEfvhj8DPga8DOCnzEEGTuFGWlcEhaBn48fQX5B+Pn4ubz8bf74+vj+PM/mh6/44mdzTPv6+JJTlMPcpLmcyq/4qrNfKk0KStVTYUFhLLh6AXetuouH/u8hxnUYh4/4kGfPI8+eR7493/Vn8c/T+cX5lJiSqjupgJ+PH4G2QAJ8Awiw/fzyt/kTYAsg2C/Ymg60BeJv88fPx8+a52/zx9/H33rvgy8HThbigy+BvgEE+QYS5OdPsF8AIf6BBI8PJEBs7N22kwfuvJuUn1LInJ3NB0uXljt5m7Z/J63e609ij/voNequi/r9Hsk+wtykuRe1jZqgSUGpeqyhf0PmD5/P9DXTWfrjUoJ8gwjyDSLQN9D62TiwMS18WzimbYEE+jpfzvcBtgCrben70g/60nmlH+4BtgBsPrbq35EOVTeJadWOKzf1ZezYsaxZs4aXX36ZhQsXVn8stZwmBaXquWC/YOYNnVfTYXhFeHg4q1atYtasWTzwwAM1Hc55udBqDudLS2crpeoVf39/Hn/8cYKDgwHHpbBPP/00+fn5NRzZucXGxhIbG+vxfjQpKKXqtbvvvpsZM2YwdOhQTpw8WdPh1DhNCkqpem3atGlERkayYcMGrv3NeLYdK656pRqwdOlSq/6RJ2lSUErVazExMSQkJNC3b1/SDh9hwNs5rEncXtNh1RhNCkqpeq9Zs2bEx8fzmzG/JrsQHv77m7zyyis1HVaN0KSglFJAYGAgr7w4k2cGB1BSYkhISMDTZYCMMRQUF1Bc8ss5ZKWXpCqllJOI8MSVATQcOJk7HnupWkp2f/7j53x/9HuyC7PJLsompyjHep9dlI29xE5sk1jev+b9atiDi6dJQSmlzjKg0RGS3n8MKcggPyOd2Z98z7SrWtK2QRHBxdmEkMOPIT3p8fCXlW6jUUAjIhpE8FPGTxzPPU4DvwY08G9A85DmNGjcgBC/EEL9Q1mXto4j2VU/NaC0zLinaVJQSimnkIZhFBpf4rK+gaxvyDJB3Lssjy+TMknce5I5f4yjfduONM/cTrPcPefcVrBfMF+N+6rKPk/nn2ZN6poq25U+CMnT9JyCUko5NQ5vTv60PWTcu4/ix9MJffoos5fvZ9CgQRzPKuT2Bckc6zCBow27eT02u93ulUqpmhSUUqqMho3DaHRpE2y+jgMpYWFhrFixgj/84Q/k5OQwZswYPv5ur9fj+uqrr6xnKniSJgWllKpCQEAA7733Hk8//TQlJSW8uHgrT39dN+9+1qSglFJuEBGefPJJ3n//ffxsPlz8dUm/THqiWSmlzsPNN9+MffO/6G770Zr30UcfkZyczIMPPmg9x7q20pGCUkqdpy6tL6VJA8d36pKSEp588kmef/552rZty0MPPcTRo0drOMILp0lBKaUugo+PDx9++CGjR48mNzeXF198kXbt2nH//fdz+PDhKtcvMQZ37pvu2LEjHTt2vPiAq6CHj5RS6iL16tWLJUuWkJSUxLPP/pXFi//L3Llz+cc/3uDeZ16myxUjOZVTSHpOIaeyC53vC0jPKSS/0SGCGhVW2UenTp28sCeaFJRS6oKUGMPTS3eSnl3IyewC6+epzpNp8cchZKxbRHFeJh8fuRSfpbsQgUuD/bk0xPHq1DyUS0P8WXncj7ySqscKpQ8BCgwM9Oh+aVJQSqnzFOzviymBTzalEtbAn7AGAbQJC+bytpcQHuJPWINowu+/gcZBvjRtGMilIf40DvbH5lP+mqWkD/xJLai6z5UrVwIwevTo6t4dF5oUlFLqPEW3bIgpDGL7tKvdam+323lg2v1MnjyZmJgYD0d3cfREs1JKXYDzKaA6Z84cXnnlFUaOHMnBgwc9F1Q10JGCUkpdLGMgPwOyj0HWUccr+ygUZEHvPzF16lS++OIL/u///o+RI0eydu1awsLCajrqCmlSUEqpC5F9HBZc7fjwzzoG9ryK213SjsAeN7N48WIGDRrE9u3bGT16NKtWrSI4ONi7MbtBk4JSSp2vNv0hLQlsfhDRCxo0g9Dm0KC542docygugn/0A1MCQOPGjfnqq6/o168fGzZsYPz48Xz66adud9mxs+fvUQBNCkopdf56TnC8zuXMoXKzWrVqxddff82AAQNYsmQJs2bNgjZgKCbpWBIn8k5wMu8kx3OPczLvJCdyT1jzzhSc4en+T9MJz96v4NGkICIjgbmADfinMWbmWcsjgXeBxs42jxhjlnkyJqWU8i7XexCioqJYunQps2bN4p577mH90ifBls3E5ROtNr7iS1hQGE2CmhARGkHPpj0Jt4XTLrCdx6P1WFIQERvwGjAcSAUSRWSJMWZXmWaPA/82xvxDRLoAy4C2nopJKaW8xubv+LnkHvjmWWjUChq2gkYRDGjYks//eguc3klP+nFgp53x7ZsTYALwN/742H0oOFFAfn4+V1xxBVf2u5KlS5eSeiyVHq17eDRsT44UegP7jDH7AURkETAGKJsUDNDQ+b4RUHWhEKWUqg1Cm8GN78OxXZCZChlpkP4j/PQdFGRazR4vNuxbns/j24sq3MwT993Kld3bgzEYLxTs9mRSaAWUPaiWCvQ5q80MYIWI3AOEAFd5MB6llPKuqNGO19nyMyEzDTLTmPnGInYUrGBMbxv+FBJIPsEmj2BbMYG+MDB9Ebz8CSbsDrJD2no85Jo+0TweeMcY86KI9AMWikhXY5yn651EZAowBSAyMrIGwlRKqWoU2NDxahpFXv9mZPr+hq0uDQxhtny6hmaRGJTBSb8zFNsa4RfYwOOheTIppAGty0xHOOeVNRkYCWCM2SAigUA4cLxsI2PMfGA+QFxcnDtVZpVSqlaY2L8NTUMDaNowgOYNA2neKJDmznpJUua26aVLl3olHk8mhUSgg4i0w5EMfg/cdFabFGAY8I6IRAGBwAkPxqSUUr8onZs3pHPzhlW281bNJI8lBWOMXUSmAl/juNz0bWPMThF5BthkjFkCPAi8JSLTcJx0nmSM0ZGAUkqdpU2bNl7px6PnFJz3HCw7a96TZd7vAgZ4MgallKoLzpw5AzjujPYkrZKqlFK1wJo1a1izZo3H+9GkoJRSyqJJQSmllEWTglJKKYsmBaWUUpaavqNZKaWUG3r08GwhvFKaFJRSqhaIiIjwSj96+EgppWqBkydPcvLkSY/3o0lBKaVqgQ0bNrBhwwaP96NJQSmllEWTglJKKYsmBaWUUhZNCkoppSx6SapSStUCvXr18ko/mhSUUqoWaN68uVf60cNHSilVCxw9epSjR496vB9NCkopVQskJiaSmJjo8X40KSillLJoUlBKKWXRpKCUUsqiSUEppZRFL0lVSqlaoF+/fl7pR5OCUkrVAuHh4V7pRw8fKaVULZCamkpqaqrH+9GRglJK1QJbtmwBPP8ENh0pKKWUsmhSUEopZdGkoJRSyqJJQSmllEVPNCulVC0wcOBAr/SjSUEppWqBxo0be6Ufjx4+EpGRIrJbRPaJyCOVtPmdiOwSkZ0i8qEn41FKqdrq4MGDHDx40OP9eGykICI24DVgOJAKJIrIEmPMrjJtOgB/AQYYY06LSFNPxaOUUrXZtm3bAGjTpo1H+/HkSKE3sM8Ys98YUwgsAsac1eZ24DVjzGkAY8xxD8ajlFKqCm6NFEQkABgHtC27jjHmmXOs1go4VGY6FehzVpuOzu2vA2zADGPM8gr6nwJMAYiMjHQnZKWUUhfA3cNHnwMZwGagoJr77wAMBiKA70SkmzHmTNlGxpj5wHyAuLg4U439K6WUKsPdpBBhjBl5nttOA1qX3YZzXlmpwPfGmCLgJxHZgyNJeP5BpEoppcpx95zCehHpdp7bTgQ6iEg7EfEHfg8sOavNYhyjBEQkHMfhpP3n2Y9SStV5Q4YMYciQIR7vx92RwhXAJBH5CcfhIwGMMSamshWMMXYRmQp8jeN8wdvGmJ0i8gywyRizxLlshIjsAoqBh40x6RexP0opVSc1aNDAK/2IMVUfoheRCq+BMsZ4/qLZs8TFxZlNmzZ5u1ullKpRP/74IwC/+tWvLmh9EdlsjImrqp1bIwVjzEERiQVK77NeY4xJvqDIlFJKnbdduxy3eF1oUnCXW+cUROQ+4AOgqfP1vojc48nAlFJKeZ+75xQmA32MMTkAIvI8sAGY56nAlFJKeZ+7Vx8JjhPBpYqd85RSStUh7o4U/gV8LyL/dU5fDyzwTEhKKaVqirsnml8SkW9xXJoK8EdjzBaPRaWUUsrF8OHDvdLPOZOCiDQ0xmSKyKXAAeerdNmlxphTng1PKaUUQGBgoFf6qWqk8CFwLY6aR2VvaBDn9GUeiksppVQZu3fvBqBTp04e7eecScEYc63zZzuPRqGUUuqc9uzZA3g+Kbh7n8IAEQlxvv+DiLwkIlrDWiml6hh3L0n9B5DrvKv5QeBHYKHHolJKKVUj3E0KduMokjQGeNUY8xoQ6rmwlFJK1QR371PIEpG/AH8ABomID+DnubCUUkrVBHeTwo3ATcBkY8xR5/mEFzwXllJKqbJGjRrllX7cvXntKPBSmekU4D1PBaWUUsqVr6+73+Evsp9zLRSRtcaYK0QkiwruUzDGNPRodEoppQDYuXMnANHR0R7tp6r7FK5w/tSTykopVYP273c8qdjTScHd+xT6ikhomelQEenjubCUUkrVhPO5TyG7zHSOc55SSqk6xO3nKZgyD3M2xpTg/pVLSimlagl3k8J+EblXRPycr/uA/Z4MTCmllPe5+23/DuAV4HEcVyF9A0zxVFBKKaVcjR492iv9uHufwnHg9x6ORSmlVA1z9+qjjiLyjYjscE7HiMjjng1NKaVUqeTkZJKTkz3ej7vnFN4C/gIUARhjtqEjB6WU8pqUlBRSUlI83o+7SSHYGJNw1jx7dQejlFKqZrmbFE6KyK9wlroQkRuAIx6LSimlVI1w9+qju4H5QGcRSQN+Am72WFRKKaVqRJVJwfnshDhjzFXOR3L6GGOyPB+aUkqpUr+IKqnguHtZRP4M/NsYk+OFmJRSSp3FW89TcPecwioReUhEWovIpaWvqlYSkZEisltE9onII+doN05EjIjEuR25Ukqpanc+T14zwF1nzb+sshVExAa8BgwHUoFEEVlijNl1VrtQ4D7ge3eDVkqp+iYpKQmAnj17erQfd0cKXXB8wCcDW4F5QFVFvXsD+4wx+40xhcAiYEwF7Z4Fngfy3YxFKaXqnbS0NNLS0jzej7tJ4V0gCkf9o3k4ksS7VazTCjhUZjrVOc8iIj2B1saYL92MQymllAe5e/ioqzGmS5npeBHZVWlrNzivanoJmORG2yk4C/BFRkZeTLdKKaXOwd2RQpKI9C2dcD51bVMV66QBrctMRzjnlQoFugLfisgBoC+wpKKTzcaY+caYOGNMXJMmTdwMWSml1Plyd6RwObBeREoLb0QCu0VkO2CMMTEVrJMIdBCRdjiSwe+Bm0oXGmMygPDSaRH5FnjIGFNVslFKqXonICDAK/24mxRGnu+GjTF2EZkKfA3YgLeNMTtF5BlgkzFmyfluUyml6qsRI0Z4pR93n6dw8EI2boxZBiw7a96TlbQdfCF9KKWUqj7unlNQSilVgxISEkhIOLtYdfXzTjENpZRSF+XYsWNe6UdHCkoppSyaFJRSSlk0KSillLLoOQWllKoFQkJCvNKPJgWllKoFhg4d6pV+9PCRUkopiyYFpZSqBdavX8/69es93o8ePlJKqVogPT3dK/3oSEEppZRFk4JSSimLJgWllFIWPaeglFK1QKNGjbzSjyYFpZSqBQYNGuSVfvTwkVJKKUudGCkUFRWRmppKfn5+TYei6oHAwEAiIiLw8/Or6VBUPfLdd98Bnh8x1ImkkJqaSmhoKG3btkVEajocVYcZY0hPTyc1NZV27drVdDiqHsnIyPBKP3Xi8FF+fj5hYWGaEJTHiQhhYWE6KlV1Vp1ICoAmBOU1+rem6rI6kxSUUkpdPE0K1eSVV14hKiqKm2++udI277zzDlOnTq1wWYMGDSqcf+utt9K0aVO6du16zv7nzJnDe++9537AXlZQUMCNN95I+/bt6dOnDwcOHKiw3dy5c+natSvR0dHMmTPHmn/jjTfSvXt3unfvTtu2benevTvguMhg4sSJdOvWjaioKP7+978DUFhYyKBBg7Db7R7fN6W8ISwsjLCwMI/3o0mhmrz++uusXLmSDz74oFq3O2nSJJYvX37ONna7nbfffpubbrrJ7e16+8NywYIFXHLJJezbt49p06Yxffr0cm127NjBW2+9RUJCAsnJyXzxxRfs27cPgI8//pitW7eydetWxo0bx9ixYwH4z3/+Q0FBAdu3b2fz5s28+eabHDhwAH9/f4YNG8bHH3/s1f1UylP69+9P//79Pd5Pnbj6qKynl+5k1+HMat1ml5YNeWp0dKXL77jjDvbv38+oUaO49dZbmThxIrfeeiv79+8nODiY+fPnExMT47LOTz/9xE033UR2djZjxoypdNuDBg2q9Ft1qdWrV9OzZ098fR3/nG+99Rbz58+nsLCQ9u3bs3DhQoKDg5k0aRKBgYFs2bKFAQMG8Oyzz3LPPfewY8cOioqKmDFjBmPGjOHAgQPccsst5OTkAPDqq69e9B/j559/zowZMwC44YYbmDp1KsYYl+Pz//vf/+jTpw/BwcEAXHnllXz22Wf8+c9/ttoYY/j3v//N6tWrAcfx/ZycHOx2O3l5efj7+9OwYUMArr/+ev7yl7+cc/SmlHKlI4Vq8MYbb9CyZUvi4+OZNm0aTz31FD169GDbtm0899xzTJgwodw69913H3feeSfbt2+nRYsWF9X/unXruPzyy63psWPHkpiYSHJyMlFRUSxYsMBalpqayvr163nppZf429/+xtChQ0lISCA+Pp6HH36YnJwcmjZtysqVK0lKSuLjjz/m3nvvrbDfgQMHWod0yr5WrVpVrm1aWhqtW7cGwNfXl0aNGpUrBdy1a1fWrFlDeno6ubm5LFu2jEOHDrm0WbNmDc2aNaNDhw6AI8GEhITQokULIiMjeeihh7j00kut7SUmJl7Ab1SpX57Vq1dbX4Y8qc6NFM71jd5b1q5dy6effgo4HqGXnp5OZqbr6GXdunVWm1tuuaXCwynuOnLkCFFRUdb0jh07ePzxxzlz5gzZ2dlcffXV1rLf/va32Gw2AFasWMGSJUuYPXs24Li0NyUlhZYtWzJ16lS2bt2KzWZjz549Ffa7Zs2aC465IlFRUUyfPp0RI0YQEhJC9+7drVhLffTRR4wfP96aTkhIwGazcfjwYU6fPs3AgQO56qqruOyyy7DZbPj7+5OVlUVoaGi1xqqUt5WO3D2OSLfiAAAXiElEQVStziWF2qS6Lm0MCgpyuW5+0qRJLF68mNjYWN555x2+/fZba1nZh38bY/j000/p1KmTy/ZmzJhBs2bNSE5OpqSkhMDAwAr7HThwIFlZWeXmz549m6uuusplXqtWrTh06BARERHY7XYyMjIqPGk2efJkJk+eDMCjjz5KRESEtcxut/PZZ5+xefNma96HH37IyJEj8fPzo2nTpgwYMIBNmzZx2WWXAY4T3JXFr5QqTw8fecDAgQOtE87ffvst4eHh1nHuUgMGDGDRokUAF31yOioqyjohC5CVlUWLFi0oKio657avvvpq5s2bhzEGgC1btgCOOydbtGiBj48PCxcupLi4uML116xZY538Lfs6OyEAXHfddbz77rsAfPLJJwwdOrTCpHj8+HEAUlJS+Oyzz1xOnq9atYrOnTu7JIrIyEhrSJ2Tk8PGjRvp3Lkz4HhSVXh4uJajUOo8aFLwgBkzZrB582ZiYmJ45JFHrA/DsubOnctrr71Gt27dSEtLq3Rb48ePp1+/fuzevZuIiAiX8wOlRo0aZdVFAXj22Wfp06cPAwYMsD4gK/LEE09QVFRETEwM0dHRPPHEEwDcddddvPvuu8TGxvLDDz+4jC4u1OTJk0lPT6d9+/a89NJLzJw5E4DDhw9zzTXXWO3GjRtHly5dGD16NK+99hqNGze2li1atMjl0BHA3XffTXZ2NtHR0fTq1Ys//vGP1kn9+Ph4fv3rX1907ErVJ1L6LbG2iIuLM5s2bXKZ97///c/lmHp99Jvf/IZZs2ZZJ2CV44T7zJkz6dixY7VvW//mlLclJCQA0Lt37wtaX0Q2G2Piqmqn5xTqiJkzZ3LkyBFNCk6FhYVcf/31HkkIStWEC00G58ujh49EZKSI7BaRfSLySAXLHxCRXSKyTUS+EZE2noynLuvUqZPXHsJRG/j7+1d4KbBS6tw8lhRExAa8BowCugDjRaTLWc22AHHGmBjgE2CWp+JRSqnabMWKFaxYscLj/XhypNAb2GeM2W+MKQQWAS637hpj4o0xuc7JjUAESimlyikoKKCgoMDj/XgyKbQCyt6OmuqcV5nJwFcVLRCRKSKySUQ2nThxohpDVEopVdYv4pJUEfkDEAe8UNFyY8x8Y0ycMSauSZMm3g1OKaXqEU8mhTSgdZnpCOc8FyJyFfAYcJ0xxvNjIw/xROnsQ4cOMWTIELp06UJ0dDRz586tdNtaOltLZytVHTx5SWoi0EFE2uFIBr8HXGo7i0gP4E1gpDHmuAdj8bjXX3+dVatWudxte7F8fX158cUX6dmzJ1lZWVx++eUMHz6cLl1cz9eXls5OSkpye9t2u92qquoNZUtnL1q0iOnTp5cra122dLa/vz8jR47k2muvpX379i5tH3zwQRo1agS4ls7Ozc2lS5cujB8/nrZt21qls7VKqqoLWrU619H36uOxTwVjjF1EpgJfAzbgbWPMThF5BthkjFmC43BRA+A/zpIHKcaY6y6q468egaPbLy74szXvBqNmVrrYU6WzW7RoYVVQDQ0NJSoqirS0tHJJQUtna+lsVff17NnTK/149KuiMWYZsOyseU+WeV++SE4t9MYbb7B8+XLi4+MJDw/nnnvuoUePHixevJjVq1czYcIEtm7d6rJOaensCRMm8Nprr1XZx4EDB9iyZQt9+vQpt6yi0tm33347AI8//jgLFizgnnvuAX4unW2z2Xj00UcZOnQob7/9NmfOnKF3795cddVVVunswMBA9u7dy/jx4zn7LnI4v4J4lZXODg8Pt9p07dqVxx57jPT0dIKCgli2bBlxca43YFZUOvvzzz+nRYsW5Obm8vLLL2vpbKUuQt27o/kc3+i9pbpLZ2dnZzNu3DjmzJlTrrAeaOlsLZ2t6oOvvnJcnDlq1CiP9lP3kkIt4k7p7KKiIsaNG8fNN99sPYLybFo6W0tnq7rPWxdN/CIuSa1rqqt0tjGGyZMnExUVxQMPPFBpf1o6W0tnK1VdNCl4QHWVzl63bh0LFy5k9erV1uWYy5YtK9dOS2dr6WylqouWzq4jtHR2eVo6W9UlS5cuBWD06NEXtL67pbN1pFBHlJbOVg5aOlvVNZGRkURGRnq8Hz3RXEd06tSp3Anj+kxLZ6u6JjY21iv96EhBKaWURZOCUkrVAkuXLrXOK3iSJgWllFIWTQpKKaUsmhSqiSdKZ+fn59O7d29iY2OJjo7mqaeeqnTb999/v8u9Cr80p06dYvjw4XTo0IHhw4dz+vTpCttNnz6drl270rVrV5fKqN988w09e/ake/fuXHHFFdbNepWV5N6+fTuTJk3y9G4pVedoUqgmr7/+OitXrjznHcTnKyAggNWrV5OcnMzWrVtZvnw5GzduLNcuPT2djRs3MmjQILe37e3nDMycOZNhw4axd+9ehg0bZt28VtaXX35JUlISW7du5fvvv2f27NlWzag777yTDz74gK1bt3LTTTfx17/+FXAtyT1t2jSrhlS3bt1ITU0lJSXFezupVB1Q5y5JfT7heX449UO1brPzpZ2Z3rvygnWeKp0tItYIoqioiKKiogpLQ3z66aeMHDnSmn7mmWdYunQpeXl59O/fnzfffBMRYfDgwXTv3p21a9cyfvx4JkyYwB133GF9cM6ZM4cBAwaQkJDAfffdR35+PkFBQfzrX/+66MtdP//8c6sG08SJExk8eDDPP/+8S5tdu3YxaNAgfH198fX1JSYmhuXLl/O73/0OEbESREZGBi1btrS2W1lJ7tGjR7No0SKX0ttK1Val9bw8TUcK1eCNN96gZcuWxMfHM23aNJ566il69OjBtm3beO655yq8Xr60dPb27dutZyZUpLi4mO7du9O0aVOGDx/uVunsqVOnkpiYyI4dO8jLy+OLL76wlhUWFrJp0yYefPBB7rvvPqZNm0ZiYiKffvopt912GwCdO3dmzZo1bNmyhWeeeYZHH320XJ9ZWVlW6Y2zX7t27SrX/tixY9Z+Nm/enGPHjpVrExsby/Lly8nNzeXkyZPEx8dz6JDjMd///Oc/ueaaa4iIiGDhwoU88sgjQOUluQHi4uKqvZKrUjUlOjqa6Ohoj/dT50YK5/pG7y3VWTrbZrOxdetWzpw5w29+8xt27NhB165dXdocOXKEss+ujo+PZ9asWeTm5nLq1Cmio6OtW+NvvPFGq92qVatcPsAzMzPJzs4mIyODiRMnsnfvXkSEoqKicnGFhoaWe0aEu0SkwhHPiBEjSExMpH///jRp0oR+/fpZpbNffvllli1bRp8+fXjhhRd44IEH+Oc//3nOfpo2bcrhw4cvKEalfmlKD/l6+omJdS4p1CbulM4u1bhxY4YMGcLy5cvLJYWypbPz8/O566672LRpE61bt2bGjBkuZbXLFrcrKSlh48aN5UpLT506lSFDhvDf//6XAwcOMHjw4HLxZGVlMXDgwApj/fDDD8s9Ha5Zs2YcOXKEFi1acOTIEZo2bVrhuo899hiPPfYYADfddBMdO3bkxIkTJCcnW6OkG2+80Tpcdq6S3KWHv5SqC0qfp3ChtY/cpYePPKC6SmefOHGCM2fOAJCXl8fKlSsrrHpatnR2aQIIDw8nOzubTz75pNI4R4wYwbx586zp0m/+GRkZ1vNg33nnnQrXLR0pVPQ6OyGAa+nsd999t8LzKMXFxdahn23btrFt2zZGjBjBJZdcQkZGhvWwn5UrV1rF6M5VknvPnj3lEqhS6tw0KXhAdZXOPnLkCEOGDCEmJoZevXoxfPhwrr322nLtfv3rX1sncRs3bsztt99O165dufrqq+nVq1elcb7yyits2rSJmJgYunTpwhtvvAHAn//8Z/7yl7/Qo0ePartK6ZFHHmHlypV06NCBVatWWecENm3aZJ3LKCoqYuDAgXTp0oUpU6bw/vvvWyed33rrLcaNG0dsbCwLFy7khRdeACovyQ1aOlupC6Gls+uIK664gi+++MLl+QP1WUFBAVdeeSVr1671yDFY/ZtT3qals9V5efHFF/Wa/DJSUlKYOXOmx0/KKVXX6P+YOqKiS1Xrsw4dOugDh1Sd4q1ng2hSUEqpWsBbz0vRw0dKKVUL5Ofnu1xe7imaFJRSqhZYuXIlK1eu9Hg/mhSUUkpZNCnUAZMmTaJdu3Z0796dnj17smHDhnLzY2Nj+eabbyrdRl0vvT1w4ECrNlPLli25/vrrrWXffvst3bt3Jzo6miuvvBJw1IgaNGiQ16vJKlXTNCnUES+88AJbt25l5syZ/OlPfyo3f86cOdxxxx0VrlsfSm+vWbPGuuO6X79+jB07FoAzZ85w1113sWTJEnbu3Ml//vMfAPz9/Rk2bJhLYlGqPqiTSaG04FpFr/nz51vt5s+ff8627jpw4ACdO3dm0qRJdOzYkZtvvplVq1YxYMAAOnToQEJCAgA5OTnceuut9O7dmx49evD5559b6w8cOJCePXvSs2dP1q9fDzi+wQ4ePJgbbriBzp07c/PNN1PVzYaDBg2ySl6U1a9fv0rvnK6o9HavXr3o2rUrU6ZMsfocPHgw999/P3FxccydO5cTJ04wbtw4evXqRa9evVi3bh0ACQkJ9OvXjx49etC/f392797t9u+yMp9//jkTJ04EHKW3Fy9eXK5N2dLbISEhVuntsjIzM1m9erU1Uvjwww8ZO3YskZGRAC41ma6//vpqfT6GUrVBnUwKNWHfvn08+OCD/PDDD/zwww98+OGHrF27ltmzZ/Pcc88B8Le//Y2hQ4eSkJBAfHw8Dz/8MDk5OTRt2pSVK1eSlJTExx9/zL333mttd8uWLcyZM4ddu3axf/9+64O3MkuXLqVbt27l5i9fvtzlkElZ9aH0dqnFixczbNgwqxbVnj17OH36NIMHD+byyy/nvffes9p27dqVxMTECn9nSnlbly5dKqwrVt08ep+CiIwE5gI24J/GmJlnLQ8A3gMuB9KBG40xBy62X3dLd0yZMoUpU6ZcbHcAtGvXzvowjo6OZtiwYYgI3bp1sx4RuWLFCpYsWcLs2bMBxyVmKSkptGzZkqlTp7J161ZsNptV+A2gd+/eREREANC9e3cOHDjAFVdcUa7/hx9+mL/+9a80adKEBQsWuMx/9NFHSU1Ntc41nK0+lN4u9dFHH1nJCxyHwTZv3sw333xDXl4e/fr1o2/fvnTs2BGbzYa/vz9ZWVmEhoZeUKxKVZdf/epXXunHY0lBRGzAa8BwIBVIFJElxpiyXwMnA6eNMe1F5PfA88CN5bf2yxcQEGC99/HxsaZ9fHys4+/GGD799NNyN6HMmDGDZs2akZycTElJiUsp67LbtdlslR7Lf+GFF7jhhhsqnT9v3jxuvfVWNm/eXK5NXS+9XerkyZMkJCTw3//+15oXERFBWFgYISEhhISEMGjQIJKTk631CgoKyu2fUjUhOzsbqPh57tXJk4ePegP7jDH7jTGFwCLg7HrJY4DSEqKfAMPkfA7m1zJXX3018+bNs0YyW7ZsARylqlu0aIGPjw8LFy6kuLi42vueOnUqJSUlfP311+WW1fXS26U++eQTrr32WpcP+TFjxrB27Vrsdju5ubl8//33VqG79PR0wsPD8fPzq/R3oJS3xMfHEx8f7/F+PJkUWgFlD+imOudV2MYYYwcygDAPxlSjnnjiCYqKioiJiSE6OponnngCgLvuuot3332X2NhYfvjhB5dv49VFRHj88ceZNWtWuWV1vfR2qUWLFjF+/HiX7UZFRTFy5EhiYmLo3bs3t912m/UMBi29reojj5XOFpEbgJHGmNuc07cAfYwxU8u02eFsk+qc/tHZ5uRZ25oCTAGIjIy8/ODBgy59aRnji6elt8sbO3YsM2fOrLAQmf7Nqdrml1A6Ow1oXWY6wjmvwjYi4gs0wnHC2YUxZr4xJs4YE1f2hKiqPlp621VhYSHXX3+91ypTKvVL4cmrjxKBDiLSDseH/++Bm85qswSYCGwAbgBWm9r21J86Qktvu/L392fChAk1HYZSXuexpGCMsYvIVOBrHJekvm2M2SkizwCbjDFLgAXAQhHZB5zCkTgutL/zuuFMqQul31tUXebR+xSMMcuAZWfNe7LM+3zgtxfbT2BgIOnp6YSFhWliUB5ljCE9PV0vU1V1Vp14yE5ERASpqamcOHGipkNR9UBgYKB1Q6FSdU2dSAp+fn60a9eupsNQSqlaT2sfKaWUsmhSUEopZdGkoJRSyuKxO5o9RUROAAerbFixcOBkla3qFt3n+kH3uX64mH1uY4yp8u7fWpcULoaIbHLnNu+6RPe5ftB9rh+8sc96+EgppZRFk4JSSilLfUsK86tuUufoPtcPus/1g8f3uV6dU1BKKXVu9W2koJRS6hzqZFIQkZEisltE9onIIxUsDxCRj53LvxeRtt6Psnq5sc8PiMguEdkmIt+ISJuaiLM6VbXPZdqNExEjIrX+ShV39llEfuf8t94pIh96O8bq5sbfdqSIxIvIFuff9zU1EWd1EZG3ReS48yFkFS0XEXnF+fvYJiI9qzUAY0ydeuEo0/0jcBngDyQDXc5qcxfwhvP974GPazpuL+zzECDY+f7O+rDPznahwHfARiCupuP2wr9zB2ALcIlzumlNx+2FfZ4P3Ol83wU4UNNxX+Q+DwJ6AjsqWX4N8BUgQF/g++rsvy6OFHoD+4wx+40xhcAi4OynvI8B3nW+/wQYJrW75naV+2yMiTfG5DonN+J4El5t5s6/M8CzwPNAvjeD8xB39vl24DVjzGkAY8xxL8dY3dzZZwM0dL5vBBz2YnzVzhjzHY7ny1RmDPCecdgINBaRFtXVf11MCq2AQ2WmU53zKmxjjLEDGUCYV6LzDHf2uazJOL5p1GZV7rNzWN3aGPOlNwPzIHf+nTsCHUVknYhsFJGRXovOM9zZ5xnAH0QkFcfzW+7xTmg15nz/v5+XOlE6W7lPRP4AxAFX1nQsniQiPsBLwKQaDsXbfHEcQhqMYzT4nYh0M8acqdGoPGs88I4x5kUR6YfjaY5djTElNR1YbVQXRwppQOsy0xHOeRW2ERFfHEPOdK9E5xnu7DMichXwGHCdMabAS7F5SlX7HAp0Bb4VkQM4jr0uqeUnm935d04FlhhjiowxPwF7cCSJ2sqdfZ4M/BvAGLMBCMRRI6iucuv/+4Wqi0khEeggIu1ExB/HieQlZ7VZAkx0vr8BWG2cZ3BqqSr3WUR6AG/iSAi1/TgzVLHPxpgMY0y4MaatMaYtjvMo1xljNtVMuNXCnb/txThGCYhIOI7DSfu9GWQ1c2efU4BhACIShSMp1OXHMC4BJjivQuoLZBhjjlTXxuvc4SNjjF1EpgJf47hy4W1jzE4ReQbYZIxZAizAMcTch+OEzu9rLuKL5+Y+vwA0AP7jPKeeYoy5rsaCvkhu7nOd4uY+fw2MEJFdQDHwsDGm1o6C3dznB4G3RGQajpPOk2rzlzwR+QhHYg93nid5CvADMMa8geO8yTXAPiAX+GO19l+Lf3dKKaWqWV08fKSUUuoCaVJQSill0aSglFLKoklBKaWURZOCUkopiyYFpbxIRNqWVr8UkcEi8kVNx6RUWZoUlHKD80Yh/f+i6jz9I1eqEs5v9btF5D1gB3CLiGwQkSQR+Y+INHC26yUi60UkWUQSRCTUue4aZ9skEelfs3ujlHvq3B3NSlWzDjhKouwDPgOuMsbkiMh04AERmQl8DNxojEkUkYZAHnAcGG6MyReRDsBHOAoRKvWLpklBqXM7aIzZKCLX4niAyzpnmRB/YAPQCThijEkEMMZkAohICPCqiHTHUW6iY00Er9T50qSg1LnlOH8KsNIYM77sQhHpVsl604BjQCyOw7R14SE/qh7QcwpKuWcjMEBE2oNjJCAiHYHdQAsR6eWcH1qmHPsRZ03/W3AUc1PqF0+TglJuMMacwPHAno9EZBuOQ0ednY+IvBGYJyLJwEocpZtfByY653Xm5xGHUr9oWiVVKaWURUcKSimlLJoUlFJKWTQpKKWUsmhSUEopZdGkoJRSyqJJQSmllEWTglJKKYsmBaWUUpb/B6IHmMhFLjbbAAAAAElFTkSuQmCC\n", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# Plot the PR curves for the negative class\n", "for j in range(k):\n", " plt.plot(recall_minus[j], precision_minus[j], label='fold %d (area = %.3f)' % (j+1, auprc_minus[j]))\n", "\n", "plt.plot(recall_minus['avg'], precision_minus['avg'], 'k--', label='mean PR (area = %.3f)' % auprc_minus['avg'], lw=2)\n", "plt.plot([0,1,1], [1,1,0], linestyle='--', color=(0.6, 0.6, 0.6)) # PR curve for a perfect classifier\n", "\n", "plt.xlim([-0.05, 1.05])\n", "plt.ylim([-0.05, 1.05])\n", "plt.xlabel('recall')\n", "plt.ylabel('precision')\n", "plt.legend(loc='lower left')\n", "\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Plot confusion matrix" ] }, { "cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [], "source": [ "# Compute a 2x2 confusion matrix\n", "confusion_matrix = np.zeros([2, 2])\n", "\n", "for j, (train_idx, val_idx) in enumerate(cv):\n", " rf.fit(X_train.iloc[train_idx], y_train.iloc[train_idx])\n", " y = rf.predict(X_train.iloc[val_idx])\n", " confusion_matrix += metrics.confusion_matrix(y_train.iloc[val_idx], y)\n", "\n", "# Average over all folds\n", "confusion_matrix /= k\n", "confusion_matrix = np.around(confusion_matrix, decimals=3)" ] }, { "cell_type": "code", "execution_count": 17, "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQ4AAAEMCAYAAAA1eViuAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAFM1JREFUeJzt3Xl4VdW9xvHvLwkgJISIiUwBUVFJRAlJBKcqKgWLCAIiWIdSI5NWrbb22tZr672OVVux4oBDKfaKQxFHFCtOTBKQ2QEoZSqgBCGACYEkrPtHNmnCEM4KnCHk/TzPeThZe+29fpuT583ea5+zjznnEBHxERftAkSk7lFwiIg3BYeIeFNwiIg3BYeIeFNwiIg3BYeIeFNwRJCZdTezcdGuQ+RQKThExJuCQ0S8md5yHn5mNhtoBCQBzYE1waL/cs5NiVphIrWk4IggM+sODHXODY1yKVILZnYjMCz4sbdzbn0064mmhGgXIFJXOOfGAGOiXUcs0ByHiHjTqYqIeNMRh4h4U3CIiDcFh4h4U3CIiDcFh4h4U3BEmJkNj3YNUnt6/SooOCJPv3h1m14/FBwiUgsx+Qaw5sekurbtjot2GWHx3aYCjklNi3YZYZUQb9EuIWw2FRSQmnbkvn5ff/VVcVFRUeLB+sXkZ1XatjuO9z6aGe0ypJbSkhtFuwSppfbHtS0MpZ9OVUTEm4JDRLwpOETEm4JDRLwpOETEm4JDRLwpOETEm4JDRLwpOETEm4JDRLwpOETEm4JDRLwpOETEm4JDRLwpOETEm4JDRLwpOETEm4JDRLwpOETEm4JDRLwpOETEm4JDRLwpOETEm4JDRLwpOETEm4JDRLwpOETEm4JDRLwpOETEm4JDRLwpOETEm4JDRLwpOETEm4JDRLwpOETEm4JDRLwpOETEm4JDRLwpOETEm4JDRLwpOETEm4JDRLwpOETEm4JDRLwpOETEm4JDRLwpOETEm4JDRLwpOETEm4IjTMY+8Rjdz+rCBWdlMyrvGkpKSqot37lzJyOuu5qzszO5pMcPWLtmVXQKrafWrl1Lj4su4PROmXQ+7VQee2z0fvt98vHH5GRn0fm0U7nwgvMr2wsLCxk86HI6ZXbktFMzmDVrFgA/HjKYnOwscrKz6HBCe3KysyKyP5GWEO0CjkQb1q/juafH8PFnC2jcuDEjfnoVb7z2CoN/fG1lnwkvjCOlWQoz533J6xNf4Z7f38nTz/8tilXXLwkJCfzhoUfIzs5m+/btdDsjhx49fkhmZmZln8LCQm762Q28Pfk92rVrx8aNGyuX3frzW+jZ62JefvXv7Nq1i+LiYgBefOnlyj63//IXNGvWLHI7FUFhP+Iws+5mNi7c48SasrIySkp2UFZWxo7iYlq0bFVt+ZR332LQlVcD0KffAKZ/8hHOuWiUWi+1atWK7OxsAJo2bUrHjhmsX7euWp8JE17ksv4DaNeuHQDHHnssAFu3bmX6tE+5Li8PgIYNG5KSklJtXeccf3/1FQYPuTLcuxIVOlUJg1at2zDqpls547STyOrYnqbJyXS/8IfV+nyzfj2t26QDFX/9kpOT2bz5u2iUW++tWrWKBQvm07Vbt2rty5ctY8uWLVx0YXe6npHDC+PHA7By5UpS09LIu+6n5OZ0Yfiw6ykqKqq27vRp0zi2RQtOOumkSO1GRMVMcJjZcDOba2Zzv9tUEO1yDklh4RamTH6L2Qu+Zv5XKykuLmbiyy9GuyzZj++//54rBg3kkT8+SnJycrVlZWVlzJv3OW++9Q6T353Cfff+L8uWLaOsrIz58+YxYuQo5n4+n8TERP7w4APV1n3ppQkMOUKPNiCMwWFms81sAfAs0NfMFgSPXvvr75wb65zLdc7lHpOaFq6yImLaxx/S9rj2HJOaRoMGDeh9aT/m5n9WrU/L1q1Zv+7fQMUv6LZt22je/JholFtvlZaWcsXlA7nyx1fRf8CAfZanp6fTs2cvEhMTSU1N5dwfnMeihQtJT08nPT2dbsERysCBlzN/3rzK9crKynh90msMumJwxPYl0sIWHM65bs65LOB64E3nXFbwmBKuMWNFm/S2zJubT3FxMc45pn/yER1O6VitT8+L+/DqhIrJ0LffeI1zz+uOmUWj3HrJOcew6/PomJHBrbfett8+l/btx4wZ0ykrK6O4uJg5+bPpmJFBy5YtSW/blqVLlwLw4YdTyagyqTr1gw84pWNH0tPTI7Iv0aCrKmGQnduVS/r2p1f3M0mIT6DT6Z25+id5/OG+u+mclUOv3n248pqh3DzyOs7OziTl6OY8+dz4aJddr8yYMYP/+9sLdDrttMpLpvfccx9r1qwBYMTIkWRkZNCr18VkZ51OXFwcP827nk6dOgHw6Og/c+01V7Fr1y5OOP4Enn3+L5Xbfvnllxg8+Mg9TQGwcM/km1l3YKhzbmio63TukuPe+2hm2GqS8EpLbhTtEqSW2h/Xdv2/165tc7B+YT/icM59DHwc7nFEJHJi5qqKiNQdCg4R8abgEBFvCg4R8abgEBFvCg4R8abgEBFvCg4R8abgEBFvCg4R8abgEBFvCg4R8abgEBFvCg4R8abgEBFvCg4R8abgEBFvCg4R8abgEBFvCg4R8abgEBFvCg4R8abgEBFvCg4R8abgEBFvCg4R8abgEBFvB/zuWDNLrmlF59y2w1+OiNQFNX3p9BeAA6xK256fHdAujHWJSAw7YHA459pGshARqTtCmuMwsyFm9pvgebqZ5YS3LBGJZQcNDjN7HLgAuCZoKgaeCmdRIhLbaprj2ONs51y2mc0HcM5tNrOGYa5LRGJYKKcqpWYWR8WEKGZ2DLA7rFWJSEwLJTjGABOBNDO7G5gOPBjWqkQkph30VMU5N97MPgd6BE2DnHNLwluWiMSyUOY4AOKBUipOV/RuU5F6LpSrKr8FJgCtgXTgRTP7dbgLE5HYFcoRx7VAF+dcMYCZ3QvMB+4PZ2EiErtCOe3YQPWASQjaRKSequlDbn+iYk5jM/CFmU0Jfu4JzIlMeSISi2o6Vdlz5eQL4J0q7Z+FrxwRqQtq+pDbc5EsRETqjoNOjprZicC9QCZw1J5259zJYaxLRGJYKJOj44C/UHEfjh8BrwAvh7EmEYlxoQRHE+fcFADn3Arn3J1UBIiI1FOhvI9jZ/AhtxVmNhJYBzQNb1kiEstCCY5bgUTgZirmOpoB14WzKBGJbaF8yG128HQ7/7mZj4jUYzW9AWwSwT049sc5NyAsFYlIzKvpiOPxiFWxl4Q4o3mSbjJWVz34zORolyC1tO7bwpD61fQGsKmHrRoROaLo3hoi4k3BISLeQg4OM2sUzkJEpO4I5Q5gXc1sMbA8+Lmzmf057JWJSMwK5YjjMaAP8B2Ac24hFV/QJCL1VCjBEeecW71XW3k4ihGRuiGUt5yvNbOugDOzeOAmYFl4yxKRWBbKEcco4DagHfAtcGbQJiL1VCifVdkIDIlALSJSR4RyB7Bn2M9nVpxzw8NSkYjEvFDmOD6o8vwooD+wNjzliEhdEMqpSrXbBJrZC1R88bSI1FO1ecv58UCLw12IiNQdocxxbOE/cxxxVHxB0x3hLEpEYluNwWFmBnSm4j6jALudcwe8uY+I1A81nqoEITHZOVcePBQaIhLSHMcCM+sS9kpEpM6o6Z6jCc65MqALMMfMVgBFVHwxk3POZUeoRhGJMTXNceQD2UDfCNUiInVETcFhUPHtbRGqRUTqiJqCI83MbjvQQufcH8NQj4jUATUFRzyQRHDkISKyR03BscE59z8Rq0RE6oyaLsfqSENE9qum4LgoYlWISJ1ywOBwzm2OZCEiUnfoC5lExJuCQ0S8KThExJuCQ0S8KThExJuCQ0S8KThExJuCQ0S8KThExJuCQ0S8KThExJuCQ0S8KThExJuCQ0S8KThExJuCQ0S8KThExJuCQ0S8KThExJuCQ0S8KThExJuCQ0S8KThExJuCQ0S81fTdsXIQJSUl9LjgfHbu3ElZeRn9Bwzkrt/dXa3PM08/xVNPPkF8fDyJSUk88eTTZGRmMic/nxtHjQDAOcedd/2Ofpf1D2mbUjuFmwuYOO4Rvt9WiJmRe+7FnH1RPzasXcEbL46hrHQXcXHx9L3yBtKPP2U/629k0guPsW1LAWBc+7O7OTq1Bc45PnhjPEvmTScuLo6u513CWRf2Zdr7E1mY/xEAu3fvpmDDWn798Is0SWwa4T0//Mw5F+0a9pGTk+tmzp4T7TIOyjlHUVERSUlJlJaWcuH5P+DhPz5KtzPPrOyzbds2kpOTAXj7rTd5+qkneeuddykuLqZhw4YkJCSwYcMGuuZksXLNOuLj4w+6zVj38HPvRruE/dq+dTPbt26mdbsO7Cwp5on7buGqkf/NO6+O5ZyLLuPkTrksXTyHae9P5PpfPLDP+s8+cgfdfzSYDpld2FmyA4szGjY8is9n/oOVSxcx4Ce3EhcXx/fbCklKTqm27teLZjNj6uvk3Xp/pHa3Vu68afB6t2t7m4P10xHHITAzkpKSACgtLaW0tBSz6t/VvSc0AIqKiiqXN2nSpLK9pKSksj2UbUrtNG3WnKbNmgPQ6KgmpLVsy7bC7zAzdpYUA1BSUkRySvN91t24fg27d5fTIbNLsH7jymX5n0zmirzbiYurOPPfOzQAFs35hNNzzz/s+xQtCo5DVF5ezlldc1mx4p+MHHUDXbt126fPU0+MYfToP7Fr1y6mvD+1sj1/9mxGDM9jzerVPD9uPAkJCSFvUw7Nlk3fsmHtv0g//hR6DxrGXx+7i3cnPofb7Rj+q4f36b9p4zoaN0nkxafuYct333Jixyx69h9KXFw8mzdtYPHcT/lywSwSmzbjkitGkNriP3+0d+0qYfkXn9NnyKhI7mJYxczkqJkNN7O5Zja3YFNBtMsJWXx8PPmfz2fFqrXMmTOHL5Ys2afPyBtu5Kul/+Te+x7g/vvurWzv2q0b8xcuYcasfB568AFKSkpC3qbU3s6SHUwYey+9rxjGUY2bkP/pZHoPGsav7v8rvQcNY9ILj+6zzu7yclYt/4KLB+Yx8o5H2bzpG+bN+gCA8rJSEho05IbfjCb33F5MemF0tXWXLsqn3YmZR8Tcxh4xExzOubHOuVznXG5aalq0y/GWkpLC+d278/777x2wzxWDh/DWm6/v094xI4PEpKR9AiKUbYqf8vIyJoy9j85dL+DULucAMH/WVDK7nA1Ap5xzWbdq2T7rNTs6lVZtT6B5Wivi4+PJ6HwWG9asACA5JbVy/cyss/nm3yurrbtozqecfsaRc5oCEQwOM7vRzBYEj9aRGjecCgoKKCwsBGDHjh1M/eADTjmlY7U+/1y+vPL5u5PfoUOHkwBYuXIlZWVlAKxevZplS7/muPbtQ9qm1I5zjknjR5PWsi3n9Ohf2Z6c0pyVyxYD8K+lCznm2H1/Pdu0P4mS4iKKtm+t7JfWqh0AGVlnsnLpIgBWLltc7TSlZEcRq5YvJqNz3ZncDkXE5jicc2OAMZEaLxK+2bCB668bSnl5ObvdbgZePojel/Th7t/fRU5OLn0u7cuTTzzOhx9OpUFCA1KOPppnnx8HwMwZ03n4oQdpkNCAuLg4Rv95DKmpqSxetGi/25RDt3rFlyyY/SEt2rTn8Xt+BsAP+/2EflffzORXnmZ3+W4SGjSg31U3AbBu9XLyP51M/2tuIS4unosH5vH8o78B52jdrgO55/YC4Lxeg3j1+YeYOfV1GjZqzGXX3Fw55pfzZ9IhM5uGjY6K/A6HkS7HymEXq5dj5eBCvRwbM3McIlJ3KDhExJuCQ0S8KThExJuCQ0S8KThExJuCQ0S8KThExJuCQ0S8KThExJuCQ0S8KThExJuCQ0S8KThExJuCQ0S8KThExJuCQ0S8KThExJuCQ0S8KThExJuCQ0S8KThExJuCQ0S8KThExJuCQ0S8KThExJuCQ0S8KThExJuCQ0S8KThExJuCQ0S8KThExJuCQ0S8KThExJuCQ0S8KThExJuCQ0S8KThExJuCQ0S8KThExJuCQ0S8KThExJuCQ0S8KThExJuCQ0S8KThExJuCQ0S8KThExJuCQ0S8KThExJuCQ0S8mXMu2jXsw8wKgNXRriNMUoFN0S5Cau1If/2Oc86lHaxTTAbHkczM5jrncqNdh9SOXr8KOlUREW8KDhHxpuCIvLGHc2NmVm5mC8xsiZm9amZNDmFb3c3s7eB5XzO7o4a+KWZ2Qy3G+L2Z/TLU9r36jDOzyz3Gam9mS3xrPIjD+vrVVQqOCHPOHe5fvB3OuSznXCdgFzCy6kKr4P06O+fedM49UEOXFMA7OOq6MLx+dZKC48gyDegQ/KVdambjgSVAWzPraWazzGxecGSSBGBmF5vZ12Y2DxiwZ0NmNtTMHg+etzCzSWa2MHicDTwAnBgc7TwU9LvdzOaY2SIzu7vKtn5rZsvMbDpwysF2wsyGBdtZaGYT9zqK6mFmc4Pt9Qn6x5vZQ1XGHnGo/5FSMwXHEcLMEoAfAYuDppOAJ5xzpwJFwJ1AD+dcNjAXuM3MjgKeAS4FcoCWB9j8Y8AnzrnOQDbwBXAHsCI42rndzHoGY3YFsoAcMzvPzHKAIUFbb+CMEHbnNefcGcF4XwF5VZa1D8a4BHgq2Ic8YKtz7oxg+8PM7PgQxpFaSoh2AXLIGpvZguD5NOA5oDWw2jn3WdB+JpAJzDAzgIbALKAjsNI5txzAzP4GDN/PGBcC1wI458qBrWZ29F59egaP+cHPSVQESVNgknOuOBjjzRD2qZOZ3UPF6VASMKXKslecc7uB5Wb2r2AfegKnV5n/aBaMvSyEsaQWFBx13w7nXFbVhiAciqo2Af9wzl25V79q6x0iA+53zj291xg/r8W2xgGXOecWmtlQoHuVZXu/8cgFY9/knKsaMJhZ+1qMLSHQqUr98Blwjpl1ADCzRDM7GfgaaG9mJwb9rjzA+lOBUcG68WbWDNhOxdHEHlOA66rMnbQxs2OBT4HLzKyxmTWl4rToYJoCG8ysAXDVXssGmVlcUPMJwNJg7FFBf8zsZDNLDGEcqSUdcdQDzrmC4C/3BDNrFDTf6ZxbZmbDgXfMrJiKU52m+9nELcBYM8sDyoFRzrlZZjYjuNz5bjDPkQHMCo54vgeuds7NM7OXgYXARmBOCCX/NzAbKAj+rVrTGiAfSAZGOudKzOxZKuY+5lnF4AXAZaH970ht6C3nIuJNpyoi4k3BISLeFBwi4k3BISLeFBwi4k3BISLeFBwi4k3BISLe/h96DsKcsMHgGAAAAABJRU5ErkJggg==\n", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# Plot the confusion matrix\n", "plt.matshow(confusion_matrix, interpolation='nearest', cmap=plt.cm.Blues, alpha=0.5)\n", "plt.xticks(np.arange(2), ('+', '-'))\n", "plt.yticks(np.arange(2), ('+', '-'))\n", "\n", "for i in range(2):\n", " for j in range(2):\n", " plt.text(j, i, confusion_matrix[i, j], ha='center', va='center')\n", "\n", "plt.xlabel('Predicted label')\n", "plt.ylabel('True label')\n", "\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# STEP 3: Evaluation" ] }, { "cell_type": "code", "execution_count": 18, "metadata": {}, "outputs": [], "source": [ "# Refit the classifier using the full training set\n", "rf = rf.fit(X_train, y_train)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Compute classifier accuracy on the test set" ] }, { "cell_type": "code", "execution_count": 19, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Test accuracy: 0.848\n" ] } ], "source": [ "# Compute classifier's accuracy on the test set\n", "test_accuracy = rf.score(X_test, y_test)\n", "print('Test accuracy: %.3f' % test_accuracy)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Generate plots for the test set" ] }, { "cell_type": "code", "execution_count": 20, "metadata": {}, "outputs": [], "source": [ "# Predict labels for the test set\n", "y = rf.predict(X_test) # Solid prediction\n", "y_prob = rf.predict_proba(X_test) # Soft prediction\n", "\n", "# Compute the ROC and PR curves for the test set\n", "fpr, tpr, _ = metrics.roc_curve(y_test, y_prob[:, 0], pos_label='+')\n", "precision_plus, recall_plus, _ = metrics.precision_recall_curve(y_test, y_prob[:, 0], pos_label='+')\n", "precision_minus, recall_minus, _ = metrics.precision_recall_curve(y_test, y_prob[:, 1], pos_label='-')\n", "\n", "# Compute the AUROC and AUPRCs for the test set\n", "auroc = metrics.auc(fpr, tpr)\n", "auprc_plus = metrics.auc(recall_plus, precision_plus)\n", "auprc_minus = metrics.auc(recall_minus, precision_minus)\n", "\n", "# Compute the confusion matrix for the test set\n", "cm = metrics.confusion_matrix(y_test, y)" ] }, { "cell_type": "code", "execution_count": 21, "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl0VNeV+PvvlgQIoREJgUCAGMQksIQQYgZjzGiD7W5shpdfJ+n0z/3ccdKJ3Wk73e7ETidvxXHiOPn93HE7sX9JZ9mxsd/rGDwkJrGNhEBmHgUCCQESoxBonlXn/VGlipAECKFbt6ru/qzFWnXvOXXvvkKqXecO+4gxBqWUUgogxO4AlFJK+Q9NCkoppbw0KSillPLSpKCUUspLk4JSSikvTQpKKaW8NCkopZTy0qSggo6InBaRBhGpFZGLIvJrEYns1GeuiHwiIjUiUiUiW0RkSqc+0SLykoic9Wyr2LOc4NsjUsp3NCmoYLXaGBMJZADTgW+3N4jIHOBj4D1gODAGOAjkichYT5/+wJ+BNGAFEA3MASqAbKuCFpEwq7atVE9oUlBBzRhzEfgj7uTQ7kfAfxljfmaMqTHGXDXGPAPkA896+vwNMAp4yBhTYIxxGWMuG2P+3RjzYXf7EpE0EdkqIldF5JKI/Itn/a9F5Psd+t0tImUdlk+LyFMicgio87x+t9O2fyYiP/e8jhGR10TkgoicE5Hvi0joHf6olAI0KaggJyLJwEqgyLMcAcwF3umm+yZgqef1vcAfjDG1PdxPFPAn4A+4Rx/jcY80emoDcB8QC7wFrPJsE88H/iPAm56+vwZaPfuYDiwD/u429qXUDWlSUMHq9yJSA5QCl4HvetYPxv17f6Gb91wA2q8XxN+gz43cD1w0xvzEGNPoGYF8fhvv/7kxptQY02CMOQPsAx7ytN0D1Btj8kVkKLAK+IYxps4Ycxn4KbD+Nval1A1pUlDB6kFjTBRwNzCJv3zYXwNcQFI370kCrnheV9ygz42MBIp7FalbaaflN3GPHgA28pdRwmigH3BBRCpFpBL4TyDxDvatlJcmBRXUjDHbcJ9u+bFnuQ7YCTzcTfdH+Mspnz8By0VkUA93VQqMvUFbHRDRYXlYd6F2Wn4HuNtz+ush/pIUSoEmIMEYE+v5F22MSethnErdlCYF5QQvAUtFJN2z/DTwRRH5uohEiUic50LwHOA5T5/f4v4A/n9FZJKIhIhIvIj8i4is6mYf7wNJIvINERng2e4sT9sB3NcIBovIMOAbtwrYGFMOfAb8H6DEGHPMs/4C7junfuK5ZTZERMaJyKJe/FyU6kKTggp6ng/Y/wK+41neDiwH/gr3dYMzuC/YzjfGnPT0acJ9sfk4sBWoBnbhPg3V5VqBMaYG90Xq1cBF4CSw2NP8W9y3vJ7G/YH+dg9Df9MTw5ud1v8N0B8owH067F1u71SXUjckOsmOUkqpdjpSUEop5aVJQSmllJcmBaWUUl6aFJRSSnkFXPGthIQEk5KSYncYSikVUPbu3XvFGDPkVv0CLimkpKSwZ88eu8NQSqmAIiJnetJPTx8ppZTy0qSglFLKS5OCUkopL00KSimlvDQpKKWU8rIsKYjI6yJyWUSO3KBdROTnIlIkIodEJNOqWJRSSvWMlSOFX+Oe8PxGVgKpnn+PAr+wMBallFI9YNlzCsaYHBFJuUmXB3BPnm6AfBGJFZEkT714S1y5coWdO3d2WT9z5kyGDRvGxYsX2b17d5f2OXPmkJCQQFlZGfv37+/SvmDBAmJjYzlz5gyHDh3q0r548WIiIyMpLi6moKCgS/vSpUsJDw+nsLCQEydOdGlfuXIlYWFhfJizmzMlp7q0NyS556TvV1VKWH3FdW0mJITGoXe52yvPENZw7fr20H40JrrnZ+l/7RShjdXXtbvCBtA0ZLK7vaKI0Obrpyx29YugKWECAAOunCCkpf669rb+kTTHj3e3lx8jpLXp+vbwaJrj3HPThF8+irS1XNfeOjCOltjR7vZLhxCX6/r2iHhaYkYCMPDCATprGTSE1ugR4Gpj4KXDXdsjh9EaNQzaWhh4+WjX9ujhtA5KRFobCS8/3qW9OSaZtogEpKWe8Ctd/++aY0fTNjCOkKZaBlwt6tLeFDcGV3gMIY1VDLhW0rV98HhcAyIJbbhG/8qut5k3JkzA9IsgtP4K/avKurYPmYQJCyes7jL9qs93aW9ITIPQfoTVXKRf7cWu7UOnQUgoYdXn6FdX3rVdf/cAX/zuHQNjmLtkJekjY7v07Ut2Prw2guunICzzrOuSFETkUdyjCUaNGuWT4PzRn49fIryhocv6N4+7P2yyo2sYH9F4XVurETYVuNvnxVYzOvz6P4wGVxP/fdTdviiuihEDmq9rr2lrZssRd/u9gytJ7H/9H87Vlhb+cNjdviK+isH9Wq9rv9zcyp8OudtXD6khKrTtuvZzTW1sO+BufyixloEh1//hnbnoIq/Svc9HhtYTJteXei86f4Vd1e5j2jis68/m+Lly9tU0ECYuHhnatf1w2SUO19YyMKSNhxK7tu8vvcixumqiQltZPaRr+64zFyhqqGRwvxZWxHdt33H6HKcbK0js38y9g7u2bysp41xTOSMGNLEormv7n06d5XJzf1LCG5kb27X9D8VnuNrSj/EDG8iO6dq+5eRpatrCmDyonulRXdv/+8QpGlyhTIusY1pk1/ZNhcW0mhAyo2qZNEh/9zry1e/e5Ih67htSjUE4VFZpeVKwdD4Fz0jhfWPM1G7a3gd+6JnwBBH5M/CUMeamjytnZWWZ3jzRXFbm/haVnJx82+/1F4+8spPQEOF3j862OxSllMWampr4/PPPOX78ONHR0SxcuJDhw4f3ensistcYk3WrfnaOFM7hnuy8XbJnnSXaT/sEclJQSjmDy+Xivffeo6qqivT0dGbMmEFYmG8+ru1MCpuBx0XkLWAWUGXl9QSllPJ3jY2NDBgwgJCQEGbOnElkZCRDhtyyhl2fsiwpiMjvgLuBBBEpA74L9AMwxrwCfAisAoqAeuDLVsWilFL+zBhDUVERO3bsIDs7m8mTJzNmzBhbYrHy7qMNt2g3wFet2r9SSgWC2tpacnNzKS0tJTExkWHDhtkaT8CVzlZKqWBRVFREbm4uxhjmzJlDWloaISH2FppwTFJYsGCB3SEopdR1BgwYQGJiIgsWLCA6OtrucAAHJYXYWGvv7VVKqVtxuVwcPnyYtrY2MjMzGTlyJMnJyYiI3aF5OSYpnDnjfhp09OjRNkeilHKiiooKtm3bxpUrVxg7dizGGETErxICOCgptJef0KSglPKltrY29u3bx4EDBwgPD+fee+9lzJgxfpcM2jkmKSillB2qqqo4ePAg48ePZ86cOYSHh9sd0k1pUlBKqT7W0tLC6dOnSU1NZfDgwTzyyCN+cyH5VjQpKKVUHyorKyM3N5eamhoSEhKIi4sLmIQAmhSUUqpPNDU1kZ+fT2FhITExMaxevZq4uDi7w7ptjkkKixcvtjsEpVSQ6ljALiMjg8zMTJ8VsOtrgRl1L0RGRtodglIqyHQsYJednU1kZCQJCQl2h3VH7H2e2oeKi4spLi62OwylVBAwxnDixAnefvttjh93z8iXkpIS8AkBHDRSaJ8Gc9y4cTZHopQKZDU1NeTm5lJWVsbQoUNJSkqyO6Q+5ZikoJRSd+rkyZNs374dYwxz584lLS3Nbx9C6y1NCkop1UPh4eEMHTqUBQsWEBUVZXc4ltCkoJRSN+ByuTh06BAul8tvC9j1NU0KSinVjStXrrBt2zYqKioYN26c3xaw62uOSQpLly61OwSlVABobW1l3759HDx4kPDwcJYuXWrb1Jh2cExS8MciVP/2+yMcKK3scf/i8lrSk3VeCKWsVF1dzaFDh0hNTWXOnDkMGDDA7pB8yjFJobCwEICJEyfaHMlfvHfgHFHh/Zg4rGcXrIZEDWB1enDd/qaUP2hpaaGkpIQJEyYEXAG7vuaYpHDixAnAv5ICwNIpQ3l2TZrdYSjlWKWlpeTm5lJbW8uQIUMCroBdX3NMUlBKqY4aGxvZuXMnJ0+eJDY2ljVr1gRkAbu+pklBKeU47QXsqqurmT59OtOnTw/YAnZ9TX8KSinHaGhoIDw8nJCQEGbNmhUUBez6mmMK4imlnMsYQ2FhIW+//TbHjh0DgqeAXV9zzEhh5cqVdoeglLJBTU0NOTk5nDt3jmHDhjF8+HC7Q/JrjkkKer5QKec5ceIE27dvR0SYP38+kydPDvonku+UYz4pjx49CkBamt7+qZRTREREkJSUxIIFC3SirR5yTFI4deoUoElBqWDmcrk4cOAAxhhmzJhBcnIyycnJdocVUByTFJRSwa1jAbvx48d7C9ip26NJQSkV0FpbW9m7dy+HDh0iPDycZcuWkZKSYndYAcvSW1JFZIWIFIpIkYg83U37KBH5VET2i8ghEVllZTxKqeBTXV3N4cOHmTBhAo888ogmhDtk2UhBREKBl4GlQBmwW0Q2G2MKOnR7BthkjPmFiEwBPgRSrIpJKRUcmpubKSkpYeLEiQwePJh169YF7Uxovmbl6aNsoMgYcwpARN4CHgA6JgUDtFeeigHOWxXM6tWrrdq0UsqHzp49S25uLvX19SQmJhIXF6cJoQ9ZmRRGAKUdlsuAWZ36PAt8LCJfAwYB93a3IRF5FHgUYNSoUX0eqFLK/zU2NrJjxw6KioqIi4vj3nvv1QJ2FrD7QvMG4NfGmJ+IyBzgtyIy1Rjj6tjJGPMq8CpAVlaW6c2ODh48CEB6evqdRayU8rmOBewyMzOZPn06oaGhdocVlKxMCueAkR2Wkz3rOvoKsALAGLNTRMKBBOByXwdz9uxZQJOCUoGkvr6egQMHEhISwuzZs4mMjCQ+Pt7usIKalXcf7QZSRWSMiPQH1gObO/U5CywBEJHJQDhQbmFMSqkAYIzh+PHjbNq0yVvAbvTo0ZoQfMCykYIxplVEHgf+CIQCrxtjjorI94A9xpjNwJPAL0Xkm7gvOn/JGNOr00NKqeBQXV1NTk4O58+fJykpiREjRtgdkqNYek3BGPMh7ttMO677TofXBcA8K2NQSgUOLWBnP7svNPuMVklVyv9FREQwfPhw5s+frwXsbOKYT0qdT0Ep/9PW1uYtYJeVlaUF7PyAY5KCUsq/XL58mW3btnHt2jVSU1O1gJ2fcExS2LdvHwCZmZk2R+LW0NxGU6uL0BD9I1DO0trayp49ezh8+DAREREsX76c0aNH2x2W8nBMUjh3zv2IhL8khd/tOktTq4vlacPsDkUpn6qurubIkSNMmjSJWbNm0b9/f7tDUh04Jin4k8aWNl7ZVsycsfFkjxlsdzhKWa5zAbv169frhWQ/pUnBBpv2lHK5pomfrZ9udyhKWa5jAbuhQ4cSGxurCcGPaVLwsabWNn7xWTHZKYOZPVZHCSp4NTQ0sHPnTm8Bu6VLlxIbG2t3WOoWHJMUBgwYYHcIALyzp4wLVY28sDZd77RQQcvlcrF582ZqamqYMWMGGRkZWsAuQDgmKSxbtszuEGhudfGLz4rJHBXLvPFaw0UFn84F7KKiohg8WEfEgcTS6TjV9f6/fWWcq2zg60tSdZSggooxhoKCAt5+++3rCthpQgg8jhkp7Nq1C4Ds7Gxb9t/S5uLlz4pIT45h0YQhtsSglBWqqqrIycnhwoULDB8+XJ9IDnCOSQqXLl2ydf+/33+O0qsNPLs6TUcJKmgUFhayfft2QkJCWLhwIRMnTtTf7wDnmKRgp9Y2F//70yKmjojmnkmJdoejVJ+JjIwkOTmZ+fPnM2jQILvDUX1Ak4IPbD54njMV9bz6P2botygV0Nra2ti/fz8AWVlZjBgxQuc7CDKaFCzW5jL870+KmJwUzdIpQ+0OR6le61jAbsKECVrALkg5JinYNbR9/9B5Tl2p4xf/V6b+AamA1NLS4i1gN2jQIFasWMGoUaPsDktZxDFJ4Z577vH5Pl0uw//6pIgJQyO18J0KWLW1tRQUFDBlyhSys7O1gF2Qc0xSsMNHRy5SdLmW/7VhOiFaIlsFkKamJkpKSpg0aRJxcXGsW7dO6xU5hGOSwo4dOwCYO3euT/bnHiWcZNyQQayaluSTfSrVF06fPs327dtpaGhg2LBhWsDOYRyTFCoqKny6v48LLnL8Yg0/W5+hE+mogNDQ0EBeXh6nTp1i8ODBLF++XAvYOZBjkoIvGWP42Z+LGJswiPvvGm53OErdksvl4r333qO2tpasrCwyMjIICdEqOE6kScECfzp2mWMXqvnJw+k6SlB+ra6ujoiICEJCQpg7dy5RUVHExcXZHZaykX4V6GPGGH7+55OMGhzBAxk6SlD+qb2A3aZNmygoKABg1KhRmhCUc0YKMTExPtnPZ4XlHD5XxY/++i7CQjXnKv9TWVlJbm4uFy5cYMSIEYwcOdLukJQfcUxSWLhwoeX7cF9LOMmI2IE8lKmP/iv/c/z4cfLy8ggNDWXRokVMmDBBH6pU13FMUvCF3JNXOFBayf/z0DT66ShB+aGoqChGjhzJ/PnziYiIsDsc5YcckxRycnIA60YM7aOE4THhrJ2h9eSVf2hra2Pfvn0AzJw5UwvYqVtyTFKoqqqydPvltU3sPXONp1ZMon+YjhKU/S5evEhOTg6VlZVMnDhRC9ipHnFMUrBam8sAEBfRz+ZIlNO1tLSwe/dujhw5QmRkJCtXrtSLyarHLP1KKyIrRKRQRIpE5Okb9HlERApE5KiIvGllPEo5QW1tLceOHSMtLY21a9dqQlC3xbKRgoiEAi8DS4EyYLeIbDbGFHTokwp8G5hnjLkmIjotmVK90NTUxKlTp5g8eTJxcXGsX79eZ0JTvWLl6aNsoMgYcwpARN4CHgAKOvT5n8DLxphrAMaYy1YFEx8fb9WmlbJVSUkJ27dvp7GxkaSkJGJjYzUhqF6zMimMAEo7LJcBszr1mQAgInlAKPCsMeYPnTckIo8CjwK9ntzDV9VRlfKV+vp68vLyKCkpIT4+npUrV2oBO3XH7L7QHAakAncDyUCOiEwzxlR27GSMeRV4FSArK8v4Okil/I3L5WLz5s3U1dUxc+ZM0tPTtYCd6hNWJoVzQMcrXMmedR2VAZ8bY1qAEhE5gTtJ7O7rYD755BPAnhnYlOortbW1DBo0yFvALjo6WkcHqk9Z+dViN5AqImNEpD+wHtjcqc/vcY8SEJEE3KeTTlkRTF1dHXV1dVZsWinLGWM4cuRIlwJ2mhBUX7NspGCMaRWRx4E/4r5e8Lox5qiIfA/YY4zZ7GlbJiIFQBvwLWOMb2fDUcrPVVZWsm3bNi5dukRycnKvr6sp1ROWXlMwxnwIfNhp3Xc6vDbAE55/SqlO2gvYhYWFcffdd5OamqpPJStL2X2hWSl1E9HR0YwaNYp58+ZpATvlE45JCkOHDrU7BKVuqbW11VvALjs7m+HDhzN8uE7WpHzHMUkhOzvb7hCUuqmLFy+ybds2qqqqmDRpkhawU7ZwTFJQyl81Nzeze/dujh49SlRUFKtWrSI5WcuvK3s4Jil8/PHHACxbtszmSJS6Xl1dHcePH2fq1KnMnDmTfv200q6yj2OSQlNTk90hKOXV2NjIqVOnmDJlCnFxcWzYsEEvJCu/cNtJQURCgA3GmDcsiEepoGaMoaSkhLy8PBobGxk+fDixsbGaEJTfuGFSEJFo4Ku4C9ttBrYCjwNPAgcBTQpK3Yb6+nq2b9/O6dOnSUhIYNWqVfpEsvI7Nxsp/Ba4BuwE/g74F0CAB40xB3wQm1JBo2MBu1mzZjFt2jQtYKf80s2SwlhjzDQAEfkVcAEYZYxp9ElkfUwnK1d26FjAbt68eURFRenoQPm1myWFlvYXxpg2ESkL1IQAkJmZaXcIykFcLhcFBQXs2rWLWbNmkZaWptNiqoBws6SQLiLVuE8ZAQzssGyMMdGWR6dUALp27Ro5OTlcunSJkSNHMnr0aLtDUqrHbpgUjDGhvgzEah999BEAK1eutDkSFcyOHTtGXl4e/fr1Y/HixYwfP16fSlYB5WZ3H4UD/zcwHjiEu/R1q68C62utrQEbugogMTExpKSkMG/ePAYOHGh3OErdtpudPvoN7usKucAqIA34R18EpVSgaG1tZe/evQDMmjVLC9ipgHezpDClw91HrwG7fBOSUoHhwoUL5OTkUFVVxeTJk7WAnQoKPb37qFV/2ZVya25uZteuXRQUFBAVFcV9992ntzyroHGzpJDhudsI3HccBfTdRzqFoeor9fX1FBYWMm3aNLKysrSAnQoqN0sKB40x030WicXS09PtDkEFsMbGRoqLi0lLSyM2NlYL2KmgdbOkYHwWhVJ+yhjDqVOnyMvLo7m5mREjRmgBOxXUbpYUEkXkiRs1GmNetCAey2zZsgWA1atX2xyJChR1dXVs376dM2fOMGTIEBYtWqQlKlTQu1lSCAUi+csTzUo5hsvlYsuWLdTV1TF79mymTp2qBeyUI9wsKVwwxnzPZ5Eo5Qdqamq8Bezmz59PVFQUMTExdoellM/c7KuPjhCUY7hcLg4dOsSmTZsoKCgAIDk5WROCcpybjRSW+CwKpWx09epVtm3bRnl5OaNGjSIlJcXukJSyzc0K4l31ZSBWGzt2rN0hKD9UUFDAjh076N+/P/fccw/jxo3Tp5KVo932HM2BKi0tze4QlB9pL0kRGxvL2LFjmTNnjhawUwoHJYX2KqlhYY45ZNWN1tZW9uzZg4hoATuluuGYT8j2+RT0OQXnOn/+PDk5OVRXVzNlyhQtYKdUNxyTFJRzNTc3k5+fz/Hjx4mOjub+++/X0YFSN6BJQQW9+vp6ioqKuOuuu8jKytJTiErdhKWPaIrIChEpFJEiEXn6Jv3+WkSMiGRZGY9yjoaGBo4cOQLgLWA3e/ZsTQhK3YJlfyEiEgq8DCwFyoDdIrLZGFPQqV8U7hndPrcqFuUcxhiKi4vJy8ujpaWF5ORkYmNj9c4ipXrIyq9N2UCRMeYUgIi8BTwAFHTq9+/A88C3LIyFCRMmWLl55Qdqa2vZvn07Z8+eJTExkYULF2oBO6Vuk5VJYQRQ2mG5DJjVsYOIZAIjjTEfiMgNk4KIPAo8Cr2fLGfixIm9ep8KDO0F7BoaGpgzZw5paWlawE6pXrDtBKuIhAAvAl+6VV9jzKvAqwBZWVm9muehsbERgPDw8N68XfmpjgXsFixYQHR0NNHRATUpoFJ+xcqvUueAkR2Wkz3r2kUBU4HPROQ0MBvYbNXF5q1bt7J161YrNq1s4HK5OHjwYJcCdpoQlLozVo4UdgOpIjIGdzJYD2xsbzTGVAEJ7csi8hnwT8aYPRbGpIJARUUFOTk5lJeXM3r0aMaMGWN3SEoFDcuSgjGmVUQeB/6Ie8Ke140xR0Xke8AeY8xmq/bdF4rLa3n4lZ3UN7f2qL/Lc1IrRJ+QtdTRo0fZsWMHAwYMYMmSJYwdO1afSlaqD1l6TcEY8yHwYad137lB37utjOV2lV6t52pdMw9mDGdodM+uQ/QLDWHJ5ESLI3Om9pIUgwcPZty4ccydO1evDyllAX2S5xb+Zm4KmaPi7A7DsVpaWti9ezchISHMnj2bpKQkkpKS7A5LqaDlmKQwZcoUu0NQt+ncuXPk5ORQU1NDWlqaFrBTygcckxTGjRtndwiqh5qamsjPz6ewsJCYmBhWr16towOlfMQxSaG2thaAyMhImyNRt9LQ0EBxcTHp6enMmDFD6xUp5UOO+Wv79NNPAZ1PwV/V19dTXFzMtGnTiI2NZePGjXohWSkbOCYpKP9kjKGoqIgdO3bQ0tLCqFGjiImJ0YSglE00KSjb1NbWkpubS2lpKUOHDmXhwoXExMTYHZZSjqZJQdmiYwG7uXPnMmXKFC1gp5Qf0KSgfKq6uprIyEhCQkJYuHAh0dHRREVF2R2WUsrDMUnhrrvusjsER3O5XBw6dIi9e/cya9Yspk6dyogRI+wOSynViWOSwujRo+0OwbGuXLlCTk4OV65cISUlhbFjx9odklLqBhyTFCorKwF0Ji4fO3LkCDt37iQ8PJx7771XE4JSfs4xSSE3NxfQ5xR8pb0kRXx8PKmpqcyePVtvM1UqADgmKSjf0AJ2SgU2TQqqz5SVlZGTk0NtbS1Tp07VAnZKBSBNCuqONTU1sXPnTk6cOEFMTAxr1qxh2LBhdoellOoFTQrqjjU0NFBSUkJGRgaZmZlawE6pAOaYv97p06fbHUJQqa+vp6ioiLvuuovY2Fg2bNigF5KVCgKOSQrJycl2hxAUjDGcPHmSnTt30trayujRo7WAnVJBxDFJ4cqVKwAkJCTYHEngqqmpITc3l7KyMoYOHcqiRYu0gJ1SQcYxSWHnzp2APqfQWy6Xi/fff5/GxkbmzZvHlClT9M4ipYKQY5KC6p2qqiqioqIICQlh0aJFREVFaQE7pYKYY5KCyxjqmlrZWVzRo/7HL9ZYHJF/c7lcHDx4kL179zJ79mymTp3K8OHD7Q5LKWUxxySF8pomSq7U8S9782/rfZEDHPMj8rpy5Qrbtm2joqKCsWPHar0ipRzEMZ94bS4DwP/58kzCw0J79J6o8DBSEyOtDMvvdCxgt3TpUsaMGWN3SEopH3JMUmDIOD4rOsO/jhlMRH/nHHZPdS5gN2fOHAYMGGB3WEopH3POp+PAWM41XbQ7Cr/T3NzMrl27CA0NZc6cOVrATimHc05SaKhkxIAmu6PwK6WlpeTm5lJbW8u0adO0gJ1SykFJobyYuwfX2x2FX2hsbGTnzp2cPHmS2NhYHnjgAYYOHWp3WEopP+CcpKC8GhsbOX36NJmZmUyfPp3Q0J5deFdKBb8QKzcuIitEpFBEikTk6W7anxCRAhE5JCJ/FhGdSNki9fX1HDx4EGMMsbGxbNy4kaysLE0ISqnrWDZSEJFQ4GVgKVAG7BaRzcaYgg7d9gNZxph6EXkM+BGwzqqYnMgYQ2FhIfn5+bS1tZGSkkJMTIzeWaSU6paVp4+ygSJjzCkAEXkLeADwJgVjzKcd+ucDX7AwHseprq4mNzeXc+fOkZSUxIIFC7SAnVLqpqw8fTQCKO2wXOZZdyNfAT7qrkFCDnr7AAASFElEQVREHhWRPSKyp7y8vHfRJE5ga4VzPhBdLhcffPABly9fZv78+dx///3ExsbaHZZSys/5xYVmEfkCkAUs6q7dGPMq8CpAVlaW6dVOwqO43Ny/tyEGjM4F7KKjo4mMdNZT2Uqp3rNypHAOGNlhOdmz7joici/wr8AaY4x1DxLUXSVlYKNlm7eby+Vi3759vPPOOxw9ehSA4cOHa0JQSt0WK0cKu4FUERmDOxmsBzZ27CAi04H/BFYYYy5bGAtUlDAvNjifUygvL2fbtm1cvXqVcePGMX78eLtDUkoFKMuSgjGmVUQeB/4IhAKvG2OOisj3gD3GmM3AC0Ak8I7nSdqzxpg1VsUUjA4fPkx+fj4DBw5k2bJlpKSk2B2SUiqAWXpNwRjzIfBhp3Xf6fD6Xiv3H8zaS1IMGTKEiRMnMmvWLL3NVCl1x/ziQrPquebmZj7//HNCQ0OZO3cuw4YNY9iwYXaHpZQKEpoUAsjZs2fJzc2lvr5eC9gppSzhnKQwbBIfnTjFv9odRy80NjayY8cOioqKiIuLY+nSpSQmJtodllIqCDknKfQfxNWWfnZH0StNTU2cOXNGC9gppSznnKRQW874iAa7o+ixuro6Tp48SXp6OjExMWzcuFEvJCulLOecpHD1LLNi/P85BWMMx48fJz8/H5fLxZgxY7SAnVLKZ5yTFAJAdXU1OTk5nD9/nqSkJBYuXKgF7JRSPqVJwU+4XC7ef/99mpqaWLBgAZMmTdI7i5RSPqdJwWaVlZVER0cTEhLC3XffrQXslFK2snTmNX8iIoT40TfvtrY29u7dy7vvvqsF7JRSfsMxI4WND64EIKK//Yd8+fJltm3bxrVr1xg/fjypqal2h6SUUoCDkoK/fANvL2AXERHB8uXLGT1ap6VWSvkPxySF4uJiAMaNG2fL/jsWsJs0aRKzZs2if//gn/RHKRVYHJMUCgrcU0P7Oik0NzeTn59PWFiYFrBTSvk9xyQFO5w5c4bc3FwaGhq46667tICdUsrvaVKwQENDAzt27KC4uJjBgwezbNkyLWCnlAoImhQs0NzcTGlpKTNmzCAjI0ML2CmlAoYmhT5SW1vLyZMnycjI8Baw0wvJSqlA45iksHTpUku2a4zh2LFjfP755xhjGDt2LDExMZoQlFIByTFJITw8vM+3WVVVRU5ODhcuXGDEiBEsWLCA6OjoPt+PUkr5imOSQmFhIQATJ07sk+25XC4++OADmpubWbhwIRMnTtQ7i5RSAc8xSeHEiRPAnSeFa9euERMTQ0hICIsXLyY6OppBgwb1RYhK9VpLSwtlZWU0NjbaHYqyWXh4OMnJyfTr17uZJh2TFO5UW1sb+/fvZ//+/cyePZtp06aRlJRkd1hKAVBWVkZUVBQpKSk6YnUwYwwVFRWUlZUxZsyYXm1Dk0IPXLp0iZycHK5du0ZqaqoWsFN+p7GxUROCQkSIj4+nvLy819vQpHALhw4dIj8/n0GDBrFixQpGjRpld0hKdUsTgoI7/z3QpHAD7SUpEhMTmTJlCtnZ2XqbqVIq6Dlmkp2VK1eycuXKW/Zrampi27Zt7NixA4Bhw4Yxf/58TQhK3URlZSX/8R//0ev3v/TSS9TX19+wfe3atZw6darX27daSUkJs2bNYvz48axbt47m5uYufZqbm/nyl7/MtGnTSE9P57PPPgOgvr6e++67j0mTJpGWlsbTTz/tfc83v/lNMjIyyMjIYMKECcTGxgJQXl7OihUrLDkWxySFsLAwwsJuPjA6ffo077zzDidOnKBfv34YY3wUnVKBzcqkcPToUdra2hg7dmyPt9fW1tbrWHrjqaee4pvf/CZFRUXExcXx2muvdenzy1/+EnDPqbJ161aefPJJXC4XAP/0T//E8ePH2b9/P3l5eXz00UcA/PSnP+XAgQMcOHCAr33ta/zVX/0VAEOGDCEpKYm8vLw+PxbHnD5qn/IyLS2tS1tDQwN5eXmcOnWK+Ph4VqxYQUJCgq9DVKpPPLflKAXnq/t0m1OGR/Pd1V3/dto9/fTTFBcXk5GRwdKlS3nhhRd44YUX2LRpE01NTTz00EM899xz1NXV8cgjj1BWVkZbWxv/9m//xqVLlzh//jyLFy8mISGBTz/99Lptv/HGGzzwwAPe5ccee4zdu3fT0NDA2rVree655wBISUlh3bp1bN26lX/+539m5syZfPWrX6W8vJyIiAh++ctfMmnSJLZs2cL3v/99mpubiY+P54033mDo0KG9/tkYY/jkk0948803AfjiF7/Is88+y2OPPXZdv4KCAu655x4AEhMTiY2NZc+ePWRnZ7N48WIA+vfvT2ZmJmVlZV3287vf/c57rAAPPvggb7zxBvPmzet17N1xTFJoH3p2lxSam5spKytj5syZpKenExLimAGUUn3ihz/8IUeOHOHAgQMAfPzxx5w8eZJdu3ZhjGHNmjXk5ORQXl7O8OHD+eCDDwB3VYCYmBhefPFFPv30026/jOXl5bFhwwbv8g9+8AMGDx5MW1sbS5Ys4dChQ9x1110AxMfHs2/fPgCWLFnCK6+8QmpqKp9//jn/8A//wCeffML8+fPJz89HRPjVr37Fj370I37yk59ct8/CwkLWrVvX7bF+9tln3tM4ABUVFcTGxnrPRCQnJ3Pu3Lku70tPT2fz5s1s2LCB0tJS9u7dS2lpKdnZ2d4+lZWVbNmyhX/8x3+87r1nzpyhpKTEm1QAsrKyeOaZZ7qN8U44Jil0Vltby4kTJ5g+fboWsFNB5Wbf6H3l448/5uOPP2b69OnAXwpGLliwgCeffJKnnnqK+++/nwULFtxyWxcuXGDIkCHe5U2bNvHqq6/S2trKhQsXKCgo8CaF9g/y2tpaduzYwcMPP+x9X1NTE+B+pmPdunVcuHCB5ubmbu/nnzhxojfB9ZW//du/5dixY2RlZTF69Gjmzp17XQXl1tZWNmzYwNe//vUup8reeust1q5de13/xMREzp8/36cxgsVJQURWAD8DQoFfGWN+2Kl9APBfwAygAlhnjDltZUzGGAoKCrzfYMaNG6cF7JTqY8YYvv3tb/P3f//3Xdr27dvHhx9+yDPPPMOSJUv4zne+c9NtDRw40PukdklJCT/+8Y/ZvXs3cXFxfOlLX7ruKe726gIul4vY2NhuP9i/9rWv8cQTT7BmzRo+++wznn322S59bmekEB8fT2VlJa2trYSFhVFWVsaIESO6vC8sLIyf/vSn3uW5c+cyYcIE7/Kjjz5Kamoq3/jGN7q896233uLll1++bl1jYyMDBw7sNsY7Ydl5EhEJBV4GVgJTgA0iMqVTt68A14wx44GfAs9bFQ+4M/GWLVvIy8sjMTGRhx9+mJiYGCt3qZQjREVFUVNT411evnw5r7/+OrW1tQCcO3eOy5cvc/78eSIiIvjCF77At771Le+pns7v72jy5MkUFRUBUF1dzaBBg4iJieHSpUveC7KdRUdHM2bMGN555x3AnaQOHjwIuE9ZtX9o/+Y3v+n2/e0jhe7+dUwI4H4uYPHixbz77rvebXa8BtKuvr6euro6ALZu3UpYWBhTprg/Ep955hmqqqp46aWXurzv+PHjXLt2jTlz5ly3/sSJE0ydOrXb+O+ElSfPs4EiY8wpY0wz8BbQ+Sf1AND+v/IusEQsegKn/fHvq1evsmjRIlatWkVUVJQVu1LKceLj45k3bx5Tp07lW9/6FsuWLWPjxo3MmTOHadOmsXbtWmpqajh8+DDZ2dlkZGTw3HPPec+JP/roo6xYscJ7wbWj++67z3v7Znp6OtOnT2fSpEls3LjxphdZ33jjDV577TXS09NJS0vjvffeA+DZZ5/l4YcfZsaMGX12Q8nzzz/Piy++yPjx46moqOArX/kKAJs3b/aOhC5fvkxmZiaTJ0/m+eef57e//S3gPp31gx/8gIKCAjIzM8nIyOBXv/qVd9tvvfUW69ev7/JQ2qeffsp9993XJ/F3JFbddikia4EVxpi/8yz/D2CWMebxDn2OePqUeZaLPX2udNrWo8CjAKNGjZpx5syZXsV08eJFoqOjiYiI6NX7lfJXx44dY/LkyXaHYYmGhgYWL15MXl6ezmLYwcKFC3nvvfeIi4vr0tbd74OI7DXGZN1quwFxm40x5lVjTJYxJqvjBafbNWzYME0ISgWYgQMH8txzz3V7R49TlZeX88QTT3SbEO6UlReazwEjOywne9Z116dMRMKAGNwXnJVSymv58uV2h+BXhgwZwoMPPmjJtq0cKewGUkVkjIj0B9YDmzv12Qx80fN6LfCJ0ceIleoV/dNRcOe/B5YlBWNMK/A48EfgGLDJGHNURL4nIms83V4D4kWkCHgCeLr7rSmlbiY8PJyKigpNDA7XfkPNnUw/bNmFZqtkZWWZPXv22B2GUn5FZ15T7W4081pPLzQ79olmpYJJv379ej3TllIdBcTdR0oppXxDk4JSSikvTQpKKaW8Au5Cs4iUA717pBkSgCu37BVc9JidQY/ZGe7kmEcbY2759G/AJYU7ISJ7enL1PZjoMTuDHrMz+OKY9fSRUkopL00KSimlvJyWFF61OwAb6DE7gx6zM1h+zI66pqCUUurmnDZSUEopdROaFJRSSnkFZVIQkRUiUigiRSLSpfKqiAwQkbc97Z+LSIrvo+xbPTjmJ0SkQEQOicifRWS0HXH2pVsdc4d+fy0iRkQC/vbFnhyziDzi+b8+KiJv+jrGvtaD3+1RIvKpiOz3/H6vsiPOviIir4vIZc/MlN21i4j83PPzOCQimX0agDEmqP4BoUAxMBboDxwEpnTq8w/AK57X64G37Y7bB8e8GIjwvH7MCcfs6RcF5AD5QJbdcfvg/zkV2A/EeZYT7Y7bB8f8KvCY5/UU4LTdcd/hMS8EMoEjN2hfBXwECDAb+Lwv9x+MI4VsoMgYc8oY0wy8BTzQqc8DwG88r98FlkjnWbEDyy2P2RjzqTGm3rOYj3smvEDWk/9ngH8HngeCoaZ0T475fwIvG2OuARhjLvs4xr7Wk2M2QLTndQxw3ofx9TljTA5w9SZdHgD+y7jlA7EiktRX+w/GpDACKO2wXOZZ120f454MqAqI90l01ujJMXf0FdzfNALZLY/ZM6weaYz5wJeBWagn/88TgAkikici+SKywmfRWaMnx/ws8AURKQM+BL7mm9Bsc7t/77dF51NwGBH5ApAFLLI7FiuJSAjwIvAlm0PxtTDcp5Duxj0azBGRacaYSlujstYG4NfGmJ+IyBzgtyIy1RjjsjuwQBSMI4VzwMgOy8medd32EZEw3EPOCp9EZ42eHDMici/wr8AaY0yTj2Kzyq2OOQqYCnwmIqdxn3vdHOAXm3vy/1wGbDbGtBhjSoATuJNEoOrJMX8F2ARgjNkJhOMuHBesevT33lvBmBR2A6kiMkZE+uO+kLy5U5/NwBc9r9cCnxjPFZwAdctjFpHpwH/iTgiBfp4ZbnHMxpgqY0yCMSbFGJOC+zrKGmNMIM/l2pPf7d/jHiUgIgm4Tyed8mWQfawnx3wWWAIgIpNxJ4Vyn0bpW5uBv/HchTQbqDLGXOirjQfd6SNjTKuIPA78EfedC68bY46KyPeAPcaYzcBruIeYRbgv6Ky3L+I718NjfgGIBN7xXFM/a4xZY1vQd6iHxxxUenjMfwSWiUgB0AZ8yxgTsKPgHh7zk8AvReSbuC86fymQv+SJyO9wJ/YEz3WS7wL9AIwxr+C+brIKKALqgS/36f4D+GenlFKqjwXj6SOllFK9pElBKaWUlyYFpZRSXpoUlFJKeWlSUEop5aVJQakeEpE2ETnQ4V+KiNwtIlWe5WMi8l1P347rj4vIj+2OX6meCLrnFJSyUIMxJqPjCk/Z9VxjzP0iMgg4ICJbPM3t6wcC+0Xkv40xeb4NWanboyMFpfqIMaYO2AuM77S+AThAHxYtU8oqmhSU6rmBHU4d/XfnRhGJx11j6Win9XG46w/l+CZMpXpPTx8p1XNdTh95LBCR/YAL+KGnDMPdnvUHcSeEl4wxF30Yq1K9oklBqTuXa4y5/0brRWQMkC8im4wxB3wdnFK3Q08fKWUxTwnrHwJP2R2LUreiSUEp33gFWOi5W0kpv6VVUpVSSnnpSEEppZSXJgWllFJemhSUUkp5aVJQSinlpUlBKaWUlyYFpZRSXpoUlFJKef3/OfmcpabJ8d8AAAAASUVORK5CYII=\n", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# Plot the ROC curve for the test set\n", "plt.plot(fpr, tpr, label='test (area = %.3f)' % auroc)\n", "plt.plot([0,1], [0,1], linestyle='--', color=(0.6, 0.6, 0.6))\n", "plt.plot([0,0,1], [0,1,1], linestyle='--', color=(0.6, 0.6, 0.6))\n", "\n", "plt.xlim([-0.05, 1.05])\n", "plt.ylim([-0.05, 1.05])\n", "plt.xlabel('FPR')\n", "plt.ylabel('TPR')\n", "plt.legend(loc='lower right')\n", "plt.title('ROC curve')\n", "\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": 22, "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl8VNX9+P/XOztkIRASdghhCYEQtrCJIosgKIulLlAo4sbXIi2fVm21VT/WT/2p1VqrYtVPq6ilLEJRsKIoywdlk7DKLkuAhJCEACEJWUhyfn/cye1kZYDMDEnez8djHpl775l73ncY5j33nHvPEWMMSimlFICPtwNQSil1/dCkoJRSyqZJQSmllE2TglJKKZsmBaWUUjZNCkoppWyaFFSdJCJ7RWTYZcq0F5FcEfH1UFi1TkSiRcSIiJ9jeZ2IPOjtuFT95eftAFT9IiLJQAugBMgDVgKzjTG5tVmPMaaHC2VOACG1Wa9S9Z2eKSh3GG+MCQH6AonAUxULiKXefP7KfsnXB/XpWNSVqzf/KdX1xxiTinWmEA9208fzIrIBuAjEiEgTEfm7iKSJSKqI/MG5uUdEHhKR/SKSIyL7RKSvY32yiNzieD5ARJJE5IKIpIvIq471FZteWovIchE5KyKHReQhp3qeFZHFIvKho669IpJY3bE5yi8RkX+IyAVghoj4iMgTInJERLIc+2vm9JobRWSjiJwXkZMiMsOx/nYR2eGI/6SIPHs177eI+IrIbx3154jINhFpV/F9cPq3eNDxfIaIbBCRP4tIFvA/jhjjncpHiki+iEQ5lseJyE5HuY0iknA1MavrjyYF5TYi0g64DdjhtPqnwEwgFDgOzAOKgc5AH2A0UPZldRfwLDAdCAMmAFlVVPUX4C/GmDCgE7C4mpAWAilAa+BO4P8TkRFO2yc4yoQDy4E3L3OIE4EljvLzgZ8DdwA3O+o4B8x1HEsHrAT5BhAJ9AZ2OvaT5zjGcOB24Gcicsdl6q7Kr4ApWO95GHA/VvJ1xUDgKFbT33PAvxz7KnM38H/GmAwR6QO8B/w/IAJ4B1guIoFXEbO63hhj9KGPWnsAyUAucB7rS/8toJFj2zrgOaeyLYDCsu2OdVOAtY7nXwJzaqjnFsfz9cDvgeYVykQDBqvvrB1WP0eo0/YXgHmO588CXztt6w7k13CczwLrK6zbD4x0Wm4FXHLU/ySwzMX38DXgzxWPwek9fLCa1x0EJlaxvtw+Ku4HmAGcqPCaW4AjTssbgOmO538F/qeKum/29udPH9f+0DMF5Q53GGPCjTEdjDGzjDH5TttOOj3vAPgDaY5miPNYvzqjHNvbAUdcqO8BoCtwQES2isi4Ksq0Bs4aY3Kc1h0H2jgtn3Z6fhEIEhE/EZnquIopV0RWVnMsZcezzOlY9mMlohY1HYuIDBSRtSKSKSLZwMNA88sedWWuvl9VqXgsa4HGjtiisc5sljm2dQAeLTtOx7G2w3qPVR2nHUrK05yH5T2JdabQ3BhTXEXZk1jNQTXv0JgfgCmOjutJwBIRiahQ7BTQTERCnRJDeyDVhf3Px2oeqrSpinjvN8ZsqFhQRE4CA6qp4p9YTVVjjTEFIvIaV5cUyt6vPRXW5zn+NgYuOJ63rFCm3LEYY0pEZDHWmVs68JnT+3YSeN4Y8/xVxKiuc3qmoLzGGJMGrAL+JCJhjo7aTiJys6PI34DHRKSf42qlzo62+XJEZJqIRBpjSrGarQBKK9R1EtgIvCAiQY6O0QeAf9TiIb0NPF8Wo6NzdqJj23zgFhG523H2ESEivR3bQrHOYgpEZADwk6us/29YncRdHO9XgohEGGMysZLfNEdn9P24kGyxktU9wFTH8zL/CzzsOIsQEQl2dJaHXmXc6jqiSUF523QgANiH1TG7BKstHmPMx8DzWF9IOcAnQLMq9jEG2CsiuVidzpMrNFmVmYLVvn4Kqynkv40xX9fisfwFq4N6lYjkAJuxOnAx1j0TtwGPAmexOpl7OV43C3jO8ZpnqL6j/HJedbx2FdYZwd+BRo5tDwGPY3XU98BKkDUyxmzBOstojdVJXrY+ybG/N7H+zQ5j9UuoekCM0Ul2lFJKWfRMQSmllE2TglJKKZsmBaWUUjZNCkoppWx17j6F5s2bm+joaG+HoZRSdcq2bdvOGGMiL1euziWF6OhokpKSvB2GUkrVKSJy3JVy2nyklFLKpklBKaWUTZOCUkopmyYFpZRSNk0KSimlbG5LCiLynohkiEjFYXzLtouIvC7WtIi7xTHNolJKKe9x55nCPKzRK6szFujieMzEms1JKaWUF7ntPgVjzHrHjE3VmQh8aKxhWjeLSLiItHKMsV/r9m/5ku279lda30Qu0FzOU2qEY6Zdpe1NJZtmkk2x8eW4aVNpe4ScI1xyKDJ+nDSVJ56KlLOESS4FJoBUU3FeE4iSM4TKRfJNIKdMi0rbW0omwZJPnmnE6SruO2kt6TSSQnJMYzJM5XlZ2shpgqSICyaETFN51Ol2cooAKea8CSXLNC23zUfgjug8GvnCwQuBHLoQVOn1Y1tn4+cDe7ODOJpTeYre8W2zAdh1rhEn8gLKbfPzsV4PsP1sY1Iv+pfbHuhrGN3KmhPmuzPBpBeU/7gG+5UyoqU178vGzGCyCstvbxJQwtCoXADWZ4SQXeRbbntEYDE3RFrzz6w5HUpecfnfSC2CihnQ3Nq+Ki2MwhIpt71N40v0bWZNgbzyVBOKy83gAO2Di+jV1BrBe0VKEyqKCS2kR5MCikut11fUNayA2LBCCkqEr9LCKm3v3qSATqGF5F7yYW165akMEprm0yG4iPNFvnyTEVJpe59mF2nb+BJnCn3ZlFl5e/+IPFo2KuZ0vh9bs4IrbR8cmUvzwBJSLvqz42zjSttvisolPKCE43kB7D7XqNL24S1yCPEv5UhOIPuyK3+2RrW6QJCvqb3PXuNmEGi9T35+fowdOxaA7du3k5pafq6lwMBARo8eDcB3331Henp6ue3BwcGMGGFN771x40aysspPHd6kSROGDh0KwPr168nOzi63PSIightuuAGANWvWkJeXV257ixYtGDDAmpNp1apVFBYWAtC5c2fi4uIqHWtt8ubNa20oPwVgimNdpaQgIjOxziZo3779VVWWfWgDoY431ll0/m66522iWPw5E3F/pe0xF5PoenEbBT6NOdvsp5W2d8rbTKf8XeT6NuF808mVtnfJXU+Hgv2c94vkQvikStu75aymTeFhsvxbkdNkQqXt3S98QYui46QHdCAvrPKJV3z2ciIupZEa2Jn80JGVtiec/xfhxZkcD4qjIGRope29zy0ipCSbI416URQ8qNJ2Nn0EpRehUT9onFh5+7H3wFyCxoOhUULl7Uffsf4GD4WgCh9mcwkOv2c9DxkJgZ3Lby+9CD98ZD0PHQMBFebXKcmGQwut52Hjwb9CUi4+AweWWs+b/Bj8KiTNS6dg/wrrefhk8K3wxZx5HPZ9YT1v+lPwqfDFl3EY9qy2nje7H6R8UiN9P+Stt55H/D8qOb0bLm6yXtes8mePtCTI32bV27TyZ49Tm6FgF/g0gSo+e6Suh8L94BsJVXz2SFkNRYfBrxVU8dnj5Bdw6Tj4d4AqPnucWA7FaRDQGar47HH8X1CSCYFxUMVnj+SFUJoNQb2gqs9eci1/9oryoEX3yuXqiLLE4+6k4Nb5FBxnCp8ZY+Kr2PYZ8KIx5lvH8mrgN44JPKqVmJho9I5m91u6LYVHP97F+seH0z6i8q9ApeqUd4dBcCRM/djbkVy1FSusHzDjx4+/qteLyDZjTBXZtTxvnimkYk32XaYtLsyXq5RSDVGbNpWbr93Bm0lhOTBbRBZiTVmY7a7+BKWUquv69vXMBZpuSwoisgAYBjQXkRTgvwF/AGPM28DnWHPWHgYuAve5KxallFKucefVR1Mus90Aj7irfqWUqk9WrlwJYF815S51buhspZRqiIqLiz1Sjw5zoZRSyqZnCkqphiHzAHz+OBQXQHEhXMq3/pYtl/tbANE3wY//19tRe5wmBaVU/dciHvYshe8/Br8g8At0+tvI+hvU5D/rU5Pg+EZvR+0VmhSUUvXfxDeth6s+eQSOrnNbOFfjakdzuFKaFJRSqg7o1auXR+rRjmallFI2TQpKKVUHrFixwh7/yJ00KSillLJpUlBKKWXTpKCUUhWJQF4mbHnXunehAdGkoJRSFQ1+BNr0g5WPwxuJsOMfUOKZYSa8TZOCUkpVFBUH930O05Za03h++gi8Nci6Aa609PKvd4OYmBhiYmLcXo8mBaWUqooIdL4FZq6De/4BPn6w5H54Zygc/ALcOGtlVXr06EGPHj3cXo8mBaWUqokIxI2Hn22ASf8LRTmw4B74+2g4tt5jYRQXF3tkpFRNCkop5QofX0i4G2YnwbjX4EIqfDAePpgAKe6fN37lypX2nArupElBKaWuhK8/JN4HP98Ot74A6XvhbyNh8XSPNym5gyYFpZS6Gv5BMHgWzNkFPe+CfZ9C4QVvR3XNNCkopdS1CAyB1n28HUWt0aSglFLKpkNnK6VUHdC1a1eP1KNJQSml6oDY2FiP1KPNR0opVQcUFBRQUFDg9no0KSilVB3w1Vdf8dVXX7m9Hk0KSimlbJoUlFJK2TQpKKWUsunVR0op5QmX8iHntPXIPQ0llyD+x9aYStcRTQpKKVVb9n9mDXVR9uWfkwa56dbfguzK5Zt1grb9XNp19+7daznYqmlSUEqpa+Xf2Pr76Szrr48/hLaC0JbQvAt0HGo9D2lp/T13DP79KJRecrmKTp06uSHwyjQpKKXUteo1xfryb9TUSgaNmlrzMFTnyJorriI3NxeAkJCQq43SJW7taBaRMSJyUEQOi8gTVWxvLyJrRWSHiOwWkdvcGY9SSrmFfxBE3wgteljTd9aUEK7UpQI4e4y1Kz9h7Vdf1N5+q+G2MwUR8QXmAqOAFGCriCw3xuxzKvYUsNgY81cR6Q58DkS7KyallLquXDwLabvgQhrknLL6IS6csvogLqRZf/PPWmUjHoYI9zchubP5aABw2BhzFEBEFgITAeekYIAwx/MmwCk3xqOUUtcHcTTSLJxScQOERFlNUE07QPtB1vOwVvBDKQQEuz00dyaFNsBJp+UUYGCFMs8Cq0Tk50AwcEtVOxKRmcBMgPbt29d6oEop5VHtBsItz1od1KGtIKy19TckyprZrSopKzwSmrc7mqcA84wxfxKRwcBHIhJvjCl1LmSMeRd4FyAxMbHuz3enlGrY/BvBjb/0dhRVcmdSSAXaOS23daxz9gAwBsAYs0lEgoDmQIYb41JKqTonISHBI/W48+qjrUAXEekoIgHAZGB5hTIngJEAIhIHBAGZboxJKaXqpA4dOtChQwe31+O2pGCMKQZmA18C+7GuMtorIs+JyARHsUeBh0RkF7AAmGGM0eYhpZSq4Pz585w/f97t9bi1T8EY8znWZabO655xer4PGOLOGJRSqj745ptvABg/frxb69FRUpVSStk0KSillLJpUlB1Tmmpdjsp5S6aFFSdUFxSysrv07j77U3EPfMFGTnun8BcqYbI2zevKVWj8xeLWLT1JB9uOk7q+Xwa+ftSWFxKVm4RUaFB3g5PKY/p06ePR+rRpKCuSz+k5/D+xmSWbU8l/1IJg2Ka8fS47pSUGh7553Zvh6eUx7Vt29Yj9WhSUNeN0lLDukMZvL8hmW9+OEOAnw939G7NjBs60r21NW7iF3vSvBylUt5x5swZAJo3b+7WejQpKK/LLSzm46STfLAxmeSsi7QIC+Sx0V2ZMqA9ESGB3g5PqevCpk2bAPffp6BJQXnN8aw85m1M5uOkFHILi+nTPpxfjY5lbHxL/H31GgilvEGTgvIoYwwbj2Tx/oZjrD6Qga8Itye04r4hHendLtzb4dVLxhikNmcCU/WaJgXlEflFJSzbkcq8jcc4lJ5LRHAAs4d3ZtqgDrQI06uIapMxhsMZuaw+kMGa/RnsOHmOzlGhDOzYjEExEQzs2IymwQHeDlNdpzQpKLc6dT6fDzcdZ+HWE5y/eIm4VmH88c4EJvRqTZC/r7fDqzcKi0vYcvQsaw5ksPpAOifP5gPQo3UYUwd24FB6Dgu3nmDexmQAYluEMiimGQNjIhjQsRnNte9GOWhSULXOGMO24+d4f0MyX+w9jTGG0d1bct+QaAZ0bKZNGbUkI6eAdQcyWX0gnW9+OMPFohKC/H24sXNzfnZzZ4Z3i6RVk0Z2+aLiUnannGfz0Sy2HDvL4qQUPth0HIAuUSEMjGnGwI4RDIxp1mDvAckvKiH9QgHtmzXGx+f6+pz279/fI/VoUlC1prC4hM92pTFvYzLfp2YTFuTHAzd25KeDOtCuWWNvh1fnGWPYe+oCq/dnsOZAOrtSsgFo3SSISX3bMLJbCwZ3iqj2DCzAz4fE6GYkRjdjNnCppJTdKdlsOZbF5qNnWbY9lX9sPgFATGQwAztGMCjGanKqb018xSWlJGflceB0DodO53AwPYeDp3M4fvYixsB7MxIZ0a2Ft8Msp2XLlh6pR5OCumYZOQXM33yC+VtOcCa3kM5RIfzhjngm9W1D4wD9iF2Li0XFbDicxZoD6aw5kEH6hUJEoE+7cB6/NZYR3aLo1jL0qs6+/H196NehKf06NGXWMOuLcs+pC9aZxNEsPtt1igXfWUkiOqKx1R/hOJtoHd6o5p1fJ4wxpJ7P56Dji//Q6RwOnM7haGYeRSXWrL8+AtHNg4lrFcZNXSL5aPNxzl+85OXIKzt9+jTg/uSg/2PVVdudcp73NyTz2e5TXCoxDI+N5L4hHbmpS3NtIroGKecusvZABqsPZLDxSBZFxaWEBPpxc9dIRnSLYlhspFvu3/Dz9aF3u3B6twvn4Zs7UVxSyr60C2w5epYtx7L4/Ps0Fm49CUD7Zo0Z2NHqkxgU04y2Tb1/JpiVW/ifL/9068v/h/RccguL7TKtmwTRtWUoN3eNJLZlKF1bhNI5KsQ+uzqelcdHm4976xBqtHXrVkDvU1DXmUslpXy59zTvb0hm2/FzBAf4MnVgB6YP7kBMZIi3w6uTSkoNO0+eczQLZXDgdA5g/Tr/6aAOjOwWRWJ0MwL8PHvvhp+vDwltw0loG85DQ2MoKTXsT7vAlmNn2XI0i1X70vl4WwoAbcIbMdDR1DSoYwTtmjVy2w+D3MJiDqWXb/Y5lJ7Dmdwiu0x4Y39iW4QyqW8bYluGEtsilC4tQmnSyN8tMdUnmhSUS87lFbFg6wk+2nSctGyrI+7pcd25K7EtYUH6H+1KXSi4xPpDmazZn8Hagxmcu3gJPx+hf3Qznro9jhHdoq67JOvrI8S3aUJ8myY8cGNHSksNB9Nz2HLU6pNYdzCTf21PBaBVkyD78teBMRFERzS+4iRRVFzKkcxcDjm++MvOAlLO5dtlGvn70rVFCMNjo6wvf0cCiAwN1LPVq6RJQdXoYHoOb607zLIdqRQWl3JDpwiemxjPiG5R+F5nV2dc745m5lqXjO7PYGvyWYpLDU0b+zM8NooRcVHc1CWyTv2S9fER4lqFEdcqjBlDrCRxODPX0SdxlvWHMlm2w0oSLcIC7SubBsVEENM82P7SLi01nDh78T9t/o6/x87kUeyYO8PPR4iJDKZ3u3DuSWxnJ4B2Ta+/q4TqOk0KqkYPfZhEoJ8PP+rThhlDounWMszbIdUZRcWlJCWftW4iO5DBsTN5AHRrGcrMoTGMjIuid7um9Sa5+vgIXVtY7fTTB0djjOFIZi6bj55l89EsNh3NYvmuUwA0Dwmkd7smZOQU8kN6LvmXSuz9tGvWiNgWoYzq3sL+8o9pHuLx5jNvKyk1nDqfz+GMXI5k5pKXmUvzUPffT6JJQVWpR5swercLZ3SPFkzp317vgHVRVm4h6w5msuZABusPZZJTWEyAnw83dIrg/iHRDO8WdV10ynqCiNA5KpTOUaFMG9QBYwzHzuSx5ZiVJL5PzaZVkyAmD2hHN0enb5cWoYQENqyvpYJLJRzJzOVIZh5HHAngcEYux87kUVhcapfr0iSUB7vEuD2ehvXuK5d1axnGJ48M8XYY1z1jDAdO5ziahdLZcfI8xkBUaCDjerViRLcWDOkcoZfmYiWJmMgQYiJDmDKgvbfD8ShjDGfzihy/+vPsL/4jmbmkns/HOGaY9RFo16wxnSJDuKlLczpHhdDJ8Z4189APM/2kKnWFCi6VsOlIFqsPpLNmfwansq2pQXu1bcJ/jezKyLgourcK07buBqik1HDy7EXHL//ccknA+d6HIH8fOkWG0Ld9U+7q18768o8KJjoiuNqbD1NSrCu93D3ZjiYFpVxwOruANQesO4m/PXyGgkulNA7w5aYuzfmvW7oyLDaSqHp216+q3sWiYo5W+MV/JCOPY2f+c1McQPOQADpFhnBbz1Z0igxx/PIPpnWTRlf8o2HHjh2AJgWlvKK01LA7NZs1+9NZfSCDvacuANC2aSMm92/PiG5RDIxpRqCfDupXXxljyMwt5EhG+S//o5l5pJ7/z2WxPgIdIoLpFBnMsNhIOkWG0Mnx5R/euO71xWlSUMoht7CYb3/IZLXj3oEzuUX4CCR2aMYTY7sxslsUnaNC9Pr3BuBPqw7x7PK9XCj4z93QjQN86RQZQv/opkyOLGvyCaFDRON69eNAk4Jq0E5kXbT6Bg5ksPloFpdKDGFBfgyLjWJkXBRDu0TqlVcNSFRoEH3bhxPo50unqGCnJp8QWoYFNYh+Ik0KqkEpLill2/FzjnkHMjickQtA56gQ7h/SkRHdoujXoSl+Oh1og9QowJd/zWrYV91pUlD13vmLRfzfIatZaN3BDC4UFOPvKwyKiWDqQKt/oENEsLfDVKpGN910k0fq0aSg6p2K01EmHT9LqbGuBLm1R0tGxkVxY5fIBneTlKrbwsM9M4e5W/9XiMgY4C+AL/A3Y8yLVZS5G3gWMMAuY8xP3BmTqp9qmo5y9vDOjIhrQUKbJg2iTVjVT8ePW0N6d+jQwa31uC0piIgvMBcYBaQAW0VkuTFmn1OZLsCTwBBjzDkRiXJXPKr+udLpKJWqy3bv3g3U4aQADAAOG2OOAojIQmAisM+pzEPAXGPMOQBjTIYb41H1yKz52+0B5lydjlIpdXnuTAptgJNOyynAwAplugKIyAasJqZnjTFfVNyRiMwEZgK0b9+wxkxR5bUOb0SArw9NG/tz5zVOR6mUqsylpCAigcCPgWjn1xhjnquF+rsAw4C2wHoR6WmMOe9cyBjzLvAuQGJiornGOlUdltA2nAP/M0b7BpRyE1fPFD4FsoFtQKGLr0kF2jktt3Wsc5YCbDHGXAKOicghrCSx1cU6VAOkCUEp93E1KbQ1xoy5wn1vBbqISEesZDAZqHhl0SfAFOB9EWmO1Zx09ArrUUqpem/48OEeqcfV2zY3ikjPK9mxMaYYmA18CewHFhtj9orIcyIywVHsSyBLRPYBa4HHjTFZV1KPUko1BCEhIYSEuH/ebjHm8k30ji/tzsAxrOYjAYwxJsG94VWWmJhokpKSPF2tUkp51ZEjRwDo1KnTVb1eRLYZYxIvV87V5qOxVxWFUkqpWrFvn3U1/9UmBVe51HxkjDkOhAPjHY9wxzqllFL1iEtJQUTmAPOBKMfjHyLyc3cGppRSyvNcbT56ABhojMkDEJGXgE3AG+4KTCmllOe5evWRACVOyyWOdUoppeoRV88U3ge2iMgyx/IdwN/dE5JSSqmKRo0a5ZF6XEoKxphXRWQdcKNj1X3GmB1ui0oppVQ5QUFBHqmnxqQgImHGmAsi0gxIdjzKtjUzxpx1b3hKKaUADh48CEBsbKxb67ncmcI/gXFYYx453+UmjuUYN8WllFLKyaFDhwAvJwVjzDjH345ujUIppdR1wdX7FIaISLDj+TQReVVEdGIDpZSqZ1y9JPWvwEUR6QU8ChwBPnJbVEoppbzC1aRQbKyR8yYCbxpj5gKh7gtLKaWUN7h6n0KOiDwJTAOGiogP4O++sJRSSjkbO9Yz45K6eqZwD9aQ2Q8YY05jzaL2stuiUkopVY6fnx9+fq7+jr+Gelwp5EgErzotnwA+dFdQSimlytu7dy8APXr0cGs9NZ4piMi3jr85InLB6ZEjIhfcGplSSinb0aNHOXrU/bMVX+4+hRsdf7VTWSmlGgBX71MYJCKhTsuhIjLQfWEppZTyhiu5TyHXaTnPsU4ppVQ94vJ8Co77FAAwxpTi+uWsSiml6ghXv9iPisgv+M/ZwSzA/T0eSimlABg/frxH6nH1TOFh4AYgFUgBBgIz3RWUUkop73D1PoUMYLKbY1FKKVWNXbt2AdCrVy+31uPq1UddRWS1iOxxLCeIyFNujUwppZTtxIkTnDhxwu31uNp89L/Ak8AlAGPMbvTMQSml6h1Xk0JjY8x3FdYV13YwSimlvMvVpHBGRDrhmJJTRO4E0twWlVJKKa9w9ZLUR4B3gW4ikgocA6a6LSqllFLleGKEVHAhKTjmTkg0xtzimJLTxxiT4/7QlFJKlblu5lNw3L38a8fzPE0ISilVf7nap/C1iDwmIu1EpFnZ43IvEpExInJQRA6LyBM1lPuxiBgRSXQ5cqWUakC2b9/O9u3b3V6Pq41U92B1Ms+qsD6muheIiC8wFxiFdRf0VhFZbozZV6FcKDAH2OJq0Eop1dCkpqYC0LdvX7fW4+qZQnesL/hdwE7gDeBy0/8MAA4bY44aY4qAhcDEKsr9D/ASUOBiLEoppdzE1aTwARAHvI6VELo71tWkDXDSaTnFsc4mIn2BdsaYf9e0IxGZKSJJIpKUmZnpYshKKaWulKvNR/HGmO5Oy2tFZF+1pV3guKrpVWDG5coaY97FuiSWxMREc5niSimlrpKrZwrbRWRQ2YJj1rWky7wmFWjntNzWsa5MKBAPrBORZGAQsFw7m5VSqrLAwEACAwPdXo+rZwr9gI0iUjYaU3vgoIh8DxhjTEIVr9kKdBGRjljJYDLwk7KNxphsoHnZsoisAx4zxlwu2SilVIMzevRoj9TjalIYc6U7NsYUi8hs4EvAF3jPGLNXRJ4Dkowxy690n0oppdzL1fkUjl/cTzyMAAAbV0lEQVTNzo0xnwOfV1j3TDVlh11NHUop1RB89501JumAAQPcWo/Os6yUUnVAenq6R+pxtaNZKaVUA6BJQSmllE2TglJKKZv2KSilVB0QHBzskXo0KSilVB0wYsQIj9SjzUdKKaVsmhSUUqoO2LhxIxs3bnR7Pdp8pJRSdUBWVpZH6tEzBaWUUjZNCkoppWyaFJRSStm0T0EppeqAJk2aeKQeTQpKKVUHDB061CP1aPORUkopmyYFpZSqA9avX8/69evdXo82HymlVB2QnZ3tkXr0TEEppZRNk4JSSimbJgWllFI27VNQSqk6ICIiwiP1aFJQSqk64IYbbvBIPdp8pJRSyqZJQSml6oA1a9awZs0at9ejzUdKKVUH5OXleaQePVNQSill06SglFLKpklBKaWUTfsUlFKqDmjRooVH6tGkoJRSdcCAAQM8Uo9bm49EZIyIHBSRwyLyRBXbfyUi+0Rkt4isFpEO7oxHKaVUzdyWFETEF5gLjAW6A1NEpHuFYjuARGNMArAE+KO74lFKqbps1apVrFq1yu31uPNMYQBw2Bhz1BhTBCwEJjoXMMasNcZcdCxuBtq6MR6llKqzCgsLKSwsdHs97kwKbYCTTsspjnXVeQBYWdUGEZkpIkkikpSZmVmLISqllHJ2XVySKiLTgETg5aq2G2PeNcYkGmMSIyMjPRucUko1IO68+igVaOe03NaxrhwRuQX4HXCzMcb950ZKKaWq5c6ksBXoIiIdsZLBZOAnzgVEpA/wDjDGGJPhxliUUqpOa9Omptb32uO2pGCMKRaR2cCXgC/wnjFmr4g8ByQZY5ZjNReFAB+LCMAJY8wEd8WklFJ1Vd++fT1Sj1tvXjPGfA58XmHdM07Pb3Fn/Uoppa7MddHRrJRSqmYrV65k5coqL9CsVTrMhVJK1QHFxcUeqUfPFJRSStk0KSillLJpUlBKKWXTPgWllKoD2rdv75F66kVSuHTpEikpKRQUFHg7FKVqXVBQEG3btsXf39/boSgv6tWrl0fqqRdJISUlhdDQUKKjo3HcBKdUvWCMISsri5SUFDp27OjtcFQDUC/6FAoKCoiIiNCEoOodESEiIkLPghUrVqxgxYoVbq+nXiQFQBOCqrf0s608qd4kBaWUUtdOk8J15O233+bDDz8EYN68eZw6dcre9uCDD7Jv3z631Ltu3TrGjRvnln0D/Nd//Rfr16932/6v1dmzZxk1ahRdunRh1KhRnDt3rlKZtWvX0rt3b/sRFBTEJ598AsCMGTPo2LGjvW3nzp3269atW0fv3r3p0aMHN998MwBFRUUMHTrUY3eoKnUlNClcRx5++GGmT58OVE4Kf/vb3+jeveIU19e/rKwsNm/ezNChQ11+jae/LF988UVGjhzJDz/8wMiRI3nxxRcrlRk+fDg7d+5k586drFmzhsaNGzN69Gh7+8svv2xv7927NwDnz59n1qxZLF++nL179/Lxxx8DEBAQwMiRI1m0aJFnDlCpK1Avrj5y9vsVe9l36kKt7rN76zD+e3yParcnJyczZswY+vXrx/bt2+nRowcffvghjRs3ZvXq1Tz22GMUFxfTv39//vrXvxIYGMgTTzzB8uXL8fPzY/To0bzyyis8++yzhISEEB0dTVJSElOnTqVRo0Zs2rSJsWPH8sorr5CUlMSRI0d4+WVrkrp58+aRlJTEm2++yT/+8Q9ef/11ioqKGDhwIG+99Ra+vr7lYt26dStz5swhLy+PwMBAVq9eXW77d999x5w5cygoKKBRo0a8//77xMbGsnfvXu677z6KioooLS1l6dKltG7dmrvvvpuUlBRKSkp4+umnueeee8rtb+nSpYwZM8Zefu6551ixYgX5+fnccMMNvPPOO4gIw4YNo3fv3nz77bdMmTKF6dOn8/DDD3PixAkAXnvtNYYMGVJtfNfi008/Zd26dQDce++9DBs2jJdeeqna8kuWLGHs2LE0bty4xv3+85//ZNKkSfb15VFRUfa2O+64gyeffJKpU6deU+yq4YiJifFIPXqmUEsOHjzIrFmz2L9/P2FhYbz11lsUFBQwY8YMFi1axPfff09xcTF//etfycrKYtmyZezdu5fdu3fz1FNPldvXnXfeSWJiIvPnz2fnzp00atTI3vbjH/+YZcuW2cuLFi1i8uTJ7N+/n0WLFrFhwwZ27tyJr68v8+fPL7ffoqIi7rnnHv7yl7+wa9cuvv7663L7BujWrRvffPMNO3bs4LnnnuO3v/0tYDVtzZkzh507d5KUlETbtm354osvaN26Nbt27WLPnj3lvvzLbNiwgX79+tnLs2fPZuvWrezZs4f8/Hw+++yzcvElJSXx6KOPMmfOHH75y1+ydetWli5dyoMPPlhjfM5ycnLKNfU4P6pqgktPT6dVq1YAtGzZkvT09EplnC1cuJApU6aUW/e73/2OhIQEfvnLX9qTqx86dIhz584xbNgw+vXrZzcNAsTHx7N169Ya61HKWY8ePejRo/ofp7Wl3p0p1PSL3p3atWvHkCFDAJg2bRqvv/46o0aNomPHjnTt2hWwfoXOnTuX2bNnExQUxAMPPMC4ceOuqD0/MjKSmJgYNm/eTJcuXThw4ABDhgxh7ty5bNu2jf79+wOQn59f7pcpWImrVatWdpmwsLBK+8/Ozubee+/lhx9+QES4dOkSAIMHD+b5558nJSWFSZMm0aVLF3r27Mmjjz7Kb37zG8aNG8dNN91UaX9paWk4z6u9du1a/vjHP3Lx4kXOnj1Ljx49GD9+PEC5s4yvv/663Bf4hQsXyM3NrTY+Z6GhoeXa9a+EiNR4tU9aWhrff/89t956q73uhRdeoGXLlhQVFTFz5kxeeuklnnnmGYqLi9m2bRurV68mPz+fwYMHM2jQILp27Yqvry8BAQHk5OQQGhp6VbGqhqWsWdXPz71f2/UuKXhLxS+Smr5Y/Pz8+O6771i9ejVLlizhzTffZM2aNS7XNXnyZBYvXky3bt340Y9+hIhgjOHee+/lhRdeuOpjAHj66acZPnw4y5YtIzk5mWHDhgHwk5/8hIEDB/Lvf/+b2267jXfeeYcRI0awfft2Pv/8c5566ilGjhzJM888U25/jRo1sq+xLygoYNasWSQlJdGuXTueffbZctffBwcH289LS0vZvHkzQUFB5fY3e/bsKuNzlpOTU2WCAqtJp2LfTIsWLUhLS6NVq1akpaVVSqbOFi9ezI9+9KNydxeXnWUEBgZy33338corrwDQtm1bIiIiCA4OJjg4mKFDh7Jr1y77R0JhYWGl41OqOmVzKZT9iHIXbT6qJSdOnGDTpk2A9cVz4403EhsbS3JyMocPHwbgo48+4uabb7Z/8d522238+c9/ZteuXZX2FxoaSk5OTpV1/ehHP+LTTz9lwYIFTJ48GYCRI0eyZMkSMjKsqa7Pnj3L8ePHy70uNjaWtLQ0u9kiJyenUqdudna2PRfsvHnz7PVHjx4lJiaGX/ziF0ycOJHdu3dz6tQpGjduzLRp03j88cfZvn17pVjj4uLs4y9LAM2bNyc3N5clS5ZU+36OHj2aN954w14u++VfXXzOys4UqnpU1Vk/YcIEPvjgAwA++OADJk6cWG1cCxYsqNR0lJaWBlh3H3/yySfEx8cDMHHiRL799luKi4u5ePEiW7ZsIS4uDrA64Js3b65DV6jrjiaFWhIbG8vcuXOJi4vj3Llz/OxnPyMoKIj333+fu+66i549e+Lj48PDDz9MTk4O48aNIyEhgRtvvJFXX3210v5mzJjBww8/TO/evcnPzy+3rWnTpsTFxXH8+HEGDBgAQPfu3fnDH/7A6NGjSUhIYNSoUfaXVZmAgAAWLVrEz3/+c3r16sWoUaMq3Sn761//mieffJI+ffqUSxiLFy8mPj6e3r17s2fPHqZPn87333/PgAED6N27N7///e8r9Y0A3H777XYnbnh4OA899BDx8fHceuutdjNWVV5//XWSkpJISEige/fuvP322zXGdy2eeOIJvvrqK7p06cLXX3/NE088AUBSUpLdlwHWBQUnT560Ly0tM3XqVHr27EnPnj05c+aM/T7ExcUxZswYEhISGDBgAA8++KCdMNauXcvtt99eK/ErVZvEGOPtGK5IYmKiSUpKKrdu//799i8wb0hOTmbcuHHs2bPHazFcz2688UY+++wzwsPDvR3KdWPSpEm8+OKLdlPS5Xj7M668r2yIi6ttPhKRbcaYxMuV0zMF5XZ/+tOf7EtLlXWV1R133OFyQlDKk7SjuRZER0frWUINBg4c6O0QrisBAQH2TYpKucpTPyI0KSilVB1wrTdpukqbj5RSqg4oKCjwyBDqmhSUUqoO+Oqrr/jqq6/cXo8mBaWUUjZNCnXE+fPneeutt+zlU6dOceedd7qtvhkzZtR4c9m1yM/P5+abb6akpMQt+68NX3zxBbGxsXTu3LnKUVMBjh8/zsiRI0lISGDYsGGkpKQANQ+zvWbNGvr27Ut8fDz33nuvfa/FZ599VulucKW8QZNCHVExKbRu3dptX9ru9t577zFp0qRKI7hWxxhDaWmpm6P6j5KSEh555BFWrlzJvn37WLBgQZUD6T322GNMnz6d3bt388wzz/Dkk08C1Q+zXVpayr333svChQvZs2cPHTp0sO+kvv3221mxYgUXL1702HEqVZX6lxRWPgHv3167j5VP1FhlcnIycXFxPPTQQ/To0YPRo0fbdyEfOXLEHlb7pptu4sCBA/b6QYMG0bNnT5566ilCQkIAyM3NZeTIkfTt25eePXvy6aefAtZdt0eOHKF37948/vjjJCcn23fHDho0iL1799rxDBs2jKSkJPLy8rj//vsZMGAAffr0sfdV0UsvvUTPnj3p1auXfTevs+eee47+/fsTHx/PzJkzKbvh8fXXX6d79+4kJCTYw2383//9n/0LuU+fPlUO1TF//nx7KInqjjc5OZnY2FimT59OfHw8J0+eZNWqVQwePJi+ffty1113kZubW2N8V+u7776jc+fOxMTEEBAQwOTJk6t87/bt28eIESMAKxFUVcZ5mO2srCwCAgLsSwtHjRrF0qVLAezhw51HjVXKG+pfUvCSH374gUceeYS9e/cSHh5u/2efOXMmb7zxBtu2beOVV15h1qxZAMyZM4c5c+bw/fff07ZtW3s/QUFBLFu2jO3bt7N27VoeffRRjDG8+OKLdOrUiZ07d9pzKZS55557WLx4MWCNw5OWlkZiYiLPP/88I0aM4LvvvmPt2rU8/vjj5OXllXvtypUr+fTTT9myZQu7du3i17/+daVjq2646xdffJEdO3awe/duexiKV155hblz57Jz506++eabSkNzFxUVcfToUaKjo2s83rL3dNasWezdu5fg4GD+8Ic/8PXXX7N9+3YSExPt4UFqGo67zPz586scSruqJrjU1FTatWtnL7dt25bU1NRK5Xr16sW//vUvAJYtW0ZOTg5ZWVnlyjgPs928eXOKi4spuyN/yZIlnDx50i6bmJjIN998U6kepcAaysYTE23Vv/sUxlbd/utuZdMxAvTr14/k5GRyc3PZuHEjd911l12ubKz9TZs22e3MP/nJT3jssccAq6nkt7/9LevXr8fHx4fU1NTLju9/9913M3r0aH7/+9+zePFi+4tu1apVLF++3B61s6CggBMnTpQbLuHrr7/mvvvusyeMadasWaX9VzfcdUJCAlOnTuWOO+7gjjvuAGDIkCH86le/YurUqUyaNKlcwgM4c+ZMueEuajreDh06MGjQIAA2b97Mvn377OHJi4qKGDx4cI3xOZs6dWqtT2jzyiuvMHv2bObNm8fQoUNp06ZNuSaxisNsiwgLFy6051wYPXp0ufJRUVHlZttTylmnTp08Uo9bk4KIjAH+AvgCfzPGvFhheyDwIdAPyALuMcYkuzMmdwkMDLSf+/r6kp+fT2lpKeHh4Vc0tv/8+fPJzMxk27Zt+Pv7Ex0dfdlrk9u0aUNERAS7d+9m0aJF9q92YwxLly69ppteahru+t///jfr169nxYoVPP/883z//fc88cQT3H777Xz++ecMGTKEL7/8km7dutn7cx5K+3LH6zyUtjGGUaNGsWDBApfjczZ//vxKZ1gAnTt3rtQ306ZNm3K/4FNSUuyRWZ21bt3aPlPIzc1l6dKl5RJeVcNsDx482D4bWLVqFYcOHSp3LBXPrJQqU9ZcWtbU7C5uaz4SEV9gLjAW6A5MEZGK5z4PAOeMMZ2BPwPVz4FYB4WFhdGxY0d7bl5jjD1M9qBBg+wmpoULF9qvyc7OJioqCn9/f9auXWsPf13TUNpgNSH98Y9/JDs7m4SEBABuvfVW3njjDbs5ZseOHZVeN2rUKN5//327g/Ps2bPltlc33HVpaSknT55k+PDhvPTSS2RnZ5Obm8uRI0fo2bMnv/nNb+jfv7/dh1KmadOmlJSU2Put7ngrGjRoEBs2bLCH4c7Ly+PQoUMuD8c9derUKofSrqp8//79+eGHHzh27BhFRUUsXLiQCRMmVCp35swZuwP8hRde4P777y+3vaphtsuGNi8sLOSll17i4YcftrcdOnTI7idSqqK1a9eydu1at9fjzj6FAcBhY8xRY0wRsBCoOFD9ROADx/MlwEipaXaaOmj+/Pn8/e9/p1evXvTo0cPujHzttdd49dVXSUhI4PDhwzRp0gSwvrySkpLo2bMnH374of0rOyIigiFDhhAfH8/jjz9eqZ4777yThQsXcvfdd9vrnn76aS5dukRCQgI9evTg6aefrvS6MWPGMGHCBBITE+ndu7fd1FSmuuGuS0pKmDZtGj179qRPnz784he/IDw8nNdee434+HgSEhLw9/dn7NixleocPXo03377bY3HW1FkZCTz5s1jypQpJCQkMHjwYA4cOHBFw3G7ys/PjzfffJNbb72VuLg47r77bnsaxGeeeYbly5cDsG7dOmJjY+natSvp6en87ne/s/dR3TDbL7/8MnFxcSQkJDB+/Hi7oxp0OG11nTDGuOUB3InVZFS2/FPgzQpl9gBtnZaPAM2r2NdMIAlIat++valo3759ldZd7/Ly8kxpaakxxpgFCxaYCRMmeDkiz9m2bZuZNm2at8O4rpw+fdqMGDGi2u118TOuri9AknHhu7tOdDQbY94F3gVrPgUvh1Mrtm3bxuzZszHGEB4eznvvveftkDymb9++DB8+nJKSEpfvVajvTpw4wZ/+9Cdvh6GUW5NCKtDOabmtY11VZVJExA9ogtXhXO/ddNNNVU7D2VBUbH9v6Gqj2Uup2uDOPoWtQBcR6SgiAcBkYHmFMsuBex3P7wTWOE5zrthVvkyp655+tpUnuS0pGGOKgdnAl8B+YLExZq+IPCciZZdy/B2IEJHDwK+Amm8drkZQUBBZWVn6n0fVO8YYsrKyCAoK8nYoqoGoF3M0X7p0iZSUFI+MNa6UpwUFBdG2bdty9zsodaVcnaO5TnQ0X46/vz8dO3b0dhhKKVXn6dhHSimlbJoUlFJK2TQpKKWUstW5jmYRyQSqHiDn8poDZ2oxnLpAj7lh0GNuGK7lmDsYYyIvV6jOJYVrISJJrvS+1yd6zA2DHnPD4Ilj1uYjpZRSNk0KSimlbA0tKbzr7QC8QI+5YdBjbhjcfswNqk9BKaVUzRramYJSSqkaaFJQSillq5dJQUTGiMhBETksIpVGXhWRQBFZ5Ni+RUSiPR9l7XLhmH8lIvtEZLeIrBaRDt6IszZd7pidyv1YRIyI1PnLF105ZhG52/FvvVdE/unpGGubC5/t9iKyVkR2OD7ft3kjztoiIu+JSIaI7Klmu4jI6473Y7eI9K3VAFyZnq0uPQBfrGk9Y4AAYBfQvUKZWcDbjueTgUXejtsDxzwcaOx4/rOGcMyOcqHAemAzkOjtuD3w79wF2AE0dSxHeTtuDxzzu8DPHM+7A8nejvsaj3ko0BfYU83224CVgACDgC21WX99PFMYABw2xhw1xhQBC4GJFcpMBD5wPF8CjBQR8WCMte2yx2yMWWuMuehY3Iw1E15d5sq/M8D/AC8B9WFcdVeO+SFgrjHmHIAxJsPDMdY2V47ZAGGO502AUx6Mr9YZY9YDZ2soMhH40Fg2A+Ei0qq26q+PSaENcNJpOcWxrsoyxpoMKBuI8Eh07uHKMTt7AOuXRl122WN2nFa3M8b825OBuZEr/85dga4iskFENovIGI9F5x6uHPOzwDQRSQE+B37umdC85kr/v1+RejGfgnKdiEwDEoGbvR2LO4mID/AqMMPLoXiaH1YT0jCss8H1ItLTGHPeq1G51xRgnjHmTyIyGPhIROKNMaXeDqwuqo9nCqlAO6flto51VZYRET+sU84sj0TnHq4cMyJyC/A7YIIxptBDsbnL5Y45FIgH1olIMlbb6/I63tnsyr9zCrDcGHPJGHMMOISVJOoqV475AWAxgDFmExCENXBcfeXS//erVR+Twlagi4h0FJEArI7k5RXKLAfudTy/E1hjHD04ddRlj1lE+gDvYCWEut7ODJc5ZmNMtjGmuTEm2hgTjdWPMsEYk1T17uoEVz7bn2CdJSAizbGak456Msha5soxnwBGAohIHFZSyPRolJ61HJjuuAppEJBtjEmrrZ3Xu+YjY0yxiMwGvsS6cuE9Y8xeEXkOSDLGLAf+jnWKeRirQ2ey9yK+di4e88tACPCxo0/9hDFmgteCvkYuHnO94uIxfwmMFpF9QAnwuDGmzp4Fu3jMjwL/KyK/xOp0nlGXf+SJyAKsxN7c0U/y34A/gDHmbax+k9uAw8BF4L5arb8Ov3dKKaVqWX1sPlJKKXWVNCkopZSyaVJQSill06SglFLKpklBKaWUTZOCUh4kItFlo1+KyDAR+czbMSnlTJOCUi5w3Cik/19UvacfcqWq4fhVf1BEPgT2AD8VkU0isl1EPhaREEe5/iKyUUR2ich3IhLqeO03jrLbReQG7x6NUq6pd3c0K1XLumANiXIY+BdwizEmT0R+A/xKRF4EFgH3GGO2ikgYkA9kAKOMMQUi0gVYgDUQoVLXNU0KStXsuDFms4iMw5rAZYNjmJAAYBMQC6QZY7YCGGMuAIhIMPCmiPTGGm6iqzeCV+pKaVJQqmZ5jr8CfGWMmeK8UUR6VvO6XwLpQC+sZtr6MMmPagC0T0Ep12wGhohIZ7DOBESkK3AQaCUi/R3rQ52GY09zjOn/U6zB3JS67mlSUMoFxphMrAl7FojIbqymo26OKSLvAd4QkV3AV1hDN78F3OtY143/nHEodV3TUVKVUkrZ9ExBKaWUTZOCUkopmyYFpZRSNk0KSimlbJoUlFJK2TQpKKWUsmlSUEopZfv/ASJdY7VeBNwNAAAAAElFTkSuQmCC\n", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# Plot the PR curve for the test set\n", "plt.plot(recall_plus, precision_plus, label='positive class (area = %.3f)' % auprc_plus)\n", "plt.plot(recall_minus, precision_minus, label='negative class (area = %.3f)' % auprc_minus)\n", "\n", "plt.plot([0,1,1], [1,1,0], linestyle='--', color=(0.6, 0.6, 0.6))\n", "\n", "plt.xlim([-0.05, 1.05])\n", "plt.ylim([-0.05, 1.05])\n", "plt.xlabel('recall')\n", "plt.ylabel('precision')\n", "plt.legend(loc='lower left')\n", "plt.title('Precision-recall curve')\n", "\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": 23, "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQ4AAAEMCAYAAAA1eViuAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAADwdJREFUeJzt3XmwlYV5x/Hfj4u4AIIsilxAVIIgDgqCC2YsKnHfYl2wpkpVqNaYtE4zNY2daTox0WTaTqw6KeoE7YKYqq1LlLrUuBRFgqiIRmMcg2gVJAiyaLg+/eO+mCuBw3munPOeA9/PzB3Oec97z/tcDvPlfd97FkeEACCjS9kDAGg+hANAGuEAkEY4AKQRDgBphANAGuEAkEY46sj2RNszyp4D+LwIB4A0wgEgzTzlvPZsPyNpR0k9JPWR9Ovipr+KiNmlDQZ0EuGoI9sTJU2JiCklj4JOsH2ZpKnF1RMj4u0y5ylT17IHAJpFRNwg6Yay52gEnOMAkMahCoA09jgApBEOAGmEA0Aa4QCQRjgApBGOOrM9rewZ0Hk8fu0IR/3xD6+58fiJcADohIZ8AljvPn1jz0FDyh6jJlYsX6beffqVPUZN7dKtpewRambZ0qXq179/2WPUzCsvv7xm9erV3be0XkO+VmXPQUN0632Plj0GOmnsoN3KHgGdNHSvwSuqWY9DFQBphANAGuEAkEY4AKQRDgBphANAGuEAkEY4AKQRDgBphANAGuEAkEY4AKQRDgBphANAGuEAkEY4AKQRDgBphANAGuEAkEY4AKQRDgBphANAGuEAkEY4AKQRDgBphANAGuEAkEY4AKQRDgBphANAGuEAkEY4AKQRDgBphANAGuEAkEY4AKQRDgBphANAGuEAkEY4AKQRDgBphANAGuEAkEY4AKQRDgBphANAGuEAkEY4AKQRDgBphKOOZt58oyZPOlznfmmCrrr8Yn20bl3ZI6FKsx98UKNG7qcRw4fp+9deU/Y4pSMcdfLe/72tWT+erhn3PaqZD/2vPmlr00P33lX2WKhCW1ubvnb5Zbr3/gf0wsJFuv32mVq0aFHZY5Wq5uGwPdH2jFpvpxm0ta3XR+vWaf369Vq3dq367TGg7JFQhblz52rffYdpn332Ubdu3XTOOZN17z3/VfZYpWKPo052HzBQ5037qk47fLROGj9SPXruqsOOPLrssVCFt5cs0aDBgz+93to6SEuWLClxovI1TDhsT7M9z/a8FcuXlT3OVrfygxV6/L8f0N1PPqf75y7S2rVr9MBdd5Q9FtApNQuH7WdsL5B0s6RTbS8ovo7b1PoRMT0ixkXEuN59+tVqrNI8++RjGjh4iHbr209dd9hBRx1/sl78+dyyx0IVBra26q3Fiz+9vmTJW2ptbS1xovLVLBwRcWhEHCTpYkn3RMRBxdfsWm2zke0xcJAWPjdP69auUUTo2ace19Bhw8seC1UYP368fvnL1/TGG2/o448/1qxZt+vkU04te6xSdS17gO3FAWPG6egTT9X5Jx2llpYWDR81Wqf/0QVlj4UqdO3aVT+87nqddMJxamtr05Q/uVCjRo0qe6xSOSJquwF7oqQpETGl2u8ZOXpM3HrfozWbCbU1dtBuZY+AThq61+C331q8eIvHYTXf44iIxyQ9VuvtAKifhvmtCoDmQTgApBEOAGmEA0Aa4QCQRjgApBEOAGmEA0Aa4QCQRjgApBEOAGmEA0Aa4QCQRjgApBEOAGmEA0Aa4QCQRjgApBEOAGmEA0Aa4QCQRjgApBEOAGmEA0Aa4QCQRjgApBEOAGmb/exY27tW+saIWLn1xwHQDCp96PRLkkKSOyzbcD0kDanhXAAa2GbDERGD6zkIgOZR1TkO25Nt/3VxeZDtg2s7FoBGtsVw2L5e0lGS/rhYtEbSj2o5FIDGVukcxwYTImKs7eckKSKW2+5W47kANLBqDlV+a7uL2k+IynZfSZ/UdCoADa2acNwg6U5J/W1/W9KTkq6t6VQAGtoWD1Ui4jbbP5c0qVh0VkQsrO1YABpZNec4JKlF0m/VfrjCs02B7Vw1v1X5lqSZkgZKGiTp321/s9aDAWhc1exxnC9pTESskSTbV0t6TtL3ajkYgMZVzWHHO/psYLoWywBspyq9yO0f1X5OY7mkl2zPLq4fK+nZ+owHoBFVOlTZ8JuTlyTd32H507UbB0AzqPQit1vqOQiA5rHFk6O295V0taT9Je20YXlEDK/hXAAaWDUnR2dI+rHa34fjBEl3SJpVw5kANLhqwrFLRMyWpIh4PSKuUntAAGynqnkex0fFi9xet32JpCWSetZ2LACNrJpw/IWk7pK+pvZzHb0kXVjLoQA0tmpe5PZMcXGVfvdmPgC2Y5WeAHa3ivfg2JSIOKMmEwFoeJX2OK6v2xQb2aGLtWf3nba8IhrStTf9tOwR0ElL3l1R1XqVngD2yFabBsA2hffWAJBGOACkVR0O2zvWchAAzaOadwA7xPaLkl4rrh9o+59qPhmAhlXNHsd1kk6W9L4kRcTzav+AJgDbqWrC0SUi3txoWVsthgHQHKp5yvli24dICtstki6X9GptxwLQyKrZ47hU0hWShkh6V9JhxTIA26lqXqvynqTJdZgFQJOo5h3AbtImXrMSEdNqMhGAhlfNOY6HO1zeSdKXJS2uzTgAmkE1hyqfeZtA2/+i9g+eBrCd6sxTzveWtMfWHgRA86jmHMdv9LtzHF3U/gFNV9ZyKACNrWI4bFvSgWp/n1FJ+iQiNvvmPgC2DxUPVYpI/DQi2oovogGgqnMcC2yPqfkkAJpGpfcc7RoR6yWNkfSs7dclrVb7BzNFRIyt04wAGkylcxxzJY2VdGqdZgHQJCqFw1L7p7fVaRYATaJSOPrbvmJzN0bEP9RgHgBNoFI4WiT1ULHnAQAbVArHOxHxd3WbBEDTqPTrWPY0AGxSpXAcU7cpADSVzYYjIpbXcxAAzYMPZAKQRjgApBEOAGmEA0Aa4QCQRjgApBEOAGmEA0Aa4QCQRjgApBEOAGmEA0Aa4QCQRjgApBEOAGmEA0Aa4QCQRjgApBEOAGmEA0Aa4QCQRjgApBEOAGmEA0Bapc+OxVY2YcwIde/RUy0tXdTS0lX3P/JU2SNhM1YsX6o7Z/y9Ply5QrY17ovHa8Ixp2nN6lWaddM1WvH+e+rdd3dNnnqldu7es+xx645w1Nms/3xAffr2K3sMbEFLS4tOOPNiDRwyTB+tW6Mbv/t1DRs5RvPnPKx9RhyoPzj+bP3swTv0+Oyf6LgzLix73LrjUAXYhJ69+mjgkGGSpB132kX9BwzWyhXv65UXntbYwydJksYePkkvP/90mWOWhnDUkW195cxTdOLRE/Rvt95S9jio0m+Wvat3Fv9Kg/beTx+uXKGevfpIknrsups+XLmi5OnK0TCHKranSZomSa2DBpc8TW3cef/DGrBnq5YtfU/nnXmKhn1hPx064Ytlj4UKPlq3VjOnX60Tz56qnXbe5TO32ZZc0mAla5g9joiYHhHjImLctnoOYMCerZKkfv1313EnnqIF8+eVPBEqaWtbr5nTv6sDDzlKo8YcIUnqsWtvrfpguSRp1QfL1aNn7zJHLE3dwmH7MtsLiq+B9dpuo1izerU+XLXq08tPPPaI9hu5f8lTYXMiQnff9kP1HzBYR0z68qfLR4w+VPPnPCxJmj/nYY0YfVhZI5aqbocqEXGDpBvqtb1Gs3Tpe5p2wWRJ0vr163X6H56ticccW/JU2Jw3X1+kBc88qj1ah+r673xVkvSl0y7Qkcedpdtvukbzn3pIvfr21+Sp3yx50nI4Isqe4feMPmhs8ByH5nXbf/xP2SOgk666/Jy34+NVrVtar2HOcQBoHoQDQBrhAJBGOACkEQ4AaYQDQBrhAJBGOACkEQ4AaYQDQBrhAJBGOACkEQ4AaYQDQBrhAJBGOACkEQ4AaYQDQBrhAJBGOACkEQ4AaYQDQBrhAJBGOACkEQ4AaYQDQBrhAJBGOACkEQ4AaYQDQBrhAJBGOACkEQ4AaYQDQBrhAJBGOACkEQ4AaYQDQBrhAJBGOACkEQ4AaYQDQBrhAJBGOACkEQ4AaYQDQBrhAJBGOACkEQ4AaYQDQBrhAJDmiCh7ht9je6mkN8ueo0b6SVpW9hDotG398dsrIvpvaaWGDMe2zPa8iBhX9hzoHB6/dhyqAEgjHADSCEf9Td+ad2a7zfYC2wtt/8T2Lp/jvibavq+4fKrtKyus29v2n3ViG39r+y+rXb7ROjNsn5nY1lDbC7MzbsFWffyaFeGos4jY2v/w1kbEQRFxgKSPJV3S8Ua3Sz/OEXFPRFxTYZXektLhaHY1ePyaEuHYtjwhaVjxP+0vbN8maaGkwbaPtT3H9vxiz6SHJNk+3vYrtudLOmPDHdmeYvv64vIetu+2/XzxNUHSNZL2LfZ2flCs9w3bz9p+wfa3O9zXt2y/avtJSftt6YewPbW4n+dt37nRXtQk2/OK+zu5WL/F9g86bPtPP+9fJCojHNsI210lnSDpxWLRFyTdGBGjJK2WdJWkSRExVtI8SVfY3knSTZJOkXSwpAGbufvrJP0sIg6UNFbSS5KulPR6sbfzDdvHFts8RNJBkg62faTtgyVNLpadKGl8FT/OXRExvtjey5Iu6nDb0GIbJ0n6UfEzXCTpg4gYX9z/VNt7V7EddFLXsgfA57az7QXF5Sck3SJpoKQ3I+LpYvlhkvaX9JRtSeomaY6kEZLeiIjXJMn2v0qatoltHC3pfEmKiDZJH9jebaN1ji2+niuu91B7SHpKujsi1hTbuKeKn+kA299R++FQD0mzO9x2R0R8Iuk1278qfoZjJY3ucP6jV7HtV6vYFjqBcDS/tRFxUMcFRRxWd1wk6aGIOHej9T7zfZ+TJX0vIv55o238eSfua4ak0yPiedtTJE3scNvGTzyKYtuXR0THwMj20E5sG1XgUGX78LSkI2wPkyTb3W0Pl/SKpKG29y3WO3cz3/+IpEuL722x3UvSKrXvTWwwW9KFHc6dtNreXdLjkk63vbPtnmo/LNqSnpLesb2DpPM2uu0s212KmfeR9Iti25cW68v2cNvdq9gOOok9ju1ARCwt/ueeaXvHYvFVEfGq7WmS7re9Ru2HOj03cRdflzTd9kWS2iRdGhFzbD9V/LrzgeI8x0hJc4o9ng8lfSUi5tueJel5Se9JeraKkf9G0jOSlhZ/dpzp15LmStpV0iURsc72zWo/9zHf7RtfKun06v520Bk85RxAGocqANIIB4A0wgEgjXAASCMcANIIB4A0wgEgjXAASPt/HQBcfRU9jKoAAAAASUVORK5CYII=\n", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# Plot the confusion matrix for the test set\n", "plt.matshow(cm, interpolation='nearest', cmap=plt.cm.Blues, alpha=0.5)\n", "plt.xticks(np.arange(2), ('+', '-'))\n", "plt.yticks(np.arange(2), ('+', '-'))\n", "\n", "for i in range(2):\n", " for j in range(2):\n", " plt.text(j, i, cm[i, j], ha='center', va='center')\n", "\n", "plt.xlabel('Predicted label')\n", "plt.ylabel('True label')\n", "\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## STEP 4: Prediction" ] }, { "cell_type": "code", "execution_count": 31, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
TTAGGGATAGGGCTAGGGGTAGGGTAAGGGTCAGGGTGAGGGTTCGGGTTGGGGTTTGGGTTAAGGTTACGGTTATGGTTAGAGTTAGCGTTAGTGTTAGGATTAGGCTTAGGTrel_TL
098.6210.0020.0290.1840.0000.8170.1300.0790.0650.0000.0130.0070.0100.0080.0000.0070.0160.0090.0032.00
194.3000.0330.4480.6510.3380.4621.0520.6780.6280.1100.2630.0610.1200.1910.0530.0560.3280.0460.182-1.01
298.6660.0020.0730.1630.0000.0760.3340.4150.0750.0000.0230.0100.0140.0530.0000.0090.0650.0130.0070.81
397.3840.0080.2750.4250.1860.2770.5480.1560.1850.0120.1130.0340.1100.0950.0000.0360.0810.0360.0370.00
496.5250.0260.2040.2300.2090.4170.9340.1970.3970.1680.1930.0260.0700.0930.0030.0340.1580.0500.066-1.04
596.1500.0670.2730.3070.2520.3551.0610.2750.4440.0780.1570.0200.0920.1080.0210.0220.2190.0330.067-0.13
\n", "
" ], "text/plain": [ " TTAGGG ATAGGG CTAGGG GTAGGG TAAGGG TCAGGG TGAGGG TTCGGG TTGGGG \\\n", "0 98.621 0.002 0.029 0.184 0.000 0.817 0.130 0.079 0.065 \n", "1 94.300 0.033 0.448 0.651 0.338 0.462 1.052 0.678 0.628 \n", "2 98.666 0.002 0.073 0.163 0.000 0.076 0.334 0.415 0.075 \n", "3 97.384 0.008 0.275 0.425 0.186 0.277 0.548 0.156 0.185 \n", "4 96.525 0.026 0.204 0.230 0.209 0.417 0.934 0.197 0.397 \n", "5 96.150 0.067 0.273 0.307 0.252 0.355 1.061 0.275 0.444 \n", "\n", " TTTGGG TTAAGG TTACGG TTATGG TTAGAG TTAGCG TTAGTG TTAGGA TTAGGC \\\n", "0 0.000 0.013 0.007 0.010 0.008 0.000 0.007 0.016 0.009 \n", "1 0.110 0.263 0.061 0.120 0.191 0.053 0.056 0.328 0.046 \n", "2 0.000 0.023 0.010 0.014 0.053 0.000 0.009 0.065 0.013 \n", "3 0.012 0.113 0.034 0.110 0.095 0.000 0.036 0.081 0.036 \n", "4 0.168 0.193 0.026 0.070 0.093 0.003 0.034 0.158 0.050 \n", "5 0.078 0.157 0.020 0.092 0.108 0.021 0.022 0.219 0.033 \n", "\n", " TTAGGT rel_TL \n", "0 0.003 2.00 \n", "1 0.182 -1.01 \n", "2 0.007 0.81 \n", "3 0.037 0.00 \n", "4 0.066 -1.04 \n", "5 0.067 -0.13 " ] }, "execution_count": 31, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Enter unlabeled data for prediction\n", "newdata = pd.read_csv(\"data/telomere_ALT/telomere_new.csv\", sep='\\t')\n", "newdata.head(6)" ] }, { "cell_type": "code", "execution_count": 32, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "array(['+', '-', '+', '-', '-', '-'], dtype=object)" ] }, "execution_count": 32, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# predict labels\n", "y_new = rf.predict(newdata)\n", "\n", "y_new" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.5" } }, "nbformat": 4, "nbformat_minor": 2 }