{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split, RandomizedSearchCV, cross_val_score\n",
    "from sklearn.naive_bayes import GaussianNB, MultinomialNB, BernoulliNB\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier, AdaBoostClassifier, GradientBoostingClassifier\n",
    "from sklearn.metrics import classification_report, accuracy_score, confusion_matrix\n",
    "import numpy as np\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Naive bayes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BernoulliNB()"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.datasets import load_iris\n",
    "iris = load_iris()\n",
    "X_train, X_test, y_train, y_test = train_test_split(iris.data,\n",
    "                                                    iris.target,\n",
    "                                                    test_size=0.2,\n",
    "                                                    random_state=0)\n",
    "\n",
    "gauss_clf = GaussianNB()\n",
    "multi_clf = MultinomialNB()\n",
    "bernl_clf = BernoulliNB()\n",
    "\n",
    "gauss_clf.fit(X_train, y_train)\n",
    "multi_clf.fit(X_train, y_train)\n",
    "bernl_clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred_gauss = gauss_clf.predict(X_test)\n",
    "y_pred_multi = multi_clf.predict(X_test)\n",
    "y_pred_bernl = bernl_clf.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00        11\n",
      "           1       0.93      1.00      0.96        13\n",
      "           2       1.00      0.83      0.91         6\n",
      "\n",
      "    accuracy                           0.97        30\n",
      "   macro avg       0.98      0.94      0.96        30\n",
      "weighted avg       0.97      0.97      0.97        30\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00        11\n",
      "           1       0.00      0.00      0.00        13\n",
      "           2       0.32      1.00      0.48         6\n",
      "\n",
      "    accuracy                           0.57        30\n",
      "   macro avg       0.44      0.67      0.49        30\n",
      "weighted avg       0.43      0.57      0.46        30\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.00      0.00      0.00        11\n",
      "           1       0.00      0.00      0.00        13\n",
      "           2       0.20      1.00      0.33         6\n",
      "\n",
      "    accuracy                           0.20        30\n",
      "   macro avg       0.07      0.33      0.11        30\n",
      "weighted avg       0.04      0.20      0.07        30\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_test, y_pred_gauss))\n",
    "print(classification_report(y_test, y_pred_multi))\n",
    "print(classification_report(y_test, y_pred_bernl))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import load_svmlight_file\n",
    "svc = SVC(kernel='rbf', random_state=101)\n",
    "X_train, y_train = load_svmlight_file('data_set/ijcnn1.bz2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SVC with rbf kernel -> cross validation accuracy: mean = 0.9625, std = 0.0185\n",
      "Wall time: 31.3 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "scores = cross_val_score(svc,\n",
    "                         X_train,\n",
    "                         y_train,\n",
    "                         cv=5,\n",
    "                         scoring='accuracy',\n",
    "                         n_jobs=-1)\n",
    "print(\n",
    "    'SVC with rbf kernel -> cross validation accuracy: mean = {:.4f}, std = {:.4f}'\n",
    "    .format(np.mean(scores), np.std(scores)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameters {'gamma': 0.1, 'C': 100}\n",
      "Cross validation accuracy: mean= 0.9625\n",
      "Wall time: 9min 7s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "svc_new = SVC(kernel='rbf', random_state=101)\n",
    "search_dict = {\n",
    "    'C': [0.01, 0.1, 1, 10, 100],\n",
    "    'gamma': [0.1, 0.01, 0.001, 0.0001]\n",
    "}\n",
    "search_func = RandomizedSearchCV(estimator=svc_new,\n",
    "                                 param_distributions=search_dict,\n",
    "                                 n_iter=10,\n",
    "                                 scoring='accuracy',\n",
    "                                 n_jobs=-1,\n",
    "                                 iid=True,\n",
    "                                 refit=True,\n",
    "                                 cv=5,\n",
    "                                 random_state=101)\n",
    "search_func.fit(X_train, y_train)\n",
    "print('Best parameters {}'.format(search_func.best_params_))\n",
    "print('Cross validation accuracy: mean= {:.4f}'.format(search_func.best_score_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SVC with rbf kernel -> cross validation accuracy: mean = 0.9625, std = 0.0185\n",
      "Wall time: 41 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "svc_best = SVC(C=100, gamma=0.1, kernel='rbf', random_state=101)\n",
    "svc_best.fit(X_train, y_train)\n",
    "print(\n",
    "    'SVC with rbf kernel -> cross validation accuracy: mean = {:.4f}, std = {:.4f}'\n",
    "    .format(np.mean(scores), np.std(scores)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## RandomForest ExtraTrees"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(581012, 54)"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.datasets import fetch_covtype\n",
    "covertype = fetch_covtype()\n",
    "covertype.data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "covertype_x = covertype.data\n",
    "covertype_y = covertype.target\n",
    "covertype_x_train, covertype_x_test_val, covertype_y_train, covertype_y_test_val = train_test_split(\n",
    "    covertype_x, covertype_y, test_size=0.4, random_state=42)\n",
    "covertype_x_test, covertype_x_val, covertype_y_test, covertype_y_val = train_test_split(\n",
    "    covertype_x_test_val, covertype_y_test_val, test_size=0.5, random_state=42)\n",
    "covertypes = [\n",
    "    'Spruce/Fir', 'Lodgepole Pine', 'Ponderosa Pine', 'Cottonwood/Willow',\n",
    "    'Aspen', 'Douglas-fir', 'Krummholz'\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(348607, 54)\n",
      "(116203, 54)\n",
      "(116202, 54)\n"
     ]
    }
   ],
   "source": [
    "print(covertype_x_train.shape)\n",
    "print(covertype_x_val.shape)\n",
    "print(covertype_x_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForestClassifier -> cross validation accurary: mean = 0.9431, std = 0.0007\n",
      "Wall time: 2min 41s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "rfc = RandomForestClassifier(n_estimators=100, random_state=101)\n",
    "scores = cross_val_score(rfc,\n",
    "                         covertype_x_train,\n",
    "                         covertype_y_train,\n",
    "                         cv=5,\n",
    "                         scoring='accuracy',\n",
    "                         n_jobs=-1)\n",
    "print(\n",
    "    'RandomForestClassifier -> cross validation accurary: mean = {:.4f}, std = {:.4f}'\n",
    "    .format(np.mean(scores), np.std(scores)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.94251456, 0.94254324, 0.94264282, 0.94429225, 0.94356076])"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ExtraTreesClassifier -> cross validation accurary: mean = 0.9426, std = 0.0008\n",
      "Wall time: 2min 49s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "etc = ExtraTreesClassifier(n_estimators=100, random_state=101)\n",
    "scores = cross_val_score(etc,\n",
    "                         covertype_x_train,\n",
    "                         covertype_y_train,\n",
    "                         cv=5,\n",
    "                         scoring='accuracy',\n",
    "                         n_jobs=-1)\n",
    "print(\n",
    "    'ExtraTreesClassifier -> cross validation accurary: mean = {:.4f}, std = {:.4f}'\n",
    "    .format(np.mean(scores), np.std(scores)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.94211296, 0.94245719, 0.94202608, 0.94426356, 0.94219819])"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CalibrationClassifierCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.calibration import CalibratedClassifierCV\n",
    "from sklearn.calibration import calibration_curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "rfc = RandomForestClassifier(n_estimators=100, random_state=101)\n",
    "calibration = CalibratedClassifierCV(rfc, method='sigmoid', cv=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "rfc.fit(covertype_x_train, covertype_y_train)\n",
    "calibration.fit(covertype_x_train, covertype_y_train)\n",
    "prob_raw = rfc.predict_proba(covertype_x_test)\n",
    "prob_cal = calibration.predict_proba(covertype_x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>raw</th>\n",
       "      <th>calibrated</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.001132</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.42</td>\n",
       "      <td>0.356083</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.001131</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.001130</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.001130</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>116197</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.001131</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>116198</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.001128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>116199</th>\n",
       "      <td>0.32</td>\n",
       "      <td>0.079379</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>116200</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.001134</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>116201</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.001130</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>116202 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         raw  calibrated\n",
       "0       0.00    0.001132\n",
       "1       0.42    0.356083\n",
       "2       0.00    0.001131\n",
       "3       0.00    0.001130\n",
       "4       0.00    0.001130\n",
       "...      ...         ...\n",
       "116197  0.00    0.001131\n",
       "116198  0.00    0.001128\n",
       "116199  0.32    0.079379\n",
       "116200  0.00    0.001134\n",
       "116201  0.00    0.001130\n",
       "\n",
       "[116202 rows x 2 columns]"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "tree_kind = covertypes.index('Ponderosa Pine')\n",
    "probs = pd.DataFrame(list(zip(prob_raw[:, tree_kind], prob_cal[:, tree_kind])),\n",
    "                     columns=['raw', 'calibrated'])\n",
    "probs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/d3fzzAAAACXBIWXMAAAsTAAALEwEAmpwYAABfNUlEQVR4nO29fXRc1ZUn+rvgGBKBlaTBaZsABpIwSSBxE5NGHejGzkcHJwwmTR5JzyR5nXTzmH4zq2Fm3iQmCkZOBjrpfkOm7ekxvSAvyfRMI5myVLb4UjBJkzjEskhkSxCTgGUkI0sqyVJVqWTJljjvj60999xT59x7Si591v6tdVdV3Tr33HNL9t5nf/12oJSCQCAQCCoXZ833AgQCgUAwvxBFIBAIBBUOUQQCgUBQ4RBFIBAIBBUOUQQCgUBQ4Vg23wsoFRdccIFas2bNfC9DIBAIFhVeeOGFQaXUhbbvFp0iWLNmDdra2uZ7GQKBQLCoEATBa67vxDUkEAgEFQ5RBAKBQFDhEEUgEAgEFY5ZUwRBEHwvCIKBIAg6Hd8HQRD8XRAErwRBcCgIgmtmay0CgUAgcGM2LYLvA/hkzPc3AXj39HEHgP8+i2sRCAQCL0xOAqOjwBtv0OvkJJ0fHwfyeTo/Nha+P3nS/j6fj16Tz9P35jh9rtFRoFAoPp/P02d93nJi1hSBUuo5ACdihtwC4IeK8AsAbw2CYNVsrUcgECxMnDplF5ZjY6FA1gWnLiwLhXCMLiz18/p4fR7XtadOAToX58QEnZ+aAoKgeP1TU8BZ05J0chJYNp2LqRTw1FPADTcAy5fT649+RPc5eRL4X/8LeOEFoLmZvnvb24Dvf5/uNzZGY/VrW1qAoSHgllto3lyufH+D+YwRXASgR/t8bPpcEYIguCMIgrYgCNoymcycLE4gqCS4dsEzud7cFU9M0Bh9d6yPOX06KmBZCAcBvR8dBZ5+OhSK119Pn4eG6PX660mI/uAHdI+xMeCZZ8LzLFyBqAB+/HEaw4KW59SF9/XXA08+Gd7r4x+na3/7W2BkhNY3PAy8+iqd/8QnaNzJk8CPfwzceitw8CApi4MHSYg/8QSt4cILgYsuAr78ZeCVV0jQX3ghfff448CmTdFrb70V2LcP2LyZ3j/7bPksg/lUBBbdCisntlLqH5RS65RS6y680FoPIRAIZohCgYQTC8Xrr6fPhYJbwOu77HweGBgA7rjD7rYYG6Pd6yuv0PejoyRsP/EJ4MCBUCDrwhyguQHgvPOAtWuBm28Gzj2XhOKvfgU8/zy9ptMkjD//eeDw4VCI6sJ1wwZ6tg0bgOuuA15/Hbj99qig/cxnSND+6lfR85/9bHh+40YS3ocP03rOOYdeDx+m8xs3kpCemAC2bLH/3g88AHzqU8Cf/AnNe/fddBw9Cnzuc8CnP01jbKirAz784fD96dNn+MdnKKVm7QCwBkCn47uHAHxe+/wygFVJc37oQx9SAoFgZpiYUCqXU2pqil7Hx5VqalKK9rbRI51WKp9XamiIXpualPrgB5U6+2x6bWhQ6he/UGr9eqU6O6PzZjJKTU4q1d2tVEeHUtls+JpO0/y1tUo99hi9Hj1K146M0P2mppQaHFRq2zalVqyg+9XX0/2qqmh8e7tSjz5K361YQWNzObpvV5dSLS00v/lcXV10je2Z165V6sgR9/lMhtZhu7ahQamBAXo/OUm/k23csmVKnT4dnVdfU9K1U1PR974A0KZcstr1RTmOBEXwKQBPgiyD6wC0+swpikAgmBlyOaV27SoWnHFCsbubBHdvLwlgc0wqpdSrr5JAvu46pVpb6ZyuMHSFksmE9zOFualgrrsuKvz5fi0tShUKoUKpqqIx5jxNTTS/uW5fIW07n8/H/175fGnKhufV15R07chI+D6X8//7z4siAPBPAI4DOA3y/38FwJ0A7pz+PgDw3wC8CqADwDqfeUURCCodLJCmpuj19Gk6r+/2R0ejY8bGlHryyejuO5ulVx+hyAK4qyvccdfWKvXQQ6FFUVenVGOjfa49e5R6/nlaC18/Pk7ja2vDeTMZpfr7aV39/fRdKkWvVVUk2JuaSEGxsKytJSVgu299PX1fLosg6feanKT327eHiipuTTaLoLbWbXU0NtLfgd+Pjfn/u5k3i2A2DlEEgkqELvyHhmg3v2oVvbIrhd0q+Ty5arq7Q/dMZycJ2cceo532L35BwkYXqC7hpwtgc8edzyt17BgJJ5d14bq+uZnWxNYEr0kfU19P17LiYYFfyg7aFOy1tWR1uAStqTj08/398ffq66P3u3fT36O+ns4vW0avpoXT1KTUwYOkOFj4s4VjXtvYSK6nDRvofTar1MmT/v+GRBEIBAsM5q6+UCje4TNGR2l3qQvIVIqEgukO2bVLqeFhEvg2oT0yotSJE3Svri6lnnmGduuunWtLS7hD5926bhE0NSn12mt038lJcjnpO3z21dt27LW1dF3Srj6dpt8nkyGFYQr/Ul09VVVKvf463dsU0sPDdD/9/M6d9FunUkrt2OGOqTQ10feNjfQ7t7fT0dNDaxwcJIFfXU3zssssmyUlms+H966uprFDQ3RtLhf+G8nlyBLIZkv7NyeKQCBYQLAJdt0vnk7TmImJcGycEPXZ1bLwM/33qRQJv1SqWCgODBRbEOZu/bXXQkWSzdp9/oOD9l20LsyTdvUcq+DdtO4+Sbp2aCj6bLoAHhggRcFWUz5PCqevLxTA7M4aHS0W2OacuRxZIO96l1I1NdEgeqEQDaiPjRW/HxtLHpPLlWYJMEQRCAQLBKdP+/mO0+kw+yZJiJ6pO2TnTtq18+7zyJFoxo25W2dB3NUVWgq1teTztykn145dP++7q29oCOMFHJjW3So2pdjSQs90+jS91tZSvIQFs35eDyxzMPb110nIsxW0ciW98py5HCmVyUm7cuFYDCsSFuajo34WYbkgikAgWCBIyjphAc67YFdwslR3yNGj8fft6gqzeOrryZ1h2627MnTq690xApfSKsUi0H8Xjlu0toauF05vNXfptqwhfZ6k4O/UFCmS3bvtY9htxYpgcJAsLLYmZrJzny2IIhAIFgh8s3T4vUtAJgl20yJIUhwsuPT6glWriq+N8+W7ns2VBVNbG2YZxWXK6JYSr3X7druVcuQIfT88TK61FSvif+tsNv53zGbJ9cX1Dy53EO/uczm670KEKAKBYIGgFIuA3RU2l87evW4Xky1GkCTwhoaiO/zGRnL5rFzpv3PXrQj94KwhU5BywHvnTvKn799PzxqXZcOZOUnPY6Zk2sZ0d5Pbx5Xy2thIAn5gIIwR2NJ2FwtEEQgEswBXPn/ceL0QSt/Jsr+dc+f37FHq8OHQvzw0RLvg1avpdXSUhJgtPXF4mK7Xz586Fe9Hd6VMtrREd+txlsW997qV0549lKHEQVhWcrq/nXfyeqzC9NuzdeDjGouzMjggPzhIv3FjY3Gq5siIUmvWhIojn5/lf1CzDFEEAkGZYcv8YeHiGt/ZGfVn8y7YzMbZvZuEE1cB6/OzqyKfp2wePWjJgnPDBtrF6ucHBkL/v5meODVFriZXsLSmhvzxu3a5d/0AjbO5UHbtoutrakLLI24nX1VFv4s5j24d+MQUXPn4qRStZ2oq/I3YxcPuncOHQyXAyqUUOoeFCFEEAkEZEZf509xMPmLdUpiYUOr48WJeHJ13Rz98UkN9Yw1scWQytDa2PmzcQaYbhoUfp0x2dtrTWfkeg4OhVcOZM319obXkE5zlddfU0L1c1oEeXzAPzizS18YCP5ulzzU1IVUDH0nKRSyCBXSIIhDMN1x+flcFbWenXWj5ZNPYBBIXhSVVuDKVxOAgFTq1tpKQNAOt+mHSH+RyYZEVoNTWrVGXVHU1PbNpvehKRY9z+Pj2WZgPDMTv6kdGil06cbxIOilcYyNZN/q1nZ1uBZ9OL76YgAlRBAJBGRGXHWMTsC7Bbvq5WWj67Jo5+Gobw2RruvXBu/UTJ5KJ5lgYM6WB7j6qq1OqrY3m7+4ma8dVactKRXf17NtHri+z+pg5hbgmoLeXlA5AcYSWFhL8zJHU0qLUZz5DVgOnbg4NRRWe6Vbav59+O36uBx6I1gKcPh1aPKZycbn8FhNEEQgEZYTLIvAV+Lbxen6+L/8PZ7y4dsSunH8fRVNfT+6kPXuKd/qtrSSkjxzxz4KqqSHh+9pr9vgHc+ccO0ZKZng4Po5SX0/PPz7uDsBzALqvL3QH5XKkRFaujD6z7v8vNQlgsUAUgUBQRrhiBK4dPu9Yeeer+7nZZaJbE3HZLiYLaC4XpjeOjNDOlfl4SrVQdNeTi8vfdB/5FrbxujOZePfLwEAY8B4ZiR/f1BRW6/ooo5077dlRS8H/7wNRBAJBmWFzIeicOnEVuOw7r6qi3Xs6Hc3GifOL24jmeDfd3Bzd7bsEfhzdBAt530rfpCyg7u7oun2sHVYirDzjxnOmT5IySqXIgrDFDpaC/98HoggEglkAuxA4I6alJdzJJ7FpDg0pdegQuUHGx6M8/TY+m3zezeIJhNlEejDWtVt3pWemUuT/37qVhKuNSbS2lgLE7GsfHaWddtwOX193KZ27uNtY0vgkZTQyQrGNQ4fsNBRLwf/vA1EEAsEsQfeRV1WRIO3p8ev81dREVoSL559z+2tq/CplWRlxhlLcePbZ62mVhw7RuYYGegYbkyivLZcLi8Ns1guzl6ZSNBfXPPhmDdXXUyDap+4gLpW0qYmUABC6p8wAsVJLNy6gI04RBPT94sG6detUW1vbfC9DIABAzd2feAK45RZg5UqgsxP42c+Af/kvqbH51FTxNcuWUSP4ujrgmmuoabqJ+npqCL9yJXDDDcDgIPC+9yXPedFFtIZ9+6hB+4UXUpN22/wdHcC3vhW+B8L1tLRQk/m4tV18MbB+PTWK/8hHgK98hc6NjACPPgrcfz9wzz3Av/k31KT+6FEgk6Fncq0pk6E1X3klsGoVsG0b8KEPUUN41zM8+CA979GjwH330fNfdRWweTNw/fVAczPw1rcCl14KfPSjQKFAv9fEBHDWWfR5717g3nvDa7dupbFVVcX3XawIguAFpdQ665cuDbFQD7EIBAsNHC/Yty8sBPPZvfsSxzU10a7bNyi6Zg1Vxrq48xsayDVUU+Ou1vVdWypFHbZ0t5OZscOduvh7mwXBlga7n/bsoZ28XpsRx0FUXU0WxOBg1I3FhWO2qmne+S/l2gEdENeQQDC74H7BeleuJDbNUvzlIyNENOfK2U+nQ8HJQept2yjzR4819PVRuqWNg38m/QHWrqVUT5trqLWV1mQqMLPaN5cLK4h7eshF1dZWLOSZ3tnMvtJjCuZ5VzN6FvI+zeiXCkQRCAQWlMsvHNdKMq5nra+/nIWZWdWrz9nbq9RLL0WrhkutXJ5JfwCd4E0PnG/dSorg0Uf9KLDN3661Nco4yimtqZR9Hr2pu7lWW4cyDhD79CNYKhBFIBAYKJU0zoU410IqFbpobHw5ExN+HPw6rbKLaI6LpXK5cMxc9Afgpja1tdGdv541VWo3NfMeelcyV1P34eFokZi+8ze5n3SFLxaBKAJBhaKcfuEkQTI4GPb8ZdoDZv7M5aKMoC7/dzpNhVU+O2vO4onL2Xf1B9i9m3beDQ3+/QG4OUt7u1IPPxy6rnThH6dUdII487fjGgQ9/9+MQWSzRG/9wgszo4aQGIEoAkGFopy7wFJZQJn5s7OThDv3/O3vD1sdbt9OyoJ3u9ksnfcJFmcy0VoGlwDevdvey7epyV25bK6toYGUwnXXhTGBtjYS3rrScu3k9Wb0LsVWV6fUgQPFQp75gtgKqKmh2MdMXH1LmV9IhygCgUDDmfiF9bgCNz8vxe3Bfn4zP//gwZCMzRTOe/bQjnvbtuSddX9/coVyUxMFZOvqiovFuGbBpMJmRWY2ljGbxtTV2esFbAHiwUGl1q+3F63xOljI6zUPAwP0nHojn+bmM9u9V3odwbwL9lIPUQSCM8VMLQLeObKrp74+Xjjrfm4WhEyJYGa+nEmjFX1n7eI70iuUJybCQK5Jf8EMnTpdhn74+vvjrBGugnYpRSaUY/ZRM0Zgo9jI5Wb5H80SgCgCgUDDTPzC+jV6IDRu193eHs8CqlcQ+zRatwl2JmZjojkfQa27j2yWxeioOx7hm1bqyv/Xf5e6uviK4B07okouibZjKe7iywlRBAKBgVL9wroVYQpbm3DWUx3jBBhbFbqAjNtp67GGyUlyTzFPP9cvuFIsOeic1NQml6M5WbnMxCIAiikdjhyJKrykubJZOnyV3FLK8JkNiCIQCCwoxS+sxxV8dsU6sVtcn1+zEYwtg8bmGtHdU7qVsXNnyPFjVhNzYNcnwM3ZQGYv46QWkffeGx8I1oW5z+9YVxdaYknjK6GnwJlAFIFAcIbQg8K+u2LO7ffNLLIVP+npk7W1YaVwEq//yIi736/v+tla0SkpUqnw3uY6T5wIG8Lr7ir90Cmwk9bR1xctoktSqGwRlKtGZKlBFIFAkACmiOBm7RMT0e9PnSJhx5k5toAvC0/e1fMOX8/kiRO8erN4TtfcsYPSJxsawu5dPrz+Q0NR1k89K6elJUodrbub9EwcPXNHbxqzejX9Pn19xUVtfX2hwLfFIbizmt6W0mVdDAxEi+gGBpJjO5VUF1AqRBEIBDHI5eytE/VMlHyedrnmOH23zLntjzwS7fObzbo5+80q3eHhkBKa+wJzFs+pU/68/lNTdNiC1Ex/kU7Ht4LkDCJTYXGxl1lcplM+u4LFDQ2k2HbsoN80n7cTyvG9fesReLdfSZXCpWLeFAGATwJ4GcArAL5m+b4awB4ABwG8CODPkuYURSAoJyYmovny+tHYGFoGceP03XJzMxWL6UK1uTmZdwggZTEyEvq2T5wgAccN23X3VJKVkc3GZwc1N9POnN0ocbty04WlK6/6etrZm/xAQHGweGgofLZcjpReHGWGrQeDrR5B3+VXEndQqZgXRQDgbACvArgcwPJpYf8+Y8w9AL49/f5CACcALI+bVxSBoJxIKghjqyBpp9nVFb63ceewUOROZGaVLjdyaW4utjhsQd5Mxt1usqGBhKyPSyppDLt6XFYA/0YdHSS89V368ePEmJrNhsJ/3z6lVq2ie7LF4LKUtm+nQrs4xlXT1SMWgRvzpQhqADytfd4MYLMxZjOAvwcQALhs2nI4K25eUQSCcsJ3B5k0bnIyfG+jPdZ7ENh2wXFpn7wD13sExHUGs7lVbGv2HWNaPSbN8+Qkrd+MsWSzdpfbwIBSDzxAz2Prl8AZS/l8SBvhm+orMQI35ksR3AbgYe3zFwBsN8acD+DHAI4DGAXwKcdcdwBoA9B2ySWXzOZvJagwxFkENTUkbFgoJfXFjbMIWGDu2GHf4fq0ldSD1JlMSM9gc6v09fnRXCdl7nBLzbgah+Hh4jhEOk3U2OY1VVV0nhXEihWUmcQNZVypnq50UNv5SuEOKhXzpQg+a1EE24wxtwF4cNoieBeALgAr4uYVi0BQTkxM2PPiWWBxGuI3vuF2xdTXUwN6M+vGbJwyNEQCvbe3eCfvsjj0egRd0O7aFWXl1A8W3D7cRHFUEHrzeZe1kkrRveLuoZ9LYiItFPz/dnFpolJHUIyF7Bp6HMAN2udnAXw4bl5RBIJyQRcWuVxI9dzTQ0J2z56oQHa5YjIZGmtm3XCHLlYQfA/bTt7lqzcFp5nqOTISjTXoCsK1Zr02YeVKEvaNjW52z7hnP3HCXi/gso7KVR0sLqDSMV+KYBmAI9O+fw4Wv98Y898B3Df9/h0AXgdwQdy8oggE5YCNQM7cWZruEBbC3d1hoVZLCwV4zVx9LoRKpYrntVXcunbKuuBM4iwaGYm2y6ytJUFusobW1ZGS4oCs2c6SXUxcN6E/O4/JZktrt8lHKdXBcZCgcOmYz/TRjQB+M5099PXpc3cCuHP6/WoALQA6AHQC+NdJc4oiEJwpXARyNpePjfKBBdy+fWGAdHCQXDGrVtGrzv+jC2SX+8iVd6+7jFxr5SKtxx4rtkpY6eguJrM5Tqndw9JpaovJ2VJJWUe6gkzKUvIV4JImWjqkoExQEfD1C8cRyPkIwpoae0YMF2rZaJUHBuIFNfvbW1pIiXCjeR9qC9+2khs2hFYDu6pOnYpnPZ2ctGf0cDbPqVPxLpqOjugzd3SUx6UjFkHpEEUgWPIohV+mVAI583xnpz3AbDZz0Q8XoVx9PVkWXImczdKueffuYiHvWqvPrpxTNZuait1KcRlB/f3FzWQmJ8k91tlJv68rS8cWzGbrZdeuM8vqkRhB6RBFIFjSSBIKZvPyfD7M5U9KsRwcLA6Quq6ZiXXB2UR6KiUrhaEhpR56iPz5DQ0UaLbN79PLeNs2twvMpaR0ygiex7Roenvp9ze5mgoFdxCZ03KHh2eWMsqQNNHSIIpAsKSR5CZgQcsC7KWXSIA9+mh8iuXOnfa+vmfatMV2PikQPDREAs6WwpqkgEZGkn3zNoWndwmLixeMjtLYnp7QWvBxOcXt2n0tPEkT9YcoAsGShi/NMx91dWFR10waq7sE70wtAm5k89hj9mvZz++qJk6iati2za8yWm9nabKP2moCeHd/4kRxvCTJ5RTnwxe3z+xAFIFgSSPJIkjKZbcRmY2Oul0b27dHBRVfH0dPHRcj8O1Q1t9vp6fYupXYSk03iR6M9lVSJi02C2WXUrz3XrfQjmu0E1c4JoHg2YEoAsGSRtwOsr6edtt6CqNPLvupU26XUSpFwqix0U3jbNJTj4xQvUEc+2hcuurkpNs1BNDchULIxsluLE5bjWtkY7KJ6rTYuRzFRLgITv8da2uTm8WYjXb0uIILkho6OxBFIFjycAUOM5nitM3BwWTG0Xyedui2atr2dqVeey1kE41zYzBRW1MTrTGbjQpqk4bC5T7ibB2ba0hnKNV377bWlg0NxdeywjIVU2OjUocP0294/HjYH4HjAO3tfi4n7qmgZxrFQSyC2YEoAkFFwAwcjo/bhXRc391UKiwUe/JJEvZMo5zN0uempnDXXIrLhat7feIZtq5humtIrxQ2FQorwJaWaKaQ7gKbnCTfPtNqmLTYOsXE/feHwXUzaygp64qztEoJ5kqMYHYgikCwZGEK/0IhmiZqE1JcxWvru8sFYevX03sXjfLKlTSXr2Dn90mKY2jI7W7iOICPAuruTrZ8WEnZYiT79lFLSraQXG6ygwdnR2hLamj5IYpAsCRhSzF0NXIxj+pqigPoPnV9Bx1HPdHURHnzPpQJZhA2rgq4sZHW4OrNq/f7LWevAdP68FWoQFhlbZLWlUNoS2poeSGKQLDkkBQgZsGW5LbQlYU+3sflY+7SbQqDC7I4/dKVrtrURHPV1ZFg1QOyutunpoYKsXw4gpKU1PCw2/rQBblPHMAs2hOhvfAgikCw5JC0Sx0YiM+U4YpjXaDqO2jf3fTWrfb+Apwd8/DDYdvGrVvDrJtMJho4npigCmKbH14P4OqVwkn8QkntLPN5ch8luXYkeLs0IIpAsOTg08jFlSmTTpMCaG6OCtRSLQIeF9eAPZcjv7+NpM7sQexDHMf3jusRwIojqZ3l5KRfsFeCt0sDoggESw46X5Ce22520zIzZfJ52n03N4ffs7Dcvj3k4PfNuz8Tnh9d2fjGGvQKX342VwZRkpIaGfGrqeA4gQRvFzdEEQiWHE6ftqc0+lAbmK4OFqhHj4Y7d84aMoOgpqvGJw4RRx/NrhsfV5SrwtfsNcBr7eyMV2Yc8PbhadIJ8eKI4gQLF6IIBEsOp0/H8+u4KnSnptxuJT1TqLY22rmLc+3N4G1cTUJDAykTl5DXBXhShS5TQCcxeurFWy+9RNe4iuI4GK2v31a/YKtTECWw+BCnCM6CQLAIMT4ObNli/+6BB4Avf7n4/FVXAWNjdFx1VfH3X/kKXcvvv/lN4FvfAi6/HKiuBl55Bbj6auDd7waWLQPWrgWuuw648Uagvp4+8/n6euAjHwF27AB6euz3KxSAu+4CamqAc84Btm61P8+WLcB3vgN897vA3Xfbx3z1q8Cb3hQ997u/C+zfD3R0ALt2ASdP0mtHB/Dqq8DjjwPr1oXrr6kBnn2WnnHTJmD5cuATn6DPe/cCVVXhesbH7esQLFK4NMRCPcQiEChVOuMo72QLBXKJ2FwmSVlDerxhaop885kMuUk6O4upFA4dCq9JakDPhHVDQ9EKX93VFdeM/tAhWofeE2BsjCwCW45/Pk+usEwm5BFy1S+YVpbw/SxOQFxDgqWGpJRGk19fLzRrbiZBafrU2R/u4/vP5cIsIPafMx9QT49Sx47ZA9J63j5nNdly+Dm2MDER5fVfuTJKe8FN6/n92Fj4/tQppa64Imx/yUonmyUF0NgYVXil0GVIyujigygCwZKDD+Mo+/b7+or93M3NxUVQuqUQVwGcTtMOPG7nXFVFwWxuRs/N4rPZ6D1c1cucHaQLZ1fzGl3JMQXG+vV0H1vgvKWFvjM7tPnWTkiMYHFCFIFgScLGR2Nm9cQxetp2tblcSC994ABx6XDnrZ4eEqwjI+6grZ7nf+gQuYg4YGxW+/rswHWFFKc4dNcNU1XwezNwfvRoeF99fh/+opn0FpaK44UBUQSCJQsWNC5q57gWkWa7xNOnyVLgnbit2Tv3ItDdNbrPn1Mrdd4iPZ9fj2347MB1t1JSZpHuuhkZCd+bbjJ9DWYdRZwVVCiUJsh9200K5gaiCARLGqV2KLPtcJl1Uy9MiyOd27YtFKStrVGu/kyG5pycJAuACdwGB6MN6H16DevNZXwD5BzM1RWeXlBmFq/pisxWONbYSL9NKZBq5IUHUQSCJY04oeNql5hKkbA/etS+80+icB4aovdbtxIjqNn8Rvfb6+/1/sJxcQhuYak3jtGD2aVYBF1d0bFJ/RgOHw6Dy7kcxU5KhfATLTyIIhAsebj463M5KqzS/fyHDoV9B1z0Dz7UEe3t4X1dwtxsAalTSXMGkcmFZMY5uNNZNqvUzp3J9zJjBK+/Hg1YDw3Zd/62+MpMBba0m1x4EEUgqAjogclCIaSTMHf7LBy3bnVTPie5bYaHyQro7i69KTyzo7K7xtZr2NbC0sV0avZHzmSI9XRwMNzV8+/Bv4VJGWGjvD4TgS0WwcJDnCII6PvFg3Xr1qm2trb5XoZgAaNQoErY7m7gwguBz32ueExTE7BmDfClLwGdnVT5e889wKWXAh/9KFXwXn01cPvtxdem01RtfPXVwOQkVQVPTRWPW7aMqnnf9Cb3+7VrgeeeA97ylvh5xseBw4epWrivD1ixAli1ChgeBs49l64fGwOCgK557TUas3o10NsL5HLA7/wO8K530e+jI5UCzj+fKqYvvpgqoR95hH7DlhbgvPNK/AOAfpcnngBuucX++23cSM8lmDsEQfCCUmqd7TuhmBAsKUxOkgC75Rbg058OKSNM3HcfCcqDB0n4HjxIQv+110gJPPggcOWVJLRM6og/+iPgL/6C5nHRRwB0vqcn/v099wBnnQWMjsbP09tLFBdKkTB/61vpunPPJcqIiy8GbrgB6OoC8nlSbjffTMrl5puBF1+ka/v7SaGsWUNzV1UBV1xBSmTTJhq/aRMpuVSK5p8Jli0jhWr+fuk0nRclsMDgMhUW6iGuIUEcdJeEb4GUfrDbRm/baLqP9HmTAr5mjAAg98yOHWFfgKmp+E5n+rXpNNUmmCmtra3k1onra9DQEFJOjIwotWZNfG1CObJ7pI5g4QASIxBUClytJ+P893zobKAuQQtEq3Fd/D+639704Q8Pk7LR+wKY/n8O7LKf/+jRcLwtHXbPHqWeeSa5hWUuR/Ps3k3ZQT402oKlAVEEgoqBbhHE7db1Bi98JNFKZDI0Znw8uns3m9+cOBHWJPT3k9DXA8GpFJ3jrJ7Dh0m5cNGZK8jNFoRpyVRVhWmpPlYQ01DzGiW7pzIQpwhmNUYQBMEngyB4OQiCV4Ig+JpjzI1BELQHQfBiEAT/PJvrESx9nHtuSOf84IMU/DUpolMpooh+8MHotV/5CnD//fZ5t2yhAC8HiNetC+edmACam4EDB4CBAaJvPmv6f9ZZZ5FYDYIwkPu3fwu8+c1AYyOwfj3w61+Tr/7AAaK9DgIKtG7aZI9hDA9H13b33eT337TJL2Zx++3Ayy/Tb5XNxo8fG4v9uQVLBS4NcaYHgLMBvArgcgDLARwE8D5jzFsBvATgkunPK5PmFYugcjBT/7JeU6C7WNjPX1dHzeSZEM5GvWDbHU9O0nu9TaatBeTQkJtZVO8XfPhwWNilu2hOnIh31/T1RS0RPYbR0uLf+ziXK7Zuyh0jECwcYKauIQCfiTsSrq0B8LT2eTOAzcaYvwTwrbh5zEMUwdKGLvyHhijXfcWK0nlqeB7uLGZy/B8/TgpBF+RJ/nWu1vWhekhiFh0djVI96G6pUjiITCZSdjuZMQtmJV25MqrYXJTcSb+1BIEXH85EEfx/08fjAIYBpKaPEwB2JVx7G4CHtc9fALDdGPNdAP8NwE8AvADgi4657gDQBqDtkksumYvfTDAPsJGU2apsfYVOoRA2o2eBPzAQ5RTSi6gmJuJ30xMT9N6n0bwZhDWJ6UZHQ0XFVkprqx+5HCs2l6Jh+gx+ZuYs4t/CVGw2Su6431jI5BYnZqwI/vcgoBnAKu3zKg9F8FmLIthmjNkO4BcAqgBcAOC3AN4TN69YBEsTvj2IfTNZTBK5rVuVamsr5gTSM4KOHqUgqqvHLwvURx5JTvXUd/X67j3OZdTaGlY7u7iAGhvJUtIzl1zKyPbb8XlmE2V3lm92kJDJLV7EKQLfYPEapdRx7XM/gPckXHMMwMXa53cC6LWMeUopVVBKDQJ4DsAHPdckWELw7UHc2UlVtDZMTlJh1htvUCXtD34AvP3tFETdtAk4dgy47bZoAPZP/oSKtf7TfwIefph6+bp6/P6P/0HB4ptuoiBvKlVcLHXllRSE1oO2d98NHD1KFc4f+xi9v/324kDw0aPAqVMUDL7sMnsf5Msuo+c87zz6LWzo7KQCs/p6CpZzUJzPNzQA73gHFXZdfTX1Qnb9pra/07332r+TXsaLGC4NoR+gnfvTAP5PAF8C8CSM3b3lmmUAjgC4DGGw+P3GmPcC2Ds99i0AOgFcFTevWARLE74Uyy6LIMmtlOTO6esLCeF27YonY2tqojaQhw9HewQXCiHZW11daOHobiK9KYxtHdyCcsUKezC6upo++1JY69xBNTVhbEJ3jaVStPZy/J0k3XThAuWoIwBwK4AHp49bPa/ZCOA3oOyhr0+fuxPAndqY/weUOdQJ4K6kOUURLE349hSwuR+S2lb6cPlPTpLLhYU69yiwkcBt2EACm3sW62R2IyOkAB56iJRKfX3UTeTDaupDN11bS0Fnl/vIrJHg1pmma4zdXr6uISGTW7wolyK4FMDHpt+/BcD5vteW8xBFsDRRKLgFG3fPsgUkORaQJDh9BJjJyqlX9OqK4Ec/IiXgEsJsKXCKqe7PT/Ltc+WvHiPQA81TU7Sj5z4IqVSx9TIwUHy+s9Mdd6ivJwvHBxIjWLw4Y0UA4C8AHADw6vTndwPY63NtuQ9RBEsTp0/bKZabmsJKW1PIsDvIJ90yk3ErmoYGEp6ulEzTNeQjzF18REmZSePjpHw4fXbHDuqdbAaXmarixz8O78eWAnc040rnoSE/6glfuHo/SNbQwkY5FEH7tJ//V9q5Dp9ry32IIli6GB2lnWt3Nwnv7m76bBMwen9hV08B3SK4/367oqmvD3sT+DaH9/GTj4yEgnflSlI0jY1KHTvmzkzKZIgzyEzL7O2NPpO+A+/oCDORzAY3eu1AuakkpI5g8aEcimD/9Ouvpl+XATjkc225D1EESxu+AiafJ4GatHtnt9KhQ1RExr2FdUXD6aNJAdhslgRv0u46m6X7sAultpZ887W1JJhZgemB4JYWt8vFxoukKzndfcRWwOgoVSizgkwKlotvf+mjHIrgOwDuAXAYwMcBNAL4zz7XlvsQRSBQinzacQVV+/aFfv5slnbo69cXC+BMhnbUgJ+LKZUiRRCX59/SEmYgpdPRAjEXW6lvP2LbmvRzOgGdriA7OsS3X+kohyI4azpOsBPAYwD+wue62ThEEQiUcu/KXYIwnQ4Ltri6N5MhBVGKiwmgubnoK47GYcMGpcbGohXEpg9ft4BK7Z1gUxAu95beK1l8+5WJciiCv/I5NxeHKAKBziNkCu04P39TEykJvbr3pptIGZipoDYXk1nd3NIS1hGMjNBnbvaiB3x1viRbg/hs1j99NsllFFenUFNDyo8VEPdJFlQGyqEIfmk59yufa8t9iCKobCQVjvk0o9GVRUuLOxWU4wum8NaZSPX7+WYdmZ3Ltm2jWEWc66azM7qT37WLdvhmENmnTkHcQZWJGSsCAJ8HsAdEOLdbO34M4Jm4a2frEEWweHGmmSY+hWM+fn5deCelgtoqdHUqaL2wyzfraO1aClTrCsLluuH02Vwumiaqk9TpdNsuQj3d+hB3UGXiTBTBpQBuBPA8gD/SjmsALIu7drYOUQSLE+VgrExyn3R3EzNnkmCfmCBXDTAz33xTEwlitgJ6eyk+kMQayu4d3pnbKCC4XzLTZ+tupcZGcu1wDEIvNHN1NDMtkYkJsQQqFXGKIKDvFw/WrVun2tra5nsZghIwOUkdt265pfi7dBrYuJGI1ZLwxhvU/Wtqqvi7ZcuI8OzZZ4m07fbbi8fU11MHsYsvBtasoS5lvb3AH/4hEb+ZWLsW2LuXyNk6O4lE7r77gMsvB264gQjgNm+m96OjwBVXxK/v5EngTW+ieXftonlsY+rq6F6f+1zxPI2NwHvfC5xzDj1HTw9w5AiR7N16q/2ZMxngwguJEO+KK4iwTlB5CILgBaXUOuuXLg2hHwCuA1UWjwI4BWAKQM7n2nIfYhEsPpSLn8YnoOpKz2QXC1NG7N5Nry0t7lRQtlgGB8O6g54equg1u5INDvpRXdgsCr1PQTabnPPPqaa889eL1+LcW83NYg1UMlAGGurtoHjBbwG8GcCfA9h2RupJUDF4y1viKZN9KZD1fsQm6uqov+673w185jPAO98JPPccWQnPPEM754suIqvk6quBSy4B7roL+Ou/JsugsTFK+cxUzR/7GPCzn9E8U1PAQw/R3G99K/C971Gf45/+lHbo555L67Bh82ayiurrgQ9/mPoSV1WRxXHVVUSTfc45ZJ1ccEH877ViRZTC+vzz48dXVQHXXgvceKOf5SWoQLg0hH5gWpNAqyYG8HOfa8t9iEWwMBEXCC4nYyUXc5n5+5wKybxETU3kD48L3k5MhNeNjBRz9uj++6Ym8tfHZQcVCiHjqM0a4QAuWxCuLmM+mU/6uaTfN5ejZxVUNlCG9NHnQFxDPwRVGd8N4KDPteU+RBEsPCQFguOyfZqbiWhN5/W3CS1d0WQylLWju2e43WKhEN7LhxyupYUEtckSahs/OEjvXdlBuRxVL7e0hMrQplQ4WOxyJcVRTOvZR3wkEeoNDkqmkKA8iuBSAOcCWAFgC4D/AuBdPteW+xBFsLDgS0tsY6xsbnbz+utsmD69jNeuJd99d3coXH1y6lMpsgZefNGvZ0FcT+HJydBS0NdhUyrcgMZ2v6oqpfbvtzN8trcX1w4wJXUcoZ759xBUHs5IEQA4G8A/Jo2bq0MUwcJCKW4f0300Ph7P68+pjkm1A7qg1oV/Ujewrq5QQLa30846ruE8WxAuAZ7JhBTTOvW0TUFmMvHBZe4jzCRyuRz1OUil7Iqjt7eYubW9PSTUs/09BJWFclgETwNY7jN2tg9RBAsLZ9K60IcjP07RmHn3uRzt7plEzkcY19YSJ9DAQOie6uoKC7bMOEBjI+3m+R4A5fUz3QRzGPX301wcC6iupmdqaCCr4brriDgvrjfBxET0t9B5lMyd/4EDND7JJSWtJCsX5VAED4HSR78B4N/z4XNtuQ9RBAsLpVgEHJzlWICPEolznxw4EO6CJydDWunjx8NqXVsqqS6MUylSAqbryUXhANB3nZ2hEhgYoHM6h5EZL+Hq4NHRsPJ3YMDdm6C9PbRY+LcAaL0tLcW9jGtq6H1SoFksgspFnCLwTR/tBdAMYiE9XzsEFY6klM5zz6X3+Tzw+ONUfLV8Ob2OjFDqpA1XXQUUCpQSahvz1a8Cq1dTeuTNN1Pq5c03Ay++SGLv+ecplfSP/5gKqjiV9KmngPXrgXXrgH/6J0q9/NnPKK1UT8n8zGeAffuAu+8uvvfWrZRC2t4O/Pa3dK/PfIZSTY8epWI2fa5bbqHU0R/8APjRj4ATJ4Avf5mK2Xp6gI4OKjA7eZJeOzqAV18FHnkk+lt0dgJ/8AdUOHb55dEU1p/9jNJEf/tb4JvfTP57CAQRuDSE7QAFi+elVzEfYhEsPCS1LpyYsMcCzN68pcQIdH+8eTQ0UOBY3zXncvZGNkmUFK4+AJOT1GxGDwr7pn3aehaYFoEeCGd3VGcnPUdzM1kA+/fbW1gytbbQTQt0oAyuoXUAOgAcnT4OAviQz7XlPkQRLEzE1REk9Q6wKRF2pUxM2BWNj0tKP+dqZONDUmebn1NJ9et95+Jq36oq6pzW2UmZSMwvpMcUuMfB+vUk9Ht7i9NkbfGP8XFpJSmIohyK4BCAG7TP10NaVQo8MTVFxGl6Bg4XV61eTUKaA60mf79ZLMaCzSfVUz/nUkalFm8B0eIy/Xrfudjn39JCVgWP0YPOeo8DJplja6dQKG+RnqAyUA5FsM/n3FwcoggWH1xuGQ6K8s4/KZWUwUohThD290fPuRRHXGYRN7Ix3Tb5fMheql8fN5dJQ53NhvObAWkfIe+jCAUCHXGKIDZYHATBNUEQXAOgNQiCh4IguDEIgj8KguDvAfykjKEKwRLG8uXEkPm5z0WDqLffDnR10fcTE1GenqoqoLaWvr/5Zvp+cpKCpi+/TEykcUHq06ej3EH5vD3o/OCDxMqZTkfHNzUR6+jq1UBLCwVyn3uOArlvvEHMo3z9pZcSh9DevcRq2tAQnau+nsY8+CBds2ULsH8/8Qu9/HJxQDqJm+nNb3YH0QE6f/Kk/TuBwAqXhiAFgh/HHM/GXTtbh1gEiw+l7nBdXD7pNLlLHntMqWPHyJ9u4x0aHCS/ut4TeO9e2oHb1rBnj1Ivvxz2LLa1l2TfPED+d90/z4VnXPyVz9P9XT7/kRFqaelyPyWxj+Zy1At55077GHYfCQQ6cKauoYV0iCJYfPCpF9B9+HGdvtJpKvaamCCFoDeB55z6VCpajHXkCPnZXYpjZIRet22L73ecy9H70VE7pUMqRQqDXT3s8x8ZoWfMZqPcRvz8ZkC6rs6uaLq6wt+qt5fmtdUg9PZKcFhQjBkrAgD/evr139uOuGtn6xBFsLjATdKTdrj79oX0CUlB174+vxaTum++tpaEq01xcG+CpPvmcpThk89T8Zg5VyYTVhy7rJpUijKFjh4N5x0asnMwpdPxaaLDw1Q8p9NKdHZKmqjAjjhFENuhLAiC/0sp9VAQBFscbiUH+/rsQTqULR4UCuQ37+4GVq6M7xq2ciUVev30p1SAdc459k5f1dU0ftmy5G5lu3aRb/6uu4A9e8jn7+pEtmsX9Shw3ZfnPHyYOoTZ7j05GV5fW+vuMtbUBNx0E41Np4GPf5yuraqi32z5cuD97wc+/3ngP/wHik3Yurs1NlLB3NQUxRXGxqhgTHoOCGw44w5lC+kQi2BxQC8Ec1E92HrqcvpkXN1BU5Mfu+fAQLhbL7WpvWtOznJKSkU157IR2B07plRbG1kROu2FzjLqQ6UtEPgAZ5A19Hdxx2xoLcHSwPg4cO+99L5QoN24TqXwk5/Q549+lL4HgAceAN71LsoQ+vrXi+e8+27KPtq0CfiHfwDuucd+7y1bgOPHycpgGoZCIT7LpqeHKB1cc27eDPzwh5Tlc9ZZtOM38cgj1NMYoH7CnPlj60R2ww3AgQNksdxyC3UQu/ZashAOHCAr6u67qb9wUvcxgeCM4dIQpEDwpbgj7trZOsQiWBxIChBzgNTcKXOGT29vsQXBvXqTrIz29rByt7WVgsrbtrlz/BsbQ6ZO25ypVJTOublZqVdftY9jAju9Z0Fc8LuxMRxr1lcMDYlFICgfMF9ZQwA+CeBlAK8A+FrMuGsBTAG4LWlOUQSLA6U0mjcDqtzacXQ0rLK1NXLR0zY5TbS2llI1p6Yo+4YDwUePhi0sTTqLbDbk5qmupnTPwcGQ0ZR5/VeupLl6emj+/v7Q/TQwQJ8nJ2ndTIvhE/zmwLZ+1NdTxfXYWDIfk0DggzNWBAAuBPC3AJ4A8CwfCdecDeBVAJeD2lweBPA+x7hnp+cWRbBEENdQhnfgcTtlrrrVFYSPctF3yvk8ZeiwolmxgiwDFvIjI7Trzucp24Zz/21c/vX1JNBtSmtgwF7zMDxM1yXFJ5hiWj9qakiZcA9mk2vJ7OImECShHIqgBcBXAPwawB8B+B6AbydcUwPgae3zZgCbLePuAvB/A/i+KIKlBRcrKe/AXS0fTcHOO3+9cYxNUHNufmMjpaOm0+6eAhyQ1WsHknbuTDSnH7W1bmqM5ubQqinFItBbVeoKbGgoDDSLJSAoFXGKwLcfwe8opR4BcFop9c9KqS8DuC7hmosA9Gifj02f+98IguAiALcC2BE3URAEdwRB0BYEQVsmk/FcsmC+8cYbxJu/ezcFiHfvps8TE0BfX9hPwIbOTkrn3LoV+PGPKdDKvQw2bQI+8AHg2WeBmhqidFizhgKyjY3ARz5CnP233OLuKbBlC7BiBfDpT1OQGogGeG3rqa4OaS8mJ+n1r/4K+M537NfU1lLvge5uNx3Gli1Aa2v03N13U18DvUdCfz+QywFBQMdZvv9zBQIfuDSEfgD4xfTr0wA+BeD3ALyacM1nATysff4CgG3GmJ0Arpt+/32IRbBoYbKDFgru3Tvz8SftwNk/7/KRp9Nh1y8XW6eLQbS6OmztyGyoekGYzVVz4gQFnk0XkB5INt0+p0+HPYXN+AQXhenrNS2TOLoNKRwTlAKUwTX0aQDVAK4C8Qy9AODmhGsSXUMAuhD2OBgFMABgU9y8oggWHtgFpAsqPS/erBdgd0htLY2zCV529SQ1oGf66lJ6Cuj1CGbVr8uV9NJLbq6ipiYKTMe5t9jnr1NMP/OMUr/8Ja2F+xFwzIKVk0lVbSpCoZIQ+KIciuAHAN6qfX47gO8lXLMMwBEAlyEMFr8/ZrxYBIsQcUFh3W+vv+cAKWfyxHXoShLy3Cx+/Xp7v4OaGqKk0M+3tLj9+jbq6VQq2c/f1xf//KyQdOppPb3V1e/YRVXN95W+AwJflEMR/MrnnGXMRgC/AWUPfX363J0A7rSMFUWwCFFKJo/+nmsCOBDKXbTMjJ2kPPrubkoRHR62Zwfx7ltnE9XrEWxznjgRTVs9dsyP/z+uatpsVWlmTcVlUOkKxXZfgcAH5VAEBwG8Tfv8dgAdPteW+xBFsLDgWzimv9+9O4wR8C7dRb08MZHc7EXP3HH51Eu1MmwKpbu7ON6h78z7+tzpp6lU9JlzObJkSu1uJhaBYKaIUwS+uQf/L4CfB0HwzSAItgL4OQBHroRgqWFyEhgdpSyg0VH6zEhqkNLTE74/fhxobqZMn5UrgbPPpu/OPhs4dSramIZx/Dg1jqmvdzd7+cpXwms548bWBOe11+j7np74NXd3A6+8Qg1pLryQaDCWL6cGOVdfTdlJOrXDPfdQVtQVV1B2UUcHPefEBK01laKMpxMnohQTb397mKWUlLF08cXF5++5hzKIBIIzhktDmAeA9wH4twD+HSyFYXN1iEUwt7AFgvWMFd8YQWMjZRKNj1NA1tyx79kTUi/rLpbOTvquttbtPtJ3+D4767iWkg0NycVuDQ1K3Xtv6AJ6/XWltm6l7zgbaWQkXGtLi32uUvodDw4Wu5727ydLRSDwAaQxjWAmiBPyesaKrXCsoYEEVU0NCa1Dh0g4mkVhTNvATVeyWfL3s8to796wCYyrAE0Xoj4ZRJw1ZCt2Y/dRkmDO56P0E7aMqFSKXrnzmZlKq2dNxSmnnTtJmZh9FGpqxDUk8IcoAsGM4NNiksF1BKdPk69cp4DeupWEpWlZ1NfTODM3X99lV1URVXNPT5S/Rz9qa8NagyQBPjRE75ublfrNb6JrHh0N6wh8aCFcVBQTE9SKcmSEYheuWIVeQcwNaBoaipWT3vXMpYwFgiSIIhDMCD4tJnWYioOpITKZZN4hm/tlYMCeYmkKy+bmkNdn+/Z4t8/oKH1//DhZJ2yhDA7Szp37ACcpFL6f6a45dIjWnc26U1R1l1lNDc3F7SyZX2hqilxp/H5oKNr7WArKBKVCFIFgRijFIlDK3ozm0UeTm8jEZcSYvnqzUfzAAAlEvY7AxjKq+9RZ6HKT+lWrQi4fVgr79sVnK9XVRZlPs1lSeDxvUt0BP7OuFPSm867YDJPQiSUgKBWiCAQzgm+MQEcuR7v87dtDAe6TrmkWglVXh5+TKot96alrakhg61XPHKQ2q4z183H1ASZBHF/vk1ZrxlFYUc3kdxcIkiCKQDBjuBhEbW6J06fJTcMB0lIyYmyNWThovGKFvWqY+w74WC7shmppUZGdeBx7KOf+s/Vx4kQYE2Bl099v51SKK4Qz6SayWbrPhg3hjr8US0wg8IEoAsEZwSSUc+1IdQGmWwFxGTGuGEFjIwVzuU+wrUCM+wfncm7OooYGUjTcN4AJ3tg940N819hI6+A0Vr3JjSsQ3NJiJ8vTCehszXj4dy4lNlPq30lQmRBFICgb4oSNLsBsDJpmcFVv0m4TwgMDIReRy03CwVVba8v6ejo/OEhB3La28F7snvFxW7W0hPGIXbuiPQxs8QNOix0YIGWgr6mjw01eV18fxipKtQiS6j0EAlEEgrIgSdjk8+R3t2UK2YK8p06R2ydOCCcFmru7w25gPT1hS8tslj7rwl8PzLJFMDKSHIPg52Gro6srfE4byR0HgjdupN+EXUC5XPL9OCU1jnrb3OlLTEHgA1EEgjOGj7A5fZp24I8+6k711N0nLn4hXSj6Foi9/npxzr7ZdJ5ZQrl4rb+flFgcFTY3vR8ZidYYuPiM9u+PrimVKj2IHFfwZtvhS0xB4ANRBIIzho+wYZpl0wrQ6Rb0gKq+yzYPpno4U8qIdJruo2cidXeHr1NTdreVmR2UTiv1jW+E646rVRgcJGoMmwvIl1yuuposJh+f/0xjCoLKgigCwRnDR9i4lIVrV887a1vsgIVwnJBnV49PwJeDy2aG0uBg6OZxKSx9HiDZksnlyFVlG+PTjKfUnbxYBAIfiCIQnDFcVcM6rbLL5+8S1LqvXefR0VtGugLN9fXk9qmrc9cR8Nq4joDpr03BzP54n8Cx7ziX4tTrDpIsEF/fvsQIBD6IUwTSAlvghXPPDRuwV1URFfNVV0VplZ98EnjuuShFMwA88gjw9a8Xz/nII8BXvwp861vU1P5Nb6LX//pf6TwAFApEA93RATz1FDA+DuzaBbz8Mn1/5ZXA66+HtNK2tV1/PfB7v0cN7t/znugaHnwwpI8uFOLpqfN5oLGRqLFd4669luYZHSUa6q4uamLPv0mhANx1F7BhA801Pg488wyQyQB//MfAu98NpNP0zMuWuf8eOpYto/HpdJSqu9R5BBUMl4ZYqIdYBPMHzhrSq4bNw1YXUFVF3D6dnWFv3p4epV58MfS3m60hM5niqt5UirKNmFNI7+7F7qM4+uimJiKEM9fGhHgcGHa5bZhWIpez78BdNQK23X6hQM+wejW9Dg2F1stMd/BSRyCIA8Q1JCgXTp9O5tEZGiomhbPRODQ2kuDcurWYYvmpp5Q6fLi4+nbNmuLKZd195KKq5rVls9FzuhKJq3fI58M1bNxI8QRz3MGD8WmfQ0M0prWVXF/CHSSYS4giqHCUe6fo07+3UAhjB8PDdhoG3qUfPFisOEZGSKjqikPn5jHXsHIlCWmftcW1yDR5inI5CihzllFHB1klk5M0Tm9PmRRE7u4uTmkVH75griCKoIJR7opTH4ugu5voGPSG8i43icm7k8uREokLfg4MRIVuKUynHPT2zevn/sX6+nt7lRobozGlNsVhN9NMsoMEgjOBKIIKRbmzSVipJPnSk8jceIyLuTOJYqGrK8rlo8cFktpQDgxEz82kaTz3NTCFv+9cehczrm0Qq0Aw2xBFUKEoZ365rddAXBHW0aMzLwTzSc8cHSX+IDMu4FpbKhXtLcxHXF4/F7WZbqOeHrIkenrIguFUV5+aB9064NqGnh7hBRLMPkQRVCjKWXHqqiNgXzrvcG0N5V1ukrj6Ah9+odZWEuw+/QjYt8/so/pYW16/XtSmKxjT1aW3kvRRkKZ1oFNjS7xAMJsQRVChOBOLQA8wFwp+HDn6uTg+fhbmrjl9d9Y6DYUPCdzOnXbKa45TcP+D/v7oTp/X5EpLTacpNXbtWnL1bN8ezuXqa6zHCLq6/P4mAsGZQBRBhcKMEegVt3EZRKOjYc4/t27MZErzpU9MxPP/cD9e25xMupbUHaymhnb5TFdty9/X3UEun78umJuaqF7ggQfCJjvc/jJJqebzofC//347DbX5DLoSFV4gwWwiThFIZfESx7XXAvX1QE0N8OyzwNVXU8Xt8uVUcfvEE1TtypicpArayclQ1I2NUQXst75lv0ddHXDBBcD27UB1NVW1jo5S1W99PX2urqbvBweBT3+a5geAe+8N56mqoirczk5a54YNwE9/Cpw8SdXEHR1UKVso0Njvfhd4/nm6x+OP03MdPAhMTdHr7bcDBw7QMwA07yWXRKtv6+uBSy+lCmMAuO8+4AtfoM+///vANdfQvG9+M11vQ2cn8Ja3UPX12Bjde8UKqrJ+4QXgJz+xPwNAFco9PeH7sbFS/roCQZng0hAL9RCLwB96f4CBAb8MokKBdtG2tM/h4eJdut7/lwukOAXU3FF3dkaZPzs7ac5Uinb3+/fTfczOXbbmNbr7qJRsnb6+4l7G+ty8QzddQUn3yGbDgO/ERDTF1tfVJTECwWwC4hqqTLg6hsXFCwqFZJcO0yHY/N9NTZTNo/cjeOght3Lp7VXq2DFyP7kUlU5bwe6tbDZaHOabv9/SEl6TzdpZRm0tLGtrSWHpcYhMJuxZzEqluZnup//2TGORSkWV6K5ddL6mRrKGBLMPUQQVClcPYTNeoHPcJPnCu7uT8/x5B873GB2Nz+0fHi4OLtvYTXfsUOrAgWKrIWk9TEM9MKDUY48l8wC1txf/Xtx6cteuUMnZrBfdItIZVFtbad7ubvqd2SISignBXEEUQYVCDxb394duIlfzdXbtxLWPTOrzW11NrhE9e0cXii5rRJ/TTNVcsSLs5Ts1RevXd/KuRvFsTYyOKrVvHykBl3tm+3YS8sPDZNGYisksWovrWbxtWzStNCnjSJSAYC4giqCCwdXAQ0Oheyau+TpXDsft9l1uJj3bx1Qw+q7bVC6Tk2HDGFPQuvL3zeK19vYwf59TOAcHQ9qK0dF4ZcS7+KkpqonI56OuKv2ZfWMSnFaaRIQnKaOCucC8KQIAnwTwMoBXAHzN8v2/AnBo+vg5gA8mzSmKoHScPk1NY3jXnCTIhobs3+n0ETZXTxythFmlq9+vry/aIEZfX9Luu7aWFMmKFdHAtMvaMa0JXRnp45uboxTZusXiG5NgIS+tJAULAfOiCACcDeBVAJcDWA7gIID3GWP+AMDbpt/fBGB/0ryiCGYGLpCqrfUjWovLf3fl+fvGDvSD8/b1hu2l8veUojiYJ8lV6asfqRS5nY4cicZOfC2CpBaeYhEI5hLzpQhqADytfd4MYHPM+LcBeD1pXlEEpdNK6+MHB+lYv95diZvLkUvjxAl3dtDu3WG/AO4j4KNgTP4fnZKZ4wu6f95n9+2yJuKEtK1C2WWtcJyDOYl800FZyEsrScFCwHwpgtsAPKx9/gKA7THj/6M+3vjuDgBtANouueSSWfuhFgNKpZW2jU+lKChqy6Dp7SXBxMpD973r1kF7OwlIvYI4iWhuYCBMPbX1JrCR0fnwDulunFLcNtksKTxdGdmU1wc/qNQ3v0kxlvr6sOahocG/7zD/HUw+I0kZFcwV5ksRfNaiCLY5xq4H8GsAv5M0byVbBKXuLOPG21pK8jyFAgn4qSmlVq0i90g2G+0Utnp1SDbHymLHDnL1JO2UXffW6ak5PXP79vi6hsFBUi6PPx62kfR12/Dvc/CgXRGYqbBmCujgYMgpNDREa62udgt5aSUpmE8saNcQgA9MxxLe4zNvJSuCUn3NSeNtfnHeZTc2hqmUZsYO58uPjoaC7cSJ8LO5862vD5lC9boAXXCaFcR6HYFtTg7+jowo9ZOfkEXQ2BjPcWS6bXQGUJtictFQc3U0N8jh5+fOZSLkBQsR86UIlgE4AuAyLVj8fmPMJdMZRX/gO28lK4JSs09KZQw1zzc2En2zOaaqitwkqVRxK8nWVrIMWBgeOULB4NZWO4VzPk+76UJBqSefLI5b1NWRn76jI1qMdfCgUm1tdC9OEx0eJmU0OJhM9sZBalYKZp/luJTX6mpat1lQxi4zCf4KFiLmM310I4DfTO/4vz597k4Ad06/fxjAMID26cO5UD4qWRHMlUWgc/N0dxeP8QmW+mbypNMk5NevDyt3TeuDC7Nsysgcn04rdfw4Cfn+/rBGYft2cmfp9QX9/dEuYbqiMWmo9WP79ngX2KlTc/rPQiDwghSULRFMTMRX0E5MRMf7xghMempdQNqsBp/MnLq6UFgmBZH7+8m141qrLaunrs79W7j6FLioqtvbycVkKjvX/IOD8f0Pcrm5+fcgEJQCUQRLBElZPDaXRC5X7CZJpyno29zsZv3U2yiagtAnM2fr1jDLxqf1pE4i58om0q/Tm9e7lJF+3zirpLGRmtHr943riTA15a523r9fCsQECxOiCJYIpqbCClqmUuZ0y+rqYgF0+rRSe/YUj2e/e2cn7YTjrAabIvAt8lq/njKMkjJ5+HsXEZyuLDhzySf+oVsiPmmo5n1Xr1ZqfLw4YyqXiyfRKxTm8B+FQOAJUQRLBOWKEeg+/iS3zfBwsQXCO2XbNTrtA++akzJ5Jibc8/A6mIvIl3H0yJEoGV0pVNVsGdh4kxobkxWbBIsFCxGiCOYR5cwdL7WOwLVr1nfHPgKypSVkCB0cpP4Cra3FBVWcNfTQQ1HqaZMUzlaY5hLmbJmYMYI4H35TE1k7x46F900ifuvri943jjfJJ3tL6gUECw1xikBaVc4iCgVqBXn99e7WkKVg2TJqc5hOR9s/Dg1R+8dTp6hF5Btv0Ov4OLWqNHHxxWHbxWPHqEWiDVddRfOsXEliDqA2kLfeCvze7wE33hi2knzqKWD9euC97wXe8Q46d//9dM3DDwOvvkptGnftirZtPHoU+O1vga4ual/Z1UXtKru6aJ3pNHDZZWErScaDD9L5pqbi1pPXXkvtNletAv7wD+k+55wDbN1qf8577gHOmv6f0NlJ973rLmrBaUNvb/xvNjJSvr+5QDAncGmIhXosFotgNvlluBUiZ8G4GqWk0xSwNVMvdYsgics/l7MHRVtbKdaQy4VB3paWkPPf1l/AtAj27KFsIdv87e0096lT7v4IzE2kp322t4eUEfpzutbAAV6demJkJN5SuvfemVVsi2UgmE9AXENzj9linGTOmm3bSOjW1obVrZxxs2ZNlBYil6PGLKtX071feinsJ5zNxrttBgeL1+8qKOM2lXV1xVlAK1dGg9YjI0Rap7tfzDRWLhDT8/ltnctGR6OBc1Z8tbUhUZx+LWcUMf11TU2xGyouuFxTE/YoNrOxbP2VJXYgWAgQRTAPmA0Oet3KeO01EjrmbrqhIcwEMoOc2SwJ9qNHScju2hXl8ndlIpnC3JXzzwoiqR0k0zvrLKOuBjRcfaz3Bx4cJEW4YkX4bAMDtD5zPfv32ykv9PUwp1E6Tb9LKpXMcaQT83EsIM56kb4DgvmGKIJ5wGxYBPqcExPxefE290Rjo1LPPENClXfKPqmgzFg6MED37O5OzkYyDxa2zc0kbB991C/P32V9mMK8sZGUi233PjAQWhA2crh8noLJuVyUnptdb76ModJ3QLCQIYpgHjAbMQLdyvBl2TTPZ7NRgeXLra9n0bj850lKJZdT6uTJMPXUp/1jKdz/ZnWwOYarftnyyWajriTzuXRXEu/+4/5u0ndAsJAhimCeUG4Oel2Az5RQbmoqem1cEFXfcfsIbd8KYpsSmqlyMbuBxT2D/rs0NJClUMq9fHb00ndAsFAhimAeUa46An2erq6ZWwS5XDE9w8qVxcHllpaoz91049h26T4WgV4dffRo6PMfGrKTvJXSaCaXo2ewBY51i6i9XakDB4qVpflcZvDa9+8nfQcECxFxikDqCGYZy5YB551HeernnUefS4VZj7BpE9Dd7c5z37IF+N737OfPPht46aUwp76qCti9GzhxgnLuly8HbriB8vAPHw5z+4eGKD+/thb48z8HNm6knPueHhrT00PXfutb9jVt3gycey49y3PPUb79LbfQNR/9KPCzn1FtQVVV9Lqenvic/Z6e8Nn6+6l24qyzgCCwr+GHP6SahnPOoRoKHQ8+CFx5JdUu1NQAzz4LXH01/d6l1ASU428uEMwpXBpioR6LzSJwwXfX6PI760FU3Q2xcyf5ynfvjp7nRjP799P3U1O0C9+3zx10bmhQ6hvfoF3+oUNEB/3oo7Rzb20tDuByxzBzTXpdwMmT7opdpqTWr+3ocPvdm5qoHwFnRDU2FmdQ7d9PsQHf4HJzMwXiCwXx9wuWFiCuoYWFUvoOx2WicEZMX1+0yTzXEbDA51x7Gw2zD2cPEKWVTmLyPHy4mKhtzx5SOnoBms1109cXBmf7+uj648ftMYzeXhLYQ0Ok+FxCe2ws7E2g00XbmtHw30AygARLDaIIFhCSMksmJqKWQlJQeHKS0kH1tEdd4G3cSMKNlY/p//YNOuvc/kmU0SZBXHMzXWPrBWAL5uo1Bd3d4X3NOoeaGlJ2SbEJVnY2umiXVTYbdSACwXxCFMECQtIOP5cLe+J2d5OgS6JPbm4Oc/x1gZdK0fUdHVHhvXVr2DrSVRNgWgQ6m6iPMDeDsC53kK2PsG5x+GQi+QaUTbeXThfNtB1sQRUK7g5lYhEIFiNEESwguHaaVVUknE2X0aFD8RZEJkOuF5ug3bo19Ovrc7a3h/xCvnn6mUzpTeH58E0B5T7CPqmqfO3Ro/7zu4R5LuffIpN/d4kRCBYbRBEsIJgWAbtoTNoGPn/0KF1j5qY3NNBO/Lrr7K0TM5lid5BNMLvqCEzaBp+OYKZS4MNnx84+/61b/VJVgTDga3IKuZSUed+pKbIE4oLXnZ1SEyBYGhBFsIBw+nRI+Kbn0Q8Ohn5+Fv6dnaGbiAnYpqbI3aPn+etuG87oaW+PFm/FCWZbBW1tbZhB09PjF0tIp5U6eDCe7dSmREZGQp//8LCdgyiul4Erg8qHBM6ne5rUBAiWAkQRLDDYXBHpNO3AH3uMUiJff91OKMdWgO6b17mDuFewyeUzE1cNN5bfto3WnNSwnbOUjh4tZgF1UV3rO3b2+W/bZi/sMllDq6tJMDc1hcR0fX10joviUin7fdm9I0FhQaVAFME8wgxCjo8nc9n7+uP5fX9/KKR1d1Acf4+PK6W6mgTso4+SxWJTTvX1dJ538bYcfiasS6KxYCXkS3vBO3adiZRdTOPjNK63N57ywcciEAiWAkQRzAFsBWK2nb9P3n5SDru+Y+ddsu4O0hvF2HbELCAbG2mO6mpi5BwcjO7oH3oorB2YmIhvDWnrO8xWjk5hbdY86G6bhoZQCZnuKpMgTlectvtyQRj3KnC5d+Keq7GRvhcIlgJEEcwyXAViHPzUXSlxgVPuuOVTO8Bz5fN0f9dufWDA7jt/6SWlXn452unMlnrKKZSueIPu549TWroSsvUISKeju33bM5fi/8/l/P35uVyoFPVguVgDgqUEUQQzwJlSQPBOub09KmBdGT7cdGVoKBxj+sj1rly88x8ejqdDaGggy4B31qOjoauKKSa4vaRtZ/3tb/sVnk1O2s/bcvhraug3ZX/+yAj9BnF5+/394TPkcqQwy9kExnThiSUgWGqIUwQVTzo3ORlt+D45Gd903hw/Pg488EDYcF1vwP7tbwMrVgAHDxLh2s03U/P3ffuoOfrTTwO//CWRm33iE3SvDRuAf/5noLEReOYZIJsF+vqAdeuI/Oycc4gU7umngbY24KabqBn8vffan+/++4EPfpAI0MbHac4bbgjvlckA73pXMdkbQERud95J5G7Hjyc3ua+tjc6jk8LpuOsu+m1+93eJaO4f/xF4/XXga1+zz795M7BtG3D55fS7nDpFpHiXXeZez9iY/TsXli8Hzj+ffqfzz6fPAkHFwKUhFupRTovA5tLp7Az94rYdcmdndPzUVHyLRRvPz4oVFOAcGqLr2SKYnKTXTMZvx84tH30yX1w1BfxcmUyxD56v7eoiN1Jc5g9n+ujunaYmSic16x84iMzZPiMjYbWzGSA2ieN6e8nd1dKS3EpSIBCEgLiGiuFy6fimVjKH/+hoPId9JkP+e1YuOo/OddeF2TGuNFF2L5m+8KoqOt/UlEwTMTiYPKa7253Fw0FuV5N7V//ffD4kxWMSuYGB0B1UW0tkdFy0pQet2e01OEjju7vp/pkM1TfU1JCCaGiQgi+BwAeiCCxwZebozdzZJz8yEhZzMQ8N77L1eaqqlGprI1oILgTLZKIpirW1tMPntE+9Obwu7G1povo6dT6epOrb2lp/Ph7XffXfZWQkvvmLjw+fq4lbW2lsd3dU4Le1Ed2EHheorVXqySejxHyFghR8CQQ+qHhFcPJkGAgsFKJBwbGx8DNn4LCQt53Xx7MiMMfo99DPsxKxZem4duMcXNUbsHd1RYOrzFNkyw7iSt9Sishs1oF+fZJSYeWWlJ+vF6MxPbVNuTQ2hvUXstsXCGaGOEWw5IPFuRyJE8bUlLtj1NlnUxAYoFcepxQFEUdHqbtVNkvnx8eB738feNvbKKDc0kJBzKefDgOyHGg+cIACwi0tFFC+4w4KIk9N0evttwOvvQbcfTfN3dkJXHwxBV+/+13g+efDYPGmTcDPf07n9eDsFVdQt7GTJ+n1iiuAiQn67pFHgHvusT/35s1hR7POTmD1aqCjgzqHcTcu/fqkrmH5fPL9zjmHupl97WvA6dP0Gw8P02/R3EzrXruWgsPXX08B4o0b7UFtgUBwhnBpiHIcAD4J4GUArwD4muX7AMDfTX9/CMA1SXOWYhGcPElplmZRF3e0+s536P3ICB2ucZ//PPmlh4fjaZh96ZZd3bFsxWJxbh8uwIprFMNjSqnW7e4unkePSWzfnkwEl8QR1NMTxhHyeXL3PPlktKEOcyo9+aS4fASCMwXmwzUE4GwArwK4HMByAAcBvM8YsxHAk9MK4ToA+5PmLUURjI25BXNjI33P719/3T1ueNhPyJfifrEVYNl89T5z+t6XlVVcdS+7k2xzcZZSV1fo3jKLsLJZpR5/3B785VhIczO5yEzhzplNEvwVCMqP+VIENQCe1j5vBrDZGPMQgM9rn18GsCpu3lIUgQ+PDL/PZuPH+Qhb34As+9Ft83R3R1MmfQq5fJq36MJ19+6wx69NkPf2Fu/kU6koP79JBMd+fr6eC96GhkgZVFfTPE1N8RW7voV8AoGgNMyXIrgNwMPa5y8A2G6MaQZwvfZ5L4B1lrnuANAGoO2SSy7xfnCf/Po4wax/5yPkz9Qi4N3vwADd78SJZGU2NJRM/8BCmgPW+/Yp9YEPhD2E+fyRI3T+0CHqaqZn8rz0Uthu0tyx5/Phbn/v3mhwXg+6S8WuQDB/iFMEsxksDizn1AzGQCn1D0qpdUqpdRdeeKH3AgqF+KAmB0K5MjZuXFKAtKeHAqT33Wcfowdkt2yhgOratRSQXrsWqK+n6uH/+T8pEHzyJAWlDxwAtm61z3nffcD27cDf/E38fX/4Q+DJJ4EXXgAefBBYtYoqmt/3Pvru4ovptbqazq9ZA1x0EQXIg4DOX3IJcN55wMc+RpXP4+PA3r303O98J3DrrcC11wK///tURc1VulVVUrErECx0zKYiOAbgYu3zOwH0zmDMjPGmN7kF5JYtYVbQli1uRbBlC2X2JAn5J54ArrkGWL+eMl10Id/QQMJ171767iMfoeyhZ54hgbp7N/D+9xMdxW23AceOhcJ1bAy48UagqSk6565dlN3z1FPAhz9MdBHpdHRMOk2ZNp//PFFJfPzjdP+ODlI055wDfPGLJMy/+EUS4EFAx5vfTOsJguJsqnPPpXMrVgBf+hJw4gTw059KVo9AsGjhMhXO9ACwDMARAJchDBa/3xjzKUSDxa1J85ZaRxDnC/+bv4lmDbnG/emfhn51WzCTK2jZL57NRt0hZh0B+9Tz+fC9XqRmNlE36xfMOcfG7GNOnqTfYHy8uCeCQCCoLCDGNRTQ97ODIAg2AvguKIPoe0qp/xwEwZ3TCmhHEAQBgO2gNNMxAH+mlGqLm3PdunWqrS12SBHGxylXvaqK3DzLltFuuFAgq4G/m5igHH/bOH4/Pk4WAo856yzaPZvvzz6bds5xY3jOsTEau2wZ3X98HHjLW6LnBQKB4EwQBMELSql11u9mUxHMBmaiCAQCgaDSEacIlnxlsUAgEAjiIYpAIBAIKhyiCAQCgaDCIYpAIBAIKhyLLlgcBEEGwGszvPwCAINlXM5igDxzZUCeuTJwJs98qVLKWpG76BTBmSAIgjZX1HypQp65MiDPXBmYrWcW15BAIBBUOEQRCAQCQYWj0hTBP8z3AuYB8syVAXnmysCsPHNFxQgEAoFAUIxKswgEAoFAYEAUgUAgEFQ4lqQiCILgk0EQvBwEwStBEHzN8n0QBMHfTX9/KAiCa+ZjneWExzP/q+lnPRQEwc+DIPjgfKyznEh6Zm3ctUEQTAVBcNtcrm824PPMQRDcGARBexAELwZB8M9zvcZyw+PfdnUQBHuCIDg4/cx/Nh/rLBeCIPheEAQDQRB0Or4vv/xy8VMv1gNEef0qgMsR9kF4nzFmI6J9EPbP97rn4Jn/AMDbpt/fVAnPrI17FsATAG6b73XPwd/5rQBeAnDJ9OeV873uOXjmewB8e/r9hQBOAFg+32s/g2f+QwDXAOh0fF92+bUULYIPA3hFKXVEKXUKwKMAbjHG3ALgh4rwCwBvDYJg1VwvtIxIfGal1M+VUsPTH38B6ga3mOHzdwaAfwcgBWBgLhc3S/B55j8FsEsp1Q0ASqnF/tw+z6wAnD/d3+Q8kCKYnNtllg9KqedAz+BC2eXXUlQEFwHo0T4fmz5X6pjFhFKf5yugHcViRuIzB0FwEYBbAeyYw3XNJnz+zu8B8LYgCH4SBMELQRB8cc5WNzvweebtAN4LanPbAeCvlFJvzM3y5gVll19LsfdVYDln5sj6jFlM8H6eIAjWgxTB9bO6otmHzzN/F8BXlVJTtFlc9PB55mUAPgTgowDeDOD5IAh+oZT6zWwvbpbg88x/DKAdwAYAVwD4URAEP1VK5WZ5bfOFssuvpagIjgG4WPv8TtBOodQxiwlezxMEwQcAPAzgJqXU0Bytbbbg88zrADw6rQQuALAxCIJJpVTTnKyw/PD9tz2olCoAKARB8ByADwJYrIrA55n/DMBfK3KgvxIEQReAfwGgdW6WOOcou/xaiq6hAwDeHQTBZUEQLAfwOQC7jTG7AXxxOvp+HYCsUur4XC+0jEh85iAILgGwC8AXFvHuUEfiMyulLlNKrVFKrQHwGIC/XMRKAPD7t50GcEMQBMuCIHgLgN8H8Os5Xmc54fPM3SALCEEQvAPAlQCOzOkq5xZll19LziJQSk0GQfBvATwNyjj4nlLqxSAI7pz+fgcog2QjgFcAjIF2FIsWns98L4DfAfD30zvkSbWImRs9n3lJweeZlVK/DoLgKQCHALwB4GGllDUNcTHA8+/8TQDfD4KgA+Q2+apSatHSUwdB8E8AbgRwQRAExwBsAfAmYPbkl1BMCAQCQYVjKbqGBAKBQFACRBEIBAJBhUMUgUAgEFQ4RBEIBAJBhUMUgUAgEFQ4RBEIBAJBhUMUgUAwA0wX88j/H8GSgPxDFgg8EQTBmiAIfh0Ewd8D+CWAR4IgaJvmwK+bHvPhIAh2Tb+/JQiCk0EQLA+C4NwgCJZytatgEWPJVRYLBLOMKwH8mVLqL4MgeLtS6kQQBGcD2DvN5fRLAL83PfYGAJ0ArgX9X9s/LysWCBIgikAgKA2vTXPAA8D/EQTBHaD/R6tADVMOTXeOei+IS/+/gBqNnA3gp/OyYoEgAeIaEghKQwEAgiC4DMB/BPBRpdQHADwO4NzpMT8FdYE7DeAZEOX39QCem/PVCgQeEEUgEMwMK0BKITvNeHmT9t1zAO4C8LxSKgMi+/sXAF6c60UKBD4Q15BAMAMopQ4GQfArkHA/AmCf9vV+AO9AaAEcAjCghOFRsEAh7KMCgUBQ4RDXkEAgEFQ4RBEIBAJBhUMUgUAgEFQ4RBEIBAJBhUMUgUAgEFQ4RBEIBAJBhUMUgUAgEFQ4/n/brBGgYVXGxwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot = probs.plot(kind='scatter', x=0, y=1, s=64, c='blue', edgecolors='white')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## AdaBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Adaboost -> cross validation accurary: mean = 0.4493, std = 0.0587\n",
      "Wall time: 6min 1s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "adbc = AdaBoostClassifier(n_estimators=300, random_state=101)\n",
    "scores = cross_val_score(adbc,\n",
    "                         covertype_x_train,\n",
    "                         covertype_y_train,\n",
    "                         cv=5,\n",
    "                         scoring='accuracy',\n",
    "                         n_jobs=-1)\n",
    "print(\n",
    "    'Adaboost -> cross validation accurary: mean = {:.4f}, std = {:.4f}'.format(\n",
    "        np.mean(scores), np.std(scores)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.38596139, 0.40148016, 0.42959797, 0.48293914, 0.54637771])"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GradientBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 19min 18s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GradientBoostingClassifier(max_depth=5, n_estimators=50, random_state=101)"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "gbc = GradientBoostingClassifier(max_depth=5, n_estimators=50, random_state=101)\n",
    "gbc.fit(covertype_x_train, covertype_y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.948254347994458"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(covertype_y_val, rfc.predict(covertype_x_val))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9477547718627907"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(covertype_y_test, rfc.predict(covertype_x_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "import xgboost as xgb\n",
    "xgb_model = xgb.XGBClassifier(object='multi:softprob',\n",
    "                              max_depth=24,\n",
    "                              gamma=0.1,\n",
    "                              subsample=0.9,\n",
    "                              learning_rate=0.01,\n",
    "                              n_estimators=500,\n",
    "                              nthread=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[20:01:03] WARNING: C:\\Users\\Administrator\\workspace\\xgboost-win64_release_1.1.0\\src\\learner.cc:480: \n",
      "Parameters: { object } might not be used.\n",
      "\n",
      "  This may not be accurate due to some parameters are only used in language bindings but\n",
      "  passed down to XGBoost core.  Or some parameters are not used but slip through this\n",
      "  verification. Please open an issue if you find above cases.\n",
      "\n",
      "\n",
      "[0]\tvalidation_0-merror:0.07679\n",
      "Will train until validation_0-merror hasn't improved in 25 rounds.\n",
      "[1]\tvalidation_0-merror:0.06606\n",
      "[2]\tvalidation_0-merror:0.06125\n",
      "[3]\tvalidation_0-merror:0.05930\n",
      "[4]\tvalidation_0-merror:0.05846\n",
      "[5]\tvalidation_0-merror:0.05813\n",
      "[6]\tvalidation_0-merror:0.05737\n",
      "[7]\tvalidation_0-merror:0.05706\n",
      "[8]\tvalidation_0-merror:0.05647\n",
      "[9]\tvalidation_0-merror:0.05622\n",
      "[10]\tvalidation_0-merror:0.05571\n",
      "[11]\tvalidation_0-merror:0.05525\n",
      "[12]\tvalidation_0-merror:0.05520\n",
      "[13]\tvalidation_0-merror:0.05461\n",
      "[14]\tvalidation_0-merror:0.05428\n",
      "[15]\tvalidation_0-merror:0.05423\n",
      "[16]\tvalidation_0-merror:0.05384\n",
      "[17]\tvalidation_0-merror:0.05371\n",
      "[18]\tvalidation_0-merror:0.05373\n",
      "[19]\tvalidation_0-merror:0.05355\n",
      "[20]\tvalidation_0-merror:0.05331\n",
      "[21]\tvalidation_0-merror:0.05326\n",
      "[22]\tvalidation_0-merror:0.05323\n",
      "[23]\tvalidation_0-merror:0.05309\n",
      "[24]\tvalidation_0-merror:0.05298\n",
      "[25]\tvalidation_0-merror:0.05294\n",
      "[26]\tvalidation_0-merror:0.05270\n",
      "[27]\tvalidation_0-merror:0.05259\n",
      "[28]\tvalidation_0-merror:0.05238\n",
      "[29]\tvalidation_0-merror:0.05220\n",
      "[30]\tvalidation_0-merror:0.05217\n",
      "[31]\tvalidation_0-merror:0.05187\n",
      "[32]\tvalidation_0-merror:0.05174\n",
      "[33]\tvalidation_0-merror:0.05150\n",
      "[34]\tvalidation_0-merror:0.05134\n",
      "[35]\tvalidation_0-merror:0.05123\n",
      "[36]\tvalidation_0-merror:0.05101\n",
      "[37]\tvalidation_0-merror:0.05097\n",
      "[38]\tvalidation_0-merror:0.05085\n",
      "[39]\tvalidation_0-merror:0.05071\n",
      "[40]\tvalidation_0-merror:0.05063\n",
      "[41]\tvalidation_0-merror:0.05036\n",
      "[42]\tvalidation_0-merror:0.05034\n",
      "[43]\tvalidation_0-merror:0.05027\n",
      "[44]\tvalidation_0-merror:0.05007\n",
      "[45]\tvalidation_0-merror:0.05005\n",
      "[46]\tvalidation_0-merror:0.04995\n",
      "[47]\tvalidation_0-merror:0.04983\n",
      "[48]\tvalidation_0-merror:0.04978\n",
      "[49]\tvalidation_0-merror:0.04973\n",
      "[50]\tvalidation_0-merror:0.04947\n",
      "[51]\tvalidation_0-merror:0.04936\n",
      "[52]\tvalidation_0-merror:0.04921\n",
      "[53]\tvalidation_0-merror:0.04908\n",
      "[54]\tvalidation_0-merror:0.04894\n",
      "[55]\tvalidation_0-merror:0.04877\n",
      "[56]\tvalidation_0-merror:0.04866\n",
      "[57]\tvalidation_0-merror:0.04861\n",
      "[58]\tvalidation_0-merror:0.04847\n",
      "[59]\tvalidation_0-merror:0.04835\n",
      "[60]\tvalidation_0-merror:0.04824\n",
      "[61]\tvalidation_0-merror:0.04821\n",
      "[62]\tvalidation_0-merror:0.04808\n",
      "[63]\tvalidation_0-merror:0.04806\n",
      "[64]\tvalidation_0-merror:0.04791\n",
      "[65]\tvalidation_0-merror:0.04780\n",
      "[66]\tvalidation_0-merror:0.04784\n",
      "[67]\tvalidation_0-merror:0.04778\n",
      "[68]\tvalidation_0-merror:0.04767\n",
      "[69]\tvalidation_0-merror:0.04750\n",
      "[70]\tvalidation_0-merror:0.04750\n",
      "[71]\tvalidation_0-merror:0.04736\n",
      "[72]\tvalidation_0-merror:0.04727\n",
      "[73]\tvalidation_0-merror:0.04718\n",
      "[74]\tvalidation_0-merror:0.04713\n",
      "[75]\tvalidation_0-merror:0.04707\n",
      "[76]\tvalidation_0-merror:0.04705\n",
      "[77]\tvalidation_0-merror:0.04698\n",
      "[78]\tvalidation_0-merror:0.04694\n",
      "[79]\tvalidation_0-merror:0.04675\n",
      "[80]\tvalidation_0-merror:0.04668\n",
      "[81]\tvalidation_0-merror:0.04660\n",
      "[82]\tvalidation_0-merror:0.04650\n",
      "[83]\tvalidation_0-merror:0.04640\n",
      "[84]\tvalidation_0-merror:0.04630\n",
      "[85]\tvalidation_0-merror:0.04623\n",
      "[86]\tvalidation_0-merror:0.04613\n",
      "[87]\tvalidation_0-merror:0.04598\n",
      "[88]\tvalidation_0-merror:0.04586\n",
      "[89]\tvalidation_0-merror:0.04583\n",
      "[90]\tvalidation_0-merror:0.04583\n",
      "[91]\tvalidation_0-merror:0.04573\n",
      "[92]\tvalidation_0-merror:0.04557\n",
      "[93]\tvalidation_0-merror:0.04546\n",
      "[94]\tvalidation_0-merror:0.04540\n",
      "[95]\tvalidation_0-merror:0.04529\n",
      "[96]\tvalidation_0-merror:0.04527\n",
      "[97]\tvalidation_0-merror:0.04522\n",
      "[98]\tvalidation_0-merror:0.04521\n",
      "[99]\tvalidation_0-merror:0.04514\n",
      "[100]\tvalidation_0-merror:0.04515\n",
      "[101]\tvalidation_0-merror:0.04514\n",
      "[102]\tvalidation_0-merror:0.04503\n",
      "[103]\tvalidation_0-merror:0.04501\n",
      "[104]\tvalidation_0-merror:0.04490\n",
      "[105]\tvalidation_0-merror:0.04480\n",
      "[106]\tvalidation_0-merror:0.04476\n",
      "[107]\tvalidation_0-merror:0.04468\n",
      "[108]\tvalidation_0-merror:0.04463\n",
      "[109]\tvalidation_0-merror:0.04452\n",
      "[110]\tvalidation_0-merror:0.04430\n",
      "[111]\tvalidation_0-merror:0.04428\n",
      "[112]\tvalidation_0-merror:0.04422\n",
      "[113]\tvalidation_0-merror:0.04419\n",
      "[114]\tvalidation_0-merror:0.04416\n",
      "[115]\tvalidation_0-merror:0.04403\n",
      "[116]\tvalidation_0-merror:0.04398\n",
      "[117]\tvalidation_0-merror:0.04397\n",
      "[118]\tvalidation_0-merror:0.04387\n",
      "[119]\tvalidation_0-merror:0.04374\n",
      "[120]\tvalidation_0-merror:0.04375\n",
      "[121]\tvalidation_0-merror:0.04372\n",
      "[122]\tvalidation_0-merror:0.04356\n",
      "[123]\tvalidation_0-merror:0.04346\n",
      "[124]\tvalidation_0-merror:0.04340\n",
      "[125]\tvalidation_0-merror:0.04333\n",
      "[126]\tvalidation_0-merror:0.04335\n",
      "[127]\tvalidation_0-merror:0.04322\n",
      "[128]\tvalidation_0-merror:0.04318\n",
      "[129]\tvalidation_0-merror:0.04313\n",
      "[130]\tvalidation_0-merror:0.04304\n",
      "[131]\tvalidation_0-merror:0.04290\n",
      "[132]\tvalidation_0-merror:0.04284\n",
      "[133]\tvalidation_0-merror:0.04269\n",
      "[134]\tvalidation_0-merror:0.04258\n",
      "[135]\tvalidation_0-merror:0.04260\n",
      "[136]\tvalidation_0-merror:0.04252\n",
      "[137]\tvalidation_0-merror:0.04240\n",
      "[138]\tvalidation_0-merror:0.04240\n",
      "[139]\tvalidation_0-merror:0.04234\n",
      "[140]\tvalidation_0-merror:0.04216\n",
      "[141]\tvalidation_0-merror:0.04208\n",
      "[142]\tvalidation_0-merror:0.04197\n",
      "[143]\tvalidation_0-merror:0.04184\n",
      "[144]\tvalidation_0-merror:0.04183\n",
      "[145]\tvalidation_0-merror:0.04177\n",
      "[146]\tvalidation_0-merror:0.04169\n",
      "[147]\tvalidation_0-merror:0.04163\n",
      "[148]\tvalidation_0-merror:0.04149\n",
      "[149]\tvalidation_0-merror:0.04148\n",
      "[150]\tvalidation_0-merror:0.04144\n",
      "[151]\tvalidation_0-merror:0.04137\n",
      "[152]\tvalidation_0-merror:0.04125\n",
      "[153]\tvalidation_0-merror:0.04120\n",
      "[154]\tvalidation_0-merror:0.04112\n",
      "[155]\tvalidation_0-merror:0.04102\n",
      "[156]\tvalidation_0-merror:0.04100\n",
      "[157]\tvalidation_0-merror:0.04089\n",
      "[158]\tvalidation_0-merror:0.04083\n",
      "[159]\tvalidation_0-merror:0.04078\n",
      "[160]\tvalidation_0-merror:0.04074\n",
      "[161]\tvalidation_0-merror:0.04073\n",
      "[162]\tvalidation_0-merror:0.04069\n",
      "[163]\tvalidation_0-merror:0.04061\n",
      "[164]\tvalidation_0-merror:0.04054\n",
      "[165]\tvalidation_0-merror:0.04049\n",
      "[166]\tvalidation_0-merror:0.04044\n",
      "[167]\tvalidation_0-merror:0.04032\n",
      "[168]\tvalidation_0-merror:0.04027\n",
      "[169]\tvalidation_0-merror:0.04026\n",
      "[170]\tvalidation_0-merror:0.04015\n",
      "[171]\tvalidation_0-merror:0.04009\n",
      "[172]\tvalidation_0-merror:0.04001\n",
      "[173]\tvalidation_0-merror:0.03998\n",
      "[174]\tvalidation_0-merror:0.03983\n",
      "[175]\tvalidation_0-merror:0.03989\n",
      "[176]\tvalidation_0-merror:0.03986\n",
      "[177]\tvalidation_0-merror:0.03977\n",
      "[178]\tvalidation_0-merror:0.03973\n",
      "[179]\tvalidation_0-merror:0.03963\n",
      "[180]\tvalidation_0-merror:0.03957\n",
      "[181]\tvalidation_0-merror:0.03956\n",
      "[182]\tvalidation_0-merror:0.03947\n",
      "[183]\tvalidation_0-merror:0.03944\n",
      "[184]\tvalidation_0-merror:0.03938\n",
      "[185]\tvalidation_0-merror:0.03930\n",
      "[186]\tvalidation_0-merror:0.03932\n",
      "[187]\tvalidation_0-merror:0.03920\n",
      "[188]\tvalidation_0-merror:0.03916\n",
      "[189]\tvalidation_0-merror:0.03904\n",
      "[190]\tvalidation_0-merror:0.03896\n",
      "[191]\tvalidation_0-merror:0.03890\n",
      "[192]\tvalidation_0-merror:0.03888\n",
      "[193]\tvalidation_0-merror:0.03885\n",
      "[194]\tvalidation_0-merror:0.03883\n",
      "[195]\tvalidation_0-merror:0.03882\n",
      "[196]\tvalidation_0-merror:0.03875\n",
      "[197]\tvalidation_0-merror:0.03870\n",
      "[198]\tvalidation_0-merror:0.03864\n",
      "[199]\tvalidation_0-merror:0.03861\n",
      "[200]\tvalidation_0-merror:0.03856\n",
      "[201]\tvalidation_0-merror:0.03855\n",
      "[202]\tvalidation_0-merror:0.03852\n",
      "[203]\tvalidation_0-merror:0.03842\n",
      "[204]\tvalidation_0-merror:0.03839\n",
      "[205]\tvalidation_0-merror:0.03831\n",
      "[206]\tvalidation_0-merror:0.03825\n",
      "[207]\tvalidation_0-merror:0.03816\n",
      "[208]\tvalidation_0-merror:0.03816\n",
      "[209]\tvalidation_0-merror:0.03816\n",
      "[210]\tvalidation_0-merror:0.03816\n",
      "[211]\tvalidation_0-merror:0.03806\n",
      "[212]\tvalidation_0-merror:0.03811\n",
      "[213]\tvalidation_0-merror:0.03808\n",
      "[214]\tvalidation_0-merror:0.03794\n",
      "[215]\tvalidation_0-merror:0.03791\n",
      "[216]\tvalidation_0-merror:0.03783\n",
      "[217]\tvalidation_0-merror:0.03783\n",
      "[218]\tvalidation_0-merror:0.03781\n",
      "[219]\tvalidation_0-merror:0.03776\n",
      "[220]\tvalidation_0-merror:0.03773\n",
      "[221]\tvalidation_0-merror:0.03761\n",
      "[222]\tvalidation_0-merror:0.03759\n",
      "[223]\tvalidation_0-merror:0.03754\n",
      "[224]\tvalidation_0-merror:0.03744\n",
      "[225]\tvalidation_0-merror:0.03743\n",
      "[226]\tvalidation_0-merror:0.03747\n",
      "[227]\tvalidation_0-merror:0.03742\n",
      "[228]\tvalidation_0-merror:0.03737\n",
      "[229]\tvalidation_0-merror:0.03734\n",
      "[230]\tvalidation_0-merror:0.03736\n",
      "[231]\tvalidation_0-merror:0.03735\n",
      "[232]\tvalidation_0-merror:0.03730\n",
      "[233]\tvalidation_0-merror:0.03724\n",
      "[234]\tvalidation_0-merror:0.03719\n",
      "[235]\tvalidation_0-merror:0.03719\n",
      "[236]\tvalidation_0-merror:0.03712\n",
      "[237]\tvalidation_0-merror:0.03712\n",
      "[238]\tvalidation_0-merror:0.03705\n",
      "[239]\tvalidation_0-merror:0.03698\n",
      "[240]\tvalidation_0-merror:0.03694\n",
      "[241]\tvalidation_0-merror:0.03690\n",
      "[242]\tvalidation_0-merror:0.03692\n",
      "[243]\tvalidation_0-merror:0.03694\n",
      "[244]\tvalidation_0-merror:0.03694\n",
      "[245]\tvalidation_0-merror:0.03689\n",
      "[246]\tvalidation_0-merror:0.03687\n",
      "[247]\tvalidation_0-merror:0.03686\n",
      "[248]\tvalidation_0-merror:0.03681\n",
      "[249]\tvalidation_0-merror:0.03677\n",
      "[250]\tvalidation_0-merror:0.03669\n",
      "[251]\tvalidation_0-merror:0.03668\n",
      "[252]\tvalidation_0-merror:0.03668\n",
      "[253]\tvalidation_0-merror:0.03663\n",
      "[254]\tvalidation_0-merror:0.03663\n",
      "[255]\tvalidation_0-merror:0.03661\n",
      "[256]\tvalidation_0-merror:0.03652\n",
      "[257]\tvalidation_0-merror:0.03650\n",
      "[258]\tvalidation_0-merror:0.03644\n",
      "[259]\tvalidation_0-merror:0.03639\n",
      "[260]\tvalidation_0-merror:0.03639\n",
      "[261]\tvalidation_0-merror:0.03635\n",
      "[262]\tvalidation_0-merror:0.03631\n",
      "[263]\tvalidation_0-merror:0.03625\n",
      "[264]\tvalidation_0-merror:0.03619\n",
      "[265]\tvalidation_0-merror:0.03613\n",
      "[266]\tvalidation_0-merror:0.03615\n",
      "[267]\tvalidation_0-merror:0.03608\n",
      "[268]\tvalidation_0-merror:0.03601\n",
      "[269]\tvalidation_0-merror:0.03605\n",
      "[270]\tvalidation_0-merror:0.03601\n",
      "[271]\tvalidation_0-merror:0.03600\n",
      "[272]\tvalidation_0-merror:0.03594\n",
      "[273]\tvalidation_0-merror:0.03590\n",
      "[274]\tvalidation_0-merror:0.03586\n",
      "[275]\tvalidation_0-merror:0.03586\n",
      "[276]\tvalidation_0-merror:0.03584\n",
      "[277]\tvalidation_0-merror:0.03583\n",
      "[278]\tvalidation_0-merror:0.03582\n",
      "[279]\tvalidation_0-merror:0.03579\n",
      "[280]\tvalidation_0-merror:0.03573\n",
      "[281]\tvalidation_0-merror:0.03574\n",
      "[282]\tvalidation_0-merror:0.03569\n",
      "[283]\tvalidation_0-merror:0.03570\n",
      "[284]\tvalidation_0-merror:0.03567\n",
      "[285]\tvalidation_0-merror:0.03569\n",
      "[286]\tvalidation_0-merror:0.03562\n",
      "[287]\tvalidation_0-merror:0.03554\n",
      "[288]\tvalidation_0-merror:0.03552\n",
      "[289]\tvalidation_0-merror:0.03542\n",
      "[290]\tvalidation_0-merror:0.03546\n",
      "[291]\tvalidation_0-merror:0.03541\n",
      "[292]\tvalidation_0-merror:0.03535\n",
      "[293]\tvalidation_0-merror:0.03537\n",
      "[294]\tvalidation_0-merror:0.03535\n",
      "[295]\tvalidation_0-merror:0.03527\n",
      "[296]\tvalidation_0-merror:0.03524\n",
      "[297]\tvalidation_0-merror:0.03527\n",
      "[298]\tvalidation_0-merror:0.03526\n",
      "[299]\tvalidation_0-merror:0.03527\n",
      "[300]\tvalidation_0-merror:0.03525\n",
      "[301]\tvalidation_0-merror:0.03524\n",
      "[302]\tvalidation_0-merror:0.03524\n",
      "[303]\tvalidation_0-merror:0.03522\n",
      "[304]\tvalidation_0-merror:0.03514\n",
      "[305]\tvalidation_0-merror:0.03511\n",
      "[306]\tvalidation_0-merror:0.03508\n",
      "[307]\tvalidation_0-merror:0.03506\n",
      "[308]\tvalidation_0-merror:0.03505\n",
      "[309]\tvalidation_0-merror:0.03506\n",
      "[310]\tvalidation_0-merror:0.03499\n",
      "[311]\tvalidation_0-merror:0.03496\n",
      "[312]\tvalidation_0-merror:0.03492\n",
      "[313]\tvalidation_0-merror:0.03490\n",
      "[314]\tvalidation_0-merror:0.03487\n",
      "[315]\tvalidation_0-merror:0.03487\n",
      "[316]\tvalidation_0-merror:0.03482\n",
      "[317]\tvalidation_0-merror:0.03484\n",
      "[318]\tvalidation_0-merror:0.03482\n",
      "[319]\tvalidation_0-merror:0.03479\n",
      "[320]\tvalidation_0-merror:0.03474\n",
      "[321]\tvalidation_0-merror:0.03471\n",
      "[322]\tvalidation_0-merror:0.03473\n",
      "[323]\tvalidation_0-merror:0.03474\n",
      "[324]\tvalidation_0-merror:0.03473\n",
      "[325]\tvalidation_0-merror:0.03472\n",
      "[326]\tvalidation_0-merror:0.03470\n",
      "[327]\tvalidation_0-merror:0.03466\n",
      "[328]\tvalidation_0-merror:0.03461\n",
      "[329]\tvalidation_0-merror:0.03458\n",
      "[330]\tvalidation_0-merror:0.03458\n",
      "[331]\tvalidation_0-merror:0.03452\n",
      "[332]\tvalidation_0-merror:0.03447\n",
      "[333]\tvalidation_0-merror:0.03439\n",
      "[334]\tvalidation_0-merror:0.03435\n",
      "[335]\tvalidation_0-merror:0.03429\n",
      "[336]\tvalidation_0-merror:0.03432\n",
      "[337]\tvalidation_0-merror:0.03428\n",
      "[338]\tvalidation_0-merror:0.03427\n",
      "[339]\tvalidation_0-merror:0.03429\n",
      "[340]\tvalidation_0-merror:0.03429\n",
      "[341]\tvalidation_0-merror:0.03425\n",
      "[342]\tvalidation_0-merror:0.03427\n",
      "[343]\tvalidation_0-merror:0.03423\n",
      "[344]\tvalidation_0-merror:0.03420\n",
      "[345]\tvalidation_0-merror:0.03418\n",
      "[346]\tvalidation_0-merror:0.03416\n",
      "[347]\tvalidation_0-merror:0.03411\n",
      "[348]\tvalidation_0-merror:0.03412\n",
      "[349]\tvalidation_0-merror:0.03410\n",
      "[350]\tvalidation_0-merror:0.03408\n",
      "[351]\tvalidation_0-merror:0.03408\n",
      "[352]\tvalidation_0-merror:0.03406\n",
      "[353]\tvalidation_0-merror:0.03404\n",
      "[354]\tvalidation_0-merror:0.03401\n",
      "[355]\tvalidation_0-merror:0.03396\n",
      "[356]\tvalidation_0-merror:0.03395\n",
      "[357]\tvalidation_0-merror:0.03393\n",
      "[358]\tvalidation_0-merror:0.03391\n",
      "[359]\tvalidation_0-merror:0.03388\n",
      "[360]\tvalidation_0-merror:0.03389\n",
      "[361]\tvalidation_0-merror:0.03387\n",
      "[362]\tvalidation_0-merror:0.03386\n",
      "[363]\tvalidation_0-merror:0.03382\n",
      "[364]\tvalidation_0-merror:0.03379\n",
      "[365]\tvalidation_0-merror:0.03373\n",
      "[366]\tvalidation_0-merror:0.03371\n",
      "[367]\tvalidation_0-merror:0.03371\n",
      "[368]\tvalidation_0-merror:0.03368\n",
      "[369]\tvalidation_0-merror:0.03364\n",
      "[370]\tvalidation_0-merror:0.03365\n",
      "[371]\tvalidation_0-merror:0.03363\n",
      "[372]\tvalidation_0-merror:0.03367\n",
      "[373]\tvalidation_0-merror:0.03358\n",
      "[374]\tvalidation_0-merror:0.03356\n",
      "[375]\tvalidation_0-merror:0.03357\n",
      "[376]\tvalidation_0-merror:0.03359\n",
      "[377]\tvalidation_0-merror:0.03350\n",
      "[378]\tvalidation_0-merror:0.03350\n",
      "[379]\tvalidation_0-merror:0.03351\n",
      "[380]\tvalidation_0-merror:0.03348\n",
      "[381]\tvalidation_0-merror:0.03345\n",
      "[382]\tvalidation_0-merror:0.03347\n",
      "[383]\tvalidation_0-merror:0.03346\n",
      "[384]\tvalidation_0-merror:0.03343\n",
      "[385]\tvalidation_0-merror:0.03344\n",
      "[386]\tvalidation_0-merror:0.03342\n",
      "[387]\tvalidation_0-merror:0.03337\n",
      "[388]\tvalidation_0-merror:0.03335\n",
      "[389]\tvalidation_0-merror:0.03337\n",
      "[390]\tvalidation_0-merror:0.03338\n",
      "[391]\tvalidation_0-merror:0.03335\n",
      "[392]\tvalidation_0-merror:0.03333\n",
      "[393]\tvalidation_0-merror:0.03330\n",
      "[394]\tvalidation_0-merror:0.03327\n",
      "[395]\tvalidation_0-merror:0.03326\n",
      "[396]\tvalidation_0-merror:0.03326\n",
      "[397]\tvalidation_0-merror:0.03326\n",
      "[398]\tvalidation_0-merror:0.03322\n",
      "[399]\tvalidation_0-merror:0.03327\n",
      "[400]\tvalidation_0-merror:0.03325\n",
      "[401]\tvalidation_0-merror:0.03322\n",
      "[402]\tvalidation_0-merror:0.03320\n",
      "[403]\tvalidation_0-merror:0.03318\n",
      "[404]\tvalidation_0-merror:0.03319\n",
      "[405]\tvalidation_0-merror:0.03313\n",
      "[406]\tvalidation_0-merror:0.03309\n",
      "[407]\tvalidation_0-merror:0.03310\n",
      "[408]\tvalidation_0-merror:0.03308\n",
      "[409]\tvalidation_0-merror:0.03303\n",
      "[410]\tvalidation_0-merror:0.03302\n",
      "[411]\tvalidation_0-merror:0.03304\n",
      "[412]\tvalidation_0-merror:0.03303\n",
      "[413]\tvalidation_0-merror:0.03304\n",
      "[414]\tvalidation_0-merror:0.03299\n",
      "[415]\tvalidation_0-merror:0.03297\n",
      "[416]\tvalidation_0-merror:0.03293\n",
      "[417]\tvalidation_0-merror:0.03293\n",
      "[418]\tvalidation_0-merror:0.03291\n",
      "[419]\tvalidation_0-merror:0.03292\n",
      "[420]\tvalidation_0-merror:0.03289\n",
      "[421]\tvalidation_0-merror:0.03291\n",
      "[422]\tvalidation_0-merror:0.03292\n",
      "[423]\tvalidation_0-merror:0.03293\n",
      "[424]\tvalidation_0-merror:0.03292\n",
      "[425]\tvalidation_0-merror:0.03289\n",
      "[426]\tvalidation_0-merror:0.03288\n",
      "[427]\tvalidation_0-merror:0.03290\n",
      "[428]\tvalidation_0-merror:0.03286\n",
      "[429]\tvalidation_0-merror:0.03283\n",
      "[430]\tvalidation_0-merror:0.03282\n",
      "[431]\tvalidation_0-merror:0.03281\n",
      "[432]\tvalidation_0-merror:0.03278\n",
      "[433]\tvalidation_0-merror:0.03279\n",
      "[434]\tvalidation_0-merror:0.03275\n",
      "[435]\tvalidation_0-merror:0.03276\n",
      "[436]\tvalidation_0-merror:0.03275\n",
      "[437]\tvalidation_0-merror:0.03271\n",
      "[438]\tvalidation_0-merror:0.03270\n",
      "[439]\tvalidation_0-merror:0.03270\n",
      "[440]\tvalidation_0-merror:0.03268\n",
      "[441]\tvalidation_0-merror:0.03267\n",
      "[442]\tvalidation_0-merror:0.03266\n",
      "[443]\tvalidation_0-merror:0.03265\n",
      "[444]\tvalidation_0-merror:0.03263\n",
      "[445]\tvalidation_0-merror:0.03265\n",
      "[446]\tvalidation_0-merror:0.03261\n",
      "[447]\tvalidation_0-merror:0.03261\n",
      "[448]\tvalidation_0-merror:0.03261\n",
      "[449]\tvalidation_0-merror:0.03261\n",
      "[450]\tvalidation_0-merror:0.03258\n",
      "[451]\tvalidation_0-merror:0.03257\n",
      "[452]\tvalidation_0-merror:0.03256\n",
      "[453]\tvalidation_0-merror:0.03254\n",
      "[454]\tvalidation_0-merror:0.03254\n",
      "[455]\tvalidation_0-merror:0.03255\n",
      "[456]\tvalidation_0-merror:0.03251\n",
      "[457]\tvalidation_0-merror:0.03248\n",
      "[458]\tvalidation_0-merror:0.03247\n",
      "[459]\tvalidation_0-merror:0.03247\n",
      "[460]\tvalidation_0-merror:0.03244\n",
      "[461]\tvalidation_0-merror:0.03241\n",
      "[462]\tvalidation_0-merror:0.03237\n",
      "[463]\tvalidation_0-merror:0.03236\n",
      "[464]\tvalidation_0-merror:0.03237\n",
      "[465]\tvalidation_0-merror:0.03237\n",
      "[466]\tvalidation_0-merror:0.03237\n",
      "[467]\tvalidation_0-merror:0.03231\n",
      "[468]\tvalidation_0-merror:0.03234\n",
      "[469]\tvalidation_0-merror:0.03234\n",
      "[470]\tvalidation_0-merror:0.03231\n",
      "[471]\tvalidation_0-merror:0.03230\n",
      "[472]\tvalidation_0-merror:0.03227\n",
      "[473]\tvalidation_0-merror:0.03223\n",
      "[474]\tvalidation_0-merror:0.03223\n",
      "[475]\tvalidation_0-merror:0.03219\n",
      "[476]\tvalidation_0-merror:0.03220\n",
      "[477]\tvalidation_0-merror:0.03220\n",
      "[478]\tvalidation_0-merror:0.03222\n",
      "[479]\tvalidation_0-merror:0.03224\n",
      "[480]\tvalidation_0-merror:0.03224\n",
      "[481]\tvalidation_0-merror:0.03223\n",
      "[482]\tvalidation_0-merror:0.03219\n",
      "[483]\tvalidation_0-merror:0.03218\n",
      "[484]\tvalidation_0-merror:0.03216\n",
      "[485]\tvalidation_0-merror:0.03216\n",
      "[486]\tvalidation_0-merror:0.03219\n",
      "[487]\tvalidation_0-merror:0.03219\n",
      "[488]\tvalidation_0-merror:0.03216\n",
      "[489]\tvalidation_0-merror:0.03216\n",
      "[490]\tvalidation_0-merror:0.03215\n",
      "[491]\tvalidation_0-merror:0.03215\n",
      "[492]\tvalidation_0-merror:0.03219\n",
      "[493]\tvalidation_0-merror:0.03211\n",
      "[494]\tvalidation_0-merror:0.03213\n",
      "[495]\tvalidation_0-merror:0.03213\n",
      "[496]\tvalidation_0-merror:0.03210\n",
      "[497]\tvalidation_0-merror:0.03208\n",
      "[498]\tvalidation_0-merror:0.03207\n",
      "[499]\tvalidation_0-merror:0.03205\n",
      "Wall time: 1h 37min 25s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "              colsample_bynode=1, colsample_bytree=1, gamma=0.1, gpu_id=-1,\n",
       "              importance_type='gain', interaction_constraints='',\n",
       "              learning_rate=0.01, max_delta_step=0, max_depth=24,\n",
       "              min_child_weight=1, missing=nan, monotone_constraints='()',\n",
       "              n_estimators=500, n_jobs=-1, nthread=-1, num_parallel_tree=1,\n",
       "              object='multi:softprob', objective='multi:softprob',\n",
       "              random_state=0, reg_alpha=0, reg_lambda=1, scale_pos_weight=None,\n",
       "              subsample=0.9, tree_method='exact', validate_parameters=1,\n",
       "              verbosity=None)"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "xgb_model.fit(covertype_x_train,\n",
    "              covertype_y_train,\n",
    "              eval_set=[(covertype_x_val, covertype_y_val)],\n",
    "              eval_metric='merror',\n",
    "              early_stopping_rounds=25,\n",
    "              verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9671606340682605"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(covertype_y_test, xgb_model.predict(covertype_x_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[40838,  1494,     0,     0,    17,     2,    64],\n",
       "       [ 1025, 55248,    67,     0,    90,    43,    22],\n",
       "       [    0,    69,  6980,    35,     6,   104,     0],\n",
       "       [    0,     0,    54,   463,     0,    12,     0],\n",
       "       [   21,   276,    31,     0,  1637,     8,     0],\n",
       "       [    3,    85,   141,    10,     3,  3304,     0],\n",
       "       [  117,    17,     0,     0,     0,     0,  3916]], dtype=int64)"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(covertype_y_test, xgb_model.predict(covertype_x_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           1       0.97      0.96      0.97     42415\n",
      "           2       0.97      0.98      0.97     56495\n",
      "           3       0.96      0.97      0.96      7194\n",
      "           4       0.91      0.88      0.89       529\n",
      "           5       0.93      0.83      0.88      1973\n",
      "           6       0.95      0.93      0.94      3546\n",
      "           7       0.98      0.97      0.97      4050\n",
      "\n",
      "    accuracy                           0.97    116202\n",
      "   macro avg       0.95      0.93      0.94    116202\n",
      "weighted avg       0.97      0.97      0.97    116202\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(\n",
    "    classification_report(covertype_y_test,\n",
    "                          xgb_model.predict(covertype_x_test)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## XGBoost gpu vs cpu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "import xgboost as xgb\n",
    "\n",
    "xgb_cpu = xgb.XGBClassifier(object='multi:softprob',\n",
    "                            max_depth=8,\n",
    "                            gamma=0.1,\n",
    "                            subsample=0.9,\n",
    "                            learning_rate=0.01,\n",
    "                            n_estimators=200,\n",
    "                            nthread=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[09:34:19] WARNING: D:\\Projects\\other_projects\\xgboost\\src\\learner.cc:529: \n",
      "Parameters: { object } might not be used.\n",
      "\n",
      "  This may not be accurate due to some parameters are only used in language bindings but\n",
      "  passed down to XGBoost core.  Or some parameters are not used but slip through this\n",
      "  verification. Please open an issue if you find above cases.\n",
      "\n",
      "\n",
      "[0]\tvalidation_0-merror:0.21399\n",
      "[1]\tvalidation_0-merror:0.21641\n",
      "[2]\tvalidation_0-merror:0.21486\n",
      "[3]\tvalidation_0-merror:0.21350\n",
      "[4]\tvalidation_0-merror:0.21427\n",
      "[5]\tvalidation_0-merror:0.21401\n",
      "[6]\tvalidation_0-merror:0.21372\n",
      "[7]\tvalidation_0-merror:0.21296\n",
      "[8]\tvalidation_0-merror:0.21235\n",
      "[9]\tvalidation_0-merror:0.21210\n",
      "[10]\tvalidation_0-merror:0.21214\n",
      "[11]\tvalidation_0-merror:0.21132\n",
      "[12]\tvalidation_0-merror:0.21171\n",
      "[13]\tvalidation_0-merror:0.21129\n",
      "[14]\tvalidation_0-merror:0.21112\n",
      "[15]\tvalidation_0-merror:0.21057\n",
      "[16]\tvalidation_0-merror:0.21017\n",
      "[17]\tvalidation_0-merror:0.21022\n",
      "[18]\tvalidation_0-merror:0.20971\n",
      "[19]\tvalidation_0-merror:0.20983\n",
      "[20]\tvalidation_0-merror:0.20920\n",
      "[21]\tvalidation_0-merror:0.20917\n",
      "[22]\tvalidation_0-merror:0.20898\n",
      "[23]\tvalidation_0-merror:0.20936\n",
      "[24]\tvalidation_0-merror:0.20937\n",
      "[25]\tvalidation_0-merror:0.20906\n",
      "[26]\tvalidation_0-merror:0.20910\n",
      "[27]\tvalidation_0-merror:0.20829\n",
      "[28]\tvalidation_0-merror:0.20810\n",
      "[29]\tvalidation_0-merror:0.20779\n",
      "[30]\tvalidation_0-merror:0.20757\n",
      "[31]\tvalidation_0-merror:0.20763\n",
      "[32]\tvalidation_0-merror:0.20725\n",
      "[33]\tvalidation_0-merror:0.20682\n",
      "[34]\tvalidation_0-merror:0.20703\n",
      "[35]\tvalidation_0-merror:0.20676\n",
      "[36]\tvalidation_0-merror:0.20666\n",
      "[37]\tvalidation_0-merror:0.20687\n",
      "[38]\tvalidation_0-merror:0.20660\n",
      "[39]\tvalidation_0-merror:0.20631\n",
      "[40]\tvalidation_0-merror:0.20617\n",
      "[41]\tvalidation_0-merror:0.20582\n",
      "[42]\tvalidation_0-merror:0.20579\n",
      "[43]\tvalidation_0-merror:0.20574\n",
      "[44]\tvalidation_0-merror:0.20528\n",
      "[45]\tvalidation_0-merror:0.20444\n",
      "[46]\tvalidation_0-merror:0.20448\n",
      "[47]\tvalidation_0-merror:0.20433\n",
      "[48]\tvalidation_0-merror:0.20384\n",
      "[49]\tvalidation_0-merror:0.20346\n",
      "[50]\tvalidation_0-merror:0.20321\n",
      "[51]\tvalidation_0-merror:0.20327\n",
      "[52]\tvalidation_0-merror:0.20314\n",
      "[53]\tvalidation_0-merror:0.20298\n",
      "[54]\tvalidation_0-merror:0.20292\n",
      "[55]\tvalidation_0-merror:0.20249\n",
      "[56]\tvalidation_0-merror:0.20237\n",
      "[57]\tvalidation_0-merror:0.20232\n",
      "[58]\tvalidation_0-merror:0.20205\n",
      "[59]\tvalidation_0-merror:0.20184\n",
      "[60]\tvalidation_0-merror:0.20129\n",
      "[61]\tvalidation_0-merror:0.20129\n",
      "[62]\tvalidation_0-merror:0.20108\n",
      "[63]\tvalidation_0-merror:0.20096\n",
      "[64]\tvalidation_0-merror:0.20069\n",
      "[65]\tvalidation_0-merror:0.20050\n",
      "[66]\tvalidation_0-merror:0.20006\n",
      "[67]\tvalidation_0-merror:0.19992\n",
      "[68]\tvalidation_0-merror:0.19953\n",
      "[69]\tvalidation_0-merror:0.19928\n",
      "[70]\tvalidation_0-merror:0.19912\n",
      "[71]\tvalidation_0-merror:0.19871\n",
      "[72]\tvalidation_0-merror:0.19857\n",
      "[73]\tvalidation_0-merror:0.19834\n",
      "[74]\tvalidation_0-merror:0.19804\n",
      "[75]\tvalidation_0-merror:0.19808\n",
      "[76]\tvalidation_0-merror:0.19775\n",
      "[77]\tvalidation_0-merror:0.19771\n",
      "[78]\tvalidation_0-merror:0.19753\n",
      "[79]\tvalidation_0-merror:0.19729\n",
      "[80]\tvalidation_0-merror:0.19698\n",
      "[81]\tvalidation_0-merror:0.19696\n",
      "[82]\tvalidation_0-merror:0.19702\n",
      "[83]\tvalidation_0-merror:0.19699\n",
      "[84]\tvalidation_0-merror:0.19676\n",
      "[85]\tvalidation_0-merror:0.19669\n",
      "[86]\tvalidation_0-merror:0.19663\n",
      "[87]\tvalidation_0-merror:0.19641\n",
      "[88]\tvalidation_0-merror:0.19632\n",
      "[89]\tvalidation_0-merror:0.19628\n",
      "[90]\tvalidation_0-merror:0.19611\n",
      "[91]\tvalidation_0-merror:0.19601\n",
      "[92]\tvalidation_0-merror:0.19586\n",
      "[93]\tvalidation_0-merror:0.19556\n",
      "[94]\tvalidation_0-merror:0.19542\n",
      "[95]\tvalidation_0-merror:0.19538\n",
      "[96]\tvalidation_0-merror:0.19531\n",
      "[97]\tvalidation_0-merror:0.19514\n",
      "[98]\tvalidation_0-merror:0.19508\n",
      "[99]\tvalidation_0-merror:0.19492\n",
      "[100]\tvalidation_0-merror:0.19481\n",
      "[101]\tvalidation_0-merror:0.19459\n",
      "[102]\tvalidation_0-merror:0.19458\n",
      "[103]\tvalidation_0-merror:0.19434\n",
      "[104]\tvalidation_0-merror:0.19416\n",
      "[105]\tvalidation_0-merror:0.19407\n",
      "[106]\tvalidation_0-merror:0.19396\n",
      "[107]\tvalidation_0-merror:0.19375\n",
      "[108]\tvalidation_0-merror:0.19356\n",
      "[109]\tvalidation_0-merror:0.19359\n",
      "[110]\tvalidation_0-merror:0.19341\n",
      "[111]\tvalidation_0-merror:0.19324\n",
      "[112]\tvalidation_0-merror:0.19301\n",
      "[113]\tvalidation_0-merror:0.19291\n",
      "[114]\tvalidation_0-merror:0.19277\n",
      "[115]\tvalidation_0-merror:0.19271\n",
      "[116]\tvalidation_0-merror:0.19252\n",
      "[117]\tvalidation_0-merror:0.19237\n",
      "[118]\tvalidation_0-merror:0.19230\n",
      "[119]\tvalidation_0-merror:0.19205\n",
      "[120]\tvalidation_0-merror:0.19196\n",
      "[121]\tvalidation_0-merror:0.19179\n",
      "[122]\tvalidation_0-merror:0.19176\n",
      "[123]\tvalidation_0-merror:0.19184\n",
      "[124]\tvalidation_0-merror:0.19169\n",
      "[125]\tvalidation_0-merror:0.19149\n",
      "[126]\tvalidation_0-merror:0.19147\n",
      "[127]\tvalidation_0-merror:0.19129\n",
      "[128]\tvalidation_0-merror:0.19123\n",
      "[129]\tvalidation_0-merror:0.19119\n",
      "[130]\tvalidation_0-merror:0.19105\n",
      "[131]\tvalidation_0-merror:0.19097\n",
      "[132]\tvalidation_0-merror:0.19077\n",
      "[133]\tvalidation_0-merror:0.19065\n",
      "[134]\tvalidation_0-merror:0.19045\n",
      "[135]\tvalidation_0-merror:0.19024\n",
      "[136]\tvalidation_0-merror:0.19006\n",
      "[137]\tvalidation_0-merror:0.18999\n",
      "[138]\tvalidation_0-merror:0.18980\n",
      "[139]\tvalidation_0-merror:0.18968\n",
      "[140]\tvalidation_0-merror:0.18956\n",
      "[141]\tvalidation_0-merror:0.18958\n",
      "[142]\tvalidation_0-merror:0.18947\n",
      "[143]\tvalidation_0-merror:0.18946\n",
      "[144]\tvalidation_0-merror:0.18924\n",
      "[145]\tvalidation_0-merror:0.18918\n",
      "[146]\tvalidation_0-merror:0.18908\n",
      "[147]\tvalidation_0-merror:0.18894\n",
      "[148]\tvalidation_0-merror:0.18882\n",
      "[149]\tvalidation_0-merror:0.18889\n",
      "[150]\tvalidation_0-merror:0.18862\n",
      "[151]\tvalidation_0-merror:0.18839\n",
      "[152]\tvalidation_0-merror:0.18836\n",
      "[153]\tvalidation_0-merror:0.18822\n",
      "[154]\tvalidation_0-merror:0.18802\n",
      "[155]\tvalidation_0-merror:0.18798\n",
      "[156]\tvalidation_0-merror:0.18796\n",
      "[157]\tvalidation_0-merror:0.18782\n",
      "[158]\tvalidation_0-merror:0.18760\n",
      "[159]\tvalidation_0-merror:0.18759\n",
      "[160]\tvalidation_0-merror:0.18752\n",
      "[161]\tvalidation_0-merror:0.18735\n",
      "[162]\tvalidation_0-merror:0.18725\n",
      "[163]\tvalidation_0-merror:0.18716\n",
      "[164]\tvalidation_0-merror:0.18705\n",
      "[165]\tvalidation_0-merror:0.18693\n",
      "[166]\tvalidation_0-merror:0.18681\n",
      "[167]\tvalidation_0-merror:0.18673\n",
      "[168]\tvalidation_0-merror:0.18655\n",
      "[169]\tvalidation_0-merror:0.18649\n",
      "[170]\tvalidation_0-merror:0.18648\n",
      "[171]\tvalidation_0-merror:0.18633\n",
      "[172]\tvalidation_0-merror:0.18628\n",
      "[173]\tvalidation_0-merror:0.18609\n",
      "[174]\tvalidation_0-merror:0.18612\n",
      "[175]\tvalidation_0-merror:0.18604\n",
      "[176]\tvalidation_0-merror:0.18596\n",
      "[177]\tvalidation_0-merror:0.18591\n",
      "[178]\tvalidation_0-merror:0.18593\n",
      "[179]\tvalidation_0-merror:0.18590\n",
      "[180]\tvalidation_0-merror:0.18576\n",
      "[181]\tvalidation_0-merror:0.18574\n",
      "[182]\tvalidation_0-merror:0.18563\n",
      "[183]\tvalidation_0-merror:0.18566\n",
      "[184]\tvalidation_0-merror:0.18562\n",
      "[185]\tvalidation_0-merror:0.18563\n",
      "[186]\tvalidation_0-merror:0.18561\n",
      "[187]\tvalidation_0-merror:0.18560\n",
      "[188]\tvalidation_0-merror:0.18557\n",
      "[189]\tvalidation_0-merror:0.18543\n",
      "[190]\tvalidation_0-merror:0.18529\n",
      "[191]\tvalidation_0-merror:0.18518\n",
      "[192]\tvalidation_0-merror:0.18505\n",
      "[193]\tvalidation_0-merror:0.18487\n",
      "[194]\tvalidation_0-merror:0.18469\n",
      "[195]\tvalidation_0-merror:0.18459\n",
      "[196]\tvalidation_0-merror:0.18444\n",
      "[197]\tvalidation_0-merror:0.18430\n",
      "[198]\tvalidation_0-merror:0.18425\n",
      "[199]\tvalidation_0-merror:0.18417\n",
      "Wall time: 12min 30s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "              colsample_bynode=1, colsample_bytree=1, gamma=0.1, gpu_id=-1,\n",
       "              importance_type='gain', interaction_constraints='',\n",
       "              learning_rate=0.01, max_delta_step=0, max_depth=8,\n",
       "              min_child_weight=1, missing=nan, monotone_constraints='()',\n",
       "              n_estimators=200, n_jobs=-1, nthread=-1, num_parallel_tree=1,\n",
       "              object='multi:softprob', objective='multi:softprob',\n",
       "              random_state=0, reg_alpha=0, reg_lambda=1, scale_pos_weight=None,\n",
       "              subsample=0.9, tree_method='exact', validate_parameters=1,\n",
       "              verbosity=None)"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "xgb_cpu.fit(covertype_x_train,\n",
    "            covertype_y_train,\n",
    "            eval_set=[(covertype_x_val, covertype_y_val)],\n",
    "            eval_metric='merror',\n",
    "            early_stopping_rounds=25,\n",
    "            verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb_gpu = xgb.XGBClassifier(object='multi:softprob',\n",
    "                            max_depth=8,\n",
    "                            gamma=0.1,\n",
    "                            subsample=0.9,\n",
    "                            learning_rate=0.01,\n",
    "                            n_estimators=200,\n",
    "                            nthread=-1,\n",
    "                            tree_method='gpu_hist',\n",
    "                            gpu_id=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[10:01:00] WARNING: D:\\Projects\\other_projects\\xgboost\\src\\learner.cc:529: \n",
      "Parameters: { object } might not be used.\n",
      "\n",
      "  This may not be accurate due to some parameters are only used in language bindings but\n",
      "  passed down to XGBoost core.  Or some parameters are not used but slip through this\n",
      "  verification. Please open an issue if you find above cases.\n",
      "\n",
      "\n",
      "[0]\tvalidation_0-merror:0.21819\n",
      "[1]\tvalidation_0-merror:0.21678\n",
      "[2]\tvalidation_0-merror:0.21659\n",
      "[3]\tvalidation_0-merror:0.21447\n",
      "[4]\tvalidation_0-merror:0.21370\n",
      "[5]\tvalidation_0-merror:0.21469\n",
      "[6]\tvalidation_0-merror:0.21351\n",
      "[7]\tvalidation_0-merror:0.21265\n",
      "[8]\tvalidation_0-merror:0.21252\n",
      "[9]\tvalidation_0-merror:0.21265\n",
      "[10]\tvalidation_0-merror:0.21286\n",
      "[11]\tvalidation_0-merror:0.21154\n",
      "[12]\tvalidation_0-merror:0.21166\n",
      "[13]\tvalidation_0-merror:0.21123\n",
      "[14]\tvalidation_0-merror:0.21070\n",
      "[15]\tvalidation_0-merror:0.21067\n",
      "[16]\tvalidation_0-merror:0.21046\n",
      "[17]\tvalidation_0-merror:0.21040\n",
      "[18]\tvalidation_0-merror:0.21009\n",
      "[19]\tvalidation_0-merror:0.20922\n",
      "[20]\tvalidation_0-merror:0.20916\n",
      "[21]\tvalidation_0-merror:0.20910\n",
      "[22]\tvalidation_0-merror:0.20902\n",
      "[23]\tvalidation_0-merror:0.20827\n",
      "[24]\tvalidation_0-merror:0.20802\n",
      "[25]\tvalidation_0-merror:0.20764\n",
      "[26]\tvalidation_0-merror:0.20784\n",
      "[27]\tvalidation_0-merror:0.20749\n",
      "[28]\tvalidation_0-merror:0.20775\n",
      "[29]\tvalidation_0-merror:0.20767\n",
      "[30]\tvalidation_0-merror:0.20751\n",
      "[31]\tvalidation_0-merror:0.20745\n",
      "[32]\tvalidation_0-merror:0.20728\n",
      "[33]\tvalidation_0-merror:0.20687\n",
      "[34]\tvalidation_0-merror:0.20703\n",
      "[35]\tvalidation_0-merror:0.20666\n",
      "[36]\tvalidation_0-merror:0.20648\n",
      "[37]\tvalidation_0-merror:0.20643\n",
      "[38]\tvalidation_0-merror:0.20605\n",
      "[39]\tvalidation_0-merror:0.20585\n",
      "[40]\tvalidation_0-merror:0.20589\n",
      "[41]\tvalidation_0-merror:0.20584\n",
      "[42]\tvalidation_0-merror:0.20589\n",
      "[43]\tvalidation_0-merror:0.20549\n",
      "[44]\tvalidation_0-merror:0.20519\n",
      "[45]\tvalidation_0-merror:0.20531\n",
      "[46]\tvalidation_0-merror:0.20495\n",
      "[47]\tvalidation_0-merror:0.20471\n",
      "[48]\tvalidation_0-merror:0.20454\n",
      "[49]\tvalidation_0-merror:0.20448\n",
      "[50]\tvalidation_0-merror:0.20423\n",
      "[51]\tvalidation_0-merror:0.20401\n",
      "[52]\tvalidation_0-merror:0.20385\n",
      "[53]\tvalidation_0-merror:0.20358\n",
      "[54]\tvalidation_0-merror:0.20333\n",
      "[55]\tvalidation_0-merror:0.20316\n",
      "[56]\tvalidation_0-merror:0.20291\n",
      "[57]\tvalidation_0-merror:0.20253\n",
      "[58]\tvalidation_0-merror:0.20228\n",
      "[59]\tvalidation_0-merror:0.20187\n",
      "[60]\tvalidation_0-merror:0.20142\n",
      "[61]\tvalidation_0-merror:0.20136\n",
      "[62]\tvalidation_0-merror:0.20111\n",
      "[63]\tvalidation_0-merror:0.20089\n",
      "[64]\tvalidation_0-merror:0.20067\n",
      "[65]\tvalidation_0-merror:0.20047\n",
      "[66]\tvalidation_0-merror:0.20017\n",
      "[67]\tvalidation_0-merror:0.19994\n",
      "[68]\tvalidation_0-merror:0.19975\n",
      "[69]\tvalidation_0-merror:0.19975\n",
      "[70]\tvalidation_0-merror:0.19958\n",
      "[71]\tvalidation_0-merror:0.19926\n",
      "[72]\tvalidation_0-merror:0.19913\n",
      "[73]\tvalidation_0-merror:0.19890\n",
      "[74]\tvalidation_0-merror:0.19879\n",
      "[75]\tvalidation_0-merror:0.19861\n",
      "[76]\tvalidation_0-merror:0.19858\n",
      "[77]\tvalidation_0-merror:0.19837\n",
      "[78]\tvalidation_0-merror:0.19821\n",
      "[79]\tvalidation_0-merror:0.19818\n",
      "[80]\tvalidation_0-merror:0.19795\n",
      "[81]\tvalidation_0-merror:0.19770\n",
      "[82]\tvalidation_0-merror:0.19742\n",
      "[83]\tvalidation_0-merror:0.19751\n",
      "[84]\tvalidation_0-merror:0.19728\n",
      "[85]\tvalidation_0-merror:0.19732\n",
      "[86]\tvalidation_0-merror:0.19709\n",
      "[87]\tvalidation_0-merror:0.19709\n",
      "[88]\tvalidation_0-merror:0.19709\n",
      "[89]\tvalidation_0-merror:0.19672\n",
      "[90]\tvalidation_0-merror:0.19663\n",
      "[91]\tvalidation_0-merror:0.19663\n",
      "[92]\tvalidation_0-merror:0.19652\n",
      "[93]\tvalidation_0-merror:0.19638\n",
      "[94]\tvalidation_0-merror:0.19605\n",
      "[95]\tvalidation_0-merror:0.19586\n",
      "[96]\tvalidation_0-merror:0.19579\n",
      "[97]\tvalidation_0-merror:0.19562\n",
      "[98]\tvalidation_0-merror:0.19545\n",
      "[99]\tvalidation_0-merror:0.19536\n",
      "[100]\tvalidation_0-merror:0.19508\n",
      "[101]\tvalidation_0-merror:0.19495\n",
      "[102]\tvalidation_0-merror:0.19479\n",
      "[103]\tvalidation_0-merror:0.19463\n",
      "[104]\tvalidation_0-merror:0.19450\n",
      "[105]\tvalidation_0-merror:0.19431\n",
      "[106]\tvalidation_0-merror:0.19416\n",
      "[107]\tvalidation_0-merror:0.19395\n",
      "[108]\tvalidation_0-merror:0.19381\n",
      "[109]\tvalidation_0-merror:0.19361\n",
      "[110]\tvalidation_0-merror:0.19346\n",
      "[111]\tvalidation_0-merror:0.19323\n",
      "[112]\tvalidation_0-merror:0.19314\n",
      "[113]\tvalidation_0-merror:0.19298\n",
      "[114]\tvalidation_0-merror:0.19273\n",
      "[115]\tvalidation_0-merror:0.19260\n",
      "[116]\tvalidation_0-merror:0.19255\n",
      "[117]\tvalidation_0-merror:0.19234\n",
      "[118]\tvalidation_0-merror:0.19215\n",
      "[119]\tvalidation_0-merror:0.19194\n",
      "[120]\tvalidation_0-merror:0.19187\n",
      "[121]\tvalidation_0-merror:0.19175\n",
      "[122]\tvalidation_0-merror:0.19167\n",
      "[123]\tvalidation_0-merror:0.19154\n",
      "[124]\tvalidation_0-merror:0.19145\n",
      "[125]\tvalidation_0-merror:0.19131\n",
      "[126]\tvalidation_0-merror:0.19121\n",
      "[127]\tvalidation_0-merror:0.19118\n",
      "[128]\tvalidation_0-merror:0.19099\n",
      "[129]\tvalidation_0-merror:0.19107\n",
      "[130]\tvalidation_0-merror:0.19086\n",
      "[131]\tvalidation_0-merror:0.19081\n",
      "[132]\tvalidation_0-merror:0.19071\n",
      "[133]\tvalidation_0-merror:0.19055\n",
      "[134]\tvalidation_0-merror:0.19039\n",
      "[135]\tvalidation_0-merror:0.19022\n",
      "[136]\tvalidation_0-merror:0.18998\n",
      "[137]\tvalidation_0-merror:0.18983\n",
      "[138]\tvalidation_0-merror:0.18975\n",
      "[139]\tvalidation_0-merror:0.18975\n",
      "[140]\tvalidation_0-merror:0.18959\n",
      "[141]\tvalidation_0-merror:0.18951\n",
      "[142]\tvalidation_0-merror:0.18946\n",
      "[143]\tvalidation_0-merror:0.18926\n",
      "[144]\tvalidation_0-merror:0.18914\n",
      "[145]\tvalidation_0-merror:0.18906\n",
      "[146]\tvalidation_0-merror:0.18895\n",
      "[147]\tvalidation_0-merror:0.18879\n",
      "[148]\tvalidation_0-merror:0.18876\n",
      "[149]\tvalidation_0-merror:0.18872\n",
      "[150]\tvalidation_0-merror:0.18855\n",
      "[151]\tvalidation_0-merror:0.18843\n",
      "[152]\tvalidation_0-merror:0.18826\n",
      "[153]\tvalidation_0-merror:0.18827\n",
      "[154]\tvalidation_0-merror:0.18821\n",
      "[155]\tvalidation_0-merror:0.18802\n",
      "[156]\tvalidation_0-merror:0.18796\n",
      "[157]\tvalidation_0-merror:0.18780\n",
      "[158]\tvalidation_0-merror:0.18777\n",
      "[159]\tvalidation_0-merror:0.18762\n",
      "[160]\tvalidation_0-merror:0.18740\n",
      "[161]\tvalidation_0-merror:0.18740\n",
      "[162]\tvalidation_0-merror:0.18740\n",
      "[163]\tvalidation_0-merror:0.18730\n",
      "[164]\tvalidation_0-merror:0.18717\n",
      "[165]\tvalidation_0-merror:0.18697\n",
      "[166]\tvalidation_0-merror:0.18678\n",
      "[167]\tvalidation_0-merror:0.18668\n",
      "[168]\tvalidation_0-merror:0.18648\n",
      "[169]\tvalidation_0-merror:0.18648\n",
      "[170]\tvalidation_0-merror:0.18658\n",
      "[171]\tvalidation_0-merror:0.18648\n",
      "[172]\tvalidation_0-merror:0.18636\n",
      "[173]\tvalidation_0-merror:0.18634\n",
      "[174]\tvalidation_0-merror:0.18631\n",
      "[175]\tvalidation_0-merror:0.18632\n",
      "[176]\tvalidation_0-merror:0.18632\n",
      "[177]\tvalidation_0-merror:0.18617\n",
      "[178]\tvalidation_0-merror:0.18617\n",
      "[179]\tvalidation_0-merror:0.18617\n",
      "[180]\tvalidation_0-merror:0.18617\n",
      "[181]\tvalidation_0-merror:0.18605\n",
      "[182]\tvalidation_0-merror:0.18593\n",
      "[183]\tvalidation_0-merror:0.18597\n",
      "[184]\tvalidation_0-merror:0.18588\n",
      "[185]\tvalidation_0-merror:0.18580\n",
      "[186]\tvalidation_0-merror:0.18582\n",
      "[187]\tvalidation_0-merror:0.18574\n",
      "[188]\tvalidation_0-merror:0.18555\n",
      "[189]\tvalidation_0-merror:0.18548\n",
      "[190]\tvalidation_0-merror:0.18537\n",
      "[191]\tvalidation_0-merror:0.18524\n",
      "[192]\tvalidation_0-merror:0.18519\n",
      "[193]\tvalidation_0-merror:0.18520\n",
      "[194]\tvalidation_0-merror:0.18508\n",
      "[195]\tvalidation_0-merror:0.18502\n",
      "[196]\tvalidation_0-merror:0.18502\n",
      "[197]\tvalidation_0-merror:0.18483\n",
      "[198]\tvalidation_0-merror:0.18473\n",
      "[199]\tvalidation_0-merror:0.18468\n",
      "Wall time: 1min 54s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "              colsample_bynode=1, colsample_bytree=1, gamma=0.1, gpu_id=0,\n",
       "              importance_type='gain', interaction_constraints='',\n",
       "              learning_rate=0.01, max_delta_step=0, max_depth=8,\n",
       "              min_child_weight=1, missing=nan, monotone_constraints='()',\n",
       "              n_estimators=200, n_jobs=-1, nthread=-1, num_parallel_tree=1,\n",
       "              object='multi:softprob', objective='multi:softprob',\n",
       "              random_state=0, reg_alpha=0, reg_lambda=1, scale_pos_weight=None,\n",
       "              subsample=0.9, tree_method='gpu_hist', validate_parameters=1,\n",
       "              verbosity=None)"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "xgb_gpu.fit(covertype_x_train,\n",
    "            covertype_y_train,\n",
    "            eval_set=[(covertype_x_val, covertype_y_val)],\n",
    "            eval_metric='merror',\n",
    "            early_stopping_rounds=25,\n",
    "            verbose=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LightGBM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "covertype_y_train_1 = covertype_y_train - 1\n",
    "covertype_y_val_1 = covertype_y_val - 1\n",
    "covertype_y_test_1 = covertype_y_test - 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import lightgbm as lgb\n",
    "\n",
    "params = {\n",
    "    'task': 'train',\n",
    "    'boosting_type': 'gbdt',\n",
    "    'objective': 'multiclass',\n",
    "    'num_class': len(np.unique(covertype_y)),\n",
    "    'metric': 'multi_logloss',\n",
    "    'learning_rate': 0.01,\n",
    "    'max_depth': 128,\n",
    "    'num_leaves': 256,\n",
    "    'feature_fraction': 0.9,\n",
    "    'bagging_fraction': 0.9,\n",
    "    'bagging_freq': 10,\n",
    "    'device': 'cpu'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = lgb.Dataset(data=covertype_x_train, label=covertype_y_train_1)\n",
    "val_data = lgb.Dataset(data=covertype_x_val, label=covertype_y_val_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 25 rounds\n",
      "[500]\tvalid_0's multi_logloss: 0.258546\n",
      "[1000]\tvalid_0's multi_logloss: 0.188758\n",
      "[1500]\tvalid_0's multi_logloss: 0.159213\n",
      "[2000]\tvalid_0's multi_logloss: 0.140833\n",
      "[2500]\tvalid_0's multi_logloss: 0.127889\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[2500]\tvalid_0's multi_logloss: 0.127889\n"
     ]
    }
   ],
   "source": [
    "bst = lgb.train(params,\n",
    "                train_data,\n",
    "                num_boost_round=2500,\n",
    "                valid_sets=val_data,\n",
    "                verbose_eval=500,\n",
    "                early_stopping_rounds=25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[250]\tcv_agg's multi_logloss: 0.383821 + 0.000454662\n",
      "[500]\tcv_agg's multi_logloss: 0.265271 + 0.000599288\n",
      "[750]\tcv_agg's multi_logloss: 0.220832 + 0.000458877\n",
      "[1000]\tcv_agg's multi_logloss: 0.196948 + 0.000144783\n",
      "[1250]\tcv_agg's multi_logloss: 0.180572 + 0.000345486\n",
      "[1500]\tcv_agg's multi_logloss: 0.167917 + 0.000484253\n",
      "[1750]\tcv_agg's multi_logloss: 0.157947 + 0.000538372\n",
      "[2000]\tcv_agg's multi_logloss: 0.150243 + 0.000658097\n",
      "[2250]\tcv_agg's multi_logloss: 0.143914 + 0.000544808\n",
      "[2500]\tcv_agg's multi_logloss: 0.139016 + 0.000600344\n"
     ]
    }
   ],
   "source": [
    "lgb_cv = lgb.cv(params,\n",
    "                train_data,\n",
    "                num_boost_round=2500,\n",
    "                nfold=3,\n",
    "                shuffle=True,\n",
    "                verbose_eval=250,\n",
    "                early_stopping_rounds=25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2499"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nround = lgb_cv['multi_logloss-mean'].index(np.min(\n",
    "    lgb_cv['multi_logloss-mean']))\n",
    "nround"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9591917522934201\n",
      "[[40188  2142     0     0    14     0    71]\n",
      " [ 1442 54835    68     0    84    45    21]\n",
      " [    2    61  7011    23     5    92     0]\n",
      " [    0     0    58   455     0    16     0]\n",
      " [   20   231    27     0  1689     6     0]\n",
      " [    2    70   110    10     3  3351     0]\n",
      " [  107    12     0     0     0     0  3931]]\n"
     ]
    }
   ],
   "source": [
    "y_probs = bst.predict(covertype_x_test, num_iteration=bst.best_iteration)\n",
    "y_preds = np.argmax(y_probs, axis=1)\n",
    "print(accuracy_score(covertype_y_test_1, y_preds))\n",
    "print(confusion_matrix(covertype_y_test_1, y_preds))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      0.95      0.95     42415\n",
      "           1       0.96      0.97      0.96     56495\n",
      "           2       0.96      0.97      0.97      7194\n",
      "           3       0.93      0.86      0.89       529\n",
      "           4       0.94      0.86      0.90      1973\n",
      "           5       0.95      0.95      0.95      3546\n",
      "           6       0.98      0.97      0.97      4050\n",
      "\n",
      "    accuracy                           0.96    116202\n",
      "   macro avg       0.96      0.93      0.94    116202\n",
      "weighted avg       0.96      0.96      0.96    116202\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(covertype_y_test_1, y_preds))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LightGBM GPU vs CPU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "import lightgbm as lgb\n",
    "\n",
    "params_cpu = {\n",
    "    'task': 'train',\n",
    "    'boosting_type': 'gbdt',\n",
    "    'objective': 'multiclass',\n",
    "    'num_class': len(np.unique(covertype_y)),\n",
    "    'metric': 'multi_logloss',\n",
    "    'learning_rate': 0.01,\n",
    "    'max_depth': 128,\n",
    "    'num_leaves': 256,\n",
    "    'feature_fraction': 0.9,\n",
    "    'bagging_fraction': 0.9,\n",
    "    'bagging_freq': 10,\n",
    "    \"n_jobs\": 8,\n",
    "    'device': 'cpu',\n",
    "    'n_jobs': -1,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] Auto-choosing col-wise multi-threading, the overhead of testing was 0.030191 seconds.\n",
      "You can set `force_col_wise=true` to remove the overhead.\n",
      "[LightGBM] [Info] Total Bins 2266\n",
      "[LightGBM] [Info] Number of data points in the train set: 348607, number of used features: 53\n",
      "[LightGBM] [Info] Start training from score -1.008130\n",
      "[LightGBM] [Info] Start training from score -0.718670\n",
      "[LightGBM] [Info] Start training from score -2.786590\n",
      "[LightGBM] [Info] Start training from score -5.333962\n",
      "[LightGBM] [Info] Start training from score -4.133503\n",
      "[LightGBM] [Info] Start training from score -3.514835\n",
      "[LightGBM] [Info] Start training from score -3.339237\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "Training until validation scores don't improve for 25 rounds\n",
      "[100]\tvalid_0's multi_logloss: 0.46432\n",
      "[200]\tvalid_0's multi_logloss: 0.316533\n",
      "[300]\tvalid_0's multi_logloss: 0.254985\n",
      "[400]\tvalid_0's multi_logloss: 0.222611\n",
      "[500]\tvalid_0's multi_logloss: 0.201845\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[500]\tvalid_0's multi_logloss: 0.201845\n",
      "Wall time: 2min 39s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "bst_cpu = lgb.train(params_cpu,\n",
    "                    train_data,\n",
    "                    num_boost_round=500,\n",
    "                    valid_sets=val_data,\n",
    "                    verbose_eval=100,\n",
    "                    early_stopping_rounds=25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "params_gpu = {\n",
    "    'task': 'train',\n",
    "    'boosting_type': 'gbdt',\n",
    "    'objective': 'multiclass',\n",
    "    'num_class': len(np.unique(covertype_y)),\n",
    "    'metric': 'multi_logloss',\n",
    "    'learning_rate': 0.01,\n",
    "    'max_depth': 128,\n",
    "    'num_leaves': 256,\n",
    "    'feature_fraction': 0.9,\n",
    "    'bagging_fraction': 0.9,\n",
    "    'bagging_freq': 10,\n",
    "    \"n_jobs\": 8,\n",
    "    'device': 'gpu',\n",
    "    'gpu_platform_id': 2,\n",
    "    'gpu_device_id': 1,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Info] This is the GPU trainer!!\n",
      "[LightGBM] [Info] Total Bins 2266\n",
      "[LightGBM] [Info] Number of data points in the train set: 348607, number of used features: 53\n",
      "[LightGBM] [Info] Using GPU Device: GeForce GTX 1060, Vendor: NVIDIA Corporation\n",
      "[LightGBM] [Info] Compiling OpenCL Kernel with 256 bins...\n",
      "[LightGBM] [Info] GPU programs have been built\n",
      "[LightGBM] [Info] Size of histogram bin entry: 8\n",
      "[LightGBM] [Info] 12 dense feature groups (3.99 MB) transferred to GPU in 0.008094 secs. 0 sparse feature groups\n",
      "[LightGBM] [Info] Start training from score -1.008130\n",
      "[LightGBM] [Info] Start training from score -0.718670\n",
      "[LightGBM] [Info] Start training from score -2.786590\n",
      "[LightGBM] [Info] Start training from score -5.333962\n",
      "[LightGBM] [Info] Start training from score -4.133503\n",
      "[LightGBM] [Info] Start training from score -3.514835\n",
      "[LightGBM] [Info] Start training from score -3.339237\n",
      "[LightGBM] [Info] Size of histogram bin entry: 8\n",
      "[LightGBM] [Info] 12 dense feature groups (3.59 MB) transferred to GPU in 0.008006 secs. 0 sparse feature groups\n",
      "Training until validation scores don't improve for 25 rounds\n",
      "[LightGBM] [Info] Size of histogram bin entry: 8\n",
      "[LightGBM] [Info] 12 dense feature groups (3.59 MB) transferred to GPU in 0.007137 secs. 0 sparse feature groups\n",
      "[LightGBM] [Info] Size of histogram bin entry: 8\n",
      "[LightGBM] [Info] 12 dense feature groups (3.59 MB) transferred to GPU in 0.007045 secs. 0 sparse feature groups\n",
      "[LightGBM] [Info] Size of histogram bin entry: 8\n",
      "[LightGBM] [Info] 12 dense feature groups (3.59 MB) transferred to GPU in 0.007813 secs. 0 sparse feature groups\n",
      "[LightGBM] [Info] Size of histogram bin entry: 8\n",
      "[LightGBM] [Info] 12 dense feature groups (3.59 MB) transferred to GPU in 0.007583 secs. 0 sparse feature groups\n",
      "[LightGBM] [Info] Size of histogram bin entry: 8\n",
      "[LightGBM] [Info] 12 dense feature groups (3.59 MB) transferred to GPU in 0.007343 secs. 0 sparse feature groups\n",
      "[LightGBM] [Info] Size of histogram bin entry: 8\n",
      "[LightGBM] [Info] 12 dense feature groups (3.59 MB) transferred to GPU in 0.007228 secs. 0 sparse feature groups\n",
      "[LightGBM] [Info] Size of histogram bin entry: 8\n",
      "[LightGBM] [Info] 12 dense feature groups (3.59 MB) transferred to GPU in 0.007928 secs. 0 sparse feature groups\n",
      "[LightGBM] [Info] Size of histogram bin entry: 8\n",
      "[LightGBM] [Info] 12 dense feature groups (3.59 MB) transferred to GPU in 0.007948 secs. 0 sparse feature groups\n",
      "[LightGBM] [Info] Size of histogram bin entry: 8\n",
      "[LightGBM] [Info] 12 dense feature groups (3.59 MB) transferred to GPU in 0.008081 secs. 0 sparse feature groups\n",
      "[100]\tvalid_0's multi_logloss: 0.464445\n",
      "[LightGBM] [Info] Size of histogram bin entry: 8\n",
      "[LightGBM] [Info] 12 dense feature groups (3.59 MB) transferred to GPU in 0.007430 secs. 0 sparse feature groups\n",
      "[LightGBM] [Info] Size of histogram bin entry: 8\n",
      "[LightGBM] [Info] 12 dense feature groups (3.59 MB) transferred to GPU in 0.008532 secs. 0 sparse feature groups\n",
      "[LightGBM] [Info] Size of histogram bin entry: 8\n",
      "[LightGBM] [Info] 12 dense feature groups (3.59 MB) transferred to GPU in 0.008299 secs. 0 sparse feature groups\n",
      "[LightGBM] [Info] Size of histogram bin entry: 8\n",
      "[LightGBM] [Info] 12 dense feature groups (3.59 MB) transferred to GPU in 0.008420 secs. 0 sparse feature groups\n",
      "[LightGBM] [Info] Size of histogram bin entry: 8\n",
      "[LightGBM] [Info] 12 dense feature groups (3.59 MB) transferred to GPU in 0.008844 secs. 0 sparse feature groups\n",
      "[LightGBM] [Info] Size of histogram bin entry: 8\n",
      "[LightGBM] [Info] 12 dense feature groups (3.59 MB) transferred to GPU in 0.007474 secs. 0 sparse feature groups\n",
      "[LightGBM] [Info] Size of histogram bin entry: 8\n",
      "[LightGBM] [Info] 12 dense feature groups (3.59 MB) transferred to GPU in 0.009012 secs. 0 sparse feature groups\n",
      "[LightGBM] [Info] Size of histogram bin entry: 8\n",
      "[LightGBM] [Info] 12 dense feature groups (3.59 MB) transferred to GPU in 0.008419 secs. 0 sparse feature groups\n",
      "[LightGBM] [Info] Size of histogram bin entry: 8\n",
      "[LightGBM] [Info] 12 dense feature groups (3.59 MB) transferred to GPU in 0.007661 secs. 0 sparse feature groups\n",
      "[LightGBM] [Info] Size of histogram bin entry: 8\n",
      "[LightGBM] [Info] 12 dense feature groups (3.59 MB) transferred to GPU in 0.007852 secs. 0 sparse feature groups\n",
      "[200]\tvalid_0's multi_logloss: 0.316437\n",
      "[LightGBM] [Info] Size of histogram bin entry: 8\n",
      "[LightGBM] [Info] 12 dense feature groups (3.59 MB) transferred to GPU in 0.008884 secs. 0 sparse feature groups\n",
      "[LightGBM] [Info] Size of histogram bin entry: 8\n",
      "[LightGBM] [Info] 12 dense feature groups (3.59 MB) transferred to GPU in 0.008355 secs. 0 sparse feature groups\n",
      "[LightGBM] [Info] Size of histogram bin entry: 8\n",
      "[LightGBM] [Info] 12 dense feature groups (3.59 MB) transferred to GPU in 0.007115 secs. 0 sparse feature groups\n",
      "[LightGBM] [Info] Size of histogram bin entry: 8\n",
      "[LightGBM] [Info] 12 dense feature groups (3.59 MB) transferred to GPU in 0.006948 secs. 0 sparse feature groups\n",
      "[LightGBM] [Info] Size of histogram bin entry: 8\n",
      "[LightGBM] [Info] 12 dense feature groups (3.59 MB) transferred to GPU in 0.007195 secs. 0 sparse feature groups\n",
      "[LightGBM] [Info] Size of histogram bin entry: 8\n",
      "[LightGBM] [Info] 12 dense feature groups (3.59 MB) transferred to GPU in 0.007332 secs. 0 sparse feature groups\n",
      "[LightGBM] [Info] Size of histogram bin entry: 8\n",
      "[LightGBM] [Info] 12 dense feature groups (3.59 MB) transferred to GPU in 0.007107 secs. 0 sparse feature groups\n",
      "[LightGBM] [Info] Size of histogram bin entry: 8\n",
      "[LightGBM] [Info] 12 dense feature groups (3.59 MB) transferred to GPU in 0.007296 secs. 0 sparse feature groups\n",
      "[LightGBM] [Info] Size of histogram bin entry: 8\n",
      "[LightGBM] [Info] 12 dense feature groups (3.59 MB) transferred to GPU in 0.007101 secs. 0 sparse feature groups\n",
      "[LightGBM] [Info] Size of histogram bin entry: 8\n",
      "[LightGBM] [Info] 12 dense feature groups (3.59 MB) transferred to GPU in 0.007279 secs. 0 sparse feature groups\n",
      "[300]\tvalid_0's multi_logloss: 0.255073\n",
      "[LightGBM] [Info] Size of histogram bin entry: 8\n",
      "[LightGBM] [Info] 12 dense feature groups (3.59 MB) transferred to GPU in 0.006899 secs. 0 sparse feature groups\n",
      "[LightGBM] [Info] Size of histogram bin entry: 8\n",
      "[LightGBM] [Info] 12 dense feature groups (3.59 MB) transferred to GPU in 0.007245 secs. 0 sparse feature groups\n",
      "[LightGBM] [Info] Size of histogram bin entry: 8\n",
      "[LightGBM] [Info] 12 dense feature groups (3.59 MB) transferred to GPU in 0.006943 secs. 0 sparse feature groups\n",
      "[LightGBM] [Info] Size of histogram bin entry: 8\n",
      "[LightGBM] [Info] 12 dense feature groups (3.59 MB) transferred to GPU in 0.008147 secs. 0 sparse feature groups\n",
      "[LightGBM] [Info] Size of histogram bin entry: 8\n",
      "[LightGBM] [Info] 12 dense feature groups (3.59 MB) transferred to GPU in 0.007125 secs. 0 sparse feature groups\n",
      "[LightGBM] [Info] Size of histogram bin entry: 8\n",
      "[LightGBM] [Info] 12 dense feature groups (3.59 MB) transferred to GPU in 0.008344 secs. 0 sparse feature groups\n",
      "[LightGBM] [Info] Size of histogram bin entry: 8\n",
      "[LightGBM] [Info] 12 dense feature groups (3.59 MB) transferred to GPU in 0.007181 secs. 0 sparse feature groups\n",
      "[LightGBM] [Info] Size of histogram bin entry: 8\n",
      "[LightGBM] [Info] 12 dense feature groups (3.59 MB) transferred to GPU in 0.009159 secs. 0 sparse feature groups\n",
      "[LightGBM] [Info] Size of histogram bin entry: 8\n",
      "[LightGBM] [Info] 12 dense feature groups (3.59 MB) transferred to GPU in 0.007620 secs. 0 sparse feature groups\n",
      "[LightGBM] [Info] Size of histogram bin entry: 8\n",
      "[LightGBM] [Info] 12 dense feature groups (3.59 MB) transferred to GPU in 0.007047 secs. 0 sparse feature groups\n",
      "[400]\tvalid_0's multi_logloss: 0.223112\n",
      "[LightGBM] [Info] Size of histogram bin entry: 8\n",
      "[LightGBM] [Info] 12 dense feature groups (3.59 MB) transferred to GPU in 0.007003 secs. 0 sparse feature groups\n",
      "[LightGBM] [Info] Size of histogram bin entry: 8\n",
      "[LightGBM] [Info] 12 dense feature groups (3.59 MB) transferred to GPU in 0.007112 secs. 0 sparse feature groups\n",
      "[LightGBM] [Info] Size of histogram bin entry: 8\n",
      "[LightGBM] [Info] 12 dense feature groups (3.59 MB) transferred to GPU in 0.008128 secs. 0 sparse feature groups\n",
      "[LightGBM] [Info] Size of histogram bin entry: 8\n",
      "[LightGBM] [Info] 12 dense feature groups (3.58 MB) transferred to GPU in 0.007451 secs. 0 sparse feature groups\n",
      "[LightGBM] [Info] Size of histogram bin entry: 8\n",
      "[LightGBM] [Info] 12 dense feature groups (3.59 MB) transferred to GPU in 0.007739 secs. 0 sparse feature groups\n",
      "[LightGBM] [Info] Size of histogram bin entry: 8\n",
      "[LightGBM] [Info] 12 dense feature groups (3.59 MB) transferred to GPU in 0.007358 secs. 0 sparse feature groups\n",
      "[LightGBM] [Info] Size of histogram bin entry: 8\n",
      "[LightGBM] [Info] 12 dense feature groups (3.59 MB) transferred to GPU in 0.007277 secs. 0 sparse feature groups\n",
      "[LightGBM] [Info] Size of histogram bin entry: 8\n",
      "[LightGBM] [Info] 12 dense feature groups (3.59 MB) transferred to GPU in 0.007078 secs. 0 sparse feature groups\n",
      "[LightGBM] [Info] Size of histogram bin entry: 8\n",
      "[LightGBM] [Info] 12 dense feature groups (3.59 MB) transferred to GPU in 0.007241 secs. 0 sparse feature groups\n",
      "[LightGBM] [Info] Size of histogram bin entry: 8\n",
      "[LightGBM] [Info] 12 dense feature groups (3.59 MB) transferred to GPU in 0.008423 secs. 0 sparse feature groups\n",
      "[500]\tvalid_0's multi_logloss: 0.202542\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[500]\tvalid_0's multi_logloss: 0.202542\n",
      "Wall time: 5min 7s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "bst_gpu = lgb.train(params_gpu,\n",
    "                    train_data,\n",
    "                    num_boost_round=500,\n",
    "                    valid_sets=val_data,\n",
    "                    verbose_eval=100,\n",
    "                    early_stopping_rounds=25)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CatBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "from catboost import CatBoostClassifier, Pool\n",
    "\n",
    "covertype_dataset = fetch_covtype(random_state=101, shuffle=True)\n",
    "label = covertype_dataset.target.astype(int) - 1\n",
    "wilderness_area = np.argmax(covertype_dataset.data[:, 10:(10 + 4)], axis=1)\n",
    "soil_type = np.argmax(covertype_dataset.data[:, (10 + 4):(10 + 4 + 40)], axis=1)\n",
    "data = (covertype_dataset.data[:, :10], wilderness_area.reshape(-1,\n",
    "                                                                1).astype(str),\n",
    "        soil_type.reshape(-1, 1).astype(str))\n",
    "data = np.hstack(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "covertype_train = Pool(data=data[:15000, :],\n",
    "                       label=label[:15000],\n",
    "                       cat_features=[10, 11])\n",
    "covertype_val = Pool(data[15000:20000, :], label[15000:20000], [10, 11])\n",
    "covertype_test_x = Pool(data[20000:25000, :], None, [10, 11])\n",
    "covertype_test_y = label[20000:25000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "cbc_cpu = CatBoostClassifier(iterations=2500,\n",
    "                             learning_rate=0.05,\n",
    "                             depth=8,\n",
    "                             custom_loss='Accuracy',\n",
    "                             eval_metric='Accuracy',\n",
    "                             use_best_model=True,\n",
    "                             loss_function='MultiClass',\n",
    "                             task_type='CPU',\n",
    "                             thread_count=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:\tlearn: 0.6639333\ttest: 0.6690000\tbest: 0.6690000 (0)\ttotal: 133ms\tremaining: 5m 32s\n",
      "500:\tlearn: 0.8766000\ttest: 0.8084000\tbest: 0.8090000 (493)\ttotal: 2m 14s\tremaining: 8m 58s\n",
      "1000:\tlearn: 0.9358000\ttest: 0.8276000\tbest: 0.8284000 (987)\ttotal: 4m 45s\tremaining: 7m 6s\n",
      "1500:\tlearn: 0.9673333\ttest: 0.8316000\tbest: 0.8318000 (1477)\ttotal: 7m 17s\tremaining: 4m 50s\n",
      "2000:\tlearn: 0.9844667\ttest: 0.8344000\tbest: 0.8344000 (1996)\ttotal: 9m 39s\tremaining: 2m 24s\n",
      "2499:\tlearn: 0.9928667\ttest: 0.8368000\tbest: 0.8368000 (2499)\ttotal: 12m 14s\tremaining: 0us\n",
      "\n",
      "bestTest = 0.8368\n",
      "bestIteration = 2499\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<catboost.core.CatBoostClassifier at 0x17880a91248>"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cbc_cpu.fit(covertype_train, eval_set=covertype_val, verbose=500, plot=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds_class_cpu = cbc_cpu.predict(covertype_test_x)\n",
    "preds_proba_cpu = cbc_cpu.predict_proba(covertype_test_x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8392\n",
      "[[1483  321    0    0    0    0   16]\n",
      " [ 237 2172   14    0    9   14    2]\n",
      " [   0   11  269    4    0   17    0]\n",
      " [   0    0    4   19    0    4    0]\n",
      " [   1   47    7    0   22    0    0]\n",
      " [   0   16   42    1    0   85    0]\n",
      " [  36    1    0    0    0    0  146]]\n"
     ]
    }
   ],
   "source": [
    "print(accuracy_score(covertype_test_y, preds_class_cpu))\n",
    "print(confusion_matrix(covertype_test_y, preds_class_cpu))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.84      0.81      0.83      1820\n",
      "           1       0.85      0.89      0.87      2448\n",
      "           2       0.80      0.89      0.84       301\n",
      "           3       0.79      0.70      0.75        27\n",
      "           4       0.71      0.29      0.41        77\n",
      "           5       0.71      0.59      0.64       144\n",
      "           6       0.89      0.80      0.84       183\n",
      "\n",
      "    accuracy                           0.84      5000\n",
      "   macro avg       0.80      0.71      0.74      5000\n",
      "weighted avg       0.84      0.84      0.84      5000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(covertype_test_y, preds_class_cpu))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "cbc_gpu = CatBoostClassifier(iterations=2500,\n",
    "                             learning_rate=0.05,\n",
    "                             depth=8,\n",
    "                             custom_loss='Accuracy',\n",
    "                             eval_metric='Accuracy',\n",
    "                             use_best_model=True,\n",
    "                             loss_function='MultiClass',\n",
    "                             task_type='GPU',\n",
    "                             thread_count=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:\tlearn: 0.6304000\ttest: 0.6248000\tbest: 0.6248000 (0)\ttotal: 36.5ms\tremaining: 1m 31s\n",
      "500:\tlearn: 0.8871333\ttest: 0.8076000\tbest: 0.8076000 (497)\ttotal: 16.7s\tremaining: 1m 6s\n",
      "1000:\tlearn: 0.9448667\ttest: 0.8272000\tbest: 0.8282000 (992)\ttotal: 37s\tremaining: 55.5s\n",
      "1500:\tlearn: 0.9734667\ttest: 0.8336000\tbest: 0.8358000 (1443)\ttotal: 54.9s\tremaining: 36.5s\n",
      "2000:\tlearn: 0.9886667\ttest: 0.8380000\tbest: 0.8396000 (1950)\ttotal: 1m 14s\tremaining: 18.6s\n",
      "2499:\tlearn: 0.9950000\ttest: 0.8394000\tbest: 0.8414000 (2348)\ttotal: 1m 35s\tremaining: 0us\n",
      "bestTest = 0.8414\n",
      "bestIteration = 2348\n",
      "Shrink model to first 2349 iterations.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<catboost.core.CatBoostClassifier at 0x17880c6f248>"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cbc_gpu.fit(covertype_train, eval_set=covertype_val, verbose=500, plot=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds_class_gpu = cbc_gpu.predict(covertype_test_x)\n",
    "preds_proba_gpu = cbc_gpu.predict_proba(covertype_test_x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.84\n",
      "[[1477  327    0    0    0    0   16]\n",
      " [ 231 2181   11    0    9   13    3]\n",
      " [   0   15  265    4    0   17    0]\n",
      " [   0    0    6   18    0    3    0]\n",
      " [   1   44    6    0   26    0    0]\n",
      " [   0   15   43    1    0   85    0]\n",
      " [  34    1    0    0    0    0  148]]\n"
     ]
    }
   ],
   "source": [
    "print(accuracy_score(covertype_test_y, preds_class_gpu))\n",
    "print(confusion_matrix(covertype_test_y, preds_class_gpu))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.85      0.81      0.83      1820\n",
      "           1       0.84      0.89      0.87      2448\n",
      "           2       0.80      0.88      0.84       301\n",
      "           3       0.78      0.67      0.72        27\n",
      "           4       0.74      0.34      0.46        77\n",
      "           5       0.72      0.59      0.65       144\n",
      "           6       0.89      0.81      0.85       183\n",
      "\n",
      "    accuracy                           0.84      5000\n",
      "   macro avg       0.80      0.71      0.74      5000\n",
      "weighted avg       0.84      0.84      0.84      5000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(covertype_test_y, preds_class_gpu))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ThunderGBM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "ename": "OSError",
     "evalue": "[WinError 126] 找不到指定的模块。",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mOSError\u001b[0m                                   Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-11-7835f13aee79>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[1;32mimport\u001b[0m \u001b[0mthundersvm\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32mD:\\Programing\\Anaconda3\\lib\\site-packages\\thundersvm\\__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     10\u001b[0m \"\"\"\n\u001b[0;32m     11\u001b[0m \u001b[0mname\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m\"thundersvm\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 12\u001b[1;33m \u001b[1;32mfrom\u001b[0m \u001b[1;33m.\u001b[0m\u001b[0mthundersvmScikit\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[1;33m*\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32mD:\\Programing\\Anaconda3\\lib\\site-packages\\thundersvm\\thundersvmScikit.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     39\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     40\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mpath\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexists\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlib_path\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 41\u001b[1;33m     \u001b[0mthundersvm\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mCDLL\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlib_path\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     42\u001b[0m \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     43\u001b[0m     \u001b[1;32mraise\u001b[0m \u001b[0mFileNotFoundError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"Please build the library first!\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Programing\\Anaconda3\\lib\\ctypes\\__init__.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, name, mode, handle, use_errno, use_last_error)\u001b[0m\n\u001b[0;32m    362\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    363\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 364\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_handle\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_dlopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_name\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    365\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    366\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_handle\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhandle\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mOSError\u001b[0m: [WinError 126] 找不到指定的模块。"
     ]
    }
   ],
   "source": [
    "import thundersvm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Object `clf` not found.\n"
     ]
    }
   ],
   "source": [
    "clf?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Package                            Version\n",
      "---------------------------------- -------------------\n",
      "aiohttp                            3.6.3\n",
      "alabaster                          0.7.12\n",
      "altgraph                           0.17\n",
      "anaconda-client                    1.7.2\n",
      "anaconda-navigator                 1.9.12\n",
      "anaconda-project                   0.8.3\n",
      "argh                               0.26.2\n",
      "argon2-cffi                        20.1.0\n",
      "asn1crypto                         1.4.0\n",
      "astroid                            2.4.2\n",
      "astropy                            4.0.1.post1\n",
      "async-timeout                      3.0.1\n",
      "atomicwrites                       1.4.0\n",
      "attrs                              20.1.0\n",
      "autopep8                           1.5.4\n",
      "Babel                              2.8.0\n",
      "backcall                           0.2.0\n",
      "backports.functools-lru-cache      1.6.1\n",
      "backports.shutil-get-terminal-size 1.0.0\n",
      "backports.tempfile                 1.0\n",
      "backports.weakref                  1.0.post1\n",
      "bcrypt                             3.2.0\n",
      "beautifulsoup4                     4.9.1\n",
      "bitarray                           1.5.3\n",
      "bkcharts                           0.2\n",
      "bleach                             3.1.5\n",
      "blis                               0.4.1\n",
      "bokeh                              2.2.1\n",
      "boto                               2.49.0\n",
      "boto3                              1.9.66\n",
      "botocore                           1.12.67\n",
      "Bottleneck                         1.3.2\n",
      "branca                             0.4.1\n",
      "brotlipy                           0.7.0\n",
      "cachetools                         4.1.1\n",
      "catalogue                          1.0.0\n",
      "catboost                           0.24\n",
      "certifi                            2020.6.20\n",
      "cffi                               1.14.2\n",
      "chardet                            3.0.4\n",
      "click                              7.1.2\n",
      "cloudpickle                        1.6.0\n",
      "clyent                             1.2.2\n",
      "colorama                           0.4.3\n",
      "comtypes                           1.1.7\n",
      "conda                              4.8.5\n",
      "conda-build                        3.20.2\n",
      "conda-pack                         0.5.0\n",
      "conda-package-handling             1.7.0\n",
      "conda-verify                       3.4.2\n",
      "confuse                            1.3.0\n",
      "contextlib2                        0.6.0.post1\n",
      "convertdate                        2.2.1\n",
      "cryptography                       3.1\n",
      "cssselect                          1.1.0\n",
      "cupy                               6.0.0\n",
      "cutecharts                         1.2.0\n",
      "cycler                             0.10.0\n",
      "cymem                              2.0.3\n",
      "Cython                             0.29.21\n",
      "cytoolz                            0.10.1\n",
      "dash                               1.16.1\n",
      "dash-core-components               1.3.1\n",
      "dash-html-components               1.0.1\n",
      "dash-renderer                      1.1.2\n",
      "dash-table                         4.4.1\n",
      "dask                               2.25.0\n",
      "decorator                          4.4.2\n",
      "defusedxml                         0.6.0\n",
      "diff-match-patch                   20200713\n",
      "distributed                        2.25.0\n",
      "docutils                           0.16\n",
      "en-core-web-lg                     2.3.1\n",
      "en-core-web-sm                     2.3.1\n",
      "entropy-based-binning              0.0.1\n",
      "entrypoints                        0.3\n",
      "ephem                              3.7.7.0\n",
      "et-xmlfile                         1.0.1\n",
      "fastcache                          1.1.0\n",
      "fastrlock                          0.4\n",
      "fbprophet                          0.6\n",
      "ffn                                0.3.4\n",
      "filelock                           3.0.12\n",
      "flake8                             3.8.3\n",
      "Flask                              1.1.2\n",
      "Flask-Compress                     1.5.0\n",
      "folium                             0.11.0\n",
      "fsspec                             0.8.0\n",
      "future                             0.18.2\n",
      "fuzzywuzzy                         0.17.0\n",
      "gensim                             3.8.0\n",
      "gevent                             20.6.2\n",
      "glob2                              0.7\n",
      "gmpy2                              2.0.8\n",
      "google-api-core                    1.22.2\n",
      "google-auth                        1.22.1\n",
      "google-cloud-core                  1.4.3\n",
      "google-cloud-storage               1.31.0\n",
      "google-crc32c                      1.0.0\n",
      "google-resumable-media             1.1.0\n",
      "googleapis-common-protos           1.52.0\n",
      "greenlet                           0.4.16\n",
      "h5py                               2.10.0\n",
      "HeapDict                           1.0.1\n",
      "holidays                           0.10.3\n",
      "html5lib                           1.1\n",
      "htmlmin                            0.1.12\n",
      "idna                               2.10\n",
      "ImageHash                          4.1.0\n",
      "imageio                            2.9.0\n",
      "imagesize                          1.2.0\n",
      "imbalanced-learn                   0.7.0\n",
      "importlib-metadata                 1.7.0\n",
      "inflection                         0.5.1\n",
      "iniconfig                          0.0.0\n",
      "intervaltree                       3.1.0\n",
      "ipykernel                          5.3.4\n",
      "ipympl                             0.5.7\n",
      "ipython                            7.18.1\n",
      "ipython-genutils                   0.2.0\n",
      "ipywidgets                         7.5.1\n",
      "isort                              5.4.2\n",
      "itsdangerous                       1.1.0\n",
      "jdcal                              1.4.1\n",
      "jedi                               0.14.1\n",
      "jieba                              0.42.1\n",
      "Jinja2                             2.11.2\n",
      "jmespath                           0.10.0\n",
      "joblib                             0.16.0\n",
      "json5                              0.9.5\n",
      "jsonschema                         3.0.2\n",
      "jupyter                            1.0.0\n",
      "jupyter-bokeh                      2.0.3\n",
      "jupyter-client                     6.1.6\n",
      "jupyter-console                    6.2.0\n",
      "jupyter-contrib-core               0.3.3\n",
      "jupyter-core                       4.6.3\n",
      "jupyter-kite                       1.0.0\n",
      "jupyter-nbextensions-configurator  0.4.1\n",
      "jupyterlab                         2.2.6\n",
      "jupyterlab-code-formatter          1.3.6\n",
      "jupyterlab-server                  1.2.0\n",
      "keyring                            21.4.0\n",
      "kiwisolver                         1.2.0\n",
      "korean-lunar-calendar              0.2.1\n",
      "lazy-object-proxy                  1.4.3\n",
      "libarchive-c                       2.9\n",
      "lightgbm                           3.0.0.99\n",
      "llvmlite                           0.33.0+1.g022ab0f\n",
      "locket                             0.2.0\n",
      "LunarCalendar                      0.0.9\n",
      "lxml                               4.5.2\n",
      "macholib                           1.14\n",
      "MarkupSafe                         1.1.1\n",
      "matplotlib                         3.3.1\n",
      "mccabe                             0.6.1\n",
      "mdlp-discretization                0.3.3\n",
      "menuinst                           1.4.16\n",
      "missingno                          0.4.2\n",
      "mistune                            0.8.4\n",
      "mkl-fft                            1.1.0\n",
      "mkl-random                         1.1.1\n",
      "mkl-service                        2.3.0\n",
      "ml-metrics                         0.1.4\n",
      "mlxtend                            0.17.3\n",
      "mock                               4.0.2\n",
      "more-itertools                     8.5.0\n",
      "MouseInfo                          0.1.3\n",
      "mpl-finance                        0.10.1\n",
      "mpmath                             1.1.0\n",
      "msgpack                            1.0.0\n",
      "multidict                          4.7.6\n",
      "multipledispatch                   0.6.0\n",
      "murmurhash                         1.0.2\n",
      "mysql-connector                    2.2.9\n",
      "mysql-connector-python             8.0.18\n",
      "navigator-updater                  0.2.1\n",
      "nb-conda-kernels                   2.2.4\n",
      "nbconvert                          5.6.1\n",
      "nbformat                           5.0.7\n",
      "nbresuse                           0.3.6\n",
      "networkx                           2.5\n",
      "nltk                               3.5\n",
      "nose                               1.3.7\n",
      "notebook                           6.1.1\n",
      "numba                              0.50.1\n",
      "numexpr                            2.7.1\n",
      "numpy                              1.19.1\n",
      "numpydoc                           1.1.0\n",
      "oauthlib                           3.1.0\n",
      "olefile                            0.46\n",
      "opencv-python                      3.4.11.41\n",
      "openpyxl                           3.0.5\n",
      "packaging                          20.4\n",
      "pandas                             1.1.1\n",
      "pandas-datareader                  0.9.0\n",
      "pandas-profiling                   2.9.0\n",
      "pandocfilters                      1.4.2\n",
      "paramiko                           2.7.2\n",
      "parsel                             1.6.0\n",
      "parso                              0.5.2\n",
      "partd                              1.1.0\n",
      "path                               15.0.0\n",
      "pathlib2                           2.3.5\n",
      "pathtools                          0.1.2\n",
      "patsy                              0.5.1\n",
      "peewee                             3.13.3\n",
      "pefile                             2019.4.18\n",
      "pep8                               1.7.1\n",
      "pexpect                            4.8.0\n",
      "phik                               0.10.0\n",
      "pickleshare                        0.7.5\n",
      "Pillow                             7.2.0\n",
      "pip                                20.2.3\n",
      "pkginfo                            1.5.0.1\n",
      "pkuseg                             0.0.25\n",
      "plac                               0.9.6\n",
      "plotly                             4.10.0\n",
      "pluggy                             0.13.1\n",
      "ply                                3.11\n",
      "preshed                            3.0.2\n",
      "prettytable                        0.7.2\n",
      "prometheus-client                  0.8.0\n",
      "prompt-toolkit                     3.0.7\n",
      "protobuf                           3.6.0\n",
      "psutil                             5.7.2\n",
      "py                                 1.9.0\n",
      "pyasn1                             0.4.8\n",
      "pyasn1-modules                     0.2.8\n",
      "PyAutoGUI                          0.9.50\n",
      "pycodestyle                        2.6.0\n",
      "pycosat                            0.6.3\n",
      "pycparser                          2.20\n",
      "pycrypto                           2.6.1\n",
      "pycryptodome                       3.9.8\n",
      "pycurl                             7.43.0.5\n",
      "pydocstyle                         5.1.1\n",
      "pyecharts                          1.8.1\n",
      "pyflakes                           2.2.0\n",
      "pyftpdlib                          1.5.6\n",
      "PyGetWindow                        0.0.8\n",
      "Pygments                           2.6.1\n",
      "PyInstaller                        3.6\n",
      "pylint                             2.6.0\n",
      "PyMeeus                            0.3.7\n",
      "pymemcache                         3.3.0\n",
      "pymongo                            3.11.0\n",
      "PyMsgBox                           1.0.8\n",
      "PyMySQL                            0.9.3\n",
      "PyNaCl                             1.4.0\n",
      "pyodbc                             4.0.0-unsupported\n",
      "pyOpenSSL                          19.1.0\n",
      "pyparsing                          2.4.7\n",
      "pyperclip                          1.8.0\n",
      "pyreadline                         2.1\n",
      "PyRect                             0.1.4\n",
      "pyrsistent                         0.16.0\n",
      "PyScreeze                          0.1.26\n",
      "PySocks                            1.7.1\n",
      "pystan                             2.19.0.0\n",
      "pytest                             6.0.1\n",
      "python-bidi                        0.4.2\n",
      "python-dateutil                    2.8.1\n",
      "python-docx                        0.8.10\n",
      "python-jsonrpc-server              0.3.4\n",
      "python-language-server             0.31.7\n",
      "python-Levenshtein                 0.12.0\n",
      "PyTweening                         1.0.3\n",
      "pytz                               2020.1\n",
      "PyWavelets                         1.1.1\n",
      "pywin32                            227\n",
      "pywin32-ctypes                     0.2.0\n",
      "pywinpty                           0.5.7\n",
      "PyYAML                             5.3.1\n",
      "pyzmq                              19.0.1\n",
      "QDarkStyle                         2.8.1\n",
      "QtAwesome                          0.7.2\n",
      "qtconsole                          4.7.6\n",
      "QtPy                               1.9.0\n",
      "Quandl                             3.5.2\n",
      "redis                              3.5.3\n",
      "regex                              2020.7.14\n",
      "requestium                         0.1.9\n",
      "requests                           2.24.0\n",
      "requests-file                      1.5.1\n",
      "retrying                           1.3.3\n",
      "rope                               0.17.0\n",
      "rsa                                4.6\n",
      "Rtree                              0.9.4\n",
      "ruamel-yaml                        0.15.87\n",
      "s3transfer                         0.1.13\n",
      "scikit-image                       0.16.2\n",
      "scikit-learn                       0.23.2\n",
      "scikit-surprise                    1.1.1\n",
      "scipy                              1.5.2\n",
      "seaborn                            0.11.0\n",
      "selenium                           3.14.1\n",
      "Send2Trash                         1.5.0\n",
      "setuptools                         49.6.0.post20200814\n",
      "simplegeneric                      0.8.1\n",
      "simplejson                         3.17.2\n",
      "singledispatch                     3.4.0.3\n",
      "six                                1.15.0\n",
      "smart-open                         3.0.0\n",
      "snowballstemmer                    2.0.0\n",
      "sortedcollections                  1.2.1\n",
      "sortedcontainers                   2.2.2\n",
      "soupsieve                          2.0.1\n",
      "spacy                              2.3.1\n",
      "spacy-lookups-data                 0.3.0\n",
      "Sphinx                             3.2.1\n",
      "sphinxcontrib-applehelp            1.0.2\n",
      "sphinxcontrib-devhelp              1.0.2\n",
      "sphinxcontrib-htmlhelp             1.0.3\n",
      "sphinxcontrib-jsmath               1.0.1\n",
      "sphinxcontrib-qthelp               1.0.3\n",
      "sphinxcontrib-serializinghtml      1.1.4\n",
      "sphinxcontrib-websupport           1.2.4\n",
      "spyder                             4.0.1\n",
      "spyder-kernels                     1.8.1\n",
      "SQLAlchemy                         1.3.19\n",
      "srsly                              1.0.2\n",
      "statsmodels                        0.11.1\n",
      "sympy                              1.6.2\n",
      "tables                             3.6.1\n",
      "tabulate                           0.8.7\n",
      "tangled-up-in-unicode              0.0.6\n",
      "tblib                              1.7.0\n",
      "terminado                          0.8.3\n",
      "testpath                           0.4.4\n",
      "thinc                              7.4.1\n",
      "threadpoolctl                      2.1.0\n",
      "thundersvm                         0.3.4\n",
      "tldextract                         2.2.3\n",
      "toml                               0.10.1\n",
      "toolz                              0.10.0\n",
      "tornado                            6.0.4\n",
      "tqdm                               4.48.2\n",
      "traitlets                          4.3.3\n",
      "tsfresh                            0.16.0\n",
      "typed-ast                          1.4.1\n",
      "typing-extensions                  3.7.4.3\n",
      "ujson                              1.35\n",
      "unicodecsv                         0.14.1\n",
      "urllib3                            1.24.3\n",
      "visions                            0.5.0\n",
      "w3lib                              1.22.0\n",
      "wasabi                             0.8.0\n",
      "watchdog                           0.10.3\n",
      "wcwidth                            0.2.5\n",
      "webencodings                       0.5.1\n",
      "Werkzeug                           1.0.1\n",
      "wheel                              0.35.1\n",
      "widgetsnbextension                 3.5.1\n",
      "win-inet-pton                      1.1.0\n",
      "win-unicode-console                0.5\n",
      "wincertstore                       0.2\n",
      "wolframclient                      1.1.4\n",
      "wordcloud                          1.8.0\n",
      "wrapt                              1.11.2\n",
      "xgboost                            1.3.0-SNAPSHOT\n",
      "xlrd                               1.2.0\n",
      "XlsxWriter                         1.3.3\n",
      "xlwings                            0.20.5\n",
      "xlwt                               1.3.0\n",
      "xmltodict                          0.12.0\n",
      "xx-ent-wiki-sm                     2.3.0\n",
      "yapf                               0.30.0\n",
      "yarl                               1.6.2\n",
      "zh-core-web-lg                     2.3.1\n",
      "zh-core-web-sm                     2.3.1\n",
      "zhon                               1.1.5\n",
      "zict                               2.0.0\n",
      "zipp                               3.1.0\n",
      "zope.event                         4.4\n",
      "zope.interface                     5.1.0\n"
     ]
    }
   ],
   "source": [
    "!pip list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}