{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MegaFon Uplift Competition Dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The dataset is provided by MegaFon at the [MegaFon Uplift Competition](https://ods.ai/competitions/megafon-df21-comp) hosted in may 2021.\n",
    "\n",
    "Here is a description of the fields:\n",
    "\n",
    "- **id** (int): client id\n",
    "\n",
    "- **treatment_group** (str): treatment/control group flag\n",
    "\n",
    "- **X_1...X_50** (float): feature values\n",
    "\n",
    "- **conversion** (binary): target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "\n",
    "# install uplift library scikit-uplift and other libraries \n",
    "!{sys.executable} -m pip install scikit-uplift dill lightgbm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklift.datasets import fetch_megafon\n",
    "import pandas as pd \n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "%matplotlib inline\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklift.models import TwoModels\n",
    "import lightgbm as lgb\n",
    "from sklift.metrics import qini_auc_score\n",
    "from sklift.viz import plot_qini_curve\n",
    "\n",
    "seed=31"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 📝 Load data\n",
    "\n",
    "Dataset can be loaded from `sklift.datasets` module using `fetch_megafon` function."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# returns sklearn Bunch object\n",
    "# with data, target, treatment keys\n",
    "# data features (pd.DataFrame), target (pd.Series), treatment (pd.Series) values \n",
    "\n",
    "dataset = fetch_megafon()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset type: <class 'sklearn.utils.Bunch'>\n",
      "\n",
      "Dataset features shape: (600000, 51)\n",
      "Dataset target shape: (600000,)\n",
      "Dataset treatment shape: (600000,)\n"
     ]
    }
   ],
   "source": [
    "print(f\"Dataset type: {type(dataset)}\\n\")\n",
    "print(f\"Dataset features shape: {dataset.data.shape}\")\n",
    "print(f\"Dataset target shape: {dataset.target.shape}\")\n",
    "print(f\"Dataset treatment shape: {dataset.treatment.shape}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 📝 EDA"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's have a look at the data features."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>X_1</th>\n",
       "      <th>X_2</th>\n",
       "      <th>X_3</th>\n",
       "      <th>X_4</th>\n",
       "      <th>X_5</th>\n",
       "      <th>X_6</th>\n",
       "      <th>X_7</th>\n",
       "      <th>X_8</th>\n",
       "      <th>X_9</th>\n",
       "      <th>...</th>\n",
       "      <th>X_41</th>\n",
       "      <th>X_42</th>\n",
       "      <th>X_43</th>\n",
       "      <th>X_44</th>\n",
       "      <th>X_45</th>\n",
       "      <th>X_46</th>\n",
       "      <th>X_47</th>\n",
       "      <th>X_48</th>\n",
       "      <th>X_49</th>\n",
       "      <th>X_50</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>39.396577</td>\n",
       "      <td>-0.186548</td>\n",
       "      <td>19.524505</td>\n",
       "      <td>21.250208</td>\n",
       "      <td>55.291264</td>\n",
       "      <td>182.966712</td>\n",
       "      <td>-5.385606</td>\n",
       "      <td>144.573379</td>\n",
       "      <td>-12.534344</td>\n",
       "      <td>...</td>\n",
       "      <td>90.877638</td>\n",
       "      <td>134.363458</td>\n",
       "      <td>-213.584582</td>\n",
       "      <td>-2.092461</td>\n",
       "      <td>-93.973258</td>\n",
       "      <td>-0.155597</td>\n",
       "      <td>-312.130733</td>\n",
       "      <td>44.798182</td>\n",
       "      <td>-125.682413</td>\n",
       "      <td>16.231365</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>38.987694</td>\n",
       "      <td>0.819522</td>\n",
       "      <td>-42.064512</td>\n",
       "      <td>-48.270949</td>\n",
       "      <td>-33.171257</td>\n",
       "      <td>179.459341</td>\n",
       "      <td>-87.151810</td>\n",
       "      <td>-162.693257</td>\n",
       "      <td>20.651652</td>\n",
       "      <td>...</td>\n",
       "      <td>-183.840746</td>\n",
       "      <td>72.864779</td>\n",
       "      <td>559.783584</td>\n",
       "      <td>1.142391</td>\n",
       "      <td>80.037124</td>\n",
       "      <td>-1.216185</td>\n",
       "      <td>-111.473936</td>\n",
       "      <td>-127.737977</td>\n",
       "      <td>-117.501171</td>\n",
       "      <td>10.732234</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>-16.693093</td>\n",
       "      <td>1.844558</td>\n",
       "      <td>-8.615192</td>\n",
       "      <td>-18.818740</td>\n",
       "      <td>-22.271188</td>\n",
       "      <td>-116.290369</td>\n",
       "      <td>-63.816746</td>\n",
       "      <td>-38.340763</td>\n",
       "      <td>24.968496</td>\n",
       "      <td>...</td>\n",
       "      <td>-203.637766</td>\n",
       "      <td>2.480242</td>\n",
       "      <td>96.998504</td>\n",
       "      <td>1.100962</td>\n",
       "      <td>-33.275159</td>\n",
       "      <td>0.920926</td>\n",
       "      <td>-679.492242</td>\n",
       "      <td>-91.009397</td>\n",
       "      <td>-18.173358</td>\n",
       "      <td>14.367636</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>-72.040154</td>\n",
       "      <td>-0.226921</td>\n",
       "      <td>39.802607</td>\n",
       "      <td>16.441262</td>\n",
       "      <td>-1.112509</td>\n",
       "      <td>68.128008</td>\n",
       "      <td>23.073147</td>\n",
       "      <td>4.688858</td>\n",
       "      <td>-49.383641</td>\n",
       "      <td>...</td>\n",
       "      <td>172.906875</td>\n",
       "      <td>83.951551</td>\n",
       "      <td>-323.642557</td>\n",
       "      <td>-0.369182</td>\n",
       "      <td>93.221948</td>\n",
       "      <td>-1.962380</td>\n",
       "      <td>-442.466684</td>\n",
       "      <td>-22.298302</td>\n",
       "      <td>-75.916603</td>\n",
       "      <td>11.634299</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>18.296973</td>\n",
       "      <td>0.996437</td>\n",
       "      <td>24.465307</td>\n",
       "      <td>-34.151971</td>\n",
       "      <td>24.623458</td>\n",
       "      <td>-155.455558</td>\n",
       "      <td>-12.159787</td>\n",
       "      <td>26.705778</td>\n",
       "      <td>105.864805</td>\n",
       "      <td>...</td>\n",
       "      <td>125.577535</td>\n",
       "      <td>-208.531112</td>\n",
       "      <td>118.902324</td>\n",
       "      <td>-0.808578</td>\n",
       "      <td>-117.497906</td>\n",
       "      <td>1.770635</td>\n",
       "      <td>627.395611</td>\n",
       "      <td>122.019189</td>\n",
       "      <td>194.091195</td>\n",
       "      <td>-11.883858</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>599995</th>\n",
       "      <td>599995</td>\n",
       "      <td>33.031482</td>\n",
       "      <td>0.584463</td>\n",
       "      <td>-25.546737</td>\n",
       "      <td>-35.129322</td>\n",
       "      <td>6.603716</td>\n",
       "      <td>-90.384229</td>\n",
       "      <td>-44.399223</td>\n",
       "      <td>71.343628</td>\n",
       "      <td>26.439937</td>\n",
       "      <td>...</td>\n",
       "      <td>-183.820274</td>\n",
       "      <td>-167.259176</td>\n",
       "      <td>531.793092</td>\n",
       "      <td>-2.307605</td>\n",
       "      <td>-48.648490</td>\n",
       "      <td>0.763070</td>\n",
       "      <td>152.501250</td>\n",
       "      <td>-49.190995</td>\n",
       "      <td>149.292732</td>\n",
       "      <td>-15.031430</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>599996</th>\n",
       "      <td>599996</td>\n",
       "      <td>-24.469552</td>\n",
       "      <td>1.175070</td>\n",
       "      <td>35.936859</td>\n",
       "      <td>19.577711</td>\n",
       "      <td>81.505532</td>\n",
       "      <td>-164.141125</td>\n",
       "      <td>-8.221423</td>\n",
       "      <td>38.864814</td>\n",
       "      <td>-2.410108</td>\n",
       "      <td>...</td>\n",
       "      <td>103.792521</td>\n",
       "      <td>-87.706751</td>\n",
       "      <td>-230.298393</td>\n",
       "      <td>0.267790</td>\n",
       "      <td>-56.102467</td>\n",
       "      <td>-1.007702</td>\n",
       "      <td>-69.854583</td>\n",
       "      <td>187.924882</td>\n",
       "      <td>109.642972</td>\n",
       "      <td>25.022399</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>599997</th>\n",
       "      <td>599997</td>\n",
       "      <td>-57.592800</td>\n",
       "      <td>1.033326</td>\n",
       "      <td>-20.608920</td>\n",
       "      <td>87.116236</td>\n",
       "      <td>-27.894756</td>\n",
       "      <td>120.864975</td>\n",
       "      <td>40.592090</td>\n",
       "      <td>-34.134983</td>\n",
       "      <td>-26.821810</td>\n",
       "      <td>...</td>\n",
       "      <td>-97.993743</td>\n",
       "      <td>170.942462</td>\n",
       "      <td>299.211339</td>\n",
       "      <td>-0.930181</td>\n",
       "      <td>168.767350</td>\n",
       "      <td>-0.004762</td>\n",
       "      <td>-249.073528</td>\n",
       "      <td>-265.077308</td>\n",
       "      <td>-110.317964</td>\n",
       "      <td>-11.892925</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>599998</th>\n",
       "      <td>599998</td>\n",
       "      <td>-16.371510</td>\n",
       "      <td>-0.876885</td>\n",
       "      <td>-1.483559</td>\n",
       "      <td>32.407631</td>\n",
       "      <td>-57.148865</td>\n",
       "      <td>92.559125</td>\n",
       "      <td>-21.464004</td>\n",
       "      <td>-110.615734</td>\n",
       "      <td>58.100487</td>\n",
       "      <td>...</td>\n",
       "      <td>-17.409713</td>\n",
       "      <td>108.897052</td>\n",
       "      <td>223.509177</td>\n",
       "      <td>-0.832695</td>\n",
       "      <td>5.545312</td>\n",
       "      <td>0.659149</td>\n",
       "      <td>-156.605507</td>\n",
       "      <td>-41.757924</td>\n",
       "      <td>-89.330302</td>\n",
       "      <td>-22.886739</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>599999</th>\n",
       "      <td>599999</td>\n",
       "      <td>62.997565</td>\n",
       "      <td>-1.299755</td>\n",
       "      <td>-9.485073</td>\n",
       "      <td>4.682186</td>\n",
       "      <td>64.339233</td>\n",
       "      <td>-14.527508</td>\n",
       "      <td>-113.750106</td>\n",
       "      <td>195.634148</td>\n",
       "      <td>104.627105</td>\n",
       "      <td>...</td>\n",
       "      <td>-64.312678</td>\n",
       "      <td>48.114939</td>\n",
       "      <td>-231.497127</td>\n",
       "      <td>-0.022551</td>\n",
       "      <td>7.373350</td>\n",
       "      <td>0.213328</td>\n",
       "      <td>-235.624729</td>\n",
       "      <td>39.129831</td>\n",
       "      <td>-47.422068</td>\n",
       "      <td>-21.564626</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10 rows × 51 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            id        X_1       X_2        X_3        X_4        X_5  \\\n",
       "0            0  39.396577 -0.186548  19.524505  21.250208  55.291264   \n",
       "1            1  38.987694  0.819522 -42.064512 -48.270949 -33.171257   \n",
       "2            2 -16.693093  1.844558  -8.615192 -18.818740 -22.271188   \n",
       "3            3 -72.040154 -0.226921  39.802607  16.441262  -1.112509   \n",
       "4            4  18.296973  0.996437  24.465307 -34.151971  24.623458   \n",
       "599995  599995  33.031482  0.584463 -25.546737 -35.129322   6.603716   \n",
       "599996  599996 -24.469552  1.175070  35.936859  19.577711  81.505532   \n",
       "599997  599997 -57.592800  1.033326 -20.608920  87.116236 -27.894756   \n",
       "599998  599998 -16.371510 -0.876885  -1.483559  32.407631 -57.148865   \n",
       "599999  599999  62.997565 -1.299755  -9.485073   4.682186  64.339233   \n",
       "\n",
       "               X_6         X_7         X_8         X_9  ...        X_41  \\\n",
       "0       182.966712   -5.385606  144.573379  -12.534344  ...   90.877638   \n",
       "1       179.459341  -87.151810 -162.693257   20.651652  ... -183.840746   \n",
       "2      -116.290369  -63.816746  -38.340763   24.968496  ... -203.637766   \n",
       "3        68.128008   23.073147    4.688858  -49.383641  ...  172.906875   \n",
       "4      -155.455558  -12.159787   26.705778  105.864805  ...  125.577535   \n",
       "599995  -90.384229  -44.399223   71.343628   26.439937  ... -183.820274   \n",
       "599996 -164.141125   -8.221423   38.864814   -2.410108  ...  103.792521   \n",
       "599997  120.864975   40.592090  -34.134983  -26.821810  ...  -97.993743   \n",
       "599998   92.559125  -21.464004 -110.615734   58.100487  ...  -17.409713   \n",
       "599999  -14.527508 -113.750106  195.634148  104.627105  ...  -64.312678   \n",
       "\n",
       "              X_42        X_43      X_44        X_45      X_46        X_47  \\\n",
       "0       134.363458 -213.584582 -2.092461  -93.973258 -0.155597 -312.130733   \n",
       "1        72.864779  559.783584  1.142391   80.037124 -1.216185 -111.473936   \n",
       "2         2.480242   96.998504  1.100962  -33.275159  0.920926 -679.492242   \n",
       "3        83.951551 -323.642557 -0.369182   93.221948 -1.962380 -442.466684   \n",
       "4      -208.531112  118.902324 -0.808578 -117.497906  1.770635  627.395611   \n",
       "599995 -167.259176  531.793092 -2.307605  -48.648490  0.763070  152.501250   \n",
       "599996  -87.706751 -230.298393  0.267790  -56.102467 -1.007702  -69.854583   \n",
       "599997  170.942462  299.211339 -0.930181  168.767350 -0.004762 -249.073528   \n",
       "599998  108.897052  223.509177 -0.832695    5.545312  0.659149 -156.605507   \n",
       "599999   48.114939 -231.497127 -0.022551    7.373350  0.213328 -235.624729   \n",
       "\n",
       "              X_48        X_49       X_50  \n",
       "0        44.798182 -125.682413  16.231365  \n",
       "1      -127.737977 -117.501171  10.732234  \n",
       "2       -91.009397  -18.173358  14.367636  \n",
       "3       -22.298302  -75.916603  11.634299  \n",
       "4       122.019189  194.091195 -11.883858  \n",
       "599995  -49.190995  149.292732 -15.031430  \n",
       "599996  187.924882  109.642972  25.022399  \n",
       "599997 -265.077308 -110.317964 -11.892925  \n",
       "599998  -41.757924  -89.330302 -22.886739  \n",
       "599999   39.129831  -47.422068 -21.564626  \n",
       "\n",
       "[10 rows x 51 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.data.head().append(dataset.data.tail())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>X_1</th>\n",
       "      <th>X_2</th>\n",
       "      <th>X_3</th>\n",
       "      <th>X_4</th>\n",
       "      <th>X_5</th>\n",
       "      <th>X_6</th>\n",
       "      <th>X_7</th>\n",
       "      <th>X_8</th>\n",
       "      <th>X_9</th>\n",
       "      <th>...</th>\n",
       "      <th>X_41</th>\n",
       "      <th>X_42</th>\n",
       "      <th>X_43</th>\n",
       "      <th>X_44</th>\n",
       "      <th>X_45</th>\n",
       "      <th>X_46</th>\n",
       "      <th>X_47</th>\n",
       "      <th>X_48</th>\n",
       "      <th>X_49</th>\n",
       "      <th>X_50</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>600000.000000</td>\n",
       "      <td>600000.000000</td>\n",
       "      <td>600000.000000</td>\n",
       "      <td>600000.000000</td>\n",
       "      <td>600000.000000</td>\n",
       "      <td>600000.000000</td>\n",
       "      <td>600000.000000</td>\n",
       "      <td>600000.000000</td>\n",
       "      <td>600000.000000</td>\n",
       "      <td>600000.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>600000.000000</td>\n",
       "      <td>600000.000000</td>\n",
       "      <td>600000.000000</td>\n",
       "      <td>600000.000000</td>\n",
       "      <td>600000.000000</td>\n",
       "      <td>600000.000000</td>\n",
       "      <td>600000.000000</td>\n",
       "      <td>600000.000000</td>\n",
       "      <td>600000.000000</td>\n",
       "      <td>600000.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>299999.500000</td>\n",
       "      <td>-3.758503</td>\n",
       "      <td>0.000405</td>\n",
       "      <td>0.356208</td>\n",
       "      <td>-1.004378</td>\n",
       "      <td>3.376919</td>\n",
       "      <td>-6.396371</td>\n",
       "      <td>-2.253712</td>\n",
       "      <td>-6.432606</td>\n",
       "      <td>-0.061507</td>\n",
       "      <td>...</td>\n",
       "      <td>6.882868</td>\n",
       "      <td>-6.632834</td>\n",
       "      <td>8.454493</td>\n",
       "      <td>0.001296</td>\n",
       "      <td>0.007967</td>\n",
       "      <td>-0.000966</td>\n",
       "      <td>-22.259600</td>\n",
       "      <td>-5.759041</td>\n",
       "      <td>6.241130</td>\n",
       "      <td>-1.176456</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>173205.225094</td>\n",
       "      <td>54.881882</td>\n",
       "      <td>0.999419</td>\n",
       "      <td>31.804123</td>\n",
       "      <td>45.291429</td>\n",
       "      <td>53.397644</td>\n",
       "      <td>140.873734</td>\n",
       "      <td>59.810396</td>\n",
       "      <td>74.840857</td>\n",
       "      <td>44.912292</td>\n",
       "      <td>...</td>\n",
       "      <td>163.776538</td>\n",
       "      <td>137.025868</td>\n",
       "      <td>262.840194</td>\n",
       "      <td>1.000368</td>\n",
       "      <td>71.553713</td>\n",
       "      <td>0.999902</td>\n",
       "      <td>500.900364</td>\n",
       "      <td>130.952113</td>\n",
       "      <td>141.211999</td>\n",
       "      <td>21.363662</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-271.659497</td>\n",
       "      <td>-4.372119</td>\n",
       "      <td>-148.870768</td>\n",
       "      <td>-244.446728</td>\n",
       "      <td>-302.574049</td>\n",
       "      <td>-683.126343</td>\n",
       "      <td>-322.731683</td>\n",
       "      <td>-506.202937</td>\n",
       "      <td>-218.466369</td>\n",
       "      <td>...</td>\n",
       "      <td>-810.842593</td>\n",
       "      <td>-633.575178</td>\n",
       "      <td>-1345.838757</td>\n",
       "      <td>-4.756720</td>\n",
       "      <td>-360.713742</td>\n",
       "      <td>-4.516004</td>\n",
       "      <td>-2506.960013</td>\n",
       "      <td>-687.526201</td>\n",
       "      <td>-702.184241</td>\n",
       "      <td>-98.094323</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>149999.750000</td>\n",
       "      <td>-40.693313</td>\n",
       "      <td>-0.673108</td>\n",
       "      <td>-20.758308</td>\n",
       "      <td>-30.644608</td>\n",
       "      <td>-31.865404</td>\n",
       "      <td>-100.762161</td>\n",
       "      <td>-42.313674</td>\n",
       "      <td>-54.840796</td>\n",
       "      <td>-30.327330</td>\n",
       "      <td>...</td>\n",
       "      <td>-102.658380</td>\n",
       "      <td>-99.033996</td>\n",
       "      <td>-167.634846</td>\n",
       "      <td>-0.673780</td>\n",
       "      <td>-48.250836</td>\n",
       "      <td>-0.675549</td>\n",
       "      <td>-357.547278</td>\n",
       "      <td>-93.163915</td>\n",
       "      <td>-88.803657</td>\n",
       "      <td>-15.580688</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>299999.500000</td>\n",
       "      <td>-3.954771</td>\n",
       "      <td>0.000915</td>\n",
       "      <td>0.372583</td>\n",
       "      <td>-0.585368</td>\n",
       "      <td>3.720738</td>\n",
       "      <td>-6.357443</td>\n",
       "      <td>-2.263690</td>\n",
       "      <td>-6.416419</td>\n",
       "      <td>-0.103742</td>\n",
       "      <td>...</td>\n",
       "      <td>7.190600</td>\n",
       "      <td>-6.784760</td>\n",
       "      <td>8.773280</td>\n",
       "      <td>0.001639</td>\n",
       "      <td>0.045537</td>\n",
       "      <td>-0.002251</td>\n",
       "      <td>-20.695017</td>\n",
       "      <td>-5.774627</td>\n",
       "      <td>6.286783</td>\n",
       "      <td>-1.199895</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>449999.250000</td>\n",
       "      <td>33.174835</td>\n",
       "      <td>0.673056</td>\n",
       "      <td>21.495530</td>\n",
       "      <td>29.027860</td>\n",
       "      <td>38.988940</td>\n",
       "      <td>88.159514</td>\n",
       "      <td>37.709780</td>\n",
       "      <td>41.962767</td>\n",
       "      <td>30.144501</td>\n",
       "      <td>...</td>\n",
       "      <td>116.755600</td>\n",
       "      <td>85.621324</td>\n",
       "      <td>185.382370</td>\n",
       "      <td>0.675779</td>\n",
       "      <td>48.221733</td>\n",
       "      <td>0.673638</td>\n",
       "      <td>313.295748</td>\n",
       "      <td>81.636824</td>\n",
       "      <td>101.558007</td>\n",
       "      <td>13.230410</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>599999.000000</td>\n",
       "      <td>250.812280</td>\n",
       "      <td>5.062006</td>\n",
       "      <td>170.053291</td>\n",
       "      <td>235.095937</td>\n",
       "      <td>284.915947</td>\n",
       "      <td>656.482242</td>\n",
       "      <td>293.909622</td>\n",
       "      <td>550.525780</td>\n",
       "      <td>219.628423</td>\n",
       "      <td>...</td>\n",
       "      <td>867.086690</td>\n",
       "      <td>689.626208</td>\n",
       "      <td>1488.759454</td>\n",
       "      <td>4.727996</td>\n",
       "      <td>384.665348</td>\n",
       "      <td>5.086304</td>\n",
       "      <td>2534.503855</td>\n",
       "      <td>595.321844</td>\n",
       "      <td>630.727101</td>\n",
       "      <td>112.233293</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 51 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                  id            X_1            X_2            X_3  \\\n",
       "count  600000.000000  600000.000000  600000.000000  600000.000000   \n",
       "mean   299999.500000      -3.758503       0.000405       0.356208   \n",
       "std    173205.225094      54.881882       0.999419      31.804123   \n",
       "min         0.000000    -271.659497      -4.372119    -148.870768   \n",
       "25%    149999.750000     -40.693313      -0.673108     -20.758308   \n",
       "50%    299999.500000      -3.954771       0.000915       0.372583   \n",
       "75%    449999.250000      33.174835       0.673056      21.495530   \n",
       "max    599999.000000     250.812280       5.062006     170.053291   \n",
       "\n",
       "                 X_4            X_5            X_6            X_7  \\\n",
       "count  600000.000000  600000.000000  600000.000000  600000.000000   \n",
       "mean       -1.004378       3.376919      -6.396371      -2.253712   \n",
       "std        45.291429      53.397644     140.873734      59.810396   \n",
       "min      -244.446728    -302.574049    -683.126343    -322.731683   \n",
       "25%       -30.644608     -31.865404    -100.762161     -42.313674   \n",
       "50%        -0.585368       3.720738      -6.357443      -2.263690   \n",
       "75%        29.027860      38.988940      88.159514      37.709780   \n",
       "max       235.095937     284.915947     656.482242     293.909622   \n",
       "\n",
       "                 X_8            X_9  ...           X_41           X_42  \\\n",
       "count  600000.000000  600000.000000  ...  600000.000000  600000.000000   \n",
       "mean       -6.432606      -0.061507  ...       6.882868      -6.632834   \n",
       "std        74.840857      44.912292  ...     163.776538     137.025868   \n",
       "min      -506.202937    -218.466369  ...    -810.842593    -633.575178   \n",
       "25%       -54.840796     -30.327330  ...    -102.658380     -99.033996   \n",
       "50%        -6.416419      -0.103742  ...       7.190600      -6.784760   \n",
       "75%        41.962767      30.144501  ...     116.755600      85.621324   \n",
       "max       550.525780     219.628423  ...     867.086690     689.626208   \n",
       "\n",
       "                X_43           X_44           X_45           X_46  \\\n",
       "count  600000.000000  600000.000000  600000.000000  600000.000000   \n",
       "mean        8.454493       0.001296       0.007967      -0.000966   \n",
       "std       262.840194       1.000368      71.553713       0.999902   \n",
       "min     -1345.838757      -4.756720    -360.713742      -4.516004   \n",
       "25%      -167.634846      -0.673780     -48.250836      -0.675549   \n",
       "50%         8.773280       0.001639       0.045537      -0.002251   \n",
       "75%       185.382370       0.675779      48.221733       0.673638   \n",
       "max      1488.759454       4.727996     384.665348       5.086304   \n",
       "\n",
       "                X_47           X_48           X_49           X_50  \n",
       "count  600000.000000  600000.000000  600000.000000  600000.000000  \n",
       "mean      -22.259600      -5.759041       6.241130      -1.176456  \n",
       "std       500.900364     130.952113     141.211999      21.363662  \n",
       "min     -2506.960013    -687.526201    -702.184241     -98.094323  \n",
       "25%      -357.547278     -93.163915     -88.803657     -15.580688  \n",
       "50%       -20.695017      -5.774627       6.286783      -1.199895  \n",
       "75%       313.295748      81.636824     101.558007      13.230410  \n",
       "max      2534.503855     595.321844     630.727101     112.233293  \n",
       "\n",
       "[8 rows x 51 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.data.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "float64    50\n",
       "int64       1\n",
       "dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.data.dtypes.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number NA: 0\n"
     ]
    }
   ],
   "source": [
    "print('Number NA:', dataset.data.isna().sum().sum())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There are 51 columns in the dataset: 50 feature values are float and id is integer. The dataset has no missing values."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  Target and treatment "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Also take a look at target and treatment."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7f15a65c8c10>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZcAAAEHCAYAAABiAAtOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAXl0lEQVR4nO3df/BddX3n8efLAGqryK/I0gQMq+mMwVXUFOja3aWwA8GZbagFC7s1qWUmjgtt3dGu2OksVGVHt1Wm+IMpLpFArYhYIDqxMRtR1yqQoJEkUMp30ZUIC8FERF3R4Hv/uJ/vcvPN/X7zJZx7v+Sb52PmzD3nfT7ncz43uZNXzo97bqoKSZK69JyZHoAkafYxXCRJnTNcJEmdM1wkSZ0zXCRJnTNcJEmdO2imB/BscdRRR9WCBQtmehiStF+58847H62quRPrhkuzYMECNm7cONPDkKT9SpL/PajuaTFJUucMF0lS5wwXSVLnDBdJUucMF0lS54YWLkmel+SOJN9KsjXJn7f68UluT3Jfkk8lOaTVn9uWx9r6BX19vavV701yZl99SauNJbm4rz5wH5Kk0RjmkcsTwGlV9SrgRGBJklOA9wOXV9VCYCdwQWt/AbCzql4GXN7akWQRcB5wArAE+GiSOUnmAB8BzgIWAee3tkyxD0nSCAwtXKrnR23x4DYVcBpwY6uvAs5u80vbMm396UnS6tdX1RNV9W1gDDipTWNVdX9V/Qy4HljatplsH5KkERjqlyjb0cWdwMvoHWX8L+AHVbWrNdkGzGvz84AHAKpqV5LHgCNb/ba+bvu3eWBC/eS2zWT7kA5I3333v5jpIehZ6Lj/snlofQ81XKrqSeDEJIcBNwEvH9SsvWaSdZPVBx11TdV+D0lWACsAjjvuuEFNnpbX/sm1z7gPzT53/sWymR6CNHIjuVusqn4AfAk4BTgsyXiozQcebPPbgGMB2voXATv66xO2maz+6BT7mDiuq6pqcVUtnjt3j0fjSJL20TDvFpvbjlhI8nzg3wL3ALcC57Rmy4Fb2vzqtkxb/8WqqlY/r91NdjywELgD2AAsbHeGHULvov/qts1k+5AkjcAwT4sdA6xq112eA9xQVZ9LcjdwfZL3At8Erm7trwauSzJG74jlPICq2prkBuBuYBdwYTvdRpKLgLXAHGBlVW1tfb1zkn1IkkZgaOFSVXcBrx5Qv5/enV4T6z8Fzp2kr8uAywbU1wBrprsPSdJo+A19SVLnDBdJUucMF0lS5wwXSVLnDBdJUucMF0lS5wwXSVLnDBdJUucMF0lS5wwXSVLnDBdJUucMF0lS5wwXSVLnDBdJUucMF0lS5wwXSVLnDBdJUucMF0lS5wwXSVLnDBdJUucMF0lS5wwXSVLnDBdJUucMF0lS5wwXSVLnhhYuSY5NcmuSe5JsTfLHrX5pku8l2dSm1/dt864kY0nuTXJmX31Jq40lubivfnyS25Pcl+RTSQ5p9ee25bG2fsGw3qckaU/DPHLZBby9ql4OnAJcmGRRW3d5VZ3YpjUAbd15wAnAEuCjSeYkmQN8BDgLWASc39fP+1tfC4GdwAWtfgGws6peBlze2kmSRmRo4VJVD1XVN9r848A9wLwpNlkKXF9VT1TVt4Ex4KQ2jVXV/VX1M+B6YGmSAKcBN7btVwFn9/W1qs3fCJze2kuSRmAk11zaaalXA7e30kVJ7kqyMsnhrTYPeKBvs22tNln9SOAHVbVrQn23vtr6x1p7SdIIDD1ckrwA+Azwtqr6IXAl8FLgROAh4APjTQdsXvtQn6qviWNbkWRjko3bt2+f8n1IkqZvqOGS5GB6wfKJqvo7gKp6uKqerKpfAB+jd9oLekcex/ZtPh94cIr6o8BhSQ6aUN+tr7b+RcCOieOrqquqanFVLZ47d+4zfbuSpGaYd4sFuBq4p6o+2Fc/pq/ZbwNb2vxq4Lx2p9fxwELgDmADsLDdGXYIvYv+q6uqgFuBc9r2y4Fb+vpa3ubPAb7Y2kuSRuCgvTfZZ68D3gRsTrKp1f6U3t1eJ9I7TfUd4C0AVbU1yQ3A3fTuNLuwqp4ESHIRsBaYA6ysqq2tv3cC1yd5L/BNemFGe70uyRi9I5bzhvg+JUkTDC1cquqrDL72sWaKbS4DLhtQXzNou6q6n6dOq/XXfwqc+3TGK0nqjt/QlyR1znCRJHXOcJEkdc5wkSR1znCRJHXOcJEkdc5wkSR1znCRJHXOcJEkdc5wkSR1znCRJHXOcJEkdc5wkSR1znCRJHXOcJEkdc5wkSR1znCRJHXOcJEkdc5wkSR1znCRJHXOcJEkdc5wkSR1znCRJHXOcJEkdc5wkSR1znCRJHVuaOGS5Ngktya5J8nWJH/c6kckWZfkvvZ6eKsnyRVJxpLcleQ1fX0tb+3vS7K8r/7aJJvbNlckyVT7kCSNxjCPXHYBb6+qlwOnABcmWQRcDKyvqoXA+rYMcBawsE0rgCuhFxTAJcDJwEnAJX1hcWVrO77dklafbB+SpBEYWrhU1UNV9Y02/zhwDzAPWAqsas1WAWe3+aXAtdVzG3BYkmOAM4F1VbWjqnYC64Albd2hVfX1qirg2gl9DdqHJGkERnLNJckC4NXA7cDRVfUQ9AIIeHFrNg94oG+zba02VX3bgDpT7EOSNAJDD5ckLwA+A7ytqn44VdMBtdqH+tMZ24okG5Ns3L59+9PZVJI0haGGS5KD6QXLJ6rq71r54XZKi/b6SKtvA47t23w+8OBe6vMH1Kfax26q6qqqWlxVi+fOnbtvb1KStIdh3i0W4Grgnqr6YN+q1cD4HV/LgVv66svaXWOnAI+1U1prgTOSHN4u5J8BrG3rHk9yStvXsgl9DdqHJGkEDhpi368D3gRsTrKp1f4UeB9wQ5ILgO8C57Z1a4DXA2PAT4A3A1TVjiTvATa0du+uqh1t/q3ANcDzgc+3iSn2IUkagaGFS1V9lcHXRQBOH9C+gAsn6WslsHJAfSPwigH17w/ahyRpNPyGviSpc4aLJKlzhoskqXOGiySpc4aLJKlzhoskqXOGiySpc4aLJKlzhoskqXOGiySpc4aLJKlzhoskqXPTCpck66dTkyQJ9vJU5CTPA34JOKr9lsr4U44PBX5lyGOTJO2n9vbI/bcAb6MXJHfyVLj8EPjIEMclSdqPTRkuVfVXwF8l+cOq+tCIxiRJ2s9N68fCqupDSf4lsKB/m6q6dkjjkiTtx6YVLkmuA14KbAKebOUCDBdJ0h6m+zPHi4FF7aeIJUma0nS/57IF+GfDHIgkafaY7pHLUcDdSe4AnhgvVtVvDWVUkqT92nTD5dJhDkKSNLtM926xLw97IJKk2WO6d4s9Tu/uMIBDgIOBH1fVocMamCRp/zXdI5cX9i8nORs4aSgjkiTt9/bpqchVdTNw2lRtkqxM8kiSLX21S5N8L8mmNr2+b927kowluTfJmX31Ja02luTivvrxSW5Pcl+STyU5pNWf25bH2voF+/IeJUn7brpPRX5D33ROkvfx1GmyyVwDLBlQv7yqTmzTmtb/IuA84IS2zUeTzEkyh94zzM4CFgHnt7YA7299LQR2Ahe0+gXAzqp6GXB5aydJGqHpHrn8u77pTOBxYOlUG1TVV4Ad0+x/KXB9VT1RVd8GxuiddjsJGKuq+6vqZ8D1wNIkoXfkdGPbfhVwdl9fq9r8jcDprb0kaUSme83lzR3u86Iky4CNwNuraicwD7itr822VgN4YEL9ZOBI4AdVtWtA+3nj21TVriSPtfaPdvgeJElTmO5psflJbmrXUB5O8pkk8/dhf1fSe0bZicBDwAfGdzGgbe1Dfaq+9pBkRZKNSTZu3759qnFLkp6G6Z4W+ziwmt7vuswDPttqT0tVPVxVT1bVL4CP8dQdZ9uAY/uazgcenKL+KHBYkoMm1Hfrq61/EZOcnquqq6pqcVUtnjt37tN9O5KkSUw3XOZW1ceralebrgGe9r/GSY7pW/xtes8sg15wndfu9DoeWAjcAWwAFrY7ww6hd9F/dXuA5q3AOW375cAtfX0tb/PnAF/0gZuSNFrTffzLo0l+D/hkWz4f+P5UGyT5JHAqvZ9I3gZcApya5ER6p6m+Q++XLqmqrUluAO4GdgEXVtWTrZ+LgLXAHGBlVW1tu3gncH2S9wLfBK5u9auB65KM0TtiOW+a71GS1JHphssfAB+md2tvAV8DprzIX1XnDyhfPaA23v4y4LIB9TXAmgH1+xnwRc6q+ilw7lRjkyQN13TD5T3A8nZnF0mOAP6SXuhIkrSb6V5zeeV4sABU1Q7g1cMZkiRpfzfdcHlOksPHF9qRy3SPeiRJB5jpBsQHgK8luZHeNZc3MuD6iCRJMP1v6F+bZCO9R64EeENV3T3UkUmS9lvTPrXVwsRAkSTt1T49cl+SpKkYLpKkzhkukqTOGS6SpM4ZLpKkzhkukqTOGS6SpM4ZLpKkzhkukqTOGS6SpM4ZLpKkzhkukqTOGS6SpM4ZLpKkzhkukqTOGS6SpM4ZLpKkzhkukqTOGS6SpM4ZLpKkzg0tXJKsTPJIki19tSOSrEtyX3s9vNWT5IokY0nuSvKavm2Wt/b3JVneV39tks1tmyuSZKp9SJJGZ5hHLtcASybULgbWV9VCYH1bBjgLWNimFcCV0AsK4BLgZOAk4JK+sLiytR3fbsle9iFJGpGhhUtVfQXYMaG8FFjV5lcBZ/fVr62e24DDkhwDnAmsq6odVbUTWAcsaesOraqvV1UB107oa9A+JEkjMuprLkdX1UMA7fXFrT4PeKCv3bZWm6q+bUB9qn3sIcmKJBuTbNy+ffs+vylJ0u6eLRf0M6BW+1B/WqrqqqpaXFWL586d+3Q3lyRNYtTh8nA7pUV7faTVtwHH9rWbDzy4l/r8AfWp9iFJGpFRh8tqYPyOr+XALX31Ze2usVOAx9oprbXAGUkObxfyzwDWtnWPJzml3SW2bEJfg/YhSRqRg4bVcZJPAqcCRyXZRu+ur/cBNyS5APgucG5rvgZ4PTAG/AR4M0BV7UjyHmBDa/fuqhq/SeCt9O5Iez7w+TYxxT4kSSMytHCpqvMnWXX6gLYFXDhJPyuBlQPqG4FXDKh/f9A+JEmj82y5oC9JmkUMF0lS5wwXSVLnDBdJUucMF0lS5wwXSVLnDBdJUucMF0lS5wwXSVLnDBdJUucMF0lS5wwXSVLnDBdJUucMF0lS5wwXSVLnDBdJUucMF0lS5wwXSVLnDBdJUucMF0lS5wwXSVLnDBdJUucMF0lS5wwXSVLnDBdJUudmJFySfCfJ5iSbkmxstSOSrEtyX3s9vNWT5IokY0nuSvKavn6Wt/b3JVneV39t63+sbZvRv0tJOnDN5JHLb1bViVW1uC1fDKyvqoXA+rYMcBawsE0rgCuhF0bAJcDJwEnAJeOB1Nqs6NtuyfDfjiRp3LPptNhSYFWbXwWc3Ve/tnpuAw5LcgxwJrCuqnZU1U5gHbCkrTu0qr5eVQVc29eXJGkEZipcCvhCkjuTrGi1o6vqIYD2+uJWnwc80Lfttlabqr5tQF2SNCIHzdB+X1dVDyZ5MbAuyT9O0XbQ9ZLah/qeHfeCbQXAcccdN/WIJUnTNiNHLlX1YHt9BLiJ3jWTh9spLdrrI635NuDYvs3nAw/upT5/QH3QOK6qqsVVtXju3LnP9G1JkpqRh0uSX07ywvF54AxgC7AaGL/jazlwS5tfDSxrd42dAjzWTputBc5Icni7kH8GsLatezzJKe0usWV9fUmSRmAmTosdDdzU7g4+CPjbqvr7JBuAG5JcAHwXOLe1XwO8HhgDfgK8GaCqdiR5D7ChtXt3Ve1o828FrgGeD3y+TZKkERl5uFTV/cCrBtS/D5w+oF7AhZP0tRJYOaC+EXjFMx6sJGmfPJtuRZYkzRKGiySpc4aLJKlzhoskqXOGiySpc4aLJKlzhoskqXOGiySpc4aLJKlzhoskqXOGiySpc4aLJKlzhoskqXOGiySpc4aLJKlzhoskqXOGiySpc4aLJKlzhoskqXOGiySpc4aLJKlzhoskqXOGiySpc4aLJKlzhoskqXOGiySpc7M2XJIsSXJvkrEkF8/0eCTpQDIrwyXJHOAjwFnAIuD8JItmdlSSdOCYleECnASMVdX9VfUz4Hpg6QyPSZIOGAfN9ACGZB7wQN/yNuDkiY2SrABWtMUfJbl3BGM7UBwFPDrTg3g2yF8un+khaHd+Nsddki56ecmg4mwNl0F/YrVHoeoq4KrhD+fAk2RjVS2e6XFIE/nZHI3ZelpsG3Bs3/J84MEZGoskHXBma7hsABYmOT7JIcB5wOoZHpMkHTBm5WmxqtqV5CJgLTAHWFlVW2d4WAcaTzfq2crP5gikao9LEZIkPSOz9bSYJGkGGS6SpM4ZLhqKJAuS/Pt93G7LMMakZ78khyX5jx3297Ykv9RVfzO9n/2J4aJhWQAMDJcks/JGEnXiMGCPcGmPdNoXbwNG8Y/+qPaz3zBcNFCSZUnuSvKtJNcleUmS9a22Pslxrd01Sa5I8rUk9yc5p3XxPuBfJdmU5D8l+f0kn07yWeAL6fmLJFuSbE7yuzP2ZvVs8j7gpe1zsyHJrUn+FtgMkOT3ktzR1v/1eOgkuTLJxiRbk/x5q/0R8CvArUlubbUfJXl/kjuT/I8kJyX5Uvvs/lZrM6d9Nje0z/tbWv3U1vbGJP+Y5BPtc7zHfgRUlZPTbhNwAnAvcFRbPgL4LLC8Lf8BcHObvwb4NL3/qCyi90w3gFOBz/X1+fv0vtx6RFv+HWAdvVvFjwa+CxxD74hny0z/GTjN2Gfv///9t8/Qj4Hj2/LL2+fw4Lb8UWBZmx//XM0BvgS8si1/Z/xz3JYLOKvN3wR8ATgYeBWwqdVXAH/W5p8LbASOb+N5jN6Xsp8DfB34jUH7carZ+T0XPWOnATdW1aMAVbUjya8Db2jrrwP+W1/7m6vqF8DdSY6eot91VbWjzf8G8MmqehJ4OMmXgV8D7uryjWi/d0dVfbvNnw68FtiQBOD5wCNt3RvbswIPoveflEUM/iz9DPj7Nr8ZeKKqfp5kM71gAzgDeGXfUfiLgIVt2zuqahtAkk1tm68+87c5+xguGiQMeBbbBP3rn5iw7WR+PM120riJn5lVVfWu/gZJjgfeAfxaVe1Mcg3wvEn6+3m1Qw3gF7TPblX9ou9aYIA/rKq1E/ZzKrt/1p/Ef0Mn5TUXDbKe3v8EjwRIcgTwNXqP0QH4D+z9f2uPAy+cYv1XgN9t57fnAv8auOMZjVqzwVSfm/XAOUleDL3PZZKXAIfSC6HH2pHzWdPsbzJrgbcmObjt51eT/PIzGPcBydTVHqpqa5LLgC8neRL4JvBHwMokfwJsB968l27uAnYl+Ra96zI7J6y/Cfh14Fv0joL+c1X9nyQLunof2v9U1feT/EO7Hf3/Ag/3rbs7yZ/RuyHkOcDPgQur6rYk3wS2AvcD/9DX5VXA55M8VFW/Oc1h/Hd6p7u+kd75t+3A2XvZZl/2M6v5+BdJUuc8LSZJ6pzhIknqnOEiSeqc4SJJ6pzhogOWD0mUhsdw0YHsgH5Iog8Q1TAZLjqQzdqHJCa5IMk/tT4+luTDrX5Nkg+2bd/fvoh4c9v3bUle2dpdmuQdff1tSe/nEBa08axq29zoUZQGmumHmzk5zdTELH1IIr3w+Q69B44eDPxP4MNt3TXA54A5bflDwCVt/rS+cV0KvKOvzy3tz2tBe1+va/WV/e2cnMYnj1ykp0z2kMRNbfmft3VvTPINek8uOIHeQxIHmfiQxC9X1c/b/IJWPwNY1vZxO3AkvYckjo9nW/UeCrqpb5u9Oanta0fb36cnrP909R4YCr0HiF4HUFVfBI5M8qK99P9AVY1/C/5vWh/SbjznKj1ltjwkcW8PBd3bA0QL2MXup82fN2H9xPbSbjxy0YFstj4k8Q7g3yQ5vIXY70zR9iv0HkQ6HmiPVtUP6Z1We02rv4beqbpxx7WfYAA4Hx85rwE8ctEBq2bpQxKr6ntJ/iu902wPAnfTu34zyKXAx5PcBfwEWN7qn+Gp03UbgH/q2+YeYHmSvwbuA67c25vUgccHV0qzUJIXVNWP2pHLTcDKqrqpg34X0PuF0Vc80740u3laTJqdLm1HHVuAbwM3z/B4dIDxyEXajyW5nd4tzP3eVFWbZ2I80jjDRZLUOU+LSZI6Z7hIkjpnuEiSOme4SJI6Z7hIkjpnuEiSOvf/AMXxfBUuReu2AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.countplot(x=dataset.treatment)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "treatment    300368\n",
       "control      299632\n",
       "Name: treatment_group, dtype: int64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.treatment.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7f15a42bb5d0>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZcAAAEKCAYAAADenhiQAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAATLUlEQVR4nO3df6xf9X3f8ecrdkiyrJRfDqM21Gj1pjppQ4ND2LJuHVRgaFejLmSgdrYyq64Q6VJ16kpWqWzJ0IKSLStdioaCi50mdVC6FC9z4lgOJKoSfpiG8nMZd6QLlllsYkJJs5BC3vvj+zF8c/2919c3n+/32tfPh/TV95z3+Zzz+VyEeHHO+ZzzTVUhSVJPr1joAUiSFh/DRZLUneEiSerOcJEkdWe4SJK6M1wkSd2NNVyS/EWSh5I8kGRPq52WZFeSx9v3qa2eJDclmUryYJI3DR1nQ2v/eJINQ/Xz2/Gn2r6ZrQ9J0mRM4szlH1fVeVW1pq1fB+yuqlXA7rYOcBmwqn02ATfDICiA64G3ABcA1w+Fxc2t7aH91h6hD0nSBCzEZbF1wJa2vAW4Yqi+tQbuBk5JchZwKbCrqg5W1TPALmBt23ZyVX2pBk+Cbp12rFF9SJImYOmYj1/AZ5MU8F+r6hbgzKp6CqCqnkryutZ2OfDk0L57W222+t4RdWbpY0ZnnHFGrVy58ij/PEk6sd1///1PV9Wy6fVxh8tbq2pf+4/7riT/c5a2GVGredTnLMkmBpfVOOecc9izZ8/R7C5JJ7wk/2dUfayXxapqX/veD3ySwT2Tr7dLWrTv/a35XuDsod1XAPuOUF8xos4sfUwf3y1Vtaaq1ixbdljwSpLmaWzhkuS1SX7o0DJwCfAwsB04NONrA3BHW94OrG+zxi4Enm2XtnYClyQ5td3IvwTY2bY9l+TCNkts/bRjjepDkjQB47wsdibwyTY7eCnwsar6TJL7gNuTbAS+BlzZ2u8ALgemgG8D7wCoqoNJ3gvc19q9p6oOtuVrgNuA1wCfbh+A983QhyRpAuIr9wfWrFlT3nORpKOT5P6hR01e4hP6kqTuDBdJUneGiySpO8NFktSd4SJJ6m7cT+ifUM7/za0LPQQdY+5///qFHoK0IDxzkSR1Z7hIkrozXCRJ3RkukqTuDBdJUneGiySpO8NFktSd4SJJ6s5wkSR1Z7hIkrozXCRJ3RkukqTuDBdJUneGiySpO8NFktSd4SJJ6s5wkSR1Z7hIkrozXCRJ3RkukqTuDBdJUneGiySpO8NFktSd4SJJ6s5wkSR1Z7hIkrozXCRJ3RkukqTuDBdJUndjD5ckS5J8Ocmn2vq5Se5J8niSjyc5qdVf1dan2vaVQ8d4d6t/JcmlQ/W1rTaV5Lqh+sg+JEmTMYkzl3cBjw2t3wh8sKpWAc8AG1t9I/BMVf0Y8MHWjiSrgauA1wNrgd9vgbUE+BBwGbAauLq1na0PSdIEjDVckqwAfg74cFsPcBHwidZkC3BFW17X1mnbL27t1wHbqur5qvoqMAVc0D5TVfVEVX0X2AasO0IfkqQJGPeZy38G/jXwvbZ+OvDNqnqhre8Flrfl5cCTAG37s639S/Vp+8xUn62P75NkU5I9SfYcOHBgvn+jJGmasYVLkp8H9lfV/cPlEU3rCNt61Q8vVt1SVWuqas2yZctGNZEkzcPSMR77rcAvJLkceDVwMoMzmVOSLG1nFiuAfa39XuBsYG+SpcAPAweH6ocM7zOq/vQsfUiSJmBsZy5V9e6qWlFVKxnckP9cVf0ScCfwttZsA3BHW97e1mnbP1dV1epXtdlk5wKrgHuB+4BVbWbYSa2P7W2fmfqQJE3AQjzn8lvAbySZYnB/5NZWvxU4vdV/A7gOoKoeAW4HHgU+A1xbVS+2s5J3AjsZzEa7vbWdrQ9J0gSM87LYS6rqLuCutvwEg5le09t8B7hyhv1vAG4YUd8B7BhRH9mHJGkyfEJfktSd4SJJ6s5wkSR1Z7hIkrozXCRJ3RkukqTuDBdJUneGiySpO8NFktSd4SJJ6s5wkSR1Z7hIkrozXCRJ3RkukqTuDBdJUneGiySpO8NFktSd4SJJ6s5wkSR1Z7hIkrozXCRJ3RkukqTuDBdJUneGiySpO8NFktSd4SJJ6s5wkSR1Z7hIkrozXCRJ3RkukqTuDBdJUneGiySpO8NFktSd4SJJ6m5s4ZLk1UnuTfLnSR5J8u9a/dwk9yR5PMnHk5zU6q9q61Nt+8qhY7271b+S5NKh+tpWm0py3VB9ZB+SpMkY55nL88BFVfVG4DxgbZILgRuBD1bVKuAZYGNrvxF4pqp+DPhga0eS1cBVwOuBtcDvJ1mSZAnwIeAyYDVwdWvLLH1IkiZgbOFSA99qq69snwIuAj7R6luAK9ryurZO235xkrT6tqp6vqq+CkwBF7TPVFU9UVXfBbYB69o+M/UhSZqAsd5zaWcYDwD7gV3A/wa+WVUvtCZ7geVteTnwJEDb/ixw+nB92j4z1U+fpQ9J0gSMNVyq6sWqOg9YweBM48dHNWvfmWFbr/phkmxKsifJngMHDoxqIkmah4nMFquqbwJ3ARcCpyRZ2jatAPa15b3A2QBt+w8DB4fr0/aZqf70LH1MH9ctVbWmqtYsW7bsB/kTJUlDxjlbbFmSU9rya4CfBR4D7gTe1pptAO5oy9vbOm3756qqWv2qNpvsXGAVcC9wH7CqzQw7icFN/+1tn5n6kCRNwNIjN5m3s4AtbVbXK4Dbq+pTSR4FtiX598CXgVtb+1uBjySZYnDGchVAVT2S5HbgUeAF4NqqehEgyTuBncASYHNVPdKO9Vsz9CFJmoCxhUtVPQj81Ij6Ewzuv0yvfwe4coZj3QDcMKK+A9gx1z4kSZPhE/qSpO4MF0lSd4aLJKk7w0WS1N2cwiXJ7rnUJEmCI8wWS/Jq4G8AZyQ5lZeffj8Z+JExj02SdJw60lTkXwV+nUGQ3M/L4fKXDN5ILEnSYWYNl6r6XeB3k/xaVf3ehMYkSTrOzekhyqr6vSR/H1g5vE9VbR3TuCRJx7E5hUuSjwB/G3gAeLGVCzBcJEmHmevrX9YAq9tLISVJmtVcn3N5GPhb4xyIJGnxmOuZyxnAo0nuBZ4/VKyqXxjLqCRJx7W5hsu/HecgJEmLy1xni31+3AORJC0ec50t9hwv/w79ScArgb+qqpPHNTBJ0vFrrmcuPzS8nuQK/DEuSdIM5vVW5Kr6E+CizmORJC0Sc70s9otDq69g8NyLz7xIkkaa62yxfzK0/ALwF8C67qORJC0Kc73n8o5xD0SStHjM9cfCViT5ZJL9Sb6e5I+TrBj34CRJx6e53tD/A2A7g991WQ7891aTJOkwcw2XZVX1B1X1QvvcBiwb47gkScexuYbL00l+OcmS9vll4BvjHJgk6fg113D5F8Dbgf8LPAW8DfAmvyRppLlORX4vsKGqngFIchrwAQahI0nS95nrmctPHgoWgKo6CPzUeIYkSTrezTVcXpHk1EMr7cxlrmc9kqQTzFwD4j8CX0zyCQavfXk7cMPYRiVJOq7N9Qn9rUn2MHhZZYBfrKpHxzoySdJxa86XtlqYGCiSpCOa1yv3JUmajeEiSepubOGS5OwkdyZ5LMkjSd7V6qcl2ZXk8fZ9aqsnyU1JppI8mORNQ8fa0No/nmTDUP38JA+1fW5Kktn6kCRNxjjPXF4A/lVV/ThwIXBtktXAdcDuqloF7G7rAJcBq9pnE3AzvDTt+XrgLQx+Wvn6obC4ubU9tN/aVp+pD0nSBIwtXKrqqar6s7b8HPAYgzcqrwO2tGZbgCva8jpgaw3cDZyS5CzgUmBXVR1sD3LuAta2bSdX1ZeqqoCt0441qg9J0gRM5J5LkpUMnui/Bzizqp6CQQABr2vNlgNPDu22t9Vmq+8dUWeWPiRJEzD2cEnyN4E/Bn69qv5ytqYjajWP+tGMbVOSPUn2HDhw4Gh2lSTNYqzhkuSVDILlo1X131r56+2SFu17f6vvBc4e2n0FsO8I9RUj6rP18X2q6paqWlNVa5Yt8+dpJKmXcc4WC3Ar8FhV/aehTduBQzO+NgB3DNXXt1ljFwLPtktaO4FLkpzabuRfAuxs255LcmHra/20Y43qQ5I0AeN8+eRbgX8OPJTkgVb7N8D7gNuTbAS+BlzZtu0ALgemgG/Tfi+mqg4meS9wX2v3nvZWZoBrgNuA1wCfbh9m6UOSNAFjC5eq+lNG3xcBuHhE+wKuneFYm4HNI+p7gDeMqH9jVB+SpMnwCX1JUneGiySpO8NFktSd4SJJ6s5wkSR1Z7hIkrozXCRJ3RkukqTuDBdJUneGiySpO8NFktSd4SJJ6s5wkSR1Z7hIkrozXCRJ3RkukqTuDBdJUneGiySpO8NFktSd4SJJ6s5wkSR1Z7hIkrozXCRJ3RkukqTuDBdJUneGiySpO8NFktSd4SJJ6s5wkSR1t3ShByBp/L72np9Y6CHoGHTO7zw0tmN75iJJ6s5wkSR1Z7hIkrozXCRJ3RkukqTuxhYuSTYn2Z/k4aHaaUl2JXm8fZ/a6klyU5KpJA8medPQPhta+8eTbBiqn5/kobbPTUkyWx+SpMkZ55nLbcDaabXrgN1VtQrY3dYBLgNWtc8m4GYYBAVwPfAW4ALg+qGwuLm1PbTf2iP0IUmakLGFS1V9ATg4rbwO2NKWtwBXDNW31sDdwClJzgIuBXZV1cGqegbYBaxt206uqi9VVQFbpx1rVB+SpAmZ9D2XM6vqKYD2/bpWXw48OdRub6vNVt87oj5bH4dJsinJniR7Dhw4MO8/SpL0/Y6VG/oZUat51I9KVd1SVWuqas2yZcuOdndJ0gwmHS5fb5e0aN/7W30vcPZQuxXAviPUV4yoz9aHJGlCJh0u24FDM742AHcM1de3WWMXAs+2S1o7gUuSnNpu5F8C7GzbnktyYZsltn7asUb1IUmakLG9uDLJHwE/A5yRZC+DWV/vA25PshH4GnBla74DuByYAr4NvAOgqg4meS9wX2v3nqo6NEngGgYz0l4DfLp9mKUPSdKEjC1cqurqGTZdPKJtAdfOcJzNwOYR9T3AG0bUvzGqD0nS5BwrN/QlSYuI4SJJ6s5wkSR1Z7hIkrozXCRJ3RkukqTuDBdJUneGiySpO8NFktSd4SJJ6s5wkSR1Z7hIkrozXCRJ3RkukqTuDBdJUneGiySpO8NFktSd4SJJ6s5wkSR1Z7hIkrozXCRJ3RkukqTuDBdJUneGiySpO8NFktSd4SJJ6s5wkSR1Z7hIkrozXCRJ3RkukqTuDBdJUneGiySpO8NFktSd4SJJ6m7RhkuStUm+kmQqyXULPR5JOpEsynBJsgT4EHAZsBq4OsnqhR2VJJ04FmW4ABcAU1X1RFV9F9gGrFvgMUnSCWOxhsty4Mmh9b2tJkmagKULPYAxyYhaHdYo2QRsaqvfSvKVsY7qxHIG8PRCD2Kh5QMbFnoIOpz/bh5y/aj/VB61Hx1VXKzhshc4e2h9BbBveqOqugW4ZVKDOpEk2VNVaxZ6HNJ0/rs5GYv1sth9wKok5yY5CbgK2L7AY5KkE8aiPHOpqheSvBPYCSwBNlfVIws8LEk6YSzKcAGoqh3AjoUexwnMy406Vvnv5gSk6rD73JIk/UAW6z0XSdICMlzUla/d0bEqyeYk+5M8vNBjOREYLurG1+7oGHcbsHahB3GiMFzUk6/d0TGrqr4AHFzocZwoDBf15Gt3JAGGi/qa02t3JC1+hot6mtNrdyQtfoaLevK1O5IAw0UdVdULwKHX7jwG3O5rd3SsSPJHwJeAv5tkb5KNCz2mxcwn9CVJ3XnmIknqznCRJHVnuEiSujNcJEndGS6SpO4MF2mRSfJhXxiqheZUZOk4lGRJVb240OOQZuKZizRPSdYneTDJnyf5SJIfTbK71XYnOae1uy3JTUm+mOSJJG9r9Y8nuXzoeLcl+adJliR5f5L72rF+tW3/mSR3JvkY8FCS1yb5H63/h5P8s9buriRr2vLVSR5q228c6utbSW5o+96d5MwJ/qPTCcBwkeYhyeuB3wYuqqo3Au8C/guwtap+EvgocNPQLmcB/wD4eeB9rbYNOBQIJwEXAzuAjcCzVfVm4M3AryQ5t+1zAfDbVbWawW+T7KuqN1bVG4DPTBvjjwA3AhcB5wFvTnJF2/xa4O429i8Av/KD/1ORXma4SPNzEfCJqnoaoKoOAn8P+Fjb/hEGYXLIn1TV96rqUeDQWcKngYuSvIrBD6x9oar+H3AJsD7JA8A9wOnAqrbPvVX11bb8EPCzSW5M8tNV9ey0Mb4ZuKuqDrRX83wU+Idt23eBT7Xl+4GV8/0HIY1iuEjzE478cwLD25+fti9V9R3gLuBSBmcw24a2/1pVndc+51bVZ9u2v3rp4FX/CzifQcj8hyS/M2KMM/nrevmG64vA0iP8LdJRMVyk+dkNvD3J6QBJTgO+yOBN0AC/BPzpHI6zDXgH8NMMXvhJ+74mySvbsf9OktdO37Fd9vp2Vf0h8AHgTdOa3AP8oyRntJ+gvhr4/Nz/RGn+/L8VaR6q6pEkNwCfT/Ii8GXgXwKbk/wmcIBBaBzJZ4GtwPb209AAH2ZwmerPkqQd64oR+/4E8P4k3wP+Grhm2hifSvJu4E4GZzE7quqOo/tLpflxKrIkqTsvi0mSujNcJEndGS6SpO4MF0lSd4aLJKk7w0WS1J3hIknqznCRJHX3/wEqUr2bAF6klgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.countplot(x=dataset.target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    477486\n",
       "1    122514\n",
       "Name: conversion, dtype: int64"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.target.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>conversion</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>treatment_group</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>control</th>\n",
       "      <td>0.820600</td>\n",
       "      <td>0.179400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>treatment</th>\n",
       "      <td>0.771081</td>\n",
       "      <td>0.228919</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "conversion              0         1\n",
       "treatment_group                    \n",
       "control          0.820600  0.179400\n",
       "treatment        0.771081  0.228919"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.crosstab(dataset.treatment, dataset.target, normalize='index')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As we can see, the target is not balanced."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Simple baseline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Convert the **treatment** to binary values."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset.treatment.loc[dataset.treatment == 'treatment'] = 1\n",
    "dataset.treatment.loc[dataset.treatment == 'control'] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    300368\n",
       "0    299632\n",
       "Name: treatment_group, dtype: int64"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.treatment.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In a binary classification problem definition we stratify train set by splitting target `0/1` column. In uplift modeling we have two columns instead of one. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train shape: (420000, 51)\n",
      "Validation shape: (180000, 51)\n"
     ]
    }
   ],
   "source": [
    "stratify_cols = pd.concat([dataset.treatment, dataset.target], axis=1)\n",
    "\n",
    "X_train, X_val, trmnt_train, trmnt_val, y_train, y_val = train_test_split(\n",
    "    dataset.data,\n",
    "    dataset.treatment,\n",
    "    dataset.target,\n",
    "    stratify=stratify_cols,\n",
    "    test_size=0.3,\n",
    "    random_state=31\n",
    ")\n",
    "\n",
    "print(f\"Train shape: {X_train.shape}\")\n",
    "print(f\"Validation shape: {X_val.shape}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "treatment_model = lgb.LGBMClassifier(random_state=31)\n",
    "control_model = lgb.LGBMClassifier(random_state=31)\n",
    "tm = TwoModels(estimator_trmnt = treatment_model, estimator_ctrl = control_model, method='vanilla')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "tm = tm.fit(X_train, y_train, trmnt_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "uplift_tm = tm.predict(X_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Qini coefficient on full data: 0.2401\n"
     ]
    }
   ],
   "source": [
    "# AUQC = area under Qini curve = Qini coefficient\n",
    "auqc = qini_auc_score(y_val, uplift_tm, trmnt_val) \n",
    "print(f\"Qini coefficient on full data: {auqc:.4f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgEAAAGSCAYAAAB6/BIRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nOzdd5xcdfX/8ddJsmTTe0jY3TSSzWwaASIBQi+B0ALYQClCbFgAxYKKIoL65edXrFjgKwqIFKWIFDEgAVQsoYM7CSHZhBRI7333/P743NmdbJ1sdvbOzryfj8c8ZubOnZmzKXvPPfdzPh9zd0RERKTwdIo7ABEREYmHkgAREZECpSRARESkQCkJEBERKVBKAkRERAqUkgAREZECpSRARESkQCkJEJEGzOyrZvZ/bb2viOQW02RBIoXHzD4CXAUcCGwEHgC+4u4b4oxLRNqXKgEiBcbMrgJuBL4I9AEOB0YAfzGzohhDa5KZdYk7BpF8pCRApICYWW/gOuCz7v5nd9/l7lXAB4CRwIei/b5pZr+NHo8wMzezi81siZmtNrOvpX1m7b5NfOdMM3vZzDaa2Vtmdmq0vcrMTmrsc9K+c5aZLQH+amZ/NrPP1PvsV8zs3Ohxwsxmm9laM5tnZh9okz80kTymJECksBwJFBPK/7XcfTPwODC9mfceBYwFTgS+YWYVLX2ZmR0G3EGoOvQFjgGq9iLeY4EK4BTgd8D5aZ89DhgOPGpmPYDZ0T6Do/1+Zmbj9+K7RAqOkgCRwjIQWO3uuxt5bQUwqJn3Xufu29z9FeAV4KAMvm8WcJu7z3b3Gndf5u7JvYj3m+6+xd23AQ8Ck81sePTah4EH3H0HcAZQ5e6/dvfd7v4icD/wvr34LpGCoyRApLCsBgY2cY19KLCqmfe+k/Z4K9Azg+8rA97KPLwG3k49cPdNwKPAedGm84C7osfDgalmtj51IyQJQ/bhu0XynpIAkcLyPLADODd9Y1ROnwE808bf9zahA6ExW4Duac8bO2DXb1+6GzjfzI4AugFPp33PM+7eN+3W090v24fYRfKekgCRAhK1AF4H/MTMTjWzIjMbAfyeUCW4q5m3t8avgEvM7EQz62RmJWaWiF57GTgvimEKmZXuHyOc9X8LuNfda6LtjwDlZnZh9HlFZvaeTMYtiBQyJQEiBcbd/x/wVeB/gU3AIsIZ+UnuvqWNv+vfwCXAD4ANhEpD6pr+1wlVgnWExOR3GXzeDsKgxpPS948uFUwnXCJYTrh0cSPQtY1+FJG8pMmCRAqcmV1KOAhPc/clcccjIu1HSYCIYGYXArvc/Z64YxGR9qMkQEREpEBpTICIiEiBUhIg0s7M7Ggzm9fW+4qI7C0lASLtzN2fc/exbb2vNGRmXc3stmjdgnfM7PPN7Huxmb0Q7bvUzP5fY5MqmdkYM9tef70EM/uQmS02sy1m9pCZ9U977TNmNtfMdpjZb9r0hxTZB0oCRCSntPGKgd8ExhDaEo8HvpRawKgR3YErCVMrTyWskfCFRva7GfhP+oZojYJfAhcC+xNmVPxZ2i7LgRuA21r5c4hkhZIAkSwxs4PN7EUz22Rm95rZPWZ2g5kdZ2ZL0/arMrMvmNmrZrYh2rc4em2PfZv5rqujFfo2mdl/zeyctNf2WOUvbYW+LtHz/mb2azNbbmbrzOyhFr5roJk9Ek3Pu9bMnjOzTtFrZWb2gJmtMrM1ZvbTaHsnM7smOlNeaWZ3mFmfevHUrhgYbT/czP4Rfc8rZnZc5n/6tS4Crnf3de5eCdwKfKSxHd3951HlZae7LyNMnDSt3s9+HrAeeKre2z8M/Mndn40WY/o6cK6Z9Yo++wF3fwhY04qfQSRrlASIZIGZ7Qc8BNwJ9CfMyPfeZt7yAeBUwnK+k2jiQNWMt4CjgT6Env/fmtnQDN97J+EseDxhBb4ftLD/VcBSwmJD+xMmHnIz60yYuW8xMAIoAVIthx+JbscDowjrDvy03ufWrhhoZiWEdQJuIPz5fQG438wGAZjZz9LXCah3ezXapx9wAGGxo5RXop8zE8cAb6SeWFiG+VvRz1/f+PTvcfe3gJ1AeYbfJRILJQEi2XE4UAT80N13ufsfqFdCrufH7r7c3dcCfwIm782Xufvvo/fXuPu9wJvAYS29L0oUZgCfjM6Wd7l7S+sH7CIsNjQ82v85D73GhxEOul+MVv7b7u5/i97zYeAmd18YnSl/hTBlcHrpP33FwAuAx9z9sehnmg3MBU6Lft5P1VsnIP02Kfq81AJHG9K+YwPQK4M/l0uAKYRZFVOuB37l7m838pae9b4n4+8SiZOSAJHsOABY5ntOxLG4mf1bs0JfLTO7yMxetroV9CYQrm23pAxY6+7r9uLrvgcsAP5iZgvN7Oq0z1rcxDLFB7Dnz78Y6EKoJKSkH1yHA++3PVcFPIqQfGRqc3TfO21bb8JUyU0ys7OB/wFmuPvqaNtkwlTFTVVJNtf7noy+SyRubTkAR0TqrABKzMzSEoFh7Nuyuo0ys+GEa90nAs+7e7WZvQxYtEtzq/W9DfQ3s77uvj6T74vm6b8KuCoaEPe0mf0n+qxhZtalkURgOXVrBkD4s9gNvAuUpj66Xlx3uvvHmviZf0GoFjRmsbuPd/d1ZrYCOAiYHb12EGkl/kY+91TCn+Xp7v5a2kvHES5xLDEzCElaZzMb5+6HRJ95UNrnjCKsWzC/qe8SyQWqBIhkx/OEg9zlZtbFzM4lg/J8K/UgHEBXQW0pe0La6y8Dx5jZsGgw3ldSL7j7CuBx4Gdm1s/C6nvHNPdlZnaGmY22cDTcCFRHt38Tkp//MbMeZlZsZqmBdXcDnzOzkWbWE/gOYRXAxqoGAL8FzjSzU8ysc/RZx5lZaRT3J6Olghu7pV/zvwO4JvrZEsDHgN808XOdQBgM+N5o4aN0txAWO5oc3X5BGLNwSvT6XVG8R1tYlvlbwANRwkT0b6AY6ExIHoqtbbsgRFpFSYBIFrj7TuBcwmC4dcAHCavfZeO7/gt8n5B4vAtMBP6e9vps4F7gVeAFwuC9dBcSrvMngZWENrnmjAGeJJTAnwd+5u5z3L0aOBMYDSwhDB78YPSe2wgDEJ8lrFq4HfhsMz/T28BMwqDDVYTKwBfZ+99Z1xKqL4sJKxh+z93/DBAlRZvNbFi079cJAysfi7ZvNrPHo3i2uvs7qVv0s29391XR628AnyQkAysJYwE+lRbHNcA24GpCBWNbtE0kVlo7QKSdWJgkZqm765e/iOQEVQJEREQKlJIAkQ4grXTd2G1Yy5+w19/31Sa+6/G2/i4RiY8uB4iIiBQoVQJEREQKlJIAERGRAlVwfaoDBw70ESNGxB2GiIhIu3jhhRdWu/ugxl4ruCRgxIgRzJ07N+4wRERE2oWZNTlluS4HiIiIFCglASIiIgVKSYCIiEiBUhIgIiJSoJQEiIiIFCglASIiIgVKSYCIiEiBUhIgIiJSoJQEiIiIFCglASIiIgVKSYCIiEiBKri1AwRwh+eegy1b4o5ERCRevXvDtGlxRxEbJQGF6Nln4bjj4o5CRCQ3vPgiHHxw3FHEQklAIXr11XD/5z9D377xxiIiEpdly+C97w2/E5UESMFIJqFPH5g+HczijkZEJB67dkFRUfidWKA0MLAQJZOQSCgBEJHCVlQEo0crCZACk0oCREQKXSKhJEAKyMaNsHy5kgAREQi/CxcsCJcGClDWkgAzKzOzp82s0szeMLMrou39zWy2mb0Z3feLtpuZ/djMFpjZq2Z2SNpnXRzt/6aZXZy2/VAzey16z4/NVN9uUSrjraiINw4RkVyQSMDu3bBwYdyRxCKblYDdwFXuXgEcDnzazMYBVwNPufsY4KnoOcAMYEx0+zjwcwhJA3AtMBU4DLg2lThE+3w87X2nZvHnyQ+pJECVABGRut+FBXpJIGtJgLuvcPcXo8ebgEqgBJgJ3B7tdjtwdvR4JnCHB/8E+prZUOAUYLa7r3X3dcBs4NTotd7u/ry7O3BH2mdJU5JJ6NIFRo2KOxIRkfiNHRvulQRkj5mNAA4G/gXs7+4rICQKwOBotxLg7bS3LY22Nbd9aSPbG/v+j5vZXDObu2rVqn39cTq2ZDKMhi0qijsSEZH49ekDQ4cqCcgWM+sJ3A9c6e4bm9u1kW3eiu0NN7rf4u5T3H3KoEGDWgo5v6kzQERkTwXcIZDVJMDMiggJwF3u/kC0+d2olE90vzLavhQoS3t7KbC8he2ljWyXpuzaFUbBKgkQEamTSgK80fPIvJbN7gADfgVUuvtNaS89DKRG+F8M/DFt+0VRl8DhwIbocsETwHQz6xcNCJwOPBG9tsnMDo++66K0z5LGLFwYEgF1BoiI1KmogPXrYeXKlvfNM9mcNngacCHwmpm9HG37KvA/wH1mNgtYArw/eu0x4DRgAbAVuATA3dea2fXAf6L9vuXua6PHlwG/AboBj0c3aYo6A0REGkrvENh//3hjaWdZSwLc/W80ft0e4MRG9nfg00181m3AbY1snwtM2IcwC0sqCUiNhhURkT2TgGOPjTeWdqYZAwtJMhlGwfbpE3ckIiK5o6QEevQoyMGBSgIKiToDREQa6tQpVEgrK+OOpN0pCSgU7koCRESaUqBtgkoCCsXKlWH0q5IAEZGGEglYvBi2bo07knalJKBQpMpcag8UEWkodYI0f368cbQzJQGFQu2BIiJNK9CFhJQEFIpkMox+LWl0eQURkcI2ZgyYKQmQPJVMhtGvnfRXLiLSQHExjBypJEDylDoDRESaV4AdAkoCCsHWrWHUq5IAEZGmJRIwbx7U1MQdSbtRElAI5s0L9+oMEBFpWiIB27fDkiVxR9JulAQUAnUGiIi0rAA7BJQEFIJkMgwIHD067khERHKXkgDJS8lkGPVaXBx3JCIiuWvgQOjfX0mA5Bl1BoiItMys4DoElATku+rqMA2mkgARkZZVVCgJkDyyZEkY7aokQESkZYkEvPsurFsXdyTtQklAvtPCQSIimUudMKVaq/OckoB8p/ZAEZHMFViHgJKAfJdMhhGvAwbEHYmISO4bMQL2209JgOQJdQaIiGSuS5ewomDqUmqeUxKQ75QEiIjsnQJqE1QSkM/WrIFVq5QEiIjsjUQC3noLdu6MO5KsUxKQz1KZrDoDREQyl0iEOVbeeivuSLJOSUA+U2eAiMjeK6AOASUB+SyZhK5dYfjwuCMREek4xo4N90oCpENLJqG8HDp3jjsSEZGOo1cvKClREiAdnDoDRERap0A6BJQE5KsdO2DhQiUBIiKtkUoC3OOOJKuUBOSrN9+EmholASIirZFIwMaN8M47cUeSVUoC8pXaA0VEWq9AOgSUBOSr1D/c8vJ44xAR6YiUBEiHlkzCsGHQo0fckYiIdDwlJdCzp5IA6aDUGSAi0npmBdEhoCQgH7krCRAR2VdKAqRDWrYMtmxREiAisi8SCViyJPw+zVNKAvJRah1sdQaIiLRe6kRq/vx448giJQH5SAsHiYjsuwLoEFASkI+SSejTB/bfP+5IREQ6rtGjoVOnuupqHlISkI9SgwLN4o5ERKTj6toVRo1SJUA6GHUGiIi0jTzvEFASkG82boTly5UEiIi0hUQiDAysro47kqxQEpBvNChQRKTtJBJhVdbFi+OOJCuUBOQbLRwkItJ28rxDQElAvkkmoUuXMJhFRET2jZIA6VCSydDWUlQUdyQiIh3fgAEwcKCSAOkg1BkgItK28rhDQElAPtm1CxYsUBIgItKWlARIh7BoUUgElASIiLSdRAJWrYI1a+KOpM0pCcgnqaktlQSIiLSd1O/UefPijSMLlATkE80RICLS9vK4Q0BJQD5JJmHo0LB4kIiItI0RI8I6AkoCJKepM0BEpO117gzl5UoCJIe5KwkQEcmWPO0QUBKQL1auhPXrlQSIiGRDIgELF4Z1BPJIi0mAmZWb2VNm9nr0fJKZXZP90GSvqDNARCR7EomwkuBbb8UdSZvKpBJwK/AVYBeAu78KnJfNoKQVtHCQiEj2pE6wUidceSKTJKC7u/+73rbd2QhG9kEyCT16QElJ3JGIiOSf8vJwn2fjAjJJAlab2YGAA5jZ+4AVWY1K9l4yCWPHQicN8xARaXM9e0JZWd4lAV0y2OfTwC1AwsyWAYuAC7Ialey9ZBKmTYs7ChGR/JWHHQItnja6+0J3PwkYBCTc/Sh3r8p6ZJK5rVth8WINChQRyaZUEuAedyRtpsVKgJn1BS4CRgBdzAwAd788q5FJ5ubPD/dKAkREsieRgM2bYfnyvBl/lcnlgMeAfwKvATXZDUdaRe2BIiLZl76GQJ4kAZmMIit298+7+6/d/fbUraU3mdltZrYyNb9AtO2bZrbMzF6ObqelvfYVM1tgZvPM7JS07adG2xaY2dVp20ea2b/M7E0zu9fM9tuLnzu/JJNhQOCYMXFHIiKSv/JwIaFMkoA7zexjZjbUzPqnbhm87zfAqY1s/4G7T45ujwGY2TjC3APjo/f8zMw6m1ln4GZgBjAOOD/aF+DG6LPGAOuAWRnElJ+SSRg5EoqL445ERCR/DR0KvXoVXBKwE/ge8DzwQnSb29Kb3P1ZYG2GccwE7nH3He6+CFgAHBbdFkSDE3cC9wAzLQxMOAH4Q/T+24GzM/yu/KM1A0REss8s7zoEMkkCPg+MdvcR7j4yuo3ah+/8jJm9Gl0u6BdtKwHeTttnabStqe0DgPXuvrve9kaZ2cfNbK6ZzV21atU+hJ6DqqvDwEAlASIi2VeAScAbwNY2+r6fAwcCkwkTDn0/2m6N7Out2N4od7/F3ae4+5RBgwbtXcS5bskS2L5dSYCISHtIJGDpUti0Ke5I2kQm3QHVwMtm9jRQu3xSa1oE3f3d1GMzuxV4JHq6FChL27UUWB49bmz7aqCvmXWJqgHp+xcWdQaIiLSf1Pos8+fDoYfGG0sbyKQS8BDwbeAf1I0JeKE1X2ZmQ9OengOkOgceBs4zs65mNhIYA/wb+A8wJuoE2I8wePBhd3fgaeB90fsvBv7Ympg6PC0cJCLSfvKsQ6DFSoC73x4dgKPVE5jn7rtaep+Z3Q0cBww0s6XAtcBxZjaZULqvAj4RfccbZnYf8F/C4kSfdvfq6HM+AzwBdAZuc/c3oq/4MnCPmd0AvAT8KqOfON8kkzBwIAwYEHckIiL578ADoXPnwkkCzOw4wuj7KsK1+DIzuzga/d8kdz+/kc1NHqjd/duEikP97Y8RJiyqv30hoXugsKkzQESk/ey3X0gECiUJIAzem+7u8wDMrBy4G+j4F0PyQTIJM2fGHYWISOHIow6BTMYEFKUSAAB3nw8UZS8kydiaNbBqlSoBIiLtKZEIAwOrq+OOZJ9lkgTMNbNfmdlx0e1WWjkwUNrYvCg3UxIgItJ+EgnYuRMWLYo7kn2WSRJwGWGugMuBKwiD9z6ZzaAkQ2oPFBFpf3nUIZBJEtAF+JG7n+vu5wA/JozUl7glk9C1K4wYEXckIiKFY+zYcF8gScBTQLe0592AJ7MTjuyVZBLKy0O7ioiItI/+/WHw4IJJAordfXPqSfS4e/ZCkoypPVBEJB550iGQSRKwxcwOST0xs0OBbdkLSTKyYwcsXKgkQEQkDnmSBGQyT8CVwO/NLDU3/1DC9L0SpwULoKZGSYCISBwSidCmvXp1mLW1g8okCXgVSABjCTMGJsmsgiDZpM4AEZH4pHcIHHVUvLHsg0wO5s+7+y53f93dX4vWDXg+24FJC1JlqNQoVRERaT950ibYZCXAzIYAJUA3MzuYUAUA6I0GBsYvmYRhw6BHj7gjEREpPMOGQXFx/iYBwCnAR4BS4Ka07ZuAr2YxJsmEOgNEROLTuXNo0c7XJMDdbwduN7P3uvv97RiTtMQ9/MObNSvuSEREClciAS907Fn0MxkYOMHMxtff6O7fykI8kolly2DLFlUCRETiVFEBf/gDbN8eLg10QJkMDNwMbIlu1cAMYEQWY5KWpMpPSgJEROKTSIRW7QUL4o6k1VqsBLj799Ofm9n/Ag9nLSJpmdoDRUTil94hMGFCvLG0Umv6/bsDo9o6ENkLyST06QNDhsQdiYhI4SovD/cdeHBgi5UAM3sN8OhpZ2AQoPEAcUp1Bpi1vK+IiGRH9+4wfHh+JwHAGWmPdwPvuvvuLMUjmUgm4eST445CREQ6+BoCLV4OcPfFQF/gTOAcYFy2g5JmbNwIy5drPICISC5IJQHuLe+bg1pMAszsCuAuYHB0u8vMPpvtwKQJ8+aFeyUBIiLxSyRCy/bSpXFH0iqZXA6YBUx19y0AZnYjYe2An2QzMGmCOgNERHJHeodAWVm8sbRCJt0BRpgfIKWaunUEpL0lk9ClCxx4YNyRiIhIB19IKJNKwK+Bf5nZg9Hzs4FfZS8kaVYyCaNHQ1FR3JGIiMj++4eW7XxNAtz9JjObAxxFqABc4u4vZTswaYIWDhIRyR1mHbpDIJNKAO7+IvBilmORluzaFaannDkz7khERCQlkYDZs+OOolVaM2OgxGXRopAIqBIgIpI7EonQur1xY9yR7DUlAR2JFg4SEck9qd/JqRbuDkRJQEei9kARkdzTgTsEmhwTYGabqFszYI+XAHf33lmLShqXTMLQoWEkqoiI5IYDDwyt2/mUBLh7r/YMRDKgzgARkdxTVBQSgQ6YBGR8OcDMBpvZsNQtm0FJI9yVBIiI5KqKivxMAszsLDN7E1gEPANUAY9nOS6pb+VKWL9eSYCISC5KJODNN2F3x1pkN5NKwPXA4cB8dx8JnAj8PatRSUPqDBARyV2JRGjhXrQo7kj2SiZJwC53XwN0MrNO7v40MDnLcUl96gwQEcldHbRDIJMkYL2Z9QSeJSwj/COgY9U78kEyCT16QGlp3JGIiEh9Y8eG+zxMAmYCW4HPAX8G3gLOyGZQ0ohkMvwj66SpHUREck7fvjBkSF4mAd9w9xp33+3ut7v7j4EvZzswqUedASIiua0DLiSUSRJwciPbZrR1INKMrVth8WIlASIiuSyRCOO3vLF59nJTczMGXgZ8ChhlZq+mvdQLdQe0r/nzw72SABGR3JVIwLp1sGoVDB4cdzQZaW4p4d8R5gP4LnB12vZN7r42q1HJntQZICKS+9I7BDpIEtDk5QB33+DuVe5+PrAU2EVYS6CnZgxsZ8lkGBA4ZkzckYiISFM6YJtgc5UAAMzsM8A3gXeBmmizA5OyF5bsIZmEkSOhuDjuSEREpCllZdCtW34lAcCVwNhowiCJgzoDRERyX6dOoZW7AyUBmXQHvA1syHYg0oTq6jAwUEmAiEju62BtgplUAhYCc8zsUWBHaqO735S1qKTOkiWwfbuSABGRjiCRgHvvhW3bwqWBHJdJJWAJMBvYj9AemLpJe9DCQSIiHUciEeYJePPNuCPJSIuVAHe/DsDMerj7luyHJHtQe6CISMeR3iEwKffHz7dYCTCzI8zsv0Bl9PwgM/tZ1iOTIJmEAQNg4MC4IxERkZaUl4NZhxkXkMnlgB8CpwBrANz9FeCYbAYlaZJJqKiIOwoREclEt24wYkReJQG4+9v1NlVnIRZpjNoDRUQ6lg7UIZBRi6CZHQm4me1nZl8gujQgWbZmTZiDWkmAiEjHkUjAvHlQU9PyvjHLJAn4JPBpoIQwffDk6Llk27x54V5JgIhIx5FIhNVfly6NO5IWZdIdsBr4cDvEIvWpM0BEpONJ7xAYlttL7WSydsBI4LPAiPT93f2s7IUlQPgH1LVrGGQiIiIdQ3oSMH16vLG0IJMZAx8CfgX8iboFhKQ9JJOh3aRz57gjERGRTA0aBP36dYjBgZkkAdvd/cdZj0QaSibh4IPjjkJERPaGWagGVOb+GPpMBgb+yMyujSYNOiR1y3pkhW7HDli4UOMBREQ6og7SJphJJWAicCFwAnWXAzx6LtmyYEFoL1ESICLS8SQS8Otfw/r10Ldv3NE0KZMk4BxglLvvzHYwkkYLB4mIdFyp393z5sHUqfHG0oxMLge8AuRuGpOvUteSysvjjUNERPZeeodADsukErA/kDSz/wA7UhvVIphlySSUlUHPnnFHIiIie2vkSCgqyosk4NrWfLCZ3QacAax09wnRtv7AvYQ5B6qAD7j7OjMz4EfAacBW4CPu/mL0nouBa6KPvcHdb4+2Hwr8BugGPAZc4e7emlhzkhYOEhHpuIqKYPTonE8CWrwc4O7PEA7YRdHj/wAvZvDZvwFOrbftauApdx8DPBU9B5gBjIluHwd+DrVJw7XAVOAw4Foz6xe95+fRvqn31f+ujstdCweJiHR0HaBDoMUkwMw+BvwB+GW0qYQwgVCz3P1ZYG29zTOB26PHtwNnp22/w4N/An3NbChhCePZ7r7W3dcBs4FTo9d6u/vz0dn/HWmf1fEtWwZbtigJEBHpyBKJ0Om1a1fckTQpk4GBnwamARsB3P1NYHArv29/d18Rfc6KtM8pAdKXK14abWtu+9JGtjfKzD5uZnPNbO6qVataGXo7UmeAiEjHl0jA7t1hzpcclUkSsCO9PdDMuhDmCWhL1sg2b8X2Rrn7Le4+xd2nDBo0qJUhtiMtHCQi0vGlxnXl8CWBTJKAZ8zsq0A3MzsZ+D1hHYHWeDcq5RPdr4y2LwXK0vYrBZa3sL20ke35IZmE3r1hyJC4IxERkdYaOzbcd/Ak4GpgFfAa8AnCSPxrmn1H0x4GLo4eXwz8MW37RRYcDmyILhc8AUw3s37RgMDpwBPRa5vM7PCos+CitM/q+FKdAdZYwUNERDqE3r3hgANyOglotkXQzDoDt7v7BcCte/PBZnY3cBww0MyWEkb5/w9wn5nNApYA7492f4zQHriA0CJ4CYC7rzWz6wkdCQDfcvfUYMPLqGsRfDy65YdkEk4+Oe4oRERkX+V4h0CzSYC7V5vZIDPbb2+nDXb385t46cRG9nXCAMTGPuc24LZGts8FJuxNTB3Cxo2wfLnGA4iI5INEAn73u9D6nYPV3UwmC6oC/m5mDwNbUhvd/aZsBVXQ5s0L90oCREQ6vkQiLCK0ciXsv3/c0TSQSRKwPLp1AnplNxxRe6CISB5JX0OgIyYB7n5dewQikcpK6NIFDjww7khERGRfpZpDe8UAACAASURBVCcBxx4bbyyNyGTGwNlm1jfteT8zeyK7YRWwZDIkAEVFcUciIiL7qqQEevSom/8lx2TSIjjI3dennkTT97Z2xkBpiRYOEhHJH506hfkCcrRDIJMkoNrMhqWemNlw2n7GQIEwv/SCBRoPICKST3K4TTCTgYFfA/5mZs9Ez48hrN4nbW3RopAIKAkQEckfqTbBrVuhe/e4o9lDJgMD/2xmhwCHE+bs/5y7r856ZIVInQEiIvkn9Tt9/nyYPDneWOrJ5HIAQFfCssAbgHFmdkz2QipgqYEjqfmmRUSk40vvEMgxLVYCzOxG4IPAG0BNtNmBZ7MYV2FKJsOiQX37tryviIh0DGPGhNkCO2ISAJwNjHX3HdkOpuCpM0BEJP8UF8PIkTmZBGRyOWAhoKb1bHMP/0A0HkBEJP/kaIdAJpWArcDLZvYUUFsNcPfLsxZVIVq5MswvrSRARCT/JBLw179CTU2YOyBHZJIEPBzdJJvUGSAikr8qKmD7dliyBEaMiDuaWpm0CN7eHoEUPCUBIiL5K71DIIeSgCZrEmZ2X3T/mpm9Wv/WfiEWiMrKMIlEaWnckYiISFvL0TbB5ioBV0T3Z7RHIAUvmQzzA+TQtSIREWkjAwfCgAEdJwlw9xXR/eL2C6eAJZMwbVrcUYiISLbkYIeATjtzwdatsHixxgOIiOQzJQHSqPnzw72SABGR/JVIwLvvwrp1cUdSq7mBgU9F9ze2XzgFSp0BIiL5L/U7ft68eONI09zAwKFmdixwlpndQ1hBsJa7v5jVyApJZWWYV3rMmLgjERGRbEnvEDj88HhjiTSXBHwDuBooBW6q95oDJ2QrqIKTTIZ5pYuL445ERLLEPdybtbzftm1QVBTmlunWDbpkMq2b5L4RI2C//epWjM0BzXUH/AH4g5l93d2vb8eYCo8WDhLJedXVYcbX7dth7lxYtQruvjt0fv3lL2EiOAgFvTffzE4Ml10Gn/scjB7dcjIhOahLl/APJIcGB2YyY+D1ZnYWcEy0aY67P5LdsApIdXUYGHjyyXFHIlLwVq+Giy6CZcvg1VZOidZWCcDYsQ0vHf/85+GWbtQoWLgQ+vWDf/1LVxVzXiIBr70WdxS1WkwCzOy7wGHAXdGmK8xsmrt/JauRFYolS8KphQYFirQLd/j73+Gaa+Ctt2DyZNi0CZ55pvn3TZgQBnV37x66eocNC2fkI0fCkCHhJK9btz3fU1EBK1aEtcEOPDBUgiE879EjxPLWW+E+VWXo0aPxwuAbb4Tbk0/Ci2kjshYuDPfr1kF5ed3200+Hs88Oce7cCSedpCuOOSGRgIceCn8pqX8QMTJPXahqaocwRfBkd6+JnncGXnL3Se0QX5ubMmWKz507N+4w6jz+OJx2Gjz3HBx1VNzRiOSVhQvhS1+C++/P/D2zZoWz8EGDwoF90KAw0Vsult/dw+Xl6uqQHPzpT2Hbpk1Nv+f440O146KLNEFpLH77W7jwQvjvf9vtMrCZveDuUxp7LdPhJn2BtdHjPm0SlQRqDxTZZ+7w9tvwkY/A009n9p7eveHKK8P9/vuHbR1taI4ZjBsXHk+cGMYLQEgCZs+G228Plzb69QuVhg0bwp/P00/DJZeEysDXvx4Gqh9/fG4mOnknvUMgB/7BZZIEfBd4ycyeJrQJHgPoUkBbqawMpxkDB8YdiUiHsHEjvPwy9OkD114Lf/xj8/t/8pMwZQp07bpnST6f9eoF554bbuk2bYJHH4UnnoDOncOf49e+1vhnnHACXHwxzJgRqiHSRsaODfc5MjiwxcsBAGY2FHgPIQn4l7u/k+3AsiXnLgccc0xI0f/2t7gjEckZ7vC738GPfgQ9e8LgwXDvvc2/56CDwjX6gw4K1+khJ060ctru3eFXz+OPw1NPNb1fSUlIuj7ykXDV8ogj2i3E/FRaCieeGEo17WCfLwdEiwk93KZRSZBMwsyZcUchEqs33oB77oEbbti79330o6EM3r+/Dvit0aULHHdcuKVUVoZBho89Blu2hMsJCxeG+y99KewzbBhMnQrveQ+ccUZof68/KFKakUNrCGgKijitWROajTUeQAqEezjzvPJKWL4c3mmmplheHg5C739/uGY/dGg409cI9+xKJVNHHlm3be1aWLo0dDM//jgsWAC//324pRKDzp1Du+JHPwqf+ESoHEgTKirgjjvCf4iYB2IoCYhTqglYSYDkqc2b4dlnw4Dou+8O1+V37Gi4X3k5nHlmOKvv3Dn8l9AgtdzRv3+4TZoE73tf+Dt85ZUwwHD9+jAIsbo6zJHw5S+HG8Chh8LnPw/vfW/4u5dIIhEGt7zzTshuY9RsEmBmnYBX3X1CO8VTWNQZIHnokUfCtfwnn2z4WvfuYTT+cceFUvLQoeHMPnUNXzqGrl3hsMPCDeC73w2dB88/Hy7tvPNOSBBeeAE+/OFwO/xwuPzykOhNKPQjSnqHQC4nAe5eY2avmNkwd1/SXkEVjMrKMFR5xIi4IxFpld27Yc6c0Jr2+uuN7zNtGhx9dKiAduqka/f5qk8fOPXUcEt55RW4777QjfDPf4YbhGaofv1CFeG++8Jgw4JaHyE9CTj++FhDyeSPfSjwhpn9G9iS2ujuZ2UtqkKRTIY6aOfOcUcikrFVq0Jb2a23Nr3PjTfCAQdAWVkY3S+F6aCDwu3b34aXXoLFi8O6C3PmhCmaoe4YWF4Od94ZKkR5fynogAPCf4wcGByYSRJwXdajKFTJJBx8cNxRiNRyD6XcRx+FBx8MU9hCGL7S2Fz6XbqEQXtXXw19+4ZtOtOXxhx8cLidfXZ4/vzzYazIqlXh39z8+aHj4IAD4JRT4LzzwgnzsGHxxp0VZjnTIZDJAkLPmNlwYIy7P2lm3QGduu6rHTtC383558cdiRS4pUvhBz+Am+ovGN6M978/tIaNH5+9uCS/HXHEnvMNPPlkWNPh+efh178ONwiXGfr1CwXT224Llw7yYrrjRCKMmo1ZJgsIfQz4ONAfOBAoAX4BnJjd0PLcggVhkiANCpR2tGoV/OEP8KlPNb/fuHFhJLh7qAJ88IPhDE3X9CVbTjop3Nzhz38OlafNm8PJ8qJFYZ9jjw33iURIQs8/P1SiSkrii7vVEolQCtmypa7kFoNMLgd8mrCK4L8A3P1NMxuc1agKgToDpJ2sWRNGZi9Y0PQ+qWv4oIO8xMssTFU8Y0bdttdfD/NK/Pe/4biZTIbb//5v3T7nnRfmKfja10IXSs5L/e6fPz/Wy8KZJAE73H2nRSM1zKwL0PJcw9K8VBKQvvanyD5yD4OuZs2qO3tqzDe+Ubfu/JAhodwqkqsmTAi36dPDRFOPPAL/+ldorqqpCasn3nNP2Pc73wn3H/hAqBxcdlmODjRMJQGVlTmfBDxjZl8FupnZycCngD9lN6wCUFmpodPSJnbsgA99CB54oOl9Tj01nCmNG5cn11OloJ1xRrilXHttWI39lltg165Q9brvvnD79KfDPr/4RUiOc6YVcfTo8J8x5sGBLS4gFE0YNAuYTlhA6Ang/zyTlYdyUM4sIDRlSpiC6y9/iTsS6WDWrg2/7Kqq4Je/bPj6JZeES4yTJoUcU8UmKTSrV4cq++WXN3xt0KAwydFDD+VAQjBmTKgC3HdfVr9mnxYQiiYMup0wJsCBeR01AcgZ7iH7mzUr7kikA1iyJIycvuMOeOaZxvd5z3vCsq99+uiavsjAgeGWOt/785/he98LsxquWhVaYIuK9nzPEUeEwfrtmhjkQJtgJt0BpxO6Ad4iVAJGmtkn3P3xbAeXt5YtCyNCNShQGrF7N3z2s6F8WV9xcTjLP//8MPhpwoTwfPjw9o9TpKNIn8nwjTfCwkePPLLnPs8/X5cYdO4cBiIOzvYQ+ESibuGFmCaNyyTn+T5wvLsvADCzA4FHASUBraXOAKlnxw745CfhN79p+FppaRgp3bs3TJ6sM32RfTF+fLh985uwfXv4dfz734epjQcODJcSqqtD62HKunV1k2G1qUQi/OdfvDi0NsQgkyRgZSoBiCwEVmYpnsKgJEAIv1h++tMwUr++sWPhwgvDGb4O+iLZUVwcEuvJk8PUxhAWPZozJ6x6mZLqnrnxxrqlk9tE+hoCuZYEmNm50cM3zOwx4D7CmID3A/9ph9jyV2VlOK0bMiTuSKSd7d4Nv/pVOOuvr2dPuPlmTcgjEqdDDw23q64K8xL88Ifw4ovhtdQyyV/9al3SsE/Sk4DTTmuDD9x7zVUCzkx7/C4QzdXEKkBdxfsimdSC6QXk2WfrZjqr74orwllIly468IvkmnHjQicOhBUQf/lLeO21MBfBd74TEvYNG/ah03vAgHANIsbBgU0mAe5+SXsGUlCSyTA/puStX/0qnEXMmVN3FpFy9tlw8smhGKQDv0jHcPjh4fbcc2HpbAgTFfXqFR5Pnw6PP96KeTgqKnIzCUgxs5HAZ4ER6ftrKeFW2rgxDDvVb/+8smVLOOD/9a+hfFhTU/dacXFY9OSii6BbNxg5MrYwRWQfHX10aD1cuhQ+//mwDhyEKV9SA/yXLAlzwWUkkQiTFsQkk4GBDwG/IswSWNPCvtKSefPCvQYFdmju8MorYRDRRz/a8PVRo0Lf8dFHwyGHaJY+kXxTWlo3x8+iRWFlzZTU8sdr12YwJXciESYvWLMmXB5oZ5kkAdvd/cdZj6RQqDOgQ1u+PFzJqaxs+Nq4ceGvdebMMEtf/clIRCQ/jRxZNzHRe98bOv4gTAr7j3/suWRyA6ljwbx5cOSRWY2zMZkkAT8ys2uBvwA7Uhvd/cWm3yJNqqwMo8AOPDDuSKQZ7vDgg+E/dHM+/3mYOFED+0QkuP/+MM/A1Knh+ZFHwllnwR//2MQb0jsEcjQJmAhcCJxA3eUAj57L3komQwKg08Sc8tZbYQaxOXOavzx33HFhAFBZmRo8RKRxnTuHysBJJ8H69fDww+F3RaMT7g8fDl27xjY4MJMk4BxglLvvzHYwBSHVHiixcQ8zN19ySRin+e9/N75feTl8+MN1c3iMHBkG+YmIZOLJJ+Hll+vGDTWaCHTuHH7Z5HAS8ArQF80SuO9Sa1zOnBl3JAXFPSy88/Ofw4oVocWnMZdeGvr5O3WCoUOzNE2oiBSUyZPD75/UXCEPPgjnnFNvp0QiZAsxyCQJ2B9Imtl/2HNMgFoE99aiRSERUCUgq9zhzTdDS96//rXna127hltZWVhx72MfCwvxdOmiJXdFJDt69AiXGc85B849N0wVft11aTskEvDAA2Edga5d2zW2TJKAa7MeRaFQZ0DW1NSE0v3KlaFXP11RUVi2+8QTwzriJSVhoh4RkfZSWgq//W34PfWtb8GVV6a1DyYSYTThggVhdaN21GIS4O5NrGAuey2VBIwdG28cecI9JM/ve1/D10pKYNas8B+vWzeN3BeR+I0dG05GnnoqtA/W1ESDi9M7BHItCTCzTYRuAID9gCJgi7vrXGpvVVaGRYN0sbnVdu8O19Tuvx/uvXfP16ZMgU98Igze00FfRHLRjTeG31UQxh+5U3ctMobBgZlUAnqlPzezs4HDshZRPlNnQKtUV4ez+V27Gr42aRJccEEYyKcDv4h0BP/+d7g0CWHc0tSpPcNApRiSgL2ezNTdH0JzBOw9dyUBe8EdnngitNB26bJnAjBlCvzgB3DnnXDbbXDCCUoARKTj6NQJbr89PL7oomhjIpGblQAzOzftaSdgCnWXB1rFzKqATUA1sNvdp5hZf+BewkJFVcAH3H2dmRnwI+A0YCvwkdRshWZ2MXBN9LE3uPvt+xJXVq1cGWaN0NGqUdXVYWKNOXNChpzeu19UFJKAn/wkXEnRH6GIdHTjx4e5R+bPDyuNHpJIwK9/Hc6A2nEWsky6A85Me7ybcIBui0b34919ddrzq4Gn3P1/zOzq6PmXgRnAmOg2Ffg5MDVKGq6lLil5wcwedvd1bRBb21NnQAP//S9ccUWYUCPdv/8dRs2OHx+W3B01Sgd+Eck/3/0unHdemENg040J2Lw5LFBSUtJuMWQyJuCS9giEkFgcFz2+HZhDSAJmAne4uwP/NLO+ZjY02ne2u68FMLPZwKnA3e0U795REsBbb4W+/Kefbnqfb3wjtPP16QMHHNB+sYmItLfRo8P95s3wTt8EQyAcK3IhCTCzbzTzPnf36/fhex34i5k58Et3vwXY391XRB++wswGR/uWAG+nvXdptK2p7Q2Y2ceBjwMMS63x2N4qK8OsNKWl8Xx/TJ55Bh57DP7f/2v89XPPhTPP1AI8IlKYrr8evv51+N4jFXwfQhJw4ont9v3NVQK2NLKtBzALGADsSxIwzd2XRwf62WbW3GiIxi6OeDPbG24MScYtAFOmTNmn8QytlkyGJtECWFh+2TJ49NHQrlffV74SzvSHDoVBg9o/NhGRXDJjRpg98Ka7h/C/vXtj7Tw4sMkkwN2/n3psZr2AK4BLgHuA7zf1vky4+/LofqWZPUhoOXzXzIZGVYCh1K1VsBQoS3t7KbA82n5cve1z9iWurIppmcj28Pe/w1FHNf5aIhGudx16qCbtERFpzGc+Az/8obG8d4KSdk4Cmj0tNbP+ZnYD8CohYTjE3b/s7q1eTMjMekRJBWbWA5gOvA48DFwc7XYxkFp9+WHgIgsOBzZElw2eAKabWT8z6xd9zhOtjSurtm6FxYvz6gj4pz/BwIFhEGt6ApC6xnX66XDrrfC1r4VxAIccklc/vohIm/ngB8P9k0sTeK5UAszse8C5hDL6RHff3EbfuT/wYOj8owvwO3f/c7RA0X1mNgtYArw/2v8xQnvgAkKL4CUA7r7WzK4H/hPt963UIMGcM39+uO/ggwK3b4dPfzr05qd06gQDBsDZZ4dCx+jR4YxfREQyU1QUEoHkvQls6R2waRP06tXyG9uAeYPFjaMXzGoIqwbuZs9r7UYYGNghpw2eMmWKz507t32/9J574Pzz4dVXYeLE9v3ufbRxYyjnN7bK5Xe+Eya50hm+iMi+2boVbj7pQe7ZeS7MnRuuobYRM3vB3ac09lpzYwLyfwRbe0kmQ918zJi4I8nY66/DhRc2PPh/4hMwbVpodBgxIpbQRETyTvfu0OPQBDwPq56rZFAbJgHNyWSyINlXlZVhaqji4rgjada2bXD55fB//1e3rVOncD3/M5+Bzp111i8i0hbcnTVr3qWqKklVVSVVVUkq17/BbuD/vvBLvnLlBe0Sh5KA9pDjawakrlakGzcuDO6bNEkHfhGR1tq9ezfLly+iqirJokWVexz0N21aX7tft249KS1N8ONOFeyacHi7xackINuqq8PAwJNOijuSPSxdCscfDwsW7Lm9X7+wOM+oUaE8JSIiLdu2bQuLF8+rPdCn7t9++0127dpZu1+/fkMYNqyC448/n7FjKxg3LsFBB1UwalQJ3boZ++3XvnErCci2JUvCsPocOZ1OJhsP5XvfgyFDciZMEZGc4+6sXbuy9iC/eHHdwf6dd5bU7tepU2eGDj2QsrIEhx9+OhUVFUyYkGDixAT779+X4uIwS2ouyJEw8liOrBngHlbhu+KKum2f/CQccUSYyDBX/kGKiMSturqa5csXpZXv60r4GzfWrVFXXNyD0tIE48YdzcyZ4ax+0qQKEokD6dOnK1275v4ksfrVn20xJwGLF4ev3r69btsXv6hr/SIioYQ/v/YAnzroL1kyv0EJv6wswXHHnUd5eaK2hD9yZAk9enRq9xJ+W1ISkG2VlWE2nYED2/VrX3stHOjTDR4clq6cNKldl6sWEYmNu7Nu3apGz+pXrFhcu1+nTp0YMiSU8KdOPY1EIsHEiRVMmDCWIUP65VQJvy3l4Y+UY9q5M2DjxrAMb7ovfxkmTNCZv4jkr+rqalasqGowMK+qqnKPEn7Xrt0pK0uQSEzjzDM/SkVFOKuvqBjdYUr4bUlJQLYlk3DWWVn/mu3bw5K8Tz5Zt+2000Lrnw7+IpIvtm/fyuLF8xsMzFuyZD47d+6o3a9fv/0pLU1w7LEfpLw8wfjxFUyalGDUqFK6d+9E164x/hA5RElANq1ZA6tWZf0o/OEPw+9+V/f81FPDNh38RaQjcnfWr1+9x1l96oDfsIQ/irKyBO95z6kkEgkmTAgH+3wu4bcl/fFk07x54T5LlwPmzw8j+1MuvRSOOSaU/kVEcl2qhJ+6Vp9+0N+4sW49uK5du1NaOpZEYhpnnDGLceMqmDgxQUXFaPr1Ky64En5bUhKQTVnqDKiuhksugTvvrNt2881w0EE5PzOxiBSg7du3sXjxvD0G5S1aVNmghN+37+CohP/+2hL+xIkJDjywjO7dwyh8DWpuW0oCsimZhP32a9OVdt55B4YOrXv+iU/AUUep9C8i8atfwk/dv/POYlIr1nbq1In99x/JsGEVTJlyColEONBPnJhg6ND+KuG3M/1RZ1NlJZSXh5V32sB//wvjx4fHkybBVVfVPRcRaQ+hhL+4wVl9VVWSDRvW1O7XtWs3SkrGkkgcwemnX1Jbwh83bgx9+xZTXKwSfi5QEpBNySRMntwmH/Xii3DkkeHxpZeGef919i8i2bJ9+zaWLJnfoN1u8eL57NxZN/tYnz6DKCur4Oij31s7F/7EiQlGjx6mEn4HoCQgW3bsgIUL4bzz9vmj5swJB/2uXcPiPhMnQt+++x6iiMj69asblO/DRDpVtSV8M6st4R9yyMkkEnVz4ZeUDFAJvwPTX1u2LFgANTX7fLp+9tnwxz+GoQXf+x4cdpj+s4nI3qmpqakt4dfvr1+/fnXtfvvtV0xp6VjKy6dy2mkXU1ERSvjjx5erhJ+ndDjJljboDLjpppAAANx4oxIAEWne9u3bePvtN/eYIjcc9OftUcLv3XsgZWUVTJt2DuXldcvZqoRfeHRIyZZUElBe3qq3z58fBv4BfP/7oQNA/ylFBGD9+jWNDsxbvnxRgxJ+WVmCgw8+sV4JfyDFxVBUFPMPIrFTEpAtlZVQVgY9e+71W9etq5sE6CtfgaOPVgIgUmhqamp4550lDQbmVVUlWbduVe1+qRL+6NHvYcaMi6ioCAvfjBs3hn79uqmEL81SEpAt+7Bw0GmnhfsvfQmmTdN/YJF8tmPH9j1K+Kn7xYvnsWPHttr9evceQFlZBUccMbNBCb9Hj84q4UurKAnIBveQBFx66V6/9Re/gH/+M3QWTpwIQ4ZkIT4RaXcbNqxtdGDe8uWLqKmpAVIl/BGUlSWYOfMExo4NZ/Uq4Uu2KAnIhmXLYMuWve4MWLECLrssPL7iCs0DINLR1NTU8O67bzey8E2SdetW1u5XVNSV0tKxjBp1KNOnX8C4cQkmTaqgomIM/ft3Vwlf2o2SgGxoZWfAmWeG+xtuCFUAEclNO3fuYMmSNxsMzFu8eB7bt2+t3a9Xr/5RCf8sxoypW8529Ojh9OypEr7ET0lANrQiCZg9G154IZz9Dx+epbhEZK9s3LiuwaC8RYsq9yjhA7Ul/LPOOo7y8roSfmnpIJXwJacpCciGZBJ69874gv6SJTB9enh85ZW6DCDSnmpqali5cmmjC9/UL+GXlJQzatShnHzyh6moCAPzxo0rVwlfOiwlAdlQWRmqABnW+VJn/tddB4ccksW4RApYXQl/z7P6+iX8nj37MWxYBYcffuYeJfwxY0bQo0dnunZVCV/yh5KAbEgm4aSTMtr1pZfC/fDhYRyAfrmI7JtNm9Y3OKuvqqpk2bKFe5TwBw8ezrBhFZx55rGMHZtgwoS6En63bqYSfg7YtWsXS5cuZfv27S3vLBQXF1NaWkrRXvzjVRLQ1jZuhOXLMxoP4B7WBgD4whdg2LAsxyaSJ9ydd999u96BPjxeu/bd2v26dNmP0tJyRow4mBNPPD9tOdtyBg7sQdeubbbSt2TB0qVL6dWrFyNGjMB0htQsd2fNmjUsXbqUkSNHZvw+JQFtbd68cJ/Bhf2bbgrjAY48Eg4+OMtxiXRAu3btrC3hp/fXL148j23bttTu17NnX8rKKjjssNMpL08wblwo4ZeXj1QJvwPbvn27EoAMmRkDBgxg1apVLe+cRklAW8uwM6CmJpz9A3zsY1BcnOW4RHLYpk3r9zibrxuFv5Dq6ura/QYPHkZZWQWnnXY0Y8eGs/pJkypUws9jSgAy15o/KyUBbS2ZDEv9HXhgs7tdcUW4/9CHYMKEdohLJGahhL+00YVv1qx5p3a/VAl/+PCDOOGEDzJuXFj4ZsKEsQwY0IPiYpXwpf2YGRdccAF33nknALt372bo0KFMnTqVRx55JOPPGTFiBHPnzmXgwIH7tE9bUxLQ1iorQwLQzCnJ5s3w05+Gx6ecojKl5Jddu3by9tsLGl34prES/pQpMxg7tqK25W7MmBH07NlFJXzJCT169OD1119n27ZtdOvWjdmzZ1NSUhJ3WG1GSUBby2DhoM9/PtxffjmMH98OMYlkwebNGxqU76uqkixb9tYeJfxBg8qiEv4sxo6tYPz4UMIfNmywSvjSIcyYMYNHH32U973vfdx9992cf/75PPfccwCsXbuWSy+9lIULF9K9e3duueUWJk2axJo1azj//PNZtWoVhx12WO0SzwC//e1v+fGPf8zOnTuZOnUqP/vZz+gcU3lLSUBb2rULFiyAs85qcpeNG+HWW8OkIu95TzvGJtIK7s7KlcsaXfhm9eoVtft16VJESUk5w4ZN5PjjP1C7nO348eUMHNhTJXzZZ1deCS+/3LafOXky/PCHLe933nnn8a1vfYszzjiDV199lUsvvbQ2Cbj22ms5+OCDeeihh/jrX//KRRddxMsvv8x1113HUUcdxTe+8Q0effRRbrnlFgAqKyu59957+fvf/05RURGf+tSnuOuuu7jooova9ofLkJKAtrRoUUgEmukMOOOMcP/xj2tmQMkdu3btZOnStxosfFNVlWTr1s21+/Xo0YeysgoOCbg7wgAAGN5JREFUPfQUxoypW862vHykSviStyZNmkRVVRV33303p6XWeo/87W9/4/777wfghBNOYM2aNWzYsIFnn32WBx54AIDTTz+dfv36AfDUU0/xwgsv8J7oLHDbtm0MHjy4HX+aPSkJaEstdAbs2AFR8si0ae0Uk0iaUMKf12Bg3tKlb1Fdvbt2v0GDyigtTTBjxqWUlycYPz4c7MvK9qd7d5Xwpf1lcsaeTWeddRZf+MIXmDNnDmvWrKndnl7mT0mN0m9stL67c/HFF/Pd7343e8HuBSUBbSmVBIwd2+jL3/52uL/mGlUBJHvcnVWrlje68E39Ev4BB4yhrGw8xx33PhKJUMKfMGGsSvgi9Vx66aX06dOHiRMnMmfOnNrtxxxzDHfddRdf//rXmTNnDgMHDqR3796126+55hoef/xx1q1bB8CJJ57IzJkz+dznPsfgwYNZu3YtmzZtYnhMK8cpCWhLyWRYNKhv3wYvbdgA118fHpeXt3Nckpd2797VoISf6rXfunVT7X7du/emrKyCQw45hTFjElRUJJg8OZTwe/UqUglfJAOlpaVckertTvPNb36TSy65hEmTJtG9e3duv/12IIwVOP/88znkkEM49thjGRZNCTtu3DhuuOEGpk+fTk1NDUVFRdx8882xJQHWWCkjn02ZMsXnzp2bnQ8/4ogw68/TTzd46bOfDW2Bl18OMY3/kA5q8+aNtQf31Fl9VVWSt99esEcJf+DAUsrKEowcWUF5eZgLf9KkBMOGDaFbN2O//WL8IURaobKykgqVTfdKY39mZvaCu09pbH9VAtqKe6gEnHdeg5d27YL77w8VAHUESGPcndWrVzQYmLdoUSWrVi2v3a9z5y5RCb+CY445d48S/qBBvVTCF5G9oiSgraxcCevXNzoo8Be/gBUrwuyASmoLW6qEX798X1WVZMuWjbX7hRJ+gsmTT2bMmETtwjeJxCh691YJX0TahpKAtpIaFNjIUf7yy8O9pgcuHFu2bGpQwl+0qLJBCX/AgBLKyhJMn37RHiX84cOHqoQvIlmnJKCtNNEeOHt2uJ8+XbMD5ptUCb+xhW9WrVpWu18o4Y+mrKyCo48+h0QiHOhDCb+3SvgiEhslAW0lmYTu3aG0tHaTezj4A5xzTpglUDqe3bt315bw6/fXp5fwu3XrFZXwT2T06LrlbBOJA+nVq4jiYpXwRSS3KAloK5WVYX6AtCP944+H+zPP1KWAjmDr1s0NzuqrqkIJf/fuXbX7DRhwAGVlFZx88oW1c+EfdFCFSvgi0uEoCWgrySQceWTtU3c4/fTw+KyzoFu3mOKSPbg7a9a80+jCNytXLq3dr1OnzhxwwGiGDatg2rSZJBJhYN7EiQkGD1YJX6RQdO7cmYkTJ7J7925GjhzJnXfeSd9G5oLZW1VVVZxxxhm8/vrrbRBl6ykJaAtbt8LixXDppbWbHn443E+dqipAHHbv3s2yZQsbXfhm8+YNtft169aTsrIKJk06njFjwnK2qRJ+7977qYQvUuC6devGy9HKRRdffDE333wzX/va12KOqu0oCWgL8+eH+7RBgV/+crifNQvNs55FW7duZvHieXu024VR+G/uUcLv338oZWUVnHjih/co4Y8YcYBK+CKSkSOOOIJXX30VgM2bNzNz5kzWrVvHrl27uOGGG5g5cyZVVVXMmDGDo446in/84x+UlJTwxz/+kW7duvHCCy9w6aWX0r17d4466qjaz92+fTuXXXYZc+fOpUuXLtx0000cf/zx/OY3v+Ghhx6iurqa119/nauuuoqdO3dy55130rVrVx577DH69++/Tz+TkoC2UK898JVXYN486NoVxo2LMa48EUr47zY6MO/dd9+u3S9Vwi8rSzBt2lmMHZtIK+H3obgYuuhfvEiHdOWVV9aekbeVyZMn88MMVyaqrq7mqaeeYtasWQAUFxfz4IMP0rt3b1avXs3hhx/OWdEy8m+++SZ33303t956Kx/4wAe4//77ueCCC7jkkkv4yU9+wrHHHssXv/jF2s+++eabAXjttddIJpNMnz6d+dHJ5euvv85LL73E9u3bGT16NDfeeCMvvfQSn/vc57jjjju48sor9+nPQL8S20IyGWrGY8YAcOGFYfOPfhRmEZbM7N69m+XLFzVY+KaqKsmmTetr9+vWrSelpQkmTDiWc8+tIJEIZ/WpEn7XrurEEJG2sW3bNiZPnkxVVRWHHnooJ598Mv+/vfsPjro+Ezj+fhJiIhjDL0FKAgSqJBFCUAoyFVFRQBrgLPTEcuipUwVOrcc4dyozjlo9bdG7HnNYS3v+qulBqlfK3JQSvVIRRgXhxB9NIOGnYARMIILyw+Bzf3w+u34TsiEJyW5293nNMNn97PfXs99l97PP97OfB9yXkwcffJC1a9eSkpLCvn372L9/PwC5ubkUFRUBcNlll7Fr1y7q6uo4fPgw48ePB2DOnDms8qPH161bx9133w1AXl4eAwcODHcCrr76ajIzM8nMzCQrK4upU6cCMHz48HBW4mxYJ6A9VFRAbi5kZPDuu/DBB3DVVTY7YCTHjn3RIIUf+vvxx5V89dXJ8HIuhZ/H1VffxNChrnZ9UVE+ubn9LYVvTJJp6Tf29hYaE1BXV0dxcTFLlizhnnvuoaSkhIMHD7Jp0ybS0tIYNGgQx48fByA9PT28fmpqKseOHUNVmywtDE2XIw4JbislJSV8PyUlhfr6+kirtZh1AtpDeXl4PECoNsBNN0G3bjE8phhTVWprD4Q/4IMD8z79dE94uZSUVPr1G0JOTh5jxxY3mAu/b9/ulsI3xnQKWVlZLF68mOnTpzNv3jzq6uro06cPaWlprFmzht27dze7fvfu3cnKymLdunVcccUVlJSUhB8LlR2+5ppr2LZtG3v27GHo0KFs3ry5o8OyTsBZO3XKDQy89loOHHBNKSnJMztgfX091dW7Tit8s2tXBZ9/fii8XEZGN7Kz8ygoGMf06e5bfWGhS+FnZaVbCt8Y0+mNHDmSESNGsGzZMmbPns3UqVMZNWoURUVF5DVRN6ax559/PjwwcNKkSeH2+fPnM3fuXIYPH06XLl144YUXGmQAOpKVEj5bO3fC4MGwdCk3/flHLFsGjzzyzRwBiSKUwm88mc6ePdsapPB79LiQnJw8Bg1y5WwvucTNmpeb259u3VIshW+MaTErJdx6Vko42vwvA74cmM+yZfCtb8GIETE+pjZSVQ4dOnjawLydO8sbpfBTuPBCl8IfM2ZKgxT+hRf2sBS+McbECXurPlu+E/DE710qaM6cBuUDOqVTp06dlsIPfegHU/jp6V3JyXEp/GnTvpkLPz//25bCN8aYBGCdgLNVUQG9evH0i72BzjU74PHjX7Jr19bTvtV//HElJ0+eCC/Xo0dfsrPzGD/+xgYp/MGDs+naNYUoXZoyxhgTZdYJOFvl5XyRk8ex91wWoAVjQ9qVqnL48GenDczbubOc6upvRqu6FP5gcnLyGT36evLz3ax5hYV5lsI3xpgkZW/7Z6uignXpbpao0aM7bp75UAq/qcI3n39eG14uPb0r2dlDycv7LsXFt1NQ4ArfFBRcRPfulsI3xhjzDesEnI2aGjh4kNfIo39/GDLk7Dd5/PixwCj8bz7s9+zZ1iCF3717H3Jy8hk//gfhFP7w4XkMGZJD165uFL4VvjHGGNMc6wScja1bAaggjzlzoE+flq966NDBRoPyQqPwd4dnj0pJSaFv31wGDMhn1KhJDcrZ9uvX01L4xhjTwYKlhPPz83nxxRfp2rVri9d/8803mTt3Lmlpabz11luc24q68itWrODiiy+moAOL0MT9R4iITAb+HUgFfq2qT0Zr31pegQAncvOb/FmgS+HvbrLwTV1dTXi59PRzyc7OIy9vLN/73q2NUvgZZGRYCt8YY2IhWEp49uzZPPvssyxYsKBF6546dYqSkhLuu+8+br311lbve8WKFRQXF1snIBIRSQWWANcBe4GNIrJSVf8ajf3vfb2CPpxD9hV9Ud1CWVnDn9vt3r2NkyePh5fPyrqAnJx8xo2bEZ4Lv7Aw31L4xhgTB8aNGxcu2vPyyy+zePFiTp48yZgxY3jmmWdITU3lvPPOY8GCBaxevZqpU6dSWlrK6tWref311ykpKWHRokWUlpZy4sQJbrjhBh555BEAXnrpJZ566ilEhMLCQubNm8fKlSt54403eOyxx3j11VcZ0h7XnBuJ604AMBqoUtUdACKyDJgORKUTsPUPyzmM8uLL5/PCb1wKX0T8KPw8Lr30OvLy8hk2zKXw+/fvZSl8Y4xpi3vvhXYuJUxREbSwMFF9fT2rVq1i8uTJlJeXs3z5ctavX09aWhrz58+npKSEm2++mS+++IJhw4bx6KOPAlBVVUVxcTEzZ86krKyMyspKNmzYgKoybdo01q5dS69evXj88cdZv349vXv3pra2lp49ezJt2rTwuh0l3j+O+gMfB+7vBcY0XkhE7gDuABgwYEC77fzTzAuoSuvK38+4iWHD3Ld6S+EbY0ziCJUSBpcJuP3221m6dCmbNm3iO75i3LFjx+jjB4WlpqYyY8aMJrdVVlZGWVkZI0eOBODo0aNUVlayZcsWZs6cSe/ebr6Znj17dnRYYfHeCWgqeX5aMQRVXQosBVc7oL12/nf7XQ2Cue21QWOMMU2LcSnhIFXllltu4Yknnjht+YyMDFJTU5vclqrywAMPcOeddzZoX7x4ccQywx0t3r+r7gVyAvezgU9idCzGGGOSwIQJE3jllVc44EvH1tbWnrGUMMCkSZN47rnnOHr0KAD79u3jwIEDTJgwgdLSUmpqasLbA8jMzOTIkSMdFIUT752AjcBFIpIrIucAs4CVMT4mY4wxCaygoIDHHnuMiRMnUlhYyHXXXUd1dfUZ15s4cSI//OEPGTt2LMOHD2fmzJkcOXKESy65hIULFzJ+/HhGjBgR/vXBrFmzWLRoESNHjmT79u0dEkvclxIWkSnAz3E/EXxOVR9vbvl2LyVsjDGmQ1gp4dZLulLCqvpH4I+xPg5jjDEm3sT75QBjjDHGtJF1AowxxpgkZZ0AY4wxnVa8j1uLprY8V9YJMMYY0yllZGRQU1NjHYEWUFVqamrIyMho1XpxPzDQGGNMYsrOzmbv3r0cPHgw1ocSFzIyMsjOzm7VOtYJMMYY0ymlpaWRm5sb68NIaHY5wBhjjElS1gkwxhhjkpR1AowxxpgkFffTBreWiBwEzlzpoeV6A5+14/Y6m0SPDxI/Rosv/iV6jIkeH8Q2xoGqekFTDyRdJ6C9ici7keZkTgSJHh8kfowWX/xL9BgTPT7ovDHa5QBjjDEmSVknwBhjjElS1gk4e0tjfQAdLNHjg8SP0eKLf4keY6LHB500RhsTYIwxxiQpywQYY4wxSco6AW0kIpNFZKuIVInI/bE+nuaISI6IrBGRchH5SER+7NsfFpF9IvKe/zclsM4DPratIjIp0N5k3CKSKyLviEiliCwXkXOiGyWIyC4R+cDH8q5v6ykir/njek1Eevh2EZHFPo73ReTSwHZu8ctXisgtgfbL/Par/LoSxdiGBs7TeyLyuYjcG+/nUESeE5EDIvJhoK3Dz1mkfUQpvkUiUuFj+L2IdPftg0TkWOBcPtvWOJp7rqIQX4e/JkUk3d+v8o8P6oj4molxeSC+XSLynm+Pu3OIqtq/Vv4DUoHtwGDgHGALUBDr42rmePsBl/rbmcA2oAB4GLivieULfEzpQK6PNbW5uIFSYJa//SwwLwZx7gJ6N2r7GXC/v30/8FN/ewqwChDgcuAd394T2OH/9vC3e/jHNgBj/TqrgOtj+Pr7FBgY7+cQuBK4FPgwmucs0j6iFN9EoIu//dNAfIOCyzXaTqviiPRcRSm+Dn9NAvOBZ/3tWcDyaL5GGz3+NPBQvJ5DywS0zWigSlV3qOpJYBkwPcbHFJGqVqvqZn/7CFAO9G9mlenAMlU9oao7gSpczE3G7Xu01wCv+PVfBP6mY6Jptem444GGxzUdeEmdt4HuItIPmAS8pqq1qnoIeA2Y7B87X1XfUvc/9CViF+MEYLuqNjfpVVycQ1VdC9Q2cewdfc4i7aNdNRWfqpapar2/+zbQbNm3NsYR6blqVxHOXyTt+ZoMxv0KMCH0zbq9NRej3+ffAv/V3DY68zm0TkDb9Ac+DtzfS/Mfqp2GT5uNBN7xTXf5VNNzgZRopPgitfcCDgfe2GL1fChQJiKbROQO39ZXVavBdYaAPr69tTH297cbt8fCLBq+6STSOYTonLNI+4i223Df9kJyReT/ROQNERnn29oSR6zfozr6NRlexz9e55ePtnHAflWtDLTF1Tm0TkDbNNXj7PQ/sxCR84BXgXtV9XPgF8AQoAioxqW1IHJ8rW2Ptu+q6qXA9cA/iMiVzSwblzH6a6LTgN/5pkQ7h81JqJhEZCFQD5T4pmpggKqOBBYAvxWR82lbHLGMPRqvyc5ybm+iYYc87s6hdQLaZi+QE7ifDXwSo2NpERFJw3UASlT1vwFUdb+qnlLVr4Ff4dJyEDm+SO2f4VJVXRq1R5WqfuL/HgB+j4tnfyiF5v8e8Iu3Nsa9NEzbxuqcXw9sVtX9kHjn0IvGOYu0j6gQN3ixGJjt08P4NHmNv70Jd538YtoWR8zeo6L0mgyv4x/PouWXJdqF3+/3geWhtng8h9YJaJuNwEV+5Oo5uPTsyhgfU0T+utV/AuWq+q+B9uD1pRuA0OjXlcAsPwI3F7gIN6ilybj9m9gaYKZf/xbgDx0ZU2Mi0k1EMkO3cYOvPvSxhEaLB49rJXCzH4F7OVDnU3GrgYki0sOnMScCq/1jR0Tkcv983kyUY/QafPNIpHMYEI1zFmkfHU5EJgP/DExT1S8D7ReISKq/PRh3zna0MY5Iz1WHi9JrMhj3TODPoc5UFF0LVKhqOM0fl+ewJaMH7V+TIz2n4EbZbwcWxvp4znCsV+DSSO8D7/l/U4DfAB/49pVAv8A6C31sWwmMgo8UN25k7wbcYJ/fAelRjnEwblTxFuCj0LHhrhP+L1Dp//b07QIs8XF8AIwKbOs2H0cVcGugfRTuDW078B/4ybaiGGNXoAbICrTF9TnEdWiqga9w33xuj8Y5i7SPKMVXhbvWG/q/GBrlPsO/drcAm4GpbY2juecqCvF1+GsSyPD3q/zjg6P5GvXtLwBzGy0bd+fQZgw0xhhjkpRdDjDGGGOSlHUCjDHGmCRlnQBjjDEmSVknwBhjjElS1gkwxhhjkpR1AoyJcyKiIvJ04P59IvJwO237BRGZeeYlW7y9B9trW2fYT5EEqte1Yr2/iMiojjgmYzoj6wQYE/9OAN8Xkd6xPpCg0KQpjbS6ExBhO2dShPvtuTGmGdYJMCb+1QNLgX9s/EDjb/IictT/vcoXOCkVkW0i8qSIzBaRDeJqng8JbOZaEXnTL1fs108VkUUislFcoZg7A9tdIyK/xU1wEjyWJ4FzxdVZL/FtK8QVfPpIvin6hIgcFZFHReQdYKyITBGRChFZJ67G+v/45bqJK1KzUVzRlul+1rlHgRv9vm5sajm//rkisszHsBw496zPhjFxpMuZFzHGxIElwPsi8rNWrDMCyMfNub4D+LWqjhaRHwN3A/f65QYB43FFYdaIyLdx057Wqep3RCQdWC8iZX750cAwdeViw1T1fhG5S1WLAs23qWqtiJwLbBSRV9XNvd4NV5f9IRHJwM2mdqWq7hSRYMGWhbgpY28Tke642eNeBx7CzbB2F4CI/Evj5UTkdeBO4EtVLRSRQtwsb8YkDcsEGJMA1FWFfAm4pxWrbVTValU9gZuaNPQh/gHugz+kVFW/VlcudQeQh5uf/2YReQ9XlroXbp50gA2NOwDNuEdEtgBv44qlhLZxClfwCr+/HYFtBjsBE4H7/XH8BTed7IAm9hNpuSuBlwFU9X3cVLfGJA3LBBiTOH6O+yb7fKCtHt/Z94VLzgk8diJw++vA/a9p+N7QeG7xUJnXu1V1dfABEbkK+KIlB+uXvRYYq6pfishfcB/OAMdV9VRo0eY2A8xQ1a2Ntj2mhcuF4jEmKVkmwJgEoaq1QCmuiEvILuAyf3s6kNaGTf9ARFL8OIHBuOIvq4F54kpUIyIXi6veeCZfhdbBlX895DsAecDlEdapAAaLyCB//8bAY6uBu30HBxEZ6duPAJktWG4tMNu3DQMKWxCDMQnDOgHGJJangeCvBH4FjBeRDcAYWvgtvZGtwBvAKlzVtOPAr4G/AptF5EPgl7Qss7gUN3ahBPgT0EVE3gd+grskcBpVPQbMB/4kIuuA/UCdf/gnuI7N+/44fuLb1wAFoYGBzSz3C+A8fwz/hBtTYEzSsCqCxphOT0TOU9Wj/pv8EqBSVf8t1sdlTLyzTIAxJh78yA/q+wh3GeGXMT4eYxKCZQKMMcaYJGWZAGOMMSZJWSfAGGOMSVLWCTDGGGOSlHUCjDHGmCRlnQBjjDEmSVknwBhjjElS/w9fVoaKFJFhqgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 576x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# with ideal Qini curve (red line)\n",
    "# perfect=True\n",
    "\n",
    "plot_qini_curve(y_val, uplift_tm, trmnt_val, perfect=True);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}