{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"import warnings\n",
"warnings.simplefilter('ignore')\n",
"\n",
"import numpy as np\n",
"import pandas as pd\n",
"import seaborn as sns\n",
"import matplotlib.pyplot as plt\n",
"from sklearn.metrics import auc, roc_curve, classification_report\n",
"\n",
"import h2o\n",
"from h2o.frame import H2OFrame\n",
"from h2o.estimators.random_forest import H2ORandomForestEstimator\n",
"\n",
"%matplotlib inline"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Load Dataset"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"
\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" user_id | \n",
" signup_time | \n",
" purchase_time | \n",
" purchase_value | \n",
" device_id | \n",
" source | \n",
" browser | \n",
" sex | \n",
" age | \n",
" ip_address | \n",
" class | \n",
"
\n",
" \n",
" \n",
" \n",
" 0 | \n",
" 22058 | \n",
" 2015-02-24 22:55:49 | \n",
" 2015-04-18 02:47:11 | \n",
" 34 | \n",
" QVPSPJUOCKZAR | \n",
" SEO | \n",
" Chrome | \n",
" M | \n",
" 39 | \n",
" 7.327584e+08 | \n",
" 0 | \n",
"
\n",
" \n",
" 1 | \n",
" 333320 | \n",
" 2015-06-07 20:39:50 | \n",
" 2015-06-08 01:38:54 | \n",
" 16 | \n",
" EOGFQPIZPYXFZ | \n",
" Ads | \n",
" Chrome | \n",
" F | \n",
" 53 | \n",
" 3.503114e+08 | \n",
" 0 | \n",
"
\n",
" \n",
" 2 | \n",
" 1359 | \n",
" 2015-01-01 18:52:44 | \n",
" 2015-01-01 18:52:45 | \n",
" 15 | \n",
" YSSKYOSJHPPLJ | \n",
" SEO | \n",
" Opera | \n",
" M | \n",
" 53 | \n",
" 2.621474e+09 | \n",
" 1 | \n",
"
\n",
" \n",
" 3 | \n",
" 150084 | \n",
" 2015-04-28 21:13:25 | \n",
" 2015-05-04 13:54:50 | \n",
" 44 | \n",
" ATGTXKYKUDUQN | \n",
" SEO | \n",
" Safari | \n",
" M | \n",
" 41 | \n",
" 3.840542e+09 | \n",
" 0 | \n",
"
\n",
" \n",
" 4 | \n",
" 221365 | \n",
" 2015-07-21 07:09:52 | \n",
" 2015-09-09 18:40:53 | \n",
" 39 | \n",
" NAUITBZFJKHWW | \n",
" Ads | \n",
" Safari | \n",
" M | \n",
" 45 | \n",
" 4.155831e+08 | \n",
" 0 | \n",
"
\n",
" \n",
"
\n",
"
"
],
"text/plain": [
" user_id signup_time purchase_time purchase_value \\\n",
"0 22058 2015-02-24 22:55:49 2015-04-18 02:47:11 34 \n",
"1 333320 2015-06-07 20:39:50 2015-06-08 01:38:54 16 \n",
"2 1359 2015-01-01 18:52:44 2015-01-01 18:52:45 15 \n",
"3 150084 2015-04-28 21:13:25 2015-05-04 13:54:50 44 \n",
"4 221365 2015-07-21 07:09:52 2015-09-09 18:40:53 39 \n",
"\n",
" device_id source browser sex age ip_address class \n",
"0 QVPSPJUOCKZAR SEO Chrome M 39 7.327584e+08 0 \n",
"1 EOGFQPIZPYXFZ Ads Chrome F 53 3.503114e+08 0 \n",
"2 YSSKYOSJHPPLJ SEO Opera M 53 2.621474e+09 1 \n",
"3 ATGTXKYKUDUQN SEO Safari M 41 3.840542e+09 0 \n",
"4 NAUITBZFJKHWW Ads Safari M 45 4.155831e+08 0 "
]
},
"execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"data = pd.read_csv('./data/Fraud/Fraud_Data.csv', parse_dates=['signup_time', 'purchase_time'])\n",
"data.head()"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" lower_bound_ip_address | \n",
" upper_bound_ip_address | \n",
" country | \n",
"
\n",
" \n",
" \n",
" \n",
" 0 | \n",
" 16777216.0 | \n",
" 16777471 | \n",
" Australia | \n",
"
\n",
" \n",
" 1 | \n",
" 16777472.0 | \n",
" 16777727 | \n",
" China | \n",
"
\n",
" \n",
" 2 | \n",
" 16777728.0 | \n",
" 16778239 | \n",
" China | \n",
"
\n",
" \n",
" 3 | \n",
" 16778240.0 | \n",
" 16779263 | \n",
" Australia | \n",
"
\n",
" \n",
" 4 | \n",
" 16779264.0 | \n",
" 16781311 | \n",
" China | \n",
"
\n",
" \n",
"
\n",
"
"
],
"text/plain": [
" lower_bound_ip_address upper_bound_ip_address country\n",
"0 16777216.0 16777471 Australia\n",
"1 16777472.0 16777727 China\n",
"2 16777728.0 16778239 China\n",
"3 16778240.0 16779263 Australia\n",
"4 16779264.0 16781311 China"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"address2country = pd.read_csv('./data/Fraud/IpAddress_to_Country.csv')\n",
"address2country.head()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Add Country to Fraud Data"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" user_id | \n",
" signup_time | \n",
" purchase_time | \n",
" purchase_value | \n",
" device_id | \n",
" source | \n",
" browser | \n",
" sex | \n",
" age | \n",
" ip_address | \n",
" class | \n",
" country | \n",
"
\n",
" \n",
" \n",
" \n",
" 0 | \n",
" 22058 | \n",
" 2015-02-24 22:55:49 | \n",
" 2015-04-18 02:47:11 | \n",
" 34 | \n",
" QVPSPJUOCKZAR | \n",
" SEO | \n",
" Chrome | \n",
" M | \n",
" 39 | \n",
" 7.327584e+08 | \n",
" 0 | \n",
" Japan | \n",
"
\n",
" \n",
" 1 | \n",
" 333320 | \n",
" 2015-06-07 20:39:50 | \n",
" 2015-06-08 01:38:54 | \n",
" 16 | \n",
" EOGFQPIZPYXFZ | \n",
" Ads | \n",
" Chrome | \n",
" F | \n",
" 53 | \n",
" 3.503114e+08 | \n",
" 0 | \n",
" United States | \n",
"
\n",
" \n",
" 2 | \n",
" 1359 | \n",
" 2015-01-01 18:52:44 | \n",
" 2015-01-01 18:52:45 | \n",
" 15 | \n",
" YSSKYOSJHPPLJ | \n",
" SEO | \n",
" Opera | \n",
" M | \n",
" 53 | \n",
" 2.621474e+09 | \n",
" 1 | \n",
" United States | \n",
"
\n",
" \n",
" 3 | \n",
" 150084 | \n",
" 2015-04-28 21:13:25 | \n",
" 2015-05-04 13:54:50 | \n",
" 44 | \n",
" ATGTXKYKUDUQN | \n",
" SEO | \n",
" Safari | \n",
" M | \n",
" 41 | \n",
" 3.840542e+09 | \n",
" 0 | \n",
" NA | \n",
"
\n",
" \n",
" 4 | \n",
" 221365 | \n",
" 2015-07-21 07:09:52 | \n",
" 2015-09-09 18:40:53 | \n",
" 39 | \n",
" NAUITBZFJKHWW | \n",
" Ads | \n",
" Safari | \n",
" M | \n",
" 45 | \n",
" 4.155831e+08 | \n",
" 0 | \n",
" United States | \n",
"
\n",
" \n",
"
\n",
"
"
],
"text/plain": [
" user_id signup_time purchase_time purchase_value \\\n",
"0 22058 2015-02-24 22:55:49 2015-04-18 02:47:11 34 \n",
"1 333320 2015-06-07 20:39:50 2015-06-08 01:38:54 16 \n",
"2 1359 2015-01-01 18:52:44 2015-01-01 18:52:45 15 \n",
"3 150084 2015-04-28 21:13:25 2015-05-04 13:54:50 44 \n",
"4 221365 2015-07-21 07:09:52 2015-09-09 18:40:53 39 \n",
"\n",
" device_id source browser sex age ip_address class country \n",
"0 QVPSPJUOCKZAR SEO Chrome M 39 7.327584e+08 0 Japan \n",
"1 EOGFQPIZPYXFZ Ads Chrome F 53 3.503114e+08 0 United States \n",
"2 YSSKYOSJHPPLJ SEO Opera M 53 2.621474e+09 1 United States \n",
"3 ATGTXKYKUDUQN SEO Safari M 41 3.840542e+09 0 NA \n",
"4 NAUITBZFJKHWW Ads Safari M 45 4.155831e+08 0 United States "
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"countries = []\n",
"for i in range(len(data)):\n",
" ip_address = data.loc[i, 'ip_address']\n",
" tmp = address2country[(address2country['lower_bound_ip_address'] <= ip_address) &\n",
" (address2country['upper_bound_ip_address'] >= ip_address)]\n",
" if len(tmp) == 1:\n",
" countries.append(tmp['country'].values[0])\n",
" else:\n",
" countries.append('NA')\n",
" \n",
"data['country'] = countries\n",
"\n",
"data.head()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Feature Engineering"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* Time difference between sign-up time and purchase time\n",
"* If the device id is unique or certain users are sharing the same device (many different user ids using the same device could be an indicator of fake accounts)\n",
"* Same for the ip address. Many different users having the same ip address could be an indicator of fake accounts\n",
"* Usual week of the year and day of the week from time variables"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"# Get the time difference between purchase time and signup time\n",
"time_diff = data['purchase_time'] - data['signup_time']\n",
"time_diff = time_diff.apply(lambda x: x.seconds)\n",
"data['time_diff'] = time_diff"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"# Check user number for unique devices\n",
"device_num = data[['user_id', 'device_id']].groupby('device_id').count().reset_index()\n",
"device_num = device_num.rename(columns={'user_id': 'device_num'})\n",
"data = data.merge(device_num, how='left', on='device_id')"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"# Check user number for unique ip_address\n",
"ip_num = data[['user_id', 'ip_address']].groupby('ip_address').count().reset_index()\n",
"ip_num = ip_num.rename(columns={'user_id': 'ip_num'})\n",
"data = data.merge(ip_num, how='left', on='ip_address')"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {
"collapsed": true,
"scrolled": false
},
"outputs": [],
"source": [
"# Signup day and week\n",
"data['signup_day'] = data['signup_time'].apply(lambda x: x.dayofweek)\n",
"data['signup_week'] = data['signup_time'].apply(lambda x: x.week)\n",
"\n",
"# Purchase day and week\n",
"data['purchase_day'] = data['purchase_time'].apply(lambda x: x.dayofweek)\n",
"data['purchase_week'] = data['purchase_time'].apply(lambda x: x.week)"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" user_id | \n",
" signup_time | \n",
" purchase_time | \n",
" purchase_value | \n",
" device_id | \n",
" source | \n",
" browser | \n",
" sex | \n",
" age | \n",
" ip_address | \n",
" class | \n",
" country | \n",
" time_diff | \n",
" device_num | \n",
" ip_num | \n",
" signup_day | \n",
" signup_week | \n",
" purchase_day | \n",
" purchase_week | \n",
"
\n",
" \n",
" \n",
" \n",
" 0 | \n",
" 22058 | \n",
" 2015-02-24 22:55:49 | \n",
" 2015-04-18 02:47:11 | \n",
" 34 | \n",
" QVPSPJUOCKZAR | \n",
" SEO | \n",
" Chrome | \n",
" M | \n",
" 39 | \n",
" 7.327584e+08 | \n",
" 0 | \n",
" Japan | \n",
" 13882 | \n",
" 1 | \n",
" 1 | \n",
" 1 | \n",
" 9 | \n",
" 5 | \n",
" 16 | \n",
"
\n",
" \n",
" 1 | \n",
" 333320 | \n",
" 2015-06-07 20:39:50 | \n",
" 2015-06-08 01:38:54 | \n",
" 16 | \n",
" EOGFQPIZPYXFZ | \n",
" Ads | \n",
" Chrome | \n",
" F | \n",
" 53 | \n",
" 3.503114e+08 | \n",
" 0 | \n",
" United States | \n",
" 17944 | \n",
" 1 | \n",
" 1 | \n",
" 6 | \n",
" 23 | \n",
" 0 | \n",
" 24 | \n",
"
\n",
" \n",
" 2 | \n",
" 1359 | \n",
" 2015-01-01 18:52:44 | \n",
" 2015-01-01 18:52:45 | \n",
" 15 | \n",
" YSSKYOSJHPPLJ | \n",
" SEO | \n",
" Opera | \n",
" M | \n",
" 53 | \n",
" 2.621474e+09 | \n",
" 1 | \n",
" United States | \n",
" 1 | \n",
" 12 | \n",
" 12 | \n",
" 3 | \n",
" 1 | \n",
" 3 | \n",
" 1 | \n",
"
\n",
" \n",
" 3 | \n",
" 150084 | \n",
" 2015-04-28 21:13:25 | \n",
" 2015-05-04 13:54:50 | \n",
" 44 | \n",
" ATGTXKYKUDUQN | \n",
" SEO | \n",
" Safari | \n",
" M | \n",
" 41 | \n",
" 3.840542e+09 | \n",
" 0 | \n",
" NA | \n",
" 60085 | \n",
" 1 | \n",
" 1 | \n",
" 1 | \n",
" 18 | \n",
" 0 | \n",
" 19 | \n",
"
\n",
" \n",
" 4 | \n",
" 221365 | \n",
" 2015-07-21 07:09:52 | \n",
" 2015-09-09 18:40:53 | \n",
" 39 | \n",
" NAUITBZFJKHWW | \n",
" Ads | \n",
" Safari | \n",
" M | \n",
" 45 | \n",
" 4.155831e+08 | \n",
" 0 | \n",
" United States | \n",
" 41461 | \n",
" 1 | \n",
" 1 | \n",
" 1 | \n",
" 30 | \n",
" 2 | \n",
" 37 | \n",
"
\n",
" \n",
"
\n",
"
"
],
"text/plain": [
" user_id signup_time purchase_time purchase_value \\\n",
"0 22058 2015-02-24 22:55:49 2015-04-18 02:47:11 34 \n",
"1 333320 2015-06-07 20:39:50 2015-06-08 01:38:54 16 \n",
"2 1359 2015-01-01 18:52:44 2015-01-01 18:52:45 15 \n",
"3 150084 2015-04-28 21:13:25 2015-05-04 13:54:50 44 \n",
"4 221365 2015-07-21 07:09:52 2015-09-09 18:40:53 39 \n",
"\n",
" device_id source browser sex age ip_address class country \\\n",
"0 QVPSPJUOCKZAR SEO Chrome M 39 7.327584e+08 0 Japan \n",
"1 EOGFQPIZPYXFZ Ads Chrome F 53 3.503114e+08 0 United States \n",
"2 YSSKYOSJHPPLJ SEO Opera M 53 2.621474e+09 1 United States \n",
"3 ATGTXKYKUDUQN SEO Safari M 41 3.840542e+09 0 NA \n",
"4 NAUITBZFJKHWW Ads Safari M 45 4.155831e+08 0 United States \n",
"\n",
" time_diff device_num ip_num signup_day signup_week purchase_day \\\n",
"0 13882 1 1 1 9 5 \n",
"1 17944 1 1 6 23 0 \n",
"2 1 12 12 3 1 3 \n",
"3 60085 1 1 1 18 0 \n",
"4 41461 1 1 1 30 2 \n",
"\n",
" purchase_week \n",
"0 16 \n",
"1 24 \n",
"2 1 \n",
"3 19 \n",
"4 37 "
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"data.head()"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" signup_day | \n",
" signup_week | \n",
" purchase_day | \n",
" purchase_week | \n",
" purchase_value | \n",
" source | \n",
" browser | \n",
" sex | \n",
" age | \n",
" country | \n",
" time_diff | \n",
" device_num | \n",
" ip_num | \n",
" class | \n",
"
\n",
" \n",
" \n",
" \n",
" 0 | \n",
" 1 | \n",
" 9 | \n",
" 5 | \n",
" 16 | \n",
" 34 | \n",
" SEO | \n",
" Chrome | \n",
" M | \n",
" 39 | \n",
" Japan | \n",
" 13882 | \n",
" 1 | \n",
" 1 | \n",
" 0 | \n",
"
\n",
" \n",
" 1 | \n",
" 6 | \n",
" 23 | \n",
" 0 | \n",
" 24 | \n",
" 16 | \n",
" Ads | \n",
" Chrome | \n",
" F | \n",
" 53 | \n",
" United States | \n",
" 17944 | \n",
" 1 | \n",
" 1 | \n",
" 0 | \n",
"
\n",
" \n",
" 2 | \n",
" 3 | \n",
" 1 | \n",
" 3 | \n",
" 1 | \n",
" 15 | \n",
" SEO | \n",
" Opera | \n",
" M | \n",
" 53 | \n",
" United States | \n",
" 1 | \n",
" 12 | \n",
" 12 | \n",
" 1 | \n",
"
\n",
" \n",
" 3 | \n",
" 1 | \n",
" 18 | \n",
" 0 | \n",
" 19 | \n",
" 44 | \n",
" SEO | \n",
" Safari | \n",
" M | \n",
" 41 | \n",
" NA | \n",
" 60085 | \n",
" 1 | \n",
" 1 | \n",
" 0 | \n",
"
\n",
" \n",
" 4 | \n",
" 1 | \n",
" 30 | \n",
" 2 | \n",
" 37 | \n",
" 39 | \n",
" Ads | \n",
" Safari | \n",
" M | \n",
" 45 | \n",
" United States | \n",
" 41461 | \n",
" 1 | \n",
" 1 | \n",
" 0 | \n",
"
\n",
" \n",
"
\n",
"
"
],
"text/plain": [
" signup_day signup_week purchase_day purchase_week purchase_value \\\n",
"0 1 9 5 16 34 \n",
"1 6 23 0 24 16 \n",
"2 3 1 3 1 15 \n",
"3 1 18 0 19 44 \n",
"4 1 30 2 37 39 \n",
"\n",
" source browser sex age country time_diff device_num ip_num class \n",
"0 SEO Chrome M 39 Japan 13882 1 1 0 \n",
"1 Ads Chrome F 53 United States 17944 1 1 0 \n",
"2 SEO Opera M 53 United States 1 12 12 1 \n",
"3 SEO Safari M 41 NA 60085 1 1 0 \n",
"4 Ads Safari M 45 United States 41461 1 1 0 "
]
},
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# Define features and target to be used\n",
"columns = ['signup_day', 'signup_week', 'purchase_day', 'purchase_week', 'purchase_value', 'source', \n",
" 'browser', 'sex', 'age', 'country', 'time_diff', 'device_num', 'ip_num', 'class']\n",
"data = data[columns]\n",
"data.head()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Fraudulent Activity Identification"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Checking whether there is an H2O instance running at http://localhost:54321..... not found.\n",
"Attempting to start a local H2O server...\n",
" Java Version: openjdk version \"1.8.0_121\"; OpenJDK Runtime Environment (Zulu 8.20.0.5-macosx) (build 1.8.0_121-b15); OpenJDK 64-Bit Server VM (Zulu 8.20.0.5-macosx) (build 25.121-b15, mixed mode)\n",
" Starting server from /anaconda/lib/python3.5/site-packages/h2o/backend/bin/h2o.jar\n",
" Ice root: /var/folders/gx/b15jqbt1567grsfr7l_znn2h0000gn/T/tmpq14c42t4\n",
" JVM stdout: /var/folders/gx/b15jqbt1567grsfr7l_znn2h0000gn/T/tmpq14c42t4/h2o_jifu_started_from_python.out\n",
" JVM stderr: /var/folders/gx/b15jqbt1567grsfr7l_znn2h0000gn/T/tmpq14c42t4/h2o_jifu_started_from_python.err\n",
" Server is running at http://127.0.0.1:54321\n",
"Connecting to H2O server at http://127.0.0.1:54321... successful.\n"
]
},
{
"data": {
"text/html": [
"H2O cluster uptime: | \n",
"03 secs |
\n",
"H2O cluster timezone: | \n",
"America/Chicago |
\n",
"H2O data parsing timezone: | \n",
"UTC |
\n",
"H2O cluster version: | \n",
"3.18.0.2 |
\n",
"H2O cluster version age: | \n",
"3 months |
\n",
"H2O cluster name: | \n",
"H2O_from_python_jifu_3z98p9 |
\n",
"H2O cluster total nodes: | \n",
"1 |
\n",
"H2O cluster free memory: | \n",
"3.556 Gb |
\n",
"H2O cluster total cores: | \n",
"8 |
\n",
"H2O cluster allowed cores: | \n",
"8 |
\n",
"H2O cluster status: | \n",
"accepting new members, healthy |
\n",
"H2O connection url: | \n",
"http://127.0.0.1:54321 |
\n",
"H2O connection proxy: | \n",
"None |
\n",
"H2O internal security: | \n",
"False |
\n",
"H2O API Extensions: | \n",
"XGBoost, Algos, AutoML, Core V3, Core V4 |
\n",
"Python version: | \n",
"3.5.4 final |
"
],
"text/plain": [
"-------------------------- ----------------------------------------\n",
"H2O cluster uptime: 03 secs\n",
"H2O cluster timezone: America/Chicago\n",
"H2O data parsing timezone: UTC\n",
"H2O cluster version: 3.18.0.2\n",
"H2O cluster version age: 3 months\n",
"H2O cluster name: H2O_from_python_jifu_3z98p9\n",
"H2O cluster total nodes: 1\n",
"H2O cluster free memory: 3.556 Gb\n",
"H2O cluster total cores: 8\n",
"H2O cluster allowed cores: 8\n",
"H2O cluster status: accepting new members, healthy\n",
"H2O connection url: http://127.0.0.1:54321\n",
"H2O connection proxy:\n",
"H2O internal security: False\n",
"H2O API Extensions: XGBoost, Algos, AutoML, Core V3, Core V4\n",
"Python version: 3.5.4 final\n",
"-------------------------- ----------------------------------------"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# Initialize H2O cluster\n",
"h2o.init()\n",
"h2o.remove_all()"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Parse progress: |█████████████████████████████████████████████████████████| 100%\n"
]
},
{
"data": {
"text/html": [
"\n",
" | signup_day | signup_week | purchase_day | purchase_week | purchase_value | source | browser | sex | age | country | time_diff | device_num | ip_num | class |
\n",
"type | enum | int | enum | int | int | enum | enum | enum | int | enum | int | int | int | enum |
\n",
"mins | | 1.0 | | 1.0 | 9.0 | | | | 18.0 | | 1.0 | 1.0 | 1.0 | |
\n",
"mean | | 16.501740430938508 | | 24.658571126052173 | 36.93537243898567 | | | | 33.14070358409671 | | 40942.584427444956 | 1.6843665625496433 | 1.6027185134205137 | |
\n",
"maxs | | 34.0 | | 51.0 | 154.0 | | | | 76.0 | | 86399.0 | 20.0 | 20.0 | |
\n",
"sigma | | 9.814287461798854 | | 11.651556782719474 | 18.32276214866212 | | | | 8.617733490961495 | | 26049.661902118358 | 2.6169536028041693 | 2.596239527375835 | |
\n",
"zeros | | 0 | | 0 | 0 | | | | 0 | | 0 | 0 | 0 | |
\n",
"missing | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 |
\n",
"0 | 1 | 9.0 | 5 | 16.0 | 34.0 | SEO | Chrome | M | 39.0 | Japan | 13882.0 | 1.0 | 1.0 | 0 |
\n",
"1 | 6 | 23.0 | 0 | 24.0 | 16.0 | Ads | Chrome | F | 53.0 | United States | 17944.0 | 1.0 | 1.0 | 0 |
\n",
"2 | 3 | 1.0 | 3 | 1.0 | 15.0 | SEO | Opera | M | 53.0 | United States | 1.0 | 12.0 | 12.0 | 1 |
\n",
"3 | 1 | 18.0 | 0 | 19.0 | 44.0 | SEO | Safari | M | 41.0 | NA | 60085.0 | 1.0 | 1.0 | 0 |
\n",
"4 | 1 | 30.0 | 2 | 37.0 | 39.0 | Ads | Safari | M | 45.0 | United States | 41461.0 | 1.0 | 1.0 | 0 |
\n",
"5 | 3 | 21.0 | 3 | 28.0 | 42.0 | Ads | Chrome | M | 18.0 | Canada | 7331.0 | 1.0 | 1.0 | 0 |
\n",
"6 | 5 | 31.0 | 3 | 35.0 | 11.0 | Ads | Chrome | F | 19.0 | NA | 17825.0 | 1.0 | 1.0 | 0 |
\n",
"7 | 0 | 15.0 | 0 | 22.0 | 27.0 | Ads | Opera | M | 34.0 | United States | 35129.0 | 1.0 | 1.0 | 0 |
\n",
"8 | 1 | 17.0 | 1 | 23.0 | 30.0 | SEO | IE | F | 43.0 | China | 51800.0 | 1.0 | 1.0 | 0 |
\n",
"9 | 6 | 4.0 | 0 | 13.0 | 62.0 | Ads | IE | M | 31.0 | United States | 18953.0 | 1.0 | 1.0 | 0 |
\n",
"
"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# Transform to H2O Frame, and make sure the target variable is categorical\n",
"h2o_df = H2OFrame(data)\n",
"\n",
"for name in ['signup_day', 'purchase_day', 'source', 'browser', 'sex', 'country', 'class']:\n",
" h2o_df[name] = h2o_df[name].asfactor()\n",
"\n",
"h2o_df.summary()"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"# Split into 70% training and 30% test dataset\n",
"strat_split = h2o_df['class'].stratified_split(test_frac=0.3, seed=42)\n",
"\n",
"train = h2o_df[strat_split == 'train']\n",
"test = h2o_df[strat_split == 'test']\n",
"\n",
"# Define features and target\n",
"feature = ['signup_day', 'signup_week', 'purchase_day', 'purchase_week', 'purchase_value', \n",
" 'source', 'browser', 'sex', 'age', 'country', 'time_diff', 'device_num', 'ip_num']\n",
"target = 'class'"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"drf Model Build progress: |███████████████████████████████████████████████| 100%\n"
]
}
],
"source": [
"# Build random forest model \n",
"model = H2ORandomForestEstimator(balance_classes=True, ntrees=100, mtries=-1, stopping_rounds=5, \n",
" stopping_metric='auc', score_each_iteration=True, seed=42)\n",
"model.train(x=feature, y=target, training_frame=train, validation_frame=test)"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {
"scrolled": false
},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" | \n",
" timestamp | \n",
" duration | \n",
" number_of_trees | \n",
" training_rmse | \n",
" training_logloss | \n",
" training_auc | \n",
" training_lift | \n",
" training_classification_error | \n",
" validation_rmse | \n",
" validation_logloss | \n",
" validation_auc | \n",
" validation_lift | \n",
" validation_classification_error | \n",
"
\n",
" \n",
" \n",
" \n",
" 0 | \n",
" | \n",
" 2018-05-20 20:25:44 | \n",
" 0.100 sec | \n",
" 0.0 | \n",
" NaN | \n",
" NaN | \n",
" NaN | \n",
" NaN | \n",
" NaN | \n",
" NaN | \n",
" NaN | \n",
" NaN | \n",
" NaN | \n",
" NaN | \n",
"
\n",
" \n",
" 1 | \n",
" | \n",
" 2018-05-20 20:25:45 | \n",
" 1.181 sec | \n",
" 1.0 | \n",
" 0.329237 | \n",
" 0.763404 | \n",
" 0.956487 | \n",
" 1.930543 | \n",
" 0.103585 | \n",
" 0.257295 | \n",
" 1.705777 | \n",
" 0.780964 | \n",
" 7.318324 | \n",
" 0.064986 | \n",
"
\n",
" \n",
" 2 | \n",
" | \n",
" 2018-05-20 20:25:46 | \n",
" 1.837 sec | \n",
" 2.0 | \n",
" 0.337703 | \n",
" 0.666988 | \n",
" 0.958739 | \n",
" 1.945962 | \n",
" 0.100437 | \n",
" 0.211104 | \n",
" 0.612876 | \n",
" 0.808146 | \n",
" 10.238311 | \n",
" 0.045618 | \n",
"
\n",
" \n",
" 3 | \n",
" | \n",
" 2018-05-20 20:25:47 | \n",
" 2.924 sec | \n",
" 3.0 | \n",
" 0.342474 | \n",
" 0.585470 | \n",
" 0.962420 | \n",
" 1.959389 | \n",
" 0.096241 | \n",
" 0.205624 | \n",
" 0.416203 | \n",
" 0.820663 | \n",
" 10.566838 | \n",
" 0.044361 | \n",
"
\n",
" \n",
" 4 | \n",
" | \n",
" 2018-05-20 20:25:48 | \n",
" 4.475 sec | \n",
" 4.0 | \n",
" 0.341220 | \n",
" 0.530667 | \n",
" 0.967584 | \n",
" 1.967190 | \n",
" 0.088710 | \n",
" 0.204276 | \n",
" 0.340733 | \n",
" 0.823580 | \n",
" 10.641350 | \n",
" 0.044140 | \n",
"
\n",
" \n",
" 5 | \n",
" | \n",
" 2018-05-20 20:25:49 | \n",
" 4.799 sec | \n",
" 5.0 | \n",
" 0.339991 | \n",
" 0.494876 | \n",
" 0.970585 | \n",
" 1.973005 | \n",
" 0.085154 | \n",
" 0.203786 | \n",
" 0.294007 | \n",
" 0.825970 | \n",
" 10.669638 | \n",
" 0.044162 | \n",
"
\n",
" \n",
" 6 | \n",
" | \n",
" 2018-05-20 20:25:49 | \n",
" 5.173 sec | \n",
" 6.0 | \n",
" 0.338576 | \n",
" 0.454248 | \n",
" 0.974636 | \n",
" 1.979192 | \n",
" 0.081914 | \n",
" 0.203566 | \n",
" 0.268536 | \n",
" 0.828691 | \n",
" 10.674389 | \n",
" 0.044096 | \n",
"
\n",
" \n",
" 7 | \n",
" | \n",
" 2018-05-20 20:25:49 | \n",
" 5.534 sec | \n",
" 7.0 | \n",
" 0.337354 | \n",
" 0.427348 | \n",
" 0.977835 | \n",
" 1.983172 | \n",
" 0.076219 | \n",
" 0.203257 | \n",
" 0.251817 | \n",
" 0.830573 | \n",
" 10.679152 | \n",
" 0.044096 | \n",
"
\n",
" \n",
" 8 | \n",
" | \n",
" 2018-05-20 20:25:50 | \n",
" 5.923 sec | \n",
" 8.0 | \n",
" 0.337933 | \n",
" 0.405616 | \n",
" 0.980248 | \n",
" 1.987241 | \n",
" 0.072625 | \n",
" 0.203124 | \n",
" 0.243703 | \n",
" 0.831204 | \n",
" 10.679152 | \n",
" 0.044118 | \n",
"
\n",
" \n",
" 9 | \n",
" | \n",
" 2018-05-20 20:25:50 | \n",
" 6.339 sec | \n",
" 9.0 | \n",
" 0.336345 | \n",
" 0.385657 | \n",
" 0.982926 | \n",
" 1.990128 | \n",
" 0.067415 | \n",
" 0.203066 | \n",
" 0.231357 | \n",
" 0.832550 | \n",
" 10.679152 | \n",
" 0.044118 | \n",
"
\n",
" \n",
" 10 | \n",
" | \n",
" 2018-05-20 20:25:51 | \n",
" 6.749 sec | \n",
" 10.0 | \n",
" 0.334526 | \n",
" 0.366322 | \n",
" 0.985589 | \n",
" 1.992683 | \n",
" 0.061423 | \n",
" 0.203097 | \n",
" 0.227298 | \n",
" 0.832316 | \n",
" 10.679152 | \n",
" 0.044427 | \n",
"
\n",
" \n",
" 11 | \n",
" | \n",
" 2018-05-20 20:25:51 | \n",
" 7.162 sec | \n",
" 11.0 | \n",
" 0.335901 | \n",
" 0.357655 | \n",
" 0.986884 | \n",
" 1.994899 | \n",
" 0.058755 | \n",
" 0.203096 | \n",
" 0.220205 | \n",
" 0.833527 | \n",
" 10.679152 | \n",
" 0.044228 | \n",
"
\n",
" \n",
" 12 | \n",
" | \n",
" 2018-05-20 20:25:51 | \n",
" 7.571 sec | \n",
" 12.0 | \n",
" 0.332650 | \n",
" 0.341858 | \n",
" 0.989303 | \n",
" 1.996165 | \n",
" 0.052590 | \n",
" 0.203156 | \n",
" 0.218058 | \n",
" 0.833289 | \n",
" 10.679152 | \n",
" 0.044162 | \n",
"
\n",
" \n",
" 13 | \n",
" | \n",
" 2018-05-20 20:25:52 | \n",
" 8.018 sec | \n",
" 13.0 | \n",
" 0.332947 | \n",
" 0.337632 | \n",
" 0.990264 | \n",
" 1.996937 | \n",
" 0.050032 | \n",
" 0.203230 | \n",
" 0.213603 | \n",
" 0.833152 | \n",
" 10.679152 | \n",
" 0.044118 | \n",
"
\n",
" \n",
" 14 | \n",
" | \n",
" 2018-05-20 20:25:52 | \n",
" 8.467 sec | \n",
" 14.0 | \n",
" 0.334967 | \n",
" 0.338206 | \n",
" 0.990769 | \n",
" 1.997604 | \n",
" 0.049137 | \n",
" 0.203201 | \n",
" 0.209188 | \n",
" 0.834405 | \n",
" 10.679152 | \n",
" 0.044471 | \n",
"
\n",
" \n",
" 15 | \n",
" | \n",
" 2018-05-20 20:25:53 | \n",
" 8.945 sec | \n",
" 15.0 | \n",
" 0.334957 | \n",
" 0.335644 | \n",
" 0.991332 | \n",
" 1.998106 | \n",
" 0.047514 | \n",
" 0.203163 | \n",
" 0.207753 | \n",
" 0.834603 | \n",
" 10.679152 | \n",
" 0.044361 | \n",
"
\n",
" \n",
" 16 | \n",
" | \n",
" 2018-05-20 20:25:53 | \n",
" 9.403 sec | \n",
" 16.0 | \n",
" 0.334266 | \n",
" 0.331837 | \n",
" 0.992059 | \n",
" 1.998392 | \n",
" 0.045455 | \n",
" 0.203135 | \n",
" 0.204570 | \n",
" 0.835777 | \n",
" 10.679152 | \n",
" 0.044118 | \n",
"
\n",
" \n",
" 17 | \n",
" | \n",
" 2018-05-20 20:25:54 | \n",
" 9.872 sec | \n",
" 17.0 | \n",
" 0.332295 | \n",
" 0.324349 | \n",
" 0.993148 | \n",
" 1.998720 | \n",
" 0.041518 | \n",
" 0.203177 | \n",
" 0.203516 | \n",
" 0.836050 | \n",
" 10.679152 | \n",
" 0.044228 | \n",
"
\n",
" \n",
" 18 | \n",
" | \n",
" 2018-05-20 20:25:54 | \n",
" 10.358 sec | \n",
" 18.0 | \n",
" 0.331662 | \n",
" 0.320754 | \n",
" 0.993694 | \n",
" 1.999123 | \n",
" 0.039879 | \n",
" 0.203168 | \n",
" 0.202761 | \n",
" 0.836510 | \n",
" 10.679152 | \n",
" 0.044625 | \n",
"
\n",
" \n",
" 19 | \n",
" | \n",
" 2018-05-20 20:25:55 | \n",
" 10.920 sec | \n",
" 19.0 | \n",
" 0.330554 | \n",
" 0.316733 | \n",
" 0.994235 | \n",
" 1.999381 | \n",
" 0.038212 | \n",
" 0.203183 | \n",
" 0.202061 | \n",
" 0.836516 | \n",
" 10.679152 | \n",
" 0.044493 | \n",
"
\n",
" \n",
" 20 | \n",
" | \n",
" 2018-05-20 20:25:55 | \n",
" 11.517 sec | \n",
" 20.0 | \n",
" 0.331997 | \n",
" 0.319453 | \n",
" 0.994307 | \n",
" 1.999455 | \n",
" 0.037777 | \n",
" 0.203136 | \n",
" 0.198964 | \n",
" 0.837179 | \n",
" 10.679152 | \n",
" 0.044846 | \n",
"
\n",
" \n",
" 21 | \n",
" | \n",
" 2018-05-20 20:25:56 | \n",
" 12.045 sec | \n",
" 21.0 | \n",
" 0.332174 | \n",
" 0.319514 | \n",
" 0.994558 | \n",
" 1.999454 | \n",
" 0.036471 | \n",
" 0.203140 | \n",
" 0.198079 | \n",
" 0.837936 | \n",
" 10.679152 | \n",
" 0.044272 | \n",
"
\n",
" \n",
" 22 | \n",
" | \n",
" 2018-05-20 20:25:56 | \n",
" 12.578 sec | \n",
" 22.0 | \n",
" 0.331975 | \n",
" 0.318617 | \n",
" 0.994821 | \n",
" 1.999529 | \n",
" 0.035568 | \n",
" 0.203130 | \n",
" 0.196660 | \n",
" 0.838474 | \n",
" 10.679152 | \n",
" 0.044272 | \n",
"
\n",
" \n",
" 23 | \n",
" | \n",
" 2018-05-20 20:25:57 | \n",
" 13.145 sec | \n",
" 23.0 | \n",
" 0.331260 | \n",
" 0.316874 | \n",
" 0.995103 | \n",
" 1.999529 | \n",
" 0.034353 | \n",
" 0.203161 | \n",
" 0.196128 | \n",
" 0.838535 | \n",
" 10.679152 | \n",
" 0.044383 | \n",
"
\n",
" \n",
" 24 | \n",
" | \n",
" 2018-05-20 20:25:58 | \n",
" 13.706 sec | \n",
" 24.0 | \n",
" 0.333191 | \n",
" 0.320829 | \n",
" 0.995035 | \n",
" 1.999566 | \n",
" 0.034467 | \n",
" 0.203116 | \n",
" 0.194919 | \n",
" 0.839078 | \n",
" 10.679152 | \n",
" 0.044780 | \n",
"
\n",
" \n",
" 25 | \n",
" | \n",
" 2018-05-20 20:25:58 | \n",
" 14.287 sec | \n",
" 25.0 | \n",
" 0.333100 | \n",
" 0.320371 | \n",
" 0.995231 | \n",
" 1.999566 | \n",
" 0.033742 | \n",
" 0.203085 | \n",
" 0.194157 | \n",
" 0.839023 | \n",
" 10.679152 | \n",
" 0.044493 | \n",
"
\n",
" \n",
" 26 | \n",
" | \n",
" 2018-05-20 20:25:59 | \n",
" 14.897 sec | \n",
" 26.0 | \n",
" 0.334370 | \n",
" 0.322707 | \n",
" 0.995213 | \n",
" 1.999566 | \n",
" 0.033783 | \n",
" 0.203029 | \n",
" 0.192694 | \n",
" 0.838821 | \n",
" 10.679152 | \n",
" 0.044493 | \n",
"
\n",
" \n",
" 27 | \n",
" | \n",
" 2018-05-20 20:25:59 | \n",
" 15.503 sec | \n",
" 27.0 | \n",
" 0.334643 | \n",
" 0.322935 | \n",
" 0.995278 | \n",
" 1.999566 | \n",
" 0.033309 | \n",
" 0.202999 | \n",
" 0.189930 | \n",
" 0.839291 | \n",
" 10.679152 | \n",
" 0.044603 | \n",
"
\n",
" \n",
" 28 | \n",
" | \n",
" 2018-05-20 20:26:00 | \n",
" 16.160 sec | \n",
" 28.0 | \n",
" 0.334961 | \n",
" 0.323930 | \n",
" 0.995309 | \n",
" 1.999566 | \n",
" 0.033121 | \n",
" 0.202994 | \n",
" 0.189183 | \n",
" 0.839264 | \n",
" 10.679152 | \n",
" 0.044427 | \n",
"
\n",
" \n",
" 29 | \n",
" | \n",
" 2018-05-20 20:26:01 | \n",
" 16.805 sec | \n",
" 29.0 | \n",
" 0.335004 | \n",
" 0.323465 | \n",
" 0.995424 | \n",
" 1.999566 | \n",
" 0.032500 | \n",
" 0.202957 | \n",
" 0.188449 | \n",
" 0.839421 | \n",
" 10.679152 | \n",
" 0.044449 | \n",
"
\n",
" \n",
" 30 | \n",
" | \n",
" 2018-05-20 20:26:01 | \n",
" 17.456 sec | \n",
" 30.0 | \n",
" 0.334927 | \n",
" 0.323080 | \n",
" 0.995487 | \n",
" 1.999604 | \n",
" 0.032255 | \n",
" 0.202957 | \n",
" 0.187168 | \n",
" 0.839391 | \n",
" 10.679152 | \n",
" 0.044692 | \n",
"
\n",
" \n",
"
\n",
"
"
],
"text/plain": [
" timestamp duration number_of_trees training_rmse \\\n",
"0 2018-05-20 20:25:44 0.100 sec 0.0 NaN \n",
"1 2018-05-20 20:25:45 1.181 sec 1.0 0.329237 \n",
"2 2018-05-20 20:25:46 1.837 sec 2.0 0.337703 \n",
"3 2018-05-20 20:25:47 2.924 sec 3.0 0.342474 \n",
"4 2018-05-20 20:25:48 4.475 sec 4.0 0.341220 \n",
"5 2018-05-20 20:25:49 4.799 sec 5.0 0.339991 \n",
"6 2018-05-20 20:25:49 5.173 sec 6.0 0.338576 \n",
"7 2018-05-20 20:25:49 5.534 sec 7.0 0.337354 \n",
"8 2018-05-20 20:25:50 5.923 sec 8.0 0.337933 \n",
"9 2018-05-20 20:25:50 6.339 sec 9.0 0.336345 \n",
"10 2018-05-20 20:25:51 6.749 sec 10.0 0.334526 \n",
"11 2018-05-20 20:25:51 7.162 sec 11.0 0.335901 \n",
"12 2018-05-20 20:25:51 7.571 sec 12.0 0.332650 \n",
"13 2018-05-20 20:25:52 8.018 sec 13.0 0.332947 \n",
"14 2018-05-20 20:25:52 8.467 sec 14.0 0.334967 \n",
"15 2018-05-20 20:25:53 8.945 sec 15.0 0.334957 \n",
"16 2018-05-20 20:25:53 9.403 sec 16.0 0.334266 \n",
"17 2018-05-20 20:25:54 9.872 sec 17.0 0.332295 \n",
"18 2018-05-20 20:25:54 10.358 sec 18.0 0.331662 \n",
"19 2018-05-20 20:25:55 10.920 sec 19.0 0.330554 \n",
"20 2018-05-20 20:25:55 11.517 sec 20.0 0.331997 \n",
"21 2018-05-20 20:25:56 12.045 sec 21.0 0.332174 \n",
"22 2018-05-20 20:25:56 12.578 sec 22.0 0.331975 \n",
"23 2018-05-20 20:25:57 13.145 sec 23.0 0.331260 \n",
"24 2018-05-20 20:25:58 13.706 sec 24.0 0.333191 \n",
"25 2018-05-20 20:25:58 14.287 sec 25.0 0.333100 \n",
"26 2018-05-20 20:25:59 14.897 sec 26.0 0.334370 \n",
"27 2018-05-20 20:25:59 15.503 sec 27.0 0.334643 \n",
"28 2018-05-20 20:26:00 16.160 sec 28.0 0.334961 \n",
"29 2018-05-20 20:26:01 16.805 sec 29.0 0.335004 \n",
"30 2018-05-20 20:26:01 17.456 sec 30.0 0.334927 \n",
"\n",
" training_logloss training_auc training_lift \\\n",
"0 NaN NaN NaN \n",
"1 0.763404 0.956487 1.930543 \n",
"2 0.666988 0.958739 1.945962 \n",
"3 0.585470 0.962420 1.959389 \n",
"4 0.530667 0.967584 1.967190 \n",
"5 0.494876 0.970585 1.973005 \n",
"6 0.454248 0.974636 1.979192 \n",
"7 0.427348 0.977835 1.983172 \n",
"8 0.405616 0.980248 1.987241 \n",
"9 0.385657 0.982926 1.990128 \n",
"10 0.366322 0.985589 1.992683 \n",
"11 0.357655 0.986884 1.994899 \n",
"12 0.341858 0.989303 1.996165 \n",
"13 0.337632 0.990264 1.996937 \n",
"14 0.338206 0.990769 1.997604 \n",
"15 0.335644 0.991332 1.998106 \n",
"16 0.331837 0.992059 1.998392 \n",
"17 0.324349 0.993148 1.998720 \n",
"18 0.320754 0.993694 1.999123 \n",
"19 0.316733 0.994235 1.999381 \n",
"20 0.319453 0.994307 1.999455 \n",
"21 0.319514 0.994558 1.999454 \n",
"22 0.318617 0.994821 1.999529 \n",
"23 0.316874 0.995103 1.999529 \n",
"24 0.320829 0.995035 1.999566 \n",
"25 0.320371 0.995231 1.999566 \n",
"26 0.322707 0.995213 1.999566 \n",
"27 0.322935 0.995278 1.999566 \n",
"28 0.323930 0.995309 1.999566 \n",
"29 0.323465 0.995424 1.999566 \n",
"30 0.323080 0.995487 1.999604 \n",
"\n",
" training_classification_error validation_rmse validation_logloss \\\n",
"0 NaN NaN NaN \n",
"1 0.103585 0.257295 1.705777 \n",
"2 0.100437 0.211104 0.612876 \n",
"3 0.096241 0.205624 0.416203 \n",
"4 0.088710 0.204276 0.340733 \n",
"5 0.085154 0.203786 0.294007 \n",
"6 0.081914 0.203566 0.268536 \n",
"7 0.076219 0.203257 0.251817 \n",
"8 0.072625 0.203124 0.243703 \n",
"9 0.067415 0.203066 0.231357 \n",
"10 0.061423 0.203097 0.227298 \n",
"11 0.058755 0.203096 0.220205 \n",
"12 0.052590 0.203156 0.218058 \n",
"13 0.050032 0.203230 0.213603 \n",
"14 0.049137 0.203201 0.209188 \n",
"15 0.047514 0.203163 0.207753 \n",
"16 0.045455 0.203135 0.204570 \n",
"17 0.041518 0.203177 0.203516 \n",
"18 0.039879 0.203168 0.202761 \n",
"19 0.038212 0.203183 0.202061 \n",
"20 0.037777 0.203136 0.198964 \n",
"21 0.036471 0.203140 0.198079 \n",
"22 0.035568 0.203130 0.196660 \n",
"23 0.034353 0.203161 0.196128 \n",
"24 0.034467 0.203116 0.194919 \n",
"25 0.033742 0.203085 0.194157 \n",
"26 0.033783 0.203029 0.192694 \n",
"27 0.033309 0.202999 0.189930 \n",
"28 0.033121 0.202994 0.189183 \n",
"29 0.032500 0.202957 0.188449 \n",
"30 0.032255 0.202957 0.187168 \n",
"\n",
" validation_auc validation_lift validation_classification_error \n",
"0 NaN NaN NaN \n",
"1 0.780964 7.318324 0.064986 \n",
"2 0.808146 10.238311 0.045618 \n",
"3 0.820663 10.566838 0.044361 \n",
"4 0.823580 10.641350 0.044140 \n",
"5 0.825970 10.669638 0.044162 \n",
"6 0.828691 10.674389 0.044096 \n",
"7 0.830573 10.679152 0.044096 \n",
"8 0.831204 10.679152 0.044118 \n",
"9 0.832550 10.679152 0.044118 \n",
"10 0.832316 10.679152 0.044427 \n",
"11 0.833527 10.679152 0.044228 \n",
"12 0.833289 10.679152 0.044162 \n",
"13 0.833152 10.679152 0.044118 \n",
"14 0.834405 10.679152 0.044471 \n",
"15 0.834603 10.679152 0.044361 \n",
"16 0.835777 10.679152 0.044118 \n",
"17 0.836050 10.679152 0.044228 \n",
"18 0.836510 10.679152 0.044625 \n",
"19 0.836516 10.679152 0.044493 \n",
"20 0.837179 10.679152 0.044846 \n",
"21 0.837936 10.679152 0.044272 \n",
"22 0.838474 10.679152 0.044272 \n",
"23 0.838535 10.679152 0.044383 \n",
"24 0.839078 10.679152 0.044780 \n",
"25 0.839023 10.679152 0.044493 \n",
"26 0.838821 10.679152 0.044493 \n",
"27 0.839291 10.679152 0.044603 \n",
"28 0.839264 10.679152 0.044427 \n",
"29 0.839421 10.679152 0.044449 \n",
"30 0.839391 10.679152 0.044692 "
]
},
"execution_count": 16,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"model.score_history()"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAqMAAAHkCAYAAAAKKiYhAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xm4XWV99//3h0FQhjBKQYQgMojI\nYAKCDEJFf1ariEBxQEXRKD4FraV92gdrUR4c6dNCqUOkFBS1CooiVBKkBBBlSCAkgAgKURSVeVaG\n5Pv7Y69TNsczbELOWWef835dV6699r3uda/v2utK+HCvtfZOVSFJkiS1YaW2C5AkSdLUZRiVJElS\nawyjkiRJao1hVJIkSa0xjEqSJKk1hlFJkiS1xjAqSZKk1hhGJUmS1BrDqCRJklqzStsFqHcbbLBB\nTZ8+ve0yJEmSRrVgwYK7qmrD0foZRvvI9OnTmT9/fttlSJIkjSrJL3rp52V6SZIktcaZ0T7yxJ33\ncOfnz2i7DEmS1Kc2POLQtkv4I86MSpIkqTWGUUmSJLXGMCpJkqTWGEYlSZLUGsOoJEmSWmMYlSRJ\nUmsMo5IkSWqNYVSSJEmtMYxKkiSpNYZRSZIktabvwmiSY5McvRzbfTzJfmNRkyRJkpbPlPlt+qr6\naNs1SJIk6an6YmY0yTFJfprkB8A2TduWSc5PsiDJpUm2TTItyZIkKzV9npPktiSrJjktyUFN+y5J\nfpTk2iRXJlkrycpJPpvkqiSLkrxvhHr2STIvyVlJbkzy1SRp1i1JskGzPDPJvGb52CSnJ5nb9HlT\nks8kWdwcx6pj+ylKkiRNPBM+jCaZAbwZ2Bl4E7BLs2o2cGRVzQCOBj5XVfcD1wKvaPq8HphTVY93\njfcs4BvAB6tqR2A/4PfA4cD9VbVLs4/3JtlihNJ2Bj4EbAe8ANijh8PZEngdsD9wBnBRVb2k2f/r\nhjn+WUnmJ5l/90MP9LALSZKk/tEPl+n3As6uqkcAkpwDrA68HDizmZAEWK15/QZwCHARnRD7uUHj\nbQP8pqquAqiqB5pxXw3sMDB7CkwDtgJuHaauK6vqV822C4HpwA9HOZbvV9XjSRYDKwPnN+2Lm+3/\nSFXNphO82WnzF9Qo40uSJPWVfgijAIND2ErAfVW10xB9zwE+mWQ9YAbw34PWZ4jxBtqPrKo5Pdb0\naNfyUp78LJ/gyRnn1YfapqqWJXm8qgbqWEb/nAtJkqQVZsJfpgcuAQ5I8uwka9G59P4IcGuSgwHS\nsSNAVT0EXAmcCJxbVUsHjXcjsEmSXZpt10qyCjAHOGLg3s0kWydZYznqXUInBAMcuBzbS5IkTRkT\nPoxW1dV0Lr0vBL4FXNqsehtweJJrgevp3Ic54BvAoc3r4PEeo3MZ/1+bbS+gM4N5CnADcHWS64Av\nsnyzlR8DTkxyKZ0ZU0mSJA0jT14p1kS30+YvqAv+7uNtlyFJkvrUhkccOm77SrKgqmaO1m/Cz4xK\nkiRp8vKhmREkeQnwlUHNj1bVy9qoR5IkabIxjI6gqhYDQz2xL0mSpBXAy/SSJElqjWFUkiRJrTGM\nSpIkqTWGUUmSJLXGB5j6yCobrjeu3w8mSZI01pwZlSRJUmsMo5IkSWqNYVSSJEmtMYxKkiSpNYZR\nSZIktcYwKkmSpNb41U595LE7lvCrk9/ddhmS1Lc2/ctT2y5B0iDOjEqSJKk1hlFJkiS1xjAqSZKk\n1hhGJUmS1BrDqCRJklpjGJUkSVJrDKOSJElqjWFUkiRJrTGMSpIkqTWGUUmSJLVm0ofRJOsk+UCz\nvEmSs8Zx3w8Ntd8kX0+yKMlfJdk2ycIk1yTZcrxqkyRJmggmfRgF1gE+AFBVt1fVQeNdQPd+k/wJ\n8PKq2qGq/hl4I/Ddqtq5qn4+3rVJkiS1aZW2CxgHnwK2TLIQuBl4UVVtn+QwOkFwZWB74J+AZwFv\nBx4FXltV9zSzlf8GbAg8Ary3qm4cakdJtgC+RudzPb+rfTpwblVtD8wFntvUczZwBLA0yd5Vte8Q\nY84CZgE8b901ntknIUmSNMFMhZnRvwN+XlU7AX8zaN32wFuBXYHjgUeqamfgx8A7mj6zgSOragZw\nNPC5EfZ1IvD5qtoF+O0wfd4wUE9VfQz4AvDPQwVRgKqaXVUzq2rmemuuPtqxSpIk9ZWpMDM6kouq\n6kHgwST3A99r2hcDOyRZE3g5cGaSgW1WG2G8PYADm+WvAJ9e8SVLkiRNHlM9jD7atbys6/0yOp/N\nSsB9zaxqr2oF1SZJkjTpTYXL9A8Cay3PhlX1AHBrkoMB0rHjCJtcBry5WX7b8uxTkiRpKpn0YbSq\n7gYuS3Id8NnlGOJtwOFJrgWuB/Yfoe8Hgf+V5Cpg2nLsS5IkaUpJlVeV+8UOm21Q//W3b2i7DEnq\nW5v+5altlyBNGUkWVNXM0fpN+plRSZIkTVxT/QGm5ZLkGODgQc1nVtXxbdQjSZLUrwyjy6EJnQZP\nSZKkZ8jL9JIkSWqNYVSSJEmtMYxKkiSpNYZRSZIktcYHmPrIs5473e/IkyRJk4ozo5IkSWqNYVSS\nJEmtMYxKkiSpNYZRSZIktcYwKkmSpNb4NH0fefCum7nolNe1XYY06ez7nvPaLkGSpixnRiVJktQa\nw6gkSZJaYxiVJElSawyjkiRJao1hVJIkSa0xjEqSJKk1hlFJkiS1xjAqSZKk1hhGJUmS1BrDqCRJ\nklrTl2E0yZIkG7RdxzOR5LQkB7VdhyRJUpsmbBhNskrbNUiSJGlsjWkYTTI9yY1JTk+yKMlZSZ7T\nPbOZZGaSec3ysUlmJ5kLfDnJyklOSLK42f7IruGPTHJ1s27bZvtdk/woyTXN6zZN+4uTXJlkYTPO\nVk37oV3tX0yy8jDH8RdJ/l+z/MEktzTLWyb5YbM8I8nFSRYkmZNk464+5zftlw7UOmj845qZ0gn7\nPweSJEljYTzCzzbA7KraAXgA+MAo/WcA+1fVW4FZwBbAzs32X+3qd1dVvRT4PHB003YjsHdV7Qx8\nFPhE0/5+4MSq2gmYCfwqyYuAQ4A9mvalwNuGqekSYK9meS/g7iTPA/YELk2yKvCvwEFVNQM4FTi+\n6T8bOLJpPxr4XPfAST4DPBd4V1UtG7zjJLOSzE8y//4HHxv+U5MkSepD43Ep/LaquqxZPgM4apT+\n51TV75vl/YAvVNUTAFV1T1e/bzevC4A3NcvTgNObmc8CVm3afwwck2RT4NtVdXOSV9IJvlclAXg2\ncMdQBVXVb5OsmWQt4PnA14C96QTTb9MJ3NsDFzRjrQz8JsmawMuBM5t2gNW6hv4H4IqqmjXch1FV\ns+kEWraZPq2G6ydJktSPxiOMDg5QBTzBk7Oyqw9a/3DXcobYfsCjzetSnjyO44CLquqAJNOBeQBV\n9bUkVwCvA+YkeU8z9ulV9fc9HsePgXcBPwUuBd4N7A78NbAZcH1V7d69QZK1gfuamdehXAXMSLLe\noKAtSZI0JYzHZfrNkgyEtLcAPwSW0JmVBDhwhG3nAu8feJgpyXqj7Gsa8Otm+bCBxiQvAG6pqpOA\nc4AdgAuBg5I8d2DsJJuPMPYldC6zXwJcA+wLPFpV99MJqBsOHGeSVZO8uKoeAG5NcnDTniQ7do15\nPvAp4Lxm1lWSJGlKGY8w+hPgnUkWAevRucfzY8CJSS6lM7M5nFOAXwKLklwLvHWUfX0G+GSSy+hc\nKh9wCHBdkoXAtsCXq+oG4CPA3Ka2C4CNRxj7UjqX6C+pqqXAbXSCNVX1GHAQ8OmmzoV0Ls9D5z7U\nw5v264H9uwetqjOBLwHnJHn2KMcnSZI0qaRq7G5DbC6Vn1tV24/ZTqaQbaZPqy98ZM+2y5AmnX3f\nc17bJUjSpJNkQVXNHK2fXyUkSZKk1ozpA0xVtYTOU+Z9o3nQabVBzW+vqsVt1CNJkjSZ+StHg1TV\ny9quQZIkaarwMr0kSZJaYxiVJElSawyjkiRJao1hVJIkSa3xAaY+stYGW/l9iJIkaVJxZlSSJEmt\nMYxKkiSpNYZRSZIktcYwKkmSpNYYRiVJktQaw6gkSZJa41c79ZG7776J005/ddtlaAwd9s65bZcg\nSdK4cmZUkiRJrTGMSpIkqTWGUUmSJLXGMCpJkqTWGEYlSZLUGsOoJEmSWmMYlSRJUmsMo5IkSWqN\nYVSSJEmtMYyOkSQfSvKctuuQJEmayAyjY+dDwJBhNMnK41yLJEnShDSlw2iSdyRZlOTaJF9JsnmS\nC5u2C5Ns1vQ7LclBXds91Lzuk2RekrOS3Jjkq+k4CtgEuCjJRQPbJPl4kiuAjyQ5u2u8VyX59rge\nvCRJ0gSwStsFtCXJi4FjgD2q6q4k6wGnA1+uqtOTvBs4CXjjKEPtDLwYuB24rBnvpCQfBvatqrua\nfmsA11XVR5ME+EmSDavqTuBdwH8MU+csYBbA+uuv/kwOWZIkacKZyjOjfwqcNRAWq+oeYHfga836\nrwB79jDOlVX1q6paBiwEpg/TbynwrWZf1Yx/aJJ1mv1+f6iNqmp2Vc2sqplrrbVqTwcmSZLUL6bs\nzCgQoEbpM7D+CZrg3sxqPqurz6Ndy0sZ/jP9Q1Ut7Xr/H8D3gD8AZ1bVEz3WLUmSNGlM5ZnRC4G/\nSLI+QHOZ/kfAm5v1bwN+2CwvAWY0y/sDvUxRPgisNdzKqrqdzqX9jwCnPb3SJUmSJocpOzNaVdcn\nOR64OMlS4BrgKODUJH8DDNzLCfAl4LtJrqQTYh/uYRezge8n+U1V7TtMn68CG1bVDc/kWCRJkvpV\nOrcvqg1JTgauqap/76X/FlusXf947G5jXJXadNg757ZdgiRJK0SSBVU1c7R+U3ZmtG1JFtCZYf3r\ntmuRJElqi2G0JVU1Y/RekiRJk9tUfoBJkiRJLTOMSpIkqTWGUUmSJLXGMCpJkqTWGEYlSZLUGp+m\n7yPrr7+130MpSZImFWdGJUmS1BrDqCRJklpjGJUkSVJrDKOSJElqjWFUkiRJrTGMSpIkqTV+tVMf\n+fW9N3PMma9puwwth+MPPr/tEiRJmpCcGZUkSVJrDKOSJElqjWFUkiRJrTGMSpIkqTWGUUmSJLXG\nMCpJkqTWGEYlSZLUGsOoJEmSWmMYlSRJUmsMo5IkSWrNlA+jSX7Udg2SJElT1ZQPo1X18rZrkCRJ\nmqqmfBhN8lDzuk+SS5KcneSGJF9IMuznk+ShJMcnuTbJ5Uk2atpPS3LQMONfnOSbSW5K8qkkb0ty\nZZLFSbYc62OVJEmaaKZ8GB1kV+CvgZcAWwJvGqHvGsDlVbUjcAnw3h7G3xH4YDP+24Gtq2pX4BTg\nyKE2SDIryfwk8x954LGeD0SSJKkfGEaf6sqquqWqlgJfB/Ycoe9jwLnN8gJgeg/jX1VVv6mqR4Gf\nA3Ob9sXDbV9Vs6tqZlXNfM7az+phF5IkSf3DMPpUNcr7bo9X1cD6pcAqzfITNJ9rkgDdCfLRruVl\nXe+XdW0vSZI0ZRhGn2rXJFs094oeAvxwOcZYAsxolvcHVl1BtUmSJE06htGn+jHwKeA64Fbg7OUY\n40vAK5JcCbwMeHjFlSdJkjS5TPlLw1W1ZtfbR6rqkKe7XVWdBZzVLP8O2K2r69837fOAeV3b7NO1\n/JR1kiRJU4Uzo5IkSWrNlJ8ZHTDc7GSSK4DVBjW/vaoWj0NZkiRJk5phdBRV9bK2a5AkSZqsvEwv\nSZKk1hhGJUmS1BrDqCRJklpjGJUkSVJrfICpjzxv3a04/uDz2y5DkiRphXFmVJIkSa0xjEqSJKk1\nhlFJkiS1xjAqSZKk1hhGJUmS1BrDqCRJklrjVzv1kZvv+zWv/c7/abuMMfNfb/xE2yVIkqRx5syo\nJEmSWmMYlSRJUmsMo5IkSWqNYVSSJEmtMYxKkiSpNYZRSZIktcYwKkmSpNYYRiVJktQaw6gkSZJa\nYxiVJElSa/oijCY5Jcl2bdexIiXZJ8m5bdchSZLUpr74bfqqek/bNUiSJGnFm3Azo0nWSHJekmuT\nXJfkkCTzksxs1h+e5Kam7UtJTm7aT0tyUpIfJbklyUFN+1NmIJOcnOSwZnlJkk8nubL588Jhalq5\nGTNJ1kmyLMnezbpLk7ywqfvUJFcluSbJ/l3bfrZpX5TkfUOMv0uzzQuGWDcryfwk8x974JFn/PlK\nkiRNJBMujAKvAW6vqh2ranvg/IEVSTYB/gHYDXgVsO2gbTcG9gT+HPhUj/t7oKp2BU4G/mWoDlW1\nFLgJ2K4ZfwGwV5LVgE2r6mfAMcB/V9UuwL7AZ5OsARwO3N+07wK8N8kWXcf0cuALwP5VdcsQ+55d\nVTOrauaz1n5Oj4ckSZLUHyZiGF0M7NfMWO5VVfd3rdsVuLiq7qmqx4EzB237napaVlU3ABv1uL+v\nd73uPkK/S4G9mz+fpBNKdwGuata/Gvi7JAuBecDqwGZN+zua9iuA9YGtmm1eBMwGXl9Vv+yxXkmS\npEljwt0zWlU3JZkBvBb4ZJK5XaszyuaPDtH3CZ4aulcfvMthlge7FHg/sAnwUeBvgH2AS7r2d2BV\n/bR7oyQBjqyqOYPa9wF+09SzM3D7CPuWJEmalCbczGhzKf6RqjoDOAF4adfqK4FXJFk3ySrAgT0M\n+QtguySrJZkGvHLQ+kO6Xn88wjhXAC8HllXVH4CFwPvohFSAOcCRTfgkyc5d7UckWbVp37q5fA9w\nH/A64BNNOJUkSZpSJtzMKPASOvdbLgMeB46gE0qpql8n+QSdYHg7cANw/3ADNdvcluSbwCLgZuCa\nQV1WS3IFnWD+lhHGeTTJbcDlTdOlTf/Fzfvj6NxzuqgJpEvo3Lt6CjAduLppvxN4Y9e4v0vyeuD7\nSd5dVVeMdDySJEmTSapGujI98SRZs6oeamZGzwZOraqzl3OsJcDMqrprRdY4Vqa9cOPa44R3tV3G\nmPmvN36i7RIkSdIKkmRBVc0crd+Eu0zfg2Obh4GuA24FvtNyPZIkSVpOE/Ey/Yiq6ugVONb0wW1J\njgEOHtR8ZlUdv6L2K0mSpI6+C6NjrQmdBk9JkqRx0I+X6SVJkjRJGEYlSZLUGsOoJEmSWuM9o31k\nq3We59cfSZKkScWZUUmSJLXGMCpJkqTWjBpGk2yU5N+TfL95v12Sw8e+NEmSJE12vcyMngbMATZp\n3t8EfGisCpIkSdLU0UsY3aCqvgksA6iqJ4ClY1qVJEmSpoRewujDSdYHCiDJbsD9Y1qVJEmSpoRe\nvtrpw8A5wJZJLgM2BA4a06o0pJvvvYvXfetLbZcxJs478L1tlyBJklowahitqquTvALYBgjw06p6\nfMwrkyRJ0qQ3bBhN8qZhVm2dhKr69hjVJEmSpClipJnR14+wrgDDqCRJkp6RYcNoVb1rPAuRJEnS\n1NPLl96vn+SkJFcnWZDkxObpekmSJOkZ6eWrnf4TuBM4kM5T9HcC3xjLoiRJkjQ19PLVTutV1XFd\n7/9vkjeOVUGSJEmaOnqZGb0oyZuTrNT8+QvgvLEuTJIkSZPfSF/t9CCdp+ZD54vvz2hWrQQ8BPzj\nmFcnSZKkSW2kp+nXGs9CJEmSNPX0cs8oSdYFtgJWH2irqkvGqqhRalkCzKyqu9rYfy/6oUZJkqSJ\nYNQwmuQ9wAeBTYGFwG7Aj4E/HauikqxSVU+M1fiSJEmaGHp5gOmDwC7AL6pqX2BnOl/vNKIk05Pc\nmOT0JIuSnJXkOUmWJNmg6TMzybxm+dgks5PMBb6cZOUkJyRZ3Gx/ZNfwRzbfe7o4ybbN9rsm+VGS\na5rXbZr2Fye5MsnCZpytmvZDu9q/mGTlYY7jiCSf6Xp/WJJ/bZa/03z36vVJZg3zGVzX9f7oJMc2\ny1smOb/Z/tKB4xhijFlJ5ieZ/9gDD472sUuSJPWVXsLoH6rqDwBJVquqG4Ftehx/G2B2Ve0APAB8\nYJT+M4D9q+qtwCxgC2DnZvuvdvW7q6peCnweOLppuxHYu6p2Bj4KfKJpfz9wYlXtBMwEfpXkRcAh\nwB5N+1LgbcPUdBbwpq73h/Dk96y+u6pmNOMe9TR/DGA2cGSz/dHA54bqVFWzq2pmVc181trexitJ\nkiaXXu4Z/VWSdYDvABckuRe4vcfxb6uqy5rlM4CjRul/TlX9vlneD/jCwOX6qrqnq9+3m9cFPBkU\npwGnNzOfBazatP8YOCbJpsC3q+rmJK+kE3yvSgLwbOCOoQqqqjuT3JJkN+BmOgF74JiOSnJAs/x8\nOvfV3j3KMZJkTeDlwJnN/gFWG207SZKkyWbUMFpVA2Hr2CQX0Ql95/c4fg3x/gmenJFdfdD6h7uW\nM8T2Ax5tXpfy5DEcB1xUVQckmQ7Ma+r/WpIrgNcBc5p7YAOcXlV/3+NxfAP4Czqzr2dXVSXZh05g\n3r2qHmluNxh8PN3HStf6lYD7mllZSZKkKWvYy/RJ1m5e1xv4AywGfgis2eP4myXZvVl+S7PtEjqz\nktD5idHhzAXen2SVgTpG2dc04NfN8mFdx/EC4JaqOgk4B9gBuBA4KMlzB8ZOsvkIY38beGNzDAOX\n6KcB9zZBdFs6D3YN9jvguUnWT7Ia8OcAVfUAcGuSg5v9J8mOoxyfJEnSpDPSPaNfa14XAPOHeO3F\nT4B3JlkErEfnHs+PAScmuZTOzOZwTgF+CSxKci3w1lH29Rngk0kuA7ofRjoEuC7JQmBb4MtVdQPw\nEWBuU9sFwMbDDVxV9wI3AJtX1ZVN8/nAKs32xwGXD7Hd48DHgSuAc+nMrA54G3B4c2zXA/uPcnyS\nJEmTTqqGuxLembEDnl9Vv3zaA3culZ9bVdsvd3V6imlbTq89P3NM22WMifMOfG/bJUiSpBUoyYKq\nmjlavxGfpq9OUj17hVUlSZIkdenlafrLk+xSVVc9nYGragnQV7OizYNOg59qf3tVLW6jHkmSpMmu\nlzC6L/C+JL+g87R76Eya7jCmlbWgql7Wdg2SJElTSS9h9M/GvApJkiRNSb18z+gvAJqvQRr8PZqS\nJEnSchv150CTvCHJzcCtwMV0vif0+2NclyRJkqaAXn6b/jg6X+h+U1VtAbySJ38OU5IkSVpuvdwz\n+nhV3Z1kpSQrVdVFST495pXpj2y17gZ+H6ckSZpUegmj9yVZE7gU+GqSO+j85rokSZL0jPRymf4S\nYB3gg3R+AvPnwOvHsihJkiRNDb2E0QBzgHnAmsA3qurusSxKkiRJU8OoYbSqPlZVLwb+F7AJcHGS\nH4x5ZZIkSZr0epkZHXAH8FvgbuC5Y1OOJEmSppJevmf0iCTzgAuBDYD3TsafApUkSdL46+Vp+s2B\nD1XVwrEuRiP72b338fqzzm67jBXuewcd0HYJkiSpJb38HOjfjUchkiRJmnqezj2jkiRJ0gplGJUk\nSVJrDKOSJElqjWFUkiRJrTGMSpIkqTWGUUmSJLXGMCpJkqTWGEYlSZLUGsOoJEmSWmMYlSRJUmsM\noytQku8kWZDk+iSzmrbDk9yUZF6SLyU5uWnfMMm3klzV/Nmj3eolSZLG36i/Ta+n5d1VdU+SZwNX\nJTkP+AfgpcCDwH8D1zZ9TwT+uap+mGQzYA7wosEDNqF2FsCzN9hwHA5BkiRp/BhGV6yjkhzQLD8f\neDtwcVXdA5DkTGDrZv1+wHZJBrZdO8laVfVg94BVNRuYDbDOli+sMa5fkiRpXBlGV5Ak+9AJmLtX\n1SNJ5gE/ZYjZzsZKTd/fj0+FkiRJE4/3jK4404B7myC6LbAb8BzgFUnWTbIKcGBX/7nAXw68SbLT\nuFYrSZI0ARhGV5zzgVWSLAKOAy4Hfg18ArgC+AFwA3B/0/8oYGaSRUluAN4//iVLkiS1y8v0K0hV\nPQr82eD2JPOranYzM3o2nRlRquou4JDxrVKSJGlicWZ07B2bZCFwHXAr8J2W65EkSZownBkdY1V1\ndNs1SJIkTVTOjEqSJKk1hlFJkiS1xjAqSZKk1hhGJUmS1BrDqCRJklrj0/R95IXrrsP3Djqg7TIk\nSZJWGGdGJUmS1BrDqCRJklpjGJUkSVJrDKOSJElqjWFUkiRJrTGMSpIkqTV+tVMf+fm9D3Pgt65s\nu4yn7VsH7tp2CZIkaYJyZlSSJEmtMYxKkiSpNYZRSZIktcYwKkmSpNYYRiVJktQaw6gkSZJaYxiV\nJElSawyjkiRJao1hVJIkSa0xjEqSJKk1fR9Gk5ySZLu26+iW5KG2a5AkSeoHff/b9FX1nrZrkCRJ\n0vLpq5nRJGskOS/JtUmuS3JIknlJZjbrD09yU9P2pSQnN+2nJTkpyY+S3JLkoKZ9nyTndo1/cpLD\nmuUlST6d5MrmzwtHqGuLJD9OclWS47ra10xyYZKrkyxOsn/TflySD3b1Oz7JUcOMPSvJ/CTzH33g\nvmf0+UmSJE00fRVGgdcAt1fVjlW1PXD+wIokmwD/AOwGvArYdtC2GwN7An8OfKrH/T1QVbsCJwP/\nMkK/E4HPV9UuwG+72v8AHFBVLwX2Bf4pSYB/B97Z1L0S8Gbgq0MNXFWzq2pmVc1cbe11eixbkiSp\nP/RbGF0M7NfMWO5VVfd3rdsVuLiq7qmqx4EzB237napaVlU3ABv1uL+vd73uPkK/Pbr6fqWrPcAn\nkiwCfgA8D9ioqpYAdyfZGXg1cE1V3d1jTZIkSZNGX90zWlU3JZkBvBb4ZJK5XaszyuaPDtH3CZ4a\nyFcfvMthlocsb4i2twEbAjOq6vEkS7r2cQpwGPAnwKmjjC1JkjQp9dXMaHMp/pGqOgM4AXhp1+or\ngVckWTfJKsCBPQz5C2C7JKslmQa8ctD6Q7pefzzCOJfRudQOnQA6YBpwRxNE9wU271p3Np3bDnYB\n5vRQqyRJ0qTTVzOjwEuAzyZZBjwOHEEnlFJVv07yCeAK4HbgBuD+4QZqtrktyTeBRcDNwDWDuqyW\n5Ao6of0tIwz1QeBrzUNJ3+pq/yrwvSTzgYXAjV37fizJRcB9VbV05MOWJEmanFI12tXn/pFkzap6\nqJkZPRs4tarOXs6xlgAzq+quFVlj1/grAVcDB1fVzb1ss+6WL6o//czpY1HOmPrWgbu2XYIkSRpn\nSRZU1czR+vXVZfoeHJtkIXAdcCvwnZbrGVLzJf0/Ay7sNYhKkiRNRv12mX5EVXX0Chxr+uC2JMcA\nBw9qPrOqjn+aY98AvGD5q5Mvoj55AAAUL0lEQVQkSZocJlUYHWtN6HxawVOSJEnDm2yX6SVJktRH\nDKOSJElqjWFUkiRJrTGMSpIkqTU+wNRHtlx3Db+zU5IkTSrOjEqSJKk1hlFJkiS1xjAqSZKk1hhG\nJUmS1BrDqCRJklrj0/R95Hf3Pc7/O/u3bZfxRz58wJ+0XYIkSepTzoxKkiSpNYZRSZIktcYwKkmS\npNYYRiVJktQaw6gkSZJaYxiVJElSawyjkiRJao1hVJIkSa0xjEqSJKk1hlFJkiS1ZlKG0SRLkmzQ\n4v4PS3JyW/uXJEnqF30bRpOs0nYNkiRJemZaDaNJpie5McnpSRYlOSvJc7pnNpPMTDKvWT42yewk\nc4EvJ1k5yQlJFjfbH9k1/JFJrm7Wbdtsv2uSHyW5pnndpml/cZIrkyxsxtmqaT+0q/2LSVYe4Vje\nleSmJBcDe3S1vz7JFc0+f5BkoyQrJbk5yYZNn5WS/KzN2VxJkqQ2TISZ0W2A2VW1A/AA8IFR+s8A\n9q+qtwKzgC2AnZvtv9rV766qeinweeDopu1GYO+q2hn4KPCJpv39wIlVtRMwE/hVkhcBhwB7NO1L\ngbcNVVCSjYGP0QmhrwK261r9Q2C3Zp//CfxtVS0Dzugabz/g2qq6a4ixZyWZn2T+ww/cPcpHI0mS\n1F8mwqXu26rqsmb5DOCoUfqfU1W/b5b3A75QVU8AVNU9Xf2+3bwuAN7ULE8DTm9mPgtYtWn/MXBM\nkk2Bb1fVzUleSSf4XpUE4NnAHcPU9DJgXlXdCZDkG8DWzbpNgW80gfVZwK1N+6nAd4F/Ad4N/MdQ\nA1fVbGA2wPNfuGMN96FIkiT1o4kwMzo4YBXwBE/Wtvqg9Q93LWeI7Qc82rwu5cnQfRxwUVVtD7x+\nYOyq+hrwBuD3wJwkf9qMfXpV7dT82aaqjn0axzHgX4GTq+olwPu69nkb8LtmXy8Dvj/C2JIkSZPS\nRAijmyXZvVl+C53L2kvozEoCHDjCtnOB9w88zJRkvVH2NQ34dbN82EBjkhcAt1TVScA5wA7AhcBB\nSZ47MHaSzYcZ9wpgnyTrJ1kVOHiYfb5z0Han0JkN/mZVLR2ldkmSpElnIoTRnwDvTLIIWI/OPZ4f\nA05Mcimdmc3hnAL8EliU5FrgraPs6zPAJ5NcBnQ/jHQIcF2ShcC2wJer6gbgI8DcprYLgI2HGrSq\nfgMcS+dy/w+Aq7tWHwuc2RzL4HtCzwHWZJhL9JIkSZNdqtq7DTHJdODc5rL5lJNkJvDPVbVXL/2f\n/8Id668+O2eMq3r6PnzAn7RdgiRJmmCSLKiqmaP1mwgPME1JSf4OOIJhntCXJEmaCloNo1W1BOir\nWdEkVwCrDWp+e1UtfjrjVNWngE+tsMIkSZL6kDOjT1NVvaztGiRJkiaLifAAkyRJkqYow6gkSZJa\nYxiVJElSawyjkiRJao0PMPWRjdZZ1e/0lCRJk4ozo5IkSWqNYVSSJEmtMYxKkiSpNYZRSZIktcYw\nKkmSpNYYRiVJktQav9qpjzx4zxPMO+POcd3nPoduOK77kyRJU4szo5IkSWqNYVSSJEmtMYxKkiSp\nNYZRSZIktcYwKkmSpNYYRiVJktQaw6gkSZJaYxiVJElSawyjkiRJas2UDKNJpie5ru06JEmSprop\nGUZ7kWTlqbBPSZKkNk3lMLpKktOTLEpyVpLnJFmS5KNJfggcnGSnJJc3fc5Osm6S5yZZAJBkxySV\nZLPm/c+bcQ5Ocl2Sa5Nc0qxbOclnk1zVjPe+pn2fJBcl+RqwuK0PQ5IkqQ2rtF1Ai7YBDq+qy5Kc\nCnygaf9DVe0JkGQRcGRVXZzk48A/VtWHkqyeZG1gL2A+sFcTYO+oqkeSfBT4/6rq10nWacY9HLi/\nqnZJshpwWZK5zbpdge2r6tbBRSaZBcwC2Gj9Tcfic5AkSWrNVJ4Zva2qLmuWzwD2bJa/AZBkGrBO\nVV3ctJ8O7N0s/wjYo3n/ieZ1L+DSZv1lwGlJ3gsMXHp/NfCOJAuBK4D1ga2adVcOFUQBqmp2Vc2s\nqpnT1l7/mRyvJEnShDOVZ0ZrmPcP97DtpXTC5+bAd4H/3Wx/LkBVvT/Jy4DXAQuT7ASEzizrnO6B\nkuzT4z4lSZImnak8M7pZkt2b5bcAP+xeWVX3A/cm2atpejswMEt6CXAocHNVLQPuAV5LZ0aUJFtW\n1RVV9VHgLuD5wBzgiCSrNn22TrLGmB2dJElSH5jKM6M/Ad6Z5IvAzcDngSMH9Xkn8IUkzwFuAd4F\nUFVLkkAnlEInyG5aVfc27z+bZCs6s6EXAtcCi4DpwNXpbHwn8MaxOTRJkqT+kKrBV6s1UW3zgp3q\nix+/YFz3uc+hG47r/iRJ0uSQZEFVzRyt31S+TC9JkqSWGUYlSZLUGsOoJEmSWmMYlSRJUmsMo5Ik\nSWqNYVSSJEmtMYxKkiSpNYZRSZIktWYq/wJT31lrvVX8EnpJkjSpODMqSZKk1hhGJUmS1BrDqCRJ\nklpjGJUkSVJrDKOSJElqjU/T95HHf/s4v/nMb5Z7+43/duMVWI0kSdIz58yoJEmSWmMYlSRJUmsM\no5IkSWqNYVSSJEmtMYxKkiSpNYZRSZIktcYwKkmSpNYYRiVJktQaw6gkSZJaYxiVJElSawyjYyiJ\nP7cqSZI0AsNolyRrJDkvybVJrktySJJXJrkmyeIkpyZZrem7JMkGzfLMJPOa5WOTzE4yF/hykpWT\nnNBsvyjJkU2/GUkuTrIgyZwk/nC8JEmacpy5e6rXALdX1esAkkwDrgNeWVU3JfkycATwL6OMMwPY\ns6p+n+QIYAtg56p6Isl6SVYF/hXYv6ruTHIIcDzw7jE6LkmSpAnJmdGnWgzsl+TTSfYCpgO3VtVN\nzfrTgb17GOecqvp9s7wf8IWqegKgqu4BtgG2By5IshD4CLDpUAMlmZVkfpL5dz989/IelyRJ0oTk\nzGiXZvZzBvBa4JPA3BG6P8GTYX71Qese7loOUIPWB7i+qnbvoabZwGyAHTfdcfA4kiRJfc2Z0S5J\nNgEeqaozgBOAlwPTk7yw6fJ24OJmeQmdy/EAB44w7Fzg/QMPMyVZD/gpsGGS3Zu2VZO8eEUeiyRJ\nUj8wjD7VS4Arm0vnx9C5fP4u4Mwki4FlwBeavh8DTkxyKbB0hDFPAX4JLEpyLfDWqnoMOAj4dNO2\nkE7wlSRJmlJS5ZXffrHjpjvW+Uedv9zbb/y3PrAvSZLGR5IFVTVztH7OjEqSJKk1hlFJkiS1xjAq\nSZKk1hhGJUmS1BrDqCRJklpjGJUkSVJrDKOSJElqjWFUkiRJrfG36fvIqn+yql9cL0mSJhVnRiVJ\nktQaw6gkSZJaYxiVJElSawyjkiRJao1hVJIkSa0xjEqSJKk1frVTH3n8jof43Uk/HLXfRkftOQ7V\nSJIkPXPOjEqSJKk1hlFJkiS1xjAqSZKk1hhGJUmS1BrDqCRJklpjGJUkSVJrDKOSJElqjWFUkiRJ\nrTGMSpIkqTWGUUmSJLXGMCpJkqTWGEZXoCRrJDkvybVJrktySJIZSS5OsiDJnCQbJ1klyVVJ9mm2\n+2SS41suX5Ikadyt0nYBk8xrgNur6nUASaYB3wf2r6o7kxwCHF9V705yGHBWkqOa7V421IBJZgGz\nADZdd6NxOARJkqTxYxhdsRYDJyT5NHAucC+wPXBBEoCVgd8AVNX1Sb4CfA/YvaoeG2rAqpoNzAbY\ncbNta8yPQJIkaRwZRlegqropyQzgtcAngQuA66tq92E2eQlwH+CUpyRJmpK8Z3QFSrIJ8EhVnQGc\nQOfS+4ZJdm/Wr5rkxc3ym4D1gb2Bk5Ks01LZkiRJrXFmdMV6CfDZJMuAx4EjgCfohM1pdD7vf0ny\nO+BTwCur6rYkJwMnAu9sqW5JkqRWGEZXoKqaA8wZYtXeQ7Rt3bXdSWNWlCRJ0gTmZXpJkiS1xjAq\nSZKk1hhGJUmS1BrDqCRJklpjGJUkSVJrDKOSJElqjWFUkiRJrTGMSpIkqTV+6X0fWfW5a7LRUXu2\nXYYkSdIK48yoJEmSWmMYlSRJUmtSVW3XoB4leRD4adt1aFQbAHe1XYRG5XnqH56r/uB56g/jeZ42\nr6oNR+vkPaP95adVNbPtIjSyJPM9TxOf56l/eK76g+epP0zE8+RlekmSJLXGMCpJkqTWGEb7y+y2\nC1BPPE/9wfPUPzxX/cHz1B8m3HnyASZJkiS1xplRSZIktcYwKkmSpNYYRieYJK9J8tMkP0vyd0Os\nXy3JN5r1VySZPv5VCno6Vx9OckOSRUkuTLJ5G3VOdaOdp65+ByWpJBPqK0+mil7OU5K/aP5OXZ/k\na+Ndozp6+LdvsyQXJbmm+ffvtW3UOZUlOTXJHUmuG2Z9kpzUnMNFSV463jV2M4xOIElWBv4N+DNg\nO+AtSbYb1O1w4N6qeiHwz8Cnx7dKQc/n6hpgZlXtAJwFfGZ8q1SP54kkawFHAVeMb4WC3s5Tkq2A\nvwf2qKoXAx8a90LV69+pjwDfrKqdgTcDnxvfKgWcBrxmhPV/BmzV/JkFfH4cahqWYXRi2RX4WVXd\nUlWPAf8J7D+oz/7A6c3yWcArk2Qca1THqOeqqi6qqkeat5cDm45zjert7xTAcXT+Z+EP41mc/kcv\n5+m9wL9V1b0AVXXHONeojl7OVQFrN8vTgNvHsT4BVXUJcM8IXfYHvlwdlwPrJNl4fKr7Y4bRieV5\nwG1d73/VtA3Zp6qeAO4H1h+X6tStl3PV7XDg+2NakYYy6nlKsjPw/Ko6dzwL01P08vdpa2DrJJcl\nuTzJSLM+Gju9nKtjgUOT/Ar4L+DI8SlNT8PT/W/YmPLnQCeWoWY4B3/3Vi99NPZ6Pg9JDgVmAq8Y\n04o0lBHPU5KV6Nzucth4FaQh9fL3aRU6lxT3oXOV4dIk21fVfWNcm56ql3P1FuC0qvqnJLsDX2nO\n1bKxL089mlBZwpnRieVXwPO73m/KH1/e+J8+SVahcwlkpKl4jY1ezhVJ9gOOAd5QVY+OU2160mjn\naS1ge2BekiXAbsA5PsQ07nr9t++7VfV4Vd0K/JROONX46uVcHQ58E6CqfgysDmwwLtWpVz39N2y8\nGEYnlquArZJskeRZdG78PmdQn3OAdzbLBwH/Xf5yQRtGPVfN5d8v0gmi3t/WjhHPU1XdX1UbVNX0\nqppO597eN1TV/HbKnbJ6+bfvO8C+AEk2oHPZ/pZxrVLQ27n6JfBKgCQvohNG7xzXKjWac4B3NE/V\n7wbcX1W/aasYL9NPIFX1RJK/BOYAKwOnVtX1ST4OzK+qc4B/p3PJ42d0ZkTf3F7FU1eP5+qzwJrA\nmc0zZr+sqje0VvQU1ON5Ust6PE9zgFcnuQFYCvxNVd3dXtVTU4/n6q+BLyX5KzqXfg9z0mR8Jfk6\nnVtaNmju3f1HYFWAqvoCnXt5Xwv8DHgEeFc7lXb4c6CSJElqjZfpJUmS1BrDqCRJklpjGJUkSVJr\nDKOSJElqjWFUkiRJrTGMStIEkGR6kuue5janJTlohPWnJNnumVf3tGr6P+O5P0n9zzAqSZNUVb2n\nqm4Yj301X569EmAYlfS0GEYlaQVIskaS85Jcm+S6JIck2SXJj5q2K5Os1cyAXprk6ubPy4cYa+Uk\nn01yVZJFSd7XtCfJyUluSHIe8NxRapo38NOmSR5K8ukkC5L8IMmuzfpbkryh6XNYku8mOT/JT5P8\nY9dYH26O67okH2rapif5SZLPAVfT+VGOZydZmOSrTZ/vNPu8PsmsrvEeSnJ889lcnmSjpn2jJGc3\n7dcOfD5JDm0+w4VJvphk5WdyviRNHP4CkyStGK8Bbq+q1wEkmQZcAxxSVVclWRv4PXAH8Kqq+kOS\nrYCvAzMHjXU4nZ/n2yXJasBlSeYCOwPbAC8BNgJuAE7tsb41gHlV9b+TnA38X+BVwHbA6Tz5k467\nAtvT+VWWq5rQW3R+oeVlQIArklwM3NvU866q+kBz3AdX1U5d+313Vd2T5NnNeN9qfjlpDeDyqjom\nyWeA9zY1nQRcXFUHNIFzzeYnJQ8B9qiqx5vw+zbgyz0eu6QJzDAqSSvGYuCEJJ8GzgXuA35TVVcB\nVNUD0JlBBU5OshOdn7XceoixXg3s0HU/6DRgK2Bv4OtVtRS4Pcl/P436HgPO76r10SbYLQamd/W7\nYOBnNpN8G9iTThg9u6oe7mrfi06A/UVVXT7Cfo9KckCz/PzmOO5u6jm3aV9AJxgD/CnwDoDmOO9P\n8nZgBp0wC/BsOqFe0iRgGJWkFaCqbkoyg87vPX8SmEsnxA32V8DvgB3p3Cr1hyH6BDiyquY8pTF5\n7TBj9uLxrt8HXwY82tS9LEn3fwsGj19NPcN5eLgVSfYB9gN2r6pHkswDVh+inqWM/N+jAKdX1d+P\n0EdSn/KeUUlaAZJsAjxSVWcAJwC7AZsk2aVZv1YT+qbRmTFdBrwdGOrexznAEUlWbbbduplRvQR4\nc3NP6cbAvmNwKK9Ksl5zWf2NwGXNft+Y5DlNHQcAlw6z/eMDddM51nubILotnc9kNBcCR8D/3Du7\ndtN2UPL/t3OHKhEGYRSG33MLBqsKVpu3YhctioLR4h0YBcNiE2xit4ggCCIrrEmb2QswLDiG+Rdh\nWQyLMiv7PnFgYKYdvsNMFrv1hSRL015Q0mxxMipJv2MNOE7yCQypgSrASRfsPqhTwlPgMskGcMPk\nyeIZtTrvp/bS79RgeEWtsZ+BV+D2D+5xB5wDq8BFKeUR6jdSwMPofKWUpyTLE/b3gEGSPrAF7CQZ\nAC/AT3X+yAHQS7JNnZjullLukxwB16kv9ofAHvA23RUlzZJ8tySSpHmWZBNYL6Xstz6LpPlhTS9J\nkqRmnIxK0j/XfdW0MrZ8OP4ASpJmkWFUkiRJzVjTS5IkqRnDqCRJkpoxjEqSJKkZw6gkSZKaMYxK\nkiSpmS+/CLSy5DQ9LwAAAABJRU5ErkJggg==\n",
"text/plain": [
"