{ "cells": [ { "cell_type": "markdown", "id": "7e8eef48", "metadata": {}, "source": [ "# Lesson 21: Ensemble learning demonstration\n", "\n", "This notebook demonstrates key concepts and tools for training ensemble models.\n", "\n", "1. Baseline models\n", " - Logistic regression\n", " - Decision tree\n", "2. Parallel ensembles\n", " - Voting ensemble\n", " - Bagging ensemble\n", " - Random forest\n", "3. Serial (sequential) ensembles\n", " - AdaBoost\n", " - Gradient boosting\n", " - Stacking ensemble\n", "4. Model comparison\n", " - Score comparison\n", " - Confusion matrix comparison\n", "5. Model metric optimization\n", " - ROC_AUC optimized thresholds\n", " - F1 optimized thresholds\n", "\n", "## Notebook set up\n", "\n", "### Imports" ] }, { "cell_type": "code", "execution_count": 1, "id": "463812a9", "metadata": {}, "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", "import pandas as pd\n", "\n", "from sklearn.calibration import CalibratedClassifierCV\n", "from sklearn.datasets import make_classification\n", "from sklearn.linear_model import LogisticRegression\n", "from sklearn.metrics import ConfusionMatrixDisplay\n", "from sklearn.model_selection import train_test_split, cross_validate, TunedThresholdClassifierCV\n", "from sklearn.svm import SVC\n", "from sklearn.tree import DecisionTreeClassifier\n", "\n", "# Ensemble models\n", "from sklearn.ensemble import (\n", " AdaBoostClassifier,\n", " BaggingClassifier,\n", " GradientBoostingClassifier,\n", " RandomForestClassifier,\n", " StackingClassifier,\n", " VotingClassifier,\n", ")" ] }, { "cell_type": "markdown", "id": "7012bb8b", "metadata": {}, "source": [ "### Dataset" ] }, { "cell_type": "code", "execution_count": 2, "id": "60a2f9f5", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
| \n", " | feature_0 | \n", "feature_1 | \n", "feature_2 | \n", "feature_3 | \n", "feature_4 | \n", "feature_5 | \n", "feature_6 | \n", "feature_7 | \n", "feature_8 | \n", "feature_9 | \n", "... | \n", "feature_11 | \n", "feature_12 | \n", "feature_13 | \n", "feature_14 | \n", "feature_15 | \n", "feature_16 | \n", "feature_17 | \n", "feature_18 | \n", "feature_19 | \n", "target | \n", "
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 6850 | \n", "2.056957 | \n", "0.222830 | \n", "-1.914398 | \n", "1.095708 | \n", "-2.389691 | \n", "-1.113905 | \n", "1.296463 | \n", "-5.769138 | \n", "-1.742394 | \n", "-1.717438 | \n", "... | \n", "0.903576 | \n", "-1.730474 | \n", "-0.211586 | \n", "0.402758 | \n", "-2.171021 | \n", "1.242654 | \n", "-0.993714 | \n", "1.083335 | \n", "1.735312 | \n", "0 | \n", "
| 5856 | \n", "1.776417 | \n", "-3.711362 | \n", "-0.042654 | \n", "-4.450805 | \n", "0.283605 | \n", "0.062951 | \n", "-2.952074 | \n", "-4.635668 | \n", "0.133406 | \n", "-1.665485 | \n", "... | \n", "-1.160329 | \n", "1.273855 | \n", "-2.754988 | \n", "-5.904544 | \n", "-0.580578 | \n", "0.952436 | \n", "-0.566598 | \n", "-1.590927 | \n", "0.632844 | \n", "0 | \n", "
| 2483 | \n", "-0.017753 | \n", "1.031828 | \n", "-2.485858 | \n", "-0.667876 | \n", "-1.021137 | \n", "-0.692445 | \n", "0.630700 | \n", "2.300769 | \n", "-1.097844 | \n", "0.717599 | \n", "... | \n", "-0.728046 | \n", "0.328055 | \n", "0.403882 | \n", "1.735435 | \n", "0.162798 | \n", "-1.306485 | \n", "-0.760256 | \n", "0.780826 | \n", "-1.476193 | \n", "0 | \n", "
| 4915 | \n", "-0.495843 | \n", "2.895735 | \n", "-2.537299 | \n", "-2.609338 | \n", "-0.875950 | \n", "2.935914 | \n", "3.253340 | \n", "6.378967 | \n", "-0.058042 | \n", "-1.338881 | \n", "... | \n", "-1.579209 | \n", "1.383636 | \n", "0.879382 | \n", "2.941039 | \n", "-0.171335 | \n", "0.508342 | \n", "-0.704955 | \n", "1.260240 | \n", "-2.110639 | \n", "0 | \n", "
| 4598 | \n", "0.899913 | \n", "-0.241683 | \n", "-1.206857 | \n", "-4.108075 | \n", "0.786454 | \n", "-0.923499 | \n", "-1.404900 | \n", "-6.952240 | \n", "-1.282080 | \n", "-1.646648 | \n", "... | \n", "3.731664 | \n", "1.857395 | \n", "-0.758554 | \n", "4.081466 | \n", "2.046172 | \n", "0.938040 | \n", "-0.088168 | \n", "-0.283488 | \n", "0.257320 | \n", "0 | \n", "
5 rows × 21 columns
\n", "| \n", " | Model | \n", "AUC Mean | \n", "AUC 95% CI Low | \n", "AUC 95% CI High | \n", "F1 Mean | \n", "F1 95% CI Low | \n", "F1 95% CI High | \n", "
|---|---|---|---|---|---|---|---|
| 0 | \n", "Logistic Regression | \n", "0.662758 | \n", "0.642092 | \n", "0.683424 | \n", "0.304572 | \n", "0.291414 | \n", "0.317729 | \n", "
| 1 | \n", "Decision Tree | \n", "0.599774 | \n", "0.587684 | \n", "0.611864 | \n", "0.307292 | \n", "0.287241 | \n", "0.327343 | \n", "
| 2 | \n", "Voting | \n", "0.764078 | \n", "0.752256 | \n", "0.775899 | \n", "0.360355 | \n", "0.343349 | \n", "0.377361 | \n", "
| 3 | \n", "Bagging | \n", "0.783047 | \n", "0.763948 | \n", "0.802146 | \n", "0.243632 | \n", "0.216746 | \n", "0.270519 | \n", "
| 4 | \n", "Random Forest | \n", "0.798929 | \n", "0.780129 | \n", "0.817729 | \n", "0.444362 | \n", "0.404471 | \n", "0.484254 | \n", "
| 5 | \n", "AdaBoost | \n", "0.683823 | \n", "0.661132 | \n", "0.706513 | \n", "0.085703 | \n", "0.067820 | \n", "0.103587 | \n", "
| 6 | \n", "Gradient Boosting | \n", "0.736354 | \n", "0.717980 | \n", "0.754728 | \n", "0.225941 | \n", "0.201503 | \n", "0.250379 | \n", "
| 7 | \n", "Stacking | \n", "0.811252 | \n", "0.794148 | \n", "0.828356 | \n", "0.520599 | \n", "0.506728 | \n", "0.534470 | \n", "