{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Import Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np                             \n",
    "from sklearn.linear_model import SGDClassifier                                          \n",
    "from sklearn.model_selection import train_test_split, cross_val_predict, KFold                            \n",
    "from sklearn.metrics import confusion_matrix, precision_score, recall_score, f1_score\n",
    "from sklearn.utils import shuffle"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Part I: \"Train-Test Split\" Method"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "QUESTION: \n",
    "Divide the data set into training set and test set. \n",
    "The following show mix percetages for training and test.\n",
    "For each case, show confusion matrix, precision, and F1 score for test set."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "a) 85,15"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('MyData.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Determine features and targets of the incoming model\n",
    "target = 'left'\n",
    "X = np.array(df.drop(columns=target))  # Parameters\n",
    "Y = np.array(df[target])               # Output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">> Confusion Matrix:\n",
      "[[TN: 1529, FP: 0],\n",
      " [FN: 310, TP: 0]]\n",
      ">> precision: 0.0\n",
      ">> recall: 0.0\n",
      ">> F1: 0.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\ADMIN\\anaconda3\\Lib\\site-packages\\sklearn\\metrics\\_classification.py:1509: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, f\"{metric.capitalize()} is\", len(result))\n"
     ]
    }
   ],
   "source": [
    "# Train-Test Split\n",
    "X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.15, random_state=42)\n",
    "\n",
    "# Initialize the classifier\n",
    "classifier = SGDClassifier(random_state=42)\n",
    "\n",
    "# Train the model with the training set\n",
    "classifier.fit(X_train, Y_train)\n",
    "\n",
    "# Predict on the test set \n",
    "Y_predict = classifier.predict(X_test)\n",
    "\n",
    "# Confusion Matrix, Precision, Recall, F1\n",
    "confusion = confusion_matrix(Y_test, Y_predict)\n",
    "precision = precision_score(Y_test, Y_predict)\n",
    "recall = recall_score(Y_test, Y_predict)\n",
    "F1 = f1_score(Y_test, Y_predict)\n",
    "\n",
    "# Show model evaluations\n",
    "print(f'>> Confusion Matrix:')\n",
    "print(f'[[TN: {confusion[0][0]}, FP: {confusion[0][1]}],\\n'\n",
    "      f' [FN: {confusion[1][0]}, TP: {confusion[1][1]}]]')\n",
    "print(f'>> precision: {precision}')\n",
    "print(f'>> recall: {recall}')\n",
    "print(f'>> F1: {F1}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "b) 75,25"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">> Confusion Matrix:\n",
      "[[TN: 42, FP: 2492],\n",
      " [FN: 1, TP: 530]]\n",
      ">> precision: 0.17538054268696227\n",
      ">> recall: 0.9981167608286252\n",
      ">> F1: 0.29833943146636643\n"
     ]
    }
   ],
   "source": [
    "# Train-Test Split\n",
    "X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.25, random_state=42)\n",
    "\n",
    "# Train the model with the training set\n",
    "classifier.fit(X_train, Y_train)\n",
    "\n",
    "# Predict on the test set \n",
    "Y_predict = classifier.predict(X_test)\n",
    "\n",
    "# Confusion Matrix, Precision, Recall, F1\n",
    "confusion = confusion_matrix(Y_test, Y_predict)\n",
    "precision = precision_score(Y_test, Y_predict)\n",
    "recall = recall_score(Y_test, Y_predict)\n",
    "F1 = f1_score(Y_test, Y_predict)\n",
    "\n",
    "# Show model evaluations\n",
    "print(f'>> Confusion Matrix:')\n",
    "print(f'[[TN: {confusion[0][0]}, FP: {confusion[0][1]}],\\n'\n",
    "      f' [FN: {confusion[1][0]}, TP: {confusion[1][1]}]]')\n",
    "print(f'>> precision: {precision}')\n",
    "print(f'>> recall: {recall}')\n",
    "print(f'>> F1: {F1}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "c) 65,35"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">> Confusion Matrix:\n",
      "[[TN: 3349, FP: 197],\n",
      " [FN: 568, TP: 177]]\n",
      ">> precision: 0.4732620320855615\n",
      ">> recall: 0.23758389261744967\n",
      ">> F1: 0.3163538873994638\n"
     ]
    }
   ],
   "source": [
    "# Train-Test Split\n",
    "X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.35, random_state=42)\n",
    "\n",
    "# Train the model with the training set\n",
    "classifier.fit(X_train, Y_train)\n",
    "\n",
    "# Predict on the test set \n",
    "Y_predict = classifier.predict(X_test)\n",
    "\n",
    "# Confusion Matrix, Precision, Recall, F1\n",
    "confusion = confusion_matrix(Y_test, Y_predict)\n",
    "precision = precision_score(Y_test, Y_predict)\n",
    "recall = recall_score(Y_test, Y_predict)\n",
    "F1 = f1_score(Y_test, Y_predict)\n",
    "\n",
    "# Show model evaluations\n",
    "print(f'>> Confusion Matrix:')\n",
    "print(f'[[TN: {confusion[0][0]}, FP: {confusion[0][1]}],\\n'\n",
    "      f' [FN: {confusion[1][0]}, TP: {confusion[1][1]}]]')\n",
    "print(f'>> precision: {precision}')\n",
    "print(f'>> recall: {recall}')\n",
    "print(f'>> F1: {F1}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Part II: \"K-Fold Cross Validation\" Method "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "QUESTION:\n",
    "Using K-Fold cross validation with 5 folds to train and validate the model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 1:\n",
      "Confusion Matrix:\n",
      "[[TN: 2039, FP: 1],\n",
      " [FN: 412, TP: 0]]\n",
      "Precision: 0.00\n",
      "Recall: 0.00\n",
      "F1-Score: 0.00\n",
      "------------------------------\n",
      "Fold 2:\n",
      "Confusion Matrix:\n",
      "[[TN: 2044, FP: 11],\n",
      " [FN: 397, TP: 0]]\n",
      "Precision: 0.00\n",
      "Recall: 0.00\n",
      "F1-Score: 0.00\n",
      "------------------------------\n",
      "Fold 3:\n",
      "Confusion Matrix:\n",
      "[[TN: 1975, FP: 53],\n",
      " [FN: 382, TP: 42]]\n",
      "Precision: 0.44\n",
      "Recall: 0.10\n",
      "F1-Score: 0.16\n",
      "------------------------------\n",
      "Fold 4:\n",
      "Confusion Matrix:\n",
      "[[TN: 1971, FP: 8],\n",
      " [FN: 473, TP: 0]]\n",
      "Precision: 0.00\n",
      "Recall: 0.00\n",
      "F1-Score: 0.00\n",
      "------------------------------\n",
      "Fold 5:\n",
      "Confusion Matrix:\n",
      "[[TN: 1595, FP: 447],\n",
      " [FN: 185, TP: 224]]\n",
      "Precision: 0.33\n",
      "Recall: 0.55\n",
      "F1-Score: 0.41\n",
      "------------------------------\n",
      "---------------------------- Average Metrics Across All Folds ----------------------------\n",
      "Average Precision: 0.16\n",
      "Average Recall: 0.13\n",
      "Average F1-Score: 0.12\n"
     ]
    }
   ],
   "source": [
    "# Determine features and targets of the incoming model\n",
    "target = 'left'\n",
    "X = np.array(df.drop(columns=target))  # Parameters\n",
    "Y = np.array(df[target])               # Output \n",
    "\n",
    "# Initialize KFold and model\n",
    "KF = KFold(n_splits=5, random_state=None, shuffle=True)\n",
    "classifier = SGDClassifier(random_state=42)   \n",
    "\n",
    "# Loop through each fold to train and validate model\n",
    "precisions = []\n",
    "recalls = []\n",
    "f1_scores = []\n",
    "fold = 1  # To track the current fold\n",
    "for (train_index, test_index) in KF.split(X):\n",
    "    # Split data into training and testing for the current fold\n",
    "    X_train, X_test = X[train_index], X[test_index]\n",
    "    Y_train, Y_test = Y[train_index], Y[test_index]\n",
    "    \n",
    "    # Train the model on the training data\n",
    "    classifier.fit(X_train, Y_train)\n",
    "    \n",
    "    # Predict on the test data\n",
    "    Y_pred = classifier.predict(X_test)\n",
    "    \n",
    "    # Calculate metrics\n",
    "    confusion = confusion_matrix(Y_test, Y_pred)\n",
    "    precision = precision_score(Y_test, Y_pred)  \n",
    "    recall = recall_score(Y_test, Y_pred)\n",
    "    f1 = f1_score(Y_test, Y_pred)\n",
    "    \n",
    "    # Append metrics for averaging later\n",
    "    precisions.append(precision)\n",
    "    recalls.append(recall)\n",
    "    f1_scores.append(f1)\n",
    "\n",
    "    # Display results for the current fold\n",
    "    print(f\"Fold {fold}:\")\n",
    "    print(\"Confusion Matrix:\")\n",
    "    print(f'[[TN: {confusion[0][0]}, FP: {confusion[0][1]}],\\n'\n",
    "          f' [FN: {confusion[1][0]}, TP: {confusion[1][1]}]]')\n",
    "    print(f\"Precision: {precision:.2f}\")\n",
    "    print(f\"Recall: {recall:.2f}\")\n",
    "    print(f\"F1-Score: {f1:.2f}\")\n",
    "    print(\"-\" * 30)\n",
    "\n",
    "    fold = fold + 1\n",
    "\n",
    "#------------------------------------------------------------\n",
    "# Calculate and display averages\n",
    "average_precision = np.mean(precisions)\n",
    "average_recall = np.mean(recalls)\n",
    "average_f1 = np.mean(f1_scores)\n",
    "\n",
    "print(\"---------------------------- Average Metrics Across All Folds ----------------------------\")\n",
    "print(f\"Average Precision: {average_precision:.2f}\")\n",
    "print(f\"Average Recall: {average_recall:.2f}\")\n",
    "print(f\"Average F1-Score: {average_f1:.2f}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}