{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from scipy.special import expit, xlogy\n",
    "from scipy.optimize import minimize\n",
    "from sklearn.base import clone\n",
    "from sklearn.datasets import load_iris\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.calibration import CalibratedClassifierCV as skCalibratedClassifierCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CalibratedClassifierCV():\n",
    "    def __init__(self, base_estimator=LinearSVC(random_state=0)):\n",
    "        self.base_estimator = base_estimator\n",
    "\n",
    "    def fit(self, X, y):\n",
    "        self.classes_, y_train = np.unique(y, return_inverse=True)\n",
    "        self.calibrated_classifiers_ = []\n",
    "        cv = StratifiedKFold()\n",
    "        for train, test in cv.split(X, y):\n",
    "            this_estimator = clone(self.base_estimator)\n",
    "            this_estimator.fit(X[train], y_train[train])\n",
    "            calibrated_classifier = _CalibratedClassifier(this_estimator)\n",
    "            calibrated_classifier.fit(X[test], y_train[test])\n",
    "            self.calibrated_classifiers_.append(calibrated_classifier)\n",
    "        return self\n",
    "\n",
    "    def predict_proba(self, X):\n",
    "        proba = np.zeros((X.shape[0], len(self.classes_)))\n",
    "        for i in range(len(self.calibrated_classifiers_)):\n",
    "            proba += self.calibrated_classifiers_[i].predict_proba(X)\n",
    "        proba /= len(self.calibrated_classifiers_)\n",
    "        return proba\n",
    "\n",
    "    def predict(self, X):\n",
    "        proba = self.predict_proba(X)\n",
    "        return self.classes_[np.argmax(proba, axis=1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class _CalibratedClassifier():\n",
    "    def __init__(self, base_estimator=LinearSVC(random_state=0)):\n",
    "        self.base_estimator = base_estimator\n",
    "\n",
    "    def fit(self, X, y):\n",
    "        self.n_classes = len(np.unique(y))\n",
    "        y_train = np.zeros((y.shape[0], self.n_classes))\n",
    "        for i in range(self.n_classes):\n",
    "            y_train[y == i, i] = 1\n",
    "        y_prob = self.base_estimator.decision_function(X)\n",
    "        if self.n_classes == 2:\n",
    "            y_train = y_train[:, 1].reshape(-1, 1)\n",
    "            y_prob = y_prob.reshape(-1, 1)\n",
    "        self.calibrators_ = []\n",
    "        for i, cur_prob in enumerate(y_prob.T):\n",
    "            calibrator = _SigmoidCalibration()\n",
    "            calibrator.fit(cur_prob, y_train[:, i])\n",
    "            self.calibrators_.append(calibrator)\n",
    "        return self\n",
    "\n",
    "    def predict_proba(self, X):\n",
    "        prob = np.zeros((X.shape[0], self.n_classes))\n",
    "        y_prob = self.base_estimator.decision_function(X)\n",
    "        if self.n_classes == 2:\n",
    "            prob[:, 1] = self.calibrators_[0].predict(y_prob)\n",
    "            prob[:, 0] = 1 - prob[:, 1]\n",
    "        else:\n",
    "            for i, cur_prob in enumerate(y_prob.T):\n",
    "                prob[:, i] = self.calibrators_[i].predict(cur_prob)\n",
    "            prob /= np.sum(prob, axis=1)[:, np.newaxis]\n",
    "        return prob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class _SigmoidCalibration():\n",
    "    @staticmethod\n",
    "    def _cost_grad(AB, X, y):\n",
    "        P = expit(-(AB[0] * X + AB[1]))\n",
    "        cost = np.sum(-(xlogy(y, P) + xlogy(1 - y, 1 - P)))\n",
    "        dA = np.dot(y - P, X)\n",
    "        dB = np.sum(y - P)\n",
    "        grad = np.array([dA, dB])\n",
    "        return cost, grad\n",
    "\n",
    "    def fit(self, X, y):\n",
    "        prior0 = np.sum(y == 0)\n",
    "        prior1 = y.shape[0] - prior0\n",
    "        y_train = np.zeros(y.shape[0])\n",
    "        y_train[y == 1] = (prior1 + 1) / (prior1 + 2)\n",
    "        y_train[y == 0] = 1 / (prior0 + 2)\n",
    "        AB0 = np.array([0, np.log((prior0 + 1) / (prior1 + 1))])\n",
    "        res = minimize(fun=self._cost_grad, jac=True, x0=AB0,\n",
    "                       args=(X, y_train), method='L-BFGS-B')\n",
    "        self.a_, self.b_ = res.x[0], res.x[1]\n",
    "        return self\n",
    "        \n",
    "    def predict(self, X):\n",
    "        return expit(-(self.a_ * X + self.b_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = load_iris(return_X_y=True)\n",
    "X, y = X[y != 2], y[y != 2]\n",
    "clf1 = CalibratedClassifierCV(base_estimator=LinearSVC(max_iter=10000, random_state=0)).fit(X, y)\n",
    "clf2 = skCalibratedClassifierCV(base_estimator=LinearSVC(max_iter=10000, random_state=0)).fit(X, y)\n",
    "prob1 = clf1.predict_proba(X)\n",
    "prob2 = clf2.predict_proba(X)\n",
    "assert np.allclose(prob1, prob2)\n",
    "pred1 = clf1.predict(X)\n",
    "pred2 = clf2.predict(X)\n",
    "assert np.allclose(pred1, pred2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = load_iris(return_X_y=True)\n",
    "X, y = X[y != 2], y[y != 2]\n",
    "clf1 = CalibratedClassifierCV(base_estimator=LogisticRegression(max_iter=10000)).fit(X, y)\n",
    "clf2 = skCalibratedClassifierCV(base_estimator=LogisticRegression(max_iter=10000)).fit(X, y)\n",
    "prob1 = clf1.predict_proba(X)\n",
    "prob2 = clf2.predict_proba(X)\n",
    "assert np.allclose(prob1, prob2, atol=1e-5)\n",
    "pred1 = clf1.predict(X)\n",
    "pred2 = clf2.predict(X)\n",
    "assert np.allclose(pred1, pred2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = load_iris(return_X_y=True)\n",
    "clf1 = CalibratedClassifierCV(base_estimator=LinearSVC(max_iter=10000, random_state=0)).fit(X, y)\n",
    "clf2 = skCalibratedClassifierCV(base_estimator=LinearSVC(max_iter=10000, random_state=0)).fit(X, y)\n",
    "prob1 = clf1.predict_proba(X)\n",
    "prob2 = clf2.predict_proba(X)\n",
    "assert np.allclose(prob1, prob2)\n",
    "pred1 = clf1.predict(X)\n",
    "pred2 = clf2.predict(X)\n",
    "assert np.allclose(pred1, pred2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = load_iris(return_X_y=True)\n",
    "clf1 = CalibratedClassifierCV(base_estimator=LogisticRegression(max_iter=10000)).fit(X, y)\n",
    "clf2 = skCalibratedClassifierCV(base_estimator=LogisticRegression(max_iter=10000)).fit(X, y)\n",
    "prob1 = clf1.predict_proba(X)\n",
    "prob2 = clf2.predict_proba(X)\n",
    "assert np.allclose(prob1, prob2)\n",
    "pred1 = clf1.predict(X)\n",
    "pred2 = clf2.predict(X)\n",
    "assert np.allclose(pred1, pred2)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "dev",
   "language": "python",
   "name": "dev"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}