{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from scipy.linalg import lstsq\n",
    "from copy import deepcopy\n",
    "from sklearn.base import BaseEstimator, RegressorMixin\n",
    "from sklearn.datasets import load_boston\n",
    "from sklearn.ensemble import BaggingRegressor as skBaggingRegressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def r2_score(y_true, y_pred):\n",
    "    numerator = np.sum((y_true - y_pred) ** 2)\n",
    "    denominator = np.sum((y_true - np.mean(y_true)) ** 2)\n",
    "    return 1 - numerator / denominator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LinearRegression(BaseEstimator, RegressorMixin):\n",
    "    def fit(self, X, y):\n",
    "        X_train = np.hstack((np.ones((X.shape[0], 1)), X))\n",
    "        coef, _, _, _ = lstsq(X_train, y)\n",
    "        self.coef_ = coef[1:]\n",
    "        self.intercept_ = coef[0]\n",
    "        return self\n",
    "\n",
    "    def predict(self, X):\n",
    "        y_pred = np.dot(X, self.coef_) + self.intercept_\n",
    "        return y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BaggingRegressor():\n",
    "    def __init__(self, base_estimator, n_estimators, oob_score, random_state):\n",
    "        self.base_estimator = base_estimator\n",
    "        self.n_estimators = n_estimators\n",
    "        self.oob_score = oob_score\n",
    "        self.random_state = random_state\n",
    "\n",
    "    def fit(self, X, y):\n",
    "        MAX_INT = np.iinfo(np.int32).max\n",
    "        rng = np.random.RandomState(self.random_state)\n",
    "        self._seeds = rng.randint(MAX_INT, size=self.n_estimators)\n",
    "        self.estimators_ = []\n",
    "        self.estimators_samples_ = []\n",
    "        for i in range(self.n_estimators):\n",
    "            est = deepcopy(self.base_estimator)\n",
    "            rng = np.random.RandomState(self._seeds[i])\n",
    "            sample_indices = rng.randint(0, X.shape[0], X.shape[0])\n",
    "            self.estimators_samples_.append(sample_indices)\n",
    "            est.fit(X[sample_indices], y[sample_indices])\n",
    "            self.estimators_.append(est)\n",
    "        if self.oob_score:\n",
    "            self._set_oob_score(X, y)\n",
    "        return self\n",
    "\n",
    "    def _set_oob_score(self, X, y):\n",
    "        predictions = np.zeros(X.shape[0])\n",
    "        n_predictions = np.zeros(X.shape[0])\n",
    "        for i in range(self.n_estimators):\n",
    "            mask = np.ones(X.shape[0], dtype=bool)\n",
    "            mask[self.estimators_samples_[i]] = False\n",
    "            predictions[mask] += self.estimators_[i].predict(X[mask])\n",
    "            n_predictions[mask] += 1\n",
    "        predictions /= n_predictions\n",
    "        self.oob_prediction_ = predictions\n",
    "        self.oob_score_ = r2_score(y, predictions)\n",
    "\n",
    "    def predict(self, X):\n",
    "        pred = np.zeros(X.shape[0])\n",
    "        for est in self.estimators_:\n",
    "            pred += est.predict(X)\n",
    "        pred /= self.n_estimators\n",
    "        return pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = load_boston(return_X_y=True)\n",
    "clf1 = BaggingRegressor(base_estimator=LinearRegression(),\n",
    "                        n_estimators=100, oob_score=True, random_state=0).fit(X, y)\n",
    "clf2 = skBaggingRegressor(base_estimator=LinearRegression(),\n",
    "                          n_estimators=100, oob_score=True, random_state=0).fit(X, y)\n",
    "assert np.allclose(clf1._seeds, clf2._seeds)\n",
    "assert np.array_equal(clf1.estimators_samples_, clf2.estimators_samples_)\n",
    "for i in range(clf1.n_estimators):\n",
    "    assert np.allclose(clf1.estimators_[i].coef_, clf2.estimators_[i].coef_)\n",
    "    assert np.allclose(clf1.estimators_[i].intercept_, clf2.estimators_[i].intercept_)\n",
    "assert np.allclose(clf1.oob_prediction_, clf2.oob_prediction_)\n",
    "assert np.allclose(clf1.oob_score_, clf2.oob_score_)\n",
    "pred1 = clf1.predict(X)\n",
    "pred2 = clf2.predict(X)\n",
    "assert np.allclose(pred1, pred2)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "dev",
   "language": "python",
   "name": "dev"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}