{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from scipy.optimize import minimize\n",
    "from sklearn.datasets import load_iris\n",
    "from sklearn.svm import LinearSVC as skLinearSVC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LinearSVC():\n",
    "    def __init__(self, C=1.0):\n",
    "        self.C = C\n",
    "\n",
    "    def _encode(self, y):\n",
    "        classes = np.unique(y)\n",
    "        y_train = np.full((y.shape[0], len(classes)), -1)\n",
    "        for i, c in enumerate(classes):\n",
    "            y_train[y == c, i] = 1\n",
    "        if len(classes) == 2:\n",
    "            y_train = y_train[:, 1].reshape(-1, 1)\n",
    "        return classes, y_train\n",
    "\n",
    "    @staticmethod\n",
    "    def _cost_grad(w, X, y, C):\n",
    "        X_train = np.c_[X, np.ones(X.shape[0])]\n",
    "        z = np.dot(X_train, w)\n",
    "        yz = y * z\n",
    "        mask = yz <= 1\n",
    "        cost = C * np.sum(np.square(1 - yz[mask])) + 0.5 * np.dot(w, w)\n",
    "        grad = w + 2 * C * np.dot(X_train[mask].T, z[mask] - y[mask])\n",
    "        return cost, grad\n",
    "        \n",
    "    def _solve_lbfgs(self, X, y):\n",
    "        result = np.zeros((y.shape[1], X.shape[1] + 1))\n",
    "        for i in range(y.shape[1]):\n",
    "            cur_y = y[:, i]\n",
    "            w0 = np.zeros(X.shape[1] + 1)\n",
    "            res = minimize(fun=self._cost_grad, jac=True, x0=w0,\n",
    "                           args=(X, cur_y, self.C), method='L-BFGS-B')\n",
    "            result[i] = res.x\n",
    "        return result[:, :-1], result[:, -1]\n",
    "\n",
    "    def fit(self, X, y):\n",
    "        self.classes_, y_train = self._encode(y)\n",
    "        self.coef_, self.intercept_ = self._solve_lbfgs(X, y_train)\n",
    "        return self\n",
    "\n",
    "    def decision_function(self, X):\n",
    "        scores = np.dot(X, self.coef_.T) + self.intercept_\n",
    "        if scores.shape[1] == 1:\n",
    "            return scores.ravel()\n",
    "        else:\n",
    "            return scores\n",
    "\n",
    "    def predict(self, X):\n",
    "        scores = self.decision_function(X)\n",
    "        if len(scores.shape) == 1:\n",
    "            indices = (scores > 0).astype(int)\n",
    "        else:\n",
    "            indices = np.argmax(scores, axis=1)\n",
    "        return self.classes_[indices]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = load_iris(return_X_y=True)\n",
    "X, y = X[y != 2], y[y != 2]\n",
    "clf1 = LinearSVC().fit(X, y)\n",
    "clf2 = skLinearSVC(dual=False).fit(X, y)\n",
    "assert np.allclose(clf1.coef_, clf2.coef_, atol=1e-2)\n",
    "assert np.allclose(clf1.intercept_, clf2.intercept_, atol=1e-3)\n",
    "prob1 = clf1.decision_function(X)\n",
    "prob2 = clf2.decision_function(X)\n",
    "assert np.allclose(prob1, prob2, atol=1e-2)\n",
    "pred1 = clf1.predict(X)\n",
    "pred2 = clf2.predict(X)\n",
    "assert np.array_equal(pred1, pred2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = load_iris(return_X_y=True)\n",
    "clf1 = LinearSVC().fit(X, y)\n",
    "clf2 = skLinearSVC(dual=False).fit(X, y)\n",
    "assert np.allclose(clf1.coef_, clf2.coef_, atol=1e-1)\n",
    "assert np.allclose(clf1.intercept_, clf2.intercept_, atol=1e-2)\n",
    "prob1 = clf1.decision_function(X)\n",
    "prob2 = clf2.decision_function(X)\n",
    "assert np.allclose(prob1, prob2, atol=1e-1)\n",
    "pred1 = clf1.predict(X)\n",
    "pred2 = clf2.predict(X)\n",
    "assert np.array_equal(pred1, pred2)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "dev",
   "language": "python",
   "name": "dev"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}