{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import os\n",
    "os.chdir('../')\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 简介\n",
    "上一讲我们实现了一个简单二元分类器：LogisticRegression，但通常情况下，我们面对的更多是多分类器的问题，而二分类转多分类的通常做法也很朴素，一般分为两种：**one-vs-rest**以及**one-vs-one**。顾名思义，one-vs-rest将多类别中的其中一类作为正类，剩余其他所有类别作为负类，对于`n_class`类别的分类问题，需要构建$n\\_class$种分类器；而one-vs-one是指进行两两分类，这样将会构造$n\\_class*(n\\_class-1)/2$种分类器，由于实现思路很简单，就直接贴出代码，将多分类实现封装到`MultiClassWrapper`类，并放到`ml_models.wrapper_models`包"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from ml_models.linear_model import *\n",
    "from ml_models.wrapper_models import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#准备手写数据\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn import model_selection\n",
    "from sklearn import datasets\n",
    "digits = datasets.load_digits()\n",
    "data = digits['data']\n",
    "target = digits['target']\n",
    "X_train, X_test, y_train, y_test = model_selection.train_test_split(data, target, test_size=0.3,\n",
    "                                                                    random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#构建初始模型\n",
    "lr = LogisticRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ovr: 0.9492701335705958\n"
     ]
    }
   ],
   "source": [
    "#进行one-vs-rest训练并评估\n",
    "ovr = MultiClassWrapper(lr, mode='ovr')\n",
    "ovr.fit(X_train, y_train)\n",
    "\n",
    "y = ovr.predict(X_test)\n",
    "print('ovr:', f1_score(y_test, y, average='macro'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ovo: 0.959902103714483\n"
     ]
    }
   ],
   "source": [
    "#进行one-vs-one训练并评估\n",
    "ovo = MultiClassWrapper(lr, mode='ovo')\n",
    "ovo.fit(X_train, y_train)\n",
    "\n",
    "y = ovo.predict(X_test)\n",
    "print('ovo:', f1_score(y_test, y, average='macro'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### `MultiClassWrapper`类实现细节"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import threading\n",
    "import copy\n",
    "import numpy as np\n",
    "\n",
    "\"\"\"\n",
    "继承Thread,获取函数的返回值\n",
    "\"\"\"\n",
    "\n",
    "\n",
    "class MyThread(threading.Thread):\n",
    "    def __init__(self, target, args, kwargs, name=''):\n",
    "        threading.Thread.__init__(self)\n",
    "        self.name = name\n",
    "        self.target = target\n",
    "        self.args = args\n",
    "        self.kwargs = kwargs\n",
    "        self.result = self.target(*self.args, **self.kwargs)\n",
    "\n",
    "    def get_result(self):\n",
    "        try:\n",
    "            return self.result\n",
    "        except:\n",
    "            return None\n",
    "\n",
    "\n",
    "class MultiClassWrapper(object):\n",
    "    def __init__(self, base_classifier, mode='ovr'):\n",
    "        \"\"\"\n",
    "        :param base_classifier: 实例化后的分类器\n",
    "        :param mode: 'ovr'表示one-vs-rest方式,'ovo'表示one-vs-one方式\n",
    "        \"\"\"\n",
    "        self.base_classifier = base_classifier\n",
    "        self.mode = mode\n",
    "\n",
    "    @staticmethod\n",
    "    def fit_base_classifier(base_classifier, x, y, **kwargs):\n",
    "        base_classifier.fit(x, y, **kwargs)\n",
    "\n",
    "    @staticmethod\n",
    "    def predict_proba_base_classifier(base_classifier, x):\n",
    "        return base_classifier.predict_proba(x)\n",
    "\n",
    "    def fit(self, x, y, **kwargs):\n",
    "        # 对y分组并行fit\n",
    "        self.n_class = np.max(y)\n",
    "        if self.mode == 'ovr':\n",
    "            # 打包数据\n",
    "            self.classifiers = []\n",
    "\n",
    "            for cls in range(0, self.n_class + 1):\n",
    "                self.classifiers.append(copy.deepcopy(self.base_classifier))\n",
    "            # 并行训练\n",
    "            tasks = []\n",
    "            for cls in range(len(self.classifiers)):\n",
    "                task = MyThread(target=self.fit_base_classifier,\n",
    "                                args=(self.classifiers[cls], x, (y == cls).astype('int')), kwargs=kwargs)\n",
    "                task.start()\n",
    "                tasks.append(task)\n",
    "            for task in tasks:\n",
    "                task.join()\n",
    "        elif self.mode == \"ovo\":\n",
    "            # 打包数据\n",
    "            self.classifiers = {}\n",
    "            for first_cls in range(0, self.n_class):\n",
    "                for second_cls in range(first_cls + 1, self.n_class + 1):\n",
    "                    self.classifiers[(first_cls, second_cls)] = copy.deepcopy(self.base_classifier)\n",
    "            # 并行训练\n",
    "            tasks = {}\n",
    "            for first_cls in range(0, self.n_class):\n",
    "                for second_cls in range(first_cls + 1, self.n_class + 1):\n",
    "                    index = np.where(y == first_cls)[0].tolist() + np.where(y == second_cls)[0].tolist()\n",
    "                    new_x = x[index, :]\n",
    "                    new_y = y[index]\n",
    "                    task = MyThread(target=self.fit_base_classifier,\n",
    "                                    args=(self.classifiers[(first_cls, second_cls)], new_x,\n",
    "                                          (new_y == first_cls).astype('int')), kwargs=kwargs)\n",
    "                    task.start()\n",
    "                    tasks[(first_cls, second_cls)] = task\n",
    "            for first_cls in range(0, self.n_class):\n",
    "                for second_cls in range(first_cls + 1, self.n_class + 1):\n",
    "                    tasks[(first_cls, second_cls)].join()\n",
    "\n",
    "    def predict_proba(self, x, **kwargs):\n",
    "        if self.mode == 'ovr':\n",
    "            tasks = []\n",
    "            probas = []\n",
    "            for cls in range(len(self.classifiers)):\n",
    "                task = MyThread(target=self.predict_proba_base_classifier, args=(self.classifiers[cls], x),\n",
    "                                kwargs=kwargs)\n",
    "                task.start()\n",
    "                tasks.append(task)\n",
    "            for task in tasks:\n",
    "                task.join()\n",
    "            for task in tasks:\n",
    "                probas.append(task.get_result())\n",
    "            total_probas = np.concatenate(probas, axis=1)\n",
    "            # 归一化\n",
    "            return total_probas / total_probas.sum(axis=1, keepdims=True)\n",
    "        elif self.mode == 'ovo':\n",
    "            tasks = {}\n",
    "            probas = {}\n",
    "            for first_cls in range(0, self.n_class):\n",
    "                for second_cls in range(first_cls + 1, self.n_class + 1):\n",
    "                    task = MyThread(target=self.predict_proba_base_classifier,\n",
    "                                    args=(self.classifiers[(first_cls, second_cls)], x), kwargs=kwargs)\n",
    "                    task.start()\n",
    "                    tasks[(first_cls, second_cls)] = task\n",
    "            for first_cls in range(0, self.n_class):\n",
    "                for second_cls in range(first_cls + 1, self.n_class + 1):\n",
    "                    tasks[(first_cls, second_cls)].join()\n",
    "            for first_cls in range(0, self.n_class):\n",
    "                for second_cls in range(first_cls + 1, self.n_class + 1):\n",
    "                    probas[(first_cls, second_cls)] = tasks[(first_cls, second_cls)].get_result()\n",
    "                    probas[(second_cls, first_cls)] = 1.0 - probas[(first_cls, second_cls)]\n",
    "            # 统计概率\n",
    "            total_probas = []\n",
    "            for first_cls in range(0, self.n_class + 1):\n",
    "                temp = []\n",
    "                for second_cls in range(0, self.n_class + 1):\n",
    "                    if first_cls != second_cls:\n",
    "                        temp.append(probas[(first_cls, second_cls)])\n",
    "                temp = np.concatenate(temp, axis=1).sum(axis=1, keepdims=True)\n",
    "                total_probas.append(temp)\n",
    "            # 归一化\n",
    "            total_probas = np.concatenate(total_probas, axis=1)\n",
    "            return total_probas / total_probas.sum(axis=1, keepdims=True)\n",
    "\n",
    "    def predict(self, x):\n",
    "        return np.argmax(self.predict_proba(x), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}