{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Support vector machines"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Pros: Low generalization error, computationally inexpensive,\n",
    "    easy to interpret results\n",
    "- Cons: Sensitive to tuning parameters and kernel choice;\n",
    "    natively only handles binary classification\n",
    "- Works with: Numeric values, nominal values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from os import listdir\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Implement SVM (without kernel)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Helper functions for the SMO algorithm\n",
    "def selectJrand(i, m):\n",
    "    j = i\n",
    "    while j == i:\n",
    "        j = np.random.randint(0, m)\n",
    "    return j\n",
    "\n",
    "\n",
    "def clipAlpha(aj, H, L):\n",
    "    if aj > H:\n",
    "        aj = H\n",
    "    if aj < L:\n",
    "        aj = L\n",
    "    return aj"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The simplified SMO algorithm\n",
    "# Create an alphas vector filled with 0s\n",
    "# While the number of iterations is less than MaxIterations:\n",
    "#     For every data vector in the dataset:\n",
    "#         If the data vector can be optimized:\n",
    "#             Select another data vector at random\n",
    "#             Optimize the two vectors together\n",
    "#             If the vectors can’t be optimized -> break\n",
    "#     If no vectors were optimized ➞ increment the iteration count\n",
    "def smoSimple(dataMatrix, labelMat, C, toler, maxIter):\n",
    "    m, n = dataMatrix.shape\n",
    "    alphas = np.zeros(m)\n",
    "    b = 0\n",
    "    n_iter = 0\n",
    "    while n_iter < maxIter:\n",
    "        alphaPairsChanged = 0\n",
    "        for i in range(m):\n",
    "            fXi = np.dot(np.multiply(alphas, labelMat),\n",
    "                         np.dot(dataMatrix, dataMatrix[i])) + b\n",
    "            Ei = fXi - labelMat[i]\n",
    "            # Enter optimization if alphas can be changed (KKT conditions)\n",
    "            if ((labelMat[i] * Ei < -toler and alphas[i] < C)\n",
    "                    or (labelMat[i] * Ei > toler and alphas[i] > 0)):\n",
    "                # Randomly select second alpha\n",
    "                j = selectJrand(i, m)\n",
    "                fXj = np.dot(np.multiply(alphas, labelMat),\n",
    "                             np.dot(dataMatrix, dataMatrix[j])) + b\n",
    "                Ej = fXj - labelMat[j]\n",
    "                alphaIold = alphas[i]\n",
    "                alphaJold = alphas[j]\n",
    "                # Guarantee alphas stay between 0 and C\n",
    "                if labelMat[i] != labelMat[j]:\n",
    "                    L = max(0, alphas[j] - alphas[i])\n",
    "                    H = min(C, C + alphas[j] - alphas[i])\n",
    "                else:\n",
    "                    L = max(0, alphas[j] + alphas[i] - C)\n",
    "                    H = min(C, alphas[j] + alphas[i])\n",
    "                if L == H:\n",
    "                    # print(\"L == H\")\n",
    "                    continue\n",
    "                eta = (2.0 * np.dot(dataMatrix[i], dataMatrix[j])\n",
    "                       - np.dot(dataMatrix[i], dataMatrix[i])\n",
    "                       - np.dot(dataMatrix[j], dataMatrix[j]))\n",
    "                if eta >= 0:\n",
    "                    # print(\"eta >= 0\")\n",
    "                    continue\n",
    "                alphas[j] -= labelMat[j] * (Ei - Ej) / eta\n",
    "                alphas[j] = clipAlpha(alphas[j], H, L)\n",
    "                if abs(alphas[j] - alphaJold) < 0.00001:\n",
    "                    # print(\"j not moving enough\")\n",
    "                    continue\n",
    "                # Update i by same amount as j in opposite direction\n",
    "                alphas[i] += (labelMat[j] * labelMat[i]\n",
    "                              * (alphaJold - alphas[j]))\n",
    "                # Set the constant term\n",
    "                b1 = (b - Ei\n",
    "                      - labelMat[i] * (alphas[i] - alphaIold)\n",
    "                      * np.dot(dataMatrix[i], dataMatrix[i])\n",
    "                      - labelMat[j] * (alphas[j] - alphaJold)\n",
    "                      * np.dot(dataMatrix[i], dataMatrix[j]))\n",
    "                b2 = (b - Ej\n",
    "                      - labelMat[i] * (alphas[i] - alphaIold)\n",
    "                      * np.dot(dataMatrix[i], dataMatrix[j])\n",
    "                      - labelMat[j] * (alphas[j] - alphaJold)\n",
    "                      * np.dot(dataMatrix[j], dataMatrix[j]))\n",
    "                if 0 < alphas[i] < C:\n",
    "                    b = b1\n",
    "                elif 0 < alphas[j] < C:\n",
    "                    b = b2\n",
    "                else:\n",
    "                    b = (b1 + b2) / 2\n",
    "                alphaPairsChanged += 1\n",
    "                # print(\"iter: %d i:%d, pairs changed %d\"\n",
    "                #       % (n_iter, i, alphaPairsChanged))\n",
    "        if alphaPairsChanged == 0:\n",
    "            n_iter += 1\n",
    "        else:\n",
    "            n_iter = 0\n",
    "        # print(\"iteration number: %d\" % n_iter)\n",
    "    return b, alphas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def calcWs(alphas, dataArr, labelMat):\n",
    "    m, n = dataArr.shape\n",
    "    w = np.zeros(n)\n",
    "    for i in range(m):\n",
    "        w += alphas[i] * labelMat[i] * dataArr[i]\n",
    "    return w"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Support functions for full Platt SMO\n",
    "class optStruct:\n",
    "    def __init__(self, dataMatIn, classLabels, C, toler):\n",
    "        self.X = dataMatIn\n",
    "        self.labelMat = classLabels\n",
    "        self.C = C\n",
    "        self.tol = toler\n",
    "        self.m = dataMatIn.shape[0]\n",
    "        self.alphas = np.zeros(self.m)\n",
    "        self.b = 0\n",
    "        # Error cache\n",
    "        # first column is a flag bit stating whether the eCache is valid\n",
    "        self.eCache = np.zeros((self.m, 2))\n",
    "\n",
    "\n",
    "def calcEk(oS, k):\n",
    "    fXk = np.dot(np.multiply(oS.alphas, oS.labelMat),\n",
    "                 np.dot(oS.X, oS.X[k])) + oS.b\n",
    "    Ek = fXk - oS.labelMat[k]\n",
    "    return Ek\n",
    "\n",
    "\n",
    "# Inner-loop heuristic\n",
    "def selectJ(i, oS, Ei):\n",
    "    maxK = -1\n",
    "    maxDeltaE = 0\n",
    "    Ej = 0\n",
    "    oS.eCache[i] = [1, Ei]\n",
    "    validEcacheList = np.nonzero(oS.eCache[:, 0])[0]\n",
    "    if len(validEcacheList) > 1:\n",
    "        for k in validEcacheList:\n",
    "            if k == i:\n",
    "                continue\n",
    "            Ek = calcEk(oS, k)\n",
    "            deltaE = abs(Ei - Ek)\n",
    "            # Choose j for maximum step size\n",
    "            if deltaE > maxDeltaE:\n",
    "                maxK = k\n",
    "                maxDeltaE = deltaE\n",
    "                Ej = Ek\n",
    "        return maxK, Ej\n",
    "    # If this is your first time through the loop, you randomly select an alpha\n",
    "    else:\n",
    "        j = selectJrand(i, oS.m)\n",
    "        Ej = calcEk(oS, j)\n",
    "    return j, Ej\n",
    "\n",
    "\n",
    "def updateEk(oS, k):\n",
    "    Ek = calcEk(oS, k)\n",
    "    oS.eCache[k] = [1, Ek]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Full Platt SMO optimization routine\n",
    "def innerL(i, oS):\n",
    "    Ei = calcEk(oS, i)\n",
    "    if ((oS.labelMat[i] * Ei < -oS.tol and oS.alphas[i] < oS.C)\n",
    "            or (oS.labelMat[i] * Ei > oS.tol and oS.alphas[i] > 0)):\n",
    "        # Second-choice heuristic\n",
    "        j, Ej = selectJ(i, oS, Ei)\n",
    "        alphaIold = oS.alphas[i]\n",
    "        alphaJold = oS.alphas[j]\n",
    "        if oS.labelMat[i] != oS.labelMat[j]:\n",
    "            L = max(0, oS.alphas[j] - oS.alphas[i])\n",
    "            H = min(oS.C, oS.C + oS.alphas[j] - oS.alphas[i])\n",
    "        else:\n",
    "            L = max(0, oS.alphas[j] + oS.alphas[i] - oS.C)\n",
    "            H = min(oS.C, oS.alphas[j] + oS.alphas[i])\n",
    "        if L == H:\n",
    "            # print(\"L == H\")\n",
    "            return 0\n",
    "        eta = (2.0 * np.dot(oS.X[i], oS.X[j])\n",
    "               - np.dot(oS.X[i], oS.X[i])\n",
    "               - np.dot(oS.X[j], oS.X[j]))\n",
    "        if eta >= 0:\n",
    "            # print(\"eta >= 0\")\n",
    "            return 0\n",
    "        oS.alphas[j] -= oS.labelMat[j] * (Ei - Ej) / eta\n",
    "        oS.alphas[j] = clipAlpha(oS.alphas[j], H, L)\n",
    "        # Updates Ecache\n",
    "        updateEk(oS, j)\n",
    "        if abs(oS.alphas[j] - alphaJold) < 0.00001:\n",
    "            # print(\"j not moving enough\")\n",
    "            return 0\n",
    "        oS.alphas[i] += (oS.labelMat[j] * oS.labelMat[i]\n",
    "                         * (alphaJold - oS.alphas[j]))\n",
    "        # Updates Ecache\n",
    "        updateEk(oS, i)\n",
    "        b1 = (oS.b - Ei\n",
    "              - oS.labelMat[i] * (oS.alphas[i] - alphaIold)\n",
    "              * np.dot(oS.X[i], oS.X[i])\n",
    "              - oS.labelMat[j] * (oS.alphas[j] - alphaJold)\n",
    "              * np.dot(oS.X[i], oS.X[j]))\n",
    "        b2 = (oS.b - Ej\n",
    "              - oS.labelMat[i] * (oS.alphas[i] - alphaIold)\n",
    "              * np.dot(oS.X[i], oS.X[j])\n",
    "              - oS.labelMat[j] * (oS.alphas[j] - alphaJold)\n",
    "              * np.dot(oS.X[j], oS.X[j]))\n",
    "        if 0 < oS.alphas[i] < oS.C:\n",
    "            oS.b = b1\n",
    "        elif 0 < oS.alphas[j] < oS.C:\n",
    "            oS.b = b2\n",
    "        else:\n",
    "            oS.b = (b1 + b2) / 2\n",
    "        return 1\n",
    "    else:\n",
    "        return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Full Platt SMO outer loop\n",
    "def smoP(dataMatIn, classLabels, C, toler, maxIter):\n",
    "    oS = optStruct(dataMatIn, classLabels, C, toler)\n",
    "    n_iter = 0\n",
    "    entireSet = True\n",
    "    alphaPairsChanged = 0\n",
    "    while n_iter < maxIter and (alphaPairsChanged > 0 or entireSet):\n",
    "        alphaPairsChanged = 0\n",
    "        # Go over all values\n",
    "        if entireSet:\n",
    "            for i in range(oS.m):\n",
    "                alphaPairsChanged += innerL(i, oS)\n",
    "                # print(\"fullSet, iter: %d i:%d, pairs changed %d\"\n",
    "                #       % (n_iter, i, alphaPairsChanged))\n",
    "            n_iter += 1\n",
    "        # Go over non-bound values\n",
    "        else:\n",
    "            nonBoundIs = np.nonzero((oS.alphas > 0) * (oS.alphas < C))[0]\n",
    "            for i in nonBoundIs:\n",
    "                alphaPairsChanged += innerL(i, oS)\n",
    "                # print(\"non-bound, iter: %d i:%d, pairs changed %d\"\n",
    "                #       % (n_iter, i, alphaPairsChanged))\n",
    "            n_iter += 1\n",
    "        if entireSet:\n",
    "            entireSet = False\n",
    "        elif alphaPairsChanged == 0:\n",
    "            entireSet = True\n",
    "        print(\"iteration number: %d\" % n_iter)\n",
    "    return oS.b, oS.alphas"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### Experiment 1: Toy dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def loadDataSet(fileName):\n",
    "    dataMat, labelMat = [], []\n",
    "    fr = open(fileName)\n",
    "    for line in fr.readlines():\n",
    "        lineArr = line.strip().split('\\t')\n",
    "        dataMat.append([float(lineArr[0]), float(lineArr[1])])\n",
    "        labelMat.append(float(lineArr[2]))\n",
    "    return np.array(dataMat), np.array(labelMat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# The simplified SMO algorithm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dataArr, labelArr = loadDataSet('testSet.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "b, alphas = smoSimple(dataArr, labelArr, 0.6, 0.001, 40)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-3.8120029344585844\n"
     ]
    }
   ],
   "source": [
    "print(b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.80774759 -0.23769058]\n"
     ]
    }
   ],
   "source": [
    "w = calcWs(alphas, dataArr, labelArr)\n",
    "print(w)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[4.658191 3.507396] -1.0\n",
      "[ 3.457096 -0.082216] -1.0\n",
      "[ 5.286862 -2.358286] 1.0\n",
      "[6.080573 0.418886] 1.0\n"
     ]
    }
   ],
   "source": [
    "# support vectors\n",
    "for i in range(len(alphas)):\n",
    "    if alphas[i] > 0:\n",
    "        print(dataArr[i], labelArr[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXkAAAD8CAYAAACSCdTiAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3X10VPW5L/DvMyGIibzkRkWFvBCv\nKB48RYlVD1a9xasUbSiNFCnUt2IE5E1Je7T03tuzkHU9xwlJSBDEKFSTBVfEAymreoRTvWopXINo\nocU3UpJQA4IQUSMGmOf+MRONYd5n7/nt2fP9rDVrkpk9+/dA9n72b35vW1QVRETkTh7TARARkX2Y\n5ImIXIxJnojIxZjkiYhcjEmeiMjFmOSJiFzMkiQvIoNE5HkReVdE9ojINVbsl4iIEtPHov1UA3hJ\nVW8Tkb4AsizaLxERJUASnQwlIgMAvAOgSDmziojIUayoyRcBOARglYh8B8AOAPNU9YueG4lIGYAy\nAMjOzh59ySWXWFA0kTudPHkSu3btQk5ODgoLC02HQw6xY8eOw6p6TiyfsaImXwxgG4AxqrpdRKoB\nHFPV/xHqM8XFxdrU1JRQuURutnjxYvz617/Grl27MHLkSNPhkEOIyA5VLY7lM1Z0vO4HsF9Vtwd+\nfx7AFRbslygtHT9+HDU1NRg3bhwTPCUs4SSvqgcAtInIxYGXxgL4a6L7JUpXDQ0NOHjwIMrLy02H\nQi5g1eiaOQAaAiNrmgHcbdF+idKKz+dDRUUFRo0ahe9///umwyEXsCTJq+rbAGJqJyKi07344ovY\ns2cP6uvrISKmwyEX4IxXIgfxer0YOnQofvKTn5gOhVyCSZ7IIZqamvDqq69i/vz5yMzMNB0OuQST\nPJFDVFRUYMCAAbj33ntNh0IuwiRP5AD79u3DunXrUFZWhgEDBpgOh1yESZ7IAaqrqyEimDt3rulQ\nyGWY5IkMO3r0KJ588kncfvvtyMvLMx0OuQyTPJFhK1euxBdffIEFCxaYDoVciEmeyKCuri5UV1fj\nxhtvxKhRo0yHQy5k1YxXIorDmjVr0N7ejlWrVpkOhVyKNXkiQ1QVXq8XI0eOxE033WQ6HHIp1uSJ\nDHn55Zexe/durF69mksYkG1YkycyxOv14oILLsCUKVNMh0IuxiRPZMDbb7+NLVu2YO7cuejbt6/p\ncMjFmOSJDKioqMBZZ52F++67z3Qo5HJM8kRJ1tbWhrVr12L69OkYNGiQ6XDI5ZjkiZJs6dKlUFXM\nmzfPdCiUBpjkiZLo008/xRNPPIFJkyahsLDQdDiUBpjkiZKorq4On332GZcwoKRhkidKkhMnTqCq\nqgo33HADiot5t0xKDk6GIkqS5557Dvv378eKFStMh0JphDV5oiToXsJgxIgR+MEPfmA6HEojrMkT\nJcEf/vAHvP3226irq4PHw7oVJY9lR5uIZIjIThHZZNU+idzC6/Vi8ODBmDp1qulQKM1YWaWYB2CP\nhfsjt2toAAoLAY/H/9zQYDoiW+zevRsvvfQS5syZg379+pkOh9KMJUleRIYCuAVAnRX7ozTQ0ACU\nlQEtLYCq/7mszJWJvqKiAllZWZgxY4bpUCgNWVWTrwLwSwC+UBuISJmINIlI06FDhywqllLWwoVA\nZ+e3X+vs9L/uIh999BEaGhpwzz33IDc313Q4lIYSTvIiciuAj1V1R7jtVHWlqharavE555yTaLGU\n6lpbY3s9RdXU1ODUqVOYP3++6VAoTVkxumYMgBIRGQ+gH4ABIlKvqtMs2De5VX6+v4km2Os2aG1t\nRWNjIzo6OjBo0CBMmDABeXl5tpTV7bPPPsOKFSvw4x//GBdeeKGtZRGFpKqWPQDcAGBTpO1Gjx6t\nlObq61WzslT9LfL+R1aW/3ULtbe3a2lpqXo8HgXw9cPj8Whpaam2t7dbWl5PVVVVCkC3bdtmWxmU\nXgA0aYx5mQN2yYypU4GVK4GCAkDE/7xypf91ixw4cABjxozB+vXrkeHzoRTArwCUAvD4fFi/fj3G\njBmDgwcPWlZmt5MnT6KyshLXXnstrrrqKsv3TxQtSydDqeqrAF61cp/kYlOnWprUe5s9ezaam5tx\nBYCNAIb2eG8/gAkA3mpuxv3334/nn3/e0rLXr1+PlpYWVFdXW7pfoliJ/xtAchUXF2tTU1PSy6X0\n0draimHDhiHD50Mzvp3gu7UBKALg83iwb98+y9roVRVXXnkljh07hnfffZczXMkyIrJDVWNa3Y5H\nH7lSY2MjfD4fShA8wQNAHvy1eZ/Ph8bGRsvKfu2117Bjxw4sWLCACZ6M4xFIrtTR0QEAuDjCdsMD\nz0ePHrWsbK/Xi7PPPht33HGHZfskiheTPLlS971T34uw3fuB55ycHEvK3bNnDzZt2oTZs2fjzDPP\ntGSfRIlgkidXKikpgcfjQSP8nazBtMHfIevxeFBSUmJJuUuWLEG/fv0wa9YsS/ZHlCgmeXKl/Px8\nTJw4ESfgb3dv6/V+G4AfATgJYOLEiZZ0uh44cADPPPMM7rrrLnBWNzkFkzy5Vm1tLYqKivAW/KNo\nboN/nPxtgd/fAlBUVIRly5ZZUt6yZctw4sQJPPDAA5bsz63SZPFRx2CSJ9c677zz8Mc//hGlpaXw\neTxYD+B/A1gP/7DJ0tJSbN26FYMHD064rC+++AKPP/44JkyYgOHDh0f+QJpKo8VHHYPj5CkttLW1\nobGxEUePHkVOTg5KSkosXbtm2bJlmD17Nt544w2MGTPGsv26TWFh8CWLCgqAffuSHU3qiWecPJM8\nUYJOnTqF4cOH49xzz8XWrVshIqZDciyPx1+D700E8IVcqJy6cTIUkQEbNmxAc3MzysvLmeB7CNb2\nHmqRUZsWHyUwyRMlRFXx2GOPoaioCD/60Y9Mh+MYodrex48HsrK+vW1WFrB4sZk40wGTPFECtm7d\niu3bt+PBBx9ERkaG6XAcI9SNv37/e9sXH6Ve2CZPlICJEyfitddeQ2trK7Kzs02H4xhse7cH2+TJ\nrDQbAP3+++9j48aNmDVrFhN8L2x7dw4mebJGGg6ArqysRGZmJmbPnm06FMdZvJht707BJE/WCNUI\nu3ChmXhsdujQIaxevRp33HGHJZOp3CYJN/6iKFl6ZyhKY62tsb2e4h5//HEcP34cDz74oOlQHMvm\nG39RlFiTJ2ukUSPsl19+idraWtx6660YMWKE6XCIwmKSJ2ukUSPsM888g8OHD6O8vNx0KEQRMcmT\nNRJthE2RkTk+nw8VFRUoLi7GddddZzocooiY5Mk6U6f6V5ny+fzPsST4FBmZ87vf/Q4ffPABlzBI\nghS57jtewkleRPJE5BUR2SMifxGReVYERmkkhUbmeL1eFBQUoLS01HQoKSOeZJ1C133Hs6ImfxLA\nAlUdAeBqAPeLyKUW7JfcrvvsD7b2LOC4kTnbtm3DG2+8gQceeAB9+nBgWjTiTdYpdN13vISTvKq2\nq+pbgZ8/A7AHwJBE90su1/PsD8VhI3MqKiowaNAg3HPPPaZDSRnxJus0G5FrK0vb5EWkEMDlALYH\nea9MRJpEpOnQoUNWFkupKNjZ35PDRubs3bsXL7zwAmbMmIH+/fubDidlxJus02hEru0sS/Iichb8\nd1abr6rHer+vqitVtVhVi3mTYwp7ljtwemRVVRUyMjIwZ84c06GklHiTdRqNyLWdJUleRDLhT/AN\nqvqCFfskh7BriEMKVck++eQTPP3005g6dSouuOAC0+GklHiTNZdFsJCqJvQAIACeAVAV7WdGjx6t\nlALq61WzslT9fWb+R1aW/3U79m1HORZ45JFHFIDu2rXLdCgpqb5etaBAVcT/7JA/a0oC0KSx5uhY\nP3DaDoBrASiAPwN4O/AYH+4zTPIO1vOMzMgInoALCqwtK1Sit6qcBHz55Zc6ePBgHTdunOlQUgIT\nur2MJPl4HkzyDhWpdt39ELG2XJHklBOHuro6BaBbtmwxHYrj9E7oM2cGP3xyc5Ob7N18oYknyfPO\nUPSNcGPWeyoo8M9otbtcq8uJkc/nw8iRI3HGGWfgrbfe4gzXHrpHwPYcICUS/G5QgL8dPhlt6sHi\nSlbZycA7Q1Fk4TpSo0nw8QxxiNR569ChFC+++CL27NnDJQyCCDYCNlx9MVkTmUKNy5+XzvPwY636\nW/Fgc40h4TpS6+tDN5tkZMT/3TfazlsHfse+4YYbdOjQodrV1WU6FMcJdajY0coXy6ERLi4HHFIJ\nA9vkKaxQnZwFBaHfE0ns7AhXpoO9+eabCkC9Xq/pUBynvj50n3y4JBvPnzzWAV4O78dPGJM8hReu\ngzPc2WlXmQ52++2364ABA/TTTz81HYqjhOubz8ryd77m5gZ/L566Qqx1hPr60Iexww+5qMST5Nkm\nn07CTT8M9V5BgX1lOtS+ffuwbt06lJWVYcCAAabDcZRQq1FkZPg7Nx9/HDh8GKivt2YiU6zLIkyd\nCuTmBn/PwYecrZjk00m4Dk67Oj8d2qkaTnV1NUQEc+fONR2K44RKrj7ft5N4vLcW6C2eOkJ1dcod\ncvaKtepvxYPNNQaF68Wyq/PTgZ2qoRw5ckSzs7N12rRppkNxpGR3scQ76TqFDrmYgG3yRIl59NFH\nFYDu3LnTdCiOZNdKFybqHqmISZ7skSZn2VdffaXnn3++3njjjaZDcZyeh0Burv9h1eFg5xJJbhNP\nkmebPIWXRvdhW7NmDdrb21FeXm46FEfpfQh88glw5Ej4yU+x4F2g7MUkT6frOUP1zjtjPwNT8A7M\nqgqv14uRI0fipptuMh2Oo4Sb3WrFNZ93gbIXkzx9W+9q26lTwbcLdQamaM3/5Zdfxu7du7mEQRCR\nkm1nJzBtWvzX8xQcZZtSmOTdJtFadKTb8nULdQam6Hdvr9eLCy64AFOmTLG1nNbWVtTW1uKRRx5B\nbW0t2trabC3PCtEm23iv5yk4yja1xNqIb8WDHa82saIHK5pFSMLtMwVnuO7cuVMB6KOPPmpbGe3t\n7VpaWqoej0cBfP3weDxaWlqq7e3ttpWdqGhXoE50+YI06NtPGDi6Js2FGsQc7QJj4RYp6/kIt48U\nXKtm2rRpetZZZ+nRo0dt2X97e7sWFRUFEnumAqUK/Crw3EcBaFFRkR44cMCW8uMVbERNNEnewdfz\nlMckn+6iSdDxru4UbbJOsfFwra2t2qdPH50/f75tZZSWlgYS/BUKtPX6L20LvA4tLS391udM1m5D\n/RntqslTdJjk0100STrcWRjpIhFtsk6h797l5eWakZGhf/vb32zZf0tLS6CJJjNIgu9+tCrQRz0e\nj7a2tqqq+WtluC+F4Q6RzMz4YkyhQ8YoJvl0l8jt+8KtH9t9YYjlzEuBs7ajo0P79++vt99+u21l\n1NTUBGrxpRH+LP7afm1traqab/UKd73PzAz9Xt++9t1ygOJL8hxd43SxjpY588xvfvaE+PP2Hi7R\nPewx2HDJrCz/koKxrDKVIsMo6+rq8Nlnn2HBggW2ldHR0RH46eIIWw4HABw9ehSAtWPHQx1C4Q6t\nUCNqcnP9K0uG0tUV+0CqFB2QlTpivSpY8WBNPkqxVHGCbdu37+nVrmCfD/fdPBmLgBvQ1dWlQ4cO\n1RtuuMHWckzX5EMdQsFuut3z0KivP/3QycyMrvM11o7XFByQZQzYXOMysZzpobbNzY3cbGL1WRZh\nfy0tLVpTU6OLFi3Smpqar9uhk6m+vl4B6KZNm2wtx3SbfKxt692HVn29v47Qu84QKcHHcyFKgTqB\nYxhL8gDGAXgPwIcAHoq0PZN8lGJJvtFuG6yt3OqzLMT+2ocOdcRYcZ/Pp6NGjdIRI0boqVOnbC/v\n26NrWoMkePtG18R6L9buwyXejte+fWNfvIxt8tEzkuQBZADYC6AIQF8A7wC4NNxnmOSjZEVNvmfV\nLNx37d7ZIJ6zrOcFo9f+2s88U4vOPVcBaCagpYD+KvDcJ5DskzVWfMuWLQpA6+rqbC9Ltfc4+T7q\nb5p5WJMxTj7emny4i0PvhNy9bW5udK2DwaRAP70jmEry1wD4jx6/Pwzg4XCfYZLvIdJC2om0yXdv\nG8uom+4z3YohEj32V3rllQpArwC0rVe5bYHXg9Vm7TBu3DgdPHiwfvnll7aX1S3SjFe7Lm7h/izh\nruvh6gyhDlk2u9jPVJK/DUBdj99/BqA23GeY5AOiSeKxVHFCLfod6Tu2FWdkmDO8u106M0iC7360\nBmr0Pdul7bBr1y4FoI888ohtZYTT2tqqtbW1umjRIq2trU1Kf8TMmaEPgVDX9XiaUNiBaj9TSX5S\nkCRfE2S7MgBNAJry8/Pt/r9IDfHcij6ahB/rYiOJnJHh2vQD++seYVIaoezSQO22e4SJHe666y7N\nysrSw4cP21ZGssX6ZbD3I1TffDSHW89tIjUBUeLYXJNqYqn6hBrTFuzMi3bma6JnZDQZpKBAFy1a\npIC/DT7ctg8HkvyiRYvi/R8N6+9//7tmZmbq7Nmzbdm/CZFq3PEcCrG0o0dTl8jNZRu7VUwl+T4A\nmgEM69Hx+g/hPsMkHxBLTT5Up2lu7unbxjqkItazO1L8vfbnlJr8Qw89pB6PRz/88ENb9m9CpEMo\n3kMhmmt9qLI9nsQPLQrO5BDK8QDeD4yyWRhpeyb5gFgaPsOdkb3FUn3r/o4dT2druAzSY39OaJM/\nduyYDho0SG+77TbL921SpC+D8X6pi6bVLtYLCJttEsfJUKko2nb2aJN8pKGSVp51MXwT6R4rfkUg\nofdO8HaPrqmqqlIAum3bNlv2b0o0I2dDNamEWz44kZp8IhcOCo9J3irJHrQbqecs3Nnk8XzzuWBz\n1aM96+L5N8cwOqgd0KI+/jHhfeBvmnkYyRknf+LECS0oKNBrr73W8n2bFu2fIFgyj2Z5g3jKTuTC\nQeExyVsh2dPvrBjfHu8jXHXPilksvfbbDmhpRoZ6RDSZY8XXrl2rAHTDhg227N+0aK7P8Yx7j6a8\nniN1uz9r1SnECVKnY5K3QrJndIQrL1INPpEE33NNWLv+zSH22zpkSNLGivt8Ph09erRedNFFSVnC\nwKmsGsMebQJPNEFzqYPgmOStkOwZHeHKC/deLAm9d+Nr7zFtdv2bHTA75tVXX1UAumLFiqSV6URW\nXceTVQfi7Nng4knyXE++t1ALaUd7y3ory7MilqwsoLoaOHz4m3Pl8GH/e92LiUe77nyskv1/GYTX\n68XZZ5+NO+64I2llOtHixf5DoaesLP/rsbBynXsnlJMWYr0qWPFwdE3ervbpeMoL916o3q3s7Oim\nKUZq67fiu7Hh79x//etfFYD+5je/SUp5TmdFGzdr8maBzTUWsWukSTzlhXov3AzYSPGHW5rQ6l4u\ng71n06dP1379+unHH3+ctDLdLtSCZzNn2l8O2+SZ5M0yUfUIlkCjOTsc0FZut/b2du3bt6/OmDHD\ndChJZ/d1deZMa1amjoSja07HJG+SUxJnuBE5PcfLufy78K9//WsVEX3vvfdMh5JUyagBp8Hh41jx\nJHl2vFrFAZ2MAML3THXfUHv8eGt64Rzqiy++wOOPP46SkhIMHz7cdDhJlYybYrNTNLUwyVvFquEL\niWhoCD1SpltnJ/D73wMrVwIFBYCI/3nlSmDq1OTEabPf/va3OHLkCMrLy02HknTJSMBOqc9QlGKt\n+lvxcGVzjarZRsRYZse6qO29t5MnT+qFF16oV111lfp8PtPhJF0ymlLYKWoO2Fxj2NSpwL59gM/n\nf05mzTjY9/RQXFzl2rhxI/bu3Yvy8nKIiOlwks7qL5QNDd9Mpygs9P8+daqrvwi6T6xXBSserq3J\nmxTtLFiXV7muueYaHTZsmJ48edJ0KMZY9YUyWI29b9/T16qh5EEcNfk+pi8yZJH8fH/Ham+5ucBZ\nZ/kbZfPz/VU6l1a5tm7dij/96U+oqalBRkaG6XCMmTrVmj9xsC+HXV3AJ5/4f+7ux+8uk5yJzTVu\nEep7enW1uSakJPN6vcjJycHdd99tOhRXiKaz1uqRO2Q9Jnm3SPOG0g8++AAbNmzArFmzkJ2dbTqc\nlBCsvb2naLtuOHTS2ZjkU0mks9JEx2+kmJKksrISmZmZmD17tpHyU01Dg7+ppaXF39re3fTS888X\n7MthMC7ux3eHWBvxrXiw4zUOThy35pCYPv74Y+3Xr5/+/Oc/T2q5qSzaoZa9bxDSe7kk04dgugGH\nUIbgkNpmQpIxlTFWDolp+fLlOH78OB588MGklpsKQh368UyaOussYPr0tG0RTF2xXhWseCS1Ju+Q\n2mbCEl0bx46JWg5Yr6ezs1PPOeccveWWW5JWZqoId+hHU5N3y6njJuACZUG4ZTWlRP4ddp2tDvi/\nfeKJJxSAvvLKK0krM1VEuq9rpEPCAX9e6iXpSR7AYwDeBfBnAP8OYFA0n0tqkndAbdMSiSRqu85W\nw1W9U6dO6fDhw3X06NFpuYRBJJEO/Uhf7txy6riJiSR/E4A+gZ//FcC/RvM51uTjFG+Ti51nq8H1\nejZu3KgAdM2aNUkrM5Ukeui76dRxC6PNNQAmAmiIZlu2ySeZS8/W733ve5qfn68nTpwwHYojJXro\n89RxnniSvJXLGtwD4P9YuD9rdHf9L1yYFlP7g1q82D8IuudImBRfP3779u14/fXXUVlZiT597F2d\no7W1FY2Njejo6MCgQYMwYcIE5OXl2VqmFRI99HnquESkqwCALQB2B3lM6LHNQvjb5CXMfsoANAFo\nys/PT85lj77hsnupTZo0SQcOHKjHjh2zrYz29nYtLS1Vj8ejAL5+eDweLS0t1fb2dtvKJgoGcdTk\nxf+5+InInQBmABirqlGtdVtcXKxNTU0JlUvpq7m5GRdddBF+8Ytf4NFHH7WljAMHDmDMmDFobm4G\nkAmgBMDFAN4DsBHASRQVFWHr1q0YPHiwLTEQ9SYiO1S1OJbPJDQZSkTGAfhnACXRJniiRFVVVSEj\nIwNz5syxrYzZs2cHEvwVAJoBPA9gceD5bwCuQHNzM+6//37bYkiEG+b/kTUSqsmLyIcAzgAQWHwU\n21R1RqTPsSZP8Tpy5Ajy8vIwadIkrF692pYyWltbMWzYMPh8GfAn+KFBtmoDUASPx4d9+/Y5qo2+\ne12a3l0wnJ2a+pJek1fV/6qqeao6KvCImOCJErFixQp0dnZiwYIFtpXR2NgIn88HfxNNsAQPAHkA\nJsDn86GxsdG2WOIRarWJadNYq09H6bF2DbnCV199haVLl+Lmm2/GZZddZls5HR0dgZ8ujrDlcADA\n0aNHbYslHuHWnwm22iS5G5M8pYyGhgYcPHgQ5eXltpYzaNCgwE/vRdjyfQBATk6OrfHEKtLSv6bX\ntaPkSnh0TTzYJk+x8vl8GDlyJPr27YudO3faepNuN7bJ9ybiv+0ApZakt8kTJctLL72EPXv2oLy8\n3NYEDwD5+fmYOHEigBMAJsCf0HtqA/AjACcxceJERyV44Ns3CQuFN/pIH0zypnGsW1S8Xi+GDBmC\nyZMnJ6W82tpaFBUVAXgLQBGA2wD8KvDsf72oqAjLli1LSjyx6r5JWH198Fv/pvBkZ4pVrLOnrHjw\nzlABXBwkKk1NTQpAH3vssaSWG2nG64EDB5IaT7xcNtk5rcHEjNd4sE0+oLDQP9yht4ICfzWMAAA/\n/elPsWnTJrS1tWHgwIFJL7+trQ2NjY04evQocnJyUFJS4rgmGkoP8bTJ27uyE4UXzz3Y0kxLSwue\ne+45zJ8/30iCB4C8vDzHzmwlioRt8iaF6v1ir9jXqqurISKYN2+e6VCIUhKTvEmLF7NXLIyOjg48\n+eSTmDx5MptHiOLEJG9Sz7FuIv5nLjDytZUrV+Lzzz+3dQkDIrdjxys5UldXF4YNG4YRI0Zgy5Yt\npsMhcgR2vJJrrF27Fh999BGeeuop06EQpTQ215DjqCq8Xi9GjhyJm2++2XQ4RCmNNXlynM2bN2PX\nrl1YtWqV7UsYELkda/LkOF6vF+effz6mTJliOhSilMckT47yzjvvYPPmzZg7dy7OOOMM0+EQpTwm\neXKUiooKZGdn47777jMdCpErMMmTY+zfvx9r1qzB9OnTHXcjDqJUxSRPjrF06VL4fD7Mnz/fdCi2\n4KrSZAJH15AjHDt2DE888QQmTZqEwsJC0+FYrvfdmrrvtQpwgjPZizV5coS6ujocO3bM9vu3mrJw\n4em34+O9VikZmOTJuBMnTqCqqgrXX389iotjmrGdMriqNJliSZIXkXIRURE524r9UXpZt24d2tra\nXFuLB7iqNJmTcJIXkTwA/x0A6yQUs+4lDC655BKMHz/edDi24arSZIoVNflKAL+E//6XRDF55ZVX\nsHPnTixYsAAej3tbD7mqNJmS0FLDIlICYKyqzhORfQCKVfVwiG3LAJQBQH5+/uiWYPc2pbQzfvx4\n7NixAy0tLejXr5/pcIgczZalhkVkC4Dzgry1EMCvANwUTUGquhLASsC/nnwMMZJL7d69Gy+++CIW\nLVrEBE9kk4hJXlVvDPa6iFwGYBiAdwIrBQ4F8JaIfFdVD1gaJbnSkiVLcOaZZ2LmzJmmQyFyrbgn\nQ6nqLgDndv8eqbmGqKf29nbU19ejrKwMubm5psMhci339nSRo9XU1ODkyZN44IEHTIdC5GqWLWug\nqoVW7Yvc7fPPP8fy5cvx4x//GBdeeKHpcIhcjTV5Srqnn34aHR0drp78ROQUTPKUVCdPnkRlZSXG\njBmDq6++2nQ4RK7HVSgpqV544QXs27cPlZWVpkMhSgusyVPSdC9hcNFFF+GHP/yh6XCI0gJr8pQ0\nr7/+Ot58800sX74cGRkZpsMhSgusyVPSeL1enH322bjjjjtMh0KUNpjkKSneffdd/O53v8P999+P\nrN7LMRKRbZjkKSmWLFmCfv36YdasWaZDIUorTPJku4MHD+KZZ57BnXfeiXPPPTfyB4jIMkzyZLtl\ny5ahq6uLSxgQGcAkT7bq7OzEsmXLUFJSgosvvth0OERph0mebLV69WocOXKESxgQGcIkT7Y5deoU\nlixZgquuugpjxowxHQ5RWmKSJ9ts3LgRe/fuRXl5OQI3liGiJGOSJ9t4vV4MGzYMEydONB0KUdri\nsgZki61bt+JPf/oTampquIRrD/eJAAAJBElEQVQBkUGsyZMtvF4vcnJycPfdd5sOhSitMcmT5T74\n4ANs2LABs2bNQnZ2tulwiNIak3wwDQ1AYSHg8fifGxpMR5RSKisrkZmZidmzZ5sOhSjtsU2+t4YG\noKwM6Oz0/97S4v8dAKZONRdXijh06BBWrVqFn/3sZzjvvPNMh0OU9liT723hwm8SfLfOTv/rFNHy\n5ctx/PhxPPjgg6ZDISIwyZ+utTW21+lrX375JWpra3HLLbfg0ksvNR0OEcGCJC8ic0TkPRH5i4j8\nmxVBGZWfH9vr9LVnn30Whw4d4hIGRA6SUJIXkf8GYAKAf1TVfwDgtSQqkxYvBnrf1CIry/86heTz\n+VBRUYHRo0fj+uuvNx0OEQUk2vE6E8CjqvoVAKjqx4mHZFh35+rChf4mmvx8f4Jnp2tYmzZtwvvv\nv481a9ZwCQMiBxFVjf/DIm8D2AhgHIDjAMpV9c0Q25YBKAOA/Pz80S0tLXGXS85z3XXXoaWlBXv3\n7kWfPhy0RWQHEdmhqsWxfCbi2SgiWwAEGwu3MPD5HABXA7gSwHMiUqRBrhyquhLASgAoLi6O/8pC\njrN9+3a8/vrrqKysZIIncpiIZ6Sq3hjqPRGZCeCFQFL/fyLiA3A2gEPWhUhOV1FRgYEDB+LnP/+5\n6VCIqJdER9dsAPB9ABCR4QD6AjicaFCUOpqbm7F+/XrMmDED/fv3Nx0OEfWS6HfrpwE8LSK7AXQB\nuDNYUw25V1VVFTIyMjBnzhzToRBREAkleVXtAjDNolgoxRw5cgRPPfUUfvrTn2LIkCGmwyGiIDjj\nleK2YsUKdHZ2YsGCBaZDIaIQmOQpLl999RWWLl2Km2++GZdddpnpcIgoBCZ5iktDQwMOHjzIJQyI\nHI5JnmLm8/ng9Xrxne98B2PHjjUdDhGFwZkrFLOXXnoJe/bswbPPPsslDIgcjjV5ipnX68WQIUMw\nefJk06EQUQRM8hSTHTt24JVXXsH8+fORmZlpOhwiioBJnmJSUVGB/v3749577zUdChFFgUmeotbS\n0oLnnnsOZWVlGDhwoOlwiCgKTPIUterqaogI5s2bZzoUIooSkzxFpaOjA08++SQmT56MvLw80+EQ\nUZSY5CkqK1euxOeff84lDIhSDJM8RdTV1YXq6mqMHTsWl19+uelwiCgGnAxFEa1duxYfffQRnnrq\nKdOhEFGMWJOnsFQVXq8XI0eOxM0332w6HCKKEWvyFNbmzZuxa9curFq1iksYEKUg1uQpLK/Xi/PP\nPx9TpkwxHQoRxYFJnkJ65513sHnzZsydOxdnnHGG6XCIKA5M8hRSRUUFsrOzcd9995kOhYjixCRP\nQe3fvx9r1qzB9OnTkZOTYzocIooTkzwFtXTpUvh8PsyfP990KESUACZ5Os2xY8fwxBNPYNKkSSgs\nLDQdDhElIKEkLyKjRGSbiLwtIk0i8l2rAiNz6urqcOzYMd6/lcgFEq3J/xuAf1HVUQD+Z+B3SmEn\nTpxAVVUVrr/+ehQXF5sOh4gSlOhkKAUwIPDzQAAfJbg/MmzdunVoa2vDsmXLTIdCRBYQVY3/wyIj\nAPwHAIH/W8E/qWpLiG3LAJQFfh0JYHfcBZt3NoDDpoNIQCrHn8qxA4zftFSP/2JV7R/LByImeRHZ\nAuC8IG8tBDAWwP9V1fUi8hMAZap6Y8RCRZpUNWXbAhi/OakcO8D4TUvH+CM214RL2iLyDIDu2wSt\nA1AXS+FERGSvRDtePwJwfeDn7wP4IMH9ERGRhRLteL0XQLWI9AFwHN+0uUeyMsFyTWP85qRy7ADj\nNy3t4k+o45WIiJyNM16JiFyMSZ6IyMWMJXkReUxE3hWRP4vIv4vIIFOxREtExonIeyLyoYg8ZDqe\nWIhInoi8IiJ7ROQvIjIv8qecR0QyRGSniGwyHUusRGSQiDwfOO73iMg1pmOKhYg8EDh2dovIGhHp\nZzqmcETkaRH5WER293jtv4jIZhH5IPDsyCVWQ8QeV840WZPfDGCkqv4jgPcBPGwwlohEJAPAMgA/\nAHApgCkicqnZqGJyEsACVR0B4GoA96dY/N3mAdhjOog4VQN4SVUvAfAdpNC/Q0SGAJgLoFhVRwLI\nAHC72agiWg1gXK/XHgLwn6p6EYD/DPzuRKtxeuxx5UxjSV5VX1bVk4FftwEYaiqWKH0XwIeq2qyq\nXQDWAphgOKaoqWq7qr4V+Pkz+BPMELNRxUZEhgK4BSk4H0NEBgC4DsBTAKCqXaraYTaqmPUBcGZg\nNF0WHL6Miaq+BuBIr5cnAPht4OffAvhRUoOKUrDY482ZTmmTvwfAi6aDiGAIgLYev+9HiiXJbiJS\nCOByANvNRhKzKgC/BOAzHUgcigAcArAq0NxUJyLZpoOKlqr+HYAXQCuAdgCfqurLZqOKy2BVbQf8\nFR8A5xqOJ15R50xbk7yIbAm03/V+TOixzUL4mxIa7IzFAhLktZQbfyoiZwFYD2C+qh4zHU+0RORW\nAB+r6g7TscSpD4ArACxX1csBfAHnNhWcJtB2PQHAMAAXAMgWkWlmo0pPsebMRCdDhRVpHRsRuRPA\nrQDGqvMH7O8HkNfj96Fw+NfV3kQkE/4E36CqL5iOJ0ZjAJSIyHgA/QAMEJF6VU2VRLMfwH5V7f72\n9DxSKMkDuBHA31T1EACIyAsA/glAvdGoYndQRM5X1XYROR/Ax6YDikU8OdPk6JpxAP4ZQImqdpqK\nIwZvArhIRIaJSF/4O50aDccUNRER+NuD96jqEtPxxEpVH1bVoapaCP///R9SKMFDVQ8AaBORiwMv\njQXwV4MhxaoVwNUikhU4lsYihTqOe2gEcGfg5zsBbDQYS0zizZnGZryKyIcAzgDwSeClbao6w0gw\nUQrUIqvgH1nwtKouNhxS1ETkWgCvA9iFb9q0f6WqvzcXVXxE5AYA5ap6q+lYYiEio+DvNO4LoBnA\n3ap61GxU0RORfwEwGf6mgp0ApqvqV2ajCk1E1gC4Af7lhQ8C+F8ANgB4DkA+/BeuSarau3PWuBCx\nP4w4ciaXNSAicjGnjK4hIiIbMMkTEbkYkzwRkYsxyRMRuRiTPBGRizHJExG5GJM8EZGL/X8OEZum\nAKnDiAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure()\n",
    "plt.scatter(dataArr[labelArr == -1][:, 0],\n",
    "            dataArr[labelArr == -1][:, 1], color=\"red\")\n",
    "plt.scatter(dataArr[labelArr == 1][:, 0],\n",
    "            dataArr[labelArr == 1][:, 1], color=\"blue\")\n",
    "plt.plot(np.array([-2, 12]),\n",
    "         ((-np.array([-2, 12]) * w[0] - b) / w[1]),\n",
    "         color=\"black\")\n",
    "for i in range(len(alphas)):\n",
    "    if alphas[i] > 0.0:\n",
    "        if labelArr[i] == -1:\n",
    "            plt.scatter(dataArr[i][0], dataArr[i][1],\n",
    "                        color=\"red\", edgecolors='black', s=100, linewidths=2)\n",
    "        else:\n",
    "            plt.scatter(dataArr[i][0], dataArr[i][1],\n",
    "                        color=\"blue\", edgecolors='black', s=100, linewidths=2)\n",
    "plt.xlim(-2, 12)\n",
    "plt.ylim(-8, 6)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Full Platt SMO"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dataArr, labelArr = loadDataSet('testSet.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "iteration number: 1\n",
      "iteration number: 2\n",
      "iteration number: 3\n",
      "iteration number: 4\n"
     ]
    }
   ],
   "source": [
    "b, alphas = smoP(dataArr, labelArr, 0.6, 0.001, 40)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-3.710327594390181\n"
     ]
    }
   ],
   "source": [
    "print(b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.79217255 -0.25433037]\n"
     ]
    }
   ],
   "source": [
    "w = calcWs(alphas, dataArr, labelArr)\n",
    "print(w)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[3.634009 1.730537] -1.0\n",
      "[4.658191 3.507396] -1.0\n",
      "[ 3.457096 -0.082216] -1.0\n",
      "[ 2.893743 -1.643468] -1.0\n",
      "[ 5.286862 -2.358286] 1.0\n",
      "[6.080573 0.418886] 1.0\n"
     ]
    }
   ],
   "source": [
    "# support vectors\n",
    "for i in range(len(alphas)):\n",
    "    if alphas[i] > 0:\n",
    "        print(dataArr[i], labelArr[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXkAAAD8CAYAAACSCdTiAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl8VNX5+PHPmYQoYTEUF1QgECsu\nuBWDVbEVl7ZKbRCDdcFq7UtTqbgvCKHwMhFxQQUJLrhQMXEFo/nZ6ler39L6pWoDiIgIImahJhCB\nKBVkm+f3x0xqHGafe+fM8rxfr3lNMnPnnidw7zPPPffcc42IoJRSKjN5bAeglFLKPZrklVIqg2mS\nV0qpDKZJXimlMpgmeaWUymCa5JVSKoM5kuSNMQXGmPnGmE+MMSuNMSc5sV6llFKJyXVoPTOB10Vk\ntDEmD8h3aL1KKaUSYBK9GMoY0xNYBhSJXlmllFIpxYlKvghoA+YaY44FFgPXicg3nRcyxpQBZQDd\nunU7/vDDD3egaaUy386dO1m+fDk/+MEPGDBggO1wlEWLFy/+UkT2i+UzTlTyxcC7wDARec8YMxP4\nWkT+GOozxcXFUl9fn1C7SmWLm266iZkzZ/LJJ5/wwx/+0HY4yiJjzGIRKY7lM06ceF0HrBOR9/y/\nzweGOLBepbLehg0bePjhhxkzZowmeBWXhJO8iLQCzcaYw/wvnQF8nOh6lVIwffp0tm/fTnl5ue1Q\nVJpyanTNNUCNf2TNWuByh9arVNZqa2tj9uzZXHTRRQwaNMh2OCpNOZLkReQDIKZ+IqVUePfffz/b\ntm3TKl4lRK94VSoFbdy4kaqqKi644AKOOOII2+GoNKZJXqkU9MADD/DNN98wadIk26GoNKdJXqkU\ns3nzZh588EFGjx7N4MGDbYej0pwmeaVSzIwZM9iyZQt//GPIS02UipomeaVSSHt7OzNnzuS8887j\n6KOPth2OygCa5JVKIQ8++CBfffWVVvHKMZrklUoRX331FQ888AAjR47kuOOOsx2OyhCa5JVKEVVV\nVbS3tzN58mTboagMokleqRSwZcsW7r//fs455xyGDNGpn5RzNMkrlQJmz57Npk2btIpXjtMkr5Rl\n//nPf5g+fTpnn302Q4cOtR2OyjCa5JWy7OGHH2bjxo1axStXaJJXyqJvvvmGe++9l5///OeceOKJ\ntsNRGUiTvFIWPfLII7S1tTFlyhTboagMpUleKUu2bt3Kvffey5lnnsnJJ59sOxyVoZy6aYhSKkZz\n5sxh/fr1vPjii7ZDURlMK3mlLNi2bRt33303p512Gj/5yU9sh6MymFbySlnw+OOP09rayrPPPms7\nFJXhtJJXKsm+/fZb7rrrLn76058yfPhw2+GoDKeVvFJJ9uSTT/LFF18wb94826GoLKCVvFJJtH37\ndqZNm8awYcM4/fTTbYejsoBW8kol0Z/+9CfWrVvHE088gTHGdjgqCzhWyRtjcowxS40xrzq1TqUy\nyY4dO7jzzjs58cQT+dnPfmY7HJUlnOyuuQ5Y6eD6VKarqYEBA8Dj8T3X1NiOyFXz5s2jqamJyZMn\naxWvksaRJG+M6Qv8EnjcifWpLFBTA2Vl0NgIIr7nsrKMTfQ7d+5k6tSpDB06lLPOOst2OCqLOFXJ\nzwBuBbyhFjDGlBlj6o0x9W1tbQ41q9JWeTls3fr917Zu9b2egaqrq2loaGDKlClaxaukMiKS2AqM\nOQcYISJ/MMYMB24WkXPCfaa4uFjq6+sTalelOY/HV8EHMga8IWuFtLRr1y4OP/xwCgoK+Ne//qVJ\nXsXNGLNYRIpj+YwTo2uGASXGmBHA3kBPY0y1iFziwLpVpurf39dFE+x1FzQ1NVFXV0d7ezsFBQWM\nHDmSfv36udJWoGeeeYbPPvuMV155RRO8Sj4RcewBDAdejbTc8ccfLyrLVVeL5OeL+Op53yM/3/e6\ng1paWqS0tFQ8Ho8A/314PB4pLS2VlpYWR9sLtHPnTjn00EPluOOOE6/X62pbKvMB9RJjXtZx8sqO\nMWN8z+Xl0NTkq+CnTv3udQe0trYybNgw1q5dSxdgFHAYsAp4xetlwYIFLF26lEWLFnHAAQc41m5n\nzz//PJ9++ikvvfSSVvHKioT75OOhffIqGUaPHs2CBQsYArwC9O303jpgJLAEKC0tZf78+Y63v3v3\nbgYPHkxeXh4ffPABHo9eYK4SY6tPXqmU09TURG1tLV3YM8Hj//1loAiora2lubnZ8T76F198kVWr\nVvHiiy9qglfW6JanMlJdXR1er5cS9kzwHfrhq+a9Xi91dXWOtu/1eqmsrGTw4MGcd955jq5bqVho\nJa8yUnt7O+Drgw9nkP958+bNjrY/f/58Pv74Y5577jmt4pVVuvWpjFRQUAD4TrKGs9r/3KtXL8fa\n7qjijzjiCEaPHu3YepWKhyZ5lZFKSkrweDzU4TvJGkwzvv56j8dDSUmJY23X1tby0UcfMWnSJHJy\nchxbr1Lx0CSvMlL//v0ZNWoUO/H1uzcHvN8MnAvsAkaNGuXYSVev10tFRQWDBg3iggsucGSdSiVC\nk7zKWFVVVRQVFbEE3yia0cBE/3MRvuGTRUVFzJ4927E26+rq+PDDD7WKDyPLJh+1TsfJq4zW2trK\nuHHjqK2txdtpThyPx8OoUaOYPXu2YxdCie9qbrZs2cLKlSvJzdVxDYE6Jh/tPDddfj7MmePodXAZ\nS8fJKxWgT58+zJ8/n+bmZurq6ti8eTO9evWipKTE8XHxr776KkuXLmXu3Lma4EMIN/moJnl3aCWv\nlANEhBNOOIFNmzbxySef0KVLF9shpaQsmnzUFfFU8tonr5QDXnvtNerr65k4caImeL9gfe+hJhl1\nafJRhSZ5pRImIlRUVFBYWMill15qO5yUEOrGXyNG+PrgO8vP981Np9yhSV6pBL3xxhu89957WsV3\nEqrv/S9/8Z1kLSz0ddEUFupJV7dpn7xSCRARhg0bxr///W8+/fRT8vLybIeUErTv3R3aJ6/sysIB\n0G+99Rb//Oc/mTBhgib4TrTvPXVoklfOCNUJm8GJXkS4/fbb6du3L5dffrntcFLK1Kna954qNMkr\nZ4QbAJ2h/va3v/HOO+9w2223sddee9kOJ6WMGaN976lC++SVM7KwE3b48OF8+umnfPbZZ+y99962\nw1FZQPvklT1Z1gm7cOFCFi5cyPjx4zXBq5SmSV45I8s6YSsqKujTpw9XXnml7VCUCkuTvHJGop2w\naTQy55133uHtt9/m1ltvpWvXrrbDUSosTfLKOWPGQEODrw++oSG2BJ9GI3MqKirYf//9+f3vf287\nlIyWRt/7KS3hJG+M6WeM+V9jzEpjzApjzHVOBKaySBqNzPnnP//Jm2++yS233EJ+YPeUCiqeZJ1m\n3/spLeHRNcaYA4EDRWSJMaYHsBg4V0Q+DvUZHV2jAN8eW17u24ODScGROWeffTb19fU0NDTQrVs3\n2+GkvHjnjx8wIPhmUVjoO0jMVlZG14hIi4gs8f+8BVgJHJzoelWG61yqAU1AFXCH/7kZUm5kzvvv\nv8/rr7/OzTffrAk+SvEepDU1xfa6Cs3RcfLGmAHA34GjROTrgPfKgDKA/v37H98YqnpT2cFfqrUC\n44BaoHPN7gFGDR1KVV0dffr0sRHhHs455xzeffddGhoa6N69u+1w0kK8l09oJR+c1XHyxpjuwALg\n+sAEDyAic0SkWESK99tvP6eaVemqqYlWYBi+jSYHKMV3D9ZSfBvmgn/9i2HDhrF+/Xp7cfrV19fz\n5z//mRtvvFETfAzivXwiy0bkusqRJG+M6YJvX60RkZecWKdKEW4Ncejfn3HAWmCI/3k+MNX//HnH\n62vXcvXVVzvTZgIqKyvp1asX48aNsx1KWok3Weu0CA4SkYQegAHmATOi/czxxx8vKg1UV4vk54v4\njrh9j/x83+sJapwxQzwgXUCaO6+/06MJJBfE4/FIU1OTA39QfJYsWSKAVFRUWIshnVVXixQWihjj\ne3Zg88laQL3EmKOdqOSHAb8BTjfGfOB/jHBgvcqGzpX7ZZe5NrSxLicHL1AC9A2xTD9gJOD1eqmr\nq0u4zXhVVlayzz77cO2111qLIV0EO/CL9/IJ5QwnRte8IyJGRI4RkeP8j784EZxKssDBybt3B1/O\ngSEO7e3tABwWYblB/ufNmzcn3GY8PvzwQ2pra7n++uvZZ599rMSQqgIT+h/+sOfY9ksugX33Te74\ndr2I6vtybQegUkiw8W7BODC0saCgAIBVEZZb7X/u1atXwm3Go7Kykp49e3LddXqNX2eB498bG+GR\nR4KPpNm40bcsuF/FB4srWW2nrFj7d5x4aJ+8ReE6SEP0jSfcJx+kzcbGRvF4PCndJ798+XIBZNKk\nSUlvO9UVFka3uXR+FBbai6t3b/fbTgbi6JPXJJ9Nwp1Ira72JeFge0hOTvxnzcK0WVpaKoAM8Sf0\nwAQ/BASQ0tJSV/45Ivn1r38tPXr0kI0bN1ppP5WF2lTCPYyJr61YTtyGiysTTvhqklfhhSpzCgtD\nv2dMYntHmDZbWlqkqKhI8FfspSAT/M+5/gRfVFQkra2tzvz9MVixYoUYY2TixIlJbzvVVVf7vvdD\nbS5OVvKxDvAKd4SRjCMJt2mSV+GF2gONCb93utWmiLS0tEhpaal4PB7Bn9jxd9GUlpZaSfAiIhdd\ndJF069ZN2trarLSfqoIl3c7Jd+xYX9dIsPfiqRXC1SWh4nP6SCKVaJJX4cVTySda/kS53qamJqmq\nqpLKykqpqqqyOi5+5cqVYoyR8ePHW4shVYX678zJ+X4Sd2psfIQaIahgXzJayWuSzw6R+uTduPDJ\nxQuq3HLJJZdIfn6+bNiwwXYoKSeepJuIeGqPNNzkoqZJXkUWrsRy69LENLrkcfXq1eLxeOTmm2+2\nHUpKcuuAL5R4E3YabXIx0SSvVIIuu+wy6dq1q7VzAanOzQO+ZNce6UiTvHJHluxla9askZycHLnh\nhhtsh5JyOm8CvXv7Hk5tDpncveK0eJK83uNVhZdF92G788476dKlC7fccovtUFJK4CawcSNs2uT7\n2QlpdPfHtKRJXu0p0UnK0nDykM8//5x58+ZRVlbGgQceaDuclBIsCXckeCe+8/UuUO7SJK++L9FJ\nytK08p82bRo5OTmMHz/edigpJ1Ky3brVNxFZvN/n8d5YREVHk3ymSbSKTnSSsjQ89m5sbGTu3Llc\nccUVHHTQQa621dTURFVVFXfccQdVVVU0Nze72p4Tok228X6f612gXBZrJ74TDz3x6hInzmCFulww\n2nUmeyC1A6666irJy8uT5uZm19qIdGVvS0uLa20nKtxVrk4Np8ySc/sJQ0fXZLlwlyNGs/eEm6Ss\n8yPcOpI9kDpBTU1N0qVLF7nqqqtca6PzHD3QRaBUYKL/OVfA3hw94QQbURNNkk/h7/O0p0k+20WT\noOOd3SnaZJ1m4+Guvvpq6dKlizQ2NrrWRsdsmzBEoDngn7TZ//qes23arG5D/Te6Vcmr6GiSz3bR\nTvIdai+M9CURbbJOk2PvdevWSV5enlx55ZWutdExb76vgg9M8B2PJoHc782bb/u7MtxBYbhNpEuX\n+GJMk03GOk3y2S7aztNgx9Ph5o/t+GKIZc9Lg7322muvldzcXPn8889da2PWrFn+Kr40wn+Lr9qv\nqqoSEfu9XuG+77t0Cf1eXp6jtxxQAeJJ8jq6JtXFOlqma9fvfvaE+O8NHC7RMewx2HDJ/Hyoro7t\nDsxpMIyypaWFOXPmcOmllzJgwADX2um4l220d7PtuJetk2PHQ21C4TatUCNqevcGY0K3tWNH7AOp\n0nBAVnqJ9VvBiYdW8lGKpcQJtmxe3p5lV7DPRzt/bLRsl6FRuOGGGyQnJ0fWrFnjaju2K/lQm9DY\nseE3rerqPTedLl2iO/ka64nXNByQZQ3aXZNhYtnTw93cMlK3idN7WYT1NTY2yqxZs6SyslJmzZqV\n9LnjW1tbpWvXrnLZZZe53pbtPvlY+9Y7Nq3qal+NEFgzRErw8XwRpUFNkDKsJXngLGAVsAa4LdLy\nmuSjFEvyjXbZYH3lTu9lIdbX0rdvSowVv+mmm8Tj8cjq1auT0t73R9c0BUnw7o2uifVerB2bS7wn\nXvPyYp+8TPvko2clyQM5wGdAEZAHLAOODPcZTfJRcqKS71yahTvWDswG8exlnb8wAtbX0rWrFO2/\nvwDSBd99XCeS/Pu5rl+/XvLz8+U3v/mNq+109v1x8rni65qZIMkYJx9vJR/uyyEwIXcs27t3dL2D\nwaTBefqUYCvJnwT8T6ffJwATwn1Gk3wnkSbSTqRPPtxdn8Il+nj2smBtdFpf6dChAsgQkOaAdpv9\nrwerZp126623isfjkU8++cTVdgLZupdtuP+WcN/r4WqGUJusdru4z1aSHw083un33wBV4T6jSd4v\nmiQeS4kTatLvSMfYTuyRYfbwjn7pLkESfMejyV/Rd+6XdlpbW5t069ZNLr74YlfWHw0b97IdOzb0\nJhDqez2eLhQ9geo+W0n+/CBJflaQ5cqAeqC+f//+bv9bpId4bkUfTcKPdbKRRPbIcH36/vV1jDAp\njdB2qb+67Rhh4rQJEyaIMUY+/vhjV9ZvS6wHg4GPUOfmo9ncOi8TqQtIJU67a9JNLKVPqDFtwfa8\naK98TXSPjCaDFBZKZWWlgK8PPtyyE/xJvrKyMt5/0ZA2btwo3bt3lwsuuMDxddsUqeKOZ1OIpR89\nmlqid2/tY3eKrSSfC6wFBnY68To43Gc0yfvFUsmHOmnau/eey8Y6pCLWvTtS/AHrS4VKftKkSQLI\n8uXLHV+3TZE2oXg3hWi+60O17fEkvmmp4GwOoRwBrPaPsimPtLwmeb9YOj7D7ZGBYinfOo6x4znZ\nGi6DdFqf7T75TZs2Sc+ePWX06NGOrjcVRDoYjPegLppeu1i/QLTbJnF6MVQ6irafPdokH2mopJN7\nXQxHIh1jxYf4E3pggndzdM2UKVMEkGXLljm+btuiGTkbqksl3PTBiVTyiXxxqPA0yTsl2YN2I505\nC7c3eTzffS7YterR7nXx/M0xjA5qASnK9Y0Jz8XXNTMB98fJb968WfbZZx8ZNWqUo+tNFdH+FwRL\n5tFMbxBP24l8cajwNMk7IdmX3zkxvj3eR7hyz4mrWALW2wJSmpMjHmMkWWPFKyoqBJClS5c6vu5U\nEc33czzj3qNpr/NI3Y7POrUL6QVSe9Ik74RkX9ERrr1IFXwiCb7znLBu/c0h1tt08MFJGSv+1Vdf\nSa9evaSkpMSV9acTp8awR5vAE03QOtVBcJrknZDsKzrCtRfuvVgSemDna+CYNrf+ZstXx0ydOlUA\nqa+vT0p7qcyp7/Fk1UB69Wxw8SR5nU8+UKiJtKO9Zb2T7TkRS34+zJwJX3753b7y5Ze+9zomE492\n3vlYJfvfspMtW7Zw33338ctf/pLjjz/e9fZS3dSpvk2hs/x83+uxcHKe+1RoJyvE+q3gxCOlK3m3\n+qfjaS/ce6HObnXrFt1lipH6+p04NrZ4zH3XXXcJIO+9957rbaULJ/q4tZK3C+2ucYhbI03iaS/U\ne+GugI0Uf7ipCZ0+y2Xh7NmWLVtk3333lbPOOsv1trJNqAnPxo51vx3tk9ckb5eN0iNYAo1m78jw\nmaTuvfdeAWTRokW2Q7HC7e/VsWOdmZk6Eh1ds6d4krzxfS65iouLpb6+Puntusrj8W3vgYwBrzd5\ncQwY4LunajCFhb5O2PLy4MsUFvru5ZrGtm7dysCBAzn22GN54403bIeTdB231+18z9T8fJgzJ/pb\n9EYSahPLgM0n5RljFotIcSyf0ROvTrF4kvF7wp2Z6rih9ogRzpyFS0GPPvooGzZsYMqUKbZDsSIZ\nN8XWk6LpRZO8U5wavpCImprQI2U6bN0Kf/mLr7QrLPQdaRQWOlvqWbJt2zbuueceTj/9dIYNG2Y7\nHCuSkYBTpZ5R0dEk75QxY+wmzo7j9N27Iy/b1OSLq6HB15XU0JD2CR5gzpw5tLa2Zm0VD8lJwKlQ\nz6gYxNqJ78QjI0+82hbLbFEZOA5t27ZtcuCBB8rw4cNth2KV06NSwg3u0pOiyUccJ15zbX/JKIdE\nezyeoSXX448/TktLCzU1NbZDsarjgKy83LdJ9O/v+++O50At8CRuYyP87ndw3XWwaZNv3U8/nREH\ngRlNR9dkilBDHnr3hu7dE9/jQ2hqaqKuro729nYKCgoYOXIk/fr1c2z90di+fTuHHHIIRUVFLFy4\nEGNMUtvPVOEGanVweuSOCi+e0TXaXZMpknz1SEtLi5SWlorH4xHYc0bJlpYWV9oN5qGHHhJA3nzz\nzaS1mQ2inSIpA3v/UhY6d00WS+KJ39bWVoYNG8aCBQvI8XopBSYCpYDH62XBggUMGzaM9evXO952\noB07djBt2jROPvlkzjjjDNfbyyQ1Nd9NXzRggO/3zqI9WatDJ1NcrN8KTjy0ko9Tipzt6nyXp8Db\n+TXj7l2eAj366KMCyOuvv+56W5kk2puNRHM7A63kkwed1iCDpchkHrbv19rZjh07pLCwUH784x+L\n1+t1rZ1MFO0sHIE3CAmcLknnk0mueJJ8dnTXRDouTQfJuJQxCnV1dXi9XkqAviGW6QeMBLxeL3V1\nda7FMm/ePBobG5k8ebKebA0h1KYfz0VT3bvDFVdk3DV0mS/WbwUnHkmt5FOkAk5YopOKOdTVU1lZ\nKYBMjHAMP8HfZVNZWRlXO5Hs2LFDBg4cKMXFxVrFhxBu04+mks+UXSeToJV8EClSAScskUsZOwY8\nNzb69tWOOWziOKIpKCgAYFWE5Vb7n3v16hVzG9Goqanh888/Z8qUKVrFhxBu04/mqtVM2XWyXqzf\nCp0fwL3AJ8CHQC1QEM3nklrJZ8q0uomUVQ5Og5wKffI7d+6UQw45RIYMGaJVfBiRNv1IB3eZsutk\nEixU8m8CR4nIMfiKtwkJrs95mTKbUiJDJB2ctap///6MGjWKnfj63ZsD3m8GzgV2AaNGjXLlwqhn\nn32Wzz77TPviI4i06UeavihTdp2sF+u3QqgHMAqoiWZZ7ZNPModvaNLS0iJFRUWCv2Iv9ffBl/p/\nB6SoqEhaW1sd/TNERHbt2iWDBg2SY489Vqv4CBLd9HXXST3YHEIJ/D/gkmiWTfoQyhQZX26NC3tr\npCte3UjwIiI1NTUCyIIFC1xZfzCNjY0ya9YsqayslFmzZrk6LNRpiW762b7rpJp4knzEuWuMMX8F\n+gR5q1xEXvEvUw4UA+dJiBUaY8qAMoD+/fsf3xhpUgzlrJoaZ2atCtDc3ExdXR2bN2+mV69elJSU\nuDZ3ze7duznqqKPIzc1l2bJleCLNnZ+g1tZWxo0bR21tLd5Od/fyeDyMGjWKqqoq+vQJtmso5Y54\n5q5JeIIyY8xlwFXAGSKyNdLyoBOUqfg8//zzXHjhhbzwwgucf/75rrbVMXXD2rVrgS5ACXAYvnFF\nrwC7KCoqYtGiRRxwwAGuxqJUh6Tf/s8YcxYwHiiJNsErFQ+v10tlZSVHHnkkpaWlrrc3btw4f4If\nAqwF5gNT/c+fA0NYu3YtV199teuxxCMTrv9Tzkh0PvkqYC/gTf8oh3dF5KqEo1IqwIIFC1ixYgXP\nPvus6900TU1N1NbW4qvgX2HPa3v7Ai8DRdTW1tLc3Jz06ZXDCTYPfFmZ72e9OjX7JLS3iMgPRaSf\niBznf2iCV47rqOIPP/xw17tp4LupG4hi8ga3p26IR6iLmC65RKv6bKR3hlIp7+WXX2b58uVUV1eT\nk5Pjenvt7e3+nw6LsOQgADZv3uxqPLEKd/mDVvXZJ/OnNVBpTUSoqKhg0KBBXHjhhUlps2Pqhmgn\nb3Br6oZ4RbpYSacmyC6a5FVKq6urY9myZZSXlyeligcoKSnx9/vXAetCLNUMvILH46GkpCQpcUUr\n2Lw0gfRGH9lDk7xKWR1V/CGHHMLFF1+ctHY7pm4giskb3Jq6IRGdZ8AIRacmyB6a5G3TsW4h/fnP\nf2bJkiWUl5eTm5vc00dVVVUUFRUBS4AiYDS+mxyO9v++hKKiImbPnp3UuKLVMS9NdXXk2SZVhov1\nElknHnpnKD+dHCQkr9crQ4cOlYEDB8qOHTusxGBr6gan6dQEmQM3pjVwg17x6jdggG+4Q6DCQl8Z\nlsVee+01RowYwWOPPcYVV1xhNZZkTt2gVDhWpjWIhyZ5P4/HV78HMsY3/2uWEhFOOukkWltbWb16\nNXl5ebZDUiolJH1aA5UgnbA7qDfffJP33nuPCRMmaIJXKkGa5G2K5h5sWUZEuP322+nXrx+XX365\n7XCUSnua5G1K5G5PGertt99m0aJFWsUr5RDtk1cpQ0Q49dRTWbt2LZ999hl77bWX7ZCUSinx9Mnr\n3DUqZSxcuJB//OMfzJo1SxO8Ug7R7hqVMm6//XYOPPBA60MmlcokWsmrlPD3v/+dv/3tb8yYMYO9\n997bdjhKZQyt5FVKqKio4IADDqCsYx5cpZQjtJJX1v3f//0fb731Fvfddx9du3a1HY5SGUUreWVd\nRUUF++23H7///e9th6JUxtEkr6x69913eeONN7jlllvo1q2b7XCUyjia5JVVFRUV7LvvvowdO9Z2\nKK7TWaWVDdonr6x5//33ee2115g2bRrdu3e3HY6ramp891btuMG23mtVJYte8aqs+dWvfsWiRYto\naGigR48etsNxlc4qrZygs1CqtLF48WJeffVVbrzxxoxP8BD6nqp6r1XlNkeSvDHmZmOMGGP2dWJ9\nKvNVVlZSUFDANddcYzuUpNBZpZUtCSd5Y0w/4GeA1iQqKh988AGvvPIKN9xwAz179rQdTlLorNLK\nFicq+QeAW/Hd/1KpiCorK9lnn3249tprbYeSNDqrtLIlodE1xpgS4N8isswYE2nZMqAMoL8eo2at\nDz/8kJdeeonJkydTUFBgO5ykGjNGk7pKvohJ3hjzV6BPkLfKgYnAz6NpSETmAHPAN7omhhhVBrnj\njjvo0aMH119/ve1QlMoKEZO8iJwZ7HVjzNHAQKCjiu8LLDHGnCAirY5GqTLCihUrmD9/PhMnTqRX\nr162w1EqK8TdXSMiy4H9O34/o/wKAAAK5UlEQVQ3xjQAxSLypQNxqQxUWVlJt27duOGGG2yHolTW\n0HHyKilWrlzJCy+8wDXXXEPv3r1th6NU1nBsWgMRGeDUulTmueOOO8jPz+fGG2+0HYpSWUUreeW6\nVatW8dxzz3H11Vez7756vZxSyaRJXrlu6tSp7L333tx00022Q1Eq62iSV65as2YNNTU1jB07lv33\n3z/yB5RSjtIkr1w1depU8vLyuPnmm22HolRW0iSvXLN27VqefvpprrrqKvr0CXY9nVLKbZrklWvu\nvPNOcnNzueWWW2yHolTW0iSvXNHQ0MBTTz1FWVkZBx10kO1wlMpamuSVK6ZNm4bH42H8+PG2Q1Eq\nq2mSV45rampi7ty5XHHFFRx88MG2w1Eqq2mSV4676667ALjtttssR6KU0iSvHLVu3TqeeOIJfve7\n39GvXz/b4SiV9TTJK0fdfffdeL1eJkyYYDsUpRSa5JWDvvjiCx577DF++9vfUlhYaDscpRSa5JWD\n7rnnHnbt2sXEiRNth6KU8tMkrxzR0tLCo48+yqWXXsrAgQNth6OU8tMkrxwxffp0du7cSXl5ue1Q\nlFKdaJJXCVu/fj0PP/wwY8aM4ZBDDrEdjlKqE03yKmHTp09n+/btTJo0yXYoSqkAmuSDqamBAQPA\n4/E919TYjihltbW18dBDD3HxxRdz6KGH2g5HKRXAsXu8ZoyaGigrg61bfb83Nvp+Bxgzxl5cKeq+\n++5j27Zt2hevVIrSSj5Qefl3Cb7D1q2+19X3fPnll1RVVXHhhRdy+OGH2w5HKRWEJvlATU2xvZ7F\nHnjgAbZu3ap98UqlsISTvDHmGmPMKmPMCmPMPU4EZVX//rG9nqU2bdrErFmzOP/88znyyCNth6OU\nCiGhJG+MOQ0YCRwjIoOB6Y5EZdPUqZCf//3X8vN9r6v/mjFjBlu2bNEqXqkUl2glPxa4S0S2A4jI\nhsRDsmzMGJgzBwoLwRjf85w5etK1k/b2dmbOnElpaSlHH3207XCUUmEkmuQHAT8xxrxnjFlojBka\nakFjTJkxpt4YU9/W1pZgsy4bMwYaGsDr9T1rgv+emTNn8vXXX/PHP/7RdihKqQgiDqE0xvwV6BPk\nrXL/53sBJwJDgReMMUUiIoELi8gcYA5AcXHxHu+r9PDVV18xY8YMzj33XI499ljb4SilIoiY5EXk\nzFDvGWPGAi/5k/r7xhgvsC+Q4qW6itesWbNob29n8uTJtkNRSkUh0e6al4HTAYwxg4A84MtEg1Kp\n6euvv+b+++/nV7/6FT/60Y9sh6OUikKiV7w+CTxpjPkI2AFcFqyrRmWG2bNns3nzZq3ilUojCSV5\nEdkBXOJQLCqF/ec//+G+++5jxIgRFBcX2w5HKRUlveJVReWhhx5i48aNWsUrlWY0yauIvvnmG6ZP\nn84vfvELfvzjH9sORykVA03yKqJHHnmEtrY2pkyZYjsUpVSMNMmrsLZu3co999zDmWeeyUknnWQ7\nHKVUjDTJq7DmzJnDhg0btIpXKk1pklchbdu2jbvvvpvTTjuNU045xXY4Sqk46J2hVEiPPfYYra2t\nPPfcc7ZDUUrFSSt5FdS3337L3Xffzamnnsqpp55qOxylVJy0kldBPfHEE3zxxRc8/fTTtkNRSiVA\nK3m1h+3bt3PXXXdxyimncNppp9kORymVAK3k1R7mzp3LunXrePLJJzHG2A5HKZUAreTV9+zYsYNp\n06Zx0kknceaZIWeZVkqlCa3k1fc89dRTNDU18eijj2oVr1QG0Epe/dfOnTu58847OeGEE/jFL35h\nOxyllAO0klf/9fTTT9PQ0EBVVZVW8UplCK3kFeCr4qdOnUpxcTEjRoywHY5SyiFaySsAnnnmGdau\nXcuMGTO0ilcqgxgbd+srLi6W+vr6pLergtu1axdHHHEEPXr0YPHixZrklUpRxpjFIhLTrdm0klc8\n99xzrFmzhtraWk3wSmUYreSz3O7duxk8eDB77bUXS5cuxePR0zRKpSqt5FXMXnjhBVatWsX8+fM1\nwSuVgbSSz2K7d+/m6KOPJicnh2XLlmmSVyrFJb2SN8YcBzwC7A3sAv4gIu8nsk6VPAsWLGDlypU8\n//zzmuCVylAJVfLGmDeAB0TkNWPMCOBWERke6XNaydvn9Xo55phj8Hq9LF++nJycHNshKaUisNEn\nL0BP/8/7AF8kuD6VJLW1taxYsYJnnnlGE7xSGSzRSv4I4H8Ag+/q2ZNFpDHEsmVAmf/Xo4CP4m7Y\nvn2BL20HkYB0jj+dYweN37Z0j/8wEekRywciJnljzF+BPkHeKgfOABaKyAJjzK+BMhGJOD+tMaY+\n1kOOVKLx25POsYPGb1s2xh+xuyZc0jbGzAOu8//6IvB4LI0rpZRyV6JDKr4AOu7yfDrwaYLrU0op\n5aBET7xeCcw0xuQC3/Jdn3skcxJs1zaN3550jh00ftuyLn4rF0MppZRKDr0CRimlMpgmeaWUymDW\nkrwx5l5jzCfGmA+NMbXGmAJbsUTLGHOWMWaVMWaNMeY22/HEwhjTzxjzv8aYlcaYFcaY6yJ/KvUY\nY3KMMUuNMa/ajiVWxpgCY8x8/3a/0hhzku2YYmGMucG/7XxkjHnWGLO37ZjCMcY8aYzZYIz5qNNr\nPzDGvGmM+dT/3MtmjKGEiD2unGmzkn8TOEpEjgFWAxMsxhKRMSYHmA2cDRwJXGSMOdJuVDHZBdwk\nIkcAJwJXp1n8Ha4DVtoOIk4zgddF5HDgWNLo7zDGHAxcCxSLyFFADnCh3agi+hNwVsBrtwFvicih\nwFv+31PRn9gz9rhyprUkLyJviMgu/6/vAn1txRKlE4A1IrJWRHYAzwEjLccUNRFpEZEl/p+34Esw\nB9uNKjbGmL7AL0nD6zGMMT2BnwJPAIjIDhFptxtVzHKBrv7RdPmk+DQmIvJ3YFPAyyOBp/w/PwWc\nm9SgohQs9nhzZqr0yf8OeM12EBEcDDR3+n0daZYkOxhjBgA/At6zG0nMZgC3Al7bgcShCGgD5vq7\nmx43xnSzHVS0ROTfwHSgCWgBvhKRN+xGFZcDRKQFfIUPsL/leOIVdc50NckbY/7q778LfIzstEw5\nvq6EGjdjcUCw++Kl3fhTY0x3YAFwvYh8bTueaBljzgE2iMhi27HEKRcYAjwsIj8CviF1uwr24O+7\nHgkMBA4CuhljLrEbVXaKNWe6emeoSPPYGGMuA84BzpDUH7C/DujX6fe+pPjhaiBjTBd8Cb5GRF6y\nHU+MhgEl/imt9wZ6GmOqRSRdEs06YJ2IdBw9zSeNkjxwJvC5iLQBGGNeAk4Gqq1GFbv1xpgDRaTF\nGHMgsMF2QLGIJ2faHF1zFjAeKBGRrbbiiMG/gEONMQONMXn4TjrVWY4pasZ3h+4ngJUicr/teGIl\nIhNEpK+IDMD3b/92GiV4RKQVaDbGHOZ/6QzgY4shxaoJONEYk+/fls4gjU4cd1IHXOb/+TLgFYux\nxCTenGntildjzBpgL2Cj/6V3ReQqK8FEyV9FzsA3suBJEZlqOaSoGWNOAf4BLOe7Pu2JIvIXe1HF\nxxgzHLhZRM6xHUss/HdSexzIA9YCl4vIZrtRRc8YcztwAb6ugqXAFSKy3W5UoRljngWG45teeD0w\nBXgZeAHoj++L63wRCTw5a12I2CcQR87UaQ2UUiqDpcroGqWUUi7QJK+UUhlMk7xSSmUwTfJKKZXB\nNMkrpVQG0ySvlFIZTJO8UkplsP8P8pKGY8kpteoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure()\n",
    "plt.scatter(dataArr[labelArr == -1][:, 0],\n",
    "            dataArr[labelArr == -1][:, 1], color=\"red\")\n",
    "plt.scatter(dataArr[labelArr == 1][:, 0],\n",
    "            dataArr[labelArr == 1][:, 1], color=\"blue\")\n",
    "plt.plot(np.array([-2, 12]),\n",
    "         ((-np.array([-2, 12]) * w[0] - b) / w[1]),\n",
    "         color=\"black\")\n",
    "for i in range(len(alphas)):\n",
    "    if alphas[i] > 0.0:\n",
    "        if labelArr[i] == -1:\n",
    "            plt.scatter(dataArr[i][0], dataArr[i][1],\n",
    "                        color=\"red\", edgecolors='black', s=100, linewidths=2)\n",
    "        else:\n",
    "            plt.scatter(dataArr[i][0], dataArr[i][1],\n",
    "                        color=\"blue\", edgecolors='black', s=100, linewidths=2)\n",
    "plt.xlim(-2, 12)\n",
    "plt.ylim(-8, 6)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-1.4069805750864268\n",
      "-1.0\n"
     ]
    }
   ],
   "source": [
    "print(np.dot(dataArr[0], w) + b)\n",
    "print(labelArr[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Implement SVM (with kernel)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Kernel transformation function\n",
    "def kernelTrans(X, A, kTup):\n",
    "    # kTup is a generic tuple that contains the information about the kernel\n",
    "    m, n = X.shape\n",
    "    K = np.zeros(m)\n",
    "    if kTup[0] == 'lin':\n",
    "        K = np.dot(X, A)\n",
    "    elif kTup[0] == 'rbf':\n",
    "        for j in range(m):\n",
    "            deltaRow = X[j] - A\n",
    "            K[j] = np.dot(deltaRow, deltaRow)\n",
    "        # Element-wise division\n",
    "        K = np.exp(K / (-1 * kTup[1] ** 2))\n",
    "    else:\n",
    "        raise NameError(\"Houston We Have a Problem -- \"\n",
    "                        \"That Kernel is not recognized\")\n",
    "    return K\n",
    "\n",
    "\n",
    "class optStruct:\n",
    "    def __init__(self, dataMatIn, classLabels, C, toler, kTup):\n",
    "        self.X = dataMatIn\n",
    "        self.labelMat = classLabels\n",
    "        self.C = C\n",
    "        self.tol = toler\n",
    "        self.m = dataMatIn.shape[0]\n",
    "        self.alphas = np.zeros(self.m)\n",
    "        self.b = 0\n",
    "        self.eCache = np.zeros((self.m, 2))\n",
    "        self.K = np.zeros((self.m, self.m))\n",
    "        for i in range(self.m):\n",
    "            self.K[:, i] = kernelTrans(self.X, self.X[i], kTup)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Changes to innerL() and calcEk() needed to user kernels\n",
    "def calcEk(oS, k):\n",
    "    fXk = np.dot(np.multiply(oS.alphas, oS.labelMat), oS.K[:, k]) + oS.b\n",
    "    Ek = fXk - oS.labelMat[k]\n",
    "    return Ek\n",
    "\n",
    "\n",
    "# Full Platt SMO optimization routine\n",
    "def innerL(i, oS):\n",
    "    Ei = calcEk(oS, i)\n",
    "    if ((oS.labelMat[i] * Ei < -oS.tol and oS.alphas[i] < oS.C)\n",
    "            or (oS.labelMat[i] * Ei > oS.tol and oS.alphas[i] > 0)):\n",
    "        # Second-choice heuristic\n",
    "        j, Ej = selectJ(i, oS, Ei)\n",
    "        alphaIold = oS.alphas[i]\n",
    "        alphaJold = oS.alphas[j]\n",
    "        if oS.labelMat[i] != oS.labelMat[j]:\n",
    "            L = max(0, oS.alphas[j] - oS.alphas[i])\n",
    "            H = min(oS.C, oS.C + oS.alphas[j] - oS.alphas[i])\n",
    "        else:\n",
    "            L = max(0, oS.alphas[j] + oS.alphas[i] - oS.C)\n",
    "            H = min(oS.C, oS.alphas[j] + oS.alphas[i])\n",
    "        if L == H:\n",
    "            # print(\"L == H\")\n",
    "            return 0\n",
    "        eta = 2.0 * oS.K[i, j] - oS.K[i, i] - oS.K[j, j]\n",
    "        if eta >= 0:\n",
    "            # print(\"eta >= 0\")\n",
    "            return 0\n",
    "        oS.alphas[j] -= oS.labelMat[j] * (Ei - Ej) / eta\n",
    "        oS.alphas[j] = clipAlpha(oS.alphas[j], H, L)\n",
    "        # Updates Ecache\n",
    "        updateEk(oS, j)\n",
    "        if abs(oS.alphas[j] - alphaJold) < 0.00001:\n",
    "            # print(\"j not moving enough\")\n",
    "            return 0\n",
    "        oS.alphas[i] += (oS.labelMat[j] * oS.labelMat[i]\n",
    "                         * (alphaJold - oS.alphas[j]))\n",
    "        # Updates Ecache\n",
    "        updateEk(oS, i)\n",
    "        b1 = (oS.b - Ei\n",
    "              - oS.labelMat[i] * (oS.alphas[i] - alphaIold) * oS.K[i, i]\n",
    "              - oS.labelMat[j] * (oS.alphas[j] - alphaJold) * oS.K[i, j])\n",
    "        b2 = (oS.b - Ej\n",
    "              - oS.labelMat[i] * (oS.alphas[i] - alphaIold) * oS.K[i, j]\n",
    "              - oS.labelMat[j] * (oS.alphas[j] - alphaJold) * oS.K[j, j])\n",
    "        if 0 < oS.alphas[i] < oS.C:\n",
    "            oS.b = b1\n",
    "        elif 0 < oS.alphas[j] < oS.C:\n",
    "            oS.b = b2\n",
    "        else:\n",
    "            oS.b = (b1 + b2) / 2\n",
    "        return 1\n",
    "    else:\n",
    "        return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Full Platt SMO outer loop\n",
    "def smoP(dataMatIn, classLabels, C, toler, maxIter, kTup):\n",
    "    oS = optStruct(dataMatIn, classLabels, C, toler, kTup)\n",
    "    n_iter = 0\n",
    "    entireSet = True\n",
    "    alphaPairsChanged = 0\n",
    "    while n_iter < maxIter and (alphaPairsChanged > 0 or entireSet):\n",
    "        alphaPairsChanged = 0\n",
    "        # Go over all values\n",
    "        if entireSet:\n",
    "            for i in range(oS.m):\n",
    "                alphaPairsChanged += innerL(i, oS)\n",
    "                # print(\"fullSet, iter: %d i:%d, pairs changed %d\"\n",
    "                #       % (n_iter, i, alphaPairsChanged))\n",
    "            n_iter += 1\n",
    "        # Go over non-bound values\n",
    "        else:\n",
    "            nonBoundIs = np.nonzero((oS.alphas > 0) * (oS.alphas < C))[0]\n",
    "            for i in nonBoundIs:\n",
    "                alphaPairsChanged += innerL(i, oS)\n",
    "                # print(\"non-bound, iter: %d i:%d, pairs changed %d\"\n",
    "                #       % (n_iter, i, alphaPairsChanged))\n",
    "            n_iter += 1\n",
    "        if entireSet:\n",
    "            entireSet = False\n",
    "        elif alphaPairsChanged == 0:\n",
    "            entireSet = True\n",
    "        print(\"iteration number: %d\" % n_iter)\n",
    "    return oS.b, oS.alphas"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Experiment 1: Toy dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Radial bias test function for classifying with a kernel\n",
    "def testRbf(k1=1.3):\n",
    "    dataArr, labelArr = loadDataSet('testSetRBF.txt')\n",
    "    b, alphas = smoP(dataArr, labelArr, 200, 0.0001, 10000, ('rbf', k1))\n",
    "    # Create matrix of support vectors\n",
    "    svInd = np.nonzero(alphas > 0)[0]\n",
    "    sVs = dataArr[svInd]\n",
    "    labelSV = labelArr[svInd]\n",
    "    print(\"there are %d Support Vectors\" % sVs.shape[0])\n",
    "\n",
    "    plt.figure()\n",
    "    plt.scatter(dataArr[labelArr == -1][:, 0],\n",
    "                dataArr[labelArr == -1][:, 1], color=\"red\")\n",
    "    plt.scatter(dataArr[labelArr == 1][:, 0],\n",
    "                dataArr[labelArr == 1][:, 1], color=\"blue\")\n",
    "    plt.scatter(sVs[labelSV == -1][:, 0], sVs[labelSV == -1][:, 1],\n",
    "                color=\"red\", edgecolors='black', s=100, linewidths=2)\n",
    "    plt.scatter(sVs[labelSV == 1][:, 0], sVs[labelSV == 1][:, 1],\n",
    "                color=\"blue\", edgecolors='black', s=100, linewidths=2)\n",
    "    plt.show()\n",
    "\n",
    "    m, n = dataArr.shape\n",
    "    errorCount = 0\n",
    "    for i in range(m):\n",
    "        kernelEval = kernelTrans(sVs, dataArr[i], ('rbf', k1))\n",
    "        predict = np.dot(kernelEval, np.multiply(labelSV, alphas[svInd])) + b\n",
    "        if np.sign(predict) != np.sign(labelArr[i]):\n",
    "            errorCount += 1\n",
    "    print(\"the training error rate is: %f\" % (errorCount / m))\n",
    "    dataArr, labelArr = loadDataSet('testSetRBF2.txt')\n",
    "    errorCount = 0\n",
    "    m, n = dataArr.shape\n",
    "    for i in range(m):\n",
    "        kernelEval = kernelTrans(sVs, dataArr[i], ('rbf', k1))\n",
    "        predict = np.dot(kernelEval, np.multiply(labelSV, alphas[svInd])) + b\n",
    "        if np.sign(predict) != np.sign(labelArr[i]):\n",
    "            errorCount += 1\n",
    "    print(\"the test error rate is: %f\" % (errorCount / m))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "iteration number: 1\n",
      "iteration number: 2\n",
      "iteration number: 3\n",
      "iteration number: 4\n",
      "iteration number: 5\n",
      "iteration number: 6\n",
      "iteration number: 7\n",
      "there are 88 Support Vectors\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAD8CAYAAABzTgP2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJztnXuQXGd14H+nJdlCTgWNH3iErJE0\nu9hgYBGycCBTFd42ssiIoQfHbFhMAlGRSIpqYwosHJaUxjIO2SpC1ErAa4jJisLG0x5rEuI4EOOi\nIlnEY0nYYFZYbks9imdkY0tQYOLY7rN/3Nua2z39uN330fd2n1/Vre6+z9O3b3/n+87rE1XFMAzD\nMMpkOi2AYRiGkSxMMRiGYRgVmGIwDMMwKjDFYBiGYVRgisEwDMOowBSDYRiGUYEpBsMwDKMCUwyG\nYRhGBaYYDMMwjAoWdlqAdjj//PN11apVnRbDMAwjVTz00EM/VdULmu2XSsWwatUqpqamOi2GYRhG\nqhCR4372M1OSYRiGUYEpBsMwDKMCUwyGYRhGBaYYDMMwjApMMRiGYRgVmGIwDMMwKjDFYBiGYVSQ\nyjwGw4iTYrHI5OQkp0+fZunSpWzcuJEVK1Z0WizDiAxTDIZRh9nZWbZs2cLExASlUunM+m3btjEy\nMkIul6O/v7+DEhpGNIRiShKRr4rIUyLywzrbRUT+SkSOisjDIrLWs+1aEXnMXa4NQx7DCMrs7CxD\nQ0Pk83kWlEpkgU8DWSBTKpHP5xkaGuLkyZMdltQwwicsH8NtwHsabF8PvMpdNgF/AyAi5wKfBX4D\nuBz4rIj0hSSTYbTNli1bKBQKrAUKwDiw0319Apz1hQKbN2/uoJSGEQ2hKAZV/R7wbINdNgJ/pw4H\ngKUisgy4Evi2qj6rqqeAb9NYwRhG5BSLRSYmJlgE7AUuqtp+EXA3jh12YmKC6enpuEU0jEiJKypp\nOeD995xw19VbbxgdY3JyklKpxDDzlUKZFTi9nVKpxOTkZHzCGUYMxKUYpMY6bbB+/glENonIlIhM\nPf3006EKZxheTp8+DcAlTfa72H09depUpPIYRtzEpRhO4HSyylwEPNlg/TxU9RZVXaeq6y64oGk5\nccNom6VLlwJwpMl+P3Ff+/rMLWZ0F6Jas4Pe+olEVgH/oKqvq7FtA7AFuArH0fxXqnq563x+CMeX\nB3AQuExVG/krWLdundp8DEZUFItFVq9ezYJSiQK1zUnTwCBQymQ4duwYqmq5DkbiEZGHVHVd0/3C\nUAwi8g3gbcD5wEmcSKNFAKr6JRERIIfjWH4O+D1VnXKP/X2cSECAnar6t82uZ4rBiJrR0VHy+Txr\ncRzN3iZ+GngfTi9mw4YNLF68eF6uQyaTsVwHI3HEqhjixhSDETXlPIZCocBCHEfzxTjmo73Ai8DK\nlStRVYrFIouAYRy/xBHPPoODg+zfv58LL7ywM1/EMDz4VQxWK8kwatDf38++ffvIZrOUMhnywOeA\nPI75KJvN8rrXvY5isWi5DkbXYSMGw2jC9PQ0k5OTnDp1ir6+PoaHh1HVlv0Q5nMwOo3fEYPVSjKM\nJqxYsWJerz+Xy1EqlRihea5D3s11sJGDkRbMlGQYbWC5DkY3Y4rBMNrAch2MbsZ8DIbRBu3kOjTy\nMdicD0YcmI/BSARBGrwkN5YDAwOMjIyQz+fZSP1chxeB7MhIXbltzgcjkahq6pbLLrtMjWQzMzOj\n2WxWM5mM4tS/UkAzmYxms1mdmZmJ5Ng4mZmZ0cHBQQV0IWgWdLv7utCVeXBwUGdnZ5sev8g97tMt\nHG8YrQJMqY82tuONfDuLKYZkE6TBS1tj2UyJNZIzm80qoGtBp8H5O7rLtLse0Gw2G+M3MroZv4rB\nfAxG6HjLSVTPZ3ACJ4TzIJDNZhkfHw/t2E5SK9ehmU/B8iCMuPHrY+h477+dxUYMyeX48eOayWR0\nUY1ecHkpur3/TCajxWIxlGPTxq5du5zRQJ3vWV6y7qghl8t1WmSjC8DniMHCVY1QCTLJTS9NkGN5\nEEaSMcVghEqQBq+XGkvLgzCSjCkGI1SCNHi91FgODw+TyWSYxPGd1GIax8+SyWQYHh6OTzij5zHF\nYIRKkAavlxrLch7ECzimsemq7d48iJEGeRBeisUiuVyOG2+8kVwux/R09VkNwyd+HBFJW8z5nGy8\nYZjFGs7jRmGYQY5NG0HzILznSUPeh9F5sDwGo1MEafDCaizTQpA8iPLxacr7MDqLKQajowRp8II2\nlmmkWCxqLpfTsbExzeVyvkNxLUnOaAW/iiGsOZ/fA3wRWADcqqo3V23/AvB29+MS4BWqutTd9hLw\niLutqKpNDceW4JYeWk38CuvYXsCS5IxWia2InogsAHYD78bxGT4oIpOq+mh5H1X9n579twJv9Jzi\nV6q6JqgcRjKpNclNHMf2AuW8j1YmC/rt3/7txBYmNJJDGNVVLweOqmoBQERux3kWH62z/weBz4Zw\nXcPoaVrN+/jrv/5r/viP/9iquBpNCSNcdTmV0XYn3HXzEJGVwGrgPs/qxSIyJSIHROR99S4iIpvc\n/aaefvrpEMQ2jHTjN+/jYff10UcfZUGpRBb4NJAFMqUS+XyeoaEhTp48GZ2wRqoIQzFIjXX1HBfX\nAOOq+pJn3YBr8/rvwF+KyH+pdaCq3qKq61R13QUXXBBMYsPoAvzmffyj+34tUADGgZ3u6xPl9YWC\nme2MM4ShGE5QOUfJRcCTdfa9BviGd4WqPum+FoD7qfQ/GIZRBz9Jclfh9NIWMb9aLe7nu3FsyhMT\nE5YUZwDhKIYHgVeJyGoROQun8Z9X3UxELgH6gAc86/pE5Gz3/fnAEPV9E4ZhVJHL5RgcHOQgTvTR\nKI6ZaNT9/EN3v14oTGiER2Dns6q+KCJbgHtxwlW/qqo/EpEdODGz5Sftg8DtWhkf+xrgyyJSwlFS\nN3ujmQyjmiRP99kJ+vv72bdv35npQfMex3Imk+HSV7+aRx99tCcKExrhEcqcz6r6j8yZMsvr/lfV\n5z+rcdx+4PVhyGB0NzY3cn36+/sZHx+vmfexd+9etm7dGklhQlPS3YvN4GYkntnZWYaGhigUCizC\nMYtcghONsxen0Nzg4CD79+/nwgsv7KSoiSOKJLh6SjqTySReSfe6MrMZ3IxEcvz4cd21a5eOjY3p\nrl27fJV+sLIPwQizMGFaazNZoUEHrFaS4Zd2GutWafeP2UvTfUZFaIUJ9+zR7JIlqVPSaVVmUWCK\nwWhKXL2oIH9Mmxs5HAIXJtyzR48vXqwZ9zdMk5K2EeccphiMhsTZiwryxxwbG1Nc2Rophu3uOcbG\nxgLL2820W8VVV67UXeXfKUVK2kaclfhVDKFEJRnpY8uWLRQKBdYyP/HpBE5c+0E3G3Z8fLzt6xSL\nRSYmJpomWA0yl2DldQb20nSfcdB2YcJikdPu2zSFvrZTaNAywG1qz57Eb2MdRjZs+Y/ZboJVL033\nmWgGBljqvk2Tkm610GASlFkSMMXQgwRtrFsh6B8zirmR005H5nbeuZPhxYvJ4JQ1+D6QA250X8sS\nJE1J24izPcyU1IPE2YsK44+Zy+U4dOgQBwsFBnEUxMXuMd48ht27d7ctZxroaJLf7/4uA8D6P/gD\nvvWrX/Hmqs3bgCtxRnUvAtmEKOnh4WG2bdvGZKnECerncSRJmSUCP46IpC3mfA5GnJE+YTn/enG6\nTy9JCLmcmZnRlStXNpQB0IGBgUT9HmHmcaQdLCrJqKacr3DdddepiMQWqRHmH7PtqJqUk4SQS78y\nbNiwITIZ2iG0PI4uwBSDcYZ6ve24elH2xwxGEkIukyBDEHp9xFnGFIOhqvVNEBs8f444Gmv7Y7ZP\nq6a/D3zgA6E3zN2SaNirI84yfhWDOZ+7nEb5CgeBdwKngbxnfSaTITsywu7du0MrSteoAmgSnJR+\n6UQRtlaDBe68807y+XyoDuluCftsO4+j1/CjPZK22IjBH36H/wtARUQ/8YlPhNeL2rNHdeVKVRHn\ndc+e4OfsIJ0swtZqb31NBKO+bhkx9DqYKcno2J95zx7VJUsqr7NkSYVyiKNwX1h0OiKoJfu++z5s\nh3TafQyGgykGo3N1hlaurH2tlStTWf64UxFBXuW5Zs0af8ECETbUFvaZfkwxGJ0bMYjUvM6M27Pu\nVM+7HTrRU24URQYNggVAZyP8bS26LP3EqhiA9+Aktx4Frq+x/SPA08Bhd/mYZ9u1wGPucq2f65li\n8EfUjVpdc1CdEUMaa/nHrVwbma0W1FAS4JiPsjWUQhSjQYsuSzexKQZgAfA4ToHMs4AfAJdW7fMR\nIFfj2HOBgvva577va3ZNUwz+iWL439QclMvN8zEcX7xYMzEm1YVF3OY4v2ar5cuXa9nRXP27Rjoa\ndOn1sM+0EqdieAtwr+fzdmB71T71FMMHgS97Pn8Z+GCza5pi8E/Yw3/fjtjduyuiknZ9+MOx9rzD\nIqnlQ0TEnMEJJcmBFXEqhlHgVs/n/1GtBFzFMAM8DIwDK9z1nwD+1LPfZ4BP1LnOJmAKmBoYGIjy\n3nUdYQ7/23XEpnXCnTh9DC2Hpfp1SCfINNfNpCGwwq9iCCPBTWqs06rPfw98Q1WfF5GPA18D3uHz\nWGel6i3ALQDr1q2ruU+v0izpKqzksiCT7qS1/HG57Hc+n2cjzvfz3jFv2e+gFUVbTSK74oor+PnP\nf97zVWfrEWcy4uzsLENDQxQKBRYBIzi/4xFgb6lEPp/n0KFD7N+/P7Sk0Ujxoz0aLfgwJVXtvwD4\nmfveTEkBiLuHEsSskuY4+Liicdq5v+YMnk8neu5JKHLoB2I0JS3EcRqvZs75/NqqfZZ53o8AB9z3\n5wJP4Die+9z35za7pikGn7Z+0NmLLgot6zioOSjNcfBxNMBBlKc5gx06kYzY7Hc7DroDJ3pMRPTA\ngQOhXbtVYlMMzrW4CmcE+zhwg7tuBzDsvv8c8CNXaXwXeLXn2N/HCXM9Cvyen+uZYmihhwLzso7b\nJagjthvi4KNugNOsPJNAJ3ru9f4XM+66jKcjgascOuVziFUxxL30umJop0SCrlwZ73XrmIPM9NGY\nblCeUeAn0qdT5spaI+kZnBF7nCMXP5hi6GJa7rmDEzYaAmH1aHvJ9NFq+KIpzzla8Rd0KtO/1nXL\n10iaz8EUQxfTsq0/pBGDqvVoWyGoEzRu5Zm0+PtW/QWdComuHqkcxxmpJzHQwhRDF9PyxC2ZjBa/\n+MXQrm892uZ4GzVYpJBV+LT7ujBRCjSp8fet+gs6WRrcK+uOslwdkKMZphi6mFZsqVH+yXvJHFTG\nb6+63FDAWoXpqp9n2l3feSdyp0uK16Mdf0EnQ6K997HsbE5iMqcphi7Hr63/kgT8ybuBVnrV5QbK\nGSlUK4XyUlRY2PGcjaTG37fb++9kVFf5GRERGzF0YjHF4NPWz1zFzSQl2aSNVs1C5UbN2d6obcjG\n3jB4SXLSYbv+giT4wB544AGVhBaM9KsYMhippL+/n3379pHNZillMuRxkkXyQAnIAvuBcvJ9uVzF\nQubKVRj+KM+bDWtxcjnHgZ3u6xPAWgqFwpm5hMulLfwWt+jU/MiTk5OUSiWGmV/epMwKnFIbpVKJ\nycnJ2GRrt4RKw/9FJkM2m428LMWb3/xm3v/+9/MCzr2r/qd5y6iMBCyjEhVh1EoyOoS3BtJ1113H\nnXfeyRpgksp6PmXKf/K8+ye3SdGbU64PhY8KUWWFW27U/DZrnaoL1WptpjgV2PDwMNu2bWOyVOIE\ntRXXNM4vkslkGB4ePrM+jNpgQess5XI5Dh06lN46Vn6GFUlbzJQ0n7RWL0067ZiF0uJj6GQUjx86\n4S8IM0IridF7xFhd1UgAaa1emnTaMQt5K7LSpCbryEi2Y6aEIL3yOIi71x12hdSwqhp3BD/aI2mL\njRjmk2RHYppp15Fc6bBe6G7fXtdh3SmSXpspzl53UiO0wgSLSuo9kv4nD5O4snSDmIWSaEqoJglR\nPH6IOmemVzpWphh6kLT8yYPQyVr7TlJasYZSaJyslvREwDQosKhJur8lLEwx9Cjd/CfvVJmJtJiF\ngpJ0BRYlvRK84VcxmPO5y0i1w6sJlfkE1aGjJ4CNFAoH2bx5M+Pj46Fdtxwbv2XLFiYmJiiV8me2\nZTIZRkay7N69Ox1TNjZgxYoVPRvCbMEblYijRNLFunXrdGpqqtNiGDFSLBZZvXo1pdICnCSzejE0\ng2QyJY4dOxaJIkyywo1zjuNuo/x8LSiVmjxdTqJcVM9X1IjIQ6q6rumOfoYVSVvMlNR7pKXMRCdI\nanXUtNELwRuYKcnoJtJSZiJuvLH3VEXfl0p7W46972VSn60cIqHUShKR94jIERE5KiLX19j+JyLy\nqIg8LCL/IiIrPdteEpHD7hJfMRYjVaSlzETctFrHyahPEuosJYXAPgYRWYDzb3w3jgfwQeCDqvqo\nZ5+3A99X1edE5A+Bt6nq77jbfqGqv9bKNc3H0HskxceQJOyeREeSfUlB8OtjCMOUdDlwVFUL7oVv\nxxmFnVEMqvpdz/4HgA+FcF2jh0hLmYk4KVdHdcxHjeujlkr5riicGJeDvZcjtCAcxbCcysqyJ4Df\naLD/R4F7PJ8Xi8gUjgnvZlW9u9ZBIrIJ2AROI2H0HmUbcKFwEOpYgZNsAw67Ueslv8vs7KwnXLh0\nZv22bdsYGRkhl8vR39/fQQm7izAUg9RYV9M+JSIfAtYBb/WsHlDVJ0VkELhPRB5R1cfnnVD1FuAW\ncExJwcVOBhZi6J+05hNE1aj1it8l7OJ2RnPC8DG8BfgzVb3S/bwdQFU/V7Xfu4BdwFtV9ak657oN\n+AdVbZid1A0+hnqNhdPAJaMHlGSlFYcNOIzvPz9qaBhPs0Z5lNNOo9YrPobR0VHy+XyDtEY4CGSz\n2VATG7uR2PIYcEYdBWA1cBbwA+C1Vfu8EXgceFXV+j7gbPf9+cBjwKXNrpn2PIakTsDula+X4+LD\n/P6VdZaqi/BNN62z1Nr5W6/jlHR6pbhdXBBnrSTgKpzx6uPADe66HcCw+/47wEngsLtMuut/E3jE\nVSaPAB/1c720K4Ykl/ftVD2ipND4+y9QQPv6+nTnzp1NG6E4Juzp9jpOvVLcLi5iVQxxL2lWDEnv\nAfnt4a5ZsybyktedoPb3n3Eb2tZGEHFla3dz4cReKW4XF34Vg2U+x0w5xLB5gGH8czO3Mr/x4cOH\nOXz4MBBdZEjcPo7a338WGMKxlraWWRxX1FA3F0604nadIZTMZ8M/SZ6AfS4ufpjmagvgCiBLqZQh\nn88zNDTEyZMnA8sxOzvL6Ogoq1evZuvWrXzmM59h69atrFq1itHRUWZnZwNfoxa1v/8WHKXQemZx\nnFFDxWKRvXv3curUKZYuXdoVSgGc6UczmQyTOI7mWnRy+tGuxc+wImlLJ01JQWcOS7LNtDxsd2zq\njcTb7u43Ns/EFNQv0kkfx/zvf9w1HzX3ETj7FXXlStU9e5zzxeVj6PZAgV4obhcXmI8hXML6AybZ\nx9CqTRxyoTVwZaKO4mnE/O/f3v1YsmROOUQZNdQrgQK9MDNhXJhiCJGww0uT2gNqpYdb7iHXaiDb\nHeXE0cNu7frtjqBUV650zhll1FAnlWjcdLODPU5MMYRIu+Gl9cxOSe4B+e3h1u5FOw1ku5EhSZhz\nofL772hJHu8ISmTunFE0ap1Wop2il6cfDQNTDCHRjunHj9kpqT0gPz1cGFSYDb3BbtXHEUVoYuX3\nX6Agvhrf6hFUecTgJcxGLQlK1EgffhWDhas2odXw0j179nDrrbf6quuSxBDDRvWIHK4CvgpUl29w\nYkOCRIYkofbP/O+vwAs0q+gK2TPbliyBnTvnnzvMip29VEDP6AB+tEfSljhHDK0m2Fy6aJHShtkp\niXh7uGvWrPGYWKIpvdAJ80ijKLNisag33XST9vX1NRxBveIVg3rRRbMqohVRSVFiIwajHTBTUji0\nGl4qkMiIo6DEVXohrto/rUSZJdHs16s+BiMYphhCohUfg5RHAz6VSNp6cXE0kHEooHbDPIP4CILm\nv9Si2wvoGeFjiiFE/IaXXuq+dntdl6gjQ6JWQHGGeUaZgNbtBfSM8DHFECK+wktBd3b5iCFuolBA\ncZpg4khAe/DBB/UNb3iDikhizFxGcjHFEDJ1e34iml2wQGdBj4Nm6E4fQ7cQp9M2ypFJvedRRHTN\nmjU6NTXVttxG9+JXMVi4qk8aVrD83vfghhugWGTkZS8j/9xzzYMbR0a6oshZ2ogrzPOBBx7grrvu\nwqlTOQLzZrudq1Q7MTHB9PS07+dh/qxwc0HRqns5fPgwV199tU11abSPH+2RtCXJ8zEkOas5rYTp\nuA06YtizxwlJrReaWu7JV5t2nAS4rDpzOwQbmfRSKQwjXDBTUudIYnhjGonCcRvEx7Bnj+pZZ1Xu\ne9ZZc8rBj09hftZ4a1ncFqZqBCFWxQC8Bydd9ShwfY3tZwN3uNu/D6zybNvurj8CXOnneklXDGWs\nrkv7ROm4bTfM87zzao8uzjuv1nnr9+QrRyutjRgssc0IQmyKAViAM9fzIHAWzvzNl1bt80fAl9z3\n1wB3uO8vdfc/G1jtnmdBs2umRTEY7RO147adMM9GDXF7lWlb79knoZ6UkV78KoYwZnC7HDiqqgVV\n/U/gduam+CqzEfia+34ceKeIiLv+dlV9XlWfwBk5XB6CTEYIFItFcrkcN954I7lcjunp6diu62+K\n0YVnHLet8prXvMZ99yKQBz7nvr7Ehg0bKhy3X/86rFrV+HytzX5Xwvk7OKEIIy0EImQy5b9s5+pJ\nGd1PGFFJy3ECbsqcAH6j3j6q+qKI/Aw4z11/oOrY5SHIZARgdnbWU0SudGZ9VHM7VzPXyDYvXVgq\n5VuaF3t+RM+7gJdwHr3/B5T48Y9/fGb/r38dNm2C556rdbYiMMnLXnaaf/qn8mPsd9LWzwIlBgcH\n2b17ty/ZAf71X//VfVee7LLW/Qle0NDobcJQDFJjnfrcx8+xzglENgGbAAYGBlqRry5xTzafBhqF\nQpZKeyuqw0YVChllSOmWLVvc77aW+aORE8BGCoWDbN68mfHxcW64oZZSmMWZC3oCKPGrX8G3vlXe\n9jVgK1BPcTo9eRHl/e/Psnv3bt/3sVgscu+99+L8bZpXfL3yyvWRPM/2v+kB/NibGi3AW4B7PZ+3\nA9ur9rkXeIv7fiHwU5ynu2Jf736NlqA+hl6YJ7ddkhAKGZWDtZ2IHpHq7TPqRBY1ijwa0NrzVTjn\nFhE9cOBAgPuywSNDvTkz0JtuuqnlazSiF/43UdS0ShLE6HxeCBRwnMdl5/Nrq/bZTKXz+Zvu+9dS\n6XwuELHzOexpOruJpIRCRiVHOwpn5cra2xwFeUCdeaHH3Nfv61zk0foa8gZTqpWO5xlXlspG2vl8\nieJxPLfT2FUfMzU11dXzS/eC0lONUTE41+IqnDHy48AN7rodwLD7fjFwJ45z+d+AQc+xN7jHHQHW\n+7leEMXQ7jSdvUCSQiGjqBzaTkTPnj2qS5aU1x93G96FClfVaZTXqzPzW1k5zJ/9rt3Gs/bvU1Rn\nStEx97V45ve56aabWm7s6jWQc0v3JdX1UmcxVsUQ99KuYmhnms5eIkmhkFFUDm1V8b33ve/VXbt2\n6Re/WHRHDuXjz9HGpqTy9vmZzxdd1H7jcvz4cRXxN5ISER0YGGgoZ/X9q587st6zLllJdWGYfnqp\ns2iKoQatTrrTa8lBSRoxqIafQd5arsH8633yk5/0rG+WxIZmMu9Vb09+8eLj+uEPt9+I7dmjumCB\nv5HU8uXLfcnpbezq+5eS9Vyohmf66bXOoimGGrQ6TWevJQclxcdQTZgZ5H5NVLBcq3vYS5cuVb89\nZ0BHR8fckcaMLlmSdXv77Tdi5XM1czwPDAy0/Ds2/u2TM5JUDdf0E1ZnMS1Oa1MMNbARQ3O6fVYw\nPyaq+VFFlSMBvz3nsbGxUEt7zEVI1XY8lxXNzp07W5Izl8s1GS0ma8QQpuknaGcxbU5rUww16LVh\nYzu0attPS0/JS2MH61VaP9S0vL//nnOY4b/zI6Qcx/PLX+6MpOZ8Ia338Bv7l8pO986PJMP+Dwfp\nLKbRaW2KoQ5+p+lMa484DPzY9sO08XZKsZRNVBs2bHDlrw4xrV7eoK30nHfs2BGqaa4yQspZlixx\n1ldua72H39y/lIyRZNij/iCKJo1Oa1MMdbD5EvxTz7YfhnkkSUNw/9FYf+T5zs0b+h07drTcQDej\n3nwQlaOJ1nv4zf1LM+qY2MKLEmuHKPyE7XQWo7Y+RNVhMsXQAJsvIRhBzSNxzIVc749Va32r0Vh+\ne85xhv/Oz9BuvYff3L/0end75+aXjsJP2E5nMSp/ZdQdJlMMPrD5ElonjMilTs2FvHz58pp/uPXr\n17szrrWaH9C45xw0/LeVXuN8/0Pz6KXGeQz1j3nooYc69r+JqqfeamcxipFLHD4LUwxGJITR2EUV\nElt/JFL2IdQfoZxzTjkprXkP228j0u53bXb+Bx98cJ7CqOV/eNnLZvRNb2ptZNzo2uvXr9edO3d2\nPNAgSj+h385iFCOGOHwWphiMSAhqHokyia7+SMTfCGVOOfjrYftpRFoN//U3PWjlUm7oc7mZmv6H\ndkbG3mNuuukmXb9+fSL8QarJ8BOGPXKJK2LSFIMRCUEb9qjs7vV75605Yhs1gO00NK2G//qfHvRC\nhSsULtVyGG0UjWEc/qB25eq0nzDMkUtcOVamGIxICGoKimrEUP+8rV8vbN9TFKanWiMHQDds2BBI\n1mriLsPeajROJ/2EYY5c4qrKYIrBiIwg2dFR+Rjqj0SSU86hWSPWenTUGq1lZnrooYdCkTfOEilJ\nCl9uhbBGLjZiMMWQeoJWPo2i7EaYI4ZO0aqZzVF65XVzvfc1a9aEIk9cRRXTmEFcTdCRi/kYTDF0\nBUF6SlGU1A7Lx9DJkOXWRwy5mt9DREL5HnHlYaQxgzgK4qjKYIrBiIV2e0pROA/rj0SSUc6hGa35\nGDI1vsvcdw1j5BPHiMHql80RR7SVKQYjFYTpPKw/EvHmMXSunIMf/JcFr9dYh+cricPHYBWPK4k6\n2sqvYliIYXSQFStWsHnz5lCbNFAFAAAUgklEQVTO1d/fz759+9iyZQsTExOUSvkz20SEV77ylczM\nzFSsz2QyjIxk2b17NxdeeGEocgQhl8tx6NAhCoWDwCCwEbgYZ+bcvcCL7vrddc7wEwD6+voCyzIw\nMMDIyAj5fN6V425ghWePaeB9wIuMjGRZsWJFrdM05PTp0wBc0mS/i93XU6dOtXyNNNHf38/4+DjT\n09NMTk5y6tQp+vr6GB4ebuv+tksgxSAi5wJ3AKuAY8DVqnqqap81wN8Avw68BOxU1TvcbbcBbwV+\n5u7+EVU9HEQmo7dp9sfq9B+uGY2Um4MA40AtJTYN7CWTyTA8PByKPH4U1eDgILt311NUjVm6dCng\nTPjeiJ+4r2EovDQQZoepHcQZXbR5sMjngWdV9WYRuR7oU9VPVe1zMaCq+piIvBJ4CHiNqp52FcM/\nqOp4K9ddt26dTk1NtS23YaSBaiV21113cd999wFrqd97P0g2m2V8vKW/VENmZ2c9iqp0Zr0z2hoJ\nNNoqFousXr2aBaUSBeCiGvtM46ikUibDsWPHEqXI04aIPKSq65ruF1AxHAHepqozIrIMuF9VG44K\nReQHwKirKG6jxxRDsVhkcnKS06dPs3TpUjZu3GgPuuGL2dlZhoaGKBQKOIP92r33/fv38/zzz4f+\nnEU12hodHSWfzzdRd4Su8HoRv4ohkBMYOF31+VST/S8Hfgxk3M+34YwiHwa+AJzt57ppdD6nNYHH\nSBbNnqOHH344dc9ZEmof9Qr4dD43HTGIyHeA/hqbbgC+pqpLPfueUtWaRsDyiAK4VlUPeNbNAmcB\ntwCPq+qOOsdvAjYBDAwMXHb8+PGGcicJb09vETCM42w7gsed6Pb0kuAANZJPrd77okWLPCOK2k9a\nUp+zKM1VxhxxjRiOAMvc98uAI3X2+3Wc0eAHGpzrbThmpa4bMVgCjxEHcdc1igKbIyVaCGvE0ET7\n/AXwjM45n89V1U9W7XMWcA/w96r6l1XblqnjnxAcU9J/qOr1za6bJh+DOdeMOCg/Z6XSAmjypGUy\nJXvOehS/I4ZMwOvcDLxbRB4D3u1+RkTWicit7j5XA78FfEREDrvLGnfb10XkEeAR4HzgxoDyJI7J\nyUlKpRLD1P6rguNs2wiUSiUmJyfjE84InWKxSC6X48YbbySXyzE9PR3LdcvPGT6eNHvOjGYEymNQ\n1WeAd9ZYPwV8zH2/B9hT5/h3BLl+EqmOOir7QtKQwGMRU+1Tz0a+bds2RkZGyOVy9PfXctWFQzlR\nzO+T1u2JYkYwLPM5JOo1DI6VLNkJPJ1u1NJOZRjpImCEstO3VNpLPp/n0KFDkTp9y4lifp+0XkkU\nM9rEjyMiaUvSnM+NygYvcB3LSS0S1g0ljztNEpy+cc6dYKQXrIhefDSLOlrqNrBRltONSnaLmGpM\nkhrkKOa5iJNWZ28zWscUQ0z4KRs85Uk0SlICj5U8Dk5ck9n4IYp5LuLAkj/jw69iCBqV1PP4iTq6\nDNjgvn9JhDzwOSCPE6KazWY7knRkEVPBSZLTt1yAL5vNksmUwPOkZTKljj1njSj7Z/L5PAtKJbLA\np4EskCmVyOfzDA0NcfLkyQ5L2luY8zkgfssG/zfgW8B1113HqlWrElHd00oeBydpTt+klG32y5Yt\nWygUCqzFyc32dlBO4HRKDhYKbN682eokxYgphoC0WjZ41apVHS2n68VKHgdneHiYbdu2USpN4jRl\n9RLLwi2H3YxOl232Q7FYZGJigkXMVwq4n+/GSf6cmJhgeno6kcqtGzFTUkCGh4fJZDKUm4VaOM0C\nsTYMfkiz7EmhPJkNvIDTv61OaPNOZjNiDZsHM2UmF1MMASk3DM2bBRLXMKRZ9iSRy+UYHBzEKQc2\nCIziWMpH3c8HA01m062YKTO5mGIIgXLDUL9ZILENQ5plTwppdPomgbIp8xCQw6mHk2N+B8VMmfET\nqIhep0hiEb00lw1Os+xJIy1O3yQwNTXFm970pnnrMzi54zkcA50VmAyPWGZw6xRJVAxl0twwpFl2\nI134maNkAHg5ToVNm70tHEwxGIaRWLzTedYNU3U/J3VyoTQSV9ltwzCMlvAbproQEODOO+80pRAz\nphgMw4iVVsJUFXjggQdik81wMMVgGEasWJhq8jHFYBhGrFjGffIx53MCsJnTjF7C5kHvHLE4n0Xk\nXBH5tog85r7WVO0i8pJnvudJz/rVIvJ99/g7ROSsIPKkjdnZWUZHR1m9ejVbt27lM5/5DFu3bmXV\nqlWMjo4yOzvbaRENI3Qs4z75BBoxiMjngWdV9WYRuR7oU9VP1djvF6r6azXWfxO4S1VvF5EvAT9Q\n1b9pdt00jRjqjQb8xHFbmJ7RrXif/4U4CuJiHPORPf/R4XfEEGjCHJx2bJn7fhlwpM5+v6ixToCf\nAgvdz28B7vVz3SRN1FOPZpOPbNiwwWZOM3qaZv+RpE0o1A3gc6KeoCOG06q61PP5lKrOMyeJyIvA\nYZyOwM2qereInA8cUNX/6u6zArhHVV/X7LpJHzH4GQ2AM2282ViNXscy7uPD74ih6XwMIvIdoL/G\nphtakGdAVZ8UkUHgPhF5BPh5jf3qaikR2QRsAsdGmWSaTT7yFvfVTxx33i03nPTa+obRLmmYO6LX\naKoYVPVd9baJyEkRWaaqMyKyDHiqzjmedF8LInI/8EacEpRLRWShqr6I00Y+2UCOW4BbwBkxNJO7\nU/jJ6rwG+N9YHLdhGMkkaB7DJHCt+/5anLawAhHpE5Gz3ffnA0PAo66967s4FZ7rHp82/GR1rnRf\nLY7bMIwkElQx3Ay8W0QeA97tfkZE1onIre4+rwGmROQHOIrgZlV91N32KeBPROQocB7wlYDydBw/\nWZ3DOJ53mznNMIwkEmjOZ1V9BnhnjfVTwMfc9/uB19c5vgBcHkSGpOEnq3MAeCXw7zh+hLtxfApl\nvHHcWYvjNgwjZqwkRsj4nUe5nLpmM6cZhpE0TDGEjN+szpeADRs2kM1mKWUynskgnRBVmw7SMIxO\nEciUZNQml8tx6NAhDhYKDFI/q/MrX/kKF154ocVxG4aRKKyIXkTYPMpGUrGijb2LTe2ZEGw0YCSF\nZp2VXC5Hf3+tXFajWzDFYBjGGaxoowEhlsQwDCP9NCvTshE4WCiwefNmxsfH657HzFC9gY0YDKPL\nCWNiHDNDdQc2YjAMA5gr0zJCe0Ubq81QI3jMUKUS+XyeQ4cO+TJD2YgjHZhiMIwux0+ZFqhftDEM\nM1S9Ece2bdtsxJFALMHNMLocP2VaoHbRRj/Vgu/G6WFOTEwwPV2d0jk34sjn8ywolcjiZPlngYw7\n4hgaGuLkyZOtfzkjEkwxGEaX47dMS62ijX6qBZfNUCXXDFWNd8RRAMaBne7rE+Csd0cc9SgWi+Ry\nOW688UZyuVxNBWSEh5mSDKMLqbblX3nlldxzzz0tF20MaobyO+IYZG7E4b2+maA6gykGw+giGkUP\nnXPOORz85S8blmmpLtoYxAwFwRzf9Zzeh4B7XRPU/fffzz//8z+zdu3aJhIarWCmJMPoEprZ8n/5\ny19yzjnn8JKI76KNQcxQEGzEUW2CyuEoqHuBssp75plnWHfZZYyOjjI7O4sRDjZiMIwuwVf00C9/\nyfr169mwYYOvMi3lasH5fL6tuUPaHXFUm6AW4kz9WID5IbPQUsis4QNVTd1y2WWXqdHdHD9+XHft\n2qVjY2O6a9cuLRaLnRYp0Rw/flwzmYwuAp0G1RpLEXQhaCaTael+zszM6ODgoOIenwXd7r4uBAV0\ncHBQZ2dnQ5Nr165dinsNdV8BXVvjPNPuekCz2Wxo97QbAabURxvb8Ua+ncUUQ/cyMzOj2WxWM5mM\n4v7ZcRuNbDarMzMznRYxkVQ3pPWWcgOby+VaOn+z36WWUiiTzWbPNOrFGkqhVqM+NjamgH4a9Dho\nBiJRer2GX8UQyJQkIucCdwCrgGPA1ap6qmqftwNf8Kx6NXCNqt4tIrcBbwV+5m77iKoeDiKTkV7C\nzLDtNYJGDzWjv7+f8fHxtqoF+52fxOv49pqgJnF8Cu1mbhtt4Ed71FuAzwPXu++vB/68yf7nAs8C\nS9zPtwGjrV7XRgzdibdnaeaC1oh6xBCUVkccXhPUJ9x9P93ku2139xsbG4v1u6UJfI4YgkYlbQS+\n5r7/Go4fqhGjwD2q+lzA6xpdRhgZtr1M0OihqCmPOI4dO0Yul2NsbIxcLsexY8cYHx+fNwL0TpF7\nu7uu3ZBZow38aI96C3C66vOpJvvfB7zX8/k2nN/7YRxz09l+rmsjhu4j6T3eNNCOLT/JeJ3emI8h\nFAhrxCAi3xGRH9ZYNraigERkGfB6nDDkMttxfA5vwjEzfarB8ZtEZEpEpp5++ulWLm2kgKht5L1A\nLpdjcHCQgziZxKM4eQyj7ueD1E5iSyr9/f3s27ePbDYLwAs4JorqsaI3ZHakRsis0TpNFYOqvktV\nX1dj2QucdBv8csP/VINTXQ1MqOoLnnOXQ0yeB/4WuLyBHLeo6jpVXXfBBRf4/X5GSgiaYWtUNqSl\nTMZ3EluSKZugpqamOP/887tG6SWdQBP1iMhfAM+o6s0icj1wrqp+ss6+B4Dtqvpdz7plqjojIoJj\nSvoPVb2+2XVtop7uI4zJZIw5unGu8WaTBe3evTtVSq8TxDLns4icB3wTGACKwAdU9VkRWQd8XFU/\n5u63CtgHrFDVkuf4+4ALAAEOu8f8otl1TTF0J6Ojo+TzedZSP8P2IJDNZhtOP2l0N92o9OIiFsXQ\nKUwxdCfePIaF1I93T5s5xDCSgl/FYEX0jMTQjTZyw0gjVkTPSBRBMmwNwwgHUwxGIlmxYoWVNTCM\nDmGmJMMwDKMCUwyGYRhGBaYYDMMwjApMMRiGYRgVmGIwDMMwKjDFYBiGYVRgisEwDMOoIJUlMUTk\naeB41erzgZ92QBy/JFm+JMsGyZYvybJBsuVLsmyQbPnalW2lqjYtT51KxVALEZnyUwOkUyRZviTL\nBsmWL8myQbLlS7JskGz5opbNTEmGYRhGBaYYDMMwjAq6STHc0mkBmpBk+ZIsGyRbviTLBsmWL8my\nQbLli1S2rvExGIZhGOHQTSMGwzAMIwRSpRhE5AMi8iMRKbnTh9bb7z0ickREjrpzUZfXrxaR74vI\nYyJyh4icFbJ854rIt93zf1tE5s1YLyJvF5HDnuU/ROR97rbbROQJz7Y1ccrm7veS5/qTnvVJuHdr\nROQB9xl4WER+x7Mt9HtX7znybD/bvRdH3XuzyrNtu7v+iIhcGVSWNmT7ExF51L1P/yIiKz3bav7G\nMcv3ERF52iPHxzzbrnWfg8dE5NoOyPYFj1w/EZHTnm2R3jsR+aqIPCUiP6yzXUTkr1zZHxaRtZ5t\n4d03VU3NArwGuAS4H1hXZ58FwOM488afBfwAuNTd9k3gGvf9l4A/DFm+zwPXu++vB/68yf7nAs8C\nS9zPtwGjEd07X7IBv6izvuP3Dmemz1e5718JzABLo7h3jZ4jzz5/BHzJfX8NcIf7/lJ3/7OB1e55\nFsQs29s9z9UflmVr9BvHLN9HgFyNY88FCu5rn/u+L07ZqvbfCnw1xnv3W8Ba4Id1tl8F3AMI8Gbg\n+1Hct1SNGFT1x6p6pMlulwNHVbWgqv8J3A5sFBEB3gGUZ5H/Gs788mGy0T2v3/OPAveo6nMhy1GL\nVmU7Q1Lunar+RFUfc98/CTwFNE3WaZOaz1EDmceBd7r3aiNwu6o+r6pPAEfd88Umm6p+1/NcHQAu\nCvH6geVrwJXAt1X1WVU9BXwbeE8HZfsg8I0Qr98QVf0eTmexHhuBv1OHA8BSEVlGyPctVYrBJ8uB\nac/nE+6684DTqvpi1fowuVBVZwDc11c02f8a5j90O90h4hdE5OwOyLZYRKZE5EDZxEUC752IXI7T\n43vcszrMe1fvOaq5j3tvfoZzr/wcG7VsXj6K08ssU+s3DhO/8mXd32tcRMrztibm3rnmt9XAfZ7V\nUd+7ZtSTP9T7lripPUXkO0B/jU03qOpeP6eosU4brG+JRvK1eJ5lwOuBez2rtwOzOA3eLcCngB0x\nyzagqk+KyCBwn4g8Avy8xn6dvnf/F7hWVUvu6kD3rtZlaqyr/s6RPmsN8H1+EfkQsA54q2f1vN9Y\nVR+vdXyE8v098A1VfV5EPo4z8nqHz2Ojlq3MNcC4qr7kWRf1vWtGLM9c4hSDqr4r4ClOAN5Z4y8C\nnsSpK7JURBa6vbvy+tDkE5GTIrJMVWfcxuupBqe6GphQ1Rc8555x3z4vIn8LfCJu2VwTDapaEJH7\ngTcCeRJy70Tk14FvAX/qDqXL5w5072pQ7zmqtc8JEVkIvBzHDODn2KhlQ0TehaN036qqz5fX1/mN\nw2zcmsqnqs94Pv4f4M89x76t6tj745TNwzVAxcTjMdy7ZtSTP9T71o2mpAeBV4kTRXMWzo87qY6H\n5rs4dn2AawE/I5BWmHTP6+f882yXboNYtum/D6gZmRCVbCLSVzbBiMj5wBDwaFLunft7TuDYWO+s\n2hb2vav5HDWQeRS4z71Xk8A14kQtrQZeBfxbQHlakk1E3gh8GRhW1ac862v+xiHK5le+ZZ6Pw8CP\n3ff3Ale4cvYBV1A5qo5cNle+S3CcuA941sVx75oxCXzYjU56M/Azt1MU7n2L0sMe9gKM4GjG54GT\nwL3u+lcC/+jZ7yrgJzia/AbP+kGcP+hR4E7g7JDlOw/4F+Ax9/Vcd/064FbPfquAfwcyVcffBzyC\n06jtAX4tTtmA33Sv/wP39aNJunfAh4AXgMOeZU1U967Wc4Rjnhp23y9278VR994Meo69wT3uCLA+\ngv9CM9m+4/5HyvdpstlvHLN8nwN+5MrxXeDVnmN/372nR4Hfi1s29/OfATdXHRf5vcPpLM64z/kJ\nHP/Qx4GPu9sF2O3K/gie6Mww75tlPhuGYRgVdKMpyTAMwwiAKQbDMAyjAlMMhmEYRgWmGAzDMIwK\nTDEYhmEYFZhiMAzDMCowxWAYhmFUYIrBMAzDqOD/A3y1piI23UVRAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the training error rate is: 0.000000\n",
      "the test error rate is: 0.080000\n"
     ]
    }
   ],
   "source": [
    "testRbf(k1=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "iteration number: 1\n",
      "iteration number: 2\n",
      "iteration number: 3\n",
      "iteration number: 4\n",
      "iteration number: 5\n",
      "iteration number: 6\n",
      "there are 24 Support Vectors\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAD8CAYAAABzTgP2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3X2QHPV95/H3d1YSRJWKJQRGBLRa\n7R12Yt+lAKu4OKo6O44fMHW1Yms3CWRdyI5dKvuEjqpUUgF0VK4QKtv5hzga3dk6xzEJKoO9WzKb\nu7g4zMOlyhgb2TzZuACxlmYVdrF8SE5SijEw3/uje6BnNI/bD9M983lVdc1MT8/Mb3pn+9u/p2+b\nuyMiIlJT6ncBREQkXxQYRESkjgKDiIjUUWAQEZE6CgwiIlJHgUFEROooMIiISB0FBhERqaPAICIi\ndVb1uwArcf755/vY2Fi/iyEiUijf+973furuF3TarpCBYWxsjCNHjvS7GCIihWJmx7vZTk1JIiJS\nR4FBRETqKDCIiEgdBQYREamjwCAiInUUGEREpE4igcHMvmRmPzGzH7R43szsL83sqJk9ZWZXRJ7b\nYWbPh8uOJMojIiIrl1SN4cvAVW2e/zBwabjsBP4HgJmdB/wZ8B+AK4E/M7P1CZVJJBmHDsHYGJRK\nwe2hQ/0ukUiqEgkM7v4PwMttNtkO/I0HHgXWmdlFwIeA+939ZXc/BdxP+wAjkq1Dh2DnTjh+HNyD\n2507FRxkoGXVx3AxsBh5fCJc12r9Wcxsp5kdMbMjJ0+eTK2gInX27IEzZ+rXnTkTrBcZUFkFBmuy\nztusP3ul+0F33+ruWy+4oGOqD5FkVCq9rRcZAFkFhhPApsjjS4AX26wXyYfR0d7WiwyArALDPHB9\nODrpN4GfufsScB/wQTNbH3Y6fzBcJ5IP+/bB2rX169auDdaLDKhEsqua2VeA9wLnm9kJgpFGqwHc\n/fPA3wNXA0eBM8DHwudeNrO9wGPhW93m7u06sUWyNTMT3O7ZEzQfjY4GQaG2XmQAmXvTJv1c27p1\nqyvttohIb8zse+6+tdN2mvkskgTNdZABUsgL9YjkSm2uQ21Ya22uA6jJSQpJNQaRuDTXQQaMAoNI\nXJrrIANGgUEkLs11kAGjwCASl+Y6yIBRYBCJa2YGDh6EzZvBLLg9eFAdz1JYGpUkkoSZGQUCGRiq\nMYjkgeZBSI4oMEh6dLDrjq75IDmjwCDpiHuwG6agonkQkjPKlSTpGBsLgkGjzZvh2LH2r22cSQzB\nKJ9B7dAtlYLg2cgMqtXsyyMDS7mSpL/iTPoatjNozYOQnFFgkHTEOdgN20xizYOQnFFgkHTEOdgN\n2xm05kFIzigwSDriHOyG8Qx6Ziboe6lWg1sFBekjTXCT9Kx00peumibSV6oxSD7pDHplhmmYr6Qm\nkcBgZleZ2bNmdtTMbmry/B1m9kS4PGdmpyPPvR55bj6J8ogMJU2Uk4TEnsdgZiPAc8AHgBPAY8B1\n7v5Mi+13A5e7+x+Gj//F3X+5l8/UPAaRJuLMHZGhkOU8hiuBo+6+4O6/AO4GtrfZ/jrgKwl8rohE\nDdswX0lNEoHhYmAx8vhEuO4sZrYZ2AI8GFl9rpkdMbNHzeyaBMojMpyGbZivpCaJwGBN1rVqn7oW\nmHX31yPrRsOqzR8Af2Fm/6bph5jtDAPIkZMnT8YrscggGsZhvpKKJALDCWBT5PElwIsttr2WhmYk\nd38xvF0AHgYub/ZCdz/o7lvdfesFF1wQt8wig6fbuSMauSQdJDGP4THgUjPbAvwjwcH/Dxo3MrO3\nA+uBb0fWrQfOuPsrZnY+sA348wTKJDKcOs0daUxQWBu5VHutCAnUGNz9NeAG4D7gR8BX3f2HZnab\nmU1ENr0OuNvrh0H9OnDEzJ4EHgI+02o0k4gkYNgSFMqKKO22FEqlUmF+fp7Tp0+zbt06tm/fzqZN\nmzq/UAJK8T3Uuh2uqpQYUgjLy8vccMMNHD58mGrkAHbjjTcyOTlJuVxm48aNfSxhQYyONp/rsMKR\nSwrUA8rdC7e8613vchkeS0tLPj4+7oCvBp8CvyW8XRWMgPPx8XFfXl7ud1Hz76673NeudQ/qDcGy\ndm2wvgdLS0s+NTXlpVLJCf8GgJdKJZ+amvKlpaWUvoDEARzxLo6xqjFIZlZ6dnnDDTewsLDAFcC9\nBMPeak4QzKb8/sICu3btYnZ2Np3CD4oEEhQuLy+zbds2FhYWWA1MAm8HngXurVaZm5vj8ccf55FH\nHuHCCy9M4UtI6rqJHnlbVGMoljhnl8ePH/dSqeSrwRejZ7mRpRLWHEqlklcqlQy/2RC66y6fWrvW\nAb+iyd9kMVwP+NTUVL9LKw3ossag7KqSqtrZ5dzcHCPVKlPALcAUUArPLrdt28ZLL73U9PXz8/NU\nq1UmqK8pRG0iqDVUq1Xm55WHMTWHDlH5xCc4fOYMqzm79kb4+OsEnZeHDx9mcXGx8V3yQXM52lJg\nkFRFm4EWgFlgX3j7YwjWh81AzZw+HSTifXuHz3lbeHvq1KkESi1N7dnD/M9/ThWKHaiVhbYjBYZh\nl+KZU6VS4fDhw7HOLtetWwcE7dftPBferl+/Pk6RpZ1KhVq+/EIHas3l6EiBYZilfOaURDPQxMQE\npVKJeYKO5mYWCQJPqVRiYmKixVYS2+go68K7hQ7UykLbkQLDMEv5zCmJZqDR0VEmJyd5lSCANNYp\nFoFrgNeAyclJjaFP0759TJx7LiUodqBWFtqOFBiGWcpnTkk1A5XLZcbHx/k+MA5ME3RgT4ePvw+M\nj49z4MCBBEotLc3MMPrFLzK5dm2xA7Wy0HbWzdClvC0arpqQzZubDv/0zZsTefskh5p2GvKqyW3Z\niU44XBVONLy5aBMO77or+J2bBbc9TvArKrocrqpcScOsMdMmBGdOzVI1r9D09DRzc3NcQdDRHD1/\nrJ1dfh+YmprqanLa4uIi8/PznDp1ivXr1zMxMZHPs9IBt7y8zA0TExx+7DGiGZZKpRKTk5McOHBA\nk9tyqNtcSX0/+1/JohpDglI+cxqIs8u8ycPZbphaowJeBt8LXl692iuf+1z2ZZGuoRqD9Etj6ot3\nv/vdfPrTnz4rAZ7OLlcgg1peV8bGmifj27wZjh3LrhzSE2VXlcy1yoBaCwDf/e53efTRRwvdDNT3\nbKLtRpJlGRg05HOwdVOtyNuipqQeZdD0MOgZUHOTTdSs+YABs2w+vyblgQuSDrpsSur7QX4liwJD\nDxJKs9zJ1NTUwCZWy1XQy8sBOaPflSRLgUECGRxIBj0Daq6CXrsDctad0nnoBJeeKDBIIIOmh/37\n9wcHxhZBobZMhQfQcrmc2GenLZdBr9kBWWfw0oVuA0MiM5/N7Coze9bMjprZTU2e/6iZnTSzJ8Ll\nE5HndpjZ8+GyI4nySEQG0/8HOQNqLtN+z8wEI3+q1eB2ZkaJ4SRRsUclmdkIcAD4AEH6lMfMbN7d\nn2nY9B53v6HhtecBfwZsJTib/F742uIcOfJu377mwxtjTP9vHJkTnIj0kPri6NFguOMKryCWpcIE\nPY0SkgQlMVz1SuCouy8AmNndBCdQjYGhmQ8B97v7y+Fr7weuAr6SQLkEErmUY0274agQJE47QfMz\n6zcSq5kxcc898POfB0/UMrpGy5ojfU37fehQ93+30dHm8wqUGE5WIImmpIupz6V1IlzXaMrMnjKz\nWTOrDfzu9rUSR7Omhx51uhIbBInTrqZDYrVf+iU21YJCTY6bPPqW9rvXlOhKDCcJSiIwWJN1jdOp\n/w4Yc/ffAL4J3NnDa4MNzXaa2REzO3Ly5MkVF1ZWppsrsQE8TYcMqI3t4DU5bfLoW9rvXvsMZmaC\n2c+bN4NZcJv1bGgZHN30ULdbgHcD90Ue3wzc3Gb7EeBn4f3rgC9EnvsCcF2nz9SopGz1MjLHwM2s\ndQbUvIzD70Ff8j3lZSKbDBQyHJX0GHCpmW0xszXAtQTX8XiDmV0UeTgB/Ci8fx/wQTNbb2brgQ+G\n6yRHehmZ48Btt91GuVxm7969lMtljh07xuzsbJAPqYBNHhs3buRb3/oWU1NTVEsl5oBPA3NAtVRi\namqKRx55JNl8T7qYzGBI8dK5aYrd+ezur5nZDQQH9BHgS+7+QzO7jSA6zQP/xcwmCGrcLwMfDV/7\nspntJQguALd52BEtPeqlo7JHvY7MAdi1a1fzjRLsDM/Sxo0bmZ2dzS7tdwqjySRjjQkPcz7Qok43\n1Yq8LWpKCtUmOtWaGFKa3DTIE9hyTTOLk9Gv/ZjDZlOUdnvANUu/3CihFMiVSoUtW7YwUq2yQOvh\nqOMETSvHjh0rXNZUGVD9TFNeKgWhoJFZMEKwD7pNu61rPhdVs1ErjRIa6dO3kTkDolKpUC6Xuf32\n2ymXyywuNu5BSU0/Z4S36g8qlXLf56AaQ1G1OhuJSvCiKbV5DAsLC6wiCBBvI5jYdS9BUBgfH0++\nE7bAOl2folwus3Hjxj6W8Gx9v95E0vp51t5NrT7jiyzp0p6DrlX7ZYoJ1Dpdk6Co11pIQ3SIK6x2\nmHK4Jbxdlc4Q1xhyc72JTnrtL+h3O3+0vCMjfe9zQNlVB1yzbJq1DuiUO9gqlYqXy2Xfu3evl8vl\nwqXRzkItVTdc4bDYcCxYDNfn4/oUubreRDsrySCbp6yzOZibosAwDDRqJXPHjx/3/fv3+969e33/\n/v1Ng2JtQmBQU2gMCrWl4rAqF9enyNX1JtpZ6dl/Xv5P+l17cQWG4ZWXf4IB00tTS214b9Bs1PxY\nECxTfR/em8vrTbSSgzPuWHJQe+k2MGhU0iDpNfGadCWaQLBaHYFICsFqtcTc3Bzbtm3jpZdeAt6c\nENjtlMB+Xp8il9ebaKXos8ELlM9KgWGQ6GItqaglEKRNCsGFhYU3ZnvXUnV3m6w70VTdPSrM9Sag\nkOlUzpJApuMsKDAMEl2sJXGVSoXDhw8DqwkG5jaeV18CfB1YxeHDh1lcXHwjVTddJOtONFX3CvT1\nehO9KtAZd9EpMAySole1c6jW1EIXjS21ppbahEC6mBLY7wmBfbvexEoV5Iy76BQYBskgVLVzZqX9\nBeVymfHxN65EQbMrVIyPj3PgwIEUSt09zWrvQkEzpMahwDBIVNVO3Er7C6KpukulKkSSdZdK1XRS\nda9QLYi1DmHkIoj1xZAO6FBKDJE2agkEg9FI7VMIlkrVpgkEM0vVHUOn9B0HDhzIRRDL3NhY82tp\nJ5huJkvdpsRQYBDpYHp6mrm5OYJRSV8n6FOoqTW2fJ+pqSlmZ2f7UcTEFCGIZSqHGVLjUGAQSUg0\ngSAtUggqgeCAGtIag/oYpHCy7gssUn+BJGxIB3TEvrSnSJb6dbXEzC/tKflQ0EvRxqWmJCmUAavZ\ni2Qq06YkM7vKzJ41s6NmdlOT5//IzJ4xs6fM7AEz2xx57nUzeyJc+piIRYpAk7tF0hc7MJjZCHAA\n+DDwDuA6M3tHw2aPA1vd/TcIEsz8eeS5f3X3y8Klz9MqJe80ubu1IZyHJSlJosZwJXDU3Rfc/RfA\n3QTDNt7g7g+5ey2726O0zi0g0taQ9gV2NKTzsCQlSQSGi6mfSX8iXNfKx4FvRB6fa2ZHzOxRM7sm\ngfLIANPk7uaUWFeSlMSoJGuyrmmPtpl9BNgKvCeyetTdXzSzceBBM3va3V9o8tqdwE4I8rsU2cBd\ncD1jMzMKBI3U9yJJSqLGcIL6qaCXAC82bmRm7wf2ABPu/kptvbu/GN4uAA8Dlzf7EHc/6O5b3X3r\nBRdckECxs7e8vMz09DRbtmxh9+7d3HrrrezevZuxsTGmp6dZXl7udxGloIau70UdKqlKIjA8Blxq\nZlvMbA1wLUEi+jeY2eXAFwiCwk8i69eb2Tnh/fOBbcAzCZQpd6JXARupViPXAINStXrWVcBEejFU\nfS/qUElfN9f/7LQAVxPkB3gB2BOuu40gEAB8E3gJeCJc5sP1vwU8DTwZ3n68m88r4jWfC3PBdUlN\n2pfjHprLfW/e3Pzaz5s397tkuUeX13zWBLcM1DJ0jlSrHfJzQrVUapqhU4qtccY2BGf06jhfgQFL\nbJcl5UrKkUJdcD3UrAm3UqlQLpe5/fbbKZfLLC42XtZlsCTZjK1RQwkaug6V7ClXUgYKdcF1muUj\nWmbHjhuoVg/j/uYZ2Y033sjk5CTlcpmNGzf2qbTpSDonk0YNJWjfvubVr4HsUOkP1RgyUKgLrtN4\ndrsMbOP11+dwH4FIt3m1WhrYTvNWZ/g7dqysBpH2Se5Q1eY0mSV93XRE5G0pWufz8ePHvVQq+eom\nHc+1pQK+CrxUKnmlUulrec2iRQs6zeEKh8WGYi+G6wev07x+HzRf1q4NOni76fS9665g+2avj2Np\nacmnpqa8VCqFf6dgKZVKPjU15UtLS/E+QAYKXXY+9/0gv5KlaIHBvX5UUqVJUMjTqKQ3B30cdyg5\nrG4SFGpLxWFV4gGt3yNsWg18aVw2bOj+gJ/0d1paWvLx8fEwGKwOg/gt4e0qB3x8fNyXl5fjfZAM\nDAWGnIn+E68CnwK/ObxdFQaFvPwTv3l2uz886Ex1OEAGQa9cLif8+Z0PtmlpVoZelixGTtZONoat\nNicr121gUB9DRqJXAauWSpFrgAVDVPN0FbBaE+5b3nI6XNNdt3lSneZ5GMHT2Iw9MtLb69PuVK5U\nKhw+fBhYTXB50cbxbpcQXJ96FYcPHx7sPgdJnAJDJwmOWaxdBezYsWOUy2X27t1LuVzm2LFjzM7O\n5iIo1MzMwO23rwsfdddtnlSneV5G8MzMBBf/qVbhzjubzyzesKH5a9MeOVkbAk0Xg6DzMgRaikPD\nVdtJ6TqSmzZtYteuXQkUMF0TExPceOONVKvzBCmxWk3Nu5dSqcTERDKX0xgdbX6Vtn4OU291hUfo\nz8jJ2hDorGtzMhxUY2inXZtGq5rEACX3Gh0dZXJyEniVYPpdY3PEInAN8BqTk5OJzdbOa96faA3i\n2LE3s7z2Y+RkbQh01rU5GRLddETkbcms87ndmMVmvaOf+lT/e00TVj/yZVXY0Xxz6iNf+j0qKe9q\nQ6D7MWJMiguNSkpAqzGLIyO9rS94cq9OY+XzMJJqGNWPSqo0CQoalST1ug0MSqLXTqvMZ43NS50M\nSHKvxcVF5ufnOXXqFOvXr2diYqLwyf4OHTq736AoE2hrqdwXFhYIugu3E/QpPEcwUuk1xsfHczPa\nTfqv2yR6fT/7X8mS6TyGZm0avdYkCl5jGFR5mC8Rl2pz0gtUY0hRq5rEjh3BuEblVi6EsbHmo582\nbw46l4tkEGtzkjzVGNLWqndUvaaF0WpsgVnyn6WfheQBqjHIIKhUKszPz3P69GnWrVvH9u3bEzsT\nXkmNYSV9ErpIj+SFagxSaFlkDe21j+Guu9zXrKnffs2azmf/aV6JUjUR6QUaripFlWXW0F4OrBs2\nND/Ab9jQ/n1bTYWJ22Q1CJ3nkq1MAwNwFcEUzKPATU2ePwe4J3z+O8BY5Lmbw/XPAh/q5vMUGAZb\nXrOGtsumGtVtZta4NYY0ayIymLoNDLFTYpjZCHAA+DDwDuA6M3tHw2YfB065+78F7gA+G772HcC1\nwDvD4PLfw/eTITUIWUObZVJplESKj7wkG5TBk0SupCuBo+6+4O6/AO4mmGkTtR24M7w/C/yOmVm4\n/m53f8Xdf0xQc7gygTJJTP26VGTesoZGU1+ZNd+mMcNquwNzkvmU0r5cqAyvJALDxdRnVzsRrmu6\njbu/BvwM2NDlayVDy8vLTE9Ps2XLFnbv3s2tt97K7t27GRsbY3p6muXl5VQ/P6usod3kOqyNJjp+\n/M2GmkarV8PnPle/rtWBecOG+gR8cV199dnr1qzpf7JBKb4k0m43O49q/BdqtU03rw3ewGwnsBOC\nrJ+SvPoUC6uBSYID9LNUq/cyNzfH448/nmqKhSyyhnabTb1Vk9DISHCAbzVcdd8++NjH4NVX69f/\n8z8Hn51EUDh0CP7qr85e/9pr8d+7G2kOI5Yc6KYjot0CvBu4L/L4ZuDmhm3uA94d3l8F/JQgKNRt\nG92u3aLO53TkodM3i6yh3XbadpoA125EU6sRTEl1DLe7JnWzzzh+/Ljv37/f9+7d6/v3719xttUs\nhhHnRVL7LE/IalRSeKBfALYAa4AngXc2bLML+Hx4/1rgq+H9d4bbnxO+fgEY6fSZKw4MGvTdUp7S\nOKedNbTbGc/tAkinoaJpz6rudhhskgfyLIcR99MgB7/MAkPwWVxNULd/AdgTrrsNmAjvnwt8jaBz\n+bvAeOS1e8LXPQt8uJvPW1Fg0KDvtvbv3x/+A0y1POAES3DQLpfLqZUl7WtAdFtjaPeT6fQeaQ8l\n7abGkPSBPA81ylaSrBHV9tlq8CnwW8LbVWGAKHLwyzQwZL2sKDBo0Hdbe/fuDf/pb+kQGG52wPfu\n3ZtqedLMGtrugN94gPnc5ypNK5ndNDOlcR7SLiiB++rVb35G3AN5tIJ98cXH3SwfNcqopM/ua/vs\nCvDFhi+4GK7vV/BLggJDoywzphVQnmoMUZVKxcvlsu/du9fL5XJiB5zGVsVyubcDTDezoJNuuew0\ncW7Dhjc/I27T4Nmflb/fR9Jn97V9trpJUKgtlfC9i3pVPAWGRqoxtJWnPoasraTJpZf0GEnp5Scc\nN9Cf/Vn5qlG6J392X9tnU+2/oE+F75vVyVGSug0MScxjKIa8XmE+J0ZHR5mcnAReJZh32DihbRG4\nBniNyclJNm3a1LdJcEm74YYbwiG6VxCMf5gF9oW3PwauYGFhgV27dr3xmpdfbv5erdYnoZeZznHn\ng5z9nukPI+5FbYZ85/nxdD1DvrbPuttjK59DUwjdRI+8LRqVlI5uO32feuqpgRm1sdKa0koqoHF/\nft185pt9EEnXGI475KdGmcbZvWoMw9iUJF3p1Jn31FNPJTbSJQ/jxFfa5LKSlN1xO6M7vUf98/EO\n5M0+a2Qk3WHEvagNlrilw0H85vD3203TVmJ9DDk+AVVgkFhadfomMWQxT+PE44zG6uX/P6kurnaf\nefZnxDuQN+ugT3MYcS/SOruP9ltUmgSFjv0WOR8Wr8AgiUuigzpvk6SyGo2VxaC4sz9jySHZA3ma\nw4h7kdYIoujvc1UYWG6mh5FOOR/kosAgiUviIJq3SVJZjcbK4njR/DOWfO3a5A/kaQ0j7kXss/sW\nYgW/nA+LV2CQxMWdBJfXIbFpp+Bwz6aFod1n5OFAnrTYZ/cdrGifpXEGkGCfhQKDJC5ujSHLSXTN\n/pda/X+lnYKjXZmSluN+z1TkpWnrDUmfAST8fgoMkri4Z/xZpd1o9r+0erX7mjWt/79yd4Dx4TvI\nx5GrGlGSf7iEayAKDJKKOM0uWdUY2uUS6vT/lZcDTDdDUy+++LjDfn/LW/b69ddnM9w3q2CVh6HM\nuZBwn4UCg6QiTrNLVn0M7VJSJ/T/lbp2J4rl8lI4p6C+dmOW7nDfLPpJ8jSUORdUY1BgKIo4zS5Z\ndPTGqTHkRevgtuSrVvVnuG/aI6sGPeX1iqiPQYGhaFbS7JJFR+9K+hjyptVBeO3a/g33TXsk5qCn\nvF4xjUpSYBgGWXT09jIqKY+aBbdzz+3vNRHSrDEMQ8rrPFBgkNzLS0dvXjUGsuuv7+81EdLsYxiG\nBHZ50G1gWIVIn2zatKkulbXUm5kJlprbb4+XSjuJ8gDs2ROk5R4dDbLWR8u4Ukp5nS8KDCIFsW5d\n/6+J0BisklL7bt19s/Sv9zDsYl2ox8zOM7P7zez58Pasv5aZXWZm3zazH5rZU2b2+5HnvmxmPzaz\nJ8LlsjjlERlkExMTlEolYB440WKrReBeSqUSExMT2RUuptp36/zNKNx3K6K4V3C7CXjA3S8FHggf\nNzoDXO/u7wSuAv7CzNZFnv8Td78sXJ6IWR6RgbWSq+wVRe27df5mFO67FZEF/RErfLHZs8B73X3J\nzC4CHnb3ts2EZvYkMO3uz5vZl4H/5e6zvXzu1q1b/ciRIysut0hRLS8vs23btvBSpKsIDqNvI2hk\nuRd4jfHxcR555BEuvPDCfha1Z9Hv1vybUdjvlhdm9j1339ppu7g1hgvdfQkgvH1rh0JdCawBXois\n3hc2Md1hZue0ee1OMztiZkdOnjwZs9gixbRx40a+9a1vMTU1RalUBeaATwNzlEpVpqamCnvgjH63\naqkU+WZQLZUK/d2KpmONwcy+CWxs8tQe4E53XxfZ9pS7N+0VqtUogB3u/mhk3TJBsDgIvODut3Uq\ntGoMIrC4uMj8/DynTp1i/fr1TExM1DWxHDqUzgiiLHT6brIy3dYYYs0nIBhEcFF4/yLg2Rbb/Qrw\nfeB327zXewmalQZvHkORZlbJQMj5FSalT+hyHkPcpqR5YEd4fwdBU2BjhFoDHAb+xt2/1vDcReGt\nEfQt/SBmefLn0CHYuROOHw/+P48fDx4fOtTvkskA27MHzpypX3fmTLBepJO4nc8bgK8Co0CFoEbw\nspltBT7p7p8ws48Afw38MPLSj7r7E2b2IHABYMAT4Wv+pdPnFqopaWwsCAaNNm+GY8eyLo0MiVIp\nOA9pZAbVavblkXzotikpVmDol0IFhgL8h1YqFebn5zl9+jTr1q1j+/btas8tOJ2PSDNZjUqSTkZH\ne1ufoeXlZaanp9myZQu7d+/m1ltvZffu3YyNjTE9Pc3y8nK/iygrtG8frF1bv27t2mC9SCcKDGnL\n6X9obcz43NwcI9UqU8AtwBRQqlaZm5tj27ZtvPTSS30tZ9EcOhScrZdKwW2/upJmZuDgwaCGYBbc\nHjxYnFFJ0mfd9FDnbcndqKROo45yOCpJue+Tp5FAknd0OSpJfQxx1UYdRYeArF2b69OzSqXCli1b\nGKlWWQAuabLNIjBOMLHo2LFj6nPogtr1Je/Ux5CVAo4LnJ+fp1qtMkHzoACwiSAlQbVaZX5+PrvC\nFVil0tt6kbxSYIirgEcD5b5PR47HGRRLXjpqhpgCQ1wFPBoo9306cjrOoFg0ITQXFBjiKuDRQLnv\n06GRQAkoYNPsIFJgiKuARwPlvk/PzEzQ0VytBrc5/hnkUwGbZgeRRiUNKeW+l1zS0K5UaVSStKXc\n95JLBWyaHUSr+l0A6Z+NGzclMDv0AAALEUlEQVQyOzur3PeSH7W2t6JeSGJAKDAImzZtYteuXf0u\nhgy7xisL/e3fKiD0iQKDiPRfYwaB2jBVUHDoA/UxiEj/aZhqrigwiEj/aZhqrigwiEj/FTCDwCBT\nYBCR/tMw1VyJFRjM7Dwzu9/Mng9vmybVMbPXzeyJcJmPrN9iZt8JX3+Pma2JUx4RKagCZhAYZHFr\nDDcBD7j7pcAD4eNm/tXdLwuXaOKdzwJ3hK8/BXw8ZnlEpKiUTyQ34gaG7cCd4f07CVLsdMXMDHgf\nMLuS1w8EpRcWkRyKGxgudPclgPD2rS22O9fMjpjZo2ZWO/hvAE67+2vh4xPAxTHLUxxKLywiOdVx\ngpuZfRPY2OSpXgYYj7r7i2Y2DjxoZk8D/9Rku5YZ/cxsJ7ATguyghddu3Laq0CLSRx1rDO7+fnf/\nd02We4GXzOwigPD2Jy3e48XwdgF4GLgc+CmwzsxqwekS4MU25Tjo7lvdfesFF1zQw1fsg26aiDRu\nW0RyKm5T0jywI7y/gyBjcx0zW29m54T3zwe2Ac94kO/7IWC63esLp9smIo3bFpGcihsYPgN8wMye\nBz4QPsbMtprZF8Ntfh04YmZPEgSCz7j7M+Fzfwr8kZkdJehz+KuY5em/bqf2a9y2iOSULtSTtFIp\nqCk0MguG4UU1ZpNUemERSVG3F+pRdtWkjY42vwJVsyaimRkFAhHJHaXESJqaiESk4BQYkqap/SJS\ncGpKSoOaiESkwFRjEBGROgoMIiJSR4FBRETqKDCIDCNl9pU21PksMmxqaVtqM/RraVtAgyYEUI1B\nZPh0m7ZFhpYCg8iwiZvZV81QA0+BQWTYxMnsqwtMDQUFBpFhEydtS1LNUKp15JoCg8iwiZO2JYkL\nTKnWkXtKuy0i3Rsba549ePNmOHYsu/eQFek27bZqDCLSvSSyB6vzO/cUGEQGUVoHzySyB6vzO/fU\nlCQyaBonsEFwVp+X9O9xyteqGWpkBO68Mx/fL8fUlCQyrPI+gS2Nzu/XX1fNIUGxagxmdh5wDzAG\nHAN+z91PNWzz28AdkVW/Blzr7l83sy8D7wF+Fj73UXd/otPnqsYg0kYv1x0vmlY1hhp1YLeVVY3h\nJuABd78UeCB8XMfdH3L3y9z9MuB9wBng/0Q2+ZPa890EBRkC6lyMJ04bft416/yO6mXYrLQUNzBs\nB+4M798JXNNh+2ngG+5+psN2MqzUuRjfIF93vNYMNTLS/PlBCH45EDcwXOjuSwDh7Vs7bH8t8JWG\ndfvM7Ckzu8PMzolZHim6vLePF8GgX3d8ZiboaB7U4JcDHfsYzOybwMYmT+0B7nT3dZFtT7n7+hbv\ncxHwFPCr7v5qZN0ysAY4CLzg7re1eP1OYCfA6Ojou463a2eU4hrk9nFJ1qFDwQlDpRLUFPbtG5zg\nl5Ju+xjidj4/C7zX3ZfCg/zD7v72FtveCLzT3Xe2eP69wB+7+3/q9LnqfB5gmhUrkpqsOp/ngR3h\n/R3AvW22vY6GZqQwmGBmRtA/8YOY5ZGiG+T2cZGCiBsYPgN8wMyeBz4QPsbMtprZF2sbmdkYsAn4\nvw2vP2RmTwNPA+cDt8csjxTdoLePixSAZj6LiAwJzXwWEZEVUWAQEZE6CgwiIlJHgUFEROooMIiI\nSB0FBhERqaPAICIidQo5j8HMTgLHCSbF/bTPxWlH5Ysv72XMe/kg/2VU+eLrtoyb3f2CThsVMjDU\nmNmRbiZr9IvKF1/ey5j38kH+y6jyxZd0GdWUJCIidRQYRESkTtEDw8F+F6ADlS++vJcx7+WD/JdR\n5Ysv0TIWuo9BRESSV/Qag4iIJCz3gcHMftfMfmhmVTNr2etuZleZ2bNmdtTMboqs32Jm3zGz583s\nHjNbk3D5zjOz+8P3v9/Mzrq0qZn9tpk9EVl+bmbXhM992cx+HHnusqzLF273eqQM85H1qe6/bsto\nZpeZ2bfD38JTZvb7kedS2YetflOR588J98nRcB+NRZ67OVz/rJl9KInyrKB8f2Rmz4T76wEz2xx5\nrunfuw9l/KiZnYyU5ROR53aEv4nnzWxH42szKt8dkbI9Z2anI8+lvg/N7Etm9hMza3oRMwv8ZVj+\np8zsishzK99/7p7rBfh14O3Aw8DWFtuMAC8A4wTXj34SeEf43FeBa8P7nwc+lXD5/hy4Kbx/E/DZ\nDtufB7wMrA0ffxmYTnH/dVU+4F9arE91/3VbRuBtwKXh/V8FloB1ae3Ddr+pyDb/Gfh8eP9a4J7w\n/jvC7c8BtoTvM9KH8v125Hf2qVr52v29+1DGjwLlJq89D1gIb9eH99dnXb6G7XcDX8p4H/5H4Arg\nBy2evxr4BmDAbwLfSWL/5b7G4O4/cvdnO2x2JXDU3Rfc/RfA3cB2MzPgfcBsuN2dBJcQTdL28H27\nff9p4BvufibhcrTSa/nekNH+gy7K6O7Pufvz4f0XgZ8AHSfqxND0N9WwTbTcs8DvhPtsO3C3u7/i\n7j8Gjobvl2n53P2hyO/sUeCShMsQu4xtfAi4391fdvdTwP3AVX0u31mXJ06bu/8DwYlkK9uBv/HA\no8A6Cy6ZHGv/5T4wdOliYDHy+ES4bgNw2t1fa1ifpAvdfQkgvH1rh+2v5ewf176wGniHmZ3Tp/Kd\na2ZHzOzRWjMX2ey/XsoIgJldSXCG90JkddL7sNVvquk24T76GcE+6+a1WZQv6uMEZ5Y1zf7eSeu2\njFPh327WzDb1+NosykfYDLcFeDCyOot92Emr7xBr/61KpGgxmdk3gY1Nntrj7vd28xZN1nmb9T1p\nV74e3+ci4N8D90VW3wwsExzoDgJ/CtzWh/KNuvuLZjYOPGjBtbj/qcl2KxrGlvA+/Ftgh7tXw9Wx\n92Gzj2qyrvG7p/q766DrzzCzjwBbgfdEVp/193b3F5q9PuUy/h3wFXd/xcw+SVADe1+Xr82ifDXX\nArPu/npkXRb7sJNUfoO5CAzu/v6Yb3EC2BR5fAnwIkHukHVmtio8o6utT6x8ZvaSmV3k7kvhQesn\nbd7q94DD7v5q5L2XwruvmNlfA3/cj/KFzTO4+4KZPQxcDsyRwP5Lqoxm9ivA/wb+a1htrr137H3Y\nRKvfVLNtTpjZKuAtBNX+bl6bRfkws/cTBN/3uPsrtfUt/t5JH9Q6ltHd/1/k4f8EPht57XsbXvtw\n1uWLuBbYFV2R0T7spNV3iLX/BqUp6THgUgtG0Kwh+CPOe9AL8xBBuz7ADqCbGkgv5sP37eb9z2qj\nDA+Etfb8a4Cmow/SLJ+Zra81v5jZ+cA24JmM9l+3ZVwDHCZoT/1aw3Np7MOmv6k25Z4GHgz32Txw\nrQWjlrYAlwLfTaBMPZXPzC4HvgBMuPtPIuub/r0TLl+3Zbwo8nAC+FF4/z7gg2FZ1wMfpL6mnUn5\nwjK+naAD99uRdVntw07mgevD0Um/CfwsPFGKt//S7lWPuwCTBNHvFeAl4L5w/a8Cfx/Z7mrgOYKI\nvSeyfpzgn/Io8DXgnITLtwF4AHg+vD0vXL8V+GJkuzHgH4FSw+sfBJ4mOJjdBfxy1uUDfissw5Ph\n7cez2n89lPEjwKvAE5HlsjT3YbPfFEET1UR4/9xwnxwN99F45LV7wtc9C3w4pf+NTuX7Zvg/U9tf\n853+3n0o46eBH4ZleQj4tchr/zDct0eBj/WjfOHj/wZ8puF1mexDghPJpfC3f4Kgr+iTwCfD5w04\nEJb/aSIjN+PsP818FhGROoPSlCQiIglRYBARkToKDCIiUkeBQURE6igwiIhIHQUGERGpo8AgIiJ1\nFBhERKTO/wcnKJlCyVwy+wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the training error rate is: 0.000000\n",
      "the test error rate is: 0.070000\n"
     ]
    }
   ],
   "source": [
    "testRbf(k1=1.3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Experiment 2: Handwriting dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Support vector machine handwriting recognition\n",
    "def img2vector(filename):\n",
    "    returnVect = np.zeros(1024)\n",
    "    fr = open(filename)\n",
    "    for i in range(32):\n",
    "        lineStr = fr.readline()\n",
    "        for j in range(32):\n",
    "            returnVect[32 * i + j] = int(lineStr[j])\n",
    "    return returnVect\n",
    "\n",
    "\n",
    "def loadImages(dirName):\n",
    "    hwLabels = []\n",
    "    trainingFileList = listdir(dirName)\n",
    "    m = len(trainingFileList)\n",
    "    trainingMat = np.zeros((m, 1024))\n",
    "    for i in range(m):\n",
    "        fileNameStr = trainingFileList[i]\n",
    "        fileStr = fileNameStr.split('.')[0]\n",
    "        classNumStr = int(fileStr.split('_')[0])\n",
    "        if classNumStr == 9:\n",
    "            hwLabels.append(-1)\n",
    "        else:\n",
    "            hwLabels.append(1)\n",
    "        trainingMat[i] = img2vector('%s/%s' % (dirName, fileNameStr))\n",
    "    return np.array(trainingMat), np.array(hwLabels)\n",
    "\n",
    "\n",
    "def testDigits(kTup=('rbf', 10)):\n",
    "    dataArr, labelArr = loadImages('trainingDigits')\n",
    "    b, alphas = smoP(dataArr, labelArr, 200, 0.0001, 10000, kTup)\n",
    "    svInd = np.nonzero(alphas > 0)[0]\n",
    "    sVs = dataArr[svInd]\n",
    "    labelSV = labelArr[svInd]\n",
    "    print(\"there are %d Support Vectors\" % sVs.shape[0])\n",
    "    m, n = dataArr.shape\n",
    "    errorCount = 0\n",
    "    for i in range(m):\n",
    "        kernelEval = kernelTrans(sVs, dataArr[i], kTup)\n",
    "        predict = np.dot(kernelEval, np.multiply(labelSV, alphas[svInd])) + b\n",
    "        if np.sign(predict) != np.sign(labelArr[i]):\n",
    "            errorCount += 1\n",
    "    print(\"the training error rate is: %f\" % (errorCount / m))\n",
    "    dataArr, labelArr = loadImages('testDigits')\n",
    "    errorCount = 0\n",
    "    m, n = dataArr.shape\n",
    "    for i in range(m):\n",
    "        kernelEval = kernelTrans(sVs, dataArr[i], kTup)\n",
    "        predict = np.dot(kernelEval, np.multiply(labelSV, alphas[svInd])) + b\n",
    "        if np.sign(predict) != np.sign(labelArr[i]):\n",
    "            errorCount += 1\n",
    "    print(\"the test error rate is: %f\" % (errorCount / m))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "iteration number: 1\n",
      "iteration number: 2\n",
      "iteration number: 3\n",
      "iteration number: 4\n",
      "there are 51 Support Vectors\n",
      "the training error rate is: 0.000000\n",
      "the test error rate is: 0.016129\n"
     ]
    }
   ],
   "source": [
    "testDigits(('rbf', 20))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "iteration number: 1\n",
      "iteration number: 2\n",
      "iteration number: 3\n",
      "iteration number: 4\n",
      "iteration number: 5\n",
      "iteration number: 6\n",
      "iteration number: 7\n",
      "there are 402 Support Vectors\n",
      "the training error rate is: 0.000000\n",
      "the test error rate is: 0.521505\n"
     ]
    }
   ],
   "source": [
    "testDigits(('rbf', 0.1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "iteration number: 1\n",
      "iteration number: 2\n",
      "iteration number: 3\n",
      "iteration number: 4\n",
      "iteration number: 5\n",
      "iteration number: 6\n",
      "iteration number: 7\n",
      "iteration number: 8\n",
      "there are 402 Support Vectors\n",
      "the training error rate is: 0.000000\n",
      "the test error rate is: 0.032258\n"
     ]
    }
   ],
   "source": [
    "testDigits(('rbf', 5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "iteration number: 1\n",
      "iteration number: 2\n",
      "iteration number: 3\n",
      "iteration number: 4\n",
      "there are 105 Support Vectors\n",
      "the training error rate is: 0.000000\n",
      "the test error rate is: 0.005376\n"
     ]
    }
   ],
   "source": [
    "testDigits(('rbf', 10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "iteration number: 1\n",
      "iteration number: 2\n",
      "iteration number: 3\n",
      "iteration number: 4\n",
      "iteration number: 5\n",
      "iteration number: 6\n",
      "there are 34 Support Vectors\n",
      "the training error rate is: 0.014925\n",
      "the test error rate is: 0.010753\n"
     ]
    }
   ],
   "source": [
    "testDigits(('rbf', 50))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "iteration number: 1\n",
      "iteration number: 2\n",
      "iteration number: 3\n",
      "iteration number: 4\n",
      "there are 26 Support Vectors\n",
      "the training error rate is: 0.044776\n",
      "the test error rate is: 0.043011\n"
     ]
    }
   ],
   "source": [
    "testDigits(('rbf', 100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "iteration number: 1\n",
      "iteration number: 2\n",
      "iteration number: 3\n",
      "iteration number: 4\n",
      "there are 33 Support Vectors\n",
      "the training error rate is: 0.007463\n",
      "the test error rate is: 0.010753\n"
     ]
    }
   ],
   "source": [
    "testDigits(('lin',))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}