{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# # Implementation of Matrix Factorization in Python\n",
    "\n",
    "Please refer to the article at http://www.albertauyeung.com/post/python-matrix-factorization/ for the detailed explanation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "class MF():\n",
    "    \n",
    "    def __init__(self, R, K, alpha, beta, iterations):\n",
    "        \"\"\"\n",
    "        Perform matrix factorization to predict empty\n",
    "        entries in a matrix.\n",
    "        \n",
    "        Arguments\n",
    "        - R (ndarray)   : user-item rating matrix\n",
    "        - K (int)       : number of latent dimensions\n",
    "        - alpha (float) : learning rate\n",
    "        - beta (float)  : regularization parameter\n",
    "        \"\"\"\n",
    "        \n",
    "        self.R = R\n",
    "        self.num_users, self.num_items = R.shape\n",
    "        self.K = K\n",
    "        self.alpha = alpha\n",
    "        self.beta = beta\n",
    "        self.iterations = iterations\n",
    "\n",
    "    def train(self):\n",
    "        # Initialize user and item latent feature matrice\n",
    "        self.P = np.random.normal(scale=1./self.K, size=(self.num_users, self.K))\n",
    "        self.Q = np.random.normal(scale=1./self.K, size=(self.num_items, self.K))\n",
    "        \n",
    "        # Initialize the biases\n",
    "        self.b_u = np.zeros(self.num_users)\n",
    "        self.b_i = np.zeros(self.num_items)\n",
    "        self.b = np.mean(self.R[np.where(self.R != 0)])\n",
    "        \n",
    "        # Create a list of training samples\n",
    "        self.samples = [\n",
    "            (i, j, self.R[i, j])\n",
    "            for i in range(self.num_users)\n",
    "            for j in range(self.num_items)\n",
    "            if self.R[i, j] > 0\n",
    "        ]\n",
    "        \n",
    "        # Perform stochastic gradient descent for number of iterations\n",
    "        training_process = []\n",
    "        for i in range(self.iterations):\n",
    "            np.random.shuffle(self.samples)\n",
    "            self.sgd()\n",
    "            mse = self.mse()\n",
    "            training_process.append((i, mse))\n",
    "            if (i+1) % 10 == 0:\n",
    "                print(\"Iteration: %d ; error = %.4f\" % (i+1, mse))\n",
    "        \n",
    "        return training_process\n",
    "\n",
    "    def mse(self):\n",
    "        \"\"\"\n",
    "        A function to compute the total mean square error\n",
    "        \"\"\"\n",
    "        xs, ys = self.R.nonzero()\n",
    "        predicted = self.full_matrix()\n",
    "        error = 0\n",
    "        for x, y in zip(xs, ys):\n",
    "            error += pow(self.R[x, y] - predicted[x, y], 2)\n",
    "        return np.sqrt(error)\n",
    "\n",
    "    def sgd(self):\n",
    "        \"\"\"\n",
    "        Perform stochastic graident descent\n",
    "        \"\"\"\n",
    "        for i, j, r in self.samples:\n",
    "            # Computer prediction and error\n",
    "            prediction = self.get_rating(i, j)\n",
    "            e = (r - prediction)\n",
    "            \n",
    "            # Update biases\n",
    "            self.b_u[i] += self.alpha * (e - self.beta * self.b_u[i])\n",
    "            self.b_i[j] += self.alpha * (e - self.beta * self.b_i[j])\n",
    "            \n",
    "            # Create copy of row of P since we need to update it but use older values for update on Q\n",
    "            P_i = self.P[i, :][:]\n",
    "            \n",
    "            # Update user and item latent feature matrices\n",
    "            self.P[i, :] += self.alpha * (e * self.Q[j, :] - self.beta * self.P[i,:])\n",
    "            self.Q[j, :] += self.alpha * (e * P_i - self.beta * self.Q[j,:])\n",
    "\n",
    "    def get_rating(self, i, j):\n",
    "        \"\"\"\n",
    "        Get the predicted rating of user i and item j\n",
    "        \"\"\"\n",
    "        prediction = self.b + self.b_u[i] + self.b_i[j] + self.P[i, :].dot(self.Q[j, :].T)\n",
    "        return prediction\n",
    "    \n",
    "    def full_matrix(self):\n",
    "        \"\"\"\n",
    "        Computer the full matrix using the resultant biases, P and Q\n",
    "        \"\"\"\n",
    "        return mf.b + mf.b_u[:,np.newaxis] + mf.b_i[np.newaxis:,] + mf.P.dot(mf.Q.T)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration: 2 ; error = 4.8575\n",
      "Iteration: 12 ; error = 0.1299\n",
      "\n",
      "P x Q:\n",
      "[[ 4.98421651  2.99708195  3.12289161  1.00971487]\n",
      " [ 3.99564826  2.1268789   3.50941346  1.01124379]\n",
      " [ 1.01613916  0.98386613  4.5843299   4.9908544 ]\n",
      " [ 0.99513504  0.75011143  3.73365012  3.98862199]\n",
      " [ 1.61407605  1.02983448  4.98695019  3.99475012]]\n",
      "\n",
      "Global bias:\n",
      "2.76923076923\n",
      "\n",
      "User bias:\n",
      "[ 0.1810972  -0.0346873   0.11877163 -0.44968314  0.29954531]\n",
      "\n",
      "Item bias:\n",
      "[ 0.34310914 -0.81457993  0.851025   -0.22669804]\n"
     ]
    }
   ],
   "source": [
    "R = np.array([\n",
    "    [5, 3, 0, 1],\n",
    "    [4, 0, 0, 1],\n",
    "    [1, 1, 0, 5],\n",
    "    [1, 0, 0, 4],\n",
    "    [0, 1, 5, 4],\n",
    "])\n",
    "\n",
    "mf = MF(R, K=2, alpha=0.1, beta=0.01, iterations=20)\n",
    "training_process = mf.train()\n",
    "print()\n",
    "print(\"P x Q:\")\n",
    "print(mf.full_matrix())\n",
    "print()\n",
    "print(\"Global bias:\")\n",
    "print(mf.b)\n",
    "print()\n",
    "print(\"User bias:\")\n",
    "print(mf.b_u)\n",
    "print()\n",
    "print(\"Item bias:\")\n",
    "print(mf.b_i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA6oAAAEKCAYAAAAb/6jZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl8VPW9//H3Z2ayAAkoSwKKFgQRRXbEBVS0dcOtrdq6\n1rVo625vvVr7a2vvbavV2muVaq1a3G/rWq24cG3RigqyL4LKIgiyrwmQbebz+2MmYRKyTCCTM0le\nz8fjPDLnzDln3hk0yfuc7zlj7i4AAAAAADJFKOgAAAAAAAAko6gCAAAAADIKRRUAAAAAkFEoqgAA\nAACAjEJRBQAAAABkFIoqAAAAACCjUFQBAAAAABmFogoAAAAAyCgUVQAAAABARokEHSBZ165dvVev\nXkHHAAAAAAA0sRkzZmxw926prJtRRbVXr16aPn160DEAAAAAAE3MzJanui5DfwEAAAAAGYWiCgAA\nAADIKBRVAAAAAEBGoagCAAAAADIKRRUAAAAAkFEoqgAAAACAjEJRBQAAAABkFIpqij5YvEETpixT\nNOZBRwEAAACAVo2imqLX563WL177ROc9/IEWrysKOg4AAAAAtFoU1RT99zcP1++/O1jLNmzX2Pvf\n1wPvfK6yiljQsQAAAACg1aGopsjM9K2hPTXpluN1yuHd9btJn+msB9/X3JVbgo4GAAAAAK0KRbWR\nuubl6IELhurP3xuhzTvK9M3xU/TriQu1sywadDQAAAAAaBUoqnvopMMKNemW43X+yAP1yHtLder9\n7+mDJRuCjgUAAAAALR5FdS90zM3Sr781UM99/yiZpAv/PFW3vzRP20rKg44GAAAAAC0WRbUJHN2n\ni9648ThdfdxB+uvHK3TSfe9q0idrg44FAAAAAC0SRbWJtMsO6/axh+qVa0dp3/bZ+v6T03XdszO1\nobg06GgAAAAA0KJQVJvYoJ776LXrR+s/Tu6ntxes1Tfue1cvzVwpdw86GgAAAAC0CBTVNMgKh3Td\niQdr4o2j1adbnm752xxd9pePtWrLzqCjAQAAAEDGo6imUd+CfD1/9dG686wB+viLTTr5vnf15Idf\nKBbj7CoAAAAA1IWimmahkOnSY3rp7ZuP0/BenfWzvy/Qd/70oRavKw46GgAAAABkJIpqM+m5b3s9\ncfkR+t15g7V4fbHG3v9vjf/XYpVHY0FHAwAAAICMQlFtRmamc4b31KSbj9dJAwp1z1uf6qwHp2je\nyq1BRwMAAACAjEFRDUC3/ByNv3CY/nTJcG0sLtU3/zhFv3ljoUrKo0FHAwAAAIDAUVQDdMqA7pp0\ny/H6zoie+tO7S3Xa/f/WR0s3Bh0LAAAAAAJFUQ1Yp3ZZ+s23B+nZq45UNOY6/5GPdMfL81RUUh50\nNAAAAAAIBEU1QxzTt6veuuk4ff/Y3npu2gqddN97emfh2qBjAQAAAECzo6hmkHbZYd1x+mF6+Yej\ntE/7LF35xHTd8NwsbSwuDToaAAAAADSbtBZVM/vCzOaZ2Wwzm57O12pNBh+wj169brRuOamf3pi/\nWt+47129MmuV3D3oaAAAAACQds1xRvUEdx/i7iOa4bVajexISDd8/WC9fsOx6tW1g27662xdMeFj\nfbVlZ9DRAAAAACCtGPqb4foV5uuFa47Rz888TB8t3aSTf/+envpouWIxzq4CAAAAaJ3SXVRd0ttm\nNsPMxqX5tVqtcMh0+ajeevvm4zT0wH30/16Zr/Mf+UhL1xcHHQ0AAAAAmlwkzfsf7e6rzKxA0iQz\nW+Tu7yWvkCiw4ySpsLBQkydPTnOklu2Kg1z9crL13KJNOvn37+qbfbN0aq8sRUIWdDQAAAAAaBLW\nXDfoMbNfSCp293vrWmfEiBE+fTr3XErFuqIS/eLVBZo4b40G7NdRd58zSIfv3ynoWAAAAABQKzOb\nkeq9i9I29NfMOphZfuVjSSdLmp+u12trCvJz9ceLhuvhi4dpXVGpzh4/RXe/uUgl5dGgowEAAADA\nXknnNaqFkt43szmSpkl63d3fTOPrtUmnHt5D/3fz8Tpn2P56aPISjb3/35q2bFPQsQAAAABgjzXb\n0N9UMPR377z/+Qbd/vJcfblpp278+sG6+aR+QUcCAAAAAEkZMvQXzW/0wV311k3H6ZxhPXX/O5/r\noclLgo4EAAAAAI2W7rv+opm1z47ot+cOUkUsprvfXKS83IguOeprQccCAAAAgJRRVFuhcMh073mD\nVVxSoZ/9fb7ycsL61tCeQccCAAAAgJQw9LeVygqHNP6iYTqqdxf9x/Nz9faCNUFHAgAAAICUUFRb\nsdyssP586Qgdvn8nXffsLE1ZvCHoSAAAAADQIIpqK5eXE9ETlx+h3l076PtPTtfMFZuDjgQAAAAA\n9aKotgH7tM/WU1eOVLf8HF32+DQtXL0t6EgAAAAAUCeKahtR0DFXT195pDrkRHTJY9O0bMP2oCMB\nAAAAQK0oqm3IAZ3b66krj5S76+JHp2rVlp1BRwIAAACA3VBU25i+BXl64oqR2razXJc8OlXri0qD\njgQAAAAA1VBU26DD9++kxy8/Ql9t3anvPT5NW3eUBx0JAAAAAKpQVNuoI3p11p8uGaHF64p0+YRp\n2lFWEXQkAAAAAJBEUW3Tju/XTX84f6hmf7lFVz81Q6UV0aAjAQAAAABFta07bWAP/fbcwfr35xt0\nw3OzVBGNBR0JAAAAQBtHUYXOHd5TvzjzML21YK1ufWGuYjEPOhIAAACANiwSdABkhstG9VZRSYV+\nN+kz5eVGdOdZA2RmQccCAAAA0AZRVFHluhP7qqi0Qo+8t1T5uRH9+JT+QUcCAAAA0AZRVFHFzHT7\naf1VVFKu8f9aovzcLF1zfJ+gYwEAAABoYyiqqMbM9N/fHKiikgrd9cYi5edGdNGRXws6FgAAAIA2\nhKKK3YRDpt9/d4h2lEX101fmKy8norOH7B90LAAAAABtBHf9Ra2ywiH98aJhOrJ3Z93ytzn6v0/W\nBh0JAAAAQBtBUUWdcrPCevTSI3T4fh31w2dn6oPFG4KOBAAAAKANoKiiXnk5EU24fKR6d+mgq56c\nrlkrNgcdCQAAAEArR1FFg/btkK2nrhyprnk5uuwvH2vh6m1BRwIAAADQilFUkZKCjrl65qoj1S4r\nrEsem6YvNmwPOhIAAACAVqreompmYTO7ubnCILMd0Lm9nr5qpGLuuujRqfpqy86gIwEAAABoheot\nqu4elXRBM2VBC9C3IF9PXjFS23aW6+LHpmpDcWnQkQAAAAC0MqkM/Z1iZg+a2bFmNqxySvUFEmdl\nZ5nZP/YiJzLI4ft30uOXH6GvtuzU9x6bpq07y4OOBAAAAKAVSaWoDpE0QNIvJf0uMd3biNe4UdLC\nxkdDJjuiV2f96ZIR+nxdka6Y8LF2lFUEHQkAAABAK9FgUXX3E2qZTkxl52bWU9Lpkh7d26DIPMf3\n66Y/nD9Us1Zs1tVPzVBpRTToSAAAAABagUhDK5hZJ0k/l3RcYtG7kn7p7ltT2P//SLpVUn49+x8n\naZwkFRYWavLkySnsFpminaTLB2TrsfkbdOEDk/TDwTkKhyzoWAAAAABasAaLqqTHJc2X9J3E/CWS\n/iLp2/VtZGZnSFrn7jPMbExd67n7I5IekaQRI0b4mDF1rooMNUbS/u8v0y//8Ykmbuise84dpBBl\nFQAAAMAeSqWo9nH3c5Lm7zSz2SlsN0rSWWY2VlKupI5m9rS7X7wnQZHZrhjdW0UlFfr9/32m/NyI\nfn7mYTKjrAIAAABovFRuprTTzEZXzpjZKEkNfoCmu9/u7j3dvZek8yX9k5Laut3w9b66anRvTfjg\nC9036bOg4wAAAABooVI5o3qNpCcT16pK0mZJl6YvEloqM9Mdpx+q4tIKPfDPxcrPjWjccX2CjgUA\nAACgham3qJpZSNIh7j7YzDpKkrtva+yLuPtkSZP3JCBaFjPTr741UMWlFfr1xEXKy8nShUceGHQs\nAAAAAC1IvUXV3WNmdqukv+1JQUXbFA6Z7vvOEG0vrdAdr8xTh5ywzh6yf9CxAAAAALQQqVyj+n9m\n9h9mdoCZda6c0p4MLVp2JKSHLh6ukb0660d/m6N3Fq4NOhIAAACAFiKVovpdSddKek/SjMQ0PZ2h\n0DrkZoX16KUjdNh+HfXDZ2bqwyUbg44EAAAAoAWot6gmrlG92N1715gOaqZ8aOHyc7P0xOUjdWDn\n9rrqiY81+8stQUcCAAAAkOHqLaruHpP0YDNlQSu1b4dsPX3VkeqSl6NLH5+mT9cUBR0JAAAAQAZL\nZejvO2Z2jplZ2tOg1SrsmKtnrjpSuVkhXfzYVH2xYXvQkQAAAABkqFSK6tWSnpdUambbzKzIzLgD\nMBrtgM7t9fSVR6oiGtNFj07V6q07g44EAAAAIAM1WFTdPd/dQ+6e7e4dE/MdmyMcWp+DC/P15BVH\natvOcl386FRtLC4NOhIAAACADFNnUTWzi5Mej6rx3HXpDIXWbWDPTnrssiO0astOfe/xadpWUh50\nJAAAAAAZpL4zqrckPX6gxnNXpCEL2pCRvTvr4YuH69M1RfrR3+YoFvOgIwEAAADIEPUVVavjcW3z\nQKONOaRAPxl7qCZ9slYPv7ck6DgAAAAAMkR9RdXreFzbPLBHLh/VS2cM6qF73/pUUxZvCDoOAAAA\ngAxQX1Htb2ZzzWxe0uPK+UOaKR9aOTPT3ecMUp9uebrhuVncCRgAAABAvUX1UElnSjoj6XHl/GHp\nj4a2okNORA9dPFwl5VH98JmZKquIBR0JAAAAQIDqLKruvry+qTlDovXrW5Cne84brFkrtuhXr38S\ndBwAAAAAAWrwc1SB5jJ2YA99/9jeeuLD5Xpl1qqg4wAAAAAICEUVGeXWU/trZK/Ouv2lefp0TVHQ\ncQAAAAAEIKWiambtzIwbKCHtssIhPXjhUOXlRnTN0zO0raQ86EgAAAAAmlmDRdXMzpQ0W9Kbifkh\nZvZquoOh7SromKvxFw7Tik079OPn58idT0MCAAAA2pJUzqj+QtJISVskyd1nS+qdxkyARvburNtP\n66+3FqzVn95bGnQcAAAAAM0olaJa7u5bayzjFBfS7srRvXX6oB767ZuL9MGSDUHHAQAAANBMUimq\nC8zsQklhMzvYzB6Q9EGacwEyM919ziD17tpBNzw3S2u2lgQdCQAAAEAzSKWoXi9pgKRSSc9K2irp\npnSGAirl5UT0p0uGa2dZVD98ZobKKmJBRwIAAACQZvUWVTMLS/qlu9/h7kckpp+6O6e20Gz6FuTr\nt+cO1swVW/TriQuDjgMAAAAgzeotqu4elTS6mbIAdTp9UA9dObq3Jnzwhf4+e1XQcQAAAACkUSSF\ndWYlPo7meUnbKxe6+0tpSwXU4rbT+mvuyi267cV56t+9ow7pnh90JAAAAABpkMo1qrmSNko6UdKZ\niemMhjYys1wzm2Zmc8xsgZnduXdR0dZlhUMaf+Ew5eVG9IOnZ6iopDzoSAAAAADSoMEzqu5++R7u\nu1TSie5ebGZZkt43szfc/aM93B+ggo65Gn/hMF3w54/04+fn6qGLh8nMgo4FAAAAoAk1eEY1cWb0\nWjP7o5k9Xjk1tJ3HFSdmsxITn7+KvTayd2fdflp/vblgjR55b2nQcQAAAAA0sVSG/j4lqbukUyS9\nK6mnpKJUdm5mYTObLWmdpEnuPnVPgwLJrhzdW2MHdtfdby7Sh0s2Bh0HAAAAQBNK5WZKfd39PDM7\n292fMLNnJf07lZ0n7ho8xMz2kfSymR3u7vOT1zGzcZLGSVJhYaEmT57cuO8AbdaZha6ZS01XPzFV\ndx6Tq31zUznuAgAAACDTpVJUK+9Ys8XMDpe0RlJBY17E3beY2b8knSppfo3nHpH0iCSNGDHCx4wZ\n05hdo43rO7BIZ4+fomeW5eq5cUcpK0xZBQAAAFq6VP6qf8TM9pX0/yS9KukTSb9taCMz65Y4kyoz\nayfpJEmL9iIrsJuDC/N19zmDNH35Zv164sKg4wAAAABoAqnc9ffRxMN3JR3UiH33kPSEmYUVL8R/\nc/d/ND4iUL8zB++nmSs26y9TvtDQA/fVWYP3CzoSAAAAgL3QYFE1s5/Vttzdf1nfdu4+V9LQPcwF\nNMpPxh6qeSu36rYX5+rQ7vk6uDA/6EgAAAAA9lAqQ3+3J01RSadJ6pXGTECjZYVDGn/RMLXPjujq\np2eoqKS84Y0AAAAAZKQGi6q7/y5p+pWkMWrcEGCgWRR2zNWDFw7V8o07dOsLc+XOx/YCAAAALdGe\n3CK1veKfpQpknKMO6qL/PPUQvTF/jR7997Kg4wAAAADYA6lcozpPUuWpqbCkbpLqvT4VCNL3jz1I\ns1Zs0V1vLtKgnp105EFdgo4EAAAAoBFSOaN6hqQzE9PJkvZz9wfTmgrYC2am3547SF/r0l7XPjtL\na7eVBB0JAAAAQCOkUlSLkqadkjqaWefKKa3pgD2Un5ulhy8eru2lFbr2mZkqj8aCjgQAAAAgRakU\n1ZmS1kv6TNLnicczEtP09EUD9k6/wnzdfe4gTV++Wb+ZuCjoOAAAAABSlEpRnSTpTHfv6u5dFB8K\n/La793Z37v6LjHbW4P102TG99PiUZfrH3K+CjgMAAAAgBakU1aPcfWLljLu/IemY9EUCmtZPxh6q\nYQfuo1tfmKvP1xYFHQcAAABAA1Ipql+Z2U/NrFdiukMSp6bQYmRHQvrjRcPVPjusa56eoeLSiqAj\nAQAAAKhHKkX1AsU/kublxFSQWAa0GN075eoPFwzVsg3b9Z8vzJW7N7wRAAAAgEA0WFTdfZO73+ju\nQyWdKOkmd9+U/mhA0zqmT1fdemp/vT5vtR57f1nQcQAAAADUoc6iamY/M7P+icc5ZvZPSYslrTWz\nbzRXQKApXX3cQTplQKF+88YiTVvG8RYAAAAgE9V3RvW7kj5NPL40sW6BpOMl/TrNuYC0MDPdc95g\nHdi5va59dqbWbSsJOhIAAACAGuorqmW+60K+UyQ95+5Rd18oKZL+aEB6dMzN0sMXD1dxSYWue3aW\nyqOxoCMBAAAASFJfUS01s8PNrJukEyS9nfRc+/TGAtLrkO75uuucgZr2xSbd/caioOMAAAAASFLf\nmdEbJb2g+B1/f+/uyyTJzMZKmtUM2YC0OnvI/pq5fLMefX+Zhh64r04f1CPoSAAAAABUT1F196mS\n+teyfKKkiekMBTSXO04/TPNWbdWtL8zRId3z1LcgP+hIAAAAQJuXyueoAq1WdiSk8RcNU25WWNc8\nPVPbSyuCjgQAAAC0eRRVtHk9OrXTAxcM1dL1xbr1xbnadQ8xAAAAAEGgqAKSjunbVT8+pb9en7ta\nj0/5Iug4AAAAQJuW0sfMmNkxknolr+/uT6YpExCIa44/SDNXbNZvJi7UoJ6ddESvzkFHAgAAANqk\nBs+omtlTku6VNFrSEYlpRJpzAc3OzPS77wxWz33b6dpnZmpdUUnQkQAAAIA2KZWhvyMkjXL3H7r7\n9YnphnQHA4LQMTdLD108XNtKynXds7NUHo0FHQkAAABoc1IpqvMldU93ECBTHNqjo37z7YGatmyT\n7nnr06DjAAAAAG1OKteodpX0iZlNk1RaudDdz0pbKiBg3xraUzOXb9Ej7y3V0AP20WkDewQdCQAA\nAGgzUimqv9iTHZvZAZKelFQoySU94u7378m+gCD89IxDNW/VVv34hbnq1z1ffbrlBR0JAAAAaBMa\nHPrr7u/WNqWw7wpJP3L3wyQdJelaMztsbwMDzSUnEtYfLxqm7EhI1zw1Q9tLK4KOBAAAALQJqdz1\n9ygz+9jMis2szMyiZratoe3cfbW7z0w8LpK0UNL+ex8ZaD777dNOD1wwVEvWF+vWF+cqGvOgIwEA\nAACtXio3U3pQ0gWSPpfUTtJVksY35kXMrJekoZKmNi4eELxRfbvq1lP76/W5q3X5hI+1dUd50JEA\nAACAVi2Va1Tl7ovNLOzuUUl/MbNZkm5PZVszy5P0oqSb3H23M7FmNk7SOEkqLCzU5MmTU80ONJv+\nki4bkK2nPlmvk+6dpBuH5mr//FSO8wAAAABoLHOvfyijmb0n6RuSHpW0RtJqSZe5++AGd26WJekf\nkt5y9/saWn/EiBE+ffr0VHIDgZixfJOueXqmdpRW6L7vDtEpA/jkJgAAACAVZjbD3Ueksm4qp4Qu\nSax3naTtkg6QdE4KIUzSY5IWplJSgZZg+Nc667XrRqtvYb6ufmqG7pv0mWJctwoAAAA0qVTu+rtc\nkknq4e53uvst7r44hX2PUrzknmhmsxPT2L3MCwSue6dc/XXcUTp3eE/94Z3PNe6pGSoq4bpVAAAA\noKmkctffMyXNlvRmYn6Imb3a0Hbu/r67m7sPcvchiWni3kcGgpebFdY95w7SnWcN0L8+Xadvjp+i\npeuLg44FAAAAtAqpDP39haSRkrZIkrvPltQ7jZmAFsHMdOkxvfT0lUdq845ynf3gFP1z0dqgYwEA\nAAAtXipFtdzdt9ZYxkV5QMLRfbro1etG6cAu7XXlE9M1/l+L1dBNygAAAADULZWiusDMLpQUNrOD\nzewBSR+kORfQovTct71euOYYnTloP93z1qe69tmZ2l5aEXQsAAAAoEVKpaheL2mApFJJz0naJumm\ndIYCWqJ22WHdf/4Q3TH2UL05f43OeegDrdi4I+hYAAAAQIvT4OeoNic+RxWtxXufrdf1z82SJD14\n4VAde3C3gBMBAAAAwWrM56jWWVQburOvu5+1B9nqRVFFa7J843aNe3KGPl9XpNtPO1RXHdtb8Y8X\nBgAAANqexhTVSD3PHS3pS8WH+05V/LNUAaToa1066KUfHqP/eH6OfjVxoRZ8tVV3nTNIuVnhoKMB\nAAAAGa2+a1S7S/qJpMMl3S/pJEkb3P1dd3+3OcIBLV2HnIj+eNEw/fiUQ/T3OV/p3Ic/0KotO4OO\nBQAAAGS0Oouqu0fd/U13v1TSUZIWS5psZtc1WzqgFTAzXXtCXz126Qgt37BDZz3wvj5aujHoWAAA\nAEDGqveuv2aWY2bflvS0pGsl/UHSy80RDGhtTuxfqFeuG6VO7bN08aNT9cQHX/B5qwAAAEAt6iyq\nZvakpA8lDZN0p7sf4e7/5e6rmi0d0Mr06ZanV64dpTGHdNPPX12g/3xxrkrKo0HHAgAAADJKfXf9\njUnanphNXskkubt3bOow3PUXbUUs5vqfdz7XH975XEMO2Ed/umS4CjvmBh0LAAAASJvG3PW3vmtU\nQ+6en5g6Jk356SipQFsSCpluOamfHr54uD5fW6QzHnhfM5ZvDjoWAAAAkBHqvUYVQHqdenh3vXzt\nKLXPDuv8Rz7U/05bEXQkAAAAIHAUVSBg/Qrz9eq1o3V0n6667aV5+ukr81RWEQs6FgAAABAYiiqQ\nATq1z9JfLjtC1xzfR09/tEIXPfqR1heVBh0LAAAACARFFcgQ4ZDpttP66w8XDNW8VVt15gPva+7K\nLUHHAgAAAJodRRXIMGcN3k8v/uAYhUOmcx/+UC/OWBl0JAAAAKBZUVSBDDRgv0567frRGn7gvvrR\n83P0y9c+UUWU61YBAADQNlBUgQzVuUO2nrxypC4f1UuPT1mm7z0+TZu2lwUdCwAAAEg7iiqQwbLC\nIf38zAG697zBmr58s8584H0t+Gpr0LEAAACAtKKoAi3AucN76vmrj1Y05jrnoQ/02pyvgo4EAAAA\npA1FFWghBh+wj167frQG7t9J1z83S3e9sUjRmAcdCwAAAGhyFFWgBemWn6NnrjpKFx91oB5+d4ku\nn/Cxtu4oDzoWAAAA0KQoqkALkx0J6b+/OVC/+fZAfbhkg84a/74+W1sUdCwAAACgyVBUgRbqgpEH\n6n/HHaUdZVF9a/wUvTl/TdCRAAAAgCZBUQVasOFf66zXrhutvoX5uubpGbpv0meKcd0qAAAAWri0\nFVUze9zM1pnZ/HS9BgCpe6dc/XXcUTpveE/94Z3PNe6pGSoq4bpVAAAAtFzpPKM6QdKpadw/gITc\nrLB+e+4g/fLsAZr86TqNvvtf+vHzc/SvT9eprCIWdDwAAACgUSLp2rG7v2dmvdK1fwDVmZm+d3Qv\nDdy/k576cLnenL9Gz89YqY65EZ10WHedPqi7RvXtqpxIOOioAAAAQL3SVlRTZWbjJI2TpMLCQk2e\nPDnYQEArcFahdFq3bC3YENbHa6J6Y+5KvThzpdpFpKEFER3RPawBXcLKDlvQUQEAAIDdmHv6bryS\nOKP6D3c/PJX1R4wY4dOnT09bHqCtKquIacqSDZo4d7Xe/mSttu4sV15ORN84tECnDeyh4/t1U24W\nZ1oBAACQPmY2w91HpLJu4GdUAaRfdiSkEw4p0AmHFOjX0Zg+WLJRE+eu1lufrNErs79Sh+ywvn5o\nocYO7K4xhxRQWgEAABAoiirQxmSFQzq+Xzcd36+b/jt6uD5aulET563WWwvW6tU5X6l9dlgn9C/Q\n6QN76IRDCtQum9IKAACA5pW2ob9m9pykMZK6Slor6efu/lh92zD0FwhORTSmqcs26fV5q/XW/DXa\nuL1M7bLCOqF/N40d2EMn9i9Q+2yObQEAAGDPNGbob1qvUW0siiqQGaIx19RlG/XGvDV6Y/4abSgu\nVW5WSGP6FWjsoB76ev8CdcihtAIAACB1FFUATSYac338xSa9MW+13pi/RuuKSpUTiQ8fPn1Q/Exr\nfm5W0DEBAACQ4SiqANIiFnNNX75ZE+et1hvzV2vttlJlR0I67uBuOn1Qd3390EJ1pLQCAACgFhRV\nAGkXi7lmrtisifPW6I35q7V6a4mywyEde3BXnTawh046rFCd2lFaAQAAEEdRBdCsYjHXrC+3VA0P\nXrVlp7LCptF946X15MMKtU/77KBjAgAAIEAUVQCBcXfNWblVE+et1utzV2vVlp2KhEyj+nbV2IHd\ndfJh3bVvB0orAABAW0NRBZAR3F3zVm3V6/NWa+K81fpy006FQ6Zj+nTR2IE9dMqA7upMaQUAAGgT\nKKoAMo67a8FX26pK6/KNOxQOmQ7q2kF9uuWpT0H8a9+CPB3ULU95fPwNAABAq0JRBZDR3F2frN6m\ntxas1cIwyX0OAAAVGklEQVTV27RkfbGWb9yhaGzXz6PuHXOrldc+3eJTYcccmVmA6QEAALAnGlNU\nOWUBoNmZmQbs10kD9utUtaysIqYVm7Zr8brtWrK+OD6tK9ZLM1epuLSiar28nIj6dKs8C5unPt06\nqG9Bng7s3EHZkVAQ3w4AAACaGEUVQEbIjoTUtyBffQvyqy13d60rKtWSdfHyunhdsZas364Pl27U\nS7NWVa0XDpm+1rm9DkoMI+5bVWTz+JgcAACAFoaiCiCjmZkKO+aqsGOujunbtdpzxaUVWlp19nV7\nVZF997N1Ko/uGkbcNS9HfRPDiJPPxO7XqZ1CIYYRAwAAZBqKKoAWKy8nokE999GgnvtUW14RjenL\nzTu1ZF2xFieGEC9ZX6zX5nylbSW7hhG3ywrroG67CmzfgvjZ2F5dOig3K9zc3w4AAAASKKoAWp1I\nOKTeXTuod9cO+oYKq5a7uzYUlyVdA7tdi9cXa8byzXp1zldV65lJB+zbvur61z7d8tS7awd175Sr\ngvxctcumxAIAAKQTRRVAm2Fm6pafo275OTrqoC7VnttZFtXSDbuuga28mdOUJRtVVhGrtm5+TqRq\nP93yc1SQn6uCjjnqlpcT/5pYtm/7LO5QDAAAsAcoqgAgqV12eLc7EUtSNOZatXmnlm3crrXbSrS+\nqLRqWldUonmrtmrdtnXaWR7dbZ9ZYVPXvBwVVJXa3KrHVV875qpbXg53LAYAAEhCUQWAeoRDpgO7\ntNeBXdrXu15xaUW8vG4r0bqqIrur0K7cvFOzVmzRxu1ltW6/T/uspBKbm1Ruq5+1zc+JcJYWAAC0\nehRVAGgCeTkR5eVE1Ltrh3rXK4/GtLG4TOuKSnYrs+u2lWp9cammLduk9cWluw05lqScSGjXMOM6\nhhx3yctWp3ZZapcVptQCAIAWiaIKAM0oKxxS90656t4pt9713F3bdlbUWmgr5xevL9aHSzdq687y\nWvcRCZk6tstSp3ZZ6pgbUcd2WfEpN0sd20XUMTfxXNLznZKez4lw0ygAABAMiioAZCAzU6f2WerU\nPksHF+bXu25JeVQbiuPldd22Um3aXqZtJeXatrNcW3eWa1tJRdXjVVt2atvO+HxZdPcztslys0KJ\n0hovsp2Sim78caTq+U41CnB+bkSRMNfdAgCAPUNRBYAWLjcrrJ77tlfPfeu/jramkvKotu0s17aS\nRKHdWVFnwd1WUq4NxWVaumF7Yt1yxbz+/eflRGo9k7ur1MYLbV5ORB1yIsrLCSsvJ0sdcsJVy7Io\nuwAAtEkUVQBoo3KzwsrNCqugY/3DkGvj7tpeFq0qrdsSxbZqvpbyu3LzDhWtjpffotKKlF4nJxJK\nKrKVpTasDjkR5edG1CE7sutxPYU3LyeinEiIa3YBAGghKKoAgEYzs6riuP8+7Rq9fUU0pqKSChWX\nxqftpTUfR3dbtr20QkUlFVpfXKovNu6IP1dSUetHA9UmErK9Kry5WWFlh0PKyQolvoaVEwkpEjIK\nMAAATYyiCgBodpFwSPt2yNa+HbL3el/RmGt7Wby0bq8qtlEVl5bXWngrC+72sgptK6nQ6q0lu7Yt\nq5A3MKS5JrP4md/K8lpbma2csiMh5UR2rVNtWY11dj2uvqyudbgmGADQmlBUAQAtWjhk8Wtec7P2\nel/urp3lURWXJBfe+OPSiqjKKmIqrYglvtacr5yi1ZaVVcT3sWl73dtWNHTBbwpCpqrCm1xwqwpz\nOFTHcyFlh6tvV7MkVz5fNZ+8Xi3PcZYZALC3KKoAACSYmdpnR9Q+O6KCZnzdimhMZdHYbkW4tEYR\nrr0gR1VaHt++tCKm0vJo1eOyGtvuLI9qy86y3ZaXVcRUmnj9ppB8ljm7xpng5FKbFY6X2nDIFAmF\nFE48js8nfw0pEjaFrMbycNLzIVNot+2q73e358LxbcOWPF9921BIVfvIClffJwAgfSiqAAAELBKO\nD91tv/cjofeKu+9WmKs9jlYvz9XXiyatV3fBLovGVFoeU0l5/DrlaMwVjbkqqr7GFI26op60PFr9\n+SY4Ab3XzFRVaLNCoarSXFliK0tvVtJ8zecrS3ZWuHopT56PJJf5pMdZ4V0Fvea8mRQyS0zxAzCh\nymWhyvldy0xJ64SSt0laJ2mf8cf1r5P8ujW/Vt9P9e05Ew+gUlqLqpmdKul+SWFJj7r7Xel8PQAA\nsOfMLHHta1j1f3pvsNxrltukEhtTvOwmL48ml9zk+eSvMUVr2zbmikZj1ebj28eXJc+XJ5XqisTz\nyfOVWSpiMZVUxPdXnryvpJy7nkvsK7Gf1i65vDZcbHeV5vrK727PhXZtH0rx9Sqfk3bft5LXU/Vl\nlfOh0K5tkzNJyfuqY9v4kYTE/ncdUKi5rWlXpuTsSvq+LGmb5O83ed6063uu/h7sOjhRPVvtBzF2\nrVO5/+rZEtGS//Wr/XdQfUn1gxjVlyfvYfd91Pzva0/2V5k/+b2p+R5Vf2+Tvnfteu/QOGkrqmYW\nljRe0kmSVkr62MxedfdP0vWaAACg9TNLnJEMB52kebm7Yi6VR2NJpbl6sS6PVp5xjq8bc1csFv/q\nlfOJ/VTur/K55Pnq61euU339htapeo1Y4nHV9xCfjyaKd+XzdW5fy+tW37/kqm2dpH3GUt9nNHHW\nvmqdmMtVuV58+6rvX/GvSs6s+OtJNd/LXf8uyd+nauwr5tptWWNv8obMVPPAQHLhryy9psqRDdVH\nO1Qv+InREbUcVPjOiJ4ad1yfYL/RJpLOM6ojJS1296WSZGb/K+lsSRRVAACARjIzhU0Kh9pYQ4ek\nXeW2WsnV7uV89+JbWYaTlscSBydqFHyvUbaTXyP5tT1pP5Xr1zx44TX249W+h137Vjxyte9z9+89\n6XHS2tWWV1tn931V22sq+0taPfn9qXr/VPeBi5oHfqptm/Qe1PU+Vtu28n1LOiCza53dc3TNy9nt\n/Wup0llU95f0ZdL8SklHpvH1AAAAgFap8kBF9YGqQOsV+M2UzGycpHGSVFhYqMmTJwcbCAAAAAAQ\nqHQW1VWSDkia75lYVo27PyLpEUkaMWKEjxkzJo2RAAAAAACZLpTGfX8s6WAz621m2ZLOl/RqGl8P\nAAAAANAKpO2MqrtXmNl1kt5S/ONpHnf3Bel6PQAAAABA65DWa1TdfaKkiel8DQAAAABA65LOob8A\nAAAAADQaRRUAAAAAkFEoqgAAAACAjGLuHnSGKma2XtLyoHPUo6ukDUGHSBFZm15LySmRNR1aSk6J\nrOnSUrK2lJwSWdOhpeSUyJouLSVrS8kpkbUpfc3du6WyYkYV1UxnZtPdfUTQOVJB1qbXUnJKZE2H\nlpJTImu6tJSsLSWnRNZ0aCk5JbKmS0vJ2lJySmQNCkN/AQAAAAAZhaIKAAAAAMgoFNXGeSToAI1A\n1qbXUnJKZE2HlpJTImu6tJSsLSWnRNZ0aCk5JbKmS0vJ2lJySmQNBNeoAgAAAAAyCmdUAQAAAAAZ\nhaKaIjM71cw+NbPFZnZb0HnqYmaPm9k6M5sfdJb6mNkBZvYvM/vEzBaY2Y1BZ6qLmeWa2TQzm5PI\nemfQmepjZmEzm2Vm/wg6S33M7Aszm2dms81setB56mNm+5jZC2a2yMwWmtnRQWeqjZkdkng/K6dt\nZnZT0LlqY2Y3J/5/mm9mz5lZbtCZ6mJmNyZyLsi097O2n/lm1tnMJpnZ54mv+waZsVIdWc9LvK8x\nM8uYu1TWkfWexM+AuWb2spntE2TGRKbacv5XIuNsM3vbzPYLMmOl+v4+MbMfmZmbWdcgstVUx/v6\nCzNblfTzdWyQGROZan1Pzez6xH+rC8zst0HlS1bHe/rXpPfzCzObHWTGSnVkHWJmH1X+zWJmI4PM\nWKmOrIPN7MPE31ivmVnHIDPuDYpqCswsLGm8pNMkHSbpAjM7LNhUdZog6dSgQ6SgQtKP3P0wSUdJ\nujaD39NSSSe6+2BJQySdamZHBZypPjdKWhh0iBSd4O5DWsBt1O+X9Ka795c0WBn6/rr7p4n3c4ik\n4ZJ2SHo54Fi7MbP9Jd0gaYS7Hy4pLOn8YFPVzswOl/R9SSMV/7c/w8z6Bpuqmgna/Wf+bZLecfeD\nJb2TmM8EE7R71vmSvi3pvWZPU78J2j3rJEmHu/sgSZ9Jur25Q9VignbPeY+7D0r8HPiHpJ81e6ra\nTVAtf5+Y2QGSTpa0orkD1WOCav9b6veVP2PdfWIzZ6rNBNXIaWYnSDpb0mB3HyDp3gBy1WaCamR1\n9+8m/c56UdJLQQSrxQTt/u//W0l3JrL+LDGfCSZo96yPSrrN3Qcq/jfAj5s7VFOhqKZmpKTF7r7U\n3csk/a/iPwQyjru/J2lT0Dka4u6r3X1m4nGR4n/47x9sqtp5XHFiNisxZeTF3WbWU9Lpiv+QQhMw\ns06SjpP0mCS5e5m7bwk2VUq+LmmJuy8POkgdIpLamVlEUntJXwWcpy6HSprq7jvcvULSu4oXq4xQ\nx8/8syU9kXj8hKRvNmuoOtSW1d0XuvunAUWqUx1Z3078NyBJH0nq2ezBaqgj57ak2Q7KkN9X9fx9\n8ntJtypDckot6m+p2nL+QNJd7l6aWGddswerRX3vqZmZpO9Ieq5ZQ9WhjqwuqfLMZCdlyO+sOrL2\n066Df5MkndOsoZoQRTU1+0v6Mml+pTK0VLVEZtZL0lBJU4NNUrfEcNrZktZJmuTumZr1fxT/hR8L\nOkgKXNLbZjbDzMYFHaYevSWtl/SXxJDqR82sQ9ChUnC+MuSXfk3uvkrxo/wrJK2WtNXd3w42VZ3m\nSzrWzLqYWXtJYyUdEHCmhhS6++rE4zWSCoMM00pdIemNoEPUxcx+ZWZfSrpImXNGdTdmdrakVe4+\nJ+gsKbouMaz68UwZUl+Lfor/zJpqZu+a2RFBB0rBsZLWuvvnQQepx02S7kn8f3WvMmNERV0WaNcJ\ntfOU+b+z6kRRRaDMLE/x4R431TgKnFHcPZoY7tFT0sjEcMCMYmZnSFrn7jOCzpKi0e4+TPEh9dea\n2XFBB6pDRNIwSQ+5+1BJ25U5QylrZWbZks6S9HzQWWqT+APvbMUPAuwnqYOZXRxsqtq5+0JJd0t6\nW9KbkmZLigYaqhE8fmv/jDlT1RqY2R2KX77yTNBZ6uLud7j7AYpnvC7oPLVJHPj5iTK4SNfwkKQ+\nil8CtFrS74KNU6eIpM6KX1b1Y0l/S5yxzGQXKEMPrCb5gaSbE/9f3azEKKsMdYWkH5rZDEn5ksoC\nzrPHKKqpWaXqRyN6JpZhL5hZluIl9Rl3z5TrEuqVGPL5L2XmdcCjJJ1lZl8oPjz9RDN7OthIdUuc\nVasclvSy4kPsM9FKSSuTzqK/oHhxzWSnSZrp7muDDlKHb0ha5u7r3b1c8euSjgk4U53c/TF3H+7u\nx0narPj1iZlsrZn1kKTE14wY+tcamNllks6QdJG3jM/3e0aZO+yvj+IHq+Ykfm/1lDTTzLoHmqoO\n7r42cdA6JunPyuzfWS8lLluapvgIq4y4SVVtEpd/fFvSX4PO0oBLtesa2ueVuf/+cvdF7n6yuw9X\n/ADAkqAz7SmKamo+lnSwmfVOnKk4X9KrAWdq0RJH9x6TtNDd7ws6T33MrFvl3R3NrJ2kkyQtCjbV\n7tz9dnfv6e69FP9v9J/unpFnqcysg5nlVz5W/EYaGXmnandfI+lLMzsksejrkj4JMFIqMv3o9ApJ\nR5lZ+8TPgq8rQ29QJUlmVpD4eqDif1A9G2yiBr2q+B9VSnz9e4BZWg0zO1XxSyvOcvcdQeepi5kd\nnDR7tjLw95Ukufs8dy9w916J31srJQ1L/MzNOJUHfxK+pQz9nSXpFUknSJKZ9ZOULWlDoInq9w1J\ni9x9ZdBBGvCVpOMTj0+UlLHDlJN+Z4Uk/VTSw8Em2nORoAO0BO5eYWbXSXpL8btTPu7uCwKOVSsz\ne07SGEldzWylpJ+7eyYOTxgl6RJJ85JuR/6TDLmLXk09JD2RuPtzSNLf3D2jP/qlBSiU9HJiNFJE\n0rPu/mawkep1vaRnEgeqlkq6POA8dUoU/5MkXR10lrq4+1Qze0HSTMWHUM6S9Eiwqer1opl1kVQu\n6dpMuplWbT/zJd2l+HC/KyUtV/wmJYGrI+smSQ9I6ibpdTOb7e6nBJcyro6st0vKkTQp8bPrI3e/\nJrCQqjPn2MSBtZji//6BZqzUgv4+qet9HWNmQxQfSv+FMuBnbB05H5f0eOLjSsokXZoJZ//r+ffP\nuPsp1PG+fl/S/YkzwCWSMuLeGnVkzTOzaxOrvCTpLwHF22uWAf/tAgAAAABQhaG/AAAAAICMQlEF\nAAAAAGQUiioAAAAAIKNQVAEAAAAAGYWiCgAAAADIKBRVAADqYWbFia+9zOzCJt73T2rMf9CU+wcA\noKWiqAIAkJpekhpVVBOfuVefakXV3Y9pZCYAAFoliioAAKm5S9KxZjbbzG42s7CZ3WNmH5vZXDO7\nWpLMbIyZ/dvMXpX0SWLZK2Y2w8wWmNm4xLK7JLVL7O+ZxLLKs7eW2Pd8M5tnZt9N2vdkM3vBzBaZ\n2TNmZpX7M7NPElnubfZ3BwCAJtTQkV4AABB3m6T/cPczJClROLe6+xFmliNpipm9nVh3mKTD3X1Z\nYv4Kd99kZu0kfWxmL7r7bWZ2nbsPqeW1vi1piKTBkromtnkv8dxQSQMkfSVpiqRRZrZQ0rck9Xd3\nN7N9mvy7BwCgGXFGFQCAPXOypO+Z2WxJUyV1kXRw4rlpSSVVkm4wszmSPpJ0QNJ6dRkt6Tl3j7r7\nWknvSjoiad8r3T0mabbiQ5K3SiqR9JiZfVvSjr3+7gAACBBFFQCAPWOSrnf3IYmpt7tXnlHdXrWS\n2RhJ35B0tLsPljRLUu5evG5p0uOopIi7V0gaKekFSWdIenMv9g8AQOAoqgAApKZIUn7S/FuSfmBm\nWZJkZv3MrEMt23WStNndd5hZf0lHJT1XXrl9Df+W9N3EdbDdJB0naVpdwcwsT1Ind58o6WbFhwwD\nANBicY0qAACpmSspmhjCO0HS/YoPu52ZuKHReknfrGW7NyVdk7iO9FPFh/9WekTSXDOb6e4XJS1/\nWdLRkuZIckm3uvuaRNGtTb6kv5tZruJnem/Zs28RAIDMYO4edAYAAAAAAKow9BcAAAAAkFEoqgAA\nAACAjEJRBQAAAABkFIoqAAAAACCjUFQBAAAAABmFogoAAAAAyCgUVQAAAABARqGoAgAAAAAyyv8H\nHhBVPeX80zgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f734efc6cf8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "x = [x for x, y in training_process]\n",
    "y = [y for x, y in training_process]\n",
    "plt.figure(figsize=((16,4)))\n",
    "plt.plot(x, y)\n",
    "plt.xticks(x, x)\n",
    "plt.xlabel(\"Iterations\")\n",
    "plt.ylabel(\"Mean Square Error\")\n",
    "plt.grid(axis=\"y\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}