{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": { "collapsed": true }, "outputs": [], "source": [ "import numpy as np\n", "import matplotlib.pyplot as plt" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Sigmoid\n", "\n", "The logistic sigmoid function is,\n", "\n", "\\begin{equation*}\n", "S\\left(x\\right) = \\frac{1}{1+e^{-x+\\alpha}}\n", "\\end{equation*}\n", "\n", "Here, α is the offset parameter to set the value at which the sigmoid evaluates to 0. Let's plot it and then discuss it's disadvantages." ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "collapsed": true }, "outputs": [], "source": [ "\"\"\" The logistic sigmoid function \"\"\"\n", "def sigmoid(arr, offset):\n", " a = []\n", " for x in arr:\n", " a.append(1 / (1+np.exp(-x+offset)))\n", " \n", " return a" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "collapsed": false }, "outputs": [], "source": [ "X = np.arange(-10, 10, .2)\n", "Y = np.linspace(0, len(X), len(X))" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "collapsed": false }, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xd4VHXa//H3Te+9l9CLVMVQ7Kyggg0BfdbVVay4xW2/\nXSEKKooK6qqrj7u6rGXRdW2EKqgggqyuhbKQhNBCDwQIvQTS5vv7Y477xDiBgDNzZpLP67pyZeac\nM+fcfOcwnzlnTu4x5xwiIiLFVfC7ABERiU0KCBERCUkBISIiISkgREQkJAWEiIiEpIAQEZGQFBBS\nppnZajMb6HcdoZjZYjO7y+86REqigJAyzTnX3Tm3ONLbMbO/m9ljEVz/FjMbHKn1i4SigBARkZAU\nEFKmFX3nbWYTzOw9M3vDzI54p58Siy17v5mlm9kBM3vdzKp5824zs8+LrduZWUczGw3cDIwxs6Nm\nNqeEWi4zs7VmdsjMXgSsyLwOZvapme0zs71m9paZ1fPmvQkkAHO89Y/xpr9vZru89S0xs+5hHTwp\n9xQQUt5cC7wD1ANmAy8Wm38zcAXQAegMjD/VCp1zU4C3gKecc7Wcc9cUX8bMGgHJ3voaARuBC4ou\nAkwCWgBnAa2BCd76bwG2Add463/Ke8yHQCegCbDCq0EkbBQQUt587pyb55wrBN4Eeheb/6Jzbrtz\nbj/wOPCTMG33SiDdOTfNOZcP/AnY9e1M51yGc26Bcy7XOZcNPAtccrIVOudec84dcc7lEgyT3mZW\nN0z1ilDJ7wJEomxXkds5QDUzq+ScK/CmbS8yfyvBd/Th0KLoup1zzsz+e9/MmgAvABcBtQm+eTtQ\n0srMrCLBALsBaAwEvFmNgENhqlnKOR1BiHxX6yK3E4Cd3u1jQI1vZ5hZs2KPO1Vb5Kyi6zYzK7at\nSd46ejnn6gA/pchnFCHWfxMwDBgM1AXafrvqU9QhUmoKCJHv+qWZtTKzBsADwLve9FVAdzM72/vg\nekKxx+0G2p9kvXO9x48ws0rAr4GiIVMbOAocNLOWwH2nWH9tIBfYRzC4nijlv0+k1BQQIt/1T2A+\nsMn7eQzAObceeBT4BNgAfF7sca8C3czsoJnNLL5S59xegqeDJhN8Ue8EfFFkkUeAPgRPD80Fphdb\nxSRgvLf+PwBvEDwFtgNIB746w3+vSIlMXxgkEmRmW4C7nHOf+F2LSCzQEYSIiISkgBARkZB0iklE\nRELSEYSIiIQU138o16hRI9e2bVu/yxARiSvLly/f65xrfKrl4jog2rZty7Jly/wuQ0QkrpjZ1tIs\np1NMIiISkgJCRERCUkCIiEhICggREQlJASEiIiFFLCDM7DUz22NmaUWmNTCzBWa2wftd35tuZvaC\nmWWYWYqZ9YlUXSIiUjqRPIL4OzCk2LQkYKFzrhOw0LsPMJRgd8tOwGjgpQjWJSIipRCxgHDOLQH2\nF5s8DJjq3Z4KXFdk+hsu6Cugnpk1j1RtIiLx6nheIZM+XEPmgZyIbyvan0E0dc5lAXi/m3jTW/Ld\nr3rM9KZ9j5mNNrNlZrYsOzs7osWKiMSSf2/cyxV/WsJfP9vEonWRf/2Llb+kDvU1iSG7CDrnpgBT\nABITE9VpUETKvMMn8pk0bw1vf7Odtg1r8M7oAQxo3zDi2412QOw2s+bOuSzvFNIeb3om3/1+3lb8\n33cBi4iUWwvSdzN+ZirZR3K55+L2/HZwZ6pXqRiVbUc7IGYDowh+7eIoYFaR6fea2TtAf+DQt6ei\nRETKo71Hc5kwezUfpGTRtVlt/nZrIr1a1YtqDRELCDN7GxgINDKzTOBhgsHwnpndCWwj+B29APOA\nK4EMIAe4PVJ1iYjEMuccM1fu4JE56RzLLeD/XdaZn13SgSqVov9naxELCOfcT0qYNSjEsg74ZaRq\nERGJBzsPHmfcjFQWrcvmnIR6PDWyF52a1vatnlj5kFpEpNwKBBxvfbONyfPWEHDw0NXdGHV+WypW\nCHX9TvQoIEREfLR57zHGJqfwzeb9XNixEZNG9KR1gxp+lwUoIEREfFFQGOCVzzfz3IL1VKlUgadG\n9uKGxFaY+XvUUJQCQkQkytJ3HmZscgqpOw5xebemTLyuB03rVPO7rO9RQIiIREluQSEvfprBS4s3\nUq9GZf58Ux+u7Nkspo4ailJAiIhEwfKtBxibnELGnqOM6NOSB6/qRv2aVfwu66QUECIiEZSTV8DT\nH6/j7//eQou61fn77X0Z2KXJqR8YAxQQIiIR8q8N2TwwI5Xt+49z63ltGDOkK7Wqxs/LbvxUKiIS\nJw7l5PPY3HTeX55J+0Y1ee+e8+jXroHfZZ02BYSISBh9vHoX42emsf9YHj8f2IHfDOpEtcrRaa4X\nbgoIEZEwyD4SbK43NzWLbs3r8PptfenRsq7fZf0gCggRkR/AOcf0FTt49IN0jucXct8VXRh9cXsq\nV4x+c71wU0CIiJyhzAM5PDAjjSXrszm3TX2eHNmLjk1q+V1W2CggREROUyDg+MfXW3nyw7U44JFr\nu3PLgDZU8Lm5XrgpIERETsPG7KMkJaewdMsBLuoUbK7Xqn5sNNcLNwWEiEgp5BcGmLJkE88v3ED1\nyhX54w29GdmnZcy2yQgHBYSIyCmk7TjE2OQUVu88zNAezXhkWHea1I695nrhpoAQESnBifxCXli4\ngb8u2UT9GlV4+ad9GNKjud9lRY0CQkQkhKVb9jM2OYVN2ce4/txWPHhVN+rWqOx3WVGlgBARKeJo\nbgFPf7SWN77aSst61Xnzzn5c1Kmx32X5QgEhIuL5bH02D0xPZeeh44w6ry33XdGFmnHUXC/cyu+/\nXETEczAnj4kfrCF5RSYdGtdk2s/O49w28ddcL9wUECJSrn2YmsWDs1ZzICePe3/UkXsv7Ri3zfXC\nTQEhIuXSniMneGjmaj5avYseLesw9Y6+dG8R3831wk0BISLlinOOacszmfhBOicKAowd0pW7L2pH\npTLQXC/cFBAiUm5s35/DAzNS+deGvfRr24DJI3vSvnHZaa4XbgoIESnzCgOON77cwtMfr8OAicO6\nc3P/stdcL9wUECJSpmXsOcLY5FSWbz3AwC6NeXx4T1rWq+53WXFBASEiZVJ+YYC/fraRFxZmUKNq\nRZ77cW+uO7tsN9cLNwWEiJQ5qZmHGJOcwpqsw1zVqzmPXNudRrWq+l1W3FFAiEiZcSK/kD99soG/\n/WsTDWtW4a+3nMsV3Zv5XVbc8iUgzOx3wF2AA1KB24HmwDtAA2AFcItzLs+P+kQk/ny9aR9J01PZ\nvPcYN/Ztzf1XnkXd6uWruV64Rf3CXzNrCfwaSHTO9QAqAjcCTwLPOec6AQeAO6Ndm4jEnyMn8hk/\nM5UfT/mKgkCAt+7qz+SRvRQOYeDXKaZKQHUzywdqAFnApcBN3vypwATgJV+qE5G4sGjtHsbNSCXr\n8AnuvLAdv7+8MzWq6Mx5uER9JJ1zO8zsj8A24DgwH1gOHHTOFXiLZQItQz3ezEYDowESEhIiX7CI\nxJz9x/KY+EE6M/6zg05NapH88/Ppk1Df77LKnKgHhJnVB4YB7YCDwPvA0BCLulCPd85NAaYAJCYm\nhlxGRMom5xxzU7N4eNZqDh3P59eXduSXl3akaiU114sEP47FBgObnXPZAGY2HTgfqGdmlbyjiFbA\nTh9qE5EYtfvwCcbPTGNB+m56tarLP+7qz1nN6/hdVpnmR0BsAwaYWQ2Cp5gGAcuARcD1BK9kGgXM\n8qE2EYkxzjneXbqdx+etIa8gwANXduWOC9RcLxr8+AziazObRvBS1gLgPwRPGc0F3jGzx7xpr0a7\nNhGJLdv25ZA0PYV/b9zHgPYNmDyiF20b1fS7rHLDl4/7nXMPAw8Xm7wJ6OdDOSISYwoDjte/2Mwz\n89dTsYLx+PAe/KRvgprrRZmuBxORmLJ+9xHGTEth5faDXNq1CY8P70Hzumqu5wcFhIjEhLyCAC9/\ntpH//XQDtapW4vkbz+ba3i3UXM9HCggR8V1K5kHGTEth7a4jXNu7BQ9f042Gaq7nOwWEiPjmeF4h\nz32ynlf+tYkmtavxyq2JDO7W1O+yxKOAEBFffLlxH0nTU9i6L4eb+ieQNLQrdaqpf1IsUUCISFQd\nPpHPpHlrefubbbRpWIN/3t2f8zs08rssCUEBISJRs3DNbsbNSGPPkRPcfVE7/t9lXaheRW0yYpUC\nQkQibt/RXB6Zk87sVTvp0rQ2L99yLme3rud3WXIKCggRiRjnHLNX7eSROekcOZHP7wZ35ucDO1Cl\nktpkxAMFhIhERNah44yfkcbCtXs4u3U9nhzZiy7NavtdlpwGBYSIhFUg4Hh76TYmzVtLQSDA+KvO\n4vYL2lFRbTLijgJCRMJmy95jJE1P4atN+zm/Q0MmjehJm4ZqrhevFBAi8oMVBhyvfb6ZZxaso3KF\nCkwe0ZMf922tNhlxTgEhIj/I2l2HGTsthVWZhxh8VlMeu64HzepW87ssCQMFhIickdyCQv68aCN/\nWZRB3eqVefGmc7iqZ3MdNZQhCggROW3/2XaAsckprN99lOHntOShq7tRv2YVv8uSMFNAiEip5eQV\n8Mz89bz2xWaa1anG67f15Uddm/hdlkSIAkJESuXfGXtJmp7Ktv053DKgDWOGdKG2muuVaQoIETmp\nQ8fzmTRvDe8s3U67RjV5d/QA+rdv6HdZEgUKCBEp0fzVuxg/M429R3O555L2/G5wZ6pVVnO98kIB\nISLfs/doLhNmr+aDlCy6NqvNK6MS6dVKzfXKGwWEiPyXc46ZK3fwyJx0cnIL+f1lnfnZwA5Urqjm\neuWRAkJEANh58DjjZqSyaF02fRKCzfU6NVVzvfJMASFSzgUCjre+2cbkeWsIOHjo6m6MOr+tmuuJ\nAkKkPNuUfZSk6al8s3k/F3ZsxKQRPWndoIbfZUmMUECIlEMFhQFe+Xwzzy1YT9VKFXjq+l7ccG4r\ntcmQ71BAiJQz6TsPMyZ5FWk7DnNF96ZMHNaDJnXUXE++TwEhUk7kFhTy4qcZvLR4I/VqVOGlm/sw\ntGdzv8uSGKaAECkHlm/dz5hpKWzMPsaIPsHmevVqqLmenJwCQqQMO5ZbwNMfr2Pql1toUbc6U+/o\nxyWdG/tdlsQJXwLCzOoBrwA9AAfcAawD3gXaAluA/3HOHfCjPpGyYMn6bO6fnsrOQ8e95npdqVVV\n7wml9Pz688jngY+cc12B3sAaIAlY6JzrBCz07ovIaTqUk89976/i1te+oWrlCrx3z3k8OqyHwkFO\nW9T3GDOrA1wM3AbgnMsD8sxsGDDQW2wqsBgYG+36ROLZR2lZPDhrNfuP5fGLgR349aBOaq4nZ8yP\ntxTtgWzgdTPrDSwHfgM0dc5lATjnssws5LeQmNloYDRAQkJCdCoWiXF7jpzg4Vmr+TBtF92a1+H1\n2/rSo2Vdv8uSOOdHQFQC+gC/cs59bWbPcxqnk5xzU4ApAImJiS4yJYrEB+ccySt2MPGDdI7nF3Lf\nFV0YfXF7NdeTsPAjIDKBTOfc1979aQQDYreZNfeOHpoDe3yoTSRuZB7IYdyMND5bn825berz5Mhe\ndGxSy++ypAyJekA453aZ2XYz6+KcWwcMAtK9n1HAZO/3rGjXJhIPAgHHP77eypMfrsUBE67pxq3n\ntaWCmutJmPl1WcOvgLfMrAqwCbid4BVV75nZncA24AafahOJWRuzj5KUnMLSLQe4uHNjnhjeg1b1\n1VxPIsOXgHDOrQQSQ8waFO1aROJBfmGAKUs28fzCDVSvXJE/3tCbkX1aqrmeRJQujBaJcWk7DjE2\nOYXVOw9zZc9mTLi2O01qq7meRJ4CQiRGncgv5IWFG/jrkk00qFmFl3/ahyE91FxPokcBIRKDlm7Z\nz9jkFDZlH+OGc1sx/qpu1K1R2e+ypJxRQIjEkKO5BTz10Vre+HIrLetV5807+3FRJzXXE38oIERi\nxGfrs3nAa653+wVt+cPlXaip/kniI+19Ij47cCyPiXPTmb5iBx0a12Taz87j3DYN/C5LRAEh4hfn\nHB+m7eKhWWkczMnnV5d25N5LO1K1kprrSWxQQIj4YM/hEzw4K42PV++mR8s6vHFHf7q1qON3WSLf\noYAQiSLnHO8vz+SxD9LJLQiQNLQrd13YjkpqricxSAEhEiXb9+fwwIxU/rVhL/3aNmDyyJ60b6zm\nehK7FBAiEVYYcLzx5Rae/ngdBky8rgc390tQcz2JeQoIkQjK2HOEMdNSWLHtIAO7NObx4T1pWa+6\n32WJlEqJAWFm84BfOOe2RK8ckbIhvzDAXz/byAsLM6hRtSLP/bg3152t5noSX052BPF3YL6ZTQWe\ncs7lR6ckkfiWmnmIMckprMk6zFW9mvPItd1pVKuq32WJnLYSA8I5956ZzQUeApaZ2ZtAoMj8Z6NQ\nn0jcOJFfyHOfrOdvSzbRqFZVptxyLpd3b+Z3WSJn7FSfQeQDx4CqQG2KBISI/J+vN+0jaXoqm/ce\n4yf9WpM09CzqVldzPYlvJ/sMYgjwLDAb6OOcy4laVSJx4siJfJ78aC3/+GobrRtU55939ef8jo38\nLkskLE52BDEOuME5tzpaxYjEk0Vr9/DAjFR2HT7BnRe24/eXd6ZGFV0YKGXHyT6DuCiahYjEi/3H\n8nh0zmpmrtxJpya1SP75+fRJqO93WSJhp7c7IqXknGNOShYTZq/m8PF8fn1pR36p5npShikgREph\n16ETjJ+ZxidrdtOrVV2eurs/XZupuZ6UbQoIkZNwzvHu0u08PncNeYUB7h/alTvVXE/KCQWESAm2\n7cshaXoK/964j/7tGvDkyF60bVTT77JEokYBIVJMYcDx+heb+eP8dVSqUIEnhvfkxr6t1VxPyh0F\nhEgR63cHm+ut3H6QQV2b8NjwHjSvq+Z6Uj4pIESAvIIALy3eyIuLNlC7WmWev/Fsru3dQs31pFxT\nQEi5t2r7QcZMS2Hd7iNc27sFD1/TjYZqrieigJDy63hesLneK//aRJPa1Xjl1kQGd2vqd1kiMUMB\nIeXSlxv3kTQ9ha37cripfwJJQ7tSp5qa64kUpYCQcuXwiXwmzVvL299so03DGrx99wDO69DQ77JE\nYpICQsqNT9J3M25mKtlHchl9cXt+N7gz1auoTYZISXwLCDOrCCwDdjjnrjazdsA7QANgBXCLcy7P\nr/qk7Nh3NJdH5qQze9VOujarzZRbEundup7fZYnEPD/7BfwGWFPk/pPAc865TsAB4E5fqpIywznH\nrJU7GPzsZ3yYlsVvB3di9r0XKhxESsmXgDCzVsBVwCvefQMuBaZ5i0wFrvOjNikbsg4d566py/jN\nOytp07Amc399Eb8d3JkqldRDSaS0/DrF9CdgDMGvMQVoCBx0zhV49zOBlqEeaGajgdEACQkJES5T\n4k0g4Hh76TYmzVtLYcDx4NXduO38tlRUmwyR0xb1gDCzq4E9zrnlZjbw28khFnWhHu+cmwJMAUhM\nTAy5jJRPW/YeI2l6Cl9t2s8FHRsyaXgvEhrW8LsskbjlxxHEBcC1ZnYlUA2oQ/CIop6ZVfKOIloB\nO32oTeJQQWGA177YzDPz11OlUgUmj+jJj/u2VpsMkR8o6gHhnLsfuB/AO4L4g3PuZjN7H7ie4JVM\no4BZ0a5N4s+arMOMTU4hJfMQg89qyuPDe9C0TjW/yxIpE2Lp7yDGAu+Y2WPAf4BXfa5HYlhuQSF/\nXrSRvyzKoG71yrx40zlc1bO5jhpEwsjXgHDOLQYWe7c3Af38rEfiw4ptBxg7LYUNe44y4pyWPHh1\nN+rXrOJ3WSJlTiwdQYicVE5eAc/MX89rX2ymeZ1qvH57X37UpYnfZYmUWQoIiQtfZOwlaXoK2/cf\n56cDEhg7pCu11VxPJKIUEBLTDh3P54m5a3h32XbaNarJu6MH0L+9muuJRIMCQmLW/NW7GD8zjX3H\n8rjnkmBzvWqV1VxPJFoUEBJzso/kMmHOauamZHFW8zq8OqovPVvV9bsskXJHASExwznHzJU7eGRO\nOjm5hfzh8s7cc0kHKldU/yQRPyggJCbsOHiccTNSWbwumz4J9Xjq+l50bFL71A8UkYhRQIivAgHH\nW99sY/K8NQQcPHR1N0apuZ5ITFBAiG82ZR8laXoq32zez4UdGzFpRE9aN1BzPZFYoYCQqCsoDPDK\n55t5bsF6qlaqwFMje3FDYiu1yRCJMQoIiar0ncHmeqk7DnFF96ZMHNaDJmquJxKTFBASFSfyC3nx\n0wxe/mwj9WpU4aWb+zC0Z3O/yxKRk1BASMQt37qfMdNS2Jh9jJF9WvHg1WdRr4aa64nEOgWERMyx\n3AKe/ngdU7/cQou61Zl6Rz8u6dzY77JEpJQUEBIR/9qQzf3TU9lx8Di3DmjDfUO6UquqdjeReKL/\nsRJWh3LymTg3nWnLM2nfuCbv3XMefds28LssETkDCggJm4/Ssnhw1mr2H8vjFwM78OtBndRcTySO\nKSDkB9tz5AQPz1rNh2m76Na8Dq/f1pceLdVcTyTeKSDkjDnnSF6xg4kfpHM8v5D7rujC6Ivbq7me\nSBmhgJAzknkghwdmpLFkfTaJbeozeWQvOjap5XdZIhJGCgg5LYGA482vtvLkR2sBeHRYd37avw0V\n1FxPpMxRQEipbcw+ythpKSzbeoCLOgWb67Wqr+Z6ImWVAkJOKb8wwJQlm3h+4QaqV67IMzf0ZkSf\nlmquJ1LGKSDkpNJ2HGLMtBTSsw5zZc9mPHJtDxrXrup3WSISBQoICelEfiEvLNzAX5dsokHNKrz8\n0z4M6aHmeiLliQJCvmfplv2MnZbCpr3HuOHcVoy/qht1a1T2uywRiTIFhPzX0dwCnvpoLW98uZVW\n9avz5p39uKiTmuuJlFcKCAFg0bo9jJueStbhE9x2flvuu6ILNdVcT6Rc0ytAOXcwJ49H56Qz/T87\n6NikFtN+dj7ntqnvd1kiEgMUEOWUc455qbt4eHYaB3PyufdHHfnVoI5UraTmeiISFPWAMLPWwBtA\nMyAATHHOPW9mDYB3gbbAFuB/nHMHol1febDn8AnGz0xjfvpuerasyxt39Kdbizp+lyUiMcaPI4gC\n4PfOuRVmVhtYbmYLgNuAhc65yWaWBCQBY32or8xyzvH+skwmzk0nryDA/UO7cueF7aik5noiEkLU\nA8I5lwVkebePmNkaoCUwDBjoLTYVWIwCImy278/h/umpfJ6xl37tGjB5RE/aN1ZzPREpma+fQZhZ\nW+Ac4GugqRceOOeyzKxJCY8ZDYwGSEhIiE6hcaww4Jj67y08/fE6KhhMHNadm9VcT0RKwbeAMLNa\nQDLwW+fc4dL29XHOTQGmACQmJrrIVRj/Nuw+wtjkFFZsO8jALo15YnhPWtSr7ndZIhInfAkIM6tM\nMBzecs5N9ybvNrPm3tFDc2CPH7WVBfmFAV5evJH//TSDmlUr8tyPe3Pd2WquJyKnx4+rmAx4FVjj\nnHu2yKzZwChgsvd7VrRrKwtSMw9x37RVrN11hGt6t+Dha7rRqJaa64nI6fPjCOIC4BYg1cxWetMe\nIBgM75nZncA24AYfaotbJ/ILee6T9fxtySYa167K325N5LJuTf0uS0TimB9XMX0OlHSuY1A0aykr\nvtq0j6TkFLbsy+En/VqTNPQs6lZXcz0R+WH0l9Rx7MiJfCZ/uJa3vt5GQoMa/POu/pzfsZHfZYlI\nGaGAiFOfrt3NuBlp7D58grsubMfvL+9C9SpqkyEi4aOAiDP7j+Xx6JzVzFy5k85Na/GXm8/nnAQ1\n1xOR8FNAxAnnHB+kZDFh9moOn8jnN4M68csfdaRKJbXJEJHIUEDEgV2Hgs31Plmzm96t6vLk9f3p\n2kzN9UQkshQQMcw5xztLt/PE3DXkBwKMu/Is7riwHRXVJkNEokABEaO27jtGUnIqX27ax4D2DZg8\nohdtG9X0uywRKUcUEDGmMOB4/YvN/HH+OipXqMCkET35cWJrNdcTkahTQMSQdbuOMCY5hVXbDzKo\naxMeG96D5nXVXE9E/KGAiAF5BQH+sjiDPy/KoHa1yrzwk3O4pldzNdcTEV8pIHy2avtBxkxLYd3u\nIww7uwUPX9OdBjWr+F2WiIgCwi/H8wp5dsE6Xv18M01qV+PVUYkMOkvN9UQkdiggfPDvjXtJSk5l\n2/4cbuqfQNLQrtSppuZ6IhJbFBBRdPhEPpPmreXtb7bRtmEN3r57AOd1aOh3WSIiISkgouST9N2M\nm5lK9pFc7rm4Pb8d3FnN9UQkpikgImzf0VwmzElnzqqddG1Wm7/dmkivVvX8LktE5JQUEBHinGP2\nqp1MmL2ao7kF/G5wZ34+sIOa64lI3FBARMDOg8cZPzONT9fu4ZyEejw5shedm9b2uywRkdOigAij\nQMDxz2+2MfnDtRQGHA9e3Y3bzm+r5noiEpcUEGGyee8xkpJT+Hrzfi7o2JBJw3uR0LCG32WJiJwx\nBcQPVFAY4NXPN/PsgvVUqVSBp0b24obEVmqTISJxTwHxA6zJOszY5BRSMg9xWbemPHZdD5rWqeZ3\nWSIiYaGAOAO5BYX8+dMM/rJ4I3WrV+bFm87hqp5qriciZYsC4jT9Z9sBxkxLYcOeo4w4pyUPXt2N\n+mquJyJlkAKilHLyCnhm/npe+2IzzetU4/Xb+/KjLk38LktEJGIUEKXwRcZekqansH3/cW4Z0IYx\nQ7pQW831RKSMU0CcxKHj+Twxdw3vLttOu0Y1ee+e8+jXroHfZYmIRIUCogQfr97FgzPT2Hcsj58P\n7MBvBnWiWmU11xOR8kMBUUz2kVwmzF7N3NQszmpeh1dH9aVnq7p+lyUiEnUKCI9zjhn/2cGjH6ST\nk1vIfVd0YfTF7alcUc31RKR8UkAAOw4eZ9yMVBavy6ZPQj2eur4XHZuouZ6IlG8xFRBmNgR4HqgI\nvOKcmxzJ7QUCjre+3srkD9figAnXdOOW89RcT0QEYiggzKwi8GfgMiATWGpms51z6ZHY3sbsoyQl\np7B0ywEu6tSIJ4b3pHUDNdcTEflWzAQE0A/IcM5tAjCzd4BhQNgD4r2l2xk/K41qlSrw9PW9uP5c\nNdcTESnYiOsSAAAHtElEQVQulgKiJbC9yP1MoH/xhcxsNDAaICEh4Yw21K5xTQZ1bcIjw7rTpLaa\n64mIhBJLARHqLbz73gTnpgBTABITE783vzT6tm1A37b6gzcRkZOJpWs4M4HWRe63Anb6VIuISLkX\nSwGxFOhkZu3MrApwIzDb55pERMqtmDnF5JwrMLN7gY8JXub6mnNutc9liYiUWzETEADOuXnAPL/r\nEBGR2DrFJCIiMUQBISIiISkgREQkJAWEiIiEZM6d0d+axQQzywa2nuHDGwF7w1hOOMVqbbFaF8Ru\nbarr9MVqbbFaF5x+bW2cc41PtVBcB8QPYWbLnHOJftcRSqzWFqt1QezWprpOX6zWFqt1QeRq0ykm\nEREJSQEhIiIhleeAmOJ3AScRq7XFal0Qu7WprtMXq7XFal0QodrK7WcQIiJycuX5CEJERE5CASEi\nIiGV6YAwsxvMbLWZBcwssdi8+80sw8zWmdkVJTy+nZl9bWYbzOxdrw15JOp818xWej9bzGxlCctt\nMbNUb7llkail2PYmmNmOIrVdWcJyQ7xxzDCzpEjX5W3zaTNba2YpZjbDzOqVsFxUxuxUY2BmVb3n\nOcPbp9pGqpYi22xtZovMbI33/+A3IZYZaGaHijzHD0W6riLbPulzY0EveGOWYmZ9olBTlyJjsdLM\nDpvZb4stE7UxM7PXzGyPmaUVmdbAzBZ4r0sLzKx+CY8d5S2zwcxGnVEBzrky+wOcBXQBFgOJRaZ3\nA1YBVYF2wEagYojHvwfc6N1+Gfh5FGp+BniohHlbgEZRHL8JwB9OsUxFb/zaA1W8ce0WhdouByp5\nt58EnvRrzEozBsAvgJe92zcC70ZhjJoDfbzbtYH1IeoaCHwQrX3qdJ4b4ErgQ4LfNjkA+DrK9VUE\ndhH8ozJfxgy4GOgDpBWZ9hSQ5N1OCrXvAw2ATd7v+t7t+qe7/TJ9BOGcW+OcWxdi1jDgHedcrnNu\nM5AB9Cu6gJkZcCkwzZs0FbgukvV62/wf4O1IbifM+gEZzrlNzrk84B2C4xtRzrn5zrkC7+5XBL+B\n0C+lGYNhBPchCO5Tg7znO2Kcc1nOuRXe7SPAGoLf/R4vhgFvuKCvgHpm1jyK2x8EbHTOnWm3hh/M\nObcE2F9sctF9qaTXpSuABc65/c65A8ACYMjpbr9MB8RJtAS2F7mfyff/4zQEDhZ5EQq1TLhdBOx2\nzm0oYb4D5pvZcjMbHeFavnWvd3j/WgmHsqUZy0i7g+A7zVCiMWalGYP/LuPtU4cI7mNR4Z3SOgf4\nOsTs88xslZl9aGbdo1UTp35u/N63bqTkN2t+jRlAU+dcFgTfBABNQiwTlrGLqS8MOhNm9gnQLMSs\ncc65WSU9LMS04tf7lmaZUitlnT/h5EcPFzjndppZE2CBma313mGcsZPVBbwETCT4755I8PTXHcVX\nEeKxYbl2ujRjZmbjgALgrRJWE/YxC1VqiGkR3Z9Oh5nVApKB3zrnDhebvYLgKZSj3mdMM4FO0aiL\nUz83fo5ZFeBa4P4Qs/0cs9IKy9jFfUA45wafwcMygdZF7rcCdhZbZi/BQ9pK3ju+UMuU2qnqNLNK\nwAjg3JOsY6f3e4+ZzSB4auMHvdiVdvzM7G/AByFmlWYsz0gpxmwUcDUwyHknXkOsI+xjFkJpxuDb\nZTK957ou3z91EHZmVplgOLzlnJtefH7RwHDOzTOzv5hZI+dcxJvSleK5idi+VQpDgRXOud3FZ/g5\nZp7dZtbcOZflnXLbE2KZTIKflXyrFcHPYk9LeT3FNBu40buypB3B9P+m6ALeC84i4Hpv0iigpCOS\ncBgMrHXOZYaaaWY1zaz2t7cJfkibFmrZcCl2vnd4CdtbCnSy4BVfVQgels+OZF1ebUOAscC1zrmc\nEpaJ1piVZgxmE9yHILhPfVpSqIWL9xnHq8Aa59yzJSzT7NvPQsysH8HXhH2RrMvbVmmem9nArd7V\nTAOAQ9+eWomCEo/m/RqzIoruSyW9Ln0MXG5m9b1Tw5d7005PND6J9+uH4ItaJpAL7AY+LjJvHMEr\nT9YBQ4tMnwe08G63JxgcGcD7QNUI1vp34GfFprUA5hWpZZX3s5rgaZZIj9+bQCqQ4u2UzYvX5d2/\nkuAVMhujUZe3zQyC51hXej8vF68tmmMWagyARwkGGEA1bx/K8Pap9lEYowsJnlZIKTJOVwI/+3Zf\nA+71xmYVwQ/7z4/S8xfyuSlWmwF/9sY0lSJXIka4thoEX/DrFpnmy5gRDKksIN97LbuT4GdXC4EN\n3u8G3rKJwCtFHnuHt79lALefyfbVakNEREIqr6eYRETkFBQQIiISkgJCRERCUkCIiEhICggREQlJ\nASESJhbsnrrZzBp49+t799v4XZvImVBAiISJc247wfYkk71Jk4EpzsdmbyI/hP4OQiSMvNYWy4HX\ngLuBc1yww6tI3In7XkwiscQ5l29m9wEfAZcrHCSe6RSTSPgNJdgeoYffhYj8EAoIkTAys7OBywh+\nA9rvovwFNyJhpYAQCROvw+dLBL93YRvwNPBHf6sSOXMKCJHwuRvY5pxb4N3/C9DVzC7xsSaRM6ar\nmEREJCQdQYiISEgKCBERCUkBISIiISkgREQkJAWEiIiEpIAQEZGQFBAiIhLS/wdXt76d5BuAxgAA\nAABJRU5ErkJggg==\n", "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "\"\"\" Let's plot the input data \"\"\"\n", "fig = plt.figure()\n", "plt.plot(X, Y)\n", "plt.xlabel('X')\n", "plt.ylabel('Y')\n", "plt.title('input data')\n", "fig.savefig('figures/data.png')\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "collapsed": false }, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XmcXHWZ7/HPU129ZOnsnX0j0AlJEAiEsMimJCGgEkdB\nwwyKwoDOiFdHx9fgcIdBvDOjg+Nc5w4OojIMICIwAkECScQgyp5Ats4Cnb3T6SWdrZP0WvXcP6rS\nUzTVSSfpU6eq+vt+vSp96pxf1Xly+nR966w/c3dEREQAImEXICIi2UOhICIiHRQKIiLSQaEgIiId\nFAoiItJBoSAiIh0UCiKAmf2tmf0s2+ZrZlvNbHYma5LezXSdgkj2MrOtwJ+7+2/DrkV6B20piIhI\nB4WC9Dpm9jdmttPMGs1so5ldYWZ3mdkjKW0+b2bbzKzBzP4udTdOsu0TZvZI8j3WmNlkM/u2mdWZ\n2Q4zm5vyXqPNbKGZ7TGzSjO7JWVa5/l+LmW+d2RqmYgcoVCQXsXMpgC3Aee5eylwJbC1U5tpwI+B\nPwNGAQOBMZ3e6hPAw8Bg4B1gMYm/pzHA3cBPUtr+EqgCRgPXAv9oZlekqW0a8B/A55JthwJjT/g/\nK3ICFArS28SAYmCamRW6+1Z339SpzbXAs+7+R3dvBe4EOh98+4O7L3b3duAJoAz4nru3AY8BE81s\nkJmNAy4G/sbdm919JfAzEh/8nV0L/MbdX3b3FuDvgHjP/LdFukehIL2Ku1cCXwfuAurM7DEzG92p\n2WhgR8prDgMNndrUpgw3AbvdPZbyHKB/8r32uHtjSvttfHDLI918D6WZr0igFArS67j7o+5+MTCB\nxBbA9zs12UXKbhsz60NiV86JqAaGmFlpyrjxwM40bXcB41Lm2/ck5ityQhQK0quY2RQz+6iZFQPN\nJL7Vxzo1exL4hJldZGZFwHcAO5H5ufsO4FXgn8ysxMzOBG4GfpGm+ZPAx83s4uR870Z/o5JhWuGk\ntykGvgfsBmqA4cDfpjZw9wrgqySODewCGoE6oOUE53k9MJHEVsNTwN+7+9LOjZLz/QrwaHK+e0kc\noBbJGF28JnIMZtYf2AeUu/uWsOsRCZK2FETSMLNPmFlfM+sH/ABYQ6dTV0XykUJBJL35JHb3VAPl\nwALXZrX0Atp9JCIiHbSlICIiHaJhF3C8hg0b5hMnTgy7DBGRnLJixYrd7l52rHY5FwoTJ05k+fLl\nYZchIpJTzGxbd9pp95GIiHRQKIiISAeFgoiIdFAoiIhIB4WCiIh0CCwUzOyBZNeEa7uYbmb2b8nu\nCVeb2TlB1SIiIt0T5JbCg8C8o0y/isTtA8qBW0l0QygiIiEK7DoFd3/ZzCYepcl84KHk/WReT3Zd\nOMrddwVVk4jkl3jcaWmP09oep6U9Rkt7nLZYnPa405oy3BaLE4s77XEn3vmnO7G4E3eIe2L8kWF3\nx5PzcSDuJMY5OEd+8r7nRxy5hdCRNkeGO6an9PD6/vEpOt2G6IqpIzhr3KAeWHJdC/PitTGkdD1I\n4r7xY0jcR/59zOxWElsTjB8/PiPFiUiw3J39TW3UN7bQcKiVvYda2XO4lX2H2zjQ3MaBpjYONLdz\nqCXxONgSo6m1ncOtMZraYrS0xWmN5X8X1pbSvdPwASV5HQrperJKe3c+d78fuB9g5syZuoOfSA5w\nd+oaW9iy+xDbGg6xteEw1fuako9m6htbuvxQLyqIMKBPIQNKovQrjtK/OMqYQYX0KYrSt7CAPkUF\nlBQWUByNUFwYoTiaGC6KRiiORigsiBCNWOJngRGNJH4WRIwCS/5MfZgRMSMSIfEzOWwYZkfGJZ9H\nEh9eZpb8+T/t4H8+xN83Ltn+yPARqR/4ZifUuV+PCzMUqkjpj5ZEn7jVIdUiIiehLRbn3dpGVlft\nZ83O/bxb08jG2kYam9s72kQjxqhBJYwe2IdZpwxhxIASykqLGda/iLL+xQzuV8TgvkUM6ltISWFB\niP+b3i3MUFgI3GZmjwHnA/t1PEEkN7TH4ryzYx+vb2rg9S0NrNi2l+a2xLf+ASVRTh81gPlnj2by\niFImDevPhKF9GTWwhGiBzoLPdoGFgpn9ErgcGGZmVcDfA4UA7n4fsAi4GqgEDgNfDKoWETl5h1ra\neXFDHS+ur2XZhjoONLdjBlNHDmDBeeOZMX4QZ40dxIShfbNmV4gcvyDPPrr+GNOdRCflIpKl4nHn\n1U0N/PrtKp5fW0NTW4wh/YqYO30kV5w+nAtPHcqgvkVhlyk9KOdunS0iwTvU0s6TK6r4z1e2sLXh\nMKUlUT45YzSfPHsMMycOoSCiLYF8pVAQkQ6NzW389A9bePCVLRxobufscYP40ZzJXDl9pA7+9hIK\nBRGhuS3GI69v495llew93MaV00dw66Wncu6EwWGXJhmmUBDp5V7dtJs7nlrLlt2HuKR8GN+6cgpn\njg32AinJXgoFkV5q/+E2/mHROh5fXsWEoX156KZZXDr5mF34Sp5TKIj0Qiu27eW2R9+mrrGFL192\nKl+7opw+RTpmIAoFkV7F3Xngla3806L1jBpUwlN/eZF2Fcn7KBREeonmthjffGIVz63exdxpI7jn\nurMY2Kcw7LIkyygURHqBA81t3PJfy3ljyx5uv+p0vnTpJF11LGkpFETyXO2BZm584E021R/kRwvO\nZv7ZY8IuSbKYQkEkj9UeaOa6+16j4WALD3zhPC4p19lFcnQKBZE8tfdQK5/7+Rs0HGzhkT8/nxnj\ndSGaHJtCQSQPHWxp5wv/+SZbGw7z4BfPUyBIt+nm5iJ5pi0W59aHlrO2+gA//tNzuOjUYWGXJDlE\noSCSZ/7hufW8uqmB73/6TGZPGxF2OZJjFAoieeSJ5Tt48NWt3HzxKVx77tiwy5EcpFAQyROrduzj\njqfXctGpQ/n2VaeHXY7kKIWCSB7Yf7iNv3hkBWX9i/l/189QX8hywnT2kUgeuOvZCmobW/j1X1zE\n0P7FYZcjOUxfJ0Ry3KI1u3jqnZ189aOncdY43dxOTo5CQSSH1TU2c8dTazhz7EC+8pHTwi5H8oBC\nQSRHuTt/++s1HG6N8cPPnEWhjiNID9BaJJKjnl9bw2/X1/GtK6dw2vDSsMuRPKFQEMlBh1vb+T+/\nWcfUUQP4wkUTwy5H8ohCQSQH/XjZJqr3N/Pd+dN1+qn0KK1NIjlmy+5D3P/yZj41YwwzJw4JuxzJ\nMwoFkRzi7nzn2QqKohFu11XLEgCFgkgO+WPlbl7aWM/Xrihn+ICSsMuRPKRQEMkR7s49izcyZlAf\nPn/RhLDLkTylUBDJEYsralldtZ+vzS6nOFoQdjmSpxQKIjkgFnd+uHQjk8r68akZY8IuR/JYoKFg\nZvPMbKOZVZrZ7WmmjzezZWb2jpmtNrOrg6xHJFctXLWTd2sP8o05k3UKqgQqsLXLzAqAe4GrgGnA\n9WY2rVOz/w087u4zgAXAj4OqRyRXtcXi/OvS95g6agBXnzEq7HIkzwX5lWMWUOnum929FXgMmN+p\njQMDksMDgeoA6xHJSc+srGb7nsN8c85kIhELuxzJc0GGwhhgR8rzquS4VHcBN5hZFbAI+Gq6NzKz\nW81suZktr6+vD6JWkawUjzv3v7yJ00eWcsXU4WGXI71AkKGQ7iuNd3p+PfCgu48FrgYeNrMP1OTu\n97v7THefWVZWFkCpItlp2cY63q09yJcum4SZthIkeEGGQhUwLuX5WD64e+hm4HEAd38NKAGGBViT\nSE75ye83M2ZQHz5+5uiwS5FeIshQeAsoN7NTzKyIxIHkhZ3abAeuADCzqSRCQfuHRIAV2/by5tY9\n3HzxKeorQTImsDXN3duB24DFwHoSZxlVmNndZnZNstk3gVvMbBXwS+AL7t55F5NIr/ST329iYJ9C\nPnveuGM3Fukh0SDf3N0XkTiAnDruzpThdcCHg6xBJBdtqj/I0vW13PaR0+hXHOifqcj7aJtUJAs9\n/No2CiMRPn/hxLBLkV5GoSCSZQ61tPPfK6q4+kMjKSstDrsc6WUUCiJZ5pmV1TS2tPO5C3UnVMk8\nhYJIFnF3HnptK1NHDeCc8YPDLkd6IYWCSBZ5e/teNtQ08rkLJuhiNQmFQkEkizz82jZKi6PMP1sX\nq0k4FAoiWWL3wRYWranh0+eO1WmoEhqFgkiW+O8VVbTG4txwwfiwS5FeTKEgkgXcnSdWVHHuhMGc\nNrw07HKkF1MoiGSBlTv2UVl3kOvOHRt2KdLLKRREssATK6ooKYzwsTPVs5qES6EgErLmthjPrqrm\nqjNGUVpSGHY50sspFERCtriihsbmdu06kqygUBAJ2RPLqxg7uA8XTBoadikiCgWRMO3c18Qrm3bz\n6XPGEonoCmYJn0JBJERPv7MTd7hWu44kSygURELi7jz9zk7OmziYcUP6hl2OCKBQEAnNhppG3qs7\nyDVnjwm7FJEOCgWRkDyzsppoxPjYh3RtgmQPhYJICOJx59lV1VxSPowh/YrCLkekg0JBJAQrtu9l\n574m5mvXkWQZhYJICBaurKakMMKcaSPCLkXkfRQKIhnWFovz3JpdzJ46Qv0mSNZRKIhk2B8rd7Pn\nUKt2HUlWUiiIZNizq6oZUBLl0snDwi5F5AMUCiIZ1NoeZ+m6WuZOH0lxtCDsckQ+QKEgkkGvVO6m\nsbmdqz80MuxSRNJSKIhk0HNrdlFaEuXi08rCLkUkLYWCSIa0tsdZUlHDnGkjKIrqT0+yk9ZMkQx5\nZdNuDjS367YWktUCDQUzm2dmG82s0sxu76LNZ8xsnZlVmNmjQdYjEqbn1+yitDjKxeU660iyV2BX\nzphZAXAvMAeoAt4ys4Xuvi6lTTnwbeDD7r7XzIYHVY9ImNpicZasq2X2tBE660iyWpBbCrOASnff\n7O6twGPA/E5tbgHudfe9AO5eF2A9IqF5dVMD+w63cbV2HUmWCzIUxgA7Up5XJcelmgxMNrNXzOx1\nM5uX7o3M7FYzW25my+vr6wMqVyQ4z6/ZRf/iKJdo15FkuSBDIV2Hs97peRQoBy4Hrgd+ZmaDPvAi\n9/vdfaa7zywr06l8klticWfpulo+cvpwSgq160iyW5ChUAWMS3k+FqhO0+YZd29z9y3ARhIhIZI3\nlm/dQ8OhVuZN1wVrkv2CDIW3gHIzO8XMioAFwMJObZ4GPgJgZsNI7E7aHGBNIhn3QkUNRdEIl0/R\nVq5kv8BCwd3bgduAxcB64HF3rzCzu83smmSzxUCDma0DlgHfcveGoGoSyTR3Z0lFLZeWD9NtsiUn\nBLqWuvsiYFGncXemDDvwjeRDJO+s3XmAnfua+Nps7RWV3KArmkUCtLiihoKIMXuqeliT3KBQEAnQ\nCxU1zJo4hCH9isIuRaRbFAoiAamsO0hl3UHmnaGzjiR3KBREArK4ogaAudO160hyh0JBJCBL1tVy\n1tiBjBrYJ+xSRLpNoSASgJr9zazasY+5umBNcoxCQSQAS9fXAnCldh1JjlEoiARgSUUNk4b149Sy\n/mGXInJcFAoiPWx/UxuvbWpgzvQRmKW7L6RI9lIoiPSwlzbW0R535k7T8QTJPQoFkR62ZF0tw/oX\nM2PcB+4CL5L1FAoiPailPcbvN9YzZ9oIIhHtOpLc02UomNkiM5uYuVJEct9rmxo42NKuC9YkZx1t\nS+FBYImZ3WFmhRmqRySnLVlXS7+iAi46dWjYpYickC5vne3uj5vZc8CdwHIzexiIp0z/YQbqE8kZ\n8WS3m5dNKaM4qm43JTcdqz+FNuAQUAyUkhIKIvJ+K6v2Ud/YwpW6illyWJehYGbzgB+S6ELzHHc/\nnLGqRHLQkopaohHj8inDwy5F5IQdbUvhDuA6d6/IVDEiuWzJuhouPHUoA/voEJzkri4PNLv7JQoE\nke6prDvI5vpDzJ2ms44kt+k6BZEesHRd4gZ4sxUKkuMUCiI9YMm6Gs5U3wmSBxQKIiep7kAz72zf\np11HkhcUCiIn6UjfCepQR/KBQkHkJC2pqGXi0L6UD1ffCZL7FAoiJ+FAcxuvbtrN3Okj1XeC5AWF\ngshJWLahjraY6ypmyRsKBZGT8MLaGoaXqu8EyR8KBZET1NwW46WN9cydrr4TJH8oFERO0Mvv1tPU\nFtOuI8krCgWRE7S4opYBJVEumKS+EyR/KBRETkBbLM5v19cye+oICgv0ZyT5I9C12czmmdlGM6s0\ns9uP0u5aM3MzmxlkPSI95c0te9jf1KYL1iTvBBYKZlYA3AtcBUwDrjezaWnalQL/C3gjqFpEetoL\na2soKYxw2eSysEsR6VFBbinMAirdfbO7twKPAfPTtPsu8M9Ac4C1iPSYeNxZXFHDZZPL6FOkbjcl\nvwQZCmOAHSnPq5LjOpjZDGCcu//maG9kZrea2XIzW15fX9/zlYochxXb91LX2MLVHxoVdikiPS7I\nUEh34rZ3TDSLAP8KfPNYb+Tu97v7THefWVamzXUJ13Ord1EUjfDR09XtpuSfIEOhChiX8nwsUJ3y\nvBQ4A3jJzLYCFwALdbBZslk87rywtoZLy8soLVG3m5J/ggyFt4ByMzvFzIqABcDCIxPdfb+7D3P3\nie4+EXgduMbdlwdYk8hJeWfHXmoONPOxM3XWkeSnwELB3duB24DFwHrgcXevMLO7zeyaoOYrEqTn\nVtdQVBDhiqnqUEfyUzTIN3f3RcCiTuPu7KLt5UHWInKy4nHn+bW7uHTyMAZo15HkKV2KKdJNK6v2\nsWt/M1edobOOJH8pFES6adHqXRQWGLPVF7PkMYWCSDe4O8+vreGS8jIG9tGuI8lfCgWRbnh7+152\n7mviY7pgTfKcQkGkG55ZWU1xNMKVZ+hUVMlvCgWRY2iLxXlu9S5mTxtB/+JAT9gTCZ1CQeQYXqnc\nTcOhVuafNTrsUkQCp1AQOYaFq6oZUBLlsim675bkP4WCyFE0t8VYvLaGqz80iuKobpMt+U+hIHIU\nL66v41BrjGu060h6CYWCyFE8s3Inw0uLOX/S0LBLEckIhYJIF/YdbuWljfV8/MzRFETSdQ8ikn8U\nCiJdWLiqmtZYnE+fO+bYjUXyhEJBpAtPLK9i2qgBTB89MOxSRDJGoSCSxoaaA6zZuZ/rZo4NuxSR\njFIoiKTxxPIqCguM+Wdr15H0LgoFkU7aYnGefmcns6eOYEi/orDLEckohYJIJ7/bUEfDoVauPVe7\njqT3USiIdPLE8irKSou5bLJuayG9j0JBJEVdYzPLNtbxqRljiBboz0N6H631Iil+9eYOYnHns+eN\nC7sUkVAoFESS2mNxHn1zO5eUD2NSWf+wyxEJhUJBJOnFDXXs2t/MDRdMCLsUkdAoFESSHnl9G6MH\nlnDF6cPDLkUkNAoFEWBz/UH+8N5u/vT88TrALL2a1n4R4JHXt1NYYHxGB5ill1MoSK93uLWdJ1bs\nYN4ZoxheWhJ2OSKhUihIr/fkiioam9v5/IU6wCyiUJBerT0W56d/2Mw54wcxc8LgsMsRCZ1CQXq1\nRWtr2LGniS9ddipm6l1NJNBQMLN5ZrbRzCrN7PY0079hZuvMbLWZvWhm2n6XjHF3fvL7TUwq68ec\nqSPCLkckKwQWCmZWANwLXAVMA643s2mdmr0DzHT3M4EngX8Oqh6Rzl6pbKCi+gBfunQSEfXBLAIE\nu6UwC6h0983u3go8BsxPbeDuy9z9cPLp64DuVSwZc9/vNzG8tJhPzlBHOiJHBBkKY4AdKc+rkuO6\ncjPwfLoJZnarmS03s+X19fU9WKL0Vqur9vHHyt3cdPEpFEcLwi5HJGsEGQrptsc9bUOzG4CZwD3p\nprv7/e4+091nlpXpHvdy8v5lybsM6lvIn54/PuxSRLJKkKFQBaReHjoWqO7cyMxmA3cA17h7S4D1\niADw1tY9/P7der582akMKCkMuxyRrBJkKLwFlJvZKWZWBCwAFqY2MLMZwE9IBEJdgLWIAIkzju55\nYSNlpcXceOHEsMsRyTqBhYK7twO3AYuB9cDj7l5hZneb2TXJZvcA/YEnzGylmS3s4u1EesTL7+3m\nza17+OpHT6NPkY4liHQWDfLN3X0RsKjTuDtThmcHOX+RVO7ODxZvZMygPiw4T8cSRNLRFc3Sazy3\nZhdrdu7n67PLKYpq1RdJR38Z0iscbm3nH59bz9RRA/gTXZcg0iWFgvQK9y6rpHp/M3fPn65OdESO\nQn8dkve27D7ET1/ewqdmjOG8iUPCLkckqykUJK+5O995toKiaITbrzo97HJEsp5CQfLa4ooaXtpY\nz9dnlzN8gHpVEzkWhYLkrd0HW7jjqbVMGzWAGy+aGHY5Ijkh0OsURMLi7tzx1Boam9t59JazKdTB\nZZFu0V+K5KVfv72TxRW1/PWVk5kysjTsckRyhkJB8s7OfU3ctbCCWROHcPPFk8IuRySnKBQkr7S0\nx/jKL94m7s4PrjuLAvWoJnJcdExB8oa7c+fTFazcsY/7bjiH8UP7hl2SSM7RloLkjV+8sZ1fLd/B\nVz5yKvPOGBV2OSI5SaEgeeGtrXv4zrMVXD6ljG/MmRJ2OSI5S6EgOW9d9QFufvAtxg3uy48WzNBx\nBJGToFCQnLZl9yE+/8Ab9CuO8tDNsxjYR91ripwMhYLkrOp9Tdzwszdwh4dvPp+xg3VgWeRkKRQk\nJ22qP8h1973GgaY2/uumWZw2vH/YJYnkBZ2SKjnnne17uenBt4iY8egtF3DGmIFhlySSNxQKklN+\nt6GWr/ziHcpKi3nopllMHNYv7JJE8opCQXJCLO7839++y78vq2T66AE88IXzGF6qW2GL9DSFgmS9\nusZmvvbLlby2uYHPzhzHd+ZPp6SwIOyyRPKSQkGylrvz1Ds7+e5v1tHUFuMH153FteeODbsskbym\nUJCstL3hMHc8vYY/vLebc8YP4vufPpPyEboFtkjQFAqSVRoOtnDvsk088vo2iqIRvjt/On92/gQi\nukpZJCMUCpIV6htbePi1rfz8j1toaotx3bnj+Ks5kxk5UAeTRTJJoSChWld9gP98ZQvPrKymNRbn\nqjNG8s25U3QxmkhIFAqScfWNLSxcVc2v366iovoAJYURPnPeWL744VM4tUxhIBImhYIEzt2prDvI\nixvq+O26Wt7evpe4w4fGDOTvPzGNT549hsH9isIuU0RQKEgA2mJxNtY0snLHPt7YsofXNzdQ39gC\nwPTRA7jto+V84sxROptIJAspFOSExeNObWMzm+sP8W5tI+/WHmRDzQHWVR+gpT0OwPDSYi6cNJQL\nJg3lsilljBnUJ+SqReRoAg0FM5sH/AgoAH7m7t/rNL0YeAg4F2gAPuvuW4OsSbqnpT3GvsNt7D7Y\nQn1j4lHX2MLOfU3s2tfEzn1NbGs43PHhDzCwTyFTRpZywwUTOHPsQM4aO4gJQ/tiptNJRXJFYKFg\nZgXAvcAcoAp4y8wWuvu6lGY3A3vd/TQzWwB8H/hsUDXlqnjcibkTiyce7Ud+xuK0xZ1YzGmLx2mL\nxWlrd1pjMVra47QmH83tcZrbYrS0xWhqi3G4NUZTa4xDre0caonR2NzOwZY29je1c6Cpjf1NbRxs\naU9by+C+hYwe1IcJQ/tx2eQyJgztx8Sh/Zg8oj9lpcUKAJEcF+SWwiyg0t03A5jZY8B8IDUU5gN3\nJYefBP7dzMzdvaeLefytHdz/h80dz7uahXfx5Migu7+vzZG3cRz3lOcp7dwT0+Md048MJ9rE44nX\nxj0xPuaOJ0Mg3uNLIqE4GqFfcZT+xVH6FUcpLY4yZlAfpo4qZWCfQob2K2JwvyKG9C1i+IBiyvqX\nUFZaTJ8i3XNIJJ8FGQpjgB0pz6uA87tq4+7tZrYfGArsTm1kZrcCtwKMHz/+hIoZ3K+IKZ0PbHbx\npTZ1dOo3X+sYl76NJf8xrKPNkZcbRiSSHDKIpLSLmBGxxHBB5H/GFZgRiRgRg2gkMVxgRrQgQjRi\nFESMwgKjIBKhsMAoKohQWBAhWmAURwsoikYoTj5KCgsoLozQtyhKn8IC9WMsImkFGQrpPnU6f+/t\nThvc/X7gfoCZM2ee0HfnOdNGMGfaiBN5qYhIrxFkd5xVwLiU52OB6q7amFkUGAjsCbAmERE5iiBD\n4S2g3MxOMbMiYAGwsFObhcCNyeFrgd8FcTxBRES6J7DdR8ljBLcBi0mckvqAu1eY2d3AcndfCPwc\neNjMKklsISwIqh4RETm2QK9TcPdFwKJO4+5MGW4GrguyBhER6b4gdx+JiEiOUSiIiEgHhYKIiHRQ\nKIiISAfLtTNAzawe2HaCLx9Gp6uls0i21patdUH21qa6jl+21patdcHx1zbB3cuO1SjnQuFkmNly\nd58Zdh3pZGtt2VoXZG9tquv4ZWtt2VoXBFebdh+JiEgHhYKIiHTobaFwf9gFHEW21patdUH21qa6\njl+21patdUFAtfWqYwoiInJ0vW1LQUREjkKhICIiHfIuFMzsOjOrMLO4mc3sNO3bZlZpZhvN7Mou\nXn+Kmb1hZu+Z2a+St/0Oos5fmdnK5GOrma3sot1WM1uTbLc8iFo6ze8uM9uZUtvVXbSbl1yOlWZ2\ne9B1Jed5j5ltMLPVZvaUmQ3qol1GltmxloGZFSd/z5XJdWpiULWkzHOcmS0zs/XJv4OvpWlzuZnt\nT/kd35nuvQKq76i/G0v4t+QyW21m52Sgpikpy2KlmR0ws693apOxZWZmD5hZnZmtTRk3xMyWJj+X\nlprZ4C5ee2OyzXtmdmO6Nsfkyf6A8+UBTAWmAC8BM1PGTwNWAcXAKcAmoCDN6x8HFiSH7wP+IgM1\n/wtwZxfTtgLDMrj87gL++hhtCpLLbxJQlFyu0zJQ21wgmhz+PvD9sJZZd5YB8JfAfcnhBcCvMrCM\nRgHnJIdLgXfT1HU58JtMrVPH87sBrgaeJ9Er4wXAGxmurwCoIXGhVyjLDLgUOAdYmzLun4Hbk8O3\np1v3gSHA5uTPwcnhwcc7/7zbUnD39e6+Mc2k+cBj7t7i7luASmBWagNLdLb8UeDJ5Kj/Aj4ZZL3J\neX4G+GWQ8+lhs4BKd9/s7q3AYySWb6DcfYm7tyefvk6iN7+wdGcZzCexDkFinbrCUjv9DoC773L3\nt5PDjcAdAbl1AAAEiklEQVR6En2h54r5wEOe8DowyMxGZXD+VwCb3P1E75pw0tz9ZT7YA2XqutTV\n59KVwFJ33+Pue4GlwLzjnX/ehcJRjAF2pDyv4oN/LEOBfSkfPOna9LRLgFp3f6+L6Q4sMbMVZnZr\nwLUccVty0/2BLjZTu7Msg3YTiW+U6WRimXVnGXS0Sa5T+0msYxmR3F01A3gjzeQLzWyVmT1vZtMz\nVRPH/t2EvW4toOsvaGEtM4AR7r4LEsEPDE/TpkeWXaCd7ATFzH4LjEwz6Q53f6arl6UZ1/l83O60\n6bZu1nk9R99K+LC7V5vZcGCpmW1IfpM4YUerC/gP4Lsk/t/fJbFr66bOb5HmtT1ybnN3lpmZ3QG0\nA7/o4m16fJmlKzXNuEDXp+NhZv2B/wa+7u4HOk1+m8TukYPJY0ZPA+WZqItj/27CXGZFwDXAt9NM\nDnOZdVePLLucDAV3n30CL6sCxqU8HwtUd2qzm8TmajT5zS5dm247Vp1mFgU+BZx7lPeoTv6sM7On\nSOy2OKkPuO4uPzP7KfCbNJO6syxPSDeW2Y3Ax4ErPLkjNc179PgyS6M7y+BIm6rk73ogH9wt0OPM\nrJBEIPzC3X/deXpqSLj7IjP7sZkNc/fAb/zWjd9NYOtWN1wFvO3utZ0nhLnMkmrNbJS770ruTqtL\n06aKxLGPI8aSOLZ6XHrT7qOFwILkGSGnkEj5N1MbJD9klgHXJkfdCHS15dETZgMb3L0q3UQz62dm\npUeGSRxoXZuubU/ptP/2T7qY31tAuSXO1Coiscm9MMi6krXNA/4GuMbdD3fRJlPLrDvLYCGJdQgS\n69TvugqynpI8ZvFzYL27/7CLNiOPHNsws1kkPgcagqwrOa/u/G4WAp9PnoV0AbD/yG6TDOhyqz2s\nZZYidV3q6nNpMTDXzAYnd/vOTY47Ppk4mp7JB4kPsiqgBagFFqdMu4PEGSMbgatSxi8CRieHJ5EI\ni0rgCaA4wFofBL7cadxoYFFKLauSjwoSu1CCXn4PA2uA1ckVcVTnupLPryZxZsumTNSVnGcliX2m\nK5OP+zrXlslllm4ZAHeTCC2AkuQ6VJlcpyZlYBldTGKXweqU5XQ18OUj6xpwW3LZrCJxwP6iDP3+\n0v5uOtVmwL3JZbqGlDMIA66tL4kP+YEp40JZZiSCaRfQlvwsu5nEsagXgfeSP4ck284Efpby2puS\n61sl8MUTmb9ucyEiIh160+4jERE5BoWCiIh0UCiIiEgHhYKIiHRQKIiISAeFgshJsMRdSbeY2ZDk\n88HJ5xPCrk3kRCgURE6Cu+8gcWuQ7yVHfQ+430O8oZrIydB1CiInKXlbiRXAA8AtwAxP3DlVJOfk\n5L2PRLKJu7eZ2beAF4C5CgTJZdp9JNIzriJxa4Izwi5E5GQoFEROkpmdDcwh0VPYX2W4UxiRHqVQ\nEDkJyTtn/geJfgu2A/cAPwi3KpETp1AQOTm3ANvdfWny+Y+B083sshBrEjlhOvtIREQ6aEtBREQ6\nKBRERKSDQkFERDooFEREpINCQUREOigURESkg0JBREQ6/H8jWdNiJAK5lQAAAABJRU5ErkJggg==\n", "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "sig = sigmoid(X, 0)\n", "\n", "fig = plt.figure()\n", "plt.plot(X, sig)\n", "plt.xlabel('X')\n", "plt.ylabel('Y')\n", "plt.title('sigmoid')\n", "fig.savefig('figures/sigmoid.png')\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "So what happens if we were to sum many of these functions, all with a different bias/offset?" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "collapsed": false }, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAEWCAYAAABliCz2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XecVOW9x/HPb3fpvax0WFERKSKwIFGjRo2ixhY1YleM\n9dpyNRqvJt4kpmk0xhIVe0FAsZFIFIzdK71Jr9LL0hZ2Yfvv/jED2ay7sOzumTPl+3699rVTnjPP\nb8+c/c6ZZ848x9wdERFJfmlhFyAiIrGhwBcRSREKfBGRFKHAFxFJEQp8EZEUocAXEUkRCnxJWWZ2\nuJnNNLOdZnbrASx3qZlNCLK2mvRrZp+a2U9jWZMkFgW+BM7MjjOz/zOzXDPbamZfmdmgsOsC7gI+\ndfdm7v5YdRdy95HufmqAdcVVv5I8FPgSKDNrDvwDeBxoDXQCfg0UhllXVDdgXthFiMSKAl+C1gPA\n3Ue5e6m773b3Ce4+B8DM/tfMXtvT2MyyzMzNLCN6/VMzeyD6DiHPzP5uZm3MbKSZ7TCzqWaWVVXn\nZna2mc0zs+3RxzoievvHwA+AJ6KP26OSZa8ys+XRIZ8VZnZpudu/LNfuVDNbFH0H8zcz+2zP0Eq0\n7Vdm9pdoDcvN7Jjo7avNbJOZXVnusVqY2StmlmNmK83sPjNLq6LfH5rZwmi/TwBW7r5Do3Xkmtlm\nMxtzQM+aJCUFvgRtMVBqZi+b2elm1qoGjzEMuJzIu4NDgK+BF4m8Y1gA3F/ZQtEQHwXcDmQC44G/\nm1l9dz8J+AK42d2buvviCss2AR4DTnf3ZsAxwKxK+mgLjAXuAdoAi6JtyzsamBO9/3VgNDAIOBS4\njMiLTtNo28eBFkB34ATgCuDqKvp9C7gPaAssA44t1+S3wASgFdA5+riS4hT4Eih33wEcBzjwLJBj\nZuPMrN0BPMyL7r7M3XOBfwLL3P0jdy8B3gT6V7HcRcD77j7R3YuBPwON+G4gV6UM6GNmjdx9vbtX\nNvxzBjDP3d+O1vMYsKFCmxXu/qK7lwJjgC7Ab9y90N0nAEXAoWaWHq35Hnff6e7fAg8TebGrrN/5\n7j42+rc9WqHfYiJDVh3dvcDdv6zkMSTFKPAlcO6+wN2vcvfOQB+gI5GAqq6N5S7vruR6UyrXEVhZ\nro4yYDWRdwr7qzmfSPjeAKw3s/fNrGcVfawut5wDa/ZTP+5e2d/QFqhfvubo5crqrazf1eXuv4vI\nEM+U6JDW8EoeQ1KMAl9iyt0XAi8RCX6AfKBxuSbt67C7dUT2cgEwMyOyd722Ogu7+4fu/kOgA7CQ\nyDuUitYTGTIp30fnStpVx2b+vWe+R9cq6l1P5G8p3+/e6+6+wd2vdfeOwPXA38zs0BrWJUlCgS+B\nMrOeZnaHmXWOXu8CXAxMijaZBRxvZl3NrAWRsfC68gZwppmdbGb1gDuIHB30f9Wou130A98m0WXy\ngNJKmr4P9DWzc6MfNP8XNXzRig75vAH8zsyamVk34L+B1ypp/j7Q28x+HO331vL9mtmFe9Y5sI3I\nkFpl9UsKUeBL0HYS+dByspnlEwn6uUTCF3efSGRcew4wncghnHXC3RcR+VD0cSJ7z2cBZ7l7UTUW\nT4vWuA7YSuQD1Jsq6WMzcCHwILAF6AVMo+aHnd5C5F3PcuBLIh/yvrCPfv8Y7fcw4KtyTQYRWed5\nwDjgNndfUcOaJEmYToAiUneih1CuAS5190/CrkekPO3hi9SSmZ1mZi3NrAHwP0Q+LJ20n8VEYk6B\nL1J73yNyHPyeYaNz3X13uCWJfJeGdEREUoT28EVEUkRG2AWU17ZtW8/Kygq7DBGRhDF9+vTN7p5Z\nnbZxFfhZWVlMmzYt7DJERBKGma3cf6sIDemIiKQIBb6ISIpQ4IuIpAgFvohIilDgi4ikCAW+iEiK\nUOCLiKQIBb6ISIimr9zGM58ti0lfCnwRkRC4O89+vpyLnvma16esIr+wJPA+4+qbtiIiqWD7riLu\nfHM2Hy3YxNDe7XnwwiNp0iD4OFbgi4jE0KINO/npK1PZkFvA/Wf14qpjsoickjh4CnwRkRiZOH8j\nt4+eSeMGGYy5/nsM6Noqpv0r8EVEAubuPPXZMh76cBF9O7VgxOXZtG/RMOZ1KPBFRAJUWubcP24u\nr01axVn9OvLQBUfSsF56KLUo8EVEArK7qJRbR89k4vyNXH9Cd+4+rSdpabEZr6+MAl9EJAC5u4sZ\n/tJUZqzaxq/P7s2Vx2SFXZICX0Skrm3JK+SKF6aweONOnrxkAGf07RB2SYACX0SkTm3cUcClz01m\n9dZdPHtFNiceflDYJe2lwBcRqSPrc3czbMQkNu8s5OXhgxnSvU3YJf0HBb6ISB3YkFvAxSMmsTWv\niNd+ejT9Y3yMfXUo8EVEamnjjgIueXYSm/OKeOWawXEZ9qDJ00REamVzXiEXPzuJjTsKeHn4oJh/\ne/ZAaA9fRKSGcncXc8XzU1i3fTevDD+agd1ah13SPmkPX0SkBnYVlTD8paks2bSTZy7PZvDB8R32\noMAXETlgRSVlXP/qdGau2sZjw/pzQo/MsEuqFg3piIgcgLIy5843Z/PFks08eMGRnB4nX6qqDu3h\ni4hUk7vzu/ELGDd7HXcP7clPsruEXdIBUeCLiFTTs18s5/kvV3DVMVnccEL3sMs5YAp8EZFqGDd7\nHb8fv5Azj+zAr37UK2ZnqapLCnwRkf2Y+u1W7nxjNoOzWvPwhf1CneK4NhT4IiL7sGJzPte+Mo3O\nrRrxzOUDQzt5SV1Q4IuIVGFrfhFXvziFNDNevHoQrZrUD7ukWgk08M3sZ2Y2z8zmmtkoM4v9SRxF\nRGqgqKSMG16bzrrcAp69IptubZqEXVKtBRb4ZtYJuBXIdvc+QDowLKj+RETqirtz37vfMGXFVh66\n4EgGdovf+XEORNBDOhlAIzPLABoD6wLuT0Sk1p77YgVvTFvDrScdyjlHdQq7nDoTWOC7+1rgz8Aq\nYD2Q6+4TKrYzs+vMbJqZTcvJyQmqHBGRavl44UZ+/88FnNm3A7ef0iPscupUkEM6rYBzgIOBjkAT\nM7usYjt3H+Hu2e6enZmZGPNRiEhyWrppJ7eOmkXvjs35cwIfflmVIId0TgFWuHuOuxcDbwPHBNif\niEiN5e4q5qcvT6NhvXRGXJ5No/qJe/hlVYIM/FXAEDNrbJGvpJ0MLAiwPxGRGikpLePmUTNYu303\nT182gI4tG4VdUiCCHMOfDIwFZgDfRPsaEVR/IiI19eCHi/hiyWYeOLcP2VnxP699TQU6PbK73w/c\nH2QfIiK1MW72OkZ8vpzLh3TjokFdwy4nUPqmrYikrPnrdnDX2NkMymrFL3/UK+xyAqfAF5GUtH1X\nEde/No0Wjerx5KUDqJ+R/HGoM16JSMopLXNuHT2LjbmFjL5+CAc1S41ZXxT4IpJy/vqvJXy+OIff\nn9eXAV2TY9qE6kj+9zAiIuX8a8FGHvvXEi4c2JmLByfWKQprS4EvIilj5ZZ8bh8ziz6dmvPbc/sk\n5FmrakOBLyIpoaC4lBtem0GaGU9dmtgnMqkpjeGLSEr45btzWbB+By9eNYgurRuHXU4otIcvIklv\nzNRVvDk9Mt3xD3oeFHY5oVHgi0hSm7s2l1++N4/vH9aW25JsuuMDpcAXkaSVu7uYm0bOoHXj+jx6\n0VGkJ9l0xwdKY/gikpTcnbvGzmbd9t2MuX4IbZo2CLuk0GkPX0SS0vNfruDDeRv5xek9GdgteWfA\nPBAKfBFJOtNXbuOP/1zIqb3acc1xB4ddTtxQ4ItIUtmWX8TNr8+gQ8uGPHRhv5T7ctW+aAxfRJJG\nWZlzx5uz2ZJXxFs3HkOLRvXCLimuaA9fRJLGiC+W8/HCTdx75hH07dwi7HLijgJfRJLCtG+38tCH\nizizbweu+F63sMuJSwp8EUl42/KLuGXUTDq3asQfzu+rcfsqaAxfRBJa+XH7t286huYNNW5fFe3h\ni0hCe+7Lf4/b9+mkcft9UeCLSMKasWobD36wiKG922vcvhoU+CKSkHJ3FXPL6zNp36Ihf7rgSI3b\nV4PG8EUk4bg7d701m407Chir4+2rTXv4IpJwXvl6JR/O28jdQ3tyVJeWYZeTMBT4IpJQ5q7N5Xfv\nL+CkngdpnpwDpMAXkYSRV1jCLaNm0rpJff58YT/SUnx++wOlMXwRSQjuzi/fncvKLfmMunYIrZvU\nD7ukhKM9fBFJCG/NWMs7M9dy28k9OLp7m7DLSUgKfBGJe8ty8vjlu3MZ0r01N590aNjlJCwFvojE\ntYLiUm5+fSaN6qfz12H9U/68tLWhMXwRiWt/GL+ABet38MJV2bRr3jDschKa9vBFJG59OG8DL3+9\nkmuOO5iTerYLu5yEp8AXkbi0dvtu7ho7h76dWnDX0MPDLicpBBr4ZtbSzMaa2UIzW2Bm3wuyPxFJ\nDiWlZdw+eiYlpWU8fnF/GmSkh11SUgh6DP+vwAfufoGZ1QcaB9yfiCSBxz5eytRvt/HoRUeR1bZJ\n2OUkjcAC38yaA8cDVwG4exFQFFR/IpIcJi3fwhMfL+H8AZ05t3+nsMtJKkEO6XQHcoAXzWymmT1n\nZt95qTaz68xsmplNy8nJCbAcEYl32/KLuH30LLLaNOE35/QOu5ykE2TgZwADgKfcvT+QD/yiYiN3\nH+Hu2e6enZmZGWA5IhLP3J2fj53D1vwiHru4P00a6KjxuhZk4K8B1rj75Oj1sUReAEREvuOVr1fy\n0YKN3H16T52qMCCBBb67bwBWm9me46lOBuYH1Z+IJK7563bwu/GRKY+HH5sVdjlJK+j3TLcAI6NH\n6CwHrg64PxFJMLuKSrhl1AxaNKrHQzpVYaACDXx3nwVkB9mHiCS23/5jPss35/PaNUfTpmmDsMtJ\navqmrYiE5v056xk1ZTXXH38Ixx7aNuxykp4CX0RCsWbbLn7x9hz6dWnJHaf2CLuclKDAF5GYKykt\n47bRs3CHx4f1p166oigWdKCriMTcY/9awvSV2/jrsKPo2kYzrsSKXlZFJKYmLd/C458s5fwBnTnn\nKE2dEEsKfBGJmW35RfxsTGTqhF9r6oSY05COiMSEu3P3W3PYnFfI2zceS1NNnRBz2sMXkZgYOXkV\nE+Zv5K7TetK3s6ZOCIMCX0QCt3jjTn77j/kc3yOTa447OOxyUpYCX0QCVVBcyi2vz6RZwwwevrAf\naWmaOiEsGkQTkUD97v0FLNq4k5eHDyazmaZOCJP28EUkMBPmbeDVSSv56XEHc0IPne8ibAp8EQnE\nhtwC7nprDn06NefnQw/f/wISOAW+iNS50jLn9jEzKSop47Fh/WmQkR52SYLG8EUkAE9/toxJy7fy\n4AVH0j2zadjlSJT28EWkTs1YtY1HJi7mrH4duXBg57DLkXIU+CJSZ3YUFHPrqJl0aNGQ353XR2ev\nijMa0hGROuHu3PfOXNbnFvDG9d+jecN6YZckFWgPX0TqxFsz1jJu9jp+dsphDOzWKuxypBIKfBGp\nteU5efzqvbkM6d6aG088NOxypAoKfBGplcKSUm4dPZP6GWn85aKjSNfUCXGrysA3s/FmlhW7UkQk\nEf35w0XMXbuDB88/kg4tGoVdjuzDvvbwXwImmNm9ZqZPX0TkOz5dtIlnv1jB5UO6cWrv9mGXI/tR\n5VE67v6Gmb0P/AqYZmavAmXl7n8kBvWJSJzK2VnInW/O5vB2zbj3zCPCLkeqYX+HZRYD+UADoBnl\nAl9EUldZmXPHm7PZWVDC69cOoWE9TZ2QCKoMfDMbCjwCjAMGuPuumFUlInHtha9W8PniHB44tw89\n2jULuxyppn3t4d8LXOju82JVjIjEv2/W5PKnDxZyWu92XHp017DLkQOwrzH878eyEBGJf3mFJdwy\nagZtmzbgT+cfqakTEoymVhCRarv/vXms2rqLUdcOoWXj+mGXIwdIX7wSkWp5b9Za3pqxhltOOoyj\nu7cJuxypAQW+iOzXyi353PvOXAZlteKWkzR1QqJS4IvIPhWVlHHrqJmkGTw6rD8Z6YqNRKUxfBHZ\np4cnLmL2mlyeunQAnVpq6oREppdqEanSF0tyeOaz5VxydFdO79sh7HKklgIPfDNLN7OZZvaPoPsS\nkbqTs7OQn42ZzWEHNeWXZ/YKuxypA7HYw78NWBCDfkSkjvx76oRinrhkAI3qa+qEZBBo4JtZZ+BM\n4Lkg+xGRuvXcl8v5fHEO9/2oF4e319QJySLoPfxHgbvYx6RrZnadmU0zs2k5OTkBlyMi+zN79XYe\n/GARp/Vux2WaOiGpBBb4ZvYjYJO7T99XO3cf4e7Z7p6dmZkZVDkiUg07C4q5dfRMDmqmqROSUZB7\n+McCZ5vZt8Bo4CQzey3A/kSkFtyd+96dy+qtu/jrxf01dUISCizw3f0ed+/s7lnAMOBjd78sqP5E\npHbGTl/De7PWcfspPRiU1TrsciQAOg5fRFi6KY9fvTePId1b818/0NQJySom37R190+BT2PRl4gc\nmILiUm4ZNZOG9dJ49KL+pKdp3D5ZaWoFkRT3+/ELWLB+B89fmU37Fg3DLkcCpCEdkRT2wdwNvPL1\nSq457mBOPqJd2OVIwBT4IilqzbZd3DV2Nkd2bsHdQ3uGXY7EgAJfJAUVl5Zx2+hZlDk8fnF/6mco\nClKBxvBFUtBfJi5m+sptPHZxf7q1aRJ2ORIjelkXSTGfL87hb58uY9igLpzdr2PY5UgMKfBFUsim\nnQX89xuz6NGuKfef1TvsciTGNKQjkiJKy5yfjZlFXmEJo64doimPU5ACXyRFPPnJUr5auoU/nd+X\nw9ppyuNUpCEdkRQwafkWHv1oMece1ZGfZHcJuxwJiQJfJMltzivk1lEzyWrThAfO66spj1OYhnRE\nklhZmfPfb8xm++5iXrp6ME0b6F8+lWkPXySJPfXZMj5fnMP9Z/WiV8fmYZcjIVPgiySpycu38PCE\nRZzdryOXDNapCkWBL5KUNucVcuvomXRr04Tf/1jj9hKhwBdJMmXR4+237yrmyUsGaNxe9lLgiySZ\nJz5ZyhdLNnP/Wb01bi//QYEvkkS+WrqZv3y0mPP6d+LiwTreXv6TAl8kSWzcUcBto2dySGZTHji3\nj8bt5Ts0uCeSBEpKy7hl1EzyC0sZde0AmmjcXiqhrUIkCTw0YRFTVmzlLxf10zw5UiUN6YgkuAnz\nNvDMZ8u59OiunNe/c9jlSBxT4IsksJVb8rnjzch5aX91Vq+wy5E4p8AXSVAFxaXc8NoM0sx48pIB\nNMjQ/PaybxrDF0lA7s6978xlwfodvHjVILq0bhx2SZIAtIcvkoBGTl7FWzPWcNvJh/GDngeFXY4k\nCAW+SIKZuWobv/77PE48PJPbTj4s7HIkgSjwRRLI5rxCbho5g/YtGvLoRUeRlqYvV0n1aQxfJEEU\nl5bxXyNnsDW/iLduPIaWjeuHXZIkGAW+SIL4w/iFTF6xlUd+0o8+nVqEXY4kIA3piCSAt2es4YWv\nVnD1sVn8eIC+XCU1o8AXiXNz1+Zyz9vfMKR7a/7njCPCLkcSmAJfJI7l7Czkulem0aZJfZ64ZAD1\n0vUvKzUX2NZjZl3M7BMzW2Bm88zstqD6EklGRSVl3PjadLbuKmLEFdm0bdog7JIkwQX5oW0JcIe7\nzzCzZsB0M5vo7vMD7FMkKbg794+by7SV23j84v76kFbqRGB7+O6+3t1nRC/vBBYAnYLqTySZvDpp\nJaOmrOamEw/hrH4dwy5HkkRMBgTNLAvoD0yu5L7rzGyamU3LycmJRTkice2LJTn8+u/zOeWIg7jz\n1MPDLkeSSOCBb2ZNgbeA2919R8X73X2Eu2e7e3ZmZmbQ5YjEtaWb8rhp5AwOO6gpjw7rr2/SSp0K\nNPDNrB6RsB/p7m8H2ZdIotuWX8Q1L0+lQUYaz12ZTVOdplDqWGBblEXOoPw8sMDdHwmqH5FkUFhS\nyvWvTmf99gJGXTeEzq003bHUvSD38I8FLgdOMrNZ0Z8zAuxPJCG5O3eNncOUb7fy0IVHMrBbq7BL\nkiQV2B6+u38JaABSZD8embiY92at4+enHc45R+lANgmOvrYnEqI3pq7m8Y+XclF2F2468ZCwy5Ek\np8AXCcknCzdxzzvf8P3D2vLAeX2IfOwlEhwFvkgIZq7axk0jZ3BEh2Y8ddlAzZEjMaGtTCTGluXk\nMfylqWQ2a8CLVw3W4ZcSMwp8kRhat303Vzw/hTQzXhk+mMxmmhBNYkeBLxIjm/MKuey5yezYXczL\nwweT1bZJ2CVJitF7SZEYyN1dzBXPT2Fd7m5eveZozX4podAevkjA8gpLuPrFKSzZtJNnLs9mUFbr\nsEuSFKU9fJEA5UfDfvaaXJ68pD8n9NAEgRIe7eGLBCQS9lOZsWo7jw3rz9A+HcIuSVKcAl8kAHmF\nJQx/aSrTVm7l0YuO4swjFfYSPg3piNSx3F3FXPniFL5Zm8tfLjpKZ6ySuKHAF6lDm/MKufz5KSzb\nlMffLh3Aab3bh12SyF4KfJE6snrrLq58cQrrtu/muSuzOV4f0EqcUeCL1IG5a3O5+qWpFBaX8uo1\nR+vQS4lL+tBWpJa+WJLDRc98Tb00Y+yNxyjsJW5pD1+kFkZOXsn9783j0IOa8tLVg2nfomHYJYlU\nSYEvUgPFpWX85u/zeXXSSk48PJPHLu5P84b1wi5LZJ8U+CIHaHNeIbe8PpOvl2/h+uO7c9fQnqSn\n6eQlEv8U+CIHYNLyLdw6aibbdxfz8IX9OH9g57BLEqk2Bb5INZSWOU9/toyHJyyiW5smvHT1YHp1\nbB52WSIHRIEvsh+rtuzizjdnM+XbrZzVryN/+HFfnaVKEpK2WpEquDujpqzmgffnk27GQxccyQUD\nO+tk45KwFPgilViycSf3vTuXySu2cuyhbXjwgn50atko7LJEakWBL1LOrqISnvh4KSM+X06TBhn8\n/ry+DBvUhTQdhSNJQIEvApSUljFm2moe/WgJOTsLuWBgZ+45vSdtmuok45I8FPiS0krLnPHfrOfR\njxazLCef7G6tePqygQzs1irs0kTqnAJfUlJRSRnvzlrL058uY/nmfA49qCkjLh/ID3u104eykrQU\n+JJSNuQW8PqUVYyasoqcnYX07th877z1+rasJDsFviS9wpJSPl6wibdnruXjhZsoc+fEHplceUwW\nJ/TI1B69pAwFviSlguJSvlyymQnzN/DB3A3sKCghs1kDrjnuYC49uivd2jQJu0SRmFPgS1Jwd5bl\n5PPV0s18sWQzXy3dzO7iUpo1yODkIw7ivAGdOfaQNmSk6xQQkroU+JKQ8gpLWLB+B7NWbWf6ym1M\nX7WNnJ2FAHRt3ZjzB3bitN7tOfrgNtTPUMiLgAJf4tzOgmJWbd3Fspx8lm7KY9mmPOav38G3W/Jx\nj7Tp2roxxx3alkFZrTnu0LZ0bdM43KJF4lSggW9mQ4G/AunAc+7+xyD7k8RRVFLG9l1FbN1VxOad\nReTkFZCzs5D1uQVsyC1g3fbdrN62m635RXuXSbNIuB/evhnn9e9E747NObJzSzKb6ctRItURWOCb\nWTrwJPBDYA0w1czGufv8oPqUA1dW5pS6U1oW/XGntNQpiV4vLi2jpMwpKS2juDRyvbi0jKKSMgr3\n/pRSUFxKQXEZu4tL2VVUyu6iEvKLSskvLCG/sISdBSXk7i7e+zuvsKTSeprUT6dDy0Z0aNGQ0zq2\noFubxnRt3ZjumU3IatOEhvXSY7yGRJJHkHv4g4Gl7r4cwMxGA+cAdR74Zz3+JQXFpQB4XT94lHvl\nj1xlf1XcUf7mPY/pQMWHd3zvbeXvc/e9j+EeaVfme9rsuRxpU1YWbetQ5pH7St3xaMCXBbWygIb1\n0mhSP4OmDTP2/u7SujHNG9ajeaMMWjeuT8sm9WnVuB6ZTRuQ2awBbZs1oFmDDB0mKRKQIAO/E7C6\n3PU1wNEVG5nZdcB1AF27dq1RR4dkNqGotOzfj0lAgVHFw1bVW1XBZf/R5t+3VWxv5RobVqFt5HKa\n7bndSLPI7bb3su1ts+e+tDQjPbpMutne6+np0d9pkZ+M9DTSzchIN+qlGxlpadRLT6N+hlEvPXK5\nQUYaDTLSqZ+RRqP66TTMSKNhvXQa1UvXZGMicSjIwK/sP/47+5TuPgIYAZCdnV2jfc5Hh/WvyWIi\nIiklyOPV1gBdyl3vDKwLsD8REdmHIAN/KnCYmR1sZvWBYcC4APsTEZF9CGxIx91LzOxm4EMih2W+\n4O7zgupPRET2LdDj8N19PDA+yD5ERKR69J1zEZEUocAXEUkRCnwRkRShwBcRSRFW1ZQBYTCzHGBl\nDRdvC2yuw3LqSrzWBfFbW7zWBfFbm+o6cPFa24HW1c3dM6vTMK4CvzbMbJq7Z4ddR0XxWhfEb23x\nWhfEb22q68DFa21B1qUhHRGRFKHAFxFJEckU+CPCLqAK8VoXxG9t8VoXxG9tquvAxWttgdWVNGP4\nIiKyb8m0hy8iIvugwBcRSREJFfhmdqGZzTOzMjPLrnDfPWa21MwWmdlpVSx/sJlNNrMlZjYmOm1z\nXdc4xsxmRX++NbNZVbT71sy+ibabVtd1VNHn/5rZ2nL1nVFFu6HR9bjUzH4Rg7oeMrOFZjbHzN4x\ns5ZVtIvJOtvf329mDaLP89Lo9pQVVC0V+u1iZp+Y2YLo/8FtlbQ50cxyyz3Hv4pRbft8biziseg6\nm2NmA2JU1+Hl1sUsM9thZrdXaBOTdWZmL5jZJjObW+621mY2MZpJE82sVRXLXhlts8TMrqxxER49\nx2ki/ABHAIcDnwLZ5W7vBcwGGgAHA8uA9EqWfwMYFr38NHBjwPU+DPyqivu+BdrGeP39L3Dnftqk\nR9dfd6B+dL32CriuU4GM6OU/AX8Ka51V5+8HbgKejl4eBoyJ0fPXARgQvdwMWFxJbScC/4jldlWd\n5wY4A/gnkTPhDQEmh1BjOrCByBeVYr7OgOOBAcDccrc9CPwievkXlW37QGtgefR3q+jlVjWpIaH2\n8N19gbsvquSuc4DR7l7o7iuApUROor6XRU7wehIwNnrTy8C5QdUa7e8nwKig+gjI3pPPu3sRsOfk\n84Fx9wnGvR1TAAAElUlEQVTuXhK9OonI2dHCUp2//xwi2w9EtqeTLQZnXnf39e4+I3p5J7CAyLmj\nE8E5wCseMQloaWYdYlzDycAyd6/pt/lrxd0/B7ZWuLn8tlRVJp0GTHT3re6+DZgIDK1JDQkV+PtQ\n2QnTK/4jtAG2lwuWytrUpe8DG919SRX3OzDBzKZHT+QeKzdH31K/UMXbx+qsyyANJ7InWJlYrLPq\n/P1720S3p1wi21fMRIeR+gOTK7n7e2Y228z+aWa9Y1TS/p6bsLcriLwbq2oHLIx1BtDO3ddD5AUd\nOKiSNnW27gI9AUpNmNlHQPtK7rrX3d+rarFKbqt4vGm1TqpeHdWs8WL2vXd/rLuvM7ODgIlmtjC6\nB1Ar+6oNeAr4LZG/+7dEhpyGV3yISpat9bG71VlnZnYvUAKMrOJhAllnFUut5LbAtqWaMLOmwFvA\n7e6+o8LdM4gMWeRFP6N5FzgsBmXt77kJe53VB84G7qnk7rDWWXXV2bqLu8B391NqsFh1Tpi+mcjb\nyIzoXlmNT6q+vxrNLAP4MTBwH4+xLvp7k5m9Q2QoodbhVd31Z2bPAv+o5K5ATj5fjXV2JfAj4GSP\nDlxW8hiBrLMKqvP372mzJvpct+C7b9UDYWb1iIT9SHd/u+L95V8A3H28mf3NzNq6e6CThFXjuQlk\nuzoApwMz3H1jxTvCWmdRG82sg7uvjw5xbaqkzRoinzPs0ZnI55gHLFmGdMYBw6JHTxxM5NV5SvkG\n0RD5BLggetOVQFXvGGrrFGChu6+p7E4za2JmzfZcJvKh5dzK2talCmOm51XRZ8xPPm9mQ4G7gbPd\nfVcVbWK1zqrz948jsv1AZHv6uKoXqboU/ZzgeWCBuz9SRZv2ez5PMLPBRP7HtwRcV3Wem3HAFdGj\ndYYAuXuGMmKkynfcYayzcspvS1Vl0ofAqWbWKjoMe2r0tgMX9CfTdflDJKTWAIXARuDDcvfdS+To\nikXA6eVuHw90jF7uTuSFYCnwJtAgoDpfAm6ocFtHYHy5OmZHf+YRGdaIxfp7FfgGmBPd0DpUrC16\n/QwiR4Asi0Vt0edjNTAr+vN0xbpiuc4q+/uB3xB5QQJoGN1+lka3p+4xev6OI/JWfk65dXUGcMOe\n7Q24Obp+ZhP5APyYGNRV6XNToS4Dnoyu028od5RdDOprTCTAW5S7LebrjMgLznqgOJpj1xD57Odf\nwJLo79bRttnAc+WWHR7d3pYCV9e0Bk2tICKSIpJlSEdERPZDgS8ikiIU+CIiKUKBLyKSIhT4IiIp\nQoEvUgWLzE65wsxaR6+3il7vFnZtIjWhwBepgruvJjIdxR+jN/0RGOEhTb4lUls6Dl9kH6JTGUwH\nXgCuBfp7ZBZNkYQTd3PpiMQTdy82s58DHwCnKuwlkWlIR2T/Tifylfg+YRciUhsKfJF9MLOjgB8S\nOUvTz0I4aYdInVHgi1QhOoPiU0TmnV8FPAT8OdyqRGpOgS9StWuBVe4+MXr9b0BPMzshxJpEakxH\n6YiIpAjt4YuIpAgFvohIilDgi4ikCAW+iEiKUOCLiKQIBb6ISIpQ4IuIpIj/B4Onl7LdQ6LYAAAA\nAElFTkSuQmCC\n", "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "nb_sum = 10 # number of logistics to sum\n", "offsets = np.linspace(0, np.max(X), 100)\n", "\n", "sig_sum = np.zeros(np.shape(X))\n", "\n", "for offset in offsets:\n", " sig_sum += sigmoid(X, offset)\n", "\n", "sig_sum = sig_sum / nb_sum\n", "\n", "fig = plt.figure()\n", "plt.plot(X, sig_sum)\n", "plt.xlabel('X')\n", "plt.ylabel('Y')\n", "plt.title('Sum of sigmoids')\n", "fig.savefig('figures/sum_sigmoids.png')\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### Problem with Sigmoid:\n", "\n", "Sigmoid gets saturated for large values of x. Thus, as the activation value increases the gradient approaches to 0, and the corresponding neurons learn nothing.\n", "\n", "The sum of sigmoids is better than sigmoid as it does not saturate at the top. But the sum is harder to calculate. It turns out that we have a better way to do this. Here is an approximation of the sum of sigmoids,\n", "\n", "\\begin{equation*}\n", "f\\left(x\\right) = ln\\left(1 + e^x\\right)\n", "\\end{equation*}" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "collapsed": true }, "outputs": [], "source": [ "def sum_of_sigmoid_approx(x):\n", " return np.log(1 + np.exp(x))" ] }, { "cell_type": "code", "execution_count": 8, "metadata": { "collapsed": false }, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl8VPX1//HXIYGwL4Gwg0EExAoIRHDr4lJrcW1rVQTr\nrnWp1Vp/Lqi1tt9Wq7XaWmnRaiuLCy4VrRtarbZWMIQdRPZdCDshkPX8/rgXO8QEQpa5k5n38/HI\nY5b7mbnvuXMzZ+793Lkfc3dERCR1NYo6gIiIREuFQEQkxakQiIikOBUCEZEUp0IgIpLiVAhERFKc\nCoHUGzM73swWm1mBmZ1zEI+7w8yeqM9sNZmvma0ws1PimSlezKxn+D6lxWl+b5jZxfGYlxyY6XcE\nUl/M7F1girs/EnWWumBmK4Ar3P2dqLM0JGZ2D3CYu4+OOotUTlsEUp8OAeZHHSJVmFl61BmkYVIh\nSBJmdquZrTWznWa2yMxODu//q5n9MqbdN8xsTcztFWZ2i5nNMbNdZvYXM+sUbrrvNLN3zKzdfuZ7\npZktMbMtZjbFzLqG9y8FDgVeDXc5ZBxE5nvMbEJMux+Y2Uoz22xmd8XuognbTjazCeHzzDWzvmZ2\nu5ltNLPVZnZqzHN1DXNuCXNfGTOt4nwvipnvmArZh5lZrpntMLMNZvZQFcunnZm9Zmb5ZrY1vN49\nZvr7ZvZrM5tuZtvN7BUzywynZZuZm9lVZrbOzNab2c0V8r4QvvYdwCVmlmFmD4ft14XXM2KW98d7\nC4aZXWNm882sacy80mNy/dLMPgrfv1fNrL2ZTQxf8ydmlh2T5ZFwWe8wsxlm9tXw/tOAO4Dzw+eZ\nHfP8V4TXG5nZneGy3mhmT5tZmwrL4GIzW2Vmmyq+F1J7KgRJwMz6AdcDR7t7K+BbwIqDeIrvAd8E\n+gJnAm8Q/PN2IFhHbqhivicBvwbOA7oAK4FnAdy9N7AKONPdW7p7UU0ym9kRwGPAqHAebYBuFZqd\nCYwH2gEzgbfC3N2Ae4E/x7R9BlgDdAXOBX61twBVMt+xwEVh2/ZA95gmjwCPuHtroDfwfGXLKMzx\nFMHWUU9gN/BohTY/AC4L51MK/L7C9BOBPsCpwG22bz/F2cALQFtgIjAGOAY4ChgEDAPuDNs+ABQD\nd5pZH+BXwGh331NF9gvC198tfI3/DV9LJrAQ+FlM20/CeWYCk4DJZtbU3d8M5/NcuB4MqmQ+l4R/\nJxJ8eWhZyTI6AegHnAzcbWb9q8gsNaBCkBzKgAzgCDNr7O4r3H3pQTz+D+6+wd3XAh8C09x9Zvjh\n/TIwuIrHjQKedPe8sO3twLGx3xTrIPO5wKvu/m93LwbuBip2bH3o7m+5eykwGcgC7nP3EoLClG1m\nbc2sB8EHyq3uvsfdZwFPEHzYVTbf19z9g/C13QWUx0wvAQ4zsw7uXuDuH1f2It19s7u/6O6F7r4T\n+D/g6xWajXf3ee6+K5zPebZvp+3P3X2Xu88l+CAeGTPtv+7+d3cvd/fdBO/Jve6+0d3zgZ/vfX3u\nXk5QdG4ApgC/cfeZleUOPeXuS919O8GXg6Xu/k7Mcv5ivXD3CeFrLXX33xK8t/3289yxRgEPufsy\ndy8gWI8usH13df3c3Xe7+2xgNkGRkzqiQpAE3H0JcCNwD7DRzJ61cBdNNW2Iub67ktstq3hcV4Kt\ngL05CoDNfPkbe20ydwVWxzyuMJzH/vJvcveymNuEr6ErsCX8QN5rZRV5K853V4X5Xk6wBfVpuJvk\njMpep5k1N7M/h7s9dgAfAG0rfNCvjrm+EmhMsDVW1fSuVUzbm3tlzO192rv7CuA9IBv4Y2WZY1R7\nvTCzm81sYbh7axvBllvsa9ifyjKnA51i7vs85nohVa+TUgMqBEnC3Se5+wkEuyAcuD+ctAtoHtO0\ncx3Odl04PwDMrAXBLpS11XnwfjLHWk/MLhkzaxbOo6Z5M82sVcx9PavIux7oETPf5rHzdffF7j4S\n6BjmfiF8/RXdTPDNeHi4G+lre58ypk2PmOs9CbY2Nu1n+rqY2xW3jvZ5Tyq2N7MRwLHAuwS7imot\n7A+4lWAXYTt3bwts53+v8UCHJlaWuZR9C4/UIxWCJGBm/czspLBTcA/Bt7W934hnASPMLNPMOhN8\nC68rk4BLzeyocN6/ItittKKWmWO9AJxpZseZWROCXR1WSbsDcvfVwEfAr8MO0oEE3+wnVjHfM8zs\nhHC+9xLz/2Jmo80sK9zdsi28u7L8rcLXti3sBP5ZJW1Gm9kRYbG5F3ghZosG4K5wy+IrwKXAc/t5\nmc8Q9AFkmVkHgl1pE8LMHYC/AFcAFxMs1xH7ea7qakXwwZ0PpJvZ3UDrmOkbCHbPVfV58wxwk5n1\nMrOW/K9PobQOskk1qBAkhwzgPoJvkZ8TfEu9I5w2nmCf6grgbfb/IXJQ3P1dgn3aLxJ8g+5N0MFY\n28yx85gP/IhgX/96YCewESiq2LaaRhLsFllH0P/xM3efWsV8ryModuuBrQSdzHudBsw3swKCjuML\nquh0fRhoRvA6PwberKTNeOCvBMuhKV/unP8XsITgW/yD7v72fl7fL4FcYA4wF8gL7wMYB7zi7q+7\n+2aCIviEmdV0C2uvtwj6ED4j2K2zh313WU0OLzebWV4lj3+SYBl8ACwPH/+jWmaSg6AflEmDEn5j\n3Ab0cfflUeepLTN7H5jg7l/6RXPY6b4caKxvx1KftEUgCc/Mzgx3jbQAHiT4prsi2lQiyUOFQBqC\nswl25awjOJ7+AtemrEid0a4hEZEUpy0CEZEU1yBOUtWhQwfPzs6OOoaISIMyY8aMTe6edaB2DaIQ\nZGdnk5ubG3UMEZEGxcxWHriVdg2JiKS8eisEZvZkeErZeTH3ZZrZVAtGrZpq+zm9sYiIxEd9bhH8\nleDXl7FuA9519z4Ev5K8rR7nLyIi1VBvhcDdPwC2VLj7bOBv4fW/AdUex1ZEROpHvPsIOrn7eoDw\nsmNVDS0YlSnXzHLz8/PjFlBEJNUkbGexu49z9xx3z8nKOuDRTyIiUkPxLgQbzKwLQHi5Mc7zFxGR\nCuJdCKYQnAed8PKVOM9fRKRB2FxQxL2vLqCwuP5PPFufh48+QzDYdT8zW2NmlxOcf/6bZraYYLD0\n++pr/iIiDVVZuXPjc7OYMG0lq7YU1vv86u2XxeEwfpU5ub7mKSKSDP7wz8V8uHgT9313AId3bn3g\nB9RSwnYWi4ikog8+y+eRdxfz3SHdOP/oHgd+QB1QIRARSRCfb9/Djc/Nok/HlvzynCMxq9Hw3AdN\nhUBEJAGUlJVz3aQ89pSU8diooTRvEr9zgjaIs4+KiCS737z5KTNWbuX3IwdzWMeWcZ23tghERCL2\n1vzPefzD5Vx0zCGcNahr3OevQiAiEqGVm3fx08mzGdi9DXee0T+SDCoEIiIR2VNSxrUT8zDgjxcO\nISM9LZIc6iMQEYnIz19dwPx1O3jiBzn0yGweWQ5tEYiIRODlmWt4Zvoqrv76oZxyRKdIs6gQiIjE\n2eINO7njpXkMy87kllP7RR1HhUBEJJ52FZVyzcQ8WmSk8YcLB5OeFv3HsPoIRETixN254+W5LMsv\nYPzlw+nUumnUkQBtEYiIxM3Eaat4ZdY6bjqlL8cf1iHqOF9QIRARiYO5a7Zz76sL+HrfLK478bCo\n4+xDhUBEpJ5tLyzh2kkzaN+yCb87/ygaNYrPyeSqS30EIiL1yN356QuzWb9tD89dfSyZLZpEHelL\ntEUgIlKPHv9wGVMXbOD2Ef0Zeki7qONUSoVARKSefLJiC/e/uYhvH9mZy47PjjpOlVQIRETqwaaC\nIq6flEf3ds24/9yBcRtkpibURyAiUsfKyp0fPzuTrYUlvHzt0bRu2jjqSPulLQIRkTr2yLuL+c+S\nzdx71lf4Stc2Ucc5IBUCEZE69MFn+fzhn4v53pDucRt8vrZUCERE6sj67bu58blZ9O3YKq6Dz9eW\nCoGISB0oKSvn+kkzKSop47HRQ2jWJJpBZmpCncUiInXg/jeCwef/MHIwvbPiO/h8bWmLQESklt6c\nt54n/r2cHxx7CGdGMPh8bakQiIjUwopNu7hl8hwGdW/DmNOjGXy+tlQIRERqaO/g840aGY9GOPh8\nbamPQESkhn7+6nwWrN/BXy6OdvD52tIWgYhIDbyUt4Znpq/mmm/05uT+0Q4+X1sqBCIiB+mzDTsZ\n8/I8hvXK5OZv9o06Tq1FUgjM7CYzm29m88zsGTNLjIE7RUQOoKColB9OmEGLjHQeHZkYg8/XVtxf\ngZl1A24Actz9SCANuCDeOUREDpa7c/tLc1mxaRe/H3kUHRNk8PnaiqqUpQPNzCwdaA6siyiHiEi1\nTfh4Ja/OXsfNp/bjuN6JM/h8bcW9ELj7WuBBYBWwHtju7m/HO4eIyMGYs2Ybv3htISf2y+Kar/eO\nOk6dimLXUDvgbKAX0BVoYWajK2l3lZnlmllufn5+vGOKiHxhe2EJ107MI6tVBg+dl3iDz9dWFLuG\nTgGWu3u+u5cALwHHVWzk7uPcPcfdc7KysuIeUkQEoLzcuXnyLDbs2MMfRw2hXQIOPl9bURSCVcAx\nZtbcgnO0ngwsjCCHiMgBjftwGe8s3MiYEf05qkfbqOPUiyj6CKYBLwB5wNwww7h45xAROZDpy7fw\nwFuLOH1AFy4+LjvqOPUmklNMuPvPgJ9FMW8RkerI3xkMPt8zszn3fW9AgxlkpiYa/i8hRETq2N7B\n57fvLuGxUUNoleCDz9eWTjonIlLBw+98xkdLN/PAuQPp36V11HHqnbYIRERivL9oI4++t4Tzcrrz\n/ZyGMfh8bakQiIiE1m3bzU3PzaJfp1b8/Kwjo44TNyoEIiJAcWk5103Ko6TMGTt6aIMafL621Ecg\nIgL8+o2FzFy1jcdGDaFXhxZRx4krbRGISMp7Y+56nvrPCi45LpsRA7pEHSfuVAhEJKUt37SLW16Y\nw1E92nLHiIY5+HxtqRCISMraU1LGNRNmkJ5m/HHUEJqkp+ZHovoIRCRl3f3KPD79fCdPXXo03do2\nizpOZFKz/IlIypucu5rnc9dw/YmHcWK/jlHHiZQKgYiknE8/38Fdr8zjuN7tuSkJBp+vLRUCEUkp\nO/eUcO2EPFo3bcwjFwwmLckGmakJ9RGISMpwd257aS4rtxQy6YrhZLXKiDpSQtAWgYikjL99tIJ/\nzFnPT0/tx/BD20cdJ2GoEIhISpi5aiv/9/pCTj68I1d/7dCo4yQUFQIRSXpbdxVz/aSZdGrdNCkH\nn68t9RGISFIrL3d+8vws8ncW8cI1x9KmeXIPMlMT2iIQkaQ29l9LeW9RPnedeQQDuyfn4PO1pUIg\nIknro6Wb+O3bizhrUFdGD+8ZdZyEpUIgIklp44493PDMLHp1aMGvv5vcg8/XlvoIRCTplJaVc/0z\nM9lVVMqkK4fTIkMfdfujpSMiSee3Uz9j+vIt/O78QfTt1CrqOAlPu4ZEJKm8u3ADY99fyshhPfnO\n4O5Rx2kQVAhEJGms3lLIT56fzVe6tuZnZx4RdZwGQ4VARJJCUWkZ103Ko9ydsaOG0rRx6gw+X1vq\nIxCRpPDL1xYyZ812/nzRUHq2bx51nAZFWwQi0uC9Mmst4z9eyZVf7cW3vtI56jgNjgqBiDRoSzbu\n5PaX5nJ0djv+32mHRx2nQVIhEJEGq7C4lGsm5NGscRp/GDmExmn6SKsJ9RGISIPk7ox5eR5L8gsY\nf9lwOrdpGnWkBkvlU0QapGemr+blmWu58eS+nNCnQ9RxGrRICoGZtTWzF8zsUzNbaGbHRpFDRBqm\neWu3c8+U+XytbxY/OumwqOM0eFHtGnoEeNPdzzWzJoCO9RKRatleWMI1E2fQvmUTHj5fg8zUhbgX\nAjNrDXwNuATA3YuB4njnEJGGx9356QuzWb9tD89dfSyZLZpEHSkpRLFr6FAgH3jKzGaa2RNm1qJi\nIzO7ysxyzSw3Pz8//ilFJOE8/uEypi7YwO0j+jP0kHZRx0kaURSCdGAIMNbdBwO7gNsqNnL3ce6e\n4+45WVlZ8c4oIglm+vIt3P/mIkYM6Mxlx2dHHSepRFEI1gBr3H1aePsFgsIgIlKp/J1FXD8pj56Z\nzbn/ewM1yEwdi3shcPfPgdVm1i+862RgQbxziEjDUFbu/PjZmWzfXcJjo4bQqqkGn69rUR019CNg\nYnjE0DLg0ohyiEiCe/idz/ho6WZ+c+5A+ndpHXWcpBRJIXD3WUBOFPMWkYbjvUUb+cM/l3BeTnfO\ny+kRdZykpV8Wi0hCWrttNzc9N4vDO7fi3rOPjDpOUlMhEJGEU1xazrUT8ygtc8aO1iAz9U0nnROR\nhPOr1xcye/U2xo4aQq8OX/qZkdQxbRGISEJ5dfY6/vrRCi4/oRffHtAl6jgpQYVARBLGko0F3Pbi\nHIYe0o7bvq1BZuJFhUBEEkJhcSnXTpxBRuM0Hr1wsAaZiSP1EYhI5PYOMrN4YwFPXzaMLm2aRR0p\npajkikjkYgeZ+WofnVss3lQIRCRSc9dokJmoqRCISGS2F5Zw7SQNMhM19RGISCTKy52bJ8/SIDMJ\nQFsEIhKJP3+wjHcWbmTM6RpkJmoqBCISd/9dupkH3vqU0wd24ZLjsqOOk/JUCEQkrjbu3MMNz84k\nu0MLDTKTIKosBGb2upllxy+KiCS70rJyrp80k4I9pYwdNZSWGeqmTAT72yL4K/C2mY0xMw0JJCK1\n9uDbnzF9+RZ+9d0j6de5VdRxJFRlOXb3583sH8DdQK6ZjQfKY6Y/FId8IpIkpi7YwJ/+tZRRw3vy\nncHdo44jMQ60XVYC7AIygFbEFAIRkepatbmQnzw/iwHd2nDXGUdEHUcqqLIQmNlpwEPAFGCIuxfG\nLZWIJI09JWX8cMIMGpnx2KghGmQmAe1vi2AM8H13nx+vMCKSfO6ZMp8F63fw5CU59MhsHnUcqcT+\n+gi+Gs8gIpJ8Jueu5tlPVnPdib056fBOUceRKuh3BCJSLxas28Gdf5/Hcb3b85Nv9os6juyHCoGI\n1Lntu0u4ZuIM2jZvzO9HDiZNJ5NLaPo1h4jUKXfnlsmzWbt1N89edQwdWmZEHUkOQFsEIlKnxn2w\njLcXbOD2Ef3Jyc6MOo5UgwqBiNSZj5dt5jdvLeL0AV247PjsqONINakQiEid2LhjD9dPmskhmc25\n73sDdDK5BkR9BCJSayVl5Vw3KY9dRaVMvGI4rZrq9GQNiQqBiNTab978lE9WbOXh84/SyeQaIO0a\nEpFaeXPeeh7/cDkXHXMI5wzuFnUcqQEVAhGpsaX5Bfx08hwG9WjLnWf0jzqO1FBkhcDM0sxsppm9\nFlUGEam5wuJSrpkwg8ZpxthRQ8hI18nkGqootwh+DCyMcP4iUkPuzu0vzWXxxgJ+P3IwXds2izqS\n1EIkhcDMugOnA09EMX8RqZ3xH6/klVnr+Mkpfflqn6yo40gtRbVF8DDw/9jPQDdmdpWZ5ZpZbn5+\nfvySich+5a3ayi9eW8CJ/bK47sTDoo4jdSDuhcDMzgA2uvuM/bVz93HunuPuOVlZ+sYhkgg2FRRx\n7YQ8OrdpysPnD6aRTiaXFKLYIjgeOMvMVgDPAieZ2YQIcojIQSgtK+eGZ2aytbCYsaOG0qa5fjSW\nLOJeCNz9dnfv7u7ZwAXAP919dLxziMjB+e3Uz/ho6WZ+ec6RHNmtTdRxpA7pdwQickBvzvucse8v\nZeSwnnw/p0fUcaSORXqKCXd/H3g/ygwisn/L8gv46eTZDOrehnvOOiLqOFIPtEUgIlUqLC7lh+GP\nxh4bPVQ/GktSOumciFTK3bntxbks2VjA05cNp5t+NJa0tEUgIpV68j8rmDJ7HTef2o8T+nSIOo7U\nIxUCEfmSacs286vXF3LqEZ249hu9o44j9UyFQET2sWHHHq4LRxp78LxBGmksBaiPQES+UFxazjUT\nZlBYXMqkK4fTWiONpQQVAhH5wr2vzSdv1TYevXAwfTtppLFUoV1DIgLA87mrmfDxKq7+2qGcMbBr\n1HEkjlQIRIQ5a7Zx59/ncfxh7bnlW/2ijiNxpkIgkuI2FxRxzYQ8slpm8PsLBpOepo+FVKM+ApEU\nVlpWzvWTZpJfUMSLPzyO9i0zoo4kEVDpF0lh973xKf9dtplff2cAA7rrjKKpSoVAJEW9MmstT/x7\nORcfewjfG9o96jgSIRUCkRQ0f912bn1xDkdnt+POM3RG0VSnQiCSYrbsKuaqp2fQtlkTHhs1lMbq\nHE556iwWSSGlZeVcNzGP/IIiJl99LFmt1Dks2iIQSSn/9/rCLzqHB/VoG3UcSRAqBCIpYnLuap76\nzwouPT5bncOyDxUCkRQwc9VWxrw8j+N6t2fMiP5Rx5EEo0IgkuQ27NjD1eNn0KlNBn+8cIh+OSxf\nojVCJIntKSnj6vEzKCgq5fEf5NCuRZOoI0kC0lFDIknK3bn9pbnMWr2NP40eyuGdW0cdSRKUtghE\nktSfP1jGyzPXcvM3+3LakZ2jjiMJTIVAJAm9u3AD97/5KWcM7ML1Jx0WdRxJcCoEIklm0ec7+fGz\nsziyaxseOFdjDsuBqRCIJJHNBUVc/rdPaN4kjXE/GEqzJmlRR5IGQJ3FIkmiqDQ4Qih/ZxHPX30s\nXdo0izqSNBAqBCJJwN2546V55K7cyqMXDtbpI+SgaNeQSBJ47P2lvJi3hptO6auB5+WgqRCINHCv\nzl7HA28t4pyjunLDyTpCSA6eCoFIAzZj5RZunjybo7Pbcf+5A3WEkNRI3AuBmfUws/fMbKGZzTez\nH8c7g0gyWLW5kKuenkGXNk3580U5ZKTrCCGpmSg6i0uBm909z8xaATPMbKq7L4ggi0iDtHVXMZc8\nNZ3ScufJS44mU+cQklqI+xaBu69397zw+k5gIdAt3jlEGqo9JWVc+XQua7bt5omLc+id1TLqSNLA\nRdpHYGbZwGBgWpQ5RBqK8nLn5udnk7tyKw+dN4ijszOjjiRJILJCYGYtgReBG919RyXTrzKzXDPL\nzc/Pj39AkQTj7vzyHwv5x9z1jBnRX4eJSp2JpBCYWWOCIjDR3V+qrI27j3P3HHfPycrKim9AkQT0\n5w+W8eR/lnPp8dlc8dVeUceRJBLFUUMG/AVY6O4PxXv+Ig3RizPWcN8bn3LmoK7cdfoROkxU6lQU\nWwTHAxcBJ5nZrPBvRAQ5RBqE9z7dyK0vzuH4w9rz4PcH0qiRioDUrbgfPuru/wa0JotUw/TlW/jh\nhBkc3qUVfxo9VL8VkHqhXxaLJKh5a7dz+V8/oVu7Zvzt0mG0ato46kiSpFQIRBLQ0vwCLn5yOq2b\nNWbC5cNp3zIj6kiSxFQIRBLMys27uPDxjzGD8ZcPo2tbjSsg9UvjEYgkkDVbC7nw8WkUlZbz7FXH\ncKh+NSxxoC0CkQTx+fY9jHpiGjv3lDDh8uEc3rl11JEkRWiLQCQBrN++m5HjPmZzQTHjLx/Gkd3a\nRB1JUogKgUjE9u4O2rqrmL9dNozBPdtFHUlSjAqBSIRWbylk5OMfs313CeOvGM5RGmtYIqBCIBKR\nJRt3MvqJ6ewuKWPSFccwoLt2B0k0VAhEIjBnzTYufnI6aY0a8exVx9C/izqGJToqBCJx9t+lm7ny\n6VzaNg9+LJbdoUXUkSTFqRCIxNErs9Zyy+Q59GzfnAmXD6dzm6ZRRxJRIRCJB3fnT/9axv1vfsqw\nXpk8flEObZrr3EGSGFQIROpZSVk5P5syn0nTVnHmoK48+P2BOouoJBQVApF6tLmgiGsn5jFt+Rau\n+UZvbjm1n8YTkISjQiBSTxas28GVT+eyqaCIh88/inMGd4s6kkilVAhE6sHzuau5+5V5tGnWmOev\nPpZB+qGYJDAVApE6VFhcyl1/n8+LeWs4rnd7Hr7gKDq20pFBkthUCETqyLy127nxuVkszS/gxyf3\n4YaT+5Cm/gBpAFQIRGqptKycse8v5ZF3F9O+ZRPGXzacE/p0iDqWSLWpEIjUwoJ1O7j95bnMXr2N\ns4/qyr1nHanfB0iDo0IgUgOFxaU8/M5i/vLv5bRt1phHLxzMGQO7Rh1LpEZUCEQOgrszZfY6fvPm\nItZu283IYT249bTDadu8SdTRRGpMhUCkmnJXbOEX/1jI7NXb6N+lNb87/yiG9cqMOpZIrakQiBzA\njJVbeeTdxXzwWT6dWmfwwLkD+e6Q7joiSJKGCoFIJdydDxdvYtwHy/j3kk1ktmjCracdzsXHHULz\nJvq3keSiNVokRkFRKa/MWstT/1nBko0FdGiZwR0jDmf0MSoAkry0ZkvKKy93pi3fwuQZq3lj7ufs\nLiljQLc2/O78QZw+oCtN0htFHVGkXqkQSEoqLStnxsqtvD53PW/M+5yNO4tolZHOOYO78f2c7gzu\n0RYz9QFIalAhkJTg7qzZupv/Lt3Mvz7L58PF+ezYU0pGeiNO7NeRbw/ozKlHdKZZE40TIKlHhUCS\n0u7iMhas38HcNduYuXob05dvYf32PQB0ap3BaUd25hv9OvL1vlm0yNC/gaQ2/QdIg7arqJSVmwtZ\nsXkXSzYWsGjDTj77fCfLNu2irNwB6Ngqg6N7ZTK8VybDe7Wnb6eW2u0jEiOSQmBmpwGPAGnAE+5+\nXxQ5JHGVlpWzpbCYLbuK2VJQTH5BERt27GHjjiLWbd/N2m17WLt1N5sKivZ5XM/M5vTt1JJvfaUz\nA7q3YWD3NnRu3VQf/CL7EfdCYGZpwB+BbwJrgE/MbIq7L4h3Fjmw8nKnzJ2ycqfcndJyp6wsvCx3\nSsrKKS13SsvKKS4rp7TMKS4rp6S0nKKycopKyikqLaOopJzdJWXBX3EZhcWlFBaXsauolIKiUnbs\nKaVgTynbd5ewfXcJBUWlleZp1jiNLm2b0q1tM/r370iPzOZkt29Bdofm9OrQQod4itRAFP81w4Al\n7r4MwMyeBc4G6rwQjHl5LtOXb/nittf1DAg6IQ/YppoTY9vFPu++94OH97gHfxUf4zHtgsu97f53\nu9yD67GXwV/wPOXOF7tW6kOzxmm0yEijeZN0Wmak07JpOl3aNOXwLq1o26wJbZo1JrNFYzJbZJDZ\nogkdW2enQmB8AAAIC0lEQVTQsVUGLTPS9e1epI5FUQi6Aatjbq8BhldsZGZXAVcB9OzZs0Yz6tq2\nGX06tdz3eamHD5FqPOX+msR+sNk+91f+eLOYV2H/e01m/2tn4f1me5/HvpjeyILrey8NI61RcBuD\nNDMamdGokYXXCa43MtLDy+B6I9LTgvsapzUK/4wm6Y1oktaIxumNaJqeRkbj4HbzJmk0a5JG0/Q0\nDeAukkCiKASVfQJ86aunu48DxgHk5OTU6KvpdSceVpOHiYiklCh+MrkG6BFzuzuwLoIcIiJCNIXg\nE6CPmfUysybABcCUCHKIiAgR7Bpy91Izux54i+Dw0SfdfX68c4iISCCSY+3c/XXg9SjmLSIi+9Jp\nFUVEUpwKgYhIilMhEBFJcSoEIiIpzqpzioSomVk+sLKGD+8AbKrDOHUlUXNB4mZL1FyQuNkSNRck\nbrZkynWIu2cdqFGDKAS1YWa57p4TdY6KEjUXJG62RM0FiZstUXNB4mZLxVzaNSQikuJUCEREUlwq\nFIJxUQeoQqLmgsTNlqi5IHGzJWouSNxsKZcr6fsIRERk/1Jhi0BERPZDhUBEJMUlRSEws++b2Xwz\nKzeznArTbjezJWa2yMy+VcXje5nZNDNbbGbPhafHruuMz5nZrPBvhZnNqqLdCjObG7bLrescVczz\nHjNbG5NvRBXtTguX4xIzuy0OuR4ws0/NbI6ZvWxmbatoF7dldqBlYGYZ4Xu9JFynsuszTzjPHmb2\nnpktDP8PflxJm2+Y2faY9/ju+s4Vzne/740Ffh8urzlmNiROufrFLItZZrbDzG6s0CZuy8zMnjSz\njWY2L+a+TDObGn4uTTWzdlU89uKwzWIzu7hGAdy9wf8B/YF+wPtATsz9RwCzgQygF7AUSKvk8c8D\nF4TX/wRcU895fwvcXcW0FUCHOC+/e4CfHqBNWrj8DgWahMv1iHrOdSqQHl6/H7g/ymVWnWUAXAv8\nKbx+AfBcHHJ1AYaE11sBn1WS6xvAa/Fcr6rz3gAjgDcIRi48BpgWQcY04HOCH19FssyArwFDgHkx\n9/0GuC28fltl6z+QCSwLL9uF19sd7PyTYovA3Re6+6JKJp0NPOvuRe6+HFgCDIttYMGAwScBL4R3\n/Q04p76yhvM7D3imvuZRT4YBS9x9mbsXA88SLN964+5vu3tpePNjgtHsolSdZXA2wToEwTp1ssUO\nSl0P3H29u+eF13cCCwnGBm8Izgae9sDHQFsz6xLnDCcDS929pmcvqDV3/wDYUuHu2HWpqs+lbwFT\n3X2Lu28FpgKnHez8k6IQ7Ec3YHXM7TV8+R+kPbAt5gOnsjZ16avABndfXMV0B942sxlmdlU95qjo\n+nDT/MkqNkGrsyzr02UE3xwrE69lVp1l8EWbcJ3aTrCOxUW4K2owMK2Sycea2Wwze8PMvhKnSAd6\nb6JeryDYcqvqi1kUy2yvTu6+HoJiD3SspE2dLL9IBqapCTN7B+hcyaQx7v5KVQ+r5L6Kx8tWp021\nVDPjSPa/NXC8u68zs47AVDP7NPy2UCv7ywaMBX5B8Lp/QbDr6rKKT1HJY2t97HF1lpmZjQFKgYlV\nPE29LLPK4lZyX72tTwfLzFoCLwI3uvuOCpPzCHZ9FIR9QH8H+sQh1oHem8iWF0DYH3gWcHslk6Na\nZgejTpZfgykE7n5KDR62BugRc7s7sK5Cm00Em6Pp4Te4ytrUSUYzSwe+Cwzdz3OsCy83mtnLBLsj\nav2hVt3lZ2aPA69VMqk6y7LOc4WdX2cAJ3u4U7SS56iXZVaJ6iyDvW3WhO93G768yV/nzKwxQRGY\n6O4vVZweWxjc/XUze8zMOrh7vZ5crRrvTb2sVwfh20Ceu2+oOCGqZRZjg5l1cff14e6yjZW0WUPQ\nl7FXd4K+0oOS7LuGpgAXhEdy9CKo5tNjG4QfLu8B54Z3XQxUtYVRW6cAn7r7msommlkLM2u19zpB\nZ+m8ytrWpQr7ZL9TxTw/AfpYcIRVE4LN6Sn1nOs04FbgLHcvrKJNPJdZdZbBFIJ1CIJ16p9VFbC6\nEvZB/AVY6O4PVdGm896+CjMbRvC/v7mec1XnvZkC/CA8eugYYPve3SFxUuUWehTLrILYdamqz6W3\ngFPNrF24S/fU8L6DE48e8fr+I/jwWgMUARuAt2KmjSE40mMR8O2Y+18HuobXDyUoEEuAyUBGPeX8\nK/DDCvd1BV6PyTE7/JtPsHskHstvPDAXmBOufF0qZgtvjyA4ImVpPLKF78dqYFb496eKueK9zCpb\nBsC9BMUKoGm4Di0J16lD47CcTiDYHTAnZlmNAH64d30Drg+Xz2yCjvfj4pCr0vemQi4D/hguz7nE\nHPUXh3zNCT7Y28TcF8kyIyhG64GS8LPscoK+pXeBxeFlZtg2B3gi5rGXhevbEuDSmsxfp5gQEUlx\nyb5rSEREDkCFQEQkxakQiIikOBUCEZEUp0IgIpLiVAhEDpIFZ/tcbmaZ4e124e1Dos4mUhMqBCIH\nyd1XE5yW477wrvuAcR7hSctEakO/IxCpgfCUDjOAJ4ErgcEenJFUpMFpMOcaEkkk7l5iZrcAbwKn\nqghIQ6ZdQyI1922C0wIcGXUQkdpQIRCpATM7CvgmwahaN0UwmIpInVEhEDlI4RkpxxKc938V8ADw\nYLSpRGpOhUDk4F0JrHL3qeHtx4DDzezrEWYSqTEdNSQikuK0RSAikuJUCEREUpwKgYhIilMhEBFJ\ncSoEIiIpToVARCTFqRCIiKS4/w+VvfcxubsF8gAAAABJRU5ErkJggg==\n", "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "ss_approx = sum_of_sigmoid_approx(X)\n", "\n", "fig = plt.figure()\n", "plt.plot(X, ss_approx)\n", "plt.xlabel('X')\n", "plt.ylabel('Y')\n", "plt.title('sum of sigmoids approximation')\n", "fig.savefig('figures/sum_sigmoids_approx.png')\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "But do we really need the log and the exp, or could we just take max(0, x)? And that works fine, thus giving you the ReLU.\n", "\n", "### ReLU\n", "\n", "In context of artificial neural networks, a rectifier is defined as,\n", "\n", "\\begin{equation*}\n", "f\\left(x\\right) = max\\left(0, x\\right)\n", "\\end{equation*}\n", "\n", "Here, x the input to a neuron. This is also known as the ramp function. A unit employing the rectifier is called the rectified linear unit (ReLU)." ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "collapsed": true }, "outputs": [], "source": [ "def relu(x):\n", " return np.maximum(0, x)" ] }, { "cell_type": "code", "execution_count": 10, "metadata": { "collapsed": false }, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHcxJREFUeJzt3Xd8lfX5//HXJUsREJCgTMGFkxFSFG0dde/WjdhabaWK\no1LbirXVqv3166qjFrS0tbaVKbjFgbutFRtCmGGvRFYQ2QIZ1++Pc0djTELIOee+z3g/H488cnKf\n++S+uM/NufK5P+fcb3N3REQke+0RdQEiIhItNQIRkSynRiAikuXUCEREspwagYhIllMjEBHJcmoE\nIiJZTo1ABDCzZWb2uZltMbPVZva0mbVqwONOMrOSOu57z8x+1ND1RaKiRiDypfPcvRXQF+gH3B5x\nPSKhUCMQqcHdVwNvEGsImFkLM3vIzFaY2Roze9LM9oq2SpHEUSMQqcHMugJnAYuCRfcDhxJrDAcD\nXYA7o6lOJPHUCES+9IKZbQaKgbXAXWZmwLXAMHdf7+6bgd8Bl0dYp0hCNY26AJEU8h13f8vMTgTG\nAB2A5kBLYFqsJwBgQJMG/L5yoFmNZc2AssSUK5IYGhGI1ODu7wNPAw8B64DPgSPdvW3wtU8wqbwr\nK4AeNZb1BJYnsFyRuKkRiNTuUeA0oDfwZ+ARM+sIYGZdzOyM6iub2Z41vgwYD1xtZgMs5lBgGDAu\n3H+KSP3UCERq4e6lwD+AXwO3EZs4/sjMNgFvAb2qrd6F2Kih+tdB7v4GMBz4G7ARmAz8HRgV0j9D\npEFMwTQiItlNIwIRkSynRiAikuXUCEREspwagYhIlkuLD5R16NDBe/ToEXUZIiJpZdq0aevcPWdX\n66VFI+jRowf5+flRlyEiklbMrEEfXtSpIRGRLJe0RmBmT5nZWjObXW1ZezObYmYLg+/tkrV9ERFp\nmGSOCJ4GzqyxbDjwtrsfArwd/CwiIhFKWiNw9w+A9TUWX0DsI/YE37+TrO2LiEjDhD1HsJ+7rwII\nvnesa0UzG2Jm+WaWX1paGlqBIiLZJmUni919lLvnuXteTs4u3/0kIiKNFHYjWGNmnQCC72tD3r6I\niNQQdiN4CbgquH0V8GLI2xcRSQufbtnBPS/PZdvO8qRvK5lvHx0L/BfoZWYlZvZD4D7gNDNbSCz0\n475kbV9EJF1VVDq3jC/kmanLWbF+W9K3l7RPFrv7oDruOiVZ2xQRyQSPv7OQfy1cx30XHs1h+7dJ\n+vZSdrJYRCQbfbCglMfeXsiFuV247BvdQtmmGoGISIpYvXE7t4wv5JCOrfjtd44iFn2dfGoEIiIp\noKyikhvGFLC9rIKRg/vTsnl41wRNi6uPiohkugden8e05Z/xh0H9OLhjq1C3rRGBiEjE3pizmj//\naynfO/YAzu/TOfTtqxGIiERo+adb+dmzM+jddR9+de7hkdSgRiAiEpHtZRUMHV2AASOuyKVF0yaR\n1KE5AhGRiNzzylzmrNzEX76fR7f2LSOrQyMCEZEIvDD9E8ZMXcF1Jx7EqUfsF2ktagQiIiFbuGYz\ntz83iwE92/Oz0w+Nuhw1AhGRMG3dUc71owvYu0UT/jioH02bRP8yrDkCEZGQuDt3PD+LJaVbeOaH\nx9CxzZ5RlwRoRCAiEpoxH6/ghcKVDDv1UI47uEPU5XxBjUBEJASzP9nI3S/N5cRDc7jh5IOjLucr\n1AhERJJs47Yyrh89jX1bNeeRy/qyxx7hXEyuoTRHICKSRO7OzybOYNWG7Yz/8UDa79086pK+RiMC\nEZEk+vO/ljBl7hpuP/tw+h/QLupyaqVGICKSJP9btp77X5/PWUftzzXH94i6nDqpEYiIJMG6LTu4\ncUwB3drtxf0X9w4tZKYx1AhERBKsotL5ybjpbNhWxojBubTZs1nUJdVLk8UiIgn22NsL+c+iT7n/\noqM5svM+UZezSxoRiIgk0AcLSnn8nYVclNuVS/PCCZ+PlxqBiEiCrNr4ObeML+TQjq1DDZ+PlxqB\niEgClFVUcuOY6ewoq2Dklbns1TyakJnG0ByBiEgC3P9aLHz+8UH9OCgn3PD5eGlEICISp9dnr+Yv\n/17K9wcewHkRhM/HS41ARCQOyz/dys+fnUGfrvtwxznRhM/HS41ARKSRqsLn99jD+GOE4fPx0hyB\niEgj3f1yLHz+r1dFGz4fL40IREQa4fnpJYz9OBY+f8rh0YbPx0uNQERkNy1cs5lfPjebAT1SI3w+\nXpE0AjMbZmZzzGy2mY01s9QI7hQR2YWvhM9fkRrh8/EK/V9gZl2Am4E8dz8KaAJcHnYdIiK7y935\nZRA+/4dB/VImfD5eUbWypsBeZtYUaAmsjKgOEZEGGz11BS8WruSnpx3KcQelTvh8vEJvBO7+CfAQ\nsAJYBWx09zfDrkNEZHfMKtnIPS/P5aReOQw9KbXC5+MVxamhdsAFQE+gM7C3mV1Zy3pDzCzfzPJL\nS0vDLlNE5AtV4fMdWjXnkUtTL3w+XlGcGjoVWOrupe5eBjwHHFdzJXcf5e557p6Xk5MTepEiIhCb\nF7j12Rms2bSdEYNzaZeC4fPxiqIRrACONbOWFrtG6ylAUQR1iIjs0qgPlvBW0Rp+efbh9OuemuHz\n8YpijmAqMBEoAGYFNYwKuw4RkV35eOl6HnhjPucc3YkfHNcj6nKSJpJLTLj7XcBdUWxbRKQh1m3Z\nwU1jC+jeviX3XXR02oTMNEb6fxJCRCTBqofPjxycS+sUD5+Ply46JyJSQ1X4/AMX9ebwTm2iLifp\nNCIQEanm/SB8/uL+Xbn0G+kRPh8vNQIRkcDKDZ9zy7jp9NqvNfdecFTU5YRGjUBEhKrw+QJ2llcy\nYnB6hc/HS3MEIiLAfa/No2DFBv54RfqFz8dLIwIRyXqvz17NX/+9lKsGHsC5vdMvfD5eagQiktWW\nrfsyfP6XaRo+Hy81AhHJWtXD50cMTt/w+XhpjkBEstbdL89h7qpNPPWDPLq2S9/w+XhpRCAiWem5\nghLGflzM0JMO4tuHpXf4fLzUCEQk6yxYs5k7np/NMT3b89PT0j98Pl5qBCKSVbbuKOf6Z6axd4um\nPD4oM8Ln46U9ICJZw90Z/twslq7byuMZFD4fLzUCEckaz3y0nJdnrOTW03sx8KB9oy4nZagRiEhW\nmFmygXtfKeKkXjlcf+JBUZeTUtQIRCTjbdxWxtDRBeS0bpGR4fPx0ucIRCSjVVY6tz5byJpN25nw\n44EZGT4fL40IRCSjjfrXEt4qWpvR4fPxUiMQkYw1dcmnPPjGfM4+ev+MDp+PlxqBiGSk0s07uGns\ndLq3b8n9F/XO6PD5eKkRiEjGqQqf3/h5doTPx0uTxSKScR57awEfLv6UBy7OjvD5eGlEICIZ5f0F\npTz+7iIu6d+VS/OyI3w+XmoEIpIxqofP35NF4fPxUiMQkYxQFT5fVuGMzLLw+XhpjkBEMkJV+PyI\nK3I5MMvC5+OlEYGIpL3XZ6/ir/9eyg+O68E5vTtFXU7aUSMQkbQWC5+fSZ9ubfnl2dkZPh8vNQIR\nSVvbyyq4fnQBTZoYI67oR/OmeklrDM0RiEja+s1LcyhatYm/Xf2NrA6fj5fap4ikpUnTShj3v2Ju\nOPkgTu7VMepy0lokjcDM2prZRDObZ2ZFZjYwijpEJD3NW72JO16YxbEHtmfYqQqfj1dUp4YeA153\n94vNrDmgMZ2INMiWHeUMHV1A6z2b8QeFzydE6I3AzNoAJwA/AHD3ncDOsOsQkfTj7gyfNJNl67Yy\n+kfH0rG1wucTIYpWeiBQCvzNzKab2V/MbO+aK5nZEDPLN7P80tLS8KsUkZTzz4+W88rMVQqfT7Ao\nGkFTIBd4wt37AVuB4TVXcvdR7p7n7nk5OTlh1ygiKWZG8QbufWUuJyt8PuGiaAQlQIm7Tw1+nkis\nMYiI1GrDtp0MHV1Ax9Z78rDC5xMu9Ebg7quBYjPrFSw6BZgbdh0ikh4qK51bJ8xg7ebtjBicq/D5\nJIjqXUM3AaODdwwtAa6OqA4RSXF/+mAJb89by2/OO4K+3dpGXU5GiqQRuHshkBfFtkUkfUxd8ikP\nvTmfc47uxFUKn08avQFXRFJS9fD5+y46WuHzSaRGICIpR+Hz4dJF50Qk5Tyq8PlQaUQgIinlvflr\nefydRVyap/D5sKgRiEjK+GTD5wwbX8hh+yt8PkxqBCKSEnaWfxk+/8SV/dmzmcLnw6I5AhFJCf/3\nWhHTg/D5nh2+dvkxSSKNCEQkcpNnreJv/1mm8PmIqBGISKSWrtvKLybOpK/C5yOjRiAikdleVsHQ\n0QU0bWKMGJyr8PmIaI5ARCJz14tfhs93abtX1OVkLbVfEYnExGkljM8v5saTD1b4fMTUCEQkdPNW\nb+JXL8xi4IH7Muw0hc9HTY1AREK1ZUc5Q5+Jhc8/NqgvTRQyE7k6G4GZTTazHuGVIiKZzt25bdJM\nln26lccH9VP4fIqob0TwNPCmmd1hZrr0n4jE7R//Xc6rM1fxszN6ceyBCp9PFXW+a8jdJ5jZq8Cd\nQL6Z/ROorHb/wyHUJyIZorB4A799dS6nHNaR605Q+Hwq2dXbR8uArUALoDXVGoGISENt2LaTG4Lw\n+d9f2kfh8ymmzkZgZmcCDwMvAbnuvi20qkQkY1RWOsPGF7J283aeve442rZU+HyqqW9EcAdwibvP\nCasYEck8T7y/mHfnl3L3+UcqfD5F1TdH8K0wCxGRzPPfxZ/y+zfnc07vTnx/4AFRlyN10OcIRCQp\n1m7ezs3jptNj3725/6LeCp9PYbrWkIgkXHlFJTePnc7m7WX884cDaNVCLzWpTM+OiCTcI28t4KMl\n63nw4t4ctr/C51OdTg2JSEK9O28tI95dzKV5XblE4fNpQY1ARBLmkw2fM2yCwufTjRqBiCTEzvJK\nbhhdQLnC59OO5ghEJCF+N7mIwuINjBys8Pl0oxGBiMTt1ZmrePrDZVx9fA/OPlrh8+lGjUBE4rJ0\n3VZumzSTft3bcvtZCp9PR2oEItJo28squP6ZaTRtYvzxCoXPp6vInjUza2Jm083slahqEJH43Pni\nbOat3swjl/VV+Hwai7J9/wQoinD7IhKHCfnFTMgvUfh8BoikEZhZV+Ac4C9RbF9E4lO0ahO/fmG2\nwuczRFQjgkeBX1BP0I2ZDTGzfDPLLy0tDa8yEanX5u1lDB1dQJu9FD6fKUJvBGZ2LrDW3afVt567\nj3L3PHfPy8nJCak6EamPuzN80iyWK3w+o0QxIjgeON/MlgHjgG+b2TMR1CEiu+nvHy7j1Vmr+PkZ\nhyl8PoOE3gjc/XZ37+ruPYDLgXfc/cqw6xCR3VNYvIH/N7mIUw7ryI9PODDqciSB9KZfEdmlz7Yq\nfD6TRXqtIXd/D3gvyhpEpH6Vlc5PJxRSunkHz143UOHzGUgjAhGpV1X4/K/OPZw+Cp/PSGoEIlKn\nqvD58/p05nvHKnw+U6kRiEit1m7ezk1jY+Hz/3fh0Qqfz2DKIxCRr6kKn9+yo4zRPzpG4fMZTs+u\niHxNVfj8Q5f0odf+raMuR5JMp4ZE5Cuqwucvy+vGxf27Rl2OhECNQES+UPLZNoZNKOTwTm24+4Ij\noy5HQqJGICJAED4/ZjoVFc4Tg3MVPp9FNEcgIkAsfH5G8QaevDKXHgqfzyoaEYjIF+HzP/xmT848\nSuHz2UaNQCTLLSndwm2TZpLbvS3Dzzos6nIkAmoEIlns850VDB1dQLMgfL5ZE70kZCPNEYhksTtf\nnM38NZt5+uoBdFb4fNZS+xfJUhPyi3l2Wgk3nXwwJx6qFMBspkYgkoWqwuePO2hffnKqwueznRqB\nSJapCp/fZ69mPHZ5P4XPi+YIRLJJVfj8ivXbGHvtseS0bhF1SZICNCIQySJPfxE+34sBPdtHXY6k\nCDUCkSwxfcVn/G5yEace3pEh31L4vHxJjUAkC1SFz+/XZk9+f0lfhc/LV2iOQCTDVVY6wyYUsm7L\nTiZeP5B9WjaLuiRJMRoRiGS4J95fzHvzS/n1eUfQu6vC5+Xr1AhEMlhV+Pz5fTpz5THdoy5HUpQa\ngUiGWrspFj7fs4PC56V+miMQyUDlFZXcNHY6W3eUM+baY9hb4fNSDx0dIhno4SkLmLp0PQ9f2odD\n91P4vNRPp4ZEMsw789Yw8r3FDBrQjQtzFT4vu6ZGIJJBitdvY9j4GRzRqQ13nafweWkYNQKRDLGj\nvIIbxxRQWek8caXC56XhNEcgkiF+92oRM0o28uSVuRywr8LnpeE0IhDJAK/MXMnf/7tc4fPSKKE3\nAjPrZmbvmlmRmc0xs5+EXYNIJllcuoXbJip8XhovilND5cCt7l5gZq2BaWY2xd3nRlCLSFr7fGcF\nQ58poHnTPRQ+L40W+lHj7qvcvSC4vRkoArqEXYdIJvj1i7NZsHYzj17eT+Hz0miR/vlgZj2AfsDU\nKOsQSUcT/lfMRIXPSwJE1gjMrBUwCbjF3TfVcv8QM8s3s/zS0tLwCxRJYXNXbuLXL87m+IMVPi/x\ni6QRmFkzYk1gtLs/V9s67j7K3fPcPS8nR3/tiFTZvL2MG8YU0LalwuclMUKfLLbYJRD/ChS5+8Nh\nb18knbk7t02ayYr12xg35Fg6tFL4vMQvihHB8cD3gG+bWWHwdXYEdYiknac/XMbkWav5xRm9+EYP\nhc9LYoQ+InD3fwMay4rspi/D5/djyAkKn5fE0ZuORdLAV8Pn+yhkRhJK1xoSSXHVw+cnXX+cwucl\n4TQiEElxI99bxHvzS7nzvCM4uus+UZcjGUiNQCSFfbh4HQ9PWcAFfTszWOHzkiRqBCIpau2m7dw8\ntpADc1rxu+8qfF6SR3MEIimovKKSG4Pw+bEKn5ck09ElkoJ+P2UBHy9dzyOX9eEQhc9LkunUkEiK\nebtoDU+8t5hBA7rz3X4Kn5fkUyMQSSHF67fx0wkzOLJzG+4674ioy5EsoUYgkiK+CJ93Z+Rghc9L\neDRHIJIivgyf76/weQmVRgQiKeDlGbHw+R99sydnHrV/1OVIllEjEInY4tItDJ80k/4HtOM2hc9L\nBNQIRCJUFT7folkT/nhFP4XPSyQ0RyASEXf/Inz+71cPoNM+Cp+XaOjPD5GITMiPhc/f/O1DOEHh\n8xIhNQKRCMxduYk7X5zDNw/uwM2nHBJ1OZLl1AhEQrZpexlDR0+jbctmPHp5X4XPS+Q0RyASInfn\ntokzKf7sc4XPS8rQiEAkRH/7zzJem72a285U+LykDjUCkZAUBOHzpx2xH9d+S+HzkjrUCERC8NnW\nndw4uoBObffkIYXPS4rRHIFIklVWOreMrxY+v5fC5yW1aEQgkmQj3l3E+wsUPi+pS41AJIk+XLSO\nR95S+LykNjUCkSRZs2k7N4+brvB5SXmaIxBJgvKKSm4aO52tOyoYe22uwuclpenoFEmCB9+cz8dL\n1/PoZX0VPi8pT6eGRBJsytw1/On9JVxxTHe+069L1OWI7JIagUgCFa/fxq0TCjmqSxvuPFfh85Ie\n1AhEEmRHeQU3jCnAgZFX9Ff4vKQNzRGIJMhvXyliZslG/vS9/nTft2XU5Yg0WCQjAjM708zmm9ki\nMxseRQ0iifTSjJX886PlXPutnpxxpMLnJb2E3gjMrAkwAjgLOAIYZGY6mSppa9HaWPh83gHt+MWZ\nCp+X9BPFqaEBwCJ3XwJgZuOAC4C5id7QHc/P4uOl6xP9a0W+onTLDvZs1oTHFT4vaSqKRtAFKK72\ncwlwTM2VzGwIMASge/fGfTS/c9u9OGS/Vo16rEhD9dq/Ndd8s6fC5yVtRdEIavucvX9tgfsoYBRA\nXl7e1+5viBtOPrgxDxMRySpRjGNLgG7Vfu4KrIygDhERIZpG8D/gEDPraWbNgcuBlyKoQ0REiODU\nkLuXm9mNwBtAE+Apd58Tdh0iIhITyQfK3H0yMDmKbYuIyFfpvW4iIllOjUBEJMupEYiIZDk1AhGR\nLGfujfqsVqjMrBRY3siHdwDWJbCcREnVuiB1a0vVuiB1a0vVuiB1a8ukug5w95xdrZQWjSAeZpbv\n7nlR11FTqtYFqVtbqtYFqVtbqtYFqVtbNtalU0MiIllOjUBEJMtlQyMYFXUBdUjVuiB1a0vVuiB1\na0vVuiB1a8u6ujJ+jkBEROqXDSMCERGphxqBiEiWy4hGYGaXmNkcM6s0s7wa991uZovMbL6ZnVHH\n43ua2VQzW2hm44PLYye6xvFmVhh8LTOzwjrWW2Zms4L18hNdRx3b/I2ZfVKtvrPrWO/MYD8uMrPh\nIdT1oJnNM7OZZva8mbWtY73Q9tmu9oGZtQie60XBMdUjmfUE2+xmZu+aWVHw/+AntaxzkpltrPYc\n35nsuoLt1vvcWMwfgv0108xyQ6qrV7V9UWhmm8zslhrrhLbPzOwpM1trZrOrLWtvZlOC16UpZtau\njsdeFayz0MyualQB7p72X8DhQC/gPSCv2vIjgBlAC6AnsBhoUsvjJwCXB7efBK5Pcr2/B+6s475l\nQIeQ999vgJ/tYp0mwf47EGge7NcjklzX6UDT4Pb9wP1R7rOG7ANgKPBkcPtyYHwIdXUCcoPbrYEF\ntdR1EvBKmMdVQ54b4GzgNWLJhccCUyOosQmwmtiHryLZZ8AJQC4wu9qyB4Dhwe3htR3/QHtgSfC9\nXXC73e5uPyNGBO5e5O7za7nrAmCcu+9w96XAImBA9RXMzIBvAxODRX8HvpOsWoPtXQqMTdY2kmQA\nsMjdl7j7TmAcsf2bNO7+pruXBz9+RCzNLkoN2QcXEDuGIHZMnRI850nj7qvcvSC4vRkoIpYNng4u\nAP7hMR8Bbc2sU8g1nAIsdvfGXr0gbu7+AbC+xuLqx1Jdr0tnAFPcfb27fwZMAc7c3e1nRCOoRxeg\nuNrPJXz9P8i+wIZqLzi1rZNI3wLWuPvCOu534E0zm2ZmQ5JYR003BkPzp+oYgjZkXybTNcT+cqxN\nWPusIfvgi3WCY2ojsWMsFMGpqH7A1FruHmhmM8zsNTM7MqSSdvXcRH1cQWzkVtcfZlHssyr7ufsq\niDV7oGMt6yRk/0USTNMYZvYWsH8td93h7i/W9bBaltV8v2xD1mmQBtY4iPpHA8e7+0oz6whMMbN5\nwV8LcamvNuAJ4F5i/+57iZ26uqbmr6jlsXG/97gh+8zM7gDKgdF1/Jqk7LPayq1lWdKOp91lZq2A\nScAt7r6pxt0FxE59bAnmgF4ADgmhrF09N5HtL4BgPvB84PZa7o5qn+2OhOy/tGkE7n5qIx5WAnSr\n9nNXYGWNddYRG442Df6Cq22dhNRoZk2BC4H+9fyOlcH3tWb2PLHTEXG/qDV0/5nZn4FXarmrIfsy\n4XUFk1/nAqd4cFK0lt+RlH1Wi4bsg6p1SoLnex++PuRPODNrRqwJjHb352reX70xuPtkMxtpZh3c\nPakXV2vAc5OU42o3nAUUuPuamndEtc+qWWNmndx9VXC6bG0t65QQm8uo0pXYXOluyfRTQy8Blwfv\n5OhJrJt/XH2F4MXlXeDiYNFVQF0jjHidCsxz95La7jSzvc2sddVtYpOls2tbN5FqnJP9bh3b/B9w\niMXeYdWc2HD6pSTXdSZwG3C+u2+rY50w91lD9sFLxI4hiB1T79TVwBIlmIP4K1Dk7g/Xsc7+VXMV\nZjaA2P/9T5NcV0Oem5eA7wfvHjoW2Fh1OiQkdY7Qo9hnNVQ/lup6XXoDON3M2gWndE8Plu2eMGbE\nk/1F7MWrBNgBrAHeqHbfHcTe6TEfOKva8slA5+D2gcQaxCLgWaBFkup8GriuxrLOwORqdcwIvuYQ\nOz0Sxv77JzALmBkcfJ1q1hb8fDaxd6QsDqO24PkoBgqDrydr1hX2PqttHwD3EGtWAHsGx9Ci4Jg6\nMIT99E1ipwNmVttXZwPXVR1vwI3B/plBbOL9uBDqqvW5qVGXASOC/TmLau/6C6G+lsRe2PeptiyS\nfUasGa0CyoLXsh8Sm1t6G1gYfG8frJsH/KXaY68JjrdFwNWN2b4uMSEikuUy/dSQiIjsghqBiEiW\nUyMQEclyagQiIllOjUBEJMupEYjsJotd7XOpmbUPfm4X/HxA1LWJNIYagchucvdiYpfluC9YdB8w\nyiO8aJlIPPQ5ApFGCC7pMA14CrgW6OexK5KKpJ20udaQSCpx9zIz+znwOnC6moCkM50aEmm8s4hd\nFuCoqAsRiYcagUgjmFlf4DRiqVrDIghTEUkYNQKR3RRckfIJYtf9XwE8CDwUbVUijadGILL7rgVW\nuPuU4OeRwGFmdmKENYk0mt41JCKS5TQiEBHJcmoEIiJZTo1ARCTLqRGIiGQ5NQIRkSynRiAikuXU\nCEREstz/BxulXZ1CoIoPAAAAAElFTkSuQmCC\n", "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "relu1 = relu(X)\n", "\n", "fig = plt.figure()\n", "plt.plot(X, relu1)\n", "plt.xlabel('X')\n", "plt.ylabel('Y')\n", "plt.title('ReLU')\n", "fig.savefig('figures/relu.png')\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### Problem with ReLU:\n", "\n", "The problem with relu is that it’s mean is not zero. A positive mean introduces a bias for the next layer which can slow down the learning. If the mean value of activation is zero we get a faster learning." ] }, { "cell_type": "code", "execution_count": 11, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/plain": [ "2.449999999999974" ] }, "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ "np.mean(relu1)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Behold the ELU\n", "\n", "With Exponential Linear Units (ELU), we can have a mean activation that is close to 0 and it is an exponential function. ELU does not saturate for large values of x. It is expressed as,\n", "\n", "\\begin{split}f(x) = \\left \\{ \\begin{array}{ll}\n", "x & {\\rm if}~ x \\ge 0 \\\\\n", "\\alpha (\\exp(x) - 1) & {\\rm if}~ x < 0,\n", "\\end{array} \\right.\\end{split}\n", "\n", "where α is a parameter. See: https://arxiv.org/abs/1511.07289\n", "\n", "In plain English, it acts like a ReLU unit if x is positive, but for negative values it is a function bounded by a fixed value -1, for alpha=1. This behavior helps to push the mean activation of neurons closer to zero which is beneficial for learning and it helps to learn representations that are more robust to noise." ] }, { "cell_type": "code", "execution_count": 12, "metadata": { "collapsed": true }, "outputs": [], "source": [ "def elu(arr, alpha):\n", " a = []\n", " for x in arr:\n", " if x >= 0:\n", " a.append(x)\n", " else:\n", " a.append(alpha * (np.exp(x)-1))\n", " \n", " return a" ] }, { "cell_type": "code", "execution_count": 13, "metadata": { "collapsed": false }, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl8VPW9//HXJwsJOwHCvgQEQRBkCSBqbStU0VptrbTo\nRUERpNZW/LVarb1ql9uqXa62vS0iWsKiKFQrWquiti63LkzYV4PsexAIa0KS+f7+mNEbYxKSkJnv\nLO/n45FHZs6c4bw5M5l3zjnfnGPOOUREJHml+A4gIiJ+qQhERJKcikBEJMmpCEREkpyKQEQkyakI\nRESSnIpARCTJqQhEqmBmW8zshJkdrfD1RzObaGbv1PCc0ZWmVTu/SKxQEYhU72vOuWYVvm71HUgk\nElQEIiJJTkUgIpLk0nwHEIlhfzOzsgr37wBKfYURiRRtEYhU7+vOuVYVvh47xfxlQHqlaemoPCTG\nqQhEGs42IKfStB7A1uhHEak9FYFI3ZmZZVb8Ck9/GphmZn0tJBe4EZjvL6rIqekYgUj1XjCz8gr3\nFwPPA+cBJyrOaGbpwGNAFvAC0B7YAdzjnHs5OnFF6sd0YRoRkeSmXUMiIklORSAikuRUBCIiSU5F\nICKS5OJi1FDbtm1dTk6O7xgiInElPz9/v3Mu+1TzxUUR5OTkEAgEfMcQEYkrZlarP2bUriERkSQX\nsSIwsyfMbJ+Zra4wrbWZLTazgvD3rEgtX0REaieSWwSzgDGVpt0FvO6c6w28Hr4vIiIeRawInHNv\nAQcqTb4SyAvfzgO+Hqnli4hI7UT7GEF759xugPD3dlFevoiIVBKzB4vNbIqZBcwsUFhY6DuOiEjC\ninYR7DWzjgDh7/uqm9E5N8M5l+ucy83OPuUwWBERqadoF8EiYEL49gRCp/QVEZFKPj5aws9eWEtx\nafmpZz5NkRw++hTwLtDHzHaY2STgAeArZlYAfCV8X0REKigpK+fmOfnMe38rHxUejfjyIvaXxc65\na6p5aFSklikiEu+cc9z97CoCWw/yx2sH079Ty4gvM2YPFouIJKM//esjnl26k9tHn8nlAztFZZkq\nAhGRGPHy6t38+pUNXHFOJ74/qlfUlqsiEBGJAat3FnH70ysY1LUVD109EDOL2rJVBCIinu09XMyk\nvCVkNUlnxvVDyUxPjery4+I01CIiierEyXJuygtwpLiMv37nPNo1z4x6BhWBiIgnwaDjBwuWs3pX\nEY9dl8tZHVt4yaFdQyIinvz3ax/y0qo93H1pX0b3a+8th4pARMSD55fv5A9vbORbuV2Y/IWeXrOo\nCEREomzptoPcsXAlI3q05hdfHxDVEUJVURGIiETRjoPHmTI7QMeWmUwfP5RGaf4/hnWwWEQkSo6W\nlDFpVoCSsiDzpwwjq2kj35EAFYGISFSUBx3ff2oZGwuPknfDcHq1a+Y70qf8b5OIiCSBX720jjfW\n7+P+K/pzQe+2vuN8hopARCTC5n+wjZnvbGbieTlcd25333E+R0UgIhJB//5oPz/522ouPDObn3z1\nLN9xqqQiEBGJkM37j/GduUvJaduUP147mLTU2PzIjc1UIiJxruh4KZNmLSHF4PEJubTITPcdqVoa\nNSQi0sBKy4Pc8mQ+2w8eZ+6kEXRv09R3pBqpCEREGpBzjvsXreF/N37Mr68eyIiebXxHOiXtGhIR\naUB5/97CvPe3cfMXezI2t6vvOLWiIhARaSD/2rCPn724lq/0a8+PLunrO06tqQhERBpAwd4jfO/J\nZfTp0IKHvz2IlBS/J5KrCxWBiMhp+vhoCTfmLSGzUSqPT8ilaUZ8HX5VEYiInIaSsnKmzs1n3+ES\nHrs+l06tGvuOVGfxVVsiIjHEOcePn13Nki0H+cM1gxnUtZXvSPWiLQIRkXqa/uYm/rp0B9NG9+Zr\n53TyHafeVAQiIvXwypo9PPTKer52TiduG9Xbd5zToiIQEamj1TuLmDZ/Oed0acWvrx7o/VKTp8tL\nEZjZ7Wa2xsxWm9lTZpbpI4eISF3tO1zM5NkBspqkM+P6oWSmp/qOdNqiXgRm1hn4PpDrnDsbSAXG\nRTuHiEhdFZeWM3l2gKITpcycMIx2zRPjd1hfu4bSgMZmlgY0AXZ5yiEiUivBoOMHC1awcmcRj4wb\nTL9OLXxHajBRLwLn3E7gN8A2YDdQ5Jx7tfJ8ZjbFzAJmFigsLIx2TBGRz3j49QL+vnI3d43py1f6\ntfcdp0H52DWUBVwJ9AA6AU3NbHzl+ZxzM5xzuc653Ozs7GjHFBH51PPLd/L71wsYO7QLUy7s6TtO\ng/Oxa2g0sNk5V+icKwWeBc7zkENE5JTytx7kjoUrGd6jNf/1jQFxP0KoKj6KYBtwrpk1sdAaHQWs\n85BDRKRGOw4e5+Y5ATq0yGT6+KE0SkvMEfc+jhG8DywElgKrwhlmRDuHiEhNjpaUcVNegJKyIE9M\nzKV100a+I0WMl3MNOefuA+7zsWwRkVMpDzpue2oZBfuO8peJw+jVrrnvSBGVmNs5IiKn4YF/rOP1\n9fu472v9uPDMxB+soiIQEang6SXbeOztzVw/sjvXj8zxHScqVAQiImHvfvQx9zy3mi/0bsu9l/fz\nHSdqVAQiIsCW/cf4zrx8urdpwh+vHUJaavJ8PCbP/1REpBpFJ0q5MW8JBjwxcRgtG6f7jhRVukKZ\niCS10vIg3523lO0HjjN30gi6t2nqO1LUqQhEJGk55/jpC2t4Z+N+Hrp6ICN6tvEdyQvtGhKRpJX3\n7y3MfW8bN1/Yk2/ldvUdxxsVgYgkpTc/LORnL65l9FntuXNMX99xvFIRiEjSKdh7hFvnLaVPhxY8\nMm4QqSmJdyK5ulARiEhSOXDsJJPyAmSkpzJzQi5NM3SoVGtARJJGSVk5U+fks+dwMU9POZfOrRr7\njhQTtEUgIknBOcc9z63mgy0H+M3YcxjcLct3pJihIhCRpPDoW5tYmL+D20b15opzOvmOE1NUBCKS\n8F5Zs4cHX17P5QM7Mm10b99xYo6KQEQS2uqdRUybv5yBXVrxm7HnJOSlJk+XikBEEta+w8VMnh2g\nVZN0HrtuKJnpqb4jxSSNGhKRhFRcWs7k2QGKTpSyYOpI2rXI9B0pZqkIRCThBIOOHyxYwcqdRTw6\nfij9O7X0HSmmadeQiCSch18v4O8rd/OjMX25uH8H33FinopARBLK88t38vvXC7h6aBduvrCn7zhx\nQUUgIglj6baD3LFwJcNzWvPLbwzQCKFaUhGISELYeegEU2bn06FFJtOvG0qjNH281ZYOFotI3Dta\nUsakWUsoKS3nqckjaN20ke9IcUVFICJxrTzomDZ/GQX7jvKXicPo3b6570hxR9tOIhLXHnx5Pa+t\n28d9X+vHhWdm+44Tl1QEIhK3nlmynRlvbeL6kd25fmSO7zhxS0UgInHpvU0f8+PnVvGF3m259/J+\nvuPENS9FYGatzGyhma03s3VmNtJHDhGJT1v2H2Pq3Hy6t2nCH68dQlqqfqc9Hb4OFj8CvOycu9rM\nGgFNPOUQkThTdKKUSXlLMOCJicNo2Tjdd6S4F/UiMLMWwIXARADn3EngZLRziEj8KSsPcuuTS9l2\n4DhzJ42ge5umviMlBB/bUz2BQuAvZrbMzGaa2edeTTObYmYBMwsUFhZGP6WIxJyfvrCWtwv281/f\nGMCInm18x0kYPoogDRgC/Nk5Nxg4BtxVeSbn3AznXK5zLjc7W0PCRJJd3r+3MOe9rdx8YU++ldvV\nd5yE4qMIdgA7nHPvh+8vJFQMIiJVevPDQn76whpGn9WeO8f09R0n4US9CJxze4DtZtYnPGkUsDba\nOUQkPhTsPcKt85bSp0MLHhk3iNQUnUiuofkaNfQ9YF54xNAm4AZPOUQkhh04dpJJeQEy0lOZOSGX\nphk6K04keFmrzrnlQK6PZYtIfCgpK2fqnHz2HC7m6Snn0rlVY9+REpb+CkNEYo5zjnueW80HWw7w\nm7HnMLhblu9ICU1FICIx59G3NrEwfwe3jerNFed08h0n4akIRCSmvLJmDw++vJ7LB3Zk2ujevuMk\nBRWBiMSMNbuKmDZ/OQO7tOI3Y8/RpSajREUgIjFh3+FibsoL0KpJOo9dN5TM9FTfkZKGxmKJiHfF\npeVMnpNP0YlSFkwdSbsWmb4jJRUVgYh45ZzjhwtWsHLHIR4dP5T+nVr6jpR0tGtIRLx6+LUCXly5\nmx+N6cvF/Tv4jpOUVAQi4s2iFbt45PUCvjmkCzdf2NN3nKSlIhARL5ZtO8gPF6xgeE5rfnnV2Roh\n5JGKQESibuehE0yenU/7FhlMv24oGWkaIeSTDhaLSFQdKynjprwAJaXlPDV5BK2bNvIdKempCEQk\nasqDjtvmL2fDnsM8MXEYvds39x1J0K4hEYmih15ez2vr9nLv5f34Up92vuNImIpARKLimcB2Hn1r\nE+PP7caE83J8x5EKVAQiEnHvb/qYe55bxQW92nLf1/prhFCMURGISERt/fgYN8/Np2vrJvzPfwwh\nPVUfO7FGr4iIREzRiVJunLUEgCcmDKNl43TPiaQqKgIRiYiy8iC3PrmUrR8f58//MZSctk19R5Jq\naPioiETEz15cy9sF+3nwmwMYeUYb33GkBtoiEJEGN/vdLcx+dytTLuzJt4d18x1HTqHaIjCzl8ws\nJ3pRRCQRvPVhIT99YS2jz2rPj8b09R1HaqGmLYJZwKtmdo+Z6QiPiJzSxn1H+O6TS+ndrhmPjBtE\naoqGicaDao8ROOeeMbO/A/cCATObAwQrPP67KOQTkThx4NhJbpwVICMthccnDqNphg5BxotTvVKl\nwDEgA2hOhSIQEfnEybIgU+fms+dwMfOnnEvnVo19R5I6qLYIzGwM8DtgETDEOXc8aqlEJG4457jn\nuVV8sPkAj4wbxJBuWb4jSR3VtEVwDzDWObcmWmFEJP7MeGsTC/J38P1RvblyUGffcaQeajpG8IVI\nLtjMUoEAsNM5d3kklyUikbF47V4eeHk9Xx3YkWmjevuOI/Xk8+8IbgPWeVy+iJyGNbuKuG3+MgZ2\nbslvx55DikYIxS0vRWBmXYCvAjN9LF9ETs++I8VMzgvQsnE6j12fS2a6LjUZz3xtETwM3EkNo5DM\nbIqZBcwsUFhYGL1kIlKj4tJyJs/O5+DxUh67Ppd2LTJ9R5LTFPUiMLPLgX3Oufya5nPOzXDO5Trn\ncrOzs6OUTkRq4pzjjoUrWbnjEA+PG8TZnVv6jiQNwMcWwfnAFWa2BZgPXGRmcz3kEJE6euT1Al5Y\nsYs7L+nLJf07+I4jDSTqReCcu9s518U5lwOMA95wzo2Pdg4RqZsXVuzi4dcK+OaQLkz9Yk/fcaQB\n6eyjInJKy7cf4ocLVjAsJ4tfXnW2LjWZYLyeDMQ59y/gXz4ziEjNdh06wU15Adq1yGD6+KFkpGmE\nUKLRWaFEpFrHSsqYlBegpLScJyePoE2zDN+RJAJUBCJSpWDQcfvTy9mw5zBPTBzGme2b+44kEaJj\nBCJSpYde2cCra/fyn5f340t92vmOIxGkIhCRz1kQ2M70Nz/iP0Z0Y+J5Ob7jSISpCETkMz7YfIAf\nP7eK83u14f4r+muEUBJQEYjIp7Z+fIyb5wTomtWEP107lPRUfUQkA73KIgLA4eJSJuUFCDp4fOIw\nWjbRpcqThYpARCgrD/LdeUvZsv8Y08cPpUfbpr4jSRRp+KiI8PMX1/J2wX4euGoAI89o4zuORJm2\nCESS3Jx3t5D37lZuuqAH44Z38x1HPFARiCSxtwsKuf+FtYzq2467LzvLdxzxREUgkqQ27jvKLfOW\n0rtdMx65ZjCputRk0lIRiCShg8dOMilvCRlpKcyckEuzDB0uTGZ69UWSzMmyIDfPzWd3UTFPTT6X\nLllNfEcSz7RFIJJEnHP85G+r+GDzAR765kCGds/yHUligIpAJIk89vYmngns4HsX9eLrgzv7jiMx\nQkUgkiQWr93Lr/6xnssGdOD20Wf6jiMxREUgkgTW7jrMbfOXMaBzS347dhApGiEkFagIRBLcviPF\n3JS3hBaZ6Tx2fS6NG+lSk/JZGjUkksCKS8uZMjufg8dLWTB1JO1bZPqOJDFIRSCSoJxz3LlwJcu3\nH2L6+CGc3bml70gSo7RrSCRB/f71jSxasYs7LunDmLM7+o4jMUxFIJKAXly5i/9+7UOuGtKZW750\nhu84EuNUBCIJZvn2Q/zgmRXkds/iV1cN0KUm5ZRUBCIJZNehE0yeHSC7eQaPXjeUjDSNEJJT08Fi\nkQRxrKSMm/ICnDhZzrybRtCmWYbvSBIntEUgkgCCQcftTy9n/Z7D/OHawZzZvrnvSBJHVAQiCeCh\nVzbw6tq9/OSr/fhyn3a+40iciXoRmFlXM/unma0zszVmdlu0M4gkkoX5O5j+5kdcO6IbN5yf4zuO\nxCEfxwjKgB8455aaWXMg38wWO+fWesgiEteWbDnA3c+u5PxebfjpFf01QkjqJepbBM653c65peHb\nR4B1gM6HK1JH2w8c5+Y5+XTNasKfrh1Keqr29Er9eH3nmFkOMBh4v4rHpphZwMwChYWF0Y4mEtMO\nF5dy46wllAcdj08cRssm6b4jSRzzVgRm1gz4KzDNOXe48uPOuRnOuVznXG52dnb0A4rEqLLyILc+\nuYzN+4/x5/FD6NG2qe9IEue8/B2BmaUTKoF5zrlnfWQQiVe/+Ps63vqwkF9dNYDzzmjrO44kAB+j\nhgx4HFjnnPtdtJcvEs/mvLeVWf/ewqQLenDN8G6+40iC8LFr6HzgOuAiM1se/rrMQw6RuPJ2QSH3\nL1rDRX3b8ePLzvIdRxJI1HcNOefeATTGTaQONu47yi3zltIruxmPjBtEqi41KQ1I481EYtzBYyeZ\nlLeEjLQUZk7IpXmmRghJw9JJ50Ri2MmyIFPn5rP7UDFPTTmXrq2b+I4kCUhFIBKjnHP8599W8/7m\nAzz87UEM7Z7lO5IkKO0aEolRM9/ezNOB7Xzvol58fbD++F4iR0UgEoNeW7uXX/5jHZcN6MDto8/0\nHUcSnIpAJMas232Y2+Yv4+xOLfnt2EGkaISQRJiKQCSGFB4p4aa8AM0z05k5IZfGjXSpSYk8HSwW\niRHFpeVMmRPgwLGTLJg6kvYtMn1HkiShIhCJAc457ly4kmXbDjF9/BDO7tzSdyRJIto1JBID/vDG\nRhat2MUdl/RhzNkdfceRJKMiEPHsxZW7+N3iD7lqSGdu+dIZvuNIElIRiHi0YvshfvDMCnK7Z/Gr\nqwboUpPihYpAxJPdRSeYPDtAdvMMHr1uKBlpGiEkfqgIRDw4frKMm/ICHD9ZzhMTh9GmWYbvSJLE\nVAQiURYMOqbNX8663Yf5w7WDObN9c9+RJMmpCESi7KFXNvDq2r385Kv9+HKfdr7jiKgIRKJpQWA7\n09/8iGtHdOOG83N8xxEBVAQiUfPB5gP8+LlVnN+rDT+9or9GCEnMUBGIRMG2j49z85wAXbOa8Kdr\nh5Keqh89iR16N4pE2OHiUiblLSHo4PGJw2jZRJealNiiIhCJoLLyILc+uYzN+4/x5/FD6NG2qe9I\nIp+jk86JRNAv/r6Otz4s5FdXDeC8M9r6jiNSJW0RiETInPe2MuvfW5h0QQ+uGd7NdxyRaqkIRCLg\nnYL93L9oDRf1bcePLzvLdxyRGqkIRBrYR4VHuWVePr2ym/HIuEGk6lKTEuNUBCIN6OCxk0yatYT0\n1BRmTsileaZGCEns08FikQZysizId+bls+tQMU9NGUHX1k18RxKpFS9bBGY2xsw2mNlGM7vLRwaR\nhuSc497nV/PepgM8ePUAhnZv7TuSSK1FvQjMLBX4H+BSoB9wjZn1i3YOkYb0+Dubmb9kO7d+uRff\nGNzFdxyROvGxRTAc2Oic2+ScOwnMB670kEOkQby+bi//9dI6LhvQgf/3lTN9xxGpMx9F0BnYXuH+\njvA0kbizfPshvvfUMs7u1JLfjh1EikYISRzyUQRV/aS4z81kNsXMAmYWKCwsjEIskbop2HuEiX/5\ngLbNMnh8Qi6NG+lSkxKffBTBDqBrhftdgF2VZ3LOzXDO5TrncrOzs6MWTqQ2dhw8znWPf0B6agpz\nJ42gXYtM35FE6s1HESwBeptZDzNrBIwDFnnIIVIv2w8cZ/zM9zl+sow5k4bTrY2GiUp8i/rfETjn\nyszsVuAVIBV4wjm3Jto5ROpj9c4ibpi1hJLScv5yw3D6dmjhO5LIafPyB2XOuZeAl3wsW6S+3vyw\nkFvm5tOqSSOevGkEvXXReUkQ+stikVMoDzqmv/kRv1v8IWe2b86sG4bRXscEJIGoCERqsKeomNuf\nXs67mz7mqwM78sBVA3T+IEk4KgKRKgSDjgX523ngH+spLg3y0DcHMja3iy44LwlJRSBSyfLth7jv\n+dWs2FHEsJwsHvjmQM7IbuY7lkjEqAhEwlbvLOKPb2zk5TV7yG6ewcPfHsSVgzppK0ASnopAklow\n6Hhn437+8r+b+eeGQppnpvH9i3ox+cKeOhYgSUNFIElp+4HjLFqxi/lLtrH9wAlaN23EHZf04bqR\n3WmhApAkoyKQpOCcY/2eI7yxfh8vr97Dqp1FAJx3RhvuvKQvF/dvT0aazhUkyUlFIAmprDzIh3uP\nsmz7QT7YfID/3bif/UdPAjCoayvuvrQvl57dUaeHEEFFIHHOOce+IyVs3n+Mgn1HWb/7MBv2HGHN\nrsOcKC0HoG2zDC7o1Zbze7Xlgt5t6diysefUIrFFRSAxq7Q8yMHjJzl4rJTCIyXsO1LM3sMl7C46\nwc6DJ9h56ATbDhzn+MnyT5/TPDONszq04NvDujK4WysGd82ia+vGGvkjUgMVgZySc46gC51qoTzo\nKAsGCQahLBikPOgoDTrKyoOUloceKy1znCwPcrIsSElZefh7kOLScorLgpSUlnOspJzjpWUcLynn\nWEkZR0rKOFZSRtGJ0k+/jhSXVZmneWYanVs1pnOrxpzbsw09s5uS06YpZ7RrRqeWmfrQF6mjhC6C\ne55bxQebD3x6/3NXv2lgzlW9hBqXW82DFSc75z69X3ERDvfp/c9MD99x4emfzBd04We5Tx4LfcA7\n98njofvB8P1y5z69HQmN0lJo0iiVZhlpn351aJFJn/bNadE4nawmjWjdrBFZTdLJbpZBuxaZtGue\nQdOMhH7bikRdQv9EdWrVmN7tP/sXoVblBdIaUDX/fE1Lre43WPvMPP93v+L8VmFGwzD77HNDzzNS\nUsJzhP8dM0gxC982UsxIsfD0lP+7n2qGmZGaEr6fkkJaipGSYqSnhqanmpGemkJaauh7o9QU0tNS\nSE81MtJSyUhLoVFaCo3TU8lMTyUzPXQ7LdXH5TBEpLKELoLvfrmX7wgiIjFPv5KJiCQ5FYGISJJT\nEYiIJDkVgYhIklMRiIgkORWBiEiSUxGIiCQ5FYGISJKz6k6LEEvMrBDYWs+ntwX2N2CchhKruSB2\ns8VqLojdbLGaC2I3WyLl6u6cyz7VTHFRBKfDzALOuVzfOSqL1VwQu9liNRfEbrZYzQWxmy0Zc2nX\nkIhIklMRiIgkuWQoghm+A1QjVnNB7GaL1VwQu9liNRfEbraky5XwxwhERKRmybBFICIiNVARiIgk\nuYQoAjMba2ZrzCxoZrmVHrvbzDaa2QYzu6Sa5/cws/fNrMDMnjazRhHI+LSZLQ9/bTGz5dXMt8XM\nVoXnCzR0jmqWeb+Z7ayQ77Jq5hsTXo8bzeyuKOT6tZmtN7OVZvacmbWqZr6orbNTrQMzywi/1hvD\n76mcSOYJL7Ormf3TzNaFfw5uq2KeL5lZUYXX+N5I5wovt8bXxkJ+H15fK81sSJRy9amwLpab2WEz\nm1ZpnqitMzN7wsz2mdnqCtNam9ni8OfSYjPLqua5E8LzFJjZhHoFCF2vNr6/gLOAPsC/gNwK0/sB\nK4AMoAfwEZBaxfOfAcaFb08HvhPhvL8F7q3msS1A2yivv/uBH55intTw+usJNAqv134RznUxkBa+\n/SDwoM91Vpt1ANwCTA/fHgc8HYVcHYEh4dvNgQ+ryPUl4MVovq9q89oAlwH/IHQF1XOB9z1kTAX2\nEPrjKy/rDLgQGAKsrjDtIeCu8O27qnr/A62BTeHvWeHbWXVdfkJsETjn1jnnNlTx0JXAfOdciXNu\nM7ARGF5xBgtdAPgiYGF4Uh7w9UhlDS/vW8BTkVpGhAwHNjrnNjnnTgLzCa3fiHHOveqcKwvffQ/o\nEsnl1UJt1sGVhN5DEHpPjbLqLkrdQJxzu51zS8O3jwDrgM6RXGYDuhKY7ULeA1qZWccoZxgFfOSc\nq+/ZC06bc+4t4EClyRXfS9V9Ll0CLHbOHXDOHQQWA2PquvyEKIIadAa2V7i/g8//gLQBDlX4wKlq\nnob0BWCvc66gmscd8KqZ5ZvZlAjmqOzW8Kb5E9VsgtZmXUbSjYR+c6xKtNZZbdbBp/OE31NFhN5j\nURHeFTUYeL+Kh0ea2Qoz+4eZ9Y9SpFO9Nr7fVxDacqvuFzMf6+wT7Z1zuyFU9kC7KuZpkPUXNxev\nN7PXgA5VPHSPc+756p5WxbTK42VrM0+t1DLjNdS8NXC+c26XmbUDFpvZ+vBvC6elpmzAn4GfE/p/\n/5zQrqsbK/8TVTz3tMce12admdk9QBkwr5p/JiLrrKq4VUyL2PuprsysGfBXYJpz7nClh5cS2vVx\nNHwM6G9A7yjEOtVr4219AYSPB14B3F3Fw77WWV00yPqLmyJwzo2ux9N2AF0r3O8C7Ko0z35Cm6Np\n4d/gqpqnQTKaWRpwFTC0hn9jV/j7PjN7jtDuiNP+UKvt+jOzx4AXq3ioNuuywXOFD35dDoxy4Z2i\nVfwbEVlnVajNOvhknh3h17sln9/kb3Bmlk6oBOY5556t/HjFYnDOvWRmfzKzts65iJ5crRavTUTe\nV3VwKbDUObe38gO+1lkFe82so3Nud3h32b4q5tlB6FjGJ7oQOlZaJ4m+a2gRMC48kqMHoTb/oOIM\n4Q+XfwJXhydNAKrbwjhdo4H1zrkdVT1oZk3NrPkntwkdLF1d1bwNqdI+2W9Us8wlQG8LjbBqRGhz\nelGEc42J+y+5AAACOUlEQVQBfgRc4Zw7Xs080VxntVkHiwi9hyD0nnqjugJrKOFjEI8D65xzv6tm\nng6fHKsws+GEfvY/jnCu2rw2i4Drw6OHzgWKPtkdEiXVbqH7WGeVVHwvVfe59ApwsZllhXfpXhye\nVjfROCIe6S9CH147gBJgL/BKhcfuITTSYwNwaYXpLwGdwrd7EiqIjcACICNCOWcBUytN6wS8VCHH\nivDXGkK7R6Kx/uYAq4CV4Tdfx8rZwvcvIzQi5aNoZAu/HtuB5eGv6ZVzRXudVbUOgJ8RKiuAzPB7\naGP4PdUzCuvpAkK7A1ZWWFeXAVM/eb8Bt4bXzwpCB97Pi0KuKl+bSrkM+J/w+lxFhVF/UcjXhNAH\ne8sK07ysM0JltBsoDX+WTSJ0bOl1oCD8vXV43lxgZoXn3hh+v20EbqjP8nWKCRGRJJfou4ZEROQU\nVAQiIklORSAikuRUBCIiSU5FICKS5FQEInVkobN9bjaz1uH7WeH73X1nE6kPFYFIHTnnthM6LccD\n4UkPADOcx5OWiZwO/R2BSD2ET+mQDzwBTAYGu9AZSUXiTtyca0gkljjnSs3sDuBl4GKVgMQz7RoS\nqb9LCZ0W4GzfQUROh4pApB7MbBDwFUJX1brdw8VURBqMikCkjsJnpPwzofP+bwN+DfzGbyqR+lMR\niNTdZGCbc25x+P6fgL5m9kWPmUTqTaOGRESSnLYIRESSnIpARCTJqQhERJKcikBEJMmpCEREkpyK\nQEQkyakIRESS3P8Hndxx3i0ZKBUAAAAASUVORK5CYII=\n", "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "elu1 = elu(X, 1.0)\n", "\n", "fig = plt.figure()\n", "plt.plot(X, elu1)\n", "plt.xlabel('X')\n", "plt.ylabel('Y')\n", "plt.title('ELU')\n", "fig.savefig('figures/elu.png')\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": 14, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/plain": [ "1.9951645051027873" ] }, "execution_count": 14, "metadata": {}, "output_type": "execute_result" } ], "source": [ "np.mean(elu1)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.0" } }, "nbformat": 4, "nbformat_minor": 2 }