{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## In this notebook, we implement a logistic regression model with a custom loss function. Our loss function is adopted from the traditional logarithmic loss. Its gradient is obtained via automatic differention using the [autograd](https://github.com/HIPS/autograd) package, which \"efficiently computes derivatives of numpy code.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%pylab inline\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "from autograd import grad\n",
    "import autograd.numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Define X, y, initial weights, and epsilon"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X = np.array([\n",
    "    [ 0.3213,  0.4856,  0.2995,  2.5044],\n",
    "    [ 0.3005,  0.4757,  0.2974,  2.4691],\n",
    "    [ 0.5638,  0.8005,  0.3381,  2.3102],\n",
    "    [ 0.5281,  0.6542,  0.3129,  2.1298],\n",
    "    [ 0.3221,  0.5126,  0.3085,  2.6147],\n",
    "    [ 0.3055,  0.4885,  0.289 ,  2.4957],\n",
    "    [ 0.3276,  0.5185,  0.3218,  2.6013],\n",
    "    [ 0.5313,  0.7028,  0.3266,  2.1543],\n",
    "    [ 0.4728,  0.6399,  0.3062,  2.0597],\n",
    "    [ 0.3221,  0.5126,  0.3085,  2.6147]\n",
    "])\n",
    "y = np.array([1., 1., 0., 0., 1., 1., 1., 1., 0., 0.])\n",
    "\n",
    "weights = np.zeros(X.shape[1])\n",
    "eps = 1e-15"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define our linear combinifier, bounded sigmoid, and custom loss function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def wTx(w, x):\n",
    "    return np.dot(x, w)\n",
    "\n",
    "def sigmoid(z):\n",
    "    return 1./(1+np.exp(-z))\n",
    "\n",
    "def logistic_predictions(w, x):\n",
    "    predictions = sigmoid(wTx(w, x))\n",
    "    return predictions.clip(eps, 1-eps)\n",
    "\n",
    "def custom_loss(y, y_predicted):\n",
    "    return -(y*np.log(y_predicted) - (1-y)*np.log(1-y_predicted)**2).mean()\n",
    "\n",
    "def custom_loss_given_weights(w):\n",
    "    y_predicted = logistic_predictions(w, X)\n",
    "    return custom_loss(y, y_predicted)\n",
    "    \n",
    "gradient = grad(custom_loss_given_weights)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Plot y_hat in [0, 1] vs. y=0 and y=1, given our custom loss function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXMAAAEaCAYAAADjQbcAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHv9JREFUeJzt3XuYHFW97vHvj8kVciMkTO6EXAC5X4IgqGkR2MB2ix5B\nDYhX1Oc8HAQ2h5sPkhGDiFvYORzZiCAXcwDdgkpgY7YRGASRQCCEEBJCgAmETIZMkgkJhEky+Z0/\nVg3TGXtmeqa7q7pr3s/zrKerqruq1lR63qxZVbXK3B0REalsuyVdARERKZzCXEQkBRTmIiIpoDAX\nEUkBhbmISAoozEVEUkBhLnkxs1oz+1bS9agEZjbQzB40syYz+23S9ZHeQWEu+fKodJuZ3WlmPypy\nffLZb52ZfTru/QJnAHsDw939SwnsPyczO9zMnjOz98xsoZkdlnSdpHgU5pJmPf4PqED7ACvcfWd3\nVzSzPiWoD2bWD3gA+DUwDLgLeMDM+pZifxI/hXkvY2aXmNl97ZbdaGaz81h9opk9aWbvmtl/m9le\nWdv4nZnVR10Lj5vZgdHy7wBnAZea2WYzeyBHnW42s39rt+wBM7swmr7MzFZH+11uZif04EfP3nZ/\nM5ttZm9H5d+jsMPMRpjZQ2a20czWm9lfs9brsh5m9kPgB8CXop/3GxZcGf2l0GBmd5nZkOjzE81s\np5l908xWAX/Jsc2XzOwzWfN9zayxmy3rDFDl7v/H3be7+/8FDCjoWEoZcXeVXlSAUcAWYGg03wdo\nAI7oYr1aYCUwBRgAPAZcm/X+14E9gL7AvwOLst67A7i6k21/Angza35P4P2orvsDbwKjovcmAJPy\n/FnfAE7Isfxq4ClgRFT+1lo/4FrgZqAqKsdHy/OuBzAT+HXW/DeBV4GJ0TG6v/X9aNlO4E5gINA/\nx/YuAX6TNX86sDhrvgnY2EG5NPrMRcDD7bY7F/jXpL+TKsUpapn3Mu6+FngCODNadAqwzt0XdbUq\ncLu7r3T3D4D/BA7P2u6d7v6eu28HfggcZmaDs9a3Trb9JOBm9olo/gzgqaiuLUB/4CAz6+vub7r7\n6/n9tB06ixDeje7eGNX3nOi9bcBoYKK7t7j736Ll3amHsevPezZwvbvXuft7wBXAl80s+/evxt23\nuntzju3dDfyzmQ2K5s8B5rS+6e7D3H3PDspPo48NAja12+67wGAkFRTmvdNdwFei6a+QFQxdWJs1\nvZUQEJhZlZn9xMxWmtkmQosYQqu3S+7uwG+AGdGiswgBhruvBC4EaoAGM7vXzEbnWd+OjAFWZc2/\nGS0D+DfCXyB/NrPXzOyyItRjdI799QGqs5a91dHK7r6G8NfDGWY2jPAf8N157rvVZmBIu2VDCYEu\nKaAw750eAA41s4OBf6b7wdDeWcBngU+7+1Bg32h5a+s0n5OQ9xLCah/go4SuiLCy+73u/gnCiUUH\nriuwvmsI3RutJkTLcPct7v6/3X0y4Wf619a+8W7Uo/3Pm2t/OwjdWx2t017rf8BnEv5qqW99w8y2\nRP3zucrl0cdeBg5tt81DgaVd7FcqREnOnEt5c/etZnY/cA+wwN1X57lqR10lg4BmYIOZ7QH8uN37\nDcCkLur0gpk1ArcB89z9XQAz2w8YR2iZNgMfdFKPXPqZ2YCs+R2E/ziuNLNno2VXEf11Ep1oXA68\nRmi1tgAt3axH++X3ApeZ2Z+ARsLx+Y277zTL+0f5A3AToTW/y38i7j4o5xq7eiz6Ob4H3AJ8l9BX\n/2i+FZDyppZ573UXcDD5d7HArq3H7Mv+fk3oRngbeAn4e7vP/go4MLpC5PedbP8ewtUV92Qt6084\nKbkOqCd03VwBYGZnm9lLXdT5YcLJ1NZyFTALWAi8GJWF0TIIJ3jnE7olngJucvfHO6tHDu0vibyd\ncJz/Crwe1eP8dp/vVHSe4veEFn5nx7Cj9bcDnwO+Sjgx+lXgc+6+o7vbkvJkobuygzfNxhN+Ufcm\nfOF+6e43mlkNcC7hiw1whbvPK3FdpYiif9vlQLW7b0m6PtI1M/sBMNXdv5p0XaT8dBXmowiXYr0Q\nnUl/jvC/+xeBze5+QzzVlGKKrqK4ARjk7ucmXR/pmpkNJ/z+nePuTyZdHyk/nfaZR5eGrY2mt5jZ\nMmBs9HZ3+i2lTER92g2EK05OaffeFnL/yX9K1iV6EjMz+zbh2v1fK8ilI522zHf5oNlE4HHgIOBi\n4BuE61YXAhe7e1NpqigiIl3J6wRo1MVyH3BB1L96M+Hys8MJJ4OuL1kNRUSkS122zKOBeB4C/uTu\n/zB+R9Rif9DdD2m3PIkBjkREKp67d7sbu9OWuYWLYH8FvJwd5O3ufPs8sKSDCqm4M3PmzMTrUC5F\nx0LHQsei89JTXd00dDzhrrMXzax17I7vAzPM7HDCybI3CDcgiIhIQrq6muVJcrfe/1Sa6oiISE/o\nDtAYZDKZpKtQNnQs2uhYtNGxKFzelyZ2e8NmXqpti4iklZnhPTgBqoG2RCR23RhgLNWK2eBVmItI\nInr7X+7F/g9NfeYiIimgMBcRSQGFuYhICijMRUSyTJw4kUceeSTpanSbwlxEJIuZVeTVNgpzEZEU\nUJiLiOSwbds2LrzwQsaOHcvYsWO56KKL2LZtGwCNjY185jOfYc8992Svvfbik5/85IfrXXfddYwb\nN44hQ4ZwwAEH8Oij8TwzW9eZi4i04+7MmjWLZ555hsWLFwNw+umnM2vWLK6++mquv/56xo8fT2Nj\nIwBPP/00AK+88go33XQTCxcuZNSoUbz55pvs2BHPM7PVMheRsmRWnNJT99xzD1dddRUjRoxgxIgR\nzJw5kzlz5gDQr18/6uvrqauro6qqiuOPPx6AqqoqmpubWbp0Kdu3b2fChAlMmjSpGIejSwpzESlL\n7sUpPbVmzRr22WefD+cnTJjAmjVrALjkkkuYMmUKJ598MpMnT+a6664DYMqUKcyePZuamhqqq6uZ\nMWMG9fX1BR2HfCnMRURyGDNmDHV1dR/Ov/nmm4wZMwaAQYMG8bOf/YzXXnuNuXPncsMNN3zYNz5j\nxgyeeOIJVq1ahZlx2WWXxVJfhbmISA4zZsxg1qxZNDY20tjYyNVXX80555wDwEMPPcTKlStxd4YM\nGUJVVRVVVVWsWLGCRx99lObmZvr378+AAQOoqqqKpb4KcxGRdsyMK6+8kmnTpnHooYdy6KGHMm3a\nNK688koAVq5cyUknncTgwYM57rjjOO+885g+fTrNzc1cccUVjBw5ktGjR9PY2Mi1114bT501nrmI\nxC0aszvpaiSqo2PQ0/HM1TIXEUkBhbmISAoozEVEUkBhLiKSAgpzEZEUUJiLiKSABtoSkURU4pjh\n5UxhLiKx6+3XmJeCullERFJAYS4ikgIKcxGRFFCYi4gk6Pzz4Y47Ct+OwlxEJEENDTBwYOHbUZiL\niCSoqQn23LPw7SjMRUQS1NQEw4YVvh2FuYhIghTmIiIpsHGjwlxEpKK5x9QyN7PxZvaYmS01s5fM\n7HvR8uFmNt/MVpjZn82sCFUREeld3n8f+vaF/v0L31ZXLfPtwEXufhBwLHCemX0EuByY7+77AY9E\n8yIi0g3FapVDF2Hu7mvd/YVoeguwDBgLfBa4K/rYXcDnilMdEZHeY+PG4lyWCN3oMzezicARwAKg\n2t0borcagOriVEdEpPcoZss8ryFwzWwQcD9wgbtvzh6H2N3dzHKOZ1lTU/PhdCaTIZPJFFJXEZFU\naWqClpZaampqC96WdTWusJn1BR4C/uTus6Nly4GMu681s9HAY+5+QLv1XGMWi4h0bM4cmDcP7r67\nbZmZ4e7dfnJHV1ezGPAr4OXWII/MBb4WTX8N+GN3dywi0tsV61Z+6Lqb5XjgK8CLZrYoWnYF8BPg\nP83sW0Ad8MXiVEdEpPeIrc/c3Z+k49b7icWpgohI79TUBKNHF2dbugNURCQhxbqVHxTmIiKJKWaf\nucJcRCQhsd0BKiIipaMwFxFJgURu5xcRkeIqZsu8yztAe7xh3QEqItKhnTvD8LfNzdAn6yLxktwB\nKiIipbF5M+yxx65BXgiFuYhIAorZXw4KcxGRRBSzvxwU5iIiiVCYi4ikgMJcRCQF1GcuIpICapmL\niKSAwlxEJAXUzSIikgJqmYuIpIDCXEQkBRTmIiIpoD5zEZEUUMtcRCQFFOYiIhVuxw7YuhUGDSre\nNhXmIiIxa2qCoUNhtyImsMJcRCRmxe5iAYW5iEjsFOYiIimgMBcRSYFiX2MOCnMRkdipZS4ikgIK\ncxGRFGhqUjeLiEjF27hRLXMRkYqnbhYRkRRQmIuIpEAifeZmdruZNZjZkqxlNWa22swWReWU4lZL\nRCS9kuozvwNoH9YO3ODuR0RlXnGrJSKSXol0s7j7E8DGHG9ZcasiIpJ+LS2wYQMMH17c7RbSZ36+\nmS02s1+ZWZH/jxERSaf6+hDkAwYUd7s9DfObgX2Bw4F64Pqi1UhEJMVWrYKJE4u/3T49Wcnd32md\nNrPbgAdzfa6mpubD6UwmQyaT6cnuRERSo65u1zCvra2ltra24O2au3f9IbOJwIPufkg0P9rd66Pp\ni4Cj3f2sdut4PtsWEelNrrkGtmyBa6/N/b6Z4e7dPifZZcvczO4FpgMjzOwtYCaQMbPDCVe1vAF8\nt7s7FhHpjerqYNq04m+3yzB39xk5Ft9e/KqIiKRfXR2ccUbxt6s7QEVEYtS+z7xY8uoz79GG1Wcu\nIrKLnTth993DHaADB+b+TE/7zNUyFxGJSX19GJOloyAvhMJcRCQmpepiAYW5iEhsFOYiIimgMBcR\nSYG6Othnn9JsW2EuIhKTUo3LAgpzEZHYlLKbRdeZi4jEIJ9rzEHXmYuIlLW1a8PThUpxjTkozEVE\nYlHKLhZQmIuIxEJhLiKSAgpzEZEUUJiLiKSAwlxEJAVKHea6zlxEpMRarzHfsCG8dkbXmYuIlKm1\na2Ho0K6DvBAKcxGREivlmCytFOYiIiW2bBlMnVrafSjMRURKbNEiOPLI0u5DYS4iUmLPP1/6MNfV\nLCIiJdTSEk5+rl4dBtrqiq5mEREpQ6++CtXV+QV5IRTmIiIlFEcXCyjMRURKatEiOOKI0u9HYS4i\nUkJxtcx1AlREpETcYa+9wnXm1dX5raMToCIiZWbVqvCYuHyDvBAKcxGREomriwUU5iIiJRPXyU9Q\nmIuIlIxa5iIiKRDHmCytFOYiIiVQXw/NzTB+fDz7U5iLiJRAa6vcun2RYc90GeZmdruZNZjZkqxl\nw81svpmtMLM/m1mJRx0QEakscZ78hPxa5ncAp7Rbdjkw3933Ax6J5kVEJLJgAUybFt/+8roD1Mwm\nAg+6+yHR/HJgurs3mNkooNbdD2i3ju4AFZFeaft2GDECVq6EkSO7t27cd4BWu3tDNN0AxHB/k4hI\nZViwACZP7n6QF6LgE6BR81tNcBGRyPz5cNJJ8e6zTw/XazCzUe6+1sxGA+/k+lBNTc2H05lMhkwm\n08PdiYhUjvnz4Uc/yu+ztbW11NbWFrzPnvaZ/xRY7+7XmdnlwDB3v7zdOuozF5Fep6kpXFu+bh0M\nGND99UvWZ25m9wJPAfub2Vtm9g3gJ8BJZrYCOCGaFxHp9R57DI47rmdBXoguu1ncfUYHb51Y5LqI\niFS8JPrLQXeAiogUlcJcRKTCvfEGbN4MhxwS/74V5iIiRTJ/Ppx4IuyWQLIqzEVEiiSpLhbQA51F\nRIpix47wrM8XX4SxY3u+HT3QWUQkQfPnw9SphQV5IRTmIiJFcM89cPbZye1f3SwiIgV6773QIn/l\nldDVUgh1s4iIJGTuXDj22MKDvBAKcxGRAiXdxQLqZhERKUhjYxi7fPVqGDy48O2pm0VEJAG/+x2c\nempxgrwQCnMRkQKUQxcLqJtFRKTHVq2Co46CNWugX7/ibFPdLCIiMbvtNvjyl4sX5IVQy1xEpAfe\nfx8mToQnn4T99ivedtUyFxGJ0Zw54dryYgZ5IXr6QGcRkV5r506YPRtuvjnpmrRRy1xEpJvmzYOB\nA2H69KRr0kZhLiLSTTfcABddBNbtnu3S0QlQEZFuWLwYTjstPCKuFFex6ASoiEgMfvYzOO+88rgc\nMZta5iIieVqyJDzjc8UKGDq0NPtQy1xEpMSuuCKUUgV5IXRpoohIHh5/HJYuhfvvT7omuallLiLS\nBXe49FKYNQv690+6NrkpzEVEunD//bBtG8yYkXRNOqYToCIindi2DQ4+GH7+czj55NLvTydARURK\n4NprYf/94aSTkq5J59QyFxHpwNKlkMnAokUwblw8+1TLXESkiFpa4Nxz4eqr4wvyQijMRURyuOkm\n6NsXvvvdpGuSH3WziIi0U1cH06bBU0/FP165ullERIqg9RLEyy8vnwdP5EMtcxGRLBdfDK+8AnPn\nwm4JNHd72jLX7fwiIpEHHoD77oPnn08myAtRUMvczOqAd4EWYLu7fzTrPbXMRaRivPEGHHNMaJEf\ne2xy9UiqZe5Axt03FLgdEZHEbNkCX/hC6CdPMsgLUYw/JMrowUkiIt2zYwd86UtwxBHhUXCVqtAw\nd+AvZrbQzL5djAqJiMTFHc4/PwT6L35RXs/07K5Cu1mOd/d6MxsJzDez5e7+ROubNTU1H34wk8mQ\nyWQK3J2ISPH89KfhWvInngg3CCWhtraW2tragrdTtEsTzWwmsMXdr4/mdQJURMrWL38J11wDf/tb\ned2uH/tNQ2a2u5kNjqb3AE4GlvR0eyIicbn11vCgiUceKa8gL0Qh3SzVwB8sdDL1Ae529z8XpVYi\nIiVy221h8KxHH4UpU5KuTfHoDlAR6TVuvhl+/OMQ5FOnJl2b3HQHqIhIB9zhBz+A3/4Wamth8uSk\na1R8CnMRSbXt2+Hb34Zly8KVKyNHJl2j0lCYi0hqrV8PX/4yDBgQulb22CPpGpVOhQ0lIyKSnxde\ngKOPhsMPhz/8Id1BDmqZi0gK3X03XHgh/Pzn4Vb93kBhLiKpsWULXHAB/PWvoVvlkEOSrlF81M0i\nIqnw7LNhsCz3MB55bwpyUMtcRCpcczP85CfwH/8RHsJ8xhlJ1ygZCnMRqVh//zuce264bvy559Jz\na35PKMxFpOJs3AhXXQX33w+zZ8OZZ1b28LXFoD5zEakYLS1htMOPfCTcDPTSS/DFLyrIQS1zEakQ\njzwCl1wCgwbBvHnh+nFpozAXkbL27LPw/e/DqlVh2Fp1qeSmbhYRKUvPPQef/3woZ54JS5eqS6Uz\nCnMRKStPPQWnnQannw6f+hSsWAHf+U5yj3WrFOpmEZHEtbTAH/8I118Pa9fCpZeG8VT690+6ZpVD\nYS4iiVm/Hu68M9zsM2oUXHwxfO5zUFWVdM0qj8JcRGLlDk8/DbfcAg88AP/yL3DPPXDssUnXrLLp\nsXEiEou1a2HOHLj99tCtcu658M1vwogRSdesvOixcSJSdt59N/SF3303LFgAX/hCeKDyccfpqpRi\nU8tcRIpq82b4r/+C++6Dv/wFPvlJOPvs0J2y++5J16789bRlrjAXkYI1NMBDD4U+8Mcfh49/PIxe\nePrpMHx40rWrLApzEYnNzp2waBE8/HAoy5bBP/0TfPaz4RrxPfdMuoaVS2EuIiX19tuh22T+/FCG\nD4dTTw1l+nTo1y/pGqaDwlxEimrt2tBlUlsbyrp1cMIJcOKJcNJJsO++SdcwnRTmItJjO3eG2+af\negqefDKUdevCyctMJpTDDoPdNABIySnMRSRv69aF0QiffTbcwLNgAQwbBh/7WDh5+fGPw0EHKbyT\noDAXkZzWrg0nK59/PpTnnoOmJpg2DY4+Go45Jtx9OWpU0jUVUJiL9Hpbt4arSl56CZYsgcWLQ2lp\nCV0kRx0FRx4ZypQpanWXK4W5SC+xaRO88gosXx7C++WXQ1m9GqZOhYMPDuWww0IZO1Z3W1YShblI\nirz3Hrz2WigrV4aTk6++Gl7ffRf23x8OOCC8HnQQHHhgaG1rzO/KpzAXqSDbt4eWdF0dvPFGeH39\n9TD9+uuhT3vSpBDQkyfDfvuFMnVqaGmriyS9FOYiZaKlJdze/vbbIbDfemvXsmoVvPNOOOG4774w\ncWIokyaFsu++MGaMAru3UpiLlNi2bSGkGxrCFSL19W2v9fWwZk0oDQ3h7shx40Irevz4MD1+PEyY\nAPvsE8K6j8YslRwU5iLd1NwcnnTT2NhW1q0L5Z132l4bGsLr5s2w996hRV1dHV5Hjw5lzJi2MmqU\nbm2Xnos9zM3sFGA2UAXc5u7XtXtfYS4l19ISTghu2hT6mTduDKV1esOGUDZuDMG9YUN4Xb8+hPle\ne4UycmR4SMKIESGwR44Mpbo6zFdXh8Gj1PUhpRZrmJtZFfAKcCLwNvAsMMPdl2V9RmEeqa2tJZPJ\nJF2NslBbW8v06Rk++AC2bAmt3dbX9uXdd0PZvDmE9aZNuwb3pk3hqo/Bg8Pdi0OHhsBtLcOGhe6O\n4cPDfGtwDx8eXgcPTvaSPX0v2uhYtIn7SUMfBVa6e120898ApwPLOlupt6qUL6p76BfeuhU++CCU\nrVvb5lunt26F999ve81V3ntv17JlS3jdsKGWlpYM/frBHnuEQB00qO11yJAwPXhwmN5773A1x9Ch\nYb61tIb34MGV+/DfSvlexEHHonA9DfOxwFtZ86uBYwqvTjq4h4GLduwI3QDNzeHP+x07/rFs3/6P\nr12Vbdt2LdnLmpt3nW6db53+4IO26ez51tc+fWDgQBgwIPfrwIHhaTGtr9nTI0eGgN599/CaPT1o\nUHi98Ua45prKDWCRctXTMO9W/8mNN8KLL2at7P847b7rdPayfMrOnfm/ti8tLbmXtS7PNZ2rtIZ3\nS0v4871Pn1B27AhPIu/TJ4RYnz7h5o7W+b592+azp/v1a5tvLa3L+vVrm+7fP4Rm9nz//mE++7V/\n/xDKuaYHDgyvpe4THjBAQS5SCj3tMz8WqHH3U6L5K4Cd2SdBzUwd5iIiPRDnCdA+hBOgnwbWAM/Q\n7gSoiIjEp0fdLO6+w8z+F/DfhEsTf6UgFxFJTsluGhIRkfgUfLrLzE4xs+Vm9qqZXdbBZ26M3l9s\nZkcUus9y1dWxMLOzo2Pwopn9zcwOTaKeccjnexF97mgz22Fm/yPO+sUpz9+RjJktMrOXzKw25irG\nJo/fkRFmNs/MXoiOxdcTqGbJmdntZtZgZks6+Uz3ctPde1wIXSwrgYlAX+AF4CPtPnMa8HA0fQzw\ndCH7LNeS57H4GDA0mj6lNx+LrM89CjwEfCHpeif4vRgGLAXGRfMjkq53gseiBri29TgA64E+Sde9\nBMfiE8ARwJIO3u92bhbaMv/w5iF33w603jyU7bPAXQDuvgAYZmbVBe63HHV5LNz97+6+KZpdAIyL\nuY5xyed7AXA+cB+wLs7KxSyfY3EWcL+7rwZw98aY6xiXfI5FPTAkmh4CrHf3HTHWMRbu/gSwsZOP\ndDs3Cw3zXDcPjc3jM2kMsXyORbZvAQ+XtEbJ6fJYmNlYwi/yzdGitJ68yed7MRUYbmaPmdlCMzsn\nttrFK59jcStwkJmtARYDF8RUt3LT7dwsdBDOfH8B218zmcZf3Lx/JjP7FPBN4PjSVSdR+RyL2cDl\n7u5mZvzjdyQt8jkWfYEjCZf67g783cyedvdXS1qz+OVzLL4PvODuGTObDMw3s8PcfXOJ61aOupWb\nhYb528D4rPnxhP9BOvvMuGhZ2uRzLIhOet4KnOLunf2ZVcnyORZHAb8JOc4I4FQz2+7uc+OpYmzy\nORZvAY3uvhXYamZ/BQ4D0hbm+RyL44BrANz9NTN7A9gfWBhLDctHt3Oz0G6WhcBUM5toZv2ALwHt\nfxnnAl+FD+8cbXL3hgL3W466PBZmNgH4PfAVd1+ZQB3j0uWxcPdJ7r6vu+9L6Df/nykMcsjvd+QB\n4ONmVmVmuxNOeL0ccz3jkM+xWE4YjZWoj3h/4PVYa1keup2bBbXMvYObh8zsu9H7t7j7w2Z2mpmt\nBN4DvlHIPstVPscCuArYE7g5apFud/ePJlXnUsnzWPQKef6OLDezecCLwE7gVndPXZjn+b34MXCH\nmS0mNDYvdfcNiVW6RMzsXmA6MMLM3gJmErrbepybumlIRCQF9NwUEZEUUJiLiKSAwlxEJAUU5iIi\nKaAwFxFJAYW5iEgKKMxFRFJAYS69QjRe+IPdXOdrZja6VHUSKSaFuUjHvg6MSboSIvlQmEtFM7Mf\nmtkFWfPXmNn3cnzUgUFm9jszW2Zm/y9rnR+Y2TNmtsTMbomWnQFMA+42s+fNbECpfxaRQijMpdLd\nTtuARLsRBm+ak+NzRniyywXAgcAkM2sdgvjn7v5Rdz8EGGhmn3H3+wgDQ53l7ke6+wel/kFECqEw\nl4rm7quA9WZ2OHAy8HwnQws/4+5rPAxI9ALh8WUAJ5jZ02b2InACIexbpXWcdUmZQsczFykHtxFG\nlasmtNQ70pw13QJURd0nNwFHufvbZjYTyO5S0Uh0UhHUMpc0+APhAdnTCMOrdkdrcK83s0HAmVnv\nbabteZQiZU0tc6l47r7dzB4FNnrHYzo7OVrZ7t5kZrcCLwFrCQ/abnUn8Aszex84Tv3mUs40nrlU\nvOjE53PAGe7+WtL1EUmCulmkopnZgYRnZf5FQS69mVrmkipmdgjw63aLP3D3jyVRH5G4KMxFRFJA\n3SwiIimgMBcRSQGFuYhICijMRURSQGEuIpIC/x+WLTlo16hxBgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x113e8af98>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df = pd.DataFrame(\n",
    "    [(y_hat, custom_loss(False, y_hat)) for y_hat in np.linspace(0, 1, 101)],\n",
    "    columns=['y_hat', 'loss']\n",
    ").plot(x='y_hat', title='y_hat vs. Loss for y=0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW0AAAEaCAYAAADaJgZbAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XmYFNW9//H3l5FFRYyIIqvIooLKoqgRRAZw16uJISou\nUbNdExOXJMblEp0YjPEXt+sTr0nMosYFF2IUNBoFRtEICMoqoIMiCogOIgSUZZjv74/T7TTjMNMz\n093V1f15PU89vVd9u2A+c+bUqVPm7oiISDy0iLoAERFJn0JbRCRGFNoiIjGi0BYRiRGFtohIjCi0\nRURiRKEt2zGzcjP7TtR1xIGZ7WxmE83sUzN7JOp6pDgotKU2TyyNZmb3mtmvMlxPOttdZmajcr1d\nYDSwN9De3c+KYPt1MrM/mtliM9tmZhdEXY9klkJbCkGTf9E0077AW+5e3dgPmtlOWagnaQ7wQ+B1\notkvkkUK7QJlZlea2eO1nrvTzO5I4+M9zOxlM1tvZs+Z2Z4p63jMzFYlugReNLN+iee/D5wD/NzM\n/mNmT9ZR091m9ttazz1pZpcn7l9lZh8ktrvYzEY24aunrru1md1hZisSy+1m1irxWgczm2Rma81s\njZm9lPK5Busws18CvwDOSnzfiywYm2j5rzaz+8ysXeL9Pcys2sy+bWbvAS/Usc4FZnZqyuOWZlZp\nZgMa873d/f/cfQqwqTGfk5hwdy0FuAD7ABuA3ROPdwJWA4Ma+Fw5UAH0BtoAU4GbUl6/ENgVaAnc\nDryR8tpfgRvqWfcwYHnK4z2AzxK1HgAsB/ZJvNYd6Jnmd30XGFnH8zcA/wY6JJZXkvUBNwF3AyWJ\nZWji+bTrAK4H7k95/G3gbaBHYh9NSL6eeK4auBfYGWhdx/quBManPD4dmJvy+FNg7Q6Wn9exvmnA\nt6L+v6gls4ta2gXK3T8k/NB+M/HUicDH7v5GQx8F/uLuFe6+CXgUGJiy3nvdfaO7bwV+CQwws91S\nPm/1rPtlwM1sWOLxaODfiVq3Aa2Bg8yspbsvd/d30vu2O3QOIaQr3b0yUe/5ide2AJ2AHu6+zd1f\nSTzfmDqM7b/vucCt7r7M3TcC1wBnm1nqz1mZu3/u7pvrWN+DwClm1jbx+Hzgb8kX3f0r7r7HDpb/\nl+5OkXhTaBe2+4DzEvfPIyUAGvBhyv3PgbYAZlZiZr8xswozW0do4UJoxTbI3R0YD4xJPHUOIahw\n9wrgcqAMWG1mD5tZpzTr3ZHOwHspj5cnngP4LeEvin+Z2VIzuyoDdXSqY3s7AR1Tnnt/Rx9295WE\nvwZGm9lXCL9oH0xz21IkFNqF7Umgv5kdDJxC8wPgHOA0YJS77w7sl3g+2dpM56DXw4RQ2hc4gtCF\nED7s/rC7DyMc4HPg5mbWu5LQLZHUPfEc7r7B3X/m7r0I3+knyb7rRtRR+/vWtb0qQrfUjj5TW/IX\n7TcJf4WsSr5gZhsS/ed1LVc3sF4pENk8gi0Rc/fPzWwC8BAww90/SPOjO+riaAtsBj4xs12BX9d6\nfTXQs4Ga5phZJfAn4Fl3Xw9gZvsDXQktzc2Eg2j1dbXU1srM2qQ8riL8ghhrZq8lnruOxF8biQN+\ni4GlwHpCt8i2RtZR+/mHgavM7J9AJWH/jHf3arO0v8oTwF2E1vl2vyzcvW2dn6hdlFlLQj99C2r2\ny+bEXzoSc2ppF777gINJv2sEtm8Npg6nu5/w5/8KYAHwaq33/hnolxiR8fd61v8QMDJxm9SacHDw\nY2AVocvlGgAzO9fMFjRQ8zOEg5rJ5TpgHDALmJdYZiWeg3Cg9XngP4SDlXe5+4v11VGH2kMN/0LY\nzy8B7yTq+HGt99crcRzh74QWe337sD7PJ7b9VeCPifvD6v2ExIal88vXzJZR0xrZ6u5HZLkuyRAz\n60ZoUXZ09w1R1yMNM7NfAH3c/VtR1yL5J93uEQdK3f2TbBYjmZUYtfBT4GEFdjyYWXvC0MHzG3qv\nFKfGdI80pn9RIpboc14PjCKMJ059bUcHtIZGUqwAYGbfI4w4+ae7vxx1PZKf0u0eeQdYR+ge+YO7\n35PtwkRE5MvS7R4Z6u6rzGwv4HkzW+zu07JZmIiIfFlaoZ0cK+ruH5vZE4TxtdMAzEzDiEREmsDd\nG93t3GCftpntkjxNOdFPejwwv9aGtbhz/fXXR15DvizaF9oX2hf1L02VTku7I/BE4uSAnYAH3f1f\nTd6iiIg0WYOh7e7vkjJhkIiIREdnRGZQaWlp1CXkDe2LGtoXNbQvmi+tIX/1rsDMm7sOEZFiY2Z4\nEw5EasIoEcmaRkyUVdAy2bBVaItIVhX7X+KZ/sWlPm0RkRhRaIuIxIhCW0QkRhTaIlKUevToweTJ\nk6Muo9EU2iJSlMwslqNbFNoiIjGi0BaRorZlyxYuv/xyunTpQpcuXbjiiivYsmULAJWVlZx66qns\nscce7LnnnhxzzDFffO7mm2+ma9eutGvXjgMPPJApU6bkpF6N0xaRouXujBs3jpkzZzJ37lwATj/9\ndMaNG8cNN9zArbfeSrdu3aisrARg+vTpACxZsoS77rqLWbNmsc8++7B8+XKqqqpyUnNGW9rLl8Pa\ntZlco4gUOrPMLE310EMPcd1119GhQwc6dOjA9ddfz9/+9jcAWrVqxapVq1i2bBklJSUMHRquyFdS\nUsLmzZtZuHAhW7dupXv37vTs2TMTu6NBGQ3ta6+FiRMzuUYRKXTumVmaauXKley7775fPO7evTsr\nV64E4Morr6R3794cf/zx9OrVi5tvvhmA3r17c8cdd1BWVkbHjh0ZM2YMq1atatZ+SFdGQ7tLF/jg\ng0yuUUQkuzp37syyZcu+eLx8+XI6d+4MQNu2bbnllltYunQpTz31FLfddtsXfddjxoxh2rRpvPfe\ne5gZV111VU7qzWhod+0KK1Zkco0iItk1ZswYxo0bR2VlJZWVldxwww2cf/75AEyaNImKigrcnXbt\n2lFSUkJJSQlvvfUWU6ZMYfPmzbRu3Zo2bdpQUlKSk3ozHtpqaYtIXJgZY8eOZfDgwfTv35/+/fsz\nePBgxo4dC0BFRQXHHXccu+22G0OGDOGSSy5h+PDhbN68mWuuuYa99tqLTp06UVlZyU033ZSbmjM5\nn/Zrr8HFF8Ps2ZkoTUTiLjFndNRlRGpH+6Cp82lnvE9b3SMiItmT0Zb2tm2w886wYQO0apWJ8kQk\nztTSzvOWdkkJ7LMPJEbLiIhIhmX8NHZ1kYiIZE/GQ1sjSEREskehLSISIxmfMEon2IhIqjjOWZ3P\nMh7aXbrAjBmZXquIxFGxjxzJBnWPiIjEiEJbRCRGMnpyDcCWLdC2LXz+eRi3LSIiX5YXJ9dAOBNy\njz3go48yvWYREcnKNSLVRSIikh1ZC20N+xMRybyshLauYCMikh3qHhERiRF1j4iIxIha2iIiMZJW\naJtZiZm9YWYT03m/+rRFRLIj3Zb2ZcCbQFpn4iTn1Na0AyIimdVgaJtZV+Bk4E9AWmfvtG0LrVvD\n2rXNrE5ERLaTTkv7duBKoLoxK1YXiYhI5tU7NauZnQp85O5vmFnpjt5XVlb2xf3S0lJKS0u/OBjZ\nv3+mShURia/y8nLKy8ubvZ56J4wys18D5wNVQBugHTDB3b+V8h6vax3f/S4ceSR873vNrlFEpOBk\nZcIod7/W3bu5+37A2cCU1MCuj4b9iYhkXmPHaac9HkR92iIimZd2aLv7i+5+Wrrv79ED3nmnSTWJ\niMgOZPwiCEmrV0PfvrBmDei6niIi28ubiyAkdewYxmqri0REJHOyFtoAAwbAnDnZ3IKISHHJemjP\nnZvNLYiIFJeshvbAgQptEZFMUveIiEiMZG30CEBVFey+exhJ0rZtszYjIlJQ8m70CMBOO0G/fjB/\nfja3IiJSPLIa2qAuEhGRTMpJaOtgpIhIZii0RURiJKsHIgHWrQuTR61bByUlzdqUiEjByMsDkRBG\nj+y1Fyxdmu0tiYgUvqyHNqiLREQkU3IS2gMHagSJiEgmqKUtIhIjCm0RkRjJSWj36AEbN8KHH+Zi\nayIihSsnod2iBQwfDlOm5GJrIiKFKyehDTBqFEyenKutiYgUppyHdjPP5RERKWo5C+0DD4QtW+Dd\nd3O1RRGRwpOz0DaDkSPVRSIi0hw5C21Qv7aISHNlfcKoVO+9B4cfHob+tcjprwsRkfyStxNGpdp3\nX2jXDhYuzOVWRUQKR87bu+oiERFpOoW2iEiM5LRPG+Djj6FPH6isDBf+FREpRrHo04ZwQYQePeC1\n13K9ZRGR+ItkDMcJJ8CkSVFsWUQk3iIJ7W9+Ex57TKe0i4g0ViShfdhhUFWlObZFRBorktA2gzPP\nhEcfjWLrIiLxFdl5icnQVheJiEj6GgxtM2tjZjPMbI6ZvWlmN2Viw4MGhds33sjE2kREikODoe3u\nm4AR7j4Q6A+MMLOjm7thdZGIiDReWt0j7v5Z4m4roAT4JBMbVxeJiEjjpBXaZtbCzOYAq4Gp7v5m\nJjY+YEA4K3L27EysTUSk8KXb0q5OdI90BY4xs9JMbDzZRfLII5lYm4hI4WvU7B/uvs7MngYGA+XJ\n58vKyr54T2lpKaWlpWmv85xz4Nhj4de/hpYtG1ONiEh8lJeXU15e3uz1NDhhlJl1AKrc/VMz2xl4\nDvilu09OvN6oCaPqcswxcOmlMHp0s1YjIhIb2ZwwqhMwJdGnPQOYmAzsTPnhD+HuuzO5RhGRwpTz\nqVnrsmULdO8O5eXhqu0iIoUuNlOz1qVVK/j2t+H3v4+6EhGR/JYXLW2AZcvCRFLLl8OuuzZ7dSIi\neS3WLW0IF0YYMgTGj4+6EhGR/JU3oQ3wgx/ogKSISH3yKrRPOAE+/RSmTYu6EhGR/JRXoV1SAldf\nDePGRV2JiEh+ypsDkUlbtkDv3vD443DEERlbrYhIXon9gcikVq3gqqvU2hYRqUvetbQBPv8cevWC\nZ56BgQMzumoRkbxQMC1tgJ13hp/9DG68MepKRETyS162tAE2boSePWHqVOjXL+OrFxGJVEG1tCGc\nFfnTn8LYsVFXIiKSP/K2pQ2waVOYQOq++2D48KxsQkQkEgXX0gZo0wZ+8xv4yU+gujrqakREopfX\noQ1w1llhGOADD0RdiYhI9PK6eyTp1VfDtSQXL9YMgCJSGAqyeyTpqKNg6FC45ZaoKxERiVYsWtoQ\n5tsePBimTw+nuYuIxFlBt7QhzLd99dVw8cWQg98RIiJ5KTahDXD55bB2Ldx/f9SViIhEIzbdI0mv\nvw4nnQTz58Pee+dssyIiGdXU7pHYhTbAlVfCypXw4IM53ayISMYUVWh/9hkccgjceit87Ws53bSI\nSEYUVWhDGLv99a+H7pLOnXO+eRGRZin40SO1HXVUuBDwBRfoFHcRKR6xDW2A//mf0FVy++1RVyIi\nkhux7R5JevfdcC3Jf/0LBg2KrAwRkUYpuu6RpP32gzvvhNGjwxhuEZFCFvuWdtJll0FFBUycCC1i\n/6tIRApd0ba0k265Bdavh1/9KupKRESyp2Ba2gCrVsHhh8Mf/gCnnBJ1NSIiO1b0LW2ATp3g0Ufh\nootg4cKoqxERybyCCm2AIUPgtttCS/vDD6OuRkQkswoutAHOOy+0tk87LYzjFhEpFAXVp53KHS68\nMBycfPxxKCmJuiIRkRrq067FDO65J4T2JZfowgkiUhgaDG0z62ZmU81soZktMLNLc1FYJrRqBf/4\nR5hU6pproq5GRKT5GuweMbN9gH3cfY6ZtQVmA19z90WJ1/OyeyTVmjUwfHjo67766qirERFpevfI\nTg29wd0/BD5M3N9gZouAzsCiRlcZkT33DHOTDBsGbdvCj34UdUUiIk3TYGinMrMewCBgRjaKyabO\nneGFF2DkyNC//eMfR12RiEjjpR3aia6Rx4HL3H1D9krKnv32g6lTQ3Bv2xYuFCwiEidphbaZtQQm\nAA+4+z9qv15WVvbF/dLSUkpLSzNUXub16AHl5TBiBFRVwc9+FnVFIlIMysvLKS8vb/Z60jkQacB9\nwBp3v6KO1/P+QGRd3n8fjj0WzjwTbrghDBEUEcmVrF0j0syOBl4C5gHJN1/j7s8mXo9laAN8/DGc\ndBIMHgx33aUTcEQkd4ruwr6Zsn59uEBw+/bwwAPQunXUFYlIMdAZkU3Urh08/XS4f9xxYUy3iEi+\nKvrQBmjTBh55BI48MswSWFERdUUiInVTaCe0aAG//S1ccQUcfTS8/HLUFYmIfJlCu5aLL4b77oMz\nzoA//jHqakREtlf0ByJ35K234PTTobQU/vd/w+RTIiKZogORGbb//jBjBqxYAaNGwcqVUVckIqLQ\nrle7dmFq1+OPD2O5p0yJuiIRKXbqHknT5Mlw/vnwwx/CtdeGA5ciIk2lk2tyYOVKOPvscALO/feH\nq7+LiDSF+rRzoHPn0EUybBgMGgQTJ0ZdkYgUG7W0m+iVV+Dcc+Hkk8P47l13jboiEYkTtbRzbOhQ\nmDMHNmyAgQPh3/+OuiIRKQZqaWfA3/8eDlBeeCGUlYXT4kVE6qOWdoTOOAPmzYO33w6t7ldeiboi\nESlUamln2IQJ4fqTo0fDjTfCbrtFXZGI5CO1tPPEN74BCxaEeboPOiicnCMikilqaWdReXmYgOqA\nA+DOO2HffaOuSETyhVraeai0FObODafAH3ZY6C7ZtCnqqkQkzhTaWda6NfziFzBrFsyeDQcfDJMm\ngf44EZGmUPdIjj33HFx2Wegque220O8tIsVH3SMxccIJMH8+nHIKjBgBP/gBfPRR1FWJSFwotCPQ\nsiVceiksXhwurtCvH4wbBxs3Rl2ZiOQ7hXaE2rcPV8WZMSMME9x//3CJs61bo65MRPKVQjsP9OoF\n48eHMd2PPQZ9+8JDD0F1ddSViUi+0YHIPDRlSrjQwmefhblMvvY1XXRBpNDoIggFxj0MDSwrg23b\n4PrrQ3hbo/+JRSQfKbQLlHu42EIyvMeODRNUlZREXZmINIdCu8C5w9NPh1Em69bBNdfAmDFhJIqI\nxI9Cu0i4hz7vG2+EpUvhpz+F73xHV84RiRudXFMkzGDUqBDcjz0GL74I++0X+rx1ko5I4VNox9gR\nR4T5u6dNg9Wrw2yC3/8+LFoUdWUiki0K7QJwwAHw+9/DkiXhivEjRsBJJ4V5TtRzJVJY1KddgDZt\nCifr3H47bNkCl1wCF1ygq+iI5BMdiJQvcYeXXoLf/Q4mT4Zzzw0XIO7bN+rKREQHIuVLzGD48HDA\ncu5c2H13GDkydJ88+mhohYtIvKilXWS2bAlznNx9dzhgeeGF8N3vQu/eUVcmUlyy1tI2s7+Y2Woz\nm9+00iSftGoFZ54JU6eG4YJVVTBkSBhG+NBDuhyaSL5rsKVtZsOADcD97n5IHa+rpR1zmzfDk0/C\nn/8cLol29tlw0UVw6KGa60QkW7J6INLMegATFdqF77334N57w7LbbqH75NxzoWPHiAsTKTA6ECkZ\nse++4ezKpUvhzjvDAcwDDgiXRxs/Hj7/POoKRYrbTplYSVlZ2Rf3S0tLKS0tzcRqJUItWkBpaVg2\nboQnnoC//jVc0/L00+G888IoFM02KJKe8vJyysvLm70edY9Io6xaFVrcDz4IK1fCWWeFPvAjjlD/\nt0hjqE9bcm7JEnj44bBs3RoC/KyzYMAABbhIQ7IW2mb2MDAc2BP4CLjO3f+a8rpCu8i5w5w58Mgj\nYUkOKxw9Gvr3V4CL1EWnsUtecIdZs0J4T5gQLtIwejR84xsaQiiSSqEteccdXn89nEY/YULoQjnj\nDPj618MJPTqIKcVMoS15zR0WLgzh/cQT4SDmaaeFixWPGgU77xx1hSK5pdCWWHn33XAW5hNPhP7w\nUaNCiJ9yCuy1V9TViWSfQltia80aeOaZEOIvvAD9+sF//ReceiocfLD6waUwKbSlIGzeHCaymjQJ\nJk6E6mo4+eTQAh85EnbZJeoKRTJDoS0Fxz1MH/v002GZPTscwDzppLDsv79a4RJfCm0peOvWhe6T\nf/4Tnn02DCc88cSwjBgB7dpFXaFI+hTaUlSSo1Geey4E+PTpMGgQHH88HHccDB6sIYWS3xTaUtQ+\n+wymTQsh/vzzsGJFaH0fe2xYevdWV4rkF4W2SIpVq0JXyuTJ4bZFixDeI0eGpXPnqCuUYqfQFtkB\n9zC51QsvhMuslZfD3nuHlviIEWH6WY0Nl1xTaIukadu2cHGHqVPDMm0adOtWM3/48OEKcck+hbZI\nE1VVwRtvhBb4iy/Cyy+H7pPhw+GYY8LSpUvUVUqhUWiLZEiyJf7iizUhvvvuMGxYzdKnjw5sSvMo\ntEWypLo6nOTz0kshwKdNC2duHn00DB0abgcNCuPGRdKl0BbJoeXLQ3i/8kpY3nkHDjssnLE5ZAgc\ndRTsuWfUVUo+U2iLRGjdOpgxoybEZ86ETp1qAvyoo8JEWDrhR5IU2iJ5ZNs2WLAAXn01LNOnw4cf\nwuGHw5FHwle/Gm733jvqSiUqCm2RPLdmTQjvGTNqlvbtw5XsjzwyLIMG6YIQxUKhLRIz1dXw1lsh\nvGfODLdvvgkHHBBa5MnloIN0kLMQKbRFCsCmTWG44cyZ8NprYVm+HAYMCJNgHX54uN1/f/WPx51C\nW6RArV8f5hKfPTtc6f611+Cjj0JXymGH1SwK8nhRaIsUkbVrw5XuZ80Kt7Nnw+rVoUU+aBAcemhY\n+vVT10q+UmiLFLlPPw2n47/xRgjy11+HZcugb98Q5IMGwcCBIdjbto26WlFoi8iXbNwI8+aFIJ8z\nJ9wuXAhdu4YAT4b4gAFhfhWdmp87Cm0RSUtVFSxeHA54zpkTlrlzw9jyZIAPGAD9+4fulTZtoq64\nMCm0RaTJ3EOf+Ny5YZk3L9xWVEDPnnDIISHEk7fdu6tV3lwKbRHJuM2bQ6t83rywzJ8flg0b4OCD\nQ4in3mq+lfQptEUkZ9asCafpz5+//e0uu4STgQ4+OCwHHRSWdu2irjj/KLRFJFLu8MEHIcAXLgzL\nggVhWtv27WsCvF+/cNu3b5invFgptEUkL1VXh6GHySBftKjmdo89aoK8b9+w9OtXHN0sCm0RiZXq\n6nCK/ptvhmXRoprbVq1CgB94YE2Y9+0bhiq2aBF15Zmh0BaRguAOq1aF8E4uixeH5dNPw4RaBx64\n/dKnT/xmR1Roi0jBW7cOliypCfHFi8PjpUthn31CoNdeunTJz9a5QltEilZVVeg3X7Jk++Wtt0LQ\n9+kTJtSqvbRvH13NWQttMzsRuAMoAf7k7jfXel2hLSJ5a/36EN5vvx1ulywJ95csCX3nffrUhHry\nfp8+sNtu2a0rK6FtZiXAEuBYYAXwGjDG3RelvEehnVBeXk5paWnUZeQF7Ysa2hc18mlfuIcpblMD\n/e23w7J0aQjtPn2gd+/tb3v1ysy486aG9k4NvH4EUOHuyxIbGQ+cDiyq70PFKp/+Q0ZN+6KG9kWN\nfNoXZtCxY1iGDdv+tepqWLkynMZfURGCfPz4EOYVFWGWxN69w9KrV81tr15huGI2T/FvKLS7AO+n\nPP4AODJ75YiIRK9FizC8sGtXqP07Jjm6JRngFRXw1FM19yHM15IM8eTSsyd069b8C1U0FNrq9xAR\nSWEGnTuHpXYL3R0++SQEenJ59VV44AF45x34yU/giiuauf0G+rS/CpS5+4mJx9cA1akHI81MwS4i\n0gTZOBC5E+FA5ChgJTCTWgciRUQkd+rtHnH3KjP7EfAcYcjfnxXYIiLRafbJNSIikjtpn9xpZiea\n2WIze9vMrtrBe+5MvD7XzAZlrsz80tC+MLNzE/tgnpm9Ymb9o6gzF9L5f5F43+FmVmVmZ+SyvlxK\n82ek1MzeMLMFZlae4xJzJo2fkQ5m9qyZzUnsiwsjKDPrzOwvZrbazObX857G5aa7N7gQukYqgB5A\nS2AO0LfWe04GnkncPxKYns6647akuS+OAnZP3D+xmPdFyvumAJOAb0Rdd4T/L74CLAS6Jh53iLru\nCPdFGXBTcj8Aa4Cdoq49C/tiGDAImL+D1xudm+m2tL84ycbdtwLJk2xSnQbcB+DuM4CvmFnHNNcf\nJw3uC3d/1d3XJR7OALrmuMZcSef/BcCPgceBj3NZXI6lsy/OASa4+wcA7l6Z4xpzJZ19sQpInlfY\nDljj7lU5rDEn3H0asLaetzQ6N9MN7bpOsumSxnsKMazS2RepvgM8k9WKotPgvjCzLoQf2LsTTxXq\nQZR0/l/0Adqb2VQzm2Vm5+esutxKZ1/cAxxkZiuBucBlOaot3zQ6Nxs6uSYp3R+02mMOC/EHNO3v\nZGYjgG8DQ7NXTqTS2Rd3AFe7u5uZ8eX/I4UinX3REjiUMIR2F+BVM5vu7m9ntbLcS2dfXAvMcfdS\nM+sFPG9mA9z9P1muLR81KjfTDe0VQLeUx90IvxHqe0/XxHOFJp19QeLg4z3Aie5e359HcZbOvjgM\nGB/ymg7ASWa21d2fyk2JOZPOvngfqHT3z4HPzewlYABQaKGdzr4YAtwI4O5Lzexd4ABgVk4qzB+N\nzs10u0dmAX3MrIeZtQLOAmr/0D0FfAu+OJPyU3dfneb646TBfWFm3YG/A+e5e0UENeZKg/vC3Xu6\n+37uvh+hX/sHBRjYkN7PyJPA0WZWYma7EA48vZnjOnMhnX2xmDB7KIk+3AOAd3JaZX5odG6m1dL2\nHZxkY2b/nXj9D+7+jJmdbGYVwEbgomZ8kbyVzr4ArgP2AO5OtDC3uvsRUdWcLWnui6KQ5s/IYjN7\nFpgHVAP3uHvBhXaa/y9+DfzVzOYSGo8/d/dPIis6S8zsYWA40MHM3geuJ3STNTk3dXKNiEiM5OGV\n00REZEcU2iIiMaLQFhGJEYW2iEiMKLRFRGJEoS0iEiMKbRGRGFFoS8FJzFk9sZGfucDMOmWrJpFM\nUWiLBBcCnaMuQqQhCm2JDTP7pZldlvL4RjO7tI63OtDWzB4zs0Vm9kDKZ35hZjPNbL6Z/SHx3Ghg\nMPCgmb1uZm2y/V1EmkqhLXHyF2om12lBmIjob3W8zwhXC7kM6Af0NLPk9Li/c/cj3P0QYGczO9Xd\nHydMcnSZrQD7AAABEUlEQVSOux/q7puy/UVEmkqhLbHh7u8Ba8xsIHA88Ho9097OdPeVHibXmUO4\n9BXASDObbmbzgJGEUE8q1Lm+pYCkO5+2SL74E2EmtI6ElveObE65vw0oSXR73AUc5u4rzOx6ILUr\nRLOnSd5TS1vi5gnCxZIHE6b+bIxkQK8xs7bAN1Ne+w811ywUyVtqaUusuPtWM5sCrPUdzyvs1NFq\ndvdPzeweYAHwIeGiy0n3Ar83s8+AIerXlnyl+bQlVhIHIGcDo919adT1iOSaukckNsysH+F6ii8o\nsKVYqaUtsWVmhwD313p6k7sfFUU9Irmg0BYRiRF1j4iIxIhCW0QkRhTaIiIxotAWEYkRhbaISIz8\nf6DtyK/EgaKHAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x116c9f160>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df = pd.DataFrame(\n",
    "    [(y_hat, custom_loss(True, y_hat)) for y_hat in np.linspace(0, 1, 101)],\n",
    "    columns=['y_hat', 'loss']\n",
    ").plot(x='y_hat', title='y_hat vs. Loss for y=1')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Gradient descent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 0    | Loss: 0.6081\n",
      "Iteration 1000 | Loss: 0.5716\n",
      "Iteration 2000 | Loss: 0.5478\n",
      "Iteration 3000 | Loss: 0.5315\n",
      "Iteration 4000 | Loss: 0.5202\n",
      "Iteration 5000 | Loss: 0.5122\n",
      "Iteration 6000 | Loss: 0.5065\n",
      "Iteration 7000 | Loss: 0.5024\n",
      "Iteration 8000 | Loss: 0.4994\n",
      "Iteration 9000 | Loss: 0.4972\n"
     ]
    }
   ],
   "source": [
    "for i in range(10000):\n",
    "    if i % 1000 == 0:\n",
    "        print('Iteration %-4d | Loss: %.4f' % (i, custom_loss_given_weights(weights)))\n",
    "    weights -= gradient(weights) * .01"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}