{ "cells": [ { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "# 09 - Introduction to Neural Networks\n", "\n", "by [Fabio A. González](http://dis.unal.edu.co/~fgonza/), Universidad Nacional de Colombia\n", "\n", "version 1.0, June 2018\n", "\n", "## Part of the class [Applied Deep Learning](https://github.com/albahnsen/AppliedDeepLearningClass)\n", "\n", "\n", "This notebook is licensed under a [Creative Commons Attribution-ShareAlike 3.0 Unported License](http://creativecommons.org/licenses/by-sa/3.0/deed.en_US). \n", "\n", "\n", "To run this notebook you need to download Pybrain (https://github.com/pybrain/pybrain) and copy the `pybrain` folder to the same folder where this notebook is.\n", "\n" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "slideshow": { "slide_type": "skip" } }, "outputs": [], "source": [ "\n", "import pybrain\n", "import numpy as np\n", "import pandas as pd\n", "from matplotlib import pyplot as plt\n", "\n", "%matplotlib inline" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "class": "center", "slide_type": "slide" } }, "source": [ "## Artificial Neuron\n", "\n", "\n", "\n", "$$o_j^{(n)} = \\varphi\\left(\\sum_{i\\; in\\; layer (n-1)}w_{ij}o_i^{(n-1)} \\right)$$" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "subslide" } }, "source": [ "## Step activation function\n", "" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "subslide" } }, "source": [ "## Logistic activation function\n", "\n", "$$\\varphi(x) = \\frac{1}{1 - e^{-(x-b)}}$$\n", "\n", "" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "### Question: How to program an artificial neuron to calculate the *and* function?" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "fragment" } }, "source": [ "
\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "
$X$$Y$$X$ and $Y$
000
010
100
111
" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "subslide" } }, "source": [ "## AND Neural Network\n", "\n", "" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "slideshow": { "slide_type": "fragment" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[ 0.453198 -1.24483399 -0.91415624]\n" ] } ], "source": [ "from pybrain.tools.shortcuts import buildNetwork\n", "net = buildNetwork(2, 1, outclass=pybrain.SigmoidLayer)\n", "print(net.params)\n" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "slideshow": { "slide_type": "subslide" } }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
XYoutput
00.00.00.611
10.01.00.387
21.00.00.312
31.01.00.154
\n", "
" ], "text/plain": [ " X Y output\n", "0 0.0 0.0 0.611\n", "1 0.0 1.0 0.387\n", "2 1.0 0.0 0.312\n", "3 1.0 1.0 0.154" ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "def print_pred2(dataset, network):\n", " df = pd.DataFrame(dataset.data['sample'][:dataset.getLength()],columns=['X', 'Y'])\n", " prediction = np.round(network.activateOnDataset(dataset),3)\n", " df['output'] = pd.DataFrame(prediction)\n", " return df\n", "\n", "from pybrain.datasets import UnsupervisedDataSet, SupervisedDataSet\n", "D = UnsupervisedDataSet(2) # define a dataset in pybrain\n", "D.addSample([0,0])\n", "D.addSample([0,1])\n", "D.addSample([1,0])\n", "D.addSample([1,1])\n", "print_pred2(D, net)" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "subslide" } }, "source": [ "## AND Neural Network\n", "" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "slideshow": { "slide_type": "fragment" } }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
XYoutput
00.00.00.0
10.01.00.0
21.00.00.0
31.01.00.0
\n", "
" ], "text/plain": [ " X Y output\n", "0 0.0 0.0 0.0\n", "1 0.0 1.0 0.0\n", "2 1.0 0.0 0.0\n", "3 1.0 1.0 0.0" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "net.params[:] = [ -150, -100, 100]\n", "print_pred2(D, net)" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "### Question: How to program an artificial neuron to calculate the *xor* function?\n", "
\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "
$X$$Y$$X$ xor $Y$
000
011
101
110
" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "subslide" } }, "source": [ "## Plotting the NN Output" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "slideshow": { "slide_type": "-" } }, "outputs": [], "source": [ "def plot_nn_prediction(N): \n", " # a function to plot the binary output of a network on the [0,1]x[0,1] space\n", " x_list = np.arange(0.0,1.0,0.025)\n", " y_list = np.arange(1.0,0.0,-0.025) \n", " z = [0.0 if N.activate([x,y])[0] <0.5 else 1.0 for y in y_list for x in x_list]\n", " z = np.array(z)\n", " grid = z.reshape((len(x_list), len(y_list)))\n", " plt.imshow(grid, extent=(x_list.min(), x_list.max(), y_list.min(), y_list.max()),cmap=plt.get_cmap('Greys_r'))\n", " plt.show()" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "subslide" } }, "source": [ "## Plotting the NN Output" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "scrolled": false, "slideshow": { "slide_type": "-" } }, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQIAAAD8CAYAAACcoKqNAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvFvnyVgAAC5ZJREFUeJzt3FGIXOd9hvHntVQl1HXSEClQtFKswBpnYwpqF+GSCzskLWtdSDchSGCKi4kgiVNoSkHFpQ3KnUsbKKhNtxCcBlJFyUWzBBtBg0yCiVLtYteNZFS2ilMtKrWTOIZiEkX034uZuuPR7M5Zec7MjvT8YGDOnE9nXo9H737nO2eVqkLS7e2OSQeQNHkWgSSLQJJFIAmLQBIWgSQaFEGSLyZ5Ocn319mfJH+VZDXJC0l+Y/QxJbWpyYzgSWBhg/0PAbPdxzHgb956LEnjNLQIqurbwE82GHIY+PvqOAf8apJfG1VASe3bPoJj7Aau9GyvdV/7z/6BSY7RmTVw5513/ua99947grfXtFtZWZl0hFvZj6pq17BBoyiCDHht4H3LVbUILALMz8/X8vLyCN5e0y4Z9BXSiPywyaBRXDVYA/b0bM8AV0dwXEljMooiWAJ+t3v14H7gtaq64bRA0tY19NQgyT8ADwI7k6wBfwb8EkBVfQF4CjgIrAKvA7/XVlhJ7RhaBFV1dMj+Aj41skS6ZbkWsHV5Z6Eki0CSRSAJi0ASFoEkRnNnoXQDrxBMF2cEkiwCSRaBJCwCSVgEkrAIJGERSMIikIRFIAmLQBLeYqxN8LbhW5czAkkWgSSLQBIWgSRcLNQ6XBi8vTgjkGQRSLIIJGERSMIikIRXDW57Xh0QOCOQhEUgCYtAEhaBJFwsvK24MKj1OCOQZBFIsggk0bAIkiwkuZRkNcnxAfv3Jjmb5LkkLyQ5OPqoktoytAiSbANOAg8Bc8DRJHN9w/4EOF1V+4EjwF+POqik9jSZERwAVqvqclVdA04Bh/vGFPCO7vN3AldHF1GblWTgQ1pPkyLYDVzp2V7rvtbrs8DDSdaAp4BPDzpQkmNJlpMsv/LKKzcRV1IbmhTBoB8l1bd9FHiyqmaAg8CXk9xw7KparKr5qprftWvX5tNKakWTIlgD9vRsz3Dj1P9R4DRAVX0XeDuwcxQBJbWvSRGcB2aT7Euyg85i4FLfmP8APgyQ5P10isC5vzQlht5iXFXXkzwGnAG2AV+sqgtJTgDLVbUE/CHwd0n+gM5pwyNV1X/6oBa4CKhRaPS7BlX1FJ1FwN7X/rTn+UXgg6ONJmlcvLNQkkUgySKQhP8ewdRwUVBtckYgySKQZBFIwiKQhEUgCa8abEleIdC4OSOQZBFIsggkYRFIwsXCiXJRUFuFMwJJFoEki0ASFoEkLAJJeNVgbLxCoK3MGYEki0CSRSAJi0ASLhaOnIuCmkbOCCRZBJIsAklYBJKwCCThVYO3xCsEulU4I5BkEUiyCCTRsAiSLCS5lGQ1yfF1xnwsycUkF5J8ZbQxJbVp6GJhkm3ASeC3gTXgfJKlqrrYM2YW+GPgg1X1apL3tBV4UlwY1K2syYzgALBaVZer6hpwCjjcN+bjwMmqehWgql4ebUxJbWpSBLuBKz3ba93Xet0D3JPk2STnkiyMKqCk9jW5j2DQnLgGHGcWeBCYAb6T5L6q+umbDpQcA44B7N27d9NhJbWjyYxgDdjTsz0DXB0w5htV9Yuq+gFwiU4xvElVLVbVfFXN79q162YzSxqxJkVwHphNsi/JDuAIsNQ35h+BDwEk2UnnVOHyKINKas/QIqiq68BjwBngReB0VV1IciLJoe6wM8CPk1wEzgJ/VFU/bit0m5IMfEi3slT1n+6Px/z8fC0vL0/kvTfiX3rdYlaqan7YIO8slGQRSLIIJHGb/3sErgdIHc4IJFkEkiwCSVgEkrAIJHGbXDXw6oC0MWcEkiwCSRaBJCwCSdyCi4UuDEqb54xAkkUgySKQhEUgiSleLHRRUBodZwSSLAJJFoEkLAJJWASSmJKrBl4hkNrljECSRSDJIpCERSCJLbZY6KKgNBnOCCRZBJIsAklYBJKwCCTRsAiSLCS5lGQ1yfENxn00SSWZH3bMlZUVkrzpIWkyhhZBkm3ASeAhYA44mmRuwLi7gN8HvjfqkJLa1WRGcABYrarLVXUNOAUcHjDuc8ATwM9GmE/SGDQpgt3AlZ7tte5rb0iyH9hTVd/c6EBJjiVZTrK86aSSWtPkzsJBJ+/1xs7kDuDzwCPDDlRVi8Bi98/VkOGSxqTJjGAN2NOzPQNc7dm+C7gPeCbJS8D9wFKTBUNJW0OTIjgPzCbZl2QHcARY+r+dVfVaVe2sqrur6m7gHHCoqpz+S1NiaBFU1XXgMeAM8CJwuqouJDmR5FDbASW1L1WTOVV3jUAai5WqGnqa7p2FkiwCSRaBJCwCSVgEkrAIJGERSMIikIRFIAmLQBIWgSQsAklYBJKwCCRhEUjCIpCERSAJi0ASFoEkLAJJWASSsAgkYRFIwiKQhEUgCYtAEhaBJCwCSVgEkrAIJGERSMIikIRFIAmLQBIWgSQsAkk0LIIkC0kuJVlNcnzA/s8kuZjkhSTfSvLe0UeV1JahRZBkG3ASeAiYA44mmesb9hwwX1W/DnwdeGLUQSW1p8mM4ACwWlWXq+oacAo43Dugqs5W1evdzXPAzGhjSmpTkyLYDVzp2V7rvraeR4GnB+1IcizJcpLl5hEltW17gzEZ8FoNHJg8DMwDDwzaX1WLwGJ37MBjSBq/JkWwBuzp2Z4BrvYPSvIR4HHggar6+WjiSRqHJqcG54HZJPuS7ACOAEu9A5LsB/4WOFRVL48+pqQ2DS2CqroOPAacAV4ETlfVhSQnkhzqDvtz4FeAryV5PsnSOoeTtAWlajKn6q4RSGOxUlXzwwZ5Z6Eki0CSRSAJi0ASFoEkLAJJWASSsAgkYRFIwiKQhEUgCYtAEhaBJCwCSVgEkrAIJGERSMIikIRFIAmLQBIWgSQsAklYBJKwCCRhEUjCIpCERSAJi0ASFoEkLAJJWASSsAgkYRFIArZP8L3/G7g0wfe/GTuBH006xCZMW16YvsxbPe97mwyaZBFcqqr5Cb7/piVZnqbM05YXpi/ztOVdj6cGkiwCSZMtgsUJvvfNmrbM05YXpi/ztOUdKFU16QySJsxTA0kWgaQxFEGShSSXkqwmOT5g/9uSfLW7/3tJ7m4700Ya5P1MkotJXkjyrSSNrtO2aVjmnnEfTVJJJn65q0nmJB/rftYXknxl3Bn7sgz7XuxNcjbJc93vxsFJ5LxpVdXaA9gG/DvwPmAH8C/AXN+YTwJf6D4/Any1zUwjyPsh4Je7zz8xybxNM3fH3QV8GzgHzG/1zMAs8Bzwru72e7Z43kXgE93nc8BLk/yMN/toe0ZwAFitqstVdQ04BRzuG3MY+FL3+deBDydJy7nWMzRvVZ2tqte7m+eAmTFn7NfkMwb4HPAE8LNxhltHk8wfB05W1asAVfXymDP2apK3gHd0n78TuDrGfG9Z20WwG7jSs73WfW3gmKq6DrwGvLvlXOtpkrfXo8DTrSYabmjmJPuBPVX1zXEG20CTz/ke4J4kzyY5l2RhbOlu1CTvZ4GHk6wBTwGfHk+00Wj7FuNBP9n7r1c2GTMujbMkeRiYBx5oNdFwG2ZOcgfweeCRcQVqoMnnvJ3O6cGDdGZd30lyX1X9tOVsgzTJexR4sqr+IslvAV/u5v2f9uO9dW3PCNaAPT3bM9w4ZXpjTJLtdKZVP2k513qa5CXJR4DHgUNV9fMxZVvPsMx3AfcBzyR5CbgfWJrwgmHT78U3quoXVfUDOr+gNjumfP2a5H0UOA1QVd8F3k7nF5KmQ8uLLNuBy8A+/n+R5QN9Yz7FmxcLT09wUahJ3v10Fo5mJ73A0zRz3/hnmPxiYZPPeQH4Uvf5TjpT83dv4bxPA490n7+fTlFk0t+Pxv+NY/gQDwL/1v3L83j3tRN0fppCpzm/BqwC/wy8b6IfyPC8/wT8F/B897E08f+JQzL3jZ14ETT8nAP8JXAR+FfgyBbPOwc82y2J54HfmfRnvJmHtxhL8s5CSRaBJCwCSVgEkrAIJGERSMIikAT8L4qgENJRh4oPAAAAAElFTkSuQmCC\n", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "net.params[:] = [-20, -50, 50]\n", "plot_nn_prediction(net)" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "subslide" } }, "source": [ "
\n", "
\n", "## Answer: It is impossible with only one neuron!\n", "
\n", "
" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "fragment" } }, "source": [ " \n", "## We need to use more than one neuron...." ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "## Multilayer Neural Network\n", "" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "subslide" } }, "source": [ "## Learning an XOR NN" ] }, { "cell_type": "code", "execution_count": 8, "metadata": { "slideshow": { "slide_type": "-" } }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
XYoutput
00.00.00.044
10.01.00.949
21.00.00.949
31.01.00.049
\n", "
" ], "text/plain": [ " X Y output\n", "0 0.0 0.0 0.044\n", "1 0.0 1.0 0.949\n", "2 1.0 0.0 0.949\n", "3 1.0 1.0 0.049" ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "Dtrain = SupervisedDataSet(2,1) # define a dataset in pybrain\n", "Dtrain.addSample([0,0],[0])\n", "Dtrain.addSample([0,1],[1])\n", "Dtrain.addSample([1,0],[1])\n", "Dtrain.addSample([1,1],[0])\n", "\n", "from pybrain.supervised.trainers import BackpropTrainer\n", "\n", "net = buildNetwork(2, 2, 1, hiddenclass=pybrain.SigmoidLayer, outclass=pybrain.SigmoidLayer)\n", "T = BackpropTrainer(net, learningrate=0.1, momentum=0.9)\n", "T.trainOnDataset(Dtrain, 1000)\n", "print_pred2(D, net)" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "subslide" } }, "source": [ "## XOR NN Output Plot" ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "slideshow": { "slide_type": "-" } }, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQIAAAD8CAYAAACcoKqNAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvFvnyVgAAC/FJREFUeJzt3F+IXOd9xvHvY6luqOukJdpA0UqxAmsc1RTUDsIlF3ZIWmRdSDchSGCCi4kgjVNoQkHFpQ3KnUsbKKhNVQhOA6mi5KJZgo2gQSbBRKlW2HUrGZWt4laLSq0kjqGYRBH99WKm7ng0qzmS5syf1fcDA3PmvJ55NFo/euc9706qCkl3trumHUDS9FkEkiwCSRaBJCwCSVgEkmhQBEm+mOS1JP+yzvkk+Yskq0leTvLr448pqU1NZgTPAHtucP5RYKl3OwT81e3HkjRJI4ugqr4N/OgGQ/YDf1tdp4FfSvIr4wooqX2bx/AcW4FLfcdrvcf+c3BgkkN0Zw3cc889v/HAAw+M4eU1i86ePTvtCOr6QVUtjBo0jiLIkMeG7luuqmPAMYBOp1MrKytjeHnNomTYj4Wm4N+bDBrHVYM1YFvf8SJweQzPK2lCxlEEy8DHelcPHgLeqKrrPhZIml0jPxok+TvgEWBLkjXgT4CfA6iqLwDPAnuBVeBN4HfaCiupHSOLoKoOjjhfwCfHlkgbwrBfb3fdYHa5s1CSRSDJIpCERSAJi0AS49lZKDWy3hflejVh+pwRSLIIJFkEkrAIJOFioWaA25GnzxmBJItAkkUgCYtAEhaBJLxqoBnlduTJckYgySKQZBFIwiKQhIuFmjNuR26HMwJJFoEki0ASFoEkLAJJeNVAG4DbkW+fMwJJFoEki0ASFoEkXCzUBuZ25OacEUiyCCRZBJJoWARJ9iS5kGQ1yeEh57cnOZXkxSQvJ9k7/qiS2jKyCJJsAo4CjwI7gYNJdg4M+yPgRFXtAg4AfznuoJLa02RGsBtYraqLVXUVOA7sHxhTwDt7998FXB5fRGl8quq6m5oVwVbgUt/xWu+xfp8FHkuyBjwLfGrYEyU5lGQlycqVK1duIa6kNjQpgmEXXgdr9CDwTFUtAnuBLye57rmr6lhVdaqqs7CwcPNpJbWiSRGsAdv6jhe5fur/BHACoKq+C7wD2DKOgJLa16QIzgBLSXYkuZvuYuDywJj/AD4EkOT9dIvAub80J0ZuMa6qa0meBE4Cm4AvVtW5JEeAlapaBj4D/E2S36f7seHxchVGc8LvM2j4uwZV9SzdRcD+x/647/554APjjSZpUtxZKMkikGQRSMLvI5DWdSd9n4EzAkkWgSSLQBIWgSQsAkl41UC6KRt1O7IzAkkWgSSLQBIWgSRcLJTGYt63IzsjkGQRSLIIJGERSMIikIRXDaTWzNN2ZGcEkiwCSRaBJCwCSbhYKE3cLG5HdkYgySKQZBFIwiKQhEUgCa8aSDNh2tuRnRFIsggkWQSSaFgESfYkuZBkNcnhdcZ8NMn5JOeSfGW8MSW1aeRiYZJNwFHgt4A14EyS5ao63zdmCfhD4ANV9XqS97QVWLqTTGo7cpMZwW5gtaouVtVV4Diwf2DMx4GjVfU6QFW9Nt6YktrUpAi2Apf6jtd6j/W7H7g/yQtJTifZM66AktrXZB/BsHnI4HxlM7AEPAIsAt9J8mBV/fhtT5QcAg4BbN++/abDSmpHkxnBGrCt73gRuDxkzDeq6mdV9X3gAt1ieJuqOlZVnarqLCws3GpmSWPWpAjOAEtJdiS5GzgALA+M+XvggwBJttD9qHBxnEEltWdkEVTVNeBJ4CTwCnCiqs4lOZJkX2/YSeCHSc4Dp4A/qKofthVaupNV1dDb7cjtPsGt6nQ6tbKyMpXXljaidS4rnq2qzqj/1p2FkiwCSRaBJPw+AmnDuJ3tyM4IJFkEkiwCSVgEkrAIJGERSMIikIRFIAmLQBIWgSQsAklYBJKwCCRhEUjCIpCERSAJi0ASFoEkLAJJWASSsAgkYRFIwiKQhEUgCYtAEhaBJCwCSVgEkrAIJGERSMIikIRFIImGRZBkT5ILSVaTHL7BuI8kqSSd8UWU1LaRRZBkE3AUeBTYCRxMsnPIuHuB3wO+N+6QktrVZEawG1itqotVdRU4DuwfMu5zwNPAT8aYT9IENCmCrcClvuO13mNvSbIL2FZV37zREyU5lGQlycqVK1duOqykdjQpggx5rN46mdwFfB74zKgnqqpjVdWpqs7CwkLzlJJa1aQI1oBtfceLwOW+43uBB4Hnk7wKPAQsj1owPHv2LEnedpM0HU2K4AywlGRHkruBA8Dy/52sqjeqaktV3VdV9wGngX1VtdJKYkljN7IIquoa8CRwEngFOFFV55IcSbKv7YCS2peqGj2qjRdOrnvhaWWRNqokZ6tq5L4edxZKsggkweZpB+i33pUDPzJI7XJGIMkikGQRSMIikMSMLRauZ9gioguI0vg4I5BkEUiyCCRhEUjCIpDEnFw1GMbtyNL4OCOQZBFIsggkYRFIYo4XC9fjdmTp5jkjkGQRSLIIJGERSGIDLhYO4y5E6cacEUiyCCRZBJKwCCRhEUjiDrlqsB63I0tdzggkWQSSLAJJWASSuMMXC4dxO7LuRM4IJDUrgiR7klxIsprk8JDzn05yPsnLSb6V5L3jjyqpLSOLIMkm4CjwKLATOJhk58CwF4FOVf0a8HXg6XEHldSeJjOC3cBqVV2sqqvAcWB//4CqOlVVb/YOTwOL440pqU1NimArcKnveK332HqeAJ4bdiLJoSQrSVaaR5TUtiZXDYYtow9dQk/yGNABHh52vqqOAcd6Y+dqGd7tyNrImhTBGrCt73gRuDw4KMmHgaeAh6vqp+OJJ2kSmnw0OAMsJdmR5G7gALDcPyDJLuCvgX1V9dr4Y0pq08giqKprwJPASeAV4ERVnUtyJMm+3rA/BX4R+FqSl5Isr/N0kmZQpvU5d97WCIZxjUCzLsnZquqMGucW49vgAqI2CrcYS7IIJFkEkrAIJGERSMKrBmPnF5toHjkjkGQRSLIIJGERSMLFwolxO7JmmTMCSRaBJItAEhaBJCwCSXjVYKrcjqxZ4YxAkkUgySKQhEUgCRcLZ5LbkTVpzggkWQSSLAJJWASSsAgk4VWDueF2ZLXJGYEki0CSRSAJi0AS010s/G/gwhRf/1ZsAX4w7RD91ltE7Jm5vA3MW+ZZz/veJoOmWQQXqqozxde/aUlW5inzvOWF+cs8b3nX40cDSRaBpOkWwbEpvvatmrfM85YX5i/zvOUdKu5Mk+RHA0kWgaQJFEGSPUkuJFlNcnjI+Z9P8tXe+e8lua/tTDfSIO+nk5xP8nKSbyVpdJ22TaMy9437SJJKMvXLXU0yJ/lo770+l+Qrk844kGXUz8X2JKeSvNj72dg7jZy3rKpauwGbgH8D3gfcDfwTsHNgzO8CX+jdPwB8tc1MY8j7QeAXevc/Mc28TTP3xt0LfBs4DXRmPTOwBLwI/HLv+D0znvcY8Ine/Z3Aq9N8j2/21vaMYDewWlUXq+oqcBzYPzBmP/Cl3v2vAx/KiO1yLRqZt6pOVdWbvcPTwOKEMw5q8h4DfA54GvjJJMOto0nmjwNHq+p1gKp6bcIZ+zXJW8A7e/ffBVyeYL7b1nYRbAUu9R2v9R4bOqaqrgFvAO9uOdd6muTt9wTwXKuJRhuZOckuYFtVfXOSwW6gyft8P3B/kheSnE6yZ2Lprtck72eBx5KsAc8Cn5pMtPFoe4vxsH/ZB69XNhkzKY2zJHkM6AAPt5potBtmTnIX8Hng8UkFaqDJ+7yZ7seDR+jOur6T5MGq+nHL2YZpkvcg8ExV/VmS3wS+3Mv7P+3Hu31tzwjWgG19x4tcP2V6a0ySzXSnVT9qOdd6muQlyYeBp4B9VfXTCWVbz6jM9wIPAs8neRV4CFie8oJh05+Lb1TVz6rq+3R/QW1pQvkGNcn7BHACoKq+C7yD7i8kzYeWF1k2AxeBHfz/IsuvDoz5JG9fLDwxxUWhJnl30V04Wpr2Ak/TzAPjn2f6i4VN3uc9wJd697fQnZq/e4bzPgc83rv/frpFkWn/fDT+M07gTdwL/Gvvf56neo8dofuvKXSb82vAKvCPwPum+oaMzvsPwH8BL/Vuy1P/SxyReWDs1Iug4fsc4M+B88A/AwdmPO9O4IVeSbwE/Pa03+ObubnFWJI7CyVZBJKwCCRhEUjCIpCERSAJi0AS8L+XL5k9PGsoLwAAAABJRU5ErkJggg==\n", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "plot_nn_prediction(net)" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "## The Little Red Riding Hood Neural Network\n", "\n", "" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "subslide" } }, "source": [ "## LRRH Network Architecture\n", "\n", "" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "subslide" } }, "source": [ "## Training\n" ] }, { "cell_type": "code", "execution_count": 10, "metadata": { "scrolled": true, "slideshow": { "slide_type": "fragment" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " Big Ears Big Teeth Handsome Wrinkled Scream Hug Food Kiss\n", "0 1.0 1.0 0.0 0.0 1.0 0.0 0.0 0.0\n", "1 0.0 1.0 1.0 0.0 0.0 0.0 1.0 1.0\n", "2 0.0 0.0 0.0 1.0 0.0 1.0 1.0 0.0\n" ] } ], "source": [ "from pybrain.tools.validation import Validator\n", "\n", "validator = Validator()\n", "Dlrrh = SupervisedDataSet(4,4) \n", "Dlrrh.addSample([1,1,0,0],[1,0,0,0])\n", "Dlrrh.addSample([0,1,1,0],[0,0,1,1])\n", "Dlrrh.addSample([0,0,0,1],[0,1,1,0])\n", "df = pd.DataFrame(Dlrrh['input'],columns=['Big Ears', 'Big Teeth', 'Handsome', 'Wrinkled'])\n", "print (df.join(pd.DataFrame(Dlrrh['target'],columns=['Scream', 'Hug', 'Food', 'Kiss'])))\n", "net = buildNetwork(4, 3, 4, hiddenclass=pybrain.SigmoidLayer, outclass=pybrain.SigmoidLayer)\n" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "subslide" } }, "source": [ "## Backpropagation" ] }, { "cell_type": "code", "execution_count": 11, "metadata": { "slideshow": { "slide_type": "fragment" } }, "outputs": [ { "data": { "text/plain": [ "[]" ] }, "execution_count": 11, "metadata": {}, "output_type": "execute_result" }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvFvnyVgAAIABJREFUeJzt3XucXWV97/HPd/bcZ5JMkpkEyEyYhIRLQAka8X4tItQWbGsPYG1ptYejR9SW09ODtUWlr/Zl7cXaSlt4tVi1KvVCbWpRVFBsUTABIYQgZhIuGQLJ5H6Z3Gbmd/5Ya8jOzp7Za0j27Nl7vu/Xa7/2Ws961tq/lQX5ZT3PWs+jiMDMzGw8dZUOwMzMpj4nCzMzK8nJwszMSnKyMDOzkpwszMysJCcLMzMrycnCzMxKcrIwM7OSnCzMzKyk+koHcLJ0dnZGb29vpcMwM6sqDzzwwLaI6CpVr2aSRW9vL6tXr650GGZmVUXSU1nquRnKzMxKKmuykHSJpMcl9Um6vsj290h6RNJDkv5b0rK8bR9K93tc0lvKGaeZmY2vbMlCUg64CbgUWAZclZ8MUl+MiBdFxHLgE8BfpfsuA64EzgUuAf4uPZ6ZmVVAOe8sLgT6ImJjRBwGbgMuz68QEXvyVtuA0fHSLwdui4hDEfEE0Jcez8zMKqCcHdwLgE156/3AywsrSXofcB3QCLwpb9/7CvZdUJ4wzcyslHLeWahI2XEzLUXETRFxBvD/gD+cyL6SrpG0WtLqgYGBEwrWzMzGVs5k0Q/05K13A5vHqX8b8LaJ7BsRt0TEiohY0dVV8jFhMzN7gcqZLFYBSyUtktRI0mG9Mr+CpKV5q28F1qfLK4ErJTVJWgQsBX5cjiB3Dx7hU99dz5r+XeU4vJlZTShbn0VEDEm6FrgTyAG3RsSjkm4EVkfESuBaSRcBR4CdwNXpvo9K+jKwDhgC3hcRw+WIU3Xwye/+jIZ68eLujnL8hJlZ1SvrG9wRcQdwR0HZDXnLHxxn3z8B/qR80SVmNjdw6qxm1m/ZV+6fMjOrWn6DG1jU2cbTOwYrHYaZ2ZTlZAHMn9nMlj0HKx2GmdmU5WRBkiy27jlExHFP55qZGU4WAMyf2cTh4RF2Dh6pdChmZlOSkwVwysxmAJ7b7aYoM7NinCyAeWmycL+FmVlxThZAZ3sjADv2H65wJGZmU5OTBTCrpQGAXQfcZ2FmVoyTBTCjOUkWu50szMyKcrIAcnViZnM9e5wszMyKcrJIzWpt8J2FmdkYnCxSs1qcLMzMxuJkkZrV0sCuQT8NZWZWjJNFyncWZmZjc7JIJcliqNJhmJlNSU4WqZktDew5cMSDCZqZFeFkkZrV0sDh4REOHCnLhHxmZlXNySI1+ha3+y3MzI7nZJHqaEnGh3KyMDM7npNF6vnxoTynhZnZcZwsUh2tboYyMxuLk0XKfRZmZmNzskjNGr2zcDOUmdlxnCxSM5rqydWJXQc85IeZWSEni5SkdHwo31mYmRUqa7KQdImkxyX1Sbq+yPbrJK2TtEbSXZJOz9s2LOmh9LOynHGO6vD4UGZmRdWX68CScsBNwJuBfmCVpJURsS6v2k+AFRExKOm9wCeAK9JtByJiebniK2amk4WZWVHlvLO4EOiLiI0RcRi4Dbg8v0JEfC8iBtPV+4DuMsZTUkerm6HMzIopZ7JYAGzKW+9Py8bybuCbeevNklZLuk/S24rtIOmatM7qgYGBEw7YzVBmZsWVrRkKUJGyokO6SnonsAJ4fV7xwojYLGkxcLekRyJiwzEHi7gFuAVgxYoVJzxcbEdroydAMjMropx3Fv1AT956N7C5sJKki4APA5dFxKHR8ojYnH5vBL4PXFDGWIF0mPKDQwyPeJhyM7N85UwWq4ClkhZJagSuBI55qknSBcDNJIlia175bElN6XIn8Gogv2O8LDrSt7j3HnRTlJlZvrI1Q0XEkKRrgTuBHHBrRDwq6UZgdUSsBP4caAe+Igng6Yi4DDgHuFnSCElC+3jBU1RlMTo+1K7BI3S0Npb758zMqkY5+yyIiDuAOwrKbshbvmiM/X4IvKicsRXzfLJwJ7eZ2TH8BneeWZ7TwsysKCeLPEfntPATUWZm+Zws8nhOCzOz4pws8nS0NFAn2Lb3UOnKZmbTiJNFnvpcHfNnNrN598FKh2JmNqU4WRQ4raOFzbsOVDoMM7MpxcmigJOFmdnxnCwKnDYraYYa8ZAfZmbPc7Io0D2nlcNDIzy35yARweGhkUqHZGZWcU4WBZadOhOAB5/eyf/83AOcc8O3+ML9T1U4KjOzyirrcB/V6NzTZtLWmOPaL/4EgO7ZLXzsP9bx2iVdLJzbWuHozMwqw3cWBZobcrzvTUtoacjxJ790Hl99z6uICD77oycrHZqZWcX4zqKI//2GJbzndWdQV5fM33TxslO4/cF+PnTp2dTnnF/NbPrx33xjGE0UAG998ansHDzC6qd2VjAiM7PKcbLI4HVndtGYq+Oux7ZUOhQzs4pwssigvamely+ew12PbS1d2cysBjlZZHTROfPZuG0/Gwb2VToUM7NJN26ykJST9LuTFcxU9nPnzANwU5SZTUvjJouIGAYun6RYprTu2a2cfcoMN0WZ2bSUpRnqXkmflvRaSS8Z/ZQ9sino9Wd18eDTO9l/aKjSoZiZTaos71m8Kv2+Ma8sgDed/HCmttcs6eTmezby4yd38Maz5lU6HDOzSVMyWUTEGycjkGrwst45NNbXce/6bU4WZjatlGyGkjRL0l9JWp1+/lLSrMkIbqppbshxQU8Hq57cUelQzMwmVZY+i1uBvcD/SD97gM+UM6ipbPnCDtY9u4dDQ8OVDsXMbNJkSRZnRMRHImJj+vkYsDjLwSVdIulxSX2Sri+y/TpJ6yStkXSXpNPztl0taX36uTr7KZXX8u4OjgwH6zbvqXQoZmaTJkuyOCDpNaMrkl4NlJx3VFIOuAm4FFgGXCVpWUG1nwArIuLFwFeBT6T7zgE+ArwcuBD4iKTZGWItu+ULOwB4aNOuCkdiZjZ5sjwN9R7gc3n9FDuBLP/SvxDoi4iNAJJuI3lnY91ohYj4Xl79+4B3pstvAb4TETvSfb8DXAJ8KcPvltWps1qYP7OJh50szGwaGTdZSKoDzoqI8yXNBIiIrO0vC4BNeev9JHcKY3k38M1x9l2Q8XfL7vzuDt9ZmNm0UuoN7hHg2nR5zwQSBYCKlEXRitI7gRXAn09kX0nXjD6lNTAwMIHQTsw5p87kqR2DHDziTm4zmx6y9Fl8R9LvSeqRNGf0k2G/fqAnb70b2FxYSdJFwIeByyLi0ET2jYhbImJFRKzo6urKENLJcca8diLgye37J+03zcwqKUuyeBfwPuAHwAPpZ3WG/VYBSyUtktQIXAmszK8g6QLgZpJEkT/o0p3AxZJmpx3bF6dlU8IZXW0AbNjqZGFm00OWPot3RsS9Ez1wRAxJupbkL/kccGtEPCrpRmB1RKwkaXZqB74iCeDpiLgsInZI+mOShANw42hn91SwqDNJFhs9XLmZTRPjJouIGJH0F8ArX8jBI+IO4I6Cshvyli8aZ99bSV4InHJaG+tZ0NHiuS3MbNrI0gz1bUm/ovSf/pZY3NXGxm1uhjKz6SHLexbXAW3AkKSDJE8qRUTMLGtkU9zpc1v5xppnKx2GmdmkyDLq7IzJCKTadM9uZdfgEfYePMKM5oZKh2NmVlZjNkOl7z6MLr+6YNu15QyqGnTPbgHgmV0lRz4xM6t64/VZXJe3/LcF295VhliqSvfsVgD6dzhZmFntGy9ZaIzlYuvTzuidxaadgxWOxMys/MZLFjHGcrH1aWduWyMtDTn6d/rOwsxq33gd3GdLWkNyF3FGuky6nmk+i1omie7ZLfT7zsLMpoHxksU5kxZFlUqShe8szKz2jZksIuKpyQykGnXPbuWBp3ZWOgwzs7LL8ga3jaFnTgt7Dg6x+8CRSodiZlZWThYnYPTx2WfcFGVmNS5TspDUIumscgdTbfz4rJlNFyWThaRfBB4CvpWuL5e0cvy9pofnX8zznYWZ1bgsdxYfBS4EdgFExENAb/lCqh6zWxtoa8yxaYfvLMystmVJFkMRsbvskVQhSfTMafWdhZnVvCxDlK+V9A4gJ2kp8AHgh+UNq3r4xTwzmw6y3Fm8HzgXOAR8EdgN/E45g6om3bNb2bRjkIhpPwKKmdWwUnNw54CPRcT/BT48OSFVl545rew/PMzOwSPMaWusdDhmZmUx7p1FRAwDL52kWKrS6OOzbooys1qWpc/iJ+mjsl8Bnp90OiJuL1tUVaQnfXx2044DvLi7o8LRmJmVR5ZkMQfYDrwprywAJwuSIT/AL+aZWW3LMgf3b01GINVqRnMDHa0NboYys5pWMllIagbeTfJEVPNoeURM+6lVR3XPbmGTp1c1sxqW5dHZzwOnAG8B7gG6gb3lDKra9MxudTOUmdW0LMliSUT8EbA/Ij4LvBV4UZaDS7pE0uOS+iRdX2T76yQ9KGlI0tsLtg1Leij9TOmxqHrmtPLMzgN+18LMalaWDu7RyRp2SToPeI4MY0Ol72jcBLwZ6AdWSVoZEevyqj0N/Cbwe0UOcSAilmeIr+K6Z7dwaGiEgb2HmDezufQOZmZVJsudxS2SZgN/BKwE1gGfyLDfhUBfRGyMiMPAbcDl+RUi4smIWAOMTCzsqWXhnOTx2Se27S9R08ysOpVMFhHxjxGxMyLuiYjFETEvIv4hw7EXAJvy1vvTsqyaJa2WdJ+ktxWrIOmatM7qgYGBCRz65Dqjqx2AjU4WZlajsjwNdUOx8oi4sdSuxXbLElRqYURslrQYuFvSIxGxoSCGW4BbAFasWFGxDoMFHS001dexYeu+SoVgZlZWWZqh9ud9hoFLyTafRT/Qk7feDWzOGlhEbE6/NwLfBy7Iuu9kq6sTi7va2TDgZGFmtSnLS3l/mb8u6S9I+i5KWQUslbQIeAa4EnhHlqDSPpLBiDgkqRN4Ndn6SSrmjK421vR72g8zq02Z5uAu0AosLlUpIoaAa4E7gceAL0fEo5JulHQZgKSXSeoHfhW4WdKj6e7nAKslPQx8D/h4wVNUU84ZXe1s2jnIwSPDlQ7FzOyky9Jn8QhH+xpyQBdQqr8CgIi4A7ijoOyGvOVVJM1Thfv9kIzvckwVZ8xrJwKe2j7IWafMqHQ4ZmYnVZb3LH4hb3kI2JLeNVieM7raANgwsM/JwsxqTpZkUTi0x0zp6INOEbHjpEZUpRZ3Jo/P+okoM6tFWZLFgyRPNe0keRy2g+TNa0iap0r2X0wHLY05FnS00OcnosysBmXp4P4W8IsR0RkRc0mapW6PiEUR4USRZ8m8dvp8Z2FmNShLsnhZ2lENQER8E3h9+UKqXkvTZDE84gEFzay2ZEkW2yT9oaReSadL+jDJzHlWYOn8dg4NjXgiJDOrOVmSxVUkj8v+G/B1YF5aZgWWzk+eglq/xU1RZlZbsrzBvQP4IDz/ZvWu8MQNRS2ZlzwRtX7rPi5aNr/C0ZiZnTxj3llIukHS2elyk6S7gT5gi6SLJivAajKzuYFTZjazfqsnEjSz2jJeM9QVwOPp8tVp3Xkkndt/Wua4qtbS+X4iysxqz3jJ4nBec9NbgC9FxHBEPEa29zOmpaXzZrB+yz5G/ESUmdWQ8ZLFIUnnSeoC3gh8O29ba3nDql5L57dz4Mgwz+w6UOlQzMxOmvGSxQeBrwI/BT4ZEU8ASPp54CeTEFtVWpp2crspysxqyZjNSRFxP3B2kfLjRpK1o5bOSx+f3bqXN549r8LRmJmdHC9kPgsbx6zWBubNaOJnftfCzGqIk0UZLJ3fzno3Q5lZDXGyKIOl82bQt2UvfnfRzGpFpkdgJb0K6M2vHxGfK1NMVW/JvHb2Hx7m2d0HOa2jpdLhmJmdsCzTqn4eOAN4CBidYDoAJ4sxnDk6RtTWfU4WZlYTstxZrACWeTyo7EYfn12/ZS+vP7OrwtGYmZ24LH0Wa4FTyh1ILZnd1khne6NHnzWzmpHlzqITWCfpx8Ch0cKIuKxsUdWAM7raPcWqmdWMLMnio+UOohYt7mrj249uqXQYZmYnRZb5LO6ZjEBqzaLONrbvP8zuA0eY1dJQ6XDMzE5IyT4LSa+QtErSPkmHJQ1L2pPl4JIukfS4pD5J1xfZ/jpJD0oakvT2gm1XS1qffq7OfkpTQ+/cNgCe3La/wpGYmZ24LB3cnyaZRnU90AL8dlo2Lkk54CbgUmAZcJWkZQXVngZ+E/hiwb5zgI8ALwcuBD6SztJXNRZ1psliu5OFmVW/TG9wR0QfkEvns/gM8IYMu10I9EXExog4DNwGXF5w3CcjYg0wUrDvW4DvRMSOiNgJfAe4JEusU0XPnFYk2DjgZGFm1S9LB/egpEbgIUmfAJ4F2jLstwDYlLfeT3KnkEWxfRdk3HdKaG7IsaCjxXcWZlYTstxZ/Hpa71pgP9AD/EqG/VSkLOuLfZn2lXSNpNWSVg8MDGQ89ORZ1NnmPgszqwklk0VEPEXyl/epEfGxiLgubZYqpZ8ksYzqBjZnjCvTvhFxS0SsiIgVXV1T703p3rltbNy23wMKmlnVy/I01C+SjAv1rXR9uaSVGY69ClgqaVHajHUlkGU/gDuBiyXNTju2L07LqkpvZxt7Dw6xY//hSodiZnZCsjRDfZSks3oXQEQ8RDIC7bgiYoik6epO4DHgyxHxqKQbJV0GIOllkvqBXwVulvRouu8O4I9JEs4q4Ma0rKos9hNRZlYjsnRwD0XEbqlYN8L4ik3BGhE35C2vImliKrbvrcCtE/7RKaQ3TRZPbBvkpafPqXA0ZmYvXJZksVbSO4CcpKXAB4Afljes2tA9u4VcnXhim8eIMrPqlqUZ6v3AuSSDCH4J2AP8TjmDqhUNuTp6Zrfw5LbBSodiZnZCsowNNQh8OP3YBC3qTJ6IMjOrZmMmi1JPPHmI8mx6O9u4/4kdRAQvpN/HzGwqGO/O4pUkb1F/Cbif4i/KWQmLOtsYPDzMwN5DzJvZXOlwzMxekPH6LE4B/gA4D/gU8GZgW0Tc42HLszt97ugTUW6KMrPqNWaySAcN/FZEXA28AugDvi/p/ZMWXQ1YNNfvWphZ9Ru3g1tSE/BWkiHKe4G/AW4vf1i147SOZhpy4gk/EWVmVWy8Du7PkjRBfRP4WESsnbSoakh9ro6eOa085TsLM6ti491Z/DrJKLNnAh/Ie5JHQETEzDLHVjMWzW1zn4WZVbUxk0VEZJoYyUo7fW4bP9yw3Y/PmlnVckKYBIs6WzlwZJgtew5VOhQzsxfEyWIS9Hr0WTOrck4Wk6B39PFZ91uYWZVyspgEp3W00Jir4wnfWZhZlXKymAS5OtEzp8V3FmZWtZwsJsmizjae2u4X88ysOjlZTJLeuW08uX0/IyNR6VDMzCbMyWKS9Ha2cfDICFv2Hqx0KGZmE+ZkMUl6PfqsmVUxJ4tJsmReOwAbtno+bjOrPk4Wk2T+zCZmNtfz+Ja9lQ7FzGzCnCwmiSTOOmUGjz/nZGFm1cfJYhKdOT9JFhF+IsrMqouTxSQ665QZ7Dk45AEFzazqlDVZSLpE0uOS+iRdX2R7k6R/TbffL6k3Le+VdEDSQ+nnH8oZ52Q5a/4MAPdbmFnVKVuykJQDbgIuBZYBV0laVlDt3cDOiFgCfBL4s7xtGyJiefp5T7ninExnjiaL5/ZUOBIzs4kp553FhUBfRGyMiMPAbcDlBXUuBz6bLn8V+DnV8OxAs9saOXVWM2ufcbIws+pSzmSxANiUt96flhWtExFDwG5gbrptkaSfSLpH0mvLGOekOr+7g4f7d1U6DDOzCSlnsih2h1D4GNBYdZ4FFkbEBcB1wBclHTfnt6RrJK2WtHpgYOCEA54Myxd28NT2QXbuP1zpUMzMMitnsugHevLWu4HNY9WRVA/MAnZExKGI2A4QEQ8AG4AzC38gIm6JiBURsaKrq6sMp3Dynd/dAeC7CzOrKuVMFquApZIWSWoErgRWFtRZCVydLr8duDsiQlJX2kGOpMXAUmBjGWOdNC/qnkWuTqx+cmelQzEzy6xsySLtg7gWuBN4DPhyRDwq6UZJl6XV/gmYK6mPpLlp9PHa1wFrJD1M0vH9nojYUa5YJ1N7Uz3Lezr4r75tlQ7FzCyz+nIePCLuAO4oKLshb/kg8KtF9vsa8LVyxlZJr1nSyd/evZ7dg0eY1dpQ6XDMzEryG9wV8NqlnYwE/HCD7y7MrDo4WVTA+T0dtDfV84P11fEEl5mZk0UFNOTqeMNZXdz56BaODI9UOhwzs5KcLCrksvNPY8f+w9zrjm4zqwJOFhXy+rO6mNFcz388/GylQzEzK8nJokKa6nNcet4pfHPts+w9eKTS4ZiZjcvJooJ+7eWnM3h4mNsffKbSoZiZjcvJooLO7+ng/J4OPvujJz17nplNaU4WFfYbrzidjQP7ubdve6VDMTMbk5NFhb31xacyp62Rz9z7RKVDMTMbk5NFhTU35PiNV57OXT/dymPPelIkM5uanCymgN98VS/tTfXc9L2+SodiZlaUk8UU0NHayDtfcTr/+cizbBzYV+lwzMyO42QxRfz2axfRVF/H331/Q6VDMTM7jpPFFNHZ3sSvvfx0bn+w330XZjblOFlMIe9/0xJmtjRw43+s83sXZjalOFlMIR2tjVz35jP50cbtfGONx4wys6nDyWKKeceFCzm/p4M//Ppant19oNLhmJkBThZTTn2ujr++YjlHhkd47788yIHDw5UOyczMyWIqWtTZxievWM7D/bt47xcecMIws4pzspii3nLuKfzpL72Ie342wJW3/Ignt+2vdEhmNo05WUxhV124kJvf+VI2btvPpZ/6Lz5993rPfWFmFaFaeURzxYoVsXr16kqHURbP7T7IH/37Wr6zbguzWhr4lZd088svWcC5p81EUqXDM7MqJumBiFhRsp6TRfVY07+Lf7hnA99dt5XDwyMs6GjhdWd28uolnSzv6WBBR4uTh5lNiJNFDds1eJj/fORZ7nl8gB9t2M7eQ0MAdLQ2cN5ps1gyr52Fc1qTz9xWFnS00NZUX+GozWwqmhLJQtIlwKeAHPCPEfHxgu1NwOeAlwLbgSsi4sl024eAdwPDwAci4s7xfms6JYt8Q8MjrN28h7XP7ObRzbtZ+8weNg7sY3/BE1QtDTnmtjfS2d5EZ3sjc9oaaW9qoL25nvamHO1NDbQ15ZjRXE9rYz1N9XU01edorK9LlhvqaMrlaGqoozFXR12d72DMakHWZFG2f25KygE3AW8G+oFVklZGxLq8au8GdkbEEklXAn8GXCFpGXAlcC5wGvBdSWdGhJ8hLVCfq2N5TwfLezqeL4sIduw/zNM7Bnl6xyCbdx1k+75DbN9/mG37DvHMroM88sxu9h8aZv/hIV7IvxcacqKpPkdDTuTq6qivE7k6UZ9Lv+tEfV3dMeu5tCxXp3S/o+v1dUISuTqoU7JcJ8jVKV1PyusEdWlZnY7WzeVtG62bG3O/o/smZRzzGxKI0W+SbyldVl7ZGOWj+x53nNHfAArLOfrbFBznaEyk2/PKC+qO7lvsPDhmH+UdL6mXLuSVjR5PBetH9yls9Sw8Xv6xxjyOm06rQjnbJi4E+iJiI4Ck24DLgfxkcTnw0XT5q8CnlfyXczlwW0QcAp6Q1Jce70dljLdmSGJuexNz25u4YOHsceuOjASDR4bZd3CIfYeSz/5DQxweGuHQ0DCHhkY4NDSSridlzy8fGWFoZIShkWB4OJLvkRGOFKwn38n64OGh55fzv48MjxABIxHpJ4nt+eUIImA4LSusa7Xj+eTG+IlqdMNY28ZLVIyVEDP8NhSrO/Zv5x/rmG0TibdgJb9cEuecOpO/veoCyqmcyWIBsClvvR94+Vh1ImJI0m5gblp+X8G+Cwp/QNI1wDUACxcuPGmBTyd1daK9qZ72Ku7TiGOSR0HCiSBGku/hODbRDI8U2W8kCCACgmT7mMvpbyfJKr8sjYnkmOSXc/Q3092OOfZoeRQcczTu0d+EvPp55fnHHCkSLzFantZ9/reOPTbHbYsidYtvy78uo+XH/9bxxxstyFp39NzyC49uy/7b+fFO5Lcp3DaBP6sx4y3yW/nbC8tHV3pmt1Bu5fwboti9ZeG/Aceqk2VfIuIW4BZI+iwmGqDVhuebd4r+Z2NmJ0M5X8rrB3ry1ruBzWPVkVQPzAJ2ZNzXzMwmSTmTxSpgqaRFkhpJOqxXFtRZCVydLr8duDuS+62VwJWSmiQtApYCPy5jrGZmNo6yNUOlfRDXAneSPDp7a0Q8KulGYHVErAT+Cfh82oG9gyShkNb7Mkln+BDwPj8JZWZWOX4pz8xsGsv6noUHEjQzs5KcLMzMrCQnCzMzK8nJwszMSqqZDm5JA8BTJ3CITmDbSQqnWvica990O1/wOU/U6RHRVapSzSSLEyVpdZYnAmqJz7n2TbfzBZ9zubgZyszMSnKyMDOzkpwsjrql0gFUgM+59k238wWfc1m4z8LMzErynYWZmZU07ZOFpEskPS6pT9L1lY7nZJHUI+l7kh6T9KikD6blcyR9R9L69Ht2Wi5Jf5P+OayR9JLKnsELJykn6SeSvpGuL5J0f3rO/5qOgkw6qvG/pud8v6TeSsb9QknqkPRVST9Nr/cra/06S/rd9L/rtZK+JKm51q6zpFslbZW0Nq9swtdV0tVp/fWSri72W1lM62Sho/OEXwosA65SMv93LRgC/k9EnAO8Anhfem7XA3dFxFLgrnQdkj+DpennGuDvJz/kk+aDwGN5638GfDI9550kc79D3hzwwCfTetXoU8C3IuJs4HySc6/Z6yxpAfABYEVEnEcyqvWV1N51/mfgkoKyCV1XSXOAj5DMUnoh8JHRBDNhyZSU0/MDvBK4M2/9Q8CHKh1Xmc7134E3A48Dp6ZlpwKPp8s3A1fl1X++XjV9SCbKugt4E/ANklkXtwH1hdecZPj8V6bL9Wk9VfocJni+M4EnCuOu5evM0emY56TX7RvAW2rxOgO9wNoXel2Bq4Cb88qPqTeRz7S+s6D4POHHzfVd7dLb7guA+4H5EfEsQPo9L61WK38Wfw2mdlnRAAAEDklEQVT8PjCSrs8FdkXEULqef17HzAEPjM4BX00WAwPAZ9Kmt3+U1EYNX+eIeAb4C+Bp4FmS6/YAtX2dR030up606z3dk0Wmub6rmaR24GvA70TEnvGqFimrqj8LSb8AbI2IB/KLi1SNDNuqRT3wEuDvI+ICYD9HmyaKqfpzTptRLgcWAacBbSTNMIVq6TqXMtY5nrRzn+7Joqbn+pbUQJIovhARt6fFWySdmm4/FdialtfCn8WrgcskPQncRtIU9ddARzrHOxx7XmPNAV9N+oH+iLg/Xf8qSfKo5et8EfBERAxExBHgduBV1PZ1HjXR63rSrvd0TxZZ5gmvSpJEMm3tYxHxV3mb8uc9v5qkL2O0/DfSpypeAewevd2tFhHxoYjojohekmt5d0T8GvA9kjne4fhzLjYHfNWIiOeATZLOSot+jmQ64pq9ziTNT6+Q1Jr+dz56zjV7nfNM9LreCVwsaXZ6R3ZxWjZxle7AqfQH+HngZ8AG4MOVjuckntdrSG431wAPpZ+fJ2mrvQtYn37PSeuL5MmwDcAjJE+aVPw8TuD83wB8I11eDPwY6AO+AjSl5c3pel+6fXGl436B57ocWJ1e668Ds2v9OgMfA34KrAU+DzTV2nUGvkTSJ3OE5A7h3S/kugLvSs+9D/itFxqP3+A2M7OSpnszlJmZZeBkYWZmJTlZmJlZSU4WZmZWkpOFmZmV5GRhVoSkfel3r6R3nORj/0HB+g9P5vHNysHJwmx8vcCEkkU6mvF4jkkWEfGqCcZkNumcLMzG93HgtZIeSudQyEn6c0mr0nkD/heApDcomT/kiyQvRSHp65IeSOdduCYt+zjQkh7vC2nZ6F2M0mOvlfSIpCvyjv19HZ2z4gvpm8tmk6a+dBWzae164Pci4hcA0r/0d0fEyyQ1AfdK+nZa90LgvIh4Il1/V0TskNQCrJL0tYi4XtK1EbG8yG/9Msnb2OcDnek+P0i3XQCcSzKuz70k42D998k/XbPifGdhNjEXk4zB8xDJkO9zSSacAfhxXqIA+ICkh4H7SAZzW8r4XgN8KSKGI2ILcA/wsrxj90fECMnQLb0n5WzMMvKdhdnECHh/RBwzGJukN5AMD56/fhHJpDuDkr5PMkZRqWOP5VDe8jD+f9cmme8szMa3F5iRt34n8N50+HcknZlONlRoFslUnoOSziaZ2nbUkdH9C/wAuCLtF+kCXkcy8J1ZxflfJ2bjWwMMpc1J/0wy33Uv8GDayTwAvK3Ift8C3iNpDckUl/flbbsFWCPpwUiGUB/1byTTgT5MMmLw70fEc2myMasojzprZmYluRnKzMxKcrIwM7OSnCzMzKwkJwszMyvJycLMzEpysjAzs5KcLMzMrCQnCzMzK+n/A3MHJI5pqIBUAAAAAElFTkSuQmCC\n", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "T = BackpropTrainer(net, learningrate=0.01, momentum=0.99)\n", "scores = []\n", "for i in range(1000):\n", " T.trainOnDataset(Dlrrh, 1)\n", " prediction = net.activateOnDataset(Dlrrh)\n", " scores.append(validator.MSE(prediction, Dlrrh.getField('target')))\n", "plt.ylabel('Mean Square Error')\n", "plt.xlabel('Iteration')\n", "plt.plot(scores)\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Learning as optimization\n", "\n", "General optimization problem:\n", "\n", "$$\\min_{f\\in H}L(f,D)$$\n", "\n", "with $H$: hypothesis space, $D$:training data, $L$:loss/error" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "subslide" } }, "source": [ "## Example, least squares linear regression:\n", "$$\\min_{f\\in H}L(f,D)$$\n", "* Hypothesis space:\n", "$f(x)=w^{T}x$\n", "* Data: $D = \\{(x_1, t_1), \\dots , (x_n, t_n)\\}$\n", "* Least squares loss: \n", "$$L(f, D)=-\\sum_{t_i \\in D}(t_{i} - f(x_i))^2$$" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "subslide" } }, "source": [ "## Example, logistic regression:\n", "$$\\min_{f\\in H}L(f,D)$$\n", "* Hypothesis space:\n", "$f(x)=P(C_{+}|x)=\\sigma(w^{T}x)$\n", "* Data: $D = \\{(x_1, t_1), \\dots , (x_n, t_n)\\}$\n", "* Cross-entropy error: \n", "$$E(f,D)=-\\ln p(D|f)=-\\sum_{t_i \\in D}(t_{n}\\ln y_{n}+(1-t_{n})\\ln(1-y_{n}))$$" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "## Gradient descent\n", "\n", "" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "subslide" } }, "source": [ "## Prediction" ] }, { "cell_type": "code", "execution_count": 12, "metadata": { "slideshow": { "slide_type": "skip" } }, "outputs": [], "source": [ "def lrrh_input(vals):\n", " return pd.DataFrame(vals,index=['big ears', 'big teeth', 'handsome', 'wrinkled'], columns=['input'])\n", "\n", "def lrrh_output(vals):\n", " return pd.DataFrame(vals,index=['scream', 'hug', 'offer food', 'kiss cheek'], columns=['output'])" ] }, { "cell_type": "code", "execution_count": 13, "metadata": { "slideshow": { "slide_type": "fragment" } }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
input
big ears0
big teeth0
handsome0
wrinkled0
\n", "
" ], "text/plain": [ " input\n", "big ears 0\n", "big teeth 0\n", "handsome 0\n", "wrinkled 0" ] }, "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ "in_vals = [0, 0, 0, 0]\n", "lrrh_input(in_vals)" ] }, { "cell_type": "code", "execution_count": 14, "metadata": { "scrolled": true, "slideshow": { "slide_type": "fragment" } }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
output
scream0.055038
hug0.786478
offer food0.947874
kiss cheek0.090418
\n", "
" ], "text/plain": [ " output\n", "scream 0.055038\n", "hug 0.786478\n", "offer food 0.947874\n", "kiss cheek 0.090418" ] }, "execution_count": 14, "metadata": {}, "output_type": "execute_result" } ], "source": [ "lrrh_output(net.activate(in_vals))" ] } ], "metadata": { "celltoolbar": "Slideshow", "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.4" }, "livereveal": { "center": "true", "theme": "black", "transition": "zoom" } }, "nbformat": 4, "nbformat_minor": 1 }