{
 "metadata": {
  "name": "32-rnn"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "Title: Multilayer Architectures\n",
      "Author: Thomas Breuel\n",
      "Institution: UniKL"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "\n",
      "import scipy,pylab,numpy\n",
      "from numpy import *\n",
      "from scipy import *\n",
      "from pylab import *\n",
      "from urllib import urlopen\n",
      "from gzip import GzipFile\n",
      "from scipy.spatial import distance\n",
      "from scipy.spatial.distance import cdist\n",
      "\n",
      "def sigmoid(x):\n",
      "    return 1/(1+exp(-x))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "N = 20\n",
      "ni = 3\n",
      "no = 2\n",
      "seq = array(rand(N)*ni,'i')%3\n",
      "output = 1*(roll(seq,1)==0)*(seq==1)\n",
      "target = zeros((N,no))\n",
      "target[output] = 1\n",
      "plot(seq[:20])\n",
      "plot(output[:20])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 2,
       "text": [
        "[<matplotlib.lines.Line2D at 0x3c30650>]"
       ]
      },
      {
       "output_type": "display_data",
       "png": "iVBORw0KGgoAAAANSUhEUgAAAXUAAAD9CAYAAABDaefJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJztnX1wVMeZ7p8RkgAhIiERBJYUvqQRCIQEJtHaGxZ5ba5j\n1w0hcWpjV8WXwuQWReHr8iZ/3NpbrrVJeV1414nLDk4uzsYuu7KLXfHexCQL2mxVVi4WB8h6xcbG\nZEYYZIQAxUhCCAmjr3P/aA4ajaZnzkf3Od193l+Vy7Y0c6Y17znPeft9T/cTsyzLAkEQBGEEeWEP\ngCAIghAHiTpBEIRBkKgTBEEYBIk6QRCEQZCoEwRBGASJOkEQhEFkFfWuri7cddddWLVqFVavXo0X\nX3wx4+see+wx1NbWorGxEe3t7VIGShAEQeQmP9svCwoK8Pzzz6OpqQnXrl3D7bffjk2bNmHlypW3\nXnPw4EGcPn0aHR0dOHbsGHbu3ImjR49KHzhBEAQxnayZ+sKFC9HU1AQAKC4uxsqVK3HhwoUprzlw\n4AC2bt0KAGhubsaVK1fQ09MjabgEQRBENhzX1Ds7O9He3o7m5uYpP+/u7kZ1dfWt/6+qqsL58+fF\njZAgCIJwTNbyi821a9fw9a9/HS+88AKKi4un/T59p4FYLDbtNZl+RhAEQeTGzW4uOTP10dFRPPDA\nA/jmN7+JLVu2TPt9ZWUlurq6bv3/+fPnUVlZyR1YWP/s328BsHDxYnhjEPnPk08+GfoYwv7n5EkW\n03/5F/ou//Ef2Xdx6VL4Y/HzfU5MWMjPt/B3fxf+36HKP27JKuqWZWH79u2or6/H448/nvE1mzdv\nxuuvvw4AOHr0KEpLS1FRUeF6ILLp7WX/TiTCHQchDorpJPZ3YH8nunL1KjA2RjH1Q9byy5EjR/DT\nn/4Ua9aswdq1awEAzzzzDM6dOwcA2LFjB+6//34cPHgQNTU1mDNnDl599VX5o/aAfbInk8DGjeGO\nhRBDakyjjv0d6C7qFFP/ZBX1L37xi5iYmMh5kL179wobkCx6e4GKCnMygJaWlrCHEDp2TP0KgAnf\nZTLJvgsVRN3P92nadRoGkVlRevkycMcd5mQAJgiRX+yY+hUA3b9Ly2Ln9R13mCHqDQ3AtWvAwIC4\nMUWJyIh6by9w553miDrBYnr77cClS8D162GPJjwuXQJmzQJqatiNTmcuXwY++1mgthbo6Ah7NHoS\nKVFvbgY6O4HR0bBHQ4jAnqovWwacPh32aMIjkQDq6oDycjUydT/09rK/o66OSjBeiZSo33Yb+6ez\nM+zRECKwBSAej/YMLJlk34FJoh71mPohUqJOGYBZpMY0ygKQTFKmTkwSCVEfGQGGhoCSEsoATOLy\n5cmsLsoCkEhQpk5MEglR7+sDysqAvDw6WUyitxeYP59iapdf5s83o1GaGlMPCyojTyRE3b77AzSt\nMwXLAvr72c06yjEdHQU+/hhYvtysTL2kBCguBtI2hSUcEDlRj3pWZwoDA8Ds2UBhIXsEbnxcf0Hz\nwtmzQGUlMHMmu8H19+ud3dK16p/IiXpVFXDlCjA4GO6YCH+kxjQWi26z1G6SAuwGN3u23ot2aFbt\nn0iIul2nA1hdvaaGFjboTmpMgeg2S+0mqc38+frOWK5fZ5t52bt7U6bujUiIeurdH6CTxQQopgy7\nSWpTXq5vs9SOqW29ENWY+iWSok7TOv2hmDLs1aQ2OjdLKaZiiKSoUwagPxRTRqZM3RRRX7YM6Opi\n60wI50RS1CkD0J/0mNbWAh99BDjYKdoYBgdZUzTVaMwkUS8sBKqrgTNnwhuTjkRC1NObarW1tLBB\nd9JjOmcOE4Sb/i2RIJlk53JeylWsc6PUXkyWSlRnYH6IhKinZwBlZey53p6e8MZE+CM9pkD0BCC9\n9ALo3Si1t31IJWoxFUEkRR2gEozuUEynN0kBs8ovQPRiKgLjRX1iYnI5eSqUAegNZer8TN0kUY9a\nTEVgvKgPDLB6a0HB1J9TBqA3vKwuSgKQuprUxjRRp+vUPcaLenpDzYYyAL3JFNcorSq1fUlra6f+\n3LRG6W23kV+pW4wX9Ux3f4BEXWeGh5moFRVN/fnixdHxK7V9SefNm/pz0xqlsRj5lbolsqJeU0N+\npbqSvpzcJj8/On6lmZqkALvRWRa78ekG71qlEow7IivqM2eSX6mu8GIKRGcGlqlJCrAbnY519bEx\ntpiqtHT676ISU1FEVtQBygB0JVdMoyAAmZqkNjqKen8/M8aYMWP67+g6dYfxos5rlAKUAehKrphG\nQQDSt9xNRcdmaaYmqQ1dp+4wXtRpqm4eFFN++QXQs1maqUlqQ36l7oi0qNO0Tk+iHtNUX9JM6Fh+\nyRZT8it1R6RFPSpZnWlki2kU/EpTfUkzYZqoA3StuiHSok5+pXqSLaZR8CvN1iQFzBT1KMzARGG8\nqGdrqpFfqZ5kiylgfrM0W5MUMK9RClCm7gajRd2yaFpnIlGPabYmKWBeoxQwP6YiMVrUh4fZdDx9\nOXkqNK3Tj6hP1XmrSW2o/BJtjBb1XCcKQBmAjlCmnjtTN03Uya/UOZEXdcoA9GJ0lO3al2k5uY3J\nfqWZfEnTMVHUya/UOUaLeq6GGkB+pbrR18d2JszLcuaa7FeayZc0ndJSduPTabO6XI1SwPwZmCiM\nFnUnmTr5leqFk5gC5gpArtILwAR/3jx2A9QBJw80AObGVDSRF3WASjA6EfWY5mqS2uhUgrl6le0N\nX1iY/XWmxlQ0JOqgDEAnoh5TJ5k6oJeoRz2moiFRh/mLVUwi6gIQdVGn6zQ3Rou6k0YpYP6ycpNw\nE1PTBMD2JXUi6jqtKnXSJAXYEz/kV5obo0U96lmdiTiNqYl+pTxf0kzotKo012pSG/IrdQaJOsiv\nVCecxtREv1KnTVLAzPILYOYMTDQk6iC/Up1wIwCmzcCcll4Ac0XdtJjKgET9JtSE0YMoCwCJOl2n\nTjBW1EdGgKEh5priBGqW6oHTRilg3lTdTfnFxEYpQNepE4wV9b4+tlo023LqVEzL6kyFMnVnrzWx\nUQqQX6kTckreI488goqKCjQ0NGT8fVtbG0pKSrB27VqsXbsWTz/9tPBBesHNxQ+Yl9WZiGUB/f3s\nZu0Ek2Kay5c0HVPLL+RXmpucor5t2za0trZmfc3GjRvR3t6O9vZ2PPHEE8IG5we3om5aVmciAwPA\n7Nm5l5PbmORXmsuXNJ2yMnYD1CGjpWtVLDlFfcOGDZiX48FYS8Ezx+2JUlXFLgLyK1UXtzGNxcwR\nADelF4Dd+GbP1mOhjhdRN2UGJoN8vweIxWJ499130djYiMrKSjz33HOor6/P+Nqnnnrq1n+3tLSg\npaXF78dzcdNQA1jt3V7YsG6dtGERPnAbU2CyBHPHHXLGFBRumqQ2drM0297zYXP9OjA2xkoqTjG9\nWdrW1oa2tjbP7/ct6uvWrUNXVxeKiopw6NAhbNmyBUnON54q6rJxe/cHJrM6EnU18RNT3UkmgcZG\nd++xm6VO6/BhYMc0FnP+nngceOcdeWMKm/SEd/fu3a7e7/vpl7lz56Lopgnofffdh9HRUfQpsJGz\nFwEwqbFmIlGOqZdMXYdmaZRjKgvfot7T03Orpn78+HFYloUyp48nSCTKWZ2pRDmmbmvqgLmiTn6l\n2clZfnnooYfwzjvv4PLly6iursbu3bsxenOTlB07duCtt97Cj370I+Tn56OoqAhvvPGG9EE7wasA\nvPiinPEQ/vES01S/UqdrFlTDiS9pJkwV9cJC9mDDmTPAihVyxqUzOUV9//79WX+/a9cu7Nq1S9iA\nROGlqZa6sMFNjY8IhsuXgdWr3b0n1a90yRIpw5KOE1/STOiwqtTNatJU7GYpifp0NM1dcuMlAyC/\nUrXxElNA/xKMl9ILoMeqUjerSVPRPaYyIVFPg5ow6hLVmHppkgLmll8A/WMqEyNFfWLC3XLyVCgD\nUBfK1N1hsqjrHlOZGCnqAwOsllpQ4P69tFpNXaIqACTq06HrlI+Rou6lSWpj+mo1nfEaV52n6m58\nSdMxuVFKfqV8jBR1r3d/QP+szlSGh5nA3Vzn5gqd/Urd+JKmY3KjlPxK+ZCop0F+pWriZTm5jc5+\npV6bpAC7AVoWuyGqip9rVecZmExI1NMgv1I18RNTQN8ZmNfSC8BugCrX1cfG2MIqrxuO6RpT2Rgr\n6l5r6gA1YVREREx1FAA/og6oXVfv72eCPmOGt/fTdZoZI0Xda53Ohpql6iEipjoKgJ/yC6B2pu53\n9kXXaWaMFPWoTtVNJqox9Zupq9ws9XujJr/SzJCoZ0DXrM5kohhTt76kmTA5Uye/0syQqGdA16zO\nZPzGVEe/Ure+pJkwWdQBulYzYayo+2mqkV+peviNqY5+pX5LL4DajVK/MQWoWZoJI0Xdb60u1a+U\nUAO/MQX0K8H4bZIC5mfq1CydjnGiblk0rTORKMZURKZucqMU0C+mQWCcqA8Ps6m2l+XkqeiW1ZmO\nqKxOp5hSpp4b3WIaBMaJuogTBaAMQDUoU/eG6aJOfqXTMVLU/TZfAGrAqMToKNuRz+tycptUv1LV\n8epLmo7pjdJUv1KCYZyoi6jTAbSwQSX6+tguhX6No1P9SlXHqy9pOiUlwNCQmhvUiZpVU7N0KsaJ\nuqgThfxK1UFUTAF9SjAiSi8AuymUlrIbo0qIeqAB0CemQUGingVqwqhBFGMqoklqo2Jd/epVtk98\nYaH/Y+kS06AgUc8CZQBqEMWYisrUATVFPYoxDQojRV1EoxSgZqkqiI6pDgIgUtRVbJbSdSoP40Rd\nVKMUoAaMKoiOqeoC4MeXNBOmZ+rkVzoV40SdpnXmITKmOviV+vElzYSKq0pF3qjJr3QqJOpZIL9S\nNRAZUx38SkU2SQHzM3VAjxlYUJCoZ4H8StVAtACoPgMTWXoBoiHqqsc0SIwUdVENGICaMCogI6Yq\nC4BoUTe9UQrQdZqKUaI+MsI29CopEXdMapaGj8j6K6D+VJ3KL+6h63QSo0TdXk4ei4k7pupZXRSI\n2lRdRvnF5EYpwBqltK0HwyhRF33xAzStCxvLYi5UZWXijqlyTEX4kqYThUy9tJTt7UN+pSTqOaFp\nXbgMDACzZ4tZTm6zYIG6fqUifEnTKStjN0aVsli6VuVhnKiLbL4A5FcaNjJiqrJfqejSC8BuiEVF\nai3OkRFXlWdgQWKUqIuu0wHkVxo2MmIKqNssFd0ktVGpBHP9OjA2xsolIqFMnWGUqMuY0gHqZnVR\nIGoxlZGpA2o1S+2YinygAVA3pkFDou4AmtaFR9RimkjIE3VVMvWoxTRoSNQdQNO68IhaTJNJ88sv\nsmK6fDn5lQIGirro5gtAGUCYyIqpin6lonxJM6HSqlJZMSW/UoZRoi6rqUZ+peEhK6Yq+pWK8iXN\nRBQydUDdGViQGCXqsk4W8isND5kCoFpjTVaTFFCrUSrrRg2oF9MwIFF3CJVgwiFKMZXVJAWik6mr\nFtMwMEbUJybELydPhaZ14RClqbqsJikQHVFXLaZhYIyoDwwAxcVAQYGc41MGEA6ymmqAelN1meWX\nKDRKAbpOAYNEXWadDqAMICxkxlWlVaWifUnTiUqmTn6lBom6zBMFUC+riwLDw0zsiorkHF8lv1LR\nvqTpRKVRSn6lJOqOWb6c/EqDRtZycpv8fGDpUjX8SmU2SQF2Y7QsdqMMmyASMFVmYGGQU9QfeeQR\nVFRUoKGhgfuaxx57DLW1tWhsbER7e7vQATpF9okyaxb5lQaN7JgC6pTVZDZJAXZjVKEEMzbGFlmV\nlsr7DFViGhY5RX3btm1obW3l/v7gwYM4ffo0Ojo68PLLL2Pnzp1CB+gUmc0Xm6hnAEETVExVEACZ\n9XQbFZql/f1M0GfMkPcZUb9O83O9YMOGDejMkp4eOHAAW7duBQA0NzfjypUr6OnpQUVFhbBBOkF2\noxRgGcAvf+lvZWleHnD33SzzD4vhT8fwDwfPYmFBra/jfOELgMwwBxXTw4flfoYT3utM4tE/XQ5A\nntqpkKkHNfv67nfZteqHVauAZcvEjClIcop6Lrq7u1FdXX3r/6uqqnD+/PmMov7UU0/d+u+Wlha0\ntLT4/fhb9PYCa9YIO1xGvvIV4PnngZdf9n6MEyeA730P+Iu/EDcut3zv5/+K7777f/ClTu+lss5O\nYNMm4PvfFzeudIIQgHgc+Pu/l/sZTjjyuc3YOu//AmiR9hkqNEuDuFGvWgU0Nvq7Tj/5hDlkHTgg\nblxOaWtrQ1tbm+f3+xZ1ALDSUtcYp7OVKuqiCUIA/vzP2T9++Ku/Cn+6/97HCaA8gbdfnEBezFuv\n/O23gR//WPDA0ghK1MOOx/Cnoxgt/ghDsxKQLepRyNSLioCf/czfMU6eBL72NTHjcUt6wrt7925X\n7/f99EtlZSW6urpu/f/58+dRKWObuRwEcbKIQIV6X0d/AmOx6+i+2u35GEH8HUHEVAW/0sMfnAVm\njOGjAblfaFREXQTLlzMDcB2fdvMt6ps3b8brr78OADh69ChKS0sDr6cDwTTVRKBCZ/7ijSQKYoVI\n9nofSBB7VwcRUxX8So/8IYnYhL94OEGFRqku16nOT7vlFPWHHnoId955JxKJBKqrq/HKK69g3759\n2LdvHwDg/vvvx7Jly1BTU4MdO3bghz/8IfdYI6Pj4kaeRhC1OhHYGW6Y2/gOFCTwJ4takOj1nhkG\nsXd1UDENe2Xpia4Eqsf9xcMJlKm7Q4VZtRdy1tT379+f8yB79+519GHHE1344uoljl7rBsvS52SZ\nP589zmU3YoLmUt81TMzsxX9feY/vzNCedaxYIWhwaQQV07Az9dP9SXx+wX345cD/xsj4CApnFEr5\nHFUapbo8UaLCrNoLga4oPfyhnG9oeJhNo2UtJxdNmCLS9vvTmDlcg/oFK31nhrL/jiBFPcyM7MJI\nEs1LGlD9mWqc7T8r7XMoU3dH2OeFVwIV9fc+lvMN6XSiAOGeLL/tSKAcccTL474zddl/R1BxDTsj\nGyxM4M9WsZjILMGQqLsj7BmcVwIVdVmNIF2aLzZhisgHF5NYPKcOS0uXovtqN26M3fB8LJl/x+go\n221P5nJymzD9Si/0DmKiYAC3xytRN79OarOUGqXuCPtm75VARb37UzlZiC5NUpswM/UzAwnUV8RR\nMKMAi0sX46P+jzwfS+bf0dfHdiyU4deZTph+pb85kcSs4Vrkz8hDvExupl5SAgwNhfuYnk6ZelUV\n29ZgcDDskbgjUFEfyJeXqetyogDhZgCfTCTxheVskxG/JRiZe1cHHdOwptrHTicxPyYmHrnIy2Mz\nn74+aR+RFZ0eaADY96XjNr6Bivr47Evouyp+82qdThQAqKlhjwKOy3vCMyMTExaGZiVxVwPbDrCu\nvA6Jy94zQ5l7V4ch6mHMnt6/mMSS4pvxkFx+AcKtq1+9yp7/LpTzcI8UdGyWBirqhUNL0fZ78ZtX\n6ybqs2ezjbCCXtjw4cd/RMyagdoq9mXFy+NI9qnZLA06pmHNnjoHE1i1kGXqt829DYM3BnH1xlVp\nnxemqOt2nQJ6NksDFfUyqw6/lfAN6dR8sQlDRNo+SGLOp5ObdteV+88MZf0dQcc0rIuXlcNYTPJi\neagtrzW2WUrXaTAEKurVRXH8V7f4tE63RikQzrTud2cSqMif3LQ7Xh73VX4B5P0dQcc0jFWlExMW\nhmcncdeayS2QRcQkG5Spu4PKLzmor6jDmQE5mbpuJ0sYGcCpPyaxvHRS1BcWL8SnY5+i/3q/52PK\nzNSDjGkYfqW/P3sJsfFZWLpo0phUREksG2GuKtU1+Uomw93Wwy2Bivr6pXH0jJGoA+FM97uGk1hz\n22T5JRaL+X7iorZWzkkfdEzD8Cttez+JuTemetiJKIllgzJ1d5SVATNnshu+LgQq6i2r4zf3jBaL\njrW6MKZ1fbEE7kjzTPMr6qWl7DnvCxf8jm4qYcQ06NnTe50JVBRMj4fM8gvV1N2jW7M02PLL4gWw\nYuPoOC/2rNIxA/jc59h0dGgomM/7dGQMI3POomVNzZSf15XX+Z7uyxDDMGIa9MX7h0+mlsOAyZts\nuvGMKChTd49uzdJART0vL4Y5n8bxm9+Ly0RGRtiGXiUlwg4ZCDNmsD3Jg1rYcORkJ2ZcX4iyz8ye\n8nNVm6Vh1F+DbpaeG06gqWpq+aV0VinmFM7BhUHBU5+bkKi7R7dmaaCiDgAL8urwuzPibnv2cnKO\ng57SBJkB/PupJErHp9vVi1jFSJm6N/rzkrizTk5MeFCj1D2UqedgaUkcH/aI+4Z0vfsDwYpI+7kk\nbptZN+3n8fI4Ovo6MGF5381K9N9hWWzPjbIyccd0QpAZ2fCnoxgt+hgbG5ZP+53MZill6u6hTD0H\nqxfF8fGQuG9I1+YLEOzJ0tGfQLx8elY4d+ZclMwsUcqvdGCA7Y0f9HLyIP1KD39wFvnXK/GZOTOn\n/U7mFrzl5eyGGcYjerpeq7r5lQYu6nfG69ALytSBYKd1F28kcfvi6Zk64H/PEdF+pWHFNEi/0iN/\nyFwOA+Rm6gUF7IYpYxO2XOh6rermVxq4qN/VWIsbRaeF+ZXqWqcDgl3YMFCQwIb6zCLiNzMU7Vca\nZkyDmj2d6Eqgalbmm6yJZhnXrwNjY+zxVx3RqQQTfKO0dA5m3JiP44kuIcfT9e4PsKloXh7zK5WJ\n7UvavKI64+/jZWo1S8OMaVCzp9P9SdTNz3yTXTZvGc4NnMPIuKCpTxphNEvtmOr4QAOgV7M0cFEH\ngM+MxoX5leos6kAw033bl7Qgf0bG39fNr1PKrzTMmAaVkV0Y4ZfDZubPlOpXGkambsJ1Spl6FhbN\njAvzK9W1+WITxMli+5Jyx6CYX2mYMQ0qI7N9SXnILMGEsarUhOuUMvUsxMvENYJ0zwCCEBHbl5SH\nan6lYca0pobt/yLTrzTVl5SHTMMMytTdQ+WXHDRVx4X5lercKAWCyQBsX1IeqvmVhhnT4mL5fqWp\nvqQ8ZPqVkqi7Rye/0lBEfUN9nTC/Ut1PliCWpqf6kvJQya807JjKzspSfUl5mLaqVPfkSye/0lBE\n/c76xcL8SsMWAL/I9itN9yXloZJfadgxld3nSPUl5UHlF/XQpVkaiqjPKswX4lc6MQFcuRL8cnKR\nyPYrTfcl5aGSX2nYTTXZmXqqLykPmX6l1Cj1hi7N0lBEHRDjVzowwBYzFBQIGlRIyBSRdF9S7hh8\nZuqAuL8j7KxO9sXrpBwm06+UMnVv6NIsDU3Uq4vi+P0FfyKie53ORqaIpPuScseg0GONYcdV5jR7\n0pfUWUxkGGaQqHuDyi85qK+ow0dX/ImICScKILdZmu5LykMVv9LhYbZtQlGRv+P4YckSeX6lmXxJ\necjyK6VGqTd08SsNTdRF+JWaIuoyM/V0X1IeqviVqrCcXKZfaSZfUh6yNvYqKmIxGh4WfmguJlyr\nuviVhibqIvxKTWi+AHKndZl8SbnjUMCvVJWYyqqfZvIl5SGr/BKLBdssHRtjz3eXlgbzeTLRoVka\nXvlFgF+pCXd/QJ5fKc+XlEdduf89YPyKoSoxlXXxZvIl5Y5Bol9pkHX1/n4m6DMybz2kFTo0S0MT\ndduv9N/e9y4iJtTpgEm/UtHTfZ4vKQ8VmqWqxFTW7OnccAKNlc7KLzL9SoMUdVVu1CLQoVkamqgD\nzK/0+EfeRcSkk0VGs5TnS8pDBb9SVWIqKyPrz0viT1cEG5NMBNksVeVGLQLK1HPg169UFQEQgYzp\nPs+XlDsGBfxKVYmpjIwsmy8pD1nNUsrUvUGZeg78+pWq0lQTgYyThedLykMFv1JVYirDrzSbLykP\nWVvwBtkoVSWmItDBrzRUUffrV2pSBiBjWpfNl5Q7Dp+GGX79SlWJqQy/0my+pDwoU1cLHfxKQxV1\n2690dMzbblYm1epkLGzI5kvKHYfPGq5fv1KVYip69pTNl5Q7BkmZOom6d1QvwYTbKL3pV3rsD+79\nSi3LrJNFtF9pLl9SHmH7laoUU9Gzp2y+pDxk+ZVSo9Q7qjdLQxV1wLtf6fAwmyKHuZxcNCKn+7l8\nSXmE7VeqkqiLzsgujCSxbrE7UZflV0qZuncoU8+BV79Sk5ovNiJPlly+pNwxhPysukpxFZ2RDRYm\nsHGVu/ILIKcEQ41S76i+qjR0UffqV2ra3R8QKyK5fEl5hOlXOjrKVtWWlHj+aKGI9Ct14kvKQ4Zh\nBmXq3qHySw6aquO4cMN9FmJanQ4QmwHk8iXlIcqv1Mvf0dfHlpPnhX5WMkT6lTrxJeUhw6+0pITd\nQIN4NM80UVfdrzT0y2dDfR2uzKBMHRC7qtSJEQMPEX6lg4PAVZemPSrGVFRW5sSXlIeMVaV5eewG\n2tcn9LDTMO2BBkB9v9LQRd2rX6lpJwogzq/UqS8pD1F+pW7FUMWYiupzvH8xicXF3kRdll9pECWY\nq1fZs92FhXI/J2hUbpaGLupe/UpNa74Ak36lH3/s7zhOfUl5hOVXqmJMRWXqnYMJrF7o7SYry680\niGapijEVgcrN0pyi3traihUrVqC2thbPPvvstN+3tbWhpKQEa9euxdq1a/H000+7HoQXv1IVszoR\niCjBOPUl5Y4hJL9SFWMq6uL1Uw6T5VcaRKauYkxFoHKzNKuoj4+P49FHH0Vrays+/PBD7N+/H6dO\nnZr2uo0bN6K9vR3t7e144oknXA/Ci1+piY1SQIyIOPUl5Y5B0GONbv8OFWMqYprtxpeUOw4Jhhkk\n6t7Rtvxy/Phx1NTUYMmSJSgoKMCDDz6It99+e9rr/G7i78Wv1NSTRUSm7tSXlIcov1Iv5RfVYirC\nr9SNLykPGX6lQawqVfFGLQKV/Urzs/2yu7sb1dWTy8yrqqpw7NixKa+JxWJ499130djYiMrKSjz3\n3HOor6/PeLynnnrq1n+3tLSgpaUFAPMr/aczP3E1cBUFQATxOJDhvumKruEkNi5/2PP7U/1Km6ua\nPR0j1a/Uqd9oby97n0qk+pU2NHg7Rtv7SRTf8H6TBVhJ7J87/tnXMdKhTN07qX6lixaJPXZbWxva\n2to8vz8cdY1IAAAQrElEQVSrqMccXI3r1q1DV1cXioqKcOjQIWzZsgVJzrw7VdRTaVkdx9ARd2kd\nNWD4uPEl5Y7Dp6in+pVWOlxvo2pM7fqpV1F/rzOBhQXeexwAi8f3f/t9X8dIZ/58+Y/lqRpTEdjX\nqmhRT014AWD37t2u3p+1/FJZWYmursnNtrq6ulBVVTXlNXPnzkXRzQ1Y7rvvPoyOjqLP5cOvXvxK\nTc0APvc5tqmXV79St76kPMLwK1U1pn5vtG58SbljkOBXSpm6P1RtlmYV9fXr16OjowOdnZ0YGRnB\nm2++ic2bN095TU9Pz60T7fjx47AsC2VlZe4G4dKvdGSEbeilynJykfj1K3XrS8ojjGapqvVXv00x\nN76kPGT4lZKo+0PVZmlWUc/Pz8fevXtx7733or6+Ht/4xjewcuVK7Nu3D/v27QMAvPXWW2hoaEBT\nUxMef/xxvPHGG54G4savtK8PmDfPea1WN/w0S936kvIQsYmU279DVQHwm5G59SXlIXplKTVK/aFq\npp61pg6wksp999035Wc7duy49d+7du3Crl27fA/EjV+pqhe/KPxM9936knLHUB5HRy/zK82LeVuj\nFo8D77zj7LWWxfbTcDnJCwQ/GZkXX1IetgvSXUvv8n0sgDJ1v2iZqQeJG79Sk5svgD9Rd+tLymPu\nzLkonVUamF/pwADbG1/F5eR+/Eq9+JLyEL0Fb3k5u5HKfCzP5GtVVb9SZUTdjV+pyXd/wF/5xYsv\nKXccAfqVqhxTP36lXnxJeYj2Ky0oYDfSgQFhh5yGynH1i6p+pcqIuhu/UpPrdIC/hQ1efEm54xDk\nV3rWgWmP6jH1OtU+0ZVA5Sxx8RC9Ba/MEsz168DYGHu01VRULMEoI+pu/EpNvvsD3v1KvfqS8hCx\nj7fTWYfqMfXaFDvdn8SK+WJmTjL8SmU2S+2YmvpAA6Bms1QZUQec+5WqLgAi8DLd9+pLykPElq9O\n/w7VY+o1I/PiS8pDhl+pzExd9ZiKgDL1HDj1KzW5+WLjRdS9+pJyxxCgX6nqMfWakXn1JeUhugQj\nc/td1WMqAhW34FVK1J36lUYhA/DSLPXqS8ojSL9S1WPqxa/0Qu8gJgqvePIl5SHaMIMydX9Q+SUH\nTv1KVW+qicBLBuDVl5RHkH6lqsfUi1/pb04kMWvImy8pD9F+pSTq/lDRr1QpUXfqVxqFk8VLpu7H\niIGH3328nfqV6hBTt1kZ8yUVN3MC9FpVqvqNWgQq+pUqJepO/UqjUKtz61fq15eUh99no536leoQ\nU7dNMT++pDxEl1+opu4f1ZqlSom6E7/SiQngyhU1l5OLxK1fqV9fUh5B+ZXqkKm7LYl1DiawyqMv\nKQ/RfqVUfvGPas1SpUQdyO1XOjDAFjPk59y1Rn/clGD8+pJyxxCQX6kOAuC2/PLJRBLNgsthov1K\nSdT9o1qzVDlRz+VXGoU6nY2bDMCvLyl3DAFtwatDXN1Ms0X4knLHIdCvlETdP1R+yUEuv9KonCiA\nuwzAry8pjyD8SoeH2ZYIN71WlMWNX6kIX1IeIv1KqVHqH9X8SpUT9fVL4+gZyy7qUWi+AO4ygK7h\nJNbcJr78kupX6pVUv9JM2DFVfTl5ql9pLkT4kvIQubGXfSMdHhZyuClE5VpN9StVAeVEvWV1HEOz\n+EoWpUzdTflFhC8pdxw+RT3VrzQTOsXUaUxE+JJyxyCw/BKLySnBjI2xR1lLS8UeV1VUapYqJ+q5\n/Ep1EgC/OPUrFeVLykO2X6lOMXVaEhPhS8pDtF+pDFHv72eCnqecwshBpWapcl95Lr/SqNTpAOd+\npaJ8SXnIbpbqFFOnJbFzwwmsqZQj6qL9SmWIuk43ahGo1CxVTtSB7H6lUTtZnGQAonxJecj2K9Up\npk4zsv68JL64Qk75BRC7slRGs1SnG7UIKFPPQTa/0qg0X2ycZACifEm5Y0jxK/V8jCyZuk4xdRIP\nkb6kPEQ2S2WsKtUppiKgTD0H2fxKdcrqROCkASPKl5SH7Vd6/up5z8fIdtLrFNMFC1gTMJsIivQl\n5SFyC14qv/hHJb9SJUU9m19p1E4WJ6tKRfqScsfhc8+RbH6lOsU0Fss91RbpS8pDZKZOou4flfxK\nlRT1bH6lUavVOVnYINKXlDsOiX6lusU011RbpC8pdwyUqSuHKiUYJUWd51dqWdE7WWy/Ul4jS7Qv\nKQ+ZfqW6xTRXpi7Sl5SHSL9SapSKQZVmqZKiDmT2Kx0eZgKn+nJy0WTLAET7kvKQ6VeqW1MtV0Ym\n0peUh0i/UmqUioEy9Rxk8ivVLaMTRbZmqWhfUu4YBKxi5J30usU1V/N6sDCBPxPoS8odh6ASDJVf\nxKDKqlJlRT2TX2kUTxQge7NUtC8pj6WlS3Fh8IJwv9LRUbZitqTE5wADpLaW71dq+5KuF+hLykOU\nYQaJuhio/JKDTH6lUazTAdkzANG+pDxk+ZX29em3nDybX6kMX1IeovxKS0rYjVXk43hRFHVV/EqV\nvZQy+ZVG8UQBsmcAMnxJecjwK9U1pryYyPAl5SFqVWleHrux9vUJGBSi+UADoI5fqbKinsmvNIrN\nF4DvVyrLl5SHDL9SXWPK6w/I8CXlIdKvVGSz9OpVZsdYWCjmeDqhQrNUWVG/5Vf6/uRuVlG8+wPs\nAlmwYLpfqSxfUh4iGnPpJ72uMeWVxDoHgymHAWL9SkXW1XWNqQhUaJYqK+rATb/SxOQ3FOWTJVOz\nVJYvKXcMAlYxppctdI0pr/zyyUQSf1ITTExE+pWSqItBhWap0qKe7lca1UYpkDkDkOVLyh2DhC14\ndY1ppmm2TF9S7jgEGWaQqIuByi85SPcrjfLJkikDkOVLykOGX6muMc3kVyrTl5SHKL9SkatKdb1R\ni0AFv1KlRT3dr1TXppoIMmUAsnxJecjwK9U1ppn8SmX6kvIQtbGXyEaprjEVgQp+pUqLerpfqa5Z\nnQgyPuMt0ZeUOw6fzVLbr/TiRfb/Osc0PSbMlzSEeFD5RSnCbpYqLerpfqVRPllsv1Lb9V22LykP\nUc1Se9ahc0zTS2LMlzS4mRMgzq+URF0cYTdLlRb1VL/SkREmaDotJxeJ7VdqL2yQ7UvKQ3SzVOf6\na3pJTKYvKQ9RfqUk6uIIu1mqtKgDk36lfX3AvHlsAUtUSc0AZPuS8hDtV6qzAKRnZLJ9SXmIuNFS\no1QclKnnwPYrjXLzxSY1A5DtS8odg0C/Ustie2XoKgCp8QjCl5SHiJIYNUrFQZl6Dmy/Up0zOlGk\nli1k+5LyEOVXmkwCAwNsb/yCAoEDDJBUv1LmS3qbVF9SHiJmT2Vl7AYr4lG8qF+rYfuVKi/qtl9p\n1E8UYOq0LghfUu44BPiVnjvHnoDROaapfqXMlzSkeAjI1AsK2A12YMD/eKJ+rYbtV6q8qNt+pZd6\nxiN9ogCT0zrLCsaXlDsOn4/R2X6lv/ud/he/HZMgfEm5Y1DILOP6dTZ7mTPH93C0JswSjPKibvuV\nHvmgS3sB8IvtV3qyIxhfUh7xMv+rGOvqgN/+Vn9RtzP1IHxJeYjyKxXRLLWz9Cg/0ACE2yxVXtQB\n5lf6biIZ6eaLTTwO/PRgML6kPET5lb77rv4NNTsjC8KXlIcov1IRzdKoN0ltKFPPwaKZcXQOJrTP\n6kQQjwOHjgfjS8odgyC/0g8+0D9Tt5u+g4XhlcMAMSUYEeWXqNfTbcJcVZpT1FtbW7FixQrU1tbi\n2Wefzfiaxx57DLW1tWhsbER7e7vwQcbL6mCVJelkAZvWnbyUROnlWaGNQZRf6cSEGgLQ1tbm+b21\ntUCik/mSfr6uStygXCJi9iRK1Ccm2vwdxACULb+Mj4/j0UcfRWtrKz788EPs378fp06dmvKagwcP\n4vTp0+jo6MDLL7+MnTt3Ch9kU3UcmE+ZOsAygPF5Ccy8PBTaGGy/0tN9p3O/mIO9ZY0KMfUj6sXF\nQOny4HxJeYjwKxUl6kNDbf4OYgBh+pVmPQuPHz+OmpoaLFmyBAUFBXjwwQfx9ttvT3nNgQMHsHXr\nVgBAc3Mzrly5gp6eHqGD3FBfB5RTpg6wDADlSVSWhftl+F3FWFnJHqEzIaYLVgTnS8pDlVWlly8z\np66oE6ZfaX62X3Z3d6O6evIJi6qqKhw7dizna86fP4+Kigphg7yzfjEwpwd/efzLKPwvYYfVkvFx\nAAvex9Lhu0Idx4r5K/DXbX+NV0684vkY+f8DeOET4B/2CxyYB5IfJPHe/vc8v/9C7UdYlfc1gSNy\nz4r5K/AfF/4DX97/Zc/HuPApcPIG8P/+0vs4rl0D5hQk8WUf36cpfHIP8LP//F9Yt+6/Bfq5WUU9\n5vC5pPQd4njvc3o8Hr/Gr3y93yR+gL/FD57/27CHgQ/wga/3HxU0Dr8k/8lflvvv+BvEvvs3gkbj\nnV8JuEb+6PP9wwB+9W8he7opwh78Cnv+Z7CfmVXUKysr0dXVdev/u7q6UFVVlfU158+fR2Vl5bRj\n+d0alCAIgshN1pr6+vXr0dHRgc7OToyMjODNN9/E5s2bp7xm8+bNeP311wEAR48eRWlpqdDSC0EQ\nBOGcrJl6fn4+9u7di3vvvRfj4+PYvn07Vq5ciX379gEAduzYgfvvvx8HDx5ETU0N5syZg1dffTWQ\ngRMEQRAZsCRz6NAhq66uzqqpqbH27Nkj++OMZ/HixVZDQ4PV1NRkff7znw97ONqxbds2a8GCBdbq\n1atv/ay3t9e65557rNraWmvTpk1Wf39/iCPUi0zf55NPPmlVVlZaTU1NVlNTk3Xo0KEQR6gP586d\ns1paWqz6+npr1apV1gsvvGBZlvvzU+qDtU6ecyfcEYvF0NbWhvb2dhw/fjzs4WjHtm3b0NraOuVn\ne/bswaZNm5BMJnH33Xdjz549IY1OPzJ9n7FYDN/+9rfR3t6O9vZ2fOlLXwppdHpRUFCA559/HidP\nnsTRo0fx0ksv4dSpU67PT6mi7uQ5d8I9FjWdPbNhwwbMmzdvys9S11ps3boVv/jFL8IYmpZk+j4B\nOke9sHDhQjQ1NQEAiouLsXLlSnR3d7s+P6WKeqZn2Lu7u2V+pPHEYjHcc889WL9+PX784x+HPRwj\n6OnpudXcr6ioEL54Lor84Ac/QGNjI7Zv344rV66EPRzt6OzsRHt7O5qbm12fn1JF3e9z6cR0jhw5\ngvb2dhw6dAgvvfQSDh8+HPaQjCIWi9F565OdO3fi7NmzOHHiBBYtWoTvfOc7YQ9JK65du4YHHngA\nL7zwAubOnTvld07OT6mi7uQ5d8IdixYtAgB89rOfxVe/+lWqqwugoqICly5dAgBcvHgRCxYsCHlE\nerNgwYJb4vOtb32LzlEXjI6O4oEHHsDDDz+MLVu2AHB/fkoVdSfPuRPOGR4exuDNHYKGhobw61//\nGg0NDSGPSn82b96M1157DQDw2muv3bqYCG9cvHjx1n///Oc/p3PUIZZlYfv27aivr8fjjz9+6+eu\nz0/Zj+kcPHjQisfj1vLly61nnnlG9scZzZkzZ6zGxkarsbHRWrVqFX2fHnjwwQetRYsWWQUFBVZV\nVZX1yiuvWL29vdbdd99NjzR6IP37/MlPfmI9/PDDVkNDg7VmzRrrK1/5inXp0qWwh6kFhw8ftmKx\nmNXY2DjlcVC352fMsqhNTRAEYQpaOB8RBEEQziBRJwiCMAgSdYIgCIMgUScIgjAIEnWCIAiDIFEn\nCIIwiP8PMmGiAUskHPEAAAAASUVORK5CYII=\n"
      }
     ],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "xs = zeros((N,ni))\n",
      "for i,n in enumerate(seq): xs[i,n] = 1.0\n",
      "imshow(xs[:20,:].T,interpolation='nearest')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 3,
       "text": [
        "<matplotlib.image.AxesImage at 0x3db1a50>"
       ]
      },
      {
       "output_type": "display_data",
       "png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAABQCAYAAADiHjXEAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAACs5JREFUeJzt3W1IW2cfx/HvmcrAdZ0KmjqVnYqOWnA+1OErsdCmnV0J\nykYf2JiopUXWjbK9mO82xx4cDMaYZXTdGK4D1zLoA6ihLSNlONrAiFCwVAdLZ1t1K1JmLawu5H7R\n+3Y6E6M5SU6S+/eBAybnSq5/rqT/xsv/uS4jGAwGERGRtPKI3QGIiEjsKbmLiKQhJXcRkTSk5C4i\nkoaU3EVE0pCSu4hIGsqM9oEzMzPs3buXGzduYJomp06dIicnZ1k70zRZv349GRkZZGVl4fV6LQUs\nIiKRRf3NvaenB6fTydjYGNu2baOnpydkO8Mw8Hg8+Hw+JXYRkQSJOrmfO3eO1tZWAFpbWzlz5kzY\ntrpOSkQksaJO7tPT0zgcDgAcDgfT09Mh2xmGwfbt26mrq+P48ePRdiciImuw4py70+lkampq2f3v\nv//+ktuGYWAYRsjnGB4eprCwkD/++AOn08mmTZtoaGhY1i7c40VEJLxwMyMrJvcLFy6EPedwOJia\nmmLDhg1MTk5SUFAQsl1hYSEA+fn5tLS04PV6Qyb3h95eIRoPsHWlcHmb7hXPR9K9Yv+pJdJYeFh5\nNJNlLKy+p5CI1+Ih3p9NSI73JBavIxIPkUYzOcYiFqznrPCinpZxuVz09fUB0NfXR3Nz87I29+/f\nZ3Z2FoC5uTnOnz9PZWVltF2KiMgqRZ3cu7q6uHDhAk8//TQ//PADXV1dANy+fZvnn38egKmpKRoa\nGqiurqa+vp7du3ezY8eO2EQuIiJhRV3nnpeXx8WLF5fd/+STTzIwMABAaWkpIyMj0Ue3hBmj5xHQ\naMaWaXcAacW0O4A0YfkKVbfbzaZNmygvL+ejjz4K2eb111+nvLycqqoqfD5flD2ZUccoy5l2B5BW\nTLsDSCum3QGkCUvJPRAIcPjwYdxuN6Ojo/T393Pt2rUlbQYHB/nll18YHx/niy++oLOz01LAIiIS\nmaXk7vV6KSsrwzRNsrKy2LdvH2fPnl3SZvHFTvX19dy9ezdsTbyIiMRG1HPuALdu3aKkpGThdnFx\nMVeuXInY5ubNmwsXQC3lWfSziX5BExH5h/+/x2pYSu6rvfDo30X24R+31Uo4IiJpzWTpV95LK7S1\nNC1TVFTExMTEwu2JiQmKi4tXbHPz5k2KioqsdCsiIhFYSu51dXWMj4/j9/t58OABJ0+exOVyLWnj\ncrn45ptvALh8+TI5OTlhpmRERCRWLE3LZGZm0tvby86dOwkEAnR0dFBRUcGxY8cAOHToELt27WJw\ncJCysjIee+wxvv7665gELiIi4VlK7vDPomGGYfDIIw9/ETh06NDCeY/Hw4kTJygtLQUelkbW1tZa\n7VZERFZgKbn/r8794sWLFBUV8eyzz+JyuaioqFjSrrGxkXPnzlkKVEREVi/ude6gzTpERBIt7nXu\nhmHw008/UVVVRVFRER9//DGbN28O+XyNixawNFl7lXsyLAOaiCVRV8PqWKTL64iFZFmuNxniSIb3\nI1aS4TO+9vH0s7TSPXwxZNzr3Gtra5mYmCA7O5uhoSGam5sZGxsL2XarlWBERNKeyWor3eNe5/74\n44+TnZ0NQFNTE/Pz88zMzFjpVkREIoh7nfv09PTCnLvX6yUYDJKXl2elWxERiSDude7ff/89n3/+\nOZmZmWRnZ/Pdd9/FJHAREQnPUnJvb29nYGCAgoICrl69unD/4jr369ev89dff5GRkcHRo0epqamx\n0qWIiKyCpWmZtrY23G532PNay11ExB6WkntDQwO5ublhz2stdxERe1jeZm8l4dZyFxGR+LK8tkwk\nq1/LXVt1iIiszM9qt+uIa3Jf61ruW+MZjIhIyjNJyEVMkWgtdxERe1j65r5//34uXbrEnTt3KCkp\nobu7m/n5eUBruYuI2MlScu/v74/Ypre310oXIiISBcvTMu3t7TgcDiorK0Oe93g8PPHEE9TU1FBT\nU8N7771ntUsREYnA8h9U29raeO2113jllVfCttFmHSIiiWX5m3ukC5lAm3WIiCRa3Ovc17JZRzeN\ni26ZJLrSPRk2Q0gW6fI6wPr7mixjkQwbfiTLWPz//lv1kxR17rC2zTpU6S4ishKTpKhzB23WISJi\nh7gnd23WISKSeJanZSJdyKTNOkREEs9Scp+YmGBqaorc3Fzy8vI4ePAg7e3tS9q8+uqrXL9+naGh\nIYLBII8++qilgEVEJDJLyT0rK4tPPvmE6upq7t27x5YtW3A6nVRUVCy0Wbxhx5UrV+js7OTy5cuW\nAxcRkfAszblv2LCB6upqANatW0dFRQW3b99e0kYbdoiIJF7MSiH9fj8+n4/6+vol94fbsCP06pCe\nRT+baEV3EZHF/CS0zv3evXu8+OKLfPrpp6xbt27Z+dVv2LE1FuGIiKQpk4TVuc/Pz/PCCy/w8ssv\n09zcvOz8WjfsEBER6ywl92AwSEdHB5s3b+bIkSMh22jDDhGRxLM0LTM8PMy3337LM888Q01NDQAf\nfPABv/32G6ANO0RE7GIpuT/11FM0Njby+++/YxgGBw8epKmpaUkbj8fDiRMnKC0tBR6WRtbW1kbR\nmx/9gTWW/Gg8Y8WPxjKW/Gg8rYt7nTvEaj13P3rDY8mPxjNW/GgsY8mPxtO6uNe5g9ZzFxFJtJgt\nHBauzn3xeu67du1idHQ0Vl2KiEg4wRiYnZ0NbtmyJXj69Oll5/7888/g3NxcMBgMBgcHB4Pl5eUh\nnwPQoUOHDh1rPMIxghbnTObn59m9ezdNTU1hyyEX27hxIz///LOW/RURiaO417lrPXcRkcSLe527\n1nMXEUk8y9My8eZ2uzly5AiBQIADBw7w1ltv2R1SSjNNk/Xr15ORkUFWVhZer9fukFJKe3s7AwMD\nFBQUcPXqVQBmZmbYu3cvN27cwDRNTp06RU5Ojs2RpoZQ4/nOO+/w5Zdfkp+fD8CHH37Ic889Z2eY\nKSnu2+xZEQgEOHz4MG63m9HRUfr7+7l27ZrdYaU0wzDweDz4fD4l9ii0tbXhdruX3NfT04PT6WRs\nbIxt27bR09NjU3SpJ9R4GobBG2+8gc/nw+fzKbFHKamTu9frpaysDNM0ycrKYt++fZw9e9busFJe\nkv+yltQaGhrIzc1dct/iPQtaW1s5c+aMHaGlpFDjCfqMxkJSJ/dQa8HfunXLxohSn2EYbN++nbq6\nOo4fP253OGlhenp6YTE8h8OhzWhi4LPPPqOqqoqOjg7u3r1rdzgpKamTe/h13yVaw8PD+Hw+hoaG\nOHr0KD/++KPdIaUVwzD0ubWos7OTX3/9lZGREQoLC3nzzTftDiklJXVy//da8BMTExQXF9sYUeor\nLCwEID8/n5aWFs27x4DD4WBqagqAyclJCgoKbI4otRUUFCz8J3ngwAF9RqOU1Mm9rq6O8fFx/H4/\nDx484OTJk7hcLrvDSln3799ndnYWgLm5Oc6fP09lZaXNUaU+l8tFX18fAH19fSE3rZHVm5ycXPj5\n9OnT+oxGKWZ7qMZDZmYmvb297Ny5k0AgQEdHx7IVJ2X1pqenaWlpAeDvv//mpZdeYseOHTZHlVr2\n79/PpUuXuHPnDiUlJbz77rt0dXWxZ88evvrqq4VSSFmdf49nd3c3Ho+HkZERDMNg48aNHDt2zO4w\nU1LS17mLiMjaJfW0jIiIREfJXUQkDSm5i4ikISV3EZE0pOQuIpKGlNxFRNLQfwCbQMj7QDMZeAAA\nAABJRU5ErkJggg==\n"
      }
     ],
     "prompt_number": 3
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "nh = 3\n",
      "ys = zeros((N,nh))\n",
      "zs = zeros((N,no))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 4
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "A = randn(nh,ni)\n",
      "a = randn(nh)\n",
      "B = randn(no,nh)\n",
      "b = randn(no)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 5
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def sigmoid(x): return 1/(1+exp(-x))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 6
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def forward():\n",
      "    for t in range(N-1):\n",
      "        ys[t] = sigmoid(dot(A,xs[t])+a)\n",
      "        zs[t] = sigmoid(dot(B,ys[t])+b)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 7
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "forward()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 8
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "dzs = zeros((N,no))\n",
      "dys = zeros((N,nh))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 20
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def backward():\n",
      "    for t in range(N-1)[::-1]:\n",
      "        dzs[t] = (zs[t]-target[t])*zs[t]*(1-zs[t])\n",
      "        dys[t] = dot(dzs[t],B)*ys[t]*(1-ys[t])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 21
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "backward()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 10
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def update(eta=0.3):\n",
      "    global A,a,B,b\n",
      "    for t in range(N-1):\n",
      "        B -= eta*outer(dzs[t],ys[t])\n",
      "        b -= eta*dzs[t]\n",
      "        A -= eta*outer(dys[t],xs[t])\n",
      "        a -= eta*dys[t]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 13
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "update()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 14
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "for i in range(1000):\n",
      "    forward()\n",
      "    backward()\n",
      "    update()\n",
      "    if i%100==0: print i,sum((zs[:N-1]-target[:N-1])**2)**.5"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "0 2.08162539801\n",
        "100"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 1.8904233524\n",
        "200"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 1.87767422863\n",
        "300"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 1.86765662972\n",
        "400"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 1.86310296207\n",
        "500"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 1.86075458681\n",
        "600"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 1.85941584069\n",
        "700"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 1.8585984244\n",
        "800"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 1.85806426965\n",
        "900"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 1.85769429465\n"
       ]
      }
     ],
     "prompt_number": 15
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "A = randn(nh,ni)\n",
      "a = randn(nh)\n",
      "B = randn(no,nh)\n",
      "b = randn(no)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 60
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "ns = 3\n",
      "states = zeros((N,ns))\n",
      "S = randn(ns,nh+ns)\n",
      "s = randn(ns)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 61
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def forward():\n",
      "    for t in range(N-1):\n",
      "        ys[t] = sigmoid(dot(A,xs[t])+a)\n",
      "        states[t] = sigmoid(dot(S,concatenate([ys[t],states[t-1]]))+s)\n",
      "        zs[t] = sigmoid(dot(B,ys[t])+b)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 62
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "forward()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 63
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "dstates = zeros((N,ns))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 64
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def backward():\n",
      "    for t in range(N-1)[::-1]:\n",
      "        dzs[t] = (zs[t]-target[t])*zs[t]*(1-zs[t])\n",
      "        dys[t] = dot(dzs[t],B)*ys[t]*(1-ys[t])\n",
      "        v = dot(states[t],S)\n",
      "        dys[t] += v[:nh]\n",
      "        dstates[t-1] = v[nh:]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 65
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "backward()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 66
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def update(eta=1.0):\n",
      "    global A,a,B,b,S,s\n",
      "    for t in range(N-1):\n",
      "        B -= eta*outer(dzs[t],ys[t])\n",
      "        b -= eta*dzs[t]\n",
      "        S -= eta*outer(dstates[t],concatenate([ys[t],states[t-1]]))\n",
      "        s -= eta*dstates[t]\n",
      "        A -= eta*outer(dys[t],xs[t])\n",
      "        a -= eta*dys[t]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 70
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "update()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 71
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "for i in range(1000):\n",
      "    forward()\n",
      "    backward()\n",
      "    update()\n",
      "    if i%100==0: print i,sum((zs[:N-1]-target[:N-1])**2)**.5"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "0 1.85758042421\n",
        "100"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 1.85705193087\n",
        "200"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 1.8567707023\n",
        "300"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 1.85659832279\n",
        "400"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 1.85648265595\n",
        "500"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 1.85640002593\n",
        "600"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 1.85633822734\n",
        "700"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 1.85629036387\n",
        "800"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 1.85625225887\n",
        "900"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 1.85622124147\n"
       ]
      }
     ],
     "prompt_number": 72
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print A"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[[ -5.88549888  -9.06246784  -5.03989056]\n",
        " [ -1.36378817  -0.28744102   3.41206429]\n",
        " [ -6.06789449 -10.10759542  -4.22821784]]\n"
       ]
      }
     ],
     "prompt_number": 73
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}