{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "class Neural_Network(object):\n",
    "    def __init__(self):        \n",
    "        #Define Hyperparameters\n",
    "        self.inputLayerSize = 2\n",
    "        self.outputLayerSize = 1\n",
    "        self.hiddenLayerSize = 3\n",
    "        \n",
    "        #Weights (parameters)\n",
    "        self.W1 = np.random.randn(self.inputLayerSize, self.hiddenLayerSize)\n",
    "        self.W2 = np.random.randn(self.hiddenLayerSize, self.outputLayerSize)\n",
    "        \n",
    "    def forward(self, X):\n",
    "        #Propagate inputs though network\n",
    "        self.z2 = np.dot(X, self.W1)\n",
    "        self.a2 = self.sigmoid(self.z2)\n",
    "        self.z3 = np.dot(self.a2, self.W2)\n",
    "        yHat = self.sigmoid(self.z3) \n",
    "        return yHat\n",
    "        \n",
    "    def sigmoid(self, z):\n",
    "        #Apply sigmoid activation function to scalar, vector, or matrix\n",
    "        return 1/(1+np.exp(-z))\n",
    "    \n",
    "    def costFunction(self, X, y):\n",
    "        #Compute cost for given X,y, use weights already stored in class.\n",
    "        self.yHat = self.forward(X)\n",
    "        #print y\n",
    "        #print self.yHat\n",
    "        J = 0.5*sum((y-self.yHat)**2)\n",
    "        return J\n",
    "    \n",
    "    def sigmoidPrime(self,z):\n",
    "        #Gradient of sigmoid\n",
    "        return np.exp(-z)/((1+np.exp(-z))**2)\n",
    "    \n",
    "    def tanh(self,x):\n",
    "        return np.tanh(x)\n",
    "\n",
    "    def tanh_deriv(self,x):\n",
    "        return 1.0 - np.tanh(x)**2\n",
    "\n",
    "    def logistic(x):\n",
    "        return 1/(1 + np.exp(-x))\n",
    "\n",
    "    def logistic_derivative(x):\n",
    "        return logistic(x)*(1-logistic(x))\n",
    "\n",
    "    def costFunctionPrime(self, X, y):\n",
    "        #Compute derivative with respect to W and W2 for a given X and y:\n",
    "        self.yHat = self.forward(X)\n",
    "        \n",
    "        delta3 = np.multiply(-(y-self.yHat), self.sigmoidPrime(self.z3))\n",
    "        #delta3 = np.multiply(-(y-self.yHat), self.z3)\n",
    "        dJdW2 = np.dot(self.a2.T, delta3)\n",
    "        \n",
    "        delta2 = np.dot(delta3, self.W2.T)*self.sigmoidPrime(self.z2)\n",
    "        dJdW1 = np.dot(X.T, delta2)  \n",
    "        \n",
    "        return dJdW1, dJdW2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3, 2)\n",
      "(3, 1)\n"
     ]
    }
   ],
   "source": [
    "X = np.array(([3,5], [5,1], [10,2]), dtype=float)\n",
    "y = np.array(([75], [82], [93]), dtype=float)\n",
    "print X.shape\n",
    "print y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10000, 2)\n",
      "(10000, 1)\n"
     ]
    }
   ],
   "source": [
    "X = np.random.rand(10000,2)\n",
    "y = np.apply_along_axis( lambda element: element[0]+element[1], axis=1, arr=X )\n",
    "y.shape=(10000,1)\n",
    "print X.shape\n",
    "print y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 950.45116221]\n",
      "[ 443.951808]\n",
      "[ 443.78483738]\n",
      "[ 443.75447796]\n",
      "[ 443.74966856]\n",
      "[ 443.74878288]\n",
      "[ 443.74846168]\n",
      "[ 443.74818329]\n",
      "[ 443.74786401]\n",
      "[ 443.74748609]\n"
     ]
    }
   ],
   "source": [
    "NN = Neural_Network()\n",
    "max_iterations = 10000\n",
    "iter = 0\n",
    "learningRate = 0.01\n",
    "while iter < max_iterations:\n",
    "      dJdW1, dJdW2 = NN.costFunctionPrime(X,y)\n",
    "\n",
    "      \n",
    "      #update\n",
    "      NN.W1 = NN.W1 - learningRate * dJdW1\n",
    "      NN.W2 = NN.W2 - learningRate * dJdW2\n",
    "      #NN.W1 = NN.W1 + upd_W1\n",
    "      #NN.W2 = NN.W2 + upd_W2\n",
    "      \n",
    "      if iter % 1000 == 0:\n",
    "            print NN.costFunction(X,y)\n",
    "    \n",
    "      iter = iter + 1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0.93202425]\n",
      " [ 0.75918096]\n",
      " [ 0.90556237]\n",
      " ..., \n",
      " [ 1.2723205 ]\n",
      " [ 1.06958507]\n",
      " [ 0.03886624]]\n",
      "---\n",
      "[[ 0.97631766]\n",
      " [ 0.93316831]\n",
      " [ 0.97124158]\n",
      " ..., \n",
      " [ 0.99544918]\n",
      " [ 0.98708883]\n",
      " [ 0.02798041]]\n"
     ]
    }
   ],
   "source": [
    "print y\n",
    "print \"---\"\n",
    "print NN.forward(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0917330467358\n",
      "0.524279660735\n"
     ]
    }
   ],
   "source": [
    "def sse(y,yhat):\n",
    "    return sum(map(lambda (a,b) : pow(a[0]-b[0],2),zip(y,yhat)))\n",
    "print sse(y,NN.forward(X))/10000\n",
    "NN = Neural_Network()\n",
    "print sse(y,NN.forward(X))/10000"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2 with Spark 2.1",
   "language": "python",
   "name": "python2-spark21"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}