{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Import Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-07-20T19:47:13.588154Z",
     "start_time": "2021-07-20T19:47:13.399659Z"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Define Classes for Building Neural Network Architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-07-20T19:47:13.618098Z",
     "start_time": "2021-07-20T19:47:13.590149Z"
    }
   },
   "outputs": [],
   "source": [
    "class Neuron:\n",
    "    def __init__(self, weights, bias):\n",
    "        self.weights = weights\n",
    "        self.bias = bias\n",
    "        self.output_weights = []\n",
    "    def forward_propogate_neuron(self, inputs, activation):\n",
    "        self.activation_obj = get_activation_obj(activation)\n",
    "        self.inputs = inputs\n",
    "        self.input_to_activation = np.dot(self.inputs, self.weights) + self.bias\n",
    "        self.activation_obj.activation(self.input_to_activation)\n",
    "        self.output = self.activation_obj.output\n",
    "    def backward_propogate_neuron_last_layer(self, diff_binary_loss, activation, learning_rate):\n",
    "        self.activation_obj = get_activation_obj(activation)\n",
    "        self.activation_obj.derivation(self.input_to_activation)\n",
    "        self.delta = diff_binary_loss * self.activation_obj.output\n",
    "        self.backward_outputs = np.multiply(self.delta, self.weights)\n",
    "        self.derivative = np.multiply(self.inputs, self.backward_outputs)\n",
    "        self.weights = self.weights - learning_rate*self.derivative[0]\n",
    "    def backward_propogate_neuron_rest_layer(self, backward_input, activation, learning_rate):\n",
    "        self.activation_obj = get_activation_obj(activation)\n",
    "        self.activation_obj.derivation(self.input_to_activation)\n",
    "        self.delta = backward_input * self.activation_obj.output\n",
    "        self.derivative = np.dot(self.delta, self.inputs)\n",
    "        self.weights = self.weights - learning_rate*self.derivative\n",
    "    def update_weights(self, learning_rate):\n",
    "        self.derivative = np.dot(self.delta, self.inputs)\n",
    "        self.weights = self.weights - learning_rate*self.derivative"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-07-20T19:47:13.648993Z",
     "start_time": "2021-07-20T19:47:13.621069Z"
    }
   },
   "outputs": [],
   "source": [
    "class Layer:\n",
    "    def __init__(self, n_inputs, n_outputs, activation='linear'):\n",
    "        self.input_shape = n_inputs\n",
    "        self.output_shape = n_outputs\n",
    "        self.activation = activation\n",
    "        self.neurons = []\n",
    "        self.deltas = []\n",
    "    def forward_propogate_layer(self, inputs):\n",
    "        neuron_count = 1\n",
    "        layer_outputs = []\n",
    "        self.inputs = inputs\n",
    "        for neuron in self.neurons:\n",
    "            neuron.forward_propogate_neuron(self.inputs, self.activation.activation_name)\n",
    "            layer_outputs.append(neuron.output)\n",
    "            neuron_count += 1\n",
    "        self.outputs = layer_outputs\n",
    "    def backward_propogate_last_layer(self, diff_binary_loss, learning_rate):\n",
    "        self.backward_outputs = []\n",
    "        neuron_count = len(self.neurons)\n",
    "        for neuron in self.neurons:\n",
    "            neuron.backward_propogate_neuron_last_layer(diff_binary_loss, self.activation.activation_name, learning_rate)\n",
    "            self.backward_outputs.append(neuron.backward_outputs)\n",
    "            neuron_count -= 1\n",
    "    def backward_propogate_rest_layers(self, prev_layer_outputs, learning_rate):\n",
    "        self.backward_outputs = [] * len(self.neurons)\n",
    "        for i in range(len(self.neurons)):\n",
    "            self.neurons[i].delta = np.array(prev_layer_outputs).T[i] * self.activation.derivation(self.neurons[i].input_to_activation)\n",
    "            self.deltas.append(self.neurons[i].delta)\n",
    "            self.neurons[i].update_weights(learning_rate)\n",
    "        supertmp = []\n",
    "        for i in range(len(self.neurons)):\n",
    "            tmp = []\n",
    "            for j in range(len(self.neurons[i].weights)):\n",
    "                tmp.extend(self.neurons[i].delta * self.neurons[i].weights[j])\n",
    "            supertmp.append(tmp)\n",
    "        self.backward_outputs = np.sum(supertmp, 0, keepdims=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-07-20T19:47:13.679911Z",
     "start_time": "2021-07-20T19:47:13.651985Z"
    }
   },
   "outputs": [],
   "source": [
    "class Network:\n",
    "    def __init__(self, epochs = 1):\n",
    "        self.network_name = 'My Network'\n",
    "        self.deltas = []\n",
    "        self.epochs = epochs\n",
    "    def initialize_neurons(self):\n",
    "        if len(self.layers) > 0:\n",
    "            layer_count = 1\n",
    "            for layer in self.layers:\n",
    "                n_inputs = layer.input_shape\n",
    "                n_neurons = layer.output_shape\n",
    "                layer_count += 1\n",
    "                neuron_count = 1\n",
    "                for i in range(n_neurons):\n",
    "                    neuron = Neuron(weights = np.random.randn(n_inputs) * 0.1, bias = np.random.randn(1)*0.1)\n",
    "                    layer.neurons.append(neuron)\n",
    "                    neuron_count += 1\n",
    "    def forward_propogate_network(self, inputs):\n",
    "        if len(self.layers) > 0:\n",
    "            layer_count = 1\n",
    "            layer_inputs = inputs\n",
    "            for layer in self.layers:\n",
    "                if (layer_count == 1):\n",
    "                    layer.forward_propogate_layer(layer_inputs)\n",
    "                else:\n",
    "                    layer.forward_propogate_layer(np.array(layer_inputs).T)\n",
    "                layer_count += 1\n",
    "                layer_inputs = layer.outputs\n",
    "    def calculate_binary_loss(self, y_prob, y):\n",
    "        m = y_prob.shape[0]\n",
    "        y_prob_clipped  = np.clip(y_prob, 1e-7, 1-1e-7) # we are clipping values so that we do not get log(0) kind of situation\n",
    "        loss = -1/m * (np.dot(np.array(y), np.log(y_prob_clipped)) + np.dot(np.array(1-y), np.log(1-y_prob_clipped)))\n",
    "        self.loss = np.squeeze(loss)\n",
    "    def calculate_diff_binary_loss(self, y_prob, y):\n",
    "        m = y_prob.shape[0]\n",
    "        self.diff_binary_loss = 1/m * ((1 - np.array(y))/(1 -np.array(y_prob)) - np.array(y)/np.array(y_prob))\n",
    "    def get_predictions(self, y_prob):\n",
    "        self.predictions = []\n",
    "        for i in y_prob:\n",
    "            if i > 0.5:\n",
    "                self.predictions.append(1)\n",
    "            else:\n",
    "                self.predictions.append(0)\n",
    "    def get_accuracy(self, y, y_prob):\n",
    "        self.accuracy = np.mean(self.predictions == y)\n",
    "    def backward_propogate(self, learning_rate, y_prob, y):\n",
    "        for i in reversed(range(len(self.layers))):\n",
    "            if i == len(self.layers) - 1:\n",
    "                self.calculate_diff_binary_loss(y_prob, y)\n",
    "                self.layers[i].backward_propogate_last_layer(self.diff_binary_loss, learning_rate)\n",
    "            else:\n",
    "                self.layers[i].backward_propogate_rest_layers(self.layers[i+1].backward_outputs, learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-07-20T19:47:13.694872Z",
     "start_time": "2021-07-20T19:47:13.681906Z"
    }
   },
   "outputs": [],
   "source": [
    "def get_activation_obj(activation):\n",
    "        if activation == 'Linear':\n",
    "            return Activation_Linear()\n",
    "        elif activation == 'ReLU':\n",
    "            return Activation_ReLU()\n",
    "        elif activation == 'Tanh':\n",
    "            return Activation_Tanh()\n",
    "        elif activation == 'Sigmoid':\n",
    "            return Activation_Sigmoid()\n",
    "        elif activation == 'Softmax':\n",
    "            return Activation_Softmax()\n",
    "        else:\n",
    "            return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-07-20T19:47:13.710828Z",
     "start_time": "2021-07-20T19:47:13.696866Z"
    }
   },
   "outputs": [],
   "source": [
    "class Activation_ReLU:\n",
    "    def __init__(self):\n",
    "        self.activation_name = 'ReLU'\n",
    "    def activation(self, inputs):\n",
    "        self.output = np.maximum(0, inputs)\n",
    "    def derivation(self, outputs):\n",
    "        if outputs <= 0:\n",
    "            self.output = 0\n",
    "        else:\n",
    "            self.output = outputs\n",
    "        return self.output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-07-20T19:47:13.725786Z",
     "start_time": "2021-07-20T19:47:13.713822Z"
    }
   },
   "outputs": [],
   "source": [
    "class Activation_Sigmoid:\n",
    "    def __init__(self):\n",
    "        self.activation_name = 'Sigmoid'\n",
    "    def activation(self, inputs):\n",
    "        self.output = 1/(1+np.exp(-inputs))\n",
    "    def derivation(self, outputs):\n",
    "        self.output = 1 - 1/(1+np.exp(-outputs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-07-20T19:47:13.740767Z",
     "start_time": "2021-07-20T19:47:13.730792Z"
    }
   },
   "outputs": [],
   "source": [
    "class Activation_Softmax:\n",
    "    def __init__(self):\n",
    "        self.activation_name = 'Softmax'\n",
    "    def activation(self, inputs):\n",
    "        exp_values = np.exp(inputs)\n",
    "        self.output = exp_values/np.sum(exp_values, axis=0, keepdims=True)\n",
    "    def derivation(self, outputs):\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-07-20T19:47:13.756706Z",
     "start_time": "2021-07-20T19:47:13.743740Z"
    }
   },
   "outputs": [],
   "source": [
    "class Activation_Tanh:\n",
    "    def __init__(self):\n",
    "        self.activation_name = 'Tanh'\n",
    "    def activation(self, inputs):\n",
    "        self.output = (np.exp(2*inputs) - 1)/(np.exp(2*inputs) + 1)\n",
    "    def derivation(self, outputs):\n",
    "        self.output = 4/(np.exp(-outputs) + np.exp(outputs))**2\n",
    "        return 4/(np.exp(-outputs) + np.exp(outputs))**2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-07-20T19:47:13.771695Z",
     "start_time": "2021-07-20T19:47:13.758701Z"
    }
   },
   "outputs": [],
   "source": [
    "class Activation_Linear:\n",
    "    def __init__(self):\n",
    "        self.activation_name = 'Linear'\n",
    "    def activation(self, inputs):\n",
    "        self.output = inputs\n",
    "    def derivation(self, outputs):\n",
    "        self.output = 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Initialize Neural Network Architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-07-20T19:47:14.304294Z",
     "start_time": "2021-07-20T19:47:13.773660Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-07-20T19:47:14.319201Z",
     "start_time": "2021-07-20T19:47:14.306236Z"
    }
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv('irisTestData.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-07-20T19:47:14.334162Z",
     "start_time": "2021-07-20T19:47:14.322194Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "np.random.seed(42)\n",
    "\n",
    "# X, y = create_data(samples = 1, classes = 1, n_features = 5)\n",
    "layers = []\n",
    "layers.append(Layer(n_inputs = len(df.columns)-1, n_outputs = 6, activation = Activation_ReLU()))\n",
    "layers.append(Layer(n_inputs = 6, n_outputs = 6, activation = Activation_Tanh()))\n",
    "layers.append(Layer(n_inputs = 6, n_outputs = 4, activation = Activation_Tanh()))\n",
    "layers.append(Layer(n_inputs = 4, n_outputs = 5, activation = Activation_Tanh()))\n",
    "layers.append(Layer(n_inputs = 5, n_outputs = 4, activation = Activation_Tanh()))\n",
    "layers.append(Layer(n_inputs = 4, n_outputs = 4, activation = Activation_Tanh()))\n",
    "layers.append(Layer(n_inputs = 4, n_outputs = 5, activation = Activation_Tanh()))\n",
    "layers.append(Layer(n_inputs = 5, n_outputs = 1, activation = Activation_Sigmoid()))\n",
    "\n",
    "network = Network(epochs = 10)\n",
    "network.layers = layers\n",
    "network.initialize_neurons()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-07-20T19:48:11.570783Z",
     "start_time": "2021-07-20T19:48:10.837680Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch #1\n",
      "Accuracy =  0.6666666666666666\n",
      "===========================================================\n",
      "Epoch #2\n",
      "Accuracy =  0.6666666666666666\n",
      "===========================================================\n",
      "Epoch #3\n",
      "Accuracy =  0.6666666666666666\n",
      "===========================================================\n",
      "Epoch #4\n",
      "Accuracy =  0.6666666666666666\n",
      "===========================================================\n",
      "Epoch #5\n",
      "Accuracy =  0.6666666666666666\n",
      "===========================================================\n",
      "Epoch #6\n",
      "Accuracy =  0.6666666666666666\n",
      "===========================================================\n",
      "Epoch #7\n",
      "Accuracy =  0.6666666666666666\n",
      "===========================================================\n",
      "Epoch #8\n",
      "Accuracy =  0.6666666666666666\n",
      "===========================================================\n",
      "Epoch #9\n",
      "Accuracy =  0.6666666666666666\n",
      "===========================================================\n",
      "Epoch #10\n",
      "Accuracy =  0.6666666666666666\n",
      "===========================================================\n"
     ]
    }
   ],
   "source": [
    "for i in range(network.epochs):\n",
    "#     print('Epoch #{0}'.format(i+1))\n",
    "    for index, row in df.iterrows():\n",
    "        lst_prob = []\n",
    "        print('Epoch #{1} and Sample # {0}'.format(index + 1, i + 1))\n",
    "        print('-----------------------------------------------------------')\n",
    "        X = row[:-1]\n",
    "        y = row[-1]\n",
    "        network.forward_propogate_network(np.array(X).reshape(1,len(X)))\n",
    "        print('Forward Propogation Completed Successfully!')\n",
    "        y_prob = network.layers[-1].neurons[0].output\n",
    "        lst_prob.append(y_prob)\n",
    "        network.calculate_binary_loss(y_prob, y)\n",
    "        print('Loss = ', network.loss)\n",
    "        network.backward_propogate(0.1, y_prob, y)\n",
    "        print('Backward Propogation Completed Successfully!')\n",
    "        network.get_predictions(y_prob)\n",
    "    network.get_accuracy(np.array(df['y']), lst_prob)\n",
    "    print('Accuracy = ', network.accuracy)\n",
    "    print('===========================================================')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}