{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# softmax \n",
    "- $softmax(x_i) = \\frac{e^{x_i}}{\\sum_j{e^{x_j}}}$\n",
    "\n",
    "- $\\begin{align} (softmax(x + c))_{i}= \\frac{e^{x_{i} + c}}{\\sum_{j} e^{x_{j} + c}} = \\frac{e^{x_{i}} \\times e^{c}}{e^{c} \\times \\sum_{j} e^{x_{j}}} \\\\ = \\frac{e^{x_{i}} \\times {e^{c}}}{{e^{c}} \\times \\sum_{j} e^{x_{j}}} = (softmax(x))_{i}  \\end{align}$\n",
    "\n",
    "so:\n",
    "\n",
    "- $softmax(x) = softmax(x + c)$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def softmax(x):\n",
    "    orig_shape = x.shape\n",
    "\n",
    "    if len(x.shape) > 1:\n",
    "        # Matrix\n",
    "        ### YOUR CODE HERE\n",
    "        x_max = np.max(x, axis=1).reshape(x.shape[0], 1)\n",
    "        x -= x_max\n",
    "        exp_sum = np.sum(np.exp(x), axis=1).reshape(x.shape[0], 1)\n",
    "        x = np.exp(x) / exp_sum \n",
    "        ### END YOUR CODE\n",
    "    else:\n",
    "        # Vector\n",
    "        ### YOUR CODE HERE\n",
    "        x_max = np.max(x)\n",
    "        x -= x_max\n",
    "        exp_sum = np.sum(np.exp(x))\n",
    "        x = np.exp(x) / exp_sum\n",
    "        ### END YOUR CODE\n",
    "        #or:  x = (np.exp(x)/sum(np.exp(x)))   \n",
    "\n",
    "    assert x.shape == orig_shape\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running basic tests...\n",
      "[0.26894142 0.73105858]\n",
      "[[0.26894142 0.73105858]\n",
      " [0.26894142 0.73105858]]\n",
      "[[0.73105858 0.26894142]]\n",
      "You should be able to verify these results by hand!\n",
      "\n"
     ]
    }
   ],
   "source": [
    "def test_softmax_basic():\n",
    "    \"\"\"\n",
    "    Some simple tests to get you started.\n",
    "    Warning: these are not exhaustive.\n",
    "    \"\"\"\n",
    "    print(\"Running basic tests...\")\n",
    "    test1 = softmax(np.array([1,2]))\n",
    "    print(test1)\n",
    "    ans1 = np.array([0.26894142,  0.73105858])\n",
    "    assert np.allclose(test1, ans1, rtol=1e-05, atol=1e-06)\n",
    "\n",
    "    test2 = softmax(np.array([[1001,1002],[3,4]]))\n",
    "    print(test2)\n",
    "    ans2 = np.array([\n",
    "        [0.26894142, 0.73105858],\n",
    "        [0.26894142, 0.73105858]])\n",
    "    assert np.allclose(test2, ans2, rtol=1e-05, atol=1e-06)\n",
    "\n",
    "    test3 = softmax(np.array([[-1001,-1002]]))\n",
    "    print(test3)\n",
    "    ans3 = np.array([0.73105858, 0.26894142])\n",
    "    assert np.allclose(test3, ans3, rtol=1e-05, atol=1e-06)\n",
    "\n",
    "    print(\"You should be able to verify these results by hand!\\n\")\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    test_softmax_basic()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = np.array([[1,2],[4,3]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([2, 4])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.max(x, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}