{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPython 3.6.8\n",
      "IPython 7.2.0\n",
      "\n",
      "numpy 1.15.4\n",
      "scipy 1.1.0\n",
      "matplotlib 3.0.2\n",
      "tensorflow 1.13.1\n"
     ]
    }
   ],
   "source": [
    "%load_ext watermark\n",
    "%watermark -v -p numpy,scipy,matplotlib,tensorflow"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**10장 – 인공 신경망 소개**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_이 노트북은 10장에 있는 모든 샘플 코드와 연습문제 해답을 가지고 있습니다._"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 설정"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "파이썬 2와 3을 모두 지원합니다. 공통 모듈을 임포트하고 맷플롯립 그림이 노트북 안에 포함되도록 설정하고 생성한 그림을 저장하기 위한 함수를 준비합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 파이썬 2와 파이썬 3 지원\n",
    "from __future__ import division, print_function, unicode_literals\n",
    "\n",
    "# 공통\n",
    "import numpy as np\n",
    "import os\n",
    "\n",
    "# 일관된 출력을 위해 유사난수 초기화\n",
    "def reset_graph(seed=42):\n",
    "    tf.reset_default_graph()\n",
    "    tf.set_random_seed(seed)\n",
    "    np.random.seed(seed)\n",
    "\n",
    "# 맷플롯립 설정\n",
    "%matplotlib inline\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "plt.rcParams['axes.labelsize'] = 14\n",
    "plt.rcParams['xtick.labelsize'] = 12\n",
    "plt.rcParams['ytick.labelsize'] = 12\n",
    "\n",
    "# 한글출력\n",
    "plt.rcParams['font.family'] = 'NanumBarunGothic'\n",
    "plt.rcParams['axes.unicode_minus'] = False\n",
    "\n",
    "# 그림을 저장할 폴더\n",
    "PROJECT_ROOT_DIR = \".\"\n",
    "CHAPTER_ID = \"ann\"\n",
    "\n",
    "def save_fig(fig_id, tight_layout=True):\n",
    "    path = os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID, fig_id + \".png\")\n",
    "    if tight_layout:\n",
    "        plt.tight_layout()\n",
    "    plt.savefig(path, format='png', dpi=300)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 퍼셉트론"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/haesun/anaconda3/envs/handson-ml/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:183: FutureWarning: max_iter and tol parameters have been added in Perceptron in 0.19. If max_iter is set but tol is left unset, the default value for tol in 0.19 and 0.20 will be None (which is equivalent to -infinity, so it has no effect) but will change in 0.21 to 1e-3. Specify tol to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from sklearn.datasets import load_iris\n",
    "from sklearn.linear_model import Perceptron\n",
    "\n",
    "iris = load_iris()\n",
    "X = iris.data[:, (2, 3)]  # 꽃잎 길이, 꽃잎 너비\n",
    "y = (iris.target == 0).astype(np.int)\n",
    "\n",
    "per_clf = Perceptron(max_iter=100, random_state=42)\n",
    "per_clf.fit(X, y)\n",
    "\n",
    "y_pred = per_clf.predict([[2, 0.5]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsgAAAEYCAYAAABBfQDEAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzs3XmYjuX7x/H3NYuxjm1QdtmSKM2EVlqkpFCUFlGiQduv0DeVyjdmYWT5WpJSWaKypEiLss1SjZQiRGWybxnbGLNcvz9mPDPG7Gaee5bP6zieg+e8t/MZ2+me875OY61FRERERERSeDidgIiIiIhIUaICWUREREQkHRXIIiIiIiLpqEAWEREREUlHBbKIiIiISDoqkEVERERE0lGBLCIiIiKSjlsLZGPME8aYX4wx0caYjcaYwVnsN8QYs8UY85sx5nNjTK3cbBMRERERuVBuK5CNMZ5AU+A6a20AcAcQZoypk2G/jsALwA3W2suBaOCdnLaJiIiIiBQE49QkPWNMWWA/0MpaG5MuPg3411o7IvV9VeAgUB0IzmqbtTbWzR9BREREREogLwevPQFYkL44TnUJ8NHZN9baf40xsUDDHLb9kv4kxpiBwECAChXK+jdvfs6N6lLt8OHjxMQcJP1/jipXrk3lyhc7mJWIiIhI9mIyVo3p1K+fm2P+xtpDJqfrOFIgG2PeAOoA92a2GUjKEEskpR0ku23nsNbOAGYA+Ps3sVFRYReYdcmyYcMOevUKJibmIACxsXto2LAt/fq9T7lyvg5nJyIiInK+wMCst40YkZtjAnJ1HbevYmGMGQe0BO611p7JZJddQP10+5cnpb1iVw7bJA/atGlMZOQ4OnZs5Yr98ssSQkLasW/fVgczExEREXGWOx/S8zDGTAfqAb3OFsfGGE9jzEpjzA2pu84GHjLGVE59PwQIt9YezGGb5FGNGpVZvvw1nnnmblds374tBAe3ZePGzxzMTEREROR8vll8kzureE7bsuK2h/SMMV2Bz0hZeSJ9m8QbwCTg/6y1n6bu+zTwGJAA7AECrbV7c9qWFbVY5GzevNUEBk7h9Om0m/pdu75Gly6v4OGh5bJFRESk+AsMNOtTV1PLlmOrWLiTCuTc2bDhT+67L4idO9NuyLdufTePPjpbfckiIiJS7OW2QNatQXFp0+YSIiPDuOmmtL7kjRuXEhzcln37tjiYmYiIiIj7qECWc/j5+bJs2Ws8+2xaX/L+/VsJDm7LL78sdTAzEREREfdQi4Vk6cMPU/qS4+LS+pLvvPNV7rxzpPqSRUREpFgYPhyOHTv7LgBro3NcB1lVjmTpgQc6sHp1MA0a1HDFli17nenTuxMXp8GFIiIiUvSlFce5pwJZsnXllSl9yTff3NoV27jxM4KD26kvWUREREokFciSIz8/Xz7//FWee667K3a2L/nnn5c4mJmIiIhIwVOBLLni5eVJcHA/PvjgOcqVKwPA6dPHmT69B0uXjiQ5OdnhDEVEREQKhgpkyZPevW9kzZoQGjas6YotX/5fpk3rxqlTRx3MTERERKRgqECWPLviikZERo7jlluucMV+/fVzgoPbsmfPZgczExERETlXkR417SQt81Y4EhOTePnl2Ywfn9aH7ONTkX79PqBNmx4OZiYiIiJyPk3Sk0J3ti95zpyhlC/vA0B8/Aneeuseli59RX3JIiIiUiypQJYLdt9917N6dTCNGtVyxZYvf4OpU+9SX7KIiIgUOyqQpUCc7Uvu1OlKV+y335YTFHQ1e/ZscjAzERERkbzxcjoBKTmqVavE0qWv8PLLcwgLWwzAwYPbCQlpT79+79OmzT0OZygiIiJFRWBg1tumT888PmgQZPb4nDEwbVrB5AW6gywFzNPTk6Cgvsydm7Ev+V4+/fRlkpOTHM5QREREiqus1pYo6DUnVCBLoejV63rWrAnhkkvS+pK/+GI0U6aoL1lERESKNhXIUmhat25IRMQ4brutjSu2adMXBAVdze7dvzmYmYiIiEjWVCBLoapWrRKffvoyw4bd64odPLid0ND2/PTTQgczExEREcmcCmQpdJ6enowe3Yd584ZRoUJZAOLjTzJjRk+WLBmhvmQREREpUtxaIBtjvI0xQ40xCcaY3lnss8IYE5XutcEY81fqNn9jzJEM259352eQ/OvZ8zrWrg2hceOLXLEVK4KYMqUrJ0/+62BmIiIiUhwYk7d4vq/jzlHTxpjBQDmgOzDFWjs/F8f8H9DUWjvYGNMZ6GmtHZCX62rUdNHy778neOSR8Xz55U+uWI0ajQkMXEKdOpc7mJmIiIiUZEVy1LS1dqq1NgzI1ffUjTHlgGeBoNSQH3CbMeb71NdoY0ylQkpXCknVqhVZsuQlhg9P35e8g9DQ9qxf/7GDmYmIiIgU/R7kQcBya+0/qe8XAQ2tte2AO4BGwAeZHWiMGWiMiTbGRB86dMw92UqueXp68sYbffjww+Hn9CW//fZ9LF78ovqSRURExDFFtkDO5O4x1to4m9oTYq09AjwP3JW67zmstTOstQHW2gA/P193pS15dO+917JuXShNmlzsin35ZTD/+9+dnDx5xMHMREREpLQqsgUyMBj4wlobk80+nsBpIN49KUlhaNmyPuHhY7n99qtcsc2bv0xdL/lXBzMTERGR0sitD+m5LmrMKmC6tXa+MaY6sBToZ639I3V7eWArcL21dme643oDK6y1R40xXsAs4IS1dlB219NDesVDUlISr78+n+DgtD7kMmXK88gjswgIuM/BzERERIqv4cPhWCbdpr6+EBrq/nzc7dzPH4C10TmueVEU7iCXBxoAldPFBpNSCO/MsG85YKUx5kcgAtgHPOeWLKXQeXp6MmrUQyxY8IKrL/nMmVPMnHk/ixa9oL5kERGRfMisOM4uXtLk53N6FXwaObPWdkz383+Auhm2j8viuFmk3DWWEqxHj2to1qwOvXoFs337HgC++iqUXbt+pn//D6lQoZrDGYqIiEhJVhTuIIucp2XL+kREhHLHHf6u2ObNXxEUFMCuXRsdzExERERKOhXIUmRVqVKRxYtfYsSItP7jQ4f+IjT0Gn78MccZMyIiIiL5ogJZijQPDw9ee+1BPvroP1SsmNaX/M47D7Bw4XCSkhIdzlBERERKGhXIUix0796edevG0rRpbVfs66/HMnnyHZw4cdjBzERERIo23yzGQWQVL2ny8zkdWebN3bTMW8kRG3uSvn3fZPnyaFesevWGDBq0hLp1r3AwMxERESnqAgPNemttQE776Q6yFCuVK1dg0aIR5/QlHz78NyEh6ksWERGRgqECWYqds33JH3/8HypVSpkynpAQl9qXPEx9ySIiInJBVCBLsdWtW3vWrQvN0Jc8jsmTb1dfsoiIiOSbI4NCRApKixb1iIgYS79+E1i27EcAtmxZSVBQAIGBi6lX70qHMxQREXGWO0ZNl7Rx1rqDLMVe5coVWLjwRV5++X5X7PDhvxk/vh0//DDPwcxERESc545R0yVtnLUKZCkRPDw8GDnyARYuHOHqS46LO8O77z7EJ588r75kERERyTUVyFKi3HVXW8LDx9KsWR1X7JtvxjNpUmdOnDjkYGYiIiJSXKhAlhLn0kvrEhExlq5d27piW7d+y5gxAcTE/ORgZiIiIlIcqECWEsnXtzyffPIfRo58wBU7cmQnY8dex/ffz3UwMxERESnqVCBLieXh4cHLL9/PokUj8PUtD0BCwmlmzXqYjz9+Tn3JIiJSKrhj1HRJG2etUdNSKmzdupuePYPYunWXK9a8+U08/vgCKlWq4WBmIiIi4i4aNS2STvPmdQgPD+Wuu9L3JX9HUJD6kkVERORcKpCl1PD1Lc/HH/+HV199AGMMAEeOxDB27HVERc12ODsREREpKlQgS6ni4eHBSy+d35f83nuP8NFHz5KUlOBwhiIiIuI0t/YgG2O8gWeAIKCPtXZ+Jvv4A18D29KFP7bWhpmU236jgPuAJOAn4Alr7cnsrqseZMnMtm0pfclbtqT1JTdr1pHHH1+Ar29NBzMTEZHipiiPWg4MzHrb9Onnx/LzWdz1+QcNgsxKV2Ng2rTc5BaAtdEmp+u4+w7yAMACUdns4wcstNa2T/c6W932BboAV1prLwMSgJBCzVhKrGbN6hAePpZu3dq7Ytu2rSIoKICdO9c7mJmIiBQ3JWnUcn4+i7s+f1b3dbO735ufHNxaIFtrp6YWu0nZ7OYH3GaM+T71NdoYUyl12/3AW9bauNT3E4EHMj2LSC5UqlSOBQuG8/rrD7n6kv/99x/Gj7+GqKgPHM5OREREnFAUe5AXAQ2tte2AO4BGwNlK5RJgR7p9dwDVjDGVM57EGDPQGBNtjIk+dKgY/vdN3MbDw4MXX+zF4sUvUblySl9yfHwC773XlwULnlZfsoiISClT5Apka22cTW2MttYeAZ4H7jLGlAMM5959Pjvp4bzPYa2dYa0NsNYG+PkV01Wqxa26dAkgImIcLVrUc8W++24yEybcyrFjBxzMTERERNypyBXImfAETgPxwC6gfrpt9YETwFEH8pISqGnT2qxbF0r37ml9yX/8sYagIH927ox2MDMRERFxF8cLZGNMdWNMuDGmaer73saYKqk/9yJlxYvZ1tpkYDbwuDGmTOrhTwGLbGkYByhuk9KX/AKjRqXvS97F2LHXExn5vsPZiYhIUVSSRi3n57O46/ObLNafyCqe3xwcGTVtjFkFTLfWzjfG1AMige7W2mhjzKPAk0AyKSterAZGWmvjjDGepCzzdicp7RWbgSettdk2GWuZN8mvL76I5pFHxhMbe8oV69jxSXr1Go+np7eDmYmIiEhe5XbUtCMFsrupQJYLsX37Xnr2DGLz5hhXrEmTGxg48GN8fWs5mJmIiIjkRW4LZMdbLESKuiZNLmbt2hB69LjGFdu+fS1BQQH8/fePDmYmIiIihUEFskguVKpUjvnzh/Pf/z58Tl/yuHE3EBExy+HsREREpCB5OZ2ASHFhjOGFF3py5ZWX0KdPGEePniQxMZ4PPniMnTvX06vXeLy8yuR8IhERyZeiPM7ZXfIzajmv8vN1vvAR0Lm7jrvoDrJIHnXufBUREeNo2TJtxcHVq6cwYcItHDu238HMRERKtpI0zjm/8jNqOa/y83UuyBHQReHXUwWySD6c7Uu+995rXbHt29cxZow/f/31vYOZiYiIyIVSgSySTxUrlmPevGGMHv2Iqy/56NHdvPnm9YSHv+twdiIiIpJfKpBFLoAxhmHD7uGzz0ZSpUoFAM6cSWT27P7MmzeYxMQzDmcoIiIieaUCWaQA3HZbGyIjx3H55Q1csTVrpvHmmzcTG7vPwcxEREQkr1QgixSQxo1T+pJ79rzOFduxI1x9ySIiBaQkjXPOr/yMWs6r/HydC3IEdFH49dQkPZECZq0lLGwxL788h+TkZAC8vMrwwANTue66/g5nJyIiUnppkp6IQ4wxDB16D5999gpVq1YEIDHxDLNnP868eYPUlywiIlLEqUAWKSSdOqX0Jbdq1dAVW7NmOuPH30Rs7F7nEhMREZFsqUAWKUSXXHIRa9YE06vX9a7Yn39GMGaMP3/+GelgZiIiIpIVFcgihaxChbLMmfM8wcH98PBI+SMXG7uXsLAOrF37tsPZiYiISEZeTicgUhoYY3juue60bt2Qhx8O48iR4yQlJTB37kBiYqK5775JeHv7OJ2miAjDh2c+6tfXF0JD3Z/PhQoMzHrb9OmZxwcNynxEsjEwbZqzx+Tn1yevx5S03wP5oTvIIm50661XEhk5jtatG7pia9fO4M03b+Lo0T3OJSYikiqzwii7eEmU1QJf2S385a5j8vPrk9dj9HtABbKI2zVqVIs1a0K4//4bXLE//4xkzBh/duyIcDAzERERARXIIo4oX96HDz54jpCQtL7kY8f2MWFCB9auneFwdiIiIqWbCmQRhxhj+L//686yZa9SvXolABISEpk79wnmzBlIQkK8wxmKiIiUTiqQRRx2yy1XEBk5jiuuaOSKrVv3NuPHd1RfsoiIiAPcWiAbY7yNMUONMQnGmN5Z7FPLGPOWMeZ3Y8wPxpi1xphWqduqG2NOGmOi0r3GufMziBSGhg1rsXp1ML173+iK/fVXFGPG+LN9e7iDmYlIaePrm7d4SWRM3uLuPCY/vz55PUa/B8DY7B6VLOiLGTMYKAd0B6ZYa+dnss8dQDlr7aLU988Dna21txljmgP/s9Z2yst1/f2b2KiosAv/ACKFzFrLxIlL+c9/3ic5ORkAT09v7r9/Ejfc8AQmu781RUREJFuBgWa9tTYgp/3cegfZWjvVWhsGJGWzzxdni+NUe0lbr9kPaG2MiTTG/GiMmWyMqZXZeYwxA40x0caY6EOHStG6JFKsGWN49tlufPHFa66+5KSkBObNG6S+ZBERETcp0j3IqcXvKOD11FA0UNtaew3QETgDLDeZ3Faz1s6w1gZYawP8/ErR9wSkRLjpptZERYVx5ZWXuGLh4TMZP74D//6728HMRERESr4iWyAbY6oDy4HXrLWrAay18dbapNSfnwSGA5cCTRxLVKSQNGhQk1WrgnjggQ6u2F9/fU9QkD/bt69zMDMREZGSrUiOmjbGXAx8AYRaa+dltyspRb56KKREKl/eh/feexZ//8a88MJ7JCUlc+zYfsaPv4n77ptIhw6D1JcsIqWGO8YsuzO3onydvCqqeeWX43eQU1emCDfGNE193wBYCfw3Y3FsjLk7tXgmta1iFLDKWrvf3XmLuIsxhqefvpsvvnids+1CycmJzJ8/hNmzHych4bTDGYqIuIc7xiznV0m7Tl4V1bzyK9M7yMaYd/N6ImvtY/nMoTzQAKic+j4MqAUMM8YMS43FW2s7ABZYZIzxBpKBjcBD+byuSLHSsWMrIiPHcd99wWzY8CcAERHvsmfPbzzxxEKqVq3rcIYiIiIlQ1YtFicL86LW2o7pfv4PUDfd+57ZHPcZ8Flh5iZSlJ3tSx40aCrz5q0G4O+/f2DMGH8GDvyEpk1vcDhDERGR4i/TAtla+5S7ExGR3ClXzodZs54lIKApw4a9S1JSMsePH2DixJvo2XMiHToMVl+yiIjIBXC8B1lE8s4Yw5NPdmXFirS+5MTEJObPf5LZs/urL1lEROQCZFogG2PaGmMWZfNa6O5EReR8HTq0IioqjKuuauyKRUTMYty4Gzly5B8HMxMRKXjuGLOcXyXtOnlVVPPKr0xHTaeOdO6dLjSSlBUjEklZWu11wNtam+yOJC+URk1LSRcXF8+QIdOZM+c7V6xSpZoMHPgxTZve6GBmIiIiRUduR01nWiCft5MxyUBZa+2ZdO+9VCCLFB3WWqZOXcbQoSl9yQAeHl706vUmHTsOUV+yiIiUerktkHPbg5yxis65qhYRtzLGMGRIV778chQ1aqSsmpicnMiCBU/x/vuPqi9ZREQkl/LykJ6KYpFi4MYbLycqahz+/mkT2KOi3mfcuBvUlywiIpILWQ0KuQtYRFpRbIBYfYtWpHioV68G3347miefnM7s2Sl9yTt3RhMU5M+AAR/TrFkHhzMUcY+SNv62JCnKY6NFshoUEgHcmt2BxaX/WKS0KlfOh5kzn8bfvwlDh75LYmISx48fZMKEW+jZczw33fSU+pKlxCtp429LkqI8Nlokq0Ehh4HVbs5FRAqYMYbBg++kVauGPPBAKAcOxJKcnMRHHz1DTMx6HnxwOmXKlHM6TRERkSIlq3WQq+X15e7ERST3brihJZGR4wgIaOqKRUV9kNqXHONgZiIiIkVPVg/pHQIO5vJ1dl8RKcLO9iX37XuLKxYTs54xY/zZunWVc4mJiIgUMVkVyI2AS1JfjUl5SO/KdLH0r7P7ikgRV7ZsGWbMeJJJkwbi5eUJwIkTh5g06RZWrpxIbtZFFxERKekyLZCttTvTvf4mZTWLndbanUAL4OYM++x0Y84icgGMMQQGduGrr0ZRs2bKeslJScl8/PGzvPdeX86ciXM4Q5GCU9LG35YkRXlstEhuJ+klAVWttceMMXcAL1hrOxZ2cgVFk/REMrdr1yHuvz+EH3/8wxWrV68NgYGLqV69gYOZiYiIFLyCnqR3hrQ1kf8Emmazr4gUE3Xr+rFy5Wj69UvrS/7nnw0EBQWwdet3DmYmIiLinFwVyNbactba46lvjwA1jRZQFSkRypYtw1tvPcnkyU+c05c8cWInvvnmTfUli4hIqZOXUdNnHQU8gYoFnIuIOMQYwxNP3MHXX/+XWrWqAJCcnMQnnzzHrFl9OHPmlMMZioiIuE9Wk/SyZK1NSL157F3w6YiIk6677jKiosK4//4QfvhhGwA//DCXvXs388QTi/Dza+hsgiIlyKBBkNk3aIyBadOK13WK8ghojbSW/MjyDrIxZmRWL9L6kfPEGONtjBlqjEkwxvTOYh9jjPmvMWarMWazMWaOMaZCTttEpGDUqVOdlStH89hjnVyxs33JW7Z862BmIiVLVt1LBd3V5I7rFOUR0BppLfmRXYvFTdm81gAJ+bjeAFKK66hs9ukLdAGutNZelnqdkFxsE5EC4uPjzbRpg5kyZRDe3infaDp58nBqX/J49SWLiEiJlmWLhbX2poK+mLV2KoAx5q5sdrsfeMtae3Yx1onASuDJHLadwxgzEBgIUL9+jQLJX6Q0McYwYEBnWrasT+/eoezb9y/WJvPJJ8+zc+d6+vR5mzJlyjudpoiISIHLz0N6he0SYEe69zuAasaYyjlsO4e1doa1NsBaG+DnpxXERfLr2mtbEBUVRrt2zV2xH3+cx9ix13Ho0F8OZiYiIlI4imKBbICkdO8TU3/0yGGbiBSS2rWr8c03b9C/f/q+5J8JCgrg99+/cTAzERGRglcUC8tdQP107+sDJ0hZXi67bSJSiFL6kocwdWr6vuQjTJ7cma+/DlNfskgeZTVNoKCnDLjjOkV5BLRGWkt+5GrUdIFf1JhVwHRr7XxjTHVgKdDPWvuHMeZR4FHgVmvtGWPM/4BK1tq+2W3L7noaNS1SsKKitnD//SHs3fuvK3b11Q/Qp89M9SWLiEiRldtR05k+pGeMGZzNMbOA0dba51L39QBWWWtvzFemUB5oAJztI/4AaAL8YIxJBDaT9hBedttExE3at7+UyMgwevcOISpqKwA//vghe/duJjBwMX5+jRzOUEREJP8yvYNsjInMYn8LdAUOAi1I6wfeZq31LJQMC4DuIIsUjvj4BJ57biZvv/2lK1ahQjX695/PZZd1yuZIERER97ugO8jW2muyOsAY40nKw3K/p/4I+RwcIiLFm4+PN1OmDOKqqxrz9NMzSEhITO1Lvp0ePYLp1GkopqAbKkVERApZfh/Ss0AloApwUcGlIyLFUf/+t7Fy5RtcfHFVAKxNZtGi4bzzzgPEx590ODsREZG8yW7U9OPGmO9SX09k3G6tPWWtPQYcLtQMRaRYaN/+UqKiwrjmmktdsejoBYSGXsvBg386mJmIiEjeZFogG2OeAl4nZXWJT4GXjDFD3ZmYiBQ/F19cja+//i9PPHG7K7Z790aCggLYvPkrBzMTERHJvazuIA8GHrTWvmmtnQDclxo7yxhjZhpjZgPzCztJESk+ypTxZvLkQN56awhlyqQ85nDq1L9MnnwHX34ZqvWSRUSkyMuqQK4LbEz3fiPn9xqfBPYB2wshLxEp5h59tBPffjuG2rWrASl9yYsXv8DMmb3VlywiIkVaVgXyJqBLuve3A9tSf26BJGvtM9baYcArpK1mISLi0rZtM6Kiwrj22hau2Pr1HxEaeg0HD+5wMDMREZGsZVUgjwRmGGM+NsbMJ2VAx8sA1tpka6332R2ttYmApgKISKYuuqgqX301isDAO1yx3bt/JSjoajZt+jKbI0VERJyRaYFsrf0KuAH4A/gb6Git/Tyrk1hrdxZKdiJSIpQp482kSU8wY8aT5/Ql/+9/d7BiRbD6kkVEpEjJcpk3a+1P1toR1tr/WGujjTF3uzMxESl5+vW7le++G0OdOtUBsNayZMmLvP32/Zw+fcLh7ERERFLkZVDI4qw2GGOuKoBcRKQUuPrqZkRGjuO669L6kn/66WNCQ6/hwAE98ysiIs7LblDIg8aYO40xbY0xNcnwIJ4xpkzqj3VIWStZRCRXLrqoKl9+OYpBg9KeBd6z5zeCg69m06YVDmYmIiKS/R3kOcBY4HMgmpTVK9L7yxjTHLgM+KVw0hORkqpMGW8mThzI228/hY9PynO/p04d5X//68KKFUHqSxYREcfk1GIRYK2tSUoRDIAxZoUxxg+oRcqQkFuA8MJLUURKsr59b8mkL3kEM2b0Ul+yiIg4IrsC2QLVjTE3Aul7jG8EygBnSBkg8n+kjKQWEcmXgICmREWFccMNLV2xDRsWEhranv37/3AwMxERKY2yK5AN0B5YmfpKHzeAJxACxFtrNxVahiJSKtSqVYUVK15nyJA7XbE9ezYRHHw1v/32hYOZiYhIaZNdgfw0sMha651+MAhpvcjGWrsZ+N0Y419oGYpIqeHt7cWbbw5g5synXX3JcXGxTJlyJ8uXj1ZfsoiIuEV2BfIkIN4Yc9AY81W6eMax0suA7gWemYiUWo88cjOrVgVRt25aX/LSpS8zY0ZPTp8+7nB2IiJS0uX4kB5wJzAxXawPcDjd+++Aaws4LxEp5fz9m2TSl7yIkBD1JYuISOHKqUDeZq39wVq77GzAWvuJtfY0sDs1tAm4PDcXM8Z0MMb8ZIzZaIyJNsa0z2SfFcaYqHSvDcaYv1K3+RtjjmTY/nzuPqqIFDc1a6b0JT/5ZFdXbO/ezYwffyW//rosmyNFRETyL7sC+RUgPquN1tpGqT8eIaUdI1vGmCrAImCItbY1MBT41BhTPsN5b7fWtj/7Aj4Azj6h4wcsTL/dWhuW07VFpPjy9vZi/PjHeeedZ1x9ybGxp5g69S6WL3+D5ORkhzMUEZGSJssC2Vo72lqblC4Uk92+ubhWZ2CrtTYy9ZhVwF5S1lHOlDGmHPAsEJQa8gNuM8Z8n/oabYyplMWxA1PvUkcfOnQsF+mJSFHWp89NrF4dRL16fsDZvuRX1JcsIiIFLqcWC5ezd4ym+iZ0AAAgAElEQVQvwCXAjgyxHanxrAwClltr/0l9vwhoaK1tB9wBNCLlDvN5rLUzrLUB1toAPz/fC8tcRIqEq65K6Uvu0CGtq+vnnxcTHNyO/fu3OZiZiIiUJLkukAuAAZIyxBKzyiGTu8dYa+Ns6jpPqa0dzwN3pe4rIqVAjRqVWb78NZ5++i5XbN++3wkKupqNGz93MDMRESkp3Fkg7wLqZ4jVT41nZjDwhbU2y9YOUoaVnCabXmkRKXm8vb0YN64/7777DGXLlgHg9OljTJ16F8uWjVJfsoiIXBB3FsifAq2NMa0AjDFtgUuBb40x4caYpmd3TH1w71lgTPoTGGN6pz7shzHGi5S7y7OttfrXUKQUevjhlL7k+vVruGKfffYqb711D3FxevZARETyx20FsrU2FugFvGuM+QGYAHQBygMNgMrpdh8MrLDW7sxwmnLASmPMj0AEsA94rrBzF5Giq02bxkRGjqNjx1au2C+/fEpISDv27dvqYGYiIlJcmdIwutXfv4mNitJqcCIlWWJiEi+++D4TJy51xcqW9eWxx+bQuvVd2RwpIiKlRWCgWW+tDchpP3e2WIiIFBovL0/Gjn2M9977vwx9yXfz+eevqy9ZRERyTQWyiJQoDz7YgdWrg2nQIK0v+fPPX2P69B7qSxYRkVxRgSwiJU6bNpcQGRnGTTel9SVv3LiU4OC27Nu3xcHMRESkOFCBLCIlkp+fL8uWvcazz97tiu3fv5WwsKv45Zel2RwpIiKlnQpkESmxvLw8CQ19jPff/z/KlUvpSz5+PI5p07rx2WevqS9ZREQypQJZREq8Bx44vy952bLXmT69O3FxsQ5mJiIiRZEKZBEpFa68MqUv+eabW7tiGzd+RnBwW/bu/d3BzEREpKhRgSwi5zhwYDXR0QMID+9BdPQADhxY7XRKBcbPz5fPP3+V557r7ort37+NkJB2/PzzEgczExGRokQFsoi4HDiwmh07phIffxCwxMcfZMeOqSWqSPby8iQ4uB+zZz/v6ks+ffo406f3YOnSkepLFhERFcgikiYmZg7JyfHnxJKT44mJmeNQRoXn/vtvYM2aEBo2rOmKLV/+X6ZNu5tTp446mJmIiDhNBbKIuMTHH8pTvLi74opGREaO45ZbrnDFfv11GcHBbdmzZ7ODmYmIiJNUIIuIi4+PX57iJUH16r589tnIc/qSDxz4g5CQdmzYsNjBzERExCkqkEXEpX79h/Hw8Dkn5uHhQ/36DzuUkXuc7UueM2co5cunfP74+BO89dY9LF36ivqSRURKGRXIIuJSs2YHGjcejI9PDcDg41ODxo0HU7NmB6dTc4v77rue1auDadSoliu2fPkbTJ16l/qSRURKEWOtdTqHQufv38RGRYU5nYaIFBNHjhynT58wvv76Z1esRo0mDBq0hNq1WzqYmYiIXIjAQLPeWhuQ0366gywikkG1apVYuvQVhg69xxU7eHA748cHsGHDIgczExERd1CBLCKSCU9PT8aMeYS5c9P6kk+cOM1bb93LkiUvkZyc5HCGIiJSWFQgi4hko1ev61m7NoRLLknrS16xYgxTptzFyZP/OpiZiIgUFhXIIiI5aNWqIRER47jttjau2KZNXxAcfDW7d//mYGYiIlIY3FogG2M6GGN+MsZsNMZEG2PaZ7KPvzHmiDEmKt3r+dRtxhjzX2PMVmPMZmPMHGNMBXd+BhE534EDq4mOHkB4eA+ioweUqNHUZ1WrVolPP32ZYcPudcUOHtxBaGh7fvppoYOZiYhIQXNbgWyMqQIsAoZYa1sDQ4FPjTHlM+zqByy01rZP9zq7BEVfoAtwpbX2MiABCHHTRxCRTBw4sJodO6YSH38QsMTHH2THjqklskj29PRk9Og+zJs3jAoVygIQH3+SGTN6smTJCPUli4iUEO68g9wZ2GqtjQSw1q4C9gK3ZNjPD7jNGPN96mu0MaZS6rb7gbestXGp7ycCDxR+6iKSlZiYOSQnx58TS06OJyZmjkMZFb6ePa9j7doQGje+yBVbsSKIKVO6qi9ZRKQEcGeBfAmwI0NsR2o8vUVAQ2ttO+AOoBHwQRbn2AFUM8ZUzngxY8zA1DaO6EOHjhVE/iKSifj4Q3mKlxSXX96AiIhxdO58lSu2adMK9SWLiJQA7iyQDZDx+4+JGXOw1sbZ1Okl1tojwPPAXcaYcpmcIzH1x/M+h7V2hrU2wFob4OfnW0AfQUQy8vHxy1O8JKlatSJLlrzE8OHn9yWvX/+xg5mJiMiFcGeBvAuonyFWPzWeHU/gNBCfyTnqAycAzYAVcUj9+g/j4eFzTszDw4f69R92KCP38vT05I03+vDhh8PP6Ut+++37WLz4RfUli4gUQ15uvNanwHhjTCtr7a/GmLbApcC3xphwoJ+19g9jTG9ghbX2qDHGCwgCZltrk40xs4HHjTHzrbVngKeARbY0zMsWKaJq1uwApPQix8cfwsfHj/r1H3bFS4t7772WSy+tS69eQWzfvheAL78M5p9/NtC//zwqVKjmcIYiRZOXVwKNG++ifPnTTqcixVxSkif791fhwAE/rL2we8BuK5CttbHGmF7Au8YYS0p7RBegPNAAONtHXA5YaYxJBiywGhiZuu0DoAnwgzEmEdgMPOmuzyAimatZs0OpK4gz07JlfcLDx9K373hWrPgJgM2bvyQo6GoCAxdTt25rhzMUKXoaN95FvXqVqFSpIcYYp9ORYspaS1JSAr6++6lYcRc7dmRsWsgbUxpuvvr7N7FRUWE57ygiUgCSkpJ4/fX5BAen9SGXL+/Dgw9+QEDAfQ5mJlL0tGnzOw0bXqriWAqEtcn8/fdWNmxoken2wECz3lobkNN5NElPRKSAeXp6MmrUQyxY8AIVK6b0JZ86Fc/MmfezaNEL6ksWyUDFsRQUYwqmtFWBLCJSSHr0uIZ168bSpEltV+yrr0KZPPkOTp484mBmIiKSHXc+pCci6Rw4sLrQH2z79deRHDu20fXe17c1rVqNKvC83PFZ3HmdgnTZZfWIiAilb983+eKL9QD8/vvXBAUFEBi4RH3JIlJq+fs35LXXxnHXXT1zfcxDD3XliisCGD78tcJLDN1BFnGEO8YzZyyOAY4d28ivv47M4oj85eWuUdPFeaR1lSoVWbz4JUaMSOs/PnToL0JDr+HHH+c7mJmI5Ef37h3573//k+X2/fv30rx5db788rMLvtbhw4d47rkBXHVVA1q2vIiAgEt48cWniIuLy/lg4Ndff2bKlHEXnEd+nDx5kqee6kejRpW47LJajB//Rrb7h4evombNotFuowJZxAHuGM+csTjOKZ7fvNw1arq4j7T28PDgtdce5KOP/uPqSz5z5hTvvPMACxcOJykpMYcziEhmWraEmjXPf7Vs6VxOPj5lufzyNlStWv2Cz9W/f09Onz7NmjWb2LRpH998s566dRuQkHAmV8dv2vQzU6c6UyAPHz6I33//lc8/D2fatLnMmDGBDz6Y4UgueaUCWcQBRXU8c37yctdnKapfs7zq3r0969aNpWnTtL7kr78ey+TJd3DixGEHMxMpng4ezFvcHapUqcrChd/Qtu21F3yuDRt+oFOnO6lYsaLr3EOGDMXXt3IORzrr4MEDLFo0j7Fjp9OyZWs6dLiVp59+kRkzJjidWq6oQBZxQFEdz5yfvNz1WYrq1yw/UvqSx9KlS9pKQ1u2fENQUAC7dv3iYGYiklehoa9x881tePXVobRseRF33NGexMREatY0hIevAuD778Pp3LktrVrVpnXrOjz99KPExuZuCPC113YkNPRVoqLWZbnP558vonPntrRpUx9//4Y8//xAYmOPMnhwH1566RkOHz5Iy5YXccstVwGQmJjIhAljaNOmPs2bV6dt2ybMmTPTdb5t236nR4+baNWqNi1bXsRjj/Vk9+5/XNfq1q0DrVvXoWXLi+jZs5NrW3q//voTZcqUoU2bq12x66+/iW3bfufEieOu2KlTp4iNPcrp02mDYhITE0lMTOTkyZPs27eH33//jeXLlxAdHZWrr1lBUIEs4gB3jGf29c384a+s4vnNy12jpkvaSOvKlSuwaNEIXnrpflfs8OG/CQm5hh9//NDBzEQkr3777WcqVqzExo27WbTou/O2v/TS09xwwy38+useIiO3ccUV/hw7Fpurc0+e/D7Nml1Gt2430rHjFcyf/z5JSWlLRS5d+jELFrzH7NmfsWFDDGvXbubUqVOMGPE0U6fOZvToiVSvXoNNm/axcmXKAKPRo0ewbNkili2LYOvWw8yZ8xkTJoxm9uy3AQgKeplatS5m48bd/PTTTrp06cGBA/sA2LLlN0aMGMOGDTH88ssuqlatlmlv8fHjx6hcueo5sWrVUm5onDx5whV76qm+NG1albFjX3PFatf2pnZtbxo1qsi1117KgAH3sWDB+5w6dTJXX7OCoFUsRBzgjvHMrVqNyvMqFvnJy12jpkviSGsPDw9effUBrryyEY89NpHjx+NISIjjnXceZOfO9fToEYynp/6aFinqLr64DkOHjsQYQ7ly5UhMPPeZgquuase6dd/y88/RXHllAP37pw0Bvu22qzO9A7tpU0pB6udXg/ffX8y2bb/zzjv/Y9iwJ5g7dybz56+gQoUKBAe/wqFDB7jppitcxyYlJVKjRq1Mc7XWMmvWFKZP/5DatesC0KxZCwYNGsrMmZPo02cAV13VjrlzZ7Ju3XfccMPN9Oz5kOv4oUNHcvjwIdasWcmuXTs5fvwYR46c3+p20UV1iI3995zYkSOH8PLyws+vpiv2zjsfu1axSE5OJiYm5eFDYwze3t54eJx7L3fOnLcpW7Zspp+tIOlvXhGHuGM8c05LumUmP3m5a9R0SR1p3a1be9atq0OvXsFs27YbgG++CWPXrp95/PH5VKxY/NpIREqT2rXrZjvsJCRkCgsWfMCwYYFYa3nyyeF0757y3aOvvvoxV9do1qwFISFTeOih/nTteh0zZ07imWdeZOfOP5k6dQ7duuVuSuehQwc5deoUdeqcO4q5Xr0G7Nz5JwBPPTWcpk0vJSxsFC+8MJiBA5/lkUcGkpSUxFNP9eW3336mU6eu1K3bgIYNG7Nly2/nXadp00s5c+YM0dFRBAS0B2DNmpU0bXopnp6emebm4eFB2bJliY+P5/TpOE6fPn+ljrFjp+PjU/gFslosRESKgBYt6hEeHsqdd6b1623ZspIxYwL455+fHcxMpGirUSNvcScYY+jduy9ffx1NSMhUXnhhMMuXL8nxOGstycnJ58Rat76KZs0uc911vvjiOmzYkLsiG6B6dT/Kly/Prl07z4n/88/f1K/fyPX+9tvvZsmSVbz//hKmTh3H229PYuHCuURErOabb37i1VdD6d9/iOsudEbVqlXnnnseZPjwQWzatJHVq79h0qQg+vd/KsccZ86cTNOmVbN8TZoUnOvPm18qkEVEiojKlSuwcOGLvPxyWl/ykSM7GT++HT/8MM/BzESKrk2b4MCB81+bNjmdWYrY2KOEhLzqKmjr1WuIt3eZXPUgHzsWS+/ed7B162ZX7NtvV7Bt22a6dOkBwDPPjGDWrCksX76E5ORkkpKSWL36G5YtWwxAuXLliYs7xYkTJ9i3bw9xcXH07/8UYWGj2LNnF5DyUN60aWE8/vjTALzxxovs2LENSCnAy5evwLFjscTHx5OYmMDRoymTQNet+47p08dnmX9o6DQuv/xKuna9jkGDHmLQoOd55JGBufq6XXttBw4csOe9OnW6M1fHXyi1WIiIFCEeHh6MHPkAbdo0pl+/Nzl+PI64uDO8++5DxMSsp0ePEPUlixQjZcuW4/TpOO6552aOHz+Gj48PvXv3o1evnB8wrlChIgEB1/Doo/dw/PgxjDHUq9eQ6dM/pGPHTgD06TMALy8vQkNf5bnnHsfbuwwtWrRixIjRANx44600b96SFi38qFXrYj79dA0jRoymUiVfunS5hri4U/j6VuGZZ0bQp88A13X79evB4cMH8fLyplOnrgwZMgxjDJGRq+nQoRVeXt60b38D/foNJiJiVRb5V2DSpFlMmjSrIL6UbmWstU7nUOj8/ZvYqKgwp9MQOcf27dPZv/8rIBnwoFat22jSJDDbY/L60J1GQBdvW7bsomfPIFdfMkDz5jczYMAC9SVLidGmze80atTC6TTEAVmNmp4yZRxffLGEuXM/P++YgQN74+/fPttR03/99TsbNmT+eyow0Ky31gZkujEdtViIOCClOF5BSnEMkMz+/SvYvn16lsfkdXS0RkAXf5deWpeIiLF07drWFdu69VvGjAkgJuYnBzMTESlcP/wQnmn/8XfffemW66tAFnFAyp3j3Mch76OjNQK6ZPD1Lc8nn/yHkSMfcMWOHNnJ2LHX8f33cx3MTETkwqxf//d5d48BhgwZmmn/8dlXdnePC4oKZBFHJOcxnncaAV1yeHh48PLL97No0Qh8fcsDkJBwmlmzHuajj/6PpKTEHM4gIiJ5oQJZxBFZ/dEruD+SGgFd8nTt2pbw8LE0b562rNK3305g4sROHD9+0MHMRERKFhXIIg6oVeu2PMUh76OjNQK6ZGrevA7h4aHcfXc7V2zbtlUEBakvWUSkoLi1QDbGdDDG/GSM2WiMiTbGtM9kn1rGmLeMMb8bY34wxqw1xrRK3VbdGHPSGBOV7jXOnZ9BpCA0aRJIrVq3k/ZH0INatW7PdhWLVq1GnVcMZ7eKRc2aHWjceDA+PjUAg49PDRo3HlwoI6DdcR1J4+tbno8+eoFXX33ANb3ryJEYxo69jqio2Q5nJyJS/LltmTdjTBVgB9DVWhtpjOkILAAaWWtPpdvvDqCctXZR6vvngc7W2tuMMc2B/1lrO+Xl2lrmTURKqmXLfqRv3zc5dsz11yg33/wM9947Fk9PbwczE8kdLfMmBa24LfPWGdhqrY0EsNauAvYCt6TfyVr7xdniONVe0gaa+AGtjTGRxpgfjTGTjTG1Cj91EZGi6c47ryYiYiyXXpq+L3kiEyfexrFjBxzMTESk+HJngXwJKXeQ09uRGs9UavE7Cng9NRQN1LbWXgN0BM4Ay83Z7zGee+zA1DaO6EOHjhVA+iIiRVOzZnUIDx9Lt25pXWtn+5J37lzvYGYiIsWTOwtkAyRliCVmlYMxpjqwHHjNWrsawFobb61NSv35SWA4cCnQJOPx1toZ1toAa22An59vwX0KEZEiqFKlcixYMJzXX3/I1Zf877//MH78NURFfeBwdiIixYtXzrsUmF3ArRli9YFPMu5ojLkY+AIItdbOy+achpQCW7eIJVPuGoGcn7HR69c/xenT/7jely1bD3//ydkeEx5+L+f+P9OT665bmM3+95HyjZazynDddR9le43vv3+MxMQjrvdeXtVo1+7dbI/RSOuiwcPDgxdf7MUVVzSib9/xxMaeIj4+gffe68vOndH07BmmvmSRAtK9e0f8/dvzyivB523bv38vN954OZMmvUfnzndd0HUOHz7E6NEvsmrVV8THx1OuXHk6dbqTkSNDKVeuXI7H//rrz6xZ8w1Dhgy9oDxKG3feQf6UlP7hsytStCXl7u+3xphwY0zT1HgDYCXw34zFsTHm7tTimdS2ilHAKmvtfjd+Dikm3DUCOT9jozMWxwCnT//D+vVPZXnM+cUxQFJqPLP9MxbHAGdS45nLWBwDJCYe4fvvH8vyGI20Lnq6dAkgImIcLVrUc8W++24yEybcqr5kKZGOHp3Ltm0N2bTJg23bGnL0qLNTJn18ynL55W2oWrX6BZ+rf/+enD59mjVrNrFp0z6++WY9des2ICEh49/vmdu06WemTtWCX3nltgLZWhsL9ALeNcb8AEwAugDlgQZA5dRdw4BawLB0S7md/RfQAouMMdHA90BN4CF3fQYpXtw1Ajk/Y6MzFsc5xVNkLI5zimf1l2fWf6lmLI5zioNGWhdVTZvWZt26ULp3T+tL/uOPNQQF+bNzZ7SDmYkUrKNH57Jnz0ASEnYCloSEnezZM9DRIrlKlaosXPgNbdtee8Hn2rDhBzp1upOKFSu6zj1kyFB8fSvncKRcCLeug2yt/c5ae7W1tq219lprbaS19h9rbV1rbXTqPj2ttdWtte3TvTqkbvvMWntNam9xW2vt49barP/lllLNfSOQC39sdFGmkdZFV0pf8guMGpW+L3kXY8deT0TEe84mJ1JADhx4iXSrxQJg7SkOHHjJLdcPDX2Nm29uw6uvDqVly4u44472JCYmUrOmITx8FQDffx9O585tadWqNq1b1+Hppx8lNvZors5/7bUdCQ19laiodVnu8/nni+jcuS1t2tTH378hzz8/kNjYowwe3IeXXnqGw4cP0rLlRdxyy1UAJCYmMmHCGNq0qU/z5tVp27YJc+bMdJ1v27bf6dHjJlq1qk3Llhfx2GM92b37H9e1unXrQOvWdWjZ8iJ69uzk2laSaJKelFjuG4Fc+GOjizKNtC7ajDH85z+9+PTTl6lSpQIAiYnxfPDBo8yf/xRJSQkOZyhyYRISYvIULwy//fYzFStWYuPG3Sxa9N1521966WluuOEWfv11D5GR27jiCn+OHYvN1bknT36fZs0uo1u3G+nY8Qrmz3+fpKS07xwuXfoxCxa8x+zZn7FhQwxr127m1KlTjBjxNFOnzmb06IlUr16DTZv2sXJlyrTN0aNHsGzZIpYti2Dr1sPMmfMZEyaMZvbstwEICnqZWrUuZuPG3fz00066dOnBgQP7ANiy5TdGjBjDhg0x/PLLLqpWrcb48W9c6JewyCkd/4JLqeSuEcj5GRtdtmy9PMVTeOYxXiaP8ZQH8vISB420Li5uv92fiIhxXHZZfVds1ar/8eabt3DsmB7jkOLL27t+nuKF4eKL6zB06Eg8PT0zfXDuqqvasW7dt/z8czQVKlSgf/8nqVevAQC33XY1LVtedN7rLD+/Grz//mLWrt1Eu3bXM2zYE3Tv3pGTJ08CEBz8Ct9/v46bbrqCli0v4uqrL+G771awcWPmSzxaa5k1awrPPz+S2rVT1k9v1qwFgwYNZebMSa58f/45mnXrvsPHx4eePR+iTZurARg6dCRNmjRnzZqVfPjhLI4fP8bff28vuC9mEaECWUosd41Azs/YaH//yecVwzmtYpGyWkXGYjjrVSxSVqvIWAxnv4pFu3bvnlcM57SKhUZaFx9NmlzMunUh3HNPWl/k9u1rGTPGn7/++sHBzETyr2bN0RhT/pyYMeWpWXO023KoXbsumYxkcAkJmUK/foMZNiyQW2/1Z8mSBa5tX331I5s27TvvlVGzZi0ICZnCsmUR/PJLtKuY3bnzT8aOnX7OsVu2HGLt2k2Z5nLo0EFOnTpFnTrn/geiXr0G7Nz5JwBPPTWc114bR1jYKK699lLee286ycnJJCQkEBj4IN263ciaNd9w5swZGjZsfM4d7ZLCncu8ibhdzZod3FJANWkSmOOybhnltKRbZrJb0i3z/bNf0i0zOS3plhl3fZ3ddZ2SrGLFcnz44TBCQxcycuRcrLUcPbqbsLAbefDBaVx77aNOpyiSJ1WqpDyrf+DASyQkxODtXZ+aNUe74kWBMYbevfvSu3df1q//ngcf7EKZMj506dI92+OstVhr8fBIu5/ZuvVVNGt2mavv9+KL67Bhw49065b1CkXpVa/uR/ny5dm1ayetWl3piv/zz9/Ur9/I9f722+/m9tvv5o8/tvDQQ12Jjz9N5cpViIhYzQ8/7KBs2bIATJoUzJYtv+X6a1Fc6A6yiEgpY4zhhRd6snTpKxn6kh/jww+fJDExd8tHiRQVVao8RLNmf9OyZTLNmv1dpIrj2NijhIS86ipo69VriLd3mVz1IB87Fkvv3newdetmV+zbb1ewbdtmunTpAcAzz4xg1qwpLF++hOTkZJKSkli9+huWLVsMQLly5YmLO8WJEyfYt28PcXFx9O//FGFho9izZxeQ8lDetGlhPP740wC88caL7NixDUgpwMuXr8CxY7HEx8eTmJjA0aMp6yOsW/cd06ePL6CvVNGiO8giIqVU585XERExjl69gti0KeWBptWrp7B79y8MHPgJvr61HM5QpPgrW7Ycp0/Hcc89N3P8+DF8fHzo3bsfvXrl/PxEhQoVCQi4hkcfvYfjx49hjKFevYZMn/4hHTt2AqBPnwF4eXkRGvoqzz33ON7eZWjRohUjRqS0mNx44600b96SFi38qFXrYj79dA0jRoymUiVfunS5hri4U/j6VuGZZ0bQp88A13X79evB4cMH8fLyplOnrgwZMgxjDJGRq+nQoRVeXt60b38D/foNJiJiVSF99ZxjrLVO51Do/P2b2KioMKfTEBEpkk6ciGPAgMksXBjhilWpUocnnlhIo0btHMxMSoM2bX6nUaMWTqchJchff/3Ohg2Z/54KDDTrrbUBOZ1DLRYiIqVcxYrlmDdvGKNHP+J60Ojo0d28+eb1hIfnvSddRKS4U4uFSAYHDqwmJmYO8fGH8PHxo379h4vMg2EpY62/ImUIiQe1at2W7cOBed1fSi9jDMOG3cMVVzSiT58w/v33BGfOJDJ7dn927ozmvvsm4OWV9RKBIiIlie4gi6Rz4MBqduyYSnz8QcASH3+QHTumcuDA6hyPLWwpxe4K0ib0JbN//wq2b59eIPuLANx2WxsiIsZy+eUNXLE1a6bx5ps3Ext7/tJTIiIlkQpkkXRiYuaQnBx/Tiw5OZ6YmDkOZZQm5U5w4cVFzmrc+GLWrg2hZ8/rXLEdO8IZM8afP/+McjAzERH3UIEskk58/KE8xd0ruZDjImkqVCjL3LlDGTPmEdcarLGxexg/vgPr1s10ODsRkcKlAlkkHR8fvzzF3SurP64FFRc5lzGGoUPv4bPPXqFq1YoAJCaeYc6cAcybN0jrJUuBKQ0raol7FNTvJf1LKZJO/foP47hQ89YAAAnzSURBVOHhc07Mw8OH+vVzXq+ysNWqdVuhxkWy0qlTGyIjx9GqVUNXbM2a6YwffxOxsXudS0xKhKQkT5KSEpxOQ0qIhIQ4EhK8L/g8KpBF0qlZswONGw/Gx6cGYPDxqUHjxoOLxCoWTZoEUqvW7aT9sfWgVq3bs1yVIq/7i2TnkksuYs2aYHr1ut4V+/PPiNS+5EgHM5Pibv/+Khw5sh9r1f4l+Wet5cyZU+zdu5uYmJoXfD4NChERkVyz1vLmm58yYsQHJCenFDSent707j2FG24Y4HB2UhwZk8wll+zC1/ek06lIMZeQ4E1MTE2OHfPNcp/cDgrROsgiIpJrxhiee647rVs35OGHwzhy5DhJSQnM/f/27j/Wq7qO4/jzBQoOtJxQumEIJkslgYkZmzN/kRbZrAzRtFxNUTNLnZuuWWk6N5xhOi1F5wzR0jkn1rSmmaJ50Uin+SOGV9KJUqYCoxTuhVd/fA90/HLv5Qu4c+798nps313O53zOOe/v9tm97314n8/n9pm89toiTjjhWnbccejmbxRRsAfR2Tm67jAiPiAlFhERscWmTp1ER8dVTJgwZmPbY4/N4eqrj2DFijfqCywi4kOQBDkiIrbK2LG7s2DBLGbMOHRj2yuvdHDFFZPp7HyixsgiIrZNpQmypMMkPS3pOUmLJE3poY8kXSZpsaQXJc2TNHxz5yIionrDhg1l7tzzufLKb29cL3nVquXMnn04CxbcWHN0ERFbp7IEWdKuwD3A2bYnABcA8yUNa+p6KjANmGR7f6ALmNXCuYiIqIEkzj33OO6//yeMGLELAOvWdXHHHWcyb95MurrWbOYOERH9S5UzyMcAi213ANh+BHgTOKqp3wzgRtvvFcfXACe1cC4iImp05JET6ei4iokTx25se/zxm5g9+/DUJUfEgFLlKhZ7A51NbZ1Fe1/9OoHdJH20r3O2V5ZvImkmMLM4XDNkyFee38b4Y2AbCfSH/aKjPhkDNVm6dCEXXTSq7jAgYyAyBgI+1UqnKhNkAeua2rrZdBa7uV938XPQZs59gO05wBwASYtaWfMu2lfGQGQMRMZAZAyEpEWt9KuyxOJ1oHmhw9FFe1/9RgOrgRWbORcRERERsc2qTJDnAxMkHQAg6WBgX+BhSX+WNK7odxtwmqQhxfE5wD1ubPnX17mIiIiIiG1WWYmF7ZWSpgO3SDKN8ohpwDBgL+CjRde5wD7AU5K6gReB77Vwri9zPrQvEgNVxkBkDETGQGQMREtjQJl8jYiIiIj4v+ykFxERERFRkgQ5IiIiIqIkCXJERERERElbJ8iSDpP0tKTnJC2SNKXumKJaknaUdIGkLkkn1h1PVE/SGZKeLX4HPCfpu3XHFNWSdFkxBp4q/iacVXdMUQ9J4yW9I+mSumOJakn6m6S/SFpYfB7uq3+VG4VUStKuwD3AsbY7JB0OzJc01vZ/640uKnQ6YGBh3YFE9SQNBsYBh9heLWkU8LKk+baX1RxeVOdd4CDbXZI+BiyV9HvbS+sOLKpT5AXXA7+uO5aoxS7ARNvrW+nczjPIxwCLbXcA2H4EeBM4qs6golq2f2H7Z2y6i2NsB2yvs32B7dVF09vAWmBwjWFFxWzPtt1VHI6hscHUO/VFFFWTNAj4FfBD4K2aw4l67AY8KukZSXdJmtRX53ZOkPcGOpvaOov2iNg+/Ry40/ZrdQcS1ZI0TtIS4A/AKbZX1h1TVOpy4EHbT9QdSNRmd9uHApNpbF73kKTmHZ43aucEWWw6a9hNe3/niOiFpMuBUbS2uVC0GdtLbI+j8b+IcyWNrzumqIak44HRtq+rO5aoj+33ip/rbd8O/JVGtUGP2rYGGXgdmNrUNhq4u4ZYIqJGkq4CPgkcb3tt3fFEfWw/I6kDOAJ4oe54ohJfBPaTtOFdlD2h8cKe7en1hRU1Gwys6u1kO8+mzgcmSDoAQNLBwL7Ag7VGFRGVkTRI0g3AJ4DpSY63P5IOkHSCJBXHo4DPAovqjSyqYvs025NtT7E9BbgZuDnJ8fZD0mckTS4dTwP2o1Fy1aO2nUG2vVLSdOAWSaZRXjHN9oqaQ4uI6kwDzqCRDD1e5EgAF9t+qLaookqvAmcCF0rqAoYAP7KdlW0ith+rgdmS9gDW0HhJ9+i+ckLZriq4iIiIiIh+r51LLCIiIiIitlgS5IiIiIiIkiTIERERERElSZAjIiIiIkqSIEdERERElCRBjogYgCQdLelJSaslLZb0/abzCyVt0a6BpbWCDyqWx9ySa2+VlJ3KIqItJEGOiKiZpIslufQ5p+nYkk4p9T8QuBe4HtiLxlrPF0ma2eLzbpfUXfp0SVoPnNXHNXtLelDS+5I6JZ24bd86IqL/SoIcEVG/WcAuwIGAgRuAGcCYov3vwMpS/+nAA7bn2n7b9iPAVcDJLT7vm8BOxWcoMJ7G4vnze+osaSjwAPAG8Gngp8Ctkg5t+RtGRAwgbbuTXkTEQGG7C+iSNAR413aXpDuAqbZflTQSeKt0ySBgXfNttuB564H1AMXOUncBl9he1sslxwEfB04vtut+WdLngPOAx1p9bkTEQJEZ5IiI/mMk8Ey5QdLOwPvA0lLz3cCxkr4uabikKcD5wJ2tPkjSIEknA08C99me1Uf3g4EniuR4gz8W7RERbSczyBERNZL0Dxp1xOW2DbPBfyo1Ly/eofuq7XuLGuDLgHnAcuA64JctPG848GMaZRr/BE4tSjT68hHg7aa2f9Mo/4iIaDtJkCMiamR7zFZedx9w31Zc9x9JS4AvA4sBJDX/LXgVOKl0vIxNZ4tHFu0REW0nCXJERD8haTpwW1PzAttH99J/MKBebvcl4L2eTti+ubh+eS/X7gCMAH5THD8PnCdpJ9vvF22fB17o5fqIiAEtCXJERP/xO2CfprY1kmS7p5fw1gCD+7jfNcC5vZ20vUdP7ZIm8cFa6N/SKMe4SdKlwCHAN4CpfTw7ImLASoIcEdEPFLXIe5aaDHQDQ4B/SRpbmr1tdLB3kHQtsLPt7xT3GQ6sBva3/dJmntnSyhe210r6AjCHxmzyMuBbtrOCRUS0pSTIERH9QHMtsqTxwIXAROCk5uS4ZC2NtYw32PB7vbvFRx8EPNtCfEtplFVERLS9JMgREf2ApBE0Nuw4gkb98H40NvK4GthT0ipgWbnUoni5rhvYqfSi3YZk2ZIG225eL7nZYHr5WyBpfbFmckTEdiXrIEdE1EzSxcBLwKU0SirOprG02oE0dtD7AY31iieXrhkDdNGYZf5a8e8uGrXCAEuAR1t4/JM0Xubr6dPjy4EREe1OPb/3ERERERGxfcoMckRERERESRLkiIiIiIiSJMgRERERESVJkCMiIiIiSpIgR0RERESUJEGOiIiIiChJghwRERERUfI/Mb1hsC+9HrgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 720x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "a = -per_clf.coef_[0][0] / per_clf.coef_[0][1]\n",
    "b = -per_clf.intercept_ / per_clf.coef_[0][1]\n",
    "\n",
    "axes = [0, 5, 0, 2]\n",
    "\n",
    "x0, x1 = np.meshgrid(\n",
    "        np.linspace(axes[0], axes[1], 500).reshape(-1, 1),\n",
    "        np.linspace(axes[2], axes[3], 200).reshape(-1, 1),\n",
    "    )\n",
    "X_new = np.c_[x0.ravel(), x1.ravel()]\n",
    "y_predict = per_clf.predict(X_new)\n",
    "zz = y_predict.reshape(x0.shape)\n",
    "\n",
    "plt.figure(figsize=(10, 4))\n",
    "plt.plot(X[y==0, 0], X[y==0, 1], \"bs\", label=\"Iris-Setosa 아님\")\n",
    "plt.plot(X[y==1, 0], X[y==1, 1], \"yo\", label=\"Iris-Setosa\")\n",
    "\n",
    "plt.plot([axes[0], axes[1]], [a * axes[0] + b, a * axes[1] + b], \"k-\", linewidth=3)\n",
    "from matplotlib.colors import ListedColormap\n",
    "custom_cmap = ListedColormap(['#9898ff', '#fafab0'])\n",
    "\n",
    "plt.contourf(x0, x1, zz, cmap=custom_cmap)\n",
    "plt.xlabel(\"꽃잎 길이\", fontsize=14)\n",
    "plt.ylabel(\"꽃잎 너비\", fontsize=14)\n",
    "plt.legend(loc=\"lower right\", fontsize=14)\n",
    "plt.axis(axes)\n",
    "\n",
    "save_fig(\"perceptron_iris_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 활성화 함수"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def logit(z):\n",
    "    return 1 / (1 + np.exp(-z))\n",
    "\n",
    "def relu(z):\n",
    "    return np.maximum(0, z)\n",
    "\n",
    "def derivative(f, z, eps=0.000001):\n",
    "    return (f(z + eps) - f(z - eps))/(2 * eps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAxAAAAEYCAYAAADMNRC5AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzs3Xd8VFXawPHfmfSEkABpFOlVQBCQrkSqiIKKWBYLNuwd39VFd10su64FRUXBuhTBRUVRQREkiALSkSZVEEJIhfQ6c94/7kyYJJNkkkxLeL587mdm7px778lNmDvPPc85R2mtEUIIIYQQQghnmLxdASGEEEIIIUT9IQGEEEIIIYQQwmkSQAghhBBCCCGcJgGEEEIIIYQQwmkSQAghhBBCCCGcJgGEaFCUUrOVUkftlr/ZvfeNUurZGu6vRtsopRKUUvdU8f5tSqkzdssmu/filVI51ew/VimllVKDnK2TEEII91NK/aPc5/tndu9NUUrtrmb7AdbP9+bur60QdSMBhGgQlFLDlFJHgcvLvTXVGki0qmb7OUqpBU4e65D1Q962VBowlKe1/khrHWm39Hd2W6trAAuwr4bbCSGEqCHrTSFdxTLUVlZr/c9yn+/X1vBw1wDpwCmX/hBCuIG/tysghCtordcCbZVScUB/IBzIAH7WWmcDKKWq2kUcsNtazh9oZF1f+n9EKXU+cD7wd8oG38VKqWuB1ZXtXCnVGvitiuM/C+yoqoJKqQuAF4E04F2l1K1a68KqthFCCFFn/wBeq+S9PGsQ8U0V299e3QGUUiOBh4Fs4BWl1BNaa0uNayqEh0gAIRoMpdQY4GtgP8ZdnNZApFJqgNb6oLVYsFIqEsjVWhdbtwsH4oFG1uDhMut+bDZaH2OAHlVUYX1lb2it/7TWxR8YCbTHCAR+1FqnWesRb32MNDbRmdbXJuAe4GXgTevjMmCLUmo68LWWGSGFEMJdirTWVaWX/ozx+R4CjAZaAknAD7btlFJTAJP1891sd2MrEPgb8CTwGLACWA70V0o9bb05JoTPkRQm0ZC0A7YBF2mt4zFaIiwYrQs2fwVOAxPt1s0CEoB8YKbW+huttdJaK+BbWyGtdQJGC0AocCtwPzAJSNFaP6u1PllV5ZRSTYGdwN3WfQwF9lnvPNmEWeuXbN3mQeBP4AngZq31k1rrdGAYMA/4EDiulBrtxPkRQgjhBkqpDsBB4GqMz/dxwGGllP1Np24Yn++brdu8AJzEuB6N0VrP1lr/AfTFCEq+VUodVEr18txPIoRzpAVCNCRzML54z1ZKXQ28g/HB/Iv1zhDAP7XWzwIopUKBmcDFwGCMAGKlUupr4BGt9WEHx/gbMAYYqrVOVEr1AX5USv2htV5hLeOvlAoGirXWZrttrwUsWuurbSusnaYfA1ZZV+VqrRvZbbMZeARYprUusq3UWpcALyul3sRo0fjJ+dMkhBDCxe4Atmutp9hWKKU+Ae6zLgB7tNb2AcU667LSPl1Ja50HPKWUehG4lKrTX4XwCmmBEA2CUmohsAt4DuMDNxK4AXgP2I7R8lDeMiAK6K+1TrE2KQ/FuPMTX8mhzgfWaK0TAbTW2zD6TthfFN7ECEYeLLftfqC1tcN3kFKqvbWuv1f2c2mtN2qtP7MPHsq9X2BtMSmobB9CCCHq5F9VdKK+yVrmd6CXUqq3UirQ2vJwEVV/vn9nXRz2ddBaZ2utl0mKqvBF0gIhGor7gACMlKUSIM96l76UUmoIRvOxzdW2PFQba4vBS3arfgKO2L1eDHxkHfHpEEbA0YuyneTu1Vq/W76CWuu1Sqn7rPtvY63LMuCf1iJnMO5G2er7MUaqlLPGaq2/q0F5IYQQVRsL+FmfXwW8CnSwe78AQGs9TynVBCOttAVGa/gHwFvWcqewpi6BMboTRiqqs7pprSsNRoTwNCWBrWhIlFIvUfWX7le01q+U28YEPAD8BePC0Azjy/wmjD4R35crPxi4E7gNo0/Eh7Z0J+tFYbGjAMJu+8aAAmz/+bT9c611rl29yrcSdscYrSkKyCz3nlnuVAkhhHtYR9t7S2sdV0WZCMp+tts/mrXW+dZyfhjXAXuXY/Rtiyq/3/I3xITwNmmBEA3NDIxRihyZx9nhWe29gtEZ+mGMFocsoDlwPbBMKXW11nq5rbDWer1SKg8jgHgDCLL2hXDWASDa+lyVe8y11dHarF2maVspZetTYZYLihBCuEclX/BN1vcqfHfSWpdY+76d4ezntiq3jz1Y013L9Y+zHdNi21dd6y+Eu0kAIRqaV4HJGMO4OrLSwbrLgde11l/YrTsG/EcpNQKjCXu5Umon0MX6vu3/zp8Y800kA/bbV2eC1rrMuOHWYVyrGktcCCGEZ6ym8hSj4vIrlFLdgKPWl7201rvLvT8FmObC+gnhVRJAiIZoodba6dmhMcbdflApdQhjLoccjDkfrsG4gLxtLTcUIxdWY9xhynfQz2KEk8cMs44Hbs9R64gQQggPsw4FXiPWFgiAcAef76F1rpQQPkQCCNEQ3WE3MkZ5m7TWw8utm4bR4vAUxgRvTTGaobcC19jSl8p3uK6jxZWsz3XhMYQQQnheZZOK7vFoLYRwI+lELYQQQgghhHCazAMhhBDCpZRSAUqpaUqpYqXUDZWUiVVKzVFK7VNKbVJKrVNK9fR0XYUQQtScBBBCCCFc7S6MvkIbqyjTB/hea91Na90f+BJjEAQhhBA+zqf7QERFRem2bdt6uxrk5uYSFhbm7Wr4FDknFck5qUjOSUW+ck62bt2aprWOrr5kzWmtZwMopa6sosyKcquSqOSapJSaCkwFCAkJ6Xveeee5qKZ1Y7FYMJnkPpw9OScVyTmpSM5JRb5yTg4cOODUtcGnA4i2bduyZcsWb1eDhIQE4uPjvV0NnyLnpCJfPifmfDO7J+wm6qooWtzTAmUqP7y5e/jyOfEWXzknSqlj3q6DjVIqFmMOlzscva+1ngvMBejXr5/2hesC+M7v0pfIOalIzklFck4q8pVz4uy1wfuhjhDC7dK/Sef0D6dJ+jDJY8GDEM5QSjUDlgPPaq3Xers+QgghqufTLRBCCNdoOrYp3RZ0w6+Rn7erIkQppVRzjHlY/qO1/sTb9RFCCOEcCSCEOAf4N/IndnKst6shzmHWloZlwBSt9UGlVBuM4OEZrfXn3q2dEEKImpAUJiGEEJ4QCrQBIqyvXwVigSeUUhuti6QwCSFEPSAtEEI0cLvG7yIwNpC2z7UlKC7I29UR5xCtdbzd8+NAK7vX13qjTkIIIepOWiCEaMDyj+aT/nU6yQuT8QuT/g9CCCGEqDsJIIRowFI+SQEg6qoo/MOlwVEIIYQQdScBhBANlNaa5AXJANKBWgghhBAuIwGEEA1Uzo4c8vblERAVQJPRTbxdHSGEEEI0EBJACNFAJS80Wh+ir4/GFCD/1YUQQgjhGvKtQogGSJt1af+H2JskfUkIIYQQriMBhBAN0JmEMxQlFRHcIZjGAxp7uzpCCCGEaEAkgBCiAbLvPK2U8nJthBBCCNGQSAAhRANjzjeT+nkqIKMvCSGEEML1JIAQooFJ/zodc7aZ8IvCCe0c6u3qCCGEEKKBqXMAoZQKUEpNU0oVK6VuqKSMUko9p5Tar5Taq5RaoJQKq+uxhRAV2UZfktYHIYQQQriDK6amvQvQwMYqytwKXA701lrnK6U+Al4CHnDB8YXwCcGnTsHSpd6uBm0v9iPEEkh0+BZYutmrdYnavRtOn/ZqHZyhNVgsUGIxYbYoSsx2i8VU5rnZokrLa4znWoNFW59b15W+Lrfu2OGj/PTzT2jAYlFoQGvH29jXz9FzqLxche2QvjBCCCFco84BhNZ6NoBS6soqil0PzNFa51tfvwGsRgII0VBoTZ/77vOJL8vh1oVvvFwRoIcL9pFPMGlEkUkEWTSudMmhEQUEV7rkE1L6vAT/ckuAC2rqrEs8eCwhhBDC9VzRAuGM9sBhu9eHgaZKqQitdaZ9QaXUVGAqQGxsLAkJCR6qYuVycnJ8oh6+RM5JWcpsZpg1eEgdOtTLtfEd5pIS/Pwdf8zkmENILIzlREEsxwvjOFkYQ2pxU9KKI0krblL6mGv2XD+OAFWMvzLjp8z4Wxfbcz+MR5OyYEKjlEZxdkFpLMqCRWm0MhOoFYEaFJDrX0J2QDEWZcGMBYtJYzGZsSgNaHrmhBr7UZotEVnk+ZeAdZ9naVrnB9MpLwQFZAQUsy0iu/S9UtZthmREEGb2QynNzvAcTgUVld3ffreeSiGEEA2YpwIIBZjtXpdYHyv0wdBazwXmAvTr10/Hx8e7vXLVSUhIwBfq4UvknJRTVGQ8+vsTvW6dV6qQ/0c+v9/yO83vbE7crXFeqUN5P/6YQNu28ezbR5nlwAFIT3duHwEBEB0NkZHQuHHlS1gYhIRAcLCx2D+3X4KCIDAQ/P3PLn5+YDIBBAABFJQUkJqbQUpuCql5qcZjbiqRwZHc0ecOAPKK8+g3tx9nCs5wpuAM+SX5Zeo976p53NzrZgBm/TqLh7972PHPZwpg3TNFpa8veOcCdqXsIsAUQLB/cJnl1l63Mv2S6QDsTd3Lk6teqVAm0C+QQL9AHh7wMLGNjH4wKw+v5FDGCQL9AgkwBRDgF8DkC5z/PQohhBD2PBVAnABa271uDeQAZzx0fCHcy2IxHv38vFaFlEUpZP6cSdB5QV4LIFJSYNMm+PVXY1m/fii5uY7LBgVB69bQpo2xtG4NcXEQE2Ms0dHGY+PG4KqpLMwWM8m5yRzKSuRE1onS5eGBD9MivAUA93xzD3O2znG4/YVxF5YGEMH+wexP349FG797f5M/TYKbEBkcSWRwJOFB4aXbDWg5gL9f8ncaBTYi8WgifXr0oVFgIxoFNiIsIAytdel8Hb/e+SuBfoH4mar+Wzo/+nyW3bjMqZ97dIfRjO4wusy6yUx2alshhBCiPLcEEEqpZsAyYIrW+iAwH7hTKbVYa10EPAh8oXX5Ln9C1FO2AMLkvZGRWz7YkqBWQYR28VzKT1YWrFkDP/xgLAcOlC/hT1wcdO8O3bqdXbp2NYIFd8xxl1WYxeGMw5iUiV5xvQDYn7afMQvGcCLrBGZtrrDNmI5jSgOIxkGN8Tf5ExMWQ3RoNDFhMaXPOzbtWLqNSZnYfe9uIoIjiAyOJMQ/pNJJ+wa0GsCAVgMASChOIL5XfKX1DwkIqe2PLoQQQniEu1ogQoE2QIT19TygI7BJKVUC7EU6UIuGxGz9UurFAMI/3J+4W9zf8pCWBl99BUuWwOrVUFJy9r2wMLjoIujfHwYMALN5PZMmDXZbXX7+82c2HN/AntQ97E/fz6GMQ6TlpQFwRecr+PrGrwGICo3iWOYxAKJDo2nVuFWZpU1Em9J9Pj/8eV4a+ZJTM3h3i+7mhp9KCCGE8G0uCyC01vF2z48Drexem4Hp1kWIhsfLLRD2KTDuUFwM334Lc+fCypVl46UhQ2DUKBg92gge7PtMJyQUOd5hDeQU5bA9aTtbk7ayM3knLwx/obS14N0t77Jw18Iy5YP9g2nfpH2ZoKBpSFMOPHCA8yLOI9g/uMrjBfoF1rnOQgghREPmqT4QQjRsXuwDoc2azb02EzEkgo6vdcQvzHV1OHUK3noLPvjAeA5GgDBmDFx7LUyYYPRVcKUzBWf4cPuHbE3ayrakbexP24+2G2XouvOvKw0gruh8BU1DmnJ+9Pl0i+pGx6YdaR7eHJMqG8gppejUrJNrKyqEEEKcoySAEMIVvJjCdHrNafL25GEpsGAKdc3xjx6Fl182AofCQmNd165w111wyy0QFeWSw5CSm8Ivf/5CTlFO6YhFAI+vfLz0eYApgB4xPejbvC8XNr+QHjFnZ5e4occN3NDjBtdURgghhBBOkQBCCFfwYgpT8oJkAGInx9Y5jSk5GZ55Bj788GxMdPXV8MgjcPHFde/0nJKbwqojq1h1ZBU///kzBzMOAtAyvCU3XXATSikigyN5auhTtI1sS9/mfekR04Mg/6C6HVgIIYQQLiMBhBCu4KUUJnOembQvjE7DsZNja72fwkJ44w14/nnIzjZ+jJtvhr/+1RhByRVmb57N/cvvL7MuNCCUQa0GMbT1UEosJQT4GTNCvzjiRdccVAghhBAuJwGEEK7gpRSm9K/TMWebCb8onNDOtRu+de1auOMOOGydK/6KK+CVV6BLl9rVKTknmWX7l7HswDI6WDoQTzwAvWJ7EewfzCVtLmFU+1EMazOM3nG9S4MGIYQQQtQPEkAI4QpeSmFKXmhNX7qp5q0P+fkwfTq8/jpoDeefDzNnGqMp1dThjMMs/X0pX/7+JeuPry/t9Nw3sm9pmYGtBpLxfxkyz8E5QCkVADwM/Au4WWu92EEZBcwArgPMwDbgbq11JVMPCiGE8BUSQAjhCl5IYSpKKyJjRQb4Qcz1MTXadts2mDwZfv/dqPL06fD00xBQi8aA+7+9n9lbZpe+DvQLZFT7UVzV9SoiUiNK1/uZ/AgxSfBwjrgL0MDGKsrcClwO9NZa5yulPgJeQuYIEg2A2Qxff20MSHHkSEvi4oyBKI69eIyS0yXVbm9jCjXR7p/tSl/btm/9VGsCmhof2KfmnSJ3V83ibkfbx94cS6MLGgFwevVpMr7LqNE+HW0fOTySZmObAZC3P4+k95OMwsfh8LeHq92no+1DOofQ4i5jJD5zrpmjzx6tUT0dbV/ZeXaWS35PDs6JN35PzpIAQghX8EILROqSVHSJpullTQmMdX7ugoUL4c47oaDAuKDNm2fM3+CMjPwMluxZwoBWA+gd1xuA/i37s2DXAsZ1GsfVXa/mso6XER4UDkBCQkJNfyzRAGitZwMopa6sotj1wBytdb719RvAahwEEEqpqcBUgNjYWJ/5u8rJyfGZuvgKOSewfn0zZs/uQGKiLa20E2+9pbnkklSe2fsHfqk12FljOHbpsbOvZwHJcLzvcbDNG/o+sK5mdXS0/fFGx8H2XXQR8EEN9+lg++Mnj4PtvtEm4BW78hyvfp+Otu8HBzodMNZllt2nUxxtX8l5dpqLfk/lz4lXfk9OkgBCCFfwQh8I2+hLMZOda30wm+HJJ43+DWAEEbNmQUg1HxoWbWHNH2t4f/v7LN23lEJzIff2u5fZ44xWh+t7XM/1Pa6vdoI2IcppD9jfbjsMNFVKRWitM+0Laq3nAnMB+vXrp+Pj4z1WyaokJCTgK3XxFef6OfnwQ6M1V2to3x5u755OxzUH+DEvlrlr29MxpoSH/lZCZKRz+zMFm2gVXzovLyf/eZKSrBJajG2Bf4TxFS71sVTyr8yvbBcOOdo+akJUaV+6zOBMMrtkVrWLChxt33hAYyIvMX7Y/Db5pCojejpy+AjtO7Svdp+Otg9uG0xMvHHdM+ebSfxPYo3q6Wj7ys6zs1zxe3J0Trzxe3KWBBBCuIKHWyDy/8gna30WplATUVdVPylDdjZMmgTff29MBDdrFtxzT9XDsiZmJfLxjo/5YPsH/HHmDwAUilHtRzG83fDSchI4iFpSGH0fbGxXa+9M5y5EHc2fbwxIAfDss0ZqaNqiYn7/upC7ri7g1yPwwc4WrP4EtmyBZjXLGAGgxd0tKqyLvqpus3k62j5iYAQRAyMclHaOo+1D2oXQ+onWABxJOELr+NY12qf99jZ+IX4V1tVEZds7Os81UZvfU3XnxFO/J2dJACGEK3i4D0TKJykARF0VhX+jqv8bnzkDY8fCxo3GrNGffQaXXFL9MV5Y9wLvbHkHgNYRrbm99+3cduFttI6o/Ye1EHZOAPZ/TK2BHOCMd6ojRO0dPAj33ms8f+UVeNw6F2azcc1gLnQf2Y41kTBqFGzdakzK+fnndZ9bRwhvkQBCCFfwYAqT1vrs5HHVjL6UlmaMqrR9O7RpA6tXQ4cOFcsVlBSwaNci4hrFMbbTWAAe7P8gqXmp3HnhnYxsPxI/k2fnuBANi1KqGbAMmKK1PgjMB+5USi3WWhcBDwJfaK21N+spRE0VF8Nf/gK5uXDDDfDYY2ffC2gaAJ2Mu+chGDdwevWCpUvh/feNQEKI+kgCCCFcwcMpTF3/25XUz1JpMrJJpWWSkmDkSNi7Fzp1glWroHW5xoPT+ad5e/PbzPp1Fql5qfRr0Y/LOl6GUopu0d1YMmmJm38ScQ4JBdoAtvbyeUBHYJNSqgTYi4zAJOqhd94xUpJatzaeV9Wq0LatUWbyZJg2Da6+GqKqz0IVwudIACGEK3gwhUkpReP+jWncv3GlZc6cMVoe9u41ZpJetQri4s6+n5iVyMyNM5mzdQ45RTkAXBh3IQ9c9AAajULa1UXdaa3j7Z4fB1rZvTYD062LEPVSZibMmGE8nzWLCp2jTyechpfh1IlTxN1kfAj/5S/G6Hfffw8vvGDMvyNEfSOd1YRwBS/NRO1IYaFxV2v3bmOY1oSEssHDqiOraPdGO17d8Co5RTmMaj+KH2/5ka1Tt3Jr71sxKe//DEIIUR+89BKkp8PFF8P48RXfz9uTB8sha31Whe2UgrffhiNHPFRZIVxIvikI4QoeSmHKWJXB9vjtpH7ueCBxiwVuvdUIGpo3h+++M5rH84vPDh03qNUgmoQ0YdL5k9hy1xZW3rySS9tdipLefEII4bS0NHj9deP5yy87Tl3SFqNLj/Ir+2avXnDLLUb/ieeec3dNhXA9CSCEcAUPpTClLEwhc20mObtyHL7/xBPw6acQHg4rVoBfkxPc+829dHyzI9mF2QCEBYZx8MGD/G/S/+jboq9b6yuEEA3V7NmQnw+XXw4DBjguo83WMQEcfNv6+9+Ne04LF8LJk+6rpxDuIAGEEK7goRaIjm90pMtHXYi7Na7Ce/PmwWuvQUAA/HdxFvNOPU7HWR15d+u7JGUnsfqP1aVlGwdV3n9CCCFE1QoK4K23jOfTplVR0JrdWr4FAoyJ5q65xmiFePNN19dRCHeSAEIIV/BQHwj/xv40n9KckHZlp4/etcuYGA5g/KPfc/uuNry28TUKzYVc1/06dt+3m6u6XuXWugkhxLli/nxITYU+faCqibcrS2Gysc0X8e67kOO4YVkInyQBhBCu4IEWCNuFqLysLLj2WqMpveXFP/B5yGWcKTjDiHYj2Dp1K59e+ynnR5/vtnoJIcS5ROuzLQaPP171sK1VpTABDBwIgwcbI+ctWODaegrhThJACOEKbu4DUZRWxIaWGzj48EHs59nSGu64U3PgAPTsCbPeMtMzticrJq/gh5t/oE/zPm6pjxBCnKs2bzZafaOijJs3VaoihcnmvvuMxw8+cE39hPAEmQdCCFdwcwpT6v9SKTpVRP7B/NLRkpJzkrnq8ZVsXHIz4eHGDKedOo1hQo9RMmu0EEK4ie2L/q23QmBg1WWrS2ECox9EZKQxGd2OHdC7t6tqKoT7SAuEEK7g5hSm5AXJAMROjqXYXMwbG9+g44zRbPzoGgBeeesMnTsbk8xJ8CCEEO6RkwOffGI8v+OO6stXl8IEEBICN91kPJdWCFFfSAAhhCu4MYUp/0g+WRuyMIWZ2Nt7LxfOuZBHVjxKzpLXoTiMcVdnM/WWyOp3JIQQok4++8wIIgYPhm7dnNjAiRQmgDvvNB4XLDBGeBLC10kAIYQruDGFKXmh0fqwv89+hn82nD2pe2i292k4eikxMZqP54a7/JhCCCEqWrjQeJwyxbnyzqQwgTGx3IUXGp2pv/uuDhUUwkMkgBDCFdyUwqS1Lg0gDg87TIh/CI91fZOCFf8E4J13FFFRLj2kEEIIB06dgh9/NObaqbbztJUzKUw2N95oPNpSpITwZRJACOEKbgggDqQfYNv328jfn09ATACPP/E4u+/dw465D5Cbq7j+eqPznRBCCPf73/+Mj/rLLoMmTZzbptXDrWAuxN1WcfLP8q6/3nj8+mvIzq5DRYXwAAkghHAFF/aBKDIX8cJPL3DBOxew7OVlAMTcEEN042g2rWzHjz8awwfKzKVCCOE5ixYZj7aWAmcENQ+CThAUF1Rt2datYehQow/EV1/VspJCeIgEEEK4gov6QGxK3ETfuX15es3TFBcVM3DbQMAYfSk3F6ZNM8r9+98QHV2nQwkhhHDS0aOwcSOEhsL48e47ji04WbzYfccQwhUkgBDCFeqYwlRsLuaZH59h0AeD2J2ymw5NOrCyw0pCzoQQ0imE8IvC+fe/ITER+vaF225zYd2FEEJU6csvjcdx4yAszPntTi04Bf+B0z+edqr8tdcaM1v/8IOkMQnfJgGEEK5QhxQmrTVjFozh+XXPo7XmsYGPseveXbRY3QKA2JtiOXpU8fLLRvlZs9w23YQQQggHbClFV11Vs+2y1mfBCsjbl+dU+ZgYY4jYoiL4/vsaVlIID5KvIUK4Qh1SmJRS3NLrFtpGtmXNrWt4dcyrBBYHkrY0DYCYv8Tw+ONQWAiTJxsXFyGEEJ6Rng7r1oG/P1x+ec22jb0pFqZBZLzzc/VMmGA8Sj8I4cskgBDCFWqYwnT0zFG+2PdF6etbe93K7nt3M6ztMGM3ISYuWHkB7f/dno1/hrJ0qdFs/tJLLq+5EEKIKnz7rXGPKD4eIms4Z2fE4AgYB2Hdnc97sgUQ33wDxcU1O54QniIBhBCu4GQKk9aaD7Z9QM93ejL5i8n8nvY7YLRChAWevcAopYgYFMF5/9eav/7VWPfUU9CypVtqL4QQohK2lgDbF3t369zZmOX6zBmj5UMIXyQBhBCu4EQLxKmcU4xfPJ47v76TnKIcLu90Oc1CmlW522XLYMsWiI2FRx5xZYWFEEJUp6DgbF+E2oy+dHr1afgK8vY71wfCRtKYhK+TAEIIV6imD8Tnez+nx+wefHPgGyKCIph/9Xw+m/QZ0WEVx2JNfCeRHSN2kL4yg2eeMdb97W81G/lDCCFE3a1eDbm50KePMU9DTZ36+BS8Dlmbsmq0nS2A+PJL0LrmxxXC3fy9XQEhGoQqWiBeXPci03+cDsDI9iP5cPyHnBdxXqW7SlmcQuZPmRw9vzm7dsF558Hdd7ul1kIIIapgG761tulL2mx8+1cmVaPt+veHuDj480/YuRN6967d8YVwF2mBEMIVqugDMbHbRJqFNOPNsW/y/U3JY1fxAAAgAElEQVTfVxk8APT4sgcd3+3MM99FAfDMMxBU/SSmQgghXMhiga+/Np7XdPhWG1sAQQ1H+DaZ4MorjeeSxiR8kQQQQriCXQpTXnEec7fORVvbnbtEdeHYI8d4oP8DmFT1/+UCmgSwMqAFuw/50aEDTJnixnoL4QZKqWFKqW1Kqd+UUluUUgMdlIlVSi1RSm1XSm1SSv2slBrqjfoK4civv0JyMrRtCz171nIn1ntLyq9mLRBwNmixtYII4UvqnMKklBoGzLTuqwh4QGu9sVyZvsAPwAG71Uu01q/W9fhC+ARrC8SvgancMudCDqQfIMAUwG0XGlNG24+wVBmtNdqsMWsTM2YY6/75TwgIcFuthXA5pVQk8AVwhdZ6g1IqHvhKKdVOa23fk/RFIA24TmutlVLXAJ8CMtaY8An2oy+pmn//B2qfwgQwfLjR923HDjh2DNq0qV0dhHCHOrVA2F0o7tdaXwBMw7hQhJYrGgV8rrUeaLdI8CAajCJzEc9cCoObLuVA+gG6R3end1zNklazt2SzocUGll9/lGPHoEsXuOEGN1VYCPcZA+zXWm8A0FonAEnAiHLlEoFIwJagF21dJ4RPWL7ceLSlEtVGbVOYAIKDYcwY4/mKFbWvgxDuUNcWiAoXCqWU7ULxtV25KGC0UupX6+tVwL+11tl1PL4QXrcnZQ83Z77K9mGgtGbaoGk8N/w5gv2Da7Sf5IXJFKcWs22tMXPQ//1ftdNKCOGL2gOHy607bF1v7x/AXCBFKXUGSAXGOdqhUmoqMBUgNjaWhIQEV9a31nJycnymLr6ioZyT9PRAdu0aTHCwGbP5ZxISajkUUqrxsGfPHiNcrqH27ZsDXViwII2uXXfXrg4+qKH8nbhSfTsndQ0gnL1QfAF8Ym2mbgq8BcwDri6/Q1+8UNS3X6onyDkx7Dyzkyd+e4JiXUzb0/Dy6WFExY9j488bq9/YnhnjfwSwOCOWqKhCWrXaWPuLlo+Qv5OKzoFzojD+ou2VULHF+ymMdKXztNaZSqm7gG+UUgO01mW211rPxQg26Nevn46Pj3dLxWsqISEBX6mLr2go5+S//zUehw/3Y/ToYbXez2+Rv5FBBj1796RZfNXz/jjSoQO88grs3BnF4MHxBAbWuio+paH8nbhSfTsndQ0gnLpQaK3z7Z5nKKUeB44rpULs37O+73MXivr2S/UEOSeGgSUDeT/pfQZmhPLaO5sIf7Af1OK8ZHyfwW+nfyMtJIQD+eG88qSq00XLV8jfSUXnwDk5AYwst6418Fm5dX8BHtdaZwJord9TSs0AegHb3F5LIaqwcqXxaEshqq26pDCBMYx39+6wZw+sX1+ry4sQblHXUZhOYFwY7LW2rq+KH1AAFNbx+EJ4lNaaeTvncTr/NADB/sFsuGMD7wVMJLyIKmeirkrygmQAluXHEhmpmDrVVTUWwuO+Ai5QSvUEUEr1B7oCPyqlflFKdbKWOwBco5QxNJlS6hKgMfCnF+osRCmLxXUBROkoTLXoRG1z2WXGo/SDEL6krgGEUxcKpdQN1g7XKKX8gX8B87XWljoeXwiPOZVzivGLx3Prl7fy4IoHS9c3CmxU7UzUVTHnmkldaiTKriaW+++H8HCXVFkIj7O2KEwCPlRKbQJeBy4HQoE2QIS16H0YAcM2a/+4l4GJWus0z9daiLO2b4e0NGPm6c6d67av0lGYajGMq83Yscbjd9/VrS5CuFKdUpiseau2C4XGSF9ydKEIAVYrpSyABtYCf6/LsYXwpM/3fs7d39xNen46EUERXNbxsrIFqpiJujppX6VhybWwm8ZkBIfw0EMuqLAXZWVlkZKSQnFxMREREezbt8/bVfIpnjonYWFhtGrVClMtW8XqQmu9BrjIwVut7MqcAm70WKWEcJJ960Nth2+10Za6pTABDB0KoaHw229w8iS0aFG3OgnhCnWeB8LJC8VHwEd1PZYQnpaRn8EDyx9g0e5FAIxsP5KPJnxEq8atyhasYibq6iQvNNKXVhPLbbdBTEydquxVWVlZJCcn07JlS0JCQsjJySFcmlPKyM7Odvs5sVgsJCYmkpaWRkx9/oMSwgu+/954HD267vtq/URrdvXbRWjn8qPbOy8oyJgT4ptvjLrddlvd6yVEXclM1EJU4nT+aXrM7sGi3YsIDQjlrbFv8f1N31cMHqDWKUxFKUVkfJ9BCYo1RPPwwy6ouBelpKTQsmVLQkNDUXW9dSdqzWQyERsbS2ZmprerIkS9kp1tdFY2mWBE+ZlLaqHZuGZwBQS1CKq+cBVs/SAkjUn4ijq3QAjRUDUJacL4LuPZk7qHjyZ8RMemHSsvXMsUppT/pYAZNtOEgWMC6dKlDhX2AcXFxYSEhHi7GgIICAigpKTE29UQol5JSIDiYhg0CJo08XZtzrIFED/8ACUl4C/f3oSXyZ+gEHZWHVlFWEAYg84bBMDrl71OgCkAP1M1qUm1TGFKnm+kL/1ALI8/WE3hekJaHnyD/B6EqDlXpi+B9SbRBig6v4jAmNpP4tChA3TqBAcPwqZNMHiwa+onRG1JCpMQQE5RDvd/ez+j5o/i5qU3k1uUCxjDtFYbPECtUpjyD+eTvSmbPPw41S6qdKQNUX9NnjyZJUuWsHjxYnr37u3t6gghashlw7da/fnvP+F1KDxe91HrJY1J+BIJIMQ5b80fa+j9bm9mb5lNgCmA23rfRpB/DfNVa5HCFNwumI/79GYWHZn6kF9tp5AQbrB48WLi4uIcvvfxxx+jlCqzfGe9oicmJkq/AyHqqT/+MO7wR0TARY6GhqmF6EnRcCUERAfUeV8SQAhfIilM4px1puAMT6x8gve3vw/ABbEXMO+qefSK61XzndUihWnvPsV/t0USFhbJpzKqhk85fPgwsbGxlb7fvXt3du/eXfr6zJkzfPfdd2RkZHiiekIIN7ClL40c6bo+Bm2easMfCX8Q3Dq4zvsaNgwCA2HLFkhPh2bNXFBBIWpJ7nmKc5LWmkv/eynvb3+fQL9AZsTPYPNdm2sXPECNWyC01rz1lvH8lluMO17CdyxZsoTdu3ezceNGh+/v2bMHf39//P39iYuLIyUlhQULFpCcnOzhmgohXMXV6UuuFhZmzAmhNaxe7e3aiHOdBBDinKSU4skhTzL4vMFsv3s7zwx7hkC/2ndwq2kfiH0PHqbT3J10JYsHHqj9YYXrvfrqq6SlpTFz5kxuvPFGTpw4UaFM9+7dKSkpoaSkhFOnTtG5c2cWLFhAt27dvFBjIURdFRef/VLuqg7UANk7smEfmAvMLtmfrW62YEcIb5EUJnFOsGgLH+/4mIz8DKYNngbAdd2vY1L3SZiUC+LoGrRAaK1J/CSVPpZCevdpz/nn1/3wPs1bowFpXaPiRUVFvPjii7z++ussX76cwYMHc+zYMYYNG8aiRYvo37+/3a41+fn5pKWlkZSUxI4dO+jVq5atV0IIr9u0CbKyoEsXaNPGdfvdc+0eOAyFIwsJ7VT7yeRsRo+GJ580AgitvffxKoQEEKLB23lqJ/cvv59fjv9CgCmAid0m0q5JO6MDLC769K1BHwilFDNa9yP4dDpTHm3kmuOLOpkzZw4zZswgOjqan376iQsuuAAwWiNatmzJ8OHDGT9+PM899xwAe/fuJSwsjLCwMFq0aEHXrl3p27evN38EIUQduHr41lLWS4Pyc821plcviImB48fh999BGj2Ft0gAIRqszIJM/r7m77y1+S0s2kJsWCyvjXmNtpFtXX+wGqQwbdsGP+8MoEmTOL6+1vVV8Tk1bAnwhh49ejBz5kwmTpyIX7kg8LHHHmPixIksWLCA5s2bM2XKFKZMmeJwPytWrCAgwBhtZcKECe6uthDCRWwBhKv7P2iz9fPPRQnjJhOMGgULFxqtEBJACG+RPhCiwdFas+C3BXR5qwuzNs0C4KH+D7H/gf38pedf3DPBlpMpTJZiC+/PMcreeisE131gDuECQ4YM4brrrqsQPNi0adOG6dOnExp6NgXh008/5ZFHHilTLiQkBH9/f7799ltuvvlmt9ZZCOEaGRmweTMEBEB8vGv3bQsgXNUCAdIPQvgGaYEQDdInuz4hOTeZwecNZvbls2s/upKznExhOj4/lVHvHSKJNtx1Vyv31kk4xWw2o2vQSuJvHd/x+PHjvPHGG7xlG07Ljtaanj17uqyOQgj3WbXKaCgdOtQY6cilXJzCBEYLBEBCAhQWQlANpy0SwhWkBUI0CCm5KRzKOAQYfQzeHPsmH034iHW3rXN/8ABOpzDtnplME11Mu/aq4XeeridGjBhBQECA08vvv/9eZlvbaEz2ywcffODFn0gIURPuHL7V1SlMAM2bwwUXQF4e/PKL6/YrRE1IC4So1wpKCpj16yxeWPcCvWJ7sXbKWpRSdGjagQ5NO3iuIk6kMBWlFBG8O4MSFH0ei/ZQxUR1Vq9eXasWCDBaL3JyciqUKSwsdEndhBDupbUbO1DjnhQmMOr6229G8DN8uEt3LYRTJIAQ9ZLWmiV7l/DXVX/l6JmjADQKbER2UTaNgxp7vkJOpDDteC0FP2Czf1Puv60Oc04Il6qs34MzEhISCA8Pd/ieDOsqhO/btw9OnDBGNnLLf1k3pDCBEUC88ooRQPz73y7dtRBOkRQmUa9orfnu0Hdc9N5FXP/Z9Rw9c5QeMT34bvJ3LJ+83DvBAzjVApH4X2OWYvOlMYTWfThw4WXTpk1Da13psmPHDm9XUQhRDVv60ujRTs8DWiPuSGECuPhiYxCO7dshJcW1+xbCGRJAiHrlTMEZJi2ZxNakrTRv1Jw5V8xh+93bGdPRDcmrNVFNH4jsfXk0OZVNHn5c+nSUBysmhBCiMu5MXwL3pTAFB8OwYcbzH35w6a6FcIoEEMLnbU7cTEFJAQBNQpowI34G/xn5Hw49dIipfafib/KBTLxqWiA2vmjcItrZOIr+F9c+ZUYIIYRrFBTA2rXGc9vIRi7nphQmkOFchXdJACF8ktaatUfXMnr+aPq/35+Ptn9U+t6jgx7liSFPEBrgQ3lAVfSB0FqT96WRvhR+dSzumIZCCCFEzfz8M+TnG30f4uLccwx3pTBB2QCiHszXKRoYH7h1K8RZWmu+P/w9z//0PL8cN8anaxTYqLQFwmdVkcKUnJBNRE4+6QQy7pkmHq6YEEIIR9w1+7S9Fne3IPFYIsrf9XeOuneHFi3g5EnYvRtk6hnhSdICIXzGioMr6DO3D2MXjuWX47/QJLgJzw57lmOPHOPRQY96u3pVqyKFadOLRuvD/pYxtOsgzQ9CCOEL3Dn/g02nWZ3gUTD5u/7rllJnWyFswZAQniIBhPAZGfkZ7Di1g5iwGF4a+RLHHjnGP+L/QdOQpt6uWvUqSWGyFFvwW2v0fzjvtlhP10oIIYQDSUnGPAqhoTBkiLdrU3vSD0J4i6QwCa/YcnILc7bMoVFgI2ZeNhOASd0nYdEWJnWfRLB/sJdrWEOVpDAd+t8ZwoqL+VOFcuVjjbxQMSE8Tyk1DJiJcY0pAh7QWm90UC4KeBfoChQCK7XWT3myruLcZPvCHR8PQUHuOYa2aLI2ZcF+IN49xxg50miJ+Oknoz9HSIh7jiNEeRJACI/JLszmk12fMHfbXLYlbQMgNCCUGZfOIDwonEC/QG7udbOXa1lLlaQwLf2zCYvozaghZm5pIulLouFTSkUCXwBXaK03KKXiga+UUu201nl25YKAb4AntNbrrOuaeaPO4txjP/+Du+hizfZB241vWne75xjR0dCnD2zdagQR7kzHEsKepDAJtzty+ghTv55Ki9dacM+397AtaRtNQ5ry6MBH2TZ1G+FBjmfyrVccpDBpDfMWKHYSySX/J9+LGjptNwxKfHw8r7zyiluP54lj1NIYYL/WegOA1joBSAJGlCt3C7AReEgptUUpNR8I8GRFxbnJYvFM/weA8P7h0MW9x5A0JuEN0gIh3CK3KJewwDAAisxFvLftPQCGth7K3X3v5trzr61/aUpVcdACsW2LZu9eRVQUXHaZl+olnNavXz+2bt1a6fubN2+mX79+/Oc//+Fvf/tbmffMZjMvvvgiTz3lXPbN4sWLeeSRRzh16pTD9z/++GNuu+22CuuHDBnCzz//7NQxvKg9cLjcusPW9fYuAXoDVwJ/AjOAhVQMNFBKTQWmAsTGxpKQkODaGtdSTk6Oz9TFV9SHc3LgQCPS0voRG1tAUtJGKvlv6Bovuf+cxMREAr1ZujSHK6/c4rbjuFJ9+DvxtPp2TiSAEC5z7MwxPt/3OUv2LqGwpJBtdxtpSl2jujJzzExGtR9F95juXq6lmzjoA3Hw+t38G03mZR0J8KU5K4RDCQkJlJSUOHyvSZOzw+8+/vjjPPTQQwAopfDz8yM6OpouXZy/zXj48GFiY6vuVN+hQwcOHTrk9D59iALM5daVULHFOwb4WGt9FEAp9RKQqZRqpLXOsS+otZ4LzAXo16+fjo+Pd0O1ay4hIQFfqYuvqA/nZMMG4/HKK4O59NJ4tx/P3edk8GB4+mn4449GdO4cT4sWbjuUy9SHvxNPq2/nRAIIUSeHMg7x+d7P+WzfZ2w5efbOR1hAGEnZSTQPbw7AIwMf8VYVPaNcC0RBRglN/zhNFJrGt8h/s/qgUSPnOrn7+fnhZ5eqtnHjRnJzcxk2bJjTx1qyZAm7d+9m48aNDBw40GGZI0eOEBxctpXu6aef5umnn3b6OF5yAhhZbl1r4LNy61KALLvXFrtFCLfxxPwPYKQ16hLt9r/owEC49FL45hsjjWnKFPceTwiQAELUweojqxk5/+z3hLCAMMZ1HsfEbhMZ12lcaQrTOaFcH4gfN/pzIwMZ1TKbJSMDvVgx36D+WXkH8jlXzGFq36kAzN06l7u/qby3of7H2X4Gfef2Le2MX1U5Z/Xr14/t27ejHEwV7ufn53A9wKxZs7jmmmto1sy5fi5vvvkmaWlpzJw5kxtvvJF169bRqlWrCuXat29fX1sgvgJeU0r11FrvUkr1xxhl6Uel1C/AFK31QYyO1n9TSi3WWmcDjwA/2ne0FsLVsrNh/XrjXs+ICslyrlWcXsz66PXQGMh077FGj5YAQniWBBCiSlprDmUcIuFoAiuPrCQiKIL3x78PwKDzBhHXKI6R7UcysdtExnQYQ0jAOTqGXLkUpvnzIYtALry3GZV87xQ+6I033uCBBx5wuvyvv/7Kp59+yvLly6stW1RUxIsvvsjrr7/O8uXLGTx4MMeOHWPYsGEsWrSI/v37V7ptfn4+mZmZJCcn89tvv9GuXTun6+hpWutMpdQk4EOllMZIX7ocCAXaABHWckuVUh2BzUqpfOAYMMU7tRbnitWrobgYBg0Cu8xE97Al8nlguBpbR+offjDuZzmY01QIl5IAQlSQmJXIysMrWXN0DWuOruFE1onS9yKCInj3incBYwjWxMcSMSn5pLJPYTqdZGbZUhOguOkmr9bKZzjbIjC179TS1ojqbJ1aeYfn2rJYLJX2gzCZTJjsrsqpqanceOONdOzYkSeffJJhw4ZVSDmymTNnDjNmzCA6OpoVK1YwaNAgAF599VVatmzJ8OHDGT9+PM899xwdOnQAjH4StlaPkJAQwsPDiYuLo2fPntx9t5vGhHQRrfUa4CIHb7UqV+5l4GWPVEoIYMUK43HsWPcfS5utn3seuER27gxt2sCxY7BjhzG0qxDuJN/8znFF5iI2J27myOkjpes+3fMpty+7nfm/zedE1gmahTRjYreJvH352+y4Zwf+prNxpwQPVnYpTAn3/Ml/CzdwX7cU2rTxbrVEzTz88MMEBAQ4XG6//fbScgUFBUyYMIG+ffuyc+dOlFLccccdlQYfPXr0YObMmWzdupUePXqUee+xxx5jz549dO/enebNjT5DU6ZMMfKnrUteXh7Jycns3LmTBQsWcPHFF9OzZ0+HqU9CCMe09nAAYfFcAKHU2VYIWx8PIdxJWiDOIWaLmUMZh9iWtI1fE3/l18Rf2Z60nUJzIdMvns7zw58HYFT7UUzoMoFL217Kpe0upUdMDwkUqmNNYdJKoVcmE0URQ8bKf6/6ZMuWs4MAvP7667z//vvs3r27Qrnk5GQmTpyIyWRi3rx5BAcH87///Y9LLrmEq666iiVLllTYZsiQIVUeu02bNkyfPr3C+vz8fMzm8gMaGf71r38R5K4pdIVogPbuhePHISbGQ3foPZjCBEYA8d57Rj8IJ0eUFqLW5BtOA3Wm4Ay7kncxtPXQ0jSIgR8MLDNSkk3XqK5EBkeWvu4Z25Mvb/jSY3VtEKwtEAfX+dO0IIt0Arl8ursTbIWn5eXlMXDgQAYNGsQHH3xASIjR56djx4789NNPvPbaa/j7l/1YNZvNZSaZKykpqbSlAiiz/aBBg9i5c2elZV9++WWmTZtW2x9HiHOKrfVhzBjP9BHwZAsEGJ3CTSb45RfIyQEnB5YTolYkgKjnMgsy+fnPn9mfvp/9afuNx/T9nMoxZsY5/uhxWjU20hy6RnUlOSeZXnG9GNByAANaDuCilheVCR5ELVkDiK3/NdEcON45hsim0nu6PoiPj2ft2rUO33M08tKGDRscDr3asWNHZs+eXWH9iBEjKt2/I/v27aNr164A7Nixo9Jy9Wm8cCF8gSfTl8CzfSDA6BTevz9s3AgJCXDFFZ45rjg3SQDhwyzaQlJ2EkfPHC2zXNj8Qu676D4A9qXt44pFFT8lgv2D6RHTg4z8jNIA4qMJH5XpvyBcyGLBjB+N9hjzX3W5v+pJwoTvWLlyJRaL8wO11zRtaPXq1WVaILKzswkPD6+0fPkWDCFE3WVnw7p1xh16W18Bt/NwChPAZZcZAcTy5RJACPeSK5UXaK3JyM8gKSeJpOwkTmafJCkniYcHPFw6DOqNn9/I53s/p9hSXGH78XnjSwOILs26MKLdCLo060KXqC6lj60jWlfotyDBgxuZzWzmBsItJZzwC+W6e6TtuL4IDHTvPB32k86BESBIkCCEZ/34ozF868CB4OSULXXm6RQmgHHj4NlnjTkh3n4bGUZcuI1cxVzgZPZJTuefJrMwk4z8DNLz0knLSyMtL42BrQYyoesEANYdW8e1S64lPS8ds67YMXLS+ZPo0NQYwlGhKLYUExMWQ9vItsYSYTz2jO1Zuk2TkCasumWVZ35QUTmLhYNM5Dwgs38sgYHyqX0uS0hIcPsx/vWvfxEbKy1dQjjD0+lLYJfC5MHLQZ8+EBtrdBbfvRt69qx+GyFqo84BhFJqGDDTuq8i4AGt9cZyZRQwA7gOo1FvG3C31jq3rsevCa01f2b+SV5xXpkltziXvOI8+rfsT+dmnQHYcHwDS/YuIbswm4MnDhJ8IpjMwkyyCrMoLCnk0ENnZ4gdNX8Ue1P3Ojzmff3uKw0ggv2DSclNAaBxUGNahLegeaPmNA9vTotGLcpMwjZr7Czeu/K9c2s253osMz+OWJoCmouejPF2dcQ5wDaXhBCiap4evrWU7T6hX5WlXMpkMlohPvzQaIWQAEK4S50CCKVUJPAFcIXWeoNSKh74SinVTmudZ1f0VoyZSHtrrfOVUh8BLwFVTvl6PPM4935zL0XmIgrNhRSZi0qXmWNm0iWqCwDPrX2ORbsXVShTWFJIj5gebLprEwAaTds32lZ6vHfHvVsaQOxJ3cPMjTPPvplatmyRuYhAPyP1oWPTjmitaRzUmKYhTWkW2oyokCiiQqPo3/Ls7LK94npx8rGTNAttVrptZaJCo6p8X/gOS5GFLYkPEIhmY6No/nrlOTobtxBC+KB9++DPPyE6Gvr29dxxvZHCBGcDiG+/leFchfvUtQViDLBfa70BQGudoJRKAkYAX9uVux6Yo7XOt75+A1hNNQFEoyONGHHjCIfvJYUmke6XDkC/wn50K+pGQvcE3h77NgCtU1vz6n9fJb1FOtxlbGNSJj574zP8zH4opVAo49H6PHR2KOv91gPQydKJ70u+p6RdCRuf2MigCwfROKgx5iFmVInC9KSp9K7C8588T/aW7Ep/jvWsL/P6D/5g4NGBmAKNT5Vd43eRvSWbHl/1oPFFjQE4Mv0Ipz46VdXpqcDR9u2eb0fz243JqVI+TeHQo4eq2kUFjraPvi4arjLez92Xy84RlQ8z6Uj0ddF0er1Tme1Du4bS+8fepWU2tNmALnZu9mKg0u0dneeacOb3dHLOSfxKmnGSYExToiXnVAghfIinh2+1MYWYCO8fTnZEza47dTVqFAQEwIYNkJ7uuT4f4txS1wCiPXC43LrD1vVVlTsMNFVKRWitM+0LKqWmAlMBOtOZqJxK7sTnQBFFAIRY/40NH8slAy7B3+RP0NEgwnLCaFbcrEw+crPsZlD5EOyl+wQIJJDAxoH0D+1PSGIIxRTDKaAEfv7pZwiwFjwCJFW+T0d+WvtThe23/boNbEld+2q+T0fb79+xn/0J+41122u+T0fbJ/6eSE5OjnFea/GzJ/6eSGJCovHCun1RcFHZvPGTVPl7Kq+y7R2d55pw6veUDln48wLdeKHLWhISGtfsIA1IREQE2dlnL5Zms7nMa+HZc1JQUOCR/hhC+LKvrbczx43z7HHDuobR99e+Hv8/GB4Ow4bBqlXw3XcwebJHDy/OEXUNIBRns/xsSqjYYFe+nO2rYYV7AVrrucBcgL69+upBK5zP8zWFmAhoYnzbsxRbKL6iGOWvCIw5my5UeKzQ6f0BKH/F+r3rS8dct20f2DywdIz4ojVFNbpbXtn2AVEBpXe7Sy4swfyW4xloK+Noe7/Gfvg3Mn7N5v5mSh6qwbdycLi9KcTELzt/IT4+HssQC8WXVRwpqiplfk/W7V3xe3K0vSd+TwuGmLlnmj+D+ImrxveG1q1rdIyGZN++fWWGKK1uyNJzkSfPSXBwMBdeeKFHjiWEL0pPN4Zv9ff3cP8HL7viCiOA+PZbCZi30dUAACAASURBVCCEe9Q1gDgBjCy3rjXwmYNyrcuVyQHOVLVzFaAIalGzMddtTAEmh9vWan92/aMdbR8YXbdhIB1t7x/hj39E7X89jrb3C/XDL7T2vbkcbV/ZeXaWS39P1Wzvrt/Tx1/7UwjczHwwzajTMYQQQrjO8uXGPJ/Dh0NEhGePrS3aGInJ+WlmXGbcOHjkESN9q6TECKCEcKW6ZgN+BVyglOoJoJTqD3QFflRK/aKU6mQtNx+4Uyll+wb2IPCFtp9dSYh66OhRWLsWgsnnWj4DPw8OtyGEEKJKX31lPE6Y4PljZ67L5KfAn+BRzx+7Y0fo0gXOnDH6QgjhanUKIKz9FyYBHyqlNgGvY4y2FAq0AWzx/jxgHbBJKbUFaIwRRAhRry1caDxeFfQdjcn2bA894RIXXXQRcXFxxMXF0bRpU5RSpa/j4uJ4+OGH67T/o0ePopTi999/d1GNhRDOKCgw+gAAXHml54+vtTYGW/HSfSVbn49vvvHO8UXDVudGLa31GuAiB2+1sitjBqZbFyEaBK1h/nzj+c2Bn0IhEkDUQ5s3by59vmrVKkaNGsWpUzUbAU0I4XsSEiA3F3r3hjZtPH/8JvFNiC+J99pABldcAa+9ZnQif+klr1RBNGDybUeIWtqyBfbvh5gYGO232lgpAYQQQvgEW/rS+PHerYe3DB0KTZoY82BIA6hwNfm2I0Qt2VofbrwR/LV1JCrpA9Hg5Ofn89RTT9GtWzfi4uJo1aoV06efbUz9+OOPiYuLY8mSJfTs2ZPo6GgGDRrE3r1lZ6ffuXMnF198Me3ataNz584sXbrU0z+KEOcMrWHZMuO5N/o/+IKAgLPBk3zcCFeTAEKIWiguhkWLjOe33AKYrUPuSgtEBUp5Z3GVzMxMzGYzq1at4v/bu+/wqKr0gePfk14IIaQBIZQgAtKkKqIYBEGxrKIgKFIUUVd0LeDqyq6o609xFV07IoioqKAINtBFiBSpQQSkSREChE4S0icz5/fHmZl00jMzyft5nvvMnTu3nLm5mXvfU48dO8a6det48803WVugZeKpU6eYO3cuCQkJnDhxgrZt2/Lgg4WbeT377LO8++67HDhwgL///e+MHTuW9PT06kuoEMIpMRGOHoWYGHBVT8YpP6ewsctGeMM1xwcYOtS8LlzoujSIukmedoSohKVL4dQpuOgi+83JZu+nTwKIOqdJkyZMmzaN1NRUFi5cyIIFCwgICGDv3sKjus+dO5fw8HCUUtx6662F2lYATJ06lY4dOwIwYsQI0tLS2LNnT619DyHqE0fpw403Vm+GQkXkpeaRsS3DDEDrIldfDcHBpsrtoUOuS4eoe+RpR4hKcDaevtN+c3IEEFKFqRitXTNVly1bttChQwf+/ve/s2PHDsLCwoiKisJqzR/oMSYmhrCwMOf7oKCgYqNNd+rUyTkfHBwMQFpaWvUlVAjh5A7Vl7TV/kPkwietwMD8AfSkGpOoThJACFFBKSnm5qRUgRE+pQpTnfXwww8zZMgQvvnmG6ZMmcLYsWOxWAqPvq7KkcVZnnWEEFW3dy/89huEhEB8vOvS4Q4BBEg1JlEz5GlHiAqaPx9ycsyNKTbWvlCqMNVZOTk5nD59GqvVSnZ2Nk888USx6ktCCPexYIF5vekm8Pd3YUIchZQuvi1cdx34+cGqVXD8uGvTIuoOedoRooLmzDGvY8cWWCgBRJ312muvsW3bNiIjI7nooosIDw+nR48erk6WEKIU8+eb1+HDXZsObXOPEoiGDWHgwMI9UwlRVVUeSE6I+mTPHli7Fho0gFtusS8sWOleAgiPNnDgQDN6bAGXXHIJmzdvLrRs8uTJzvmxY8cytlA0WXg/rVq1KrZPoMRldYVS6krgVcw9JheYqLVed571nwamAG211n/WSiJFnbRnD2zZAqGhpgGxK7lLFSYw1Zi+/960g7jnHlenRtQFEkAIUQFz55rXW281PVsAztIHrZTUcxf1nlKqEbAQuF5rvVYpFQ8sVkq11lpnlrD+DUBT4EjtplTURW5TfQncpgoTmN6ovLxg2TJITTUBlhBV4QaXtRCewWbLDyDGjCnyAbiur0Ah3MtgYLfWei2A1joBSAYGFF1RKdUOeBj4W20mUNRd7lJ9CdyrBCIyEvr1M2MYOUboFqIqpARCiHJasQKSkqBVK/ND7OQogfD2RkIIIYgD9hVZts++3Ekp1RCYBdyhtc45X+mdUmoCMAEgOjqahISE6kxvpaWnp7tNWtyFK8/JoUNBbN3amwYNLPj6/kJCgourCe40LxarxS2uk27dmpKQ0I433zxDixZbXZoW+d8pztPOiQQQQpSTo/H06NFFmjrYu3DVUgIhBIAiv/KGQx4F8mGViRY+BJ7TWh8sa4da6/eA9wB69uyp413ZN2cBCQkJuEta3IUrz8lzz5nXYcN8ufrqK12ShoKO7j7KHvbg6+/rFtdJ587wxhuQmNiYDh3iiY52XVrkf6c4TzsnblCwJoT7S0uDL78086NHF/lQemASoqDDQIsiy1rYlzuEABcDzyil1iml1mHaQXyllBpbK6kUdY47VV8C96rCBBAebgaVs9ng889dnRrh6dzkshbCvX3xBWRlwRVXQJs2RT50VGGSAEIIgMVAF6VUZwClVG+gPbBcKbVGKdVWa52mtW6ttb7UMWHaSdystZ7juqQLT7VjB2zfDmFhMKBYaxsX8rZPbsIx+Oknn7g2HcLzyROPEOXw4YfmtUhvnYZjFGqpwiQEWutUYBgwWym1AXgNGAIEAS0B6f9FVLtPPzWvN98Mvr6uTYtDzF9jiM+Lh4dcnZJ8N9xguiHfsMGM2C1EZUkAIUQZ9u+HlSshMNB031qMlEAIUYjWeoXWupfWurfW+jKt9VqtdZLWurnWelMp27SSMSBEZdhs+Zk8d97p2rS4u6AgE2QBzJvn2rQIzyZPPEKUwdF16y23mBE9i5FuXIUQwmWWLzc95LVuXaSHPFGigtWY6vB4lqKGSQAhxHmUOvZD0ZUw3bgKIYSoXY4e8saMca++LJJnJ7Ox60b4wtUpKWzAAIiKMqN2b97s6tQIT+VG/2pCuJ9Vq+DAAYiNhf79S1lJunH1ePHx8TRo0IAmTZoQExND+/btefbZZ8nJySlz2zlz5tCkSZNSP1dKsXTp0mLLR40axdgSG9UIIcorNRUWLjTzxXrIc7HcY7lkbM2As65OSWE+PjBihJmXxtSisiSAEOI83n/fvI4eDaUWMEg3rnXCxIkTOXbsGEeOHGHp0qUsWrSIRx55xNXJEkKcx4IFpoe8+HhThcmdNB3flJ5besLNrk5JcY5qTB99BOXIJxGiGHniEaIUZ86YmxPA3XefZ0VpRF3ntGrViscff5z5jo7lhRBuyVF9yR0L8/yi/GjQtQFEuDolxfXqBV26wKlTsHixq1MjPJE88QhRio8/NjkzgwaVkbMl3bjWSZmZmfj4+ADw888/069fP1q2bEmLFi247bbbOHz4cBl7EELUpD17YM0a0y1piT3kiVIpBRMmmPkZM1ybFuGZfFydACHckdbw3ntm3vEjWyopgShTgkqo0PoNujegZ2LPYtvH63jnsk09NpG+Ob3E7QuuV1E2m43Nmzfz4osvMm7cONatW8dTTz3F3LlziYuLw2KxMGXKFG6//XZWrlxZ6eMIIarG0XXrsGEQHOzatJTk5JcnOf3daWgLxLs6NcWNGgWTJ5terP74A9q2dXWKhCeRAEKIEqxbB7//bnqquOGGMlaWAKJOePPNN5kzZw7nzp3Dx8eHF198kfvuu49rr72Wbdu2cdlllznXtVqtWCwWrI7Sp0pSUmolRKVYLGUM8OkGzm06x7EPjsH5qsC6UGioaUz9wQcwcya89JKrUyQ8iQQQQpTAUfowbhz4+ZWxsjSiLlNVSgRK275gCUV1mDhxIi+++CJ//PEHnTp1omvXriil2L9/P0899RSPP/54pfbbsGFDMjIyii1PS0sjLi6uqskWol5atAiOHIH27eGKK1ydmpJpq32QBTfu4XvCBBNAfPABPPcc+Pu7OkXCU8gTjxBFpKTA55+b+fHjy7GBdONap7Rt25bx48dz//33Y7VaiYmJYePGjVXa3++//15omdaa33//nZYtW1Y1uULUS2+8YV4nTnTf5mfaZg8g3PhJ65JL8htTL1rk6tQIT+LGl7UQrjFvnukW8Kqr4IILyrGBjERd5zz99NMcOHCA119/nSeffJKFCxcyc+ZM8vLysNlsrF+/njmO7l/K8PDDD/P666+zbNkybDYb6enp/OMf/yAtLY2RI0fW7BcRog7assWM0RMS4n5jPxTiqOHoxreGgo2pHSXvQpSHBBBCFKB1fo8UZTaedpCRqOucqKgoJk2axNNPP02nTp1YtGgRs2bNomnTpsTExDBp0iRiYmKc6588eZImTZoUmoYPHw6YAeNeffVVJk+eTEREBB07dmTXrl2sXr36vAPQCSFK5ih9uOsuE0S4K2cVJjd/0ho1CgIDTWPq3btdnRrhKaQNhBAFrFoFW7eaxtM33VTOjaQbV4+XkJBQbNm//vUv/vWvfwHQrFkzbiilNf3YsWPLHFH6zjvv5M477wTg3LlzhLjzU48QbuzUKVNKDPDAA65NS1k8oQ0EmMbUo0aZhtTTp0u3rqJ83DwuFqJ2OXK27r23Ao3JpBcmIYSoFe+/D9nZcO21HtDtqL12qyc8aT36qHn98EM4fty1aRGewQMuayFqR1ISfPUV+PjAffdVYEMJIIQQosbl5cHbb5v5hx5ybVrKw1kC4QGF0+3bw403msFT33zT1akRnkCeeISwe+cdUxvpllugWbMKbChVmIQQosYtWmQyetq2hUGDXJ2asnlKFSYHR0/Vb70F6SWP0SmEkwQQQmCKxGfONPMVztmSEgghhKhRWsMLL5j5hx/2jGF3PKURtUPfvtCnD5w9C7Nnuzo1wt1JI2ohgM8+M43zunc3P6AVIgPJCSFEjfrxR9i82XRwMW6cq1OTz6ZtJJ9L5kDKAY6nH+d01mlOZ54mLSeNEZYRZiUFn23/jD2n9xDsG0xYYBjNQpoRExJDTMMYwgLC3GZU+smTYehQePVV+OtfTZVeIUoil4ao97SG11838w8+WImaSFICUYjW2m1uhvWZ1trVSRCi2jhKHx591HQ56gpHzx3FarMSGxoLwNK9S7nps5vIseaUuP7Q3KFmxhvm/z6fr3Z9VeJ6A+MG8r87/weY/9vE5ES6RHfBz9uv+r9EGW680VQR++MP+OILGDGi1pMgPIQEEKLeW7MGfv0VIiIq+WMpbSCcfH19ycrKIigoyNVJqfcsFgs+kn0o6oCffzZTaCjcf3/tHTcpNYll+5ex8tBKVh5cyf6z+5nYayJvDDHd9UUHR5NjzSEqOIrWjVrTNKQp4YHhhAeGExoQSouRLQh6KojEQ4mMaDOCjpEdybBkcDrrNEfSjnD03FGS0pJoFpLf6O5g6kF6zeyFn7cf3Zt2p3+r/gyMG8hlsZcR4BNQ49/Z2xsmTTI9ET7/PAwfLoXromRydxH13ksvmdd774WAyvw+SwmEU1RUFEeOHCEmJoZAV2UTCmw2G8ePHyc0NNTVSRGiSrQG+3AsPPIINGxY88d8bd1rzPp1FttPbC+0PMQvBC+V/zvfOboz5548RwO/BuffYQoM7zic4R2HF/tIa01WXpbz/YmME7SPaM+uU7tYd3gd6w6v44XVLxDgE0B8q3jeue4dWjVqVaXvV5YxY+C552D7dlMKMbx4soWQAELUb9u3wzffmMCh0t0CSgDh1NB+dz969CgWi4Xs7GwCKhWV1V21dU6Cg4OJiIio8eMIUZOWL4eVKyEszDSerm5aa9YdXkf7iPaEBYYB8GfKn2w/sZ0Gfg24qvVV9G/Vn34t+9Elugs+XvmPTT5ePmUHD2VQShHkm19i2zumNzsf2ElKdgrrDq/jp/0/sezAMrYc20LCnwlEBkU61/1h7w90iOxAi9AWVUpDUf7+MGWK6c586lTTM6G3h/QkJWqPBBCiXvvPf8zrXXeZxnmVIlWYCmnYsKEzkEhISKBbt24uTpF7kXMiRPlobR5kwTTurc4CtR0ndzBnyxw+//1zDqUeYsb1M5jQYwIA9/a4lxvb3cjlLS6vUjuEw68fJn1LOlwKxFds20YBjbjmgmu45oJrAFMyseXYFoL9ggHIteYy4ssRpGSncFXrqxjbdSxDOwx1fl5V48bBiy/Czp3wyScwenS17FbUIVXKMlVKXamU2qyU2qqU2qSUurSU9Xoopc4opdYVmB6ryrGFqKpDh2DevPw6n5UmJRBCCFHtvvwS1q0zmTsTJ1Z9fxm5GXzw6wf0nd2Xjm935D+//IdDqYeICYkp1OlAh8gOXNX6qio3Yj677CzHPjgGJ6qacogKjmJQm/zBL85mnWVA6wEE+ASw/MByRi8aTZNXmnD34rtZdXBVlTtR8PMzpQ9ggrisrPOuLuqhSj/xKKUaAQuBB7TWXYBJwGKlVEmtJyOAL7XWlxaYXqnssYWoDtOnm5FNhw+H1q2rsCPpxlWIQsqTuaSUilZKzVBK7VRKbVBKrVJKdXZFeoX7yc2FJ54w8888AyEhVd/nzZ/fzF1f38UvSb/QwK8BE7pPYNW4VRx65BD39ry36gcoIuahGNq93w4uqPZdE90gmi+Gf0HyY8nMuH4GfZr3IT03ndlbZtNvTj8SkxOrfIxRo6BLFzN4n6OnQiEcqlKFaTCwW2u9FkBrnaCUSgYGAN8UWTcCGKSUWm9/vwx4UWt9ruhOlVITgAkA0dHRJCQkVCGJ1SM9Pd0t0uFOPP2cpKb6MGNGH8Cbq67aSEJCRqX3Fb5lC50Bi83m0eekJnj6dVIT6vo5KZC5dL3Weq1SKh6TudRaa51ZYNXuwA9a63vt2z0GvAJ4wBjDoqa98w7s2wft2sHdd1d8e601P+77kbbhbYkLiwPg9s63k2HJYHy38QzrOKzK7RfK0nhgYwB2J+yusWM0CmjEhB4TmNBjArtP7WbOljlsPbGVHk17ONd5e+PbDIwbyIXhF1Zo397epprv4MHwf/9nqvpGRpa9nagfygwglFJ+wMoSPloC7CuybB8QV8K6C4F5WmutlGoMvAnMBW4uuqLW+j3gPYCePXvq+Pj4spJY4xISEnCHdLgTTz8nzzxjRp++5hoYP75X1XZ29iwAPr6+Hn1OaoKnXyc1oR6ck3JlLmmtlxTZLhlplyeA48fze1566SXw9S3/tjl5OXy6/VOmr53OthPbuLfHvbx7/bsAjOk6hrEXj63+BLuJdhHteGHgC4WW7Tq1iwe+fwCAQW0GMbHXRK678LpCvUmdz6BB5j65dCk8+SS8/361J1t4qDJ/rLXWuZgmQIUopf4BWIsszqOEalFa66wC82fsOU1JSqnAgp8JURtSUuC118z83/9eDTuUNhBCFBRH+TOXAFOdCXgWKDGv2R1LpqHulyZVRnWck2nT2pGW1pRLLjlNSMg2yrO7NEsaXx/9mq+OfsWZ3DMAhPuF453i7bq/0SrgFGR0y3BZGpKzkhnSZAjLTizjx30/8uO+H4kNjGVY82EMbjIYP6+y23mMHBnIsmW9mDXLix49EunQoVjlkQqT/53iPO2cVCW35zAwsMiyFsAX5djWG8gGSh6+UYga9MorJoiIj4crr6yGHUoAIURBinJmLgEopcKB74GpWuufS1rHHUumoV6UJlVYVc/JmjUmt9vPDz76KJy2bcve1ydbP2HCtxPItJgacp2jOvNYn8cY2XmkS0ZzdtjyzBZSElIInh5M/Nh4l6VjJCM5k3WG2b/O5o0Nb3Ao9RDT/5jOguML2PvQ3nKdox07YNo0mDWrB+vWQVXHqJT/neI87ZxU5YlnMdDF0ehNKdUbaA/8TykVrpRao5Rqa/9shL1eLEopH+AF4COtta1qyReiYk6ezC99eP75aup5VbpxFaKgw5jMpIJa2JcXopRqCvwEvKK1/rgW0ibcWHY2jB9v5h9/HNq2LX3d1OxU53zn6M5kWjIZ3GYwP476kd/u+40xF49xafAAoK32npDc4NbQOLAxky6bxL6H9jFv6Dy6NenGoDaDnOfIarNy4OyBUrefMgViYyExEf7739pKtXBnlQ4gtNapwDBgtlJqA/AaMERrnQIEAS0BR6/NgcBPSqmNwC/AMeDRqiRciMqYNg3S02HIELjssmraqZRACFFQaZlLy4tkLLXEBA/Paa3nuSy1wm08/zzs2mUaTj/1VPHP82x5LPh9AX1m9WHIvCHO5V2iu7D/of0sHbWUq9tcjXKTzBxnAOFGg7D5ePkwsvNIEick8vq1+V0rLdy5kAveuIDhC4az4ciGYts1aADvmqYk/POfpoG7qN+qVAiltV4BFGuBqrVOApoXeP8B8EFVjiVEVR09Cm+9Zeafe64adyzduArhpLVOVUo5Mpc0pvrSEIpnLL0CRAOTlVKT7ctytNbVUbFQeJhNm8zAZWAa6hYcrP1czjlm/zqb/67/LwdSTC55WEAYR88dpVlIMwBah1WlL+6a4Qwg3PDWUHQE7L1n9uKtvFmwYwELdiygX8t+TOozqVCD6yFD4PbbzfhJ48bBihUyQnV9Jj1eiHrj+edNEfktt0D37tW4Y3sVJu0muV5CuFppmUsUzli6tfZSJNxZZqYZcyAvD/72N7j8crP8bNZZXlz9IjMSZ5CaY6ostQlrwyOXPsLYi8dW26jLNcZRSdsNA4iinrziSUZ3Hc3r61/n3cR3WXlwJSsPrqRdeDue7f8swzsOB0z1peXLYdUq08WrY6wOUf94wGUtRNXt3w8zZ5pmCs8+W807lxIIIYSotMceg927oWPH/FIIAC/lxTub3iE1J5XLW1zOV7d9xe6Ju3mg9wPuHzzg3iUQJYlpGMO0q6eR9EgSrwx6hdiGsew+vZvkc8nOdSIi4AN7fZJ//cuUHIn6yUMuayGq5rHHwGKBO++Eiy6q5p1LGwghhKiUTz81dev9/DT3Pb+aUV/fSpbF9O4eGhDK29e9zbq717Fq3Cpuan8T3l6eU2fG0wIIh4b+DXm0z6POBtd3dbvL+dkLq15gcd79jBp/FosFhg93DoUk6hmpwiTqvGXLYNEiCA6GF14oe/0Kc1RhkgBCCCHKbedOuOceDSga3fQMD255BoDBbQZzT497ABjVZZQLU1hFjs6MPfTW4Ovty8jOI53vc625TF83nVOZpyB6DqGtt3PgQBvGjtV89ZWSQvh6Rv7cok6zWEydWjDd0DVrVgMHcVRhkjYQQghRLtv+PMJlA0+RkaGg0zxOdHiG5g2bM23gNIZ1HObq5FULbfPMEojS+Hn7sXLsSsZ3G49fgI3UG66GgLN8/bVi6D27sdqKDv8i6rI6clkLUbJ33jED4LRpA488UkMHkSpMQghRbhYL9L32MClHI6DJr/S+/30+H/Y5+x/az+N9H6dRQCNXJ7FaeGoVpvPpENmBmTfO5NDDh/jnTaMIuWM8eFlYPLsdT0wrfRwJUffUoctaiMJOnoSnnzbzr74K/v41dCBpRC2EEKXKzsvm460fs//sfmw2uPtuOLfrEgIapbBoEaz/63KGdxyOr7evq5NavTy8CtP5RDeI5tn+z5L89lzueHI1AK/+sw1Ll5rPp6+dzsYjG12YQlHTpA2EqLOefBJSUmDwYLj++ho8kHTjKoQQxew+tZsZiTP48LcPOZN1hom9HiRgxet89JFpk7bix0b06tXN1cmsMXWtClNJgv2C+fjf/WkFPP+8Ytgw+PjrJCatnIRG0zumNw/0eoDhHYcT4BNQ1u6EB6nDl7Woz378EWbNAj8/eO21Gm6eICUQQggBQK4tl8+2f0b/D/vT/q32vLruVc5knaFbk+4c+uoeXn4ZfHzgyy+hV0kjhdQhcS/G0e79dlA3amSd13PPmUHm0tNh7NAYbm/8KmEBYWw4soExi8bQ7JVmPPj9g2w5tsXVSRXVRJ54RJ2Tlgb3mA48eOYZaN++hg8obSCEEAKAd/e9y8gvR5LwZwJBvkGM7zaeDeM3MvjPRL5+vzPe3mYk48GDXZ3Smhc1LIqmdzcF9x+yosqUMuND3HQTpKR48c2Uv/FFn6PMunEW3Zt252z2Wd7c+CaXvn8pKdkprk6uqAZShUnUOY8/DocOQc+eMGlSLRxQunEVQtRDyeeSmbdtHu0j2nPdhdcBcHX01ezN28t9Pe/jjs53EOgVyj33wNy54O0Nn3wCw+pGJ0uiCD8/mD/fjCo+fz7cOCSAb7+9i8QJd7Hl2BZmbZ5Fni3P2UjeYrXw4JIHubn9zQyIG4CPlzySehL5a4k6ZdkymDHD/JB98IEpKq9x0o2rEKKeSM9N55vd3zB361x+3PcjNm1jQOsBzgCifUh7frvvN5RSpKXBdbeY3+WgIFiwAIYMcfEXqEXJs5OxZdsgztUpqT2+viZI9PeHjz6Ca64x9+QxYy7mjSFvFFp3yd4lzEicwYzEGUQGRTLsomGM7DySy2Ivw0tJhpy7kwBC1BmpqTB+vJl/+mno1KmWDixVmIQQddzyA8t5c8ObLNm7hOy8bAB8vXz5S7u/MKbrGOd6SimUUuzdC7feCr/9BlFR8N13plS4Ptn/xH4sJy2w0NUpqV0+PjBnDoSGwptvwtixsHkzvPyyCTAcujftztQrp/Lp9k/ZfXo3b296m7c3vU1sw1hGdBrBc/2fw9+nprpPFFUlTzyiTtAaxo2DgwehRw9TjanW2KswSSNqIURdcTbrLMfTjzvf7zm9h692fUV2XjZ9mvfhrSFvkfxYMgtvW8hf2v+l0Laffgrdu5vgoW1bWLu2/gUPAE3GNaHZ/c2gHj4De3nBG2/Ae++ZoOH11027l+P5lxTNGzbn6fin2fnATjZP2MzkyyYT2zCWpLQkFu9ejJ+3n3Pd9YfXk2fLc8E3EaWR3vRQeAAAGuZJREFUEghRJ7z6Knz1lcnx+PzzWqq65OAogZAqTEIID6W1ZsfJHXz/x/cs2buEVYdWMbHXRF695lUAhnYYisVqYWiHocQ0jClxH5mZ8PLLF/Ldd+b9rbfCzJnQqB70QlSSNtPaAHA04aiLU+I699wDF10Et9wCK1ZAx47w9tumVMpBKUW3pt3o1rQbLw58kV+SfiE1OxVlv6f+mfInl866lLCAMK678Dr+0u4vDG4zmBD/EBd9KwESQIg6YPXq/BKHOXPMqNO1yhFAeHvX8oGFEKJqVhxYwee/f873f3xPUlqSc7mX8uJM9hnn+6jgKB685MFS9/Ptt/Dgg/Dnn83w9zfdZ997rzQNE9C3LyQmmqpMy5bBbbfBlVdexIIFEBlZeF0v5cXlLS4vtOxI2hHahbdj9+ndfLz1Yz7e+jF+3n70je3L1XFX80DvB2jo37D2vpAApAqT8HAnTpgfI6sVJk82XcjVOmlELYTwAJmWTH7a/xPH0o85ly3evZgZiTNISksiKjiKMV3H8Nktn3Fq8ik+vOnDMvd56BDcfDPccAP8+Se0aZPOhg1w333yk5i2Po209Wn5I1LXYzExZnymd981gwj+/HMU7dqZQDM39/zb9m3Rl10Td7HrgV28NPAl+sb2xWK1sOLPFTzz8zP4euU3rPhh7w/sP7sfrXUNfyMhJRDCY2VlwdChcPQoXHEFPP+8ixIi3bgKIdxQanYqG45sYOXBlSQcTGD94fVYbBZmXD+DCT0mADCy00jCA8MZ0nYI3Zp2K3fvN8eOwbRp5oEwOxsaNDCDiXXunEiXLlfW5NfyGL9e+Ss6R8NSV6fEPShlSqUGDYJhw86QmNiYRx4xDa2nTTP38/MFne0i2jE5YjKT+07mdOZplh9YTlJaEoG+gQDk2fIY/sVw0nLSaBbSjMtbXE7f2L70je1L1yZdpZvYaiZnU3gkqxXuuAPWrIHmzeGzzwr37lCrpARCCOFiNm0r9PDfZ1Yf1h9ejyY/J9ZLedGjaQ+CffNHNruk+SVc0vySch/nyBGTa/zWWyYTB2D4cJg+3eQyJyRIzq+To+RB8pYKad0a/vOfrWRkxDN5MuzaZdrLdOpkahKMHFn2/Tw8KJxhHQsPKHI26yxXx13Nij9XcPTcUeb/Pp/5v88HINg3mE+GfuJs8J9rzcXXy9fZzkJUnAQQwuNoDQ89ZBpNN2oES5dCs2YuTJB04yqEqEUZuRlsO7GN3479xtbjW9lyfAs7T+7k2KRjzp5rgnyD8PX2pWt0V65ocQXxreK5ouUVzkG8KkJrWLnSBA0LF+Z3PPeXv8DUqXDxxdX45eoQbbMHU3JrKEYpuP560zPTzJmmBsH27TBmDEyZAg88AKNHQ9Om5d9nZHAkXwz/Apu2sfvUbtYkrWH1odWsSVrD3jN7aR3W2rnuE8ueYN62eXRv2p0eTXvQo1kPujftTmzDWAkqykkCCOFxpk0zvTj4+8PXX5teHVxKunEVQtSA9Nx0sixZRAablqa/JP3CmEVj2HdmX6GSBYcdJ3dwcRPzND/3prlEBEVUqR/9P/4wpbvz5plcYjCjSd96KzzxhOkyW5RMaw32wmkJIErn6wt//asZw2nePHjpJdi501xfTz1lBqIbO9YMQBgUVL59eikvOkR2oENkB8Z3N4NDHU8/TkRQhHOdXad2cTzjOEv2LmHJ3iXO5SF+IYzsNJIZN8wAwGqzkpyeTExIjAQWRUgAITzKf/4DTz5pci8+/ti0fXA56cZVCFEFB84eYOepnew5vYfdp3az+/Ru9pzew5FzRxjfbTwzb5wJQHhgOHvP7MXHy4eLIi+iS3QXukZ3pWt0V7o37U54ULhzn6V1tXo+Nhts2gRLlsA335iecxyio2HCBDM1b17lr1z3OYIHZZ/Eefn5mUBh9GhTq2DWLJNB+N13ZgoMNG0nbr4Zrr22cDew5RHdILrQ++9u/44DKQdIPJrI5uTNJCYnsuXYFk5mniwUnP9x5g86vNWBEL8Q2jRuQ5uwNlzQ+ALahLWhTeM2XBJzCcF+wUUPVy9IACE8gtbwzDNmUso03Lv1Vlenyk6qMAkhSmDTNk5lniL5XDIHUw9yMOWgeU09yPs3vE9oQCgA9393Pz/s+6HY9n7efuRYc5zvL2h8AVvu3UKHyA6FBtmqLKsVtm0zXWGvXg0//QSnTuV/HhJiHthGjICBA13YzswDSfWlyvHyMqUNQ4bAyZPwySdmYMING2DxYjOBaS9x1VUQHw+9e5tqzBXJw1NKERcWR1xYXKG2FCczTpJrze8W6nj6ccIDwzmddZotx7aw5diWQvvZfv92OkaZahDT105n58mdxIbG0rxhc2Ib2l9DY2ng16DS58RdSQAh3J7WZpyHl182Py5z5sCdd7o6VQVIFSYh6g2L1cKZrDOcyjzFbym/cXbnWY5nHCf5XDIXN7mYmzvcDMDqQ6vp/2H/UkfPnXLFFLo26QpA75jeWGwWLmx8Ie0i2tEuvB0Xhl9Iy0YtC/Uc4+3l7dymojIyTB3z334rPKWnF16vVSuTw3vttSZoCAys1OHqr5degl690H36AaC8VfHqZitWwMaN+QMYiRJFRsLDD5vpyJH8AGLVKnMtb99uRrgGaNLEjHbes6cZBb1DB3MtV3RQWUd1QYcrW13JycknOZ11mn1n9rHv7D72ntnrfI0Li3Ou+/0f3/PTgZ9K3O/QDkP5cviXgGnsPTVhKlHBUUQFRxHdIJqo4CiOZh0lPTedYN9gj6guJQGEcGvZ2aY/8Q8/ND8E8+bBsGFlb1erpAqTEB4jz5bHuZxz+Hr7OnMFD6Ue4pekX0jLSXNOqdmpnM46zems03x3+3fOHo76zOpDYnKBuj2/5c+O7jraGUCEB4aTZ8sjLCCMpiFNaRHagpahLc3UqCXNG+bXA3q2/7NV/l5ZWaZr1eRkSEqCffvyp/374fBhkxlTVFycGejr8suhXz9o1046lKuSXr1Mt1QfzgdU8QBixQrz+fz5LkuiJ4qJMW0l/vpXyMmB9eth+XJTcpaYaK79b781k4OvL1xwAVx4obmuW7aE2FhTBS82FsLDy3etK6WICIogIijivD2WTek3hVs63EJSWhKH0w5zOO2wcz48ML964eG0w7y+4fWSd7IBVoxZQXyreADe2vAWP+z7gbDAMMICwmgU0IiwgDDCAsNo3rA5V7W+yrnp8fTjBPsFE+QbVO7umKtCAgjhto4cMf1Cb9hgcsHmzze9NrgdRzeuMhK1EAAopa4EXsXcY3KBiVrrdUXWUcCzwHBMh5ebgXu11hnn23emJZP3Et8j05LpnLIsWWRaMmno35BpV09zrjtg7gBOZpwk05JJhiWDtJw0Mi2ZALx89cs8dtljAKw5tIbbF95e6jFTslNoHNgYMDmUEUERhAeG42PxIa5JHNHB0TQNaUrvmN7ObS4Mv5Csp7II8Ako51kzhZlZWWZKTYWUlMJTwWWnT5tgwRE0pKaef98+PiZXtmvXwlNF65KLMvTvD/Pno28dDXwE2krLuR+Zop6AANNH6fz5Zj1RKf7+JtjtZwp50NoEyZs2menXX2HPHhNI79xpppIEBJi2PRERprQjIiJ/ioyE0FBTja+kKSCgePAR3yre+eBfkNa6UFXEyOBIpg+azomME5zIOMHxjOOcyDjBodOHSLWmEhWc/0+5KXkT3+z5psT092jag00TNgGmsXeTV5oAoFAE+wXTwK+Bc5p65VRnF7YrDqzgs+2fEeQbRKBvIIE+gc7XIN9ytlRHAgjhptatM8FDcrLJNVi0yI27CpQSCCGclFKNgIXA9VrrtUqpeGCxUqq11jqzwKpjgCHAxVrrLKXUB8A0YOL59n/yeDbvTv4VZ+tU7Xj1JcTfj0Wrz6K1eajI+eUyfHKzOBl6hjS/XEDRND2UljkR7DzRiVk7zXpJh3px2+a5+HkF4Ofth6+XH37e/vh7BxLgHcCCR614e53FZoMbrZ9wnVWR2TCA7aePExPTCq+0XPyOZbDW15dvG4HFAjoHwpOzyLNmYbVCXh7kWSAn1+Sg5uZAtv3Vsex0ni/7MaUivljpRBoWvNhOqPP7dyYFnwI52oFAHODjDY3DoXFj8wDUrBk0awrNWnkRd2MosbEmRzZlVQraogm9PBQvP5NLmf5bOpbTlgr9nUvaPrhLfmPSrP1ZZP+ZXaF9BncJxi/Cr9D2Aa0DCGxt6lHlnswlY9t548tiStreN8KXBl3MebZmW0n7Ja1C+yxpey9/L0L72v9O/fuTOmkO/ANUdiat5swxFfmtVvj+ewkeqplS0KaNmW67LX95RobpSWzPnvyAwjEdPmwC8YMHzVRR3t5m8MSAABPQ+PsXni/8XuHvH4C/vwnkvb2b4O39CN7eEOENTXzM/pKSDhAX14pFM+Fb+7KozP/jnryHybZmkGXNINuWQWZeOpmWDCIzI/jwQ/P9c615NNx1P9nWbHKt2aQrTToalAY0P51rjGW7Wfe7P9L5YMsJKPC549XHu/xhgQQQwq1YLPDCC2ZE07w800Bq/nyTG+C2pA2EEAUNBnZrrdcCaK0TlFLJwACgYFbabcAMrbV9ODL+C/xEGQGEPh7M9OO3lb7Cj/l1iv7NAACeoDPrMVUIruEAoznI7JWtGG9fry+h/JvYEnaWY5+KZ+/PphUf0cq+fSr/5ndWE84MOgMQiYX5Bes3lcNqwvk/2hBIFq04zSscJRXNh5ylESk0IoVr6IQfJTSgtgIn7FMB/pwg7pH88/U7C7EQxmXcjB8pAOznBc5waYXSWtL2nXmCeNYDcIxxHGR0hfbZmScIL7J9K/uZBkilL7/z7wrts6Ttw1lNZ/4JgIVIfqNi1YlK2t6fE/Qh/zzvYiEQhiIPpbWpjwtw9dUVOlZdFV8LxwgGLrZPJTlHA04SySkinK+O6SSRpBLKOUJKnHKt/qSmll3yVzGtS1jW1D6VLP8H1R94u9T13vgC3nC+u8E+FWdabJUvM1QCCOE2duwwg8hsMiVyPPKIGfPB7Xv+kBIIIQqKA/YVWbbPvvx86+0DGiulQrXWhW7LSqkJwASAaNpyECuqQC58WfOXsJw25OKFjdYEkUwwF7CRcSxHoYnCn2M0LnU/Bd8rNArNpSzlAo7ii4VGhHCOtlzITt7hLXyx4IcPOfRzru+YvLHihc35auZteGHldrYyhQ8AyKURO/gnkaSykPw2Ett5hjzK36OLb5HgJ5Rt5NEARX7j7mD2YyspKDmPkrb3JT8nP4BjNGJzhfZZ0vYBHCvweWqF91nS9g3Y71zmhaXC+yxp+9LOcyQrK7RvUXtCSCeEdOI4UOFtc/ElnQbk4F9oyiag2LKCy614O6c8fAq9L+9yG17OXxSgyC9M1aclZXx3BwkghMtlZJjxHV580RTjt2hhelrymFJe6cZViIIUJj+8oDyKd2hZdD3HE2mxfySt9XvAewA9e/bUYzYNqFCC7qjQ2uWXkJBAfHx8Ne91NgB+FMw5fcY516lS+yxp+0edc20qtc+Stn/UeU7On29a9j7zt89f1ojSc5PLs8/C25d0niuijL/Tt9+aHj+yC1TjCgiABQvctDFf7aqZ/53a4wc0ruZ9uss5KW9eqDzxCJex2Uwfz+3amfEdcnLgrrtMv+QeEzyAVGESorDDQIsiy1rYl59vvRZAOtjrxQjhyQICzL0hIMCUThd4L0RdIE88otbZbKZRdO/eMGqU6W2pe3dYudKMPtmwoatTWEFShUmIghYDXZRSnQGUUr2B9sBypdQapVRb+3ofAeOVUo66Mw8CC7UuqbNRITzIihWmt6Xvv4cFC/hz3DhT8vD992b5ihWuTqEQVSZVmEStyc01DaJfeMG0dwAz+MsLL5jh6z02A1+6cRXCSWudqpQaBsxWSmlM1aQhQBDQEpxdCs0FLgA2KKXygB2U0YBaCLdXcJwHe1H6wQYNaO2omjJ/frHPhfBEEkCIGrdnD8ycaQaDO3nSLIuNhcmT4e67Iaj83Q67J3sVJimBEMLQWq8AepXwUfMC61iBp+yTEHXDxo3nDw7s40SwcaMEEMKjSQAhakRSEnz5pSm1/eWX/OWdO5vele64A/wq1umH+3KUQHhsEYoQQohq8fjjZa/Tv78ED8LjSQAhqkVenhkx+scfYckSM+8QFAQjRsCECabdQ53LqJc2EEIIIYSoRySAEJWSmurD0qWwfr2Z1qyBtAKDeQYGwnXXwa23wpAhZuj3Oku6cRVCCCFEPSIBhDivnBzThmHnzvwpMRH27r282LoXXgiDBpmBNgcMgOBgFyTYFaQbVyGEEELUIxJA1GN5eXDqFJw4AcePm3YLBw+a6dAh8/rnn/lV/Avy87PSq5c3l1xiqiX16WMGgKuXpAqTEEIIIeqRKgUQSilf4G/AC8CdWuvPSllPAc8CwzEjj24G7tVaZ1Tl+PVRXp4Z2DInx7wWnM/IMNWISptSU/MDhpMn4cyZso/n5QVt20KHDvlT585w5sxqBg68sua/sCeQblyFEEIIUY9UtQTiHkAD68pYbwymH/CLtdZZSqkPgGmU0ef30T3nmHrVShzDCmlUOeZBa1Vo3jEqUWXnU9M0n4cklrjv0uatNoVVe5FnU1htXli1Is/mdZ55+/rW/GUWqzfZed5k5/mQY/UhO88Hq636qsl4KRvhQVlEBWcQFZxJ84ZptGyUSstGqbRolEaL0FRah6Xg72PN3yjJTAcOHIB1q6otLR5t715ASiCEEEIIUT9UKYDQWr8NoJS6oYxVbwNmaK2z7O//C/xEGQFE8rkQnlnRrypJrHO8sBJAdqHJnxwCyCaITBqSdt4pglNEcpIoTtBYn8E7wwaVKAdqXf1fzeNZ602jDyGEEELUZ2UGEEopP2BlCR/101rnlvM4ccC+Au/3AY2VUqFa69Qix5sATABo7NuSUe2/QqGdXX8qU95gX5cC8zp/H2jne+V8X3x7imyvStneas3Dx8f7/McvcgwvZcPHy4aXsuGtbHgrq/3VhreXeTXrWPFW9vWVtcD6Nvy88/D3tuDnZcHf20w+XiU0SCi3QCAWiCUTyKzCnnJzc/GrMwM5VF1OZCRHmzcnLSHB1UlxK+np6STIOSlEzokQQghPV2YAYQ8SLq3icRSm7YNDnv21WH0crfV7wHsAPXv21P/ddHMVD111CQkJxDuGoReAnJOSHJVzUoxcJ8XJORFCCOHpaqvfycNAwT56WgDpQEotHV8IIYQQQghRDWokgFBKhSul1iil2toXfQSMt1eHAngQWKi11iXvQQghhBBCCOGOamociCCgJRBqfz8XuADYoJTKA3ZQRgNqIYQQQgghhPuplgBCax1f5H0S0LzAeyvwlH0SQgghhBBCeKjaagMhhBBCCCGEqAMkgBBCCCGEEEKUmwQQQgghhBBCiHKTAEIIIYQQQghRbhJACCGEEEIIIcpNAgghhBBCCCFEuUkAIYQQQgghhCg3CSCEEEIIIYQQ5SYBhBBCCCGEEKLcJIAQQgghhBBClJsEEEIIIaqVUupKpdRmpdRWpdQmpdSlpawXrZSaoZTaqZTaoJRapZTqXNvpFUIIUTESQAghhKg2SqlGwELgAa11F2ASsFgpFVTC6t2BH7TWHbTWvYFFwCu1l1ohhBCV4ePqBAghhKhTBgO7tdZrAbTWCUqpZGAA8E3BFbXWS4psm4zcl4QQwu259Q91YmLiKaXUQVenA4gATrk6EW5Gzklxck6Kk3NSnLuck5ZV2Vgp5QesLOGjJcC+Isv2AXFl7C8aeBa4u4TPJgAT7G/TlVK7K5zgmuEuf0t3IuekODknxck5Kc5dzkm57g1uHUBorSNdnQYApdQmrXVPV6fDncg5KU7OSXFyToqrK+dEa50LFGvboJT6B2AtsjiP81SZVUqFA98DU7XWP5dwrPeA96qU4BpQV/6W1UnOSXFyToqTc1Kcp50TaQMhhBCiOh0GWhRZ1sK+vBilVFPgJ+AVrfXHNZw2IYQQ1UACCCGEENVpMdDF0ZuSUqo30B74n1IqXCm1RinV1v5ZS0zw8JzWep7LUiyEEKJC3LoKkxtxu6JzNyDnpDg5J8XJOSmuTp8TrXWqUmoYMFsppTHVl4ZorVOUUrGY+rWh9tVfAaKByUqpyfZlOVrrK2s94ZVTp/+WlSTnpDg5J8XJOSnOo86J0lq7Og1CCCGEEEIIDyFVmIQQQgghhBDlJgGEEEIIIYQQotwkgBBCCCGEEEKUmwQQFaSU6qiUOqOUmurqtLiaUupepdRvSqlNSqmtSqm/ujpNrqCUulIptdl+DjYppYr1jV/fyLVROvkNqZvk75pP/v8NuTcUJ9dGyTzx90N6YaoApVQj4C3gU1enxdWUUt5AW6Cv1jpdKRUD7FVKLdZaH3Fx8mqN/ZpYCFyvtV6rlIoHFiulWmutM12bOteQa6N08htSN8nfNZ/8/xtybyhOro2Seervh5RAlJNSygv4EPgHcNLFyXE5rbVVaz1Ja51uX3QayAW8XZgsVxgM7NZarwXQWicAycAAVybKleTaKJn8htRN8nctTP7/neTeUIRcG8V58u+HlEAUoJTyA1aW8FE/YCrwP631L0qpQbWaMBc63znRWucWeP8a8LnW+lDtpMxtxAH7iizbZ18ujPp6bRT1b+rhb0hdIPeG4uTeUCa5N5Stvl4bBXnsfUECiALsP3rF6igqpW4BWmit/1H7qXKt0s5JQUqpfwMxwC21kij3ogBrkWV5SOkeUO+vDaf6/BtSF8i9oTi5N5RJ7g3nUc+vDcDzfz8kgCifa4EOSql19vfNwTR60VoPc12yXE8p9TLQBrilSK5TfXEYGFhkWQvgCxekxa3ItVGI/IbUTfJ3LYX8/8u9oTRybTh59O+HjERdCY5W8lrrqa5NievY6+29DYQBd2it81ycJJdQSoViiqX7a623KaV6Az8ArbXWKa5NnWvItVE2+Q2pm+TvKv//DnJvKE6ujfPztN8PKYEQlTUEuBfYBKxWSjmWT9FaL3NZqmqZ1jpVKTUMmK2U0pgi6iH19QZhJ9eGEPWX/P8j94ZSyLVRh0gJhBBCCCGEEKLcpDGPEEIIIYQQotwkgBBCCCGEEEKUmwQQQgghhBBCiHKTAEIIIYQQQghRbhJACCGEEEIIIcpNAgghhBBCCCFEuUkAIYQQQgghhCg3CSCEEEIIIYQQ5fb/9OoP4zkR5r0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 792x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "z = np.linspace(-5, 5, 200)\n",
    "\n",
    "plt.figure(figsize=(11,4))\n",
    "\n",
    "plt.subplot(121)\n",
    "plt.plot(z, np.sign(z), \"r-\", linewidth=2, label=\"스텝\")\n",
    "plt.plot(z, logit(z), \"g--\", linewidth=2, label=\"로지스틱\")\n",
    "plt.plot(z, np.tanh(z), \"b-\", linewidth=2, label=\"Tanh\")\n",
    "plt.plot(z, relu(z), \"m-.\", linewidth=2, label=\"ReLU\")\n",
    "plt.grid(True)\n",
    "plt.legend(loc=\"center right\", fontsize=14)\n",
    "plt.title(\"활성화 함수\", fontsize=14)\n",
    "plt.axis([-5, 5, -1.2, 1.2])\n",
    "\n",
    "plt.subplot(122)\n",
    "plt.plot(z, derivative(np.sign, z), \"r-\", linewidth=2, label=\"Step\")\n",
    "plt.plot(0, 0, \"ro\", markersize=5)\n",
    "plt.plot(0, 0, \"rx\", markersize=10)\n",
    "plt.plot(z, derivative(logit, z), \"g--\", linewidth=2, label=\"Logit\")\n",
    "plt.plot(z, derivative(np.tanh, z), \"b-\", linewidth=2, label=\"Tanh\")\n",
    "plt.plot(z, derivative(relu, z), \"m-.\", linewidth=2, label=\"ReLU\")\n",
    "plt.grid(True)\n",
    "plt.title(\"도함수\", fontsize=14)\n",
    "plt.axis([-5, 5, -0.2, 1.2])\n",
    "\n",
    "save_fig(\"activation_functions_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def heaviside(z):\n",
    "    return (z >= 0).astype(z.dtype)\n",
    "\n",
    "def sigmoid(z):\n",
    "    return 1/(1+np.exp(-z))\n",
    "\n",
    "def mlp_xor(x1, x2, activation=heaviside):\n",
    "    return activation(-activation(x1 + x2 - 1.5) + activation(x1 + x2 - 0.5) - 0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmUAAAEKCAYAAABTxSI1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3XuUXHWd7/33N5fu0AlJOhcSQ+gEItLjKPBwO4zjDByV8QyznAOIy9uJg3LMzBEFRpEHZ0ZlZI6XKAreRpklqEGO5wwEUI+3KKIOyMMlKg9HwzxGEggYbjYJSSd9/T5/7CqoVKq7qvb98nmt1SupXbv2/qa78utP/fb+7m3ujoiIiIhka0bWBYiIiIiIQpmIiIhILiiUiYiIiOSAQpmIiIhIDiiUiYiIiOSAQlnOmNnnzWxbw9ffNTz3LTO7vMvtdfUaM7vdzP5mmuffambPNHzd3fDc6Wa2p832l5mZm9kfdVpTh3V/2cw+21yDmQ2a2cua1n25ma1pePyAmZ3Xxb7czE7qsr5tZnZuN68RKRqNX9GY2Xlm9kAS245C41d6FMpywsxOM7NtwJlNT62r/YdY2eb1XzSz6zvc129qA0v9a8pBrJm7X+fuCxu+Tun0tTXnAJPAr7t8HQBmdllj7R285A3Ah5qW/RPwmg729Ukz+0aHdZ1RC3f7zexnZnZ8B6/Z1vRzaP6a9mcukhcav7pjZn9tZr/qcN1PtBknvtriNfvbvOaFTetr/MqJWVkXIAF3/zGw2syWA6cAhwK/B/7N3Z8FMLPpNrEceKC23ixgXm35cz9jM3sx8GLgAxwYyMdqn4J+ONXGzWwAuH+a/V8O/GK6As3sWODDwFPAF8zsr9x9ZLrXNL1+PnAd8PWGZQuB3Z1uo0uTwHAHdQ0CtwJ/B3wDuAD4npkNuvtQm5efB9w0xXN7Oy9VJDsav0LZ1+F6HwA+OsVzVzH1OPEa4N+ali0CtjYu0PiVLwplOWJmrwa+CTwIPA0MAAvN7D+4+/9XW21OLYjsdfex2usOBU4H5tUGtP9U207dXbU/DwNeMk0Jd071hLs/XKtlFvAq4CiCwek2d3+qVsfptT8XBi/xXbXHM4C/AT4OfKb25zeAe83s74Fveu0qxmY2F/gVcJO7v7upjPuBVS3KO3Kaf1MUhwPbO1jvQuBH7n5V7fF7zOwvgbXAp9u8dsTdpz1kIlIEGr/ajl+NVgAPT/N8Y+3DTPHh0Mz6gMemeOked3+maf1Wv/M1fuWIDl/my5HAZuBkdz+d4BPnJMGnyLr/GxgCXtuw7NPA7QSfvD7l7t9yd3N3A/53fSV3v53gk14f8FcEn4heBzzh7pe7+1T/uQEws0XAL4G/rm3j5cCvzexVDavNrdX3eO017yIYfN4LrHX3y9z9aeA04KvAtcAjZvZntdePEQzqjzTv391X1/5NPcDS+r/R3bdNVzfwyqZDnqe1Wb8+EJ9GZ/9HTiH4/jf6UW25SFVo/Jpm/GpyWm3dqPqBJyJuQ+NXjmimLF++SDAYfN7Mzgb+mWDwusPMDqmt84/ufjk89ynpU8CfAC8jGNS+b2bfBC52960c7O+AVwMvd/dHzewE4DYze8jdv1NbZ5aZzQHG3H2i4bXnApPufnZ9gQUnxr4b+EFt0V53n9fwmnuAi4FvuPtofaG7jwMfN7PPEHxy/Ult+SjwZ0zBzN5LcI7YTDO7n2BQniA43DHVJ7ZNwF82PP72VNtv8MfAfOC/mNk/1T81N/67aodj/lttvaebnn8K+MMO9iNSFhq/2oxftX0eRhAIx83sCHdvF+Cms4hagIxA41eOaKYsJ8zsa8D/C1wB/EdgIcFJ6v8C/JzgE2azbwBLgFPc/YnauRsvJziP4PQpdvVigqnqRwHcfTPBuRyNhwU+QzBAvqvptQ8CAxac1NtrZkfVat0y1b/L3e9y9xsbB7Sm5/fXPhnvn2obdbUB+APASQSfdO8DPktwuOC107x0sraf/bX9TLbbF8H3+1+AfyU4b6PZfyT4lHot8CiwuOn5JbXl7fyPaU6UfXkHrxfJnMav9uNXgwsJvlf/TDB+RbEC2DHFcz9qHlOAJ1usp/ErRzRTlh/vAGYTBIZxYLj2aew5ZvbHBFPrdWfXT6Ktq30y/FjDop8Av214/HXgOgs6pX5DMAgeB7ytYZ3/5u5faC7Q3X9sZu+obX9VrZZvAP9YW+UZ4KcN9X6Z4DBDp/7c3b87zfMvBn7t7v+ntv3/BWxw93m1fcXCzN5OMHV/HsHP4n4zu8TdP9Gw2nPna1jQwv5KgnNNsGAK7ZXAJ9vs6lSm/z8Y9bCESFo0frUfv7DgUjqXAn9BMAt3v5n9g7v/Uxf7qW9rEUF4anVu2lyg3llxJfBC4D/Xn2z62Wj8yhN311eOvggGjJ3TfF3S4jUzCD593UXwSWiSoPPpu8CrW6z/MoIZHgf+O7Cm4bnbgb9pU+N8YEHtz/kEnVbzal9zm+qa1fR1XG2/i1s8Vz9f7HvA37bY7x8SHKKsH1q8nuCEWoAvE3zqPJ0gMNVfc3ltf81fFzes8wBwXu3vawm6hv6s4fn/C9gFXF2r0YGTGp4fJPhk/m6CE4g/RTAg9Tessw04N+v3l770leSXxq9px6/jCE7K/1jDspMJusfXAzMJPgg+0OH3+mUE56XNbrPeVcC3pnle41eOvjRTlj8fovaJpYWv8nyreKNPEJxbdRHBJ8vdwAuA1wPfMLOz3f2586jc/U4zGwbeShA0emuHBjv178DS2t+t6c+99RrdfZKmQ4VmVj/HY8KbPknXnp8NrCGYlj+Au/8fM7uIoAV7EXAbwSA2nQ8RXJes2UTzAgsuuXEp8GZ3/37Dfn9uZqcR/ILw5tZ+d99iZq8hGPw+QnCy85/5FO3kZjaT579fbbX6PonklMavFuNXrXHoywSnQ1zW8G+5p9Zo8G6CUNb4mpkEs49T+UOCsDSztu44QWBsHlss2NzBnZfuPq7xK18UyvLnSuDNHHziZd33Wyw7E7jK3Tc2LNsOrDezVwJ/DnzbzH4JHFN7vv6zf5jgU+njQOPr2/nP7v6txgW1lvJvtV69M+6+l2CqfarnvwR8ycxm+oEn8U61/kED6zTr7jazY7320bDpuV8wzXWM3P0HTN+u32grrS/t0ZKZHeLdnbMikhWNXy3GL3efNLM/9uDyFs3P3U1w/l3ztdzWElyXsZ369c4+VtvOVGPLQd2e9bFF41d+KJTl09fcveOrVAPfAd5lZr8huFbPHoJr+pxD0Hr9udp6Lyf4NOYEQWVf86eY2iDYibkWXM+nUatPwYnoJJCF3G4ndwmIuo/VSe9DJEMav1poFcjarP9lgtm1blzWfpVoNH4lS6Esn843s/8yxXN3u/srmpZdQvDJ8n0E5wQsIjhp9T7gnPrUvzedVBvR16dYrqs4i1Sbxi+RkCyFiQERERERaSPydcrMbLaZXWJmY2b2hinWWWbBDWd/bWZ3m9lPzeylUfctIhKFxi8RyZM4Lh77doJj/HdNs84JwPfc/Q/c/RTgFoITQkVEsqTxS0RyI/I5Ze7+eYBaS+1U63ynadHvptq3ma0D1gHMmdN74sojDotaYmzcZ2GWn+7eOOsZnZwFE4ZNdNSoOKWZs2YwMR5tG3FSPdPLWz0A2x/Z9pS7L22/ZnRJj19H5Gj8gnTGsPEuPuvP8JlMWiI9O12Z8KDmWW6MWzan9HiL/4azbAbjrZ6IZYcdX9XiObNnGGOT3X1/kv52zpphjCc8hlkXp3lFHb9SP9HfzJYRXMvm/FbPu/s1wDUAR79owG/8QX7OeXtsy9+yYnB91mU8J956xtgw9DI2XXsqK256KPRWzrn4ZDauvyemmqJTPdPLWz0A2/nM9qxrmEo349eLXjTg3/xhfj7EAWzbcgmrBz+ayr5u3N3+0mFrHjmLrUfckkI17W3aOcibho/nhr4pr3yTuG07Dvxd/rfzVnHlnuT+O/Q+3NPV+he+8HA+/ZtO7r50oEO3J/d7/Pw/OZwv/fRRFmwdSWwfdT1bprqj1fO28/lIP7BU731pZosJbgZ9ubv/OM19S3tr++/kjLfdxYOXdnwJGpHK0PjVnXPnb866hK6csXzKW2CmZvXKVremTM7IwCgjAy1v6xmrZ1cZz67qfmauG7vW9Ca6fYDRwZWJ7yO1UGZmLwB+CFzp7tentV/pztr+O7n5rKsUzEQaaPwKp2jBbP7s/ZmHs7SDGZBKMANSCWZJh7Okg1kioczMFpvZHWZ2dO3xKoIB7Qp3vyGJfUq86sFs30sOz7oUkVRp/IpX0YIZZD9rtnrlk5nMmqUh6WAGyc+aJRnMkpop6yO4DcOC2uMrgWXAe83srtqXpv9z7uazruL3F+xVMJOq0fgVMwWzcHp70j0nUcGsc0kFs9hCmbuf7u5fr/39EXdf6e731h6f6+6L3f3Uhq/T4tq3JGfDcdfx+wv28thrj8y6FJHEaPxKnoJZOJoxC6+IwSzVE/2lmDYcdx1nvO0uBTMRieTc+ZsLF84UzJJTlmAWZzhTKJOOqDNTROKiYNY9dWaGV6TOTIUy6Zg6M0UkLkUMZlmHM3VmhleUzkyFMumaOjNFJA79M4ezLqFreQhmOpwZXhqzZlEolEko6swUkTgUbcYMsg9moPPMoshzMFMok9DUmSkicVAwCyftYDbZk859cqsczBTKJBJ1ZopIHNSZGY5mzMLLYzBTKJPI1JkpInFRMOueOjPDy1swUyiTWNQ7M0eW92RdiogUXBGDWdbhTJ2Z4aXRmdkphTKJ1ZqFT6gzU0QiK1owg+xnzdSZGU0egplCmcROnZkiEgcFs3AUzMLLOpgplEki1JkpInFQMAtHwSy8LIOZQpkkRp2ZIhIHdWaGo2AWXlbBTKFMEqXOTBGJi4JZ99SZGV4WwWxW6nssqMnxCXpmXc3k+AQzZs3MupxCWdt/J2vPupOzuZhj1m/PupxKuO3NX2K0r/0tbL47BLy9/fZ6hvt4xdfOj16YZGZifILZMz/NxPgEMws8hp07fzM37j4h6zI6Vg9mm3YOZlbD6pVPsm3H0lT3OTIwSu/D4bvxt05+kAmebbveO7YBR7Tf3qyJQznuscu7rqMezBZsHen6tWFopqxD+4Z2M8N+y76h3VmXUli6Z2Z6OglkWW5P0rdnaA9mv2XP0J6sS4msaDNmkP2sWdE6MzsJZN0Ynxlte2nNmimUdWByfILRPXsxc0b37GVyfCLrkgpLnZki6ZsYn2D42WHMnOFnh5kowRimYBZOkYJZ3qQRzBTKOrBvaDd47YGj2bKI1Jkpkq49Q3sOGMPKMFsGCmZhKZiFl3QwUyhroz5L1kizZdGpM1MkHfVZskZlmS0DdWaGpWAWXpLBTKGsjQNmyeo0WxYLdWaKJO+AWbK6Es2W1SmYda+snZlpSCqYKZRNo9UsWZ1my+JRv2emgplI/FrNktWVabasrojBLOtwVuZ7ZiYtiXtmKpRNo+UsWZ1my2KlzkyR+LWcJasr4WwZFC+YQfazZkXrzMybOINZ5FBmZrPN7BIzGzOzN0yxjpnZFWb2oJn9ysyuN7O5UfedpOlmyeo0WxYvdWZK2so6fsH0s2R1ZZwtAwWzsBTMwosrmMUxU/Z2gs9id02zzl8BZwLHu/uLgTHgYzHsOzHTzpLVabYsdurMlJSVcvyCNrNkdSWdLQMFs7AUzMKLI5iZe7v/tR1uyOx24Avu/vUWz30HuNndr6k9Ph74obsvbrHuOmAdwNKlS078yvUfiKW+7uxizuwPYTbWdk332ewf+yAwP/mymoztX8bsOY+nvt/pxFXT0xNz2f3UPGYPRbuKcv/yuQztnH7GM01p1bNu6C2xb/Oa/q/Gvs1W1l30lvvc/aRUdlaT1Pi14WvvT7Tuqe2iZ9YVHY9ho+MfIIsxbGT/cnrn7Ex8P0MTfR2t1zu6kJGeZxKupr3dY3MAWDTZx+9nZHPh5pHRg2/4s2xmL49PJHNl+xmjB84RvfPfXx/7Pj77ov/JjNF4Ms9U3nHeGyKNX2ndZukoYGvD463AIjNb4O67GlesDXzXABz9ogFfMbg+pRKft/fJIUafbT+YAZiNsWDxR5i7tD/hqg722JZLyeL7M524aloBbBh6GRt/fEqkWzOdc+nJbFx/T+R64pJaPR3cOqlbefo+pizU+PWiFw346sGPpldlg11P7mJ4d+djWP/ij7Bg6YKEqzrYti2Xkcb3aDV0dGumNY+cxdYjbkm8nk5s2jnIm4aP54a+X2RTQB8H3ZrpPfNWceWeZG+VF+XWTO18+jePPvf3Q7cnG87CSutEfwMaT1wYT3n/HevkXLJmOrcsGerMlJwozPgFnZ1L1qys55Y1KtrhzDOWb2H+7P2Z1lDmzsykb2YeVlqDyg5goOHxALAHyH6euElH55I107lliVJnpmSsMOMXdHguWbMSn1vWqGjBDLI/z6zMnZl5DGaJhDIzW2xmd5jZ0bVFG4D/amb1ecl3ARs9rhPaYhJmlqxOs2XJUmempKWo4xeEmyWrq8JsGSiYhZXFrFka8hbMkpop6wNWAfWTFL4K/BS428zuJTij9F0J7Tu0ULNkdZotS5w6MyUlhRy/IOQsWV1FZstAwSys3p7x9isVUJ6CWWwn+rv76Q1/fwRY2fB4Avj72lcuRZklqxvds5dD+uczY9bMmKqSZhuOu44NAy9jE6ey4qaHsi5HSqLo4xdEmyWrG352mHn985hZgTGsHsw6aQDIizOWb2HTzsFMa1i98smDGgDK4NlVlouT/3N5omoWIs2S1Wm2LBW6Z6bIwSLNktVVaLasrmizZnmYMSvzocysZ80UyohnlqxO55alQ52ZIs+LY5asrirnljUqYjDLOpyVNZhBtoczFcqIaZasTrNlqVJnpkhMs2R1FZwtg+IFM8h+1iyLzsy0ZBXMFMqAiZF422/j3p5MT52ZB+sZ7uwK5lltT+I1uj/eMSfu7RVF/8xsrp4fRdbBDJKZNZsxY16s25vJoV2/JotgltYV/XNt/splHa2XxyvoS2DDcdex9oK3Yj84Ug0AwCu+dn5H6+XtjgcSztIjOjvxOq0r6BfZufM3F+rkfyhnA8ARA//Qdp3eh3u48IWHH3Cl/ril3QCgmTIpjQ3HXccZb7tLl8wQkUjOnb+5cIczyzpjNp0yXmRWoUxKRZ2ZIhIXBbPupR3MJnsmUwlnaXVmKpRJ6agzU0TiUsRglnU40z0zw1Mok9K6+ayrGFneowYAEYmkaMEMsp810z0zw1Eok1Jbs/AJdWaKSGQKZuEomHVHoUxKT/fMFJE4KJiFo2DWOYUyqQR1ZopIHNSZGY6CWWcUyqQy1JkpInFRMOteFsGsaJ2ZCmVSKerMFJG4FDGYZR3O1Jk5PYUyqSTdM1NE4lC0YAbZz5qVuTMzKoUyqSzdM1NE4qBgFo6C2cEUyqTS1JkpSRlnRuHuoSjhKZiFo2B2IIUyqTx1ZkqSFMyqQ52Z4SiYPU+hTITnOzMVzCQJCmbVomDWvbJ2ZnZLoUykZm3/nXz1PZ9UZ6YkQsGsWooYzLIOZ2XuzOyUQplIE3VmSlIUzKqlaMEMsp81q3pnpkKZSAvqzJSkKJhVi4JZOFUNZpFDmZmdZmabzex+M7vXzE5tsc4yM/tXM/u5md1tZv9mZi+Pum+RJKkzs/yyGr9u3H2CwlmFKJiFU8VgFimUmdlCYCNwgbsfC1wC3GpmfU2rfhh4CjjB3U8BPgn8zyj7FkmDOjPLKw/jl4JZdagzM5yqBTNz9/AvNns9cJG7v6xh2S+A97v7NxuWfQg4Gniru+83s78Gzq8NcM3bXAesA1i6dMmJX7n+A6Hri9vY/mXMnvN41mU8J2/1QP5qiquepyfmsvupecweGom0nf7lcxnauTdyPXHJWz0A6y56y33uflLS+0l6/FqydMmJn/nqRzqqpX/mcKR/S6dG9i+nd87OVPbViarWMzTRnPtb6x1dyEjPMwlX097usTkALJrs4/cz0nmvNhsZndVy+bKZvTw+EW1cnsqM0e7nrd75ptdHGr9a/ys7dxSwtWnZ1tryRh8ErgGeMLNngCeBv2i1QXe/prYuR79owFcMro9YYnwe23Ipqmd6easprnpW1P48+5aLOWb99tDbOefSk9m4/p7I9cQlb/WkLNHx66gXrfatR9zScTFpzKJs23IZqwc/mvh+OlXVelbT2SzpmkfOopv3UOJ++wZu6PtFNvvug207lh60+D3zVnHlnvBjcju9D/cktu1Wop5TZsBE07LxFtt9H3A4cIS7DwBfAL5lZjMj7l8kVerMLJVcjV86lFktRTuUCTB/9v5M91+FzsyooWwHMNC0bKC2vNGbgKvdfReAu/8LwSB3XMT9i6ROnZmlkbvxS8GsWooYzHSeWbKihrJbgWPN7KUAZnYKMAjcZmZ3mNnRtfX+HTjHzGbU1vtTYD7wcMT9i2RCnZmlkMvxS52Z1aJgFk5Zg1mkUFb75Pg64Fozuxu4CjgT6ANWAQtqq76DYBDbbGb/D/Bx4LXu/lSU/YtkSZ2ZxZb38UvBrDrUmRlOGYNZ1BP9cfcfASe3eGplwzo7gTdG3ZdI3qztvxPeBps4lRU3PZR1OdKlvI9fN+4+oXC/rCW8c+dvLlQYP2P5FjbtHMy0ht6e8VT3Vw9mSTUA6Ir+IhHpnpmSpCL9kpboihbCdc/MeCmUicREnZmSFAWzailaMIPsD2eWpTNToUwkRurMlKQomFWLglk4RQ9mCmUiMVNnpiRFnZnVktadHuKkYBaNQplIAtSZKUlSMKsOdWaGU9RgplAmkpC1/XcqmEliFMyqRcGse0UMZgplIglSZ6YkScGsWooYzLIOZ1l0ZkahUCaSgnpnph8yO+tSpGQUzKqlaMEMsp81y6IzMyyFMpGU3HzWVYwfNqnOTImdglm1KJiFU4RgplAmkqIjD3lanZmSCHVmVouCWTh5D2a5DmUjEzrUI+WjzkxJkoJZdagzM5w8B7Nch7IZY5OcfcvFbBh6WdaliMRKnZmSJAWzalEw615eg1muQxnAMeu3s+naUxXMpHTUmSlJUjCrliIGs6zDWR6DWe5DGcCKmx5SMJPS0j0zJSkKZtVStGAG2c+a5a0zsxChDJ4PZmt/+dasSxGJne6ZKUlRMKsWBbNw8hLMChPKIAhmiz43V8FMSkn3zJSkqDOzWhTMwslDMCtUKAM45IFHWfS5uZx9y8VZlyISO3VmSpKGJvqyLkFSos7McLIOZoULZRAEs2PWb1dnppSSOjMlSZoxqxYFs+5lGcwKGcrq1JkpZaXOTEmSglm1FDGYZR3OsgpmhQ5loM5MKTd1ZkpSFMyqpWjBDLKfNcuiM7PwoQzUmSnlps5MSYqCWbUomIWTZjArRSgDdWZKuakzU5KizsxqUTALJ61gFjmUmdlpZrbZzO43s3vN7NQp1ltiZjea2QNmdp+ZfSTqvpupM1PKTJ2Z8cvT+JU1BbPqUGdmOGkEs0ihzMwWAhuBC9z9WOAS4FYz62tarxf4FnC1u7/E3U8EPhFl31NRZ6aUmToz45P0+DXhxTsQoWBWLQpm3Us6mEUdNV4NPOjuPwNw99uB3wGvbFrvLcBdwIW1T6MbgNkR9z0tdWZKWakzMzaJj1+bdg7GV21KFMyqpYjBLOtwlmQwM3cP/2Kz9wEvdve1DctuAn7i7lc3LNsAHA+8BngY+BDwR+7ePPhhZuuAdQBLliw58cPv/2To+gDG+nuZv2QPi2fujbQdgLH9y5g95/HI24lL3uqB/NVU9nq2PnMYPbsc2zcW6vX9y+cytDP6/404rbvoLfe5+0lJ7yfx8WvpkhP/+5c+BcD82fsT/Jd0rnd0ISM9z3S0bv/M4YSrgZH9y+mdszPx/XSqyvV0cmHhbt4/adg9NodFk338fkby79WpjIzOOuDxhee+MdL4Nav9KtMyYKJp2TgHz8AdBnzZ3bcBmNnHgF1mNs/d9zSu6O7XANcArB440jeuvydiifDYa4/EXzXEhuOui7adLZeyYnB95Hrikrd6IH81lb2eFcDaX76VRZ+byyEPPNr168+59GTi+D9WUImOX6uOPspv6PvFARvK+hP+mkfOYusRt3S8ftKzKNu2XMbqwY8muo9uVLme1bSfJe32/ZOK376B5v9nqeqDbTuWxra5qIcvdwADTcsGassbPQHsbng82fCVOHVmSpmpMzO01Mevoh3OVGdmtRTtUCbkYxY6zsOZUUPZrcCxZvZSADM7BRgEbjOzO8zs6Np6G4F1ZnZo7fHFwG3untqcozozpczUmRlKJuNX0YIZ6DyzKlFnZjhxBbNIoczddwGvA641s7uBq4AzgT5gFbCgtt7NwP8C7jGznwMnA+dF2XcY6syUMlNnZneyHL8UzCTvFMy6F0cwi3pOGe7+I4JBqtnKpvU+Dnw86v7icMz67Wzaeiq8LfhFJlIWa/vvZO177uTsNRdzzPrtWZeTe1mOX5t2DubiF0k3btx9QuF+WUt4587fXKgwXv//VMQPPXXFu5BOTHTPTCkz3TOzGIr4y6NIv6QluiKG8KJ92GlU2VAGumemlJvumVkMm3YOFi6cKZhVi4JZeiodykCdmVJu6swsjiIGM4Wz6lAwS0flQxmoM1PKTZ2ZxVG0YAaaNauS/pnDhQtnRQtmCmU16syUMlNnZnEomEneKZglR6Gsie6ZKWWle2YWh4KZ5F0Rg1kRwplCWQvqzJQyU2dmMSiYSd4VLZhB/mfNFMqmoM5MKbN6Z6YfMjvrUmQa6syUvFMwi5dC2TTUmSlltuG46xg/bFLnmRVAEYOZwll1KJjFR6GsDXVmSpkdecjTagAoiKIFM9CsWZXonpnxUCjrQL0zc+szh+k8MykddWYWh4KZ5J2CWTQKZV3o3TmqBgApJXVmFoeCmeRdEYNZXsKZQlmX1JkpZabOzGJQMJO8K1owg3zMmimUhaDOTCkz3TOzGNSZKXmnYNY9hbKQ1JkpZaZ7ZhZHEYOZwll1KJh1R6EsAnVmSpnpnpnFUbRgBpo1qxJ1ZnZOoSwi3TNTykydmcVRxGA2NNGXdQmSIgWz9hTKYqJ7ZkpZqTOzOIoYzDRjVi1FDGZwpKIzAAAYOklEQVRphjOFshipM1PKTJ2ZxaBgJnlXtGAG6c2aKZTFTJ2ZUmbqzCwGdWZK3imYtaZQlgB1ZkqZqTOzMz6ZdQXFmzVTZ2a1KJgdTKEsIerMlDJTZ2Zntu1YmnUJhQtmoFmzKlFn5oEihzIzO83MNpvZ/WZ2r5md2mb9D5rZmJmtjrrvvFNnppRZGToz0xi/FMzCUTCrFgWzQKRQZmYLgY3ABe5+LHAJcKuZtexzNrPXAC8AHo2y36JRZ6aUVZE7M9McvxTMwlEwq5YiBrO4w1nUmbJXAw+6+88A3P124HfAK5tXNLNjgIuBiyLus5DUmSllVtDOzFTHr207lmYeznaPzcl0/2EomFVL0YIZxDtrZu4e/sVm7wNe7O5rG5bdBPzE3a9uWDYf+DbwZnffbmbbgNPdfVuLba4D1gEsWbLkxA+//5Oh64tb//K5DO3cG2kbY/29MH+cIw95OnI9Y/uXMXvO45G3E6e81aR6phd3PQ/tW8ysJ2Zg+8ZCb2PdRW+5z91Piq2oKSQ+fi1dcuIH/vkzLffd2zMe47+kc4sm+/j9jGEA5s/en0kNjXpHFzLS80xH6/bPHE64GhjZv5zeOTsT30+nqlxPpxcW7uY9lLTdY3P4679cG2n8mhWxBgMmmpaN0zADZ2YGfAW4wt23t9ugu18DXAOweuBI37j+noglxuecS08mjnr2veRwfn/BXjYcd12k7Ty25VJWDK6PXE+c8laT6ple3PWsANb+8q3YD/pZcdNDsW03IYmOXwNrjvIr90z9ktUrnwxRcjRvGj6eG/p+8dzjrG++vOaRs9h6xC1dvSbJmZRtWy5j9eBHE9t+t6pcz2o6myUN8x7Ks6iHL3cAA03LBmrL6w4Fjgf+0czuMrO7CM7LuNnMzou4/0JSZ6aUWYE6MzMdv7I+lAk6z0zyrYidmVFFDWW3Asea2UsBzOwUYBC4zczuMLOj3X23ux/p7qfWvwjO2zjb3b8ccf+Fpc5MKbOCdGZmPn4pmIWjYFYtVQpmkUKZu+8CXgdca2Z3A1cBZwJ9wCpgQeQKS06dmVJW9WCW187MvIxfCmbhKJhVS1WCWdRzynD3HwEnt3hq5TSvWR11v2Wy4qaH2MSp8LbgF5lIWaztv5O1Z93J2VzMwLfHOeSBfF0NJy/jVz2YZXGeWd2mnYOZn2PWrRt3n1CZX9YSBLOyh3Fd0T8ndM9MKTPdM7MzWc+a6Z6ZkndlD+EKZTmie2ZKmememZ3JOphB8Q5n6p6Z1VLmYKZQljPqzJQyK1BnZqYUzMJRMKuOsnZmKpTlkDozpcwK0pmZOQWzcBTMqiWNiwqnSaEsx9SZKWWV987MvFAwC0fBrFrKNGOmUJZzumemlNXa/juLes/MVOXhnpkKZpJ3ZQlmCmUFoM5MKTN1ZnYmD8GsaOFMwaxayhDMFMoKQp2ZUmbqzOxM1sEMijdrps7Mail6MFMoKxB1ZkqZ1TszZXoKZuEomFVHkTszFcoKprEz8+mJuVmXIxIr3dGiMwpm4SiYVUsRg5lCWUEds347u5+apwYAkYpSMAtHwaxaihbMFMoKbPbQiDozRSpMnZnhKJhVS5GCmUJZwakzU0TyEMyKFs4UzKqlKMFMoawE1JkpkkNuqe4u62AGxZs1U2dmtRQhmCmUlYQ6M0Xyp/fhnlT3p2AWjoJZdeS9M1OhrER0z0yR/FEwKwYFs2rJazBTKCsh3TNTJF8UzIphaKIv6xIkRXkMZgplJaV7ZorkSxbBLOtwVsRgphmzaslbMFMoKzF1ZorkS+/DPZWbNVNnpuRdnoKZQlnJqTNTJH+qFsygeLNm6syslrwEM4WyClBnpkj+KJgVg4JZdeShM1OhrCLUmSmSP2kHs5HRWanurxUFM8m7LINZ5FBmZqeZ2WYzu9/M7jWzU1uss8zMvmhmvzazu83sp2b20qj7lu6pM1PkeXkYvzRjVgwKZtWSVTCLFMrMbCGwEbjA3Y8FLgFuNbPmvuITgO+5+x+4+ynALcCVUfYt4akzUyRf45c6M4tBwaxasghmUWfKXg086O4/A3D324HfAa9sXMndv+PuGxsW/Q7Ifh69wtSZKZKv8auKnZm7x+YULpwpmFVL2sHM3D38i83eB7zY3dc2LLsJ+Im7Xz3Fa5YBdwDnu/uPWzy/DlgHsGTJkhM//P5Phq4vbv3L5zK0c2/WZTwnjnr8kNmMHzbJkYc8HUtNY/uXMXvO47FsKw6qZ3p5qwfgzFdfeJ+7n5T0fpIfv5aeePmnPxuqtsmeyVCva2fZzF4enxg5aHlvz3gi+2tn0WQfv58xDMD82fszqaFR7+hCRnqe6Xj9/pnDCVYDI/uX0ztnZ6L76Ebe6oH0aur0wsJv/PO3Rxq/on7aM2Ciadk4U8zAmdli4NvA5a0GNAB3vwa4BmD1wJG+cf09EUuMzzmXnkwZ69n3ksN5+MxZ3HzWVZG39diWS1kxuD7yduKieqaXt3pSluj4NXDUGv/0bx4NXdzIwGjo107lPfNWceWe7S2fW73yydj3186bho/nhr5fPPf4jOVbUq+h0ZpHzmLrEbd09ZokZ1K2bbmM1YMfTWz73cpbPZBeTatrfyY9Uxr18OUOYKBp2UBt+QHM7AXAD4Er3f36iPuVGDV2ZopUSK7Hr6odygSdZyb5l/ThzKih7Fbg2HonkpmdAgwCt5nZHWZ2dG35KoIB7Qp3vyHiPiUhx6zfzluufLcaAKQqcj9+KZgVg4JZtSQZzCKFMnffBbwOuNbM7gauAs4E+oBVwILaqlcCy4D3mtldta+W0/+SLXVmSlUUZfxSZ2YxKJhVS1LBLHIHkbv/CDi5xVMrG9Y5N+p+JD0rbnqITZzK9191DBuOuy7rckQSU5Txqx7MkjjPbCrbdizN5Dyzunowy/o8s27cuPuEzK8IL+k5d/7m2MO4rugvLememSL5o8OZ+ad7ZlZL3CFcoUympHtmiuSPglkxKJhVR5z3zFQok2mpM1MkfxTMikHBrFriCGYKZdIRdWaK5IuCWTEomEk3FMqkY+rMFMkXdWYWg4KZdEqhTLqie2aKdMbC38GuK1W8Z+amnYOFC2cKZtIJhTLpmjozRTpz6Hbn0O3ppLOqBTMo3qyZOjOlHYUyCUWdmSKdUzBLTtGCGWjWTKamUCahqTNTpHMKZslRMJOyUCiTyNSZKdIZBbPkKJhJGSiUSSzqDQBPT8zNuhSRXCtzMMs6nCmYSdEplElsVtz0ELufmqcGAJE20gxmVZs1U2emFJlCmcRq9tCIOjNFOqDOzGQVMZgpnIlCmcROnZkinVMwS07Rghlo1qzqFMokEerMFOlcWsFsxmi6Q76CWTgKZtU1K+sC8uC2N3+J0b7htut9dwh4e/vt9Qz38YqvnR+9sBI4Zv123rL13ZzxtrtY239n1uVIg8nxCXpmXc3k+AQzZs3MupzKO3S78+wqC/XarZMfZIJn2673zn/vbHszZszjiIF/CFVLs207lrJ65ZOxbCusTTsHOWP5lkxr6NaNu0/gpKyLyLmJ8Qlmz/w0E+MTzCzJGKaZMugokGW5vaLTPTPzad/QbmbYb9k3tDvrUqQm7IxZJ4GsG5OTe2Ldnjozwxma6Mu6hFzbM7QHs9+yZyje92uWFMokFbpnZr5Mjk8wumcvZs7onr1Mjk9kXZLUpHUoMwt5CGZFC2c6lNnaxPgEw88OY+YMPzvMREnGMIUySY3umZkf+4Z2Q/13v6PZspxJszMzbVkHMyjerJk6Mw+2Z2jPAWNYWWbLFMokVerMzF59lqyRZsvyScEsOUULZqBZs7r6LFmjssyWKZRJ6tSZma0DZsnqNFuWWwpmyVEwK6YDZsnqSjJbplAmmdE9M9PXapasTrNl+VXWYDYymv0FABTMiqXVLFldGWbLIocyMzvNzDab2f1mdq+ZndpiHTOzK8zsQTP7lZldb2a6SaKoMzNlLWfJ6io4W1ak8auswUydmeFUNZi1nCWrK8FsWaRQZmYLgY3ABe5+LHAJcKuZNffx/hVwJnC8u78YGAM+FmXfUh7qzEzHdLNkdVWaLSvi+JVmMKvaHQA27Rxk99icTGvoVtWC2XSzZHVFny2LOlP2auBBd/8ZgLvfDvwOeGXTeq8Hvuju+2qPrwbeGHHfUiLqzEzetLNkddWaLSvk+KV7ZiaraLNmVerMnHaWrK7gs2VRD+gfBWxtWra1tny69bYCi8xsgbvvalzRzNYB6wCWLFnCOe8/OWKJ7X13KP5tnnNp8nX3L5+byn66EbUm3zmbn+77BGsWPhFLPWP7l/HYlktj2VYcsqtnF3Nmfwjr4ILxI7tH2fX03wDzE6+qtQvT2lHi49f5f3J4vBU3mewJfqCdXqm/Gxe+8PnaJ3sm498BsGxmL++Zt+rAhc+sordnPJH9tbNoso83DR8Pvz2e+bP3Z1JDo97Rhax55KyO1v05Z9E/M9kLl4/sX862LZcluo+p7aJn1hUdjWF7d40x9PQ7yGYMuyjSq6OGMgOa5wnHOXgGrnm9+v+4g2bq3P0a4BqA1QNH+sb190QssQMd3DqpW2nUfc6lJ6eyn27EVdODl67i5rOuirydx7ZcyorB9ZG3E5es6tn75BCjz451tK7ZGAsWf4S5S/sTripziY5fq1Yd5V/66aPxVDqNsLdmaufTvzmw9pGB0dj38Z55q7hyz/aWz2Vxa6Y3DR/PDX2/eO5x1rdmWvPIWWw94pauXnPu/M0JVQPbtlzG6sGPJrb96ex6chfDuzsfw/oXf4QFSxckXFX8oh6+3AEMNC0bqC2fbr0BYA/wTMT9S0mpMzM+nZxL1qwi55YlPn4t2DoSpb6O6FBmcop2KBPKeZ5ZJ+eSNSvquWVRQ9mtwLFm9lIAMzsFGARuM7M7zOzo2nobgP9qZvX/1e8CNrp7OduJJBbqzIxHR+eSNavGuWWpjF9pBLO0ZBHMsg5nCmbZ6+hcsmYFPbcs0uFLd99lZq8DrjUzJ5jWPxPoA1YB9bnDrwIvBO42s3HgV8A7o+xbqmHFTQ+xiVP5/quOYcNx12VdTuGEmSWrG92zl0P65zNj1syYq8qHNMevejDbtaY3nuIzVA9mSRzOnMq2HUszOZxZVw9mWR/O7MaNu09I9FBmWsLMktUNPzvMvP55zCzQGBb5yn3u/iOg1ZndKxvWmQD+vvYl0pUVNz3EvgcPZ+0Fb1Uw61KoWbK62mxZmc8tS3v8WrB1pBTBDIJwVqVgBkE4K1owg2TPM0taqFmyutpsWZHOLdMV/aUQdM/M7kWZJauryLllqdLhzPCyPpQJOpyZpiizZHVFO7dMoUwKQ/fM7E6kWbK6apxbljoFs/AUzMIpYjCLNEtWV7BzyxTKpHDUmdleHLNkdZotS4aCWXgKZuEUKZjFMUtWV6TZMoUyKSR1Zk4vllmyOs2WJUbBLDx1ZoZTlGAWyyxZXYFmyxTKgJ7h5lvd5Wt70prumTm1iZF4T8COe3vyvAVbRyKHs1kTh8ZUTWAm4bbX+3BP5WbNNu0cLFw4K0IwG90f75gT9/aSErn7sgxe8bXzO1ovj1fQrzp1ZrY2f+WyjtbL2x0PqixKZ+Zxj13e0Xrn/8nhXPXwY6H20Q11ZuZf3jszlx7RWdjO8i4DSdBMmRSeOjOlLHQHgPCynjEDHc6U6BTKpBTUmSlloWAWnoJZOApm+aFQJqWizkwpAwWz8BTMwlEwyweFMikddWZKGSiYhafOzHAUzLKnUCalVA9mD+1bnHUpIqHF0ZnZTprBrGqzZurMlG4plElprbjpIWY9MUOXzJDCSyOYlXnWLGtFDGYKZ9lQKJNSs31j6syUUtDhzPAUzMJRMEufQpmUnjozpSwUzMJTMAtHwSxdCmVSGerMlDIoUzCbMZruryAFs3AUzNKjUCaVos5MKYMyBTN1ZhaDglk6FMqkcnTPTEmDebKhRp2Z0eQhmBUtnA1N6L7OSVMok0pacdNDLPrcXAUzSVTPlh2J70OdmeFlHcygeLNm6sxMlkKZVJbumSlp6NmyI/FwpsOZ4SmYhaNglgyFMqk0dWZKWhTMOqdgVgwKZvFTKBNBnZmSDgWzzimYFYOCWbwUykRq1JkpaVAw61zawWxkdFbm4UzBrNoihTIzO83MNpvZ/WZ2r5mdOsV6y8zsi2b2azO728x+amYvjbJvkSSoM7NashrD0ghm6swMLw/BrGjhTMEsHqFDmZktBDYCF7j7scAlwK1m1qpn9gTge+7+B+5+CnALcGXYfYskSZ2Z1ZD1GKbOzO5ULZgB7B6bk3UJXVFnZnRRZspeDTzo7j8DcPfbgd8Br2xe0d2/4+4bGxb9DpgVYd8iiVJnZiVkPoapM7M7VQxmRZsxA82aRWHe5gKHZtYD/KTFU98BXujuaxvWvQn4ibtfPc32lgF3AOe7+49bPL8OWFd7+BLggXb/iBQtAZ7KuogGeasH8leT6ple3uoBOMbdD41rY2mOYTkfvyB/P2/VMz3V017eaoo0frX9pOfuo8BB51mY2d8BE02Lx5lm9s3MFgPfBi5vFchq+7sGuKa2/r3uflK7GtOietrLW02qZ3p5qweCmuLcXppjWJ7HL8hfTapneqqnvbzVFHX8inL4cgcw0LRsoLb8IGb2AuCHwJXufn2E/YqIxEFjmIjkSpRQditwbL0DycxOAQaBTWa22MzuMLOja8+tIhjMrnD3G6IWLSISA41hIpIroU9UdfddZvY64Fozc4Jp/zPd/RkzOwJYBSyorX4lsAx4r5m9t7ZsxN1Pa7Oba8LWlxDV017ealI908tbPZBSTSmMYZX93nZB9UxP9bSXt5oi1dP2RH8RERERSZ6u6C8iIiKSAwplIiIiIjmgUCYiIiKSA7kKZXm4l2YnNVjgCjN70Mx+ZWbXm9ncOPYfsp7U7i3a6c+oYf0PmtmYma1Oop5uajKzJWZ2o5k9YGb3mdlHsqqn9jP7VzP7ee1n9m9m9vKE6pltZpfUfg5vmGKdNN/TndRTuPvlavwKXU+qP+u8jWEav9rWk6vxq4uawr2v3T0XX8BC4Gngj2qPTwceB/parPvnwDkNj98DfD+tGoDzgPuAQ2qPrwM+m9X3JKnvR5SfUe351wBfALYBq7N83wC9wF3AnzQsW5xhPV8C/pnnm23OAR5N6Hv0jtp74qfAG6ZYJ5X3dBf1pPKeTvvnnuS/TeNXvD+n2vOJjmEavzqqKVfjVxc1hXpf52mmLPP70HVRw+uBL7r7vtrjq4E3xrD/UPUk+P0IVQ+AmR0DXAxclEAdYWp6C8GgdmHt098GYHaG9TxKMAD21h4vrS2Lnbt/3t2v5OCr1zdK6z3dUT0pvqfjovErZD0p/6zzNoZp/Gojb+NXpzWFfV+nPsjZ9Peh29q0bCtwVJvtLQM+BJwfQ3lHdVhD83pbgUVmtsDdd8VQR7f1PCfm70eoesxsPsEnqTe7+4iZJVBKdzUBfwocT/DJ92GC79HXaDEYp1TPBwmuZ/OEmT0DPAn8Rcy1dCOt93TXEn5Pd1uLxq/463lOCj/rvI1hGr/ikdvxC7p7X6ceyjzle2l2yTqsoXm98dqfcc88dlpPsHL834+u67Fg9PoKwZXPtydQQ9c11RwGfNndtwGY2ceAXWY2z933ZFDP+4DDgSM8uIjp24Fvmdl/cPfpPhEmJa33dFdSeE93ReNXIvUEK6fzs87bGKbxKx65HL+g+/d15gU3yMN96DqtoXm9AWAP8ExMdXRbT1r35euknkMJPtH9o5ndZWZ3AS8Abjaz8zKqCeAJYHfD48mGryzqeRNwdf1TnLv/C8Egd1zM9XQqrfd0x1J6T8dF41f4etL8WedtDNP4FY/cjV8Q8n2d1Ilw3X4R3M7kKeCltcenAEMEx60XA3cAR9eeWwX8CnhtSjU07/+tBIcwemqPPwt8JcXvSSrfj7D1tHjdNpI70b/T79HZwD3AobXHf08yzRCd1rMR+CIwo/b4T4G9wJIEf363UzspNav3dBf1pPKeTuHnrvErJ+NXNzW1eF0iY5jGr65qy9X41UFNod7XuTlx1tO5l2aoGoC+pv1/FXghcLeZjRN8498ZZd8R60nk+xGhntR0WpO732xmLwTuMbN9wHaCjp1M6iHo3vkUsNnMRmrLXuvuT8Vd0xQyeU93UU8q7+m4aPyKVE9qP+u8jWEav0LL2/jVqqZQ72vd+1JEREQkB/J0TpmIiIhIZSmUiYiIiOSAQpmIiIhIDiiUiYiIiOSAQpmIiIhIDiiUiYiIiOSAQpmIiIhIDiiUiYiIiOTA/w/I266Sc3h/QgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 720x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "x1s = np.linspace(-0.2, 1.2, 100)\n",
    "x2s = np.linspace(-0.2, 1.2, 100)\n",
    "x1, x2 = np.meshgrid(x1s, x2s)\n",
    "\n",
    "z1 = mlp_xor(x1, x2, activation=heaviside)\n",
    "z2 = mlp_xor(x1, x2, activation=sigmoid)\n",
    "\n",
    "plt.figure(figsize=(10,4))\n",
    "\n",
    "plt.subplot(121)\n",
    "plt.contourf(x1, x2, z1)\n",
    "plt.plot([0, 1], [0, 1], \"gs\", markersize=20)\n",
    "plt.plot([0, 1], [1, 0], \"y^\", markersize=20)\n",
    "plt.title(\"활성화 함수: 헤비사이드\", fontsize=14)\n",
    "plt.grid(True)\n",
    "\n",
    "plt.subplot(122)\n",
    "plt.contourf(x1, x2, z2)\n",
    "plt.plot([0, 1], [0, 1], \"gs\", markersize=20)\n",
    "plt.plot([0, 1], [1, 0], \"y^\", markersize=20)\n",
    "plt.title(\"활성화 함수: 시그모이드\", fontsize=14)\n",
    "plt.grid(True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MNIST를 위한 FNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "주의: `tf.examples.tutorials.mnist`은 삭제될 예정이므로 대신 `tf.keras.datasets.mnist`를 사용하겠습니다. `tf.contrib.learn` API는 `tf.estimator`와 `tf.feature_column`로 옮겨졌고 상당히 많이 바뀌었습니다. 특히 `infer_real_valued_columns_from_input()` 함수와 `SKCompat` 클래스가 없습니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "(X_train, y_train), (X_test, y_test) = tf.keras.datasets.mnist.load_data()\n",
    "X_train = X_train.astype(np.float32).reshape(-1, 28*28) / 255.0\n",
    "X_test = X_test.astype(np.float32).reshape(-1, 28*28) / 255.0\n",
    "y_train = y_train.astype(np.int32)\n",
    "y_test = y_test.astype(np.int32)\n",
    "X_valid, X_train = X_train[:5000], X_train[5000:]\n",
    "y_valid, y_train = y_train[:5000], y_train[5000:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Estimator API를 사용"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using default config.\n",
      "WARNING:tensorflow:Using temporary folder as model directory: /tmp/tmpojfdm9i4\n",
      "INFO:tensorflow:Using config: {'_model_dir': '/tmp/tmpojfdm9i4', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': None, '_save_checkpoints_secs': 600, '_session_config': allow_soft_placement: true\n",
      "graph_options {\n",
      "  rewrite_options {\n",
      "    meta_optimizer_iterations: ONE\n",
      "  }\n",
      "}\n",
      ", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7fd26e16a2b0>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}\n",
      "WARNING:tensorflow:From /home/haesun/anaconda3/envs/handson-ml/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:tensorflow:From /home/haesun/anaconda3/envs/handson-ml/lib/python3.6/site-packages/tensorflow_estimator/python/estimator/inputs/queues/feeding_queue_runner.py:62: QueueRunner.__init__ (from tensorflow.python.training.queue_runner_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "To construct input pipelines, use the `tf.data` module.\n",
      "WARNING:tensorflow:From /home/haesun/anaconda3/envs/handson-ml/lib/python3.6/site-packages/tensorflow_estimator/python/estimator/inputs/queues/feeding_functions.py:500: add_queue_runner (from tensorflow.python.training.queue_runner_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "To construct input pipelines, use the `tf.data` module.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "WARNING:tensorflow:From /home/haesun/anaconda3/envs/handson-ml/lib/python3.6/site-packages/tensorflow/python/feature_column/feature_column_v2.py:2703: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "WARNING:tensorflow:From /home/haesun/anaconda3/envs/handson-ml/lib/python3.6/site-packages/tensorflow/python/training/monitored_session.py:809: start_queue_runners (from tensorflow.python.training.queue_runner_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "To construct input pipelines, use the `tf.data` module.\n",
      "INFO:tensorflow:Saving checkpoints for 0 into /tmp/tmpojfdm9i4/model.ckpt.\n",
      "INFO:tensorflow:loss = 118.85941, step = 1\n",
      "INFO:tensorflow:global_step/sec: 477.233\n",
      "INFO:tensorflow:loss = 4.778356, step = 101 (0.210 sec)\n",
      "INFO:tensorflow:global_step/sec: 501.009\n",
      "INFO:tensorflow:loss = 19.603981, step = 201 (0.200 sec)\n",
      "INFO:tensorflow:global_step/sec: 541.6\n",
      "INFO:tensorflow:loss = 6.543038, step = 301 (0.185 sec)\n",
      "INFO:tensorflow:global_step/sec: 574.88\n",
      "INFO:tensorflow:loss = 5.7384977, step = 401 (0.174 sec)\n",
      "INFO:tensorflow:global_step/sec: 526.035\n",
      "INFO:tensorflow:loss = 2.469788, step = 501 (0.190 sec)\n",
      "INFO:tensorflow:global_step/sec: 555.513\n",
      "INFO:tensorflow:loss = 16.599104, step = 601 (0.180 sec)\n",
      "INFO:tensorflow:global_step/sec: 469.997\n",
      "INFO:tensorflow:loss = 16.176723, step = 701 (0.213 sec)\n",
      "INFO:tensorflow:global_step/sec: 542.576\n",
      "INFO:tensorflow:loss = 7.090806, step = 801 (0.184 sec)\n",
      "INFO:tensorflow:global_step/sec: 540.736\n",
      "INFO:tensorflow:loss = 6.0114694, step = 901 (0.185 sec)\n",
      "INFO:tensorflow:global_step/sec: 566.645\n",
      "INFO:tensorflow:loss = 5.358244, step = 1001 (0.177 sec)\n",
      "INFO:tensorflow:global_step/sec: 534.892\n",
      "INFO:tensorflow:loss = 3.2692354, step = 1101 (0.187 sec)\n",
      "INFO:tensorflow:global_step/sec: 555.532\n",
      "INFO:tensorflow:loss = 3.0086937, step = 1201 (0.180 sec)\n",
      "INFO:tensorflow:global_step/sec: 533.514\n",
      "INFO:tensorflow:loss = 8.29101, step = 1301 (0.187 sec)\n",
      "INFO:tensorflow:global_step/sec: 567.324\n",
      "INFO:tensorflow:loss = 0.9659397, step = 1401 (0.177 sec)\n",
      "INFO:tensorflow:global_step/sec: 522.631\n",
      "INFO:tensorflow:loss = 8.458067, step = 1501 (0.191 sec)\n",
      "INFO:tensorflow:global_step/sec: 570.015\n",
      "INFO:tensorflow:loss = 6.0303555, step = 1601 (0.175 sec)\n",
      "INFO:tensorflow:global_step/sec: 567.522\n",
      "INFO:tensorflow:loss = 1.9793772, step = 1701 (0.176 sec)\n",
      "INFO:tensorflow:global_step/sec: 530.652\n",
      "INFO:tensorflow:loss = 0.96449625, step = 1801 (0.188 sec)\n",
      "INFO:tensorflow:global_step/sec: 576.999\n",
      "INFO:tensorflow:loss = 5.321749, step = 1901 (0.173 sec)\n",
      "INFO:tensorflow:global_step/sec: 547.224\n",
      "INFO:tensorflow:loss = 3.0814586, step = 2001 (0.183 sec)\n",
      "INFO:tensorflow:global_step/sec: 569.76\n",
      "INFO:tensorflow:loss = 3.7164736, step = 2101 (0.176 sec)\n",
      "INFO:tensorflow:global_step/sec: 567.939\n",
      "INFO:tensorflow:loss = 4.883929, step = 2201 (0.176 sec)\n",
      "INFO:tensorflow:global_step/sec: 507.357\n",
      "INFO:tensorflow:loss = 0.23744398, step = 2301 (0.197 sec)\n",
      "INFO:tensorflow:global_step/sec: 507.144\n",
      "INFO:tensorflow:loss = 0.69871193, step = 2401 (0.197 sec)\n",
      "INFO:tensorflow:global_step/sec: 565.449\n",
      "INFO:tensorflow:loss = 1.6435666, step = 2501 (0.177 sec)\n",
      "INFO:tensorflow:global_step/sec: 516.793\n",
      "INFO:tensorflow:loss = 3.2308083, step = 2601 (0.194 sec)\n",
      "INFO:tensorflow:global_step/sec: 487.837\n",
      "INFO:tensorflow:loss = 3.1514878, step = 2701 (0.205 sec)\n",
      "INFO:tensorflow:global_step/sec: 538.172\n",
      "INFO:tensorflow:loss = 2.877276, step = 2801 (0.186 sec)\n",
      "INFO:tensorflow:global_step/sec: 556.85\n",
      "INFO:tensorflow:loss = 5.2355924, step = 2901 (0.179 sec)\n",
      "INFO:tensorflow:global_step/sec: 522.545\n",
      "INFO:tensorflow:loss = 3.2547688, step = 3001 (0.192 sec)\n",
      "INFO:tensorflow:global_step/sec: 547.245\n",
      "INFO:tensorflow:loss = 4.4220386, step = 3101 (0.183 sec)\n",
      "INFO:tensorflow:global_step/sec: 558.688\n",
      "INFO:tensorflow:loss = 4.284453, step = 3201 (0.179 sec)\n",
      "INFO:tensorflow:global_step/sec: 535.234\n",
      "INFO:tensorflow:loss = 0.99215746, step = 3301 (0.187 sec)\n",
      "INFO:tensorflow:global_step/sec: 555.942\n",
      "INFO:tensorflow:loss = 0.850379, step = 3401 (0.180 sec)\n",
      "INFO:tensorflow:global_step/sec: 500.857\n",
      "INFO:tensorflow:loss = 2.3322434, step = 3501 (0.200 sec)\n",
      "INFO:tensorflow:global_step/sec: 533.184\n",
      "INFO:tensorflow:loss = 0.14512159, step = 3601 (0.188 sec)\n",
      "INFO:tensorflow:global_step/sec: 582.852\n",
      "INFO:tensorflow:loss = 2.8052263, step = 3701 (0.172 sec)\n",
      "INFO:tensorflow:global_step/sec: 578.804\n",
      "INFO:tensorflow:loss = 3.5911367, step = 3801 (0.173 sec)\n",
      "INFO:tensorflow:global_step/sec: 506.16\n",
      "INFO:tensorflow:loss = 1.5496864, step = 3901 (0.197 sec)\n",
      "INFO:tensorflow:global_step/sec: 568.5\n",
      "INFO:tensorflow:loss = 1.2705369, step = 4001 (0.176 sec)\n",
      "INFO:tensorflow:global_step/sec: 532.195\n",
      "INFO:tensorflow:loss = 2.8989897, step = 4101 (0.188 sec)\n",
      "INFO:tensorflow:global_step/sec: 549.987\n",
      "INFO:tensorflow:loss = 0.43264794, step = 4201 (0.182 sec)\n",
      "INFO:tensorflow:global_step/sec: 538.191\n",
      "INFO:tensorflow:loss = 0.8565729, step = 4301 (0.186 sec)\n",
      "INFO:tensorflow:global_step/sec: 539.969\n",
      "INFO:tensorflow:loss = 1.8702921, step = 4401 (0.185 sec)\n",
      "INFO:tensorflow:global_step/sec: 555.682\n",
      "INFO:tensorflow:loss = 0.8081864, step = 4501 (0.180 sec)\n",
      "INFO:tensorflow:global_step/sec: 571.317\n",
      "INFO:tensorflow:loss = 0.85746926, step = 4601 (0.175 sec)\n",
      "INFO:tensorflow:global_step/sec: 564.586\n",
      "INFO:tensorflow:loss = 2.4415886, step = 4701 (0.177 sec)\n",
      "INFO:tensorflow:global_step/sec: 552.317\n",
      "INFO:tensorflow:loss = 0.95544356, step = 4801 (0.181 sec)\n",
      "INFO:tensorflow:global_step/sec: 559.994\n",
      "INFO:tensorflow:loss = 0.67740196, step = 4901 (0.179 sec)\n",
      "INFO:tensorflow:global_step/sec: 574.802\n",
      "INFO:tensorflow:loss = 0.6857321, step = 5001 (0.174 sec)\n",
      "INFO:tensorflow:global_step/sec: 572.373\n",
      "INFO:tensorflow:loss = 0.3742535, step = 5101 (0.175 sec)\n",
      "INFO:tensorflow:global_step/sec: 580.907\n",
      "INFO:tensorflow:loss = 1.4654251, step = 5201 (0.172 sec)\n",
      "INFO:tensorflow:global_step/sec: 563.196\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:loss = 2.2117522, step = 5301 (0.178 sec)\n",
      "INFO:tensorflow:global_step/sec: 582.861\n",
      "INFO:tensorflow:loss = 3.696426, step = 5401 (0.172 sec)\n",
      "INFO:tensorflow:global_step/sec: 541.519\n",
      "INFO:tensorflow:loss = 1.3339893, step = 5501 (0.185 sec)\n",
      "INFO:tensorflow:global_step/sec: 556.986\n",
      "INFO:tensorflow:loss = 0.6472928, step = 5601 (0.179 sec)\n",
      "INFO:tensorflow:global_step/sec: 572.359\n",
      "INFO:tensorflow:loss = 1.742102, step = 5701 (0.175 sec)\n",
      "INFO:tensorflow:global_step/sec: 560.263\n",
      "INFO:tensorflow:loss = 0.501868, step = 5801 (0.179 sec)\n",
      "INFO:tensorflow:global_step/sec: 544.08\n",
      "INFO:tensorflow:loss = 1.6975393, step = 5901 (0.184 sec)\n",
      "INFO:tensorflow:global_step/sec: 573.919\n",
      "INFO:tensorflow:loss = 0.30660093, step = 6001 (0.174 sec)\n",
      "INFO:tensorflow:global_step/sec: 584.75\n",
      "INFO:tensorflow:loss = 0.88393766, step = 6101 (0.171 sec)\n",
      "INFO:tensorflow:global_step/sec: 589.588\n",
      "INFO:tensorflow:loss = 0.44216326, step = 6201 (0.170 sec)\n",
      "INFO:tensorflow:global_step/sec: 587.092\n",
      "INFO:tensorflow:loss = 1.4926738, step = 6301 (0.173 sec)\n",
      "INFO:tensorflow:global_step/sec: 557.747\n",
      "INFO:tensorflow:loss = 1.4066219, step = 6401 (0.176 sec)\n",
      "INFO:tensorflow:global_step/sec: 573.112\n",
      "INFO:tensorflow:loss = 0.6162974, step = 6501 (0.180 sec)\n",
      "INFO:tensorflow:global_step/sec: 539.349\n",
      "INFO:tensorflow:loss = 0.099980876, step = 6601 (0.180 sec)\n",
      "INFO:tensorflow:global_step/sec: 553.395\n",
      "INFO:tensorflow:loss = 0.30662355, step = 6701 (0.180 sec)\n",
      "INFO:tensorflow:global_step/sec: 544.257\n",
      "INFO:tensorflow:loss = 0.678537, step = 6801 (0.184 sec)\n",
      "INFO:tensorflow:global_step/sec: 522.288\n",
      "INFO:tensorflow:loss = 0.84153837, step = 6901 (0.192 sec)\n",
      "INFO:tensorflow:global_step/sec: 557.681\n",
      "INFO:tensorflow:loss = 0.048569877, step = 7001 (0.179 sec)\n",
      "INFO:tensorflow:global_step/sec: 556.499\n",
      "INFO:tensorflow:loss = 0.9179066, step = 7101 (0.180 sec)\n",
      "INFO:tensorflow:global_step/sec: 560.042\n",
      "INFO:tensorflow:loss = 0.32493937, step = 7201 (0.178 sec)\n",
      "INFO:tensorflow:global_step/sec: 566.07\n",
      "INFO:tensorflow:loss = 0.5088048, step = 7301 (0.177 sec)\n",
      "INFO:tensorflow:global_step/sec: 555.531\n",
      "INFO:tensorflow:loss = 0.042212833, step = 7401 (0.180 sec)\n",
      "INFO:tensorflow:global_step/sec: 534.688\n",
      "INFO:tensorflow:loss = 1.0725307, step = 7501 (0.187 sec)\n",
      "INFO:tensorflow:global_step/sec: 527.979\n",
      "INFO:tensorflow:loss = 0.39079815, step = 7601 (0.189 sec)\n",
      "INFO:tensorflow:global_step/sec: 554.79\n",
      "INFO:tensorflow:loss = 0.37993446, step = 7701 (0.180 sec)\n",
      "INFO:tensorflow:global_step/sec: 585.273\n",
      "INFO:tensorflow:loss = 0.053365562, step = 7801 (0.171 sec)\n",
      "INFO:tensorflow:global_step/sec: 522.373\n",
      "INFO:tensorflow:loss = 2.1146438, step = 7901 (0.191 sec)\n",
      "INFO:tensorflow:global_step/sec: 541.801\n",
      "INFO:tensorflow:loss = 2.594081, step = 8001 (0.186 sec)\n",
      "INFO:tensorflow:global_step/sec: 541.03\n",
      "INFO:tensorflow:loss = 0.09510458, step = 8101 (0.184 sec)\n",
      "INFO:tensorflow:global_step/sec: 579.733\n",
      "INFO:tensorflow:loss = 1.561128, step = 8201 (0.172 sec)\n",
      "INFO:tensorflow:global_step/sec: 557.858\n",
      "INFO:tensorflow:loss = 0.67071486, step = 8301 (0.179 sec)\n",
      "INFO:tensorflow:global_step/sec: 514.659\n",
      "INFO:tensorflow:loss = 1.66127, step = 8401 (0.194 sec)\n",
      "INFO:tensorflow:global_step/sec: 562.431\n",
      "INFO:tensorflow:loss = 0.6046287, step = 8501 (0.178 sec)\n",
      "INFO:tensorflow:global_step/sec: 510.354\n",
      "INFO:tensorflow:loss = 0.07571233, step = 8601 (0.196 sec)\n",
      "INFO:tensorflow:global_step/sec: 548.878\n",
      "INFO:tensorflow:loss = 0.2349731, step = 8701 (0.182 sec)\n",
      "INFO:tensorflow:global_step/sec: 565.689\n",
      "INFO:tensorflow:loss = 2.2023585, step = 8801 (0.177 sec)\n",
      "INFO:tensorflow:global_step/sec: 530.328\n",
      "INFO:tensorflow:loss = 0.38220543, step = 8901 (0.189 sec)\n",
      "INFO:tensorflow:global_step/sec: 562.358\n",
      "INFO:tensorflow:loss = 0.36914846, step = 9001 (0.178 sec)\n",
      "INFO:tensorflow:global_step/sec: 512.184\n",
      "INFO:tensorflow:loss = 0.5194205, step = 9101 (0.195 sec)\n",
      "INFO:tensorflow:global_step/sec: 523.832\n",
      "INFO:tensorflow:loss = 0.4209218, step = 9201 (0.191 sec)\n",
      "INFO:tensorflow:global_step/sec: 542.804\n",
      "INFO:tensorflow:loss = 0.09953248, step = 9301 (0.184 sec)\n",
      "INFO:tensorflow:global_step/sec: 544.32\n",
      "INFO:tensorflow:loss = 0.1184893, step = 9401 (0.184 sec)\n",
      "INFO:tensorflow:global_step/sec: 520.985\n",
      "INFO:tensorflow:loss = 0.05812273, step = 9501 (0.192 sec)\n",
      "INFO:tensorflow:global_step/sec: 530.232\n",
      "INFO:tensorflow:loss = 0.07530656, step = 9601 (0.190 sec)\n",
      "INFO:tensorflow:global_step/sec: 521.136\n",
      "INFO:tensorflow:loss = 0.44742566, step = 9701 (0.191 sec)\n",
      "INFO:tensorflow:global_step/sec: 572.217\n",
      "INFO:tensorflow:loss = 0.082484074, step = 9801 (0.175 sec)\n",
      "INFO:tensorflow:global_step/sec: 524.194\n",
      "INFO:tensorflow:loss = 0.25080732, step = 9901 (0.191 sec)\n",
      "INFO:tensorflow:global_step/sec: 542.645\n",
      "INFO:tensorflow:loss = 0.08440753, step = 10001 (0.184 sec)\n",
      "INFO:tensorflow:global_step/sec: 560.199\n",
      "INFO:tensorflow:loss = 0.9891646, step = 10101 (0.179 sec)\n",
      "INFO:tensorflow:global_step/sec: 516.885\n",
      "INFO:tensorflow:loss = 4.606642, step = 10201 (0.193 sec)\n",
      "INFO:tensorflow:global_step/sec: 522.503\n",
      "INFO:tensorflow:loss = 0.012688531, step = 10301 (0.191 sec)\n",
      "INFO:tensorflow:global_step/sec: 581.323\n",
      "INFO:tensorflow:loss = 0.013619176, step = 10401 (0.172 sec)\n",
      "INFO:tensorflow:global_step/sec: 580.563\n",
      "INFO:tensorflow:loss = 0.6381511, step = 10501 (0.172 sec)\n",
      "INFO:tensorflow:global_step/sec: 567.726\n",
      "INFO:tensorflow:loss = 0.058108587, step = 10601 (0.176 sec)\n",
      "INFO:tensorflow:global_step/sec: 509.767\n",
      "INFO:tensorflow:loss = 0.35373282, step = 10701 (0.196 sec)\n",
      "INFO:tensorflow:global_step/sec: 580.241\n",
      "INFO:tensorflow:loss = 0.5197712, step = 10801 (0.172 sec)\n",
      "INFO:tensorflow:global_step/sec: 572.133\n",
      "INFO:tensorflow:loss = 0.2736507, step = 10901 (0.175 sec)\n",
      "INFO:tensorflow:global_step/sec: 574.474\n",
      "INFO:tensorflow:loss = 0.30080464, step = 11001 (0.174 sec)\n",
      "INFO:tensorflow:global_step/sec: 555.688\n",
      "INFO:tensorflow:loss = 0.061243724, step = 11101 (0.180 sec)\n",
      "INFO:tensorflow:global_step/sec: 512.196\n",
      "INFO:tensorflow:loss = 0.21038066, step = 11201 (0.195 sec)\n",
      "INFO:tensorflow:global_step/sec: 525.363\n",
      "INFO:tensorflow:loss = 0.53266364, step = 11301 (0.190 sec)\n",
      "INFO:tensorflow:global_step/sec: 532.274\n",
      "INFO:tensorflow:loss = 1.0308418, step = 11401 (0.188 sec)\n",
      "INFO:tensorflow:global_step/sec: 550.086\n",
      "INFO:tensorflow:loss = 0.49383116, step = 11501 (0.182 sec)\n",
      "INFO:tensorflow:global_step/sec: 552.971\n",
      "INFO:tensorflow:loss = 0.11138415, step = 11601 (0.181 sec)\n",
      "INFO:tensorflow:global_step/sec: 509.298\n",
      "INFO:tensorflow:loss = 0.1296062, step = 11701 (0.196 sec)\n",
      "INFO:tensorflow:global_step/sec: 573.402\n",
      "INFO:tensorflow:loss = 0.15011884, step = 11801 (0.174 sec)\n",
      "INFO:tensorflow:global_step/sec: 542.27\n",
      "INFO:tensorflow:loss = 0.27008277, step = 11901 (0.184 sec)\n",
      "INFO:tensorflow:global_step/sec: 524.846\n",
      "INFO:tensorflow:loss = 0.06225844, step = 12001 (0.191 sec)\n",
      "INFO:tensorflow:global_step/sec: 548.171\n",
      "INFO:tensorflow:loss = 1.8348533, step = 12101 (0.182 sec)\n",
      "INFO:tensorflow:global_step/sec: 559.22\n",
      "INFO:tensorflow:loss = 0.05205598, step = 12201 (0.179 sec)\n",
      "INFO:tensorflow:global_step/sec: 546.364\n",
      "INFO:tensorflow:loss = 0.57541645, step = 12301 (0.183 sec)\n",
      "INFO:tensorflow:global_step/sec: 580.001\n",
      "INFO:tensorflow:loss = 0.2991472, step = 12401 (0.172 sec)\n",
      "INFO:tensorflow:global_step/sec: 577.381\n",
      "INFO:tensorflow:loss = 0.09210443, step = 12501 (0.173 sec)\n",
      "INFO:tensorflow:global_step/sec: 572.561\n",
      "INFO:tensorflow:loss = 0.23399316, step = 12601 (0.175 sec)\n",
      "INFO:tensorflow:global_step/sec: 548.644\n",
      "INFO:tensorflow:loss = 0.15280849, step = 12701 (0.182 sec)\n",
      "INFO:tensorflow:global_step/sec: 545.805\n",
      "INFO:tensorflow:loss = 0.06541799, step = 12801 (0.183 sec)\n",
      "INFO:tensorflow:global_step/sec: 553.353\n",
      "INFO:tensorflow:loss = 0.14217947, step = 12901 (0.182 sec)\n",
      "INFO:tensorflow:global_step/sec: 519.781\n",
      "INFO:tensorflow:loss = 0.13706292, step = 13001 (0.191 sec)\n",
      "INFO:tensorflow:global_step/sec: 541.452\n",
      "INFO:tensorflow:loss = 0.2248143, step = 13101 (0.185 sec)\n",
      "INFO:tensorflow:global_step/sec: 520.247\n",
      "INFO:tensorflow:loss = 0.03255721, step = 13201 (0.192 sec)\n",
      "INFO:tensorflow:global_step/sec: 554.524\n",
      "INFO:tensorflow:loss = 0.0017399766, step = 13301 (0.180 sec)\n",
      "INFO:tensorflow:global_step/sec: 527.271\n",
      "INFO:tensorflow:loss = 0.31406155, step = 13401 (0.190 sec)\n",
      "INFO:tensorflow:global_step/sec: 583.138\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:loss = 0.011241686, step = 13501 (0.171 sec)\n",
      "INFO:tensorflow:global_step/sec: 554.975\n",
      "INFO:tensorflow:loss = 0.15866129, step = 13601 (0.180 sec)\n",
      "INFO:tensorflow:global_step/sec: 536.806\n",
      "INFO:tensorflow:loss = 0.10693681, step = 13701 (0.186 sec)\n",
      "INFO:tensorflow:global_step/sec: 570.001\n",
      "INFO:tensorflow:loss = 0.0871465, step = 13801 (0.177 sec)\n",
      "INFO:tensorflow:global_step/sec: 504.71\n",
      "INFO:tensorflow:loss = 0.07074093, step = 13901 (0.197 sec)\n",
      "INFO:tensorflow:global_step/sec: 575.315\n",
      "INFO:tensorflow:loss = 0.22189362, step = 14001 (0.174 sec)\n",
      "INFO:tensorflow:global_step/sec: 549.441\n",
      "INFO:tensorflow:loss = 0.11603419, step = 14101 (0.182 sec)\n",
      "INFO:tensorflow:global_step/sec: 547.624\n",
      "INFO:tensorflow:loss = 0.14664313, step = 14201 (0.183 sec)\n",
      "INFO:tensorflow:global_step/sec: 539.533\n",
      "INFO:tensorflow:loss = 0.42555675, step = 14301 (0.185 sec)\n",
      "INFO:tensorflow:global_step/sec: 527.497\n",
      "INFO:tensorflow:loss = 0.37917304, step = 14401 (0.190 sec)\n",
      "INFO:tensorflow:global_step/sec: 576.774\n",
      "INFO:tensorflow:loss = 0.5171, step = 14501 (0.173 sec)\n",
      "INFO:tensorflow:global_step/sec: 518.837\n",
      "INFO:tensorflow:loss = 0.3486355, step = 14601 (0.193 sec)\n",
      "INFO:tensorflow:global_step/sec: 553.058\n",
      "INFO:tensorflow:loss = 0.13664484, step = 14701 (0.181 sec)\n",
      "INFO:tensorflow:global_step/sec: 570.151\n",
      "INFO:tensorflow:loss = 0.1113739, step = 14801 (0.175 sec)\n",
      "INFO:tensorflow:global_step/sec: 519.464\n",
      "INFO:tensorflow:loss = 0.06201718, step = 14901 (0.192 sec)\n",
      "INFO:tensorflow:global_step/sec: 546.288\n",
      "INFO:tensorflow:loss = 0.106239654, step = 15001 (0.183 sec)\n",
      "INFO:tensorflow:global_step/sec: 571.922\n",
      "INFO:tensorflow:loss = 1.088209, step = 15101 (0.175 sec)\n",
      "INFO:tensorflow:global_step/sec: 583.281\n",
      "INFO:tensorflow:loss = 0.019456884, step = 15201 (0.171 sec)\n",
      "INFO:tensorflow:global_step/sec: 576.624\n",
      "INFO:tensorflow:loss = 0.2174708, step = 15301 (0.174 sec)\n",
      "INFO:tensorflow:global_step/sec: 572.117\n",
      "INFO:tensorflow:loss = 0.08925899, step = 15401 (0.175 sec)\n",
      "INFO:tensorflow:global_step/sec: 514.098\n",
      "INFO:tensorflow:loss = 0.032817803, step = 15501 (0.195 sec)\n",
      "INFO:tensorflow:global_step/sec: 539.578\n",
      "INFO:tensorflow:loss = 0.18805078, step = 15601 (0.185 sec)\n",
      "INFO:tensorflow:global_step/sec: 536.787\n",
      "INFO:tensorflow:loss = 0.026527144, step = 15701 (0.186 sec)\n",
      "INFO:tensorflow:global_step/sec: 543.382\n",
      "INFO:tensorflow:loss = 0.029808998, step = 15801 (0.184 sec)\n",
      "INFO:tensorflow:global_step/sec: 525.825\n",
      "INFO:tensorflow:loss = 1.0226713, step = 15901 (0.190 sec)\n",
      "INFO:tensorflow:global_step/sec: 561.218\n",
      "INFO:tensorflow:loss = 0.12774374, step = 16001 (0.178 sec)\n",
      "INFO:tensorflow:global_step/sec: 524.603\n",
      "INFO:tensorflow:loss = 0.0591494, step = 16101 (0.191 sec)\n",
      "INFO:tensorflow:global_step/sec: 531.935\n",
      "INFO:tensorflow:loss = 0.21184368, step = 16201 (0.188 sec)\n",
      "INFO:tensorflow:global_step/sec: 574.553\n",
      "INFO:tensorflow:loss = 0.10544134, step = 16301 (0.174 sec)\n",
      "INFO:tensorflow:global_step/sec: 574.819\n",
      "INFO:tensorflow:loss = 0.7078202, step = 16401 (0.174 sec)\n",
      "INFO:tensorflow:global_step/sec: 530.327\n",
      "INFO:tensorflow:loss = 0.091435984, step = 16501 (0.189 sec)\n",
      "INFO:tensorflow:global_step/sec: 565.877\n",
      "INFO:tensorflow:loss = 0.042299915, step = 16601 (0.177 sec)\n",
      "INFO:tensorflow:global_step/sec: 551.435\n",
      "INFO:tensorflow:loss = 0.055315495, step = 16701 (0.181 sec)\n",
      "INFO:tensorflow:global_step/sec: 578.877\n",
      "INFO:tensorflow:loss = 0.0063198307, step = 16801 (0.173 sec)\n",
      "INFO:tensorflow:global_step/sec: 574.383\n",
      "INFO:tensorflow:loss = 0.11429474, step = 16901 (0.174 sec)\n",
      "INFO:tensorflow:global_step/sec: 542.293\n",
      "INFO:tensorflow:loss = 0.011431064, step = 17001 (0.184 sec)\n",
      "INFO:tensorflow:global_step/sec: 576.225\n",
      "INFO:tensorflow:loss = 0.046713084, step = 17101 (0.174 sec)\n",
      "INFO:tensorflow:global_step/sec: 522.602\n",
      "INFO:tensorflow:loss = 0.009620156, step = 17201 (0.191 sec)\n",
      "INFO:tensorflow:global_step/sec: 538.774\n",
      "INFO:tensorflow:loss = 0.11900287, step = 17301 (0.186 sec)\n",
      "INFO:tensorflow:global_step/sec: 528.6\n",
      "INFO:tensorflow:loss = 0.07075401, step = 17401 (0.189 sec)\n",
      "INFO:tensorflow:global_step/sec: 547.485\n",
      "INFO:tensorflow:loss = 0.09028783, step = 17501 (0.183 sec)\n",
      "INFO:tensorflow:global_step/sec: 576.401\n",
      "INFO:tensorflow:loss = 0.001982708, step = 17601 (0.174 sec)\n",
      "INFO:tensorflow:global_step/sec: 580.22\n",
      "INFO:tensorflow:loss = 0.10898565, step = 17701 (0.172 sec)\n",
      "INFO:tensorflow:global_step/sec: 521.483\n",
      "INFO:tensorflow:loss = 0.18678357, step = 17801 (0.192 sec)\n",
      "INFO:tensorflow:global_step/sec: 481.735\n",
      "INFO:tensorflow:loss = 0.12262266, step = 17901 (0.207 sec)\n",
      "INFO:tensorflow:global_step/sec: 514.98\n",
      "INFO:tensorflow:loss = 0.094105355, step = 18001 (0.194 sec)\n",
      "INFO:tensorflow:global_step/sec: 580.389\n",
      "INFO:tensorflow:loss = 0.07503551, step = 18101 (0.173 sec)\n",
      "INFO:tensorflow:global_step/sec: 547.095\n",
      "INFO:tensorflow:loss = 0.046804003, step = 18201 (0.183 sec)\n",
      "INFO:tensorflow:global_step/sec: 578.402\n",
      "INFO:tensorflow:loss = 0.008770084, step = 18301 (0.173 sec)\n",
      "INFO:tensorflow:global_step/sec: 559.426\n",
      "INFO:tensorflow:loss = 0.058044855, step = 18401 (0.179 sec)\n",
      "INFO:tensorflow:global_step/sec: 495.634\n",
      "INFO:tensorflow:loss = 0.08846693, step = 18501 (0.202 sec)\n",
      "INFO:tensorflow:global_step/sec: 552.27\n",
      "INFO:tensorflow:loss = 0.031553775, step = 18601 (0.181 sec)\n",
      "INFO:tensorflow:global_step/sec: 533.96\n",
      "INFO:tensorflow:loss = 0.10631254, step = 18701 (0.187 sec)\n",
      "INFO:tensorflow:global_step/sec: 557.611\n",
      "INFO:tensorflow:loss = 0.00194736, step = 18801 (0.179 sec)\n",
      "INFO:tensorflow:global_step/sec: 562.611\n",
      "INFO:tensorflow:loss = 0.010946507, step = 18901 (0.178 sec)\n",
      "INFO:tensorflow:global_step/sec: 547.963\n",
      "INFO:tensorflow:loss = 0.03273103, step = 19001 (0.183 sec)\n",
      "INFO:tensorflow:global_step/sec: 508.351\n",
      "INFO:tensorflow:loss = 0.02912912, step = 19101 (0.197 sec)\n",
      "INFO:tensorflow:global_step/sec: 562.083\n",
      "INFO:tensorflow:loss = 0.017483419, step = 19201 (0.178 sec)\n",
      "INFO:tensorflow:global_step/sec: 579.578\n",
      "INFO:tensorflow:loss = 0.08909572, step = 19301 (0.173 sec)\n",
      "INFO:tensorflow:global_step/sec: 568.024\n",
      "INFO:tensorflow:loss = 0.06861903, step = 19401 (0.176 sec)\n",
      "INFO:tensorflow:global_step/sec: 529.35\n",
      "INFO:tensorflow:loss = 0.07836465, step = 19501 (0.189 sec)\n",
      "INFO:tensorflow:global_step/sec: 536.665\n",
      "INFO:tensorflow:loss = 0.09608982, step = 19601 (0.186 sec)\n",
      "INFO:tensorflow:global_step/sec: 571.57\n",
      "INFO:tensorflow:loss = 0.06135221, step = 19701 (0.175 sec)\n",
      "INFO:tensorflow:global_step/sec: 562.998\n",
      "INFO:tensorflow:loss = 0.050788857, step = 19801 (0.177 sec)\n",
      "INFO:tensorflow:global_step/sec: 552.326\n",
      "INFO:tensorflow:loss = 0.007914622, step = 19901 (0.181 sec)\n",
      "INFO:tensorflow:global_step/sec: 524.976\n",
      "INFO:tensorflow:loss = 0.003741128, step = 20001 (0.191 sec)\n",
      "INFO:tensorflow:global_step/sec: 549.404\n",
      "INFO:tensorflow:loss = 0.08882706, step = 20101 (0.182 sec)\n",
      "INFO:tensorflow:global_step/sec: 562.149\n",
      "INFO:tensorflow:loss = 0.117087275, step = 20201 (0.178 sec)\n",
      "INFO:tensorflow:global_step/sec: 541\n",
      "INFO:tensorflow:loss = 0.6327322, step = 20301 (0.185 sec)\n",
      "INFO:tensorflow:global_step/sec: 545.288\n",
      "INFO:tensorflow:loss = 0.018492205, step = 20401 (0.183 sec)\n",
      "INFO:tensorflow:global_step/sec: 534.688\n",
      "INFO:tensorflow:loss = 0.022824822, step = 20501 (0.187 sec)\n",
      "INFO:tensorflow:global_step/sec: 497.889\n",
      "INFO:tensorflow:loss = 0.3480062, step = 20601 (0.201 sec)\n",
      "INFO:tensorflow:global_step/sec: 544.866\n",
      "INFO:tensorflow:loss = 0.008415331, step = 20701 (0.183 sec)\n",
      "INFO:tensorflow:global_step/sec: 541.398\n",
      "INFO:tensorflow:loss = 0.06143046, step = 20801 (0.185 sec)\n",
      "INFO:tensorflow:global_step/sec: 495.767\n",
      "INFO:tensorflow:loss = 0.0755091, step = 20901 (0.202 sec)\n",
      "INFO:tensorflow:global_step/sec: 536.415\n",
      "INFO:tensorflow:loss = 0.04431589, step = 21001 (0.186 sec)\n",
      "INFO:tensorflow:global_step/sec: 482.33\n",
      "INFO:tensorflow:loss = 0.020321662, step = 21101 (0.207 sec)\n",
      "INFO:tensorflow:global_step/sec: 541.244\n",
      "INFO:tensorflow:loss = 0.035308134, step = 21201 (0.185 sec)\n",
      "INFO:tensorflow:global_step/sec: 556.491\n",
      "INFO:tensorflow:loss = 0.09718782, step = 21301 (0.180 sec)\n",
      "INFO:tensorflow:global_step/sec: 543.984\n",
      "INFO:tensorflow:loss = 0.0144804735, step = 21401 (0.184 sec)\n",
      "INFO:tensorflow:global_step/sec: 552.718\n",
      "INFO:tensorflow:loss = 0.017475953, step = 21501 (0.181 sec)\n",
      "INFO:tensorflow:global_step/sec: 542.681\n",
      "INFO:tensorflow:loss = 0.020672055, step = 21601 (0.185 sec)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:global_step/sec: 527.613\n",
      "INFO:tensorflow:loss = 0.016053233, step = 21701 (0.189 sec)\n",
      "INFO:tensorflow:global_step/sec: 547.151\n",
      "INFO:tensorflow:loss = 0.022209248, step = 21801 (0.183 sec)\n",
      "INFO:tensorflow:global_step/sec: 528.985\n",
      "INFO:tensorflow:loss = 0.028935524, step = 21901 (0.189 sec)\n",
      "INFO:tensorflow:global_step/sec: 547.061\n",
      "INFO:tensorflow:loss = 0.06261704, step = 22001 (0.183 sec)\n",
      "INFO:tensorflow:global_step/sec: 559.372\n",
      "INFO:tensorflow:loss = 0.068284065, step = 22101 (0.179 sec)\n",
      "INFO:tensorflow:global_step/sec: 570.905\n",
      "INFO:tensorflow:loss = 0.14644632, step = 22201 (0.175 sec)\n",
      "INFO:tensorflow:global_step/sec: 566.876\n",
      "INFO:tensorflow:loss = 0.04378733, step = 22301 (0.177 sec)\n",
      "INFO:tensorflow:global_step/sec: 547.441\n",
      "INFO:tensorflow:loss = 0.021930942, step = 22401 (0.183 sec)\n",
      "INFO:tensorflow:global_step/sec: 554.175\n",
      "INFO:tensorflow:loss = 0.025536126, step = 22501 (0.180 sec)\n",
      "INFO:tensorflow:global_step/sec: 570.44\n",
      "INFO:tensorflow:loss = 0.08840186, step = 22601 (0.175 sec)\n",
      "INFO:tensorflow:global_step/sec: 520.922\n",
      "INFO:tensorflow:loss = 0.054924298, step = 22701 (0.192 sec)\n",
      "INFO:tensorflow:global_step/sec: 551.295\n",
      "INFO:tensorflow:loss = 0.04061536, step = 22801 (0.181 sec)\n",
      "INFO:tensorflow:global_step/sec: 559.148\n",
      "INFO:tensorflow:loss = 0.020501666, step = 22901 (0.179 sec)\n",
      "INFO:tensorflow:global_step/sec: 536.529\n",
      "INFO:tensorflow:loss = 0.07039086, step = 23001 (0.187 sec)\n",
      "INFO:tensorflow:global_step/sec: 550.534\n",
      "INFO:tensorflow:loss = 0.0053439504, step = 23101 (0.182 sec)\n",
      "INFO:tensorflow:global_step/sec: 574.23\n",
      "INFO:tensorflow:loss = 0.0038134716, step = 23201 (0.174 sec)\n",
      "INFO:tensorflow:global_step/sec: 542.121\n",
      "INFO:tensorflow:loss = 0.0016957168, step = 23301 (0.185 sec)\n",
      "INFO:tensorflow:global_step/sec: 506.422\n",
      "INFO:tensorflow:loss = 0.015078062, step = 23401 (0.197 sec)\n",
      "INFO:tensorflow:global_step/sec: 570.188\n",
      "INFO:tensorflow:loss = 0.04223028, step = 23501 (0.176 sec)\n",
      "INFO:tensorflow:global_step/sec: 581.652\n",
      "INFO:tensorflow:loss = 0.012350606, step = 23601 (0.172 sec)\n",
      "INFO:tensorflow:global_step/sec: 554.671\n",
      "INFO:tensorflow:loss = 0.024547208, step = 23701 (0.180 sec)\n",
      "INFO:tensorflow:global_step/sec: 539.084\n",
      "INFO:tensorflow:loss = 0.032275073, step = 23801 (0.186 sec)\n",
      "INFO:tensorflow:global_step/sec: 527.022\n",
      "INFO:tensorflow:loss = 0.01018185, step = 23901 (0.190 sec)\n",
      "INFO:tensorflow:global_step/sec: 527.727\n",
      "INFO:tensorflow:loss = 0.20077734, step = 24001 (0.189 sec)\n",
      "INFO:tensorflow:global_step/sec: 569.838\n",
      "INFO:tensorflow:loss = 0.16565718, step = 24101 (0.176 sec)\n",
      "INFO:tensorflow:global_step/sec: 530.536\n",
      "INFO:tensorflow:loss = 0.10936381, step = 24201 (0.189 sec)\n",
      "INFO:tensorflow:global_step/sec: 549.47\n",
      "INFO:tensorflow:loss = 0.0066721803, step = 24301 (0.182 sec)\n",
      "INFO:tensorflow:global_step/sec: 574.046\n",
      "INFO:tensorflow:loss = 0.13768645, step = 24401 (0.174 sec)\n",
      "INFO:tensorflow:global_step/sec: 552.011\n",
      "INFO:tensorflow:loss = 0.04029975, step = 24501 (0.181 sec)\n",
      "INFO:tensorflow:global_step/sec: 526.206\n",
      "INFO:tensorflow:loss = 0.014089955, step = 24601 (0.190 sec)\n",
      "INFO:tensorflow:global_step/sec: 533.582\n",
      "INFO:tensorflow:loss = 0.06257168, step = 24701 (0.187 sec)\n",
      "INFO:tensorflow:global_step/sec: 534.945\n",
      "INFO:tensorflow:loss = 0.020908294, step = 24801 (0.187 sec)\n",
      "INFO:tensorflow:global_step/sec: 539.051\n",
      "INFO:tensorflow:loss = 0.047648206, step = 24901 (0.186 sec)\n",
      "INFO:tensorflow:global_step/sec: 544.808\n",
      "INFO:tensorflow:loss = 0.00054284785, step = 25001 (0.183 sec)\n",
      "INFO:tensorflow:global_step/sec: 554.069\n",
      "INFO:tensorflow:loss = 0.016776541, step = 25101 (0.181 sec)\n",
      "INFO:tensorflow:global_step/sec: 516.626\n",
      "INFO:tensorflow:loss = 0.016490135, step = 25201 (0.194 sec)\n",
      "INFO:tensorflow:global_step/sec: 565.889\n",
      "INFO:tensorflow:loss = 0.05278136, step = 25301 (0.177 sec)\n",
      "INFO:tensorflow:global_step/sec: 577.663\n",
      "INFO:tensorflow:loss = 0.018266264, step = 25401 (0.173 sec)\n",
      "INFO:tensorflow:global_step/sec: 584.948\n",
      "INFO:tensorflow:loss = 0.053608123, step = 25501 (0.171 sec)\n",
      "INFO:tensorflow:global_step/sec: 576.999\n",
      "INFO:tensorflow:loss = 0.083988555, step = 25601 (0.173 sec)\n",
      "INFO:tensorflow:global_step/sec: 543.986\n",
      "INFO:tensorflow:loss = 0.008863233, step = 25701 (0.184 sec)\n",
      "INFO:tensorflow:global_step/sec: 581.882\n",
      "INFO:tensorflow:loss = 0.0111049805, step = 25801 (0.172 sec)\n",
      "INFO:tensorflow:global_step/sec: 561.577\n",
      "INFO:tensorflow:loss = 0.015411706, step = 25901 (0.178 sec)\n",
      "INFO:tensorflow:global_step/sec: 584.834\n",
      "INFO:tensorflow:loss = 0.062174626, step = 26001 (0.171 sec)\n",
      "INFO:tensorflow:global_step/sec: 529.132\n",
      "INFO:tensorflow:loss = 0.050509952, step = 26101 (0.189 sec)\n",
      "INFO:tensorflow:global_step/sec: 535.618\n",
      "INFO:tensorflow:loss = 0.003832923, step = 26201 (0.186 sec)\n",
      "INFO:tensorflow:global_step/sec: 547.101\n",
      "INFO:tensorflow:loss = 0.039984077, step = 26301 (0.183 sec)\n",
      "INFO:tensorflow:global_step/sec: 586.946\n",
      "INFO:tensorflow:loss = 0.0072427224, step = 26401 (0.171 sec)\n",
      "INFO:tensorflow:global_step/sec: 538.088\n",
      "INFO:tensorflow:loss = 0.0077223936, step = 26501 (0.186 sec)\n",
      "INFO:tensorflow:global_step/sec: 568.608\n",
      "INFO:tensorflow:loss = 0.06776569, step = 26601 (0.176 sec)\n",
      "INFO:tensorflow:global_step/sec: 528.767\n",
      "INFO:tensorflow:loss = 0.0153904995, step = 26701 (0.189 sec)\n",
      "INFO:tensorflow:global_step/sec: 522.588\n",
      "INFO:tensorflow:loss = 0.011445919, step = 26801 (0.191 sec)\n",
      "INFO:tensorflow:global_step/sec: 554.135\n",
      "INFO:tensorflow:loss = 0.034391914, step = 26901 (0.180 sec)\n",
      "INFO:tensorflow:global_step/sec: 576.812\n",
      "INFO:tensorflow:loss = 0.053680126, step = 27001 (0.174 sec)\n",
      "INFO:tensorflow:global_step/sec: 553.864\n",
      "INFO:tensorflow:loss = 0.04531908, step = 27101 (0.180 sec)\n",
      "INFO:tensorflow:global_step/sec: 552.101\n",
      "INFO:tensorflow:loss = 0.011397259, step = 27201 (0.181 sec)\n",
      "INFO:tensorflow:global_step/sec: 586.441\n",
      "INFO:tensorflow:loss = 0.16199645, step = 27301 (0.170 sec)\n",
      "INFO:tensorflow:global_step/sec: 562.935\n",
      "INFO:tensorflow:loss = 0.0120202415, step = 27401 (0.178 sec)\n",
      "INFO:tensorflow:global_step/sec: 504.426\n",
      "INFO:tensorflow:loss = 0.05711146, step = 27501 (0.198 sec)\n",
      "INFO:tensorflow:global_step/sec: 521.226\n",
      "INFO:tensorflow:loss = 0.0063743177, step = 27601 (0.192 sec)\n",
      "INFO:tensorflow:global_step/sec: 540.632\n",
      "INFO:tensorflow:loss = 0.005149548, step = 27701 (0.185 sec)\n",
      "INFO:tensorflow:global_step/sec: 575.854\n",
      "INFO:tensorflow:loss = 0.030661171, step = 27801 (0.174 sec)\n",
      "INFO:tensorflow:global_step/sec: 542.873\n",
      "INFO:tensorflow:loss = 0.0055489326, step = 27901 (0.184 sec)\n",
      "INFO:tensorflow:global_step/sec: 520.737\n",
      "INFO:tensorflow:loss = 0.030552872, step = 28001 (0.192 sec)\n",
      "INFO:tensorflow:global_step/sec: 531.773\n",
      "INFO:tensorflow:loss = 0.048291147, step = 28101 (0.188 sec)\n",
      "INFO:tensorflow:global_step/sec: 583.437\n",
      "INFO:tensorflow:loss = 0.028656349, step = 28201 (0.171 sec)\n",
      "INFO:tensorflow:global_step/sec: 539.551\n",
      "INFO:tensorflow:loss = 0.020492617, step = 28301 (0.185 sec)\n",
      "INFO:tensorflow:global_step/sec: 501.137\n",
      "INFO:tensorflow:loss = 0.036006976, step = 28401 (0.200 sec)\n",
      "INFO:tensorflow:global_step/sec: 509.129\n",
      "INFO:tensorflow:loss = 0.039369524, step = 28501 (0.196 sec)\n",
      "INFO:tensorflow:global_step/sec: 562.729\n",
      "INFO:tensorflow:loss = 0.0018159291, step = 28601 (0.178 sec)\n",
      "INFO:tensorflow:global_step/sec: 512.064\n",
      "INFO:tensorflow:loss = 0.0012205822, step = 28701 (0.195 sec)\n",
      "INFO:tensorflow:global_step/sec: 532.469\n",
      "INFO:tensorflow:loss = 0.012099271, step = 28801 (0.188 sec)\n",
      "INFO:tensorflow:global_step/sec: 537.327\n",
      "INFO:tensorflow:loss = 0.012919017, step = 28901 (0.187 sec)\n",
      "INFO:tensorflow:global_step/sec: 533.974\n",
      "INFO:tensorflow:loss = 0.049701054, step = 29001 (0.186 sec)\n",
      "INFO:tensorflow:global_step/sec: 554.251\n",
      "INFO:tensorflow:loss = 0.024024993, step = 29101 (0.181 sec)\n",
      "INFO:tensorflow:global_step/sec: 541.398\n",
      "INFO:tensorflow:loss = 0.047240384, step = 29201 (0.185 sec)\n",
      "INFO:tensorflow:global_step/sec: 574.729\n",
      "INFO:tensorflow:loss = 0.0001422137, step = 29301 (0.174 sec)\n",
      "INFO:tensorflow:global_step/sec: 571.934\n",
      "INFO:tensorflow:loss = 0.036594465, step = 29401 (0.175 sec)\n",
      "INFO:tensorflow:global_step/sec: 549.47\n",
      "INFO:tensorflow:loss = 0.020672482, step = 29501 (0.182 sec)\n",
      "INFO:tensorflow:global_step/sec: 539.151\n",
      "INFO:tensorflow:loss = 0.010345614, step = 29601 (0.186 sec)\n",
      "INFO:tensorflow:global_step/sec: 532.483\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:loss = 0.021201316, step = 29701 (0.187 sec)\n",
      "INFO:tensorflow:global_step/sec: 522.232\n",
      "INFO:tensorflow:loss = 0.0077635464, step = 29801 (0.191 sec)\n",
      "INFO:tensorflow:global_step/sec: 568.277\n",
      "INFO:tensorflow:loss = 0.022789955, step = 29901 (0.176 sec)\n",
      "INFO:tensorflow:global_step/sec: 542.034\n",
      "INFO:tensorflow:loss = 0.001063808, step = 30001 (0.185 sec)\n",
      "INFO:tensorflow:global_step/sec: 535.901\n",
      "INFO:tensorflow:loss = 0.0066127866, step = 30101 (0.186 sec)\n",
      "INFO:tensorflow:global_step/sec: 525.904\n",
      "INFO:tensorflow:loss = 0.081749484, step = 30201 (0.191 sec)\n",
      "INFO:tensorflow:global_step/sec: 557.617\n",
      "INFO:tensorflow:loss = 0.0075727557, step = 30301 (0.179 sec)\n",
      "INFO:tensorflow:global_step/sec: 559.862\n",
      "INFO:tensorflow:loss = 0.010309324, step = 30401 (0.178 sec)\n",
      "INFO:tensorflow:global_step/sec: 569.734\n",
      "INFO:tensorflow:loss = 0.014869677, step = 30501 (0.176 sec)\n",
      "INFO:tensorflow:global_step/sec: 563.707\n",
      "INFO:tensorflow:loss = 0.10320924, step = 30601 (0.177 sec)\n",
      "INFO:tensorflow:global_step/sec: 579.856\n",
      "INFO:tensorflow:loss = 0.06666302, step = 30701 (0.172 sec)\n",
      "INFO:tensorflow:global_step/sec: 530.703\n",
      "INFO:tensorflow:loss = 0.098159865, step = 30801 (0.189 sec)\n",
      "INFO:tensorflow:global_step/sec: 571.45\n",
      "INFO:tensorflow:loss = 0.017723754, step = 30901 (0.176 sec)\n",
      "INFO:tensorflow:global_step/sec: 512.035\n",
      "INFO:tensorflow:loss = 0.03997017, step = 31001 (0.194 sec)\n",
      "INFO:tensorflow:global_step/sec: 563.239\n",
      "INFO:tensorflow:loss = 0.018782405, step = 31101 (0.178 sec)\n",
      "INFO:tensorflow:global_step/sec: 531.837\n",
      "INFO:tensorflow:loss = 0.003263375, step = 31201 (0.188 sec)\n",
      "INFO:tensorflow:global_step/sec: 501.902\n",
      "INFO:tensorflow:loss = 0.010298785, step = 31301 (0.199 sec)\n",
      "INFO:tensorflow:global_step/sec: 560.114\n",
      "INFO:tensorflow:loss = 0.014827792, step = 31401 (0.179 sec)\n",
      "INFO:tensorflow:global_step/sec: 554.147\n",
      "INFO:tensorflow:loss = 0.022893084, step = 31501 (0.181 sec)\n",
      "INFO:tensorflow:global_step/sec: 552.669\n",
      "INFO:tensorflow:loss = 0.036395874, step = 31601 (0.181 sec)\n",
      "INFO:tensorflow:global_step/sec: 576.895\n",
      "INFO:tensorflow:loss = 0.017323071, step = 31701 (0.173 sec)\n",
      "INFO:tensorflow:global_step/sec: 583.446\n",
      "INFO:tensorflow:loss = 0.049461555, step = 31801 (0.171 sec)\n",
      "INFO:tensorflow:global_step/sec: 581.223\n",
      "INFO:tensorflow:loss = 0.022591686, step = 31901 (0.172 sec)\n",
      "INFO:tensorflow:global_step/sec: 512.086\n",
      "INFO:tensorflow:loss = 0.04490753, step = 32001 (0.195 sec)\n",
      "INFO:tensorflow:global_step/sec: 582.076\n",
      "INFO:tensorflow:loss = 0.06807555, step = 32101 (0.172 sec)\n",
      "INFO:tensorflow:global_step/sec: 567.055\n",
      "INFO:tensorflow:loss = 0.017051337, step = 32201 (0.176 sec)\n",
      "INFO:tensorflow:global_step/sec: 567.322\n",
      "INFO:tensorflow:loss = 0.0035310462, step = 32301 (0.176 sec)\n",
      "INFO:tensorflow:global_step/sec: 536.923\n",
      "INFO:tensorflow:loss = 0.024808906, step = 32401 (0.186 sec)\n",
      "INFO:tensorflow:global_step/sec: 579.977\n",
      "INFO:tensorflow:loss = 0.021814056, step = 32501 (0.172 sec)\n",
      "INFO:tensorflow:global_step/sec: 565.741\n",
      "INFO:tensorflow:loss = 0.015415751, step = 32601 (0.177 sec)\n",
      "INFO:tensorflow:global_step/sec: 549.23\n",
      "INFO:tensorflow:loss = 0.10920554, step = 32701 (0.182 sec)\n",
      "INFO:tensorflow:global_step/sec: 524.825\n",
      "INFO:tensorflow:loss = 0.03339508, step = 32801 (0.191 sec)\n",
      "INFO:tensorflow:global_step/sec: 582.662\n",
      "INFO:tensorflow:loss = 0.010500197, step = 32901 (0.172 sec)\n",
      "INFO:tensorflow:global_step/sec: 561.608\n",
      "INFO:tensorflow:loss = 0.03074598, step = 33001 (0.178 sec)\n",
      "INFO:tensorflow:global_step/sec: 539.716\n",
      "INFO:tensorflow:loss = 0.0003077746, step = 33101 (0.185 sec)\n",
      "INFO:tensorflow:global_step/sec: 552.945\n",
      "INFO:tensorflow:loss = 0.022750063, step = 33201 (0.181 sec)\n",
      "INFO:tensorflow:global_step/sec: 565.564\n",
      "INFO:tensorflow:loss = 0.0132475775, step = 33301 (0.177 sec)\n",
      "INFO:tensorflow:global_step/sec: 554.968\n",
      "INFO:tensorflow:loss = 0.009440497, step = 33401 (0.181 sec)\n",
      "INFO:tensorflow:global_step/sec: 535.27\n",
      "INFO:tensorflow:loss = 0.1535424, step = 33501 (0.186 sec)\n",
      "INFO:tensorflow:global_step/sec: 550.464\n",
      "INFO:tensorflow:loss = 0.005948379, step = 33601 (0.182 sec)\n",
      "INFO:tensorflow:global_step/sec: 543.211\n",
      "INFO:tensorflow:loss = 0.0019750833, step = 33701 (0.184 sec)\n",
      "INFO:tensorflow:global_step/sec: 540.84\n",
      "INFO:tensorflow:loss = 0.009366742, step = 33801 (0.185 sec)\n",
      "INFO:tensorflow:global_step/sec: 549.297\n",
      "INFO:tensorflow:loss = 0.019191831, step = 33901 (0.182 sec)\n",
      "INFO:tensorflow:global_step/sec: 530.356\n",
      "INFO:tensorflow:loss = 0.05053223, step = 34001 (0.189 sec)\n",
      "INFO:tensorflow:global_step/sec: 527.292\n",
      "INFO:tensorflow:loss = 0.012844564, step = 34101 (0.190 sec)\n",
      "INFO:tensorflow:global_step/sec: 511.818\n",
      "INFO:tensorflow:loss = 0.037054155, step = 34201 (0.195 sec)\n",
      "INFO:tensorflow:global_step/sec: 536.589\n",
      "INFO:tensorflow:loss = 0.022539515, step = 34301 (0.187 sec)\n",
      "INFO:tensorflow:global_step/sec: 578.302\n",
      "INFO:tensorflow:loss = 0.01174491, step = 34401 (0.173 sec)\n",
      "INFO:tensorflow:global_step/sec: 523.453\n",
      "INFO:tensorflow:loss = 0.028177984, step = 34501 (0.191 sec)\n",
      "INFO:tensorflow:global_step/sec: 543.232\n",
      "INFO:tensorflow:loss = 0.005267547, step = 34601 (0.184 sec)\n",
      "INFO:tensorflow:global_step/sec: 499.025\n",
      "INFO:tensorflow:loss = 0.024686066, step = 34701 (0.200 sec)\n",
      "INFO:tensorflow:global_step/sec: 528.895\n",
      "INFO:tensorflow:loss = 0.027395472, step = 34801 (0.189 sec)\n",
      "INFO:tensorflow:global_step/sec: 564.18\n",
      "INFO:tensorflow:loss = 0.06456257, step = 34901 (0.177 sec)\n",
      "INFO:tensorflow:global_step/sec: 579.037\n",
      "INFO:tensorflow:loss = 0.0018642172, step = 35001 (0.173 sec)\n",
      "INFO:tensorflow:global_step/sec: 542.829\n",
      "INFO:tensorflow:loss = 0.010013457, step = 35101 (0.184 sec)\n",
      "INFO:tensorflow:global_step/sec: 524.474\n",
      "INFO:tensorflow:loss = 0.054200776, step = 35201 (0.191 sec)\n",
      "INFO:tensorflow:global_step/sec: 588.273\n",
      "INFO:tensorflow:loss = 0.009455023, step = 35301 (0.170 sec)\n",
      "INFO:tensorflow:global_step/sec: 503.434\n",
      "INFO:tensorflow:loss = 0.028001023, step = 35401 (0.199 sec)\n",
      "INFO:tensorflow:global_step/sec: 522.784\n",
      "INFO:tensorflow:loss = 0.005491089, step = 35501 (0.191 sec)\n",
      "INFO:tensorflow:global_step/sec: 533.168\n",
      "INFO:tensorflow:loss = 0.013958635, step = 35601 (0.187 sec)\n",
      "INFO:tensorflow:global_step/sec: 550.822\n",
      "INFO:tensorflow:loss = 0.004931296, step = 35701 (0.182 sec)\n",
      "INFO:tensorflow:global_step/sec: 549.072\n",
      "INFO:tensorflow:loss = 0.0066981493, step = 35801 (0.182 sec)\n",
      "INFO:tensorflow:global_step/sec: 523.492\n",
      "INFO:tensorflow:loss = 0.068456076, step = 35901 (0.191 sec)\n",
      "INFO:tensorflow:global_step/sec: 579.32\n",
      "INFO:tensorflow:loss = 0.003665933, step = 36001 (0.173 sec)\n",
      "INFO:tensorflow:global_step/sec: 562.556\n",
      "INFO:tensorflow:loss = 0.07057111, step = 36101 (0.177 sec)\n",
      "INFO:tensorflow:global_step/sec: 579.809\n",
      "INFO:tensorflow:loss = 0.017230734, step = 36201 (0.173 sec)\n",
      "INFO:tensorflow:global_step/sec: 566.645\n",
      "INFO:tensorflow:loss = 0.00053055363, step = 36301 (0.177 sec)\n",
      "INFO:tensorflow:global_step/sec: 521.967\n",
      "INFO:tensorflow:loss = 0.0044449912, step = 36401 (0.191 sec)\n",
      "INFO:tensorflow:global_step/sec: 563.766\n",
      "INFO:tensorflow:loss = 0.012352949, step = 36501 (0.177 sec)\n",
      "INFO:tensorflow:global_step/sec: 576.484\n",
      "INFO:tensorflow:loss = 0.0062317317, step = 36601 (0.175 sec)\n",
      "INFO:tensorflow:global_step/sec: 547.842\n",
      "INFO:tensorflow:loss = 0.0041069584, step = 36701 (0.182 sec)\n",
      "INFO:tensorflow:global_step/sec: 527.732\n",
      "INFO:tensorflow:loss = 0.007429436, step = 36801 (0.189 sec)\n",
      "INFO:tensorflow:global_step/sec: 536.139\n",
      "INFO:tensorflow:loss = 0.007568854, step = 36901 (0.187 sec)\n",
      "INFO:tensorflow:global_step/sec: 551.6\n",
      "INFO:tensorflow:loss = 0.0018288522, step = 37001 (0.181 sec)\n",
      "INFO:tensorflow:global_step/sec: 568.248\n",
      "INFO:tensorflow:loss = 0.0015033914, step = 37101 (0.176 sec)\n",
      "INFO:tensorflow:global_step/sec: 534.508\n",
      "INFO:tensorflow:loss = 0.013135002, step = 37201 (0.187 sec)\n",
      "INFO:tensorflow:global_step/sec: 532.975\n",
      "INFO:tensorflow:loss = 0.014331066, step = 37301 (0.188 sec)\n",
      "INFO:tensorflow:global_step/sec: 536.982\n",
      "INFO:tensorflow:loss = 0.014240445, step = 37401 (0.186 sec)\n",
      "INFO:tensorflow:global_step/sec: 519.287\n",
      "INFO:tensorflow:loss = 0.009031234, step = 37501 (0.193 sec)\n",
      "INFO:tensorflow:global_step/sec: 578.978\n",
      "INFO:tensorflow:loss = 0.020729389, step = 37601 (0.174 sec)\n",
      "INFO:tensorflow:global_step/sec: 508.348\n",
      "INFO:tensorflow:loss = 0.0104787145, step = 37701 (0.196 sec)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:global_step/sec: 547.962\n",
      "INFO:tensorflow:loss = 0.006090905, step = 37801 (0.183 sec)\n",
      "INFO:tensorflow:global_step/sec: 512.349\n",
      "INFO:tensorflow:loss = 0.027446099, step = 37901 (0.195 sec)\n",
      "INFO:tensorflow:global_step/sec: 549.77\n",
      "INFO:tensorflow:loss = 0.010040886, step = 38001 (0.182 sec)\n",
      "INFO:tensorflow:global_step/sec: 565.766\n",
      "INFO:tensorflow:loss = 0.0040907385, step = 38101 (0.177 sec)\n",
      "INFO:tensorflow:global_step/sec: 554.925\n",
      "INFO:tensorflow:loss = 0.0064082625, step = 38201 (0.180 sec)\n",
      "INFO:tensorflow:global_step/sec: 555.812\n",
      "INFO:tensorflow:loss = 0.016764032, step = 38301 (0.180 sec)\n",
      "INFO:tensorflow:global_step/sec: 532.655\n",
      "INFO:tensorflow:loss = 0.015445124, step = 38401 (0.188 sec)\n",
      "INFO:tensorflow:global_step/sec: 571.298\n",
      "INFO:tensorflow:loss = 0.005340596, step = 38501 (0.175 sec)\n",
      "INFO:tensorflow:global_step/sec: 585.672\n",
      "INFO:tensorflow:loss = 0.010536489, step = 38601 (0.171 sec)\n",
      "INFO:tensorflow:global_step/sec: 567.296\n",
      "INFO:tensorflow:loss = 0.00047024354, step = 38701 (0.177 sec)\n",
      "INFO:tensorflow:global_step/sec: 512.224\n",
      "INFO:tensorflow:loss = 0.0044144113, step = 38801 (0.195 sec)\n",
      "INFO:tensorflow:global_step/sec: 582.423\n",
      "INFO:tensorflow:loss = 0.0033733537, step = 38901 (0.172 sec)\n",
      "INFO:tensorflow:global_step/sec: 528.837\n",
      "INFO:tensorflow:loss = 0.012416655, step = 39001 (0.189 sec)\n",
      "INFO:tensorflow:global_step/sec: 512.397\n",
      "INFO:tensorflow:loss = 0.0012967619, step = 39101 (0.195 sec)\n",
      "INFO:tensorflow:global_step/sec: 540.912\n",
      "INFO:tensorflow:loss = 0.035943344, step = 39201 (0.185 sec)\n",
      "INFO:tensorflow:global_step/sec: 516.765\n",
      "INFO:tensorflow:loss = 0.0027524238, step = 39301 (0.193 sec)\n",
      "INFO:tensorflow:global_step/sec: 550.879\n",
      "INFO:tensorflow:loss = 0.0010044426, step = 39401 (0.181 sec)\n",
      "INFO:tensorflow:global_step/sec: 519.646\n",
      "INFO:tensorflow:loss = 0.013848992, step = 39501 (0.193 sec)\n",
      "INFO:tensorflow:global_step/sec: 521.074\n",
      "INFO:tensorflow:loss = 0.042146713, step = 39601 (0.192 sec)\n",
      "INFO:tensorflow:global_step/sec: 564.182\n",
      "INFO:tensorflow:loss = 0.0022740923, step = 39701 (0.177 sec)\n",
      "INFO:tensorflow:global_step/sec: 539.981\n",
      "INFO:tensorflow:loss = 0.007475619, step = 39801 (0.185 sec)\n",
      "INFO:tensorflow:global_step/sec: 539.563\n",
      "INFO:tensorflow:loss = 0.013219342, step = 39901 (0.185 sec)\n",
      "INFO:tensorflow:global_step/sec: 575.774\n",
      "INFO:tensorflow:loss = 0.005512397, step = 40001 (0.174 sec)\n",
      "INFO:tensorflow:global_step/sec: 532.441\n",
      "INFO:tensorflow:loss = 0.027387971, step = 40101 (0.188 sec)\n",
      "INFO:tensorflow:global_step/sec: 530.021\n",
      "INFO:tensorflow:loss = 0.045892507, step = 40201 (0.189 sec)\n",
      "INFO:tensorflow:global_step/sec: 547.763\n",
      "INFO:tensorflow:loss = 0.013453027, step = 40301 (0.182 sec)\n",
      "INFO:tensorflow:global_step/sec: 565.221\n",
      "INFO:tensorflow:loss = 0.013412428, step = 40401 (0.177 sec)\n",
      "INFO:tensorflow:global_step/sec: 533.169\n",
      "INFO:tensorflow:loss = 0.04231118, step = 40501 (0.188 sec)\n",
      "INFO:tensorflow:global_step/sec: 531.827\n",
      "INFO:tensorflow:loss = 0.044293113, step = 40601 (0.188 sec)\n",
      "INFO:tensorflow:global_step/sec: 550.118\n",
      "INFO:tensorflow:loss = 0.0067848177, step = 40701 (0.182 sec)\n",
      "INFO:tensorflow:global_step/sec: 584.693\n",
      "INFO:tensorflow:loss = 0.032841533, step = 40801 (0.171 sec)\n",
      "INFO:tensorflow:global_step/sec: 535.443\n",
      "INFO:tensorflow:loss = 0.00746412, step = 40901 (0.187 sec)\n",
      "INFO:tensorflow:global_step/sec: 567.931\n",
      "INFO:tensorflow:loss = 0.021296762, step = 41001 (0.176 sec)\n",
      "INFO:tensorflow:global_step/sec: 533.462\n",
      "INFO:tensorflow:loss = 0.008851138, step = 41101 (0.187 sec)\n",
      "INFO:tensorflow:global_step/sec: 541.683\n",
      "INFO:tensorflow:loss = 0.005806368, step = 41201 (0.185 sec)\n",
      "INFO:tensorflow:global_step/sec: 515.098\n",
      "INFO:tensorflow:loss = 0.008331387, step = 41301 (0.194 sec)\n",
      "INFO:tensorflow:global_step/sec: 562.414\n",
      "INFO:tensorflow:loss = 0.007950978, step = 41401 (0.178 sec)\n",
      "INFO:tensorflow:global_step/sec: 569.406\n",
      "INFO:tensorflow:loss = 0.08545297, step = 41501 (0.176 sec)\n",
      "INFO:tensorflow:global_step/sec: 520.468\n",
      "INFO:tensorflow:loss = 0.011263645, step = 41601 (0.192 sec)\n",
      "INFO:tensorflow:global_step/sec: 541.947\n",
      "INFO:tensorflow:loss = 0.006076463, step = 41701 (0.185 sec)\n",
      "INFO:tensorflow:global_step/sec: 568.799\n",
      "INFO:tensorflow:loss = 0.009995136, step = 41801 (0.176 sec)\n",
      "INFO:tensorflow:global_step/sec: 499.653\n",
      "INFO:tensorflow:loss = 0.012567114, step = 41901 (0.200 sec)\n",
      "INFO:tensorflow:global_step/sec: 511.896\n",
      "INFO:tensorflow:loss = 0.02049037, step = 42001 (0.195 sec)\n",
      "INFO:tensorflow:global_step/sec: 569.908\n",
      "INFO:tensorflow:loss = 0.010155819, step = 42101 (0.175 sec)\n",
      "INFO:tensorflow:global_step/sec: 569.764\n",
      "INFO:tensorflow:loss = 0.0035195504, step = 42201 (0.176 sec)\n",
      "INFO:tensorflow:global_step/sec: 533.654\n",
      "INFO:tensorflow:loss = 0.0019364416, step = 42301 (0.187 sec)\n",
      "INFO:tensorflow:global_step/sec: 534.942\n",
      "INFO:tensorflow:loss = 0.0045584384, step = 42401 (0.187 sec)\n",
      "INFO:tensorflow:global_step/sec: 539.342\n",
      "INFO:tensorflow:loss = 0.006920579, step = 42501 (0.185 sec)\n",
      "INFO:tensorflow:global_step/sec: 564.25\n",
      "INFO:tensorflow:loss = 0.009331305, step = 42601 (0.177 sec)\n",
      "INFO:tensorflow:global_step/sec: 540.177\n",
      "INFO:tensorflow:loss = 0.0036617266, step = 42701 (0.185 sec)\n",
      "INFO:tensorflow:global_step/sec: 501.728\n",
      "INFO:tensorflow:loss = 0.0027828994, step = 42801 (0.199 sec)\n",
      "INFO:tensorflow:global_step/sec: 545.326\n",
      "INFO:tensorflow:loss = 0.032777697, step = 42901 (0.183 sec)\n",
      "INFO:tensorflow:global_step/sec: 586.393\n",
      "INFO:tensorflow:loss = 0.00010311474, step = 43001 (0.170 sec)\n",
      "INFO:tensorflow:global_step/sec: 549.013\n",
      "INFO:tensorflow:loss = 0.008474188, step = 43101 (0.182 sec)\n",
      "INFO:tensorflow:global_step/sec: 520.239\n",
      "INFO:tensorflow:loss = 0.01432121, step = 43201 (0.192 sec)\n",
      "INFO:tensorflow:global_step/sec: 521.753\n",
      "INFO:tensorflow:loss = 0.0067292107, step = 43301 (0.192 sec)\n",
      "INFO:tensorflow:global_step/sec: 574.87\n",
      "INFO:tensorflow:loss = 0.012926378, step = 43401 (0.174 sec)\n",
      "INFO:tensorflow:global_step/sec: 558.866\n",
      "INFO:tensorflow:loss = 0.0008270074, step = 43501 (0.179 sec)\n",
      "INFO:tensorflow:global_step/sec: 518.361\n",
      "INFO:tensorflow:loss = 0.0009860055, step = 43601 (0.193 sec)\n",
      "INFO:tensorflow:global_step/sec: 570.306\n",
      "INFO:tensorflow:loss = 0.016725576, step = 43701 (0.175 sec)\n",
      "INFO:tensorflow:global_step/sec: 541.511\n",
      "INFO:tensorflow:loss = 0.029001798, step = 43801 (0.185 sec)\n",
      "INFO:tensorflow:global_step/sec: 558.246\n",
      "INFO:tensorflow:loss = 0.0050913715, step = 43901 (0.179 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 44000 into /tmp/tmpojfdm9i4/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 0.007587311.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow_estimator.python.estimator.canned.dnn.DNNClassifier at 0x7fd26e24c978>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feature_cols = [tf.feature_column.numeric_column(\"X\", shape=[28 * 28])]\n",
    "dnn_clf = tf.estimator.DNNClassifier(hidden_units=[300,100], n_classes=10,\n",
    "                                     feature_columns=feature_cols)\n",
    "\n",
    "input_fn = tf.estimator.inputs.numpy_input_fn(\n",
    "    x={\"X\": X_train}, y=y_train, num_epochs=40, batch_size=50, shuffle=True)\n",
    "dnn_clf.train(input_fn=input_fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2019-03-05T07:47:05Z\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "WARNING:tensorflow:From /home/haesun/anaconda3/envs/handson-ml/lib/python3.6/site-packages/tensorflow/python/training/saver.py:1266: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use standard file APIs to check for files with this prefix.\n",
      "INFO:tensorflow:Restoring parameters from /tmp/tmpojfdm9i4/model.ckpt-44000\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2019-03-05-07:47:05\n",
      "INFO:tensorflow:Saving dict for global step 44000: accuracy = 0.9805, average_loss = 0.10441127, global_step = 44000, loss = 13.216616\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 44000: /tmp/tmpojfdm9i4/model.ckpt-44000\n"
     ]
    }
   ],
   "source": [
    "test_input_fn = tf.estimator.inputs.numpy_input_fn(\n",
    "    x={\"X\": X_test}, y=y_test, shuffle=False)\n",
    "eval_results = dnn_clf.evaluate(input_fn=test_input_fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accuracy': 0.9805,\n",
       " 'average_loss': 0.10441127,\n",
       " 'loss': 13.216616,\n",
       " 'global_step': 44000}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eval_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /tmp/tmpojfdm9i4/model.ckpt-44000\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'logits': array([ -1.8939482 ,   1.5261059 ,  -4.122183  ,   0.98258054,\n",
       "         -6.645769  ,  -3.2733636 , -16.164238  ,  24.242487  ,\n",
       "          1.1082871 ,   1.5906262 ], dtype=float32),\n",
       " 'probabilities': array([4.4574934e-12, 1.3627025e-10, 4.8015384e-13, 7.9131764e-11,\n",
       "        3.8494690e-14, 1.1220653e-12, 2.8286756e-18, 1.0000000e+00,\n",
       "        8.9731507e-11, 1.4535223e-10], dtype=float32),\n",
       " 'class_ids': array([7]),\n",
       " 'classes': array([b'7'], dtype=object)}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred_iter = dnn_clf.predict(input_fn=test_input_fn)\n",
    "y_pred = list(y_pred_iter)\n",
    "y_pred[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## `tf.contrib.learn`을 사용"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# from tensorflow.examples.tutorials.mnist import input_data\n",
    "# mnist = input_data.read_data_sets(\"/tmp/data/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# X_train = mnist.train.images\n",
    "# X_test = mnist.test.images\n",
    "# y_train = mnist.train.labels.astype(\"int\")\n",
    "# y_test = mnist.test.labels.astype(\"int\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n",
      "WARNING:tensorflow:From <ipython-input-18-0efa993935b9>:1: RunConfig.__init__ (from tensorflow.contrib.learn.python.learn.estimators.run_config) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "When switching to tf.estimator.Estimator, use tf.estimator.RunConfig instead.\n",
      "WARNING:tensorflow:From <ipython-input-18-0efa993935b9>:3: infer_real_valued_columns_from_input (from tensorflow.contrib.learn.python.learn.estimators.estimator) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please specify feature columns explicitly.\n",
      "WARNING:tensorflow:From /home/haesun/anaconda3/envs/handson-ml/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/estimators/estimator.py:143: setup_train_data_feeder (from tensorflow.contrib.learn.python.learn.learn_io.data_feeder) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tensorflow/transform or tf.data.\n",
      "WARNING:tensorflow:From /home/haesun/anaconda3/envs/handson-ml/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/learn_io/data_feeder.py:100: extract_pandas_data (from tensorflow.contrib.learn.python.learn.learn_io.pandas_io) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please access pandas data directly.\n",
      "WARNING:tensorflow:From /home/haesun/anaconda3/envs/handson-ml/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/learn_io/data_feeder.py:159: DataFeeder.__init__ (from tensorflow.contrib.learn.python.learn.learn_io.data_feeder) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tensorflow/transform or tf.data.\n",
      "WARNING:tensorflow:From /home/haesun/anaconda3/envs/handson-ml/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/learn_io/data_feeder.py:340: check_array (from tensorflow.contrib.learn.python.learn.learn_io.data_feeder) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please convert numpy dtypes explicitly.\n",
      "WARNING:tensorflow:From /home/haesun/anaconda3/envs/handson-ml/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/estimators/estimator.py:183: infer_real_valued_columns_from_input_fn (from tensorflow.contrib.learn.python.learn.estimators.estimator) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please specify feature columns explicitly.\n",
      "WARNING:tensorflow:From /home/haesun/anaconda3/envs/handson-ml/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/estimators/dnn.py:378: multi_class_head (from tensorflow.contrib.learn.python.learn.estimators.head) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please switch to tf.contrib.estimator.*_head.\n",
      "WARNING:tensorflow:From /home/haesun/anaconda3/envs/handson-ml/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/estimators/estimator.py:1179: BaseEstimator.__init__ (from tensorflow.contrib.learn.python.learn.estimators.estimator) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please replace uses of any Estimator from tf.contrib.learn with an Estimator from tf.estimator.*\n",
      "WARNING:tensorflow:Using temporary folder as model directory: /tmp/tmpgxyz4krv\n",
      "INFO:tensorflow:Using config: {'_task_type': None, '_task_id': 0, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7fd25e595198>, '_master': '', '_num_ps_replicas': 0, '_num_worker_replicas': 0, '_environment': 'local', '_is_chief': True, '_evaluation_master': '', '_train_distribute': None, '_eval_distribute': None, '_device_fn': None, '_tf_config': gpu_options {\n",
      "  per_process_gpu_memory_fraction: 1.0\n",
      "}\n",
      ", '_tf_random_seed': 42, '_save_summary_steps': 100, '_save_checkpoints_secs': 600, '_log_step_count_steps': 100, '_protocol': None, '_session_config': None, '_save_checkpoints_steps': None, '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_model_dir': '/tmp/tmpgxyz4krv'}\n",
      "WARNING:tensorflow:From <ipython-input-18-0efa993935b9>:6: SKCompat.__init__ (from tensorflow.contrib.learn.python.learn.estimators.estimator) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please switch to the Estimator interface.\n",
      "WARNING:tensorflow:From /home/haesun/anaconda3/envs/handson-ml/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/learn_io/data_feeder.py:102: extract_pandas_labels (from tensorflow.contrib.learn.python.learn.learn_io.pandas_io) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please access pandas data directly.\n",
      "WARNING:tensorflow:From /home/haesun/anaconda3/envs/handson-ml/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/estimators/head.py:677: ModelFnOps.__new__ (from tensorflow.contrib.learn.python.learn.estimators.model_fn) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "When switching to tf.estimator.Estimator, use tf.estimator.EstimatorSpec. You can use the `estimator_spec` method to create an equivalent one.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 0 into /tmp/tmpgxyz4krv/model.ckpt.\n",
      "INFO:tensorflow:loss = 2.3451102, step = 1\n",
      "INFO:tensorflow:global_step/sec: 492.083\n",
      "INFO:tensorflow:loss = 0.31802297, step = 101 (0.204 sec)\n",
      "INFO:tensorflow:global_step/sec: 548.838\n",
      "INFO:tensorflow:loss = 0.30232447, step = 201 (0.182 sec)\n",
      "INFO:tensorflow:global_step/sec: 551.427\n",
      "INFO:tensorflow:loss = 0.38436472, step = 301 (0.181 sec)\n",
      "INFO:tensorflow:global_step/sec: 551.788\n",
      "INFO:tensorflow:loss = 0.2465452, step = 401 (0.181 sec)\n",
      "INFO:tensorflow:global_step/sec: 589.248\n",
      "INFO:tensorflow:loss = 0.25312555, step = 501 (0.170 sec)\n",
      "INFO:tensorflow:global_step/sec: 573.652\n",
      "INFO:tensorflow:loss = 0.056727532, step = 601 (0.174 sec)\n",
      "INFO:tensorflow:global_step/sec: 535.505\n",
      "INFO:tensorflow:loss = 0.13340972, step = 701 (0.187 sec)\n",
      "INFO:tensorflow:global_step/sec: 607.266\n",
      "INFO:tensorflow:loss = 0.20130588, step = 801 (0.165 sec)\n",
      "INFO:tensorflow:global_step/sec: 562.028\n",
      "INFO:tensorflow:loss = 0.12044342, step = 901 (0.178 sec)\n",
      "INFO:tensorflow:global_step/sec: 589.589\n",
      "INFO:tensorflow:loss = 0.20554487, step = 1001 (0.170 sec)\n",
      "INFO:tensorflow:global_step/sec: 584.346\n",
      "INFO:tensorflow:loss = 0.19413483, step = 1101 (0.171 sec)\n",
      "INFO:tensorflow:global_step/sec: 571.292\n",
      "INFO:tensorflow:loss = 0.15674618, step = 1201 (0.175 sec)\n",
      "INFO:tensorflow:global_step/sec: 582.003\n",
      "INFO:tensorflow:loss = 0.19098362, step = 1301 (0.172 sec)\n",
      "INFO:tensorflow:global_step/sec: 540.3\n",
      "INFO:tensorflow:loss = 0.055449057, step = 1401 (0.185 sec)\n",
      "INFO:tensorflow:global_step/sec: 549.329\n",
      "INFO:tensorflow:loss = 0.09881707, step = 1501 (0.182 sec)\n",
      "INFO:tensorflow:global_step/sec: 569.226\n",
      "INFO:tensorflow:loss = 0.1162917, step = 1601 (0.176 sec)\n",
      "INFO:tensorflow:global_step/sec: 603.679\n",
      "INFO:tensorflow:loss = 0.046873286, step = 1701 (0.166 sec)\n",
      "INFO:tensorflow:global_step/sec: 566.413\n",
      "INFO:tensorflow:loss = 0.12048383, step = 1801 (0.177 sec)\n",
      "INFO:tensorflow:global_step/sec: 550.63\n",
      "INFO:tensorflow:loss = 0.069441885, step = 1901 (0.181 sec)\n",
      "INFO:tensorflow:global_step/sec: 562.451\n",
      "INFO:tensorflow:loss = 0.085839644, step = 2001 (0.178 sec)\n",
      "INFO:tensorflow:global_step/sec: 542.098\n",
      "INFO:tensorflow:loss = 0.033949643, step = 2101 (0.184 sec)\n",
      "INFO:tensorflow:global_step/sec: 587.169\n",
      "INFO:tensorflow:loss = 0.030171711, step = 2201 (0.170 sec)\n",
      "INFO:tensorflow:global_step/sec: 620.304\n",
      "INFO:tensorflow:loss = 0.061152652, step = 2301 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 597.122\n",
      "INFO:tensorflow:loss = 0.039993852, step = 2401 (0.168 sec)\n",
      "INFO:tensorflow:global_step/sec: 588.535\n",
      "INFO:tensorflow:loss = 0.075442255, step = 2501 (0.170 sec)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:global_step/sec: 613.202\n",
      "INFO:tensorflow:loss = 0.05732553, step = 2601 (0.163 sec)\n",
      "INFO:tensorflow:global_step/sec: 579.699\n",
      "INFO:tensorflow:loss = 0.030963503, step = 2701 (0.172 sec)\n",
      "INFO:tensorflow:global_step/sec: 579.885\n",
      "INFO:tensorflow:loss = 0.041068997, step = 2801 (0.172 sec)\n",
      "INFO:tensorflow:global_step/sec: 616.332\n",
      "INFO:tensorflow:loss = 0.072006404, step = 2901 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 598.507\n",
      "INFO:tensorflow:loss = 0.022036154, step = 3001 (0.167 sec)\n",
      "INFO:tensorflow:global_step/sec: 570.647\n",
      "INFO:tensorflow:loss = 0.05341647, step = 3101 (0.175 sec)\n",
      "INFO:tensorflow:global_step/sec: 588.379\n",
      "INFO:tensorflow:loss = 0.017184652, step = 3201 (0.170 sec)\n",
      "INFO:tensorflow:global_step/sec: 612.442\n",
      "INFO:tensorflow:loss = 0.020275978, step = 3301 (0.163 sec)\n",
      "INFO:tensorflow:global_step/sec: 566.463\n",
      "INFO:tensorflow:loss = 0.159838, step = 3401 (0.177 sec)\n",
      "INFO:tensorflow:global_step/sec: 587.199\n",
      "INFO:tensorflow:loss = 0.09738517, step = 3501 (0.170 sec)\n",
      "INFO:tensorflow:global_step/sec: 584.06\n",
      "INFO:tensorflow:loss = 0.14237829, step = 3601 (0.171 sec)\n",
      "INFO:tensorflow:global_step/sec: 612.104\n",
      "INFO:tensorflow:loss = 0.036337003, step = 3701 (0.163 sec)\n",
      "INFO:tensorflow:global_step/sec: 569.587\n",
      "INFO:tensorflow:loss = 0.014561833, step = 3801 (0.176 sec)\n",
      "INFO:tensorflow:global_step/sec: 564.302\n",
      "INFO:tensorflow:loss = 0.057594523, step = 3901 (0.177 sec)\n",
      "INFO:tensorflow:global_step/sec: 557.231\n",
      "INFO:tensorflow:loss = 0.10828104, step = 4001 (0.179 sec)\n",
      "INFO:tensorflow:global_step/sec: 558.003\n",
      "INFO:tensorflow:loss = 0.031443182, step = 4101 (0.179 sec)\n",
      "INFO:tensorflow:global_step/sec: 571.632\n",
      "INFO:tensorflow:loss = 0.042048283, step = 4201 (0.175 sec)\n",
      "INFO:tensorflow:global_step/sec: 557.663\n",
      "INFO:tensorflow:loss = 0.11971554, step = 4301 (0.179 sec)\n",
      "INFO:tensorflow:global_step/sec: 618.754\n",
      "INFO:tensorflow:loss = 0.13275144, step = 4401 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 619.969\n",
      "INFO:tensorflow:loss = 0.027946675, step = 4501 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 619.538\n",
      "INFO:tensorflow:loss = 0.021402497, step = 4601 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 576.311\n",
      "INFO:tensorflow:loss = 0.010770836, step = 4701 (0.173 sec)\n",
      "INFO:tensorflow:global_step/sec: 588.946\n",
      "INFO:tensorflow:loss = 0.029937793, step = 4801 (0.170 sec)\n",
      "INFO:tensorflow:global_step/sec: 554.656\n",
      "INFO:tensorflow:loss = 0.06171802, step = 4901 (0.180 sec)\n",
      "INFO:tensorflow:global_step/sec: 534.011\n",
      "INFO:tensorflow:loss = 0.07065291, step = 5001 (0.187 sec)\n",
      "INFO:tensorflow:global_step/sec: 588.273\n",
      "INFO:tensorflow:loss = 0.009426829, step = 5101 (0.170 sec)\n",
      "INFO:tensorflow:global_step/sec: 577.549\n",
      "INFO:tensorflow:loss = 0.047650453, step = 5201 (0.173 sec)\n",
      "INFO:tensorflow:global_step/sec: 561.435\n",
      "INFO:tensorflow:loss = 0.035001382, step = 5301 (0.178 sec)\n",
      "INFO:tensorflow:global_step/sec: 580.966\n",
      "INFO:tensorflow:loss = 0.02803141, step = 5401 (0.172 sec)\n",
      "INFO:tensorflow:global_step/sec: 600.08\n",
      "INFO:tensorflow:loss = 0.03274735, step = 5501 (0.167 sec)\n",
      "INFO:tensorflow:global_step/sec: 606.303\n",
      "INFO:tensorflow:loss = 0.05977501, step = 5601 (0.165 sec)\n",
      "INFO:tensorflow:global_step/sec: 573.619\n",
      "INFO:tensorflow:loss = 0.011078225, step = 5701 (0.174 sec)\n",
      "INFO:tensorflow:global_step/sec: 618.428\n",
      "INFO:tensorflow:loss = 0.011373464, step = 5801 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 613.541\n",
      "INFO:tensorflow:loss = 0.072194636, step = 5901 (0.163 sec)\n",
      "INFO:tensorflow:global_step/sec: 588.366\n",
      "INFO:tensorflow:loss = 0.08580964, step = 6001 (0.171 sec)\n",
      "INFO:tensorflow:global_step/sec: 585.879\n",
      "INFO:tensorflow:loss = 0.016766634, step = 6101 (0.170 sec)\n",
      "INFO:tensorflow:global_step/sec: 577.96\n",
      "INFO:tensorflow:loss = 0.014873489, step = 6201 (0.173 sec)\n",
      "INFO:tensorflow:global_step/sec: 605.04\n",
      "INFO:tensorflow:loss = 0.045027524, step = 6301 (0.165 sec)\n",
      "INFO:tensorflow:global_step/sec: 588.952\n",
      "INFO:tensorflow:loss = 0.036739245, step = 6401 (0.170 sec)\n",
      "INFO:tensorflow:global_step/sec: 629.081\n",
      "INFO:tensorflow:loss = 0.0196109, step = 6501 (0.159 sec)\n",
      "INFO:tensorflow:global_step/sec: 594.989\n",
      "INFO:tensorflow:loss = 0.012298155, step = 6601 (0.168 sec)\n",
      "INFO:tensorflow:global_step/sec: 589.404\n",
      "INFO:tensorflow:loss = 0.010495988, step = 6701 (0.170 sec)\n",
      "INFO:tensorflow:global_step/sec: 580.961\n",
      "INFO:tensorflow:loss = 0.013333989, step = 6801 (0.172 sec)\n",
      "INFO:tensorflow:global_step/sec: 538.957\n",
      "INFO:tensorflow:loss = 0.008744521, step = 6901 (0.185 sec)\n",
      "INFO:tensorflow:global_step/sec: 618.507\n",
      "INFO:tensorflow:loss = 0.025936853, step = 7001 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 618.054\n",
      "INFO:tensorflow:loss = 0.005803064, step = 7101 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 608.39\n",
      "INFO:tensorflow:loss = 0.042505816, step = 7201 (0.165 sec)\n",
      "INFO:tensorflow:global_step/sec: 562.192\n",
      "INFO:tensorflow:loss = 0.004921202, step = 7301 (0.178 sec)\n",
      "INFO:tensorflow:global_step/sec: 576.096\n",
      "INFO:tensorflow:loss = 0.016862536, step = 7401 (0.174 sec)\n",
      "INFO:tensorflow:global_step/sec: 565.508\n",
      "INFO:tensorflow:loss = 0.005889205, step = 7501 (0.177 sec)\n",
      "INFO:tensorflow:global_step/sec: 603.045\n",
      "INFO:tensorflow:loss = 0.034333896, step = 7601 (0.166 sec)\n",
      "INFO:tensorflow:global_step/sec: 603.486\n",
      "INFO:tensorflow:loss = 0.0077809785, step = 7701 (0.166 sec)\n",
      "INFO:tensorflow:global_step/sec: 615.063\n",
      "INFO:tensorflow:loss = 0.004997916, step = 7801 (0.163 sec)\n",
      "INFO:tensorflow:global_step/sec: 567.656\n",
      "INFO:tensorflow:loss = 0.020938748, step = 7901 (0.176 sec)\n",
      "INFO:tensorflow:global_step/sec: 583.101\n",
      "INFO:tensorflow:loss = 0.005703023, step = 8001 (0.172 sec)\n",
      "INFO:tensorflow:global_step/sec: 584.888\n",
      "INFO:tensorflow:loss = 0.015334785, step = 8101 (0.171 sec)\n",
      "INFO:tensorflow:global_step/sec: 603.16\n",
      "INFO:tensorflow:loss = 0.019969221, step = 8201 (0.166 sec)\n",
      "INFO:tensorflow:global_step/sec: 578.731\n",
      "INFO:tensorflow:loss = 0.045110244, step = 8301 (0.173 sec)\n",
      "INFO:tensorflow:global_step/sec: 576.411\n",
      "INFO:tensorflow:loss = 0.0050535067, step = 8401 (0.174 sec)\n",
      "INFO:tensorflow:global_step/sec: 612.831\n",
      "INFO:tensorflow:loss = 0.007333615, step = 8501 (0.163 sec)\n",
      "INFO:tensorflow:global_step/sec: 608.972\n",
      "INFO:tensorflow:loss = 0.0036406762, step = 8601 (0.164 sec)\n",
      "INFO:tensorflow:global_step/sec: 557.772\n",
      "INFO:tensorflow:loss = 0.001994214, step = 8701 (0.180 sec)\n",
      "INFO:tensorflow:global_step/sec: 576.959\n",
      "INFO:tensorflow:loss = 0.008269597, step = 8801 (0.173 sec)\n",
      "INFO:tensorflow:global_step/sec: 587.816\n",
      "INFO:tensorflow:loss = 0.003621273, step = 8901 (0.170 sec)\n",
      "INFO:tensorflow:global_step/sec: 536.069\n",
      "INFO:tensorflow:loss = 0.0045940853, step = 9001 (0.187 sec)\n",
      "INFO:tensorflow:global_step/sec: 549.008\n",
      "INFO:tensorflow:loss = 0.005486887, step = 9101 (0.182 sec)\n",
      "INFO:tensorflow:global_step/sec: 542.454\n",
      "INFO:tensorflow:loss = 0.006724998, step = 9201 (0.184 sec)\n",
      "INFO:tensorflow:global_step/sec: 606.428\n",
      "INFO:tensorflow:loss = 0.014083808, step = 9301 (0.165 sec)\n",
      "INFO:tensorflow:global_step/sec: 559.246\n",
      "INFO:tensorflow:loss = 0.036959503, step = 9401 (0.179 sec)\n",
      "INFO:tensorflow:global_step/sec: 620.778\n",
      "INFO:tensorflow:loss = 0.0062059658, step = 9501 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 622.896\n",
      "INFO:tensorflow:loss = 0.009496834, step = 9601 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 562.215\n",
      "INFO:tensorflow:loss = 0.008183774, step = 9701 (0.178 sec)\n",
      "INFO:tensorflow:global_step/sec: 538.749\n",
      "INFO:tensorflow:loss = 0.00429501, step = 9801 (0.186 sec)\n",
      "INFO:tensorflow:global_step/sec: 525.39\n",
      "INFO:tensorflow:loss = 0.019238504, step = 9901 (0.190 sec)\n",
      "INFO:tensorflow:global_step/sec: 596.887\n",
      "INFO:tensorflow:loss = 0.0076530827, step = 10001 (0.168 sec)\n",
      "INFO:tensorflow:global_step/sec: 621.385\n",
      "INFO:tensorflow:loss = 0.004769493, step = 10101 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 579.676\n",
      "INFO:tensorflow:loss = 0.0067348373, step = 10201 (0.172 sec)\n",
      "INFO:tensorflow:global_step/sec: 555.509\n",
      "INFO:tensorflow:loss = 0.0044888672, step = 10301 (0.180 sec)\n",
      "INFO:tensorflow:global_step/sec: 564.635\n",
      "INFO:tensorflow:loss = 0.0026622459, step = 10401 (0.177 sec)\n",
      "INFO:tensorflow:global_step/sec: 620.918\n",
      "INFO:tensorflow:loss = 0.002560972, step = 10501 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 566.355\n",
      "INFO:tensorflow:loss = 0.010731762, step = 10601 (0.177 sec)\n",
      "INFO:tensorflow:global_step/sec: 569.153\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:loss = 0.02476872, step = 10701 (0.176 sec)\n",
      "INFO:tensorflow:global_step/sec: 605.303\n",
      "INFO:tensorflow:loss = 0.005142131, step = 10801 (0.165 sec)\n",
      "INFO:tensorflow:global_step/sec: 555.011\n",
      "INFO:tensorflow:loss = 0.0014687899, step = 10901 (0.180 sec)\n",
      "INFO:tensorflow:global_step/sec: 603.142\n",
      "INFO:tensorflow:loss = 0.021500275, step = 11001 (0.166 sec)\n",
      "INFO:tensorflow:global_step/sec: 565.798\n",
      "INFO:tensorflow:loss = 0.0057202443, step = 11101 (0.178 sec)\n",
      "INFO:tensorflow:global_step/sec: 557.872\n",
      "INFO:tensorflow:loss = 0.0013711127, step = 11201 (0.178 sec)\n",
      "INFO:tensorflow:global_step/sec: 554.369\n",
      "INFO:tensorflow:loss = 0.0060190703, step = 11301 (0.180 sec)\n",
      "INFO:tensorflow:global_step/sec: 610.71\n",
      "INFO:tensorflow:loss = 0.0090283295, step = 11401 (0.164 sec)\n",
      "INFO:tensorflow:global_step/sec: 613.598\n",
      "INFO:tensorflow:loss = 0.012596136, step = 11501 (0.163 sec)\n",
      "INFO:tensorflow:global_step/sec: 550.469\n",
      "INFO:tensorflow:loss = 0.00064278586, step = 11601 (0.182 sec)\n",
      "INFO:tensorflow:global_step/sec: 601.298\n",
      "INFO:tensorflow:loss = 0.0032277638, step = 11701 (0.167 sec)\n",
      "INFO:tensorflow:global_step/sec: 600.687\n",
      "INFO:tensorflow:loss = 0.00027793134, step = 11801 (0.166 sec)\n",
      "INFO:tensorflow:global_step/sec: 607.145\n",
      "INFO:tensorflow:loss = 0.0037988091, step = 11901 (0.165 sec)\n",
      "INFO:tensorflow:global_step/sec: 563.581\n",
      "INFO:tensorflow:loss = 0.00030214098, step = 12001 (0.177 sec)\n",
      "INFO:tensorflow:global_step/sec: 592.417\n",
      "INFO:tensorflow:loss = 0.0012906775, step = 12101 (0.169 sec)\n",
      "INFO:tensorflow:global_step/sec: 600.366\n",
      "INFO:tensorflow:loss = 0.0041380553, step = 12201 (0.167 sec)\n",
      "INFO:tensorflow:global_step/sec: 599.099\n",
      "INFO:tensorflow:loss = 0.0039740186, step = 12301 (0.167 sec)\n",
      "INFO:tensorflow:global_step/sec: 615.799\n",
      "INFO:tensorflow:loss = 0.0008845813, step = 12401 (0.163 sec)\n",
      "INFO:tensorflow:global_step/sec: 562.53\n",
      "INFO:tensorflow:loss = 0.006382004, step = 12501 (0.177 sec)\n",
      "INFO:tensorflow:global_step/sec: 611.096\n",
      "INFO:tensorflow:loss = 0.0021671809, step = 12601 (0.164 sec)\n",
      "INFO:tensorflow:global_step/sec: 547.824\n",
      "INFO:tensorflow:loss = 0.002999243, step = 12701 (0.183 sec)\n",
      "INFO:tensorflow:global_step/sec: 570.045\n",
      "INFO:tensorflow:loss = 0.0053828782, step = 12801 (0.174 sec)\n",
      "INFO:tensorflow:global_step/sec: 534.289\n",
      "INFO:tensorflow:loss = 0.0030581472, step = 12901 (0.187 sec)\n",
      "INFO:tensorflow:global_step/sec: 577.816\n",
      "INFO:tensorflow:loss = 0.0032504764, step = 13001 (0.173 sec)\n",
      "INFO:tensorflow:global_step/sec: 580.441\n",
      "INFO:tensorflow:loss = 0.0015980683, step = 13101 (0.172 sec)\n",
      "INFO:tensorflow:global_step/sec: 612.77\n",
      "INFO:tensorflow:loss = 0.004015964, step = 13201 (0.163 sec)\n",
      "INFO:tensorflow:global_step/sec: 580.735\n",
      "INFO:tensorflow:loss = 0.0038952255, step = 13301 (0.172 sec)\n",
      "INFO:tensorflow:global_step/sec: 556.595\n",
      "INFO:tensorflow:loss = 0.0071106986, step = 13401 (0.180 sec)\n",
      "INFO:tensorflow:global_step/sec: 578.827\n",
      "INFO:tensorflow:loss = 0.0056826817, step = 13501 (0.172 sec)\n",
      "INFO:tensorflow:global_step/sec: 592.26\n",
      "INFO:tensorflow:loss = 0.0058757868, step = 13601 (0.169 sec)\n",
      "INFO:tensorflow:global_step/sec: 567.649\n",
      "INFO:tensorflow:loss = 0.0019712867, step = 13701 (0.176 sec)\n",
      "INFO:tensorflow:global_step/sec: 547.201\n",
      "INFO:tensorflow:loss = 0.0029933276, step = 13801 (0.183 sec)\n",
      "INFO:tensorflow:global_step/sec: 619.975\n",
      "INFO:tensorflow:loss = 0.002517118, step = 13901 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 594.389\n",
      "INFO:tensorflow:loss = 0.0020566876, step = 14001 (0.168 sec)\n",
      "INFO:tensorflow:global_step/sec: 601.399\n",
      "INFO:tensorflow:loss = 0.005713878, step = 14101 (0.166 sec)\n",
      "INFO:tensorflow:global_step/sec: 615.628\n",
      "INFO:tensorflow:loss = 0.004194613, step = 14201 (0.163 sec)\n",
      "INFO:tensorflow:global_step/sec: 529.633\n",
      "INFO:tensorflow:loss = 0.00066771626, step = 14301 (0.189 sec)\n",
      "INFO:tensorflow:global_step/sec: 588.809\n",
      "INFO:tensorflow:loss = 0.0007644817, step = 14401 (0.170 sec)\n",
      "INFO:tensorflow:global_step/sec: 604.576\n",
      "INFO:tensorflow:loss = 0.0010867632, step = 14501 (0.165 sec)\n",
      "INFO:tensorflow:global_step/sec: 564.36\n",
      "INFO:tensorflow:loss = 0.0056726024, step = 14601 (0.177 sec)\n",
      "INFO:tensorflow:global_step/sec: 581.413\n",
      "INFO:tensorflow:loss = 0.0004162171, step = 14701 (0.172 sec)\n",
      "INFO:tensorflow:global_step/sec: 553.605\n",
      "INFO:tensorflow:loss = 0.0006648994, step = 14801 (0.181 sec)\n",
      "INFO:tensorflow:global_step/sec: 596.712\n",
      "INFO:tensorflow:loss = 0.0019335021, step = 14901 (0.168 sec)\n",
      "INFO:tensorflow:global_step/sec: 581.687\n",
      "INFO:tensorflow:loss = 0.00064709253, step = 15001 (0.172 sec)\n",
      "INFO:tensorflow:global_step/sec: 588.46\n",
      "INFO:tensorflow:loss = 0.001985025, step = 15101 (0.170 sec)\n",
      "INFO:tensorflow:global_step/sec: 585.244\n",
      "INFO:tensorflow:loss = 0.0023689247, step = 15201 (0.171 sec)\n",
      "INFO:tensorflow:global_step/sec: 585.036\n",
      "INFO:tensorflow:loss = 0.0020909624, step = 15301 (0.171 sec)\n",
      "INFO:tensorflow:global_step/sec: 540.38\n",
      "INFO:tensorflow:loss = 0.0032329368, step = 15401 (0.185 sec)\n",
      "INFO:tensorflow:global_step/sec: 587.736\n",
      "INFO:tensorflow:loss = 0.0045683496, step = 15501 (0.173 sec)\n",
      "INFO:tensorflow:global_step/sec: 528.923\n",
      "INFO:tensorflow:loss = 0.0026687828, step = 15601 (0.186 sec)\n",
      "INFO:tensorflow:global_step/sec: 553.657\n",
      "INFO:tensorflow:loss = 0.0069907536, step = 15701 (0.181 sec)\n",
      "INFO:tensorflow:global_step/sec: 565.784\n",
      "INFO:tensorflow:loss = 0.0020118984, step = 15801 (0.177 sec)\n",
      "INFO:tensorflow:global_step/sec: 600.593\n",
      "INFO:tensorflow:loss = 0.0007071229, step = 15901 (0.166 sec)\n",
      "INFO:tensorflow:global_step/sec: 556.363\n",
      "INFO:tensorflow:loss = 0.0040952037, step = 16001 (0.180 sec)\n",
      "INFO:tensorflow:global_step/sec: 582.518\n",
      "INFO:tensorflow:loss = 0.0034461406, step = 16101 (0.172 sec)\n",
      "INFO:tensorflow:global_step/sec: 580.442\n",
      "INFO:tensorflow:loss = 0.00014968254, step = 16201 (0.172 sec)\n",
      "INFO:tensorflow:global_step/sec: 574.646\n",
      "INFO:tensorflow:loss = 0.0021619012, step = 16301 (0.174 sec)\n",
      "INFO:tensorflow:global_step/sec: 612.848\n",
      "INFO:tensorflow:loss = 0.0012628483, step = 16401 (0.163 sec)\n",
      "INFO:tensorflow:global_step/sec: 573.874\n",
      "INFO:tensorflow:loss = 0.001991824, step = 16501 (0.174 sec)\n",
      "INFO:tensorflow:global_step/sec: 531.659\n",
      "INFO:tensorflow:loss = 0.002245477, step = 16601 (0.188 sec)\n",
      "INFO:tensorflow:global_step/sec: 462.623\n",
      "INFO:tensorflow:loss = 0.0025873198, step = 16701 (0.216 sec)\n",
      "INFO:tensorflow:global_step/sec: 606.165\n",
      "INFO:tensorflow:loss = 0.0019077093, step = 16801 (0.165 sec)\n",
      "INFO:tensorflow:global_step/sec: 577.656\n",
      "INFO:tensorflow:loss = 0.0027277085, step = 16901 (0.173 sec)\n",
      "INFO:tensorflow:global_step/sec: 557.662\n",
      "INFO:tensorflow:loss = 0.0033591322, step = 17001 (0.179 sec)\n",
      "INFO:tensorflow:global_step/sec: 612.466\n",
      "INFO:tensorflow:loss = 0.00081264717, step = 17101 (0.163 sec)\n",
      "INFO:tensorflow:global_step/sec: 549.72\n",
      "INFO:tensorflow:loss = 0.0028142184, step = 17201 (0.182 sec)\n",
      "INFO:tensorflow:global_step/sec: 619.118\n",
      "INFO:tensorflow:loss = 0.0015742044, step = 17301 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 614.853\n",
      "INFO:tensorflow:loss = 0.001303041, step = 17401 (0.164 sec)\n",
      "INFO:tensorflow:global_step/sec: 571.001\n",
      "INFO:tensorflow:loss = 0.0012680978, step = 17501 (0.174 sec)\n",
      "INFO:tensorflow:global_step/sec: 581.156\n",
      "INFO:tensorflow:loss = 0.00044543087, step = 17601 (0.172 sec)\n",
      "INFO:tensorflow:global_step/sec: 583.566\n",
      "INFO:tensorflow:loss = 0.00048143737, step = 17701 (0.171 sec)\n",
      "INFO:tensorflow:global_step/sec: 516.066\n",
      "INFO:tensorflow:loss = 0.000113162714, step = 17801 (0.194 sec)\n",
      "INFO:tensorflow:global_step/sec: 551.238\n",
      "INFO:tensorflow:loss = 0.0022201536, step = 17901 (0.181 sec)\n",
      "INFO:tensorflow:global_step/sec: 557.068\n",
      "INFO:tensorflow:loss = 0.00030681453, step = 18001 (0.180 sec)\n",
      "INFO:tensorflow:global_step/sec: 608.674\n",
      "INFO:tensorflow:loss = 0.00073158706, step = 18101 (0.164 sec)\n",
      "INFO:tensorflow:global_step/sec: 540.467\n",
      "INFO:tensorflow:loss = 0.0037087274, step = 18201 (0.185 sec)\n",
      "INFO:tensorflow:global_step/sec: 618.651\n",
      "INFO:tensorflow:loss = 0.010514365, step = 18301 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 594.421\n",
      "INFO:tensorflow:loss = 0.0027738074, step = 18401 (0.168 sec)\n",
      "INFO:tensorflow:global_step/sec: 600.108\n",
      "INFO:tensorflow:loss = 0.0005838919, step = 18501 (0.167 sec)\n",
      "INFO:tensorflow:global_step/sec: 547.106\n",
      "INFO:tensorflow:loss = 0.001977162, step = 18601 (0.183 sec)\n",
      "INFO:tensorflow:global_step/sec: 597.354\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:loss = 0.00084020704, step = 18701 (0.167 sec)\n",
      "INFO:tensorflow:global_step/sec: 576.933\n",
      "INFO:tensorflow:loss = 0.0021770515, step = 18801 (0.173 sec)\n",
      "INFO:tensorflow:global_step/sec: 555.231\n",
      "INFO:tensorflow:loss = 0.0031298078, step = 18901 (0.180 sec)\n",
      "INFO:tensorflow:global_step/sec: 607.102\n",
      "INFO:tensorflow:loss = 0.0009287165, step = 19001 (0.165 sec)\n",
      "INFO:tensorflow:global_step/sec: 529.179\n",
      "INFO:tensorflow:loss = 0.00043610687, step = 19101 (0.189 sec)\n",
      "INFO:tensorflow:global_step/sec: 630.364\n",
      "INFO:tensorflow:loss = 0.00049805274, step = 19201 (0.159 sec)\n",
      "INFO:tensorflow:global_step/sec: 601.334\n",
      "INFO:tensorflow:loss = 0.0047699637, step = 19301 (0.166 sec)\n",
      "INFO:tensorflow:global_step/sec: 613.145\n",
      "INFO:tensorflow:loss = 0.00069915206, step = 19401 (0.163 sec)\n",
      "INFO:tensorflow:global_step/sec: 603.905\n",
      "INFO:tensorflow:loss = 0.0023377547, step = 19501 (0.166 sec)\n",
      "INFO:tensorflow:global_step/sec: 541.987\n",
      "INFO:tensorflow:loss = 0.0005096203, step = 19601 (0.184 sec)\n",
      "INFO:tensorflow:global_step/sec: 574.996\n",
      "INFO:tensorflow:loss = 0.0002367841, step = 19701 (0.174 sec)\n",
      "INFO:tensorflow:global_step/sec: 573.716\n",
      "INFO:tensorflow:loss = 0.0005644985, step = 19801 (0.174 sec)\n",
      "INFO:tensorflow:global_step/sec: 576.369\n",
      "INFO:tensorflow:loss = 0.0011820027, step = 19901 (0.173 sec)\n",
      "INFO:tensorflow:global_step/sec: 587.91\n",
      "INFO:tensorflow:loss = 0.0015222441, step = 20001 (0.170 sec)\n",
      "INFO:tensorflow:global_step/sec: 582.523\n",
      "INFO:tensorflow:loss = 0.00070636696, step = 20101 (0.172 sec)\n",
      "INFO:tensorflow:global_step/sec: 590.359\n",
      "INFO:tensorflow:loss = 0.0021241598, step = 20201 (0.171 sec)\n",
      "INFO:tensorflow:global_step/sec: 583.196\n",
      "INFO:tensorflow:loss = 0.0015302488, step = 20301 (0.170 sec)\n",
      "INFO:tensorflow:global_step/sec: 544.823\n",
      "INFO:tensorflow:loss = 0.00021270182, step = 20401 (0.184 sec)\n",
      "INFO:tensorflow:global_step/sec: 586.984\n",
      "INFO:tensorflow:loss = 0.0010946243, step = 20501 (0.170 sec)\n",
      "INFO:tensorflow:global_step/sec: 589.415\n",
      "INFO:tensorflow:loss = 0.0012895975, step = 20601 (0.170 sec)\n",
      "INFO:tensorflow:global_step/sec: 612.847\n",
      "INFO:tensorflow:loss = 0.0006077198, step = 20701 (0.163 sec)\n",
      "INFO:tensorflow:global_step/sec: 568.112\n",
      "INFO:tensorflow:loss = 0.0012670392, step = 20801 (0.176 sec)\n",
      "INFO:tensorflow:global_step/sec: 593.911\n",
      "INFO:tensorflow:loss = 0.0008119062, step = 20901 (0.168 sec)\n",
      "INFO:tensorflow:global_step/sec: 598.572\n",
      "INFO:tensorflow:loss = 0.0024994535, step = 21001 (0.167 sec)\n",
      "INFO:tensorflow:global_step/sec: 543.97\n",
      "INFO:tensorflow:loss = 0.001212596, step = 21101 (0.184 sec)\n",
      "INFO:tensorflow:global_step/sec: 572.878\n",
      "INFO:tensorflow:loss = 0.0044880807, step = 21201 (0.175 sec)\n",
      "INFO:tensorflow:global_step/sec: 569.419\n",
      "INFO:tensorflow:loss = 0.0005579956, step = 21301 (0.176 sec)\n",
      "INFO:tensorflow:global_step/sec: 576.432\n",
      "INFO:tensorflow:loss = 0.001666267, step = 21401 (0.173 sec)\n",
      "INFO:tensorflow:global_step/sec: 608.997\n",
      "INFO:tensorflow:loss = 0.0003389172, step = 21501 (0.164 sec)\n",
      "INFO:tensorflow:global_step/sec: 577.587\n",
      "INFO:tensorflow:loss = 0.0006626778, step = 21601 (0.174 sec)\n",
      "INFO:tensorflow:global_step/sec: 574.402\n",
      "INFO:tensorflow:loss = 0.0011631178, step = 21701 (0.173 sec)\n",
      "INFO:tensorflow:global_step/sec: 612.871\n",
      "INFO:tensorflow:loss = 0.00034612181, step = 21801 (0.163 sec)\n",
      "INFO:tensorflow:global_step/sec: 605.966\n",
      "INFO:tensorflow:loss = 0.00080818025, step = 21901 (0.166 sec)\n",
      "INFO:tensorflow:global_step/sec: 577.271\n",
      "INFO:tensorflow:loss = 0.00022657546, step = 22001 (0.172 sec)\n",
      "INFO:tensorflow:global_step/sec: 623.857\n",
      "INFO:tensorflow:loss = 0.00012089937, step = 22101 (0.160 sec)\n",
      "INFO:tensorflow:global_step/sec: 612.942\n",
      "INFO:tensorflow:loss = 0.001126931, step = 22201 (0.163 sec)\n",
      "INFO:tensorflow:global_step/sec: 611.9\n",
      "INFO:tensorflow:loss = 0.0019331959, step = 22301 (0.163 sec)\n",
      "INFO:tensorflow:global_step/sec: 614.261\n",
      "INFO:tensorflow:loss = 0.001663991, step = 22401 (0.163 sec)\n",
      "INFO:tensorflow:global_step/sec: 605.744\n",
      "INFO:tensorflow:loss = 0.001471436, step = 22501 (0.165 sec)\n",
      "INFO:tensorflow:global_step/sec: 609.61\n",
      "INFO:tensorflow:loss = 0.0024152768, step = 22601 (0.164 sec)\n",
      "INFO:tensorflow:global_step/sec: 604.792\n",
      "INFO:tensorflow:loss = 0.001117092, step = 22701 (0.165 sec)\n",
      "INFO:tensorflow:global_step/sec: 559.905\n",
      "INFO:tensorflow:loss = 0.0011551083, step = 22801 (0.179 sec)\n",
      "INFO:tensorflow:global_step/sec: 598.39\n",
      "INFO:tensorflow:loss = 0.0018551925, step = 22901 (0.167 sec)\n",
      "INFO:tensorflow:global_step/sec: 592.198\n",
      "INFO:tensorflow:loss = 0.00072174164, step = 23001 (0.169 sec)\n",
      "INFO:tensorflow:global_step/sec: 609.964\n",
      "INFO:tensorflow:loss = 0.0036146098, step = 23101 (0.164 sec)\n",
      "INFO:tensorflow:global_step/sec: 584.132\n",
      "INFO:tensorflow:loss = 0.0020106062, step = 23201 (0.171 sec)\n",
      "INFO:tensorflow:global_step/sec: 611.048\n",
      "INFO:tensorflow:loss = 0.0014956103, step = 23301 (0.164 sec)\n",
      "INFO:tensorflow:global_step/sec: 619.174\n",
      "INFO:tensorflow:loss = 0.0008220522, step = 23401 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 609.429\n",
      "INFO:tensorflow:loss = 0.00064344076, step = 23501 (0.164 sec)\n",
      "INFO:tensorflow:global_step/sec: 614.133\n",
      "INFO:tensorflow:loss = 0.00045825407, step = 23601 (0.163 sec)\n",
      "INFO:tensorflow:global_step/sec: 575.3\n",
      "INFO:tensorflow:loss = 0.00022047936, step = 23701 (0.174 sec)\n",
      "INFO:tensorflow:global_step/sec: 564.108\n",
      "INFO:tensorflow:loss = 0.0015265867, step = 23801 (0.177 sec)\n",
      "INFO:tensorflow:global_step/sec: 586.258\n",
      "INFO:tensorflow:loss = 0.0011534224, step = 23901 (0.171 sec)\n",
      "INFO:tensorflow:global_step/sec: 605.237\n",
      "INFO:tensorflow:loss = 0.0014475276, step = 24001 (0.166 sec)\n",
      "INFO:tensorflow:global_step/sec: 588.37\n",
      "INFO:tensorflow:loss = 0.0006489145, step = 24101 (0.170 sec)\n",
      "INFO:tensorflow:global_step/sec: 576.579\n",
      "INFO:tensorflow:loss = 0.0014438891, step = 24201 (0.173 sec)\n",
      "INFO:tensorflow:global_step/sec: 582.279\n",
      "INFO:tensorflow:loss = 0.00013492178, step = 24301 (0.172 sec)\n",
      "INFO:tensorflow:global_step/sec: 591.572\n",
      "INFO:tensorflow:loss = 0.0018164318, step = 24401 (0.169 sec)\n",
      "INFO:tensorflow:global_step/sec: 588.075\n",
      "INFO:tensorflow:loss = 0.0007827875, step = 24501 (0.170 sec)\n",
      "INFO:tensorflow:global_step/sec: 557.94\n",
      "INFO:tensorflow:loss = 0.0005252632, step = 24601 (0.179 sec)\n",
      "INFO:tensorflow:global_step/sec: 568.769\n",
      "INFO:tensorflow:loss = 0.00082520884, step = 24701 (0.176 sec)\n",
      "INFO:tensorflow:global_step/sec: 601.958\n",
      "INFO:tensorflow:loss = 0.0010905435, step = 24801 (0.166 sec)\n",
      "INFO:tensorflow:global_step/sec: 580.404\n",
      "INFO:tensorflow:loss = 0.0013812265, step = 24901 (0.172 sec)\n",
      "INFO:tensorflow:global_step/sec: 581.72\n",
      "INFO:tensorflow:loss = 0.000296785, step = 25001 (0.172 sec)\n",
      "INFO:tensorflow:global_step/sec: 532.304\n",
      "INFO:tensorflow:loss = 0.0011830023, step = 25101 (0.188 sec)\n",
      "INFO:tensorflow:global_step/sec: 617.722\n",
      "INFO:tensorflow:loss = 0.0014861291, step = 25201 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 596.738\n",
      "INFO:tensorflow:loss = 0.00016240163, step = 25301 (0.168 sec)\n",
      "INFO:tensorflow:global_step/sec: 559.637\n",
      "INFO:tensorflow:loss = 0.00070019526, step = 25401 (0.179 sec)\n",
      "INFO:tensorflow:global_step/sec: 602.358\n",
      "INFO:tensorflow:loss = 0.0008364168, step = 25501 (0.167 sec)\n",
      "INFO:tensorflow:global_step/sec: 563.528\n",
      "INFO:tensorflow:loss = 0.0007035256, step = 25601 (0.177 sec)\n",
      "INFO:tensorflow:global_step/sec: 574.091\n",
      "INFO:tensorflow:loss = 0.00064905005, step = 25701 (0.175 sec)\n",
      "INFO:tensorflow:global_step/sec: 569.65\n",
      "INFO:tensorflow:loss = 0.0010136625, step = 25801 (0.174 sec)\n",
      "INFO:tensorflow:global_step/sec: 589.046\n",
      "INFO:tensorflow:loss = 0.0016925582, step = 25901 (0.170 sec)\n",
      "INFO:tensorflow:global_step/sec: 609.377\n",
      "INFO:tensorflow:loss = 4.019986e-05, step = 26001 (0.164 sec)\n",
      "INFO:tensorflow:global_step/sec: 599.429\n",
      "INFO:tensorflow:loss = 0.0010253192, step = 26101 (0.167 sec)\n",
      "INFO:tensorflow:global_step/sec: 582.55\n",
      "INFO:tensorflow:loss = 0.0004353946, step = 26201 (0.171 sec)\n",
      "INFO:tensorflow:global_step/sec: 622.232\n",
      "INFO:tensorflow:loss = 0.0012551624, step = 26301 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 558.681\n",
      "INFO:tensorflow:loss = 0.0011457175, step = 26401 (0.179 sec)\n",
      "INFO:tensorflow:global_step/sec: 575.226\n",
      "INFO:tensorflow:loss = 0.00077512173, step = 26501 (0.174 sec)\n",
      "INFO:tensorflow:global_step/sec: 591.649\n",
      "INFO:tensorflow:loss = 0.0006089196, step = 26601 (0.169 sec)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:global_step/sec: 562.834\n",
      "INFO:tensorflow:loss = 2.1493912e-05, step = 26701 (0.178 sec)\n",
      "INFO:tensorflow:global_step/sec: 581.117\n",
      "INFO:tensorflow:loss = 0.00045070797, step = 26801 (0.172 sec)\n",
      "INFO:tensorflow:global_step/sec: 579.912\n",
      "INFO:tensorflow:loss = 0.00093468383, step = 26901 (0.172 sec)\n",
      "INFO:tensorflow:global_step/sec: 596.649\n",
      "INFO:tensorflow:loss = 0.0006957704, step = 27001 (0.169 sec)\n",
      "INFO:tensorflow:global_step/sec: 567.435\n",
      "INFO:tensorflow:loss = 0.00052795606, step = 27101 (0.175 sec)\n",
      "INFO:tensorflow:global_step/sec: 618.532\n",
      "INFO:tensorflow:loss = 0.00044472318, step = 27201 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 620.087\n",
      "INFO:tensorflow:loss = 0.0009544973, step = 27301 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 572.733\n",
      "INFO:tensorflow:loss = 0.00013921122, step = 27401 (0.176 sec)\n",
      "INFO:tensorflow:global_step/sec: 578.235\n",
      "INFO:tensorflow:loss = 0.0014383527, step = 27501 (0.172 sec)\n",
      "INFO:tensorflow:global_step/sec: 625.161\n",
      "INFO:tensorflow:loss = 0.0013214696, step = 27601 (0.160 sec)\n",
      "INFO:tensorflow:global_step/sec: 565.016\n",
      "INFO:tensorflow:loss = 0.00026849075, step = 27701 (0.177 sec)\n",
      "INFO:tensorflow:global_step/sec: 613.106\n",
      "INFO:tensorflow:loss = 0.00035008916, step = 27801 (0.163 sec)\n",
      "INFO:tensorflow:global_step/sec: 597.176\n",
      "INFO:tensorflow:loss = 0.00042687863, step = 27901 (0.168 sec)\n",
      "INFO:tensorflow:global_step/sec: 629.476\n",
      "INFO:tensorflow:loss = 0.0017999372, step = 28001 (0.159 sec)\n",
      "INFO:tensorflow:global_step/sec: 567.065\n",
      "INFO:tensorflow:loss = 0.00041657663, step = 28101 (0.176 sec)\n",
      "INFO:tensorflow:global_step/sec: 576.685\n",
      "INFO:tensorflow:loss = 0.00047744508, step = 28201 (0.173 sec)\n",
      "INFO:tensorflow:global_step/sec: 600.838\n",
      "INFO:tensorflow:loss = 0.0009236564, step = 28301 (0.166 sec)\n",
      "INFO:tensorflow:global_step/sec: 588.246\n",
      "INFO:tensorflow:loss = 0.0009110287, step = 28401 (0.170 sec)\n",
      "INFO:tensorflow:global_step/sec: 526.444\n",
      "INFO:tensorflow:loss = 0.00018912686, step = 28501 (0.190 sec)\n",
      "INFO:tensorflow:global_step/sec: 580.034\n",
      "INFO:tensorflow:loss = 0.00019538727, step = 28601 (0.173 sec)\n",
      "INFO:tensorflow:global_step/sec: 609.796\n",
      "INFO:tensorflow:loss = 0.0010311238, step = 28701 (0.164 sec)\n",
      "INFO:tensorflow:global_step/sec: 629.891\n",
      "INFO:tensorflow:loss = 0.0010478238, step = 28801 (0.159 sec)\n",
      "INFO:tensorflow:global_step/sec: 579.509\n",
      "INFO:tensorflow:loss = 0.00026054308, step = 28901 (0.173 sec)\n",
      "INFO:tensorflow:global_step/sec: 538.373\n",
      "INFO:tensorflow:loss = 0.001478945, step = 29001 (0.186 sec)\n",
      "INFO:tensorflow:global_step/sec: 593.139\n",
      "INFO:tensorflow:loss = 0.0015984685, step = 29101 (0.169 sec)\n",
      "INFO:tensorflow:global_step/sec: 596.163\n",
      "INFO:tensorflow:loss = 0.0012184778, step = 29201 (0.167 sec)\n",
      "INFO:tensorflow:global_step/sec: 547.281\n",
      "INFO:tensorflow:loss = 0.001256704, step = 29301 (0.183 sec)\n",
      "INFO:tensorflow:global_step/sec: 619.531\n",
      "INFO:tensorflow:loss = 0.0007791214, step = 29401 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 584.175\n",
      "INFO:tensorflow:loss = 0.00071598747, step = 29501 (0.171 sec)\n",
      "INFO:tensorflow:global_step/sec: 616.946\n",
      "INFO:tensorflow:loss = 0.00025601368, step = 29601 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 612.323\n",
      "INFO:tensorflow:loss = 0.00016068027, step = 29701 (0.163 sec)\n",
      "INFO:tensorflow:global_step/sec: 563.005\n",
      "INFO:tensorflow:loss = 0.00055066944, step = 29801 (0.178 sec)\n",
      "INFO:tensorflow:global_step/sec: 582.274\n",
      "INFO:tensorflow:loss = 0.0007775384, step = 29901 (0.172 sec)\n",
      "INFO:tensorflow:global_step/sec: 600.885\n",
      "INFO:tensorflow:loss = 4.824771e-05, step = 30001 (0.166 sec)\n",
      "INFO:tensorflow:global_step/sec: 536.742\n",
      "INFO:tensorflow:loss = 0.00079905934, step = 30101 (0.186 sec)\n",
      "INFO:tensorflow:global_step/sec: 573.282\n",
      "INFO:tensorflow:loss = 0.000464034, step = 30201 (0.175 sec)\n",
      "INFO:tensorflow:global_step/sec: 547.319\n",
      "INFO:tensorflow:loss = 0.0010382755, step = 30301 (0.182 sec)\n",
      "INFO:tensorflow:global_step/sec: 620.4\n",
      "INFO:tensorflow:loss = 0.001261039, step = 30401 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 606.614\n",
      "INFO:tensorflow:loss = 0.00069867, step = 30501 (0.165 sec)\n",
      "INFO:tensorflow:global_step/sec: 606.105\n",
      "INFO:tensorflow:loss = 0.0008429569, step = 30601 (0.165 sec)\n",
      "INFO:tensorflow:global_step/sec: 558.439\n",
      "INFO:tensorflow:loss = 0.00054088724, step = 30701 (0.179 sec)\n",
      "INFO:tensorflow:global_step/sec: 615.945\n",
      "INFO:tensorflow:loss = 0.0006515493, step = 30801 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 601.606\n",
      "INFO:tensorflow:loss = 0.0011512174, step = 30901 (0.166 sec)\n",
      "INFO:tensorflow:global_step/sec: 606.441\n",
      "INFO:tensorflow:loss = 0.0011126797, step = 31001 (0.165 sec)\n",
      "INFO:tensorflow:global_step/sec: 581.595\n",
      "INFO:tensorflow:loss = 0.0009732157, step = 31101 (0.172 sec)\n",
      "INFO:tensorflow:global_step/sec: 595.703\n",
      "INFO:tensorflow:loss = 0.00028798505, step = 31201 (0.168 sec)\n",
      "INFO:tensorflow:global_step/sec: 588.536\n",
      "INFO:tensorflow:loss = 0.00073854194, step = 31301 (0.170 sec)\n",
      "INFO:tensorflow:global_step/sec: 571.048\n",
      "INFO:tensorflow:loss = 0.00086828315, step = 31401 (0.175 sec)\n",
      "INFO:tensorflow:global_step/sec: 549.234\n",
      "INFO:tensorflow:loss = 0.00046636164, step = 31501 (0.182 sec)\n",
      "INFO:tensorflow:global_step/sec: 596.682\n",
      "INFO:tensorflow:loss = 0.00024272705, step = 31601 (0.168 sec)\n",
      "INFO:tensorflow:global_step/sec: 604.276\n",
      "INFO:tensorflow:loss = 0.0006858927, step = 31701 (0.165 sec)\n",
      "INFO:tensorflow:global_step/sec: 607.909\n",
      "INFO:tensorflow:loss = 0.00027849182, step = 31801 (0.164 sec)\n",
      "INFO:tensorflow:global_step/sec: 616.76\n",
      "INFO:tensorflow:loss = 0.0008028148, step = 31901 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 567.692\n",
      "INFO:tensorflow:loss = 0.00021827081, step = 32001 (0.176 sec)\n",
      "INFO:tensorflow:global_step/sec: 605.639\n",
      "INFO:tensorflow:loss = 0.00025145116, step = 32101 (0.165 sec)\n",
      "INFO:tensorflow:global_step/sec: 558.086\n",
      "INFO:tensorflow:loss = 0.0010610253, step = 32201 (0.179 sec)\n",
      "INFO:tensorflow:global_step/sec: 549.596\n",
      "INFO:tensorflow:loss = 0.0004342794, step = 32301 (0.183 sec)\n",
      "INFO:tensorflow:global_step/sec: 558.624\n",
      "INFO:tensorflow:loss = 0.00012760163, step = 32401 (0.178 sec)\n",
      "INFO:tensorflow:global_step/sec: 563.098\n",
      "INFO:tensorflow:loss = 0.00048173414, step = 32501 (0.178 sec)\n",
      "INFO:tensorflow:global_step/sec: 581.996\n",
      "INFO:tensorflow:loss = 8.136925e-05, step = 32601 (0.172 sec)\n",
      "INFO:tensorflow:global_step/sec: 578.376\n",
      "INFO:tensorflow:loss = 0.0007714884, step = 32701 (0.173 sec)\n",
      "INFO:tensorflow:global_step/sec: 622.035\n",
      "INFO:tensorflow:loss = 0.0011828557, step = 32801 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 553.014\n",
      "INFO:tensorflow:loss = 0.00059744803, step = 32901 (0.181 sec)\n",
      "INFO:tensorflow:global_step/sec: 556.893\n",
      "INFO:tensorflow:loss = 0.00026821953, step = 33001 (0.180 sec)\n",
      "INFO:tensorflow:global_step/sec: 563.61\n",
      "INFO:tensorflow:loss = 0.0018447504, step = 33101 (0.177 sec)\n",
      "INFO:tensorflow:global_step/sec: 562.931\n",
      "INFO:tensorflow:loss = 0.0006075809, step = 33201 (0.178 sec)\n",
      "INFO:tensorflow:global_step/sec: 582.692\n",
      "INFO:tensorflow:loss = 0.00017816921, step = 33301 (0.172 sec)\n",
      "INFO:tensorflow:global_step/sec: 544.115\n",
      "INFO:tensorflow:loss = 0.0010752638, step = 33401 (0.184 sec)\n",
      "INFO:tensorflow:global_step/sec: 594.049\n",
      "INFO:tensorflow:loss = 0.00016672479, step = 33501 (0.168 sec)\n",
      "INFO:tensorflow:global_step/sec: 564.132\n",
      "INFO:tensorflow:loss = 0.0006773797, step = 33601 (0.177 sec)\n",
      "INFO:tensorflow:global_step/sec: 534.434\n",
      "INFO:tensorflow:loss = 0.0007475652, step = 33701 (0.187 sec)\n",
      "INFO:tensorflow:global_step/sec: 584.275\n",
      "INFO:tensorflow:loss = 0.00081024895, step = 33801 (0.171 sec)\n",
      "INFO:tensorflow:global_step/sec: 586.596\n",
      "INFO:tensorflow:loss = 0.0007606104, step = 33901 (0.170 sec)\n",
      "INFO:tensorflow:global_step/sec: 541.474\n",
      "INFO:tensorflow:loss = 0.0007453459, step = 34001 (0.185 sec)\n",
      "INFO:tensorflow:global_step/sec: 580.558\n",
      "INFO:tensorflow:loss = 0.00056487654, step = 34101 (0.172 sec)\n",
      "INFO:tensorflow:global_step/sec: 619.68\n",
      "INFO:tensorflow:loss = 0.0009299599, step = 34201 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 579.864\n",
      "INFO:tensorflow:loss = 0.00021587823, step = 34301 (0.172 sec)\n",
      "INFO:tensorflow:global_step/sec: 552.252\n",
      "INFO:tensorflow:loss = 0.00040929543, step = 34401 (0.181 sec)\n",
      "INFO:tensorflow:global_step/sec: 580.58\n",
      "INFO:tensorflow:loss = 0.0005763147, step = 34501 (0.172 sec)\n",
      "INFO:tensorflow:global_step/sec: 620.445\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:loss = 0.0004739043, step = 34601 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 543.304\n",
      "INFO:tensorflow:loss = 0.0015119809, step = 34701 (0.184 sec)\n",
      "INFO:tensorflow:global_step/sec: 597.189\n",
      "INFO:tensorflow:loss = 0.000674001, step = 34801 (0.167 sec)\n",
      "INFO:tensorflow:global_step/sec: 581.494\n",
      "INFO:tensorflow:loss = 0.00034475912, step = 34901 (0.172 sec)\n",
      "INFO:tensorflow:global_step/sec: 569.123\n",
      "INFO:tensorflow:loss = 0.00055940147, step = 35001 (0.176 sec)\n",
      "INFO:tensorflow:global_step/sec: 586.16\n",
      "INFO:tensorflow:loss = 0.00087014853, step = 35101 (0.171 sec)\n",
      "INFO:tensorflow:global_step/sec: 549.67\n",
      "INFO:tensorflow:loss = 0.00023894513, step = 35201 (0.182 sec)\n",
      "INFO:tensorflow:global_step/sec: 572.214\n",
      "INFO:tensorflow:loss = 0.000138041, step = 35301 (0.175 sec)\n",
      "INFO:tensorflow:global_step/sec: 556.875\n",
      "INFO:tensorflow:loss = 0.0011388202, step = 35401 (0.181 sec)\n",
      "INFO:tensorflow:global_step/sec: 582.317\n",
      "INFO:tensorflow:loss = 0.00018684927, step = 35501 (0.171 sec)\n",
      "INFO:tensorflow:global_step/sec: 608.476\n",
      "INFO:tensorflow:loss = 0.00018745154, step = 35601 (0.164 sec)\n",
      "INFO:tensorflow:global_step/sec: 575.008\n",
      "INFO:tensorflow:loss = 0.0006908841, step = 35701 (0.174 sec)\n",
      "INFO:tensorflow:global_step/sec: 559.336\n",
      "INFO:tensorflow:loss = 0.00070670916, step = 35801 (0.179 sec)\n",
      "INFO:tensorflow:global_step/sec: 617.563\n",
      "INFO:tensorflow:loss = 0.00054090185, step = 35901 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 606.564\n",
      "INFO:tensorflow:loss = 0.0003731058, step = 36001 (0.165 sec)\n",
      "INFO:tensorflow:global_step/sec: 616.641\n",
      "INFO:tensorflow:loss = 0.0011918206, step = 36101 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 619.707\n",
      "INFO:tensorflow:loss = 0.00064709573, step = 36201 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 575.716\n",
      "INFO:tensorflow:loss = 0.00025932075, step = 36301 (0.175 sec)\n",
      "INFO:tensorflow:global_step/sec: 546.966\n",
      "INFO:tensorflow:loss = 0.0007245174, step = 36401 (0.182 sec)\n",
      "INFO:tensorflow:global_step/sec: 600.053\n",
      "INFO:tensorflow:loss = 0.00018499527, step = 36501 (0.168 sec)\n",
      "INFO:tensorflow:global_step/sec: 573.538\n",
      "INFO:tensorflow:loss = 0.00025028634, step = 36601 (0.174 sec)\n",
      "INFO:tensorflow:global_step/sec: 575.703\n",
      "INFO:tensorflow:loss = 0.00044182275, step = 36701 (0.173 sec)\n",
      "INFO:tensorflow:global_step/sec: 619.262\n",
      "INFO:tensorflow:loss = 0.0008609672, step = 36801 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 580.882\n",
      "INFO:tensorflow:loss = 0.0005228994, step = 36901 (0.172 sec)\n",
      "INFO:tensorflow:global_step/sec: 571.551\n",
      "INFO:tensorflow:loss = 0.000739621, step = 37001 (0.175 sec)\n",
      "INFO:tensorflow:global_step/sec: 574.717\n",
      "INFO:tensorflow:loss = 0.00022675189, step = 37101 (0.174 sec)\n",
      "INFO:tensorflow:global_step/sec: 567.49\n",
      "INFO:tensorflow:loss = 0.00021757817, step = 37201 (0.176 sec)\n",
      "INFO:tensorflow:global_step/sec: 597.996\n",
      "INFO:tensorflow:loss = 0.00048570606, step = 37301 (0.167 sec)\n",
      "INFO:tensorflow:global_step/sec: 585.657\n",
      "INFO:tensorflow:loss = 0.0003734218, step = 37401 (0.172 sec)\n",
      "INFO:tensorflow:global_step/sec: 593.716\n",
      "INFO:tensorflow:loss = 0.00033477123, step = 37501 (0.167 sec)\n",
      "INFO:tensorflow:global_step/sec: 556.965\n",
      "INFO:tensorflow:loss = 0.00023778708, step = 37601 (0.180 sec)\n",
      "INFO:tensorflow:global_step/sec: 555.381\n",
      "INFO:tensorflow:loss = 7.513482e-05, step = 37701 (0.180 sec)\n",
      "INFO:tensorflow:global_step/sec: 552.038\n",
      "INFO:tensorflow:loss = 0.00058054813, step = 37801 (0.181 sec)\n",
      "INFO:tensorflow:global_step/sec: 608.115\n",
      "INFO:tensorflow:loss = 0.0012024788, step = 37901 (0.164 sec)\n",
      "INFO:tensorflow:global_step/sec: 617.472\n",
      "INFO:tensorflow:loss = 0.00015282346, step = 38001 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 564.76\n",
      "INFO:tensorflow:loss = 0.001171692, step = 38101 (0.177 sec)\n",
      "INFO:tensorflow:global_step/sec: 558.202\n",
      "INFO:tensorflow:loss = 0.0005760422, step = 38201 (0.179 sec)\n",
      "INFO:tensorflow:global_step/sec: 562.643\n",
      "INFO:tensorflow:loss = 0.00012748956, step = 38301 (0.178 sec)\n",
      "INFO:tensorflow:global_step/sec: 532.459\n",
      "INFO:tensorflow:loss = 0.00020312564, step = 38401 (0.188 sec)\n",
      "INFO:tensorflow:global_step/sec: 585.301\n",
      "INFO:tensorflow:loss = 0.00043357856, step = 38501 (0.171 sec)\n",
      "INFO:tensorflow:global_step/sec: 614.041\n",
      "INFO:tensorflow:loss = 0.0007085696, step = 38601 (0.163 sec)\n",
      "INFO:tensorflow:global_step/sec: 606.99\n",
      "INFO:tensorflow:loss = 0.0005537599, step = 38701 (0.165 sec)\n",
      "INFO:tensorflow:global_step/sec: 555.094\n",
      "INFO:tensorflow:loss = 5.7141384e-05, step = 38801 (0.180 sec)\n",
      "INFO:tensorflow:global_step/sec: 581.784\n",
      "INFO:tensorflow:loss = 0.00131531, step = 38901 (0.172 sec)\n",
      "INFO:tensorflow:global_step/sec: 622.613\n",
      "INFO:tensorflow:loss = 8.0722086e-05, step = 39001 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 622.775\n",
      "INFO:tensorflow:loss = 0.00080231635, step = 39101 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 578.969\n",
      "INFO:tensorflow:loss = 0.00035499, step = 39201 (0.173 sec)\n",
      "INFO:tensorflow:global_step/sec: 584.063\n",
      "INFO:tensorflow:loss = 0.00020848228, step = 39301 (0.171 sec)\n",
      "INFO:tensorflow:global_step/sec: 614.922\n",
      "INFO:tensorflow:loss = 0.00030716695, step = 39401 (0.163 sec)\n",
      "INFO:tensorflow:global_step/sec: 580.459\n",
      "INFO:tensorflow:loss = 0.00016974908, step = 39501 (0.172 sec)\n",
      "INFO:tensorflow:global_step/sec: 574.219\n",
      "INFO:tensorflow:loss = 0.00039360396, step = 39601 (0.174 sec)\n",
      "INFO:tensorflow:global_step/sec: 604.972\n",
      "INFO:tensorflow:loss = 0.00011240821, step = 39701 (0.165 sec)\n",
      "INFO:tensorflow:global_step/sec: 571.511\n",
      "INFO:tensorflow:loss = 0.0012242816, step = 39801 (0.175 sec)\n",
      "INFO:tensorflow:global_step/sec: 611.909\n",
      "INFO:tensorflow:loss = 0.0013449484, step = 39901 (0.163 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 40000 into /tmp/tmpgxyz4krv/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 0.0004099389.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "SKCompat()"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "config = tf.contrib.learn.RunConfig(tf_random_seed=42) # 책에는 없음\n",
    "\n",
    "feature_cols = tf.contrib.learn.infer_real_valued_columns_from_input(X_train)\n",
    "dnn_clf = tf.contrib.learn.DNNClassifier(hidden_units=[300,100], n_classes=10,\n",
    "                                         feature_columns=feature_cols, config=config)\n",
    "dnn_clf = tf.contrib.learn.SKCompat(dnn_clf) # if TensorFlow >= 1.1\n",
    "tf.logging.set_verbosity(tf.logging.INFO)\n",
    "dnn_clf.fit(X_train, y_train, batch_size=50, steps=40000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /tmp/tmpgxyz4krv/model.ckpt-40000\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.9817"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "y_pred = dnn_clf.predict(X_test)\n",
    "accuracy_score(y_test, y_pred['classes'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.07212044412362126"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import log_loss\n",
    "\n",
    "y_pred_proba = y_pred['probabilities']\n",
    "log_loss(y_test, y_pred_proba)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## 저수준의 TensorFlow API 사용"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "n_inputs = 28*28  # MNIST\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 100\n",
    "n_outputs = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int32, shape=(None), name=\"y\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def neuron_layer(X, n_neurons, name, activation=None):\n",
    "    with tf.name_scope(name):\n",
    "        n_inputs = int(X.get_shape()[1])\n",
    "        stddev = 2 / np.sqrt(n_inputs)\n",
    "        init = tf.truncated_normal((n_inputs, n_neurons), stddev=stddev)\n",
    "        W = tf.Variable(init, name=\"kernel\")\n",
    "        b = tf.Variable(tf.zeros([n_neurons]), name=\"bias\")\n",
    "        Z = tf.matmul(X, W) + b\n",
    "        if activation is not None:\n",
    "            return activation(Z)\n",
    "        else:\n",
    "            return Z"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"dnn\"):\n",
    "    hidden1 = neuron_layer(X, n_hidden1, name=\"hidden1\",\n",
    "                           activation=tf.nn.relu)\n",
    "    hidden2 = neuron_layer(hidden1, n_hidden2, name=\"hidden2\",\n",
    "                           activation=tf.nn.relu)\n",
    "    logits = neuron_layer(hidden2, n_outputs, name=\"outputs\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y,\n",
    "                                                              logits=logits)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = 0.01\n",
    "\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "    training_op = optimizer.minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs = 40\n",
    "batch_size = 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def shuffle_batch(X, y, batch_size):\n",
    "    rnd_idx = np.random.permutation(len(X))\n",
    "    n_batches = len(X) // batch_size\n",
    "    for batch_idx in np.array_split(rnd_idx, n_batches):\n",
    "        X_batch, y_batch = X[batch_idx], y[batch_idx]\n",
    "        yield X_batch, y_batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 배치 데이터 정확도: 0.9 검증 세트 정확도: 0.9146\n",
      "1 배치 데이터 정확도: 0.92 검증 세트 정확도: 0.936\n",
      "2 배치 데이터 정확도: 0.96 검증 세트 정확도: 0.9448\n",
      "3 배치 데이터 정확도: 0.92 검증 세트 정확도: 0.951\n",
      "4 배치 데이터 정확도: 0.98 검증 세트 정확도: 0.9558\n",
      "5 배치 데이터 정확도: 0.96 검증 세트 정확도: 0.9566\n",
      "6 배치 데이터 정확도: 1.0 검증 세트 정확도: 0.9612\n",
      "7 배치 데이터 정확도: 0.96 검증 세트 정확도: 0.9632\n",
      "8 배치 데이터 정확도: 0.98 검증 세트 정확도: 0.9648\n",
      "9 배치 데이터 정확도: 0.96 검증 세트 정확도: 0.9662\n",
      "10 배치 데이터 정확도: 0.92 검증 세트 정확도: 0.9686\n",
      "11 배치 데이터 정확도: 0.98 검증 세트 정확도: 0.9688\n",
      "12 배치 데이터 정확도: 0.98 검증 세트 정확도: 0.9666\n",
      "13 배치 데이터 정확도: 0.98 검증 세트 정확도: 0.9706\n",
      "14 배치 데이터 정확도: 1.0 검증 세트 정확도: 0.9712\n",
      "15 배치 데이터 정확도: 0.94 검증 세트 정확도: 0.9732\n",
      "16 배치 데이터 정확도: 1.0 검증 세트 정확도: 0.9732\n",
      "17 배치 데이터 정확도: 1.0 검증 세트 정확도: 0.9742\n",
      "18 배치 데이터 정확도: 1.0 검증 세트 정확도: 0.9748\n",
      "19 배치 데이터 정확도: 0.98 검증 세트 정확도: 0.9744\n",
      "20 배치 데이터 정확도: 1.0 검증 세트 정확도: 0.9748\n",
      "21 배치 데이터 정확도: 1.0 검증 세트 정확도: 0.976\n",
      "22 배치 데이터 정확도: 0.98 검증 세트 정확도: 0.9766\n",
      "23 배치 데이터 정확도: 0.98 검증 세트 정확도: 0.9752\n",
      "24 배치 데이터 정확도: 0.98 검증 세트 정확도: 0.9766\n",
      "25 배치 데이터 정확도: 1.0 검증 세트 정확도: 0.9768\n",
      "26 배치 데이터 정확도: 0.98 검증 세트 정확도: 0.9778\n",
      "27 배치 데이터 정확도: 1.0 검증 세트 정확도: 0.9774\n",
      "28 배치 데이터 정확도: 0.96 검증 세트 정확도: 0.9754\n",
      "29 배치 데이터 정확도: 0.98 검증 세트 정확도: 0.9776\n",
      "30 배치 데이터 정확도: 1.0 검증 세트 정확도: 0.9754\n",
      "31 배치 데이터 정확도: 0.98 검증 세트 정확도: 0.9772\n",
      "32 배치 데이터 정확도: 0.98 검증 세트 정확도: 0.9776\n",
      "33 배치 데이터 정확도: 0.98 검증 세트 정확도: 0.9792\n",
      "34 배치 데이터 정확도: 1.0 검증 세트 정확도: 0.9784\n",
      "35 배치 데이터 정확도: 1.0 검증 세트 정확도: 0.978\n",
      "36 배치 데이터 정확도: 0.98 검증 세트 정확도: 0.9784\n",
      "37 배치 데이터 정확도: 1.0 검증 세트 정확도: 0.9778\n",
      "38 배치 데이터 정확도: 1.0 검증 세트 정확도: 0.9792\n",
      "39 배치 데이터 정확도: 1.0 검증 세트 정확도: 0.978\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size):\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_batch = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_valid = accuracy.eval(feed_dict={X: X_valid, y: y_valid})\n",
    "        print(epoch, \"배치 데이터 정확도:\", acc_batch, \"검증 세트 정확도:\", acc_valid)\n",
    "\n",
    "    save_path = saver.save(sess, \"./my_model_final.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_model_final.ckpt\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, \"./my_model_final.ckpt\") # 또는 save_path를 사용합니다\n",
    "    X_new_scaled = X_test[:20]\n",
    "    Z = logits.eval(feed_dict={X: X_new_scaled})\n",
    "    y_pred = np.argmax(Z, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "예측 클래스: [7 2 1 0 4 1 4 9 6 9 0 6 9 0 1 5 9 7 3 4]\n",
      "진짜 클래스: [7 2 1 0 4 1 4 9 5 9 0 6 9 0 1 5 9 7 3 4]\n"
     ]
    }
   ],
   "source": [
    "print(\"예측 클래스:\", y_pred)\n",
    "print(\"진짜 클래스:\", y_test[:20])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow_graph_in_jupyter import show_graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe seamless style=\"width:1200px;height:620px;border:0\" srcdoc=\"\n",
       "        <script src=&quot;//cdnjs.cloudflare.com/ajax/libs/polymer/0.3.3/platform.js&quot;></script>\n",
       "        <script>\n",
       "          function load() {\n",
       "            document.getElementById(&quot;graph0.2851015593374667&quot;).pbtxt = 'node {\\n  name: &quot;X&quot;\\n  op: &quot;Placeholder&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: -1\\n        }\\n        dim {\\n          size: 784\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;y&quot;\\n  op: &quot;Placeholder&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        unknown_rank: true\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/truncated_normal/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;\\\\020\\\\003\\\\000\\\\000,\\\\001\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/truncated_normal/mean&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/truncated_normal/stddev&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.0714285746216774\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/truncated_normal/TruncatedNormal&quot;\\n  op: &quot;TruncatedNormal&quot;\\n  input: &quot;dnn/hidden1/truncated_normal/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 42\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 5\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/truncated_normal/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;dnn/hidden1/truncated_normal/TruncatedNormal&quot;\\n  input: &quot;dnn/hidden1/truncated_normal/stddev&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/truncated_normal&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;dnn/hidden1/truncated_normal/mul&quot;\\n  input: &quot;dnn/hidden1/truncated_normal/mean&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 784\\n        }\\n        dim {\\n          size: 300\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden1/kernel&quot;\\n  input: &quot;dnn/hidden1/truncated_normal&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;dnn/hidden1/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/zeros&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 300\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 300\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden1/bias&quot;\\n  input: &quot;dnn/hidden1/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;dnn/hidden1/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;X&quot;\\n  input: &quot;dnn/hidden1/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/add&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;dnn/hidden1/MatMul&quot;\\n  input: &quot;dnn/hidden1/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/Relu&quot;\\n  op: &quot;Relu&quot;\\n  input: &quot;dnn/hidden1/add&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/truncated_normal/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;,\\\\001\\\\000\\\\000d\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/truncated_normal/mean&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/truncated_normal/stddev&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.1154700517654419\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/truncated_normal/TruncatedNormal&quot;\\n  op: &quot;TruncatedNormal&quot;\\n  input: &quot;dnn/hidden2/truncated_normal/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 42\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 21\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/truncated_normal/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;dnn/hidden2/truncated_normal/TruncatedNormal&quot;\\n  input: &quot;dnn/hidden2/truncated_normal/stddev&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/truncated_normal&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;dnn/hidden2/truncated_normal/mul&quot;\\n  input: &quot;dnn/hidden2/truncated_normal/mean&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 300\\n        }\\n        dim {\\n          size: 100\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden2/kernel&quot;\\n  input: &quot;dnn/hidden2/truncated_normal&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;dnn/hidden2/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/zeros&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 100\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 100\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden2/bias&quot;\\n  input: &quot;dnn/hidden2/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;dnn/hidden2/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  input: &quot;dnn/hidden2/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/add&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;dnn/hidden2/MatMul&quot;\\n  input: &quot;dnn/hidden2/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/Relu&quot;\\n  op: &quot;Relu&quot;\\n  input: &quot;dnn/hidden2/add&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/truncated_normal/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;d\\\\000\\\\000\\\\000\\\\n\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/truncated_normal/mean&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/truncated_normal/stddev&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.20000000298023224\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/truncated_normal/TruncatedNormal&quot;\\n  op: &quot;TruncatedNormal&quot;\\n  input: &quot;dnn/outputs/truncated_normal/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 42\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 37\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/truncated_normal/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;dnn/outputs/truncated_normal/TruncatedNormal&quot;\\n  input: &quot;dnn/outputs/truncated_normal/stddev&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/truncated_normal&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;dnn/outputs/truncated_normal/mul&quot;\\n  input: &quot;dnn/outputs/truncated_normal/mean&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 100\\n        }\\n        dim {\\n          size: 10\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/outputs/kernel&quot;\\n  input: &quot;dnn/outputs/truncated_normal&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;dnn/outputs/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/zeros&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 10\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 10\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/outputs/bias&quot;\\n  input: &quot;dnn/outputs/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;dnn/outputs/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  input: &quot;dnn/outputs/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/add&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;dnn/outputs/MatMul&quot;\\n  input: &quot;dnn/outputs/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  op: &quot;SparseSoftmaxCrossEntropyWithLogits&quot;\\n  input: &quot;dnn/outputs/add&quot;\\n  input: &quot;y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tlabels&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/loss&quot;\\n  op: &quot;Mean&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  input: &quot;loss/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/Shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n          }\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/grad_ys_0&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/Fill&quot;\\n  op: &quot;Fill&quot;\\n  input: &quot;train/gradients/Shape&quot;\\n  input: &quot;train/gradients/grad_ys_0&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;index_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Reshape/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/Fill&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Reshape/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Tile&quot;\\n  op: &quot;Tile&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Reshape&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tmultiples&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Shape_1&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Shape_2&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n          }\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Prod&quot;\\n  op: &quot;Prod&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Shape_1&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Const_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Prod_1&quot;\\n  op: &quot;Prod&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Shape_2&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Const_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Maximum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Maximum&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Prod_1&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Maximum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/floordiv&quot;\\n  op: &quot;FloorDiv&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Prod&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Maximum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Cast&quot;\\n  op: &quot;Cast&quot;\\n  input: &quot;train/gradients/loss/loss_grad/floordiv&quot;\\n  attr {\\n    key: &quot;DstT&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;SrcT&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Truncate&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/truediv&quot;\\n  op: &quot;RealDiv&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Tile&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Cast&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/zeros_like&quot;\\n  op: &quot;ZerosLike&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/PreventGradient&quot;\\n  op: &quot;PreventGradient&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;message&quot;\\n    value {\\n      s: &quot;Currently there is no way to take the second derivative of sparse_softmax_cross_entropy_with_logits due to the fused implementation\\\\\\'s interaction with tf.gradients()&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims/dim&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: -1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims&quot;\\n  op: &quot;ExpandDims&quot;\\n  input: &quot;train/gradients/loss/loss_grad/truediv&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims/dim&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tdim&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/PreventGradient&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;dnn/outputs/MatMul&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/Shape_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 10\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/BroadcastGradientArgs&quot;\\n  op: &quot;BroadcastGradientArgs&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Shape&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/Sum&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/BroadcastGradientArgs&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Sum&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/Sum_1&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/BroadcastGradientArgs:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/Reshape_1&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Sum_1&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/outputs/add_grad/Reshape&quot;\\n  input: &quot;^train/gradients/dnn/outputs/add_grad/Reshape_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Reshape&quot;\\n  input: &quot;^train/gradients/dnn/outputs/add_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/add_grad/Reshape&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Reshape_1&quot;\\n  input: &quot;^train/gradients/dnn/outputs/add_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/add_grad/Reshape_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/outputs/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  op: &quot;ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;dnn/hidden2/MatMul&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/Shape_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 100\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/BroadcastGradientArgs&quot;\\n  op: &quot;BroadcastGradientArgs&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Shape&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/Sum&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/BroadcastGradientArgs&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Sum&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/Sum_1&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/BroadcastGradientArgs:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/Reshape_1&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Sum_1&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/add_grad/Reshape&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/add_grad/Reshape_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Reshape&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/add_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/add_grad/Reshape&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Reshape_1&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/add_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/add_grad/Reshape_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden2/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  op: &quot;ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;dnn/hidden1/MatMul&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/Shape_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 300\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/BroadcastGradientArgs&quot;\\n  op: &quot;BroadcastGradientArgs&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Shape&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/Sum&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/BroadcastGradientArgs&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Sum&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/Sum_1&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/BroadcastGradientArgs:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/Reshape_1&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Sum_1&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/add_grad/Reshape&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/add_grad/Reshape_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Reshape&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/add_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/add_grad/Reshape&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Reshape_1&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/add_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/add_grad/Reshape_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden1/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;X&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/learning_rate&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.009999999776482582\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_dnn/hidden1/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;dnn/hidden1/kernel&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_dnn/hidden1/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;dnn/hidden1/bias&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_dnn/hidden2/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;dnn/hidden2/kernel&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_dnn/hidden2/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;dnn/hidden2/bias&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_dnn/outputs/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;dnn/outputs/kernel&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_dnn/outputs/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;dnn/outputs/bias&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/GradientDescent/update_dnn/hidden1/bias/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_dnn/hidden1/kernel/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_dnn/hidden2/bias/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_dnn/hidden2/kernel/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_dnn/outputs/bias/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_dnn/outputs/kernel/ApplyGradientDescent&quot;\\n}\\nnode {\\n  name: &quot;eval/in_top_k/InTopKV2/k&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/in_top_k/InTopKV2&quot;\\n  op: &quot;InTopKV2&quot;\\n  input: &quot;dnn/outputs/add&quot;\\n  input: &quot;y&quot;\\n  input: &quot;eval/in_top_k/InTopKV2/k&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/Cast&quot;\\n  op: &quot;Cast&quot;\\n  input: &quot;eval/in_top_k/InTopKV2&quot;\\n  attr {\\n    key: &quot;DstT&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;SrcT&quot;\\n    value {\\n      type: DT_BOOL\\n    }\\n  }\\n  attr {\\n    key: &quot;Truncate&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/Mean&quot;\\n  op: &quot;Mean&quot;\\n  input: &quot;eval/Cast&quot;\\n  input: &quot;eval/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;init&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^dnn/hidden1/bias/Assign&quot;\\n  input: &quot;^dnn/hidden1/kernel/Assign&quot;\\n  input: &quot;^dnn/hidden2/bias/Assign&quot;\\n  input: &quot;^dnn/hidden2/kernel/Assign&quot;\\n  input: &quot;^dnn/outputs/bias/Assign&quot;\\n  input: &quot;^dnn/outputs/kernel/Assign&quot;\\n}\\nnode {\\n  name: &quot;save/filename/input&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n        }\\n        string_val: &quot;model&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/filename&quot;\\n  op: &quot;PlaceholderWithDefault&quot;\\n  input: &quot;save/filename/input&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Const&quot;\\n  op: &quot;PlaceholderWithDefault&quot;\\n  input: &quot;save/filename&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 6\\n          }\\n        }\\n        string_val: &quot;dnn/hidden1/bias&quot;\\n        string_val: &quot;dnn/hidden1/kernel&quot;\\n        string_val: &quot;dnn/hidden2/bias&quot;\\n        string_val: &quot;dnn/hidden2/kernel&quot;\\n        string_val: &quot;dnn/outputs/bias&quot;\\n        string_val: &quot;dnn/outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 6\\n          }\\n        }\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2&quot;\\n  op: &quot;SaveV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/SaveV2/tensor_names&quot;\\n  input: &quot;save/SaveV2/shape_and_slices&quot;\\n  input: &quot;dnn/hidden1/bias&quot;\\n  input: &quot;dnn/hidden1/kernel&quot;\\n  input: &quot;dnn/hidden2/bias&quot;\\n  input: &quot;dnn/hidden2/kernel&quot;\\n  input: &quot;dnn/outputs/bias&quot;\\n  input: &quot;dnn/outputs/kernel&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;^save/SaveV2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@save/Const&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  device: &quot;/device:CPU:0&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 6\\n          }\\n        }\\n        string_val: &quot;dnn/hidden1/bias&quot;\\n        string_val: &quot;dnn/hidden1/kernel&quot;\\n        string_val: &quot;dnn/hidden2/bias&quot;\\n        string_val: &quot;dnn/hidden2/kernel&quot;\\n        string_val: &quot;dnn/outputs/bias&quot;\\n        string_val: &quot;dnn/outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  device: &quot;/device:CPU:0&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 6\\n          }\\n        }\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2/tensor_names&quot;\\n  input: &quot;save/RestoreV2/shape_and_slices&quot;\\n  device: &quot;/device:CPU:0&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden1/bias&quot;\\n  input: &quot;save/RestoreV2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_1&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden1/kernel&quot;\\n  input: &quot;save/RestoreV2:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_2&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden2/bias&quot;\\n  input: &quot;save/RestoreV2:2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_3&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden2/kernel&quot;\\n  input: &quot;save/RestoreV2:3&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_4&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/outputs/bias&quot;\\n  input: &quot;save/RestoreV2:4&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_5&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/outputs/kernel&quot;\\n  input: &quot;save/RestoreV2:5&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_all&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^save/Assign&quot;\\n  input: &quot;^save/Assign_1&quot;\\n  input: &quot;^save/Assign_2&quot;\\n  input: &quot;^save/Assign_3&quot;\\n  input: &quot;^save/Assign_4&quot;\\n  input: &quot;^save/Assign_5&quot;\\n}\\n';\n",
       "          }\n",
       "        </script>\n",
       "        <link rel=&quot;import&quot; href=&quot;https://tensorboard.appspot.com/tf-graph-basic.build.html&quot; onload=load()>\n",
       "        <div style=&quot;height:600px&quot;>\n",
       "          <tf-graph-basic id=&quot;graph0.2851015593374667&quot;></tf-graph-basic>\n",
       "        </div>\n",
       "    \"></iframe>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_graph(tf.get_default_graph())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## `neuron_layer()` 대신 `dense()` 사용"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_inputs = 28*28  # MNIST\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 100\n",
    "n_outputs = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int32, shape=(None), name=\"y\") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-38-28239acea1fc>:3: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.dense instead.\n"
     ]
    }
   ],
   "source": [
    "with tf.name_scope(\"dnn\"):\n",
    "    hidden1 = tf.layers.dense(X, n_hidden1, name=\"hidden1\",\n",
    "                              activation=tf.nn.relu)\n",
    "    hidden2 = tf.layers.dense(hidden1, n_hidden2, name=\"hidden2\",\n",
    "                              activation=tf.nn.relu)\n",
    "    logits = tf.layers.dense(hidden2, n_outputs, name=\"outputs\")\n",
    "    y_proba = tf.nn.softmax(logits)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = 0.01\n",
    "\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "    training_op = optimizer.minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 배치 데이터 정확도: 0.9 검증 세트 정확도: 0.9024\n",
      "1 배치 데이터 정확도: 0.92 검증 세트 정확도: 0.9254\n",
      "2 배치 데이터 정확도: 0.94 검증 세트 정확도: 0.9372\n",
      "3 배치 데이터 정확도: 0.9 검증 세트 정확도: 0.9416\n",
      "4 배치 데이터 정확도: 0.94 검증 세트 정확도: 0.947\n",
      "5 배치 데이터 정확도: 0.94 검증 세트 정확도: 0.951\n",
      "6 배치 데이터 정확도: 1.0 검증 세트 정확도: 0.9548\n",
      "7 배치 데이터 정확도: 0.94 검증 세트 정확도: 0.9612\n",
      "8 배치 데이터 정확도: 0.96 검증 세트 정확도: 0.9622\n",
      "9 배치 데이터 정확도: 0.94 검증 세트 정확도: 0.965\n",
      "10 배치 데이터 정확도: 0.92 검증 세트 정확도: 0.9654\n",
      "11 배치 데이터 정확도: 0.98 검증 세트 정확도: 0.9668\n",
      "12 배치 데이터 정확도: 0.98 검증 세트 정확도: 0.9686\n",
      "13 배치 데이터 정확도: 0.98 검증 세트 정확도: 0.9702\n",
      "14 배치 데이터 정확도: 1.0 검증 세트 정확도: 0.9696\n",
      "15 배치 데이터 정확도: 0.94 검증 세트 정확도: 0.9718\n",
      "16 배치 데이터 정확도: 0.98 검증 세트 정확도: 0.9728\n",
      "17 배치 데이터 정확도: 1.0 검증 세트 정확도: 0.9728\n",
      "18 배치 데이터 정확도: 0.98 검증 세트 정확도: 0.975\n",
      "19 배치 데이터 정확도: 0.98 검증 세트 정확도: 0.9754\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 20\n",
    "n_batches = 50\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size):\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_batch = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_valid = accuracy.eval(feed_dict={X: X_valid, y: y_valid})\n",
    "        print(epoch, \"배치 데이터 정확도:\", acc_batch, \"검증 세트 정확도:\", acc_valid)\n",
    "\n",
    "    save_path = saver.save(sess, \"./my_model_final.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe seamless style=\"width:1200px;height:620px;border:0\" srcdoc=\"\n",
       "        <script src=&quot;//cdnjs.cloudflare.com/ajax/libs/polymer/0.3.3/platform.js&quot;></script>\n",
       "        <script>\n",
       "          function load() {\n",
       "            document.getElementById(&quot;graph0.7224827313584268&quot;).pbtxt = 'node {\\n  name: &quot;X&quot;\\n  op: &quot;Placeholder&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: -1\\n        }\\n        dim {\\n          size: 784\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;y&quot;\\n  op: &quot;Placeholder&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        unknown_rank: true\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;\\\\020\\\\003\\\\000\\\\000,\\\\001\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/min&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -0.07439795136451721\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/max&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.07439795136451721\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  op: &quot;RandomUniform&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 42\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 5\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/sub&quot;\\n  op: &quot;Sub&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/max&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/sub&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/mul&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 784\\n        }\\n        dim {\\n          size: 300\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden1/kernel&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden1/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/bias/Initializer/zeros&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 300\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 300\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden1/bias&quot;\\n  input: &quot;hidden1/bias/Initializer/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden1/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;X&quot;\\n  input: &quot;hidden1/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/BiasAdd&quot;\\n  op: &quot;BiasAdd&quot;\\n  input: &quot;dnn/hidden1/MatMul&quot;\\n  input: &quot;hidden1/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/Relu&quot;\\n  op: &quot;Relu&quot;\\n  input: &quot;dnn/hidden1/BiasAdd&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;,\\\\001\\\\000\\\\000d\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/min&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -0.12247448414564133\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/max&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.12247448414564133\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  op: &quot;RandomUniform&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 42\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 22\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/sub&quot;\\n  op: &quot;Sub&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/max&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/sub&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/mul&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 300\\n        }\\n        dim {\\n          size: 100\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden2/kernel&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden2/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/bias/Initializer/zeros&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 100\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 100\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden2/bias&quot;\\n  input: &quot;hidden2/bias/Initializer/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden2/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  input: &quot;hidden2/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/BiasAdd&quot;\\n  op: &quot;BiasAdd&quot;\\n  input: &quot;dnn/hidden2/MatMul&quot;\\n  input: &quot;hidden2/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/Relu&quot;\\n  op: &quot;Relu&quot;\\n  input: &quot;dnn/hidden2/BiasAdd&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;d\\\\000\\\\000\\\\000\\\\n\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/min&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -0.23354968428611755\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/max&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.23354968428611755\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  op: &quot;RandomUniform&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 42\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 39\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/sub&quot;\\n  op: &quot;Sub&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/max&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/sub&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/mul&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 100\\n        }\\n        dim {\\n          size: 10\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;outputs/kernel&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;outputs/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/bias/Initializer/zeros&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 10\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 10\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;outputs/bias&quot;\\n  input: &quot;outputs/bias/Initializer/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;outputs/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  input: &quot;outputs/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/BiasAdd&quot;\\n  op: &quot;BiasAdd&quot;\\n  input: &quot;dnn/outputs/MatMul&quot;\\n  input: &quot;outputs/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/Softmax&quot;\\n  op: &quot;Softmax&quot;\\n  input: &quot;dnn/outputs/BiasAdd&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  op: &quot;SparseSoftmaxCrossEntropyWithLogits&quot;\\n  input: &quot;dnn/outputs/BiasAdd&quot;\\n  input: &quot;y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tlabels&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/loss&quot;\\n  op: &quot;Mean&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  input: &quot;loss/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/Shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n          }\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/grad_ys_0&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/Fill&quot;\\n  op: &quot;Fill&quot;\\n  input: &quot;train/gradients/Shape&quot;\\n  input: &quot;train/gradients/grad_ys_0&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;index_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Reshape/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/Fill&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Reshape/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Tile&quot;\\n  op: &quot;Tile&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Reshape&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tmultiples&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Shape_1&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Shape_2&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n          }\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Prod&quot;\\n  op: &quot;Prod&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Shape_1&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Const_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Prod_1&quot;\\n  op: &quot;Prod&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Shape_2&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Const_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Maximum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Maximum&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Prod_1&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Maximum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/floordiv&quot;\\n  op: &quot;FloorDiv&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Prod&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Maximum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Cast&quot;\\n  op: &quot;Cast&quot;\\n  input: &quot;train/gradients/loss/loss_grad/floordiv&quot;\\n  attr {\\n    key: &quot;DstT&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;SrcT&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Truncate&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/truediv&quot;\\n  op: &quot;RealDiv&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Tile&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Cast&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/zeros_like&quot;\\n  op: &quot;ZerosLike&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/PreventGradient&quot;\\n  op: &quot;PreventGradient&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;message&quot;\\n    value {\\n      s: &quot;Currently there is no way to take the second derivative of sparse_softmax_cross_entropy_with_logits due to the fused implementation\\\\\\'s interaction with tf.gradients()&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims/dim&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: -1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims&quot;\\n  op: &quot;ExpandDims&quot;\\n  input: &quot;train/gradients/loss/loss_grad/truediv&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims/dim&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tdim&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/PreventGradient&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/BiasAdd_grad/BiasAddGrad&quot;\\n  op: &quot;BiasAddGrad&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/outputs/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  input: &quot;^train/gradients/dnn/outputs/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^train/gradients/dnn/outputs/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/BiasAdd_grad/BiasAddGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency&quot;\\n  input: &quot;outputs/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  input: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  op: &quot;ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/BiasAddGrad&quot;\\n  op: &quot;BiasAddGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/BiasAdd_grad/BiasAddGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency&quot;\\n  input: &quot;hidden2/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  input: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  op: &quot;ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/BiasAddGrad&quot;\\n  op: &quot;BiasAddGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/BiasAdd_grad/BiasAddGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency&quot;\\n  input: &quot;hidden1/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;X&quot;\\n  input: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/learning_rate&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.009999999776482582\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_hidden1/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;hidden1/kernel&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_hidden1/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;hidden1/bias&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_hidden2/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;hidden2/kernel&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_hidden2/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;hidden2/bias&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_outputs/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;outputs/kernel&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_outputs/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;outputs/bias&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/GradientDescent/update_hidden1/bias/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_hidden1/kernel/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_hidden2/bias/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_hidden2/kernel/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_outputs/bias/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_outputs/kernel/ApplyGradientDescent&quot;\\n}\\nnode {\\n  name: &quot;eval/in_top_k/InTopKV2/k&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/in_top_k/InTopKV2&quot;\\n  op: &quot;InTopKV2&quot;\\n  input: &quot;dnn/outputs/BiasAdd&quot;\\n  input: &quot;y&quot;\\n  input: &quot;eval/in_top_k/InTopKV2/k&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/Cast&quot;\\n  op: &quot;Cast&quot;\\n  input: &quot;eval/in_top_k/InTopKV2&quot;\\n  attr {\\n    key: &quot;DstT&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;SrcT&quot;\\n    value {\\n      type: DT_BOOL\\n    }\\n  }\\n  attr {\\n    key: &quot;Truncate&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/Mean&quot;\\n  op: &quot;Mean&quot;\\n  input: &quot;eval/Cast&quot;\\n  input: &quot;eval/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;init&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^hidden1/bias/Assign&quot;\\n  input: &quot;^hidden1/kernel/Assign&quot;\\n  input: &quot;^hidden2/bias/Assign&quot;\\n  input: &quot;^hidden2/kernel/Assign&quot;\\n  input: &quot;^outputs/bias/Assign&quot;\\n  input: &quot;^outputs/kernel/Assign&quot;\\n}\\nnode {\\n  name: &quot;save/filename/input&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n        }\\n        string_val: &quot;model&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/filename&quot;\\n  op: &quot;PlaceholderWithDefault&quot;\\n  input: &quot;save/filename/input&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Const&quot;\\n  op: &quot;PlaceholderWithDefault&quot;\\n  input: &quot;save/filename&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 6\\n          }\\n        }\\n        string_val: &quot;hidden1/bias&quot;\\n        string_val: &quot;hidden1/kernel&quot;\\n        string_val: &quot;hidden2/bias&quot;\\n        string_val: &quot;hidden2/kernel&quot;\\n        string_val: &quot;outputs/bias&quot;\\n        string_val: &quot;outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 6\\n          }\\n        }\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2&quot;\\n  op: &quot;SaveV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/SaveV2/tensor_names&quot;\\n  input: &quot;save/SaveV2/shape_and_slices&quot;\\n  input: &quot;hidden1/bias&quot;\\n  input: &quot;hidden1/kernel&quot;\\n  input: &quot;hidden2/bias&quot;\\n  input: &quot;hidden2/kernel&quot;\\n  input: &quot;outputs/bias&quot;\\n  input: &quot;outputs/kernel&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;^save/SaveV2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@save/Const&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  device: &quot;/device:CPU:0&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 6\\n          }\\n        }\\n        string_val: &quot;hidden1/bias&quot;\\n        string_val: &quot;hidden1/kernel&quot;\\n        string_val: &quot;hidden2/bias&quot;\\n        string_val: &quot;hidden2/kernel&quot;\\n        string_val: &quot;outputs/bias&quot;\\n        string_val: &quot;outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  device: &quot;/device:CPU:0&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 6\\n          }\\n        }\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2/tensor_names&quot;\\n  input: &quot;save/RestoreV2/shape_and_slices&quot;\\n  device: &quot;/device:CPU:0&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden1/bias&quot;\\n  input: &quot;save/RestoreV2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_1&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden1/kernel&quot;\\n  input: &quot;save/RestoreV2:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_2&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden2/bias&quot;\\n  input: &quot;save/RestoreV2:2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_3&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden2/kernel&quot;\\n  input: &quot;save/RestoreV2:3&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_4&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;outputs/bias&quot;\\n  input: &quot;save/RestoreV2:4&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_5&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;outputs/kernel&quot;\\n  input: &quot;save/RestoreV2:5&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_all&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^save/Assign&quot;\\n  input: &quot;^save/Assign_1&quot;\\n  input: &quot;^save/Assign_2&quot;\\n  input: &quot;^save/Assign_3&quot;\\n  input: &quot;^save/Assign_4&quot;\\n  input: &quot;^save/Assign_5&quot;\\n}\\n';\n",
       "          }\n",
       "        </script>\n",
       "        <link rel=&quot;import&quot; href=&quot;https://tensorboard.appspot.com/tf-graph-basic.build.html&quot; onload=load()>\n",
       "        <div style=&quot;height:600px&quot;>\n",
       "          <tf-graph-basic id=&quot;graph0.7224827313584268&quot;></tf-graph-basic>\n",
       "        </div>\n",
       "    \"></iframe>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_graph(tf.get_default_graph())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# 연습문제 정답"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. to 8."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "부록 A 참조."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 9."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_깊은 다층 퍼셉트론을 MNIST 데이터셋에 훈련시키고 98% 정확도를 얻을 수 있는지 확인해보세요. 9장의 마지막 연습문제에서와 같이 모든 부가 기능을 추가해보세요(즉, 체크포인트를 저장하고, 중지되었을 때 마지막 체크포인트를 복원하고, 서머리를 추가하고, 텐서보드를 사용해 학습 곡선을 그려보세요)._"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "먼저 심층 신경망을 만듭니다. 한가지 추가된 것 외에는 앞서 했던 것과 동일합니다. 텐서보드에서 학습 곡선을 볼 수 있도록 훈련하는 동안 손실과 정확도를 기록하는 `tf.summary.scalar()` 추가합니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_inputs = 28*28  # MNIST\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 100\n",
    "n_outputs = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int32, shape=(None), name=\"y\") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"dnn\"):\n",
    "    hidden1 = tf.layers.dense(X, n_hidden1, name=\"hidden1\",\n",
    "                              activation=tf.nn.relu)\n",
    "    hidden2 = tf.layers.dense(hidden1, n_hidden2, name=\"hidden2\",\n",
    "                              activation=tf.nn.relu)\n",
    "    logits = tf.layers.dense(hidden2, n_outputs, name=\"outputs\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")\n",
    "    loss_summary = tf.summary.scalar('log_loss', loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = 0.01\n",
    "\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "    training_op = optimizer.minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
    "    accuracy_summary = tf.summary.scalar('accuracy', accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "텐서보드 로그를 기록할 디렉토리를 정의합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime\n",
    "\n",
    "def log_dir(prefix=\"\"):\n",
    "    now = datetime.utcnow().strftime(\"%Y%m%d%H%M%S\")\n",
    "    root_logdir = \"tf_logs\"\n",
    "    if prefix:\n",
    "        prefix += \"-\"\n",
    "    name = prefix + \"run-\" + now\n",
    "    return \"{}/{}/\".format(root_logdir, name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "logdir = log_dir(\"mnist_dnn\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "텐서보드 로그를 작성하는 데 필요한 `FileWriter` 객체를 만듭니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_writer = tf.summary.FileWriter(logdir, tf.get_default_graph())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "잠시만요! 조기 종료를 구현하는 것이 좋겠죠? 이렇게 하려면 검증 세트가 필요합니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "# X_valid = mnist.validation.images\n",
    "# y_valid = mnist.validation.labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "m, n = X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "에포크: 0 \t검증 세트 정확도: 90.240% \t손실: 0.35380\n",
      "에포크: 5 \t검증 세트 정확도: 95.100% \t손실: 0.17919\n",
      "에포크: 10 \t검증 세트 정확도: 96.540% \t손실: 0.12785\n",
      "에포크: 15 \t검증 세트 정확도: 97.180% \t손실: 0.10325\n",
      "에포크: 20 \t검증 세트 정확도: 97.480% \t손실: 0.09168\n",
      "에포크: 25 \t검증 세트 정확도: 97.620% \t손실: 0.08212\n",
      "에포크: 30 \t검증 세트 정확도: 97.760% \t손실: 0.07890\n",
      "에포크: 35 \t검증 세트 정확도: 97.800% \t손실: 0.07426\n",
      "에포크: 40 \t검증 세트 정확도: 97.840% \t손실: 0.07170\n",
      "에포크: 45 \t검증 세트 정확도: 98.080% \t손실: 0.06751\n",
      "에포크: 50 \t검증 세트 정확도: 98.040% \t손실: 0.06737\n",
      "에포크: 55 \t검증 세트 정확도: 98.040% \t손실: 0.06689\n",
      "에포크: 60 \t검증 세트 정확도: 98.040% \t손실: 0.06732\n",
      "에포크: 65 \t검증 세트 정확도: 98.220% \t손실: 0.06677\n",
      "에포크: 70 \t검증 세트 정확도: 98.180% \t손실: 0.06617\n",
      "에포크: 75 \t검증 세트 정확도: 98.100% \t손실: 0.06657\n",
      "에포크: 80 \t검증 세트 정확도: 98.160% \t손실: 0.06677\n",
      "에포크: 85 \t검증 세트 정확도: 98.260% \t손실: 0.06613\n",
      "에포크: 90 \t검증 세트 정확도: 98.200% \t손실: 0.06754\n",
      "에포크: 95 \t검증 세트 정확도: 98.140% \t손실: 0.06903\n",
      "에포크: 100 \t검증 세트 정확도: 98.220% \t손실: 0.06893\n",
      "에포크: 105 \t검증 세트 정확도: 98.220% \t손실: 0.07091\n",
      "에포크: 110 \t검증 세트 정확도: 98.200% \t손실: 0.07081\n",
      "에포크: 115 \t검증 세트 정확도: 98.280% \t손실: 0.07086\n",
      "에포크: 120 \t검증 세트 정확도: 98.240% \t손실: 0.07330\n",
      "에포크: 125 \t검증 세트 정확도: 98.280% \t손실: 0.07168\n",
      "조기 종료\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 10001\n",
    "batch_size = 50\n",
    "n_batches = int(np.ceil(m / batch_size))\n",
    "\n",
    "checkpoint_path = \"/tmp/my_deep_mnist_model.ckpt\"\n",
    "checkpoint_epoch_path = checkpoint_path + \".epoch\"\n",
    "final_model_path = \"./my_deep_mnist_model\"\n",
    "\n",
    "best_loss = np.infty\n",
    "epochs_without_progress = 0\n",
    "max_epochs_without_progress = 50\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    if os.path.isfile(checkpoint_epoch_path):\n",
    "        # 체크포인트 파일이 있으면 모델을 복원하고 에포크 숫자를 로드합니다\n",
    "        with open(checkpoint_epoch_path, \"rb\") as f:\n",
    "            start_epoch = int(f.read())\n",
    "        print(\"이전 훈련이 중지되었습니다. 에포크 {}에서 시작합니다\".format(start_epoch))\n",
    "        saver.restore(sess, checkpoint_path)\n",
    "    else:\n",
    "        start_epoch = 0\n",
    "        sess.run(init)\n",
    "\n",
    "    for epoch in range(start_epoch, n_epochs):\n",
    "        for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size):\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        accuracy_val, loss_val, accuracy_summary_str, loss_summary_str = sess.run([accuracy, loss, accuracy_summary, loss_summary], feed_dict={X: X_valid, y: y_valid})\n",
    "        file_writer.add_summary(accuracy_summary_str, epoch)\n",
    "        file_writer.add_summary(loss_summary_str, epoch)\n",
    "        if epoch % 5 == 0:\n",
    "            print(\"에포크:\", epoch,\n",
    "                  \"\\t검증 세트 정확도: {:.3f}%\".format(accuracy_val * 100),\n",
    "                  \"\\t손실: {:.5f}\".format(loss_val))\n",
    "            saver.save(sess, checkpoint_path)\n",
    "            with open(checkpoint_epoch_path, \"wb\") as f:\n",
    "                f.write(b\"%d\" % (epoch + 1))\n",
    "            if loss_val < best_loss:\n",
    "                saver.save(sess, final_model_path)\n",
    "                best_loss = loss_val\n",
    "            else:\n",
    "                epochs_without_progress += 5\n",
    "                if epochs_without_progress > max_epochs_without_progress:\n",
    "                    print(\"조기 종료\")\n",
    "                    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.remove(checkpoint_epoch_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_deep_mnist_model\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, final_model_path)\n",
    "    accuracy_val = accuracy.eval(feed_dict={X: X_test, y: y_test})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9796"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_val"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  },
  "nav_menu": {
   "height": "264px",
   "width": "369px"
  },
  "toc": {
   "navigate_menu": true,
   "number_sections": true,
   "sideBar": true,
   "threshold": 6,
   "toc_cell": false,
   "toc_section_display": "block",
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}