{ "nbformat": 4, "nbformat_minor": 0, "metadata": { "kernelspec": { "display_name": "TensorFlow 2.3 on Python 3.6 (CUDA 10.1)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.9" }, "colab": { "name": "9-1.weight_initialization.ipynb", "provenance": [] }, "accelerator": "GPU" }, "cells": [ { "cell_type": "markdown", "metadata": { "id": "0cwi1yTfzXOE" }, "source": [ "# 가중치 초기화" ] }, { "cell_type": "markdown", "metadata": { "id": "e6UbOPmAzXOL" }, "source": [ "이 노트북에서는 가중치 초기화가 층의 출력에 어떤 영향을 미치는지 알아 봅니다." ] }, { "cell_type": "markdown", "metadata": { "id": "MgdZpxD0zXOM" }, "source": [ "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/rickiepark/dl-illustrated/blob/master/notebooks/9-1.weight_initialization.ipynb)" ] }, { "cell_type": "markdown", "metadata": { "id": "lYtDagbOzXOM" }, "source": [ "#### 라이브러리를 적재합니다." ] }, { "cell_type": "code", "metadata": { "id": "E3h3NczrzXOM" }, "source": [ "import numpy as np\n", "import matplotlib.pyplot as plt\n", "\n", "from tensorflow.keras import Sequential\n", "from tensorflow.keras.layers import Dense, Activation\n", "from tensorflow.keras.initializers import Zeros, RandomNormal, glorot_normal, glorot_uniform" ], "execution_count": 1, "outputs": [] }, { "cell_type": "markdown", "metadata": { "id": "8ieDVCRpzXON" }, "source": [ "#### 뉴런의 개수를 지정합니다." ] }, { "cell_type": "code", "metadata": { "id": "P1v3ZBUhzXON" }, "source": [ "n_input = 784\n", "n_dense = 256" ], "execution_count": 2, "outputs": [] }, { "cell_type": "markdown", "metadata": { "id": "uvSpE-mlzXON" }, "source": [ "#### 절편의 초깃값을 지정합니다." ] }, { "cell_type": "code", "metadata": { "id": "PHzeN5tezXON" }, "source": [ "b_init = Zeros()" ], "execution_count": 3, "outputs": [] }, { "cell_type": "markdown", "metadata": { "id": "SR_B3hupzXOO" }, "source": [ "#### 가중치 초기화를 선택합니다." ] }, { "cell_type": "code", "metadata": { "id": "PPA9GguezXOO" }, "source": [ "w_init = RandomNormal(stddev=1.0)\n", "# w_init = glorot_normal()\n", "# w_init = glorot_uniform()" ], "execution_count": 4, "outputs": [] }, { "cell_type": "markdown", "metadata": { "id": "RpDUl7hazXOO" }, "source": [ "#### 신경망을 만듭니다." ] }, { "cell_type": "code", "metadata": { "id": "e7voAI6dzXOO" }, "source": [ "model = Sequential()\n", "model.add(Dense(n_dense, \n", " input_dim=n_input,\n", " kernel_initializer=w_init,\n", " bias_initializer=b_init))\n", "model.add(Activation('sigmoid'))\n", "# model.add(Activation('tanh'))\n", "# model.add(Activation('relu'))" ], "execution_count": 5, "outputs": [] }, { "cell_type": "markdown", "metadata": { "id": "xHSsM8QKzXOP" }, "source": [ "#### 랜덤한 *x* 값을 생성합니다." ] }, { "cell_type": "code", "metadata": { "id": "VOXC5EtBzXOP" }, "source": [ "x = np.random.random((1,n_input))" ], "execution_count": 6, "outputs": [] }, { "cell_type": "markdown", "metadata": { "id": "ucgRuwWGzXOP" }, "source": [ "#### 신경망의 정방향 계산을 수행합니다." ] }, { "cell_type": "code", "metadata": { "id": "ddTN1W_AzXOP", "outputId": "cf0c33b3-202d-43d9-c0e4-fdc93c1fb6b5", "colab": { "base_uri": "https://localhost:8080/" } }, "source": [ "a = model.predict(x)" ], "execution_count": 7, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "1/1 [==============================] - 4s 4s/step\n" ] } ] }, { "cell_type": "code", "metadata": { "id": "XgoTL6gGzXOQ", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "1f110d0f-81ae-4b64-917b-d353bd53131b" }, "source": [ "a" ], "execution_count": 8, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ "array([[9.9999976e-01, 2.0004527e-06, 1.0000000e+00, 7.6007054e-06,\n", " 7.2447336e-01, 1.0000000e+00, 1.0000000e+00, 7.2155090e-06,\n", " 9.9873585e-01, 1.0000000e+00, 9.9276406e-01, 1.7446166e-07,\n", " 9.9991155e-01, 1.7818085e-08, 6.0669146e-03, 9.9694365e-01,\n", " 5.5367585e-02, 8.4336215e-01, 1.0000000e+00, 9.1620052e-01,\n", " 3.7742063e-02, 1.1626942e-01, 6.0207140e-02, 9.9999952e-01,\n", " 9.9999988e-01, 1.1345611e-18, 8.5575145e-04, 4.0285154e-12,\n", " 2.7216640e-07, 1.2736185e-01, 9.9425274e-01, 9.8856676e-01,\n", " 1.0000000e+00, 1.0000000e+00, 8.7992710e-01, 7.5850660e-01,\n", " 8.3993769e-01, 1.5167581e-02, 9.9999368e-01, 2.5858393e-08,\n", " 3.2360949e-05, 2.6045323e-04, 9.9999988e-01, 6.9003995e-08,\n", " 4.6693298e-01, 1.7045186e-05, 1.0000000e+00, 4.6300636e-05,\n", " 2.7897401e-02, 1.0000000e+00, 1.0000000e+00, 4.3196255e-01,\n", " 9.9982542e-01, 1.7630829e-03, 1.2610410e-05, 9.9998820e-01,\n", " 9.9999988e-01, 1.0000000e+00, 1.9858588e-09, 9.9865794e-01,\n", " 1.0000000e+00, 9.9999988e-01, 9.9999356e-01, 5.2389276e-01,\n", " 1.0000000e+00, 2.2728916e-03, 9.9999940e-01, 1.0000000e+00,\n", " 9.6458012e-01, 1.0000000e+00, 9.9812490e-01, 9.9949884e-01,\n", " 4.7519954e-04, 3.3310425e-01, 9.0552413e-01, 8.5556632e-01,\n", " 9.9999988e-01, 7.2879493e-02, 4.2009432e-10, 1.3394852e-03,\n", " 3.3087630e-09, 9.9996769e-01, 5.1238286e-01, 3.7398124e-06,\n", " 9.3211509e-02, 5.1276106e-04, 4.3272531e-01, 1.7245174e-03,\n", " 4.9460980e-01, 9.0364385e-01, 1.0000000e+00, 4.0471383e-07,\n", " 3.6911390e-04, 1.0000000e+00, 2.4616565e-03, 7.4751504e-02,\n", " 9.9999809e-01, 9.9999869e-01, 6.0375658e-07, 1.5204600e-17,\n", " 1.0145973e-08, 3.2308731e-15, 2.2927362e-23, 1.0000000e+00,\n", " 9.9996197e-01, 9.4917184e-01, 2.4504557e-02, 9.9999917e-01,\n", " 5.0322893e-08, 4.0138799e-09, 1.0000000e+00, 5.7756886e-02,\n", " 9.9624120e-10, 9.6115343e-08, 6.7261944e-04, 9.9793267e-01,\n", " 4.0458761e-10, 9.9998069e-01, 1.6944791e-09, 5.7222009e-09,\n", " 2.9329160e-06, 9.9243402e-01, 1.0670438e-07, 2.3417340e-10,\n", " 9.9979085e-01, 9.9999475e-01, 2.7916862e-11, 9.9081635e-01,\n", " 9.9996519e-01, 1.0000000e+00, 3.3258403e-09, 9.9999547e-01,\n", " 6.4479635e-04, 1.3639749e-20, 2.5365388e-02, 4.9270267e-12,\n", " 1.0000000e+00, 1.0000000e+00, 1.0000000e+00, 8.1613696e-01,\n", " 9.9900800e-01, 3.6039779e-05, 1.0000000e+00, 9.9927586e-01,\n", " 5.0434846e-02, 1.0000000e+00, 1.0000000e+00, 3.9598272e-06,\n", " 9.8010617e-01, 9.8909539e-01, 7.0482641e-02, 9.8838860e-01,\n", " 9.9990189e-01, 6.2087643e-01, 1.1945281e-04, 4.1250487e-07,\n", " 1.0000000e+00, 8.5153884e-10, 1.1645834e-01, 1.0000000e+00,\n", " 4.0245816e-01, 1.0000000e+00, 9.7169703e-01, 9.9998891e-01,\n", " 1.0000000e+00, 1.0000000e+00, 9.9997938e-01, 1.8650668e-11,\n", " 1.5423197e-01, 1.0000000e+00, 9.7142178e-01, 9.9932158e-01,\n", " 1.0000000e+00, 2.2391982e-02, 9.9999058e-01, 1.4179017e-08,\n", " 9.6793604e-09, 9.2963719e-01, 2.3809712e-14, 9.8940842e-11,\n", " 1.0000000e+00, 8.0581201e-05, 9.9527836e-01, 9.9998581e-01,\n", " 1.1921300e-02, 8.3797222e-01, 7.2229195e-01, 2.0421967e-02,\n", " 2.5230616e-05, 2.8043921e-04, 9.1558671e-01, 1.0000000e+00,\n", " 9.9992621e-01, 9.9997973e-01, 9.9999952e-01, 1.0000000e+00,\n", " 1.0000000e+00, 1.8243941e-02, 6.1026850e-07, 6.0636532e-01,\n", " 2.2425574e-04, 1.5451181e-04, 5.9624430e-02, 9.9996448e-01,\n", " 9.9997723e-01, 9.9996424e-01, 1.2161328e-07, 2.6257087e-08,\n", " 1.3579285e-10, 2.6606099e-08, 8.7479502e-01, 5.0915283e-04,\n", " 5.9039354e-07, 5.8859558e-04, 3.1329486e-09, 2.4465851e-03,\n", " 9.9977654e-01, 9.9999428e-01, 9.9907744e-01, 9.9978012e-01,\n", " 1.0000000e+00, 8.5014262e-06, 9.9947661e-01, 1.3509205e-09,\n", " 7.6480833e-10, 4.2969562e-04, 9.1507763e-01, 1.5044299e-09,\n", " 8.7820133e-04, 9.8001146e-01, 4.0643033e-01, 4.7102249e-03,\n", " 9.9734432e-01, 1.1301318e-03, 6.6847545e-01, 9.9999225e-01,\n", " 3.2009734e-07, 1.0000000e+00, 9.9856037e-01, 2.0048836e-01,\n", " 2.8661656e-01, 1.5250902e-06, 1.5025468e-01, 1.0000000e+00,\n", " 1.1106618e-04, 9.7996815e-07, 5.6953407e-08, 7.7879751e-01,\n", " 9.9998009e-01, 3.1049974e-04, 9.9977845e-01, 3.6121635e-03,\n", " 3.9790309e-04, 9.9987197e-01, 9.9999952e-01, 4.0278728e-03]],\n", " dtype=float32)" ] }, "metadata": {}, "execution_count": 8 } ] }, { "cell_type": "code", "metadata": { "id": "8PW4WCkczXOR", "colab": { "base_uri": "https://localhost:8080/", "height": 268 }, "outputId": "9c3942f7-cc06-4df1-aa09-9819da77978c" }, "source": [ "_ = plt.hist(np.transpose(a))" ], "execution_count": 9, "outputs": [ { "output_type": "display_data", "data": { "text/plain": [ "
" ], "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD7CAYAAACRxdTpAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAOu0lEQVR4nO3dfYxld13H8feHri2CSAs7NnW3OiUs6lo1NJOmhAQrS7QU0m1i0ywBWXDjBkREMYEif9ToP21UEBJEN7SwGOyDFe1G8KEubRoJuziltQ9bSpc+wNZtdxBaH4hA5esf94DjdLZz55577zC/fb+SyZyH3znn+9s7+9nf/O65Z1NVSJLa8oy1LkCSNH6GuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSg1YM9yRXJzmW5O5F234/yeeT3Jnkr5Kcumjfu5IcTnJfkl+YVOGSpOMbZuT+EeCCJdtuAs6uqp8GvgC8CyDJVmAH8JPdMX+c5KSxVStJGsqGlRpU1a1JZpds+4dFqweAS7rl7cC1VfUN4MEkh4Fzgc883TU2btxYs7OzT9dEkrTEbbfd9pWqmllu34rhPoRfBq7rljcxCPvvONJte1qzs7PMz8+PoRRJOnEkefh4+3q9oZrk3cCTwMdGOHZ3kvkk8wsLC33KkCQtMXK4J3kD8GrgtfV/D6h5BDhzUbPN3banqKo9VTVXVXMzM8v+ViFJGtFI4Z7kAuAdwEVV9fVFu/YBO5KckuQsYAvw2f5lSpJWY8U59yTXAOcDG5McAS5ncHfMKcBNSQAOVNWbquqeJNcDhxhM17ylqv5nUsVLkpaX74VH/s7NzZVvqErS6iS5rarmltvnJ1QlqUGGuyQ1yHCXpAYZ7pLUoHF8QlWS1rXZyz6xZtd+6IpXTeS8jtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSg9b9rZAt3sIkSX05cpekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBK4Z7kquTHEty96Jtz0tyU5L7u++ndduT5P1JDie5M8k5kyxekrS8YUbuHwEuWLLtMmB/VW0B9nfrAK8EtnRfu4EPjqdMSdJqrBjuVXUr8NUlm7cDe7vlvcDFi7Z/tAYOAKcmOWNcxUqShjPqnPvpVXW0W34UOL1b3gR8eVG7I902SdIU9X5DtaoKqNUel2R3kvkk8wsLC33LkCQtMmq4P/ad6Zbu+7Fu+yPAmYvabe62PUVV7amquaqam5mZGbEMSdJyRg33fcDObnkncOOi7a/v7po5D3hi0fSNJGlKNqzUIMk1wPnAxiRHgMuBK4Drk+wCHgYu7Zp/ErgQOAx8HXjjBGqWJK1gxXCvqtccZ9e2ZdoW8Ja+RUmS+vETqpLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSg3qFe5LfTHJPkruTXJPkmUnOSnIwyeEk1yU5eVzFSpKGM3K4J9kE/DowV1VnAycBO4ArgfdW1QuBrwG7xlGoJGl4fadlNgDfn2QD8CzgKPBy4IZu/17g4p7XkCSt0sjhXlWPAH8AfIlBqD8B3AY8XlVPds2OAJv6FilJWp0+0zKnAduBs4AfBp4NXLCK43cnmU8yv7CwMGoZkqRl9JmWeQXwYFUtVNW3gI8DLwVO7aZpADYDjyx3cFXtqaq5qpqbmZnpUYYkaak+4f4l4Lwkz0oSYBtwCLgZuKRrsxO4sV+JkqTV6jPnfpDBG6efA+7qzrUHeCfw9iSHgecDV42hTknSKmxYucnxVdXlwOVLNj8AnNvnvJKkfvyEqiQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUoF7hnuTUJDck+XySe5O8JMnzktyU5P7u+2njKlaSNJy+I/f3AX9XVT8O/AxwL3AZsL+qtgD7u3VJ0hSNHO5Jngu8DLgKoKq+WVWPA9uBvV2zvcDFfYuUJK1On5H7WcAC8OEktyf5UJJnA6dX1dGuzaPA6X2LlCStTp9w3wCcA3ywql4M/BdLpmCqqoBa7uAku5PMJ5lfWFjoUYYkaak+4X4EOFJVB7v1GxiE/WNJzgDovh9b7uCq2lNVc1U1NzMz06MMSdJSI4d7VT0KfDnJj3WbtgGHgH3Azm7bTuDGXhVKklZtQ8/j3wp8LMnJwAPAGxn8g3F9kl3Aw8ClPa8hSVqlXuFeVXcAc8vs2tbnvJKkfvyEqiQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUoN7hnuSkJLcn+Ztu/awkB5McTnJdkpP7lylJWo1xjNzfBty7aP1K4L1V9ULga8CuMVxDkrQKvcI9yWbgVcCHuvUALwdu6JrsBS7ucw1J0ur1Hbn/EfAO4Nvd+vOBx6vqyW79CLCp5zUkSas0crgneTVwrKpuG/H43Unmk8wvLCyMWoYkaRl9Ru4vBS5K8hBwLYPpmPcBpybZ0LXZDDyy3MFVtaeq5qpqbmZmpkcZkqSlRg73qnpXVW2uqllgB/CpqnotcDNwSddsJ3Bj7yolSasyifvc3wm8PclhBnPwV03gGpKkp7Fh5SYrq6pbgFu65QeAc8dxXknSaPyEqiQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUoJHDPcmZSW5OcijJPUne1m1/XpKbktzffT9tfOVKkobRZ+T+JPBbVbUVOA94S5KtwGXA/qraAuzv1iVJUzRyuFfV0ar6XLf8H8C9wCZgO7C3a7YXuLhvkZKk1RnLnHuSWeDFwEHg9Ko62u16FDh9HNeQJA2vd7gn+QHgL4HfqKp/X7yvqgqo4xy3O8l8kvmFhYW+ZUiSFukV7km+j0Gwf6yqPt5tfizJGd3+M4Bjyx1bVXuqaq6q5mZmZvqUIUlaos/dMgGuAu6tqvcs2rUP2Nkt7wRuHL08SdIoNvQ49qXALwF3Jbmj2/bbwBXA9Ul2AQ8Dl/YrUZK0WiOHe1X9E5Dj7N426nklSf35CVVJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ3q83+oStJYzV72ibUuoRmO3CWpQY7ctS6s5YjuoStetWbXlkblyF2SGuTIvYe1Gk06kpS0EkfuktQgR+6S/h/vWGmD4S59jzJk1YfTMpLUoImN3JNcALwPOAn4UFVdMalrnWi8LXC6HEFrPZrIyD3JScAHgFcCW4HXJNk6iWtJkp5qUiP3c4HDVfUAQJJrge3AoQldT1PiKFZaHyY1574J+PKi9SPdNknSFKzZ3TJJdgO7u9X/THLfiKfaCHxlPFWtG/b5xGCfTwC5sleff/R4OyYV7o8AZy5a39xt+66q2gPs6XuhJPNVNdf3POuJfT4x2OcTw6T6PKlpmX8GtiQ5K8nJwA5g34SuJUlaYiIj96p6MsmvAX/P4FbIq6vqnklcS5L0VBObc6+qTwKfnNT5F+k9tbMO2ecTg30+MUykz6mqSZxXkrSGfPyAJDVo3YR7kguS3JfkcJLLltl/SpLruv0Hk8xOv8rxGqLPb09yKMmdSfYnOe5tUevFSn1e1O4Xk1SSdX9nxTB9TnJp91rfk+TPp13juA3xs/0jSW5Ocnv3833hWtQ5LkmuTnIsyd3H2Z8k7+/+PO5Mck7vi1bV9/wXgzdlvwi8ADgZ+Bdg65I2vwr8Sbe8A7hureueQp9/DnhWt/zmE6HPXbvnALcCB4C5ta57Cq/zFuB24LRu/YfWuu4p9HkP8OZueSvw0FrX3bPPLwPOAe4+zv4Lgb8FApwHHOx7zfUycv/u4wyq6pvAdx5nsNh2YG+3fAOwLUmmWOO4rdjnqrq5qr7erR5g8HmC9WyY1xng94Argf+eZnETMkyffwX4QFV9DaCqjk25xnEbps8F/GC3/FzgX6dY39hV1a3AV5+myXbgozVwADg1yRl9rrlewn2Yxxl8t01VPQk8ATx/KtVNxmof4bCLwb/869mKfe5+XT2zqlp5yM0wr/OLgBcl+XSSA90TV9ezYfr8O8DrkhxhcNfdW6dT2poZ+yNb/M86GpDkdcAc8LNrXcskJXkG8B7gDWtcyrRtYDA1cz6D385uTfJTVfX4mlY1Wa8BPlJVf5jkJcCfJTm7qr691oWtF+tl5L7i4wwWt0mygcGvcv82leomY5g+k+QVwLuBi6rqG1OqbVJW6vNzgLOBW5I8xGBuct86f1N1mNf5CLCvqr5VVQ8CX2AQ9uvVMH3eBVwPUFWfAZ7J4LkzrRrq7/tqrJdwH+ZxBvuAnd3yJcCnqnunYp1asc9JXgz8KYNgX+/zsLBCn6vqiaraWFWzVTXL4H2Gi6pqfm3KHYthfrb/msGonSQbGUzTPDDNIsdsmD5/CdgGkOQnGIT7wlSrnK59wOu7u2bOA56oqqO9zrjW7yKv4t3mCxmMWL4IvLvb9rsM/nLD4MX/C+Aw8FngBWtd8xT6/I/AY8Ad3de+ta550n1e0vYW1vndMkO+zmEwHXUIuAvYsdY1T6HPW4FPM7iT5g7g59e65p79vQY4CnyLwW9iu4A3AW9a9Bp/oPvzuGscP9d+QlWSGrRepmUkSatguEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1KD/BcBe97FSZKzXAAAAAElFTkSuQmCC\n" }, "metadata": { "needs_background": "light" } } ] } ] }