{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "TensorFlow 2.3 on Python 3.6 (CUDA 10.1)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.9"
    },
    "colab": {
      "name": "11-6.bi_lstm_sentiment_classifier.ipynb",
      "provenance": []
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jqMY1A0ut70x"
      },
      "source": [
        "# 양방향 LSTM 감성 분류기"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UTHMNabDt701"
      },
      "source": [
        "이 노트북에서 *양방향* LSTM을 만들어 감성에 따라 IMDB 영화 리뷰를 분류합니다."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "94L0jGEft701"
      },
      "source": [
        "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/rickiepark/dl-illustrated/blob/master/notebooks/11-6.bi_lstm_sentiment_classifier.ipynb)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9ksCVoEut701"
      },
      "source": [
        "#### 라이브러리 적재"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jJYPAGVQt702"
      },
      "source": [
        "from tensorflow import keras\n",
        "from tensorflow.keras.datasets import imdb\n",
        "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
        "from tensorflow.keras.models import Sequential\n",
        "from tensorflow.keras.layers import Dense, Dropout, Embedding, SpatialDropout1D, LSTM\n",
        "from tensorflow.keras.layers import Bidirectional # new! \n",
        "from tensorflow.keras.callbacks import ModelCheckpoint\n",
        "import os\n",
        "from sklearn.metrics import roc_auc_score \n",
        "import matplotlib.pyplot as plt \n",
        "%matplotlib inline"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "I2CbnSalt702"
      },
      "source": [
        "#### 하이퍼파라미터 설정"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CpELYZDPt702"
      },
      "source": [
        "# 출력 디렉토리\n",
        "output_dir = 'model_output/biLSTM'\n",
        "\n",
        "# 훈련\n",
        "epochs = 6\n",
        "batch_size = 128\n",
        "\n",
        "# 벡터 공간 임베딩\n",
        "n_dim = 64 \n",
        "n_unique_words = 10000 \n",
        "max_review_length = 200 # 두베!\n",
        "pad_type = trunc_type = 'pre'\n",
        "drop_embed = 0.2 \n",
        "\n",
        "# LSTM 층 구조\n",
        "n_lstm = 256 \n",
        "drop_lstm = 0.2"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ki8tpkjft703"
      },
      "source": [
        "#### 데이터 적재"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OLH6E5Wat703",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f32f2369-927c-4635-e603-316218504461"
      },
      "source": [
        "(x_train, y_train), (x_valid, y_valid) = imdb.load_data(num_words=n_unique_words) # n_words_to_skip 삭제"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/imdb.npz\n",
            "17464789/17464789 [==============================] - 2s 0us/step\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XVkBZCMUt703"
      },
      "source": [
        "#### 데이터 전처리"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NyYU0OqGt703"
      },
      "source": [
        "x_train = pad_sequences(x_train, maxlen=max_review_length, padding=pad_type, truncating=trunc_type, value=0)\n",
        "x_valid = pad_sequences(x_valid, maxlen=max_review_length, padding=pad_type, truncating=trunc_type, value=0)"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "collapsed": true,
        "id": "NVJhEcK7t703"
      },
      "source": [
        "#### 신경망 만들기"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G7MUVnwNt704"
      },
      "source": [
        "model = Sequential()\n",
        "model.add(Embedding(n_unique_words, n_dim, input_length=max_review_length)) \n",
        "model.add(SpatialDropout1D(drop_embed))\n",
        "model.add(Bidirectional(LSTM(n_lstm, dropout=drop_lstm)))\n",
        "model.add(Dense(1, activation='sigmoid'))"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lRZjI_1zt704",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a03f7c85-825c-4978-c3e4-8ae5086594ff"
      },
      "source": [
        "# 양 방향으로 가중치가 있기 때문에 LSTM 층 파라미터가 두 배가 됩니다.\n",
        "model.summary() "
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " embedding (Embedding)       (None, 200, 64)           640000    \n",
            "                                                                 \n",
            " spatial_dropout1d (SpatialD  (None, 200, 64)          0         \n",
            " ropout1D)                                                       \n",
            "                                                                 \n",
            " bidirectional (Bidirectiona  (None, 512)              657408    \n",
            " l)                                                              \n",
            "                                                                 \n",
            " dense (Dense)               (None, 1)                 513       \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 1,297,921\n",
            "Trainable params: 1,297,921\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5NIlx8HXt704"
      },
      "source": [
        "#### 모델 설정"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-46STerot705"
      },
      "source": [
        "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4DFs0E_Et705"
      },
      "source": [
        "modelcheckpoint = ModelCheckpoint(filepath=output_dir+\"/weights.{epoch:02d}.hdf5\")\n",
        "if not os.path.exists(output_dir):\n",
        "    os.makedirs(output_dir)"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "f_lLNZADt705"
      },
      "source": [
        "#### 훈련!"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cGv6vIgRt705",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "59257a63-e7bc-4607-94ec-05595fa50197"
      },
      "source": [
        "# 데이터셋이 작기 때문에 긴 문장에 걸쳐 단어의 복잡한 상호작용이 잘 학습되지 않습니다.\n",
        "# CNN은 리뷰의 감성을 예측하는 위치에 상관없는 2개에서 4개까지 단어 조각을 선택합니다.\n",
        "# 이 작업이 더 간단하기 때문에 데이터에서 학습하기 쉽습니다.\n",
        "# 따라서 CNN이 IMDB 데이터셋에서 성능이 더 좋습니다.\n",
        "model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1, validation_data=(x_valid, y_valid), callbacks=[modelcheckpoint])"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/6\n",
            "196/196 [==============================] - 22s 71ms/step - loss: 0.5663 - accuracy: 0.7075 - val_loss: 0.3506 - val_accuracy: 0.8566\n",
            "Epoch 2/6\n",
            "196/196 [==============================] - 13s 68ms/step - loss: 0.2967 - accuracy: 0.8795 - val_loss: 0.3144 - val_accuracy: 0.8679\n",
            "Epoch 3/6\n",
            "196/196 [==============================] - 14s 70ms/step - loss: 0.2261 - accuracy: 0.9156 - val_loss: 0.3119 - val_accuracy: 0.8745\n",
            "Epoch 4/6\n",
            "196/196 [==============================] - 14s 72ms/step - loss: 0.1781 - accuracy: 0.9325 - val_loss: 0.3227 - val_accuracy: 0.8624\n",
            "Epoch 5/6\n",
            "196/196 [==============================] - 14s 72ms/step - loss: 0.1487 - accuracy: 0.9449 - val_loss: 0.3631 - val_accuracy: 0.8681\n",
            "Epoch 6/6\n",
            "196/196 [==============================] - 14s 72ms/step - loss: 0.1243 - accuracy: 0.9556 - val_loss: 0.3986 - val_accuracy: 0.8627\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7fba23b71040>"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "collapsed": true,
        "id": "v2s_iNBat705"
      },
      "source": [
        "#### 평가"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XxmeMhdqt705"
      },
      "source": [
        "model.load_weights(output_dir+\"/weights.04.hdf5\") "
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L10_WLaBt706",
        "outputId": "b7e36fd8-6f95-4586-8990-5933eddf61e0",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "y_hat = model.predict(x_valid)"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "782/782 [==============================] - 8s 9ms/step\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HaMMUWICt706",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 265
        },
        "outputId": "28804b53-bc64-4439-89bc-02f11cc545b9"
      },
      "source": [
        "plt.hist(y_hat)\n",
        "_ = plt.axvline(x=0.5, color='orange')"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD4CAYAAAAAczaOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAU3klEQVR4nO3dcZBd5X3e8e9jZOzYsS1hFA2V5IqMFafYHdt0B/C4kzqWIwTOWMzUZvA0tcJoqk5K0iTNtMXtH2rBdPC0DQ0zMYka1AhPYkxoXDQxDVVlPJ52CmYxDjEQyhobIxXQBgm5CWOncn79475yrvEue5e9e9fL+/3M7Nz3vOc957wvK5579r3nnpOqQpLUh1esdAckSZNj6EtSRwx9SeqIoS9JHTH0Jakja1a6Ay/m7LPPri1btqx0N6Tv981HB6+vf8vK9kOaw/333/+nVbV+rnU/0KG/ZcsWpqenV7ob0vf77+8ZvL7v8yvZC2lOSZ6Yb53TO5LUEUNfkjpi6EtSRwx9SerISKGf5JeTPJTkK0k+leTVSc5Ncm+SmSSfTnJma/uqtjzT1m8Z2s9HW/2jSS5eniFJkuazYOgn2Qj8Y2Cqqt4GnAFcAXwcuKGq3gycAHa3TXYDJ1r9Da0dSc5r270V2AF8IskZ4x2OJOnFjDq9swb4oSRrgNcATwHvBW5v6w8Al7XyzrZMW78tSVr9rVX17ar6GjADXLD0IUiSRrVg6FfVUeDfAd9gEPYngfuB56rqVGt2BNjYyhuBJ9u2p1r7Nw7Xz7GNJGkCRpneWcfgLP1c4K8Br2UwPbMskuxJMp1kenZ2drkOI0ldGuUbue8DvlZVswBJfh94N7A2yZp2Nr8JONraHwU2A0fadNAbgGeH6k8b3ua7qmofsA9gampqSU942XL1Z5ey+Uv29evfvyLHlaSFjDKn/w3goiSvaXPz24CHgbuBD7Y2u4A7WvlgW6at/1wNHs91ELiiXd1zLrAV+OJ4hiFJGsWCZ/pVdW+S24EvAaeABxiciX8WuDXJx1rdzW2Tm4FPJpkBjjO4YoeqeijJbQzeME4BV1XVd8Y8HknSixjphmtVtRfY+4Lqx5nj6puq+hbwoXn2cx1w3SL7KEkaE7+RK0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSR36gH4wuSStppW7lAst3OxfP9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSMLhn6StyT58tDPN5P8UpKzkhxK8lh7XdfaJ8mNSWaSPJjk/KF97WrtH0uya/6jSpKWw4KhX1WPVtU7quodwN8Cngc+A1wNHK6qrcDhtgxwCYOHnm8F9gA3ASQ5i8EjFy9k8JjFvaffKCRJk7HY6Z1twFer6glgJ3Cg1R8ALmvlncAtNXAPsDbJOcDFwKGqOl5VJ4BDwI4lj0CSNLLFhv4VwKdaeUNVPdXKTwMbWnkj8OTQNkda3Xz1kqQJGTn0k5wJfAD4vReuq6oCahwdSrInyXSS6dnZ2XHsUpLULOZM/xLgS1X1TFt+pk3b0F6PtfqjwOah7Ta1uvnqv0dV7auqqaqaWr9+/SK6J0layGJC/8P81dQOwEHg9BU4u4A7huo/0q7iuQg42aaB7gK2J1nXPsDd3uokSRMy0kNUkrwW+CngHw5VXw/clmQ38ARweau/E7gUmGFwpc+VAFV1PMm1wH2t3TVVdXzJI5AkjWyk0K+qPwfe+IK6ZxlczfPCtgVcNc9+9gP7F99NSdI4+I1cSeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdGSn0k6xNcnuSP0nySJJ3JTkryaEkj7XXda1tktyYZCbJg0nOH9rPrtb+sSS75j+iJGk5jHqm/2vAH1bVjwNvBx4BrgYOV9VW4HBbBrgE2Np+9gA3ASQ5C9gLXAhcAOw9/UYhSZqMBUM/yRuAnwBuBqiqv6iq54CdwIHW7ABwWSvvBG6pgXuAtUnOAS4GDlXV8ao6ARwCdox1NJKkFzXKmf65wCzwn5I8kOS3krwW2FBVT7U2TwMbWnkj8OTQ9kda3Xz13yPJniTTSaZnZ2cXNxpJ0osaJfTXAOcDN1XVO4E/56+mcgCoqgJqHB2qqn1VNVVVU+vXrx/HLiVJzSihfwQ4UlX3tuXbGbwJPNOmbWivx9r6o8Dmoe03tbr56iVJE7Jg6FfV08CTSd7SqrYBDwMHgdNX4OwC7mjlg8BH2lU8FwEn2zTQXcD2JOvaB7jbW50kaULWjNjuF4DfSXIm8DhwJYM3jNuS7AaeAC5vbe8ELgVmgOdbW6rqeJJrgftau2uq6vhYRiFJGslIoV9VXwam5li1bY62BVw1z372A/sX00FJ0vj4jVxJ6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqyEihn+TrSf44yZeTTLe6s5IcSvJYe13X6pPkxiQzSR5Mcv7Qfna19o8l2TXf8SRJy2MxZ/o/WVXvqKrTj028GjhcVVuBw20Z4BJga/vZA9wEgzcJYC9wIXABsPf0G4UkaTKWMr2zEzjQygeAy4bqb6mBe4C1Sc4BLgYOVdXxqjoBHAJ2LOH4kqRFGjX0C/hvSe5PsqfVbaiqp1r5aWBDK28Enhza9kirm6/+eyTZk2Q6yfTs7OyI3ZMkjWLNiO3+dlUdTfIjwKEkfzK8sqoqSY2jQ1W1D9gHMDU1NZZ9SpIGRjrTr6qj7fUY8BkGc/LPtGkb2uux1vwosHlo802tbr56SdKELBj6SV6b5HWny8B24CvAQeD0FTi7gDta+SDwkXYVz0XAyTYNdBewPcm69gHu9lYnSZqQUaZ3NgCfSXK6/e9W1R8muQ+4Lclu4Ang8tb+TuBSYAZ4HrgSoKqOJ7kWuK+1u6aqjo9tJJKkBS0Y+lX1OPD2OeqfBbbNUV/AVfPsaz+wf/HdlCSNg9/IlaSOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI6MHPpJzkjyQJI/aMvnJrk3yUySTyc5s9W/qi3PtPVbhvbx0Vb/aJKLxz0YSdKLW8yZ/i8Cjwwtfxy4oareDJwAdrf63cCJVn9Da0eS84ArgLcCO4BPJDljad2XJC3GSKGfZBPwfuC32nKA9wK3tyYHgMtaeWdbpq3f1trvBG6tqm9X1dcYPDj9gnEMQpI0mlHP9P8D8M+Av2zLbwSeq6pTbfkIsLGVNwJPArT1J1v779bPsc13JdmTZDrJ9Ozs7CKGIklayIKhn+SngWNVdf8E+kNV7auqqaqaWr9+/SQOKUndWDNCm3cDH0hyKfBq4PXArwFrk6xpZ/ObgKOt/VFgM3AkyRrgDcCzQ/WnDW8jSZqABc/0q+qjVbWpqrYw+CD2c1X194C7gQ+2ZruAO1r5YFumrf9cVVWrv6Jd3XMusBX44thGIkla0Chn+vP558CtST4GPADc3OpvBj6ZZAY4zuCNgqp6KMltwMPAKeCqqvrOEo4vSVqkRYV+VX0e+HwrP84cV99U1beAD82z/XXAdYvtpCRpPPxGriR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHVkwdBP8uokX0zyR0keSvKvW/25Se5NMpPk00nObPWvasszbf2WoX19tNU/muTi5RqUJGluo5zpfxt4b1W9HXgHsCPJRcDHgRuq6s3ACWB3a78bONHqb2jtSHIeg+flvhXYAXwiyRnjHIwk6cUtGPo18Gdt8ZXtp4D3Are3+gPAZa28sy3T1m9LklZ/a1V9u6q+BswwxzN2JUnLZ6Q5/SRnJPkycAw4BHwVeK6qTrUmR4CNrbwReBKgrT8JvHG4fo5tho+1J8l0kunZ2dnFj0iSNK+RQr+qvlNV7wA2MTg7//Hl6lBV7auqqaqaWr9+/XIdRpK6tKird6rqOeBu4F3A2iRr2qpNwNFWPgpsBmjr3wA8O1w/xzaSpAkY5eqd9UnWtvIPAT8FPMIg/D/Ymu0C7mjlg22Ztv5zVVWt/op2dc+5wFbgi+MaiCRpYWsWbsI5wIF2pc0rgNuq6g+SPAzcmuRjwAPAza39zcAnk8wAxxlcsUNVPZTkNuBh4BRwVVV9Z7zDkSS9mAVDv6oeBN45R/3jzHH1TVV9C/jQPPu6Drhu8d2UJI2D38iVpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0JekjozyjNzNSe5O8nCSh5L8Yqs/K8mhJI+113WtPkluTDKT5MEk5w/ta1dr/1iSXfMdU5K0PEY50z8F/EpVnQdcBFyV5DzgauBwVW0FDrdlgEsYPPR8K7AHuAkGbxLAXuBCBo9Z3Hv6jUKSNBkLhn5VPVVVX2rl/ws8AmwEdgIHWrMDwGWtvBO4pQbuAdYmOQe4GDhUVcer6gRwCNgx1tFIkl7Uoub0k2xh8JD0e4ENVfVUW/U0sKGVNwJPDm12pNXNV//CY+xJMp1kenZ2djHdkyQtYOTQT/LDwH8Gfqmqvjm8rqoKqHF0qKr2VdVUVU2tX79+HLuUJDUjhX6SVzII/N+pqt9v1c+0aRva67FWfxTYPLT5plY3X70kaUJGuXonwM3AI1X1q0OrDgKnr8DZBdwxVP+RdhXPRcDJNg10F7A9ybr2Ae72VidJmpA1I7R5N/D3gT9O8uVW9y+A64HbkuwGngAub+vuBC4FZoDngSsBqup4kmuB+1q7a6rq+FhGIellbcvVn13pLrxsLBj6VfU/gMyzetsc7Qu4ap597Qf2L6aDkqTx8Ru5ktQRQ1+SOjLKnL4WaaXmH79+/ftX5LiSVg/P9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI547x1JI/O+9qufZ/qS1BHP9F9GVvIszDt8SqvDKM/I3Z/kWJKvDNWdleRQksfa67pWnyQ3JplJ8mCS84e22dXaP5Zk11zHkiQtr1Gmd34b2PGCuquBw1W1FTjclgEuAba2nz3ATTB4kwD2AhcCFwB7T79RSJImZ8HQr6ovAC98gPlO4EArHwAuG6q/pQbuAdYmOQe4GDhUVcer6gRwiO9/I5EkLbOXOqe/oaqeauWngQ2tvBF4cqjdkVY3X/33SbKHwV8JvOlNb3qJ3ZNevryCRkux5Kt3qqqAGkNfTu9vX1VNVdXU+vXrx7VbSRIv/Uz/mSTnVNVTbfrmWKs/Cmwearep1R0F3vOC+s+/xGPrB5DPBZZWh5ca+geBXcD17fWOofqfT3Irgw9tT7Y3hruAfzP04e124KMvvdvSyrvn8We5wqkWrTILhn6STzE4Sz87yREGV+FcD9yWZDfwBHB5a34ncCkwAzwPXAlQVceTXAvc19pdU1Uv/HBYWrSV+gvj1h99dkWOKy3VgqFfVR+eZ9W2OdoWcNU8+9kP7F9U7yRJY+VtGCSpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0JakjEw/9JDuSPJpkJsnVkz6+JPVsoqGf5Azg14FLgPOADyc5b5J9kKSeTfpM/wJgpqoer6q/AG4Fdk64D5LUrQUfjD5mG4Enh5aPABcON0iyB9jTFv8syaNLON7ZwJ8uYfvVprfxwgqN+V3fLf30pA8N/p67kI8vacx/fb4Vkw79BVXVPmDfOPaVZLqqpsaxr9Wgt/GCY+6FYx6fSU/vHAU2Dy1vanWSpAmYdOjfB2xNcm6SM4ErgIMT7oMkdWui0ztVdSrJzwN3AWcA+6vqoWU85FimiVaR3sYLjrkXjnlMUlXLsV9J0g8gv5ErSR0x9CWpI6s+9Be6rUOSVyX5dFt/b5Itk+/leI0w5n+S5OEkDyY5nGTea3ZXi1Fv35Hk7yapJKv+8r5Rxpzk8va7fijJ7066j+M2wr/tNyW5O8kD7d/3pSvRz3FJsj/JsSRfmWd9ktzY/ns8mOT8JR+0qlbtD4MPg78K/ChwJvBHwHkvaPOPgN9o5SuAT690vycw5p8EXtPKP9fDmFu71wFfAO4Bpla63xP4PW8FHgDWteUfWel+T2DM+4Cfa+XzgK+vdL+XOOafAM4HvjLP+kuB/woEuAi4d6nHXO1n+qPc1mEncKCVbwe2JckE+zhuC465qu6uqufb4j0Mvg+xmo16+45rgY8D35pk55bJKGP+B8CvV9UJgKo6NuE+jtsoYy7g9a38BuD/TLB/Y1dVXwCOv0iTncAtNXAPsDbJOUs55moP/blu67BxvjZVdQo4CbxxIr1bHqOMedhuBmcKq9mCY25/9m6uqs9OsmPLaJTf848BP5bkfya5J8mOifVueYwy5n8F/EySI8CdwC9MpmsrZrH/vy/oB+42DBqfJD8DTAF/Z6X7spySvAL4VeBnV7grk7aGwRTPexj8NfeFJH+zqp5b0V4trw8Dv11V/z7Ju4BPJnlbVf3lSndstVjtZ/qj3Nbhu22SrGHwJ+GzE+nd8hjpVhZJ3gf8S+ADVfXtCfVtuSw05tcBbwM+n+TrDOY+D67yD3NH+T0fAQ5W1f+rqq8B/5vBm8BqNcqYdwO3AVTV/wJezeBmbC9XY791zWoP/VFu63AQ2NXKHwQ+V+0TklVqwTEneSfwmwwCf7XP88ICY66qk1V1dlVtqaotDD7H+EBVTa9Md8dilH/b/4XBWT5JzmYw3fP4JDs5ZqOM+RvANoAkf4NB6M9OtJeTdRD4SLuK5yLgZFU9tZQdrurpnZrntg5JrgGmq+ogcDODPwFnGHxgcsXK9XjpRhzzvwV+GPi99pn1N6rqAyvW6SUaccwvKyOO+S5ge5KHge8A/7SqVu1fsSOO+VeA/5jklxl8qPuzq/kkLsmnGLxxn90+p9gLvBKgqn6DwecWlwIzwPPAlUs+5ir+7yVJWqTVPr0jSVoEQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR15P8Dws83Ervo6GAAAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S7kj1NQot706",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "outputId": "3544c3f4-07bb-47f9-ff39-46dfec1a70b1"
      },
      "source": [
        "\"{:0.2f}\".format(roc_auc_score(y_valid, y_hat)*100.0)"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'93.74'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    }
  ]
}