{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "**Important: This notebook will only work with fastai-0.7.x. Do not try to run any fastai-1.x code from this path in the repository because it will load fastai-0.7.x**" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import torch\n", "from torch.autograd import Variable\n", "from fastai.rnn_reg import *\n", "import numpy as np" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "def embedded_dropout(embed, words, dropout=0.1, scale=None):\n", " \"\"\" Applies dropout in the embedding layer by zeroing out some elements of the embedding vector.\n", " Uses the dropout_mask custom layer to achieve this.\n", "\n", " Args:\n", " embed (torch.nn.Embedding): An embedding torch layer\n", " words (torch.nn.Variable): A torch variable\n", " dropout (float): dropout fraction to apply to the embedding weights\n", " scale (float): additional scaling to apply to the modified embedding weights\n", "\n", " Returns:\n", " tensor of size: (batch_size x seq_length x embedding_size)\n", "\n", " Example:\n", "\n", " >> embed = torch.nn.Embedding(10,3)\n", " >> words = Variable(torch.LongTensor([[1,2,4,5] ,[4,3,2,9]]))\n", " >> words.size()\n", " (2,4)\n", "\n", " >> dropout_out_ = embedded_dropout(embed, words, dropout=0.40)\n", " >> dropout_out_\n", " Variable containing:\n", " (0 ,.,.) =\n", " 1.2549 1.8230 1.9367\n", " 0.0000 -0.0000 0.0000\n", " 2.2540 -0.1299 1.5448\n", " 0.0000 -0.0000 -0.0000\n", "\n", " (1 ,.,.) =\n", " 2.2540 -0.1299 1.5448\n", " -4.0457 2.4815 -0.2897\n", " 0.0000 -0.0000 0.0000\n", " 1.8796 -0.4022 3.8773\n", " [torch.FloatTensor of size 2x4x3]\n", " \"\"\"\n", "\n", " if dropout:\n", " mask = Variable(dropout_mask(embed.weight.data, (embed.weight.size(0), 1), dropout))\n", " masked_embed_weight = mask * embed.weight\n", " else: masked_embed_weight = embed.weight\n", " if scale: masked_embed_weight = scale * masked_embed_weight\n", "\n", " padding_idx = embed.padding_idx\n", " if padding_idx is None: padding_idx = -1\n", " X = embed._backend.Embedding.apply(words, masked_embed_weight,\n", " padding_idx, embed.max_norm, embed.norm_type,\n", " embed.scale_grad_by_freq, embed.sparse\n", " )\n", " return X\n", "\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Test 1" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### Initialize embedding matrix and input" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "embed = torch.nn.Embedding(10,3)\n", "words = torch.autograd.Variable(torch.LongTensor([[1,2,4,5] ,[4,3,2,9]]))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### propagate the input via the old method (embedded_dropout)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "Variable containing:\n", "(0 ,.,.) = \n", " -0.0000 -0.0000 0.0000\n", " -0.0000 -0.0000 0.0000\n", " -0.8727 -1.3131 -1.2833\n", " -0.0000 -0.0000 -0.0000\n", "\n", "(1 ,.,.) = \n", " -0.8727 -1.3131 -1.2833\n", " -0.7425 -4.4689 0.9931\n", " -0.0000 -0.0000 0.0000\n", " -1.3907 -0.6359 -3.4528\n", "[torch.FloatTensor of size 2x4x3]" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "torch.manual_seed(88123)\n", "dropout_out_old = embedded_dropout(embed, words, dropout=0.40)\n", "dropout_out_old" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### propagate the input via the forward method in the new layer (EmbeddingDropout)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "Variable containing:\n", "(0 ,.,.) = \n", " -0.0000 -0.0000 0.0000\n", " -0.0000 -0.0000 0.0000\n", " -0.8727 -1.3131 -1.2833\n", " -0.0000 -0.0000 -0.0000\n", "\n", "(1 ,.,.) = \n", " -0.8727 -1.3131 -1.2833\n", " -0.7425 -4.4689 0.9931\n", " -0.0000 -0.0000 0.0000\n", " -1.3907 -0.6359 -3.4528\n", "[torch.FloatTensor of size 2x4x3]" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "torch.manual_seed(88123)\n", "embed_dropout_layer = EmbeddingDropout(embed)\n", "dropout_out_new = embed_dropout_layer(words, dropout=0.4)\n", "dropout_out_new" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "None\n" ] } ], "source": [ "print(np.testing.assert_array_equal(to_np(dropout_out_old),to_np(dropout_out_new)))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Test 2" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### Initialize embedding and matrix" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "embed = torch.nn.Embedding(10,7)\n", "words = torch.autograd.Variable(torch.LongTensor([[1,2,4,5,2,8] ,[4,3,2,9,7,6]]))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### get the input by propagating via the old method" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "Variable containing:\n", "(0 ,.,.) = \n", " -0.1285 0.7315 0.1410 -1.6623 -3.7160 -1.1500 1.9748\n", " -0.0000 -0.0000 -0.0000 -0.0000 0.0000 -0.0000 0.0000\n", " -1.4062 -0.2985 -1.2622 -1.2019 -3.9978 2.7126 4.9408\n", " 0.0000 -0.0000 -0.0000 -0.0000 -0.0000 -0.0000 0.0000\n", " -0.0000 -0.0000 -0.0000 -0.0000 0.0000 -0.0000 0.0000\n", " -0.0000 0.0000 0.0000 -0.0000 -0.0000 0.0000 0.0000\n", "\n", "(1 ,.,.) = \n", " -1.4062 -0.2985 -1.2622 -1.2019 -3.9978 2.7126 4.9408\n", " 0.0000 -0.0000 0.0000 0.0000 -0.0000 0.0000 -0.0000\n", " -0.0000 -0.0000 -0.0000 -0.0000 0.0000 -0.0000 0.0000\n", " -6.4958 2.6005 -2.2331 -7.8277 -0.9653 3.3815 1.9252\n", " -0.0000 -0.0000 0.0000 0.0000 -0.0000 0.0000 0.0000\n", " 0.0000 0.0000 0.0000 0.0000 -0.0000 -0.0000 0.0000\n", "[torch.FloatTensor of size 2x6x7]" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "torch.manual_seed(7123)\n", "dropout_out_old = embedded_dropout(embed, words, dropout=0.64)\n", "dropout_out_old" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### get the input by propagating input via the embedding layer" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "Variable containing:\n", "(0 ,.,.) = \n", " -0.1285 0.7315 0.1410 -1.6623 -3.7160 -1.1500 1.9748\n", " -0.0000 -0.0000 -0.0000 -0.0000 0.0000 -0.0000 0.0000\n", " -1.4062 -0.2985 -1.2622 -1.2019 -3.9978 2.7126 4.9408\n", " 0.0000 -0.0000 -0.0000 -0.0000 -0.0000 -0.0000 0.0000\n", " -0.0000 -0.0000 -0.0000 -0.0000 0.0000 -0.0000 0.0000\n", " -0.0000 0.0000 0.0000 -0.0000 -0.0000 0.0000 0.0000\n", "\n", "(1 ,.,.) = \n", " -1.4062 -0.2985 -1.2622 -1.2019 -3.9978 2.7126 4.9408\n", " 0.0000 -0.0000 0.0000 0.0000 -0.0000 0.0000 -0.0000\n", " -0.0000 -0.0000 -0.0000 -0.0000 0.0000 -0.0000 0.0000\n", " -6.4958 2.6005 -2.2331 -7.8277 -0.9653 3.3815 1.9252\n", " -0.0000 -0.0000 0.0000 0.0000 -0.0000 0.0000 0.0000\n", " 0.0000 0.0000 0.0000 0.0000 -0.0000 -0.0000 0.0000\n", "[torch.FloatTensor of size 2x6x7]" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "torch.manual_seed(7123)\n", "embed_dropout_layer = EmbeddingDropout(embed)\n", "dropout_out_new = embed_dropout_layer(words, dropout=0.64)\n", "dropout_out_new" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "None\n" ] } ], "source": [ "print(np.testing.assert_array_equal(to_np(dropout_out_old),to_np(dropout_out_new)))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" } }, "nbformat": 4, "nbformat_minor": 2 }