{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "substantial-albuquerque",
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "from nbdev.showdoc import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loss Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "present-authentication",
   "metadata": {},
   "outputs": [],
   "source": [
    "import timm\n",
    "import torch\n",
    "import torch.nn.functional as F\n",
    "from timm.loss import LabelSmoothingCrossEntropy, SoftTargetCrossEntropy\n",
    "from timm.data.mixup import mixup_target"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "necessary-workplace",
   "metadata": {},
   "source": [
    "## LabelSmoothingCrossEntropy"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "coated-cosmetic",
   "metadata": {},
   "source": [
    "Same as NLL loss with label smoothing. Label smoothing increases loss when the model is correct `x` and decreases loss when model is incorrect `x_i`. Use this to not punish model as harshly, such as when incorrect labels are expected. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "mighty-maria",
   "metadata": {},
   "outputs": [],
   "source": [
    "x = torch.eye(2)\n",
    "x_i = 1 - x\n",
    "y = torch.arange(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "duplicate-brief",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor(0.3133), tensor(1.3133))"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "LabelSmoothingCrossEntropy(0.0)(x,y),LabelSmoothingCrossEntropy(0.0)(x_i,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "attended-astronomy",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor(0.3633), tensor(1.2633))"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "LabelSmoothingCrossEntropy(0.1)(x,y),LabelSmoothingCrossEntropy(0.1)(x_i,y)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "promotional-tours",
   "metadata": {},
   "source": [
    "## SoftTargetCrossEntropy"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "martial-stereo",
   "metadata": {},
   "source": [
    "`log_softmax` family loss function to be used with mixup.  Use __[mixup_target](https://github.com/rwightman/pytorch-image-models/blob/9a38416fbdfd0d38e6922eee5d664e8ec7fbc356/timm/data/mixup.py#L22)__ to add label smoothing and adjust the amount of mixing of the target labels. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "complete-stage",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[[0., 1., 0., 0., 1.]],\n",
       " \n",
       "         [[1., 1., 1., 1., 1.]]], device='cuda:0'),\n",
       " tensor([[0.0000, 0.7000, 0.0000, 0.0000, 0.3000],\n",
       "         [0.0000, 0.3000, 0.0000, 0.0000, 0.7000]], device='cuda:0'))"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x=torch.tensor([[[0,1.,0,0,1.]],[[1.,1.,1.,1.,1.]]],device='cuda')\n",
    "y=mixup_target(torch.tensor([1,4],device='cuda'),5, lam=0.7)\n",
    "x,y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "equipped-geneva",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor(1.1326, device='cuda:0'), tensor(1.6094, device='cuda:0'))"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SoftTargetCrossEntropy()(x[0],y),SoftTargetCrossEntropy()(x[1],y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "spread-cleaners",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Converted 00_core.ipynb.\n",
      "Converted 01_training.ipynb.\n",
      "Converted 19_loss.cross_entropy.ipynb.\n",
      "Converted index.ipynb.\n"
     ]
    }
   ],
   "source": [
    "#hide\n",
    "from nbdev.export import notebook2script\n",
    "notebook2script()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}