{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "04ae7e4c-7642-49a3-827c-452df0b17108",
   "metadata": {},
   "source": [
    "# 1. Using the trained model, find semantically similar words for other input words. Can you improve the results by tuning hyperparameters?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "id": "b12770d2-c299-49a7-916c-a9e85c4673dd",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import time\n",
    "import collections\n",
    "import math\n",
    "import os\n",
    "import random\n",
    "import torch\n",
    "import warnings\n",
    "import sys\n",
    "import pandas as pd\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "sys.path.append('/home/jovyan/work/d2l_solutions/notebooks/exercises/d2l_utils/')\n",
    "import d2l\n",
    "from torchsummary import summary\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "#@save\n",
    "d2l.DATA_HUB['ptb'] = (d2l.DATA_URL + 'ptb.zip',\n",
    "                       '319d85e578af0cdc590547f26231e4e31cdf1e42')\n",
    "#@save\n",
    "class RandomGenerator:\n",
    "    \"\"\"Randomly draw among {1, ..., n} according to n sampling weights.\"\"\"\n",
    "    def __init__(self, sampling_weights,k=10000):\n",
    "        # Exclude\n",
    "        self.population = list(range(1, len(sampling_weights) + 1))\n",
    "        self.sampling_weights = sampling_weights\n",
    "        self.candidates = []\n",
    "        self.i = 0\n",
    "        self.k = k\n",
    "\n",
    "    def draw(self):\n",
    "        if self.i == len(self.candidates):\n",
    "            # Cache `k` random sampling results\n",
    "            self.candidates = random.choices(\n",
    "                self.population, self.sampling_weights, k=self.k)\n",
    "            self.i = 0\n",
    "        self.i += 1\n",
    "        return self.candidates[self.i - 1]\n",
    "    \n",
    "#@save\n",
    "def subsample(sentences, vocab,flag=True):\n",
    "    \"\"\"Subsample high-frequency words.\"\"\"\n",
    "    # Exclude unknown tokens ('<unk>')\n",
    "    sentences = [[token for token in line if vocab[token] != vocab.unk]\n",
    "                 for line in sentences]\n",
    "    counter = collections.Counter([\n",
    "        token for line in sentences for token in line])\n",
    "    num_tokens = sum(counter.values())\n",
    "\n",
    "    # Return True if `token` is kept during subsampling\n",
    "    def keep(token):\n",
    "        return(random.uniform(0, 1) <\n",
    "               math.sqrt(1e-4 / counter[token] * num_tokens))\n",
    "    if flag:\n",
    "        return ([[token for token in line if keep(token)] for line in sentences],\n",
    "            counter)\n",
    "    return (sentences,counter)\n",
    "\n",
    "#@save\n",
    "def get_centers_and_contexts(corpus, max_window_size):\n",
    "    \"\"\"Return center words and context words in skip-gram.\"\"\"\n",
    "    centers, contexts = [], []\n",
    "    for line in corpus:\n",
    "        # To form a \"center word--context word\" pair, each sentence needs to\n",
    "        # have at least 2 words\n",
    "        if len(line) < 2:\n",
    "            continue\n",
    "        centers += line\n",
    "        for i in range(len(line)):  # Context window centered at `i`\n",
    "            window_size = random.randint(1, max_window_size)\n",
    "            indices = list(range(max(0, i - window_size),\n",
    "                                 min(len(line), i + 1 + window_size)))\n",
    "            # Exclude the center word from the context words\n",
    "            indices.remove(i)\n",
    "            contexts.append([line[idx] for idx in indices])\n",
    "    return centers, contexts\n",
    "\n",
    "#@save\n",
    "def read_ptb():\n",
    "    \"\"\"Load the PTB dataset into a list of text lines.\"\"\"\n",
    "    data_dir = d2l.download_extract('ptb')\n",
    "    # Read the training set\n",
    "    with open(os.path.join(data_dir, 'ptb.train.txt')) as f:\n",
    "        raw_text = f.read()\n",
    "    return [line.split() for line in raw_text.split('\\n')]\n",
    "\n",
    "#@save\n",
    "def get_negatives(all_contexts, vocab, counter, K, k=10000):\n",
    "    \"\"\"Return noise words in negative sampling.\"\"\"\n",
    "    # Sampling weights for words with indices 1, 2, ... (index 0 is the\n",
    "    # excluded unknown token) in the vocabulary\n",
    "    sampling_weights = [counter[vocab.to_tokens(i)]**0.75\n",
    "                        for i in range(1, len(vocab))]\n",
    "    all_negatives, generator = [], RandomGenerator(sampling_weights,k)\n",
    "    for contexts in all_contexts:\n",
    "        negatives = []\n",
    "        while len(negatives) < len(contexts) * K:\n",
    "            neg = generator.draw()\n",
    "            # Noise words cannot be context words\n",
    "            if neg not in contexts:\n",
    "                negatives.append(neg)\n",
    "        all_negatives.append(negatives)\n",
    "    return all_negatives\n",
    "\n",
    "#@save\n",
    "def batchify(data):\n",
    "    \"\"\"Return a minibatch of examples for skip-gram with negative sampling.\"\"\"\n",
    "    max_len = max(len(c) + len(n) for _, c, n in data)\n",
    "    centers, contexts_negatives, masks, labels = [], [], [], []\n",
    "    for center, context, negative in data:\n",
    "        cur_len = len(context) + len(negative)\n",
    "        centers += [center]\n",
    "        contexts_negatives += [context + negative + [0] * (max_len - cur_len)]\n",
    "        masks += [[1] * cur_len + [0] * (max_len - cur_len)]\n",
    "        labels += [[1] * len(context) + [0] * (max_len - len(context))]\n",
    "    return (torch.tensor(centers).reshape((-1, 1)), torch.tensor(\n",
    "        contexts_negatives), torch.tensor(masks), torch.tensor(labels))\n",
    "\n",
    "#@save\n",
    "def load_data_ptb(batch_size, max_window_size, num_noise_words, flag=True, k=10000):\n",
    "    \"\"\"Download the PTB dataset and then load it into memory.\"\"\"\n",
    "    # num_workers = d2l.get_dataloader_workers()\n",
    "    sentences = read_ptb()\n",
    "    vocab = d2l.Vocab(sentences, min_freq=10)\n",
    "    subsampled, counter = subsample(sentences, vocab, flag)\n",
    "    corpus = [vocab[line] for line in subsampled]\n",
    "    all_centers, all_contexts = get_centers_and_contexts(\n",
    "        corpus, max_window_size)\n",
    "    all_negatives = get_negatives(\n",
    "        all_contexts, vocab, counter, num_noise_words, k=k)\n",
    "\n",
    "    class PTBDataset(torch.utils.data.Dataset):\n",
    "        def __init__(self, centers, contexts, negatives):\n",
    "            assert len(centers) == len(contexts) == len(negatives)\n",
    "            self.centers = centers\n",
    "            self.contexts = contexts\n",
    "            self.negatives = negatives\n",
    "\n",
    "        def __getitem__(self, index):\n",
    "            return (self.centers[index], self.contexts[index],\n",
    "                    self.negatives[index])\n",
    "\n",
    "        def __len__(self):\n",
    "            return len(self.centers)\n",
    "\n",
    "    dataset = PTBDataset(all_centers, all_contexts, all_negatives)\n",
    "\n",
    "    data_iter = torch.utils.data.DataLoader(dataset, batch_size, shuffle=True,\n",
    "                                      collate_fn=batchify)\n",
    "    return data_iter, vocab\n",
    "\n",
    "def skip_gram(center, contexts_and_negatives, embed_v, embed_u):\n",
    "    v = embed_v(center)\n",
    "    u = embed_u(contexts_and_negatives)\n",
    "    pred = torch.bmm(v, u.permute(0, 2, 1))\n",
    "    return pred\n",
    "\n",
    "class SigmoidBCELoss(nn.Module):\n",
    "    # Binary cross-entropy loss with masking\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "\n",
    "    def forward(self, inputs, target, mask=None):\n",
    "        out = nn.functional.binary_cross_entropy_with_logits(\n",
    "            inputs, target, weight=mask, reduction=\"none\")\n",
    "        return out.mean(dim=1)\n",
    "\n",
    "def train(net, data_iter, lr, num_epochs, device='cpu'):\n",
    "    def init_weights(module):\n",
    "        if type(module) == nn.Embedding:\n",
    "            nn.init.xavier_uniform_(module.weight)\n",
    "    net.apply(init_weights)\n",
    "    net = net.to(device)\n",
    "    optimizer = torch.optim.Adam(net.parameters(), lr=lr)\n",
    "    animator = d2l.Animator(xlabel='epoch', ylabel='loss',\n",
    "                            xlim=[1, num_epochs])\n",
    "    # Sum of normalized losses, no. of normalized losses\n",
    "    metric = d2l.Accumulator(2)\n",
    "    loss = SigmoidBCELoss()\n",
    "    for epoch in range(num_epochs):\n",
    "        timer, num_batches = d2l.Timer(), len(data_iter)\n",
    "        for i, batch in enumerate(data_iter):\n",
    "            optimizer.zero_grad()\n",
    "            center, context_negative, mask, label = [\n",
    "                data.to(device) for data in batch]\n",
    "\n",
    "            pred = skip_gram(center, context_negative, net[0], net[1])\n",
    "            l = (loss(pred.reshape(label.shape).float(), label.float(), mask)\n",
    "                     / mask.sum(axis=1) * mask.shape[1])\n",
    "            l.sum().backward()\n",
    "            optimizer.step()\n",
    "            metric.add(l.sum(), l.numel())\n",
    "            if (i + 1) % (num_batches // 5) == 0 or i == num_batches - 1:\n",
    "                animator.add(epoch + (i + 1) / num_batches,\n",
    "                             (metric[0] / metric[1],))\n",
    "    print(f'loss {metric[0] / metric[1]:.3f}, '\n",
    "          f'{metric[1] / timer.stop():.1f} tokens/sec on {str(device)}')\n",
    "    return metric[0] / metric[1]\n",
    "\n",
    "def get_similar_tokens(query_token, k, embed):\n",
    "    W = embed.weight.data\n",
    "    x = W[vocab[query_token]]\n",
    "    # Compute the cosine similarity. Add 1e-9 for numerical stability\n",
    "    cos = torch.mv(W, x) / torch.sqrt(torch.sum(W * W, dim=1) *\n",
    "                                      torch.sum(x * x) + 1e-9)\n",
    "    topk = torch.topk(cos, k=k+1)[1].cpu().numpy().astype('int32')\n",
    "    for i in topk[1:]:  # Remove the input words\n",
    "        print(f'cosine sim={float(cos[i]):.3f}: {vocab.to_tokens(i)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "eaa1e620-8a88-4b7a-bc0e-2a020db974c2",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss 0.410, 55995.3 tokens/sec on cpu\n"
     ]
    },
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       "  \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<svg height=\"180.25pt\" version=\"1.1\" viewBox=\"0 0 251.325 180.25\" width=\"251.325pt\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       " <metadata>\n",
       "  <rdf:RDF xmlns:cc=\"http://creativecommons.org/ns#\" xmlns:dc=\"http://purl.org/dc/elements/1.1/\" xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\">\n",
       "   <cc:Work>\n",
       "    <dc:type rdf:resource=\"http://purl.org/dc/dcmitype/StillImage\"/>\n",
       "    <dc:date>2023-09-28T12:26:13.031876</dc:date>\n",
       "    <dc:format>image/svg+xml</dc:format>\n",
       "    <dc:creator>\n",
       "     <cc:Agent>\n",
       "      <dc:title>Matplotlib v3.4.0, https://matplotlib.org/</dc:title>\n",
       "     </cc:Agent>\n",
       "    </dc:creator>\n",
       "   </cc:Work>\n",
       "  </rdf:RDF>\n",
       " </metadata>\n",
       " <defs>\n",
       "  <style type=\"text/css\">*{stroke-linecap:butt;stroke-linejoin:round;}</style>\n",
       " </defs>\n",
       " <g id=\"figure_1\">\n",
       "  <g id=\"patch_1\">\n",
       "   <path d=\"M 0 180.25 \n",
       "L 251.325 180.25 \n",
       "L 251.325 0 \n",
       "L 0 0 \n",
       "z\n",
       "\" style=\"fill:#ffffff;\"/>\n",
       "  </g>\n",
       "  <g id=\"axes_1\">\n",
       "   <g id=\"patch_2\">\n",
       "    <path d=\"M 46.325 145.8 \n",
       "L 241.625 145.8 \n",
       "L 241.625 7.2 \n",
       "L 46.325 7.2 \n",
       "z\n",
       "\" style=\"fill:#ffffff;\"/>\n",
       "   </g>\n",
       "   <g id=\"matplotlib.axis_1\">\n",
       "    <g id=\"xtick_1\">\n",
       "     <g id=\"line2d_1\">\n",
       "      <path clip-path=\"url(#pb8a768a74a)\" d=\"M 46.325 145.8 \n",
       "L 46.325 7.2 \n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\n",
       "     </g>\n",
       "     <g id=\"line2d_2\">\n",
       "      <defs>\n",
       "       <path d=\"M 0 0 \n",
       "L 0 3.5 \n",
       "\" id=\"m0569e64c40\" style=\"stroke:#000000;stroke-width:0.8;\"/>\n",
       "      </defs>\n",
       "      <g>\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"46.325\" xlink:href=\"#m0569e64c40\" y=\"145.8\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_1\">\n",
       "      <!-- 1 -->\n",
       "      <g transform=\"translate(43.825 159.675)scale(0.1 -0.1)\">\n",
       "       <defs>\n",
       "        <path d=\"M 1400 3600 \n",
       "Q 1075 3275 575 2975 \n",
       "L 575 3450 \n",
       "Q 1200 3875 1600 4450 \n",
       "L 1900 4450 \n",
       "L 1900 150 \n",
       "L 1400 150 \n",
       "L 1400 3600 \n",
       "z\n",
       "\" id=\"SimHei-31\" transform=\"scale(0.015625)\"/>\n",
       "       </defs>\n",
       "       <use xlink:href=\"#SimHei-31\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"xtick_2\">\n",
       "     <g id=\"line2d_3\">\n",
       "      <path clip-path=\"url(#pb8a768a74a)\" d=\"M 95.15 145.8 \n",
       "L 95.15 7.2 \n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\n",
       "     </g>\n",
       "     <g id=\"line2d_4\">\n",
       "      <g>\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"95.15\" xlink:href=\"#m0569e64c40\" y=\"145.8\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_2\">\n",
       "      <!-- 2 -->\n",
       "      <g transform=\"translate(92.65 159.675)scale(0.1 -0.1)\">\n",
       "       <defs>\n",
       "        <path d=\"M 300 250 \n",
       "Q 325 625 650 925 \n",
       "Q 975 1225 1475 1862 \n",
       "Q 1975 2500 2125 2850 \n",
       "Q 2275 3200 2237 3450 \n",
       "Q 2200 3700 2000 3862 \n",
       "Q 1800 4025 1537 4000 \n",
       "Q 1275 3975 1037 3800 \n",
       "Q 800 3625 675 3275 \n",
       "L 200 3350 \n",
       "Q 400 3925 712 4187 \n",
       "Q 1025 4450 1450 4475 \n",
       "Q 1700 4500 1900 4462 \n",
       "Q 2100 4425 2312 4287 \n",
       "Q 2525 4150 2662 3875 \n",
       "Q 2800 3600 2762 3212 \n",
       "Q 2725 2825 2375 2287 \n",
       "Q 2025 1750 1025 600 \n",
       "L 2825 600 \n",
       "L 2825 150 \n",
       "L 300 150 \n",
       "L 300 250 \n",
       "z\n",
       "\" id=\"SimHei-32\" transform=\"scale(0.015625)\"/>\n",
       "       </defs>\n",
       "       <use xlink:href=\"#SimHei-32\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"xtick_3\">\n",
       "     <g id=\"line2d_5\">\n",
       "      <path clip-path=\"url(#pb8a768a74a)\" d=\"M 143.975 145.8 \n",
       "L 143.975 7.2 \n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\n",
       "     </g>\n",
       "     <g id=\"line2d_6\">\n",
       "      <g>\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"143.975\" xlink:href=\"#m0569e64c40\" y=\"145.8\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_3\">\n",
       "      <!-- 3 -->\n",
       "      <g transform=\"translate(141.475 159.675)scale(0.1 -0.1)\">\n",
       "       <defs>\n",
       "        <path d=\"M 250 1225 \n",
       "L 700 1300 \n",
       "Q 800 975 1025 762 \n",
       "Q 1250 550 1587 562 \n",
       "Q 1925 575 2125 837 \n",
       "Q 2325 1100 2300 1437 \n",
       "Q 2275 1775 2037 1962 \n",
       "Q 1800 2150 1275 2225 \n",
       "L 1275 2550 \n",
       "Q 1800 2600 2037 2825 \n",
       "Q 2275 3050 2250 3412 \n",
       "Q 2225 3775 1925 3937 \n",
       "Q 1625 4100 1287 3975 \n",
       "Q 950 3850 750 3275 \n",
       "L 300 3350 \n",
       "Q 450 3800 712 4100 \n",
       "Q 975 4400 1425 4450 \n",
       "Q 1875 4500 2212 4337 \n",
       "Q 2550 4175 2687 3837 \n",
       "Q 2825 3500 2725 3100 \n",
       "Q 2625 2700 2150 2400 \n",
       "Q 2500 2250 2687 1950 \n",
       "Q 2875 1650 2812 1162 \n",
       "Q 2750 675 2375 375 \n",
       "Q 2000 75 1525 87 \n",
       "Q 1050 100 700 387 \n",
       "Q 350 675 250 1225 \n",
       "z\n",
       "\" id=\"SimHei-33\" transform=\"scale(0.015625)\"/>\n",
       "       </defs>\n",
       "       <use xlink:href=\"#SimHei-33\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"xtick_4\">\n",
       "     <g id=\"line2d_7\">\n",
       "      <path clip-path=\"url(#pb8a768a74a)\" d=\"M 192.8 145.8 \n",
       "L 192.8 7.2 \n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\n",
       "     </g>\n",
       "     <g id=\"line2d_8\">\n",
       "      <g>\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"192.8\" xlink:href=\"#m0569e64c40\" y=\"145.8\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_4\">\n",
       "      <!-- 4 -->\n",
       "      <g transform=\"translate(190.3 159.675)scale(0.1 -0.1)\">\n",
       "       <defs>\n",
       "        <path d=\"M 2000 1100 \n",
       "L 75 1100 \n",
       "L 75 1525 \n",
       "L 2100 4450 \n",
       "L 2475 4450 \n",
       "L 2475 1525 \n",
       "L 3075 1525 \n",
       "L 3075 1100 \n",
       "L 2475 1100 \n",
       "L 2475 150 \n",
       "L 2000 150 \n",
       "L 2000 1100 \n",
       "z\n",
       "M 2000 1525 \n",
       "L 2000 3500 \n",
       "L 600 1525 \n",
       "L 2000 1525 \n",
       "z\n",
       "\" id=\"SimHei-34\" transform=\"scale(0.015625)\"/>\n",
       "       </defs>\n",
       "       <use xlink:href=\"#SimHei-34\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"xtick_5\">\n",
       "     <g id=\"line2d_9\">\n",
       "      <path clip-path=\"url(#pb8a768a74a)\" d=\"M 241.625 145.8 \n",
       "L 241.625 7.2 \n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\n",
       "     </g>\n",
       "     <g id=\"line2d_10\">\n",
       "      <g>\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"241.625\" xlink:href=\"#m0569e64c40\" y=\"145.8\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_5\">\n",
       "      <!-- 5 -->\n",
       "      <g transform=\"translate(239.125 159.675)scale(0.1 -0.1)\">\n",
       "       <defs>\n",
       "        <path d=\"M 550 1325 \n",
       "Q 725 650 1150 575 \n",
       "Q 1575 500 1837 662 \n",
       "Q 2100 825 2212 1087 \n",
       "Q 2325 1350 2312 1675 \n",
       "Q 2300 2000 2137 2225 \n",
       "Q 1975 2450 1725 2525 \n",
       "Q 1475 2600 1162 2525 \n",
       "Q 850 2450 650 2175 \n",
       "L 225 2225 \n",
       "Q 275 2375 700 4375 \n",
       "L 2675 4375 \n",
       "L 2675 3925 \n",
       "L 1075 3925 \n",
       "Q 950 3250 825 2850 \n",
       "Q 1200 3025 1525 3012 \n",
       "Q 1850 3000 2150 2862 \n",
       "Q 2450 2725 2587 2487 \n",
       "Q 2725 2250 2787 2012 \n",
       "Q 2850 1775 2837 1500 \n",
       "Q 2825 1225 2725 937 \n",
       "Q 2625 650 2425 462 \n",
       "Q 2225 275 1937 162 \n",
       "Q 1650 50 1275 75 \n",
       "Q 900 100 562 350 \n",
       "Q 225 600 100 1200 \n",
       "L 550 1325 \n",
       "z\n",
       "\" id=\"SimHei-35\" transform=\"scale(0.015625)\"/>\n",
       "       </defs>\n",
       "       <use xlink:href=\"#SimHei-35\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"text_6\">\n",
       "     <!-- epoch -->\n",
       "     <g transform=\"translate(131.475 171.8)scale(0.1 -0.1)\">\n",
       "      <defs>\n",
       "       <path d=\"M 2850 1075 \n",
       "Q 2800 625 2450 350 \n",
       "Q 2100 75 1625 75 \n",
       "Q 1025 75 637 462 \n",
       "Q 250 850 250 1525 \n",
       "Q 250 2200 637 2587 \n",
       "Q 1025 2975 1625 2975 \n",
       "Q 2150 2975 2487 2637 \n",
       "Q 2825 2300 2825 1525 \n",
       "L 800 1525 \n",
       "Q 800 975 1037 750 \n",
       "Q 1275 525 1625 525 \n",
       "Q 1900 525 2075 662 \n",
       "Q 2250 800 2300 1075 \n",
       "L 2850 1075 \n",
       "z\n",
       "M 2250 1925 \n",
       "Q 2200 2275 2025 2412 \n",
       "Q 1850 2550 1575 2550 \n",
       "Q 1325 2550 1125 2412 \n",
       "Q 925 2275 825 1925 \n",
       "L 2250 1925 \n",
       "z\n",
       "\" id=\"SimHei-65\" transform=\"scale(0.015625)\"/>\n",
       "       <path d=\"M 2825 1525 \n",
       "Q 2825 800 2475 425 \n",
       "Q 2125 50 1625 50 \n",
       "Q 1350 50 1150 162 \n",
       "Q 950 275 800 500 \n",
       "L 800 -800 \n",
       "L 300 -800 \n",
       "L 300 2925 \n",
       "L 800 2925 \n",
       "L 800 2550 \n",
       "Q 950 2775 1150 2875 \n",
       "Q 1350 2975 1625 2975 \n",
       "Q 2125 2975 2475 2612 \n",
       "Q 2825 2250 2825 1525 \n",
       "z\n",
       "M 2275 1525 \n",
       "Q 2275 2000 2087 2262 \n",
       "Q 1900 2525 1525 2525 \n",
       "Q 1225 2525 1012 2262 \n",
       "Q 800 2000 800 1525 \n",
       "Q 800 1050 1012 775 \n",
       "Q 1225 500 1525 500 \n",
       "Q 1900 500 2087 775 \n",
       "Q 2275 1050 2275 1525 \n",
       "z\n",
       "\" id=\"SimHei-70\" transform=\"scale(0.015625)\"/>\n",
       "       <path d=\"M 2925 1525 \n",
       "Q 2925 875 2525 475 \n",
       "Q 2125 75 1575 75 \n",
       "Q 1025 75 625 475 \n",
       "Q 225 875 225 1525 \n",
       "Q 225 2175 625 2575 \n",
       "Q 1025 2975 1575 2975 \n",
       "Q 2125 2975 2525 2575 \n",
       "Q 2925 2175 2925 1525 \n",
       "z\n",
       "M 2375 1525 \n",
       "Q 2375 2025 2125 2275 \n",
       "Q 1875 2525 1575 2525 \n",
       "Q 1275 2525 1025 2275 \n",
       "Q 775 2025 775 1525 \n",
       "Q 775 1025 1025 775 \n",
       "Q 1275 525 1575 525 \n",
       "Q 1875 525 2125 775 \n",
       "Q 2375 1025 2375 1525 \n",
       "z\n",
       "\" id=\"SimHei-6f\" transform=\"scale(0.015625)\"/>\n",
       "       <path d=\"M 2850 1250 \n",
       "Q 2850 725 2487 400 \n",
       "Q 2125 75 1575 75 \n",
       "Q 1025 75 625 462 \n",
       "Q 225 850 225 1525 \n",
       "Q 225 2200 625 2587 \n",
       "Q 1025 2975 1575 2975 \n",
       "Q 2125 2975 2450 2687 \n",
       "Q 2775 2400 2775 2000 \n",
       "L 2225 2000 \n",
       "Q 2200 2300 2012 2412 \n",
       "Q 1825 2525 1575 2525 \n",
       "Q 1275 2525 1025 2287 \n",
       "Q 775 2050 775 1525 \n",
       "Q 775 1000 1025 762 \n",
       "Q 1275 525 1575 525 \n",
       "Q 1900 525 2100 700 \n",
       "Q 2300 875 2300 1250 \n",
       "L 2850 1250 \n",
       "z\n",
       "\" id=\"SimHei-63\" transform=\"scale(0.015625)\"/>\n",
       "       <path d=\"M 2800 125 \n",
       "L 2300 125 \n",
       "L 2300 1925 \n",
       "Q 2300 2225 2150 2400 \n",
       "Q 2000 2575 1750 2575 \n",
       "Q 1425 2575 1137 2237 \n",
       "Q 850 1900 850 1400 \n",
       "L 850 125 \n",
       "L 350 125 \n",
       "L 350 4400 \n",
       "L 850 4400 \n",
       "L 850 2400 \n",
       "Q 1050 2675 1287 2825 \n",
       "Q 1525 2975 1900 2975 \n",
       "Q 2350 2975 2575 2725 \n",
       "Q 2800 2475 2800 2100 \n",
       "L 2800 125 \n",
       "z\n",
       "\" id=\"SimHei-68\" transform=\"scale(0.015625)\"/>\n",
       "      </defs>\n",
       "      <use xlink:href=\"#SimHei-65\"/>\n",
       "      <use x=\"50\" xlink:href=\"#SimHei-70\"/>\n",
       "      <use x=\"100\" xlink:href=\"#SimHei-6f\"/>\n",
       "      <use x=\"150\" xlink:href=\"#SimHei-63\"/>\n",
       "      <use x=\"200\" xlink:href=\"#SimHei-68\"/>\n",
       "     </g>\n",
       "    </g>\n",
       "   </g>\n",
       "   <g id=\"matplotlib.axis_2\">\n",
       "    <g id=\"ytick_1\">\n",
       "     <g id=\"line2d_11\">\n",
       "      <path clip-path=\"url(#pb8a768a74a)\" d=\"M 46.325 114.814029 \n",
       "L 241.625 114.814029 \n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\n",
       "     </g>\n",
       "     <g id=\"line2d_12\">\n",
       "      <defs>\n",
       "       <path d=\"M 0 0 \n",
       "L -3.5 0 \n",
       "\" id=\"m482452a682\" style=\"stroke:#000000;stroke-width:0.8;\"/>\n",
       "      </defs>\n",
       "      <g>\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"46.325\" xlink:href=\"#m482452a682\" y=\"114.814029\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_7\">\n",
       "      <!-- 0.45 -->\n",
       "      <g transform=\"translate(19.325 118.251529)scale(0.1 -0.1)\">\n",
       "       <defs>\n",
       "        <path d=\"M 225 2537 \n",
       "Q 250 3200 412 3587 \n",
       "Q 575 3975 875 4225 \n",
       "Q 1175 4475 1612 4475 \n",
       "Q 2050 4475 2375 4112 \n",
       "Q 2700 3750 2800 3200 \n",
       "Q 2900 2650 2862 1937 \n",
       "Q 2825 1225 2612 775 \n",
       "Q 2400 325 1975 150 \n",
       "Q 1550 -25 1125 187 \n",
       "Q 700 400 525 750 \n",
       "Q 350 1100 275 1487 \n",
       "Q 200 1875 225 2537 \n",
       "z\n",
       "M 750 2687 \n",
       "Q 675 2000 800 1462 \n",
       "Q 925 925 1212 700 \n",
       "Q 1500 475 1800 612 \n",
       "Q 2100 750 2237 1162 \n",
       "Q 2375 1575 2375 2062 \n",
       "Q 2375 2550 2337 2950 \n",
       "Q 2300 3350 2112 3675 \n",
       "Q 1925 4000 1612 4012 \n",
       "Q 1300 4025 1062 3700 \n",
       "Q 825 3375 750 2687 \n",
       "z\n",
       "\" id=\"SimHei-30\" transform=\"scale(0.015625)\"/>\n",
       "        <path d=\"M 1075 125 \n",
       "L 500 125 \n",
       "L 500 675 \n",
       "L 1075 675 \n",
       "L 1075 125 \n",
       "z\n",
       "\" id=\"SimHei-2e\" transform=\"scale(0.015625)\"/>\n",
       "       </defs>\n",
       "       <use xlink:href=\"#SimHei-30\"/>\n",
       "       <use x=\"50\" xlink:href=\"#SimHei-2e\"/>\n",
       "       <use x=\"100\" xlink:href=\"#SimHei-34\"/>\n",
       "       <use x=\"150\" xlink:href=\"#SimHei-35\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"ytick_2\">\n",
       "     <g id=\"line2d_13\">\n",
       "      <path clip-path=\"url(#pb8a768a74a)\" d=\"M 46.325 83.739484 \n",
       "L 241.625 83.739484 \n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\n",
       "     </g>\n",
       "     <g id=\"line2d_14\">\n",
       "      <g>\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"46.325\" xlink:href=\"#m482452a682\" y=\"83.739484\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_8\">\n",
       "      <!-- 0.50 -->\n",
       "      <g transform=\"translate(19.325 87.176984)scale(0.1 -0.1)\">\n",
       "       <use xlink:href=\"#SimHei-30\"/>\n",
       "       <use x=\"50\" xlink:href=\"#SimHei-2e\"/>\n",
       "       <use x=\"100\" xlink:href=\"#SimHei-35\"/>\n",
       "       <use x=\"150\" xlink:href=\"#SimHei-30\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"ytick_3\">\n",
       "     <g id=\"line2d_15\">\n",
       "      <path clip-path=\"url(#pb8a768a74a)\" d=\"M 46.325 52.664939 \n",
       "L 241.625 52.664939 \n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\n",
       "     </g>\n",
       "     <g id=\"line2d_16\">\n",
       "      <g>\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"46.325\" xlink:href=\"#m482452a682\" y=\"52.664939\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_9\">\n",
       "      <!-- 0.55 -->\n",
       "      <g transform=\"translate(19.325 56.102439)scale(0.1 -0.1)\">\n",
       "       <use xlink:href=\"#SimHei-30\"/>\n",
       "       <use x=\"50\" xlink:href=\"#SimHei-2e\"/>\n",
       "       <use x=\"100\" xlink:href=\"#SimHei-35\"/>\n",
       "       <use x=\"150\" xlink:href=\"#SimHei-35\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"ytick_4\">\n",
       "     <g id=\"line2d_17\">\n",
       "      <path clip-path=\"url(#pb8a768a74a)\" d=\"M 46.325 21.590394 \n",
       "L 241.625 21.590394 \n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\n",
       "     </g>\n",
       "     <g id=\"line2d_18\">\n",
       "      <g>\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"46.325\" xlink:href=\"#m482452a682\" y=\"21.590394\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_10\">\n",
       "      <!-- 0.60 -->\n",
       "      <g transform=\"translate(19.325 25.027894)scale(0.1 -0.1)\">\n",
       "       <defs>\n",
       "        <path d=\"M 250 1612 \n",
       "Q 275 1975 387 2225 \n",
       "Q 500 2475 725 2850 \n",
       "L 1750 4450 \n",
       "L 2325 4450 \n",
       "L 1275 2800 \n",
       "Q 1950 2975 2350 2750 \n",
       "Q 2750 2525 2887 2237 \n",
       "Q 3025 1950 3037 1612 \n",
       "Q 3050 1275 2937 950 \n",
       "Q 2825 625 2537 362 \n",
       "Q 2250 100 1737 75 \n",
       "Q 1225 50 862 262 \n",
       "Q 500 475 362 862 \n",
       "Q 225 1250 250 1612 \n",
       "z\n",
       "M 1025 787 \n",
       "Q 1250 550 1625 525 \n",
       "Q 2000 500 2250 775 \n",
       "Q 2500 1050 2500 1575 \n",
       "Q 2500 2100 2187 2300 \n",
       "Q 1875 2500 1487 2450 \n",
       "Q 1100 2400 925 2075 \n",
       "Q 750 1750 775 1387 \n",
       "Q 800 1025 1025 787 \n",
       "z\n",
       "\" id=\"SimHei-36\" transform=\"scale(0.015625)\"/>\n",
       "       </defs>\n",
       "       <use xlink:href=\"#SimHei-30\"/>\n",
       "       <use x=\"50\" xlink:href=\"#SimHei-2e\"/>\n",
       "       <use x=\"100\" xlink:href=\"#SimHei-36\"/>\n",
       "       <use x=\"150\" xlink:href=\"#SimHei-30\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"text_11\">\n",
       "     <!-- loss -->\n",
       "     <g transform=\"translate(14.075 86.5)rotate(-90)scale(0.1 -0.1)\">\n",
       "      <defs>\n",
       "       <path d=\"M 1825 125 \n",
       "L 1325 125 \n",
       "L 1325 4400 \n",
       "L 1825 4400 \n",
       "L 1825 125 \n",
       "z\n",
       "\" id=\"SimHei-6c\" transform=\"scale(0.015625)\"/>\n",
       "       <path d=\"M 2750 900 \n",
       "Q 2750 500 2437 287 \n",
       "Q 2125 75 1650 75 \n",
       "Q 1050 75 725 312 \n",
       "Q 400 550 400 1000 \n",
       "L 900 1000 \n",
       "Q 900 700 1112 600 \n",
       "Q 1325 500 1625 500 \n",
       "Q 1925 500 2075 612 \n",
       "Q 2225 725 2225 900 \n",
       "Q 2225 1025 2100 1150 \n",
       "Q 1975 1275 1475 1350 \n",
       "Q 900 1425 687 1637 \n",
       "Q 475 1850 475 2200 \n",
       "Q 475 2500 762 2737 \n",
       "Q 1050 2975 1600 2975 \n",
       "Q 2100 2975 2387 2750 \n",
       "Q 2675 2525 2675 2150 \n",
       "L 2175 2150 \n",
       "Q 2175 2375 2012 2462 \n",
       "Q 1850 2550 1600 2550 \n",
       "Q 1275 2550 1137 2437 \n",
       "Q 1000 2325 1000 2175 \n",
       "Q 1000 2000 1125 1900 \n",
       "Q 1250 1800 1650 1750 \n",
       "Q 2300 1650 2525 1437 \n",
       "Q 2750 1225 2750 900 \n",
       "z\n",
       "\" id=\"SimHei-73\" transform=\"scale(0.015625)\"/>\n",
       "      </defs>\n",
       "      <use xlink:href=\"#SimHei-6c\"/>\n",
       "      <use x=\"50\" xlink:href=\"#SimHei-6f\"/>\n",
       "      <use x=\"100\" xlink:href=\"#SimHei-73\"/>\n",
       "      <use x=\"150\" xlink:href=\"#SimHei-73\"/>\n",
       "     </g>\n",
       "    </g>\n",
       "   </g>\n",
       "   <g id=\"line2d_19\">\n",
       "    <path clip-path=\"url(#pb8a768a74a)\" d=\"M 7.206614 13.5 \n",
       "L 16.913229 60.627183 \n",
       "L 26.619843 79.524938 \n",
       "L 36.326457 89.582659 \n",
       "L 46.033072 96.008288 \n",
       "L 46.325 96.154587 \n",
       "L 56.031614 100.963952 \n",
       "L 65.738229 104.684661 \n",
       "L 75.444843 107.71552 \n",
       "L 85.151457 110.322627 \n",
       "L 94.858072 112.612341 \n",
       "L 95.15 112.672752 \n",
       "L 104.856614 115.12263 \n",
       "L 114.563229 117.296055 \n",
       "L 124.269843 119.290078 \n",
       "L 133.976457 121.128253 \n",
       "L 143.683072 122.823953 \n",
       "L 143.975 122.869853 \n",
       "L 153.681614 124.97817 \n",
       "L 163.388229 126.882726 \n",
       "L 173.094843 128.604914 \n",
       "L 182.801457 130.180356 \n",
       "L 192.508072 131.622353 \n",
       "L 192.8 131.66477 \n",
       "L 202.506614 133.550789 \n",
       "L 212.213229 135.257975 \n",
       "L 221.919843 136.788523 \n",
       "L 231.626457 138.186138 \n",
       "L 241.333072 139.46671 \n",
       "L 241.625 139.5 \n",
       "\" style=\"fill:none;stroke:#1f77b4;stroke-linecap:square;stroke-width:1.5;\"/>\n",
       "   </g>\n",
       "   <g id=\"patch_3\">\n",
       "    <path d=\"M 46.325 145.8 \n",
       "L 46.325 7.2 \n",
       "\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n",
       "   </g>\n",
       "   <g id=\"patch_4\">\n",
       "    <path d=\"M 241.625 145.8 \n",
       "L 241.625 7.2 \n",
       "\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n",
       "   </g>\n",
       "   <g id=\"patch_5\">\n",
       "    <path d=\"M 46.325 145.8 \n",
       "L 241.625 145.8 \n",
       "\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n",
       "   </g>\n",
       "   <g id=\"patch_6\">\n",
       "    <path d=\"M 46.325 7.2 \n",
       "L 241.625 7.2 \n",
       "\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n",
       "   </g>\n",
       "  </g>\n",
       " </g>\n",
       " <defs>\n",
       "  <clipPath id=\"pb8a768a74a\">\n",
       "   <rect height=\"138.6\" width=\"195.3\" x=\"46.325\" y=\"7.2\"/>\n",
       "  </clipPath>\n",
       " </defs>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<Figure size 350x250 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data_iter, vocab = load_data_ptb(512, 5, 5)\n",
    "lr, num_epochs = 0.002, 5\n",
    "embed_size = 100\n",
    "net = nn.Sequential(nn.Embedding(num_embeddings=len(vocab),\n",
    "                                 embedding_dim=embed_size),\n",
    "                    nn.Embedding(num_embeddings=len(vocab),\n",
    "                                 embedding_dim=embed_size))\n",
    "train(net, data_iter, lr, num_epochs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "ba4f941f-a8e4-493a-be24-2d78884ecd0e",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss 0.384, 36166.7 tokens/sec on cpu\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.3843912484866804"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       "  \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<svg height=\"180.737522pt\" version=\"1.1\" viewBox=\"0 0 251.325 180.737522\" width=\"251.325pt\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       " <metadata>\n",
       "  <rdf:RDF xmlns:cc=\"http://creativecommons.org/ns#\" xmlns:dc=\"http://purl.org/dc/elements/1.1/\" xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\">\n",
       "   <cc:Work>\n",
       "    <dc:type rdf:resource=\"http://purl.org/dc/dcmitype/StillImage\"/>\n",
       "    <dc:date>2023-09-28T12:35:10.325423</dc:date>\n",
       "    <dc:format>image/svg+xml</dc:format>\n",
       "    <dc:creator>\n",
       "     <cc:Agent>\n",
       "      <dc:title>Matplotlib v3.4.0, https://matplotlib.org/</dc:title>\n",
       "     </cc:Agent>\n",
       "    </dc:creator>\n",
       "   </cc:Work>\n",
       "  </rdf:RDF>\n",
       " </metadata>\n",
       " <defs>\n",
       "  <style type=\"text/css\">*{stroke-linecap:butt;stroke-linejoin:round;}</style>\n",
       " </defs>\n",
       " <g id=\"figure_1\">\n",
       "  <g id=\"patch_1\">\n",
       "   <path d=\"M 0 180.737522 \n",
       "L 251.325 180.737522 \n",
       "L 251.325 0 \n",
       "L 0 0 \n",
       "z\n",
       "\" style=\"fill:#ffffff;\"/>\n",
       "  </g>\n",
       "  <g id=\"axes_1\">\n",
       "   <g id=\"patch_2\">\n",
       "    <path d=\"M 46.325 146.287522 \n",
       "L 241.625 146.287522 \n",
       "L 241.625 7.687522 \n",
       "L 46.325 7.687522 \n",
       "z\n",
       "\" style=\"fill:#ffffff;\"/>\n",
       "   </g>\n",
       "   <g id=\"matplotlib.axis_1\">\n",
       "    <g id=\"xtick_1\">\n",
       "     <g id=\"line2d_1\">\n",
       "      <path clip-path=\"url(#pb3af3af454)\" d=\"M 46.325 146.287522 \n",
       "L 46.325 7.687522 \n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\n",
       "     </g>\n",
       "     <g id=\"line2d_2\">\n",
       "      <defs>\n",
       "       <path d=\"M 0 0 \n",
       "L 0 3.5 \n",
       "\" id=\"m9bd6e18cf5\" style=\"stroke:#000000;stroke-width:0.8;\"/>\n",
       "      </defs>\n",
       "      <g>\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"46.325\" xlink:href=\"#m9bd6e18cf5\" y=\"146.287522\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_1\">\n",
       "      <!-- 1 -->\n",
       "      <g transform=\"translate(43.825 160.162522)scale(0.1 -0.1)\">\n",
       "       <defs>\n",
       "        <path d=\"M 1400 3600 \n",
       "Q 1075 3275 575 2975 \n",
       "L 575 3450 \n",
       "Q 1200 3875 1600 4450 \n",
       "L 1900 4450 \n",
       "L 1900 150 \n",
       "L 1400 150 \n",
       "L 1400 3600 \n",
       "z\n",
       "\" id=\"SimHei-31\" transform=\"scale(0.015625)\"/>\n",
       "       </defs>\n",
       "       <use xlink:href=\"#SimHei-31\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"xtick_2\">\n",
       "     <g id=\"line2d_3\">\n",
       "      <path clip-path=\"url(#pb3af3af454)\" d=\"M 95.15 146.287522 \n",
       "L 95.15 7.687522 \n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\n",
       "     </g>\n",
       "     <g id=\"line2d_4\">\n",
       "      <g>\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"95.15\" xlink:href=\"#m9bd6e18cf5\" y=\"146.287522\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_2\">\n",
       "      <!-- 2 -->\n",
       "      <g transform=\"translate(92.65 160.162522)scale(0.1 -0.1)\">\n",
       "       <defs>\n",
       "        <path d=\"M 300 250 \n",
       "Q 325 625 650 925 \n",
       "Q 975 1225 1475 1862 \n",
       "Q 1975 2500 2125 2850 \n",
       "Q 2275 3200 2237 3450 \n",
       "Q 2200 3700 2000 3862 \n",
       "Q 1800 4025 1537 4000 \n",
       "Q 1275 3975 1037 3800 \n",
       "Q 800 3625 675 3275 \n",
       "L 200 3350 \n",
       "Q 400 3925 712 4187 \n",
       "Q 1025 4450 1450 4475 \n",
       "Q 1700 4500 1900 4462 \n",
       "Q 2100 4425 2312 4287 \n",
       "Q 2525 4150 2662 3875 \n",
       "Q 2800 3600 2762 3212 \n",
       "Q 2725 2825 2375 2287 \n",
       "Q 2025 1750 1025 600 \n",
       "L 2825 600 \n",
       "L 2825 150 \n",
       "L 300 150 \n",
       "L 300 250 \n",
       "z\n",
       "\" id=\"SimHei-32\" transform=\"scale(0.015625)\"/>\n",
       "       </defs>\n",
       "       <use xlink:href=\"#SimHei-32\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"xtick_3\">\n",
       "     <g id=\"line2d_5\">\n",
       "      <path clip-path=\"url(#pb3af3af454)\" d=\"M 143.975 146.287522 \n",
       "L 143.975 7.687522 \n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\n",
       "     </g>\n",
       "     <g id=\"line2d_6\">\n",
       "      <g>\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"143.975\" xlink:href=\"#m9bd6e18cf5\" y=\"146.287522\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_3\">\n",
       "      <!-- 3 -->\n",
       "      <g transform=\"translate(141.475 160.162522)scale(0.1 -0.1)\">\n",
       "       <defs>\n",
       "        <path d=\"M 250 1225 \n",
       "L 700 1300 \n",
       "Q 800 975 1025 762 \n",
       "Q 1250 550 1587 562 \n",
       "Q 1925 575 2125 837 \n",
       "Q 2325 1100 2300 1437 \n",
       "Q 2275 1775 2037 1962 \n",
       "Q 1800 2150 1275 2225 \n",
       "L 1275 2550 \n",
       "Q 1800 2600 2037 2825 \n",
       "Q 2275 3050 2250 3412 \n",
       "Q 2225 3775 1925 3937 \n",
       "Q 1625 4100 1287 3975 \n",
       "Q 950 3850 750 3275 \n",
       "L 300 3350 \n",
       "Q 450 3800 712 4100 \n",
       "Q 975 4400 1425 4450 \n",
       "Q 1875 4500 2212 4337 \n",
       "Q 2550 4175 2687 3837 \n",
       "Q 2825 3500 2725 3100 \n",
       "Q 2625 2700 2150 2400 \n",
       "Q 2500 2250 2687 1950 \n",
       "Q 2875 1650 2812 1162 \n",
       "Q 2750 675 2375 375 \n",
       "Q 2000 75 1525 87 \n",
       "Q 1050 100 700 387 \n",
       "Q 350 675 250 1225 \n",
       "z\n",
       "\" id=\"SimHei-33\" transform=\"scale(0.015625)\"/>\n",
       "       </defs>\n",
       "       <use xlink:href=\"#SimHei-33\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"xtick_4\">\n",
       "     <g id=\"line2d_7\">\n",
       "      <path clip-path=\"url(#pb3af3af454)\" d=\"M 192.8 146.287522 \n",
       "L 192.8 7.687522 \n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\n",
       "     </g>\n",
       "     <g id=\"line2d_8\">\n",
       "      <g>\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"192.8\" xlink:href=\"#m9bd6e18cf5\" y=\"146.287522\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_4\">\n",
       "      <!-- 4 -->\n",
       "      <g transform=\"translate(190.3 160.162522)scale(0.1 -0.1)\">\n",
       "       <defs>\n",
       "        <path d=\"M 2000 1100 \n",
       "L 75 1100 \n",
       "L 75 1525 \n",
       "L 2100 4450 \n",
       "L 2475 4450 \n",
       "L 2475 1525 \n",
       "L 3075 1525 \n",
       "L 3075 1100 \n",
       "L 2475 1100 \n",
       "L 2475 150 \n",
       "L 2000 150 \n",
       "L 2000 1100 \n",
       "z\n",
       "M 2000 1525 \n",
       "L 2000 3500 \n",
       "L 600 1525 \n",
       "L 2000 1525 \n",
       "z\n",
       "\" id=\"SimHei-34\" transform=\"scale(0.015625)\"/>\n",
       "       </defs>\n",
       "       <use xlink:href=\"#SimHei-34\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"xtick_5\">\n",
       "     <g id=\"line2d_9\">\n",
       "      <path clip-path=\"url(#pb3af3af454)\" d=\"M 241.625 146.287522 \n",
       "L 241.625 7.687522 \n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\n",
       "     </g>\n",
       "     <g id=\"line2d_10\">\n",
       "      <g>\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"241.625\" xlink:href=\"#m9bd6e18cf5\" y=\"146.287522\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_5\">\n",
       "      <!-- 5 -->\n",
       "      <g transform=\"translate(239.125 160.162522)scale(0.1 -0.1)\">\n",
       "       <defs>\n",
       "        <path d=\"M 550 1325 \n",
       "Q 725 650 1150 575 \n",
       "Q 1575 500 1837 662 \n",
       "Q 2100 825 2212 1087 \n",
       "Q 2325 1350 2312 1675 \n",
       "Q 2300 2000 2137 2225 \n",
       "Q 1975 2450 1725 2525 \n",
       "Q 1475 2600 1162 2525 \n",
       "Q 850 2450 650 2175 \n",
       "L 225 2225 \n",
       "Q 275 2375 700 4375 \n",
       "L 2675 4375 \n",
       "L 2675 3925 \n",
       "L 1075 3925 \n",
       "Q 950 3250 825 2850 \n",
       "Q 1200 3025 1525 3012 \n",
       "Q 1850 3000 2150 2862 \n",
       "Q 2450 2725 2587 2487 \n",
       "Q 2725 2250 2787 2012 \n",
       "Q 2850 1775 2837 1500 \n",
       "Q 2825 1225 2725 937 \n",
       "Q 2625 650 2425 462 \n",
       "Q 2225 275 1937 162 \n",
       "Q 1650 50 1275 75 \n",
       "Q 900 100 562 350 \n",
       "Q 225 600 100 1200 \n",
       "L 550 1325 \n",
       "z\n",
       "\" id=\"SimHei-35\" transform=\"scale(0.015625)\"/>\n",
       "       </defs>\n",
       "       <use xlink:href=\"#SimHei-35\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"text_6\">\n",
       "     <!-- epoch -->\n",
       "     <g transform=\"translate(131.475 172.287522)scale(0.1 -0.1)\">\n",
       "      <defs>\n",
       "       <path d=\"M 2850 1075 \n",
       "Q 2800 625 2450 350 \n",
       "Q 2100 75 1625 75 \n",
       "Q 1025 75 637 462 \n",
       "Q 250 850 250 1525 \n",
       "Q 250 2200 637 2587 \n",
       "Q 1025 2975 1625 2975 \n",
       "Q 2150 2975 2487 2637 \n",
       "Q 2825 2300 2825 1525 \n",
       "L 800 1525 \n",
       "Q 800 975 1037 750 \n",
       "Q 1275 525 1625 525 \n",
       "Q 1900 525 2075 662 \n",
       "Q 2250 800 2300 1075 \n",
       "L 2850 1075 \n",
       "z\n",
       "M 2250 1925 \n",
       "Q 2200 2275 2025 2412 \n",
       "Q 1850 2550 1575 2550 \n",
       "Q 1325 2550 1125 2412 \n",
       "Q 925 2275 825 1925 \n",
       "L 2250 1925 \n",
       "z\n",
       "\" id=\"SimHei-65\" transform=\"scale(0.015625)\"/>\n",
       "       <path d=\"M 2825 1525 \n",
       "Q 2825 800 2475 425 \n",
       "Q 2125 50 1625 50 \n",
       "Q 1350 50 1150 162 \n",
       "Q 950 275 800 500 \n",
       "L 800 -800 \n",
       "L 300 -800 \n",
       "L 300 2925 \n",
       "L 800 2925 \n",
       "L 800 2550 \n",
       "Q 950 2775 1150 2875 \n",
       "Q 1350 2975 1625 2975 \n",
       "Q 2125 2975 2475 2612 \n",
       "Q 2825 2250 2825 1525 \n",
       "z\n",
       "M 2275 1525 \n",
       "Q 2275 2000 2087 2262 \n",
       "Q 1900 2525 1525 2525 \n",
       "Q 1225 2525 1012 2262 \n",
       "Q 800 2000 800 1525 \n",
       "Q 800 1050 1012 775 \n",
       "Q 1225 500 1525 500 \n",
       "Q 1900 500 2087 775 \n",
       "Q 2275 1050 2275 1525 \n",
       "z\n",
       "\" id=\"SimHei-70\" transform=\"scale(0.015625)\"/>\n",
       "       <path d=\"M 2925 1525 \n",
       "Q 2925 875 2525 475 \n",
       "Q 2125 75 1575 75 \n",
       "Q 1025 75 625 475 \n",
       "Q 225 875 225 1525 \n",
       "Q 225 2175 625 2575 \n",
       "Q 1025 2975 1575 2975 \n",
       "Q 2125 2975 2525 2575 \n",
       "Q 2925 2175 2925 1525 \n",
       "z\n",
       "M 2375 1525 \n",
       "Q 2375 2025 2125 2275 \n",
       "Q 1875 2525 1575 2525 \n",
       "Q 1275 2525 1025 2275 \n",
       "Q 775 2025 775 1525 \n",
       "Q 775 1025 1025 775 \n",
       "Q 1275 525 1575 525 \n",
       "Q 1875 525 2125 775 \n",
       "Q 2375 1025 2375 1525 \n",
       "z\n",
       "\" id=\"SimHei-6f\" transform=\"scale(0.015625)\"/>\n",
       "       <path d=\"M 2850 1250 \n",
       "Q 2850 725 2487 400 \n",
       "Q 2125 75 1575 75 \n",
       "Q 1025 75 625 462 \n",
       "Q 225 850 225 1525 \n",
       "Q 225 2200 625 2587 \n",
       "Q 1025 2975 1575 2975 \n",
       "Q 2125 2975 2450 2687 \n",
       "Q 2775 2400 2775 2000 \n",
       "L 2225 2000 \n",
       "Q 2200 2300 2012 2412 \n",
       "Q 1825 2525 1575 2525 \n",
       "Q 1275 2525 1025 2287 \n",
       "Q 775 2050 775 1525 \n",
       "Q 775 1000 1025 762 \n",
       "Q 1275 525 1575 525 \n",
       "Q 1900 525 2100 700 \n",
       "Q 2300 875 2300 1250 \n",
       "L 2850 1250 \n",
       "z\n",
       "\" id=\"SimHei-63\" transform=\"scale(0.015625)\"/>\n",
       "       <path d=\"M 2800 125 \n",
       "L 2300 125 \n",
       "L 2300 1925 \n",
       "Q 2300 2225 2150 2400 \n",
       "Q 2000 2575 1750 2575 \n",
       "Q 1425 2575 1137 2237 \n",
       "Q 850 1900 850 1400 \n",
       "L 850 125 \n",
       "L 350 125 \n",
       "L 350 4400 \n",
       "L 850 4400 \n",
       "L 850 2400 \n",
       "Q 1050 2675 1287 2825 \n",
       "Q 1525 2975 1900 2975 \n",
       "Q 2350 2975 2575 2725 \n",
       "Q 2800 2475 2800 2100 \n",
       "L 2800 125 \n",
       "z\n",
       "\" id=\"SimHei-68\" transform=\"scale(0.015625)\"/>\n",
       "      </defs>\n",
       "      <use xlink:href=\"#SimHei-65\"/>\n",
       "      <use x=\"50\" xlink:href=\"#SimHei-70\"/>\n",
       "      <use x=\"100\" xlink:href=\"#SimHei-6f\"/>\n",
       "      <use x=\"150\" xlink:href=\"#SimHei-63\"/>\n",
       "      <use x=\"200\" xlink:href=\"#SimHei-68\"/>\n",
       "     </g>\n",
       "    </g>\n",
       "   </g>\n",
       "   <g id=\"matplotlib.axis_2\">\n",
       "    <g id=\"ytick_1\">\n",
       "     <g id=\"line2d_11\">\n",
       "      <path clip-path=\"url(#pb3af3af454)\" d=\"M 46.325 130.623374 \n",
       "L 241.625 130.623374 \n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\n",
       "     </g>\n",
       "     <g id=\"line2d_12\">\n",
       "      <defs>\n",
       "       <path d=\"M 0 0 \n",
       "L -3.5 0 \n",
       "\" id=\"m379999c32d\" style=\"stroke:#000000;stroke-width:0.8;\"/>\n",
       "      </defs>\n",
       "      <g>\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"46.325\" xlink:href=\"#m379999c32d\" y=\"130.623374\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_7\">\n",
       "      <!-- 0.40 -->\n",
       "      <g transform=\"translate(19.325 134.060874)scale(0.1 -0.1)\">\n",
       "       <defs>\n",
       "        <path d=\"M 225 2537 \n",
       "Q 250 3200 412 3587 \n",
       "Q 575 3975 875 4225 \n",
       "Q 1175 4475 1612 4475 \n",
       "Q 2050 4475 2375 4112 \n",
       "Q 2700 3750 2800 3200 \n",
       "Q 2900 2650 2862 1937 \n",
       "Q 2825 1225 2612 775 \n",
       "Q 2400 325 1975 150 \n",
       "Q 1550 -25 1125 187 \n",
       "Q 700 400 525 750 \n",
       "Q 350 1100 275 1487 \n",
       "Q 200 1875 225 2537 \n",
       "z\n",
       "M 750 2687 \n",
       "Q 675 2000 800 1462 \n",
       "Q 925 925 1212 700 \n",
       "Q 1500 475 1800 612 \n",
       "Q 2100 750 2237 1162 \n",
       "Q 2375 1575 2375 2062 \n",
       "Q 2375 2550 2337 2950 \n",
       "Q 2300 3350 2112 3675 \n",
       "Q 1925 4000 1612 4012 \n",
       "Q 1300 4025 1062 3700 \n",
       "Q 825 3375 750 2687 \n",
       "z\n",
       "\" id=\"SimHei-30\" transform=\"scale(0.015625)\"/>\n",
       "        <path d=\"M 1075 125 \n",
       "L 500 125 \n",
       "L 500 675 \n",
       "L 1075 675 \n",
       "L 1075 125 \n",
       "z\n",
       "\" id=\"SimHei-2e\" transform=\"scale(0.015625)\"/>\n",
       "       </defs>\n",
       "       <use xlink:href=\"#SimHei-30\"/>\n",
       "       <use x=\"50\" xlink:href=\"#SimHei-2e\"/>\n",
       "       <use x=\"100\" xlink:href=\"#SimHei-34\"/>\n",
       "       <use x=\"150\" xlink:href=\"#SimHei-30\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"ytick_2\">\n",
       "     <g id=\"line2d_13\">\n",
       "      <path clip-path=\"url(#pb3af3af454)\" d=\"M 46.325 100.626906 \n",
       "L 241.625 100.626906 \n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\n",
       "     </g>\n",
       "     <g id=\"line2d_14\">\n",
       "      <g>\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"46.325\" xlink:href=\"#m379999c32d\" y=\"100.626906\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_8\">\n",
       "      <!-- 0.45 -->\n",
       "      <g transform=\"translate(19.325 104.064406)scale(0.1 -0.1)\">\n",
       "       <use xlink:href=\"#SimHei-30\"/>\n",
       "       <use x=\"50\" xlink:href=\"#SimHei-2e\"/>\n",
       "       <use x=\"100\" xlink:href=\"#SimHei-34\"/>\n",
       "       <use x=\"150\" xlink:href=\"#SimHei-35\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"ytick_3\">\n",
       "     <g id=\"line2d_15\">\n",
       "      <path clip-path=\"url(#pb3af3af454)\" d=\"M 46.325 70.630437 \n",
       "L 241.625 70.630437 \n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\n",
       "     </g>\n",
       "     <g id=\"line2d_16\">\n",
       "      <g>\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"46.325\" xlink:href=\"#m379999c32d\" y=\"70.630437\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_9\">\n",
       "      <!-- 0.50 -->\n",
       "      <g transform=\"translate(19.325 74.067937)scale(0.1 -0.1)\">\n",
       "       <use xlink:href=\"#SimHei-30\"/>\n",
       "       <use x=\"50\" xlink:href=\"#SimHei-2e\"/>\n",
       "       <use x=\"100\" xlink:href=\"#SimHei-35\"/>\n",
       "       <use x=\"150\" xlink:href=\"#SimHei-30\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"ytick_4\">\n",
       "     <g id=\"line2d_17\">\n",
       "      <path clip-path=\"url(#pb3af3af454)\" d=\"M 46.325 40.633969 \n",
       "L 241.625 40.633969 \n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\n",
       "     </g>\n",
       "     <g id=\"line2d_18\">\n",
       "      <g>\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"46.325\" xlink:href=\"#m379999c32d\" y=\"40.633969\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_10\">\n",
       "      <!-- 0.55 -->\n",
       "      <g transform=\"translate(19.325 44.071469)scale(0.1 -0.1)\">\n",
       "       <use xlink:href=\"#SimHei-30\"/>\n",
       "       <use x=\"50\" xlink:href=\"#SimHei-2e\"/>\n",
       "       <use x=\"100\" xlink:href=\"#SimHei-35\"/>\n",
       "       <use x=\"150\" xlink:href=\"#SimHei-35\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"ytick_5\">\n",
       "     <g id=\"line2d_19\">\n",
       "      <path clip-path=\"url(#pb3af3af454)\" d=\"M 46.325 10.6375 \n",
       "L 241.625 10.6375 \n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\n",
       "     </g>\n",
       "     <g id=\"line2d_20\">\n",
       "      <g>\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"46.325\" xlink:href=\"#m379999c32d\" y=\"10.6375\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_11\">\n",
       "      <!-- 0.60 -->\n",
       "      <g transform=\"translate(19.325 14.075)scale(0.1 -0.1)\">\n",
       "       <defs>\n",
       "        <path d=\"M 250 1612 \n",
       "Q 275 1975 387 2225 \n",
       "Q 500 2475 725 2850 \n",
       "L 1750 4450 \n",
       "L 2325 4450 \n",
       "L 1275 2800 \n",
       "Q 1950 2975 2350 2750 \n",
       "Q 2750 2525 2887 2237 \n",
       "Q 3025 1950 3037 1612 \n",
       "Q 3050 1275 2937 950 \n",
       "Q 2825 625 2537 362 \n",
       "Q 2250 100 1737 75 \n",
       "Q 1225 50 862 262 \n",
       "Q 500 475 362 862 \n",
       "Q 225 1250 250 1612 \n",
       "z\n",
       "M 1025 787 \n",
       "Q 1250 550 1625 525 \n",
       "Q 2000 500 2250 775 \n",
       "Q 2500 1050 2500 1575 \n",
       "Q 2500 2100 2187 2300 \n",
       "Q 1875 2500 1487 2450 \n",
       "Q 1100 2400 925 2075 \n",
       "Q 750 1750 775 1387 \n",
       "Q 800 1025 1025 787 \n",
       "z\n",
       "\" id=\"SimHei-36\" transform=\"scale(0.015625)\"/>\n",
       "       </defs>\n",
       "       <use xlink:href=\"#SimHei-30\"/>\n",
       "       <use x=\"50\" xlink:href=\"#SimHei-2e\"/>\n",
       "       <use x=\"100\" xlink:href=\"#SimHei-36\"/>\n",
       "       <use x=\"150\" xlink:href=\"#SimHei-30\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"text_12\">\n",
       "     <!-- loss -->\n",
       "     <g transform=\"translate(14.075 86.987522)rotate(-90)scale(0.1 -0.1)\">\n",
       "      <defs>\n",
       "       <path d=\"M 1825 125 \n",
       "L 1325 125 \n",
       "L 1325 4400 \n",
       "L 1825 4400 \n",
       "L 1825 125 \n",
       "z\n",
       "\" id=\"SimHei-6c\" transform=\"scale(0.015625)\"/>\n",
       "       <path d=\"M 2750 900 \n",
       "Q 2750 500 2437 287 \n",
       "Q 2125 75 1650 75 \n",
       "Q 1050 75 725 312 \n",
       "Q 400 550 400 1000 \n",
       "L 900 1000 \n",
       "Q 900 700 1112 600 \n",
       "Q 1325 500 1625 500 \n",
       "Q 1925 500 2075 612 \n",
       "Q 2225 725 2225 900 \n",
       "Q 2225 1025 2100 1150 \n",
       "Q 1975 1275 1475 1350 \n",
       "Q 900 1425 687 1637 \n",
       "Q 475 1850 475 2200 \n",
       "Q 475 2500 762 2737 \n",
       "Q 1050 2975 1600 2975 \n",
       "Q 2100 2975 2387 2750 \n",
       "Q 2675 2525 2675 2150 \n",
       "L 2175 2150 \n",
       "Q 2175 2375 2012 2462 \n",
       "Q 1850 2550 1600 2550 \n",
       "Q 1275 2550 1137 2437 \n",
       "Q 1000 2325 1000 2175 \n",
       "Q 1000 2000 1125 1900 \n",
       "Q 1250 1800 1650 1750 \n",
       "Q 2300 1650 2525 1437 \n",
       "Q 2750 1225 2750 900 \n",
       "z\n",
       "\" id=\"SimHei-73\" transform=\"scale(0.015625)\"/>\n",
       "      </defs>\n",
       "      <use xlink:href=\"#SimHei-6c\"/>\n",
       "      <use x=\"50\" xlink:href=\"#SimHei-6f\"/>\n",
       "      <use x=\"100\" xlink:href=\"#SimHei-73\"/>\n",
       "      <use x=\"150\" xlink:href=\"#SimHei-73\"/>\n",
       "     </g>\n",
       "    </g>\n",
       "   </g>\n",
       "   <g id=\"line2d_21\">\n",
       "    <path clip-path=\"url(#pb3af3af454)\" d=\"M 7.206614 13.987522 \n",
       "L 16.913229 56.23942 \n",
       "L 26.619843 72.470271 \n",
       "L 36.326457 81.35474 \n",
       "L 46.033072 87.243381 \n",
       "L 46.325 87.38379 \n",
       "L 56.031614 92.324132 \n",
       "L 65.738229 96.313184 \n",
       "L 75.444843 99.66666 \n",
       "L 85.151457 102.536124 \n",
       "L 94.858072 105.12276 \n",
       "L 95.15 105.18858 \n",
       "L 104.856614 108.432239 \n",
       "L 114.563229 111.273599 \n",
       "L 124.269843 113.806376 \n",
       "L 133.976457 116.051247 \n",
       "L 143.683072 118.067472 \n",
       "L 143.975 118.122089 \n",
       "L 153.681614 121.00422 \n",
       "L 163.388229 123.558549 \n",
       "L 173.094843 125.824495 \n",
       "L 182.801457 127.827496 \n",
       "L 192.508072 129.634236 \n",
       "L 192.8 129.680538 \n",
       "L 202.506614 132.211941 \n",
       "L 212.213229 134.498683 \n",
       "L 221.919843 136.517786 \n",
       "L 231.626457 138.316312 \n",
       "L 241.333072 139.947711 \n",
       "L 241.625 139.987522 \n",
       "\" style=\"fill:none;stroke:#1f77b4;stroke-linecap:square;stroke-width:1.5;\"/>\n",
       "   </g>\n",
       "   <g id=\"patch_3\">\n",
       "    <path d=\"M 46.325 146.287522 \n",
       "L 46.325 7.687522 \n",
       "\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n",
       "   </g>\n",
       "   <g id=\"patch_4\">\n",
       "    <path d=\"M 241.625 146.287522 \n",
       "L 241.625 7.687522 \n",
       "\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n",
       "   </g>\n",
       "   <g id=\"patch_5\">\n",
       "    <path d=\"M 46.325 146.287522 \n",
       "L 241.625 146.287522 \n",
       "\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n",
       "   </g>\n",
       "   <g id=\"patch_6\">\n",
       "    <path d=\"M 46.325 7.687522 \n",
       "L 241.625 7.687522 \n",
       "\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n",
       "   </g>\n",
       "  </g>\n",
       " </g>\n",
       " <defs>\n",
       "  <clipPath id=\"pb3af3af454\">\n",
       "   <rect height=\"138.6\" width=\"195.3\" x=\"46.325\" y=\"7.687522\"/>\n",
       "  </clipPath>\n",
       " </defs>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<Figure size 350x250 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "lr, num_epochs = 0.002, 5\n",
    "embed_size = 200\n",
    "net = nn.Sequential(nn.Embedding(num_embeddings=len(vocab),\n",
    "                                 embedding_dim=embed_size),\n",
    "                    nn.Embedding(num_embeddings=len(vocab),\n",
    "                                 embedding_dim=embed_size))\n",
    "train(net, data_iter, lr, num_epochs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "d2391db9-f3f4-456c-ad03-b4fa9e9359ab",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cosine sim=0.551: workstations\n",
      "cosine sim=0.537: microprocessor\n",
      "cosine sim=0.525: compaq\n"
     ]
    }
   ],
   "source": [
    "get_similar_tokens('intel', 3, net[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "793a6be3-a982-40b3-8063-6b68585e44a4",
   "metadata": {},
   "source": [
    "# 2. When a training corpus is huge, we often sample context words and noise words for the center words in the current minibatch when updating model parameters. In other words, the same center word may have different context words or noise words in different training epochs. What are the benefits of this method? Try to implement this training method."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a3ced8d4-728e-41db-8a46-2f7a6b20fd5b",
   "metadata": {},
   "source": [
    "The benefit of sampling context words and noise words is that it allows the model to capture more information from different contexts and handle larger training corpora. By varying the context words and noise words for the same center word in different epochs, the model can learn more diverse and robust representations of words that reflect their meanings in various situations4. This can also help the model avoid overfitting to a specific context or corpus."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "id": "6d4f22f5-3316-4382-9a11-e20899f21518",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import random\n",
    "\n",
    "class Vocab:\n",
    "    \"\"\"Vocabulary for text.\"\"\"\n",
    "    def __init__(self, filename, min_freq=0, reserved_tokens=[]):\n",
    "        # Flatten a 2D list if needed\n",
    "        # if tokens and isinstance(tokens[0], list):\n",
    "        #     tokens = [token for line in tokens for token in line]\n",
    "        self.token_freqs = collections.Counter([])\n",
    "        with open(filename, 'rb') as f:\n",
    "            for line in f:\n",
    "                counter = collections.Counter(line.decode().strip().split())\n",
    "                self.token_freqs.update(counter)\n",
    "        # Count token frequencies\n",
    "        \n",
    "        # counter = collections.Counter(tokens)\n",
    "        # self.token_freqs = sorted(counter.items(), key=lambda x: x[1],\n",
    "        #                           reverse=True)\n",
    "        # The list of unique tokens\n",
    "        filter_tokens = list(set(['<unk>'] + reserved_tokens + [\n",
    "            token for token in self.token_freqs if self.token_freqs[token] >= min_freq]))\n",
    "        self.idx_to_token = {}\n",
    "        self.token_to_idx = {}\n",
    "        for idx, token in enumerate(filter_tokens):\n",
    "            self.idx_to_token[idx]=token\n",
    "            self.token_to_idx[token]=idx\n",
    "        self.cur_id = len(self.idx_to_token)\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.idx_to_token)\n",
    "\n",
    "    def __getitem__(self, tokens):\n",
    "        if not isinstance(tokens, (list, tuple)):\n",
    "            return self.token_to_idx.get(tokens, self.unk)\n",
    "        return [self.__getitem__(token) for token in tokens]\n",
    "\n",
    "    def to_tokens(self, indices):\n",
    "        if hasattr(indices, '__len__') and len(indices) > 1:\n",
    "            return [self.idx_to_token[int(index)] for index in indices]\n",
    "        return self.idx_to_token[indices]\n",
    "    \n",
    "    def update(self, tokens):\n",
    "        # 统计tokens中每个词的出现次数，并更新到原有的词频字典中\n",
    "        counter = collections.Counter(tokens)\n",
    "        self.token_freqs.update(counter)\n",
    "        # 遍历新出现的词，给每个词分配一个id，并更新两个映射字典\n",
    "        for token in counter:\n",
    "            if token not in self.token_to_idx:\n",
    "                self.token_to_idx[token] = self.cur_id\n",
    "                self.idx_to_token[self.cur_id] = token\n",
    "                self.cur_id += 1\n",
    "    \n",
    "    @property\n",
    "    def unk(self):  # Index for the unknown token\n",
    "        return self.token_to_idx['<unk>']\n",
    "\n",
    "\n",
    "def get_random_line(filepath,n):\n",
    "    file_size = os.path.getsize(filepath)\n",
    "    res = []\n",
    "    row = 0\n",
    "    with open(filepath, 'rb') as f:\n",
    "        while True:\n",
    "            pos = random.randint(0, file_size)\n",
    "            if not pos:  # the first line is chosen\n",
    "                return res  # return str\n",
    "            f.seek(pos)  # seek to random position\n",
    "            f.readline()  # skip possibly incomplete line\n",
    "            line = f.readline()  # read next (full) line\n",
    "            if line:\n",
    "                res.append(line.decode().strip().split())\n",
    "                row += 1\n",
    "                if row >= n:\n",
    "                    return res\n",
    "                \n",
    "#@save\n",
    "def subsample(sentences, vocab,flag=True):\n",
    "    \"\"\"Subsample high-frequency words.\"\"\"\n",
    "    # Exclude unknown tokens ('<unk>')\n",
    "    sentences = [[token for token in line if vocab[token] != vocab.unk]\n",
    "                 for line in sentences]\n",
    "    # print(sentences)\n",
    "    counter = collections.Counter([\n",
    "        token for line in sentences for token in line])\n",
    "    num_tokens = sum(counter.values())\n",
    "\n",
    "    # Return True if `token` is kept during subsampling\n",
    "    def keep(token):\n",
    "        # print(token, counter[token]/num_tokens,math.sqrt(1e-4 / counter[token] * num_tokens))\n",
    "        return(random.uniform(0, 1) <\n",
    "               math.sqrt(1e-4 / counter[token] * num_tokens))\n",
    "    if flag:\n",
    "        return ([[token for token in line if keep(token)] for line in sentences],\n",
    "            counter)\n",
    "    return (sentences,counter)\n",
    "\n",
    "def batchify(all_centers, all_contexts,all_negatives):\n",
    "    \"\"\"Return a minibatch of examples for skip-gram with negative sampling.\"\"\"\n",
    "    max_len = max(len(c) + len(n) for _, c, n in data)\n",
    "    centers, contexts_negatives, masks, labels = [], [], [], []\n",
    "    for center, context, negative in zip(all_centers, all_contexts,all_negatives):\n",
    "        cur_len = len(context) + len(negative)\n",
    "        centers += [center]\n",
    "        contexts_negatives += [context + negative + [0] * (max_len - cur_len)]\n",
    "        masks += [[1] * cur_len + [0] * (max_len - cur_len)]\n",
    "        labels += [[1] * len(context) + [0] * (max_len - len(context))]\n",
    "    # print(contexts_negatives)\n",
    "    return (torch.tensor(centers).reshape((-1, 1)), torch.tensor(\n",
    "        contexts_negatives), torch.tensor(masks), torch.tensor(labels))\n",
    "\n",
    "def random_batch(filename,vocab,n,max_window_size,num_noise_words):\n",
    "    sentences = get_random_line(filename,n)\n",
    "    subsampled, counter = subsample(sentences, vocab, False)\n",
    "    corpus = [vocab[line] for line in subsampled]\n",
    "    all_centers, all_contexts = get_centers_and_contexts(\n",
    "        corpus, max_window_size)\n",
    "    all_negatives = get_negatives(all_contexts, vocab, counter, num_noise_words, k=100)\n",
    "    return batchify(all_centers, all_contexts,all_negatives)\n",
    "\n",
    "def train(net, filename,vocab,n ,max_window_size,num_noise_words,lr, num_epochs, num_batches, device='cpu'):\n",
    "    def init_weights(module):\n",
    "        if type(module) == nn.Embedding:\n",
    "            nn.init.xavier_uniform_(module.weight)\n",
    "    net.apply(init_weights)\n",
    "    net = net.to(device)\n",
    "    optimizer = torch.optim.Adam(net.parameters(), lr=lr)\n",
    "    animator = d2l.Animator(xlabel='epoch', ylabel='loss',\n",
    "                            xlim=[1, num_epochs])\n",
    "    # Sum of normalized losses, no. of normalized losses\n",
    "    metric = d2l.Accumulator(2)\n",
    "    loss = SigmoidBCELoss()\n",
    "    for epoch in range(num_epochs):\n",
    "        timer = d2l.Timer()\n",
    "        for i in range(num_batches):\n",
    "            optimizer.zero_grad()\n",
    "            center, context_negative, mask, label = random_batch(filename,vocab,n,max_window_size,num_noise_words)\n",
    "            pred = skip_gram(center, context_negative, net[0], net[1])\n",
    "            l = (loss(pred.reshape(label.shape).float(), label.float(), mask)\n",
    "                     / mask.sum(axis=1) * mask.shape[1])\n",
    "            l.sum().backward()\n",
    "            optimizer.step()\n",
    "            metric.add(l.sum(), l.numel())\n",
    "            if (i + 1) % (num_batches // 5) == 0 or i == num_batches - 1:\n",
    "                animator.add(epoch + (i + 1) / num_batches,\n",
    "                             (metric[0] / metric[1],))\n",
    "    print(f'loss {metric[0] / metric[1]:.3f}, '\n",
    "          f'{metric[1] / timer.stop():.1f} tokens/sec on {str(device)}')\n",
    "    return metric[0] / metric[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "id": "3bd83c85-3d63-418d-a5e4-0e73220a2fe0",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "data_dir = d2l.download_extract('ptb')\n",
    "filename = os.path.join(data_dir, 'ptb.train.txt')\n",
    "vocab = Vocab(filename=filename)\n",
    "# sentences = get_random_line(filename,100)\n",
    "# sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "id": "78975b9d-45cf-4364-9f58-8373c793cbad",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss 0.444, 24885.4 tokens/sec on cpu\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.4439971089160742"
      ]
     },
     "execution_count": 224,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       "  \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<svg height=\"182.870271pt\" version=\"1.1\" viewBox=\"0 0 253.825 182.870271\" width=\"253.825pt\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       " <metadata>\n",
       "  <rdf:RDF xmlns:cc=\"http://creativecommons.org/ns#\" xmlns:dc=\"http://purl.org/dc/elements/1.1/\" xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\">\n",
       "   <cc:Work>\n",
       "    <dc:type rdf:resource=\"http://purl.org/dc/dcmitype/StillImage\"/>\n",
       "    <dc:date>2023-10-19T02:52:40.264552</dc:date>\n",
       "    <dc:format>image/svg+xml</dc:format>\n",
       "    <dc:creator>\n",
       "     <cc:Agent>\n",
       "      <dc:title>Matplotlib v3.4.0, https://matplotlib.org/</dc:title>\n",
       "     </cc:Agent>\n",
       "    </dc:creator>\n",
       "   </cc:Work>\n",
       "  </rdf:RDF>\n",
       " </metadata>\n",
       " <defs>\n",
       "  <style type=\"text/css\">*{stroke-linecap:butt;stroke-linejoin:round;}</style>\n",
       " </defs>\n",
       " <g id=\"figure_1\">\n",
       "  <g id=\"patch_1\">\n",
       "   <path d=\"M 0 182.870271 \n",
       "L 253.825 182.870271 \n",
       "L 253.825 -0 \n",
       "L 0 -0 \n",
       "z\n",
       "\" style=\"fill:#ffffff;\"/>\n",
       "  </g>\n",
       "  <g id=\"axes_1\">\n",
       "   <g id=\"patch_2\">\n",
       "    <path d=\"M 46.325 148.420271 \n",
       "L 241.625 148.420271 \n",
       "L 241.625 9.820271 \n",
       "L 46.325 9.820271 \n",
       "z\n",
       "\" style=\"fill:#ffffff;\"/>\n",
       "   </g>\n",
       "   <g id=\"matplotlib.axis_1\">\n",
       "    <g id=\"xtick_1\">\n",
       "     <g id=\"line2d_1\">\n",
       "      <path clip-path=\"url(#pacf88afa7c)\" d=\"M 68.025 148.420271 \n",
       "L 68.025 9.820271 \n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\n",
       "     </g>\n",
       "     <g id=\"line2d_2\">\n",
       "      <defs>\n",
       "       <path d=\"M 0 0 \n",
       "L 0 3.5 \n",
       "\" id=\"mce001325b8\" style=\"stroke:#000000;stroke-width:0.8;\"/>\n",
       "      </defs>\n",
       "      <g>\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"68.025\" xlink:href=\"#mce001325b8\" y=\"148.420271\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_1\">\n",
       "      <!-- 2 -->\n",
       "      <g transform=\"translate(65.525 162.295271)scale(0.1 -0.1)\">\n",
       "       <defs>\n",
       "        <path d=\"M 300 250 \n",
       "Q 325 625 650 925 \n",
       "Q 975 1225 1475 1862 \n",
       "Q 1975 2500 2125 2850 \n",
       "Q 2275 3200 2237 3450 \n",
       "Q 2200 3700 2000 3862 \n",
       "Q 1800 4025 1537 4000 \n",
       "Q 1275 3975 1037 3800 \n",
       "Q 800 3625 675 3275 \n",
       "L 200 3350 \n",
       "Q 400 3925 712 4187 \n",
       "Q 1025 4450 1450 4475 \n",
       "Q 1700 4500 1900 4462 \n",
       "Q 2100 4425 2312 4287 \n",
       "Q 2525 4150 2662 3875 \n",
       "Q 2800 3600 2762 3212 \n",
       "Q 2725 2825 2375 2287 \n",
       "Q 2025 1750 1025 600 \n",
       "L 2825 600 \n",
       "L 2825 150 \n",
       "L 300 150 \n",
       "L 300 250 \n",
       "z\n",
       "\" id=\"SimHei-32\" transform=\"scale(0.015625)\"/>\n",
       "       </defs>\n",
       "       <use xlink:href=\"#SimHei-32\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"xtick_2\">\n",
       "     <g id=\"line2d_3\">\n",
       "      <path clip-path=\"url(#pacf88afa7c)\" d=\"M 111.425 148.420271 \n",
       "L 111.425 9.820271 \n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\n",
       "     </g>\n",
       "     <g id=\"line2d_4\">\n",
       "      <g>\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"111.425\" xlink:href=\"#mce001325b8\" y=\"148.420271\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_2\">\n",
       "      <!-- 4 -->\n",
       "      <g transform=\"translate(108.925 162.295271)scale(0.1 -0.1)\">\n",
       "       <defs>\n",
       "        <path d=\"M 2000 1100 \n",
       "L 75 1100 \n",
       "L 75 1525 \n",
       "L 2100 4450 \n",
       "L 2475 4450 \n",
       "L 2475 1525 \n",
       "L 3075 1525 \n",
       "L 3075 1100 \n",
       "L 2475 1100 \n",
       "L 2475 150 \n",
       "L 2000 150 \n",
       "L 2000 1100 \n",
       "z\n",
       "M 2000 1525 \n",
       "L 2000 3500 \n",
       "L 600 1525 \n",
       "L 2000 1525 \n",
       "z\n",
       "\" id=\"SimHei-34\" transform=\"scale(0.015625)\"/>\n",
       "       </defs>\n",
       "       <use xlink:href=\"#SimHei-34\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"xtick_3\">\n",
       "     <g id=\"line2d_5\">\n",
       "      <path clip-path=\"url(#pacf88afa7c)\" d=\"M 154.825 148.420271 \n",
       "L 154.825 9.820271 \n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\n",
       "     </g>\n",
       "     <g id=\"line2d_6\">\n",
       "      <g>\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"154.825\" xlink:href=\"#mce001325b8\" y=\"148.420271\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_3\">\n",
       "      <!-- 6 -->\n",
       "      <g transform=\"translate(152.325 162.295271)scale(0.1 -0.1)\">\n",
       "       <defs>\n",
       "        <path d=\"M 250 1612 \n",
       "Q 275 1975 387 2225 \n",
       "Q 500 2475 725 2850 \n",
       "L 1750 4450 \n",
       "L 2325 4450 \n",
       "L 1275 2800 \n",
       "Q 1950 2975 2350 2750 \n",
       "Q 2750 2525 2887 2237 \n",
       "Q 3025 1950 3037 1612 \n",
       "Q 3050 1275 2937 950 \n",
       "Q 2825 625 2537 362 \n",
       "Q 2250 100 1737 75 \n",
       "Q 1225 50 862 262 \n",
       "Q 500 475 362 862 \n",
       "Q 225 1250 250 1612 \n",
       "z\n",
       "M 1025 787 \n",
       "Q 1250 550 1625 525 \n",
       "Q 2000 500 2250 775 \n",
       "Q 2500 1050 2500 1575 \n",
       "Q 2500 2100 2187 2300 \n",
       "Q 1875 2500 1487 2450 \n",
       "Q 1100 2400 925 2075 \n",
       "Q 750 1750 775 1387 \n",
       "Q 800 1025 1025 787 \n",
       "z\n",
       "\" id=\"SimHei-36\" transform=\"scale(0.015625)\"/>\n",
       "       </defs>\n",
       "       <use xlink:href=\"#SimHei-36\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"xtick_4\">\n",
       "     <g id=\"line2d_7\">\n",
       "      <path clip-path=\"url(#pacf88afa7c)\" d=\"M 198.225 148.420271 \n",
       "L 198.225 9.820271 \n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\n",
       "     </g>\n",
       "     <g id=\"line2d_8\">\n",
       "      <g>\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"198.225\" xlink:href=\"#mce001325b8\" y=\"148.420271\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_4\">\n",
       "      <!-- 8 -->\n",
       "      <g transform=\"translate(195.725 162.295271)scale(0.1 -0.1)\">\n",
       "       <defs>\n",
       "        <path d=\"M 175 1375 \n",
       "Q 175 1675 325 1962 \n",
       "Q 475 2250 825 2425 \n",
       "Q 525 2600 425 2812 \n",
       "Q 325 3025 312 3300 \n",
       "Q 300 3575 387 3775 \n",
       "Q 475 3975 650 4150 \n",
       "Q 825 4325 1037 4387 \n",
       "Q 1250 4450 1500 4450 \n",
       "Q 1750 4450 1950 4400 \n",
       "Q 2150 4350 2375 4187 \n",
       "Q 2600 4025 2700 3725 \n",
       "Q 2800 3425 2687 3025 \n",
       "Q 2575 2625 2100 2400 \n",
       "Q 2525 2275 2700 2012 \n",
       "Q 2875 1750 2875 1375 \n",
       "Q 2875 1000 2762 775 \n",
       "Q 2650 550 2512 400 \n",
       "Q 2375 250 2137 162 \n",
       "Q 1900 75 1537 75 \n",
       "Q 1175 75 912 162 \n",
       "Q 650 250 475 425 \n",
       "Q 300 600 237 837 \n",
       "Q 175 1075 175 1375 \n",
       "z\n",
       "M 687 1400 \n",
       "Q 675 1100 787 875 \n",
       "Q 900 650 1200 587 \n",
       "Q 1500 525 1825 600 \n",
       "Q 2150 675 2275 950 \n",
       "Q 2400 1225 2362 1500 \n",
       "Q 2325 1775 2050 1962 \n",
       "Q 1775 2150 1450 2125 \n",
       "Q 1125 2100 912 1900 \n",
       "Q 700 1700 687 1400 \n",
       "z\n",
       "M 775 3350 \n",
       "Q 775 3100 950 2875 \n",
       "Q 1125 2650 1500 2650 \n",
       "Q 1875 2650 2062 2875 \n",
       "Q 2250 3100 2237 3412 \n",
       "Q 2225 3725 2012 3875 \n",
       "Q 1800 4025 1437 4000 \n",
       "Q 1075 3975 925 3787 \n",
       "Q 775 3600 775 3350 \n",
       "z\n",
       "\" id=\"SimHei-38\" transform=\"scale(0.015625)\"/>\n",
       "       </defs>\n",
       "       <use xlink:href=\"#SimHei-38\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"xtick_5\">\n",
       "     <g id=\"line2d_9\">\n",
       "      <path clip-path=\"url(#pacf88afa7c)\" d=\"M 241.625 148.420271 \n",
       "L 241.625 9.820271 \n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\n",
       "     </g>\n",
       "     <g id=\"line2d_10\">\n",
       "      <g>\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"241.625\" xlink:href=\"#mce001325b8\" y=\"148.420271\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_5\">\n",
       "      <!-- 10 -->\n",
       "      <g transform=\"translate(236.625 162.295271)scale(0.1 -0.1)\">\n",
       "       <defs>\n",
       "        <path d=\"M 1400 3600 \n",
       "Q 1075 3275 575 2975 \n",
       "L 575 3450 \n",
       "Q 1200 3875 1600 4450 \n",
       "L 1900 4450 \n",
       "L 1900 150 \n",
       "L 1400 150 \n",
       "L 1400 3600 \n",
       "z\n",
       "\" id=\"SimHei-31\" transform=\"scale(0.015625)\"/>\n",
       "        <path d=\"M 225 2537 \n",
       "Q 250 3200 412 3587 \n",
       "Q 575 3975 875 4225 \n",
       "Q 1175 4475 1612 4475 \n",
       "Q 2050 4475 2375 4112 \n",
       "Q 2700 3750 2800 3200 \n",
       "Q 2900 2650 2862 1937 \n",
       "Q 2825 1225 2612 775 \n",
       "Q 2400 325 1975 150 \n",
       "Q 1550 -25 1125 187 \n",
       "Q 700 400 525 750 \n",
       "Q 350 1100 275 1487 \n",
       "Q 200 1875 225 2537 \n",
       "z\n",
       "M 750 2687 \n",
       "Q 675 2000 800 1462 \n",
       "Q 925 925 1212 700 \n",
       "Q 1500 475 1800 612 \n",
       "Q 2100 750 2237 1162 \n",
       "Q 2375 1575 2375 2062 \n",
       "Q 2375 2550 2337 2950 \n",
       "Q 2300 3350 2112 3675 \n",
       "Q 1925 4000 1612 4012 \n",
       "Q 1300 4025 1062 3700 \n",
       "Q 825 3375 750 2687 \n",
       "z\n",
       "\" id=\"SimHei-30\" transform=\"scale(0.015625)\"/>\n",
       "       </defs>\n",
       "       <use xlink:href=\"#SimHei-31\"/>\n",
       "       <use x=\"50\" xlink:href=\"#SimHei-30\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"text_6\">\n",
       "     <!-- epoch -->\n",
       "     <g transform=\"translate(131.475 174.420271)scale(0.1 -0.1)\">\n",
       "      <defs>\n",
       "       <path d=\"M 2850 1075 \n",
       "Q 2800 625 2450 350 \n",
       "Q 2100 75 1625 75 \n",
       "Q 1025 75 637 462 \n",
       "Q 250 850 250 1525 \n",
       "Q 250 2200 637 2587 \n",
       "Q 1025 2975 1625 2975 \n",
       "Q 2150 2975 2487 2637 \n",
       "Q 2825 2300 2825 1525 \n",
       "L 800 1525 \n",
       "Q 800 975 1037 750 \n",
       "Q 1275 525 1625 525 \n",
       "Q 1900 525 2075 662 \n",
       "Q 2250 800 2300 1075 \n",
       "L 2850 1075 \n",
       "z\n",
       "M 2250 1925 \n",
       "Q 2200 2275 2025 2412 \n",
       "Q 1850 2550 1575 2550 \n",
       "Q 1325 2550 1125 2412 \n",
       "Q 925 2275 825 1925 \n",
       "L 2250 1925 \n",
       "z\n",
       "\" id=\"SimHei-65\" transform=\"scale(0.015625)\"/>\n",
       "       <path d=\"M 2825 1525 \n",
       "Q 2825 800 2475 425 \n",
       "Q 2125 50 1625 50 \n",
       "Q 1350 50 1150 162 \n",
       "Q 950 275 800 500 \n",
       "L 800 -800 \n",
       "L 300 -800 \n",
       "L 300 2925 \n",
       "L 800 2925 \n",
       "L 800 2550 \n",
       "Q 950 2775 1150 2875 \n",
       "Q 1350 2975 1625 2975 \n",
       "Q 2125 2975 2475 2612 \n",
       "Q 2825 2250 2825 1525 \n",
       "z\n",
       "M 2275 1525 \n",
       "Q 2275 2000 2087 2262 \n",
       "Q 1900 2525 1525 2525 \n",
       "Q 1225 2525 1012 2262 \n",
       "Q 800 2000 800 1525 \n",
       "Q 800 1050 1012 775 \n",
       "Q 1225 500 1525 500 \n",
       "Q 1900 500 2087 775 \n",
       "Q 2275 1050 2275 1525 \n",
       "z\n",
       "\" id=\"SimHei-70\" transform=\"scale(0.015625)\"/>\n",
       "       <path d=\"M 2925 1525 \n",
       "Q 2925 875 2525 475 \n",
       "Q 2125 75 1575 75 \n",
       "Q 1025 75 625 475 \n",
       "Q 225 875 225 1525 \n",
       "Q 225 2175 625 2575 \n",
       "Q 1025 2975 1575 2975 \n",
       "Q 2125 2975 2525 2575 \n",
       "Q 2925 2175 2925 1525 \n",
       "z\n",
       "M 2375 1525 \n",
       "Q 2375 2025 2125 2275 \n",
       "Q 1875 2525 1575 2525 \n",
       "Q 1275 2525 1025 2275 \n",
       "Q 775 2025 775 1525 \n",
       "Q 775 1025 1025 775 \n",
       "Q 1275 525 1575 525 \n",
       "Q 1875 525 2125 775 \n",
       "Q 2375 1025 2375 1525 \n",
       "z\n",
       "\" id=\"SimHei-6f\" transform=\"scale(0.015625)\"/>\n",
       "       <path d=\"M 2850 1250 \n",
       "Q 2850 725 2487 400 \n",
       "Q 2125 75 1575 75 \n",
       "Q 1025 75 625 462 \n",
       "Q 225 850 225 1525 \n",
       "Q 225 2200 625 2587 \n",
       "Q 1025 2975 1575 2975 \n",
       "Q 2125 2975 2450 2687 \n",
       "Q 2775 2400 2775 2000 \n",
       "L 2225 2000 \n",
       "Q 2200 2300 2012 2412 \n",
       "Q 1825 2525 1575 2525 \n",
       "Q 1275 2525 1025 2287 \n",
       "Q 775 2050 775 1525 \n",
       "Q 775 1000 1025 762 \n",
       "Q 1275 525 1575 525 \n",
       "Q 1900 525 2100 700 \n",
       "Q 2300 875 2300 1250 \n",
       "L 2850 1250 \n",
       "z\n",
       "\" id=\"SimHei-63\" transform=\"scale(0.015625)\"/>\n",
       "       <path d=\"M 2800 125 \n",
       "L 2300 125 \n",
       "L 2300 1925 \n",
       "Q 2300 2225 2150 2400 \n",
       "Q 2000 2575 1750 2575 \n",
       "Q 1425 2575 1137 2237 \n",
       "Q 850 1900 850 1400 \n",
       "L 850 125 \n",
       "L 350 125 \n",
       "L 350 4400 \n",
       "L 850 4400 \n",
       "L 850 2400 \n",
       "Q 1050 2675 1287 2825 \n",
       "Q 1525 2975 1900 2975 \n",
       "Q 2350 2975 2575 2725 \n",
       "Q 2800 2475 2800 2100 \n",
       "L 2800 125 \n",
       "z\n",
       "\" id=\"SimHei-68\" transform=\"scale(0.015625)\"/>\n",
       "      </defs>\n",
       "      <use xlink:href=\"#SimHei-65\"/>\n",
       "      <use x=\"50\" xlink:href=\"#SimHei-70\"/>\n",
       "      <use x=\"100\" xlink:href=\"#SimHei-6f\"/>\n",
       "      <use x=\"150\" xlink:href=\"#SimHei-63\"/>\n",
       "      <use x=\"200\" xlink:href=\"#SimHei-68\"/>\n",
       "     </g>\n",
       "    </g>\n",
       "   </g>\n",
       "   <g id=\"matplotlib.axis_2\">\n",
       "    <g id=\"ytick_1\">\n",
       "     <g id=\"line2d_11\">\n",
       "      <path clip-path=\"url(#pacf88afa7c)\" d=\"M 46.325 139.037194 \n",
       "L 241.625 139.037194 \n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\n",
       "     </g>\n",
       "     <g id=\"line2d_12\">\n",
       "      <defs>\n",
       "       <path d=\"M 0 0 \n",
       "L -3.5 0 \n",
       "\" id=\"m5ef138e4e6\" style=\"stroke:#000000;stroke-width:0.8;\"/>\n",
       "      </defs>\n",
       "      <g>\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"46.325\" xlink:href=\"#m5ef138e4e6\" y=\"139.037194\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_7\">\n",
       "      <!-- 0.45 -->\n",
       "      <g transform=\"translate(19.325 142.474694)scale(0.1 -0.1)\">\n",
       "       <defs>\n",
       "        <path d=\"M 1075 125 \n",
       "L 500 125 \n",
       "L 500 675 \n",
       "L 1075 675 \n",
       "L 1075 125 \n",
       "z\n",
       "\" id=\"SimHei-2e\" transform=\"scale(0.015625)\"/>\n",
       "        <path d=\"M 550 1325 \n",
       "Q 725 650 1150 575 \n",
       "Q 1575 500 1837 662 \n",
       "Q 2100 825 2212 1087 \n",
       "Q 2325 1350 2312 1675 \n",
       "Q 2300 2000 2137 2225 \n",
       "Q 1975 2450 1725 2525 \n",
       "Q 1475 2600 1162 2525 \n",
       "Q 850 2450 650 2175 \n",
       "L 225 2225 \n",
       "Q 275 2375 700 4375 \n",
       "L 2675 4375 \n",
       "L 2675 3925 \n",
       "L 1075 3925 \n",
       "Q 950 3250 825 2850 \n",
       "Q 1200 3025 1525 3012 \n",
       "Q 1850 3000 2150 2862 \n",
       "Q 2450 2725 2587 2487 \n",
       "Q 2725 2250 2787 2012 \n",
       "Q 2850 1775 2837 1500 \n",
       "Q 2825 1225 2725 937 \n",
       "Q 2625 650 2425 462 \n",
       "Q 2225 275 1937 162 \n",
       "Q 1650 50 1275 75 \n",
       "Q 900 100 562 350 \n",
       "Q 225 600 100 1200 \n",
       "L 550 1325 \n",
       "z\n",
       "\" id=\"SimHei-35\" transform=\"scale(0.015625)\"/>\n",
       "       </defs>\n",
       "       <use xlink:href=\"#SimHei-30\"/>\n",
       "       <use x=\"50\" xlink:href=\"#SimHei-2e\"/>\n",
       "       <use x=\"100\" xlink:href=\"#SimHei-34\"/>\n",
       "       <use x=\"150\" xlink:href=\"#SimHei-35\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"ytick_2\">\n",
       "     <g id=\"line2d_13\">\n",
       "      <path clip-path=\"url(#pacf88afa7c)\" d=\"M 46.325 113.357255 \n",
       "L 241.625 113.357255 \n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\n",
       "     </g>\n",
       "     <g id=\"line2d_14\">\n",
       "      <g>\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"46.325\" xlink:href=\"#m5ef138e4e6\" y=\"113.357255\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_8\">\n",
       "      <!-- 0.50 -->\n",
       "      <g transform=\"translate(19.325 116.794755)scale(0.1 -0.1)\">\n",
       "       <use xlink:href=\"#SimHei-30\"/>\n",
       "       <use x=\"50\" xlink:href=\"#SimHei-2e\"/>\n",
       "       <use x=\"100\" xlink:href=\"#SimHei-35\"/>\n",
       "       <use x=\"150\" xlink:href=\"#SimHei-30\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"ytick_3\">\n",
       "     <g id=\"line2d_15\">\n",
       "      <path clip-path=\"url(#pacf88afa7c)\" d=\"M 46.325 87.677316 \n",
       "L 241.625 87.677316 \n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\n",
       "     </g>\n",
       "     <g id=\"line2d_16\">\n",
       "      <g>\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"46.325\" xlink:href=\"#m5ef138e4e6\" y=\"87.677316\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_9\">\n",
       "      <!-- 0.55 -->\n",
       "      <g transform=\"translate(19.325 91.114816)scale(0.1 -0.1)\">\n",
       "       <use xlink:href=\"#SimHei-30\"/>\n",
       "       <use x=\"50\" xlink:href=\"#SimHei-2e\"/>\n",
       "       <use x=\"100\" xlink:href=\"#SimHei-35\"/>\n",
       "       <use x=\"150\" xlink:href=\"#SimHei-35\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"ytick_4\">\n",
       "     <g id=\"line2d_17\">\n",
       "      <path clip-path=\"url(#pacf88afa7c)\" d=\"M 46.325 61.997378 \n",
       "L 241.625 61.997378 \n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\n",
       "     </g>\n",
       "     <g id=\"line2d_18\">\n",
       "      <g>\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"46.325\" xlink:href=\"#m5ef138e4e6\" y=\"61.997378\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_10\">\n",
       "      <!-- 0.60 -->\n",
       "      <g transform=\"translate(19.325 65.434878)scale(0.1 -0.1)\">\n",
       "       <use xlink:href=\"#SimHei-30\"/>\n",
       "       <use x=\"50\" xlink:href=\"#SimHei-2e\"/>\n",
       "       <use x=\"100\" xlink:href=\"#SimHei-36\"/>\n",
       "       <use x=\"150\" xlink:href=\"#SimHei-30\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"ytick_5\">\n",
       "     <g id=\"line2d_19\">\n",
       "      <path clip-path=\"url(#pacf88afa7c)\" d=\"M 46.325 36.317439 \n",
       "L 241.625 36.317439 \n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\n",
       "     </g>\n",
       "     <g id=\"line2d_20\">\n",
       "      <g>\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"46.325\" xlink:href=\"#m5ef138e4e6\" y=\"36.317439\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_11\">\n",
       "      <!-- 0.65 -->\n",
       "      <g transform=\"translate(19.325 39.754939)scale(0.1 -0.1)\">\n",
       "       <use xlink:href=\"#SimHei-30\"/>\n",
       "       <use x=\"50\" xlink:href=\"#SimHei-2e\"/>\n",
       "       <use x=\"100\" xlink:href=\"#SimHei-36\"/>\n",
       "       <use x=\"150\" xlink:href=\"#SimHei-35\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"ytick_6\">\n",
       "     <g id=\"line2d_21\">\n",
       "      <path clip-path=\"url(#pacf88afa7c)\" d=\"M 46.325 10.6375 \n",
       "L 241.625 10.6375 \n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\n",
       "     </g>\n",
       "     <g id=\"line2d_22\">\n",
       "      <g>\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"46.325\" xlink:href=\"#m5ef138e4e6\" y=\"10.6375\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_12\">\n",
       "      <!-- 0.70 -->\n",
       "      <g transform=\"translate(19.325 14.075)scale(0.1 -0.1)\">\n",
       "       <defs>\n",
       "        <path d=\"M 850 150 \n",
       "Q 1300 2050 2425 3925 \n",
       "L 275 3925 \n",
       "L 275 4375 \n",
       "L 2950 4375 \n",
       "L 2950 3950 \n",
       "Q 1775 2050 1400 150 \n",
       "L 850 150 \n",
       "z\n",
       "\" id=\"SimHei-37\" transform=\"scale(0.015625)\"/>\n",
       "       </defs>\n",
       "       <use xlink:href=\"#SimHei-30\"/>\n",
       "       <use x=\"50\" xlink:href=\"#SimHei-2e\"/>\n",
       "       <use x=\"100\" xlink:href=\"#SimHei-37\"/>\n",
       "       <use x=\"150\" xlink:href=\"#SimHei-30\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"text_13\">\n",
       "     <!-- loss -->\n",
       "     <g transform=\"translate(14.075 89.120271)rotate(-90)scale(0.1 -0.1)\">\n",
       "      <defs>\n",
       "       <path d=\"M 1825 125 \n",
       "L 1325 125 \n",
       "L 1325 4400 \n",
       "L 1825 4400 \n",
       "L 1825 125 \n",
       "z\n",
       "\" id=\"SimHei-6c\" transform=\"scale(0.015625)\"/>\n",
       "       <path d=\"M 2750 900 \n",
       "Q 2750 500 2437 287 \n",
       "Q 2125 75 1650 75 \n",
       "Q 1050 75 725 312 \n",
       "Q 400 550 400 1000 \n",
       "L 900 1000 \n",
       "Q 900 700 1112 600 \n",
       "Q 1325 500 1625 500 \n",
       "Q 1925 500 2075 612 \n",
       "Q 2225 725 2225 900 \n",
       "Q 2225 1025 2100 1150 \n",
       "Q 1975 1275 1475 1350 \n",
       "Q 900 1425 687 1637 \n",
       "Q 475 1850 475 2200 \n",
       "Q 475 2500 762 2737 \n",
       "Q 1050 2975 1600 2975 \n",
       "Q 2100 2975 2387 2750 \n",
       "Q 2675 2525 2675 2150 \n",
       "L 2175 2150 \n",
       "Q 2175 2375 2012 2462 \n",
       "Q 1850 2550 1600 2550 \n",
       "Q 1275 2550 1137 2437 \n",
       "Q 1000 2325 1000 2175 \n",
       "Q 1000 2000 1125 1900 \n",
       "Q 1250 1800 1650 1750 \n",
       "Q 2300 1650 2525 1437 \n",
       "Q 2750 1225 2750 900 \n",
       "z\n",
       "\" id=\"SimHei-73\" transform=\"scale(0.015625)\"/>\n",
       "      </defs>\n",
       "      <use xlink:href=\"#SimHei-6c\"/>\n",
       "      <use x=\"50\" xlink:href=\"#SimHei-6f\"/>\n",
       "      <use x=\"100\" xlink:href=\"#SimHei-73\"/>\n",
       "      <use x=\"150\" xlink:href=\"#SimHei-73\"/>\n",
       "     </g>\n",
       "    </g>\n",
       "   </g>\n",
       "   <g id=\"line2d_23\">\n",
       "    <path clip-path=\"url(#pacf88afa7c)\" d=\"M 28.965 16.120271 \n",
       "L 33.305 25.579081 \n",
       "L 37.645 40.933982 \n",
       "L 41.985 55.294021 \n",
       "L 46.325 66.631933 \n",
       "L 50.665 75.628376 \n",
       "L 55.005 83.018855 \n",
       "L 59.345 89.095005 \n",
       "L 63.685 94.204506 \n",
       "L 68.025 98.553971 \n",
       "L 72.365 102.350418 \n",
       "L 76.705 105.574798 \n",
       "L 81.045 108.505042 \n",
       "L 85.385 111.062084 \n",
       "L 89.725 113.37227 \n",
       "L 94.065 115.485161 \n",
       "L 98.405 117.379487 \n",
       "L 102.745 119.122563 \n",
       "L 107.085 120.688686 \n",
       "L 111.425 122.148514 \n",
       "L 115.765 123.478724 \n",
       "L 120.105 124.718625 \n",
       "L 124.445 125.825883 \n",
       "L 128.785 126.896437 \n",
       "L 133.125 127.866817 \n",
       "L 137.465 128.825465 \n",
       "L 141.805 129.703801 \n",
       "L 146.145 130.548616 \n",
       "L 150.485 131.347782 \n",
       "L 154.825 132.109774 \n",
       "L 159.165 132.805192 \n",
       "L 163.505 133.476933 \n",
       "L 167.845 134.138924 \n",
       "L 172.185 134.765551 \n",
       "L 176.525 135.362339 \n",
       "L 180.865 135.932571 \n",
       "L 185.205 136.491483 \n",
       "L 189.545 137.019456 \n",
       "L 193.885 137.544083 \n",
       "L 198.225 138.024447 \n",
       "L 202.565 138.507698 \n",
       "L 206.905 138.952709 \n",
       "L 211.245 139.382068 \n",
       "L 215.585 139.810628 \n",
       "L 219.925 140.222053 \n",
       "L 224.265 140.64032 \n",
       "L 228.605 141.035515 \n",
       "L 232.945 141.421851 \n",
       "L 237.285 141.774685 \n",
       "L 241.625 142.120271 \n",
       "\" style=\"fill:none;stroke:#1f77b4;stroke-linecap:square;stroke-width:1.5;\"/>\n",
       "   </g>\n",
       "   <g id=\"patch_3\">\n",
       "    <path d=\"M 46.325 148.420271 \n",
       "L 46.325 9.820271 \n",
       "\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n",
       "   </g>\n",
       "   <g id=\"patch_4\">\n",
       "    <path d=\"M 241.625 148.420271 \n",
       "L 241.625 9.820271 \n",
       "\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n",
       "   </g>\n",
       "   <g id=\"patch_5\">\n",
       "    <path d=\"M 46.325 148.420271 \n",
       "L 241.625 148.420271 \n",
       "\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n",
       "   </g>\n",
       "   <g id=\"patch_6\">\n",
       "    <path d=\"M 46.325 9.820271 \n",
       "L 241.625 9.820271 \n",
       "\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n",
       "   </g>\n",
       "  </g>\n",
       " </g>\n",
       " <defs>\n",
       "  <clipPath id=\"pacf88afa7c\">\n",
       "   <rect height=\"138.6\" width=\"195.3\" x=\"46.325\" y=\"9.820271\"/>\n",
       "  </clipPath>\n",
       " </defs>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<Figure size 350x250 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# data_iter, vocab = load_data_ptb(512, 5, 5)\n",
    "lr, num_epochs = 0.002, 10\n",
    "embed_size = 100\n",
    "n = 100\n",
    "max_window_size = 5\n",
    "num_noise_words = 5\n",
    "num_batches = 100\n",
    "net = nn.Sequential(nn.Embedding(num_embeddings=len(vocab),\n",
    "                                 embedding_dim=embed_size),\n",
    "                    nn.Embedding(num_embeddings=len(vocab),\n",
    "                                 embedding_dim=embed_size))\n",
    "# train(net, data_iter, lr, num_epochs)\n",
    "train(net, filename,vocab,n,max_window_size,num_noise_words, lr, num_epochs, num_batches)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "id": "56389e7f-b616-4b29-ac5d-e1bea3ed80f3",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cosine sim=0.981: lighting\n",
      "cosine sim=0.977: reliance\n",
      "cosine sim=0.976: vanguard\n"
     ]
    }
   ],
   "source": [
    "get_similar_tokens('intel', 3, net[0])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:d2l]",
   "language": "python",
   "name": "conda-env-d2l-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}