{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "cambiar mT5-large al ESP.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "8cad48bccf994bca9ef5c4110cf7e534": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_cd7299a39f2249afafc1045138aa881b",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_2b741c0b58cb4e29a1a650e469b8b8e6",
              "IPY_MODEL_89ec4f14405e4df09149be036fdeeb5d",
              "IPY_MODEL_34284d236f8c43b0b3066bef8bdc98cf"
            ]
          }
        },
        "cd7299a39f2249afafc1045138aa881b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2b741c0b58cb4e29a1a650e469b8b8e6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_c43e5a1fdf9d4a2481b6ac3c2403dc61",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2091196b6bc14b569ba5072b8c3e4205"
          }
        },
        "89ec4f14405e4df09149be036fdeeb5d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_5e3885db9d5f452c94b379a30f1def75",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 4309802,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 4309802,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1c836c595d634111b04bc983ad5db372"
          }
        },
        "34284d236f8c43b0b3066bef8bdc98cf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_2aea0ff0775f4364ad9b622b0b4ff8bb",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 4.31M/4.31M [00:00<00:00, 17.5MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4a22a0829ace4e7a8c804e56eb8debf0"
          }
        },
        "c43e5a1fdf9d4a2481b6ac3c2403dc61": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2091196b6bc14b569ba5072b8c3e4205": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5e3885db9d5f452c94b379a30f1def75": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1c836c595d634111b04bc983ad5db372": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2aea0ff0775f4364ad9b622b0b4ff8bb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4a22a0829ace4e7a8c804e56eb8debf0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2b7ddc64af344c78adc1457bd8c73ee9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_9406c92706664a1dbddbdc362ed45599",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_fbb858408de04686a6a183cf3a6ae444",
              "IPY_MODEL_db3cd507ded14c96afa325655efa0875",
              "IPY_MODEL_e2fafb5f704b4b10bba874e78b1df1cd"
            ]
          }
        },
        "9406c92706664a1dbddbdc362ed45599": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fbb858408de04686a6a183cf3a6ae444": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_0914bfb7ce564e9a89d5635462d672c9",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_942749c7b1004323a4e5ec1be68e7c30"
          }
        },
        "db3cd507ded14c96afa325655efa0875": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_41349f564d41451d81f0781f3d1cbeba",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 224480,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 224480,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_40f91c7615d64463b844406a8d161f0e"
          }
        },
        "e2fafb5f704b4b10bba874e78b1df1cd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_749fbf1e0e5144dcb77efb493f317af8",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 224480/224480 [00:46<00:00, 20170.69it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b678648bfe114127bae8ff235854d476"
          }
        },
        "0914bfb7ce564e9a89d5635462d672c9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "942749c7b1004323a4e5ec1be68e7c30": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "41349f564d41451d81f0781f3d1cbeba": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "40f91c7615d64463b844406a8d161f0e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "749fbf1e0e5144dcb77efb493f317af8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b678648bfe114127bae8ff235854d476": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 395,
          "referenced_widgets": [
            "8cad48bccf994bca9ef5c4110cf7e534",
            "cd7299a39f2249afafc1045138aa881b",
            "2b741c0b58cb4e29a1a650e469b8b8e6",
            "89ec4f14405e4df09149be036fdeeb5d",
            "34284d236f8c43b0b3066bef8bdc98cf",
            "c43e5a1fdf9d4a2481b6ac3c2403dc61",
            "2091196b6bc14b569ba5072b8c3e4205",
            "5e3885db9d5f452c94b379a30f1def75",
            "1c836c595d634111b04bc983ad5db372",
            "2aea0ff0775f4364ad9b622b0b4ff8bb",
            "4a22a0829ace4e7a8c804e56eb8debf0",
            "809710f020124e64b2094be82510813d",
            "c77fa699e17c4422a2cf4f4673621f63",
            "a122423e8af54b4b992199f0f29704eb",
            "9680c986255046debd765afc6ff0704b"
          ]
        },
        "id": "hVVGGIpK_eWg",
        "outputId": "62c670ed-f5c3-4fc3-b6fd-b7179784bfb5"
      },
      "source": [
        "!pip install --quiet transformers==4.5.0\n",
        "!pip install --quiet pytorch-lightning==1.2.7\n",
        "!pip install --quiet sentencepiece\n",
        "import torch\n",
        "from transformers import MT5ForConditionalGeneration, T5Tokenizer\n",
        "tokenizer = T5Tokenizer.from_pretrained(\"google/mt5-large\")\n",
        "model = MT5ForConditionalGeneration.from_pretrained('google/mt5-large')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[K     |████████████████████████████████| 2.1 MB 5.3 MB/s \n",
            "\u001b[K     |████████████████████████████████| 3.3 MB 31.7 MB/s \n",
            "\u001b[K     |████████████████████████████████| 895 kB 47.5 MB/s \n",
            "\u001b[K     |████████████████████████████████| 830 kB 5.3 MB/s \n",
            "\u001b[K     |████████████████████████████████| 125 kB 43.6 MB/s \n",
            "\u001b[K     |████████████████████████████████| 596 kB 21.9 MB/s \n",
            "\u001b[K     |████████████████████████████████| 282 kB 45.2 MB/s \n",
            "\u001b[K     |████████████████████████████████| 829 kB 31.9 MB/s \n",
            "\u001b[K     |████████████████████████████████| 1.3 MB 25.9 MB/s \n",
            "\u001b[K     |████████████████████████████████| 160 kB 46.5 MB/s \n",
            "\u001b[K     |████████████████████████████████| 271 kB 47.6 MB/s \n",
            "\u001b[?25h  Building wheel for future (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[K     |████████████████████████████████| 1.2 MB 5.2 MB/s \n",
            "\u001b[?25h"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "8cad48bccf994bca9ef5c4110cf7e534",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/4.31M [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "809710f020124e64b2094be82510813d",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/65.0 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c77fa699e17c4422a2cf4f4673621f63",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/376 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "a122423e8af54b4b992199f0f29704eb",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/642 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "9680c986255046debd765afc6ff0704b",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/4.92G [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zUOOEKO71Ffn",
        "outputId": "539de749-8584-4b91-f4ca-f2d9a731c88c"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "6WOm_8P91z_f",
        "outputId": "555e259b-a5ae-4b97-9e1c-2af3f1d28f41"
      },
      "source": [
        "import shutil\n",
        "\n",
        "# Copia el archivo desde el drive al entorno de ejecucion\n",
        "\n",
        "shutil.copy(\"./drive/MyDrive/spa_news_2020_1M.tar.gz\", \"./spa_news_2020_1M.tar.gz\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'./spa_news_2020_1M.tar.gz'"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NmVGLxim2QLK"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YbYUgCyDulVL"
      },
      "source": [
        "# importing the \"tarfile\" module\n",
        "import tarfile\n",
        "  \n",
        "# open file\n",
        "file = tarfile.open('spa_news_2020_1M.tar.gz')\n",
        "  \n",
        "# extracting file\n",
        "file.extractall('.')\n",
        "  \n",
        "file.close()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vcpqPHQ9_684",
        "outputId": "e437709b-0787-4236-af9a-289cbb86da7e"
      },
      "source": [
        "def msize(m):\n",
        "    return sum(p.numel() for p in m.parameters())\n",
        "print(msize(model.shared) / msize(model))   \n",
        "print(msize(model.lm_head) / msize(model))  "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0.2082942262544732\n",
            "0.2082942262544732\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 66,
          "referenced_widgets": [
            "933009db7dd043559dee189a56e91267"
          ]
        },
        "id": "aNsaPoDO_-9f",
        "outputId": "5b32091a-fc60-4f65-b2e0-aa6634b51610"
      },
      "source": [
        "import pandas as pd\n",
        "import csv\n",
        "from collections import Counter\n",
        "from tqdm.auto import tqdm, trange\n",
        "df_es = pd.read_csv('./spa_news_2020_1M/spa_news_2020_1M-sentences.txt', sep='\\t', header=None, quoting=csv.QUOTE_NONE)\n",
        "df_es.columns = ['idx', 'text']\n",
        "cnt_es = Counter()\n",
        "for text in tqdm(df_es.text):\n",
        "    cnt_es.update(tokenizer.encode(text))\n",
        "print(len(cnt_es), len(cnt_es)/tokenizer.vocab_size)  \n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "933009db7dd043559dee189a56e91267",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "  0%|          | 0/1000000 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "64840 0.2592562974810076\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "leo835PdAFW3",
        "outputId": "d1ea2118-ed93-4188-b2f7-e5e73923c3df"
      },
      "source": [
        "for top in 10_000, 20_000, 30_000:\n",
        "    print(top, sum(v for k, v in cnt_es.most_common(top)) / sum(cnt_es.values()))\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "10000 0.9709232293434434\n",
            "20000 0.9908633305930422\n",
            "30000 0.996493022559441\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xBJ044mfk0B_",
        "outputId": "272d1bcc-950d-46f7-b3f7-8ba9a6c6d18f"
      },
      "source": [
        "cnt_es.most_common(2)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[(259, 4129329), (269, 1622306)]"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "IzBqoqXil7hl",
        "outputId": "5a3e168b-7ae3-4432-ce19-5da8e2c8f378"
      },
      "source": [
        "tokenizer.decode(269)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'de'"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1Ou_XuyFjpsp"
      },
      "source": [
        "#with open(f'masComunesESP.txt', 'w', encoding='utf-8') as fp:\n",
        "#    fp.write(''.join(cnt_es.most_common(25_000)))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ua8-t9jgAJMX",
        "outputId": "eed6fed5-a8eb-46f5-afab-1176ec295848"
      },
      "source": [
        "new_tokens = set(range(1000))\n",
        "#for i, (k, v) in enumerate(cnt_en.most_common(10_000)):\n",
        "#    if k not in new_tokens:\n",
        "#        new_tokens.add(k)\n",
        "for i, (k, v) in enumerate(cnt_es.most_common(25_000)):\n",
        "    if len(new_tokens) == 29_900:\n",
        "        print(i, 'Esp tokens are included')\n",
        "        break\n",
        "    if k not in new_tokens:\n",
        "        new_tokens.add(k)\n",
        "for t in range(tokenizer.vocab_size - 100, tokenizer.vocab_size):\n",
        "    new_tokens.add(t)\n",
        "print(len(new_tokens))\n",
        "kept_ids = sorted(new_tokens)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "25620\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qQVpne_YAOFq"
      },
      "source": [
        "new_size = len(kept_ids)\n",
        "new_emb = torch.nn.Embedding(new_size, model.shared.embedding_dim)\n",
        "new_head = torch.nn.Linear(in_features=model.lm_head.in_features, out_features=new_size, bias=False)\n",
        "for new_id, old_id in enumerate(kept_ids):\n",
        "    new_emb.weight.data[new_id] = model.shared.weight.data[old_id]\n",
        "    new_head.weight.data[new_id] = model.lm_head.weight.data[old_id]\n",
        "model.shared.weight = new_emb.weight\n",
        "model.lm_head.weight = new_head.weight\n",
        "model.config.__dict__['vocab_size'] = new_size\n",
        "model.config.__dict__['_name_or_path'] = 'cointegrated/es5-large'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 284,
          "referenced_widgets": [
            "2b7ddc64af344c78adc1457bd8c73ee9",
            "9406c92706664a1dbddbdc362ed45599",
            "fbb858408de04686a6a183cf3a6ae444",
            "db3cd507ded14c96afa325655efa0875",
            "e2fafb5f704b4b10bba874e78b1df1cd",
            "0914bfb7ce564e9a89d5635462d672c9",
            "942749c7b1004323a4e5ec1be68e7c30",
            "41349f564d41451d81f0781f3d1cbeba",
            "40f91c7615d64463b844406a8d161f0e",
            "749fbf1e0e5144dcb77efb493f317af8",
            "b678648bfe114127bae8ff235854d476"
          ]
        },
        "id": "cU1s25H3AYkh",
        "outputId": "d7ebf177-767a-4e41-b7f8-08ae4faebf27"
      },
      "source": [
        "! wget https://raw.githubusercontent.com/google/sentencepiece/master/src/sentencepiece_model.proto\n",
        "! protoc --python_out=. sentencepiece_model.proto\n",
        "import sentencepiece_model_pb2 as spmp\n",
        "smp = tokenizer.sp_model.serialized_model_proto()\n",
        "m = spmp.ModelProto()\n",
        "m.ParseFromString(smp)\n",
        "print('the loaded model has pieces:', len(m.pieces))\n",
        "new_pieces = [m.pieces[idx] for idx in kept_ids]\n",
        "print('the new pieces:', len(new_pieces))\n",
        "# replace the content of the first 30K pieces\n",
        "for i, p in enumerate(new_pieces):\n",
        "    m.pieces[i].piece = p.piece\n",
        "    m.pieces[i].score = p.score\n",
        "    m.pieces[i].type = p.type\n",
        "# drop the remaining pieces\n",
        "n = len(new_pieces)\n",
        "for i in trange(len(m.pieces) - n):\n",
        "    m.pieces.pop(len(m.pieces) - 1)\n",
        "print(len(m.pieces))\n",
        "with open('new_sp.model', 'wb') as f:\n",
        "    f.write(m.SerializeToString())\n",
        "new_tokenizer = T5Tokenizer('new_sp.model', extra_ids=100)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2021-10-15 14:13:55--  https://raw.githubusercontent.com/google/sentencepiece/master/src/sentencepiece_model.proto\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.110.133, 185.199.108.133, 185.199.111.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.110.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 12872 (13K) [text/plain]\n",
            "Saving to: ‘sentencepiece_model.proto’\n",
            "\n",
            "sentencepiece_model 100%[===================>]  12.57K  --.-KB/s    in 0s      \n",
            "\n",
            "2021-10-15 14:13:55 (89.3 MB/s) - ‘sentencepiece_model.proto’ saved [12872/12872]\n",
            "\n",
            "the loaded model has pieces: 250100\n",
            "the new pieces: 25620\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "2b7ddc64af344c78adc1457bd8c73ee9",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "  0%|          | 0/224480 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "25620\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_ktnEtEJAe8h"
      },
      "source": [
        "new_tokenizer.save_pretrained('espt5-large')\n",
        "model.save_pretrained('espt5-large')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "ril8cbe7Mma3",
        "outputId": "f78ce065-a8bd-4c17-d733-82360745b65b"
      },
      "source": [
        "shutil.copytree(\"./espt5-large\", \"./drive/MyDrive/espt5-large\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'./drive/MyDrive/espt5-large'"
            ]
          },
          "metadata": {},
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zpnRbZBKWPQ5"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}