"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"from IPython.display import display, HTML\n",
"display(HTML(''))"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "aEeFv1rMo9U6",
"outputId": "77b4cad4-6a6e-49cd-8d6f-164fd1c9c245"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Sun Feb 25 13:20:57 2024 \n",
"+---------------------------------------------------------------------------------------+\n",
"| NVIDIA-SMI 535.129.03 Driver Version: 535.129.03 CUDA Version: 12.2 |\n",
"|-----------------------------------------+----------------------+----------------------+\n",
"| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
"| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n",
"| | | MIG M. |\n",
"|=========================================+======================+======================|\n",
"| 0 NVIDIA GeForce RTX 4090 On | 00000000:01:00.0 Off | Off |\n",
"| 0% 36C P8 31W / 450W | 8684MiB / 24564MiB | 0% Default |\n",
"| | | N/A |\n",
"+-----------------------------------------+----------------------+----------------------+\n",
" \n",
"+---------------------------------------------------------------------------------------+\n",
"| Processes: |\n",
"| GPU GI CI PID Type Process name GPU Memory |\n",
"| ID ID Usage |\n",
"|=======================================================================================|\n",
"+---------------------------------------------------------------------------------------+\n"
]
}
],
"source": [
"!nvidia-smi"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
"\u001b[0m\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
"\u001b[0m\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
"\u001b[0m\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
"\u001b[0m"
]
}
],
"source": [
"!pip -q uninstall transformers -y\n",
"!pip -q install transformers[sentencepiece]\n",
"!pip -q install transformers\n",
"!pip -q install accelerate -U"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
"\u001b[0m"
]
}
],
"source": [
"!pip -q install pandas matplotlib torch nltk tqdm transformers datasets transformers[sentencepiece]"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "uqOtyVtZqKnK",
"outputId": "fefd4b58-2229-4345-d939-7b504bf05587"
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"[nltk_data] Downloading package punkt to /root/nltk_data...\n",
"[nltk_data] Package punkt is already up-to-date!\n",
"2024-02-25 13:21:18.076170: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
"2024-02-25 13:21:18.076240: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
"2024-02-25 13:21:18.077366: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
"2024-02-25 13:21:18.084780: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
"To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
"2024-02-25 13:21:19.164485: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n"
]
}
],
"source": [
"# Importing\n",
"import pandas as pd\n",
"import matplotlib.pyplot as plt\n",
"import torch\n",
"import nltk\n",
"\n",
"from nltk.tokenize import sent_tokenize\n",
"from tqdm import tqdm\n",
"nltk.download('punkt')\n",
"\n",
"# model download\n",
"from transformers import pipeline\n",
"from transformers import AutoTokenizer, AutoModelForSeq2SeqLM\n",
"from datasets import load_dataset, load_from_disk, load_metric\n",
"\n",
"# finetuning\n",
"from transformers import TrainingArguments, Trainer, DataCollatorForSeq2Seq\n",
"\n",
"import warnings\n",
"warnings.filterwarnings('ignore')"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "3viz1DDJqNRf",
"outputId": "a785e0e5-ce91-41da-d845-faa2e67489d8"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"gpu\n"
]
}
],
"source": [
"device='gpu' if torch.cuda.is_available() else 'cpu'\n",
"print(device)"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 706,
"referenced_widgets": [
"7b9695f9865e446aa754809eaf55e721",
"e0bdcb553f984708b8f27f02b1d2f81a",
"4cc45ad0cf594a0690113056d69d3733",
"da6ced8b0d764b79a660d926326e8c5e",
"bb6c9fed02284727903f833e9cba92d1",
"fc2c296479c147cc9abc27a0d733fa09",
"c43fc7d5821e414584063b61a404cdb7",
"b0faaad8b7444d6e9efe193116ba1579",
"45ffd3a5d95f424bb944702566e1702d",
"47c3749bfd3748c4a9422f894a9556d0",
"91e34f02a4cd4ae1b15a7161f8817217",
"e2c46cd9b1ea4d5b807ab51e6a0bac32",
"30008248f38d4970a5fa19f9d1d900b5",
"d9f1afd1e8b7431198fc240eee6a3d53",
"ca2ba491119645cda546b4bd2beb6167",
"c7f7b8791ba949899eebfbc90bccb4e9",
"1717c86dda5147e4a60acf7ce1c35345",
"317da8f59e2a4a938d51ac7724c83d05",
"6a6d7ada3bd34864a430b6847939bad4",
"9dc6baa3433e44dcbcf8de6ea73d9e19",
"82d4f240ce1f451fbe409738bfc51131",
"d0a7efdd3e4b40aba08312ab9c913f8b",
"14e37541c4e14a3d919b6b26e8356512",
"19a9e14bf58c43f1b5f6387c610d3c8e",
"0682ff2c837648edae487f9b352927e1",
"59e0bdb9ef4e4d76b785523cf842aa69",
"a652c05864024971b482167c7babc511",
"3634b73c77a14fca9c5c3b1a3486b21e",
"ad7ca3a6207c47abb7bdcd4c0f214741",
"07e53614051242818ed80df790a99a2c",
"e55c1e1d3e7449a0bb2c1128db39bf86",
"c42dacac96de49a198003763c55b94e1",
"984569ba89454f3bb5920d77b37dc4c1",
"9d2efc8aebca45b19f9265bea5e03708",
"1ee8014ed433466f8c863efbcca6105c",
"9c201a2e9b354757b2a316be7b2eecf2",
"8092d0ed7c584ce1b51aded31514222f",
"54ad0d74fe2646cc9cb898eb4c6bb010",
"5318a779763e465d87484167b9d6af51",
"2b25cd0c66d044518193e34edc53bfbe",
"0d15060ffcc0439fa61a050d9762e501",
"ca6ed50074f247479b4010e5a924eedc",
"6f9642cfcb1940e2affa0e2e1d7dd15d",
"c40e7e5d3f93413cb00e94e1b8bea28b",
"50e554b5a11a47cbae2a416e916b82da",
"56dad24a02444183b19132e75a8628b7",
"ba8288fac69f4784a60870d6ca4e2a12",
"25af764bf16f480687b51949441077dd",
"4ba9471e5f22408686aab425531bf8e4",
"bad2c5ad07c4498ba613ba199c3d88f9",
"28fe1b3cedcf4a5bacbec6ed2296d21a",
"804e6edf5ca2478cbfc3cebb26355ee4",
"9cea5e5986f44840b3d412cb4779b107",
"926f7bec7cdd4557b54cd1b93e755b01",
"84b91d991432460ca8f291842fe3c950",
"3ad9b947f051471180398c06108c1aaf",
"fa7f86bd160941c0902b7e69ed199844",
"e77f8c37a4e24c528fdc74c252f9a699",
"c7db02f33ad144c289223b31e0ee2533",
"914fd769008e489b9f6db4d710c9297d",
"09a714ca038a47058f642b5e930824b0",
"c31394ff745547288ab13629df046289",
"a9518cf4a351442ab2d98f415391f6a5",
"0f3956ace1ba46108b86d368569586f9",
"58465c8b171146d18415dd54fcc8647f",
"72486f059283449a8d507c86ace7c359"
]
},
"id": "ICJRXvIaqPXR",
"outputId": "6238d8bc-56cf-45ae-bd50-311ca63635a4"
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Some weights of PegasusForConditionalGeneration were not initialized from the model checkpoint at google/pegasus-cnn_dailymail and are newly initialized: ['model.decoder.embed_positions.weight', 'model.encoder.embed_positions.weight']\n",
"You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
]
}
],
"source": [
"model='google/pegasus-cnn_dailymail'\n",
"tokenizer=AutoTokenizer.from_pretrained(model)\n",
"model_pegasus=AutoModelForSeq2SeqLM.from_pretrained(model).to('cuda')"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {
"id": "ot0b0u3ys3X2"
},
"outputs": [
{
"data": {
"text/plain": [
"DatasetDict({\n",
" train: Dataset({\n",
" features: ['id', 'dialogue', 'summary'],\n",
" num_rows: 14732\n",
" })\n",
" test: Dataset({\n",
" features: ['id', 'dialogue', 'summary'],\n",
" num_rows: 819\n",
" })\n",
" validation: Dataset({\n",
" features: ['id', 'dialogue', 'summary'],\n",
" num_rows: 818\n",
" })\n",
"})"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"dataset=load_dataset('samsum')\n",
"dataset"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {
"id": "xY8UFUhatU9o"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Fetch first dialogue and summary\n",
"Amanda: I baked cookies. Do you want some?\n",
"Jerry: Sure!\n",
"Amanda: I'll bring you tomorrow :-) Amanda baked cookies and will bring Jerry some tomorrow.\n"
]
}
],
"source": [
"# train dataset\n",
"train = dataset['train']\n",
"\n",
"# dialogues in train dataset\n",
"dialogue = train['dialogue']\n",
"\n",
"# summary in train dataset\n",
"summary = train['summary']\n",
"\n",
"print('Fetch first dialogue and summary')\n",
"print(dialogue[0], summary[0])"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {
"id": "zYUGzUvnzq-i"
},
"outputs": [],
"source": [
"def convert_examples_to_features(data_in_batch):\n",
" # Tokenize the dialogues in the batch\n",
" input_encoding = tokenizer(data_in_batch['dialogue'],\n",
" max_length=1024,\n",
" truncation=True)\n",
"\n",
" # Tokenize the summaries in the batch\n",
" target_encoding = tokenizer(data_in_batch['summary'],\n",
" max_length=128,\n",
" truncation=True)\n",
"\n",
" # Return a dictionary containing input and target tokenized sequences\n",
" return {\n",
" 'input_ids': input_encoding['input_ids'], # Input token IDs for dialogues\n",
" 'attention_mask': input_encoding['attention_mask'], # Attention mask for dialogue inputs\n",
" 'labels': target_encoding['input_ids'] # Target token IDs for summaries (used as labels)\n",
" }"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {
"id": "Wh1VhhIotf1Z"
},
"outputs": [],
"source": [
"dataset_en = dataset.map(convert_examples_to_features, batched = True)"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {
"id": "JkuS9Md53ndP"
},
"outputs": [
{
"data": {
"text/plain": [
"Dataset({\n",
" features: ['id', 'dialogue', 'summary', 'input_ids', 'attention_mask', 'labels'],\n",
" num_rows: 14732\n",
"})"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/plain": [
"[12195,\n",
" 151,\n",
" 125,\n",
" 7091,\n",
" 3659,\n",
" 107,\n",
" 842,\n",
" 119,\n",
" 245,\n",
" 181,\n",
" 152,\n",
" 10508,\n",
" 151,\n",
" 7435,\n",
" 147,\n",
" 12195,\n",
" 151,\n",
" 125,\n",
" 131,\n",
" 267,\n",
" 650,\n",
" 119,\n",
" 3469,\n",
" 29344,\n",
" 1]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/plain": [
"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/plain": [
"[12195, 7091, 3659, 111, 138, 650, 10508, 181, 3469, 107, 1]"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# Display the first example from the training dataset\n",
"display(\n",
" dataset_en['train'], # Display the training dataset\n",
" dataset_en['train']['input_ids'][0], # Display the input token IDs of the first example\n",
" dataset_en['train']['attention_mask'][0], # Display the attention mask of the first example\n",
" dataset_en['train']['labels'][0] # Display the target token IDs (labels) of the first example\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {
"id": "CfOP_ysM7KaT"
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Detected kernel version 5.4.0, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher.\n"
]
},
{
"data": {
"text/html": [
"\n",
" \n",
" \n",
"
\n",
" [920/920 23:22, Epoch 0/1]\n",
"
\n",
" \n",
" \n",
" | Step | \n",
" Training Loss | \n",
" Validation Loss | \n",
"
\n",
" \n",
" \n",
" \n",
" | 500 | \n",
" 1.665300 | \n",
" 1.482481 | \n",
"
\n",
" \n",
"
"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/plain": [
"TrainOutput(global_step=920, training_loss=1.8287915426751842, metrics={'train_runtime': 1404.8123, 'train_samples_per_second': 10.487, 'train_steps_per_second': 0.655, 'total_flos': 5528248038285312.0, 'train_loss': 1.8287915426751842, 'epoch': 1.0})"
]
},
"execution_count": 13,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# reference: https://huggingface.co/docs/transformers/v4.38.1/en/main_classes/trainer#transformers.Seq2SeqTrainingArguments\n",
"seq2seq_data_collator = DataCollatorForSeq2Seq(tokenizer, model=model)\n",
"\n",
"training_args=TrainingArguments(\n",
" output_dir='pegasus-samsum',\n",
" num_train_epochs=1,\n",
" warmup_steps=500,\n",
" per_device_train_batch_size=1,\n",
" per_device_eval_batch_size=1,\n",
" weight_decay=0.01,\n",
" logging_steps=10,\n",
" evaluation_strategy='steps',\n",
" eval_steps=500,\n",
" save_steps=1e6,\n",
" gradient_accumulation_steps=16\n",
")\n",
"trainer=Trainer(model=model_pegasus,\n",
" args=training_args,\n",
" tokenizer=tokenizer,\n",
" data_collator=seq2seq_data_collator,\n",
" train_dataset=dataset_en['train'],\n",
" eval_dataset=dataset_en['validation'])\n",
"\n",
"trainer.train()"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {
"id": "h7DQmI2z8jky"
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Some non-default generation parameters are set in the model config. These should go into a GenerationConfig file (https://huggingface.co/docs/transformers/generation_strategies#save-a-custom-decoding-strategy-with-your-model) instead. This warning will be raised to an exception in v4.41.\n",
"Non-default generation parameters: {'max_length': 128, 'min_length': 32, 'num_beams': 8, 'length_penalty': 0.8, 'forced_eos_token_id': 1}\n"
]
},
{
"data": {
"text/plain": [
"('samsum-tokenizer/tokenizer_config.json',\n",
" 'samsum-tokenizer/special_tokens_map.json',\n",
" 'samsum-tokenizer/spiece.model',\n",
" 'samsum-tokenizer/added_tokens.json',\n",
" 'samsum-tokenizer/tokenizer.json')"
]
},
"execution_count": 14,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# Save the Pegasus model\n",
"model_pegasus.save_pretrained('pegasus-samsum-model')\n",
"\n",
"# Save the tokenizer used with the Pegasus model\n",
"tokenizer.save_pretrained('samsum-tokenizer')"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {
"id": "2BPx6ee3_s16"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Sample text: Tim: Hi, what's up?\n",
"Kim: Bad mood tbh, I was going to do lots of stuff but ended up procrastinating\n",
"Tim: What did you plan on doing?\n",
"Kim: Oh you know, uni stuff and unfucking my room\n",
"Kim: Maybe tomorrow I'll move my ass and do everything\n",
"Kim: We were going to defrost a fridge so instead of shopping I'll eat some defrosted veggies\n",
"Tim: For doing stuff I recommend Pomodoro technique where u use breaks for doing chores\n",
"Tim: It really helps\n",
"Kim: thanks, maybe I'll do that\n",
"Tim: I also like using post-its in kaban style\n"
]
}
],
"source": [
"# Load tokenizer and model from the pretrained directories\n",
"tokenizer = AutoTokenizer.from_pretrained('samsum-tokenizer')\n",
"model = AutoModelForSeq2SeqLM.from_pretrained('pegasus-samsum-model')\n",
"\n",
"# Create a pipeline object for summarization using the loaded model and tokenizer\n",
"pipeline_obj = pipeline('summarization', model=model, tokenizer=tokenizer)\n",
"\n",
"# Define a sample text for summarization\n",
"sample_text_for_text = dataset['train']['dialogue'][2]\n",
"actual_summary = dataset['train']['summary'][2]\n",
"print(f'Sample text: {sample_text_for_text}')"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Actual summary: Kim may try the pomodoro technique recommended by Tim to get more stuff done.\n"
]
}
],
"source": [
"print(f'Actual summary: {actual_summary}')"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {
"id": "aqWPIU6MCAO4"
},
"outputs": [
{
"data": {
"text/plain": [
"[{'summary_text': \"Kim was going to do lots of stuff but ended up procrastinating. She'll move her ass tomorrow and do everything. Tim recommends Pomodoro technique for doing chores.\"}]"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# Generate a summary using the pipeline object\n",
"gen_kwargs = {'length_penalty': 0.8, 'num_beams':8, 'max_length': 128}\n",
"display(pipeline_obj(sample_text_for_text, **gen_kwargs))\n",
"prediction = pipeline_obj(sample_text_for_text, **gen_kwargs)[0]['summary_text']"
]
},
{
"cell_type": "code",
"execution_count": 21,
"metadata": {
"id": "8V7-E1kLDd3k"
},
"outputs": [],
"source": [
"# Evaluation\n",
"# evalation of model\n",
"# Evaluation\n",
"\n",
"def generate_batch_sized_chunks(list_of_elements, batch_size):\n",
" '''split the dataset into smaller batches that we can process simultaneously\n",
" Yield successive batch-sized chunks from list_of_elements.'''\n",
" for i in range(0, len(list_of_elements), batch_size):\n",
" yield list_of_elements[i : i + batch_size]\n",
"\n",
"\n",
"\n",
"def calculate_metric_on_test_ds(dataset, metric, model, tokenizer,\n",
" batch_size=16, device=device,\n",
" column_text='article',\n",
" column_summary='highlights'):\n",
" # Divide input text data and target summaries into batches\n",
" article_batches = list(generate_batch_sized_chunks(dataset[column_text], batch_size))\n",
" target_batches = list(generate_batch_sized_chunks(dataset[column_summary], batch_size))\n",
"\n",
" # Iterate over batches and track progress using tqdm\n",
" for article_batch, target_batch in tqdm(zip(article_batches, target_batches), total=len(article_batches)):\n",
"\n",
" # Tokenize the input text data for model input\n",
" inputs = tokenizer(article_batch, max_length=1024, truncation=True,\n",
" padding='max_length', return_tensors='pt')\n",
"\n",
" # Generate summaries using the model\n",
" summaries = model.generate(input_ids=inputs['input_ids'].to('cuda'),\n",
" attention_mask=inputs['attention_mask'].to('cuda'),\n",
" length_penalty=0.8, num_beams=8, max_length=128)\n",
"\n",
" ''' Parameter for length penalty ensures that the model\n",
" does not generate sequences that are too long. '''\n",
"\n",
" # Decode the generated summaries for evaluation\n",
" decoded_summaries = [tokenizer.decode(s, skip_special_tokens=True,\n",
" clean_up_tokenization_spaces=True)\n",
" for s in summaries]\n",
"\n",
" decoded_summaries = [d.replace('', ' ') for d in decoded_summaries]\n",
"\n",
" # Add the generated summaries and references to the metric for evaluation\n",
" metric.add_batch(predictions=decoded_summaries, references=target_batch)\n",
"\n",
" # Finally compute and return the ROUGE scores.\n",
" score = metric.compute()\n",
" return score"
]
},
{
"cell_type": "code",
"execution_count": 22,
"metadata": {
"id": "RVTphNwbGuNN"
},
"outputs": [],
"source": [
"rouge_names=['rouge1', 'rouge2', 'rougeL', 'rougeLsum']\n",
"rouge_metric = load_metric('rouge')"
]
},
{
"cell_type": "code",
"execution_count": 23,
"metadata": {
"id": "ogY_PBYzHBXN"
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████| 5/5 [00:03<00:00, 1.30it/s]\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" rouge1 | \n",
" rouge2 | \n",
" rougeL | \n",
" rougeLsum | \n",
"
\n",
" \n",
" \n",
" \n",
" | pegasus | \n",
" 0.0247 | \n",
" 0.0 | \n",
" 0.024573 | \n",
" 0.024587 | \n",
"
\n",
" \n",
"
\n",
"