{ "cells": [ { "cell_type": "markdown", "id": "c7d02608-381e-487c-bfc8-7a534e794e21", "metadata": {}, "source": [ "# TTM-R3 Sample Notebook" ] }, { "cell_type": "markdown", "id": "d8258884-05e5-4e1d-9007-5027e5396cb9", "metadata": {}, "source": [ "# 🚀 TTM-R3 GIFT Evaluation Notebook\n", "\n", "---\n", "\n", "### 📦 Model\n", "- **TTM-R3 Model Card** \n", " 👉 https://huggingface.co/ibm-research/ttm-r3\n", "\n", "---\n", "\n", "### ⚙️ Setup Instructions\n", "- Follow the setup guide: \n", " 👉 https://github.com/ibm-granite/granite-tsfm/blob/ttm-r3-release-mq2/notebooks/hfdemo/tinytimemixer/full_benchmarking/gift_leaderboard_ttm_r3_nc/README.md\n", "- It requires cloning of `granite-tsfm` repo and moving this notebook to `gift_leaderboard_ttm_r3_nc` folder as per the instructions mentioned in the link.\n", "---\n", "\n", "### 📝 Notes\n", "- This notebook runs **GIFT evaluation workflows** for TTM-R3\n", "- Demonstrates **pre-trained and fine-tuned workflows**\n", "- Ensure environment setup is completed before execution\n", "\n", "---" ] }, { "cell_type": "markdown", "id": "8474f7e6-9d37-4468-9d47-c7ea1865fc11", "metadata": {}, "source": [ "# Importing libraries" ] }, { "cell_type": "code", "execution_count": 55, "id": "288c242c-79da-4e3e-8a8c-5b20603b3251", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Loaded predictor from: /dccstor/tsfm-irl/vijaye12/opensource/granite-tsfm/notebooks/hfdemo/tinytimemixer/full_benchmarking/gift_leaderboard_ttm_r3_nc/src/ttm_gluonts_predictor.py\n" ] } ], "source": [ "\n", "\n", "# -------------------------\n", "# Cell 1: Imports\n", "# -------------------------\n", "import csv\n", "import json\n", "import os\n", "import sys\n", "from copy import deepcopy\n", "from pprint import pprint\n", "\n", "import pandas as pd\n", "import torch\n", "from dotenv import load_dotenv\n", "from gift_eval.data import Dataset\n", "from gluonts.ev.metrics import (\n", " MAE,\n", " MAPE,\n", " MASE,\n", " MSE,\n", " MSIS,\n", " ND,\n", " NRMSE,\n", " RMSE,\n", " SMAPE,\n", " MeanWeightedSumQuantileLoss,\n", ")\n", "from gluonts.model import evaluate_model\n", "from gluonts.time_feature import get_seasonality\n", "\n", "from notebooks.hfdemo.tinytimemixer.full_benchmarking.gift_leaderboard_ttm_r3_nc.src import (\n", " ttm_gluonts_predictor,\n", ")\n", "from notebooks.hfdemo.tinytimemixer.full_benchmarking.gift_leaderboard_ttm_r3_nc.src.ttm_gluonts_predictor import (\n", " TTM_MAX_FORECAST_HORIZON,\n", " TTMGluonTSPredictor,\n", ")\n", "from notebooks.hfdemo.tinytimemixer.full_benchmarking.gift_leaderboard_ttm_r3_nc.src.utils import (\n", " delete_if_empty_csv,\n", " get_args,\n", " set_seed,\n", ")\n", "\n", "print(\"Loaded predictor from:\", ttm_gluonts_predictor.__file__)" ] }, { "cell_type": "markdown", "id": "fdaf0e52-d8f0-46ac-9da0-a322288e73a8", "metadata": {}, "source": [ "# Checking gift-eval env" ] }, { "cell_type": "code", "execution_count": 56, "id": "71cd8377-b4af-4757-934d-9bb0e9b9daf0", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "GIFT_EVAL = /dccstor/tsfm23/datasets/GIFT\n" ] } ], "source": [ "import os\n", "from dotenv import load_dotenv\n", "\n", "load_dotenv()\n", "\n", "print(\"GIFT_EVAL =\", os.getenv(\"GIFT_EVAL\"))" ] }, { "cell_type": "markdown", "id": "829a02f6-302d-494d-a74d-2f141629060f", "metadata": {}, "source": [ "# Select the workflow (pretrain or finetune)" ] }, { "cell_type": "code", "execution_count": 57, "id": "8cf2aa54-43dd-4d6a-b6fd-3be850b88280", "metadata": {}, "outputs": [], "source": [ "# -------------------------\n", "# Cell 3: Config\n", "# -------------------------\n", "RUN_MODE = \"pretrain\" # \"pretrain\" or \"finetune\"\n", "SEED = 42\n", "OUT_ROOT = \"../results\"\n" ] }, { "cell_type": "markdown", "id": "051ef62f-84fa-4d29-948d-52177522df6b", "metadata": {}, "source": [ "# Input args" ] }, { "cell_type": "code", "execution_count": 58, "id": "5105b2a2-002f-47dd-9fa8-6b07b69e7a31", "metadata": {}, "outputs": [], "source": [ "\n", "def get_args_notebook_safe():\n", " \"\"\"\n", " Call existing get_args() safely inside notebook by temporarily\n", " replacing sys.argv so Jupyter args do not interfere.\n", " \"\"\"\n", " old_argv = deepcopy(sys.argv)\n", " try:\n", " sys.argv = [\"notebook\"]\n", " args = get_args()\n", " finally:\n", " sys.argv = old_argv\n", "\n", " return args\n", "\n", "# -------------------------\n", "# Cell 4: Build args from existing parser defaults + override only what you need\n", "# -------------------------\n", "args = get_args_notebook_safe()\n", "\n", "args.upper_bound_fewshot_samples = 1\n", "args.auto_sampler = 1\n", "args.rolling_norm = 1\n", "args.ft_zs_ensemble = 1\n", "args.ft_zs_ensemble_mode = \"backtest_mean\"\n", "args.ensemble_shrink_lambda = 0.7\n", "\n" ] }, { "cell_type": "markdown", "id": "eb09e708-d39d-4325-96c0-6e8604c96819", "metadata": {}, "source": [ "# Configuration" ] }, { "cell_type": "code", "execution_count": 60, "id": "6679c225-0550-4ea8-80e4-aea9b7fc9694", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "RUN_MODE: pretrain\n", "ttm_version: TTM-R3-Pretrained-Test-5\n", "num_epochs: 0\n", "OUT_DIR: ../results/TTM-R3-Pretrained-Test-5\n" ] } ], "source": [ "if RUN_MODE == \"pretrain\":\n", " args.ttm_version = \"TTM-R3-Pretrained-Test-5\"\n", " args.num_epochs = 0\n", "\n", "elif RUN_MODE == \"finetune\":\n", " args.ttm_version = \"TTM-R3-Finetuned-Test-5\"\n", " args.num_epochs = 5\n", " args.decoder_tune = 0\n", " args.bias_tune = 1\n", " args.automate_fewshot_fraction = 1\n", " args.head_tune = 0\n", " args.prefix_tune = 0\n", " args.plot_predictions = 0\n", " args.quantile_tune = 0\n", " args.few_shot_data_limit_config = \"resources/fewshot_data_limit.json\"\n", "\n", "else:\n", " raise ValueError(f\"Unsupported RUN_MODE: {RUN_MODE}\")\n", "\n", "\n", "OUT_DIR = os.path.join(OUT_ROOT, args.ttm_version)\n", "os.makedirs(OUT_DIR, exist_ok=True)\n", "\n", "print(\"RUN_MODE:\", RUN_MODE)\n", "print(\"ttm_version:\", args.ttm_version)\n", "print(\"num_epochs:\", args.num_epochs)\n", "print(\"OUT_DIR:\", OUT_DIR)" ] }, { "cell_type": "markdown", "id": "fea7b6d2-3271-459c-a9cf-7ce71aef4fc1", "metadata": {}, "source": [ "# Choose Datasets" ] }, { "cell_type": "code", "execution_count": 61, "id": "d2c8d281-db3a-4a99-b1f1-c215cc22c87b", "metadata": {}, "outputs": [], "source": [ "short_datasets = \"solar/W\"\n", "med_long_datasets = \"\"" ] }, { "cell_type": "code", "execution_count": 62, "id": "87eafde4-932e-44b5-bd4f-d806fc388106", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Datasets: ['solar/W']\n" ] } ], "source": [ "# -------------------------\n", "# Cell 5: Environment, FS mode, datasets\n", "# -------------------------\n", "set_seed(SEED)\n", "load_dotenv()\n", "\n", "FS_MODE = None\n", "if getattr(args, \"few_shot_data_limit_config\", None):\n", " with open(args.few_shot_data_limit_config, \"r\") as f:\n", " FS_MODE = json.load(f)\n", "\n", "with open(\"resources/dataset_properties.json\", \"r\") as f:\n", " dataset_properties_map = json.load(f)\n", "\n", "pretty_names = {\n", " \"saugeenday\": \"saugeen\",\n", " \"temperature_rain_with_missing\": \"temperature_rain\",\n", " \"kdd_cup_2018_with_missing\": \"kdd_cup_2018\",\n", " \"car_parts_with_missing\": \"car_parts\",\n", "}\n", "\n", "\n", "all_datasets = sorted(set(short_datasets.split() + med_long_datasets.split()))\n", "\n", "print(\"Datasets:\", all_datasets)" ] }, { "cell_type": "markdown", "id": "0dd1d003-ef0a-445d-aeff-597a87009fd1", "metadata": {}, "source": [ "# Metrics" ] }, { "cell_type": "code", "execution_count": 63, "id": "3db67f73-5c0f-46b6-8de0-64f0e714c9e8", "metadata": {}, "outputs": [], "source": [ "# -------------------------\n", "# Cell 6: Metrics\n", "# -------------------------\n", "metrics = [\n", " MSE(forecast_type=\"mean\"),\n", " MSE(forecast_type=0.5),\n", " MAE(forecast_type=\"mean\"),\n", " MAE(forecast_type=0.5),\n", " MASE(),\n", " MAPE(),\n", " SMAPE(),\n", " MSIS(),\n", " RMSE(),\n", " NRMSE(),\n", " ND(),\n", " MeanWeightedSumQuantileLoss(\n", " quantile_levels=[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]\n", " ),\n", "]" ] }, { "cell_type": "code", "execution_count": 64, "id": "260a5bd2-06ac-4257-99d2-ab0640f53b9f", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "File does not exist: ../results/TTM-R3-Pretrained-Test-5/all_results.csv\n", "Already done datasets: []\n", "Writing to: ../results/TTM-R3-Pretrained-Test-5/all_results.csv\n" ] } ], "source": [ "# -------------------------\n", "# Cell 7: Results CSV setup\n", "# -------------------------\n", "csv_file_path = os.path.join(OUT_DIR, \"all_results.csv\")\n", "delete_if_empty_csv(csv_file_path)\n", "\n", "DONE_DATASETS = []\n", "if os.path.exists(csv_file_path):\n", " try:\n", " df_res = pd.read_csv(csv_file_path)\n", " if \"dataset\" in df_res.columns:\n", " DONE_DATASETS = df_res[\"dataset\"].astype(str).tolist()\n", " except Exception as e:\n", " print(\"Warning: could not read existing CSV:\", e)\n", "\n", "print(\"Already done datasets:\", DONE_DATASETS)\n", "\n", "file_exists = os.path.exists(csv_file_path)\n", "csv_file = open(csv_file_path, \"a\", newline=\"\")\n", "csv_writer = csv.writer(csv_file)\n", "\n", "if not file_exists:\n", " csv_writer.writerow(\n", " [\n", " \"dataset\",\n", " \"model\",\n", " \"eval_metrics/MSE[mean]\",\n", " \"eval_metrics/MSE[0.5]\",\n", " \"eval_metrics/MAE[mean]\",\n", " \"eval_metrics/MAE[0.5]\",\n", " \"eval_metrics/MASE[0.5]\",\n", " \"eval_metrics/MAPE[0.5]\",\n", " \"eval_metrics/sMAPE[0.5]\",\n", " \"eval_metrics/MSIS\",\n", " \"eval_metrics/RMSE[mean]\",\n", " \"eval_metrics/NRMSE[mean]\",\n", " \"eval_metrics/ND[0.5]\",\n", " \"eval_metrics/mean_weighted_sum_quantile_loss\",\n", " \"domain\",\n", " \"num_variates\",\n", " \"horizon\",\n", " \"ttm_context_len\",\n", " \"available_context_len\",\n", " \"finetune_success\",\n", " \"finetune_train_num_samples\",\n", " \"finetune_valid_num_samples\",\n", " \"insample_error\",\n", " ]\n", " )\n", " csv_file.flush()\n", "\n", "print(\"Writing to:\", csv_file_path)" ] }, { "cell_type": "markdown", "id": "f81e375e-d0dd-47b6-9e7a-011a8c26654c", "metadata": {}, "source": [ "# Main Run Loop" ] }, { "cell_type": "code", "execution_count": 65, "id": "6e6d2cb1-4153-4906-8521-ad209ad52aa9", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "INFO:p-1956010:t-23001482699456:get_model.py:get_model:Loading model from: ibm-research/ttm-r3\n", "INFO:p-1956010:t-23001482699456:get_model.py:get_model:Loading model from: ibm-granite/granite-timeseries-ttm-r2\n", "WARNING:p-1956010:t-23001482699456:get_model.py:get_model:Requested `context_length` is shorter than the shortest TTM available, hence, zero-padding must be done outside.\n", "WARNING:p-1956010:t-23001482699456:get_model.py:get_model:Requested `prediction_length` (8) is not exactly equal to any of the available TTM prediction lengths. Hence, TTM will forecast using the `prediction_filter_length` argument to provide the requested prediction length. Check the model card to know more about the supported context lengths and forecast/prediction lengths.\n", "INFO:p-1956010:t-23001482699456:get_model.py:get_model:Loading model from: ibm-research/ttm-r3\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "================================================================================\n", "Processing dataset: solar/W\n", "================================================================================\n", "\n", "--- dataset=solar/W, term=short ---\n", "Freq=W-FRI, Horizon=8\n", "Min context length: 44\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "INFO:p-1956010:t-23001482699456:get_model.py:get_model:Model loaded successfully from ibm-research/ttm-r3, revision = 156-16-dec-52-r3.\n", "INFO:p-1956010:t-23001482699456:get_model.py:get_model:[TTM] context_length = 156, prediction_length = 16\n", "INFO:p-1956010:t-23001482699456:ttm_gluonts_predictor.py:_get_gift_model:The TTM has Prefix Tuning = False\n", "INFO:p-1956010:t-23001482699456:ttm_gluonts_predictor.py:train:Number of series: Train = 137, Valid = 137\n", "137it [00:00, 27173.92it/s]\n", "137it [00:00, 219069.63it/s]\n", "137it [00:00, 214410.32it/s]\n", "INFO:p-1956010:t-23001482699456:ttm_gluonts_predictor.py:train:Global scaling done successfully.\n", "INFO:p-1956010:t-23001482699456:ttm_gluonts_predictor.py:train:Increasing fewshot fraction to 0.9 due to small dataset size.\n", "INFO:p-1956010:t-23001482699456:ttm_gluonts_predictor.py:train:Length of orginal train set = 137\n", "INFO:p-1956010:t-23001482699456:ttm_gluonts_predictor.py:train:Length of 90.0 % train set = 123\n", "INFO:p-1956010:t-23001482699456:ttm_gluonts_predictor.py:train:Number of train samples = 260, valid samples = 14\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "train_num_samples 260\n", "Training failed. Falling back to evaluation: Forcing zeroshot since number of finetune samples is very low or force_zerohot is set to true\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "137it [00:00, 27986.54it/s]\n", "137it [00:00, 215536.25it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "bs = 64\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "3it [00:00, 49.40it/s]\n", "100%|█████████████████████████████████████████████████████████████████████████████████████████| 137/137 [00:00<00:00, 145105.97it/s]\n", "INFO:p-1956010:t-23001482699456:ttm_gluonts_predictor.py:predict:[Predict] Dataset=solar/W, Num series=137, NM=1\n", "137it [00:00, 1075.54it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Saved result for solar/W/short\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "# -------------------------\n", "# Cell 8: Main run loop \n", "# -------------------------\n", "for ds_name in all_datasets:\n", " set_seed(SEED)\n", "\n", " print(\"\\n\" + \"=\" * 80)\n", " print(f\"Processing dataset: {ds_name}\")\n", " print(\"=\" * 80)\n", "\n", " terms = [\"short\", \"medium\", \"long\"]\n", "\n", " for term in terms:\n", " if term in [\"medium\", \"long\"] and ds_name not in med_long_datasets.split():\n", " continue\n", "\n", " print(f\"\\n--- dataset={ds_name}, term={term} ---\")\n", "\n", " # -------------------------\n", " # dataset parsing\n", " # -------------------------\n", " if \"/\" in ds_name:\n", " ds_key = ds_name.split(\"/\")[0].lower()\n", " ds_freq = ds_name.split(\"/\")[1]\n", " ds_key = pretty_names.get(ds_key, ds_key)\n", " else:\n", " ds_key = pretty_names.get(ds_name.lower(), ds_name.lower())\n", " ds_freq = dataset_properties_map[ds_key][\"frequency\"]\n", "\n", " ds_config = f\"{ds_key}/{ds_freq}/{term}\"\n", "\n", " if ds_config in DONE_DATASETS:\n", " print(f\"Skipping {ds_config} (already done)\")\n", " continue\n", "\n", " dataset = Dataset(name=ds_name, term=term, to_univariate=False)\n", " season_length = get_seasonality(dataset.freq)\n", "\n", " print(f\"Freq={dataset.freq}, Horizon={dataset.prediction_length}\")\n", "\n", " # -------------------------\n", " # context and channel discovery\n", " # -------------------------\n", " all_lengths = []\n", " num_channels = None\n", " last_item = None\n", "\n", " for x in dataset.test_data:\n", " last_item = x\n", " target = x[0][\"target\"]\n", "\n", " if len(target.shape) == 1:\n", " all_lengths.append(len(target))\n", " num_channels = 1\n", " else:\n", " all_lengths.append(target.shape[1])\n", " num_channels = target.shape[0]\n", "\n", " min_context_length = min(all_lengths)\n", " print(\"Min context length:\", min_context_length)\n", "\n", " num_prediction_channels = num_channels\n", " prediction_channel_indices = list(range(num_channels))\n", "\n", " # -------------------------\n", " # exogenous variables\n", " # -------------------------\n", " past_feat_dynamic_real_exist = False\n", " if args.use_exogs and last_item is not None and \"past_feat_dynamic_real\" in last_item[0]:\n", " num_exogs = last_item[0][\"past_feat_dynamic_real\"].shape[0]\n", " print(f\"Using {num_exogs} past dynamic real exogenous channels\")\n", " num_channels += num_exogs\n", " past_feat_dynamic_real_exist = True\n", "\n", " if dataset.prediction_length > TTM_MAX_FORECAST_HORIZON:\n", " prediction_channel_indices = list(range(num_channels))\n", "\n", " # -------------------------\n", " # predictor\n", " # -------------------------\n", " predictor = TTMGluonTSPredictor(\n", " context_length=min_context_length,\n", " prediction_length=min(dataset.prediction_length, TTM_MAX_FORECAST_HORIZON),\n", " test_data_label=dataset.test_data.label,\n", " random_seed=SEED,\n", " term=term,\n", " ds_name=ds_name,\n", " out_dir=OUT_DIR,\n", " scale=True,\n", " upper_bound_fewshot_samples=args.upper_bound_fewshot_samples,\n", " force_short_context=(args.force_short_context if term == \"short\" else False),\n", " min_context_mult=args.min_context_mult,\n", " past_feat_dynamic_real_exist=past_feat_dynamic_real_exist,\n", " num_prediction_channels=num_prediction_channels,\n", " freq=dataset.freq,\n", " use_valid_from_train=args.use_valid_from_train,\n", " insample_forecast=args.insample_forecast,\n", " insample_use_train=args.insample_use_train,\n", " head_dropout=args.head_dropout,\n", " decoder_mode=args.decoder_mode,\n", " num_input_channels=num_channels,\n", " huber_delta=args.huber_delta,\n", " quantile=args.quantile,\n", " loss=args.loss,\n", " prediction_channel_indices=prediction_channel_indices,\n", " use_mask=True,\n", " force_zeroshot=(args.num_epochs == 0),\n", " auto_sampler=args.auto_sampler,\n", " fs_mode_dict=FS_MODE,\n", " use_lite=args.use_lite,\n", " plot_predictions=args.plot_predictions,\n", " rolling_norm=args.rolling_norm,\n", " ft_zs_ensemble=args.ft_zs_ensemble,\n", " ft_zs_ensemble_mode=args.ft_zs_ensemble_mode,\n", " ensemble_shrink_lambda=args.ensemble_shrink_lambda,\n", " use_zs_anchor=args.use_zs_anchor,\n", " zs_anchor_weight=args.zs_anchor_weight,\n", " disable_extra_point_weightage=args.disable_extra_point_weightage,\n", " )\n", "\n", " # -------------------------\n", " # training\n", " # -------------------------\n", " batch_size = args.batch_size\n", " optimize_batch_size = batch_size is None\n", "\n", " finetune_success = False\n", " finetune_train_num_samples = 0\n", " finetune_valid_num_samples = 0\n", "\n", " try:\n", " if dataset.prediction_length > TTM_MAX_FORECAST_HORIZON:\n", " predictor.prediction_length = dataset.prediction_length\n", "\n", " predictor.train(\n", " train_dataset=dataset.training_dataset,\n", " valid_dataset=dataset.validation_dataset,\n", " batch_size=batch_size,\n", " optimize_batch_size=optimize_batch_size,\n", " learning_rate=args.learning_rate,\n", " num_epochs=args.num_epochs,\n", " fewshot_fraction=args.fewshot_fraction,\n", " fewshot_location=args.fewshot_location,\n", " automate_fewshot_fraction=args.automate_fewshot_fraction,\n", " automate_fewshot_fraction_threshold=args.automate_fewshot_fraction_threshold,\n", " fewshot_max_samples=args.fewshot_max_samples,\n", " valid_max_samples=args.valid_max_samples,\n", " bias_tune=args.bias_tune,\n", " norm_tune=args.norm_tune,\n", " patch_tune=args.patch_tune,\n", " backbone_tune=args.backbone_tune,\n", " prefix_tune=args.prefix_tune,\n", " decoder_tune=args.decoder_tune,\n", " head_tune=args.head_tune,\n", " quantile_tune=args.quantile_tune,\n", " enable_staging=args.enable_staging,\n", " )\n", " finetune_success = True\n", " finetune_train_num_samples = getattr(predictor, \"train_num_samples\", 0)\n", " finetune_valid_num_samples = getattr(predictor, \"valid_num_samples\", 0)\n", "\n", " except Exception as e:\n", " print(\"Training failed. Falling back to evaluation:\", e)\n", " finetune_success = False\n", "\n", " # -------------------------\n", " # evaluation\n", " # -------------------------\n", " res = evaluate_model(\n", " predictor,\n", " test_data=dataset.test_data,\n", " metrics=metrics,\n", " batch_size=batch_size,\n", " axis=None,\n", " mask_invalid_label=True,\n", " allow_nan_forecast=False,\n", " seasonality=season_length,\n", " )\n", "\n", " insample_error = 0\n", "\n", " result = [\n", " ds_config,\n", " args.ttm_version,\n", " res[\"MSE[mean]\"].iloc[0],\n", " res[\"MSE[0.5]\"].iloc[0],\n", " res[\"MAE[mean]\"].iloc[0],\n", " res[\"MAE[0.5]\"].iloc[0],\n", " res[\"MASE[0.5]\"].iloc[0],\n", " res[\"MAPE[0.5]\"].iloc[0],\n", " res[\"sMAPE[0.5]\"].iloc[0],\n", " res[\"MSIS\"].iloc[0],\n", " res[\"RMSE[mean]\"].iloc[0],\n", " res[\"NRMSE[mean]\"].iloc[0],\n", " res[\"ND[0.5]\"].iloc[0],\n", " res[\"mean_weighted_sum_quantile_loss\"].iloc[0],\n", " dataset_properties_map[ds_key][\"domain\"],\n", " dataset_properties_map[ds_key][\"num_variates\"],\n", " dataset.prediction_length,\n", " predictor.ttm.config.context_length,\n", " min_context_length,\n", " finetune_success,\n", " finetune_train_num_samples,\n", " finetune_valid_num_samples,\n", " insample_error,\n", " ]\n", "\n", " csv_writer.writerow(result)\n", " csv_file.flush()\n", "\n", " DONE_DATASETS.append(ds_config)\n", " print(f\"Saved result for {ds_config}\")" ] }, { "cell_type": "code", "execution_count": 66, "id": "36e4e296-caa9-422d-b956-a5a62f1854a1", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CSV closed: ../results/TTM-R3-Pretrained-Test-5/all_results.csv\n" ] } ], "source": [ "# Cell 9: Close CSV\n", "# -------------------------\n", "csv_file.close()\n", "print(\"CSV closed:\", csv_file_path)" ] }, { "cell_type": "markdown", "id": "2a0e49b5-c291-4caa-b281-f6de2622adcf", "metadata": {}, "source": [ "# Results" ] }, { "cell_type": "code", "execution_count": 67, "id": "42938e18-d003-4ea1-84d7-6daa2a019a06", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[{'dataset': 'solar/W/short',\n", " 'eval_metrics/MASE[0.5]': 1.344993018008766,\n", " 'eval_metrics/NRMSE[mean]': 0.3103785363996592,\n", " 'eval_metrics/mean_weighted_sum_quantile_loss': 0.1865114347329705,\n", " 'model': 'TTM-R3-Pretrained-Test-5'}]\n" ] } ], "source": [ "# -------------------------\n", "# Cell 10: Read results\n", "# -------------------------\n", "df = pd.read_csv(csv_file_path)\n", "df = df.sort_values(by=\"dataset\")\n", "\n", "display_cols = [\n", " \"dataset\",\n", " \"model\",\n", " \"eval_metrics/MASE[0.5]\",\n", " \"eval_metrics/NRMSE[mean]\",\n", " \"eval_metrics/mean_weighted_sum_quantile_loss\",\n", "]\n", "\n", "pprint(df[display_cols].to_dict(orient=\"records\"))\n" ] }, { "cell_type": "code", "execution_count": 68, "id": "810867ad-249f-45fc-b615-8ef9d8128c9d", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
datasetmodeleval_metrics/MASE[0.5]eval_metrics/NRMSE[mean]eval_metrics/mean_weighted_sum_quantile_loss
0solar/W/shortTTM-R3-Pretrained-Test-51.3449930.3103790.186511
\n", "
" ], "text/plain": [ " dataset model eval_metrics/MASE[0.5] \\\n", "0 solar/W/short TTM-R3-Pretrained-Test-5 1.344993 \n", "\n", " eval_metrics/NRMSE[mean] eval_metrics/mean_weighted_sum_quantile_loss \n", "0 0.310379 0.186511 " ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "display(df[display_cols])" ] }, { "cell_type": "code", "execution_count": null, "id": "864c37da-1e09-4c52-a50f-8de1e33463ac", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.15" } }, "nbformat": 4, "nbformat_minor": 5 }