{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Collect metrics using a pre-trained model\n", "\n", "This notebook demonstrates how to use a pre-trained model to collect metrics on a dataset.\n", "\n", "![](../images/cifar10-collect-metrics.png)\n", "\n", "" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Project setup" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "tags": [ "parameters" ] }, "outputs": [], "source": [ "PROJECT_NAME = \"3LC Tutorials - CIFAR-10\"\n", "RUN_NAME = \"collect-metrics-cifar-10\"\n", "NUM_WORKERS = 4\n", "INSTALL_DEPENDENCIES = True" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Install dependencies" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "if INSTALL_DEPENDENCIES:\n", " %pip install -q 3lc\n", " %pip install -q timm\n", " %pip install -q git+https://github.com/3lc-ai/3lc-examples.git" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Imports" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import timm\n", "import tlc\n", "import torch\n", "import torch.nn as nn\n", "import torchvision\n", "\n", "from tlc_tools.common import infer_torch_device" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Prepare Tables\n", "\n", "We will reuse the tables from the notebook [create-table-from-torch.ipynb](../1-create-tables/create-table-from-torch-dataset.ipynb),\n", "and use a pre-trained model from Hugging Face Hub." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "device = infer_torch_device()\n", "\n", "# Use a resnet18 model from timm, already trained on CIFAR-10\n", "model = timm.create_model(\"hf_hub:FredMell/resnet18-cifar10\", pretrained=True).to(device)\n", "\n", "# Load the tables\n", "train_table = tlc.Table.from_names(\n", " table_name=\"initial\", dataset_name=\"CIFAR-10-train\", project_name=\"3LC Tutorials - CIFAR-10\"\n", ")\n", "val_table = tlc.Table.from_names(\n", " table_name=\"initial\", dataset_name=\"CIFAR-10-val\", project_name=\"3LC Tutorials - CIFAR-10\"\n", ")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "image_transform = torchvision.transforms.Compose(\n", " [\n", " torchvision.transforms.Resize(224),\n", " torchvision.transforms.ToTensor(),\n", " torchvision.transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),\n", " ]\n", ")\n", "\n", "\n", "def transform(sample):\n", " image = sample[\"Image\"]\n", " label = sample[\"Label\"]\n", " return (image_transform(image), label)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# Build non-mutating views that apply the model-compatible transforms on read\n", "\n", "train_view = train_table.with_transform(transform)\n", "val_view = val_table.with_transform(transform)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Setup Metrics Collector\n", "\n", "We use a `tlc.metrics.FunctionalMetricsCollector` to compute per-sample loss, predicted label, confidence, and accuracy from the model's raw logits." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "classes = list(train_table.get_simple_value_map(\"Label\").values())\n", "criterion = nn.CrossEntropyLoss(reduction=\"none\")\n", "\n", "\n", "def compute_metrics(batch, predictor_output):\n", " predictions = predictor_output.forward\n", " labels = batch[1].to(predictions.device)\n", "\n", " softmax = torch.softmax(predictions, dim=1)\n", " predicted = torch.argmax(predictions, dim=1)\n", " confidence = torch.gather(softmax, 1, predicted.unsqueeze(1)).squeeze(1)\n", "\n", " return {\n", " \"loss\": criterion(predictions, labels).cpu().numpy(),\n", " \"predicted\": predicted.cpu().numpy(),\n", " \"confidence\": confidence.cpu().numpy(),\n", " \"accuracy\": predicted.eq(labels).float().cpu().numpy(),\n", " }\n", "\n", "\n", "metrics_collector = tlc.metrics.FunctionalMetricsCollector(\n", " collection_fn=compute_metrics,\n", " schema={\n", " \"predicted\": tlc.schemas.CategoricalLabelSchema(\n", " classes=classes,\n", " display_name=\"predicted label\",\n", " ),\n", " },\n", ")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Collect metrics" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# Create a 3LC run and collect metrics\n", "run = tlc.init(\n", " project_name=PROJECT_NAME,\n", " run_name=RUN_NAME,\n", " description=\"Only collect metrics with trained model on CIFAR-10\",\n", " if_exists=\"overwrite\",\n", ")\n", "\n", "dataloader_args = {\n", " \"batch_size\": 128,\n", " \"num_workers\": NUM_WORKERS,\n", " \"pin_memory\": True,\n", "}\n", "\n", "tlc.collect_metrics(\n", " table=train_view,\n", " predictor=model,\n", " metrics_collectors=metrics_collector,\n", " dataloader_args=dataloader_args,\n", " split=\"train\",\n", ")\n", "\n", "tlc.collect_metrics(\n", " table=val_view,\n", " predictor=model,\n", " metrics_collectors=metrics_collector,\n", " dataloader_args=dataloader_args,\n", " split=\"val\",\n", ")\n", "\n", "run.set_status_completed()" ] } ], "metadata": { "kernelspec": { "display_name": ".venv", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.12.9" }, "test_marks": [ "slow" ] }, "nbformat": 4, "nbformat_minor": 2 }