{"nbformat":4,"nbformat_minor":0,"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":2},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython2","version":"2.7.6"},"colab":{"name":"tutorial_getting_started.ipynb","provenance":[],"collapsed_sections":[],"toc_visible":true},"accelerator":"GPU"},"cells":[{"cell_type":"markdown","metadata":{"id":"bukochgPFg7s"},"source":["# Getting started\n","\n","This notebook shows how to get started with Quantus, using a very simple example. For this purpose, we use a LeNet model and MNIST dataset.\n","\n","- Make sure to have GPUs enabled to speed up computation.\n","- Skip running the first cell if you do not use Google Colab."]},{"cell_type":"code","metadata":{"id":"A8-GsXIAcigw"},"source":["# Mount Google Drive.\n","from google.colab import drive\n","drive.mount('/content/drive', force_remount=True)\n","\n","# Install packages.\n","from IPython.display import clear_output\n","!pip install captum opencv-python\n","!pip install torch==1.8.0+cu111 torchvision==0.9.0+cu111 -f https://download.pytorch.org/whl/torch_stable.html\n","clear_output()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"MuA8v9jLp-pf"},"source":["# Imports general.\n","import sys\n","import gc\n","import warnings\n","import pathlib\n","import numpy as np\n","import pandas as pd\n","import matplotlib.pyplot as plt\n","import torch\n","import torchvision\n","from torchvision import transforms\n","import captum\n","from captum.attr import *\n","import random\n","import os\n","import cv2\n","\n","# Import package.\n","path = \"/content/drive/MyDrive/Projects\"\n","sys.path.append(f'{path}/quantus')\n","import quantus\n","\n","# Collect garbage.\n","gc.collect()\n","torch.cuda.empty_cache()\n","\n","# Notebook settings.\n","device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\") \n","warnings.filterwarnings(\"ignore\", category=UserWarning)\n","%load_ext autoreload\n","%autoreload 2\n","clear_output()"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"mGhP4bTuoWYF"},"source":["## 1. Preliminaries"]},{"cell_type":"markdown","metadata":{"id":"XqKzag4VFjHT"},"source":["### 1.1 Load datasets\n","\n","We will then load a batch of input, output pairs that we generate explanations for, then to evaluate."]},{"cell_type":"code","metadata":{"id":"TmsZxFhuc0mm"},"source":["# Load datasets and make loaders.\n","test_samples = 24\n","transformer = transforms.Compose([transforms.ToTensor()])\n","train_set = torchvision.datasets.MNIST(root='./sample_data', train=True, transform=transformer, download=True)\n","test_set = torchvision.datasets.MNIST(root='./sample_data', train=False, transform=transformer, download=True)\n","train_loader = torch.utils.data.DataLoader(train_set, batch_size=128, shuffle=True, pin_memory=True) # num_workers=4,\n","test_loader = torch.utils.data.DataLoader(test_set, batch_size=200, pin_memory=True)\n","\n","# Load a batch of inputs and outputs to use for evaluation.\n","x_batch, y_batch = iter(test_loader).next()\n","x_batch, y_batch = x_batch.to(device), y_batch.to(device)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":191},"id":"aAR67-cnOS67","executionInfo":{"status":"ok","timestamp":1640102275943,"user_tz":-60,"elapsed":501,"user":{"displayName":"Anna Hedström","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhbfsluHeZ1mzN6Bsf-1zU62lYHcz183jYjeS63=s64","userId":"05540180366077551505"}},"outputId":"694e08b4-89fd-4be9-d19a-161e9d41dcd5"},"source":["# Plot some inputs!\n","nr_images = 5\n","fig, axes = plt.subplots(nrows=1, ncols=nr_images, figsize=(nr_images*3, int(nr_images*2/3)))\n","for i in range(nr_images):\n","    axes[i].imshow((np.reshape(x_batch[i].cpu().numpy(), (28, 28)) * 255).astype(np.uint8), vmin=0.0, vmax=1.0, cmap=\"gray\")\n","    axes[i].title.set_text(f\"MNIST class - {y_batch[i].item()}\")\n","    axes[i].axis(\"off\")\n","plt.show()"],"execution_count":null,"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAA1MAAACuCAYAAADTXFfGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAMqElEQVR4nO3dXaxsZ1kH8P9TEEHbomKCSArE1kQtFBNRjCSgtoRATTQQ/ADtIdpESlQSE0LSaK1WueDCGIMJogalVaBFTUyU1E9IudAIKjWN5aIUQdsTmxZ6WvqF7ePFmhN3m9nn7P3O7Pnav19yLmZmnXXetfez1qz/vPM+p7o7AAAAHM456x4AAADANhKmAAAABghTAAAAA4QpAACAAcIUAADAAGEKAABggDC1RFXVVXXRuscBB6Vm2TZqlm2jZtk2avZwNipMVdXnquqxqvrGpzz/r7Nf7Itmj/9w9vh79mxzUVX1nscfq6or9zy+uqrurKoHq+q/qurDs+dvmz33YFU9XlWP7Hl89VEf81GpqjfvOY4Hq+qh2c/su9Y9tl2iZpenqr63qv6mqu6rqnuq6qaqet66x7Vr1OzyVNUzquojs59pV9X3r3tMu0jNLldVXVpVt8/uC/6hql647jHtGjV7NKrqmtnP67J1j2WvjQpTM3cm+YnTD6rqJUm+Zs529yX59YPssKpOJPmpJJd197lJXpbk75Kkuy/u7nNnz9+S5OdOP+7udy12KOvT3X+85zjOTfK2JJ9N8i9rHtouUrPL8fVJ3pfkRUlemOSBJO9f54B2mJpdnk8k+ckkJ9c9kB2nZpdgdnP/Z0l+Ock3JPlkkg+vdVC7S80uUVVdmOSNSe5e91ieahPD1PVJrtjz+ESSD8zZ7o+SXFJVrzrAPr87yc3dfUeSdPfJ7n7fyOCq6mmzTwXuqKoHqupTVXXBnO0un30CcaqqvlBV1+557ZlVdUNV3VtVX6qqf66q585ee0tVfXa27zur6s0j45zjRJIPdHefdUsOS80uoWa7+6PdfVN3n+ruh5K8J8krRvbFWanZ5dTsY939W939iSSPj+yDA1Ozy7k3eH2S22bX2keSXJvkpVX1bYP7Y39qdrn3s7+T5J1JHltwP0u3iWHqH5OcX1XfXlVPS/LjSW6Ys91DSd6V5DcOuM8rquodVfWy2X5H/WKmTxpel+T8JD89G8tTfTnTSfR1SS5PclVV/cjstRNJnp3kgiTPSfLWJA9X1dcm+e0kr+3u85J8X5J/W2CsSZKapvBfmfknMYtTs0uu2ZlXJrltSfviydTs0dQsR0fNLqdmL07y6dMPuvvLSe6YPc9yqdklXWer6o1JHu3uvxrdx1HaxDCV/H+af3WS/0jy3/ts97tJXlBVrz3Tzrr7hiQ/n+Q1ST6e5H+q6p2DY7syyS9192d68unuvnfOv/mx7v737n6iu29N8sEkpz91+Eqmoruoux/v7k9196nZa08keXFVPau77+7uZdxMXpHklu6+cwn7Yj41u8SarapLklyT5B2L7ot9qdnlXmc5emp28Zo9N8n9T3nu/iTnDe6PM1OzC9ZsVZ2XKWy+feTvr8Imh6k3JXlLzjCb0t2PJrlu9ueMelpDdFmmZP3WJNdV1WsGxnZBpk9xzqiqXl7Tws57qur+2b95eiHi9UluTvKhqrqrqt5dVV81+4Tox2bb3l1Vf7nf1Hs9ubnEC84ynCsyTSNzdNTskmq2pg5CH03y9u6+5VBHymGo2eVeZzl6anbxmn0w0yzEXudnWqPK8qnZxWv22iTXd/fnDn2EK7KRYaq7/zPTwr3XZVooeSbvz1RQrz/gvr/S3TcluTXJiweG94UkFx5guz9J8hdJLujuZyd5b5LaM4Zf7e7vyDT1+UOZfa+2u2/u7lcneV6S25P83j7Hce6eP5/fbxBV9Yok35zkIwc9QA5PzS6nZmdfSf3bJNd19/WHOUgOR80u7zrLaqjZpdTsbUleevrB7OtYF8ZXqo+Eml1KzV6a5Beq6mRVncwUAm9cYEZu6TYyTM38TJIfnKXbfXX3/yb5lUyL0uaqaRHc5VV1XlWdM5tGvTjJPw2M6/czfQrwrTW5pKqeM2e785Lc192P1NTy8k17xvMDVfWS2XddT2WaJn2iqp5bVT88u7g9mukTpCcGxrjXiSR/2t0+dTp6anaBmq2q5yf5+yTv6e73juyDQ1OzC15nq+qrq+qZs4fPqGlBdo3uj7NSs4vV7J9n+urVG2Z1e02SW7v79sH9cXZqdrGavTRTWPzO2Z+7kvxspoYUG2Fjw1R339Hdnzzg5h/MmVslnkpydZLPJ/lSkncnuaqnDkyH9ZtJbkzy17P9/kGSZ83Z7m1Jfq2qHsh0sbpxz2vflGmm6FSm79B+PNNU6TmZFgTelalV5quSXDUwxiRTl5UkPxpf8VsJNbtwzV6Z5FuSXLt32n9wXxyAml38OpvkM0keTvL8TF93eThTa3+OgJpdrGa7+54kb8jU7OCLSV6eqTECR0TNLlyz9/bUtfBkd5/M1Dn1i929MfcH1TplAwAAHNrGzkwBAABsMmEKAABggDAFAAAwQJgCAAAY8PQzvVhVulOwkO5eaYtgNcuiVl2zibplca61bBs1y7bZr2bNTAEAAAwQpgAAAAYIUwAAAAOEKQAAgAHCFAAAwABhCgAAYIAwBQAAMECYAgAAGCBMAQAADBCmAAAABghTAAAAA4QpAACAAcIUAADAAGEKAABggDAFAAAw4OnrHgCwv+5e9xCepKrWPQQAgI1hZgoAAGCAMAUAADBAmAIAABggTAEAAAzQgAI2xKY1m5hn3hg1pWCTLHIeqWXOZlXXabUI28PMFAAAwABhCgAAYIAwBQAAMECYAgAAGKABBazYNjSaAAA210HvJTatmclh7oE2bez7MTMFAAAwQJgCAAAYIEwBAAAMEKYAAAAGaEABR2idzSbmLdzU/AIm27KwmfVxvQQOwswUAADAAGEKAABggDAFAAAwQJgCAAAYsLENKDZt4T5sCvUJEw0CWJZNq6WDjsf7wfGwafW5KvOOexNr3swUAADAAGEKAABggDAFAAAwQJgCAAAYsLENKNZp1xb6beJiveNi3s/+MPXldwfAfvZ7P/HeAatjZgoAAGCAMAUAADBAmAIAABggTAEAAAwQpgAAAAbo5gcrtqouS7vWlRJgGVwb2VRqczuZmQIAABggTAEAAAwQpgAAAAYIUwAAAAM2tgGFRfpwcKuq41Wdl7DXovWtbo+n4/z+Pu/YnQe7b9N+x8flHDQzBQAAMECYAgAAGCBMAQAADBCmAAAABmxsA4pVsVgP5tu0cwNgP9vy3nnQ6+pRHM9B9+navxoa6+wOM1MAAAADhCkAAIABwhQAAMAAYQoAAGDAsW9AsU5HscDUgsTdty0LrQGOg6N43523T9d+Ntlxvqc1MwUAADBAmAIAABggTAEAAAwQpgAAAAZoQAEbbFULjrdlkSe7zyJ7NplrJSMWva4dh7rb5mM0MwUAADBAmAIAABggTAEAAAwQpgAAAAZoQLEix/l/hmZzqBl2nRpnhLoBRpmZAgAAGCBMAQAADBCmAAAABghTAAAAA4QpAACAAbr5wYY4io6PAMB2m3d/sM4OlO5XnszMFAAAwABhCgAAYIAwBQAAMECYAgAAGKABxRFY9sK8dS4yZPlWtXBT3bDpFj0X1Di7wGL+3TfvWrXo713dbA4zUwAAAAOEKQAAgAHCFAAAwABhCgAAYIAGFBvGgmoAYBXcc8DizEwBAAAMEKYAAAAGCFMAAAADhCkAAIABGlAswP8+zaawiBhg863zvsH7xGY5zO9jFXWzqvHsYh2amQIAABggTAEAAAwQpgAAAAYIUwAAAAM0oDigo1j8t4uL8FiPVS1qnlez29yIxTkIJNt9HZvHtW237NLvc79zbZuP0cwUAADAAGEKAABggDAFAAAwQJgCAAAYIEwBAAAM0M1vjl3r6gPL4txglNoBYJu79u3HzBQAAMAAYQoAAGCAMAUAADBAmAIAABigAcWK7OKCOwAAOM7MTAEAAAwQpgAAAAYIUwAAAAOEKQAAgAHHvgFFd697CBwz+zUjOa61qDnL7ll2LasRzmZejWzzNVXNw/YwMwUAADBAmAIAABggTAEAAAwQpgAAAAYc+wYUR8HCUUaoG4DlOcw1dZubVQDrZWYKAABggDAFAAAwQJgCAAAYIEwBAAAMOFYNKCwwBTh6mqmwbdQsLN9xOa/MTAEAAAwQpgAAAAYIUwAAAAOEKQAAgAHHqgEFAAAw7rg0ljgoM1MAAAADhCkAAIABwhQAAMAAYQoAAGCAMAUAADBAN78F6GYCAADHl5kpAACAAcIUAADAAGEKAABggDAFAAAw4Fg1oNAwAgAAWBYzUwAAAAOEKQAAgAHCFAAAwABhCgAAYEB197rHAAAAsHXMTAEAAAwQpgAAAAYIUwAAAAOEKQAAgAHCFAAAwABhCgAAYMD/Ac/SlKuddICZAAAAAElFTkSuQmCC\n","text/plain":["<Figure size 1080x216 with 5 Axes>"]},"metadata":{"needs_background":"light"}}]},{"cell_type":"markdown","metadata":{"id":"vmccxpA0n6MY"},"source":["### 1.2 Train a LeNet model\n","\n","(or any other model of choice). \n","Network architecture from: https://github.com/ChawDoe/LeNet5-MNIST-PyTorch."]},{"cell_type":"code","metadata":{"id":"CUghaOhXddLU","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1640102275944,"user_tz":-60,"elapsed":8,"user":{"displayName":"Anna Hedström","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhbfsluHeZ1mzN6Bsf-1zU62lYHcz183jYjeS63=s64","userId":"05540180366077551505"}},"outputId":"cb625cea-1bc4-477e-a549-4bd1e1a8e158"},"source":["class LeNet(torch.nn.Module):\n","    \"\"\"Network architecture from: https://github.com/ChawDoe/LeNet5-MNIST-PyTorch.\"\"\"\n","    def __init__(self):\n","        super().__init__()\n","        self.conv_1 = torch.nn.Conv2d(1, 6, 5)\n","        self.pool_1 = torch.nn.MaxPool2d(2, 2)\n","        self.relu_1 = torch.nn.ReLU()\n","        self.conv_2 = torch.nn.Conv2d(6, 16, 5)\n","        self.pool_2 = torch.nn.MaxPool2d(2, 2)\n","        self.relu_2 = torch.nn.ReLU()\n","        self.fc_1 = torch.nn.Linear(256, 120)\n","        self.relu_3 = torch.nn.ReLU()\n","        self.fc_2 = torch.nn.Linear(120, 84)\n","        self.relu_4 = torch.nn.ReLU()\n","        self.fc_3 = torch.nn.Linear(84, 10)\n","\n","    def forward(self, x):\n","        x = self.pool_1(self.relu_1(self.conv_1(x)))\n","        x = self.pool_2(self.relu_2(self.conv_2(x)))\n","        x = x.view(x.shape[0], -1)\n","        x = self.relu_3(self.fc_1(x))\n","        x = self.relu_4(self.fc_2(x))\n","        x = self.fc_3(x)\n","        return x\n","\n","# Load model architecture.\n","model = LeNet()\n","print(f\"\\n Model architecture: {model.eval()}\\n\")"],"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["\n"," Model architecture: LeNet(\n","  (conv_1): Conv2d(1, 6, kernel_size=(5, 5), stride=(1, 1))\n","  (pool_1): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n","  (relu_1): ReLU()\n","  (conv_2): Conv2d(6, 16, kernel_size=(5, 5), stride=(1, 1))\n","  (pool_2): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n","  (relu_2): ReLU()\n","  (fc_1): Linear(in_features=256, out_features=120, bias=True)\n","  (relu_3): ReLU()\n","  (fc_2): Linear(in_features=120, out_features=84, bias=True)\n","  (relu_4): ReLU()\n","  (fc_3): Linear(in_features=84, out_features=10, bias=True)\n",")\n","\n"]}]},{"cell_type":"code","metadata":{"id":"olAfyOHzevne"},"source":["def train_model(model, \n","                train_data: torchvision.datasets,\n","                test_data: torchvision.datasets, \n","                device: torch.device, \n","                epochs: int = 20,\n","                criterion: torch.nn = torch.nn.CrossEntropyLoss(), \n","                optimizer: torch.optim = torch.optim.SGD(model.parameters(), lr=0.001, momentum=0.9), \n","                evaluate: bool = False):\n","    \"\"\"Train torch model.\"\"\"\n","    \n","    model.train()\n","    \n","    for epoch in range(epochs):\n","\n","        for images, labels in train_data:\n","            images, labels = images.to(device), labels.to(device)\n","        \n","            optimizer.zero_grad()\n","            \n","            logits = model(images)\n","            loss = criterion(logits, labels)\n","            loss.backward()\n","            optimizer.step()\n","\n","        # Evaluate model!\n","        if evaluate:\n","            predictions, labels = evaluate_model(model, test_data, device)\n","            test_acc = np.mean(np.argmax(predictions.cpu().numpy(), axis=1) == labels.cpu().numpy())\n","        \n","        print(f\"Epoch {epoch+1}/{epochs} - test accuracy: {(100 * test_acc):.2f}% and CE loss {loss.item():.2f}\")\n","\n","    return model\n","\n","def evaluate_model(model, data, device):\n","    \"\"\"Evaluate torch model.\"\"\"\n","    model.eval()\n","    logits = torch.Tensor().to(device)\n","    targets = torch.LongTensor().to(device)\n","\n","    with torch.no_grad():\n","        for images, labels in data:\n","            images, labels = images.to(device), labels.to(device)\n","            logits = torch.cat([logits, model(images)])\n","            targets = torch.cat([targets, labels])\n","    \n","    return torch.nn.functional.softmax(logits, dim=1), targets"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"t6_qEwhee1WH","executionInfo":{"status":"ok","timestamp":1640102277771,"user_tz":-60,"elapsed":1832,"user":{"displayName":"Anna Hedström","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhbfsluHeZ1mzN6Bsf-1zU62lYHcz183jYjeS63=s64","userId":"05540180366077551505"}},"outputId":"fabc06df-34b7-49fc-ba4f-583707c862bb"},"source":["path_model_weights = \"drive/MyDrive/Projects/quantus/tutorials/assets/mnist\"\n","\n","if pathlib.Path(path_model_weights).is_file():\n","    model.load_state_dict(torch.load(path_model_weights))\n","    \n","else:\n","\n","    # Train and evaluate model.\n","    model = train_model(model=model.to(device),\n","                        train_data=train_loader,\n","                        test_data=test_loader,\n","                        device=device,\n","                        epochs=20,\n","                        criterion=torch.nn.CrossEntropyLoss().to(device),\n","                        optimizer=torch.optim.SGD(model.parameters(), lr=0.01, momentum=0.9),\n","                        evaluate=True)\n","\n","    # Save model.\n","    torch.save(model.state_dict(), path_model_weights)\n","\n","# Model to GPU and eval mode.\n","model.to(device)\n","model.eval()\n","\n","# Check test set performance.\n","predictions, labels = evaluate_model(model, test_loader, device)\n","test_acc = np.mean(np.argmax(predictions.cpu().numpy(), axis=1) == labels.cpu().numpy())        \n","print(f\"Model test accuracy: {(100 * test_acc):.2f}%\")"],"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Model test accuracy: 99.01%\n"]}]},{"cell_type":"markdown","metadata":{"id":"4vY9mZQanaxr"},"source":["### 1.3 Generate explanations\n","\n","There exist multiple ways to generate explanations for neural network models e.g., using `captum` or `innvestigate` libraries. In this example, we rely on the `quantus.explain` functionality (a simple wrapper around `captum`) however use whatever approach or library you'd like to create your explanations.\n","\n","**Requirements.**\n","\n","* **Data type.** Similar to the x-y pairs, the attributions should also be of type `np.ndarray`\n","* **Shape.** Sharing all the same dimensions as the input (expect for nr_channels which for explanations is equal to 1). For example, if x_batch is of size (128, 3, 224, 224) then the attributions should be of size (128, 1, 224, 224)."]},{"cell_type":"code","metadata":{"id":"gNxAtc2Co1pL"},"source":["# Generate normalised Saliency and Integrated Gradients attributions of the first batch of the test set.\n","a_batch_saliency = quantus.normalise_by_negative(Saliency(model).attribute(inputs=x_batch, target=y_batch, abs=True).sum(axis=1).cpu().numpy())\n","a_batch_intgrad = quantus.normalise_by_negative(IntegratedGradients(model).attribute(inputs=x_batch, target=y_batch, baselines=torch.zeros_like(x_batch)).sum(axis=1).cpu().numpy())\n","\n","# Save x_batch and y_batch as numpy arrays that will be used to call metric instances.\n","x_batch, y_batch = x_batch.cpu().numpy(), y_batch.cpu().numpy()\n","\n","\n","# Quick assert.\n","assert [isinstance(obj, np.ndarray) for obj in [x_batch, y_batch, a_batch_saliency, a_batch_intgrad]]"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"iRDwzUUp8bR2"},"source":["Visualise attributions given model and pairs of input-output."]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":628},"id":"82WWNmyoilXo","executionInfo":{"status":"ok","timestamp":1640102281314,"user_tz":-60,"elapsed":3146,"user":{"displayName":"Anna Hedström","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhbfsluHeZ1mzN6Bsf-1zU62lYHcz183jYjeS63=s64","userId":"05540180366077551505"}},"outputId":"46910bcf-1acd-46a0-f479-8512f967143c"},"source":["# Plot explanations!\n","nr_images = 3\n","fig, axes = plt.subplots(nrows=nr_images, ncols=3, figsize=(nr_images*2.5, int(nr_images*3)))\n","for i in range(nr_images):\n","    axes[i, 0].imshow((np.reshape(x_batch[i], (28, 28)) * 255).astype(np.uint8), vmin=0.0, vmax=1.0, cmap=\"gray\")\n","    axes[i, 0].title.set_text(f\"MNIST digit {y_batch[i].item()}\")\n","    axes[i, 0].axis(\"off\")\n","    axes[i, 1].imshow(a_batch_saliency[i], cmap=\"seismic\")\n","    axes[i, 1].title.set_text(f\"Saliency\")\n","    axes[i, 1].axis(\"off\")\n","    a = axes[i, 2].imshow(a_batch_intgrad[i], cmap=\"seismic\")\n","    axes[i, 2].title.set_text(f\"Integrated Gradients\")\n","    axes[i, 2].axis(\"off\")\n","plt.savefig(f'{path}/quantus/tutorials/assets/mnist_example.png', dpi=400)\n","plt.tight_layout()\n","plt.show()"],"execution_count":null,"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAhMAAAJkCAYAAACrhKNdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzde7xVZbX/8e8IMRREELyAGnhLM0lPRxMzPVZ4Sz1a3shCPak/Ne2qVsc6aV67aOWl0mOl5j0tS61IqEOJqWmJec8LEhQioAgiKOL4/TEnumCPsfdePBv2hc/79eKl+7ufPddca8+19lhzPWM+5u4CAABYXm/p7B0AAADdG8UEAAAoQjEBAACKUEwAAIAiFBMAAKAIxQQAAChCMdGJzGw3M5vW8PXDZrZbO3+23WOBjJk9Y2aj6v8/1cx+2Nn7BHQEMxtuZm5mq3XyfqwSz7EeU0zUv7BXzWzwMvn99QE1vP76ivrr9zSM2dzMvOHrCWZ2dMPXp5rZZDN7ycymmdkNdf5wnb1kZovNbGHD16c2ex/c/Z3uPqHZsWZ2upld3dr4hv1q3N+Lmt1HdE1m9j4z+5OZvWhmz5vZnWa2QzPbcPdz3P3otkeiu2r8w9aOsUu9Dq5s7Xld64DbGG1m95jZfDN7rv7/T5qZrYjb66jnWFcplBr1mGKiNlnSR5d8YWYjJK0ZjHte0lnt2aCZHSFpjKRR7t5P0vaSfie98Qe9X53fIenEJV+7+zlld6VjNexXP0kbSFog6cZO3i10ADPrL+k2SRdJWkfShpK+JumVztwvINMV/gia2UmSLpD0LVWvietLOk7SzpJWT36m10rbwW6mpxUTV0k6vOHrIyT9JBh3paR3mdl/tGObO0j6rbs/JUnu/qy7/+/y7JyZrVGfGXnBzB6pt934/cbTYWuY2ZX12EfN7AvLfCTyjJmNMrO9JJ0q6dD6jMMD7diVAyU9p6oAQvf3dkly9+vcfbG7L3D32939b2a2mZn93sxmm9ksM7vGzAZEG1n2naCZjazPdswxswcaP1ar37WeWZ8BmWdmtzeeFWw4UzLHzKaa2ZFmtoOZzWh8QTazj7TzmEUHq38nE83svPp1ZrKZ7V1/72xJu0i6uH5dubjOtzKzcfXZr8fN7JCG7Q0ys1vNbK6Z3WtmZ5nZxIbvu5mdYGZPSHqizi6oj4+5ZvYXM9ulzsPXNTNb28x+ZGbTzeyf9W30qr/Xq74vs8zsaUn7tHLf15Z0hqRPuvtN7j7PK/e7+8fc/ZV63BVm9gMz+7WZzZf0fjPbx6oz3nPrfT99mW2PMbMp9XPuy8t8r6OeY3+s/zunfnx2suoM+x+sOjs5y+oz6CtLTysm7pbU38zeUR9goyVFp8lelnSOpLPbuc3DzewUM9veyirT0yRtVv/bU1Wx09rY4ZI2lbS7pI9Hg9x9rKr7ckN95mHbduzHEZJ+4lxLvaf4u6TFdfG5t5kNbPieSTpX0lBJ75C0saTT29qgmW0o6VeqzuCtI+lkST8zs3Ubhh0m6b8krafqndzJ9c8Ok/QbVWdK1pW0naRJ7n6vpNmS9mjYxhjFBT9Wjh0lPS5psKRvSvqRmZm7f1lLn2090cz6Shon6VpVv/PRkr5vZlvX2/qepPmq3uUfofj17YD6Npf8zL2qjo916u3eaGZ9Wnldu0LSa5I2l/Rvqo6lJR8bHCNp3zrfXtJBrdzvnSS9VdIv23yEquP8bElrSZpY38fDJQ1QVbAcb2YHSFL9WPxA1XE9VNIgSRtFGy15jknatf7vgPrxuUvSmZJulzSwvs2V+jF2TysmpDfPTuwu6VFJ/0zGXSrpbUsq8Yy7Xy3pU6r++P9B0nNm9sXl3LdDJJ3t7s+7+1RJF7Yx9hx3f8Hdp7Uxtt3qF/r/UHV2Bj2Au8+V9D5JLukySTPN7BYzW9/dn3T3ce7+irvPlPRtVb//tnxc0q/d/dfu/rq7j5N0n6QPNYy53N3/7u4LJP1U1R8FqXoBHF+fKVnk7rPdfVL9vSvrbcvM1lH1vLq26AFAiSnufpm7L1b1uxmi6nR/ZF9Jz7j75e7+mrvfL+lnkg6u32QdKOk0d3/Z3R9R/Bpzbv36t0CqXl/r4+M1dz9f1R/4LaMbN7P1VR1/n3X3+e7+nKTvqCpqpOo187vuPtXdn1dVRGcGS5rl7q81bH/JGYIFZrZrw9hfuvud9fNgobtPcPcH66//Juk6vfmcOkjSbe7+x/rsxv9Iej3Zh5LnWGSRpGGShtb7ObGVsR2upxYTh0k6Uq2846l/0WfW/1rl7te4+yhVlehxks40sz2XY9+GSpra8PWUJsZOzQY2aYykie4+uYO2hy7A3R919yPdfSNJ26g6fr5rZuub2fX1KeG5qs7UDW51Y5Vhqv5IzFnyT1XBMqRhzLMN//+ypH71/28s6alku1dL2q9+l3uIpDvcfXp77yc63Bu/Q3d/uf7ffsnYYZJ2XOaY+JiqMxHrSlpNbb9mLZWZ2clWfYz7Yr29tZUfn8Mk9ZY0veH2L1X1rl1q7vV1tqTB1jB3w93f6+4D6u81/m1cdp93NLP/M7OZZvaiqr8JS/Z5qX1w9/n19rL7s7zPscgXVJ2J/LNVzQGfaGVsh+txxYS7T1E1EfNDkn7exvDLVRUIH2nnthe5+42S/qbqBbtZ01W90C7xtjbGNp4e2zgbqOodaXsdLs5K9Gju/piq08HbqDpV7JJGuHt/Ve+G2jNTfaqkq9x9QMO/vu7+9Xb+7GbJvv1T0l2qnnNjVBX/6JqWfV2ZKukPyxwT/dz9eEkzVX380NZrVmPX3C6q/gAeImlg/Yf8Rb15fEa3/4qkwQ2339/d31l/v5nX17vqbe3fypgW+1y7VtItkjZ297UlXdKwz0vtg5mtqeqjjkjJc6zFa349n+8Ydx8q6VhVH0Ft3o5tdYgeV0zUjpL0gboqTNWnuE6TlH5sYdUkpX3MbC0ze0v9scg7Jd2zHPv1U0n/bWYDzWwjVR+ftGfshpJObGXsDEnDzazV36eZvVfVTH+6OHoQqybFnVQfUzKzjVV1Nd2t6nPelyS9WB9Hp7Rzs0vOIOxZT2zrY9V1UcLPf5dxjaRRZnaIma1m1cS8xtOzP1H1R2SE2i740XlmqJqztcRtkt5eTzDsXf/bwczeUX9M8nNJp5vZmma2lZaeDB9ZS1UBMlPSamb2VUn9l7n9N17X6jNYt0s638z616/Hm9mbE+l/KunTZrZRPW/oS9kNu/scVR1P3zezgxpe37eT1Lcd+/28uy+06hIDhzV87yZJ+1o1AXl1VZM8s9flkufYTFUfn7zx+zGzgxt+9gVVBUf2EUuH65HFhLs/5e73tXP4daqqycxcVbOK/yFpjqpJSscv5+dRX1N16m2yqidFa+/KzpA0rR47XtVBmrX6LSkOZpvZX1vZ5hGSfu7u85rZaXR581RNarvHqhnnd0t6SNJJqo65d6t6x/crtfOPdz2nZ39Vx/5MVe+iTlE7XjPc/R+qzgyepKoNe5KkxonBN6s6xXtzw6l1dD0XSDrIqk6PC+vXjT1UzVH4l6pT8N9QNc9Bqt7wrF3nV6l6bW2tPfm3ksaqmkA8RdJCLf2RQvS6driqiYiPqPqDeZPe/FjgsnqbD0j6q9o41t39m5I+r6qwnVH/u1TVm8s/tfKjn5R0hpnNk/RVVUXMkm0+LOkEVWcvptf7OC3aSOFz7GVVk0LvrD8iGamqO/AeM3tJ1ZmTz7j7021tq6MYE/q7BzM7XtJod2/P5DmgSzOzpyQd6+7jO3tfsGKY2TckbeDurXWtoYfokWcmegIzG2JmO9en3rZU9S7v5s7eL6CUmR2o6hTs7zt7X9Bx6o/b3mWV96j6uJnXrFVEp1+FDKnVVZ1y20TVxyvXS/p+p+4RUMjMJqi6xsAYd19pn+dipVhL1UcbQ1V9ZHC+2ncdB/QAfMwBAACK8DEHAAAoQjEBAACKtDpnwhqW5Qa6Andvamlgs/2aPIbDxQJVXdssMjDI+geZVHW+RV5I8uwyKQuSvNllYxYl+asdsO3sMeid5HOaHD8kybMLBC5O8llBlj0u0djmud/a7mP4hhuauiAdsMIdemh80TvOTAAAgCIUEwAAoAjFBAAAKEIxAQAAinDRKmApWX2dTcyMJlVmE/gyzzU5fr0kzyYIZvuTTaqM8mwi5BpJ3ifJs0mo2QTJbM2lbN+zyamZaJJrNAEVQGs4MwEAAIpQTAAAgCIUEwAAoAjFBAAAKEIxAQAAitDNgR4u6wbIuhCyy0A3cznmbGx06W1J2jzJt0zyrDtjhyR/OsmHJfnjTWzjgSTfJ8l3TvL1k/zBJL83yccneXaZ7ahLJ+tEaXbFdLpCsOrgzAQAAChCMQEAAIpQTAAAgCIUEwAAoAjFBAAAKEI3B1ZR2VoTc5vcTtQVkm0jW98j66p4f5iabRLmo0bFW5k1691hfv/9s5PbnRRkTyZjsw6SrDsjc3uS/znJH0nyXZN8SpJHa4VMT8ZmXToAODMBAACKUEwAAIAiFBMAAKAIxQQAAChCMQEAAIrQzYEeLpqtL+XrW2RreQxK8mi9hgHJ2PlJnnUP/DFMd9st7ubokywpcf/9WSfGzUk+LciydUKitUkk6XtJPjTJs06JLB+R5JkXkjz6nbCmBtAszkwAAIAiFBMAAKAIxQQAAChCMQEAAIowARM9XP8k75XkC5I8m7C5XpAdE4685pohYT5+fLzlyy9/IMzvuisev3BhfMlos83D3D2Zsak9giy6xLY0YsR5YX700fGWt98+zm+7Lc4nTIjzu+66Lv6Gkg2lBgZZNmm3WVx+G6sOzkwAAIAiFBMAAKAIxQQAAChCMQEAAIpQTAAAgCJ0c6CHy7o5sstmxx0X0s5hOmbMDi2yE06ItzBopIX5YaecEubT9/pmmI8dm11+e06YHnvssDC/9NJPhbn73UH64XDsvvvGe3LccXG++pOPhPl7z9oq/oGHHgpj2zben7ybI7v89tNBlnVhrJ7k2Xsyujmw6uDMBAAAKEIxAQAAilBMAACAIhQTAACgCMUEAAAo0m26Ody9s3dhKWbxzHx0NfObHD87yaeF6W67tezm2HGL5+NNbLBBnD/7bBjvt188fOzYeA0O6ZYwveSSuCtEGpnk0XoVN4cjzz033sa5596ZbPtLYXrccfH7mhEj3pVs57tJflKSZ+4Nsngdkryb419N3mb3dOiU+Dia9MUvhvl2x8Rr1Oiss8L42vHROjdSn2QJmUXZcjnoFJyZAAAARSgmAABAEYoJAABQhGICAAAUoZgAAABFuk03R1fT1bpLVqTu3bnyUpLPTfJZSR53UBx1VMtugKOOitfxkMaG6W8/vm2Yz7s/3op/6tr4G8m09znfiscP2C5eJ0O//GWL6CcTPh4OPXze9+JtjPlJnK8Wv+RY3zvi8fpxkmdrq2yZ5NkaLdFx8GqT2+iX5AuTvJsaPDiM4x4MSVdcEcbjL7sszLNH974kT5aF0dujbW+8cTh22tSpYb7RWmvFGx8Zdy8Nfej2MB89Ot7MQQfFeb/kUHr00TjvSjgzAQAAilBMAACAIhQTAACgCMUEAAAoQjEBAACK0M2BHi6bI56ts9ByrY1KPJN9xIj/bJE9+OCTyTZODtM994yndm+xRdxB8cWr41yjRoVxtnLEgM99Lsxt2JAWWTa5/fC+8ToLevjhMJ71gx+E+eLFcXdUr17Z3m+U5H9N8qznIFpzJev0ydZt6Z3kPcstgz8R5ps/HOd9kqVoRl0XdwC9duKJYf5Ysj9Zr0z/Cy5oGU6cGI69OunmePe8eWH+yLhxYf6vdePj6/Xzngvzt0z4fZi/uv0HwpxuDgAA0ONRTAAAgCIUEwAAoAjFBAAAKEIxAQAAinSbbo7OWh9iVVqDo2fK5nz3SvIXkjzuHoiWmrjmms3DsXPm/CLMTzjh7jB/4on/CXPb8d1hPmZM3IVw1VWTw/xzkzYJ88lq+Vwb/r69w7G/PnF6mO+zz2lhbhY/n876ehjrggsODfPPfGZR/APpWh5ZV0+0Fkt2bGS3mY3vWRYsiPMHH2wu1+AT4vz6OH/7GvHwC6+O8z8Gy/F8feynw7Hb7RJv4+2fjfM9tkpuM1nSZ9etopVClHZePTY47uboDjgzAQAAilBMAACAIhQTAACgCMUEAAAoQjEBAACKWGvdCpZNvUaXsqI7Tjqrkybi7k3tjNkByYPT7Az89yb5+4NsWDiyT59BYb5w4UXJtuOukLxj4fUwPeaYn4X5xZfFD+U5QbZrcovP/Ch+eI866t7kJx4J0913PyLMx42bkGwnXt9B+kiSZ++bbg+ybA2OZo+ZV8PU/dZ2H8M33CBeg7uBgw+O87fc9NMwf3r7Q8L83uxp04UcemjQ7iXOTAAAgEIUEwAAoAjFBAAAKEIxAQAAilBMAACAIt1mbQ6sWl0bHad3kq+e5NkaDn9K8seDLF4jY+HC7ZJtxGtzSNOSPL6u/1prHRvml112dpj/7/z5YX769de3yH6sT4RjjzrqN2H+0Y/Ga3nstdcOYX7EEdkaKtkCD/H2pcVJPiDJ1w+yZDGIVLaeS9zNge5r443j/C1XJB1W/fqF8cSJ8fC3vnU5dqqL4MwEAAAoQjEBAACKUEwAAIAiFBMAAKAIxQQAAChCNwd6uKxLIMs3anL7UefGwGTsJkm+dZJn3QCLwnTevEuT8ZPC1Pq+loyP1huZHo584IG4q2LbbaeE+XXX3Zzc5oQk/88kvyXJP5rk8f7H3T5bJmPjxzHr3kHPs8EGcf78VnG30xNPxOMHJ8u/zJu3HDvVRXBmAgAAFKGYAAAARSgmAABAEYoJAABQhAmYXRCXze5M2cTMYUk+IsiyCXmvJ3l8WWtpfJJv0eR2skuE3xum6677wRbZzJnHh2O33fbDYd679x5hvmhRNLlTknZO8u8meXzpcCm+XemhJB8cZMmsuXCslD++TyY5urrsstmbTvxJ/I1f/CKMe3/l52HenSdaZjgzAQAAilBMAACAIhQTAACgCMUEAAAoQjEBAACK0M0BLGVWki9I8ugyzS8kY7Muj8VJ/tkkz7oKbgvT/fb7UZjfeuuPw3zmzJuCdLvkNn8TpgcdFHdVXHHFtmF+c3KV7dGj430fMqRPmB95ZLydm2+Ob/exx6LLp2eX074hybmcdk/z3m3i3+mzOx8R5htsEXdYTZvWYbvU5XFmAgAAFKGYAAAARSgmAABAEYoJAABQhGICAAAUoZujE7EGx8qwepPjs5n5rzYxvlcydr0mb/OWJD86yfcN0732ikffd98nwnz69N8FabyPu+zynTD/4Q/j21x99EfCfPgX4zUMDj447to48MB4+zNmxPmQIXH+4ouDWmTTp0+JByvLszVR0NWtsUbyjRNPDOOxyfAjH3sszBfc2Pw+dVecmQAAAEUoJgAAQBGKCQAAUIRiAgAAFKGYAAAARejm6AHo2lgeWR2ddWIsTPLNg2x2Mjbr2hiY5Fn3QDanfJMwPeGEs5LxWyf5P4Ksfzhy9Oh4C2ueGHeK6EtfCuOnnoyHz5sX56NH3xl/I9x3KV9z5ekgy35P2bGRra2Cru4/t/9XmE/b/6owP3L48DC/4Ubel/MIAACAIhQTAACgCMUEAAAoQjEBAACKUEwAAIAidHOsBCt6DQ60Jptpn621kcnW+IjaEOLOB2mHJJ+W5Nsmedy1kXeRDEvyuIVi3XWvaZH9+7/HWzjhhEfiXEkbxuUXJ/uS/Z4+nOQ3JHl2Xw9L8u2C7C/J2LuTPOvyQFex335xPve1oWF+xjHxa/Z20eEiqeUKL6sezkwAAIAiFBMAAKAIxQQAAChCMQEAAIpQTAAAgCJ0c3SwFdm5wRocy6N3kvdJ8mbXWRgSZNnaDv2SPJ5RLsXrBkiDkzzurNhii/PC/IknHgzziRNbZiNHJjeZdFWMGHFjmE+YEG9l0KDnku0/keRfSPJ3J3n82MTHR7YOS9bRkx1j85McK9IWW7TM1jzvjHDshQO+GuaXXRZ3Ol1/fbQWDyTOTAAAgEIUEwAAoAjFBAAAKEIxAQAAilBMAACAInRzLCfW2+guFnXQdrKZ/JE1kvylJN8jyddL8q8n+dvC9Iknsg6VU8N0yy13DtK4M8GnHBNvev9/C+Nrx96f7MvvkjzrlLg9yfdN8juTvG+QPZ6Mzbo8ml3nBR2hd3JovHvO71tk124ed2185mPxoh0f/vCty71fqyrOTAAAgCIUEwAAoAjFBAAAKEIxAQAAilBMAACAInRzdEGswdGRml1rIxvfTFdIryTP1uyI1veQpLuTPFsfIOs4+XGSZ+tbtNyfPn2S2zzv02H89xviro2Pbfn95DajDhIpX1MjW/fitiTP1v5o5vgYmOTZOi/Z7xsd4aWkOer/jf1Ai+ymm7Kt/DxMDz10+fZpVcaZCQAAUIRiAgAAFKGYAAAARSgmAABAEYoJAABQhG6ONrAGR3eXdVZkeTa7PxsfzfAflowdkOSZ6UmerdmRrUuRrWMxKMl/0yJZuLB/OPL4RReG+SVbPplsO/OXdu9LJeuuyX5/2Rof0e8v68J4IcmzYwMdYX7SuDNpUpwfd1zLbLvt4rGDBmXHBZrFmQkAAFCEYgIAABShmAAAAEUoJgAAQBGKCQAAUIRujk7EGhwrQza7v6PW7JgRZNns/ieS/HdJHndQSH2bHJ+t2bFGkg8Osrjb4pJLjkq2kXVbfDDJs7UzXk/y7DFY2OT4BUGWzfBv9phBR3jooTifMyfOVwv+qg3KGpfQYTgzAQAAilBMAACAIhQTAACgCMUEAAAowgTMGpfNRqUjLo38dJLvluS7JvlFTd7uzUmeTcB8W5JHkxKzS3vvneTZJMbJSf5gkmeTSrP3QX2SPJvIGU3YzPY9y7NtM2GzGTOiucyS9torzo88Ms632qplNnx4PLYXV0LvMJyZAAAARSgmAABAEYoJAABQhGICAAAUoZgAAABF6OYAltLsDPyBQdYvGftIkm+d5FknQ9ZZkY2P9lGSvp3k5wXZmGTsrCS/M8mnJXnWcZJ1UGT3dX6SZ5fZjrafjcWKNDi6irukfsnTafz4OD/66JbZQQfFYzfcsO39QvtwZgIAABShmAAAAEUoJgAAQBGKCQAAUIRiAgAAFKGbYyUws87eBaxU2foQWZfA2CTvm+RZh0P23uDxJF8vyc8KsvWTsUOSPDMoybMOlWz87CR/Ncmzxz7q3skWbGCtjRUp69q44444n5807nzmMx2zP2gOZyYAAEARigkAAFCEYgIAABShmAAAAEUoJgAAQBG6OWp0XGD5ZGtTrMhtZF0eWRfC5kmedSdEnRWTm9xGsx0R2fjnmtwOuqsFC+J8+PCVuhtYTpyZAAAARSgmAABAEYoJAABQhGICAAAUoZgAAABFzN07ex8AAEA3xpkJAABQhGICAAAUoZgAAABFKCYAAEARigkAAFCEYgIAABShmAAAAEUoJgAAQBGKCQAAUIRiAgAAFKGYAAAARSgmAABAEYoJAABQhGICAAAUoZgAAABFKCYAAEARigkAAFCEYgIAABShmAAAAEUoJgAAQBGKCQAAUIRiAgAAFKGYAAAARSgmAABAEYoJAABQhGICAAAUoZgAAABFKCYAAEARiolOZGa7mdm0hq8fNrPd2vmz7R4LZMzsGTMbVf//qWb2w87eJ6AjmNlwM3MzW62T92OVeI71mGKi/oW9amaDl8nvrw+o4fXXV9Rfv6dhzOZm5g1fTzCzoxu+PtXMJpvZS2Y2zcxuqPOH6+wlM1tsZgsbvj612fvg7u909wnNjjWz083s6mysmb3VzH5kZlPMbJ6ZTTKzvZvdP3RdZvY+M/uTmb1oZs+b2Z1mtkMz23D3c9z96LZHortq/MPWjrFLvQ6ubG29rnXQbYw2s3vMbL6ZPVf//yfNzFbE7XXUc6yrFEqNekwxUZss6aNLvjCzEZLWDMY9L+ms9mzQzI6QNEbSKHfvJ2l7Sb+T3viD3q/O75B04pKv3f2csrvSoVaTNFXSf0haW9JXJP10SYGF7s3M+ku6TdJFktaRtKGkr0l6pTP3C8h0hT+CZnaSpAskfUvSBpLWl3ScpJ0lrZ78TK+VtoPdTE8rJq6SdHjD10dI+kkw7kpJ7zKz/2jHNneQ9Ft3f0qS3P1Zd//f5dk5M1ujPjPygpk9Um+78fuNp8PWMLMr67GPmtkXlvlI5BkzG2Vme0k6VdKh9RmRB5a9XXef7+6nu/sz7v66u9+mqvD69+W5H+hy3i5J7n6duy929wXufru7/83MNjOz35vZbDObZWbXmNmAaCPLvhM0s5H12Y45ZvZA48dq9bvWM+szIPPM7PbGs4INZ0rmmNlUMzvSzHYwsxmNL8hm9pHomMWKV/9OJprZefXrzOQlZyzN7GxJu0i6uH5dubjOtzKzcfXZr8fN7JCG7Q0ys1vNbK6Z3WtmZ5nZxIbvu5mdYGZPSHqizi6oj4+5ZvYXM9ulzsPXNTNbuz7LOt3M/lnfRq/6e73q+zLLzJ6WtE8r931tSWdI+qS73+Tu87xyv7t/zN1fqcddYWY/MLNfm9l8Se83s32sOuM9t97305fZ9hirzgLPNrMvL/O9jnqO/bH+75z68dnJqjPsf7Dq7OQsq8+gryw9rZi4W1J/M3tHfYCNlhSdJntZ0jmSzm7nNg83s1PMbHsrq0xPk7RZ/W9PVcVOa2OHS9pU0u6SPh4Ncvexqu7LDfUZkW3b2gkzW1/VH6CHm9l5dFl/l7S4Lj73NrOBDd8zSedKGirpHZI2lnR6Wxs0sw0l/UrVGbx1JJ0s6Wdmtm7DsMMk/Zek9VS9kzu5/tlhkn6j6kzJupK2kzTJ3e+VNFvSHg3bGKO44MfKsaOkxyUNlvRNST8yM3P3L2vps60nmllfSeMkXavqdz5a0vfNbOt6W9+TNF/Vu/wjFL++HVDf5pKfuVfV8bFOvd0bzaxPK69rV0h6TdLmkv5N1bG05GODYyTtW+fbSzqolfu9k6S3Svplm49QdZyfLWktSRPr+3i4pAGqCpbjzewASaofix+oOq6HShokaaNooyXPMUm71v8dUD8+d0k6U9LtkgbWt3lRO+5bhxXspJkAACAASURBVOlpxYT05tmJ3SU9KumfybhLJb3N2pg74O5XS/qUqj/+f5D0nJl9cTn37RBJZ7v78+4+VdKFbYw9x91fcPdpbYxtNzPrLekaSVe6+2MdsU10LnefK+l9klzSZZJmmtktZra+uz/p7uPc/RV3nynp26o+7mrLxyX92t1/XZ/NGifpPkkfahhzubv/3d0XSPqpqj8KUvUCOL4+U7LI3We7+6T6e1fW25aZraPqeXVt0QOAElPc/TJ3X6zqdzNE1en+yL6SnnH3y939NXe/X9LPJB1cv8k6UNJp7v6yuz9Sb29Z59avfwuk6vW1Pj5ec/fzVf2B3zK68fpN0IckfbY+2/qcpO+oKmqk6jXzu+4+1d2fV1VEZwZLmuXurzVsf8kZggVmtmvD2F+6+53182Chu09w9wfrr/8m6Tq9+Zw6SNJt7v7H+uzG/0h6PdmHkudYZJGkYZKG1vs5sZWxHa6nFhOHSTpSrbzjqX/RZ9b/WuXu17j7KFWV6HGSzjSzPZdj34aqmruwxJQmxk7NBraXmb1F1ePzqqQTS7eHrsPdH3X3I919I0nbqDp+vmtm65vZ9fUp4bmqztQNbnVjlWGq/kjMWfJPVcEypGHMsw3//7KkfvX/byzpqWS7V0var36Xe4ikO9x9envvJzrcG79Dd3+5/t9+ydhhknZc5pj4mKozEevqzblZS0SvWUtlZnayVR/jvlhvb23lx+cwSb0lTW+4/UtVvWuXmnt9nS1psDXM3XD397r7gPp7jX8bl93nHc3s/8xsppm9qOpvwpJ9Xmof3H1+vb3s/izvcyzyBVVnIv9sVXPAJ1oZ2+F6XDHh7lNUzQf4kKSftzH8clUFwkfaue1F7n6jpL+pesFu1nRVL7RLvK2NsY2nxzbOBqp6R9oqMzNJP1L1ruNAd1/U1s+ge6rPOF2h6hg9R9XxMcLd+6t6N9SemepTJV3l7gMa/vV196+382c3S/btn5LuUvWcG6OquEXXtOzrylRJf1jmmOjn7sdLmqnq44e2XrMau+Z2UfUH8BBJA+s/5C/qzeMzuv1XJA1uuP3+7v7O+vvNvL7eVW9r/1bGtNjn2rWSbpG0sbuvLemShn1eah/MbE1VH3VESp5jLV7zvZrPd4y7D5V0rKqPoDZvx7Y6RI8rJmpHSfpAXRWm6lNcp0lKP7awapLSPma2lpm9pf5Y5J2S7lmO/fqppP82s4FmtpGqj0/aM3ZDtX4mYYak4fWZh8wPVH1mvt+SU4zoGayaFHdSfUzJzDZW1dV0t6rPeV+S9GJ9HJ3Szs0uOYOwZz2xrY9V10UJP/9dxjWSRpnZIWa2mlUT8xpPz/5E1R+REWq74EfnmaFqztYSt0l6ez3BsHf9bwcze0f9McnPJZ1uZmua2VZaejJ8ZC1VBchMSauZ2Vcl9V/m9t94XavPYN0u6Xwz61+/Hm9mb06k/6mkT5vZRvW8oS9lN+zuc1R1PH3fzA5qeH3fTlLfduz38+6+0KpLDBzW8L2bJO1r1QTk1VVN8sxel0ueYzNVfXzyxu/HzA5u+NkXVBUc2UcsHa5HFhPu/pS739fO4depqiYzc1XNKv6HpDmqJikdv5yfR31N1am3yaqeFK29KztD0rR67HhVB2nW6ndj/d/ZZvbXZb9ZT4g7VtXnbc/am9fC+Nhy3Ad0PfNUTWq7x6oZ53dLekjSSaqOuXeresf3K7Xzj3c9p2d/Vcf+TFXvok5RO14z3P0fqs4MnqSqDXuSpMaJwTerOsV7c8OpdXQ9F0g6yKpOjwvdfZ6qCY+jJf1L1Sn4b6ia5yBVb3jWrvOrVL22ttae/FtJY1VNIJ4iaaGW/kghel07XNVExEdU/cG8SW9+LHBZvc0HJP1VbRzr7v5NSZ9XVdjOqP9dqurN5Z9a+dFPSjrDzOZJ+qqqImbJNh+WdIKqsxfT632cFm2k8Dn2sqpJoXfWH5GMVNUdeI+ZvaTqzMln3P3ptrbVUcy9zTPk6ALM7HhJo929PZPngC7NzJ6SdKy7j+/sfcGKYWbfkLSBu7fWtYYeokeemegJzGyIme1cn3rbUtW7vJs7e7+AUmZ2oKpTsL/v7H1Bx6k/bnuXVd6j6uNmXrNWEZ1+FTKkVld1ym0TVR+vXC/p+526R0AhM5ug6hoDY9x9pX2ei5ViLVUfbQxV9ZHB+WrfdRzQA/AxBwAAKMLHHAAAoAjFBAAAKNLqnAlrWJYb6Arcvamlgc32a/IYDhcLVF53R0u1LE7GrtHcrqSyy4QsTPLsPmX7GenT5Daya6I1c5tS/PhK1YUQmxmf5a8GWTaVIxrbPPdb230M77132xekA1am3/wmvugdZyYAAEARigkAAFCEYgIAABShmAAAAEW4aBVWUdmEvGZFEw2zSYbZRMiBSZ7tY7YOUTY+W+8u289ocmO2783uS3Zfs+Vx/pXk2STJ7HabmSjaUccGsOrgzAQAAChCMQEAAIpQTAAAgCIUEwAAoAjFBAAAKEI3B7CU7JLJWTdDJOscyC53nV0eO7v89qZJvl6T+zM5yWcleTPbzro2su6PTZI8e7/zXJL3T/KsoyX7nUSyS4qv2MtvA90BZyYAAEARigkAAFCEYgIAABShmAAAAEUoJgAAQBG6OWru3tm70CYz6+xd6IaaXash69rIZvLPbWIbmWh9CEmaneRZl0CWZ/f1wST/eJCtn4zN1s64PclHJnnWVTE0ybPujKybI+tQidYhaXaNE9by6Agvvtjc+A02iPN9922ZDR4cj7377jh/7LE4X5A1XoEzEwAAoAzFBAAAKEIxAQAAilBMAACAIhQTAACgyCrXzdEdujYy2b7T5dGRmu3yiNagGNLktrdL8mwdi2wNjmlN5qeE6Q477NwiO/LIeAsnnJB1YWT36ewk3yjJn07yrZM8u6/Z9qPx2Zoo2e816/KYnuSrtt12i/PeUWONpGeeifNttonzAw5oma0z7W/JNt4V5lk3x3e/G+e9aOjhzAQAAChDMQEAAIpQTAAAgCIUEwAAoEiPnYDZnSdaojNlkySbkc3GyibwfTBM99orHr/FFvFWxo2L88ce+3Fyu7eE6b33tpyAOGjQu5NtZJepziYfNjupNLvk93NJvmmSP5Lk/YIsuyR3dOl0Kb8c+qrtU5+K8w+Nii/7/vdn4knOF18cb2f8+DiPJmzuv3880XJgcpX8ftFhIalPMn4RhwBnJgAAQBmKCQAAUIRiAgAAFKGYAAAARSgmAABAkW7fzdHVujaiS1t3tX1ctSTTr9NLJmf1ddahEc38zzoWsm3EnQlDhsTdHNtvH2/loYfi/LHHtkxud0qS/65FMnZsNva2JM8ud31WkmcdFHGnS35Z7hlJPjLJ1w+yrEPlySTPLim+ahgYXVFe0qhRyQ88+2wYz5r1tjCfllwhfcGCOL///vZlkjR8eJyPGBHnO7e80rwkacKEOF+VcGYCAAAUoZgAAABFKCYAAEARigkAAFCEYgIAABTp9t0cK1rUnYHuJOugyOro7CL7WT47yLKOgmTae9I9cPnl8XYuvzzZTNCFUdkhyY8M091336RFNmlSvIWZMwcn2/6/MB0zZlCY/+IX8VbmzWv2PmVrczyd5NEaIi8lY7Muj/lJvmp47bU4v/rqOH/ppbhr47akMahX9hRuwm9/G+dvfWucn3JKnM+ZU74vPRVnJgAAQBGKCQAAUIRiAgAAFKGYAAAARSgmAABAkW7fzZF1WzS7HgZdGz1Vtm7C4ibH924iz7a9UZLHa3BI8ZoaZsPCfJtt9gjzBx+Muw0uuCBeQ+TTw29pGfaJ1zixPbOuijvD9KCD4tFXXTUh2c69SX5Fkv8xyecm+eQgWy8Zm/2esm3Ha670NPPmxfkPfxjnqyV/dfpmS9p0APczwnzhwuPC/Fvfio+B44/vsF3qcTgzAQAAilBMAACAIhQTAACgCMUEAAAoQjEBAACKdPtujkxndWc020WCFS3rrMjq6Gx8lkedFdm09LgjYuDAT4T5fffFW9l0cNw98Omv9A/z4cOTro1B18Q3MP6eOA989KNxB8l118X3df/9s8fxhiSPty99OsmzdTKyfOsgyzp3pjS57VXb2mt3zu2OHXtHkGZdQSOS/MNh+uijy7NHqwbOTAAAgCIUEwAAoAjFBAAAKEIxAQAAilBMAACAIj22m2NF64yuDdYPWR7Zehhxt4G0KMmz9RdmNbEv8TZeeGF2mN9226BkO3HXxsUXx6PPOy/ZzIABYfzz3S5skR144O3JRk5N8l5J/t0kn5bkf03y7PeXbSfbnxlBNj0ZGz/u+baxIo0dm6198ucguz4cud9+cafTouxlACnOTAAAgCIUEwAAoAjFBAAAKEIxAQAAilBMAACAInRzdEF0bXSkrAsjk62zkHVtrB5k2doc0VhJOjJMP/OZfZPx2doR8RT0X/zi2DAfcOQ+YX7UUVcn229p443PCfOpU7MujDOSPFojQ8ofsweTPFtrIfv9RWuFROutSNKCJJ+T5Fixok4cSTq0RXLccfFzcsiQeAt33bWcu7QK48wEAAAoQjEBAACKUEwAAIAiFBMAAKAIxQQAAChCN0cbOmMNDnSkrDsjs16SZzP5ow6Kx5OxmyR51v2R2TnJp4TpHXfEXR533PGVML/mmm+0yDbbLL7F88+P86lTfxfmW2zxizB/8sl4O+7Hx9/QyCTP1ubIxkfdHGskY/+S5IOT/JEkRzPGjo3XSll33bhz57lnXm6R3ZM0/3yj5aGO5cSZCQAAUIRiAgAAFKGYAAAARSgmAABAEYoJAABQhG6OWmd0bbAGx8qQrWPxapJna3n0SvKoUyLrCJmd5Nn6E32S/LkkjzoTJOngMDWLOysO2/7vLbIzrn97OPbGGxcmtzkpTJ944sfJ+PWTPHssN0ryrJMmWYRB/wiy7NjIfk/Nrv+CyPyk8WqnneLf3Q9/mGxo4sQW0aXX7xEOXZA1aaFpnJkAAABFKCYAAEARigkAAFCEYgIAABRhAuZKwETLzpRNmsvEl56WsomG0WS9YU2MlaQ5SZ5N7HtvmG6ySTxJ9OkJ74w3c+v3wvirV5/QIjvzzPjy2NJlSZ5NeLwzybPHPXvMoomTkvRCkl+a5AODbFYytn+SN3vJdkSuuCLONz35I/E3PvtSGN9y4u0tsunxFbnRgTgzAQAAilBMAACAIhQTAACgCMUEAAAoQjEBAACKrHLdHJ1x2Wx0RVmXQNZVkIk6KPomY7NLPe+d5PElqaWbw/Tpi9YI89OH7Rvf6t3xc+HMkaOD9KRw7O67Xx/m48b9Kszzxz27bHZ2GfNpSZ51Vmye5NGlybPHvdmWAC6zHRk+PM43PeBdYX7Ogw+G+akPPBDmF5/cMuuVHUboMJyZAAAARSgmAABAEYoJAABQhGICAAAUoZgAAABFemw3R2d1bbAOR1ezOMmzjotsnYwtkzyaJj4iGZut7/FIkm8apv/850Fh/v0N42PvA8nWR46Mu0J6927ZobFo0eRw7LhxWYdKvG6CFHec5I/BVkkePzbSX5J8dpJHv79sH1ngoRmLk6feD7aJ14Q5/ZK4a2PrZPv/e3fc/UHnRufgzAQAAChCMQEAAIpQTAAAgCIUEwAAoAjFBAAAKNJjuzmASrYWRLaGQ9blkXkhyCYkY9+b5Nk6FrHzzovzbw8YEObHj472UdIlZ4TxokXvbpGNGLFJOHZ0tIyHpNdei7tfTjvtr/EP6KYkH5vk2fug1ZP80CTvH2S9k7F9mrzNVXttjsGDk288HncAjUyG7zU/fq7ucUA8nm6OzsGZCQAAUIRiAgAAFKGYAAAARSgmAABAEYoJAABQhG6O5cQaHN1FNqU869rI1s+YkeTRGhHTkrFZ18a2Sb5rmE6YEI+2OU/G37gku92ss+JTLZJ9941HnnpAvKbG3I3iFRVOO+2y5DZ3S/LnknxIkt+Z5IOSPFpHJVsnpF+SZ90fq3Y3x3e+k3zjV9uF8V6z4/VTPvLxNcOcro2uhTMTAACgCMUEAAAoQjEBAACKUEwAAIAiFBMAAKAI3RzLyd1X+m1mHSSdsS8rWsd1y2RrcGTrLGR5M7JOg+lJfkuSDw3T++8/ORn/4TA9+OBPhPmNN45PtvN6i+Tcc+N9/8pX4q6NtftmXTHxCgxbbHFEmD/xRNwRse660Zoa0syZ2VosWYdG1M2R/Z6yYyNrK+iIY6n7yrqO7pwUH48XHRU/V4ckT6ettorz115rY8cazJkT5xtsEOfNdpD0Thp9Vkv+8mb5s8/G+WOPtcy2i5tltNFGcT49O9ybxJkJAABQhGICAAAUoZgAAABFKCYAAEARigkAAFCEbo5upCd2bax4C5I8Wz+jZSdDZfMknxRkA5OxWYfDHkmerdlxYZjutNMmYf7Zz8ZbufHGLyTbj9bPeDoc2bdv1i2zepJfHKbZ7Pkjj4y7Nj74wXj8yJFRd4Yk/TjJo3VLdkjGZrIOkuxYWjUsXhznF12UrRUTd9xMn/7HJI+PX7NdWmQnJw1Q8+bFedaFkXWQZA48MM7Xt3jNmQuvXy/MJ06MtxN1aGRdGwuTl5/svi5aFOcZzkwAAIAiFBMAAKAIxQQAAChCMQEAAIpQTAAAgCJ0c6CHyzorsqnK2boa8RoR8Yz9LZOxg5M8mfaedpzEa3mstto5YT4+WYKjd+/4vi5a9ESLbJddfhSOPfroeNtXXx3n553X3Pgvfzn+PX35y9m6Ipl43RLpr0GWdahkXRvZsbFqy9a92GSTfcJ88uTsccyeTzPCdNSoltk3v5Js+6qr4nz48DifFHVvSa//95fDPFuf5PyxcddG1nGxedJMNi14iYiy1rbdbNdGhjMTAACgCMUEAAAoQjEBAACKUEwAAIAiFBMAAKBIj+3mMLMwZ32LXPaYdW/ZehhZl8esJM/WmohmZUcdAlLeKRKvP5F3nPQK0zvuiNbUkO64Y9NkO/FaCNF9veOOyeHIUaPi9UDGjYsf9223zWbs35DkWyf50CQ/LckPTfLodzI7GZsdG9kaHFmXzqrh1lvjfLfd4vytb/1omM+YEef9+sXb6dOnZXbPo/FzbMd11403skvL9T0k6dcWd6KMT9b+uO22OB8wIM4HDYrz7oAzEwAAoAjFBAAAKEIxAQAAilBMAACAIhQTAACgiLXW3WBmtD6gS3H3plpOzA5IjuFmZ9pn3RzRdrK1BIJp5pKkZBGDtHsgWzsi6xaJ1wHI1wp5Osiyfcw6UeLZ8FkXTZ8+Hwzz/faLtzJxYpxPn/7F5Hazx2x6kGVrcMRdNLn4GHO/td3H8N57i9dgdCm/+Y3C45czEwAAoAjFBAAAKEIxAQAAilBMAACAIj32ctpApaMuaZxNyosmIGa3mV2meUGSZ5M+m5Vtp5lLh783GbtFkvdN8niSaHL1Yj32WJxPn35dsv0pSZ5Nis3GR1bty2MDreHMBAAAKEIxAQAAilBMAACAIhQTAACgCMUEAAAoQjcHerhmL4Hcu8nx2WWaI1k3wKImt51dlvuFJH8uybP7Gl1mO74MtjQjyW9O8rizZNy4bN93S/IJTW1feijJo8cy20b2+8h+r80ee0D3xZkJAABQhGICAAAUoZgAAABFKCYAAEARigkAAFCEbg70cB010z4b/3qQPZmMzbowsi6BrIMiW8ujX5Jnj0GWTwuyq5Ox0dokkrR+kmfrgWT39Y4k3zTJ5zaZR50bC5OxzWItD6w6ODMBAACKUEwAAIAiFBMAAKAIxQQAAChCMQEAAIrQzYFVVLMdDh0h6xLIOiJeTfKssyTrlMi6SJrpUMned2RdEtm6H+slebYexqAkzx7LrNOl2f0H0AzOTAAAgCIUEwAAoAjFBAAAKEIxAQAAilBMAACAIubunb0PAACgG+PMBAAAKEIxAQAAilBMAACAIhQTAACgCMUEAAAoQjEBAACKUEwAAIAiFBMAAKAIxQQAAChCMQEAAIpQTAAAgCIUEwAAoAjFBAAAKEIxAQAAilBMAACAIhQTAACgCMUEAAAoQjEBAACKUEwAAIAiFBMAAKAIxQQAAChCMQEAAIpQTAAAgCIUEwAAoAjFBAAAKEIxAQAAilBMAACAIhQTAACgCMVEJzKz3cxsWsPXD5vZbu382XaPBTJm9oyZjar//1Qz+2Fn7xPQEcxsuJm5ma3WyfuxSjzHekwxUf/CXjWzwcvk99cH1PD66yvqr9/TMGZzM/OGryeY2dENX59qZpPN7CUzm2ZmN9T5w3X2kpktNrOFDV+f2ux9cPd3uvuEZsea2elmdnVr483sRDO7z8xeMbMrmt03dG1m9j4z+5OZvWhmz5vZnWa2QzPbcPdz3P3otkeiu2r8w9aOsUu9Dq5s7Xld64DbGG1m95jZfDN7rv7/T5qZrYjb66jnWFcplBr1mGKiNlnSR5d8YWYjJK0ZjHte0lnt2aCZHSFpjKRR7t5P0vaSfie98Qe9X53fIenEJV+7+zlld6XD/UvVff5xZ+8IOpaZ9Zd0m6SLJK0jaUNJX5P0SmfuF5DpCn8EzewkSRdI+pakDSStL+k4STtLWj35mV4rbQe7mZ5WTFwl6fCGr4+Q9JNg3JWS3mVm/9GObe4g6bfu/pQkufuz7v6/y7NzZrZGfWbkBTN7pN524/cbT4etYWZX1mMfNbMvLPORyDNmNsrM9pJ0qqRD6zMiD0S37e4/d/dfSJq9PPuOLu3tkuTu17n7Yndf4O63u/vfzGwzM/u9mc02s1lmdo2ZDYg2suw7QTMbWZ/tmGNmDzR+rFa/az2zPgMyz8xubzwr2HCmZI6ZTTWzI81sBzOb0fiCbGYfyY5ZrFj172SimZ1Xv85MNrO96++dLWkXSRfXrysX1/lWZjauPvv1uJkd0rC9QWZ2q5nNNbN7zewsM5vY8H03sxPM7AlJT9TZBfXxMdfM/mJmu9R5+LpmZmub2Y/MbLqZ/bO+jV7193rV92WWmT0taZ9W7vvaks6Q9El3v8nd53nlfnf/mLu/Uo+7wsx+YGa/NrP5kt5vZvtYdcZ7br3vpy+z7TFmNqV+zn15me911HPsj/V/59SPz05WnWH/g1VnJ2dZfQZ9ZelpxcTdkvqb2TvqA2y0pOg02cuSzpF0dju3ebiZnWJm21tZZXqapM3qf3uqKnZaGztc0qaSdpf08WiQu49VdV9uqM+IbFuwf+ie/i5pcV187m1mAxu+Z5LOlTRU0jskbSzp9LY2aGYbSvqVqrNZ60g6WdLPzGzdhmGHSfovSeupeid3cv2zwyT9RtWZknUlbSdpkrvfq6qY3aNhG2MUF/xYOXaU9LikwZK+KelHZmbu/mUtfbb1RDPrK2mcpGtV/c5HS/q+mW1db+t7kuarepd/hOLXtwPq21zyM/eqOj7Wqbd7o5n1aeV17QpJr0naXNK/qTqWlnxscIykfet8e0kHtXK/d5L0Vkm/bPMRqo7zsyWtJWlifR8PlzRAVcFyvJkdIEn1Y/EDVcf1UEmDJG0UbbTkOSZp1/q/A+rH5y5JZ0q6XdLA+jYvasd96zA9rZiQ3jw7sbukRyX9Mxl3qaS3LanEM+5+taRPqfrj/wdJz5nZF5dz3w6RdLa7P+/uUyVd2MbYc9z9BXef1sZYrMLcfa6k90lySZdJmmlmt5jZ+u7+pLuPc/dX3H2mpG9Las8ZuY9L+rW7/9rdX3f3cZLuk/ShhjGXu/vf3X2BpJ+q+qMgVS+A4+szJYvcfba7T6q/d2W9bZnZOqqeV9cWPQAoMcXdL3P3xap+N0NUne6P7CvpGXe/3N1fc/f7Jf1M0sH1m6wDJZ3m7i+7+yP19pZ1bv36t0CqXl/r4+M1dz9f1R/4LaMbN7P1VR1/n3X3+e7+nKTvqCpqpOo187vuPtXdn1dVRGcGS5rl7q81bH/JGYIFZrZrw9hfuvud9fNgobtPcPcH66//Juk6vfmcOkjSbe7+x/rsxv9Iej3Zh5LnWGSRpGGShtb7ObGVsR2upxYTh0k6Uq2846l/0WfW/1rl7te4+yhVlehxks40sz2XY9+GSpra8PWUJsZOzQYC7v6oux/p7htJ2kbV8fNdM1vfzK6vTwnPVXWmbnCrG6sMU/VHYs6Sf6oKliENY55t+P+XJfWr/39jSU8l271a0n71u9xDJN3h7tPbez/R4d74Hbr7y/X/9kvGDpO04zLHxMdUnYlYV9Jqavs1a6nMzE626mPcF+vtra38+Bwmqbek6Q23f6mqd+1Sc6+vsyUNtoa5G+7+XncfUH+v8W/jsvu8o5n9n5nNNLMXVf1NWLLPS+2Du89X/tFyyXMs8gVVZyL/bFVzwCdaGdvhelwx4e5TVE3E/JCkn7cx/HJVBcJH2rntRe5+o6S/qXrBbtZ0VS+0S7ytjbGNp8c2zgaqekcKSJLc/TFVp4O3UXWq2CWNcPf+qt4NtWem+lRJV7n7gIZ/fd396+382c2SffunpLtUPefGqCr+0TUt+7oyVdIfljkm+rn78ZJmqvr4oa3XrMauuV1U/QE8RNLA+g/5i3rz+Ixu/xVJgxtuv7+7v7P+fjOvr3fV29q/lTEt9rl2raRbJG3s7mtLuqRhn5faBzNbU9VHHZGS51iL1/x6Pt8x7j5U0rGqPoLavB3b6hA9rpioHSXpA3VVmKpPcZ0mKf3YwqpJSvuY2Vpm9pb6Y5F3SrpnOfbrp5L+28wGmtlGqj4+ac/YDSWd2MrYGZKGm1n6+zSz1cysj6ReknqZWR/rAjOqUc6qSXEn1ceUzGxjVV1Nd6v6nPclSS/Wx9Ep7dzskjMIe9YT2/pYdV2U8PPfZVwjaZSZHVIfd4PMrPH07E9U/REZobYLfnSeGarmbC1xm6S31xMMe9f/djCzd9Qfk/xc0ulmtqaZbaWlJ8NH1lJVgMyUtJqZfVVS/2Vu/43XtfoM1u2Szjez/vXr8Wb25kT6n0r6tJltVM8b+lJ2w+4+R1XH0/fNZ1eO/QAAD85JREFU7KCG1/ftJPVtx34/7+4LrbrEwGEN37tJ0r5WTUBeXdUkz+x1ueQ5NlPVxydv/H7M7OCGn31BVcGRfcTS4XpkMeHuT7n7fe0cfp2qajIzV9Ws4n9ImqNqktLxy/l51NdUnXqbrOpJ0dq7sjMkTavHjld1kGatfjfW/51tZn9NxnxF0gJVT7CP1///lWZ2Hl3WPFWT2u6xasb53ZIeknSSqmPu3are8f1K7fzjXc/p2V/VsT9T1buoU9SO1wx3/4eqM4MnqWrDniSpcWLwzapO8d7ccGodXc8Fkg6yqtPjQnefp2rC42hVrebPSvqGqnkOUvWGZ+06v0rVa2tr7cm/lTRW1QTiKZIWaumPFKLXtcNVTUR8RNUfzJv05scCl9XbfEDSX9XGse7u35T0eVWF7Yz636Wq3lz+qZUf/aSkM8xsnqSvqipilmzzYUknqDp7Mb3ex2nRRgqfYy+rmhR6Z/0RyUhV3YH3mNlLqs6cfMbdn25rWx3F3DlD3h2Y2fGSRrt7eybPAV2amT0l6Vh3H9/Z+4IVw8y+IWkDd2+taw09RI88M9ETmNkQM9u5PvW2pap3eTd39n4BpczsQFWnYH/f2fuCjlN/3PYuq7xH1cfNvGatIvjMvOtaXdUpt01UfbxyvaTvd+oeAYXMbIKqawyMcfeV9nkuVoq1VH20MVTVRwbnq33XcUAPwMccAACgCB9zAACAIhQTAACgSKtzJqxhWW6gK3D3ppYGNtsvOYazJVZ6J3m4iKCqbrZlLU7GZteuyS7416eJ25SWbtFv1OrlVgLR/mf7+FKSZ9MhsvHPJXmz+96s6L5mx0b2e22O+63tPoY//3kuSIeu5dvfji96x5kJAABQhGICAAAUoZgAAABFKCYAAEARLlqFHi6bTNdsnnk1yLJ1grJJnNnEvmyiZTYpMdtOli9K8kg2ATNbEXlWk+Ozxz2bmPlCkmePcfR7ynTMREtgVcKZCQAAUIRiAgAAFKGYAAAARSgmAABAEYoJAABQhG4O9HDNXhq52Ty65HXWUZB1OGT5sCTPujmy+/p0kg9M8rntzFrTzOMlSRs1mWeXPV8jyRckedTR0lGX8G6mgwTo3jgzAQAAilBMAACAIhQTAACgCMUEAAAoQjEBAACK0M2BHi6b9Z91G2Tj10vyaO2IbM2LrDuj2Y6FbJ2M7D5l7xmyzoeoy+OlJsZKeUfE3Uk+NMkz2Rof2X3N1vKIulSyjpNm1jIBVi2cmQAAAEUoJgAAQBGKCQAAUIRiAgAAFKGYAAAARejm6ILcvUO2Y2Ydsp3urdmujfWbHB+th7FJMvaDST4iTNdaK+7ymDdvSrKdhUl+c5JnBgXZyGRs3yTPOiImJXn2+L43yeck+bQm82gdlWxtlaz7JVuDo6PW+MDK9uyzcb755nE+OGmweuaZDtmdboEzEwAAoAjFBAAAKEIxAQAAilBMAACAIkzARA8XTZCU8omZ05O8mYmG2UTIh5I8nkw4b96AZPw/kjybCPh0kkeXApek/kF2SzJ2SJJnj8EWSd7s7yO7LHd2n7IJodFky2xfssf39SRHd7X22nF+d3LY7bXXituX7oIzEwAAoAjFBAAAKEIxAQAAilBMAACAIhQTAACgCN0cPQCXzV4eA5N8bpJnM/mjbpGsg2RBkmeXx/5zki9qYl8k6d+T/F9JPjnIsg6H5PrC6eO7dZI/mOS3J3l2qeqs6yYbHz1mWedHvyTPjo0sR1e3225x/oP/ip+Tn7/+PStuZ7oJzkwAAIAiFBMAAKAIxQQAAChCMQEAAIpQTAAAgCJ0c3Qid+/sXViFZWtHZJ0S2Qz/yAtJnnVPDE7yrCNi/SSfluRZt8iMJO8dZLsmY9dI8rirok+fkWG+cOHjyXaidUKkvHMl68bJHuPovmadH9FYKf99o6sbkCx/c+guyXP1oXgdHXBmAgAAFKKYAAAARSgmAABAEYoJAABQhGICAAAUoZsDPVzWtZHJ1lPI6u6sSyDyyQ7YhiRNSvJmuwqy2406IuJ1RT73uVPC/IAD4i0PHx7nw4btnOzLk0meda5smuSZ6H5lx0wzHSFSvp4JuoqTT47zR/puGOZ9nqIDL8OZCQAAUIRiAgAAFKGYAAAARSgmAABAEYoJAABQhG4O9HDZjPps/YXVkzxbsyNaIyJel0KanuTZPmbjH0jybN+zNUH2D9Pevf+zRbZo0efCsd/5zuww//Y2vwzzL9z2iWRfbk/yQUn+YJJn3TiZ6DjIfn9ZBwldG93Vmpd8O8xfTsZff8UK25VujzMTAACgCMUEAAAoQjEBAACKUEwAAIAiFBMAAKAI3RwrgXvHXM/dzDpkO6uWrMMhq6NfT/I+SR6tY5GNzToQRib540merQWRrR0xIEwPPrhl14YkPfZYy+zBB+POkksvjbst1jgh7tpYuPC6MP/Zz+J1Sw48MOtoyR6zm5M86rqR4sc4+p1K+bGUbZsuj65icPLUeOmkk8J809nxa/ZLZ3XUHvU8nJkAAABFKCYAAEARigkAAFCEYgIAABShmAAAAEXo5kAPl82o76iZ9tFM/qwLo3+SP5LkayT5kCTPukX2DtNnnkm28uDXW2RXXnl9OPbw0fFaGMcee3mYv//9x4b5R7Z7OszXXXfTMJ8583dhLm2d5Pcm+XpB9lwyNkPXRld36peSLq1vxZ1Oh382Hp51hYAzEwAAoBDFBAAAKEIxAQAAilBMAACAIhQTAACgCN0c6OGydRMycXdCPmM/Xpsitm+SZ2t5ZGtBTGpyX+L1Le699zfJ+JZT2T90RLwuzOlHJJtIHseTT45Hf/7irGsj64y5Icmzjpmsy+OFINs2GZvtS3aMRdtGpzjuuDA+fc6cMB+ZLP3y5JMdtUM9D2cmAABAEYoJAABQhGICAAAUoZgAAABFKCYAAEARujk6mLt39i5gKVl3RiabmZ91XPQNsqyjYG6SZzV9ttZG3PkgTUnyEWG60057hPmffthyrZA/vzPe8vDkFqW7w3TQoF3C/DvfuSnZTrZuyX8mefb7izta4vGzm9w2a3N0Jbvt1jJ7ev/LwrGjk238+JmO2ptVB2cmAABAEYoJAABQhGICAAAUoZgAAABFmID5/9u7nxBP6zoO4N8ptx1RZGUZQ1hRxLCDZSZ2kjBCEWSIzSgPQnTwEh4iFNYOsYQGQYdAEBRM6OZJcRIWh+oQxFxKbSU0B8FY3HLtD+LCwg7+OqyX5fm8jWc/M862+3od3/vdZx9+vx32sw/f9/M9Dy0t1a8u5pOQNtmdDnn1Ot706uZqs+YYY1wb8rRpMG3MTBsw63dev/xyvfqdfdP7/8ojj5Rrv3T4p2X+vb2PlfmTT9YbMMd4KeQPhPzKkL8X8gMhrzaKpteYp2vbgHk+ue66aXb96mq9+NChMt5K+4GJPJkAAFoMEwBAi2ECAGgxTAAALYYJAKBFmwPOsifkl4a8el13egV0yn/3sXc0lV73nFohx8r01KlXy3xj455J9s2N+vXYe/emJsNmmT7zTH0vY3w25G+F/PKQp2346XutXnF+MqxNUgOI3XDTTdPs9bW1cu07P3hhh+/m4uHJBADQYpgAAFoMEwBAi2ECAGgxTAAALdoc52ixWLSv4QyO3ZR24Ked/Km18LkiS2dwXB/ydKZGOn/i1yFPf+4TIb+xTO+996pJtrr623CNn4T8ZyGv2hNjjHF3yNP5JDeE/I6Zf+5vimw5rE0+DLkzO3bScviaPvWff02yzz/9dLn2qfSjxGyeTAAALYYJAKDFMAEAtBgmAIAWwwQA0KLNwQUutTZSnhoR1RkcY4xxRZEdCGvT7v507Q9CntoGN4f8YMh/GfLpZ7O2lhoht5Xp6uq0EXLmOqmdkc7auDrkp0Kevr90xkd1n+l7SmeisBvuuy/8wkMPTaK/Hgp/11/bvvu52HkyAQC0GCYAgBbDBADQYpgAAFoMEwBAizYHF7i0Mz/lc+fr6lyNdO03Qr4n5OmckK+FPDUi0vkZnwn5/iJ7r1x5553fL/N0bsIYb4b82ZDfFfL0Wab2x7GQV+dqpBZN4gyO3XAglaa2tiZRUfAYY4xxQzrihdk8mQAAWgwTAECLYQIAaDFMAAAthgkAoEWbgwtcaiwkp2eur87VSM2BL4T86yHfCHlqRKRWSPLlkL80SW655cf1yp//ucyXbp7uqD/jxZB/J+T/Dnn6XtNnn1TNjXTuB+eTjfDj8Yu//2qSbW7Wa7U5to8nEwBAi2ECAGgxTAAALYYJAKDFMAEAtGhz/A+LxWK3b4GWdG7Cp0OeGhFp/a1Fls6HSA2E+tyLMd6fmR8JeWqR1J/Nww8/MMnSWRvffvSL4do/CvkPy3Rlpb7HEyd+H65zRcj/EPL0/VXNjfQ9pf97Ved7jFE3fZhr3746f/zxOl9fX51kKytr23hHVDyZAABaDBMAQIthAgBoMUwAAC2GCQCgRZuDC1xqc6Q8STv83yqydJ5EOvPhaMhTA2Huve+ftfqVV6bZ+vrJsDrd+1Uhf61MT5xILYw/hjx9BqnpklTfqxbG+eT48To/cuTd8Du+O0nuv3/77oeaJxMAQIthAgBoMUwAAC2GCQCgxTABALRoc3xkJ8/gWFpa2rFrc65SU+KykKf2QHVGxNth7TdCfkfIN0Ke3BXydI7FY2W6vv6XIq2yMca4NORpfWq6/C3kSTrPJEltnDnNmHQN7Y+ddM01db68XDeGVla+tYN3Q+LJBADQYpgAAFoMEwBAi2ECAGgxTAAALdocXKTmtDPGyDv5ry2yG8Paf4Q8rf9qyOtd7AcP1tfZ3KyvcvTowXD9Z4vsn2Ht1SFPrZjLQ35byN8IefqejoU8fd9Vqyc1fbQ2dtIl4V+jBx+s89tvr/Pnn9+e+2EeTyYAgBbDBADQYpgAAFoMEwBAiw2YH/HKa85Ir2m+MuTVa6PrDZJjHA/53Fd7v1mmzz2XXlV9NOTJn4osbZB8N+TpXtIGzNdDnjZ+7g95+v9Ryk8X2ZxXbLNdtrbq/PDhT/Q2OEeeTAAALYYJAKDFMAEAtBgmAIAWwwQA0KLNAWdJr80+FfKqzZDWJi+EPL2+ObVFwnuzx/shT6+ePlBkr85Y+3H5ByFPr8dOToY8NWOq1kaSrqHlAYknEwBAi2ECAGgxTAAALYYJAKDFMAEAtGhzwFnSjv3lkKezJiqpnbFvxjXGyE2Gt0Oe2hyptZDOJ6mkMzhSa+PDkKfPJtmuxoWGBmwHTyYAgBbDBADQYpgAAFoMEwBAi2ECAGjR5oCzpKZEyquzOZLUCEkNhyQ1EC4LeWo+7An5nLNFtCcATyYAgCbDBADQYpgAAFoMEwBAi2ECAGhZWiwWu30PAMD/MU8mAIAWwwQA0GKYAABaDBMAQIthAgBoMUwAAC3/BTbpYnfOgxHsAAAAAElFTkSuQmCC\n","text/plain":["<Figure size 540x648 with 9 Axes>"]},"metadata":{"needs_background":"light"}}]},{"cell_type":"markdown","metadata":{"id":"tuBkEBv3mihR"},"source":["## 2. Quantative evaluation using Quantus\n","\n","We can evaluate our explanations on a variety of quantuative criteria but as a motivating example we test the Max-Sensitivity (Yeh at el., 2019) of the explanations. This metric tests how the explanations maximally change while subject to slight perturbations."]},{"cell_type":"code","source":["# Define params for evaluation.\n","params_eval = {\n","    \"nr_samples\": 10,\n","    \"perturb_radius\": 0.1,\n","    \"norm_numerator\": quantus.fro_norm,\n","    \"norm_denominator\": quantus.fro_norm,\n","    \"perturb_func\": quantus.uniform_sampling,\n","    \"similarity_func\": quantus.difference,\n","    \"disable_warnings\": True,\n","}"],"metadata":{"id":"aLjrKsT6mS9X"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# Return max sensitivity scores in an one-liner - by calling the metric instance.\n","scores_saliency = quantus.MaxSensitivity(**params_eval)(model=model, \n","   x_batch=x_batch,\n","   y_batch=y_batch,\n","   a_batch=a_batch_saliency,\n","   **{\"explain_func\": quantus.explain, \"method\": \"Saliency\", \"device\": device, \"img_size\": 28, \"normalise\": False, \"abs\": False})"],"metadata":{"id":"NlV_43TAmJll"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# Return max sensitivity scores in an one-liner - by calling the metric instance.\n","scores_intgrad = quantus.MaxSensitivity(**params_eval)(model=model, \n","   x_batch=x_batch,\n","   y_batch=y_batch,\n","   a_batch=a_batch_intgrad,\n","   **{\"explain_func\": quantus.explain, \"method\": \"IntegratedGradients\", \"device\": device, \"img_size\": 28, \"normalise\": False, \"abs\": False})"],"metadata":{"id":"iq7qqDfSmIdj"},"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"3kBrG51Lpuq9"},"source":["print(f\"max-Sensitivity scores by Yeh et al., 2019\\n\" \\\n","      f\"\\n • Saliency = {np.mean(scores_saliency):.2f} ({np.std(scores_saliency):.2f}).\" \\\n","      f\"\\n • Integrated Gradients = {np.mean(scores_intgrad):.2f} ({np.std(scores_intgrad):.2f}).\"\n","      )"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"uGYHGu1Esya7"},"source":["metrics = {\"max-Sensitivity\": quantus.MaxSensitivity(**params_eval)}\n","\n","xai_methods = {\"Saliency\": a_batch_saliency,\n","               \"IntegratedGradients\": a_batch_intgrad}\n","\n","results = quantus.evaluate(evaluation_metrics=metrics,\n","                           explanation_methods=xai_methods,\n","                           model=model,\n","                           x_batch=x_batch,\n","                           y_batch=y_batch,\n","                           agg_func=np.mean,\n","                           **{\"explain_func\": quantus.explain, \"device\": device, \"img_size\": 28, \"normalise\": False, \"abs\": False})\n","\n","df = pd.DataFrame(results)\n","df"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"X9DkYz4BuySK"},"source":[""],"execution_count":null,"outputs":[]}]}