{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "## Matrix multiplication from foundations" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "The *foundations* we'll assume throughout this course are:\n", "\n", "- Python\n", "- matplotlib\n", "- The Python standard library\n", "- Jupyter notebooks and nbdev" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from pathlib import Path\n", "import pickle, gzip, math, os, time, shutil, matplotlib as mpl, matplotlib.pyplot as plt" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Get data" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "MNIST_URL='https://github.com/mnielsen/neural-networks-and-deep-learning/blob/master/data/mnist.pkl.gz?raw=true'\n", "path_data = Path('data')\n", "path_data.mkdir(exist_ok=True)\n", "path_gz = path_data/'mnist.pkl.gz'" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "[urlretrieve](https://docs.python.org/3/library/urllib.request.html#urllib.request.urlretrieve) - (read the docs!)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from urllib.request import urlretrieve\n", "if not path_gz.exists(): urlretrieve(MNIST_URL, path_gz)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "total 16656\r\n", "-rw-rw-r-- 1 jhoward jhoward 17051982 Sep 30 04:37 mnist.pkl.gz\r\n" ] } ], "source": [ "!ls -l data" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "with gzip.open(path_gz, 'rb') as f: ((x_train, y_train), (x_valid, y_valid), _) = pickle.load(f, encoding='latin-1')" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[0.0,\n", " 0.0,\n", " 0.0,\n", " 0.19140625,\n", " 0.9296875,\n", " 0.98828125,\n", " 0.98828125,\n", " 0.98828125,\n", " 0.98828125,\n", " 0.98828125]" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "lst1 = list(x_train[0])\n", "vals = lst1[200:210]\n", "vals" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "def chunks(x, sz):\n", " for i in range(0, len(x), sz): yield x[i:i+sz]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[[0.0, 0.0, 0.0, 0.19140625, 0.9296875],\n", " [0.98828125, 0.98828125, 0.98828125, 0.98828125, 0.98828125]]" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "list(chunks(vals, 5))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAN80lEQVR4nO3df6hcdXrH8c+ncf3DrBpTMYasNhuRWBWbLRqLSl2RrD9QNOqWDVgsBrN/GHChhEr6xyolEuqP0qAsuYu6sWyzLqgYZVkVo6ZFCF5j1JjU1YrdjV6SSozG+KtJnv5xT+Su3vnOzcyZOZP7vF9wmZnzzJnzcLife87Md879OiIEYPL7k6YbANAfhB1IgrADSRB2IAnCDiRxRD83ZpuP/oEeiwiPt7yrI7vtS22/aftt27d281oAesudjrPbniLpd5IWSNou6SVJiyJia2EdjuxAj/XiyD5f0tsR8U5EfCnpV5Ku6uL1APRQN2GfJekPYx5vr5b9EdtLbA/bHu5iWwC61M0HdOOdKnzjND0ihiQNSZzGA03q5si+XdJJYx5/R9L73bUDoFe6CftLkk61/V3bR0r6kaR19bQFoG4dn8ZHxD7bSyU9JWmKpAci4o3aOgNQq46H3jraGO/ZgZ7ryZdqABw+CDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUii4ymbcXiYMmVKsX7sscf2dPtLly5tWTvqqKOK686dO7dYv/nmm4v1u+66q2Vt0aJFxXU///zzYn3lypXF+u23316sN6GrsNt+V9IeSfsl7YuIs+toCkD96jiyXxQRH9TwOgB6iPfsQBLdhj0kPW37ZdtLxnuC7SW2h20Pd7ktAF3o9jT+/Ih43/YJkp6x/V8RsWHsEyJiSNKQJNmOLrcHoENdHdkj4v3qdqekxyTNr6MpAPXrOOy2p9o++uB9ST+QtKWuxgDUq5vT+BmSHrN98HX+PSJ+W0tXk8zJJ59crB955JHF+nnnnVesX3DBBS1r06ZNK6577bXXFutN2r59e7G+atWqYn3hwoUta3v27Cmu++qrrxbrL7zwQrE+iDoOe0S8I+kvauwFQA8x9AYkQdiBJAg7kARhB5Ig7EASjujfl9om6zfo5s2bV6yvX7++WO/1ZaaD6sCBA8X6jTfeWKx/8sknHW97ZGSkWP/www+L9TfffLPjbfdaRHi85RzZgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJxtlrMH369GJ948aNxfqcOXPqbKdW7XrfvXt3sX7RRRe1rH355ZfFdbN+/6BbjLMDyRF2IAnCDiRB2IEkCDuQBGEHkiDsQBJM2VyDXbt2FevLli0r1q+44opi/ZVXXinW2/1L5ZLNmzcX6wsWLCjW9+7dW6yfccYZLWu33HJLcV3UiyM7kARhB5Ig7EAShB1IgrADSRB2IAnCDiTB9ewD4JhjjinW200vvHr16pa1xYsXF9e9/vrri/W1a9cW6xg8HV/PbvsB2zttbxmzbLrtZ2y/Vd0eV2ezAOo3kdP4X0i69GvLbpX0bEScKunZ6jGAAdY27BGxQdLXvw96laQ11f01kq6uty0Adev0u/EzImJEkiJixPYJrZ5oe4mkJR1uB0BNen4hTEQMSRqS+IAOaFKnQ287bM+UpOp2Z30tAeiFTsO+TtIN1f0bJD1eTzsAeqXtabzttZK+L+l429sl/VTSSkm/tr1Y0u8l/bCXTU52H3/8cVfrf/TRRx2ve9NNNxXrDz/8cLHebo51DI62YY+IRS1KF9fcC4Ae4uuyQBKEHUiCsANJEHYgCcIOJMElrpPA1KlTW9aeeOKJ4roXXnhhsX7ZZZcV608//XSxjv5jymYgOcIOJEHYgSQIO5AEYQeSIOxAEoQdSIJx9knulFNOKdY3bdpUrO/evbtYf+6554r14eHhlrX77ruvuG4/fzcnE8bZgeQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJxtmTW7hwYbH+4IMPFutHH310x9tevnx5sf7QQw8V6yMjIx1vezJjnB1IjrADSRB2IAnCDiRB2IEkCDuQBGEHkmCcHUVnnnlmsX7PPfcU6xdf3Plkv6tXry7WV6xYUay/9957HW/7cNbxOLvtB2zvtL1lzLLbbL9ne3P1c3mdzQKo30RO438h6dJxlv9LRMyrfn5Tb1sA6tY27BGxQdKuPvQCoIe6+YBuqe3XqtP841o9yfYS28O2W/8zMgA912nYfybpFEnzJI1IurvVEyNiKCLOjoizO9wWgBp0FPaI2BER+yPigKSfS5pfb1sA6tZR2G3PHPNwoaQtrZ4LYDC0HWe3vVbS9yUdL2mHpJ9Wj+dJCknvSvpxRLS9uJhx9sln2rRpxfqVV17ZstbuWnl73OHir6xfv75YX7BgQbE+WbUaZz9iAisuGmfx/V13BKCv+LoskARhB5Ig7EAShB1IgrADSXCJKxrzxRdfFOtHHFEeLNq3b1+xfskll7SsPf/888V1D2f8K2kgOcIOJEHYgSQIO5AEYQeSIOxAEoQdSKLtVW/I7ayzzirWr7vuumL9nHPOaVlrN47eztatW4v1DRs2dPX6kw1HdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgnH2SW7u3LnF+tKlS4v1a665plg/8cQTD7mnidq/f3+xPjJS/u/lBw4cqLOdwx5HdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgnH2w0C7sexFi8abaHdUu3H02bNnd9JSLYaHh4v1FStWFOvr1q2rs51Jr+2R3fZJtp+zvc32G7ZvqZZPt/2M7beq2+N63y6ATk3kNH6fpL+PiD+X9FeSbrZ9uqRbJT0bEadKerZ6DGBAtQ17RIxExKbq/h5J2yTNknSVpDXV09ZIurpHPQKowSG9Z7c9W9L3JG2UNCMiRqTRPwi2T2ixzhJJS7rsE0CXJhx229+W9Iikn0TEx/a4c8d9Q0QMSRqqXoOJHYGGTGjozfa3NBr0X0bEo9XiHbZnVvWZknb2pkUAdWh7ZPfoIfx+Sdsi4p4xpXWSbpC0srp9vCcdTgIzZswo1k8//fRi/d577y3WTzvttEPuqS4bN24s1u+8886WtccfL//KcIlqvSZyGn++pL+V9LrtzdWy5RoN+a9tL5b0e0k/7EmHAGrRNuwR8Z+SWr1Bv7jedgD0Cl+XBZIg7EAShB1IgrADSRB2IAkucZ2g6dOnt6ytXr26uO68efOK9Tlz5nTSUi1efPHFYv3uu+8u1p966qli/bPPPjvkntAbHNmBJAg7kARhB5Ig7EAShB1IgrADSRB2IIk04+znnntusb5s2bJiff78+S1rs2bN6qinunz66acta6tWrSque8cddxTre/fu7agnDB6O7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQRJpx9oULF3ZV78bWrVuL9SeffLJY37dvX7FeuuZ89+7dxXWRB0d2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUjCEVF+gn2SpIcknSjpgKShiPhX27dJuknS/1ZPXR4Rv2nzWuWNAehaRIw76/JEwj5T0syI2GT7aEkvS7pa0t9I+iQi7ppoE4Qd6L1WYZ/I/Owjkkaq+3tsb5PU7L9mAXDIDuk9u+3Zkr4naWO1aKnt12w/YPu4FusssT1se7i7VgF0o+1p/FdPtL8t6QVJKyLiUdszJH0gKST9k0ZP9W9s8xqcxgM91vF7dkmy/S1JT0p6KiLuGac+W9KTEXFmm9ch7ECPtQp729N425Z0v6RtY4NefXB30EJJW7ptEkDvTOTT+Ask/Yek1zU69CZJyyUtkjRPo6fx70r6cfVhXum1OLIDPdbVaXxdCDvQex2fxgOYHAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJ9HvK5g8k/c+Yx8dXywbRoPY2qH1J9NapOnv7s1aFvl7P/o2N28MRcXZjDRQMam+D2pdEb53qV2+cxgNJEHYgiabDPtTw9ksGtbdB7Uuit071pbdG37MD6J+mj+wA+oSwA0k0Enbbl9p+0/bbtm9toodWbL9r+3Xbm5uen66aQ2+n7S1jlk23/Yztt6rbcefYa6i322y/V+27zbYvb6i3k2w/Z3ub7Tds31Itb3TfFfrqy37r+3t221Mk/U7SAknbJb0kaVFEbO1rIy3YflfS2RHR+BcwbP+1pE8kPXRwai3b/yxpV0SsrP5QHhcR/zAgvd2mQ5zGu0e9tZpm/O/U4L6rc/rzTjRxZJ8v6e2IeCcivpT0K0lXNdDHwIuIDZJ2fW3xVZLWVPfXaPSXpe9a9DYQImIkIjZV9/dIOjjNeKP7rtBXXzQR9lmS/jDm8XYN1nzvIelp2y/bXtJ0M+OYcXCarer2hIb7+bq203j309emGR+YfdfJ9OfdaiLs401NM0jjf+dHxF9KukzSzdXpKibmZ5JO0egcgCOS7m6ymWqa8Uck/SQiPm6yl7HG6asv+62JsG+XdNKYx9+R9H4DfYwrIt6vbndKekyjbzsGyY6DM+hWtzsb7ucrEbEjIvZHxAFJP1eD+66aZvwRSb+MiEerxY3vu/H66td+ayLsL0k61fZ3bR8p6UeS1jXQxzfYnlp9cCLbUyX9QIM3FfU6STdU92+Q9HiDvfyRQZnGu9U042p43zU+/XlE9P1H0uUa/UT+vyX9YxM9tOhrjqRXq583mu5N0lqNntb9n0bPiBZL+lNJz0p6q7qdPkC9/ZtGp/Z+TaPBmtlQbxdo9K3ha5I2Vz+XN73vCn31Zb/xdVkgCb5BByRB2IEkCDuQBGEHkiDsQBKEHUiCsANJ/D+f1mbt6t55/AAAAABJRU5ErkJggg==\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "mpl.rcParams['image.cmap'] = 'gray'\n", "plt.imshow(list(chunks(lst1, 28)));" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "[islice](https://docs.python.org/3/library/itertools.html#itertools.islice)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from itertools import islice" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "it = iter(vals)\n", "islice(it, 5)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[0.0, 0.0, 0.0, 0.19140625, 0.9296875]" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "list(islice(it, 5))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[0.98828125, 0.98828125, 0.98828125, 0.98828125, 0.98828125]" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "list(islice(it, 5))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[]" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "list(islice(it, 5))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "it = iter(lst1)\n", "img = list(iter(lambda: list(islice(it, 28)), []))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAN80lEQVR4nO3df6hcdXrH8c+ncf3DrBpTMYasNhuRWBWbLRqLSl2RrD9QNOqWDVgsBrN/GHChhEr6xyolEuqP0qAsuYu6sWyzLqgYZVkVo6ZFCF5j1JjU1YrdjV6SSozG+KtJnv5xT+Su3vnOzcyZOZP7vF9wmZnzzJnzcLife87Md879OiIEYPL7k6YbANAfhB1IgrADSRB2IAnCDiRxRD83ZpuP/oEeiwiPt7yrI7vtS22/aftt27d281oAesudjrPbniLpd5IWSNou6SVJiyJia2EdjuxAj/XiyD5f0tsR8U5EfCnpV5Ku6uL1APRQN2GfJekPYx5vr5b9EdtLbA/bHu5iWwC61M0HdOOdKnzjND0ihiQNSZzGA03q5si+XdJJYx5/R9L73bUDoFe6CftLkk61/V3bR0r6kaR19bQFoG4dn8ZHxD7bSyU9JWmKpAci4o3aOgNQq46H3jraGO/ZgZ7ryZdqABw+CDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUii4ymbcXiYMmVKsX7sscf2dPtLly5tWTvqqKOK686dO7dYv/nmm4v1u+66q2Vt0aJFxXU///zzYn3lypXF+u23316sN6GrsNt+V9IeSfsl7YuIs+toCkD96jiyXxQRH9TwOgB6iPfsQBLdhj0kPW37ZdtLxnuC7SW2h20Pd7ktAF3o9jT+/Ih43/YJkp6x/V8RsWHsEyJiSNKQJNmOLrcHoENdHdkj4v3qdqekxyTNr6MpAPXrOOy2p9o++uB9ST+QtKWuxgDUq5vT+BmSHrN98HX+PSJ+W0tXk8zJJ59crB955JHF+nnnnVesX3DBBS1r06ZNK6577bXXFutN2r59e7G+atWqYn3hwoUta3v27Cmu++qrrxbrL7zwQrE+iDoOe0S8I+kvauwFQA8x9AYkQdiBJAg7kARhB5Ig7EASjujfl9om6zfo5s2bV6yvX7++WO/1ZaaD6sCBA8X6jTfeWKx/8sknHW97ZGSkWP/www+L9TfffLPjbfdaRHi85RzZgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJxtlrMH369GJ948aNxfqcOXPqbKdW7XrfvXt3sX7RRRe1rH355ZfFdbN+/6BbjLMDyRF2IAnCDiRB2IEkCDuQBGEHkiDsQBJM2VyDXbt2FevLli0r1q+44opi/ZVXXinW2/1L5ZLNmzcX6wsWLCjW9+7dW6yfccYZLWu33HJLcV3UiyM7kARhB5Ig7EAShB1IgrADSRB2IAnCDiTB9ewD4JhjjinW200vvHr16pa1xYsXF9e9/vrri/W1a9cW6xg8HV/PbvsB2zttbxmzbLrtZ2y/Vd0eV2ezAOo3kdP4X0i69GvLbpX0bEScKunZ6jGAAdY27BGxQdLXvw96laQ11f01kq6uty0Adev0u/EzImJEkiJixPYJrZ5oe4mkJR1uB0BNen4hTEQMSRqS+IAOaFKnQ287bM+UpOp2Z30tAeiFTsO+TtIN1f0bJD1eTzsAeqXtabzttZK+L+l429sl/VTSSkm/tr1Y0u8l/bCXTU52H3/8cVfrf/TRRx2ve9NNNxXrDz/8cLHebo51DI62YY+IRS1KF9fcC4Ae4uuyQBKEHUiCsANJEHYgCcIOJMElrpPA1KlTW9aeeOKJ4roXXnhhsX7ZZZcV608//XSxjv5jymYgOcIOJEHYgSQIO5AEYQeSIOxAEoQdSIJx9knulFNOKdY3bdpUrO/evbtYf+6554r14eHhlrX77ruvuG4/fzcnE8bZgeQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJxtmTW7hwYbH+4IMPFutHH310x9tevnx5sf7QQw8V6yMjIx1vezJjnB1IjrADSRB2IAnCDiRB2IEkCDuQBGEHkmCcHUVnnnlmsX7PPfcU6xdf3Plkv6tXry7WV6xYUay/9957HW/7cNbxOLvtB2zvtL1lzLLbbL9ne3P1c3mdzQKo30RO438h6dJxlv9LRMyrfn5Tb1sA6tY27BGxQdKuPvQCoIe6+YBuqe3XqtP841o9yfYS28O2W/8zMgA912nYfybpFEnzJI1IurvVEyNiKCLOjoizO9wWgBp0FPaI2BER+yPigKSfS5pfb1sA6tZR2G3PHPNwoaQtrZ4LYDC0HWe3vVbS9yUdL2mHpJ9Wj+dJCknvSvpxRLS9uJhx9sln2rRpxfqVV17ZstbuWnl73OHir6xfv75YX7BgQbE+WbUaZz9iAisuGmfx/V13BKCv+LoskARhB5Ig7EAShB1IgrADSXCJKxrzxRdfFOtHHFEeLNq3b1+xfskll7SsPf/888V1D2f8K2kgOcIOJEHYgSQIO5AEYQeSIOxAEoQdSKLtVW/I7ayzzirWr7vuumL9nHPOaVlrN47eztatW4v1DRs2dPX6kw1HdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgnH2SW7u3LnF+tKlS4v1a665plg/8cQTD7mnidq/f3+xPjJS/u/lBw4cqLOdwx5HdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgnH2w0C7sexFi8abaHdUu3H02bNnd9JSLYaHh4v1FStWFOvr1q2rs51Jr+2R3fZJtp+zvc32G7ZvqZZPt/2M7beq2+N63y6ATk3kNH6fpL+PiD+X9FeSbrZ9uqRbJT0bEadKerZ6DGBAtQ17RIxExKbq/h5J2yTNknSVpDXV09ZIurpHPQKowSG9Z7c9W9L3JG2UNCMiRqTRPwi2T2ixzhJJS7rsE0CXJhx229+W9Iikn0TEx/a4c8d9Q0QMSRqqXoOJHYGGTGjozfa3NBr0X0bEo9XiHbZnVvWZknb2pkUAdWh7ZPfoIfx+Sdsi4p4xpXWSbpC0srp9vCcdTgIzZswo1k8//fRi/d577y3WTzvttEPuqS4bN24s1u+8886WtccfL//KcIlqvSZyGn++pL+V9LrtzdWy5RoN+a9tL5b0e0k/7EmHAGrRNuwR8Z+SWr1Bv7jedgD0Cl+XBZIg7EAShB1IgrADSRB2IAkucZ2g6dOnt6ytXr26uO68efOK9Tlz5nTSUi1efPHFYv3uu+8u1p966qli/bPPPjvkntAbHNmBJAg7kARhB5Ig7EAShB1IgrADSRB2IIk04+znnntusb5s2bJiff78+S1rs2bN6qinunz66acta6tWrSque8cddxTre/fu7agnDB6O7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQRJpx9oULF3ZV78bWrVuL9SeffLJY37dvX7FeuuZ89+7dxXWRB0d2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUjCEVF+gn2SpIcknSjpgKShiPhX27dJuknS/1ZPXR4Rv2nzWuWNAehaRIw76/JEwj5T0syI2GT7aEkvS7pa0t9I+iQi7ppoE4Qd6L1WYZ/I/Owjkkaq+3tsb5PU7L9mAXDIDuk9u+3Zkr4naWO1aKnt12w/YPu4FusssT1se7i7VgF0o+1p/FdPtL8t6QVJKyLiUdszJH0gKST9k0ZP9W9s8xqcxgM91vF7dkmy/S1JT0p6KiLuGac+W9KTEXFmm9ch7ECPtQp729N425Z0v6RtY4NefXB30EJJW7ptEkDvTOTT+Ask/Yek1zU69CZJyyUtkjRPo6fx70r6cfVhXum1OLIDPdbVaXxdCDvQex2fxgOYHAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJ9HvK5g8k/c+Yx8dXywbRoPY2qH1J9NapOnv7s1aFvl7P/o2N28MRcXZjDRQMam+D2pdEb53qV2+cxgNJEHYgiabDPtTw9ksGtbdB7Uuit071pbdG37MD6J+mj+wA+oSwA0k0Enbbl9p+0/bbtm9toodWbL9r+3Xbm5uen66aQ2+n7S1jlk23/Yztt6rbcefYa6i322y/V+27zbYvb6i3k2w/Z3ub7Tds31Itb3TfFfrqy37r+3t221Mk/U7SAknbJb0kaVFEbO1rIy3YflfS2RHR+BcwbP+1pE8kPXRwai3b/yxpV0SsrP5QHhcR/zAgvd2mQ5zGu0e9tZpm/O/U4L6rc/rzTjRxZJ8v6e2IeCcivpT0K0lXNdDHwIuIDZJ2fW3xVZLWVPfXaPSXpe9a9DYQImIkIjZV9/dIOjjNeKP7rtBXXzQR9lmS/jDm8XYN1nzvIelp2y/bXtJ0M+OYcXCarer2hIb7+bq203j309emGR+YfdfJ9OfdaiLs401NM0jjf+dHxF9KukzSzdXpKibmZ5JO0egcgCOS7m6ymWqa8Uck/SQiPm6yl7HG6asv+62JsG+XdNKYx9+R9H4DfYwrIt6vbndKekyjbzsGyY6DM+hWtzsb7ucrEbEjIvZHxAFJP1eD+66aZvwRSb+MiEerxY3vu/H66td+ayLsL0k61fZ3bR8p6UeS1jXQxzfYnlp9cCLbUyX9QIM3FfU6STdU92+Q9HiDvfyRQZnGu9U042p43zU+/XlE9P1H0uUa/UT+vyX9YxM9tOhrjqRXq583mu5N0lqNntb9n0bPiBZL+lNJz0p6q7qdPkC9/ZtGp/Z+TaPBmtlQbxdo9K3ha5I2Vz+XN73vCn31Zb/xdVkgCb5BByRB2IEkCDuQBGEHkiDsQBKEHUiCsANJ/D+f1mbt6t55/AAAAABJRU5ErkJggg==\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "plt.imshow(img);" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Matrix and tensor" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "0.98828125" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "img[20][15]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "class Matrix:\n", " def __init__(self, xs): self.xs = xs\n", " def __getitem__(self, idxs): return self.xs[idxs[0]][idxs[1]]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "0.98828125" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "m = Matrix(img)\n", "m[20,15]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import torch\n", "from torch import tensor" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor([1, 2, 3])" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "tensor([1,2,3])" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "torch.Size([50000, 784])" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "x_train,y_train,x_valid,y_valid = map(tensor, (x_train,y_train,x_valid,y_valid))\n", "x_train.shape" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'torch.FloatTensor'" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "x_train.type()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "[Tensor](https://pytorch.org/docs/stable/tensors.html)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "torch.Size([50000, 28, 28])" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "imgs = x_train.reshape((-1,28,28))\n", "imgs.shape" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAN80lEQVR4nO3df6hcdXrH8c+ncf3DrBpTMYasNhuRWBWbLRqLSl2RrD9QNOqWDVgsBrN/GHChhEr6xyolEuqP0qAsuYu6sWyzLqgYZVkVo6ZFCF5j1JjU1YrdjV6SSozG+KtJnv5xT+Su3vnOzcyZOZP7vF9wmZnzzJnzcLife87Md879OiIEYPL7k6YbANAfhB1IgrADSRB2IAnCDiRxRD83ZpuP/oEeiwiPt7yrI7vtS22/aftt27d281oAesudjrPbniLpd5IWSNou6SVJiyJia2EdjuxAj/XiyD5f0tsR8U5EfCnpV5Ku6uL1APRQN2GfJekPYx5vr5b9EdtLbA/bHu5iWwC61M0HdOOdKnzjND0ihiQNSZzGA03q5si+XdJJYx5/R9L73bUDoFe6CftLkk61/V3bR0r6kaR19bQFoG4dn8ZHxD7bSyU9JWmKpAci4o3aOgNQq46H3jraGO/ZgZ7ryZdqABw+CDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUii4ymbcXiYMmVKsX7sscf2dPtLly5tWTvqqKOK686dO7dYv/nmm4v1u+66q2Vt0aJFxXU///zzYn3lypXF+u23316sN6GrsNt+V9IeSfsl7YuIs+toCkD96jiyXxQRH9TwOgB6iPfsQBLdhj0kPW37ZdtLxnuC7SW2h20Pd7ktAF3o9jT+/Ih43/YJkp6x/V8RsWHsEyJiSNKQJNmOLrcHoENdHdkj4v3qdqekxyTNr6MpAPXrOOy2p9o++uB9ST+QtKWuxgDUq5vT+BmSHrN98HX+PSJ+W0tXk8zJJ59crB955JHF+nnnnVesX3DBBS1r06ZNK6577bXXFutN2r59e7G+atWqYn3hwoUta3v27Cmu++qrrxbrL7zwQrE+iDoOe0S8I+kvauwFQA8x9AYkQdiBJAg7kARhB5Ig7EASjujfl9om6zfo5s2bV6yvX7++WO/1ZaaD6sCBA8X6jTfeWKx/8sknHW97ZGSkWP/www+L9TfffLPjbfdaRHi85RzZgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJxtlrMH369GJ948aNxfqcOXPqbKdW7XrfvXt3sX7RRRe1rH355ZfFdbN+/6BbjLMDyRF2IAnCDiRB2IEkCDuQBGEHkiDsQBJM2VyDXbt2FevLli0r1q+44opi/ZVXXinW2/1L5ZLNmzcX6wsWLCjW9+7dW6yfccYZLWu33HJLcV3UiyM7kARhB5Ig7EAShB1IgrADSRB2IAnCDiTB9ewD4JhjjinW200vvHr16pa1xYsXF9e9/vrri/W1a9cW6xg8HV/PbvsB2zttbxmzbLrtZ2y/Vd0eV2ezAOo3kdP4X0i69GvLbpX0bEScKunZ6jGAAdY27BGxQdLXvw96laQ11f01kq6uty0Adev0u/EzImJEkiJixPYJrZ5oe4mkJR1uB0BNen4hTEQMSRqS+IAOaFKnQ287bM+UpOp2Z30tAeiFTsO+TtIN1f0bJD1eTzsAeqXtabzttZK+L+l429sl/VTSSkm/tr1Y0u8l/bCXTU52H3/8cVfrf/TRRx2ve9NNNxXrDz/8cLHebo51DI62YY+IRS1KF9fcC4Ae4uuyQBKEHUiCsANJEHYgCcIOJMElrpPA1KlTW9aeeOKJ4roXXnhhsX7ZZZcV608//XSxjv5jymYgOcIOJEHYgSQIO5AEYQeSIOxAEoQdSIJx9knulFNOKdY3bdpUrO/evbtYf+6554r14eHhlrX77ruvuG4/fzcnE8bZgeQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJxtmTW7hwYbH+4IMPFutHH310x9tevnx5sf7QQw8V6yMjIx1vezJjnB1IjrADSRB2IAnCDiRB2IEkCDuQBGEHkmCcHUVnnnlmsX7PPfcU6xdf3Plkv6tXry7WV6xYUay/9957HW/7cNbxOLvtB2zvtL1lzLLbbL9ne3P1c3mdzQKo30RO438h6dJxlv9LRMyrfn5Tb1sA6tY27BGxQdKuPvQCoIe6+YBuqe3XqtP841o9yfYS28O2W/8zMgA912nYfybpFEnzJI1IurvVEyNiKCLOjoizO9wWgBp0FPaI2BER+yPigKSfS5pfb1sA6tZR2G3PHPNwoaQtrZ4LYDC0HWe3vVbS9yUdL2mHpJ9Wj+dJCknvSvpxRLS9uJhx9sln2rRpxfqVV17ZstbuWnl73OHir6xfv75YX7BgQbE+WbUaZz9iAisuGmfx/V13BKCv+LoskARhB5Ig7EAShB1IgrADSXCJKxrzxRdfFOtHHFEeLNq3b1+xfskll7SsPf/888V1D2f8K2kgOcIOJEHYgSQIO5AEYQeSIOxAEoQdSKLtVW/I7ayzzirWr7vuumL9nHPOaVlrN47eztatW4v1DRs2dPX6kw1HdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgnH2SW7u3LnF+tKlS4v1a665plg/8cQTD7mnidq/f3+xPjJS/u/lBw4cqLOdwx5HdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgnH2w0C7sexFi8abaHdUu3H02bNnd9JSLYaHh4v1FStWFOvr1q2rs51Jr+2R3fZJtp+zvc32G7ZvqZZPt/2M7beq2+N63y6ATk3kNH6fpL+PiD+X9FeSbrZ9uqRbJT0bEadKerZ6DGBAtQ17RIxExKbq/h5J2yTNknSVpDXV09ZIurpHPQKowSG9Z7c9W9L3JG2UNCMiRqTRPwi2T2ixzhJJS7rsE0CXJhx229+W9Iikn0TEx/a4c8d9Q0QMSRqqXoOJHYGGTGjozfa3NBr0X0bEo9XiHbZnVvWZknb2pkUAdWh7ZPfoIfx+Sdsi4p4xpXWSbpC0srp9vCcdTgIzZswo1k8//fRi/d577y3WTzvttEPuqS4bN24s1u+8886WtccfL//KcIlqvSZyGn++pL+V9LrtzdWy5RoN+a9tL5b0e0k/7EmHAGrRNuwR8Z+SWr1Bv7jedgD0Cl+XBZIg7EAShB1IgrADSRB2IAkucZ2g6dOnt6ytXr26uO68efOK9Tlz5nTSUi1efPHFYv3uu+8u1p966qli/bPPPjvkntAbHNmBJAg7kARhB5Ig7EAShB1IgrADSRB2IIk04+znnntusb5s2bJiff78+S1rs2bN6qinunz66acta6tWrSque8cddxTre/fu7agnDB6O7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQRJpx9oULF3ZV78bWrVuL9SeffLJY37dvX7FeuuZ89+7dxXWRB0d2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUjCEVF+gn2SpIcknSjpgKShiPhX27dJuknS/1ZPXR4Rv2nzWuWNAehaRIw76/JEwj5T0syI2GT7aEkvS7pa0t9I+iQi7ppoE4Qd6L1WYZ/I/Owjkkaq+3tsb5PU7L9mAXDIDuk9u+3Zkr4naWO1aKnt12w/YPu4FusssT1se7i7VgF0o+1p/FdPtL8t6QVJKyLiUdszJH0gKST9k0ZP9W9s8xqcxgM91vF7dkmy/S1JT0p6KiLuGac+W9KTEXFmm9ch7ECPtQp729N425Z0v6RtY4NefXB30EJJW7ptEkDvTOTT+Ask/Yek1zU69CZJyyUtkjRPo6fx70r6cfVhXum1OLIDPdbVaXxdCDvQex2fxgOYHAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJ9HvK5g8k/c+Yx8dXywbRoPY2qH1J9NapOnv7s1aFvl7P/o2N28MRcXZjDRQMam+D2pdEb53qV2+cxgNJEHYgiabDPtTw9ksGtbdB7Uuit071pbdG37MD6J+mj+wA+oSwA0k0Enbbl9p+0/bbtm9toodWbL9r+3Xbm5uen66aQ2+n7S1jlk23/Yztt6rbcefYa6i322y/V+27zbYvb6i3k2w/Z3ub7Tds31Itb3TfFfrqy37r+3t221Mk/U7SAknbJb0kaVFEbO1rIy3YflfS2RHR+BcwbP+1pE8kPXRwai3b/yxpV0SsrP5QHhcR/zAgvd2mQ5zGu0e9tZpm/O/U4L6rc/rzTjRxZJ8v6e2IeCcivpT0K0lXNdDHwIuIDZJ2fW3xVZLWVPfXaPSXpe9a9DYQImIkIjZV9/dIOjjNeKP7rtBXXzQR9lmS/jDm8XYN1nzvIelp2y/bXtJ0M+OYcXCarer2hIb7+bq203j309emGR+YfdfJ9OfdaiLs401NM0jjf+dHxF9KukzSzdXpKibmZ5JO0egcgCOS7m6ymWqa8Uck/SQiPm6yl7HG6asv+62JsG+XdNKYx9+R9H4DfYwrIt6vbndKekyjbzsGyY6DM+hWtzsb7ucrEbEjIvZHxAFJP1eD+66aZvwRSb+MiEerxY3vu/H66td+ayLsL0k61fZ3bR8p6UeS1jXQxzfYnlp9cCLbUyX9QIM3FfU6STdU92+Q9HiDvfyRQZnGu9U042p43zU+/XlE9P1H0uUa/UT+vyX9YxM9tOhrjqRXq583mu5N0lqNntb9n0bPiBZL+lNJz0p6q7qdPkC9/ZtGp/Z+TaPBmtlQbxdo9K3ha5I2Vz+XN73vCn31Zb/xdVkgCb5BByRB2IEkCDuQBGEHkiDsQBKEHUiCsANJ/D+f1mbt6t55/AAAAABJRU5ErkJggg==\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "plt.imshow(imgs[0]);" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor(0.9883)" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "imgs[0,20,15]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(tensor([5, 0, 4, ..., 8, 4, 8]), torch.Size([50000]))" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "n,c = x_train.shape\n", "y_train, y_train.shape" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(tensor(0), tensor(9))" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "min(y_train),max(y_train)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(tensor(0), tensor(9))" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "y_train.min(), y_train.max()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Random numbers" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Based on the Wichmann Hill algorithm used before Python 2.3." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "rnd_state = None\n", "def seed(a):\n", " global rnd_state\n", " a, x = divmod(a, 30268)\n", " a, y = divmod(a, 30306)\n", " a, z = divmod(a, 30322)\n", " rnd_state = int(x)+1, int(y)+1, int(z)+1" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(4976, 20238, 499)" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "seed(457428938475)\n", "rnd_state" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "def rand():\n", " global rnd_state\n", " x, y, z = rnd_state\n", " x = (171 * x) % 30269\n", " y = (172 * y) % 30307\n", " z = (170 * z) % 30323\n", " rnd_state = x,y,z\n", " return (x/30269 + y/30307 + z/30323) % 1.0" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(0.7645251082582081, 0.7920889799553945, 0.06912886811267205)" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "rand(),rand(),rand()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "In parent: 0.9559050644103264\n", "In child: 0.9559050644103264\n" ] } ], "source": [ "if os.fork(): print(f'In parent: {rand()}')\n", "else:\n", " print(f'In child: {rand()}')\n", " os._exit(os.EX_OK)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "In parent: tensor([0.3242])\n", "In child: tensor([0.3242])\n" ] } ], "source": [ "if os.fork(): print(f'In parent: {torch.rand(1)}')\n", "else:\n", " print(f'In child: {torch.rand(1)}')\n", " os._exit(os.EX_OK)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAABVWUlEQVR4nO29eZQj13Xm+b1AAIE1E7lWZWVWsbakuFUVSdGUuGmxPZIoeUwv8lh0t6T2xpZb8tg9do/Vs7SP20fdM+6xx21LNpte2lZ7UcuWZMs2rWVESyRFkVJRYlWJW1UWl8qsJfcF+/rmj4gXCACxvAhEBBLI9ztHR5UAmAgkgBs3vnvvdwmlFAKBQCAYfKR+H4BAIBAI/EEEdIFAIBgSREAXCASCIUEEdIFAIBgSREAXCASCIUHu1xNPTk7Sw4cP9+vpBQKBYCB59tln1yilU2b39S2gHz58GKdPn+7X0wsEAsFAQgh53eo+IbkIBALBkCACukAgEAwJIqALBALBkCACukAgEAwJIqALBALBkOAY0Akhf0wIWSGEfNfifkII+R1CyAIh5Cwh5Hb/D1MgEAgETvBk6H8C4F02998PYF7730MAfr/3wxIIBAKBWxwDOqX0cQAbNg95AMAnqcrTALKEkBm/DlAgEAQDpRSfeXYJhUq934ci8Ak/NPRZAIuGn5e027oghDxECDlNCDm9urrqw1MLBAKvLKzk8Ut/dQZfeuFavw9F4BN+BHRicpvp1gxK6SOU0jsopXdMTZlOrgoEgpC4tlMGAOQrjT4ficAv/AjoSwAOGn6eA3DFh98rEAgCZDVXAQCUqyKgDwt+BPTPA/iA1u3yZgDblNKrPvxegUAQICtaQC+KgD40OJpzEUL+EsDbAEwSQpYA/CqAKABQSh8G8CiAdwNYAFAE8JNBHaxAIPCPlR01oJdqIqAPC44BnVL6oMP9FMCHfTsigUAQCqt5LaBXRZfLsCAmRQWCPcqKVhQVGfrwIAK6QLBHWRUa+tAhArpAsEfRu1z2WIbebFJ86L89i29cXO/3ofiOCOgCwR6kVG0gp02I9kNyoZTi4a9dxNXtUujPna/W8YXnr+Gbr9oNwA8mIqALBHuQlVxZ/3c/JJfX14v4v/7xJfzjufCnVIvaIFW5PnxXJiKgCwR7ECa3xKMSSn0I6Jc2igDQFx+ZgtbVM4xSkwjoAsEehA0VXTee6ovkwgJ6vg8tk3qGXmuG/txBIwK6QLAHYS2LB8eTfcnQF/uYoee156wIyUUgEAwDq/kKZIlgNhvva4Ze6IMxWFG7KqiIDF0gEAwDKzsVTKYVpBS5rxp6vi8aOpNcRIYuEAiGgJVcBVMZBYloBPUmRa0RXrZKKcWldTWgF/uioTPJRWToAoFgCFjNVTCdUZCIRQCE27q4XarpPfD98GJnVwUiQxcIBEPBSq6C6ZFWQA8zuC1uqMNEMVnqS1GUnbxEH7pAIBh46o0m1gsVTGXiSETVgB6mjs708+v3pfvahy6KogKBYOBZL1RBKTCVUZDsg+TCAvqN+0f6UhQVk6ICgWBoYFOi0xkFcZahhyi5XNooYjwVw76ROAqVOtSVCuFR0DV0kaELBIIBh/m4TGcUJGPqjpswJZfFjSIOjieRUmQ0afiBVYz+CwSCoYGtnmNti0D4Gfqh8STSivrchZBbF5m8JNoWBQLBwMMkl6m2tsVwgmq90cSVrRIOjiWQUtSrg7ALo+z5qvUmms1w5Z6gEQFdINhjrOQqyCajUORI6G2LV7fLqDcpDo0ndbkn7MKosQA8bFm645JogUAwXKzkyphKKwAQetsiM+U6NJ4ES47D9nMxnkAq9YZ+UhsGRIYuEOwxVrWhIgCttsWQMnTWsqgWRTUNvQ8ZejRCAAxfp4sI6ALBHmMlV8F0Jg4AUGQJhADlkDL0SxtFyBLBzGgcaaU/kkuhUsdYMgZg+DpdREAfYv7+7BX83lcXsFGo9vtQBLsESqluzAUAhBAkopHQBosubRQxO5aAHJH6UhStN5qo1JsYT6kBXWjogoHh448t4KVrOfzOVy7gvW+cw0/fexRHJlP9PixBH9kp11GtNzGtBXRA1dHDaltc1FoWAbQCeog98Oy5JtIiQ98VPPbSMu77jcfw+nqh34ey6ylWG3jz0XH84KkD+PS3lvC9v/lVPPTJ0/jWaxuhT+cJdger2lDRlCGgx8MM6JslzI1pAT0WvobO2jPHU+rrFwG9z9QbFIsbJeTK4XtADBrFah3HptL4jfeewpMffTs+8vbj+OZrG/ixh7+BX/zvz/X78AR9wDhUxEjGIqF0ueTKNWwUqnqGLkckKCE7LrKOmglNcikPmeQycAG9X8MIg0ih0tD/XtOZOH7pHW/AUx/9Xrzn5AwePXdVZOl7kNU883GJ67clYuFk6Mw2lwV0AEgrcqhF0VaGrmnoIkPvL4mQ26wGlUaTolRr6G1pjGRMxu2HxlBrUGyXan06OkG/YBk6a1sEEFpR9JKhB52RUuRQkzN28hgTGfruIKVNlxX7sOlkkGCZCPt7GWGX22wEXLB3WMmVocgSMkrrc5GIRULRkhctAnqYW4tY3JhMiaLoroBlnGEb+gwaLONKKSYBPS0C+l6FDRURQvTbEtFwNPRLG0WMxGWMJqP6bWklEq6G3im57MUMnRDyLkLIy4SQBULIR03uHyWE/B0h5Awh5HlCyE/6f6gqLKD3Y1P5IMG+JGwazwi73F4RAX3PYRwqYiRi4UkuhyaSbbelFDnURdHsde5ZDZ0QEgHwCQD3A7gJwIOEkJs6HvZhAC9QSk8BeBuA3ySExHw+VgDG3lWRodvBPrhJIbkIDKzkKvoVGiMRDUly2Sy2yS0Ak1zC7HJpz9D3ouRyJ4AFSukrlNIqgE8BeKDjMRRAhqjXcWkAGwACeZcUWYJEhIbuBPuSpEyMhzKKDEWW9I4Hwd7B6OPCSIaQoTebFEsbJRwc6wjosUio5lzsuUYTUUhkb0ouswAWDT8vabcZ+TiAGwFcAXAOwC9QSrv+UoSQhwghpwkhp1dXVz0dMCEEyZgc6g7EQYRdxiZNNHRCCKYyisjQh4xnX9/AHzz+iuX95VoD26Va25Qo0JoUDbKNdTlXRrXRxEGTDD3swaJ4VIIckRAP6cokTHgCOjG5rfOdfyeA5wAcAHArgI8TQka6/iNKH6GU3kEpvWNqasrlobZQMwohudjBMpG0iYYOQAT0IeR3H1vAxx590dK7x7jYwkhCk+WCzFYvrXd3uABqH3qhGt5e0UK1rnd+KbK0J90WlwAcNPw8BzUTN/KTAD5LVRYAvArgBn8OsZuUIofq/zCI6Bm6iYYOqJ0uIqAPD+VaA0+/sg4A+MbFddPHmA0VAUAiqoaBIK96zXrQAeh7RcOyHihWGkhqSU48GkGlPlxxhCegfwvAPCHkiFbofB+Az3c85hKA7wMAQsg+AG8AYH3t1yOJaARFMSlqC8vQzfrQAbXThS0LFgw+33ptQ882n1xYM32M2dg/0BrWCzKoLm4UIRHgQDbRdnsqZAvdfKWVoauSy3Bl6I5ui5TSOiHkIwC+CCAC4I8ppc8TQj6k3f8wgF8H8CeEkHNQJZpfoZSaf6p8IKWEZ/c5qLAM3Woby1Q6js1iDdV6EzF54MYRBB08fn4VsYiE7zkyhqcumn/1mDFXl4auBbhSgDLm4mYJM6OJrs+avii60gAygT29TrHassNQJZfhiiNc9rmU0kcBPNpx28OGf18B8A5/D82aZEzGVlF4fNuRrzQQi0iWwZplaeuFCmZGE6aPEQwOj59fw51HxvF9N07j1/7uBSxuFLsKkKu5CiQCTJi0LQJAqRqghr7R3bIItCTBsAqjhWpdX6yhRCNi9H83kFIiQkN3oFit61qhGaIXfXi4tl3Gy8s5vOX6SdxzfBIATLP0lVwFE2kFEam9z0FfQxdghn5po4iD492JQzpks71ipdGSXGRp7w0W7UYSUVlMijpQMHxwzRABfXh4/LzaAvyW66cwP53GVEbB1xe6C6NmQ0WAqiUDwWnopWoDq7mKaYYe9qBgvtJKdESGvktQM3RRFLWjWK2bjv0zpkVAHxq+dmEV+0YUvGFfBoQQ3HNsAk9dXOtqBVzJlbuGioCW5BKUnry42VoM3QnT0MMy6Coa2hZFhr5LSMZkMSnqQKHasGxZBForuISfy2DTaFI8eWENb5mf0g237j4+ibV8FS8v59oeu5qrdBVEAaPk4vydKlUbruURqx50IPz9BgVDUVRtWxQZet9JxiKoNpqoNYbrzfCTQsU+Q1fkCLLJaOgZ+vnlHP7b06+H+pzDzJmlLWyXanjL9a1BPaajP3mhpaM3mhRr+WpXDzrgrm3xo589i3/15992dYxWPehAuAG91miiWm/qdhjx6PB1uQxsQAeCHYQYdAqVum2GDvRnuOivTi/iV//2u2Jbkk88fn4VhAD3akEcAGazCRyZTOEpw4DRRqGKRpN29aADhoDO8X26tFHEy9dyjo8zsrhZRCoW0Q2xjDD5I4w+dHZVn9TbFvfm6P+ug53Vxfi/NcVqw9SYy8hURgndoCtfaaBJMTADHX/5zUu4ul3q92FY8vj5VZyay+obeBj3HJ/AM6+s61ex7MRtJrm02hadg9tOqYaVXBl1F1fHrIXS6MHOiEgE8agUSnJWqLYb1sWjkpBcdgMiQ3dGLYo6ZOh98HNhl9aDUNTeKdfwbz97Dn99eqnfh2LKdrGG5xa32uQWxj3HJlGoNnBmcQsA9Klgs6JoNCJBlgiX5JIr19GkcJUIXDLpiTcS1l5RfYuXYpwUDdaULGwGNKCLNXROGBdEWzGtBfQwP9B6QB8A6wZ2jLvVZvjJhTU0KfDW6ye77rvr2AQIgd6+yIrfU+luDR3gX3KxU1b30F7d5rONoJRaDhUxwnJc1O0wWNuiLKFJgVrDn8//er6Cr768go8/dgEPffI07v/PT2BhJe/L7+aFa1J0t5EKeA1dudbAWr6CuTHrD+FuxmpBdCdTGQWlWgP5Sh2ZeNT2sX6R1wP67j8Zs2Nc26UB/fHzq8jEZZyay3bdl03GcMuBUXx9YQ2/8P3zLcnFJEMH+JZcVOtNXSq7ulUGDjkf42q+gnKtaR/QY2EF9HbDOtZ/X643PNtfXFjO4be+fB5nl7Zxeaslzc1mE7i8VcKZxS0cn073eOT8DGRATwasof/5M5fwW196Gd/5d+8YSJ8TuwXRRozDRWEFdHYSHoT6BwsAa7ndZzNBKcXXzq/ivvlJyBHzz+jdxyfwx0++ikKlrr3Hsh7EOuFZcpHTsnMA3HUFs8XQnYQlubDpct0+V/tbVGpNwPzCxZYLyzm875Gn0aAU981P4YN3X4cTs1ncPDuCRoPitl//MrZLNedf5CODF60QvIa+vFNGodrYtZfaTujr52zaFoHW5XeYOjrLenm/wJV6A+//o2dwbmk7yMMyhZ18dmOGfmElj2s7Zbxl3nqvwL3HJ1FrUHzztQ2s5MqmHS6MuLbkwo5cufWeXeOUXJY21cA/N2btF5RSwtla1NLQW5IL4G2gamEljwf/4BlIEsFnf+5u/O6Dt+GhtxzDXccmMBKPYiShJkgioHOgB/SAPgQs2AzqFCXLLNMcRVEgXI2Y/W15T8ZXt8p44sIaHr/gbcNVL7DP1248sRvH/a2447pxxCISnlpYsxwqYiRizpLLjjFD33Eb0HeTht4uubj1RH91rYCf+IOnAVD85c++CUenuiWViESQUWQR0Hlgl0xBaeh5LRNZ4fzQ7jbsFkQb6Yefi9uiKDsBXNkKv3WQfb5y5fqu61f+2vlVzE+nu/zFjSRiEdx+XRZfX1jHSq5iOlTE4JNc1L9HTJZwlfP9WNosYjIds7RxBjQNPQQJrrX0RWtb1DN0/tbF19cLePCRp1FvUvzFz74Zx6etPX9HElHsiIDuTCJgyWW3dzc4wY7fqQ89m4giGiGhBfRmk+rvGW9AL/QxoBs/X+sWa936QanawDOvbthm54x7j0/ihas7uLJVspVcEtGIYx86C07z02lXksusQ3OBmqEHf8LMWxRFeTP0xY0iHnzkaZTrDfz5z7wJ1++zN3AfTUSxJQK6M4osISKRwApruQrL0AczoLc0dPsMXZIIJtNKaH4uxiyM1/6Y/Te8bXJ+YjzprO0i+e2ZV9dRrTe5Avrd2gRprUEdJBeZW0O/fl8Gy7kKGk3ndr+lzZKtfg6oBl1h7BUtVhtIRCO6fbDiIkO/vFXCg3/wNArVBv7sp9+EG2e6ViZ3kU1GheTCAyEEyVhwhZRBz9DznBk6EO5wkfH94j0ZMxe+y/2QXAzHu5sKo187vwpFlvCmI+OOjz05O4qMdmK3alkE1L2ijhm6pqFfvy+DRpM6fm6aTYrLHAE9pcigNPhBwU5/IzcZ+n/52kWs5ir4s59+E26ZHeV6vtGECOjcJGPOl4heKQx8hq5dWjpk6EC4fi7GzhbekzF7L3LlelvbXBgYTzq7KaB/+9IWbj80ZtmCaESOSHjT0QkA1kNFgCpDOGXoO+U6CIHeV+3Uuriar6DaaDrOc4Rl0FXscCDV+9A5MvSNQhWz2QROzPEFc0AEdFcEWUhpdbkMZlGUBcu0Q1EUCNfPpdAW0N1p6ED4skuhWtevctbyu0dDX94um27/seK+eVV2mclaB/Q4p4aeVmTMaoVYJx19SfNBd5ZcwjHoylfqbcN2btoW1eze3djOaCKK7WIt1EnsgRwsAtQe66Au0Qa9bdFpQbSRqYyC9byqh3auJvMbY3Dmfe+MX/IrWyXHQpSfFCsNjKVikEht13wWGk2K1XwF+0b4J2Hed+dBzGYTOGbSXsdIRFVL6nqjaTmolCvXMRKPYmZUfW6nEyxrWTzIIbkAwU8Pd/obucnQ1f0Czt8nI6PJKKoNdbqW57voBwOboSejwfSu1rU3gBD1knEQjXsKVfsF0UamMwqaVL2kDBoWnGWJcGdjhbaAHm6Gnq+oC4UnM8qukVzWC+rJ167A2YkiR/D9N+2zfUySwxN9p1xDJi4jm4xCkSVHyYUF9Nmsg+QSsJUHo9PfKB5Vvx88GjqP2V0no30YLhrcgK44T7Z5gWUJs9kEag2KrWK4GpgfFCr2C6KNsFa2lRDkJfaFncooroqi2WQUEuEfN/eLopaVTaZjuyags7rOtIsMnYc4T0Av1TCSiIIQggPZBEeG7tyDDoSpodfbGgUUmT9DL3KY3XUiAroLgjL0yVXUP/6RyRSAwVzR5rQg2kiYw0WsY2U6o7gqio4motg3Eg+906WgZWWTaWXXaOjL2rCbG8mFhySHJ7oquaifq/0jcQ4N3bkHHWgF9KA19EKlvSjqRkPPV+pcXWNGREB3Ac9kmxdYoDmqBfTdop26wWlBtJEw/VzYCXh6JM59eV2oqEt9D2QTqsNfiBQrLEPfPZLLspah77NpQfQCzxq6nXINI5qJ28xonEtDdyqIAq2iaNAaeqHjeyFJBDGZb8lFZ4cMDyygbxXDSwZEQO+AZQmtDH3wOl2cFkQbmcyom27C6HQpVNS2t8k0f4bOdGw1gPQhQ4+pGfpWsbYrdtgu75T1v6Gf8GwtypXryLAMfTSO5Z2y5XARbw860DLLClxyMZFN4rLzXlFKKQrVOtKcSRIjm1C/WyJD5yCpyIFMiuoBXesIGETJpeiwINpIMiYjrcghSS5qgEwrEe73jmVVs9kErmyXQ1/GkVJk/aS3vgtkl5VcBRMpBVGLThSvOO0VpZQiV67pLoIz2QTqTYp1i0SAtwcdaI3iBym5VOtNVBvNLtlEiUYci6KlWgOU8s11GBGSiwtSsQhqDYqqzzsBWZawb0RBMhYZSMnFTYYOtDYXBQ2b1EvGZBSrDTQ5RsdZZ8LMaBzVejNUT5VCtYGkEtGz4d0gu6zslH2XWwBDhm6RrRaq6i5YlqHPjNi3LvL2oAOqM2EiGgk0Q28Zc3Vk6FFJ9UO3QXdpdKmhZ+IyCEGoBl0DG9AT2hvj97Qoc1pMKzKmMuH5nPhJwWUBZzK0gK4GZ6aZFjmLUWlFxow2zBKWSVet0US13tQlF2B3WEEs58quWhZ5cdoxwIIS09D3673o5u8Hbw86I6UE67ioL7fouHKNyxGUHTL0zl2kvEiahW6YBl0DG9CD6l1ll31pRdYy18HT0IvVuqvLw7D8XFhwZi2VRY6MjMkes3pAD+f9aFkQRzDFMvRdcHJf3nE3VMRL3CFDZ8ZcbLMVs+21ztD5etAZ6YCXXLDPWmdQVqKSY9tip0ujG7LJmJBceAhqDV3B8MYPbobecFxuYSQsPxfWsZLi1EyZ3S6TXIDwMnRjVsY09H63LtYbTazlK773oAMtDd2qQMiMuUYS6ns3lowiJkuWrYu8PeiMoJdcdK6fY8Rl58UeRYvsnoew/Vy4Ajoh5F2EkJcJIQuEkI9aPOZthJDnCCHPE0K+5u9hdsP6Zv3udMlX6lBkCdGIhOlMHKsDZtDFuyDayFRGQa5SD8zsjJHXsu2UfjK2fz529ZVWIhhPxbimE9nz9NoqZtxuk4zJSMYifdfQ1/JVUOp/yyLgLLkwYzSWoRNCbFsXeXvQGamA94oW9Sy7Q3KJRhzbFjuXS7th1wV0QkgEwCcA3A/gJgAPEkJu6nhMFsDvAfhBSunNAH7M/0NtJ6m3Ovkf0Fl2G1ag8xN2ycw7WAS0houCDlis9UuXyxy+wMagyqYTeSSX//1z5/Azf3q6t2PtsCDeDb3orIV2n83mIa/EZfsul52S+vdgg0WAOlxkp6HzFEQZ6YA19LyV5MLRtqib3bnU0IFdGNAB3AlggVL6CqW0CuBTAB7oeMxPAPgspfQSAFBKV/w9zG5YwPJbcslX6kjHWwEdGKzhIj0TcXF5OK2P/wcc0LWiKJPLnL7AxnoGABzIxnGFI0N/9vVNvLpW6O1YO7oidsP4f2uoyP+ALkkE8ahko6G3Z+gALMf/3fSgM4LeWtSSTTq7XJwll9Znwb3kMqI5LoYFT0CfBbBo+HlJu83I9QDGCCFfJYQ8Swj5gF8HaIU+jOBz9sx0XqAV6Fbzg1MYbS23cJ+hB33iYlc/ac6rq0LHa5kZTThq6DvlGpY2S9goVnsaBCpW2nXTqYyCtVx/NXQ29m+3qKIX7NbQ7ehFUUOGrg0XdbafuulBZ6RikUAlFxaUu/vQnSdFrQqqPLCtRWHNT/AEdDNP1c6jkwG8EcB7ALwTwP9JCLm+6xcR8hAh5DQh5PTqam9b3Ftti/5+CHLldskFGKxFF1aZiB2tgB7ciUtvA9Q0acD56qrQ8UU6kE1gJVexDdQvX8sBACjtbRCoO0MPzzfeipWdMiQCTKRigfz+RNTa8G6nXENMltqWasyMxlFr0K7ZADc96Iygi6LsBN3Z/aVm6A4aeo9F0bphl27Q8AT0JQAHDT/PAbhi8pgvUEoLlNI1AI8DONX5iyilj1BK76CU3jE15bwP0Y6WDutzhl5tSS5sS3q/v8hu4F0QbWQipUAiwWboRYMe3upysX/vuiSX0TgotV+s8NLVHf3fvbwe9gVkzz2ZVrBZrKLex/H/5Z0KJtOKpV95ryRstoDtlOp6Dzpj/4h5L7rbHnRA/VzwDpt5gX2WEh1bnhRZQsVRQ69DlghiHv7uYU+L8hzhtwDME0KOEEJiAN4H4PMdj/lbAPcRQmRCSBLAmwC86O+htsOb5bnF6Jk8noohIpGBzNDd9KFHJILxVLAZaN7QscLbh65fJmuPd+p9BoAXrub0f/fiw1PoqEVMZhTQkHzjrVjOlQPRzxmJmHWGnivX2gqigPX74bYHHYAuw/EMm3mhWK23LYhm8HS5MBtlQtwvgGkZdO2SgE4prQP4CIAvQg3Sn6aUPk8I+RAh5EPaY14E8AUAZwF8E8AfUkq/G9xhAzFZgiwR3y9ljJJLRCKYSMUGyqDLSit0IujhIqN8EtWWbzjVP/Id3QUHss696C9d28GhcTWQ9PJ62JUfa4+dSodnYmaFOlQUjH4OqEtjrBKknXIdmURHhq7NBnReMbntQQeC90QvVM39zOOyuqnJymSMHZOXDhdgd2booJQ+Sim9nlJ6jFL6Me22hymlDxse858opTdRSm+hlP52QMfbRhCOi+qb1/ogTo+Et0QZANbzFXzm2SXP/30rs3T3AQzaz6WzbSwVczbo6tTQZ0a1aVGLTpdmk+Llazncq+3Q7E1yUecRmLzR8nPpX4a+misHMlTEiMciKFnoyTul7gx9PBlDLCJ1vR9ue9CB4PeKWhnWKRxbiwouJ6+N7MqAvlvxu5BSbzRRqjWQVlqZyFQ63GnRz377Mn7pr854bpHzaiQUVobOvrg8gyTMbpe1i6UUGaOJqGWGfmmjiGK1gVNzo8gmoz29b2y5BWOyz+P/tUYTa/lqID3ojGQ0YtlkkDN4oTMkiWD/aPeiC7c96ECrkymoDD1fMTesi2tLLuwMutSFMd52grKAHpZB10AH9EQs4qvmZlbNns7Ew83QNY3W6xndylXOiamMYrlDtdmk+PILy1ybXazobEFMxWS9UGoFs9s1apd2iy5euqYWRG/YP9KznYHqnd36HEyGNHxlBXstQbUsAvYa+k65ro/9G9nfMS3qpQcdaNUqAsvQq+aGdfqiaJsM3cs+UcZoUmTo3KhBwb8PQGcWCWj9x/mKrcbmJ2xk3esZ3c2CaCNTacVyh+rfnrmMn/3kaXz5hWVPxwR06+FJJeI4WFQwuUw+MBrHFYui6AtXc5AIcP2+jCqV9RB82cmEkYpFEI9KfQvordVzwQX0eDSCUtU8U82Va21DRYyZjgzdSw86EPzWIisNnUkudq2LVtk9D+mYDIkAW6VwpLqBDujJWMTXwSK9Tc6gFU6PKGiG2N3AnocNcril6GJBtBG9F70jYJVrDfw/Xzyv3tdTkbG9Y4VnJ2znlnYA2vi/ueTy0tUdHJ5MIaE5JPZSzGadDQxCSF93i7Ip0ekgJZeYueRSrTdRrjW7NHQAuuTC2g299KADrTpJEEtrAGsNPa4vinbK0L1JLpJEQh3/H+iAnlJkX31WzPweWmPx4XS6sAy5lwzdzZQow2pa9JPfeE1fztyL4VVXUVRxLmjnTboLZrJxbJdqpieDl67lcOP+EQCtmoDXCb1ODR3or5+L7uMSZNuiNljU+TczG/tnHBhNoNpoYkP7bHjpQQeCL4oWKnVzDV2TXOxaFzuXS7tFDejBrtdjDHRAT8ScL9vdwJZbZDokFyC8VXSbTHIpewzolbonz4lpk4C+Vazi448t4G1vmMJYMorNHnpp2XAG27SeijmbMRU6ZA8Aui965zBLrlzDpY0ibpzJaK8njnKt6TlAsAXRRiZDshk2Y2WnorfRBkUiFkGTdgc3drVopaEDrdZFLz3oQEhtiybfi5bkYtPlUnG/T9SIyNA5ScUijoU1N3S2yQGGadGwA7rHM7qVVuiEWYb+iX9aQK5Sx0fvvwFjyZiehXk6Ls06lxU4kxwLDZjdrhG9dbGjMHp+WR0ousGQoQPeT8RmGfpUJtZHyUXdVCRJ7odbeGFTlJ3BTc/QFXMNHWgNF3npQQda/f5O08NesSpsKg6SS8uO2nuGrhp0CQ3dkSRHlueGnEVRFAgnoFNK9SzYa4buZkG0kbQiIx6V9Ev7xY0i/vSp1/He2+dww/4RZJPRHiWX9qUbPC2nZpvWrYaL2ITojQfaA7rX983s6mAyrWCjEF6B3MhyrhLI6jkj+qLojuCmW+cmzAJ6+xWTlx50QNWak7Fg9opW603UGtR8sEjvQzeXXHQ7apGhB09S857wy8nMrMslHo0gE5dDCeg75boeLHrR0L1kE4SQtl703/zSyyAE+F/eoXqsjadi2Cz0JrkYvxSpmIxKvWnrjWJWFN03Egch6Op0eenqDjJxGQe0jNFMQnJ1vNqCaCOTabVAvhlStmVkZSfYoSLAeslFS0Pv/lxNpGKIRoghQ3ffssgIyqCraGN/G7e4KtH/2x6cFhnMcTEMBjqgpxQZ9SZF1SfDJDPJBVCDQxhFUWMG7LnLxaLflocpzVHwu5e38TfPXcFP3XtEz8CyyVhPgaxQbS9KsS+XXZeSWVE0GpGwLxPvytBZQZRJOr1ILsYF0UZa06Lh6+jLO+VAWxYBw17RjvektX6uO0OXJIJ9I61OFy896Ix0QFuL7CylFYfBIi921J2MJqLYKddDsdAd6ICuZxQ+6W65Sh0xubuHO6wlysaio+cMveJ9THkqo2Blp4L/8OiLGEtG8XNvO6bfpxZFe+tyMQbntEObmtFut5OZbPumnGaT4qWrO3pBFFC/RLGI5Ol9My6INjKp+bmE7YteqTewWawFOiUKtF5vp+SSM/FCN6Kuoit57kFn8HQ+eaFlWGedoVuN/lt9Ftwwmoii0aSB+r0zhiOg+zQtamXCM52Jh9Llsqn1oKcVuYcuF+9jytOZOBZW83jq4jp+/nvn20a9x1IxlGtNz22inZKLvrXI4mRsdbUEoGsV3dJmCYVqAzfMjOi3dUpIbihWu6U3oH/ToisBbioykrDK0Es1EKIOyZgxM6puLvLag85IxYLJ0O0+Sy3JxTxDN5Nh3RKm4+KAB3Qty/PpQ5Avmwd0lrkGfcnEMuBD40lPGTqryHvV+6Y0i9hD40n88zdf13bfWDLWdoxu6dTDnfaKtrzQu09OB0ZVyYW9Hy9oHug3GgI6oAZgL1KZlcFZvySXlRDG/gGD5NJZFNW+F1YdNmxZ9OKGtx50RjowDd16zy6TXKw0dH3RiQ8BPQwdfaADut9r6PImRThA1dBLtYbv6+46YVOihyeTnjR0LwuijbCFBf/mnW/okp3GNE8KrwG9U3LR+44tJBfjguhODmQTqNSb+t/rpWs7IAS4fl+67XFe/VysDM5G4rIq44SeoQc/VAQYJBcTDb3TmMvI/tE4qvUmzl3eBuC+B50RVFGUJQdmskk0IiEiEUsvF69md0ZGE2oyFIZBl/fTzi4gEfV3XNhqgKC1iq6M9FS6636/2CrWIBF1eOYrJfd7tr0siDbynpMzGEnIeOfN+7vuYxm6l8tGSqneh87Ql3xbSC5WW9oBY6tcGRNpBS9dzeHwRKqru2d6RMFzi5uuj7dz/RxDHf+Pha6h67tE+9S2mCvXLfVzoNWL/q3XNjz1oDNSSiSQPvRi1b5TJS5LlkXRosjQwyOl+FsUNeuqAMIbLtosVjGWjCGbjKFSb7p2NyzYXFrykFJkvOuWGdPNLGPahKIXT5tKvYl6k7b9bZP61ZVVhm6tXbJpUWZJ8OK19oIoYyqtYL3gfm1c54JoI5OZ8Mf/l3MVRCNEP6kGRVJPkLo1dLMOFwY7wT5/ZcdTDzqDx9/HCwWb9xPQ9opaZOi6qVwvXS4hOi4OdEBnGZRfw0WdWSSDaZdBF0Y3i1Vkk1HdBCnnUnYp2Fxa9ko2yQo77gO62Z5TJ3e9TrtdIzPacNHVrRIKlTpeXy/qE6JGWE2gc4mx4/HaZHT98HNRp0TjgU6JAkA8Zq4n58p1U2MuBsvQG03quSAKaN5MtYbvg1t6hm4RlBVZsiyKsqter1cdgKEoKgK6PeyM65dBV65ifmk5lQ5nWnSzUMNYMqZnQ247XfyoyFvBssMND8NFZnp4a4jFqShqPswSkyVc3S7jpWvahOhMd0D3OlzU0k3NAnqsL10uQRdEASAWkSAR9xr6RFqBrJ1segnoTq2sXmFZdueCaIbdXlGvdtRGUrEIZImIDN0JdonoV7HSbNwbULPTaISEkqGPpWL6l8dtEcXLgmheohEJGUX2VBQ1C87s6sqqTa3TbtcIIQQHRuO4vFUyLLUwkVw8OmW2dFMTySWtYD1fDWw7vRkruXLgPeiA+ndNxmRTycVOQ49ow0UAPPegA0aDLn919KJmWGd1haNEI9ZdLh6tNIwQEp6F7kAH9IQ+WNT7Gb3RpChWG21e6AxCSM/+2jyoGnpUd7Vz2+nidUE0L9mUNz8XMwkjIhEkotaDJK3tUfa9zy9dzSGjyKaZoVc/l84F0UYm0wrqTWr55fzyC8v4/a9edPV8TgS9HNpIPNq+taipDcTYaehAS3bpTXIJZmuRk2GdKrlYty32YszFEAGdg5gsIRZx3h7PQ8FimIQxNRLsKjpmzNVThl4JLkMH1KXAGx66XKw6VlKKtRlTvsNutxO26OLFqzu4YSZjWsj1GtA7F0QbsRsuopTiPz76Iv7zV877lsGXaw1sl2qB+7gwOpdcFKp1NKn1lCiD2eh67UEHjHUVfwO6kx1GPCpZSi6dqwi9ojouioDuSMJiy4pb7KbJAFWPDTKgF6sNVOvN3jT0oDP0ZKynomjnyTJp09XQabfbyYFsHMs7qoZuVhAFVGvU0YT7ZdGqy6P554CN/5v1op9d2sYrawWUa029A6dXVvRNReFk6ImODJ0V5u00dEA9wRr/3wt6k4PfGbrFcgtGPBpBxSZD78WYiyEydE5SPq2hY8stLDP0gAM606bHklFDhu61yyWYDN2rn4uVHp5SZMv3zqqFlHEgm0CTqo+7waRlkeHlfStUup0W9d+nT4t2/x0+953L+r8vrORcPacVyyFsKjKSiLXLYHbGXEbe/+br8Ns/fmtPn72gthYVHLJsuy4Xq7qaW8JyXBz4gJ5UZF+q4nZdFYCaIa0Xqqj55OzYCbOmHUvGEI9KiEaIhwy994q8HWMeLXQ7F0QzUrGI5XvnVIximi1g3uHC8HJlZfcl1sf/O35nrdHE3525gnuOTwAALiznXT2nFcshTYkyEh0FQidjLsbB8SR+6LbZnp475TCb4BWr5RYMuz70zt2yXhEZOifJmD8ObWYLoo0wPXY9oI01eoaeioEQgpF41IOG7m1BNC9jyRjylTqqNvsXzbCSs5KKbDkZaOaFboQNFxECvGGffYbuVnKx+xKPJqKQJdKloT95YQ3rhSo+eNdhTGcUXFjxK6AzY66QJJdYu+TCPoNOkosftDJ0f7tcnPbsxuWI5aSonfzmBtVCtxZ4d9RwBHQfPgB2gyxAa1o0qE6XluSiarQjmoeyG7wuiOaFTYtulVwO6mi2xNGOIqO6QtC6KGr3RZrRAvp140nbwM/8XNwYq9npppJEMGHSi/6571xGNhnF294wjfl9ad8C+kqujJgs6cMpQdMpufBm6H7A/uZ+me0xWNuiFfGoZOvl4keSNJqIglL3w4JuGfiAzrNsmAcrWYDR8nMJRkdn1rnMBGskLnvoQ/e2IJoX3aDLpexiFZxTSnfPM8NJu0wrMkYTUcuCKGN6xL2xWrFif2JUp0VbJ7V8pY4vvXANP3ByBjFZwvx0BgvLOV/cOVe0lkWr4rDfJKIRlD1o6H49NyH+F0XNdtMacexD96ltEQh+/H+gzbkA9bLdj0nRvPbBtZJc9KnDgKYE2XIL9saPaJdobshXGoG1LALeLXSLVfOiVCoWsR0scuou+E/vPYlDE/aDLF6M1fIO0lXn+P8XvnsN5VoTP6xpyMen0yhUG7i6Xe6p6wNojf2HRTIWadsvEGaGLkkEyai/Bl2UUkcdPC6rbYuU0rYTZ73RRKXe9K0PHQg+oA98hp6MRnzJ0FuDLOZvPCuGBZahF6uqPqvJEl41dDO3SL9oOS66C+h5iyzHrqCd53gt77h5v2OGPpV2b6ym9i07ZOiG3/e57yzh0HgStx8aAwDMT6snDj9klzBWzxlJRCNtCdJOqQZFlqDIwX2ujPhtoVttqMZwThk6pehaZekUE9wgAjonScUfDT1XriMWsf7gxmQJY8koVvNBaeg1XdIAgJGE7N6cy+OCaF7GUurxufVzsdoElVZk1Bq0q8hKKXWc7uOFeaC4ubIyWxBtZDITw1q+Ckoprm2X8dTFdfzQbbN6djevFWkvLPfeuriyUwk1Q2e+Jqx4t1OuIxNCQZSRVmTkfexyKXL4mbeWXLR/Dp1sd93AHBfd1p/cMvABnWnoveqVPJ4N05l4YBn6VrGKrMEedSTuXnLpZUE0D14lFyv5RF8U3ZGRVepNNByyKl6mXF5ZWS2I7vyd1UYTO+U6Pn/mMiiFLrcAwHgqholUDAs9ZujFah25Sj20lkWge6/oTrmmW1GEgd8Zur7cwqFtEejeK6pbQPjUtgjskgydEPIuQsjLhJAFQshHbR73PYSQBiHkvf4doj2JWARNCsvRXV7ylbqlfs7w0gLHy0ahivGUIaAnoijXmpbLa80oBKyhx6MRJKIRT5KLaVHUwv7YaSbADcxYjTdDLzp4yAAtXX4tX8Fnv30Ztx7M4shkqu0xx6fTPQf0lZBbFoHuJRe5kDN0OzsIL9itn2PoAb0jQ/fTvTSrbS3qe0AnhEQAfALA/QBuAvAgIeQmi8f93wC+6PdB2pHSbVh7k12sdF4jQY7/bxVruuc4AE+e6EFn6IDa6eJecrEoiup2qZ2ZkX+XusxYjfd9M/Nu74TVU76+sIaXruXwI7d3D9Sw1sVerhzDHioCuhdF75Rqtl7ofqPuFfWvKNoyhuORXDo+hxabq7wQj6q+U30P6ADuBLBAKX2FUloF8CkAD5g87ucBfAaA+91pPdDaHt/bWd1K5zUyNeK+p5mXjUIV48n2DB3gN+hqNlk1P9gvnxc/F0vJxcJdz25BtBfcXFnxrBxjAf2PnnwVskTwnhMzXY+Zn85gu1TrqStqObcbMnR7L3S/Ue0gAtDQuSSXDg3dYdORGwghaufaLgjoswAWDT8vabfpEEJmAfwwgIftfhEh5CFCyGlCyOnV1VW3x2pKp+bnFS7JhWmnLj1WnCjXGijVGvrgDtCazOMdLmKtZkEstzAynoq50tDVAqe95NJZ1LZbEO0FN34uPEuBmUHX6+tFvPX6KUykuwPuca3TZaEHCwC2HHoqxKJoV4ZeroeqodsZtnnBbkE0Ix61z9D9+hyOJmRPO3ndwBPQzSYaOlPU3wbwK5RS26hKKX2EUnoHpfSOqakpzkO0J+WTQ5vT8AEA3cLU72lR9iZnO7pcAP4MvdcF0bxkk1G9Z56HUq2BJjX/Ulh5d/gpuQBqQFzlfM94LrPHkjFEtGUJP2witwD+tC4u75QRj0qhSh6JDglTXW4RZpeL9WyCF5zWzwGtDL2zy8Vuc5UXwvBz4TnSJQAHDT/PAbjS8Zg7AHxKa9uaBPBuQkidUvo3fhykHUm/NPRyHRmHAHJUK3ydWdrWW9P8gC1eHk+aZeh8H4BeF0TzMpZ0l6FbeaEDhgw9wKIooGbobFm0mce5kaLDxDCgDsCMp2IoVxv4/hv3WT7nSFzuyXVRXWwRD21KFGhl6OVaA5V6A5V6M9QTSkqRUa41ud4rHpwWpQDWGrrd5iovZJMxvS4SFDx/sW8BmCeEHCGExAC8D8DnjQ+glB6hlB6mlB4G8NcA/lUYwRxoZVK9BnSeycSbD4xgNpvAo+eu9vRcnTBNOmuqofNlK0EuiDYylophu1TjXuRb0ANk93G1NPTgiqJAa1n0Bsey6ALnl/je45P4wN3X6dldJ4QQzO/L9OS6uLwTzuo5I8bvU2tKNNw+dMC/tZLFinNR1EpDt9tc5YUwMnTHgE4prQP4CNTulRcBfJpS+jwh5EOEkA8FenQcsC9eLxa6zSbfIAshBO8+sR9PXFj19Y3Z0AL6uKmGzim5cGQifjCWVE2GeF+/nemZvhTYqijq09XGtL5b1FlH573M/n9//Fb8m3feYPuY+R5bF1dz4SyHNqJr6LVWQA+7Dx3wz8+lUKmDENVR0Qp2n5mGHo+ab67ywmgIW4u4jpRS+iil9HpK6TFK6ce02x6mlHYVQSml/4JS+td+H6gVLQ3d+xmdZWVOkgsAvPvEDGoNiv/vhWXPz9cJ06SNk6K6J7rbwBl4QHc3XGQnn8RlzYypq23Rv+4CwN0qOj8vs49Pp7FeqGLdQ6cLpTR0HxcAiMfUkFCq1vXPXkYJt8sF6C1BM1KoNpCMWi+IBgCFFUW7Bov8MeZijCSiyFXq3Fe3Xhj4SVE/MnQ3XRW3HsziwGjcV9llq9Atueie6NwaerDr5xisE2eTQ74A7E80zIypMxsr2Oz09AKbFuUJ6H5eZrM6i5csfbtUQ6HawIFsfySX9gw93KIo4N4T3aqV2Gm5BdDK0DsHi4oOFhBuybpsRfbC4Af0aO9F0XzF3mnRCCEE95+YwRMX1lyP5luxUawirchdm4bUvlXOtsWAF0QzdAtdzktHu6Iou73zZMwzE+AG3XGRo9Ol6ONldi+dLosb6k7SuTF7N0m/abUtNvXPdxhOiwwve0X/7swV3PUfHzO9EnJalAJYZ+g8w4ZuCGP8f+ADuqytXOtlGCFvU7gz490nZlBtNH2TXTqnRBkjcXn3ZeguJZeCQ9dIymQykKdA7YZ4NIKRuMyXoTs4LbphZjSOVCziKUNf2iwCAA6O92a/65aIRBCTJRRrdeRC9EJneNkr+k8vr+DaThn/5fFXuu7j2RFgZ87l5+dQBHROUj1uLWp5NvB9cG87mMWMj7JLp48Lw81kGbtCCXpS1LvkYv6lSsa6JZc8R1bllqmMwjW16deGGkC9mju+L+OpdXFRC+hhZ+hAa8kFuzoMu20RcJehn13aBgB88huvdV2F8WTZhBAosmRqzuVn11jLcVEEdFuSMevNNzwwrZC3CCdJBPffMoPHz6/pWUwvdDotMlQNne+Dna/UEY2QwBZEM1KxCKIR4lpysTrRmG2cKgTg687rlOl3IWx+Ou2pdXFxo4SRuBza6jkjbE9vrlwDIcHPNhjRh804A3q+UsfF1Tx+5LZZ1BoUv/dPF9vu59XB49HuvaJ+S38iQ+ckabM9ngcvrmrvObkf1UYTX3mxd+uaTi90xkiCfw1d0WeZwgpCiCs/l0KljkQ0ok9WdpJSupd82+309Apvhu7XlnfG/HQaK7mK63a1pc0iDo6Hn50D2pKLWkP1Qldk2w4Rv3G7KPq7l7dBKfA/njqA994+h7945hKubpf0+3nlu3hUMhks8tcbKSsCOh9JRe5pEMHLZOJtB8ewfySOf/BBdtksVHVt2oi7LpdgF0QbGU/GuIZ0AOfgnFRkU3OuQAI6r4bu43PP79M8XVbdyS6LmyUc7IPcAqjZarnWwE453LF/QD2ZSIS/a+2cJrecmBvFz3/fcVBQfPyxBf3+YrXBVVdS5O69oupnwb+Tu1vDPS8MR0CPRlDqqSjqvodbkgjuP7EfXzu/2pPsUms0kavUzQO6C0/0oBdEG8kmo9wmQ/lKw1Y+Mat/FCp134aKGNMZBcVqw7HY5rQg2i3z02x7Eb/sQinF0mYRc2PhFkQZTHLZKdVDLYgC6hVgKtZ9krfizNIWZrMJTKYVzI0l8ePfcxCfPr2IxQ21BpGv1Lmy7HhUMnVb9Ls4r8iSa7dSNwxFQFdN8XvL0KMRole7eXn3iRlU60089pJ32WVTnxI173IB+DzRg15uYcSN46LTJa+ZXSpPq5lbeIeLnBZEu2U2m0A8KrlqXVzLV1GuNfsnucQiWh96LdSWRUZKkbnbdc9d3sbJuVH954+8fR6EEPzuYxf0BdE8WTa7KmFU601UG03fu8aCHv8fioCuFkV709BTiuzaBOmNh8awb0TBP5z1Lru0nBbNM3SA7xItjOUWjKwLgy4n+SSl2aWywZCW3a6/r4U3oDstiHaLJBEcn067CuiLfWpZZLBF0Tvleqhe6Iz5fWmcWdpyfNxWsYrX14s4OZfVb9s/Gsc/e9MhfObbl3F+Oc+9ylCRpba2RX1i2OcrRRHQOTArrLnBakWaE6zb5avnVz1bfrL2PysNHeDzRA+i1c+KMU1y4Vn04dQpkFTaVwgWqw1QC7vdXmAj9E7DRU4Lor0wP53BgouF0Uwu6EfLItCeoYfZssi4b34SCyv5tuKmGaxd0ZihA8DPve0YohGC//DoiwD4unTU5ditGNJyafT3s5BNioDuSCLaW9tivuy9PYnJLl950duQEct0x8wkFxee6GFm6OOpGOpNihzHScxJckl39B0H5UnDk6GzBdF+6/fHp9O4sl3mrrUsbbIp0T5n6KVa6Bo6ANxzfBIA8OSFNdvHnbusBvRbZtsD+nQmjg/cdRhfO68u0eGpLalF0VaGHtTnUM3Q/V2QY2QoAnpKiaBQrXteDddLZ8Md141hOqN4HjJqGXN1Z+gZF46LYWroTB7iGS5yKoomO8zV/PZCZ2QTUcgSsQ3o+nCWz8/NLAAurha4Hr+0WcREKhb4kJgVCa0omq/U+6Kh37h/BBOpGL6+YB/Qzyxu4chkyrRX/1++5ageyLkkl6jUNvpv5xLaCyOJKLZFUdSeZEwGpd1+xrz0kqGrsst+fPXlVU+Wnxs8kgvHGT3MDN2Nn4vToA47ZlYY9Xv9HEOSiONuUZ4F0V5gJl0XOGWXxY0S5vpUEAXUDD1fqaNJ0RcNXZII7jk+iScX1m2TtM6CqJGJtIKfvOcwAL7PUlxuHyxqTV6LomjosD+6Vw9lrxo6490nZlCpN/FPL7vvdtkqVhGPSvrqLyO65OKQoYe1IJqhj/87ZBqNJkWpZq/td9qltlpI/T85OfWi8yyI9sLBsQRissTt6bK0WcTBPsktQHsQ60eGDgD3zk9iLV/BS9fMT4IruTKubpdxYtY8oAPAh956DL/wffP4nsNjjs+nti22MnQvrcw8ZBMxFKoN1Brekk8nhiqge9XRC5VGTwH9jdeNIRohuqbnBnVKtDs7B9RMSZacPdHZgugggqAZY5ySC8u67f62qQ67VC9Tu7xMpe0DOs+CaC/IEQlHJ1NcnS6NJsXlrVLfCqIA2rYw9UNDB9SNUAAsZRc2UHTqYNbyd2TiUfzr/+F6zj70iGmXi/8aOn9dzAtDEdB1Qx+PrYu9TibKEQnXTaTwKqdGasRqShTQPNETztOixYCyCSt4JReewpK+8owVRQP6IgHA9IiD5BJQqxqgyi48Jl3LO2XUGrRvLYsA2q4W+5WhH8gmcHQqhScsCqNnlrYhEXUtpB+obYtGDT2Yk3vQBl1DEdA7N5W7QV0/13vf85HJFF5d8xDQi1XTDhfGSNx5yCKsBdGMkXgUEoHjxJuT0yLQvUMyqKIooGbo64UK6haXuzwLor0yP53G4kbJcV6Cdbj0a+wfaJdc+qGhM+47PolnXl03nZQ+u7SF+emMbyffeDSCepPqn40gu1yA4PxchiKg69vjPUyLFmtq3zPPcgs7jk6m8Pp60fV6qc1izXSoiMGToYe1IJohSapBl5Ofi5MXOtBd/whyld7cWBKUApe3zPubeRdEe+GG/Wph9MWr9ll6qwe9jxl6tP8ZOgDcOz+Fcq2Jb7++1XY7pRTnlqwLol6Ia0suWGMFSzASPi2IZoiAzkGyo1PCDX4FkKNTKVQbTVzetB+G6GSzWMW4XUCPO3uih7Ug2giPnwvP37ZTLsv7uAKuk6NTKQDAKxbSGO+CaC+wacZzDhOQi5tFEALM9jOgG15/vzR0AHjz0XFEJIInF1bbbr+8VcJ6oeprQFc6FkUXK6o3kt9Ok6MJ9bsuNHQbWEAveZBcmE9Kr5fZRya1XuM1/hHvRpNiu2RuncsYSciOk6It7TecDB1QHRedulx45BNFlhCRiH51pbY5+v9FAoCjU6wf3Pw98nNBdCf7RhRMZRScdSicL26UsC8T1wNMP9gtGXomHsVtB7NdA0bn9AnRrG/PZZahB5EgiQydg16Kon51VbDsz01hdLtUA6WtNkAzeDL0IDtDrOCSXDgKnIQQJGMRPfj7vX7OyHgqhmwyilcsah1+LojuhBCCE7OjejCyop8uiwwW0BVZ6uuJBVCnRs9e3m6r15xZ2kY0QnDDTMa352GdPSxDZ4mF37CAzutW6pahCOh626IHDd2vYDiRiiETl10VRvWx/x419LAWRBsZ45Bc8hW+dsqUwVyt15kAJ45OpvCKTYbu14JoM07MjuLiat52XmJps9Q3l0UGazII2wvdjPvmJ0Ep8I2L6/pt5y5v4Yb9I76ebDr3iqp21P5/DmOyhExcRtXjEKQTQxLQ2XCKB8nFJw2dEKIGCxeSC+vjNlsQzRiJy46e6GEtiDYynopho1i1neTjPVmq1g2GzCjIgD6VthzB93vLeycn50bRpMALV3dM7681mri6XerrUBHQCuhssK2fnDqYRVqR8YTWj95sUpz1uSAKAIqWobPvmWrhHMz36eyvvgO//M43BPK7hyKgRyTVy9yLha6fcsXRqbQryYX1cZstiGawopSdJ3pYC6KNZJMxVOtNlGo2J5pKHRJx7hRIKbKhyyW4LxKgSmOruYqpURbv/kmvsKnGsxayy9WtMpq0fy6LDCY59bNlkRGNSHjz0QldR39tvYBcue57QI/rRVGmoQeXWLi16XbDUAR0wHxRAg964c6H4s+RyRSubJe5i7Nckovu52ItbxRCWhBthGe4iGW8Th/gpGFrUfCSi1oYNet08XtBdCfTI3HsH4lbdrowH/S5Pg4VAUbJpf8ZOqDKLpc2iri0XtSnsf0siAKtomi5btTQd8frd8PQBHS2Nsstfg6yHJnUCqOcOrruhW6boTM/F/sMPWxnPt3PxaYwyiufpGKtk3GQmREAHGOtiybSmN8Los04MTdq2enCetD7OVQEqHoyIf1tWTSi2+kurOHM4jbiUUl3sPQLpsdXai1f/jC7xvxiuAK6x6KoLLlfP2eG3unCG9CLNUQjxFb75snQg85qzdD9XGxaF3nlk3bJJdiAfmgiCYlYZOgBn0wAVXZ5da1gKvksbZYQkQhmRuOBHoMThBAkopG+LLcw49hUCjOjcTy5sIpzl7dw84FR3wvXrbbF1pVimHMdfjFEAd2j5FL2tn7OjFaGzlcY3SxUkU3GbJ9bX0Nn0+kS5oJoBq/kwnOiMRZFgz45KXIEh8aTfZFcADVDpxR4/kp3YXRxs4iZ0XhgXTZu+PDbj+MHT832+zAAqCeYe49P4usL6/ju5R1bh0WvGNsW3ewi3W30/5PjE17X0OV7dFo0kozJmBmNW04iduI0JQrweaKHudyC4afkkozJKFbqqDeaKNeagQdVtdOl+6Sr/h0Dlly0YGTWj764Uey73ML48NuP465jE/0+DJ175yexXaqhVGvg1EH/A7qxbbFSb6LRpH1bMNILQxPQva6hy1dqvmaERyZTloMrnWwVa7YtiwCfJ3qYyy0Y2QTL0K0DOu9la0qRUaw1DMstgn0tRydTeG29gGaH747fC6LNmEwrmM0mTHX0pc1S34eKditMRwf8L4gCrQy9Um/0ZVDPL7gCOiHkXYSQlwkhC4SQj5rc/88IIWe1/z1FCDnl/6Hao2boXtoW/b20OqINrvCsw9soVm1bFgE+T/RCJfyiqByRMBKXbYeLVBdLnqJoBJQCq3nV2jboL9LRqTTKtSaudCwhDmJBtBnqxOhW223lWgMruUrfh4p2K5NpBTfOjCCjyDgykfL99xsz9KC2FYWBY0AnhEQAfALA/QBuAvAgIeSmjoe9CuCtlNKTAH4dwCN+H6gTyZisZ3huyFXqSPvYb3t0Ko2dct1xLB5Q7WftnBYBPk90P+x/vTCWsh//5z1ZMrloZacMIHiTMTOTrqAWRJtxYm4Ur60X2/w8dNvcPrcs7mb+9ffP45ff+YZAfH7kiARZIijXGoF68gcNT4Z+J4AFSukrlNIqgE8BeMD4AErpU5TSTe3HpwHM+XuYzqRiEZQ8Dhb5GQyPcrYuUkq1bUXOJxMnT/R+aOiAOlzkh+TC/v5s+UTwGToL6C0dPUz7BKajP2+QXZZYD/ou0dB3I++4eT8+ePfhwH5/PBpBpd5sefoMY4YOYBbAouHnJe02K34awD+a3UEIeYgQcpoQcnp1ddXsIZ5JxiIo1hpduqgTBZ+7KvRg4RDQd8p1NJrUUXIBnP1c+qGhA8C4jZ+Lm4yXyUUruXAy9Km0gowit71HYdon6BOjhoC+uAsWW+x14lF1a9Gwa+hm1zemUZMQ8naoAf1XzO6nlD5CKb2DUnrH1NQU/1FykFRkUNqa9OKFtS36xWw2gWiEOHa6MPc4J8kFsHdcLNcaWotV+B++MRvHRTc+86wQubxT0f6bYIMqIQRHp1Jt71FQC6LNGEvFcHA80dbpsrRRRCwiYTqjBP78AnMUOaJp6MGtIgwanoC+BOCg4ec5AFc6H0QIOQngDwE8QCld77w/aFIe1tBRSpHnLNzxIkckHBpPOvais0DIJbnYeKJ/59IWAOCWA/63cjmRTcYs19C5mcBNhSy5AGqtwyi5BLVD0oqTs1mcvbyl/7y0WcLsWCIQfVjAhxKVtC6XcJeu+wlPQP8WgHlCyBFCSAzA+wB83vgAQsghAJ8F8H5K6Xn/D9OZhIc1dMWqtn7O5wBydCrtqKEzqcJu7J9hl6F/4+IaJALceXTc/YH2yHgqikK1YeoE2fpS8LUtAuEVRQG11nFlu6xnY2EXwk7MjWJxo6T38S/uAh/0vU5cy9CHuihKKa0D+AiALwJ4EcCnKaXPE0I+RAj5kPawfwdgAsDvEUKeI4ScDuyILUh5WEMX1P5Ktc/Zfr9oK0PvTUN/6uI6Tsxl++KMx+QiMx09r/9tObpcYv3J0IFW8TrI9XNmnGQDRpqOvrhRFC2LfSbemaEPqeQCSumjlNLrKaXHKKUf0257mFL6sPbvn6GUjlFKb9X+d0eQB20G0z7d9KKzoOO3q9yRyRSq9SauWCwjBloDOU6TooC1J3qhUsdzi1u4p08TfXZ+Lm5OlmlDhh7xyVfHic7WxSDXz5lxsyGg5yt1bBZrIkPvM6qG3kCxqto+M3+XQWLwjtiCpAcNXc8ifT4TO+2uBNSsViJ8JxMrT/RvvraBepPi7mOTZv9Z4IyltGnRQneGXnDxt2XFp0K1gVQsEqhfNOPIZArEYNIVdlY2moji8EQS55a29ZZF0eHSX9QMvclt+7wbGbqA7ma4yE8vdCM8Nrob2lARTxHMynHxGxfXEYtIeON1Yz0crXfsMnQ3RdGYLCEaIdyP94N4NIIDowndRjfsDB0ATsxlce7yNhY31Cs5kaH3l3hUy9BD8PQJiqEJ6CyzYoGEh3w5mH7TybT9ftFmk+KZV9ZxeIIvI7PyRH/q4hpuO5TVFxKEDZ/kwndsTJoJsxBlbF3sh256cnYUl7dKOLO4BQBCQ+8ziizpRdFB1M+BIQro+0fjiEaIrczRSVDVbH2/qEUv+uMXVnFxtYD333Ud1+8zy9C3ilU8f2WnzbQobJixmFlRlNnh8v5t2RcozIB+TGtdpJSioC2IjoTYNnhCW6P26HevIhGNYIKj40kQHOqkaCNwT/4gGZqAHo9G8Ib9GZy1WO9lRlAZOqDKLlYZ+h89+SqmMwrec+IA1+8y80R/+pV1UArc3UeL03g0gqmMgj9/+nU8dXGt7b68y8UhTDILczrv6FQKhapqitWPlWM3HxjRdfy5scRAarbDhCq5NFWTtgEc+weGKKADqq3m2aVt7vH/vHaZHUQQOTqVxuWtEsodS5TPL+fwxIU1fOCu67h3gJp5oj91cR3JWCQQK1E3PPL+N0KJRvATf/AMfu3vntdfL8tyeINUUpdcwvsisf2iF1fzgS+INiMTj+reP0Ju6T+KNvpfDGFzVVAMVUA/NTeKXLmO19b5/MgLlToiEgmkPcmqMPpfv/4aFFnCg3ce4v5dZp7oT11cx/ccHg91MbQZtx0awz/8z/fig3ddh//69dfw7t95As8tbrnePMQMusLW0AE1Q+7XUmDm63JQFET7jiK3zLlEhr4LYNnqWZNtMGao7UnBtMmZBfSNQhWf/fYSfvi2WUyk+T07Oj3RV3bKWFjJ91VuMZKMyfi1B27Bn/30m1CqNvCjv/8UHj+/6irbZq2LYUou+0fiSEQjeGW10Dc/nBPaZ1a4LPYflthtFqsDacwFDFlAn59OIx6VcIZTRw9yf6VZQP/Lb15Cpd7ET917xNXv6vRE/8YrqlVOPwuiZtw7P4kv/OJb8MCtB7CWr7qaXmWTvmEGVUki2oapPAp92MsKALcfygIAjk37v7RB4I64rL7/W8XaQBpzAcBgHrUFckTCzQdG+TP0ct33HnRGSpGxfySud91U60386VOv4b75SVy/L+P69xk90b++sIbRRBQ3zoz4esx+MJqI4rf+p1vxI7fNuZrA1dsWQw6qx6bTOLO4hXhUwr5MPNTnBlTJ6jM/dzduO5gN/bkF7SgG6XUQjbmAIcvQAeDk3Ciev7KNeqPp+NgXr+3gUIDFqKNTrU6XR89dxUqu4jo7Zxgz9KcuruPNR8dDbbFzy73zkzjlIkj1ow8dUH13ljaL2CjU+jZM8sbrxoTL4i6AZejAYBpzAUMY0E/NZVGuNXFhxb4f/dJ6Ea+vF3FvgLIFa12klOKPnnwVR6dSeOu8Nx945ri4uFHE0mapb+P+QZHsg+QCqCfdJgXW8pWBHSYR+ANbFA2Ef6XoF0MX0E9qwxpO/ehPLKgbk+673t9FG0aOTKawVazhSy8s49zlbfzUPUc8Z2LME531e99zfHcURP2C1TLCLkYd03x3gMHNygT+YJyZGFQNfegC+uGJFDJxGWccdPQnzq/hwGhc7wMOAhYsfv3vX8BoIoofud1uc589LEN/6uI6pjJKWyAaBpJ9mBQFWsVrYHCzMoE/tGXoA3pyH7qALkkEJ+dGbTP0RpPiqYtruHd+MtDpPBYsljZLePDOQz2d9UcSUWxrAf3uYxNDN1XIilB+Luzme161eA2Es35OsHuJi6Lo7uTkXBYvXc11TWkyzi5tYadcx30e9Wxe5sbU/aIRieCDd/P5tlgxEpdRqTexmqvsmv5zP7lldhQ37M/o05thwgaMRIa+tzFm6EJy2UWcmhtFvUnx4tUd0/ufuLAGQoLv45YjEk7NZfGjt89iZrS3SUDm5wJg6AqigCpPfeEX38K1ks9vWEAXGfrexqihD+pg0WAetQPGidHbDnV7hT95YQ23HBjFeAjB47//y7t8+T1sSOfgeEL4fvgMuyoIW+4R7C7aM/TB/CwMZYY+MxrHZFoxnRjNV+r49qVN3DsfTpYbkYgv/eLMz+Xuo8OXnfebG2bUQa/xFL8dg2D4aB8sGsxcdygDOiGsMNrd6fL0xXXUmxT37bKxeSdYsLl7yNoVdwN3HZ3AZ37uLpzSWl4FexNFFhn6ruXk3Cgurua7Nhg9ubCGeFTCGw/3Z22bV07NjeKR978RP3CSz0NdwA8hBG+8bnzoOocE7mBdLm58/Hcbg3nUHJyay4JS4FxHlv74hVW86chE29l4ECCE4B0379/V4/4CwSATi0ggRM3OB/XkPrQB3Wxi9MpWCa+sFnBfSPq5QCAYHAhRM/NB1c+BIQ7oE2kFs9kEzl5uZehPXlDH5oPuPxcIBINJPBoRAX23cupg+8To4xdWMZ1RcP2+4RqbFwgE/hCXIwM9YDbUAf3kXBaLGyVsFKpoNim+vhD8uL9AIBhc4lFpYKdEgSEdLGIYdfSJlILNYk3o5wKBwJJ0XEY2yb9pa7cx1AH9xOwoCFEnRuWImpXvtrVtAoFg9/AbP3pqYHvQgSEP6Jl4FEcnUzi7tIVitYEb9mcw3Yc1YwKBYDC46cDuW+vohqHW0AFVR//2pS2cfm1TyC0CgWCo2QMBfRQbhSqqjaZoVxQIBEMNV0AnhLyLEPIyIWSBEPJRk/sJIeR3tPvPEkJu9/9QvcGcF2OyhDuPjPf3YAQCgSBAHAM6ISQC4BMA7gdwE4AHCSE3dTzsfgDz2v8eAvD7Ph+nZ24+MAJZIrjz8HibPaZAIBAMGzxF0TsBLFBKXwEAQsinADwA4AXDYx4A8ElKKQXwNCEkSwiZoZRe9f2IXRKPRvB/vOdG3Dgz2MUOgUAgcIInoM8CWDT8vATgTRyPmQXQFtAJIQ9BzeBx6NAht8fqmX9xz5HQnksgEAj6BY+GbjZWST08BpTSRyild1BK75iaEgVKgUAg8BOegL4E4KDh5zkAVzw8RiAQCAQBwhPQvwVgnhByhBASA/A+AJ/veMznAXxA63Z5M4Dt3aCfCwQCwV7CUUOnlNYJIR8B8EUAEQB/TCl9nhDyIe3+hwE8CuDdABYAFAH8ZHCHLBAIBAIzuEb/KaWPQg3axtseNvybAviwv4cmEAgEAjcM/aSoQCAQ7BVEQBcIBIIhQQR0gUAgGBKIKn/34YkJWQXwusf/fBLAmo+HM0js1dcuXvfeQrxua66jlJoO8vQtoPcCIeQ0pfSOfh9HP9irr1287r2FeN3eEJKLQCAQDAkioAsEAsGQMKgB/ZF+H0Af2auvXbzuvYV43R4YSA1dIBAIBN0MaoYuEAgEgg5EQBcIBIIhYeACutN+02GBEPLHhJAVQsh3DbeNE0K+TAi5oP3/WD+PMQgIIQcJIf9ECHmREPI8IeQXtNuH+rUTQuKEkG8SQs5or/vXtNuH+nUzCCERQsh3CCF/r/089K+bEPIaIeQcIeQ5Qshp7baeXvdABXTO/abDwp8AeFfHbR8F8BVK6TyAr2g/Dxt1AL9EKb0RwJsBfFh7j4f9tVcAfC+l9BSAWwG8S7OiHvbXzfgFAC8aft4rr/vtlNJbDb3nPb3ugQroMOw3pZRWAbD9pkMHpfRxABsdNz8A4E+1f/8pgB8K85jCgFJ6lVL6be3fOahf8lkM+WunKnntx6j2P4ohf90AQAiZA/AeAH9ouHnoX7cFPb3uQQvoVrtL9wr72OIQ7f+n+3w8gUIIOQzgNgDPYA+8dk12eA7ACoAvU0r3xOsG8NsA/lcATcNte+F1UwBfIoQ8q+1bBnp83Vx+6LsIrt2lgsGHEJIG8BkAv0gp3SHE7K0fLiilDQC3EkKyAD5HCLmlz4cUOISQHwCwQil9lhDytj4fTtjcQym9QgiZBvBlQshLvf7CQcvQ9/ru0mVCyAwAaP+/0ufjCQRCSBRqMP9zSulntZv3xGsHAErpFoCvQq2hDPvrvgfADxJCXoMqoX4vIeTPMPyvG5TSK9r/rwD4HFRJuafXPWgBnWe/6TDzeQAf1P79QQB/28djCQSipuJ/BOBFSulvGe4a6tdOCJnSMnMQQhIAvh/ASxjy100p/beU0jlK6WGo3+fHKKX/HEP+ugkhKUJIhv0bwDsAfBc9vu6BmxQlhLwbqubG9pt+rL9HFAyEkL8E8DaodprLAH4VwN8A+DSAQwAuAfgxSmln4XSgIYTcC+AJAOfQ0lT/N6g6+tC+dkLISahFsAjUROvTlNJ/TwiZwBC/biOa5PLLlNIfGPbXTQg5CjUrB1Tp+y8opR/r9XUPXEAXCAQCgTmDJrkIBAKBwAIR0AUCgWBIEAFdIBAIhgQR0AUCgWBIEAFdIBAIhgQR0AUCgWBIEAFdIBAIhoT/HwifBFTd20qGAAAAAElFTkSuQmCC\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "plt.plot([rand() for _ in range(50)]);" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD4CAYAAAAAczaOAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAPHUlEQVR4nO3cf6zdd13H8efLlg0GIp29W2rb2WIq0BEJ4zonKEFrsjGMnQlLisIasqRRJ05jIh1/uD9Mk5EYg0QHaQZSIlnTjMXVH6BLEdHANu/Y2NbVuivV7rq6XkABMRm0vP3jfE2O3b3rueecey63n+cjac45n/P99vv55DbP++333vNNVSFJasP3rfQEJEmTY/QlqSFGX5IaYvQlqSFGX5IasnalJ3A+69evry1btqz0NCRpVXn44Ye/UlVT545/z0d/y5YtzMzMrPQ0JGlVSfJvC417eUeSGmL0JakhRl+SGmL0JakhRl+SGmL0JakhRl+SGmL0JakhRl+SGvI9/4lcLc2WvX+5Isf91zvetiLHhTbX3JqV+hrDhfd19kxfkhpi9CWpIV7e0Vis5H+/JQ3uvGf6ST6a5HSSJ/rGLk1yf5Knusd1fe/dlmQ2yfEk1/aNvyHJ4917H0yS8S9HkvRCBrm88zHgunPG9gJHqmobcKR7TZLtwC7gym6fO5Os6fb5ELAH2Nb9OffvlCQts/NGv6o+B3ztnOGdwIHu+QHghr7xg1X1XFWdAGaBq5NsAF5eVV+oqgI+3rePJGlChr2mf3lVnQKoqlNJLuvGNwIP9G031419p3t+7viCkuyh978CrrjiiiGnKGnc/NnN6jfuH+QudJ2+XmB8QVW1H9gPMD09veh2UquMr4Y17K9sPttdsqF7PN2NzwGb+7bbBDzTjW9aYFySNEHDRv8wsLt7vhu4r298V5KLk2yl9wPbh7pLQd9Mck33Wzs39e0jSZqQ817eSXI38BZgfZI54HbgDuBQkpuBk8CNAFV1NMkh4EngDHBLVZ3t/qpfpfebQC8BPtX9kSRN0HmjX1XvWOStHYtsvw/Yt8D4DPDaJc1OklbYhXZvJz+Ruwz8IVsb/DprNfLeO5LUkAv6TN8zMUn6/zzTl6SGGH1JaojRl6SGGH1JaojRl6SGGH1JaojRl6SGGH1JaojRl6SGGH1JaojRl6SGGH1JaojRl6SGGH1JaojRl6SGGH1JaojRl6SGGH1JaojRl6SGGH1JaojRl6SGGH1JaojRl6SGGH1JaojRl6SGGH1JaojRl6SGGH1JashI0U/yW0mOJnkiyd1JXpzk0iT3J3mqe1zXt/1tSWaTHE9y7ejTlyQtxdDRT7IR+A1guqpeC6wBdgF7gSNVtQ040r0myfbu/SuB64A7k6wZbfqSpKUY9fLOWuAlSdYClwDPADuBA937B4Abuuc7gYNV9VxVnQBmgatHPL4kaQmGjn5V/Tvw+8BJ4BTw9ar6G+DyqjrVbXMKuKzbZSPwdN9fMdeNPU+SPUlmkszMz88PO0VJ0jlGubyzjt7Z+1bgh4CXJnnnC+2ywFgttGFV7a+q6aqanpqaGnaKkqRzjHJ55+eAE1U1X1XfAe4F3gg8m2QDQPd4utt+Dtjct/8mepeDJEkTMkr0TwLXJLkkSYAdwDHgMLC722Y3cF/3/DCwK8nFSbYC24CHRji+JGmJ1g67Y1U9mOQe4IvAGeARYD/wMuBQkpvpfWO4sdv+aJJDwJPd9rdU1dkR5y9JWoKhow9QVbcDt58z/By9s/6Ftt8H7BvlmJKk4fmJXElqiNGXpIYYfUlqiNGXpIYYfUlqiNGXpIYYfUlqiNGXpIYYfUlqiNGXpIYYfUlqiNGXpIYYfUlqiNGXpIYYfUlqiNGXpIYYfUlqiNGXpIYYfUlqiNGXpIYYfUlqiNGXpIYYfUlqiNGXpIYYfUlqiNGXpIYYfUlqiNGXpIYYfUlqiNGXpIYYfUlqyEjRT/KKJPck+ackx5L8ZJJLk9yf5KnucV3f9rclmU1yPMm1o09fkrQUo57p/yHw6ap6NfA64BiwFzhSVduAI91rkmwHdgFXAtcBdyZZM+LxJUlLMHT0k7wceDPwEYCq+nZV/RewEzjQbXYAuKF7vhM4WFXPVdUJYBa4etjjS5KWbpQz/VcC88CfJHkkyV1JXgpcXlWnALrHy7rtNwJP9+0/1409T5I9SWaSzMzPz48wRUlSv1Givxa4CvhQVb0e+BbdpZxFZIGxWmjDqtpfVdNVNT01NTXCFCVJ/UaJ/hwwV1UPdq/vofdN4NkkGwC6x9N922/u238T8MwIx5ckLdHQ0a+q/wCeTvKqbmgH8CRwGNjdje0G7uueHwZ2Jbk4yVZgG/DQsMeXJC3d2hH3fw/wiSQXAV8G3k3vG8mhJDcDJ4EbAarqaJJD9L4xnAFuqaqzIx5fkrQEI0W/qh4Fphd4a8ci2+8D9o1yTEnS8PxEriQ1xOhLUkOMviQ1xOhLUkOMviQ1xOhLUkOMviQ1xOhLUkOMviQ1xOhLUkOMviQ1xOhLUkOMviQ1xOhLUkOMviQ1xOhLUkOMviQ1xOhLUkOMviQ1xOhLUkOMviQ1xOhLUkOMviQ1xOhLUkOMviQ1xOhLUkOMviQ1xOhLUkOMviQ1xOhLUkOMviQ1ZOToJ1mT5JEkf9G9vjTJ/Ume6h7X9W17W5LZJMeTXDvqsSVJSzOOM/1bgWN9r/cCR6pqG3Cke02S7cAu4ErgOuDOJGvGcHxJ0oBGin6STcDbgLv6hncCB7rnB4Ab+sYPVtVzVXUCmAWuHuX4kqSlGfVM/wPA7wDf7Ru7vKpOAXSPl3XjG4Gn+7ab68aeJ8meJDNJZubn50ecoiTp/wwd/SQ/D5yuqocH3WWBsVpow6raX1XTVTU9NTU17BQlSedYO8K+bwJ+Icn1wIuBlyf5U+DZJBuq6lSSDcDpbvs5YHPf/puAZ0Y4viRpiYY+06+q26pqU1VtofcD2s9U1TuBw8DubrPdwH3d88PAriQXJ9kKbAMeGnrmkqQlG+VMfzF3AIeS3AycBG4EqKqjSQ4BTwJngFuq6uwyHF+StIixRL+qPgt8tnv+VWDHItvtA/aN45iSpKXzE7mS1BCjL0kNMfqS1BCjL0kNMfqS1BCjL0kNMfqS1BCjL0kNMfqS1BCjL0kNMfqS1BCjL0kNMfqS1BCjL0kNMfqS1BCjL0kNMfqS1BCjL0kNMfqS1BCjL0kNMfqS1BCjL0kNMfqS1BCjL0kNMfqS1BCjL0kNMfqS1BCjL0kNMfqS1BCjL0kNMfqS1JCho59kc5K/TXIsydEkt3bjlya5P8lT3eO6vn1uSzKb5HiSa8exAEnS4EY50z8D/HZVvQa4BrglyXZgL3CkqrYBR7rXdO/tAq4ErgPuTLJmlMlLkpZm6OhX1amq+mL3/JvAMWAjsBM40G12ALihe74TOFhVz1XVCWAWuHrY40uSlm4s1/STbAFeDzwIXF5Vp6D3jQG4rNtsI/B0325z3dhCf9+eJDNJZubn58cxRUkSY4h+kpcBnwR+s6q+8UKbLjBWC21YVfurarqqpqempkadoiSpM1L0k7yIXvA/UVX3dsPPJtnQvb8BON2NzwGb+3bfBDwzyvElSUszym/vBPgIcKyq/qDvrcPA7u75buC+vvFdSS5OshXYBjw07PElSUu3doR93wS8C3g8yaPd2PuAO4BDSW4GTgI3AlTV0SSHgCfp/ebPLVV1doTjS5KWaOjoV9U/sPB1eoAdi+yzD9g37DElSaPxE7mS1BCjL0kNMfqS1BCjL0kNMfqS1BCjL0kNMfqS1BCjL0kNMfqS1BCjL0kNMfqS1BCjL0kNMfqS1BCjL0kNMfqS1BCjL0kNMfqS1BCjL0kNMfqS1BCjL0kNMfqS1BCjL0kNMfqS1BCjL0kNMfqS1BCjL0kNMfqS1BCjL0kNMfqS1BCjL0kNMfqS1JCJRz/JdUmOJ5lNsnfSx5eklk00+knWAH8MvBXYDrwjyfZJzkGSWjbpM/2rgdmq+nJVfRs4COyc8BwkqVlrJ3y8jcDTfa/ngJ84d6Mke4A93cv/TnJ8yOOtB74y5L6rlWtuQ2trbm295P0jr/mHFxqcdPSzwFg9b6BqP7B/5IMlM1U1Perfs5q45ja0tubW1gvLt+ZJX96ZAzb3vd4EPDPhOUhSsyYd/X8EtiXZmuQiYBdweMJzkKRmTfTyTlWdSfLrwF8Da4CPVtXRZTzkyJeIViHX3IbW1tzaemGZ1pyq511SlyRdoPxEriQ1xOhLUkMuiOif79YO6flg9/5jSa5aiXmOywDr/eVunY8l+XyS163EPMdp0Nt3JPnxJGeTvH2S81sOg6w5yVuSPJrkaJK/m/Qcx22Af9s/kOTPk3ypW/O7V2Ke45Lko0lOJ3likffH366qWtV/6P1A+F+AVwIXAV8Ctp+zzfXAp+h9TuAa4MGVnvcyr/eNwLru+VtX83oHXXPfdp8B/gp4+0rPewJf51cATwJXdK8vW+l5T2DN7wPe3z2fAr4GXLTScx9hzW8GrgKeWOT9sbfrQjjTH+TWDjuBj1fPA8ArkmyY9ETH5LzrrarPV9V/di8foPd5iNVs0Nt3vAf4JHB6kpNbJoOs+ZeAe6vqJEBVrfZ1D7LmAr4/SYCX0Yv+mclOc3yq6nP01rCYsbfrQoj+Qrd22DjENqvFUtdyM70zhdXsvGtOshH4ReDDE5zXchrk6/yjwLokn03ycJKbJja75THImv8IeA29D3U+DtxaVd+dzPRWxNjbNenbMCyHQW7tMNDtH1aJgdeS5GfoRf+nlnVGy2+QNX8AeG9Vne2dBK56g6x5LfAGYAfwEuALSR6oqn9e7sktk0HWfC3wKPCzwI8A9yf5+6r6xjLPbaWMvV0XQvQHubXDhXT7h4HWkuTHgLuAt1bVVyc0t+UyyJqngYNd8NcD1yc5U1V/NpEZjt+g/66/UlXfAr6V5HPA64DVGv1B1vxu4I7qXfCeTXICeDXw0GSmOHFjb9eFcHlnkFs7HAZu6n4Sfg3w9ao6NemJjsl515vkCuBe4F2r+Kyv33nXXFVbq2pLVW0B7gF+bRUHHwb7d30f8NNJ1ia5hN4da49NeJ7jNMiaT9L7nw1JLgdeBXx5orOcrLG3a9Wf6dcit3ZI8ivd+x+m99sc1wOzwP/QO1tYlQZc7+8CPwjc2Z35nqlVfIfCAdd8QRlkzVV1LMmngceA7wJ3VdWCv/q3Ggz4df494GNJHqd36eO9VbVqb7mc5G7gLcD6JHPA7cCLYPna5W0YJKkhF8LlHUnSgIy+JDXE6EtSQ4y+JDXE6EtSQ4y+JDXE6EtSQ/4XLf0Ao/3fCcsAAAAASUVORK5CYII=\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "plt.hist([rand() for _ in range(10000)]);" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "2.67 ms ± 15.9 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" ] } ], "source": [ "%timeit -n 10 list(chunks([rand() for _ in range(7840)], 10))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "37.8 µs ± 12.9 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" ] } ], "source": [ "%timeit -n 10 torch.randn(784,10)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Matrix multiplication" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "torch.manual_seed(1)\n", "weights = torch.randn(784,10)\n", "bias = torch.zeros(10)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "m1 = x_valid[:5]\n", "m2 = weights" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(torch.Size([5, 784]), torch.Size([784, 10]))" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "m1.shape,m2.shape" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "((5, 784), (784, 10))" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "ar,ac = m1.shape # n_rows * n_cols\n", "br,bc = m2.shape\n", "(ar,ac),(br,bc)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "torch.Size([5, 10])" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "t1 = torch.zeros(ar, bc)\n", "t1.shape" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "for i in range(ar): # 5\n", " for j in range(bc): # 10\n", " for k in range(ac): # 784\n", " t1[i,j] += m1[i,k] * m2[k,j]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor([[-10.9417, -0.6844, -7.0038, -4.0066, -2.0857, -3.3588, 3.9127,\n", " -3.4375, -11.4696, -2.1153],\n", " [ 14.5430, 5.9977, 2.8914, -4.0777, 6.5914, -14.7383, -9.2787,\n", " 2.1577, -15.2772, -2.6758],\n", " [ 2.2204, -3.2171, -4.7988, -6.0453, 14.1661, -8.9824, -4.7922,\n", " -5.4446, -20.6758, 13.5657],\n", " [ -6.7097, 8.8998, -7.4611, -7.8966, 2.6994, -4.7260, -11.0278,\n", " -12.9776, -6.4443, 3.6376],\n", " [ -2.4444, -6.4034, -2.3984, -9.0371, 11.1772, -5.7724, -8.9214,\n", " -3.7862, -8.9827, 5.2797]])" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "t1" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "torch.Size([5, 10])" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "t1.shape" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor([[-10.94, -0.68, -7.00, -4.01, -2.09, -3.36, 3.91, -3.44, -11.47, -2.12],\n", " [ 14.54, 6.00, 2.89, -4.08, 6.59, -14.74, -9.28, 2.16, -15.28, -2.68],\n", " [ 2.22, -3.22, -4.80, -6.05, 14.17, -8.98, -4.79, -5.44, -20.68, 13.57],\n", " [ -6.71, 8.90, -7.46, -7.90, 2.70, -4.73, -11.03, -12.98, -6.44, 3.64],\n", " [ -2.44, -6.40, -2.40, -9.04, 11.18, -5.77, -8.92, -3.79, -8.98, 5.28]])" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "torch.set_printoptions(precision=2, linewidth=140, sci_mode=False)\n", "t1" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", "np.set_printoptions(precision=2, linewidth=140)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "def matmul(a,b):\n", " (ar,ac),(br,bc) = a.shape,b.shape\n", " c = torch.zeros(ar, bc)\n", " for i in range(ar):\n", " for j in range(bc):\n", " for k in range(ac): c[i,j] += a[i,k] * b[k,j]\n", " return c" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CPU times: user 421 ms, sys: 131 µs, total: 421 ms\n", "Wall time: 421 ms\n" ] } ], "source": [ "%time _=matmul(m1, m2)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "39200" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "ar*bc*ac" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Numba" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from numba import njit" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "@njit\n", "def dot(a,b):\n", " res = 0.\n", " for i in range(len(a)): res+=a[i]*b[i]\n", " return res" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from numpy import array" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CPU times: user 184 ms, sys: 12 ms, total: 196 ms\n", "Wall time: 196 ms\n" ] }, { "data": { "text/plain": [ "20.0" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "%time dot(array([1.,2,3]),array([2.,3,4]))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CPU times: user 15 µs, sys: 0 ns, total: 15 µs\n", "Wall time: 17.6 µs\n" ] }, { "data": { "text/plain": [ "20.0" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "%time dot(array([1.,2,3]),array([2.,3,4]))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Now only two of our loops are running in Python, not three:" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "def matmul(a,b):\n", " (ar,ac),(br,bc) = a.shape,b.shape\n", " c = torch.zeros(ar, bc)\n", " for i in range(ar):\n", " for j in range(bc): c[i,j] = dot(a[i,:], b[:,j])\n", " return c" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "m1a,m2a = m1.numpy(),m2.numpy()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from fastcore.test import *" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "test_close(t1,matmul(m1a, m2a))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "236 µs ± 3.51 µs per loop (mean ± std. dev. of 7 runs, 50 loops each)\n" ] } ], "source": [ "%timeit -n 50 matmul(m1a,m2a)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Elementwise ops" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "[TryAPL](https://tryapl.org/)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(tensor([10., 6., -4.]), tensor([2., 8., 7.]))" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "a = tensor([10., 6, -4])\n", "b = tensor([2., 8, 7])\n", "a,b" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor([12., 14., 3.])" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "a + b" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor(0.67)" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "(a < b).float().mean()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor([[1., 2., 3.],\n", " [4., 5., 6.],\n", " [7., 8., 9.]])" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "m = tensor([[1., 2, 3], [4,5,6], [7,8,9]]); m" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Frobenius norm:\n", "\n", "$$\\| A \\|_F = \\left( \\sum_{i,j=1}^n | a_{ij} |^2 \\right)^{1/2}$$\n", "\n", "*Hint*: you don't normally need to write equations in LaTeX yourself, instead, you can click 'edit' in Wikipedia and copy the LaTeX from there (which is what I did for the above equation). Or on arxiv.org, click \"Download: Other formats\" in the top right, then \"Download source\"; rename the downloaded file to end in `.tgz` if it doesn't already, and you should find the source there, including the equations to copy and paste. This is the source LaTeX that I pasted to render the equation above:\n", "\n", "```latex\n", "$$\\| A \\|_F = \\left( \\sum_{i,j=1}^n | a_{ij} |^2 \\right)^{1/2}$$\n", "```" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor(285.)" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "sf = (m*m).sum()\n", "sf" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor(16.88)" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "sf.sqrt()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(tensor([7., 8., 9.]), tensor([3., 6., 9.]))" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "m[2,:],m[:,2]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor([7., 8., 9.])" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "m[2]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "def matmul(a,b):\n", " (ar,ac),(br,bc) = a.shape,b.shape\n", " c = torch.zeros(ar, bc)\n", " for i in range(ar):\n", " for j in range(bc): c[i,j] = (a[i,:] * b[:,j]).sum()\n", " return c" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "test_close(t1,matmul(m1, m2))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "598 µs ± 4.2 µs per loop (mean ± std. dev. of 7 runs, 50 loops each)\n" ] } ], "source": [ "%timeit -n 50 _=matmul(m1, m2)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "def matmul(a,b):\n", " (ar,ac),(br,bc) = a.shape,b.shape\n", " c = torch.zeros(ar, bc)\n", " for i in range(ar):\n", " for j in range(bc): c[i,j] = torch.dot(a[i,:], b[:,j])\n", " return c" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "test_close(t1,matmul(m1, m2))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "481 µs ± 4.9 µs per loop (mean ± std. dev. of 7 runs, 50 loops each)\n" ] } ], "source": [ "%timeit -n 50 _=matmul(m1, m2)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Broadcasting" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "The term **broadcasting** describes how arrays with different shapes are treated during arithmetic operations.\n", "\n", "From the [Numpy Documentation](https://docs.scipy.org/doc/numpy-1.10.0/user/basics.broadcasting.html):\n", "\n", " The term broadcasting describes how numpy treats arrays with \n", " different shapes during arithmetic operations. Subject to certain \n", " constraints, the smaller array is “broadcast” across the larger \n", " array so that they have compatible shapes. Broadcasting provides a \n", " means of vectorizing array operations so that looping occurs in C\n", " instead of Python. It does this without making needless copies of \n", " data and usually leads to efficient algorithm implementations.\n", " \n", "In addition to the efficiency of broadcasting, it allows developers to write less code, which typically leads to fewer errors.\n", "\n", "*This section was adapted from [Chapter 4](http://nbviewer.jupyter.org/github/fastai/numerical-linear-algebra/blob/master/nbs/4.%20Compressed%20Sensing%20of%20CT%20Scans%20with%20Robust%20Regression.ipynb#4.-Compressed-Sensing-of-CT-Scans-with-Robust-Regression) of the fast.ai [Computational Linear Algebra](https://github.com/fastai/numerical-linear-algebra) course.*" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Broadcasting with a scalar" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor([10., 6., -4.])" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "a" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor([ True, True, False])" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "a > 0" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "How are we able to do `a > 0`? 0 is being **broadcast** to have the same dimensions as a.\n", "\n", "For instance you can normalize our dataset by subtracting the mean (a scalar) from the entire data set (a matrix) and dividing by the standard deviation (another scalar), using broadcasting.\n", "\n", "Other examples of broadcasting with a scalar:" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor([11., 7., -3.])" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "a + 1" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor([[1., 2., 3.],\n", " [4., 5., 6.],\n", " [7., 8., 9.]])" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "m" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor([[ 2., 4., 6.],\n", " [ 8., 10., 12.],\n", " [14., 16., 18.]])" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "2*m" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Broadcasting a vector to a matrix" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Although broadcasting a scalar is an idea that dates back to APL, the more powerful idea of broadcasting across higher rank tensors [comes from](https://mail.python.org/pipermail/matrix-sig/1995-November/000143.html) a little known language called [Yorick](https://software.llnl.gov/yorick-doc/manual/yorick_50.html).\n", "\n", "We can also broadcast a vector to a matrix:" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor([10., 20., 30.])" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "c = tensor([10.,20,30]); c" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor([[1., 2., 3.],\n", " [4., 5., 6.],\n", " [7., 8., 9.]])" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "m" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(torch.Size([3, 3]), torch.Size([3]))" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "m.shape,c.shape" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor([[11., 22., 33.],\n", " [14., 25., 36.],\n", " [17., 28., 39.]])" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "m + c" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor([[11., 22., 33.],\n", " [14., 25., 36.],\n", " [17., 28., 39.]])" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "c + m" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "t = c.expand_as(m)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor([[10., 20., 30.],\n", " [10., 20., 30.],\n", " [10., 20., 30.]])" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "t" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor([[11., 22., 33.],\n", " [14., 25., 36.],\n", " [17., 28., 39.]])" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "m + t" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "We don't really copy the rows, but it looks as if we did. In fact, the rows are given a *stride* of 0." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ " 10.0\n", " 20.0\n", " 30.0\n", "[torch.storage._TypedStorage(dtype=torch.float32, device=cpu) of size 3]" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "t.storage()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "((0, 1), torch.Size([3, 3]))" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "t.stride(), t.shape" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "You can index with the special value [None] or use `unsqueeze()` to convert a 1-dimensional array into a 2-dimensional array (although one of those dimensions has value 1)." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(tensor([[10., 20., 30.]]), tensor([[10., 20., 30.]]))" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "c.unsqueeze(0), c[None, :]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(torch.Size([3]), torch.Size([1, 3]))" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "c.shape, c.unsqueeze(0).shape" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(tensor([[10.],\n", " [20.],\n", " [30.]]),\n", " tensor([[10.],\n", " [20.],\n", " [30.]]))" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "c.unsqueeze(1), c[:, None]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(torch.Size([3]), torch.Size([3, 1]))" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "c.shape, c.unsqueeze(1).shape" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "You can always skip trailling ':'s. And '...' means '*all preceding dimensions*'" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(torch.Size([1, 3]), torch.Size([3, 1]))" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "c[None].shape,c[...,None].shape" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor([[10., 10., 10.],\n", " [20., 20., 20.],\n", " [30., 30., 30.]])" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "c[:,None].expand_as(m)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor([[11., 12., 13.],\n", " [24., 25., 26.],\n", " [37., 38., 39.]])" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "m + c[:,None]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor([[11., 22., 33.],\n", " [14., 25., 36.],\n", " [17., 28., 39.]])" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "m + c[None,:]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Broadcasting Rules" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor([[10., 20., 30.]])" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "c[None,:]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "torch.Size([1, 3])" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "c[None,:].shape" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor([[10.],\n", " [20.],\n", " [30.]])" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "c[:,None]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "torch.Size([3, 1])" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "c[:,None].shape" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor([[100., 200., 300.],\n", " [200., 400., 600.],\n", " [300., 600., 900.]])" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "c[None,:] * c[:,None]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor([[False, True, True],\n", " [False, False, True],\n", " [False, False, False]])" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "c[None] > c[:,None]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor([[ 1., 4., 9.],\n", " [16., 25., 36.],\n", " [49., 64., 81.]])" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "m*m" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "When operating on two arrays/tensors, Numpy/PyTorch compares their shapes element-wise. It starts with the **trailing dimensions**, and works its way forward. Two dimensions are **compatible** when\n", "\n", "- they are equal, or\n", "- one of them is 1, in which case that dimension is broadcasted to make it the same size\n", "\n", "Arrays do not need to have the same number of dimensions. For example, if you have a `256*256*3` array of RGB values, and you want to scale each color in the image by a different value, you can multiply the image by a one-dimensional array with 3 values. Lining up the sizes of the trailing axes of these arrays according to the broadcast rules, shows that they are compatible:\n", "\n", " Image (3d array): 256 x 256 x 3\n", " Scale (1d array): 3\n", " Result (3d array): 256 x 256 x 3\n", "\n", "The [numpy documentation](https://docs.scipy.org/doc/numpy-1.13.0/user/basics.broadcasting.html#general-broadcasting-rules) includes several examples of what dimensions can and can not be broadcast together." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Matmul with broadcasting" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(torch.Size([784]), torch.Size([784, 10]))" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "digit = m1[0]\n", "digit.shape,m2.shape" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "torch.Size([784, 1])" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "digit[:,None].shape" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "torch.Size([784, 10])" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "digit[:,None].expand_as(m2).shape" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "torch.Size([784, 10])" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "(digit[:,None]*m2).shape" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "def matmul(a,b):\n", " (ar,ac),(br,bc) = a.shape,b.shape\n", " c = torch.zeros(ar, bc)\n", " for i in range(ar):\n", "# c[i,j] = (a[i,:] * b[:,j]).sum() # previous version\n", " c[i] = (a[i,:,None] * b).sum(dim=0) # broadcast version\n", " return c" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "test_close(t1,matmul(m1, m2))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "70.1 µs ± 1.97 µs per loop (mean ± std. dev. of 7 runs, 50 loops each)\n" ] } ], "source": [ "%timeit -n 50 _=matmul(m1, m2)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Our time has gone from ~500ms to <0.1ms, an over 5000x improvement! We can run on the whole dataset now." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor([[ 0.96, -2.96, -2.11, ..., -15.09, -17.69, 0.60],\n", " [ 6.89, -0.34, 0.79, ..., -17.13, -25.36, 16.23],\n", " [-10.18, 7.38, 4.13, ..., -6.73, -6.79, -1.58],\n", " ...,\n", " [ 7.40, 7.64, -3.50, ..., -1.02, -16.22, 2.07],\n", " [ 3.25, 9.52, -9.37, ..., 2.98, -19.58, -1.96],\n", " [ 15.70, 4.12, -5.62, ..., 8.08, -12.21, 0.42]])" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "tr = matmul(x_train, weights)\n", "tr" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "torch.Size([50000, 10])" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "tr.shape" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CPU times: user 6.59 s, sys: 200 ms, total: 6.79 s\n", "Wall time: 663 ms\n" ] } ], "source": [ "%time _=matmul(x_train, weights)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Einstein summation" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "[Einstein summation](https://ajcr.net/Basic-guide-to-einsum/) ([`einsum`](https://numpy.org/doc/stable/reference/generated/numpy.einsum.html)) is a compact representation for combining products and sums in a general way. The key rules are:\n", "\n", "- Repeating letters between input arrays means that values along those axes will be multiplied together.\n", "- Omitting a letter from the output means that values along that axis will be summed." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(torch.Size([5, 784]), torch.Size([784, 10]))" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "m1.shape,m2.shape" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "torch.Size([5, 784, 10])" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# c[i,j] += a[i,k] * b[k,j]\n", "# c[i,j] = (a[i,:] * b[:,j]).sum()\n", "mr = torch.einsum('ik,kj->ikj', m1, m2)\n", "mr.shape" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor([[-10.94, -0.68, -7.00, -4.01, -2.09, -3.36, 3.91, -3.44, -11.47, -2.12],\n", " [ 14.54, 6.00, 2.89, -4.08, 6.59, -14.74, -9.28, 2.16, -15.28, -2.68],\n", " [ 2.22, -3.22, -4.80, -6.05, 14.17, -8.98, -4.79, -5.44, -20.68, 13.57],\n", " [ -6.71, 8.90, -7.46, -7.90, 2.70, -4.73, -11.03, -12.98, -6.44, 3.64],\n", " [ -2.44, -6.40, -2.40, -9.04, 11.18, -5.77, -8.92, -3.79, -8.98, 5.28]])" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "mr.sum(1)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor([[-10.94, -0.68, -7.00, -4.01, -2.09, -3.36, 3.91, -3.44, -11.47, -2.12],\n", " [ 14.54, 6.00, 2.89, -4.08, 6.59, -14.74, -9.28, 2.16, -15.28, -2.68],\n", " [ 2.22, -3.22, -4.80, -6.05, 14.17, -8.98, -4.79, -5.44, -20.68, 13.57],\n", " [ -6.71, 8.90, -7.46, -7.90, 2.70, -4.73, -11.03, -12.98, -6.44, 3.64],\n", " [ -2.44, -6.40, -2.40, -9.04, 11.18, -5.77, -8.92, -3.79, -8.98, 5.28]])" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "torch.einsum('ik,kj->ij', m1, m2)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "def matmul(a,b): return torch.einsum('ik,kj->ij', a, b)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "test_close(tr, matmul(x_train, weights), eps=1e-3)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "15.1 ms ± 176 µs per loop (mean ± std. dev. of 7 runs, 5 loops each)\n" ] } ], "source": [ "%timeit -n 5 _=matmul(x_train, weights)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## pytorch op" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "We can use pytorch's function or operator directly for matrix multiplication." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "test_close(tr, x_train@weights, eps=1e-3)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "15.2 ms ± 96.2 µs per loop (mean ± std. dev. of 7 runs, 5 loops each)\n" ] } ], "source": [ "%timeit -n 5 _=torch.matmul(x_train, weights)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## CUDA" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "def matmul(grid, a,b,c):\n", " i,j = grid\n", " if i < c.shape[0] and j < c.shape[1]:\n", " tmp = 0.\n", " for k in range(a.shape[1]): tmp += a[i, k] * b[k, j]\n", " c[i,j] = tmp" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor([[-10.94, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00],\n", " [ 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00],\n", " [ 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00],\n", " [ 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00],\n", " [ 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00]])" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "res = torch.zeros(ar, bc)\n", "matmul((0,0), m1, m2, res)\n", "res" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "def launch_kernel(kernel, grid_x, grid_y, *args, **kwargs):\n", " for i in range(grid_x):\n", " for j in range(grid_y): kernel((i,j), *args, **kwargs)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor([[-10.94, -0.68, -7.00, -4.01, -2.09, -3.36, 3.91, -3.44, -11.47, -2.12],\n", " [ 14.54, 6.00, 2.89, -4.08, 6.59, -14.74, -9.28, 2.16, -15.28, -2.68],\n", " [ 2.22, -3.22, -4.80, -6.05, 14.17, -8.98, -4.79, -5.44, -20.68, 13.57],\n", " [ -6.71, 8.90, -7.46, -7.90, 2.70, -4.73, -11.03, -12.98, -6.44, 3.64],\n", " [ -2.44, -6.40, -2.40, -9.04, 11.18, -5.77, -8.92, -3.79, -8.98, 5.28]])" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "res = torch.zeros(ar, bc)\n", "launch_kernel(matmul, ar, bc, m1, m2, res)\n", "res" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from numba import cuda" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "def matmul(grid, a,b,c):\n", " i,j = grid\n", " if i < c.shape[0] and j < c.shape[1]:\n", " tmp = 0.\n", " for k in range(a.shape[1]): tmp += a[i, k] * b[k, j]\n", " c[i,j] = tmp" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "@cuda.jit\n", "def matmul(a,b,c):\n", " i, j = cuda.grid(2)\n", " if i < c.shape[0] and j < c.shape[1]:\n", " tmp = 0.\n", " for k in range(a.shape[1]): tmp += a[i, k] * b[k, j]\n", " c[i,j] = tmp" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "r = np.zeros(tr.shape)\n", "m1g,m2g,rg = map(cuda.to_device, (x_train,weights,r))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(50000, 10)" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "r.shape" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(3125, 1)" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "TPB = 16\n", "rr,rc = r.shape\n", "blockspergrid = (math.ceil(rr / TPB), math.ceil(rc / TPB))\n", "blockspergrid" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "matmul[blockspergrid, (TPB,TPB)](m1g,m2g,rg)\n", "r = rg.copy_to_host()\n", "test_close(tr, r, eps=1e-3)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "3.61 ms ± 708 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" ] } ], "source": [ "%%timeit -n 10\n", "matmul[blockspergrid, (TPB,TPB)](m1g,m2g,rg)\n", "r = rg.copy_to_host()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "m1c,m2c = x_train.cuda(),weights.cuda()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "r=(m1c@m2c).cpu()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "458 µs ± 93.1 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" ] } ], "source": [ "%timeit -n 10 r=(m1c@m2c).cpu()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Our broadcasting version was >500ms, and our CUDA version is around 0.5ms, which is another 1000x improvement compared to broadcasting. So our total speedup is around 5 million times!" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" } }, "nbformat": 4, "nbformat_minor": 2 }