{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# A CNN Mnist Model" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Installing packages:\n", "\t.package(path: \"/home/jupyter/notebooks/swift/FastaiNotebook_05b_early_stopping\")\n", "\t\tFastaiNotebook_05b_early_stopping\n", "With SwiftPM flags: []\n", "Working in: /tmp/tmph4filx0a/swift-install\n", "warning: /home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)\n", "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", "warning: /home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)\n", "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", "warning: /home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)\n", "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", "warning: /home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)\n", "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", "warning: /home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)\n", "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", "warning: /home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)\n", "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", "warning: /home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)\n", "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", "warning: /home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)\n", "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", "warning: /home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)\n", "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", "warning: /home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)\n", "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", "warning: /home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)\n", "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", "warning: /home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)\n", "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", "warning: /home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)\n", "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", "/home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)\n", "/home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)[1/10] Compiling FastaiNotebook_05b_early_stopping 03_minibatch_training.swift\n", "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", "[2/10] Compiling FastaiNotebook_05b_early_stopping 02_fully_connected.swift\n", "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", "[3/10] Compiling FastaiNotebook_05b_early_stopping 05b_early_stopping.swift\n", "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", "[4/10] Compiling FastaiNotebook_05b_early_stopping 05_anneal.swift\n", "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", "[5/10] Compiling FastaiNotebook_05b_early_stopping 02a_why_sqrt5.swift\n", "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", "[6/10] Compiling FastaiNotebook_05b_early_stopping 00_load_data.swift\n", "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", "[7/10] Compiling FastaiNotebook_05b_early_stopping 01_matmul.swift\n", "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", "[8/10] Compiling FastaiNotebook_05b_early_stopping 04_callbacks.swift\n", "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", "[9/10] Compiling FastaiNotebook_05b_early_stopping 01a_fastai_layers.swift\n", "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", "[10/11] Merging module FastaiNotebook_05b_early_stopping\n", "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", "/home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)[11/12] Compiling jupyterInstalledPackages jupyterInstalledPackages.swift\n", "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", "[12/13] Merging module jupyterInstalledPackages\n", "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", "/home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)\n", "/home/jupyter/swift/usr/bin/swift-autolink-extract: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift-autolink-extract)\n", "[13/13] Linking libjupyterInstalledPackages.so\n", "Initializing Swift...\n", "Installation complete!\n" ] } ], "source": [ "%install-location $cwd/swift-install\n", "%install '.package(path: \"$cwd/FastaiNotebook_05b_early_stopping\")' FastaiNotebook_05b_early_stopping" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "//export\n", "import Path\n", "import TensorFlow\n", "import Python" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import FastaiNotebook_05b_early_stopping" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "('inline', 'module://ipykernel.pylab.backend_inline')\n" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "%include \"EnableIPythonDisplay.swift\"\n", "IPythonDisplay.shell.enable_matplotlib(\"inline\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Load data" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "let data = mnistDataBunch(flat: false, bs: 512)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Batch size: 512\r\n", "Example side size: 28\r\n", "Class count: 512\r\n" ] } ], "source": [ "let firstBatch = data.train.ds.first(where: { _ in true })!\n", "let batchShape = firstBatch.xb.shape\n", "let batchSize = batchShape.dimensions[0]\n", "let exampleSideSize = batchShape.dimensions[1]\n", "assert(exampleSideSize == batchShape.dimensions[2])\n", "print(\"Batch size: \\(batchSize)\")\n", "print(\"Example side size: \\(exampleSideSize)\")\n", "\n", "let classCount = firstBatch.yb.shape.dimensions[0]\n", "print(\"Class count: \\(classCount)\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "▿ [512, 28, 28]\n", " ▿ dimensions : 3 elements\n", " - 0 : 512\n", " - 1 : 28\n", " - 2 : 28\n" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "firstBatch.xb.shape" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "// export\n", "extension Learner {\n", " public class AddChannel: Delegate {\n", " public override func batchWillStart(learner: Learner) {\n", " learner.currentInput = learner.currentInput!.expandingShape(at: -1)\n", " }\n", " }\n", " \n", " public func makeAddChannel() -> AddChannel { return AddChannel() }\n", "}" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "//export \n", "public struct CnnModel: Layer {\n", " public var convs: [FAConv2D]\n", " public var pool = FAGlobalAvgPool2D()\n", " public var linear: FADense\n", " \n", " public init(channelIn: Int, nOut: Int, filters: [Int]){\n", " let allFilters = [channelIn] + filters\n", " convs = Array(0..(filters.last!, nOut)\n", " }\n", " \n", " @differentiable\n", " public func callAsFunction(_ input: TF) -> TF {\n", " return linear(pool(convs(input)))\n", " }\n", "}" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "let model = CnnModel(channelIn: 1, nOut: 10, filters: [8, 16, 32, 32])" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[512, 10]\r\n", "[-0.23888946, -0.17904489, 0.27589405, 0.22169928, 0.2614417, -0.2746509, -0.0112212,\r\n", " -0.13185278, -0.13168347, 0.16951849]\r\n" ] } ], "source": [ "// Test that data goes through the model as expected.\n", "let predictions = model(firstBatch.xb.expandingShape(at: -1))\n", "print(predictions.shape)\n", "print(predictions[0])" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Compare training on CPU and GPU" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "func optFunc(_ model: CnnModel) -> SGD { return SGD(for: model, learningRate: 0.4)}\n", "func modelInit() -> CnnModel { return CnnModel(channelIn: 1, nOut: 10, filters: [8, 16, 32, 32]) }\n", "let learner = Learner(data: data, lossFunc: softmaxCrossEntropy, optFunc: optFunc, modelInit: modelInit)\n", "let recorder = learner.makeDefaultDelegates(metrics: [accuracy])\n", "learner.addDelegates([learner.makeNormalize(mean: mnistStats.mean, std: mnistStats.std),\n", " learner.makeAddChannel()])" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Epoch 0: [0.38877672, 0.8857] \n", "average: 2399.418786 ms, min: 2399.418786 ms, max: 2399.418786 ms \n" ] } ], "source": [ "// This happens on the GPU (if you have one and it's configured correctly).\n", "// I tried this on a GCE 8vCPU 30GB + Tesla P100:\n", "// - time: ~4.3s\n", "// - nvidia-smi shows ~10% GPU-Util while this is running\n", "time { try! learner.fit(1) }" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Epoch 0: [0.38036653, 0.8828] \n", "average: 3779.773271 ms, min: 3779.773271 ms, max: 3779.773271 ms \n" ] } ], "source": [ "// This happens on the CPU.\n", "// I tried this on a GCE 8vCPU 30GB + Tesla P100:\n", "// - time: ~6.3s\n", "// - nvidia-smi shows 0% GPU-Util while this is running\n", "time {\n", " withDevice(.cpu) { try! learner.fit(1) }\n", "}" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Collect Layer Activation Statistics" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "class ActivationStatsHook {\n", " var means: [Float] = []\n", " var stds: [Float] = [] \n", " func update(_ act: TF) {\n", " means.append(act.mean().scalarized())\n", " stds.append (act.std() .scalarized())\n", " }\n", "}" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "let learner = Learner(data: data, lossFunc: softmaxCrossEntropy, optFunc: optFunc, modelInit: modelInit)\n", "let recorder = learner.makeDefaultDelegates(metrics: [accuracy])\n", "learner.addDelegates([learner.makeNormalize(mean: mnistStats.mean, std: mnistStats.std),\n", " learner.makeAddChannel()])" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "var statHooks: [ActivationStatsHook] = (0.." ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" }, { "data": { "text/plain": [ "None\n" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "for stat in statHooks {\n", " plt.plot(stat.means)\n", "}\n", "plt.legend(Array(1...statHooks.count))\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3dd3gc1fXw8e/dql6takmWG24UGxtsMBibkoDpJQRD3gQSMPCDBEJC781AKIGEagKhhpCEZmMCmIDBGIxxo7jLVZLVe9163z9mVSxLsspqd7U6n+fZZ2Zn7s6cXa3O3rl35o7SWiOEEGLwMwU7ACGEEP4hCV0IIcKEJHQhhAgTktCFECJMSEIXQogwYQnWjocNG6Zzc3ODtXshhBiU1qxZU661TulsXY8SulJqF1AHeAC31npah/UKeAKYCzQCF2ut13a3zdzcXFavXt2T3QshhPBRSu3ual1vauhztNblXaw7BRjre0wHnvFNhRBCBIi/2tDPBF7RhpVAglIqw0/bFkII0QM9Tega+FgptUYpNb+T9cOB/HbPC3zL9qGUmq+UWq2UWl1WVtb7aIUQQnSpp00ux2itC5VSqcBSpdRmrfUXvd2Z1nohsBBg2rRp+4054HK5KCgooLm5ubebDqiIiAiysrKwWq3BDkUIIVr1KKFrrQt901Kl1DvAkUD7hF4IZLd7nuVb1isFBQXExsaSm5uL0c8aerTWVFRUUFBQwMiRI4MdjhBCtDpgk4tSKlopFdsyD/wE+LFDsUXAL5VhBlCjtS7qbTDNzc0kJyeHbDIHUEqRnJwc8kcRQoihpyc19DTgHV+StQD/0Fp/qJS6AkBr/SzwAcYpi3kYpy1e0teAQjmZtxgMMQohhp4DJnSt9Q7gsE6WP9tuXgNX+Tc0IYTwL601Ne++h330KCIPPTTY4fidXPrfwa9//WtSU1M5+OCDgx2KEMLPyp96mqKbb2bX+T8n/8r/o3nLlmCH5FeS0Du4+OKL+fDDD4MdhhDCz6r+9S/Kn3yS+DPPJOXaa2hcvZqdZ55F4XXX4dixM9jh+YUk9A5mzZpFUlJSsMMQQvhR3aefUXzX3UTPOpaM++5l2BVXMOaTpSRfcTl1yz5nx2mnsfeWW3EW9PrkvJAStMG5DuTuxRvYuLfWr9ucmBnHnadP8us2hRChrWn9egqvu46IiRPJ+vOfUb7rR8zx8aReey1J/+//UbHweareeIOaxYtJ/Nl5JF9+Bda01CBH3ntSQxdChC3Hjp3kX3ElltRUsp97FlN09H5lLMnJpN18E6M//oiEc86h6l//ZvtPfkLJnx7GXVUVhKj7LmRr6FKTFkL0h6u0lPzLLgOTiZy/PY8lObnb8tb0dDLuvovkS39D+ZNPUfnSS1T/858kXfwrki65BHNsbIAi7zupoQshwo6nvp78y6/AXVVF9nPPYcvJ6fFrbdnZZD70IKMWLyL62GMpf/oZ8k48ifKFz+NtbBzAqPtPEnoH8+bN46ijjmLLli1kZWXxwgsvBDskIUQvaKeTwt/9DsfWrWQ9/mciD+nbKcj20aPJeuJxRr79FlGTJ1P22GPknfQTKl95Be10+jlq/wjZJpdgeeONN4IdghCij7TXy95bb6Phq6/JWLCAmFmz+r3NiIkTyX7uWRrXrqPsiScoWfAAjrztZNxztx8i9i+poQshwkbZY49Ru3gxKddeS8I5Z/t121GHT2HEyy+R9KtfUv3vf9P0ww9+3b4/SEIXQoSFyldepeJvL5B44TySL+/stg3+Mey3v8U8LJnie+9De70Dtp++kIQuhBj0av/7X0oeeIDYk04k7dZbB3QAPXNMDGnXX0/z999T8/bbA7afvpCELoQY1BpWrWLvDTcSOWUKmQ8/jDKbB3yfcaefTuTUqZQ++hiempoB319PSUIXQgxazVu2UnDV1Vhzcsh++ilMEREB2a9SivTbb8NTU0PZX/4akH32hCR0IcSg5CoqIn/+fEyRkeQ8vxBzQkJA9x8xfjyJ8+ZR9cYbNG/aFNB9d0USegf5+fnMmTOHiRMnMmnSJJ544olghySE6MBTXc2eyy7D29BA9vMLsWZmBiWOlN/9FnN8vNFBqve7TXLASULvwGKx8Oijj7Jx40ZWrlzJU089xcaNG4MdlhDCx+twkH/V1bh27yHrySeJGDcuaLGY4+NJ/cN1NK1dS+3ixUGLo0WPE7pSyqyUWqeUer+TdRcrpcqUUut9j0v9G2bgZGRkcPjhhwMQGxvLhAkTKCwc3ENqCjGYeR0OHDt2UP/551S+9jr5V1xB05o1ZD70INEzpgc7POLPOYeIQw+l5OGH8dTXBzWW3lwpeg2wCYjrYv2bWuur+x+Sz39vgmI/n7iffgic8mCPi+/atYt169YxfXrwvzRChCutNZ6KCpz5+bgKCozpnnycBfm48gtwl5TsU94UFUXaHbcTN3dukCLelzKZSL/9Nnad/3PKn3yKtJtuDFosPUroSqks4FTgfuC6AY0oRNTX13Puuefy+OOPExfX1W+YEKK3aj/6mKa1a3EWFODaswdnQQG6qWmfMpa0NKzZWUQfdRTW7CxsOTlYs7KwZWdjTk4OuRu1Rx5yCAnnnUflq6+ScO452MeODUocPa2hPw7cAHQ3fuS5SqlZwFbg91rr/I4FlFLzgfkAOQca/awXNWl/c7lcnHvuuVx00UWcc845QYtDiHBTv3w5hddcg4qIwJadhTUrm+ijj8KalW0k7uxsrMOHB+z0Q39Kue731H78McX3LyDn7y8G5UfngAldKXUaUKq1XqOUmt1FscXAG1prh1LqcuBl4PiOhbTWC4GFANOmTQt+l3AntNb85je/YcKECVx33ZA4GBEiILTTScn9C7CNGMHIxYsw2WzBDsmvLImJpFzzO0ruuZe6jz4i7uSTAx5DTzpFZwJnKKV2Af8EjldKvda+gNa6Qmvt8D39GzDVr1EG0IoVK3j11Vf59NNPmTx5MpMnT+aDDz4IdlhCDHqVr7yCc9cu0m69JeySeYvEn/8c+4QJlDz4UFDGTj9gDV1rfTNwM4Cvhv5HrfUv2pdRSmVorYt8T8/A6DwdlI455piQOJ9UiHDiKiml/OlniJkzxy9D2oYqZTaTfvtt7L7wIsqffY7U634f0P33+Tx0pdQ9SqkzfE9/p5TaoJT6DvgdcLE/ghPhR2tN0w8/4G1uDnYoIoBKH3kE7XaTdvNNwQ5lwEUdfjjxZ55Jxd//jmPnzoDuu1cJXWu9TGt9mm/+Dq31It/8zVrrSVrrw7TWc7TWmwciWDG4abeb4jvuZNfPzmfbMcdSdMedNK5bJ0dEYa5xzRpqFy8m6deX9OpWcINZ6h//gMlup2TBAwH9fsuVoiIgvI2NFFx1NdX//jeJF11E7AnHU7NoEbvnXciOU+ZS/txCXB3ONxaDn/Z4KL7vfiwZGQybP3BjlIcaS0oKKb+9mobly6n/7LOA7VcSuhhw7vJydv/yV9QvX076XXeSfvttZD70EGO/XE7G/fdhHpZM2Z//TN6c49lz6WXULFkiTTJhovpf/8KxaRNpN1yPKSoq2OEEVOKFF2IfO4aS+xcE7PssCV0MKMfOneyadyGOvDyynvwriRdc0LrOHBNDwrnnkvvaa4z+6EOSL5+PY/t29v7hj2w7dhZFd91F03ffSZPMIOWuqqLs8SeImj6d2CCcwhdsymol7dbbcBUWUvG3wNxsXhK6GDBN69eze96FeOvrGfHyS8Qev9+lCa1sI0aQes01jPnfJ+T8/UVi5sym5t332PXzC9hx6mmUP/88rpLSAEYv+qvsiSfw1NeTdustIXdlZ6BEz5hO3NxTqHj+eZwFBQO+P0noHTQ3N3PkkUdy2GGHMWnSJO68885ghzQo1f3vf+y++BJMcXHkvvEPIg87rEevUyYT0UcdxfA//YmxXy4n/d57MCckUPboY+TNmcOe+fOp/fBDqbWHuOaNG6l+818kXnghEQcdFOxwgir1hhvAbKbkwYG/+l0Segd2u51PP/2U7777jvXr1/Phhx+ycuXKYIc1qFS98QYFv/0d9rFjyX3jH9hyc/u0HXNMDIk/+xm5/3id0R/+l+T5l+HYuo3Ca39PxXPP+Tdo4Tdaa4rvux9zYiIpv/XfeH2DlTU9nWFXXkH9J/+jfvnyAd2XJPQOlFLExMQAxpguLpdryB4u9pbWmtJHH6P47nuImTWLES+/hCU52S/btuXmknrttYz53yfEzZ1L2ZNP0Szj1Iek2sWLaVq7ltTrfo9ZBrYDIOlXv8KWm0vJfffjdToHbD+9GT43oB5a9RCbK/17Ovv4pPHceOSBh7b0eDxMnTqVvLw8rrrqKhk+twe008ne226jdtFiEs4/n/Q7bkdZ/P/1UmYz6XfcTuPq1ey98UZy//MfTHa73/cj+sZT30Dpw48QccghxMvAdq1MNhtpt95K/mWXUfnSywybf9nA7GdAtjrImc1m1q9fT0FBAatWreLHH38MdkghzVNXx57LL6d20WJSrr2G9LvvGpBk3sKckEDG/ffj2JZH2RN/GbD9iN4rf+Zp3GVlpN92K8ok6aW9mGOPIfakEyl/5hlcxcUDso+QraH3pCY90BISEpgzZw4ffvghBx98cLDDCUmukhLy51+OY/t2Mh54gISzzwrIfmOOPYaEeRdQ+fe/EztnNlFHHBGQ/YquOXbspPKVV4k/55wed4IPNak33kT9F6dS/uyzZNx1l9+3Lz+hHZSVlVFdXQ1AU1MTS5cuZfz48UGOKjQ5tm1j1wXzcOXnk/3sswFL5i3Srr8ea3Y2e2+6Oei3/hrqtNaULFiAyW4P+IBUg4ktazjZzz5D2vXXD8j2JaF3UFRUxJw5czj00EM54ogjOOmkkzjttNOCHVbIaVi1il0XXgRuNyNef42YY2YGPAZTVBSZDz6Iq6goIKeEia7Vf/opDV9+Scpvr8YybFiwwwlp0UcdhSk6ekC2HbJNLsFy6KGHsm7dumCHEbK8Tid1H31M0S23YM3JIWfhc1iHDw9aPFGHTyH50kupWLiQ2ONPIPb4OUGLZajyNjdT8sCD2MaMJvHCC4MdzpAmCX2I0243nupq3BWVeKoqcVdU4KmoxF1ZgaeyyphWVOKprMRdWYm3rg6AyGlTyX7yScwJCUF+B5By9VXUf/EFRbffTuTkRViSkoId0pBS8eKLuAoKyHnp7yirNdjhDGmS0IcYV0kJxXfdjTN/j5Goq6uhs6suTSbMSUlYkpIwJyURMWkS5uRkLMlJWNLSiZt7SsicLqhsNjIfepBd5/2M4jvvYvhfnpBrBwLEVVhIxcLnif3pT4meMSPY4Qx5ktCHEK01xXfdTcPXXxMzaxbmI47AkpSMObktcVuSkzEnJWGOjx9Up51FjBtHyjW/o/SRR6ldvJj4M8448ItEv5X86WEA0m68IciRCJCEPqTUffQR9Z99RuqNN5J8ycXBDsfvki65hLpPP6P43vuIOuIIrBkZwQ4prDV8/TV1H33EsN/9FmtmZrDDEfTiLBellFkptU4p9X4n6+xKqTeVUnlKqW+UUrn+DFL0n6emhuL77idi0iSS/t8vDvyCQUiZzWQ++ADa42HvLbegvd5ghxS2tMtF8f33Y83KIvk3vwl2OMKnN8fU19D1zZ9/A1RprccAfwYe6m9gwr9KH3kET1UVGffdO6BXcQabLSeHtBtvpPHrlVT9441ghxO2qv7xD5x520m7+aaQ6UsRPUzoSqks4FTgb10UORN42Tf/H+AENch7pTweD1OmTAmLc9AbvllF9b//Q/IlFxMxYUKwwxlwCef/jOhZx1L6yCM4dgT2Jr1Dgbu8nLK/Pkn0MccQ080Y9yLwelpDfxy4AejqGHY4kA+gtXYDNcB+w+wppeYrpVYrpVaXlZX1IdzAeeKJJ5gQBsnP29xM8R13YM3OZthVVwU7nIBQSpFx332Y7Hb23nQT2u0OdkhhQXu9vou4HsLrcJB2y9C9cUWoOuCxt1LqNKBUa71GKTW7PzvTWi8EFgJMmzYtZO9QUFBQwJIlS7j11lt57LHHgh1Ov5Q/8yzO3bvJefEFTJGRwQ4nYKypqaTfdSeFv7+OiuefZ9iVVwY7pEHDXVWFc9cunLt2+6a+x+7daN+9MZMvuxT7qJFBjlR01JPG1JnAGUqpuUAEEKeUek1r3b5nrRDIBgqUUhYgHqjoT2DFCxbg2OTf4XPtE8aTfsstByx37bXX8qc//Yk630U0g1Xzli1UvPAC8WefTfTRRwc7nICLO+UU6j75H2VPPU30rFlETpoU7JBChrepCeeePTh37jSS9c62xO2pqWkraLFgy8rClptL9FFHYcvNxTZqpAyGFqIOmNC11jcDNwP4auh/7JDMARYBvwK+Bs4DPtWD9B5h77//PqmpqUydOpVly5YFO5w+0x4PRbffgTkujtQbBmYgoMEg/fbbaPz2W/beeCMj33orrDrwtNbopiY8tbV4amrx1tW2zdfW4Kmtw1PbyXx1De4OTZ6W9HRsubnEnnKykbRzc7Hn5mIdPlyu/hxE+ny6g1LqHmC11noR8ALwqlIqD6gELuj2xT3Qk5r0QFixYgWLFi3igw8+oLm5mdraWn7xi1/w2muvBSWevqp6/XWav/+ezEcewZKYGOxwgqZl7PT8yy6j7PEnBvUFMK6SEmrefY/aDz/EXVKCp64OXK5uX2OKjcUcG4spPh5zXBy23FxMcXGttW5bbi62nJwBGyxKBJYKVkV62rRpevXq1fss27RpU0h1RC5btoxHHnmE99/f79T7kIu1PVdhIdtPP4OoaVPJfu456bgCiu6+m+p/vknOyy8RfeSRPX6d1hp3SQmOLVto3rIVx5YtOPfsIWLSRGJPOonoI48c0Bqs1+mk/tPPqH7nbRqWfwleL5FTp2IfOwZzbBzm+DhMcXGYfQ9TXDzm+DgjicfGoszmAYtNBIdSao3Welpn68L3hOQhSmtN0T33AJBx552SzH3Srr+ehq++ouimmxm56D3MvvvGtudtbMSxbRvNW7bg2LIVx9atNG/dirddm7IlMwPb8Cxq3ltE9T/fxBQXR+yc2cSceCIxxxzjt47n5i1bqH7rLWoXLcZTXY0lLY3k+ZeRcPbZ2EaM8Ms+RPiRhN6N2bNnM3v27GCH0Su1H3xAw+dfkHbLzUEd1jbUtIydvvuiX1Cy4AGGXT6f5q1bjcS9ZQvNW7fg2pPfOlCZKSoK+0EHEXfyydgPGkvEuHHYDzqo9abH3uZmGlasoO7jpdQtW0bNe4tQERG+24ydRMzs2b2+QbKnpoaaJUuoeettmjdsQFmtxJxwAgnnnkP00UdLbVsckDS59FEoxuquqmLHqadhzcoi941/SALoROmfH6fiuefaFiiFbcQI7OPGtSXuceOMzsAeDk6mXS4av/2Wuk8+oW7pJ0aHo8VC9PTpxJ50IrEnnIAlJaXz13q9NHz9NTVvv0Pd0qVopxP7+PEknHsucaedOqT7P0TnumtykYTeR6EY696bb6Fm8WJGvvUfIsaNC3Y4IUk7nVS+8grmhAQjiY8Z49fz87XXS/P331P3ySfULl2Ka/ceUIrIyZOJPekkYk86EVt2Ns6CAmrefofqd9/BvbcIU3w88aedRsK55xAxcaLf4hHhZ1Al9PHjx4d8u6/Wms2bN4dUQm/4+mv2XPJrkufPl3s6hgitNY5t26hbupS6T/6HY5MxFJI1KwtXQQEoRfTRR5Nw7jnEnHBCWJ1SKQbOoEnoO3fuJDY2luTk5JBN6lprKioqqKurY+TI0LhSztvUxI4zzwIFo957D1NERLBDEp1w5udT98n/aFy5ksjJhxF/5pky7KzotUFzlktWVhYFBQWE+jgvERERZGVlBTuMVuVPPYVrzx5yXnpJknkIs2Vnk3zJxWE5Fr0IDSGV0K1Wa8jUegeL5o0bqfj7S8Sfdy7RM6YHOxwhRBANnnuMif1ot9u4vD8xkbTrh+7l/UIIQ0jV0EXvVL76Gs0bNjD8z49hjo8PdjhCiCCTGvog5SwooOwvfyFm9mxiTz452OEIIUKAJPRBSGtN8Z13oZQi/c47QvaMICFEYElCH4RqFy+mYcUKUq67Tu5sL4RoJW3oIU5rjae8HGd+Pq78fJx78ql6/XUiDzuMxHn9HqVYCBFGJKGHAO104iwsNBJ2fj6uPfk4Cwpw7dmDs6AA3dTUVlgpbLm5ZNx/n4zVIoTYhyR0P9Naoxsb8TQ04K1vwNvQgLeh3jdtwFNfj7e21kjY+QU48/fgLipuHeUPQEVEYMvOwpqdQ/TRR2HNzml9bh2eKZeICyE6JQm9F7yNjdQvW0b9ihV4a2uN5NzQaCTr+rakTQ+GUzAnJ2PLziZq6jRs2dlYc7KNaXY2lpQU6egUQvTaARO6UioC+AKw+8r/R2t9Z4cyFwMPY9wsGuBJrfXf/BuqoXHtOkoWLCDmuOOIOW4WEQcf3ONhTvvC63DQsHw5tR98QN1ny9BNTZgTErCkpGCKicEcF4c1IwNTTDSmaONhjolpnTdFx7Rb7lsWGyuX6Ash/K4nNXQHcLzWul4pZQW+VEr9V2u9skO5N7XWV/s/xH1ppxNlsVD+9NOUP/UU5qQkYo49lpjjZhE9c6ZfLrDRLhcNX39N7ZIPqPvf//DW12NOTCT+zDOImzuXqKlTpf1aCBFyDpjQtTEcY73vqdX3CM4QjUD0jOlE//MN3FVVNHy5gvrPP6d+2TJq3nsPzGYiJ09urb3bDzqox00X2uOh8dtvjST+8cd4amowxcYS+5OfEDd3LtEzpqMs0kIlhAhdPRo+VyllBtYAY4CntNY3dlh/MfAAUAZsBX6vtc7vZDvzgfkAOTk5U3fv3t3f+AEjGTd9/z31X3xB/eef49hojDttSUsjZtYsYmYfR/SMGfvd2Vx7vTStX0/tkg+o/egjPOXlqKgoYo8/3kjix8zEZLP5JUYhhPAHv42HrpRKAN4Bfqu1/rHd8mSgXmvtUEpdDvxca318d9vqbDx0f3GVlNLw5XLqP/+ChhUr8DY0oKxWoo6YRvSsWUSMH0/9519Q++GHuIuKUHY7MccdR9zcucQcN8uvd7ARQgh/8usNLpRSdwCNWutHulhvBiq11t02Zg9kQm9PO500rl3XWnt3bt9urLBaiZk5k7hT5xIz53jMMdHdb0gIIUJAv25woZRKAVxa62qlVCRwEvBQhzIZWusi39MzgE39jNlvlM1mtLvPmE7aDdfjLCjAsXUrUVOnygiFQoiw0pNevgzgZV/N2wT8S2v9vlLqHmC11noR8Dul1BmAG6gELh6ogPvLlpWFLYTuNiSEEP4SUvcUFUII0b3umlxktEUhhAgTktCFECJMSEIXQogwIQldCCHChCR0IYQIE5LQhRAiTEhCF0KIMCEJXQghwoQkdCGECBOS0IUQIkxIQhdCiDAhCV0IIcKEJHQhhAgTktCFECJMSEIXQogwIQldCCHChCR0IYQIEwdM6EqpCKXUKqXUd0qpDUqpuzspY1dKvamUylNKfaOUyh2IYIUQQnStJzV0B3C81vowYDJwslJqRocyvwGqtNZjgD/T4SbSQgghBt4BE7o21PueWn2PjjciPRN42Tf/H+AEpZTyW5RCCCEOqEdt6Eops1JqPVAKLNVaf9OhyHAgH0Br7QZqgOROtjNfKbVaKbW6rKysf5ELIYTYR48Sutbao7WeDGQBRyqlDu7LzrTWC7XW07TW01JSUvqyCSGEEF3o1VkuWutq4DPg5A6rCoFsAKWUBYgHKvwRoBBCiJ7pyVkuKUqpBN98JHASsLlDsUXAr3zz5wGfaq07trMLIYQYQJYelMkAXlZKmTF+AP6ltX5fKXUPsFprvQh4AXhVKZUHVAIXDFjEQgghOnXAhK61/h6Y0snyO9rNNwM/829oQgghekOuFBVCiDAhCV0IIcKEJHQhhAgTktCFECJMSEIXQogwIQldCCHChCR0IYQIE5LQhRAiTEhCF0KIMCEJXQghwoQkdCGECBOS0IUQIkxIQhdCiDAhCV0IIcKEJHQhhAgTktCFECJMSEIXQgwpze5mXB5XsMMYEAe8Y5FSKht4BUgDNLBQa/1EhzKzgfeAnb5Fb2ut7/FvqEII0XebKzfz+qbX+WDHBzi9TmKtsSRGJJIUkdQ6bXkkRiSSGJFIckRy67zVZA32WzigntxT1A38QWu9VikVC6xRSi3VWm/sUG651vo0/4cohBB94/a6+XTPp7y+6XXWlq4l0hLJmWPOJD06ncrmytZHYX0hP5T/QFVzFR7t6XRbsbZYkiOSSYpI4ryDzuP00acH+N0cWE/uKVoEFPnm65RSm4DhQMeELkSXtNZ8X/4972x7h88LPsdqshJvjyfeFk+cPa51Pt4e3+XyCEtEsN+GGCSqmqt4a9tb/HPzPylpLGF4zHCun3Y9Z409izhbXJev82ovdc66fZJ9VXMVFc0VVDVXUdlcyfbq7dzy5S18W/wtN0+/mUhLZADfWfd6UkNvpZTKxbhh9DedrD5KKfUdsBf4o9Z6Qyevnw/MB8jJyeltrGIQKmssY/GOxbyb9y47a3YSaYnkuKzjsJlt1DpqqXHWsL16OzWOGmqcNbi97i63ZTfbibfFkxyZzK8P/jUnjzw5gO9E+IPWmm+Lv8VmtjEuaZzfk+Hmys38Y9M/WLJjCU6vkxkZM7htxm0cO/xYzCbzAV9vUqbWSsXI+JGdlvF4PTz93dM8//3z/FD+A4/NfqzLsoGmtNY9K6hUDPA5cL/W+u0O6+IAr9a6Xik1F3hCaz22u+1NmzZNr169uo9hi1Dm8rhYVrCMd/PeZUXhCjzaw5TUKZw15ix+mvtToq3Rnb5Oa02Tu6k1udc4avaZb/kB2FC+gS1VWzhl5CncOv1W4u3xAX6Hoi9qHDXc9dVdfLLnE8BInqPiRzEhaQITkycyIXkC45PGd/n96Irb6+az/M94fdPrrClZQ6QlktNHnc6FEy5kdMLogXgrAKwoXMHNy2/G4XFw51F3MnfU3AHbV3tKqTVa62mdrutJQldKWYH3gY+01o/1oPwuYJrWuryrMpLQg6fJ3YTdbMek/HuS05bKLbyb9y5LdiyhylFFajAR2h0AAB+eSURBVGQqp48+nTPHnOnXGozb6+bFH1/kmfXPkBSRxD0z72Hm8Jl+277wv2+Lv+Wm5TdR2VzJ1ZOvZlT8KDZWbmRTxSY2VmykrKkMAIViRNwIJiRPYFLyJCYkTWB88vhOm0lamlXe3PImxQ3FDI8Zzrzx8zhrzFkB+5Evbijmhi9uYF3pOs4/6HxuOPIG7Gb7gO6zXwldKaWAl4FKrfW1XZRJB0q01lopdSTwH2CE7mbjktADr9ndzBNrn+D1Ta9jN9vJis0iOzabnNgccuJyyIrNIic2h/TodCymnrXG1ThqWLJjCe/mvcumyk1YTBbmZM/hrDFncXTm0T3eTl9sqtjELV/eQl51HucfdD5/mPYHoqxRA7Y/0Xsur4tn1j/D3374GzlxOTw06yEmJU/ar1xZYxmbKo3kvqliExsrN1LcUNy6Pjs2u7UmPzJ+JJ8XfM6SHUtweBxMz5jOReMvYlbWrB41q/iby+vir+v+yt9//DsTkibw6HGPkh2XPWD7629CPwZYDvwAeH2LbwFyALTWzyqlrgauxDgjpgm4Tmv9VXfblYQeWBsrNnLz8pvZUbODs8ecTZwtjj11e8ivyye/Lh+Hx9Fa1mKykBWT1Zrgc+JyyI7NJjs2m+ExwzErM1/t/Yp3897ls/zPcHldjE8az1ljzuLUkaeSEJEQsPfl8Dh4ct2TvLzhZbJis1hwzAImp04O2P5F1/Jr87lx+Y38UP4D54w9hxuPuLFXP7iVzZVsqtjUmug3VmyksL4QgAhzBKePPp154+cxNrHb1t2AWZa/jFu/vBWv9nLvzHs5ccSJA7Kffje5DARJ6IHh9rp54YcXePa7Z0mKSOLeY+7l6Myj9ynj1V5KG0tbk/ue2j3sqdtDQV0Be+r20OBqaC1rUiaiLFHUu+pJsCdw6qhTOWvMWYxPGh/ot7aP1cWruW3FbRQ1FHHJpEv4v8n/h81sC2pMQ9ni7Yu5b+V9mE1m7jzqTn6a+1O/bLfGYXSij04YHZJ9J4X1hVz/+fX8UP4Dv5jwC66beh1Ws3/PX5eEPkTtrt3NLV/ewvdl33NK7incOqP3HYhaayqbK41E76vRlzWWcXTm0czOnh1SSbPB1cDD3z7MW9ve4qDEg1hwzALGJY0LdlhDSp2zjvu/uZ8lO5ZweOrhPHjsg2TEZAQ7rIByeVw8tuYxXtv0GocMO4RHjnuEzJhMv21fEvoQo7XmX1v+xaNrHsVisnD7jNs5ZeQpwQ4rYD7P/5w7v7qTGmcNV0++mosnXRyUttWhZn3pem5afhPFDcVcediVXHrIpUP6c1+6eyl3rLgDkzJx/zH3Mzt7tl+2Kwl9CCltLOWOr+5gReEKjs48mnuOvoe06LRghxVwVc1V3LvyXpbuXsqU1CncP/P+Ae2oGso8Xg/P//A8z373LOnR6Tx47IPSj+Gzp3YPf/j8D2yu3Mwlky7ht4f/tt9DCEhCHyI+2vUR9668F4fbwXXTruOCcRdgnKQ0NGmtWbJzCQtWLsCt3fxx2h/52UE/G9Kfib8V1Rdx0/KbWFu6lrkj53LbjNuItcUGO6yQ4vA4eGjVQ/x767+ZkjqFP836E+nR6X3eniT0MFfrrGXBNwtYsmMJBycfzIJjF4TMlWuhoLihmNtX3M7KopXMHD6Te46+h9So1GCHNeh9tOsj7v76bjxeD7fNuC0kxzYJJUt2LOHur+8mwhzBA8c+0OdrJyShh7GVRSu57cvbKG8q5/JDL+fSQy8dFKPCBZpXe3lzy5s8tvoxrGYrU9Omtp6KmRWT1XpKpr/PSAhHja5GHlz1IO/kvcOhww7lwWMflOasHtpRs4M/LPsDZ485m19O+mWftiEJPQw1u5t5fO3jvL7pdXLjcnng2Ac4eNjBwQ4r5O2q2cVT658irzqPwvpCmtxNretMykR6VHrrBVdZsVmt89mx2d0O6hRILo+LRncjTo8Tp9dpTFseXicOjwOXx4XD48DpdbbNtyvv9rpxeV24vK7WebfXjcvjwq2N6X7rfOVbBqy69JBLuXLylVKB6KVmdzN2s73PTX+S0MPMhvIN3Pzlzeys2cm88fP4/dTfh9SIb4OF1prypnIK6gtaz8HPr8unoM54XtlcuU/5eHt8a20+MyaTzOhMMmIyyIjOIDMms9djkHTH5XFRUF/Anto97KrdxZ7aPeyu3c3uut37XEHZVxZlwWKyYDVZsZqtWJQFq9mK1WRtXd66vv282YrNbOPcsedyRPoRfninore6S+gDd112mNJaU+uspcndhNPj3Kfm0zLv8Dj2mW+pPe2zrJOalcvjalve8rxjLczrxKu9pEal8txJz+13kZDoOaUUKVEppESlMCV1yn7rG1wNFNQVtCb4lseGig18sueT/UaGjLPFkRmT2ZrgW6YtiT/RnrhPrczj9VDUUNSWtOt8Sbt2N3vr9+4zLnecLY4RcSOYljaNnNgcYmwx2Ew2bGbjYTfbsZltWE3WTudbH77X+HscHxEaBl0NfXXxav667q+tNaOM6AzSo9Nb52NsMf2Oze11U9RQRH5t/j7/yPn1Ru2t/WF6bygUdrPdqOW0+2e0mqz7/LNZzVbsJnuX62NtsZwz9pyQvFJuqPBqL+VN5eyt30tRQ1Gn00Z34z6viTBHkB6dTmpUKuVN5eTX5ePytt0KLcoSxYi4EeTE5TAiboQxH5tDblxuQIdTEKEtrGroHu1BKcX60vV81PARbr1vLSnWGkt6THqnyT4jOoOUqBQsJguNrsbWQ+2ONbCi+qJ9tmsz2VrbUqenTyczJpMoS1RrzaglSbfM77Pc1LbcYrLIKXNhwqRMpEalkhqVymT2P+e65Uius2Rf2ljKyPiRHJd9HCNiR7Qm72GRw+T7Ifpl0NXQ2/N4PZQ3lVPUUERxQzFFDUWtj5bnNY6afV5jVmZibbFUO6r3WR5ri20debD17AdfEk+NSpVDVCFESAirGnp7ZpOZtOi0bq+EbHQ17pfsq5qrSI9Ob03c2bHZ0nwhhBj0BnVC74koaxSjEkYxKmFUsEMRQogBJe0IQggRJiShCyFEmJCELoQQYeKACV0pla2U+kwptVEptUEpdU0nZZRS6i9KqTyl1PdKqcMHJlwhhBBd6UmnqBv4g9Z6rVIqFlijlFqqtd7YrswpwFjfYzrwjG8qhBAiQA5YQ9daF2mt1/rm64BNwPAOxc4EXtGGlUCCUmpo3XdKCCGCrFdt6EqpXGAK8E2HVcOB/HbPC9g/6aOUmq+UWq2UWl1WVta7SIUQQnSrxwldKRUDvAVcq7Wu7cvOtNYLtdbTtNbTUlJS+rIJIYQQXehRQldKWTGS+eta67c7KVIItB/hPsu3TAghRID05CwXBbwAbNJaP9ZFsUXAL31nu8wAarTWRX6MUwghxAH05CyXmcD/A35QSq33LbsFyAHQWj8LfADMBfKARuAS/4cqhBCiOwdM6FrrL4Fux/TUxpCNV/krKBFGvB4o2wJ718LedVC4Fko3GstNFt/D3MW0w3rlm1ojYMIZcNg8sEUF+x0KETLCfnAu0YHXC2WbwBYNMWlg9eOt67SGqp1G0m5J3kXfgavBWG+LhczJMO3XYLGD120k9k6n7R7au+/zumJYch18eh8c8Rs44jKI7XrETSGGCknoQ4WzEb7/J6x8Bsq3ti23x0NMKsSmGwk+Js1Ijq3zvuWRidDx5gu1e33Je21bEm/2jTNvtkPGoTDlIsg8HIYfDsljweSH0Sa0ht1fwddPwRePwIon4JDz4aj/g7RJ/d++EIPUoL7BheiB2r2w6nlY83doqoKMyTDtEqPpoq4Y6kuh3jetK4b6EnA17r8dk7Ut2dtjoXSz8TowmkLSJkLmlLbknToRzAG4G3x5HnzzDKx7HdxNMPp4OOoqGH3C/j9AQgB4XMaRXmsz3uAa0qq7G1xIQg9XhWth5dOw4R2jyWL8qTDjKsiZ0X2i0xqc9VBXYiT3jsm+vsT4YRh2UFvyTj/Ev003fdFYCatfhFULjRhTJhiJ/ZCfGW3uYmjyuKBkg3H02NKPU7rJSOitVIc+G9O+z5V5334dswVyZ8H0yyFxRMDfkiT0ocLrgc3vw9dPQ/5Ko8368F/C9PmQmBvs6ALD7YAf34avn4SSHyE6BY6cb7TbRw8LdnSicgd8/y9QJogbDvHDIS4L4jL738Ht9RjNiS39N3vXQvGP4HEY6yMSfEeRkyEivl2fTft+Gw/ojv04HfpwnPWw8wujojThDDjqasg+ov+fTQ9JQg93zTWw9lVY9RxU74GEETD9CpjyC4iIC3Z0waE17PwcvnoS8paCJQIOu8A4Skk5KNjRDS1eL2z/1Dh62vaxcYSovfuXi0xqS/Dxw30JP6st8cdmgsVmlNXa+HHYu66LDvgYo3kxc7JxFJk5BRJH+q8ZrqbAeD+rXwJHDWQdaRwRjj/NqMEPIEno4apyB3zzHKx7zag1jJgJM66EcXONQ0RhKN1sND9990+jtjb2pzB5Hoz9iXG2jxgYTdWw/h/w7fPGdzUmDaZeAlMvNjrZ6/ZCTSHUFhoJsrZw3+fN1ftvM9rXgV+926jIgPFjnX6I0QSYOcXXAT8mMP8DjnpY/7rx/araBQk5MP3KAa1MSUIfLLRud/jn67jxuPd/XlsA374Am5cYbXoHn2Mk8swpwX4Hoa2+DFa/YHx2DaVgjYKxJ8HEs+Cgn0py95eSjUYS/+5No8acPd1o9ppwRlsNuyecDb4EX7Bvoq8rNmrsLQk8dUJgOuC74/XAlg+MM6/2fA32OF9z5xWQkH3g1/dCeCX00s1GR19UMkQnG9OoYb5pcu++MD3haoamSqPTranKmG+uMZa7m402W3fH+R5Mu0rYPRWZZLQLH3EpxMlIxb3i9cDuFbDhXdi02EjulkgjuU86y6jB22OCHeXg4nEbCW3VQti13Kg1H3KecY1A5uRgRxdYBWtg5VPG9wuM79SMqyBrql82H14J/ce34D+/7nq9PQ6ikvZN8h0Tvz0WHHX7J+qmqnbPffPupgPHZLIYX2CLvWdTs82oUZgsxumAJnM3z1t61ts9t0XDqNnBP7MkHHg9xjntG9+FjYt8yT0CxpwIk842au722GBHGboaymHNS8YZRrWFEJ9jXOx1+C+N/8OhrDrf6Nda8zI4aiF7hq+d/dR+NQeFV0IHozbQVAWN5dBYYTwayo0E3NUyd3PX2zNZjDa9yCTjS9g6n9g2H5noW5cEkQnG4brFblxAM8CdICJAvB7Ys7ItudcXtyX3iWfBuJMlubcoXGNc3/DjW+BxGhWMIy83fgCl/2Zfjjqjn2vl08ZJC4m5MOc2OPRnfdpc+CX03tLauFimJck76owOi5ZEbY+Vi1DEvrxe49TPDe/CpkVQV2T8eI85ESacDsPG+n7kk41T4ALx/fG4jI7G5mpwNRmJtKUJz+Pcd+pu3n9Z+/La2+4UvfbTzpZ7933uqDNOD7TFwOQLjWa/lHED//4HO48btiwxzrw69Hw48rI+bUYSuhD94fVCwSojuW98zzg7o72WI7yo5LajvKjktmlkUlvzX1SScT60q9E4ymyubmvia6oyEnbLfOs639RZ3/f30NIsaLYZj5YLaFoumtln2tlyU9tzs9W4EvewC4buabH95fX2+QpVSehC+IvXa1ywVFfc1rTXVOmb9/XJtF/Wm45uMJJtZGLbIyJh3+eRCcYya6Svyc/WbhrRyTJfs+Agu7xddK27hC6Nv0L0hslkDDqWceiBy2ptNE+0JPuWJN9UbVwV2VnCtkZK85/oM0noQgwUpYwmiYg4SBoZ7GjEENCTW9C9qJQqVUr92MX62UqpGqXUet/jDv+HKYQQ4kB6UkN/CXgSeKWbMsu11qf5JSIhhBB9csAautb6C6AyALEIIYToB391fR+llPpOKfVfpVSXt4xRSs1XSq1WSq0uKyvz066FEEKAfxL6WmCE1vow4K/Au10V1Fov1FpP01pPS0lJ8cOuhRBCtOh3Qtda12qt633zHwBWpZTcSUAIIQKs3wldKZWulHHirFLqSN82K/q7XSGEEL1zwLNclFJvALOBYUqpAuBOwAqgtX4WOA+4UinlBpqAC3SwLj8VQogh7IAJXWs97wDrn8Q4rVEIIUQQyQAPQggRJiShCyFEmJCELoQQYUISuhBChAlJ6EIIESZk+FwREF6vZldFA5uK6theVo/dYiIu0kp8pJW4CN800kJ8pJXYCCtmk4wJLkRvSUIXftfk9LC5uJaNRbVs3FvLpqJaNhfX0ej09HgbMfaW5G7xJfu2xJ8YZWXqiESm5iZit8gNiYVoIQl9CCmoauSrvAq+K6jGbjGTEGUkyIQoa2ttOcE3jY+0YjF33yKntaaszsGGdol7Y1EtO8sbaLm0LNZuYUJmHOdPy2ZiRhwTM+MYkxqD26upbXJR2+yiptFFbbObmiZX27ImF7VNvmXNLvIrG33r3NQ7jNu6RVhNTB+ZzLFjhzHroBTGpsag5G4/YgiThB7GKuodfL2jghV5FXy1vZzdFY0AxEZY8Ho1DQeoMbfUklseLT8AEVYz28vq2bi3looGZ2v5rMRIJmbEcfqhmUzMjGNiRhxZiZFdJtkYu4VMInv9vhocbr7ZWcEXW8tZvq2M+5ZsgiWbSIuzc+zYFI4dO4yZY4YxLMbe620LMZjJTaLDSL3DzaqdFXyVV8GK7RVsKqoFjFry9FFJHD3aSHQHpRk1WZfHS02Tq+3RaEyrG53UNLmpbnK21pqrW9Y1uWhyesgdFsWE9LjWxD0+I474SGtQ3ndhdRNfbivji23lrMgrp7rRBcCkzDiOHZvCrLHDpHlGoLWmtM7B9rJ60BBttxBtNxNttxBlsxBtMx/wqDQUdHeTaEnog5jD7WHdnmq+yitnxfYKvsuvxu3V2CwmpuYkcszYYRw9OplDhscPii+qP3i8mh8La/gyr5wvtpaxZncVbq+W5pkhxOvVFFY3sa20jrzSeraV1JNXVk9eST11vua6rtgtJmLsFqLsZqJtFl/SN5J9++m49Fimj0wmPT4iQO+qjST0AaC1xu3VeLy+qUfj9nrbnrdOvbi9Gren3XOPxqM1Xi++aVt5rzbmW6Zt821l65pdfLOzkm93VdLs8mJScEhWAjNHJzNzzDCmjkgkwiq1UTCOWr7ZUcHybeV8sa2MHWUNAGTGR3DcuFTmjEth5phhRNul9XEgOdwefiysYdXOKtbnV6FQxEVa2p3htO+ZTu2X2y2mTn98XR4vuysayWtJ3KX15JXWs72snmaXt7XcsBg7Y1NjGJMaw9i0GEanxGA2KRqdbuodHhodRr9Mo9NDg8NNg9NNo8PTusyYumlweGhwuqlvduP2GnkzNzmK6SOTmTE6iekjk8lM6H0TYm+FVUL/aEMxf/z3d0TZjF/QKLuZKGvbL2qkzUy0zUyk7xAqym4hymZuK28zY7eaaHJ6aXC6fX9A44/aOt/ZH9pXttHhodHlweMN7oCSY1NjmDnGqIFPH5UctOaOwaawuokvtpaxbEspX24rp8HpwWY2ceTIJGaPS2H2uFRGp0RL7b2f6ppdrN1Tzbc7K1m1q5Lv8qtxuI0km5schc1iau30bnJ135djMxunuLYk+xi7heKaZnaWN7QmVoDhCZGMaUncvumY1BgSomx+fW8er2ZTUS0rd1Swckclq3ZWUNts1PxzkqKYPjKJGaOSmT4qiazEKL/uG8Isof9YWMN/1hTQ5Euyjb4E3PLraiz30OT04PR4D7zBDloPrXw/BC2HWVF2CzEtPyA2M1azCYtJYTa1TBUWs2/acfk+643lJmUsM5tonW+ZWkwKk0lhblnumzeZwKwUNouJ2AhJ4P3ldHtZvauSZVvL+GxzKdtK6wHITopkzrhU5oxLZcaoZCJtcrRzIGV1Dr7dVdn62Li3Fq8Gs0kxKTOOI3KTOCI3kWm5Sft1VjvdXmqbjb6amqauz3hqeV7b7CY11r5P4h6dEhO0oyyPV7O5uJZvdlSyckcFq3ZVtvbjDE+IbE3uR41K7vYkgZ4Kq4TeG063lyanh0aXcbjUkvibXR6ifLX1lvayGLuFCIsZk1zQMmTlVzaybGsZyzaX8tX2CppcHuwWEzNGJTNnXApzxqcyIjk62GEGndaaPZWNrNrZksCr2FluNGVFWE1MyU7kiNxEjhiZxJScRGKGWHOW16vZUlLHNy01+F2VVPrOBsuMj2DGqGTOnDKc4w7q2204h2xCF6Kvml0eVu2s5LMtpSzbUtaasEYNi2bWQSmkxUUQZTMTaTUT6WvSa5mPtBnNgBE2E1E2C5FWc7dXvnq8miaXUeFocnpocHhocrUcfRpHmy1HokYFxYPb492nr8bbWd+Nx9cP4+uPaXnu9nrxaCMxt/TPtM1rtMaY9/XzGOuMPhytNQ63lzpfE0N8pNVI3rlJHDEyiYMz47FZhkYHfE95vZptpfV8s7OClTsq+GZHJZfMzOXq48f2aXuS0IXop13lDSzbUspnW8r4ekcFTnfvmvNsFlNr0o+wmnG6va1HjI5ebstiMprdzB2b9EymfZ53tb6tGc9o7lOqrenP5Gv6MymjeU+1zJva5i0mxdi0WI4cmcSYlBg5qu2llh/Fvp640F1C78kt6F4ETgNKtdYHd7JeAU8Ac4FG4GKt9do+RSpEiModFs3Fw0Zy8cyReL2aZndbzbnZ5WmtTbfMN7k8NDndvpq3UbZlvtnlaU3wLTX4qJYOfN98pG9dS4d+lK/Dv6X/RgxeSqkBOwutJ41bL2HcYu6VLtafAoz1PaYDz/imQoQlk0n5kq2F5GAHI0Q7B/yp11p/AVR2U+RM4BVtWAkkKKUy/BWgEEKInvHHsdtwIL/d8wLfsv0opeYrpVYrpVaXlZX5YddCCCFaBLQxTmu9UGs9TWs9LSWlb6fsCCGE6Jw/EnohkN3ueZZvmRBCiADyR0JfBPxSGWYANVrrIj9sVwghRC/05LTFN4DZwDClVAFwJ2AF0Fo/C3yAccpiHsZpi5cMVLBCCCG6dsCErrWed4D1GrjKbxEJIYToE7lCQQghwkTQLv1XSpUBu/v48mFAuR/D8bdQjw9CP0aJr38kvv4J5fhGaK07PU0waAm9P5RSq7sayyAUhHp8EPoxSnz9I/H1T6jH1xVpchFCiDAhCV0IIcLEYE3oC4MdwAGEenwQ+jFKfP0j8fVPqMfXqUHZhi6EEGJ/g7WGLoQQogNJ6EIIESZCOqErpU5WSm1RSuUppW7qZL1dKfWmb/03SqncAMaWrZT6TCm1USm1QSl1TSdlZiulapRS632POwIVn2//u5RSP/j2vd/9/nzj7/zF9/l9r5Q6PICxjWv3uaxXStUqpa7tUCbgn59S6kWlVKlS6sd2y5KUUkuVUtt808QuXvsrX5ltSqlfBTC+h5VSm31/w3eUUgldvLbb78MAxneXUqqw3d9xbhev7fb/fQDje7NdbLuUUuu7eO2Af379pn03fg21B2AGtgOjABvwHTCxQ5n/A571zV8AvBnA+DKAw33zscDWTuKbDbwfxM9wFzCsm/Vzgf8CCpgBfBPEv3UxxgUTQf38gFnA4cCP7Zb9CbjJN38T8FAnr0sCdvimib75xADF9xPA4pt/qLP4evJ9GMD47gL+2IPvQLf/7wMVX4f1jwJ3BOvz6+8jlGvoRwJ5WusdWmsn8E+MuyO1dybwsm/+P8AJvnucDjitdZH23TtVa10HbKKLG3uEsFC529QJwHatdV+vHPYb3fkdutp/z14GzurkpT8FlmqtK7XWVcBS4ORAxKe1/lhr7fY9XYkxhHVQdPH59URP/t/7rbv4fLnjfOANf+83UEI5offkTkitZXxf6BoI/G0efU09U4BvOll9lFLqO6XUf5VSkwIaGGjgY6XUGqXU/E7W9/huUwPsArr+Jwrm59ciTbcNCV0MpHVSJlQ+y19jHHV15kDfh4F0ta9J6MUumqxC4fM7FijRWm/rYn0wP78eCeWEPigopWKAt4Brtda1HVavxWhGOAz4K/BugMM7Rmt9OMaNvK9SSs0K8P4PSCllA84A/t3J6mB/fvvRxrF3SJ7rq5S6FXADr3dRJFjfh2eA0cBkoAijWSMUzaP72nnI/z+FckLvyZ2QWssopSxAPFARkOiMfVoxkvnrWuu3O67XWtdqret98x8AVqXUsEDFp7Uu9E1LgXcwDmvbC4W7TZ0CrNVal3RcEezPr52SlqYo37S0kzJB/SyVUhcDpwEX+X509tOD78OA0FqXaK09Wmsv8HwX+w3252cBzgHe7KpMsD6/3gjlhP4tMFYpNdJXi7sA4+5I7S0CWs4mOA/4tKsvs7/52tteADZprR/rokx6S5u+UupIjM87ID84SqlopVRsyzxGx9mPHYqFwt2muqwVBfPz66D99+xXwHudlPkI+IlSKtHXpPAT37IBp5Q6GbgBOENr3dhFmZ58HwYqvvb9Mmd3sd+e/L8PpBOBzVrrgs5WBvPz65Vg98p298A4C2MrRu/3rb5l92B8cQEiMA7V84BVwKgAxnYMxqH398B632MucAVwha/M1cAGjB77lcDRAYxvlG+/3/liaPn82sengKd8n+8PwLQA/32jMRJ0fLtlQf38MH5cigAXRjvubzD6Zf4HbAM+AZJ8ZacBf2v32l/7vot5wCUBjC8Po/255XvYcuZXJvBBd9+HAMX3qu/79T1Gks7oGJ/v+X7/74GIz7f8pZbvXbuyAf/8+vuQS/+FECJMhHKTixBCiF6QhC6EEGFCEroQQoQJSehCCBEmJKELIUSYkIQuhBBhQhK6EEKEif8PfMYQ4J8tZKgAAAAASUVORK5CYII=\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" }, { "data": { "text/plain": [ "None\n" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "for stat in statHooks {\n", " plt.plot(stat.stds)\n", "}\n", "plt.legend(Array(1...statHooks.count))\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Export" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "success\r\n" ] } ], "source": [ "import NotebookExport\n", "let exporter = NotebookExport(Path.cwd/\"06_cuda.ipynb\")\n", "print(exporter.export(usingPrefix: \"FastaiNotebook_\"))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Swift", "language": "swift", "name": "swift" } }, "nbformat": 4, "nbformat_minor": 2 }