{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Annealing" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Installing packages:\n", "\t.package(path: \"/home/ubuntu/fastai_docs/dev_swift/FastaiNotebook_04_callbacks\")\n", "\t\tFastaiNotebook_04_callbacks\n", "With SwiftPM flags: []\n", "Working in: /tmp/tmpul25xo8z/swift-install\n", "Compile Swift Module 'FastaiNotebook_04_callbacks' (7 sources)\n", "Compile Swift Module 'jupyterInstalledPackages' (1 sources)\n", "Linking ./.build/x86_64-unknown-linux/debug/libjupyterInstalledPackages.so\n", "Initializing Swift...\n", "Installation complete!\n" ] } ], "source": [ "%install-location $cwd/swift-install\n", "%install '.package(path: \"$cwd/FastaiNotebook_04_callbacks\")' FastaiNotebook_04_callbacks" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "//export\n", "import Path\n", "import TensorFlow" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import FastaiNotebook_04_callbacks" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "('inline', 'module://ipykernel.pylab.backend_inline')\n" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "%include \"EnableIPythonDisplay.swift\"\n", "IPythonDisplay.shell.enable_matplotlib(\"inline\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Load data" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "let data = mnistDataBunch(flat: true)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "let (n,m) = (60000,784)\n", "let c = 10\n", "let nHid = 50" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "func optFunc(_ model: BasicModel) -> SGD {return SGD(for: model, learningRate: 1e-2)}" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "func modelInit() -> BasicModel {return BasicModel(nIn: m, nHid: nHid, nOut: c)}" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "let learner = Learner(data: data, lossFunc: softmaxCrossEntropy, optFunc: optFunc, modelInit: modelInit)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "learner.delegates = [learner.makeTrainEvalDelegate(), learner.makeAvgMetric(metrics: [accuracy]),\n", " learner.makeNormalize(mean: mnistStats.mean, std: mnistStats.std)]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "//Crashes!\n", "//learner.delegates = [type(of: learner).TrainEvalDelegate(), type(of: learner).AvgMetric(metrics: [accuracy])]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Epoch 0: [0.29990172, 0.9156]\n", "Epoch 1: [0.24770017, 0.9311]\n" ] } ], "source": [ "learner.fit(2)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Recoder" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "The Recorder's role is to keep track of the loss and our scheduled learning rate. " ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "// export\n", "import Python\n", "public let np = Python.import(\"numpy\")\n", "public let plt = Python.import(\"matplotlib.pyplot\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "// export\n", "public func plot(_ arr1: [S1], _ arr2: [S2], logScale:Bool = false, xLabel: String=\"\", yLabel: String = \"\") \n", " where S1:PythonConvertible, S2:PythonConvertible{\n", " plt.figure(figsize: [6,4])\n", " let (npArr1, npArr2) = (np.array(arr1), np.array(arr2))\n", " if logScale {plt.xscale(\"log\")} \n", " if !xLabel.isEmpty {plt.xlabel(xLabel)}\n", " if !yLabel.isEmpty {plt.ylabel(yLabel)} \n", " let fig = plt.plot(npArr1, npArr2)\n", " plt.show(fig)\n", "}" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "// export\n", "extension Learner where Opt.Scalar: PythonConvertible{\n", " public class Recorder: Delegate {\n", " public var losses: [Loss] = []\n", " public var lrs: [Opt.Scalar] = []\n", " \n", " public override func batchDidFinish(learner: Learner) {\n", " if learner.inTrain {\n", " losses.append(learner.currentLoss)\n", " lrs.append(learner.opt.learningRate)\n", " }\n", " }\n", " \n", " public func plotLosses(){\n", " plot(Array(0.. Recorder {\n", " return Recorder()\n", " }\n", "}" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "let learner = Learner(data: data, lossFunc: softmaxCrossEntropy, optFunc: optFunc, modelInit: modelInit)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Utility optional property to get backour `Recorder` if it was created by a utility function. This doesn't always work properly for unkwnon reasons" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "//TODO: Fix\n", "extension Learner where Opt.Scalar: PythonConvertible{\n", " public var recorder: Learner.Recorder? {\n", " for callback in learner.delegates {\n", " if let recorder = callback as? Learner.Recorder { return recorder }\n", " }\n", " return nil\n", " }\n", "}" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "learner.delegates = [learner.makeTrainEvalDelegate(), learner.makeAvgMetric(metrics: [accuracy]), \n", " learner.makeNormalize(mean: mnistStats.mean, std: mnistStats.std), learner.makeRecorder()]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Epoch 0: [0.32225573, 0.9083]\n", "Epoch 1: [0.26439548, 0.9237]\n" ] } ], "source": [ "learner.fit(2)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xd4FOX6N/DvnULvEDoSilSRYlAQUBAEBLsesBxR1IO9HCvYy09Ffa3HdrAce8eOgqAIKAIGpApIQJBIi9QU0jbP+8fMbGZ3Z3dnNzs7m+z3c125spmdnb0zm8w9TxelFIiIiAAgxe0AiIgocTApEBGRF5MCERF5MSkQEZEXkwIREXkxKRARkZdjSUFEOojIfBFZLyLrROQGi32Gi8hBEVmpf93jVDxERBRemoPHLgdws1JqhYg0BLBcROYqpX7z22+RUupUB+MgIiKbHCspKKV2KqVW6I/zAawH0M6p9yMioqpzsqTgJSKZAPoDWGrx9GARWQVgB4BblFLrQh2rRYsWKjMzM9YhEhHVaMuXL/9bKZURbj/Hk4KINAAwE8CNSqlDfk+vANBRKVUgIuMAfAbgSItjTAEwBQCOOOIIZGdnOxw1EVHNIiLb7OznaO8jEUmHlhDeUUp94v+8UuqQUqpAf/w1gHQRaWGx3wylVJZSKisjI2yiIyKiKDnZ+0gAvApgvVLqySD7tNb3g4gcq8ez16mYiIgoNCerj4YAuAjAGhFZqW+7A8ARAKCUegnAuQCuEpFyAIcBnKc4bSsRkWscSwpKqR8BSJh9ngPwnFMxEBFRZDiimYiIvJgUiIjIi0mBiIi8kiopzN+4B4tz/nY7DCKihBWXEc2JwFOhMPl/vwAAtk4f73I0RESJKWlKCmWeCrdDICJKeEwKRETklURJgWPiiIjCSZqkUM6SAhFRWEmTFEqZFIiIwkqapFDO6iMiorCSJimwoZmIKLykSQqFpR63QyAiSnhJkxTyi8u8jzk7NxGRtaRJCgXF5d7H5RVMCkREVpImKRzZqiEa100HwEZnIqJgkiYpdG3ZANed1BUAUFbBRmciIitJkxQAIC1FWwiOJQUiImvJlRRStV+Xo5uJiKwlVVJIT9VKCmVsaCYispRUSSEtRft18/JLXI6EiCgxJVVSMOY/uuR/y1yOhIgoMSVVUjDGrB0oKgu9IxFRkkqqpKB3PiIioiCSKikQEVFoSZUU2OeIiCi0pEoKREQUWlIlhdP6tgUAjOndyuVIiIgSU1IlhQa109C8fi20aFDb7VCIiBJSUiUFAEhLFXg4opmIyFLyJYWUFO8gNiIi8pV0SaFN4zrI3XfY7TCIiBJS0iWFJvXSUVhaHn5HIqIklHRJITVFuJ4CEVEQjiUFEekgIvNFZL2IrBORGyz2ERF5VkRyRGS1iAxwKh5DWmoKNu7ORxFLC0REAZwsKZQDuFkp1RPAIADXiEgvv31OAXCk/jUFwIsOxgMAOHRYmwzv5g9XOf1WRETVjmNJQSm1Uym1Qn+cD2A9gHZ+u50B4E2lWQKgiYi0cSomACgu8wAA1u045OTbEBFVS3FpUxCRTAD9ASz1e6odgO2mn3MRmDhiGwu0qVJTOWUqEVEAx5OCiDQAMBPAjUop/9tzqytzQCuwiEwRkWwRyc7Ly6tiPNp35gQiokCOJgURSYeWEN5RSn1isUsugA6mn9sD2OG/k1JqhlIqSymVlZGREZPYjKU5iYiokpO9jwTAqwDWK6WeDLLbFwAm6b2QBgE4qJTa6VRMAJCiFxVSWFQgIgqQ5uCxhwC4CMAaEVmpb7sDwBEAoJR6CcDXAMYByAFQBGCyg/EAqKw+8lRwqgsiIn+OJQWl1I+wbjMw76MAXONUDKH8vrvAjbclIkpoSVexrjiYmYgoqORLCqbOTYoZgojIR/IlBVMeYBUSEZGvpE4KZVxXgYjIR/IlBVP1UTlXYCMi8pF8ScGUB9gtlYjIV/IlBdNj1h4REflKuqTw5IS+3sflLCkQEflIuqTQsXl97+O9BaUuRkJElHiSLimY5eWXuB0CEVFCSeqkwOojIiJfSZ0USsuZFIiIzJI7KXg4ToGIyCy5kwJLCkREPpI6Kby0YDO+XBWw0BsRUdJK6qQAAI98vd7tEIiIEkZSJoUPpgzyPhbhspxERIakTArHdW7ufezhpHhERF5JmRTMOH02EVGlpE8Kews51QURkSHpkwIAHDxc5nYIREQJgUkBXKuZiMiQtEnh4ysHux0CEVHCSdqkkJXZzO0QiIgSTtImBTP2SiUi0jApAKhgmwIREQAmBQBMCkREBiYFAFxrh4hIw6QAlhSIiAxMCgAe+WaD2yEQESUEJgWAayoQEemYFIiIyItJgYiIvJgUiIjIy7GkICKvicgeEVkb5PnhInJQRFbqX/c4FQsREdnjZEnhdQBjw+yzSCnVT/96wMFYLF1/Ulfv41mrd8b77YmIEo5jSUEptRDAPqeOHwutG9f1Pv5i1V8uRkJElBjcblMYLCKrROQbEekdbCcRmSIi2SKSnZeXF7M3Nw9a41rNRETuJoUVADoqpfoC+A+Az4LtqJSaoZTKUkplZWRkxCyA2mmVvz6TAhGRi0lBKXVIKVWgP/4aQLqItIhnDA1qp3kflzMpEBG5lxREpLWIiP74WD2WvfGM4ZiOTb2POf8RERGQFn6X6IjIewCGA2ghIrkA7gWQDgBKqZcAnAvgKhEpB3AYwHkqzoslt2xUx/v4p5y9WLn9APp1aBLPEIiIEopjSUEpdX6Y558D8JxT7x+NSa8uxer7xrgdBhGRa9zufZRQUlPE7RCIiFzFpGDCpEBEyY5JwSRFmBSIKLkxKZiksaRAREmOScEkhUmBiJIck4IJ2xSIKNkxKZikimDplr0oKi13OxQiIlcwKZjk5Zdg4owluPXj1W6HQkTkCiYFk8NlHgDAhp2HXI6EiMgdSZ8URvVs5X3MhmYiSna2koKI3CAijUTzqoisEJHRTgcXD69cnOV9bHRJ3ZxX6FY4RESusltSuFQpdQjAaAAZACYDmO5YVC4p93CmVCJKbnaTglGvMg7A/5RSq0zbaoxST4XbIRARucpuUlguIt9CSwpzRKQhAF5BiYhqGLtTZ18GoB+ALUqpIhFpBq0KiYiIahC7JYXBADYqpQ6IyD8B3AXgoHNhERGRG+wmhRcBFIlIXwC3AdgG4E3HoiIiIlfYTQrl+lKZZwB4Rin1DICGzoVFRERusNumkC8i0wBcBGCYiKRCX2+ZiIhqDrslhYkASqCNV9gFoB2Axx2LKs6W3jEyYNvybftdiISIyF22koKeCN4B0FhETgVQrJSqMW0KrRrVCdh2zouLXYiEiMhddqe5mABgGYB/AJgAYKmInOtkYEREFH922xTuBDBQKbUHAEQkA8A8AB87FRgREcWf3TaFFCMh6PZG8NpqSymF0vIKLN2y1+1QiIjiwu6FfbaIzBGRS0TkEgCzAHztXFiJocyj8NCs3zBxxhKs5xoLRJQE7DY03wpgBoCjAfQFMEMpdbuTgSWCUk8FNuzKBwDsLyoNeL6wpBzfrtsV77CIiBxjt00BSqmZAGY6GEvCKfdUwJhMWywmhZ36yRp8uWoHvv33CejWimP5iKj6C5kURCQfgNUiAwJAKaUaORJVgsgvLvc+FouJwv/cqy3GU1hSHvgkEVE1FDIpKKWS+vb3n68uRauG2hiGUItHiFXGICKqhmxXHyWjbXuLUFTqAcALPxElhxrfrdSu96cMstyel18S9DVcvJOIahomBd2gzs3RvmndoM+LAO8s3YZ9hZW9kJSeFViGIKKagknBJCVEFdHvu/Nx56drceMHK+MYERFRfDEpmKSmBE8KpeXaktT7CoNXJxERVXeOJQUReU1E9ojI2iDPi4g8KyI5IrJaRAY4FYtdIXKCl9V4BbZBE1FN4WRJ4XUAY0M8fwqAI/WvKdCW/HRVqJLC/V/+FrBNsamZiGoYx5KCUmohgH0hdjkDwJtKswRAExFp41Q8doRqUzCYd6lsaGZRgYhqBjfbFNoB2G76OVffFkBEpohItohk5+XlORbQpUM7OXZsIqLqwM2kYHV7bVkfo5SaoZTKUkplZWRkOBbQhKwOuHZE15D7CIDt+4owc3muY3EQEbnFzRHNuQA6mH5uD2CHS7F42WknOOuFxfi7oAS92mhTP7GhmYhqCjdLCl8AmKT3QhoE4KBSaqeL8QAAKmy0Hf9dUKLvy4ZmIqpZHCspiMh7AIYDaCEiuQDuBZAOAEqpl6At0jMOQA6AIgCTnYolEuGu8+t2VC62w5xARDWNY0lBKXV+mOcVgGucev9oDercDC8t2Bz0+XJTUWLj7vx4hEREFDcc0exnePeWWHv/mIhew2okIqopmBQsNKgdWQFq18FivL/sT4eiISKKH66nEANT3loOADipZ0u01BflISKqjlhSiCUFHCouw4ZdhwKfUgr7TdNuExElIiaFGBIRTHp1GcY+vQjKr53hrSXb0P/BudiSV+BSdERE4TEpxJAIsHL7AQBAqafC57n5G/YAALbuLYx7XEREdjEphPH65IG29xUAtVK1U1pU4vF5jv2TiKg6YFII4uRerVCvViq6ZDSI6HW107VTWlhabvk8Z1QlokTG3kdBvDwpCwAC2gZCUQDS9DUZyjwsGxBR9cOSQhgigqyOTW3tW6GUd+6kSJIJEVGiYFKwYVwfe2v/3PrRahw8XAYg+MR6XK2NiBIZk4INk4dk2tpvwe/mBYB48Sei6odJwQaJYsGEzXmVXU/z8kssl+7ctrcQhSXWDdJERG5gUnDIFfrUFxt35WPgQ/P8ShGaEx//AZNeWxbv0IiIgmJSsOnmk7tF9boxTy8M+fzybfujOm4o5Z4KHC71hN+RiMgPk4JN1408MuLXGI3OVqzmR4qVy9/MRs97Zjt2fCKquZgUHNT3/m8DN+pNChe8vDSqY27clY+KMGuG/rCxsqrqlUVbcNvHq6J6L7s27c5HfnHwBEhE1QeTgks8dhaD9rNux0GMeXohnpufY/s1/zdrPT7Mzo34vSJx8lML8c9XoktyRJRYmBRcEs1qbTsOFAMAVumT7iWSVbkH3Q6BiGKASSEC7/1rUJWPcf17v+LtJdvCVgFZScRZk+yO3N51sBh/HTgccp+cPfk4xGooIlcxKUSgT/vGVT5GfnE57vpsLTymi2lpeQW63fkNPv01dDVPpGnkkxXOVhsBgN0Cz6BHvsOQ6d+H3GfUkwsx4aWfYxBV9aKUiuomgcgJTAoRMCa7i4UK03ILy/7Yh1JPBe79fB2mzlyNfg9oDdTZW/d512EAgJw92gI9dsfS3fRhbBuYZ6/dibNe+MmndOCJ8RxPG3blx/R41cFTc39H5zu+RnEZuxG7SSmFVxZtwZ78YrdDcRWTQgTSUwNPV9301KiOZW5TuPvztQCAQ8XleP+X7ThQVIan5/2Oc1/6GZNf/8W736OzNwAACkyjoLfvK0Lu/qKoYojUlW+vwK9/HvCZ1ymaBnPy9cbP2wCAY0tclrOnAP83az2uffdXt0NxFZNCBFItSgpdWtaP6ljhGpqfnrfJ1nGGPTYfQx+db/t99xeWInPqLHzwy5+2X+PPXFJI5MlglVK46YOVWLplr+Xzuw4W4/E5GxKm6iaK2VQohozVEg+FGF+UDJgUqshv1U3bzNehSK8FSmkXvPu/XBfx+27XSxVvLdmmH0vh8TkbkLPHfrWNT0nBLyss+2Mfno+gy6yTSsor8Mmvf+Ei01QiZZ4K5OWXAAD+/cFKPD9/M1bmJl5vLjveWrIND371m9th1DjRzHVWkzApVJGnIsqsUAUKQO7+w/jfT1sjfm2K/gdvXMv3Fpbi+fmbcWEE4wzMpRz/6qMJ//0Zj8/ZGHFc8XLnp2sw8KF5KC7zoKRcq65JlJJCpO7+bC1e/fEPt8OocZJ9LRQmhQjVr6W1IdTRl908olk9n+cfPadP5Ae1cWNiNDIDAFT0VQ3G6/yvg2UehZw9BSiLsOhj/AOFimfr34XBn7RpxZ/7MXvtriof55s12jFKyiu8CdLtnBCLi9AHv/yJy9/IjkE0yYtL5WqYFCJk/PsuvG0ENjw4Fk9M6IfT+rat0jG35IW+aCqlMOrJBd6fK5TyXtAiZfzhK6Xwx9+FmPSqVrWyr7AUo55cgIe/Xh/2GFYlhVDRDP9/P0QVq9nZLyzGlW8vj+g1ltdaU6DGKUyUO8OqXJRun7kG89bvjmE00ducV4DMqbOw0GJmYEp8TAoRMq4fddNTUSc9FY3rpmNY1xaOvqf/nWyFUlH3+klJqTzGI1+vx287fSfm+3mzdaPsn3srezgZb51fXIaScq1kEW2ScpI3eVmdKmVKkPELKaSasipf9tZ9AICvVu9wORKKBpNClMwXwaJSZxfK8a/SqVDWXUGVUngmTK+lsU8v0ve17k0VLNmc8HhlD6ey8goopdDnvm9xqd5ltqo5wYm79VBHrFDKVFKI+VtHxW6e73XPbEz/ZoOzwVQjC37Pw6gnF6C0vGrtezUlKVcVk0KEsjKbAvC9oBaZBh05cYHx72W0cvsBlFs0cB8qLsdT8363dcxNewqsk4LFL+B/we7/4FxM18dMGIPNyjwKmVNn2XpvK06cN6OkYP5nN35jj09ScPdiYLy73TiKSj14acFm5wKqZu78dA1y9hRg96HYDDpj7yOKyIv/PAZfXTcUdUyD1pwedPTesu0B26wG2ER6cbNKChUVCvsLS/HEtxtRrpdQyjyBx32tir1eFv6eh4+XV07DYTfy/OIyTHkz29ao01Cno6JCJWD1EVVFopT4qjsmhQg1qJ2Go9r5zoFkdXF1mv90EBUVCh9mByYPw66DgRfRYCWFJ+ZuxH++z0HXO78BABSXByY9q0QRiUmvLcMtH1VOw2E3oX2UnYtvf9uNF+aHv1M2jmk+tHEX6FHK275i92JSUeHQHEVG00c1u6it33ko7CSHgDa310fZ2x0rkSX5jX3MOZoURGSsiGwUkRwRmWrx/CUikiciK/Wvy52MxylXntjF7RCw4+BhPPx18Hrmn7f8HbAt1eK/qaICSEvx/bOIpkvpd+t3Y8xTC72ljXDsXi7sXgAOFJUid3/wC5bHp6QQ+t2/W78bmVNnofvd36DH3fZWtMv6v7m46NXKsR+HisvQedoszN+4J+hr/OPwVCgcKCq19X7hbP27MCCh7S8sRXGZB3vyi5E5dRYW5wT+jYRyyjOLwk5yuPPgYVzxVjZu/Xg1Plqe6zNFSzh/F5Sg9z2zw04VX92SaaJzLCmISCqA5wGcAqAXgPNFpJfFrh8opfrpX684FY+T6qSnYvldo7DqntHef+uz+7eLawzZW0Ov9VzuUfjs1798tlmVFFJSAreviGId6cveyMbG3fnYXxQ4ZcCXqwJ7pZi7uX6xagfOm6HNllpYUu5TzWS1v5Vhj87Hqf/5EUBlwlm346B3idSKCthuaP5Cj7fMo7xTIYTzd0EpFm2qvMhu2p2PCgX85zvfjgA7Dx5Gvn6h9I/jwa9+Q78H5qIwggup2eLNfyNnTwEOFJVi+P/7AbfNXO3zfP8H5+K8GUuwYpt20f3f4q1RvU8wSgEXvrIU8/WVAG/7eDWOuncOAG3+rnClrsWb96Kw1IOXF22JaVzhlJZ7HJlPbHNegaPL8MaKkyWFYwHkKKW2KKVKAbwP4AwH389VzRvURuN66d6f412ldOMHK0M+/936PQH7WCYFkYAeSB2bRze/EwDLKq3r3rNqD6l8fP17v2LJln14b9mfOPP5n3yqmYyI3/x5G/YVBr+Lzre4kI5/9kfvY3NSiXTBo3JPha1qE0NBSTkKS/TR035v9brpQuwfxlerdwIACiPo3Wauorng5aUY9eQCFOptXnPWBQ7+W7n9gGXp65l5m7AkyJxRdnkqlOUYnANFpTjq3jl4+rvQPeWMsMJ9PP7xz1m3C9e8uyJsfHn5JZYlsc15hRj66PwqtRUu37YfF76yxKfn4MgnFnh7/yUyJ5NCOwDmK0Kuvs3fOSKyWkQ+FpEODsYTF+2a1AUAdG/d0OVIfM22uCBYlS5SRQIuktGsEmfwn/Li7s/W2n7ttE/WYJN5JLefHTYvzFZ12VrvI9+G5sypsyznbfK/Zj741W8YMv177C0osfX+R907B5P0+Zf8YzFPxx60GiuC02/1UXnvyIMcxziP5tc+Ne93nDdjif03trD0j32W241k/sXKvyyfN3inZAlxApZv24ft+yr/DorLPLjireWYpSfUUAY+NA/9Hpjr/dn/3FWli+utH63CTzl78ee+InyYvd1nCvxE52RSsLpV9v90vwSQqZQ6GsA8AG9YHkhkiohki0h2Xl5ij5I8oVsGZl41GJcO6eR2KGFt3B04Cd7hMg/e1KdyNsSybdWYiM+f3bxjp7ugnXYMj6nReL1pAN/jczZi/oY9Ptv8GVNdHyqOvFrHow8aXK1Pwpdqar/xPwfe6q0Ijh8qgQd75v4vnZlUL5LSlBXvlCwhPs5zXvRdlKnnPfbafOyYtWYnMqfOwoT/Rr/wk1JatZl5CnxDzp585O4vwuFSD+76bI23atNtTiaFXADmO//2AHwqk5VSe5VSxu3WywCOsTqQUmqGUipLKZWVkZHhSLCxdEzHZkhxoUdSLOz066X04g+bq1RSsMvuwCE761MbvaZCmfbJGvyoN6w+Ntu3NDP59V9wyjOxKeYv3uzbeFvuUfjvwi044/mfAPiWFCqUQu97ZmPKm9ocRubqk1cWbcHc38JPY2GVqOx/frH5nAtKQle7fKq3bZnHZxglqPU7D+HG93+Fp0J5q2/s/m0oKJ/EatVmUVhSjsyps3CZxUXa3x2frgGgzfxrZfm2/di+L0jbg41//1FPLsTQR+dj5opcvL3kTzw1194YI6c5mRR+AXCkiHQSkVoAzgPwhXkHEWlj+vF0AOEn3qmGjo7BMp5ueXT2Bny9JnxRPFpKKRwqLsOdn9qrVtqcV1mdtDr3YNTvu9xG4/mKP/dXeUDUBS/7zj5rXKCNa0ZaqnlkvAeFpR58+9tueCoUyisqB9/936z1+Neb1hPemaukBjw4N+B5b+1RnLrphJvO22grMcLpNO1rb2nl2ndX4LOVO7AlrwA3621Jdudo9M8BVgMxjVLwdzGozjnnxcUY9tj8kPvYmd7eGPOUKOs4OJYUlFLlAK4FMAfaxf5DpdQ6EXlARE7Xd7teRNaJyCoA1wO4xKl43PDChQNw+9geeH/KILdDqZLPVzo3h41SwKRXl3nvHsMpNY2PMO7ktOMoTP9mQ1Sjqrtk1Le8YJ79wmKMfGJBTEe4Vl7oNeaSwuinFnof/+vNbG/d++KcygZfq7tf/01v/rzV52fz77ZxVz5eCdKbx9jtvWWBCzDZbT+xw7j4mUsARoO71ble6Vc63HOoGLd+tMo79bnhd7/q0HKLsTSxKPX+vjvf9ngVcw+0YBrU1pJCJN11nZTm5MGVUl8D+Npv2z2mx9MATHMyBjeN69Mm/E4hHNWuEdb+lfhd2KrCoxQa1rH/Z1hqMZAO0PrMB1vfOdy/b592jYMOxgv1jxqsq2ioO3L/i0lqivV92femO9mbTb2vjOlFfI7p9373fL4OkwZnmp7X4wJw2n9+RKmnApcP6xwYN4Dc/UWY9skan+3Lt+3HOS8uxrPn98fpFjMC//rnfrRqVAfNG9TC7oPhk4dxB28sFhXO334J6YGvfsNXq3diWDffquQr3vKdRVebCsZ3udxQ72dn2vg1uQdx2nM/4vaxPbzb9hwqRstGdcK+NhhjXFC5EwMjo8ARzXGSnhp4B5TZvJ7FnpqzB7TDWf3bOxlSQhjwwFz0atvI9v6bg0wzHiwhANrFJ9RqcArAuS8tth2D4dT//Ihv/KrWvly1A/d9EbzKwPjHN/4aLP4sQpqxMPAu32oSw4dmVVbhGHfaSlUuOWnVGK+UsmzU/W2HVk0XbFnTs15YjOOnf4/ud832mTgxGI8pATt5HfQ/L/sKS7ErRHXgWS+E/xswxi+s3F5Z/XjSEwsC9gv1se44cBhnv/CT92fzzMWJgEkhTrLvPDlg26dXDwm6/9RTelgmkpomv6Qc/13g/OCkUKvBbdpdELJ9ItSncNU7vv3hr3vvV2/vJCvGKGvj3z81ter/glYXk5cXVc5NZR7nYbAahKcQ2Oc/WI+Y4rLo+/CX6ZnHbknB4KlQPgsthXutkYA35xUgv7gMAx6ci7eXRLc2uTETslgszORfmjxUXBZykOMri/7Aij8rq8SsjukmJoU4aVwvHUumjcSi20Z4tzWtX8tnH/9V22p+SkgM/mtK+CsJU62wavsBPPvdpojmRVIKOP25H7Gzit02AaDXPXNs7XfYdCG32wf/2e824e7PA0s+j1pUY9lVXFb53sFOmdXmnnfPxpVvL/c2VIfrEuypUMjLL8HIJxZEtNysldOf0+7sw42CV0rh6Pu+9Rk74e9Pvx5LlUvkBh50/oY92GTRddxJjrYpkK/WjQPrHdfcNxp97vsWADBx4BF4fM5G/F1QyqUBE0i4gVBG91L/8R3hrM49iHU73GkzMg/aMpR5KgJ60xgL5vgL2hUzAkopn1JOt7u+CZms/O++ww2ELK9QGPXEDwCq1lMN0JbD3bQ73/tfmec3S6+nQiFF4B09Hor/Cnkz9WlcrBKNMb5h6/TxkQcdJZYUXNawTrrPz9eM6KpvT8NpfduiX4cmboRVbSzdsrdKVRmx5N8gake0K+g5YV+hvS6Rc9btwqHDVe8ps+NgsU/VizkhRLpWuBWPR/mUjoIp91TYml/qZFPvsFV+SabLHV/j9pmrbU8AaWbMrZUobQosKSSYyUM6YbI+GrpOeio+u0Zrd7js9V9i0rfan0j1nmVy4owlaNGgVvgdKSyrUdzmXlnvLP0T7yyNrk4+mAkvWY8WDjfWwQ67c0aNfmohtticCThUb7QPs3Nxy+juto5jxUgK2/cVYdhj8/HSPy3H8jqOJYVqokOzwJ5KZ/YL7B7or0566I+4e6vEmqMpGn8XxGZ6aQoUrr2lqoJdjP1HEUczTsLu6nR2EwIA3PRhYKO92bEPf2f7WP6MQqNxzmeuqJwIh3W1AAAVtklEQVQdePu+IuTsyY9LyZJJoRp7YkI/PHNev5D7JErtBBdCoUj4/92e82LkXYYTZS4hu5b9sQ8zl+d6p7g3V6cNe2w+Rj25EE/bXG63Klh95BJjNlVDuDt6K6kpgi4ZDYI+36tNI+SEmGU0Hlo3qoOrR3TBnkMleC7EWAHy1aZxnYB5qJLZ1r2RN2wnyg1RJMwDFRf8Hjj5Z7h1U2KBJQUXfHfziZh1/VDvzwtuHY6fbj8p5GtaNqoNAKhfy3eE5lHtGuPyob4zsvbQp+0+pmNT3HVqz4Bj9WxTOVisnt/xYi2jYW1MGpzpM8cPhZceg/ELyc6RpVNdZndywKrgX54LumQ0QJN6lY2jHZvXR/MGtUO+Zsqwznh6Yj8su3NUwHN3ndoL44+unFIjo6F2rAqlMGlwZkB3tg+uqJyLKSuzWVS/g10PnNEbgO8cP0TxsD9GS5kmknjkOSaFaiItNQVn9m+H+rWta/zM6zfUTtM+VnOdZAPT6xqaHt82JvreEnb0P6IpAGDS8ZlRvT7eK9glgrQUCRjgRJFzawxIdcekUA3996JjMP1s39HPx3RsipE9WgIAaulJwdzX+5sbhnkfiwieOa8f5t8yHGlxqqZo5Dcew663LzsuxpEkvkSZGI0SEEsKZGVM79Y479gjArYbfy+19At9meni4t+l9Yx+7dCpRfRrL0djePfIF0gylxRuGHlkLMMhqnbiMcCNSaEGMeZOqZ2mNR6XlPmOrvxgyiDcMrpbwOs+ufp4vD9lEMZXcarv1ycPDPn8rSGqqjY8ONZye4dmlb20akfRQyvWjNIYAFw9vEvErx/U2dk2HKrZioNMHR9L7JJagxj3EC0aao3Yewt9B/wc17k5juvcPOB1A/R6/0Gdm+PM33bjsdkbsClIV9a0FLGs3vjhluHIDFPy6NWmER48o7flBGvG6lP+2jSuix9vH4G9BaX4Jcg8PPFkXmY1WMyhLNkS+nfo3bYR68IpqMM25laqKvdvvSjm+rZvgvFHt8HDZ/UJv7Ofk3u1wpfXDQ3YPqpnS1w8uKPPwkGPmNo1wiUEQGvLuGhwJn65c5S3isuO9k3roW+HJqgdxUU41lJFMDFLW3q8Y4j1MKI+fhI2rEeqcd3o2qdqAo5opog8fFYfnH9sB4zo0RLPXzDAZzxCJMyjj9+9/Di8fdlxeOXigbj/jKN8/ijPt2jXOK5T+OqRjIa1MapXy7D73TGuh8/P/fXJAZ+/YEDY15qFWsyokcWqb20sZrM1Bhumpgimn9MH715+nOUqZME8eOZR+P7mE8Pul1LFod8DM5uir74meDRtONVBTRodv+mhUyLaP9gKgbHEpFCDtG1SF4+cfXSVBz4Z03anCHB81xYYemQL73NN62t3aU9N7AsAuOT4THQ2lRI+uGIwFt02Aq0b1fFZO8KfnTueI1v6zst0VLvGWH3faJ8xGUumjQx7nJQQd9//m3xswLYOTeth88PjfN7n/GM7eI8lIji+awvbazdPP7sPLhrUEZ0zGmCY6VxaqWpJoWvLhrjyRK2tI5LSmJNaNgw9BidSoc5Q8/qRT45odOF2Q6T/q6EW74mVxPiroYRiXOvaNw28w75jXE88NbEvzuzXDgBw3+m98f0tw3326dCsHpbcMdJyEj/DJcd3CvpcKEbX1tQUQb1aqbbWd04NcfE+pmPTwP1TBKkpglYNK0sMxj9vsGv2OQOCL53atWXlVCSnHh26Md+cFO45tVfIfa00qpOGfkdoJapJgzMx5YTAtZjj7e3LY9utuF6tNLRqZJ1oTu7VKuLj3TCq+vRqG3dUa8ffg0mBAqSnpuC5C/rjwysGBzxXr1Yazurf3vZdcjCDuzQPv3BIiLdYe98YrLj7ZJ9pOv57kTbV8FHtfKvNzBfaL68dilE9Q1dd+d+tN62Xju761CEdgyS6nm0aYt5N1tVD5lHjwe4Mrxmh3d0bCaxJvXRcOrRT2OmTr/frptuvQxO0aVwXW6ePx9AjW6B+rfj3JRnatQX+eGQcumRoJchICj9WPdQmZLXHjIsqz0PzBrW8i937qxXFXf/4Pm1wfJfADhiJKNoq4UgwKZClU49ua7lSXKx9ds0Q/Hh78GqmYOrWSkWd9FSICObceAI+vGIwxvRujT8eGYeB+kX4tL5tMe+mE30SWJ/2jb319sHuoo3dje9XD++KE7tlYMZFx+C6IGMlaqel+JQIDP7Tm/tPhGgo96srPlCkzfA51uLO0FwtZK6OmnfTiTjFr1vxP7LaI7N5Pdx3WuSlDittbfxNXDS4I0TEcu3hcNVjkwZ3DNj22Ll9vVO3AFq7S7CLf6j1DoJJS02JyaI+0Yrk/orTXFCN169DE8tqqkh0b90Qx+oN3CLivWgO7twcXVs2CLhTNRqSh3bVLqjXjOji07hsJA3zy0QEo3u3DrjTN8ZRGBepS/ym87j3tN4+P1t1CQYq1+s2SiShLLxthHfakhQRPHbu0fjwisGWSaltk7r44dYRuGRI+Oq6ZXdUts8smTYSr12SFbCPcU26ym+Mxt16VdfpfdtiTG8tkRnnzzzeas6NwxBKnfTUgEkfAd8G+KyOTdGknnUPpGiWsU1LEZx7TPDqv0ituPvkiPZPD1LqscLBa5SUTuhWtV4z1488Eted1BX/yNL+0f3vTqeN64nHzz3ae5d965ge+NnUYG3VqyoYY4yHkRSMtoXT+rbF1unjvRf7cC4b2glvXHos7j2tF/570THY/PC4gH3e/ddx+P7mE9G6cR3vHXFqimBCVgdvUgxl3k0nBBxv88PjMFhPVA3rpHvPVevGdXBSj1beOvrUFMG9p/XCpMGZACpn4jX0atMIL144wKebstnLk7Jw1/ieqGeqzrpWX3rWLC1FLMd/GHGlpQimntIDt42p7JlmTIUyqmcr3DW+J/49KnCAZiipKYKJA4/A1unjsXjqSfjk6uMt93v0HOvfzb/018ziMz9vYIeg7x9J43GotqtYYVKghBVtq0X92mm4eXR37139pX53yXXSU/GPrA4B7SKvTx6IEd0zMKa3b2NlqOmKjV5UxijyPu0b46mJfYNeHK08eEZvpKem4MRuGRARjOnd2rKa5fguLdDZb/2MgmL71SVd/XpzpYjWoP7SRcfg4ysHo26tVHz77xPwwoWVXX6z9Ib4z68ZgslDOuGq4V2wdfr4gAZ+EeCUPm18Jmw0Tq+Cwsm9WuHyYZ29bUCDOzfHLab2g6+uG4o7x/WEiFj2FjNKCl1bNkBaagoGd2mOAXqDep30FKy7fwxe/OcANK1fy6fh+Jnz+mHYkS0w86rKC/3NJ3fz6aVk7ojQtkldb6IHgM0Pj/N+FhMH+t4sGIm4QZBJKs0eshgzZIwj+tcwe50ubh3THXUdnuoe4IhmSiDf3DAMtdJScN8XgSOeq+LM/u1w4wcrw+43vHtLDO9e2Qhtp673nlN7oVHddIzqWZlIzuof+m7u1Yuz8FF2LrbuLUR+cTku0u++o2F1V2qXcYfbuG66tzG8S0YDn4WbppzQGSf3ahWQjIZ3a4m7xvfEJyv+wm87D1mOrzC2mWs8mtSrhf9dMtDnwgto3Y2PaqeNr7h0SCc8OnuDz/PdWzfEOQPa44oTK9uBjKSugKCzB5/ety3O0HvK3X96b5zYLQOZLerjjZ+3efdJDbHWR2qKYP7Nw/HHXt8lO7dOH48Nuw5h7NOLLF+XIuHbUi44Tksyd47vhSOa1bMc6e8GJgVKGEbPiqr2bIqnlo3qRDxyfGTPVhjZM/Kuk1Z6tY2uN8ryu0aFXcMD0D4L/4QAaOM1Lh/WGYUlHvy285DlgD+j+7B/whjRI3Tvr6uGd8FVw7tg+jcb0K6JdtzUFMETE/r6xhY2et+/pYtN7T2Th2Ti8TkbMbJHSzSw6KH10FlHeacvP6J5PRwRYgCkVUly4W0jMHXmGvyY87eNKJFQI/KYFCjhTD+7D/7z/SYM6Rp6oFckMpvX85miIxJxaNuL2MdXDkbu/sMRv65h7TTkl5TbSgh2XHdSV0wY2B5tGgf2qnrugv749Ne/0K1V8CVjQ5l6So+Qz/do0xDZ2/ZHNe3F1cO74OrhXYLegFx4XGAvKH9Go7ZS2uDG95Zt9z7Xvmk9vHXZsZg4Y4k3YY7onoFjOzVHhVIB42OG6+1o007pgdG9W+OLlTvw1LzfUTc9FYfLnJ/vyIxJgRKOMTI7ln64NfJur4lSYnntkiw0redbTZSV2QxZmZEfa/G0kwK6v1ZFSopYJgRAK0VdcWLkM8nadfepvXDq0W3RrVVgj61m9WthX2Hwldei/WxfnpTlLUFUtpkAj5x9ND5fuQNFpgnrRMRnrI/V6HlDh2b1fMbtnH9cB7yyaAvev2IQxj/7I4CqVRVGgkmBKAjjQuz2BGwn9YhNVROg9TCqKWqnpWJQkC6+s28Yhu1RlKTCMY+YruxyqyXZn6eN9FntsCpaNqyDNfeP8dlmTMToNCYFoiAuH9YJTeql4x9x+mdMRkvvGIlDh8tiftyWjeqgZSNnB18avaSMLrRO3TzcPrYH5qzbFXIOr1gSlYgVpiFkZWWp7Oxst8MgoiSnlMKz3+XgnGPaVXkAZjyIyHKlVOCIRD8sKRARRUFEqtVkenZx8BoREXk5mhREZKyIbBSRHBGZavF8bRH5QH9+qYhkOhkPERGF5lhSEJFUAM8DOAVALwDni4j/VI2XAdivlOoK4CkAjzoVDxERhedkSeFYADlKqS1KqVIA7wM4w2+fMwC8oT/+GMBISZTO4UREScjJpNAOwHbTz7n6Nst9lFLlAA4CqB6rXRAR1UBOJgWrO37//q929oGITBGRbBHJzsvLi0lwREQUyMmkkAvAPOqnPYAdwfYRkTQAjQHs8z+QUmqGUipLKZWVkVG1ufaJiCg4J5PCLwCOFJFOIlILwHkAvvDb5wsAF+uPzwXwvapuo+mIiGoQR0c0i8g4AE8DSAXwmlLqIRF5AEC2UuoLEakD4C0A/aGVEM5TSm0Jc8w8ANtC7RNCCwA257J1DWOsukSPD0j8GBM9PiDxY0y0+DoqpcJWtVS7aS6qQkSy7QzzdhNjrLpEjw9I/BgTPT4g8WNM9PiC4YhmIiLyYlIgIiKvZEsKM9wOwAbGWHWJHh+Q+DEmenxA4seY6PFZSqo2BSIiCi3ZSgpERBRC0iSFcDO2ximGDiIyX0TWi8g6EblB336fiPwlIiv1r3Gm10zTY94oImOCHz2mcW4VkTV6LNn6tmYiMldENunfm+rbRUSe1WNcLSIDHI6tu+k8rRSRQyJyo9vnUEReE5E9IrLWtC3icyYiF+v7bxKRi63eK8YxPi4iG/Q4PhWRJvr2TBE5bDqfL5lec4z+95Gj/x4xma8sSHwRf65O/q8HifEDU3xbRWSlvj3u5zAmlFI1/gvaOInNADoDqAVgFYBeLsTRBsAA/XFDAL9Dm0H2PgC3WOzfS4+1NoBO+u+QGoc4twJo4bftMQBT9cdTATyqPx4H4BtoU5YMArA0zp/rLgAd3T6HAE4AMADA2mjPGYBmALbo35vqj5s6HONoAGn640dNMWaa9/M7zjIAg/X4vwFwioPxRfS5Ov2/bhWj3/NPALjHrXMYi69kKSnYmbHVcUqpnUqpFfrjfADrEThJoNkZAN5XSpUopf4AkAPtd3GDeUbbNwCcadr+ptIsAdBERNrEKaaRADYrpUINZozLOVRKLUTgFC2RnrMxAOYqpfYppfYDmAtgrJMxKqW+VdpklACwBNp0NEHpcTZSSv2stKvbm6bfK+bxhRDsc3X0fz1UjPrd/gQA74U6hpPnMBaSJSnYmbE1rkRbUKg/gKX6pmv1IvxrRjUD3ItbAfhWRJaLyBR9Wyul1E5AS24AWrocI6BNnWL+B0ykcwhEfs7c/ju9FNpdq6GTiPwqIgtEZJi+rZ0elyEeMUbyubp5DocB2K2U2mTalijn0LZkSQq2ZmONFxFpAGAmgBuVUocAvAigC4B+AHZCK4IC7sU9RCk1ANoCSdeIyAkh9nUlRtHm0zodwEf6pkQ7h6EEi8m1WEXkTgDlAN7RN+0EcIRSqj+AmwC8KyKNXIgx0s/Vzc/7fPjepCTKOYxIsiQFOzO2xoWIpENLCO8opT4BAKXUbqWURylVAeBlVFZvuBK3UmqH/n0PgE/1eHYb1UL69z1uxggtYa1QSu3WY02oc6iL9Jy5EqveoH0qgAv16gzo1TJ79cfLodXTd9NjNFcxORpjFJ+rW+cwDcDZAD4wtiXKOYxUsiQFOzO2Ok6vc3wVwHql1JOm7eY6+LMAGD0bvgBwnmhrWXcCcCS0BionY6wvIg2Nx9AaItfCd0bbiwF8bopxkt6jZhCAg0aVicN87soS6RyaRHrO5gAYLSJN9WqS0fo2x4jIWAC3AzhdKVVk2p4h2pK6EJHO0M7bFj3OfBEZpP89TzL9Xk7EF+nn6tb/+igAG5RS3mqhRDmHEXO7pTteX9B6fPwOLVvf6VIMQ6EVE1cDWKl/jYM2U+waffsXANqYXnOnHvNGxKGHArReG6v0r3XGuYK2It53ADbp35vp2wXaWtyb9d8hKw4x1gOwF0Bj0zZXzyG0BLUTQBm0O8HLojln0Or1c/SvyXGIMQdaHbzx9/iSvu85+ue/CsAKAKeZjpMF7eK8GcBz0AfBOhRfxJ+rk//rVjHq218HcKXfvnE/h7H44ohmIiLySpbqIyIisoFJgYiIvJgUiIjIi0mBiIi8mBSIiMiLSYGSlogs1r9nisgFMT72HVbvRZTo2CWVkp6IDIc2E+epEbwmVSnlCfF8gVKqQSziI4onlhQoaYlIgf5wOoBh+pz3/xaRVNHWGfhFn4jtCn3/4aKth/EutAFVEJHP9IkD1xmTB4rIdAB19eO9Y34vfRTz4yKyVp9Pf6Lp2D+IyMeirW/wjj7alSiu0twOgCgBTIWppKBf3A8qpQaKSG0AP4nIt/q+xwI4SmnTNQPApUqpfSJSF8AvIjJTKTVVRK5VSvWzeK+zoU3u1hdAC/01C/Xn+gPoDW0enJ8ADAHwY+x/XaLgWFIgCjQa2txEK6FNbd4c2rw1ALDMlBAA4HoRWQVtLYIOpv2CGQrgPaVN8rYbwAIAA03HzlXa5G8roS3SQhRXLCkQBRIA1ymlfCaj09seCv1+HgVgsFKqSER+AFDHxrGDKTE99oD/n+QClhSIgHxoy6Ma5gC4Sp/mHCLSTZ8x1l9jAPv1hNAD2tKahjLj9X4WApiot1tkQFveMV6zthKFxTsRIm0GznK9Guh1AM9Aq7pZoTf25sF6ucTZAK4UkdXQZupcYnpuBoDVIrJCKXWhafun0NbmXQVtxtzblFK79KRC5Dp2SSUiIi9WHxERkReTAhEReTEpEBGRF5MCERF5MSkQEZEXkwIREXkxKRARkReTAhERef1/8c+WuhOkFtkAAAAASUVORK5CYII=\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "learner.recorder!.plotLosses()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Progress bar" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "It's nice to keep track of where we're at in the training with a progress bar." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "// export\n", "import Foundation" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "// export\n", "func formatTime(_ t: Float) -> String {\n", " let t = Int(t)\n", " let (h,m,s) = (t/3600, (t/60)%60, t%60)\n", " return h != 0 ? String(format: \"%02d:%02d:%02d\", h, m, s) : String(format: \"%02d:%02d\", m, s)\n", "}" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "\"01:18\"\n" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "formatTime(78.23)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "// export\n", "public struct ProgressBar{\n", " let total: Int\n", " let length: Int = 50\n", " let showEvery: Float = 0.2\n", " let fillChar: Character = \"X\"\n", " public var comment: String = \"\"\n", " private var waitFor: Int = 0\n", " private var startTime: UInt64 = 0\n", " private var lastPrint: UInt64 = 0\n", " private var lastShow: UInt64 = 0\n", " private var estimatedTotal: Float = 0.0\n", " private var bar: String = \"\"\n", " \n", " public init(_ c: Int) { total = c }\n", " \n", " public mutating func update(_ val: Int){\n", " lastShow = DispatchTime.now().uptimeNanoseconds\n", " if val == 0 { startTime = lastShow } \n", " else {\n", " let averageTime = Float(lastShow - startTime) / (1e9 * Float(val))\n", " estimatedTotal = Float(total) * averageTime\n", " }\n", " if val == 0 || lastShow - lastPrint >= Int(1e9 * showEvery) { update_bar(val) }\n", " }\n", " \n", " public mutating func update_bar(_ val: Int){\n", " lastPrint = lastShow\n", " let prevLength = bar.count\n", " bar = String(repeating: fillChar, count: (val * length) / total)\n", " bar += String(repeating: \"-\", count: length - (val * length) / total)\n", " let pct = String(format: \"%.2f\", 100.0 * Float(val)/Float(total))\n", " let elapsedTime = Float(lastShow - startTime) / 1e9\n", " let remaingTime = estimatedTotal - elapsedTime\n", " bar += \" \\(pct)% [\\(val)/\\(total) \\(formatTime(elapsedTime))<\\(formatTime(remaingTime))\"\n", " bar += comment.isEmpty ? \"]\" : \" \\(comment)]\"\n", " if bar.count < prevLength { bar += String(repeating: \" \", count: prevLength-bar.count) }\n", " print(bar, terminator:\"\\r\")\n", " fflush(stdout)\n", " }\n", " \n", " public func remove(){\n", " print(String(repeating: \" \", count: bar.count), terminator:\"\\r\")\n", " fflush(stdout)\n", " }\n", "}" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " \r" ] } ], "source": [ "var tst = ProgressBar(100)\n", "for i in 0...100{\n", " tst.update(i)\n", " usleep(50000)\n", "}\n", "tst.remove()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "// export\n", "extension Learner {\n", " public class ShowProgress: Delegate {\n", " var pbar: ProgressBar? = nil\n", " var iter: Int = 0\n", " \n", " public override func epochWillStart(learner: Learner) {\n", " pbar = ProgressBar(learner.data.train.count)\n", " }\n", " \n", " public override func validationWillStart(learner: Learner) {\n", " if pbar != nil { pbar!.remove() }\n", " pbar = ProgressBar(learner.data.valid.count)\n", " }\n", " \n", " public override func epochDidFinish(learner: Learner) {\n", " if pbar != nil { pbar!.remove() }\n", " }\n", " \n", " public override func batchWillStart(learner: Learner) {\n", " if learner.currentIter == 0 {pbar!.update(0)}\n", " }\n", " \n", " public override func batchDidFinish(learner: Learner) {\n", " pbar!.update(learner.currentIter)\n", " }\n", " \n", " public override func trainingDidFinish(learner: Learner) {\n", " if pbar != nil { pbar!.remove() }\n", " }\n", " }\n", " \n", " public func makeShowProgress() -> ShowProgress { return ShowProgress() }\n", "}" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "let learner = Learner(data: data, lossFunc: softmaxCrossEntropy, optFunc: optFunc, modelInit: modelInit)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "learner.delegates = [learner.makeTrainEvalDelegate(), learner.makeShowProgress(), \n", " learner.makeAvgMetric(metrics: [accuracy]), learner.makeRecorder(),\n", " learner.makeNormalize(mean: mnistStats.mean, std: mnistStats.std)]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Epoch 0: [0.30917937, 0.9136] \n", "Epoch 1: [0.25163874, 0.9296] \n", " \r" ] } ], "source": [ "learner.fit(2)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Annealing" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "// export\n", "/// A non-generalized learning rate scheduler\n", "extension Learner where Opt.Scalar: BinaryFloatingPoint {\n", " public class LRScheduler: Delegate {\n", " public override var order: Int { return 1 }\n", " public typealias ScheduleFunc = (Float) -> Float\n", "\n", " // A learning rate schedule from step to float.\n", " public var scheduler: ScheduleFunc\n", " \n", " public init(scheduler: @escaping (Float) -> Float) {\n", " self.scheduler = scheduler\n", " }\n", " \n", " override public func batchWillStart(learner: Learner) {\n", " learner.opt.learningRate = Opt.Scalar(scheduler(learner.pctEpochs/Float(learner.epochCount)))\n", " }\n", " }\n", " \n", " public func makeLRScheduler(scheduler: @escaping (Float) -> Float) -> LRScheduler {\n", " return LRScheduler(scheduler: scheduler)\n", " }\n", "}" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "// export\n", "public func linearSchedule(start: Float, end: Float, pct: Float) -> Float {\n", " return start + pct * (end - start)\n", "}\n", "\n", "public func makeAnnealer(start: Float, end: Float, schedule: @escaping (Float, Float, Float) -> Float) -> (Float) -> Float { \n", " return { pct in return schedule(start, end, pct) }\n", "}" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "0.037\n" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "let annealer = makeAnnealer(start: 1e-2, end: 0.1, schedule: linearSchedule)\n", "annealer(0.3)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "let learner = Learner(data: data, lossFunc: softmaxCrossEntropy, optFunc: optFunc, modelInit: modelInit)\n", "let recorder = learner.makeRecorder()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "learner.delegates = [learner.makeTrainEvalDelegate(), learner.makeShowProgress(), \n", " learner.makeAvgMetric(metrics: [accuracy]), recorder,\n", " learner.makeNormalize(mean: mnistStats.mean, std: mnistStats.std),\n", " learner.makeLRScheduler(scheduler: annealer)]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Epoch 0: [0.22716658, 0.9338] \n", "Epoch 1: [0.15640278, 0.952] \n", " \r" ] } ], "source": [ "learner.fit(2)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xd4VAXa/vHvQ+i9d0LoHVEDiL2LroiIu6KuYlnR3XXrb4VgWbGDZV3fVdcX26pr2yWgWLH3RlBIo4XQQodACCWkPb8/Zth3ZIEQyORMkvtzXbkyc+bM5M6ZJHfOmZlnzN0RERE5mFpBBxARkdinshARkTKpLEREpEwqCxERKZPKQkREyqSyEBGRMqksRESkTCoLEREpk8pCRETKVDvoABWldevWnpCQEHQMEZEqZd68eZvdvU1Z61WbskhISCAlJSXoGCIiVYqZrTyU9XQYSkREyqSyEBGRMqksRESkTCoLEREpU1TLwsxGmtliM8sys6T9XH6ymX1vZsVmdvE+l403s6Xhj/HRzCkiIgcXtbIwszjgMeBcoD9wqZn132e1VcBVwEv7XLclcDswHBgG3G5mLaKVVUREDi6aexbDgCx3z3b3QuAVYHTkCu6+wt1TgdJ9rnsO8L6757r7VuB9YGQUs4qIyEFEsyw6AasjzueEl0X7uiIiNYK78+rcVXyQuSHqXyuaZWH7WXaob/h9SNc1swlmlmJmKZs2bSpXOBGRqmzVll1c/tS3TEpO47X5a6L+9aL5Cu4coEvE+c7A2nJc99R9rvvJviu5+3RgOkBiYuKhFpGISJVVUur846sVPDhnMXG1jHvGDOTSofFR/7rRLIu5QC8z6wasAcYBlx3idecA90Y8qH02MLniI4qIVB1LNuQzcUYq81dv4/S+bblnzEA6NGtQKV87amXh7sVmdiOhP/xxwDPunmFmdwIp7j7bzIYCs4AWwCgzu8PdB7h7rpndRahwAO5099xoZRURiWWFxaX8/ZNlPPrxUprUr8Mj44ZwwVEdMdvfEfvoMPfqcfQmMTHRNUhQRKqbBau3MSk5lUXr87ngqI7cPqo/rRrXq7DbN7N57p5Y1nrVZuqsiEh1sruwhIc/WMJTn2fTtkl9nroykTP7twssj8pCRCTGfL1sC5NnprJiyy4uHRbP5PP60rR+nUAzqSxERGLE9oIipr6ziJe+XUXXVg156brhHN+jddCxAJWFiEhM+HDhBm6Zlc7G/AImnNydP5zZmwZ144KO9R8qCxGRAG3ZsYc73shk9oK19GnXhCeuOJYhXZoHHeu/qCxERALg7sxesJY73sgkv6CIP5zZm1+e2oO6tWPznSNUFiIilWxd3m5unZXOh4s2clSX5tw/djB92jcJOtZBqSxERCpJaanzytzV3Pf2QopKS7n1J/24+oRuxNWqvBfXHS6VhYhIJVixeSdJM1P5JjuX43u0YupFg4lv1TDoWIdMZSEiEkXFJaU88+VyHnpvCXXjajH1okFcMrRLpY7qqAgqCxGRKFm0fjuTZqSyICePM/u14+4LB9K+Wf2gYx0WlYWISAXbU1zCYx8v4/GPs2jWoA5/u/Rozh/cocrtTURSWYiIVKAfVm1lUnIqSzbsYMzRnbjt/P60bFQ36FhHTGUhIlIBdhUW89B7S3jmy+W0b1qfZ68ayml92wYdq8KoLEREjtBXWZtJmpnGqtxd/Py4eCaN7EuTgAf/VTSVhYjIYcrbXcR9by/klbmr6da6Ea9OOI7h3VsFHSsqVBYiIofhvYz13PpaOpt37OH6U0KD/+rXiZ3BfxVNZSEiUg6bd+xhyuwM3kxdR9/2TXhqfCKDO8fe4L+KprIQETkE7s5r89dwxxuZ7NpTwv87qzc3nNqDOnGxOfivoqksRETKsHbbbm6ZlcbHizdxTHxzpo0dTK92sT34r6KpLEREDqC01Hnxu1VMfXshpQ63j+rPlSMSqsTgv4qmshAR2Y/sTTtISk7juxW5nNizNfddNIguLavO4L+KprIQEYlQXFLKU18s5+H3l1Cvdi3uv3gwPz22c5Ue1VERVBYiImGZa7czMXkB6Wu2c86Adtw1eiBtm1bNwX8VTWUhIjXenuISHv0oi79/sozmDevw+OXHcO7A9jV+byKSykJEarR5K3OZlJxG1sYdjD2mM7ed34/mDav+4L+KprIQkRpp555iHpizmOe+XkHHZg147pphnNK7TdCxYpbKQkRqnM+XbmLyzDRytu5m/Iiu3DSyL43r6c/hwWjriEiNkberiLvfyuTf83Lo3qYR/75hBEMTWgYdq0pQWYhIjfBu+npuez2d3J2F/OrUHvz2jF7VevBfRVNZiEi1tjG/gCmzM3g7bT39OzTl2auGMrBTs6BjVTkqCxGpltyd5O/XcNebmewuKuGmc/ow4eTuNWbwX0VTWYhItZOzdRc3z0rnsyWbSOzagqljB9OzbeOgY1VpKgsRqTZKS50XvlnJtHcXAXDHBQO44riu1KqBg/8qmspCRKqFZZt2MGlGKikrt3Jy7zbcO2YgnVvU3MF/FS2qB+/MbKSZLTazLDNL2s/l9czs1fDl35pZQnh5HTN7zszSzGyhmU2OZk4RqbqKSkp57OMszn3kc5Zu3MGDPz2K564eqqKoYFHbszCzOOAx4CwgB5hrZrPdPTNitWuBre7e08zGAdOAS4CfAvXcfZCZNQQyzexld18RrbwiUvWkr8lj4oxUMtdt57xB7ZlywQDaNtHgv2iI5mGoYUCWu2cDmNkrwGggsixGA1PCp2cAj1pocpcDjcysNtAAKAS2RzGriFQhBUUlPPLhUqZ/lk3LRnV54ufHMHJgh6BjVWvRLItOwOqI8znA8AOt4+7FZpYHtCJUHKOBdUBD4A/unhvFrCJSRcxdkcukGalkb97JT4/tzK0/6U+zhnWCjlXtRbMs9vf0Az/EdYYBJUBHoAXwuZl9sHcv5T9XNpsATACIj48/4sAiErt27Cnm/ncX8fzXK+ncogEvXDuMk3pp8F9liWZZ5ABdIs53BtYeYJ2c8CGnZkAucBnwrrsXARvN7EsgEfhRWbj7dGA6QGJi4r5FJCLVxKdLNnHzzDTW5u3mquMTuOmcPjTS4L9KFc1nQ80FeplZNzOrC4wDZu+zzmxgfPj0xcBH7u7AKuB0C2kEHAcsimJWEYlBW3cW8sd/zWf8M99Rv04tZtwwgikXDFBRBCBqWzz8GMSNwBwgDnjG3TPM7E4gxd1nA08DL5hZFqE9inHhqz8GPAukEzpU9ay7p0Yrq4jEFnfnnfT1/Pn1dLbtKuI3p/fk16f11OC/AFnoH/mqLzEx0VNSUoKOISJHaOP2Am57PZ05GRsY1KkZ08YOpn/HpkHHqrbMbJ67J5a1nvblRCQmuDv/npfD3W9msqe4lKRz+/KLE7tRW4P/YoLKQkQCtzp3F5NnpvFF1maGJbRk6thBdG+jwX+xRGUhIoEpKXWe/3oF97+7mFoGd104kMuHxWvwXwxSWYhIIJZuyGdScirfr9rGqX3acM+YQXRq3iDoWHIAKgsRqVRFJaU88cky/vZRFo3qxfHXS4YwekhHQpN+JFapLESk0qTl5HHTjAUsWp/P+YM7MOWCAbRuXC/oWHIIVBYiEnUFRSU8/MESnvwsm9aN6zH9imM5e0D7oGNJOagsRCSqvs3eQtLMNJZv3sm4oV2YfF4/mjXQ4L+qRmUhIlGRX1DEtHcX8c9vVtGlZQNe/MVwTujZOuhYcphUFiJS4T5etJGbZ6WxYXsBvzixG388uzcN6+rPTVWme09EKkzuzkLufCOD1+avpVfbxjz+y+M5Or5F0LGkAqgsROSIuTtvpq5jyuwM8nYX8bszevGr03pQr7YG/1UXKgsROSIbthdwy6x0Pli4gcGdm/HidcPp216D/6oblYWIHBZ359W5q7nn7YUUFpdyy3n9uPqEBA3+q6ZUFiJSbiu37GTyzDS+WraF4d1aMm3sYBJaNwo6lkSRykJEDllJqfPsl8t58L3F1KlVi3vHDGLc0C4a/FcDqCxE5JAsXp/PxORUFqzexhl923L3mIF0aKbBfzWFykJEDqqwuJTHP8nisY+zaFK/Do+MG8IFR2nwX02jshCRA1qwehsTZ6SyeEM+o4d05M/n96eVBv/VSCoLEfkvuwtL+Mv7i3n6i+W0bVKfp65M5Mz+7YKOJQFSWYjIj3y1bDOTZ6axcssuLhseT9K5fWlaX4P/ajqVhYgAsL2giPveXsTL362ia6uGvHzdcYzo0SroWBIjVBYiwgeZG7jltTQ25e9hwsnd+cOZvWlQV6M65P+oLERqsC079nDHG5nMXrCWvu2bMP2KRI7q0jzoWBKDVBYiNZC7M3vBWqbMzmDHnmL+cGZvfnlqD+rW1qgO2T+VhUgNsy5vN7fOSufDRRsZ0qU59188mN7tmgQdS2KcykKkhigtdV6eu4r73l5ESalz2/n9uer4BOI0qkMOgcpCpAZYvnknScmpfLs8lxN6tuK+MYOJb9Uw6FhShagsRKqx4pJSnvlyOQ+9t4S6tWsxbewgfpbYRaM6pNxUFiLV1MJ125mUnEpqTh5n9W/H3RcOpF3T+kHHkipKZSFSzewpLuGxj5fx+MdZNGtQh0cvO5qfDOqgvQk5IioLkWrk+1VbmTQjlaUbdzDm6E78+fz+tGhUN+hYUg2oLESqgV2FxTw4ZwnPfrWcDk3r8+xVQzmtb9ugY0k1orIQqeK+zNpM0sxUVufu5orjujJxZB+aaPCfVDCVhUgVlbe7iHvfWsirKavp1roRr044juHdNfhPoiOqZWFmI4FHgDjgKXefus/l9YDngWOBLcAl7r4ifNlg4H+BpkApMNTdC6KZV6SqeC9jPbe+ls6WnYXccEoPfn9mL+rX0eA/iZ6olYWZxQGPAWcBOcBcM5vt7pkRq10LbHX3nmY2DpgGXGJmtYF/Ale4+wIzawUURSurSFWxKX8PU97I4K3UdfTr0JSnxw9lUOdmQceSGiCaexbDgCx3zwYws1eA0UBkWYwGpoRPzwAetdDz+84GUt19AYC7b4liTpGY5+7M+mENd76Zya49Jfzp7N5cf0oP6sRp8J9UjmiWRSdgdcT5HGD4gdZx92IzywNaAb0BN7M5QBvgFXe/P4pZRWLWmm27uWVWGp8s3sQx8aHBfz3bavCfVK5olsX+XgHkh7hObeBEYCiwC/jQzOa5+4c/urLZBGACQHx8/BEHFoklpaXOi9+uZOo7i3Bgyqj+XDFCg/8kGNEsixygS8T5zsDaA6yTE36cohmQG17+qbtvBjCzt4FjgB+VhbtPB6YDJCYm7ltEIlVW9qYdJCWn8d2KXE7q1Zp7xwyiS0sN/pPgRLMs5gK9zKwbsAYYB1y2zzqzgfHA18DFwEfuvvfw00QzawgUAqcAD0cxq0hMKC4p5cnPl/PwB0uoX7sWD1w8mIuP7axRHRK4qJVF+DGIG4E5hJ46+4y7Z5jZnUCKu88GngZeMLMsQnsU48LX3WpmfyFUOA687e5vRSurSCzIWJvHpORU0tds55wB7bhr9EDaavCfxAhzrx5HbxITEz0lJSXoGCLlVlBUwt8+WsoTn2bTomFd7ho9gHMHdQg6ltQQ4ceDE8taT6/gFgnQvJW5TJyRyrJNOxl7TGduO78fzRtq8J/EHpWFSAB27inmgTmLee7rFXRs1oDnrhnGKb3bBB1L5IBUFiKV7LMlm5g8M421ebu58riu3DSyL43r6VdRYpt+QkUqSd6uIu56K5MZ83Lo3qYR/7p+BEMTWgYdS+SQqCxEKsG76eu47fUMcncW8qtTe/DbMzT4T6qWMsvCzGoRmtM0sBLyiFQrG/MLuP31DN5JX8+Ajk159qqhDOykwX9S9ZRZFu5eamYLzCze3VdVRiiRqs7dmTEvh7vfWsjuohImjuzDdSd11+A/qbIO9TBUByDDzL4Ddu5d6O4XRCWVSBW2OncXN89K4/Olmxma0IKpYwfTo03joGOJHJFDLYs7oppCpBooLXWe/3oF989ZjAF3jh7Az4d3pZYG/0k1cEhl4e6fRjuISFWWtXEHScmppKzcysm923DvmIF0bqHBf1J9HLQszCyf/x4rDqHR4u7uTaOSSqSKKCopZfpn2TzywVIa1I3joZ8exUXHdNLgP6l2DloW7q53WBE5gPQ1eUyckUrmuu38ZFAHplwwgDZN6gUdSyQq9DoLkXIqKCrhkQ+XMv2zbFo2qssTPz+WkQPbBx1LJKpUFiLlMHdFLpNmpJK9eSc/S+zMLef1p1nDOkHHEok6lYXIIdixp5j7313E81+vpHOLBvzz2uGc2Kt10LFEKo3KQqQMHy/eyC0z01i3vYCrT0jgT2f3oZEG/0kNo594kQPYurOQu97MZOYPa+jZtjEzbjieY7u2CDqWSCBUFiL7cHfeTlvP7bPT2bariN+e3pNfn96TerU1+E9qLpWFSISN2wu49bV03svcwKBOzXj+muH076iXE4moLEQI7U38OyWHu97KpLC4lMnn9uXaE7tRW4P/RACVhQirtoQG/32RtZlh3Voy9aJBdNfgP5EfUVlIjVVS6vzjqxU8OGcxcbWMuy8cyGXD4jX4T2Q/VBZSIy3dkM/E5FR+WLWN0/q04Z4xg+jYvEHQsURilspCapTC4lKe+HQZj36URaN6cfz1kiGMHtJRg/9EyqCykBojNWcbE2eksmh9PqOO6sjto/rTurEG/4kcCpWFVHsFRSU8/P4Snvw8mzZN6vHklYmc1b9d0LFEqhSVhVRr32RvISk5lRVbdnHpsC4knduPZg00+E+kvFQWUi3lFxQx9Z1FvPjtKuJbNuSlXwzn+J4a/CdyuFQWUu18tGgDt8xKZ8P2An5xYjf+eHZvGtbVj7rIkdBvkFQbuTsLufONDF6bv5be7Rrz+OXHc3S8Bv+JVASVhVR57s4bqeuYMjuD/IIifndGL359Wk/q1taoDpGKorKQKm19Xmjw3wcLN3BU52ZMu3g4fdtr8J9IRVNZSJXk7rwydzX3vrWQotJSbjmvH9ec2I04jeoQiQqVhVQ5K7fsJCk5ja+zt3Bc95ZMvWgwCa0bBR1LpFpTWUiVUVLqPPvlch58bzF1atXivosGcUliFw3+E6kEUX0E0MxGmtliM8sys6T9XF7PzF4NX/6tmSXsc3m8me0wsz9FM6fEvsXr87no719x91sLObFna97/4ylcqgmxIpUmansWZhYHPAacBeQAc81strtnRqx2LbDV3Xua2ThgGnBJxOUPA+9EK6PEvsLiUh7/JIvHPs6iSf06/M+lRzNqcAcN/hOpZNE8DDUMyHL3bAAzewUYDUSWxWhgSvj0DOBRMzN3dzO7EMgGdkYxo8Sw+au3MWlGKos35DN6SEduHzWAlo3qBh1LpEaKZll0AlZHnM8Bhh9oHXcvNrM8oJWZ7QYmEdor0SGoGmZ3YQkPvbeYZ75cTtsm9Xl6fCJn9NPgP5EgRbMs9necwA9xnTuAh919x8EON5jZBGACQHx8/GHGlFjy1bLNJCWnsSp3F5cNjyfp3L40ra/BfyJBi2ZZ5ABdIs53BtYeYJ0cM6sNNANyCe2BXGxm9wPNgVIzK3D3RyOv7O7TgekAiYmJ+xaRVCHbC4q47+2FvPzdahJaNeTl645jRI9WQccSkbBolsVcoJeZdQPWAOOAy/ZZZzYwHvgauBj4yN0dOGnvCmY2Bdixb1FI9fFB5gZueS2NTfl7uP7k7vz+zN40qBsXdCwRiRC1sgg/BnEjMAeIA55x9wwzuxNIcffZwNPAC2aWRWiPYly08kjs2bJjD1PeyOSNBWvp274JT16ZyODOzYOOJSL7YaF/5Ku+xMRET0lJCTqGHAJ35/X5a7njjQx27CnmN6f34oZTemjwn0gAzGyeuyeWtZ5ewS2Vau223dz6WjofLdrIkC7Nuf/iwfRu1yToWCJSBpWFVIrSUuel71Yx9Z1FlJQ6t53fn6uOT9DgP5EqQmUhUbd8806SklP5dnkuJ/RsxX1jBhPfqmHQsUSkHFQWEjXFJaU8/cVy/vL+EurWrsX9Ywfz08TOGtUhUgWpLCQqMtduZ1JyKmlr8jirfzvuvnAg7ZrWDzqWiBwmlYVUqD3FJTz6URZ//2QZzRvW4bHLjuG8Qe21NyFSxakspMLMW7mVScmpZG3cwUVHd+K28/vTQoP/RKoFlYUcsV2FxTwwZzH/+GoFHZrW59mrh3Jan7ZBxxKRCqSykCPyxdLNJM1MJWfrbq4c0ZWJI/vSuJ5+rESqG/1Wy2HJ213EPW9l8q+UHLq1bsS/rh/BsG4tg44lIlGispBym5OxntteS2fLzkJ+eWoPfndGL+rX0eA/kepMZSGHbFP+HqbMzuCttHX069CUp8cPZVDnZkHHEpFKoLKQMrk7M79fw51vZrK7sISbzunDhJO7UydOg/9EagqVhRzUmm27uXlmGp8u2cSxXVswbewgerbV4D+RmkZlIftVWur889uVTHtnEQ5MGdWfK0ckUEuD/0RqJJWF/Jdlm3aQlJzK3BVbOalXa+4dM4guLTX4T6QmU1nIfxSVlPLk59n89YOl1K9diwcuHszFx2rwn4ioLCQsfU0ek5JTyVi7nZED2nPnhQNo20SD/0QkRGVRwxUUlfC3j5byxKfZtGhYl79ffgznDuoQdCwRiTEqixosZUUuE5NTyd60k4uP7cytP+lH84Ya/Cci/01lUQPt3BMa/Pfc1yvo2KwBz18zjJN7twk6lojEMJVFDfPpkk3cPDONtXm7GT8igZvO6UMjDf4TkTLor0QNsW1XIXe9uZDk73Po3qYR/75+BIkJGvwnIodGZVEDvJO2jttez2DrrkJ+fVoPfnO6Bv+JSPmoLKqxjdsL+PPrGbybsZ4BHZvy3DVDGdBRg/9EpPxUFtWQuzNjXg53vZlJQXEpk0b25bqTulFbg/9E5DCpLKqZ1bm7uHlWGp8v3czQhBZMHTuYHm0aBx1LRKo4lUU1UVLqPP/1Ch6YsxgD7ho9gMuHd9XgPxGpECqLaiBrYz6TktOYt3Irp/Ruwz1jBtK5hQb/iUjFUVlUYUUlpfzvp8v4nw+zaFgvjr/87CjGHN1Jg/9EpMKpLKqo9DV53DQjlYXrtvOTwR2YMmoAbZrUCzqWiFRTKosqpqCohL9+sJQnP8+mZaO6/O8Vx3LOgPZBxxKRak5lUYV8tzyXpORUsjfv5JLELtx8Xj+aNawTdCwRqQFUFlVAfkER97+7mBe+WUnnFg3457XDObFX66BjiUgNorKIcR8v3sgtM9NYt72Aa07oxp/O6U3DurrbRKRyRfUlvWY20swWm1mWmSXt5/J6ZvZq+PJvzSwhvPwsM5tnZmnhz6dHM2cs2rqzkD++Op+rn51Lw3q1mXHD8fx5VH8VhYgEImp/ecwsDngMOAvIAeaa2Wx3z4xY7Vpgq7v3NLNxwDTgEmAzMMrd15rZQGAO0ClaWWOJu/NW2jpufz2DvN1F/Pb0nvz69J7Uq63BfyISnGj+mzoMyHL3bAAzewUYDUSWxWhgSvj0DOBRMzN3/yFinQygvpnVc/c9UcwbuA3bC7jttXTey9zAoE7N+OcvhtOvQ9OgY4mIRLUsOgGrI87nAMMPtI67F5tZHtCK0J7FXmOBH6pzUbg7/0pZzd1vLaSwuJTJ5/bl2hM1+E9EYkc0y2J/LyP28qxjZgMIHZo6e79fwGwCMAEgPj7+8FIGbNWWXSTNTOWrZVsY1q0l08YOplvrRkHHEhH5kWiWRQ7QJeJ8Z2DtAdbJMbPaQDMgF8DMOgOzgCvdfdn+voC7TwemAyQmJu5bRDGtpNT5x1creHDOYuJqGXdfOJDLhsVr8J+IxKRolsVcoJeZdQPWAOOAy/ZZZzYwHvgauBj4yN3dzJoDbwGT3f3LKGYMxJIN+Uyckcr81ds4rU8b7hkziI7NGwQdS0TkgKJWFuHHIG4k9EymOOAZd88wszuBFHefDTwNvGBmWYT2KMaFr34j0BO4zcxuCy872903RitvZSgsLuWJT5fxt4+W0rhebR4ZN4QLjuqowX8iEvPMvUodvTmgxMRET0lJCTrGAS1YvY1JyaksWp/PqKM6MmVUf1o11uA/EQmWmc1z98Sy1tMrvKJsd2EJD3+whKc+z6ZNk3o8eWUiZ/VvF3QsEZFyUVlE0dfLtjB5Ziortuzi0mFdmHxeP5rW1+A/Eal6VBZRsL2giKnvLOKlb1cR37IhL/1iOMf31OA/Eam6VBYV7KNFG7h5Zjob8wu47qRu/PGsPjSoq1EdIlK1qSwqyJYde7jzzUxen7+WPu2a8MQVxzKkS/OgY4mIVAiVxRFyd2YvWMsdb2SSX1DE78/sxa9O7Und2hrVISLVh8riCKzL282ts9L5cNFGjurSnPvHDqZP+yZBxxIRqXAqi8NQWuq8Mnc19729kKLSUm79ST+uPqEbcRrVISLVlMqinFZs3knSzFS+yc5lRPdWTB07iK6tNPhPRKo3lcUhKil1nvliOQ+9v5g6tWpx30WDGDe0i0Z1iEiNoLI4BIvWb2fSjFQW5ORxZr+23H3hINo3qx90LBGRSqOyOIg9xSU89vEyHv84i2YN6vC3S4/m/MEdtDchIjWOyuIAfli1lUnJqSzZsIMLh3Tkz6MG0LJR3aBjiYgEQmWxj12FxTz03hKe+XI57ZvW55mrEjm9rwb/iUjNprKI8FXWZpJmprEqdxeXD48n6dy+NNHgPxERlQVA3u4i7nt7Ia/MXU1Cq4a8MuE4juveKuhYIiIxo8aXRWrONq57PoVN+Xu4/pTu/OHM3tSvo8F/IiKRanxZxLdsSO92TXjyykQGd9bgPxGR/anxZdG8YV1euHZ40DFERGKaRqOKiEiZVBYiIlImlYWIiJRJZSEiImVSWYiISJlUFiIiUiaVhYiIlEllISIiZTJ3DzpDhTCzTcDKI7iJ1sDmCooTDbGeD2I/Y6zng9jPGOv5QBnLq6u7tylrpWpTFkfKzFLcPTHoHAcS6/kg9jPGej6I/Yyxng+UMVp0GEpERMqkshARkTKpLP7P9KADlCHW80HsZ4z1fBD7GWM9HyhjVOgxCxERKZP2LEREpEw1vizMbKQJPf1FAAAGsklEQVSZLTazLDNLCihDFzP72MwWmlmGmf0uvHyKma0xs/nhj/MirjM5nHmxmZ1TSTlXmFlaOEtKeFlLM3vfzJaGP7cILzcz+59wxlQzO6YS8vWJ2FbzzWy7mf0+yO1oZs+Y2UYzS49YVu5tZmbjw+svNbPxlZDxATNbFM4xy8yah5cnmNnuiG35RMR1jg3/fGSFvw+LcsZy36/R+n0/QL5XI7KtMLP54eWBbMMj5u419gOIA5YB3YG6wAKgfwA5OgDHhE83AZYA/YEpwJ/2s37/cNZ6QLfw9xBXCTlXAK33WXY/kBQ+nQRMC58+D3gHMOA44NsA7tv1QNcgtyNwMnAMkH642wxoCWSHP7cIn24R5YxnA7XDp6dFZEyIXG+f2/kOGBHO/w5wbpQzlut+jebv+/7y7XP5Q8Cfg9yGR/pR0/cshgFZ7p7t7oXAK8Doyg7h7uvc/fvw6XxgIdDpIFcZDbzi7nvcfTmQReh7CcJo4Lnw6eeACyOWP+8h3wDNzaxDJeY6A1jm7gd7oWbUt6O7fwbk7ufrlmebnQO87+657r4VeB8YGc2M7v6euxeHz34DdD7YbYRzNnX3rz30V+/5iO8rKhkP4kD3a9R+3w+WL7x38DPg5YPdRrS34ZGq6WXRCVgdcT6Hg/+RjjozSwCOBr4NL7oxfCjgmb2HKwgutwPvmdk8M5sQXtbO3ddBqPSAtgFn3GscP/7ljKXtWN5tFvS2vIbQf7l7dTOzH8zsUzM7KbysUzjXXpWVsTz3a1Db8SRgg7svjVgWS9vwkNT0stjf8cDAnh5mZo2BZOD37r4d+DvQAxgCrCO0KwvB5T7B3Y8BzgV+bWYnH2TdwLatmdUFLgD+HV4Ua9vxQA6UJ8hteQtQDLwYXrQOiHf3o4E/Ai+ZWdOAMpb3fg1qO17Kj/9xiaVteMhqelnkAF0izncG1gYRxMzqECqKF919JoC7b3D3EncvBZ7k/w6RBJLb3deGP28EZoXzbNh7eCn8eWOQGcPOBb539w3hvDG1HSn/NgskZ/iB9POBy8OHRQgf2tkSPj2P0GMAvcMZIw9VRT3jYdyvlb4dzaw2cBHwakTumNmG5VHTy2Iu0MvMuoX/Gx0HzK7sEOFjmk8DC939LxHLI4/xjwH2PtNiNjDOzOqZWTegF6EHxqKZsZGZNdl7mtADoOnhLHufnTMeeD0i45XhZ/gcB+TtPfRSCX70n1wsbceIr1uebTYHONvMWoQPtZwdXhY1ZjYSmARc4O67Ipa3MbO48OnuhLZZdjhnvpkdF/55vjLi+4pWxvLer0H8vp8JLHL3/xxeiqVtWC5BP8Ie9AehZ6AsIdTutwSU4URCu5upwPzwx3nAC0BaePlsoEPEdW4JZ15MJTxjgtAzSBaEPzL2biugFfAhsDT8uWV4uQGPhTOmAYmVtC0bAluAZhHLAtuOhEprHVBE6D/Haw9nmxF63CAr/HF1JWTMInR8f+/P4xPhdceG7/8FwPfAqIjbSST0B3sZ8CjhF/1GMWO579do/b7vL194+T+AG/ZZN5BteKQfegW3iIiUqaYfhhIRkUOgshARkTKpLEREpEwqCxERKZPKQkREyqSyENkPM/sq/DnBzC6r4Nu+eX9fSySW6amzIgdhZqcSmmx6fjmuE+fuJQe5fIe7N66IfCKVRXsWIvthZjvCJ6cCJ4Xfd+APZhZnofd6mBseYHd9eP1TLfSeJC8ReqEYZvZaeOhixt7Bi2Y2FWgQvr0XI79W+JXbD5hZevg9DS6JuO1PzGyGhd5j4sXwK3xFKk3toAOIxLgkIvYswn/089x9qJnVA740s/fC6w4DBnpoLDbANe6ea2YNgLlmluzuSWZ2o7sP2c/XuojQULyjgNbh63wWvuxoYAChWUFfAicAX1T8tyuyf9qzECmfswnNb5pPaIx8K0KzfQC+iygKgN+a2QJC7wfRJWK9AzkReNlDw/E2AJ8CQyNuO8dDQ/PmE3oDHZFKoz0LkfIx4Dfu/qNBfuHHNnbuc/5MYIS77zKzT4D6h3DbB7In4nQJ+t2VSqY9C5GDyyf0Vrd7zQF+GR4pj5n1Dk/h3VczYGu4KPoSepvUvYr2Xn8fnwGXhB8XaUPorTorYwquSJn034nIwaUCxeHDSf8AHiF0COj78IPMm9j/W1++C9xgZqmEJp9+E3HZdCDVzL5398sjls8i9P7LCwhNIZ7o7uvDZSMSKD11VkREyqTDUCIiUiaVhYiIlEllISIiZVJZiIhImVQWIiJSJpWFiIiUSWUhIiJlUlmIiEiZ/j8+Z9asAcq3CAAAAABJRU5ErkJggg==\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "recorder.plotLRs()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "More annealing functions" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "// export\n", "public func constantSchedule(start: Float, end: Float, pct: Float) -> Float {\n", " return start\n", "}\n", "\n", "public func cosineSchedule(start: Float, end: Float, pct: Float) -> Float {\n", " return start + (1 + cos(Float.pi*(1-pct))) * (end-start) / 2\n", "}\n", "\n", "public func expSchedule(start: Float, end: Float, pct: Float) -> Float {\n", " return start * pow(end / start, pct)\n", "}" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "// export\n", "public func combineSchedules(pcts: [Float], schedules: [(Float) -> Float]) -> ((Float) -> Float){\n", " var cumPcts: [Float] = [0]\n", " for pct in pcts {cumPcts.append(cumPcts.last! + pct)}\n", " func inner(pct: Float) -> Float{\n", " if (pct == 0.0) { return schedules[0](0.0) }\n", " if (pct > 1.0) { return schedules.last!(1.0) }\n", " let i = cumPcts.firstIndex(where: {$0 >= pct})! - 1\n", " let actualPos = (pct-cumPcts[i]) / (cumPcts[i+1]-cumPcts[i])\n", " return schedules[i](actualPos)\n", " }\n", " return inner\n", "}" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "let mySchedule = combineSchedules(pcts: [0.3, 0.7], \n", " schedules: [makeAnnealer(start: 0.3, end: 0.6, schedule: cosineSchedule),\n", " makeAnnealer(start: 0.6, end: 0.2, schedule: cosineSchedule)])" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "let learner = Learner(data: data, lossFunc: softmaxCrossEntropy, optFunc: optFunc, modelInit: modelInit)\n", "let recorder = learner.makeRecorder()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "learner.delegates = [learner.makeTrainEvalDelegate(), learner.makeShowProgress(), \n", " learner.makeAvgMetric(metrics: [accuracy]), recorder,\n", " learner.makeNormalize(mean: mnistStats.mean, std: mnistStats.std),\n", " learner.makeLRScheduler(scheduler: mySchedule)]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Epoch 0: [0.37618634, 0.8993] \n", "Epoch 1: [0.11197992, 0.9672] \n", " \r" ] } ], "source": [ "learner.fit(2)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xd8VfX9x/HXJ5uRQCBhJoGw9wygIq4qwwEoVHC0Wgdqpe5arKMtrVat1VbFgXXVioCjSiuKVEEQRRIg7JWwEvZeIfvz++Me+rumgSSQk3Nz7+f5eNxH7j33e27eOTe5n5zzPef7FVXFGGOMOZUwrwMYY4wJfFYsjDHGVMiKhTHGmApZsTDGGFMhKxbGGGMqZMXCGGNMhaxYGGOMqZAVC2OMMRWyYmGMMaZCEV4HqC4JCQnaunVrr2MYY0ytsnjx4r2qmlhRu6ApFq1btyYjI8PrGMYYU6uIyJbKtLPDUMYYYypkxcIYY0yFrFgYY4ypkBULY4wxFbJiYYwxpkKuFgsRGSoi60QkS0QmnKTN1SKyWkRWicgUv+U3iMgG53aDmzmNMcacmmunzopIODAJuATIBdJFZIaqrvZr0x54CBioqgdEpImzvBHwGyANUGCxs+4Bt/IaY4w5OTevs+gPZKnqRgARmQqMAFb7tbkVmHSiCKjqbmf5EGC2qu531p0NDAXeczGvcUlhcSkrth1iw64j7D1aQEkpREYI0RHhNK4XRaN6USTUjyalcV3qRwfNpT/GBBU3/zJbAjl+j3OBAWXadAAQkQVAOPBbVf38JOu2LPsNRGQcMA4gJSWl2oKb6pG+eT/vLtzCrFW7OF5UUql1msXF0CaxHl1bxNEnJZ4+reJpGhfjclJjTEXcLBZSzjIt5/u3By4AkoD5ItKtkuuiqpOByQBpaWn/87zxxpodh3li5hrmb9hLbEwEI3u35PwOCXRt0YAmcdGEi1BcqhwvLGF/XiH7jxWy+3ABm/cdI3vPUbJ3H+Xtb7fw2vxNALRqXJcLOzbhgo6JnNWmMTGR4R7/hMaEHjeLRS6Q7Pc4CdheTpuFqloEbBKRdfiKRy6+AuK/7lzXkppqUVRSystzs3n+yw3ExkTwyGWduW5AK+pE/e+He0Q4xESGE18virbljEpTUFzC6u2HWbL1IAuy9jI1fStvfbuZulHhDO3ajCv7tOSctgmEh5X3f4UxprqJqjv/kItIBLAe+BGwDUgHrlXVVX5thgLXqOoNIpIALAV64XRqA32cpkuAvif6MMqTlpamNjaUdw7mFfLzd5fwbfY+RvRqwe+Gd6Vh3ahqe/38ohK+27iPz1fsZObKHRzJL6ZJbDRj+iVz/Vmt7FCVMadJRBaralqF7dwqFk6IS4G/4OuPeENVHxeRiUCGqs4QEQH+jK/zugR4XFWnOuveBPzaeanHVfXNU30vKxbe2bz3GDe+uYjtB/N54qrujO6b5Or3yy8q4au1u/lgcS5z1u0mXIRLuzfnpnNT6ZXc0NXvbUywCYhiUZOsWHhj095jjJ38HYXFpbz20zTSWjeq0e+/Zd8x3v52C+9n5HCkoJjzOyRyz8Xt6Z0SX6M5jKmtrFgY123Zd4yrX/2OohJlyq0D6NQszrMsRwuKeee7LUyel82BvCIu6JjIg0M60aWFd5mMqQ2sWBhXHThWyFUvf8vBvEKmjjubjs1ivY4EwLGCYv7+3RZenZfN4eNFjOmXwv2DO5BQP9rraMYEpMoWCxsbylRZflEJ497JYNvB47z207SAKRQA9aIjuOOCtnz9wIXceE4q72fkcOGf5vLavI0Ul5R6Hc+YWsuKhamy33yyivTNB/jzj3vWeB9FZTWoG8ljV3Rh1r3n0S+1EY/PXMPIlxawctshr6MZUytZsTBV8uHiXKZl5HDnhW25omcLr+NUqG1ifV6/IY2Xr+vDrsMFjJi0gD9+tobjhZW7otwY42PFwlTahl1HeOTjlQxIbcS9F3fwOk6liQjDujfnP/eez+g+Sbz69UYuf2G+7WUYUwVWLEylFBaX8ov3llIvOpwXrulNRHjt+9VpUDeSp0b34B83D+BoQTFXvrSAyfOyKS0NjpM8jHFT7fuLN5544asNrN15hKdH96BJLb9a+tz2CXx+93lc1KkJT8xcy0/e+J7dR/K9jmVMQLNiYSq0PPcgL83NZlSfJC7q1NTrONUivl4Ur1zflyev6s7iLQe4/PlvyNh80tFkjAl5VizMKRUUl/DL95eTUD+Kx67o4nWcaiUijO2fwsd3DqRuVDhjJy/kzQWbCJZrj4ypTlYszCn9bf4m1u06wh+v6k6DOpFex3FFp2ZxfDL+XC7omMjv/rWae6Zl2tlSxpRhxcKc1LaDx3nxqyyGdG0aNIefTqZBnUgm/ySNBwZ3YMay7Yx9baH1Yxjjx4qFOak//Hs1ivLo5cF1+OlkwsKE8Re155Xr+7J+5xGunPQt63Ye8TqWMQHBioUp17z1e/hs5U7GX9iOpPi6XsepUUO6NmP6bWdTVFLKqJe/5ev1e7yOZIznrFiY/1FcUsrEf6+mdeO63HpeG6/jeKJ7UgM+GT+Q5EZ1uemtdKan51S8kjFBzNViISJDRWSdiGSJyIRynr9RRPaISKZzu8XvuRK/5TPczGl+6P3FuWTtPspDl3YmOiJ057tu3qAOH9x+Nue0bcyDHy5n8rxsryMZ4xnX5uAWkXBgEnAJvjm100VkhqquLtN0mqqOL+cljqtqL7fymfLlFRbz7Oz19G0Vz+Auwd2pXRn1oiN4/YZ+3Ds9kydmrmX/sSJ+NbQjvkkejQkdrhULoD+QpaobAURkKjACKFssTAB5ff4m9hwp4JXr+9gHoiMqIoznx/amQZ1IXvk6m4N5hTx+ZXfCw2z7mNDh5mGoloD/gd5cZ1lZo0RkuYh8ICLJfstjRCRDRBaKyEgXcxrH3qMFvDpvI0O6NqVvq8Acetwr4WHC4yO7Mf7CdkxNz+GeaZk2P4YJKW7uWZT3b1fZS2P/BbynqgUicjvwNnCR81yKqm4XkTbAVyKyQlV/cNBYRMYB4wBSUlKqN30IemlONseLSnhwaCevowQkEeGBIR2pHxPBk5+tpVSVv47pVSsHVTSmqtz8Lc8F/PcUkoDt/g1UdZ+qFjgPXwP6+j233fm6EZgL9C77DVR1sqqmqWpaYmJi9aYPMbsP5/Pu91u4qndL2ibW9zpOQLv9/Lb8+tJOfLp8B3dPzaTI9jBMCHCzWKQD7UUkVUSigLHAD85qEpHmfg+HA2uc5fEiEu3cTwAGYn0drnrl640UlyrjL2rndZRaYdx5bXn40s58umIHd09dagXDBD3XDkOparGIjAdmAeHAG6q6SkQmAhmqOgO4S0SGA8XAfuBGZ/XOwKsiUoqvoD1ZzllUppr471W0alzP6zi1xq3ntUEE/vDpGlSX1tp5PoypDDf7LFDVmcDMMsse87v/EPBQOet9C3R3M5v5f7ZXcfpuGeS7aPEPn67hwQ+X88zonoTZWVImCLlaLEzg233Et1dxpe1VnLZbBrUhr7CEZ2evJzY6gt8O72qnHZugY8UixE0+sVdxoe1VnIlfXNSOowXFTJ63kfoxEfxyiJ1RZoKLFYsQdiiviCmLtnJFj+a0TrC9ijMhIjw0rBNH8ouZNCeb2JhIbj+/rdexjKk2VixC2D++30JeYQm32YdatRAR/jCyG0cLinnys7U0qBPJNf3t+h8THKxYhKj8ohLeXLCJ8zsk0rl5nNdxgkZ4mPDs1T05kl/Ew/9cQZPYaH7U2cbYMrWfnecXoj5aso29Rwu57fzQHILcTZHhYUy6tg9dWzRg/JSlZOYc9DqSMWfMikUIKilVJs/LpkdSA85u09jrOEGpXnQEb9zYj8TYaG56K53Ne495HcmYM2LFIgR9sWonm/flcdt5be0UTxclxkbz1s/6oarc8OYi9h4tqHglYwKUFYsQo6q8Mm8jKY3qMrRbM6/jBL02ifV5/cZ+7Dqcz81vpZNXWOx1JGNOixWLELNk6wGW5Rzk1kGpNh9DDemTEs8L1/RhxbZD3DdtGaWlZQdfNibwWbEIMW8u2ExsTASj+iZ5HSWkXNKlKQ9f1oXPV+3kz7PXeR3HmCqzU2dDyI5Dx/ls5U5uGtiaulH21te0mwa2Jmv3ESbNyaZdk/pc2dsKtqk9bM8ihPxj4RZUlZ+e3drrKCFJRJg4ohtntWnErz5YweIt+72OZEylWbEIEflFJUz5fisXd25KcqO6XscJWZHhYbxyfV9aNIxh3N8Xk7M/z+tIxlSKFYsQMSNzOwfyirhxYGuvo4S8hnWjeP3GfhSVlHLL2xkcLbAzpEzgs2IRAlSVN7/dTMemsXYRXoBom1ifl67rS9aeo9w7LdPOkDIBz9ViISJDRWSdiGSJyIRynr9RRPaISKZzu8XvuRtEZINzu8HNnMFu0ab9rNlxmJ8NbG0X4QWQc9sn8MhlnZm9ehcvzsnyOo4xp+TaKTEiEg5MAi4BcoF0EZlRzvSo01R1fJl1GwG/AdIABRY76x5wK28we3PBZhrWjWREr5ZeRzFl3HhOa1bkHuK5/6yna4s4G3TQBCw39yz6A1mqulFVC4GpwIhKrjsEmK2q+50CMRsY6lLOoLb94HG+WL2Tsf1SqBMV7nUcU4aI8MRV3enSPI57pmaycc9RryMZUy43i0VLIMfvca6zrKxRIrJcRD4QkeQqrmsqMD0jBwWuG2DzKgSqmMhwXv1JXyLChdveWWwd3iYguVksyjs4XrYX719Aa1XtAfwHeLsK6yIi40QkQ0Qy9uzZc0Zhg1FJqTItPYdB7RPtdNkAlxRfl0nX9iF7z1EemL4MVevwNoHFzWKRCyT7PU4Ctvs3UNV9qnpiKM7XgL6VXddZf7KqpqlqWmJiYrUFDxZfr9/NjkP5XNs/ueLGxnPntEvg15d25vNVO3lpbrbXcYz5ATeLRTrQXkRSRSQKGAvM8G8gIs39Hg4H1jj3ZwGDRSReROKBwc4yUwVTvt9KQn2bqa02ufncVK7o2YJnvljH/A22t2wCh2vFQlWLgfH4PuTXANNVdZWITBSR4U6zu0RklYgsA+4CbnTW3Q/8Hl/BSQcmOstMJe04dJyv1u7m6rQkIsPtcpraQkR4alR32jepzz1TM9l5KN/rSMYAIMFybDQtLU0zMjK8jhEw/vqfDTz3n/XM++WFpDS2/oraJmv3EYa/uICuLeKYcutZVvCNa0RksaqmVdTOfgODkK9jeyuD2idYoail2jWJ5Y9XdSd98wGemWVDmhvvWbEIQvPW72H7oXyu6W+ny9ZmI3q15LoBKbw6byOzV+/yOo4JcVYsgtCURVtJqB/FxdaxXes9enkXurWM4/7pmTZCrfGUFYsgs/NQPl+t3c3ovslERdjbW9vFRIbz0rV9UeDOKUsoKC7xOpIJUfZpEmTez8ihpFQZ28+urQgWKY3r8syPe7I89xCPf7qm4hWMcYEViyBSUqpMTc9hYLvGtE6o53UcU42GdG3GLeem8vfvtvDp8h1exzEhyIpFEJm/YQ/bDh63ju0g9athneiV3JAJHy23/gtT46xYBJH3Fm2lcb0oBndp5nUU44LI8DBeuKY3KNw1dSlFJaVeRzIhxIpFkNh9OJ//rNnN6L5J1rEdxJIb1eWJq7qzdOtBnpu93us4JoTYp0qQeH9xLiWlyhjr2A56V/RswZi0ZF7+OpsFWXu9jmNChBWLIFBaqry3aCtnt2lMm8T6XscxNeA3w7vQJqEe90zLZO/RgopXMOYMWbEIAt9k7SX3wHGutQmOQkbdqAhevLYPh44X8cD7yygtDY4x3kzgsmIRBN5btJVG9aIY3NWu2A4lnZvH8chlnZm7bg9vLNjkdRwT5KxY1HK7j+Qze/UuRvdNIjrC5tgONT85qxWDuzTlqc/XsiL3kNdxTBCzYlHLvZ+RS7FdsR2yRISnR/cgoX40v3hvic3fbVxjxaIWKy1VpqZv5aw2jaxjO4Q1rBvFX8f2Zuv+PH47Y5XXcUyQcrVYiMhQEVknIlkiMuEU7UaLiIpImvO4tYgcF5FM5/aKmzlrqwXZe8nZb1dsG+if2og7L2zHB4tzmbnChgMx1S/CrRcWkXBgEnAJkAuki8gMVV1dpl0svilVvy/zEtmq2sutfMHgvUVbia8byZCudsW2gbt+1J556/fw0Ecr6JMST7MGMV5HMkHEzT2L/kCWqm5U1UJgKjCinHa/B54GbLLhKthzpIAvVu1iVJ8kYiKtY9v4hgN5bkwvCotLuf/9TDud1lQrN4tFSyDH73Gus+y/RKQ3kKyq/y5n/VQRWSoiX4vIoPK+gYiME5EMEcnYs2dPtQWvDT5Y7HRs2yEo46dNYn0eu6ILC7L22em0plq5WSyknGX//VdHRMKA54D7y2m3A0hR1d7AfcAUEYn7nxdTnayqaaqalpiYWE2xA9+Jju3+qY1o18Q6ts0Pje2XzCVdmvL05+tYvf2w13FMkHCzWOQC/udzJgHb/R7HAt2AuSKyGTgLmCEiaapaoKr7AFR1MZANdHAxa63y3cZ9bNmXx7W2V2HKISI8NaoHDepGcs+0peQX2ex65sy5WSzSgfYikioiUcBYYMaJJ1X1kKomqGprVW0NLASGq2qGiCQ6HeSISBugPbDRxay1ypRFW2lQJ5Kh3axj25SvUb0onvlxT9bvOspTn6/1Oo4JAq4VC1UtBsYDs4A1wHRVXSUiE0VkeAWrnwcsF5FlwAfA7aq6362stcneowV8sWqndWybCp3fIZEbz2nNmws2M299aPXpmern2qmzAKo6E5hZZtljJ2l7gd/9D4EP3cxWW324OJeiEuWa/nbFtqnYhGGd+DZ7L/e/v4xZ95xHo3pRXkcytZRdwV2LqPqGIu/XOp72TWO9jmNqgZjIcP4ypjeH8oqY8OFyVO10WnN6rFjUIt9t3MfmfXl2xbapki4t4vjlkI58sXoX09JzKl7BmHJYsahF3luUQ1xMBJd2b+51FFPL3HxuKgPbNWbiv1ezdV+e13FMLWTFopbYd7SAWSt3cpV1bJvTEBYm/Gl0T8LDhPumZ1JiV3ebKrJiUUt8sDiXwpJSmw3PnLYWDeswcURXMrYc4LX5dia6qRorFrXAiTm201rF08E6ts0ZGNmrJcO6NePZL9azZodd3W0qz4pFLXCiY9v2KsyZEhEev7I7cXUiuXdaJgXFdnW3qRwrFrXAlO99V2xbx7apDo3qRfHUqO6s3XmE52Zv8DqOqSWsWAS4PUcKmGVXbJtq9qPOTRnbL5lX52WTvtkGRzAVs2IR4N5fnENxqXLtALti21SvRy7vQlJ8He6fvszm7jYVsmIRwEpLlamLchiQ2oh2Taxj21Sv+tER/PnHvcg5kMfjn67xOo4JcFYsAtg3WXvZut86to17+qc2Ytx5bXhv0Va+WrvL6zgmgFmxCGBTvt9Ko3pRNhS5cdV9l3SgU7NYHvxgBfuPFXodxwQoKxYBavfhfGav2cXovklER1jHtnFPdEQ4z17di0PHC3nk4xU22KAplxWLADU9I4eSUrVBA02N6NIijnsv6cDMFTv5JHN7xSuYkONqsRCRoSKyTkSyRGTCKdqNFhEVkTS/ZQ85660TkSFu5gw0JaXKe4tyOKdtY1IT6nkdx4SI285rS99W8Tz6yUq2HzzudRwTYFwrFs60qJOAYUAX4BoR6VJOu1jgLuB7v2Vd8E3D2hUYCrx0YprVUDBvwx62HTxuHdumRoWHCc9e3ZOSUuWXHyyj1AYbNH7c3LPoD2Sp6kZVLQSmAiPKafd74Gkg32/ZCGCqqhao6iYgy3m9kPDOd1tIqB/N4C7WsW1qVqvG9Xjksi4syNrH37/b7HUcE0AqLBYiEiYiK0/jtVsC/jOt5DrL/F+7N5Csqv+u6rrBasu+Y8xZt5trB6QQFWFdSqbmXdM/mQs6JvLHz9aStfuo13FMgKjw00hVS4FlIlLVYyJS3sv990mRMOA54P6qruv3GuNEJENEMvbsCY4J6f+xcAvhIlxnh6CMR0SEp0f1oE5UOPdPz6SopNTrSCYAVPZf1+bAKhH5UkRmnLhVsE4u4D9GRRLgf5pFLNANmCsim4GzgBlOJ3dF6wKgqpNVNU1V0xITEyv5owSu44UlTEvPYUi3ZjSNi/E6jglhTeJieHxkd5blHmLSnCyv45gAEFHJdr87jddOB9qLSCqwDV+H9bUnnlTVQ0DCicciMhd4QFUzROQ4MEVEngVaAO2BRaeRoVb5OHMbh/OLufGc1l5HMYbLejTni9UteOGrLC7s2ISeyQ29jmQ8VKlioapfV/WFVbVYRMYDs4Bw4A1VXSUiE4EMVT3pnonTbjqwGigG7lTVoB54X1V5+9vNdG4eR1qreK/jGAPAxOHdWLRpP/dOz+TTXwyiTlTInJRoyjjlYSgROSIih8u5HRGRCqfZUtWZqtpBVduq6uPOssfKKxSqeoGqZvg9ftxZr6OqfnY6P1xtkr75AGt3HuGGs1shUl6XjTE1r0HdSJ75cU827jnGU5+v9TqO8dApi4WqxqpqXDm3WFWNq6mQoeDt7zYTFxPBiF4hcdKXqUUGtkvgZwNb89a3m5m/IThOJDFVZ+dmBoCdh/KZtXInY/ol226+CUi/GtqJdk3q88D7yziYZ4MNhiIrFgHgHwu3UKLK9We18jqKMeWKiQznuat7se9oIY9+ssrrOMYDViw8lldYzD++38IlnZvSqrGNA2UCV/ekBtz9o/b8a9l2Psnc5nUcU8OsWHjsw8W5HMwr4tbz2ngdxZgK3XFBW3qnNOTRj1ey45ANNhhKrFh4qKRUef2bTfRMbminy5paISI8jOeu7kVRifLgB8ttsMEQYsXCQ/9Zs4vN+/K4dVCqnS5rao3WCfV45PLOzN+w1wYbDCFWLDz0t/kbadmwDkO72uiypna5tn8KF9pggyHFioVHMnMOkr75ADedm0pEuL0NpnYREZ4a1YO6UeHcO80GGwwF9inlkdfmbyQ2JoIx/ZIrbmxMAGoSF8MTV3ZnxbZDvPCVDTYY7KxYeGDz3mN8tmIH1w5IoX50ZcdyNCbwDOvenKv6tGTSnCyWbj3gdRzjIisWHnh5bjaR4WHcfG6q11GMOWO/Hd6VZnEx3Dd9GXmFxV7HMS6xYlHDth08zodLchnbL5kmsTZnhan94mJ8gw1u3neMP860wQaDlRWLGjb562xEYNz5bb2OYky1ObttY24emMo7C7cwd91ur+MYF1ixqEG7j+TzXnoOV/VOomXDOl7HMaZaPTCkIx2a1ufBD5Zz4JgNNhhsrFjUoNfnb6K4pJQ7LrC9ChN8YiLDeW5MLw7kFfLIxytRtau7g4mrxUJEhorIOhHJEpEJ5Tx/u4isEJFMEflGRLo4y1uLyHFneaaIvOJmzpqw92gB7yzcwuU9WtA6wQYMNMGpa4sG3HtJBz5dsYNPMrd7HcdUI9eKhYiEA5OAYUAX4JoTxcDPFFXtrqq9gKeBZ/2ey1bVXs7tdrdy1pSX52aTX1TC3Re39zqKMa667by2pLWK59GPV5KzP8/rOKaauLln0R/IUtWNqloITAVG+DdQVf+pWesBQbnfuuPQcd5ZuIVRfZJom1jf6zjGuCo8THhuTC8A7p2WSbFd3R0U3CwWLYEcv8e5zrIfEJE7RSQb357FXX5PpYrIUhH5WkQGuZjTdc9/mYWq2l6FCRnJjery+5HdyNhygElzsr2OY6qBm8WivGFU/2fPQVUnqWpb4FfAI87iHUCKqvYG7gOmiMj/zPktIuNEJENEMvbsCcy5gTfvPcb7GTlc2z+FpPi6XscxpsaM7N2SK3u35K9frmfxlv1exzFnyM1ikQv4D3yUBJyqx2sqMBJAVQtUdZ9zfzGQDXQou4KqTlbVNFVNS0xMrLbg1enZ2euJCBfuvKid11GMqXETR3SlZXwd7p6ayeH8Iq/jmDPgZrFIB9qLSKqIRAFjgRn+DUTE/7jMZcAGZ3mi00GOiLQB2gMbXczqiiVbDzBj2XZuObeNXa1tQlJsTCR/GdObHYfyefTjlV7HMWfAtWKhqsXAeGAWsAaYrqqrRGSiiAx3mo0XkVUikonvcNMNzvLzgOUisgz4ALhdVWvVfmxpqTLxX6tJjI226ypMSOvbKp67f9SeTzK388+luV7HMafJ1SFPVXUmMLPMssf87t99kvU+BD50M5vbZizbTmbOQf40ugf1bGRZE+LuvLAd32zYy6Mfr6JvSiNSGlv/XW1jV3C74HhhCU9+tpZuLeMY1SfJ6zjGeC48THhubC9E4K6pS22ypFrIioUL/vrlBnYezuexy7sSFmZzaxsD0LJhHZ64sjuZOQd5/ssNXscxVWTFopqt2XGY1+ZvZHTfJPqnNvI6jjEB5YqeLRjdN4lJc7JYtKlWdUOGPCsW1aikVHnooxU0qBPJw5d29jqOMQHpt8O7ktyoLvdMXcrBPBudtrawYlGN3v1+C5k5B3n08s7E14vyOo4xAal+dATPj+3NnqMFPPD+chudtpawYlFNNu31zRI2qH0CI3v9z6gmxhg/PZMbMmFYZ/6zZhdvLNjsdRxTCVYsqkFRSSn3TMskKiKMp0f3QMQ6tY2pyE0DW3Nx56Y8+dkaluUc9DqOqYAVi2rwwldZLMs5yBNXdqd5A5sBz5jKEBGe+XEPmsTGcOeUJRw6bsOBBDIrFmfomw17efGrDVzVuyWX9WjudRxjapWGdaN4/pre7DyUz4QPrf8ikFmxOAM5+/MY/94S2jWpz+9HdvM6jjG1Ut9W8fxySEc+W7mTdxZu8TqOOQkrFqfpWEExt72zmJJSZfJP0mxID2POwK2D2nBhx0T+8O81rNx2yOs4phxWLE5DUUkpd7y7hLU7D/P82N42p7YxZygsTPjz1b1oVC+KO6cs4YgNZx5wrFhUUUmp8sv3lzFv/R6euLI7F3Zq4nUkY4JCo3q+/ovcA8eZ8NEK678IMFYsqqCopJR7p2XyceZ2fjmkI2P7p3gdyZig0j+1EfcP7sCny3fwpl1/EVCsWFTS0YJibn9nMTOWbWfCsE7ceaHNfGeMG24/ry0Xd27KEzPXkL7Zxo8KFFYsKiF7z1GunLSAOesSWAotAAASgklEQVR284eR3bj9fJvMyBi3+PovepIUX4c7313C7iP5XkcyuFwsRGSoiKwTkSwRmVDO87eLyAoRyRSRb0Ski99zDznrrRORIW7mPJniklImz8vmsufns+9YIf+4eQDXn9XKiyjGhJQGdSJ55Sd9OZxfxPgpNv9FIHCtWDhzaE8ChgFdgGv8i4Fjiqp2V9VewNPAs866XfDN2d0VGAq8dGJO7pqQX1TCR0tyGfyXeTwxcy3ntktk5l2DOKddQk1FMCbkdWoWx5NX9WDRpv089dlar+OEPDcvDugPZKnqRgARmQqMAFafaKCqh/3a1wNOnP4wApiqqgXAJhHJcl7vu+oOWVBcQubWgxzIK2L7weMszTnI3LW7OVJQTKdmsbz6k74M7tLUxnsyxgMje7dk6dYD/O2bTfRKacjlPVp4HSlkuVksWgI5fo9zgQFlG4nIncB9QBRwkd+6C8us+z9DuYrIOGAcQErK6Z2ZdCS/mDGT//9bNYuLYXDXZozs3YKBbRNspjtjPPbwZV1Yse0QD36wnE7NYmnXJNbrSCHJzWJR3qfs/5w4raqTgEkici3wCHBDFdadDEwGSEtLO62TsuPrRvHuLQNoUCeSJrHRNImLOZ2XMca4JCoijJeu68vlL8xn3N8X8887B9KgTqTXsUKOmx3cuUCy3+MkYPsp2k8FRp7muqctPEwY2C6Bbi0bWKEwJkA1axDDpGv7sHV/Hne9t5SSUrtgr6a5WSzSgfYikioiUfg6rGf4NxCR9n4PLwNOzOI+AxgrItEikgq0Bxa5mNUYE+AGtGnMxBHd+Hr9Hp7+3Dq8a5prh6FUtVhExgOzgHDgDVVdJSITgQxVnQGMF5GLgSLgAL5DUDjtpuPrDC8G7lTVEreyGmNqh2sHpLB252FenbeRjs1iuapPkteRQoYEy/graWlpmpGR4XUMY4zLikpK+enri1i89QDTxp1F75R4ryPVaiKyWFXTKmpnV3AbY2qVyPAwJl3Xh6Zx0dz2zmJ2HrIrvGuCFQtjTK3TqF4Uf/tpP2demQzyi+wotdusWBhjaqWOzWJ5bkwvlm87xP3Tl1FqZ0i5yoqFMabWGty1Gb8e1plPV+zgqVl2hpSbbC5QY0ytdsugVLbuz+PVrzeSHF/XBvt0iRULY0ytJiL85ooubDt4nMc+WUnLhnVsBksX2GEoY0ytFxEexgvX9KZz8zjunLKEldsOeR0p6FixMMYEhXrREbxxYz8a1onk5rfT2X7wuNeRgooVC2NM0GgaF8MbP+tHXkEJN7yxiAPHCr2OFDSsWBhjgkqnZnFM/mkaW/bn8bO30jlWUOx1pKBgxcIYE3TObtuYF6/pzfLcg9z+j8UUFNtFe2fKioUxJigN7tqMJ0f1YP6Gvdw3bZkNa36G7NRZY0zQujotmUN5RTw+cw0N6kby+MhuNkXyabJiYYwJaree14b9eYW8PDeb+tERPDSskxWM02DFwhgT9B4c0pGj+cVMnreR8DDhwSEdrWBUkRULY0zQExF+N7wrJaq8PDebcBHuH9zBCkYVuNrBLSJDRWSdiGSJyIRynr9PRFaLyHIR+VJEWvk9VyIimc5tRtl1jTGmKsLChD+M6MbYfsm8OCeLv365oeKVzH+5tmchIuHAJOASIBdIF5EZqrrar9lSIE1V80TkDuBpYIzz3HFV7eVWPmNM6AkLE564sjvFpcpf/rOBMBHu+lF7r2PVCm4ehuoPZKnqRgARmQqMwDevNgCqOsev/ULgehfzGGMMYWHCU6N6UKrKs7PXc7yoxPowKsHNYtESyPF7nAsMOEX7m4HP/B7HiEgGUAw8qaofl11BRMYB4wBSUlLOOLAxJjSEhwl/Gt2TmMhwXp6bzdH8Yn43vCthYVYwTsbNYlHeVi/3qhgRuR5IA873W5yiqttFpA3wlYisUNXsH7yY6mRgMkBaWppdcWOMqbTwMOHxkd2IjY7g1XkbOVZQzNOjexARbtcql8fNYpELJPs9TgK2l20kIhcDDwPnq2rBieWqut35ulFE5gK9geyy6xtjzOkSESYM60RsTATPfLGeowXFPH9Nb2Iiw72OFnDcLKHpQHsRSRWRKGAs8IOzmkSkN/AqMFxVd/stjxeRaOd+AjAQv74OY4ypLiLC+Iva89sruvDF6l389PVFHMyz0WrLcq1YqGoxMB6YBawBpqvqKhGZKCLDnWZ/AuoD75c5RbYzkCEiy4A5+PosrFgYY1xz48BUnr+mN5k5B7nq5W/Zui/P60gBRVSD41B/WlqaZmRkeB3DGFPLLdq0n1v/nkFkuPC3G/rRK7mh15FcJSKLVTWtonbWk2OMMX76pzbiwzvOoU5UOGMnf8dnK3Z4HSkgWLEwxpgy2jWpzz9/PpDOzeO4490lPDNrXcgPcW7FwhhjypFQP5qp485iTJpveJCb307n0PEir2N5xoqFMcacRHREOE+O6s7jV3ZjQdZeRrz4DWt3HvY6liesWBhjzCmICNcNaMV7t57FscISRry4gHcWbiFYTg6qLCsWxhhTCWmtGzHzrkEMaNOYRz9eyW3vLObAsdC5HsOKhTHGVFJibDRv3diPRy7rzJx1uxn21/ksyNrrdawaYcXCGGOqICxMuGVQGz66YyB1o8K57m/f89BHyzmcH9yd31YsjDHmNHRPasDMuwdx23ltmJaew+Bn5/Hlml1ex3KNFQtjjDlNMZHhPHRpZ/7584E0qBPJzW9ncOvfM9iy75jX0aqdFQtjjDlDPZMb8q9fnMuDQzuyIGsvlzw7j6c/X8uxgmKvo1UbKxbGGFMNoiLC+PkF7ZjzwAVc3qM5L83N5oJn5vLmgk3kF5V4He+MWbEwxphq1DQuhmfH9OKjn59D28R6/O5fqzn/T3N457vNFBTX3qJho84aY4yLvs3ey7NfrCdjywES6kdx/VmtuG5AKxJjo72OBlR+1FkrFsYY4zJV5dvsffxt/kbmrNtDVHgYl/dszui+SZyV2tjTub8rWyzcnFYVERkK/BUIB/6mqk+Wef4+4BagGNgD3KSqW5znbgAecZr+QVXfdjOrMca4RUQY2C6Bge0SyN5zlLcWbOafS7fx0ZJttGxYhxG9WjCsW3O6tojztHCcimt7FiISDqwHLsE3H3c6cI3/jHciciHwvarmicgdwAWqOkZEGgEZQBqgwGKgr6oeONn3sz0LY0xtcrywhNlrdvHRklzmrd9DqfquEL+oYxMGdUigb6t4mjeo43qOQNiz6A9kqepGJ9BUYAR+c2mr6hy/9guB6537Q4DZqrrfWXc2MBR4z8W8xhhTY+pEhTO8ZwuG92zBvqMFzF23h6/W7Wbmih1My8gBoEWDGHomN6R9k/q0bVKfNgn1aRIXTaN6UUSG1+z5SW4Wi5ZAjt/jXGDAKdrfDHx2inVbVms6Y4wJEI3rRzOqbxKj+iZRVFLKmh2HWbzlAIu3HGDFtkPMWrWTsnMvNawbSb2oCKIiwujWsgEvXNPb1YxuFovyDryVe8xLRK7Hd8jp/KqsKyLjgHEAKSkpp5fSGGMCSGR4GD2SGtIjqSE/G5gKQH5RCVv25bFp7zH2Hi1g79EC9h0tJK+whMKSUpLj3T9c5WaxyAWS/R4nAdvLNhKRi4GHgfNVtcBv3QvKrDu37LqqOhmYDL4+i+oIbYwxgSYmMpyOzWLp2CzWswxuHvRKB9qLSKqIRAFjgRn+DUSkN/AqMFxVd/s9NQsYLCLxIhIPDHaWGWOM8YBrexaqWiwi4/F9yIcDb6jqKhGZCGSo6gzgT0B94H0RAdiqqsNVdb+I/B5fwQGYeKKz2xhjTM2zi/KMMSaEVfbUWRsbyhhjTIWsWBhjjKmQFQtjjDEVsmJhjDGmQlYsjDHGVChozoYSkT3AljN4iQRgbzXFcUOg54PAzxjo+cAyVodAzweBlbGVqiZW1ChoisWZEpGMypw+5pVAzweBnzHQ84FlrA6Bng9qR8ay7DCUMcaYClmxMMYYUyErFv9vstcBKhDo+SDwMwZ6PrCM1SHQ80HtyPgD1mdhjDGmQrZnYYwxpkIhXyxEZKiIrBORLBGZ4GGOZBGZIyJrRGSViNztLP+tiGwTkUzndqnfOg85udeJyJAayLhZRFY4OTKcZY1EZLaIbHC+xjvLRUSed/ItF5E+NZCvo992yhSRwyJyj9fbUETeEJHdIrLSb1mVt5uI3OC03yAiN7ic708istbJ8E8Raegsby0ix/225St+6/R1fj+ynJ+hvEnMqjNjld9Xt/7eT5Jvml+2zSKS6Sz3ZBueMVUN2Ru+odOzgTZAFLAM6OJRluZAH+d+LLAe6AL8FnignPZdnLzRQKrzc4S7nHEzkFBm2dPABOf+BOAp5/6l+KbJFeAs4HsP3tudQCuvtyFwHtAHWHm62w1oBGx0vsY79+NdzDcYiHDuP+WXr7V/uzKvswg428n+GTDM5W1YpffVzb/38vKVef7PwGNebsMzvYX6nkV/IEtVN6pqITAVGOFFEFXdoapLnPtHgDWcet7xEcBUVS1Q1U1AFr6fp6aNAN527r8NjPRb/nf1WQg0FJHmNZjrR0C2qp7qQs0a2YaqOg8oOx9LVbfbEGC2qu5X1QPAbGCoW/lU9QtVLXYeLsQ3W+VJORnjVPU79X3q/d3vZ3Il4ymc7H117e/9VPmcvYOrgfdO9Rpub8MzFerFoiWQ4/c4l1N/QNcIEWkN9Aa+dxaNdw4HvHHicAXeZFfgCxFZLL75zwGaquoO8BU8oImH+fyN5Yd/nIGyDU+o6nbzMutN+P7LPSFVRJaKyNciMshZ1tLJVNP5qvK+erUNBwG7VHWD37JA2oaVEurForzjgZ6eHiYi9YEPgXtU9TDwMtAW6AXswLc7C95kH6iqfYBhwJ0ict4p2nq2bcU3je9w4H1nUSBtw4qcLJMnWUXkYaAYeNdZtANIUdXewH3AFBGJ8yhfVd9Xr97va/jhPy6BtA0rLdSLRS6Q7Pc4CdjuURZEJBJfoXhXVT8CUNVdqlqiqqXAa/z/YZIaz66q252vu4F/Oll2nTi85Hw9MZe6l9t2GLBEVXc5eQNmG/qp6nar8axOJ/rlwHXOYRGcQzv7nPuL8fUBdHDy+R+qqonfx6q+r15swwjgKmCaX+6A2YZVEerFIh1oLyKpzn+jY4EZXgRxjmu+DqxR1Wf9lvsf578SOHG2xQxgrIhEi0gq0B5f55hb+eqJSOyJ+/g6QFc6OU6cmXMD8Ilfvp86Z/ecBRw6cdilBvzgP7lA2YZlVHW7zQIGi0i8c7hlsLPMFSIyFPgVMFxV8/yWJ4pIuHO/Db5tttHJeEREznJ+l3/q9zO5lbGq76sXf+8XA2tV9b+HlwJpG1aJ1z3sXt/wnX2yHl91f9jDHOfi2+VcDmQ6t0uBd4AVzvIZQHO/dR52cq/D5bMm8J1Bssy5rTqxrYDGwJfABudrI2e5AJOcfCuAtBrajnWBfUADv2WebkN8hWsHUITvv8ebT2e74es7yHJuP3M5Xxa+4/snfhdfcdqOct7/ZcAS4Aq/10nD94GdDbyIc9Gvixmr/L669fdeXj5n+VvA7WXaerINz/RmV3AbY4ypUKgfhjLGGFMJViyMMcZUyIqFMcaYClmxMMYYUyErFsYYYypkxcKYcojIt87X1iJybTW/9q/L+17GBDI7ddaYUxCRC/CNbHp5FdYJV9WSUzx/VFXrV0c+Y2qK7VkYUw4ROercfRIY5Mw7cK+IhItvrod0ZwC725z2F4hvPpIp+C4UQ0Q+dgZdXHVi4EUReRKo47zeu/7fy7lq+08istKZ02CM32vPFZEPxDfHxLvOFb7G1JgIrwMYE+Am4Ldn4XzoH1LVfiISDSwQkS+ctv2BbuobFhvgJlXdLyJ1gHQR+VBVJ4jIeFXtVc73ugrfoHg9gQRnnXnOc72BrvjGCloADAS+qf4f15jy2Z6FMVUzGN/YTZn4hpBvjG9sH4BFfoUC4C4RWYZvPohkv3Yncy7wnvoGx9sFfA3083vtXPUNmpeJbwIdY2qM7VkYUzUC/EJVfzCIn9O3cazM44uBs1U1T0TmAjGVeO2TKfC7X4L97ZoaZnsWxpzaEXzT3J4wC7jDGU4eEengjMJbVgPggFMoOuGbIvWEohPrlzEPGOP0iyTim6qzpkbBNeaU7L8TY05tOVDsHE56C/grvkNAS5xO5j2UP/Xl58DtIrIc38inC/2emwwsF5Elqnqd3/J/4pt/eRm+EYgfVNWdTrExxlN26qwxxpgK2WEoY4wxFbJiYYwxpkJWLIwxxlTIioUxxpgKWbEwxhhTISsWxhhjKmTFwhhjTIWsWBhjjKnQ/wGqCY2hbWy5UgAAAABJRU5ErkJggg==\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "recorder.plotLRs()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "//Needs fixing \n", "//learner.recorder!.plotLRs()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Export" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "success\r\n" ] } ], "source": [ "import NotebookExport\n", "let exporter = NotebookExport(Path.cwd/\"05_anneal.ipynb\")\n", "print(exporter.export(usingPrefix: \"FastaiNotebook_\"))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Swift", "language": "swift", "name": "swift" } }, "nbformat": 4, "nbformat_minor": 1 }