{ "cells": [ { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Installing packages:\n", "\t.package(path: \"/home/sgugger/git/course-v3/nbs/swift/FastaiNotebook_08a_heterogeneous_dictionary\")\n", "\t\tFastaiNotebook_08a_heterogeneous_dictionary\n", "With SwiftPM flags: []\n", "Working in: /tmp/tmpjnhcoy0k/swift-install\n", "warning: /home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc)\n", "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n", "warning: /home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc)\n", "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n", "warning: /home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc)\n", "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n", "warning: /home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc)\n", "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n", "warning: /home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc)\n", "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n", "warning: /home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc)\n", "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n", "warning: /home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc)\n", "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n", "warning: /home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc)\n", "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n", "warning: /home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc)\n", "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n", "warning: /home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc)\n", "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n", "warning: /home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc)\n", "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n", "warning: /home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc)\n", "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n", "warning: /home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc)\n", "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n", "warning: /home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc)\n", "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n", "warning: /home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc)\n", "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n", "warning: /home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc)\n", "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n", "warning: /home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc)\n", "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n", "/home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc)\n", "/home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc)[1/13] Compiling FastaiNotebook_08_data_block 01_matmul.swift\n", "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n", "[2/13] Compiling FastaiNotebook_08_data_block 03_minibatch_training.swift\n", "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n", "[3/13] Compiling FastaiNotebook_08_data_block 02_fully_connected.swift\n", "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n", "[4/13] Compiling FastaiNotebook_08_data_block 05b_early_stopping.swift\n", "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n", "[5/13] Compiling FastaiNotebook_08_data_block 06_cuda.swift\n", "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n", "[6/13] Compiling FastaiNotebook_08_data_block 05_anneal.swift\n", "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n", "[7/13] Compiling FastaiNotebook_08_data_block 02a_why_sqrt5.swift\n", "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n", "[8/13] Compiling FastaiNotebook_08_data_block 00_load_data.swift\n", "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n", "[9/13] Compiling FastaiNotebook_08_data_block 04_callbacks.swift\n", "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n", "[10/13] Compiling FastaiNotebook_08_data_block 08_data_block.swift\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n", "[11/13] Compiling FastaiNotebook_08_data_block 01a_fastai_layers.swift\n", "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n", "[12/13] Compiling FastaiNotebook_08_data_block 07_batchnorm.swift\n", "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n", "[13/14] Merging module FastaiNotebook_08_data_block\n", "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n", "/home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc)[14/15] Compiling FastaiNotebook_08a_heterogeneous_dictionary 08_data_block.swift\n", "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n", "[15/25] Compiling FastaiNotebook_08a_heterogeneous_dictionary 03_minibatch_training.swift\n", "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n", "[16/25] Compiling FastaiNotebook_08a_heterogeneous_dictionary 02_fully_connected.swift\n", "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n", "[17/25] Compiling FastaiNotebook_08a_heterogeneous_dictionary 05b_early_stopping.swift\n", "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n", "[18/25] Compiling FastaiNotebook_08a_heterogeneous_dictionary 06_cuda.swift\n", "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n", "[19/25] Compiling FastaiNotebook_08a_heterogeneous_dictionary 05_anneal.swift\n", "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n", "[20/25] Compiling FastaiNotebook_08a_heterogeneous_dictionary 02a_why_sqrt5.swift\n", "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n", "[21/25] Compiling FastaiNotebook_08a_heterogeneous_dictionary 00_load_data.swift\n", "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n", "[22/25] Compiling FastaiNotebook_08a_heterogeneous_dictionary 04_callbacks.swift\n", "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n", "[23/25] Compiling FastaiNotebook_08a_heterogeneous_dictionary 01a_fastai_layers.swift\n", "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n", "[24/25] Compiling FastaiNotebook_08a_heterogeneous_dictionary 07_batchnorm.swift\n", "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n", "[25/26] Merging module FastaiNotebook_08a_heterogeneous_dictionary\n", "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n", "/home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc)[26/27] Compiling jupyterInstalledPackages jupyterInstalledPackages.swift\n", "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n", "[27/28] Merging module jupyterInstalledPackages\n", "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n", "/home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc)\n", "/home/sgugger/swift/usr/bin/swift-autolink-extract: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift-autolink-extract)\n", "[28/28] Linking libjupyterInstalledPackages.so\n", "Initializing Swift...\n", "Installation complete!\n" ] } ], "source": [ "%install-location $cwd/swift-install\n", "%install '.package(path: \"$cwd/FastaiNotebook_08a_heterogeneous_dictionary\")' FastaiNotebook_08a_heterogeneous_dictionary" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "// export\n", "import Path\n", "import TensorFlow" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import FastaiNotebook_08a_heterogeneous_dictionary" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "('inline', 'module://ipykernel.pylab.backend_inline')\n" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "%include \"EnableIPythonDisplay.swift\"\n", "IPythonDisplay.shell.enable_matplotlib(\"inline\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Load data" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "let path = downloadImagenette()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "let il = ItemList(fromFolder: path, extensions: [\"jpeg\", \"jpg\"])\n", "let sd = SplitData(il, fromFunc: {grandParentSplitter(fName: $0, valid: \"val\")})\n", "var procLabel = CategoryProcessor()\n", "let sld = makeLabeledData(sd, fromFunc: parentLabeler, procLabel: &procLabel)\n", "let rawData = sld.toDataBunch(itemToTensor: pathsToTensor, labelToTensor: intsToTensor)\n", "let data = transformData(rawData, tfmItem: { openAndResize(fname: $0, size: 128) })" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "func modelInit() -> CNNModel { return CNNModel(channelIn: 3, nOut: 10, filters: [64, 64, 128, 256]) }" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Stateful optimizer" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Before we begin, we create this structure to contain the names of our hyper-parameters. This will give us some tab completion and typo-proof way of handling them." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "//export\n", "public struct HyperParams {\n", " public static let lr = \"learningRate\"\n", "}" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Like in the python version, we create `statDelegates` that will be responsible for computing/updating statistics in the state (like the moving average of gradients) and `stepDelegates` that will be responsible for performing a part of the update of the weights. \n", "\n", "In PyTorch we created a basic class with functions that needed to be implemented. In swift this is what protocols are for." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "//export\n", "public protocol StatDelegate {\n", " var name: String {get}\n", " var defaultHPs: [String:Float] {get}\n", " \n", " func update(_ state: inout [String:TF], p: TF, 𝛁p: TF, hps: inout [String:Float])\n", "}\n", "\n", "public protocol StepDelegate {\n", " var defaultHPs: [String:Float] {get}\n", " \n", " func update(_ p: inout TF, 𝛁p: inout TF, state: [String:TF], hps: inout [String:Float])\n", "}" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Those are helper functions to merge dictionaries that we'll use in the `StatefulOptimizer`." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "//export\n", "public func mergeDicts(_ dicts: inout [[String:Float]], with newDict: [String:Float]) {\n", " for i in dicts.indices { \n", " dicts[i].merge(newDict) { (_, new) in new } \n", " }\n", "}\n", "\n", "public func mergeDicts(_ dicts: inout [[String:Float]], with newDicts: [[String:Float]]) {\n", " for i in dicts.indices { \n", " dicts[i].merge(newDicts[i]) { (_, new) in new } \n", " }\n", "}" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Those two extensions are there to initialize dicts easily." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "//export\n", "extension Dictionary where Value == Int{\n", " public init(mapFromArrays arrays: [[Key]]){\n", " self.init(uniqueKeysWithValues: arrays.enumerated().flatMap { i, arr in arr.map { ($0, i) } })\n", " }\n", "}\n", "\n", "extension Dictionary {\n", " public init(constant: Value, keys: [Key]){\n", " self.init(uniqueKeysWithValues: keys.map { ($0, constant) })\n", " }\n", "}" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "This is the initial state of our StatefulOptimizer. It's a dictionary keyPath (see below) to dictionary that maps names to tensor of floats." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "//export\n", "public func initState(for model: Model, names: [String]) \n", "-> [WritableKeyPath: [String:TF]] {\n", " return [WritableKeyPath: [String:TF]](\n", " constant: [String: TF](constant: TF(0), keys: names),\n", " keys: model.variables.keyPaths)\n", "}" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "And we can define the main `StatefulOptimizer`. It takes a model, hyperparameters for each parameter group, some `steppers` and `stats` and a `splitArray` that defines our different parameter groups.\n", "\n", "To understand how this work, you need to know a little bit about keyPaths. This is a tool in swift to access any elements in a nested structure like our models: one model typically has a few attributes that are modules which in turn contain other modules and so forth until we reach the primitives layers like `Conv2d` or `Dense`. KeyPaths will allow us to index in that nested structure the objects of a particular type. \n", "\n", "For instance, the shortcut `keyPaths` you can apply to any `Layer` will find all the tensors of floats. If we apply it to a `Model.AllDifferentiableVariables` object, we will find all the parameters of the model (since `model.allDifferentiableVariables` only contain the trainable parameters).\n", "\n", "That's why the inner loop of our `StatefulOptimizer` is over `variables.keyPaths`. The same keyPath index will give us the gradients. Then we create a `state` to be a dictionary of such keyPaths to `[String:TF]` and the `splitArray` we provide is an array of different keyPaths (each giving us a parameter group) from which we build a `splitDict` that maps our keyPaths to the index of the corresponding group." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "//export\n", "public class StatefulOptimizer\n", " where Model.AllDifferentiableVariables == Model.TangentVector {\n", " public typealias ModelKeyPath = WritableKeyPath\n", " public typealias SplitDict = [ModelKeyPath: Int]\n", " public var hpGroups: [[String:Float]]\n", " public var splitDict: SplitDict\n", " public var states: [ModelKeyPath: [String: TF]]\n", " public var stats: [StatDelegate]\n", " public var steppers: [StepDelegate]\n", " public init( \n", " for model: __shared Model,\n", " steppers: [StepDelegate],\n", " stats: [StatDelegate],\n", " hpGroups: [[String:Float]],\n", " splitArray: [[ModelKeyPath]]\n", " ) {\n", " self.hpGroups = Array(repeating: [:], count: hpGroups.count)\n", " (self.steppers,self.stats) = (steppers,stats)\n", " self.splitDict = SplitDict(mapFromArrays: splitArray)\n", " states = [:]\n", " steppers.forEach { mergeDicts(&self.hpGroups, with: $0.defaultHPs) }\n", " stats.forEach { mergeDicts(&self.hpGroups, with: $0.defaultHPs) }\n", " states = initState(for: model, names: stats.map { $0.name })\n", " mergeDicts(&self.hpGroups, with: hpGroups)\n", " }\n", " \n", " public func update(\n", " _ variables: inout Model.AllDifferentiableVariables,\n", " along direction: Model.TangentVector\n", " ) {\n", " for kp in variables.keyPaths {\n", " var 𝛁p = direction[keyPath: kp]\n", " var hps = hpGroups[splitDict[kp]!]\n", " stats.forEach() { $0.update(&states[kp]!, p: variables[keyPath: kp], 𝛁p: 𝛁p, hps: &hps) }\n", " steppers.forEach() { $0.update(&variables[keyPath: kp], 𝛁p: &𝛁p, state: states[kp]!, hps: &hps) }\n", " hpGroups[splitDict[kp]!] = hps\n", " }\n", " }\n", "}" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "To make `StatefulOptimizer` conform to the `Optimizer` protocol, we need to add a `learningRate` property." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "//export\n", "extension StatefulOptimizer: Optimizer{\n", " public var learningRate: Float {\n", " get { return hpGroups.last![HyperParams.lr]! } \n", " set { \n", " for i in hpGroups.indices {self.hpGroups[i][HyperParams.lr] = newValue }\n", " }\n", " }\n", " //For discriminative learning rates\n", " public var learningRates: [Float] {\n", " get { return hpGroups.map { $0[HyperParams.lr]! } }\n", " set { \n", " for i in hpGroups.indices {self.hpGroups[i][HyperParams.lr] = newValue[i] } \n", " }\n", " }\n", "}" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "When we don't have any parameter groups, we just use one with all the `keyPaths`. This convenience init automatically does that for us." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "//export\n", "extension StatefulOptimizer{\n", " public convenience init (for model: __shared Model,\n", " steppers: [StepDelegate],\n", " stats: [StatDelegate],\n", " hps: [String:Float]) {\n", " self.init(for: model,\n", " steppers: steppers,\n", " stats: stats,\n", " hpGroups: [hps],\n", " splitArray: [model.variables.keyPaths])\n", " }\n", "}" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "We are now ready to define `steppers` and `stats`. Let's begin with basic SGD:" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "//export\n", "public struct SGDStep: StepDelegate {\n", " public var defaultHPs: [String: Float] { return [HyperParams.lr: 3e-3] }\n", " public init() {}\n", " public func update(_ p: inout TF, 𝛁p: inout TF, state: [String:TF], hps: inout [String:Float]) {\n", " p -= 𝛁p * hps[HyperParams.lr]!\n", " }\n", "}" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "We can check all is working and train:" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "var hps: [String:Float] = [HyperParams.lr: 0.01]\n", "func optFunc(_ model: CNNModel) -> StatefulOptimizer {\n", " return StatefulOptimizer(for: model, steppers: [SGDStep()], stats: [], hps: hps)\n", "}" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "var learner = Learner(data: data, lossFunc: softmaxCrossEntropy, optFunc: optFunc, modelInit: modelInit)\n", "var recorder = learner.makeDefaultDelegates(metrics: [accuracy])\n", "learner.delegates.append(learner.makeNormalize(mean: mnistStats.mean, std: mnistStats.std))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Epoch 0: [1.5975714, 0.45] \n", " \r" ] } ], "source": [ "learner.fit(1)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Then we can add weight decay and L2 regularization." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "//export\n", "public extension HyperParams {\n", " static let wd = \"weightDecay\"\n", "}\n", "\n", "public struct WeightDecay: StepDelegate {\n", " public var defaultHPs: [String: Float] { return [HyperParams.wd: 0] }\n", " public init() {}\n", " public func update(_ p: inout TF, 𝛁p: inout TF, state: [String:TF], hps: inout [String:Float]) {\n", " p *= 1 - hps[HyperParams.lr]! * hps[HyperParams.wd]!\n", " }\n", "}" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "//export\n", "public struct L2Regularization: StepDelegate {\n", " public var defaultHPs: [String: Float] { return [HyperParams.wd: 0] }\n", " public init() {}\n", " public func update(_ p: inout TF, 𝛁p: inout TF, state: [String:TF], hps: inout [String:Float]) {\n", " 𝛁p += hps[HyperParams.wd]! * p\n", " }\n", "}" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "The next step is SGD with momentum. For this we need a statistic that keeps track of the moving average of the gradients." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "//export\n", "//Expandable enum to have tab completes/typo-proof for state variable names.\n", "public struct StateKeys {\n", " public static let avgGrad = \"averageGrad\"\n", "}" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "//export\n", "public extension HyperParams {\n", " static let mom = \"momentum\"\n", " static let momDamp = \"dampening\"\n", "}\n", "\n", "public struct AverageGrad: StatDelegate {\n", " public var defaultHPs: [String: Float] { return [HyperParams.mom: 0.9] }\n", " public let dampened: Bool\n", " public init(dampened: Bool = false) { self.dampened = dampened }\n", " public var name: String { return StateKeys.avgGrad }\n", " public func update(_ state: inout [String: TF], p: TF, 𝛁p: TF, hps: inout [String:Float]) {\n", " state[StateKeys.avgGrad]! *= hps[HyperParams.mom]!\n", " hps[HyperParams.momDamp] = 1.0 - (dampened ? hps[HyperParams.mom]! : 0.0)\n", " state[StateKeys.avgGrad]! += hps[HyperParams.momDamp]! * 𝛁p\n", " }\n", "}" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "//export\n", "public struct MomentumStep: StepDelegate {\n", " public var defaultHPs: [String: Float] = [:]\n", " public init() {}\n", " public func update(_ p: inout TF, 𝛁p: inout TF, state: [String: TF], hps: inout [String:Float]) {\n", " p -= state[StateKeys.avgGrad]! * hps[HyperParams.lr]!\n", " }\n", "}" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "And we can check it trains properly." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "let hps: [String:Float] = [HyperParams.lr: 0.01]\n", "func optFunc(_ model: CNNModel) -> StatefulOptimizer {\n", " return StatefulOptimizer(for: model, steppers: [MomentumStep()], stats: [AverageGrad()], hps: hps)\n", "}" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "var learner = Learner(data: data, lossFunc: softmaxCrossEntropy, optFunc: optFunc, modelInit: modelInit)\n", "var recorder = learner.makeDefaultDelegates(metrics: [accuracy])\n", "learner.delegates.append(learner.makeNormalize(mean: mnistStats.mean, std: mnistStats.std))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Epoch 0: [1.335211, 0.544] \n", " \r" ] } ], "source": [ "learner.fit(1)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "The hyper-parameters have taken the default values provided (except for learning rates)." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "▿ 3 elements\n", " ▿ 0 : 2 elements\n", " - key : \"dampening\"\n", " - value : 1.0\n", " ▿ 1 : 2 elements\n", " - key : \"momentum\"\n", " - value : 0.9\n", " ▿ 2 : 2 elements\n", " - key : \"learningRate\"\n", " - value : 0.01\n" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "learner.opt.hpGroups[0]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "The next step is Adam. For that we need to keep track of the averages of the gradients squared." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "//export\n", "public extension HyperParams {\n", " static let ²mom = \"momentumSquares\"\n", " static let ²momDamp = \"dampeningSquares\"\n", "}\n", "\n", "public extension StateKeys {\n", " static let avgSqr = \"averageSquaredGrad\"\n", "}\n", "\n", "public struct AverageSquaredGrad: StatDelegate {\n", " let dampened: Bool\n", " public init(dampened: Bool = true) { self.dampened = dampened }\n", " public var name: String { return StateKeys.avgSqr }\n", " public var defaultHPs: [String: Float] { return [HyperParams.²mom: 0.99] }\n", " public func update(_ state: inout [String: TF], p: TF, 𝛁p: TF, hps: inout [String:Float]) {\n", " state[StateKeys.avgSqr]! *= hps[HyperParams.²mom]!\n", " hps[HyperParams.²momDamp] = 1.0 - (dampened ? hps[HyperParams.²mom]! : 0.0)\n", " state[StateKeys.avgSqr]! += hps[HyperParams.²momDamp]! * 𝛁p.squared()\n", " }\n", "}" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "And we also need to keep track of the number of iterations we did." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "//export\n", "public extension StateKeys {\n", " static let step = \"stepCount\"\n", "}\n", "\n", "public struct StepCount: StatDelegate {\n", " public var name: String { return StateKeys.step }\n", " public var defaultHPs: [String:Float] = [:]\n", " public init() {}\n", " public func update(_ state: inout [String: TF], p: TF, 𝛁p: TF, hps: inout [String:Float]) {\n", " state[StateKeys.step]! += 1.0\n", " }\n", "}" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "//export\n", "//public struct Epsilon: HetDictKey { public static var defaultValue: Float = 1e-5 }\n", "public extension HyperParams {\n", " static let eps = \"epsilon\"\n", "}" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "//export\n", "public struct AdamStep: StepDelegate {\n", " public var defaultHPs: [String: Float] { return [HyperParams.eps: 1e-5] }\n", " public init() {}\n", " public func update(_ p: inout TF, 𝛁p: inout TF, state: [String: TF], hps: inout [String:Float]) {\n", " let step = state[StateKeys.step]!\n", " let (mom,damp) = (hps[HyperParams.mom]!,hps[HyperParams.momDamp]!)\n", " let debias1 = damp * (1 - pow(mom, step)) / (1 - mom)\n", " let num = debias1 * state[StateKeys.avgGrad]!\n", " \n", " let (²mom,²damp) = (hps[HyperParams.²mom]!,hps[HyperParams.²momDamp]!)\n", " let debias2 = ²damp * (1 - pow(²mom, step)) / (1 - ²mom)\n", " let denom = sqrt(state[StateKeys.avgSqr]!/debias2) + hps[HyperParams.eps]!\n", " \n", " p -= hps[HyperParams.lr]! * num / denom\n", " }\n", "}" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Again let's check it's all training properly." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "func optFunc(_ model: CNNModel) -> StatefulOptimizer {\n", " return StatefulOptimizer(\n", " for: model,\n", " steppers: [AdamStep()], \n", " stats: [AverageGrad(dampened: true), AverageSquaredGrad(), StepCount()], \n", " hps: [HyperParams.lr: 1e-3])\n", "}" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "let learner = Learner(data: data, lossFunc: softmaxCrossEntropy, optFunc: optFunc, modelInit: modelInit)\n", "let recorder = learner.makeDefaultDelegates(metrics: [accuracy])\n", "learner.delegates.append(learner.makeNormalize(mean: mnistStats.mean, std: mnistStats.std))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Epoch 0: [1.208637, 0.596] \n", " \r" ] } ], "source": [ "learner.fit(1)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "We can also check the values of the hyper-parameters have been set properly." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "▿ 6 elements\n", " ▿ 0 : 2 elements\n", " - key : \"dampeningSquares\"\n", " - value : 0.00999999\n", " ▿ 1 : 2 elements\n", " - key : \"momentumSquares\"\n", " - value : 0.99\n", " ▿ 2 : 2 elements\n", " - key : \"dampening\"\n", " - value : 0.100000024\n", " ▿ 3 : 2 elements\n", " - key : \"learningRate\"\n", " - value : 0.001\n", " ▿ 4 : 2 elements\n", " - key : \"epsilon\"\n", " - value : 1e-05\n", " ▿ 5 : 2 elements\n", " - key : \"momentum\"\n", " - value : 0.9\n" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "learner.opt.hpGroups[0]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Defining the Lamb optimizer is as easy as before." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "public struct LambStep: StepDelegate {\n", " public var defaultHPs: [String: Float] { return [HyperParams.eps: 1e-6, HyperParams.wd: 0.0] }\n", " public func update(_ p: inout TF, 𝛁p: inout TF, state: [String: TF], hps: inout [String:Float]) {\n", " let stepCount = state[StateKeys.step]!\n", " let (mom,damp) = (hps[HyperParams.mom]!,hps[HyperParams.momDamp]!)\n", " let debias1 = damp * (1 - pow(mom, stepCount)) / (1 - mom)\n", " let num = debias1 * state[StateKeys.avgGrad]!\n", " \n", " let (²mom,²damp) = (hps[HyperParams.²mom]!,hps[HyperParams.²momDamp]!)\n", " let debias2 = ²damp * (1 - pow(²mom, stepCount)) / (1 - ²mom)\n", " let denom = sqrt(state[StateKeys.avgSqr]!/debias2) + hps[HyperParams.eps]!\n", " \n", " let step = num / denom + hps[HyperParams.wd]! * p\n", " let r1 = sqrt((p * p).mean())\n", " let r2 = sqrt((step * step).mean())\n", " let factor = min(r1 / r2, Float(10.0))\n", " p -= hps[HyperParams.lr]! * factor * step\n", " }\n", "}" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Making convenience functions" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "To easily create our optimizers, we have two convenience functions." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "// export\n", "public func sgdOpt(lr: Float, mom: Float = 0.9, wd: Float = 0.0, dampening: Bool = false\n", " ) -> ((Model) -> StatefulOptimizer) {\n", " var steppers: [StepDelegate] = (mom != 0) ? [MomentumStep()] : [SGDStep()]\n", " if wd != 0 { steppers.append(WeightDecay()) }\n", " let stats = (mom != 0) ? [AverageGrad(dampened: dampening)] : []\n", " var hps: [String: Float] = [HyperParams.lr: lr]\n", " if mom != 0 { hps[HyperParams.mom] = mom }\n", " if wd != 0 { hps[HyperParams.wd ] = wd }\n", " return {model in \n", " return StatefulOptimizer(for: model, steppers: steppers, stats: stats, hps: hps)}\n", "}" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "// export\n", "public func adamOpt(lr: Float, mom: Float = 0.9, beta: Float=0.99, wd: Float = 0.0, eps: Float = 1e-5\n", " ) -> ((Model) -> StatefulOptimizer) {\n", " var steppers: [StepDelegate] = [AdamStep()]\n", " if wd != 0 { steppers.append(WeightDecay()) }\n", " let stats: [StatDelegate] = [AverageGrad(dampened: true), AverageSquaredGrad(), StepCount()]\n", " var hps: [String: Float] = [HyperParams.lr: lr]\n", " hps[HyperParams.mom] = mom\n", " hps[HyperParams.²mom] = beta\n", " hps[HyperParams.eps] = eps\n", " if wd != 0 { hps[HyperParams.wd ] = wd }\n", " return {model in \n", " return StatefulOptimizer(for: model, steppers: steppers, stats: stats, hps: hps)}\n", "}" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Schedule the hyperparams" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "The next thing is that we need to schedule our hyper-parameters. The following function allows us to schedule any of them, as long as they are present in the `hpGroups` dictionaries." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "// export\n", "public extension StatefulOptimizer {\n", " func setParam(_ hp: String, _ val: Float) {\n", " for i in 0.. Float\n", "\n", " // A learning rate schedule from step to float.\n", " public var scheduler: ScheduleFunc\n", " public let hp: String\n", " \n", " public init(scheduler: @escaping (Float) -> Float, hp: String) {\n", " (self.scheduler,self.hp) = (scheduler,hp)\n", " }\n", " \n", " override public func batchWillStart(learner: Learner) {\n", " let val = scheduler(learner.pctEpochs/Float(learner.epochCount))\n", " (learner.opt as! StatefulOptimizer).setParam(hp, val)\n", " }\n", " }\n", " \n", " public func makeParamScheduler(_ scheduler: @escaping (Float) -> Float, hp: String) -> ParamScheduler {\n", " return ParamScheduler(scheduler: scheduler, hp: hp)\n", " }\n", "}" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "We can then define a helper function to schedule a 1cycle policy." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "// export \n", "public func oneCycleSchedulers(_ lrMax: Float, pctStart:Float=0.25, divStart: Float = 10, divEnd: Float = 1e5, \n", " moms: (Float,Float,Float) = (0.95,0.85,0.95)) \n", "-> ((Float) -> Float, (Float) -> Float){\n", " let lrSched = combineSchedules(\n", " pcts: [pctStart, 1-pctStart], \n", " schedules: [makeAnnealer(start: lrMax/divStart, end: lrMax, schedule: cosineSchedule),\n", " makeAnnealer(start: lrMax, end: lrMax/divEnd, schedule: cosineSchedule)])\n", " let momSched = combineSchedules(\n", " pcts: [pctStart, 1-pctStart], \n", " schedules: [makeAnnealer(start: moms.0, end: moms.1, schedule: cosineSchedule),\n", " makeAnnealer(start: moms.1, end: moms.2, schedule: cosineSchedule)])\n", " return (lrSched, momSched)\n", "}" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "// export\n", "extension Learner where Opt.Scalar: BinaryFloatingPoint, \n", " Opt.Model.AllDifferentiableVariables == Opt.Model.TangentVector{\n", "\n", " public func addOneCycleDelegates(_ lrMax: Float, pctStart:Float=0.25, divStart: Float = 10, divEnd: Float = 1e5, \n", " moms: (Float,Float,Float) = (0.95,0.85,0.95)) {\n", " let scheds = oneCycleSchedulers(lrMax, pctStart: pctStart, divStart: divStart, divEnd: divEnd, moms: moms)\n", " addDelegates([makeParamScheduler(scheds.0 , hp: HyperParams.lr), \n", " makeParamScheduler(scheds.1 , hp: HyperParams.mom)])\n", " }\n", "}" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "And check it's all training properly." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "let optFunc: (CNNModel) -> StatefulOptimizer = adamOpt(lr: 1e-3, mom: 0.9, beta: 0.99, wd: 1e-2, eps: 1e-6)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "let learner = Learner(data: data, lossFunc: softmaxCrossEntropy, optFunc: optFunc, modelInit: modelInit)\n", "let recorder = learner.makeDefaultDelegates(metrics: [accuracy])\n", "learner.delegates.append(learner.makeNormalize(mean: mnistStats.mean, std: mnistStats.std))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Epoch 0: [1.3358655, 0.582] \n", " \r" ] } ], "source": [ "learner.addOneCycleDelegates(1e-3)\n", "learner.fit(1)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZgAAAEKCAYAAAAvlUMdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xd8VVW6+P/Pk5NCCgTSaCEkQOidSBMro2IZGREHUBEdEBs6cx1HceZex58zjnqdaxvLiKKDihTLKDoqFpBgoYQSeorU0BJICAmBhCTr+8fZ+IsxhUD22ac879eLFzvrrL3y7J2T82TvtddaYoxBKaWUam5BTgeglFLKP2mCUUopZQtNMEoppWyhCUYppZQtNMEopZSyhSYYpZRSttAEo5RSyhaaYJRSStlCE4xSSilbBDsdgJPi4uJMcnKy02EopZRPWbNmzSFjTHxj9QI6wSQnJ5ORkeF0GEop5VNEZNfp1NNbZEoppWyhCUYppZQtNMEopZSyhSYYpZRSttAEo5RSyha2JhgRGSMiWSKSKyIz63g9TEQWWK+vFJHkGq89aJVnichlNcpfE5F8EdlUq60YEflCRHKs/9vYeWxKKaUaZluCEREX8AJwOdAbmCQivWtVmwoUGWO6AU8DT1j79gYmAn2AMcCLVnsA/7LKapsJfGWMSQW+sr5WSinlEDvHwQwFco0x2wFEZD4wFthSo85Y4GFr+13geRERq3y+MaYc2CEiuVZ73xtj0mte6dRq60Jrew7wNfBA8x2OaqqqasPmfcWs33OE0vLKH8ujwoKJDg+hdUQobVuF0alNBJFhAT0kSym/ZOdvdUdgT42v84Bh9dUxxlSKSDEQa5WvqLVvx0a+X1tjzH6rrf0iklBXJRGZDkwHSEpKOr0jUU1SVW1YsHoPs9J/YOfhstPap01ECF3jo+jVvhU927dkQGJrerVvhStIbI5WKWUXOxNMXZ8M5jTrnM6+Z8QYMwuYBZCWltYsbar/X15RGfcuyGTVzkIGdGrNU6NTGd4llpjIUACMgWMVlRwpO8mRsgr2F58gr+g4e4rKyDlYwgfr9lKywn21ExUWzODObTi3aywX90ygW0IU7gtcpZQvsDPB5AGdanydCOyrp06eiAQD0UDhae5b20ERaW9dvbQH8s8meNV0m/YWc/Prqyk/WcXfrxvAtYM71pkQwkNdxEWF1dmGMYa8ouOs3V3E6p2FrNpRyGOfbuOxT7eR2CacS3q35eoBHRjYqbUmG6W8nJ0JZjWQKiIpwF7cnfbX16qzCJgCfA+MB5YYY4yILALeFpGngA5AKrCqke93qq3Hrf8/bK4DUY3bvK+YSbNW0LJFMPOnj6RbQsszakdE6BQTQaeYCMYOdN8V3V98nKXbCliy7SBzV+zm9W930ikmnGsGJTLhnE50bB3enIeilGomYox9d4lE5ArgGcAFvGaMeVREHgEyjDGLRKQF8CYwCPeVy8QaDwX8CfgNUAn8zhjzqVU+D3dnfhxwEPizMWa2iMQCC4EkYDdwnTGmsKH40tLSjE52efb2FJYx7qXvCA4S3rtjJB1s/MA/euIkizcdYFHmPr7JPYQAF/ZIYPKIzlzYPV6vapTyABFZY4xJa7SenQnG22mCOXsnTlZx7UvfsaewjHfvGEn3tmd25XIm9hSWsTBjDwtW7yG/pJye7Vpyx4VdubJfe4JdOoZYKbtogjkNmmDO3v98sIk3V+zi1ZvS+EXvto7EUFFZzaLMfby87Ady8ktJbBPO9PO7MOGcToQFuxpvQCnVJKebYPTPPHXGlmbl8+aKXdx6XopjyQUgNDiI8UMSWfy783nlpjQSWobx0IebGf1/y/hg3V6qqwP3jyilnKQJRp2R0vJK/vT+RlITorjvsh5OhwNAUJBwSe+2vHfHSN6aOozo8BB+t2A9v3z+G77NPeR0eEoFHE0w6oz8fXEW+4+e4Inx/b3uNpSIMCo1jo9mjOKZCQM5UnaSG15dyS2vr2L3aQ78VEqdPU0wqsly80t4c8Uurh+axOAk751TNChI+NWgjnz1+wv44xU9WbWjkEueXsYLS3OpqKx2Ojyl/J4mGNVkf/l4KxGhLu69pLvToZyWFiEupp/flS9/fwEX9UjgycVZXPWP5aze2eBT7Eqps6QJRjXJd7mHWJZdwG9HpxJbz2h8b9U+Opx/Th7C7ClpHCuv4rp/fs+fP9zE8Yoqp0NTyi9pglGnzRjD3z/PokN0CyaP6Ox0OGdsdK+2fHHv+dxybjJzvt/Flf9Yzoa8I06HpZTf0QSjTtvXWQWs3X2Eu0enel3HflNFhAbz51/2Ye60YRyvqGLci9/x7Jc5VFZp34xSzUUTjDpt/1iSQ6eYcMYPSXQ6lGZzbrc4Pvvt+VzZvz1Pf5nNdS9/z94jx50OSym/oAlGnZY1uwpZu/sI00Z1IcTPpmGJjgjh2YmDeG7SIHIOlnLVc8tZll3gdFhK+Tz/+qRQtnklfQfR4SFcl+Y/Vy+1XT2gA4tmnEvbVi24+fVVPPVFNlU6C4BSZ0wTjGrUrsPHWLzlADcOTyIi1L+XNu4SH8W/7zyXcYMSee6rHKa8torDpeVOh6WUT9IEoxo1+5sdhAQFMWVEstOheER4qIu/X9efJ67tx6qdhYx94VuyDpQ4HZZSPkcTjGpQ0bEK3snIY+zADiS0auF0OB4jIkw4J4mFt42gorKaa1/6jiXbDjodllI+RROMatDbq3Zz/GQV087r4nQojhjYqTWLZowiOS6CqXMyeCV9O4G8xIVSTaEJRtWrqtrw9srdjOoWR492nltIzNu0i27BO7eN5PK+7Xj0k6088N4GTup4GaUapQlG1evb3EPsPXKciUM7OR2K48JDXTw/aTD3XNyNhRl5TJuTwbHySqfDUsqraYJR9Vqweg9tIkK4xMHFxLxJUJBw76U9eHxcP5bnFHD9Kyv0CTOlGqAJRtXpcGk5n285wLjBiT4/LUxzmzg0iVmT09h2oIRrX/pO15hRqh6aYFSd/r1uLyerDBPO0dtjdflF77a8feswjhw/ybiXvmPT3mKnQ1LK62iCUT9jjGH+6j0MTmpN97aB27nfmCGdY3j39hGEuoRJr6xg7e4ip0NSyqtoglE/s3Z3Ebn5pUw8J8npULxet4SWLLx9BDGRoUx+dSUrth92OiSlvIYmGPUzC1bvITLUxZX92zsdik9IbBPBwttG0L51ODe/vkonylTKoglG/cSJk1V8uvEAY/q2JzLMv+cda05tW7VgwfThpMRFceucDD7ffMDpkJRynCYY9RNfZxVQUl7J2IEdnA7F58RGhTH/1uH06tCKO+eu5bNNmmRUYNMEo37io8x9xEWFMrJrrNOh+KToiBDemjqUfonR3D1vLV9u0fnLVODSBKN+VFpeyZdbD3JFv/YE+9miYp7UskUIc34zlN7t3VcyS7flOx2SUo7QTxH1o883H6C8slpvjzWDVi1CeOM3w+jeLorb3lpDunb8qwCkCUb9aFHmPjq2DmdwUhunQ/EL7ttlw+gWH8Wtb2Twbe4hp0NSyqNsTTAiMkZEskQkV0Rm1vF6mIgssF5fKSLJNV570CrPEpHLGmtTREaLyFoRWS8i34hINzuPzd8cLi1nec4hfjmgAyLidDh+o3VEKG9NG0ZybCS3vpHBOh2MqQKIbQlGRFzAC8DlQG9gkoj0rlVtKlBkjOkGPA08Ye3bG5gI9AHGAC+KiKuRNl8CbjDGDATeBv7brmPzR59sOkBVteHqAXp7rLnFRIby5tShxEWFccu/VpN9UFfHVIHBziuYoUCuMWa7MaYCmA+MrVVnLDDH2n4XGC3uP5/HAvONMeXGmB1ArtVeQ20aoJW1HQ3ss+m4/NLHmfvolhBFr/Y6NYwdElq14K2pwwh1BTF59kr2FOoEmcr/2ZlgOgJ7anydZ5XVWccYUwkUA7EN7NtQm9OAT0QkD5gMPN4sRxEADpWWs3pnIVf0a6+3x2yUFBvBG1OHcryiismzV1JQolP9K/9mZ4Kp65Oq9lqz9dVpajnAfwFXGGMSgdeBp+oMSmS6iGSISEZBgT7ZA/DlloNUG7isj677Yree7Vrx+i1DOXi0nJteW0Xx8ZNOh6SUbexMMHlAzbneE/n5basf64hIMO5bW4UN7FtnuYjEAwOMMSut8gXAyLqCMsbMMsakGWPS4uPjz+S4/M5nmw/QKSac3u1bNV5ZnbUhndvwz8lDyM0vYeq/VnO8osrpkJSyhZ0JZjWQKiIpIhKKu9N+Ua06i4Ap1vZ4YIkxxljlE62nzFKAVGBVA20WAdEi0t1q6xJgq43H5jeOnjjJt7mHGNOnnd4e86ALusfz9ISBrNldxIy311JZVe10SEo1O9tmMzTGVIrIDGAx4AJeM8ZsFpFHgAxjzCJgNvCmiOTivnKZaO27WUQWAluASuAuY0wVQF1tWuW3Au+JSDXuhPMbu47Nnyzdls/JKsOYvu2cDiXgXNW/A0VlJ/mfDzbx8Eeb+cvYvprklV8R9wVDYEpLSzMZGRlOh+GoO+euYfXOIlY+OJqgIP1wc8Jjn27l5WXbmXl5T26/oKvT4SjVKBFZY4xJa6yezscewE6crGLptgKuHdJRk4uDHrisJ3uLjvP4p9vo0DpcxyIpv6EJJoClZxdw/GQVY/rowmJOCgoS/n7dAPKPlnPfwkzatgxjWBedzVr5Pp2LLIB9tvkA0eEhDOsS43QoAa9FiItZNw0hMSac6W+uITe/1OmQlDprmmACVFW1Yem2fC7umUCITs3vFVpHhDLnlqGEuISbX1+lAzGVz9NPlgC1fk8RRWUnGd0rwelQVA2dYiJ47eZzOFxawbQ5qzlxUsfIKN+lCSZAfbU1n+Ag4bxUHWzqbfontubZiQPZsLeYP7y7gUB+0lP5Nk0wAWrJtnzSktsQHR7idCiqDpf2acf9l/Xko8x9/GNJrtPhKHVGNMEEoLyiMrYdKGF0T517zJvdfkEXxg3uyFNfZPOfDfudDkepJtMEE4BOrRF/sfa/eDUR4bFx/RjSuQ2/f2c9G/OKnQ5JqSbRBBOAlmzLJzk2gi5xkU6HohoRFuzi5clDiI0MY9obqzl49ITTISl12jTBBJiyikq+/eEwF/dsq/Ne+Yi4qDBenZJGyYlKbn0jQ2dfVj5DE0yA+S73MBWV1VzcU2+P+ZJe7Vvx7MRBbNxbzP3v6ZNlyjdoggkwX23LJzLUxdAUHb3vay7p3ZY/XNaDjzL3MSt9u9PhKNUoTTABxBj36P3zu8cTGqw/el90xwVdubJfe574bBvLc3RFVuXd9FMmgGzZf5QDR0/o7TEfJiL87/j+pCa05O5569hTWOZ0SErVSxNMAFmyNR8RuLCHJhhfFhkWzKybhlBdbbj1jQzKKiqdDkmpOmmCCSBfbcunf2Jr4luGOR2KOkudYyN5btIgsg6W8MB7G7XTX3klTTABoqCknMy8I4zW22N+48IeCdx3qbvT/9XlO5wOR6mf0QQTINKzCzAG7X/xM3de2JUr+rXjsU+38k3OIafDUeonNMEEiPScAuKiQundvpXToahmJCI8OX4A3RKimDFvrXb6K6+iCSYAVFcblucc4rzUeIKCdPS+v4kMC2bW5DSqqw23vblGR/orr6EJJgBs3neUwmMVnN89zulQlE2S4yJ5dtIgth44yp8+0E5/5R00wQSAdGtAni4u5t8u6pHAb0en8v7avcxbtcfpcJTSBBMIlmUX0KdDK+Ki9PFkf3fPxamc3z2ehxdtZkPeEafDUQFOE4yfKzlxkrW7iji/u169BIKgIOGZCQOJiwrljrfWcqSswumQVADTBOPnvv/hMJXVhvP19ljAiIkM5cUbh5BfcoL/WrCe6mrtj1HO0ATj59JzCogMdTGkcxunQ1EeNLBTax66qjdLswp48etcp8NRAUoTjJ9Lzz7EiK6xOntyALpxeGfGDuzAU19k822uDsJUnqefOn5s56Fj7C4s0/6XACUiPDauH13jo7hn3joOFOtyy8qzNMH4sVOPJ2v/S+CKCA3mpRuHcOJkFXe9vZaTVdVOh6QCiCYYP7Ysq4CkmAiS4yKdDkU5qFtCFE+M78+aXUU89sk2p8NRAcTWBCMiY0QkS0RyRWRmHa+HicgC6/WVIpJc47UHrfIsEbmssTbF7VERyRaRrSJyj53H5u0qKqv5fvthHb2vALiqfwduHpnMa9/u4LNNB5wORwUI2xKMiLiAF4DLgd7AJBHpXavaVKDIGNMNeBp4wtq3NzAR6AOMAV4UEVcjbd4MdAJ6GmN6AfPtOjZfkLGrkLKKKr09pn70xyt6MSAxmj+8m6mTYiqPsPMKZiiQa4zZboypwP2BP7ZWnbHAHGv7XWC0iIhVPt8YU26M2QHkWu011OYdwCPGmGoAY0y+jcfm9dKzDxEcJIzoGut0KMpLhAYH8fz1gwGY8fZaKiq1P0bZy84E0xGoOSFSnlVWZx1jTCVQDMQ2sG9DbXYFJohIhoh8KiKpdQUlItOtOhkFBQVndGC+ID27gMGd29CyRYjToSgv0ikmgifHDyAzr5jHPt3qdDjKz9mZYOqaF772kOL66jS1HCAMOGGMSQNeAV6rKyhjzCxjTJoxJi0+3j9vHxWUlLNl/1Eu0MeTVR3G9G3HLecm8/q3O7U/RtnKzgSTh7tP5JREYF99dUQkGIgGChvYt6E284D3rO1/A/3P+gh81HJ9PFk14sHLe9E/MZr7tT9G2cjOBLMaSBWRFBEJxd1pv6hWnUXAFGt7PLDEuBeyWARMtJ4ySwFSgVWNtPkBcLG1fQGQbdNxeb307AJiI0Pp00FXr1R1Cw0O4vlJgzFof4yyj20JxupTmQEsBrYCC40xm0XkERG52qo2G4gVkVzgXmCmte9mYCGwBfgMuMsYU1Vfm1ZbjwPXishG4DFgml3H5s1OrV45KjVOV69UDUqKjeDJ8f3JzCvm8U91fIxqfsF2Nm6M+QT4pFbZQzW2TwDX1bPvo8Cjp9OmVX4EuPIsQ/Z5W/Yf5fCxCr09pk7LmL7tfxwfM6xLDJf1aed0SMqP6Eh+P7Ms21q9UgdYqtP04BU96dcxmj+8o/0xqnlpgvEz6dkF9GrfioSWLZwORfmIsGAXL1w/GGNgxrx12h+jmo0mGD9SWl7Jml1FOj2MarKk2Aj+d3x/Mvcc4YnPtD9GNQ9NMH7k1OqVF2j/izoDl/drz5QRnZn9zQ4+36zjY9TZ0wTjR9KzCwgPcTEkWVevVGfmj1f2om/HVtyn/TGqGWiC8SPpOQWM6BpLWLDL6VCUj6rZH3O39seos6QJxk/sOnyMXYfLOD9V+1/U2ekcG8nj1/Zn/Z4j/P3zLKfDUT5ME4yfSLceT9blkVVzuLJ/e24cnsSs9O0s2XbQ6XCUj9IE4yeWZR8isU04Kbp6pWom/31lb3q1b8W9CzPZd+S40+EoH6QJxg9UVFbz/Q+HOL97PO7ldJQ6ey1CXLxw/SBOVlZzz7x1VFZpf4xqGk0wfmDt7iKO6eqVygZd4qP427h+ZOwq4qkvAnb+WHWGNMH4geU5BbiChJHddPVK1fzGDuzIhLROvPj1Dz9ORaTU6dAE4wfSsw8xOKk1rXT1SmWTh6/uQ4+2Lbl3wXoOHj3hdDjKRzSaYEQkSEQ2eSIY1XSHS8vZtK9Yb48pW4WHunj++kGUVVTx2/nrqKquvTitUj/XaIIxxlQDmSKS5IF4VBN9k3sIY/TxZGW/1LYteWRsH1ZsL+S5r3KcDkf5gNNdD6Y9sFlEVgHHThUaY66ufxflCcuyC2gTEULfjtFOh6ICwHVpnfh++2GeW5LDsJQYRnbTgb2qfqebYP4/W6NQZ8SYU6tXxuPS1SuVh/xlbF8y9xzhtwvW88k95xHfMszpkJSXOq1OfmPMsrr+2R2catjW/SUUlJTr9DDKoyLDgnn++sEcPX6Sexeup1r7Y1Q9GkwwIlIiIkfr+FciIkc9FaSqW3qOTg+jnNGrfSv+/Ms+LM85xEvLfnA6HOWlGrxFZoxp6alAVNOlZxfQs11L2rbS1SuV500a6u6P+b/PszgnOYahKTFOh6S8jI6D8VFlFZVk7CziPL09phwiIvztmr4kxURwz7x1FB6rcDok5WU0wfioFdsPU1FVrbfHlKNatgjh+esHU3isgvveydT+GPUTmmB8VHr2IVqEBHFOst6WUM7q2zGaP13ZiyXb8nn1m+1Oh6O8iCYYH5WeU8CwlFhahOjqlcp5N43ozJg+7fjfz7JYu7vI6XCUl9AE44PyisrYXnBMb48pryEiPDG+P+2iW3D32+soLjvpdEjKC2iC8UHp2YcAuKC7dvAr7xEd7u6PyS85wR/ezcQY7Y8JdJpgfFB6dgEdolvQNT7K6VCU+omBnVrzwJiefL7lIP/6bqfT4SiHaYLxMSerqvlWV69UXmzqqBR+0SuBv32ylQ15R5wORzlIE4yPydhZRMmJSi7skeB0KErVSUT4+3UDiI8KY8bb6zh6QvtjApUmGB+zNCufEJcwSgdYKi/WOiKUf1w/iL1HjvPgexu1PyZA2ZpgRGSMiGSJSK6IzKzj9TARWWC9vlJEkmu89qBVniUilzWhzX+ISKldx+S0JdvyGZYSS1TY6U6ErZQzhnSO4b5Le/CfjfuZu3K30+EoB9iWYETEBbwAXA70BiaJSO9a1aYCRcaYbsDTwBPWvr2BiUAfYAzwooi4GmtTRNKA1nYdk9P2FJaRm1/KRT319pjyDbed34ULusfzyMdb2Lyv2OlwlIfZeQUzFMg1xmw3xlQA84GxteqMBeZY2+8Co8Xdcz0WmG+MKTfG7AByrfbqbdNKPk8C99t4TI5ampUPwMWaYJSPCAoSnvr1ANpEhHD32+soLa90OiTlQXYmmI7Anhpf51llddYxxlQCxUBsA/s21OYMYJExZn8zxe91lmzLJyUukpS4SKdDUeq0xUaF8ezEQew8fIz//rf2xwQSOxNMXc/Q1n5n1VenSeUi0gG4DvhHo0GJTBeRDBHJKCgoaKy61zheUcX3PxzmIn16TPmg4V1i+d0vuvPB+n28k5HndDjKQ+xMMHlApxpfJwL76qsjIsFANFDYwL71lQ8CugG5IrITiBCR3LqCMsbMMsakGWPS4uN9Z6qV7344RHllNRf19J2Ylarprou6MbJrLA8t2kT2wRKnw1EeYGeCWQ2kikiKiITi7rRfVKvOImCKtT0eWGLc18+LgInWU2YpQCqwqr42jTH/Mca0M8YkG2OSgTLrwQG/sWRbPhGhLl3USfksV5DwzMSBRIWFcNfctZRVaH+Mv7MtwVh9KjOAxcBWYKExZrOIPCIiV1vVZgOx1tXGvcBMa9/NwEJgC/AZcJcxpqq+Nu06Bm9hjGHptnxGdYsjLFhnT1a+K6FlC56ZMJDcglIe+tDvf3UDnq2DKYwxnwCf1Cp7qMb2Cdx9J3Xt+yjw6Om0WUcdv5qkK+tgCfuKT3DP6FSnQ1HqrI1KjePui7rx3JJczkluw4RzkpwOSdlER/L7gKXb3A8j6PgX5S9++4vunJcax/98uJlNe3V8jL/SBOMDlm7Lp0+HVrRt1cLpUJRqFq4g4dmJg4iLDOX2t9ZwpKzC6ZCUDTTBeLnispOs2V2kgyuV34mJDOWFGwZz8OgJ7l2YSXW1jo/xN5pgvNyynAKqqo3eHlN+aVBSGx66qjdLtuXz4td1jixQPkwTjJf7YstBYiNDGZDot1OsqQB34/DO/GpgB576Iptvcg45HY5qRppgvFh5ZRVLt+VzSe+2uIJ0cTHln0SEv43rR7eEKO6Zv479xcedDkk1E00wXuz7Hw5TWl7JpX3aOh2KUraKCA3mpRuHUH6yijvnrqWistrpkFQz0ATjxT7fcpDIUBcju+riYsr/dY2P4snrBrBu9xH+9slWp8NRzUATjJeqrjZ8seUgF/ZIoEWIjt5XgeGKfu2ZNiqFf323kw/X73U6HHWWNMF4qXV7jlBQUq63x1TAeeDyngxLieH+dzfoIEwfpwnGS32+5QAhLtHHk1XACXEF8cINg4mNDOW2N9dwuLTc6ZDUGdIE44WMMXy++SDDu8TSqkWI0+Eo5XFxUWH8c/IQCkrLmfH2OiqrtNPfF2mC8UK5+aXsOHSMy/q0czoUpRzTP7E1j13Tj++3H+Zvn2xzOhx1BmydTVmdmc82HQDgkt7a/6IC27VDEtm0r5jXvt1B346tGDc40emQVBPoFYwX+njDfs5JbqOTWyoF/PGKXgzvEsOD729kY552+vsSTTBeJvtgCVkHS7iqfwenQ1HKK4S4gnjh+sHERYVx25sZHNJOf5+hCcbLfJy5jyCBy/tp/4tSp8RGhfHy5CEcPlbBXXPXclI7/X2CJhgvYozh4w37GZYSS0JLvT2mVE19O0bz+LX9WLmjkL9+vMXpcNRp0ATjRbbsP8r2Q8e4akB7p0NRyitdMyiRaaNSmPP9Luau3OV0OKoRmmC8yMcb9uMKEi7vqwlGqfo8eEUvLuwRz0Mfbua7XJ3e35tpgvESxhg+ytzHud3iiIkMdTocpbyWK0j4x6RBdI2P5Pa31rC9oNTpkFQ9NMF4icy8YvKKjnNVf716UaoxLVuEMHvKOQS7gpg6J4MjZRVOh6TqoAnGS3y4fi8hLuGy3vr0mFKno1NMBC9PHsLeouPcqU+WeSVNMF6gorKaD9fv45LebYmO0LnHlDpd5yTH8Ni4fnz3w2H+vGgzxhinQ1I16FQxXmDJtnwKj1UwfohOg6FUU107JJHcglJe+voHusVH8ZtRKU6HpCyaYLzAu2v2kNAyjPNT450ORSmf9IdLe7C9oJS//mcLiW3CuVQnivUKeovMYfklJ1iaVcA1gzsS7NIfh1JnIihIeGbCIPontuae+etYu7vI6ZAUmmAc98G6vVRVG64b0snpUJTyaeGhLmZPSaNtqxZMm5PBzkPHnA4p4GmCcZAxhncy8hiU1JpuCVFOh6OUz4uNCmPOLUMBmPL6Kl0N02GaYByUmVdMTn6pXr0o1YyS4yJ5dUoaB4+e4DdzMjheUeV0SAFLE4yDFqzeTVhwkM49plQzG5zUhucmDmJj3hHunreOqmp9fNkJtiYYERkjIlkikisiM+sK/GXkAAARc0lEQVR4PUxEFlivrxSR5BqvPWiVZ4nIZY21KSJzrfJNIvKaiHj1gJKiYxW8v3Yv1wzqSKsWXh2qUj7p0j7tePjqPny59SB/XrRJx8g4wLYEIyIu4AXgcqA3MElEeteqNhUoMsZ0A54GnrD27Q1MBPoAY4AXRcTVSJtzgZ5APyAcmGbXsTWHeat3U15Zzc3nJjsdilJ+66YRydx2QRfeWrGbp7/McTqcgGPnOJihQK4xZjuAiMwHxgI1F3IYCzxsbb8LPC8iYpXPN8aUAztEJNdqj/raNMZ8cqpREVkFeO2oxZNV1bz5/S5Gdo2lZ7tWToejlF+bOaYnRccqeO6rHKLDQ5iqAzE9xs5bZB2BPTW+zrPK6qxjjKkEioHYBvZttE3r1thk4LO6ghKR6SKSISIZBQUFTTyk5rF48wH2F5/glnP1ja6U3USEx8b15/K+7fjLx1tYmLGn8Z1Us7AzwUgdZbVvgtZXp6nlNb0IpBtjltcVlDFmljEmzRiTFh/vzMj517/dSVJMBBf3THDk+ysVaFxBwjMTB3Jeahwz39vAZ5v2Ox1SQLAzweQBNZ+/TQT21VdHRIKBaKCwgX0bbFNE/gzEA/c2yxHYYEPeEdbsKmLKyGRcQXXlS6WUHcKCXbw8eQgDO7Xmnnnr+SZHFyuzm50JZjWQKiIpIhKKu9N+Ua06i4Ap1vZ4YIlxP+qxCJhoPWWWAqQCqxpqU0SmAZcBk4wxXjtv9yvLdxAZ6uK6NK/tIlLKb0WEBvP6zUPpEh/J9DczWLNLp5Sxk20JxupTmQEsBrYCC40xm0XkERG52qo2G4i1OvHvBWZa+24GFuJ+IOAz4C5jTFV9bVpt/RNoC3wvIutF5CG7ju1M5eaX8vGGfUwekayPJivlkOiIEN6YOpSElmHc8voqNu0tdjokvyWB/Gx4WlqaycjI8Nj3u3fBej7ddIDlD1xEXFSYx76vUurn8orKmPDyCkrLK5k7bRh9O0Y7HZLPEJE1xpi0xurpSH4Pyc0v5YP1e7lhWJImF6W8QGKbCOZPH05kqIsbZ69ky76jTofkdzTBeMiTi7cRHuLi9gu7Oh2KUsrSKSaC+dNHEB7i4oZXV7B1vyaZ5qQJxgPW7Cpk8eaD3HZBV716UcrLJMW6r2TCgl3c8OpKth3QJNNcNMHYrKra8MhHW4hvGca083RgpVLeqHNsJPOnDyfUFcTEWSvYkHfE6ZD8giYYm81btZvMvGL++8peRITqCtVKeavkuEgW3jaCqLBgrn9lJat2FDodks/TBGOj/KMneHJxFiO6xHL1gA5Oh6OUakRSbATv3D6ChFZh3PTaStKznZlOyl9ogrGJMYb739vAiZNV/OVXfXHP4amU8nbto8NZeNsIUuKimDYng8WbDzgdks/SBGOTN1fs4uusAv54RS9dDlkpHxMXFcb8W4fTp2Mr7py7lgWrdzsdkk/SBGODVTsKeeSjLVzUI56bRnR2Ohyl1BmIjgjhranDOLdbHA+8t5FnvszWRcuaSBNMM/uhoJQ73lpDUkwEz0wcpLfGlPJhkWHBzJ6SxvghiTzzZQ4Pvr+RyiqvnerQ6+hjTc1oe0Epk2atAOCVKWlEh+t8Y0r5uhBXEE+O70+H6BY8tySX/JJynr9+kD4Vehr0CqaZLMsu4JoXv6Oy2vD2rcPpGq/9Lkr5CxHh3kt78Og1ffk6K59fv/w9+44cdzosr6cJ5iztKSzj9wszufn1VbSPbsH7d4ykR7uWToellLLBDcM68+qUNHYeKuPq57/V6f4bobMpn8Fsym+u2MWanYX8UHCMjXuLCXUFccuoZH47OlUvm5UKADkHS5j2Rgb7j5zgb+P6MX5IYK3vdLqzKeun4RlYtaOQdbuL6Ng6nPvH9OBXAzvSoXW402EppTwktW1LPrzrXO56ey33vZPJtv1HmXl5T4JdelOoJr2C8eB6MEop/3KyqppH/7OVf323k2EpMTw3aRBtW7VwOizb6XowSillsxBXEA9f3Yenfj2ADXnFXPnccr7JOeR0WF5DE4xSSp2lcYMTWTTjXNpEhDL5tZU8/UU2VdWBe3foFE0wSinVDFLbtuTDGedyzaCOPPtVDje8uoK8ojKnw3KUJhillGomEaHBPPXrgTw5vj8b84oZ88xyFq7eE7BTzGiCUUqpZnZdWic++9359OnQivvf28C0ORnkl5xwOiyP0wSjlFI26BQTwbxbh/M/V/Xmm9xDXPZ0Ou+uyQuoqxlNMEopZZOgIGHqqBT+c895pMRFct87mUx4eQVZB0qcDs0jNMEopZTNuiVE8e7tI3ni2n7k5JdwxXPL+dsnWzlWXul0aLbSBKOUUh4QFCRMOCeJJb+/kF+nJTIrfTsX/f1r5q7cxUk/XQJAE4xSSnlQm8hQHhvXn/fvHElSTAR/+vcmLnlqGR9l7qPaz8bOaIJRSikHDE5qwzu3j2D2lDTCgl3cPW8dv3z+Gz7bdMBvBmnqXGQ6F5lSymFV1YZFmXt56ots9hQeJyUukqmjUhg/JJEWIS6nw/uZ052LTBOMJhillJeorKpm8eaDzEr/gcy8YmIiQ7lxWBK/PqcTiW0inA7vR5pgToMmGKWUNzLGsGpHIbPSt7MkKx+AUd3iuC6tE6N7JhAZ5uxKK14xm7KIjBGRLBHJFZGZdbweJiILrNdXikhyjdcetMqzROSyxtoUkRSrjRyrzVA7j00ppewiIgzrEsvsm89h+f0Xcc/FqWwvOMY989Yx5K9fcMdba1iUuY8jZRVOh9og265gRMQFZAOXAHnAamCSMWZLjTp3Av2NMbeLyETgGmPMBBHpDcwDhgIdgC+B7tZudbYpIguB940x80Xkn0CmMealhmLUKxillK+oqjas3lnIJxv388nGAxwqLSdIoH9ia85LjWNI5zYM6tSG6IgQ22PxhhUthwK5xpjtVkDzgbHAlhp1xgIPW9vvAs+LiFjl840x5cAOEcm12qOuNkVkK3AxcL1VZ47VboMJRimlfIUrSBjeJZbhXWL58y/7sH7PEdKzC0jPKeCFpbmcevCsS3wkPdu1JDWhJd0SoujQOpwOrVuQ0LIFriDxaMx2JpiOwJ4aX+cBw+qrY4ypFJFiINYqX1Fr347Wdl1txgJHjDGVddRXSim/4goShnRuw5DObfivS7pTWl7Jhj1HWLu7iMy8YrbsO8qnmw5Q8waVCESFBhPVIpiosGBeuSmN5LhIW+O0M8HUlSpr34+rr0595XX1GTVU/+dBiUwHpgMkJSXVVUUppXxKVFgwI7vFMbJb3I9lJ05WsfPwMfYfOcH+4hMcPHqCkhOVlJafpOREJRFh9j/+bGeCyQM61fg6EdhXT508EQkGooHCRvatq/wQ0FpEgq2rmLq+FwDGmFnALHD3wTT9sJRSyvu1CHHRs10rerZr5VgMdj5FthpItZ7uCgUmAotq1VkETLG2xwNLjPupg0XAROspsxQgFVhVX5vWPkutNrDa/NDGY1NKKdUI265grD6VGcBiwAW8ZozZLCKPABnGmEXAbOBNqxO/EHfCwKq3EPcDAZXAXcaYKoC62rS+5QPAfBH5K7DOalsppZRDdKClPqaslFJN4hUDLZVSSgUuTTBKKaVsoQlGKaWULTTBKKWUsoUmGKWUUrYI6KfIRKQA2HWGu8fhHuDpbTSupvHGuLwxJtC4msqf4+psjIlvrFJAJ5izISIZp/OYnqdpXE3jjXF5Y0ygcTWVxqW3yJRSStlEE4xSSilbaII5c7OcDqAeGlfTeGNc3hgTaFxNFfBxaR+MUkopW+gVjFJKKVtogjkDIjJGRLJEJFdEZjoUQycRWSoiW0Vks4j81ip/WET2ish6698VDsS2U0Q2Wt8/wyqLEZEvRCTH+r+Nh2PqUeOcrBeRoyLyOyfOl4i8JiL5IrKpRlmd50fcnrPeaxtEZLCH43pSRLZZ3/vfItLaKk8WkeM1zts/PRxXvT83EXnQOl9ZInKZh+NaUCOmnSKy3ir3yPlq4HPBmfeXMUb/NeEf7mUCfgC6AKFAJtDbgTjaA4Ot7ZZANtAbeBi4z+FztBOIq1X2v8BMa3sm8ITDP8MDQGcnzhdwPjAY2NTY+QGuAD7FvWrrcGClh+O6FAi2tp+oEVdyzXoOnK86f27W70AmEAakWL+rLk/FVev1/wMe8uT5auBzwZH3l17BNN1QINcYs90YUwHMB8Z6OghjzH5jzFpruwTYCnT0dBxNMBaYY23PAX7lYCyjgR+MMWc6yPasGGPSca9/VFN952cs8IZxW4F75db2norLGPO5ca8SC7AC92qxHlXP+arPWGC+MabcGLMDyMX9O+vRuEREgF8D8+z43g3EVN/ngiPvL00wTdcR2FPj6zwc/mAXkWRgELDSKpphXe6+5ulbURYDfC4ia0RkulXW1hizH9y/BECCA3GdMpGf/uI7fb6g/vPjTe+33+D+a/eUFBFZJyLLROQ8B+Kp6+fmLefrPOCgMSanRplHz1etzwVH3l+aYJpO6ihz7FE8EYkC3gN+Z4w5CrwEdAUGAvtxX6Z72rnGmMHA5cBdInK+AzHUSdxLbV8NvGMVecP5aohXvN9E5E+4V5edaxXtB5KMMYOAe4G3RcSTi7/X93PzivMFTOKnf8R49HzV8blQb9U6yprtfGmCabo8oFONrxOBfU4EIiIhuN9Ec40x7wMYYw4aY6qMMdXAK9h0e6Ahxph91v/5wL+tGA6euvS2/s/3dFyWy4G1xpiDVoyOny9LfefH8febiEwBrgJuMNaNe+sW1GFrew3uvo7unoqpgZ+bN5yvYGAcsOBUmSfPV12fCzj0/tIE03SrgVQRSbH+Gp4ILPJ0ENY93tnAVmPMUzXKa94/vQbYVHtfm+OKFJGWp7ZxdxJvwn2OpljVpgAfejKuGn7yl6XT56uG+s7PIuAm62mf4UDxqVsdniAiY4AHgKuNMWU1yuNFxGVtdwFSge0ejKu+n9siYKKIhIlIihXXKk/FZfkFsM0Yk3eqwFPnq77PBZx6f9n9VIM//sP95EU27r9C/uRQDKNwX8puANZb/64A3gQ2WuWLgPYejqsL7qd4MoHNp84PEAt8BeRY/8c4cM4igMNAdI0yj58v3AluP3AS91+QU+s7P7hvYbxgvdc2AmkejisX9z36U++xf1p1r7V+vpnAWuCXHo6r3p8b8CfrfGUBl3syLqv8X8Dttep65Hw18LngyPtLR/IrpZSyhd4iU0opZQtNMEoppWyhCUYppZQtNMEopZSyhSYYpZRSttAEo1QzEJHvrP+TReT6Zm77j3V9L6W8nT6mrFQzEpELcc/ye1UT9nEZY6oaeL3UGBPVHPEp5Ul6BaNUMxCRUmvzceA8a82P/xIRl7jXVFltTcx4m1X/QmvdjrdxD3BDRD6wJgjdfGqSUBF5HAi32ptb83tZo6+fFJFN4l5/Z0KNtr8WkXfFvZbLXGuEt1IeFex0AEr5mZnUuIKxEkWxMeYcEQkDvhWRz626Q4G+xj2tPMBvjDGFIhIOrBaR94wxM0VkhjFmYB3faxzuyR4HAHHWPunWa4OAPrjnlfoWOBf4pvkPV6n66RWMUva6FPdcT+txT5sei3seKoBVNZILwD0ikol73ZVONerVZxQwz7gnfTwILAPOqdF2nnFPBrke94JXSnmUXsEoZS8B7jbGLP5Jobuv5litr38BjDDGlInI10CL02i7PuU1tqvQ33XlAL2CUap5leBeqvaUxcAd1hTqiEh3a5bp2qKBIiu59MS9fO0pJ0/tX0s6MMHq54nHvYSvp2cOVqpe+leNUs1rA1Bp3er6F/As7ttTa62O9gLqXi76M+B2EdmAexbgFTVemwVsEJG1xpgbapT/GxiBe4ZeA9xvjDlgJSilHKePKSullLKF3iJTSillC00wSimlbKEJRimllC00wSillLKFJhillFK20ASjlFLKFppglFJK2UITjFJKKVv8P4SAxSQ8wo+/AAAAAElFTkSuQmCC\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "recorder.plotLRs()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Differential learning rates" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "To train at differential learning rates (or freeze part of the models) we need to pass to our optimizer arrays of KeyPaths (which will define our layer groups). For instance, we can begin with the firt 9 keyPaths (which corresponds to the first three ConvBNs):" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "func modelInit() -> CNNModel { return CNNModel(channelIn: 3, nOut: 10, filters: [64, 64, 128, 256]) }" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "var model = modelInit()\n", "let splitArray = [Array(model.variables.keyPaths[0..<9]), Array(model.variables.keyPaths[9...])]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "let hpGroups: [[String: Float]] = [[HyperParams.lr: 0], [HyperParams.lr: 0.1]]\n", "func optFunc(_ model: CNNModel) -> StatefulOptimizer {\n", " return StatefulOptimizer(for: model, steppers: [SGDStep()], stats: [], hpGroups: hpGroups, splitArray: splitArray)\n", "}" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "let learner = Learner(data: data, lossFunc: softmaxCrossEntropy, optFunc: optFunc, modelInit: modelInit)\n", "let recorder = learner.makeDefaultDelegates(metrics: [accuracy])\n", "learner.delegates.append(learner.makeNormalize(mean: mnistStats.mean, std: mnistStats.std))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "First parameter shouldn't change since the corresponding layer group as a LR of 0., second should." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]\n" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "learner.model.convs[0].norm.scale" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0,\n", " 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0,\n", " 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]\n" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "learner.model.convs[3].norm.scale" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Epoch 0: [1.3130397, 0.534] \n", " \r" ] } ], "source": [ "learner.fit(1)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[0.99947506, 1.0001966, 0.9900738, 0.98156047, 0.99899787, 1.0270612, 0.9824792, 1.0029291,\n", " 1.0091724, 1.0046912, 1.0073055, 1.0055526, 0.97067755, 0.9816713, 1.016569, 0.9986565]\n" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "learner.model.convs[0].norm.scale" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[ 1.0017759, 1.0072277, 0.9770908, 1.0054005, 1.0019907, 1.0047579, 1.0007318, 1.0017214,\n", " 0.97550786, 1.0026606, 1.0116018, 1.0013392, 0.9883987, 0.98546517, 0.9893817, 1.0137036,\n", " 0.999046, 1.0305443, 1.005647, 1.0052477, 1.0025676, 0.99569345, 0.9984126, 1.0053662,\n", " 0.97381794, 0.9985082, 0.9871409, 0.99648446, 0.9966728, 1.0009459, 0.9931889, 1.0144972,\n", " 1.0004792, 0.9952349, 1.0133064, 0.9818646, 0.99662995, 0.9993304, 1.001136, 0.985157,\n", " 0.99845725, 0.98352814, 0.98484004, 0.97534674, 0.98715824, 0.9976744, 0.9670125, 0.97067595,\n", " 0.9903402, 1.0086904, 0.9866701, 1.0071692, 1.0245737, 0.9928977, 1.009554, 1.0141283,\n", " 1.0014772, 0.97363013, 1.0075214, 0.9731512, 0.9915053, 1.001776, 1.0038495, 1.0007021]\n" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "learner.model.convs[3].norm.scale" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Another way to get those keyPaths is to use the keyPaths to certain layers then append the keyPaths of all the parameters inside. This function takes a model, a layer and the keyPath that points from `model.variables` to `layer.variables` and returns the keyPaths of all the parameters of that layer. " ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "public func parameterKeyPaths(\n", " _ model: M1,\n", " _ kp: WritableKeyPath,\n", " _ layer: M2) -> [WritableKeyPath]\n", "where M1: Layer, M2: Layer {\n", " return model.variables[keyPath: kp].keyPaths.map { kp.appending(path: $0) }\n", "}" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "To access a keyPath directly, we use \\ commands. Here is the keyPath to the array of convs, which lets us easily get the keyPaths for all the body of our CNN:" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "let kp = \\(CNNModel.AllDifferentiableVariables).convs\n", "let conv = model.convs\n", "let bodyKeyPaths = parameterKeyPaths(model, kp, conv)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Then we could split body and head:" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "▿ 2 elements\n", " - 0 : 21\n", " - 1 : 2\n" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "let splitArray = [bodyKeyPaths, model.variables.keyPaths.filter { return !bodyKeyPaths.contains($0) }]\n", "splitArray.map { $0.count }" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "If we want to refine this a bit and split our body between the first 4 convs and the last 3 we can proceed like this:" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "1 4\r\n", "2 5\r\n", "3 6\r\n" ] }, { "data": { "text/plain": [ "▿ 3 elements\n", " - 0 : 0 elements\n", " - 1 : 0 elements\n", " - 2 : 0 elements\n" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "let x = [1,2,3]\n", "let y = [4,5,6]\n", "zip(x,y).map { print($0, $1) }" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "let deepBody = (0..<4).map { parameterKeyPaths(\n", " model, \n", " \\(CNNModel.AllDifferentiableVariables).convs.base[$0], \n", " model.convs[$0]\n", ") }.reduce([], +)\n", "\n", "let upperBody = (4..<7).map { parameterKeyPaths(\n", " model, \n", " \\(CNNModel.AllDifferentiableVariables).convs.base[$0], \n", " model.convs[$0]\n", ") }.reduce([], +)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "▿ 3 elements\n", " - 0 : 12\n", " - 1 : 9\n", " - 2 : 2\n" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "let splitArray = [deepBody, upperBody, model.variables.keyPaths.filter { return !bodyKeyPaths.contains($0) }]\n", "splitArray.map { $0.count }" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Now let's say we want a parameter group will all the batchnorm layers. KeyPaths allow us to get all the batchnorm layers by saying `(to: FABatchNorm.self)`. The `.keyPaths` method we have been using is jsut a shortcut for `recursivelyAllWritableKeyPaths(to: TF.self)`, which grabs all the keypaths to all the tensors." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "let bns = model.recursivelyAllWritableKeyPaths(to: FABatchNorm.self).map { model[keyPath: $0] }" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Then we need the keypaths from model.variables to those batchnorms." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "let bnKeyPaths = model.variables.recursivelyAllWritableKeyPaths(to: FABatchNorm.AllDifferentiableVariables.self)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "14\n" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "let bnParameters = zip(bnKeyPaths, bns).map { parameterKeyPaths(model, $0, $1) }.reduce([], +)\n", "bnParameters.count" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Export" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "success\r\n" ] } ], "source": [ "import NotebookExport\n", "let exporter = NotebookExport(Path.cwd/\"09_optimizer.ipynb\")\n", "print(exporter.export(usingPrefix: \"FastaiNotebook_\"))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Swift", "language": "swift", "name": "swift" } }, "nbformat": 4, "nbformat_minor": 1 }