{ "metadata" : { "kernelspec" : { "name" : "swift", "display_name" : "Swift", "language" : "swift" } }, "cells" : [ { "source" : [ "# Early stopping" ], "metadata" : { }, "cell_type" : "markdown" }, { "metadata" : { }, "source" : [ "%install-location $cwd\/swift-install\n", "%install '.package(path: \"$cwd\/FastaiNotebook_05_anneal\")' FastaiNotebook_05_anneal" ], "outputs" : [ ], "execution_count" : null, "cell_type" : "code" }, { "metadata" : { }, "source" : [ "\/\/ export\n", "import Path\n", "import TensorFlow\n", "import Python" ], "outputs" : [ ], "execution_count" : null, "cell_type" : "code" }, { "metadata" : { }, "source" : [ "import FastaiNotebook_05_anneal" ], "outputs" : [ ], "execution_count" : null, "cell_type" : "code" }, { "metadata" : { }, "source" : [ "%include \"EnableIPythonDisplay.swift\"\n", "IPythonDisplay.shell.enable_matplotlib(\"inline\")" ], "outputs" : [ ], "execution_count" : null, "cell_type" : "code" }, { "source" : [ "## Load data" ], "metadata" : { }, "cell_type" : "markdown" }, { "cell_type" : "code", "source" : [ "let data = mnistDataBunch(flat: true)" ], "outputs" : [ ], "execution_count" : null, "metadata" : { } }, { "cell_type" : "code", "source" : [ "let (n,m) = (60000,784)\n", "let c = 10\n", "let nHid = 50" ], "outputs" : [ ], "execution_count" : null, "metadata" : { } }, { "metadata" : { }, "source" : [ "func optFunc(_ model: BasicModel) -> SGD {return SGD(for: model, learningRate: 1e-2)}" ], "outputs" : [ ], "execution_count" : null, "cell_type" : "code" }, { "cell_type" : "code", "source" : [ "func modelInit() -> BasicModel {return BasicModel(nIn: m, nHid: nHid, nOut: c)}" ], "outputs" : [ ], "execution_count" : null, "metadata" : { } }, { "metadata" : { }, "source" : [ "let learner = Learner(data: data, lossFunc: softmaxCrossEntropy, optFunc: optFunc, modelInit: modelInit)\n", "let recorder = learner.makeRecorder()" ], "outputs" : [ ], "execution_count" : null, "cell_type" : "code" }, { "source" : [ "Check the previous callbacks load." ], "metadata" : { }, "cell_type" : "markdown" }, { "cell_type" : "code", "source" : [ "learner.delegates = [learner.makeTrainEvalDelegate(), learner.makeShowProgress(),\n", " learner.makeNormalize(mean: mnistStats.mean, std: mnistStats.std),\n", " learner.makeAvgMetric(metrics: [accuracy]), recorder]" ], "outputs" : [ ], "execution_count" : null, "metadata" : { } }, { "metadata" : { }, "source" : [ "learner.fit(2)" ], "outputs" : [ ], "execution_count" : null, "cell_type" : "code" }, { "source" : [ "Make an extension to quickly load them. " ], "metadata" : { }, "cell_type" : "markdown" }, { "metadata" : { }, "source" : [ "\/\/ export\n", "\/\/TODO: when recorder can be accessed as a property, remove it from the return\n", "extension Learner where Opt.Scalar: PythonConvertible {\n", " public func makeDefaultDelegates(metrics: [(Output, Label) -> TF] = []) -> Recorder {\n", " let recorder = makeRecorder()\n", " delegates = [makeTrainEvalDelegate(), makeShowProgress(), recorder]\n", " if !metrics.isEmpty { delegates.append(makeAvgMetric(metrics: metrics)) }\n", " return recorder\n", " }\n", "}" ], "outputs" : [ ], "execution_count" : null, "cell_type" : "code" }, { "source" : [ "## Control Flow test" ], "metadata" : { }, "cell_type" : "markdown" }, { "metadata" : { }, "source" : [ "extension Learner {\n", " public class TestControlFlow: Delegate {\n", " public override var order: Int { return 3 }\n", " \n", " var skipAfter,stopAfter: Int\n", " public init(skipAfter:Int, stopAfter: Int){ (self.skipAfter,self.stopAfter) = (skipAfter,stopAfter) }\n", " \n", " public override func batchWillStart(learner: Learner) throws {\n", " print(\"batchWillStart\")\n", " if learner.currentIter >= stopAfter {\n", " throw LearnerAction.stop(reason: \"*** stopped: \\(learner.currentIter)\")\n", " }\n", " if learner.currentIter >= skipAfter {\n", " throw LearnerAction.skipBatch(reason: \"*** skipBatch: \\(learner.currentIter)\")\n", " }\n", " }\n", " \n", " public override func trainingDidFinish(learner: Learner) {\n", " print(\"trainingDidFinish\")\n", " }\n", " \n", " public override func batchSkipped(learner: Learner, reason: String) {\n", " print(reason)\n", " }\n", " }\n", "}" ], "outputs" : [ ], "execution_count" : null, "cell_type" : "code" }, { "cell_type" : "code", "source" : [ "let learner = Learner(data: data, lossFunc: softmaxCrossEntropy, optFunc: optFunc, modelInit: modelInit)" ], "outputs" : [ ], "execution_count" : null, "metadata" : { } }, { "cell_type" : "code", "source" : [ "learner.delegates = [type(of: learner).TestControlFlow(skipAfter:5, stopAfter: 8),\n", " learner.makeTrainEvalDelegate()]" ], "outputs" : [ ], "execution_count" : null, "metadata" : { } }, { "metadata" : { }, "source" : [ "learner.fit(5)" ], "outputs" : [ ], "execution_count" : null, "cell_type" : "code" }, { "source" : [ "Check if the orders were taken into account:" ], "metadata" : { }, "cell_type" : "markdown" }, { "metadata" : { }, "source" : [ "(learner.delegates[0].order,learner.delegates[1].order)" ], "outputs" : [ ], "execution_count" : null, "cell_type" : "code" }, { "source" : [ "### LR Finder" ], "metadata" : { }, "cell_type" : "markdown" }, { "metadata" : { }, "source" : [ "\/\/ export\n", "extension Learner where Opt.Scalar: BinaryFloatingPoint {\n", " public class LRFinder: Delegate {\n", " public typealias ScheduleFunc = (Float) -> Float\n", "\n", " \/\/ A learning rate schedule from step to float.\n", " private var scheduler: ScheduleFunc\n", " private var numIter: Int\n", " private var minLoss: Float? = nil\n", " \n", " public init(start: Float = 1e-5, end: Float = 10, numIter: Int = 100) {\n", " scheduler = makeAnnealer(start: start, end: end, schedule: expSchedule)\n", " self.numIter = numIter\n", " }\n", " \n", " override public func batchWillStart(learner: Learner) {\n", " learner.opt.learningRate = Opt.Scalar(scheduler(Float(learner.currentIter)\/Float(numIter)))\n", " }\n", " \n", " override public func batchDidFinish(learner: Learner) throws {\n", " if minLoss == nil {minLoss = learner.currentLoss.scalar}\n", " else { \n", " if learner.currentLoss.scalarized() < minLoss! { minLoss = learner.currentLoss.scalarized()}\n", " if learner.currentLoss.scalarized() > 4 * minLoss! { \n", " throw LearnerAction.stop(reason: \"Loss diverged\")\n", " }\n", " if learner.currentIter >= numIter { \n", " throw LearnerAction.stop(reason: \"Finished the range.\") \n", " }\n", " }\n", " }\n", " \n", " override public func validationWillStart(learner: Learner) throws {\n", " \/\/Skip validation during the LR range test\n", " throw LearnerAction.skipEpoch(reason: \"No validation in the LR Finder.\")\n", " }\n", " }\n", " \n", " public func makeLRFinder(start: Float = 1e-5, end: Float = 10, numIter: Int = 100) -> LRFinder {\n", " return LRFinder(start: start, end: end, numIter: numIter)\n", " }\n", "}" ], "outputs" : [ ], "execution_count" : null, "cell_type" : "code" }, { "cell_type" : "code", "source" : [ "let learner = Learner(data: data, lossFunc: softmaxCrossEntropy, optFunc: optFunc, modelInit: modelInit)\n", "let recorder = learner.makeDefaultDelegates()" ], "outputs" : [ ], "execution_count" : null, "metadata" : { } }, { "cell_type" : "code", "source" : [ "learner.delegates.append(learner.makeNormalize(mean: mnistStats.mean, std: mnistStats.std))\n", "learner.delegates.append(learner.makeLRFinder())" ], "outputs" : [ ], "execution_count" : null, "metadata" : { } }, { "cell_type" : "code", "source" : [ "learner.fit(2)" ], "outputs" : [ ], "execution_count" : null, "metadata" : { } }, { "cell_type" : "code", "source" : [ "recorder.plotLRFinder()" ], "outputs" : [ ], "execution_count" : null, "metadata" : { } }, { "cell_type" : "code", "source" : [ "\/\/ export\n", "\/\/TODO: when Recorder is a property of Learner don't return it.\n", "extension Learner where Opt.Scalar: PythonConvertible & BinaryFloatingPoint {\n", " public func lrFind(start: Float = 1e-5, end: Float = 10, numIter: Int = 100) -> Recorder {\n", " let epochCount = data.train.count\/numIter + 1\n", " let recorder = makeDefaultDelegates()\n", " delegates.append(makeLRFinder(start: start, end: end, numIter: numIter))\n", " try! self.fit(epochCount)\n", " return recorder\n", " }\n", "}" ], "outputs" : [ ], "execution_count" : null, "metadata" : { } }, { "metadata" : { }, "source" : [ "let recorder = learner.lrFind()" ], "outputs" : [ ], "execution_count" : null, "cell_type" : "code" }, { "cell_type" : "code", "source" : [ "recorder.plotLRFinder()" ], "outputs" : [ ], "execution_count" : null, "metadata" : { } }, { "source" : [ "## Export" ], "metadata" : { }, "cell_type" : "markdown" }, { "metadata" : { }, "source" : [ "import NotebookExport\n", "let exporter = NotebookExport(Path.cwd\/\"05b_early_stopping.ipynb\")\n", "print(exporter.export(usingPrefix: \"FastaiNotebook_\"))" ], "outputs" : [ ], "execution_count" : null, "cell_type" : "code" }, { "cell_type" : "code", "source" : [ ], "outputs" : [ ], "execution_count" : null, "metadata" : { } } ], "nbformat" : 4, "nbformat_minor" : 2 }