{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# \"fast.ai\" layers\n", "\n", "This notebook defines layers similar to those defined in the [Swift for TensorFlow Deep Learning Library](https://github.com/tensorflow/swift-apis), but with some experimental extra features for the fast.ai course." ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Installing packages:\n", "\t.package(path: \"/home/ubuntu/fastai_docs/dev_swift/FastaiNotebook_00_load_data\")\n", "\t\tFastaiNotebook_00_load_data\n", "With SwiftPM flags: []\n", "Working in: /tmp/tmpdhnj0dux/swift-install\n", "Fetching https://github.com/mxcl/Path.swift\n", "Fetching https://github.com/JustHTTP/Just\n", "Completed resolution in 1.22s\n", "Cloning https://github.com/JustHTTP/Just\n", "Resolving https://github.com/JustHTTP/Just at 0.7.1\n", "Cloning https://github.com/mxcl/Path.swift\n", "Resolving https://github.com/mxcl/Path.swift at 0.16.2\n", "Compile Swift Module 'Path' (9 sources)\n", "Compile Swift Module 'Just' (1 sources)\n", "Compile Swift Module 'FastaiNotebook_00_load_data' (1 sources)\n", "Compile Swift Module 'jupyterInstalledPackages' (1 sources)\n", "Linking ./.build/x86_64-unknown-linux/debug/libjupyterInstalledPackages.so\n", "Initializing Swift...\n", "Installation complete!\n" ] } ], "source": [ "%install '.package(path: \"$cwd/FastaiNotebook_00_load_data\")' FastaiNotebook_00_load_data" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Protocol" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "//export\n", "\n", "import TensorFlow\n", "\n", "public protocol FALayer: Layer {\n", " var delegate: LayerDelegate { get set }\n", " \n", " @differentiable\n", " func forward(_ input: Input) -> Output\n", "}\n", "\n", "// TODO: This doesn't actually work. So we'll just paste it into every layer definition for now.\n", "// extension FALayer {\n", "// @differentiable\n", "// public func applied(to input: Input, in context: Context) -> Output {\n", "// let activation = forward(input, in: context)\n", "// delegate.didProduceActivation(activation, in: context)\n", "// return activation\n", "// }\n", "// }\n", "\n", "open class LayerDelegate {\n", " public init() {}\n", " \n", " open func didProduceActivation(_ activation: Output) {}\n", "}" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Dense" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "//export\n", "\n", "@_fixed_layout\n", "public struct FADense: FALayer { \n", " public var weight: Tensor\n", " public var bias: Tensor\n", " public typealias Activation = @differentiable (Tensor) -> Tensor\n", " @noDerivative public let activation: Activation\n", " \n", " @noDerivative public var delegate: LayerDelegate = LayerDelegate()\n", "\n", " public init(\n", " weight: Tensor,\n", " bias: Tensor,\n", " activation: @escaping Activation\n", " ) {\n", " self.weight = weight\n", " self.bias = bias\n", " self.activation = activation\n", " }\n", "\n", " @differentiable\n", " public func forward(_ input: Tensor) -> Tensor {\n", " return activation(matmul(input, weight) + bias)\n", " }\n", " \n", " @differentiable\n", " public func applied(to input: Tensor) -> Tensor {\n", " let activation = forward(input)\n", " delegate.didProduceActivation(activation)\n", " return activation\n", " }\n", "}\n", "\n", "public extension FADense {\n", " init(\n", " inputSize: Int,\n", " outputSize: Int,\n", " activation: @escaping Activation = identity,\n", " seed: (Int64, Int64) = (Int64.random(in: Int64.min..: FALayer {\n", " public var filter: Tensor\n", " public typealias Activation = @differentiable (Tensor) -> Tensor\n", " @noDerivative public let activation: Activation\n", " @noDerivative public let strides: (Int, Int)\n", " @noDerivative public let padding: Padding\n", " \n", " @noDerivative public var delegate: LayerDelegate = LayerDelegate()\n", "\n", " public init(\n", " filter: Tensor,\n", " activation: @escaping Activation,\n", " strides: (Int, Int),\n", " padding: Padding\n", " ) {\n", " self.filter = filter\n", " self.activation = activation\n", " self.strides = strides\n", " self.padding = padding\n", " }\n", "\n", " @differentiable\n", " public func forward(_ input: Tensor) -> Tensor {\n", " return activation(input.convolved2D(withFilter: filter,\n", " strides: (1, strides.0, strides.1, 1),\n", " padding: padding))\n", " }\n", " \n", " @differentiable\n", " public func applied(to input: Tensor) -> Tensor {\n", " let activation = forward(input)\n", " delegate.didProduceActivation(activation)\n", " return activation\n", " }\n", "}\n", "\n", "public extension FANoBiasConv2D {\n", " init(\n", " filterShape: (Int, Int, Int, Int),\n", " strides: (Int, Int) = (1, 1),\n", " padding: Padding = .valid,\n", " activation: @escaping Activation = identity,\n", " generator: inout G\n", " ) {\n", " let filterTensorShape = TensorShape([\n", " filterShape.0, filterShape.1,\n", " filterShape.2, filterShape.3])\n", " self.init(\n", " filter: Tensor(glorotUniform: filterTensorShape, generator: &generator),\n", " activation: activation,\n", " strides: strides,\n", " padding: padding)\n", " }\n", "}\n", "\n", "public extension FANoBiasConv2D {\n", " init(\n", " filterShape: (Int, Int, Int, Int),\n", " strides: (Int, Int) = (1, 1),\n", " padding: Padding = .valid,\n", " activation: @escaping Activation = identity,\n", " seed: (Int64, Int64) = (Int64.random(in: Int64.min..: FALayer {\n", " public var filter: Tensor\n", " public var bias: Tensor\n", " public typealias Activation = @differentiable (Tensor) -> Tensor\n", " @noDerivative public let activation: Activation\n", " @noDerivative public let strides: (Int, Int)\n", " @noDerivative public let padding: Padding\n", " \n", " @noDerivative public var delegate: LayerDelegate = LayerDelegate()\n", "\n", " public init(\n", " filter: Tensor,\n", " bias: Tensor,\n", " activation: @escaping Activation,\n", " strides: (Int, Int),\n", " padding: Padding\n", " ) {\n", " self.filter = filter\n", " self.bias = bias\n", " self.activation = activation\n", " self.strides = strides\n", " self.padding = padding\n", " }\n", "\n", " @differentiable\n", " public func forward(_ input: Tensor) -> Tensor {\n", " return activation(input.convolved2D(withFilter: filter,\n", " strides: (1, strides.0, strides.1, 1),\n", " padding: padding) + bias)\n", " }\n", " \n", " @differentiable\n", " public func applied(to input: Tensor) -> Tensor {\n", " let activation = forward(input)\n", " delegate.didProduceActivation(activation)\n", " return activation\n", " }\n", "}\n", "\n", "public extension FAConv2D {\n", " init(\n", " filterShape: (Int, Int, Int, Int),\n", " strides: (Int, Int) = (1, 1),\n", " padding: Padding = .valid,\n", " activation: @escaping Activation = identity,\n", " generator: inout G\n", " ) {\n", " let filterTensorShape = TensorShape([\n", " filterShape.0, filterShape.1,\n", " filterShape.2, filterShape.3])\n", " self.init(\n", " filter: Tensor(glorotUniform: filterTensorShape, generator: &generator),\n", " bias: Tensor(zeros: TensorShape([filterShape.3])),\n", " activation: activation,\n", " strides: strides,\n", " padding: padding)\n", " }\n", "}\n", "\n", "public extension FAConv2D {\n", " init(\n", " filterShape: (Int, Int, Int, Int),\n", " strides: (Int, Int) = (1, 1),\n", " padding: Padding = .valid,\n", " activation: @escaping Activation = identity,\n", " seed: (Int64, Int64) = (Int64.random(in: Int64.min..: FALayer {\n", " @noDerivative let poolSize: (Int, Int, Int, Int)\n", " @noDerivative let strides: (Int, Int, Int, Int)\n", " @noDerivative let padding: Padding\n", " \n", " @noDerivative public var delegate: LayerDelegate = LayerDelegate()\n", "\n", " public init(\n", " poolSize: (Int, Int, Int, Int),\n", " strides: (Int, Int, Int, Int),\n", " padding: Padding\n", " ) {\n", " self.poolSize = poolSize\n", " self.strides = strides\n", " self.padding = padding\n", " }\n", "\n", " public init(poolSize: (Int, Int), strides: (Int, Int), padding: Padding = .valid) {\n", " self.poolSize = (1, poolSize.0, poolSize.1, 1)\n", " self.strides = (1, strides.0, strides.1, 1)\n", " self.padding = padding\n", " }\n", "\n", " @differentiable\n", " public func forward(_ input: Tensor) -> Tensor {\n", " return input.averagePooled(kernelSize: poolSize, strides: strides, padding: padding)\n", " }\n", " \n", " @differentiable\n", " public func applied(to input: Tensor) -> Tensor {\n", " let activation = forward(input)\n", " delegate.didProduceActivation(activation)\n", " return activation\n", " }\n", "}" ] }, { "cell_type": "code", "execution_count": 21, "metadata": {}, "outputs": [], "source": [ "//export\n", "\n", "@_fixed_layout\n", "public struct FAAdaptiveAvgPool2D: FALayer {\n", " @noDerivative public var delegate: LayerDelegate = LayerDelegate()\n", " \n", " public init() {}\n", "\n", " @differentiable\n", " public func forward(_ input: Tensor) -> Tensor {\n", " return input.mean(alongAxes: [1,2])\n", " }\n", " \n", " @differentiable\n", " public func applied(to input: Tensor) -> Tensor {\n", " let activation = forward(input)\n", " delegate.didProduceActivation(activation)\n", " return activation\n", " }\n", "}" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Export" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "import FastaiNotebook_00_load_data\n", "import Path\n", "notebookToScript(fname: (Path.cwd / \"01a_fastai_layers.ipynb\").string)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Swift", "language": "swift", "name": "swift" }, "language_info": { "file_extension": ".swift", "mimetype": "text/x-swift", "name": "swift", "version": "" } }, "nbformat": 4, "nbformat_minor": 2 }