{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": { "collapsed": true }, "outputs": [], "source": [ "abstract type AbstractLayer end" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "collapsed": true }, "outputs": [], "source": [ "mutable struct MulLayer{T<:AbstractFloat} <: AbstractLayer\n", " x::T\n", " y::T\n", " (::Type{MulLayer{T}})() where {T<:AbstractFloat} = new{T}()\n", "end" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "forward (generic function with 2 methods)" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "function forward(self::MulLayer{T}, x::T, y::T) where {T<:AbstractFloat}\n", " self.x = x\n", " self.y = y\n", " out = x * y\n", " return out\n", "end\n", "@inline forward(lyr::MulLayer{T}, x, y) where T = forward(lyr, T(x), T(y))" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "backward (generic function with 2 methods)" ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "function backward(self::MulLayer{T}, dout::T) where {T<:AbstractFloat}\n", " dx = dout * self.y\n", " dy = dout * self.x\n", " return dx, dy\n", "end\n", "@inline backward(lyr::MulLayer{T}, dout) where T = backward(lyr, T(dout))" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "1.1" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "apple = 100\n", "apple_num = 2\n", "tax = 1.1" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "MulLayer{Float32}(7.0f-45, 0.0f0)" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "mul_apple_layer = MulLayer{Float32}()\n", "mul_tax_layer = MulLayer{Float32}()" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "220.0f0" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "apple_price = forward(mul_apple_layer, apple, apple_num)\n", "price = forward(mul_tax_layer, apple_price, tax)" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(2.2f0, 110.0f0, 200.0f0)" ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "dprice = 1\n", "dapple_price, dtax = backward(mul_tax_layer, dprice)\n", "dapple, dapple_num = backward(mul_apple_layer, dapple_price)\n", "(dapple, dapple_num, dtax)" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "backward (generic function with 4 methods)" ] }, "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ "mutable struct AddLayer{T<:AbstractFloat} <: AbstractLayer end\n", "\n", "function forward(self::AddLayer{T}, x::T, y::T) where {T<:AbstractFloat}\n", " out = x + y\n", " return out\n", "end\n", "@inline forward(lyr::AddLayer{T}, x, y) where T = forward(lyr, T(x), T(y))\n", " \n", "function backward(self::AddLayer{T}, dout::T) where {T<:AbstractFloat}\n", " dx = dout * 1\n", " dy = dout * 1\n", " return dx, dy\n", "end\n", "@inline backward(lyr::AddLayer{T}, dout) where T = backward(lyr, T(dout))" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "1.1" ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "apple = 100\n", "apple_num = 2\n", "orange = 150\n", "orange_num = 3\n", "tax = 1.1" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "MulLayer{Float32}(1.160496f20, 4.5761f-41)" ] }, "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ "mul_apple_layer = MulLayer{Float32}()\n", "mul_orange_layer = MulLayer{Float32}()\n", "add_apple_orange_layer = AddLayer{Float32}()\n", "mul_tax_layer = MulLayer{Float32}()" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "715.0f0" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "apple_price = forward(mul_apple_layer, apple, apple_num)\n", "orange_price = forward(mul_orange_layer, orange, orange_num)\n", "all_price = forward(add_apple_orange_layer, apple_price, orange_price)\n", "price = forward(mul_tax_layer, all_price, tax)" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(2.2f0, 110.0f0, 3.3000002f0, 165.0f0, 650.0f0)" ] }, "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ "dprice = 1\n", "dall_price, dtax = backward(mul_tax_layer, dprice)\n", "dapple_price, dorange_price = backward(add_apple_orange_layer, dall_price)\n", "dorange, dorange_num = backward(mul_orange_layer, dorange_price)\n", "dapple, dapple_num = backward(mul_apple_layer, dapple_price)\n", "(dapple, dapple_num, dorange, dorange_num, dtax)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Relu" ] }, { "cell_type": "code", "execution_count": 14, "metadata": { "collapsed": true }, "outputs": [], "source": [ "mutable struct ReluLayer{T<:AbstractFloat} <: AbstractLayer\n", " mask::AbstractArray{Bool}\n", " (::Type{ReluLayer{T}})() where {T} = new{T}()\n", "end" ] }, { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "forward (generic function with 5 methods)" ] }, "execution_count": 15, "metadata": {}, "output_type": "execute_result" } ], "source": [ "function forward(self::ReluLayer{T}, x::AbstractArray{T}) where {T<:AbstractFloat}\n", " mask = self.mask = (x .<= 0)\n", " out = copy(x)\n", " out[mask] .= zero(T)\n", " out\n", "end" ] }, { "cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "backward (generic function with 5 methods)" ] }, "execution_count": 16, "metadata": {}, "output_type": "execute_result" } ], "source": [ "function backward(self::ReluLayer{T}, dout::AbstractArray{T}) where {T<:AbstractFloat}\n", " dout[self.mask] .= zero(T)\n", " dout\n", "end" ] }, { "cell_type": "code", "execution_count": 17, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "2×2 Array{Float32,2}:\n", " 1.0 0.0\n", " 0.0 3.0" ] }, "execution_count": 17, "metadata": {}, "output_type": "execute_result" } ], "source": [ "relulyr = ReluLayer{Float32}()\n", "forward(relulyr, Float32[1.0 -0.5; -2.0 3.0])" ] }, { "cell_type": "code", "execution_count": 18, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "2×2 Array{Bool,2}:\n", " false true\n", " true false" ] }, "execution_count": 18, "metadata": {}, "output_type": "execute_result" } ], "source": [ "relulyr.mask" ] }, { "cell_type": "code", "execution_count": 19, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "2×2 Array{Float32,2}:\n", " 1.0 0.0\n", " 0.0 1.0" ] }, "execution_count": 19, "metadata": {}, "output_type": "execute_result" } ], "source": [ "backward(relulyr, Float32[1.0 1.0; 1.0 1.0])" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Sigmoid" ] }, { "cell_type": "code", "execution_count": 20, "metadata": { "collapsed": true }, "outputs": [], "source": [ "mutable struct SigmoidLayer{T<:AbstractFloat} <: AbstractLayer\n", " out::AbstractArray{T}\n", " (::Type{SigmoidLayer{T}})() where {T} = new{T}()\n", "end" ] }, { "cell_type": "code", "execution_count": 21, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "forward (generic function with 6 methods)" ] }, "execution_count": 21, "metadata": {}, "output_type": "execute_result" } ], "source": [ "function forward(self::SigmoidLayer{T}, x::A) where {T<:AbstractFloat, A<:AbstractArray{T}}\n", " self.out = one(T) ./ (one(T) .+ exp.(.-x))\n", "end" ] }, { "cell_type": "code", "execution_count": 22, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "backward (generic function with 6 methods)" ] }, "execution_count": 22, "metadata": {}, "output_type": "execute_result" } ], "source": [ "function backward(self::SigmoidLayer{T}, dout::A) where {T<:AbstractFloat, A<:AbstractArray{T}}\n", " dout .* (one(T) .- self.out) .* self.out\n", "end" ] }, { "cell_type": "code", "execution_count": 23, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "11-element Array{Float32,1}:\n", " 0.00669285\n", " 0.0179862 \n", " 0.0474259 \n", " 0.119203 \n", " 0.268941 \n", " 0.5 \n", " 0.731059 \n", " 0.880797 \n", " 0.952574 \n", " 0.982014 \n", " 0.993307 " ] }, "execution_count": 23, "metadata": {}, "output_type": "execute_result" } ], "source": [ "sigmoidlyr = SigmoidLayer{Float32}()\n", "forward(sigmoidlyr, Float32[-5:5;])" ] }, { "cell_type": "code", "execution_count": 24, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "11-element Array{Float32,1}:\n", " 0.00664806\n", " 0.0176627 \n", " 0.0451767 \n", " 0.104994 \n", " 0.196612 \n", " 0.25 \n", " 0.196612 \n", " 0.104994 \n", " 0.0451767 \n", " 0.0176627 \n", " 0.00664803" ] }, "execution_count": 24, "metadata": {}, "output_type": "execute_result" } ], "source": [ "backward(sigmoidlyr, ones(Float32,11))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### 5.6.2 バッチ版 Affine レイヤ" ] }, { "cell_type": "code", "execution_count": 25, "metadata": { "collapsed": true }, "outputs": [], "source": [ "mutable struct AffineLayer{T<:AbstractFloat} <: AbstractLayer\n", " W::AbstractMatrix{T}\n", " b::AbstractVector{T}\n", " x::AbstractArray{T}\n", " dW::AbstractMatrix{T}\n", " db::AbstractVector{T}\n", " function (::Type{AffineLayer})(W::AbstractMatrix{T}, b::AbstractVector{T}) where {T}\n", " lyr = new{T}()\n", " lyr.W = W\n", " lyr.b = b\n", " lyr\n", " end\n", "end" ] }, { "cell_type": "code", "execution_count": 26, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "forward (generic function with 7 methods)" ] }, "execution_count": 26, "metadata": {}, "output_type": "execute_result" } ], "source": [ "function forward(self::AffineLayer{T}, x::A) where {T<:AbstractFloat, A<:AbstractArray{T}}\n", " self.x = x\n", " self.W * x .+ self.b\n", "end" ] }, { "cell_type": "code", "execution_count": 27, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "_sumvec (generic function with 3 methods)" ] }, "execution_count": 27, "metadata": {}, "output_type": "execute_result" } ], "source": [ "function backward(self::AffineLayer{T}, dout::A) where {T<:AbstractFloat, A<:AbstractArray{T}}\n", " dx = self.W' * dout\n", " self.dW = dout * self.x'\n", " self.db = _sumvec(dout)\n", " dx\n", "end\n", "@inline _sumvec{T}(dout::AbstractVector{T}) = dout\n", "@inline _sumvec{T}(dout::AbstractMatrix{T}) = vec(mapslices(sum, dout, 2))\n", "@inline _sumvec{T,N}(dout::AbstractArray{T,N}) = vec(mapslices(sum, dout, 2:N))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### 5.6.3 Softmax-with-Loss レイヤ" ] }, { "cell_type": "code", "execution_count": 28, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "softmax (generic function with 2 methods)" ] }, "execution_count": 28, "metadata": {}, "output_type": "execute_result" } ], "source": [ "function softmax(a::AbstractVector{T}) where {T<:AbstractFloat}\n", " c = maximum(a) # オーバーフロー対策\n", " exp_a = exp.(a .- c)\n", " exp_a ./ sum(exp_a)\n", "end\n", "\n", "function softmax(a::AbstractMatrix{T}) where {T<:AbstractFloat}\n", " mapslices(softmax, a, 1)\n", "end" ] }, { "cell_type": "code", "execution_count": 29, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "crossentropyerror (generic function with 3 methods)" ] }, "execution_count": 29, "metadata": {}, "output_type": "execute_result" } ], "source": [ "function crossentropyerror(y::Vector, t::Vector)\n", " δ = 1e-7 # アンダーフロー対策\n", " # -sum(t .* log.(y .+ δ))\n", " -(t ⋅ log.(y .+ δ))\n", "end\n", "function crossentropyerror(y::Matrix, t::Matrix)\n", " batch_size = size(y, 2)\n", " δ = 1e-7 # アンダーフロー対策\n", " # -sum(t .* log(y .+ δ)) / batch_size\n", " -vecdot(t, log.(y .+ δ)) / batch_size\n", "end\n", "function crossentropyerror(y::Matrix, t::Vector)\n", " batch_size = size(y, 2)\n", " δ = 1e-7 # アンダーフロー対策\n", " -sum([log.(y[t[i]+1, i]) for i=1:batch_size] .+ δ) / batch_size\n", "end" ] }, { "cell_type": "code", "execution_count": 30, "metadata": { "collapsed": true }, "outputs": [], "source": [ "mutable struct SoftmaxWithLossLayer{T<:AbstractFloat} <: AbstractLayer\n", " loss::T\n", " y::AbstractArray{T}\n", " t::AbstractArray{T}\n", " (::Type{SoftmaxWithLossLayer{T}})() where {T} = new{T}()\n", "end" ] }, { "cell_type": "code", "execution_count": 31, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "forward (generic function with 8 methods)" ] }, "execution_count": 31, "metadata": {}, "output_type": "execute_result" } ], "source": [ "function forward(self::SoftmaxWithLossLayer{T}, x::AbstractArray{T}, t::AbstractArray{T}) where {T<:AbstractFloat}\n", " self.t = t\n", " y = self.y = softmax(x)\n", " self.loss = crossentropyerror(y, t)\n", "end" ] }, { "cell_type": "code", "execution_count": 32, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "_swlvec (generic function with 2 methods)" ] }, "execution_count": 32, "metadata": {}, "output_type": "execute_result" } ], "source": [ "function backward(lyr::SoftmaxWithLossLayer{T}, dout::T=one(T)) where {T<:AbstractFloat}\n", " dout .* _swlvec(lyr.y, lyr.t)\n", "end\n", "@inline _swlvec(y::AbstractArray{T}, t::AbstractVector{T}) where {T<:AbstractFloat} = y .- t\n", "@inline _swlvec(y::AbstractArray{T}, t::AbstractMatrix{T}) where {T<:AbstractFloat} = (y .- t) / size(t)[2]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### 5.7.2 誤差逆伝播法に対応したニューラルネットワークの実装" ] }, { "cell_type": "code", "execution_count": 33, "metadata": { "collapsed": true }, "outputs": [], "source": [ "mutable struct TwoLayerNet{T<:AbstractFloat}\n", " a1lyr::AffineLayer{T}\n", " relu1lyr::ReluLayer\n", " a2lyr::AffineLayer{T}\n", " softmaxlyr::SoftmaxWithLossLayer{T}\n", "end\n", "\n", "function (::Type{TwoLayerNet{T}})(input_size::Int, hidden_size::Int, output_size::Int,\n", " weight_init_std::T=T(0.01)) where {T<:AbstractFloat}\n", " W1 = weight_init_std .* randn(T, hidden_size, input_size)\n", " b1 = zeros(T, hidden_size)\n", " W2 = weight_init_std .* randn(T, output_size, hidden_size)\n", " b2 = zeros(T, output_size)\n", " a1lyr = AffineLayer(W1, b1)\n", " relu1lyr = ReluLayer{T}()\n", " a2lyr = AffineLayer(W2, b2)\n", " softmaxlyr = SoftmaxWithLossLayer{T}()\n", " TwoLayerNet(a1lyr, relu1lyr, a2lyr, softmaxlyr)\n", "end" ] }, { "cell_type": "code", "execution_count": 34, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "predict (generic function with 1 method)" ] }, "execution_count": 34, "metadata": {}, "output_type": "execute_result" } ], "source": [ "function predict(net::TwoLayerNet{T}, x::AbstractArray{T}) where {T<:AbstractFloat}\n", " a1 = forward(net.a1lyr, x)\n", " z1 = forward(net.relu1lyr, a1)\n", " a2 = forward(net.a2lyr, z1)\n", " # softmax(a2)\n", " a2\n", "end" ] }, { "cell_type": "code", "execution_count": 35, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "loss (generic function with 1 method)" ] }, "execution_count": 35, "metadata": {}, "output_type": "execute_result" } ], "source": [ "function loss(net::TwoLayerNet{T}, x::AbstractArray{T}, t::AbstractArray{T}) where {T<:AbstractFloat}\n", " y = predict(net, x)\n", " forward(net.softmaxlyr, y, t)\n", "end" ] }, { "cell_type": "code", "execution_count": 36, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "accuracy (generic function with 1 method)" ] }, "execution_count": 36, "metadata": {}, "output_type": "execute_result" } ], "source": [ "function accuracy(net::TwoLayerNet{T}, x::AbstractArray{T}, t::AbstractArray{T}) where {T<:AbstractFloat}\n", " y = vec(mapslices(indmax, predict(net, x), 1))\n", " if ndims(t) > 1 t = vec(mapslices(indmax, t, 1)) end\n", " mean(y .== t)\n", "end" ] }, { "cell_type": "code", "execution_count": 37, "metadata": { "collapsed": true }, "outputs": [], "source": [ "struct TwoLayerNetGrads{T}\n", " W1::AbstractMatrix{T}\n", " b1::AbstractVector{T}\n", " W2::AbstractMatrix{T}\n", " b2::AbstractVector{T}\n", "end" ] }, { "cell_type": "code", "execution_count": 38, "metadata": { "collapsed": true }, "outputs": [], "source": [ "function Base.gradient(net::TwoLayerNet{T}, x::AbstractArray{T}, t::AbstractArray{T}) where {T<:AbstractFloat}\n", " # forward\n", " loss(net, x, t)\n", " # backward\n", " dout = one(T)\n", " dz2 = backward(net.softmaxlyr, dout)\n", " da2 = backward(net.a2lyr, dz2)\n", " dz1 = backward(net.relu1lyr, da2)\n", " da1 = backward(net.a1lyr, dz1)\n", " TwoLayerNetGrads(net.a1lyr.dW, net.a1lyr.db, net.a2lyr.dW, net.a2lyr.db)\n", "end" ] }, { "cell_type": "code", "execution_count": 39, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "applygradient! (generic function with 1 method)" ] }, "execution_count": 39, "metadata": {}, "output_type": "execute_result" } ], "source": [ "function applygradient!(net::TwoLayerNet{T}, grads::TwoLayerNetGrads{T}, learning_rate::T) where {T<:AbstractFloat}\n", " net.a1lyr.W -= learning_rate .* grads.W1\n", " net.a1lyr.b -= learning_rate .* grads.b1\n", " net.a2lyr.W -= learning_rate .* grads.W2\n", " net.a2lyr.b -= learning_rate .* grads.b2\n", "end" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### 5.7.3 誤差逆伝播法の勾配確認" ] }, { "cell_type": "code", "execution_count": 40, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "LoadMnist" ] }, "execution_count": 40, "metadata": {}, "output_type": "execute_result" } ], "source": [ "include(\"dataset/load_mnist.jl\")" ] }, { "cell_type": "code", "execution_count": 41, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "((Float32[0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0; … ; 0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0], UInt8[0x00 0x01 … 0x00 0x00; 0x00 0x00 … 0x00 0x00; … ; 0x00 0x00 … 0x00 0x01; 0x00 0x00 … 0x00 0x00]), (Float32[0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0; … ; 0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0], UInt8[0x00 0x00 … 0x00 0x00; 0x00 0x00 … 0x00 0x00; … ; 0x00 0x00 … 0x00 0x00; 0x00 0x00 … 0x00 0x00]))" ] }, "execution_count": 41, "metadata": {}, "output_type": "execute_result" } ], "source": [ "(x_train, t_train), (x_test, t_test) = LoadMnist.load_mnist(;\n", " normalize=true, flatten=true, one_hot_label=true)" ] }, { "cell_type": "code", "execution_count": 42, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(10, 60000)" ] }, "execution_count": 42, "metadata": {}, "output_type": "execute_result" } ], "source": [ "size(t_train)" ] }, { "cell_type": "code", "execution_count": 43, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "TwoLayerNet{Float32}(AffineLayer{Float32}(Float32[0.00318988 0.00485958 … 0.00432392 -0.00266472; 0.00109636 0.00661024 … 0.0103529 0.00247615; … ; -0.000739207 0.0207509 … -6.73199f-5 0.0100186; 0.011015 0.0093262 … 0.00189109 -0.0164121], Float32[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 … 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], #undef, #undef, #undef), ReluLayer{Float32}(#undef), AffineLayer{Float32}(Float32[0.00970453 -0.00344387 … -0.00698218 0.00117947; 0.00413081 0.000712915 … -0.00246677 -0.00750007; … ; 0.00190007 -0.00820374 … 0.00863559 -0.0117912; -0.0139849 0.00173204 … -0.000736556 0.00177398], Float32[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], #undef, #undef, #undef), SoftmaxWithLossLayer{Float32}(7.6892265f20, #undef, #undef))" ] }, "execution_count": 43, "metadata": {}, "output_type": "execute_result" } ], "source": [ "network = TwoLayerNet{Float32}(784, 50, 10)" ] }, { "cell_type": "code", "execution_count": 44, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "784×3 Array{Float32,2}:\n", " 0.0 0.0 0.0\n", " 0.0 0.0 0.0\n", " 0.0 0.0 0.0\n", " 0.0 0.0 0.0\n", " 0.0 0.0 0.0\n", " 0.0 0.0 0.0\n", " 0.0 0.0 0.0\n", " 0.0 0.0 0.0\n", " 0.0 0.0 0.0\n", " 0.0 0.0 0.0\n", " 0.0 0.0 0.0\n", " 0.0 0.0 0.0\n", " 0.0 0.0 0.0\n", " ⋮ \n", " 0.0 0.0 0.0\n", " 0.0 0.0 0.0\n", " 0.0 0.0 0.0\n", " 0.0 0.0 0.0\n", " 0.0 0.0 0.0\n", " 0.0 0.0 0.0\n", " 0.0 0.0 0.0\n", " 0.0 0.0 0.0\n", " 0.0 0.0 0.0\n", " 0.0 0.0 0.0\n", " 0.0 0.0 0.0\n", " 0.0 0.0 0.0" ] }, "execution_count": 44, "metadata": {}, "output_type": "execute_result" } ], "source": [ "x_batch = x_train[:, 1:3]" ] }, { "cell_type": "code", "execution_count": 45, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "10×3 Array{Float32,2}:\n", " 0.0 1.0 0.0\n", " 0.0 0.0 0.0\n", " 0.0 0.0 0.0\n", " 0.0 0.0 0.0\n", " 0.0 0.0 1.0\n", " 1.0 0.0 0.0\n", " 0.0 0.0 0.0\n", " 0.0 0.0 0.0\n", " 0.0 0.0 0.0\n", " 0.0 0.0 0.0" ] }, "execution_count": 45, "metadata": {}, "output_type": "execute_result" } ], "source": [ "t_batch = Matrix{Float32}(t_train[:, 1:3])" ] }, { "cell_type": "code", "execution_count": 46, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "10×3 Array{Float32,2}:\n", " -0.0011316 -0.0014604 -0.000707305\n", " -0.00289681 0.0045625 0.00177277 \n", " -0.000804678 -0.00404845 0.00219935 \n", " 0.00101501 0.00697281 0.000847883\n", " -0.00777562 -0.00674609 -0.00108261 \n", " -0.00570799 -0.0101445 -0.000680426\n", " -0.00433399 -0.00503931 -0.000889094\n", " 0.00661214 0.00467918 0.00023113 \n", " -0.00730404 -0.00778527 0.000421932\n", " 0.00122218 -0.00650803 -0.00211031 " ] }, "execution_count": 46, "metadata": {}, "output_type": "execute_result" } ], "source": [ "predict(network, x_batch)" ] }, { "cell_type": "code", "execution_count": 47, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "2.303789029815617" ] }, "execution_count": 47, "metadata": {}, "output_type": "execute_result" } ], "source": [ "loss(network, x_batch, t_batch)\n", "# loss(network, x_batch, Matrix{Float32}(t_batch))" ] }, { "cell_type": "code", "execution_count": 48, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "TwoLayerNetGrads{Float32}(Float32[0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0; … ; 0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0], Float32[-0.00222329, 0.00170545, -0.00132295, 0.0, -0.00491989, 0.0, 0.00502056, 0.0, 0.00248478, 0.00177194 … -0.00875813, 0.00109634, 0.00871906, -0.00546616, 0.000614705, -0.000547221, 0.00571311, 0.00147637, -0.00292858, 0.0], Float32[0.00182903 -0.0109375 … -0.0491178 0.0; 0.00183358 0.00400405 … 0.0101247 0.0; … ; 0.0018311 0.0039746 … 0.0100359 0.0; 0.00182647 0.00399781 … 0.0100695 0.0], Float32[-0.233289, 0.10027, 0.100066, 0.100451, -0.233698, -0.233729, 0.0998127, 0.100541, 0.0996665, 0.0999085])" ] }, "execution_count": 48, "metadata": {}, "output_type": "execute_result" } ], "source": [ "grad_backprop = gradient(network, x_batch, t_batch)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### 5.7.4 誤差逆伝播法を使った学習" ] }, { "cell_type": "code", "execution_count": 49, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "TwoLayerNet{Float32}(AffineLayer{Float32}(Float32[-0.0145858 0.0112641 … -0.00181048 -0.00982263; -0.00418988 0.000436219 … 0.0169963 -0.00666518; … ; -0.00234774 -0.0183891 … -0.0095593 -0.00152059; -0.0075102 0.00935047 … 0.000530805 -0.0067455], Float32[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 … 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], #undef, #undef, #undef), ReluLayer{Float32}(#undef), AffineLayer{Float32}(Float32[-0.00512687 -0.0145102 … -0.00207356 0.0202991; -0.00433898 0.0068244 … 0.00575347 -0.0205735; … ; -0.0189138 0.0150627 … -0.0172908 0.000555763; 0.00474322 -0.0054643 … -0.00044954 -0.0116008], Float32[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], #undef, #undef, #undef), SoftmaxWithLossLayer{Float32}(7.6892265f20, #undef, #undef))" ] }, "execution_count": 49, "metadata": {}, "output_type": "execute_result" } ], "source": [ "network = TwoLayerNet{Float32}(784, 50, 10)" ] }, { "cell_type": "code", "execution_count": 50, "metadata": { "collapsed": true }, "outputs": [], "source": [ "iters_num = 10000;\n", "train_size = size(x_train, 2); # => 60000\n", "batch_size = 100;\n", "learning_rate = Float32(0.1);" ] }, { "cell_type": "code", "execution_count": 51, "metadata": { "collapsed": true }, "outputs": [], "source": [ "train_loss_list = Float32[];\n", "train_acc_list = Float32[];\n", "test_acc_list = Float32[];" ] }, { "cell_type": "code", "execution_count": 52, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "600" ] }, "execution_count": 52, "metadata": {}, "output_type": "execute_result" } ], "source": [ "iter_per_epoch = max(train_size ÷ batch_size, 1)" ] }, { "cell_type": "code", "execution_count": 53, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "0: train_acc=0.15588333333333335 / test_acc=0.1581\n", "600: train_acc=0.9048833333333334 / test_acc=0.9073\n", "1200: train_acc=0.92345 / test_acc=0.925\n", "1800: train_acc=0.9356833333333333 / test_acc=0.9366\n", "2400: train_acc=0.9428166666666666 / test_acc=0.9426\n", "3000: train_acc=0.951 / test_acc=0.9468\n", "3600: train_acc=0.9546666666666667 / test_acc=0.9517\n", "4200: train_acc=0.9602 / test_acc=0.9552\n", "4800: train_acc=0.9620666666666666 / test_acc=0.9577\n", "5400: train_acc=0.96585 / test_acc=0.9614\n", "6000: train_acc=0.9685833333333334 / test_acc=0.9629\n", "6600: train_acc=0.9716166666666667 / test_acc=0.9634\n", "7200: train_acc=0.9726833333333333 / test_acc=0.9635\n", "7800: train_acc=0.9729 / test_acc=0.965\n", "8400: train_acc=0.9748666666666667 / test_acc=0.9662\n", "9000: train_acc=0.9767666666666667 / test_acc=0.9692\n", "9600: train_acc=0.9782333333333333 / test_acc=0.9665\n" ] } ], "source": [ "for i = 1:iters_num\n", " batch_mask = rand(1:train_size, batch_size)\n", " x_batch = x_train[:, batch_mask]\n", " t_batch = Matrix{Float32}(t_train[:, batch_mask])\n", " \n", " # 誤差逆伝播法によって勾配を求める\n", " grads = gradient(network, x_batch, t_batch)\n", " \n", " # 更新\n", " applygradient!(network, grads, learning_rate)\n", " \n", " _loss = loss(network, x_batch, t_batch)\n", " push!(train_loss_list, _loss)\n", "\n", " if i % iter_per_epoch == 1\n", " train_acc = accuracy(network, x_train, Matrix{Float32}(t_train))\n", " test_acc = accuracy(network, x_test, Matrix{Float32}(t_test))\n", " push!(train_acc_list, train_acc)\n", " push!(test_acc_list, test_acc)\n", " println(\"$(i-1): train_acc=$(train_acc) / test_acc=$(test_acc)\")\n", " end\n", "end" ] }, { "cell_type": "code", "execution_count": 54, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "final: train_acc=0.9794666666666667 / test_acc=0.9689\n" ] } ], "source": [ "final_train_acc = accuracy(network, x_train, Matrix{Float32}(t_train))\n", "final_test_acc = accuracy(network, x_test, Matrix{Float32}(t_test))\n", "push!(train_acc_list, final_train_acc)\n", "push!(test_acc_list, final_test_acc)\n", "println(\"final: train_acc=$(final_train_acc) / test_acc=$(final_test_acc)\")" ] }, { "cell_type": "code", "execution_count": 55, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "PyPlot" ] }, "execution_count": 55, "metadata": {}, "output_type": "execute_result" } ], "source": [ "using PyPlot\n", "const plt = PyPlot" ] }, { "cell_type": "code", "execution_count": 56, "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAicAAAGgCAYAAABxDccgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAPYQAAD2EBqD+naQAAIABJREFUeJzt3Xd0VGXixvFnkpBAgAQpKUCAKEWaVMGABRRFRFfWXVdXV7HuT8VVxBVl7RXWglhQVFTsiiK6CtJCh9AJPaG3kELNhPRk7u+PJMNMMpPMhCR3Qr6fc3JO5s57733noszDWy2GYRgCAADwEX5mVwAAAMAR4QQAAPgUwgkAAPAphBMAAOBTCCcAAMCnEE4AAIBPIZwAAACfQjgBAAA+hXACAAB8CuEEAAD4FMIJAADwKQFmV8ATNptNR44cUePGjWWxWMyuDgAA8IBhGMrIyFDLli3l5+d5e0itCCdHjhxRVFSU2dUAAACVcOjQIbVu3drj8rUinDRu3FhS0YcLCQkxuTYAAMATVqtVUVFR9u9xT9WKcFLSlRMSEkI4AQCglvF2SAYDYgEAgE8hnAAAAJ9COAEAAD6FcAIAAHwK4QQAAPgUwgkAAPAphBMAAOBTCCcAAMCnEE4AAIBPIZwAAACfQjgBAAA+hXACAAB8Sp0OJ7vTMtTuqVnadiTd7KoAAIBidTqcDJm4VJI0/N3lJtcEAACUqLPhxDAMDekcbn+9+fApE2sDAABK1NlwYrFY9MmdfeyvtyZZTawNAAAoUWfDiVQUUO67NFqSlJhCOAEAwBfU6XAiSR3CG0mSvog7YHJNAACARDjRuv0nza4CAABwUOfDicVy5vdCm2FeRQAAgCTCiR4c1N7+++GTWSbWBAAASIQTRTdvaP89J99mYk0AAIBEOJEktQytL0nKyS80uSYAAIBwIql+PX9JUm4BLScAAJiNcCIpMKDoMdByAgCA+QgnouUEAABfQjiRlFccSqzZ+SbXBAAAEE4kbU8uWrr+tdk7TK4JAAAgnDg4nplndhUAAKjzCCeSGgUFSJKu6x5hck0AAADhRNJf+7SW5LwgGwAAMAfhRNLutNOSpMmL9phcEwAAQDiRtHz3MbOrAAAAihFOAACATyGcAAAAn0I4AQAAPoVwAgAAfArhBAAA+BTCCQAA8CmEE0mjBl9gdhUAAEAxwomkyzq0MLsKAACgGOFE0s7UDLOrAAAAihFOJKWk55hdBQAAUIxwImlEr1ZmVwEAABQjnEhqUM9fkhQc6G9yTQAAAOFEkr+fRZJUYDNMrgkAACCc6Ew4KSScAABgOsKJnMOJYRBQAAAwE+FEkr/FYv+dxhMAAMxFOJHk738mnBTYbCbWBAAAEE4kBfg5tJyQTQAAMBXhRJKfhZYTAAB8BeFEzi0nzNgBAMBchBOdma0jEU4AADCbV+Fk/Pjxuvjii9W4cWOFhYVpxIgRSkxMrPC8xYsXq3fv3goKClL79u01bdq0yta3WlgsFpXkE8IJAADm8iqcLFmyRKNGjdKqVas0f/585efn65prrlFmZqbbc/bt26fhw4dr8ODBio+P1+jRo3Xfffdp7ty5Z135qhTgV/QoWCUWAABzBXhTeM6cOU6vp02bprCwMK1fv16XX365y3OmTJmi6OhovfXWW5Kkzp07a/ny5Xr77bc1dOjQSla76vn7WaRCWk4AADDbWY05SU9PlyQ1bdrUbZm4uDgNGTLE6djQoUMVFxfn9pzc3FxZrVann+rGEvYAAPiGSocTm82m0aNHa+DAgerWrZvbcikpKQoPD3c6Fh4eLqvVquzsbJfnjB8/XqGhofafqKioylbTY2z+BwCAb6h0OBk1apS2bt2q77//virrI0kaN26c0tPT7T+HDh2q8nuURssJAAC+wasxJyUefvhh/f7771q6dKlat25dbtmIiAilpqY6HUtNTVVISIgaNGjg8pygoCAFBQVVpmqVRjgBAMA3eNVyYhiGHn74Yc2cOVMLFy5UdHR0hefExMQoNjbW6dj8+fMVExPjXU2rWQDhBAAAn+BVOBk1apS+/vprffvtt2rcuLFSUlKUkpLiNHZk3LhxuvPOO+2vH3jgAe3du1djx45VQkKCPvjgA02fPl2PPfZY1X2KKlCyhD3L1wMAYC6vwsmHH36o9PR0DRo0SJGRkfafH374wV4mOTlZBw8etL+Ojo7WrFmzNH/+fPXo0UNvvfWWpk6d6lPTiCUpoHhnYptBywkAAGbyasyJ4cEXt6vVXwcNGqSNGzd6c6saZ5+tU0g4AQDATOytU8zfwpgTAAB8AeGkmH22Dt06AACYinBSrGTMCYuwAQBgLsJJsa1JRUvkJ6ZkmFwTAADqNsJJKRP+SDC7CgAA1GmEEwAA4FMIJwAAwKcQTgAAgE8hnJRyRccWZlcBAIA6jXBSbFi3CEnS4E6EEwAAzEQ4KVayCBurnAAAYC7CSSksEAsAgLkIJ8UsxXvrAAAAcxFOipVEExpOAAAwF+GkWEnDiUG/DgAApiKcAAAAn0I4KbZu/0lJ0qq9J0yuCQAAdRvhpFjSqWxJ0oIdqSbXBACAuo1wAgAAfArhBAAA+BTCSbGru4RLkq7rHmFyTQAAqNsIJ8W6RIZIkpo2DDS5JgAA1G2Ek2J+xQudFNpMrggAAHUc4aSYf/GTYBE2AADMRTgpVrK3jo1wAgCAqQgnxejWAQDANxBOitGtAwCAbyCcFLO3nBBOAAAwFeGk2JkxJyZXBACAOo5wUsy/KJswIBYAAJMRTor5+RW3nNB0AgCAqQgnxfyYSgwAgE8gnBTzY8wJAAA+gXBSzK9kzAnpBAAAUxFOitnHnNCtAwCAqQgnxc6sc2JyRQAAqOMIJ8VYIRYAAN9AOCnGbB0AAHwD4aSYxb7xH+EEAAAzEU6K+ReHk1V7T5hcEwAA6jbCSbG9R0+bXQUAACDCiV2KNcfsKgAAABFOAACAjyGcFCsecgIAAExGOClmEekEAABfQDgBAAA+hXBSjG4dAAB8A+GkGNkEAADfQDgpZqHpBAAAn0A4AQAAPoVwUoyGEwAAfAPhpNhferc2uwoAAECEE7tmjQIlSfX8aUIBAMBMhBMAAOBTCCfFSlaINQyTKwIAQB1HOAEAAD6FcFKsZLYODScAAJiLcAIAAHwK4aRYyRwdg0EnAACYinBSghnEAAD4BMJJKTYaTgAAMBXhxIX8QpvZVQAAoM7yOpwsXbpUN9xwg1q2bCmLxaJffvml3PKLFy+WxWIp85OSklLpSlcHP4fNdY5m5JpYEwAA6javw0lmZqZ69OihyZMne3VeYmKikpOT7T9hYWHe3rpaNW8UZP89gCXsAQAwTYC3JwwbNkzDhg3z+kZhYWFq0qSJ1+fVpAA/iwpsBqvEAgBgohobc9KzZ09FRkbq6quv1ooVK8otm5ubK6vV6vRTE0q6dmykEwAATFPt4SQyMlJTpkzRjBkzNGPGDEVFRWnQoEHasGGD23PGjx+v0NBQ+09UVFR1V1PSmVVimbEDAIB5vO7W8VanTp3UqVMn++sBAwZoz549evvtt/XVV1+5PGfcuHEaM2aM/bXVaq2RgGIPJ6QTAABMU+3hxJV+/fpp+fLlbt8PCgpSUFCQ2/eri+OMHQAAYA5T1jmJj49XZGSkGbcuF2NOAAAwn9ctJ6dPn9bu3bvtr/ft26f4+Hg1bdpUbdq00bhx45SUlKQvv/xSkjRp0iRFR0era9euysnJ0dSpU7Vw4ULNmzev6j5FFWHMCQAA5vM6nKxbt06DBw+2vy4ZGzJy5EhNmzZNycnJOnjwoP39vLw8Pf7440pKSlJwcLAuuugiLViwwOkavoKWEwAAzGcxasE2vFarVaGhoUpPT1dISEi13afnS/N0KitfC8ZcrvZhjavtPgAA1AWV/f5mbx0HJS0nvh/XAAA4dxFOHJzIzJMkxe09bnJNAACouwgnLrz023azqwAAQJ1FOHGBAbEAAJiHcOIC0QQAAPMQTlyg4QQAAPMQTgAAgE8hnAAAAJ9COAEAAD6FcAIAAHwK4QQAAPgUwgkAAPAphBMAAOBTCCcAAMCnEE4AAIBPIZwAAACfQjgBAAA+hXACAAB8CuEEAAD4FMIJAADwKYQTAADgUwgnLnSJDDG7CgAA1FmEEwfXdY+QJF3TNdzkmgAAUHcRThw0bRgoSTIMkysCAEAdRjhxYM0ukCTF7Tluck0AAKi7CCcO/rfpiCRpzf4TJtcEAIC6i3ACAAB8CuEEAAD4FMIJAADwKYQTAADgUwgnAADApxBOAACATyGcAAAAn0I4AQAAPoVw4iAytL7ZVQAAoM4jnDj4z3WdJUlRTRuYXBMAAOouwomD4EB/SdKpzHyTawIAQN1FOHGQnl0USjJyC0yuCQAAdRfhxEFaRq79d5vNMLEmAADUXYQTB4H+Zx5Hvs1mYk0AAKi7CCcOAgMcwkkhLScAAJiBcOLAKZwU0HICAIAZCCcO/C0W++82g5YTAADMQDhx4OfwNIgmAACYg3DiwCJaTgAAMBvhxIFDr47IJgAAmINw4gbhBAAAcxBOHFgYEAsAgOkIJw4cenUYEAsAgEkIJw4cx5ywfD0AAOYgnDhwnK1Drw4AAOYgnDhwmq1Dxw4AAKYgnDhwHHNCrw4AAOYgnDhwbDnJzC0wryIAANRhhBMnZ9LJtJX7zasGAAB1GOHEgR8tJwAAmI5w4oBF2AAAMB/hxIEfe+sAAGA6wokDxwGxAADAHIQTB06LsJlYDwAA6jLCiYNebZrYf6dbBwAAcxBOHDQJDnR4RToBAMAMhBM3aDkBAMAcXoeTpUuX6oYbblDLli1lsVj0yy+/VHjO4sWL1bt3bwUFBal9+/aaNm1aZeoKAADqAK/DSWZmpnr06KHJkyd7VH7fvn0aPny4Bg8erPj4eI0ePVr33Xef5s6d63VlaxINJwAAmCPA2xOGDRumYcOGeVx+ypQpio6O1ltvvSVJ6ty5s5YvX663335bQ4cOdXlObm6ucnNz7a+tVqu31TxrO5Jr/p4AAKAGxpzExcVpyJAhTseGDh2quLg4t+eMHz9eoaGh9p+oqKjqrmYZyek5NX5PAABQA+EkJSVF4eHhTsfCw8NltVqVnZ3t8pxx48YpPT3d/nPo0KHqriYAAPARXnfr1ISgoCAFBQWZXQ0AAGCCam85iYiIUGpqqtOx1NRUhYSEqEGDBtV9ewAAUMtUeziJiYlRbGys07H58+crJiamum8NAABqIa/DyenTpxUfH6/4+HhJRVOF4+PjdfDgQUlF40XuvPNOe/kHHnhAe/fu1dixY5WQkKAPPvhA06dP12OPPVZFHwEAAJxLvA4n69atU69evdSrVy9J0pgxY9SrVy8999xzkqTk5GR7UJGk6OhozZo1S/Pnz1ePHj301ltvaerUqW6nEQMAgLrNYhi+v1C71WpVaGio0tPTFRISUq33avfULPvv+ycMr9Z7AQBwLqvs9zd76wAAAJ9COAEAAD6FcAIAAHwK4QQAAPgUwgkAAPAphBMAAOBTCCcAAMCnEE7KkZaRY3YVAACocwgn5ej3aqyy8grMrgYAAHUK4aQCiSkZZlcBAIA6hXBSAZ9f2x8AgHMM4QQAAPgUwkkFLGZXAACAOoZwUgG6dQAAqFmEEwAA4FMIJ6X0atPE7CoAAFCnEU5KYYwJAADmIpyU0rxRkNlVAACgTiOclHLvpdFmVwEAgDqNcFJK/Xr+Tq8NpusAAFCjCCcAAMCnEE5KsTAiFgAAUxFOSqEbBwAAcxFOKmCQVgAAqFGEkwrkFdjMrgIAAHUK4QQAAPgUwklFGCALAECNIpwAAACfQjgBAAA+hXBSESbrAABQowgnFViYkGZ2FQAAqFMIJxWYunyf2VUAAKBOIZyU0qVliNlVAACgTiOclFLPn0cCAICZ+CYGAAA+hXDiga1J6WZXAQCAOoNw4oGbPlxpdhUAAKgzCCceYPM/AABqDuEEAAD4FMIJAADwKYQTD+XkF1bqvPxCuoQAAPAG4cRDd366xutzXpu9Q52e+UO7UjOqoUYAAJybCCceWrP/hAzDu10AP166VzZDenvBzmqqFQAA5x7CiRce+T7e7CoAAHDOI5x44bdNR8yuAgAA5zzCCQAA8CmEk0pKTMnQ5yv2qcCD2TgWWWqgRgAAnBsCzK5AbTV00lL773cPjC63rCHvBtICAFCX0XJylrawKSAAAFWKcOLCk9deaHYVAACoswgnLtxzabsqvR5jTgAA8BzhxIWgAH+zqwAAQJ1FOKkBDIgFAMBzhBMAAOBTCCc1oLwxJ56skwIAQF1COPFSdl6h0+us3EIlplRu1+GPluxRp2fnaMPBk07HC210AwEA6i7CiRttmwW7PG7NyXd6PWdbioZOWqoVu495fY/xfySo0GbomZlb7ce2HE5Xt+fnauqyvV5fDwCAcwHhxI0OYY1cHu//WqwMo2zLxu+bk5WRk+/yvYo4njFu5mZl5xfqlVk7vL4OAADnApavd6Npw0C37206XHZV2O/WHNR3aw7qpl6tFNmkvrq3auLxvSoTaAAAOFcRTtz415UdNH3dYZfvlR534ujnjUlVWo/jp3P1+Yr9urlva7Vt1rBKrw0AgC+iW8eN0OB6bt+rzgGrpRtRHpu+Se8v2q0/f7Cy2u4JAIAvIZy4Ud6C8zW5qNrafSckSScy82rsngAAmKlS4WTy5Mlq166d6tevr/79+2vNmjVuyy5evFgWi6XMT0pKSqUrXRP8LO7jydifNnt3MS+21il9W1aXBQDUNV6Hkx9++EFjxozR888/rw0bNqhHjx4aOnSo0tLSyj0vMTFRycnJ9p+wsLBKV7omlBdOktNzvLuYF/niXBgbeyIzTz+uO6SsvAKzqwIAqIW8DicTJ07U/fffr7vvvltdunTRlClTFBwcrM8++6zc88LCwhQREWH/8fPz7R6lcrJJpdlshrYfseqvH67U7C3J9uPuAsnEeYnKyfduBdmsvALd8elqfRW3v/IVPUt3fLpaT/y0Wc//us20OgAAai+vEkJeXp7Wr1+vIUOGnLmAn5+GDBmiuLi4cs/t2bOnIiMjdfXVV2vFihXlls3NzZXVanX6qWnVEU7u/WKtrnt3mdYdOKmHvtng9N6+Y5nKL7WU/bsLd7u8zn/nJOiG95YrJ7/srKEvVh7Qsl3H9KyJwWDbkaI/L8cABgCAp7wKJ8eOHVNhYaHCw8OdjoeHh7sdQxIZGakpU6ZoxowZmjFjhqKiojRo0CBt2LDBZXlJGj9+vEJDQ+0/UVFR3lSzSpTXreO14kstSjzq8u3E1AwNfnOx7v58rUeX+3DxHm1JStdMF9OWT+fmuzgDAIDao9rXOenUqZM6depkfz1gwADt2bNHb7/9tr766iuX54wbN05jxoyxv7ZarTUeUKo0nHho+e5j6toyxOPyBT6+B49v1w4A4Ku8CifNmzeXv7+/UlNTnY6npqYqIiLC4+v069dPy5cvd/t+UFCQgoKCvKlalfOrymxShd/SGTm0jAAAzm1edesEBgaqT58+io2NtR+z2WyKjY1VTEyMx9eJj49XZGSkN7eucRYTWk6kM+M1XDEMQ91fmGd/bU4NAQCoXl5364wZM0YjR45U37591a9fP02aNEmZmZm6++67JRV1ySQlJenLL7+UJE2aNEnR0dHq2rWrcnJyNHXqVC1cuFDz5s0r7zbnFkvRgNezVboXx4xuk3E/b9HRjBx9cmdf0wIcAODc5nU4ueWWW3T06FE999xzSklJUc+ePTVnzhz7INnk5GQdPHjQXj4vL0+PP/64kpKSFBwcrIsuukgLFizQ4MGDq+5T1AKD31x81teozg0CDcPQHZ+uUYC/RZ/fdbHb4PHdmqI/24SUDHWO9Hx8DAAAnqrUgNiHH35YDz/8sMv3pk2b5vR67NixGjt2bGVug1JKRxNP2i1sNkMZOQXl7hUkSWkZuVq++5gkyZpToNAG5Zevzv2FAAB1m2+vhGayL+7pZ3YVnFSm4eTeL9aqx0vztO1IernlbA4X96S3xpMy58JqtwCAmkc4KccVHVuYXQUnldlnp2Rtla9XHayg5BmetMhYGI4LAKgmhJNapCpaImxuumNKXzs9O19jf9qklcVdPUVlaAoBAFQ/wkkFNjx7tdlVqCKGXvptu3q/Ml+p1oo3LnxrXqKmrzus26auPnOFKsomOfmFevaXrVq2y/WKuVUtPSufMTIAUIsQTirQtGHgWV9j1uaq2WPGViod7Dl6Wl/G7Xf64nXsbvlx3SGn8p+t2KdTWfn6ZOnecu9jsVh06ERWBWU8rLQLHy3Zq69WHdAdn66p/EU8tO9Ypnq8NE+3fFT+3k8AAN9R7cvXo+qUbrn4fMV+SUUtEf+8/IKiMg7jUp74aXMF1zO0NcmqjhGNPBrN4ljG03Ayc+NhtWkarD5tm9qPHTpZfvCpSj9vOCxJWnfgZI3dEwBwdggntYi7APHLxiNKTDmt67q730LAVZfMZyv26+Xft2twpxZ6eUQ3+/FCm+Fyk0Jvx5xk5xfqsR82SZLeubWnftuUrLdv6cFQWgBAuQgntYi7cLA92artyVbN2HBYF7c7z+Prfb5in6SyuyVvcNPK4NRy4mXEePT7eElFOyoDAFAexpzUIp60W6zd79vdFyez8s9qvAoA4NxHOPHA+Ju6m10FSdW7qJnh3CxSYZnKBgxfDibbjqQrbs9xs6sBAHUe4cQDLRoFmV0FSdJvm45U+tyKgs2HSyrubqnMInClVSabJKRY9f7CXcrJLzzr+5dn+LvL9fdPViklveKp1gCA6kM4qUWe+WVrtV3729VnVpD1bIXYyjl4Isvr8SrXTlqmN+ft1Luxu+zHsvMK9fTMLVq6s+rXSkk6lV3l16ytTmbmEdYA1DjCiQf8eEqSqqZbadmuY5Xu2tmSdGZ/oI+W7tE3qw/qzs+qf62UuqzXy/N1yfhYpWflm10VAHUIX7seqB/gb3YVztq6Ayeq9HozNiTpyrcWa+/R01V6XU8dOuF968bKPccqLuSh9Kx8fbh4T51pZdl7zJw/ZwB1E+HEA5ec38zsKpy1PUczPS5rcdO04dhyMmXJHu09mqmnft5ytlWrMbd9srriQh56csZm/XdOgv764coquyYAoAjhxAN+fj48xcRkuQU2U+5bFYNzS3y16oD+9P5yr84p2RcouZaNxzhwPFP/99U6bTzo21POAdRthBN4rLxAsGTnUf2+2bPZRI4NM7/GJykrr0AFhTbtTM2wLzTnbvfkqmIYhv7Ykqx9xzL17C9btflwesUnnQMe/HqD5m5L1Z8/oMUHgO9ihdg6yFDRl7O7ganu2oncDYhNz87XyOKBqZ0jQzyowZk7lKwcW+KZ4Z11a782umbiEl1yQTNN/FtP+3vLdh3TEz9u0hs39/BsRbpyLEpM04PfbHBduypuKCu0GTIMQwH+5v9bYP9xz7v3AMAs5v9tiRr36fJ9+uuUuEoNKi1t06FT2uYwi2aJiz15Sjuakev2vfcX7dbvm47oSHqOft6QVOb9H9cf1gv/26afNzq/l51XqHumrdXXqw54VO8NB065fc8wpIycfD387QbN25aiY6dzlVtQuTVWDMPQtZOW6rLXF6mgsGq6wBYnplV6sbjSO1vXNoZh6PDJLK/3eQJQu9By4qF+0U21Zl/Vzngx0/pydundleY8MyOvwKYHvl6vLm5aRW6bemag6Uu/b6/w3gt2pLp975QHU1anrdzv9Hrqsr1KTs/RwoQ0LUxI0z8uaaujGbmaOH+nvltz0OU1Khqz8v6i3fp9c7J+35wsSWoZWl8rx11VYd1KK7AZ9ud56GS2ops39Poajo6fztVdn6+VJO197Tqvx0Pl5JszRqiqvLdwtybO36nRQzpo9JCOZlcHQDWh5cRD0+6+2Owq1JiXSwWMh75Zr4UJaXp/0e4auf/BE1n23z1ZFfaVWTv06fJ9Tsce/3GT22BSEYtFWpSQ5nTsSKmBr+5mNJWnon/tr9h9THsqmJp9MivvzPW8roFvmL72kO7/cl2lVvydOH+nJGnSgl0VlARQmxFOPBQcGKD7Lo02uxqmWLAjreJCVegDh52Ls/Mq152y+bD7bhtPeDP1uirsSLbq9qmrddVbS2r0vmYYO2Oz5m9P1VdxnnXBAah7CCde6H8OrHdS25zOLajUeRW1a1Q0ZKGwmmcLlbYj2Vqj9/MFGTnmrTp7/HSuXpu9Q7vTMkyrAwD3CCdeYBBezRv85uJqua6v/UlW1Eu0cs8x/bC2ct1UKGvsT5v18dK9GvbOMrOrAsAFwokXfO0LrS4oqEQLxtr9Jyo1JsRT2XmFlW7RcaeizRBv+2S1npyxxeV6LOnZ+RoycYn+/MGKclt8fC5cV+OfUUU2FXf75Rf62DNBrbblcLpe+X27rCa2Cp4rmK2Dc87NU+LKfT8nv1CLPZjy7M6k2J1ljjl+8e9IztD5LRqqfj1/jzdL9PR7+oiLvXwGTlhoD0vbj1jVvXVomTLzt6fq8enxZY6biXWXca65oXil6cy8Ao2/6SKTa1O7EU684Gv/8ETlXPfuMu2txIDXQyeyFBjgp02HnAfbFtoMjZi8Qs0aBepPPVpqzPRNimraQI2D6um2/m3s5TYfTtf5LRpVeJ+tSenq1qpswCiqw5lwYjMMHU3PLdOKk5aRo/u/WKfb+rfRLRcX3f/+L9e5vF5GTr5unhKnIZ3D9e+hnSqsW1XyNJDlFdh04Him2odV/OxQeySmZGhXWoauv6il2VWpcgkpjGU6W4QTL3Rr5cnqp/B1FQUTdyH0stcXuTyemJKhLcUL0Vmzi5pzi0JEtp75Zau93Ogf4nX9RZEuV4p17Ia6/r3l+vb+/urXrqlyCmw6fvrMonU/rDtk//3R7zdq9paUMtd6c26iNh1O16bDW+zhxJX9xzI1b3uKElIylJCSUSacnMjMc3meYRiyZhcoNLie22tXpXumrdXy3cf09i09auR+nliUmKYliUf1n+s6KzCg7J/nycw8rdxzXEO6hCnoHNjVvDoMnbRUktQ0OFAD2jc3uTZVq/TfIYZh6PW5ibqoVaiGdY80p1K1DGNOvND6vGDNf+xyfXFPP7Mo6fYgAAAgAElEQVSrgmr0yHcbvSrvuKDbhoPlT2H+fm1RuDAMQ8dP52r2lmSXK8fe9slqPfvrVg0YH6sr3ljs8lqugonk+QynQW8u1ucr9rt9f9+xMyFupcOKtA9/t1E9Xpqn9QfOblHCisbZlFi++5gk6YuVnk89zsor0AeLd2t3WvnrxlTW3Z+v1bSV+92uSHzLx3Ea9e0GTZxXtgsQzrbXgZlqC3ak6cPFe9xumYGyCCde6hDeWFd0bGF2NVBJN3qw+3CSi3Ed5fGmu6+kJeUfn65Wn1cW6KFvNuipn7e4/Jr+bs0hWXPOfuBteTsQO+6qXLqlxHHx2XnbzgShWcWr5n6y1Hnhu9IWJqQqsZzmbU+6df7Ykmz/3Zte1Tfn7tTrcxI1ZGL1rhvjagyQJO1MLQpFJSsM11U5+YU1Pi3fF6Vl1K7dy30B4aSSLoxobHYVUAmbqmH34evfqzjwlLZi95mWiJ/WH3ZaFfdszN2W4tSi8sSPmzzegbj3y/OdVqh17Goq+Xpx3GPIr/hvjzX7TuifX67T4ZNnPsPWpHTdM22dveneFU/aTUpWhJW8m2203k0gyy+06dvVB3XstOsuK0lKSc/RoSr68/A11b3bt6PsvEJ1f2Gurq7mgIhzE+EE8AFvzE2skuuU3mLgx/WHvTr/qreW6JeNSdqddtpleOj/Wqz995Jumb99FKd521M12mGHaU8GBHrScuL4VepNC5WrS+cX2vTo9xv1n5lb3N/PMHTJ+Fhd9vqiKp8ubrakU9nq++oCvVlF/61VZNPhU8ovNLT3GDthw3uEkyoy48EYs6sAVInRP8RryMQlTrtHbz6crjfmJpTZmNGxNWPdgZPKK7Bp4vyd5W4sWZ7M3AK3O0BXtFljRZ6eucXtOJ0Sjg0LKeneN8VvOZyuf7qZGeWJnzcctneblZZmzdEnS/fqVJb7Vp8S+45l6taP47Rs15kp8+8s2KkTmXle7ZG1/YhV932xtlIrGDNVHGeD2TpVpE/bpmZXAbWEt2NazLK71CaEkxftcS5gcR4oK0kdn/nD5bUMwyizMJ7FYtGcrck6fDJb9112vrLyCtT1+blqElxPG5+9WtZs55aLDC/G38QfKjswefo6161IWXkFCg4s+1fhf+ckqFWTBnrhT109vu8NHoxpcufY6VyNmb5JkjS067Ays7pum7pau9NOK27vcX12V/kbkf7ruw3ammTVqr1rtH/C8ErX6W8fxel0boHW7DuhzS8MrfR16hpG2Zw9Wk6qwJR/9DG7CqhFBk5YaHYVqsztU1dXWGZxYpp6vTzfaVBtiQe+3qBXZu3QlsPp9sGzp7Ly9eJv29XjpXlOs20OHHc9DsQwDOUV2Oy/e7tZ5NUTXY+Lmb89VdNW7leq1X0LytTl+8qse1NZjuHL1ZdbybNYmFDxRpyOrV5no6RrqzIDs6tzlebymLlnE6oO4aSSWjVpYP/92m4RkqSZDw0wqzpAlfvvnIRy33fX/VDaXZ+v1amsfP3zq/VOxx3H2Rw77fxlOm3lfo+uXVBo08WvLlDHZ/7QfV+s04AJC53WlpGk53/dqs9XuJ9ZlHQq2z6jZPaWsp+pJPi4c+PkFW7f87ktAyowfe0h3fDe8nIDmafMyCYrdh9T9xfm6YX/bav5mzugS+vsEU4q6bWbumto13B9e19/+7Febc4zsUZA1arp71VvbzdxXqI+WrrXPvNmwY5UJafnaMYG5+6bL+IO6MXftpd7rYtemKtxP2/Rv1yscTN12V7tr+SgziPpOW4XsytP6Wc/ddneSt3f0YnMilsUxs7YrC1J6Xp11o6zvl9VmL89VQ99s17p2Z61hpQEak/DbXWpXZHUNxFOKik8pL4+uqPvObeyIVCd3HW5/Lb5iFbtPe7yPXfeXbi7ymY5ZeYV6rs1rnd9/iLugAa/tVj5hTa3LSELE1LdvjdtxT6dysrTx0v3OLVI7D+Wqay8ou6SQpvhNOjUcfDv1qR0vVIqLFTUdeFqgbsFO1LLPcdRZgUzlSpqTSqqw9m7/8t1mr0lRW/P92wxO1oszh2Ekxp2/UUsXYy6q/Nzc1we/3lDkl6fUzNTXCvDMKQeL87TPdPWunz/nmnr9Jmb1XbfXbhbY6Zv0muzE3TbJ6skSZsPn9KgNxdr8JuLJUkv/bZND7lZPdTVOjqvzS4KKxPn79R/Zm6plu6jfBcrF0vSr/FJ6vjMH/p5g/tp6mv3n9C3q8+EPU/rtyPZqsd+iNfBUuOLjp72cAyNm76krUnpWpxY8VidmlAdA+K3JqUr1ovwWRsQTmpQ/Xp+ev+23low5gpJ0mUdaHUBaousvEItKmc365d/d991VDKIdU/xvk5ziwcHp1qLvnS/iHNeBj/NWv6XccleTu/G7tK3qw9qR3LRYOLM3AK3ocKdvAKb1h844XRebEKaOjzteubVo8Xr2ZTMLHLl5ilx+nljkv21p9npxvdXaObGJN36cZyue2eZZycVO3wyS6kO078dBypf/95y3fX5Wu09Wj3bGXhj4ISFSquCMT2Orn9vue79Yp12p1Vuw8EDxzO1onibCF/BVOIactWFYZp8e29JUvuwRto/Ybi2HE7Xsl2Vn3oIoOZVxV/ijrtLu1re/bLXF8nfz6IBFzRzef7WJKtOOoxlycjJV3pWvnq8NE9tmwWXKV/el+GTMzZrpkOQKE/p4DNmerx+3lB07tPXddb9l5/v8jxP23Xyiq9/JD1HR7xYZyY7r1CX/td5Y84bJ6/QvvHXOc0aWrrzqA6dzK5wC5L07Hw1DPR3uUmnRypIY1uS0nVVSH1JRa1F249YdVPvVmc9w2nfsSy1D/N+9fKS/bt+HTVQPaKanFUdqgotJ9XogSsusP8eHBSg+vWcdyc1aaYdgLPgyfTp8vy++Yj+t+mI/XUXN11dhTZDy3a5D0Kvzz0zm8qQ9EXcfkllp1znFhSqn8PKvpJ085SV9gXaPA0mNptRZqPDkmAiSa/Odj+I1lbOl/WSnUc9HvBaWn6hTZm5BWVme5WYu825q+OF37Zr5Gdryt208sipbPV4cV6ltqVwZcvhdKcWHcl5sb9h7yzT4z9ucpoinpNfqE+X73M5EHtRQpom/JFQLXsWbT5cNdPiqwLhpJq0bRasp4ZdaHY1APiYh791nhGU68HgUleOZpxpOflk6V6nfYgc/bap7PTotftP6o5P13g1VuWR7zdWaqVYqfyGhGW7jummD9xPx3a+jqHv1xzUgu2pyi+0adAbi9X1+bmKczOY2nG/J0d/+TDOPjg5J995kHbJwOGElAwdPplVZs2YQptR5hx3dqed1g3vL9e7C51X5XUV1hyf7Tuxu/Ty79s1qHhMUon/zNyiu6et1ZQle/TQN+tV2tmOPfKlWUaEkyr24e29Fd28oT4o7sIp0axhYKWut/VFVmUEUJbj7JvYUguzFdjOBJ7yZtZEj5vt8f1+35zschaQJ0pvPVB6ttGeo5kejZV5e/5OPfXzFt335Tr1fmm+fXDpOwt2uSz/yqwd+qpUa0+Jf/+4SVsOp+vCZ+c4rYvi59Ckfel/F+niVxc4nTfsnaXq/sJc+0yrEluTym4q6mqlYsl1iMjJtyk9O18rdx/TIhcL7eXkFzoNMi5pFXJsQdlzNFPHPRw8bBiGvl19UBvK2bXcTISTKjase6QW/XuQurYMlVQUVoZ0DtdjQzqWKevYrfPs9V1cXq9RUICuujCsWuoK4NzkuOuyXxV2H3vSFe1qb6Sbp8Q5tQzc8tGqMmUmLXA/XXjW5mSlZ+U7tUBkeLgx47OlFuUrsf94pt6cVzRDzHFdFD83HzInv1BfrNyvnamnlV9o2IPH1qR0WXPynbqBjlewto2rHpn3F+1Wjxfn6bapqz3aOFMqCmu/xp/pWvvvnAT1eWWBdqZmaNuR8ndgX7zzqP4zc4tucti13JfWDGRAbDUb1j1Sw7q7nj7cPqyRggP91bRhoO69NFr3XhotSWr31Cyncm/f2lMXvTBPkvSvK9vrvYWeb9wFoG6ryrFtFV3r1Vnb9cmysqvxbj6crpGfrdGap4do3rYUbXfRPVRm76ZSHv/R/eygyrDI4vR5Vu4+pojQ+vrMzWrCb85N1NTlzu8t2XlUIz9bo/CQIKfjh09mF9/DtZIQcLbdMO/Eum4xuubtoi0ZNj1/jUIb1HNZZk9a2ZlLvrSiMeHEREEB/tr43NXyL/V//JUXhmlhQpp91H1I/Xr67v5LNG97ikYNbq+vVx3QySz2jwBQsZz8yo1pceW7NYfKfd9VMCmRlpGrWz+O06q97gejlqe8ReQq+6V6xGHNkdvKGeg8f3tqmWBis0l/FG93kOpm6re7MFcy5qRkWnZ5XG2a6amjGblO4WTW5mQt2ZmmV0Z0d32vSt2lehBOTBYU4F/m2Pu39dKafScU4zCNMOaCZvbXv4waaJ/65U7joACPmz0BnLueN3mfGUeVDSYV8WbacQmLRdqZ6tm6J/d/ua7MsX98ulqNgtx/hf60/rAOnXA9ILcknDjO2nLnuV+36eUR3SrV5ZJmzVH7sEaKP3RKB45n2sPQ9HWH1SS4bIuKDzWcMObEFwUHBmhQpzCXwUWS2jZr6PL4lH+cGYTbvLFzM+PA9q7XSwCAuqgqertOl/MPwH//uMltt4s3IaBkQG9507HduW3qan26fJ9GTF5RppXmlIvWdx/KJrSc1FZx467Uit3H9W+Hfthru50Z21L6f7zX/9pDAycsrKHaAYBv23/cdatGTXhjbqLW7ve8Fan0OERvlLdycWm+NOaElpNaKjK0gf7ap7U6hDVyXaBUOmnVpEH1VwoAUKGkU9n6ZrXrjSZRhHBSy/m7mSf4yJUd7L/3alO0HHFJ/+jQruFnfd+O4Y2U8PK1Z30dAIBv8KGGE7p1aruru4QrISVDYcVjTDY+e7WOZ+aqfVhjtW0WrNO5Bbq4XVNJ0oIxV2jt/hMa1i1Cr89NVNOGgfpy5f5KDSaTpPr1/PXyjV317K++M+AOAFA5pRfLMxPhpJb715UddEGLRhpQPOD1vIaBOq94Ndpebc5zKhsRWl839GgpSfrPdZ0lSf93+fm6/I1FOnE6T89c30WdIhpr1uZkfVo8ba5bqxBtTXK/ZPUdMe20/sBJ/RJf8ahzb03/vxg1rh+gYV7uTgoAqN0IJ7VcYICfRvRqVenzLRaLFj0+SIWGYZ8d1CUyRF0iQ3R5xxZq0ThIf3p/uTYfdr/a4Nu39NTj13RS3J7jGjtjs9tyd1zS1j7y/Jou4Zq33f26BeEhQeoX3dTpWKOggHJHxwMAKs+XunUYcwIF+Ps5TVuuX89ff+nTWi2Ku4o+uqOP7h7YTq/92fXCPRaLRVFNg/XXPq317PVd9PEdfcqUmf3IZXp5RDf76+6tQjX1zr4aNfiCMmXdefyajgoK4D9ZAKgOPpRNCCeoWGRoAz1/Q1fd1r+NGgYWhZiB7ZuXKefnZ9G9l0brmq4R2j9huH57+FL7e11ahjiVtVikIV3C9cTQindurudfNOj3uu6RmjP68rP5KJKkhJevdRpIPKRzuBaMuVy3929z1tcGgNqKlhPUWnNGX64XbuiisR6Eiu6tQzXzoQFa/Z+rPL7+E0M7SZIm/q2n/djWF4dqw7NXKzykvto2DVafts5jaV4e0U3f//MSBfqf+c95yRODXF5/4t96qH69om0DSvx7aEe1D2usS10ELqmoC6pEVNMG2vriUPdTuIuVBCpJZeoLAL6o0FZ1Wx2cLcIJvBLVNFh3DYxWg0DXq9eW1qvNeQoPqW9/3Sm8saSiVhBXRg1ur12vDnNqmQkKKNocUSpqnfnpgRhtf2mourcK1b+ubK87LmmrS85vpkuKl/cPaxykts0aKuHla7XtxaFqUK+orhe0aKibereWVLRf0Z0xbXVt1wh7nVz5+t7+Tq+Xjb1SjYIC9NldF5f7uTc8e7Vu6tVKPz0QoxkPDtDe167Tr6MG6pEr29vLvHxj13Kv8d+/uO5GA4DqkFfgO+GEAbGoUb8/cqms2flq1ijIbZl6/uVnZovFouDAAP32r0udjk+6pae+jNuvvxQHkPrFoWTmqAH6aMlejR7Swan8Szd2kzsXtztPdw+M1qUdmis2oezA3aimwVow5nINmbi0zHsN6vmrcf16mnjLmdYfPz+LekQ1UavzGti3fR/WPVJXdg6Xv8WiS8bHOl1jWLcIDbjAdUtOv+imWrOvaHXJt27uobCQIDVtGKjh7y53Wb60QZ1aaHHiUY/KeuvHB2J085S4ark2gOrl52bdLDMQTlCj6vn7lQkmV3RsoSU7j6p38WJxldW0YaBGD+lY5viFESF62yEouONYrx8fGGD//bGrO8qaXaAbe7Z0Kt8+rLHWPTNEfV9Z4HTcscuoTB2DA3VBi4Yyin8v+ctg+ZODdfBEVplA8shVHeRnkSYtOLNHx4ierXTPwHZau/+kRvRqZR8/88TQTnpjbqLT+Zd1aK7LOjTXewt365UR3TSsW6QCA/z03K9b9WXcAaeyzwzvrIZBAZqyZI/+fU0n/eu7jfb3XE0p97NId8a00029WynpZLbaNAtW15ahemjQBfpg8R6Xnz8owE8zHxqo694tmh5+/2XR5e5kC6DmVPQPw5pkMXxpMX03rFarQkNDlZ6erpCQkIpPQK2SnpWvXzclaXj3yHJbVKqbYRiatGCXLghrpD/1aFnxCcX2Hj2tP7amKDjQX5GhDXRtt4hyyxfaiv6Xc7e6ryv/99U6zd1W1IKz5YVr1Lh+2R1FS8TtOa51+08oI7dAY4d2UoC/n2w2o8y/igoKbQrw91NGTr4ycgrUstQWB2N/2qTAAD+9cENR99PYGZu1MzVD0+7up5OZeWof1sjtVu7ZeYXKyM3XjuQMHTyeqS/jDmj2o5fZ//Kbty1FIQ3q6ZLzm+loRq62J1vVNDhQU5buUbtmwZq8yHW4cfTyiG7ak3Za01budzoeHOivrLxCt+c9clUHvetmQ7YSF7UO1f8evlTXvL2kwp1rB7Zvpr/1jXLaWO3v/aL03ZpDFX6Gr+/tr398utrle2/e3ENbk9KVdCpb88uZdg9UlSeGdtKowe0rLuiFyn5/VyqcTJ48WW+88YZSUlLUo0cPvffee+rXr5/b8osXL9aYMWO0bds2RUVF6ZlnntFdd93l8f0IJ6jrMnLy1f2FeZKkXa8O86l/4VSHDQdP6qYPVpY5/vEdffTmvER9cHtvtQ9rrPxCmzYfPqVbPlqlguLQt/XFobr9k1W6pmuEU0vSc9d30cgB7eRnkQ4cz1LbZsHKyC3QrtTTuuPT1fZA07tNE33/zxgFBvhpZ2qGrnn7TNfdLX2j9NpN3eVnKepePH461x6oSzZnO795Q8177HJd9+4y7Uw9rcev7qiOEY31f1+tlyStGneVNh0+pc4RIWrTLNh+3iXnN9VdA9rpga836M+9Wjm19qWk5+j/vlqnVuc10OwtKU7PpHH9AL379156/tdtmnxbb83YcLhMYHPl2/v664PFe7R897EKy0rSrRdHae/RTK0ptWHd1/f21z3T1iqv8Mx4hYtah+rqzuF6a/5Oj65dnrHXdtLrcxIrLoiz9vR1nXX/5edX6TVrLJz88MMPuvPOOzVlyhT1799fkyZN0o8//qjExESFhYWVKb9v3z5169ZNDzzwgO677z7FxsZq9OjRmjVrloYOHerRPQkngBR/6JQC/Czq1irU7KrUiLSMHPV7tWgszoieLXVjr1Ya3Kns3zFS0Zf3JeNj1SS4nuKfu8Z+fN3+E3pzXqJe/FM3dYpwP/A5O69Qq/cd1wUtGimqabDTewkpVr05d6cev6ajOke6//vnh7UH9ea8nZp298Xq2jJU+YU2JaZkqEtkiCwW6b2Fu9UporGGdnVuWftw8R4t331Un468WPXr+Ss9O18h9QNctkoZhqFtR6zKzi/U1qR03RnTzmULXE5+oZbvOqZLLmimj5fs0ZQle/XsDV10NCNXXSIb68oLwxVYvGbQlW8t1t6jmfZzNz1/jQ6fzFJQgL/mbU/Rn3q0VNOGgQoODFChzdC/vtugbq1CdXmHFsotKFSftkWLJc7dlmIPYPsnDLdf72hGrg4cz9TqfSc0Y8Nhp3vd1r+NjpzKVsfwxrp7YDvFjC/aOf2S85tqwk0XqV3zhk7BvMTLI7qpW8sQXdS6iU7nFKjHS0XvNwoKUOzjV2j7EaviD51SwyB/HTudp1mbk5V0KtvpGn3bnqd1B066/fMs8fldF2vBjlSnjfoujGishJSMCs81w/pnhujpmVs1Z1tKxYVLef+2Xrr+Is9bjT1RY+Gkf//+uvjii/X+++9Lkmw2m6KiovSvf/1LTz31VJnyTz75pGbNmqWtW7faj9166606deqU5syZ49E9CSdA3XToRJYaBgXYZ2uV52hGrhrXD7APhDaDYRhuu7rMVF69Dp3I0n/nJKiev58ev6ajWp8X7LKcJ/f4Y2uKurYMUdtmDV2WOXwySx8v3atBnVpowAXNy/xZHTqRpXr+fooIre90/ODxLBXYbNqRnKHzWzQsExIPn8xSQaGh1uc1UICLVsX0rHx9tmKf/tSzpZ7/dZtan9dAE/5ykaw5+Zq8aLeW7TymVGuOXrupuz1gzR19uTqGn+m6zMkv1FdxB3Rz39ZqEhyo9QdO6IuVB9SlZYgm/JGgfu2a6uUR3TR2xmbdcUlb7U47rcs7NlfM+c1UYDM0acHOMt2V8x67XC2bNFC35+dKkl7/y0X628VRkqSNB09q4vydOpqRq8eu7qj/xR/RZR2aa/CFYer/WlFof+nGrrq2a4RembVD/9t0RDMejLGHxU2HTmlLUrqW7jyqedtT9ciV7XXf5efr4PEsbU+2auxPZVfzfvfvvbzq0vZEjYSTvLw8BQcH66efftKIESPsx0eOHKlTp07p119/LXPO5Zdfrt69e2vSpEn2Y59//rlGjx6t9HTXS6Ln5uYqNzfX/tpqtSoqKopwAgDwOYdOZCkitH6F3a0p6Tlave+4ruse6VR2yc6jstkMDb7QdctgabkFhbLIYm/98pbNZmjJzqOKCK2vCyMa62hGrnILbGrWqKiFrCpVNpx4VYtjx46psLBQ4eHhTsfDw8OVkJDg8pyUlBSX5a1Wq7Kzs9WgQYMy54wfP14vvviiN1UDAMAUpbsC3YkIra8be5bdC+2Kji28up/jdiOV4edncQpCYSH1yyltDp8cVTdu3Dilp6fbfw4dqnjUOwAAODd41XLSvHlz+fv7KzXVeVpbamqqIiJcT5+MiIhwWT4kJMRlq4kkBQUFKSjIvCmlAADAPF61nAQGBqpPnz6KjT2zmqXNZlNsbKxiYmJcnhMTE+NUXpLmz5/vtjwAAKjbvO7WGTNmjD755BN98cUX2rFjhx588EFlZmbq7rvvllTUJXPnnXfayz/wwAPau3evxo4dq4SEBH3wwQeaPn26Hnvssar7FAAA4Jzh9bDcW265RUePHtVzzz2nlJQU9ezZU3PmzLEPek1OTtbBg2fmg0dHR2vWrFl67LHH9M4776h169aaOnWqx2ucAACAuoXl6wEAQLWo7Pe3T87WAQAAdRfhBAAA+BTCCQAA8CmEEwAA4FMIJwAAwKcQTgAAgE8hnAAAAJ9StXsjV5OSpVisVqvJNQEAAJ4q+d72dkm1WhFOMjIyJElRUVEm1wQAAHgrIyNDoaGhHpevFSvE2mw2HTlyRI0bN5bFYqmy61qtVkVFRenQoUOsPFvNeNY1g+dcM3jONYPnXDOq8zkbhqGMjAy1bNlSfn6ejySpFS0nfn5+at26dbVdPyQkhP/wawjPumbwnGsGz7lm8JxrRnU9Z29aTEowIBYAAPgUwgkAAPAp/i+88MILZlfCTP7+/ho0aJACAmpFD1etxrOuGTznmsFzrhk855rha8+5VgyIBQAAdQfdOgAAwKcQTgAAgE8hnAAAAJ9COAEAAD6FcAIAAHxKnQ4nkydPVrt27VS/fn31799fa9asMbtKPmv8+PG6+OKL1bhxY4WFhWnEiBFKTEx0KmMYhp577jlFRkaqQYMGGjJkiHbt2uVUJicnR6NGjVKzZs3UqFEj/eUvf1FqaqpTmRMnTuj2229XSEiImjRponvvvVenT5+u9s/oiyZMmCCLxaLRo0fbj/Gcq0ZSUpL+8Y9/qFmzZmrQoIG6d++udevW2d/nOZ+9wsJCPfvss4qOjlaDBg10wQUX6OWXX3baBI7nXDlLly7VDTfcoJYtW8piseiXX35xer8mn+vBgwc1fPhwBQcHKywsTE888YQKCgrO7gMaddT3339vBAYGGp999pmxbds24/777zeaNGlipKamml01nzR06FDj888/N7Zu3WrEx8cb1113ndGmTRvj9OnT9jITJkwwQkNDjV9++cXYtGmT8ac//cmIjo42srOz7WUeeOABIyoqyoiNjTXWrVtnXHLJJcaAAQOc7nXttdcaPXr0MFatWmUsW7bMaN++vfH3v/+9xj6rr1izZo3Rrl0746KLLjIeffRR+3Ge89k7ceKE0bZtW+Ouu+4yVq9ebezdu9eYO3eusXv3bnsZnvPZe/XVV41mzZoZv//+u7Fv3z7jxx9/NBo1amS888479jI858qZPXu28fTTTxs///yzIcmYOXOm0/s19VwLCgqMbt26GUOGDDE2btxozJ4922jevLkxbty4s/p8dTac9OvXzxg1apT9dWFhodGyZUtj/PjxJtaq9khLSzMkGUuWLDEMwzBsNpsRERFhvPHGG/Yyp06dMoKCgozvvvvO/rpevXrGjz/+aC+zY8cOQ5IRFxdnGIZhbN++3ZBkrF271l7mjz/+MCwWi5GUlFQTH80nZGRkGB06dDDmz59vXHHFFfZwwnOuGk8++aRx6aWXun2f51w1hg8fbtxzzz1Ox2666Sbj9ttvNwyD51xVSoeTmnUP9/QAAAWoSURBVHyus2fPNvz8/IyUlBR7mQ8//NAICQkxcnNzK/2Z6mS3Tl5entavX68hQ4bYj/n5+WnIkCGKi4szsWa1R3p6uiSpadOmkqR9+/YpJSXF6ZmGhoaqf//+9me6fv165efnO5W58MIL1aZNG3uZuLg4NWnSRH379rWXGTJkiPz8/LR69epq/1y+YtSoURo+fLjTs5J4zlXlf//7n/r27aubb75ZYWFh6tWrlz755BP7+zznqjFgwADFxsZq586dkqRNmzZp+fLlGjZsmCSec3WpyecaFxen7t27Kzw83F5m6NChslqt2rZtW6U/g2+sU1vDjh07psLCQqeHKUnh4eFKSEgwqVa1h81m0+jRozVw4EB169ZNkpSSkiJJLp9pyXspKSkKDAxUkyZNyi0TFhbm9H5AQICaNm1qL3Ou+/7777VhwwatXbu2zHs856qxd+9effjhhxozZoz+85//aO3atXrkkUcUGBiokSNH8pyryFNPPSWr1aoLL7xQ/v7+Kiws1Kuvvqrbb79dEv89V5eafK4pKSku7+NYj8qok+EEZ2fUqFHaunWrli9fbnZVzjmHDh3So48+qvnz56t+/fpmV+ecZbPZ1LdvX7322muSpF69emnr1q2aMmWKRo4caXLtzh3Tp0/XN998o2+//VZdu3ZVfHy8Ro8erZYtW/KcUa462a3TvHlz+fv7lxmVnJqaqoiICJNqVTs8/PDD+v3337Vo0SK1bt3afrzkuZX3TCMiIpSXl6dTp06VWyYtLc3p/YKCAp04caJO/NmsX79eaWlp6t27twICAhQQEKAlS5bo3XffVUBAgP1fJDznsxMZGakuXbo4HevcubMOHjwoif+eq8oTTzyhJ598Urfeequ6d++uO+64Q4899pjGjx8viedcXWryuUZERLi8j2M9KqNOhpPAwED16dNHsbGx9mM2m02xsbGKiYkxsWa+yzAMPfzww5o5c6YWLlyo6Ohop/ejo6MVERHh9EytVqtWr15tf6Z9+vRRvXr1nMokJibq4MGD9jIxMTE6deqU1q9fby+zcOFC2Ww29e/fvzo/ok+46qqrtGXLFsXHx9t/+vbtq9tvv13x8fE6//zzec5VYODAgWWmwu/cuVNt27aVxH/PVSUrK6vMLrf+/v6y2WySeM7VpSafa0xMjLZs2eIUYubPn6+QkJAy/wDwSqWH0tZy33//vREUFGRMmzbN2L59u/HPf/7TaNKkidOIY5zx4IMPGqGhocbixYuN5ORk+09WVpa9zIQJE4wmTZoYv/76q7F582bjxhtvdDl1rU2bNsbChQuNdevWGTExMUZMTIzTva699lqjV69exurVq43ly5cbHTp0OKenBFbEcbaOYfCcq8KaNWuMgIAA49VXXzV27dplfPPNN0ZwcLDx9ddf28vwnM/eyJEjjVatWtmnEv/8889G8+bNjbFjx9rL8JwrJyMjw9i4caOxceNGQ5IxceJEY+PGjcaBAwcMw6i551oylfiaa64x4uPjjTlz5hgtWrRgKvHZeO+994w2bdoYgYGBRr9+/YxVq1aZXSWfJcnlz+eff24vY7PZjGeffdYIDw83goKCjKuuuspITEx0uk52drbx0EMPGeedd54RHBxs/PnPfzaSk5Odyhw/ftz4+9//bjRq1MgICQkx7r77biMjI6MmPqZPKh1OeM5V47fffjO6detmBAUFGRdeeKHx8ccfO73Pcz57VqvVePTRR402bdoY9evXN84//3zj6aefdppiynOunEWLFrn8O3nkyJGGYdTsc92/f78xbNgwo0GDBkbz5s2Nxx9/3MjPzz+rz2cxDIel+gAAAExWJ8ecAAAA30U4AQAAPoVwAgAAfArhBAAA+BTCCQAA8CmEEwAA4FMIJwAAwKcQTgAAgE8hnAAAAJ9COAEAAD6FcAIAAHzK/wNYj227N+tygQAAAABJRU5ErkJggg==", "text/plain": [ "PyPlot.Figure(PyObject )" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/plain": [ "1-element Array{Any,1}:\n", " PyObject " ] }, "execution_count": 56, "metadata": {}, "output_type": "execute_result" } ], "source": [ "plt.plot(1:length(train_loss_list), train_loss_list)" ] }, { "cell_type": "code", "execution_count": 57, "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAicAAAGgCAYAAABxDccgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAPYQAAD2EBqD+naQAAIABJREFUeJzt3Xl0XPV9///XnX1G0owW27KMjWWw2b2wuoamkGBwCKVAmuJATjAm0JBAC3UIyy/fYEwpNAk7dSBtMUuThiUEkpbU1BgMwXEhYJywxXaMwSaWZcu2NFpGs35+f9zRWLK1jTSjq+X5OOce3blz7+g9VwPz8l3eH8sYYwQAADBMuJwuAAAAoDPCCQAAGFYIJwAAYFghnAAAgGGFcAIAAIYVwgkAABhWCCcAAGBYIZwAAIBhhXACAACGFcIJAAAYVggnAABgWPE4XUB/ZDIZ7dixQ2VlZbIsy+lyAABAPxhj1NzcrEmTJsnl6v/xkBERTnbs2KEpU6Y4XQYAABiA7du3a/Lkyf1ef0SEk7KyMkn2mwuHww5XAwAA+iMajWrKlCm57/H+GhHhpONUTjgcJpwAADDC5HtJBhfEAgCAYSXvcPLaa6/pvPPO06RJk2RZlp5//vk+t1mzZo1OOOEE+f1+TZ8+XY899thAagUAAGNA3uGktbVVs2fP1vLly/u1/tatW3Xuuefqs5/9rDZs2KDrrrtOV1xxhV588cW8iwUAAKNf3tecnHPOOTrnnHP6vf7DDz+sadOm6e6775YkHX300Xr99dd17733asGCBfn+egAAMMoV/ZqTdevWaf78+V2WLViwQOvWretxm3g8rmg02mUCAABjQ9HDyc6dO1VdXd1lWXV1taLRqGKxWLfb3HnnnYpEIrmJHicAAIwdw/JunZtvvllNTU25afv27U6XBAAAhkjR+5xMnDhR9fX1XZbV19crHA4rGAx2u43f75ff7y92aQAAYBgq+pGTefPmafXq1V2WrVq1SvPmzSv2rwYAACNQ3uGkpaVFGzZs0IYNGyTZtwpv2LBB27Ztk2Sfkrn00ktz61911VX66KOPdMMNN+gPf/iDfvjDH+rpp5/WP/zDPxToLQAAgNEk73Dy1ltv6fjjj9fxxx8vSVqyZImOP/543XLLLZKkurq6XFCRpGnTpumFF17QqlWrNHv2bN19993693//d24jBgAA3bKMMcbpIvoSjUYViUTU1NTE2DoAAIwQA/3+HhED/wEAMFakM0bJdCY7GaUyGaXSRqm0UTI7n0xnlMoYpTPZdTo9l0pnlDzguVRu3t6uu2VfOnGyjjsk4vTbl0Q4AQCMMZmMUSKdUTyVUTyVViJlz3f9aS8/cFk8lVEinVEylQ0QmU7zafu5jvBgP86Ghex8MpXJhYJEKpNbL5W2a0qmM8o4dD7jhKkVhBMAwNhijFEqY7p+2ScP/vLvHBjs5zuWd7Nu0g4E8WS6a7BId37O/tnxXDI97K9m6MKyJK/LJY/bksdlyevumO9+mfeA5zzursu8Lpfcbkve7HMdy2ZMKHX6reYQTgBgiKQzRrFkWm3xlFoTabXGU2pLpNWaSKktnlZ7Mm0f0s9klM4ees89TpvsYXyTPSyf6fI43XFIP7fs4Mf26QL7ccYYdVxxaGQHB0kyRjI6+DmTfdDl8QHr57bpeK3se+4cOJw6KtATy5J8bpf8Hpd8Hrf8no55V3beLV+nxz6PSz63S96On+6OYOCSLzvvzT7vzYYGe91scPDY2/iy23TMd7dNR+hwuyynd9OQI5wAwAGMsb9Q2xJptSVSiiXSak3sDxVtiZRa4wf8zAaM1kQ2cBwQPFoTKbUnM06/tWHF67ZyX/7+A8KA3+OS35t97O6Ytx/vDw/u3HLfAdt295odr2P/dOfChWUd8OWfTkqJFinR2s3UYj/vDUq+Eskb6jQftB/7QpInYCef4SSTkVIxKdme/dlpSsWkCcdIpROcrlIS4QTACNQRHtqT6WyASCuWSNtHJbJhoi37uGO+LZlSe24+u35uPtVl3VgyrWLex+iypBK/RyGfWyU+j0J+t0I+jwJet7wuS26Xtf+w/QGP3S77UL2745B9xzqdH7ut/etlD/N3fuxxWbIs5b6ULdnfo5ayyzsKPWBZbn1r/zbKPr//daxOz0kuy1LAuz9UdD4SkdcRAWPsUJBJSZmklM7+zKSyyxNSsq1rkGjrFCgSrdnnewodndbLJAf5F87ujY6g4g1K3pLsfKjT8tAB4ebA+ZD9/joCRbJNSrUfHCpy8708l4xJ6XjvJf/NY9KxFxbgvQ8e4QTAkDLGqC2RVnN7Ss3tSUWzP+3H++db4ilFc8sPfj41ROcH/B7XQUGixJd97O/60548KsmGjZ7W93tcB/9r3RjJZPbP2zNdnx/Isi4pq9Oyji/1dCI7JQ/4mciGgG6Wd1nW3fJO852DRC5cdASKpJRJ759PpzqFjwPWNekB/f0Gxe2zg4KvNPszGxrcvuwXfqv9M9G2fz7Vvn9fJ1vtaThy++wg5AlK3oD9vjzdDynjBMIJgJzcXQzJjOLprncxdJlPp/dfbHjARYotB4SI5vauIaMlnhrgdQdGIcVVpjbVWm0KWgl5lVLQnVapJ6NSj1GpJ6OgO62Q2yjkTivozijkzshvpRV0pRVwpeV3peRTWn4rJa+Vlk8peZWWRyl5lJTHpOQ2SblNSq5MUlY6sT84mIx9aDyWkdoy+5d1fr7bqdPz6mFd5M9ySS6P5PJKbk/2iETJ/kCRe9wpXPhCBzwu3R86uiwvkdze/GvKpLPBJXsU56D51myY6Rxuskd1upt3ebJHXoL2qSJvqFOYCHT/nCe4f3kugHReLyi53IX/exQQ4QQY4YwxisZSamiNa09LQnta4trTmrDns8ua4ynFk+lu71zoHDyKdTTCp6RKFVOF1aYpiinsalPYatd4X1xVnrgq3e0qd8cUcbUrbLWpVDGFTKuCmTYF0q3yplvlTbXI6u1LPJWdkB+33/5XtNub/dl53tPNsl7mXd2t780u99ohwuXeP+/OPs7NH/i8p1P4yL5Ol9cq+vBw+XO5JX+pPWHACCfAMNNx2qNzuNjTGldDix049rba4aMhG0T2tiYKECqM/EqqRAkFFVfASihkJVTmSqrMk7R/upMqcSVU4koqlH0+5EooKHsKKKGA2lVi2hQ0bQpmWuXPhgp3JtHTr5WS2am/LLcUCNvn8Lv9kuzrS9Tbvy/ajvmOL0Grt8nq4/nsl2hf63Q+1dPltE93y/tYt6/tO4LAcLtoExDhBCiaTMaoNWFfO9H5eon9j5P7j3BkQ0ZDNoj0fleHfXqjVDGVWjFNVEylrpjGexOa6E9ovD+pKk9cFZ64yl3tKrNiCigub6Zd3ky7POl2uTtNrlRMViomS70EnEx2GixfqeQvk/xh+2cg3Gk+0s1zZZI/0mk+bB+S5gsVGNUIJ8ABjDFqT2bUHM9emJkNFi3x/ddQ2AEjmb1os2Od/cGjpT2llkTqoDs+PEqpXK0qt5pVoRaVWjGVKaaJVkzTs2GjTDGVemIKu9pV4Ykrkg0YJaZNAROTL90mV09JIZGdBsPl2X/XQMetkR0/c+e4Qwf8DOyf94d7DiDD/Dw3gOGBcIIxJ5HKaEdjTJ/ui+nTfW3avq8tO28/3tPSv9MkfiVUoWZVWC2qsJpVI/tnuVpUaTWr3NOiCjWr0mpRpatF5WpWmdryK7a3IxaWq2sIyB2V6GbquDCwc6DoctFcpzAykIsAAaCACCcYdZLpjOoa27Oho2vw2L43pvrm9i5HNDxKqUxtilitqlabjrJaVO5qVqWrRdWeVo13t6rK1aoKq0XlJqoy06zSdFQ+095zEb2ypGC5FKywT2X4SjsdbeiYDlh24OkQfxmnNwCMWoQTjDjJdEY7m9r3H/HY26Zde/aocW+DWhoblGzdpzK1KqxWRSx7mqk2/bnVqrDaFPa2qtxqU4WrTWVqVdDEev+F6ezUHZdHClZKocr9P3PzVd3PB8s5vQEAvSCcYFgxmYya9u5WQ/2nijZ8qra9O5RsqlemeZcysX2y2qPypaIKq0UT1aYjs4HDa3VKD77+/rIDHvvK7OAQKJdCFXagyAWOqm7CR6V9JIOjFwBQUIQTFJ8xUrxZmeZ6RffsUHT3p2rbW6dk006Z5nq5Y7sViDeoLLVX5ZlGlVtplff2ej20NshYHqX9EVnBiNyhClmBiB00ApH9oSM33/m5CjtkuPnPAQCGA/5vjIFLxaXoDql1t9LNO9W2Z4da99Yp0VQn07xLnrbdCiQaVJrcJ7/ickkqz049yh6EiKpETa4KtfkqFQ+MlykZL39ZlUrCVYpUjldppEquUMX+kBEsl8sbkoujGAAw4hFO0DdjpKbtUv0Hin36O7Vu+53cDR8q3PqJ3NmLMdySyrJTT1pMQLtNRI3uSrV5K5UIjJMpmSB3uFr+8hqVVk1SxYTJqqo+ROFgicJD8d4AAMMO4QRdtUelXR/K1L+n1m2/U2LHewo1blQg3SJJCman3OrGq92mXLsVUYOJqNVbpXhgnEzJeLnKJipQkQ0d4w/RhHGVOqQsoGmeYdhyGgAwbBBOxqp0Str7kVT/ntJ176n109/LvesDlcT+JMk+u9J5ZIikceuPZpL+YA5VfeBwpcYdrdCU2ao+ZJpqKoKqiQQ0q9Qvj5vgAQAYHMLJWNCyW6p/T6p/X4kd7ymx410FGjfJkx3vxC11OYVSZyr1h8wUbdZUNYVnyF1znMZNPU5HTR6nM2vKFA7QpAsAUDyEk9Fm3yfSJ2tldr6n+I53ZdW/L398T+5pn/bfadtm/NpopujDzBR94qlVsuoYBafM1GFTpuiYSWGdNr5UPk7BAACGGOFkpGuPSh//WuaPLyux6SX5ox9Lsk/LBLKrZIylj021/mAO1cbMFO0KTZdr4nEaf+gROmZSuf5iUliHlAdlcacLAGAYIJyMNOmUtOMdacvLSm1+Sa4db8tl0rIk+WVfG7LBHK73MtO0SVPVWn6kSiYfp+mTq3VMTViX14QVCXFaBgAwfBFORoK9W6UtL8tseUWZj16VOxGVtP+PtyVTo19nZuot12y5DvuMTjmqVidPKdcl1aXye2iTDgAYWQgnw1GsUfr419KWl5X+48tyN34syT5V45bUaEr0euY4/TozSzsq/0zHHH2sTj9yvC6ZWsk1IgCAEY9wMhykU9Kf3s4eHXlZ+tPbssz+5mZJ49Z6M0OvpWfpbfdsRQ4/WacfVaNrjxyvSeXB3l8bAIARhnDilL0fSVtelra8IvPRq7ISzZJy3du1JVOj1zKz9OvMTO0dd7LmHjVVpx85XtdydAQAMMoRToZKe1T6aE3u6IjV+EnuKUvSPlOqtZnj9FpmptZ7jtfhM47UGUdO0O1HcHQEADC2EE6GQqJVmQdPlKt1lyQ7jCSMW+vNEXotPVO/zsxSavxx+oujJuqCI8frdo6OAADGMMLJEIjVb1awdZfixqv/TH9Or2Vm6T3vTJ0w4xCdceQE/YijIwAA5BBOhsCuuk81VdLHmqidpy3T3x4xQSdOreDoCAAA3SCcDIFY405JUpu3Ujefc7TD1QAAMLzxT/chkIza15rEvBUOVwIAwPBHOBkC6WY7nCQCVQ5XAgDA8Ec4GQKutt2SpExwnMOVAAAw/BFOhoAntseeKR3vbCEAAIwAhJMhEEjslSR5w9UOVwIAwPBHOBkCJal9kiR/ZKLDlQAAMPwRTorNGEUyjZKkkkrCCQAAfSGcFJmJNyughCQpMm6Sw9UAADD8EU6KrK2xXpLUavyqrCh3uBoAAIY/wkmRRRt2SJL2KqKQj4a8AAD0hXBSZK177db1UTdHTQAA6A/CSZHFm+xw0uqhdT0AAP1BOCmyVHZcnbiv0uFKAAAYGQgnxdZqt65P0roeAIB+IZwUmTvWIEkyJYQTAAD6g3BSZN52u3W9u3SCw5UAADAyEE6KLJS0B/3zRRhXBwCA/iCcFFlZ2m5dHyqndT0AAP1BOCmmdEph0yxJKqV1PQAA/UI4KaJUS4NcMsoYS+VVnNYBAKA/CCdF1LSno3V9mSpKgw5XAwDAyEA4KaKWPXWSpCYrIrfLcrgaAABGBsJJEbXvs0ckjrppXQ8AQH8RToooEbXDSYzW9QAA9BvhpIjSzfa4Ogl/lcOVAAAwchBOisjVZo+rkwkRTgAA6C/CSRF52u3usFbpeIcrAQBg5CCcFFEgnh1Xp4weJwAA9BfhpIhKUvskSQFa1wMA0G+EkyIKZ+xxdUoqCCcAAPQX4aRITLxZQcUlSRHG1QEAoN8IJ0USyzZgixmfKivKHa4GAICRg3BSJNFs6/o9iijk9zpcDQAAIwfhpEha9trhJOriqAkAAPkgnBRJvNE+rdPqZVwdAADyQTgpklSzHU7aGVcHAIC8EE6KpcVuXZ8MjHO4EAAARpYBhZPly5ertrZWgUBAc+fO1Ztvvtnr+j/5yU80e/ZshUIh1dTU6PLLL9eePXsGVPBI4Yo1SJJMCeEEAIB85B1OnnrqKS1ZskRLly7V+vXrNXv2bC1YsEC7du3qdv21a9fq0ksv1de+9jW9//77euaZZ/Tmm2/qyiuvHHTxw5kvO66Ou2yCw5UAADCy5B1O7rnnHl155ZVavHixjjnmGD388MMKhUJasWJFt+uvW7dOtbW1+vu//3tNmzZNf/7nf66vf/3rfR5tGelCSXtcHV+YcAIAQD7yCieJREJvv/225s+fv/8FXC7Nnz9f69at63abefPmafv27frVr34lY4zq6+v1zDPP6Atf+EKPvycejysajXaZRpqytN26PlhR43AlAACMLHmFk4aGBqXTaVVXdx1lt7q6Wjt37ux2m9NOO00/+clPtHDhQvl8Pk2cOFHl5eVavnx5j7/nzjvvVCQSyU1TpkzJp0znZdIKGztQlVYRTgAAyEfR79b54IMPdO211+qWW27R22+/rZUrV+rjjz/WVVdd1eM2N998s5qamnLT9u3bi11mQaVaGuSSUcZYKq9i0D8AAPLhyWflcePGye12q76+vsvy+vp6TZzY/ZfwnXfeqVNPPVXf/va3JUmzZs1SSUmJPvOZz+j2229XTc3BRxb8fr/8fn8+pQ0r0T11qpS0T6WqLAs5XQ4AACNKXkdOfD6fTjzxRK1evTq3LJPJaPXq1Zo3b16327S1tcnj6ZqB3G63JMkYk2+9I0LLnh2SpEYrIrfLcrgaAABGlrxP6yxZskT/9m//pscff1wffvihvvGNb6i1tVWLFy+WZJ+SufTSS3Prn3feeXr22Wf10EMP6aOPPtLatWv193//9zrllFM0adKkwr2TYSS2z77+ptlN63oAAPKV12kdSVq4cKF2796tW265RTt37tScOXO0cuXK3EWydXV12rZtW279yy67TM3NzfqXf/kXfetb31J5ebk+97nP6Xvf+17h3sUwk4jaPV/avLSuBwAgX5YZAedWotGoIpGImpqaFA6HnS6nTxseW6I5Hz+iNeUX6ozrHnO6HAAAHDHQ72/G1ikCq81uXZ8J0roeAIB8EU6KwJsdV8cqHe9wJQAAjDyEkyLwJ7Lj6oSr+1gTAAAciHBSBCVJu3W9P0I4AQAgX4STIghn7HBSUknregAA8kU4KbREq0JqlyRFqkZnHxcAAIqJcFJgsUa7AVu78aqygiZsAADki3BSYNGGOknSHkUU8ufd4w4AgDGPcFJgLXvtcNLkKpdlMa4OAAD5IpwUWHujPWJzq5dTOgAADAThpMDSUTucxH2MqwMAwEAQTgrMtO6WJCUDVQ5XAgDAyEQ4KTBXx7g6JRMcrgQAgJGJcFJgvni2dT3j6gAAMCCEkwILJvZJkny0rgcAYEAIJwVWlrbDSbB8osOVAAAwMhFOCimTVthEJUklVYyrAwDAQBBOCijdulduZSRJ5eM4cgIAwEAQTgooumeHJGmvKVVlacjhagAAGJkIJwXUssduXd9olcvjZtcCADAQfIMWUNs+O5w0u8sdrgQAgJGLcFJAieguSVKbl9b1AAAMFOGkgNLZcJKgdT0AAANGOCkgq80eVycdHOdwJQAAjFyEkwLyxuzW9SohnAAAMFCEkwLyJ+xw4gnTuh4AgIEinBRQSdJuXe9nXB0AAAaMcFJA4UyjJKmkktb1AAAMFOGkUBJtCqldkhQeRzgBAGCgCCcFEmvcKUmKG68qK7iVGACAgSKcFEi0we4O26CISvweh6sBAGDkIpwUSMteO5xEXeWyLMvhagAAGLkIJwUSb6yXJLV6GFcHAIDBIJwUSKrZDiftfq43AQBgMAgnBZJpsVvXJ/0M+gcAwGAQTgrEFWuQJGVKJjhcCQAAIxvhpEB87XbrenfpeIcrAQBgZCOcFEgwuVeS5KV1PQAAg0I4KZCylD2uTrCccAIAwGAQTgohk1HERCVJpVWTHC4GAICRjXBSAOm2vXIrI0kqHzfR4WoAABjZCCcF0NywQ5K0z5SqsqzE4WoAABjZCCcF0JxtXb/PisjjZpcCADAYfJMWQNs+O5w0u2ldDwDAYBFOCiDRtEuS1OalOywAAINFOCmAdLMdThKMqwMAwKARTgrAarVb16eDhBMAAAaLcFIAnnZ70D/Ruh4AgEEjnBSAP263rveUMegfAACDRTgpgJJs63pfhAZsAAAMFuGkAMLpRklSaSXhBACAwSKcDFYyphLFJElljKsDAMCgEU4Gqb2xXpIUNx5VVnK3DgAAg0U4GaSm7Lg6exRRacDrcDUAAIx8hJNBas2OqxN1lcuyLIerAQBg5COcDFLHaZ0WT4XDlQAAMDoQTgYplW1d3+4jnAAAUAiEk0HKtNjhJBkY53AlAACMDoSTQXK12ePqZEoIJwAAFALhZJB87XskSe5SWtcDAFAIhJNBCibtcXW8EcIJAACFQDgZpLLsuDoBxtUBAKAgCCeDkckobJokSaVVNQ4XAwDA6EA4GYRM2z55lJEkVYwjnAAAUAiEk0GIZrvDNpoSVYRLHa4GAIDRgXAyCC177HF19lkRed3sSgAACoFv1EFo27tTktTsLne4EgAARg/CySDEm+xxdVq9lQ5XAgDA6EE4GYR0dlydhL/K4UoAABg9CCeDYLXtliSlg7SuBwCgUAgng+CO2a3rVTLe2UIAABhFBhROli9frtraWgUCAc2dO1dvvvlmr+vH43F95zvf0dSpU+X3+1VbW6sVK1YMqODhJBDPjqtTRjgBAKBQPPlu8NRTT2nJkiV6+OGHNXfuXN13331asGCBNm7cqAkTuh9f5qKLLlJ9fb0eeeQRTZ8+XXV1dcpkMoMu3mmhbOt6fzmt6wEAKJS8w8k999yjK6+8UosXL5YkPfzww3rhhRe0YsUK3XTTTQetv3LlSr366qv66KOPVFlp39VSW1vb6++Ix+OKx+O5x9FoNN8yh0Q43ShJKqkgnAAAUCh5ndZJJBJ6++23NX/+/P0v4HJp/vz5WrduXbfb/PKXv9RJJ52k73//+zrkkEN0xBFH6Prrr1csFuvx99x5552KRCK5acqUKfmUOTSS7SpVmyQpPG6Sw8UAADB65BVOGhoalE6nVV1d3WV5dXW1du7c2e02H330kV5//XW99957eu6553TffffpZz/7mb75zW/2+HtuvvlmNTU15abt27fnU+aQaM/2OEkYtyoquVsHAIBCyfu0Tr4ymYwsy9JPfvITRSIRSfapoS996Uv64Q9/qGAweNA2fr9ffr+/2KUNSlPDnxSQtFcRVQe8TpcDAMCokdeRk3Hjxsntdqu+vr7L8vr6ek2c2P11FzU1NTrkkENywUSSjj76aBlj9Omnnw6g5OGhNdu6vtFVLsuyHK4GAIDRI69w4vP5dOKJJ2r16tW5ZZlMRqtXr9a8efO63ea0007Tjh071NLSklu2adMmuVwuTZ48eYBlO6+9Mdu63lPhcCUAAIwuefc5WbJkif7t3/5Njz/+uD788EN94xvfUGtra+7unZtvvlmXXnppbv1LLrlEVVVVWrx4sT744AO99tpr+va3v63LL7+821M6I0Uyareuj/kYVwcAgELK+5qThQsXavfu3brlllu0c+dOzZkzRytXrsxdJFtXV6dt27bl1i8tLdWqVav0d3/3dzrppJNUVVWliy66SLfffnvh3oUDTIsdTpIBxtUBAKCQLGOMcbqIvkSjUUUiETU1NSkcDjtdjiTp9w8s1Ky9K7X60L/TmZeP7KAFAEAxDPT7m7F1BsjX0bqecXUAACgowskABRJ7JUneSPct+wEAwMAQTgaoNDuuToBxdQAAKCjCyUBkMio3TZKkkooah4sBAGB0IZwMQCbWKI/SkqSKCYyrAwBAIRFOBqB5T50kqcmEVFFW6nA1AACMLoSTAWjeu0OStM+KyOdhFwIAUEh8sw5AW3Zcnaib1vUAABQa4WQA4k12OIl5CScAABQa4WQA0s27JUlxP63rAQAoNMLJAFitdjhJBcY5XAkAAKMP4WQA3O1263qVEk4AACg0wskABLLhxF1G63oAAAqNcDIAoY7W9RFa1wMAUGiEkwEIp+1wEqoknAAAUGiEk3yl4ipVmySprJJxdQAAKDTCSZ7am+olSUnjVkUV15wAAFBohJM8RRvs1vV7FVY46HW4GgAARh/CSZ5a99qD/jW6ymVZlsPVAAAw+hBO8tTeaJ/WafHQuh4AgGIgnOQpEbXDSbu30uFKAAAYnQgneTLZcXUSAcbVAQCgGAgneXLF7HBiSmhdDwBAMRBO8uRt3ytJskq5jRgAgGIgnOQpmLDDibes2uFKAAAYnQgneSrtGFengnACAEAxEE7yYYwipkkSresBACgWwkkeMm2N8iolSYqMm+RwNQAAjE6Ekzy0ZLvDRk1IlZEyh6sBAGB0IpzkIbrHDid7rYh8HnYdAADFwDdsHtr22eGk2VXucCUAAIxehJM8JLLj6rR5GVcHAIBiIZzkId2yS5IU99O6HgCAYiGc5MG02q3rU0Fa1wMAUCyEkzx4YnvsGcbVAQCgaAgnefDHGyRJ7jLG1QEAoFgIJ3kIJbOt6yO0rgcAoFgIJ3kIpxslScFKusMCAFAshJP+SiVUplZJUlnVRIeLAQBg9CKv/AvDAAAgAElEQVSc9FM8avc4SRq3Kiu55gQAgGIhnPRTtGGHJGmvyhQO+RyuBgCA0Ytw0k8te3ZKkhpdFbIsy+FqAAAYvQgn/RRrssNJi5txdQAAKCbCST+lonbr+nZfpcOVAAAwuhFO+inTnL0gNsC4OgAAFBPhpJ9cbXZ32EyI1vUAABQT4aSfvO32uDpWKbcRAwBQTISTfgok7Nb13jCt6wEAKCbCST+VprPj6pQTTgAAKCbCSX8Yo0jGHlenrKrG4WIAABjdCCf9YNob5VNKkhQeRzgBAKCYCCf90NEdNmqCqoyEHa4GAIDRjXDSD9E9dZKkfVZEfo/b4WoAABjdCCf90LrXDifNLlrXAwBQbISTfkg02d1h27wVDlcCAMDoRzjph3RzdlwdP91hAQAoNsJJP5jW3ZKkVJBxdQAAKDbCST94Yva4OioZ72whAACMAYSTfvDH7XF1PGWEEwAAio1w0g+hpN263hehdT0AAMVGOOmHsrTduj5YTndYAACKjXDSl1RCYbVIkspoXQ8AQNERTvoQz95GnDIuVVZNcLgaAABGP8JJH6INOyRJexVWJOR3uBoAAEY/wkkfWrOD/jW5IrIsy+FqAAAY/QgnfYg12uGk2V3pcCUAAIwNhJM+JKP2NScxH+PqAAAwFAgnfci02OEkGaB1PQAAQ4Fw0gdXmz2uTibEoH8AAAwFwkkfvO1263qrlNuIAQAYCgMKJ8uXL1dtba0CgYDmzp2rN998s1/brV27Vh6PR3PmzBnIr3VEILFXkuQpI5wAADAU8g4nTz31lJYsWaKlS5dq/fr1mj17thYsWKBdu3b1ul1jY6MuvfRSnXnmmQMu1gklqY7W9RMdrgQAgLEh73Byzz336Morr9TixYt1zDHH6OGHH1YoFNKKFSt63e6qq67SJZdconnz5g242CFnjMoz9qB/pZW0rgcAYCjkFU4SiYTefvttzZ8/f/8LuFyaP3++1q1b1+N2jz76qD766CMtXbq0X78nHo8rGo12mZxg2pvkU0qSFB5POAEAYCjkFU4aGhqUTqdVXV3dZXl1dbV27tzZ7TabN2/WTTfdpB//+MfyeDz9+j133nmnIpFIbpoyZUo+ZRZMy95sAzYTVGUk4kgNAACMNUW9WyedTuuSSy7RsmXLdMQRR/R7u5tvvllNTU25afv27UWssmfRPXWSpH0KK+B1O1IDAABjTf8OZWSNGzdObrdb9fX1XZbX19dr4sSDLxhtbm7WW2+9pXfeeUfXXHONJCmTycgYI4/Ho//93//V5z73uYO28/v98vudH2SvLXvkJOoud7gSAADGjryOnPh8Pp144olavXp1blkmk9Hq1au7vdA1HA7r3Xff1YYNG3LTVVddpSOPPFIbNmzQ3LlzB/8OiijeZIeTNi/j6gAAMFTyOnIiSUuWLNGiRYt00kkn6ZRTTtF9992n1tZWLV68WJJ9SuZPf/qTnnjiCblcLh133HFdtp8wYYICgcBBy4ejVLN9e3TcRzgBAGCo5B1OFi5cqN27d+uWW27Rzp07NWfOHK1cuTJ3kWxdXZ22bdtW8EId0Wq3rk8GaV0PAMBQsYwxxuki+hKNRhWJRNTU1KRwODxkv/fd+y7QzMZX9PK0b+lzi24Zst8LAMBoMNDvb8bW6YW/ndb1AAAMNcJJL0JJO5z4ItV9rAkAAAqFcNKLsnR2XJ0KusMCADBUCCc9SScVUbMkqayKcAIAwFAhnPQgEbXv1EkbSxVVnNYBAGCoEE56EG3YIUnaq7AiIee71QIAMFYQTnrQstceV6fRisjlshyuBgCAsYNw0oNYY3ZEYg/dYQEAGEqEkx4km+zBDdt9FQ5XAgDA2EI46UGmJdu6PkDregAAhhLhpAeutuzdOsEqhysBAGBsIZz0wNO+R5JkldK6HgCAoUQ46UEgkR1XJ0w4AQBgKBFOelCa2idJCpTTgA0AgKFEOOmOMYpkmiRJpZW0rgcAYCgRTrph4lH5lZAkRcZNcrgaAADGFsJJN1r22Q3YWkxAleXlDlcDAMDYQjjpRnS3Pa7OPkUU8LodrgYAgLGFcNKNtn12d9iom6MmAAAMNcJJNxJN9qB/rV5a1wMAMNQIJ91IRe3usHEfg/4BADDUCCfdMK3ZcXWCjKsDAMBQI5x0wx1rsGdKxjtbCAAAYxDhpBv+uN263l1KOAEAYKgRTroRStrhxBehdT0AAEONcNKNsrQ9rk6wfKLDlQAAMPYQTg6UTilimiVJYVrXAwAw5AgnB0i22HfqpI2liipO6wAAMNQIJweINnS0ri9TpCTgcDUAAIw9hJMDNO+xw0mjVS6Xy3K4GgAAxh7CyQHas+PqNHtoXQ8AgBMIJwdIRO1wEvPSuh4AACcQTg6QyV4QmwwQTgAAcALh5ABWW/ZunRDj6gAA4ATCyQG8sT2SJItxdQAAcATh5ACBhN263hOmOywAAE4gnBygJJVtXV9BAzYAAJxAOOnMGEUyjZKkkgqOnAAA4ATCSScm3qyAEpIYVwcAAKcQTjpp3bfT/mn8qiynCRsAAE4gnHQSbaiTJO1VREGf2+FqAAAYmwgnnbTts8NJ1F3ucCUAAIxdhJNO2ht3SZLaGFcHAADHEE46STfb4+q0+2ldDwCAUwgnnZhWu3V9KkjregAAnEI46cTd1iBJMiFa1wMA4BTCSSe+uN263l1KOAEAwCmEk05CSXvQP1+E1vUAADiFcNJJWdpuXR+kdT0AAI4hnHRIpxQ2zZKksipa1wMA4BTCSVaypUEuGWWMpYpxHDkBAMAphJOsaMMOSdJelam8JOBwNQAAjF2Ek6yWPXbr+iYrIpfLcrgaAADGLsJJVqzRHpE4Sut6AAAcRTjJSkTt1vUxL63rAQBwksfpAoaLTLPduj4ZqHK4EgAYezKZjBKJhNNlIE9er1dut7vgr0s4ybKyrevTjKsDAEMqkUho69atymQyTpeCASgvL9fEiRNlWYW7XpNwkuWN2eHEKiWcAMBQMcaorq5ObrdbU6ZMkcvF1QYjhTFGbW1t2rVrlySppqamYK9NOMnyJ+xxdTxhWtcDwFBJpVJqa2vTpEmTFAqFnC4HeQoGg5KkXbt2acKECQU7xUNEzSpN7ZMkBSI0YAOAoZJOpyVJPp/P4UowUB2hMplMFuw1CSdZ4Yw9rk5JZeEOSwEA+qeQ1ytgaBXjb0c4kWTizQoqLkkKVxFOAABwEuFEUtu+bI8T41NlRbnD1QAAMLYRTrR/XJ09iijk9zpcDQBgLKmtrdV9993ndBnDCnfrSGrdZ4+rE3Vz1AQA0LczzjhDc+bMKUio+O1vf6uSkpICVDV6EE4ktTfa92i3Mq4OAKAAjDFKp9PyePr+mh0/fvwQVDSycFpHUjo7rk67j3F1AMBJxhi1JVKOTMaYftV42WWX6dVXX9X9998vy7JkWZYee+wxWZal//mf/9GJJ54ov9+v119/XVu2bNH555+v6upqlZaW6uSTT9ZLL73U5fUOPK1jWZb+/d//XRdeeKFCoZBmzJihX/7yl/2qLZ1O62tf+5qmTZumYDCoI488Uvfff/9B661YsULHHnus/H6/ampqdM011+Sea2xs1Ne//nVVV1crEAjouOOO03//93/36/cXCkdOJJlWe1ydFK3rAcBRsWRax9zyoiO/+4PbFijk6/tr8f7779emTZt03HHH6bbbbpMkvf/++5Kkm266SXfddZcOO+wwVVRUaPv27frCF76gf/qnf5Lf79cTTzyh8847Txs3btShhx7a4+9YtmyZvv/97+sHP/iBHnzwQX3lK1/RJ598osrK3v8RnclkNHnyZD3zzDOqqqrSb37zG/3t3/6tampqdNFFF0mSHnroIS1ZskT//M//rC984Qtqbm7W66+/ntv+nHPOUXNzs3784x/r8MMP18aNG/u1/wqJcCLJnR1Xx5RwaA0A0LtIJCKfz6dQKKSJE+3GnX/4wx8kSbfddpvOOuus3LqVlZWaPXt27vE//uM/6rnnntMvf/nLLkcrDnTZZZfp4osvliTdcccdeuCBB/Tmm2/q85//fK+1eb1eLVu2LPd42rRpWrdunZ5++ulcOLn99tv1rW99S9dee21uvRNOOEGS9NJLL+nNN9/Uhx9+qCOOOEKSdNhhh/W9UwqMcCLJF98jSXKXEk4AwElBr1sf3LbAsd89WCeddFKXxy0tLbr11lv1wgsvqK6uTqlUSrFYTNu2bev1dWbNmpWbLykpUTgczo1h05fly5drxYoV2rZtm2KxmBKJhObMmSPJbjO/Y8cOnXnmmd1uu2HDBk2ePDkXTJxCOJEUTNqt670RxtUBACdZltWvUyvD1YF33Vx//fVatWqV7rrrLk2fPl3BYFBf+tKXlEgken0dr7drWwvLsvo1avOTTz6p66+/XnfffbfmzZunsrIy/eAHP9Abb7whaf9YOD3p6/mhMqALYpcvX67a2loFAgHNnTtXb775Zo/r/vznP9dZZ52l8ePHKxwOa968eXrxRWfOJ/akLG2Hk1A54+oAAPrm8/ly4wL1Zu3atbrssst04YUXaubMmZo4caI+/vjjotW1du1anXrqqfrmN7+p448/XtOnT9eWLVtyz5eVlam2tlarV6/udvtZs2bp008/1aZNm4pWY3/kHU6eeuopLVmyREuXLtX69es1e/ZsLViwoMfDTa+99prOOuss/epXv9Lbb7+tz372szrvvPP0zjvvDLr4gsikFTFRSVJp1SSHiwEAjAS1tbV644039PHHH6uhoaHHoxozZszQz3/+c23YsEG/+93vdMkll/TrCMhAzZgxQ2+99ZZefPFFbdq0Sd/97nf129/+tss6t956q+6++2498MAD2rx5s9avX68HH3xQknT66afrL/7iL/TXf/3XWrVqlbZu3ar/+Z//0cqVK4tWc3fyDif33HOPrrzySi1evFjHHHOMHn74YYVCIa1YsaLb9e+77z7dcMMNOvnkkzVjxgzdcccdmjFjhv7rv/6rx98Rj8cVjUa7TMWSammQS0YZY6liHKd1AAB9u/766+V2u3XMMcdo/PjxPV5Dcs8996iiokKnnnqqzjvvPC1YsCB38WkxfP3rX9cXv/hFLVy4UHPnztWePXv0zW9+s8s6ixYt0n333acf/vCHOvbYY/WXf/mX2rx5c+75Z599VieffLIuvvhiHXPMMbrhhhv6dZSokCzT3xu7JSUSCYVCIf3sZz/TBRdckFu+aNEiNTY26he/+EWfr5HJZFRbW6sbbrihxyuVb7311i5XG3doampSOBzub7n9smfrO6p6/AztMWUqX7pdbhcjYwLAUGlvb9fWrVs1bdo0BQIBp8vBAPT2N4xGo4pEInl/f+d15KShoUHpdFrV1V2PMFRXV2vnzp39eo277rpLLS0tuVuaunPzzTerqakpN23fvj2fMvPS0mC3rm+0IgQTAACGgSHtEPuf//mfWrZsmZ5++mlNmDChx/X8fr/C4XCXqVhijXaoanHTuh4AMLxdddVVKi0t7Xa66qqrnC6vYPK6X2vcuHFyu92qr6/vsry+vj7XiKYnTz75pK644go988wzmj9/fv6VFkmiyb6Qt43W9QCAYe62227T9ddf3+1zxfyH/FDLK5z4fD6deOKJWr16de6ak0wmo9WrV/fa6e6nP/2pLr/8cj355JM699xzB1dxgWVa7HCS8Fc5XAkAAL2bMGFCr2ceRou8O90sWbJEixYt0kknnaRTTjlF9913n1pbW7V48WJJ9vUif/rTn/TEE09Isk/lLFq0SPfff7/mzp2buzYlGAwqEokU8K0MjJVtXZ8OEU4AABgO8g4nCxcu1O7du3XLLbdo586dmjNnjlauXJm7SLaurq7LLVX/+q//qlQqpauvvlpXX311bvmiRYv02GOPDf4dDJI3Zg/6Z5WM/iQKAMBIMKAewddcc02Pp3EODBxr1qwZyK8YMv74XkmSJ0w4AQBgOBjSu3WGo5KU3bo+EKF1PQAAw8GYDyfhTJMkKVRJOAEAYDgY0+HExFsUUrskKTKOcXUAABgOxnQ4iTXa/VrajVeVFTRhAwD0zxlnnKHrrruuYK932WWXdRkWZqwb0+GkqWGHJGmPyhXyex2uBgAASGM8nLTutXuuNLmd77cCAJBkjJRodWbq5zi4l112mV599VXdf//9sixLlmXp448/1nvvvadzzjlHpaWlqq6u1le/+lU1NDTktvvZz36mmTNnKhgMqqqqSvPnz1dra6tuvfVWPf744/rFL36Re73+3Ol644036ogjjlAoFNJhhx2m7373u0omk13W+a//+i+dfPLJCgQCGjdunC688MLcc/F4XDfeeKOmTJkiv9+v6dOn65FHHunf36nIBnQr8WgRb7LDSauHUzoAMCwk26Q7HLoG8P/bIflK+lzt/vvv16ZNm3TcccfptttukyR5vV6dcsopuuKKK3TvvfcqFovpxhtv1EUXXaSXX35ZdXV1uvjii/X9739fF154oZqbm/XrX/9axhhdf/31+vDDDxWNRvXoo49Kkior+x5SpaysTI899pgmTZqkd999V1deeaXKysp0ww03SJJeeOEFXXjhhfrOd76jJ554QqlUSr/61a9y21966aVat26dHnjgAc2ePVvbtm07aHgap4zpcJKK2q3r44yrAwDop0gkIp/Pp1AolBtX7vbbb9fxxx+vO+64I7feihUrNGXKFG3atEktLS1KpVL64he/qKlTp0qSZs6cmVs3GAwqHo/3OU5dZ//v//2/3Hxtba2uv/56Pfnkk7lw8k//9E/68pe/rGXLluXW6/idmzZt0tNPP61Vq1blxrs77LDD8t0VRTOmw4lpsbvDJgPjHK4EACBJ8obsIxhO/e4B+t3vfqdXXnlFpaWlBz23ZcsWnX322TrzzDM1c+ZMLViwQGeffba+9KUvqWIQN2M89dRTeuCBB7Rly5Zc+Ok8+N+GDRt05ZVXdrvthg0b5Ha7dfrppw/49xfTmA4n7ph9LtCUjHe4EgCAJMmy+nVqZbhpaWnReeedp+9973sHPVdTUyO3261Vq1bpN7/5jf73f/9XDz74oL7zne/ojTfe0LRp0/L+fevWrdNXvvIVLVu2TAsWLFAkEtGTTz6pu+++O7dOMBjscfvenhsOxvQFse96Z+nZ9GcUrzrG6VIAACOIz+dTOp3OPT7hhBP0/vvvq7a2VtOnT+8ylZTYYcuyLJ122mlatmyZ3nnnHfl8Pj333HPdvl5ffvOb32jq1Kn6zne+o5NOOkkzZszQJ5980mWdWbNmafXq1d1uP3PmTGUyGb366qv5vvUhMabDSeTPr9TmU+9S9ZyznS4FADCC1NbW6o033tDHH3+shoYGXX311dq7d68uvvhi/fa3v9WWLVv04osvavHixUqn03rjjTd0xx136K233tK2bdv085//XLt379bRRx+de73f//732rhxoxoaGg666+ZAM2bM0LZt2/Tkk09qy5YteuCBB3JBp8PSpUv105/+VEuXLtWHH36od999N3dkp7a2VosWLdLll1+u559/Xlu3btWaNWv09NNPF2eH5cuMAE1NTUaSaWpqcroUAEABxWIx88EHH5hYLOZ0KXnZuHGj+bM/+zMTDAaNJLN161azadMmc+GFF5ry8nITDAbNUUcdZa677jqTyWTMBx98YBYsWGDGjx9v/H6/OeKII8yDDz6Ye71du3aZs846y5SWlhpJ5pVXXumzhm9/+9umqqrKlJaWmoULF5p7773XRCKRLus8++yzZs6cOcbn85lx48aZL37xi7nnYrGY+Yd/+AdTU1NjfD6fmT59ulmxYkXe+6K3v+FAv78tY/p5Y7eDotGoIpGImpqaulzsAwAY2drb27V161ZNmzZNgUDA6XIwAL39DQf6/T2mT+sAAIDhh3ACAMAwc8cdd6i0tLTb6ZxzznG6vKIb07cSAwAwHF111VW66KKLun1uuN8GXAiEEwAAhpnKysp+tbAfrTitAwBw3Ai4NwM9yGQyBX9NjpwAABzj9XplWZZ2796t8ePHy7Isp0tCPxljlEgktHv3brlcLvl8voK9NuEEAOAYt9utyZMn69NPP9XHH3/sdDkYgFAopEMPPVQuV+FOxhBOAACOKi0t1YwZM/rsiorhx+12y+PxFPyIF+EEAOA4t9stt9vtdBkYJrggFgAADCuEEwAAMKwQTgAAwLAyIq456bj/PRqNOlwJAADor47v7Xz72IyIcNLc3CxJmjJlisOVAACAfDU3NysSifR7fcuMgLZ8mUxGO3bsUFlZ2Zhq0BONRjVlyhRt3749r6GmRxv2w37sCxv7wcZ+2I99YRtu+8EYo+bmZk2aNCmvPigj4siJy+XS5MmTnS7DMeFweFh8yJzGftiPfWFjP9jYD/uxL2zDaT/kc8SkAxfEAgCAYYVwAgAAhhX3rbfeeqvTRaBnbrdbZ5xxhjyeEXEGrmjYD/uxL2zsBxv7YT/2hW007IcRcUEsAAAYOzitAwAAhhXCCQAAGFYIJwAAYFghnAAAgGGFcAIAAIYVwolD7rzzTp188skqKyvThAkTdMEFF2jjxo29brNmzRpZlnXQtHPnziGquvBuvfXWg97PUUcd1es2a9as0QknnCC/36/p06frscceG5pii6y2trbbv+/VV1/d7fqj5fPw2muv6bzzztOkSZNkWZaef/75Ls8bY3TLLbeopqZGwWBQ8+fP1+bNm/t83WeeeUZHHXWUAoGAZs6cqV/96lfFegsF0dt+SCaTuvHGGzVz5kyVlJRo0qRJuvTSS7Vjx45eX/Oxxx476PMRCASK/VYGra/PxGWXXXbQ+/r85z/f5+uOps+EpG7/+7csSz/4wQ96fM2R8pkgnDjk1Vdf1dVXX63/+7//06pVq5RMJnX22WertbW1z203btyourq63DRhwoQhqLh4jj322C7v5/XXX+9x3a1bt+rcc8/VZz/7WW3YsEHXXXedrrjiCr344otDWHFx/Pa3v+2yH1atWiVJ+pu/+Ztetxvpn4fW1lbNnj1by5cv7/b573//+3rggQf08MMP64033lBJSYkWLFig9vb2Hl/zN7/5jS6++GJ97Wtf0zvvvKMLLrhAF1xwgd57771ivY1B620/tLW1af369frud7+r9evX6+c//7k2btyov/qrv+rzdcPhcJfPxyeffFKM8guqr8+EJH3+85/v8r5++tOf9vqao+0zIanL+6+rq9OKFStkWZb++q//utfXHRGfCYNhYdeuXUaSefXVV3tc55VXXjGSzL59+4awsuJaunSpmT17dr/Xv+GGG8yxxx7bZdnChQvNggULCl2a46699lpz+OGHm0wm0+3zo/HzIMk899xzuceZTMZMnDjR/OAHP8gta2xsNH6/3/z0pz/t8XUuuugic+6553ZZNnfuXPP1r3+98EUXwYH7oTtvvvmmkWQ++eSTHtd59NFHTSQSKXR5Q6q7fbFo0SJz/vnn5/U6Y+Ezcf7555vPfe5zva4zUj4THDkZJpqamiRJlZWVfa47Z84c1dTU6KyzztLatWuLXVrRbd68WZMmTdJhhx2mr3zlK9q2bVuP665bt07z58/vsmzBggVat25dscscUolEQj/+8Y91+eWX9zkS92j7PHS2detW7dy5s8vfPBKJaO7cub3+zcfC56SpqUmWZam8vLzX9VpaWjR16lRNmTJF559/vt5///0hqrC41qxZowkTJujII4/UN77xDe3Zs6fX9Uf7Z6K+vl4vvPCCvva1r/W57kj4TBBOhoFMJqPrrrtOp512mo477rge16upqdHDDz+sZ599Vs8++6ymTJmiM844Q+vXrx/Cagtr7ty5euyxx7Ry5Uo99NBD2rp1qz7zmc+oubm52/V37typ6urqLsuqq6sVjUYVi8WGouQh8fzzz6uxsVGXXXZZj+uMxs/DgTqun+nub97btTU9fU5G2vU4PWlvb9eNN96oiy++uNeRZ4888kitWLFCv/jFL/TjH/9YmUxGp556qj799NMhrLbwPv/5z+uJJ57Q6tWr9b3vfU+vvvqqzjnnHKXT6R63Ge2ficcff1xlZWX64he/2Ot6I+UzMXIb748iV199td57771er7WQ7A/VkUcemXt86qmnasuWLbr33nv1H//xH8UusyjOOeec3PysWbM0d+5cTZ06VU8//XS//gUwWj3yyCM655xzNGnSpB7XGY2fB/QtmUzqoosukjFGDz30UK/rzps3T/Pmzcs9PvXUU3X00UfrRz/6kf7xH/+x2KUWzZe//OXc/MyZMzVr1iwdfvjhWrNmjc4880wHK3POihUr9JWvfKXPi1tHymeCIycOu+aaa/Tf//3feuWVVzR58uS8tz/llFP0xz/+sQiVOaO8vFxHHHFEj+9p4sSJqq+v77Ksvr5e4XBYwWBwKEosuk8++UQvvfSSrrjiiry3HW2fh4kTJ0pSt3/zjud62i7fbUaCjmDyySefaNWqVb0eNemO1+vV8ccfP6o+I5J02GGHady4cb2+r9H6mZCkX//619q4ceOA/p8xXD8ThBOHGGN0zTXX6LnnntPLL7+sadOmDeh1NmzYoJqamgJX55yWlhb98Y9/7PE9zZs3T6tXr+6ybNWqVV3+JTDSPfroo5owYYLOPffcvLcdbZ+HadOmaeLEiV3+5tFoVG+88Uavf/PR+DnpCCabN2/WSy+9pKqqqrxfI51O69133x1VnxFJ+vTTT7Vnz55e39do/Ex0eOSRR3TiiSdq9uzZeW87bD8TTl+RO1Z94xvfMJFIxKxZs8bU1dXlpra2ttw6N910k/nqV7+ae3zvvfea559/3mzevNm8++675tprrzUul8u89NJLTryFgvjWt75l1qxZY7Zu3WrWrl1r5s+fb8aNG2d27dpljDl4H3z00UcmFAqZb3/72+bDDz80y5cvN26326xcudKpt1BQ6QIQt1wAAAJbSURBVHTaHHrooebGG2886LnR+nlobm4277zzjnnnnXeMJHPPPfeYd955J3cXyj//8z+b8vJy84tf/ML8/ve/N+eff76ZNm2aicViudf46le/am666abc47Vr1xqPx2Puuusu8+GHH5qlS5car9dr3n333SF/f/3V235IJBLmr/7qr8zkyZPNhg0buvw/Ix6P517jwP2wbNky8+KLL5otW7aYt99+23z5y182gUDAvP/++068xX7rbV80Nzeb66+/3qxbt85s3brVvPTSS+aEE04wM2bMMO3t7bnXGO2fiQ5NTU0mFAqZhx56qNvXGKmfCcKJQyR1Oz366KO5dRYtWmROP/303OPvfe975vDDDzeBQMBUVlaaM844w7z88stDX3wBLVy40NTU1Bifz2cOOeQQs3DhQvPHP/4x9/yB+8AY+xbaOXPmGJ/PZw477LAu+2yke/HFF40ks3HjxoOeG62fh45bog+cFi1aZIyxbyf+7ne/a6qrq43f7zdnnnnmQfvn9NNPz63f4emnnzZHHHGE8fl85thjjzUvvPDCEL2jgeltP2zdurXH/2e88sorudc4cD9cd9115tBDDzU+n89UV1ebL3zhC2b9+vVD/+by1Nu+aGtrM2effbYZP3688Xq9ZurUqebKK680O3fu7PIao/0z0eFHP/qRCQaDprGxsdvXGKmfCcsYY4p6aAYAACAPXHMCAACGFcIJAAAYVggnAABgWCGcAACAYYVwAgAAhhXCCQAAGFYIJwAAYFghnAAAgGGFcAIAAIYVwgkAABhWCCcAAGBY+f8BBpbnprbV5gMAAAAASUVORK5CYII=", "text/plain": [ "PyPlot.Figure(PyObject )" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/plain": [ "PyObject " ] }, "execution_count": 57, "metadata": {}, "output_type": "execute_result" } ], "source": [ "x1 = [1:length(train_acc_list);]\n", "plt.plot(x1, train_acc_list, label=\"train_acc\")\n", "plt.plot(x1, test_acc_list, label=\"test_acc\")\n", "plt.legend()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Julia 0.6.0", "language": "julia", "name": "julia-0.6" }, "language_info": { "file_extension": ".jl", "mimetype": "application/julia", "name": "julia", "version": "0.6.0" } }, "nbformat": 4, "nbformat_minor": 2 }