#+TITLE: fanfic_classifier * @todo ** Batch normalization? ** Try recurrent cells ** tokenization *** naive Bayes ** DONE Dropout ** DONE Shuffle the training data ** DONE Make the random split between the training and validation data deterministic based on a seed ** DONE Use =Float32= instead of =UInt8= for the characters? #+begin_example : ┌ Warning: Slow fallback implementation invoked for conv! You probably don't want this; check your datatypes. : │ yT = Float32 : │ T1 = UInt8 : │ T2 = Float32 #+end_example * Remotes ** Check the remote #+begin_src jupyter-julia :session (night/org-babel-session-name-get "j1") :async yes :pandoc t run(`hostname`) #+end_src #+RESULTS: :RESULTS: : daf087556f92 : Process(`hostname`, ProcessExited(0)) :END: ** Move data to the remote(s) #+begin_example zsh host=ubuntu@194.5.193.126 host_port=22 #+end_example #+begin_example zsh host=walle@51.178.215.202 host_port=2380 #+end_example #+begin_example zsh rsp-safe -e "ssh -p ${host_port:-22}" --rsync-path="mkdir -p ~/code/julia && rsync" ~cod/julia/NightCommon ${host}:code/julia/ rsp-safe -e "ssh -p ${host_port:-22}" --exclude=models --rsync-path="mkdir -p ~/base/archives && rsync" ~base/archives/nlp_data ${host}:base/archives/ #+end_example *** Get stuff from the remote #+begin_example zsh tmuxnewsh2 save-j1 lo_s=10 loop rsp-safe -e "ssh -p ${host_port:-22}" ${host}:'logs/j1' ./ #+end_example **** Models #+begin_example zsh tmuxnewsh2 save-all lo_s=10 loop rsp-safe -e "ssh -p ${host_port:-22}" ${host}:'base/arch ives/nlp_data/processBooks/' ./all/ #+end_example ***** Find the best model #+begin_example zsh c ls -clt **/*(.) | perl -lne '/accu=(.*)/ && print $1' | sort #+end_example ***** Copy #+begin_example zsh rsp-safe -e "ssh -p ${host_port:-22}" ${host}: ./ #+end_example **** To local #+begin_example zsh rsp-safe 'zii@51.178.215.202:base/saved_models/elizium_vs_luminary_m1/all/models/21-11-10 19:09/230000_accu=0.899' . #+end_example * GPU #+begin_src jupyter-julia :session (night/org-babel-session-name-get "j1") :async yes :pandoc t # import Pkg; Pkg.add("CUDA") #+end_src #+RESULTS: #+begin_src jupyter-julia :session (night/org-babel-session-name-get "j1") :async yes :pandoc t GC.gc(true) CUDA.reclaim() #+end_src #+RESULTS: #+begin_src jupyter-julia :session (night/org-babel-session-name-get "j1") :async yes :pandoc t CUDA.memory_status() #+end_src #+RESULTS: : Effective GPU memory usage: 69.73% (7.504 GiB/10.761 GiB) : Memory pool usage: 6.642 GiB (6.750 GiB reserved) *** _ #+begin_src jupyter-julia :session (night/org-babel-session-name-get "j1") :async yes :pandoc t [10 3 2] |> gpu #+end_src #+RESULTS: : 1×3 CuArray{Int64, 2, CUDA.Mem.DeviceBuffer}: : 10 3 2 #+begin_src jupyter-julia :session (night/org-babel-session-name-get "j1") :async yes :pandoc t [10 3 2] |> cu #+end_src #+RESULTS: : 1×3 CuArray{Int64, 2, CUDA.Mem.DeviceBuffer}: : 10 3 2 #+begin_src jupyter-julia :session (night/org-babel-session-name-get "j1") :async yes :pandoc t Float32.([10 3 2]) |> gpu #+end_src #+RESULTS: : 1×3 CuArray{Float32, 2, CUDA.Mem.DeviceBuffer}: : 10.0 3.0 2.0 *** _ #+begin_src jupyter-julia :session (night/org-babel-session-name-get "j1") :async yes :pandoc t ps #+end_src #+RESULTS: : Params([Float32[-0.111778125; 0.002819123; -0.027244896; -0.03988738] : : Float32[-0.18919002; -0.05827969; 0.12246067; -0.21493939] : : Float32[0.7942748; -1.4989135; -0.5318076; -0.7112862] : : ... : : Float32[-0.1343732; -0.008487504; -0.13199817; -0.009081438] : : Float32[0.41980788; -0.9344694; -0.5839063; 0.024473608] : : Float32[0.0014797748; -0.0027378402; -0.027659174; 0.0004863238], Float32[-0.12336355, -0.25500798, -0.004385097, -0.20307787, -1.6159056, -0.41346544, 0.0, -0.38553542, -0.005019192, 0.3411631 … -1.3360305, -1.7169491, -0.16208945, 0.30585626, -0.07662875, 0.0, -0.0040730415, -0.008172875, -0.005298142, -0.13486442], Float32[0.0046841805 -0.02351334 … 0.017824527 0.0055555017; -0.0045686923 0.03180616 … -0.0575844 -0.015381001; … ; -0.016207157 0.018622335 … -0.019795312 -0.01859735; -0.114179954 0.03541677 … -0.035208307 -0.06850835], Float32[0.036304567, -0.19586259, -0.26909864, -0.26280588, -0.31083316, -0.7186801, -0.08919432, -0.20911779, 0.0048079244, -0.017520554 … -0.86946017, -0.32969135, -0.037706837, 0.01641627, 0.08448448, -0.8033937, 0.06943873, -0.15353437, -0.03882099, -0.4388559], Float32[0.03218485 -0.16204661 … -0.014277367 0.028125435; 0.26154378 -0.800345 … 0.12894902 0.2691888; … ; -0.09592571 0.06082246 … 0.049979135 0.4438708; -0.028685609 -0.25797015 … -0.08898944 0.29308218], Float32[-0.025797812, -0.4099781, 0.3404507, -0.6801296, -0.42796856, 0.57648206, 0.2200871, 2.2506657, -0.34521192, 0.09023797 … -1.7964157, -0.2455771, -0.70767885, 0.17814818, -1.926309, -2.1069493, -0.08664764, -1.6999221, -0.3778407, 1.0155528], Float32[0.04485587 0.1187261 … 0.088028684 -0.15526259; 0.071831286 -0.16534011 … -0.08978507 0.12799264; … ; -0.07809766 0.2991697 … -0.43194246 0.007871836; 0.06602923 0.52628297 … 0.03215004 -0.29033986], Float32[-0.34063843, -0.16484079, 0.4424414, -0.11640712, -0.9418747, -0.15433855, -0.93596625, -0.12772629, -0.7169618, -0.31299147 … -0.50175637, -0.5899134, -4.7391863, -0.6534439, 0.0564369, 0.4552015, -0.32347828, -0.39680782, -0.5125247, -0.06212805], Float32[-0.014492278 -0.03157189 … 0.061014976 0.034709286], Float32[-0.5395494]]) #+begin_src jupyter-julia :session (night/org-babel-session-name-get "j1") :async yes :pandoc t cpu(ps) #+end_src #+RESULTS: :RESULTS: # [goto error] : MethodError: in(::Zygote.Params, ::Base.KeySet{Any, IdDict{Any, Any}}) is ambiguous. Candidates: : in(k, v::Base.KeySet{var"#s77", var"#s76"} where {var"#s77", var"#s76"<:IdDict}) in Base at iddict.jl:174 : in(k, v::Base.KeySet) in Base at abstractdict.jl:69 : in(x::Zygote.Params, args...; kwargs...) in Zygote at /home/walle/.julia/packages/MacroTools/PP9IQ/src/examples/forward.jl:17 : Possible fix, define : in(::Zygote.Params, ::Base.KeySet{var"#s77", T} where {var"#s77", T<:(IdDict{var"#s77", V} where V)}) : : Stacktrace: : [1] haskey(d::IdDict{Any, Any}, k::Zygote.Params) : @ Base ./abstractdict.jl:17 : [2] fmap(f::Flux.var"#130#131", x::Zygote.Params; exclude::typeof(Functors.isleaf), walk::typeof(Functors._default_walk), cache::IdDict{Any, Any}) : @ Functors ~/.julia/packages/Functors/hIysk/src/functor.jl:120 : [3] fmap(f::Function, x::Zygote.Params) : @ Functors ~/.julia/packages/Functors/hIysk/src/functor.jl:120 : [4] cpu(x::Zygote.Params) : @ Flux ~/.julia/packages/Flux/BPPNj/src/functor.jl:115 : [5] top-level scope : @ In[35]:1 : [6] eval : @ ./boot.jl:360 [inlined] : [7] include_string(mapexpr::typeof(REPL.softscope), mod::Module, code::String, filename::String) : @ Base ./loading.jl:1116 :END: #+begin_src jupyter-julia :session (night/org-babel-session-name-get "j1") :async yes :pandoc t Flux.params(model1) #+end_src #+RESULTS: : Params([Float32[-0.111778125; 0.002819123; -0.027244896; -0.03988738] : : Float32[-0.18919002; -0.05827969; 0.12246067; -0.21493939] : : Float32[0.7942748; -1.4989135; -0.5318076; -0.7112862] : : ... : : Float32[-0.1343732; -0.008487504; -0.13199817; -0.009081438] : : Float32[0.41980788; -0.9344694; -0.5839063; 0.024473608] : : Float32[0.0014797748; -0.0027378402; -0.027659174; 0.0004863238], Float32[-0.12336355, -0.25500798, -0.004385097, -0.20307787, -1.6159056, -0.41346544, 0.0, -0.38553542, -0.005019192, 0.3411631 … -1.3360305, -1.7169491, -0.16208945, 0.30585626, -0.07662875, 0.0, -0.0040730415, -0.008172875, -0.005298142, -0.13486442], Float32[0.0046841805 -0.02351334 … 0.017824527 0.0055555017; -0.0045686923 0.03180616 … -0.0575844 -0.015381001; … ; -0.016207157 0.018622335 … -0.019795312 -0.01859735; -0.114179954 0.03541677 … -0.035208307 -0.06850835], Float32[0.036304567, -0.19586259, -0.26909864, -0.26280588, -0.31083316, -0.7186801, -0.08919432, -0.20911779, 0.0048079244, -0.017520554 … -0.86946017, -0.32969135, -0.037706837, 0.01641627, 0.08448448, -0.8033937, 0.06943873, -0.15353437, -0.03882099, -0.4388559], Float32[0.03218485 -0.16204661 … -0.014277367 0.028125435; 0.26154378 -0.800345 … 0.12894902 0.2691888; … ; -0.09592571 0.06082246 … 0.049979135 0.4438708; -0.028685609 -0.25797015 … -0.08898944 0.29308218], Float32[-0.025797812, -0.4099781, 0.3404507, -0.6801296, -0.42796856, 0.57648206, 0.2200871, 2.2506657, -0.34521192, 0.09023797 … -1.7964157, -0.2455771, -0.70767885, 0.17814818, -1.926309, -2.1069493, -0.08664764, -1.6999221, -0.3778407, 1.0155528], Float32[0.04485587 0.1187261 … 0.088028684 -0.15526259; 0.071831286 -0.16534011 … -0.08978507 0.12799264; … ; -0.07809766 0.2991697 … -0.43194246 0.007871836; 0.06602923 0.52628297 … 0.03215004 -0.29033986], Float32[-0.34063843, -0.16484079, 0.4424414, -0.11640712, -0.9418747, -0.15433855, -0.93596625, -0.12772629, -0.7169618, -0.31299147 … -0.50175637, -0.5899134, -4.7391863, -0.6534439, 0.0564369, 0.4552015, -0.32347828, -0.39680782, -0.5125247, -0.06212805], Float32[-0.014492278 -0.03157189 … 0.061014976 0.034709286], Float32[-0.5395494]]) #+begin_src jupyter-julia :session (night/org-babel-session-name-get "j1") :async yes :pandoc t cpu(Flux.params(model1)) #+end_src #+RESULTS: :RESULTS: # [goto error] : MethodError: in(::Zygote.Params, ::Base.KeySet{Any, IdDict{Any, Any}}) is ambiguous. Candidates: : in(k, v::Base.KeySet{var"#s77", var"#s76"} where {var"#s77", var"#s76"<:IdDict}) in Base at iddict.jl:174 : in(k, v::Base.KeySet) in Base at abstractdict.jl:69 : in(x::Zygote.Params, args...; kwargs...) in Zygote at /home/walle/.julia/packages/MacroTools/PP9IQ/src/examples/forward.jl:17 : Possible fix, define : in(::Zygote.Params, ::Base.KeySet{var"#s77", T} where {var"#s77", T<:(IdDict{var"#s77", V} where V)}) : : Stacktrace: : [1] haskey(d::IdDict{Any, Any}, k::Zygote.Params) : @ Base ./abstractdict.jl:17 : [2] fmap(f::Flux.var"#130#131", x::Zygote.Params; exclude::typeof(Functors.isleaf), walk::typeof(Functors._default_walk), cache::IdDict{Any, Any}) : @ Functors ~/.julia/packages/Functors/hIysk/src/functor.jl:120 : [3] fmap(f::Function, x::Zygote.Params) : @ Functors ~/.julia/packages/Functors/hIysk/src/functor.jl:120 : [4] cpu(x::Zygote.Params) : @ Flux ~/.julia/packages/Flux/BPPNj/src/functor.jl:115 : [5] top-level scope : @ In[38]:1 : [6] eval : @ ./boot.jl:360 [inlined] : [7] include_string(mapexpr::typeof(REPL.softscope), mod::Module, code::String, filename::String) : @ Base ./loading.jl:1116 :END: #+begin_src jupyter-julia :session (night/org-babel-session-name-get "j1") :async yes :pandoc t Flux.params(cpu(model1)) #+end_src #+RESULTS: : Params([Float32[-0.111778125; 0.002819123; -0.027244896; -0.03988738] : : Float32[-0.18919002; -0.05827969; 0.12246067; -0.21493939] : : Float32[0.7942748; -1.4989135; -0.5318076; -0.7112862] : : ... : : Float32[-0.1343732; -0.008487504; -0.13199817; -0.009081438] : : Float32[0.41980788; -0.9344694; -0.5839063; 0.024473608] : : Float32[0.0014797748; -0.0027378402; -0.027659174; 0.0004863238], Float32[-0.12336355, -0.25500798, -0.004385097, -0.20307787, -1.6159056, -0.41346544, 0.0, -0.38553542, -0.005019192, 0.3411631 … -1.3360305, -1.7169491, -0.16208945, 0.30585626, -0.07662875, 0.0, -0.0040730415, -0.008172875, -0.005298142, -0.13486442], Float32[0.0046841805 -0.02351334 … 0.017824527 0.0055555017; -0.0045686923 0.03180616 … -0.0575844 -0.015381001; … ; -0.016207157 0.018622335 … -0.019795312 -0.01859735; -0.114179954 0.03541677 … -0.035208307 -0.06850835], Float32[0.036304567, -0.19586259, -0.26909864, -0.26280588, -0.31083316, -0.7186801, -0.08919432, -0.20911779, 0.0048079244, -0.017520554 … -0.86946017, -0.32969135, -0.037706837, 0.01641627, 0.08448448, -0.8033937, 0.06943873, -0.15353437, -0.03882099, -0.4388559], Float32[0.03218485 -0.16204661 … -0.014277367 0.028125435; 0.26154378 -0.800345 … 0.12894902 0.2691888; … ; -0.09592571 0.06082246 … 0.049979135 0.4438708; -0.028685609 -0.25797015 … -0.08898944 0.29308218], Float32[-0.025797812, -0.4099781, 0.3404507, -0.6801296, -0.42796856, 0.57648206, 0.2200871, 2.2506657, -0.34521192, 0.09023797 … -1.7964157, -0.2455771, -0.70767885, 0.17814818, -1.926309, -2.1069493, -0.08664764, -1.6999221, -0.3778407, 1.0155528], Float32[0.04485587 0.1187261 … 0.088028684 -0.15526259; 0.071831286 -0.16534011 … -0.08978507 0.12799264; … ; -0.07809766 0.2991697 … -0.43194246 0.007871836; 0.06602923 0.52628297 … 0.03215004 -0.29033986], Float32[-0.34063843, -0.16484079, 0.4424414, -0.11640712, -0.9418747, -0.15433855, -0.93596625, -0.12772629, -0.7169618, -0.31299147 … -0.50175637, -0.5899134, -4.7391863, -0.6534439, 0.0564369, 0.4552015, -0.32347828, -0.39680782, -0.5125247, -0.06212805], Float32[-0.014492278 -0.03157189 … 0.061014976 0.034709286], Float32[-0.5395494]]) *** @broken Use =IdDict= instead :PROPERTIES: :visibility: folded :END: #+begin_src jupyter-julia :session (night/org-babel-session-name-get "j1") :async yes :pandoc t is = Dict{Book, Int}() is[elizium] = 2 #+end_src #+RESULTS: :RESULTS: # [goto error] : Scalar indexing is disallowed. : Invocation of getindex resulted in scalar indexing of a GPU array. : This is typically caused by calling an iterating implementation of a method. : Such implementations *do not* execute on the GPU, but very slowly on the CPU, : and therefore are only permitted from the REPL for prototyping purposes. : If you did intend to index this array, annotate the caller with @allowscalar. : : Stacktrace: : [1] error(s::String) : @ Base ./error.jl:33 : [2] assertscalar(op::String) : @ GPUArrays ~/.julia/packages/GPUArrays/3sW6s/src/host/indexing.jl:53 : [3] getindex : @ ~/.julia/packages/GPUArrays/3sW6s/src/host/indexing.jl:86 [inlined] : [4] getindex : @ ./subarray.jl:309 [inlined] : [5] hash(A::SubArray{Float32, 1, CuArray{Float32, 1, CUDA.Mem.DeviceBuffer}, Tuple{UnitRange{Int64}}, true}, h::UInt64) : @ Base ./abstractarray.jl:2432 : [6] hash : @ ./pair.jl:52 [inlined] : [7] hash(A::Vector{SubArray{Float32, 1, CuArray{Float32, 1, CUDA.Mem.DeviceBuffer}, Tuple{UnitRange{Int64}}, true}}, h::UInt64) : @ Base ./abstractarray.jl:2433 : [8] hash(x::Vector{SubArray{Float32, 1, CuArray{Float32, 1, CUDA.Mem.DeviceBuffer}, Tuple{UnitRange{Int64}}, true}}) : @ Base ./hashing.jl:18 : [9] hash(a::Dict{Symbol, Any}, h::UInt64) : @ Base ./abstractdict.jl:489 : [10] hash : @ ./hashing.jl:18 [inlined] : [11] hashindex : @ ./dict.jl:169 [inlined] : [12] ht_keyindex2!(h::Dict{Dict{Symbol, Any}, Int64}, key::Dict{Symbol, Any}) : @ Base ./dict.jl:310 : [13] setindex!(h::Dict{Dict{Symbol, Any}, Int64}, v0::Int64, key::Dict{Symbol, Any}) : @ Base ./dict.jl:383 : [14] top-level scope : @ In[52]:2 : [15] eval : @ ./boot.jl:360 [inlined] : [16] include_string(mapexpr::typeof(REPL.softscope), mod::Module, code::String, filename::String) : @ Base ./loading.jl:1116 :END: * Bootstrap #+begin_src jupyter-julia :session (night/org-babel-session-name-get "j1") :async yes :pandoc t # using Revise # @redundant const LOCAL_PACKAGES = expanduser("~/code/julia/") push!(LOAD_PATH, LOCAL_PACKAGES) using NightCommon #+end_src #+RESULTS: #+begin_src jupyter-julia :session (night/org-babel-session-name-get "j1") :async yes :pandoc t using Random using Base.Iterators using LinearAlgebra using Distributions using Flux using UnicodePlots import Zygote import Dates import BSON using CUDA CUDA.allowscalar(false) #+end_src #+RESULTS: :RESULTS: # [goto error] : ArgumentError: Package CUDA not found in current path: : - Run `import Pkg; Pkg.add("CUDA")` to install the CUDA package. : : : Stacktrace: : [1] require(into::Module, mod::Symbol) : @ Base ./loading.jl:871 : [2] eval : @ ./boot.jl:360 [inlined] : [3] include_string(mapexpr::typeof(REPL.softscope), mod::Module, code::String, filename::String) : @ Base ./loading.jl:1094 :END: #+begin_src jupyter-julia :session (night/org-babel-session-name-get "j1") :async yes :pandoc t logdir = "$(homedir())/logs/" ensureDir(logdir) log_file = open("$(logdir)/j1","a+") function eclog(str) println(log_file, str) flush(log_file) end #+end_src #+RESULTS: : eclog (generic function with 1 method) #+begin_src jupyter-julia :session (night/org-babel-session-name-get "j1") :async yes :pandoc t eclog("\n----------------------\nStarted\n") #+end_src #+RESULTS: * Hello Flux ** Synthesis #+begin_src jupyter-julia :session (night/org-babel-session-name-get "j1") :async yes :pandoc t W_truth = [1 2 3.1 4 5; 5 4 300 2.9 1] b_truth = [-100.78; -2.3] ground_truth(x) = W_truth*x .+ b_truth #+end_src #+RESULTS: : ground_truth (generic function with 1 method) ** Training data #+begin_src jupyter-julia :session (night/org-babel-session-name-get "j1") :async yes :pandoc t xs_train = [(rand(Float64,5)*10 .+ 100) for i in 1:100_000] #+end_src #+RESULTS: : 100000-element Vector{Vector{Float64}}: : [100.36979583608108, 108.17272057468041, 108.13216857593622, 102.55596012750429, 109.57065453997049] : [100.44472492413394, 108.43515993819202, 101.55092748751085, 107.11292635602152, 101.79524735256447] : [109.00658525869122, 101.58210206368936, 105.51660384612553, 104.85685363518186, 105.71371287219621] : [103.38486855513372, 106.23320922228451, 105.19031888773029, 108.42068483761616, 106.11467018171516] : [100.40140602704624, 101.37256451640205, 102.52920830317863, 105.23863831522823, 104.83088460568285] : [104.80846091334296, 109.27158423640607, 101.52866219119274, 104.77331164184251, 109.79490005189709] : [102.56373548911344, 101.151263919452, 101.41860045778343, 106.95126662881235, 108.48505788579226] : [101.20496406962556, 103.60817175638607, 103.69585673727094, 104.02418314236131, 107.91481286996839] : [107.7867213587923, 100.16638353637622, 100.35141373810708, 102.46760181769352, 101.38451658839273] : [103.19874403546231, 102.63568583998585, 108.09753420836485, 100.24899867337709, 103.82380950581822] : [104.13298867582984, 101.14242483570106, 107.78259278397968, 105.97879713656089, 101.74337654001522] : [109.24940620372966, 107.31769036135181, 106.60242441026423, 101.76931114450122, 103.36013588784681] : [101.88180568480068, 109.14786765753644, 103.04037298174951, 105.64675710710799, 100.9450805031335] : ⋮ : [109.94447818861076, 107.09827056798805, 103.77614264679953, 109.01375840291946, 107.52853204135349] : [107.36558112948407, 101.00581096874745, 107.22778201804897, 101.24518101088728, 107.41083732917485] : [101.27711770559824, 108.90314300550493, 101.26908359163107, 106.11375689379527, 104.91302013168084] : [102.65765726181017, 100.2741528047537, 103.29590706522308, 109.87161477587163, 101.18621849888797] : [106.94522472801248, 106.18849924905513, 106.43574990026252, 101.76302028623583, 103.44737444435215] : [106.10642522346733, 104.52425393677184, 101.98878194967945, 107.59272618260448, 101.04000773405382] : [104.50968872888951, 104.587626328234, 106.17164465478386, 101.75536086491522, 101.08851089598623] : [106.18546137244768, 104.12298899367362, 100.25319859168985, 107.65588578488328, 105.24238314527345] : [105.48988440772116, 108.45673061525132, 108.81394685548135, 106.63472001009077, 107.8847675126022] : [106.30806672811083, 108.08169022780484, 106.23400345085997, 108.2345596802688, 101.45577117772316] : [100.73556967603075, 100.95680374945198, 102.03643016951354, 108.26393448927217, 104.09133922175752] : [106.96704452177362, 104.6505635196802, 109.36190747398466, 100.43728062981657, 108.47428791160505] #+begin_src jupyter-julia :session (night/org-babel-session-name-get "j1") :async yes :pandoc t ys_train = [ground_truth(x) + rand(2) for x in xs_train] #+end_src #+RESULTS: : 100000-element Vector{Vector{Float64}}: : [1509.4908685790447, 33778.97067262285] : [1469.6211967501786, 31811.706692890388] : [1486.5306955089723, 33013.89232426207] : [1505.8763650121678, 32917.8887159962] : [1466.2755911236095, 32074.877144340353] : [1506.046785276518, 31831.70647463568] : [1489.0449477527452, 31759.922559892806] : [1485.0073539132018, 32436.60406649058] : [1435.795246318564, 31441.288875989052] : [1462.963445642635, 33748.5562942912] : [1473.0044565127232, 33667.65712376475] : [1477.6231879593809, 33353.05484326523] : [1466.3133666095014, 32263.20325021832] : ⋮ : [1519.3740578988936, 32532.482028641873] : [1483.6234664182145, 33508.613084596494] : [1482.0110160685174, 31733.57925731069] : [1468.1221131769246, 32321.23512803118] : [1473.287784814114, 33287.34822054001] : [1466.129435911688, 31956.222287420984] : [1454.5671691518387, 33187.15389698428] : [1481.3119664037233, 31438.731098602915] : [1525.6184776258503, 34021.03552173618] : [1491.971879484515, 33247.437112798674] : [1472.6724468204152, 31934.494673164743] : [1499.4709234539773, 34159.453716272714] #+begin_src jupyter-julia :session (night/org-babel-session-name-get "j1") :async yes :pandoc t # train_data = zip(xs_train, ys_train) train_data = zip(x_train, y_train) first(train_data) #+end_src #+RESULTS: | 9.731267315697135 | 9.958680324172958 | 5.107991701340638 | 8.030787679260214 | 8.84921805213325 | | 20.803722553240227 | 1650.4759777536717 | | | | ** Model #+begin_src jupyter-julia :session (night/org-babel-session-name-get "j1") :async yes :pandoc t function ps_init() global W = rand(2,5) global b = rand(2,1) global ps = Flux.params(W, b) end ps_init() #+end_src #+begin_src jupyter-julia :session (night/org-babel-session-name-get "j1") :async yes :pandoc t model(x) = W*x + b #+end_src #+RESULTS: : model (generic function with 1 method) ** Loss and optimizer #+begin_src jupyter-julia :session (night/org-babel-session-name-get "j1") :async yes :pandoc t function loss(x, target, pred) se = (pred .- target).^2 res = sum(se) # println("se: $se, loss: $res") return res end #+end_src #+RESULTS: : loss (generic function with 2 methods) ** Training #+begin_src jupyter-julia :session (night/org-babel-session-name-get "j1") :async yes :pandoc t ps_init() losses = Float64[] min_loss = Inf best_W = W best_b = b #+end_src #+RESULTS: : 2×1 Matrix{Float64}: : 0.21621735587862734 : 0.6929369880227094 #+begin_src jupyter-julia :session (night/org-babel-session-name-get "j1") :async yes :pandoc t # opt = Descent(0.0001) decay = 0.99999999999999 decay = 0.1 for _ in 1:50 for (x, target) in train_data current_loss = nothing pred = nothing gs = Flux.gradient(ps) do pred = model(x) current_loss = loss(x, target, pred) # for p in ps # current_loss += 0.001*sum(abs2,p) # end current_loss end push!(losses, current_loss) if current_loss < min_loss min_loss = current_loss best_W = W best_b = b end if rand() <= 0.0001 println("---------------") @labeled x @labeled pred @labeled target @labeled current_loss @labeled gs[ps[1]] @labeled gs[ps[2]] @labeled ps[2] # @labeled gs[b] # @labeled b end #### Optimization # Flux.Optimise.update!(opt, ps, gs) ### W_step = decay*0.01*normalize(gs[W])*min(1000,norm(gs[W])) b_step = decay*0.01*normalize(gs[b])*min(1000,norm(gs[b])) ## W .-= W_step b .-= b_step ## # Flux.update!(W, W_step) # Flux.update!(b, b_step) ## # decay *= decay #### end end nothing #+end_src #+RESULTS: : --------------- : x => [9.745641720016033, 9.10097490619058, 9.880925610044365, 8.309731956730428, 7.989954647936419] : pred => [18.044926138820614; 2866.787490630857] : target => [30.80810046870502, 3079.278856287415] : current_loss => 45315.47909756394 : gs[ps[1]] => [-248.7706484583183 -232.3146585992275 -252.22395220323133 -212.11711519672383 -203.95436811896542; -4141.729436571462 -3867.757173244997 -4199.222753658366 -3531.4925834511764 -3395.592749347941] : gs[ps[2]] => [-25.526348659768814; -424.98273131311544] : ps[2] => [-0.059457474882017344; 134.11003329266492] : --------------- : x => [9.982730581522594, 6.88840451423814, 7.913588415335067, 9.602511046964946, 5.761185136026533] : pred => [7.240336198184552; 2471.1289315630265] : target => [14.852228708663867, 2482.8474003988827] : current_loss => 195.26341944802238 : gs[ps[1]] => [-151.9749442952493 -104.8675894621624 -120.47476877940973 -146.1865638403747 -87.70704397681024; -233.96463443264088 -161.44310725774062 -185.47027844979212 -225.05345289964583 -135.02453694824897] : gs[ps[2]] => [-15.22378502095863; -23.436937671712258] : ps[2] => [-3.119628166105348; 148.13484811329025] : --------------- : x => [6.292123889448201, 5.572606910644091, 5.764044712518562, 9.971191443949667, 9.592466647407212] : pred => [17.00450526428546; 1810.6994431358282] : target => [22.624574283112857, 1819.2064748032285] : current_loss => 103.95476356653455 : gs[ps[1]] => [-70.72434106742315 -62.636870905228626 -64.78865822392288 -112.07756822987669 -107.82064923845677; -107.0545943654832 -94.81268691764573 -98.06982180341298 -169.6504827507809 -163.2068350759481] : gs[ps[2]] => [-11.240138037654795; -17.01406333480054] : ps[2] => [-18.146868548552415; 124.98303659079237] : --------------- : x => [7.40065335859841, 6.656590616380632, 8.518018686250146, 7.124480350724576, 5.316820485034204] : pred => [6.816690612570522; 2651.2168470154475] : target => [1.4111392274886356, 2642.6873703588367] : current_loss => 101.97195781242871 : gs[ps[1]] => [80.0092240261651 71.96508525259883 92.08917541522574 77.02348925569582 57.480692674216776; 126.24740013166644 113.55446855006578 145.3084830898899 121.53617768397474 90.69939242897837] : gs[ps[2]] => [10.811102770163773; 17.058953313221537] : ps[2] => [-30.14924415492992; 106.49490946062708] : --------------- : x => [9.348874769609289, 9.48132295040376, 6.346399846275067, 7.739239533252926, 7.722334184745371] : pred => [4.518774679678664; 1997.8313763652184] : target => [16.52393366597871, 2016.4390206129237] : current_loss => 490.3682667355033 : gs[ps[1]] => [-224.46945590433745 -227.6495788401051 -152.37907829032468 -185.82160205951988 -185.41569926641586; -347.9210716584762 -352.85016891743476 -236.18310118635694 -288.01803196509553 -287.3888945432714] : gs[ps[2]] => [-24.01031797260009; -37.21528849541073] : ps[2] => [-36.98949317203729; 95.9584190985206] : --------------- : x => [6.040859718406763, 7.804729929473719, 5.708607145066713, 9.549115318058782, 6.271290774830542] : pred => [12.441603022679082; 1812.1985042693975] : target => [8.270622148975587, 1805.5824770806391] : current_loss => 61.1688972111905 : gs[ps[1]] => [50.39262069240098 65.10675852051223 47.62098243512075 79.6583547048245 52.314867750502735; 79.93298428090887 103.27261082862881 75.53660016340346 126.35441314573205 82.98206054977696] : gs[ps[2]] => [8.341961747406991; 13.23205437751676] : ps[2] => [-41.145433308806616; 89.55515056429233] : --------------- : x => [9.50184162759799, 8.538448279152352, 5.937053605566673, 7.83533876155829, 5.359575526197977] : pred => [6.6231562766045045; 1894.979568704417] : target => [1.9817384247489067, 1888.8402211307045] : current_loss => 59.23434830637321 : gs[ps[1]] => [88.20403471167593 79.26101254000687 55.1126931846016 72.73416240646557 49.75205905132731; 116.67021668438801 104.84100345176707 72.89927129667342 96.20773602997654 65.80859400578473] : gs[ps[2]] => [9.282835703711196; 12.278695147424969] : ps[2] => [-45.68203870585836; 82.56504696631826] : --------------- : x => [9.187257892352852, 8.66032735591489, 9.312711881756764, 7.295707589427934, 5.650836692076445] : pred => [9.572642495432127; 2895.0255629696167] : target => [11.825361270220537, 2898.8434301618286] : current_loss => 19.650851775652217 : gs[ps[1]] => [-41.392616685852516 -39.01856406096626 -41.95784180045712 -32.87035492414119 -25.459491819007674; -70.15146098720773 -66.12795937192548 -71.10939432776232 -55.70808529929665 -43.14828803045183] : gs[ps[2]] => [-4.505437549576818; -7.635734384423813] : ps[2] => [-46.79707835221256; 80.84695862939975] : --------------- : x => [7.568334209639169, 9.626727194587396, 5.279340383568343, 5.694535246625735, 9.465431609541453] : pred => [12.475318214882762; 1683.6710095330102] : target => [12.575350580303429, 1683.8494785428566] : current_loss => 0.04185766160719646 : gs[ps[1]] => [-1.5141567465687167 -1.9259685850680726 -1.0562098128583828 -1.1392756613826642 -1.8936990272599605; -2.701426225161379 -3.4361449409582145 -1.8843973017947635 -2.032596134001065 -3.378572414246735] : gs[ps[2]] => [-0.20006473084133347; -0.3569380196927341] : ps[2] => [-54.86276071130685; 68.42414570915939] : --------------- : x => [6.587916984199005, 8.688125009052486, 5.53021936320296, 7.3362137696440275, 8.66437854019398] : pred => [15.634249169187086; 1758.3022137908981] : target => [12.773398483277054, 1754.340911545644] : current_loss => 23.87638212532665 : gs[ps[1]] => [37.69409364592814 49.71085678283982 31.642263716904257 41.975624389737476 49.57498657939622; 52.19346068210957 68.83257821081489 43.813740760406574 58.12192015470907 68.64444433000277] : gs[ps[2]] => [5.721701371820064; 7.922604490508093] : ps[2] => [-68.7910133401225; 46.9647268373293] : --------------- : x => [5.201049930563784, 5.879123339142632, 5.879212493497006, 8.078001904382251, 6.594085038051688] : pred => [3.4788968073592628; 1846.8725408483065] : target => [-0.308038160544388, 1841.2060727855267] : current_loss => 46.44973675763503 : gs[ps[1]] => [39.392075703729695 44.52771550723541 44.52839075071965 61.181735764994855 49.94274242385642; 58.94316664892569 66.62772927679028 66.62873965739371 91.54747960451307 74.7303445427482] : gs[ps[2]] => [7.573869935807301; 11.332936125559627] : ps[2] => [-71.3053726699468; 43.09245267241971] : --------------- : x => [9.942897933129355, 6.519864023243275, 9.707348288113408, 9.971068130102418, 6.892915305429175] : pred => [22.335712325607645; 3014.810505386612] : target => [26.902524621928332, 3021.278698682901] : current_loss => 62.693299067983446 : gs[ps[1]] => [-90.81469708415338 -59.54999038337252 -88.66327505364778 -91.0719930880061 -62.95730074866205; -128.62517151350556 -84.34348153571561 -125.57801024383478 -128.98959207193818 -89.1694171409293] : gs[ps[2]] => [-9.133624592641375; -12.936386592577946] : ps[2] => [-71.50395546884347; 42.7834425780336] : --------------- : x => [8.107277331071227, 9.462915797757965, 6.390597192104631, 6.32478809506127, 7.539976643075683] : pred => [10.192611566695419; 2021.1965782219545] : target => [8.80129951383563, 2019.2156382711123] : current_loss => 5.859872317275557 : gs[ps[1]] => [22.559505333192675 26.33173760923592 17.78262979669419 17.599507816885694 20.980920763584972; 32.12005911535239 37.49093591046879 25.318778575159904 25.058050836235815 29.87248192137113] : gs[ps[2]] => [2.7826241057195773; 3.9618799016843695] : ps[2] => [-73.6610948231352; 39.46272063768593] : --------------- : x => [9.770189872539635, 8.941814727903868, 9.488389307408992, 8.943402024428142, 9.799053905492174] : pred => [36.75015560684993; 2958.207671457844] : target => [41.247334275018815, 2964.31356171575] : current_loss => 57.50651181502637 : gs[ps[1]] => [-87.87657895748985 -80.42587689809528 -85.34196397712293 -80.44015361023334 -88.13619238403282; -119.31141432126597 -109.19547887021973 -115.8701276706587 -109.21486258698768 -119.66389555648355] : gs[ps[2]] => [-8.994357336337771; -12.21178051581228] : ps[2] => [-74.21558195821082; 38.6040652815749] : --------------- : x => [8.07667954611113, 7.021152483346666, 5.580177533263063, 6.150606465775391, 5.366222841989607] : pred => [-6.1056169875356545; 1769.4504581614583] : target => [-9.78715823304588, 1763.1028940691106] : current_loss => 53.84531584885535 : gs[ps[1]] => [59.469257751553855 51.69732491691458 41.08730749195494 45.28742277730795 39.51194145076767; 102.53448214458888 89.13443078037889 70.84106907813239 78.0827374966354 68.12488684669893] : gs[ps[2]] => [7.3630824910204495; 12.695128184695477] : ps[2] => [-78.36459472528888; 32.211056543389205] : --------------- : x => [8.136792166458934, 8.146177756790511, 8.101230058270698, 5.660649201764269, 7.350719438883338] : pred => [8.322644738914931; 2525.8140958952467] : target => [8.300881163930084, 2525.238613983031] : current_loss => 0.33165308448363356 : gs[ps[1]] => [0.3541713729016937 0.3545799008996104 0.35262345568534587 0.24639192673102497 0.3199558674014239; 9.365153430512075 9.375955905454655 9.324222730466895 6.515202454027881 8.460412157700345] : gs[ps[2]] => [0.043527149969694534; 1.1509638244315283] : ps[2] => [-79.24179628137244; 30.863577831249916] : --------------- : x => [8.132391790415454, 5.383832652302875, 8.143537790116412, 6.350657058765372, 5.597714892775953] : pred => [-1.1778630878008869; 2530.2469836995365] : target => [-3.462211549008061, 2527.072711516078] : current_loss => 15.294251786897787 : gs[ps[1]] => [37.1544333447388 24.597099669370024 37.205356039269795 29.014227359690313 25.57426280317846; 51.6288500906036 34.1795004572009 51.69961096421877 40.3174280966462 35.53734135013983] : gs[ps[2]] => [4.5686969224143485; 6.34854436691694] : ps[2] => [-81.6129494381351; 27.206924603661232] : --------------- : x => [7.063537815191207, 6.248436155696099, 9.620026699616378, 9.491664638755989, 6.954066473752799] : pred => [20.097978002803558; 2976.7093353244522] : target => [21.427326982167923, 2978.3467480563363] : current_loss => 4.44828916347313 : gs[ps[1]] => [-18.779813570652056 -16.612704452396013 -25.57674534918595 -25.23546939999822 -18.488762338630465; -23.131853501477327 -20.46253783140308 -31.50390839803308 -31.08354505254572 -22.773353964981972] : gs[ps[2]] => [-2.6586979587287303; -3.2748254637681384] : ps[2] => [-82.35769348554672; 26.060974008698864] : --------------- : x => [9.80627757692316, 6.521133710660103, 6.216187064928497, 6.175196972506677, 7.073714788845585] : pred => [2.898672652112353; 1964.7049754630411] : target => [1.2626134698182747, 1963.1160590409074] : current_loss => 5.201345044494948 : gs[ps[1]] => [32.08730094769932 21.337921372585832 20.340099852867887 20.205975418688283 23.146032066440473; 31.162710963949486 20.72307288759511 19.754003421040032 19.623743759052413 22.479083186973632] : gs[ps[2]] => [3.2721183645881564; 3.1778328442674137] : ps[2] => [-82.55914441200129; 25.752166287651313] : --------------- : x => [7.45869893961911, 9.103239556791012, 5.6800176965957805, 8.411872179071423, 7.559582399177387] : pred => [14.499244053424533; 1808.6188117033882] : target => [14.263352773692556, 1807.4494577690166] : current_loss => 1.423033319684062 : gs[ps[1]] => [3.51888407600458 4.29474965751637 2.679733286700507 3.96857458652594 3.566479132762562; 17.443717900674617 21.28981798252253 13.28390208162982 19.6729116560573 17.679654841369608] : gs[ps[2]] => [0.47178255946395353; 2.338707868743313] : ps[2] => [-89.4026336656995; 15.207885277038] : --------------- : x => [5.655402223251164, 6.83668082665157, 5.940260360136811, 8.594210659399256, 7.290828306284258] : pred => [9.867187486796198; 1871.082198930316] : target => [8.081444620695557, 1867.7353782754183] : current_loss => 14.390086079879758 : gs[ps[1]] => [20.19818835020094 24.417108028000147 21.21555512178947 30.694100749576613 26.039089271823464; 37.85523394506368 45.76228920316225 39.76197213755285 57.526563494840254 48.80218953357115] : gs[ps[2]] => [3.571485732201282; 6.6936413097955665] : ps[2] => [-91.14563798080482; 12.519431513108765] : --------------- : x => [5.198783834523098, 6.496326924071557, 8.2926328763796, 6.0049991480570135, 6.558376007711266] : pred => [1.6921246164799015; 2564.1429591384826] : target => [-0.11114771114579197, 2561.646265772856] : current_loss => 9.485268849543816 : gs[ps[1]] => [18.74964605220659 23.429293146775954 29.907750778268777 21.657297582214152 23.653075937739995; 25.95953821795992 32.438672664540135 41.4083229720658 29.9852830670931 32.74850773547307] : gs[ps[2]] => [3.606544655251387; 4.99338673125294] : ps[2] => [-92.46305632533752; 10.488590041390175] : --------------- : x => [5.635568953845485, 6.865266879284139, 6.662201393357896, 9.24418683527761, 7.240782194560622] : pred => [12.14707983380211; 2085.776404583582] : target => [12.399712764581135, 2086.1883456126466] : current_loss => 0.23351880914077244 : gs[ps[1]] => [-2.8474606028345444 -3.468784984587454 -3.3661829268882264 -4.670772025730137 -3.6585200538888696; -4.643044148422843 -5.656170206110361 -5.488868195630456 -7.6161196755787195 -5.965550536919359] : gs[ps[2]] => [-0.5052658615580512; -0.8238820581291293] : ps[2] => [-95.2061336188353; 6.264384565818323] : --------------- : x => [5.672489502180572, 7.137538125150752, 6.935684562216434, 8.784215244369795, 6.230858087675193] : pred => [7.561234739783629; 2167.865680760087] : target => [7.009523862194727, 2167.0349845260234] : current_loss => 0.994441125737119 : gs[ps[1]] => [6.259148322723754 7.875714845702334 7.6529852330004555 9.69269420280214 6.875264367366376; 9.424231334451742 11.858252082094094 11.522894092970436 14.594029045401522 10.351900696831178] : gs[ps[2]] => [1.1034217551778038; 1.6613924681269054] : ps[2] => [-95.3128722726062; 6.099751559836653] : --------------- : x => [6.347503217357707, 5.654960144930728, 7.050410826369625, 9.965139415589395, 6.11296602421222] : pred => [9.0599399779273; 2202.201997211129] : target => [9.363964481319211, 2202.1136274697155] : current_loss => 0.10024010986015476 : gs[ps[1]] => [-3.8595930268714724 -3.438492899527231 -4.28699530039196 -6.059293124111455 -3.7169829195254938; 1.1218544358761922 0.9994547314202803 1.2460859631678638 1.7612335866065687 1.0804024536560486] : gs[ps[2]] => [-0.6080490067838227; 0.17673948282663332] : ps[2] => [-96.12058588104517; 4.857463911068353] : --------------- : x => [9.777351884559106, 9.073579506315435, 5.305247273917343, 9.797673867105852, 8.597419986732175] : pred => [24.995395486617994; 1710.5245594789826] : target => [26.175752339486234, 1711.4031199233164] : current_loss => 2.16511075446092 : gs[ps[1]] => [-23.08152859968708 -21.420123500648486 -12.524169951857766 -23.12950298241292 -20.29604719665139; -17.179989232211224 -15.943376085532071 -9.321960804546402 -17.215697412243244 -15.106706247334452] : gs[ps[2]] => [-2.3607137057364795; -1.7571208886674867] : ps[2] => [-96.1996183396579; 4.731817060521341] : --------------- : x => [8.42072836367306, 9.3647694010882, 9.056452360882627, 7.070909529088602, 5.10805767797557] : pred => [8.872050009185457; 2820.0739164231486] : target => [8.228341134298768, 2819.997365346627] : current_loss => 0.4202211829245356 : gs[ps[1]] => [10.840995161412838 12.056370349495564 11.659437519377317 9.10321443479039 6.57620412149194; 1.2892316426738177 1.4337663580636075 1.3865623553880981 1.0825714728801084 0.7820546283690498] : gs[ps[2]] => [1.287417749773379; 0.15310215304361918] : ps[2] => [-96.30601598590196; 4.563419597349832] : --------------- : x => [8.198395896123102, 7.6617586537426465, 7.674924747478488, 8.761466236547234, 7.11284783949618] : pred => [17.051911377317182; 2404.009347739471] : target => [17.503578582974917, 2404.149604236478] : current_loss => 0.22367514961925794 : gs[ps[1]] => [-7.405893130555526 -6.92113024311982 -6.93302362865401 -7.91453394505176 -6.425280215867792; -2.299756578928717 -2.149222859369477 -2.152916119742779 -2.457705125961238 -1.9952462434188636] : gs[ps[2]] => [-0.9033344113154698; -0.28051299401340657] : ps[2] => [-98.55471918239859; 1.1029164940471525] : --------------- : x => [8.230861939702576, 9.74546753527152, 8.47309801046038, 7.936561351394312, 8.350143260516882] : pred => [26.28860592163025; 2650.7336174552042] : target => [26.63895660260686, 2651.1074067697837] : current_loss => 0.262464051354532 : gs[ps[1]] => [-5.7673761711985305 -6.82866237483566 -5.937111315892721 -5.561159348147296 -5.850956755148694; -6.153216485678621 -7.285503260530378 -6.334306995388567 -5.933203655310486 -6.242388651977238] : gs[ps[2]] => [-0.7007013619532216; -0.747578629158852] : ps[2] => [-98.81046563001061; 0.7130485210397078] : --------------- : x => [5.421239780260826, 8.150093089994947, 9.1101057209511, 9.727875792137121, 7.5981067869973415] : pred => [26.19284877750644; 2826.0797768500565] : target => [26.27162948310371, 2826.5048854436805] : current_loss => 0.1869237159473087 : gs[ps[1]] => [-0.8541781902018497 -1.2841401686264369 -1.4354011135244675 -1.5327378377342857 -1.19716842776608; -4.609231237369434 -6.929349222783525 -7.745568461597461 -8.270807193887135 -6.46004098084957] : gs[ps[2]] => [-0.1575614111945356; -0.8502171872478357] : ps[2] => [-99.17811478936568; 0.13975183684583337] : --------------- : x => [8.403214542851863, 5.4637730272593, 9.464059637811491, 6.126108473550499, 5.407092310295194] : pred => [-0.14329376630925594; 2924.3891542677743] : target => [-0.6818618703441419, 2924.026673619358] : current_loss => 0.42144782316002916 : gs[ps[1]] => [9.051406648284217 5.885227760335981 10.194081311218449 6.598653251424283 5.824174907794588; 6.092005312548359 3.961023979440824 6.861076948328782 4.441191543522283 3.919932653365125] : gs[ps[2]] => [1.077136208069772; 0.7249612968325891] : ps[2] => [-99.79450445103765; -0.8077620029139135] : --------------- : x => [6.471913704985612, 7.705444392672555, 8.178922503664019, 5.911026114540293, 9.944276461952983] : pred => [19.813280977510445; 2541.668151941998] : target => [19.684002471287062, 2541.4838274851063] : current_loss => 0.05068843757979609 : gs[ps[1]] => [1.6733586723743545 1.9922966817440941 2.1147177675809847 1.5283372526703483 2.571162412947251; 2.385863957443785 2.840603705578246 3.015150896895534 2.1790933564714483 3.6659467160622423] : gs[ps[2]] => [0.2585570124467651; 0.36864891378354514] : ps[2] => [-99.91763907223711; -0.9944910196665342] : --------------- : x => [8.982544537444614, 7.898214971330536, 8.679172719712724, 6.454547055484834, 7.037261642773518] : pred => [12.01945759114166; 2703.704905395801] : target => [11.72489433031473, 2703.6882975767726] : current_loss => 0.08704333428186725 : gs[ps[1]] => [5.291855218945625 4.6530479133344 5.1131308351974285 3.802544855648944 4.145837473575291; 0.2983609481811649 0.2623442497792771 0.28828425968687327 0.21439189881292312 0.23374713563448282] : gs[ps[2]] => [0.5891265216538599; 0.03321563805639016] : ps[2] => [-100.08541260976502; -1.2531691550687722] : --------------- : x => [9.62192957185627, 6.795427168025086, 5.535096872847279, 5.652413184753744, 8.09077580140869] : pred => [2.836471574461953; 1758.059190352223] : target => [2.463298895407459, 1758.3788398127879] : current_loss => 0.24143362603217877 : gs[ps[1]] => [7.181282472006535 5.071735523223233 4.131093857733145 4.218652342555001 6.0385129628819065; -6.151289194475796 -4.344309257135392 -3.5385814591608646 -3.613581650793609 -5.172424240944647] : gs[ps[2]] => [0.7463453581089885; -0.6392989211299209] : ps[2] => [-100.16194692257643; -1.3720610680984633] : --------------- : x => [7.543307299653746, 5.5183314269251245, 5.22812758569566, 5.694926870940346, 8.056046224747252] : pred => [-2.874177599184037; 1650.5646596861911] : target => [-2.738913133178481, 1650.5286359135782] : current_loss => 0.019594187957034704 : gs[ps[1]] => [-2.040682867606956 -1.4928683074094118 -1.4143597721760823 -1.540642484276878 -2.179393581413028; 0.543476773824018 0.39758223305241996 0.3766737586767351 0.4103055012917794 0.5804183547187278] : gs[ps[2]] => [-0.2705289320111124; 0.07204754522581425] : ps[2] => [-100.16647844556252; -1.377799000976817] : --------------- : x => [6.3590988150611985, 8.130220736669003, 9.052446654141534, 5.859871712468653, 7.289372329248824] : pred => [9.649101526285534; 2801.942283658593] : target => [10.091998531634031, 2801.912337720273] : current_loss => 0.19705451656853173 : gs[ps[1]] => [-5.632851643811562 -7.201700834185906 -8.018603028392615 -5.190639266357471 -6.456882350989006; 0.380858361773182 0.486934177416507 0.5421680182999778 0.35095871372936727 0.43657418832636163] : gs[ps[2]] => [-0.8857940106969941; 0.059891876639994734] : ps[2] => [-100.32858735270564; -1.6278600889354387] : --------------- : x => [9.395982160473306, 9.170646509830085, 8.330450988994329, 6.598354009530433, 6.42025396213381] : pred => [11.183056201914184; 2606.0939161139045] : target => [11.676625600255612, 2606.1110710677417] : current_loss => 0.24390504342027708 : gs[ps[1]] => [-9.275138523543205 -9.052700960517507 -8.223311365101374 -6.513491237055373 -6.33768177057911; -0.3223752804367996 -0.31464403506746424 -0.2858170043190868 -0.2263889168704534 -0.22027832068745665] : gs[ps[2]] => [-0.9871387966828564; -0.03430990767446929] : ps[2] => [-100.36677151184932; -1.6869893447500204] : --------------- : x => [7.8143192892247235, 9.699765853357817, 8.712952444014755, 8.749681553134003, 5.743756934009722] : pred => [17.20477631443562; 2720.2568796872197] : target => [17.28821778751129, 2720.6319515890254] : current_loss => 0.1476414109532334 : gs[ps[1]] => [-1.3040766251530647 -1.6187255025865162 -1.454043173533698 -1.4601726354730322 -0.9585350791247285; -5.8618631942540285 -7.276219251379129 -6.535967287039657 -6.563519400657633 -4.308643673498187] : gs[ps[2]] => [-0.1668829461513397; -0.750143803611536] : ps[2] => [-100.45063374673684; -1.823121277927638] : --------------- : x => [6.499635068282499, 5.863772932337639, 5.2887956176081, 8.466573530561128, 8.286587426936345] : pred => [9.186250344373548; 1673.1612766753735] : target => [9.163313904542052, 1673.0648465927702] : current_loss => 0.009824841103026833 : gs[ps[1]] => [0.2981569773406767 0.26898815009622595 0.24261228492868833 0.38838610872529195 0.3801296278527017; 1.2535206930518277 1.1308882164648555 1.0199979965560468 1.6328647698381764 1.5981526201582017] : gs[ps[2]] => [0.045872879662990584; 0.19286016520663907] : ps[2] => [-100.47581014655853; -1.8548122777316274] : --------------- : x => [5.804718584794035, 8.639169240343685, 8.842222548367678, 8.27331998891977, 6.405703743564591] : pred => [14.90090155350056; 2744.6001930777165] : target => [14.861448260060026, 2744.302813225994] : current_loss => 0.08999133857369739 : gs[ps[1]] => [0.4580305313312012 0.6816873582434302 0.6977096017345137 0.6528194425005752 0.5054522189759626; 3.4524127040717585 5.138229735394967 5.258997660658772 4.920637343112631 3.8098544588767713] : gs[ps[2]] => [0.07890658688106811; 0.5947597034446517] : ps[2] => [-100.47955551759053; -1.8585956651272353] : --------------- : x => [5.313006670707398, 5.247913972040842, 8.298549744225653, 8.029192055983291, 9.618171187501382] : pred => [21.065758660540325; 2567.9746089219157] : target => [21.00949074262798, 2567.587004121633] : current_loss => 0.15340355978829534 : gs[ps[1]] => [0.5979036464301949 0.5905783851796697 0.933884231599177 0.9035718390170173 1.082388933690391; 4.1186937789992 4.06823329406619 6.433115432491416 6.224306766579895 7.456098644430436] : gs[ps[2]] => [0.11253583582468707; 0.7752096005651765] : ps[2] => [-100.58508042641984; -2.0210692128264585] : --------------- : x => [6.96601544544508, 5.310725844282581, 9.451107815687687, 5.034701243274945, 6.844130094837791] : pred => [0.5939202131917654; 2910.493330967293] : target => [0.46989408186335274, 2910.353351957762] : current_loss => 0.034976604361597385 : gs[ps[1]] => [1.7279358929450443 1.3173375620243728 2.344368678294936 1.2488690351954812 1.6977019559421858; 1.9501918848633415 1.4867802871476654 2.645913422022995 1.4095049866371394 1.9160691035546258] : gs[ps[2]] => [0.24805226265682523; 0.2799580190621782] : ps[2] => [-100.66222511768291; -2.146486661104506] : --------------- : x => [6.562446417120794, 6.0453200159748555, 7.395388433397859, 6.792970917412018, 5.976875645133751] : pred => [-2.096214983851766; 2298.897696541392] : target => [-1.9307823568936415, 2298.906089393899] : current_loss => 0.02743839403546788 : gs[ps[1]] => [-2.1712855001124507 -2.0001863420905037 -2.446877071825474 -2.247558047435223 -1.9775404779530235; -0.11015528972611521 -0.10147495850167428 -0.12413680870487287 -0.11402480598643827 -0.10032607148288603] : gs[ps[2]] => [-0.3308652539162491; -0.016785705013717234] : ps[2] => [-100.6811195999645; -2.172353972508324] : --------------- : x => [7.874275154308644, 6.683472074434978, 6.851042532548933, 7.659376266944267, 9.805294264589367] : pred => [21.18904053611243; 2151.1971539169717] : target => [21.548044567043128, 2150.941100494756] : current_loss => 0.1944472492528616 : gs[ps[1]] => [-5.653793042108503 -4.79878683066983 -4.9191037705254566 -5.499493908495836 -7.040280330898485; 4.0324702014575795 3.4226517938842833 3.5084657724089445 3.922419010377589 5.021358304560171] : gs[ps[2]] => [-0.7180080618613971; 0.5121068444314005] : ps[2] => [-100.6830257672136; -2.1736162612243386] : --------------- : x => [7.742415649380012, 5.747407829497573, 8.90776997733522, 9.088094349939304, 5.324337702186051] : pred => [9.166965250764562; 2763.426188920554] : target => [9.338329522509808, 2763.209476943293] : current_loss => 0.0763297947192303 : gs[ps[1]] => [-2.6535468386099956 -1.9698007142495473 -3.052947030080424 -3.114749339658858 -1.824802505721733; 3.355748408310154 2.4910642297136585 3.8608408895522572 3.9389977922230837 2.3076955020940173] : gs[ps[2]] => [-0.342728543490491; 0.43342395452236815] : ps[2] => [-100.694224341659; -2.190896264442863] #+begin_src jupyter-julia :session (night/org-babel-session-name-get "j1") :async yes :pandoc t println(lineplot(losses)) @labeled minimum(losses) @labeled length(losses) # we need to use the loss of a batch for this to work # W = best_W # b = best_b bella() #+end_src * Hello Flux test #+begin_src jupyter-julia :session (night/org-babel-session-name-get "j1") :async yes :pandoc t using Flux W_truth = [1 2 3.1 4 5; 5 4 300 2.9 1] b_truth = [-100.78; -2.3] ground_truth(x) = W_truth*x .+ b_truth x_train = [((rand(5).*5) .+ 5) for _ in 1:10_000] y_train = [ ground_truth(x) + 0.2 .* randn(2) for x in x_train ] model(x) = W*x .+ b W = rand(2, 5) b = rand(2) function loss(x, y) pred = model(x) # sum(sqrt.((y .- pred).^2)) sum(((y .- pred).^2)) end opt = Descent(0.01) train_data = zip(x_train, y_train) ps = Flux.params(W, b) for (x,y) in train_data gs = Flux.gradient(ps) do loss(x,y) end Flux.Optimise.update!(opt, ps, gs) end println(ps[1] - W_truth) println(ps[2] - b_truth) nothing #+end_src #+RESULTS: : [NaN NaN NaN NaN NaN; NaN NaN NaN NaN NaN] : [NaN, NaN] * Classifying books ** Data preprocessing #+begin_src jupyter-julia :session (night/org-babel-session-name-get "j1") :async yes :pandoc t PART_ONE_SIZE = 500 PART_BIG_SIZE = PART_ONE_SIZE * 4 SAMPLE_SIZE = 650 # Can cause GPU-OOM (550 is around 7GB) rng = MersenneTwister(1234); Book = Dict{Symbol, Any} function bigPartSplit(parts) return vcat((map(parts) do s collect(Iterators.partition(s, PART_ONE_SIZE)) end)...) end function text_to_bytes(text) convert(Vector{Float32}, codeunits(text)) end function book_get(path; part_size=PART_BIG_SIZE, sample_size=SAMPLE_SIZE) book = Book() book[:path] = path book[:text] = open(f->read(f,String), book[:path]) book[:bytes] = text_to_bytes(book[:text]) book[:parts] = (collect(Iterators.partition(book[:bytes], part_size))) pop!(book[:parts]) # the last entry might not be complete, so we just skip it instead of padding etc sample_size = min(sample_size, length(book[:parts])) train_size = floor(Int, sample_size*0.8) # valid_size = floor(Int, sample_size*0.2) # @todo I think only :train_samples need to be moved to the gpu book[:part_samples] = map(gpu, sample(rng, book[:parts], sample_size; replace=false)) book[:train_samples] = book[:part_samples][1:train_size] book[:train_samples_split] = bigPartSplit(book[:train_samples]) book[:valid_samples] = book[:part_samples][(train_size+1):end] book[:valid_samples_split] = bigPartSplit(book[:valid_samples]) #: You can use =String(book[:parts][1])= to get the string back return book end #+end_src #+RESULTS: : book_get (generic function with 1 method) #+begin_src jupyter-julia :session (night/org-babel-session-name-get "j1") :async yes :pandoc t dir_base = "$(homedir())/base/archives/nlp_data" Books = Dict{Symbol, Book} books = Books() function books_load(path::AbstractString ; name::Union{Symbol,Nothing}=nothing , verbosity::UInt8=0x01) if name === nothing name = Symbol(replace(fileNameNoExt(path), r"\s+" => "_")) end book = book_get(path) book[:name] = name books[name] = book if verbosity >= 1 println("Loaded book $(name)") @labeled length(book[:part_samples]) display(book) end return book end elizium = nothing # for GC luminary = nothing nothing #+end_src #+RESULTS: #+begin_src jupyter-julia :session (night/org-babel-session-name-get "j1") :async yes :pandoc t elizium = books_load("$(dir_base)/Voice of the Nephilim/Elizium for the Sleepless Souls.txt"; name=:elizium) #+end_src #+RESULTS: :RESULTS: : Dict{Symbol, Any} with 10 entries: : :part_samples => SubArray{Float32, 1, Vector{Float32}, Tuple{UnitRange… : :valid_samples => SubArray{Float32, 1, Vector{Float32}, Tuple{UnitRange… : :path => "/Users/evar/base/archives/nlp_data/Voice of the Neph… : :name => :elizium : :text => "Elizium for the Sleepless Souls\n\nBy: Voice of the … : :valid_samples_split => SubArray{Float32, 1, Vector{Float32}, Tuple{UnitRange… : :bytes => Float32[69.0, 108.0, 105.0, 122.0, 105.0, 117.0, 109.… : :train_samples => SubArray{Float32, 1, Vector{Float32}, Tuple{UnitRange… : :train_samples_split => SubArray{Float32, 1, Vector{Float32}, Tuple{UnitRange… : :parts => SubArray{Float32, 1, Vector{Float32}, Tuple{UnitRange…Loaded book elizium : length(book[:part_samples]) => 150 : Dict{Symbol, Any} with 10 entries: : :part_samples => SubArray{Float32, 1, Vector{Float32}, Tuple{UnitRange… : :valid_samples => SubArray{Float32, 1, Vector{Float32}, Tuple{UnitRange… : :path => "/Users/evar/base/archives/nlp_data/Voice of the Neph… : :name => :elizium : :text => "Elizium for the Sleepless Souls\n\nBy: Voice of the … : :valid_samples_split => SubArray{Float32, 1, Vector{Float32}, Tuple{UnitRange… : :bytes => Float32[69.0, 108.0, 105.0, 122.0, 105.0, 117.0, 109.… : :train_samples => SubArray{Float32, 1, Vector{Float32}, Tuple{UnitRange… : :train_samples_split => SubArray{Float32, 1, Vector{Float32}, Tuple{UnitRange… : :parts => SubArray{Float32, 1, Vector{Float32}, Tuple{UnitRange… :END: #+begin_src jupyter-julia :session (night/org-babel-session-name-get "j1") :async yes :pandoc t hsep() luminary = books_load("$(dir_base)/YakAge/Black Luminary.txt"; name=:luminary) #+end_src #+RESULTS: :RESULTS: : ---------------------------------------- : Dict{Symbol, Any} with 10 entries: : :part_samples => SubArray{Float32, 1, Vector{Float32}, Tuple{UnitRange… : :valid_samples => SubArray{Float32, 1, Vector{Float32}, Tuple{UnitRange… : :path => "/Users/evar/base/archives/nlp_data/YakAge/Black Lumi… : :name => :luminary : :text => "Black Luminary\n\nBy: YakAge\n\nThe war against the … : :valid_samples_split => SubArray{Float32, 1, Vector{Float32}, Tuple{UnitRange… : :bytes => Float32[66.0, 108.0, 97.0, 99.0, 107.0, 32.0, 76.0, 1… : :train_samples => SubArray{Float32, 1, Vector{Float32}, Tuple{UnitRange… : :train_samples_split => SubArray{Float32, 1, Vector{Float32}, Tuple{UnitRange… : :parts => SubArray{Float32, 1, Vector{Float32}, Tuple{UnitRange…Loaded book luminary : length(book[:part_samples]) => 650 : Dict{Symbol, Any} with 10 entries: : :part_samples => SubArray{Float32, 1, Vector{Float32}, Tuple{UnitRange… : :valid_samples => SubArray{Float32, 1, Vector{Float32}, Tuple{UnitRange… : :path => "/Users/evar/base/archives/nlp_data/YakAge/Black Lumi… : :name => :luminary : :text => "Black Luminary\n\nBy: YakAge\n\nThe war against the … : :valid_samples_split => SubArray{Float32, 1, Vector{Float32}, Tuple{UnitRange… : :bytes => Float32[66.0, 108.0, 97.0, 99.0, 107.0, 32.0, 76.0, 1… : :train_samples => SubArray{Float32, 1, Vector{Float32}, Tuple{UnitRange… : :train_samples_split => SubArray{Float32, 1, Vector{Float32}, Tuple{UnitRange… : :parts => SubArray{Float32, 1, Vector{Float32}, Tuple{UnitRange… :END: *** Assertions #+begin_src jupyter-julia :session (night/org-babel-session-name-get "j1") :async yes :pandoc t for book in values(books) for ds in [:valid_samples, :train_samples] for (i, part) in enumerate(book[ds]) @assert length(part) == PART_BIG_SIZE "$(ds): i=$i, length(part)=$(length(part))" end end for ds in [:valid_samples_split, :train_samples_split] for (i, part) in enumerate(book[ds]) @assert length(part) == PART_ONE_SIZE "$(ds): i=$i, length(part)=$(length(part))" end end end #+end_src #+RESULTS: #+begin_src jupyter-julia :session (night/org-babel-session-name-get "j1") :async yes :pandoc t length(elizium[:train_samples_split][end]) #+end_src #+RESULTS: : 500 ** Model #+begin_src jupyter-julia :session (night/org-babel-session-name-get "j1") :async yes :pandoc t model_selected = :m1 function ps_init() eclog("\n---------------------------\nps_init()") ## global i_last = 0 global losses = Float64[] global accu_v = Bool[] global book_names = collect(keys(books)) global books_v = collect(values(books)) global is = IdDict{Book, Int}() ## out_len = length(books_v) if out_len == 2 out_len -= 1 # @todo Can't we always decrease this even for multiple classes? end if model_selected == :m1 lc1_s = 4 lc1_c = 32 global model1 = Chain( # x -> reshape(x, :, 1), # Fake batching x -> reshape(x, :, 1, size(x, 2)), Conv((lc1_s,), 1=>lc1_c, relu), x -> reshape(x, :, size(x, 3)), Dense((PART_ONE_SIZE - (lc1_s - 1))*lc1_c, 128, relu), Dense(128, 64, relu), Dense(64, 32, relu), Dense(32, out_len), ) |> gpu global ps = Flux.params(model1) elseif model_selected == :m0 rnd = Flux.glorot_uniform l1 = 128 global W1 = rnd(l1, PART_ONE_SIZE) |> gpu global b1 = rnd(l1, 1) |> gpu l2 = 64 global W2 = rnd(l2, l1) |> gpu global b2 = rnd(l2, 1) |> gpu l3 = 16 global W3 = rnd(l3, l2) |> gpu global b3 = rnd(l3, 1) |> gpu global W_last = rnd(out_len, l3) |> gpu global b_last = rnd(out_len, 1) |> gpu global ps = Flux.params(W1, b1, W2, b2, W3, b3, W_last, b_last) end ## end # ps_init() function predict_title_raw(text_part) if model_selected == :m0 l1 = relu.(W1 * text_part .+ b1) l2 = relu.(W2 * l1 .+ b2) l3 = relu.(W3 * l2 .+ b3) l_last = W_last * l3 .+ b_last elseif model_selected == :m1 model1(text_part) end # using =logitcrossentropy= instead of =softmax= end function title_from_raw(raw) raw = cpu(raw) if length(books_v) == 2 raw = sigmoid.(raw) res = map(raw) do x if x >= 0.5f0 return book_names[1], x else return book_names[2], (1-x) end end # These list-comprehensions are returning 1xn matrices instead of vectors. This is because =res= itself is a matrix. return reshape([i[1] for i in res], :), reshape([i[2] for i in res], :) else # @todo/batch raw = softmax(raw) max_i = 0 max_p = -Inf for (i, p) in enumerate(raw) if p > max_p max_p = p max_i = i end end return book_names[max_i], max_p end end function pad_constant_to_length(arr, len; constant=0, kwargs...) pad_n = len - length(arr) if pad_n > 0 return Flux.pad_constant(arr, (0, pad_n), constant; kwargs...) elseif pad_n < 0 return @view arr[1:len] else return arr end end function predict_title(text::AbstractString) text_bytes = text_to_bytes(text) text_bytes = pad_constant_to_length(text_bytes, PART_ONE_SIZE) predict_title(text_bytes) end function predict_title(text_bytes::Union{AbstractVector{UInt8}, AbstractVector{Float32}}) @assert length(text_bytes) == PART_ONE_SIZE "predict_title: length(text_bytes)=$(length(text_bytes))" predict_title(reshape(text_bytes, :, 1)) end function predict_title(batch::AbstractMatrix) raw = predict_title_raw(batch) title_from_raw(raw) end function loss_title(pred, target) loss = 0 # loss = (10^-1)*norm(pred) # @todo decrease this further? # if length(books_v) == 2 loss += Flux.Losses.logitbinarycrossentropy(pred, reshape(target[1, :], 1, :)) # else # loss += Flux.Losses.logitcrossentropy(pred, target) # end return loss end function validate() hsep(log_file) accu_mean = -Inf for ds in (:train_samples_split, :valid_samples_split) eclog("validate: $(ds)") accuracies = Float64[] for book in values(books) eclog("validate: $(book[:name])") samples = book[ds][1:end] accu = sum(map(predict_title(hcat(samples...))[1]) do x x == book[:name] end) / length(samples) push!(accuracies, accu) eclog(accu) end accu_mean = mean(accuracies) eclog("mean(accuracies)=$(accu_mean)") end hsep(log_file) flush(log_file) return accu_mean end #+end_src #+RESULTS: : validate (generic function with 1 method) #+begin_src jupyter-julia :session (night/org-babel-session-name-get "j1") :async yes :pandoc t if @isdefined books_v # @labeled title_from_raw([0.2,0.8]) # @labeled title_from_raw([0.8,0.2]) # @labeled title_from_raw([10,9]) @labeled title_from_raw(Float32.([10 3 2]) |> gpu) @labeled title_from_raw([-10 -4 1]) end nothing #+end_src #+RESULTS: #+begin_src jupyter-julia :session (night/org-babel-session-name-get "j1") :async yes :pandoc t function processBooks(dataset::Symbol=:train_samples ; freeze=false , is = IdDict{Book, Int}() , valid_mode=false , losses=Float64[] , accu_v=Bool[] , batch_size = 64 , i_t_start = 0 , n=10^3 , checkpoint=10^4 , model_dir= "$(dir_base)/processBooks/models/$(Dates.format(Dates.now(), "yy-mm-dd HH:MM") )") interrupted = false i_t = i_t_start # n = ceil(n / batch_size) # checkpoint_n_batches = ceil(checkpoint / batch_size) checkpoint_n_batches = checkpoint # val_n_batches = ceil(10^5 / batch_size) val_n_batches = 10^4 diag_n_batches = 10^4 function model_save() accu = validate() if valid_mode return end dest = "$(model_dir)/$(i_t)_accu=$(round(accu, digits=3))" ensureDir("$(model_dir)/") BSON.@save dest ps=Flux.params(cpu(model1)) i_last=i_t # =opt= not saved # cpu(ps) errors eclog("Saved the model to: $(dest)") hsep(log_file) end try i_books = 1 while i_t <= (i_t_start + n) i_t += 1 input_batch = Matrix{Float32}(undef, PART_ONE_SIZE, batch_size) |> gpu targets = Vector{Symbol}(undef, batch_size) for i_batch in 1:batch_size book = books_v[i_books] i_books = (i_books % length(books_v)) + 1 data = book[dataset] i = get!(is, book, 1) is[book] = (i % length(data)) + 1 d = data[i] d_size = rand(100:(PART_ONE_SIZE - 1)) # @hyperParam i_d_start = rand(1:(PART_BIG_SIZE - (d_size))) d = @view d[i_d_start:(i_d_start + d_size)] d = pad_constant_to_length(d, PART_ONE_SIZE) input_batch[:, i_batch] .= d targets[i_batch] = book[:name] end target_onehot = Flux.onehotbatch(targets, book_names) |> gpu loss_c = nothing pred_raw = nothing loss_calculate() = begin pred_raw = predict_title_raw(input_batch) loss_c = loss_title(pred_raw, target_onehot) return loss_c end if ! valid_mode # target_onehot = Flux.label_smoothing(target_onehot, 0.2f0) # @? gs = gradient(loss_calculate, ps) if ! (freeze) Flux.Optimise.update!(opt, ps, gs) end else loss_calculate() end push!(losses, loss_c) pred, pred_p = title_from_raw(pred_raw) success = (pred .== targets) append!(accu_v, success) if true # ! valid_mode eclog("$(i_t):. accu=$(mean(success)); loss: $(loss_c), mean(p): $(mean(pred_p)), mean(pred_raw): $(mean(pred_raw))") if i_t == i_t_start + 1 || i_t % diag_n_batches == 0 eclog("p:\n$(pred_p)\npred_raw:\n$(pred_raw)\npred:\n$(pred)\ntargets:\n$(targets)\nsuccess:\n$(success)") # size(success): $(size(success)) end if i_t % checkpoint_n_batches == 0 model_save() elseif i_t % val_n_batches == 0 validate() end end end catch ex if ex isa InterruptException || ex isa Flux.Optimise.StopException eclog("Interrupted") interrupted = true else rethrow(ex) end end model_save() return losses, accu_v, i_t, interrupted end #+end_src #+RESULTS: : processBooks (generic function with 2 methods) ** Training #+begin_src jupyter-julia :session (night/org-babel-session-name-get "j1") :async yes :pandoc t ps_init() #+end_src #+RESULTS: : Params([Float32[0.055319645; -0.085456744; 0.19957349; 0.08606427] : : Float32[0.10556889; -0.041268174; -0.14451694; -0.15412125] : : Float32[-0.16258563; 0.11842584; 0.20073889; 0.118710846] : : ... : : Float32[0.18832041; -0.10462217; -0.1294518; -0.0067606773] : : Float32[-0.18592195; 0.08853288; -0.21216692; 0.019245688] : : Float32[-0.13922325; -0.026169788; -0.024823325; -0.076162495], Float32[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 … 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], Float32[-0.007998933 0.008388935 … 0.002452117 -0.007796853; 0.010862431 0.015860489 … 0.017589653 -0.011679318; … ; -0.006528453 0.0041661477 … 5.0855728f-5 -0.005960212; -0.013393072 0.009681716 … 0.010192597 0.009112317], Float32[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 … 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], Float32[0.1741361 0.09063144 … 0.15173556 -0.14428733; -0.11622107 0.091664165 … -0.08898337 0.045376055; … ; -0.15532534 0.15237123 … 0.01752234 -0.056859806; -0.13324068 0.14229198 … -0.07071508 0.123857066], Float32[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 … 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], Float32[-0.0887956 0.17488807 … 0.05877477 0.011693835; -0.20825922 -0.13366467 … 0.009864926 0.038101554; … ; 0.16201395 0.1249339 … -0.0790984 -0.014540076; 0.15982467 -0.1264543 … -0.12867141 -0.1762023], Float32[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 … 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], Float32[-0.4137816 0.09582225 … -0.13326377 -0.108566195], Float32[0.0]]) #+begin_src jupyter-julia :session (night/org-babel-session-name-get "j1") :async yes :pandoc t # opt = Flux.Optimise.ADAM(10^-4) opt = Flux.Optimise.ADAM() #+end_src #+RESULTS: : ADAM(0.001, (0.9, 0.999), IdDict{Any, Any}()) *** Load #+begin_src jupyter-julia :session (night/org-babel-session-name-get "j1") :async yes :pandoc t # BSON.@load "$(dir_base)/processBooks/models/21-11-06 00:39/150000_accu=0.82" ps i_last # opt BSON.@load "$(dir_base)/processBooks/models/good/230000_accu=0.899" ps i_last # opt # BSON.@load "$(dir_base)/processBooks/models/21-11-10 18:58/120000_accu=0.819" ps i_last # opt Flux.loadparams!(model1, ps) model1 |> gpu ps = Flux.params(model1) validate() #+end_src #+RESULTS: : 0.9067307692307692 *** Epoch #+begin_src jupyter-julia :session (night/org-babel-session-name-get "j1") :async yes :pandoc t for i in 1:1 _, _, i_last, interrupted = @time processBooks(; n=10^6, losses, accu_v, is, i_t_start=i_last) if interrupted break end end nothing #+end_src #+RESULTS: : 43cdf771-e9ed-4c08-9597-0637f7935829 #+begin_src jupyter-julia :session (night/org-babel-session-name-get "j1") :async yes :pandoc t @labeled minimum(losses) @labeled length(accu_v) @labeled mean(accu_v) @labeled mean(accu_v[end-200:end]) lineplot(losses) #+end_src #+RESULTS: :RESULTS: : minimum(losses) => 0.2101973593235016 : length(accu_v) => 1011200 : mean(accu_v) => 0.7409533227848101 : mean(accu_v[end - 200:end]) => 0.7562189054726368 : ┌────────────────────────────────────────┐ : 60 │⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀│ : │⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀│ : │⡆⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀│ : │⡇⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀│ : │⡇⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀│ : │⡇⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀│ : │⡇⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀│ : │⡇⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀│ : │⡇⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀│ : │⡇⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀│ : │⡇⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀│ : │⡇⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀│ : │⡇⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀│ : │⡇⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀│ : 0 │⣧⣀⣀⣀⣀⣀⣀⣀⣀⣀⣀⣀⣀⣀⣀⣀⣀⣀⣀⣀⣀⣀⣀⣀⣀⣀⣀⣀⣀⣀⣀⣀⠀⠀⠀⠀⠀⠀⠀⠀│ : └────────────────────────────────────────┘ : ⠀0⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀20000⠀ :END: ** Validation #+begin_src jupyter-julia :session (night/org-babel-session-name-get "j1") :async yes :pandoc t valid_losses, valid_accu_v, _ = processBooks(:valid_samples; valid_mode=true, n=100) #+end_src #+RESULTS: | (0.3621591031551361 0.5376628041267395 0.44329220056533813) | Bool | (0 1 1 1 1 0 1 1 1 1 … 1 1 1 1 1 1 0 1 1 1) | 3 | #+begin_src jupyter-julia :session (night/org-babel-session-name-get "j1") :async yes :pandoc t @labeled mean(valid_losses) @labeled minimum(valid_losses) @labeled maximum(valid_losses) @labeled mean(valid_accu_v) lineplot(valid_losses) #+end_src #+RESULTS: :RESULTS: : mean(valid_losses) => 0.4477047026157379 : minimum(valid_losses) => 0.3621591031551361 : maximum(valid_losses) => 0.5376628041267395 : mean(valid_accu_v) => 0.8020833333333334 : ┌────────────────────────────────────────┐ : 0.6 │⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀│ : │⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀│ : │⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀│ : │⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⣀⠔⠉⠒⠤⣀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀│ : │⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⡠⠊⠀⠀⠀⠀⠀⠀⠉⠒⠤⣀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀│ : │⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢀⠤⠊⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠉⠑⠢⢄⡀⠀⠀⠀⠀⠀⠀⠀│ : │⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢀⠔⠁⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠈⠑⠢⢄⡀⠀⠀⠀│ : │⠀⠀⠀⠀⠀⠀⠀⠀⠀⡠⠔⠁⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠈⠉⠒⢤│ : │⠀⠀⠀⠀⠀⠀⠀⡠⠊⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀│ : │⠀⠀⠀⠀⢀⠔⠊⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀│ : │⠀⠀⢀⠔⠁⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀│ : │⡠⠊⠁⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀│ : │⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀│ : │⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀│ : 0.3 │⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀│ : └────────────────────────────────────────┘ : ⠀1⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀3⠀ :END: #+begin_src jupyter-julia :session (night/org-babel-session-name-get "j1") :async yes :pandoc t validate() #+end_src #+RESULTS: : 0.4984848484848485 #+begin_src jupyter-julia :session (night/org-babel-session-name-get "j1") :async yes :pandoc t @labeled book_names getindex.(books_v,:name) #+end_src #+RESULTS: :RESULTS: : book_names => [:elizium, :luminary] : 2-element Vector{Symbol}: : :elizium : :luminary :END: *** Manual testing #+begin_src jupyter-julia :session (night/org-babel-session-name-get "j1") :async yes :pandoc t @labeled predict_title("Daphne") @labeled predict_title("Sturges") @labeled predict_title("Former") @labeled predict_title("hjsiunx") @labeled predict_title("Sturges closed his eyes and let out a deep sigh before responding.") @labeled predict_title("""Sturges gave a curt nod, before pointing his wand at the stone. Under his command, it rose into the air, made a loop and fell back to the ground. "It worked," the Auror said, as if he couldn't believe it. Harry snorted.""") @labeled predict_title(""""I'm not a Healer, I wouldn't know," the Auror evasively replied. "Besides, what does it matter now? The boat's gone, Azkaban is sinking, and we're standing around talking." "It's important because there might be someone else loose on this island! If they found Personal Effects, then they're armed too." Sturges froze for a moment, becoming a statue, before shaking his head. "Shite, you've got a point - I can't say you're wrong. Bollocks, it's possible. I did a sweep around the Abyss, looking for others, but that doesn't mean I found them all." """) @labeled predict_title(""" As a way to light one's path, there was no finer spell than the Thief's Guide. It was a conjured light invisible to others, but it came with a price. The spell took almost thirty seconds to form, and required the same amount of time to wind down, before any other magic would channel through the wand. He moved to within a foot of the high cliff face. Dolohov had the high ground, providing an eagle eye view of the island and Harry hoped to hide by sticking close to the rocky terrain. He flew along the shore, traveling in a counter-clockwise trajectory around the island, towards the north side. From behind him a bright light flashed, bathing everything in harsh white light. Harry started to bring his wand up, before the jagged roar of a thunderclap echoed through the air, so close it rattled his teeth within their sockets, filling his skull with ringing pain. Agony ripped through his head as he rose upward, until the overhang atop the walls of the Abyss stretched overhead, providing temporary reprieve from the storm. """) nothing #+end_src #+RESULTS: : predict_title("Daphne") => ([:luminary], Float32[0.8800906]) : predict_title("Sturges") => ([:luminary], Float32[0.91894454]) : predict_title("Former") => ([:luminary], Float32[0.87178886]) : predict_title("hjsiunx") => ([:luminary], Float32[0.91926265]) : predict_title("Sturges closed his eyes and let out a deep sigh before responding.") => ([:luminary], Float32[0.6961006]) : predict_title("Sturges gave a curt nod, before pointing his wand at the stone. Under his command, it rose into the air, made a loop and fell back to the ground.\n\n\"It worked,\" the Auror said, as if he couldn't believe it.\n\nHarry snorted.") => ([:elizium], Float32[0.9949986]) : predict_title("Death Eaters aren't worth the regard you'd show a cockroach. The things they've done… they're not even human anymore. They all deserve to suffer, to live every moment in fear, in agony.") => ([:luminary], Float32[0.8097029]) : predict_title("\"Death Eaters aren't worth the regard you'd show a cockroach. The things they've done… they're not even human anymore. They all deserve to suffer, to live every moment in fear, in agony.\"\n\nThe Auror opened his eyes, keeping his gaze level as he continued.\n\n\"For most of them, the Dementors do the job. But day after day, when you see that Bolshevik piece of shit sitting in his cell, like he's on a fucking vacation… someone has to balance the scales, Potter.\"\n") => ([:elizium], Float32[0.7296372]) : predict_title("\"I'm not a Healer, I wouldn't know,\" the Auror evasively replied. \"Besides, what does it matter now? The boat's gone, Azkaban is sinking, and we're standing around talking.\"\n\n\"It's important because there might be someone else loose on this island! If they found Personal Effects, then they're armed too.\"\n\nSturges froze for a moment, becoming a statue, before shaking his head.\n\n\"Shite, you've got a point - I can't say you're wrong. Bollocks, it's possible. I did a sweep around the Abyss, looking for others, but that doesn't mean I found them all.\" ") => ([:elizium], Float32[0.9999281]) : predict_title("As a way to light one's path, there was no finer spell than the Thief's Guide. It was a conjured light invisible to others, but it came with a price. The spell took almost thirty seconds to form, and required the same amount of time to wind down, before any other magic would channel through the wand.\n\nHe moved to within a foot of the high cliff face. Dolohov had the high ground, providing an eagle eye view of the island and Harry hoped to hide by sticking close to the rocky terrain. He flew along the shore, traveling in a counter-clockwise trajectory around the island, towards the north side.\n\nFrom behind him a bright light flashed, bathing everything in harsh white light. Harry started to bring his wand up, before the jagged roar of a thunderclap echoed through the air, so close it rattled his teeth within their sockets, filling his skull with ringing pain. Agony ripped through his head as he rose upward, until the overhang atop the walls of the Abyss stretched overhead, providing temporary reprieve from the storm.\n") => ([:elizium], Float32[0.5888242]) **** _ #+begin_src jupyter-julia :session (night/org-babel-session-name-get "j1") :async yes :pandoc t predict_title("""Death Eaters aren't worth the regard you'd show a cockroach. The things they've done… they're not even human anymore. They all deserve to suffer, to live every moment in fear, in agony.""") #+end_src #+RESULTS: | (:luminary) | Float32 | (0.8097029) | #+begin_src jupyter-julia :session (night/org-babel-session-name-get "j1") :async yes :pandoc t predict_title(""""Death Eaters aren't worth the regard you'd show a cockroach. The things they've done… they're not even human anymore. They all deserve to suffer, to live every moment in fear, in agony." The Auror opened his eyes, keeping his gaze level as he continued. "For most of them, the Dementors do the job. But day after day, when you see that Bolshevik piece of shit sitting in his cell, like he's on a fucking vacation… someone has to balance the scales, Potter." """) #+end_src #+RESULTS: | (:elizium) | Float32 | (0.7296372) | #+begin_src jupyter-julia :session (night/org-babel-session-name-get "j1") :async yes :pandoc t predict_title(""""Daphnes aren't worth the regard you'd show a cockroach. The things they've done… they're not even human anymore. They all deserve to suffer, to live every moment in fear, in agony." The Auror opened his eyes, keeping his gaze level as he continued. "For most of them, the Dementors do the job. But day after day, when you see that Bolshevik piece of shit sitting in his cell, like he's on a fucking vacation… someone has to balance the scales, Potter." """) #+end_src #+RESULTS: | (:luminary) | Float32 | (0.7834104) | #+begin_src jupyter-julia :session (night/org-babel-session-name-get "j1") :async yes :pandoc t predict_title(""""Death Eaters aren't worth the regard you'd show Daphne. The things they've done… they're not even human anymore. They all deserve to suffer, to live every moment in fear, in agony." The Auror opened his eyes, keeping his gaze level as he continued. "For most of them, the Dementors do the job. But day after day, when you see that Bolshevik piece of shit sitting in his cell, like he's on a fucking vacation… someone has to balance the scales, Potter." """) #+end_src #+RESULTS: | (:luminary) | Float32 | (0.6188742) | #+begin_src jupyter-julia :session (night/org-babel-session-name-get "j1") :async yes :pandoc t predict_title(""""Death Eaters aren't worth the regard you'd show a cockroach. The things they've done… they're not even human anymore. They all deserve to suffer, to live every moment in fear, in agony." Daphne opened his eyes, keeping his gaze level as he continued. "For most of them, the Dementors do the job. But day after day, when you see that Bolshevik piece of shit sitting in his cell, like he's on a fucking vacation… someone has to balance the scales, Potter." """) #+end_src #+RESULTS: | (:luminary) | Float32 | (0.9481187) | #+begin_src jupyter-julia :session (night/org-babel-session-name-get "j1") :async yes :pandoc t for s in ("Sturges ", "Sturges", "\tSturges\t", "\nSturges\n", "Daphne ", "Daphne", "Harry ", "Harry Potter ", "former death eater ", "former Death Eater ", "former Death Eater") for i in 1:3:1000 sr = repeat(s, i) sr_len = length(codeunits(sr)) println("\"$(s)\"|$(i) ($(sr_len)): $(predict_title(sr))") if sr_len >= 500 break end end end #+end_src #+RESULTS: : "Sturges "|1 (8): ([:luminary], Float32[0.9084659]) : "Sturges "|4 (32): ([:luminary], Float32[0.78637934]) : "Sturges "|7 (56): ([:luminary], Float32[0.7511015]) : "Sturges "|10 (80): ([:luminary], Float32[0.7167213]) : "Sturges "|13 (104): ([:luminary], Float32[0.7197282]) : "Sturges "|16 (128): ([:luminary], Float32[0.5586547]) : "Sturges "|19 (152): ([:luminary], Float32[0.69433963]) : "Sturges "|22 (176): ([:elizium], Float32[0.5284413]) : "Sturges "|25 (200): ([:elizium], Float32[0.7057922]) : "Sturges "|28 (224): ([:elizium], Float32[0.6231144]) : "Sturges "|31 (248): ([:elizium], Float32[0.64291555]) : "Sturges "|34 (272): ([:elizium], Float32[0.93219006]) : "Sturges "|37 (296): ([:elizium], Float32[0.94266087]) : "Sturges "|40 (320): ([:elizium], Float32[0.9496109]) : "Sturges "|43 (344): ([:elizium], Float32[0.9398242]) : "Sturges "|46 (368): ([:elizium], Float32[0.95951897]) : "Sturges "|49 (392): ([:elizium], Float32[0.97920173]) : "Sturges "|52 (416): ([:elizium], Float32[0.9904779]) : "Sturges "|55 (440): ([:elizium], Float32[0.9872892]) : "Sturges "|58 (464): ([:elizium], Float32[0.9981779]) : "Sturges "|61 (488): ([:elizium], Float32[0.9985221]) : "Sturges "|64 (512): ([:elizium], Float32[0.99998176]) : "Sturges"|1 (7): ([:luminary], Float32[0.91894454]) : "Sturges"|4 (28): ([:luminary], Float32[0.69747823]) : "Sturges"|7 (49): ([:luminary], Float32[0.7664588]) : "Sturges"|10 (70): ([:luminary], Float32[0.7631384]) : "Sturges"|13 (91): ([:luminary], Float32[0.89710164]) : "Sturges"|16 (112): ([:luminary], Float32[0.816814]) : "Sturges"|19 (133): ([:luminary], Float32[0.7467115]) : "Sturges"|22 (154): ([:luminary], Float32[0.81807005]) : "Sturges"|25 (175): ([:luminary], Float32[0.817155]) : "Sturges"|28 (196): ([:luminary], Float32[0.87153524]) : "Sturges"|31 (217): ([:luminary], Float32[0.837211]) : "Sturges"|34 (238): ([:luminary], Float32[0.82632333]) : "Sturges"|37 (259): ([:luminary], Float32[0.7658739]) : "Sturges"|40 (280): ([:luminary], Float32[0.78688633]) : "Sturges"|43 (301): ([:luminary], Float32[0.79343736]) : "Sturges"|46 (322): ([:luminary], Float32[0.8122535]) : "Sturges"|49 (343): ([:luminary], Float32[0.77966666]) : "Sturges"|52 (364): ([:luminary], Float32[0.7658739]) : "Sturges"|55 (385): ([:luminary], Float32[0.7658739]) : "Sturges"|58 (406): ([:luminary], Float32[0.7280034]) : "Sturges"|61 (427): ([:luminary], Float32[0.71619225]) : "Sturges"|64 (448): ([:luminary], Float32[0.7658739]) : "Sturges"|67 (469): ([:luminary], Float32[0.7926266]) : "Sturges"|70 (490): ([:luminary], Float32[0.9476907]) : "Sturges"|73 (511): ([:luminary], Float32[0.82291657]) : " Sturges "|1 (9): ([:luminary], Float32[0.98466915]) : " Sturges "|4 (36): ([:luminary], Float32[0.99738425]) : " Sturges "|7 (63): ([:luminary], Float32[0.9153329]) : " Sturges "|10 (90): ([:elizium], Float32[0.9438124]) : " Sturges "|13 (117): ([:elizium], Float32[0.9999902]) : " Sturges "|16 (144): ([:elizium], Float32[1.0]) : " Sturges "|19 (171): ([:elizium], Float32[1.0]) : " Sturges "|22 (198): ([:elizium], Float32[1.0]) : " Sturges "|25 (225): ([:elizium], Float32[1.0]) : " Sturges "|28 (252): ([:elizium], Float32[1.0]) : " Sturges "|31 (279): ([:elizium], Float32[1.0]) : " Sturges "|34 (306): ([:elizium], Float32[1.0]) : " Sturges "|37 (333): ([:elizium], Float32[1.0]) : " Sturges "|40 (360): ([:elizium], Float32[1.0]) : " Sturges "|43 (387): ([:elizium], Float32[1.0]) : " Sturges "|46 (414): ([:elizium], Float32[1.0]) : " Sturges "|49 (441): ([:elizium], Float32[1.0]) : " Sturges "|52 (468): ([:elizium], Float32[1.0]) : " Sturges "|55 (495): ([:elizium], Float32[1.0]) : " Sturges "|58 (522): ([:elizium], Float32[1.0]) : " : Sturges : "|1 (9): ([:luminary], Float32[0.9837257]) : " : Sturges : "|4 (36): ([:luminary], Float32[0.99785995]) : " : Sturges : "|7 (63): ([:luminary], Float32[0.9215182]) : " : Sturges : "|10 (90): ([:elizium], Float32[0.9145907]) : " : Sturges : "|13 (117): ([:elizium], Float32[0.9999672]) : " : Sturges : "|16 (144): ([:elizium], Float32[1.0]) : " : Sturges : "|19 (171): ([:elizium], Float32[1.0]) : " : Sturges : "|22 (198): ([:elizium], Float32[1.0]) : " : Sturges : "|25 (225): ([:elizium], Float32[1.0]) : " : Sturges : "|28 (252): ([:elizium], Float32[1.0]) : " : Sturges : "|31 (279): ([:elizium], Float32[1.0]) : " : Sturges : "|34 (306): ([:elizium], Float32[1.0]) : " : Sturges : "|37 (333): ([:elizium], Float32[1.0]) : " : Sturges : "|40 (360): ([:elizium], Float32[1.0]) : " : Sturges : "|43 (387): ([:elizium], Float32[1.0]) : " : Sturges : "|46 (414): ([:elizium], Float32[1.0]) : " : Sturges : "|49 (441): ([:elizium], Float32[1.0]) : " : Sturges : "|52 (468): ([:elizium], Float32[1.0]) : " : Sturges : "|55 (495): ([:elizium], Float32[1.0]) : " : Sturges : "|58 (522): ([:elizium], Float32[1.0]) : "Daphne "|1 (7): ([:luminary], Float32[0.8615514]) : "Daphne "|4 (28): ([:luminary], Float32[0.57944894]) : "Daphne "|7 (49): ([:elizium], Float32[0.5802219]) : "Daphne "|10 (70): ([:elizium], Float32[0.7709762]) : "Daphne "|13 (91): ([:elizium], Float32[0.856806]) : "Daphne "|16 (112): ([:elizium], Float32[0.8962437]) : "Daphne "|19 (133): ([:elizium], Float32[0.9360701]) : "Daphne "|22 (154): ([:elizium], Float32[0.9451292]) : "Daphne "|25 (175): ([:elizium], Float32[0.9580155]) : "Daphne "|28 (196): ([:elizium], Float32[0.97587323]) : "Daphne "|31 (217): ([:elizium], Float32[0.9858964]) : "Daphne "|34 (238): ([:elizium], Float32[0.99068415]) : "Daphne "|37 (259): ([:elizium], Float32[0.9954436]) : "Daphne "|40 (280): ([:elizium], Float32[0.99323386]) : "Daphne "|43 (301): ([:elizium], Float32[0.99595743]) : "Daphne "|46 (322): ([:elizium], Float32[0.997063]) : "Daphne "|49 (343): ([:elizium], Float32[0.996554]) : "Daphne "|52 (364): ([:elizium], Float32[0.9976845]) : "Daphne "|55 (385): ([:elizium], Float32[0.9985098]) : "Daphne "|58 (406): ([:elizium], Float32[0.9980161]) : "Daphne "|61 (427): ([:elizium], Float32[0.9989992]) : "Daphne "|64 (448): ([:elizium], Float32[0.9989015]) : "Daphne "|67 (469): ([:elizium], Float32[0.99903274]) : "Daphne "|70 (490): ([:elizium], Float32[0.9989741]) : "Daphne "|73 (511): ([:elizium], Float32[0.99969304]) : "Daphne"|1 (6): ([:luminary], Float32[0.8800906]) : "Daphne"|4 (24): ([:luminary], Float32[0.80955136]) : "Daphne"|7 (42): ([:luminary], Float32[0.87359834]) : "Daphne"|10 (60): ([:luminary], Float32[0.7579535]) : "Daphne"|13 (78): ([:luminary], Float32[0.71352124]) : "Daphne"|16 (96): ([:luminary], Float32[0.66352475]) : "Daphne"|19 (114): ([:luminary], Float32[0.7255676]) : "Daphne"|22 (132): ([:luminary], Float32[0.79945016]) : "Daphne"|25 (150): ([:luminary], Float32[0.7297182]) : "Daphne"|28 (168): ([:luminary], Float32[0.8448794]) : "Daphne"|31 (186): ([:luminary], Float32[0.8565506]) : "Daphne"|34 (204): ([:luminary], Float32[0.8138204]) : "Daphne"|37 (222): ([:luminary], Float32[0.8336204]) : "Daphne"|40 (240): ([:luminary], Float32[0.82503927]) : "Daphne"|43 (258): ([:luminary], Float32[0.8094206]) : "Daphne"|46 (276): ([:luminary], Float32[0.90847826]) : "Daphne"|49 (294): ([:luminary], Float32[0.77766144]) : "Daphne"|52 (312): ([:luminary], Float32[0.8013817]) : "Daphne"|55 (330): ([:luminary], Float32[0.82758427]) : "Daphne"|58 (348): ([:luminary], Float32[0.82149667]) : "Daphne"|61 (366): ([:luminary], Float32[0.67405784]) : "Daphne"|64 (384): ([:luminary], Float32[0.7831406]) : "Daphne"|67 (402): ([:luminary], Float32[0.7658739]) : "Daphne"|70 (420): ([:luminary], Float32[0.7658739]) : "Daphne"|73 (438): ([:luminary], Float32[0.7658739]) : "Daphne"|76 (456): ([:luminary], Float32[0.80686116]) : "Daphne"|79 (474): ([:luminary], Float32[0.7658739]) : "Daphne"|82 (492): ([:luminary], Float32[0.99999803]) : "Daphne"|85 (510): ([:luminary], Float32[0.82291657]) : "Harry "|1 (6): ([:luminary], Float32[0.90520597]) : "Harry "|4 (24): ([:luminary], Float32[0.8497299]) : "Harry "|7 (42): ([:luminary], Float32[0.9232881]) : "Harry "|10 (60): ([:luminary], Float32[0.899124]) : "Harry "|13 (78): ([:luminary], Float32[0.9456601]) : "Harry "|16 (96): ([:luminary], Float32[0.97567326]) : "Harry "|19 (114): ([:luminary], Float32[0.9579711]) : "Harry "|22 (132): ([:luminary], Float32[0.98196566]) : "Harry "|25 (150): ([:luminary], Float32[0.9895813]) : "Harry "|28 (168): ([:luminary], Float32[0.98945683]) : "Harry "|31 (186): ([:luminary], Float32[0.99331117]) : "Harry "|34 (204): ([:luminary], Float32[0.9968096]) : "Harry "|37 (222): ([:luminary], Float32[0.9982335]) : "Harry "|40 (240): ([:luminary], Float32[0.9980396]) : "Harry "|43 (258): ([:luminary], Float32[0.99790424]) : "Harry "|46 (276): ([:luminary], Float32[0.9982446]) : "Harry "|49 (294): ([:luminary], Float32[0.9992256]) : "Harry "|52 (312): ([:luminary], Float32[0.9994071]) : "Harry "|55 (330): ([:luminary], Float32[0.9998256]) : "Harry "|58 (348): ([:luminary], Float32[0.99977195]) : "Harry "|61 (366): ([:luminary], Float32[0.9998938]) : "Harry "|64 (384): ([:luminary], Float32[0.9998329]) : "Harry "|67 (402): ([:luminary], Float32[0.99977374]) : "Harry "|70 (420): ([:luminary], Float32[0.99994034]) : "Harry "|73 (438): ([:luminary], Float32[0.99977124]) : "Harry "|76 (456): ([:luminary], Float32[0.9998982]) : "Harry "|79 (474): ([:luminary], Float32[0.99990094]) : "Harry "|82 (492): ([:luminary], Float32[0.9999482]) : "Harry "|85 (510): ([:luminary], Float32[0.99983907]) : "Harry Potter "|1 (13): ([:luminary], Float32[0.82890207]) : "Harry Potter "|4 (52): ([:luminary], Float32[0.8515082]) : "Harry Potter "|7 (91): ([:luminary], Float32[0.93710357]) : "Harry Potter "|10 (130): ([:luminary], Float32[0.91286415]) : "Harry Potter "|13 (169): ([:luminary], Float32[0.92091745]) : "Harry Potter "|16 (208): ([:luminary], Float32[0.9332891]) : "Harry Potter "|19 (247): ([:luminary], Float32[0.96434355]) : "Harry Potter "|22 (286): ([:luminary], Float32[0.9825712]) : "Harry Potter "|25 (325): ([:luminary], Float32[0.99254185]) : "Harry Potter "|28 (364): ([:luminary], Float32[0.99375033]) : "Harry Potter "|31 (403): ([:luminary], Float32[0.9981972]) : "Harry Potter "|34 (442): ([:luminary], Float32[0.9988368]) : "Harry Potter "|37 (481): ([:luminary], Float32[0.9984812]) : "Harry Potter "|40 (520): ([:luminary], Float32[0.9716197]) : "former death eater "|1 (19): ([:luminary], Float32[0.89179957]) : "former death eater "|4 (76): ([:luminary], Float32[0.87866825]) : "former death eater "|7 (133): ([:luminary], Float32[0.79445416]) : "former death eater "|10 (190): ([:luminary], Float32[0.84965074]) : "former death eater "|13 (247): ([:luminary], Float32[0.85968727]) : "former death eater "|16 (304): ([:luminary], Float32[0.8823935]) : "former death eater "|19 (361): ([:luminary], Float32[0.9766743]) : "former death eater "|22 (418): ([:luminary], Float32[0.955864]) : "former death eater "|25 (475): ([:luminary], Float32[0.9771394]) : "former death eater "|28 (532): ([:luminary], Float32[0.9284054]) : "former Death Eater "|1 (19): ([:luminary], Float32[0.89483047]) : "former Death Eater "|4 (76): ([:luminary], Float32[0.91536283]) : "former Death Eater "|7 (133): ([:luminary], Float32[0.59517527]) : "former Death Eater "|10 (190): ([:luminary], Float32[0.6829245]) : "former Death Eater "|13 (247): ([:luminary], Float32[0.6325604]) : "former Death Eater "|16 (304): ([:luminary], Float32[0.78703505]) : "former Death Eater "|19 (361): ([:luminary], Float32[0.8949729]) : "former Death Eater "|22 (418): ([:luminary], Float32[0.89971715]) : "former Death Eater "|25 (475): ([:luminary], Float32[0.9180496]) : "former Death Eater "|28 (532): ([:elizium], Float32[0.5988412]) : "former Death Eater"|1 (18): ([:luminary], Float32[0.86180055]) : "former Death Eater"|4 (72): ([:luminary], Float32[0.7002988]) : "former Death Eater"|7 (126): ([:luminary], Float32[0.59264016]) : "former Death Eater"|10 (180): ([:luminary], Float32[0.60319144]) : "former Death Eater"|13 (234): ([:luminary], Float32[0.6880965]) : "former Death Eater"|16 (288): ([:luminary], Float32[0.66495836]) : "former Death Eater"|19 (342): ([:luminary], Float32[0.7718569]) : "former Death Eater"|22 (396): ([:luminary], Float32[0.86179024]) : "former Death Eater"|25 (450): ([:luminary], Float32[0.708271]) : "former Death Eater"|28 (504): ([:elizium], Float32[0.8494184]) *** _ #+begin_src jupyter-julia :session (night/org-babel-session-name-get "j1") :async yes :pandoc t bella(:mbp) #+end_src #+RESULTS: : /usr/bin/env: invalid option -- 'S' : Try '/usr/bin/env --help' for more information.