{
"cells": [
{
"cell_type": "markdown",
"id": "693de1e8",
"metadata": {},
"source": [
"## Andrej's makemore lecture - 3\n",
"\n",
"Implementation following Andrej Karpathy's lecture [Building makemore Part 3: Activations & Gradients, BatchNorm](https://youtu.be/P6sfmUTpUmc).\n",
"\n",
"I have some liberty to refactor and pythonise his implementation."
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "1cdd0780",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
" \n",
" "
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"import black\n",
"import jupyter_black\n",
"\n",
"jupyter_black.load(\n",
" lab=False,\n",
" line_length=79,\n",
" target_version=black.TargetVersion.PY310,\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "246c1917",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
""
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"from IPython.display import display, HTML, clear_output\n",
"\n",
"display(HTML(\"\"))\n",
"\n",
"\n",
"from dataclasses import dataclass, field\n",
"import typing as t\n",
"import itertools as it\n",
"import collections as c\n",
"import json\n",
"from copy import deepcopy\n",
"import math\n",
"import time\n",
"import functools as ft\n",
"import numpy as np\n",
"import random\n",
"from tqdm.notebook import tqdm\n",
"import heapq\n",
"import torch as T\n",
"import torch.nn.functional as F\n",
"import matplotlib.pyplot as plt\n",
"import seaborn as sns\n",
"import torch.utils.tensorboard as tb\n",
"\n",
"plt.rcParams[\"figure.figsize\"] = (12, 4)\n",
"plt.rcParams[\"font.size\"] = 14"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "cae62e50",
"metadata": {},
"outputs": [],
"source": [
"# Number of past tokens to use to predict next token\n",
"CTX_WIN_SZ = 3"
]
},
{
"cell_type": "markdown",
"id": "d0a4107a",
"metadata": {},
"source": [
"### Load data"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "918744d9",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"['emma', 'olivia', 'ava', 'isabella']"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"DOT = \".\"\n",
"words = open(\"names.txt\").read().splitlines()\n",
"words[:4]"
]
},
{
"cell_type": "markdown",
"id": "130b61aa",
"metadata": {},
"source": [
"#### Build mapping of character to index"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "40d8add8",
"metadata": {},
"outputs": [],
"source": [
"def build_ixes(words):\n",
" chars = [DOT] + sorted(set(it.chain.from_iterable(words)))\n",
" nchars = len(chars)\n",
" ctoix = {c: i for i, c in enumerate(chars)}\n",
" ixtoc = dict(enumerate(chars))\n",
" return (ctoix, ixtoc)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "7c18c2f8",
"metadata": {},
"outputs": [],
"source": [
"(ctoix, ixtoc) = build_ixes(words)"
]
},
{
"cell_type": "markdown",
"id": "03333c0e",
"metadata": {},
"source": [
"#### Create training data with context window size"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "ca5067d1",
"metadata": {},
"outputs": [],
"source": [
"def build_train_data(words, ctoix, ctx_win=CTX_WIN_SZ):\n",
" Xs, Ys = [], []\n",
" pad = DOT * ctx_win\n",
" for wnum, w in enumerate(words):\n",
" pw = pad + w + DOT\n",
" if wnum < 2:\n",
" print(pw)\n",
" for i in range(len(w) + 1):\n",
" if wnum < 2:\n",
" print(pw[i : i + ctx_win], \"--->\", pw[i + ctx_win])\n",
" Xs.append([ctoix[c] for c in pw[i : i + ctx_win]])\n",
" Ys.append([ctoix[pw[i + ctx_win]]])\n",
" return T.tensor(Xs, dtype=int), T.tensor(Ys, dtype=int).flatten()"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "784d2742",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"...emma.\n",
"... ---> e\n",
"..e ---> m\n",
".em ---> m\n",
"emm ---> a\n",
"mma ---> .\n",
"...olivia.\n",
"... ---> o\n",
"..o ---> l\n",
".ol ---> i\n",
"oli ---> v\n",
"liv ---> i\n",
"ivi ---> a\n",
"via ---> .\n"
]
},
{
"data": {
"text/plain": [
"(228146, 3)"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"Xs, Ys = build_train_data(words, ctoix, ctx_win=CTX_WIN_SZ)\n",
"n, m = Xs.shape\n",
"n, m"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "2bff6fc0",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"tensor([ 5, 13, 13, 1, 0, 15, 12, 9, 22, 9, 1, 0, 1, 22, 1, 0, 9, 19,\n",
" 1, 2, 5, 12, 12, 1, 0, 19, 15, 16, 8, 9])"
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"Ys[:30]"
]
},
{
"cell_type": "markdown",
"id": "5efb3e62",
"metadata": {},
"source": [
"#### Train, validation, test split"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "88e03a4b",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"27"
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"NCHARS = len(ctoix)\n",
"NCHARS"
]
},
{
"cell_type": "code",
"execution_count": 11,
"id": "f3b7b224",
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"ntrain=182516, nval=22814, ntest=22816\n"
]
}
],
"source": [
"ntrain, nval = int(n * 0.8), int(n * 0.1)\n",
"ntest = n - ntrain - nval\n",
"print(f\"{ntrain=}, {nval=}, {ntest=}\")\n",
"ixes = list(range(0, n))\n",
"random.shuffle(ixes)\n",
"valend = ntrain + nval\n",
"ixtr, ixval, ixtest = ixes[:ntrain], ixes[ntrain:valend], ixes[valend:]"
]
},
{
"cell_type": "code",
"execution_count": 12,
"id": "63041722",
"metadata": {},
"outputs": [],
"source": [
"(Xtr, Ytr), (Xval, Yval) = (Xs[ixtr], Ys[ixtr]), (Xs[ixval], Ys[ixval])\n",
"(Xtest, Ytest) = (Xs[ixtest], Ys[ixtest])"
]
},
{
"cell_type": "code",
"execution_count": 13,
"id": "6e639831",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(tensor([[ 0, 0, 12],\n",
" [ 0, 13, 9],\n",
" [ 0, 20, 1],\n",
" ...,\n",
" [15, 13, 1],\n",
" [ 1, 22, 1],\n",
" [12, 15, 18]]),\n",
" tensor([15, 18, 2, ..., 25, 0, 1]))"
]
},
"execution_count": 13,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"Xtest, Ytest"
]
},
{
"cell_type": "code",
"execution_count": 14,
"id": "fc404c8e",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"torch.Size([182516, 3]) torch.Size([22814, 3]) torch.Size([22816, 3])\n"
]
}
],
"source": [
"print(Xtr.shape, Xval.shape, Xtest.shape)"
]
},
{
"cell_type": "markdown",
"id": "c044dfc3",
"metadata": {},
"source": [
"### Model layer implementations\n",
"\n",
"Requires building following layers\n",
"\n",
"- Embedding\n",
"- Linear layer\n",
"- BatchNorm1D\n",
"- Tanh"
]
},
{
"cell_type": "code",
"execution_count": 15,
"id": "e43faff1",
"metadata": {},
"outputs": [],
"source": [
"@dataclass\n",
"class Embedding:\n",
"\n",
" num_embed: int\n",
" embed_dim: int\n",
" E: T.Tensor = field(init=False, repr=False)\n",
" _params: list[T.Tensor] = field(init=False, repr=False)\n",
" #: last forward pass, mutated during forward pass\n",
" out: t.Optional[T.Tensor] = field(default=None, repr=False)\n",
"\n",
" def __post_init__(self):\n",
" self.E = T.randn(self.num_embed, self.embed_dim)\n",
" self._params = [self.E]\n",
" self.parameters = lambda: self._params\n",
"\n",
" def __call__(self, X):\n",
" batch_sz, num_terms = X.shape\n",
" self.out = self.E[X].view(batch_sz, -1)\n",
" return self.out\n",
"\n",
"\n",
"@dataclass\n",
"class Linear:\n",
"\n",
" fanin: int\n",
" fanout: int\n",
" bias: bool = True\n",
" # gain used in kaiming he activation\n",
" wt_gain: float = 1.0\n",
" # if not set we use kaiming he activation\n",
" init_wt_scale: t.Optional[float] = None\n",
" b: t.Optional[T.Tensor] = field(init=False, repr=False)\n",
" W: T.Tensor = field(init=False, repr=False)\n",
" _params: list[T.Tensor] = field(init=False, repr=False)\n",
" #: last forward pass, mutated during forward pass\n",
" out: t.Optional[T.Tensor] = field(default=None, repr=False)\n",
"\n",
" def __post_init__(self):\n",
" if self.init_wt_scale is None:\n",
" self.init_wt_scale = self.wt_gain / (self.fanin**0.5)\n",
" # sample from uniform random\n",
" self.W = T.FloatTensor(self.fanin, self.fanout).uniform_(\n",
" -self.init_wt_scale, self.init_wt_scale\n",
" )\n",
" if self.bias:\n",
" self.b = T.ones(1, self.fanout) * 0.01\n",
" self._params = [self.W, self.b]\n",
" else:\n",
" self.b = None\n",
" self._params = [self.W]\n",
" self.parameters = lambda: self._params\n",
"\n",
" def __call__(self, X):\n",
" if self.bias:\n",
" self.out = X @ self.W + self.b\n",
" else:\n",
" self.out = X @ self.W\n",
" return self.out\n",
"\n",
"\n",
"@dataclass\n",
"class BatchNorm1D:\n",
"\n",
" size: int\n",
" momentum: float = 0.01\n",
" eps: float = 1e-5\n",
" #: scaling after standardising\n",
" gamma: T.Tensor = field(init=False, repr=False)\n",
" #: shift after standardising\n",
" beta: T.Tensor = field(init=False, repr=False)\n",
" _params: list[T.Tensor] = field(init=False, repr=False)\n",
" #: running averages of mean and variance\n",
" buffer_mean: T.Tensor = field(init=False, repr=False)\n",
" buffer_var: T.Tensor = field(init=False, repr=False)\n",
" #: last forward pass, mutated during forward pass\n",
" out: t.Optional[T.Tensor] = field(default=None, repr=False)\n",
"\n",
" def __post_init__(self):\n",
" self.gamma = T.ones(1, self.size)\n",
" self.beta = T.zeros(1, self.size)\n",
" self._params = [self.gamma, self.beta]\n",
" self.parameters = lambda: self._params\n",
" self.buffer_mean = T.zeros(1, self.size, requires_grad=False)\n",
" self.buffer_var = T.ones(1, self.size, requires_grad=False)\n",
"\n",
" def __call__(self, X, inference=False):\n",
" fwd_fn = self._fwd_inference if inference else self._fwd_train\n",
" return fwd_fn(X)\n",
"\n",
" def _fwd_train(self, X):\n",
" mu = X.mean(dim=0, keepdims=True)\n",
" var = X.var(dim=0, keepdims=True) + self.eps\n",
" with T.no_grad():\n",
" mom_old, mom_new = 1 - self.momentum, self.momentum\n",
" self.buffer_mean = mom_old * self.buffer_mean + mom_new * mu\n",
" self.buffer_var = mom_old * self.buffer_var + mom_new * var\n",
" self.out = BatchNorm1D._fwd(\n",
" X=X, mu=mu, var=var, gamma=self.gamma, beta=self.beta\n",
" )\n",
" return self.out\n",
"\n",
" def _fwd_inference(self, X):\n",
" with T.no_grad():\n",
" self.out = BatchNorm1D._fwd(\n",
" X=X,\n",
" mu=self.buffer_mean,\n",
" var=self.buffer_var,\n",
" gamma=self.gamma,\n",
" beta=self.beta,\n",
" )\n",
" return self.out\n",
"\n",
" @staticmethod\n",
" def _fwd(X, mu, var, gamma, beta):\n",
" X_std = (X - mu) / T.sqrt(var)\n",
" return (X_std * gamma) + beta\n",
"\n",
"\n",
"@dataclass\n",
"class Tanh:\n",
"\n",
" #: last forward pass, mutated during forward pass\n",
" out: t.Optional[T.Tensor] = field(default=None, repr=False)\n",
"\n",
" def parameters(self):\n",
" return []\n",
"\n",
" def __call__(self, X):\n",
" self.out = T.tanh(X)\n",
" return self.out"
]
},
{
"cell_type": "markdown",
"id": "c1b154ed",
"metadata": {},
"source": [
"### Training loop\n",
"- Split data into batches\n",
"- Fwd pass, zero grad, loss.backward, batch gradient update.\n",
"- Keep track of learning losses for later plotting."
]
},
{
"cell_type": "code",
"execution_count": 16,
"id": "c39a65b9",
"metadata": {},
"outputs": [],
"source": [
"def train_loop(\n",
" Xs,\n",
" Ys,\n",
" mdl,\n",
" lr,\n",
" num_iter,\n",
" max_sub_iter=None,\n",
" batch_sz=128,\n",
" losses=None,\n",
" wt_update_ratios=None,\n",
" verbose=True,\n",
"):\n",
" # train_ix to loss\n",
" wt_update_ratios = wt_update_ratios if wt_update_ratios is not None else {}\n",
" losses = losses if losses is not None else []\n",
" max_sub_iter = max_sub_iter or np.inf\n",
" nrows = Xs.shape[0]\n",
" ixes = list(range(nrows))\n",
" if verbose:\n",
" itrs = tqdm(range(num_iter))\n",
" else:\n",
" itrs = range(num_iter)\n",
" total_iter = len(losses)\n",
" for i in itrs:\n",
" total_iter += 1\n",
" random.shuffle(ixes)\n",
" for sub_iter, begix in enumerate(\n",
" T.arange(0, nrows, batch_sz), start=1\n",
" ):\n",
" batch_ix = ixes[begix : begix + batch_sz]\n",
" p = fwd_pass(Xs=Xs[batch_ix], mdl=mdl)\n",
" loss = F.cross_entropy(input=p, target=Ys[batch_ix])\n",
" losses.append(loss.item())\n",
" # The output of each layer is an intermediate computation\n",
" # the gradient is only needed for model params. We force\n",
" # pytorch to keep these grads.\n",
" retain_out_grad(mdl)\n",
" _zero_grad(mdl=mdl)\n",
" loss.backward()\n",
" _update_params(\n",
" mdl=mdl,\n",
" lr=_get_lr(lr=lr, it=total_iter),\n",
" wt_update_ratios=wt_update_ratios,\n",
" )\n",
" if sub_iter >= max_sub_iter:\n",
" break\n",
" if verbose:\n",
" itrs.set_description(f\"Loss: {loss.item():.2f}\")\n",
" return losses\n",
"\n",
"\n",
"def fwd_pass(Xs, mdl, act_fn=None, return_intermediates=False):\n",
" x = Xs\n",
" for layer in mdl[\"layers\"]:\n",
" x = layer(x)\n",
" return x\n",
"\n",
"\n",
"def retain_out_grad(mdl):\n",
" for layer in mdl[\"layers\"]:\n",
" layer.out.retain_grad()\n",
"\n",
"\n",
"def fwd_proba(Xs, mdl, act_fn=None):\n",
" return F.softmax(fwd_pass(Xs=Xs, mdl=mdl, act_fn=act_fn), dim=1)\n",
"\n",
"\n",
"def _get_lr(lr, it):\n",
" if isinstance(lr, (int, float, T.TensorType, T.Tensor)):\n",
" return lr\n",
" elif isinstance(lr, dict):\n",
" for (min_it, max_it), _lr in lr.items():\n",
" if min_it <= _lr < max_it:\n",
" return _lr\n",
" else:\n",
" raise ValueError(f\"Iteration {it} not in any range in {lr}\")\n",
" else:\n",
" raise NotImplementedError(\n",
" f\"Don't know how to handle learning {lr} of type {type(lr)}\"\n",
" )\n",
"\n",
"\n",
"def _zero_grad(mdl):\n",
" for param in mdl[\"params\"]:\n",
" param.grad = None\n",
"\n",
"\n",
"def _update_params(mdl, lr, wt_update_ratios):\n",
" lyr_num = 0\n",
" for param in mdl[\"params\"]:\n",
" param.data -= lr * param.grad\n",
" if param.ndim != 2 or param.shape[0] == 1:\n",
" # only pick linear layer, don't choose gradients for batchnorm\n",
" continue\n",
" # weight updates only for W matrices\n",
" lyr_num += 1\n",
" wname = f\"W_{lyr_num}\"\n",
" ratios = wt_update_ratios.get(wname, [])\n",
" ratios.append(\n",
" ((lr * param.grad).std() / param.data.std()).log10().item()\n",
" )\n",
" wt_update_ratios[wname] = ratios\n",
"\n",
"\n",
"def _loss(Xs, Ys, mdl):\n",
" logits = fwd_pass(Xs=Xs, mdl=mdl)\n",
" return F.cross_entropy(input=logits, target=Ys).item()\n",
"\n",
"\n",
"def _plot_losses(losses: list[int]):\n",
" plt.plot(losses)\n",
" plt.title(\"Loss vs iteration\")\n",
" plt.xlabel(\"Iter\")\n",
" plt.ylabel(\"Cross entropy or NLL loss\")\n",
" plt.grid()\n",
"\n",
"\n",
"@T.no_grad()\n",
"def generate_words(mdl, nwords, ctoix, ctx_win=CTX_WIN_SZ, generator=None):\n",
" st_X = _ctx_to_X(ctx_chars=DOT * CTX_WIN_SZ, ctoix=ctoix).repeat(nwords, 1)\n",
" lst_X = st_X\n",
" words = [[] for _ in range(nwords)]\n",
" for i in range(30):\n",
" # print(f\"{lst_X=}\")\n",
" new_X = T.multinomial(\n",
" input=F.softmax(fwd_pass(lst_X, mdl), dim=1),\n",
" num_samples=1,\n",
" replacement=True,\n",
" generator=generator,\n",
" )\n",
" # print(f\"{new_X=}\")\n",
" char_added = False\n",
" for w, ix in zip(words, new_X):\n",
" if w and w[-1] == DOT:\n",
" continue\n",
" char_added |= True\n",
" w.append(ixtoc[ix.item()])\n",
" lst_X[:, :-1] = lst_X[:, 1:]\n",
" lst_X[:, -1] = new_X.squeeze()\n",
" # print(f\"{lst_X=}\\n\\n\")\n",
" if not char_added:\n",
" break\n",
" return [\"\".join(w[:-1]) for w in words]\n",
"\n",
"\n",
"def num_params(mdl):\n",
" return sum(T.numel(param) for param in mdl[\"params\"])\n",
"\n",
"\n",
"def _ctx_to_X(ctx_chars, ctoix):\n",
" return T.tensor([ctoix[c] for c in ctx_chars]).unsqueeze(dim=0)"
]
},
{
"cell_type": "markdown",
"id": "fb91021b",
"metadata": {},
"source": [
"#### Train validation and test losses"
]
},
{
"cell_type": "code",
"execution_count": 17,
"id": "e36e774d",
"metadata": {},
"outputs": [],
"source": [
"@T.no_grad()\n",
"def _split_loss(Xtr, Ytr, Xval, Yval, Xtest, Ytest, mdl, split):\n",
" match split:\n",
" case \"train\":\n",
" spl, loss = \"Train\", _loss(Xtr, Ytr, mdl)\n",
" case \"val\":\n",
" spl, loss = \"Validation\", _loss(Xval, Yval, mdl)\n",
" case \"test\":\n",
" spl, loss = \"Test\", _loss(Xtest, Ytest, mdl)\n",
" case _:\n",
" raise NotImplementedError(\"Split should be train, val or test\")\n",
" print(f\"{spl} loss={loss:.4f}\")\n",
" return loss\n",
"\n",
"\n",
"split_loss = ft.partial(\n",
" _split_loss,\n",
" Xtr=Xtr,\n",
" Ytr=Ytr,\n",
" Xval=Xval,\n",
" Yval=Yval,\n",
" Xtest=Xtest,\n",
" Ytest=Ytest,\n",
")"
]
},
{
"cell_type": "markdown",
"id": "797704e9",
"metadata": {},
"source": [
"### Build model by stacking Layers"
]
},
{
"cell_type": "code",
"execution_count": 18,
"id": "19c04dd3",
"metadata": {},
"outputs": [],
"source": [
"EMBED_DIM = 5\n",
"HIDDEN_DIM = 50\n",
"NUM_HIDDEN = 1"
]
},
{
"cell_type": "code",
"execution_count": 19,
"id": "098fba0f",
"metadata": {},
"outputs": [],
"source": [
"def build_model(\n",
" nchrs=NCHARS,\n",
" ctx_win=CTX_WIN_SZ,\n",
" embed_dim=EMBED_DIM,\n",
" hidden_dim=HIDDEN_DIM,\n",
" num_hidden=NUM_HIDDEN,\n",
" lin_wt_gain=1.0,\n",
" init_wt_scale=None,\n",
" batchnorm=False,\n",
"):\n",
" assert hidden_dim > 0, \"at least one hidden dim is needed\"\n",
" lkwrgs = dict(wt_gain=lin_wt_gain, init_wt_scale=init_wt_scale)\n",
" if batchnorm:\n",
" activation_fn = lambda: [BatchNorm1D(size=hidden_dim), Tanh()]\n",
" else:\n",
" activation_fn = lambda: [Tanh()]\n",
" layers = [\n",
" Embedding(num_embed=nchrs, embed_dim=embed_dim),\n",
" Linear(fanin=embed_dim * ctx_win, fanout=hidden_dim, **lkwrgs),\n",
" ] + activation_fn()\n",
" for _ in range(num_hidden - 1):\n",
" layers.extend([Linear(fanin=hidden_dim, fanout=hidden_dim, **lkwrgs)])\n",
" layers.extend(activation_fn())\n",
" if batchnorm:\n",
" layers.append(Linear(fanin=hidden_dim, fanout=nchrs, wt_gain=1.0))\n",
" bn = BatchNorm1D(size=layers[-1].fanout)\n",
" # last layer we scale the gamma not the weights as they are standardised\n",
" bn.gamma *= 0.01\n",
" layers.append(bn)\n",
" else:\n",
" layers.append(Linear(fanin=hidden_dim, fanout=nchrs, wt_gain=0.1))\n",
" params = [p for lyr in layers for p in lyr.parameters()]\n",
" for p in params:\n",
" p.requires_grad = True\n",
" lyrs_str = \"\\n\\t\".join(str(l) for l in layers)\n",
" print(f\"Layers: \\n\\t{lyrs_str}\")\n",
" return dict(layers=layers, params=params)"
]
},
{
"cell_type": "markdown",
"id": "03e0ee0f",
"metadata": {},
"source": [
"### Test if it works and try to overfit"
]
},
{
"cell_type": "code",
"execution_count": 20,
"id": "34d5c073",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Layers: \n",
"\tEmbedding(num_embed=27, embed_dim=5)\n",
"\tLinear(fanin=15, fanout=50, bias=True, wt_gain=1.6666666666666667, init_wt_scale=0.43033148291193524)\n",
"\tBatchNorm1D(size=50, momentum=0.01, eps=1e-05)\n",
"\tTanh()\n",
"\tLinear(fanin=50, fanout=27, bias=True, wt_gain=1.0, init_wt_scale=0.1414213562373095)\n",
"\tBatchNorm1D(size=27, momentum=0.01, eps=1e-05)\n"
]
}
],
"source": [
"mdl = build_model(num_hidden=1, lin_wt_gain=5 / 3, batchnorm=True)\n",
"Xtr_sml, Ytr_sml = Xtr[:50], Ytr[:50]"
]
},
{
"cell_type": "code",
"execution_count": 21,
"id": "20fde998",
"metadata": {},
"outputs": [],
"source": [
"losses = []"
]
},
{
"cell_type": "code",
"execution_count": 22,
"id": "4cd6b15d",
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "10249d59e96c48b29b826fe79bdca94f",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/500 [00:00, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"We expect this to be large as we overfitted on a small batch.\n",
"Train loss=5.8337\n",
"Validation loss=5.8518\n"
]
},
{
"data": {
"text/plain": [
"5.851779460906982"
]
},
"execution_count": 22,
"metadata": {},
"output_type": "execute_result"
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAAsTAAALEwEAmpwYAABAIklEQVR4nO3dd5hU1fnA8e+7haV3XOlFEOwKiKIiixVEY0RjjyUa1Fij/ozGbmIkGk1iF2M3sTdURBBZRQWlSO/gooDS29J39/39ce/szsxOuTM7bXfez/PMszN3bnnPzOw995R7jqgqxhhjsldOugMwxhiTXpYRGGNMlrOMwBhjspxlBMYYk+UsIzDGmCxnGYExxmQ5ywiMSQARGSAiC9Mcw59F5D/pjMHUTmL3EZhMISIlwOWq+lm6Y6mpZKdFRIqAV1W1QzL2b7KLlQiMyTDisP9NkzL2YzMZT0QKRORfIrLKffxLRArc91qLyEcisklENojIRN9JVET+JCIrRWSriCwUkeND7PsIEflFRHL9lp0hIrPc5/1EZKqIbBGR1SLySJgYi0Rkhfv8FaAT8KGIlIrILe7yI0XkGzfWme5VvW/7YhG5X0S+BrYD3UTkUhGZ78a/TESucNdtBHwCtHP3Xyoi7UTkHhF51W+fvxKRue7xikVkP7/3SkTkZhGZJSKbReQNEakf3zdkajvLCExtcDtwJHAocAjQD7jDfe8mYAXQBigE/gyoiPQErgEOV9UmwMlASfCOVfVbYBtwnN/i84H/uc//DfxbVZsC+wBvRgtWVX8L/AicpqqNVfVBEWkPfAz8FWgJ3Ay8IyJt/Db9LTAcaAIsB9YApwJNgUuBf4pIb1XdBgwBVrn7b6yqq/xjEJF9gdeAG9zPZjROxlTPb7WzgcFAV+Bg4JJoaTN1k2UEpja4ALhPVdeo6lrgXpyTJsAeoC3QWVX3qOpEdRq+yoECYH8RyVfVElVdGmb/rwHnAYhIE+AUd5lv/91FpLWqlqrq5DjTcCEwWlVHq2qFqo4DprrH8nlRVeeqapmblo9Vdak6vgDGAgM8Hu8c4GNVHaeqe4B/AA2Ao/zWeVRVV6nqBuBDnIzWZCHLCExt0A7nCtlnubsM4CFgCTDWrT65FUBVl+BcDd8DrBGR10WkHaH9DxjmVjcNA6arqu94lwH7AgtEZIqInBpnGjoDv3GraTaJyCbgGJxMzOcn/w1EZIiITHarvDbhZBqtPR4v4DNT1Qp3/+391vnF7/l2oLHHfZs6xjICUxuswjmR+nRyl6GqW1X1JlXtBvwKuNHXFqCq/1PVY9xtFfh7qJ2r6jyck+YQAquFUNXFqnoesJe7/dtuHX00wd3xfgJeUdXmfo9Gqjoi1DZupvQOzpV8oao2x6nekTD7DxbwmYmIAB2BlR5iN1nGMgKTafJFpL7fIw+nmuYOEWkjIq2Bu4BXAUTkVBHp7p7oNuNUCVWISE8ROc49oe4EdgAVEY77P+B64FjgLd9CEblQRNq4V9Sb3MWR9uOzGujm9/pV4DQROVlEct20FYlIuO6f9XCqttYCZSIyBDgpaP+tRKRZmO3fBIaKyPEiko/TlrIL+MZD7CbLWEZgMs1onJO273EPTgPrVGAWMBuY7i4D6AF8BpQCk4AnVXUCzkl0BLAOpwpkL+C2CMd9DRgIfK6q6/yWDwbmikgpTsPxuaq6w0M6HsDJvDaJyM2q+hNwOk5j9lqcEsL/EeZ/UFW3AtfhnNA34pRURvm9v8CNeZl7jHZB2y/EaZd4zP0MTsNpvN7tIXaTZeyGMmOMyXJWIjDGmCxnGYExxmQ5ywiMMSbLWUZgjDFZLi/dAcSqdevW2qVLl7i23bZtG40aeekCXndYmrODpTk71CTN06ZNW6eqbUK9V+sygi5dujB16tS4ti0uLqaoqCixAWU4S3N2sDRnh5qkWUSWh3vPqoaMMSbLWUZgjDFZzjICY4zJcpYRGGNMlrOMwBhjspxlBMYYk+UsIzDGmCyXNRnB7rIKJvy4h517ytMdijHGZJSsyQhe+PoHXpq3mzem/BR9ZWOMySJZkxH87piuANz74dw0R2KMMZklazKC/FwnqRUK60t3pTkaY4zJHFmTEfj7asm66CsZY0yWyKqMYEjXfACuf31GegMxxpgMklUZwa+751c+X7R6axojMcaYzJFVGUFBrlQ+X71lZxojMcaYzJFVGYG/R8cvTncIxhiTEbI2I5hSsjHdIRhjTEbIuoxg4i2DKp/vKa9IYyTGGJMZsi4j6NiyYeXzUTNWpTESY4zJDFmXEfi76a2Z6Q7BGGPSLqszAoAdu20QOmNMdsvKjGDEsIMqn+931xiWrS1NYzTGGJNeWZkR1MsLTPbStdvSFIkxxqRf0jICEakvIt+JyEwRmSsi94ZYp0BE3hCRJSLyrYh0SVY8/vZv1zTgdV6OhFnTGGPqvmSWCHYBx6nqIcChwGAROTJoncuAjaraHfgn8PckxlOp196BGUGuZQTGmCyWtIxAHb7K93z3oUGrnQ685D5/GzheRFJ+VrYSgTEmm4lq8Lk5gTsXyQWmAd2BJ1T1T0HvzwEGq+oK9/VS4AhVXRe03nBgOEBhYWGf119/Pa54SktLady4MQCXjKlqF7itX316tsyNa5+Zzj/N2cLSnB0szbEZNGjQNFXtG/JNVU36A2gOTAAODFo+B+jg93op0DrSvvr06aPxmjBhQuXznXvKtPOfPtLOf/pIp5asj3ufmc4/zdnC0pwdLM2xAaZqmPNqSnoNqeomNyMYHPTWSqAjgIjkAc2A9amIqSCvqgRw5lOTOPOpb1JxWGOMyTjJ7DXURkSau88bACcCC4JWGwVc7D4/C/jczblSbtpyG4TOGJOd8pK477bAS247QQ7wpqp+JCL34RRRRgHPAa+IyBJgA3BuEuOJqqJCybGGY2NMlklaRqCqs4DDQiy/y+/5TuA3yYohVk8WL+Ga43qkOwxjjEmprLyz2OdPg3sFvLbqIWNMNsrqjKB3p+YBr2ev3JyeQIwxJo2yOiMIvndtXenuNEVijDHpk9UZwbbdZekOwRhj0i6rM4L9gsYcMsaYbJTVGcHezeqnOwRjjEm7rM4IQvG/n23JmlKm/2g9iYwxdZtlBEEe+KTq5ucTHvmCYU/a0BPGmLrNMoIgI79clu4QjDEmpSwjMMaYLGcZgTHGZDnLCIwxJstZRmCMMVnOMgJjjMlyUTMCETlaRBq5zy8UkUdEpHPyQ0uNi/tXT8oXi9amIRJjjEkPLyWCp4DtInIIcBPOvMIvJzWqFBo+cJ9qyy5+/rs0RGKMMenhJSMoc6ePPB14XFWfAJokN6zUad+8Ab32rjPJMcaYmHmZoWyriNwGXAgcKyI5QH5yw0qtpvXrVHKMMSYmXkoE5wC7gMtU9RegA/BQUqNKsQq/8YWMMSbbeCoRAP9W1XIR2RfoBbyW3LBSq9wyAmNMFvNSIvgSKBCR9sBY4LfAi8kMKtXaNW+Q7hCMMSZtvGQEoqrbgWHAk6r6G+DAqBuJdBSRCSIyT0Tmisj1IdYpEpHNIjLDfdwVexJqbsSwg9JxWGOMyQieMgIR6Q9cAHwcw3ZlwE2quj9wJHC1iOwfYr2Jqnqo+7jPU9QJ1qR+PqOvG5COQxtjTNp5OaHfANwGvKeqc0WkGzAh2kaq+rOqTnefbwXmA+1rEGtS5QR9EmrtBsaYLCFeT3gi0hhAVUtjPohIF5y2hgNVdYvf8iLgHWAFsAq4WVXnhth+ODAcoLCwsM/rr78eawgAlJaW0rhx45DvrdhawR1f76h8/fzJDfndp9sBeHFwo7iOlwkipbmusjRnB0tzbAYNGjRNVfuGei9qryEROQjnTuKWzktZC1wU6oQdZvvGOCf7G/wzAdd0oLOqlorIKcD7QI/gfajqSGAkQN++fbWoqMjLoaspLi4m3LaLV2+Fr7+sfP3QrKqPJt7jZYJIaa6rLM3ZwdKcOF6qhp4BblTVzqraCWeYiWe97FxE8nEygf+q6rvB76vqFl8JQ1VHA/ki0tpz9AkkIgGv5/8cnGcZY0zd5CUjaKSqlW0CqloMRK0rEefM+hwwX1UfCbPO3u56iEg/N571HmJKuByJvo4xxtRFXm4oWyYidwKvuK8vBLxM7Hs0zj0Hs0Vkhrvsz0AnAFV9GjgLuEpEyoAdwLmaplbaHLGcwBiTnbxkBL8D7gV8VTsT3WURqepXQMSzq6o+DjzuIYaks3zAGJOtomYEqroRuC4FsaSVRM6zjDGmzgqbEYjIh0DYahpV/VVSIkoTKxEYY7JVpBLBP1IWRQbIsdZiY0yWCpsRqOoXqQwk3SwfMMZkK5u83tWmcQEN8nPTHYYxxqScZQSuvNwc5v9lMCfuX5juUIwxJqXiyghEpM62H9hYc8aYbBNvieDshEaRQc7r17HastvenRXweuWmHWzbVZaqkIwxJqnizQjqbNPq8ftVrxp67bufAl4fPeJzzh05OVUhGWNMUkW6j6BluLeowxmBV7NXbk53CMYYkxCR7iOYhnNDWaiT/p7khJMZ7hi6H3/9eH66wzDGmJSIdB9B13DvuRPZ11mXD+hmGYExJmvE20YwKaFRGGOMSRtrLPbomS+WpjsEY4xJingzgqzrbf/AJwvSHYIxxiRFpF5DjxH6hC9A82QFZIwxJrUi9RqaGud7ddaGbbtp2aheusMwxpiEitRr6KVUBlIbzPhpI8f1srGIjDF1S6SqoRcI3xagqnpZckLKXMvWbuO4XumOwhhjEitS1dBHIZZ1BP4IZOV4zX/9eD6XD+iW7jCMMSahIlUNveN7LiLdgD8DxwIjgOeSH1rtcO+Hc5m9YjNvX3VUukMxxpi4ROw+KiK9RORV4EPgK2B/VX1KVXdH27GIdBSRCSIyT0Tmisj1IdYREXlURJaIyCwR6R13StLkha9LmLp8Y7rDMMaYuEVqI3gL6AM8jFMdVA40FXeWd1XdEGXfZcBNqjpdRJoA00RknKrO81tnCNDDfRwBPOX+NcYYkyKR2ggOx2ksvhm4yV3mu6NYgYiV5ar6M/Cz+3yriMwH2gP+GcHpwMuqqsBkEWkuIm3dbTPSxm1RC0PGGFOrRGoj6JKog4hIF+Aw4Nugt9oD/oP9r3CXZWxGcNhfxqU7BGOMSahIJYKEEJHGwDvADaq6Jc59DAeGAxQWFlJcXBxXLKWlpXFvG4r/vhK530RKdJprA0tzdrA0J05SMwIRycfJBP6rqu+GWGUlTpdUnw7usgCqOhIYCdC3b18tKiqKK57i4mI8bzvm46irFBUVVa4Xb0zJFlOa6whLc3awNCdOvIPORSVOq/JzwHxVfSTMaqOAi9zeQ0cCmzOlfaBFw/x0h2CMMSkRrftorojEO+zm0cBvgeNEZIb7OEVErhSRK911RgPLgCXAs8Af4jxWwj1xfnw9WScsWMPOPeUJjsYYY5InYtWQqpaLyEIR6aSqP8ayY1X9iijzFri9ha6OZb+pUq6xj7Q9b9UWLn1xCmf37cCDZx2ShKiMMSbxvLQRtADmish3wDbfQlX9VdKiygAVccy4sGWnM5XzjJ82sXnHHpo1sOolY0zm85IR3Jn0KDJQRYw5wdvTVtCxRQMAFq0u5ZB7x1IyYmgyQjPGmISK2lisql8AC4Am7mO+u6xOK4sxI7j5rZn8smVnkqIxxpjkiZoRiMjZwHfAb4CzgW9F5KxkB5ZuvTs1j7rOS9+UBLwuj6c+yRhj0sxL99HbgcNV9WJVvQjoRxZUF7VqXMDk246PuM7do+amKBpjjEkeLxlBjqqu8Xu93uN2WeeVycvTHYIxxsTMywl9jIh8KiKXiMglwMc4/f/rvMKmBdx04r6e1//+x03JC8YYY5LES2Px/wHPAAe7j5Gq+qdkB5YJRIRrj++R7jCMMSapPI015I4TFGqsIGOMMbWc1fUbY0yWs4zAGGOynJf7CE4TEcswjDGmjvJygj8HWCwiD4pIr2QHlImKby7irSv7pzsMY4xJCi+9hi7EmWZyKfCiiEwSkeHuhPRZoUvrRhQ2qZ/uMIwxJik8Vfm4U0y+DbwOtAXOAKaLyLVJjC2jSMQBtY0xpvby0kbwKxF5DygG8oF+qjoEOAS4KbnhZY6cnMzMCeau2syj4xenOwxjTC3mpURwJvBPVT1IVR/yDTehqtuBy5IaXQaJJx946NMFHPvghMQH4+f0x7/mkXGLknqMVLv/43kMe/LrdIdhTNaIekOZql4sInuLyK8ABaao6i/ue+OTHWCmyImjbuiJCUujrrNtVxnlqjStH98kNrEOl10bPDvxh3SHYExW8VI1dBnOMNTDgLOAySLyu2QHlmmS1UZw+P2fcfA9Y2u8H41jak1jjAFvQ0zcAhymqusBRKQV8A3wfDIDyzTxlAh8tuzcE/aKf/vu+Ca6L1m3jfr5uXHHZIwxPl4ygvXAVr/XW91lWaUmGcFj4xdz+9D9ExgNFP2jOOC1qvVsMsbEx0tj8RKcWcnuEZG7gcnAIhG5UURuDLeRiDwvImtEZE6Y94tEZLOIzHAfd8WXhNSoSaeh8orExWGMMYnmpUSw1H34fOD+jXZD2YvA48DLEdaZqKqneogh7aQGl9tK8uvvrYXAGBMvL72G7gUQkcbu61IvO1bVL0WkS42iyyAZehuBMcbUmJdeQweKyPfAXGCuiEwTkQMSdPz+IjJTRD5J4D6ToiZtBKno0GO9howx8fJSNTQSuFFVJ4BTtw88CxxVw2NPBzqraqmInAK8D4ScDkxEhgPDAQoLCykuLo7rgKWlpXFvu6s8/hPt2FnLKWq6NuI68cbl88UXX5AbothSkzSnWzq+59rK0pwdkpVmLxlBI18mAKCqxSLSqKYHdscv8j0fLSJPikhrVV0XYt2ROBkSffv21aKioriOWVxcTLzb7txTDuPGxLXtqlKlqKiIN6f+RFHPNuzlP4DdmI8BYo/L3c7n2IEDyc+tXsCrSZrTJt7PxFUr01xDlubskKw0e+k1tExE7hSRLu7jDmBZTQ/s3q0s7vN+bix1tlvqmq07ueXtWVz24tR0h2KMMQG8lAh+B9yLM2exAhPdZRGJyGtAEdBaRFYAd+MMWoeqPo1zl/JVIlIG7ADO1Qyu6PZVuzTIz2XHnthvAit3h4JYu3VXQuPyydxPzhiT6SJmBCKSC7yrqoNi3bGqnhfl/cdxupfWCvm5ObxzVX/KK+DsZybFvP33P24CUtOV1BhjYhExI1DVchGpEJFmqro5VUFlqj6dW7Lwl63RVwzhD/+dHvB6+o8bKV6wJhFhAZbBGGPi56VqqBSYLSLjgG2+hap6XdKiymA1HcbBV4Uz7Mlvah6MMcYkgJeM4F334S9rLz99+UCT+nnk5+awYdvumLYP98F9MGMlHVs2pHenFnHFZW0Exph4eek11FxVX/J/APGdreoAX4lgryYFfH7TwJi3D3fCvv71GVZKMMakhZeM4OIQyy5JcBy1iJMTOKN9Jn7cCVXlgxkr2V1mI9UZY1IjbNWQiJwHnA90FZFRfm81ATYkO7BM5Tv3K/G1F+wqK6ciwqxi4+ev4frXZ7CwaCu3DO4VX5DGGBODSG0E3wA/A62Bh/2WbwVmJTOoTOY796sq8ZQHtu4s48R/fhH2/W27ywD4ccP2OPZujDGxC5sRqOpyYDnQP3XhZD5fdZASf9XQ0rXbwr7nm3Vs557YqoassdgYEy8vo48OE5HF7iQyW0Rkq4hsibZdXVVVIiCuEkE0BXnOV7KrrJwPZqzkw5mrknCU9Bg79xeemLAk3WEYY4J46T76IHCaqs5PdjC1QVUbgSZlakjfUBaqTk8igNMOaRd1u9pwQ9nwV6YBcPWg7mmOxBjjz0uvodWWCVRpUM+puunepnFSqmPELWds3bkn8Ts3xpgQvJQIporIGzjzBVSOmKaqwTeZZYW9mtTn1cuO4JCOzZKyf18po2R9VWNxWXkFE5esY1DPvcJuZ20Exph4eSkRNAW2AycBp7mPWjHPcLIc06M1Ternk5eb+Loh3x4bF1Tl0Y+OX8ylL0xh4mJncpvN22tXaWHrzj3sKbf7IozJVF7mLL40FYHURgV5ubz3h6M4I5F3BIfIWx793GlgXV/qDGfR72+fVVsnkwsEB90zlkE926Q7DGNMGF56De0rIuNFZI77+mB3choDHBbn2EDh+OZGXrlpR7X3fNVGuxJ01/Hb01bw/vcrE7KvaCYsjDxVpzEmfbxUDT0L3AbsAVDVWcC5yQwqm8Vb2RTPnD43vzWTG96YEecRjTF1hZeMoKGqfhe0rCwZwZjkjF8UqyVrtkYcBsMYU7d4yQjWicg+uNXQInIWztATJgki5QPzVm3hH58uDPleok7bc1Zu5oRHvmTkxBpPS22MqSW8dB+9GhgJ9BKRlcAPwAVJjaqWOah9M2avTMwEble4N12F8syXyT85r9jodFudvnxj0o+VTjv3lJOfm1N5A58x2SxqiUBVl6nqCUAboJeqHuOOQ2RcH157DMMOa5+QfcU60Y1PpCaCN6f+xGH3jfVY3VM1llIsZvy0idkras9spr3uHMPvXpwS9v1pyzdw/8fzUhiRMenjpWoIAFXdpqrxTdibBTK5Rv2O9+awcfsednvoy++7QI617fnXT3zNaY9/FUd06fPFovA9mc58ahLPTvwhhdEYkz6eMwITWTy9dhIbQIT3Yqj9qBxdNUx6Zvy0KWVp/WjWKr5esi4lx8ok3yxdZ4Pz4XRauP/jeen/38oCScsIROR5EVnju/8gxPsiIo+KyBIRmSUivZMVSypkwk/1rKe+4b/fVq+18x8x9YHR8/nDf8O3Q1SuG+K9z+at5tdPfM1r3/1U41iDHfvgBB4bvzhg2TX/+54L/vNtwo/lLxNPMuc/+y0PhekUkAzL1pZy+uNfsXlHZt2xfvHzU3h24g+s2rwz3aHUeV5uKPuNiDRxn98hIu96PGm/CAyO8P4QoIf7GA485WGfGSvd5xNFmbp8I7e/Vz3f9R8x9ZkvlzF69i+89/2KkPvJcX8RoU6QJeudeRSWrCkF4M7353Dvh3MTEL0zEc/D4xYlZF+x6HXnmJQfM5wTH/kibK+wZJi9YjP/+mwRxz38BTNXbObzBatTdmwvKtzfoDXnJ5+XXkN3qupbInIMcALwEM5J+4hIG6nqlyLSJcIqpwMvq3PGmSwizUWkrarWyq6pFenOCTy45n/fVz7/7+QfQ67jG/00UruyL2N5ZbJT+vjjifuGXG/S0vXs17ZJHJGmTqLu0q6p2Ss2s3hNKYvXpK5KKLhNpxb8hJOqdFdZwBhf2cRLqsvdv0OBkar6sYj8NQHHbg/41zGscJdVywhEZDhOqYHCwkKKi4vjOmBpaWnc20azZk16i68PvFE1/aV/GktLS6kod87cny9YU7l8ql/3UN/6ZRXKQ1OcdKzfsKHaZ7X4B6fqYMWKnygurtrXuAkTK59PmDABEaG8Qrls7HY6N61e6Px0/AQKwgzYF+r7Cfedbd6lvDxvF43yhSFd8mnbOKcyzbF8z5HW/f1Tn3Jer3qVQ3/4bNxZwcZdSrdmuZ6PE8klY6rPWhdLGrykefLPZbRtJHRuGjrm+fPnU7wlc9omdu1yBjueNGkSrRpU/x0l8v/5+zVl/Hv6Lu44oj7dWyTmO61Q5bUFuzmhUz6FjRJTC5+sc5iXjGCliDwDnAj8XUQKSHEjs6qOxLmXgb59+2pRUVFc+ykuLibebaN5a+V0+CV9hZk3FlZ1O/VP49jxE9ijkec/9q3/9BdLWbhxAQAtWrSgqCiw0Lc4ZxksnE/HDh0pKtofxnwMwBFHHglfTABg4MAicnLEGW107Ccs31L9irvlPocwasYq1pXu4qkL+zgL3X0VFRVVPg+VHn93vD+baaudks2PO+pR/H/Oep6+Z79j+K975/tz6NCiAeB8DuOWl3HzGUfRc+/Aks2+t3/C7vIKSkYMjXwcr4LSHBxXNF7SfMmtzjEqYw46Zk6LDjy3dAvPXtS3csrUVFq6tpRb35nFi5f2o1FBHvW+GQ87d9K/f3/aNW9Qbf1E/j8Xj5oLlJDTphtFx3RNyD7nrdrCuE8nsnJ3Q0ZfPyCufXw2bzWdWjVk30Ln95esc5iXE/rZwKfAyaq6CWgJ/F8Cjr0S6Oj3uoO7rFbKxBnCVJXh47Z7LvJv311e+Xzi4nWVN5cBvDKphPtHO/MTicC3y9ZXvvej39wJXg6l6lQrfTLnF2+B4fQgee270NVZXo/rxSuTl/PAJwuiruelK244a7fuonjhmugrxqn3X8bx8qSSmLd75stlTFy8juk/Ju9mwp17ylmzJXTp+e+fLGBKyUYmLk5fT7FEjvDiOyfU5Ld5+ctTOemfXyYmoAi8ZARtgY9VdbGIFAG/AYLHHorHKOAit/fQkcDm2to+AJlXv3r5S1PoeYe3htAxc5yPPfgm20fGOo23a7bs5M4PqhqFn534A+eMnFz5+ny/nj27yyr412eL2L6rKlNJhCH/nsht784O+34yP/9EZPKPjFtU2Rh7/rOTueSFKZQnaTynDdt2c9cHiWnET7Thr0yj39/GByz7avE63pxavSdaKofdysTeY6tCjECcLF6qht4B+opId5zqmQ+A/wGnRNpIRF4DioDWIrICuBvIB1DVp4HR7j6W4Ex8U6vnPci0xuLP5nu/4nx72koGH9i2sqHYp0KVR8Yt4ukvlnrf1/QV/OuzxWyKMHlOPP90e8ojb5OJJbKde8r5ZM7P/PrQ9jzqdo0tGTGUpWtL0xxZ+nwZ4ia+C59zLiRO2r8wYHk6/qUSmffUNP6jRnyemEA88JIRVKhqmYgMAx5T1cdE5PtoG6nqeVHeV5xxjOqEwzq14NO5mdX9LlbBV2AzV2zm/RmrYtrH9l3OwLQ794QvESTq/9v/Hy2pJYI49/230fN5edJy9mpSP8x+ldrQOfLd6St4edJy3r/66IDl3/+4kQb1cum1d1NP+5mwwOvFSeZl6jWR+d+wt6qhPSJyHnAR8JG7LD95IdVOwwd0Y9/CxukOA4g8dEJozj9ecNVQPFUXZe42mTCcdqJ4zQhK1m3jilemsnNPOarKy5Oc7rVbdwaWjirv3k5olNUd/3AxF/xncvQVo7jxzZnM+GlTteVnPPkNg/81sfoGYUxdviHi++n8yaQj6znuH8Wc+VQCZzesAS8ZwaVAf+B+Vf1BRLoCryQ3rNonJ0fo0qpRusMA4OLn42vCCT55xzMwpy/ziLRtIq7eP571M//9tqrxuCb7jHYTl9dqpzs/mMOnc1fz7Q8bGDuvqnT43FehxyxKRinGv9pt6dptfL1kfYS1Uyu46tHzdinIIHwfWyovYJat28a0DBnl18voo/OAm4HZInIgsEJV/570yGqhTGsniMXqLTurDWsQzz9FVYkg/Do1qc/39VB6/uvqJ9cxc36O6wr48QSP67Njdzkb/UaRnVIS+p89ke0aFaqUV2hCMpctO8rYVVa9ai9c2048PZRioQqfL1jN82Ey1Hh8OHMVXW79mO27A+fY8v1u127dxVdRei9NLdkQ0HsunNpQOPYyxEQRsBh4AngSWCQixyY3rNopWb1AUuHPIXrkxPP7LXO7VS5dU/0GqUo1+JjGznO6nAbHtnLTDq58dbqnK+A9MXb9jPXkeuWr08ImcXeIO5lDNaDGavi47Rz3cHGN9wNO/OeNrJ6hhvscwpV4gvmfEHfuKef971d67jjwuxenct9HiRsW/J/ucCarNjldWYMz5bOfmVTZiB3OWU9PCug9F6yqlFGDQFPES2Pxw8BJqroQnMnsgdeAPskMrDaK0rElY4XtYRTHD9hXIpgU4Urp/KCB5GLpRVTmIbMNt79PZv/Mjj3lzAxR3/3j+u10atXQcxyh+JegwiXJfw4E3zoXeajKU1VK1m+na+tGlO4qI0egYb2qf9+yCli+fnvCyhjTf9zEZS9O4blLDq9cVqFKTogfhdevz3/Lv49ZwAtfl9CiUb3o2yXjRFq5T7evvwYu/mFdhAuZOshLG0G+LxMAUNVFWGNxSId2aBZyeSsPP/a6ItZS0bZdZQEnkmjb79xTzsNjFwbc/BYs3B6u+u90bnxzJu9Or37fYqS5FCKd6Bb+spW1W3eFiCH0Rl/FMaz21JIN/PGNGQz6RzFzVm7mwLs/5dD7xgGwvjTw2KEywTVb4xv+ZHxQL5/yMB9EpCqu8grliQlLKN1VFnBGX+3eVObfkJ7KcZ/8R+SNJJ6uzis37ci4kVyj8ZIRTBOR/4hIkft4Fpia7MBqo+tP2JcxNwyg+16BvYfevuqoNEVUMzWpGvJqd1lFQNvKyCjTcf7rs8U89vkS5v28JXwMFfDo+MVs21UW8v2tIZbH+4978r++5NgHneE1/D+vcIP6+a/n5RxTUaGc9fSkym68P21w2kh2l1Uwedl6+vz1M8ZEuUN76KOJmTAoXLzBy9ds3VnZfXjMnF946NOFPDB6fsDnE6rhuHjh2ojHiVWXWz/m9vdC34ToteeWfyzrS3dR4qGkcPSIzxn8ry8T2gZUXqFMi9Lrqia8ZARXAvOA69zHPOCqpEVUi+XmSMg+1W2aFKQhmpqLp7H4zamhh7cOf4zAf8YfN1T/R/tgRmwjj3yzqoxHxi3ikQQNa60oY+f+wtlPT6J0VxnL1wfGuCPEPRORMiqfLTv3MHZu+JP4i1//QLc/jw6KpcqsFZsAp8QQ6n2ftVt3VZvrIR5eO0P0u388l7zwHX3/Oo6r/zcdcEp+oX5OXnY52UODbGWMFco6v1KSf88yf8GZcWUYQUH6h3fUiM8p+kexpzh+9ptDId7eUv4e+3wxZz41icUbE3vHvk/ENgIRyQVmqmov4JGkRFDHvX1l/1o7tG08P99QJ8XIx5CAE0yoqqHrX58R0z53lsUXS6RG5CtfnUaFwtlPT/J0kvfi2te+57sfwl/l3fNh9cbRaCfO4EzKJxFzPYSrtQsV0+Rl3q5eQ+1y2vKNDDmobeXrf3/mPRN7eNxCnpiwlCm3nxBxPd/5fvvuMr73G1tpWskGfntk58rXFarkuv8J0aqubnxjBt397iX6OYET6iz8xZkleOOu5DRERiwRqGo5sFBEOiXl6HVUy4ZVbQJ9u7SMuG6Lhlne3CLw5aKqevNEdLyasdbJCZasjm0oh1CNyOCc6Hylo0iZgNcClK/BO9xJOxL/6oZQV5qjZ3sfyM+LcX73Q9S0V5zXK+P/fPVDQAYZPAR4JOPdjg/+pYIut34ctivoGU9+wxlPfsOclZsBqt1JHy7jnR/id/Du9yt5cExVF+wrXnFmAkxke0GyOiB5qRpqAcwVkfEiMsr3SFI8dcITF1SfwO2Vy/px4v6F9CwMHM44k+/ATUknKIXfv1zV5PT2tNiqlkJZsMG5cvuuZEPEoS68UqLfXPfi1z9U1nF7FU+VQcCwGlS/ZyPRt7KM/LJqnClfw2lFUIYQT4OqL/biMMNO+N/kF8+/SHA11l8+msfcVZt5/3unmjH4s18dZkRURdm8Yw/fBDXyD/l31R3Vr05eHnZE1Xj8tGG707ieQp5mKEt6FHVMqDaBAT3aMKBHG8C5QvGJ5+7dVPFNSZlMw19Jbr+Dez+cxwPDDvK0brjB9VTVPXGEP+GFqsZJBv8T3N9GO8Nlf7O0qg491I12NeF/Pt28Yw+CcMh9Y7nv9AOq1nH/nvrYRFo0DN1DrkIDT+i+ksu730dv/4l0sTRv1RZOedQ5KX9+08DK6r3gvGnh6q2Vjea/Pqx9tcwlUkP48Jen8m2EKrw73p/DHe+HnJodiP1/fMCDE9ivrbfxmxIlbIlARLqLyNGq+oX/A2fGsppfthlXBucEKRDpHywRIs1hECziiK1J+JrKKhLTXXLuqqpqimR2W7x71NzK/T85wb+k4Pyds3JL2LkERs1cFfNH6Cs1RDqR+nf7Pe7hL1i6dltATN6PFWa5wqLVW2PbWRAvVVs/rNsW0MstVNVTMkWqGvoXECqaze57JgEyuGbIuJI1Rui60t3RVwqSiKqfnXvKWeZxKGz/KU237NjD5S87N8T5l0wU5efNyRs7P9KJNFy7RbQeTsGljHCr73fXmGrrRuuuW/1g0VcZ9I9iLnzu28q2ilSLVDVUqKrVOuGq6uwok9Ibjzq3ahjxxiiTGPvfNYaj9mkd9/aqmTPJfSL6pt/05kw+nh37HFAVCovcBnj/8+/qLbvo/0D0sfNjveipSbfLqBlBtSXh1w8ukVz56rS4Yorm+x83cepjoe/5SPYwZpFKBM0jvFd9AlET4KNrj2H8TQNDvndkN6cn0QPDDsryiqHU2L67nM/mxz9XRKgB2NIlEbVJ8WQCQMBQ1PE0EMfaMSLW7r/+onVwCg7Fv3Tm34bnrh13HEBlHvP1knUx33CZKpFKBFNF5Peq+qz/QhG5HEhOlliHHNg+9HATkHnTWprINmyLvQonWRLdGByvZI+0u2H77sr2iAW/BNbRr96ykzVbqg/r4S/aTWix5EmJqL71ZS7XHtedm07qGfDeBA/zVyeqPSmcSBnBDcB7InIBVSf+vkA94IykRpUlBIm7oH9Au6YBjYQmeTJpVNlM+c43RpiKNJxYTqiReqwd++AEdpVV8PDA8BUTwUOq+/vdi1NYsdF7m0ZN8wH/X0+oaUovfWFKtWXBYpl6Nh5hq4ZUdbWqHgXcC5S4j3tVtb+qJvaulSwT6bSSn+vtZ3ftcT0SE4yJKtY7m01oP6xNzIievvaam76Ir4H68wVrIs6pHaymJQL/kUxrWpBK2w1lqjpBVR9zH6mbTbkOu/u0/Tm8SwsO69Q84Icx5oYBnHpwO0/7yOT7D4wJ5a0E3CyYjvaasgSOL+/7f7/1nVlxbb9kU3LS7+XOYpNgB7RrxltXHkX9/Fx85YNrj+seMGDdrw/1liEYk00OuXdsyo+5PoFtRIry6uTlvD7lp7i2n7GmFmYEIjJYRBaKyBIRuTXE+5eIyFoRmeE+Lk9mPJnoN307AvCHou4Bywf0aEPJiKHs3bR+yO0ypTujMam0c0/t/t1v2LY74l3I0eQk6YydtGEx3ZFLnwBOxLkTeYqIjHLnQPb3hqpek6w4Mt3/ndST64/v4ZYOqgtXBdSkfu0c0dSYbBZu/mqvcpN0B2oySwT9gCWqukxVdwOvA6cn8Xi1Uk6OBGQCwV+zf9/r3/TpwNCDneF5gye/McbUfclqG0zmZWV7wL8ibAVwRIj1zhSRY4FFwB9VtVrlmYgMB4YDFBYWUlxcHFdApaWlcW+bKr+sdvpHL1gwn+KtS9i9q2pUw9Wrf+Hi/etxbLMGLJkZfZ5bY0zdIlqelHNYuusXPgReU9VdInIF8BJwXPBKqjoSGAnQt29fLSoqiutgxcXFxLttqoxaPQNWraRnr/0o6tOBhlMmwA5nesJ2bdty/HEHV608xrlJZd59J7P/XZ8mLIam9fPYsjO1w+AaY6LLz81NyjksmVVDK4GOfq87uMsqqep6VfXdIvgfoE8S46kVOrdqBEDrxs5wvv4DboWrHmxYr3p+XpNeR61r6dSaxtR1uUk6YyczI5gC9BCRriJSDzgXCJjQRkTa+r38FTA/ifHUClcP2ocXLjmcop57AcEnf28VhO2bN6BZg/hmPht/00Ab/8iYDOXxftOYJS0jUNUy4BrgU5wT/JuqOldE7hORX7mrXScic0VkJnAdcEmy4qkt8nJzGNRrr8rXrRtVXZ3v387bZBXPXtQ37uPv06ZxRs+aZkw2S1avoaS2EajqaGB00LK7/J7fBtyWzBhquycu6M2YOT9zeNeW1aa57FnYhIUhJs1o26x+aqaZNMakVG3sNWQSoE2TAn7bv0vI9967+ii27ap+p6H/yJAFubCrHC45qgs3n9yTA++O3qhs5QFjMlNtbCMwSdawXl7I+ZGVqjFNmtRzTut5OULjgjxKRgyNul+rGTImM9W6NgKTWo+cfUjl8wb5uZx/RCfycoTeezk3q8VSVZSXrPvYjTE1UpCknMD+4+uIYb07sOT+IUy5/QQaFeSxX9umLPnbKbRq4HzFoSYS+fymgTx45sHVlvt3H23VqF7ygjbGxCRZbQSWEdQhebk51aqK+hTmkp8rnHt4p8plr/3+SD65fgDd2jTm7MM7Bu+G/Bz/exesnsiYTGGNxSYurRvksPj+UwKW9d+nVcRt/E/+Nu+BMZmjY5PkXLtbicBUOsidZ9n/5J+ToBLBHUP3C3h9SMfmCdlvXVIvWV1CTJ0xqGNyrt3tl2cqPXexcyOaBGQEidl3XtCO9m5qw1gEO6p75JKaqRtaN47/t5+sqlrLCExlg3BBntPDKHB8o6rnLRrGN2wFQG5uDo+ffxgAF/XvTNtm4SceT4WapCVZknXXqMkso645Ot0hVGMZgeHLWwbxzlVH0cw9Oeb4Xb0fu28bAP40uFeN7lbOyxFOPbgdJSOGct/pB3LrkF4xbX9ev07RV4rBUxdm3viG1jBfM0d2a5nuEDxpWC/0JFTpZBmBoVFBHn06t6h87TsdPXreYXRo4Vy5N8jPqRwZ1ScvR/j8poGejnFoUJtA/fxcptx+As9fEnpcpEZB/yz186P/VFs0zOfl3/XzFE/Hlg09rZdK1kRQM/m15AOslxdbnPu19TbGWE3Ujk/OpJSvakhVuXxAV24b0osLjuzMC5cczn/8BrRb8rdT6NYm8kxpA3q0ZuFfB4f8MbdpUsBxvQpDblcRVPzwUiL45tbjK0sw0bRvHrlqKrhxO5REzxKXW4e6aKVjBr1EdWxItli/51SkyjICU43vh1qhSkFeLlcM3If83BxaNqrHCftXP3HfeOK+XDOoOz88cEq195o3rFfZ9hAL/xvgZt59EvsGDbgXSoMwRe5+XWOvMmhcEL13xtMJrl6qS1VDoW5grIkrBnaLuk5tGWgx1gwrFTf6W0ZgqvH9TisqvK1/3fE9uPnkntVOZId0bM79Zxzo+bidWzWkk1tlM6x3B0pGDKVkxNDKuRX+8ZtDeOHSw0Nuu0+bqmqr/VoG/qwvOMIpTeSHuT3fVy12y+CelcuO8xsKPJRXLzuCRJ964mksDu6NFUq0tNSEf8kp4GbGBJ+VD2rfjAfPqn4XvL/uUUqnqXRuiBs1fWLOCFJwgWAZganG98OryVVdj70a88HVR9O0fvTeOX8o2oenL+zDZzcOZO+m9QE4PcQMa2f16cCgnqFPauNvKqp8fvEBgd3zTjmoLVcO3Idnfhv6Cv7FSw/nk+sH8Iei7nxz63EsuX8IBfnhSzGTbzueY3q0pjTEyK814X9S//uZB1V7/5jurastC05TQYj6Z03w1Xk4J4YoLQJ8dO0xNd63IBwRVLLbt0VVWj+85hhuGdyTW4f0Yljv9jU+Xk0NipD5xloDmIqSomUEphpffX64evSm9aNXm/Tu1CLqOj63DO7F4AP3Jj83h1MPcSatq0ljbvB5Lz83h1uH9OLQjk5M/zznkID3m9TPr0xzu+YNyMvNCbiX4o6h+3HDCT0qX+/dzMmsDmzXlKEHtyVY707NmXn3SZ6ql/xP/gV+DeL9u1U/6b96+RHVlpUHNaaEOmcEt7fE4pKjusS1nf8hI13R/jdEmnz6d6u6ryJHnMzAn3//gYM6NKN+fi5XDtyHR84+NNZwq/nVIfFP9Qpw8gF7h30v1hO7tRGYtLj0qC58cPXRHBXiChSc7qZf/WlQ2O1zc4T7fn1AXMf+7ZGdmX/f4KiNuV75n8haNqpHyYihnHFYh8pl4a4efVfW5/XryOUDuoU8meXl5nD/r6tXff3znENp1iDf03De0+48kSm3n8Cce0/2lOZPbzg24HVwqe3CIzpX2+a0oJNayYihYavJgh3o3m3u7w9F+1Q+92WKwfxLIZE+h6PD/MYAGhXkcpJbyhAJ3M+NJ+7L7w9K3k2JzT3eZ1IY542R3fyqMqP9TlLRh8AyAlNNTo5EHAKiecN6dGgR+or9m1uPY9bdJ8XVQAzO1VK4Rt9QPr3h2IAhuAEau3Mw/H5AV+75VfgMqWTE0LBXjwV5ucy460T+crpzog9XTda8oXMzXhO/q39fN1vf/2+kqpFmDfJp06SAxgV5XDFwn7DrXXZMVwDaNQ888ZYHteP8+ZT9WPjXwQHLzurTgaV/C2zID5WcpX87hQeGHcSgnlU9rxoXhP8uzurTgaEHtaVZg3xuGdwz4MrV/z4R34muIC+Hv595EAe4U64eEDT1aqgSlF92EnB/y3XH96B5/dSevkpGDOWJ83sHLPvbGQdx/hHRe7RVK2H4ff63Do58T00q2ghs0DmTUO0SdCUfyYfXHMOCX7awY085PfduQs+9A3sUNaknfHf78bRqVLMrRt9JHiL3UfdN9rN8/TZWbtpRufzQTi34ctFaurRuRMmIoXS59eOIx8vPzaFjywb8tGEH6nemmH3PSTSqlxcyjrKgFv2cHKEgp/rJOzdH6NKqKvM+vEtLJi1bT9fWjfhh3bbKdc7r14nz+nWqjNXXDVQEzj28E2u27Ky8CDi6eytEhJl3nwTAD+u2MWnpet68sn9lA39ujlQOqXDZMV055/BOvPB1CeA0/vubc+/J9Lh9NHvKnbT7Z1Y5Au1ClD7O69cx7EVJKBf178zLk5ZXW/7G8CM5Z+TkgGUvXno4xQvX8uI3JZXLTjqgkAE9WjNx8Tr2adOI4/crZNLS9VGPG/y9NfarXu3TuQUlI4by7vQVHNGtFZu37+GURydWvu/7GE7YrxAojZ7IOFhGYGqdgzo046AO1ass/O3VJHSVRbwuO6Yra7fu4qqi8FftnVs1Crjp7skLerNkTWnlle4Llx5O0/p5nPnUpLD7uLqoO7e+O5s2TQp456qjaNO4gCZ+De7183N5+8r+LFu7jVvemVXtqjqSCTcXVZ5cn724L8vXb+O6174PuW775g1YuWkH3fdqwld/GkS7Zg0qr8hVlQ4tGjCgR2C1TtfWjfj85iIA9rhFFcEZW2fqHSfQws1Yfe/52keuHrQPDd2MLrik4qtiEhFEhCsGduMVvxP5A8NC9yRa+NfBvPRNCX8bvSBg+X2nH8iJ+xfy2+e+C1h+RLdWXHFsN575clll3EU996Ko514BGUF+bg53nbo/J/7zy8plfzxxX179djmN6uWxftvuyuWHdGzOzJ82VfZaAxh2mFMVecvJvbju9e9568r+7OP2dhrW26myDK4iLN1ZBsDJBxRCqWUExqRN/fzciNVMoTQuyAu4ozq4x1OoaUPP7deJc92b5/zv9vbXt0tL+nZpybDe7cnzu9L070L7+PmHcc3/Ak/yzsm0KrYD2jXj3MM7cf/o+bwY1C13ws1FldVhwVfcIhL1xr28HOHIbi257Bin/7//QGu+GfB8sf/fyaGrRk4+YG86tmzI+AVrOKxTcwBuG7Iftw2JfrNfQV4ulx7tVKd9NOtnZq3YXPnegB5Vsf/11wfyxaK1AE7PMjcjKPQrfVx3fA9GzVhZ+bqVm5YhBzodBRoV5LHgL0MAAkp9H1xdNabQHe/PBqhMxzE9WjP9zhPDxt+sQT6bd+yhTZMCurVpxMLVW6OmuSaSmhGIyGDg30Au8B9VHRH0fgHwMtAHWA+co6olyYzJmHQ7r18nTg3R2yhWvhNp68YFrCvdFdCF9tSD29FjrybM/GlTxH1cPqArlx3TNaD+HWIfBiGYiPD68P4h3xt5UR/enPpTQFWVzz/POZRrX/ueEcMOqpw0ycs826Hk5+Yw/Nh9eO/7VQB8fF1VW80tg3vStH4+Fx7ZmQuPdBrYWzSqx+c3DWTWis0B38+NJ+7LjSfuW/m6ZaN6zLzrJJqE6D13Xr+OfDZ/TbXlvoywaQNvjdDf3X48qs4FyE1vzgSSe8Nc0jICEckFngBOBFYAU0RklKrO81vtMmCjqnYXkXOBvwPnJCsmYzLBA8Oq3yNQE6OuOZo3xn5dbXmo9pNg/qWEVOncqlHYUsBph7Sr1supph4662D+PX5xwN3pfyjqHnLdbm0aRx02BagcoDHYA8MO5oEQy/9Q1J12zRpw2sHe0ubf2cLXrdjLzYPxSmazez9giaouU9XdwOvA6UHrnA685D5/Gzhe6tJ99sakQLvmDThsL6vlDefA9s149qK+aR2Url5eDmcf3rFaycuLPw3uxRUDuyU8g/QnybrrUETOAgar6uXu698CR6jqNX7rzHHXWeG+Xuqusy5oX8OB4QCFhYV9Xn/99bhiKi0tpXHjzLkNPRUszdnB0pwdapLmQYMGTVPVkMP91orLCFUdCYwE6Nu3rxYVFcW1n+LiYuLdtrayNGcHS3N2SFaak1lWWgn4j7zUwV0Wch0RyQOa4TQaG2OMSZFkZgRTgB4i0lVE6gHnAqOC1hkFXOw+Pwv4XFM1QpYxxhggiVVDqlomItcAn+J0H31eVeeKyH3AVFUdBTwHvCIiS4ANOJmFMcaYFEpqG4GqjgZGBy27y+/5TuA3yYzBGGNMZDbonDHGZDnLCIwxJstZRmCMMVkuaTeUJYuIrAWqjyPrTWtgXdS16hZLc3awNGeHmqS5s6qGHC2w1mUENSEiU8PdWVdXWZqzg6U5OyQrzVY1ZIwxWc4yAmOMyXLZlhGMTHcAaWBpzg6W5uyQlDRnVRuBMcaY6rKtRGCMMSaIZQTGGJPlsiYjEJHBIrJQRJaIyK3pjieRRKRERGaLyAwRmeouayki40Rksfu3hbtcRORR93OYJSK90xu9NyLyvIiscScz8i2LOY0icrG7/mIRuTjUsTJFmDTfIyIr3e96hoic4vfebW6aF4rIyX7La8VvX0Q6isgEEZknInNF5Hp3eZ39niOkObXfs6rW+QfO6KdLgW5APWAmsH+640pg+kqA1kHLHgRudZ/fCvzdfX4K8AkgwJHAt+mO32MajwV6A3PiTSPQEljm/m3hPm+R7rTFmOZ7gJtDrLu/+7suALq6v/fc2vTbB9oCvd3nTYBFbrrq7PccIc0p/Z6zpUTgZf7kusZ/PuiXgF/7LX9ZHZOB5iLSNg3xxURVv8QZqtxfrGk8GRinqhtUdSMwDhic9ODjFCbN4ZwOvK6qu1T1B2AJzu++1vz2VfVnVZ3uPt8KzAfaU4e/5whpDicp33O2ZATtgZ/8Xq8g8odd2ygwVkSmufM7AxSq6s/u81+AQvd5XfosYk1jXUn7NW5VyPO+ahLqWJpFpAtwGPAtWfI9B6UZUvg9Z0tGUNcdo6q9gSHA1SJyrP+b6pQp63Q/4WxIo+spYB/gUOBn4OG0RpMEItIYeAe4QVW3+L9XV7/nEGlO6fecLRmBl/mTay1VXen+XQO8h1NMXO2r8nH/rnFXr0ufRaxprPVpV9XVqlquqhXAszjfNdSRNItIPs4J8b+q+q67uE5/z6HSnOrvOVsyAi/zJ9dKItJIRJr4ngMnAXMInA/6YuAD9/ko4CK3x8WRwGa/YndtE2saPwVOEpEWblH7JHdZrRHUnnMGzncNTprPFZECEekK9AC+oxb99kVEcKavna+qj/i9VWe/53BpTvn3nO5W81Q9cHoYLMJpWb893fEkMF3dcHoIzATm+tIGtALGA4uBz4CW7nIBnnA/h9lA33SnwWM6X8MpIu/Bqf+8LJ40Ar/DaWBbAlya7nTFkeZX3DTNcv/R2/qtf7ub5oXAEL/lteK3DxyDU+0zC5jhPk6py99zhDSn9Hu2ISaMMSbLZUvVkDHGmDAsIzDGmCxnGYExxmQ5ywiMMSbLWUZgjDFZzjICY2IgIqXu3y4icn664zEmESwjMCY+XYCYMgIRyUtOKMbUjGUExsRnBDDAHSv+jyKSKyIPicgUd6CwKwBEpEhEJorIKGBeekM2JjS7QjEmPrfijBd/KoA76utmVT1cRAqAr0VkrLtub+BAdYYNNibjWEZgTGKcBBwsIme5r5vhjAOzG/jOMgGTySwjMCYxBLhWVQMGNxORImBbOgIyxitrIzAmPltxphb0+RS4yh1SGBHZ1x0N1piMZyUCY+IzCygXkZnAi8C/cXoSTXeHFl5L1ZSKxmQ0G33UGGOynFUNGWNMlrOMwBhjspxlBMYYk+UsIzDGmCxnGYExxmQ5ywiMMSbLWUZgjDFZ7v8B76HAl29HH1oAAAAASUVORK5CYII=\n",
"text/plain": [
"