{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"name": "2021-07-12-yet-another-movie-recommender-pytorch.ipynb",
"provenance": [],
"toc_visible": true,
"authorship_tag": "ABX9TyNM3CW6ibwkJwT/j4siHHGp"
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"language_info": {
"name": "python"
}
},
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "qKn2XvzuSaqU"
},
"source": [
"# Yet another Movie Recommender from scratch\n",
"> Building and training Item-popularity and MLP model on movielens dataset in pure pytorch\n",
"\n",
"- toc: true\n",
"- badges: true\n",
"- comments: true\n",
"- categories: [PyTorch, Movie, MLP]\n",
"- author: \"Harshdeep Gupta\"\n",
"- image:"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "GUoQMgjCPmkp"
},
"source": [
"## Setup"
]
},
{
"cell_type": "code",
"metadata": {
"id": "Q6wvep55K6of"
},
"source": [
"import math\n",
"import torch\n",
"import heapq\n",
"import pickle\n",
"import argparse\n",
"import numpy as np\n",
"import pandas as pd\n",
"from torch import nn\n",
"import seaborn as sns\n",
"from time import time\n",
"import scipy.sparse as sp\n",
"import matplotlib.pyplot as plt\n",
"import torch.nn.functional as F\n",
"from torch.autograd import Variable\n",
"from torch.utils.data import Dataset, DataLoader"
],
"execution_count": 8,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "jz4ocKNnLN4P"
},
"source": [
"np.random.seed(7)\n",
"torch.manual_seed(0)\n",
"\n",
"_model = None\n",
"_testRatings = None\n",
"_testNegatives = None\n",
"_topk = None\n",
"\n",
"use_cuda = torch.cuda.is_available()\n",
"device = torch.device(\"cuda:0\" if use_cuda else \"cpu\")"
],
"execution_count": 9,
"outputs": []
},
{
"cell_type": "markdown",
"metadata": {
"id": "srnZMdMoPh9V"
},
"source": [
"## Data Loading"
]
},
{
"cell_type": "code",
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "J52BdmTvKvUv",
"outputId": "69b3556b-7460-4720-88b8-3a4fb33970b0"
},
"source": [
"!wget https://github.com/HarshdeepGupta/recommender_pytorch/raw/master/Data/movielens.train.rating\n",
"!wget https://github.com/HarshdeepGupta/recommender_pytorch/raw/master/Data/movielens.test.rating\n",
"!wget https://github.com/HarshdeepGupta/recommender_pytorch/raw/master/Data/u.data"
],
"execution_count": 4,
"outputs": [
{
"output_type": "stream",
"text": [
"--2021-07-12 05:23:32-- https://github.com/HarshdeepGupta/recommender_pytorch/raw/master/Data/movielens.train.rating\n",
"Resolving github.com (github.com)... 140.82.114.3\n",
"Connecting to github.com (github.com)|140.82.114.3|:443... connected.\n",
"HTTP request sent, awaiting response... 302 Found\n",
"Location: https://raw.githubusercontent.com/HarshdeepGupta/recommender_pytorch/master/Data/movielens.train.rating [following]\n",
"--2021-07-12 05:23:32-- https://raw.githubusercontent.com/HarshdeepGupta/recommender_pytorch/master/Data/movielens.train.rating\n",
"Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.108.133, 185.199.109.133, 185.199.110.133, ...\n",
"Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.108.133|:443... connected.\n",
"HTTP request sent, awaiting response... 200 OK\n",
"Length: 1960426 (1.9M) [text/plain]\n",
"Saving to: ‘movielens.train.rating’\n",
"\n",
"movielens.train.rat 100%[===================>] 1.87M --.-KB/s in 0.06s \n",
"\n",
"2021-07-12 05:23:33 (30.1 MB/s) - ‘movielens.train.rating’ saved [1960426/1960426]\n",
"\n",
"--2021-07-12 05:23:33-- https://github.com/HarshdeepGupta/recommender_pytorch/raw/master/Data/movielens.test.rating\n",
"Resolving github.com (github.com)... 140.82.113.4\n",
"Connecting to github.com (github.com)|140.82.113.4|:443... connected.\n",
"HTTP request sent, awaiting response... 302 Found\n",
"Location: https://raw.githubusercontent.com/HarshdeepGupta/recommender_pytorch/master/Data/movielens.test.rating [following]\n",
"--2021-07-12 05:23:33-- https://raw.githubusercontent.com/HarshdeepGupta/recommender_pytorch/master/Data/movielens.test.rating\n",
"Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.108.133, 185.199.109.133, 185.199.110.133, ...\n",
"Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.108.133|:443... connected.\n",
"HTTP request sent, awaiting response... 200 OK\n",
"Length: 18747 (18K) [text/plain]\n",
"Saving to: ‘movielens.test.rating’\n",
"\n",
"movielens.test.rati 100%[===================>] 18.31K --.-KB/s in 0s \n",
"\n",
"2021-07-12 05:23:33 (41.1 MB/s) - ‘movielens.test.rating’ saved [18747/18747]\n",
"\n",
"--2021-07-12 05:23:33-- https://github.com/HarshdeepGupta/recommender_pytorch/raw/master/Data/u.data\n",
"Resolving github.com (github.com)... 140.82.113.3\n",
"Connecting to github.com (github.com)|140.82.113.3|:443... connected.\n",
"HTTP request sent, awaiting response... 302 Found\n",
"Location: https://raw.githubusercontent.com/HarshdeepGupta/recommender_pytorch/master/Data/u.data [following]\n",
"--2021-07-12 05:23:34-- https://raw.githubusercontent.com/HarshdeepGupta/recommender_pytorch/master/Data/u.data\n",
"Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.108.133, 185.199.109.133, 185.199.110.133, ...\n",
"Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.108.133|:443... connected.\n",
"HTTP request sent, awaiting response... 200 OK\n",
"Length: 1979173 (1.9M) [text/plain]\n",
"Saving to: ‘u.data’\n",
"\n",
"u.data 100%[===================>] 1.89M --.-KB/s in 0.07s \n",
"\n",
"2021-07-12 05:23:34 (27.6 MB/s) - ‘u.data’ saved [1979173/1979173]\n",
"\n"
],
"name": "stdout"
}
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "csjr7o5wPd7n"
},
"source": [
"## Eval Methods"
]
},
{
"cell_type": "code",
"metadata": {
"id": "et-6h-pkLLMk"
},
"source": [
"def evaluate_model(model, full_dataset: MovieLensDataset, topK: int):\n",
" \"\"\"\n",
" Evaluate the performance (Hit_Ratio, NDCG) of top-K recommendation\n",
" Return: score of each test rating.\n",
" \"\"\"\n",
" global _model\n",
" global _testRatings\n",
" global _testNegatives\n",
" global _topk\n",
" _model = model\n",
" _testRatings = full_dataset.testRatings\n",
" _testNegatives = full_dataset.testNegatives\n",
" _topk = topK\n",
"\n",
" hits, ndcgs = [], []\n",
" for idx in range(len(_testRatings)):\n",
" (hr, ndcg) = eval_one_rating(idx, full_dataset)\n",
" hits.append(hr)\n",
" ndcgs.append(ndcg)\n",
" return (hits, ndcgs)\n",
"\n",
"\n",
"def eval_one_rating(idx, full_dataset: MovieLensDataset):\n",
" rating = _testRatings[idx]\n",
" items = _testNegatives[idx]\n",
" u = rating[0]\n",
"\n",
" gtItem = rating[1]\n",
" items.append(gtItem)\n",
" # Get prediction scores\n",
" map_item_score = {}\n",
" users = np.full(len(items), u, dtype='int32')\n",
"\n",
" feed_dict = {\n",
" 'user_id': users,\n",
" 'item_id': np.array(items),\n",
" }\n",
" predictions = _model.predict(feed_dict)\n",
" for i in range(len(items)):\n",
" item = items[i]\n",
" map_item_score[item] = predictions[i]\n",
"\n",
" # Evaluate top rank list\n",
" ranklist = heapq.nlargest(_topk, map_item_score, key=map_item_score.get)\n",
" hr = getHitRatio(ranklist, gtItem)\n",
" ndcg = getNDCG(ranklist, gtItem)\n",
" return (hr, ndcg)"
],
"execution_count": 10,
"outputs": []
},
{
"cell_type": "markdown",
"metadata": {
"id": "SvpXLWBpPYKk"
},
"source": [
"## Eval Metrics"
]
},
{
"cell_type": "code",
"metadata": {
"id": "vvU46669Mnmz"
},
"source": [
"def getHitRatio(ranklist, gtItem):\n",
" for item in ranklist:\n",
" if item == gtItem:\n",
" return 1\n",
" return 0\n",
"\n",
"\n",
"def getNDCG(ranklist, gtItem):\n",
" for i in range(len(ranklist)):\n",
" item = ranklist[i]\n",
" if item == gtItem:\n",
" return math.log(2) / math.log(i+2)\n",
" return 0"
],
"execution_count": 11,
"outputs": []
},
{
"cell_type": "markdown",
"metadata": {
"id": "Rv_-b2rnPQXI"
},
"source": [
"## Pytorch Dataset"
]
},
{
"cell_type": "code",
"metadata": {
"id": "grg5RywRK1H8"
},
"source": [
"class MovieLensDataset(Dataset):\n",
" 'Characterizes the dataset for PyTorch, and feeds the (user,item) pairs for training'\n",
"\n",
" def __init__(self, file_name, num_negatives_train=5, num_negatives_test=100):\n",
" 'Load the datasets from disk, and store them in appropriate structures'\n",
"\n",
" self.trainMatrix = self.load_rating_file_as_matrix(\n",
" file_name + \".train.rating\")\n",
" self.num_users, self.num_items = self.trainMatrix.shape\n",
" # make training set with negative sampling\n",
" self.user_input, self.item_input, self.ratings = self.get_train_instances(\n",
" self.trainMatrix, num_negatives_train)\n",
" # make testing set with negative sampling\n",
" self.testRatings = self.load_rating_file_as_list(\n",
" file_name + \".test.rating\")\n",
" self.testNegatives = self.create_negative_file(\n",
" num_samples=num_negatives_test)\n",
" assert len(self.testRatings) == len(self.testNegatives)\n",
"\n",
" def __len__(self):\n",
" 'Denotes the total number of rating in test set'\n",
" return len(self.user_input)\n",
"\n",
" def __getitem__(self, index):\n",
" 'Generates one sample of data'\n",
"\n",
" # get the train data\n",
" user_id = self.user_input[index]\n",
" item_id = self.item_input[index]\n",
" rating = self.ratings[index]\n",
"\n",
" return {'user_id': user_id,\n",
" 'item_id': item_id,\n",
" 'rating': rating}\n",
"\n",
" def get_train_instances(self, train, num_negatives):\n",
" user_input, item_input, ratings = [], [], []\n",
" num_users, num_items = train.shape\n",
" for (u, i) in train.keys():\n",
" # positive instance\n",
" user_input.append(u)\n",
" item_input.append(i)\n",
" ratings.append(1)\n",
" # negative instances\n",
" for _ in range(num_negatives):\n",
" j = np.random.randint(1, num_items)\n",
" # while train.has_key((u, j)):\n",
" while (u, j) in train:\n",
" j = np.random.randint(1, num_items)\n",
" user_input.append(u)\n",
" item_input.append(j)\n",
" ratings.append(0)\n",
" return user_input, item_input, ratings\n",
"\n",
" def load_rating_file_as_list(self, filename):\n",
" ratingList = []\n",
" with open(filename, \"r\") as f:\n",
" line = f.readline()\n",
" while line != None and line != \"\":\n",
" arr = line.split(\"\\t\")\n",
" user, item = int(arr[0]), int(arr[1])\n",
" ratingList.append([user, item])\n",
" line = f.readline()\n",
" return ratingList\n",
"\n",
" def create_negative_file(self, num_samples=100):\n",
" negativeList = []\n",
" for user_item_pair in self.testRatings:\n",
" user = user_item_pair[0]\n",
" item = user_item_pair[1]\n",
" negatives = []\n",
" for t in range(num_samples):\n",
" j = np.random.randint(1, self.num_items)\n",
" while (user, j) in self.trainMatrix or j == item:\n",
" j = np.random.randint(1, self.num_items)\n",
" negatives.append(j)\n",
" negativeList.append(negatives)\n",
" return negativeList\n",
"\n",
" def load_rating_file_as_matrix(self, filename):\n",
" '''\n",
" Read .rating file and Return dok matrix.\n",
" The first line of .rating file is: num_users\\t num_items\n",
" '''\n",
" # Get number of users and items\n",
" num_users, num_items = 0, 0\n",
" with open(filename, \"r\") as f:\n",
" line = f.readline()\n",
" while line != None and line != \"\":\n",
" arr = line.split(\"\\t\")\n",
" u, i = int(arr[0]), int(arr[1])\n",
" num_users = max(num_users, u)\n",
" num_items = max(num_items, i)\n",
" line = f.readline()\n",
" # Construct matrix\n",
" mat = sp.dok_matrix((num_users+1, num_items+1), dtype=np.float32)\n",
" with open(filename, \"r\") as f:\n",
" line = f.readline()\n",
" while line != None and line != \"\":\n",
" arr = line.split(\"\\t\")\n",
" user, item, rating = int(arr[0]), int(arr[1]), float(arr[2])\n",
" if (rating > 0):\n",
" mat[user, item] = 1.0\n",
" line = f.readline()\n",
" return mat"
],
"execution_count": 12,
"outputs": []
},
{
"cell_type": "markdown",
"metadata": {
"id": "M_CJ2wlKPS-w"
},
"source": [
"## Utils"
]
},
{
"cell_type": "code",
"metadata": {
"id": "YhQs5tfvK9Pf"
},
"source": [
"def train_one_epoch(model, data_loader, loss_fn, optimizer, epoch_no, device, verbose = 1):\n",
" 'trains the model for one epoch and returns the loss'\n",
" print(\"Epoch = {}\".format(epoch_no))\n",
" # Training\n",
" # get user, item and rating data\n",
" t1 = time()\n",
" epoch_loss = []\n",
" # put the model in train mode before training\n",
" model.train()\n",
" # transfer the data to GPU\n",
" for feed_dict in data_loader:\n",
" for key in feed_dict:\n",
" if type(feed_dict[key]) != type(None):\n",
" feed_dict[key] = feed_dict[key].to(dtype = torch.long, device = device)\n",
" # get the predictions\n",
" prediction = model(feed_dict)\n",
" # print(prediction.shape)\n",
" # get the actual targets\n",
" rating = feed_dict['rating']\n",
" \n",
" \n",
" # convert to float and change dim from [batch_size] to [batch_size,1]\n",
" rating = rating.float().view(prediction.size()) \n",
" loss = loss_fn(prediction, rating)\n",
" # clear the gradients\n",
" optimizer.zero_grad()\n",
" # backpropagate\n",
" loss.backward()\n",
" # update weights\n",
" optimizer.step()\n",
" # accumulate the loss for monitoring\n",
" epoch_loss.append(loss.item())\n",
" epoch_loss = np.mean(epoch_loss)\n",
" if verbose:\n",
" print(\"Epoch completed {:.1f} s\".format(time() - t1))\n",
" print(\"Train Loss: {}\".format(epoch_loss))\n",
" return epoch_loss\n",
" \n",
"\n",
"def test(model, full_dataset : MovieLensDataset, topK):\n",
" 'Test the HR and NDCG for the model @topK'\n",
" # put the model in eval mode before testing\n",
" if hasattr(model,'eval'):\n",
" # print(\"Putting the model in eval mode\")\n",
" model.eval()\n",
" t1 = time()\n",
" (hits, ndcgs) = evaluate_model(model, full_dataset, topK)\n",
" hr, ndcg = np.array(hits).mean(), np.array(ndcgs).mean()\n",
" print('Eval: HR = %.4f, NDCG = %.4f [%.1f s]' % (hr, ndcg, time()-t1))\n",
" return hr, ndcg\n",
" \n",
"\n",
"def plot_statistics(hr_list, ndcg_list, loss_list, model_alias, path):\n",
" 'plots and saves the figures to a local directory'\n",
" plt.figure()\n",
" hr = np.vstack([np.arange(len(hr_list)),np.array(hr_list)]).T\n",
" ndcg = np.vstack([np.arange(len(ndcg_list)),np.array(ndcg_list)]).T\n",
" loss = np.vstack([np.arange(len(loss_list)),np.array(loss_list)]).T\n",
" plt.plot(hr[:,0], hr[:,1],linestyle='-', marker='o', label = \"HR\")\n",
" plt.plot(ndcg[:,0], ndcg[:,1],linestyle='-', marker='v', label = \"NDCG\")\n",
" plt.plot(loss[:,0], loss[:,1],linestyle='-', marker='s', label = \"Loss\")\n",
"\n",
" plt.xlabel(\"Epochs\")\n",
" plt.ylabel(\"Value\")\n",
" plt.legend()\n",
" plt.savefig(path+model_alias+\".jpg\")\n",
" return\n",
"\n",
"\n",
"def get_items_interacted(user_id, interaction_df):\n",
" # returns a set of items the user has interacted with\n",
" userid_mask = interaction_df['userid'] == user_id\n",
" interacted_items = interaction_df.loc[userid_mask].courseid\n",
" return set(interacted_items if type(interacted_items) == pd.Series else [interacted_items])\n",
"\n",
"\n",
"def save_to_csv(df,path, header = False, index = False, sep = '\\t', verbose = False):\n",
" if verbose:\n",
" print(\"Saving df to path: {}\".format(path))\n",
" print(\"Columns in df are: {}\".format(df.columns.tolist()))\n",
"\n",
" df.to_csv(path, header = header, index = index, sep = sep)"
],
"execution_count": 13,
"outputs": []
},
{
"cell_type": "markdown",
"metadata": {
"id": "qEl945qyM1FB"
},
"source": [
"## Item Popularity Model"
]
},
{
"cell_type": "code",
"metadata": {
"id": "dzjxYN-mM3uv"
},
"source": [
"def parse_args():\n",
" parser = argparse.ArgumentParser(description=\"Run ItemPop\")\n",
" parser.add_argument('--path', nargs='?', default='/content/',\n",
" help='Input data path.')\n",
" parser.add_argument('--dataset', nargs='?', default='movielens',\n",
" help='Choose a dataset.')\n",
" parser.add_argument('--num_neg_test', type=int, default=100,\n",
" help='Number of negative instances to pair with a positive instance while testing')\n",
" \n",
" return parser.parse_args(args={})"
],
"execution_count": 18,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "xAr3XZ4sM73x"
},
"source": [
"class ItemPop():\n",
" def __init__(self, train_interaction_matrix: sp.dok_matrix):\n",
" \"\"\"\n",
" Simple popularity based recommender system\n",
" \"\"\"\n",
" self.__alias__ = \"Item Popularity without metadata\"\n",
" # Sum the occurences of each item to get is popularity, convert to array and \n",
" # lose the extra dimension\n",
" self.item_ratings = np.array(train_interaction_matrix.sum(axis=0, dtype=int)).flatten()\n",
"\n",
" def forward(self):\n",
" pass\n",
"\n",
" def predict(self, feeddict) -> np.array:\n",
" # returns the prediction score for each (user,item) pair in the input\n",
" items = feeddict['item_id']\n",
" output_scores = [self.item_ratings[itemid] for itemid in items]\n",
" return np.array(output_scores)\n",
"\n",
" def get_alias(self):\n",
" return self.__alias__"
],
"execution_count": 19,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "0EDavlooLWT0",
"outputId": "a1f13e86-030e-4530-e2d9-34b0d676570a"
},
"source": [
"args = parse_args()\n",
"path = args.path\n",
"dataset = args.dataset\n",
"num_negatives_test = args.num_neg_test\n",
"print(\"Model arguments: %s \" %(args))\n",
"\n",
"topK = 10\n",
"\n",
"# Load data\n",
"\n",
"t1 = time()\n",
"full_dataset = MovieLensDataset(path + dataset, num_negatives_test=num_negatives_test)\n",
"train, testRatings, testNegatives = full_dataset.trainMatrix, full_dataset.testRatings, full_dataset.testNegatives\n",
"num_users, num_items = train.shape\n",
"print(\"Load data done [%.1f s]. #user=%d, #item=%d, #train=%d, #test=%d\"\n",
" % (time()-t1, num_users, num_items, train.nnz, len(testRatings)))\n",
"\n",
"model = ItemPop(train)\n",
"test(model, full_dataset, topK)"
],
"execution_count": 20,
"outputs": [
{
"output_type": "stream",
"text": [
"Model arguments: Namespace(dataset='movielens', num_neg_test=100, path='/content/') \n",
"Load data done [4.3 s]. #user=944, #item=1683, #train=99057, #test=943\n",
"Eval: HR = 0.4062, NDCG = 0.2199 [0.1 s]\n"
],
"name": "stdout"
},
{
"output_type": "execute_result",
"data": {
"text/plain": [
"(0.4061505832449629, 0.21988638109018463)"
]
},
"metadata": {
"tags": []
},
"execution_count": 20
}
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "IaJQ8h8kNWmr"
},
"source": [
"## MLP Model"
]
},
{
"cell_type": "code",
"metadata": {
"id": "F_GYre42NhDX"
},
"source": [
"def parse_args():\n",
" parser = argparse.ArgumentParser(description=\"Run MLP.\")\n",
" parser.add_argument('--path', nargs='?', default='/content/',\n",
" help='Input data path.')\n",
" parser.add_argument('--dataset', nargs='?', default='movielens',\n",
" help='Choose a dataset.')\n",
" parser.add_argument('--epochs', type=int, default=30,\n",
" help='Number of epochs.')\n",
" parser.add_argument('--batch_size', type=int, default=256,\n",
" help='Batch size.')\n",
" parser.add_argument('--layers', nargs='?', default='[16,32,16,8]',\n",
" help=\"Size of each layer. Note that the first layer is the concatenation of user and item embeddings. So layers[0]/2 is the embedding size.\")\n",
" parser.add_argument('--weight_decay', type=float, default=0.00001,\n",
" help=\"Regularization for each layer\")\n",
" parser.add_argument('--num_neg_train', type=int, default=4,\n",
" help='Number of negative instances to pair with a positive instance while training')\n",
" parser.add_argument('--num_neg_test', type=int, default=100,\n",
" help='Number of negative instances to pair with a positive instance while testing')\n",
" parser.add_argument('--lr', type=float, default=0.001,\n",
" help='Learning rate.')\n",
" parser.add_argument('--dropout', type=float, default=0,\n",
" help='Add dropout layer after each dense layer, with p = dropout_prob')\n",
" parser.add_argument('--learner', nargs='?', default='adam',\n",
" help='Specify an optimizer: adagrad, adam, rmsprop, sgd')\n",
" parser.add_argument('--verbose', type=int, default=1,\n",
" help='Show performance per X iterations')\n",
" parser.add_argument('--out', type=int, default=1,\n",
" help='Whether to save the trained model.')\n",
" return parser.parse_args(args={})"
],
"execution_count": 21,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "BJVmkqWGNoXM"
},
"source": [
"class MLP(nn.Module):\n",
"\n",
" def __init__(self, n_users, n_items, layers=[16, 8], dropout=False):\n",
" \"\"\"\n",
" Simple Feedforward network with Embeddings for users and items\n",
" \"\"\"\n",
" super().__init__()\n",
" assert (layers[0] % 2 == 0), \"layers[0] must be an even number\"\n",
" self.__alias__ = \"MLP {}\".format(layers)\n",
" self.__dropout__ = dropout\n",
"\n",
" # user and item embedding layers\n",
" embedding_dim = int(layers[0]/2)\n",
" self.user_embedding = torch.nn.Embedding(n_users, embedding_dim)\n",
" self.item_embedding = torch.nn.Embedding(n_items, embedding_dim)\n",
"\n",
" # list of weight matrices\n",
" self.fc_layers = torch.nn.ModuleList()\n",
" # hidden dense layers\n",
" for _, (in_size, out_size) in enumerate(zip(layers[:-1], layers[1:])):\n",
" self.fc_layers.append(torch.nn.Linear(in_size, out_size))\n",
" # final prediction layer\n",
" self.output_layer = torch.nn.Linear(layers[-1], 1)\n",
"\n",
" def forward(self, feed_dict):\n",
" users = feed_dict['user_id']\n",
" items = feed_dict['item_id']\n",
" user_embedding = self.user_embedding(users)\n",
" item_embedding = self.item_embedding(items)\n",
" # concatenate user and item embeddings to form input\n",
" x = torch.cat([user_embedding, item_embedding], 1)\n",
" for idx, _ in enumerate(range(len(self.fc_layers))):\n",
" x = self.fc_layers[idx](x)\n",
" x = F.relu(x)\n",
" x = F.dropout(x, p=self.__dropout__, training=self.training)\n",
" logit = self.output_layer(x)\n",
" rating = torch.sigmoid(logit)\n",
" return rating\n",
"\n",
" def predict(self, feed_dict):\n",
" # return the score, inputs and outputs are numpy arrays\n",
" for key in feed_dict:\n",
" if type(feed_dict[key]) != type(None):\n",
" feed_dict[key] = torch.from_numpy(\n",
" feed_dict[key]).to(dtype=torch.long, device=device)\n",
" output_scores = self.forward(feed_dict)\n",
" return output_scores.cpu().detach().numpy()\n",
"\n",
" def get_alias(self):\n",
" return self.__alias__"
],
"execution_count": 45,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 1000
},
"id": "lA9s7rv0LspV",
"outputId": "57d64091-be5b-493d-d20a-5f8d991e69ef"
},
"source": [
"print(\"Device available: {}\".format(device))\n",
"\n",
"args = parse_args()\n",
"path = args.path\n",
"dataset = args.dataset\n",
"layers = eval(args.layers)\n",
"weight_decay = args.weight_decay\n",
"num_negatives_train = args.num_neg_train\n",
"num_negatives_test = args.num_neg_test\n",
"dropout = args.dropout\n",
"learner = args.learner\n",
"learning_rate = args.lr\n",
"batch_size = args.batch_size\n",
"epochs = args.epochs\n",
"verbose = args.verbose\n",
"\n",
"topK = 10\n",
"print(\"MLP arguments: %s \" % (args))\n",
"model_out_file = '%s_MLP_%s_%d.h5' %(args.dataset, args.layers, time())\n",
"\n",
"# Load data\n",
"\n",
"t1 = time()\n",
"full_dataset = MovieLensDataset(\n",
" path + dataset, num_negatives_train=num_negatives_train, num_negatives_test=num_negatives_test)\n",
"train, testRatings, testNegatives = full_dataset.trainMatrix, full_dataset.testRatings, full_dataset.testNegatives\n",
"num_users, num_items = train.shape\n",
"print(\"Load data done [%.1f s]. #user=%d, #item=%d, #train=%d, #test=%d\"\n",
" % (time()-t1, num_users, num_items, train.nnz, len(testRatings)))\n",
"\n",
"training_data_generator = DataLoader(\n",
" full_dataset, batch_size=batch_size, shuffle=True, num_workers=0)\n",
"\n",
"# Build model\n",
"model = MLP(num_users, num_items, layers=layers, dropout=dropout)\n",
"# Transfer the model to GPU, if one is available\n",
"model.to(device)\n",
"if verbose:\n",
" print(model)\n",
"\n",
"loss_fn = torch.nn.BCELoss()\n",
"# Use Adam optimizer\n",
"optimizer = torch.optim.Adam(model.parameters(), weight_decay=weight_decay)\n",
"\n",
"# Record performance\n",
"hr_list = []\n",
"ndcg_list = []\n",
"BCE_loss_list = []\n",
"\n",
"# Check Init performance\n",
"hr, ndcg = test(model, full_dataset, topK)\n",
"hr_list.append(hr)\n",
"ndcg_list.append(ndcg)\n",
"BCE_loss_list.append(1)\n",
"\n",
"# do the epochs now\n",
"\n",
"for epoch in range(epochs):\n",
" epoch_loss = train_one_epoch( model, training_data_generator, loss_fn, optimizer, epoch, device)\n",
"\n",
" if epoch % verbose == 0:\n",
" hr, ndcg = test(model, full_dataset, topK)\n",
" hr_list.append(hr)\n",
" ndcg_list.append(ndcg)\n",
" BCE_loss_list.append(epoch_loss)\n",
" if hr > max(hr_list):\n",
" if args.out > 0:\n",
" model.save(model_out_file, overwrite=True)\n",
"\n",
"print(\"hr for epochs: \", hr_list)\n",
"print(\"ndcg for epochs: \", ndcg_list)\n",
"print(\"loss for epochs: \", BCE_loss_list)\n",
"plot_statistics(hr_list, ndcg_list, BCE_loss_list, model.get_alias(), \"/content\")\n",
"with open(\"metrics\", 'wb') as fp:\n",
" pickle.dump(hr_list, fp)\n",
" pickle.dump(ndcg_list, fp)\n",
"\n",
"best_iter = np.argmax(np.array(hr_list))\n",
"best_hr = hr_list[best_iter]\n",
"best_ndcg = ndcg_list[best_iter]\n",
"print(\"End. Best Iteration %d: HR = %.4f, NDCG = %.4f. \" %\n",
" (best_iter, best_hr, best_ndcg))\n",
"if args.out > 0:\n",
" print(\"The best MLP model is saved to %s\" %(model_out_file))"
],
"execution_count": 46,
"outputs": [
{
"output_type": "stream",
"text": [
"Device available: cpu\n",
"MLP arguments: Namespace(batch_size=256, dataset='movielens', dropout=0, epochs=30, layers='[16,32,16,8]', learner='adam', lr=0.001, num_neg_test=100, num_neg_train=4, out=1, path='/content/', verbose=1, weight_decay=1e-05) \n",
"Load data done [3.8 s]. #user=944, #item=1683, #train=99057, #test=943\n",
"MLP(\n",
" (user_embedding): Embedding(944, 8)\n",
" (item_embedding): Embedding(1683, 8)\n",
" (fc_layers): ModuleList(\n",
" (0): Linear(in_features=16, out_features=32, bias=True)\n",
" (1): Linear(in_features=32, out_features=16, bias=True)\n",
" (2): Linear(in_features=16, out_features=8, bias=True)\n",
" )\n",
" (output_layer): Linear(in_features=8, out_features=1, bias=True)\n",
")\n",
"Eval: HR = 0.0848, NDCG = 0.0386 [0.6 s]\n",
"Epoch = 0\n",
"Epoch completed 5.8 s\n",
"Train Loss: 0.4429853802195507\n",
"Eval: HR = 0.3945, NDCG = 0.2187 [0.6 s]\n",
"Epoch = 1\n",
"Epoch completed 5.6 s\n",
"Train Loss: 0.3646208482657292\n",
"Eval: HR = 0.3818, NDCG = 0.2133 [0.6 s]\n",
"Epoch = 2\n",
"Epoch completed 5.6 s\n",
"Train Loss: 0.35764367812979747\n",
"Eval: HR = 0.3924, NDCG = 0.2137 [0.6 s]\n",
"Epoch = 3\n",
"Epoch completed 5.7 s\n",
"Train Loss: 0.35384849094297227\n",
"Eval: HR = 0.3796, NDCG = 0.2103 [0.6 s]\n",
"Epoch = 4\n",
"Epoch completed 5.7 s\n",
"Train Loss: 0.35072445729290175\n",
"Eval: HR = 0.3818, NDCG = 0.2143 [0.6 s]\n",
"Epoch = 5\n",
"Epoch completed 5.8 s\n",
"Train Loss: 0.3481164647319212\n",
"Eval: HR = 0.3881, NDCG = 0.2171 [0.7 s]\n",
"Epoch = 6\n",
"Epoch completed 5.8 s\n",
"Train Loss: 0.3454590990638856\n",
"Eval: HR = 0.4157, NDCG = 0.2292 [0.6 s]\n",
"Epoch = 7\n",
"Epoch completed 5.8 s\n",
"Train Loss: 0.3422531268162321\n",
"Eval: HR = 0.4231, NDCG = 0.2371 [0.6 s]\n",
"Epoch = 8\n",
"Epoch completed 5.8 s\n",
"Train Loss: 0.3384355346053762\n",
"Eval: HR = 0.4443, NDCG = 0.2508 [0.6 s]\n",
"Epoch = 9\n",
"Epoch completed 5.8 s\n",
"Train Loss: 0.3335341374156395\n",
"Eval: HR = 0.4677, NDCG = 0.2598 [0.6 s]\n",
"Epoch = 10\n",
"Epoch completed 5.8 s\n",
"Train Loss: 0.3280563016347491\n",
"Eval: HR = 0.4719, NDCG = 0.2652 [0.6 s]\n",
"Epoch = 11\n",
"Epoch completed 5.7 s\n",
"Train Loss: 0.3223747977760719\n",
"Eval: HR = 0.4995, NDCG = 0.2748 [0.6 s]\n",
"Epoch = 12\n",
"Epoch completed 5.8 s\n",
"Train Loss: 0.3164166678753934\n",
"Eval: HR = 0.5090, NDCG = 0.2817 [0.6 s]\n",
"Epoch = 13\n",
"Epoch completed 5.7 s\n",
"Train Loss: 0.31102338709726507\n",
"Eval: HR = 0.5143, NDCG = 0.2829 [0.6 s]\n",
"Epoch = 14\n",
"Epoch completed 5.7 s\n",
"Train Loss: 0.30582732322604156\n",
"Eval: HR = 0.5175, NDCG = 0.2908 [0.6 s]\n",
"Epoch = 15\n",
"Epoch completed 5.6 s\n",
"Train Loss: 0.3016319169092548\n",
"Eval: HR = 0.5429, NDCG = 0.2963 [0.6 s]\n",
"Epoch = 16\n",
"Epoch completed 5.7 s\n",
"Train Loss: 0.2980319341254789\n",
"Eval: HR = 0.5493, NDCG = 0.2978 [0.6 s]\n",
"Epoch = 17\n",
"Epoch completed 5.7 s\n",
"Train Loss: 0.29476294266469105\n",
"Eval: HR = 0.5504, NDCG = 0.3014 [0.6 s]\n",
"Epoch = 18\n",
"Epoch completed 5.6 s\n",
"Train Loss: 0.2921119521985682\n",
"Eval: HR = 0.5589, NDCG = 0.3108 [0.6 s]\n",
"Epoch = 19\n",
"Epoch completed 5.8 s\n",
"Train Loss: 0.28990745035406845\n",
"Eval: HR = 0.5620, NDCG = 0.3092 [0.6 s]\n",
"Epoch = 20\n",
"Epoch completed 5.7 s\n",
"Train Loss: 0.2876521824250234\n",
"Eval: HR = 0.5514, NDCG = 0.3097 [0.6 s]\n",
"Epoch = 21\n",
"Epoch completed 5.6 s\n",
"Train Loss: 0.2858751243245078\n",
"Eval: HR = 0.5578, NDCG = 0.3122 [0.6 s]\n",
"Epoch = 22\n",
"Epoch completed 5.6 s\n",
"Train Loss: 0.2843063232125546\n",
"Eval: HR = 0.5567, NDCG = 0.3043 [0.6 s]\n",
"Epoch = 23\n",
"Epoch completed 5.6 s\n",
"Train Loss: 0.28271066885277896\n",
"Eval: HR = 0.5663, NDCG = 0.3141 [0.6 s]\n",
"Epoch = 24\n",
"Epoch completed 5.6 s\n",
"Train Loss: 0.2813221255630178\n",
"Eval: HR = 0.5610, NDCG = 0.3070 [0.6 s]\n",
"Epoch = 25\n",
"Epoch completed 5.7 s\n",
"Train Loss: 0.28002421261420235\n",
"Eval: HR = 0.5610, NDCG = 0.3110 [0.6 s]\n",
"Epoch = 26\n",
"Epoch completed 5.9 s\n",
"Train Loss: 0.27882074906998516\n",
"Eval: HR = 0.5610, NDCG = 0.3095 [0.6 s]\n",
"Epoch = 27\n",
"Epoch completed 5.8 s\n",
"Train Loss: 0.27783915350449484\n",
"Eval: HR = 0.5663, NDCG = 0.3115 [0.6 s]\n",
"Epoch = 28\n",
"Epoch completed 5.7 s\n",
"Train Loss: 0.2768868865122783\n",
"Eval: HR = 0.5631, NDCG = 0.3109 [0.6 s]\n",
"Epoch = 29\n",
"Epoch completed 5.8 s\n",
"Train Loss: 0.2760479487343968\n",
"Eval: HR = 0.5631, NDCG = 0.3092 [0.6 s]\n",
"hr for epochs: [0.08483563096500531, 0.3944856839872747, 0.38176033934252385, 0.39236479321314954, 0.3796394485683987, 0.38176033934252385, 0.38812301166489926, 0.41569459172852596, 0.42311770943796395, 0.4443266171792153, 0.4676564156945917, 0.471898197242842, 0.49946977730646874, 0.5090137857900318, 0.5143160127253447, 0.5174973488865323, 0.542948038176034, 0.5493107104984093, 0.5503711558854719, 0.5588547189819725, 0.5620360551431601, 0.5514316012725344, 0.5577942735949099, 0.5567338282078473, 0.5662778366914104, 0.5609756097560976, 0.5609756097560976, 0.5609756097560976, 0.5662778366914104, 0.5630965005302226, 0.5630965005302226]\n",
"ndcg for epochs: [0.03855482836637224, 0.2186689741068423, 0.21325592738572174, 0.21374918741658008, 0.21033736603276898, 0.21431768576892837, 0.21714573069782853, 0.2292039485312514, 0.23708514689275148, 0.2507826695009706, 0.2598176007060155, 0.2652029648171546, 0.2747717153150814, 0.2817258947342069, 0.28289172403583096, 0.2907608027818361, 0.29626902860751664, 0.29775495439534627, 0.3014327139896777, 0.31075028453364517, 0.30917060839326094, 0.3096903348455541, 0.31217614966561463, 0.3043410687051171, 0.314059797472155, 0.3070033682048637, 0.31104383409268926, 0.3094572048871119, 0.3115140344405953, 0.31090220293994014, 0.3092050624323008]\n",
"loss for epochs: [1, 0.4429853802195507, 0.3646208482657292, 0.35764367812979747, 0.35384849094297227, 0.35072445729290175, 0.3481164647319212, 0.3454590990638856, 0.3422531268162321, 0.3384355346053762, 0.3335341374156395, 0.3280563016347491, 0.3223747977760719, 0.3164166678753934, 0.31102338709726507, 0.30582732322604156, 0.3016319169092548, 0.2980319341254789, 0.29476294266469105, 0.2921119521985682, 0.28990745035406845, 0.2876521824250234, 0.2858751243245078, 0.2843063232125546, 0.28271066885277896, 0.2813221255630178, 0.28002421261420235, 0.27882074906998516, 0.27783915350449484, 0.2768868865122783, 0.2760479487343968]\n",
"End. Best Iteration 24: HR = 0.5663, NDCG = 0.3141. \n",
"The best MLP model is saved to movielens_MLP_[16,32,16,8]_1626069383.h5\n"
],
"name": "stdout"
},
{
"output_type": "display_data",
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXhU1fnA8e87SzKTkIUsEAggKKAiu5FWUVFaXKq11FoU7aK1om3pz4ob1mrRat1aF6zVutO6rxQrVlsBl7ogCMjmkoCyCCQkJCH7Muf3x50kk2RmMgkzmST3/TwPT2bunNx7bkbPe8+5575HjDEopZSyL0e8K6CUUiq+NBAopZTNaSBQSimb00CglFI2p4FAKaVszhXvCnRWVlaWGT58eLyroZRSvcrq1av3GmOyg33W6wLB8OHDWbVqVbyroZRSvYqIfBXqMx0aUkopm9NAoJRSNqeBQCmlbK7X3SNQSqlQ6uvr2bFjBzU1NfGuStx4PB6GDBmC2+2O+Hc0ECil+owdO3aQkpLC8OHDEZF4V6fbGWMoLi5mx44djBgxIuLfi1kgEJFHgdOBQmPM2CCfC3AP8B2gCjjfGPNxtOtxwrMnUFxT3G57pieTFWeviPbhlFJxVFNTY9sgACAiZGZmUlRU1Knfi+U9gseBU8J8fiowyv9vDnB/LCoRLAiE266U6t3sGgSadOX8YxYIjDFvAyVhinwP+LuxfACki8igWNVHKaVUcPGcNZQLbA94v8O/rR0RmSMiq0RkVWe7PEop1Z369evX6v3jjz/O3LlzAViwYAG5ublMnDiRMWPG8PTTT8ejiu30iumjxpgHjTF5xpi87OygT0grpVSnLV6zk6m3LmPE/FeZeusyFq/ZGfNjXnbZZaxdu5Z//vOfXHzxxdTX18f8mB2JZyDYCQwNeD/Ev00ppWJu8ZqdXPPSenaWVmOAnaXVXPPS+m4JBgCjRo0iKSmJffv2dcvxwonn9NElwFwReQb4BlBmjNkV7YNkejJDzhpSSvVdN7yykU1fl4f8fM22Uuoafa22Vdc3ctULn/D0ym1Bf2fM4FR+/90jwh63urqaiRMnNr8vKSnhjDPOaFfu448/ZtSoUQwYMCDs/rpDLKePPg2cAGSJyA7g94AbwBjzALAUa+poPtb00QtiUY+mKaLF1cWc8NwJXDPlGs49/NxYHEop1Yu0DQIdbY+U1+tl7dq1ze8ff/zxVoky77rrLh577DE+//xzXnnllQM6VrTELBAYY2Z38LkBfhWr47eVnpiOQxw6bVQpm+joyn3qrcvYWVrdbntuupdnLz46VtXisssu44orrmDJkiVceOGFFBQU4PF4Yna8SPSKm8XR4HQ4SU9Mp6Qm3IxWpZRdXHnyoXjdzlbbvG4nV558aLcc/4wzziAvL49FixZ1y/HCsU0gAMj0ZlJcrT0CpRTMnJTLLWeOIzfdi2D1BG45cxwzJwWdxR4T119/PXfeeSc+34ENRx0osUZoeo+8vDzT1YVpLnrjIqoaqnjyO09GuVZKqZ5g8+bNHH744fGuRtwF+zuIyGpjTF6w8rbrEZRU69CQUkoFslUgyPBk6M1ipZRqw1aBINOTSXVDNVX1VfGuilJK9Rj2CgRe6yEynTmklFItbBUIMjwZgKagVkqpQLYKBE09Ap1CqpRSLewVCDw6NKSUii0R4fLLL29+/6c//YkFCxYArdNQjxo1ijPPPJNNmzY1l62vr2f+/PmMGjWKyZMnc/TRR/Paa68BUFFRwS9+8QsOOeQQJk+ezJFHHslDDz0UlTrbas3i5qEh7REopR44Fnavb789Zxxc8m6Xd5uYmMhLL73ENddcQ1ZWVrvPm1JMADz77LNMnz6d9evXk52dzXXXXceuXbvYsGEDiYmJ7Nmzh7feeguAn//85xx88MF88cUXOBwOioqKePTRR7tcz0C26hEkOBNISUjRewRKKRgyBZwJrbc5E6ztB8DlcjFnzhzuuuuuDsueffbZnHTSSTz11FNUVVXx0EMPce+995KYmAjAwIEDmTVrFgUFBaxcuZKbbroJh8NqtrOzs7n66qsPqK7NdY7KXnqRTE+mDg0pZQevzQ9+xd+koQ58Da23+Rqs33nstOC/kzMOTr21w0P/6le/Yvz48Vx11VUdlp08eTKffvop+fn5DBs2jNTU1HZlNm7cyIQJE5qDQLTZqkcA/ofKdGhIKeVKgOQBQNNi72K9b9tL6ILU1FR+8pOfsHDhwg7LdiXNz80338zEiRMZPHhwV6rXjv16BN5M8kvz410NpVSsRXDlzv7dcM8EaKgBVyJc/DakDIzK4X/zm98wefJkLrgg/FIra9asIS8vj5EjR7Jt2zbKy8vb9QrGjBnDunXr8Pl8OBwOrr32Wq699tp26yN3le16BDo0pJRqlpIDE88DcVg/oxQEADIyMpg1axaPPPJIyDIvvvgib7zxBrNnzyYpKYkLL7yQSy+9lLq6OgCKiop4/vnnGTlyJHl5efzud7+jsbERgJqami71JoKxXSDI8GZQVltGvS/+C0YrpXqAaVfBsG/CtOjceA10+eWXs3fv3lbb7rrrrubpo0888QTLli0jOzsbgJtuuons7GzGjBnD2LFjOf3005t7Bw8//DDFxcXNQWHGjBncfvvtUamnrdJQAzz32XP84YM/8N+z/svA5OhFf6VU/Gkaaoumoe6A5htSSqnW7BcI/E8X67MESillsW8g0CmkSikF2DEQ6NCQUkq1YrtA4HV58Tg92iNQSik/2wUCESHTm6n3CJRSys92gQD0oTKlVOxE62nf7mS7FBNg5RvaVbkr3tVQSsXRCc+eEHRkINOTyYqzV3R/heLInj0CHRpSyvZCtQGxaBvWrl3LN7/5TcaPH8/3v/999u3bB8DChQsZM2YM48eP55xzzgHgrbfeYuLEiUycOJFJkyaxf//+qNenLdv2CPbV7MNnfDjElrFQqT7vtpW38WnJp1363Qv+HTxR3GEZh3H1lM6novjJT37Cvffey7Rp07j++uu54YYbuPvuu7n11lvZunUriYmJlJaWAtaKZvfddx9Tp06loqICj8fTpXPoDFu2gpneTBpNI2W1ZfGuilKqjysrK6O0tJRp06YB8NOf/pS3334bgPHjx3PeeefxxBNP4HJZ1+VTp05l3rx5LFy4kNLS0ubtsWTLHkHgQ2X9Pf3jXBulVCx0dOU+btG4kJ89dspj0a5OUK+++ipvv/02r7zyCjfffDPr169n/vz5nHbaaSxdupSpU6fy+uuvc9hhh8W0HrbtEYCmmVBKxV5aWhr9+/fnnXfeAeAf//gH06ZNw+fzsX37dk488URuu+02ysrKqKiooKCggHHjxnH11Vdz1FFH8emnXRve6gxb9giaFrHXKaRK2VemJ/ikkaYRg66qqqpiyJAhze/nzZvHokWLuOSSS6iqquLggw/mscceo7GxkR/96EeUlZVhjOH//u//SE9P57rrrmP58uU4HA6OOOIITj311AOqTyRiGghE5BTgHsAJPGyMubXN58OARUC6v8x8Y8zSWNYJNN+QUoqYTRH1+XxBt3/wwQfttr377rvttt17771Rr1NHYjY0JCJO4D7gVGAMMFtExrQp9jvgOWPMJOAc4K+xqk+g1MRUXOLSoSGllCK29wimAPnGmC3GmDrgGeB7bcoYoGlxzjTg6xjWp5lDHGR4MnRoSCmliG0gyAW2B7zf4d8WaAHwIxHZASwFfh1sRyIyR0RWiciqoqKiqFQuw5uhQ0NK9UG9bdXFaOvK+cd71tBs4HFjzBDgO8A/RNo/4WWMedAYk2eMyWta2/NAZXoyNRAo1cd4PB6Ki4ttGwyMMRQXF3f6IbRY3izeCQwNeD/Evy3QhcApAMaY90XEA2QBhTGsF2BNId1atjXWh1FKdaMhQ4awY8cOojVy0Bt5PJ5Ws5YiEctA8BEwSkRGYAWAc4Bz25TZBnwLeFxEDgc8QLd8gxmeDIprrCsHEemOQyqlYsztdjNixIh4V6PXidnQkDGmAZgLvA5sxpodtFFEbhSRM/zFLgcuEpF1wNPA+aab+nSZnkxqG2uprK/sjsMppVSPFdPnCPzPBCxts+36gNebgKmxrEMogUtW9kvoffnDlVIqWuJ9szhump4u1mcJlFJ2Z9tA0JxvSGcOKaVszr6BwNMyNKSUUnZm20CQ7kkHtEeglFK2DQRuh5v0xHS9R6CUsj3bBgKwhod0aEgpZXe2DgSab0gppWweCEItTKGUUnZi70DgzaSkWoeGlFL2ZutAkOHJYH/9fmoba+NdFaWUihtbB4LmZwm0V6CUsjF7BwKvPlSmlFK2DgSab0gppWweCDTfkFJK2TwQaI9AKaVsHgi8Li9JriTtESilbM3WgQCs4SHtESil7EwDgeYbUkrZnO0DQYZH8w0ppezN9oEg06s9AqWUvWkg8Gayr2YfDb6GeFdFKaXiwvaBIMOTgcFQWlsa76oopVRc2D4QNOUb0vsESim70kDQ9HSxTiFVStmUBgKPJp5TStmb7QNBhtefZkKHhpRSNmX7QJDiTsHtcOvQkFLKtmwfCEREl6xUStma7QMB+J8u1h6BUsqmNBBg3TDWewRKKbvSQICmmVBK2ZsGAqyhoZKaEowx8a6KUkp1u5gGAhE5RUQ+E5F8EZkfoswsEdkkIhtF5KlY1ieUTE8m9b56yuvK43F4pZSKK1esdiwiTuA+YAawA/hIRJYYYzYFlBkFXANMNcbsE5EBsapPOE1PF5fUlJCWmBaPKiilVNzEskcwBcg3xmwxxtQBzwDfa1PmIuA+Y8w+AGNMYQzrE1Lz2sV6w1gpZUOxDAS5wPaA9zv82wKNBkaLyP9E5AMROSXYjkRkjoisEpFVRUVFUa+o5htSStlZvG8Wu4BRwAnAbOAhEUlvW8gY86AxJs8Yk5ednR31Smi+IaWUncUyEOwEhga8H+LfFmgHsMQYU2+M2Qp8jhUYulV6YjoOcejQkFLKlmIZCD4CRonICBFJAM4BlrQpsxirN4CIZGENFW2JYZ2CcjqcpCem69CQUsqWYjZryBjTICJzgdcBJ/CoMWajiNwIrDLGLPF/dpKIbAIagSuNMXFpjTXfkFK9w+I1O7nj9c/4urSawelerjz5UGZOanv7MX56ev2CiVkgADDGLAWWttl2fcBrA8zz/4srzTekVM9vxBav2ck1L62nur4RgJ2l1Vzz0nqAHlHPztYv0r93rL+XiAOBiCQZY6qiduQeJtOTySdFn8S7GkrFRCQNSSwa2c40YJGUveP1T5vr16S6vpE7Xv8s6PlEs5GNpNzt/w5ev1te28wJh2aT4nHjdEjz/iL5e3dH8JOO0iqIyDHAw0A/Y8wwEZkAXGyM+WVUatBJeXl5ZtWqVVHf7+0f3c6Ln7/Ih+d9GPV9KxVPbRsSAK/byY3fO4Jpo7Mpq66nvKaeOX9fTXFlXbvfz0338r/509vts7OBpem4t5w5LsKyDq446VAGpXvZsLOMjV+X89bnoaePjxrQj5w0DzmpHsqq61j+WRH1jS3tW4LLwZzjRnDcqJaZh+98UcSD72ylrsHX6XIuh3DU8P4kup3sLqthd3kNpVX1IevXJCXRRarXzZ7yGhp87dtfj9vBtNEtx37r8yJq6n3tygX7XsIRkdXGmLygn0UQCD4EzsKa3TPJv22DMWZsxDWIolgFgofXP8w9H9/DyvNW4nV5o75/pTojGleplbUNfFFYwfmPrqS0uuMGKpyfTR3BxGHpTBqazqovS/jtyxtCNvDGGIor6zj17ncoqqhtt69+iS5+NnU4bqcDt8uByyHcuyyfsjB1dDmEkQP6sa2kiqq6xnafJyc6OXZkFrvLa9ldVs2e8vbHjQWHwJjBqeSkeslJS2TJ2q8pr2loV65/kptfTx9FeU29FXirG3jx4x0h93tYTkrz60937w9aRoCtt54WcV3DBYKIhoaMMdtFJHBT+2+il2t6lqC4upghKUPiXBvVV0VriMYYw8sf7+C3izc0Xy3uLK3miufX8dA7BZTXNLC9pLrD+tw0cyxpXjepXjdXPLcuaMOd4HTw5Idf8ej/tgJW49f2Qra6vpH5L37CX1fks2NfddDGuklFbQMLl+V3WLcmS+ZOZfTAFDxuZ8hexs0zW/cyRsx/lWCXuAI8+fNvNL8/9+HgIwCRljMG/vXr45rf5x2UEbR+v//uEe2+5w+2FLOztP13lJvu5d+/Ob75/dRblwUtNzg9eheskQSC7f7hISMibuBSYHPUatBDBOYb0kCgYiFUA19b38iEYelsL6lme0kVf/7PZ0HHmS97bi3Xvrye+kZDvc9HsM58g8/w2e4KThmbw6wjhzI6J4Xr/7kh6BVybrqXH33zoOb31552eMihnNPGD+Kz3ftZs72U6xZvCHp+NQ0+hmUkM3VkFsMykrh3WT4lIYaa3r36RBp9pvlcTrrrbXaX1QQtO35IyzOmTY1pR8F0cLo3ZON5zMisVvs/0HKBIq0fwJUnHxr0733lyYd2qdyBiCQQXALcg5UeYifwBvCrqNWghwjsESgVC3e8HryBv9p/td8RY2D2lGG4nA4SnBLyqrrRZ/jLuZNbjlHXGFFD0lEjNjY3jbG5aTywoiDklezDP20ZeeiflBDyuCKCyym4nODFyfxTDou4sZs5KbfDm6TRbmQ70xhHUr+mctBx0OhMcOmqDgOBMWYvcF7UjthDab4hFUsllXVBG88m986exNCMJIb293LGX95lZ2nwq+PfnT6m+f2LH++M+lVqNBvZzh430rKRiHYjG6vGuDNBI5bTYyO5WfwYtB9uM8b8LFaVCidWN4vrGus48okjmTtxLhdPuDjq+1f2VFZdzyPvbOGRd7dSGWLcvO3sj0hn23RmVk609fTnDVR7B3qz+F8Brz3A94Gvo1GxniTBmUCKO0UTz6lOC9YozhgzkMff+5K/vWXduD1t3CDG5aZyz5v5UbuS7o4hg1BifYWqulckQ0MvBr4XkaeBd2NWozjK9Gbq0JBq1tUZPle+sI4Ep4PKuka+ffgALpsxmiMGWwse5aR5ozZE05lySoXTlRQTo4C4rCQWaxmeDL1ZbAPRmMJpjKGitoFbXtvc7gZwfaPBIYaXf3kMk4b1b/WZNtyqJ+owEIjIfqx7BOL/uRu4Osb1iotMbyYFpQXxroaKoXAN/Cljc9hTXsOushpueGVj0Bk+lz+/jgWvbKS8ur7dXPpAdQ2+dkFAqZ4qkqGhlI7K9BUZngxW1qyMdzVUF0VypX9biFww855by2+e7fgYjT7Dd8cPJtXrIs3r5q/LC4I+tRvNh32UirWQgUBEJof6DMAY83H0qxNfmd5MymrLqPfV43a4410d1QnBrvTnv/gJm3aVkeZNYOPXZWzYWc6uIA8tgfWk7BUnjSYnzUtOqod5z62lcH/wh7D+MLMlu8qAFE/MH/ZRKtbC9Qj+HOYzA0Se7aiXaHqobF/NPgYk9cnbIH1WsIe1ahp8PPi2lRbhoMwkxuamUlpVFzQXTG66l7nTWxbH++13gj9leyBz5ZXqqUIGAmPMid1ZkZ4g8OliDQS9hzEm5MNaAqxbcBKpHquHF2ru/YE+DKUNv+rNIpo1JCJjgTFYzxEAYIz5e6wqFS/6dHHvs/HrMm5+NXTqq8Hp3uYgANrAKxVMJLOGfo+1rvAYrNXGTsV6jqDvBQJPS+I51bPtLqvhT298xosf7yDN6+bMSYNZumF3q7ztB5oLRim7iKRHcBYwAVhjjLlARAYCT8S2WvGR4c0ANPFcTxM4GygnzcP43DTe/mIvDT4fPz92BHNPHEVakpvjR2vaA6W6IpJAUGOM8YlIg4ikAoXA0BjXKy6SXEl4nB4NBN2kKw927Sqz5vlPHJLGwtmTGZaZ1FxWr/SV6ppw00fvA54GVopIOvAQsBqoAN7vnup1LxEh05upQ0PdINh0z6te+IT3t+xlWEYyxRV1FFfW8u8Nu6ltaL9MX1FFXasgoJTqunA9gs+BO4DBQCVWUJgBpBpj+uwq7xmeDL1Z3A1ufa39g111jT6e/chavi85wUlmv8SgQQDg6zApnZVSnRNu+ug9wD0ichBwDvAo4AWeFpFqY8wX3VTHbpXpyWRX5a54V6PP2rq3kr+9VcDu8uAPdgmw6cZT8CY4ge5Zpk8pu4skxcRXwG3AbSIyCSsgXA84Y1y3uMj0ZrKxeGO8q9GrBRv7HzmgH/e/VcBr63fhcjpITnAGzc8/ON3bHASge5bpU8ruIpk+6sKaMnoO8C1gBbAgprWKgxOePaHVkNC4ReMAq4ew4uwVcapV7xNs7H/ec2vxGeiX6GLO8Yfws2OH815+sT65q1QPEe5m8QxgNvAdYCXwDDDHGFPZTXXrVqHuC+j9gs4JlurBZyDV4+Kdq6eT5rUe7tIHu5TqOcL1CK4BngIuN8bs66b6qF5sf019yFQP+2samoNAE23gleoZwt0s7nNJ5VRsbN1byaL3vuSF1TtCltGbu0r1XF1ZoUzZUNsbwFfMGE1GSiKP/28ryz8rwu0UTh8/mBFZSdy/Yove3FWqF9FAYHNdXbZx3vPrMEBWv0R+8+1RnPuNYQxIsXISDstI1pu7SvUiGgj8Mj3BF65PcCTEoTbdI9yyjaeOy2FbcRVb9lZy/T83tLsBbID+SW7emz+dBJej1Wc69q9U76KBwC/YFNGH1z/MPR/fw/JtyzlxWN9bniHYDJ+mdXmbpnyGU1pV3y4IKKV6n5j+Xywip4jIZyKSLyLzw5T7gYgYEcmLZX0666dH/JRR/Udx84c3U1nf92bNhkrT0OgzzJ0+irvPnsiSuVMZlOYJWk5vACvVN8QsEIiIE7gP62G0McBsERkTpFwKcCnwYazq0lVuh5vfH/17CqsKuXfNvfGuTtT4fIaX1+zAIcE/z033Mm/GaGZOymX8kHSuPuUwvO7WD5LrDWCl+o5YDg1NAfKNMVsAROQZ4HvApjbl/oCVwuLKGNYlIsFvnE5g1qGzeGrzU5x+8OmMzRrb8Y463Gf8xs9Xbi3hplc38cmOMoakeyisqKOuIfxiLvp0r1J9WywDQS6wPeD9DuAbgQVEZDIw1BjzqojENRCEu3F66eRLWb5tOTe8fwNPn/Y0Lkdkf7Zw+4x1I9o2AJ1/zHBWfVXC6xv3MCjNw52zJjBzYi5L1n2tT/cqZXNxu1ksIg7gTuD8CMrOAeYADBs2LCb1CXXj9KZXN3HK2Olc841ruGzFZTyx6QnOH3t+h1f6xhhueW1z0H3e8fpnXW5Uuzrd8+alm0lwClecNJoLjz24ObGbNvBKqVgGgp20XslsiH9bkxRgLLBCRABygCUicoYxZlXgjowxDwIPAuTl5XUwl6VrQt043VtRx4Qb3uDI4ekMTz2Ke9f8hcb9R/CnpcWtGtr5L33C53v2k5zoYu32UtZuL6Vof23Qfe4sreb5Vds5ZmQWuf4brl1t4Jt6GN+dMJg95TVsL6liwSsb2wUggIzkROZOH9W1P5BSqs8SY2LSrjZlLf0cK2PpTuAj4FxjTNAczyKyAriibRBoKy8vz6xaFbZIl4TKe5+ZnMDMSbm8V1DMp0XbSD74Thqrh1O9/QKs7PntjchKZuLQdJZ9WkhZdX27zx1C89TMgzKTGJzmYdVX+6hvbPkuPG4HV518KMeNyqaqrpGqukbmPvUxxZV17fbndAgOodXvByPA1ltPC1tGKdU3ichqY0zQmZkx6xEYYxpEZC7wOtbaBY8aYzaKyI3AKmPMklgduytC5b2/7vQxzVfmxRW1/PnDMl7ZcT+u1E9oKJ/Qbj9rr59BepL1EFrbK/imff5x5lgOz03lvfxi3iso5s3Ne2jbhNfU+7jxX5uBzR3WvdFnuGjaIQzN8DK0fxJXvrCOPeXteyM63VMpFUzMegSxEqseAVgNd9ODVLkhhmcafY1M/PtkkPZLKEpjCp/87L12++xoyGfE/FfbBYIm986eRFKCE2+Ck0ufXktRRfsGPjfdy//mt+QIDBWAbjlznN4PUMqm4tIj6I2OG5WFz8DvTjucnx93cNAyToczaBAAMM79rd43L3YzCPoNgnLguk/g7s9bL3YzON0bdFgqN93LdycMbn5/7WmH62IuSqmo00AQIL+wAoCRA/p1eR+flnzK0JShJLuTI17sxjd0ASmD2i/54HP1B1qu9HUxF6VULGggCJBfdOCB4Iev/BCALG9W2HKlNaWkJKTgdDipaAi+7k/b7ZH2MNouu9lEl91USgWjgSBAfmEFXreTwWldv6n652l/Ztv+bWwr38bL+S+HLHfcs8chCGmJaWH398/8f5KakEpKQkrEPYxIy0UaMDSwKNW3aSAIUFBUySEDknGESsITgZOGn9T8OlwgmD9lPvtq9lFaW8qznz0bstzv/ve7iI573qvnkeROop87fG/mo90fkexOpp+7X9QDC2jQUKo30kAQoKCwgqOG9++wXKi1CzI9mREf67zDz2t+HS4QLP3+Usrry9lft5+L3rgoZLl+Cf2orK9kb/XesMf92es/i6h+81bMIzUhldSE1LDljDH4HwgEot8bUUrFngYCv8raBnaWVnNO9tAOy0baUEUjYAxN7bg+AH+b8bfm1+MWjQtZ7pGTHqGivoLK+kp+++5vQ5bLL82nvLac8rrysMc96smjSE9Mp7+nf4fDXA2+huY8TRowlOo5NBD4bSmy1hs4kBvFbXVnwIjUlEFTml+HCwRLZrY87xcusJx72Lnsq91HaU0ppbWlYY89+R+T6e/pz4CkAWHLBfYyOjMspZTqGg0EfvlF1jMA0QwEkYp2wOjOwDIvb16r9+GCxiUTLqGwqpC91Xv5tOTTkOWmPDmFgckDyUnKibge2nNQqus0EPgVFFbidAgHZSbHuyohRdqg9dTA8suJv2x+HS5gzDp0Fnuq9rC7cnfY/c1cPJOc5BxyknN0qMkuHjgWdq9vvz1nHFzybvfXp4/QQOCXX1jBQZlJtlqDN9qBBaITNK48qmVpinABY3jacHZV7mJzSfh8TFe+dSWDkgcxqN8gewaMSBvPeJXrjCFToOgzaAxIvuhMsLZ3RTzPOZ5/xzY0EPjlF1VwSHb3Dwv1Nd15Xw7+qXIAABcSSURBVOTuE+9ufh0uYGwq3sSb296k3tc+E2yg+9fez8DkgQxMGti3AkakjWe8ykHkjd03fwFr/tG6jAhMu6pr+4vnOcdin12kgQCob/Tx5d5KZowZGO+q2EZ3BoxXz3wVn/FRXF3M9Oenhyx3/7r7MSHT/7V4avNTDEwayICkAfF9xqKjxs7ng31brfemzfoUvgbY9yU8fwG4POBKBF9j+3LGB64EePPGls+Nz3rddn+7N8CDJ0BtBdRVQE1568YLoLEeqkrgnTsh+1DIOhT6Dw/R2LkhbQj87x7YtQ6+XgslBe3Pt6EWFk6GjIMhYwRkHgKeNHC4rHo1cbggeQB89AhUFUPlXti/q3WZpjruXg9P/KDlb2NMkHNuhLLt8MRZUF8FdZVQuz/IOdfBxpdg8yvgcII4rZ8Y61ht/44lW+H588Hl9R/b1/57EQdMu7r936KLNBAAXxVX0eAzjNQeQY8TratqhzjITsoOW2b1j1ZTVF3Enqo9/OS1n4Qsd8vKWyI65qKNi8jwZDT/i7iX8fh4iqV9QMo0worzP2kpl1RN8Yj2K/Zl+spY8cjJsGeD1SC3I5CUCRWFULrNakgbqq2fvjbH9TXABw/4GzBHSyMmjtaNkyfdujJPyoT0gyCxHySkwLb3rUbVNFq/02+gtW1TwMOWDrf1O8Ea5M9es/6lDYVBE2DCbEgfBkt+DY211pXxCfOtRr24AAo3W+WD9f58DVDwpvUPIDENkjMhKQsqiwBj/W1SB1tBqHqf/29TY/10uqEh4JzdXuvv506y/vUbYAU1ESjZYjXg4oDsw2DY0dbfwBcQSH0N1t9z35ctx07KgqoiK8A01EB9tfXTF5Do0pkAE8+DlOhduGogAAqikGNIxVc0eg5up5vB/QYzuN/gsOWWz1pOYVUhhVWF/HrZr0OW+9OqP0V87Fs+vIX0xDRSXclBgwBgbX/rDqgugaoSik37RYoAih2AD5h4LuSM44TPHqS4rqxduUyPixVnt6R071SPZf9uuGeC1Ui5PPDLD4I3TM3lGq0GbM5bVrmaMtj7hdUL2Pu59a+yEGqbnlsRyD0STvytFQCS2+Tu2v4hrH4MJv0Yjru89WeNDVZD+trVkP9fqwF2OGHkDJj+O6uxTcq0ejrtziURLlrewbn4z/nXH3dczpkAP14cutFue+xL3u14n1HuDYAGAqAl6+ghGgh6re4casryZpHlzWJM5piw5d47+UlKSj5n376tFJdv5zeFy0OWfWXTE+x3dDxR4eQvHiXVQKq4wB263PunLmhJJbL+jqBlDjhH1dCA50Fe+nboHFWhyg3Js/41lR2cDqQHHKmQzNV/CP7dTrsKijYHbxCdLmuI6IyFLUHI4YbvLgzeyKbkWFfYqx8Lf6Ud7XKx2mcXaCDASi2Rk+qhX6L+Ofq6FfsaYPe29h/ktH4qOtNIyOGZZtXBs8Y2SXngOFKAg5o2BBnGafLekLNocLopdwjTvnomZLkjR59BeX2l9cR34ZqQ5eb8Z07YujX58dIf43V5SXInhS3336/+S5IriSR3UkxyVHXp5rwDeOnbQJhEiZEGK4DhQ6D4P7BoXOj7NuECUFfKxWqfnaQtH9aMIR0W6uWiPEtkRea3YO0TrcuJE7JGweOnW/uoLCRzaC7FLme7w2Y6PDDzAUjLhdRca9z5qTCzPGbcgAvIAFgUOhD88fjbml+Hmyn1+CmPU1lfSWV9JVe9fVXIcomuRCobKimqLgpdN+CyFZeF/bzJic+dSKIzEY/TE7bcbStvI9GZaP1zJYYtu2Hvhuay0Q5CMQtAHZSL1T67yvaBwBhDQWEFP8yLLKeP6qHCNfB1ldZMjJIt4O3ffvZHYx1sXAwbXrBuUDbWtb9xCdZYc/lOSEyF0SdB1qGsyBoNSf1h0RktY8eXfhK0696dT3wfOfDI5tfhAsHDJz3c/DpcYHnhuy9Q1VBFVX0Vl/z3kpDlpg2ZRm1jLTUNNRSUBZnh4/dy/svUNtbSEOzv3MbsV2d3WAasIJTgSCDBmRC23LXvXovb4cbtCDO2BizOX4zb4SbBmRA2YGwt24rL4cLtcMekF9QdaVZsHwh2l9dQWdeo9wd6qkiv9I+/EtY80bqMrwE2LYFVj7Te7kyERh/WTA0HZI6EEcdagcPp9v9MgC/esKYsmkZr6uG4WTDzr9askLYiGL9tvnrbvxteuADOevyAAkZ3BpZDMw7tuBCw4JgFza/DBZYPzv0AsNYAr/PVMeXJ0L2lv0z/C7WNtdQ21obNjzVtyDTqffXUNdbxZfmXIcut2r3KKucLfrO9yXX/uy7s503OWHxGROWmPDkFl7hwOVzNyRdDOf/f51vlpHuaaNsHgublKXXqaPc60KGcrMNg/QvW/PLdn1g/G2tb7ys525opkjHCP8fcP8+8vjpgpkYCnP+v4I33kecH3Gx0wbcXBA8C0Lnx25QcuOC1kB/39lQineF0OPE6wi8ENW3otObX4QJBYBD695f/Dlnu9bNeb34dLli9duZr1PvqqffV84MlPwhZ7pbjbqHB10CDr4Eb3r8hZLlZo2fRYBqay774xYshywpCXWMdVb6qkGWiSQNB84yhnptjqE/qaKzeGOtm7OFnwJq/t/7dxjrY8Lz1z5kAA8bA4d+F/iNgxS3W5y4PXPxO8Abekxab2R9hGvd46g2BJR5BqCNDUoZEVO70g09vfh0uEFxx1BWt3ocLBI+d8ljz63DBKlo0EBRWkOpxkd0v/A0rFaEDGspptJ5OXTgZyr+2HnJqR2DA4XD0ryBnvPWwjitgTLhsR2QNdw+YqdFbxSJHVbyCUE8MQPGggaDQmjEkobr8qnNCXelnHGw1/Ls3WE+87tnQfijH7bWGXgaNh0NPtWbapA62ru6f+6lV3pUY/gGdSBvuSK/ge/CVvh1FOwj1hl5QdwQrMabj3Co9SV5enlm1alXHBSPd303/Zfph2dx+1oSo7dPWyr6GhRPa51tp4k6yhnJyxkLaMHjr1pahnBCzbQD41zzrSv/IC+D0O2NXf6X6KBFZbYzJC/aZrXsEZVX17K2o1WcIOtLRcE9FIRQsg/w3YcvyNkHAf4V/7GUwcJx1s9YRMO++fGd0h3KUUp1m60DQtCqZpp/uQLDhHocLkNZBIikTDpkOuXnwn+tbhnLOfb77hnKUUp1m70BQaPNkc5Hc2K2rgsNOg4/bzNzxNcCejTDsmzD9Ohj5LciZAE35cvZ+3utn2yhlF7YPBAkuB0P6h8+z0mcFu9IXpzV184kfQNHnUBYkL484rCv/sx4DT2rwfetQjlK9hq0DQUFRJQdnJeN0SN9aCzWSc9n3FWSNbp9uwTTC3nxr9s7QKTD5x1Y5bwY89cOW1Lrf+2voIAB6pa9UL2LrQJBfWMH4If6sk7FYVi/aweVAnsZ1uK2FOF6aA1+9Z+VrB+scm9ItOFww9kyY+beWIZ5AMUyDq5SKH9sGgpr6Rrbvq+LMybnWhmlXwdonWxfyNVpXxh/+rWVOe2ou5B4V3bVGD7SBzxwF2z6wlgasLYf0odYqSK3OpR6+etdKu3DQVJh6KRx0DHj6w72TrCt9hwtm3BQ8CDT9jXS4R6k+x7aBYEtRJcYEzBhqTifwuH8JPrEecPr47+3nxIuz/RqijfVWY/zAcf41Rn3W8nbB1iSt2w//XWBlwvSkW41z2/VVxWmtWfqf31vrq1YVW0/btttfvbUe6saXwpytAw46Gr57j5Vgre3Dc30gjYJSqutiGghE5BTgHsAJPGyMubXN5/OAnwMNQBHwM2PMV7GsU5P8YMtTTrsKVj1qvXYlWkvR9Rvgb4R3Wg1x089N/7TWSA1c5zQtt2VdVxFrvrzxWePx+KxyCSlQsNzKoxMuBa9phB0r4es11jJ9SVmQlAEZw639GZ91nIOOhqMugsQUK4dOYqr1uqEa/np0S2K1sx478CmcSqk+KWaBQEScwH3ADGAH8JGILDHGbAootgbIM8ZUicgvgNuBs2NVp0AFhRU4BEZkBSSbcybQ3LAHXh0nZ1n/BgU8fTxlThfWOU2EuR9Z5Yyx8uRX77P+rfgjfPEfKzg4XHDEmXDan61GPfAKvtV6qG74waOhG3i90ldKRaDjRVK7bgqQb4zZYoypA54BvhdYwBiz3BjTlGf1AyCydH9RkF9UwdCMJDzugKdct75l/cwZG9kDThPPs3oAkWSwbFtOBBL7WeP5g8bD6Xf7H9LC+nnSTdasnLbDOJEeF6wr/WHf1Ct9pVRYsQwEucD2gPc7/NtCuRAIelkqInNEZJWIrCoqCr+kXqQKCivar0FQsMyaWXPRishmxUTa0EZSLhYNfNOVvs7wUUqF0SNuFovIj4A8YFqwz40xDwIPgpV07kCP1+gzbNlbyfGjswMPYo3dH3w8OCP8s0Q7g6WmW1BKxUEsewQ7gcCFgIf4t7UiIt8GrgXOMMbUtv08FraXVFHX4GvdIyjOt+bWHzK9O6oQnF7BK6XiIJaB4CNglIiMEJEE4BxgSWABEZkE/A0rCBTGsC6tFBQ1rUoWEAjy37R+xjMQKKVUHMQsEBhjGoC5wOvAZuA5Y8xGEblRRJpWe74D6Ac8LyJrRWRJiN1FVdBkcwXLIOMQ6D+8O6qglFI9RkzvERhjlgJL22y7PuD1t2N5/FDyCyvITkkkzeu2NjTUwpfvWDdplVLKZmI5NNRj5RdVcEh2wPMD21dCfZUOCymlbMl2gcAY07xOcbOCZdbc/eHHxq9iSikVJ7YLBEUVteyvaWg9Y6hgmZXQLVxaZaWU6qNsFwhabhSnWBsq98KudTospJSyLdsFgoK2M4a2rACMBgKllG3ZLhDkF1bQL9HFwNREa0PBMisd9OCJ8a2YUkrFif0CgX/GkIj400osg4NPsFJGK6WUDdkvEBRWtDxRXPQp7N+lw0JKKVuzVSDYX1PPnvLalvsDBcusnwefGL9KKaVUnNkqEBQUVQK0TB0tWAZZo601AZRSyqZsFQha5Riqr4Ev/6fDQkop27NdIHA7hWEZSbD9A2td30O+Fe9qKaVUXNkuEAzPTMbldFhppx1uGD413tVSSqm4slUg2FIUkGOoYLm13GNCcvhfUkqpPs4WgWDxmp0cc8ubbNlbybtf7OW199fCnvV6f0AppbBBIFi8ZifXvLSer8tqANhf28Dy1563PtRAoJRSfT8Q3PH6Z1TXN7ba9g2zln2kQs74ONVKKaV6jj4fCL4urW6zxXC8Yz1vN44FR58/faWU6lCfbwkHp3tbvT9MtpMtZaz3HBmnGimlVM/S5wPBlScfitfdklDueMc6APJO/EG8qqSUUj1KTBev7wlmTsoFrHsFX5dWMyNxE2XJozjlmElxrplSSvUMfT4QgBUMZk7KhboquO0CGHtRvKuklFI9Rp8fGmpl23vQWAuHaLZRpZRqYq9AULAcnIkw7Jh410QppXqMvj809MCxsHt9621/HAQ54+CSd+NTJ6WU6kH6fo9gyBRwJrTe5kywtiullLJBIJh2FUib0xQHTLs6PvVRSqkepu8HgpQcmHgeOPyjYM4E633KwPjWSymleoi+HwjA6hU0BQLtDSilVCv2CARNvQJxaG9AKaXa6PuzhppMuwqKNmtvQCml2rBPIEjJgQtei3ctlFKqx4np0JCInCIin4lIvojMD/J5oog86//8QxEZHsv6KKWUai9mgUBEnMB9wKnAGGC2iIxpU+xCYJ8xZiRwF3BbrOqjlFIquFj2CKYA+caYLcaYOuAZ4HttynwPWOR//QLwLRGRGNZJKaVUG7EMBLnA9oD3O/zbgpYxxjQAZUBm2x2JyBwRWSUiq4qKimJUXaWUsqdeMX3UGPOgMSbPGJOXnZ0d7+oopVSfEstZQzuBoQHvh/i3BSuzQ0RcQBpQHG6nq1ev3isiX3WxTlnA3i7+bk+j59Lz9JXzAD2XnupAzuWgUB/EMhB8BIwSkRFYDf45wLltyiwBfgq8D5wFLDPGmHA7NcZ0uUsgIquMMXld/f2eRM+l5+kr5wF6Lj1VrM4lZoHAGNMgInOB1wEn8KgxZqOI3AisMsYsAR4B/iEi+UAJVrBQSinVjWL6QJkxZimwtM226wNe1wA/jGUdlFJKhdcrbhZH0YPxrkAU6bn0PH3lPEDPpaeKyblIB0PySiml+ji79QiUUkq1oYFAKaVszjaBoKMEeL2JiHwpIutFZK2IrIp3fTpDRB4VkUIR2RCwLUNE/iMiX/h/9o9nHSMR4jwWiMhO//eyVkS+E886RkpEhorIchHZJCIbReRS//Ze9b2EOY9e972IiEdEVorIOv+53ODfPsKfoDPfn7AzoaN9RXQ8O9wj8CfA+xyYgZXq4iNgtjFmU1wr1kUi8iWQZ4zpdQ/JiMjxQAXwd2PMWP+224ESY8yt/iDd3xjToxeOCHEeC4AKY8yf4lm3zhKRQcAgY8zHIpICrAZmAufTi76XMOcxi172vfhzriUbYypExA28C1wKzANeMsY8IyIPAOuMMfcf6PHs0iOIJAGe6gbGmLexnhkJFJh8cBHW/7w9Wojz6JWMMbuMMR/7X+8HNmPlAetV30uY8+h1jKXC/9bt/2eA6VgJOiGK34ldAkEkCfB6EwO8ISKrRWROvCsTBQONMbv8r3cDvXkt0bki8ol/6KhHD6UE418TZBLwIb34e2lzHtALvxcRcYrIWqAQ+A9QAJT6E3RCFNsxuwSCvuZYY8xkrLUefuUfpugT/ClGeut45f3AIcBEYBfw5/hWp3NEpB/wIvAbY0x54Ge96XsJch698nsxxjQaYyZi5WmbAhwWq2PZJRBEkgCv1zDG7PT/LARexvqPpDfb4x/fbRrnLYxzfbrEGLPH/z+vD3iIXvS9+MehXwSeNMa85N/c676XYOfRm78XAGNMKbAcOBpI9yfohCi2Y3YJBM0J8Px32c/BSnjX64hIsv9GGCKSDJwEbAj/Wz1eU/JB/D//Gce6dFlTo+n3fXrJ9+K/MfkIsNkYc2fAR73qewl1Hr3xexGRbBFJ97/2Yk102YwVEM7yF4vad2KLWUMA/iljd9OSAO/mOFepS0TkYKxeAFi5op7qTeciIk8DJ2Cl090D/B5YDDwHDAO+AmYZY3r0jdgQ53EC1vCDAb4ELg4YY++xRORY4B1gPeDzb/4t1vh6r/lewpzHbHrZ9yIi47FuBjuxLtifM8bc6P///xkgA1gD/MgYU3vAx7NLIFBKKRWcXYaGlFJKhaCBQCmlbE4DgVJK2ZwGAqWUsjkNBEopZXMaCJTyE5HGgAyVa6OZpVZEhgdmKlWqJ4npmsVK9TLV/kf6lbIV7REo1QH/+g+3+9eAWCkiI/3bh4vIMn8yszdFZJh/+0ARedmfS36diBzj35VTRB7y55d/w//EKCLyf/4c+p+IyDNxOk1lYxoIlGrhbTM0dHbAZ2XGmHHAX7CeUAe4F1hkjBkPPAks9G9fCLxljJkATAY2+rePAu4zxhwBlAI/8G+fD0zy7+eSWJ2cUqHok8VK+YlIhTGmX5DtXwLTjTFb/EnNdhtjMkVkL9ZCKPX+7buMMVkiUgQMCXz0358W+T/GmFH+91cDbmPMTSLyb6xFbhYDiwPy0CvVLbRHoFRkTIjXnRGYE6aRlnt0pwH3YfUePgrILqlUt9BAoFRkzg74+b7/9XtYmWwBzsNKeAbwJvALaF5cJC3UTkXEAQw1xiwHrgbSgHa9EqViSa88lGrh9a8I1eTfxpimKaT9ReQTrKv62f5tvwYeE5ErgSLgAv/2S4EHReRCrCv/X2AtiBKME3jCHywEWOjPP69Ut9F7BEp1wH+PIM8YszfedVEqFnRoSCmlbE57BEopZXPaI1BKKZvTQKCUUjangUAppWxOA4FSStmcBgKllLK5/weA/uHgFsh5DAAAAABJRU5ErkJggg==\n",
"text/plain": [
""
]
},
"metadata": {
"tags": [],
"needs_background": "light"
}
}
]
}
]
}