{
"cells": [
{
"cell_type": "raw",
"metadata": {},
"source": [
"%env MKL_NUM_THREADS=12\n",
"%env OMP_NUM_THREADS=12"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"from collections import defaultdict\n",
"\n",
"import numpy as np\n",
"import scipy as sp\n",
"import pandas as pd\n",
"from ipypb import track\n",
"\n",
"from polara.evaluation import evaluation_engine as ee\n",
"from polara.evaluation.pipelines import random_grid, find_optimal_config\n",
"\n",
"from lce import LCEModel, LCEModelItemColdStart\n",
"from data_preprocessing import (get_yahoo_music_data,\n",
" get_similarity_data,\n",
" prepare_data_model,\n",
" prepare_cold_start_data_model)\n",
"from utils import (report_results, save_results,\n",
" apply_config, print_data_stats,\n",
" save_training_time, save_cv_training_time)\n",
"\n",
"%matplotlib inline"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"from polara.recommender import defaults\n",
"defaults.memory_hard_limit = 15 # allowed memory usage during recommendations generationa\n",
"defaults.max_test_workers = 6 # use this manyparallel thread for evaluation each using up to {memory_hard_limit} Gb of RAM"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"seed = 42"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"experiment_name = 'lce'"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Experiment setup"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"data_labels = ['YaMus']"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"init_config = dict(seed = seed,\n",
" max_iterations = 75,\n",
" alpha = 0.1,\n",
" beta = 0.05,\n",
" max_neighbours=10,\n",
" )\n",
"lce_init_config = dict.fromkeys(data_labels, {'LCE': init_config, # standard scenario\n",
" 'LCE(cs)': init_config}) # cold start"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"params = {\n",
" 'regularization': [1, 3, 10, 30],\n",
" 'rank': [100] # for initial tuning (exploration)\n",
" }"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"coeffs = {\n",
" 'alpha': [0.1, 0.3, 0.5, 0.7, 0.9],\n",
" 'beta': [0, 0.05, 0.1, 0.3]\n",
"}"
]
},
{
"cell_type": "code",
"execution_count": 38,
"metadata": {},
"outputs": [],
"source": [
"ranks_grid = [1, 50, 100, 500, 750, 1000, 1500, 2000, 2500, 3000]"
]
},
{
"cell_type": "code",
"execution_count": 39,
"metadata": {},
"outputs": [],
"source": [
"lce_ranks = {'YaMus': ranks_grid}"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
"topk_values = [1, 3, 10, 20, 30]"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [],
"source": [
"target_metric = 'mrr'"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [],
"source": [
"data_dict = dict.fromkeys(data_labels)\n",
"meta_dict = dict.fromkeys(data_labels)\n",
"similarities = dict.fromkeys(data_labels)\n",
"feature_idx = dict.fromkeys(data_labels)\n",
"sim_indices = dict.fromkeys(data_labels)"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [],
"source": [
"all_data = [data_dict, similarities, sim_indices, meta_dict]"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Yahoo Music"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [],
"source": [
"lbl = 'YaMus'"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [],
"source": [
"data_dict[lbl], meta_dict[lbl] = get_yahoo_music_data('/data/recsys/yahoo_music/yamus_train0_rating5.gz',\n",
" meta_path='/data/recsys/yahoo_music/yamus_attrs.gz',\n",
" implicit=True,\n",
" pcore=5,\n",
" filter_data={'genreid': [0]}, # filter unknown genre\n",
" filter_no_meta=True)"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [],
"source": [
"similarities[lbl], sim_indices[lbl], feature_idx[lbl] = get_similarity_data(meta_dict[lbl])"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"0.0"
]
},
"execution_count": 18,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"(meta_dict[lbl].applymap(len).sum(axis=1)==0).mean()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Data stats"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"YaMus\n",
"{'userid': 183003, 'songid': 134059}\n",
"density 0.09740952587383789\n",
"similarity matrix density 0.4576464914574314\n"
]
}
],
"source": [
"print_data_stats(data_labels, all_data)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Standard experiment"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {},
"outputs": [],
"source": [
"def prepare_recommender_models(data_label, data_models, config):\n",
" data_model = data_models[data_label]\n",
" lce = LCEModel(data_model, item_features=meta_dict[data_label])\n",
" lce.method = 'LCE'\n",
" models = [lce]\n",
" apply_config(models, config, data_label)\n",
" return models\n",
"\n",
"def fine_tune_lce(model, params, label, ntrials=60, record_time_as=None):\n",
" param_grid, param_names = random_grid(params, n=ntrials)\n",
" best_lce_config, lce_scores = find_optimal_config(model, param_grid, param_names,\n",
" target_metric,\n",
" return_scores=True,\n",
" force_build=True,\n",
" iterator=lambda x: track(x, label=label))\n",
" model_config = {model.method: dict(zip(param_names, best_lce_config))}\n",
" model_scores = {model.method: lce_scores}\n",
" try:\n",
" if record_time_as:\n",
" save_training_time(f'{experiment_name}_{record_time_as}', model, lce_scores.index, label)\n",
" finally:\n",
" return model_config, model_scores"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## tuning"
]
},
{
"cell_type": "code",
"execution_count": 21,
"metadata": {},
"outputs": [],
"source": [
"config = {}\n",
"scores = {}\n",
"data_models = {}"
]
},
{
"cell_type": "code",
"execution_count": 22,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'seed': 42,\n",
" 'max_iterations': 75,\n",
" 'alpha': 0.1,\n",
" 'beta': 0.05,\n",
" 'max_neighbours': 10}"
]
},
"execution_count": 22,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"lce_init_config['YaMus']['LCE']"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### regularization"
]
},
{
"cell_type": "code",
"execution_count": 23,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div><span class=\"Text-label\" style=\"display:inline-block; overflow:hidden; white-space:nowrap; text-overflow:ellipsis; min-width:0; max-width:15ex; vertical-align:middle; text-align:right\"></span>\n",
"<progress style=\"width:60ex\" max=\"1\" value=\"1\" class=\"Progress-main\"/></progress>\n",
"<span class=\"Progress-label\"><strong>100%</strong></span>\n",
"<span class=\"Iteration-label\">1/1</span>\n",
"<span class=\"Time-label\">[47:14<47:14, 2833.77s/it]</span></div>"
],
"text/plain": [
"\u001b[A\u001b[A\u001b[2K\r",
" [████████████████████████████████████████████████████████████] 1/1 [47:14<47:14, 2833.77s/it]\u001b[B"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"<div><span class=\"Text-label\" style=\"display:inline-block; overflow:hidden; white-space:nowrap; text-overflow:ellipsis; min-width:15ex; max-width:15ex; vertical-align:middle; text-align:right\">YaMus</span>\n",
"<progress style=\"width:45ex\" max=\"4\" value=\"4\" class=\"Progress-main\"/></progress>\n",
"<span class=\"Progress-label\"><strong>100%</strong></span>\n",
"<span class=\"Iteration-label\">4/4</span>\n",
"<span class=\"Time-label\">[46:45<12:04, 701.19s/it]</span></div>"
],
"text/plain": [
"\u001b[A\u001b[2K\r",
" YaMus [█████████████████████████████████████████████] 4/4 [46:45<12:04, 701.19s/it]"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"_config = {}\n",
"_scores = {}\n",
"for label in track(data_labels):\n",
" data_models[label] = prepare_data_model(label, *all_data, seed)\n",
" model, = prepare_recommender_models(label, data_models, lce_init_config)\n",
" _config[label], _ = fine_tune_lce(model, params, label)\n",
"del model"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'YaMus': {'LCE': {'regularization': 10, 'rank': 40}}}"
]
},
"execution_count": 24,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"_config # will also reuse it in coldstart"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### $\\alpha, \\beta$"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div><span class=\"Text-label\" style=\"display:inline-block; overflow:hidden; white-space:nowrap; text-overflow:ellipsis; min-width:0; max-width:15ex; vertical-align:middle; text-align:right\"></span>\n",
"<progress style=\"width:60ex\" max=\"1\" value=\"0\" class=\"Progress-main\"/></progress>\n",
"<span class=\"Progress-label\"><strong>0%</strong></span>\n",
"<span class=\"Iteration-label\">0/1</span>\n",
"<span class=\"Time-label\">[0<0, 0.00s/it]</span></div>"
],
"text/plain": [
"\u001b[2K\r",
" [############################################################] 0/1 [0<0, 0.00s/it]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"<div><span class=\"Text-label\" style=\"display:inline-block; overflow:hidden; white-space:nowrap; text-overflow:ellipsis; min-width:15ex; max-width:15ex; vertical-align:middle; text-align:right\">YaMus</span>\n",
"<progress style=\"width:45ex\" max=\"20\" value=\"2\" class=\"Progress-main\"/></progress>\n",
"<span class=\"Progress-label\"><strong>10%</strong></span>\n",
"<span class=\"Iteration-label\">2/20</span>\n",
"<span class=\"Time-label\">[23:48<11:39, 713.76s/it]</span></div>"
],
"text/plain": [
"\u001b[A\u001b[2K\r",
" YaMus [████#########################################] 2/20 [23:48<11:39, 713.76s/it]"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"for label in track(data_labels):\n",
" data_models[label] = prepare_data_model(label, *all_data, seed)\n",
" model, = prepare_recommender_models(label, data_models, [lce_init_config, _config])\n",
" config[label], scores[label] = fine_tune_lce(model, coeffs, label)\n",
" # make sure to save all parameters\n",
" config[label][model.method].update(_config[label][model.method])\n",
"del model"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"report_results('tuning', scores);"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"config"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### saving data"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"save_results(f'{experiment_name}_param', config=config, tuning=scores)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## rank estimation"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"rank_config = {}\n",
"rank_scores = {}"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"for label in track(data_labels):\n",
" model, = prepare_recommender_models(label, data_models,\n",
" [lce_init_config, config]) # initiate with optimal config\n",
" rank_config[label], rank_scores[label] = fine_tune_lce(model, {'rank': lce_ranks[label]},\n",
" label, ntrials=0, record_time_as='rank')\n",
"del model"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"report_results('rank', {lbl: v.sort_index() for lbl, scr in rank_scores.items() for k, v in scr.items()});"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"rank_config"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### saving data"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"save_results(f'{experiment_name}_rank', config=rank_config, tuning=rank_scores)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## cross-validation"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"result = {}\n",
"for label in track(data_labels):\n",
" models = prepare_recommender_models(label, data_models, [lce_init_config, config, rank_config])\n",
" result[label] = ee.run_cv_experiment(models,\n",
" fold_experiment=ee.topk_test,\n",
" topk_list=topk_values,\n",
" ignore_feedback=True,\n",
" iterator=lambda x: track(x, label=label))\n",
" save_cv_training_time(experiment_name, models, label)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"report_results('topn', result, target_metric);"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"pd.concat({lbl: res.mean(level='top-n').loc[10, :'ranking'] for lbl, res in result.items()}, axis=1)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### saving data"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"save_results(experiment_name, cv=result)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Cold start"
]
},
{
"cell_type": "raw",
"metadata": {},
"source": [
"import gc\n",
"del data_models, models\n",
"gc.collect()"
]
},
{
"cell_type": "code",
"execution_count": 40,
"metadata": {},
"outputs": [],
"source": [
"def prepare_cold_start_recommender_models(data_label, data_models, config):\n",
" data_model = data_models[data_label]\n",
" lce = LCEModelItemColdStart(data_model, item_features=meta_dict[data_label])\n",
" lce.method = 'LCE(cs)'\n",
" models = [lce]\n",
" apply_config(models, config, data_label)\n",
" return models"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## tuning"
]
},
{
"cell_type": "code",
"execution_count": 41,
"metadata": {},
"outputs": [],
"source": [
"config_cold = {}\n",
"scores_cold = {}\n",
"data_models_cold = {}"
]
},
{
"cell_type": "code",
"execution_count": 42,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'seed': 42,\n",
" 'max_iterations': 75,\n",
" 'alpha': 0.1,\n",
" 'beta': 0.05,\n",
" 'max_neighbours': 10}"
]
},
"execution_count": 42,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"lce_init_config['YaMus']['LCE(cs)']"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### $\\alpha, \\beta$"
]
},
{
"cell_type": "code",
"execution_count": 43,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div><span class=\"Text-label\" style=\"display:inline-block; overflow:hidden; white-space:nowrap; text-overflow:ellipsis; min-width:0; max-width:15ex; vertical-align:middle; text-align:right\"></span>\n",
"<progress style=\"width:60ex\" max=\"1\" value=\"1\" class=\"Progress-main\"/></progress>\n",
"<span class=\"Progress-label\"><strong>100%</strong></span>\n",
"<span class=\"Iteration-label\">1/1</span>\n",
"<span class=\"Time-label\">[03:58:60<03:58:60, 14339.53s/it]</span></div>"
],
"text/plain": [
"\u001b[A\u001b[A\u001b[2K\r",
" [████████████████████████████████████████████████████████████] 1/1 [03:58:60<03:58:60, 14339.53s/it]\u001b[B"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"<div><span class=\"Text-label\" style=\"display:inline-block; overflow:hidden; white-space:nowrap; text-overflow:ellipsis; min-width:15ex; max-width:15ex; vertical-align:middle; text-align:right\">YaMus</span>\n",
"<progress style=\"width:45ex\" max=\"20\" value=\"20\" class=\"Progress-main\"/></progress>\n",
"<span class=\"Progress-label\"><strong>100%</strong></span>\n",
"<span class=\"Iteration-label\">20/20</span>\n",
"<span class=\"Time-label\">[03:58:34<12:07, 715.69s/it]</span></div>"
],
"text/plain": [
"\u001b[A\u001b[2K\r",
" YaMus [█████████████████████████████████████████████] 20/20 [03:58:34<12:07, 715.69s/it]"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"_config_cold = {}\n",
"for label in track(data_labels):\n",
" # reuse regularization param from standard scenario\n",
" _config_cold[label] = {f'{k}(cs)' if k=='LCE' else k: v for k, v in _config[label].items()}\n",
" data_models_cold[label] = prepare_cold_start_data_model(label, *all_data, seed)\n",
" model, = prepare_cold_start_recommender_models(label, data_models_cold, [lce_init_config, _config_cold])\n",
" config_cold[label], scores_cold[label] = fine_tune_lce(model, coeffs, label, record_time_as=None)\n",
" # make sure to save all parameters\n",
" config_cold[label][model.method].update(_config_cold[label][model.method])\n",
"del model"
]
},
{
"cell_type": "code",
"execution_count": 44,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/opt/conda/envs/py36/lib/python3.6/site-packages/pandas/plotting/_core.py:1001: UserWarning: Attempted to set non-positive left xlim on a log-scaled axis.\n",
"Invalid limit will be ignored.\n",
" ax.set_xlim(left, right)\n"
]
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAEWCAYAAABmE+CbAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xl8VfWd//HXJxskQAKEsAUwYVEIIqgpCIpLccFOFetYG1tbVJRpq6OOS9U6ttY6P+vSUh2tDlVGxNaNqQ51ikuLOygEBGUnhC2sIeyEEJJ8fn/ck3gTb8jNAkng/Xw8eOQs3+/3nBNy7/ue7/ecc83dERERiWnuHRARkZZBgSAiIoACQUREAgoEEREBFAgiIhJQIIiICKBAEBGRgAJBjltm9iczm1Jj2TlmVmRmPQ5TL87M3Mw2mVls2PKEoG7ZkdxvkSNFgSDHs5uBb5nZBQBm1hb4I3C7u2+Oov5e4MKw+W8D25t8L0WOEgWCHLfcvQj4V2CymbUDfgmsdvfnzWykmX1qZrvMbLOZPWFm8TWamAb8KGz+R8AL4QXMrMDMzg2bf9DMng+mk8zsz8FZxS4zm2tmXZr+SEWio0CQ45q7vwbMB14CJgL/EqwqA24BugBnAmPD1lX6C/BNM0s2s1TgDODNemz+WiAJ6AWkAj8FShp2JCKNF9fcOyDSAtwIrAbudff1AO4+L2x9vplNBs4BngxbXgzMBL4LJAKvAwfrsd1DhAKnv7t/CeQ2+AhEmoACQY577r7VzLYDSyqXmdlA4LfA6YQ+xccBn0Wo/gKhrqZEQmcU9fE80BN41cySCXVB/bu7a1BamoW6jEQi+y9gMaFP78nALwCLUO494ASgo7vPibB+P6FAqdS9csLdS939fncfBJwFfAf4QRPtv0i9KRBEIusA7Ab2m9kgvj5+AICHnh//beCyWtpZCOQEl6oOBy6vXGFm3zSzk80sBthDqAupvAmPQaReFAgikd0OjCd0ael/Aa/UVtDdF7v70lpW3wsMBHYB9wF/DlvXk9DA9B5C3VV/JzS4LdIsTF+QIyIioDMEEREJKBBERARQIIiISECBICIiQCu7Ma1Lly6ekZHR3LshItKqzJ8/f7u7p9VVrlUFQkZGBrm5urtfRCRcfuE+Ln96NhdmdeP+SweTlFD9rd3M1kXTTqsKhIKdB7j91UUNqmsGPTsm0i+tHf3S2tM3rd3Xfmki0nJs21PCv770Ob+8ZDBZPZObe3daLHfn/r8u5UBpOa/NL2DB+l08+f1TGdi9/r+zVvWOuO9gGZ/mFzWobnmFs21vCRVht130TGlLv67tqwKiX1poultyG8wiPaVARI6WSX9fyWdrdvDEP1bxzA9Pb+7dabHeWbqVD1cWct+3sxjYvQO3vrKQcU9+wi8uyeL7w/vU672sVd2Ylp2d7Y3pMio5VM66omJWF+4jv3Afqwv3s7pwH6u37WN/6VdPDGiXEEu/ru3p2yUIiSA0TkhNom187GG2ICJNIW/bPi6c9AEdkxLYWVzK+3ecywmp7Zp7t1qckkPljPntB7RrE8v/3Tya+NgYCvce5LZXF/LRqu3805AePPTPQ0hJTJjv7tl1tdeqzhAaq218LCd178BJ3TtUW+7ubNt7kNXb9oUCIgiKeWt38sbCTVXlzKB3p6Sqbqeq0OjantR2CTqrEGkij7y1nKSEOP50/QguffJjpny8hl+NO7m5d6vF+cP7q9m46wAv3XAG8bGhi0bTOrRh6rXD+a8P83nsnRV8sXFX1O0dV4FQGzOjW3JbuiW3ZVT/6l9YVVxaRn4QEJU/VxfuZ/bqIg6WVVSVS0mMp19aO/oG3U790kJB0adzUtV/lEh9FO49yH/OWoU7/HDkCZzYrUPdlY4BuWt38M7Srdx+wYkM6pHMuGHpvJpbwL9dcCIdkxKae/dajPVFxTzzwWouGdqTkf1Sq62LiTF+cm4/hmd24t7XF0fd5nHVZdSUKiqcTbsPhM4mqs4sQqGxbe9X35ESF2P0SU2qGp+oDI3+ae1JSar5jYxyOO7O5t0lVd18eYX7WL1tPxt3HeCcE9OYcFYmGV1af7dCRYXz0rz1PDxzOQcOlRNjxsGyCkb1S2X8qAzOH9SN2Jhj82zU3fnuM3NYv6OY9+88l6SEOJZv2cPY33/EnRedxI3n9W/uXWwxrp+ay+zV25l1+7l0T2lbazl3JyYmRl1GR1JMjNGrUxK9OiVxzonVL+/dU3IodDaxbR/520NvWqsL9/H+im0cKv8qgLu0TwjOKNqFBUZ70jslHrMv+GiUllWwrij0O8vbFjojy9sWGvcJH+vp0DaO/l3bc2K39rwybwMvfraOi7K6c8PZfTn9hE7NeAQNt3TTHu5940s+X7+LM/p25sHLhpDaLoGX521g2py1/Mu0+fTqlMgPzziB732j9zH3ifndpVvJXbeT//edIVVXAQ7snszoAV14fvZarh+dSZs4jeO9t3wbf1+2lbsvHnjYMAA0qNxSlZVXULDzQNXZRGVQ5G/fz479pVXlEuJiyExtR7+uXwVF3+DMon2bYyfD95QcCs6u9gdv/KFP/ut2FFMedjlY+NVg/cN+dmn/1bjNtj0lTJ2zlhc/Xc/uA4c4/YRO3DA6kwuyureKcN1/sIzf/30lUz5ZS0piPPd+axCXn5Ze7cVcVl7B35dt5fnZa/k0fwdt42P4zqnpjB+V0aBLDFuasvIKxj7+ERXuvHPr2cSFdbV+uLKQH02Zy6NXnMJ3s3s3417WX2lZBRt3HWDDjmI27Cxm/Y5iCnYcoGBnMYN6JHPL+QPokZIYdXslh8q56PcfEhdjzLzlbBLi6u6SNrOozhAUCC3Ejv2lwZVP1ccq1td4c+ye3LYqKCoHtPultadHStsWOajt7mzdc7DqDT/8Z3jXWnyskZHartobfmUQtqtHCBaXlvFabgHPfpzPhh0HOCE1ievPyuSK03uTmNAyP1m+s2QL989YwqbdJVw1vDd3jR1Y5yf/ZZv38MKctbz++UZKDlUwIrMz154Z6k6KOwJjVu7Oqm37mJ23nV0HDjHx7L5Nfh/Py3PXc/dfvuSZq09n7Mndq61zdy5+/CMAZt4yusX9rVf+nX/1YS/0+l2zfT+bdx+odrl7QmwM6Z0S6Z7clvnrdmIGE87K5Mfn9iO5bd3dyE/OWsVj76xk2oThjB5Q583HgALhmHGwrJz1RcVfXSIbBEX+tn3sPfjVV+8mJcSS2SWs6ykIjcwu7Y7KpbKHyitYV1Rc7ZN+5b7uC9vPDm3i6Ne1fY03/nb06ZzUpG9k5RXO20u2MPnDfBZu2EWnpHiuPuMEfjQyg7QObZpsO42xcdcB7p+xhHeXbuWkbh34j++cTHZG53q1sau4lFfmbeCFOevYuOsAPVPacvXIE8j5Rh86t2t4d5K7s66omNmri5i9ejuf5hexfd9XZ7GjB3Th2fHZTdZ9c6C0nHMfe4/0jon8z09GRXzDnz6/gDteW8QL1w3n7BOjeyM8UioqnA9WFvLXRZuCsazq3Znt28RVjRf27pxEn85J9O6USJ/UJLp1aEtMcNa6YUcxv31nBW8s3ESnpHhuHjOAH4w44Wuf+svKK1i6eQ9zVhcx6e8rOe+krjx9dfT3ZigQjnHuTuG+g191O4UFxsZdB6j8bzWD9I6J1bqeKgMjrX39b8DbW3KoaiA9L+yNf11RMWVhH4N6pLSt9obfr2toID2tw9G96c/dmb9uJ5M/zOfdZVuJj43h8lPTuX50Jv27Ns9VO4fKK/jvT9Yw6d1VANx6/gCuOyuzUVejlVc4/wi6k2avLqJNXAzjhvVk/KgMBvdMiaqNjbsOMCcIgDmri9i8uwSAbsltGNWvCyP7pTKybypz8ov42fQvuGhwN576/mlNEuRPvZfHo2+v4NV/GcnwzMihWFpWwVkPz+Kk7h2YNmFEo7fZEHtLDjF9fgFTZ69lbVExqe0SyOqZHHa/Uuj11bWef+dfFuzmoZnLmL26iD6dk/jZ2JPISG3HnNVFfJpfxNw1O6o+AA7qkcxz47Pp2TH6biYFwnHsQGk5a7bvrzagXRkaBw5VH5SteZd2/67t6NO5HTuLSyN282zdU/0Kqowu7eiXVr2rp6WOdazZvp/nPs7ntdwCDpZV8M2BXblhdF/O6Nv5qIXU/HU7uff1L1m+ZS/nD+rK/ZcOplenpCbdxsqte5k6ey1/WbCRA4fKGZ7RmfGjMrhwcLdqoVO49yBz8ouYs3o7s1cXsa6oGIDO7RIY2TeVkf1SGdUvlcwu7b72+5ny8RoeeHMpl5+WzmNXDK36xNsQO/eXcvYj7zGib2eeHf+Nw5atDI6Zt4xmUI+GjZu8tXgzv3t3JQO7JzOsd0eG9elIVo/kw55Jr92+n+dnr2X6/AL2HSzjtD4dufbMTMae3L3JLit3D511/GbmcpZv2Vu1vG+XdowI/j/OyOxM1+TDDyJHokCQr6mocLbs+eqyzfzt+6sGt7fsKam1Xvugm6fmG39rvceiaN9BXvx0PS/MWUvR/lKGpKdww9l9+dbJ3Y9I/zvA7uJD/Oat5bw0dz09Utpy/6WDuTCr2xENot3Fh3htfqg7af2OYront+XKb/Rmd3Eps1cXsWrbPiD0wWBEZujNf1T/VE7s2iGqN/gn/rGK3727kh+NPIFfXTq4wcfy6zeX8t+frOHtW89mQB33WuwqLmXkQ7P41pAe/PbKofXe1oYdxXzr8Y9IToynPHg9QGgMa1CPZIb26siw3h0Z2rsjmV3a8Unedp6fvZb3VmwjLsb49ik9uWZUBkN7d2zQsUajsruztKyCM/qm1nkVUTQUCFIv+w6WkR+cRazZvp/U9glVb/z1Pf1tLUoOlfP65xv540f55BfuJ71jIteemUHO8D5Ndobj7ryxcCMPvrmMXQcOce2oDP7tghPrNVDeWOUVznvLtzF1zlo+WrWdxPhYvpHZORQA/VIZ3DOlQVdiuTsPzVzO5A/zufG8ftx50cB6t7FhRzFjfvsB3zk1nYevOCWqOvfPWMKfPlvHx3d9k271+LRcVl7B9yZ/ysote/nbLaPp3TmJLbtLWLhhF4sKdrFw/S6+KNhVNRaQEBtDaXkFXdq34eoz+vD9EX3o2qHxb87NQYEgEqWKCmfW8m1M/iifuWt20KFtHN8f0YdrR2U26tPZ6sJ93PfGYmavLmJY7478x3dOjro//0gp3HuQlMT4qC5VjIa78/PXF/PS3PX8bOxJ/PTc+t049m+vLORvX27m/TvPjfrSy/VFxZz72Hv8yzn9uGts9CE06d2VPP6PVTyeM4xxw9IjlimvcPIL9/H5hl0s3bSHU3ql8E+n9Gj19z5EGwgtr6NX5CiLiTHOz+rG+VndWLRhF3/8KJ8/fpjPcx+t4dKhPbl+dN96PX655FA5f3h/Nc+8v5o28TE8eNnJXDW8T4u4H6Kpr7AyMx687GT2HyzjkbdW0KFNHD8cmRFV3SWbdvPGwo38+Jx+9boOv09qEhcN7s6fPl3HTef1j+psa97aHfznrFVcflp6rWEAEBtjDOjWoc6uq2OVAkEkzNDeHXny+6exYUcxUz5ZwyvzNvCXzzcyekAXrh/dl7MHdDls99lHqwq5743FrC0qZtywntz7T4NabTdDtGJjjN9eOZTi0jLu+98ltGsTx+Wn9aqz3sNvrSAlMZ4fn9Ov3tu8fnRfZi7ewmu5G7jmzMzDlt194BC3vryQXp2SeEAPyDus1jciKHIU9O6cxC8vGcycu8dw19iBrNiyl/FT5nLx4x8xfX4BpWEPNgTYtreEm1/6nB8+Nxcz48UJI3g859RjPgwqxcfG8OT3T2NUv1TunP4Fby3ectjyn+Rt58OVhdx0Xn9SEuv/TK/TT+jEaX06MuWTtdVu3KzJ3bn39S/ZsqeEx3OGtcir31oSBYLIYaQkxfOTc/vx8V3f5LHvDsUd7nhtEaMfmcUf3s9j5/5Spn26jjG//YC3Fm/hljEDmHnLaM4a0KXuxo8xbeNj+eOPshmSnsLNL33OR6sKI5arqHB+M3M56R0TufqMExq8vYln92X9jmLeWVJ7+PzPgo28+cVmbrvgRE7t0zqfb3U0aVBZpB7cnQ9XbefZj/L5aNV2YgwqHM7sn8qvx51M37T2zb2LzW5XcSk5kz9lXVEx0yYM/9rd1zMWbeLmlz7nd1cOjaprqTblFc43f/s+ndsl8PpPz/za+rXb9/OtJz5iSHoKf77hjBYxhtNcoh1U1hmCSD2YGeecmMa0CSP4282juWZUJo/nDOPFCSMUBoGOSQlMmzCC7iltufb5eSzeuLtqXWlZBY+9vYJBPZK57DCDu9GIjTGuOzOTz9fvYv66HdXWlZZVcPPLnxMfG8Ok7w07rsOgPhQIIg2U1TOZX1ySxbhh6cfkfRqNkdahDS9eP4IObeL40ZS55G0L3Xn7589CN8ndNfakRt3dXOm72b1ISYznjx+uqbZ80t9X8kXBbh66fEi9HvFwvFMgiMgRkd4xkRevH0GMwdXPzmXZ5j08MSuPUf1Sv/YdIg2VlBDHD0b04e2lW1hXtB+A2au388wHq8n5Rm++NaRHk2zneKFAEJEjpm9ae6ZNGEFxaRmXPvkxO/aXcvfFA5v0jOqaURnExRhTPl7Dzv2l3PbKIjJT2/GLS7KabBvHi6gCwczGmtkKM8szs7sjrG9jZq8E6z8zs4xgeaqZvWdm+8zsybDySWb2f2a23MyWmNlvmuqARKRlGdQjmanXDSchNobLhvXklF5N+xygrsltq753+dZXFlK0/yBPXHVqk39fw/GgzkAws1jgKeBiIAu4ysxqRu8EYKe79wcmAQ8Hy0uA+4A7IjT9mLsPBE4FzjSzixt2CCLS0p3apxOf3B26dPdIuH50JgcOlfPBykLuvOgkTk5v3keEtFbRROhwIM/d8wHM7GVgHLA0rMw44P5gejrwpJmZu+8HPjazag84cfdi4L1gutTMFgANv/5MRFq8I/n9zwO7J/PPp/WipKyc68/qe8S2c6yLJhDSgQ1h8wVAzW+nqCrj7mVmthtIBbbX1biZdQQuAR6vZf1EYCJAnz59othdETkeNeRx2FJdNGMIkUZ/at7NFk2ZrzdsFge8BDxReQbytUbcJ7t7trtnp6U179fmiYgcy6IJhAKgd9h8L2BTbWWCN/kUYAd1mwyscvffR1FWRESOoGgCYR4wwMwyzSwByAFm1CgzAxgfTF8BzPI6nolhZg8SCo5b67fLIiJyJNQ5hhCMCdwEvA3EAlPcfYmZPQDkuvsM4DlgmpnlETozyKmsb2ZrgWQgwcwuAy4E9gD3AsuBBcE1yU+6+7NNeXAiIhK9qC7Udfe/AX+rsewXYdMlwHdrqZtRS7O6119EpAXRncoiIgIoEEREJKBAEBERQIEgIiIBBYKIiAAKBBERCSgQREQEUCCIiEhAgSAiIoACQUREAgoEEREBFAgiIhJQIIiICKBAEBGRgAJBREQABYKIiAQUCCIiAigQREQkoEAQERFAgSAiIgEFgoiIAAoEEREJRBUIZjbWzFaYWZ6Z3R1hfRszeyVY/5mZZQTLU83sPTPbZ2ZP1qhzupl9GdR5wsysKQ5IREQaps5AMLNY4CngYiALuMrMsmoUmwDsdPf+wCTg4WB5CXAfcEeEpp8GJgIDgn9jG3IAIiLSNKI5QxgO5Ll7vruXAi8D42qUGQdMDaanA2PMzNx9v7t/TCgYqphZDyDZ3ee4uwMvAJc15kBERKRxogmEdGBD2HxBsCxiGXcvA3YDqXW0WVBHmwCY2UQzyzWz3MLCwih2V0REGiKaQIjUt+8NKNOg8u4+2d2z3T07LS3tME2KiEhjRBMIBUDvsPlewKbayphZHJAC7KijzV51tCkiIkdRNIEwDxhgZplmlgDkADNqlJkBjA+mrwBmBWMDEbn7ZmCvmZ0RXF30I+B/6733IiLSZOLqKuDuZWZ2E/A2EAtMcfclZvYAkOvuM4DngGlmlkfozCCnsr6ZrQWSgQQzuwy40N2XAj8BngcSgZnBPxERaSZ2mA/yLU52drbn5uY2926IiLQqZjbf3bPrKqc7lUVEBFAgiIhIQIEgIiKAAkFERAIKBBERARQIIiISUCCIiAigQBARkYACQUREAAWCiIgEFAgiIgIoEEREJKBAEBERQIEgIiIBBYKIiAAKBBERCSgQREQEUCCIiEhAgSAiIoACQUREAgoEEREBFAgiIhJQIIiICBBlIJjZWDNbYWZ5ZnZ3hPVtzOyVYP1nZpYRtu6eYPkKM7sobPm/mdkSM1tsZi+ZWdumOCAREWmYOgPBzGKBp4CLgSzgKjPLqlFsArDT3fsDk4CHg7pZQA4wGBgL/MHMYs0sHbgZyHb3k4HYoJyIiDSTaM4QhgN57p7v7qXAy8C4GmXGAVOD6enAGDOzYPnL7n7Q3dcAeUF7AHFAopnFAUnApsYdioiINEY0gZAObAibLwiWRSzj7mXAbiC1trruvhF4DFgPbAZ2u/s7kTZuZhPNLNfMcgsLC6PYXRERaYhoAsEiLPMoy0RcbmadCJ09ZAI9gXZmdnWkjbv7ZHfPdvfstLS0KHZXREQaIppAKAB6h8334uvdO1Vlgi6gFGDHYeqeD6xx90J3PwT8BRjVkAMQEZGmEU0gzAMGmFmmmSUQGvydUaPMDGB8MH0FMMvdPVieE1yFlAkMAOYS6io6w8ySgrGGMcCyxh+OiIg0VFxdBdy9zMxuAt4mdDXQFHdfYmYPALnuPgN4DphmZnmEzgxygrpLzOxVYClQBtzo7uXAZ2Y2HVgQLP8cmNz0hyciItGy0Af51iE7O9tzc3ObezdERFoVM5vv7tl1ldOdyiIiAigQREQkoEAQERFAgSAiIgEFgoiIAAoEEREJKBBERARQIIiISECBICIigAJBREQCCgQREQEUCCIiElAgiIgIoEAQEZGAAkFERAAFgoiIBBQIIiICKBBERCSgQBAREUCBICIiAQWCiIgACgQREQlEFQhmNtbMVphZnpndHWF9GzN7JVj/mZllhK27J1i+wswuClve0cymm9lyM1tmZiOb4oBERKRh6gwEM4sFngIuBrKAq8wsq0axCcBOd+8PTAIeDupmATnAYGAs8IegPYDHgbfcfSAwFFjW+MMREZGGiuYMYTiQ5+757l4KvAyMq1FmHDA1mJ4OjDEzC5a/7O4H3X0NkAcMN7Nk4GzgOQB3L3X3XY0/HBERaahoAiEd2BA2XxAsi1jG3cuA3UDqYer2BQqB/zazz83sWTNrF2njZjbRzHLNLLewsDCK3RURkYaIJhAswjKPskxty+OA04Cn3f1UYD/wtbEJAHef7O7Z7p6dlpYWxe6KiEhDRBMIBUDvsPlewKbayphZHJAC7DhM3QKgwN0/C5ZPJxQQIiLSTKIJhHnAADPLNLMEQoPEM2qUmQGMD6avAGa5uwfLc4KrkDKBAcBcd98CbDCzk4I6Y4CljTwWERFphLi6Crh7mZndBLwNxAJT3H2JmT0A5Lr7DEKDw9PMLI/QmUFOUHeJmb1K6M2+DLjR3cuDpv8V+FMQMvnAtU18bCIiUg8W+iDfOmRnZ3tubm5z74aISKtiZvPdPbuucrpTWUREAAWCiIgEFAgiIgIoEEREJKBAEBERQIEgIiIBBYKIiAAKBBERCSgQREQEUCCIiEhAgSAiIoACQUREAgoEEREBFAgiIhJQIIiICKBAEBGRgAJBREQABYKIiAQUCCIiAigQREQkoEAQERFAgSAiIgEFgoiIAFEGgpmNNbMVZpZnZndHWN/GzF4J1n9mZhlh6+4Jlq8ws4tq1Is1s8/N7M3GHoiIiDROnYFgZrHAU8DFQBZwlZll1Sg2Adjp7v2BScDDQd0sIAcYDIwF/hC0V+kWYFljD0JERBovmjOE4UCeu+e7eynwMjCuRplxwNRgejowxswsWP6yux909zVAXtAeZtYL+Cfg2cYfhoiINFY0gZAObAibLwiWRSzj7mXAbiC1jrq/B34GVBxu42Y20cxyzSy3sLAwit0VEZGGiCYQLMIyj7JMxOVm9m1gm7vPr2vj7j7Z3bPdPTstLa3uvRURkQaJJhAKgN5h872ATbWVMbM4IAXYcZi6ZwKXmtlaQl1Q3zSzFxuw/yIi0kSiCYR5wAAzyzSzBEKDxDNqlJkBjA+mrwBmubsHy3OCq5AygQHAXHe/x917uXtG0N4sd7+6CY5HREQaKK6uAu5eZmY3AW8DscAUd19iZg8Aue4+A3gOmGZmeYTODHKCukvM7FVgKVAG3Oju5UfoWEREpBEs9EG+dcjOzvbc3Nzm3g0RkVbFzOa7e3Zd5XSnsoiIAAoEEREJKBBERARQIIiISECBICIigAJBREQCCgQREQEUCCIiElAgiIgIoEAQEZGAAkFERAAFgoiIBBQIIiICKBBERCSgQBAREUCBICIiAQWCiIgACgQREQkoEEREBFAgiIhIQIEgIiKAAkFERAJRBYKZjTWzFWaWZ2Z3R1jfxsxeCdZ/ZmYZYevuCZavMLOLgmW9zew9M1tmZkvM7JamOiAREWmYOgPBzGKBp4CLgSzgKjPLqlFsArDT3fsDk4CHg7pZQA4wGBgL/CForwy43d0HAWcAN0ZoU0REjqJozhCGA3nunu/upcDLwLgaZcYBU4Pp6cAYM7Ng+cvuftDd1wB5wHB33+zuCwDcfS+wDEhv/OGIiEhDRRMI6cCGsPkCvv7mXVXG3cuA3UBqNHWD7qVTgc8ibdzMJppZrpnlFhYWRrG7IiLSENEEgkVY5lGWOWxdM2sP/A9wq7vvibRxd5/s7tnunp2WlhbF7oqISENEEwgFQO+w+V7AptrKmFkckALsOFxdM4snFAZ/cve/NGTnRUSk6UQTCPOAAWaWaWYJhAaJZ9QoMwMYH0xfAcxydw+W5wRXIWUCA4C5wfjCc8Ayd/9dUxyIiIg0TlxdBdy9zMxuAt4GYoEp7r7EzB4Act19BqE392lmlkfozCAnqLvEzF4FlhK6suhGdy83s7OAHwJfmtnCYFM/d/e/NfUBiohIdCz0Qb51yM7O9tzc3ObeDRGRVsXM5rt7dl3ldKeyiIgACgQREQkoEEREBIhiULkyOjo6AAAL/UlEQVSlO3ToEAUFBZSUlDT3rrRIbdu2pVevXsTHxzf3rohIC9fqA6GgoIAOHTqQkZFB6GpWqeTuFBUVUVBQQGZmZnPvjoi0cK2+y6ikpITU1FSFQQRmRmpqqs6eRCQqrT4QAIXBYeh3IyLROiYCQUREGk+B0ATat28fcfkLL7zAySefzODBg8nKyuKxxx4D4JprriEzM5Nhw4YxbNgwRo0aVVXnjTfe4IEHHqj3PuTk5LBq1aqGHYCICAqEI2bmzJn8/ve/55133mHJkiUsWLCAlJSUqvWPPvooCxcuZOHChcyePbtq+SOPPMJPf/rTem/vJz/5CY888kiT7LuIHJ9a/VVG4X711yUs3RTxKdoNltUzmV9eMrje9R566CEee+wxevbsCYQu/7zhhhsOW2flypW0adOGLl26ALB161Z+/OMfk5+fD8DTTz/N0KFDufLKKykoKKC8vJz77ruP733ve4wePZprrrmGsrIy4uKOqf9WETlKdIZwhCxevJjTTz+91vV33nlnVZfRD37wAwA++eQTTjvttKoyN998M+eccw6LFi1iwYIFDB48mLfeeouePXuyaNEiFi9ezNixYwGIiYmhf//+LFq06MgemIgcs46pj5IN+STfXB599FGuuOKKass2b95M+JcAzZo1ixdeeAGA2NhYUlJSGDJkCHfccQd33XUX3/72txk9enRV+a5du7Jp06bDBpGISG10hnCEDB48mPnz59erTmJiYp33DJx44onMnz+fIUOGcM8991QbgC4pKSExMbFB+ysiokA4Qu655x5+9rOfsWXLFgAOHjzIE088cdg6gwYNIi8vr2p+zJgxPP300wCUl5ezZ88eNm3aRFJSEldffTV33HEHCxYsqCq/cuVKBg9uPWdJItKyHFNdRs2luLiYXr16Vc3fdttt3HbbbWzdupXzzz8fd8fMuO6666rK3HnnnTz44INV83PnzuXss8/m9ttvryr/+OOPM3HiRJ577jliY2N5+umn2bNnD3feeScxMTHEx8dXBcbWrVtJTEykR48eR+/AReSY0uq/IGfZsmUMGjSomfao6d1yyy1ccsklnH/++fWqN2nSJJKTk5kwYcLX1h1rvyMRqR99QU4r9fOf/5zi4uJ61+vYsSPjx4+vu6CISC3UZdTCdOvWjUsvvbTe9a699tojsDcicjw5Js4QWlO319Gm342IRKvVB0Lbtm0pKirSG18Eld+H0LZt2+beFRFpBVp9l1GvXr0oKCigsLCwuXelRar8xjQRkbq0+kCIj4/Xt4GJiDSBqLqMzGysma0wszwzuzvC+jZm9kqw/jMzywhbd0+wfIWZXRRtmyIicnTVGQhmFgs8BVwMZAFXmVlWjWITgJ3u3h+YBDwc1M0CcoDBwFjgD2YWG2WbIiJyFEVzhjAcyHP3fHcvBV4GxtUoMw6YGkxPB8ZY6LsbxwEvu/tBd18D5AXtRdOmiIgcRdGMIaQDG8LmC4ARtZVx9zIz2w2kBss/rVE3PZiuq00AzGwiMDGY3WdmK8JWpwC7oziGSl2A7fUofzyr7++2uTXn/h7pbTd1+41trzH1G1K3PnX0Go/shGgKRRMIkb6lveY1nrWVqW15pDOTiNeNuvtkYHLEHTOb7O4TI62rpXxuNLdvS/1/t82tOff3SG+7qdtvbHuNqd+QuvWpo9d440TTZVQA9A6b7wVsqq2MmcURSvQdh6kbTZvR+GsD6kh0Wtvvtjn390hvu6nbb2x7janfkLqt7W+x1arz4XbBG/xKYAywEZgHfN/dl4SVuREY4u4/NrMc4HJ3v9LMBgN/JjRm0BP4BzCA0JnDYds8EvTpQeTYptd449TZZRSMCdwEvA3EAlPcfYmZPQDkuvsM4DlgmpnlETozyAnqLjGzV4GlQBlwo7uXA0Rqs+kP72sidj2JyDFDr/FGaFWPvxYRkSOn1T/LSEREmoYCQUREgBYUCGaWaGYfBHcxY2bjzWxV8C/iN7+Y2a/N7AszW2hm75hZzyi2E027nc3s3aDMu2bWKVh+rpntDra30Mx+ESxPMLMPgwF4EalDA1/vQ81sjpl9aWZ/NbPkKLZT5yNyanv0jpkNMbPnG3yQrZG7t4h/wI3ALcF0ZyA/+NkpmO4UoU5y2PTNwDN1bCPadh8B7g6m7wYeDqbPBd6spe1fAj9o7t+j/ulfa/jXwNf7POCcYPo64Nd1bCMWWA30BRKARUBWhHI/rXzvIHRBzCth6/4O9Gnu39fR+tdizhCAHwD/G0xfBLzr7jvcfSfwLqFnIVXj7nvCZttRy81tYaJql+qP4pgKXBbF/r8RHIOI1K3er3fgJODDYPpd4J/r2Ea0j8ip7dE7ELoHIieK4zkmtIhAMLMEoK+7rw0WRXpcRnrNekHd/zCzDYT+wH5Rx6aibbebu28GCH52DVs30swWmdnM4D6LSouBb9SxfZHjXiNe74uByu+X/S7Vb26NJNp2qz16h9BjMlKDdbnA6Dq2c8xoEYFA6Pkju8Lmo3lcRmih+73u3hv4E3BTHduJut1aLABOcPehwH8SOiuo3I9yoNTMOtSjPZHjUUNf79cBN5rZfKADUFrHdqJt93DlthG6qfa40FIC4QAQ/j2PDXm0xZ+p+xQy2na3mlkPgODnNgh1Ubn7vmD6b0C8mXUJq9cGKKljH0SOdw16vbv7cne/0N1PB14iND5wONG+3mt79A7Bfh6oYzvHjBYRCEG/YayZVf6RvA1caGadgit8LgyWVWNmA8JmLwWWB8uHm9kLETYVVbvADKDySofxBH2dZta9sm/RzIYT+v0VBfOpQKG7H4r+yEWOP414vXcNfsYA/w48E8ynm9k/ImxqHjDAzDKDbqocQq/tmsJf71cAszwYUQZOJNRVdVxoEYEQeAc4C8DddwC/JvQfOg94IFiGmT1rZpXPKvmNmS02sy8I/RHdEizvQ4RUr0+7wAVmtgq4IJiH0B/LYjNbBDwB5IT94ZwH/K3xvwaR40JDXu9XmdlKQh/8NgH/HSzvQejRONUE4wGVj8hZBrzqwSNyzOwBM6scj3gOSA0evXMboSsLK50H/F+THHEr0GIeXWFmpwK3ufsPm6CtR4Fp7v5F4/cs6m3+BbjH3VfUWVjkONfEr/ebgPUeeq5akzGzNsAHwFlBuBzzWkwgAJjZdcDUYIC21ag8HXX3SN1UIhJBS3+9B13S6e7+fnPvy9HSogJBRESaT0saQxARkWakQBAREUCBICIiAQWCHJfMbG2NmwobVOYwdc81szfrWedWM0tqyPZEmoICQaTluBVQIEizUSDIMc/M3jCz+Wa2xMwm1liXYWbLzWxq8N0a02t8Sv9XM1sQPIN/YFBnuJnNNrPPg58n1bLpZDN73cyWmtkzwR22mNmFwXP9F5jZa2bW3sxuJvTMnPfM7L2g3NNmlhvs96+a/jcjUp0CQY4H1wXPv8kGbg4eMxLuJGCyu58C7CH0fPxK2939NOBp4I5g2XLgbHc/ldATdv9fLdsdDtwODAH6AZcHXVD/DpwftJtL6AatJwjdfXueu58X1L/X3bOBU4BzzOyUBh6/SFT0DV9yPLjZzL4TTPcGBtRYv8HdPwmmXyT0ZUuPBfN/CX7OBy4PplOAqcGNSw7E17Ldue6eD2BmLxF6VEMJkAV8EjwWKwGYU0v9K4MzmjhCj2fIAo7a3fdy/FEgyDHNzM4FzgdGunuxmb1P9SdtwtcfiRw+fzD4Wc5Xr5dfA++5+3eCr1t8v5bNR2rXCH0ZzFV17HcmoTOSb7j7zuCrHGvut0iTUpeRHOtSgJ1BGAwEzohQpo+ZjQymrwI+jqLNjcH0NZULIzx1c3jwpM0Y4HtBu58CZ5pZ/6BOkpmdGJTfS+g5/wDJwH5gt5l1Ay6u+1BFGkeBIMe6t4C44Im4vyb0hlzTMmB8UKYzofGCw3kEeMjMPiH0vb2Vaj51cw6hJ+UuBtYAr7t7IaEQeSnY3qfAwKD8ZGCmmb3n7ouAz4ElwBTgE0SOMD3LSI5rQZfPm+5+chO0dUSeuilytGgMQaSJuPuTzb0PIo2hMwQREQE0hiAiIgEFgoiIAAoEEREJKBBERARQIIiISOD/A6xpZiqsrURWAAAAAElFTkSuQmCC\n",
"text/plain": [
"<Figure size 432x288 with 1 Axes>"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"report_results('tuning', scores_cold);"
]
},
{
"cell_type": "code",
"execution_count": 45,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'YaMus': {'LCE(cs)': {'alpha': 0.1,\n",
" 'beta': 0.1,\n",
" 'regularization': 10,\n",
" 'rank': 40}}}"
]
},
"execution_count": 45,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"config_cold"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### saving data"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"save_results(f'{experiment_name}_coldstart_param', config=config_cold, tuning=scores_cold)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## rank estimation"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"rank_config_cold = {}\n",
"rank_scores_cold = {}"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div><span class=\"Text-label\" style=\"display:inline-block; overflow:hidden; white-space:nowrap; text-overflow:ellipsis; min-width:0; max-width:15ex; vertical-align:middle; text-align:right\"></span>\n",
"<progress style=\"width:60ex\" max=\"1\" value=\"0\" class=\"Progress-main\"/></progress>\n",
"<span class=\"Progress-label\"><strong>0%</strong></span>\n",
"<span class=\"Iteration-label\">0/1</span>\n",
"<span class=\"Time-label\">[0<0, 0.00s/it]</span></div>"
],
"text/plain": [
"\u001b[2K\r",
" [############################################################] 0/1 [0<0, 0.00s/it]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"<div><span class=\"Text-label\" style=\"display:inline-block; overflow:hidden; white-space:nowrap; text-overflow:ellipsis; min-width:15ex; max-width:15ex; vertical-align:middle; text-align:right\">YaMus</span>\n",
"<progress style=\"width:45ex\" max=\"10\" value=\"1\" class=\"Progress-main\"/></progress>\n",
"<span class=\"Progress-label\"><strong>10%</strong></span>\n",
"<span class=\"Iteration-label\">1/10</span>\n",
"<span class=\"Time-label\">[06:43<06:43, 402.93s/it]</span></div>"
],
"text/plain": [
"\u001b[A\u001b[2K\r",
" YaMus [████#########################################] 1/10 [06:43<06:43, 402.93s/it]"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"for label in track(data_labels):\n",
" model, = prepare_cold_start_recommender_models(label, data_models_cold,\n",
" [lce_init_config, config_cold]) # initiate with optimal config\n",
" rank_config_cold[label], rank_scores_cold[label] = fine_tune_lce(model, {'rank': lce_ranks[label]},\n",
" label, ntrials=0)\n",
"del model"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"report_results('rank', {lbl: v.sort_index() for lbl, scr in rank_scores_cold.items() for k, v in scr.items()});"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"rank_config_cold"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### saving data"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"save_results(f'{experiment_name}_coldstart_rank', config=rank_config_cold, tuning=rank_scores_cold)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## cross validation"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"result_cold = {}\n",
"for label in track(data_labels):\n",
" models_cold = prepare_cold_start_recommender_models(label, data_models_cold,\n",
" [lce_init_config, config_cold, rank_config_cold])\n",
" result_cold[label] = ee.run_cv_experiment(models_cold,\n",
" fold_experiment=ee.topk_test,\n",
" topk_list=topk_values,\n",
" ignore_feedback=True,\n",
" iterator=lambda x: track(x, label=label))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"report_results('topn', result_cold, target_metric);"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"report_results('topn', result_cold, 'coverage');"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### saving data"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"save_results(f'{experiment_name}_coldstart', cv=result_cold)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"1"
]
},
"execution_count": 52,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"1"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"1"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.8"
}
},
"nbformat": 4,
"nbformat_minor": 2
}