{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "env: MKL_NUM_THREADS=12\n", "env: OMP_NUM_THREADS=12\n" ] } ], "source": [ "%env MKL_NUM_THREADS=12\n", "%env OMP_NUM_THREADS=12" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "from collections import defaultdict\n", "\n", "import numpy as np\n", "import scipy as sp\n", "import pandas as pd\n", "from ipypb import track\n", "\n", "from polara.evaluation import evaluation_engine as ee\n", "from polara.evaluation.pipelines import random_grid, find_optimal_config\n", "from polara.recommender.external.turi.turiwrapper import TuriFactorizationRecommender\n", "\n", "from data_preprocessing import (get_amazon_data,\n", " get_similarity_data,\n", " prepare_data_model)\n", "from utils import (report_results, save_results,\n", " apply_config, print_data_stats,\n", " save_training_time, save_cv_training_time)\n", "\n", "%matplotlib inline" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "seed = 42" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "experiment_name = 'sgd'" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Experiment setup" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "data_labels = ['AMZe', 'AMZvg']" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "# according to https://apple.github.io/turicreate/docs/api/generated/turicreate.recommender.ranking_factorization_recommender.RankingFactorizationRecommender.html\n", "init_config = dict(with_data_feedback = False, # implicit case\n", " ranking_optimization = True,\n", " solver = 'adagrad',\n", " sgd_step_size = 0, # let Turi autotune it\n", " seed = seed,\n", " max_iterations = 25,\n", " other_tc_params = {}\n", " )\n", "mf_init_config = dict.fromkeys(data_labels, {'SGD': init_config}) # standard scenario" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [], "source": [ "params = {\n", " 'regularization': [1e-10, 3e-10, 1e-9, 3e-9, 1e-8, 3e-8, 1e-7, 3e-7, 1e-6, 3e-6],\n", " 'linear_regularization': [1e-10, 3e-10, 1e-9, 3e-9, 1e-8, 3e-8, 1e-7, 3e-7, 1e-6, 3e-6],\n", " 'rank': [40] # for initial tuning (exploration)\n", " }\n", "\n", "if init_config['solver'] == 'adagrad':\n", " params.update({\n", " 'adagrad_momentum_weighting': [0.9, 0.95, 0.99]\n", " }) " ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [], "source": [ "ranks_grid = [1, 5, 15, 30, 50, 75, 100, 150, 200, 250, 300, 400,\n", " 500, 750, 1000, 1250, 1500, 1750, 2000, 2500, 3000]" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [], "source": [ "mf_ranks = {'AMZe': ranks_grid,\n", " 'AMZvg': ranks_grid\n", " }" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [], "source": [ "topk_values = [1, 3, 10, 20, 30]" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [], "source": [ "target_metric = 'mrr'" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [], "source": [ "data_dict = dict.fromkeys(data_labels)\n", "meta_dict = dict.fromkeys(data_labels)\n", "similarities = dict.fromkeys(data_labels)\n", "sim_indices = dict.fromkeys(data_labels)\n", "feature_idx = dict.fromkeys(data_labels)" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [], "source": [ "all_data = [data_dict, similarities, sim_indices, meta_dict]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Amazon Electronics" ] }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [], "source": [ "lbl = 'AMZe'" ] }, { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [], "source": [ "data_dict[lbl], meta_dict[lbl] = get_amazon_data('/data/recsys/amazon/ratings_Electronics.csv',\n", " meta_path='/data/recsys/amazon/meta/meta_Electronics.json.gz',\n", " implicit=True,\n", " pcore=5,\n", " filter_no_meta=True,\n", " flat_categories=True) # take only bottom level categories" ] }, { "cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [], "source": [ "similarities[lbl], sim_indices[lbl], feature_idx[lbl] = get_similarity_data(meta_dict[lbl])" ] }, { "cell_type": "code", "execution_count": 17, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "0.0" ] }, "execution_count": 17, "metadata": {}, "output_type": "execute_result" } ], "source": [ "(meta_dict[lbl].applymap(len).sum(axis=1)==0).mean()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Amazon Video Games" ] }, { "cell_type": "code", "execution_count": 18, "metadata": {}, "outputs": [], "source": [ "lbl = 'AMZvg'" ] }, { "cell_type": "code", "execution_count": 19, "metadata": {}, "outputs": [], "source": [ "data_dict[lbl], meta_dict[lbl] = get_amazon_data('/data/recsys/amazon/ratings_Video_Games.csv',\n", " meta_path='/data/recsys/amazon/meta/meta_Video_Games.json.gz',\n", " implicit=True,\n", " pcore=5,\n", " filter_data={'categories': ['Games']}, # filter uniformative category\n", " filter_no_meta=True,\n", " flat_categories=True) # take only bottom level categories" ] }, { "cell_type": "code", "execution_count": 20, "metadata": {}, "outputs": [], "source": [ "similarities[lbl], sim_indices[lbl], feature_idx[lbl] = get_similarity_data(meta_dict[lbl])" ] }, { "cell_type": "code", "execution_count": 21, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "0.0" ] }, "execution_count": 21, "metadata": {}, "output_type": "execute_result" } ], "source": [ "(meta_dict[lbl].applymap(len).sum(axis=1)==0).mean()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Data stats" ] }, { "cell_type": "code", "execution_count": 22, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "AMZe\n", "{'userid': 124895, 'asin': 44843}\n", "density 0.019153791836615672\n", "similarity matrix density 1.1054998336712965\n", "AMZvg\n", "{'userid': 14251, 'asin': 6858}\n", "density 0.13281340440589384\n", "similarity matrix density 9.081814734274188\n" ] } ], "source": [ "print_data_stats(data_labels, all_data)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Standard experiment" ] }, { "cell_type": "code", "execution_count": 23, "metadata": {}, "outputs": [], "source": [ "def prepare_recommender_models(data_label, data_models, config):\n", " data_model = data_models[data_label]\n", " mf = TuriFactorizationRecommender(data_model, item_side_info=None)\n", " mf.method = 'SGD'\n", " models = [mf]\n", " apply_config(models, config, data_label)\n", " return models\n", "\n", "def fine_tune_mf(model, params, label, ntrials=60, record_time_as=None):\n", " param_grid, param_names = random_grid(params, n=ntrials)\n", " best_mf_config, mf_scores = find_optimal_config(model, param_grid, param_names,\n", " target_metric,\n", " return_scores=True,\n", " force_build=True,\n", " iterator=lambda x: track(x, label=label))\n", " model_config = {model.method: dict(zip(param_names, best_mf_config))}\n", " model_scores = {model.method: mf_scores}\n", " try:\n", " if record_time_as:\n", " save_training_time(f'{experiment_name}_{record_time_as}', model, mf_scores.index, label)\n", " finally:\n", " return model_config, model_scores" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## tuning" ] }, { "cell_type": "code", "execution_count": 24, "metadata": {}, "outputs": [], "source": [ "config = {}\n", "scores = {}\n", "times = {}\n", "data_models = {}" ] }, { "cell_type": "code", "execution_count": 25, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'with_data_feedback': False,\n", " 'ranking_optimization': True,\n", " 'solver': 'adagrad',\n", " 'sgd_step_size': 0,\n", " 'seed': 42,\n", " 'max_iterations': 25,\n", " 'other_tc_params': {}}" ] }, "execution_count": 25, "metadata": {}, "output_type": "execute_result" } ], "source": [ "mf_init_config['AMZe']['SGD']" ] }, { "cell_type": "code", "execution_count": 26, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
| \n", " | \n", " | AMZe | \n", "AMZvg | \n", "
|---|---|---|---|
| type | \n", "metric | \n", "\n", " | \n", " |
| relevance | \n", "hr | \n", "0.050090 | \n", "0.143850 | \n", "
| ranking | \n", "mrr | \n", "0.021451 | \n", "0.062757 | \n", "