{ "cells": [ { "cell_type": "markdown", "metadata": { "id": "mzsjxPxPuRYX" }, "source": [ "# Contextual bandit with changing context\n", "> Customizing the context and changing it midway to see how fast the agent can adapt to the new context and start recommending better products as per the context\n", "\n", "- toc: true\n", "- badges: true\n", "- comments: true\n", "- categories: [contextual bandit]\n", "- image: " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "7SCDe05G1QSx", "outputId": "aa23d3f1-5df9-43df-e742-4710a370c32f" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\u001b[K |████████████████████████████████| 2.6MB 7.0MB/s \n", "\u001b[?25h" ] } ], "source": [ "!pip install -q vowpalwabbit" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "U17IKmm51Sa_" }, "outputs": [], "source": [ "from vowpalwabbit import pyvw\n", "import random\n", "import matplotlib.pyplot as plt\n", "import pandas as pd\n", "from itertools import product" ] }, { "cell_type": "markdown", "metadata": { "id": "Mh7X3wHlrvk7" }, "source": [ "### Setting the context" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "AFdEwthO1U6N" }, "outputs": [], "source": [ "USER_LIKED_ARTICLE = -1.0\n", "USER_DISLIKED_ARTICLE = 0.0" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Joz5em0y7kfi" }, "outputs": [], "source": [ "users = ['A','B','C']\n", "items = ['Item1','Item2','Item3','Item4','Item5','Item6']\n", "context1 = ['morning','evening']\n", "context2 = ['summer','winter']\n", "\n", "context = pd.DataFrame(list(product(users, context1, context2, items)), columns=['users', 'context1', 'context2', 'items'])\n", "context['reward'] = 0\n", "\n", "#user 1 likes Item 1 in morning, and Item 6 in summer\n", "context.loc[(context.users=='A') & \\\n", " (context.context1=='morning') & \\\n", " (context['items']=='Item1'), \\\n", " 'reward'] = 1\n", "context.loc[(context.users=='A') & \\\n", " (context.context2=='summer') & \\\n", " (context['items']=='Item6'), \\\n", " 'reward'] = 1\n", "\n", "#user 2 likes Item 2 in winter, and Item 5 in summer morning\n", "context.loc[(context.users=='B') & \\\n", " (context.context2=='winter') & \\\n", " (context['items']=='Item2'), \\\n", " 'reward'] = 1\n", "context.loc[(context.users=='B') & \\\n", " (context.context1=='morning') & \\\n", " (context.context2=='summer') & \\\n", " (context['items']=='Item5'), \\\n", " 'reward'] = 1\n", "\n", "\n", "#user 3 likes Item 2 in morning, Item 3 in evening, and item 4 in winter morning\n", "context.loc[(context.users=='C') & \\\n", " (context.context1=='morning') & \\\n", " (context['items']=='Item2'), \\\n", " 'reward'] = 1\n", "context.loc[(context.users=='C') & \\\n", " (context.context1=='evening') & \\\n", " (context['items']=='Item3'), \\\n", " 'reward'] = 1\n", "context.loc[(context.users=='C') & \\\n", " (context.context1=='morning') & \\\n", " (context.context2=='winter') & \\\n", " (context['items']=='Item4'), \\\n", " 'reward'] = 1\n", "\n", "context['cost'] = context['reward']*-1\n", "\n", "contextdf = context.copy()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "rpgSzAFjv_Rh", "outputId": "7398f2a8-d5c0-4281-e98d-7c08b6363373" }, "outputs": [ { "data": { "text/plain": [ " 0 60\n", "-1 12\n", "Name: cost, dtype: int64" ] }, "execution_count": 5, "metadata": { "tags": [] }, "output_type": "execute_result" } ], "source": [ "contextdf.cost.value_counts()" ] }, { "cell_type": "markdown", "metadata": { "id": "_nTz89_drrWM" }, "source": [ "### Cost function util" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "QlOd83hd1Z3F" }, "outputs": [], "source": [ "def get_cost(context,action):\n", " return contextdf.loc[(contextdf['users']==context['user']) & \\\n", " (contextdf.context1==context['context1']) & \\\n", " (contextdf.context2==context['context2']) & \\\n", " (contextdf['items']==action), \\\n", " 'cost'].values[0]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "6P_HYhhW8UWO", "outputId": "06eb7e38-ede8-4013-8767-8e230092bb1a" }, "outputs": [ { "data": { "text/plain": [ "0" ] }, "execution_count": 7, "metadata": { "tags": [] }, "output_type": "execute_result" } ], "source": [ "get_cost({'user':'A','context1':'morning','context2':'summer'},'Item2')" ] }, { "cell_type": "markdown", "metadata": { "id": "wws6kC3ur2R0" }, "source": [ "### Vowpalwabbit format util" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "pTCHTNFf3jRe" }, "outputs": [], "source": [ "# This function modifies (context, action, cost, probability) to VW friendly format\n", "def to_vw_example_format(context, actions, cb_label = None):\n", " if cb_label is not None:\n", " chosen_action, cost, prob = cb_label\n", " example_string = \"\"\n", " example_string += \"shared |User users={} context1={} context2={}\\n\".format(context[\"user\"], context[\"context1\"], context[\"context2\"])\n", " for action in actions:\n", " if cb_label is not None and action == chosen_action:\n", " example_string += \"0:{}:{} \".format(cost, prob)\n", " example_string += \"|Action items={} \\n\".format(action)\n", " #Strip the last newline\n", " return example_string[:-1]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "k8dLifwF3eDJ", "outputId": "29547726-b54f-42bf-cfbd-190195a43243" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "shared |User users=A context1=morning context2=summer\n", "|Action items=Item1 \n", "|Action items=Item2 \n", "|Action items=Item3 \n", "|Action items=Item4 \n", "|Action items=Item5 \n", "|Action items=Item6 \n" ] } ], "source": [ "context = {\"user\":\"A\",\"context1\":\"morning\",\"context2\":\"summer\"}\n", "\n", "print(to_vw_example_format(context,items))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "dM_aaYyb5xe9" }, "outputs": [], "source": [ "def sample_custom_pmf(pmf):\n", " total = sum(pmf)\n", " scale = 1 / total\n", " pmf = [x * scale for x in pmf]\n", " draw = random.random()\n", " sum_prob = 0.0\n", " for index, prob in enumerate(pmf):\n", " sum_prob += prob\n", " if(sum_prob > draw):\n", " return index, prob" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "1xeAG23o2EGj" }, "outputs": [], "source": [ "def get_action(vw, context, actions):\n", " vw_text_example = to_vw_example_format(context,actions)\n", " pmf = vw.predict(vw_text_example)\n", " chosen_action_index, prob = sample_custom_pmf(pmf)\n", " return actions[chosen_action_index], prob" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "az_BHiJ32EwX" }, "outputs": [], "source": [ "def choose_user(users):\n", " return random.choice(users)\n", "\n", "def choose_context1(context1):\n", " return random.choice(context1)\n", "\n", "def choose_context2(context2):\n", " return random.choice(context2)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "XqWD3pqt2GTr" }, "outputs": [], "source": [ "def run_simulation(vw, num_iterations, users, contexts1, contexts2, actions, cost_function, do_learn = True):\n", " cost_sum = 0.\n", " ctr = []\n", "\n", " for i in range(1, num_iterations+1):\n", " user = choose_user(users)\n", " context1 = choose_context1(contexts1)\n", " context2 = choose_context2(contexts2)\n", "\n", " context = {'user': user, 'context1': context1, 'context2': context2}\n", " # print(context)\n", " action, prob = get_action(vw, context, actions)\n", " # print(action, prob)\n", "\n", " cost = cost_function(context, action)\n", " # print(cost)\n", " cost_sum += cost\n", "\n", " if do_learn:\n", " # 5. Inform VW of what happened so we can learn from it\n", " vw_format = vw.parse(to_vw_example_format(context, actions, (action, cost, prob)),pyvw.vw.lContextualBandit)\n", " # 6. Learn\n", " vw.learn(vw_format)\n", " # 7. Let VW know you're done with these objects\n", " vw.finish_example(vw_format)\n", "\n", " # We negate this so that on the plot instead of minimizing cost, we are maximizing reward\n", " ctr.append(-1*cost_sum/i)\n", "\n", " return ctr" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "fPDQ5H512KEj" }, "outputs": [], "source": [ "def plot_ctr(num_iterations, ctr):\n", " plt.plot(range(1,num_iterations+1), ctr)\n", " plt.xlabel('num_iterations', fontsize=14)\n", " plt.ylabel('ctr', fontsize=14)\n", " plt.ylim([0,1])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 287 }, "id": "wPhRpLyo2MmF", "outputId": "51f7cefd-986b-4366-c767-f9b6938a432c" }, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAEOCAYAAACNY7BQAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXRc5X3/8fd3RvtiW7blBe+AWRyg4DqGUAImkLClkOXXAiWnScoP2iTkJGTpgeZXmpI00DRtSprQhGbfIDTN4iZOgcRkTygGbMCsNnjFtixbsnZplu/vj3slz8gjW7I1d0a6n9c5Opq76M73yvJ85j7PM881d0dERGRQotQFiIhIeVEwiIhIHgWDiIjkUTCIiEgeBYOIiORRMIiISJ7IgsHMvmxmLWb29Ajbzcw+Y2abzOxJM1seVW0iInJQlFcMXwUuPcz2y4Cl4deNwL9HUJOIiAwTWTC4+y+B/YfZ5Srg6x74PTDNzOZGU52IiAyqKHUBOeYB23OWd4Trdg3f0cxuJLiqoL6+/g9POeWUMT/Z3s5+dnf08arjppAwO7qKRUQmqMcee6zV3ZsLbSunYBg1d78HuAdgxYoVvm7dujEf4wu/2MwdP3mOX99+CXVVE/LXICJy1Mxs60jbymlU0k5gQc7y/HCdiIhEqJyCYTXw5+HopHOAA+5+SDPSeNMcgiIi+SJrQzGze4FVwEwz2wH8HVAJ4O6fB9YAlwObgB7gncWtp5hHFxGZuCILBne/9gjbHXhPROWIiMgIyqkpqSTUkiQiki+2wWCoLUlEpJDYBoOIiBSmYBARkTyxDwbd81pEJF9sg0HDVUVECottMIiISGGxDwY1JImI5It9MIiISD4Fg4iI5FEwiIhIntgHg0ariojki20wmMariogUFNtgEBGRwhQMakoSEckT22BQQ5KISGGxDQYRESlMwSAiInliHwyuTgYRkTyxDQaNVhURKSy2wSAiIoXFPhj0yWcRkXyxDQa1JImIFBbbYBARkcIUDCIikif2waAuBhGRfLENBs2uKiJSWGyDQURECot9MLjGq4qI5IltMKglSUSksNgGg4iIFKZgEBGRPLEPBvUwiIjki20wqItBRKSw2AaDiIgUFmkwmNmlZva8mW0ys1sKbF9oZg+b2RNm9qSZXV7smjRaVUQkX2TBYGZJ4HPAZcAy4FozWzZst/8H3O/uZwHXAHcXsaCiHVpEZCKL8ophJbDJ3V9y9wHgPuCqYfs4MCV8PBV4JcL6RESEaINhHrA9Z3lHuC7XR4G3mdkOYA3w3kIHMrMbzWydma3bu3dvMWoVEYmtcut8vhb4qrvPBy4HvmFmh9To7ve4+wp3X9Hc3HxMT+gasCoikifKYNgJLMhZnh+uy3U9cD+Au/8OqAFmFqMY9TCIiBQWZTA8Ciw1syVmVkXQubx62D7bgIsAzOxUgmBQW5GISIQiCwZ3TwM3AQ8AzxKMPtpoZreb2ZXhbh8EbjCzDcC9wDu82NOfqiVJRCRPRZRP5u5rCDqVc9fdlvP4GeCPoqhFo1VFRAort85nEREpMQWDiIjkiX0wqItBRCRfbIPBNGBVRKSg2AaDiIgUFvtg0OyqIiL5YhsMGq4qIlJYbINBREQKUzCIiEie2AeDZlcVEckX22BQF4OISGGxDQYRESks9sGg4aoiIvliGwwarioiUlhsg0FERApTMIiISJ7YB4O6GERE8sU2GDS7qohIYbENBhERKSz2weAaryoikie+waCWJBGRgipKXYCISLnqS2XoT2WpqUpQXZEsWR2ZrJNMGJmss31/Dzvbe9nZ3svyhU2cOKth3J8v9sGgliSRycHd6exP01hdgZlxoDfF+u3ttHT04cCO/T20dPZTU5mkZyBNU10VDhzoSdE9kMaB1s5+ugfSJMx4pb2P1q5+ABIGU2srcaC5oZqmuiraegZorKmgqz9NRSJBJuscN62GeU219KWyGNCTypA0o766gqm1ldRXJUkkguaK3Qf6ONCbIpXJUluZZH/PAFtau2msqaQiabR1D/BKex9ZdzLuNDdU09GXoi+VHTrnj7/pNAXDeFJLkkh0+tMZqpJBy7WF0w64Oz0DGTr6Ury0t5uO3hT96SxN9VVUJg3DmFZXydZ93eztGqAyYbR29WNm1FUlOdCborWrn637eujoTfHCni56UxlqK5Oks1lSmfx3fQmD6fXV9KUyVFUkONCbAqCprorGmgrS2SzNDdU0N1STzjonzW5k8Yw6qiuSdPSl2N89QDrj7Ovup70nxdxptfT0p1k4vZ50NktFwti+v5d1W9uor6oglcnSWFNBxp2O3jQ9A+m8mhqrK5jRUEUyYXT3Z5hWV8mrjpvKgd4UWXfmzJ3C606ZTWWFkTBjX1c/9dUVnDpnCsdNq2XRjDqa6quK8u8V22AQiSt3Z8u+Htp6BmiorqCzL0VzQw1zp9VQGb54D6SzvNTaxdTaSuZMqRl6MR/UO5Chtauf/d0D7Onoo61ngJrKJE/tOMCezn5aOvro6EsDsHVfNz0DQTAkE0ZFwuhLZ3CHdPbYLtmn1FSwZGY9NZVJrn71AmY2VNHS2U9tZZIptZWctWAac6fVkjCYPaWGmsqDzUED6WwQQBHOj9OXygCQymRpCK9sypGCQWSC6U9nGEhnMTMG0lme2NZGR1+KZCJBS0cfW/f10NWfZtv+Hna19zKzsZr+VJbmxmpau/rZ2dZLZ3/6kOMmE8acKTVUJI1dB/oYSAdNFjPqq6itSpLKZFk8o56OvjTP7e4o2AxbXZGgubGamQ3VzGqsBuDsJdOZWltJR1+KgXSWrDuVyQQViQTT6iqZXl/F4hn1TK2tpLLC2N89AEA64+zvHmDRjDrmTq0l4870uioy7vQMpJlRX00ycfQvrFUV0Y+9GQym3IAqRwoGkQj1pzO0dARNESfPaSSZsLwXt75Uhu7+NOms88S2Nna09dLRm2Jnex/b9/ewo62HPZ39ZA7zTrumMkFDdQULptexdHYjbT0DzGioorWrn5kN1Zy1sIllcxuZXl891PTS3jvAi3u62NvVTyqd5fylzZw0p5GuvjQbtreH7+yd7ft7mdlYxU0XnsicqTVMr6uiubGa+uoKsu4sndUYyQtuQ7Veuooptr/dcr2Ek4npQG+Knz6zh+d2d/D8ni6qKxLMaqwm67C5pYtt+3vY3zMw9C58uNlTqqlMJtjR1ltwe2N1BfOaajlrUROzG2toqqskkTDM4PR5U2mqq6KzL838ploWTK8r5qlKDMQ2GESORn86Q0dvmu1tPTy/u5N1W9rYsKOdTS1dQ/s0N1bTN5ChJ5XB3TlxVgMLZ9RxyatmM7W2kql1VVRXJHhmV9AcU12RYEdb0PxzxRlzaaqrImnGq46bwuKZ9TQ3Vg+1/YtEIfbBoOGqUoi788qBPrbu66a9J8WG7e384oW9PLe7M2+/ZMI4e8l03rBsNq9d2sypcxuZVheMFMlknVQmW/btySLDxTYY1JAkAM/t7uDlvd048NjWNh56Zg/d/WlqKpPsbM9v1jm+uZ43nzWP46bVML+pjpNmN3LmgmkjdoAG/QcKBZl4YhsMMrmlMlme3NHOmQuaeLGlkye2tbO5pYuMO5ms09LRzyMv76OtJzX0M2Zw7gkzmFpbSTrjXHfOQqbVVtFQU8GyuY2cOKuxhGckEh0Fg0wI2azTNZCmpz/D1n3dHDctv5N1MAge29rGr15s5Vcvth72eHVVSf5wUROrTp7F/KZaqpIJTp07hTlTa4p9KiJlL/bB4LpVT8k9vfMAP312D5XJBNedvZBnXungm49sZdv+oIN3+CdYBx3fHIx937avh33h2HcIRvi8+ax5zJ5SQ2dfioXT61i5ZDpLZtazr3uAabWVzGiojur0RCacSIPBzC4F7gKSwBfd/c4C+/wp8FGCm6ttcPc/K04txTiqALR09LF6wyskzFixuInT502loy/N5r1dfPD+DcybVssLezpp6ew/5Gf/6YHnD1l35oJpnNDcQEXCmDO1hrlTa/jh+ldo702xfX8vS2c38IaZ9Zw8u5GLTp3N/KbaEYcjD3YMi8jIIgsGM0sCnwNeD+wAHjWz1e7+TM4+S4FbgT9y9zYzmxVVfXLsfru5lXd98/GhOWhG8nJrd97ytSsXcPGps3lg426e39NFZcK4+tULuPCUWcwc4Z39NSsXjlvdIpIvyiuGlcAmd38JwMzuA64CnsnZ5wbgc+7eBuDuLcUuSsNVj90P1+/kffetH1quSia4duUCNuw4wJM72hn8kO6bz5rHzRefxNTaSqoqEtRW5Y/YuejU2VGWLSIjiDIY5gHbc5Z3AGcP2+ckADP7DUFz00fd/X+GH8jMbgRuBFi48OjeOaop6eikM1kqkgnu/d9tbNjezo+f2kVnOFnaa5fO5J//5A+YNUUduCITWbl1PlcAS4FVwHzgl2Z2uru35+7k7vcA9wCsWLFC7/kjcNO3H+dHT+4quO2vLjiB//vaJSM2+4jIxBJlMOwEFuQszw/X5doBPOLuKeBlM3uBICgejaZEGe7pnQd447/9uuC2my48kavOPI6lszW+X2QyiTIYHgWWmtkSgkC4Bhg+4ugHwLXAV8xsJkHT0kvFLEqXG/myWedXm1p5YXcn/7Dm2aH1i2bUsfaDq45pmmMRmRgiCwZ3T5vZTcADBP0HX3b3jWZ2O7DO3VeH295gZs8AGeDD7r6vGPWYJsU4xLcf2cbffP+pQ9Z//m3LufS0uSWoSERKIdI+BndfA6wZtu62nMcOfCD8kiJr7xlgzVO7aesZOOTzAzPqq7j59SfxtnMWlag6ESmVcut8jpzHZLxqfzrD3Q9v5nWnzOKTDzzHbzYVvhB78ObzOUl9BiKxFttgiMNw1b2d/Xz/iR18Ys1zQ+vu+tmLBfe994ZzeM0JM6IqTUTKWGyDYTJzd5bcumbE7afNm8LrTp7Fuy88UfcKEJFDKBgmiQM9Ka770u95emfHIdvu/8vXUJk0zlrYVILKRGSiiX0wTIYehse3tfGWu397yPqNf38J9bppuoiMkV41JqiPfP8pvvXItkPWf+fGczj7ePUViMjRUzBMIJ19KT67dhNVFYlDQuGWy07hry44oUSVichkEvtgmAijVXsG0iy77YFD1v/hoiZWLGriw5ecTEUyUYLKRGQyim0wjHQjl3LTl8oUDIU733K67kkgIkUxqmAws0qCKbMvcveNxS1JBv34yV2859uPDy1vufOKElYjInExqvaHcLbTFJNjEM+E0NmXyguFjX9/SQmrEZE4GUvD9L8Bt5rZJGt+Kr+s++3mVk7/6IMAnLVwGi/fcbmGnYpIZMbyavNa4AJgp5k9DeTduNfdrxzPwoqtHHsYdh/oo6Gmgj/7j0eG1n3vXedOmP4QEZkcxhIMrcB/FauQuNvX1c85d/wsb90LH79MoSAikRt1MLj7O4tZSKmUw3DVlo6+Q+6DsPkTl+umOCJSEqMOBjNbC7xl+P2XzWwK8AN3f914F1dM5fBGvLWrnxUf/2neugdvPp/jptUqFESkZMbS+bwKqCqwvoag/0FG4SdP7WJfVz+ZrB8SCu+58AROmt1IgzqaRaSEjvgKZGbLcxbPMLP9OctJ4BKCezjLMD97dg97O/vZ3dHHH//Bcdy/bjtf+MWht7C++7rlXHbaHPUniEhZGM1b03UEYzodeLDA9l7gveNZVJSK1cXg7lz/tXVDy//608I3yPnRe8/jtHlTi1SFiMjYjaYpaQlwAsEIz1eHy4Nf84Ap7v7lolVYJFbkAav/8/TuguvnTKnh369bTsLgpx84X6EgImXniMHg7lvdfQtwJ/DqcHnwaxdwg5l9rNiFTjRf+90WABbPqOP77z53aP0D7z+fy06fy0t3XMGJs3RvZREpP2Pp5Xwb8CcF1j8O3Ar87bhUFLHxHK66fX8PN3x9Hc/t7hxa9/MPXwjALz68ipdbu5laVzl+TygiUgRjCYZZwN4C61uB2eNTTnSK0c/72k8+nLecO+R00Yx6Fs2oH/8nFREZZ2MZrrqNwsNSzwd2jE85E9fiW358yLrNn7i8BJWIiBybsVwxfAH4tJlVAWvDdRcBdwD/ON6FTVTvu2gpN7/+pFKXISJy1MYyJcY/m9lM4DMc/KDbAHCXu3+yGMVFwcdhwOrj29oAmNVYrVAQkQlvTB+xdfdbzezjwLJw1bPu3jX+ZRXfeHYxfPqhFwA4eY5GGYnIxDfmuRfcvRt4tAi1TFi/erEVgK++c2WJKxEROXaxv4P8sQ5XbensG3qsie9EZDKIbTCM13DVz67dBASdziIik0Fsg2G8rN7wCgB/dcEJJa5ERGR8KBjG6PFtbfzoyVeGlmfUV3HxqbOorUqWsCoRkfET+4n/x9rH8Ja7fwvAFafPpa0nxea93bzxjOOKUJmISGnE+Irh2DoZ9nUPsPxjDwEwvb7Q/YtERCam2F8xjNauA73c+ZPnhpaf2HbwDqdvWT6vFCWJiBRFpFcMZnapmT1vZpvM7JbD7PdWM3MzW1Hsmkb7yefX3LGWH64/2Ldww9eDm/CsWNREY41mTBWRySOyYDCzJPA54DKCT05fa2bLCuzXCLwPeKS49YzPcd50lq4WRGRyifKKYSWwyd1fcvcB4D7gqgL7fYxgUr6+AttK5twTZgw9XnVy89DjP1u5sBTliIgUTZR9DPOA7TnLO4Czc3cws+XAAnf/sZl9eKQDmdmNwI0ACxcW/4XZ3fnt5n2cPLuRB24+H4C//cHTmEFCn3YWkUmmbDqfzSwB/AvwjiPt6+73APcArFix4pgmtRjNcNXt+3sBeH7PwTuzfexNpx3L04qIlK0om5J2AgtylueH6wY1AqcBPzezLcA5wOpidUCP5X1+a3c/AO9epU83i8jkF2UwPAosNbMl4c1+rgFWD2509wPuPtPdF7v7YuD3wJXuvi7CGgu6+TvrAfRBNhGJhciCwd3TwE3AA8CzwP3uvtHMbjezK6Oq42hs3dcDwJypNSWuRESk+CLtY3D3NcCaYetuG2HfVcWsxY5ivGpTnT6vICKTX4ynxBiddCYLwPKF044qTEREJprYB8ORRiX9alNwd7aegUwE1YiIlF5sg2G07/0T4VXCB15/UvGKEREpI7ENhtHa1xUMVV06u7HElYiIREPBcAStYTDMbNDU2iISD7EPhiPNrrphxwEAGqrL5kPiIiJFFdtgGO0Aox8/uSvcXyOSRCQeYhsMIiJSWOzbR440XHVWYzUXnjwrmmJERMpAbK8YRtMylM06LZ39zFDHs4jESGyDYTQu/8yvAFi3pa3ElYiIREfBcBjP7Q7uv7Bgel2JKxERiU7sg2E0d/n51J+cUfQ6RETKRWyDwY4wKcYr7b0H99VQVRGJkdgGw5Gce+faUpcgIlISsQ8GP8J41evPWxJRJSIi5SG+wXCE1qGFYYfzO85dXPxaRETKSOw/4DaSE2c1MKW2QiOSRCR24nvFcARd/Wnqq5SbIhI/sQ+GkXoYuvvTmlFVRGIptsFwpAGoPQMZ6hQMIhJDsQ2GI+nqT9NQnSx1GSIikYt9MIw0WrWnP02d+hhEJIZiGwyH+zTz/u4BugcyR5ySW0RkMoptMBzO3Q9vAuB7T+wocSUiItFTMAyzqaWLL/76ZQDeunx+iasREYlebINhsCFp+JQYX/nNy0OPP3zJyRFWJCJSHmIbDImwjyE3FvpSGb71yLah5ZpKjUoSkfiJcTAE3zPZg9HwX48f7FM4bmpN1CWJiJSF2I7HTITJkA2bkj5w/3q+9/jOoe3ffde5JalLRKTUYnzFEDYlhRcMuaEAakYSkfiKcTAE33ObknJNr6+KsBoRkfIR22AY/IBb1v2QkUlrP3hBKUoSESkLsQ2GZOJgU9KvN7XmbRvhIkJEJBYiDQYzu9TMnjezTWZ2S4HtHzCzZ8zsSTP7mZktKlYtg01JwRVD/rZFM3RzHhGJr8iCwcySwOeAy4BlwLVmtmzYbk8AK9z9DOC7wCeLVc9g53Mm69TnzKJ6xvypVCZjeyElIhLpFcNKYJO7v+TuA8B9wFW5O7j7w+7eEy7+HijanBSDwbCve4Cv/GYLAM2N1Xz66jOL9ZQiIhNClJ9jmAdsz1neAZx9mP2vB35SaIOZ3QjcCLBw4cKjKiYRRuKt33tqaN1dV5/JCc0NR3U8EZHJoizbTMzsbcAK4J8KbXf3e9x9hbuvaG5uPqrnSBSYdvuFPZ1HdSwRkckkyiuGncCCnOX54bo8ZnYx8BHgAnfvL1YxiQK3Y5jZWF2spxMRmTCivGJ4FFhqZkvMrAq4Blidu4OZnQV8AbjS3VuKWczgFUNVTkfzxafOLuZTiohMCJEFg7ungZuAB4BngfvdfaOZ3W5mV4a7/RPQAPynma03s9UjHO6YDQZDXc6IpCqNRhIRiXYSPXdfA6wZtu62nMcXR1XLYDDUV1XQ3pMK1hVqXxIRiZnYvkUeHJVUXRHbX4GISEGxfVUc+oDb8I89i4jEXOyDIZ1RMIiI5IpxMATfU5lsaQsRESkz8Q2GMBkUDCIi+eIbDMOakpbMrC9lOSIiZSPGwRB8T2WzVCUTPPyhVSWtR0SkXET6OYZyMtiU1JdSU5KISK4YXzHow2wiIoXEOBhKXYGISHmKcTAoGUREColtMGjCPBGRwmL76qgJ80RECottMIiISGEKBhERyaNgEBGRPAoG4NqVC468k4hITCgYgHecu6TUJYiIlA0FA9BQE9uZQUREDqFgADRwVUTkIAUD+hS0iEguBQOgXBAROUjBgIJBRCRXrIOhMhkkgqmXQURkSKyDobGmEgB3L3ElIiLlI9bBsOqkZkAT6omI5Ir1AP473no6N5x/PDMbqktdiohI2Yj1FUN1RZJT504pdRkiImUl1sEgIiKHUjCIiEgeBYOIiORRMIiISB4Fg4iI5FEwiIhIHgWDiIjkiTQYzOxSM3vezDaZ2S0Ftleb2XfC7Y+Y2eIo6xMRkQiDwcySwOeAy4BlwLVmtmzYbtcDbe5+IvBp4B+jqk9ERAJRXjGsBDa5+0vuPgDcB1w1bJ+rgK+Fj78LXGSmSbFFRKIU5VxJ84DtOcs7gLNH2sfd02Z2AJgBtObuZGY3AjeGi11m9vxR1jRz+LFjQOccDzrneDiWc1400oYJOYmeu98D3HOsxzGzde6+YhxKmjB0zvGgc46HYp1zlE1JO4EFOcvzw3UF9zGzCmAqsC+S6kREBIg2GB4FlprZEjOrAq4BVg/bZzXw9vDx/wHWuu6iIyISqciaksI+g5uAB4Ak8GV332hmtwPr3H018CXgG2a2CdhPEB7FdMzNUROQzjkedM7xUJRzNr0hFxGRXPrks4iI5FEwiIhIntgGw5Gm55hIzOzLZtZiZk/nrJtuZg+Z2Yvh96ZwvZnZZ8LzftLMluf8zNvD/V80s7cXeq5yYGYLzOxhM3vGzDaa2fvC9ZP5nGvM7H/NbEN4zn8frl8STh+zKZxOpipcP+L0MmZ2a7j+eTO7pDRnNHpmljSzJ8zsR+HypD5nM9tiZk+Z2XozWxeui/Zv291j90XQ+b0ZOB6oAjYAy0pd1zGcz/nAcuDpnHWfBG4JH98C/GP4+HLgJ4AB5wCPhOunAy+F35vCx02lPrcRzncusDx83Ai8QDDNymQ+ZwMawseVwCPhudwPXBOu/zzwrvDxu4HPh4+vAb4TPl4W/r1XA0vC/wfJUp/fEc79A8C3gR+Fy5P6nIEtwMxh6yL9247rFcNopueYMNz9lwSjuHLlTi/yNeBNOeu/7oHfA9PMbC5wCfCQu+939zbgIeDS4lc/du6+y90fDx93As8SfGp+Mp+zu3tXuFgZfjnwOoLpY+DQcy40vcxVwH3u3u/uLwObCP4/lCUzmw9cAXwxXDYm+TmPINK/7bgGQ6HpOeaVqJZime3uu8LHu4HZ4eORzn1C/k7C5oKzCN5BT+pzDptU1gMtBP/RNwPt7p4Od8mtP296GWBwepkJdc7AvwJ/DWTD5RlM/nN24EEze8yC6X8g4r/tCTklhoyNu7uZTbpxyWbWAPwX8H5377Cc+RYn4zm7ewY408ymAd8HTilxSUVlZm8EWtz9MTNbVep6InSeu+80s1nAQ2b2XO7GKP6243rFMJrpOSa6PeElJeH3lnD9SOc+oX4nZlZJEArfcvfvhasn9TkPcvd24GHgNQRNB4Nv8HLrH2l6mYl0zn8EXGlmWwiae18H3MXkPmfcfWf4vYXgDcBKIv7bjmswjGZ6jokud3qRtwM/zFn/5+FohnOAA+El6gPAG8ysKRzx8IZwXdkJ242/BDzr7v+Ss2kyn3NzeKWAmdUCryfoW3mYYPoYOPScC00vsxq4JhzBswRYCvxvNGcxNu5+q7vPd/fFBP9H17r7dUziczazejNrHHxM8Df5NFH/bZe6B75UXwS9+S8QtNN+pNT1HOO53AvsAlIEbYnXE7St/gx4EfgpMD3c1whumLQZeApYkXOcvyDomNsEvLPU53WY8z2PoB32SWB9+HX5JD/nM4AnwnN+GrgtXH88wYvcJuA/gepwfU24vCncfnzOsT4S/i6eBy4r9bmN8vxXcXBU0qQ95/DcNoRfGwdfm6L+29aUGCIikieuTUkiIjICBYOIiORRMIiISB4Fg4iI5FEwiIhIHgWDyBGY2WIzczMrqxvNl2tdMvEpGESObDvBjK7rAcxsVfiCPDOqAszs52b22cPVJTJeNFeSyBF4MEfR7mIc28wq3T11ND9bzLok3nTFIGUlfGd8t5l9wsxaLbgB0afMLBFu32JmHyrwM5/NWd5iZreZ2VfNrNPMtpvZ1WY2zczuM7Ou8OYlbxhlTUNNNuFsrg+Hm/aG678a7mdm9tdmttnMei242crbChznWjNba2a9wF+a2Qwzu9fMdoQ/t9HM3pnzc18FLgDeE/68h8c6pCnJzM634CY1fWa2x8w+HU77Mqrfb7jPWyy46Uuvme03s1+Y2eBsnhIDCgYpR9cBaeBc4Cbg/cDVYzzG+wmmRVhOcGOXrxHc7GUNcCbwS+CbZlYzxuNuB94aPn4VQVPO+8LljxNMR/IegpvD3AF8wcyuGHaMO4C7w31+QDCVwydKjhgAAAL+SURBVOPAG8Nj3hX+3EXh/u8Dfgd8JXy+ueRPqQyAmc0juGnLEwRTkV8PXBs+X64Rf79mNodgwrqvAacS3ATqG6P4vchkUuq5QfSlr9wv4OfA74atewj4Yvh4C/ChAj/z2ZzlLcC9OcsNBHMrfSZn3eJw3YpR1JS3L8G8PU7OXbaAeqAXeO2wn/1XYM2w43xwFM953+A5FzrHEer6B4K5dBI5+7wD6AfqRvn7XR4ec1Gp/xb0Vbov9TFIOXpy2PIrwKyjPYa7d5lZD8EkY4P2hN/HetyRLCN45/8/lj9XfiVBUOVal7tgZkmC2zVeTXAzlWqCW87+fIw1nAr83t2zOet+HR7rRA7+Tg73+91AMEnb02b2YPj4u+6+d4y1yASmYJByNLwz1jnY7JklmFEyV+Uoj5Eatgzj15w6eJw/BrYdoZbuYcsfAj5I0GT0FNAFfILxCy04eL6F6hn6/bp7Jux7OYdgqubrgTvM7AJ33zCO9UgZUzDIRLOXoI0dgLCP4BSCdvWoDITfkznrniFoslnk7mvHeLzzgP9292/A0P0mTgLahz1nssDP5noW+FMzS+RcNZwX/uzm0Rbj7k7Qp/E7M7udYPrnqwmuJiQGFAwy0awF/sLMVhOExEeI/u94K8G77CvM7L+BXnfvNLNPAZ8KX9h/SdC3cQ6Qdfd7DnO8F4Crzew8oBV4L7CE/LDbAqwMR0V1AfsLHOdugo7ku83sLoK5/e8k6JvoGc2JhTd7uZjgpi57CDqxFxAEn8SERiXJRHMHQTj8EHiQoA09yqsFPLj14t8RdPbuAQaHyv4t8FGCpqGNBJ26bwVePsIhP04wguonBIHSDXxr2D6fInjn/wxBIC4coa7LCF7M1wNfJriJ09+M4fQOENxS80cEHdn/DHzM3b85hmPIBKcb9YiISB5dMYiISB4Fg8SemX0+/DR0oa/Pl7o+kaipKUliz8xmAVNG2Nzh7i1R1iNSagoGERHJo6YkERHJo2AQEZE8CgYREcmjYBARkTz/H7mLwQFNE+6oAAAAAElFTkSuQmCC\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light", "tags": [] }, "output_type": "display_data" } ], "source": [ "# Instantiate learner in VW\n", "vw = pyvw.vw(\"--cb_explore_adf -q UA --quiet --epsilon 0.2\")\n", "\n", "num_iterations = 5000\n", "ctr = run_simulation(vw, num_iterations, users, context1, context2, items, get_cost)\n", "\n", "plot_ctr(num_iterations, ctr)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 287 }, "id": "TL81VOqK2Tbq", "outputId": "acae0673-e691-419d-e766-604b1a9807ef" }, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAEOCAYAAACNY7BQAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAfXElEQVR4nO3deZhcdb3n8fe3lt7T3Vk6kI0kkGAIPAiZBhFQELcAXnDGGZa5jujlER3FR0ccLrjgFRcWd0e8yNzLoCgiLvcaFS+KgNyrBGiWACEmNFlIQkg66Wy91vadP87ppE6nOulO0qe6U5/X89TTdZY69f3VU12fOr9z6nfM3RERERmQKHcBIiIytigYREQkQsEgIiIRCgYREYlQMIiISISCQUREImILBjO708y2mNkLQyw3M/uOmbWb2XNmtiiu2kREZK849xjuAhbvZ/n5wPzwdhXwjzHUJCIig8QWDO7+KNC5n1UuBn7ogaVAs5lNi6c6EREZkCp3AUVmAOuLpjeE8zYNXtHMriLYq6C+vv4/LViwIJYCRUSOFE899dRWd28ptWwsBcOwufsdwB0Ara2t3tbWVuaKRETGFzNbN9SysXRW0kZgVtH0zHCeiIjEaCwFwxLgfeHZSWcAO919n24kEREZXbF1JZnZT4BzgSlmtgH4PJAGcPfbgfuBC4B2oAf4QFy1iYjIXrEFg7tffoDlDnw0pnJERGQIY6krSURExgAFg4iIRCgYREQkQsEgIiIRCgYREYlQMIiISISCQUREIhQMIiISoWAQEZEIBYOIiEQoGEREJELBICIiEQoGERGJUDCIiEiEgkFERCIUDCIiEqFgEBGRCAWDiIhEKBhERCRCwSAiIhEKBhERiVAwiIhIhIJBREQiFAwiIhKhYBARkQgFg4iIRCgYREQkQsEgIiIRCgYREYlQMIiISISCQUREIhQMIiISoWAQEZEIBYOIiEQoGEREJCLWYDCzxWa20szazey6EsuPMbOHzewZM3vOzC6Isz4REYkxGMwsCdwGnA8sBC43s4WDVvsscJ+7nwpcBnwvrvpERCQQ5x7D6UC7u6929wxwL3DxoHUcaAzvNwGvxlifiIgQbzDMANYXTW8I5xX7B+C9ZrYBuB/4WKkNmdlVZtZmZm0dHR2jUauISMUaawefLwfucveZwAXA3Wa2T43ufoe7t7p7a0tLS+xFiogcyeIMho3ArKLpmeG8YlcC9wG4+2NADTAllupERASINxieBOab2VwzqyI4uLxk0DqvAG8FMLMTCIJBfUUiIjGKLRjcPQdcDTwArCA4+2i5md1oZheFq10DfNDMlgE/Ad7v7h5XjSIiAqk4n8zd7yc4qFw874ai+y8CZ8VZk4iIRI21g88iIlJmCgYREYlQMIiISISCQUREIhQMIiISoWAQEZEIBYOIiEQoGEREJELBICIiEQoGERGJUDCIiEiEgkFERCIUDCIiEqFgEBGRCAWDiIhEKBhERCRCwSAiIhEKBhERiVAwiIhIhIJBREQiFAwiIhKhYBARkQgFg4iIRCgYREQkQsEgIiIRCgYREYlQMIiISISCQUREIhQMIiISoWAQEZEIBYOIiEQoGEREJELBICIiEQoGERGJiDUYzGyxma00s3Yzu26IdS4xsxfNbLmZ3RNnfSIiAqm4nsjMksBtwNuBDcCTZrbE3V8sWmc+cD1wlrtvN7OpcdUnIiKBOPcYTgfa3X21u2eAe4GLB63zQeA2d98O4O5bYqxPRESINxhmAOuLpjeE84odDxxvZn82s6VmtrjUhszsKjNrM7O2jo6OUSpXRKQyjbWDzylgPnAucDnwf82sefBK7n6Hu7e6e2tLS0vMJYqIHNniDIaNwKyi6ZnhvGIbgCXunnX3NcAqgqAQEZGYxBkMTwLzzWyumVUBlwFLBq3zrwR7C5jZFIKupdUx1igiUvFiCwZ3zwFXAw8AK4D73H25md1oZheFqz0AbDOzF4GHgf/t7tviqlFERMDcvdw1HJLW1lZva2srdxkiIuOKmT3l7q2llo21g88iIlJmCgYREYlQMIiISISCQUREIhQMIiISoWAQEZEIBYOIiEQoGEREJELBICIiEcMKBjNLm9lrZnbiaBckIiLlNaxgcPcskAXG9/gZIiJyQCPpSvo/wPVmFtvlQEVEJH4j+ZB/E3AOsNHMXgC6ixe6+0UlHyUiIuPKSIJhK/CL0SpERETGhmEHg7t/YDQLERGRsWHYxxjM7KFS1182s0Yze+jwliUiIuUykoPP5wJVJebXEBx/EBGRI8ABu5LMbFHR5Mlm1lk0nQTeCWw83IWJiEh5DOcYQxvB7xcc+H2J5b3Axw5nUSIiUj7DCYa5gAGrgdMIzk4akAG2uHt+FGoTEZEyOGAwuPs6ADO7GTjN3W8vXm5mHzazGe7+uVGqUUREYjSSg8/vBZ4pMf9p4H2HpxwRESm3kQTDVKCjxPytwFGHpxwRESm3kQTDK5Q+LfXNwIbDU46IiJTbSIbE+D7wTTOrAgZ+0PZW4CbglsNdmIiIlMdIhsT4uplNAb7D3h+6ZYBvu/uto1GciIjEb0RDaLv79Wb2JWBhOGuFu3cd/rJERKRcRnxtBXfvBp4chVpERGQM0DWfRUQkQsEgIiIRCgYREYlQMIiISISCQUREIhQMIiISoWAQEZGIWIPBzBab2Uozazez6/az3nvMzM2sNc76REQkxmAwsyRwG3A+wS+nLzezhSXWmwB8HHg8rtpERGSvOPcYTgfa3X21u2eAe4GLS6z3RYJB+fpirE1EREJxBsMMYH3R9IZw3h5mtgiY5e6/3d+GzOwqM2szs7aOjlKXiBARkYM1Zg4+m1kC+AZwzYHWdfc73L3V3VtbWlpGvzgRkQoSZzBsBGYVTc8M5w2YAJwEPGJma4EzgCU6AC0iEq84g+FJYL6ZzQ0v9nMZsGRgobvvdPcp7j7H3ecAS4GL3L0txhpFRCpebMHg7jngauABYAVwn7svN7MbzeyiuOoQEZH9G/H1GA6Fu98P3D9o3g1DrHtuHDWJiEjUmDn4LCIiY4OCQUREIhQMIiISoWAQEZEIBYOIiEQoGEREJELBICIiERUdDB27+7nn8VfKXYaIyJgS6w/cxpqP3vM0T6zp5Kx5k5k9ub7c5YiIjAkVvcfQ2Z0BIJMrlLkSEZGxo6KDIWHB34KXtw4RkbGkwoMhSIaCKxlERAZUdDDYEMGwvrOHZet3lKMkEZGyq+iDzwNdST9+/BW+/O6T6Njdz+lf+eOe5WtvvrBMlYmIlE9F7zEMdCXd8/grPLhiC2fd8lBkeS6vg9IiUnkqPBj23t/RkyGbj3YpbdzRG3NFIiLlV9HBMHCMAeC5DTv3WX7OVx+JsRoRkbGhoo8xFP9+4e6l60quc9+T67nktFlxlTQs7s5jq7dx6qyJVKcSJIp3fYR8wXF3Usn9f+/pzeTJ5Ao8sbaTv7y8lZc7ulnd0cWu3iwT66uY0lBNXVWSmnSSiXVp6qpSTGuqIVdwuvtzLJzeSFdfjnWdPfzl5W1s784wsb6KloYquvvzbN7dx9GNNZy3YConz2xmQk2K/lyBvmyeU2Y1U5NO7re+vmye6lQi8gXmUPRm8mzc0Ut3f46jm2qoTiVIJxNs68rwSmcPjtOTybNx+9510skEO3oy5ArOjp4sHbv7qa1KMqWhijfNb+GkGU0k9f474piP81M1W1tbva2t7aAeO+e63w5rvb9+cfEB/4kPxiMrt3DGsZOpSSd5afNuLr1jKe9742w+/tb5+/0w+PufP8dP29ZH5l39lnn8r7cfP+b+SQsF53O/eoEfh0OPXH/+Ai5pnUVNOslT67bz+llNrNi0m9bZE/cEXKHgPPXKdhqqU7y2qw+AloZqpjZW09WXY9XmLpau3sajqzpoqkvTVJumuTZNXXWKrr4cf27fys7eLNOba5k9uQ4zozad4OSZzQA8saaT9i1d+3QVTq6vYuH0RmrSSTK5Apt29tKXLdCTyZPJ5dnVlxuynVMaqlk4vZG+TJ4N23sAOG5qA+u29fBKZ88+6ycTxpnHTeakGU0U3JneVEtvNs+ajm56s3lWbd7NX1/bTWNNinQyQTJhJBOGAVMba5jSUEV1KsnC6Y1s786wszdLwozG2hTTmmpxoCadYFpTDW1rt/PEmk6eXb+D3CH8aMcMJtZV0dWf2/Olygwm1VUxc1Idp82eGAROOklfJs/UxmqOa2lg9uQ6GqpT9GTy/PW1XcxoruOoxmrMjELByRYKvLazjxdf3cX2niy5QoGm2jSzJ9dTm04yqb6KZMJIGGzZ3U82XyBhxqT6KtZu7aY7k2NGcx0Jgx29WVa+tpvt3Rlq0knMoDqdZGZzLTXpJH3ZPOlkgqObagDYsjt4f9Wkk2za0Ud/Lk9jTZrJDVXUV6dorkvTUJ0iYYZ78JoG7Q7eq9l8gV29WZ5dv4Nt3Rm6+3NUp5JMqEnRm8lTHa7fWJumUPAgaHuzdHb1k8kXqK1K0ZfJ053J0Z8r0FiTpiadwB2mN9fQWJtmUn0V6WSCCdUpdvVlqatKYQZdfTmmN9dSX31w3+/N7Cl3by25TMGwr7s+cBpf//0qnt8Y7V668ORp3PbfFx3Ucw1wd360dB3PrN/BL5/eCMBnLzyBL/12xZCPWfb5d9BUmwbgku8/xhNrOodc95cfOZNFx0wEgg/YgQ/bf3+pg+nNtcydXH9Iexgbd/RSKDhTG6vJ5Ao0VKfI5Avkww+cN9/6MFu7Mge17blT6unP5nl1Z9+w1p9YlyaTKzCxvorNu/rI5p2m2jQLjp7AjOZatnZnWLetm86uDL3Z/J4PxYbqFItmT2T+1AZq00nmTW3gLa+bSlNder/P5+5s3NFLJldgWlMtz67fQcJgwdGN+33s2q3drNq8m119uT3/9E+t287DK7ewbtu+oTG5voppzTWcNmcSXX05cgWnN5MnkQj2hrZ2ZXhtZx+5QoHNu/oBqK9KkjCjK5Nj8L90wuDE6U0sOqaZE2c0UZ0K9hK6+oN1a6sSzJpYR0NNipp0kpaGao5uqmHD9l7AmVCTpiqZIJU0JtQE7dza1c+f27fy4qZddHZlWLutm2Xrd5IZ4oSNhEV/SNpYk6IqlWRrV/9+X/OxpiadIJt3jCDc3RmyzSOVStiIg/vGi0/kfW+cc1DPp2AYQqlg+B9nzOaL7z4Jd2fu9ffvs/yP15zDcS0NB/V8AP/2wmt8+EdPHfTjB/zLR87k9TObMYNXd/Zx1s0PDbnucS31vNzRXXLZSTMauek/n8ycKXVMqEmzvrOHD/6wjVveczJbu/qZN7WBtrXbueZnyw661rPnTeG9ZxzD2fNb+PQvn2fJslcjy98wdxIbd/SGH0SBaxe/DsPY0ZvhuJYG8gVny65+6quTTGuq5czjJjOxviqyneIgLGXN1m46dvdH9k7Kyd3J5AukEgk27ewlYca0ppoRdR11dmdoqk3v2VN0d7Z1Zyi4058tsH57D7Mn1zOjuXa0mrFHNl9gZ2+Wvmye+qoUm3b28UpnNy9t7qI3myeZMOYfNYHOrn5Wbt5Nf67AtKYaalJJmuvSLJzeyNFNtaQStqd7K5MvsK2rn4IHbWuZUE1VMkGu4HR2Z5g6oZqWCdVsCoOyqTbNvJYJHN1UQ28mT7ZQIJsv8OqOXnoyeWrSSbK5Ah1d/bhDc12adDJBTybPtKYa6qqS7OgNusz6cwU6u/rpzRbIFwq4w87eLOlUEO65fIFkwpjaWMPrjprAjIm1NFSnyBWC16G+Kug6dHd29QV7dNm8U5NOMKO5lup0kp5wDyOdMuqqUuzuy5LNB12hm3b2sasvy/buLNl8gd39OSZUp+jOBHuu9VUpXj+rmblTDm6cNwXDEEoFw4fefCzXX3ACEPzQ7U23PrzPOiP9fcNldzzG0tWd3PCuhdz4mxf3zE8ljI+cexzfeagdgFVfOp+qVIKXO7ro7s9hGB+95+l9uiJ+87GzOWlG0z7Ps7qji/O+/qcR1Xa4nT1vCt+67BSaa9MH7OMfyoE+4EXk0O0vGCr64HMpv3h6455gmF70LeuUWc08G/4auj+Xpzo1vGMO7Vu6WLo66PopDoW2z76NKQ3VAHzyHa+LPKZ4j+TRa9/Crr4sXgh2Wbd197Pg6MaSz3VsSwPLv/BO/v2lDt554tGYGc+u38Gnf/k81y5+Hecc30JXf45UIkEmX+ClzbtZ/uouPr9keWQ7p8+dxBNrOkkmjHzBqatK8vMPn8mxLfX7HGtx98N2cHSAQkGkvCp6j+H0Lz/Ilt379nEOtUdwxZ1P8KdVHXzhohPZvKuPaxcv2O/2123rLnnK65+vOy+WXXsRkaFoj2EIMybW7hMMZ82bPOT6X/tvr+e0Lz+45xt2dSrJh889lof/2sFdf1nDqcdM5O8XL2BnT5a/+e5/RLqAVn/lAiA4i+Nwf8MWETmcKjoY0ol9+8Bvec/JQ67fMqE6Mv3NB1fxzQdX7ZleurqTf3zk5X0ed+f7W9U9IiLjRkX/8rnUOf8Tqvd/yuKHzjl2RM/x5GfexnkLjhrRY0REyqmigyGV3BsMVeEZNLVV+z+ofP35J7D25gu5tWjP4ux5U7jijbP3TM+b2sCamy5g7c0X7rOXISIy1lV0V9LAHsO7Tp7GR86dxx9XbKYqNbysvOS0Wfz6uVc5dVbznrOKvnDxSbRv6eKYSXU6jiAi41ZFB4M7zJxYyzcuOYWqVIKF00ufBjqUu698wz7z5k09+B+/iYiMBRXdleTA5IbqYe8liIhUgor+RHQPxjwREZG9KjoYIHqxHhERqfBgKIzCcA4iIuNdrMFgZovNbKWZtZvZdSWWf9LMXjSz58zsj2Y2u9R2Dhd31JUkIjJIbMFgZkngNuB8YCFwuZktHLTaM0Cru58M/By4dTRrcg+GqBARkb3i3GM4HWh399XungHuBS4uXsHdH3b3gQGGlgIzR6uYXL7AY6u3RS7vKSIi8QbDDKD4epQbwnlDuRL4XakFZnaVmbWZWVtHR8dBFfOLpzcAsGzDzgOsKSJSWcbkwWczey/QCny11HJ3v8PdW929taWl5aCeozeTP4QKRUSOXHH+8nkjMKtoemY4L8LM3gZ8BjjH3UftgrClBtATEZF49xieBOab2VwzqwIuA5YUr2BmpwLfBy5y9y2jWo2OOouIlBRbMLh7DrgaeABYAdzn7svN7EYzuyhc7atAA/AzM3vWzJYMsblDph0GEZHSYh1Ez93vB+4fNO+Govtvi6sW0y8YRERKGpMHn+OgPQYRkdIqNxiUDCIiJVVsMKQUDCIiJVVsMOh0VRGR0io2GNLJim26iMh+Veyno/YYRERKq9hg0DEGEZHSKjcY1JUkIlJSxX46ao9BRKS0ig0GHWMQESmtYoMhoUH0RERKqthgcPdylyAiMiZVbDCIiEhpCgYREYmo2GBQR5KISGkVGwwiIlKagkFERCIqNhh0UpKISGkVGwwDrnn78eUuQURkTKn4YGidM6ncJYiIjCkVGwyu85JEREqq2GAYoJExRESiKj4YREQkqnKDQT1JIiIlVW4whNSTJCISVfHBICIiURUbDOpJEhEprWKDYYDptCQRkYiKDQYNiSEiUlrFBsMA7TCIiERVfDCIiEhUxQaDhsQQESmtYoNhgHqSRESiKj4YREQkqmKDQWcliYiUFmswmNliM1tpZu1mdl2J5dVm9tNw+eNmNmf0axrtZxARGV9iCwYzSwK3AecDC4HLzWzhoNWuBLa7+zzgm8Ato1VPTyY/WpsWERnX4txjOB1od/fV7p4B7gUuHrTOxcAPwvs/B95qo/TT5D+t2gJAOlmxvWkiIiWlYnyuGcD6oukNwBuGWsfdc2a2E5gMbC1eycyuAq4KJ7vMbOVB1jTl9bdEt10BpoDaXAHU5spwKG2ePdSCOIPhsHH3O4A7DnU7Ztbm7q2HoaRxQ22uDGpzZRitNsfZj7IRmFU0PTOcV3IdM0sBTcC2WKoTEREg3mB4EphvZnPNrAq4DFgyaJ0lwBXh/f8KPOSuE0tFROIUW1dSeMzgauABIAnc6e7LzexGoM3dlwD/DNxtZu1AJ0F4jKZD7o4ah9TmyqA2V4ZRabPpC7mIiBTTuZoiIhKhYBARkYiKDYYDDc8xnpjZnWa2xcxeKJo3ycz+YGYvhX8nhvPNzL4Ttvs5M1tU9JgrwvVfMrMrSj3XWGBms8zsYTN70cyWm9nHw/lHcptrzOwJM1sWtvkL4fy54fAx7eFwMlXh/CGHlzGz68P5K83sneVp0fCZWdLMnjGz34TTR3SbzWytmT1vZs+aWVs4L973trtX3I3g4PfLwLFAFbAMWFjuug6hPW8GFgEvFM27FbguvH8dcEt4/wLgdwQjjp8BPB7OnwSsDv9ODO9PLHfbhmjvNGBReH8CsIpgmJUjuc0GNIT308DjYVvuAy4L598O/M/w/keA28P7lwE/De8vDN/v1cDc8P8gWe72HaDtnwTuAX4TTh/RbQbWAlMGzYv1vV2pewzDGZ5j3HD3RwnO4ipWPLzID4B3F83/oQeWAs1mNg14J/AHd+909+3AH4DFo1/9yLn7Jnd/Ory/G1hB8Kv5I7nN7u5d4WQ6vDlwHsHwMbBvm0sNL3MxcK+797v7GqCd4P9hTDKzmcCFwD+F08YR3uYhxPrertRgKDU8x4wy1TJajnL3TeH914CjwvtDtX1cviZhd8GpBN+gj+g2h10qzwJbCP7RXwZ2uHsuXKW4/sjwMsDA8DLjqs3At4BrgUI4PZkjv80O/N7MnrJg+B+I+b09LofEkJFxdzezI+68ZDNrAH4BfMLdd1nReItHYpvdPQ+cYmbNwL8AC8pc0qgys3cBW9z9KTM7t9z1xOhsd99oZlOBP5jZX4sXxvHertQ9huEMzzHebQ53KQn/bgnnD9X2cfWamFmaIBR+7O6/DGcf0W0e4O47gIeBNxJ0HQx8wSuuf6jhZcZTm88CLjKztQTdvecB3+bIbjPuvjH8u4XgC8DpxPzertRgGM7wHONd8fAiVwC/Kpr/vvBshjOAneEu6gPAO8xsYnjGwzvCeWNO2G/8z8AKd/9G0aIjuc0t4Z4CZlYLvJ3g2MrDBMPHwL5tLjW8zBLgsvAMnrnAfOCJeFoxMu5+vbvPdPc5BP+jD7n733IEt9nM6s1swsB9gvfkC8T93i73Efhy3QiO5q8i6Kf9TLnrOcS2/ATYBGQJ+hKvJOhb/SPwEvAgMClc1wgumPQy8DzQWrSdvyM4MNcOfKDc7dpPe88m6Id9Dng2vF1whLf5ZOCZsM0vADeE848l+JBrB34GVIfza8Lp9nD5sUXb+kz4WqwEzi9324bZ/nPZe1bSEdvmsG3Lwtvygc+muN/bGhJDREQiKrUrSUREhqBgEBGRCAWDiIhEKBhERCRCwSAiIhEKBpEDMLM5ZuZmNqYuND9W65LxT8EgcmDrCUZ0fRbAzM4NP5CnxFWAmT1iZt/dX10ih4vGShI5AA/GKHptNLZtZml3zx7MY0ezLqls2mOQMSX8Zvw9M/uKmW214AJEXzOzRLh8rZl9qsRjvls0vdbMbjCzu8xst5mtN7NLzazZzO41s67w4iXvGGZNe7pswtFcHw4XdYTz7wrXMzO71sxeNrNeCy628t4S27nczB4ys17gQ2Y22cx+YmYbwsctN7MPFD3uLuAc4KPh4z3c1j5dSWb2ZgsuUtNnZpvN7JvhsC/Den3Ddf6LBRd96TWzTjP7k5kNjOYpFUDBIGPR3wI54EzgauATwKUj3MYnCIZFWERwYZcfEFzs5X7gFOBR4EdmVjPC7a4H3hPeP5GgK+fj4fSXCIYj+SjBxWFuAr5vZhcO2sZNwPfCdf6VYCiHp4F3hdv8dvi4t4brfxx4DPh/4fNNIzqkMgBmNoPgoi3PEAxFfiVwefh8xYZ8fc3saIIB634AnEBwEai7h/G6yJGk3GOD6KZb8Q14BHhs0Lw/AP8U3l8LfKrEY75bNL0W+EnRdAPB2ErfKZo3J5zXOoyaIusSjNvjFF1lC6gHeoE3DXrst4D7B23nmmE8570DbS7VxiHq+jLBWDqJonXeD/QDdcN8fReF25xd7veCbuW76RiDjEXPDZp+FZh6sNtw9y4z6yEYZGzA5vDvSLc7lIUE3/z/zaJj5acJgqpYW/GEmSUJLtd4KcHFVKoJLjn7yAhrOAFY6u6Fonn/EW5rHntfk/29vssIBml7wcx+H97/ubt3jLAWGccUDDIWDT4Y6+zt9iwQjChZLD3MbWQHTcPh604d2M7fAK8coJbuQdOfAq4h6DJ6HugCvsLhCy3Y295S9ex5fd09Hx57OYNgqOYrgZvM7Bx3X3YY65ExTMEg400HQR87AOExggUE/epxyYR/k0XzXiTospnt7g+NcHtnA79297thz/Umjgd2DHrOZInHFlsBXGJmiaK9hrPDx7483GLc3QmOaTxmZjcSDP98KcHehFQABYOMNw8Bf2dmSwhC4jPE/z5eR/At+0Iz+zXQ6+67zexrwNfCD/ZHCY5tnAEU3P2O/WxvFXCpmZ0NbAU+BswlGnZrgdPDs6K6gM4S2/kewYHk75nZtwnG9r+Z4NhEz3AaFl7s5W0EF3XZTHAQexZB8EmF0FlJMt7cRBAOvwJ+T9CHHufeAh5cevHzBAd7NwMDp8p+DvgHgq6h5QQHdd8DrDnAJr9EcAbV7wgCpRv48aB1vkbwzf9FgkA8Zoi6zif4MH8WuJPgIk6fHkHzdhJcUvM3BAeyvw580d1/NIJtyDinC/WIiEiE9hhERCRCwSAVz8xuD38NXep2e7nrE4mbupKk4pnZVKBxiMW73H1LnPWIlJuCQUREItSVJCIiEQoGERGJUDCIiEiEgkFERCL+P26AFClHRRwPAAAAAElFTkSuQmCC\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light", "tags": [] }, "output_type": "display_data" } ], "source": [ "# Instantiate learner in VW but without -q\n", "vw = pyvw.vw(\"--cb_explore_adf --quiet --epsilon 0.2\")\n", "\n", "num_iterations = 5000\n", "ctr = run_simulation(vw, num_iterations, users, context1, context2, items, get_cost)\n", "\n", "plot_ctr(num_iterations, ctr)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 287 }, "id": "2vU5Kozv2WDg", "outputId": "a77bdbf6-e776-4dc5-bc9d-3daaa335156f" }, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAEOCAYAAACNY7BQAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAepklEQVR4nO3deZhcdZ3v8fe39l6zbySBBAlicANaxBEHrjgQ0IH7jHcUrlwVuYM64tVn9HrhOsMoOqJe3FC4yCiDAyqD62QYGHBk07mCdGTLQkIgIQuQdPZea/3eP87ppE5TnXSH9Knurs/reepJnaVOfX+V6vrU+Z1Tv2PujoiIyKBEvQsQEZHxRcEgIiIRCgYREYlQMIiISISCQUREIhQMIiISEVswmNnNZrbdzFYOs9zM7DozW29mT5rZyXHVJiIiB8S5x3ALsOwgy88FloS3y4D/G0NNIiIyRGzB4O4PAbsOssoFwD964GFgqpnNi6c6EREZlKp3AVXmA5urpreE814cuqKZXUawV0FLS8spJ5xwQiwFiohMFitWrNjh7rNqLRtPwTBi7n4TcBNAR0eHd3Z21rkiEZGJxcyeH27ZeDoraSuwsGp6QThPRERiNJ6CYTnw/vDspNOAve7+sm4kEREZW7F1JZnZj4EzgZlmtgX4WyAN4O43AncB5wHrgT7gkrhqExGRA2ILBne/6BDLHfhYTOWIiMgwxlNXkoiIjAMKBhERiVAwiIhIhIJBREQiFAwiIhKhYBARkQgFg4iIRCgYREQkQsEgIiIRCgYREYlQMIiISISCQUREIhQMIiISoWAQEZEIBYOIiEQoGEREJELBICIiEQoGERGJUDCIiEiEgkFERCIUDCIiEqFgEBGRCAWDiIhEKBhERCRCwSAiIhEKBhERiVAwiIhIhIJBREQiFAwiIhKhYBARkQgFg4iIRCgYREQkQsEgIiIRCgYREYlQMIiISESswWBmy8xsrZmtN7Mraiw/2szuN7PHzOxJMzsvzvpERCTGYDCzJHA9cC6wFLjIzJYOWe2vgTvc/STgQuCGuOoTEZFAnHsMpwLr3f05dy8AtwMXDFnHgfbw/hTghRjrExER4g2G+cDmqukt4bxqnwMuNrMtwF3Ax2ttyMwuM7NOM+vs6uoai1pFRBrWeDv4fBFwi7svAM4DbjWzl9Xo7je5e4e7d8yaNSv2IkVEJrM4g2ErsLBqekE4r9qlwB0A7v47IAfMjKU6EREB4g2GR4ElZrbYzDIEB5eXD1lnE3AWgJm9hiAY1FckIhKj2ILB3UvA5cA9wBqCs49WmdnVZnZ+uNqngL8wsyeAHwMfdHePq0YREYFUnE/m7ncRHFSunndV1f3VwFvjrElERKLG28FnERGpMwWDiIhEKBhERCRCwSAiIhEKBhERiVAwiIhIhIJBREQiFAwiIhKhYBARkQgFg4iIRCgYREQkQsEgIiIRCgYREYlQMIiISISCQUREIhQMIiISoWAQEZEIBYOIiEQoGEREJELBICIiEQoGERGJUDCIiEiEgkFERCIUDCIiEqFgEBGRCAWDiIhEKBhERCRCwSAiIhEKBhERiVAwiIhIhIJBREQiFAwiIhKhYBARkQgFg4iIRMQaDGa2zMzWmtl6M7timHXeY2arzWyVmf0ozvpERARScT2RmSWB64E/AbYAj5rZcndfXbXOEuBK4K3uvtvMZsdVn4iIBOLcYzgVWO/uz7l7AbgduGDIOn8BXO/uuwHcfXuM9YmICPEGw3xgc9X0lnBeteOB483sP8zsYTNbVmtDZnaZmXWaWWdXV9cYlSsi0pjG28HnFLAEOBO4CPh7M5s6dCV3v8ndO9y9Y9asWTGXKCIyucUZDFuBhVXTC8J51bYAy9296O4bgHUEQSEiIjGJMxgeBZaY2WIzywAXAsuHrPNLgr0FzGwmQdfSczHWKCLS8GILBncvAZcD9wBrgDvcfZWZXW1m54er3QPsNLPVwP3A/3T3nXHVKCIiYO5e7xpekY6ODu/s7Kx3GSIiE4qZrXD3jlrLxtvBZxERqTMFg4iIRCgYREQkQsEgIiIRCgYREYlQMIiISISCQUREIhQMIiISoWAQEZGIEQWDmaXN7CUzO3GsCxIRkfoaUTC4exEoAhN7/AwRETmk0XQlfRu40sxiuxyoiIjEbzQf8m8DzgC2mtlKoLd6obufX/NRIiIyoYwmGHYAPxurQkREZHwYcTC4+yVjWYiIiIwPIz7GYGb31br+spm1m9l9R7YsERGpl9EcfD4TyNSYnyM4/iAiIpPAIbuSzOzkqsnXm9muqukkcA6w9UgXJiIi9TGSYwydBL9fcODeGsv7gY8fyaJERKR+RhIMiwEDngPeRHB20qACsN3dy2NQm4iI1MEhg8Hdnwcwsy8Db3L3G6uXm9lHzGy+u//NGNUoIiIxGs3B54uBx2rM/wPw/iNTjoiI1NtogmE20FVj/g5gzpEpR0RE6m00wbCJ2qel/jGw5ciUIyIi9TaaITG+C3zDzDLA4A/azgKuAb5ypAsTEZH6GM2QGF8zs5nAdRz4oVsB+Ja7f3UsihMRkfiNaghtd7/SzL4ILA1nrXH3niNfloiI1Muor63g7r3Ao2NQi4iIjAO65rOIiEQoGEREJELBICIiEQoGERGJUDCIiEiEgkFERCIUDCIiEhFrMJjZMjNba2brzeyKg6z3bjNzM+uIsz4REYkxGMwsCVwPnEvwy+mLzGxpjfXagE8Aj8RVm4iIHBDnHsOpwHp3f87dC8DtwAU11vsCwaB8AzHWJiIioTiDYT6wuWp6SzhvPzM7GVjo7v96sA2Z2WVm1mlmnV1dtS4RISIih2vcHHw2swTwdeBTh1rX3W9y9w5375g1a9bYFyci0kDiDIatwMKq6QXhvEFtwGuBB8xsI3AasFwHoEVE4hVnMDwKLDGzxeHFfi4Elg8udPe97j7T3Re5+yLgYeB8d++MsUYRkYYXWzC4ewm4HLgHWAPc4e6rzOxqMzs/rjpEROTgRn09hlfC3e8C7hoy76ph1j0zjppERCRq3Bx8FhGR8UHBICIiEQoGERGJUDCIiEiEgkFERCIUDCIiEqFgEBGRCAWDiIhEKBhERCRCwSAiIhEKBhERiVAwiIhIhIJBREQiFAwiIhKhYBARkQgFg4iIRCgYREQkQsEgIiIRCgYREYlQMIiISISCQUREIhQMIiISoWAQEZEIBYOIiEQoGEREJKKhg+G3z+xgT1+h3mWIiIwrDRsMvfkSF3//ET50y6P1LkVEZFxp2GAouwPwzLaeOlciIjK+NGwwJMyAAwEhIiKBBg6G4N+KgkFEJKJhg2EwD5QLIiJRDRsMg7THICIS1bDBMBgH5YqCQUSkWsMGwyDlgohIVKzBYGbLzGytma03sytqLP8rM1ttZk+a2a/N7JixqsXVhSQiUlNswWBmSeB64FxgKXCRmS0dstpjQIe7vx74KfDVsapHsSAiUlucewynAuvd/Tl3LwC3AxdUr+Du97t7Xzj5MLBgrIoplxUNIiK1xBkM84HNVdNbwnnDuRS4u9YCM7vMzDrNrLOrq+uwivnZH7Yc1uNERCa7cXnw2cwuBjqA/1Nrubvf5O4d7t4xa9asw3qOvkL5FVQoIjJ5pWJ8rq3AwqrpBeG8CDN7B/BZ4Ax3z49VMYO/fBYRkag49xgeBZaY2WIzywAXAsurVzCzk4DvAue7+/axLMZsfCVD90CRnT1jloMiIiMW2x6Du5fM7HLgHiAJ3Ozuq8zsaqDT3ZcTdB21Aj8JP7g3ufv5cdU4FgqlCo6TSSa4o3Mzr18wlUUzWsilg0z+1E+e4JRjpvHZX6zc/5ir3rWUD52+uF4li0wYg6edj7cvehNdnF1JuPtdwF1D5l1Vdf8dcdYzVorlCuu2dfOjRzbxw0c2HXL9n/8h2qN29Z2r+dq9a/n5X76VV89tG6syG0ZvvsSGHb08t6OXma0ZFk5rpjUbvPUzqQS5dJKefImdPXmOmtpELp0c8bYHimWSCSOdTPDS3gG27O5joFjBDNpzaZoySZoySWa0ZHBn/xeCw/kgG/wQHChW6MmXGCiWyZcqbNrVS/dAiXyxQjJhPLCui929BXryJXb05NnXX6Qlm2J2W5aF05uZ057j2FktpBMJcpkkmaQxb0oTM9uy5FIJpjVnANi6p5+9/UWyqQRTmtLMbM1SdieVMF7cO8ALe/rpzpcoliq0ZlOU3dnRkyeZSDBvSo6EGaVyhRmtWfoLZfYNFMmlk+TSCbbtG6B7oMRzXb1s2NFLe1OK5kyK3nyJ2W05dvcV2NGTZ3pLhmTC2NNXpDWbojWXYqBYZtXWffTkS2zZ3UdPvkRLJsW8qTnmTWlibnuOmW0ZevNlCuUKSTN29RXoGShRLFdoz6VJJo257Tmmt2TYN1AkX6zg7pgZs9uz5FJJWrMpcpkk6YTRVyiTTSeY3pIhk0zQnS+RSyX3X+grnUyQSSVIJYxSxcmmErTl0rTlUpQrzpSmNGbQky/Rlkvj7vQVylTcaUonSSUSQR2l4P/QCEZ+TicSlCoV+gpldvUW2NtfDNZPGtlUkuPntDKjNTvq99KhxBoM40niCH/DWPbNhwC47b+/mY4v/vuoHtuSSVIsO099/mzueHQzf/+bDWza1cc533yIL//Z63hPx4FDM4mqgyPb9w3w1XvW8vjmPVx57gnMnZLj+DltpBLGFT97it19BcoVZ932bm74r6eQThm/XrOdpfPaeetxM8mkjmxP4t7+Il+4czXPbOtmTnuOE+a1ky+WKVecTbv6aM2meGFvP82ZFK+Z18as1iznvHYuc9tzrHh+Nyue382SOa305stkUwlue2QTqYQxpz3L9JYMqUTwhze7Pcvxc9oYKFZIWPDh/vsNu2jNpegZKLH6xX0MFMtUHDo37mJ3X3HEbUgnjWNmtHDM9Gamt2RoyabYsruPbCpJSzbJxp197OkrUCo73fkSXd15zMAY+a/oc+kE7tCSTdGWS1EqO8fMaGZma5bmTJJ9A0V29hTo6skzUCjTnS8xpSnNnr4iPfnSIbc/tTnN4pktpBLGGxZMpTWboliusK17gCe27GH7vjz5UuWg2zB7+QCTCQvamEzYERtKxgzmtefoLZTpyZdoziTpHiiRSSaY3Z5ld2+BUsWZ1pyhv1imr1Ci4vC6+VOY057ldfOn7A+BF/b088Lefta8uI8dPXlaMilSScOBKU3p8MPZ2LZvgFLFuW/NdvqLZTLJBOmkkbBg3ZG8xuPFFy44kf/2lkVHfLs20X8B3NHR4Z2dnaN+3I0PPsuX734agI1ffudhPfeD67r49q+fIZkwHtmwq+Y6i2e28L43H033QIkzXj2LWa1Z2nNpANIpozlTO5sfXNfFJf/w+zEfsmPw28lJR0/ltGNn8OEzXrX/2/Sh7O4t8IvHtuLAtfespb945M/0qvUBdSiZVIKZLRl68iWWzGnjxKPaWTKnjUUzmimWK2za2Ud/sUK+VMYw+otlCqUK86bk2NGbZ82L3Wzd3ceOngK9+RLzpuaoVGBff5H505poz6VpziZpz6WZ3ZYlmTD29hc5ekYzx81qJZ1MkEoa+/pL9OZLDJTK7OkrkjDYNxB800+YMVAMPgwTZmzY0cvuvuBbflM6ydwpOWa1Zsmmk2RTCQqlCqmEMX9aE5lksJfTlkuRSSWY0ZplRkuGVNIolZ3jZrcedK+nUnG27unHHQZKQdu37umnqzvPQLHMzt7gC8XstiwLpjWRL1XY01dky+4+culk+EGd5ujpzbTn0mTTSfKlMqWyM6c9hxm8uHeAYqlCNp1g+7482XSCGS1ZBkpl+vJlpjWnmdmWZXZblqnhHsrgN/aBYvDFoNZelbtTrjip5MG/1FQqHvkSVYu701so05JJRp6rr1Cir1Cmv1BmoFimr1BmSlOafKnCzp48hXKFpnSSgVKFac1pEhYEZbFcoVh20kljoFhh30CRnoHS/vdHxZ3mTIq+Qgl3aM0Ff2eFUoViuUJLNkVT+PoakEhAqewkE8HewczWDNNaMvQXypQqzkCxzOKZLRw1temg7RyOma1w946ayxo1GG566Fm+dNfhBYO7c843H2LdQa7+9tEzX8Unzloyqm6JoXrzJS686WGe2rr3oOv9j7cfx4pNu/mP9Tsj8z+z7NWsfamb1x41hVUv7OWXj7/A2UvnkDDj31a9dNBtzmzNcOri6Zy6aDqLZ7Xy/d9u4KF1XbxtyUwe37SHpUe11wzDPz9lAX/9rqWkEsau3gLJhDGtOcPGnb2cMLdt/x/gum3dbN3Tz8ote3n6pW7M4M3HzqA1m6SrO0+5Am9aNI2ORdNxd/b1l1i7rZtSuUI6lWDdtm4GihVaMknK7px+3Ex29xVpz6U4ZkYLySN02tngh5XIZKNgqOF7v3mOL/7rGuDgwXDzbzfQ1ZPnfy07AYBLb3mUXz9d+4SpDdecx0duW8E9q7bx9BeWvaJQGGrwA6p7oMhtD2/i7SfM5pgZza/oOUrlCn3FYBe8PZfm1oefZ+XWvfzmmR0j3sZJR0+l45hpvGnRdM4+ce5h1yIi8TpYMOgYwzC27O6jXHGuvnM1AB/7T8fRmk29LBQ++Y4lnHT0NFqzKcyMGy8+hb39xSMaCnDgYGVbLs1Hz3zVEdlmKpmgPZmgfW7QtTUYfhAcvxgoVrjuvmfoK5SY0ZLlqj9dyjPbetjTX+C186eQSgzfFSYiE1fD/lUfqqfh9K/cH5m+7+ntnP+Go/ZPt+dS/L8rz3pZf7yZ7e8znchmt+cAuPbP3xCZv/So9nqUIyIxathgmNZS+8O7VK7w+s/f+7L5dz/1IruqfoD25OfOGbPaRETqqWGDYbgj+bv6Ci8bR2n+1CbuXvkSd68MDth+bci3aBGRyWRcDqIXh+GOuRdqnN993UUnRabffcqYjQYuIlJ3DbvHMJyB4oFgWHP1MpoySdydUxdN5/cbd/Gzj/5RHasTERl7CgYOnArq7rzj6w8CwW8AmjLBmUVmxh0feUs9SxQRiU0DdyUd6EsqlIO9hH0DB34KPzc8K0dEpNFojwF4YG0X86c28a5v/3b/PA1eJyKNSsEAfPjWFZHp9735aE48akqdqhERqa/G7Uo6yLLDHZRKRGQyaNhgOJjqXziLiDQaBUMNC6c317sEEZG6adhgGO4Hbk9cdXa8hYiIjDMNGwy13PzBDqY0p+tdhohIXemspND6vzv3kFeFEhFpBA37SehDzktSKIiIBBp+j+GcE+fw0TOPq3cZIiLjRsN/Tb7krYt548Kp9S5DRGTcaNxgmNiXuhYRGTONGwyhQ1zhU0Sk4TR8MIiISFTDBoN6kkREamvYYBhkps4kEZFqDRsMww2JISLS6Bo2GAZph0FEJKrhg0FERKIaNhiGDokhIiKBhg2GQepJEhGJavhgEBGRqIYNBp2VJCJSW6zBYGbLzGytma03sytqLM+a2T+Fyx8xs0VjX9NYP4OIyMQSWzCYWRK4HjgXWApcZGZLh6x2KbDb3Y8DvgF8Zazq6SuUx2rTIiITWpx7DKcC6939OXcvALcDFwxZ5wLgB+H9nwJn2Rj9NPnBddsBSOsCPSIiEXFeqGc+sLlqegvw5uHWcfeSme0FZgA7qlcys8uAy8LJHjNbe5g1zXzDV6LbbgAzQW1uAGpzY3glbT5muAUT8gpu7n4TcNMr3Y6Zdbp7xxEoacJQmxuD2twYxqrNcfajbAUWVk0vCOfVXMfMUsAUYGcs1YmICBBvMDwKLDGzxWaWAS4Elg9ZZznwgfD+fwHuc9eJpSIicYqtKyk8ZnA5cA+QBG5291VmdjXQ6e7Lge8Dt5rZemAXQXiMpVfcHTUBqc2NQW1uDGPSZtMXchERqaZzNUVEJELBICIiEQ0bDIcanmMiMbObzWy7ma2smjfdzH5lZs+E/04L55uZXRe2+0kzO7nqMR8I13/GzD5Q67nGAzNbaGb3m9lqM1tlZp8I50/mNufM7Pdm9kTY5s+H8xeHw8esD4eTyYTzhx1exsyuDOevNbNz6tOikTOzpJk9ZmZ3htOTus1mttHMnjKzx82sM5wX73vb3RvuRnDw+1ngWCADPAEsrXddr6A9fwycDKysmvdV4Irw/hXAV8L75wF3E4w4fhrwSDh/OvBc+O+08P60erdtmPbOA04O77cB6wiGWZnMbTagNbyfBh4J23IHcGE4/0bgo+H9vwRuDO9fCPxTeH9p+H7PAovDv4Nkvdt3iLb/FfAj4M5welK3GdgIzBwyL9b3dqPuMYxkeI4Jw90fIjiLq1r18CI/AP5z1fx/9MDDwFQzmwecA/zK3Xe5+27gV8Cysa9+9Nz9RXf/Q3i/G1hD8Kv5ydxmd/eecDId3hx4O8HwMfDyNtcaXuYC4HZ3z7v7BmA9wd/DuGRmC4B3At8Lp41J3uZhxPrebtRgqDU8x/w61TJW5rj7i+H9l4A54f3h2j4hX5Owu+Akgm/Qk7rNYZfK48B2gj/0Z4E97l4KV6muPzK8DDA4vMyEajPwTeAzQCWcnsHkb7MD95rZCguG/4GY39sTckgMGR13dzObdOclm1kr8DPgk+6+z6rGW5yMbXb3MvBGM5sK/AI4oc4ljSkzexew3d1XmNmZ9a4nRqe7+1Yzmw38ysyerl4Yx3u7UfcYRjI8x0S3LdylJPx3ezh/uLZPqNfEzNIEofBDd/95OHtSt3mQu+8B7gfeQtB1MPgFr7r+4YaXmUhtfitwvpltJOjufTvwLSZ3m3H3reG/2wm+AJxKzO/tRg2GkQzPMdFVDy/yAeCfq+a/Pzyb4TRgb7iLeg9wtplNC894ODucN+6E/cbfB9a4+9erFk3mNs8K9xQwsybgTwiOrdxPMHwMvLzNtYaXWQ5cGJ7BsxhYAvw+nlaMjrtf6e4L3H0Rwd/ofe7+PiZxm82sxczaBu8TvCdXEvd7u95H4Ot1Iziav46gn/az9a7nFbblx8CLQJGgL/FSgr7VXwPPAP8OTA/XNYILJj0LPAV0VG3nQwQH5tYDl9S7XQdp7+kE/bBPAo+Ht/MmeZtfDzwWtnklcFU4/1iCD7n1wE+AbDg/F06vD5cfW7Wtz4avxVrg3Hq3bYTtP5MDZyVN2jaHbXsivK0a/GyK+72tITFERCSiUbuSRERkGAoGERGJUDCIiEiEgkFERCIUDCIiEqFgEDkEM1tkZm5m4+pC8+O1Lpn4FAwih7aZYETXxwHM7MzwA3lmXAWY2QNm9p2D1SVypGisJJFD8GCMopfGYttmlnb34uE8dizrksamPQYZV8JvxjeY2ZfMbIcFFyC61swS4fKNZvbpGo/5TtX0RjO7ysxuMbNuM9tsZu81s6lmdruZ9YQXLzl7hDXt77IJR3O9P1zUFc6/JVzPzOwzZvasmfVbcLGVi2ts5yIzu8/M+oEPm9kMM/uxmW0JH7fKzC6petwtwBnAx8LHe7itl3UlmdkfW3CRmgEz22Zm3wiHfRnR6xuu82cWXPSl38x2mdmDZjY4mqc0AAWDjEfvA0rAHwGXA58E3jvKbXySYFiEkwku7PIDgou93AW8EXgIuM3McqPc7mbg3eH9Ewm6cj4RTn+RYDiSjxFcHOYa4Ltm9s4h27gGuCFc55cEQzn8AXhXuM1vhY87K1z/E8DvgH8In28e0SGVATCz+QQXbXmMYCjyS4GLwuerNuzra2ZzCQas+wHwGoKLQN06gtdFJpN6jw2im27VN+AB4HdD5v0K+F54fyPw6RqP+U7V9Ebgx1XTrQRjK11XNW9ROK9jBDVF1iUYt8epusoW0AL0A28b8thvAncN2c6nRvCctw+2uVYbh6nr7wjG0klUrfNBIA80j/D1PTnc5jH1fi/oVr+bjjHIePTkkOkXgNmHuw137zGzPoJBxgZtC/8d7XaHs5Tgm/+/WXSs/DRBUFXrrJ4wsyTB5RrfS3AxlSzBJWcfGGUNrwEedvdK1bzfhts6jgOvycFe3ycIBmlbaWb3hvd/6u5do6xFJjAFg4xHQw/GOge6PSsEI0pWS49wG8Uh03DkulMHt/OnwKZD1NI7ZPrTwKcIuoyeAnqAL3HkQgsOtLdWPftfX3cvh8deTiMYqvlS4BozO8PdnziC9cg4pmCQiaaLoI8dgPAYwQkE/epxKYT/JqvmrSbosjnG3e8b5fZOB/7F3W+F/debOB7YM+Q5kzUeW20N8B4zS1TtNZwePvbZkRbj7k5wTON3ZnY1wfDP7yXYm5AGoGCQieY+4ENmtpwgJD5L/O/j5wm+Zb/TzP4F6Hf3bjO7Frg2/GB/iODYxmlAxd1vOsj21gHvNbPTgR3Ax4HFRMNuI3BqeFZUD7CrxnZuIDiQfIOZfYtgbP8vExyb6BtJw8KLvbyD4KIu2wgOYi8kCD5pEDorSSaaawjC4Z+Bewn60OPcW8CDSy/+LcHB3m3A4KmyfwN8jqBraBXBQd13AxsOsckvEpxBdTdBoPQCPxyyzrUE3/xXEwTi0cPUdS7Bh/njwM0EF3H636No3l6CS2reSXAg+2vAF9z9tlFsQyY4XahHREQitMcgIiIRCgZpeGZ2Y/hr6Fq3G+tdn0jc1JUkDc/MZgPtwyze5+7b46xHpN4UDCIiEqGuJBERiVAwiIhIhIJBREQiFAwiIhLx/wGl4u+uZJ0q3gAAAABJRU5ErkJggg==\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light", "tags": [] }, "output_type": "display_data" } ], "source": [ "# Instantiate learner in VW\n", "vw = pyvw.vw(\"--cb_explore_adf -q UA --quiet --epsilon 0.2\")\n", "\n", "num_iterations = 5000\n", "ctr = run_simulation(vw, num_iterations, users, context1, context2, items, get_cost, do_learn=False)\n", "\n", "plot_ctr(num_iterations, ctr)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "UzavSLvC2ZHm" }, "outputs": [], "source": [ "users = ['A','B','C']\n", "items = ['Item1','Item2','Item3','Item4','Item5','Item6']\n", "context1 = ['morning','evening']\n", "context2 = ['summer','winter']\n", "\n", "context = pd.DataFrame(list(product(users, context1, context2, items)), columns=['users', 'context1', 'context2', 'items'])\n", "context['reward'] = 0\n", "\n", "#user 1 likes Item 2 in morning, and Item 5 in summer\n", "context.loc[(context.users=='A') & \\\n", " (context.context1=='morning') & \\\n", " (context['items']=='Item2'), \\\n", " 'reward'] = 1\n", "context.loc[(context.users=='A') & \\\n", " (context.context2=='summer') & \\\n", " (context['items']=='Item5'), \\\n", " 'reward'] = 1\n", "\n", "#user 2 likes Item 2 in summer, and Item 5 in morning\n", "context.loc[(context.users=='B') & \\\n", " (context.context2=='summer') & \\\n", " (context['items']=='Item2'), \\\n", " 'reward'] = 1\n", "context.loc[(context.users=='B') & \\\n", " (context.context1=='morning') & \\\n", " (context['items']=='Item5'), \\\n", " 'reward'] = 1\n", "\n", "\n", "#user 3 likes Item 4 in morning, Item 3 in evening, and item 4 in winter evening\n", "context.loc[(context.users=='C') & \\\n", " (context.context1=='morning') & \\\n", " (context['items']=='Item4'), \\\n", " 'reward'] = 1\n", "context.loc[(context.users=='C') & \\\n", " (context.context1=='evening') & \\\n", " (context['items']=='Item3'), \\\n", " 'reward'] = 1\n", "context.loc[(context.users=='C') & \\\n", " (context.context1=='evening') & \\\n", " (context.context2=='winter') & \\\n", " (context['items']=='Item4'), \\\n", " 'reward'] = 1\n", "\n", "context['cost'] = context['reward']*-1\n", "\n", "contextdf_new = context.copy()\n", "\n", "def get_cost_new1(context,action):\n", " return contextdf_new.loc[(contextdf_new['users']==context['user']) & \\\n", " (contextdf_new.context1==context['context1']) & \\\n", " (contextdf_new.context2==context['context2']) & \\\n", " (contextdf_new['items']==action), \\\n", " 'cost'].values[0]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Mqvn5gLc2ik2" }, "outputs": [], "source": [ "def run_simulation_multiple_cost_functions(vw, num_iterations, users, contexts1, contexts2, actions, cost_functions, do_learn = True):\n", " cost_sum = 0.\n", " ctr = []\n", "\n", " start_counter = 1\n", " end_counter = start_counter + num_iterations\n", " for cost_function in cost_functions:\n", " for i in range(start_counter, end_counter):\n", " user = choose_user(users)\n", " context1 = choose_context1(contexts1)\n", " context2 = choose_context2(contexts2)\n", "\n", " context = {'user': user, 'context1': context1, 'context2': context2}\n", " \n", " action, prob = get_action(vw, context, actions)\n", " cost = cost_function(context, action)\n", " cost_sum += cost\n", "\n", " if do_learn:\n", " vw_format = vw.parse(to_vw_example_format(context, actions, (action, cost, prob)),pyvw.vw.lContextualBandit)\n", " vw.learn(vw_format)\n", "\n", " ctr.append(-1*cost_sum/i)\n", " start_counter = end_counter\n", " end_counter = start_counter + num_iterations\n", "\n", " return ctr" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 287 }, "id": "ExchBfTr2lBS", "outputId": "3c23545d-652c-4deb-971c-8da1b409fbd9" }, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAEOCAYAAACNY7BQAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de5gcdZ3v8fe3u6dnMpPJZXIhIQlk0EAIFw8xIqwICIghKuyj5yi4rIooezzqUdc9LqzHG6KI62VhRSHHC8qq6KrHDQjiBVhWF5EgBEhCNDeYXEgmmWSSufZ093f/qJpJ19CTzMBMdc/U5/U889B16ZpvdYX+TP1+Vb8yd0dERKRfqtIFiIhIdVEwiIhIhIJBREQiFAwiIhKhYBARkQgFg4iIRMQWDGb2LTPbbWZPDbHczOwmM9toZk+Y2dK4ahMRkUPiPGO4DVh+mOUXAYvCn6uAr8dQk4iIDBJbMLj7g0DbYVa5BPiuB34PTDOzufFUJyIi/TKVLqDEPKClZHpbOG/n4BXN7CqCswoaGhpevnjx4lgKFBGZKB599NE97j6r3LJqCoZhc/eVwEqAZcuW+erVqytckYjI+GJmzwy1rJquStoOLCiZnh/OExGRGFVTMKwC3h5enXQG0O7uz2tGEhGRsRVbU5KZ/QA4F5hpZtuATwI1AO5+C3A3sALYCHQBV8RVm4iIHBJbMLj7ZUdY7sD7YipHRESGUE1NSSIiUgUUDCIiEqFgEBGRCAWDiIhEKBhERCRCwSAiIhEKBhERiVAwiIhIhIJBREQiFAwiIhKhYBARkQgFg4iIRCgYREQkQsEgIiIRCgYREYlQMIiISISCQUREIhQMIiISoWAQEZEIBYOIiEQoGEREJELBICIiEQoGERGJUDCIiEiEgkFERCIUDCIiEqFgEBGRCAWDiIhEKBhERCRCwSAiIhEKBhERiVAwiIhIhIJBREQiFAwiIhKhYBARkYhYg8HMlpvZBjPbaGZXl1l+jJndb2aPmdkTZrYizvpERCTGYDCzNHAzcBGwBLjMzJYMWu3/Aj9y99OAS4GvxVWfiIgE4jxjOB3Y6O6b3T0H3AFcMmgdB6aEr6cCO2KsT0REiDcY5gEtJdPbwnmlPgVcbmbbgLuBD5TbkJldZWarzWx1a2vrWNQqIpJY1db5fBlwm7vPB1YAt5vZ82p095Xuvszdl82aNSv2IkVEJrI4g2E7sKBken44r9SVwI8A3P0hoA6YGUt1IiICxBsMjwCLzKzZzLIEncurBq3zLHA+gJmdSBAMaisSEYlRbMHg7nng/cC9wHqCq4/Wmtm1ZnZxuNpHgPeY2RrgB8A73d3jqlFERCAT5y9z97sJOpVL532i5PU64FVx1iQiIlHV1vksIiIVpmAQEZEIBYOIiEQoGEREJELBICIiEQoGERGJUDCIiEiEgkFERCIUDCIiEqFgEBGRCAWDiIhEKBhERCRCwSAiIhEKBhERiVAwiIhIhIJBREQiFAwiIhKhYBARkQgFg4iIRCgYREQkQsEgIiIRmUoXIFIJhaKz60APW/Z0sm1fF4+37KetM0dbZ46+gpNOGdl0ivbuPhxo78qRShknHNXI5LoMbZ05ajMpevqKFN3JpFP09hVoashSdKfosKejlzeeejQrTpnLnKl1ld5lkWEzd690DS/KsmXLfPXq1ZUuQyqkN19gx/4eunJ53GHO1Doe2dLGbzfuoakhy+zGWna297C3I8eO9m6efu4gtZkU3bkCeztzA9vJZlLMbMgytT5LY22Ggju9+QKNtTUU3Jk1uZauXJ5NrZ309BWYObmWfLGIO0zKpunszdNXcPKFImZGOmXkC0V2tPeQzaS4+GVHc/7i2bxm8WzqatIV/MREAmb2qLsvK7dMZwxSMflCkb2dOZoastSkD7VqtnXm2L6vm90He3h4Sxs723vYtq+Lnr4izTPrmVSTYcf+brbs6eS5Az1lt12bSdGbLw5M19WkaMhmOHX+VIoeBMEp86Zy3KwGmmc2sHjOFNIpG9X9c3c27DrI/3twC3eu2cGPH93GpJo07351M8c01TNv2iSWLWwim1GLrlQXnTFIrPZ29PLZn6/ntxv3sPtg77Dek82kWDynkamTatiyp5OuXIEpdRkWNNVzyrypzG6spT6boa9YZG9HjuOPmsx5i4+irTNHZy7PUVPqmFxb2b+B+gpF7nt6N9/8jy38YWvbwPyUwbJjm7j8zGM5bcE0FjTVV7BKSZLDnTEoGGRUtR7s5UerW3hmbyc/e3wHuXyR6fU17Ovqi6y3cEY9k+sytLR1097dx+zGWuZOm0RbZy8nzZ3KK49rYlZjLeccP4vGupoK7c3oc3f2dfXxXHsPLfu6+I8/t3LPk88NNGs1z2zgZfOnctSUOqY3ZDlx7hROnNPI7Cnqo5DRpWCQUVEoOmt3tHPCnEY6ewt89ufr+ckftw0sn91Ye8SzgKXHTOO6vzyFJUdPGetyx41cvsiT29t5eMteHtq0l8db9nOwJx9Z56WzJ7Ps2Omc+ZIZTJlUgwF1NWmm12c5dka9+i1kxBQM8oI8197D/7j1P2lp6x72e/o7cS9YchQfvuB4OnN5ptVnK96UM97kC0Xau/tYu+MAjz27n/s37GbdzgPkSvpN+tWkjWOa6llydNCsVig6k2szpFPGpGya9u4+9hzsJWXGS2dPZkFTPScdPYWjp00a9X4VGT8UDDLA3enNF5/3F+bKBzexpqWdx57dx4728h26pRprM5xzwiw++rrFHDMjaBfPF4pk0upIHStduTybWzvpzRfozhUpuLP7QA8bd3fw9HMHefq5AxzoDs40uvsKkffOaqylWPTIlVgpg0wqxbT6GuZOraOpIcvkuhrqa9KkUpBNp1g4s4ET5jQyo6EWM5jRkKWpIQuAmUJlPNNVScK3f7eFT9+5bsTv+/JbXsbFLzt6WF/4CoWxVZ/NcPK8qUdcz90pFIM/+Pr/COg/M9gd3rvxp10H2ba/m55cga5cgWfbuth1oJdNrZ105Q5detuZKzxv+/XZNPmCc9ysBgpFp7Wjl7pMmlc0N7Fo9mROb27iuJkN6hcZxxQME1hnb55rfvokq9bsGNb6H11+AmcvmsXcqXUUPfgrU8YfMyOTDoJgcFjPnlLH7Cl1vPK4GcPa1pY9nWzd00lHb56+QpG2zhzb9nXj7mzb100mbSw9Zjr7u3M83rKPO0v+rdWkjca6GuZPn0RTQ5ad+3twnGn1WZpnNHDszHpmNGTJF5150ybRWJehoTbDS2dNBqC1o5d8wYObCWtSGMZRU2rpKziNdRn1q4whBcMEtXZHO6+/6beReX/8+GsHmgEAikUnX3RdRy9Dap4Z3OcxXHs6elm74wDrdx6grTPHvs4c2/d3s2N/Nwum15NKGfs6c/xq/S7aSpq1RsoMZk6uZeqkoBmsriZNvlAcuEeloyfPvq4cfYXgjCmTTlFfk6Ymk6Kts5ddB3rJpoNmtIUzGmisyzC5NkNjXQ0NtWmm1Wc5elrdwKXOuXyRbCZFJmXkCsWBK+WCZreJ16SmPoZxrq9Q5HN3rwegO1fgjkdaIstftmAaK06ew1VnH6c2Yakq+7tydPTmSaeMrXu66MkX2HOwl+37uykUndlT6kibMa0++BLuzRfY25Ejm0mxtyPHzvbgUufnDvTS21cYaC7rC7+4p9dnyaSMzlyeojudvQUKRWd6Q5Y5U2rJF4JmsGf2dtHdV+BAd1/kpsjhSKeMTMqYOqlm4I+u5pkNTKmroa9QpCtXoLEuw5QwwI6d0UAmZRw3q4Gjp02K3NgZN/UxTFAPbNjNO7/9yJDLv/uu0zn7+FkxViQyfNPqs0yrD75M506dVOFqAt25wkBIPNPWRVtnLx29BbJpo68Q9N2kU0Z7dx8pMzp6+4KAOdjLgZ4ggP606yDt3XlqMynqalIc7MlzsCf/vAsC0mGg9JszpY76bBozwqvKUuQKRQrFIo21NTRNDq7umzk5y/T6LF25AmctmslLwqa30RRrMJjZcuBGIA18w90/X2adtwCfAhxY4+5vi7PG8eL3m/c+LxTOWzyb5SfPAeCNpx7NpKzaYEVGYlI2zaRsmqaGLAtH0IR2JP03Nj6zt5N80dnS2skzbZ3s7+rDDApF2HWgh+6ws3/3wV4KRac2kyKTTvFc+0H2b+3jYG8+csnyx9+wZHwHg5mlgZuB1wLbgEfMbJW7rytZZxFwDfAqd99nZrPjqm+86OjNc/In7x2YvuXypSw/eW4FKxKRIzEzmkou9X3FwqYXtB1350BPnv1dOWrSKeaO0ai9cZ4xnA5sdPfNAGZ2B3AJUHoN5XuAm919H4C7746xvqrn7nz4h48PTF//plMUCiIJYhY0P5U2QY2FOINhHlDaM7oNeOWgdY4HMLPfETQ3fcrdfzF4Q2Z2FXAVwDHHHDMmxVaTXL7IdT9fx3cfegaA1586l5vftrTCVYnIRFVtnc8ZYBFwLjAfeNDMTnH3/aUruftKYCUEVyXFXWScfv7ETt73/T9G5l13yckVqkZEkiDOYNgOLCiZnh/OK7UNeNjd+4AtZvYngqAY+tKbCSpfKPLSj90TmXfz25ay4pQ5uuxURMZUnBfRPgIsMrNmM8sClwKrBq3zM4KzBcxsJkHT0uYYa6wK7h4JhTcvnc+W61fw+lPnKhREZMzFdsbg7nkzez9wL0H/wbfcfa2ZXQusdvdV4bILzWwdUAD+j7vvjavGarCvM8dpn/nVwPTaT7+OBo1MKiIx0p3PVeS59h7OuP43A9MbrltObUb3IojI6NOdz+PAHX94lqt/+uTA9NbPv76C1YhIkikYKiyXL3Lpyof447OHLrzacv2KClYkIkmnYKigG37xNF9/YFNk3vprl6uDWUQqSsEQs47ePI9sbeOKQeMcffudr+A1izUCiIhUnoIhRj19hcg4RwAvmdXALz98jp69KyJVQ8EQk82tHZz3pX+PzPvt37+G+dPrK1SRiEh5CoYxtn7nAd79ndVs3989MG/z51ZMyKc+icjEoGAYQ/c8uZP3fi86ztHjn3itQkFEqpqCYYysadn/vFDY9LkV6ksQkao3rGAwsxqCIbPPd/e1Y1vSxHDJzb8beK2b1URkPBnWIHrhaKd9BI/blCNoaesaeP3nz15UwUpEREZuJKOr/jNwjZmp+ekIzvvSAwDc+tcvpyYd5wC2IiIv3ki+5F8NnANsN7OngM7She5+8WgWNl59/+Fn6SsEJ1avO2lOhasRERm5kQTDHuAnY1XIRNDWmeMf/n8wEN5n/lJPWROR8WnYweDuV4xlIePJb/+8h8u/+TDvPfcl/P3yxezp6GXT7g7euvL3A+v89RnHVrBCEZEXbtjBYGb3AW8a/PxlM5sC/Mzdzxvt4sbSmpb9/PN9G7nl8qVkRtgPcPk3Hwbg6w9set4geBA8R0FEZLwaSVPSuUC2zPw6gv6HceWDdzzG1r1dtOzrpnlmw6htd80nL9TDdURkXDtiMJjZ0pLJU82srWQ6DbwO2D7ahVWr9u6+5817zQmzuPGy08imU9TVKBREZHwbzhnDaoL7Fxz4ZZnl3cAHRrOoOI300aYv+3TwEXxsxYm85+zjxqIkEZGKGk4wNAMGbAZeQXB1Ur8csNvdC2NQ25h6IQ/D+d8/eGzg9aWnLxjNckREqsYRg8HdnwEws88Dr3D3W0qXm9n/NLN57v7xMaqxKuzY382qNTsGphvraipYjYjI2BnJ5TiXA4+Vmf9H4O2jU078htuQ9Befv2/g9VffdtrYFCMiUgVGclXSbKC1zPw9wFGjU058RtKQ1NN3qKVszScuZGq9zhZEZOIayRnDs5S/LPVsYNvolFOdPnPXuoHXCgURmehGcsZwK/AVM8sC/e0q5wPXAzeMdmFxOdJFSTvbu/new88CcNcHzoqhIhGRyhrJkBhfMrOZwE0cutEtB9zo7l8Yi+LG1DDbks68/lDfwpK5U8aoGBGR6jGiIbTd/Rozuw5YEs5a7+4do19WddIjOUUkCUb8bAV37wQeGYNaKmR41yXpgTsikhSJfejOcP/2nzqphrOPn6UH7ohIYiT222445wl/c/tq2rv7uLPkxjYRkYkuscFwJMWic+/aXQC8Zdn8ClcjIhKfxAbDkZqSbvjF0wOvz1o0a2yLERGpIokNhn5D3cdw64ObB16/4ZS5MVUjIlJ5iQ2G4Y6uetsVr9BlqiKSKIkNhuE694TZlS5BRCRWsQaDmS03sw1mttHMrj7Mem82MzezZWNdU7mWpLbOHAAXnDjuxgYUEXnRYgsGM0sDNwMXEdw5fZmZLSmzXiPwQeDhMa3nMMvedVtw/96v1+8ayxJERKpSnGcMpwMb3X2zu+eAO4BLyqz3GYJB+XpirC3i8Zb9AFx7yUmVKkFEpGLiDIZ5QEvJ9LZw3gAzWwoscPefH25DZnaVma02s9WtreUeETF8g69KamnrGnj9ttOPeVHbFhEZj6qm89nMUsCXgY8caV13X+nuy9x92axZL+weg6EuStp98NCJSkbDYIhIAsX5zbcdWFAyPT+c168ROBl4wMy2AmcAq+LogC715q8/BMArm5vi/LUiIlUjzmB4BFhkZs3hw34uBVb1L3T3dnef6e4L3X0h8HvgYndfPZZF+RCjJl3/plPG8teKiFSt2ILB3fPA+4F7gfXAj9x9rZlda2YXx1VHPzvCoBjNMxtiqkREpLrEOuy2u98N3D1o3ieGWPfcMa2lzJnC3o7egdfDvTNaRGSiUe9qiZdf9+tKlyAiUnGJDYYjNSWJiCRVYoPhcNZ88sJKlyAiUjGJD4Zyw25PnVQTfyEiIlUiscGgvmURkfISGwz9hnpQj4hIUiU+GPq5EkJEBFAwDOjKFQC45qLFFa5ERKSyEh8M/Te6dfTmAZhcF+s9fyIiVSexwTD4zuZ1Ow8A8OS29kqUIyJSNRIbDIN1hmcMR02pq3AlIiKVlfhg6O9zbqgNmpDOOeGFPd9BRGSiSHww9OvqDTqfJ9eqj0FEki3xwdDf1dCZC5qS6rPpClYjIlJ5iQ+G/qak7/znVgCyepyniCRcYr8FB4+I0XoweBZDY53GSRKRZEtsMJT64r0b2B0GwyQ1JYlIwiU+GPoKRb56/8ZKlyEiUjUSGwz9nc79Q2GIiEggscHQr6dPwSAiUkrB0FcceH328bq5TUQkscHQ35TUsq9rYN6Df2qtUDUiItUjscHQ71frdlW6BBGRqpLYYLDwToZn2w6dMZx53IxKlSMiUjUSGwz9+m9sA7j9ytMrWImISHVIfDD0+9n7XkVGw2GIiCgY+hX1zGcREUDBQG0m+AiWzJ1S4UpERKpD4oNh8ZxGmhqy1NVojCQREYDEPpWm/z6GNXrGs4hIROLPGEREJErBICIiEYkNhsEP6hERkUBig0FERMqLNRjMbLmZbTCzjWZ2dZnlf2tm68zsCTP7jZkdG2d9IiISYzCYWRq4GbgIWAJcZmZLBq32GLDM3U8Ffgx8YQwLGrNNi4iMZ3GeMZwObHT3ze6eA+4ALildwd3vd/f+Ue1+D8yPsT4RESHeYJgHtJRMbwvnDeVK4J5yC8zsKjNbbWarW1tf2DMUSs8XTm9uekHbEBGZiKqy89nMLgeWAf9Ybrm7r3T3Ze6+bNasF/bUtVRJMrz9THVliIj0i/PO5+3AgpLp+eG8CDO7APgYcI679w5ePlpSJX0MmZT6G0RE+sV5xvAIsMjMms0sC1wKrCpdwcxOA24FLnb33WNZTGkwbGrtHMtfJSIyrsQWDO6eB94P3AusB37k7mvN7Fozuzhc7R+BycC/mtnjZrZqiM29aKUXJT2p8ZJERAbEOoieu98N3D1o3idKXl8QVy2lZww9+UJcv1ZEpOpVZedzHFIle75wRkPlChERqTLJDYaSM4YLTzqqgpWIiFSXxAaDlQTDgun1FaxERKS6JDYY0iWdzwuaFAwiIv0SGwwpjZUkIlJWYoPBFAwiImUlNhh0s7OISHkJDgYlg4hIOckNhsTuuYjI4SX261FnDCIi5SU2GNT5LCJSXmKDQZ3PIiLlJTgYlAwiIuUkNhiUCyIi5SU2GHTGICJSXmKDIa1gEBEpK7HBoPsYRETKS+zXoy5XFREpL7HBoMtVRUTKS2wwqI9BRKS8xAZDNpPYXRcROazEfjvWZtKVLkFEpColOBgSu+siIoeV2G9HNSWJiJSnb0cREYlQMIiISETig+E9r26udAkiIlUl8cHw4dceX+kSRESqSuKDQUREohQMIiISoWAQEZEIBYOIiEQoGEREJELBICIiEQoGERGJiDUYzGy5mW0ws41mdnWZ5bVm9sNw+cNmtjDO+kREJMZgMLM0cDNwEbAEuMzMlgxa7Upgn7u/FPgKcENc9YmISCDOM4bTgY3uvtndc8AdwCWD1rkE+E74+sfA+aaHM4uIxCoT4++aB7SUTG8DXjnUOu6eN7N2YAawp3QlM7sKuCqc7DCzDS+wppkNN0S3nQAzQfucANrnZHgx+3zsUAviDIZR4+4rgZUvdjtmttrdl41CSeOG9jkZtM/JMFb7HGdT0nZgQcn0/HBe2XXMLANMBfbGUp2IiADxBsMjwCIzazazLHApsGrQOquAd4Sv/ztwn7t7jDWKiCRebE1JYZ/B+4F7gTTwLXdfa2bXAqvdfRXwTeB2M9sItBGEx1h60c1R45D2ORm0z8kwJvts+oNcRERK6c5nERGJUDCIiEhEYoPhSMNzjBdmtsDM7jezdWa21sw+GM5vMrNfmdmfw/9OD+ebmd0U7vcTZra0ZFvvCNf/s5m9Y6jfWS3MLG1mj5nZXeF0cziUysZwaJVsOH/IoVbM7Jpw/gYze11l9mR4zGyamf3YzJ42s/VmduZEP85m9uHw3/VTZvYDM6ubaMfZzL5lZrvN7KmSeaN2XM3s5Wb2ZPiem4Z107C7J+6HoPN7E3AckAXWAEsqXdcL3Je5wNLwdSPwJ4IhR74AXB3Ovxq4IXy9ArgHMOAM4OFwfhOwOfzv9PD19Erv3xH2/W+B7wN3hdM/Ai4NX98CvDd8/b+AW8LXlwI/DF8vCY99LdAc/ptIV3q/DrO/3wHeHb7OAtMm8nEmuOF1CzCp5Pi+c6IdZ+BsYCnwVMm8UTuuwB/CdS1870VHrKnSH0qFDsSZwL0l09cA11S6rlHat38DXgtsAOaG8+YCG8LXtwKXlay/IVx+GXBryfzIetX2Q3AfzG+A84C7wn/0e4DM4GNMcCXcmeHrTLieDT7upetV2w/BPT1bCC8YGXz8JuJx5tBICE3hcbsLeN1EPM7AwkHBMCrHNVz2dMn8yHpD/SS1Kanc8BzzKlTLqAlPnU8DHgaOcved4aLngKPC10Pt+3j7TP4J+ChQDKdnAPvdPR9Ol9YfGWoF6B9qZTztczPQCnw7bD77hpk1MIGPs7tvB74IPAvsJDhujzKxj3O/0Tqu88LXg+cfVlKDYcIxs8nAT4APufuB0mUe/KkwYa5LNrM3ALvd/dFK1xKjDEFzw9fd/TSgk6CJYcAEPM7TCQbWbAaOBhqA5RUtqgIqcVyTGgzDGZ5j3DCzGoJQ+J67/zScvcvM5obL5wK7w/lD7ft4+kxeBVxsZlsJRuk9D7gRmGbBUCoQrX+ooVbG0z5vA7a5+8Ph9I8JgmIiH+cLgC3u3urufcBPCY79RD7O/UbruG4PXw+ef1hJDYbhDM8xLoRXGHwTWO/uXy5ZVDq8yDsI+h765789vLrhDKA9PGW9F7jQzKaHf6ldGM6rOu5+jbvPd/eFBMfuPnf/K+B+gqFU4Pn7XG6olVXApeHVLM3AIoKOuqrj7s8BLWZ2QjjrfGAdE/g4EzQhnWFm9eG/8/59nrDHucSoHNdw2QEzOyP8DN9esq2hVbrTpYKdPSsIruDZBHys0vW8iP04i+A08wng8fBnBUHb6m+APwO/BprC9Y3ggUmbgCeBZSXbehewMfy5otL7Nsz9P5dDVyUdR/A//EbgX4HacH5dOL0xXH5cyfs/Fn4WGxjG1RoV3tf/BqwOj/XPCK4+mdDHGfg08DTwFHA7wZVFE+o4Az8g6EPpIzgzvHI0jyuwLPz8NgFfZdAFDOV+NCSGiIhEJLUpSUREhqBgEBGRCAWDiIhEKBhERCRCwSAiIhEKBpEjMLOFZuZmVlUPmq/WumT8UzCIHFkLwWBkjwOY2bnhF/LMuAowswfM7KuHq0tktMT2zGeR8crdCwQDmY06M6vxYLiHERvLuiTZdMYgVSX8y/hrZvY5M9sTPsDki2aWCpdvNbO/K/Oer5ZMbzWzT5jZbWZ20MxazOytFjzo5g4z6wgfZnLhMGsaaLIJR7C9P1zUGs6/LVzPzOyjZrbJzLrDh6NcXmY7l5nZfWbWDfyNmc2w4CE028L3rTWzK0redxtwDvC+8P0ebut5TUlmdrYFD6npMbNdZvaVcNiXYX2+4TpvsuAhMN1m1mZm/25m/aN7SgIoGKQa/RWQB/4CeD/wIeCtI9zGhwiGRVhK8GCX7xA81OdugqElHgT+xczqRrjdFuDN4euTCJpyPhhOX0cwnMH7CB4Ocz1wq5m9ftA2rge+Fq7zM4KhHP4IvCHc5o3h+84P1/8g8BDw7fD3zSU6xDIAZjaP4EEsjxEMv34lwfj71w9adcjP18zmEAxM+B3gRIKHyNw+jM9FJpJKjxOiH/2U/gAPAA8Nmvcr4Bvh663A35V5z1dLprcCPyiZnkwwntRNJfMWhvOWDaOmyLoE4zM5MLNknQagG3j1oPf+E3D3oO18ZBi/847+fS63j0PU9VmCsXVSJeu8E+gF6of5+S4Nt3lspf8t6KdyP+pjkGr0xKDpHcDsF7oNd+8wsy6CQcf67Qr/O9LtDmUJwV/+vzCz0gHIagiCqtTq0gkzSxM8W+GtBA9RqSV4dOcDI6zhROD37l4smffbcFsv5dBncrjPdw3BoG1Pmdkvw9c/dvfWEdYi45iCQarR4M5Y51CzZ5FghMlSNcPcRt+gaRi95tT+7byRYLjow9XSOWj674CPEDQZPQl0AJ9j9EILog96GfLzdfdC2PdyBsHQzVcC15vZOe6+ZhTrkSqmYJDxppWgjR2AsI9gMUG7eo6Gi/0AAAF+SURBVFxy4X/TJfPWETTZHOvu941we2cBd7r77TDwjI3jgf2Dfme6zHtLrQfeYmapkrOGs8L3bhpuMe7uBH0aD5nZtcBagrMZBUNCKBhkvLkPeJeZrSIIiY8R/7/jZwj+yn69md0JdLv7QTP7IvDF8Iv9QYK+jTOAoruvPMz2/gS81czOIniA/QcIHmdZGnZbgdPDq6I6gLYy2/kaQUfy18zsRoLnFnyeoG+iazg7Fj785QKCB7/sIujEXkAQfJIQuipJxpvrCcLh34BfErShx3m2gAcPqf8kQWfvLoKHnwB8HPgUQdPQWoJO3TcDW46wyesIrqC6hyBQOoHvDVrniwR/+a8jCMRjhqjrIoIv88eBbxE8BOYfRrB77QSPz7yLoCP7S8Bn3P1fRrANGef0oB4REYnQGYOIiEQoGCTxzOyW8G7ocj+3VLo+kbipKUkSz8xmA1OGWHzA3XfHWY9IpSkYREQkQk1JIiISoWAQEZEIBYOIiEQoGEREJOK/AKfepxmKuEpwAAAAAElFTkSuQmCC\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light", "tags": [] }, "output_type": "display_data" } ], "source": [ "# use first reward function initially and then switch to second reward function\n", "\n", "# Instantiate learner in VW\n", "vw = pyvw.vw(\"--cb_explore_adf -q UA --quiet --epsilon 0.2\")\n", "\n", "num_iterations_per_cost_func = 5000\n", "cost_functions = [get_cost, get_cost_new1]\n", "total_iterations = num_iterations_per_cost_func * len(cost_functions)\n", "\n", "ctr = run_simulation_multiple_cost_functions(vw, num_iterations_per_cost_func, users, context1, context2, items, cost_functions)\n", "\n", "plot_ctr(total_iterations, ctr)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 287 }, "id": "mgaEviR42mlI", "outputId": "00ccc48f-f222-4a58-fe14-d8675d4b13cc" }, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAEOCAYAAACNY7BQAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAdxUlEQVR4nO3deZhddZ3n8ffnbrVlJQtCEiFIHIzthtUILd1i4wLqwEzrKLS2Gy0zo/jotI4PtNPajXajNmrruADt2mhLq+1oVBzsFtDHaaQpVJDFaIBAEiApElJJarvbd/44pyr3FFVJBSqnKnU/r+e5T852T33PPTf3c8/5nfs7igjMzMzGFGa7ADMzm1scDGZmluFgMDOzDAeDmZllOBjMzCzDwWBmZhm5BYOkL0jaIemOKeZL0iclbZJ0u6ST86rNzMz2y/OI4UvAWQeYfzawLn1cCHw2h5rMzGyC3IIhIn4C7DrAIucC/xCJnwFLJB2TT3VmZjamNNsFtFgFbGkZ35pOe2jigpIuJDmqoKen57knnXRSLgWamc0Xt9566yMRsWKyeXMpGKYtIq4CrgLo7e2Nvr6+Wa7IzOzIIun+qebNpauStgFrWsZXp9PMzCxHcykYNgCvT69OOhUYiIjHnEYyM7PDK7dTSZK+BpwBLJe0FXg/UAaIiCuAa4GXAZuAIeBNedVmZmb75RYMEXH+QeYH8LacyjEzsynMpVNJZmY2BzgYzMwsw8FgZmYZDgYzM8twMJiZWYaDwczMMhwMZmaW4WAwM7MMB4OZmWU4GMzMLMPBYGZmGQ4GMzPLcDCYmVmGg8HMzDIcDGZmluFgMDOzDAeDmZllOBjMzCzDwWBmZhkOBjMzy3AwmJlZhoPBzMwyHAxmZpbhYDAzswwHg5mZZTgYzMwsw8FgZmYZDgYzM8twMJiZWYaDwczMMhwMZmaW4WAwM7MMB4OZmWU4GMzMLMPBYGZmGbkGg6SzJG2UtEnSxZPMf7KkGyT9QtLtkl6WZ31mZpZjMEgqAp8GzgbWA+dLWj9hsf8FfD0ingOcB3wmr/rMzCyR5xHDKcCmiLg3IqrANcC5E5YJYFE6vBh4MMf6zMyMfINhFbClZXxrOq3VXwKvk7QVuBZ4+2QrknShpD5Jff39/YejVjOztjXXGp/PB74UEauBlwFXS3pMjRFxVUT0RkTvihUrci/SzGw+yzMYtgFrWsZXp9NaXQB8HSAibgI6geW5VGdmZkC+wXALsE7SWkkVksblDROWeQA4E0DS00iCweeKzMxylFswREQduAi4Drib5OqjOyVdKumcdLF3AW+RdBvwNeCNERF51WhmZlDK849FxLUkjcqt097XMnwX8Pw8azIzs6y51vhsZmazzMFgZmYZDgYzM8twMJiZWYaDwczMMhwMZmaW4WAwM7MMB4OZmWU4GMzMLMPBYGZmGQ4GMzPLcDCYmVmGg8HMzDIcDGZmluFgMDOzDAeDmZllOBjMzCzDwWBmZhkOBjMzy3AwmJlZhoPBzMwyHAxmZpbhYDAzswwHg5mZZTgYzMwsw8FgZmYZDgYzM8twMJiZWYaDwczMMhwMZmaW4WAwM7MMB4OZmWU4GMzMLMPBYGZmGbkGg6SzJG2UtEnSxVMs82pJd0m6U9I/5lmfmZlBKa8/JKkIfBp4MbAVuEXShoi4q2WZdcAlwPMj4lFJK/Oqz8zMEnkeMZwCbIqIeyOiClwDnDthmbcAn46IRwEiYkeO9ZmZGfkGwypgS8v41nRaq6cCT5X0/yT9TNJZk61I0oWS+iT19ff3H6Zyzcza01xrfC4B64AzgPOBv5e0ZOJCEXFVRPRGRO+KFStyLtHMbH7LMxi2AWtaxlen01ptBTZERC0i7gN+QxIUZmaWkzyD4RZgnaS1kirAecCGCct8m+RoAUnLSU4t3ZtjjWZmbS+3YIiIOnARcB1wN/D1iLhT0qWSzkkXuw7YKeku4Abgf0bEzrxqNDMzUETMdg1PSG9vb/T19c12GWZmRxRJt0ZE72Tz5lrjs5mZzTIHg5mZZTgYzMwsw8FgZmYZDgYzM8twMJiZWYaDwczMMhwMZmaW4WAwM7OMaQWDpLKkhyU9/XAXZGZms2tawRARNaAGHNn9Z5iZ2UEdyqmk/w1cIim324GamVn+DuVD/veBFwDbJN0BDLbOjIhzJn2WmZkdUQ4lGB4B/vlwFWJmZnPDtIMhIt50OAsxM7O5YdptDJKun+z+y5IWSbp+ZssyM7PZciiNz2cAlUmmd5K0P5iZ2Txw0FNJkk5uGX2mpF0t40XgpcC2mS7MzMxmx3TaGPpIfr8QwA8nmT8MvH0mizIzs9kznWBYCwi4F/hdkquTxlSBHRHROAy1mZnZLDhoMETE/QCSPgT8bkRc0Tpf0n+TtCoi/uIw1WhmZjk6lMbn1wG/mGT6z4HXz0w5ZmY22w4lGFYC/ZNMfwQ4embKMTOz2XYowfAAk1+W+gfA1pkpx8zMZtuhdIlxJfBxSRVg7AdtZwKXAR+e6cLMzGx2HEqXGB+VtBz4JPt/6FYFPhERHzkcxZmZWf4OqQvtiLhE0geB9emkuyNi38yXZWZms+WQ760QEYPALYehFjMzmwN8z2czM8twMJiZWYaDwczMMhwMZmaW4WAwM7MMB4OZmWU4GMzMLCPXYJB0lqSNkjZJuvgAy71SUkjqzbM+MzPLMRgkFYFPA2eT/HL6fEnrJ1luIfAO4Oa8ajMzs/3yPGI4BdgUEfdGRBW4Bjh3kuU+QNIp30iOtZmZWSrPYFgFbGkZ35pOGyfpZGBNRHz/QCuSdKGkPkl9/f2T3SLCzMwerznT+CypAHwMeNfBlo2IqyKiNyJ6V6xYcfiLMzNrI3kGwzZgTcv46nTamIXA7wA3StoMnApscAO0mVm+8gyGW4B1ktamN/s5D9gwNjMiBiJieUQcHxHHAz8DzomIvhxrNDNre7kFQ0TUgYuA64C7ga9HxJ2SLpV0Tl51mJnZgR3y/RieiIi4Frh2wrT3TbHsGXnUZGZmWXOm8dnMzOYGB4OZmWU4GMzMLMPBYGZmGQ4GMzPLcDCYmVmGg8HMzDIcDGZmluFgMDOzDAeDmZllOBjMzCzDwWBmZhkOBjMzy3AwmJlZhoPBzMwyHAxmZpbhYDAzswwHg5mZZTgYzMwsw8FgZmYZDgYzM8twMJiZWYaDwczMMhwMZmaW4WAwM7MMB4OZmWU4GMzMLMPBYGZmGQ4GMzPLcDCYmVmGg8HMzDIcDGZmluFgMDOzDAeDmZll5BoMks6StFHSJkkXTzL/zyTdJel2ST+SdFye9ZmZWY7BIKkIfBo4G1gPnC9p/YTFfgH0RsQzgW8CH8mrPjMzS+R5xHAKsCki7o2IKnANcG7rAhFxQ0QMpaM/A1bnWJ+ZmZFvMKwCtrSMb02nTeUC4AeTzZB0oaQ+SX39/f0zWKKZmc3JxmdJrwN6gb+dbH5EXBURvRHRu2LFinyLMzOb50o5/q1twJqW8dXptAxJLwLeC7wgIkZzqs3MzFJ5HjHcAqyTtFZSBTgP2NC6gKTnAFcC50TEjhxrO2TNZlBrNGe7DDOzGZfbEUNE1CVdBFwHFIEvRMSdki4F+iJiA8mpowXANyQBPBAR5+RV41Q+/9P7+MD37hof/+v//Du89//cAcDdl55FV6XIb7bv5d7+Qb7Rt4UXnrSSVz13NZ3lIgARQaMZ3L9riJ/+9hFef9pxpNtnZjbnKCJmu4YnpLe3N/r6+g7b+m/cuIM3fvGWx/Xcd75oHWuX9/COa3456fyOUoGb//xMlnRXMtNrjSbr3vvYdvfFXWW+e9HpPHlZ9+Oqx2y+aTaDejMoF3XAL1sRwVC1QbEgKsUChYJoNpPPPokZ+6LWbAYBFAQRMFxrsH3PCLVGUCqKiKCjVGRRZ5lSUXSUCpSKs9PUK+nWiOiddF47B8N3b3uQ009cztKe7Afzd365jaFqg0u+9auZKHFaXtO7hnOefSyv/dzN037O2uU9XP5fnkVEsO7ohXSUCuNHKRPt3DfKzx/Yzd0P7eHbv9zGaScsY3FXmUeHqhQL4is/e4DFXWWesWox645ewK7BKr/3lGWc86xVdFX2r3NwtE5PR55NU3ao9o7UGBiujX/YDVfr7B6q0VkusrirTKMZVBtNCgIQEIzUmuwZqbF3pE5PpcSxSzpZ1tPBoq4Se0bqDFXrPDpYo1gQxYIYGK7Rv3eE0XqTBR0lSsUCw9U6g6MNao0mtWZQbzSp1pvsGqoyXG2wa7BKAETypah/3yhd5SKj9Sb9e0cZqtYpKFl/qVigWm8AMFJrMlpvUm82iYB6egq3UBAjtQa1RlApFVjUWaJUKNBRLtBVLlIpFdgzXGO03mRwtM6ekTqQBEFPpcRQtU4zkvFyoUCt2aRcLNBZKtBVKVJrBAVBR6mIBF3lIh3lAj2V/a/Jos4yXZUiu4eqNAPu3zmYhEBB1JvT+2ztKhfp6SjRXSmytLtMpVSgu1JiYWeJeiPYOThKM6AoUS6JrnKRcrHAaL3Jn5x6HC88aeXjep84GCaxacc+XvSxHwOw+UMvz8w7/uLvP2b5K//kubz4aUcjwbW/epiHBoa54PS1/PHf38xN9+7kU3/8HF769CeNvyEmfuPf/KGXj59S2rh9Lz9/YDdX37SZ32zfN2WNmz/0ch4eGKFab7LmqC427xzihZffeNBtW7u8hxevP5qVCzv4/q8e4hcP7D74C3IQq5Z08dDAMGPv9UWdJUbrTZ69ZgndleSN+tDACB2lAqeesIzjlnUzMFzj1w/vZfXSLpb1VFjaU2H10m6W9VRYvbSL27cOsGuwyqNDVZYt6ODpxy5iWU/lsJ9miwge2DVEpVSgUixw1GH+m9V6k3sf2cfAUI2FnWVG6w2Gqg2qjSZbdw3x0MDI+LfZxV1lVi7soKtSZOXCTlYs7GDvSI19o8mH5tZHh3hkX5WHB0YoFwvJh0ezycMDo4zUGmzdPcztW3czU/+ty0VRazyxlVWKBSql5HUuFQSC4WqDZQsqVOtNSoUCyxd2sKSrTDOCeiOoN5tUSgUi9n8glwoFCoJioYAEjWbQWS6yoKPI3tE6e0fq1BtJiAxVG1TrTRZ1lelMvzAdu6SLZgSjtQb7Rht0VQoUCwUazSb1ZlApFqjWm+NBUk7//mitQQAjtQYjtQb7Russ7irTXSmxZ6TG0GiDJd1lJHjyUd30dJSo1pOQ6aoUWb6gg85ygXojxsNsrNbhWoPB0Tr7RusMVRvsHqpRrTcZqiZBViqIpd0VSkXRSNs1R2pJSJaLBS564Ymc/YxjHtd+cTBM4t/v28Wrr7wJmF4wTFxmukZqDTpKhSk/ePaN1rnpnp285R+SbVi1pIsvv/kUTly5YMp1NpvJG+ybt27l6ps2c9vWgWnVctoJy+jpKPK2F57IqiVd3LBxB2uWdjNUbXDKCUexqLPMtt3DFASPDtbYM1Ljy/+2mdu27ObBgREgCYSRWpNq+q3t2MWdSKLaaCKSb1879o4+7g+mJd1lnrSok8VdZUZqDSSxLD2iO6qnQk9HiVoj+U9RazTH//PUm8HaZd0ct6yHrkqRfaN1tj06TEQw2mjyyN4qD+waZKTWZPueEXbs3X/BW3elyNGLOlnUWWKo2mC41qCUfjM+ZnEXKxd1MFJrsHNflSXd5fFTBI1m0F0pUm0Ey3oqLOws0Yzgwd0jLOws8fDACNt2D7Nt9/ABX4+x0xsSDFUb03qdFnaWaDaDwWqDguBJizrprBRZ0lXm+ScuZ1lPhY5yEcH4kcJovTF+JNFVLtJsKapcLLCku8yCjhKDow227R5m91CV/n2jLOmq0NNRZMWCDhrpl5uFnSVWLuyks1xg70idWiN5LZKjB1EuFigXC5SKYlFn+XG9F+zwcjBMom/zLl51RRIMH/qjZ3D2M45hcVeZeqPJiRO+7b/1jKfwnrNOmpF689BoBjdu3MHfXreR971iPac9ZVmujd27Bqvc27+PrkqRk560iAd3DzMwnJym2Dk4yqNDNbbsGmJpd4W1y7tZ1FlmsNrgtzv2svmRQfr3jjIwXKNULCDg/p1DdJYL7Ng7ylC1kZ4jhmYTVizsYM9wjQWdJbbvGWGqo/el3WWOX95DZ6nI0p4ypxx/FJVSkZFag/seGWT7nhH2jNQoFwvjH2SNZvDgwDAPDYxQKRZY1FVm32iNciH5wBs7DVIsiF2DVXYP1SgXxfIFHQxW66xZ2s2S7gpPPXoBxy3rZuWizqTWjhLdlRISHLO4kycf1T2+f0brDXbsSb79PzSQBFhXuciKhR3UGk1WL+1iaU9lvMaxc9rFgi9msEPjYJjErfc/yis/+2/j44u7ytz2/pdwT/8+zvzoj/lPzz6Wj7/m2Tw4MMKqJV0zWbI9TmPvVUmZ4TEjtQZbdg0xWG2wqLPE0Ys6x4/W/MFplnWgYGjbVsTShA+KgeEa9+8c5MyPJu0Ozzsh+ZbtUJg7WkNgsiOgznKRdUcvzLMks3lpTnaJkYfJvkHe+eCe8eHbtjzxBlszsyNR2wbDZKfc3/rVn48Pv+KZx+ZYjZnZ3NG2wXCgc87PPW4pp69bnmM1ZmZzR/sGwwGu0lk24QdvZmbtpG2D4UCXby7obNs2eTOz9g0GyF6m+9U/fd748OWvelbexZiZzRltHAxZRy/qHB8u+Jp3M2tjbXvOZOLv+jpKBX79gbMouDtsM2tz7RsME8YlpuyZ1MysnbTtqaSJRwxLu30lkpkZtHMwtBwzLF/Q4XsMmJml2vbTcOyI4bOvPflx92duZjYfte8RQxoMbms2M8tq22DYz8lgZtaqbYMhHnNdkpmZQTsHg08lmZlNqm2DYYxzwcwsq22DYf8Rg6PBzKxV+wZD2sbgWDAzy2rfYHAbg5nZpNo2GMY4GMzMsto2GHyxqpnZ5No3GGKsjcGHDGZmrdo3GMYGnAtmZhntGwxjjc+zW4aZ2ZzTtsEwdszg3zGYmWW1bTD4iMHMbHJtGwxjfMBgZpbVtsHgy1XNzCaXazBIOkvSRkmbJF08yfwOSf+Uzr9Z0vGHq5b9p5J8yGBm1iq3YJBUBD4NnA2sB86XtH7CYhcAj0bEicDHgQ8frnoGq/W0rsP1F8zMjkx5HjGcAmyKiHsjogpcA5w7YZlzgS+nw98EztRhumzoX+/aDkCl1LZn08zMJlXK8W+tAra0jG8FnjfVMhFRlzQALAMeaV1I0oXAhenoPkkbH2dNy0/5cHbdbWA5eJvbgLe5PTyRbT5uqhl5BsOMiYirgKue6Hok9UVE7wyUdMTwNrcHb3N7OFzbnOd5lG3Ampbx1em0SZeRVAIWAztzqc7MzIB8g+EWYJ2ktZIqwHnAhgnLbADekA6/Crg+xnq7MzOzXOR2KiltM7gIuA4oAl+IiDslXQr0RcQG4PPA1ZI2AbtIwuNwesKno45A3ub24G1uD4dlm+Uv5GZm1srXapqZWYaDwczMMto2GA7WPceRQtIaSTdIukvSnZLekU4/StK/SPpt+u/SdLokfTLd7tslndyyrjeky/9W0hum+ptzhaSipF9I+l46vjbtSmVT2rVKJZ0+ZVcrki5Jp2+U9NLZ2ZLpkbRE0jcl/VrS3ZJOm+/7WdL/SN/Xd0j6mqTO+bafJX1B0g5Jd7RMm7H9Kum5kn6VPueT0/rRcES03YOk8fse4ASgAtwGrJ/tuh7nthwDnJwOLwR+Q9LlyEeAi9PpFwMfTodfBvyApMfxU4Gb0+lHAfem/y5Nh5fO9vYdZNv/DPhH4Hvp+NeB89LhK4D/ng6/FbgiHT4P+Kd0eH267zuAtel7ojjb23WA7f0y8KfpcAVYMp/3M8kPXu8Dulr27xvn234G/gA4GbijZdqM7Vfg39NllT737IPWNNsvyiztiNOA61rGLwEume26ZmjbvgO8GNgIHJNOOwbYmA5fCZzfsvzGdP75wJUt0zPLzbUHye9gfgT8IfC99E3/CFCauI9JroQ7LR0upctp4n5vXW6uPUh+03Mf6QUjE/fffNzP7O8J4ah0v30PeOl83M/A8ROCYUb2azrv1y3TM8tN9WjXU0mTdc+xapZqmTHpofNzgJuBoyPioXTWw8DR6fBU236kvSZ/B7wHaKbjy4DdEVFPx1vrz3S1Aox1tXIkbfNaoB/4Ynr67HOSepjH+zkitgGXAw8AD5Hst1uZ3/t5zEzt11Xp8MTpB9SuwTDvSFoA/DPwzojY0zovkq8K8+a6ZEmvAHZExK2zXUuOSiSnGz4bEc8BBklOMYybh/t5KUnHmmuBY4Ee4KxZLWoWzMZ+bddgmE73HEcMSWWSUPhqRHwrnbxd0jHp/GOAHen0qbb9SHpNng+cI2kzSS+9fwh8AliipCsVyNY/VVcrR9I2bwW2RsTN6fg3SYJiPu/nFwH3RUR/RNSAb5Hs+/m8n8fM1H7dlg5PnH5A7RoM0+me44iQXmHweeDuiPhYy6zW7kXeQNL2MDb99enVDacCA+kh63XASyQtTb+pvSSdNudExCURsToijifZd9dHxGuBG0i6UoHHbvNkXa1sAM5Lr2ZZC6wjaaibcyLiYWCLpP+QTjoTuIt5vJ9JTiGdKqk7fZ+PbfO83c8tZmS/pvP2SDo1fQ1f37Kuqc12o8ssNva8jOQKnnuA9852PU9gO04nOcy8Hfhl+ngZybnVHwG/Bf4VOCpdXiQ3TLoH+BXQ27KuNwOb0sebZnvbprn9Z7D/qqQTSP7DbwK+AXSk0zvT8U3p/BNanv/e9LXYyDSu1pjlbX020Jfu62+TXH0yr/cz8FfAr4E7gKtJriyaV/sZ+BpJG0qN5Mjwgpncr0Bv+vrdA3yKCRcwTPZwlxhmZpbRrqeSzMxsCg4GMzPLcDCYmVmGg8HMzDIcDGZmluFgMDsIScdLCklz6kbzc7UuO/I5GMwObgtJZ2S/BJB0RvqBvDyvAiTdKOlTB6rLbKbkds9nsyNVRDRIOjKbcZLKkXT3cMgOZ13W3nzEYHNK+s34M5L+RtIj6Q1MLpdUSOdvlvTuSZ7zqZbxzZLeJ+lLkvZK2iLpNUpudHONpH3pzUxeMs2axk/ZpD3Y3pDO6k+nfyldTpLeI+keScPpzVFeN8l6zpd0vaRh4L9KWqbkJjRb0+fdKelNLc/7EvAC4G3p8yNd12NOJUn6AyU3qRmRtF3Sx9NuX6b1+qbL/JGSm8AMS9ol6ceSxnr3tDbgYLC56LVAHfg94CLgncBrDnEd7yTpFuFkkhu7fJnkpj7XknQt8RPgK5I6D3G9W4BXpsNPJzmV8450/IMk3Rm8jeTmMJcBV0p6+YR1XAZ8Jl3m2yRdOfwceEW6zk+kzzszXf4dwE3AF9O/dwzZLpYBkLSK5EYsvyDpfv0Ckv73L5uw6JSvr6QnkXRM+GXgaSQ3kbl6Gq+LzSez3U+IH360PoAbgZsmTPsX4HPp8Gbg3ZM851Mt45uBr7WMLyDpT+qTLdOOT6f1TqOmzLIk/TMFsLxlmR5gGPj9Cc/9O+DaCet51zT+5jVj2zzZNk5R11+T9K1TaFnmjcAo0D3N1/fkdJ3HzfZ7wY/Ze7iNweai2yeMPwisfLzriIh9koZIOh0bsz3991DXO5X1JN/8/6+k1g7IyiRB1aqvdURSkeTeCq8huYlKB8mtO288xBqeBvwsIpot036arutE9r8mB3p9byPptO0OST9Mh78ZEf2HWIsdwRwMNhdNbIwN9p/2bJL0MNmqPM111CaMw8ydTh1bz38k6S76QLUMThh/N/AuklNGvwL2AX/DzIUWZG/0MuXrGxGNtO3lVJKumy8ALpP0goi4bQbrsTnMwWBHmn6Sc+wApG0EJ5GcV89LNf232DLtLpJTNsdFxPWHuL7Tge9GxNUwfo+NpwK7J/zN4iTPbXU38GpJhZajhtPT594z3WIiIkjaNG6SdClwJ8nRjIOhTTgY7EhzPfBmSRtIQuK95P8+vp/kW/bLJX0XGI6IvZIuBy5PP9h/QtK2cSrQjIirDrC+3wCvkXQ6yQ3s305yO8vWsNsMnJJeFbUP2DXJej5D0pD8GUmfILlvwYdI2iaGprNh6c1fXkRy45ftJI3Ya0iCz9qEr0qyI81lJOHwHeCHJOfQ8zxaIJKb1L+fpLF3O8nNTwD+AvhLklNDd5I06r4SuO8gq/wgyRVUPyAJlEHgqxOWuZzkm/9dJIH45CnqOpvkw/yXwBdIbgLz54eweQMkt8/8HklD9keBD0TEVw5hHXaE8416zMwsw0cMZmaW4WCwtifpivTX0JM9rpjt+szy5lNJ1vYkrQQWTTF7T0TsyLMes9nmYDAzswyfSjIzswwHg5mZZTgYzMwsw8FgZmYZ/x9fpcms79TW+AAAAABJRU5ErkJggg==\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light", "tags": [] }, "output_type": "display_data" } ], "source": [ "# Do not learn\n", "# use first reward function initially and then switch to second reward function\n", "\n", "# Instantiate learner in VW\n", "vw = pyvw.vw(\"--cb_explore_adf -q UA --quiet --epsilon 0.2\")\n", "\n", "num_iterations_per_cost_func = 5000\n", "cost_functions = [get_cost, get_cost_new1]\n", "total_iterations = num_iterations_per_cost_func * len(cost_functions)\n", "\n", "ctr = run_simulation_multiple_cost_functions(vw, num_iterations_per_cost_func, users, context1, context2, items, cost_functions, do_learn=False)\n", "plot_ctr(total_iterations, ctr)" ] } ], "metadata": { "colab": { "collapsed_sections": [], "name": "2021-06-12-vowpalwabbit-changing-context.ipynb", "provenance": [] }, "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.3" } }, "nbformat": 4, "nbformat_minor": 4 }