estimand,group_id,method,supported,matching_decisions,treated_decisions,skip_decisions,total_episodes,estimate,lower,upper,standard_error blip_randomized,domain:device_state,AIPW,True,1988,961,1027,3096,1.5173358973126525,-0.7377872966496262,3.7724590912749303,1.1501442395801236 blip_randomized,domain:device_state,SNMM,True,1988,961,1027,3096,1.4471723800714689,-0.8121857368536765,3.7065304969966144,1.1523041092776405 blip_no_future,domain:device_state,SNMM,True,1988,961,1027,3096,2.7508328867847647,-1.314021624067411,6.815687397636941,2.0731324181778215 selective_policy,domain:device_state,SNMM,True,1988,961,1027,3096,2.3175909422097907,-1.671083198943256,6.306265083362837,2.0342793685470424 blip_randomized,operation:connectivity,AIPW,True,928,456,472,3096,0.4261203394958833,-2.216092095150191,3.068332774141958,1.3475651439315508 blip_randomized,operation:connectivity,SNMM,True,928,456,472,3096,0.4134786278850842,-2.2171231156968405,3.0440803714670093,1.3416435297684182 blip_no_future,operation:connectivity,SNMM,True,928,456,472,3096,1.452863218276898,-1.8767710973769989,4.782497533930795,1.6981598780548453 selective_policy,operation:connectivity,SNMM,True,928,456,472,3096,0.9909140035484154,-1.3371945947239554,3.319022601820786,1.1873678123611693 blip_randomized,tool:get_cellular_service_status,AIPW,True,85,44,41,3096,4.5350559288988705,-6.668713565559157,15.7388254233569,5.714078494751164 blip_randomized,tool:get_cellular_service_status,SNMM,True,85,44,41,3096,4.53505592889884,-6.668713565559198,15.73882542335688,5.71407849475117 blip_no_future,tool:get_cellular_service_status,SNMM,True,85,44,41,3096,6.334154614316089,-5.410212769440735,18.078521998072915,5.989790948053432 selective_policy,tool:get_cellular_service_status,SNMM,True,85,44,41,3096,0.107161620848839,-0.2623258494870514,0.4766490911847295,0.1884437563063637 blip_randomized,tool:set_cellular_service_status,AIPW,True,247,118,129,3096,-1.5075016151098857,-4.717633311063455,1.7026300808436832,1.6372118775061155 blip_randomized,tool:set_cellular_service_status,SNMM,True,247,118,129,3096,-1.5075016151099063,-4.717633311063476,1.7026300808436632,1.6372118775061155 blip_no_future,tool:set_cellular_service_status,SNMM,True,247,118,129,3096,-0.5524017232336866,-4.439570811936866,3.334767365469493,1.9825103779765672 selective_policy,tool:set_cellular_service_status,SNMM,True,247,118,129,3096,0.1491065814862358,-0.5894008853341612,0.8876140483066328,0.3766490944372756 blip_randomized,tool:get_wifi_status,AIPW,True,167,82,85,3096,5.310500307153223,-2.8019211041637564,13.4229217184702,4.137447914266254 blip_randomized,tool:get_wifi_status,SNMM,True,167,82,85,3096,5.310500307153185,-2.801921104163796,13.422921718470166,4.137447914266254 blip_no_future,tool:get_wifi_status,SNMM,True,167,82,85,3096,7.044947972854594,-1.379960600520126,15.469856546229316,4.296820719417519 selective_policy,tool:get_wifi_status,SNMM,True,167,82,85,3096,0.6482776937578594,-0.3751179193905382,1.6716733069062568,0.5219460171513303 blip_randomized,tool:set_wifi_status,AIPW,True,431,213,218,3096,-1.2242587318384182,-5.054326150955975,2.6058086872791395,1.95338773114284 blip_randomized,tool:set_wifi_status,SNMM,True,431,213,218,3096,-1.224258731838428,-5.054326150955981,2.605808687279125,1.9533877311428376 blip_no_future,tool:set_wifi_status,SNMM,True,431,213,218,3096,-0.5679686500838003,-4.37213457297173,3.2361972728041284,1.940177607294718 selective_policy,tool:set_wifi_status,SNMM,True,431,213,218,3096,0.1135293953824279,-1.1607809724241391,1.387839763188995,0.6499160369127344 blip_randomized,operation:location_access,AIPW,True,734,340,394,3096,2.0749997333942667,-2.950712934930437,7.10071240171897,2.563183461876627 blip_randomized,operation:location_access,SNMM,True,734,340,394,3096,2.016503582186165,-2.998236365636006,7.031243530008336,2.557587221585942 blip_no_future,operation:location_access,SNMM,True,734,340,394,3096,4.333116952054499,-2.4703321201375155,11.136566024246513,3.469853789189164 selective_policy,operation:location_access,SNMM,True,734,340,394,3096,1.6909948947650744,-1.3281613722068295,4.710151161736978,1.5398117487093033 blip_randomized,tool:get_location_service_status,AIPW,True,258,110,148,3096,7.2341224052333635,-2.6913668034651783,17.159611613931904,5.062137744387129 blip_randomized,tool:get_location_service_status,SNMM,True,258,110,148,3096,7.2341224052333315,-2.691366803465213,17.159611613931876,5.0621377443871305 blip_no_future,tool:get_location_service_status,SNMM,True,258,110,148,3096,8.292618237960717,-3.712818994158244,20.29805547007968,6.122940207049874 selective_policy,tool:get_location_service_status,SNMM,True,258,110,148,3096,1.3571202793462134,-0.725980631543572,3.4402211902359987,1.0624104791873603 blip_randomized,tool:set_location_service_status,AIPW,True,477,230,247,3096,-0.8069551023471894,-6.335703120966511,4.721792916272133,2.8197384970940083 blip_randomized,tool:set_location_service_status,SNMM,True,477,230,247,3096,-0.8069551023472337,-6.335703120966548,4.721792916272082,2.819738497094005 blip_no_future,tool:set_location_service_status,SNMM,True,477,230,247,3096,2.184779735459583,-4.093666149759256,8.463225620678422,3.2020948512849916 selective_policy,tool:set_location_service_status,SNMM,True,477,230,247,3096,0.7713505309589697,-1.4216775105246982,2.9643785724426377,1.118474846918856 blip_randomized,operation:battery_mode,AIPW,True,341,171,170,3096,2.7838467237787077,-2.184564939138812,7.752258386696227,2.533959150997439 blip_randomized,operation:battery_mode,SNMM,True,341,171,170,3096,2.7838467237786886,-2.1845649391388293,7.752258386696206,2.533959150997438 blip_no_future,operation:battery_mode,SNMM,True,341,171,170,3096,2.46653658213965,-3.500852189941413,8.433925354220712,3.043451391001455 selective_policy,operation:battery_mode,SNMM,True,341,171,170,3096,0.7496696351704246,-1.2946867188185152,2.7940259891593646,1.0426502155113453 blip_randomized,tool:get_low_battery_mode_status,AIPW,True,48,20,28,3096,1.8354474318402776,-9.739695104229176,13.410589967909733,5.903483918671378 blip_randomized,tool:get_low_battery_mode_status,SNMM,True,48,20,28,3096,1.8354474318402825,-9.739695104229153,13.410589967909717,5.903483918671369 blip_no_future,tool:get_low_battery_mode_status,SNMM,True,48,20,28,3096,5.209082185974811,-9.222328411651448,19.640492783601072,7.360220413817903 selective_policy,tool:get_low_battery_mode_status,SNMM,True,48,20,28,3096,0.2962692185967452,-0.5486711576118917,1.1412095948053822,0.4309313606843608 blip_randomized,tool:set_low_battery_mode_status,AIPW,True,293,151,142,3096,2.9392158910587245,-2.448988593614327,8.327420375731776,2.748059337209832 blip_randomized,tool:set_low_battery_mode_status,SNMM,True,293,151,142,3096,2.939215891058702,-2.44898859361435,8.327420375731753,2.748059337209832 blip_no_future,tool:set_low_battery_mode_status,SNMM,True,293,151,142,3096,2.017245834753687,-4.2037707793845875,8.23826244889196,3.17280512312582 selective_policy,tool:set_low_battery_mode_status,SNMM,True,293,151,142,3096,0.5428042372656927,-1.0634884740481916,2.1490969485795772,0.8192316561434404 blip_randomized,domain:contacts,AIPW,True,1623,839,784,3096,5.7476495073218965,3.73390321863471,7.761395796009082,1.027038655852768 blip_randomized,domain:contacts,SNMM,True,1623,839,784,3096,5.743032835543929,3.7316919481838218,7.7543737229040355,1.0258118676721375 blip_no_future,domain:contacts,SNMM,True,1623,839,784,3096,6.947916176598905,2.6924217540544624,11.203410599143346,2.170361428483762 selective_policy,domain:contacts,SNMM,True,1623,839,784,3096,3.641638161845384,0.9835026680610692,6.299773655629699,1.355686125877625 blip_randomized,operation:find_person,AIPW,True,795,410,385,3096,4.550855198136941,1.6472681968712302,7.454442199402653,1.48086981348361 blip_randomized,operation:find_person,SNMM,True,795,410,385,3096,4.550855198136972,1.6472681968712624,7.454442199402682,1.4808698134836096 blip_no_future,operation:find_person,SNMM,True,795,410,385,3096,6.043730877187339,0.1014926198963518,11.985969134478324,3.030624278147609 selective_policy,operation:find_person,SNMM,True,795,410,385,3096,1.7080152555223993,-0.3523501194509914,3.76838063049579,1.0508150391963131 blip_randomized,tool:search_contacts,AIPW,True,795,410,385,3096,4.550855198136941,1.6472681968712302,7.454442199402653,1.48086981348361 blip_randomized,tool:search_contacts,SNMM,True,795,410,385,3096,4.550855198136972,1.6472681968712624,7.454442199402682,1.4808698134836096 blip_no_future,tool:search_contacts,SNMM,True,795,410,385,3096,6.043730877187339,0.1014926198963518,11.985969134478324,3.030624278147609 selective_policy,tool:search_contacts,SNMM,True,795,410,385,3096,1.7080152555223993,-0.3523501194509914,3.76838063049579,1.0508150391963131 blip_randomized,operation:add_contact,AIPW,True,91,46,45,3096,6.940471735513445,-3.954130967311759,17.83507443833865,5.556399124764645 blip_randomized,operation:add_contact,SNMM,True,91,46,45,3096,6.940471735513467,-3.9541309673117766,17.83507443833871,5.556399124764664 blip_no_future,operation:add_contact,SNMM,True,91,46,45,3096,7.284825895808393,-5.469730197764923,20.03938198938171,6.505001260552107 selective_policy,operation:add_contact,SNMM,True,91,46,45,3096,0.4553903042721852,-0.4914243009966148,1.4022049095409852,0.482888636468192 blip_randomized,tool:add_contact,AIPW,True,91,46,45,3096,6.940471735513445,-3.954130967311759,17.83507443833865,5.556399124764645 blip_randomized,tool:add_contact,SNMM,True,91,46,45,3096,6.940471735513467,-3.9541309673117766,17.83507443833871,5.556399124764664 blip_no_future,tool:add_contact,SNMM,True,91,46,45,3096,7.284825895808393,-5.469730197764923,20.03938198938171,6.505001260552107 selective_policy,tool:add_contact,SNMM,True,91,46,45,3096,0.4553903042721852,-0.4914243009966148,1.4022049095409852,0.482888636468192 blip_randomized,operation:change_remove_contact,AIPW,True,739,385,354,3096,6.872178437931671,4.141318282860001,9.60303859300334,1.392776716085295 blip_randomized,operation:change_remove_contact,SNMM,True,739,385,354,3096,6.872178437931674,4.141318282860003,9.603038593003346,1.3927767160852955 blip_no_future,operation:change_remove_contact,SNMM,True,739,385,354,3096,7.859537499665725,4.062444911451273,11.656630087880178,1.9365701080897513 selective_policy,operation:change_remove_contact,SNMM,True,739,385,354,3096,2.2053741573867303,0.1338755703964834,4.276872744376977,1.0564931323947222 blip_randomized,tool:modify_contact,AIPW,True,468,245,223,3096,3.860982818758132,0.8809606656440914,6.841004971872174,1.5198528055592349 blip_randomized,tool:modify_contact,SNMM,True,468,245,223,3096,3.8609828187581177,0.880960665644076,6.8410049718721595,1.519852805559235 blip_no_future,tool:modify_contact,SNMM,True,468,245,223,3096,4.59571795187932,0.7737705728148726,8.417665330943766,1.9492463976151424 selective_policy,tool:modify_contact,SNMM,True,468,245,223,3096,1.0232400667305463,-0.6062704728360773,2.65275060629717,0.8310730719436699 blip_randomized,tool:remove_contact,AIPW,True,271,140,131,3096,12.07232437805424,6.896455938103041,17.24819281800544,2.6397649968625214 blip_randomized,tool:remove_contact,SNMM,True,271,140,131,3096,12.072324378054246,6.896455938103041,17.24819281800545,2.639764996862524 blip_no_future,tool:remove_contact,SNMM,True,271,140,131,3096,13.495949117245178,6.797457057761645,20.194441176728713,3.416324250805898 selective_policy,tool:remove_contact,SNMM,True,271,140,131,3096,1.259019611973311,0.3783379393601143,2.139701284586508,0.4491599196761219 blip_randomized,domain:messaging,AIPW,True,815,419,396,3096,7.121026543151579,4.061257980672417,10.18079510563074,1.5605245850895315 blip_randomized,domain:messaging,SNMM,True,815,419,396,3096,7.121026543151541,4.061257980672389,10.180795105630692,1.5605245850895266 blip_no_future,domain:messaging,SNMM,True,815,419,396,3096,8.464450272508456,3.043937784816203,13.88496276020071,2.7645368687541656 selective_policy,domain:messaging,SNMM,True,815,419,396,3096,2.3084995637787764,0.3016463580436949,4.315352769513858,1.023523087536351 blip_randomized,operation:find_messages,AIPW,True,545,280,265,3096,6.3992455945297495,2.268708670723589,10.52978251833591,2.1066313636469673 blip_randomized,operation:find_messages,SNMM,True,545,280,265,3096,6.399245594529724,2.2687086707235813,10.529782518335864,2.106631363646958 blip_no_future,operation:find_messages,SNMM,True,545,280,265,3096,8.630529276483127,1.3226530045521754,15.938405548414082,3.7271186870096313 selective_policy,operation:find_messages,SNMM,True,545,280,265,3096,1.4061856067154377,-0.1832648889755515,2.995636102406427,0.8106418915876495 blip_randomized,tool:search_messages,AIPW,True,545,280,265,3096,6.3992455945297495,2.268708670723589,10.52978251833591,2.1066313636469673 blip_randomized,tool:search_messages,SNMM,True,545,280,265,3096,6.399245594529724,2.2687086707235813,10.529782518335864,2.106631363646958 blip_no_future,tool:search_messages,SNMM,True,545,280,265,3096,8.630529276483127,1.3226530045521754,15.938405548414082,3.7271186870096313 selective_policy,tool:search_messages,SNMM,True,545,280,265,3096,1.4061856067154377,-0.1832648889755515,2.995636102406427,0.8106418915876495 blip_randomized,operation:send_message,AIPW,True,270,139,131,3096,8.5779547542586,4.554712053407327,12.601197455109874,2.0519098154840187 blip_randomized,operation:send_message,SNMM,True,270,139,131,3096,8.577954754258556,4.554712053407282,12.60119745510983,2.051909815484019 blip_no_future,operation:send_message,SNMM,True,270,139,131,3096,8.129216727448487,3.7854696867765263,12.472963768120444,2.2153714929622734 selective_policy,operation:send_message,SNMM,True,270,139,131,3096,0.956504794121238,0.2014583452256752,1.7115512430168005,0.3850842056601123 blip_randomized,tool:send_message_with_phone_number,AIPW,True,270,139,131,3096,8.5779547542586,4.554712053407327,12.601197455109874,2.0519098154840187 blip_randomized,tool:send_message_with_phone_number,SNMM,True,270,139,131,3096,8.577954754258556,4.554712053407282,12.60119745510983,2.051909815484019 blip_no_future,tool:send_message_with_phone_number,SNMM,True,270,139,131,3096,8.129216727448487,3.7854696867765263,12.472963768120444,2.2153714929622734 selective_policy,tool:send_message_with_phone_number,SNMM,True,270,139,131,3096,0.956504794121238,0.2014583452256752,1.7115512430168005,0.3850842056601123 blip_randomized,domain:reminders,AIPW,True,2177,1085,1092,3096,6.421779313819352,3.845052128837878,8.998506498800825,1.3141667544861952 blip_randomized,domain:reminders,SNMM,True,2177,1085,1092,3096,6.421779313819333,3.845052128837861,8.998506498800804,1.314166754486194 blip_no_future,domain:reminders,SNMM,True,2177,1085,1092,3096,5.474526572432804,0.7948627453449202,10.154190399520688,2.386693732877564 selective_policy,domain:reminders,SNMM,True,2177,1085,1092,3096,5.348982493698983,1.1032727423069322,9.594692245091036,2.1653711099094872 blip_randomized,operation:find_reminders,AIPW,True,1153,561,592,3096,1.7787387182630472,-1.8502728390154208,5.407750275541515,1.8508464411826453 blip_randomized,operation:find_reminders,SNMM,True,1153,561,592,3096,1.7787387182630323,-1.8502728390154344,5.407750275541499,1.850846441182645 blip_no_future,operation:find_reminders,SNMM,True,1153,561,592,3096,1.6110565458147643,-5.1259127318751325,8.348025823504662,3.4359481680240793 selective_policy,operation:find_reminders,SNMM,True,1153,561,592,3096,1.062399928644775,-1.8868874875827983,4.011687344872349,1.5041776616559557 blip_randomized,tool:search_reminder,AIPW,True,1153,561,592,3096,1.7787387182630472,-1.8502728390154208,5.407750275541515,1.8508464411826453 blip_randomized,tool:search_reminder,SNMM,True,1153,561,592,3096,1.7787387182630323,-1.8502728390154344,5.407750275541499,1.850846441182645 blip_no_future,tool:search_reminder,SNMM,True,1153,561,592,3096,1.6110565458147643,-5.1259127318751325,8.348025823504662,3.4359481680240793 selective_policy,tool:search_reminder,SNMM,True,1153,561,592,3096,1.062399928644775,-1.8868874875827983,4.011687344872349,1.5041776616559557 blip_randomized,operation:create_reminders,AIPW,True,735,380,355,3096,13.258856766843552,8.849988917795454,17.66772461589165,2.2485840122743643 blip_randomized,operation:create_reminders,SNMM,True,735,380,355,3096,13.258856766843536,8.849988917795441,17.66772461589163,2.2485840122743626 blip_no_future,operation:create_reminders,SNMM,True,735,380,355,3096,10.499445050894185,5.870740764259532,15.128149337528834,2.3607036574524796 selective_policy,operation:create_reminders,SNMM,True,735,380,355,3096,3.6877294562853824,1.3289564400954794,6.046502472475286,1.2030070926108485 blip_randomized,tool:add_reminder,AIPW,True,735,380,355,3096,13.258856766843552,8.849988917795454,17.66772461589165,2.2485840122743643 blip_randomized,tool:add_reminder,SNMM,True,735,380,355,3096,13.258856766843536,8.849988917795441,17.66772461589163,2.2485840122743626 blip_no_future,tool:add_reminder,SNMM,True,735,380,355,3096,10.499445050894185,5.870740764259532,15.128149337528834,2.3607036574524796 selective_policy,tool:add_reminder,SNMM,True,735,380,355,3096,3.6877294562853824,1.3289564400954794,6.046502472475286,1.2030070926108485 blip_randomized,operation:change_remove_reminders,AIPW,True,289,144,145,3096,7.557329067119127,1.4896733032439484,13.624984830994306,3.094588953393073 blip_randomized,operation:change_remove_reminders,SNMM,True,289,144,145,3096,7.55732906711909,1.48967330324392,13.624984830994258,3.0945889533930684 blip_no_future,operation:change_remove_reminders,SNMM,True,289,144,145,3096,8.108664492922408,2.0054794363131903,14.211849549531626,3.1127093875599594 selective_policy,operation:change_remove_reminders,SNMM,True,289,144,145,3096,1.2678690502494483,-0.5543651871697478,3.0901032876686445,0.9293648422155446 blip_randomized,tool:modify_reminder,AIPW,True,171,85,86,3096,14.750102694418896,6.1038962483675965,23.39630914047019,4.40968571684726 blip_randomized,tool:modify_reminder,SNMM,True,171,85,86,3096,14.750102694418866,6.103896248367572,23.39630914047016,4.409685716847258 blip_no_future,tool:modify_reminder,SNMM,True,171,85,86,3096,14.716985451385517,5.526681591508119,23.907289311262907,4.687182976401851 selective_policy,tool:modify_reminder,SNMM,True,171,85,86,3096,1.3828011512947105,-0.2284096149604033,2.9940119175498237,0.8217399326649795 blip_randomized,tool:remove_reminder,AIPW,True,118,59,59,3096,-2.8660971215949456,-9.91185322403949,4.179658980849599,3.593433749610331 blip_randomized,tool:remove_reminder,SNMM,True,118,59,59,3096,-2.866097121594984,-9.911853224039533,4.179658980849565,3.593433749610333 blip_no_future,tool:remove_reminder,SNMM,True,118,59,59,3096,-1.4678006248503848,-7.426456079686767,4.490854829985999,3.038997277563978 selective_policy,tool:remove_reminder,SNMM,True,118,59,59,3096,-0.067290167816858,-0.6857616327324495,0.5511812970987334,0.3154290615350065 blip_randomized,domain:time_calculations,AIPW,True,2735,1399,1336,3096,1.8173146995958152,-0.15711359904697,3.7917429982386,1.0069859332864315 blip_randomized,domain:time_calculations,SNMM,True,2735,1399,1336,3096,1.7975842360134342,-0.1756310858696121,3.77079955789648,1.0063672982439231 blip_no_future,domain:time_calculations,SNMM,True,2735,1399,1336,3096,3.09907200573948,-1.2657916648054244,7.463935676284385,2.226141268322102 selective_policy,domain:time_calculations,SNMM,True,2735,1399,1336,3096,2.3893231737402876,-2.5146950615946047,7.29334140907518,2.501117606021418 blip_randomized,operation:current_time,AIPW,True,834,423,411,3096,-0.1881546876740833,-3.383880185350733,3.0075708100025667,1.6298645157270302 blip_randomized,operation:current_time,SNMM,True,834,423,411,3096,-0.1881546876740799,-3.383880185350737,3.007570810002577,1.6298645157270335 blip_no_future,operation:current_time,SNMM,True,834,423,411,3096,1.7696757123684348,-5.8094187195098055,9.348770144246677,3.8654437262660504 selective_policy,operation:current_time,SNMM,True,834,423,411,3096,0.6536856492548648,-2.7697733165869716,4.077144615096701,1.7460117564946556 blip_randomized,tool:get_current_timestamp,AIPW,True,834,423,411,3096,-0.1881546876740833,-3.383880185350733,3.0075708100025667,1.6298645157270302 blip_randomized,tool:get_current_timestamp,SNMM,True,834,423,411,3096,-0.1881546876740799,-3.383880185350737,3.007570810002577,1.6298645157270335 blip_no_future,tool:get_current_timestamp,SNMM,True,834,423,411,3096,1.7696757123684348,-5.8094187195098055,9.348770144246677,3.8654437262660504 selective_policy,tool:get_current_timestamp,SNMM,True,834,423,411,3096,0.6536856492548648,-2.7697733165869716,4.077144615096701,1.7460117564946556 blip_randomized,operation:construct_dates,AIPW,True,1173,589,584,3096,2.627744188307859,-0.7490071124579809,6.004495489073699,1.72219019673454 blip_randomized,operation:construct_dates,SNMM,True,1173,589,584,3096,2.627744188307825,-0.7490071124580114,6.004495489073662,1.7221901967345383 blip_no_future,operation:construct_dates,SNMM,True,1173,589,584,3096,3.1536766502383142,-1.231602522927526,7.538955823404154,2.2365534590177005 selective_policy,operation:construct_dates,SNMM,True,1173,589,584,3096,1.684841979793017,-1.984282935084133,5.353966894670167,1.871304812280004 blip_randomized,tool:datetime_info_to_timestamp,AIPW,True,429,210,219,3096,6.915941375310447,1.228478759639575,12.603403990981318,2.9006851522589474 blip_randomized,tool:datetime_info_to_timestamp,SNMM,True,429,210,219,3096,6.915941375310402,1.2284787596395577,12.60340399098125,2.9006851522589336 blip_no_future,tool:datetime_info_to_timestamp,SNMM,True,429,210,219,3096,8.4123988979905,1.7776972330168714,15.047100562964127,3.383790261096359 selective_policy,tool:datetime_info_to_timestamp,SNMM,True,429,210,219,3096,1.53918625583781,-0.3484088518469038,3.426781363522524,0.9626997964349396 blip_randomized,tool:timestamp_to_datetime_info,AIPW,True,744,379,365,3096,0.1551143587055609,-4.341382208569162,4.651610925980284,2.2932758791133825 blip_randomized,tool:timestamp_to_datetime_info,SNMM,True,744,379,365,3096,0.1551143587055501,-4.341382208569172,4.651610925980272,2.2932758791133816 blip_no_future,tool:timestamp_to_datetime_info,SNMM,True,744,379,365,3096,0.1214295477038058,-4.760734885184168,5.00359398059178,2.489971862381098 selective_policy,tool:timestamp_to_datetime_info,SNMM,True,744,379,365,3096,0.5425942919771831,-2.292476790954002,3.3776653749083687,1.4459257408036876 blip_randomized,operation:time_arithmetic,AIPW,True,519,274,245,3096,0.4166762531932041,-4.197416813292444,5.0307693196788525,2.353251731662968 blip_randomized,operation:time_arithmetic,SNMM,True,519,274,245,3096,0.4166762531931888,-4.197416813292476,5.030769319678853,2.353251731662977 blip_no_future,operation:time_arithmetic,SNMM,True,519,274,245,3096,2.3890124177990204,-3.233071702626009,8.011096538224049,2.867341209054096 selective_policy,operation:time_arithmetic,SNMM,True,519,274,245,3096,0.6563402416421948,-1.7469575738109464,3.059638057095336,1.2257153604022404 blip_randomized,tool:shift_timestamp,AIPW,True,342,182,160,3096,-2.951545649825792,-8.541016378247901,2.637925078596316,2.8507079248744307 blip_randomized,tool:shift_timestamp,SNMM,True,342,182,160,3096,-2.9515456498258312,-8.541016378247962,2.637925078596299,2.850707924874442 blip_no_future,tool:shift_timestamp,SNMM,True,342,182,160,3096,-0.9476641971107314,-7.056627368406962,5.161298974185499,3.1156563065312564 selective_policy,tool:shift_timestamp,SNMM,True,342,182,160,3096,0.2092777519487554,-1.5781467622760088,1.9967022661735196,0.9116113985364386 blip_randomized,tool:timestamp_diff,AIPW,True,177,92,85,3096,6.924766031907878,-0.7387593511557446,14.5882914149715,3.908504687373687 blip_randomized,tool:timestamp_diff,SNMM,True,177,92,85,3096,6.924766031907891,-0.7387593511557544,14.588291414971538,3.908504687373699 blip_no_future,tool:timestamp_diff,SNMM,True,177,92,85,3096,8.836150283895813,-0.744302746626134,18.416603314417763,4.886164487653737 selective_policy,tool:timestamp_diff,SNMM,True,177,92,85,3096,0.5279175127689162,-0.5460642947992468,1.601899320337079,0.5477456808991665 blip_randomized,operation:convert_units,AIPW,True,64,34,30,3096,6.871788628420752,-6.297021750310675,20.040599007152174,6.716276715955876 blip_randomized,operation:convert_units,SNMM,True,64,34,30,3096,6.871788628420758,-6.297021750310722,20.04059900715224,6.716276715955901 blip_no_future,operation:convert_units,SNMM,True,64,34,30,3096,9.200953700612544,-6.863708057424395,25.26561545864948,8.19320125449357 selective_policy,operation:convert_units,SNMM,True,64,34,30,3096,0.4763675125143862,-0.4917840533798939,1.4445190784086663,0.4937707835807077 blip_randomized,tool:unit_conversion,AIPW,True,55,29,26,3096,7.824346359646856,-6.664230027708713,22.31292274700243,7.389375762818013 blip_randomized,tool:unit_conversion,SNMM,True,55,29,26,3096,7.824346359646876,-6.664230027708758,22.31292274700251,7.389375762818045 blip_no_future,tool:unit_conversion,SNMM,True,55,29,26,3096,9.249676488224669,-7.596953833280898,26.09630680973023,8.592016113572718 selective_policy,tool:unit_conversion,SNMM,True,55,29,26,3096,0.4379588055737238,-0.4624870866110834,1.3384046977585309,0.4592399469451176 blip_randomized,tool:seconds_to_hours_minutes_seconds,AIPW,False,9,5,4,3096,1.050602493150104,-31.227481803826183,33.3286867901264,16.462272579273513 blip_randomized,tool:seconds_to_hours_minutes_seconds,SNMM,False,9,5,4,3096,1.050602493150022,-31.22748180382628,33.32868679012632,16.462272579273517 blip_no_future,tool:seconds_to_hours_minutes_seconds,SNMM,False,9,5,4,3096,8.903203331871795,-38.51154548079544,56.31795214453903,24.18218231448863 selective_policy,tool:seconds_to_hours_minutes_seconds,SNMM,False,9,5,4,3096,0.040958607500896,-0.1792105254769697,0.2611277404787618,0.1122893244616587 blip_randomized,operation:find_holidays,AIPW,True,146,79,67,3096,9.131828177214503,-1.5037435444302014,19.76739989885921,5.424289716429359 blip_randomized,operation:find_holidays,SNMM,True,146,79,67,3096,9.131828177214476,-1.5037435444302112,19.767399898859164,5.424289716429349 blip_no_future,operation:find_holidays,SNMM,True,146,79,67,3096,10.045263103814186,-0.2639660616467573,20.35449226927513,5.257850467287883 selective_policy,operation:find_holidays,SNMM,True,146,79,67,3096,0.1374403267815571,-0.3548515952774438,0.6297322488405581,0.2510757371765376 blip_randomized,tool:search_holiday,AIPW,True,146,79,67,3096,9.131828177214503,-1.5037435444302014,19.76739989885921,5.424289716429359 blip_randomized,tool:search_holiday,SNMM,True,146,79,67,3096,9.131828177214476,-1.5037435444302112,19.767399898859164,5.424289716429349 blip_no_future,tool:search_holiday,SNMM,True,146,79,67,3096,10.045263103814186,-0.2639660616467573,20.35449226927513,5.257850467287883 selective_policy,tool:search_holiday,SNMM,True,146,79,67,3096,0.1374403267815571,-0.3548515952774438,0.6297322488405581,0.2510757371765376 blip_randomized,domain:location_information,AIPW,True,1603,802,801,3096,5.657845803646225,2.9713788918396764,8.344312715452773,1.3701355436853306 blip_randomized,domain:location_information,SNMM,True,1603,802,801,3096,5.727291280329234,3.043267063790102,8.411315496868365,1.3688897350756766 blip_no_future,domain:location_information,SNMM,True,1603,802,801,3096,6.533706264246053,2.216344881315867,10.85106764717624,2.201914439999276 selective_policy,domain:location_information,SNMM,True,1603,802,801,3096,4.031368231199028,0.3422882438213218,7.720448218576735,1.8814821772010728 blip_randomized,operation:find_locations,AIPW,True,654,317,337,3096,6.677666818208272,2.0237740855094866,11.33155955090706,2.3735501157844245 blip_randomized,operation:find_locations,SNMM,True,654,317,337,3096,6.677666818208245,2.0237740855094666,11.331559550907022,2.373550115784421 blip_no_future,operation:find_locations,SNMM,True,654,317,337,3096,8.372481246072345,1.9731989819973912,14.7717635101473,3.2637230875805416 selective_policy,operation:find_locations,SNMM,True,654,317,337,3096,2.229313191767124,-0.3373604205797558,4.795986804114004,1.3090392924105811 blip_randomized,tool:get_current_location,AIPW,True,295,136,159,3096,1.6218854293141314,-4.1281574372666645,7.371928295894928,2.9326019518769164 blip_randomized,tool:get_current_location,SNMM,True,295,136,159,3096,1.621885429314111,-4.128157437266687,7.371928295894908,2.9326019518769173 blip_no_future,tool:get_current_location,SNMM,True,295,136,159,3096,4.312177329383224,-3.3594224680340705,11.983777126800518,3.91262275115917 selective_policy,tool:get_current_location,SNMM,True,295,136,159,3096,0.3991720596056236,-0.7073930927275598,1.505737211938807,0.5643636405689868 blip_randomized,tool:search_lat_lon,AIPW,True,359,181,178,3096,10.832138990140782,3.7597448784912153,17.90453310179035,3.607019505334482 blip_randomized,tool:search_lat_lon,SNMM,True,359,181,178,3096,10.832138990140743,3.759744878491185,17.904533101790303,3.6070195053344785 blip_no_future,tool:search_lat_lon,SNMM,True,359,181,178,3096,11.708942681791818,3.84114670112108,19.576738662462557,4.012685537352273 selective_policy,tool:search_lat_lon,SNMM,True,359,181,178,3096,1.8682114844285056,-0.2594871271625682,3.99591009601958,1.0851559277275715 blip_randomized,operation:nearby_places,AIPW,True,190,97,93,3096,4.41956197174177,-3.294867519396074,12.133991462879612,3.93446649151338 blip_randomized,operation:nearby_places,SNMM,True,190,97,93,3096,4.419561971741782,-3.294867519396078,12.133991462879642,3.9344664915133896 blip_no_future,operation:nearby_places,SNMM,True,190,97,93,3096,5.515771163400029,-3.0488951776988493,14.080437504498908,4.368099127583813 selective_policy,operation:nearby_places,SNMM,True,190,97,93,3096,0.7379185946864375,-0.6335264290423767,2.109363618415252,0.6994560643807144 blip_randomized,tool:search_location_around_lat_lon,AIPW,True,190,97,93,3096,4.41956197174177,-3.294867519396074,12.133991462879612,3.93446649151338 blip_randomized,tool:search_location_around_lat_lon,SNMM,True,190,97,93,3096,4.419561971741782,-3.294867519396078,12.133991462879642,3.9344664915133896 blip_no_future,tool:search_location_around_lat_lon,SNMM,True,190,97,93,3096,5.515771163400029,-3.0488951776988493,14.080437504498908,4.368099127583813 selective_policy,tool:search_location_around_lat_lon,SNMM,True,190,97,93,3096,0.7379185946864375,-0.6335264290423767,2.109363618415252,0.6994560643807144 blip_randomized,operation:weather,AIPW,True,504,257,247,3096,6.242586308396126,1.5382900103885446,10.946882606403706,2.399256636141902 blip_randomized,operation:weather,SNMM,True,504,257,247,3096,6.242586308396119,1.5382900103885482,10.946882606403689,2.399256636141897 blip_no_future,operation:weather,SNMM,True,504,257,247,3096,5.9804789586271205,1.097103166859044,10.863854750395197,2.4905896722824568 selective_policy,operation:weather,SNMM,True,504,257,247,3096,1.613348502439771,-0.0695616622699367,3.2962586671494787,0.858307624547488 blip_randomized,tool:search_weather_around_lat_lon,AIPW,True,504,257,247,3096,6.242586308396126,1.5382900103885446,10.946882606403706,2.399256636141902 blip_randomized,tool:search_weather_around_lat_lon,SNMM,True,504,257,247,3096,6.242586308396119,1.5382900103885482,10.946882606403689,2.399256636141897 blip_no_future,tool:search_weather_around_lat_lon,SNMM,True,504,257,247,3096,5.9804789586271205,1.097103166859044,10.863854750395197,2.4905896722824568 selective_policy,tool:search_weather_around_lat_lon,SNMM,True,504,257,247,3096,1.613348502439771,-0.0695616622699367,3.2962586671494787,0.858307624547488 blip_randomized,operation:financial_information,AIPW,True,99,53,46,3096,0.6398139111749096,-3.3969534225607134,4.6765812449105315,2.058807566633942 blip_randomized,operation:financial_information,SNMM,True,99,53,46,3096,0.6398139111748958,-3.3969534225607347,4.676581244910527,2.058807566633946 blip_no_future,operation:financial_information,SNMM,True,99,53,46,3096,1.5556587736360676,-3.06992917636622,6.181246723638355,2.3591142823638367 selective_policy,operation:financial_information,SNMM,True,99,53,46,3096,0.1019553491463542,-0.6126082929767497,0.8165189912694583,0.3644374103382257 blip_randomized,tool:search_stock,AIPW,True,21,12,9,3096,-1.576375021918619,-14.661003903613224,11.508253859775984,6.673342979938546 blip_randomized,tool:search_stock,SNMM,True,21,12,9,3096,-1.5763750219186616,-14.661003903613253,11.508253859775929,6.673342979938539 blip_no_future,tool:search_stock,SNMM,True,21,12,9,3096,0.0911959618838767,-12.041286238596069,12.223678162363818,6.187734910469729 selective_policy,tool:search_stock,SNMM,True,21,12,9,3096,0.0524578442501755,-0.6213929393819823,0.7263086278823333,0.343673285435617 blip_randomized,tool:convert_currency,AIPW,True,78,41,37,3096,1.236480162392398,-2.531453589433335,5.004413914218131,1.9216986953903767 blip_randomized,tool:convert_currency,SNMM,True,78,41,37,3096,1.236480162392391,-2.531453589433357,5.004413914218139,1.9216986953903843 blip_no_future,tool:convert_currency,SNMM,True,78,41,37,3096,1.9499372229539649,-2.8387831856422876,6.738657631550217,2.4423141084499367 selective_policy,tool:convert_currency,SNMM,True,78,41,37,3096,0.0906662225815813,-0.1194885996461054,0.3008210448092681,0.1071818864939588 blip_randomized,operation:distance,AIPW,True,160,80,80,3096,4.7039294865950225,-1.950894488161528,11.358753461351572,3.3940529193608113 blip_randomized,operation:distance,SNMM,True,160,80,80,3096,4.703929486594998,-1.9508944881615629,11.358753461351558,3.394052919360816 blip_no_future,operation:distance,SNMM,True,160,80,80,3096,4.685787617839759,-2.0959452700698753,11.467520505749397,3.4587782327295833 selective_policy,operation:distance,SNMM,True,160,80,80,3096,0.1767607678933247,-0.3928823468261858,0.7464038826128352,0.2905259228255197 blip_randomized,tool:calculate_lat_lon_distance,AIPW,True,160,80,80,3096,4.7039294865950225,-1.950894488161528,11.358753461351572,3.3940529193608113 blip_randomized,tool:calculate_lat_lon_distance,SNMM,True,160,80,80,3096,4.703929486594998,-1.9508944881615629,11.358753461351558,3.394052919360816 blip_no_future,tool:calculate_lat_lon_distance,SNMM,True,160,80,80,3096,4.685787617839759,-2.0959452700698753,11.467520505749397,3.4587782327295833 selective_policy,tool:calculate_lat_lon_distance,SNMM,True,160,80,80,3096,0.1767607678933247,-0.3928823468261858,0.7464038826128352,0.2905259228255197