{"expected": "track_order", "family": "intent", "group": null, "id": "easy-intent-00", "labels": ["track_order", "cancel_order", "change_address", "report_damage", "billing_question"], "provenance": {"exclude_reason": null, "label_basis": "Answer named or stated explicitly in the text; authored and reviewed before inference", "license": "MIT", "source": "JevBench v1.1 easy tier, original authored item"}, "question": {"criteria": {"billing_question": "Asks about a charge, invoice or payment", "cancel_order": "Wants to cancel an order", "change_address": "Wants to change the delivery address", "report_damage": "Received an item that is broken or damaged", "track_order": "Wants to know where an order is or when it arrives"}, "instructions": "Which intent does the user's message express?", "type": "choice"}, "split": "public", "state": "Where is my package? I ordered it last week and it still hasn't arrived."}
{"expected": "cancel_order", "family": "intent", "group": null, "id": "easy-intent-01", "labels": ["track_order", "cancel_order", "change_address", "report_damage", "billing_question"], "provenance": {"exclude_reason": null, "label_basis": "Answer named or stated explicitly in the text; authored and reviewed before inference", "license": "MIT", "source": "JevBench v1.1 easy tier, original authored item"}, "question": {"criteria": {"billing_question": "Asks about a charge, invoice or payment", "cancel_order": "Wants to cancel an order", "change_address": "Wants to change the delivery address", "report_damage": "Received an item that is broken or damaged", "track_order": "Wants to know where an order is or when it arrives"}, "instructions": "Which intent does the user's message express?", "type": "choice"}, "split": "public", "state": "Please cancel my order #4471, I don't need it anymore."}
{"expected": "change_address", "family": "intent", "group": null, "id": "easy-intent-02", "labels": ["track_order", "cancel_order", "change_address", "report_damage", "billing_question"], "provenance": {"exclude_reason": null, "label_basis": "Answer named or stated explicitly in the text; authored and reviewed before inference", "license": "MIT", "source": "JevBench v1.1 easy tier, original authored item"}, "question": {"criteria": {"billing_question": "Asks about a charge, invoice or payment", "cancel_order": "Wants to cancel an order", "change_address": "Wants to change the delivery address", "report_damage": "Received an item that is broken or damaged", "track_order": "Wants to know where an order is or when it arrives"}, "instructions": "Which intent does the user's message express?", "type": "choice"}, "split": "public", "state": "I moved. Can you ship my order to 12 Elm Street instead of the old address?"}
{"expected": "report_damage", "family": "intent", "group": null, "id": "easy-intent-03", "labels": ["track_order", "cancel_order", "change_address", "report_damage", "billing_question"], "provenance": {"exclude_reason": null, "label_basis": "Answer named or stated explicitly in the text; authored and reviewed before inference", "license": "MIT", "source": "JevBench v1.1 easy tier, original authored item"}, "question": {"criteria": {"billing_question": "Asks about a charge, invoice or payment", "cancel_order": "Wants to cancel an order", "change_address": "Wants to change the delivery address", "report_damage": "Received an item that is broken or damaged", "track_order": "Wants to know where an order is or when it arrives"}, "instructions": "Which intent does the user's message express?", "type": "choice"}, "split": "public", "state": "The mug I received arrived smashed into pieces."}
{"expected": "billing_question", "family": "intent", "group": null, "id": "easy-intent-04", "labels": ["track_order", "cancel_order", "change_address", "report_damage", "billing_question"], "provenance": {"exclude_reason": null, "label_basis": "Answer named or stated explicitly in the text; authored and reviewed before inference", "license": "MIT", "source": "JevBench v1.1 easy tier, original authored item"}, "question": {"criteria": {"billing_question": "Asks about a charge, invoice or payment", "cancel_order": "Wants to cancel an order", "change_address": "Wants to change the delivery address", "report_damage": "Received an item that is broken or damaged", "track_order": "Wants to know where an order is or when it arrives"}, "instructions": "Which intent does the user's message express?", "type": "choice"}, "split": "public", "state": "Why was I charged twice on my credit card statement for one order?"}
{"expected": "track_order", "family": "intent", "group": null, "id": "easy-intent-05", "labels": ["track_order", "cancel_order", "change_address", "report_damage", "billing_question"], "provenance": {"exclude_reason": null, "label_basis": "Answer named or stated explicitly in the text; authored and reviewed before inference", "license": "MIT", "source": "JevBench v1.1 easy tier, original authored item"}, "question": {"criteria": {"billing_question": "Asks about a charge, invoice or payment", "cancel_order": "Wants to cancel an order", "change_address": "Wants to change the delivery address", "report_damage": "Received an item that is broken or damaged", "track_order": "Wants to know where an order is or when it arrives"}, "instructions": "Which intent does the user's message express?", "type": "choice"}, "split": "public", "state": "When will my order be delivered? The tracking page shows nothing."}
{"expected": "set_alarm", "family": "intent", "group": null, "id": "easy-intent-06", "labels": ["set_alarm", "play_music", "weather", "send_message", "turn_off_lights"], "provenance": {"exclude_reason": null, "label_basis": "Answer named or stated explicitly in the text; authored and reviewed before inference", "license": "MIT", "source": "JevBench v1.1 easy tier, original authored item"}, "question": {"criteria": {"play_music": "Play a song, album, artist or playlist", "send_message": "Send a text message to someone", "set_alarm": "Set an alarm or wake-up time", "turn_off_lights": "Switch lights off", "weather": "Ask about the weather or forecast"}, "instructions": "Which intent does the user's message express?", "type": "choice"}, "split": "public", "state": "Wake me up at 6:30 tomorrow morning."}
{"expected": "play_music", "family": "intent", "group": null, "id": "easy-intent-07", "labels": ["set_alarm", "play_music", "weather", "send_message", "turn_off_lights"], "provenance": {"exclude_reason": null, "label_basis": "Answer named or stated explicitly in the text; authored and reviewed before inference", "license": "MIT", "source": "JevBench v1.1 easy tier, original authored item"}, "question": {"criteria": {"play_music": "Play a song, album, artist or playlist", "send_message": "Send a text message to someone", "set_alarm": "Set an alarm or wake-up time", "turn_off_lights": "Switch lights off", "weather": "Ask about the weather or forecast"}, "instructions": "Which intent does the user's message express?", "type": "choice"}, "split": "public", "state": "Play some Taylor Swift."}
{"expected": "weather", "family": "intent", "group": null, "id": "easy-intent-08", "labels": ["set_alarm", "play_music", "weather", "send_message", "turn_off_lights"], "provenance": {"exclude_reason": null, "label_basis": "Answer named or stated explicitly in the text; authored and reviewed before inference", "license": "MIT", "source": "JevBench v1.1 easy tier, original authored item"}, "question": {"criteria": {"play_music": "Play a song, album, artist or playlist", "send_message": "Send a text message to someone", "set_alarm": "Set an alarm or wake-up time", "turn_off_lights": "Switch lights off", "weather": "Ask about the weather or forecast"}, "instructions": "Which intent does the user's message express?", "type": "choice"}, "split": "public", "state": "Will it rain in Berlin tomorrow?"}
{"expected": "send_message", "family": "intent", "group": null, "id": "easy-intent-09", "labels": ["set_alarm", "play_music", "weather", "send_message", "turn_off_lights"], "provenance": {"exclude_reason": null, "label_basis": "Answer named or stated explicitly in the text; authored and reviewed before inference", "license": "MIT", "source": "JevBench v1.1 easy tier, original authored item"}, "question": {"criteria": {"play_music": "Play a song, album, artist or playlist", "send_message": "Send a text message to someone", "set_alarm": "Set an alarm or wake-up time", "turn_off_lights": "Switch lights off", "weather": "Ask about the weather or forecast"}, "instructions": "Which intent does the user's message express?", "type": "choice"}, "split": "public", "state": "Text Anna that I'll be ten minutes late."}
{"expected": "turn_off_lights", "family": "intent", "group": null, "id": "easy-intent-10", "labels": ["set_alarm", "play_music", "weather", "send_message", "turn_off_lights"], "provenance": {"exclude_reason": null, "label_basis": "Answer named or stated explicitly in the text; authored and reviewed before inference", "license": "MIT", "source": "JevBench v1.1 easy tier, original authored item"}, "question": {"criteria": {"play_music": "Play a song, album, artist or playlist", "send_message": "Send a text message to someone", "set_alarm": "Set an alarm or wake-up time", "turn_off_lights": "Switch lights off", "weather": "Ask about the weather or forecast"}, "instructions": "Which intent does the user's message express?", "type": "choice"}, "split": "public", "state": "Turn off the lights in the bedroom."}
{"expected": "set_alarm", "family": "intent", "group": null, "id": "easy-intent-11", "labels": ["set_alarm", "play_music", "weather", "send_message", "turn_off_lights"], "provenance": {"exclude_reason": null, "label_basis": "Answer named or stated explicitly in the text; authored and reviewed before inference", "license": "MIT", "source": "JevBench v1.1 easy tier, original authored item"}, "question": {"criteria": {"play_music": "Play a song, album, artist or playlist", "send_message": "Send a text message to someone", "set_alarm": "Set an alarm or wake-up time", "turn_off_lights": "Switch lights off", "weather": "Ask about the weather or forecast"}, "instructions": "Which intent does the user's message express?", "type": "choice"}, "split": "public", "state": "Set an alarm for 7 am."}
{"expected": "yes", "family": "fact", "group": null, "id": "easy-fact-00", "labels": ["no", "yes"], "provenance": {"exclude_reason": null, "label_basis": "Answer named or stated explicitly in the text; authored and reviewed before inference", "license": "MIT", "source": "JevBench v1.1 easy tier, original authored item"}, "question": {"criteria": {"false": "The text states that this is not so", "true": "The text states that this is so"}, "instructions": "Has the order been shipped? Answer strictly from the facts stated in the text.", "type": "noul"}, "split": "public", "state": "Order #1182. Status: shipped on 3 September. Carrier: DHL."}
{"expected": "no", "family": "fact", "group": null, "id": "easy-fact-01", "labels": ["no", "yes"], "provenance": {"exclude_reason": null, "label_basis": "Answer named or stated explicitly in the text; authored and reviewed before inference", "license": "MIT", "source": "JevBench v1.1 easy tier, original authored item"}, "question": {"criteria": {"false": "The text states that this is not so", "true": "The text states that this is so"}, "instructions": "Has the order been shipped? Answer strictly from the facts stated in the text.", "type": "noul"}, "split": "public", "state": "Order #1183. Status: not yet shipped, waiting for stock."}
{"expected": "yes", "family": "fact", "group": null, "id": "easy-fact-02", "labels": ["no", "yes"], "provenance": {"exclude_reason": null, "label_basis": "Answer named or stated explicitly in the text; authored and reviewed before inference", "license": "MIT", "source": "JevBench v1.1 easy tier, original authored item"}, "question": {"criteria": {"false": "The text states that this is not so", "true": "The text states that this is so"}, "instructions": "Is the invoice paid? Answer strictly from the facts stated in the text.", "type": "noul"}, "split": "public", "state": "Invoice 2026-044. Amount: 120 EUR. Payment status: paid in full."}
{"expected": "no", "family": "fact", "group": null, "id": "easy-fact-03", "labels": ["no", "yes"], "provenance": {"exclude_reason": null, "label_basis": "Answer named or stated explicitly in the text; authored and reviewed before inference", "license": "MIT", "source": "JevBench v1.1 easy tier, original authored item"}, "question": {"criteria": {"false": "The text states that this is not so", "true": "The text states that this is so"}, "instructions": "Is the invoice paid? Answer strictly from the facts stated in the text.", "type": "noul"}, "split": "public", "state": "Invoice 2026-045. Amount: 80 EUR. Payment status: unpaid, overdue since 1 August."}
{"expected": "yes", "family": "fact", "group": null, "id": "easy-fact-04", "labels": ["no", "yes"], "provenance": {"exclude_reason": null, "label_basis": "Answer named or stated explicitly in the text; authored and reviewed before inference", "license": "MIT", "source": "JevBench v1.1 easy tier, original authored item"}, "question": {"criteria": {"false": "The text states that this is not so", "true": "The text states that this is so"}, "instructions": "Is the guest allergic to peanuts? Answer strictly from the facts stated in the text.", "type": "noul"}, "split": "public", "state": "Guest profile: Maria Lopez. Allergies: peanuts. Diet: vegetarian."}
{"expected": "no", "family": "fact", "group": null, "id": "easy-fact-05", "labels": ["no", "yes"], "provenance": {"exclude_reason": null, "label_basis": "Answer named or stated explicitly in the text; authored and reviewed before inference", "license": "MIT", "source": "JevBench v1.1 easy tier, original authored item"}, "question": {"criteria": {"false": "The text states that this is not so", "true": "The text states that this is so"}, "instructions": "Is the guest allergic to peanuts? Answer strictly from the facts stated in the text.", "type": "noul"}, "split": "public", "state": "Guest profile: Jonas Weber. Allergies: none. Diet: no restrictions."}
{"expected": "yes", "family": "fact", "group": null, "id": "easy-fact-06", "labels": ["no", "yes"], "provenance": {"exclude_reason": null, "label_basis": "Answer named or stated explicitly in the text; authored and reviewed before inference", "license": "MIT", "source": "JevBench v1.1 easy tier, original authored item"}, "question": {"criteria": {"false": "The text states that this is not so", "true": "The text states that this is so"}, "instructions": "Is meeting room B booked from 14:00 to 15:00? Answer strictly from the facts stated in the text.", "type": "noul"}, "split": "public", "state": "Meeting room B: booked from 14:00 to 15:00 by the sales team."}
{"expected": "no", "family": "fact", "group": null, "id": "easy-fact-07", "labels": ["no", "yes"], "provenance": {"exclude_reason": null, "label_basis": "Answer named or stated explicitly in the text; authored and reviewed before inference", "license": "MIT", "source": "JevBench v1.1 easy tier, original authored item"}, "question": {"criteria": {"false": "The text states that this is not so", "true": "The text states that this is so"}, "instructions": "Is meeting room C booked this afternoon? Answer strictly from the facts stated in the text.", "type": "noul"}, "split": "public", "state": "Meeting room C: free all afternoon, no bookings."}
{"expected": "yes", "family": "fact", "group": null, "id": "easy-fact-08", "labels": ["no", "yes"], "provenance": {"exclude_reason": null, "label_basis": "Answer named or stated explicitly in the text; authored and reviewed before inference", "license": "MIT", "source": "JevBench v1.1 easy tier, original authored item"}, "question": {"criteria": {"false": "The text states that this is not so", "true": "The text states that this is so"}, "instructions": "Is two-factor authentication enabled? Answer strictly from the facts stated in the text.", "type": "noul"}, "split": "public", "state": "Account settings: two-factor authentication is enabled."}
{"expected": "no", "family": "fact", "group": null, "id": "easy-fact-09", "labels": ["no", "yes"], "provenance": {"exclude_reason": null, "label_basis": "Answer named or stated explicitly in the text; authored and reviewed before inference", "license": "MIT", "source": "JevBench v1.1 easy tier, original authored item"}, "question": {"criteria": {"false": "The text states that this is not so", "true": "The text states that this is so"}, "instructions": "Is two-factor authentication enabled? Answer strictly from the facts stated in the text.", "type": "noul"}, "split": "public", "state": "Account settings: two-factor authentication is disabled."}
{"expected": "yes", "family": "fact", "group": null, "id": "easy-fact-10", "labels": ["no", "yes"], "provenance": {"exclude_reason": null, "label_basis": "Answer named or stated explicitly in the text; authored and reviewed before inference", "license": "MIT", "source": "JevBench v1.1 easy tier, original authored item"}, "question": {"criteria": {"false": "The text states that this is not so", "true": "The text states that this is so"}, "instructions": "Did the user agree to subscribe to the newsletter? Answer strictly from the facts stated in the text.", "type": "noul"}, "split": "public", "state": "The user wrote: 'Yes, please subscribe me to the newsletter.'"}
{"expected": "no", "family": "fact", "group": null, "id": "easy-fact-11", "labels": ["no", "yes"], "provenance": {"exclude_reason": null, "label_basis": "Answer named or stated explicitly in the text; authored and reviewed before inference", "license": "MIT", "source": "JevBench v1.1 easy tier, original authored item"}, "question": {"criteria": {"false": "The text states that this is not so", "true": "The text states that this is so"}, "instructions": "Did the user agree to subscribe to the newsletter? Answer strictly from the facts stated in the text.", "type": "noul"}, "split": "public", "state": "The user wrote: 'No thanks, I do not want the newsletter.'"}
{"expected": "paypal", "family": "extraction", "group": null, "id": "easy-extraction-00", "labels": ["credit_card", "paypal", "bank_transfer", "cash"], "provenance": {"exclude_reason": null, "label_basis": "Answer named or stated explicitly in the text; authored and reviewed before inference", "license": "MIT", "source": "JevBench v1.1 easy tier, original authored item"}, "question": {"criteria": {"bank_transfer": "Bank transfer / wire", "cash": "Cash", "credit_card": "Paid or wants to pay by credit card", "paypal": "PayPal"}, "instructions": "Which payment method does the customer name?", "type": "choice"}, "split": "public", "state": "I paid with PayPal yesterday evening."}
{"expected": "cash", "family": "extraction", "group": null, "id": "easy-extraction-01", "labels": ["credit_card", "paypal", "bank_transfer", "cash"], "provenance": {"exclude_reason": null, "label_basis": "Answer named or stated explicitly in the text; authored and reviewed before inference", "license": "MIT", "source": "JevBench v1.1 easy tier, original authored item"}, "question": {"criteria": {"bank_transfer": "Bank transfer / wire", "cash": "Cash", "credit_card": "Paid or wants to pay by credit card", "paypal": "PayPal"}, "instructions": "Which payment method does the customer name?", "type": "choice"}, "split": "public", "state": "I'll pay cash when the courier arrives."}
{"expected": "bank_transfer", "family": "extraction", "group": null, "id": "easy-extraction-02", "labels": ["credit_card", "paypal", "bank_transfer", "cash"], "provenance": {"exclude_reason": null, "label_basis": "Answer named or stated explicitly in the text; authored and reviewed before inference", "license": "MIT", "source": "JevBench v1.1 easy tier, original authored item"}, "question": {"criteria": {"bank_transfer": "Bank transfer / wire", "cash": "Cash", "credit_card": "Paid or wants to pay by credit card", "paypal": "PayPal"}, "instructions": "Which payment method does the customer name?", "type": "choice"}, "split": "public", "state": "I sent the money by bank transfer on Monday."}
{"expected": "high", "family": "extraction", "group": null, "id": "easy-extraction-03", "labels": ["low", "medium", "high", "urgent"], "provenance": {"exclude_reason": null, "label_basis": "Answer named or stated explicitly in the text; authored and reviewed before inference", "license": "MIT", "source": "JevBench v1.1 easy tier, original authored item"}, "question": {"criteria": {"high": "High priority", "low": "Low priority", "medium": "Medium priority", "urgent": "Urgent / critical"}, "instructions": "Which priority does the ticket state?", "type": "choice"}, "split": "public", "state": "Ticket #88 - Priority: HIGH - Printer on floor 2 is jammed."}
{"expected": "low", "family": "extraction", "group": null, "id": "easy-extraction-04", "labels": ["low", "medium", "high", "urgent"], "provenance": {"exclude_reason": null, "label_basis": "Answer named or stated explicitly in the text; authored and reviewed before inference", "license": "MIT", "source": "JevBench v1.1 easy tier, original authored item"}, "question": {"criteria": {"high": "High priority", "low": "Low priority", "medium": "Medium priority", "urgent": "Urgent / critical"}, "instructions": "Which priority does the ticket state?", "type": "choice"}, "split": "public", "state": "Ticket #89 - Priority: low - Please update my desk phone label."}
{"expected": "urgent", "family": "extraction", "group": null, "id": "easy-extraction-05", "labels": ["low", "medium", "high", "urgent"], "provenance": {"exclude_reason": null, "label_basis": "Answer named or stated explicitly in the text; authored and reviewed before inference", "license": "MIT", "source": "JevBench v1.1 easy tier, original authored item"}, "question": {"criteria": {"high": "High priority", "low": "Low priority", "medium": "Medium priority", "urgent": "Urgent / critical"}, "instructions": "Which priority does the ticket state?", "type": "choice"}, "split": "public", "state": "Ticket #90 - Priority: URGENT - Production database is down."}
{"expected": "M", "family": "extraction", "group": null, "id": "easy-extraction-06", "labels": ["S", "M", "L", "XL"], "provenance": {"exclude_reason": null, "label_basis": "Answer named or stated explicitly in the text; authored and reviewed before inference", "license": "MIT", "source": "JevBench v1.1 easy tier, original authored item"}, "question": {"criteria": {"L": "Large", "M": "Medium", "S": "Small", "XL": "Extra large"}, "instructions": "Which shirt size does the customer ask for?", "type": "choice"}, "split": "public", "state": "Could I get the blue shirt in size M, please?"}
{"expected": "XL", "family": "extraction", "group": null, "id": "easy-extraction-07", "labels": ["S", "M", "L", "XL"], "provenance": {"exclude_reason": null, "label_basis": "Answer named or stated explicitly in the text; authored and reviewed before inference", "license": "MIT", "source": "JevBench v1.1 easy tier, original authored item"}, "question": {"criteria": {"L": "Large", "M": "Medium", "S": "Small", "XL": "Extra large"}, "instructions": "Which shirt size does the customer ask for?", "type": "choice"}, "split": "public", "state": "I need the jacket in extra large (XL)."}
{"expected": "thursday", "family": "extraction", "group": null, "id": "easy-extraction-08", "labels": ["monday", "tuesday", "wednesday", "thursday", "friday"], "provenance": {"exclude_reason": null, "label_basis": "Answer named or stated explicitly in the text; authored and reviewed before inference", "license": "MIT", "source": "JevBench v1.1 easy tier, original authored item"}, "question": {"criteria": {"friday": "Friday", "monday": "Monday", "thursday": "Thursday", "tuesday": "Tuesday", "wednesday": "Wednesday"}, "instructions": "Which day does the person propose for the meeting?", "type": "choice"}, "split": "public", "state": "Let's meet on Thursday at 10."}
{"expected": "monday", "family": "extraction", "group": null, "id": "easy-extraction-09", "labels": ["monday", "tuesday", "wednesday", "thursday", "friday"], "provenance": {"exclude_reason": null, "label_basis": "Answer named or stated explicitly in the text; authored and reviewed before inference", "license": "MIT", "source": "JevBench v1.1 easy tier, original authored item"}, "question": {"criteria": {"friday": "Friday", "monday": "Monday", "thursday": "Thursday", "tuesday": "Tuesday", "wednesday": "Wednesday"}, "instructions": "Which day does the person propose for the meeting?", "type": "choice"}, "split": "public", "state": "Does Monday work for you? I'm free all day."}
{"expected": "friday", "family": "extraction", "group": null, "id": "easy-extraction-10", "labels": ["monday", "tuesday", "wednesday", "thursday", "friday"], "provenance": {"exclude_reason": null, "label_basis": "Answer named or stated explicitly in the text; authored and reviewed before inference", "license": "MIT", "source": "JevBench v1.1 easy tier, original authored item"}, "question": {"criteria": {"friday": "Friday", "monday": "Monday", "thursday": "Thursday", "tuesday": "Tuesday", "wednesday": "Wednesday"}, "instructions": "Which day does the person propose for the meeting?", "type": "choice"}, "split": "public", "state": "How about Friday afternoon for the review?"}
{"expected": "credit_card", "family": "extraction", "group": null, "id": "easy-extraction-11", "labels": ["credit_card", "paypal", "bank_transfer", "cash"], "provenance": {"exclude_reason": null, "label_basis": "Answer named or stated explicitly in the text; authored and reviewed before inference", "license": "MIT", "source": "JevBench v1.1 easy tier, original authored item"}, "question": {"criteria": {"bank_transfer": "Bank transfer / wire", "cash": "Cash", "credit_card": "Paid or wants to pay by credit card", "paypal": "PayPal"}, "instructions": "Which payment method does the customer name?", "type": "choice"}, "split": "public", "state": "Please charge my credit card ending in 4242."}
{"expected": "get_weather", "family": "tool_selection", "group": null, "id": "easy-tool_selection-00", "labels": ["get_weather", "send_email", "create_calendar_event", "convert_currency", "translate_text"], "provenance": {"exclude_reason": null, "label_basis": "Answer named or stated explicitly in the text; authored and reviewed before inference", "license": "MIT", "source": "JevBench v1.1 easy tier, original authored item"}, "question": {"criteria": {"convert_currency": "Convert an amount from one currency to another", "create_calendar_event": "Put an appointment or meeting into the calendar", "get_weather": "Current weather or forecast for a place", "send_email": "Send an email to a recipient", "translate_text": "Translate text into another language"}, "instructions": "Which single tool should be called to handle this request?", "type": "choice"}, "split": "public", "state": "What's the weather like in Paris right now?"}
{"expected": "send_email", "family": "tool_selection", "group": null, "id": "easy-tool_selection-01", "labels": ["get_weather", "send_email", "create_calendar_event", "convert_currency", "translate_text"], "provenance": {"exclude_reason": null, "label_basis": "Answer named or stated explicitly in the text; authored and reviewed before inference", "license": "MIT", "source": "JevBench v1.1 easy tier, original authored item"}, "question": {"criteria": {"convert_currency": "Convert an amount from one currency to another", "create_calendar_event": "Put an appointment or meeting into the calendar", "get_weather": "Current weather or forecast for a place", "send_email": "Send an email to a recipient", "translate_text": "Translate text into another language"}, "instructions": "Which single tool should be called to handle this request?", "type": "choice"}, "split": "public", "state": "Email Sarah the quarterly report and say it's attached."}
{"expected": "create_calendar_event", "family": "tool_selection", "group": null, "id": "easy-tool_selection-02", "labels": ["get_weather", "send_email", "create_calendar_event", "convert_currency", "translate_text"], "provenance": {"exclude_reason": null, "label_basis": "Answer named or stated explicitly in the text; authored and reviewed before inference", "license": "MIT", "source": "JevBench v1.1 easy tier, original authored item"}, "question": {"criteria": {"convert_currency": "Convert an amount from one currency to another", "create_calendar_event": "Put an appointment or meeting into the calendar", "get_weather": "Current weather or forecast for a place", "send_email": "Send an email to a recipient", "translate_text": "Translate text into another language"}, "instructions": "Which single tool should be called to handle this request?", "type": "choice"}, "split": "public", "state": "Add a dentist appointment on Friday at 3 pm to my calendar."}
{"expected": "convert_currency", "family": "tool_selection", "group": null, "id": "easy-tool_selection-03", "labels": ["get_weather", "send_email", "create_calendar_event", "convert_currency", "translate_text"], "provenance": {"exclude_reason": null, "label_basis": "Answer named or stated explicitly in the text; authored and reviewed before inference", "license": "MIT", "source": "JevBench v1.1 easy tier, original authored item"}, "question": {"criteria": {"convert_currency": "Convert an amount from one currency to another", "create_calendar_event": "Put an appointment or meeting into the calendar", "get_weather": "Current weather or forecast for a place", "send_email": "Send an email to a recipient", "translate_text": "Translate text into another language"}, "instructions": "Which single tool should be called to handle this request?", "type": "choice"}, "split": "public", "state": "How much is 250 US dollars in euros?"}
{"expected": "translate_text", "family": "tool_selection", "group": null, "id": "easy-tool_selection-04", "labels": ["get_weather", "send_email", "create_calendar_event", "convert_currency", "translate_text"], "provenance": {"exclude_reason": null, "label_basis": "Answer named or stated explicitly in the text; authored and reviewed before inference", "license": "MIT", "source": "JevBench v1.1 easy tier, original authored item"}, "question": {"criteria": {"convert_currency": "Convert an amount from one currency to another", "create_calendar_event": "Put an appointment or meeting into the calendar", "get_weather": "Current weather or forecast for a place", "send_email": "Send an email to a recipient", "translate_text": "Translate text into another language"}, "instructions": "Which single tool should be called to handle this request?", "type": "choice"}, "split": "public", "state": "Translate 'good morning' into Japanese."}
{"expected": "create_calendar_event", "family": "tool_selection", "group": null, "id": "easy-tool_selection-05", "labels": ["get_weather", "send_email", "create_calendar_event", "convert_currency", "translate_text"], "provenance": {"exclude_reason": null, "label_basis": "Answer named or stated explicitly in the text; authored and reviewed before inference", "license": "MIT", "source": "JevBench v1.1 easy tier, original authored item"}, "question": {"criteria": {"convert_currency": "Convert an amount from one currency to another", "create_calendar_event": "Put an appointment or meeting into the calendar", "get_weather": "Current weather or forecast for a place", "send_email": "Send an email to a recipient", "translate_text": "Translate text into another language"}, "instructions": "Which single tool should be called to handle this request?", "type": "choice"}, "split": "public", "state": "Schedule a team meeting next Tuesday at 10 in my calendar."}
{"expected": "search_flights", "family": "tool_selection", "group": null, "id": "easy-tool_selection-06", "labels": ["search_flights", "book_restaurant", "set_reminder", "get_stock_price", "call_taxi"], "provenance": {"exclude_reason": null, "label_basis": "Answer named or stated explicitly in the text; authored and reviewed before inference", "license": "MIT", "source": "JevBench v1.1 easy tier, original authored item"}, "question": {"criteria": {"book_restaurant": "Reserve a table at a restaurant", "call_taxi": "Order a taxi to a location", "get_stock_price": "Look up the current price of a stock", "search_flights": "Find flights between two cities", "set_reminder": "Remind the user of something at a given time"}, "instructions": "Which single tool should be called to handle this request?", "type": "choice"}, "split": "public", "state": "Find me flights from Munich to Lisbon on 12 October."}
{"expected": "book_restaurant", "family": "tool_selection", "group": null, "id": "easy-tool_selection-07", "labels": ["search_flights", "book_restaurant", "set_reminder", "get_stock_price", "call_taxi"], "provenance": {"exclude_reason": null, "label_basis": "Answer named or stated explicitly in the text; authored and reviewed before inference", "license": "MIT", "source": "JevBench v1.1 easy tier, original authored item"}, "question": {"criteria": {"book_restaurant": "Reserve a table at a restaurant", "call_taxi": "Order a taxi to a location", "get_stock_price": "Look up the current price of a stock", "search_flights": "Find flights between two cities", "set_reminder": "Remind the user of something at a given time"}, "instructions": "Which single tool should be called to handle this request?", "type": "choice"}, "split": "public", "state": "Book a table for four at Luigi's tonight at 8."}
{"expected": "set_reminder", "family": "tool_selection", "group": null, "id": "easy-tool_selection-08", "labels": ["search_flights", "book_restaurant", "set_reminder", "get_stock_price", "call_taxi"], "provenance": {"exclude_reason": null, "label_basis": "Answer named or stated explicitly in the text; authored and reviewed before inference", "license": "MIT", "source": "JevBench v1.1 easy tier, original authored item"}, "question": {"criteria": {"book_restaurant": "Reserve a table at a restaurant", "call_taxi": "Order a taxi to a location", "get_stock_price": "Look up the current price of a stock", "search_flights": "Find flights between two cities", "set_reminder": "Remind the user of something at a given time"}, "instructions": "Which single tool should be called to handle this request?", "type": "choice"}, "split": "public", "state": "Remind me to call my mother at 6 pm."}
{"expected": "get_stock_price", "family": "tool_selection", "group": null, "id": "easy-tool_selection-09", "labels": ["search_flights", "book_restaurant", "set_reminder", "get_stock_price", "call_taxi"], "provenance": {"exclude_reason": null, "label_basis": "Answer named or stated explicitly in the text; authored and reviewed before inference", "license": "MIT", "source": "JevBench v1.1 easy tier, original authored item"}, "question": {"criteria": {"book_restaurant": "Reserve a table at a restaurant", "call_taxi": "Order a taxi to a location", "get_stock_price": "Look up the current price of a stock", "search_flights": "Find flights between two cities", "set_reminder": "Remind the user of something at a given time"}, "instructions": "Which single tool should be called to handle this request?", "type": "choice"}, "split": "public", "state": "What is Apple's stock price right now?"}
{"expected": "call_taxi", "family": "tool_selection", "group": null, "id": "easy-tool_selection-10", "labels": ["search_flights", "book_restaurant", "set_reminder", "get_stock_price", "call_taxi"], "provenance": {"exclude_reason": null, "label_basis": "Answer named or stated explicitly in the text; authored and reviewed before inference", "license": "MIT", "source": "JevBench v1.1 easy tier, original authored item"}, "question": {"criteria": {"book_restaurant": "Reserve a table at a restaurant", "call_taxi": "Order a taxi to a location", "get_stock_price": "Look up the current price of a stock", "search_flights": "Find flights between two cities", "set_reminder": "Remind the user of something at a given time"}, "instructions": "Which single tool should be called to handle this request?", "type": "choice"}, "split": "public", "state": "Get me a taxi to the main station."}
{"expected": "set_reminder", "family": "tool_selection", "group": null, "id": "easy-tool_selection-11", "labels": ["search_flights", "book_restaurant", "set_reminder", "get_stock_price", "call_taxi"], "provenance": {"exclude_reason": null, "label_basis": "Answer named or stated explicitly in the text; authored and reviewed before inference", "license": "MIT", "source": "JevBench v1.1 easy tier, original authored item"}, "question": {"criteria": {"book_restaurant": "Reserve a table at a restaurant", "call_taxi": "Order a taxi to a location", "get_stock_price": "Look up the current price of a stock", "search_flights": "Find flights between two cities", "set_reminder": "Remind the user of something at a given time"}, "instructions": "Which single tool should be called to handle this request?", "type": "choice"}, "split": "public", "state": "Remind me tomorrow at 9 to water the plants."}