# (C) 2026 GoodData Corporation import json import logging from collections.abc import Callable from typing import Any from urllib.parse import unquote import httpx import pytest from gooddata_eval.core import _retry as retry_mod from gooddata_eval.core.chat import sse_client as sse_mod from gooddata_eval.core.chat.sse_client import ( ChatClient, ChatError, TransientChatError, TurnIncompleteError, parse_sse_lines, ) from gooddata_eval.core.langfuse.experiment import experiment_id_for from gooddata_eval.core.langfuse.item_scope import ( ItemTraceScope, JoinedRun, item_scope, joined_run, joined_turns, trace_ids_for, ) from gooddata_eval.core.models import DatasetItem, ReasoningStepEvent, ToolCallEvent, build_latency_breakdown from gooddata_eval.core.timing import TraceRecorder def test_parse_sse_lines_collects_text_and_visualization(fixtures_dir): lines = (fixtures_dir / "sse_visualization_stream.txt").read_text().splitlines() result = parse_sse_lines(lines) assert result.text_response == "Here is your chart" assert result.created_visualizations is not None assert result.created_visualizations.objects[0].id == "v1" assert result.created_visualizations.objects[0].type == "column_chart" def test_parse_sse_lines_raises_on_error_event(): # 400: a code outside _RETRYABLE_STATUS_CODES, so this exercises the terminal path. lines = ['data: {"statusCode": 400, "detail": "boom"}'] with pytest.raises(RuntimeError, match="SSE error 400"): parse_sse_lines(lines) def test_parse_sse_lines_error_carries_partial_result_with_tool_calls_already_seen(): # A statusCode error ends the stream before _build_chat_result ever runs -- without # partial_result, a tool call that already succeeded (e.g. KDA's own create/execute) # before a LATER, unrelated error killed the turn would be silently discarded, making # the run look like the agent never called the tool at all. lines = [ json.dumps( { "item": { "role": "assistant", "content": {"type": "toolCall", "callId": "c1", "name": "create_key_driver_analysis"}, } } ), "", json.dumps( { "item": { "role": "tool", "content": { "type": "toolResult", "callId": "c1", "result": json.dumps({"success": True}), }, } } ), "", json.dumps({"statusCode": 400, "detail": "boom"}), ] lines = [f"data: {line}" if line else line for line in lines] with pytest.raises(ChatError) as ei: parse_sse_lines(lines) partial = ei.value.partial_result assert partial is not None assert len(partial.tool_call_events) == 1 assert partial.tool_call_events[0].function_name == "create_key_driver_analysis" assert partial.tool_call_events[0].result == '{"success": true}' def test_parse_sse_lines_stamps_call_and_result_receipt_time(monkeypatch): # t0 (accumulator construction) = 100.0, tool_call received at 105.0, tool_result # received at 130.5 -- call_ts/result_ts are offsets from t0, so 5.0 and 30.5. monkeypatch.setattr(sse_mod.time, "monotonic", iter([100.0, 105.0, 130.5]).__next__) lines = [ json.dumps( { "item": { "role": "assistant", "content": {"type": "toolCall", "callId": "c1", "name": "create_key_driver_analysis"}, } } ), "", json.dumps( { "item": { "role": "tool", "content": {"type": "toolResult", "callId": "c1", "result": json.dumps({"success": True})}, } } ), ] lines = [f"data: {line}" if line else line for line in lines] result = parse_sse_lines(lines) tc = result.tool_call_events[0] assert tc.call_ts == 5.0 assert tc.result_ts == 30.5 assert tc.index == 0 def test_build_latency_breakdown_gives_repeated_tool_calls_separate_entries_in_order(): # Same tool called twice, back-to-back (no gap between them). A dict keyed by tool # name would sum these into one number and lose which call was slower -- each call # must stay its own entry, in the order it actually ran, identifiable by its index. events = [ ToolCallEvent(function_name="search_tool", function_arguments="{}", call_ts=0.0, result_ts=2.5, index=0), ToolCallEvent(function_name="search_tool", function_arguments="{}", call_ts=2.5, result_ts=3.5, index=1), ToolCallEvent( function_name="create_metric_alert", function_arguments="{}", call_ts=3.5, result_ts=63.7, index=2 ), ] assert build_latency_breakdown(events) == [ {"seq": 0, "kind": "tool", "name": "search_tool", "index": 0, "duration_s": 2.5}, {"seq": 1, "kind": "tool", "name": "search_tool", "index": 1, "duration_s": 1.0}, {"seq": 2, "kind": "tool", "name": "create_metric_alert", "index": 2, "duration_s": 60.2}, ] def test_build_latency_breakdown_attributes_gaps_to_reasoning_steps(): # search_tool runs 0-2.5s. Then a gap: the model emits a reasoning step at 2.5s, then # goes idle until the next tool call starts at 5.0s -- that whole 2.5s idle gap belongs # to the reasoning step, not to "search_tool" (which already finished) or nothing. tool_events = [ ToolCallEvent(function_name="search_tool", function_arguments="{}", call_ts=0.0, result_ts=2.5, index=0), ToolCallEvent( function_name="create_metric_alert", function_arguments="{}", call_ts=5.0, result_ts=65.0, index=1 ), ] reasoning_events = [ReasoningStepEvent(summary="Picking the right metric", ts=2.5, index=0)] result = build_latency_breakdown(tool_events, reasoning_events) assert result == [ {"seq": 0, "kind": "tool", "name": "search_tool", "index": 0, "duration_s": 2.5}, {"seq": 1, "kind": "reasoning", "name": "Picking the right metric", "index": 0, "duration_s": 2.5}, {"seq": 2, "kind": "tool", "name": "create_metric_alert", "index": 1, "duration_s": 60.0}, ] def test_build_latency_breakdown_uses_bold_title_as_reasoning_name(): # Real reasoning summaries are a bolded title followed by a full paragraph -- "name" # must be just the title, not the whole thing. tool_events = [ ToolCallEvent(function_name="search_tool", function_arguments="{}", call_ts=0.0, result_ts=2.5, index=0), ToolCallEvent( function_name="create_metric_alert", function_arguments="{}", call_ts=5.0, result_ts=6.0, index=1 ), ] reasoning_events = [ ReasoningStepEvent(summary="**Picking the right metric**\n\nLots more detail follows here.", ts=2.5, index=0) ] result = build_latency_breakdown(tool_events, reasoning_events) reasoning_step = next(s for s in result if s["kind"] == "reasoning") assert reasoning_step["name"] == "Picking the right metric" assert "Lots more detail" not in reasoning_step["name"] def test_build_latency_breakdown_disambiguates_reasoning_steps_sharing_a_title_by_index(): # Two reasoning steps sharing the same title (a real, common occurrence) must stay # distinguishable via "index" even though "name" repeats. tool_events = [ ToolCallEvent(function_name="search_tool", function_arguments="{}", call_ts=0.0, result_ts=1.0, index=0), ToolCallEvent(function_name="search_tool", function_arguments="{}", call_ts=4.0, result_ts=5.0, index=1), ToolCallEvent( function_name="create_metric_alert", function_arguments="{}", call_ts=8.0, result_ts=9.0, index=2 ), ] reasoning_events = [ ReasoningStepEvent(summary="**Considering data analysis**", ts=1.0, index=0), ReasoningStepEvent(summary="**Considering data analysis**", ts=5.0, index=1), ] result = build_latency_breakdown(tool_events, reasoning_events) reasoning_steps = [s for s in result if s["kind"] == "reasoning"] assert [s["index"] for s in reasoning_steps] == [0, 1] assert all(s["name"] == "Considering data analysis" for s in reasoning_steps) assert [s["duration_s"] for s in reasoning_steps] == [3.0, 3.0] def test_build_latency_breakdown_truncates_untitled_reasoning_names(): tool_events = [ ToolCallEvent(function_name="search_tool", function_arguments="{}", call_ts=0.0, result_ts=2.5, index=0), ToolCallEvent( function_name="create_metric_alert", function_arguments="{}", call_ts=5.0, result_ts=6.0, index=1 ), ] long_summary = "x" * 200 reasoning_events = [ReasoningStepEvent(summary=long_summary, ts=2.5, index=0)] result = build_latency_breakdown(tool_events, reasoning_events) reasoning_step = next(s for s in result if s["kind"] == "reasoning") assert len(reasoning_step["name"]) < len(long_summary) def test_build_latency_breakdown_seq_reflects_true_execution_order(): # Interleaved on purpose: reasoning, tool, reasoning, tool -- seq must follow actual # chronological order, not group all tools first or all reasoning first. tool_events = [ ToolCallEvent(function_name="search_tool", function_arguments="{}", call_ts=1.0, result_ts=2.0, index=0), ToolCallEvent( function_name="create_metric_alert", function_arguments="{}", call_ts=4.0, result_ts=5.0, index=1 ), ] reasoning_events = [ ReasoningStepEvent(summary="**First**", ts=0.0, index=0), ReasoningStepEvent(summary="**Second**", ts=3.0, index=1), ] result = build_latency_breakdown(tool_events, reasoning_events) # "First" appears twice: once for the 0.0-1.0 gap before search_tool starts, and again # for the 2.0-3.0 gap after it resolves -- "First" is still the last-emitted reasoning # step until "Second" itself arrives at 3.0, so that gap is correctly its too. assert [(s["seq"], s["kind"], s["name"]) for s in result] == [ (0, "reasoning", "First"), (1, "tool", "search_tool"), (2, "reasoning", "First"), (3, "reasoning", "Second"), (4, "tool", "create_metric_alert"), ] def test_build_latency_breakdown_gap_with_no_reasoning_events_gets_a_catch_all_name(): # A gap between two tool calls with zero reasoning events supplied at all (e.g. an # older chat backend, or reasoning capture disabled) must not be silently dropped -- # it needs a name that doesn't fake having a real summary for it. tool_events = [ ToolCallEvent(function_name="search_tool", function_arguments="{}", call_ts=0.0, result_ts=2.5, index=0), ToolCallEvent( function_name="create_metric_alert", function_arguments="{}", call_ts=5.0, result_ts=65.0, index=1 ), ] result = build_latency_breakdown(tool_events, reasoning_step_events=None) assert result == [ {"seq": 0, "kind": "tool", "name": "search_tool", "index": 0, "duration_s": 2.5}, {"seq": 1, "kind": "reasoning", "name": "(before first step)", "index": None, "duration_s": 2.5}, {"seq": 2, "kind": "tool", "name": "create_metric_alert", "index": 1, "duration_s": 60.0}, ] def test_build_latency_breakdown_skips_calls_missing_a_timestamp(): events = [ ToolCallEvent(function_name="search_tool", function_arguments="{}", call_ts=0.0, result_ts=None), ToolCallEvent(function_name="create_metric_alert", function_arguments="{}"), ] assert build_latency_breakdown(events) == [] def test_parse_sse_lines_raw_transport_error_also_carries_partial_result(): # A connection drop mid-stream (httpx.RemoteProtocolError/ReadError) has no statusCode # payload -- it's a raw exception from iterating `lines` itself, not one this module # raises. Must still be rescued the same way a statusCode-shaped error is. def _lines(): yield ( 'data: {"item": {"role": "assistant", "content": ' + json.dumps({"type": "toolCall", "callId": "c1", "name": "create_key_driver_analysis"}) + "}}" ) yield "" raise RuntimeError("connection dropped") with pytest.raises(ChatError) as ei: parse_sse_lines(_lines()) assert not isinstance(ei.value, TransientChatError) # not retried -- same as before this fix partial = ei.value.partial_result assert partial is not None assert len(partial.tool_call_events) == 1 assert partial.tool_call_events[0].function_name == "create_key_driver_analysis" def test_parse_sse_lines_remote_protocol_error_mid_stream_is_retryable(): # httpx.RemoteProtocolError raised from `next(it)` (mid-stream, not at connect time) must # come out as TransientChatError -- otherwise _is_retryable_exc never sees the raw # RemoteProtocolError (only the ChatError parse_sse_lines wraps it in) and the retry this # class exists for never fires. Same partial_result guarantee as any other transport error. def _lines(): yield ( 'data: {"item": {"role": "assistant", "content": ' + json.dumps({"type": "toolCall", "callId": "c1", "name": "create_key_driver_analysis"}) + "}}" ) yield "" raise httpx.RemoteProtocolError("peer closed connection without sending complete message body") with pytest.raises(TransientChatError) as ei: parse_sse_lines(_lines()) partial = ei.value.partial_result assert partial is not None assert len(partial.tool_call_events) == 1 assert partial.tool_call_events[0].function_name == "create_key_driver_analysis" def test_parse_sse_lines_a_real_parsing_bug_propagates_uncaught_not_as_a_chat_error(): # A malformed payload (here: "item" is a string, not a dict) crashes the processing # code itself with a plain AttributeError -- must surface loudly as that bug, not get # silently relabeled as a ChatError/"SSE stream error" indistinguishable from a # genuine network blip. Only a failure from iterating `lines` itself is rescued. lines = ['data: {"item": "not-a-dict"}'] with pytest.raises(AttributeError): parse_sse_lines(lines) def test_parse_sse_lines_ignores_non_data_lines(): result = parse_sse_lines(["event: ping", "", ": comment"]) assert result.text_response is None assert result.created_visualizations is None def test_parse_sse_lines_stream_ended_false_when_response_ended_never_arrives(): # A turn cut off mid-stream (connection dropped, process killed) never gets to emit # gen-ai's own "response_ended" event -- text_response can still be non-empty from # whatever text arrived before the cutoff. lines = [ "event: item", 'data: {"item": {"role": "assistant", "content": {"type": "text", "text": "partial answ"}}}', "", ] result = parse_sse_lines(lines) assert result.text_response == "partial answ" assert result.stream_ended is False def test_parse_sse_lines_stream_ended_true_when_response_ended_event_arrives(): lines = [ "event: item", 'data: {"item": {"role": "assistant", "content": {"type": "text", "text": "full answer"}}}', "", "event: response_ended", "data: {}", "", ] result = parse_sse_lines(lines) assert result.text_response == "full answer" assert result.stream_ended is True def test_parse_sse_lines_stream_ended_defaults_false_with_no_events_at_all(): assert parse_sse_lines([]).stream_ended is False def test_parse_sse_lines_stream_ended_true_when_response_ended_has_no_data_line(): lines = [ "event: item", 'data: {"item": {"role": "assistant", "content": {"type": "text", "text": "x"}}}', "", "event: response_ended", "", ] assert parse_sse_lines(lines).stream_ended is True def test_parse_sse_lines_falls_back_to_adhoc_viz_when_multipart_viz_is_null(): """Visualization from create_adhoc_visualization args used when multipart viz is null.""" viz_def = { "type": "line_chart", "query": {"fields": {"m": {"using": "metric/total_sales"}}, "filter_by": {}}, "metrics": ["m"], "view_by": [], } lines = [ # agent calls create_adhoc_visualization — stash the viz f'data: {{"item": {{"role": "assistant", "content": {{"type": "toolCall", "callId": "c1", "name": "create_adhoc_visualization", "arguments": {{"visualization": {json.dumps(viz_def)}}}}}}}}}', # data source fails 'data: {"item": {"role": "tool", "content": {"type": "toolResult", "callId": "c1", "result": "{"status": "error", "message": "Data source does not exist"}"}}}', # final multipart — visualization is null 'data: {"item": {"role": "assistant", "content": {"type": "multipart", "parts": [{"type": "text", "text": "Could not create"}, {"type": "visualization", "visualization": null}]}}}', ] result = parse_sse_lines(lines) assert result.created_visualizations is not None assert result.created_visualizations.objects[0].type == "line_chart" assert result.created_visualizations.objects[0].metrics == ["m"] def test_parse_sse_lines_adhoc_fallback_synthesizes_id_when_args_have_none(caplog): """A create_adhoc_visualization definition carries no `id` -- the server mints it. The fallback marks the chart with a stand-in id so a report can tell it was never persisted, and the taking of the fallback is logged. """ viz_def = { "type": "line_chart", "query": {"fields": {"m": {"using": "metric/total_sales"}}, "filter_by": {}}, "metrics": ["m"], } lines = [ f'data: {{"item": {{"role": "assistant", "content": {{"type": "toolCall", "callId": "c1", "name": "create_adhoc_visualization", "arguments": {{"visualization": {json.dumps(viz_def)}}}}}}}}}', 'data: {"item": {"role": "assistant", "content": {"type": "multipart", "parts": [{"type": "visualization", "visualization": null}]}}}', ] with caplog.at_level("WARNING", logger="gooddata_eval.core.chat.sse_client"): result = parse_sse_lines(lines) assert result.created_visualizations is not None assert result.created_visualizations.objects[0].id == "adhoc-visualization-not-persisted" assert "no visualization part" in caplog.text assert result.created_visualizations.objects[0].type == "line_chart" def test_parse_sse_lines_counts_reasoning_steps(): lines = [ 'data: {"item": {"role": "assistant", "content": {"type": "reasoning", "summary": "step one"}}}', 'data: {"item": {"role": "assistant", "content": {"type": "reasoning", "summary": "step two"}}}', 'data: {"item": {"role": "assistant", "content": {"type": "text", "text": "Done"}}}', ] result = parse_sse_lines(lines) assert result.reasoning_step_count == 2 assert result.reasoning_steps == ["step one", "step two"] assert result.text_response == "Done" def test_parse_sse_lines_reasoning_steps_empty_when_no_reasoning_events(): lines = ['data: {"item": {"role": "assistant", "content": {"type": "text", "text": "Done"}}}'] result = parse_sse_lines(lines) assert result.reasoning_step_count == 0 assert result.reasoning_steps == [] def test_parse_sse_lines_stamps_reasoning_step_receipt_time(monkeypatch): monkeypatch.setattr(sse_mod.time, "monotonic", iter([100.0, 104.5, 110.0]).__next__) lines = [ 'data: {"item": {"role": "assistant", "content": {"type": "reasoning", "summary": "step one"}}}', 'data: {"item": {"role": "assistant", "content": {"type": "reasoning", "summary": "step two"}}}', ] result = parse_sse_lines(lines) assert [e.summary for e in result.reasoning_step_events] == ["step one", "step two"] assert [e.ts for e in result.reasoning_step_events] == [4.5, 10.0] assert [e.index for e in result.reasoning_step_events] == [0, 1] def test_parse_sse_lines_prefers_multipart_viz_over_adhoc_fallback(): """Real multipart visualization takes priority over adhoc tool call stash.""" adhoc_viz = {"type": "table", "query": {"fields": {}, "filter_by": {}}} real_viz = { "id": "real", "type": "column_chart", "query": {"fields": {"m": {"using": "metric/rev"}}, "filter_by": {}}, "metrics": ["m"], "view_by": [], } lines = [ f'data: {{"item": {{"role": "assistant", "content": {{"type": "toolCall", "callId": "c1", "name": "create_adhoc_visualization", "arguments": {{"visualization": {json.dumps(adhoc_viz)}}}}}}}}}', f'data: {{"item": {{"role": "assistant", "content": {{"type": "multipart", "parts": [{{"type": "visualization", "visualization": {json.dumps(real_viz)}}}]}}}}}}', ] result = parse_sse_lines(lines) assert result.created_visualizations.objects[0].id == "real" def test_parse_sse_lines_collects_alert_proposal_without_text_part(): """The alert skill's confirmation turn emits ONLY an alertProposal part (GDAI-2032). Pins the wire contract the simulated-user loops depend on: part ``type`` is ``alertProposal`` and the payload lives under the ``alertProposal`` key. """ proposal = { "title": "# of Orders Alert - Greater Than 500", "cta": "Should I create this alert?", "recipients": [{"email": "admin@gooddata.com"}], "alert": {"trigger": "ALWAYS", "execution": {"measures": [{"opaque": "afm"}]}}, } lines = [ 'data: {"item": {"role": "assistant", "content": {"type": "toolCall", "callId": "c1", ' '"name": "prepare_metric_alert_proposal", "arguments": {}}}}', f'data: {{"item": {{"role": "assistant", "content": {{"type": "multipart", ' f'"parts": [{{"type": "alertProposal", "alertProposal": {json.dumps(proposal)}}}]}}}}}}', ] result = parse_sse_lines(lines) assert result.text_response is None assert result.alert_proposals == [proposal] def test_parse_sse_lines_keeps_alert_proposal_part_when_payload_is_null(): """Presence of the part is the confirmation signal even if the server did not resolve it.""" lines = [ 'data: {"item": {"role": "assistant", "content": {"type": "multipart", ' '"parts": [{"type": "alertProposal", "alertProposal": null}]}}}', ] result = parse_sse_lines(lines) assert result.alert_proposals == [{}] def test_parse_sse_lines_has_no_alert_proposals_by_default(): lines = ['data: {"item": {"role": "assistant", "content": {"type": "text", "text": "Done"}}}'] assert parse_sse_lines(lines).alert_proposals == [] @pytest.mark.parametrize("code", [429, 500, 502, 503, 504]) def test_parse_sse_lines_transient_status_codes(code): with pytest.raises(TransientChatError) as ei: parse_sse_lines([f'data: {{"statusCode": {code}, "detail": null}}']) assert ei.value.status_code == code def test_parse_sse_lines_metadata_sync_is_transient(): with pytest.raises(TransientChatError): parse_sse_lines(['data: {"reasonCode": "METADATA_SYNC_IN_PROGRESS"}']) def test_parse_sse_lines_metadata_sync_marker_in_malformed_json_is_transient(): # marker present but the data payload is not valid JSON -> still transient, not swallowed with pytest.raises(TransientChatError): parse_sse_lines(["data: {bad json METADATA_SYNC_IN_PROGRESS"]) @pytest.mark.parametrize("reason", ["max_iterations", "max_tokens", "content_filter"]) def test_parse_sse_lines_turn_incomplete_is_not_transient(reason: str) -> None: """gen-ai's "no final answer" error is a 502 with a reason; retrying it would resend the question.""" lines = [ 'data: {"item": {"role": "assistant", "content": {"type": "text", "text": "Let me check."}}}', f'data: {{"statusCode": 502, "detail": "iteration limit", "reason": "{reason}"}}', ] with pytest.raises(TurnIncompleteError) as ei: parse_sse_lines(lines) assert not isinstance(ei.value, TransientChatError) assert ei.value.reason == reason assert ei.value.partial_result is not None assert ei.value.partial_result.text_response == "Let me check." @pytest.mark.parametrize("reason", [None, "unknown"]) def test_parse_sse_lines_502_without_a_definite_reason_stays_transient(reason: str | None) -> None: payload = {"statusCode": 502, "detail": "LLM provider is not responding"} if reason: payload["reason"] = reason with pytest.raises(TransientChatError): parse_sse_lines([f"data: {json.dumps(payload)}"]) def test_parse_sse_lines_non_retryable_status_is_chat_error_not_transient(): with pytest.raises(ChatError) as ei: parse_sse_lines(['data: {"statusCode": 400, "detail": "bad"}']) assert not isinstance(ei.value, TransientChatError) assert ei.value.status_code == 400 @pytest.mark.parametrize( ("code", "error_type"), [(400, ChatError), (503, TransientChatError)], ids=["non-retryable", "transient"] ) def test_parse_sse_lines_keeps_the_error_reason(code: int, error_type: type[ChatError]) -> None: payload = {"statusCode": code, "detail": "user-facing text", "reason": "DATA_OBFUSCATION_CONTENT_REJECTED"} with pytest.raises(error_type) as ei: parse_sse_lines([f"data: {json.dumps(payload)}"]) assert ei.value.reason == "DATA_OBFUSCATION_CONTENT_REJECTED" def _client_with_handler(handler, **kwargs): client = ChatClient(host="https://example.invalid", token="t", workspace_id="w", **kwargs) client._client = httpx.Client(transport=httpx.MockTransport(handler)) return client _TRANSIENT_SSE = b'data: {"statusCode": 503, "detail": null}\n' _NONRETRY_SSE = b'data: {"statusCode": 400, "detail": "bad"}\n' _OK_SSE = b'data: {"item": {"role": "assistant", "content": {"type": "text", "text": "ok"}}}\n' def test_send_message_retries_transient_then_succeeds(monkeypatch): sleeps = [] monkeypatch.setattr(sse_mod.time, "sleep", lambda s: sleeps.append(s)) calls = {"n": 0} def handler(request): calls["n"] += 1 return httpx.Response(200, content=_TRANSIENT_SSE if calls["n"] < 3 else _OK_SSE) client = _client_with_handler(handler) result = client.send_message("conv", "q") assert result.text_response == "ok" assert calls["n"] == 3 assert sleeps == [5, 10] def test_send_message_backoff_schedule_then_raises(monkeypatch): sleeps = [] monkeypatch.setattr(sse_mod.time, "sleep", lambda s: sleeps.append(s)) calls = {"n": 0} def handler(request): calls["n"] += 1 return httpx.Response(200, content=_TRANSIENT_SSE) client = _client_with_handler(handler) with pytest.raises(TransientChatError): client.send_message("conv", "q") assert calls["n"] == 6 # 1 initial + 5 retries assert sleeps == [5, 10, 20, 40, 60] def test_send_message_does_not_retry_non_transient(monkeypatch): sleeps = [] monkeypatch.setattr(sse_mod.time, "sleep", lambda s: sleeps.append(s)) calls = {"n": 0} def handler(request): calls["n"] += 1 return httpx.Response(200, content=_NONRETRY_SSE) client = _client_with_handler(handler) with pytest.raises(ChatError) as ei: client.send_message("conv", "q") assert not isinstance(ei.value, TransientChatError) assert calls["n"] == 1 assert sleeps == [] def test_send_message_sets_turn_wall_clock_sec_on_success(monkeypatch): # 3 monotonic() calls per attempt now: t0, _SseAccumulator's own t0, final wall-clock. monkeypatch.setattr(sse_mod.time, "monotonic", iter([100.0, 100.0, 102.5]).__next__) client = _client_with_handler(lambda request: httpx.Response(200, content=_OK_SSE)) result = client.send_message("conv", "q") assert result.turn_wall_clock_sec == pytest.approx(2.5) def test_send_message_wall_clock_excludes_retry_backoff(monkeypatch): # t0 must be per-attempt, set inside _do() before the connection opens -- not around # the whole send_message() call -- or a failed attempt's time plus the backoff sleep # between attempts (harness/network overhead, not gen-ai's time) would inflate the # reported latency. monkeypatch.setattr(sse_mod.time, "sleep", lambda s: None) # 3 monotonic() calls per attempt now: t0, _SseAccumulator's own t0, final wall-clock. monkeypatch.setattr(sse_mod.time, "monotonic", iter([1000.0, 1000.0, 1000.5, 2000.0, 2000.0, 2001.2]).__next__) calls = {"n": 0} def handler(request): calls["n"] += 1 return httpx.Response(200, content=_TRANSIENT_SSE if calls["n"] < 2 else _OK_SSE) client = _client_with_handler(handler) result = client.send_message("conv", "q") assert calls["n"] == 2 assert result.turn_wall_clock_sec == pytest.approx(1.2) # attempt 2 alone, not spanning attempt 1 + backoff def test_send_message_stamps_turn_wall_clock_sec_on_partial_result_too(monkeypatch): # 3 monotonic() calls now: t0, _SseAccumulator's own t0, partial-result wall-clock. monkeypatch.setattr(sse_mod.time, "monotonic", iter([50.0, 50.0, 51.0]).__next__) client = _client_with_handler(lambda request: httpx.Response(200, content=_NONRETRY_SSE)) with pytest.raises(ChatError) as ei: client.send_message("conv", "q") assert ei.value.partial_result is not None assert ei.value.partial_result.turn_wall_clock_sec == pytest.approx(1.0) def test_send_message_records_the_turn_as_a_trace_latency(monkeypatch): monkeypatch.setattr(sse_mod.time, "monotonic", iter([100.0, 100.0, 102.5]).__next__) client = _client_with_handler(lambda request: httpx.Response(200, content=_OK_SSE)) recorder = TraceRecorder() client.send_message("conv", "q") assert recorder.stop() == [pytest.approx(2.5)] def test_send_message_records_a_turn_that_ended_without_a_final_answer(monkeypatch) -> None: # The agent ran out of iterations: the slowest kind of turn, and evaluators score its partial # result as a run, so the response population has to hold it too. incomplete = b'data: {"statusCode": 502, "detail": "iteration limit", "reason": "max_iterations"}\n' monkeypatch.setattr(sse_mod.time, "monotonic", iter([50.0, 50.0, 51.0]).__next__) client = _client_with_handler(lambda request: httpx.Response(200, content=incomplete)) recorder = TraceRecorder() with pytest.raises(TurnIncompleteError): client.send_message("conv", "q") assert recorder.stop() == [pytest.approx(1.0)] def test_send_message_records_nothing_for_a_turn_that_ended_in_an_error_event(monkeypatch) -> None: # An error event is a failed request, not a response: its time says nothing about how long the # agent takes to answer, so it stays out of the percentiles. The partial result still carries it. monkeypatch.setattr(sse_mod.time, "monotonic", iter([50.0, 50.0, 51.0]).__next__) client = _client_with_handler(lambda request: httpx.Response(200, content=_NONRETRY_SSE)) recorder = TraceRecorder() with pytest.raises(ChatError): client.send_message("conv", "q") assert recorder.stop() == [] def test_send_message_records_nothing_when_a_503_outlasts_its_retries(monkeypatch) -> None: # The last attempt fails at once, so the figure on its partial result is ~0s. Recording it would # put a very fast response into the percentiles for a turn that never answered. monkeypatch.setattr(sse_mod.time, "sleep", lambda s: None) client = _client_with_handler(lambda request: httpx.Response(200, content=_TRANSIENT_SSE)) recorder = TraceRecorder() with pytest.raises(TransientChatError) as ei: client.send_message("conv", "q") assert ei.value.partial_result is not None assert recorder.stop() == [] def test_send_message_records_a_turn_cut_off_by_its_deadline(monkeypatch): """The slowest turns are the ones that hit the budget; leaving them out would make the tail look shorter exactly when the agent is slowest.""" # monotonic() calls: t0, the accumulator's own t0, the deadline check on the first line, the wall-clock. monkeypatch.setattr(sse_mod.time, "monotonic", iter([0.0, 0.0, 61.0, 61.0]).__next__) client = _client_with_handler(lambda request: httpx.Response(200, content=_OK_SSE), turn_timeout_s=60) recorder = TraceRecorder() with pytest.raises(sse_mod.TurnTimeoutError): client.send_message("conv", "q") assert recorder.stop() == [pytest.approx(61.0)] def test_send_message_records_nothing_for_a_turn_that_failed_without_a_partial_result(): # An HTTP error before any stream opens leaves no partial result: no response to time. client = _client_with_handler(lambda request: httpx.Response(400)) recorder = TraceRecorder() with pytest.raises(httpx.HTTPStatusError): client.send_message("conv", "q") assert recorder.stop() == [] def test_send_message_records_only_the_successful_attempt_of_a_retried_turn(monkeypatch): monkeypatch.setattr(sse_mod.time, "sleep", lambda s: None) monkeypatch.setattr(sse_mod.time, "monotonic", iter([1000.0, 1000.0, 1000.5, 2000.0, 2000.0, 2001.2]).__next__) calls = {"n": 0} def handler(request): calls["n"] += 1 return httpx.Response(200, content=_TRANSIENT_SSE if calls["n"] < 2 else _OK_SSE) client = _client_with_handler(handler) recorder = TraceRecorder() client.send_message("conv", "q") # One trace, timed on the attempt that answered: the failed attempt and the backoff are not it. assert recorder.stop() == [pytest.approx(1.2)] def test_create_conversation_retries_then_succeeds(monkeypatch): sleeps = [] monkeypatch.setattr(sse_mod.time, "sleep", lambda s: sleeps.append(s)) calls = {"n": 0} def handler(request): calls["n"] += 1 if calls["n"] < 3: return httpx.Response(503) return httpx.Response(200, json={"conversationId": "abc"}) client = _client_with_handler(handler) assert client.create_conversation() == "abc" assert calls["n"] == 3 assert sleeps == [5, 10] def test_create_conversation_retries_remote_protocol_error(monkeypatch): # "peer closed connection without sending complete message body" -- a pure # network flake, not a real agent/content failure. Previously not retried # at all: hard-failed on the first occurrence with zero retry attempts. sleeps = [] monkeypatch.setattr(sse_mod.time, "sleep", lambda s: sleeps.append(s)) calls = {"n": 0} def handler(request): calls["n"] += 1 if calls["n"] < 3: raise httpx.RemoteProtocolError("peer closed connection without sending complete message body") return httpx.Response(200, json={"conversationId": "abc"}) client = _client_with_handler(handler) assert client.create_conversation() == "abc" assert calls["n"] == 3 assert sleeps == [5, 10] def test_create_conversation_does_not_retry_4xx(monkeypatch): sleeps = [] monkeypatch.setattr(sse_mod.time, "sleep", lambda s: sleeps.append(s)) calls = {"n": 0} def handler(request): calls["n"] += 1 return httpx.Response(400) client = _client_with_handler(handler) with pytest.raises(httpx.HTTPStatusError): client.create_conversation() assert calls["n"] == 1 assert sleeps == [] def test_create_conversation_omits_agent_id_by_default(): # No agent_id given -> unchanged, existing behavior: GoodData's own # last-used/last-edited default-agent resolution still applies. seen = {} def handler(request): seen["body"] = request.content return httpx.Response(200, json={"conversationId": "abc"}) client = _client_with_handler(handler) client.create_conversation() assert seen["body"] in (b"", b"{}") def test_create_conversation_sends_agent_id_when_given(): seen = {} def handler(request): seen["body"] = json.loads(request.content) return httpx.Response(200, json={"conversationId": "abc"}) client = _client_with_handler(handler, agent_id="agent-123") client.create_conversation() assert seen["body"] == {"agentId": "agent-123"} def test_int_env_uses_default_when_unset(monkeypatch): monkeypatch.delenv("GD_TEST_INT", raising=False) assert sse_mod._int_env("GD_TEST_INT", 5) == 5 def test_int_env_uses_default_when_blank(monkeypatch): monkeypatch.setenv("GD_TEST_INT", "") assert sse_mod._int_env("GD_TEST_INT", 5) == 5 def test_int_env_reads_override(monkeypatch): monkeypatch.setenv("GD_TEST_INT", "2") assert sse_mod._int_env("GD_TEST_INT", 5) == 2 def test_float_env_uses_default_when_unset(monkeypatch): monkeypatch.delenv("GD_TEST_FLOAT", raising=False) assert sse_mod._float_env("GD_TEST_FLOAT", 5.0) == 5.0 def test_float_env_reads_override(monkeypatch): monkeypatch.setenv("GD_TEST_FLOAT", "1.5") assert sse_mod._float_env("GD_TEST_FLOAT", 5.0) == 1.5 def test_parse_sse_lines_captures_response_id_from_event_data(): """response_id is captured from the top-level event_data.""" lines = [ 'data: {"responseId": "resp-123", "item": {"role": "assistant", "content": {"type": "text", "text": "hi"}}}', ] result = parse_sse_lines(lines) assert result.response_id == "resp-123" def test_parse_sse_lines_captures_response_id_from_item(): """response_id is captured from item when not present at top level.""" lines = [ 'data: {"item": {"role": "assistant", "responseId": "resp-456", "content": {"type": "text", "text": "hi"}}}', ] result = parse_sse_lines(lines) assert result.response_id == "resp-456" def test_parse_sse_lines_first_response_id_wins(): """Only the first responseId encountered is kept.""" lines = [ 'data: {"responseId": "first", "item": {"role": "assistant", "content": {"type": "text", "text": "a"}}}', 'data: {"responseId": "second", "item": {"role": "assistant", "content": {"type": "text", "text": "b"}}}', ] result = parse_sse_lines(lines) assert result.response_id == "first" def test_ask_attaches_conversation_id_to_result(monkeypatch): """ChatClient.ask() sets conversation_id on the returned ChatResult.""" calls = [] def handler(request): if request.method == "POST" and request.url.path.endswith("/conversations"): return httpx.Response(200, json={"conversationId": "conv-abc"}) if request.method == "POST" and "messages" in str(request.url): return httpx.Response(200, content=_OK_SSE) if request.method == "DELETE": calls.append("delete") return httpx.Response(204) return httpx.Response(404) client = _client_with_handler(handler) item = DatasetItem(id="t1", dataset_name="d", test_kind="visualization", question="q", expected_output={}) result = client.ask(item) assert result.conversation_id == "conv-abc" assert "delete" in calls # conversation cleaned up on success def test_ask_preserve_failed_keeps_conversation_on_error(monkeypatch): """With preserve_failed=True, failed conversations are not deleted.""" calls = [] def handler(request): if request.method == "POST" and request.url.path.endswith("/conversations"): return httpx.Response(200, json={"conversationId": "conv-fail"}) if request.method == "POST" and "messages" in str(request.url): return httpx.Response(200, content=_NONRETRY_SSE) if request.method == "DELETE": calls.append("delete") return httpx.Response(204) return httpx.Response(404) monkeypatch.setattr(sse_mod.time, "sleep", lambda s: None) client = ChatClient(host="https://example.invalid", token="t", workspace_id="w", preserve_failed=True) client._client = httpx.Client(transport=httpx.MockTransport(handler)) item = DatasetItem(id="t1", dataset_name="d", test_kind="visualization", question="q", expected_output={}) with pytest.raises(ChatError): client.ask(item) assert "delete" not in calls # conversation preserved @pytest.fixture def keep_conversations(monkeypatch): """Turn the process-wide keep flag on for one test, and off again afterwards.""" monkeypatch.setattr(sse_mod, "_KEEP_CONVERSATIONS", True) def _delete_recording_handler(calls, conversation_id, sse_body): """A transport handler that appends to `calls` on DELETE, so a test can assert on it.""" def handler(request): if request.method == "POST" and request.url.path.endswith("/conversations"): return httpx.Response(200, json={"conversationId": conversation_id}) if request.method == "POST" and "messages" in str(request.url): return httpx.Response(200, content=sse_body) if request.method == "DELETE": calls.append("delete") return httpx.Response(204) return httpx.Response(404) return handler def test_keep_conversations_keeps_the_conversation_on_success(keep_conversations): """A passing run is kept too -- that is the whole difference from preserve_failed.""" calls: list[str] = [] client = _client_with_handler(_delete_recording_handler(calls, "conv-keep-ok", _OK_SSE)) item = DatasetItem(id="t1", dataset_name="d", test_kind="visualization", question="q", expected_output={}) result = client.ask(item) assert result.conversation_id == "conv-keep-ok" assert "delete" not in calls def test_keep_conversations_keeps_the_conversation_on_failure(monkeypatch, keep_conversations): """Set on its own, with preserve_failed left off, it still keeps a failed conversation.""" calls: list[str] = [] monkeypatch.setattr(sse_mod.time, "sleep", lambda s: None) client = _client_with_handler(_delete_recording_handler(calls, "conv-keep-fail", _NONRETRY_SSE)) item = DatasetItem(id="t1", dataset_name="d", test_kind="visualization", question="q", expected_output={}) with pytest.raises(ChatError): client.ask(item) assert "delete" not in calls def test_keep_conversations_blocks_a_direct_delete_conversation_call(keep_conversations): """The gate is in delete_conversation itself, which is what the agentic evaluators call. Each of them creates its own ChatClient deep in the call tree and deletes by hand in a finally block -- twenty call sites that never pass through ``ask``. If this stops holding, --keep-conversations silently covers the single-turn kinds only, which is the gap --preserve-failed already had. """ calls: list[str] = [] client = _client_with_handler(_delete_recording_handler(calls, "conv-direct", _OK_SSE)) client.delete_conversation("conv-direct") assert calls == [] def test_set_keep_conversations_affects_an_already_built_client(monkeypatch): """The agentic clients may already exist when the CLI applies the flag.""" calls: list[str] = [] monkeypatch.setattr(sse_mod, "_KEEP_CONVERSATIONS", False) client = _client_with_handler(_delete_recording_handler(calls, "conv-late", _OK_SSE)) sse_mod.set_keep_conversations(True) client.delete_conversation("conv-late") assert calls == [] sse_mod.set_keep_conversations(False) client.delete_conversation("conv-late") assert calls == ["delete"] def test_keep_conversations_restores_the_previous_setting(monkeypatch): """One run asking to keep must not leave the next run keeping too.""" monkeypatch.setattr(sse_mod, "_KEEP_CONVERSATIONS", False) with sse_mod.keep_conversations(True): assert sse_mod._KEEP_CONVERSATIONS is True assert sse_mod._KEEP_CONVERSATIONS is False with pytest.raises(RuntimeError), sse_mod.keep_conversations(True): raise RuntimeError("run blew up") assert sse_mod._KEEP_CONVERSATIONS is False def test_keep_conversations_false_does_not_clear_an_env_set_flag(monkeypatch): """``keep=False`` means "did not ask", not "delete" -- an exported env var survives a run that passes no flag.""" monkeypatch.setattr(sse_mod, "_KEEP_CONVERSATIONS", True) with sse_mod.keep_conversations(False): assert sse_mod._KEEP_CONVERSATIONS is True assert sse_mod._KEEP_CONVERSATIONS is True @pytest.mark.parametrize( ("raw", "expected"), [("1", True), ("true", True), ("yes", True), ("0", False), ("false", False), ("off", False), ("", False)], ) def test_keep_conversations_env_var(monkeypatch, raw, expected): """``GOODDATA_EVAL_KEEP_CONVERSATIONS=0`` turns it off rather than reading as truthy.""" monkeypatch.setenv("GOODDATA_EVAL_KEEP_CONVERSATIONS", raw) assert sse_mod._bool_env("GOODDATA_EVAL_KEEP_CONVERSATIONS", False) is expected def test_conversations_are_deleted_when_the_flag_is_off(monkeypatch): """The default is unchanged: a run that does not ask to keep state leaves none behind.""" calls: list[str] = [] # Set explicitly rather than assumed: with GOODDATA_EVAL_KEEP_CONVERSATIONS exported, the # module initialises to True and this test would fail when run on its own. monkeypatch.setattr(sse_mod, "_KEEP_CONVERSATIONS", False) client = _client_with_handler(_delete_recording_handler(calls, "conv-default", _OK_SSE)) client.delete_conversation("conv-default") assert calls == ["delete"] def test_ask_without_preserve_failed_deletes_on_error(monkeypatch): """Without preserve_failed, conversations are deleted even on error.""" calls = [] def handler(request): if request.method == "POST" and request.url.path.endswith("/conversations"): return httpx.Response(200, json={"conversationId": "conv-del"}) if request.method == "POST" and "messages" in str(request.url): return httpx.Response(200, content=_NONRETRY_SSE) if request.method == "DELETE": calls.append("delete") return httpx.Response(204) return httpx.Response(404) monkeypatch.setattr(sse_mod.time, "sleep", lambda s: None) client = _client_with_handler(handler) item = DatasetItem(id="t1", dataset_name="d", test_kind="visualization", question="q", expected_output={}) with pytest.raises(ChatError): client.ask(item) assert "delete" in calls # conversation deleted def test_ask_attaches_conversation_id_to_exception(monkeypatch): """On failure, conversation_id is attached to the raised exception.""" def handler(request): if request.method == "POST" and request.url.path.endswith("/conversations"): return httpx.Response(200, json={"conversationId": "conv-exc"}) if request.method == "POST" and "messages" in str(request.url): return httpx.Response(200, content=_NONRETRY_SSE) if request.method == "DELETE": return httpx.Response(204) return httpx.Response(404) monkeypatch.setattr(sse_mod.time, "sleep", lambda s: None) client = _client_with_handler(handler) item = DatasetItem(id="t1", dataset_name="d", test_kind="visualization", question="q", expected_output={}) with pytest.raises(ChatError) as ei: client.ask(item) assert ei.value.conversation_id == "conv-exc" def _capture_body_client( captured: list[dict[str, Any]], *, reasoning_effort: str | None = None, document_format: str | None = None, ) -> ChatClient: def handler(request: httpx.Request) -> httpx.Response: captured.append(json.loads(request.content)) return httpx.Response(200, content=_OK_SSE) client = ChatClient( host="https://example.invalid", token="t", workspace_id="w", reasoning_effort=reasoning_effort, document_format=document_format, ) client._client = httpx.Client(transport=httpx.MockTransport(handler)) return client def test_send_message_omits_options_when_no_reasoning_effort(): """Default must stay byte-identical to the pre-feature payload.""" captured = [] _capture_body_client(captured).send_message("conv", "q") assert captured == [{"item": {"role": "user", "content": {"type": "text", "text": "q"}}}] @pytest.mark.parametrize("effort", ["LOW", "MEDIUM", "HIGH"]) def test_send_message_sends_reasoning_effort(effort): captured = [] _capture_body_client(captured, reasoning_effort=effort).send_message("conv", "q") assert captured[0]["options"] == {"reasoningEffort": effort} assert captured[0]["item"]["content"]["text"] == "q" def test_reasoning_effort_applies_to_every_message_in_a_conversation(): """Multi-turn evaluators call send_message repeatedly on one client.""" captured = [] client = _capture_body_client(captured, reasoning_effort="LOW") client.send_message("conv", "first") client.send_message("conv", "follow-up") assert [c["options"] for c in captured] == [{"reasoningEffort": "LOW"}] * 2 def test_ask_propagates_reasoning_effort(): """ask() creates, sends and deletes — the effort must survive that whole path.""" captured = [] def handler(request): if request.method == "POST" and request.url.path.endswith("/conversations"): return httpx.Response(201, json={"conversationId": "c1"}) if request.method == "POST": captured.append(json.loads(request.content)) return httpx.Response(200, content=_OK_SSE) return httpx.Response(204) client = _client_with_handler(handler, reasoning_effort="HIGH") item = DatasetItem(id="t1", dataset_name="d", test_kind="visualization", question="q", expected_output={}) client.ask(item) assert captured[0]["options"] == {"reasoningEffort": "HIGH"} @pytest.mark.parametrize(("given", "sent"), [("low", "LOW"), (" High ", "HIGH"), ("MEDIUM", "MEDIUM")]) def test_send_message_normalizes_reasoning_effort(given, sent): """The endpoint enum is uppercase, so casing is canonicalized before the request.""" captured = [] _capture_body_client(captured, reasoning_effort=given).send_message("conv", "q") assert captured[0]["options"] == {"reasoningEffort": sent} @pytest.mark.parametrize("blank", ["", " "]) def test_blank_reasoning_effort_is_treated_as_unset(blank): """Previously a blank value was sent but skipped by the Langfuse writers, so the request and the recorded run disagreed. It now means 'unset' on both paths.""" captured = [] _capture_body_client(captured, reasoning_effort=blank).send_message("conv", "q") assert "options" not in captured[0] def test_invalid_reasoning_effort_fails_at_construction(): """Fail locally rather than as an out-of-enum request partway through a run.""" with pytest.raises(ValueError, match="Invalid reasoning effort"): ChatClient(host="https://example.invalid", token="t", workspace_id="w", reasoning_effort="maximum") @pytest.mark.parametrize("document_format", ["aac-v1", "aac-v2"]) def test_send_message_sends_document_format_on_every_message(document_format: str) -> None: """gen-ai keeps a draft in the format of the turn that wrote it, so follow-ups need it too.""" captured: list[dict[str, Any]] = [] client = _capture_body_client(captured, document_format=document_format) client.send_message("conv", "first") client.send_message("conv", "follow-up") assert [c["options"] for c in captured] == [{"documentFormat": document_format}] * 2 def test_send_message_sends_document_format_with_reasoning_effort() -> None: captured: list[dict[str, Any]] = [] _capture_body_client(captured, reasoning_effort="LOW", document_format="aac-v2").send_message("conv", "q") assert captured[0]["options"] == {"reasoningEffort": "LOW", "documentFormat": "aac-v2"} @pytest.mark.parametrize(("given", "sent"), [("AAC-V2", "aac-v2"), (" aac-v1 ", "aac-v1")]) def test_send_message_normalizes_document_format(given: str, sent: str) -> None: captured: list[dict[str, Any]] = [] _capture_body_client(captured, document_format=given).send_message("conv", "q") assert captured[0]["options"] == {"documentFormat": sent} @pytest.mark.parametrize("blank", ["", " "]) def test_blank_document_format_is_treated_as_unset(blank: str) -> None: captured: list[dict[str, Any]] = [] _capture_body_client(captured, document_format=blank).send_message("conv", "q") assert "options" not in captured[0] @pytest.mark.parametrize("bad", ["v2", "aac-v3", "aac_v2"]) def test_invalid_document_format_fails_at_construction(bad: str) -> None: """gen-ai rejects any other value; fail before the first request instead.""" with pytest.raises(ValueError, match="Invalid document format"): ChatClient(host="https://example.invalid", token="t", workspace_id="w", document_format=bad) def test_turn_timeout_aborts_a_streaming_turn_and_is_not_retried(monkeypatch): """A chatty-but-slow agent must be cut off: httpx's per-read timeout never fires for one, so only the wall-clock budget bounds the item.""" clock = iter([0.0, 0.0, 1.0, 61.0, 61.0, 61.0]) monkeypatch.setattr(sse_mod.time, "monotonic", lambda: next(clock)) def forever(): while True: yield 'data: {"role":"assistant","content":{"type":"reasoning","text":"thinking"}}' with pytest.raises(sse_mod.TurnTimeoutError, match="exceeded the 60s turn budget"): sse_mod.parse_sse_lines(sse_mod._until_deadline(forever(), deadline=60.0, budget=60.0)) assert sse_mod._is_retryable_exc(sse_mod.TurnTimeoutError("x")) is False def _silent_after_one_line(now: list[float], silent_until: float): """One event, then a read timeout that ends with the clock at ``silent_until``.""" yield 'data: {"role":"assistant","content":{"type":"reasoning","text":"thinking"}}' now[0] = silent_until raise httpx.ReadTimeout("timed out") def test_a_silent_agent_past_its_budget_is_a_turn_timeout(monkeypatch): """The read timeout is lowered to the budget, so a silent agent ends in a read timeout, not at the deadline check -- it is the same timeout all the same.""" now = [0.0] monkeypatch.setattr(sse_mod.time, "monotonic", lambda: now[0]) lines = sse_mod._until_deadline(_silent_after_one_line(now, 61.0), deadline=60.0, budget=60.0) with pytest.raises(sse_mod.TurnTimeoutError, match="exceeded the 60s turn budget") as exc_info: sse_mod.parse_sse_lines(lines) assert isinstance(exc_info.value.__cause__, httpx.ReadTimeout) def test_a_read_timeout_before_the_deadline_stays_a_stream_error(monkeypatch): now = [0.0] monkeypatch.setattr(sse_mod.time, "monotonic", lambda: now[0]) lines = sse_mod._until_deadline(_silent_after_one_line(now, 10.0), deadline=60.0, budget=60.0) with pytest.raises(sse_mod.ChatError) as exc_info: sse_mod.parse_sse_lines(lines) assert not isinstance(exc_info.value, sse_mod.TurnTimeoutError) assert isinstance(exc_info.value.__cause__, httpx.ReadTimeout) def _headers_never_came(now: list[float], silent_until: float): def handler(request): now[0] = silent_until raise httpx.ReadTimeout("timed out", request=request) return handler def test_a_read_timeout_awaiting_the_headers_past_the_budget_is_a_turn_timeout(monkeypatch): """A server that holds the headers back until the first event times out before the body.""" now = [0.0] monkeypatch.setattr(sse_mod.time, "monotonic", lambda: now[0]) client = _client_with_handler(_headers_never_came(now, 61.0), turn_timeout_s=60) with pytest.raises(sse_mod.TurnTimeoutError, match="exceeded the 60s turn budget") as exc_info: client.send_message("conv", "q") assert isinstance(exc_info.value.__cause__, httpx.ReadTimeout) def test_a_read_timeout_awaiting_the_headers_past_the_budget_records_the_turn(monkeypatch): """The turn carries no partial result, but it is one of the slowest: the tail needs it.""" now = [0.0] monkeypatch.setattr(sse_mod.time, "monotonic", lambda: now[0]) client = _client_with_handler(_headers_never_came(now, 61.0), turn_timeout_s=60) recorder = TraceRecorder() with pytest.raises(sse_mod.TurnTimeoutError): client.send_message("conv", "q") assert recorder.stop() == [pytest.approx(61.0)] def test_a_read_timeout_awaiting_the_headers_before_the_budget_is_left_as_it_is(monkeypatch): now = [0.0] monkeypatch.setattr(sse_mod.time, "monotonic", lambda: now[0]) client = _client_with_handler(_headers_never_came(now, 10.0), turn_timeout_s=60) recorder = TraceRecorder() with pytest.raises(httpx.ReadTimeout): client.send_message("conv", "q") assert recorder.stop() == [] def test_no_turn_timeout_leaves_the_stream_untouched(): lines = ['data: {"role":"assistant","content":{"type":"text","text":"hi"}}'] assert list(sse_mod._until_deadline(iter(lines), deadline=None)) == lines def test_default_turn_timeout_reaches_clients_built_later(monkeypatch): """The agentic evaluators build their own ChatClient, so the CLI flag has to be a module default rather than a constructor argument threaded through them.""" monkeypatch.setattr(sse_mod, "_TURN_TIMEOUT_S", 0.0) sse_mod.set_default_turn_timeout(60) client = sse_mod.ChatClient(host="http://h", token="t", workspace_id="w") assert client._turn_timeout_s == 60 sse_mod.set_default_turn_timeout(None) assert sse_mod.ChatClient(host="http://h", token="t", workspace_id="w")._turn_timeout_s is None def test_item_budget_shrinks_across_turns_of_one_conversation(monkeypatch): """A per-turn cap alone lets a 4-turn agentic item run to 4x the budget. The item cap is anchored at conversation creation, so later turns inherit what is left of it.""" monkeypatch.setattr(sse_mod, "_TURN_TIMEOUT_S", 0.0) monkeypatch.setattr(sse_mod, "_ITEM_TIMEOUT_S", 0.0) client = sse_mod.ChatClient(host="http://h", token="t", workspace_id="w", turn_timeout_s=60, item_timeout_s=90) client._conversation_started = 0.0 # First turn: the turn cap (60s) bites before the item cap (90s). assert client._deadline(0.0) == (60.0, 60, "turn") # Third turn, 80s already spent: the item cap is what is left, and it is what fires. assert client._deadline(80.0) == (90.0, 90, "item") def test_item_timeout_alone_still_caps_a_turn(monkeypatch): monkeypatch.setattr(sse_mod, "_TURN_TIMEOUT_S", 0.0) monkeypatch.setattr(sse_mod, "_ITEM_TIMEOUT_S", 0.0) client = sse_mod.ChatClient(host="http://h", token="t", workspace_id="w", item_timeout_s=300) assert client._deadline(0.0) == (None, 0.0, "turn") # no conversation yet client._conversation_started = 10.0 assert client._deadline(20.0) == (310.0, 300, "item") _ATTACHMENT = {"referencedObjects": [{"objects": [{"type": "WIDGET", "id": "campaign_spend"}]}]} def _capture_body(store): def handler(request): store["body"] = json.loads(request.read()) return httpx.Response(200, content=_OK_SSE) return handler def test_send_message_puts_the_user_context_on_the_wire(): captured = {} client = _client_with_handler(_capture_body(captured)) client.send_message("conv", "q", user_context=_ATTACHMENT) assert captured["body"]["userContext"] == _ATTACHMENT def test_send_message_omits_user_context_entirely_when_there_is_no_attachment(): """An explicit ``"userContext": null`` would be ACCEPTED by gen-ai, so a sloppy unconditional assignment would silently alter every request in every existing dataset rather than failing loudly. This is the guard against that.""" captured = {} client = _client_with_handler(_capture_body(captured)) client.send_message("conv", "q") assert "userContext" not in captured["body"] def test_a_client_user_context_goes_on_every_message() -> None: """An agentic run's follow-up and clarification turns are sent by the same client, and gen-ai treats a message without a context as cleared.""" bodies: list[dict] = [] def handler(request: httpx.Request) -> httpx.Response: bodies.append(json.loads(request.read())) return httpx.Response(200, content=_OK_SSE) client = _client_with_handler(handler, user_context=_ATTACHMENT) client.send_message("conv", "q1") client.send_message("conv", "q2") assert [b["userContext"] for b in bodies] == [_ATTACHMENT, _ATTACHMENT] def test_a_per_call_user_context_overrides_the_clients() -> None: other = {"view": {"dashboard": {"id": "dashboard_000"}}} captured = {} client = _client_with_handler(_capture_body(captured), user_context=_ATTACHMENT) client.send_message("conv", "q", user_context=other) assert captured["body"]["userContext"] == other def test_ask_puts_the_item_attachment_on_the_wire(): """The single-turn path goes through ask(), so an item's user_context has to be forwarded there too, or every non-agentic item with an attachment is asked bare.""" captured = {} def handler(request): if request.method == "POST" and request.url.path.endswith("/conversations"): return httpx.Response(200, json={"conversationId": "conv-abc"}) if request.method == "POST" and "messages" in str(request.url): captured["body"] = json.loads(request.read()) return httpx.Response(200, content=_OK_SSE) return httpx.Response(204) client = _client_with_handler(handler) item = DatasetItem( id="t1", dataset_name="d", test_kind="general_question", question="q", expected_output="e", user_context=_ATTACHMENT, ) client.ask(item) assert captured["body"]["userContext"] == _ATTACHMENT def _baggage_of(requests: list[httpx.Request]) -> list[dict[str, str] | None]: """Each request's `baggage` header as {key: decoded value}, or None when absent.""" out: list[dict[str, str] | None] = [] for r in requests: raw = r.headers.get("baggage") out.append(None if raw is None else {k: unquote(v) for k, v in (e.split("=", 1) for e in raw.split(","))}) return out def _record_requests(requests: list[httpx.Request]) -> Callable[[httpx.Request], httpx.Response]: def handler(request: httpx.Request) -> httpx.Response: requests.append(request) if request.url.path.endswith("/conversations"): return httpx.Response(200, json={"conversationId": "c1"}) return httpx.Response(200, content=_OK_SSE) return handler def test_trace_labels_ride_every_request_as_langfuse_baggage(monkeypatch: pytest.MonkeyPatch) -> None: """gen-ai's Langfuse span processor copies `langfuse_*` baggage onto every span it starts, so the labels reach the root generation and its cost-bearing children at creation.""" monkeypatch.setenv("GOODDATA_EVAL_TRACE_LABELS", "model_version=gpt-5.2 openai,github_run_id=42") requests: list[httpx.Request] = [] client = _client_with_handler(_record_requests(requests)) client.send_message(client.create_conversation(), "q") expected = { "langfuse_version": "gpt-5.2 openai", "langfuse_metadata_model_version": "gpt-5.2 openai", "langfuse_metadata_github_run_id": "42", } assert _baggage_of(requests) == [expected, expected] # Encoded, not raw: a space or comma in a value would otherwise split or end the entry. assert " " not in requests[0].headers["baggage"] def test_trace_labels_without_a_model_version_set_no_langfuse_version(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("GOODDATA_EVAL_TRACE_LABELS", "github_run_id=42") requests: list[httpx.Request] = [] _client_with_handler(_record_requests(requests)).create_conversation() assert _baggage_of(requests) == [{"langfuse_metadata_github_run_id": "42"}] @pytest.mark.parametrize("key", ["modèle", "run id", "c++"]) def test_trace_label_with_a_non_token_key_is_skipped_with_one_warning( monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, key: str ) -> None: """httpx refuses a non-ASCII header, gen-ai's OTel propagator drops a key with a space, and it reads `+` as a space; each would otherwise fail the run or mislabel the trace.""" monkeypatch.setenv("GOODDATA_EVAL_TRACE_LABELS", f"github_run_id=42,{key}=1") requests: list[httpx.Request] = [] for _ in range(2): _client_with_handler(_record_requests(requests)).create_conversation() assert _baggage_of(requests) == [{"langfuse_metadata_github_run_id": "42"}] * 2 assert [r.levelno for r in caplog.records if repr(key) in r.getMessage()] == [logging.WARNING] @pytest.mark.parametrize("raw", [None, "", "no-equals-sign", " , "]) def test_no_trace_labels_send_no_baggage(monkeypatch: pytest.MonkeyPatch, raw: str | None) -> None: if raw is None: monkeypatch.delenv("GOODDATA_EVAL_TRACE_LABELS", raising=False) else: monkeypatch.setenv("GOODDATA_EVAL_TRACE_LABELS", raw) requests: list[httpx.Request] = [] _client_with_handler(_record_requests(requests)).create_conversation() assert _baggage_of(requests) == [None] def _started_sse(trace_id: str | None) -> bytes: data = {"responseId": "r1", **({"traceId": trace_id} if trace_id else {})} return f"event: response_started\ndata: {json.dumps(data)}\n\n".encode() + _OK_SSE def test_parse_sse_lines_reads_the_trace_id_from_response_started(): lines = ["event: response_started", 'data: {"responseId": "r1", "traceId": "' + "a" * 32 + '"}', ""] assert parse_sse_lines(lines).trace_id == "a" * 32 def test_parse_sse_lines_ignores_a_trace_id_outside_response_started(): assert parse_sse_lines(['data: {"traceId": "abc", "responseId": "r1"}']).trace_id is None _SCOPE = ItemTraceScope(base_name="ds_ts_model", suffix_runs=True, dataset_id="ds-1", item_id="item-1") def _scoped_client(monkeypatch: pytest.MonkeyPatch, requests: list[httpx.Request], *, echo_trace: bool = True): """A client whose fake gen-ai reports, per conversation, the trace id the request asked for.""" monkeypatch.setenv("GOODDATA_EVAL_JOIN_GENAI_TRACE", "1") def handler(request: httpx.Request) -> httpx.Response: requests.append(request) asked = (_baggage_of([request])[0] or {}).get("langfuse_trace_id") return httpx.Response(200, content=_started_sse(asked if echo_trace else "f" * 32)) return _client_with_handler(handler) def test_each_conversation_is_sent_its_own_trace_and_root(monkeypatch: pytest.MonkeyPatch) -> None: requests: list[httpx.Request] = [] client = _scoped_client(monkeypatch, requests) with item_scope(_SCOPE): client.send_message("conv-a-distinct", "q") client.send_message("conv-b-distinct", "q") a, b = _baggage_of(requests) assert (a["langfuse_trace_id"], a["langfuse_experiment_item_root_observation_id"]) == trace_ids_for( "conv-a-distinct" ) assert a["langfuse_trace_id"] != b["langfuse_trace_id"] assert a["langfuse_experiment_item_root_observation_id"] != b["langfuse_experiment_item_root_observation_id"] assert (a["langfuse_experiment_dataset_id"], a["langfuse_experiment_item_id"]) == ("ds-1", "item-1") def test_run_index_follows_the_order_conversations_are_first_sent(monkeypatch: pytest.MonkeyPatch) -> None: requests: list[httpx.Request] = [] client = _scoped_client(monkeypatch, requests) with item_scope(_SCOPE): # run 0 takes two turns before run 1 starts; its second turn keeps run 0's name. for conversation_id in ("conv-run0", "conv-run0", "conv-run1"): client.send_message(conversation_id, "q") names = [b["langfuse_experiment_name"] for b in _baggage_of(requests)] assert names == ["ds_ts_model_run0", "ds_ts_model_run0", "ds_ts_model_run1"] ids = [b["langfuse_experiment_id"] for b in _baggage_of(requests)] assert ids == [experiment_id_for(name) for name in names] def test_labels_and_scope_share_one_baggage_header(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("GOODDATA_EVAL_TRACE_LABELS", "github_run_id=42") requests: list[httpx.Request] = [] client = _scoped_client(monkeypatch, requests) with item_scope(_SCOPE): client.send_message("conv-labels", "q") (baggage,) = _baggage_of(requests) assert baggage["langfuse_metadata_github_run_id"] == "42" assert "langfuse_trace_id" in baggage def test_a_conversation_is_joined_only_when_gen_ai_reports_the_trace_it_was_asked_for( monkeypatch: pytest.MonkeyPatch, ) -> None: client = _scoped_client(monkeypatch, []) with item_scope(_SCOPE): result = client.send_message("conv-joined", "q") assert result.trace_id == trace_ids_for("conv-joined")[0] assert joined_run("conv-joined") == JoinedRun("ds_ts_model_run0", "ds-1") other = _scoped_client(monkeypatch, [], echo_trace=False) with item_scope(_SCOPE): other.send_message("conv-not-joined", "q") assert joined_run("conv-not-joined") is None @pytest.mark.parametrize("switch", ["", "0", "false"]) def test_switch_off_sends_no_new_baggage_even_inside_a_scope(monkeypatch: pytest.MonkeyPatch, switch: str) -> None: monkeypatch.setenv("GOODDATA_EVAL_JOIN_GENAI_TRACE", switch) monkeypatch.delenv("GOODDATA_EVAL_TRACE_LABELS", raising=False) requests: list[httpx.Request] = [] client = _client_with_handler(_record_requests(requests)) with item_scope(_SCOPE): client.send_message("conv-off", "q") assert _baggage_of(requests) == [None] assert joined_run("conv-off") is None def test_switch_on_without_a_scope_sends_no_new_baggage(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("GOODDATA_EVAL_JOIN_GENAI_TRACE", "1") monkeypatch.delenv("GOODDATA_EVAL_TRACE_LABELS", raising=False) requests: list[httpx.Request] = [] _client_with_handler(_record_requests(requests)).send_message("conv-unscoped", "q") assert _baggage_of(requests) == [None] def test_a_turn_that_fails_after_response_started_still_counts_as_joined(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("GOODDATA_EVAL_JOIN_GENAI_TRACE", "1") asked = trace_ids_for("conv-failed-turn")[0] started = f"event: response_started\ndata: {json.dumps({'responseId': 'r1', 'traceId': asked})}\n\n".encode() client = _client_with_handler(lambda _request: httpx.Response(200, content=started + _NONRETRY_SSE)) with item_scope(_SCOPE), pytest.raises(ChatError): client.send_message("conv-failed-turn", "q") assert joined_turns("conv-failed-turn") == 1 def test_retry_names_external_callers_import_stay_on_sse_client() -> None: assert sse_mod._RETRYABLE_STATUS_CODES is retry_mod.RETRYABLE_STATUS_CODES assert sse_mod._retry_transient is retry_mod.retry_transient def test_parse_sse_lines_adhoc_fallback_keeps_its_id_over_one_in_the_arguments(): """An `id` in the tool arguments must not hide that the chart was never persisted.""" viz_def = { "id": "v1", "type": "line_chart", "query": {"fields": {"m": {"using": "metric/total_sales"}}}, "metrics": ["m"], } lines = [ f'data: {{"item": {{"role": "assistant", "content": {{"type": "toolCall", "callId": "c1", "name": "create_adhoc_visualization", "arguments": {{"visualization": {json.dumps(viz_def)}}}}}}}}}', 'data: {"item": {"role": "assistant", "content": {"type": "multipart", "parts": [{"type": "visualization", "visualization": null}]}}}', ] result = parse_sse_lines(lines) assert result.created_visualizations is not None assert result.created_visualizations.objects[0].id == "adhoc-visualization-not-persisted" def test_parse_sse_lines_keeps_the_item_id_of_the_questionnaire(): card = {"type": "clarifyingQuestions", "questions": [{"text": "Which one?", "control": {"options": []}}]} lines = [ 'data: {"item": {"itemId": "i-text", "role": "assistant", "content": ' '{"type": "multipart", "parts": [{"type": "text", "text": "Two measures."}]}}}', "data: " + json.dumps( { "item": { "itemId": "i-card", "role": "assistant", "content": {"type": "multipart", "parts": [{"type": "text", "text": "Pick one."}, card]}, } } ), ] result = parse_sse_lines(lines) assert result.clarifying_questions_item_id == "i-card" assert result.unhandled_parts == [{**card, "itemId": "i-card"}] def test_parse_sse_lines_has_no_questionnaire_item_without_one(): assert parse_sse_lines(_OK_SSE.decode().splitlines()).clarifying_questions_item_id is None def test_send_clarifying_answers_sends_the_chat_panel_shape(): captured = [] answers = [ { "questionIndex": 0, "questionText": "Which one?", "answer": {"type": "singleChoice", "optionIndex": 1, "optionText": "{metric/net_sales}"}, } ] _capture_body_client(captured, reasoning_effort="LOW").send_clarifying_answers("conv", "i-card", answers) assert captured == [ { "item": { "role": "user", "content": {"type": "clarifyingAnswers", "questionsItemId": "i-card", "answers": answers}, }, "options": {"reasoningEffort": "LOW"}, } ]