# (C) 2026 GoodData Corporation """Pydantic models for the eval dataset envelope and the agent's AAC output. Ported from gdc-nas tavern-e2e app/llm_as_judge/schemas/chat.py. """ import json import re from enum import Enum from typing import Any, Literal from pydantic import BaseModel, ConfigDict, Field, SerializerFunctionWrapHandler, field_validator, model_serializer from gooddata_eval.core.timing import PhaseTimings class LoopExit(str, Enum): """Why an agentic evaluator's simulated-user loop stopped. Every agentic kind drives the agent through a loop of simulated-user turns that can end several ways, but the result only ever recorded *whether* the agent produced its output -- so a run that ran out of turns while doing the right thing was reported identically to one that refused, and identically to one that answered wrongly. Downstream checks are all of the form ``produced_output and ``, so an exhausted run also reports every per-field check as False: specific-sounding content failures for work the agent was never given the chance to do. This does not change any verdict -- an exhausted run still fails. It makes these cases countable, so "is the turn budget too tight" becomes a question the data can answer rather than one that has to be argued. Which members label a run ERROR (the harness failed) rather than FAIL is decided in one place, ``core/outcome.py``. The label decides no verdict. Only SIMULATED_USER_FAILED is an ERROR: CHAT_ERROR, AGENT_TIMEOUT and TURN_INCOMPLETE are on the product path, and a customer hits them too. """ SUCCESS = "success" """The agent produced the expected output; the loop broke early.""" AGENT_SILENT = "agent_silent" """The agent returned neither text nor a tool call -- genuinely stuck.""" BUDGET_EXHAUSTED = "budget_exhausted" """The loop hit ``max_iterations`` without the agent producing its output. Says nothing about whether the agent was on track; it may have been one turn away.""" STOPPED_EARLY = "stopped_early" """The agent replied and the harness had nothing to send back, so the loop stopped with turns left: a question the case has no reply for, a second question after the case's reply was spent, a reply that neither asked nor acted, or a refusal case answered in one turn, where any follow-up would press the agent to build what it must not.""" SIMULATED_USER_FAILED = "simulated_user_failed" """The harness's own simulated-user model failed to produce a follow-up. A harness-side fault, not an agent one, so the run is labelled ERROR (``core/outcome.py``).""" CHAT_ERROR = "chat_error" """The chat call itself raised mid-conversation and was not one of the agent's own failures below: a gen-ai error event, a broken stream, or a 5xx still failing after its retries. It is on the product path -- a customer would see it too -- so the run is a FAIL, not a harness ERROR.""" AGENT_TIMEOUT = "agent_timeout" """The agent exceeded the per-turn or per-item wall-clock budget (``TurnTimeoutError``). The agent's own failure to answer in time, so the run is a FAIL.""" TURN_INCOMPLETE = "turn_incomplete" """The agent ended its turn without a final answer, e.g. at its own iteration limit (``TurnIncompleteError``). The agent's own failure, so the run is a FAIL.""" NOT_RUN = "not_run" """The loop never started -- e.g. an agentic_conversation turn whose $ref to an earlier turn's output could not be resolved, so the turn was skipped before any message was sent. Distinct from every other member, which all imply at least one turn happened. A FAIL: its cause is usually the earlier turn's failure, which the item already reports.""" class AacQueryField(BaseModel): model_config = ConfigDict(extra="allow") using: str title: str | None = None aggregation: str | None = None # Set by the convertor on a derived measure (`previous_period`, `same_period_previous_year`) # and by a measure-level filter. Declared rather than left to `extra` so callers can read # them by attribute; nothing scores on either. type: str | None = None filter_by: dict | None = None class AacBucketRef(BaseModel): model_config = ConfigDict(extra="allow") field: str class AacQuery(BaseModel): fields: dict[str, AacQueryField | str] filter_by: dict[str, dict] = Field(default_factory=dict) # Entries are `{"type": "metric_sort", "direction", "metrics": [alias]}` or # `{"type": "attribute_sort", "direction", "by": alias}`. Kept as raw dicts for the # same reason as `filter_by`: the agent adds keys (`aggregation`) this does not read, # and a typed model would reject a chart that is otherwise correct. sort_by: list[dict] = Field(default_factory=list) @field_validator("filter_by", mode="before") @classmethod def _coerce_filter_by(cls, v: object) -> object: return v if v is not None else {} @field_validator("sort_by", mode="before") @classmethod def _coerce_sort_by(cls, v: object) -> object: return v if v is not None else [] class CreatedVisualization(BaseModel): """Visualization in the AAC format (agent output and dataset expected output).""" model_config = ConfigDict(extra="ignore", populate_by_name=True) # Optional on purpose: nothing scores on it, and the agent sometimes omits it. A # required field here turns a scorable chart into a parse error and an errored item. id: str | None = None title: str | None = None type: str query: AacQuery # A null entry holds a position: a bubble chart with no x measure is `[null, y]`. metrics: list[AacBucketRef | str | None] = Field(default_factory=list) view_by: list[AacBucketRef | str] = Field(default_factory=list) segment_by: list[AacBucketRef | str] = Field(default_factory=list) rows: list[AacBucketRef | str] = Field(default_factory=list) columns: list[AacBucketRef | str] = Field(default_factory=list) # The metric that sizes each bubble of a bubble chart. size_by: list[AacBucketRef | str] = Field(default_factory=list) # The source and the destination of a sankey or dependency wheel: one bucket item each, not a list. from_: AacBucketRef | str | None = Field(default=None, alias="from") to: AacBucketRef | str | None = None config: dict | None = None # Expectation switches. An agent never sends them, and a recorded output leaves them out. buckets_strict: bool = Field(default=False, exclude=True) metrics_ordered: bool = Field(default=False, exclude=True) @field_validator("metrics", "view_by", "segment_by", "rows", "columns", "size_by", mode="before") @classmethod def _coerce_list_fields(cls, v: object) -> object: return v if v is not None else [] @model_serializer(mode="wrap") def _dump_from_under_its_aac_name(self, handler: SerializerFunctionWrapHandler) -> dict[str, Any]: # `from` is a keyword, so the field is `from_`. A dump without `by_alias` must still # write the AAC key: a dataset or a fixture holding `from_` loses the source bucket in # every reader that does not know the Python name. return {"from" if key == "from_" else key: value for key, value in handler(self).items()} class CreatedVisualizations(BaseModel): model_config = ConfigDict(extra="ignore") objects: list[CreatedVisualization] = Field(default_factory=list) reasoning: str = "" class ToolCallEvent(BaseModel): model_config = ConfigDict(populate_by_name=True) function_name: str = Field(alias="functionName") function_arguments: str = Field(alias="functionArguments") result: str | None = None # Client-observed receipt time (seconds since the turn's first SSE line), not a # server-side execution measurement -- set by sse_client.py as the tool-call and # tool-result events stream in. None when the call never got a result (stalled turn). call_ts: float | None = None result_ts: float | None = None # This call's 0-based position among tool calls in the turn (mirrors # ReasoningStepEvent.index) -- lets build_latency_breakdown's timeline point back at # this exact ToolCallEvent (its arguments/result) even when the same tool is called # more than once. Optional/None for callers that build a ToolCallEvent by hand (most # existing tests, and any real event from a chat backend older than this capture). index: int | None = None def parsed_arguments(self) -> dict[str, Any]: try: return json.loads(self.function_arguments) if self.function_arguments else {} except json.JSONDecodeError: return {} def parsed_result(self) -> dict[str, Any] | None: if not self.result: return None try: return json.loads(self.result) except json.JSONDecodeError: return None class ReasoningStepEvent(BaseModel): """One reasoning step with its client-observed receipt time (see ToolCallEvent.call_ts). ``index`` is this step's 0-based position among reasoning steps in the turn -- the same position it occupies in ``ChatResult.reasoning_steps`` and, downstream, in the ``.reasoning.json`` sidecar's ``reasoning`` list. It's embedded in ``build_latency_breakdown``'s label precisely so the two files can be cross-referenced by an exact index instead of fuzzy-matching on title text (which is not unique -- the same title can recur, e.g. two distinct "Considering data analysis" steps). """ summary: str ts: float index: int # Reasoning summaries are full paragraphs, e.g. "**Identifying analytics needs**\n\nI'm # analyzing..." -- using the whole thing as a latency_breakdown label would make every # entry an unreadable wall of text. The bolded title is the summary's own heading, and # downstream reporting keys off it for the same reason. _REASONING_TITLE_RE = re.compile(r"^\*\*(.+?)\*\*") _REASONING_LABEL_MAX_LEN = 60 def _reasoning_title(summary: str) -> str: m = _REASONING_TITLE_RE.match(summary.strip()) title = m.group(1) if m else summary.strip().replace("\n", " ") return title if len(title) <= _REASONING_LABEL_MAX_LEN else title[:_REASONING_LABEL_MAX_LEN] + "…" def build_latency_breakdown( tool_call_events: list[ToolCallEvent], reasoning_step_events: list[ReasoningStepEvent] | None = None, ) -> list[dict]: """The turn's tool calls and reasoning steps, in EXECUTION ORDER, each with its own wall time -- reconstructs the actual pipeline, not just a per-name total. Each entry: ``{"seq", "kind", "name", "index", "duration_s"}``. - ``seq``: 0-based position in execution order across tools AND reasoning combined. This is what answers "what ran before what" -- unlike a dict keyed by name, nothing here is aggregated together just for sharing a label, so the same tool called twice produces two separate entries in their real order, not one summed one. - ``kind``: ``"tool"`` or ``"reasoning"``. - ``name``: the tool's ``function_name``, or the reasoning step's title (see ``_reasoning_title``). - ``index``: this step's position within its OWN kind's source list -- ``ToolCallEvent.index`` for tools, ``ReasoningStepEvent.index`` for reasoning (the same position it occupies in the ``.reasoning.json`` sidecar's ``reasoning`` list). Use it to look up the full record -- the tool call's arguments/result, or the reasoning step's full paragraph in the sidecar -- since this function only ever keeps the short name/title. ``None`` for the "nothing reasoned yet" catch-all before the first real reasoning step. - ``duration_s``: wall time attributed to this step. Without ``reasoning_step_events``, only tool-call entries are produced (call receipt to result receipt) -- the gaps between them (the model "thinking") are left out entirely rather than invented as a fake step. With them, every point in the turn -- a tool call starting, a tool call's result arriving, or a reasoning step being emitted -- is merged into one timeline and sorted. The gap between consecutive points becomes one entry, attributed to whatever was "active" during it: the tool while its call is outstanding, or the most recently emitted reasoning step otherwise (its gap runs until the next point, whatever that turns out to be). This accounts for effectively the whole turn, not just its tool-call portion; only the time before the first point and after the last (connection setup/teardown) is left out, since this function has no reference to the turn's total duration. Tool calls missing either timestamp (stalled, or from a chat backend that predates this capture) are skipped entirely rather than producing a zero-duration entry. """ points: list[tuple[float, str, str, int | None]] = [] # (ts, point_kind, name, index) for tc in tool_call_events: if tc.call_ts is None or tc.result_ts is None: continue points.append((tc.call_ts, "tool_start", tc.function_name, tc.index)) points.append((tc.result_ts, "tool_end", tc.function_name, tc.index)) points.extend((rs.ts, "reasoning", _reasoning_title(rs.summary), rs.index) for rs in reasoning_step_events or []) points.sort(key=lambda p: p[0]) steps: list[dict] = [] current_reasoning_name, current_reasoning_index = "(before first step)", None seq = 0 for (ts, point_kind, name, index), (next_ts, _, _, _) in zip(points, points[1:]): gap = next_ts - ts if gap <= 0: continue if point_kind == "tool_start": step_kind, step_name, step_index = "tool", name, index elif point_kind == "reasoning": step_kind, step_name, step_index = "reasoning", name, index else: # tool_end -- control passes to whatever reasoning is (or isn't yet) active step_kind, step_name, step_index = "reasoning", current_reasoning_name, current_reasoning_index steps.append( {"seq": seq, "kind": step_kind, "name": step_name, "index": step_index, "duration_s": round(gap, 2)} ) seq += 1 if point_kind == "reasoning": current_reasoning_name, current_reasoning_index = name, index return steps # A tool result can be a whole visualization definition or a page of query rows. Kept whole # they would dominate the JSON report and the HTML built from it, so each side is clipped # and told how much was cut -- enough to see what the agent asked for and what came back, # without the report becoming a data dump. _TOOL_PAYLOAD_MAX_LEN = 2000 def _clip(text: str) -> str: if len(text) <= _TOOL_PAYLOAD_MAX_LEN: return text return text[:_TOOL_PAYLOAD_MAX_LEN] + f"… [clipped, {len(text)} chars total]" def build_tool_calls(tool_call_events: list[ToolCallEvent]) -> list[dict]: """The turn's tool calls with their arguments and results, in execution order. The counterpart to the ``.reasoning`` list: ``build_latency_breakdown`` keeps only a tool's *name*, and its entries point back here by ``index`` exactly as reasoning entries point into ``reasoning``. That is what lets a latency timeline answer "what did this call actually ask for" without every timeline entry carrying its payload. Each entry: ``{"index", "name", "arguments", "result"}``. ``arguments`` is the parsed object when it parses and is small enough, otherwise the raw (clipped) string. Calls whose position is unknown (``index is None`` -- a hand-built event, or a chat backend older than the index capture) are skipped: without an index nothing can join to them, and a positional guess would silently attribute the wrong args to a step. """ calls: list[dict] = [] for tc in tool_call_events: if tc.index is None: continue raw_args = tc.function_arguments or "" calls.append( { "index": tc.index, "name": tc.function_name, "arguments": _clip(raw_args) if len(raw_args) > _TOOL_PAYLOAD_MAX_LEN else (tc.parsed_arguments() or raw_args), "result": _clip(tc.result) if tc.result else None, } ) return calls def timeline_detail( tool_call_events: list[ToolCallEvent], reasoning_step_events: list[ReasoningStepEvent] | None = None, ) -> dict: """The `detail` keys describing how a turn actually ran: the timeline and what fills it. Every evaluator wants both and they must be built from the same events to stay index-aligned, so they are produced together rather than at a dozen call sites that could drift apart. """ return { "latency_breakdown": build_latency_breakdown(tool_call_events, reasoning_step_events), "tool_calls": build_tool_calls(tool_call_events), } class ChatResult(BaseModel): """Subset of the agent chat response needed for Phase 1 evaluation.""" model_config = ConfigDict(extra="ignore", populate_by_name=True) text_response: str | None = Field(default=None, alias="textResponse") created_visualizations: CreatedVisualizations | None = Field(default=None, alias="createdVisualizations") # Alert-proposal parts of the agent's multipart response. The alert skill's confirmation # step emits ONLY this part (no text part), so its `cta` is the only "the agent is asking # a question" signal the simulated-user loops can key off. alert_proposals: list[dict] = Field(default_factory=list, alias="alertProposals") search_results: list[dict] = Field(default_factory=list, alias="searchResults") unhandled_parts: list[dict] = Field(default_factory=list, alias="unhandledParts") # The itemId of the assistant item holding a clarifyingQuestions part: a clarifyingAnswers # message names it as questionsItemId. clarifying_questions_item_id: str | None = Field(default=None, alias="clarifyingQuestionsItemId") tool_call_events: list[ToolCallEvent] = Field(default_factory=list, alias="toolCallEvents") reasoning_step_count: int = Field(default=0, alias="reasoningStepCount") reasoning_steps: list[str] = Field(default_factory=list, alias="reasoningSteps") reasoning_step_events: list[ReasoningStepEvent] = Field(default_factory=list, alias="reasoningStepEvents") conversation_id: str | None = Field(default=None, alias="conversationId") response_id: str | None = Field(default=None, alias="responseId") # The Langfuse trace gen-ai opened the turn in, from its response_started event. trace_id: str | None = Field(default=None, alias="traceId") # True once gen-ai's response_ended event arrived. stream_ended: bool = False # Wall-clock seconds for the whole chat turn, timed by the client. turn_wall_clock_sec: float | None = None # The endpoint's whole JSON response, for single-shot endpoints whose answer carries # more than its text (the takeaway endpoint's `noData`). None for chat responses. raw_response: dict[str, Any] | None = None def shift_and_index_events( result: ChatResult, *, turn_offset: float, tool_index_offset: int, reasoning_index_offset: int, ) -> tuple[float, int, int]: """Rebase one turn's tool-call/reasoning-step timestamps and indices onto a shared, conversation-wide timeline, mutating the events in place. Each turn's own SSE stream times its events from ~0 and indexes them from 0 within that turn alone -- call_ts/result_ts/ts and index must be shifted by every prior turn's contribution before events from multiple turns can be merged into one chronologically ordered ``build_latency_breakdown`` across a whole multi-turn run. Returns the updated ``(turn_offset, tool_index_offset, reasoning_index_offset)`` to pass into the next turn's call -- the caller still owns accumulating the shifted events themselves (e.g. into an ``all_tool_call_events`` list) since callers differ in whether/how they also need the raw per-turn events for other purposes. Note: ``turn_offset`` advances by ``turn_wall_clock_sec`` only, so time spent generating a simulated-user reply between turns is not represented -- inter-turn gaps compress in the reconstructed timeline. """ for tc in result.tool_call_events or []: if tc.call_ts is not None: tc.call_ts += turn_offset if tc.result_ts is not None: tc.result_ts += turn_offset if tc.index is not None: tc.index += tool_index_offset for rs in result.reasoning_step_events or []: rs.ts += turn_offset rs.index += reasoning_index_offset tool_index_offset += len(result.tool_call_events or []) reasoning_index_offset += len(result.reasoning_step_events or []) turn_offset += result.turn_wall_clock_sec or 0.0 return turn_offset, tool_index_offset, reasoning_index_offset class AgenticAssertionError(AssertionError): """Base for every agentic kind's failure, carrying what the runner reports about it. Set by ``evaluate_agentic_*`` so a failing item still reports what the agent did and how many of its K runs passed -- ``cli/agentic_runner`` reads these off the exception exactly as it reads them off an ``AgenticEvalOutcome`` on the success path. Declared once because the payload is identical for every kind: while it lived in eight copies, two declared ``timings`` and six did not, though the runner reads it from all eight. Bare annotations, so no class attributes are created and ``getattr(exc, name, default)`` still sees only what the raising code actually set. """ __tracebackhide__ = True reasoning_steps: list[str] conversation_id: str response_id: str | None detail: dict timings: PhaseTimings runs_passed: int runs_effective: int failed_runs: list[dict] run_latencies_s: list[float] best_run_latency_s: float | None harness_error: dict """``{"status": "error", "reason": ...}``, set only when every run of the item errored (``core/outcome.py``): the item is a harness error, not an agent failure. A runner labels it; it stays a failure.""" shadow: dict """The phase-2 candidate verdict (``core.outcome.shadow_verdict``). Recorded, never used.""" class AgenticEvalOutcome(BaseModel): """Reasoning trace, trace-lookup IDs, and per-kind diagnostics from an evaluate_agentic_* call. ``detail`` mirrors the single-shot path's ``ItemEvaluation.detail`` -- a kind-specific dict of whatever diagnostic fields that evaluator already tracks internally (e.g. ``actual_maql`` for metric_skill, the full per-check breakdown for visualization). Both ``reasoning_steps``/etc. and ``detail`` are populated on success; on failure the same fields are attached directly to the raised ``*AssertionError`` instead. """ reasoning_steps: list[str] = Field(default_factory=list) conversation_id: str | None = None response_id: str | None = None detail: dict = Field(default_factory=dict) # Per-phase latency for the whole item (summed across its K runs). Kept beside # ``detail`` rather than inside it so reporting can read it without guessing at keys. timings: PhaseTimings = Field(default_factory=PhaseTimings) # How many of the item's runs passed, and how many it actually ran. pass_at_k answers # only "did any run pass", so without these a 5/5 item and a 1/5 item are identical in # every output. ``runs_effective`` exists because the requested K is not always what # ran -- agentic_conversation drives its fixture once whatever --runs says. runs_passed: int = 0 runs_effective: int = 0 # One record per run that did not pass. ``detail`` above describes only the winning run, # so without this a 1/K item and a 0/K item carry identical diagnostics and neither says # anything about the K-1 attempts that failed. Built by # ``core.agentic._failed_runs.build_failed_runs``; mirrors the single-shot path's # ``ItemReport.failed_runs``, which ``core/runner.py`` fills for the non-agentic kinds. failed_runs: list[dict] = Field(default_factory=list) # The wall-clock seconds of every run, passing or not, in run order. ``timings`` and # ``best_run_latency_s`` each describe one number per item; the latency distribution # needs one per run. Empty means the kind recorded none, which the report reads as "no # sample", not as zero-second runs. run_latencies_s: list[float] = Field(default_factory=list) # The rank-selected run's own wall time -- mirrors the single-shot path's # ItemReport.best_run_latency_s (see core/runner.py's _run_one_item), which the # agentic path never populated because its K runs happen inside the evaluator, # not in a loop the runner itself can time. None for a kind that has not yet been # wired to measure it (see AgenticAssertionError.best_run_latency_s, same default). best_run_latency_s: float | None = None # The phase-2 candidate verdict (``core.outcome.shadow_verdict``), as on # AgenticAssertionError. Recorded on a passing item too: under pass@K an item can pass with # most of its runs errored, which the candidate rule would not call a pass. Never used. shadow: dict | None = None class SummaryInput(BaseModel): """Structured input for the `dashboard_summary` test kind. Maps onto the dedicated summary endpoint's request body (`POST /api/v1/ai/workspaces/{ws}/summary`). Authored in snake_case in the dataset; the SummaryClient maps it to the endpoint's camelCase fields. """ model_config = ConfigDict(extra="ignore") dashboard_id: str visualizations: list[str] | None = None filter_context: list[dict] | None = None tab_id: str | None = None format_hint: str | None = None class TakeawayVisualization(BaseModel): """A takeaway visualization placed with its own filters.""" model_config = ConfigDict(extra="forbid") visualization_id: str filters: list[dict] = Field(default_factory=list) class TakeawayInstructions(BaseModel): model_config = ConfigDict(extra="forbid") document: str | None = None slot: str | None = None format_hint: str | None = None class TakeawayInput(BaseModel): """Structured input for the `document_takeaway` test kind. Maps onto the takeaway endpoint's request body (`POST /api/v1/ai/workspaces/{ws}/takeaway`). Authored in snake_case in the dataset; the TakeawayClient maps it to the endpoint's camelCase fields. Unknown keys are rejected at every level so a typo in a dataset item fails loudly instead of being dropped. """ model_config = ConfigDict(extra="forbid") visualizations: list[str | TakeawayVisualization] = Field(min_length=1) filter_context: list[dict] = Field(default_factory=list) instructions: TakeawayInstructions = Field(default_factory=TakeawayInstructions) max_characters: int | None = None agent_id: str | None = None class CatalogInput(BaseModel): """Structured input for the `catalog_metadata` test kind. Names one Analytics Catalog object and which text to generate for it: `title` calls `generateTitle`, `description` calls `generateDescription`. Unknown keys are rejected so a typo in a dataset item fails loudly. """ model_config = ConfigDict(extra="forbid") object_type: Literal["Visualization", "Dashboard", "Metric", "Fact", "Attribute"] object_id: str = Field(min_length=1) output: Literal["title", "description"] class DatasetItem(BaseModel): """Common dataset envelope. `expected_output` stays raw; each evaluator parses its own shape.""" model_config = ConfigDict(extra="ignore") id: str dataset_name: str test_kind: str question: str expected_output: Any # Only used by the `dashboard_summary` test kind; ignored by all others. summary_input: SummaryInput | None = None # Only used by the `document_takeaway` test kind; ignored by all others. takeaway_input: TakeawayInput | None = None # Only used by the `catalog_metadata` test kind; ignored by all others. catalog_input: CatalogInput | None = None # Relayed verbatim as the chat request's `userContext`. Deliberately opaque: gen-ai owns # that schema (a discriminated union of view/widget descriptors), so re-modelling it here # would only create a second copy to keep in sync. user_context: dict[str, Any] | None = None