# (C) 2026 GoodData Corporation """`gd-eval` command-line entry point.""" import argparse import os import sys import threading from datetime import UTC, datetime from pathlib import Path from typing import Any, get_args import httpx from gooddata_api_client.exceptions import ApiException, ApiTypeError from rich.console import Console from rich.table import Table from gooddata_eval.cli.agentic_runner import AGENTIC_TEST_KINDS, UNGATED_AGENTIC_TEST_KINDS, run_agentic_items from gooddata_eval.core.chat.sse_client import ( ChatClient, keep_conversations, set_default_item_timeout, set_default_turn_timeout, ) from gooddata_eval.core.config import ( DEFAULT_GATE, DEFAULT_JUDGE_MODEL, JUDGE_MODEL_ENV_VAR, EvalGate, ReasoningEffort, RunConfig, normalize_gate, ) from gooddata_eval.core.connection import ConnectionError_, resolve_connection from gooddata_eval.core.dataset.from_insights import generate as generate_from_insights from gooddata_eval.core.dataset.local import load_local_dataset from gooddata_eval.core.evaluators import supported_test_kinds from gooddata_eval.core.langfuse.sink import LangfuseSink from gooddata_eval.core.models import ChatResult, DatasetItem from gooddata_eval.core.reporting.console import render_comparison, render_console from gooddata_eval.core.reporting.html_report import load_report_files, write_html_report from gooddata_eval.core.reporting.json_report import build_multi_model_report, write_multi_model_report from gooddata_eval.core.runner import ItemReport, run_items from gooddata_eval.core.summary.http_client import SummaryClient from gooddata_eval.core.takeaway.http_client import TakeawayClient from gooddata_eval.core.timing import TIMERS_ENV_VAR from gooddata_eval.core.workspace import ModelResolutionError, WorkspaceModelController _EXIT_OK = 0 _EXIT_OPERATIONAL_ERROR = 2 class _RoutingBackend: """Dispatch each item to the right backend by test_kind. Kinds in `by_kind` go to their dedicated endpoint client (`dashboard_summary` to the summary endpoint, `document_takeaway` to the takeaway endpoint); everything else uses the conversational chat endpoint. """ def __init__(self, chat: ChatClient, by_kind: dict[str, Any]): self._chat = chat self._by_kind = by_kind def ask(self, item: DatasetItem) -> ChatResult: return self._by_kind.get(item.test_kind, self._chat).ask(item) def close(self) -> None: for backend in (self._chat, *self._by_kind.values()): if hasattr(backend, "close"): backend.close() def _build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(prog="gd-eval", description="Evaluate the GoodData AI agent.") sub = parser.add_subparsers(dest="command", required=True) run = sub.add_parser("run", help="Run an evaluation dataset.") run.add_argument("--host", help="GoodData host URL.") run.add_argument("--token", help="API token (or set GOODDATA_TOKEN).") run.add_argument("--profile", help="Profile name in ~/.gooddata/profiles.yaml.") run.add_argument("--workspace", required=True, help="Workspace id.") source = run.add_mutually_exclusive_group(required=True) source.add_argument("--dataset", help="Path to a folder of dataset JSON files.") source.add_argument("--langfuse-dataset", dest="langfuse_dataset", help="Langfuse dataset name.") run.add_argument( "--kind", dest="kind", default="visualization", metavar="TEST_KIND", help=( "Default test kind for dataset items that don't embed one. " "Use 'vis_agentic', 'agentic_visualization', 'agentic_metric_skill', etc. for multi-turn agentic eval. " "(default: visualization)" ), ) run.add_argument( "--model", action="append", dest="models", metavar="MODEL", help=( "Model id to evaluate (e.g. --model gpt-5.2). " "Prefix with provider name or id to disambiguate: " "--model ProviderName/gpt-5.2 or --model provider_id/gpt-5.2. " "Repeat to compare multiple models. " "Default: workspace's current active model." ), ) run.add_argument("--runs", type=int, default=2, help="Independent runs per item. Default 2.") run.add_argument( "--gate", choices=get_args(EvalGate), default=DEFAULT_GATE, help="Which verdict decides an item: 'any' = pass@K (a run passing is enough, the " "default and historic behaviour), 'power' = pass^K (every run must pass, so the verdict " "measures stability). Identical at --runs 1. Agentic kinds only.", ) run.add_argument( "--concurrency", type=int, default=1, help="Number of items evaluated concurrently (default 1 = sequential). " "Increase to load-test the agent under simultaneous requests. Agentic kinds that " "create workspace objects (metric_skill, alert_skill, conversation, kda_skill) always " "run one at a time regardless, to avoid cross-test contamination.", ) run.add_argument( "--judge-model", dest="judge_model", metavar="MODEL", help=f"OpenAI model for LLM-as-judge scoring (default: {DEFAULT_JUDGE_MODEL}). " "Also settable via GD_EVAL_JUDGE_MODEL. Two things to weigh before changing it: a " "model that rejects temperature=0 (the gpt-5 family does) makes verdicts " "non-deterministic, and picking the same model the agent runs means the judge " "grades its own family's output.", ) run.add_argument( "--timers", action="store_true", help="Print per-turn [timer] diagnostics (agent response, judge, simulated user). " "Off by default because a large run emits hundreds of lines; the same measurements " "are always in the JSON report's latency_breakdown_s. Equivalent to GD_EVAL_TIMERS=1.", ) run.add_argument( "--turn-timeout", dest="turn_timeout", type=float, help="Wall-clock seconds a single agent turn may take before the item is failed and the " "run moves on (or set GOODDATA_EVAL_CHAT_TURN_TIMEOUT_S). Default: uncapped.", ) run.add_argument( "--item-timeout", dest="item_timeout", type=float, help="Wall-clock seconds one item may take across ALL its turns before it is failed and " "the run moves on (or set GOODDATA_EVAL_CHAT_ITEM_TIMEOUT_S). Default: uncapped.", ) run.add_argument("--json", dest="json_path", help="Write a JSON report to this path.") run.add_argument( "--html", dest="html_path", help="Write a self-contained HTML report to this path. Same output as `gd-eval report`, " "for the single-run case where you do not want to keep the JSON around.", ) run.add_argument( "--redact", action="store_true", help="Customer-safe HTML: drop conversation/response ids and raw reasoning, and replace " "model names with 'Model A', 'Model B', ...", ) run.add_argument("--quiet", action="store_true", help="Suppress per-item progress output.") run.add_argument( "--preserve-failed", action="store_true", dest="preserve_failed", help="Keep failed conversations on the server for post-mortem inspection. Single-turn " "kinds only; use --keep-conversations to cover the agentic ones.", ) run.add_argument( "--keep-conversations", action="store_true", dest="keep_conversations", help="Keep every conversation on the server, passed or failed, so the AI Interaction " "Intelligence endpoints can be queried after the run (or set " "GOODDATA_EVAL_KEEP_CONVERSATIONS=1). Covers the agentic kinds, which --preserve-failed " "does not. Leaves state behind: only for a diagnostic run.", ) run.add_argument( "--reasoning-effort", dest="reasoning_effort", choices=list(get_args(ReasoningEffort)), help="Reasoning effort requested per message. Requires the enableGenAiReasoningEffort " "feature flag on the target organization; without it the server ignores the value.", ) run.add_argument( "--langfuse", action="store_true", help="Log scores and traces to Langfuse (requires --langfuse-dataset and LANGFUSE_* env vars).", ) run.add_argument( "--tag", dest="tags", action="append", default=[], type=_parse_tag, metavar="TAG", help="Custom tag for every Langfuse trace gd-eval writes, next to the automatic ones. " "Repeat for several tags. Applied wherever traces are written: single-turn kinds only " "with --langfuse, agentic kinds whenever LANGFUSE_* env vars are set.", ) run.add_argument( "--agent-id", dest="agent_id", help=( "AI Hub agent id every conversation should target (or set GD_EVAL_AGENT_ID). " "GoodData has no admin-settable default agent -- without this, each conversation " "falls back to whichever agent the platform's last-used/last-edited heuristic " "resolves, which may not have every skill under test enabled." ), ) report = sub.add_parser( "report", help="Render JSON report(s) as one self-contained HTML file.", description="Render JSON report(s) as one self-contained HTML file. Pass several files to " "compare runs side by side -- each becomes its own column, keyed by file name.", ) report.add_argument("json_paths", nargs="+", metavar="REPORT.json", help="JSON report file(s) from `run --json`.") report.add_argument("-o", "--out", required=True, help="Path to write the HTML file to.") report.add_argument("--title", default="gd-eval report", help="Title shown in the report header.") report.add_argument( "--redact", action="store_true", help="Customer-safe output: drop conversation/response ids and raw reasoning, and replace " "model names with 'Model A', 'Model B', ...", ) gen = sub.add_parser( "generate", help="Generate a visualization dataset by reverse-engineering a workspace's insights.", ) gen.add_argument("--host", help="GoodData host URL.") gen.add_argument("--token", help="API token (or set GOODDATA_TOKEN).") gen.add_argument("--profile", help="Profile name in ~/.gooddata/profiles.yaml.") gen.add_argument("--workspace", help="Workspace id to read insights from.") gen.add_argument( "--dataset-name", dest="dataset_name", required=True, help="`dataset_name` written into every item." ) gen.add_argument("--out", help="Output folder for the dataset JSON files (default: ./).") gen.add_argument( "--dashboard", action="append", default=[], help="Restrict to insights placed on this dashboard (repeatable). Default: the whole workspace.", ) gen.add_argument( "--snapshot-in", dest="snapshot_in", help="Replay a saved model snapshot instead of calling the API." ) gen.add_argument("--snapshot-out", dest="snapshot_out", help="Save the fetched model snapshot for later replay.") gen.add_argument("--langfuse-out", dest="langfuse_out", help="Also write a Langfuse-importable dataset JSON here.") gen.add_argument( "--id-prefix", dest="id_prefix", default="", help="Prefix every exported Langfuse item id. Langfuse ids are unique per PROJECT, so " "carrying an item into a second dataset under its original id is a 409.", ) gen.add_argument( "--no-phrase", dest="no_phrase", action="store_true", help="Skip the LLM step; emit mechanical questions." ) gen.add_argument( "--phrase-model", dest="phrase_model", default="gpt-4o", help="OpenAI model for the phrasing step." ) gen.add_argument( "--no-viz-type", dest="no_viz_type", action="store_true", help="Always blank the expected chart type." ) gen.add_argument( "--min-questions", dest="min_questions", type=int, default=15, help="Fail below this many questions." ) gen.add_argument( "--min-filtered", dest="min_filtered", type=int, default=1, help="Fail below this many questions carrying a filter.", ) gen.add_argument( "--skip-ambiguous", dest="skip_ambiguous", action="store_true", help="Drop items whose metric or dimension name matches more than one object in the model " "(loop has six labels titled 'Product Title'). Such a question cannot say which object it " "means, so a defensible answer still scores zero. Reported either way.", ) gen.add_argument("--dry-run", dest="dry_run", action="store_true", help="Report only; write nothing.") models_cmd = sub.add_parser("models", help="List LLM providers and models configured in the org.") models_cmd.add_argument("--host", help="GoodData host URL.") models_cmd.add_argument("--token", help="API token (or set GOODDATA_TOKEN).") models_cmd.add_argument("--profile", help="Profile name in ~/.gooddata/profiles.yaml.") models_cmd.add_argument( "--workspace", help="Workspace id. When provided, marks the currently active model.", ) return parser def parse_args(argv: list[str]) -> argparse.Namespace: return _build_parser().parse_args(argv) def _apply_judge_model(model: str | None) -> None: """Publish --judge-model for the deep LLMJudge call sites; an explicit flag wins.""" if model: os.environ[JUDGE_MODEL_ENV_VAR] = model def _apply_timer_flag(enabled: bool) -> None: """Publish --timers for the deep [timer] call sites. An env var rather than a parameter because the emitters sit four layers down, and it is how this package already gates cross-cutting behaviour (see TAVERN_E2E_SKIP_TRACE_LINK). Only ever sets it -- never clears a value the caller exported themselves. """ if enabled: os.environ[TIMERS_ENV_VAR] = "1" def _reject_power_gate_on_ungated_items(config: RunConfig, items: list) -> None: """Refuse a pass^K request the run cannot honour for every item. Two kinds of item are never gated: everything on the non-agentic path, because `run_items` has no gate and always decides on pass@K, and agentic_conversation, which drives its fixture once whatever --runs says and so has no K to gate over. Running a mixed dataset anyway would decide part of it under each rule and label the whole report `power`. test_kind is resolved per item, so a dataset does not have to be homogeneous. Kinds no evaluator supports are not counted: those items are skipped rather than decided, so refusing on them would make --gate power fail where --gate any runs. """ if normalize_gate(config.gate) != "power": return supported = supported_test_kinds() ungated = [ i for i in items if i.test_kind in UNGATED_AGENTIC_TEST_KINDS or (i.test_kind not in AGENTIC_TEST_KINDS and i.test_kind in supported) ] if not ungated: return kinds = sorted({i.test_kind for i in ungated}) raise ValueError( f"--gate power applies to kinds that repeat K runs, but this dataset has {len(ungated)} " f"item(s) of kind {kinds}, which are always decided on pass@K. Run them separately, or " f"use --gate any." ) def _warn_if_local_dataset_cannot_link(config: RunConfig, agentic_items: list) -> None: """Say up front that experiment assembly will fail, rather than after the run. --langfuse is refused outright with a local dataset because local item ids cannot be linked. But every evaluate_agentic_* falls back to try_make_langfuse_client() when the caller passes none, so with LANGFUSE_* exported the linking runs anyway and every dataset-item lookup 404s -- arriving in a block at the very end of the run, long after the flag that would have prevented it could be changed. The fallback is deliberate (direct library and tavern callers rely on it), so this warns instead of disabling it. """ from gooddata_eval.core.agentic._langfuse import SKIP_ENV_VAR, langfuse_credentials_present # noqa: PLC0415 from gooddata_eval.core.config import env_flag # noqa: PLC0415 if not agentic_items or config.dataset_folder is None: return if not langfuse_credentials_present() or env_flag(SKIP_ENV_VAR): return print( f"warning: --dataset is a local folder, so its item ids are not Langfuse dataset item ids. " f"Traces will be found and scored, but the per-run grouping that makes models comparable " f"cannot be created and each conversation will report that its item does not exist in Langfuse. " f"Use --langfuse-dataset for comparable runs, or set {SKIP_ENV_VAR}=1 to skip trace linking.", file=sys.stderr, ) def _truncate(text: str, limit: int = 80) -> str: return text if len(text) <= limit else text[: limit - 1] + "…" def _parse_model_arg(val: str) -> tuple[str | None, str]: """Parse a model argument that may include a provider prefix. Accepts two forms: "gpt-5.2" → (None, "gpt-5.2") "ProviderName/gpt-5.2" → ("ProviderName", "gpt-5.2") "provider_id.../model_id" → ("provider_id...", "model_id") The provider part (if present) is passed to resolve_and_activate and accepted as either a provider name or provider id. """ if "/" in val: provider_ref, _, model_id = val.partition("/") return provider_ref.strip() or None, model_id.strip() return None, val def _parse_tag(val: str) -> str: """Strip one --tag value and reject a blank one, which Langfuse would store as an empty tag.""" tag = val.strip() if not tag: raise argparse.ArgumentTypeError("tag must not be blank") return tag def _make_progress_callbacks(console: Console): """Build (on_item_start, on_run_done, on_item_done) callbacks that stream progress. A threading lock guards all console.print() calls so that concurrent ``--concurrency 2+`` workers do not deadlock when stdout is piped (e.g. running in a background process). """ _print_lock = threading.Lock() def on_item_start(index: int, total: int, item: DatasetItem) -> None: with _print_lock: console.print(f"[dim]\\[{index}/{total}][/dim] [cyan]{item.id}[/cyan] {_truncate(item.question)}") def on_run_done(index: int, total: int, run_index: int, runs: int, passed: bool, latency: float) -> None: tag = "[green]pass[/green]" if passed else "[red]fail[/red]" with _print_lock: console.print( f"[dim]\\[{index}/{total}][/dim] run {run_index}/{runs} {tag} [dim]{latency:.2f}s[/dim]" ) def on_item_done(index: int, total: int, report: ItemReport) -> None: if report.skipped: tag = "[yellow]SKIP[/yellow]" elif report.error: tag = "[red]ERR [/red]" elif report.passed: tag = "[green]PASS[/green]" else: tag = "[red]FAIL[/red]" if report.skipped: suffix = "" else: quality_str = f"{report.quality_score:.0%}" suffix = ( f" [dim]({report.latency_s:.2f}s total, {report.avg_latency_s:.2f}s avg, " f"quality={quality_str}, {report.runs} run(s))[/dim]" ) with _print_lock: console.print(f"[dim]\\[{index}/{total}][/dim] -> {tag} [cyan]{report.id}[/cyan]{suffix}") return on_item_start, on_run_done, on_item_done def _load_dataset(config: RunConfig): if config.dataset_folder is not None: return load_local_dataset(config.dataset_folder) from gooddata_eval.core.dataset.langfuse_source import load_langfuse_dataset # noqa: PLC0415 if config.langfuse_dataset is None: # pragma: no cover - argparse mutually-exclusive group guarantees one is set raise ValueError("Either --dataset or --langfuse-dataset is required.") return load_langfuse_dataset(config.langfuse_dataset, default_test_kind=config.kind) def _list_models(host: str, token: str, workspace_id: str | None) -> int: """List all LLM providers and their models; mark the active one if --workspace given.""" from gooddata_eval.core.workspace import WorkspaceModelController # noqa: PLC0415 controller = WorkspaceModelController(host, token, workspace_id or "") info = controller._provider_info() # {provider_id: {name, models: [{id, family}]}} active_provider_id: str | None = None active_model_id: str | None = None if workspace_id: active = controller.get_active() if active: active_provider_id = active.provider_id active_model_id = active.default_model_id console = Console() if not info: console.print("[yellow]No LLM providers configured in this organisation.[/yellow]") return _EXIT_OK table = Table(title=f"LLM Providers and Models{f' (workspace: {workspace_id})' if workspace_id else ''}") table.add_column("Provider") table.add_column("Provider ID") table.add_column("Model ID") table.add_column("Family") table.add_column("Active") for provider_id, pinfo in sorted(info.items(), key=lambda kv: kv[1].get("name") or kv[0]): name = pinfo.get("name") or provider_id models = pinfo.get("models") or [] if not models: is_active_provider = provider_id == active_provider_id table.add_row(name, provider_id, "[dim](none listed)[/dim]", "", "◀" if is_active_provider else "") for i, model in enumerate(models): model_id = model.get("id", "?") if isinstance(model, dict) else str(model) family = model.get("family", "") if isinstance(model, dict) else "" is_active = provider_id == active_provider_id and model_id == active_model_id active_marker = "[green]◀ active[/green]" if is_active else "" table.add_row( name if i == 0 else "", provider_id if i == 0 else "", model_id, family, active_marker, ) console.print(table) return _EXIT_OK def _run(config: RunConfig) -> int: # Applies to the agentic evaluators' own clients too, which this function never sees. set_default_turn_timeout(config.turn_timeout_s) set_default_item_timeout(config.item_timeout_s) if config.log_to_langfuse and config.langfuse_dataset is None: print( "error: --langfuse requires --langfuse-dataset (local datasets have no Langfuse item ids to link to).", file=sys.stderr, ) return _EXIT_OPERATIONAL_ERROR items = _load_dataset(config) agentic_items = [i for i in items if i.test_kind in AGENTIC_TEST_KINDS] non_agentic_items = [i for i in items if i.test_kind not in AGENTIC_TEST_KINDS] _reject_power_gate_on_ungated_items(config, items) _warn_if_local_dataset_cannot_link(config, agentic_items) models = config.models or [] run_ts = datetime.now(UTC).strftime("%Y-%m-%d-%H-%M") n_models = len(models) if models else 1 controller = WorkspaceModelController(config.host, config.token, config.workspace_id) original_active = controller.get_active() progress_console = Console(stderr=True) if not config.quiet else None if progress_console: multi_suffix = f" — {n_models} model(s)" if n_models > 1 else "" progress_console.print(f"Evaluating {len(items)} item(s) on workspace '{config.workspace_id}'{multi_suffix}") reports: list = [] try: for k, model_id in enumerate(models or [None], start=1): provider_ref, bare_model_id = _parse_model_arg(model_id or "") effective_provider = provider_ref try: resolved = controller.resolve_and_activate(bare_model_id or None, effective_provider) except (ModelResolutionError, httpx.HTTPError, ApiException, RuntimeError) as exc: print(f"warning: skipping model '{model_id}': {exc}", file=sys.stderr) continue if progress_console: if n_models > 1: progress_console.print(f"\n── Model {k}/{n_models}: {resolved.model_id} " + "─" * 40) else: switched = " [switched active provider]" if resolved.switched else "" provider_display = resolved.provider_name or resolved.provider_id progress_console.print(f"Provider={provider_display}, model={resolved.model_id}{switched}") run_name = f"gd-eval-{run_ts}-{resolved.model_id}" if config.reasoning_effort: # Without this two runs differing only by effort share a name and are # indistinguishable in the report, which is the comparison this exists for. run_name = f"{run_name}-effort-{config.reasoning_effort.lower()}" if progress_console and config.log_to_langfuse: progress_console.print(f"Logging to Langfuse run '{run_name}'...") on_item_start, on_run_done, on_item_done = (None, None, None) if progress_console: on_item_start, on_run_done, on_item_done = _make_progress_callbacks(progress_console) on_langfuse_item_done = None if config.log_to_langfuse: assert config.langfuse_dataset is not None sink = LangfuseSink( dataset_name=config.langfuse_dataset, run_name=run_name, model_id=resolved.model_id, provider_type=resolved.provider_type, reasoning_effort=config.reasoning_effort, tags=config.tags, ) def on_langfuse_item_done( index: int, total: int, report: ItemReport, _sink: LangfuseSink = sink, _model_id: str = resolved.model_id, _provider_type: str = resolved.provider_type, ) -> None: _sink.log_item(report, dataset_item_id=report.id) # --- agentic items (multi-turn, use evaluate_agentic_*) --- agentic_report = None if agentic_items: agentic_report = run_agentic_items( agentic_items, host=config.host, token=config.token, workspace_id=config.workspace_id, k=config.runs, gate=config.gate, model_version=resolved.model_id, reasoning_effort=config.reasoning_effort, use_langfuse=config.log_to_langfuse, tags=config.tags, run_ts=run_ts, on_item_start=on_item_start, on_item_done=on_item_done, agent_id=config.agent_id, concurrency=config.concurrency, ) # --- non-agentic items (single-turn, use Evaluator) --- backend = _RoutingBackend( ChatClient( host=config.host, token=config.token, workspace_id=config.workspace_id, preserve_failed=config.preserve_failed, reasoning_effort=config.reasoning_effort, agent_id=config.agent_id, turn_timeout_s=config.turn_timeout_s, item_timeout_s=config.item_timeout_s, ), { "dashboard_summary": SummaryClient( host=config.host, token=config.token, workspace_id=config.workspace_id ), "document_takeaway": TakeawayClient( host=config.host, token=config.token, workspace_id=config.workspace_id ), }, ) try: single_report = run_items( non_agentic_items, backend, runs=config.runs, model=resolved.model_id, provider_name=resolved.provider_name or resolved.provider_id, provider_type=resolved.provider_type, workspace_id=config.workspace_id, on_item_start=on_item_start, on_run_done=on_run_done, on_item_done=on_item_done, on_langfuse_item_done=on_langfuse_item_done, concurrency=config.concurrency, ) finally: if hasattr(backend, "close"): backend.close() # merge into a single report for display/export from gooddata_eval.core.runner import EvalReport # noqa: PLC0415 report = EvalReport( model=resolved.model_id, provider_name=resolved.provider_name or resolved.provider_id, provider_type=resolved.provider_type, workspace_id=config.workspace_id, gate=config.gate, ) if agentic_report is not None: report.items.extend(agentic_report.items) report.items.extend(single_report.items) report.wall_clock_s = (agentic_report.wall_clock_s if agentic_report else 0.0) + single_report.wall_clock_s # The two phases run back to back, so the merged run spans the earliest start to the latest finish. phases = [r for r in (agentic_report, single_report) if r is not None] report.started_at = min((r.started_at for r in phases if r.started_at), default=None) report.finished_at = max((r.finished_at for r in phases if r.finished_at), default=None) skipped_kinds = sorted({i.test_kind for i in report.items if i.skipped}) if skipped_kinds: print( f"warning: skipped {sum(i.skipped for i in report.items)} item(s) with " f"unsupported test_kind(s): {', '.join(skipped_kinds)}", file=sys.stderr, ) render_console(report) reports.append(report) finally: try: controller.restore(original_active) except Exception as _restore_exc: print( f"warning: failed to restore workspace active model: {_restore_exc}", file=sys.stderr, ) if not reports: print("error: no models evaluated successfully.", file=sys.stderr) return _EXIT_OPERATIONAL_ERROR if len(reports) > 1: render_comparison(reports) if config.json_path is not None: write_multi_model_report(reports, config.json_path) if config.html_path is not None: write_html_report(build_multi_model_report(reports), config.html_path, redact=config.redact) return _EXIT_OK def _report(args: argparse.Namespace) -> int: paths = [Path(p) for p in args.json_paths] write_html_report(load_report_files(paths), Path(args.out), redact=args.redact, title=args.title) print(f"Wrote {args.out}") return _EXIT_OK def _generate(args: argparse.Namespace) -> int: """`gd-eval generate` -- reverse-engineer a dataset from a workspace's insights.""" if not args.snapshot_in and not args.workspace: print("error: generate needs --workspace, or --snapshot-in to replay a saved model.", file=sys.stderr) return _EXIT_OPERATIONAL_ERROR if args.out is None: args.out = args.dataset_name def sdk_factory(): from gooddata_sdk import GoodDataSdk # noqa: PLC0415 host, token = resolve_connection(host=args.host, token=args.token, profile=args.profile) return GoodDataSdk.create(host, token) return generate_from_insights(args, sdk_factory) def main(argv: list[str] | None = None) -> int: args = parse_args(argv if argv is not None else sys.argv[1:]) _apply_timer_flag(getattr(args, "timers", False)) _apply_judge_model(getattr(args, "judge_model", None)) if hasattr(args, "concurrency") and args.concurrency < 1: print("error: --concurrency must be >= 1.", file=sys.stderr) return _EXIT_OPERATIONAL_ERROR try: # Rendering existing JSON needs no host, token or workspace -- dispatch before # resolve_connection so `report` works on a laptop with no credentials at all. if args.command == "report": return _report(args) if args.command == "generate": return _generate(args) host, token = resolve_connection(host=args.host, token=args.token, profile=args.profile) if args.command == "models": return _list_models(host, token, getattr(args, "workspace", None)) config = RunConfig( host=host, token=token, workspace_id=args.workspace, dataset_folder=Path(args.dataset) if args.dataset else None, langfuse_dataset=args.langfuse_dataset, models=args.models or [], runs=args.runs, concurrency=args.concurrency, json_path=Path(args.json_path) if args.json_path else None, html_path=Path(args.html_path) if args.html_path else None, redact=args.redact, log_to_langfuse=args.langfuse, quiet=args.quiet, kind=args.kind, preserve_failed=args.preserve_failed, keep_conversations=args.keep_conversations, reasoning_effort=args.reasoning_effort, gate=normalize_gate(args.gate), agent_id=args.agent_id or os.environ.get("GD_EVAL_AGENT_ID"), turn_timeout_s=args.turn_timeout, item_timeout_s=args.item_timeout, tags=tuple(dict.fromkeys(args.tags)), ) # Scoped to this run: main() is re-entrant under test and as a library call, and a # leaked True would leave the next run's conversations on the server unasked. with keep_conversations(config.keep_conversations): return _run(config) except ( ConnectionError_, ModelResolutionError, FileNotFoundError, ValueError, httpx.HTTPError, ApiException, # A host pointing at the UI (or any non-API endpoint) deserializes as HTML, not # a model -- an operator error, not a bug worth a traceback. ApiTypeError, RuntimeError, ) as e: print(f"error: {e}", file=sys.stderr) return _EXIT_OPERATIONAL_ERROR if __name__ == "__main__": raise SystemExit(main())