{ "openapi": "3.1.0", "info": { "title": "PhariaStudio API", "version": "0.1.0" }, "paths": { "/projects/{project_id}/traces": { "post": { "tags": [ "Traces" ], "summary": "Create Trace", "description": "Create a new `Trace`. All submitted spans must have the same trace id.", "operationId": "create_trace_projects__project_id__traces_post", "parameters": [ { "name": "project_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Project Id" } } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/SingularTraceSpanList" } } } }, "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "type": "string", "format": "uuid", "title": "Response Create Trace Projects Project Id Traces Post" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } }, "get": { "tags": [ "Traces" ], "summary": "Get Many Traces", "description": "List paginated `Traces`.", "operationId": "get_many_traces_projects__project_id__traces_get", "parameters": [ { "name": "project_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Project Id" } }, { "name": "page", "in": "query", "required": false, "schema": { "type": "integer", "exclusiveMinimum": 0, "description": "The page number to retrieve. Starts at 1.", "default": 1, "title": "Page" }, "description": "The page number to retrieve. Starts at 1." }, { "name": "size", "in": "query", "required": false, "schema": { "type": "integer", "maximum": 100, "exclusiveMinimum": 0, "description": "The number of items per page.", "default": 25, "title": "Size" }, "description": "The number of items per page." }, { "name": "only_root", "in": "query", "required": false, "schema": { "type": "boolean", "description": "Return only the root span instead of all spans", "default": false, "title": "Only Root" }, "description": "Return only the root span instead of all spans" } ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/PagedResponse_TraceResponse_" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/projects/{project_id}/traces/{trace_id}": { "get": { "tags": [ "Traces" ], "summary": "Get Trace", "description": "Get a `Trace` by its id.", "operationId": "get_trace_projects__project_id__traces__trace_id__get", "parameters": [ { "name": "project_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Project Id" } }, { "name": "trace_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Trace Id" } } ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/TraceResponse" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/projects/{project_id}/traces/{trace_id}/spans": { "post": { "tags": [ "Spans" ], "summary": "Create Span", "description": "Create a new `Span`. You can only add a `Span` to an active `Trace`. The client is responsible to create and manage the ID of each span, the server does not accept spans with the same ID.", "operationId": "create_span_projects__project_id__traces__trace_id__spans_post", "parameters": [ { "name": "project_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Project Id" } }, { "name": "trace_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Trace Id" } } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/SpanResponse-Input" } } } }, "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": {} } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } }, "get": { "tags": [ "Spans" ], "summary": "Get Many Spans", "description": "Get all `Span`.", "operationId": "get_many_spans_projects__project_id__traces__trace_id__spans_get", "parameters": [ { "name": "project_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Project Id" } }, { "name": "trace_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Trace Id" } } ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/PagedResponse_SpanResponse_" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/projects/{project_id}/traces/{trace_id}/spans/{span_id}": { "get": { "tags": [ "Spans" ], "summary": "Get Span", "description": "Get a `Span` by its id.", "operationId": "get_span_projects__project_id__traces__trace_id__spans__span_id__get", "parameters": [ { "name": "project_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Project Id" } }, { "name": "trace_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Trace Id" } }, { "name": "span_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Span Id" } } ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/SpanResponse-Output" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/projects/{project_id}/traces/{trace_id}/spans/{span_id}/events": { "post": { "tags": [ "Events" ], "summary": "Create Span Event", "description": "Create a new `Event`. You can only add an `Event` to an active `Span`. The client is responsible to create and manage the ID of each event, the server does not accept events with the same ID.", "operationId": "create_span_event_projects__project_id__traces__trace_id__spans__span_id__events_post", "parameters": [ { "name": "project_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Project Id" } }, { "name": "trace_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Trace Id" } }, { "name": "span_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Span Id" } } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/EventResponse-Input" } } } }, "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "type": "string", "format": "uuid", "title": "Response Create Span Event Projects Project Id Traces Trace Id Spans Span Id Events Post" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } }, "get": { "tags": [ "Events" ], "summary": "Get Many Events", "description": "Get all events of a span.", "operationId": "get_many_events_projects__project_id__traces__trace_id__spans__span_id__events_get", "parameters": [ { "name": "project_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Project Id" } }, { "name": "trace_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Trace Id" } }, { "name": "span_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Span Id" } }, { "name": "page", "in": "query", "required": false, "schema": { "type": "integer", "exclusiveMinimum": 0, "description": "The page number to retrieve. Starts at 1.", "default": 1, "title": "Page" }, "description": "The page number to retrieve. Starts at 1." }, { "name": "size", "in": "query", "required": false, "schema": { "type": "integer", "maximum": 100, "exclusiveMinimum": 0, "description": "The number of items per page.", "default": 25, "title": "Size" }, "description": "The number of items per page." } ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/PagedResponse_EventResponse_" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/projects/{project_id}/traces/{trace_id}/spans/{span_id}/events/{event_id}": { "get": { "tags": [ "Events" ], "summary": "Get Event", "description": "Get a `Event` by its id.", "operationId": "get_event_projects__project_id__traces__trace_id__spans__span_id__events__event_id__get", "parameters": [ { "name": "project_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Project Id" } }, { "name": "trace_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Trace Id" } }, { "name": "span_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Span Id" } }, { "name": "event_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Event Id" } } ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/EventResponse-Output" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/models": { "get": { "tags": [ "Models" ], "summary": "Get All Model Cards", "description": "Get all `ModelCard` from the list of models that are available.", "operationId": "get_all_model_cards_models_get", "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "items": { "$ref": "#/components/schemas/ModelCardResponse" }, "type": "array", "title": "Response Get All Model Cards Models Get" } } } } } } }, "/models/complete": { "post": { "tags": [ "Models" ], "summary": "Complete", "description": "Send a streamed complete request against the inference api. This always sets `stream=True` in the request body.", "operationId": "complete_models_complete_post", "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ModelCompleteRequest" } } }, "required": true }, "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": {} } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/models/chat/completions": { "post": { "tags": [ "Models" ], "summary": "Chat Completion", "operationId": "chat_completion_models_chat_completions_post", "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ChatCompletionRequest" } } }, "required": true }, "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": {} } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/projects/{project_id}/evaluation/benchmarks": { "post": { "tags": [ "Benchmarks" ], "summary": "Create Benchmark", "description": "Create a new `Benchmark`.", "operationId": "create_benchmark_projects__project_id__evaluation_benchmarks_post", "parameters": [ { "name": "project_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Project Id" } } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/PostBenchmarkRequest" } } } }, "responses": { "201": { "description": "Successful Response", "content": { "application/json": { "schema": { "type": "string", "title": "Response Create Benchmark Projects Project Id Evaluation Benchmarks Post" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } }, "get": { "tags": [ "Benchmarks" ], "summary": "Get Many Benchmarks", "description": "Get a list of `Benchmark`s .", "operationId": "get_many_benchmarks_projects__project_id__evaluation_benchmarks_get", "parameters": [ { "name": "project_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Project Id" } }, { "name": "search", "in": "query", "required": false, "schema": { "anyOf": [ { "type": "string", "minLength": 1 }, { "type": "null" } ], "description": "Search term to find results containing this text in any searchable field (case insensitive).", "title": "Search" }, "description": "Search term to find results containing this text in any searchable field (case insensitive)." }, { "name": "sort_by", "in": "query", "required": false, "schema": { "$ref": "#/components/schemas/BenchmarkSortableColumns", "description": "The column to sort by.", "default": "created_at" }, "description": "The column to sort by." }, { "name": "sort_direction", "in": "query", "required": false, "schema": { "$ref": "#/components/schemas/SortDirection", "description": "The direction to sort by.", "default": "desc" }, "description": "The direction to sort by." }, { "name": "page", "in": "query", "required": false, "schema": { "type": "integer", "exclusiveMinimum": 0, "description": "The page number to retrieve. Starts at 1.", "default": 1, "title": "Page" }, "description": "The page number to retrieve. Starts at 1." }, { "name": "size", "in": "query", "required": false, "schema": { "type": "integer", "maximum": 100, "exclusiveMinimum": 0, "description": "The number of items per page.", "default": 25, "title": "Size" }, "description": "The number of items per page." } ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/PagedResponse_GetBenchmarkResponse_" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/projects/{project_id}/evaluation/benchmarks/{benchmark_id}": { "get": { "tags": [ "Benchmarks" ], "summary": "Get Benchmark", "description": "Get a `Benchmark` by its ID.", "operationId": "get_benchmark_projects__project_id__evaluation_benchmarks__benchmark_id__get", "parameters": [ { "name": "project_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Project Id" } }, { "name": "benchmark_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Benchmark Id" } } ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "anyOf": [ { "$ref": "#/components/schemas/GetBenchmarkResponse" }, { "type": "null" } ], "title": "Response Get Benchmark Projects Project Id Evaluation Benchmarks Benchmark Id Get" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } }, "delete": { "tags": [ "Benchmarks" ], "summary": "Delete Benchmark", "description": "Delete a specific`Benchmark`.", "operationId": "delete_benchmark_projects__project_id__evaluation_benchmarks__benchmark_id__delete", "parameters": [ { "name": "project_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Project Id" } }, { "name": "benchmark_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Benchmark Id" } } ], "responses": { "204": { "description": "Successful Response" }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } }, "patch": { "tags": [ "Benchmarks" ], "summary": "Patch Benchmark", "description": "Update a specific`Benchmark`.", "operationId": "patch_benchmark_projects__project_id__evaluation_benchmarks__benchmark_id__patch", "parameters": [ { "name": "project_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Project Id" } }, { "name": "benchmark_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Benchmark Id" } } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/PatchBenchmarkRequest" } } } }, "responses": { "204": { "description": "Successful Response" }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/projects/{project_id}/evaluation/benchmarks/{benchmark_id}/executions": { "post": { "tags": [ "Benchmark Executions" ], "summary": "Create Benchmark Execution", "description": "Create a new `BenchmarkExecution`.", "operationId": "create_benchmark_execution_projects__project_id__evaluation_benchmarks__benchmark_id__executions_post", "parameters": [ { "name": "project_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Project Id" } }, { "name": "benchmark_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Benchmark Id" } } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/PostBenchmarkExecutionRequest" } } } }, "responses": { "201": { "description": "Successful Response", "content": { "application/json": { "schema": { "type": "string", "format": "uuid", "title": "Response Create Benchmark Execution Projects Project Id Evaluation Benchmarks Benchmark Id Executions Post" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } }, "get": { "tags": [ "Benchmark Executions" ], "summary": "Get Many Benchmark Executions", "description": "Get multiple `BenchmarkExecution`s. Results are sorted by descending creation date.", "operationId": "get_many_benchmark_executions_projects__project_id__evaluation_benchmarks__benchmark_id__executions_get", "parameters": [ { "name": "project_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Project Id" } }, { "name": "benchmark_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Benchmark Id" } }, { "name": "search", "in": "query", "required": false, "schema": { "anyOf": [ { "type": "string", "minLength": 1 }, { "type": "null" } ], "description": "Search term to find results containing this text in any searchable field (case insensitive).", "title": "Search" }, "description": "Search term to find results containing this text in any searchable field (case insensitive)." }, { "name": "sort_by", "in": "query", "required": false, "schema": { "anyOf": [ { "$ref": "#/components/schemas/BenchmarkExecutionSortableColumns" }, { "type": "string" } ], "description": "The column to sort by. Can be a predefined column or 'statistics.KEY' for JSONB sorting.", "default": "created_at", "title": "Sort By" }, "description": "The column to sort by. Can be a predefined column or 'statistics.KEY' for JSONB sorting." }, { "name": "sort_direction", "in": "query", "required": false, "schema": { "$ref": "#/components/schemas/SortDirection", "description": "The direction to sort by.", "default": "desc" }, "description": "The direction to sort by." }, { "name": "page", "in": "query", "required": false, "schema": { "type": "integer", "exclusiveMinimum": 0, "description": "The page number to retrieve. Starts at 1.", "default": 1, "title": "Page" }, "description": "The page number to retrieve. Starts at 1." }, { "name": "size", "in": "query", "required": false, "schema": { "type": "integer", "maximum": 100, "exclusiveMinimum": 0, "description": "The number of items per page.", "default": 25, "title": "Size" }, "description": "The number of items per page." } ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/PagedResponse_BenchmarkExecutionResponse_" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/projects/{project_id}/evaluation/benchmarks/{benchmark_id}/executions/{execution_id}": { "get": { "tags": [ "Benchmark Executions" ], "summary": "Get Benchmark Execution", "description": "Get a `BenchmarkExecution` by ID. Gives an error if the project or benchmark do not exist and `None` if they do but `BenchmarkExecution` was not found.", "operationId": "get_benchmark_execution_projects__project_id__evaluation_benchmarks__benchmark_id__executions__execution_id__get", "parameters": [ { "name": "project_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Project Id" } }, { "name": "benchmark_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Benchmark Id" } }, { "name": "execution_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Execution Id" } } ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "anyOf": [ { "$ref": "#/components/schemas/BenchmarkExecutionResponse" }, { "type": "null" } ], "title": "Response Get Benchmark Execution Projects Project Id Evaluation Benchmarks Benchmark Id Executions Execution Id Get" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } }, "delete": { "tags": [ "Benchmark Executions" ], "summary": "Delete Benchmark Execution", "description": "Delete a `BenchmarkExecution`.", "operationId": "delete_benchmark_execution_projects__project_id__evaluation_benchmarks__benchmark_id__executions__execution_id__delete", "parameters": [ { "name": "project_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Project Id" } }, { "name": "benchmark_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Benchmark Id" } }, { "name": "execution_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Execution Id" } } ], "responses": { "204": { "description": "Successful Response" }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/projects/{project_id}/evaluation/benchmarks/{benchmark_id}/executions/{execution_id}/lineages": { "post": { "tags": [ "Lineages" ], "summary": "Create Benchmark Lineages", "description": "Create new `BenchmarkLineage`s. Accepts gzip encoding.", "operationId": "create_benchmark_lineages_projects__project_id__evaluation_benchmarks__benchmark_id__executions__execution_id__lineages_post", "parameters": [ { "name": "project_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Project Id" } }, { "name": "benchmark_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Benchmark Id" } }, { "name": "execution_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Execution Id" } } ], "responses": { "201": { "description": "Successful Response", "content": { "application/json": { "schema": { "type": "array", "items": { "type": "string", "format": "uuid" }, "title": "Response Create Benchmark Lineages Projects Project Id Evaluation Benchmarks Benchmark Id Executions Execution Id Lineages Post" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } }, "requestBody": { "content": { "application/json": { "schema": { "type": "array", "items": { "properties": { "trace_id": { "title": "Trace Id", "type": "string" }, "input": { "title": "Input" }, "expected_output": { "title": "Expected Output" }, "example_metadata": { "anyOf": [ { "additionalProperties": true, "type": "object" }, { "type": "null" } ], "title": "Example Metadata" }, "output": { "title": "Output" }, "evaluation": { "title": "Evaluation" }, "run_latency": { "title": "Run Latency", "type": "integer" }, "run_tokens": { "title": "Run Tokens", "type": "integer" } }, "required": [ "trace_id", "input", "expected_output", "output", "evaluation", "run_latency", "run_tokens" ], "title": "BenchmarkLineageRequest", "type": "object" }, "example": [ { "trace_id": "db78d79f-5cac-4e79-842c-07c496e765f7", "input": "input", "expected_output": "test", "output": "output data", "evaluation": "evaluation data", "run_latency": 0, "run_tokens": 0 } ] } } }, "required": true } }, "get": { "tags": [ "Lineages" ], "summary": "Get Many Benchmark Lineages", "description": "Get many `BenchmarkLineage`s. Results are sorted by descending creation date.", "operationId": "get_many_benchmark_lineages_projects__project_id__evaluation_benchmarks__benchmark_id__executions__execution_id__lineages_get", "parameters": [ { "name": "project_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Project Id" } }, { "name": "benchmark_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Benchmark Id" } }, { "name": "execution_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Execution Id" } }, { "name": "search", "in": "query", "required": false, "schema": { "anyOf": [ { "type": "string", "minLength": 1 }, { "type": "null" } ], "description": "Search term to find results containing this text in any searchable field (case insensitive).", "title": "Search" }, "description": "Search term to find results containing this text in any searchable field (case insensitive)." }, { "name": "sort_by", "in": "query", "required": false, "schema": { "anyOf": [ { "$ref": "#/components/schemas/BenchmarkLineageSortableColumns" }, { "type": "string" } ], "description": "The column to sort by. Can be a predefined column or 'evaluation.KEY' for JSONB sorting.", "default": "created_at", "title": "Sort By" }, "description": "The column to sort by. Can be a predefined column or 'evaluation.KEY' for JSONB sorting." }, { "name": "sort_direction", "in": "query", "required": false, "schema": { "$ref": "#/components/schemas/SortDirection", "description": "The direction to sort by.", "default": "desc" }, "description": "The direction to sort by." }, { "name": "page", "in": "query", "required": false, "schema": { "type": "integer", "exclusiveMinimum": 0, "description": "The page number to retrieve. Starts at 1.", "default": 1, "title": "Page" }, "description": "The page number to retrieve. Starts at 1." }, { "name": "size", "in": "query", "required": false, "schema": { "type": "integer", "maximum": 100, "exclusiveMinimum": 0, "description": "The number of items per page.", "default": 25, "title": "Size" }, "description": "The number of items per page." } ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/PagedResponse_BenchmarkLineageResponse_" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/projects/{project_id}/evaluation/benchmarks/{benchmark_id}/executions/{execution_id}/lineages/{lineage_id}": { "get": { "tags": [ "Lineages" ], "summary": "Get Benchmark Lineage", "description": "Get a `BenchmarkLineage` by ID.", "operationId": "get_benchmark_lineage_projects__project_id__evaluation_benchmarks__benchmark_id__executions__execution_id__lineages__lineage_id__get", "parameters": [ { "name": "project_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Project Id" } }, { "name": "benchmark_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Benchmark Id" } }, { "name": "execution_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Execution Id" } }, { "name": "lineage_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Lineage Id" } } ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "anyOf": [ { "$ref": "#/components/schemas/BenchmarkLineageResponse" }, { "type": "null" } ], "title": "Response Get Benchmark Lineage Projects Project Id Evaluation Benchmarks Benchmark Id Executions Execution Id Lineages Lineage Id Get" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/projects/{project_id}/evaluation/datasets": { "get": { "tags": [ "Evaluation Datasets" ], "summary": "Get Many Datasets", "operationId": "get_many_datasets_projects__project_id__evaluation_datasets_get", "parameters": [ { "name": "project_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Project Id" } }, { "name": "page", "in": "query", "required": false, "schema": { "type": "integer", "exclusiveMinimum": 0, "description": "The page number to retrieve. Starts at 1.", "default": 1, "title": "Page" }, "description": "The page number to retrieve. Starts at 1." }, { "name": "size", "in": "query", "required": false, "schema": { "type": "integer", "maximum": 100, "exclusiveMinimum": 0, "description": "The number of items per page.", "default": 25, "title": "Size" }, "description": "The number of items per page." }, { "name": "benchmarks", "in": "query", "required": false, "schema": { "type": "boolean", "description": "If set, sets `associatedBenchmarks` on `Dataset` objects . Otherwise this is null.", "default": false, "title": "Benchmarks" }, "description": "If set, sets `associatedBenchmarks` on `Dataset` objects . Otherwise this is null." } ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/PagedResponse_Dataset_" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } }, "post": { "tags": [ "Evaluation Datasets" ], "summary": "Create Dataset", "operationId": "create_dataset_projects__project_id__evaluation_datasets_post", "parameters": [ { "name": "project_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Project Id" } } ], "requestBody": { "required": true, "content": { "application/x-www-form-urlencoded": { "schema": { "$ref": "#/components/schemas/Body_create_dataset_projects__project_id__evaluation_datasets_post" } } } }, "responses": { "201": { "description": "Successful Response", "content": { "application/json": { "schema": { "type": "string", "format": "uuid", "title": "Response Create Dataset Projects Project Id Evaluation Datasets Post" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/projects/{project_id}/evaluation/datasets/{dataset_id}": { "get": { "tags": [ "Evaluation Datasets" ], "summary": "Get Dataset", "operationId": "get_dataset_projects__project_id__evaluation_datasets__dataset_id__get", "parameters": [ { "name": "project_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Project Id" } }, { "name": "dataset_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Dataset Id" } }, { "name": "benchmarks", "in": "query", "required": false, "schema": { "type": "boolean", "description": "If set, sets `associatedBenchmarks` on `Dataset` objects . Otherwise this is null.", "default": false, "title": "Benchmarks" }, "description": "If set, sets `associatedBenchmarks` on `Dataset` objects . Otherwise this is null." } ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Dataset" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } }, "delete": { "tags": [ "Evaluation Datasets" ], "summary": "Delete Dataset", "operationId": "delete_dataset_projects__project_id__evaluation_datasets__dataset_id__delete", "parameters": [ { "name": "dataset_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Dataset Id" } }, { "name": "project_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Project Id" } } ], "responses": { "204": { "description": "Successful Response" }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } }, "patch": { "tags": [ "Evaluation Datasets" ], "summary": "Update Dataset", "operationId": "update_dataset_projects__project_id__evaluation_datasets__dataset_id__patch", "parameters": [ { "name": "dataset_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Dataset Id" } }, { "name": "project_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Project Id" } } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/DatasetMetadataUpdate" } } } }, "responses": { "204": { "description": "Successful Response" }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/projects/{project_id}/evaluation/datasets/{dataset_id}/datapoints": { "get": { "tags": [ "Evaluation Datasets" ], "summary": "Get Dataset Datapoints", "operationId": "get_dataset_datapoints_projects__project_id__evaluation_datasets__dataset_id__datapoints_get", "parameters": [ { "name": "dataset_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Dataset Id" } }, { "name": "project_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Project Id" } }, { "name": "page", "in": "query", "required": false, "schema": { "type": "integer", "exclusiveMinimum": 0, "description": "The page number to retrieve. Starts at 1.", "default": 1, "title": "Page" }, "description": "The page number to retrieve. Starts at 1." }, { "name": "size", "in": "query", "required": false, "schema": { "type": "integer", "maximum": 100, "exclusiveMinimum": 0, "description": "The number of items per page.", "default": 25, "title": "Size" }, "description": "The number of items per page." } ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/PagedResponse_Example_JsonSerializable__JsonSerializable__" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/projects/{project_id}/evaluation/datasets/{dataset_id}/downloads": { "post": { "tags": [ "Evaluation Datasets" ], "summary": "Create Dataset Download Link", "operationId": "create_dataset_download_link_projects__project_id__evaluation_datasets__dataset_id__downloads_post", "parameters": [ { "name": "dataset_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Dataset Id" } }, { "name": "project_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Project Id" } } ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/DownloadLinkResponse" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/projects": { "post": { "tags": [ "Projects" ], "summary": "Create Project", "description": "Create a new `Project`.", "operationId": "create_project_projects_post", "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ProjectRequest" } } } }, "responses": { "201": { "description": "Created", "content": { "application/json": { "schema": { "type": "string", "format": "uuid", "title": "Response Create Project Projects Post" } } } }, "400": { "description": "Bad Request" }, "401": { "description": "Unauthorized" }, "403": { "description": "Forbidden" }, "409": { "description": "Prerequisites not met or project name already exists" }, "500": { "description": "Internal Server Error" }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } }, "get": { "tags": [ "Projects" ], "summary": "Get Many Projects", "operationId": "get_many_projects_projects_get", "parameters": [ { "name": "with_user_details", "in": "query", "required": false, "schema": { "type": "boolean", "description": "Whether to include the user details in the response.", "default": false, "title": "With User Details" }, "description": "Whether to include the user details in the response." } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "type": "array", "items": { "$ref": "#/components/schemas/ProjectResponseV2" }, "title": "Response Get Many Projects Projects Get" } } } }, "401": { "description": "Unauthorized" }, "403": { "description": "Forbidden" }, "404": { "description": "Workspace not found" }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/projects/{project_id}": { "get": { "tags": [ "Projects" ], "summary": "Get Project", "description": "Get a `Project` by its id.", "operationId": "get_project_projects__project_id__get", "parameters": [ { "name": "project_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "description": "The UUID of the project.", "title": "Project Id" }, "description": "The UUID of the project." }, { "name": "with_user_details", "in": "query", "required": false, "schema": { "type": "boolean", "description": "Whether to include the user details in the response.", "default": false, "title": "With User Details" }, "description": "Whether to include the user details in the response." } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ProjectResponseV2" } } } }, "401": { "description": "Unauthorized" }, "403": { "description": "Forbidden" }, "404": { "description": "Project not found" }, "409": { "description": "Prerequisites not met" }, "500": { "description": "Internal Server Error" }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } }, "delete": { "tags": [ "Projects" ], "summary": "Delete Project", "description": "Delete a `Project` by its id.", "operationId": "delete_project_projects__project_id__delete", "parameters": [ { "name": "project_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "description": "The UUID of the project.", "title": "Project Id" }, "description": "The UUID of the project." } ], "responses": { "204": { "description": "No content" }, "401": { "description": "Unauthorized" }, "403": { "description": "Forbidden" }, "404": { "description": "Project not found" }, "409": { "description": "Prerequisites not met" }, "500": { "description": "Internal Server Error" }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } }, "patch": { "tags": [ "Projects" ], "summary": "Update Project", "description": "Update a project's details.", "operationId": "update_project_projects__project_id__patch", "parameters": [ { "name": "project_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "description": "The UUID of the project.", "title": "Project Id" }, "description": "The UUID of the project." } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/PatchProjectRequest" } } } }, "responses": { "204": { "description": "No content" }, "400": { "description": "Bad Request" }, "401": { "description": "Unauthorized" }, "403": { "description": "Forbidden" }, "404": { "description": "Project not found" }, "409": { "description": "Prerequisites not met" }, "500": { "description": "Internal Server Error" }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/workspaces/{workspace_id}/projects/{project_id}/userlist": { "get": { "tags": [ "Projects" ], "summary": "Get Project Userlist", "description": "Get a list of all non-service users of a Project by the `Project` identifier.", "operationId": "get_project_userlist_workspaces__workspace_id__projects__project_id__userlist_get", "parameters": [ { "name": "workspace_id", "in": "path", "required": true, "schema": { "anyOf": [ { "type": "string", "format": "uuid" }, { "const": "default", "type": "string" } ], "title": "Workspace Id" } }, { "name": "project_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "description": "The UUID of the project.", "title": "Project Id" }, "description": "The UUID of the project." } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/GetProjectUserlistResponse" } } } }, "400": { "description": "Bad Request" }, "401": { "description": "Unauthorized" }, "404": { "description": "List of Project user not found" }, "500": { "description": "Internal Server Error" }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } }, "patch": { "tags": [ "Projects" ], "summary": "Update Project Userlist", "description": "Update the list of users of a `Project`", "operationId": "update_project_userlist_workspaces__workspace_id__projects__project_id__userlist_patch", "parameters": [ { "name": "workspace_id", "in": "path", "required": true, "schema": { "anyOf": [ { "type": "string", "format": "uuid" }, { "const": "default", "type": "string" } ], "title": "Workspace Id" } }, { "name": "project_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "description": "The UUID of the project.", "title": "Project Id" }, "description": "The UUID of the project." } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/UpdateProjectUserlistRequest" } } } }, "responses": { "204": { "description": "No content" }, "400": { "description": "Bad Request" }, "401": { "description": "Unauthorized" }, "403": { "description": "Forbidden" }, "409": { "description": "Prerequisites not met" }, "500": { "description": "Internal Server Error" }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/workspaces/{workspace_id}/userlist": { "get": { "tags": [ "Workspaces" ], "summary": "Get Workspace Userlist With Groups", "description": "Get a list of all users of a Workspace by the `Workspace` identifier", "operationId": "get_workspace_userlist_with_groups_workspaces__workspace_id__userlist_get", "parameters": [ { "name": "workspace_id", "in": "path", "required": true, "schema": { "anyOf": [ { "type": "string", "format": "uuid" }, { "const": "default", "type": "string" } ], "title": "Workspace Id" } }, { "name": "page", "in": "query", "required": false, "schema": { "type": "integer", "exclusiveMinimum": 0, "description": "The page number to retrieve. Starts at 1.", "default": 1, "title": "Page" }, "description": "The page number to retrieve. Starts at 1." }, { "name": "size", "in": "query", "required": false, "schema": { "type": "integer", "maximum": 100, "exclusiveMinimum": 0, "description": "The number of items per page.", "default": 25, "title": "Size" }, "description": "The number of items per page." }, { "name": "email_filter", "in": "query", "required": false, "schema": { "type": "string", "description": "Return users whose email addresses contain the filter as a substring. Case insensitive. If not provided, all results are returned.", "examples": [ "foo" ], "default": "", "title": "Email Filter" }, "description": "Return users whose email addresses contain the filter as a substring. Case insensitive. If not provided, all results are returned." } ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/PagedResponse_Union_User__Group__" } } } }, "404": { "description": "List of Workspace users not found" }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/projects/{project_id}/traces_v2": { "get": { "tags": [ "Application Traces" ], "summary": "Get Traces", "operationId": "get_traces_projects__project_id__traces_v2_get", "parameters": [ { "name": "project_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Project Id" } }, { "name": "limit", "in": "query", "required": false, "schema": { "type": "integer", "default": 20, "title": "Limit" } }, { "name": "offset", "in": "query", "required": false, "schema": { "type": "integer", "default": 0, "title": "Offset" } }, { "name": "start_time", "in": "query", "required": false, "schema": { "anyOf": [ { "type": "string", "format": "date-time" }, { "type": "null" } ], "title": "Start Time" } }, { "name": "end_time", "in": "query", "required": false, "schema": { "anyOf": [ { "type": "string", "format": "date-time" }, { "type": "null" } ], "title": "End Time" } }, { "name": "application_id", "in": "query", "required": false, "schema": { "anyOf": [ { "type": "string", "format": "uuid" }, { "type": "null" } ], "title": "Application Id" } }, { "name": "status", "in": "query", "required": false, "schema": { "anyOf": [ { "$ref": "#/components/schemas/TraceStatus" }, { "type": "null" } ], "title": "Status" } } ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/TraceListResponse" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } }, "post": { "tags": [ "Application Traces" ], "summary": "Receive Otlp Traces", "description": "OTLP traces receiver endpoint. Currently only supports http/protobuf format. This endpoint is a receiver for traces from OpenTelemetry collectors or other OTLP compatible exporters.", "operationId": "receive_otlp_traces_projects__project_id__traces_v2_post", "parameters": [ { "name": "project_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Project Id" } } ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": {} } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/projects/{project_id}/traces_v2/{trace_id}": { "get": { "tags": [ "Application Traces" ], "summary": "Get Trace By Id", "description": "Get comprehensive details for a specific `Trace`.", "operationId": "get_trace_by_id_projects__project_id__traces_v2__trace_id__get", "parameters": [ { "name": "project_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Project Id" } }, { "name": "trace_id", "in": "path", "required": true, "schema": { "type": "string", "title": "Trace Id" } } ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/TraceResponseV2" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } } }, "components": { "schemas": { "AggregationLogic": { "properties": { "logic": { "type": "string", "title": "Logic", "description": "The code of the aggregation logic.", "examples": [ "logic" ] }, "evaluation_schema": { "additionalProperties": true, "type": "object", "title": "Evaluation Schema", "description": "The evaluation schema of the aggregation logic as a JSON schema https://json-schema.org/draft/2020-12/release-notes..", "examples": [ { "properties": { "name": { "title": "Name", "type": "string" }, "email": { "title": "Email", "type": "string" } }, "required": [ "name", "email" ], "title": "Test", "type": "object" } ] }, "aggregation_schema": { "additionalProperties": true, "type": "object", "title": "Aggregation Schema", "description": "The aggregated evaluation schema of the aggregation logic as a JSON schema https://json-schema.org/draft/2020-12/release-notes..", "examples": [ { "properties": { "name": { "title": "Name", "type": "string" }, "email": { "title": "Email", "type": "string" } }, "required": [ "name", "email" ], "title": "Test", "type": "object" } ] } }, "type": "object", "required": [ "logic", "evaluation_schema", "aggregation_schema" ], "title": "AggregationLogic" }, "AssociatedBenchmark": { "properties": { "id": { "type": "string", "title": "Id" }, "name": { "type": "string", "title": "Name" } }, "type": "object", "required": [ "id", "name" ], "title": "AssociatedBenchmark", "description": "Benchmark associated with a dataset." }, "BenchmarkExecutionResponse": { "properties": { "name": { "type": "string", "title": "Name", "description": "The name of the benchmark execution. This field is searchable.", "examples": [ "Benchmark Execution" ] }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Description", "description": "The description of the benchmark execution. This field is searchable.", "examples": [ "This is a benchmark execution." ] }, "labels": { "anyOf": [ { "items": { "type": "string" }, "type": "array" }, { "type": "null" } ], "title": "Labels" }, "metadata": { "anyOf": [ { "additionalProperties": true, "type": "object" }, { "type": "null" } ], "title": "Metadata" }, "start": { "type": "string", "format": "date-time", "title": "Start", "description": "Start of the benchmark execution in UTC." }, "end": { "type": "string", "format": "date-time", "title": "End", "description": "End of the benchmark execution in UTC. Does not include upload time." }, "run_start": { "type": "string", "format": "date-time", "title": "Run Start", "description": "Start of running the given task on the benchmark dataset in UTC." }, "run_end": { "type": "string", "format": "date-time", "title": "Run End", "description": "End of running the given task on the benchmark dataset in UTC." }, "run_successful_count": { "type": "integer", "title": "Run Successful Count", "description": "Number of examples that the task was successfully executed on." }, "run_failed_count": { "type": "integer", "title": "Run Failed Count", "description": "Number of examples that the task raised an exception on. These will be skipped in the evaluation." }, "run_success_avg_latency": { "type": "number", "title": "Run Success Avg Latency", "description": "Average time it takes the task to generate an output in ns. Does not include failed runs." }, "run_success_avg_token_count": { "type": "number", "title": "Run Success Avg Token Count", "description": "Average amount of tokens it takes the task to generate an output. Includes tokens from intermediate steps. Does not include failed runs." }, "eval_start": { "type": "string", "format": "date-time", "title": "Eval Start", "description": "Start of evaluation process of the given task on the benchmark dataset in UTC." }, "eval_end": { "type": "string", "format": "date-time", "title": "Eval End", "description": "End of evaluation process of the given task on the benchmark dataset in UTC." }, "eval_successful_count": { "type": "integer", "title": "Eval Successful Count", "description": "Number of evaluations of task runs that succeeded. Failed runs are skipped and not included." }, "eval_failed_count": { "type": "integer", "title": "Eval Failed Count", "description": "Number of evaluations of task runs that failed. Failed runs are skipped and not included." }, "aggregation_start": { "type": "string", "format": "date-time", "title": "Aggregation Start", "description": "Start of the aggregation process in UTC." }, "aggregation_end": { "type": "string", "format": "date-time", "title": "Aggregation End", "description": "End of the aggregation process in UTC." }, "statistics": { "$ref": "#/components/schemas/JsonSerializable-Output", "description": "Data that is the result of the aggregation process." }, "updated_at": { "type": "string", "format": "date-time", "title": "Updated At", "description": "The timestamp when this resource was last updated in UTC." }, "updated_by": { "type": "string", "title": "Updated By", "description": "The ID of the user that updated the resource.", "examples": [ "12345" ] }, "created_at": { "type": "string", "format": "date-time", "title": "Created At", "description": "The timestamp when this resource was created in UTC." }, "created_by": { "type": "string", "title": "Created By", "description": "The ID of the user that created the resource.", "examples": [ "12345" ] }, "id": { "type": "string", "title": "Id" }, "benchmark_id": { "type": "string", "title": "Benchmark Id" } }, "type": "object", "required": [ "name", "start", "end", "run_start", "run_end", "run_successful_count", "run_failed_count", "run_success_avg_latency", "run_success_avg_token_count", "eval_start", "eval_end", "eval_successful_count", "eval_failed_count", "aggregation_start", "aggregation_end", "statistics", "updated_at", "updated_by", "created_at", "created_by", "id", "benchmark_id" ], "title": "BenchmarkExecutionResponse" }, "BenchmarkExecutionSortableColumns": { "type": "string", "enum": [ "name", "description", "eval_successful_count", "run_success_avg_latency", "run_success_avg_token_count", "created_at", "start", "end", "run_start", "run_end", "eval_start", "eval_end", "aggregation_start", "aggregation_end", "updated_at", "created_by", "updated_by" ], "title": "BenchmarkExecutionSortableColumns" }, "BenchmarkLineageResponse": { "properties": { "created_at": { "type": "string", "format": "date-time", "title": "Created At", "description": "The timestamp when this resource was created in UTC." }, "created_by": { "type": "string", "title": "Created By", "description": "The ID of the user that created the resource.", "examples": [ "12345" ] }, "id": { "type": "string", "title": "Id" }, "trace_id": { "type": "string", "title": "Trace Id" }, "benchmark_execution_id": { "type": "string", "title": "Benchmark Execution Id" }, "input": { "title": "Input", "description": "The input of the benchmark lineage. This field is searchable." }, "expected_output": { "title": "Expected Output", "description": "The expected output of the benchmark lineage. This field is searchable." }, "example_metadata": { "anyOf": [ { "additionalProperties": true, "type": "object" }, { "type": "null" } ], "title": "Example Metadata" }, "output": { "title": "Output", "description": "The output of the benchmark lineage. This field is searchable." }, "evaluation": { "title": "Evaluation" }, "run_latency": { "type": "integer", "title": "Run Latency" }, "run_tokens": { "type": "integer", "title": "Run Tokens" } }, "type": "object", "required": [ "created_at", "created_by", "id", "trace_id", "benchmark_execution_id", "input", "expected_output", "output", "evaluation", "run_latency", "run_tokens" ], "title": "BenchmarkLineageResponse" }, "BenchmarkLineageSortableColumns": { "type": "string", "enum": [ "run_latency", "run_tokens", "created_by", "created_at" ], "title": "BenchmarkLineageSortableColumns", "description": "Sortable columns for benchmark lineage queries." }, "BenchmarkSortableColumns": { "type": "string", "enum": [ "name", "description", "last_executed_at", "created_at", "updated_at", "created_by", "updated_by" ], "title": "BenchmarkSortableColumns" }, "Body_create_dataset_projects__project_id__evaluation_datasets_post": { "properties": { "source_data": { "type": "string", "contentMediaType": "application/octet-stream", "title": "Source Data", "description": "File to upload. This needs to be in jsonl format." }, "name": { "type": "string", "title": "Name" }, "total_datapoints": { "type": "integer", "title": "Total Datapoints" }, "metadata": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Metadata", "description": "The format needs to be a valid dictionary parsed as a string. E.g., \"{'a': 'b'}\"." }, "labels": { "anyOf": [ { "items": { "type": "string" }, "type": "array" }, { "type": "null" } ], "title": "Labels" } }, "type": "object", "required": [ "source_data", "name", "total_datapoints" ], "title": "Body_create_dataset_projects__project_id__evaluation_datasets_post" }, "ChatCompletionRequest": { "properties": { "messages": { "items": { "$ref": "#/components/schemas/Message" }, "type": "array", "title": "Messages" }, "model": { "type": "string", "title": "Model" }, "stream": { "type": "boolean", "title": "Stream", "default": true } }, "additionalProperties": true, "type": "object", "required": [ "messages", "model" ], "title": "ChatCompletionRequest", "description": "Request model for chat completion." }, "Context": { "properties": { "trace_id": { "type": "string", "format": "uuid", "title": "Trace Id" }, "span_id": { "type": "string", "format": "uuid", "title": "Span Id" } }, "type": "object", "required": [ "trace_id", "span_id" ], "title": "Context" }, "Dataset": { "properties": { "repositoryId": { "type": "string", "title": "Repositoryid" }, "datasetId": { "type": "string", "title": "Datasetid" }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Name" }, "labels": { "anyOf": [ { "items": { "type": "string" }, "type": "array" }, { "type": "null" } ], "title": "Labels" }, "version": { "type": "string", "title": "Version" }, "totalDatapoints": { "type": "integer", "title": "Totaldatapoints" }, "metadata": { "anyOf": [ { "additionalProperties": true, "type": "object" }, { "type": "null" } ], "title": "Metadata" }, "createdAt": { "type": "string", "format": "date-time", "title": "Createdat" }, "updatedAt": { "type": "string", "format": "date-time", "title": "Updatedat" }, "associatedBenchmarks": { "anyOf": [ { "items": { "$ref": "#/components/schemas/AssociatedBenchmark" }, "type": "array" }, { "type": "null" } ], "title": "Associatedbenchmarks" } }, "additionalProperties": true, "type": "object", "required": [ "repositoryId", "datasetId", "version", "totalDatapoints", "createdAt", "updatedAt" ], "title": "Dataset", "description": "Studio API model for datasets." }, "DatasetMetadataUpdate": { "properties": { "labels": { "anyOf": [ { "items": { "type": "string" }, "type": "array" }, { "type": "null" } ], "title": "Labels" }, "metadata": { "anyOf": [ { "additionalProperties": true, "type": "object" }, { "type": "null" } ], "title": "Metadata" }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Name" }, "totalDatapoints": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Totaldatapoints" } }, "type": "object", "title": "DatasetMetadataUpdate", "description": "Model for updating dataset metadata." }, "DownloadLinkResponse": { "properties": { "downloadId": { "type": "string", "title": "Downloadid" }, "datasetVersion": { "type": "string", "title": "Datasetversion" }, "totalDatapoints": { "type": "integer", "title": "Totaldatapoints" }, "mediaType": { "type": "string", "title": "Mediatype" }, "url": { "type": "string", "title": "Url" }, "createdAt": { "type": "string", "format": "date-time", "title": "Createdat" }, "expireAt": { "type": "string", "format": "date-time", "title": "Expireat" } }, "type": "object", "required": [ "downloadId", "datasetVersion", "totalDatapoints", "mediaType", "url", "createdAt", "expireAt" ], "title": "DownloadLinkResponse", "description": "Response model for dataset download links." }, "EvaluationLogic": { "properties": { "logic": { "type": "string", "title": "Logic", "description": "The code of the evaluation logic.", "examples": [ "logic" ] }, "input_schema": { "additionalProperties": true, "type": "object", "title": "Input Schema", "description": "The input schema of the evaluation logic as a JSON schema https://json-schema.org/draft/2020-12/release-notes.", "examples": [ { "properties": { "name": { "title": "Name", "type": "string" }, "email": { "title": "Email", "type": "string" } }, "required": [ "name", "email" ], "title": "Test", "type": "object" } ] }, "output_schema": { "additionalProperties": true, "type": "object", "title": "Output Schema", "description": "The output schema of the evaluation logic as a JSON schema https://json-schema.org/draft/2020-12/release-notes.", "examples": [ { "properties": { "name": { "title": "Name", "type": "string" }, "email": { "title": "Email", "type": "string" } }, "required": [ "name", "email" ], "title": "Test", "type": "object" } ] }, "expected_output_schema": { "additionalProperties": true, "type": "object", "title": "Expected Output Schema", "description": "The expected output schema of the evaluation logic as a JSON schema https://json-schema.org/draft/2020-12/release-notes.", "examples": [ { "properties": { "name": { "title": "Name", "type": "string" }, "email": { "title": "Email", "type": "string" } }, "required": [ "name", "email" ], "title": "Test", "type": "object" } ] }, "evaluation_schema": { "additionalProperties": true, "type": "object", "title": "Evaluation Schema", "description": "The evaluation schema of the evaluation logic, as a JSON schema https://json-schema.org/draft/2020-12/release-notes.", "examples": [ { "properties": { "name": { "title": "Name", "type": "string" }, "email": { "title": "Email", "type": "string" } }, "required": [ "name", "email" ], "title": "Test", "type": "object" } ] } }, "type": "object", "required": [ "logic", "input_schema", "output_schema", "expected_output_schema", "evaluation_schema" ], "title": "EvaluationLogic" }, "Event": { "properties": { "name": { "type": "string", "title": "Name" }, "message": { "type": "string", "title": "Message" }, "body": { "$ref": "#/components/schemas/JsonSerializable-Input" }, "timestamp": { "type": "string", "format": "date-time", "title": "Timestamp" } }, "type": "object", "required": [ "name", "message", "body" ], "title": "Event" }, "EventResponse-Input": { "properties": { "created_at": { "type": "string", "format": "date-time", "title": "Created At", "description": "The timestamp when this resource was created in UTC." }, "created_by": { "type": "string", "title": "Created By", "description": "The ID of the user that created the resource.", "examples": [ "12345" ] }, "id": { "type": "string", "format": "uuid", "title": "Id", "description": "The unique id of the event.", "examples": [ "123e4567-e89b-12d3-a456-426614174000" ] }, "name": { "type": "string", "title": "Name", "description": "The name of the event.", "examples": [ "Event" ] }, "body": { "anyOf": [ { "$ref": "#/components/schemas/JsonSerializable-Input" }, { "type": "null" } ], "description": "The body of the event.", "examples": [ { "key": "value" } ] }, "message": { "type": "string", "title": "Message", "description": "The message of the event.", "examples": [ "Message" ] }, "timestamp": { "type": "string", "format": "date-time", "title": "Timestamp", "description": "The timestamp when this event was created.", "examples": [ "2024-01-01 12:34:56.123456+00:00" ] } }, "type": "object", "required": [ "created_at", "created_by", "id", "name", "body", "message", "timestamp" ], "title": "EventResponse" }, "EventResponse-Output": { "properties": { "created_at": { "type": "string", "format": "date-time", "title": "Created At", "description": "The timestamp when this resource was created in UTC." }, "created_by": { "type": "string", "title": "Created By", "description": "The ID of the user that created the resource.", "examples": [ "12345" ] }, "id": { "type": "string", "format": "uuid", "title": "Id", "description": "The unique id of the event.", "examples": [ "123e4567-e89b-12d3-a456-426614174000" ] }, "name": { "type": "string", "title": "Name", "description": "The name of the event.", "examples": [ "Event" ] }, "body": { "anyOf": [ { "$ref": "#/components/schemas/JsonSerializable-Output" }, { "type": "null" } ], "description": "The body of the event.", "examples": [ { "key": "value" } ] }, "message": { "type": "string", "title": "Message", "description": "The message of the event.", "examples": [ "Message" ] }, "timestamp": { "type": "string", "format": "date-time", "title": "Timestamp", "description": "The timestamp when this event was created.", "examples": [ "2024-01-01 12:34:56.123456+00:00" ] } }, "type": "object", "required": [ "created_at", "created_by", "id", "name", "body", "message", "timestamp" ], "title": "EventResponse" }, "Example_JsonSerializable_JsonSerializable_": { "properties": { "input": { "$ref": "#/components/schemas/JsonSerializable-Output" }, "expected_output": { "$ref": "#/components/schemas/JsonSerializable-Output" }, "id": { "type": "string", "title": "Id" }, "metadata": { "anyOf": [ { "additionalProperties": { "$ref": "#/components/schemas/JsonSerializable-Output" }, "type": "object" }, { "type": "null" } ], "title": "Metadata" } }, "type": "object", "required": [ "input", "expected_output" ], "title": "Example[JsonSerializable, JsonSerializable]" }, "GetBenchmarkResponse": { "properties": { "updated_at": { "type": "string", "format": "date-time", "title": "Updated At", "description": "The timestamp when this resource was last updated in UTC." }, "updated_by": { "type": "string", "title": "Updated By", "description": "The ID of the user that updated the resource.", "examples": [ "12345" ] }, "created_at": { "type": "string", "format": "date-time", "title": "Created At", "description": "The timestamp when this resource was created in UTC." }, "created_by": { "type": "string", "title": "Created By", "description": "The ID of the user that created the resource.", "examples": [ "12345" ] }, "id": { "type": "string", "title": "Id", "description": "The unique id of the benchmark.", "examples": [ "123e4567-e89b-12d3-a456-426614174000" ] }, "project_id": { "type": "string", "format": "uuid", "title": "Project Id", "description": "The universal unique id of the project.", "examples": [ "123e4567-e89b-12d3-a456-426614174000" ] }, "dataset_id": { "type": "string", "title": "Dataset Id", "description": "The unique id of the dataset.", "examples": [ "123e4567-e89b-12d3-a456-426614174000" ] }, "dataset_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Dataset Name", "description": "The name of the dataset.", "examples": [ "Dataset" ] }, "name": { "type": "string", "maxLength": 100, "title": "Name", "description": "The name of the benchmark. This field is searchable.", "examples": [ "Benchmark" ] }, "description": { "anyOf": [ { "type": "string", "maxLength": 1000 }, { "type": "null" } ], "title": "Description", "description": "The description of the benchmark. This field is searchable.", "examples": [ "This is a benchmark." ] }, "benchmark_metadata": { "anyOf": [ { "additionalProperties": true, "type": "object" }, { "type": "null" } ], "title": "Benchmark Metadata", "description": "The metadata of the benchmark.", "examples": [ { "key": "value" } ] }, "evaluation_logic": { "$ref": "#/components/schemas/EvaluationLogic", "description": "The evaluation logic of the benchmark." }, "aggregation_logic": { "$ref": "#/components/schemas/AggregationLogic", "description": "The aggregation logic of the benchmark." }, "last_executed_at": { "anyOf": [ { "type": "string", "format": "date-time" }, { "type": "null" } ], "title": "Last Executed At", "description": "The timestamp when this benchmark was last executed in UTC.", "examples": [ "2024-01-01 12:34:56.123456+00:00" ] } }, "type": "object", "required": [ "updated_at", "updated_by", "created_at", "created_by", "id", "project_id", "dataset_id", "name", "description", "benchmark_metadata", "evaluation_logic", "aggregation_logic", "last_executed_at" ], "title": "GetBenchmarkResponse" }, "GetProjectUserlistResponse": { "items": { "$ref": "#/components/schemas/ProjectUser" }, "type": "array", "title": "GetProjectUserlistResponse", "description": "Response model for getting project user list." }, "Group": { "properties": { "type": { "type": "string", "const": "group", "title": "Type", "default": "group" }, "id": { "type": "string", "title": "Id", "description": "The unique identifier of the group", "examples": [ "123456789" ] }, "display_name": { "type": "string", "title": "Display Name", "description": "The display name of the group", "examples": [ "Kung Fu fighters" ] }, "created_at": { "type": "string", "format": "date-time", "title": "Created At", "description": "The UTC timestamp of the time the group has been created in ISO 8601 format.", "examples": [ "2024-08-08T15:09:00Z" ] }, "updated_at": { "type": "string", "format": "date-time", "title": "Updated At", "description": "The UTC timestamp of the time the group has been updated in ISO 8601 format.", "examples": [ "2024-08-08T15:09:00Z" ] } }, "type": "object", "required": [ "id", "display_name", "created_at", "updated_at" ], "title": "Group" }, "HTTPValidationError": { "properties": { "detail": { "items": { "$ref": "#/components/schemas/ValidationError" }, "type": "array", "title": "Detail" } }, "type": "object", "title": "HTTPValidationError" }, "JsonSerializable-Input": { "anyOf": [ { "type": "integer" }, { "type": "number" }, { "type": "string" }, { "type": "boolean" }, { "items": { "$ref": "#/components/schemas/JsonSerializable-Input" }, "type": "array" }, { "additionalProperties": { "$ref": "#/components/schemas/JsonSerializable-Input" }, "type": "object" }, { "type": "null" } ] }, "JsonSerializable-Output": { "anyOf": [ { "type": "integer" }, { "type": "number" }, { "type": "string" }, { "type": "boolean" }, { "items": { "$ref": "#/components/schemas/JsonSerializable-Output" }, "type": "array" }, { "additionalProperties": { "$ref": "#/components/schemas/JsonSerializable-Output" }, "type": "object" }, { "type": "null" } ] }, "Message": { "properties": { "role": { "type": "string", "title": "Role" }, "content": { "type": "string", "title": "Content" } }, "type": "object", "required": [ "role", "content" ], "title": "Message", "description": "Chat message model." }, "ModelCardResponse": { "properties": { "name": { "type": "string", "title": "Name", "description": "The name of the model", "examples": [ "luminous-base-control", "llama-3.1-70b-instruct" ] }, "status": { "type": "string", "enum": [ "available", "unavailable" ], "title": "Status", "description": "The current availability status of the model. Currently supported states are 'available' and 'unavailable'", "examples": [ "available", "unavailable" ] }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Description", "description": "A brief description of the model's capabilities and features", "examples": [ "A model that can generate text" ] }, "max_context_size": { "type": "integer", "title": "Max Context Size", "description": "The maximum number of tokens the model can process in a single input", "examples": [ 2048, 8192 ] }, "aligned": { "type": "boolean", "title": "Aligned", "description": "" }, "semantic_embedding": { "type": "boolean", "title": "Semantic Embedding", "description": "Whether the model can generate semantic embeddings" }, "worker_type": { "type": "string", "title": "Worker Type", "description": "Type of worker that serves the model. Can either be 'luminous' or 'vllm'. If 'luminous' is set, the model supports advanced completion parameters. If called with these parameters and it is not supported, it will raise an error. This is being deprecated because the Inference API does not expose what worker the model is running on anymore.", "deprecated": true, "examples": [ "vllm", "luminous", "" ] }, "multimodal": { "type": "boolean", "title": "Multimodal", "description": "Whether the model can process multiple types of input data (e.g., text, images)" }, "chat": { "type": "boolean", "title": "Chat", "description": "Whether the model is supported by the chat endpoint" }, "completion_type": { "type": "string", "enum": [ "full", "none" ], "title": "Completion Type", "description": "The completion type supported by the model. It states if the model has not been trained to support completions.", "examples": [ "full", "none" ] }, "prompt_template": { "type": "string", "title": "Prompt Template", "description": "The prompt template that should be used to prompt the model", "examples": [ "{{instruction}} \n #Output: " ] }, "category": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Category", "description": "The category of the model", "examples": [ "TextGeneration" ] }, "link": { "type": "string", "title": "Link", "description": "The link to the model card", "examples": [ "https://api.aleph-alpha.com" ] }, "maximum_completion_tokens": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Maximum Completion Tokens", "description": "The maximum number of tokens the model will provide for the output.", "examples": [ 2048, 8192 ] } }, "type": "object", "required": [ "name", "status", "description", "max_context_size", "aligned", "semantic_embedding", "worker_type", "multimodal", "chat", "completion_type", "prompt_template", "link" ], "title": "ModelCardResponse", "description": "Response model for model cards." }, "ModelCompleteRequest": { "properties": { "model": { "type": "string", "title": "Model" }, "prompt": { "type": "string", "title": "Prompt" } }, "additionalProperties": true, "type": "object", "required": [ "model", "prompt" ], "title": "ModelCompleteRequest", "description": "Request model for model completion." }, "PagedResponse_BenchmarkExecutionResponse_": { "properties": { "total": { "type": "integer", "title": "Total" }, "page": { "type": "integer", "title": "Page" }, "size": { "type": "integer", "title": "Size" }, "num_pages": { "type": "integer", "title": "Num Pages" }, "items": { "items": { "$ref": "#/components/schemas/BenchmarkExecutionResponse" }, "type": "array", "title": "Items", "default": [] } }, "type": "object", "required": [ "total", "page", "size", "num_pages" ], "title": "PagedResponse[BenchmarkExecutionResponse]" }, "PagedResponse_BenchmarkLineageResponse_": { "properties": { "total": { "type": "integer", "title": "Total" }, "page": { "type": "integer", "title": "Page" }, "size": { "type": "integer", "title": "Size" }, "num_pages": { "type": "integer", "title": "Num Pages" }, "items": { "items": { "$ref": "#/components/schemas/BenchmarkLineageResponse" }, "type": "array", "title": "Items", "default": [] } }, "type": "object", "required": [ "total", "page", "size", "num_pages" ], "title": "PagedResponse[BenchmarkLineageResponse]" }, "PagedResponse_Dataset_": { "properties": { "total": { "type": "integer", "title": "Total" }, "page": { "type": "integer", "title": "Page" }, "size": { "type": "integer", "title": "Size" }, "num_pages": { "type": "integer", "title": "Num Pages" }, "items": { "items": { "$ref": "#/components/schemas/Dataset" }, "type": "array", "title": "Items", "default": [] } }, "type": "object", "required": [ "total", "page", "size", "num_pages" ], "title": "PagedResponse[Dataset]" }, "PagedResponse_EventResponse_": { "properties": { "total": { "type": "integer", "title": "Total" }, "page": { "type": "integer", "title": "Page" }, "size": { "type": "integer", "title": "Size" }, "num_pages": { "type": "integer", "title": "Num Pages" }, "items": { "items": { "$ref": "#/components/schemas/EventResponse-Output" }, "type": "array", "title": "Items", "default": [] } }, "type": "object", "required": [ "total", "page", "size", "num_pages" ], "title": "PagedResponse[EventResponse]" }, "PagedResponse_Example_JsonSerializable__JsonSerializable__": { "properties": { "total": { "type": "integer", "title": "Total" }, "page": { "type": "integer", "title": "Page" }, "size": { "type": "integer", "title": "Size" }, "num_pages": { "type": "integer", "title": "Num Pages" }, "items": { "items": { "$ref": "#/components/schemas/Example_JsonSerializable_JsonSerializable_" }, "type": "array", "title": "Items", "default": [] } }, "type": "object", "required": [ "total", "page", "size", "num_pages" ], "title": "PagedResponse[Example[JsonSerializable, JsonSerializable]]" }, "PagedResponse_GetBenchmarkResponse_": { "properties": { "total": { "type": "integer", "title": "Total" }, "page": { "type": "integer", "title": "Page" }, "size": { "type": "integer", "title": "Size" }, "num_pages": { "type": "integer", "title": "Num Pages" }, "items": { "items": { "$ref": "#/components/schemas/GetBenchmarkResponse" }, "type": "array", "title": "Items", "default": [] } }, "type": "object", "required": [ "total", "page", "size", "num_pages" ], "title": "PagedResponse[GetBenchmarkResponse]" }, "PagedResponse_SpanResponse_": { "properties": { "total": { "type": "integer", "title": "Total" }, "page": { "type": "integer", "title": "Page" }, "size": { "type": "integer", "title": "Size" }, "num_pages": { "type": "integer", "title": "Num Pages" }, "items": { "items": { "$ref": "#/components/schemas/SpanResponse-Output" }, "type": "array", "title": "Items", "default": [] } }, "type": "object", "required": [ "total", "page", "size", "num_pages" ], "title": "PagedResponse[SpanResponse]" }, "PagedResponse_TraceResponse_": { "properties": { "total": { "type": "integer", "title": "Total" }, "page": { "type": "integer", "title": "Page" }, "size": { "type": "integer", "title": "Size" }, "num_pages": { "type": "integer", "title": "Num Pages" }, "items": { "items": { "$ref": "#/components/schemas/TraceResponse" }, "type": "array", "title": "Items", "default": [] } }, "type": "object", "required": [ "total", "page", "size", "num_pages" ], "title": "PagedResponse[TraceResponse]" }, "PagedResponse_Union_User__Group__": { "properties": { "total": { "type": "integer", "title": "Total" }, "page": { "type": "integer", "title": "Page" }, "size": { "type": "integer", "title": "Size" }, "num_pages": { "type": "integer", "title": "Num Pages" }, "items": { "items": { "anyOf": [ { "$ref": "#/components/schemas/User" }, { "$ref": "#/components/schemas/Group" } ] }, "type": "array", "title": "Items", "default": [] } }, "type": "object", "required": [ "total", "page", "size", "num_pages" ], "title": "PagedResponse[Union[User, Group]]" }, "PatchBenchmarkRequest": { "properties": { "name": { "anyOf": [ { "type": "string", "maxLength": 100 }, { "type": "null" } ], "title": "Name", "description": "The name of the benchmark.", "examples": [ "Benchmark" ] }, "description": { "anyOf": [ { "type": "string", "maxLength": 1000 }, { "type": "null" } ], "title": "Description", "description": "The description of the benchamrk.", "examples": [ "This is a benchmark." ] }, "metadata": { "anyOf": [ { "additionalProperties": true, "type": "object" }, { "type": "null" } ], "title": "Metadata", "description": "The metadata of the benchmark." } }, "type": "object", "title": "PatchBenchmarkRequest" }, "PatchProjectRequest": { "properties": { "name": { "anyOf": [ { "type": "string", "maxLength": 100 }, { "type": "null" } ], "title": "Name", "description": "The name of the project.", "examples": [ "Project" ] }, "description": { "anyOf": [ { "type": "string", "maxLength": 1000 }, { "type": "null" } ], "title": "Description", "description": "The description of the project.", "examples": [ "This is a project." ] }, "archived": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "title": "Archived", "description": "The archival status of the project" } }, "type": "object", "title": "PatchProjectRequest", "description": "Request model for partially updating a project." }, "PostBenchmarkExecutionRequest": { "properties": { "name": { "type": "string", "title": "Name", "description": "The name of the benchmark execution. This field is searchable.", "examples": [ "Benchmark Execution" ] }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Description", "description": "The description of the benchmark execution. This field is searchable.", "examples": [ "This is a benchmark execution." ] }, "labels": { "anyOf": [ { "items": { "type": "string" }, "type": "array" }, { "type": "null" } ], "title": "Labels" }, "metadata": { "anyOf": [ { "additionalProperties": true, "type": "object" }, { "type": "null" } ], "title": "Metadata" }, "start": { "type": "string", "format": "date-time", "title": "Start", "description": "Start of the benchmark execution in UTC." }, "end": { "type": "string", "format": "date-time", "title": "End", "description": "End of the benchmark execution in UTC. Does not include upload time." }, "run_start": { "type": "string", "format": "date-time", "title": "Run Start", "description": "Start of running the given task on the benchmark dataset in UTC." }, "run_end": { "type": "string", "format": "date-time", "title": "Run End", "description": "End of running the given task on the benchmark dataset in UTC." }, "run_successful_count": { "type": "integer", "title": "Run Successful Count", "description": "Number of examples that the task was successfully executed on." }, "run_failed_count": { "type": "integer", "title": "Run Failed Count", "description": "Number of examples that the task raised an exception on. These will be skipped in the evaluation." }, "run_success_avg_latency": { "type": "number", "title": "Run Success Avg Latency", "description": "Average time it takes the task to generate an output in ns. Does not include failed runs." }, "run_success_avg_token_count": { "type": "number", "title": "Run Success Avg Token Count", "description": "Average amount of tokens it takes the task to generate an output. Includes tokens from intermediate steps. Does not include failed runs." }, "eval_start": { "type": "string", "format": "date-time", "title": "Eval Start", "description": "Start of evaluation process of the given task on the benchmark dataset in UTC." }, "eval_end": { "type": "string", "format": "date-time", "title": "Eval End", "description": "End of evaluation process of the given task on the benchmark dataset in UTC." }, "eval_successful_count": { "type": "integer", "title": "Eval Successful Count", "description": "Number of evaluations of task runs that succeeded. Failed runs are skipped and not included." }, "eval_failed_count": { "type": "integer", "title": "Eval Failed Count", "description": "Number of evaluations of task runs that failed. Failed runs are skipped and not included." }, "aggregation_start": { "type": "string", "format": "date-time", "title": "Aggregation Start", "description": "Start of the aggregation process in UTC." }, "aggregation_end": { "type": "string", "format": "date-time", "title": "Aggregation End", "description": "End of the aggregation process in UTC." }, "statistics": { "$ref": "#/components/schemas/JsonSerializable-Input", "description": "Data that is the result of the aggregation process." } }, "type": "object", "required": [ "name", "start", "end", "run_start", "run_end", "run_successful_count", "run_failed_count", "run_success_avg_latency", "run_success_avg_token_count", "eval_start", "eval_end", "eval_successful_count", "eval_failed_count", "aggregation_start", "aggregation_end", "statistics" ], "title": "PostBenchmarkExecutionRequest" }, "PostBenchmarkRequest": { "properties": { "dataset_id": { "type": "string", "title": "Dataset Id", "description": "The unique id of the dataset.", "examples": [ "123e4567-e89b-12d3-a456-426614174000" ] }, "name": { "type": "string", "maxLength": 100, "title": "Name", "description": "The name of the benchmark.", "examples": [ "Benchmark" ] }, "description": { "anyOf": [ { "type": "string", "maxLength": 1000 }, { "type": "null" } ], "title": "Description", "description": "The description of the benchmark.", "examples": [ "This is a benchmark." ] }, "benchmark_metadata": { "anyOf": [ { "additionalProperties": true, "type": "object" }, { "type": "null" } ], "title": "Benchmark Metadata", "description": "The metadata of the benchmark.", "examples": [ { "key": "value" } ] }, "evaluation_logic": { "$ref": "#/components/schemas/EvaluationLogic", "description": "The evaluation logic of the benchmark." }, "aggregation_logic": { "$ref": "#/components/schemas/AggregationLogic", "description": "The aggregation logic of the benchmark." } }, "type": "object", "required": [ "dataset_id", "name", "evaluation_logic", "aggregation_logic" ], "title": "PostBenchmarkRequest" }, "ProjectRequest": { "properties": { "name": { "type": "string", "maxLength": 100, "title": "Name", "description": "The name of the project.", "examples": [ "Project" ] }, "description": { "anyOf": [ { "type": "string", "maxLength": 1000 }, { "type": "null" } ], "title": "Description", "description": "The description of the project.", "examples": [ "This is a project." ] } }, "type": "object", "required": [ "name" ], "title": "ProjectRequest", "description": "Request model for creating or updating a project." }, "ProjectResponseV2": { "properties": { "updated_at": { "type": "string", "format": "date-time", "title": "Updated At", "description": "The timestamp when this resource was last updated in UTC." }, "updated_by": { "type": "string", "title": "Updated By", "description": "The ID of the user that updated the resource.", "examples": [ "12345" ] }, "id": { "type": "integer", "title": "Id", "description": "The unique id of the project.", "examples": [ 1 ] }, "name": { "type": "string", "title": "Name", "description": "The unique name of the project.", "examples": [ "Project" ] }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Description", "description": "The description of the project.", "examples": [ "This is a project." ] }, "project_id": { "type": "string", "format": "uuid", "title": "Project Id", "description": "The universal unique id of the project.", "examples": [ "123e4567-e89b-12d3-a456-426614174000" ] }, "error": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Error", "description": "The error message if the user details could not be retrieved." }, "created_at": { "type": "string", "format": "date-time", "title": "Created At", "description": "The timestamp when this resource was created in UTC." }, "created_by": { "anyOf": [ { "type": "string" }, { "$ref": "#/components/schemas/User" } ], "title": "Created By", "description": "The ID of the user that created the resource.", "examples": [ "12345", { "created_at": "2224-08-08T15:09:00Z", "email": "john.doe@example.com", "id": "12345", "profile": { "given_name": "John", "last_name": "Doe" }, "type": "user", "updated_at": "2224-08-08T15:09:00Z", "user_name": "john.doe" } ] } }, "type": "object", "required": [ "updated_at", "updated_by", "id", "name", "project_id", "created_at", "created_by" ], "title": "ProjectResponseV2", "description": "Response model for project operations (v2) with enhanced user details." }, "ProjectUser": { "properties": { "type": { "type": "string", "const": "user", "title": "Type", "default": "user" }, "id": { "type": "string", "title": "Id", "description": "The unique identifier of the user", "examples": [ "123456789" ] }, "user_name": { "type": "string", "title": "User Name", "description": "The username of the user", "examples": [ "Chuck Norris" ] }, "created_at": { "type": "string", "format": "date-time", "title": "Created At", "description": "The UTC timestamp of the time the user has been created in ISO 8601 format.", "examples": [ "2024-08-08T15:09:00Z" ] }, "updated_at": { "type": "string", "format": "date-time", "title": "Updated At", "description": "The UTC timestamp of the time the user has been updated in ISO 8601 format.", "examples": [ "2024-08-08T15:09:00Z" ] }, "email": { "type": "string", "title": "Email", "description": "The email of the user", "examples": [ "chuck@norris.com" ] }, "profile": { "$ref": "#/components/schemas/UserProfile", "description": "The profile of the user" }, "role": { "$ref": "#/components/schemas/WPUserRole", "description": "The role of the user in a project", "examples": [ "owner", "member" ] } }, "type": "object", "required": [ "id", "user_name", "created_at", "updated_at", "email", "profile", "role" ], "title": "ProjectUser", "description": "User model with project role information." }, "SingularTraceSpanList": { "items": { "$ref": "#/components/schemas/SpanRequest" }, "type": "array", "title": "SingularTraceSpanList" }, "SortDirection": { "type": "string", "enum": [ "asc", "desc" ], "title": "SortDirection" }, "SpanAttributes": { "properties": { "type": { "type": "string", "const": "SPAN", "title": "Type", "default": "SPAN" } }, "type": "object", "title": "SpanAttributes" }, "SpanRequest": { "properties": { "context": { "$ref": "#/components/schemas/Context" }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Name" }, "parent_id": { "anyOf": [ { "type": "string", "format": "uuid" }, { "type": "null" } ], "title": "Parent Id" }, "start_time": { "type": "string", "format": "date-time", "title": "Start Time" }, "end_time": { "type": "string", "format": "date-time", "title": "End Time" }, "attributes": { "oneOf": [ { "$ref": "#/components/schemas/SpanAttributes" }, { "$ref": "#/components/schemas/TaskSpanAttributes" } ], "title": "Attributes", "discriminator": { "propertyName": "type", "mapping": { "SPAN": "#/components/schemas/SpanAttributes", "TASK_SPAN": "#/components/schemas/TaskSpanAttributes" } } }, "events": { "items": { "$ref": "#/components/schemas/Event" }, "type": "array", "title": "Events" }, "status": { "$ref": "#/components/schemas/SpanStatus" } }, "type": "object", "required": [ "context", "name", "parent_id", "start_time", "end_time", "attributes", "events", "status" ], "title": "SpanRequest" }, "SpanResponse-Input": { "properties": { "created_at": { "type": "string", "format": "date-time", "title": "Created At", "description": "The timestamp when this resource was created in UTC." }, "created_by": { "type": "string", "title": "Created By", "description": "The ID of the user that created the resource.", "examples": [ "12345" ] }, "id": { "type": "string", "format": "uuid", "title": "Id", "description": "The unique id of the span.", "examples": [ "123e4567-e89b-12d3-a456-426614174000" ] }, "parent_id": { "anyOf": [ { "type": "string", "format": "uuid" }, { "type": "null" } ], "title": "Parent Id", "description": "The unique id of the parent element of the span.", "examples": [ "123e4567-e89b-12d3-a456-426614174000" ] }, "name": { "type": "string", "title": "Name", "description": "The name of the span.", "examples": [ "Span" ] }, "start_time": { "type": "string", "format": "date-time", "title": "Start Time", "description": "The timestamp when this span was started.", "examples": [ "2024-01-01 12:34:56.123456+00:00" ] }, "end_time": { "anyOf": [ { "type": "string", "format": "date-time" }, { "type": "null" } ], "title": "End Time", "description": "The timestamp when this span was ended.", "examples": [ "2024-01-01 12:34:56.123456+00:00" ] }, "status": { "$ref": "#/components/schemas/SpanStatus", "description": "The status code of the span.", "examples": [ "OK" ] }, "type": { "$ref": "#/components/schemas/SpanType", "description": "The type of the span.", "examples": [ "TaskSpan" ] }, "attributes": { "additionalProperties": { "$ref": "#/components/schemas/JsonSerializable-Input" }, "type": "object", "title": "Attributes", "description": "The `Attributes` of the `Span`.", "examples": [ { "key": "value" } ] } }, "type": "object", "required": [ "created_at", "created_by", "id", "parent_id", "name", "start_time", "end_time", "status", "type", "attributes" ], "title": "SpanResponse" }, "SpanResponse-Output": { "properties": { "created_at": { "type": "string", "format": "date-time", "title": "Created At", "description": "The timestamp when this resource was created in UTC." }, "created_by": { "type": "string", "title": "Created By", "description": "The ID of the user that created the resource.", "examples": [ "12345" ] }, "id": { "type": "string", "format": "uuid", "title": "Id", "description": "The unique id of the span.", "examples": [ "123e4567-e89b-12d3-a456-426614174000" ] }, "parent_id": { "anyOf": [ { "type": "string", "format": "uuid" }, { "type": "null" } ], "title": "Parent Id", "description": "The unique id of the parent element of the span.", "examples": [ "123e4567-e89b-12d3-a456-426614174000" ] }, "name": { "type": "string", "title": "Name", "description": "The name of the span.", "examples": [ "Span" ] }, "start_time": { "type": "string", "format": "date-time", "title": "Start Time", "description": "The timestamp when this span was started.", "examples": [ "2024-01-01 12:34:56.123456+00:00" ] }, "end_time": { "anyOf": [ { "type": "string", "format": "date-time" }, { "type": "null" } ], "title": "End Time", "description": "The timestamp when this span was ended.", "examples": [ "2024-01-01 12:34:56.123456+00:00" ] }, "status": { "$ref": "#/components/schemas/SpanStatus", "description": "The status code of the span.", "examples": [ "OK" ] }, "type": { "$ref": "#/components/schemas/SpanType", "description": "The type of the span.", "examples": [ "TaskSpan" ] }, "attributes": { "additionalProperties": { "$ref": "#/components/schemas/JsonSerializable-Output" }, "type": "object", "title": "Attributes", "description": "The `Attributes` of the `Span`.", "examples": [ { "key": "value" } ] } }, "type": "object", "required": [ "created_at", "created_by", "id", "parent_id", "name", "start_time", "end_time", "status", "type", "attributes" ], "title": "SpanResponse" }, "SpanStatus": { "type": "string", "enum": [ "ERROR", "OK", "ACTIVE" ], "title": "SpanStatus" }, "SpanType": { "type": "string", "enum": [ "SPAN", "TASK_SPAN" ], "title": "SpanType" }, "TaskSpanAttributes": { "properties": { "type": { "type": "string", "const": "TASK_SPAN", "title": "Type", "default": "TASK_SPAN" }, "input": { "$ref": "#/components/schemas/JsonSerializable-Input" }, "output": { "$ref": "#/components/schemas/JsonSerializable-Input" } }, "type": "object", "required": [ "input", "output" ], "title": "TaskSpanAttributes" }, "TraceEvent": { "properties": { "id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Id" }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Name" }, "body": { "anyOf": [ { "$ref": "#/components/schemas/JsonSerializable-Output" }, { "type": "null" } ] }, "message": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Message" }, "timestamp": { "type": "string", "format": "date-time", "title": "Timestamp" }, "created_at": { "type": "string", "title": "Created At" }, "created_by": { "type": "string", "title": "Created By" } }, "type": "object", "required": [ "timestamp", "created_at", "created_by" ], "title": "TraceEvent", "description": "Studio Event model." }, "TraceListResponse": { "properties": { "traces": { "items": { "$ref": "#/components/schemas/TraceSummary" }, "type": "array", "title": "Traces" }, "total": { "type": "integer", "title": "Total" }, "page": { "type": "integer", "title": "Page" }, "size": { "type": "integer", "title": "Size" }, "num_pages": { "type": "integer", "title": "Num Pages" } }, "type": "object", "required": [ "traces", "total", "page", "size", "num_pages" ], "title": "TraceListResponse", "description": "Response model for trace listing." }, "TraceResponse": { "properties": { "created_at": { "type": "string", "format": "date-time", "title": "Created At", "description": "The timestamp when this resource was created in UTC." }, "created_by": { "type": "string", "title": "Created By", "description": "The ID of the user that created the resource.", "examples": [ "12345" ] }, "id": { "type": "string", "format": "uuid", "title": "Id", "description": "A unique id for the opened `Trace`.", "examples": [ "123e4567-e89b-12d3-a456-426614174000" ] }, "spans": { "items": { "$ref": "#/components/schemas/SpanResponse-Output" }, "type": "array", "title": "Spans" } }, "type": "object", "required": [ "created_at", "created_by", "id", "spans" ], "title": "TraceResponse", "description": "A trace represents a single tree of spans." }, "TraceResponseV2": { "properties": { "id": { "type": "string", "title": "Id" }, "spans": { "items": { "$ref": "#/components/schemas/TraceSpan" }, "type": "array", "title": "Spans" } }, "type": "object", "required": [ "id", "spans" ], "title": "TraceResponseV2", "description": "Studio Trace model for detailed view." }, "TraceSpan": { "properties": { "id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Id" }, "parent_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Parent Id" }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Name" }, "start_time": { "type": "string", "format": "date-time", "title": "Start Time" }, "end_time": { "type": "string", "format": "date-time", "title": "End Time" }, "status": { "$ref": "#/components/schemas/SpanStatus" }, "type": { "$ref": "#/components/schemas/SpanType" }, "attributes": { "additionalProperties": true, "type": "object", "title": "Attributes" }, "created_at": { "type": "string", "title": "Created At" }, "created_by": { "type": "string", "title": "Created By" }, "events": { "anyOf": [ { "items": { "$ref": "#/components/schemas/TraceEvent" }, "type": "array" }, { "type": "null" } ], "title": "Events" } }, "type": "object", "required": [ "start_time", "end_time", "status", "type", "attributes", "created_at", "created_by" ], "title": "TraceSpan", "description": "Studio Span model." }, "TraceStatus": { "type": "string", "enum": [ "ERROR", "OK", "ACTIVE" ], "title": "TraceStatus", "description": "Status of a trace." }, "TraceSummary": { "properties": { "trace_id": { "type": "string", "title": "Trace Id" }, "root_service_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Root Service Name" }, "root_trace_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Root Trace Name" }, "start_time": { "type": "string", "format": "date-time", "title": "Start Time" }, "duration_ms": { "type": "integer", "title": "Duration Ms" } }, "type": "object", "required": [ "trace_id", "start_time", "duration_ms" ], "title": "TraceSummary", "description": "Summary information for a trace." }, "UpdateProjectUserlistRequest": { "properties": { "upsert": { "items": { "$ref": "#/components/schemas/UpsertOrRemoveProjectUser" }, "type": "array", "title": "Upsert", "description": "The list of users to be added to the project and their roles" }, "remove": { "items": { "$ref": "#/components/schemas/UpsertOrRemoveProjectUser" }, "type": "array", "title": "Remove", "description": "The list of users to be removed from the project" } }, "type": "object", "required": [ "upsert", "remove" ], "title": "UpdateProjectUserlistRequest", "description": "Request model for updating project user list." }, "UpsertOrRemoveProjectUser": { "properties": { "id": { "type": "string", "title": "Id", "description": "The unique identifier of the user", "examples": [ "12345" ] }, "role": { "type": "string", "enum": [ "owner", "member" ], "title": "Role", "description": "The role of the user", "examples": [ "owner", "member" ] } }, "type": "object", "required": [ "id", "role" ], "title": "UpsertOrRemoveProjectUser", "description": "Model for adding or removing users from a project." }, "User": { "properties": { "type": { "type": "string", "const": "user", "title": "Type", "default": "user" }, "id": { "type": "string", "title": "Id", "description": "The unique identifier of the user", "examples": [ "123456789" ] }, "user_name": { "type": "string", "title": "User Name", "description": "The username of the user", "examples": [ "Chuck Norris" ] }, "created_at": { "type": "string", "format": "date-time", "title": "Created At", "description": "The UTC timestamp of the time the user has been created in ISO 8601 format.", "examples": [ "2024-08-08T15:09:00Z" ] }, "updated_at": { "type": "string", "format": "date-time", "title": "Updated At", "description": "The UTC timestamp of the time the user has been updated in ISO 8601 format.", "examples": [ "2024-08-08T15:09:00Z" ] }, "email": { "type": "string", "title": "Email", "description": "The email of the user", "examples": [ "chuck@norris.com" ] }, "profile": { "$ref": "#/components/schemas/UserProfile", "description": "The profile of the user" } }, "type": "object", "required": [ "id", "user_name", "created_at", "updated_at", "email", "profile" ], "title": "User" }, "UserProfile": { "properties": { "given_name": { "type": "string", "title": "Given Name", "description": "The given name of the profile", "examples": [ "Chuck" ] }, "last_name": { "type": "string", "title": "Last Name", "description": "The last name of the profile", "examples": [ "Norris" ] } }, "type": "object", "required": [ "given_name", "last_name" ], "title": "UserProfile" }, "ValidationError": { "properties": { "loc": { "items": { "anyOf": [ { "type": "string" }, { "type": "integer" } ] }, "type": "array", "title": "Location" }, "msg": { "type": "string", "title": "Message" }, "type": { "type": "string", "title": "Error Type" }, "input": { "title": "Input" }, "ctx": { "type": "object", "title": "Context" } }, "type": "object", "required": [ "loc", "msg", "type" ], "title": "ValidationError" }, "WPUserRole": { "type": "string", "enum": [ "owner", "member" ] } }, "securitySchemes": { "BearerAuth": { "type": "http", "scheme": "bearer", "bearerFormat": "JWT" } } }, "tags": [ { "name": "Projects", "description": "A Project is an abstraction representing a group of running operations associated with the same task in the Studio. Endpoints under this tag allow you to create, update, delete, and retrieve projects. Projects help in organizing and managing tasks efficiently." }, { "name": "Traces", "description": "Trace management endpoints. A Trace provides a mechanism similar to OpenTelemetry for tracking the execution flow of a task. It allows you to monitor and analyze the sequence of operations, their timings, and other related metrics. Tracing is essential for performance monitoring and debugging." }, { "name": "Spans", "description": "Endpoints for managing Spans. A Span is a single unit of work within a Trace and represents an individual operation in the execution sequence. Spans help in breaking down the Trace into more manageable parts, providing detailed insights into each operation's performance and behavior." }, { "name": "Events", "description": "Endpoints for handling Events. Events capture detailed information about specific occurrences within a Span. This includes logs, exceptions, and other relevant data points that provide a granular view of the operations within a Span. Events are crucial for diagnosing issues and understanding the execution context." }, { "name": "Evaluation Datasets", "description": "Endpoints for handling evaluation datasets, which are specific datasets to be used for evaluation in the Pharia Studio SDK." }, { "name": "Models", "description": "List of models available for use in the Studio. Models are pre-trained language models that can be used to generate text based on input prompts. The Studio provides a variety of models for different use cases, such as text generation, summarization, and translation." }, { "name": "Benchmarks", "description": "Endpoints for handling benchmarks. A Benchmark provides a way to compare and evaluate the quality of your task results for a specific dataset." }, { "name": "Benchmark Executions", "description": "Endpoints for handling execution instances of benchmarks. These provides an overview of all evaluation steps during execution (run, evaluation, aggregation)." }, { "name": "Lineages", "description": "Lineages represent the results of a benchmark execution. A single lineage represents an evaluation of a single example with a task and evaluation logic. They contain all task and evaluation logic input and outputs." }, { "name": "Workspaces", "description": "A Workspace is a shared environment where users can collaborate on projects. Workspaces help in organizing and managing projects in large organizations efficiently." } ], "security": [ { "BearerAuth": [] } ], "servers": [ { "url": "{host}/v1/studio", "variables": { "host": { "default": "https://api.pharia.example.com" } } } ] }