openapi: 3.0.0
info:
title: Traceloop API
version: 1.0.0
contact: {}
paths:
/v2/auto-monitor-setups:
get:
description: List all auto monitor setups for the organization with optional
filters
parameters:
- description: Filter by status (e.g., pending, completed)
in: query
name: status
schema:
type: string
- description: Filter by external ID
in: query
name: external_id
schema:
type: string
responses:
'200':
description: OK
content:
application/json:
schema:
items:
$ref: '#/components/schemas/response.AutoMonitorSetupResponse'
type: array
'500':
description: Internal error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
summary: List auto monitor setups
tags:
- auto-monitor-setups
post:
description: 'Create a new auto monitor setup for automatic monitor creation.
The `selector` field is a map of span attribute key-value pairs used to filter
which spans
this monitor applies to. Keys are span attribute names and values can be strings,
numbers, or booleans.
Common selector keys:
- `gen_ai.system` — LLM provider (e.g. "openai", "anthropic")
- `gen_ai.request.model` — model name (e.g. "gpt-4o", "claude-3-5-sonnet")
- `gen_ai.request.max_tokens` — max tokens (number)
- Any other span attribute from your traces'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.CreateAutoMonitorSetupInput'
description: Auto monitor setup configuration
required: true
responses:
'201':
description: Created
content:
application/json:
schema:
$ref: '#/components/schemas/response.AutoMonitorSetupResponse'
'400':
description: Invalid input
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
summary: Create an auto monitor setup
tags:
- auto-monitor-setups
/v2/auto-monitor-setups/{external_id}:
delete:
description: Delete an auto monitor setup by its external ID
parameters:
- description: Auto monitor setup external ID
in: path
name: external_id
required: true
schema:
type: string
responses:
'204':
description: No content
'404':
description: Not found
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
summary: Delete an auto monitor setup by external ID
tags:
- auto-monitor-setups
get:
description: Get a specific auto monitor setup by its external ID
parameters:
- description: Auto monitor setup external ID
in: path
name: external_id
required: true
schema:
type: string
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.AutoMonitorSetupResponse'
'404':
description: Not found
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
summary: Get an auto monitor setup by external ID
tags:
- auto-monitor-setups
put:
description: Update an existing auto monitor setup by its external ID
parameters:
- description: Auto monitor setup external ID
in: path
name: external_id
required: true
schema:
type: string
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.UpdateAutoMonitorSetupInput'
description: Fields to update
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.AutoMonitorSetupResponse'
'400':
description: Invalid input
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'404':
description: Not found
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
summary: Update an auto monitor setup by external ID
tags:
- auto-monitor-setups
/v2/evaluators:
post:
description: Create a new custom evaluator at the organization level without
project or environment bindings. The evaluator uses LLM-as-a-judge to evaluate
inputs based on the provided configuration. The response format is automatically
generated from the output schema.
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.CreateCustomEvaluatorInput'
description: Custom evaluator creation input
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.CreateEvaluatorResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
summary: Create a custom LLM-as-a-judge evaluator
tags:
- evaluators
/v2/evaluators/agent-efficiency/execute:
post:
description: 'Evaluate agent efficiency - detect redundant calls, unnecessary
follow-ups
**Request Body:**
- `input.trajectory_prompts` (string, required): JSON array of prompts in
the agent trajectory
- `input.trajectory_completions` (string, required): JSON array of completions
in the agent trajectory'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.AgentEfficiencyRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.AgentEfficiencyResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
security:
- BearerAuth: []
summary: Execute agent-efficiency evaluator
tags:
- evaluators
/v2/evaluators/agent-flow-quality/execute:
post:
description: 'Validate agent trajectory against user-defined conditions
**Request Body:**
- `input.trajectory_prompts` (string, required): JSON array of prompts in
the agent trajectory
- `input.trajectory_completions` (string, required): JSON array of completions
in the agent trajectory
- `config.conditions` (array of strings, required): Array of evaluation conditions/rules
to validate against
- `config.threshold` (number, required): Score threshold for pass/fail determination
(0.0-1.0)'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.AgentFlowQualityRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.AgentFlowQualityResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
security:
- BearerAuth: []
summary: Execute agent-flow-quality evaluator
tags:
- evaluators
/v2/evaluators/agent-goal-accuracy/execute:
post:
description: 'Evaluate agent goal accuracy
**Request Body:**
- `input.question` (string, required): The original question or goal
- `input.completion` (string, required): The agent''s completion/response
- `input.reference` (string, required): The expected reference answer'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.AgentGoalAccuracyRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.AgentGoalAccuracyResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
security:
- BearerAuth: []
summary: Execute agent-goal-accuracy evaluator
tags:
- evaluators
/v2/evaluators/agent-goal-completeness/execute:
post:
description: 'Measure if agent accomplished all user goals
**Request Body:**
- `input.trajectory_prompts` (string, required): JSON array of prompts in
the agent trajectory
- `input.trajectory_completions` (string, required): JSON array of completions
in the agent trajectory
- `config.threshold` (number, required): Score threshold for pass/fail determination
(0.0-1.0)'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.AgentGoalCompletenessRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.AgentGoalCompletenessResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
security:
- BearerAuth: []
summary: Execute agent-goal-completeness evaluator
tags:
- evaluators
/v2/evaluators/agent-tool-error-detector/execute:
post:
description: 'Detect errors or failures during tool execution
**Request Body:**
- `input.tool_input` (string, required): JSON string of the tool input
- `input.tool_output` (string, required): JSON string of the tool output'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.AgentToolErrorDetectorRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.AgentToolErrorDetectorResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
security:
- BearerAuth: []
summary: Execute agent-tool-error-detector evaluator
tags:
- evaluators
/v2/evaluators/agent-tool-trajectory/execute:
post:
description: 'Compare actual tool calls against expected reference tool calls
**Request Body:**
- `input.executed_tool_calls` (string, required): JSON array of actual tool
calls made by the agent
- `input.expected_tool_calls` (string, required): JSON array of expected/reference
tool calls
- `config.threshold` (float, optional): Score threshold for pass/fail determination
(default: 0.5)
- `config.mismatch_sensitive` (bool, optional): Whether tool calls must match
exactly (default: false)
- `config.order_sensitive` (bool, optional): Whether order of tool calls matters
(default: false)
- `config.input_params_sensitive` (bool, optional): Whether to compare input
parameters (default: true)'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.AgentToolTrajectoryRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.AgentToolTrajectoryResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
security:
- BearerAuth: []
summary: Execute agent-tool-trajectory evaluator
tags:
- evaluators
/v2/evaluators/answer-completeness/execute:
post:
description: 'Evaluate whether the answer is complete and contains all the necessary
information
**Request Body:**
- `input.question` (string, required): The original question
- `input.completion` (string, required): The completion to evaluate for completeness
- `input.context` (string, required): The context that provides the complete
information'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.AnswerCompletenessRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.AnswerCompletenessResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
security:
- BearerAuth: []
summary: Execute answer-completeness evaluator
tags:
- evaluators
/v2/evaluators/answer-correctness/execute:
post:
description: 'Evaluate factual accuracy by comparing answers against ground
truth
**Request Body:**
- `input.question` (string, required): The original question
- `input.completion` (string, required): The completion to evaluate
- `input.ground_truth` (string, required): The expected correct answer'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.AnswerCorrectnessRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.AnswerCorrectnessResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
security:
- BearerAuth: []
summary: Execute answer-correctness evaluator
tags:
- evaluators
/v2/evaluators/answer-relevancy/execute:
post:
description: 'Check if an answer is relevant to a question
**Request Body:**
- `input.answer` (string, required): The answer to evaluate for relevancy
- `input.question` (string, required): The question that the answer should
be relevant to'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.AnswerRelevancyRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.AnswerRelevancyResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
security:
- BearerAuth: []
summary: Execute answer-relevancy evaluator
tags:
- evaluators
/v2/evaluators/char-count/execute:
post:
description: 'Count the number of characters in text
**Request Body:**
- `input.text` (string, required): The text to count characters in'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.CharCountRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.CharCountResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
security:
- BearerAuth: []
summary: Execute char-count evaluator
tags:
- evaluators
/v2/evaluators/char-count-ratio/execute:
post:
description: 'Calculate the ratio of characters between two texts
**Request Body:**
- `input.numerator_text` (string, required): The numerator text (will be divided
by denominator)
- `input.denominator_text` (string, required): The denominator text (divides
the numerator)'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.CharCountRatioRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.CharCountRatioResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
security:
- BearerAuth: []
summary: Execute char-count-ratio evaluator
tags:
- evaluators
/v2/evaluators/context-relevance/execute:
post:
description: 'Evaluate whether retrieved context contains sufficient information
to answer the query
**Request Body:**
- `input.query` (string, required): The query/question to evaluate context
relevance for
- `input.completion` (string, required): The completion to evaluate for context
relevance
- `input.context` (string, required): The context to evaluate for relevance
to the query
- `config.model` (string, optional): Model to use for evaluation (default:
gpt-4o)'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.ContextRelevanceRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.ContextRelevanceResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
security:
- BearerAuth: []
summary: Execute context-relevance evaluator
tags:
- evaluators
/v2/evaluators/conversation-quality/execute:
post:
description: 'Evaluate conversation quality based on tone, clarity, flow, responsiveness,
and transparency
**Request Body:**
- `input.prompts` (string, required): JSON array of prompts in the conversation
- `input.completions` (string, required): JSON array of completions in the
conversation'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.ConversationQualityRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.ConversationQualityResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
security:
- BearerAuth: []
summary: Execute conversation-quality evaluator
tags:
- evaluators
/v2/evaluators/faithfulness/execute:
post:
description: 'Check if a completion is faithful to the provided context
**Request Body:**
- `input.completion` (string, required): The LLM completion to check for faithfulness
- `input.context` (string, required): The context that the completion should
be faithful to
- `input.question` (string, required): The original question asked'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.FaithfulnessRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.FaithfulnessResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
security:
- BearerAuth: []
summary: Execute faithfulness evaluator
tags:
- evaluators
/v2/evaluators/html-comparison/execute:
post:
description: 'Compare two HTML documents for structural and content similarity
**Request Body:**
- `input.html1` (string, required): The first HTML document to compare
- `input.html2` (string, required): The second HTML document to compare'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.HtmlComparisonRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.HtmlComparisonResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
security:
- BearerAuth: []
summary: Execute html-comparison evaluator
tags:
- evaluators
/v2/evaluators/instruction-adherence/execute:
post:
description: 'Evaluate how well responses follow given instructions
**Request Body:**
- `input.instructions` (string, required): The instructions that should be
followed
- `input.response` (string, required): The response to evaluate for instruction
adherence'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.InstructionAdherenceRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.InstructionAdherenceResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
security:
- BearerAuth: []
summary: Execute instruction-adherence evaluator
tags:
- evaluators
/v2/evaluators/intent-change/execute:
post:
description: 'Detect changes in user intent between prompts and completions
**Request Body:**
- `input.prompts` (string, required): JSON array of prompts in the conversation
- `input.completions` (string, required): JSON array of completions in the
conversation'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.IntentChangeRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.IntentChangeResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
security:
- BearerAuth: []
summary: Execute intent-change evaluator
tags:
- evaluators
/v2/evaluators/json-validator/execute:
post:
description: 'Validate JSON syntax
**Request Body:**
- `input.text` (string, required): The text to validate as JSON
- `config.enable_schema_validation` (bool, optional): Enable JSON schema validation
- `config.schema_string` (string, optional): JSON schema to validate against'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.JSONValidatorRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.JSONValidatorResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
security:
- BearerAuth: []
summary: Execute json-validator evaluator
tags:
- evaluators
/v2/evaluators/perplexity/execute:
post:
description: 'Measure text perplexity from logprobs
**Request Body:**
- `input.logprobs` (string, required): JSON array of log probabilities from
the model'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.PerplexityRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.PerplexityResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
security:
- BearerAuth: []
summary: Execute perplexity evaluator
tags:
- evaluators
/v2/evaluators/pii-detector/execute:
post:
description: 'Detect personally identifiable information in text
**Request Body:**
- `input.text` (string, required): The text to scan for personally identifiable
information
- `config.probability_threshold` (float, optional): Detection threshold (default:
0.8)'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.PIIDetectorRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.PIIDetectorResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
security:
- BearerAuth: []
summary: Execute pii-detector evaluator
tags:
- evaluators
/v2/evaluators/placeholder-regex/execute:
post:
description: 'Validate text against a placeholder regex pattern
**Request Body:**
- `input.placeholder_name` (string, required): The name of the placeholder
to substitute
- `input.placeholder_value` (string, required): The value to substitute into
the regex placeholder
- `input.text` (string, required): The text to validate against the regex
pattern
- `config.should_match` (bool, optional): Whether the text should match the
regex
- `config.case_sensitive` (bool, optional): Case-sensitive matching
- `config.dot_include_nl` (bool, optional): Dot matches newlines
- `config.multi_line` (bool, optional): Multi-line mode'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.PlaceholderRegexRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.PlaceholderRegexResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
security:
- BearerAuth: []
summary: Execute placeholder-regex evaluator
tags:
- evaluators
/v2/evaluators/profanity-detector/execute:
post:
description: 'Detect profanity in text
**Request Body:**
- `input.text` (string, required): The text to scan for profanity'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.ProfanityDetectorRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.ProfanityDetectorResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
security:
- BearerAuth: []
summary: Execute profanity-detector evaluator
tags:
- evaluators
/v2/evaluators/prompt-injection/execute:
post:
description: 'Detect prompt injection attempts
**Request Body:**
- `input.prompt` (string, required): The prompt to check for injection attempts
- `config.threshold` (float, optional): Detection threshold (default: 0.5)'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.PromptInjectionRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.PromptInjectionResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
security:
- BearerAuth: []
summary: Execute prompt-injection evaluator
tags:
- evaluators
/v2/evaluators/prompt-perplexity/execute:
post:
description: 'Measure prompt perplexity to detect potential injection attempts
**Request Body:**
- `input.prompt` (string, required): The prompt to calculate perplexity for'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.PromptPerplexityRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.PromptPerplexityResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
security:
- BearerAuth: []
summary: Execute prompt-perplexity evaluator
tags:
- evaluators
/v2/evaluators/regex-validator/execute:
post:
description: 'Validate text against a regex pattern
**Request Body:**
- `input.text` (string, required): The text to validate against a regex pattern
- `config.regex` (string, optional): The regex pattern to match against
- `config.should_match` (bool, optional): Whether the text should match the
regex
- `config.case_sensitive` (bool, optional): Case-sensitive matching
- `config.dot_include_nl` (bool, optional): Dot matches newlines
- `config.multi_line` (bool, optional): Multi-line mode'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.RegexValidatorRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.RegexValidatorResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
security:
- BearerAuth: []
summary: Execute regex-validator evaluator
tags:
- evaluators
/v2/evaluators/secrets-detector/execute:
post:
description: 'Detect secrets and credentials in text
**Request Body:**
- `input.text` (string, required): The text to scan for secrets (API keys,
passwords, etc.)'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.SecretsDetectorRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.SecretsDetectorResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
security:
- BearerAuth: []
summary: Execute secrets-detector evaluator
tags:
- evaluators
/v2/evaluators/semantic-similarity/execute:
post:
description: 'Calculate semantic similarity between completion and reference
**Request Body:**
- `input.completion` (string, required): The completion text to compare
- `input.reference` (string, required): The reference text to compare against'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.SemanticSimilarityRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.SemanticSimilarityResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
security:
- BearerAuth: []
summary: Execute semantic-similarity evaluator
tags:
- evaluators
/v2/evaluators/sexism-detector/execute:
post:
description: 'Detect sexist language and bias
**Request Body:**
- `input.text` (string, required): The text to scan for sexist content
- `config.threshold` (float, optional): Detection threshold (default: 0.5)'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.SexismDetectorRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.SexismDetectorResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
security:
- BearerAuth: []
summary: Execute sexism-detector evaluator
tags:
- evaluators
/v2/evaluators/sql-validator/execute:
post:
description: 'Validate SQL query syntax
**Request Body:**
- `input.text` (string, required): The text to validate as SQL'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.SQLValidatorRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.SQLValidatorResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
security:
- BearerAuth: []
summary: Execute sql-validator evaluator
tags:
- evaluators
/v2/evaluators/tone-detection/execute:
post:
description: 'Detect the tone of the text
**Request Body:**
- `input.text` (string, required): The text to detect the tone of'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.ToneDetectionRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.ToneDetectionResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
security:
- BearerAuth: []
summary: Execute tone-detection evaluator
tags:
- evaluators
/v2/evaluators/topic-adherence/execute:
post:
description: 'Evaluate topic adherence
**Request Body:**
- `input.question` (string, required): The original question
- `input.completion` (string, required): The completion to evaluate
- `input.reference_topics` (string, required): Comma-separated list of expected
topics'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.TopicAdherenceRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.TopicAdherenceResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
security:
- BearerAuth: []
summary: Execute topic-adherence evaluator
tags:
- evaluators
/v2/evaluators/toxicity-detector/execute:
post:
description: 'Detect toxic or harmful language
**Request Body:**
- `input.text` (string, required): The text to scan for toxic content
- `config.threshold` (float, optional): Detection threshold (default: 0.5)'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.ToxicityDetectorRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.ToxicityDetectorResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
security:
- BearerAuth: []
summary: Execute toxicity-detector evaluator
tags:
- evaluators
/v2/evaluators/uncertainty-detector/execute:
post:
description: 'Detect uncertainty in the text
**Request Body:**
- `input.prompt` (string, required): The text to detect uncertainty in'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.UncertaintyDetectorRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.UncertaintyDetectorResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
security:
- BearerAuth: []
summary: Execute uncertainty-detector evaluator
tags:
- evaluators
/v2/evaluators/word-count/execute:
post:
description: 'Count the number of words in text
**Request Body:**
- `input.text` (string, required): The text to count words in'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.WordCountRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.WordCountResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
security:
- BearerAuth: []
summary: Execute word-count evaluator
tags:
- evaluators
/v2/evaluators/word-count-ratio/execute:
post:
description: 'Calculate the ratio of words between two texts
**Request Body:**
- `input.numerator_text` (string, required): The numerator text (will be divided
by denominator)
- `input.denominator_text` (string, required): The denominator text (divides
the numerator)'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.WordCountRatioRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.WordCountRatioResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
security:
- BearerAuth: []
summary: Execute word-count-ratio evaluator
tags:
- evaluators
/v2/metrics:
post:
description: Retrieves metrics data with support for filtering, sorting, and
pagination. Metrics are grouped by metric name with individual data points.
Supports filtering by direct column fields (bool_value, trace_id, etc.), label
fields (labels.agent_name, labels.trace_id), and attribute fields (attributes.*).
operationId: get-metrics
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.GetMetricsRequest'
description: Metrics query parameters including filters, environments, and
pagination
required: true
responses:
'200':
description: Grouped metrics with data points
content:
application/json:
schema:
$ref: '#/components/schemas/response.GetMetricsResponse'
'400':
description: Invalid request parameters
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'404':
description: Not Found
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal server error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
summary: Get metrics with filtering and grouping
tags:
- metrics
/v2/metrics_hwm:
get:
description: Returns the timestamp of the last successfully processed evaluation
(high water mark)
operationId: get-metrics-hwm
responses:
'200':
description: High water mark timestamp in milliseconds
content:
application/json:
schema:
$ref: '#/components/schemas/response.MetricsHWMResponse'
'500':
description: Internal server error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
summary: Get metrics high water mark
tags:
- metrics
/v2/organizations:
post:
description: Create a new organization with environments and API keys.
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.CreateOrganizationRequest'
description: Organization creation request
required: true
responses:
'201':
description: Created
content:
application/json:
schema:
$ref: '#/components/schemas/response.CreateOrganizationResponse'
'400':
description: Invalid request body or validation error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'403':
description: Not allowed to create organizations
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal server error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
summary: Create a new organization
tags:
- organizations
servers:
- url: https://api.traceloop.com
components:
securitySchemes:
BearerAuth:
description: Type "Bearer" followed by a space and JWT token.
in: header
name: Authorization
type: apiKey
schemas:
evaluator.Property:
properties:
description:
type: string
enum_values:
description: The allowed values for this property. Only applicable when
`type` is `enum`.
items:
type: string
type: array
label:
type: string
name:
type: string
type:
$ref: '#/components/schemas/evaluator.PropertyType'
required:
- name
- type
type: object
evaluator.PropertyType:
enum:
- string
- int
- float
- boolean
- enum
type: string
x-enum-varnames:
- StringType
- IntType
- FloatType
- BooleanType
- EnumType
model.LLMMessage:
properties:
content:
type: string
role:
type: string
tool_call_id:
type: string
type: object
request.AgentEfficiencyInput:
properties:
trajectory_completions:
example: '["User found", "Email updated", "Changes saved"]'
type: string
trajectory_prompts:
example: '["Find user info", "Update email", "Save changes"]'
type: string
required:
- trajectory_completions
- trajectory_prompts
type: object
request.AgentEfficiencyRequest:
properties:
input:
$ref: '#/components/schemas/request.AgentEfficiencyInput'
required:
- input
type: object
request.AgentFlowQualityConfigRequest:
properties:
conditions:
example:
- no tools called
- agent completed task
items:
type: string
type: array
threshold:
example: 0.5
type: number
required:
- conditions
- threshold
type: object
request.AgentFlowQualityInput:
properties:
trajectory_completions:
example: '["Found 5 flights", "Selected $299 flight", "Booking confirmed"]'
type: string
trajectory_prompts:
example: '["Search for flights", "Select the cheapest option", "Confirm
booking"]'
type: string
required:
- trajectory_completions
- trajectory_prompts
type: object
request.AgentFlowQualityRequest:
properties:
config:
$ref: '#/components/schemas/request.AgentFlowQualityConfigRequest'
input:
$ref: '#/components/schemas/request.AgentFlowQualityInput'
required:
- config
- input
type: object
request.AgentGoalAccuracyInput:
properties:
completion:
example: I have booked your flight from New York to Los Angeles departing
Monday at 9am.
type: string
question:
example: Book a flight from NYC to LA for next Monday
type: string
reference:
example: 'Flight booked: NYC to LA, Monday departure'
type: string
required:
- completion
- question
- reference
type: object
request.AgentGoalAccuracyRequest:
properties:
input:
$ref: '#/components/schemas/request.AgentGoalAccuracyInput'
required:
- input
type: object
request.AgentGoalCompletenessConfigRequest:
properties:
threshold:
example: 0.5
type: number
required:
- threshold
type: object
request.AgentGoalCompletenessInput:
properties:
trajectory_completions:
example: '["Account created", "Preferences saved", "Notifications enabled"]'
type: string
trajectory_prompts:
example: '["Create new account", "Set preferences", "Enable notifications"]'
type: string
required:
- trajectory_completions
- trajectory_prompts
type: object
request.AgentGoalCompletenessRequest:
properties:
config:
$ref: '#/components/schemas/request.AgentGoalCompletenessConfigRequest'
input:
$ref: '#/components/schemas/request.AgentGoalCompletenessInput'
required:
- input
type: object
request.AgentToolErrorDetectorInput:
properties:
tool_input:
example: '{"action": "search", "query": "flights to Paris"}'
type: string
tool_output:
example: '{"status": "success", "results": [{"flight": "AF123", "price":
450}]}'
type: string
required:
- tool_input
- tool_output
type: object
request.AgentToolErrorDetectorRequest:
properties:
input:
$ref: '#/components/schemas/request.AgentToolErrorDetectorInput'
required:
- input
type: object
request.AgentToolTrajectoryConfigRequest:
properties:
input_params_sensitive:
example: true
type: boolean
mismatch_sensitive:
example: false
type: boolean
order_sensitive:
example: false
type: boolean
threshold:
example: 0.5
type: number
type: object
request.AgentToolTrajectoryInput:
properties:
executed_tool_calls:
example: '[{"name": "search", "input": {"query": "weather"}}]'
type: string
expected_tool_calls:
example: '[{"name": "search", "input": {"query": "weather"}}]'
type: string
required:
- executed_tool_calls
- expected_tool_calls
type: object
request.AgentToolTrajectoryRequest:
properties:
config:
$ref: '#/components/schemas/request.AgentToolTrajectoryConfigRequest'
input:
$ref: '#/components/schemas/request.AgentToolTrajectoryInput'
required:
- input
type: object
request.AnswerCompletenessInput:
properties:
completion:
example: Paris.
type: string
context:
example: The capital of France is Paris.
type: string
question:
example: What is the capital of France?
type: string
required:
- completion
- context
- question
type: object
request.AnswerCompletenessRequest:
properties:
input:
$ref: '#/components/schemas/request.AnswerCompletenessInput'
required:
- input
type: object
request.AnswerCorrectnessInput:
properties:
completion:
example: World War II ended in 1945.
type: string
ground_truth:
example: '1945'
type: string
question:
example: What year did World War II end?
type: string
required:
- completion
- ground_truth
- question
type: object
request.AnswerCorrectnessRequest:
properties:
input:
$ref: '#/components/schemas/request.AnswerCorrectnessInput'
required:
- input
type: object
request.AnswerRelevancyInput:
properties:
answer:
example: The capital of France is Paris.
type: string
question:
example: What is the capital of France?
type: string
required:
- answer
- question
type: object
request.AnswerRelevancyRequest:
properties:
input:
$ref: '#/components/schemas/request.AnswerRelevancyInput'
required:
- input
type: object
request.CharCountInput:
properties:
text:
example: Hello, world! This is a sample text.
type: string
required:
- text
type: object
request.CharCountRatioInput:
properties:
denominator_text:
example: This is a longer text for comparison
type: string
numerator_text:
example: Short text
type: string
required:
- denominator_text
- numerator_text
type: object
request.CharCountRatioRequest:
properties:
input:
$ref: '#/components/schemas/request.CharCountRatioInput'
required:
- input
type: object
request.CharCountRequest:
properties:
input:
$ref: '#/components/schemas/request.CharCountInput'
required:
- input
type: object
request.ContextRelevanceConfigRequest:
properties:
model:
example: gpt-4o
type: string
type: object
request.ContextRelevanceInput:
properties:
completion:
example: Our store is open Monday to Friday from 9am to 6pm, and Saturday
from 10am to 4pm. We are closed on Sundays.
type: string
context:
example: Our store is open Monday to Friday from 9am to 6pm, and Saturday
from 10am to 4pm. We are closed on Sundays.
type: string
query:
example: What are the business hours?
type: string
required:
- completion
- context
- query
type: object
request.ContextRelevanceRequest:
properties:
config:
$ref: '#/components/schemas/request.ContextRelevanceConfigRequest'
input:
$ref: '#/components/schemas/request.ContextRelevanceInput'
required:
- input
type: object
request.ConversationQualityInput:
properties:
completions:
example: '["Hi! I''d be happy to assist you today.", "We offer consulting,
development, and support services."]'
type: string
prompts:
example: '["Hello, how can I help?", "What services do you offer?"]'
type: string
required:
- completions
- prompts
type: object
request.ConversationQualityRequest:
properties:
input:
$ref: '#/components/schemas/request.ConversationQualityInput'
required:
- input
type: object
request.CreateAutoMonitorSetupInput:
properties:
evaluators:
description: List of evaluator slugs to run on matched spans
example:
- hallucination
- toxicity
items:
type: string
minItems: 1
type: array
external_id:
description: Unique identifier for the auto monitor setup, used to reference
it in future requests
example: my-agent-monitor-1
type: string
selector:
description: 'Map of span attributes to filter which spans this monitor
applies to.
Keys are span attribute names (e.g. gen_ai.system, gen_ai.request.model)
and
values can be strings, numbers, or booleans.
Example: {"gen_ai.system": "openai", "gen_ai.request.model": "gpt-4o",
"gen_ai.request.max_tokens": 1000}'
type: object
required:
- evaluators
- external_id
type: object
request.CreateCustomEvaluatorInput:
properties:
description:
type: string
frequency_penalty:
type: number
input_schema:
description: Schemas
items:
$ref: '#/components/schemas/evaluator.Property'
type: array
logprobs:
type: boolean
max_tokens:
type: integer
messages:
description: LLM Message configuration
items:
$ref: '#/components/schemas/model.LLMMessage'
type: array
mode:
type: string
model:
description: LLM parameters (from BaseLLMConfig)
type: string
name:
description: Basic fields
type: string
output_schema:
items:
$ref: '#/components/schemas/evaluator.Property'
type: array
presence_penalty:
type: number
provider:
description: Provider configuration
type: string
slug:
type: string
stop:
items:
type: string
type: array
temperature:
type: number
top_p:
type: number
required:
- input_schema
- messages
- model
- name
- output_schema
- provider
type: object
request.CreateOrganizationRequest:
properties:
envs:
items:
type: string
type: array
org_name:
type: string
retention_hours:
type: integer
description: Data retention period in hours (1-720). Defaults to 720 (1
month).
minimum: 1
maximum: 720
default: 720
required:
- org_name
type: object
request.FaithfulnessInput:
properties:
completion:
example: The Eiffel Tower is located in Paris and was built in 1889.
type: string
context:
example: The Eiffel Tower is a wrought-iron lattice tower on the Champ de
Mars in Paris, France. It was constructed from 1887 to 1889.
type: string
question:
example: When was the Eiffel Tower built?
type: string
required:
- completion
- context
- question
type: object
request.FaithfulnessRequest:
properties:
input:
$ref: '#/components/schemas/request.FaithfulnessInput'
required:
- input
type: object
request.GetMetricsRequest:
properties:
from_timestamp_sec:
type: integer
description: Start time in Unix seconds timestamp.
example: 1772905218
to_timestamp_sec:
type: integer
description: End time in Unix seconds timestamp.
example: 1772905218
cursor:
type: integer
description: Pagination cursor for fetching the next page of results. The
cursor is returned in the response to the previous request.
example: 1772905218000
environments:
items:
type: string
type: array
description: List of environments to filter by
example:
- prd
- stg
- dev
filters:
items:
$ref: '#/components/schemas/shared.FilterCondition'
type: array
limit:
type: integer
description: Maximum number of metrics to return per page.
example: 50
logical_operator:
$ref: '#/components/schemas/types.LogicalOperator'
metric_name:
type: string
description: Metric name to filter by. The metric_name is a metric attribute
that is set on the metric object.
example: answer relevancy
metric_source:
type: string
description: Metric source to filter by. The metric_source is a metric attribute
that is set on the metric object.
example: llm_as_a_judge
sort_by:
type: string
description: Field to sort by.
example: timestamp
sort_order:
type: string
description: Sort order.
example: ASC
type: object
request.HtmlComparisonInput:
properties:
html1:
example: Hello, world!
type: string
html2:
example: Hello, world!
type: string
required:
- html1
- html2
type: object
request.HtmlComparisonRequest:
properties:
input:
$ref: '#/components/schemas/request.HtmlComparisonInput'
required:
- input
type: object
request.InstructionAdherenceInput:
properties:
instructions:
example: Respond in exactly 3 bullet points and use formal language.
type: string
response:
example: '- First point about the topic
- Second relevant consideration
- Final concluding thought'
type: string
required:
- instructions
- response
type: object
request.InstructionAdherenceRequest:
properties:
input:
$ref: '#/components/schemas/request.InstructionAdherenceInput'
required:
- input
type: object
request.IntentChangeInput:
properties:
completions:
example: '["Sure, I can help with hotel booking", "No problem, let me search
for flights"]'
type: string
prompts:
example: '["I want to book a hotel", "Actually, I need a flight instead"]'
type: string
required:
- completions
- prompts
type: object
request.IntentChangeRequest:
properties:
input:
$ref: '#/components/schemas/request.IntentChangeInput'
required:
- input
type: object
request.JSONValidatorConfigRequest:
properties:
enable_schema_validation:
example: true
type: boolean
schema_string:
example: '{}'
type: string
type: object
request.JSONValidatorInput:
properties:
text:
example: '{"name": "John", "age": 30}'
type: string
required:
- text
type: object
request.JSONValidatorRequest:
properties:
config:
$ref: '#/components/schemas/request.JSONValidatorConfigRequest'
input:
$ref: '#/components/schemas/request.JSONValidatorInput'
required:
- input
type: object
request.PIIDetectorConfigRequest:
properties:
probability_threshold:
example: 0.8
type: number
type: object
request.PIIDetectorInput:
properties:
text:
example: Please contact John Smith at john.smith@email.com or call 555-123-4567.
type: string
required:
- text
type: object
request.PIIDetectorRequest:
properties:
config:
$ref: '#/components/schemas/request.PIIDetectorConfigRequest'
input:
$ref: '#/components/schemas/request.PIIDetectorInput'
required:
- input
type: object
request.PerplexityInput:
properties:
logprobs:
example: '[-2.3, -1.5, -0.8, -1.2, -0.5]'
type: string
required:
- logprobs
type: object
request.PerplexityRequest:
properties:
input:
$ref: '#/components/schemas/request.PerplexityInput'
required:
- input
type: object
request.PlaceholderRegexConfigRequest:
properties:
case_sensitive:
example: true
type: boolean
dot_include_nl:
example: true
type: boolean
multi_line:
example: true
type: boolean
should_match:
example: true
type: boolean
type: object
request.PlaceholderRegexInput:
properties:
placeholder_name:
example: name
type: string
placeholder_value:
example: Hello ${name}
type: string
text:
example: Hello John Doe
type: string
required:
- placeholder_name
- placeholder_value
- text
type: object
request.PlaceholderRegexRequest:
properties:
config:
$ref: '#/components/schemas/request.PlaceholderRegexConfigRequest'
input:
$ref: '#/components/schemas/request.PlaceholderRegexInput'
required:
- input
type: object
request.ProfanityDetectorInput:
properties:
text:
example: This is a clean and professional message.
type: string
required:
- text
type: object
request.ProfanityDetectorRequest:
properties:
input:
$ref: '#/components/schemas/request.ProfanityDetectorInput'
required:
- input
type: object
request.PromptInjectionConfigRequest:
properties:
threshold:
example: 0.5
type: number
type: object
request.PromptInjectionInput:
properties:
prompt:
example: What is the weather like today?
type: string
required:
- prompt
type: object
request.PromptInjectionRequest:
properties:
config:
$ref: '#/components/schemas/request.PromptInjectionConfigRequest'
input:
$ref: '#/components/schemas/request.PromptInjectionInput'
required:
- input
type: object
request.PromptPerplexityInput:
properties:
prompt:
example: What is the capital of France?
type: string
required:
- prompt
type: object
request.PromptPerplexityRequest:
properties:
input:
$ref: '#/components/schemas/request.PromptPerplexityInput'
required:
- input
type: object
request.RegexValidatorConfigRequest:
properties:
case_sensitive:
example: true
type: boolean
dot_include_nl:
example: true
type: boolean
multi_line:
example: true
type: boolean
regex:
example: .*
type: string
should_match:
example: true
type: boolean
type: object
request.RegexValidatorInput:
properties:
text:
example: user@example.com
type: string
required:
- text
type: object
request.RegexValidatorRequest:
properties:
config:
$ref: '#/components/schemas/request.RegexValidatorConfigRequest'
input:
$ref: '#/components/schemas/request.RegexValidatorInput'
required:
- input
type: object
request.SQLValidatorInput:
properties:
text:
example: SELECT * FROM users WHERE id = 1;
type: string
required:
- text
type: object
request.SQLValidatorRequest:
properties:
input:
$ref: '#/components/schemas/request.SQLValidatorInput'
required:
- input
type: object
request.SecretsDetectorInput:
properties:
text:
example: Here is some text without any API keys or passwords.
type: string
required:
- text
type: object
request.SecretsDetectorRequest:
properties:
input:
$ref: '#/components/schemas/request.SecretsDetectorInput'
required:
- input
type: object
request.SemanticSimilarityInput:
properties:
completion:
example: The cat sat on the mat.
type: string
reference:
example: A feline was resting on the rug.
type: string
required:
- completion
- reference
type: object
request.SemanticSimilarityRequest:
properties:
input:
$ref: '#/components/schemas/request.SemanticSimilarityInput'
required:
- input
type: object
request.SexismDetectorConfigRequest:
properties:
threshold:
example: 0.5
type: number
type: object
request.SexismDetectorInput:
properties:
text:
example: All team members should be treated equally regardless of gender.
type: string
required:
- text
type: object
request.SexismDetectorRequest:
properties:
config:
$ref: '#/components/schemas/request.SexismDetectorConfigRequest'
input:
$ref: '#/components/schemas/request.SexismDetectorInput'
required:
- input
type: object
request.ToneDetectionInput:
properties:
text:
example: The capital of France is Paris.
type: string
required:
- text
type: object
request.ToneDetectionRequest:
properties:
input:
$ref: '#/components/schemas/request.ToneDetectionInput'
required:
- input
type: object
request.TopicAdherenceInput:
properties:
completion:
example: Machine learning is a subset of AI that enables systems to learn
from data.
type: string
question:
example: Tell me about machine learning
type: string
reference_topics:
example: artificial intelligence, data science, algorithms
type: string
required:
- completion
- question
- reference_topics
type: object
request.TopicAdherenceRequest:
properties:
input:
$ref: '#/components/schemas/request.TopicAdherenceInput'
required:
- input
type: object
request.ToxicityDetectorConfigRequest:
properties:
threshold:
example: 0.5
type: number
type: object
request.ToxicityDetectorInput:
properties:
text:
example: Thank you for your help with this project.
type: string
required:
- text
type: object
request.ToxicityDetectorRequest:
properties:
config:
$ref: '#/components/schemas/request.ToxicityDetectorConfigRequest'
input:
$ref: '#/components/schemas/request.ToxicityDetectorInput'
required:
- input
type: object
request.UncertaintyDetectorInput:
properties:
prompt:
example: I am not sure, I think the capital of France is Paris.
type: string
required:
- prompt
type: object
request.UncertaintyDetectorRequest:
properties:
input:
$ref: '#/components/schemas/request.UncertaintyDetectorInput'
required:
- input
type: object
request.UpdateAutoMonitorSetupInput:
properties:
evaluators:
description: List of evaluator slugs to run on matched spans
example:
- hallucination
- toxicity
items:
type: string
type: array
selector:
description: 'Map of span attributes to filter which spans this monitor
applies to.
Keys are span attribute names (e.g. gen_ai.system, gen_ai.request.model)
and
values can be strings, numbers, or booleans.
Example: {"gen_ai.system": "openai", "gen_ai.request.model": "gpt-4o",
"gen_ai.request.max_tokens": 1000}'
type: object
type: object
request.WordCountInput:
properties:
text:
example: This is a sample text with several words.
type: string
required:
- text
type: object
request.WordCountRatioInput:
properties:
denominator_text:
example: This is a longer input text for comparison
type: string
numerator_text:
example: Short response
type: string
required:
- denominator_text
- numerator_text
type: object
request.WordCountRatioRequest:
properties:
input:
$ref: '#/components/schemas/request.WordCountRatioInput'
required:
- input
type: object
request.WordCountRequest:
properties:
input:
$ref: '#/components/schemas/request.WordCountInput'
required:
- input
type: object
response.AgentEfficiencyResponse:
properties:
step_efficiency_reason:
example: Agent completed task with minimal redundant steps
type: string
step_efficiency_score:
example: 0.85
type: number
task_completion_reason:
example: All required tasks were completed successfully
type: string
task_completion_score:
example: 0.92
type: number
type: object
response.AgentFlowQualityResponse:
properties:
reason:
example: Agent followed the expected flow correctly
type: string
score:
example: 0.89
type: number
success:
example: true
type: boolean
type: object
response.AgentGoalAccuracyResponse:
properties:
accuracy_score:
example: 0.88
type: number
type: object
response.AgentGoalCompletenessResponse:
properties:
reason:
example: All user goals were accomplished
type: string
score:
example: 0.95
type: number
success:
example: true
type: boolean
type: object
response.AgentToolErrorDetectorResponse:
properties:
reason:
example: Tool executed successfully without errors
type: string
success:
example: true
type: boolean
type: object
response.AgentToolTrajectoryResponse:
properties:
reason:
example: Tool calls match the expected trajectory
type: string
score:
example: 0.85
type: number
success:
example: true
type: boolean
type: object
response.AnswerCompletenessResponse:
properties:
answer_completeness_score:
example: 0.95
type: number
type: object
response.AnswerCorrectnessResponse:
properties:
correctness_score:
example: 0.91
type: number
type: object
response.AnswerRelevancyResponse:
properties:
is_relevant:
example: true
type: boolean
type: object
response.AutoMonitorEvaluatorResponse:
properties:
binding_id:
type: string
error_message:
type: string
evaluator_id:
type: string
evaluator_type:
type: string
input_schema:
items:
$ref: '#/components/schemas/evaluator.Property'
type: array
output_schema:
items:
$ref: '#/components/schemas/evaluator.Property'
type: array
processed_at:
type: string
status:
type: string
type: object
response.AutoMonitorSetupResponse:
properties:
created_at:
type: string
env_project_id:
type: string
evaluators:
items:
$ref: '#/components/schemas/response.AutoMonitorEvaluatorResponse'
type: array
external_id:
type: string
id:
type: string
init_rules:
items:
$ref: '#/components/schemas/types.Rule'
type: array
org_id:
type: string
project_id:
type: string
status:
type: string
updated_at:
type: string
type: object
response.CharCountRatioResponse:
properties:
char_ratio:
example: 0.75
type: number
type: object
response.CharCountResponse:
properties:
char_count:
example: 42
type: integer
type: object
response.ContextRelevanceResponse:
properties:
relevance_score:
example: 0.88
type: number
type: object
response.ConversationQualityResponse:
properties:
conversation_quality_score:
example: 0.82
type: number
type: object
response.CreateEvaluatorResponse:
properties:
bindings:
items:
type: string
type: array
evaluator_id:
type: string
slug:
type: string
type:
$ref: '#/components/schemas/types.EvaluatorType'
version_id:
type: string
type: object
response.CreateOrganizationResponse:
properties:
environments:
items:
$ref: '#/components/schemas/response.EnvironmentWithKeyResponse'
type: array
org_id:
type: string
type: object
response.EnvironmentWithKeyResponse:
properties:
api_key:
type: string
slug:
type: string
type: object
response.ErrorResponse:
description: Standard error response structure
properties:
error:
example: error message
type: string
type: object
response.FaithfulnessResponse:
properties:
is_faithful:
example: true
type: boolean
type: object
response.GetMetricsResponse:
properties:
metrics:
$ref: '#/components/schemas/response.PaginatedMetricsResponse'
type: object
response.HtmlComparisonResponse:
properties:
similarity_score:
example: 0.92
type: number
type: object
response.InstructionAdherenceResponse:
properties:
instruction_adherence_score:
example: 0.87
type: number
type: object
response.IntentChangeResponse:
properties:
reason:
example: User intent remained consistent throughout the conversation
type: string
score:
example: 1
type: integer
success:
example: true
type: boolean
type: object
response.JSONValidatorResponse:
properties:
is_valid_json:
example: true
type: boolean
type: object
response.MetricGroup:
properties:
metric_name:
type: string
description: Metric name.
example: answer relevancy
organization_id:
type: string
points:
items:
$ref: '#/components/schemas/response.MetricPoint'
type: array
description: 'List of data points for this metric. The `unit` field in each
point''s `labels` indicates the data type and determines which value field
is populated: `bool_value` for booleans, `numeric_value` for numbers,
or `enum_value` for strings.'
type: object
response.MetricPoint:
description: 'A single metric data point. The `unit` field in `labels` indicates
the data type and determines which value field is populated: `bool_value`
for booleans unitless, `numeric_value` for number units, or `enum_value` for
strings.'
properties:
bool_value:
type: boolean
enum_value:
type: string
numeric_value:
type: number
event_time:
type: integer
labels:
description: The labels are the attributes of the metric point. The labels
are key-value pairs that are used to identify the metric point.
example:
agent_name: Travel Planner Agent
entity_type: span
env_project_id: '1111'
environment: dev
evaluator_name: ''
insert_time: '1772616490000'
metric_source: word_count
metric_type: numeric
model: gpt-4o-2024-08-06
org_id: '12345'
run_id: '54321'
service_name: travel-agent-demo
span_id: e118a1985c9cff23
trace_id: 2c98a42d1225cb786f3395d97ca3014d
unit: words
vendor: openai
additionalProperties:
type: string
type: object
type: object
response.MetricsHWMResponse:
properties:
high_water_mark:
type: integer
type: object
response.PIIDetectorResponse:
properties:
has_pii:
example: false
type: boolean
type: object
response.PaginatedMetricsResponse:
properties:
data:
items:
$ref: '#/components/schemas/response.MetricGroup'
type: array
next_cursor:
type: string
total_points:
type: integer
total_results:
type: integer
type: object
response.PerplexityResponse:
properties:
perplexity_score:
example: 12.5
type: number
type: object
response.PlaceholderRegexResponse:
properties:
is_valid_regex:
example: true
type: boolean
type: object
response.ProfanityDetectorResponse:
properties:
is_safe:
example: false
type: boolean
type: object
response.PromptInjectionResponse:
properties:
has_injection:
example: false
type: boolean
type: object
response.PromptPerplexityResponse:
properties:
perplexity_score:
example: 8.3
type: number
type: object
response.RegexValidatorResponse:
properties:
is_valid_regex:
example: true
type: boolean
type: object
response.SQLValidatorResponse:
properties:
is_valid_sql:
example: true
type: boolean
type: object
response.SecretsDetectorResponse:
properties:
has_secret:
example: false
type: boolean
type: object
response.SemanticSimilarityResponse:
properties:
similarity_score:
example: 0.92
type: number
type: object
response.SexismDetectorResponse:
properties:
is_safe:
example: false
type: boolean
type: object
response.ToneDetectionResponse:
properties:
score:
example: 0.95
type: number
tone:
example: neutral
type: string
type: object
response.TopicAdherenceResponse:
properties:
adherence_score:
example: 0.95
type: number
type: object
response.ToxicityDetectorResponse:
properties:
is_safe:
example: false
type: boolean
type: object
response.UncertaintyDetectorResponse:
properties:
answer:
example: Paris
type: string
uncertainty:
example: 0.95
type: number
type: object
response.WordCountRatioResponse:
properties:
word_ratio:
example: 0.85
type: number
type: object
response.WordCountResponse:
properties:
word_count:
example: 10
type: integer
type: object
shared.FilterCondition:
properties:
field:
type: string
description: Field key to filter by. The key is labels attribute on the
metric point from the response.metrics object
example: labels.agent_name
operator:
type: string
description: Comparison operator.
enum:
- equals
- not_equals
- contains
- not_contains
- in
- not_in
example: equals
value:
type: string
description: Value to filter by.
example: Travel Planner Agent
values:
items:
type: string
type: array
description: List of values to filter by. This is only used for the in and
not_in operators.
example:
- Travel Planner Agent
- Calendar Planner Agent
type: object
types.ComparisonOperator:
enum:
- equals
- not_equals
- contains
- not_contains
- in
- not_in
type: string
x-enum-varnames:
- ComparisonOperatorEquals
- ComparisonOperatorNotEquals
- ComparisonOperatorContains
- ComparisonOperatorNotContains
- ComparisonOperatorIn
- ComparisonOperatorNotIn
types.EvaluatorType:
enum:
- AGENT_GOAL_ACCURACY
- CHAR_COUNT
- CHAR_COUNT_RATIO
- ANSWER_COMPLETENESS
- FAITHFULNESS
- JSON_VALIDATOR
- PERPLEXITY
- PII_DETECTOR
- PLACEHOLDER_REGEX
- PROFANITY_DETECTOR
- SEXISM_DETECTOR
- PROMPT_INJECTION
- TONE_DETECTION
- PROMPT_PERPLEXITY
- TOXICITY_DETECTOR
- ANSWER_RELEVANCY
- ANSWER_CORRECTNESS
- REGEX_VALIDATOR
- SECRETS_DETECTOR
- SQL_VALIDATOR
- SEMANTIC_SIMILARITY
- TOPIC_ADHERENCE
- UNCERTAINTY_DETECTOR
- WORD_COUNT
- WORD_COUNT_RATIO
- INSTRUCTION_ADHERENCE
- AGENT_FLOW_QUALITY
- AGENT_EFFICIENCY
- AGENT_GOAL_COMPLETENESS
- CONVERSATION_QUALITY
- INTENT_CHANGE
- CONTEXT_RELEVANCE
- AGENT_TOOL_ERROR_DETECTOR
- AGENT_TOOL_TRAJECTORY
- HTML_COMPARISON
- LLM_AS_A_JUDGE
type: string
x-enum-varnames:
- AgentGoalAccuracy
- CharCount
- CharCountRatio
- AnswerCompleteness
- Faithfulness
- JsonValidator
- Perplexity
- PIIDetector
- PlaceholderRegex
- ProfanityDetector
- SexismDetector
- PromptInjection
- ToneDetection
- PromptPerplexity
- ToxicityDetector
- AnswerRelevancy
- AnswerCorrectness
- RegexValidator
- SecretsDetector
- SQLValidator
- SemanticSimilarity
- TopicAdherence
- UncertaintyDetector
- WordCount
- WordCountRatio
- InstructionAdherence
- AgentFlowQuality
- AgentEfficiency
- AgentGoalCompleteness
- ConversationQuality
- IdentIntentChange
- ContextRelevance
- AgentToolErrorDetector
- AgentToolTrajectory
- HtmlComparison
- LLMAsAJudge
types.LogicalOperator:
enum:
- AND
- OR
type: string
x-enum-varnames:
- LogicalOperatorAnd
- LogicalOperatorOr
types.Rule:
properties:
key:
type: string
op:
$ref: '#/components/schemas/types.ComparisonOperator'
source:
type: string
value:
type: string
value_type:
type: string
type: object