openapi: 3.0.0
info:
  title: Traceloop auto-monitor-setups evaluators API
  version: 1.0.0
  contact: {}
servers:
- url: https://api.traceloop.com
tags:
- name: evaluators
paths:
  /v2/evaluators:
    post:
      description: Create a new custom evaluator at the organization level without project or environment bindings. The evaluator uses LLM-as-a-judge to evaluate inputs based on the provided configuration. The response format is automatically generated from the output schema.
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/request.CreateCustomEvaluatorInput'
        description: Custom evaluator creation input
        required: true
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.CreateEvaluatorResponse'
        '400':
          description: Bad Request
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
        '500':
          description: Internal Server Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
      summary: Create a custom LLM-as-a-judge evaluator
      tags:
      - evaluators
  /v2/evaluators/agent-efficiency/execute:
    post:
      description: 'Evaluate agent efficiency - detect redundant calls, unnecessary follow-ups


        **Request Body:**

        - `input.trajectory_prompts` (string, required): JSON array of prompts in the agent trajectory

        - `input.trajectory_completions` (string, required): JSON array of completions in the agent trajectory'
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/request.AgentEfficiencyRequest'
        description: Request body
        required: true
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.AgentEfficiencyResponse'
        '400':
          description: Bad Request
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
        '500':
          description: Internal Server Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
      security:
      - BearerAuth: []
      summary: Execute agent-efficiency evaluator
      tags:
      - evaluators
  /v2/evaluators/agent-flow-quality/execute:
    post:
      description: 'Validate agent trajectory against user-defined conditions


        **Request Body:**

        - `input.trajectory_prompts` (string, required): JSON array of prompts in the agent trajectory

        - `input.trajectory_completions` (string, required): JSON array of completions in the agent trajectory

        - `config.conditions` (array of strings, required): Array of evaluation conditions/rules to validate against

        - `config.threshold` (number, required): Score threshold for pass/fail determination (0.0-1.0)'
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/request.AgentFlowQualityRequest'
        description: Request body
        required: true
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.AgentFlowQualityResponse'
        '400':
          description: Bad Request
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
        '500':
          description: Internal Server Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
      security:
      - BearerAuth: []
      summary: Execute agent-flow-quality evaluator
      tags:
      - evaluators
  /v2/evaluators/agent-goal-accuracy/execute:
    post:
      description: 'Evaluate agent goal accuracy


        **Request Body:**

        - `input.question` (string, required): The original question or goal

        - `input.completion` (string, required): The agent''s completion/response

        - `input.reference` (string, required): The expected reference answer'
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/request.AgentGoalAccuracyRequest'
        description: Request body
        required: true
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.AgentGoalAccuracyResponse'
        '400':
          description: Bad Request
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
        '500':
          description: Internal Server Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
      security:
      - BearerAuth: []
      summary: Execute agent-goal-accuracy evaluator
      tags:
      - evaluators
  /v2/evaluators/agent-goal-completeness/execute:
    post:
      description: 'Measure if agent accomplished all user goals


        **Request Body:**

        - `input.trajectory_prompts` (string, required): JSON array of prompts in the agent trajectory

        - `input.trajectory_completions` (string, required): JSON array of completions in the agent trajectory

        - `config.threshold` (number, required): Score threshold for pass/fail determination (0.0-1.0)'
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/request.AgentGoalCompletenessRequest'
        description: Request body
        required: true
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.AgentGoalCompletenessResponse'
        '400':
          description: Bad Request
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
        '500':
          description: Internal Server Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
      security:
      - BearerAuth: []
      summary: Execute agent-goal-completeness evaluator
      tags:
      - evaluators
  /v2/evaluators/agent-tool-error-detector/execute:
    post:
      description: 'Detect errors or failures during tool execution


        **Request Body:**

        - `input.tool_input` (string, required): JSON string of the tool input

        - `input.tool_output` (string, required): JSON string of the tool output'
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/request.AgentToolErrorDetectorRequest'
        description: Request body
        required: true
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.AgentToolErrorDetectorResponse'
        '400':
          description: Bad Request
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
        '500':
          description: Internal Server Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
      security:
      - BearerAuth: []
      summary: Execute agent-tool-error-detector evaluator
      tags:
      - evaluators
  /v2/evaluators/agent-tool-trajectory/execute:
    post:
      description: 'Compare actual tool calls against expected reference tool calls


        **Request Body:**

        - `input.executed_tool_calls` (string, required): JSON array of actual tool calls made by the agent

        - `input.expected_tool_calls` (string, required): JSON array of expected/reference tool calls

        - `config.threshold` (float, optional): Score threshold for pass/fail determination (default: 0.5)

        - `config.mismatch_sensitive` (bool, optional): Whether tool calls must match exactly (default: false)

        - `config.order_sensitive` (bool, optional): Whether order of tool calls matters (default: false)

        - `config.input_params_sensitive` (bool, optional): Whether to compare input parameters (default: true)'
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/request.AgentToolTrajectoryRequest'
        description: Request body
        required: true
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.AgentToolTrajectoryResponse'
        '400':
          description: Bad Request
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
        '500':
          description: Internal Server Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
      security:
      - BearerAuth: []
      summary: Execute agent-tool-trajectory evaluator
      tags:
      - evaluators
  /v2/evaluators/answer-completeness/execute:
    post:
      description: 'Evaluate whether the answer is complete and contains all the necessary information


        **Request Body:**

        - `input.question` (string, required): The original question

        - `input.completion` (string, required): The completion to evaluate for completeness

        - `input.context` (string, required): The context that provides the complete information'
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/request.AnswerCompletenessRequest'
        description: Request body
        required: true
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.AnswerCompletenessResponse'
        '400':
          description: Bad Request
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
        '500':
          description: Internal Server Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
      security:
      - BearerAuth: []
      summary: Execute answer-completeness evaluator
      tags:
      - evaluators
  /v2/evaluators/answer-correctness/execute:
    post:
      description: 'Evaluate factual accuracy by comparing answers against ground truth


        **Request Body:**

        - `input.question` (string, required): The original question

        - `input.completion` (string, required): The completion to evaluate

        - `input.ground_truth` (string, required): The expected correct answer'
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/request.AnswerCorrectnessRequest'
        description: Request body
        required: true
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.AnswerCorrectnessResponse'
        '400':
          description: Bad Request
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
        '500':
          description: Internal Server Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
      security:
      - BearerAuth: []
      summary: Execute answer-correctness evaluator
      tags:
      - evaluators
  /v2/evaluators/answer-relevancy/execute:
    post:
      description: 'Check if an answer is relevant to a question


        **Request Body:**

        - `input.answer` (string, required): The answer to evaluate for relevancy

        - `input.question` (string, required): The question that the answer should be relevant to'
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/request.AnswerRelevancyRequest'
        description: Request body
        required: true
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.AnswerRelevancyResponse'
        '400':
          description: Bad Request
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
        '500':
          description: Internal Server Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
      security:
      - BearerAuth: []
      summary: Execute answer-relevancy evaluator
      tags:
      - evaluators
  /v2/evaluators/char-count/execute:
    post:
      description: 'Count the number of characters in text


        **Request Body:**

        - `input.text` (string, required): The text to count characters in'
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/request.CharCountRequest'
        description: Request body
        required: true
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.CharCountResponse'
        '400':
          description: Bad Request
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
        '500':
          description: Internal Server Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
      security:
      - BearerAuth: []
      summary: Execute char-count evaluator
      tags:
      - evaluators
  /v2/evaluators/char-count-ratio/execute:
    post:
      description: 'Calculate the ratio of characters between two texts


        **Request Body:**

        - `input.numerator_text` (string, required): The numerator text (will be divided by denominator)

        - `input.denominator_text` (string, required): The denominator text (divides the numerator)'
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/request.CharCountRatioRequest'
        description: Request body
        required: true
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.CharCountRatioResponse'
        '400':
          description: Bad Request
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
        '500':
          description: Internal Server Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
      security:
      - BearerAuth: []
      summary: Execute char-count-ratio evaluator
      tags:
      - evaluators
  /v2/evaluators/context-relevance/execute:
    post:
      description: 'Evaluate whether retrieved context contains sufficient information to answer the query


        **Request Body:**

        - `input.query` (string, required): The query/question to evaluate context relevance for

        - `input.completion` (string, required): The completion to evaluate for context relevance

        - `input.context` (string, required): The context to evaluate for relevance to the query

        - `config.model` (string, optional): Model to use for evaluation (default: gpt-4o)'
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/request.ContextRelevanceRequest'
        description: Request body
        required: true
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ContextRelevanceResponse'
        '400':
          description: Bad Request
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
        '500':
          description: Internal Server Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
      security:
      - BearerAuth: []
      summary: Execute context-relevance evaluator
      tags:
      - evaluators
  /v2/evaluators/conversation-quality/execute:
    post:
      description: 'Evaluate conversation quality based on tone, clarity, flow, responsiveness, and transparency


        **Request Body:**

        - `input.prompts` (string, required): JSON array of prompts in the conversation

        - `input.completions` (string, required): JSON array of completions in the conversation'
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/request.ConversationQualityRequest'
        description: Request body
        required: true
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ConversationQualityResponse'
        '400':
          description: Bad Request
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
        '500':
          description: Internal Server Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
      security:
      - BearerAuth: []
      summary: Execute conversation-quality evaluator
      tags:
      - evaluators
  /v2/evaluators/faithfulness/execute:
    post:
      description: 'Check if a completion is faithful to the provided context


        **Request Body:**

        - `input.completion` (string, required): The LLM completion to check for faithfulness

        - `input.context` (string, required): The context that the completion should be faithful to

        - `input.question` (string, required): The original question asked'
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/request.FaithfulnessRequest'
        description: Request body
        required: true
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.FaithfulnessResponse'
        '400':
          description: Bad Request
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
        '500':
          description: Internal Server Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
      security:
      - BearerAuth: []
      summary: Execute faithfulness evaluator
      tags:
      - evaluators
  /v2/evaluators/html-comparison/execute:
    post:
      description: 'Compare two HTML documents for structural and content similarity


        **Request Body:**

        - `input.html1` (string, required): The first HTML document to compare

        - `input.html2` (string, required): The second HTML document to compare'
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/request.HtmlComparisonRequest'
        description: Request body
        required: true
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.HtmlComparisonResponse'
        '400':
          description: Bad Request
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
        '500':
          description: Internal Server Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
      security:
      - BearerAuth: []
      summary: Execute html-comparison evaluator
      tags:
      - evaluators
  /v2/evaluators/instruction-adherence/execute:
    post:
      description: 'Evaluate how well responses follow given instructions


        **Request Body:**

        - `input.instructions` (string, required): The instructions that should be followed

        - `input.response` (string, required): The response to evaluate for instruction adherence'
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/request.InstructionAdherenceRequest'
        description: Request body
        required: true
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.InstructionAdherenceResponse'
        '400':
          description: Bad Request
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
        '500':
          description: Internal Server Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
      security:
      - BearerAuth: []
      summary: Execute instruction-adherence evaluator
      tags:
      - evaluators
  /v2/evaluators/intent-change/execute:
    post:
      description: 'Detect changes in user intent between prompts and completions


        **Request Body:**

        - `input.prompts` (string, required): JSON array of prompts in the conversation

        - `input.completions` (string, required): JSON array of completions in the conversation'
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/request.IntentChangeRequest'
        description: Request body
        required: true
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.IntentChangeResponse'
        '400':
          description: Bad Request
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
        '500':
          description: Internal Server Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
      security:
      - BearerAuth: []
      summary: Execute intent-change evaluator
      tags:
      - evaluators
  /v2/evaluators/json-validator/execute:
    post:
      description: 'Validate JSON syntax


        **Request Body:**

        - `input.text` (string, required): The text to validate as JSON

        - `config.enable_schema_validation` (bool, optional): Enable JSON schema validation

        - `config.schema_string` (string, optional): JSON schema to validate against'
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/request.JSONValidatorRequest'
        description: Request body
        required: true
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.JSONValidatorResponse'
        '400':
          description: Bad Request
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
        '500':
          description: Internal Server Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
      security:
      - BearerAuth: []
      summary: Execute json-validator evaluator
      tags:
      - evaluators
  /v2/evaluators/perplexity/execute:
    post:
      description: 'Measure text perplexity from logprobs


        **Request Body:**

        - `input.logprobs` (string, required): JSON array of log probabilities from the model'
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/request.PerplexityRequest'
        description: Request body
        required: true
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.PerplexityResponse'
        '400':
          description: Bad Request
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
        '500':
          description: Internal Server Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
      security:
      - BearerAuth: []
      summary: Execute perplexity evaluator
      tags:
      - evaluators
  /v2/evaluators/pii-detector/execute:
    post:
      description: 'Detect personally identifiable information in text


        **Request Body:**

        - `input.text` (string, required): The text to scan for personally identifiable information

        - `config.probability_threshold` (float, optional): Detection threshold (default: 0.8)'
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/request.PIIDetectorRequest'
        description: Request body
        required: true
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.PIIDetectorResponse'
        '400':
          description: Bad Request
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
        '500':
          description: Internal Server Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
      security:
      - BearerAuth: []
      summary: Execute pii-detector evaluator
      tags:
      - evaluators
  /v2/evaluators/placeholder-regex/execute:
    post:
      description: 'Validate text against a placeholder regex pattern


        **Request Body:**

        - `input.placeholder_name` (string, required): The name of the placeholder to substitute

        - `input.placeholder_value` (string, required): The value to substitute into the regex placeholder

        - `input.text` (string, required): The text to validate against the regex pattern

        - `config.should_match` (bool, optional): Whether the text should match the regex

        - `config.case_sensitive` (bool, optional): Case-sensitive matching

        - `config.dot_include_nl` (bool, optional): Dot matches newlines

        - `config.multi_line` (bool, optional): Multi-line mode'
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/request.PlaceholderRegexRequest'
        description: Request body
        required: true
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.PlaceholderRegexResponse'
        '400':
          description: Bad Request
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
        '500':
          description: Internal Server Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
      security:
      - BearerAuth: []
      summary: Execute placeholder-regex evaluator
      tags:
      - evaluators
  /v2/evaluators/profanity-detector/execute:
    post:
      description: 'Detect profanity in text


        **Request Body:**

        - `input.text` (string, required): The text to scan for profanity'
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/request.ProfanityDetectorRequest'
        description: Request body
        required: true
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ProfanityDetectorResponse'
        '400':
          description: Bad Request
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
        '500':
          description: Internal Server Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
      security:
      - BearerAuth: []
      summary: Execute profanity-detector evaluator
      tags:
      - evaluators
  /v2/evaluators/prompt-injection/execute:
    post:
      description: 'Detect prompt injection attempts


        **Request Body:**

        - `input.prompt` (string, required): The prompt to check for injection attempts

        - `config.threshold` (float, optional): Detection threshold (default: 0.5)'
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/request.PromptInjectionRequest'
        description: Request body
        required: true
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.PromptInjectionResponse'
        '400':
          description: Bad Request
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
        '500':
          description: Internal Server Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
      security:
      - BearerAuth: []
      summary: Execute prompt-injection evaluator
      tags:
      - evaluators
  /v2/evaluators/prompt-perplexity/execute:
    post:
      description: 'Measure prompt perplexity to detect potential injection attempts


        **Request Body:**

        - `input.prompt` (string, required): The prompt to calculate perplexity for'
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/request.PromptPerplexityRequest'
        description: Request body
        required: true
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.PromptPerplexityResponse'
        '400':
          description: Bad Request
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
        '500':
          description: Internal Server Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
      security:
      - BearerAuth: []
      summary: Execute prompt-perplexity evaluator
      tags:
      - evaluators
  /v2/evaluators/regex-validator/execute:
    post:
      description: 'Validate text against a regex pattern


        **Request Body:**

        - `input.text` (string, required): The text to validate against a regex pattern

        - `config.regex` (string, optional): The regex pattern to match against

        - `config.should_match` (bool, optional): Whether the text should match the regex

        - `config.case_sensitive` (bool, optional): Case-sensitive matching

        - `config.dot_include_nl` (bool, optional): Dot matches newlines

        - `config.multi_line` (bool, optional): Multi-line mode'
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/request.RegexValidatorRequest'
        description: Request body
        required: true
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.RegexValidatorResponse'
        '400':
          description: Bad Request
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
        '500':
          description: Internal Server Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
      security:
      - BearerAuth: []
      summary: Execute regex-validator evaluator
      tags:
      - evaluators
  /v2/evaluators/secrets-detector/execute:
    post:
      description: 'Detect secrets and credentials in text


        **Request Body:**

        - `input.text` (string, required): The text to scan for secrets (API keys, passwords, etc.)'
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/request.SecretsDetectorRequest'
        description: Request body
        required: true
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.SecretsDetectorResponse'
        '400':
          description: Bad Request
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
        '500':
          description: Internal Server Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
      security:
      - BearerAuth: []
      summary: Execute secrets-detector evaluator
      tags:
      - evaluators
  /v2/evaluators/semantic-similarity/execute:
    post:
      description: 'Calculate semantic similarity between completion and reference


        **Request Body:**

        - `input.completion` (string, required): The completion text to compare

        - `input.reference` (string, required): The reference text to compare against'
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/request.SemanticSimilarityRequest'
        description: Request body
        required: true
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.SemanticSimilarityResponse'
        '400':
          description: Bad Request
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
        '500':
          description: Internal Server Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
      security:
      - BearerAuth: []
      summary: Execute semantic-similarity evaluator
      tags:
      - evaluators
  /v2/evaluators/sexism-detector/execute:
    post:
      description: 'Detect sexist language and bias


        **Request Body:**

        - `input.text` (string, required): The text to scan for sexist content

        - `config.threshold` (float, optional): Detection threshold (default: 0.5)'
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/request.SexismDetectorRequest'
        description: Request body
        required: true
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.SexismDetectorResponse'
        '400':
          description: Bad Request
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
        '500':
          description: Internal Server Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
      security:
      - BearerAuth: []
      summary: Execute sexism-detector evaluator
      tags:
      - evaluators
  /v2/evaluators/sql-validator/execute:
    post:
      description: 'Validate SQL query syntax


        **Request Body:**

        - `input.text` (string, required): The text to validate as SQL'
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/request.SQLValidatorRequest'
        description: Request body
        required: true
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.SQLValidatorResponse'
        '400':
          description: Bad Request
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
        '500':
          description: Internal Server Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
      security:
      - BearerAuth: []
      summary: Execute sql-validator evaluator
      tags:
      - evaluators
  /v2/evaluators/tone-detection/execute:
    post:
      description: 'Detect the tone of the text


        **Request Body:**

        - `input.text` (string, required): The text to detect the tone of'
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/request.ToneDetectionRequest'
        description: Request body
        required: true
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ToneDetectionResponse'
        '400':
          description: Bad Request
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
        '500':
          description: Internal Server Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
      security:
      - BearerAuth: []
      summary: Execute tone-detection evaluator
      tags:
      - evaluators
  /v2/evaluators/topic-adherence/execute:
    post:
      description: 'Evaluate topic adherence


        **Request Body:**

        - `input.question` (string, required): The original question

        - `input.completion` (string, required): The completion to evaluate

        - `input.reference_topics` (string, required): Comma-separated list of expected topics'
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/request.TopicAdherenceRequest'
        description: Request body
        required: true
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.TopicAdherenceResponse'
        '400':
          description: Bad Request
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
        '500':
          description: Internal Server Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
      security:
      - BearerAuth: []
      summary: Execute topic-adherence evaluator
      tags:
      - evaluators
  /v2/evaluators/toxicity-detector/execute:
    post:
      description: 'Detect toxic or harmful language


        **Request Body:**

        - `input.text` (string, required): The text to scan for toxic content

        - `config.threshold` (float, optional): Detection threshold (default: 0.5)'
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/request.ToxicityDetectorRequest'
        description: Request body
        required: true
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ToxicityDetectorResponse'
        '400':
          description: Bad Request
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
        '500':
          description: Internal Server Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
      security:
      - BearerAuth: []
      summary: Execute toxicity-detector evaluator
      tags:
      - evaluators
  /v2/evaluators/uncertainty-detector/execute:
    post:
      description: 'Detect uncertainty in the text


        **Request Body:**

        - `input.prompt` (string, required): The text to detect uncertainty in'
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/request.UncertaintyDetectorRequest'
        description: Request body
        required: true
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.UncertaintyDetectorResponse'
        '400':
          description: Bad Request
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
        '500':
          description: Internal Server Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
      security:
      - BearerAuth: []
      summary: Execute uncertainty-detector evaluator
      tags:
      - evaluators
  /v2/evaluators/word-count/execute:
    post:
      description: 'Count the number of words in text


        **Request Body:**

        - `input.text` (string, required): The text to count words in'
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/request.WordCountRequest'
        description: Request body
        required: true
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.WordCountResponse'
        '400':
          description: Bad Request
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
        '500':
          description: Internal Server Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
      security:
      - BearerAuth: []
      summary: Execute word-count evaluator
      tags:
      - evaluators
  /v2/evaluators/word-count-ratio/execute:
    post:
      description: 'Calculate the ratio of words between two texts


        **Request Body:**

        - `input.numerator_text` (string, required): The numerator text (will be divided by denominator)

        - `input.denominator_text` (string, required): The denominator text (divides the numerator)'
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/request.WordCountRatioRequest'
        description: Request body
        required: true
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.WordCountRatioResponse'
        '400':
          description: Bad Request
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
        '500':
          description: Internal Server Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/response.ErrorResponse'
      security:
      - BearerAuth: []
      summary: Execute word-count-ratio evaluator
      tags:
      - evaluators
components:
  schemas:
    request.AgentGoalCompletenessInput:
      properties:
        trajectory_completions:
          example: '["Account created", "Preferences saved", "Notifications enabled"]'
          type: string
        trajectory_prompts:
          example: '["Create new account", "Set preferences", "Enable notifications"]'
          type: string
      required:
      - trajectory_completions
      - trajectory_prompts
      type: object
    response.WordCountRatioResponse:
      properties:
        word_ratio:
          example: 0.85
          type: number
      type: object
    request.PromptInjectionRequest:
      properties:
        config:
          $ref: '#/components/schemas/request.PromptInjectionConfigRequest'
        input:
          $ref: '#/components/schemas/request.PromptInjectionInput'
      required:
      - input
      type: object
    request.CharCountInput:
      properties:
        text:
          example: Hello, world! This is a sample text.
          type: string
      required:
      - text
      type: object
    request.SecretsDetectorInput:
      properties:
        text:
          example: Here is some text without any API keys or passwords.
          type: string
      required:
      - text
      type: object
    response.AgentGoalCompletenessResponse:
      properties:
        reason:
          example: All user goals were accomplished
          type: string
        score:
          example: 0.95
          type: number
        success:
          example: true
          type: boolean
      type: object
    request.AgentToolErrorDetectorRequest:
      properties:
        input:
          $ref: '#/components/schemas/request.AgentToolErrorDetectorInput'
      required:
      - input
      type: object
    request.HtmlComparisonRequest:
      properties:
        input:
          $ref: '#/components/schemas/request.HtmlComparisonInput'
      required:
      - input
      type: object
    response.ProfanityDetectorResponse:
      properties:
        is_safe:
          example: false
          type: boolean
      type: object
    request.AnswerRelevancyInput:
      properties:
        answer:
          example: The capital of France is Paris.
          type: string
        question:
          example: What is the capital of France?
          type: string
      required:
      - answer
      - question
      type: object
    types.EvaluatorType:
      enum:
      - AGENT_GOAL_ACCURACY
      - CHAR_COUNT
      - CHAR_COUNT_RATIO
      - ANSWER_COMPLETENESS
      - FAITHFULNESS
      - JSON_VALIDATOR
      - PERPLEXITY
      - PII_DETECTOR
      - PLACEHOLDER_REGEX
      - PROFANITY_DETECTOR
      - SEXISM_DETECTOR
      - PROMPT_INJECTION
      - TONE_DETECTION
      - PROMPT_PERPLEXITY
      - TOXICITY_DETECTOR
      - ANSWER_RELEVANCY
      - ANSWER_CORRECTNESS
      - REGEX_VALIDATOR
      - SECRETS_DETECTOR
      - SQL_VALIDATOR
      - SEMANTIC_SIMILARITY
      - TOPIC_ADHERENCE
      - UNCERTAINTY_DETECTOR
      - WORD_COUNT
      - WORD_COUNT_RATIO
      - INSTRUCTION_ADHERENCE
      - AGENT_FLOW_QUALITY
      - AGENT_EFFICIENCY
      - AGENT_GOAL_COMPLETENESS
      - CONVERSATION_QUALITY
      - INTENT_CHANGE
      - CONTEXT_RELEVANCE
      - AGENT_TOOL_ERROR_DETECTOR
      - AGENT_TOOL_TRAJECTORY
      - HTML_COMPARISON
      - LLM_AS_A_JUDGE
      type: string
      x-enum-varnames:
      - AgentGoalAccuracy
      - CharCount
      - CharCountRatio
      - AnswerCompleteness
      - Faithfulness
      - JsonValidator
      - Perplexity
      - PIIDetector
      - PlaceholderRegex
      - ProfanityDetector
      - SexismDetector
      - PromptInjection
      - ToneDetection
      - PromptPerplexity
      - ToxicityDetector
      - AnswerRelevancy
      - AnswerCorrectness
      - RegexValidator
      - SecretsDetector
      - SQLValidator
      - SemanticSimilarity
      - TopicAdherence
      - UncertaintyDetector
      - WordCount
      - WordCountRatio
      - InstructionAdherence
      - AgentFlowQuality
      - AgentEfficiency
      - AgentGoalCompleteness
      - ConversationQuality
      - IdentIntentChange
      - ContextRelevance
      - AgentToolErrorDetector
      - AgentToolTrajectory
      - HtmlComparison
      - LLMAsAJudge
    request.AnswerCompletenessInput:
      properties:
        completion:
          example: Paris.
          type: string
        context:
          example: The capital of France is Paris.
          type: string
        question:
          example: What is the capital of France?
          type: string
      required:
      - completion
      - context
      - question
      type: object
    request.PromptPerplexityInput:
      properties:
        prompt:
          example: What is the capital of France?
          type: string
      required:
      - prompt
      type: object
    response.UncertaintyDetectorResponse:
      properties:
        answer:
          example: Paris
          type: string
        uncertainty:
          example: 0.95
          type: number
      type: object
    request.ToxicityDetectorInput:
      properties:
        text:
          example: Thank you for your help with this project.
          type: string
      required:
      - text
      type: object
    request.InstructionAdherenceInput:
      properties:
        instructions:
          example: Respond in exactly 3 bullet points and use formal language.
          type: string
        response:
          example: '- First point about the topic

            - Second relevant consideration

            - Final concluding thought'
          type: string
      required:
      - instructions
      - response
      type: object
    request.AgentToolTrajectoryRequest:
      properties:
        config:
          $ref: '#/components/schemas/request.AgentToolTrajectoryConfigRequest'
        input:
          $ref: '#/components/schemas/request.AgentToolTrajectoryInput'
      required:
      - input
      type: object
    request.PerplexityInput:
      properties:
        logprobs:
          example: '[-2.3, -1.5, -0.8, -1.2, -0.5]'
          type: string
      required:
      - logprobs
      type: object
    response.PromptInjectionResponse:
      properties:
        has_injection:
          example: false
          type: boolean
      type: object
    request.TopicAdherenceInput:
      properties:
        completion:
          example: Machine learning is a subset of AI that enables systems to learn from data.
          type: string
        question:
          example: Tell me about machine learning
          type: string
        reference_topics:
          example: artificial intelligence, data science, algorithms
          type: string
      required:
      - completion
      - question
      - reference_topics
      type: object
    request.WordCountRequest:
      properties:
        input:
          $ref: '#/components/schemas/request.WordCountInput'
      required:
      - input
      type: object
    request.FaithfulnessRequest:
      properties:
        input:
          $ref: '#/components/schemas/request.FaithfulnessInput'
      required:
      - input
      type: object
    response.AgentToolErrorDetectorResponse:
      properties:
        reason:
          example: Tool executed successfully without errors
          type: string
        success:
          example: true
          type: boolean
      type: object
    request.ProfanityDetectorInput:
      properties:
        text:
          example: This is a clean and professional message.
          type: string
      required:
      - text
      type: object
    request.AnswerCorrectnessRequest:
      properties:
        input:
          $ref: '#/components/schemas/request.AnswerCorrectnessInput'
      required:
      - input
      type: object
    response.AgentGoalAccuracyResponse:
      properties:
        accuracy_score:
          example: 0.88
          type: number
      type: object
    request.ContextRelevanceConfigRequest:
      properties:
        model:
          example: gpt-4o
          type: string
      type: object
    response.AnswerCorrectnessResponse:
      properties:
        correctness_score:
          example: 0.91
          type: number
      type: object
    response.JSONValidatorResponse:
      properties:
        is_valid_json:
          example: true
          type: boolean
      type: object
    request.AgentFlowQualityRequest:
      properties:
        config:
          $ref: '#/components/schemas/request.AgentFlowQualityConfigRequest'
        input:
          $ref: '#/components/schemas/request.AgentFlowQualityInput'
      required:
      - config
      - input
      type: object
    request.ToneDetectionInput:
      properties:
        text:
          example: The capital of France is Paris.
          type: string
      required:
      - text
      type: object
    request.PlaceholderRegexRequest:
      properties:
        config:
          $ref: '#/components/schemas/request.PlaceholderRegexConfigRequest'
        input:
          $ref: '#/components/schemas/request.PlaceholderRegexInput'
      required:
      - input
      type: object
    response.IntentChangeResponse:
      properties:
        reason:
          example: User intent remained consistent throughout the conversation
          type: string
        score:
          example: 1
          type: integer
        success:
          example: true
          type: boolean
      type: object
    response.AgentToolTrajectoryResponse:
      properties:
        reason:
          example: Tool calls match the expected trajectory
          type: string
        score:
          example: 0.85
          type: number
        success:
          example: true
          type: boolean
      type: object
    model.LLMMessage:
      properties:
        content:
          type: string
        role:
          type: string
        tool_call_id:
          type: string
      type: object
    request.WordCountInput:
      properties:
        text:
          example: This is a sample text with several words.
          type: string
      required:
      - text
      type: object
    response.TopicAdherenceResponse:
      properties:
        adherence_score:
          example: 0.95
          type: number
      type: object
    request.SemanticSimilarityInput:
      properties:
        completion:
          example: The cat sat on the mat.
          type: string
        reference:
          example: A feline was resting on the rug.
          type: string
      required:
      - completion
      - reference
      type: object
    request.SQLValidatorInput:
      properties:
        text:
          example: SELECT * FROM users WHERE id = 1;
          type: string
      required:
      - text
      type: object
    request.PIIDetectorRequest:
      properties:
        config:
          $ref: '#/components/schemas/request.PIIDetectorConfigRequest'
        input:
          $ref: '#/components/schemas/request.PIIDetectorInput'
      required:
      - input
      type: object
    evaluator.PropertyType:
      enum:
      - string
      - int
      - float
      - boolean
      - enum
      type: string
      x-enum-varnames:
      - StringType
      - IntType
      - FloatType
      - BooleanType
      - EnumType
    request.HtmlComparisonInput:
      properties:
        html1:
          example: <html><body><h1>Hello, world!</h1></body></html>
          type: string
        html2:
          example: <html><body><h1>Hello, world!</h1></body></html>
          type: string
      required:
      - html1
      - html2
      type: object
    request.CharCountRatioInput:
      properties:
        denominator_text:
          example: This is a longer text for comparison
          type: string
        numerator_text:
          example: Short text
          type: string
      required:
      - denominator_text
      - numerator_text
      type: object
    request.PIIDetectorConfigRequest:
      properties:
        probability_threshold:
          example: 0.8
          type: number
      type: object
    request.SexismDetectorRequest:
      properties:
        config:
          $ref: '#/components/schemas/request.SexismDetectorConfigRequest'
        input:
          $ref: '#/components/schemas/request.SexismDetectorInput'
      required:
      - input
      type: object
    request.AgentToolErrorDetectorInput:
      properties:
        tool_input:
          example: '{"action": "search", "query": "flights to Paris"}'
          type: string
        tool_output:
          example: '{"status": "success", "results": [{"flight": "AF123", "price": 450}]}'
          type: string
      required:
      - tool_input
      - tool_output
      type: object
    request.PIIDetectorInput:
      properties:
        text:
          example: Please contact John Smith at john.smith@email.com or call 555-123-4567.
          type: string
      required:
      - text
      type: object
    request.SecretsDetectorRequest:
      properties:
        input:
          $ref: '#/components/schemas/request.SecretsDetectorInput'
      required:
      - input
      type: object
    request.AgentGoalCompletenessConfigRequest:
      properties:
        threshold:
          example: 0.5
          type: number
      required:
      - threshold
      type: object
    request.ConversationQualityRequest:
      properties:
        input:
          $ref: '#/components/schemas/request.ConversationQualityInput'
      required:
      - input
      type: object
    request.IntentChangeInput:
      properties:
        completions:
          example: '["Sure, I can help with hotel booking", "No problem, let me search for flights"]'
          type: string
        prompts:
          example: '["I want to book a hotel", "Actually, I need a flight instead"]'
          type: string
      required:
      - completions
      - prompts
      type: object
    request.AnswerCorrectnessInput:
      properties:
        completion:
          example: World War II ended in 1945.
          type: string
        ground_truth:
          example: '1945'
          type: string
        question:
          example: What year did World War II end?
          type: string
      required:
      - completion
      - ground_truth
      - question
      type: object
    request.PromptInjectionConfigRequest:
      properties:
        threshold:
          example: 0.5
          type: number
      type: object
    request.PerplexityRequest:
      properties:
        input:
          $ref: '#/components/schemas/request.PerplexityInput'
      required:
      - input
      type: object
    request.AnswerCompletenessRequest:
      properties:
        input:
          $ref: '#/components/schemas/request.AnswerCompletenessInput'
      required:
      - input
      type: object
    request.ContextRelevanceRequest:
      properties:
        config:
          $ref: '#/components/schemas/request.ContextRelevanceConfigRequest'
        input:
          $ref: '#/components/schemas/request.ContextRelevanceInput'
      required:
      - input
      type: object
    request.RegexValidatorConfigRequest:
      properties:
        case_sensitive:
          example: true
          type: boolean
        dot_include_nl:
          example: true
          type: boolean
        multi_line:
          example: true
          type: boolean
        regex:
          example: .*
          type: string
        should_match:
          example: true
          type: boolean
      type: object
    response.PerplexityResponse:
      properties:
        perplexity_score:
          example: 12.5
          type: number
      type: object
    request.FaithfulnessInput:
      properties:
        completion:
          example: The Eiffel Tower is located in Paris and was built in 1889.
          type: string
        context:
          example: The Eiffel Tower is a wrought-iron lattice tower on the Champ de Mars in Paris, France. It was constructed from 1887 to 1889.
          type: string
        question:
          example: When was the Eiffel Tower built?
          type: string
      required:
      - completion
      - context
      - question
      type: object
    request.ContextRelevanceInput:
      properties:
        completion:
          example: Our store is open Monday to Friday from 9am to 6pm, and Saturday from 10am to 4pm. We are closed on Sundays.
          type: string
        context:
          example: Our store is open Monday to Friday from 9am to 6pm, and Saturday from 10am to 4pm. We are closed on Sundays.
          type: string
        query:
          example: What are the business hours?
          type: string
      required:
      - completion
      - context
      - query
      type: object
    response.WordCountResponse:
      properties:
        word_count:
          example: 10
          type: integer
      type: object
    request.SexismDetectorConfigRequest:
      properties:
        threshold:
          example: 0.5
          type: number
      type: object
    response.FaithfulnessResponse:
      properties:
        is_faithful:
          example: true
          type: boolean
      type: object
    response.ErrorResponse:
      description: Standard error response structure
      properties:
        error:
          example: error message
          type: string
      type: object
    request.UncertaintyDetectorInput:
      properties:
        prompt:
          example: I am not sure, I think the capital of France is Paris.
          type: string
      required:
      - prompt
      type: object
    response.ToxicityDetectorResponse:
      properties:
        is_safe:
          example: false
          type: boolean
      type: object
    response.RegexValidatorResponse:
      properties:
        is_valid_regex:
          example: true
          type: boolean
      type: object
    request.RegexValidatorInput:
      properties:
        text:
          example: user@example.com
          type: string
      required:
      - text
      type: object
    request.PromptPerplexityRequest:
      properties:
        input:
          $ref: '#/components/schemas/request.PromptPerplexityInput'
      required:
      - input
      type: object
    request.AgentToolTrajectoryConfigRequest:
      properties:
        input_params_sensitive:
          example: true
          type: boolean
        mismatch_sensitive:
          example: false
          type: boolean
        order_sensitive:
          example: false
          type: boolean
        threshold:
          example: 0.5
          type: number
      type: object
    request.CharCountRequest:
      properties:
        input:
          $ref: '#/components/schemas/request.CharCountInput'
      required:
      - input
      type: object
    response.SexismDetectorResponse:
      properties:
        is_safe:
          example: false
          type: boolean
      type: object
    request.AgentFlowQualityConfigRequest:
      properties:
        conditions:
          example:
          - no tools called
          - agent completed task
          items:
            type: string
          type: array
        threshold:
          example: 0.5
          type: number
      required:
      - conditions
      - threshold
      type: object
    request.ProfanityDetectorRequest:
      properties:
        input:
          $ref: '#/components/schemas/request.ProfanityDetectorInput'
      required:
      - input
      type: object
    request.UncertaintyDetectorRequest:
      properties:
        input:
          $ref: '#/components/schemas/request.UncertaintyDetectorInput'
      required:
      - input
      type: object
    response.AnswerCompletenessResponse:
      properties:
        answer_completeness_score:
          example: 0.95
          type: number
      type: object
    request.PromptInjectionInput:
      properties:
        prompt:
          example: What is the weather like today?
          type: string
      required:
      - prompt
      type: object
    request.CreateCustomEvaluatorInput:
      properties:
        description:
          type: string
        frequency_penalty:
          type: number
        input_schema:
          description: Schemas
          items:
            $ref: '#/components/schemas/evaluator.Property'
          type: array
        logprobs:
          type: boolean
        max_tokens:
          type: integer
        messages:
          description: LLM Message configuration
          items:
            $ref: '#/components/schemas/model.LLMMessage'
          type: array
        mode:
          type: string
        model:
          description: LLM parameters (from BaseLLMConfig)
          type: string
        name:
          description: Basic fields
          type: string
        output_schema:
          items:
            $ref: '#/components/schemas/evaluator.Property'
          type: array
        presence_penalty:
          type: number
        provider:
          description: Provider configuration
          type: string
        slug:
          type: string
        stop:
          items:
            type: string
          type: array
        temperature:
          type: number
        top_p:
          type: number
      required:
      - input_schema
      - messages
      - model
      - name
      - output_schema
      - provider
      type: object
    request.AnswerRelevancyRequest:
      properties:
        input:
          $ref: '#/components/schemas/request.AnswerRelevancyInput'
      required:
      - input
      type: object
    request.AgentGoalCompletenessRequest:
      properties:
        config:
          $ref: '#/components/schemas/request.AgentGoalCompletenessConfigRequest'
        input:
          $ref: '#/components/schemas/request.AgentGoalCompletenessInput'
      required:
      - input
      type: object
    request.ToxicityDetectorConfigRequest:
      properties:
        threshold:
          example: 0.5
          type: number
      type: object
    evaluator.Property:
      properties:
        description:
          type: string
        enum_values:
          description: The allowed values for this property. Only applicable when `type` is `enum`.
          items:
            type: string
          type: array
        label:
          type: string
        name:
          type: string
        type:
          $ref: '#/components/schemas/evaluator.PropertyType'
      required:
      - name
      - type
      type: object
    request.PlaceholderRegexConfigRequest:
      properties:
        case_sensitive:
          example: true
          type: boolean
        dot_include_nl:
          example: true
          type: boolean
        multi_line:
          example: true
          type: boolean
        should_match:
          example: true
          type: boolean
      type: object
    response.AgentFlowQualityResponse:
      properties:
        reason:
          example: Agent followed the expected flow correctly
          type: string
        score:
          example: 0.89
          type: number
        success:
          example: true
          type: boolean
      type: object
    request.TopicAdherenceRequest:
      properties:
        input:
          $ref: '#/components/schemas/request.TopicAdherenceInput'
      required:
      - input
      type: object
    request.ToneDetectionRequest:
      properties:
        input:
          $ref: '#/components/schemas/request.ToneDetectionInput'
      required:
      - input
      type: object
    request.AgentEfficiencyRequest:
      properties:
        input:
          $ref: '#/components/schemas/request.AgentEfficiencyInput'
      required:
      - input
      type: object
    request.JSONValidatorInput:
      properties:
        text:
          example: '{"name": "John", "age": 30}'
          type: string
      required:
      - text
      type: object
    request.AgentFlowQualityInput:
      properties:
        trajectory_completions:
          example: '["Found 5 flights", "Selected $299 flight", "Booking confirmed"]'
          type: string
        trajectory_prompts:
          example: '["Search for flights", "Select the cheapest option", "Confirm booking"]'
          type: string
      required:
      - trajectory_completions
      - trajectory_prompts
      type: object
    response.PromptPerplexityResponse:
      properties:
        perplexity_score:
          example: 8.3
          type: number
      type: object
    request.WordCountRatioRequest:
      properties:
        input:
          $ref: '#/components/schemas/request.WordCountRatioInput'
      required:
      - input
      type: object
    response.SecretsDetectorResponse:
      properties:
        has_secret:
          example: false
          type: boolean
      type: object
    response.HtmlComparisonResponse:
      properties:
        similarity_score:
          example: 0.92
          type: number
      type: object
    response.CharCountRatioResponse:
      properties:
        char_ratio:
          example: 0.75
          type: number
      type: object
    request.CharCountRatioRequest:
      properties:
        input:
          $ref: '#/components/schemas/request.CharCountRatioInput'
      required:
      - input
      type: object
    request.AgentEfficiencyInput:
      properties:
        trajectory_completions:
          example: '["User found", "Email updated", "Changes saved"]'
          type: string
        trajectory_prompts:
          example: '["Find user info", "Update email", "Save changes"]'
          type: string
      required:
      - trajectory_completions
      - trajectory_prompts
      type: object
    response.SQLValidatorResponse:
      properties:
        is_valid_sql:
          example: true
          type: boolean
      type: object
    response.InstructionAdherenceResponse:
      properties:
        instruction_adherence_score:
          example: 0.87
          type: number
      type: object
    response.CreateEvaluatorResponse:
      properties:
        bindings:
          items:
            type: string
          type: array
        evaluator_id:
          type: string
        slug:
          type: string
        type:
          $ref: '#/components/schemas/types.EvaluatorType'
        version_id:
          type: string
      type: object
    response.AnswerRelevancyResponse:
      properties:
        is_relevant:
          example: true
          type: boolean
      type: object
    request.JSONValidatorRequest:
      properties:
        config:
          $ref: '#/components/schemas/request.JSONValidatorConfigRequest'
        input:
          $ref: '#/components/schemas/request.JSONValidatorInput'
      required:
      - input
      type: object
    request.PlaceholderRegexInput:
      properties:
        placeholder_name:
          example: name
          type: string
        placeholder_value:
          example: Hello ${name}
          type: string
        text:
          example: Hello John Doe
          type: string
      required:
      - placeholder_name
      - placeholder_value
      - text
      type: object
    response.SemanticSimilarityResponse:
      properties:
        similarity_score:
          example: 0.92
          type: number
      type: object
    response.PIIDetectorResponse:
      properties:
        has_pii:
          example: false
          type: boolean
      type: object
    request.SemanticSimilarityRequest:
      properties:
        input:
          $ref: '#/components/schemas/request.SemanticSimilarityInput'
      required:
      - input
      type: object
    request.AgentGoalAccuracyInput:
      properties:
        completion:
          example: I have booked your flight from New York to Los Angeles departing Monday at 9am.
          type: string
        question:
          example: Book a flight from NYC to LA for next Monday
          type: string
        reference:
          example: 'Flight booked: NYC to LA, Monday departure'
          type: string
      required:
      - completion
      - question
      - reference
      type: object
    response.ToneDetectionResponse:
      properties:
        score:
          example: 0.95
          type: number
        tone:
          example: neutral
          type: string
      type: object
    request.RegexValidatorRequest:
      properties:
        config:
          $ref: '#/components/schemas/request.RegexValidatorConfigRequest'
        input:
          $ref: '#/components/schemas/request.RegexValidatorInput'
      required:
      - input
      type: object
    request.AgentToolTrajectoryInput:
      properties:
        executed_tool_calls:
          example: '[{"name": "search", "input": {"query": "weather"}}]'
          type: string
        expected_tool_calls:
          example: '[{"name": "search", "input": {"query": "weather"}}]'
          type: string
      required:
      - executed_tool_calls
      - expected_tool_calls
      type: object
    response.AgentEfficiencyResponse:
      properties:
        step_efficiency_reason:
          example: Agent completed task with minimal redundant steps
          type: string
        step_efficiency_score:
          example: 0.85
          type: number
        task_completion_reason:
          example: All required tasks were completed successfully
          type: string
        task_completion_score:
          example: 0.92
          type: number
      type: object
    request.IntentChangeRequest:
      properties:
        input:
          $ref: '#/components/schemas/request.IntentChangeInput'
      required:
      - input
      type: object
    request.InstructionAdherenceRequest:
      properties:
        input:
          $ref: '#/components/schemas/request.InstructionAdherenceInput'
      required:
      - input
      type: object
    request.WordCountRatioInput:
      properties:
        denominator_text:
          example: This is a longer input text for comparison
          type: string
        numerator_text:
          example: Short response
          type: string
      required:
      - denominator_text
      - numerator_text
      type: object
    request.ConversationQualityInput:
      properties:
        completions:
          example: '["Hi! I''d be happy to assist you today.", "We offer consulting, development, and support services."]'
          type: string
        prompts:
          example: '["Hello, how can I help?", "What services do you offer?"]'
          type: string
      required:
      - completions
      - prompts
      type: object
    request.SQLValidatorRequest:
      properties:
        input:
          $ref: '#/components/schemas/request.SQLValidatorInput'
      required:
      - input
      type: object
    request.SexismDetectorInput:
      properties:
        text:
          example: All team members should be treated equally regardless of gender.
          type: string
      required:
      - text
      type: object
    response.ContextRelevanceResponse:
      properties:
        relevance_score:
          example: 0.88
          type: number
      type: object
    response.PlaceholderRegexResponse:
      properties:
        is_valid_regex:
          example: true
          type: boolean
      type: object
    response.CharCountResponse:
      properties:
        char_count:
          example: 42
          type: integer
      type: object
    request.ToxicityDetectorRequest:
      properties:
        config:
          $ref: '#/components/schemas/request.ToxicityDetectorConfigRequest'
        input:
          $ref: '#/components/schemas/request.ToxicityDetectorInput'
      required:
      - input
      type: object
    request.JSONValidatorConfigRequest:
      properties:
        enable_schema_validation:
          example: true
          type: boolean
        schema_string:
          example: '{}'
          type: string
      type: object
    request.AgentGoalAccuracyRequest:
      properties:
        input:
          $ref: '#/components/schemas/request.AgentGoalAccuracyInput'
      required:
      - input
      type: object
    response.ConversationQualityResponse:
      properties:
        conversation_quality_score:
          example: 0.82
          type: number
      type: object
  securitySchemes:
    BearerAuth:
      description: Type "Bearer" followed by a space and JWT token.
      in: header
      name: Authorization
      type: apiKey