openapi: 3.1.0
info:
  title: Arthur GenAI Engine Agent Discovery LLMEvals API
  version: 2.1.688
tags:
- name: LLMEvals
paths:
  /api/v1/tasks/{task_id}/llm_evals/{eval_name}/versions/{eval_version}:
    get:
      tags:
      - LLMEvals
      summary: Get an llm eval
      description: Get an llm eval by name and version
      operationId: get_llm_eval_api_v1_tasks__task_id__llm_evals__eval_name__versions__eval_version__get
      security:
      - API Key: []
      parameters:
      - name: eval_name
        in: path
        required: true
        schema:
          type: string
          title: Eval Name
      - name: eval_version
        in: path
        required: true
        schema:
          type: string
          title: LLM Eval Version
          description: The version of the llm eval to retrieve. Can be 'latest', a version number (e.g. '1', '2', etc.), an ISO datetime string (e.g. '2025-01-01T00:00:00'), or a tag.
        description: The version of the llm eval to retrieve. Can be 'latest', a version number (e.g. '1', '2', etc.), an ISO datetime string (e.g. '2025-01-01T00:00:00'), or a tag.
      - name: task_id
        in: path
        required: true
        schema:
          type: string
          format: uuid
          title: Task Id
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Eval'
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
    delete:
      tags:
      - LLMEvals
      summary: Delete an llm eval version
      description: Deletes a specific version of an llm eval
      operationId: soft_delete_llm_eval_version_api_v1_tasks__task_id__llm_evals__eval_name__versions__eval_version__delete
      security:
      - API Key: []
      parameters:
      - name: eval_name
        in: path
        required: true
        schema:
          type: string
          title: Eval Name
      - name: eval_version
        in: path
        required: true
        schema:
          type: string
          title: LLM Eval Version
          description: The version of the llm eval to delete. Can be 'latest', a version number (e.g. '1', '2', etc.), an ISO datetime string (e.g. '2025-01-01T00:00:00'), or a tag.
        description: The version of the llm eval to delete. Can be 'latest', a version number (e.g. '1', '2', etc.), an ISO datetime string (e.g. '2025-01-01T00:00:00'), or a tag.
      - name: task_id
        in: path
        required: true
        schema:
          type: string
          format: uuid
          title: Task Id
      responses:
        '204':
          description: LLM eval version deleted.
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
  /api/v1/tasks/{task_id}/llm_evals/{eval_name}/versions:
    get:
      tags:
      - LLMEvals
      summary: List all versions of an llm eval
      description: List all versions of an llm eval with optional filtering.
      operationId: get_all_llm_eval_versions_api_v1_tasks__task_id__llm_evals__eval_name__versions_get
      security:
      - API Key: []
      parameters:
      - name: eval_name
        in: path
        required: true
        schema:
          type: string
          title: Eval Name
      - name: task_id
        in: path
        required: true
        schema:
          type: string
          format: uuid
          title: Task Id
      - name: sort
        in: query
        required: false
        schema:
          $ref: '#/components/schemas/PaginationSortMethod'
          description: Sort the results (asc/desc)
          default: desc
        description: Sort the results (asc/desc)
      - name: page_size
        in: query
        required: false
        schema:
          type: integer
          description: Page size. Default is 10. Must be greater than 0 and less than 5000.
          default: 10
          title: Page Size
        description: Page size. Default is 10. Must be greater than 0 and less than 5000.
      - name: page
        in: query
        required: false
        schema:
          type: integer
          description: Page number
          default: 0
          title: Page
        description: Page number
      - name: model_provider
        in: query
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          description: Filter by model provider (e.g., 'openai', 'anthropic', 'azure').
          title: Model Provider
        description: Filter by model provider (e.g., 'openai', 'anthropic', 'azure').
      - name: model_name
        in: query
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          description: Filter by model name (e.g., 'gpt-4', 'claude-3-5-sonnet').
          title: Model Name
        description: Filter by model name (e.g., 'gpt-4', 'claude-3-5-sonnet').
      - name: created_after
        in: query
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          description: Inclusive start date for prompt creation in ISO8601 string format. Use local time (not UTC).
          title: Created After
        description: Inclusive start date for prompt creation in ISO8601 string format. Use local time (not UTC).
      - name: created_before
        in: query
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          description: Exclusive end date for prompt creation in ISO8601 string format. Use local time (not UTC).
          title: Created Before
        description: Exclusive end date for prompt creation in ISO8601 string format. Use local time (not UTC).
      - name: exclude_deleted
        in: query
        required: false
        schema:
          type: boolean
          description: Whether to exclude deleted prompt versions from the results. Default is False.
          default: false
          title: Exclude Deleted
        description: Whether to exclude deleted prompt versions from the results. Default is False.
      - name: min_version
        in: query
        required: false
        schema:
          anyOf:
          - type: integer
            minimum: 1
          - type: 'null'
          description: Minimum version number to filter on (inclusive).
          title: Min Version
        description: Minimum version number to filter on (inclusive).
      - name: max_version
        in: query
        required: false
        schema:
          anyOf:
          - type: integer
            minimum: 1
          - type: 'null'
          description: Maximum version number to filter on (inclusive).
          title: Max Version
        description: Maximum version number to filter on (inclusive).
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/LLMEvalsVersionListResponse'
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
  /api/v1/tasks/{task_id}/llm_evals:
    get:
      tags:
      - LLMEvals
      summary: Get all llm evals
      description: Get all llm evals for a given task with optional filtering.
      operationId: get_all_llm_evals_api_v1_tasks__task_id__llm_evals_get
      security:
      - API Key: []
      parameters:
      - name: task_id
        in: path
        required: true
        schema:
          type: string
          format: uuid
          title: Task Id
      - name: sort_by
        in: query
        required: false
        schema:
          $ref: '#/components/schemas/LLMMetadataSortField'
          description: Field to sort the metadata list by. 'name' (default) preserves the historical alphabetical ordering. 'latest_version_created_at' orders by the most recent version's creation timestamp so that pagination matches the 'last updated' display.
          default: name
        description: Field to sort the metadata list by. 'name' (default) preserves the historical alphabetical ordering. 'latest_version_created_at' orders by the most recent version's creation timestamp so that pagination matches the 'last updated' display.
      - name: sort
        in: query
        required: false
        schema:
          $ref: '#/components/schemas/PaginationSortMethod'
          description: Sort the results (asc/desc)
          default: desc
        description: Sort the results (asc/desc)
      - name: page_size
        in: query
        required: false
        schema:
          type: integer
          description: Page size. Default is 10. Must be greater than 0 and less than 5000.
          default: 10
          title: Page Size
        description: Page size. Default is 10. Must be greater than 0 and less than 5000.
      - name: page
        in: query
        required: false
        schema:
          type: integer
          description: Page number
          default: 0
          title: Page
        description: Page number
      - name: llm_asset_names
        in: query
        required: false
        schema:
          anyOf:
          - type: array
            items:
              type: string
          - type: 'null'
          description: LLM asset names to filter on using partial matching. If provided, llm assets matching any of these name patterns will be returned
          title: Llm Asset Names
        description: LLM asset names to filter on using partial matching. If provided, llm assets matching any of these name patterns will be returned
      - name: model_provider
        in: query
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          description: Filter by model provider (e.g., 'openai', 'anthropic', 'azure').
          title: Model Provider
        description: Filter by model provider (e.g., 'openai', 'anthropic', 'azure').
      - name: model_name
        in: query
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          description: Filter by model name (e.g., 'gpt-4', 'claude-3-5-sonnet').
          title: Model Name
        description: Filter by model name (e.g., 'gpt-4', 'claude-3-5-sonnet').
      - name: created_after
        in: query
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          description: Inclusive start date for prompt creation in ISO8601 string format. Use local time (not UTC).
          title: Created After
        description: Inclusive start date for prompt creation in ISO8601 string format. Use local time (not UTC).
      - name: created_before
        in: query
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          description: Exclusive end date for prompt creation in ISO8601 string format. Use local time (not UTC).
          title: Created Before
        description: Exclusive end date for prompt creation in ISO8601 string format. Use local time (not UTC).
      - name: tags
        in: query
        required: false
        schema:
          anyOf:
          - type: array
            items:
              type: string
          - type: 'null'
          description: List of tags to filter for items that have any matching tag across any version.
          title: Tags
        description: List of tags to filter for items that have any matching tag across any version.
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/LLMGetAllMetadataListResponse'
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
  /api/v1/tasks/{task_id}/llm_evals/{eval_name}/versions/{eval_version}/completions:
    post:
      tags:
      - LLMEvals
      summary: Run a saved llm eval
      description: Run a saved llm eval
      operationId: run_saved_llm_eval_api_v1_tasks__task_id__llm_evals__eval_name__versions__eval_version__completions_post
      security:
      - API Key: []
      parameters:
      - name: eval_name
        in: path
        required: true
        schema:
          type: string
          title: Eval Name
      - name: eval_version
        in: path
        required: true
        schema:
          type: string
          title: LLM Eval Version
          description: The version of the llm eval to run. Can be 'latest', a version number (e.g. '1', '2', etc.), an ISO datetime string (e.g. '2025-01-01T00:00:00'), or a tag.
        description: The version of the llm eval to run. Can be 'latest', a version number (e.g. '1', '2', etc.), an ISO datetime string (e.g. '2025-01-01T00:00:00'), or a tag.
      - name: task_id
        in: path
        required: true
        schema:
          type: string
          format: uuid
          title: Task Id
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/BaseCompletionRequest'
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/EvalRunResponse'
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
  /api/v1/tasks/{task_id}/llm_evals/{eval_name}:
    post:
      tags:
      - LLMEvals
      summary: Save an llm eval
      description: Save an llm eval to the database
      operationId: save_llm_eval_api_v1_tasks__task_id__llm_evals__eval_name__post
      security:
      - API Key: []
      parameters:
      - name: eval_name
        in: path
        required: true
        schema:
          type: string
          title: Eval Name
      - name: task_id
        in: path
        required: true
        schema:
          type: string
          format: uuid
          title: Task Id
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/CreateEvalRequest'
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Eval'
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
    delete:
      tags:
      - LLMEvals
      summary: Delete an llm eval
      description: Deletes an entire llm eval
      operationId: delete_llm_eval_api_v1_tasks__task_id__llm_evals__eval_name__delete
      security:
      - API Key: []
      parameters:
      - name: eval_name
        in: path
        required: true
        schema:
          type: string
          title: Eval Name
      - name: task_id
        in: path
        required: true
        schema:
          type: string
          format: uuid
          title: Task Id
      responses:
        '204':
          description: LLM eval deleted.
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
  /api/v1/tasks/{task_id}/llm_evals/{eval_name}/versions/tags/{tag}:
    get:
      tags:
      - LLMEvals
      summary: Get an llm eval by name and tag
      description: Get an llm eval by name and tag
      operationId: get_llm_eval_by_tag_api_v1_tasks__task_id__llm_evals__eval_name__versions_tags__tag__get
      security:
      - API Key: []
      parameters:
      - name: eval_name
        in: path
        required: true
        schema:
          type: string
          title: Eval Name
      - name: tag
        in: path
        required: true
        schema:
          type: string
          title: Tag
      - name: task_id
        in: path
        required: true
        schema:
          type: string
          format: uuid
          title: Task Id
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Eval'
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
  /api/v1/tasks/{task_id}/llm_evals/{eval_name}/versions/{eval_version}/tags:
    put:
      tags:
      - LLMEvals
      summary: Add a tag to an llm eval version
      description: Add a tag to an llm eval version
      operationId: add_tag_to_llm_eval_version_api_v1_tasks__task_id__llm_evals__eval_name__versions__eval_version__tags_put
      security:
      - API Key: []
      parameters:
      - name: eval_name
        in: path
        required: true
        schema:
          type: string
          title: Eval Name
      - name: eval_version
        in: path
        required: true
        schema:
          type: string
          title: LLM Eval Version
          description: The version of the llm eval to retrieve. Can be 'latest', a version number (e.g. '1', '2', etc.), an ISO datetime string (e.g. '2025-01-01T00:00:00'), or a tag.
        description: The version of the llm eval to retrieve. Can be 'latest', a version number (e.g. '1', '2', etc.), an ISO datetime string (e.g. '2025-01-01T00:00:00'), or a tag.
      - name: task_id
        in: path
        required: true
        schema:
          type: string
          format: uuid
          title: Task Id
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/Body_add_tag_to_llm_eval_version_api_v1_tasks__task_id__llm_evals__eval_name__versions__eval_version__tags_put'
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Eval'
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
  /api/v1/tasks/{task_id}/llm_evals/{eval_name}/versions/{eval_version}/tags/{tag}:
    delete:
      tags:
      - LLMEvals
      summary: Remove a tag from an llm eval version
      description: Remove a tag from an llm eval version
      operationId: delete_tag_from_llm_eval_version_api_v1_tasks__task_id__llm_evals__eval_name__versions__eval_version__tags__tag__delete
      security:
      - API Key: []
      parameters:
      - name: eval_name
        in: path
        required: true
        schema:
          type: string
          title: Eval Name
      - name: eval_version
        in: path
        required: true
        schema:
          type: string
          title: LLM Eval Version
          description: The version of the llm eval to retrieve. Can be 'latest', a version number (e.g. '1', '2', etc.), an ISO datetime string (e.g. '2025-01-01T00:00:00'), or a tag.
        description: The version of the llm eval to retrieve. Can be 'latest', a version number (e.g. '1', '2', etc.), an ISO datetime string (e.g. '2025-01-01T00:00:00'), or a tag.
      - name: tag
        in: path
        required: true
        schema:
          type: string
          title: Tag
      - name: task_id
        in: path
        required: true
        schema:
          type: string
          format: uuid
          title: Task Id
      responses:
        '204':
          description: Tag removed.
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
components:
  schemas:
    EvalKind:
      type: string
      enum:
      - llm_as_a_judge
      - pii
      - pii_v1
      - toxicity
      - prompt_injection
      title: EvalKind
      description: Discriminator for all eval types stored in the llm_evals table.
    ValidationError:
      properties:
        loc:
          items:
            anyOf:
            - type: string
            - type: integer
          type: array
          title: Location
        msg:
          type: string
          title: Message
        type:
          type: string
          title: Error Type
        input:
          title: Input
        ctx:
          type: object
          title: Context
      type: object
      required:
      - loc
      - msg
      - type
      title: ValidationError
    HTTPValidationError:
      properties:
        detail:
          items:
            $ref: '#/components/schemas/ValidationError'
          type: array
          title: Detail
      type: object
      title: HTTPValidationError
    LLMMetadataSortField:
      type: string
      enum:
      - name
      - latest_version_created_at
      title: LLMMetadataSortField
      description: Sort field options for the LLM evals/prompts metadata list endpoints.
    LLMBaseConfigSettings:
      properties:
        timeout:
          anyOf:
          - type: number
          - type: 'null'
          title: Timeout
          description: Request timeout in seconds
        temperature:
          anyOf:
          - type: number
          - type: 'null'
          title: Temperature
          description: Sampling temperature (0.0 to 2.0). Higher values make output more random
        top_p:
          anyOf:
          - type: number
          - type: 'null'
          title: Top P
          description: Top-p sampling parameter (0.0 to 1.0). Alternative to temperature
        max_tokens:
          anyOf:
          - type: integer
          - type: 'null'
          title: Max Tokens
          description: Maximum number of tokens to generate in the response
        stop:
          anyOf:
          - type: string
          - type: 'null'
          title: Stop
          description: Stop sequence(s) where the model should stop generating
        presence_penalty:
          anyOf:
          - type: number
          - type: 'null'
          title: Presence Penalty
          description: Presence penalty (-2.0 to 2.0). Positive values penalize new tokens based on their presence
        frequency_penalty:
          anyOf:
          - type: number
          - type: 'null'
          title: Frequency Penalty
          description: Frequency penalty (-2.0 to 2.0). Positive values penalize tokens based on frequency
        seed:
          anyOf:
          - type: integer
          - type: 'null'
          title: Seed
          description: Random seed for reproducible outputs
        logprobs:
          anyOf:
          - type: boolean
          - type: 'null'
          title: Logprobs
          description: Whether to return log probabilities of output tokens
        top_logprobs:
          anyOf:
          - type: integer
          - type: 'null'
          title: Top Logprobs
          description: Number of most likely tokens to return log probabilities for (1-20)
        logit_bias:
          anyOf:
          - items:
              $ref: '#/components/schemas/LogitBiasItem'
            type: array
          - type: 'null'
          title: Logit Bias
          description: Modify likelihood of specified tokens appearing in completion
        max_completion_tokens:
          anyOf:
          - type: integer
          - type: 'null'
          title: Max Completion Tokens
          description: Maximum number of completion tokens (alternative to max_tokens)
        reasoning_effort:
          anyOf:
          - $ref: '#/components/schemas/ReasoningEffortEnum'
          - type: 'null'
          description: Reasoning effort level for models that support it (e.g., OpenAI o1 series)
        thinking:
          anyOf:
          - $ref: '#/components/schemas/AnthropicThinkingParam-Output'
          - type: 'null'
          description: Anthropic-specific thinking parameter for Claude models
      additionalProperties: false
      type: object
      title: LLMBaseConfigSettings
    ModelProvider:
      type: string
      enum:
      - anthropic
      - openai
      - gemini
      - bedrock
      - vertex_ai
      - hosted_vllm
      - azure
      title: ModelProvider
    ReasoningEffortEnum:
      type: string
      enum:
      - none
      - minimal
      - low
      - medium
      - high
      - default
      title: ReasoningEffortEnum
    BaseCompletionRequest:
      properties:
        variables:
          anyOf:
          - items:
              $ref: '#/components/schemas/VariableTemplateValue'
            type: array
          - type: 'null'
          title: Variables
          description: List of VariableTemplateValue fields that specify the values to fill in for each template in the prompt
          default: []
      type: object
      title: BaseCompletionRequest
    VariableTemplateValue:
      properties:
        name:
          type: string
          title: Name
          description: Name of the variable
        value:
          type: string
          title: Value
          description: Value of the variable
      type: object
      required:
      - name
      - value
      title: VariableTemplateValue
    LogitBiasItem:
      properties:
        token_id:
          type: integer
          title: Token Id
          description: Token ID to bias
        bias:
          type: number
          maximum: 100.0
          minimum: -100.0
          title: Bias
          description: Bias value between -100 and 100
      type: object
      required:
      - token_id
      - bias
      title: LogitBiasItem
    Eval:
      properties:
        name:
          type: string
          title: Name
          description: Name of the llm eval
        eval_kind:
          type: string
          title: Eval Kind
          description: Eval kind discriminator (e.g. 'llm_as_a_judge', 'pii', 'toxicity')
          default: llm_as_a_judge
        model_name:
          anyOf:
          - type: string
          - type: 'null'
          title: Model Name
          description: Name of the LLM model (e.g., 'gpt-4o', 'claude-3-sonnet'). None for ML evals.
        model_provider:
          anyOf:
          - $ref: '#/components/schemas/ModelProvider'
          - type: 'null'
          description: Provider of the LLM model (e.g., 'openai', 'anthropic', 'azure'). None for ML evals.
        instructions:
          anyOf:
          - type: string
          - type: 'null'
          title: Instructions
          description: Instructions for the llm eval. None for ML evals.
        variables:
          items:
            type: string
          type: array
          title: Variables
          description: List of variable names for the llm eval
        tags:
          items:
            type: string
          type: array
          title: Tags
          description: List of tags for this llm eval version
        config:
          anyOf:
          - $ref: '#/components/schemas/LLMBaseConfigSettings'
          - additionalProperties: true
            type: object
          - type: 'null'
          title: EvalConfig
          description: Eval configuration. LLMBaseConfigSettings for LLM evals; type-specific dict for ML evals.
        created_at:
          type: string
          format: date-time
          title: Created At
          description: Timestamp when the llm eval was created.
        deleted_at:
          anyOf:
          - type: string
            format: date-time
          - type: 'null'
          title: Deleted At
          description: Time that this llm eval was deleted
        version:
          type: integer
          title: Version
          description: Version of the llm eval
          default: 1
      type: object
      required:
      - name
      - created_at
      title: Eval
    LLMVersionResponse:
      properties:
        version:
          type: integer
          title: Version
          description: Version number of the llm eval
        eval_kind:
          $ref: '#/components/schemas/EvalKind'
          description: Eval kind discriminator (e.g. 'llm_as_a_judge', 'pii', 'toxicity')
          default: llm_as_a_judge
        created_at:
          type: string
          format: date-time
          title: Created At
          description: Timestamp when the llm eval version was created
        deleted_at:
          anyOf:
          - type: string
            format: date-time
          - type: 'null'
          title: Deleted At
          description: Timestamp when the llm eval version was deleted (None if not deleted)
        model_provider:
          anyOf:
          - $ref: '#/components/schemas/ModelProvider'
          - type: 'null'
          description: Model provider chosen for this version of the llm eval. None for ML evals.
        model_name:
          anyOf:
          - type: string
          - type: 'null'
          title: Model Name
          description: Model name chosen for this version of the llm eval. None for ML evals.
        tags:
          items:
            type: string
          type: array
          title: Tags
          description: List of tags for the llm asset
      type: object
      required:
      - version
      - created_at
      - deleted_at
      title: LLMVersionResponse
    PaginationSortMethod:
      type: string
      enum:
      - asc
      - desc
      title: PaginationSortMethod
    LLMRequestConfigSettings:
      properties:
        timeout:
          anyOf:
          - type: number
          - type: 'null'
          title: Timeout
          description: Request timeout in seconds
        temperature:
          anyOf:
          - type: number
          - type: 'null'
          title: Temperature
          description: Sampling temperature (0.0 to 2.0). Higher values make output more random
        top_p:
          anyOf:
          - type: number
          - type: 'null'
          title: Top P
          description: Top-p sampling parameter (0.0 to 1.0). Alternative to temperature
        max_tokens:
          anyOf:
          - type: integer
          - type: 'null'
          title: Max Tokens
          description: Maximum number of tokens to generate in the response
        stop:
          anyOf:
          - type: string
          - type: 'null'
          title: Stop
          description: Stop sequence(s) where the model should stop generating
        presence_penalty:
          anyOf:
          - type: number
          - type: 'null'
          title: Presence Penalty
          description: Presence penalty (-2.0 to 2.0). Positive values penalize new tokens based on their presence
        frequency_penalty:
          anyOf:
          - type: number
          - type: 'null'
          title: Frequency Penalty
          description: Frequency penalty (-2.0 to 2.0). Positive values penalize tokens based on frequency
        seed:
          anyOf:
          - type: integer
          - type: 'null'
          title: Seed
          description: Random seed for reproducible outputs
        logprobs:
          anyOf:
          - type: boolean
          - type: 'null'
          title: Logprobs
          description: Whether to return log probabilities of output tokens
        top_logprobs:
          anyOf:
          - type: integer
          - type: 'null'
          title: Top Logprobs
          description: Number of most likely tokens to return log probabilities for (1-20)
        logit_bias:
          anyOf:
          - items:
              $ref: '#/components/schemas/LogitBiasItem'
            type: array
          - type: 'null'
          title: Logit Bias
          description: Modify likelihood of specified tokens appearing in completion
        max_completion_tokens:
          anyOf:
          - type: integer
          - type: 'null'
          title: Max Completion Tokens
          description: Maximum number of completion tokens (alternative to max_tokens)
        reasoning_effort:
          anyOf:
          - $ref: '#/components/schemas/ReasoningEffortEnum'
          - type: 'null'
          description: Reasoning effort level for models that support it (e.g., OpenAI o1 series)
        thinking:
          anyOf:
          - $ref: '#/components/schemas/AnthropicThinkingParam-Input'
          - type: 'null'
          description: Anthropic-specific thinking parameter for Claude models
      type: object
      title: LLMRequestConfigSettings
    LLMGetAllMetadataListResponse:
      properties:
        llm_metadata:
          items:
            $ref: '#/components/schemas/LLMGetAllMetadataResponse'
          type: array
          title: Llm Metadata
          description: List of llm asset metadata
        count:
          type: integer
          title: Count
          description: Total number of llm assets matching filters
      type: object
      required:
      - llm_metadata
      - count
      title: LLMGetAllMetadataListResponse
    CreateEvalRequest:
      properties:
        model_name:
          type: string
          title: Model Name
          description: Name of the LLM model (e.g., 'gpt-4o', 'claude-3-sonnet')
        model_provider:
          $ref: '#/components/schemas/ModelProvider'
          description: Provider of the LLM model (e.g., 'openai', 'anthropic', 'azure')
        instructions:
          type: string
          title: Instructions
          description: Instructions for the llm eval
        config:
          anyOf:
          - $ref: '#/components/schemas/LLMRequestConfigSettings'
          - type: 'null'
          description: LLM configurations for this eval (e.g. temperature, max_tokens, etc.)
      type: object
      required:
      - model_name
      - model_provider
      - instructions
      title: CreateEvalRequest
    LLMGetAllMetadataResponse:
      properties:
        name:
          type: string
          title: Name
          description: Name of the llm asset
        eval_kind:
          $ref: '#/components/schemas/EvalKind'
          description: Eval kind discriminator (e.g. 'llm_as_a_judge', 'pii', 'toxicity')
          default: llm_as_a_judge
        versions:
          type: integer
          title: Versions
          description: Number of versions of the llm asset
        tags:
          items:
            type: string
          type: array
          title: Tags
          description: List of tags for the llm asset
        created_at:
          type: string
          format: date-time
          title: Created At
          description: Timestamp when the llm asset was created
        latest_version_created_at:
          type: string
          format: date-time
          title: Latest Version Created At
          description: Timestamp when the last version of the llm asset was created
        deleted_versions:
          items:
            type: integer
          type: array
          title: Deleted Versions
          description: List of deleted versions of the llm asset
      type: object
      required:
      - name
      - versions
      - created_at
      - latest_version_created_at
      - deleted_versions
      title: LLMGetAllMetadataResponse
    LLMEvalsVersionListResponse:
      properties:
        versions:
          items:
            $ref: '#/components/schemas/LLMVersionResponse'
          type: array
          title: Versions
          description: List of llm eval version metadata
        count:
          type: integer
          title: Count
          description: Total number of llm evals matching filters
      type: object
      required:
      - versions
      - count
      title: LLMEvalsVersionListResponse
    AnthropicThinkingParam-Output:
      properties:
        type:
          type: string
          enum:
          - enabled
          - adaptive
          title: Type
        budget_tokens:
          type: integer
          title: Budget Tokens
      additionalProperties: false
      type: object
      title: AnthropicThinkingParam
    AnthropicThinkingParam-Input:
      properties:
        type:
          type: string
          enum:
          - enabled
          - adaptive
          title: Type
        budget_tokens:
          type: integer
          title: Budget Tokens
      type: object
      title: AnthropicThinkingParam
    Body_add_tag_to_llm_eval_version_api_v1_tasks__task_id__llm_evals__eval_name__versions__eval_version__tags_put:
      properties:
        tag:
          type: string
          title: Tag
          description: Tag to add to this llm eval version
      type: object
      required:
      - tag
      title: Body_add_tag_to_llm_eval_version_api_v1_tasks__task_id__llm_evals__eval_name__versions__eval_version__tags_put
    EvalRunResponse:
      properties:
        reason:
          type: string
          title: Reason
          description: Explanation for how the llm arrived at this answer.
        score:
          type: integer
          title: Score
          description: Score for this llm eval
        cost:
          type: string
          title: Cost
          description: Cost of this llm completion
      type: object
      required:
      - reason
      - score
      - cost
      title: EvalRunResponse
  securitySchemes:
    API Key:
      type: http
      description: Bearer token authentication with an API key
      scheme: bearer