openapi: 3.2.0
info:
  title: LiteLLM Evals API
  description: 'Proxy Server to call 100+ LLMs in the OpenAI format. **Customize Swagger Docs**


    👉 ```LiteLLM Admin Panel on /ui```. Create, Edit Keys with SSO. Having issues? Try ```Fallback Login```


    💸 ```LiteLLM Model Cost Map```.


    🔎 ```LiteLLM Model Hub```. See available models on the proxy. **Docs**'
  version: 1.102.1
tags:
- name: Evals
paths:
  /v1/evals:
    get:
      description: 'List evaluations with pagination.


        Model-based routing (for multi-account support):

        - Pass model via header: `x-litellm-model: gpt-4-account-1`

        - Pass model via query: `?model=gpt-4-account-1`

        - Pass model via body: `{"model": "gpt-4-account-1"}`


        Example usage:

        ```bash

        curl "http://localhost:4000/v1/evals?limit=10" -H "Authorization: Bearer your-key"

        ```


        Returns: ListEvalsResponse with list of evaluations'
      operationId: list_evals_v1_evals_get
      parameters:
      - in: query
        name: limit
        required: false
        schema:
          anyOf:
          - type: integer
          - type: 'null'
          default: 20
          title: Limit
      - in: query
        name: after
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          title: After
      - in: query
        name: before
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          title: Before
      - in: query
        name: order
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          title: Order
      - in: query
        name: order_by
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          title: Order By
      - in: query
        name: custom_llm_provider
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          default: openai
          title: Custom Llm Provider
      responses:
        '200':
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ListEvalsResponse'
          description: Successful Response
        '422':
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
          description: Validation Error
      security:
      - APIKeyHeader: []
      summary: List Evals
      tags:
      - Evals
    post:
      description: 'Create a new evaluation.


        Model-based routing (for multi-account support):

        - Pass model via header: `x-litellm-model: gpt-4-account-1`

        - Pass model via query: `?model=gpt-4-account-1`

        - Pass model via body: `{"model": "gpt-4-account-1"}`


        Example usage:

        ```bash

        curl -X POST "http://localhost:4000/v1/evals" -H "Authorization: Bearer your-key" -H "Content-Type: application/json" -d ''{

        "name": "Test Eval",

        "data_source_config": {"type": "file", "file_id": "file-abc123"},

        "testing_criteria": {"graders": [{"type": "llm_as_judge"}]}

        }''

        ```


        Returns: Eval object with id, status, timestamps, etc.'
      operationId: create_eval_v1_evals_post
      parameters:
      - in: query
        name: custom_llm_provider
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          default: openai
          title: Custom Llm Provider
      responses:
        '200':
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Eval'
          description: Successful Response
        '422':
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
          description: Validation Error
      security:
      - APIKeyHeader: []
      summary: Create Eval
      tags:
      - Evals
  /v1/evals/{eval_id}:
    delete:
      description: 'Delete an evaluation.


        Model-based routing (for multi-account support):

        - Pass model via header: `x-litellm-model: gpt-4-account-1`

        - Pass model via query: `?model=gpt-4-account-1`

        - Pass model via body: `{"model": "gpt-4-account-1"}`


        Example usage:

        ```bash

        curl -X DELETE "http://localhost:4000/v1/evals/eval_123" -H "Authorization: Bearer your-key"

        ```


        Returns: DeleteEvalResponse with deletion confirmation'
      operationId: delete_eval_v1_evals__eval_id__delete
      parameters:
      - in: path
        name: eval_id
        required: true
        schema:
          title: Eval Id
          type: string
      - in: query
        name: custom_llm_provider
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          default: openai
          title: Custom Llm Provider
      responses:
        '200':
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/DeleteEvalResponse'
          description: Successful Response
        '422':
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
          description: Validation Error
      security:
      - APIKeyHeader: []
      summary: Delete Eval
      tags:
      - Evals
    get:
      description: 'Get a specific evaluation by ID.


        Model-based routing (for multi-account support):

        - Pass model via header: `x-litellm-model: gpt-4-account-1`

        - Pass model via query: `?model=gpt-4-account-1`

        - Pass model via body: `{"model": "gpt-4-account-1"}`


        Example usage:

        ```bash

        curl "http://localhost:4000/v1/evals/eval_123" -H "Authorization: Bearer your-key"

        ```


        Returns: Eval object'
      operationId: get_eval_v1_evals__eval_id__get
      parameters:
      - in: path
        name: eval_id
        required: true
        schema:
          title: Eval Id
          type: string
      - in: query
        name: custom_llm_provider
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          default: openai
          title: Custom Llm Provider
      responses:
        '200':
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Eval'
          description: Successful Response
        '422':
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
          description: Validation Error
      security:
      - APIKeyHeader: []
      summary: Get Eval
      tags:
      - Evals
    post:
      description: 'Update an evaluation.


        Model-based routing (for multi-account support):

        - Pass model via header: `x-litellm-model: gpt-4-account-1`

        - Pass model via query: `?model=gpt-4-account-1`

        - Pass model via body: `{"model": "gpt-4-account-1"}`


        Example usage:

        ```bash

        curl -X POST "http://localhost:4000/v1/evals/eval_123" -H "Authorization: Bearer your-key" -H "Content-Type: application/json" -d ''{"name": "Updated Name"}''

        ```


        Returns: Updated Eval object'
      operationId: update_eval_v1_evals__eval_id__post
      parameters:
      - in: path
        name: eval_id
        required: true
        schema:
          title: Eval Id
          type: string
      - in: query
        name: custom_llm_provider
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          default: openai
          title: Custom Llm Provider
      responses:
        '200':
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Eval'
          description: Successful Response
        '422':
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
          description: Validation Error
      security:
      - APIKeyHeader: []
      summary: Update Eval
      tags:
      - Evals
  /v1/evals/{eval_id}/cancel:
    post:
      description: 'Cancel a running evaluation.


        Model-based routing (for multi-account support):

        - Pass model via header: `x-litellm-model: gpt-4-account-1`

        - Pass model via query: `?model=gpt-4-account-1`

        - Pass model via body: `{"model": "gpt-4-account-1"}`


        Example usage:

        ```bash

        curl -X POST "http://localhost:4000/v1/evals/eval_123/cancel" -H "Authorization: Bearer your-key"

        ```


        Returns: CancelEvalResponse with cancellation confirmation'
      operationId: cancel_eval_v1_evals__eval_id__cancel_post
      parameters:
      - in: path
        name: eval_id
        required: true
        schema:
          title: Eval Id
          type: string
      - in: query
        name: custom_llm_provider
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          default: openai
          title: Custom Llm Provider
      responses:
        '200':
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/CancelEvalResponse'
          description: Successful Response
        '422':
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
          description: Validation Error
      security:
      - APIKeyHeader: []
      summary: Cancel Eval
      tags:
      - Evals
  /v1/evals/{eval_id}/runs:
    get:
      description: 'List all runs for an evaluation with pagination.


        Model-based routing (for multi-account support):

        - Pass model via header: `x-litellm-model: gpt-4-account-1`

        - Pass model via query: `?model=gpt-4-account-1`


        Example usage:

        ```bash

        curl "http://localhost:4000/v1/evals/eval_123/runs?limit=10" -H "Authorization: Bearer your-key"

        ```


        Returns: ListRunsResponse with list of runs'
      operationId: list_runs_v1_evals__eval_id__runs_get
      parameters:
      - in: path
        name: eval_id
        required: true
        schema:
          title: Eval Id
          type: string
      - in: query
        name: limit
        required: false
        schema:
          anyOf:
          - type: integer
          - type: 'null'
          default: 20
          title: Limit
      - in: query
        name: after
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          title: After
      - in: query
        name: before
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          title: Before
      - in: query
        name: order
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          title: Order
      - in: query
        name: custom_llm_provider
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          default: openai
          title: Custom Llm Provider
      responses:
        '200':
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ListRunsResponse'
          description: Successful Response
        '422':
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
          description: Validation Error
      security:
      - APIKeyHeader: []
      summary: List Runs
      tags:
      - Evals
    post:
      description: 'Create a new run for an evaluation.


        Model-based routing (for multi-account support):

        - Pass model via header: `x-litellm-model: gpt-4-account-1`

        - Pass model via query: `?model=gpt-4-account-1`

        - Pass model via body: `{"model": "gpt-4-account-1"}`

        - Pass model via completion.model: `{"completion": {"model": "gpt-4-account-1"}}`


        Example usage:

        ```bash

        curl -X POST "http://localhost:4000/v1/evals/eval_123/runs" -H "Authorization: Bearer your-key" -H "Content-Type: application/json" -d ''{

        "data_source": {"type": "dataset", "dataset_id": "dataset_123"},

        "completion": {"model": "gpt-4", "temperature": 0.7}

        }''

        ```


        Returns: Run object with id, status, timestamps, etc.'
      operationId: create_run_v1_evals__eval_id__runs_post
      parameters:
      - in: path
        name: eval_id
        required: true
        schema:
          title: Eval Id
          type: string
      - in: query
        name: custom_llm_provider
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          default: openai
          title: Custom Llm Provider
      responses:
        '200':
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Run'
          description: Successful Response
        '422':
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
          description: Validation Error
      security:
      - APIKeyHeader: []
      summary: Create Run
      tags:
      - Evals
  /v1/evals/{eval_id}/runs/{run_id}:
    delete:
      description: 'Delete a run.


        Model-based routing (for multi-account support):

        - Pass model via header: `x-litellm-model: gpt-4-account-1`

        - Pass model via query: `?model=gpt-4-account-1`


        Example usage:

        ```bash

        curl -X DELETE "http://localhost:4000/v1/evals/eval_123/runs/run_456" -H "Authorization: Bearer your-key"

        ```


        Returns: RunDeleteResponse with deletion confirmation'
      operationId: delete_run_v1_evals__eval_id__runs__run_id__delete
      parameters:
      - in: path
        name: eval_id
        required: true
        schema:
          title: Eval Id
          type: string
      - in: path
        name: run_id
        required: true
        schema:
          title: Run Id
          type: string
      - in: query
        name: custom_llm_provider
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          default: openai
          title: Custom Llm Provider
      responses:
        '200':
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/RunDeleteResponse'
          description: Successful Response
        '422':
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
          description: Validation Error
      security:
      - APIKeyHeader: []
      summary: Delete Run
      tags:
      - Evals
    get:
      description: 'Get a specific run by ID.


        Model-based routing (for multi-account support):

        - Pass model via header: `x-litellm-model: gpt-4-account-1`

        - Pass model via query: `?model=gpt-4-account-1`


        Example usage:

        ```bash

        curl "http://localhost:4000/v1/evals/eval_123/runs/run_456" -H "Authorization: Bearer your-key"

        ```


        Returns: Run object with full details'
      operationId: get_run_v1_evals__eval_id__runs__run_id__get
      parameters:
      - in: path
        name: eval_id
        required: true
        schema:
          title: Eval Id
          type: string
      - in: path
        name: run_id
        required: true
        schema:
          title: Run Id
          type: string
      - in: query
        name: custom_llm_provider
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          default: openai
          title: Custom Llm Provider
      responses:
        '200':
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Run'
          description: Successful Response
        '422':
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
          description: Validation Error
      security:
      - APIKeyHeader: []
      summary: Get Run
      tags:
      - Evals
    post:
      description: 'Cancel a running run.


        Model-based routing (for multi-account support):

        - Pass model via header: `x-litellm-model: gpt-4-account-1`

        - Pass model via query: `?model=gpt-4-account-1`


        Example usage:

        ```bash

        curl -X POST "http://localhost:4000/v1/evals/eval_123/runs/run_456/cancel" -H "Authorization: Bearer your-key"

        ```


        Returns: CancelRunResponse with cancellation confirmation'
      operationId: cancel_run_v1_evals__eval_id__runs__run_id__post
      parameters:
      - in: path
        name: eval_id
        required: true
        schema:
          title: Eval Id
          type: string
      - in: path
        name: run_id
        required: true
        schema:
          title: Run Id
          type: string
      - in: query
        name: custom_llm_provider
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          default: openai
          title: Custom Llm Provider
      responses:
        '200':
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/CancelRunResponse'
          description: Successful Response
        '422':
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
          description: Validation Error
      security:
      - APIKeyHeader: []
      summary: Cancel Run
      tags:
      - Evals
components:
  schemas:
    DeleteEvalResponse:
      description: Response from deleting an evaluation
      properties:
        deleted:
          title: Deleted
          type: boolean
        eval_id:
          title: Eval Id
          type: string
        object:
          default: eval.deleted
          title: Object
          type: string
      required:
      - eval_id
      - deleted
      title: DeleteEvalResponse
      type: object
    ResultCounts:
      description: Result counts for a run
      properties:
        error:
          default: 0
          title: Error
          type: integer
        failed:
          default: 0
          title: Failed
          type: integer
        passed:
          default: 0
          title: Passed
          type: integer
        total:
          title: Total
          type: integer
      required:
      - total
      title: ResultCounts
      type: object
    ListRunsResponse:
      description: Response from listing runs
      properties:
        data:
          items:
            $ref: '#/components/schemas/Run'
          title: Data
          type: array
        first_id:
          anyOf:
          - type: string
          - type: 'null'
          title: First Id
        has_more:
          default: false
          title: Has More
          type: boolean
        last_id:
          anyOf:
          - type: string
          - type: 'null'
          title: Last Id
        object:
          default: list
          title: Object
          type: string
      required:
      - data
      title: ListRunsResponse
      type: object
    PerTestingCriteriaResult:
      description: Results for a specific testing criteria
      properties:
        average_score:
          anyOf:
          - type: number
          - type: 'null'
          title: Average Score
        result_counts:
          $ref: '#/components/schemas/ResultCounts'
        testing_criteria_index:
          title: Testing Criteria Index
          type: integer
      required:
      - testing_criteria_index
      - result_counts
      title: PerTestingCriteriaResult
      type: object
    CancelRunResponse:
      description: Response from cancelling a run
      properties:
        id:
          title: Id
          type: string
        object:
          default: eval.run
          title: Object
          type: string
        status:
          const: cancelled
          title: Status
          type: string
      required:
      - id
      - status
      title: CancelRunResponse
      type: object
    Run:
      description: Represents a run from the OpenAI Evals API
      properties:
        completed_at:
          anyOf:
          - type: integer
          - type: 'null'
          title: Completed At
        created_at:
          title: Created At
          type: integer
        data_source:
          additionalProperties: true
          title: Data Source
          type: object
        error:
          anyOf:
          - additionalProperties: true
            type: object
          - type: 'null'
          title: Error
        eval_id:
          title: Eval Id
          type: string
        id:
          title: Id
          type: string
        metadata:
          anyOf:
          - additionalProperties: true
            type: object
          - type: 'null'
          title: Metadata
        model:
          anyOf:
          - type: string
          - type: 'null'
          title: Model
        name:
          anyOf:
          - type: string
          - type: 'null'
          title: Name
        object:
          default: eval.run
          title: Object
          type: string
        per_model_usage:
          anyOf:
          - {}
          - type: 'null'
          title: Per Model Usage
        per_testing_criteria_results:
          anyOf:
          - items:
              $ref: '#/components/schemas/PerTestingCriteriaResult'
            type: array
          - type: 'null'
          title: Per Testing Criteria Results
        report_url:
          anyOf:
          - type: string
          - type: 'null'
          title: Report Url
        result_counts:
          anyOf:
          - additionalProperties:
              type: integer
            type: object
          - type: 'null'
          title: Result Counts
        shared_with_openai:
          anyOf:
          - type: boolean
          - type: 'null'
          title: Shared With Openai
        started_at:
          anyOf:
          - type: integer
          - type: 'null'
          title: Started At
        status:
          enum:
          - queued
          - running
          - completed
          - failed
          - cancelled
          title: Status
          type: string
      required:
      - id
      - created_at
      - status
      - data_source
      - eval_id
      title: Run
      type: object
    CancelEvalResponse:
      description: Response from cancelling an evaluation
      properties:
        id:
          title: Id
          type: string
        object:
          default: eval
          title: Object
          type: string
        status:
          const: cancelled
          title: Status
          type: string
      required:
      - id
      - status
      title: CancelEvalResponse
      type: object
    ListEvalsResponse:
      description: Response from listing evaluations
      properties:
        data:
          items:
            $ref: '#/components/schemas/Eval'
          title: Data
          type: array
        first_id:
          anyOf:
          - type: string
          - type: 'null'
          title: First Id
        has_more:
          default: false
          title: Has More
          type: boolean
        last_id:
          anyOf:
          - type: string
          - type: 'null'
          title: Last Id
        object:
          default: list
          title: Object
          type: string
      required:
      - data
      title: ListEvalsResponse
      type: object
    ValidationError:
      properties:
        loc:
          items:
            anyOf:
            - type: string
            - type: integer
          type: array
          title: Location
        msg:
          type: string
          title: Message
        type:
          type: string
          title: Error Type
        input:
          title: Input
        ctx:
          type: object
          title: Context
      type: object
      required:
      - loc
      - msg
      - type
      title: ValidationError
    RunDeleteResponse:
      description: Response from deleting a run
      properties:
        deleted:
          anyOf:
          - type: boolean
          - type: 'null'
          default: true
          title: Deleted
        object:
          anyOf:
          - type: string
          - type: 'null'
          default: eval.run.deleted
          title: Object
        run_id:
          title: Run Id
          type: string
      required:
      - run_id
      title: RunDeleteResponse
      type: object
    Eval:
      description: Represents an evaluation from the OpenAI Evals API
      properties:
        created_at:
          title: Created At
          type: integer
        data_source_config:
          additionalProperties: true
          title: Data Source Config
          type: object
        id:
          title: Id
          type: string
        metadata:
          anyOf:
          - additionalProperties: true
            type: object
          - type: 'null'
          title: Metadata
        name:
          anyOf:
          - type: string
          - type: 'null'
          title: Name
        object:
          default: eval
          title: Object
          type: string
        testing_criteria:
          items:
            additionalProperties: true
            type: object
          title: Testing Criteria
          type: array
        updated_at:
          anyOf:
          - type: integer
          - type: 'null'
          title: Updated At
      required:
      - id
      - created_at
      - data_source_config
      - testing_criteria
      title: Eval
      type: object
    HTTPValidationError:
      properties:
        detail:
          items:
            $ref: '#/components/schemas/ValidationError'
          type: array
          title: Detail
      type: object
      title: HTTPValidationError
  securitySchemes:
    APIKeyHeader:
      type: apiKey
      description: Bearer token
      in: header
      name: x-litellm-api-key