openapi: 3.1.0
info:
  title: Prime Intellect Evaluations API
  version: 0.1.0
  description: Run and manage model evaluations against the Environments Hub. Includes both client-driven evaluations (push
    samples, finalize, retrieve) and hosted evaluations executed by Prime Intellect with logs, cancellation, and selectable
    inference models.
  contact:
    name: Prime Intellect
    url: https://www.primeintellect.ai
servers:
- url: https://api.primeintellect.ai
security:
- HTTPBearer: []
tags:
- name: evals
- name: hosted-evaluations
- name: Feedback
paths:
  /api/v1/evaluations/:
    post:
      tags:
      - evals
      summary: Create Evaluation
      description: 'Create a new evaluation


        This endpoint supports:

        - Environment evaluations: Provide environments

        - Prime RL evaluations: Provide run_id

        - Suite evaluations: Provide suite_id


        Ownership:

        - If team_id is provided in request, the evaluation will be owned by the team

        - Otherwise, the evaluation will be owned by the authenticated user'
      operationId: create_evaluation_api_v1_evaluations__post
      security:
      - HTTPBearer: []
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/CreateEvaluationRequest'
      responses:
        '201':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/CreateEvaluationResponse'
        '401':
          description: Authorization failed
        '422':
          description: Invalid request data
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ErrorResponse'
    get:
      tags:
      - evals
      summary: List Evaluations
      description: 'Get a list of evaluations owned by the authenticated user or their teams


        By default, returns all evaluations the user has access to (personal + all teams).'
      operationId: list_evaluations_api_v1_evaluations__get
      security:
      - HTTPBearer: []
      parameters:
      - name: team_id
        in: query
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          description: Filter by specific team ID
          title: Team Id
        description: Filter by specific team ID
      - name: environment_id
        in: query
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          description: Filter by environment ID
          title: Environment Id
        description: Filter by environment ID
      - name: environment_name
        in: query
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          description: Filter by environment name
          title: Environment Name
        description: Filter by environment name
      - name: suite_id
        in: query
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          description: Filter by suite ID
          title: Suite Id
        description: Filter by suite ID
      - name: skip
        in: query
        required: false
        schema:
          type: integer
          minimum: 0
          default: 0
          title: Skip
      - name: limit
        in: query
        required: false
        schema:
          type: integer
          maximum: 100
          minimum: 1
          default: 50
          title: Limit
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ListEvaluationsResponse'
        '401':
          description: Authorization failed
        '422':
          description: Invalid request data
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ErrorResponse'
  /api/v1/evaluations/bulk-delete:
    post:
      tags:
      - evals
      summary: Bulk Delete Evaluations
      operationId: bulk_delete_evaluations_api_v1_evaluations_bulk_delete_post
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/BulkDeleteEvaluationsRequest'
        required: true
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/BulkDeleteEvaluationsResponse'
        '401':
          description: Authorization failed
        '422':
          description: Invalid request data
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ErrorResponse'
      security:
      - HTTPBearer: []
  /api/v1/evaluations/{evaluation_id}:
    put:
      tags:
      - evals
      summary: Update Evaluation
      description: Update an existing evaluation
      operationId: update_evaluation_api_v1_evaluations__evaluation_id__put
      security:
      - HTTPBearer: []
      parameters:
      - name: evaluation_id
        in: path
        required: true
        schema:
          type: string
          title: Evaluation Id
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/UpdateEvaluationRequest'
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/UpdateEvaluationResponse'
        '401':
          description: Authorization failed
        '422':
          description: Invalid request data
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ErrorResponse'
    get:
      tags:
      - evals
      summary: Get Evaluation
      description: Get detailed information about a specific evaluation.
      operationId: get_evaluation_api_v1_evaluations__evaluation_id__get
      security:
      - HTTPBearer: []
      parameters:
      - name: evaluation_id
        in: path
        required: true
        schema:
          type: string
          title: Evaluation Id
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/GetEvaluationResponse'
        '401':
          description: Authorization failed
        '422':
          description: Invalid request data
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ErrorResponse'
    delete:
      tags:
      - evals
      summary: Delete Evaluation
      operationId: delete_evaluation_api_v1_evaluations__evaluation_id__delete
      security:
      - HTTPBearer: []
      parameters:
      - name: evaluation_id
        in: path
        required: true
        schema:
          type: string
          title: Evaluation Id
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema: {}
        '401':
          description: Authorization failed
        '422':
          description: Invalid request data
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ErrorResponse'
  /api/v1/evaluations/{evaluation_id}/finalize:
    post:
      tags:
      - evals
      summary: Finalize Evaluation
      description: Mark an evaluation as complete and compute final statistics.
      operationId: finalize_evaluation_api_v1_evaluations__evaluation_id__finalize_post
      security:
      - HTTPBearer: []
      parameters:
      - name: evaluation_id
        in: path
        required: true
        schema:
          type: string
          title: Evaluation Id
      requestBody:
        content:
          application/json:
            schema:
              anyOf:
              - $ref: '#/components/schemas/FinalizeEvaluationRequest'
              - type: 'null'
              title: Request
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/FinalizeEvaluationResponse'
        '401':
          description: Authorization failed
        '422':
          description: Invalid request data
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ErrorResponse'
  /api/v1/evaluations/{evaluation_id}/samples:
    post:
      tags:
      - evals
      summary: Push Samples
      description: 'Push evaluation samples


        This endpoint can be called multiple times to stream samples as they''re generated.'
      operationId: push_samples_api_v1_evaluations__evaluation_id__samples_post
      security:
      - HTTPBearer: []
      parameters:
      - name: evaluation_id
        in: path
        required: true
        schema:
          type: string
          title: Evaluation Id
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/PushSamplesRequest'
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/PushSamplesResponse'
        '401':
          description: Authorization failed
        '422':
          description: Invalid request data
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ErrorResponse'
    get:
      tags:
      - evals
      summary: Get Samples
      description: Get samples for a specific evaluation
      operationId: get_samples_api_v1_evaluations__evaluation_id__samples_get
      security:
      - HTTPBearer: []
      parameters:
      - name: evaluation_id
        in: path
        required: true
        schema:
          type: string
          title: Evaluation Id
      - name: page
        in: query
        required: false
        schema:
          type: integer
          minimum: 1
          default: 1
          title: Page
      - name: limit
        in: query
        required: false
        schema:
          type: integer
          maximum: 1000
          minimum: 1
          default: 100
          title: Limit
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/GetSamplesResponse'
        '401':
          description: Authorization failed
        '422':
          description: Invalid request data
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ErrorResponse'
  /api/v1/feedback:
    post:
      tags:
      - Feedback
      summary: Submit Feedback
      operationId: submit_feedback_api_v1_feedback_post
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/FeedbackRequest'
        required: true
      responses:
        '201':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/FeedbackResponse'
        '401':
          description: Authorization failed
        '422':
          description: Invalid request data
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ErrorResponse'
      security:
      - HTTPBearer: []
  /api/v1/hosted-evaluations:
    post:
      tags:
      - hosted-evaluations
      summary: Create Hosted Evaluation
      description: Create and start a hosted evaluation.
      operationId: create_hosted_evaluation_api_v1_hosted_evaluations_post
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/CreateHostedEvaluationRequest'
        required: true
      responses:
        '201':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/CreateHostedEvaluationResponse'
        '401':
          description: Authorization failed
        '422':
          description: Invalid request data
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ErrorResponse'
      security:
      - HTTPBearer: []
  /api/v1/hosted-evaluations/models:
    get:
      tags:
      - hosted-evaluations
      summary: Get Inference Models
      description: Get available models from Prime Inference API for hosted evaluations
      operationId: get_inference_models_api_v1_hosted_evaluations_models_get
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ListInferenceModelsResponse'
        '401':
          description: Authorization failed
        '422':
          description: Invalid request data
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ErrorResponse'
      security:
      - HTTPBearer: []
  /api/v1/hosted-evaluations/{evaluation_id}/cancel:
    patch:
      tags:
      - hosted-evaluations
      summary: Cancel Hosted Evaluation Route
      description: Cancel a running hosted evaluation.
      operationId: cancel_hosted_evaluation_route_api_v1_hosted_evaluations__evaluation_id__cancel_patch
      security:
      - HTTPBearer: []
      parameters:
      - name: evaluation_id
        in: path
        required: true
        schema:
          type: string
          title: Evaluation Id
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/CancelHostedEvaluationResponse'
        '401':
          description: Authorization failed
        '422':
          description: Invalid request data
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ErrorResponse'
  /api/v1/hosted-evaluations/{evaluation_id}/logs:
    get:
      tags:
      - hosted-evaluations
      summary: Get Hosted Evaluation Logs Route
      description: Get real-time logs from the sandbox running a hosted evaluation.
      operationId: get_hosted_evaluation_logs_route_api_v1_hosted_evaluations__evaluation_id__logs_get
      security:
      - HTTPBearer: []
      parameters:
      - name: evaluation_id
        in: path
        required: true
        schema:
          type: string
          title: Evaluation Id
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/GetHostedEvaluationLogsResponse'
        '401':
          description: Authorization failed
        '422':
          description: Invalid request data
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ErrorResponse'
components:
  securitySchemes:
    HTTPBearer:
      type: http
      scheme: bearer
  schemas:
    FinalizeEvaluationResponse:
      properties:
        evaluation_id:
          type: string
          title: Evaluation Id
        status:
          $ref: '#/components/schemas/EvaluationStatus'
        message:
          anyOf:
          - type: string
          - type: 'null'
          title: Message
        total_samples:
          anyOf:
          - type: integer
          - type: 'null'
          title: Total Samples
        avg_score:
          anyOf:
          - type: number
          - type: 'null'
          title: Avg Score
        min_score:
          anyOf:
          - type: number
          - type: 'null'
          title: Min Score
        max_score:
          anyOf:
          - type: number
          - type: 'null'
          title: Max Score
        viewer_url:
          anyOf:
          - type: string
          - type: 'null'
          title: Viewer Url
      type: object
      required:
      - evaluation_id
      - status
      title: FinalizeEvaluationResponse
      description: Response after finalizing an evaluation
    EvaluationStatus:
      type: string
      enum:
      - PENDING
      - RUNNING
      - PROCESSING
      - COMPLETED
      - FAILED
      - TIMEOUT
      - CANCELLED
      title: EvaluationStatus
      description: Evaluation status enum
    ListInferenceModelsResponse:
      properties:
        models:
          items:
            type: object
          type: array
          title: Models
          description: List of available models
      type: object
      required:
      - models
      title: ListInferenceModelsResponse
      description: Response for listing available inference models
    CreateHostedEvaluationRequest:
      properties:
        environment_ids:
          items:
            type: string
          type: array
          minItems: 1
          title: Environment Ids
          description: List of environment IDs to evaluate
        inference_model:
          type: string
          title: Inference Model
          description: Model ID for inference
        eval_config:
          $ref: '#/components/schemas/HostedEvalConfig'
          description: Evaluation configuration
        team_id:
          anyOf:
          - type: string
          - type: 'null'
          title: Team Id
          description: Optional team ID to own the hosted evaluation
        name:
          anyOf:
          - type: string
          - type: 'null'
          title: Name
          description: Optional custom evaluation name
      type: object
      required:
      - environment_ids
      - inference_model
      - eval_config
      title: CreateHostedEvaluationRequest
      description: Request to create and start a hosted evaluation
    HostedEvalConfig:
      properties:
        num_examples:
          type: integer
          maximum: 9007199254740991.0
          minimum: -1.0
          title: Num Examples
          description: Number of examples to evaluate (-1 for all)
        rollouts_per_example:
          type: integer
          maximum: 2048.0
          minimum: 1.0
          title: Rollouts Per Example
          description: Rollouts per example
        env_args:
          anyOf:
          - type: object
          - type: 'null'
          title: Env Args
          description: Optional environment arguments to pass to the evaluation
        allow_sandbox_access:
          anyOf:
          - type: boolean
          - type: 'null'
          title: Allow Sandbox Access
          description: Allow sandbox read/write access
          default: false
        allow_instances_access:
          anyOf:
          - type: boolean
          - type: 'null'
          title: Allow Instances Access
          description: Allow instance creation and management access
          default: false
        allow_tunnel_access:
          anyOf:
          - type: boolean
          - type: 'null'
          title: Allow Tunnel Access
          description: Allow tunnel creation and management access
          default: false
        timeout_minutes:
          anyOf:
          - type: integer
            maximum: 1440.0
            minimum: 120.0
          - type: 'null'
          title: Timeout Minutes
          description: 'Custom timeout in minutes for the hosted eval run (default: 1440 = 24 hours, min: 120, max: 1440)'
        custom_secrets:
          anyOf:
          - additionalProperties:
              type: string
            type: object
          - type: 'null'
          title: Custom Secrets
          description: Custom secrets to set in the evaluation sandbox (e.g., API keys, tokens)
        sampling_args:
          anyOf:
          - type: object
          - type: 'null'
          title: Sampling Args
          description: Optional sampling arguments forwarded to `prime eval run --sampling-args`
        max_concurrent:
          anyOf:
          - type: integer
            maximum: 9007199254740991.0
            minimum: 1.0
          - type: 'null'
          title: Max Concurrent
          description: Optional max concurrency forwarded to `prime eval run --max-concurrent`
        max_retries:
          anyOf:
          - type: integer
            maximum: 9007199254740991.0
            minimum: 0.0
          - type: 'null'
          title: Max Retries
          description: Optional max retries forwarded to `prime eval run --max-retries`
        state_columns:
          anyOf:
          - items:
              type: string
            type: array
          - type: 'null'
          title: State Columns
          description: Optional state columns forwarded to `prime eval run --state-columns`
        independent_scoring:
          anyOf:
          - type: boolean
          - type: 'null'
          title: Independent Scoring
          description: Forward `--independent-scoring` to the hosted eval runner
          default: false
        verbose:
          anyOf:
          - type: boolean
          - type: 'null'
          title: Verbose
          description: Forward `--verbose` to the hosted eval runner
          default: false
        headers:
          anyOf:
          - items:
              type: string
            type: array
          - type: 'null'
          title: Headers
          description: Optional repeated headers forwarded to `prime eval run --header`
        extra_env_kwargs:
          anyOf:
          - type: object
          - type: 'null'
          title: Extra Env Kwargs
          description: Optional environment constructor kwargs forwarded to `prime eval run --extra-env-kwargs`
        api_client_type:
          anyOf:
          - type: string
          - type: 'null'
          title: Api Client Type
          description: Optional API client type forwarded to `prime eval run --api-client-type`
        api_base_url:
          anyOf:
          - type: string
          - type: 'null'
          title: Api Base Url
          description: Optional inference base URL forwarded to `prime eval run --api-base-url`
        api_key_var:
          anyOf:
          - type: string
          - type: 'null'
          title: Api Key Var
          description: Optional API key env var forwarded to `prime eval run --api-key-var`
      type: object
      required:
      - num_examples
      - rollouts_per_example
      title: HostedEvalConfig
      description: Hosted evaluation configuration
    CreateHostedEvaluationResponse:
      properties:
        evaluation_id:
          type: string
          title: Evaluation Id
          description: ID of the created evaluation
        sandbox_id:
          anyOf:
          - type: string
          - type: 'null'
          title: Sandbox Id
          description: ID of the sandbox running the evaluation
        status:
          type: string
          title: Status
          description: Current status of the evaluation
        evaluation_ids:
          anyOf:
          - items:
              type: string
            type: array
          - type: 'null'
          title: Evaluation Ids
          description: List of evaluation IDs if multiple environments were provided
        error:
          anyOf:
          - type: string
          - type: 'null'
          title: Error
          description: Error message if creation failed
      type: object
      required:
      - evaluation_id
      - status
      title: CreateHostedEvaluationResponse
      description: Response after creating a hosted evaluation
    BulkDeleteEvaluationsRequest:
      properties:
        evaluation_ids:
          items:
            type: string
          type: array
          maxItems: 100
          minItems: 1
          title: Evaluation Ids
      type: object
      required:
      - evaluation_ids
      title: BulkDeleteEvaluationsRequest
    UpdateEvaluationRequest:
      properties:
        name:
          anyOf:
          - type: string
          - type: 'null'
          title: Name
          description: Name of the evaluation
        description:
          anyOf:
          - type: string
          - type: 'null'
          title: Description
          description: Description of the evaluation
        tags:
          anyOf:
          - items:
              type: string
            type: array
          - type: 'null'
          title: Tags
          description: Tags for categorization
        metadata:
          anyOf:
          - type: object
          - type: 'null'
          title: Metadata
          description: Additional metadata
        metrics:
          anyOf:
          - type: object
          - type: 'null'
          title: Metrics
          description: High-level metrics summary
        model_name:
          anyOf:
          - type: string
          - type: 'null'
          title: Model Name
          description: Model name
        dataset:
          anyOf:
          - type: string
          - type: 'null'
          title: Dataset
          description: Dataset name
        framework:
          anyOf:
          - type: string
          - type: 'null'
          title: Framework
          description: Framework used
        task_type:
          anyOf:
          - type: string
          - type: 'null'
          title: Task Type
          description: Type of task
        inference_model:
          anyOf:
          - type: string
          - type: 'null'
          title: Inference Model
          description: Prime Inference model ID
        eval_config:
          anyOf:
          - $ref: '#/components/schemas/HostedEvalConfig'
          - type: 'null'
          description: Hosted evaluation configuration
        is_public:
          anyOf:
          - type: boolean
          - type: 'null'
          title: Is Public
          description: Whether the evaluation is publicly shareable by link; setting true without show_on_leaderboard keeps
            it off leaderboards by default
        show_on_leaderboard:
          anyOf:
          - type: boolean
          - type: 'null'
          title: Show On Leaderboard
          description: Whether this public evaluation appears on environment leaderboards
      type: object
      title: UpdateEvaluationRequest
      description: Request to update an existing evaluation
    FeedbackRequest:
      properties:
        message:
          type: string
          maxLength: 2048
          minLength: 1
          title: Message
        category:
          $ref: '#/components/schemas/FeedbackCategory'
        cli_version:
          type: string
          maxLength: 64
          minLength: 1
          title: Cli Version
        run_id:
          anyOf:
          - type: string
            maxLength: 128
          - type: 'null'
          title: Run Id
      type: object
      required:
      - message
      - category
      - cli_version
      title: FeedbackRequest
    FeedbackCategory:
      type: string
      enum:
      - bug
      - feature
      - general
      title: FeedbackCategory
    FinalizeEvaluationRequest:
      properties:
        metrics:
          anyOf:
          - type: object
          - type: 'null'
          title: Metrics
          description: Final metrics to attach to the evaluation
      type: object
      title: FinalizeEvaluationRequest
      description: Request to finalize an evaluation (optional, can include final metrics)
    UpdateEvaluationResponse:
      properties:
        evaluation_id:
          type: string
          title: Evaluation Id
        name:
          type: string
          title: Name
        status:
          $ref: '#/components/schemas/EvaluationStatus'
        updated_at:
          type: string
          format: date-time
          title: Updated At
      type: object
      required:
      - evaluation_id
      - name
      - status
      - updated_at
      title: UpdateEvaluationResponse
      description: Response after updating an evaluation
    GetEvaluationResponse:
      properties:
        evaluation_id:
          type: string
          title: Evaluation Id
        name:
          type: string
          title: Name
        status:
          $ref: '#/components/schemas/EvaluationStatus'
        eval_type:
          $ref: '#/components/schemas/EvaluationType'
        user_id:
          anyOf:
          - type: string
          - type: 'null'
          title: User Id
          description: User ID of evaluation owner
        team_id:
          anyOf:
          - type: string
          - type: 'null'
          title: Team Id
          description: Team ID if evaluation is owned by a team
        environment_ids:
          anyOf:
          - items:
              type: string
            type: array
          - type: 'null'
          title: Environment Ids
        environment_names:
          anyOf:
          - items:
              type: string
            type: array
          - type: 'null'
          title: Environment Names
        suite_id:
          anyOf:
          - type: string
          - type: 'null'
          title: Suite Id
        run_id:
          anyOf:
          - type: string
          - type: 'null'
          title: Run Id
        version_id:
          anyOf:
          - type: string
          - type: 'null'
          title: Version Id
        is_hosted:
          type: boolean
          title: Is Hosted
          default: false
        sandbox_id:
          anyOf:
          - type: string
          - type: 'null'
          title: Sandbox Id
        inference_model:
          anyOf:
          - type: string
          - type: 'null'
          title: Inference Model
        eval_config:
          anyOf:
          - type: object
          - type: 'null'
          title: Eval Config
        error_message:
          anyOf:
          - type: string
          - type: 'null'
          title: Error Message
        logs:
          anyOf:
          - type: string
          - type: 'null'
          title: Logs
        model_name:
          anyOf:
          - type: string
          - type: 'null'
          title: Model Name
        dataset:
          anyOf:
          - type: string
          - type: 'null'
          title: Dataset
        framework:
          anyOf:
          - type: string
          - type: 'null'
          title: Framework
        task_type:
          anyOf:
          - type: string
          - type: 'null'
          title: Task Type
        description:
          anyOf:
          - type: string
          - type: 'null'
          title: Description
        tags:
          items:
            type: string
          type: array
          title: Tags
          default: []
        total_samples:
          type: integer
          title: Total Samples
        avg_score:
          anyOf:
          - type: number
          - type: 'null'
          title: Avg Score
        min_score:
          anyOf:
          - type: number
          - type: 'null'
          title: Min Score
        max_score:
          anyOf:
          - type: number
          - type: 'null'
          title: Max Score
        started_at:
          anyOf:
          - type: string
            format: date-time
          - type: 'null'
          title: Started At
        completed_at:
          anyOf:
          - type: string
            format: date-time
          - type: 'null'
          title: Completed At
        created_at:
          type: string
          format: date-time
          title: Created At
        updated_at:
          type: string
          format: date-time
          title: Updated At
        metadata:
          anyOf:
          - type: object
          - type: 'null'
          title: Metadata
        metrics:
          anyOf:
          - type: object
          - type: 'null'
          title: Metrics
        viewer_url:
          anyOf:
          - type: string
          - type: 'null'
          title: Viewer Url
        is_public:
          type: boolean
          title: Is Public
          description: Whether this evaluation is publicly shareable by link
          default: false
        show_on_leaderboard:
          type: boolean
          title: Show On Leaderboard
          description: Whether this public evaluation appears on environment leaderboards
          default: false
        statistics:
          anyOf:
          - type: object
          - type: 'null'
          title: Statistics
          description: Statistics format (totalResults, totalReward, avgReward)
        chartData:
          anyOf:
          - type: object
          - type: 'null'
          title: Chartdata
          description: Histogram and statistics for visualization
        rubricInfo:
          anyOf:
          - type: object
          - type: 'null'
          title: Rubricinfo
          description: Rubric with detected metrics and weights
        detailedMetrics:
          anyOf:
          - type: object
          - type: 'null'
          title: Detailedmetrics
          description: Detailed statistics for each detected metric
        rewardStats:
          anyOf:
          - type: object
          - type: 'null'
          title: Rewardstats
          description: Reward statistics for backward compatibility
      type: object
      required:
      - evaluation_id
      - name
      - status
      - eval_type
      - total_samples
      - created_at
      - updated_at
      title: GetEvaluationResponse
      description: Response for getting evaluation details
    EvaluationType:
      type: string
      enum:
      - suite
      - training
      - environment
      title: EvaluationType
      description: Evaluation type enum
    GetHostedEvaluationLogsResponse:
      properties:
        logs:
          type: string
          title: Logs
          description: Sandbox logs
        sandbox_id:
          anyOf:
          - type: string
          - type: 'null'
          title: Sandbox Id
          description: Sandbox ID
        evaluation_id:
          type: string
          title: Evaluation Id
          description: Evaluation ID
      type: object
      required:
      - logs
      - evaluation_id
      title: GetHostedEvaluationLogsResponse
    BulkDeleteEvaluationsResponse:
      properties:
        success_count:
          type: integer
          title: Success Count
          description: Number of evaluations successfully deleted
        failure_count:
          type: integer
          title: Failure Count
          description: Number of evaluations that failed to delete
        successful_ids:
          items:
            type: string
          type: array
          title: Successful Ids
          description: IDs of successfully deleted evaluations
        failed_ids:
          items:
            type: string
          type: array
          title: Failed Ids
          description: IDs of evaluations that failed to delete
      type: object
      required:
      - success_count
      - failure_count
      - successful_ids
      - failed_ids
      title: BulkDeleteEvaluationsResponse
    FeedbackResponse:
      properties:
        message:
          type: string
          title: Message
      type: object
      required:
      - message
      title: FeedbackResponse
    CreateEvaluationRequest:
      properties:
        name:
          type: string
          title: Name
          description: Name of the evaluation
        team_id:
          anyOf:
          - type: string
          - type: 'null'
          title: Team Id
          description: Team ID if creating evaluation for a team
        environments:
          anyOf:
          - items:
              $ref: '#/components/schemas/EnvironmentReference'
            type: array
          - type: 'null'
          title: Environments
          description: List of environment references with optional version IDs
        suite_id:
          anyOf:
          - type: string
          - type: 'null'
          title: Suite Id
          description: Suite ID if this evaluation is part of a suite
        run_id:
          anyOf:
          - type: string
          - type: 'null'
          title: Run Id
          description: Run ID for Prime RL runs (optional)
        is_hosted:
          type: boolean
          title: Is Hosted
          description: Whether this is a hosted evaluation
          default: false
        inference_model:
          anyOf:
          - type: string
          - type: 'null'
          title: Inference Model
          description: Prime Inference model ID
        eval_config:
          anyOf:
          - $ref: '#/components/schemas/HostedEvalConfig'
          - type: 'null'
          description: Hosted evaluation configuration
        model_name:
          anyOf:
          - type: string
          - type: 'null'
          title: Model Name
          description: Model name
        dataset:
          anyOf:
          - type: string
          - type: 'null'
          title: Dataset
          description: Dataset name
        framework:
          anyOf:
          - type: string
          - type: 'null'
          title: Framework
          description: Framework used (e.g., 'prime-rl', 'openai/evals')
        task_type:
          anyOf:
          - type: string
          - type: 'null'
          title: Task Type
          description: Type of task (e.g., 'classification', 'generation')
        description:
          anyOf:
          - type: string
          - type: 'null'
          title: Description
          description: Description of the evaluation
        tags:
          items:
            type: string
          type: array
          title: Tags
          description: Tags for categorization
          default: []
        metadata:
          anyOf:
          - type: object
          - type: 'null'
          title: Metadata
          description: Additional metadata
        metrics:
          anyOf:
          - type: object
          - type: 'null'
          title: Metrics
          description: High-level metrics summary
        is_public:
          type: boolean
          title: Is Public
          description: Whether this evaluation is publicly shareable by link
          default: false
        show_on_leaderboard:
          type: boolean
          title: Show On Leaderboard
          description: Whether this public evaluation appears on environment leaderboards
          default: false
      type: object
      required:
      - name
      title: CreateEvaluationRequest
      description: Request to create a new evaluation
    EnvironmentReference:
      properties:
        id:
          type: string
          title: Id
        version_id:
          anyOf:
          - type: string
          - type: 'null'
          title: Version Id
      type: object
      required:
      - id
      title: EnvironmentReference
      description: Environment reference with optional version
    CreateEvaluationResponse:
      properties:
        evaluation_id:
          type: string
          title: Evaluation Id
          description: ID of the created evaluation
        name:
          type: string
          title: Name
        status:
          $ref: '#/components/schemas/EvaluationStatus'
        eval_type:
          $ref: '#/components/schemas/EvaluationType'
          description: 'Type: ''prime_rl'', ''environment'', or ''suite'''
        environment_ids:
          anyOf:
          - items:
              type: string
            type: array
          - type: 'null'
          title: Environment Ids
        suite_id:
          anyOf:
          - type: string
          - type: 'null'
          title: Suite Id
        run_id:
          anyOf:
          - type: string
          - type: 'null'
          title: Run Id
        version_id:
          anyOf:
          - type: string
          - type: 'null'
          title: Version Id
        viewer_url:
          anyOf:
          - type: string
          - type: 'null'
          title: Viewer Url
        created_at:
          type: string
          format: date-time
          title: Created At
      type: object
      required:
      - evaluation_id
      - name
      - status
      - eval_type
      - created_at
      title: CreateEvaluationResponse
      description: Response after creating an evaluation
    ErrorResponse:
      properties:
        errors:
          items:
            $ref: '#/components/schemas/ErrorDetail'
          type: array
          title: Errors
      type: object
      required:
      - errors
      title: ErrorResponse
    ErrorDetail:
      properties:
        param:
          type: string
          title: Param
        details:
          type: string
          title: Details
      type: object
      required:
      - param
      - details
      title: ErrorDetail
    PushSamplesRequest:
      properties:
        samples:
          items:
            type: object
          type: array
          title: Samples
          description: List of evaluation samples
      type: object
      required:
      - samples
      title: PushSamplesRequest
      description: Request to push evaluation samples (replaces traces)
    GetSamplesResponse:
      properties:
        evaluation_id:
          type: string
          title: Evaluation Id
        samples:
          items:
            type: object
          type: array
          title: Samples
        total:
          type: integer
          title: Total
        page:
          type: integer
          title: Page
        limit:
          type: integer
          title: Limit
        total_pages:
          type: integer
          title: Total Pages
      type: object
      required:
      - evaluation_id
      - samples
      - total
      - page
      - limit
      - total_pages
      title: GetSamplesResponse
      description: Response for getting evaluation samples
    PushSamplesResponse:
      properties:
        evaluation_id:
          type: string
          title: Evaluation Id
        samples_pushed:
          type: integer
          title: Samples Pushed
        status:
          $ref: '#/components/schemas/EvaluationStatus'
      type: object
      required:
      - evaluation_id
      - samples_pushed
      - status
      title: PushSamplesResponse
      description: Response after pushing samples
    ListEvaluationsResponse:
      properties:
        evaluations:
          items:
            $ref: '#/components/schemas/GetEvaluationResponse'
          type: array
          title: Evaluations
        total:
          type: integer
          title: Total
        skip:
          type: integer
          title: Skip
        limit:
          type: integer
          title: Limit
      type: object
      required:
      - evaluations
      - total
      - skip
      - limit
      title: ListEvaluationsResponse
      description: Response for listing evaluations
    CancelHostedEvaluationResponse:
      properties:
        success:
          type: boolean
          title: Success
        message:
          type: string
          title: Message
        evaluation_id:
          type: string
          title: Evaluation Id
        status:
          type: string
          title: Status
      type: object
      required:
      - success
      - message
      - evaluation_id
      - status
      title: CancelHostedEvaluationResponse