openapi: 3.0.3
info:
  title: Coval Agents Runs API
  version: 1.0.0
  description: '

    Manage configurations for simulations and evaluations.

    '
  contact:
    name: Coval API Support
    email: support@coval.dev
    url: https://docs.coval.ai
  license:
    name: Proprietary
    url: https://coval.dev/terms
servers:
- url: https://api.coval.dev/v1
  description: Production API
security:
- ApiKeyAuth: []
tags:
- name: Runs
  description: Launch and manage simulation runs
paths:
  /runs:
    get:
      operationId: listRuns
      summary: List runs
      description: 'Retrieve a paginated list of simulation runs with optional filtering and sorting.

        '
      tags:
      - Runs
      security:
      - ApiKeyAuth: []
      parameters:
      - name: filter
        in: query
        required: false
        schema:
          type: string
        description: 'Filter expression syntax.


          Supported fields: `status`, `agent_id`, `persona_id`, `test_set_id`, `create_time`, `update_time`, `created_by`, `tag`


          Operators: `=`, `!=`, `>`, `<`, `>=`, `<=`, `AND`, `OR`


          Values may be unquoted or double-quoted. Values containing spaces must be quoted (e.g., `status="IN PROGRESS"`).

          '
        examples:
          byStatus:
            value: status="COMPLETED" AND agent_id=gk3jK9mPq2xRt5vW8yZaBc
            summary: Filter by status and agent
          byTag:
            value: tag="regression"
            summary: Filter by tag
      - name: page_size
        in: query
        required: false
        schema:
          type: integer
          minimum: 1
          maximum: 1000
          default: 50
        description: Maximum number of results per page
        example: 50
      - name: page_token
        in: query
        required: false
        schema:
          type: string
        description: Opaque pagination token from previous response
      - name: order_by
        in: query
        required: false
        schema:
          type: string
          default: -create_time
        description: 'Sort order specification.


          Format: `field` or `-field` (descending)


          Supported fields: `create_time`, `update_time`, `status`, `agent_id`, `persona_id`, `test_set_id`

          '
        example: -create_time
      responses:
        '200':
          description: List of runs
          content:
            application/json:
              schema:
                type: object
                required:
                - runs
                properties:
                  runs:
                    type: array
                    items:
                      $ref: '#/components/schemas/Run'
                  next_page_token:
                    type: string
                    description: Token for fetching the next page of results
                    example: eyJvZmZzZXQiOiA1MH0=
        '400':
          $ref: '#/components/responses/BadRequest'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '500':
          $ref: '#/components/responses/InternalError'
    post:
      operationId: launchRun
      summary: Launch run
      description: 'Launch a new simulation run to evaluate an agent against test cases using a persona.

        '
      tags:
      - Runs
      security:
      - ApiKeyAuth: []
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/LaunchRunRequest'
            examples:
              minimal:
                summary: Minimal request (uses agent defaults)
                value:
                  agent_id: gk3jK9mPq2xRt5vW8yZaBc
                  persona_id: hL4kL0nQr3ySt6vX9zAcDd
                  test_set_id: aB1cD2eF
              full:
                summary: Full request with all options
                value:
                  agent_id: gk3jK9mPq2xRt5vW8yZaBc
                  persona_id: hL4kL0nQr3ySt6vX9zAcDd
                  test_set_id: aB1cD2eF
                  metric_ids:
                  - iM5lM1oRs4zTu7wY0aBdEe
                  - jN6mN2pSt5aUv8xZ1bCeFf
                  options:
                    iteration_count: 3
                    concurrency: 2
                  metadata:
                    display_name: Q4 Voice Agent Regression Test
                    customer:
                      campaign_id: q4_2025
                    created_by: automated_pipeline
                    tags:
                    - regression
                    - v2.1
              singleMutation:
                summary: Run with single mutation (A/B test)
                description: Runs base agent + one mutation variant
                value:
                  agent_id: gk3jK9mPq2xRt5vW8yZaBc
                  persona_id: hL4kL0nQr3ySt6vX9zAcDd
                  test_set_id: aB1cD2eF
                  mutation_id: 01ARZ3NDEKTSV4RRFFQ69G5FAV
                  options:
                    iteration_count: 3
                    concurrency: 5
                  metadata:
                    display_name: GPT-4 Fast VAD A/B Test
              multiMutation:
                summary: Run with multiple mutations (multi-variant test)
                description: 'Runs base agent + multiple mutations.

                  Total simulations = test_cases × iterations × (1 + mutations)

                  '
                value:
                  agent_id: gk3jK9mPq2xRt5vW8yZaBc
                  persona_id: hL4kL0nQr3ySt6vX9zAcDd
                  test_set_id: aB1cD2eF
                  mutation_ids:
                  - 01ARZ3NDEKTSV4RRFFQ69G5FAV
                  - 01ARZ3NDEKTSV4RRFFQ69G5FAW
                  - 01ARZ3NDEKTSV4RRFFQ69G5FAX
                  options:
                    iteration_count: 2
                    concurrency: 10
                  metadata:
                    display_name: Multi-Model Comparison Test
                    customer:
                      experiment_id: exp_2025_model_compare
              selectedTestCases:
                summary: Run a specific subset of test cases
                description: 'Restrict execution to a hand-picked list of test case IDs from

                  the test set. Useful for re-running failures or smoke-testing

                  a known-good slice. All IDs must belong to the specified

                  test_set_id and be active. Limit: 1–100 IDs.

                  '
                value:
                  agent_id: gk3jK9mPq2xRt5vW8yZaBc
                  persona_id: hL4kL0nQr3ySt6vX9zAcDd
                  test_set_id: aB1cD2eF
                  options:
                    test_case_ids:
                    - kP7mN3qStUvWxYzAbCdEfG
                    - lQ8nO4rTuVwXyZaBcDeFgH
                    - mR9oP5sUvWxYzAbCdEfGhI
                  metadata:
                    display_name: Re-run failed cases from nightly
      responses:
        '200':
          description: Run launched successfully
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/LaunchRunResponse'
              examples:
                pending:
                  summary: Newly launched run
                  value:
                    run:
                      name: runs/3zfmuDbVQsi4GaseDtiVcS
                      run_id: 3zfmuDbVQsi4GaseDtiVcS
                      status: PENDING
                      create_time: '2025-10-14T12:00:00Z'
        '400':
          $ref: '#/components/responses/BadRequest'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '403':
          $ref: '#/components/responses/Forbidden'
        '404':
          $ref: '#/components/responses/NotFound'
        '503':
          $ref: '#/components/responses/ServiceUnavailable'
        '500':
          $ref: '#/components/responses/InternalError'
  /runs/{run_id}:
    get:
      operationId: getRun
      summary: Get run
      description: 'Retrieve detailed information about a specific simulation run.

        '
      tags:
      - Runs
      security:
      - ApiKeyAuth: []
      parameters:
      - name: run_id
        in: path
        required: true
        schema:
          type: string
          minLength: 22
          maxLength: 22
        description: Unique identifier for the run
        example: 4AgpnEcWRtj5HbtfEujWdT
      responses:
        '200':
          description: Run details
          content:
            application/json:
              schema:
                type: object
                required:
                - run
                properties:
                  run:
                    $ref: '#/components/schemas/Run'
              examples:
                completed:
                  summary: Completed run with results
                  value:
                    run:
                      name: runs/5BhqoFdXSuk6IcugFvkXeU
                      run_id: 5BhqoFdXSuk6IcugFvkXeU
                      status: COMPLETED
                      create_time: '2025-10-14T12:00:00Z'
                      update_time: '2025-10-14T12:15:00Z'
                      agent_id: gk3jK9mPq2xRt5vW8yZaBc
                      persona_id: hL4kL0nQr3ySt6vX9zAcDd
                      test_set_id: aB1cD2eF
                      tags:
                      - regression
                      - v2.1
                      progress:
                        total_test_cases: 10
                        completed_test_cases: 10
                        failed_test_cases: 0
                        in_progress_test_cases: 0
                      results:
                        output_ids:
                        - sim_output_1
                        - sim_output_2
                        metrics:
                          availability:
                            mean: 0.95
                            min: 0.85
                            max: 1.0
                      metadata:
                        campaign_id: q4_2025
                in_progress:
                  summary: Run in progress
                  value:
                    run:
                      name: runs/6CirpGeYTvl7JdvhGwlYfV
                      run_id: 6CirpGeYTvl7JdvhGwlYfV
                      status: IN PROGRESS
                      create_time: '2025-10-14T12:00:00Z'
                      update_time: '2025-10-14T12:05:00Z'
                      agent_id: gk3jK9mPq2xRt5vW8yZaBc
                      persona_id: hL4kL0nQr3ySt6vX9zAcDd
                      test_set_id: aB1cD2eF
                      progress:
                        total_test_cases: 10
                        completed_test_cases: 5
                        failed_test_cases: 0
                        in_progress_test_cases: 1
                      metadata: {}
        '401':
          $ref: '#/components/responses/Unauthorized'
        '404':
          $ref: '#/components/responses/NotFound'
        '500':
          $ref: '#/components/responses/InternalError'
    delete:
      operationId: deleteRun
      summary: Delete run
      description: 'Delete or cancel a simulation run based on its current status.

        '
      tags:
      - Runs
      security:
      - ApiKeyAuth: []
      parameters:
      - name: run_id
        in: path
        required: true
        schema:
          type: string
          minLength: 22
          maxLength: 22
        description: Unique identifier for the run
        example: 7DjsqHfZUwm8KewjHxmZgW
      responses:
        '200':
          description: Run deleted or cancelled successfully
          content:
            application/json:
              schema:
                type: object
                description: Empty response on success
                example: {}
        '401':
          $ref: '#/components/responses/Unauthorized'
        '404':
          $ref: '#/components/responses/NotFound'
        '500':
          $ref: '#/components/responses/InternalError'
    patch:
      operationId: updateRun
      summary: Update run
      description: 'Replace the tag set for an existing simulation run.

        Provide an empty list to clear all tags.

        '
      tags:
      - Runs
      security:
      - ApiKeyAuth: []
      parameters:
      - name: run_id
        in: path
        required: true
        schema:
          type: string
          minLength: 22
          maxLength: 22
        description: Unique identifier for the run
        example: 7DjsqHfZUwm8KewjHxmZgW
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/UpdateRunRequest'
            examples:
              markBaseline:
                summary: Mark a run as a baseline
                value:
                  tags:
                  - baseline
                  - production
              clearTags:
                summary: Clear all run tags
                value:
                  tags: []
      responses:
        '200':
          description: Run updated successfully
          content:
            application/json:
              schema:
                type: object
                required:
                - run
                properties:
                  run:
                    $ref: '#/components/schemas/Run'
        '400':
          $ref: '#/components/responses/BadRequest'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '403':
          $ref: '#/components/responses/Forbidden'
        '404':
          $ref: '#/components/responses/NotFound'
        '500':
          $ref: '#/components/responses/InternalError'
components:
  schemas:
    Results:
      type: object
      nullable: true
      description: Results summary (only for COMPLETED runs)
      required:
      - output_ids
      - metrics
      properties:
        output_ids:
          type: array
          items:
            type: string
          description: IDs of simulation outputs (test case results)
          example:
          - sim_output_abc123
          - sim_output_def456
        metrics:
          type: object
          additionalProperties:
            $ref: '#/components/schemas/MetricSummary'
          description: Aggregated metric results
          example:
            availability:
              mean: 0.95
              min: 0.85
              max: 1.0
    Progress:
      type: object
      description: Progress information for the run
      required:
      - total_test_cases
      - completed_test_cases
      - failed_test_cases
      - in_progress_test_cases
      properties:
        total_test_cases:
          type: integer
          description: Total number of test cases
          example: 10
        completed_test_cases:
          type: integer
          description: Number of completed test cases
          example: 8
        failed_test_cases:
          type: integer
          description: Number of failed test cases
          example: 1
        in_progress_test_cases:
          type: integer
          description: Number of test cases currently running
          example: 1
    LaunchRunResponse:
      type: object
      required:
      - run
      properties:
        run:
          $ref: '#/components/schemas/Run'
    LaunchMetadata:
      type: object
      description: Metadata for tracking and organization purposes.
      properties:
        customer:
          type: object
          additionalProperties: true
          description: Custom customer metadata for tracking and organization
          example:
            campaign_id: summer_2025
            environment: production
        created_by:
          type: string
          description: Identifier for who/what created this simulation
          example: user@example.com
        display_name:
          type: string
          maxLength: 200
          description: 'Human-readable name for the run. If not provided, a timestamp-based

            name is auto-generated (e.g., "Simulation Run 2025-10-14 12:00").

            '
          example: Q4 Voice Agent Regression Test
        tags:
          type: array
          items:
            type: string
            maxLength: 200
          maxItems: 20
          nullable: true
          description: 'Tags for categorizing and filtering runs. Each tag max 200 characters,

            max 20 tags. Duplicate tags are automatically removed. Leading/trailing

            whitespace is stripped.

            '
          example:
          - regression
          - v2.1
          - nightly
    ErrorDetail:
      type: object
      properties:
        field:
          type: string
          description: The field that caused the error (if applicable)
          example: agent_id
        description:
          type: string
          description: Human-readable description of the error
          example: Agent 'gk3jK9mPq2xRt5vW8yZaBc' does not exist or is not accessible by your organization
    Run:
      type: object
      required:
      - name
      - run_id
      - status
      - create_time
      properties:
        name:
          type: string
          description: Resource name in format "runs/{run_id}"
          example: runs/abc123xyz789
        run_id:
          type: string
          minLength: 22
          maxLength: 22
          description: Unique identifier for this simulation run
          example: 8EktrIgaVxn9LfxkIynagX
        display_name:
          type: string
          nullable: true
          description: Human-readable name for the run, set via `metadata.display_name` at launch.
          example: Nightly regression – checkout flow
        status:
          type: string
          enum:
          - PENDING
          - IN QUEUE
          - IN PROGRESS
          - COMPLETED
          - FAILED
          - CANCELLED
          - DELETED
          description: Current status of the simulation run
          example: COMPLETED
        create_time:
          type: string
          format: date-time
          description: Timestamp when the run was created (ISO 8601)
          example: '2025-10-14T12:00:00Z'
        update_time:
          type: string
          format: date-time
          description: Timestamp when the run was last updated (ISO 8601)
          example: '2025-10-14T12:15:00Z'
        agent_id:
          type: string
          nullable: true
          description: ID of the agent being tested
          example: gk3jK9mPq2xRt5vW8yZaBc
        persona_id:
          type: string
          nullable: true
          description: ID of the simulated persona used
          example: hL4kL0nQr3ySt6vX9zAcDd
        test_set_id:
          type: string
          nullable: true
          description: ID of the test set containing test cases
          example: aB1cD2eF
        tags:
          type: array
          items:
            type: string
            maxLength: 200
          maxItems: 20
          description: Tags for categorizing and filtering runs
          example:
          - regression
          - v2.1
        progress:
          $ref: '#/components/schemas/Progress'
        results:
          $ref: '#/components/schemas/Results'
        metadata:
          type: object
          additionalProperties: true
          description: Custom metadata provided during launch
          example:
            campaign_id: q4_2025
            environment: production
        error:
          type: string
          nullable: true
          description: Error message if status is FAILED
          example: Agent connection timeout
    MetricSummary:
      type: object
      description: Aggregated metric statistics
      required:
      - mean
      properties:
        mean:
          type: number
          format: float
          description: Mean value across all test cases
          example: 0.95
        min:
          type: number
          format: float
          description: Minimum value
          example: 0.85
        max:
          type: number
          format: float
          description: Maximum value
          example: 1.0
    UpdateRunRequest:
      type: object
      required:
      - tags
      properties:
        tags:
          type: array
          items:
            type: string
            maxLength: 200
          maxItems: 20
          description: 'Full replacement list of run tags. Duplicate tags are automatically

            removed. Leading/trailing whitespace is stripped. Provide an empty

            list to clear all tags.

            '
          example:
          - baseline
          - production
    Error:
      type: object
      required:
      - code
      - message
      properties:
        code:
          type: string
          enum:
          - INVALID_ARGUMENT
          - UNAUTHENTICATED
          - PERMISSION_DENIED
          - NOT_FOUND
          - INTERNAL
          description: Machine-readable error code
          example: INVALID_ARGUMENT
        message:
          type: string
          description: Human-readable error message
          example: 'Invalid agent_id: Agent not found'
        details:
          type: array
          items:
            $ref: '#/components/schemas/ErrorDetail'
    LaunchOptions:
      type: object
      description: Execution options for launching simulations.
      properties:
        iteration_count:
          type: integer
          minimum: 1
          maximum: 50
          default: 1
          description: Number of times to run each test case.
          example: 3
        concurrency:
          type: integer
          minimum: 1
          maximum: 100
          default: 1
          description: Number of simulations to run concurrently.
          example: 5
        sub_sample_size:
          type: integer
          minimum: 0
          default: 0
          description: 'Number of test cases to randomly sample from the test set.

            Set to 0 (default) to use all test cases. When set to a positive value,

            a random subset of that size will be selected for the run.

            '
          example: 10
        sub_sample_seed:
          type: integer
          format: int32
          minimum: 0
          maximum: 2147483647
          nullable: true
          default: null
          description: 'Random seed for reproducible sub-sampling. Providing the same seed

            with the same sub_sample_size will select the same test cases.

            Only used when sub_sample_size > 0. If null, a random seed is used.

            Valid range: 0 to 2,147,483,647 (2^31 - 1).

            '
          example: 847293
        test_case_ids:
          type: array
          items:
            type: string
          minItems: 1
          maxItems: 100
          nullable: true
          default: null
          description: 'Specific test case IDs to run. When provided, only these test cases

            will be executed. All IDs must belong to the specified test set and

            be active. Mutually exclusive with `sub_sample_size`.

            '
          example:
          - kP7mN3qStUvWxYzAbCdEfG
          - lQ8nO4rTuVwXyZaBcDeFgH
          - mR9oP5sUvWxYzAbCdEfGhI
    LaunchRunRequest:
      type: object
      required:
      - agent_id
      - persona_id
      - test_set_id
      properties:
        agent_id:
          type: string
          minLength: 22
          maxLength: 22
          description: The agent to test. Must be owned by the authenticated organization.
          example: gk3jK9mPq2xRt5vW8yZaBc
        persona_id:
          type: string
          minLength: 22
          maxLength: 22
          description: The simulated persona to use for testing.
          example: hL4kL0nQr3ySt6vX9zAcDd
        test_set_id:
          type: string
          minLength: 8
          maxLength: 8
          description: The test set containing test cases to run.
          example: aB1cD2eF
        metric_ids:
          type: array
          items:
            type: string
            minLength: 22
            maxLength: 22
          description: Optional list of metric IDs to evaluate. If not provided, uses agent's default metrics.
          example:
          - iM5lM1oRs4zTu7wY0aBdEe
          - jN6mN2pSt5aUv8xZ1bCeFf
        mutation_id:
          type: string
          minLength: 26
          maxLength: 26
          nullable: true
          description: 'Single mutation ID to run in addition to the base agent.

            Mutually exclusive with `mutation_ids`. The base agent always runs.

            '
          example: 01ARZ3NDEKTSV4RRFFQ69G5FAV
        mutation_ids:
          type: array
          items:
            type: string
            minLength: 26
            maxLength: 26
          maxItems: 100
          description: 'List of mutation IDs to run in addition to the base agent.

            Mutually exclusive with `mutation_id`. Max 100 mutations.

            The base agent always runs alongside all mutations.


            **Total simulations** = test_cases × iterations × (1 + len(mutation_ids))

            '
          example:
          - 01ARZ3NDEKTSV4RRFFQ69G5FAV
          - 01ARZ3NDEKTSV4RRFFQ69G5FAW
        persona_metrics:
          type: array
          items:
            type: string
          description: 'List of metric names that should evaluate the persona instead of the agent.

            Each entry should be the base metric name (e.g., ''latency'', not ''persona:latency'').

            '
          example:
          - latency
          - interruption_rate
        options:
          $ref: '#/components/schemas/LaunchOptions'
        metadata:
          $ref: '#/components/schemas/LaunchMetadata'
  responses:
    InternalError:
      description: Internal Server Error
      content:
        application/json:
          schema:
            type: object
            required:
            - error
            properties:
              error:
                $ref: '#/components/schemas/Error'
          example:
            error:
              code: INTERNAL
              message: Internal server error
              details:
              - description: An unexpected error occurred. Please contact support.
    ServiceUnavailable:
      description: Service temporarily unavailable
      content:
        application/json:
          schema:
            type: object
            required:
            - error
            properties:
              error:
                $ref: '#/components/schemas/Error'
          example:
            error:
              code: INTERNAL
              message: Service temporarily unavailable
              details:
              - description: Database routing is temporarily unavailable. Please retry.
    Unauthorized:
      description: Unauthorized
      content:
        application/json:
          schema:
            type: object
            required:
            - error
            properties:
              error:
                $ref: '#/components/schemas/Error'
          example:
            error:
              code: UNAUTHENTICATED
              message: Invalid or missing API key
              details:
              - field: X-API-Key
                description: API key is required in the X-API-Key header
    BadRequest:
      description: Bad Request
      content:
        application/json:
          schema:
            type: object
            required:
            - error
            properties:
              error:
                $ref: '#/components/schemas/Error'
          example:
            error:
              code: INVALID_ARGUMENT
              message: Request validation failed
              details:
              - field: iteration_count
                description: Value must be between 1 and 10
    Forbidden:
      description: Forbidden
      content:
        application/json:
          schema:
            type: object
            required:
            - error
            properties:
              error:
                $ref: '#/components/schemas/Error'
          example:
            error:
              code: PERMISSION_DENIED
              message: Access denied
              details:
              - description: Your organization does not have access to this resource
    NotFound:
      description: Not Found
      content:
        application/json:
          schema:
            type: object
            required:
            - error
            properties:
              error:
                $ref: '#/components/schemas/Error'
          example:
            error:
              code: NOT_FOUND
              message: Run not found
              details:
              - field: run_id
                description: Run 'abc123xyz789' does not exist or is not accessible by your organization
  securitySchemes:
    ApiKeyAuth:
      type: apiKey
      in: header
      name: x-api-key
      description: API key for authentication
x-visibility: external