openapi: 3.0.3
info:
  title: Coval Agents Test Sets API
  version: 1.0.0
  description: '

    Manage configurations for simulations and evaluations.

    '
  contact:
    name: Coval API Support
    email: support@coval.dev
    url: https://docs.coval.ai
  license:
    name: Proprietary
    url: https://coval.dev/terms
servers:
- url: https://api.coval.dev/v1
  description: Production API
security:
- ApiKeyAuth: []
tags:
- name: Test Sets
  description: Operations for managing test sets
paths:
  /test-sets:
    get:
      summary: List test sets
      description: List test sets for your organization.
      operationId: listTestSets
      tags:
      - Test Sets
      parameters:
      - name: filter
        in: query
        description: 'Filter expression syntax.

          Values may be unquoted or double-quoted. Values containing spaces must be quoted.

          Example: `test_set_type=SCENARIO`

          '
        schema:
          type: string
        example: test_set_type=SCENARIO
      - name: page_size
        in: query
        description: Maximum number of test sets to return (default 50, max 100)
        schema:
          type: integer
          default: 50
          maximum: 100
      - name: page_token
        in: query
        description: Token for retrieving the next page of results
        schema:
          type: string
      - name: order_by
        in: query
        description: 'Field to order results by. Prefix with `-` for descending order.

          Example: `-create_time` for newest first

          '
        schema:
          type: string
          default: -update_time
        example: -create_time
      - name: tag_filters
        in: query
        description: 'Filter test sets by tags. A resource matches when it has ALL the listed tags (AND-semantics).


          Repeat the parameter for each tag (e.g., `?tag_filters=production&tag_filters=regression`).

          '
        required: false
        style: form
        explode: true
        schema:
          type: array
          items:
            type: string
          maxItems: 20
        example:
        - regression
      responses:
        '200':
          description: Successful response
          content:
            application/json:
              schema:
                type: object
                properties:
                  test_sets:
                    type: array
                    items:
                      $ref: '#/components/schemas/TestSetResource'
                  next_page_token:
                    type: string
                    description: Token for retrieving the next page (empty if last page)
        '401':
          $ref: '#/components/responses/Unauthorized'
        '500':
          $ref: '#/components/responses/InternalError'
    post:
      summary: Create test set
      description: Create a new test set.
      operationId: createTestSet
      tags:
      - Test Sets
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/CreateTestSetRequest'
      responses:
        '201':
          description: Test set created successfully
          content:
            application/json:
              schema:
                type: object
                properties:
                  test_set:
                    $ref: '#/components/schemas/TestSetResource'
        '400':
          $ref: '#/components/responses/BadRequest'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '409':
          description: Slug or display_name already exists
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ErrorResponse'
        '500':
          $ref: '#/components/responses/InternalError'
  /test-sets/{test_set_id}:
    parameters:
    - name: test_set_id
      in: path
      required: true
      description: Test set ID (8-character ID)
      schema:
        type: string
        minLength: 8
        maxLength: 8
      example: abc12345
    get:
      summary: Get test set
      description: Retrieve a test set by ID.
      operationId: getTestSet
      tags:
      - Test Sets
      responses:
        '200':
          description: Successful response
          content:
            application/json:
              schema:
                type: object
                properties:
                  test_set:
                    $ref: '#/components/schemas/TestSetResource'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '404':
          $ref: '#/components/responses/NotFound'
        '500':
          $ref: '#/components/responses/InternalError'
    patch:
      summary: Update test set
      description: Update a test set.
      operationId: updateTestSet
      tags:
      - Test Sets
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/UpdateTestSetRequest'
      responses:
        '200':
          description: Test set updated successfully
          content:
            application/json:
              schema:
                type: object
                properties:
                  test_set:
                    $ref: '#/components/schemas/TestSetResource'
        '400':
          $ref: '#/components/responses/BadRequest'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '404':
          $ref: '#/components/responses/NotFound'
        '409':
          description: Slug or display_name conflict
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ErrorResponse'
        '500':
          $ref: '#/components/responses/InternalError'
    delete:
      summary: Delete test set
      description: 'Delete a test set.

        '
      operationId: deleteTestSet
      tags:
      - Test Sets
      responses:
        '200':
          description: Test set deleted successfully (or already deleted)
          content:
            application/json:
              schema:
                type: object
                description: Empty object
                example: {}
        '401':
          $ref: '#/components/responses/Unauthorized'
        '404':
          description: Test set belongs to different organization
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ErrorResponse'
        '500':
          $ref: '#/components/responses/InternalError'
  /test-sets/{test_set_id}/duplicate:
    post:
      operationId: duplicateTestSet
      summary: Duplicate a test set
      description: Clone an existing test set and its active test cases into a new test set. Returns the new test set.
      tags:
      - Test Sets
      parameters:
      - name: test_set_id
        in: path
        required: true
        schema:
          type: string
        description: ID of the test set to duplicate
      responses:
        '201':
          description: Test set duplicated successfully
          content:
            application/json:
              schema:
                type: object
                properties:
                  test_set:
                    $ref: '#/components/schemas/TestSetResource'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '404':
          $ref: '#/components/responses/NotFound'
        '500':
          $ref: '#/components/responses/InternalError'
  /test-sets/{test_set_id}/versions:
    get:
      summary: List test set versions
      description: List the prior-state version history for a test set, newest first. The live test set is the current version and is not included here. Each entry captures the test set's config and an ordered snapshot of its test-case rows.
      operationId: listTestSetVersions
      tags:
      - Test Sets
      parameters:
      - name: test_set_id
        in: path
        required: true
        description: Test set ID (8-character ID)
        schema:
          type: string
          minLength: 8
          maxLength: 8
        example: abc12345
      responses:
        '200':
          description: Successful response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ListTestSetVersionsResponse'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '404':
          $ref: '#/components/responses/NotFound'
        '500':
          $ref: '#/components/responses/InternalError'
  /test-sets/{test_set_id}/versions/{version_id}/revert:
    post:
      summary: Revert test set version
      description: 'Re-apply a prior version''s content (config + test-case grid) to the live test set. A revert is forward-only: it mints a new version (change_type=revert) and advances the test set, so the response reflects the test set''s new live config. Reverting to the version the test set already points at is rejected with 400. Test cases are not included in the response; fetch them separately via GET /test-cases.'
      operationId: revertTestSetVersion
      tags:
      - Test Sets
      parameters:
      - name: test_set_id
        in: path
        required: true
        description: Test set ID (8-character ID)
        schema:
          type: string
          minLength: 8
          maxLength: 8
        example: abc12345
      - name: version_id
        in: path
        required: true
        description: ULID of the target version to re-apply
        schema:
          type: string
          minLength: 26
          maxLength: 26
          pattern: ^[0-9A-HJKMNP-TV-Z]{26}$
        example: 01KKWQYSF737ZN6X1Q1RYX8M22
      responses:
        '200':
          description: Test set reverted successfully
          content:
            application/json:
              schema:
                type: object
                properties:
                  test_set:
                    $ref: '#/components/schemas/TestSetResource'
        '400':
          $ref: '#/components/responses/BadRequest'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '404':
          $ref: '#/components/responses/NotFound'
        '500':
          $ref: '#/components/responses/InternalError'
  /test-sets/{test_set_id}/agents:
    parameters:
    - name: test_set_id
      in: path
      required: true
      description: Test set ID (8-character ID)
      schema:
        type: string
        minLength: 8
        maxLength: 8
      example: abc12345
    get:
      summary: List associated agents
      description: List the agents currently associated with a test set. Returns every associated agent (not paginated); associations are bounded per test set. An unknown test_set_id returns an empty list rather than 404.
      operationId: listTestSetAgents
      tags:
      - Test Sets
      responses:
        '200':
          description: Successful response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ListTestSetAgentsResponse'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '500':
          $ref: '#/components/responses/InternalError'
  /test-sets/{test_set_id}/agents:add:
    post:
      summary: Associate agents with a test set
      description: 'Associate one or more agents with a test set. The operation is idempotent: already-associated agents are a no-op, and unknown or inactive agent IDs are ignored. The response reflects the resulting association set, so callers can confirm which IDs took effect.'
      operationId: addTestSetAgents
      tags:
      - Test Sets
      parameters:
      - name: test_set_id
        in: path
        required: true
        description: Test set ID (8-character ID)
        schema:
          type: string
          minLength: 8
          maxLength: 8
        example: abc12345
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/AddTestSetAgentsRequest'
      responses:
        '200':
          description: Agents associated successfully
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ListTestSetAgentsResponse'
        '400':
          $ref: '#/components/responses/BadRequest'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '404':
          description: Test set not found for this organization
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ErrorResponse'
        '500':
          $ref: '#/components/responses/InternalError'
  /test-sets/{test_set_id}/agents/{agent_id}:
    delete:
      summary: Remove an agent association
      description: 'Remove one agent''s association with a test set. Idempotent: returns 200 whether or not the association existed.'
      operationId: removeTestSetAgent
      tags:
      - Test Sets
      parameters:
      - name: test_set_id
        in: path
        required: true
        description: Test set ID (8-character ID)
        schema:
          type: string
          minLength: 8
          maxLength: 8
        example: abc12345
      - name: agent_id
        in: path
        required: true
        description: Agent ID (22-character ShortUUID)
        schema:
          type: string
        example: abc123def456ghi789jklm
      responses:
        '200':
          description: Association removed successfully (or already absent)
          content:
            application/json:
              schema:
                type: object
                description: Empty object
                example: {}
        '401':
          $ref: '#/components/responses/Unauthorized'
        '500':
          $ref: '#/components/responses/InternalError'
components:
  responses:
    InternalError:
      description: Internal server error
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/ErrorResponse'
          example:
            error:
              code: INTERNAL
              message: Internal server error
              details:
              - description: An unexpected error occurred while processing the request
    BadRequest:
      description: Invalid request parameters
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/ErrorResponse'
          example:
            error:
              code: INVALID_ARGUMENT
              message: Invalid request parameters
              details:
              - field: display_name
                description: display_name is required and cannot be empty
    Unauthorized:
      description: Authentication failed or missing API key
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/ErrorResponse'
          example:
            error:
              code: UNAUTHENTICATED
              message: Authentication failed
              details:
              - field: X-API-Key
                description: Invalid or missing API key
    NotFound:
      description: Resource not found or doesn't belong to organization
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/ErrorResponse'
          example:
            error:
              code: NOT_FOUND
              message: Test set not found
              details:
              - field: test_set_id
                description: Test set 'abc12345' not found
  schemas:
    TestSetResource:
      type: object
      description: 'Test set resource representation.

        '
      properties:
        name:
          type: string
          description: Resource name in format `test-sets/{id}`
          example: test-sets/abc12345
        id:
          type: string
          description: Test set ID (8-character ID)
          minLength: 8
          maxLength: 8
          example: abc12345
        slug:
          type: string
          description: URL-friendly identifier (unique per organization)
          example: customer-support-scenarios
        display_name:
          type: string
          description: Human-readable test set name
          example: Customer Support Scenarios
        description:
          type: string
          nullable: true
          description: Test set description
          example: Test cases for customer support agent
        test_set_type:
          type: string
          nullable: true
          description: Test set type (e.g., DEFAULT, SCENARIO, TRANSCRIPT, WORKFLOW)
          example: SCENARIO
        test_set_metadata:
          type: object
          description: Additional test set configuration (JSON)
          additionalProperties: true
          example:
            category: support
            priority: high
        parameters:
          type: object
          description: 'Test case parameterization (e.g., {"name": ["Alice", "Bob"]})'
          additionalProperties: true
          example:
            customer_name:
            - Alice
            - Bob
            issue_type:
            - billing
            - technical
        test_case_count:
          type: integer
          description: Number of active test cases (GET endpoint only)
          example: 42
        tags:
          type: array
          description: Tags associated with this test set
          items:
            type: string
          default: []
          example:
          - regression
          - voice
        create_time:
          type: string
          format: date-time
          description: Timestamp when test set was created
          example: '2025-10-14T12:00:00Z'
        update_time:
          type: string
          format: date-time
          nullable: true
          description: Timestamp when test set was last updated
          example: '2025-10-15T14:30:00Z'
    AddTestSetAgentsRequest:
      type: object
      description: Associate one or more agents with the test set.
      required:
      - agent_ids
      properties:
        agent_ids:
          type: array
          description: Agent IDs (22-char ShortUUIDs) to associate. Already-associated IDs are a no-op, and unknown or inactive IDs are ignored; the response reflects the resulting association set.
          minItems: 1
          maxItems: 1000
          items:
            type: string
            minLength: 1
            maxLength: 22
            pattern: ^[A-Za-z0-9]+$
          example:
          - abc123def456ghi789jklm
    TestSetVersionResource:
      type: object
      description: A prior, displaced snapshot of a test set's content (config + test-case grid).
      required:
      - name
      - ulid
      - version_number
      - change_type
      - created_by
      properties:
        name:
          type: string
          description: Resource name
          example: test-sets/abc12345/versions/01KKWQYSF737ZN6X1Q1RYX8M2D
        ulid:
          type: string
          description: Version identifier (26-char ULID)
          example: 01KKWQYSF737ZN6X1Q1RYX8M2D
        version_number:
          type: integer
          description: Per-test-set monotonic version number, 1-based
          example: 2
        change_type:
          type: string
          description: How this version came about. A revert is a save whose new state mirrors an older version.
          enum:
          - save
          - revert
        label:
          type: string
          nullable: true
          description: Optional user-supplied tag
          example: v1.2 prod
        test_set_metadata:
          type: object
          additionalProperties: true
          description: Verbatim config snapshot
        parameters:
          type: object
          additionalProperties: true
          description: Verbatim parameter-sweep snapshot
        test_set_type:
          type: string
          nullable: true
        test_cases:
          type: array
          description: Ordered snapshot of the test set's active rows at this version
          items:
            type: object
            additionalProperties: true
        created_by:
          type: string
          description: Who created this version (user ULID)
        create_time:
          type: string
          format: date-time
          nullable: true
          description: When this version became current
    TestSetAgentResource:
      type: object
      description: An agent associated with a test set.
      required:
      - name
      - agent_id
      - display_name
      properties:
        name:
          type: string
          description: Resource name, "test-sets/{test_set_id}/agents/{agent_id}".
          example: test-sets/abc12345/agents/abc123def456ghi789jklm
        agent_id:
          type: string
          description: Agent identifier (22-char ShortUUID).
          example: abc123def456ghi789jklm
        display_name:
          type: string
          description: Human-readable agent name.
          example: Support Bot
    UpdateTestSetRequest:
      type: object
      description: Update request. Only provided fields will be updated.
      properties:
        display_name:
          type: string
          minLength: 1
          maxLength: 100
          description: Human-readable test set name
          example: Updated Test Set Name
        slug:
          type: string
          maxLength: 100
          description: URL-friendly identifier
          example: updated-slug
        description:
          type: string
          nullable: true
          description: Test set description
          example: Updated description
        test_set_type:
          type: string
          maxLength: 50
          description: Test set type
          example: WORKFLOW
        test_set_metadata:
          type: object
          description: Additional configuration (null = don't update, {} = clear field)
          additionalProperties: true
          example:
            category: support
            updated: true
        parameters:
          type: object
          description: Test case parameterization (null = don't update, {} = clear field)
          additionalProperties: true
          example:
            name:
            - Alice
            - Bob
        tags:
          type: array
          nullable: true
          description: Tags to associate with this test set. Null or omitted leaves tags unchanged. Pass [] to clear all tags.
          items:
            type: string
          example:
          - regression
    ListTestSetAgentsResponse:
      type: object
      description: The agents currently associated with the test set.
      required:
      - agents
      properties:
        agents:
          type: array
          items:
            $ref: '#/components/schemas/TestSetAgentResource'
    CreateTestSetRequest:
      type: object
      description: Request body for creating a test set
      required:
      - display_name
      properties:
        display_name:
          type: string
          minLength: 1
          maxLength: 100
          description: Human-readable test set name
          example: Customer Support Scenarios
        slug:
          type: string
          maxLength: 100
          description: URL-friendly identifier (auto-generated from display_name if not provided)
          example: customer-support-scenarios
        description:
          type: string
          nullable: true
          description: Test set description
          example: Test cases for customer support agent
        test_set_type:
          type: string
          maxLength: 50
          description: Test set type (e.g., DEFAULT, SCENARIO, TRANSCRIPT, WORKFLOW)
          example: SCENARIO
        test_set_metadata:
          type: object
          description: Additional test set configuration (JSON)
          additionalProperties: true
          default: {}
          example:
            category: support
            priority: high
        parameters:
          type: object
          description: Test case parameterization
          additionalProperties: true
          default: {}
          example:
            customer_name:
            - Alice
            - Bob
            issue_type:
            - billing
            - technical
        tags:
          type: array
          nullable: true
          description: Tags to associate with this test set. Null or omitted creates the test set with no tags. Pass [] for an empty tag list.
          items:
            type: string
          example:
          - regression
    ErrorResponse:
      type: object
      description: Standard error response
      required:
      - error
      properties:
        error:
          type: object
          required:
          - code
          - message
          - details
          properties:
            code:
              type: string
              description: Machine-readable error code
              example: INVALID_ARGUMENT
            message:
              type: string
              description: Human-readable error message
              example: Invalid request parameters
            details:
              type: array
              description: Detailed error information
              items:
                type: object
                properties:
                  field:
                    type: string
                    description: Field that caused the error
                    example: slug
                  description:
                    type: string
                    description: Description of the error
                    example: slug already exists for this organization
    ListTestSetVersionsResponse:
      type: object
      description: Response for GET /v1/test-sets/{test_set_id}/versions (newest first)
      required:
      - versions
      properties:
        versions:
          type: array
          items:
            $ref: '#/components/schemas/TestSetVersionResource'
  securitySchemes:
    ApiKeyAuth:
      type: apiKey
      in: header
      name: x-api-key
      description: API key for authentication
x-visibility: external