openapi: 3.0.3
info:
  title: Coval Agents Review Annotations API
  version: 1.0.0
  description: '

    Manage configurations for simulations and evaluations.

    '
  contact:
    name: Coval API Support
    email: support@coval.dev
    url: https://docs.coval.ai
  license:
    name: Proprietary
    url: https://coval.dev/terms
servers:
- url: https://api.coval.dev/v1
  description: Production API
security:
- ApiKeyAuth: []
tags:
- name: Review Annotations
  description: Annotation CRUD operations
paths:
  /review-annotations:
    get:
      operationId: listReviewAnnotations
      summary: List review annotations
      description: List review annotations for your organization with optional filtering, sorting, and pagination.
      tags:
      - Review Annotations
      security:
      - ApiKeyAuth: []
      parameters:
      - name: page_size
        in: query
        required: false
        schema:
          type: integer
          minimum: 1
          maximum: 100
          default: 50
        description: Maximum number of results per page
      - name: page_token
        in: query
        required: false
        schema:
          type: string
        description: Opaque pagination token from previous response
      - name: order_by
        in: query
        required: false
        schema:
          type: string
          example: -create_time
        description: 'Sort field and direction. Prefix with `-` for descending.

          Valid fields: `create_time`, `update_time`, `assignee`, `priority`.

          '
      - name: filter
        in: query
        required: false
        schema:
          type: string
        description: 'Filter expression. Supported fields:

          - `simulation_output_id` — exact match

          - `metric_id` — exact match

          - `assignee` — exact match

          - `status` — `ACTIVE` or `ARCHIVED`

          - `completion_status` — `PENDING` or `COMPLETED`

          - `project_id` — annotations belonging to a project

          '
      responses:
        '200':
          description: Annotations retrieved successfully
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ListReviewAnnotationsResponse'
              examples:
                success:
                  $ref: '#/components/examples/ListAnnotationsSuccess'
        '400':
          $ref: '#/components/responses/BadRequest'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '500':
          $ref: '#/components/responses/InternalError'
    post:
      operationId: createReviewAnnotation
      summary: Create review annotation
      description: Create a new review annotation for your organization.
      tags:
      - Review Annotations
      security:
      - ApiKeyAuth: []
      parameters:
      - name: project_id
        in: query
        required: false
        schema:
          type: string
        description: Optional project ID to apply project-rule-aware completion status
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/CreateReviewAnnotationRequest'
            examples:
              basic:
                summary: Create a basic annotation
                value:
                  simulation_output_id: sim-output-abc123
                  metric_id: metric-accuracy-001
                  assignee: reviewer@company.com
              with_ground_truth:
                summary: Create with ground truth (auto-completes)
                value:
                  simulation_output_id: sim-output-abc123
                  metric_id: metric-accuracy-001
                  assignee: reviewer@company.com
                  ground_truth_float_value: 0.95
                  reviewer_notes: Verified correct response
      responses:
        '201':
          description: Annotation created successfully
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/CreateReviewAnnotationResponse'
        '400':
          $ref: '#/components/responses/BadRequest'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '404':
          $ref: '#/components/responses/NotFound'
        '500':
          $ref: '#/components/responses/InternalError'
  /review-annotations/{annotation_id}:
    get:
      operationId: getReviewAnnotation
      summary: Get review annotation
      description: Get a single review annotation by ID.
      tags:
      - Review Annotations
      security:
      - ApiKeyAuth: []
      parameters:
      - name: annotation_id
        in: path
        required: true
        schema:
          type: string
        description: The annotation ID
      - name: project_id
        in: query
        required: false
        schema:
          type: string
        description: Optional project ID to apply project-rule-aware completion status
      responses:
        '200':
          description: Annotation retrieved successfully
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/GetReviewAnnotationResponse'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '404':
          $ref: '#/components/responses/NotFound'
        '500':
          $ref: '#/components/responses/InternalError'
    patch:
      operationId: updateReviewAnnotation
      summary: Update review annotation
      description: 'Partially update a review annotation.

        Providing a ground truth value auto-sets `completion_status` to `COMPLETED`.

        '
      tags:
      - Review Annotations
      security:
      - ApiKeyAuth: []
      parameters:
      - name: annotation_id
        in: path
        required: true
        schema:
          type: string
        description: The annotation ID
      - name: project_id
        in: query
        required: false
        schema:
          type: string
        description: Optional project ID to apply project-rule-aware completion status
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/UpdateReviewAnnotationRequest'
            examples:
              set_ground_truth:
                summary: Set ground truth (auto-completes)
                value:
                  ground_truth_float_value: 0.85
              update_assignee:
                summary: Reassign annotation
                value:
                  assignee: new-reviewer@company.com
      responses:
        '200':
          description: Annotation updated successfully
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/UpdateReviewAnnotationResponse'
        '400':
          $ref: '#/components/responses/BadRequest'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '404':
          $ref: '#/components/responses/NotFound'
        '500':
          $ref: '#/components/responses/InternalError'
    delete:
      operationId: deleteReviewAnnotation
      summary: Delete review annotation
      description: Delete a review annotation.
      tags:
      - Review Annotations
      security:
      - ApiKeyAuth: []
      parameters:
      - name: annotation_id
        in: path
        required: true
        schema:
          type: string
        description: The annotation ID
      responses:
        '200':
          description: Annotation deleted successfully
          content:
            application/json:
              schema:
                type: object
              example: {}
        '401':
          $ref: '#/components/responses/Unauthorized'
        '404':
          $ref: '#/components/responses/NotFound'
        '500':
          $ref: '#/components/responses/InternalError'
  /review-annotations/metric-health-stats:
    get:
      operationId: getReviewMetricHealthStats
      summary: Get metric health stats
      description: Compact machine-vs-human and human-vs-human agreement health for one metric, computed from its review annotations. Metric-scoped (no project).
      tags:
      - Review Annotations
      security:
      - ApiKeyAuth: []
      parameters:
      - name: metric_id
        in: query
        required: true
        schema:
          type: string
        description: The metric ID (22-char ShortUUID).
      responses:
        '200':
          description: Health stats computed successfully
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/GetMetricHealthStatsResponse'
        '400':
          $ref: '#/components/responses/BadRequest'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '403':
          $ref: '#/components/responses/Forbidden'
        '503':
          $ref: '#/components/responses/ServiceUnavailable'
        '500':
          $ref: '#/components/responses/InternalError'
  /review-annotations:withMetricOutputs:
    post:
      operationId: getReviewAnnotationsWithMetricOutputs
      summary: Get annotations with metric outputs
      description: Review annotations grouped by metric plus the latest pipeline metric outputs, scoped by any of metric_ids / simulation_output_ids / project_ids (all optional). Offset-paginated via page_token / page_size.
      tags:
      - Review Annotations
      security:
      - ApiKeyAuth: []
      requestBody:
        required: false
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/GetAnnotationsWithMetricOutputsRequest'
      responses:
        '200':
          description: Annotations retrieved successfully
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/GetAnnotationsWithMetricOutputsResponse'
        '400':
          $ref: '#/components/responses/BadRequest'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '403':
          $ref: '#/components/responses/Forbidden'
        '503':
          $ref: '#/components/responses/ServiceUnavailable'
        '500':
          $ref: '#/components/responses/InternalError'
components:
  responses:
    InternalError:
      description: Internal server error
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/ErrorResponse'
          example:
            error:
              code: INTERNAL
              message: Internal server error
              details:
              - description: An unexpected error occurred
    Forbidden:
      description: Insufficient permissions
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/ErrorResponse'
          example:
            error:
              code: PERMISSION_DENIED
              message: Insufficient permissions
              details:
              - field: permissions
                description: The API key does not include the required reviews permission.
    ServiceUnavailable:
      description: Service temporarily unavailable
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/ErrorResponse'
          example:
            error:
              code: INTERNAL
              message: Service temporarily unavailable
              details:
              - description: Database routing is temporarily unavailable. Please retry.
    Unauthorized:
      description: Authentication failed
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/ErrorResponse'
          example:
            error:
              code: UNAUTHENTICATED
              message: Authentication failed
              details:
              - field: X-API-Key
                description: Invalid or missing API key
    NotFound:
      description: Resource not found
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/ErrorResponse'
          example:
            error:
              code: NOT_FOUND
              message: Resource not found
              details:
              - field: annotation_id
                description: Annotation not found or not accessible by your organization
    BadRequest:
      description: Invalid request
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/ErrorResponse'
          example:
            error:
              code: INVALID_ARGUMENT
              message: Invalid request body
              details:
              - field: simulation_output_id
                description: simulation_output_id is required
  schemas:
    UpdateReviewAnnotationResponse:
      type: object
      required:
      - review_annotation
      properties:
        review_annotation:
          $ref: '#/components/schemas/ReviewAnnotationResource'
    GetMetricHealthStatsResponse:
      type: object
      required:
      - conversations
      - annotations
      - agreement_rate
      - agreements
      - disagreements
      - human_agreement
      properties:
        conversations:
          type: integer
        annotations:
          type: integer
        agreement_rate:
          type: number
        agreements:
          type: integer
        disagreements:
          type: integer
        human_agreement:
          $ref: '#/components/schemas/HumanAgreementStatsResource'
    ListReviewAnnotationsResponse:
      type: object
      required:
      - review_annotations
      properties:
        review_annotations:
          type: array
          items:
            $ref: '#/components/schemas/ReviewAnnotationResource'
        next_page_token:
          type: string
          nullable: true
          description: Opaque token for fetching the next page
    HumanAgreementStatsResource:
      type: object
      required:
      - human_agreement_rate
      - human_agreement_total
      - human_agreements
      - human_disagreements
      properties:
        human_agreement_rate:
          type: number
        human_agreement_total:
          type: integer
        human_agreements:
          type: integer
        human_disagreements:
          type: integer
    AnnotationStatus:
      type: string
      enum:
      - ACTIVE
      - ARCHIVED
      description: 'Annotation lifecycle status: ACTIVE (default) or ARCHIVED'
    ErrorResponse:
      type: object
      properties:
        error:
          type: object
          properties:
            code:
              type: string
              description: Error code
              example: INVALID_ARGUMENT
            message:
              type: string
              description: Human-readable error message
              example: Invalid request parameter
            details:
              type: array
              description: Detailed error information
              items:
                type: object
                properties:
                  field:
                    type: string
                    nullable: true
                    description: Field that caused the error
                  description:
                    type: string
                    description: Detailed error description
    ReviewMetricOutputForAnnotationResource:
      type: object
      required:
      - created_at
      properties:
        output_type:
          type: string
          nullable: true
        float_value:
          type: number
          nullable: true
        string_value:
          type: string
          nullable: true
        set_value:
          type: array
          nullable: true
          items:
            type: string
        error_status:
          type: string
          nullable: true
        created_at:
          type: string
          format: date-time
        explanation:
          type: string
          nullable: true
        result_json:
          nullable: true
        subvalues_by_timestamp:
          nullable: true
        metric_version_number:
          type: integer
          nullable: true
        metric_version_ulid:
          type: string
          nullable: true
    UpdateReviewAnnotationRequest:
      type: object
      properties:
        ground_truth_float_value:
          type: number
          format: float
          nullable: true
          description: Ground truth numeric value (auto-completes annotation)
        ground_truth_string_value:
          type: string
          nullable: true
          description: Ground truth string value (auto-completes annotation)
        ground_truth_subvalues_by_timestamp:
          type: array
          nullable: true
          items:
            type: object
          description: Ground truth subvalues
        reviewer_notes:
          type: string
          nullable: true
          description: Reviewer notes
        priority:
          $ref: '#/components/schemas/AnnotationPriority'
        assignee:
          type: string
          nullable: true
          description: Reassign to a different reviewer
    GetAnnotationsWithMetricOutputsRequest:
      type: object
      additionalProperties: false
      properties:
        metric_id:
          type: string
          nullable: true
        metric_ids:
          type: array
          nullable: true
          maxItems: 10000
          items:
            type: string
        simulation_output_ids:
          type: array
          nullable: true
          maxItems: 10000
          items:
            type: string
        project_ids:
          type: array
          nullable: true
          maxItems: 1000
          items:
            type: string
        filter_archived:
          type: boolean
          default: false
        include_metric_output_details:
          type: boolean
          default: true
        include_audio_lengths:
          type: boolean
          default: true
        page_size:
          type: integer
          nullable: true
          minimum: 1
          maximum: 1000
        page_token:
          type: string
          nullable: true
    GetReviewAnnotationResponse:
      type: object
      required:
      - review_annotation
      properties:
        review_annotation:
          $ref: '#/components/schemas/ReviewAnnotationResource'
    CreateReviewAnnotationRequest:
      type: object
      required:
      - simulation_output_id
      - metric_id
      - assignee
      properties:
        simulation_output_id:
          type: string
          minLength: 1
          description: Simulation output ID to link
        metric_id:
          type: string
          minLength: 1
          description: Metric ID to link
        assignee:
          type: string
          minLength: 1
          description: Email of the reviewer to assign
        ground_truth_float_value:
          type: number
          format: float
          nullable: true
          description: Optional ground truth numeric value (auto-completes annotation)
        ground_truth_string_value:
          type: string
          nullable: true
          description: Optional ground truth string value (auto-completes annotation)
        ground_truth_subvalues_by_timestamp:
          type: array
          nullable: true
          items:
            type: object
          description: Optional ground truth subvalues
        reviewer_notes:
          type: string
          nullable: true
          description: Optional reviewer notes
        priority:
          $ref: '#/components/schemas/AnnotationPriority'
    GetAnnotationsWithMetricOutputsResponse:
      type: object
      required:
      - annotations
      - metric_outputs
      - audio_lengths_by_simulation_output_id
      properties:
        annotations:
          type: object
          description: Annotations keyed by metric ID.
          additionalProperties:
            type: array
            items:
              $ref: '#/components/schemas/ReviewAnnotationResource'
        metric_outputs:
          type: object
          description: Latest pipeline metric output keyed by metric ID then simulation output ID.
          additionalProperties:
            type: object
            additionalProperties:
              $ref: '#/components/schemas/ReviewMetricOutputForAnnotationResource'
        audio_lengths_by_simulation_output_id:
          type: object
          additionalProperties:
            type: number
            nullable: true
        next_page_token:
          type: string
          nullable: true
    CreateReviewAnnotationResponse:
      type: object
      required:
      - review_annotation
      properties:
        review_annotation:
          $ref: '#/components/schemas/ReviewAnnotationResource'
    AnnotationPriority:
      type: string
      enum:
      - PRIORITY_PRIMARY
      - PRIORITY_STANDARD
      default: PRIORITY_STANDARD
      description: Annotation priority level
    ReviewAnnotationResource:
      type: object
      description: A single review annotation resource.
      required:
      - name
      - id
      - simulation_output_id
      - metric_id
      - assignee
      - status
      - completion_status
      - priority
      - create_time
      - update_time
      properties:
        name:
          type: string
          description: 'Resource name: review-annotations/{id}'
          example: review-annotations/abc123def456ghi789jklm
        id:
          type: string
          description: Unique annotation ID
          example: abc123def456ghi789jklm
        simulation_output_id:
          type: string
          description: Linked simulation output ID
        metric_id:
          type: string
          description: Linked metric ID
        assignee:
          type: string
          description: Email of the assigned reviewer
        ground_truth_float_value:
          type: number
          format: float
          nullable: true
          description: Reviewer-provided ground truth numeric value
        ground_truth_string_value:
          type: string
          nullable: true
          description: Reviewer-provided ground truth string value
        ground_truth_subvalues_by_timestamp:
          type: array
          nullable: true
          description: Reviewer-provided ground truth subvalues keyed by timestamp
          items:
            type: object
        reviewer_notes:
          type: string
          nullable: true
          description: Free-text reviewer notes
        status:
          $ref: '#/components/schemas/AnnotationStatus'
        completion_status:
          $ref: '#/components/schemas/CompletionStatus'
        priority:
          $ref: '#/components/schemas/AnnotationPriority'
        create_time:
          type: string
          format: date-time
          description: Creation timestamp (ISO 8601)
        update_time:
          type: string
          format: date-time
          description: Last update timestamp (ISO 8601)
    CompletionStatus:
      type: string
      enum:
      - PENDING
      - COMPLETED
      description: 'Annotation completion status: PENDING or COMPLETED'
  examples:
    ListAnnotationsSuccess:
      summary: Successful annotation list response
      value:
        review_annotations:
        - name: review-annotations/abc123def456ghi789jklm
          id: abc123def456ghi789jklm
          simulation_output_id: sim-output-001
          metric_id: metric-accuracy
          assignee: alice@company.com
          ground_truth_float_value: null
          ground_truth_string_value: null
          reviewer_notes: null
          status: ACTIVE
          completion_status: PENDING
          priority: PRIORITY_STANDARD
          create_time: '2026-01-15T12:00:00Z'
          update_time: '2026-01-15T12:00:00Z'
        next_page_token: null
  securitySchemes:
    ApiKeyAuth:
      type: apiKey
      in: header
      name: x-api-key
      description: API key for authentication
x-visibility: external