openapi: 3.0.0
info:
  title: Plum API
  version: 1.1.0
servers:
  - url: https://beta.getplum.ai/v1
paths:
  /data/seeds:
    get:
      summary: List all seed datasets
      security:
        - ApiKeyAuth: []
      responses:
        "200":
          description: Successfully retrieved list of seed datasets
          content:
            application/json:
              schema:
                type: object
                properties:
                  datasets:
                    type: array
                    items:
                      type: object
                      properties:
                        id:
                          type: string
                        created_at:
                          type: string
  /data/seed/{id}:
    get:
      summary: Retrieve a specific seed dataset by ID
      security:
        - ApiKeyAuth: []
      parameters:
        - name: id
          in: path
          required: true
          schema:
            type: string
          description: ID of the seed dataset or "latest" to get the most recent dataset
      responses:
        "200":
          description: Successfully retrieved dataset
          content:
            application/json:
              schema:
                type: object
                properties:
                  id:
                    type: string
                  data:
                    type: array
                    items:
                      type: object
                      properties:
                        input:
                          type: string
                        output:
                          type: string
                  system_prompt:
                    type: string
                  created_at:
                    type: string
    post:
      summary: Insert or overwrite seed data by ID
      security:
        - ApiKeyAuth: []
      parameters:
        - name: id
          in: path
          required: true
          schema:
            type: string
      requestBody:
        required: true
        content:
          application/json:
            schema:
              type: object
              properties:
                data:
                  type: array
                  items:
                    type: object
                    properties:
                      input:
                        type: string
                      output:
                        type: string
                      id:
                        type: string
                system_prompt:
                  type: string
              required:
                - data
                - system_prompt
      responses:
        "200":
          description: Data successfully inserted
          content:
            application/json:
              schema:
                type: object
                properties:
                  id:
                    type: string
  /data/seed/pair:
    post:
      summary: Add a single pair to a dataset (creates new dataset if the system_prompt_template that's provided doesn't correspond to an existing dataset)
      security:
        - ApiKeyAuth: []
      requestBody:
        required: true
        content:
          application/json:
            schema:
              type: object
              properties:
                id:
                  type: string
                  description: Optional ID for the pair, will be generated if not provided
                input:
                  type: string
                output:
                  type: string
                labels:
                  type: array
                  items:
                    type: string
                input_media:
                  type: string
                  format: byte
                  description: Optional base64 encoded media data
                system_prompt_template:
                  type: string
                  description: If this prompt already exists, the pair will be added to the existing dataset. If not, a new dataset will be created.
              required:
                - input
                - output
                - system_prompt_template
      responses:
        "200":
          description: Pair successfully added to dataset (existing or newly created)
          content:
            application/json:
              schema:
                type: object
                properties:
                  dataset_id:
                    type: string
                  pair_id:
                    type: string
        "400":
          description: Invalid request - missing required fields
          content:
            application/json:
              schema:
                type: object
                properties:
                  error:
                    type: string
  /data/seed/{id}/pair:
    post:
      summary: Add a single pair to an existing seed dataset by ID
      security:
        - ApiKeyAuth: []
      parameters:
        - name: id
          in: path
          required: true
          schema:
            type: string
          description: ID of the existing seed dataset
      requestBody:
        required: true
        content:
          application/json:
            schema:
              type: object
              properties:
                id:
                  type: string
                  description: Optional ID for the pair, will be generated if not provided
                input:
                  type: string
                output:
                  type: string
                labels:
                  type: array
                  items:
                    type: string
                input_media:
                  type: string
                  format: byte
                  description: Optional base64 encoded media data
                system_prompt_template:
                  type: string
                  description: Optional - if provided when using a specific dataset ID, it will be ignored
              required:
                - input
                - output
      responses:
        "200":
          description: Pair successfully added to dataset
          content:
            application/json:
              schema:
                type: object
                properties:
                  dataset_id:
                    type: string
                  pair_id:
                    type: string
        "400":
          description: Invalid request - missing required fields or dataset ID doesn't exist
          content:
            application/json:
              schema:
                type: object
                properties:
                  error:
                    type: string
  /data/seed:
    post:
      summary: Upload a new seed dataset
      security:
        - ApiKeyAuth: []
      requestBody:
        required: true
        content:
          application/json:
            schema:
              type: object
              properties:
                data:
                  type: array
                  items:
                    type: object
                    properties:
                      input:
                        type: string
                      output:
                        type: string
                      id:
                        type: string
                system_prompt:
                  type: string
              required:
                - data
                - system_prompt
      responses:
        "200":
          description: Dataset successfully uploaded
          content:
            application/json:
              schema:
                type: object
                properties:
                  id:
                    type: string
  /data/synthetics:
    get:
      summary: List all synthetic datasets
      security:
        - ApiKeyAuth: []
      responses:
        "200":
          description: Successfully retrieved list of synthetic datasets
          content:
            application/json:
              schema:
                type: object
                properties:
                  datasets:
                    type: array
                    items:
                      type: object
                      properties:
                        id:
                          type: string
                        created_at:
                          type: string
  /data/synthetic/{id}:
    get:
      summary: Retrieve synthetic data by ID
      security:
        - ApiKeyAuth: []
      parameters:
        - name: id
          in: path
          required: true
          schema:
            type: string
          description: ID of the synthetic dataset or "latest" to get the most recent dataset
      responses:
        "200":
          description: Successfully retrieved synthetic data
          content:
            application/json:
              schema:
                type: object
                properties:
                  id:
                    type: string
                  data:
                    type: array
                    items:
                      type: object
                      properties:
                        input:
                          type: string
                        output:
                          type: string
                  system_prompt:
                    type: string
                  created_at:
                    type: string
    put:
      summary: Insert or overwrite synthetic data by ID
      security:
        - ApiKeyAuth: []
      parameters:
        - name: id
          in: path
          required: true
          schema:
            type: string
      requestBody:
        required: true
        content:
          application/json:
            schema:
              type: object
              properties:
                data:
                  type: array
                  items:
                    type: object
                    properties:
                      input:
                        type: string
                      output:
                        type: string
                system_prompt:
                  type: string
              required:
                - data
      responses:
        "200":
          description: Data successfully inserted
          content:
            application/json:
              schema:
                type: object
                properties:
                  id:
                    type: string
  /data/synthetic:
    post:
      summary: Upload new synthetic data
      security:
        - ApiKeyAuth: []
      requestBody:
        required: true
        content:
          application/json:
            schema:
              type: object
              properties:
                data:
                  type: array
                  items:
                    type: object
                    properties:
                      input:
                        type: string
                      output:
                        type: string
                system_prompt:
                  type: string
              required:
                - data
      responses:
        "200":
          description: Synthetic data successfully uploaded
          content:
            application/json:
              schema:
                type: object
                properties:
                  id:
                    type: string
  /data/synthetic_openai/{id}:
    get:
      summary: Retrieve synthetic data formatted for OpenAI by ID
      security:
        - ApiKeyAuth: []
      parameters:
        - name: id
          in: path
          required: true
          schema:
            type: string
      responses:
        "200":
          description: Successfully retrieved OpenAI formatted synthetic data
          content:
            text/plain:
              schema:
                type: string
                description: JSONL format for OpenAI fine-tuning
  /question/{id}:
    get:
      summary: Get a specific metric definition by ID
      security:
        - ApiKeyAuth: []
      parameters:
        - name: id
          in: path
          required: true
          schema:
            type: string
      responses:
        "200":
          description: Successfully retrieved metric definition
          content:
            application/json:
              schema:
                type: object
                properties:
                  metrics_id:
                    type: string
                  definitions:
                    type: array
                    items:
                      type: object
                      properties:
                        name:
                          type: string
                        description:
                          type: string
                  system_prompt:
                    type: string
                  metric_count:
                    type: integer
                  created_at:
                    type: string
  /evaluation/{id}:
    get:
      summary: Get a specific evaluation result by ID
      security:
        - ApiKeyAuth: []
      parameters:
        - name: id
          in: path
          required: true
          schema:
            type: string
      responses:
        "200":
          description: Successfully retrieved evaluation result
          content:
            application/json:
              schema:
                type: object
                properties:
                  results_id:
                    type: string
                  created_at:
                    type: string
                  dataset_id:
                    type: string
                  metrics_id:
                    type: string
                  metrics_definitions:
                    type: array
                    items:
                      type: string
                  pair_count:
                    type: integer
                  system_prompt:
                    type: string
                  score_means:
                    type: array
                    items:
                      type: number
                  score_medians:
                    type: array
                    items:
                      type: number
                  score_mins:
                    type: array
                    items:
                      type: number
                  score_maxes:
                    type: array
                    items:
                      type: number
                  score_std_devs:
                    type: array
                    items:
                      type: number
                  score_confidence_intervals:
                    type: array
                    items:
                      type: object
                      properties:
                        ci_low:
                          type: number
                        ci_high:
                          type: number
                        ci_confidence:
                          type: number
                  min_scoring_pairs:
                    type: array
                    items:
                      type: array
                      items:
                        type: object
                        properties:
                          reason:
                            type: string
                          pair_id:
                            type: string
                  human_critique:
                    type: array
                    items:
                      type: object
                      properties:
                        id:
                          type: string
                        pair_id:
                          type: string
                        metric_idx:
                          type: integer
                        comment:
                          type: string
                        vote:
                          type: integer
                          enum: [-1, 0, 1]
                        user:
                          type: string
                        user_email:
                          type: string
                        time:
                          type: string
  /vote_eval_pair/{id}:
    put:
      summary: Update evaluation result with human feedback
      security:
        - ApiKeyAuth: []
      parameters:
        - name: id
          in: path
          required: true
          schema:
            type: string
          description: ID of the evaluation result to update
      requestBody:
        required: true
        content:
          application/json:
            schema:
              type: object
              properties:
                pair_id:
                  type: string
                  description: ID of the pair to provide feedback on
                metric_idx:
                  type: integer
                  description: Index of the metric in the Metrics.Definitions array
                vote:
                  type: integer
                  enum: [-1, 0, 1]
                  description: Vote for the pair (-1 = negative, 0 = neutral, 1 = positive)
                comment:
                  type: string
                  description: Optional comment providing feedback on the pair
              required:
                - pair_id
                - metric_idx
                - vote
      responses:
        "200":
          description: Evaluation result successfully updated with human feedback
          content:
            application/json:
              schema:
                type: object
                properties:
                  success:
                    type: boolean
                  results_id:
                    type: string
                  pair_id:
                    type: string
        "400":
          description: Bad request - invalid input
          content:
            application/json:
              schema:
                type: object
                properties:
                  error:
                    type: string
        "500":
          description: Internal server error
          content:
            application/json:
              schema:
                type: object
                properties:
                  error:
                    type: string
  /questions:
    post:
      summary: Generate evaluation metrics
      security:
        - ApiKeyAuth: []
      requestBody:
        required: true
        content:
          application/json:
            schema:
              type: object
              properties:
                system_prompt:
                  type: string
              required:
                - system_prompt
      responses:
        "200":
          description: Metrics successfully generated
          content:
            application/json:
              schema:
                type: object
                properties:
                  metrics_id:
                    type: string
                  definitions:
                    type: array
                    items:
                      type: object
                  created_at:
                    type: string
  /list_questions:
    get:
      summary: List all available evaluation metrics
      security:
        - ApiKeyAuth: []
      responses:
        "200":
          description: Successfully retrieved list of evaluation metrics
          content:
            application/json:
              schema:
                type: object
                properties:
                  metrics:
                    type: object
                    additionalProperties:
                      type: object
                      properties:
                        metrics_id:
                          type: string
                        definitions:
                          type: array
                          items:
                            type: string
                        system_prompt:
                          type: string
                        metric_count:
                          type: integer
                        created_at:
                          type: string
                  total_count:
                    type: integer
  /list_evaluations:
    get:
      summary: List all available evaluation results
      security:
        - ApiKeyAuth: []
      parameters:
        - name: hydrate
          in: query
          required: false
          schema:
            type: boolean
            default: false
          description: Whether to return full evaluation results or just metadata
      responses:
        "200":
          description: Successfully retrieved list of evaluation results
          content:
            application/json:
              schema:
                type: object
                properties:
                  results:
                    type: object
                    additionalProperties:
                      type: object
                      properties:
                        results_id:
                          type: string
                        created_at:
                          type: string
                        dataset_id:
                          type: string
                        metrics_id:
                          type: string
                        metrics_definitions:
                          type: array
                          items:
                            type: string
                        pair_count:
                          type: integer
                        system_prompt:
                          type: string
                        score_means:
                          type: array
                          items:
                            type: number
                        score_medians:
                          type: array
                          items:
                            type: number
                        score_mins:
                          type: array
                          items:
                            type: number
                        score_maxes:
                          type: array
                          items:
                            type: number
                        score_std_devs:
                          type: array
                          items:
                            type: number
                        score_confidence_intervals:
                          type: array
                          items:
                            type: object
                            properties:
                              ci_low:
                                type: number
                              ci_high:
                                type: number
                              ci_confidence:
                                type: number
                        min_scoring_pairs:
                          type: array
                          items:
                            type: array
                            items:
                              type: object
                              properties:
                                reason:
                                  type: string
                                pair_id:
                                  type: string
                  total_count:
                    type: integer
  /specify_questions:
    post:
      summary: Create custom metric definitions
      security:
        - ApiKeyAuth: []
      requestBody:
        required: true
        content:
          application/json:
            schema:
              type: object
              properties:
                metrics:
                  type: array
                  items:
                    type: string
                    description: String description of each metric
              required:
                - metrics
      responses:
        "200":
          description: Custom metrics successfully created
          content:
            application/json:
              schema:
                type: object
                properties:
                  metrics_id:
                    type: string
                  created_at:
                    type: string
  /evaluate:
    post:
      summary: Evaluate datasets with specified metrics
      security:
        - ApiKeyAuth: []
      requestBody:
        required: true
        content:
          application/json:
            schema:
              type: object
              properties:
                synthetic_data_id:
                  type: string
                  description: ID of synthetic dataset to evaluate (either this or seed_data_id is required)
                seed_data_id:
                  type: string
                  description: ID of seed dataset to evaluate (either this or synthetic_data_id is required)
                metrics_id:
                  type: string
                  description: ID of metrics to use for evaluation (will use latest if not provided)
                pair_query:
                  type: object
                  description: Optional query parameters to filter dataset pairs
                  properties:
                    latest_n_pairs:
                      type: integer
                      description: Maximum number of latest pairs to include (defaults to 150 if not provided)
                    pair_label:
                      type: string
                      description: Filter pairs by label (optional)
                    last_n_seconds:
                      type: integer
                      description: Filter pairs created in the last N seconds (optional)
      responses:
        "200":
          description: Evaluation successfully completed
          content:
            application/json:
              schema:
                type: object
                properties:
                  eval_results_id:
                    type: string
                  dataset_id:
                    type: string
                  created_at:
                    type: string
                  scores:
                    type: array
                    items:
                      type: object
                      properties:
                        metric:
                          type: string
                        mean_score:
                          type: number
                        std_dev:
                          type: number
                        ci_low:
                          type: number
                        ci_high:
                          type: number
                        ci_confidence:
                          type: number
                        median_score:
                          type: number
                        min_score:
                          type: number
                        max_score:
                          type: number
                        lowest_scoring_pairs:
                          type: array
                          items:
                            type: object
                            properties:
                              pair_id:
                                type: string
                              score_reason:
                                type: string
                  pair_count:
                    type: integer
        "403":
          description: Forbidden - dataset size exceeds plan limit
          content:
            application/json:
              schema:
                type: object
                properties:
                  error:
                    type: string
        "500":
          description: Internal server error
          content:
            application/json:
              schema:
                type: object
                properties:
                  error:
                    type: string
  /augment:
    post:
      summary: Augment seed data to generate synthetic data
      security:
        - ApiKeyAuth: []
      requestBody:
        required: true
        content:
          application/json:
            schema:
              type: object
              properties:
                multiple:
                  type: integer
                  default: 1
                  maximum: 50
                  description: Number of synthetic examples to generate per seed example (max 50)
                eval_results_id:
                  type: string
                  description: ID of evaluation results to use for target metrics (will use latest if not provided)
                include_gathered:
                  type: boolean
                  default: false
                  description: If true, includes gathered pairs (high-scoring and positively critiqued) in the synthetic dataset
                pair_query:
                  type: object
                  description: Optional query parameters to filter seed dataset pairs
                  properties:
                    latest_n_pairs:
                      type: integer
                      description: Maximum number of latest pairs to include (defaults to MaxCorpusSize if not provided)
                    pair_label:
                      type: string
                      description: Filter pairs by label (optional)
                target_metric_idx:
                  type: array
                  items:
                    type: integer
                  description: Array of indices of target metrics for redrafting synthetic data (from the evaluation results)
              required:
                - target_metric_idx
      responses:
        "200":
          description: Data successfully augmented
          content:
            application/json:
              schema:
                type: object
                properties:
                  synthetic_data_id:
                    type: string
                    description: ID of the generated synthetic dataset
                  created_at:
                    type: string
                    description: Timestamp when the synthetic data was created
                  seed_data_size:
                    type: integer
                    description: Number of pairs in the original seed dataset
                  synthetic_data_size:
                    type: integer
                    description: Total number of pairs in the synthetic dataset (including gathered pairs if include_gathered is true)
                  system_prompt:
                    type: string
                    description: System prompt used for the synthetic data
                  target_metrics:
                    type: array
                    items:
                      type: string
                    description: Array of target metrics that were used for redrafting
        "400":
          description: Bad request - malformed JSON or missing required fields
          content:
            application/json:
              schema:
                type: object
                properties:
                  error:
                    type: string
        "403":
          description: Forbidden - token count exceeds plan limit
          content:
            application/json:
              schema:
                type: object
                properties:
                  error:
                    type: string
        "500":
          description: Internal server error - various failures during augmentation process
          content:
            application/json:
              schema:
                type: object
                properties:
                  error:
                    type: string
  /gather/{id}:
    get:
      summary: Gather high-scoring and positively critiqued pairs from evaluation results
      security:
        - ApiKeyAuth: []
      parameters:
        - name: id
          in: path
          required: true
          schema:
            type: string
          description: ID of the evaluation results to gather pairs from
      responses:
        "200":
          description: Successfully gathered pairs from evaluation results
          content:
            application/json:
              schema:
                type: object
                properties:
                  eval_results_id:
                    type: string
                  dataset_id:
                    type: string
                  high_scoring_pairs:
                    type: object
                    additionalProperties:
                      type: object
                      properties:
                        id:
                          type: string
                        input:
                          type: string
                        output:
                          type: string
                        metadata:
                          type: object
                          properties:
                            created_at:
                              type: string
                            labels:
                              type: array
                              items:
                                type: string
                  positively_critiqued_pairs:
                    type: object
                    additionalProperties:
                      type: object
                      properties:
                        id:
                          type: string
                        input:
                          type: string
                        output:
                          type: string
                        metadata:
                          type: object
                          properties:
                            created_at:
                              type: string
                            labels:
                              type: array
                              items:
                                type: string
                  high_scoring_pair_count:
                    type: integer
                  positive_critique_count:
                    type: integer
                  system_prompt:
                    type: string
        "400":
          description: Bad request - evaluation results id is required
          content:
            application/json:
              schema:
                type: object
                properties:
                  error:
                    type: string
        "500":
          description: Internal server error
          content:
            application/json:
              schema:
                type: object
                properties:
                  error:
                    type: string
  /gather_openai/{id}:
    get:
      summary: Gather high-scoring and positively critiqued pairs formatted for OpenAI fine-tuning
      security:
        - ApiKeyAuth: []
      parameters:
        - name: id
          in: path
          required: true
          schema:
            type: string
          description: ID of the evaluation results to gather pairs from
      responses:
        "200":
          description: Successfully gathered pairs in OpenAI JSONL format
          content:
            text/plain:
              schema:
                type: string
                description: JSONL format for OpenAI fine-tuning, combining high-scoring and positively critiqued pairs (deduplicated)
        "400":
          description: Bad request - evaluation results id is required
          content:
            application/json:
              schema:
                type: object
                properties:
                  error:
                    type: string
        "500":
          description: Internal server error
          content:
            application/json:
              schema:
                type: object
                properties:
                  error:
                    type: string
  /compare_prompts:
    post:
      summary: Compare multiple prompts against a dataset
      security:
        - ApiKeyAuth: []
      requestBody:
        required: true
        content:
          application/json:
            schema:
              type: object
              properties:
                system_prompts:
                  type: array
                  items:
                    type: string
                seed_data_id:
                  type: string
                  description: ID of seed dataset to use (either this or synthetic_data_id is required)
                synthetic_data_id:
                  type: string
                  description: ID of synthetic dataset to use (either this or seed_data_id is required)
                metrics_id:
                  type: string
                model_name:
                  type: string
                model_provider:
                  type: string
              required:
                - system_prompts
                - model_name
                - model_provider
      responses:
        "200":
          description: Prompts successfully compared
          content:
            application/json:
              schema:
                type: object
                properties:
                  result:
                    type: object
                    properties:
                      prompts:
                        type: array
                        items:
                          type: string
                      comparisons:
                        type: array
                        items:
                          type: object
                      metrics_id:
                        type: string
  /improve_prompt:
    post:
      summary: Generate an improved prompt based on comparison results
      security:
        - ApiKeyAuth: []
      requestBody:
        required: true
        content:
          application/json:
            schema:
              type: object
              properties:
                prompt_comparison_results:
                  type: object
                  description: Results from the /compare_prompts endpoint
                seed_data_id:
                  type: string
                  description: Optional ID of seed dataset to use for prompt improvement
              required:
                - prompt_comparison_results
      responses:
        "200":
          description: Improved prompt successfully generated
          content:
            application/json:
              schema:
                type: object
                properties:
                  improved_prompt:
                    type: string
                  original_prompt:
                    type: string
                  metrics_id:
                    type: string

components:
  securitySchemes:
    ApiKeyAuth:
      type: apiKey
      in: header
      name: Authorization