openapi: 3.0.0
info:
  title: Plum API
  version: 1.1.0
servers:
  - url: https://beta.getplum.ai/v1
paths:
  /data/seeds:
    get:
      summary: List all seed datasets
      security:
        - ApiKeyAuth: []
      responses:
        "200":
          description: Successfully retrieved list of seed datasets
          content:
            application/json:
              schema:
                type: object
                properties:
                  datasets:
                    type: array
                    items:
                      type: object
                      properties:
                        id:
                          type: string
                        created_at:
                          type: string
  /data/seed/{id}:
    get:
      summary: Retrieve a specific seed dataset by ID
      security:
        - ApiKeyAuth: []
      parameters:
        - name: id
          in: path
          required: true
          schema:
            type: string
          description: ID of the seed dataset or "latest" to get the most recent dataset
      responses:
        "200":
          description: Successfully retrieved dataset
          content:
            application/json:
              schema:
                type: object
                properties:
                  id:
                    type: string
                  data:
                    type: array
                    items:
                      type: object
                      properties:
                        input:
                          type: string
                        output:
                          type: string
                  system_prompt:
                    type: string
                  created_at:
                    type: string
    post:
      summary: Insert or overwrite seed data by ID
      security:
        - ApiKeyAuth: []
      parameters:
        - name: id
          in: path
          required: true
          schema:
            type: string
      requestBody:
        required: true
        content:
          application/json:
            schema:
              type: object
              properties:
                data:
                  type: array
                  items:
                    type: object
                    properties:
                      input:
                        type: string
                      output:
                        type: string
                      id:
                        type: string
                system_prompt:
                  type: string
              required:
                - data
                - system_prompt
      responses:
        "200":
          description: Data successfully inserted
          content:
            application/json:
              schema:
                type: object
                properties:
                  id:
                    type: string
  /data/seed/{id}/pair:
    post:
      summary: Add a single pair to an existing seed dataset
      security:
        - ApiKeyAuth: []
      parameters:
        - name: id
          in: path
          required: true
          schema:
            type: string
          description: ID of the existing seed dataset
      requestBody:
        required: true
        content:
          application/json:
            schema:
              type: object
              properties:
                id:
                  type: string
                  description: Optional ID for the pair, will be generated if not provided
                input:
                  type: string
                output:
                  type: string
                labels:
                  type: array
                  items:
                    type: string
              required:
                - input
                - output
      responses:
        "200":
          description: Pair successfully added to dataset
          content:
            application/json:
              schema:
                type: object
                properties:
                  dataset_id:
                    type: string
                  pair_id:
                    type: string
  /data/seed:
    post:
      summary: Upload a new seed dataset
      security:
        - ApiKeyAuth: []
      requestBody:
        required: true
        content:
          application/json:
            schema:
              type: object
              properties:
                data:
                  type: array
                  items:
                    type: object
                    properties:
                      input:
                        type: string
                      output:
                        type: string
                      id:
                        type: string
                system_prompt:
                  type: string
              required:
                - data
                - system_prompt
      responses:
        "200":
          description: Dataset successfully uploaded
          content:
            application/json:
              schema:
                type: object
                properties:
                  id:
                    type: string
  /data/synthetics:
    get:
      summary: List all synthetic datasets
      security:
        - ApiKeyAuth: []
      responses:
        "200":
          description: Successfully retrieved list of synthetic datasets
          content:
            application/json:
              schema:
                type: object
                properties:
                  datasets:
                    type: array
                    items:
                      type: object
                      properties:
                        id:
                          type: string
                        created_at:
                          type: string
  /data/synthetic/{id}:
    get:
      summary: Retrieve synthetic data by ID
      security:
        - ApiKeyAuth: []
      parameters:
        - name: id
          in: path
          required: true
          schema:
            type: string
          description: ID of the synthetic dataset or "latest" to get the most recent dataset
      responses:
        "200":
          description: Successfully retrieved synthetic data
          content:
            application/json:
              schema:
                type: object
                properties:
                  id:
                    type: string
                  data:
                    type: array
                    items:
                      type: object
                      properties:
                        input:
                          type: string
                        output:
                          type: string
                  system_prompt:
                    type: string
                  created_at:
                    type: string
    put:
      summary: Insert or overwrite synthetic data by ID
      security:
        - ApiKeyAuth: []
      parameters:
        - name: id
          in: path
          required: true
          schema:
            type: string
      requestBody:
        required: true
        content:
          application/json:
            schema:
              type: object
              properties:
                data:
                  type: array
                  items:
                    type: object
                    properties:
                      input:
                        type: string
                      output:
                        type: string
                system_prompt:
                  type: string
              required:
                - data
      responses:
        "200":
          description: Data successfully inserted
          content:
            application/json:
              schema:
                type: object
                properties:
                  id:
                    type: string
  /data/synthetic:
    post:
      summary: Upload new synthetic data
      security:
        - ApiKeyAuth: []
      requestBody:
        required: true
        content:
          application/json:
            schema:
              type: object
              properties:
                data:
                  type: array
                  items:
                    type: object
                    properties:
                      input:
                        type: string
                      output:
                        type: string
                system_prompt:
                  type: string
              required:
                - data
      responses:
        "200":
          description: Synthetic data successfully uploaded
          content:
            application/json:
              schema:
                type: object
                properties:
                  id:
                    type: string
  /data/synthetic_openai/{id}:
    get:
      summary: Retrieve synthetic data formatted for OpenAI by ID
      security:
        - ApiKeyAuth: []
      parameters:
        - name: id
          in: path
          required: true
          schema:
            type: string
      responses:
        "200":
          description: Successfully retrieved OpenAI formatted synthetic data
          content:
            text/plain:
              schema:
                type: string
                description: JSONL format for OpenAI fine-tuning
  /question/{id}:
    get:
      summary: Get a specific metric definition by ID
      security:
        - ApiKeyAuth: []
      parameters:
        - name: id
          in: path
          required: true
          schema:
            type: string
      responses:
        "200":
          description: Successfully retrieved metric definition
          content:
            application/json:
              schema:
                type: object
                properties:
                  metrics_id:
                    type: string
                  definitions:
                    type: array
                    items:
                      type: object
                      properties:
                        name:
                          type: string
                        description:
                          type: string
                  system_prompt:
                    type: string
                  metric_count:
                    type: integer
                  created_at:
                    type: string
  /evaluation/{id}:
    get:
      summary: Get a specific evaluation result by ID
      security:
        - ApiKeyAuth: []
      parameters:
        - name: id
          in: path
          required: true
          schema:
            type: string
      responses:
        "200":
          description: Successfully retrieved evaluation result
          content:
            application/json:
              schema:
                type: object
                properties:
                  results_id:
                    type: string
                  created_at:
                    type: string
                  dataset_id:
                    type: string
                  metrics_id:
                    type: string
                  metrics_definitions:
                    type: array
                    items:
                      type: string
                  pair_count:
                    type: integer
                  system_prompt:
                    type: string
                  score_means:
                    type: array
                    items:
                      type: number
                  score_medians:
                    type: array
                    items:
                      type: number
                  score_mins:
                    type: array
                    items:
                      type: number
                  score_maxes:
                    type: array
                    items:
                      type: number
                  score_std_devs:
                    type: array
                    items:
                      type: number
                  score_confidence_intervals:
                    type: array
                    items:
                      type: object
                      properties:
                        ci_low:
                          type: number
                        ci_high:
                          type: number
                        ci_confidence:
                          type: number
                  min_scoring_pairs:
                    type: array
                    items:
                      type: array
                      items:
                        type: object
                        properties:
                          reason:
                            type: string
                          pair_id:
                            type: string
  /questions:
    post:
      summary: Generate evaluation metrics
      security:
        - ApiKeyAuth: []
      requestBody:
        required: true
        content:
          application/json:
            schema:
              type: object
              properties:
                system_prompt:
                  type: string
              required:
                - system_prompt
      responses:
        "200":
          description: Metrics successfully generated
          content:
            application/json:
              schema:
                type: object
                properties:
                  metrics_id:
                    type: string
                  definitions:
                    type: array
                    items:
                      type: object
                  created_at:
                    type: string
  /list_questions:
    get:
      summary: List all available evaluation metrics
      security:
        - ApiKeyAuth: []
      responses:
        "200":
          description: Successfully retrieved list of evaluation metrics
          content:
            application/json:
              schema:
                type: object
                properties:
                  metrics:
                    type: object
                    additionalProperties:
                      type: object
                      properties:
                        metrics_id:
                          type: string
                        definitions:
                          type: array
                          items:
                            type: string
                        system_prompt:
                          type: string
                        metric_count:
                          type: integer
                        created_at:
                          type: string
                  total_count:
                    type: integer
  /list_evaluations:
    get:
      summary: List all available evaluation results
      security:
        - ApiKeyAuth: []
      responses:
        "200":
          description: Successfully retrieved list of evaluation results
          content:
            application/json:
              schema:
                type: object
                properties:
                  results:
                    type: object
                    additionalProperties:
                      type: object
                      properties:
                        results_id:
                          type: string
                        created_at:
                          type: string
                        dataset_id:
                          type: string
                        metrics_id:
                          type: string
                        metrics_definitions:
                          type: array
                          items:
                            type: string
                        pair_count:
                          type: integer
                        system_prompt:
                          type: string
                        score_means:
                          type: array
                          items:
                            type: number
                        score_medians:
                          type: array
                          items:
                            type: number
                        score_mins:
                          type: array
                          items:
                            type: number
                        score_maxes:
                          type: array
                          items:
                            type: number
                        score_std_devs:
                          type: array
                          items:
                            type: number
                        score_confidence_intervals:
                          type: array
                          items:
                            type: object
                            properties:
                              ci_low:
                                type: number
                              ci_high:
                                type: number
                              ci_confidence:
                                type: number
                        min_scoring_pairs:
                          type: array
                          items:
                            type: array
                            items:
                              type: object
                              properties:
                                reason:
                                  type: string
                                pair_id:
                                  type: string
                  total_count:
                    type: integer
  /specify_questions:
    post:
      summary: Create custom metric definitions
      security:
        - ApiKeyAuth: []
      requestBody:
        required: true
        content:
          application/json:
            schema:
              type: object
              properties:
                metrics:
                  type: array
                  items:
                    type: string
                    description: String description of each metric
              required:
                - metrics
      responses:
        "200":
          description: Custom metrics successfully created
          content:
            application/json:
              schema:
                type: object
                properties:
                  metrics_id:
                    type: string
                  created_at:
                    type: string
  /evaluate:
    post:
      summary: Evaluate datasets with specified metrics
      security:
        - ApiKeyAuth: []
      requestBody:
        required: true
        content:
          application/json:
            schema:
              type: object
              properties:
                synthetic_data_id:
                  type: string
                  description: ID of synthetic dataset to evaluate (either this or seed_data_id is required)
                seed_data_id:
                  type: string
                  description: ID of seed dataset to evaluate (either this or synthetic_data_id is required)
                metrics_id:
                  type: string
                  description: ID of metrics to use for evaluation (will use latest if not provided)
                pair_query:
                  type: object
                  description: Optional query parameters to filter dataset pairs
                  properties:
                    latest_n_pairs:
                      type: integer
                      description: Maximum number of latest pairs to include (defaults to 150 if not provided)
                    pair_label:
                      type: string
                      description: Filter pairs by label (optional)
      responses:
        "200":
          description: Evaluation successfully completed
          content:
            application/json:
              schema:
                type: object
                properties:
                  eval_results_id:
                    type: string
                  dataset_id:
                    type: string
                  created_at:
                    type: string
                  scores:
                    type: array
                    items:
                      type: object
                      properties:
                        metric:
                          type: string
                        mean_score:
                          type: number
                        std_dev:
                          type: number
                        ci_low:
                          type: number
                        ci_high:
                          type: number
                        ci_confidence:
                          type: number
                        median_score:
                          type: number
                        min_score:
                          type: number
                        max_score:
                          type: number
                        lowest_scoring_pairs:
                          type: array
                          items:
                            type: object
                            properties:
                              pair_id:
                                type: string
                              score_reason:
                                type: string
                  pair_count:
                    type: integer
  /augment:
    post:
      summary: Augment seed data to generate synthetic data
      security:
        - ApiKeyAuth: []
      requestBody:
        required: true
        content:
          application/json:
            schema:
              type: object
              properties:
                seed_data_id:
                  type: string
                  description: ID of seed dataset to augment (will use latest if not provided)
                multiple:
                  type: integer
                  default: 1
                  maximum: 50
                  description: Number of synthetic examples to generate per seed example (max 50)
                eval_results_id:
                  type: string
                  description: ID of evaluation results to use for target metric (will use latest if not provided)
                pair_query:
                  type: object
                  description: Optional query parameters to filter seed dataset pairs
                  properties:
                    latest_n_pairs:
                      type: integer
                      description: Maximum number of latest pairs to include (defaults to 150 if not provided)
                    pair_label:
                      type: string
                      description: Filter pairs by label (optional)
                target_metric:
                  type: string
                  description: Target metric for redrafting synthetic data (will use lowest scoring metric if not provided)
      responses:
        "200":
          description: Data successfully augmented
          content:
            application/json:
              schema:
                type: object
                properties:
                  synthetic_data_id:
                    type: string
                  created_at:
                    type: string
                  seed_data_size:
                    type: integer
                  synthetic_data_size:
                    type: integer
                  system_prompt:
                    type: string
                  target_metric:
                    type: object
  /compare_prompts:
    post:
      summary: Compare multiple prompts against a dataset
      security:
        - ApiKeyAuth: []
      requestBody:
        required: true
        content:
          application/json:
            schema:
              type: object
              properties:
                system_prompts:
                  type: array
                  items:
                    type: string
                seed_data_id:
                  type: string
                  description: ID of seed dataset to use (either this or synthetic_data_id is required)
                synthetic_data_id:
                  type: string
                  description: ID of synthetic dataset to use (either this or seed_data_id is required)
                metrics_id:
                  type: string
                model_name:
                  type: string
                model_provider:
                  type: string
              required:
                - system_prompts
                - model_name
                - model_provider
      responses:
        "200":
          description: Prompts successfully compared
          content:
            application/json:
              schema:
                type: object
                properties:
                  result:
                    type: object
                    properties:
                      prompts:
                        type: array
                        items:
                          type: string
                      comparisons:
                        type: array
                        items:
                          type: object
                      metrics_id:
                        type: string
  /improve_prompt:
    post:
      summary: Generate an improved prompt based on comparison results
      security:
        - ApiKeyAuth: []
      requestBody:
        required: true
        content:
          application/json:
            schema:
              type: object
              properties:
                prompt_comparison_results:
                  type: object
                  description: Results from the /compare_prompts endpoint
                seed_data_id:
                  type: string
                  description: Optional ID of seed dataset to use for prompt improvement
              required:
                - prompt_comparison_results
      responses:
        "200":
          description: Improved prompt successfully generated
          content:
            application/json:
              schema:
                type: object
                properties:
                  improved_prompt:
                    type: string
                  original_prompt:
                    type: string
                  metrics_id:
                    type: string

components:
  securitySchemes:
    ApiKeyAuth:
      type: apiKey
      in: header
      name: Authorization