openapi: 3.0.1
info:
  title: Openlayer API
  description: >-
    The Openlayer REST API for AI evaluation, testing, and observability. Manage
    projects, commit model versions and datasets, run and evaluate tests, create
    inference pipelines for production monitoring, and stream production inference
    data into Openlayer. All requests are authenticated with a Bearer API key.
  termsOfService: https://www.openlayer.com/terms
  contact:
    name: Openlayer Support
    url: https://www.openlayer.com/docs
    email: support@openlayer.com
  version: '1.0'
servers:
  - url: https://api.openlayer.com/v1
    description: Openlayer production API
security:
  - api_key: []
tags:
  - name: Projects
    description: Create, list, and delete projects.
  - name: Commits
    description: Project versions (commits) and their test results.
  - name: Inference Pipelines
    description: Production monitoring pipelines and their rows, sessions, and users.
  - name: Data Stream
    description: Publish production inference data to a pipeline.
  - name: Tests
    description: Define, evaluate, and read project tests.
  - name: Storage
    description: Presigned URLs for uploading datasets and artifacts.
paths:
  /projects:
    get:
      operationId: listProjects
      tags:
        - Projects
      summary: List projects
      description: List the projects in the authenticated user's workspace.
      parameters:
        - name: name
          in: query
          description: Filter projects by name.
          required: false
          schema:
            type: string
        - name: taskType
          in: query
          description: Filter by task type (e.g. llm-base, tabular-classification).
          required: false
          schema:
            type: string
        - name: page
          in: query
          required: false
          schema:
            type: integer
            default: 1
        - name: perPage
          in: query
          required: false
          schema:
            type: integer
            default: 100
      responses:
        '200':
          description: A paginated list of projects.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ProjectList'
        '401':
          $ref: '#/components/responses/Unauthorized'
    post:
      operationId: createProject
      tags:
        - Projects
      summary: Create a project
      description: Create a new project in the authenticated user's workspace.
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/CreateProjectRequest'
      responses:
        '201':
          description: The created project.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Project'
        '401':
          $ref: '#/components/responses/Unauthorized'
  /projects/{projectId}:
    delete:
      operationId: deleteProject
      tags:
        - Projects
      summary: Delete a project
      description: Delete a project by its id.
      parameters:
        - $ref: '#/components/parameters/ProjectId'
      responses:
        '204':
          description: Project deleted.
        '401':
          $ref: '#/components/responses/Unauthorized'
        '404':
          $ref: '#/components/responses/NotFound'
  /projects/{projectId}/versions:
    get:
      operationId: listCommits
      tags:
        - Commits
      summary: List project commits
      description: List the commits (versions) for a project.
      parameters:
        - $ref: '#/components/parameters/ProjectId'
        - name: page
          in: query
          required: false
          schema:
            type: integer
            default: 1
        - name: perPage
          in: query
          required: false
          schema:
            type: integer
            default: 100
      responses:
        '200':
          description: A paginated list of commits.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/CommitList'
        '401':
          $ref: '#/components/responses/Unauthorized'
    post:
      operationId: createCommit
      tags:
        - Commits
      summary: Create a project commit
      description: Create a new commit (version) within a project.
      parameters:
        - $ref: '#/components/parameters/ProjectId'
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/CreateCommitRequest'
      responses:
        '201':
          description: The created commit.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Commit'
        '401':
          $ref: '#/components/responses/Unauthorized'
  /projects/{projectId}/inference-pipelines:
    get:
      operationId: listInferencePipelines
      tags:
        - Inference Pipelines
      summary: List inference pipelines
      description: List the inference pipelines in a project.
      parameters:
        - $ref: '#/components/parameters/ProjectId'
        - name: name
          in: query
          required: false
          schema:
            type: string
        - name: page
          in: query
          required: false
          schema:
            type: integer
            default: 1
        - name: perPage
          in: query
          required: false
          schema:
            type: integer
            default: 100
      responses:
        '200':
          description: A paginated list of inference pipelines.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/InferencePipelineList'
        '401':
          $ref: '#/components/responses/Unauthorized'
    post:
      operationId: createInferencePipeline
      tags:
        - Inference Pipelines
      summary: Create an inference pipeline
      description: Create a new inference pipeline within a project.
      parameters:
        - $ref: '#/components/parameters/ProjectId'
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/CreateInferencePipelineRequest'
      responses:
        '201':
          description: The created inference pipeline.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/InferencePipeline'
        '401':
          $ref: '#/components/responses/Unauthorized'
  /projects/{projectId}/tests:
    get:
      operationId: listTests
      tags:
        - Tests
      summary: List project tests
      description: List the tests defined within a project.
      parameters:
        - $ref: '#/components/parameters/ProjectId'
        - name: includeArchived
          in: query
          required: false
          schema:
            type: boolean
        - name: type
          in: query
          description: Filter by test type (integrity, consistency, performance).
          required: false
          schema:
            type: string
      responses:
        '200':
          description: A list of tests.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/TestList'
        '401':
          $ref: '#/components/responses/Unauthorized'
    post:
      operationId: createTest
      tags:
        - Tests
      summary: Create a project test
      description: Create a new test within a project.
      parameters:
        - $ref: '#/components/parameters/ProjectId'
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/CreateTestRequest'
      responses:
        '201':
          description: The created test.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Test'
        '401':
          $ref: '#/components/responses/Unauthorized'
    put:
      operationId: updateTests
      tags:
        - Tests
      summary: Update project tests
      description: Update one or more tests within a project.
      parameters:
        - $ref: '#/components/parameters/ProjectId'
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/UpdateTestsRequest'
      responses:
        '200':
          description: The updated tests.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/TestList'
        '401':
          $ref: '#/components/responses/Unauthorized'
  /versions/{projectVersionId}:
    get:
      operationId: retrieveCommit
      tags:
        - Commits
      summary: Retrieve a commit
      description: Retrieve a single commit (project version) by its id.
      parameters:
        - $ref: '#/components/parameters/ProjectVersionId'
      responses:
        '200':
          description: The requested commit.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Commit'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '404':
          $ref: '#/components/responses/NotFound'
  /versions/{projectVersionId}/results:
    get:
      operationId: listCommitTestResults
      tags:
        - Commits
      summary: List commit test results
      description: List the test results for a commit (project version).
      parameters:
        - $ref: '#/components/parameters/ProjectVersionId'
        - name: status
          in: query
          description: Filter by result status (running, passing, failing, error).
          required: false
          schema:
            type: string
        - name: type
          in: query
          required: false
          schema:
            type: string
      responses:
        '200':
          description: A list of test results.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/TestResultList'
        '401':
          $ref: '#/components/responses/Unauthorized'
  /inference-pipelines/{inferencePipelineId}:
    get:
      operationId: retrieveInferencePipeline
      tags:
        - Inference Pipelines
      summary: Retrieve an inference pipeline
      parameters:
        - $ref: '#/components/parameters/InferencePipelineId'
      responses:
        '200':
          description: The requested inference pipeline.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/InferencePipeline'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '404':
          $ref: '#/components/responses/NotFound'
    put:
      operationId: updateInferencePipeline
      tags:
        - Inference Pipelines
      summary: Update an inference pipeline
      parameters:
        - $ref: '#/components/parameters/InferencePipelineId'
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/UpdateInferencePipelineRequest'
      responses:
        '200':
          description: The updated inference pipeline.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/InferencePipeline'
        '401':
          $ref: '#/components/responses/Unauthorized'
    delete:
      operationId: deleteInferencePipeline
      tags:
        - Inference Pipelines
      summary: Delete an inference pipeline
      parameters:
        - $ref: '#/components/parameters/InferencePipelineId'
      responses:
        '204':
          description: Inference pipeline deleted.
        '401':
          $ref: '#/components/responses/Unauthorized'
        '404':
          $ref: '#/components/responses/NotFound'
  /inference-pipelines/{inferencePipelineId}/data-stream:
    post:
      operationId: streamData
      tags:
        - Data Stream
      summary: Stream production data
      description: >-
        Publish production inference data (rows) to an inference pipeline. The
        config object maps the columns in each row to Openlayer concepts such as
        input variables, output, ground truth, latency, cost, and token counts.
      parameters:
        - $ref: '#/components/parameters/InferencePipelineId'
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/DataStreamRequest'
      responses:
        '200':
          description: Data accepted for ingestion.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/DataStreamResponse'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '422':
          $ref: '#/components/responses/ValidationError'
  /inference-pipelines/{inferencePipelineId}/rows:
    post:
      operationId: streamRows
      tags:
        - Inference Pipelines
      summary: Create inference pipeline rows
      parameters:
        - $ref: '#/components/parameters/InferencePipelineId'
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/DataStreamRequest'
      responses:
        '200':
          description: Rows created.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/DataStreamResponse'
        '401':
          $ref: '#/components/responses/Unauthorized'
    put:
      operationId: updateRow
      tags:
        - Inference Pipelines
      summary: Update an inference pipeline row
      description: Update a previously published row, for example to attach ground truth.
      parameters:
        - $ref: '#/components/parameters/InferencePipelineId'
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/UpdateRowRequest'
      responses:
        '200':
          description: Row updated.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Row'
        '401':
          $ref: '#/components/responses/Unauthorized'
  /inference-pipelines/{inferencePipelineId}/rows/{inferenceId}:
    get:
      operationId: retrieveRow
      tags:
        - Inference Pipelines
      summary: Retrieve an inference pipeline row
      parameters:
        - $ref: '#/components/parameters/InferencePipelineId'
        - $ref: '#/components/parameters/InferenceId'
      responses:
        '200':
          description: The requested row.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Row'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '404':
          $ref: '#/components/responses/NotFound'
    delete:
      operationId: deleteRow
      tags:
        - Inference Pipelines
      summary: Delete an inference pipeline row
      parameters:
        - $ref: '#/components/parameters/InferencePipelineId'
        - $ref: '#/components/parameters/InferenceId'
      responses:
        '204':
          description: Row deleted.
        '401':
          $ref: '#/components/responses/Unauthorized'
        '404':
          $ref: '#/components/responses/NotFound'
  /inference-pipelines/{inferencePipelineId}/results:
    get:
      operationId: listInferencePipelineTestResults
      tags:
        - Inference Pipelines
      summary: List inference pipeline test results
      parameters:
        - $ref: '#/components/parameters/InferencePipelineId'
        - name: status
          in: query
          required: false
          schema:
            type: string
        - name: type
          in: query
          required: false
          schema:
            type: string
      responses:
        '200':
          description: A list of test results.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/TestResultList'
        '401':
          $ref: '#/components/responses/Unauthorized'
  /tests/{testId}/evaluate:
    post:
      operationId: evaluateTest
      tags:
        - Tests
      summary: Evaluate a test
      description: Trigger an evaluation of a test by its id.
      parameters:
        - $ref: '#/components/parameters/TestId'
      responses:
        '200':
          description: The evaluation result.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/TestResult'
        '401':
          $ref: '#/components/responses/Unauthorized'
  /tests/{testId}/results:
    get:
      operationId: listTestResults
      tags:
        - Tests
      summary: List results for a test
      parameters:
        - $ref: '#/components/parameters/TestId'
      responses:
        '200':
          description: A list of results for the test.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/TestResultList'
        '401':
          $ref: '#/components/responses/Unauthorized'
  /storage/presigned-url:
    post:
      operationId: createPresignedUrl
      tags:
        - Storage
      summary: Create a presigned URL
      description: >-
        Create a presigned URL used to upload a dataset or artifact to Openlayer
        object storage before committing it to a project.
      parameters:
        - name: objectName
          in: query
          description: The name of the object to upload.
          required: true
          schema:
            type: string
      responses:
        '200':
          description: A presigned upload URL.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/PresignedUrl'
        '401':
          $ref: '#/components/responses/Unauthorized'
components:
  securitySchemes:
    api_key:
      type: http
      scheme: bearer
      bearerFormat: apiKey
      description: Pass your Openlayer API key as a Bearer token in the Authorization header.
  parameters:
    ProjectId:
      name: projectId
      in: path
      required: true
      description: The unique identifier of the project.
      schema:
        type: string
        format: uuid
    ProjectVersionId:
      name: projectVersionId
      in: path
      required: true
      description: The unique identifier of the project version (commit).
      schema:
        type: string
        format: uuid
    InferencePipelineId:
      name: inferencePipelineId
      in: path
      required: true
      description: The unique identifier of the inference pipeline.
      schema:
        type: string
        format: uuid
    InferenceId:
      name: inferenceId
      in: path
      required: true
      description: The unique identifier of a streamed inference row.
      schema:
        type: string
    TestId:
      name: testId
      in: path
      required: true
      description: The unique identifier of the test.
      schema:
        type: string
        format: uuid
  responses:
    Unauthorized:
      description: Authentication failed or the API key is missing or invalid.
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/Error'
    NotFound:
      description: The requested resource was not found.
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/Error'
    ValidationError:
      description: The request body failed validation.
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/Error'
  schemas:
    Error:
      type: object
      properties:
        code:
          type: string
        message:
          type: string
      required:
        - message
    Project:
      type: object
      properties:
        id:
          type: string
          format: uuid
        name:
          type: string
        description:
          type: string
          nullable: true
        taskType:
          type: string
          description: The task type, e.g. llm-base, tabular-classification, text-classification.
        creatorId:
          type: string
          format: uuid
          nullable: true
        workspaceId:
          type: string
          format: uuid
          nullable: true
        dateCreated:
          type: string
          format: date-time
        dateUpdated:
          type: string
          format: date-time
        source:
          type: string
          nullable: true
        versionCount:
          type: integer
          nullable: true
        inferencePipelineCount:
          type: integer
          nullable: true
        links:
          type: object
          properties:
            app:
              type: string
      required:
        - id
        - name
        - taskType
    ProjectList:
      type: object
      properties:
        items:
          type: array
          items:
            $ref: '#/components/schemas/Project'
    CreateProjectRequest:
      type: object
      properties:
        name:
          type: string
        taskType:
          type: string
          description: The task type, e.g. llm-base, tabular-classification, text-classification.
        description:
          type: string
      required:
        - name
        - taskType
    Commit:
      type: object
      properties:
        id:
          type: string
          format: uuid
        projectId:
          type: string
          format: uuid
        status:
          type: string
          description: Commit status, e.g. queued, running, paused, failed, completed, unknown.
        statusMessage:
          type: string
          nullable: true
        commit:
          type: object
          properties:
            id:
              type: string
            authorId:
              type: string
              nullable: true
            message:
              type: string
            dateCreated:
              type: string
              format: date-time
        dateCreated:
          type: string
          format: date-time
        passingGoalCount:
          type: integer
          nullable: true
        failingGoalCount:
          type: integer
          nullable: true
        totalGoalCount:
          type: integer
          nullable: true
        links:
          type: object
          properties:
            app:
              type: string
      required:
        - id
        - projectId
        - status
    CommitList:
      type: object
      properties:
        items:
          type: array
          items:
            $ref: '#/components/schemas/Commit'
    CreateCommitRequest:
      type: object
      properties:
        commit:
          type: object
          properties:
            message:
              type: string
          required:
            - message
        storageUri:
          type: string
          description: The storage URI (e.g. s3://...) where the committed bundle was uploaded.
        archived:
          type: boolean
          nullable: true
        deploymentStatus:
          type: string
          nullable: true
      required:
        - commit
        - storageUri
    InferencePipeline:
      type: object
      properties:
        id:
          type: string
          format: uuid
        projectId:
          type: string
          format: uuid
          nullable: true
        name:
          type: string
        description:
          type: string
          nullable: true
        dateCreated:
          type: string
          format: date-time
        dateUpdated:
          type: string
          format: date-time
        dateLastSampleReceived:
          type: string
          format: date-time
          nullable: true
        dateLastEvaluated:
          type: string
          format: date-time
          nullable: true
        passingGoalCount:
          type: integer
          nullable: true
        failingGoalCount:
          type: integer
          nullable: true
        totalGoalCount:
          type: integer
          nullable: true
        status:
          type: string
          nullable: true
        statusMessage:
          type: string
          nullable: true
        links:
          type: object
          properties:
            app:
              type: string
      required:
        - id
        - name
    InferencePipelineList:
      type: object
      properties:
        items:
          type: array
          items:
            $ref: '#/components/schemas/InferencePipeline'
    CreateInferencePipelineRequest:
      type: object
      properties:
        name:
          type: string
        description:
          type: string
      required:
        - name
    UpdateInferencePipelineRequest:
      type: object
      properties:
        name:
          type: string
        description:
          type: string
          nullable: true
        referenceDatasetUri:
          type: string
          nullable: true
    DataStreamConfig:
      type: object
      description: >-
        Maps the columns present in each streamed row to Openlayer concepts. The
        exact fields depend on the project task type (LLM vs tabular).
      properties:
        inputVariableNames:
          type: array
          items:
            type: string
          description: Names of the columns that are input variables to the model.
        outputColumnName:
          type: string
          nullable: true
          description: Name of the column containing the model output.
        groundTruthColumnName:
          type: string
          nullable: true
          description: Name of the column containing ground truth labels.
        numOfTokenColumnName:
          type: string
          nullable: true
          description: Name of the column containing token counts.
        costColumnName:
          type: string
          nullable: true
          description: Name of the column containing the request cost.
        latencyColumnName:
          type: string
          nullable: true
          description: Name of the column containing latency in milliseconds.
        timestampColumnName:
          type: string
          nullable: true
          description: Name of the column containing the UNIX timestamp.
        inferenceIdColumnName:
          type: string
          nullable: true
          description: Name of the column containing a unique inference id for the row.
        contextColumnName:
          type: string
          nullable: true
          description: Name of the column containing retrieved context (for RAG/LLM tasks).
        questionColumnName:
          type: string
          nullable: true
        prompt:
          type: array
          nullable: true
          items:
            type: object
            properties:
              role:
                type: string
              content:
                type: string
    DataStreamRequest:
      type: object
      properties:
        config:
          $ref: '#/components/schemas/DataStreamConfig'
        rows:
          type: array
          description: The rows of production data being published.
          items:
            type: object
            additionalProperties: true
      required:
        - config
        - rows
    DataStreamResponse:
      type: object
      properties:
        success:
          type: string
          description: Indicates the data was accepted (e.g. "success").
    Row:
      type: object
      properties:
        inferenceId:
          type: string
        row:
          type: object
          additionalProperties: true
    UpdateRowRequest:
      type: object
      properties:
        inferenceId:
          type: string
        row:
          type: object
          additionalProperties: true
        config:
          $ref: '#/components/schemas/DataStreamConfig'
      required:
        - inferenceId
        - row
    Test:
      type: object
      properties:
        id:
          type: string
          format: uuid
        projectVersionId:
          type: string
          format: uuid
          nullable: true
        name:
          type: string
        description:
          type: string
          nullable: true
        type:
          type: string
          description: The test type (integrity, consistency, performance).
        subtype:
          type: string
          nullable: true
        archived:
          type: boolean
        dateCreated:
          type: string
          format: date-time
        dateUpdated:
          type: string
          format: date-time
        creatorId:
          type: string
          format: uuid
          nullable: true
        usesMlModel:
          type: boolean
          nullable: true
        usesValidationDataset:
          type: boolean
          nullable: true
        usesTrainingDataset:
          type: boolean
          nullable: true
        usesProductionData:
          type: boolean
          nullable: true
        thresholds:
          type: array
          items:
            type: object
            properties:
              measurement:
                type: string
              operator:
                type: string
              value:
                oneOf:
                  - type: number
                  - type: string
                  - type: boolean
                  - type: array
                    items: {}
      required:
        - id
        - name
        - type
    TestList:
      type: object
      properties:
        items:
          type: array
          items:
            $ref: '#/components/schemas/Test'
    CreateTestRequest:
      type: object
      properties:
        name:
          type: string
        description:
          type: string
          nullable: true
        type:
          type: string
        subtype:
          type: string
        thresholds:
          type: array
          items:
            type: object
            properties:
              measurement:
                type: string
              operator:
                type: string
              value:
                oneOf:
                  - type: number
                  - type: string
                  - type: boolean
                  - type: array
                    items: {}
      required:
        - name
        - type
        - thresholds
    UpdateTestsRequest:
      type: object
      properties:
        tests:
          type: array
          items:
            $ref: '#/components/schemas/Test'
      required:
        - tests
    TestResult:
      type: object
      properties:
        id:
          type: string
          format: uuid
        projectVersionId:
          type: string
          format: uuid
          nullable: true
        inferencePipelineId:
          type: string
          format: uuid
          nullable: true
        status:
          type: string
          description: Result status, e.g. running, passing, failing, skipped, error.
        statusMessage:
          type: string
          nullable: true
        dateCreated:
          type: string
          format: date-time
        dateUpdated:
          type: string
          format: date-time
        dateDataStarts:
          type: string
          format: date-time
          nullable: true
        dateDataEnds:
          type: string
          format: date-time
          nullable: true
        goal:
          $ref: '#/components/schemas/Test'
        goalId:
          type: string
          format: uuid
          nullable: true
      required:
        - id
        - status
    TestResultList:
      type: object
      properties:
        items:
          type: array
          items:
            $ref: '#/components/schemas/TestResult'
    PresignedUrl:
      type: object
      properties:
        url:
          type: string
          description: The presigned URL to which the object should be uploaded.
        storageUri:
          type: string
          description: The storage URI (e.g. s3://...) referencing the uploaded object.
        fields:
          type: object
          additionalProperties: true
          description: Form fields to include with the upload request, when applicable.