{ "openapi": "3.0.1", "info": { "title": "PhariaData API", "description": "The PhariaData API provides a comprehensive suite of endpoints to manage data workflows within the Pharia Data Platform. This API enables users to organize, store, retrieve, and manipulate datasets across repositories with efficiency and control. Key functionalities include the creation and management of datasets, stages, and repositories, as well as streamlined access to data through downloads and real-time data streaming.", "contact": { "name": "PhariaData API Support", "url": "https://aleph-alpha.atlassian.net/servicedesk/customer/portals", "email": "pharia-data@aleph-alpha.com" }, "license": { "name": "Open Aleph License" }, "version": "1.0.0" }, "tags": [ { "name": "Repositories", "description": "A collection of data organized by a common type, modality, and schema, stored within datasets (lists of data points) in the repositories. To share datasets externally, they must first be exported as files, which will then be stored in the internal storage of the data platform." }, { "name": "Datasets", "description": "Represents the primary data abstraction within the data platform, serving as a structured collection of data points. Datasets can be either manually uploaded or generated through data transformations, providing a versatile means for organizing, storing, and retrieving data across various use cases." }, { "name": "Stages", "description": "Stages serve as the data platform's entry point for data collection, where source files are securely stored, enabling subsequent transformations and dataset generation. Files can be uploaded directly into stages via the HTTP API or through connectors, which support automated and continuous synchronization with your source systems. Stages provide secure, persistent storage and act as an interface for importing and exporting data files within the platform. To improve data discoverability, comprehension, and quality for end users, the data platform offers datasets as the recommended abstraction for organizing and sharing data." }, { "name": "Downloads", "description": "Endpoints for managing and tracking dataset download requests within the data platform. These endpoints allow users to initiate new downloads, retrieve details of specific download requests, and list download activity for datasets, with optional filters by date and repository. Download management helps streamline access to dataset files and provides caching capabilities." }, { "name": "Transformations", "description": "Available transformations can be applied to an input data object of type A to produce an output data object of type B. The output data object will consist of a sequence of items, each of which will conform to an Avro schema specified in the transformation as OutputSchema. Currently, the input data object can be a File located in a Stage within the Data Platform. Future versions will support receiving Datasets from Repositories in the Data Platform as input data objects. The output data object can be loaded into various types of data storage within the Data Platform, with Repositories being the first supported storage type. When triggering a transformation, this has to be specified as destination. After loading the output into the destination, it can also be sent to an external service through available connectors in the Data Platform. The first available connector will be the Document Index Collection Connector." }, { "name": "Connectors", "description": "Available connectors for the Data Platform to ingest data from external sources." }, { "name": "Search Store", "description": "A Search Store is a structured data repository optimized for indexing and retrieving searchable entities. It supports full-text search, metadata-based filtering, and relevance ranking. Each Search Store can manage multiple indexed documents, making it ideal for fast, scalable information retrieval across different modalities." }, { "name": "Document", "description": "A Document represents an individual item stored within a Search Store. Documents can contain content from various modalities, such as text, images, or pre-chunked data." } ], "security": [ { "Bearer": [] } ], "paths": { "/transformations": { "get": { "tags": [ "Transformations" ], "summary": "Retrieve a list of available transformations", "description": "Fetches a paginated list of all available transformations in the Data Platform.", "parameters": [ { "name": "page", "in": "query", "description": "The page number to retrieve, used for paginated responses.", "schema": { "type": "integer", "example": 0 } }, { "name": "size", "in": "query", "description": "The number of items to include per page in the response.", "schema": { "type": "integer", "example": 10 } }, { "name": "name", "in": "query", "description": "Search for transformations by their specific name.", "schema": { "type": "string", "example": "DocumentToText" } } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/TransformationListWithPagination" } } } }, "401": { "description": "UNAUTHORIZED When access token is invalid.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "500": { "description": "INTERNAL SERVER ERROR", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "400": { "description": "BAD REQUEST", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } } } } }, "/transformations/{transformationID}": { "get": { "tags": [ "Transformations" ], "summary": "Retrieve transformation details by ID", "description": "Retrieves detailed information for a specific transformation identified by its unique ID. Includes properties such as name, input media type, supported destinations, and output schema.", "parameters": [ { "name": "transformationID", "in": "path", "description": "Unique identifier for the transformation.", "required": true, "schema": { "type": "string", "example": "cb26237c-3beb-40f3-8e13-77d862f722b5" } } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Transformation" } } } }, "400": { "description": "Bad Request", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "401": { "description": "Unauthorized", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "404": { "description": "Not Found", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "500": { "description": "INTERNAL SERVER ERROR", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } } } } }, "/transformations/{transformationID}/runs": { "post": { "tags": [ "Transformations" ], "summary": "Trigger a transformation", "description": "Initiates a transformation process for a specific transformation, identified by its unique ID. The operation applies the transformation to the input data object and stores the result in the specified destination. If connector configurations are provided, the output can also be delivered to external services. Some transformations accept parameters: the list of the allowed parameters for each transformation can be retrieved by the list transformations endpoint. ```Note: If a file is TXT or Markdown format, only files with UTF-8 encoding and LF as line feed character are supported.```\n", "parameters": [ { "name": "transformationID", "in": "path", "description": "Unique identifier for the transformation.", "required": true, "schema": { "type": "string", "example": "cb26237c-3beb-40f3-8e13-77d862f722b5" } } ], "requestBody": { "description": "Transformation parameters", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/TransformationRunCreate" } } }, "required": true }, "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/TransformationRun" } } } }, "400": { "description": "Bad Request", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "401": { "description": "Unauthorized", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "404": { "description": "Not Found", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "413": { "description": "Request Entity Too Large", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } } } }, "get": { "tags": [ "Transformations" ], "summary": "Retrieve a list of created runs given a transformation ID", "description": "Fetches a paginated list of all created runs in the Data Platform for a specific transformation.", "parameters": [ { "name": "transformationID", "in": "path", "description": "Unique identifier for the transformation.", "required": true, "schema": { "type": "string", "example": "cb26237c-3beb-40f3-8e13-77d862f722b5" } }, { "name": "page", "in": "query", "description": "The page number to retrieve, used for paginated responses.", "schema": { "type": "integer", "example": 0 } }, { "name": "size", "in": "query", "description": "The number of items to include per page in the response.", "schema": { "type": "integer", "example": 10 } }, { "name": "file_id", "in": "query", "description": "A file ID to filter the list of runs that received this file as input.", "schema": { "type": "string", "example": "cb26237c-3beb-40f3-8e13-77d862f722b5" } } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/RunListWithPagination" } } } }, "400": { "description": "BAD REQUEST", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "401": { "description": "UNAUTHORIZED When access token is invalid.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "500": { "description": "INTERNAL SERVER ERROR", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "404": { "description": "Not Found", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } } } } }, "/transformations/{transformationID}/runs/events": { "get": { "tags": [ "Transformations" ], "summary": "Stream real-time run updates for a transformation", "description": "Opens a Server-Sent Events (SSE) connection that streams live run updates for a specific transformation.\n\n**SSE Events:**\n- `connected`: Connection established\n- `runs:new`: New runs appeared on the current page\n- `runs:updated`: Existing runs changed status\n- `runs:removed`: Runs fell off the current page window (e.g. pushed out by newer runs)\n- `done`: All currently visible runs reached a terminal state\n- `error`: Stream error occurred\n\nThe stream closes automatically when all runs reach a terminal state or when the client disconnects.", "parameters": [ { "name": "transformationID", "in": "path", "description": "Unique identifier for the transformation.", "required": true, "schema": { "type": "string", "example": "cb26237c-3beb-40f3-8e13-77d862f722b5" } }, { "name": "page", "in": "query", "description": "The page number to retrieve, used for paginated responses.", "schema": { "type": "integer", "example": 0 } }, { "name": "size", "in": "query", "description": "The number of items to include per page in the response.", "schema": { "type": "integer", "example": 10 } }, { "name": "file_id", "in": "query", "description": "A file ID to filter the list of runs that received this file as input.", "schema": { "type": "string", "example": "cb26237c-3beb-40f3-8e13-77d862f722b5" } } ], "responses": { "200": { "description": "OK", "content": { "text/event-stream": { "schema": { "type": "string", "description": "Server-Sent Events stream with real-time run updates", "example": "event: connected\ndata: {\"message\":\"Connected to runs stream\",\"transformationId\":\"...\"}\n\nevent: runs:new\ndata: {\"runs\":[...],\"count\":1,\"total\":5}\n\nevent: runs:updated\ndata: {\"runs\":[...],\"count\":1,\"total\":5}\n\nevent: runs:removed\ndata: {\"runIds\":[\"...\"],\"count\":1,\"total\":4}\n\n: keepalive at 2026-01-01T00:00:00Z\n\nevent: done\ndata: {\"message\":\"All runs have reached a terminal state\"}\n" } } } }, "400": { "description": "BAD REQUEST", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "401": { "description": "UNAUTHORIZED When access token is invalid.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "404": { "description": "Not Found", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "500": { "description": "INTERNAL SERVER ERROR", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } } } } }, "/transformations/{transformationID}/runs/{runID}": { "get": { "tags": [ "Transformations" ], "summary": "Retrieve transformation run details by ID", "description": "Fetches detailed information for a specific transformation run identified by its unique ID. Includes properties such as status, start time, end time, and output destination.", "parameters": [ { "name": "transformationID", "in": "path", "description": "Unique identifier for the transformation.", "required": true, "schema": { "type": "string", "example": "cb26237c-3beb-40f3-8e13-77d862f722b5" } }, { "name": "runID", "in": "path", "description": "Unique identifier for the transformation run.", "required": true, "schema": { "type": "string", "example": "cb26237c-3beb-40f3-8e13-77d862f722b5" } } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/TransformationRun" } } } }, "400": { "description": "Bad Request", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "401": { "description": "Unauthorized", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "404": { "description": "Not Found", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "500": { "description": "INTERNAL SERVER ERROR", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } } } } }, "/transformations/{transformationID}/runs/batch": { "post": { "tags": [ "Transformations" ], "summary": "Trigger multiple transformations in batch", "description": "Initiates multiple transformation processes for a specific transformation, identified by its unique ID. This endpoint allows you to execute multiple transformation runs simultaneously by providing an array of transformation configurations. Each transformation in the batch applies to its respective input data object and stores the result in the specified destination. If connector configurations are provided, the outputs can also be delivered to external services. Some transformations accept parameters: the list of the allowed parameters for each transformation can be retrieved by the list transformations endpoint. ```Note: If a file is TXT or Markdown format, only files with UTF-8 encoding and LF as line feed character are supported.```\n", "parameters": [ { "name": "transformationID", "in": "path", "description": "Unique identifier for the transformation.", "required": true, "schema": { "type": "string", "example": "cb26237c-3beb-40f3-8e13-77d862f722b5" } } ], "requestBody": { "description": "Array of transformation parameters for batch processing", "content": { "application/json": { "schema": { "type": "array", "items": { "$ref": "#/components/schemas/TransformationRunCreate" } } } }, "required": true }, "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "type": "array", "items": { "$ref": "#/components/schemas/TransformationRun" } } } } }, "400": { "description": "Bad Request", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "401": { "description": "Unauthorized", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "404": { "description": "Not Found", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "413": { "description": "Request Entity Too Large", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } } }, "security": [ { "Bearer": [] } ] } }, "/repositories": { "get": { "tags": [ "Repositories" ], "summary": "Retrieve a list of repositories", "description": "Fetches a paginated list of all available repositories with optional filters for media type, modality, and name.", "parameters": [ { "name": "page", "in": "query", "description": "The page number to retrieve, used for paginated responses.", "schema": { "type": "integer", "example": 0 } }, { "name": "size", "in": "query", "description": "The number of items to include per page in the response.", "schema": { "type": "integer", "example": 10 } }, { "name": "media_type", "in": "query", "description": "Filter repositories by their specific media type, such as application/x-ndjson, application/csv", "schema": { "type": "string", "example": "application/x-ndjson" } }, { "name": "modality", "in": "query", "description": "Filter repositories by modality, indicating the format or approach, like text.", "schema": { "type": "string", "example": "text" } }, { "name": "name", "in": "query", "description": "Search for repositories by their specific name.", "schema": { "type": "string", "example": "repository" } } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/RepositoryListWithPagination" } } } }, "401": { "description": "UNAUTHORIZED When access token is invalid.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "500": { "description": "INTERNAL SERVER ERROR", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "400": { "description": "BAD REQUEST", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } } } }, "post": { "tags": [ "Repositories" ], "summary": "Create a new repository", "description": "Creates a new repository to store datasets with specified properties such as name, media type, and modality.", "requestBody": { "description": "The details required to create a new repository, including name, media type, modality, and optional schema.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/RepositoryCreate" } } }, "required": true }, "responses": { "201": { "description": "Created", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Repository" } } } }, "400": { "description": "Bad Request", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "401": { "description": "Unauthorized", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "500": { "description": "INTERNAL SERVER ERROR", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } } } } }, "/repositories/{repositoryID}": { "get": { "tags": [ "Repositories" ], "summary": "Retrieve repository details by ID", "description": "Retrieves detailed information for a specific repository identified by its unique ID. Includes properties such as name, media type, modality, schema, creation date, and last updated date.", "parameters": [ { "name": "repositoryID", "in": "path", "description": "Unique identifier for the repository.", "required": true, "schema": { "type": "string", "example": "cb26237c-3beb-40f3-8e13-77d862f722b5" } } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Repository" } } } }, "400": { "description": "Bad Request", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "401": { "description": "Unauthorized", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "404": { "description": "Not Found", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "500": { "description": "INTERNAL SERVER ERROR", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } } } }, "delete": { "tags": [ "Repositories" ], "summary": "Delete repository by ID", "description": "Delete the repository by ID. All datasets in this repository will be immediately soft-deleted and hard-deleted asynchronously, along with all existing downloads. This operation is not reversible.", "parameters": [ { "name": "repositoryID", "in": "path", "description": "Unique identifier for the repository.", "required": true, "schema": { "type": "string", "example": "cb26237c-3beb-40f3-8e13-77d862f722b5" } } ], "responses": { "204": { "description": "OK" }, "400": { "description": "Bad Request", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "404": { "description": "Not Found", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "500": { "description": "INTERNAL SERVER ERROR", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } } } } }, "/repositories/{repositoryID}/datasets": { "get": { "tags": [ "Datasets" ], "summary": "Retrieve a list of datasets", "description": "Lists all datasets within a specified repository, with options to filter by creation date and labels. Use this endpoint to browse datasets and locate relevant data collections by repository. Returns 404 if the repository does not exist.", "parameters": [ { "name": "repositoryID", "in": "path", "description": "The unique identifier of the repository containing the datasets.", "required": true, "schema": { "type": "string", "example": "cb26237c-3beb-40f3-8e13-77d862f722b5" } }, { "name": "created_after", "in": "query", "description": "Filters datasets created after the specified date (in ISO 8601 format).", "schema": { "type": "string", "example": "2024-01-01" } }, { "name": "created_before", "in": "query", "description": "Filters datasets created before the specified date (in ISO 8601 format).", "schema": { "type": "string", "example": "2024-12-31" } }, { "name": "label", "in": "query", "description": "Filters datasets by one or more labels. Use this to narrow down results to datasets tagged with specific labels.", "style": "form", "explode": false, "schema": { "type": "array", "items": { "type": "string" }, "example": [ "pharia-finetuning" ] } }, { "name": "page", "in": "query", "description": "The page number to retrieve, used for paginated responses.", "schema": { "type": "integer", "example": 0 } }, { "name": "size", "in": "query", "description": "The number of items to include per page in the response.", "schema": { "type": "integer", "example": 10 } } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/DatasetListResponse" } } } }, "400": { "description": "Bad Request", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponseDataset" } } } }, "401": { "description": "Unauthorized", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponseDataset" } } } }, "404": { "description": "Repository not found - The specified repository does not exist", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponseDataset" } } } }, "500": { "description": "INTERNAL SERVER ERROR", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponseDataset" } } } } } }, "post": { "tags": [ "Datasets" ], "summary": "Create a dataset", "description": "Adds a new dataset to the specified repository. This endpoint accepts a file containing dataset content in JSON Lines format, along with optional license, metadata, and labels to better classify and describe the dataset.", "parameters": [ { "name": "repositoryID", "in": "path", "description": "Unique identifier of the repository in which the dataset will be created.", "required": true, "schema": { "type": "string", "example": "cb26237c-3beb-40f3-8e13-77d862f722b5" } } ], "requestBody": { "required": true, "content": { "multipart/form-data": { "schema": { "type": "object", "required": [ "sourceData", "totalDatapoints" ], "properties": { "sourceData": { "type": "string", "description": "File containing dataset content, serialized in JSON Lines format. Each line represents a distinct data point in JSON format.", "format": "binary" }, "name": { "type": "string", "description": "Name assigned to the dataset for identification.", "example": "pharia-llm-dataset-evaluation" }, "metadata": { "type": "object", "description": "Additional key-value metadata related to the dataset, used to provide descriptive information and context.", "example": { "source": "studio", "task": "evaluation-pharia-llm-1" } }, "license": { "type": "object", "description": "License information for the dataset, ensuring transparency and regulatory compliance. Recommended licenses include the Open Aleph License (OAL-1.0), Creative Commons CC0, and Open Data Commons PDDL. If included, please provide the license ID and URL.", "example": { "id": "OAL-1.0", "url": "https://github.com/Aleph-Alpha/.github/blob/main/OAL-1.0.pdf" } }, "labels": { "type": "string", "description": "A comma-separated list of labels associated with the dataset to enhance searchability and classification.", "example": "finetuning, pharia, aleph-alpha" }, "totalDatapoints": { "type": "integer", "description": "Total number of data points contained in the dataset, useful for tracking dataset size.", "example": 1000 } } } } } }, "responses": { "201": { "description": "Created", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Dataset" } } } }, "400": { "description": "Bad Request", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponseDataset" } } } }, "401": { "description": "Unauthorized", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponseDataset" } } } }, "404": { "description": "Not Found", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponseDataset" } } } }, "422": { "description": "UnprocessableEntity", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponseDataset" } } } }, "500": { "description": "INTERNAL SERVER ERROR", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponseDataset" } } } } } } }, "/repositories/{repositoryID}/datasets/{datasetID}": { "get": { "tags": [ "Datasets" ], "summary": "Retrieve a dataset by ID", "description": "Fetches details of a specific dataset within a repository by its unique identifier. This endpoint provides information about the dataset's metadata, labels, and other properties.", "parameters": [ { "name": "repositoryID", "in": "path", "description": "Unique identifier of the repository containing the dataset.", "required": true, "schema": { "type": "string", "example": "cb26237c-3beb-40f3-8e13-77d862f722b5" } }, { "name": "datasetID", "in": "path", "description": "Unique identifier of the dataset to be retrieved.", "required": true, "schema": { "type": "string", "example": "3fa85f64-5717-4562-b3fc-2c963f66afa6" } } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Dataset" } } } }, "400": { "description": "Bad Request", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponseDataset" } } } }, "401": { "description": "Unauthorized", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponseDataset" } } } }, "404": { "description": "Not Found", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponseDataset" } } } }, "500": { "description": "INTERNAL SERVER ERROR", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponseDataset" } } } } } }, "delete": { "tags": [ "Datasets" ], "summary": "Delete a dataset by ID", "description": "Permanently removes a dataset from the specified repository. This operation is irreversible and will delete all associated data points within the dataset.", "parameters": [ { "name": "repositoryID", "in": "path", "description": "Unique identifier of the repository containing the dataset to be deleted.", "required": true, "schema": { "type": "string", "example": "cb26237c-3beb-40f3-8e13-77d862f722b5" } }, { "name": "datasetID", "in": "path", "description": "Unique identifier of the dataset to delete.", "required": true, "schema": { "type": "string", "example": "3fa85f64-5717-4562-b3fc-2c963f66afa6" } } ], "responses": { "204": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponseDataset" } } } }, "400": { "description": "Bad Request", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponseDataset" } } } }, "404": { "description": "Not Found", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponseDataset" } } } }, "500": { "description": "INTERNAL SERVER ERROR", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponseDataset" } } } } } }, "patch": { "tags": [ "Datasets" ], "summary": "Update dataset metadata", "description": "Update the metadata fields of the latest version of a specified dataset within a repository. This operation allows partial updates to the dataset's metadata, including fields such as name, labels, totalDatapoints, version, and additional metadata.", "parameters": [ { "name": "repositoryID", "in": "path", "description": "Unique identifier of the repository containing the dataset to update.", "required": true, "schema": { "type": "string", "example": "cb26237c-3beb-40f3-8e13-77d862f722b5" } }, { "name": "datasetID", "in": "path", "description": "Unique identifier of the dataset to update.", "required": true, "schema": { "type": "string", "example": "3fa85f64-5717-4562-b3fc-2c963f66afa6" } } ], "requestBody": { "description": "JSON object containing the metadata fields to be updated for the dataset.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/DatasetMetadataUpdate" } } }, "required": true }, "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Dataset" } } } }, "400": { "description": "Bad Request", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponseDownload" } } } }, "401": { "description": "Unauthorized", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponseDownload" } } } }, "404": { "description": "Not Found", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponseDownload" } } } }, "422": { "description": "UnprocessableEntity", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponseDownload" } } } }, "503": { "description": "Service Unavailable", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponseDownload" } } } } } } }, "/repositories/{repositoryID}/datasets/{datasetID}/datapoints": { "get": { "tags": [ "Datasets" ], "summary": "Stream dataset data points", "description": "Streams data points of the specified dataset, providing real-time access to its content. This operation allows continuous retrieval of data points in a JSON format.", "parameters": [ { "name": "repositoryID", "in": "path", "description": "Unique identifier of the repository containing the dataset to stream.", "required": true, "schema": { "type": "string", "example": "cb26237c-3beb-40f3-8e13-77d862f722b5" } }, { "name": "datasetID", "in": "path", "description": "Unique identifier of the dataset from which data points will be streamed.", "required": true, "schema": { "type": "string", "example": "3fa85f64-5717-4562-b3fc-2c963f66afa6" } }, { "name": "version", "in": "query", "description": "Specific version of the dataset to stream. If omitted, the latest version will be streamed by default.", "schema": { "type": "string", "example": "3fa85f64-5717-40f3-8e13-77d862f722b5" } }, { "name": "start", "in": "query", "description": "Starting index for streaming data points, used to control where the stream begins.", "schema": { "type": "integer", "example": 0 } }, { "name": "size", "in": "query", "description": "Number of data points to retrieve in the stream, allowing control over the stream length.", "schema": { "type": "integer", "example": 100 } } ], "responses": { "200": { "description": "Successful data point stream", "headers": { "Transfer-Encoding": { "description": "Indicates chunked transfer encoding for streaming response.", "schema": { "type": "string", "example": "chunked" } } }, "content": { "application/json": { "schema": { "type": "string", "description": "Streamed data points in JSON Lines format. Each line represents a JSON object containing individual data points.", "example": "{\"id\": \"0\", \"key\": \"value\"}\n{\"id\": \"1\", \"key\": \"another_value\"}\n{\"id\": \"2\", \"key\": \"more_data\"}\n" } } } }, "400": { "description": "Bad Request", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponseDataset" } } } }, "404": { "description": "Dataset or repository not found", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponseDataset" } } } } } }, "put": { "tags": [ "Datasets" ], "summary": "Update dataset data points", "description": "Uploads new data points to update an existing dataset within the specified repository. This operation replaces the current data points with the new content provided in the request and generate a new version.", "parameters": [ { "name": "repositoryID", "in": "path", "description": "Unique identifier of the repository containing the dataset to be updated.", "required": true, "schema": { "type": "string", "example": "cb26237c-3beb-40f3-8e13-77d862f722b5" } }, { "name": "datasetID", "in": "path", "description": "Unique identifier of the dataset from which data points will be streamed.", "required": true, "schema": { "type": "string", "example": "3fa85f64-5717-4562-b3fc-2c963f66afa6" } } ], "requestBody": { "required": true, "content": { "multipart/form-data": { "schema": { "type": "object", "required": [ "sourceData" ], "properties": { "sourceData": { "type": "string", "description": "A file containing the new dataset content, serialized in JSON Lines format. Each line represents a data point in JSON format.", "format": "binary", "example": "my_data.jsonl" } } } } } }, "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Dataset" } } } }, "400": { "description": "Bad Request", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponseDataset" } } } }, "401": { "description": "Unauthorized", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponseDataset" } } } }, "404": { "description": "Not Found", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponseDataset" } } } }, "500": { "description": "INTERNAL SERVER ERROR", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } } } } }, "/repositories/{repositoryID}/datasets/{datasetID}/downloads": { "get": { "tags": [ "Downloads" ], "summary": "List download records for a specific dataset", "description": "Retrieves a list of downloads for the specified dataset within a repository. This endpoint supports filtering by date range.", "parameters": [ { "name": "repositoryID", "in": "path", "description": "The unique identifier of the repository containing the dataset.", "required": true, "schema": { "type": "string", "example": "cb26237c-3beb-40f3-8e13-77d862f722b5" } }, { "name": "datasetID", "in": "path", "description": "The unique identifier of the dataset for which downloads are being listed.", "required": true, "schema": { "type": "string", "example": "3fa85f64-5717-4562-b3fc-2c963f66afa6" } }, { "name": "created_after", "in": "query", "description": "Filters downloads created after the specified date (in ISO 8601 format).", "schema": { "type": "string", "example": "2024-01-01" } }, { "name": "created_before", "in": "query", "description": "Filters downloads created before the specified date (in ISO 8601 format).", "schema": { "type": "string", "example": "2024-12-31" } } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/DownloadListResponse" } } } }, "400": { "description": "Bad Request", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponseDownload" } } } }, "401": { "description": "Unauthorized", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponseDownload" } } } }, "404": { "description": "Not Found", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponseDownload" } } } }, "500": { "description": "INTERNAL SERVER ERROR", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponseDownload" } } } } } }, "post": { "tags": [ "Downloads" ], "summary": "Create a download request for a specific dataset", "description": "Creates a new download request for the specified dataset within a repository. The request can include parameters to define the scope and details of the download.", "parameters": [ { "name": "repositoryID", "in": "path", "description": "The unique identifier of the repository containing the dataset to be downloaded.", "required": true, "schema": { "type": "string", "example": "cb26237c-3beb-40f3-8e13-77d862f722b5" } }, { "name": "datasetID", "in": "path", "description": "The unique identifier of the dataset for which the download is being created.", "required": true, "schema": { "type": "string", "example": "3fa85f64-5717-4562-b3fc-2c963f66afa6" } } ], "requestBody": { "description": "Download Parameters", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/DownloadCreate" } } }, "required": true }, "responses": { "201": { "description": "Created", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Download" } } } }, "400": { "description": "Bad Request", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponseDownload" } } } }, "401": { "description": "Unauthorized", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponseDownload" } } } }, "404": { "description": "Not Found", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponseDownload" } } } }, "500": { "description": "INTERNAL SERVER ERROR", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponseDownload" } } } } } } }, "/repositories/{repositoryID}/datasets/{datasetID}/downloads/{downloadID}": { "get": { "tags": [ "Downloads" ], "summary": "Retrieve download details by ID", "description": "Fetches details of a specific download request using the unique repository, dataset, and download identifiers. This endpoint provides information about the version, file location, and metadata of the download.", "parameters": [ { "name": "repositoryID", "in": "path", "description": "The unique identifier of the repository containing the dataset for the download request.", "required": true, "schema": { "type": "string", "example": "cb26237c-3beb-40f3-8e13-77d862f722b5" } }, { "name": "datasetID", "in": "path", "description": "The unique identifier of the dataset for which the download was requested.", "required": true, "schema": { "type": "string", "example": "3fa85f64-5717-4562-b3fc-2c963f66afa6" } }, { "name": "downloadID", "in": "path", "description": "The unique identifier of the download request.", "required": true, "schema": { "type": "string", "example": "a1b2c3d4-e5f6-7890-abcd-1234567890ef" } } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Download" } } } }, "400": { "description": "Bad Request", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponseDownload" } } } }, "401": { "description": "Unauthorized", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponseDownload" } } } }, "404": { "description": "Not Found", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponseDownload" } } } }, "503": { "description": "Service Unavailable", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponseDownload" } } } } } } }, "/stages": { "get": { "tags": [ "Stages" ], "summary": "Retrieve a list of stages", "description": "Retrieves a paginated list of stages in the data platform. Optionally, filters can be applied based on stage name.", "parameters": [ { "name": "page", "in": "query", "description": "The page number to retrieve in the paginated response (starting from 0).", "schema": { "type": "integer", "example": 0 } }, { "name": "size", "in": "query", "description": "The number of stages to include in each page of results.", "schema": { "type": "integer", "example": 10 } }, { "name": "name", "in": "query", "description": "An optional filter to retrieve stages that match the specified name.", "schema": { "type": "string", "example": "My Stage" } }, { "name": "withSearchStore", "in": "query", "description": "Filter stages that have or don't have a search store configured.", "schema": { "type": "boolean", "example": true } } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/StageListWithPagination" } } } }, "400": { "description": "Bad Request", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "401": { "description": "Unauthorized", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "500": { "description": "INTERNAL SERVER ERROR", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } } } }, "post": { "tags": [ "Stages" ], "summary": "Create a new stage", "description": "Creates a new stage in the data platform with the specified name. This stage can be used to store source files for further data processing.\n\n**Access Policy:** By default, stages are created as private (owner-only access). Admin users can optionally create public stages by setting `accessPolicy: \"public\"`. Public stages grant read-only access to all authenticated users, while write access remains with the owner. Non-admin users attempting to create public stages will receive a 403 Forbidden error.", "requestBody": { "description": "The details required to create a new stage, including the name of the stage. This field is mandatory.", "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/StageCreate" }, "example": { "name": "MyPublicCollection", "triggers": [ { "transformationName": "DocumentToText", "destinationType": "DataPlatform:Repository:CREATE", "connectorType": "DataPlatform:SearchStore:CREATE", "name": "PDFTriggerMyStage" } ], "retentionPolicy": { "retentionPeriod": 30 }, "accessPolicy": "public", "searchStore": { "chunkingStrategy": { "maxChunkSizeTokens": 500, "chunkOverlapTokens": 250 }, "embeddingStrategy": { "type": "vllm", "config": { "model": "qwen3-embedding-8b" } }, "metadata": { "topic": "movies" }, "metadataSchema": { "topic": "string", "approved": "boolean" } } } } } }, "responses": { "201": { "description": "Created", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/StageCreatedResponse" }, "example": { "stageId": "fd8b3886-c8c0-4728-8b66-7922703b1ef5", "name": "MyPublicCollection", "createdAt": "2026-02-16T13:03:48Z", "updatedAt": "2026-02-16T13:03:48Z", "triggers": [ { "transformationName": "DocumentToText", "destinationType": "DataPlatform:Repository", "connectorType": "DataPlatform:SearchStore", "name": "PDFTriggerMyStage", "repositoryId": "815f77f9-6a68-4c42-86a4-9bd603395a49" } ], "retentionPolicy": { "retentionPeriod": 30 }, "searchStore": { "id": "41804fe6-34b2-4db6-a4b1-dcd50fa7cf6a", "chunkingStrategy": { "maxChunkSizeTokens": 500, "chunkOverlapTokens": 250 }, "embeddingStrategy": { "type": "vllm", "config": { "model": "qwen3-embedding-8b" } }, "metadata": { "topic": "movies" }, "metadataSchema": { "topic": "string", "approved": "boolean" } }, "accessPolicy": "public", "filesCount": 0, "ownerId": "316056799180494087" } } } }, "400": { "description": "Bad Request", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "401": { "description": "Unauthorized", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "403": { "description": "Forbidden - Non-admin user attempted to create a public stage", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "409": { "description": "Conflict", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "500": { "description": "INTERNAL SERVER ERROR", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } } } } }, "/stages/{stageID}": { "get": { "tags": [ "Stages" ], "summary": "Retrieve stage details by ID", "description": "Retrieves detailed information about a specific stage identified by the provided stage ID, including its name, creation date, and last updated date.", "parameters": [ { "name": "stageID", "in": "path", "description": "The unique identifier of the stage to be deleted.", "required": true, "schema": { "type": "string", "example": "cb26237c-3beb-40f3-8e13-77d862f722b5" } } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Stage" } } } }, "400": { "description": "Bad Request", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "401": { "description": "Unauthorized", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "404": { "description": "Not Found", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "500": { "description": "INTERNAL SERVER ERROR", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } } } }, "delete": { "tags": [ "Stages" ], "summary": "Delete a specific stage", "description": "Permanently deletes the stage identified by the specified stage ID. This action cannot be undone and will remove all associated data with the stage.", "parameters": [ { "name": "stageID", "in": "path", "description": "The unique identifier of the stage to be deleted.", "required": true, "schema": { "type": "string", "example": "cb26237c-3beb-40f3-8e13-77d862f722b5" } } ], "responses": { "204": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponseDataset" } } } }, "400": { "description": "Bad Request", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "404": { "description": "Not Found", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "500": { "description": "INTERNAL SERVER ERROR", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } } } }, "patch": { "tags": [ "Stages" ], "summary": "Update Stage Configuration", "description": "Updates a stage's configuration including triggers, access policy, and retention policy. This is a partial update operation where only the provided fields will be updated. If triggers are provided, they will completely replace the existing triggers configuration. for updating access policy users must be either admin or owner of the stage.", "parameters": [ { "name": "stageID", "in": "path", "description": "Unique identifier of the stage to update", "required": true, "schema": { "type": "string", "format": "uuid", "example": "cb26237c-3beb-40f3-8e13-77d862f722b5" } } ], "requestBody": { "description": "Stage update configuration. All fields are optional - only provided fields will be updated.", "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/StageUpdate" }, "examples": { "setRetention30Days": { "summary": "Set retention to 30 days", "value": { "retentionPolicy": { "retentionPeriod": 30 } } }, "clearRetentionWithNull": { "summary": "Clear retention (unrestricted) using null", "value": { "retentionPolicy": null } }, "clearRetentionWithEmptyObject": { "summary": "Clear retention (unrestricted) using empty object", "value": { "retentionPolicy": {} } }, "clearRetentionWithZero": { "summary": "Clear retention (unrestricted) using retentionPeriod = 0", "value": { "retentionPolicy": { "retentionPeriod": 0 } } }, "noChangeOmittedField": { "summary": "No change (omit retentionPolicy)", "value": { "accessPolicy": "private" } } } } } }, "responses": { "200": { "description": "Stage updated successfully", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Stage" } } } }, "400": { "description": "Bad Request - Invalid stage ID format or malformed request body", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "401": { "description": "Unauthorized - Authentication required", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "403": { "description": "Forbidden - Insufficient permissions to update this stage", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "404": { "description": "Not Found - Stage with the specified ID does not exist", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "422": { "description": "Unprocessable Entity - Invalid transformation names or unsupported destination types", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "500": { "description": "Internal Server Error - An unexpected error occurred while updating the stage", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } } } } }, "/stages/{stageID}/runs": { "get": { "tags": [ "Stages" ], "summary": "Retrieve a list of runs for a stage", "description": "Retrieves a paginated list of runs associated with the specified stage.", "parameters": [ { "name": "stageID", "in": "path", "description": "The unique identifier of the stage to retrieve runs for.", "required": true, "schema": { "type": "string", "example": "cb26237c-3beb-40f3-8e13-77d862f722b5" } }, { "name": "page", "in": "query", "description": "The page number to retrieve in the paginated response (starting from 0).", "schema": { "type": "integer", "example": 0 } }, { "name": "size", "in": "query", "description": "The number of runs to include in each page of results.", "schema": { "type": "integer", "example": 10 } }, { "name": "status", "in": "query", "description": "The status of the runs to filter by.", "schema": { "type": "string", "example": "completed" } }, { "name": "transformationID", "in": "query", "description": "The ID of the transformation to filter by.", "schema": { "type": "string", "example": "3fa85f64-5717-4562-b3fc-2c963f66afa6" } } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/RunListWithPagination" } } } }, "400": { "description": "Bad Request", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "401": { "description": "Unauthorized", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "404": { "description": "Not Found", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "500": { "description": "INTERNAL SERVER ERROR", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } } }, "security": [ { "Bearer": [] } ] } }, "/stages/{stageID}/runs/events": { "get": { "tags": [ "Stages" ], "summary": "Stream real-time run updates for a stage", "description": "Opens a Server-Sent Events (SSE) connection that streams live run updates for a specific stage.\n\n**SSE Events:**\n- `connected`: Connection established\n- `runs:new`: New runs appeared on the current page\n- `runs:updated`: Existing runs changed status\n- `runs:removed`: Runs fell off the current page window (e.g. pushed out by newer runs)\n- `done`: All currently visible runs reached a terminal state\n- `error`: Stream error occurred\n\nThe stream closes automatically when all runs reach a terminal state or when the client disconnects.", "parameters": [ { "name": "stageID", "in": "path", "description": "The unique identifier of the stage to stream runs for.", "required": true, "schema": { "type": "string", "example": "cb26237c-3beb-40f3-8e13-77d862f722b5" } }, { "name": "page", "in": "query", "description": "The page number to retrieve in the paginated response (starting from 0).", "schema": { "type": "integer", "example": 0 } }, { "name": "size", "in": "query", "description": "The number of runs to include in each page of results.", "schema": { "type": "integer", "example": 10 } }, { "name": "status", "in": "query", "description": "The status of the runs to filter by.", "schema": { "type": "string", "example": "completed" } }, { "name": "transformationID", "in": "query", "description": "The ID of the transformation to filter by.", "schema": { "type": "string", "example": "3fa85f64-5717-4562-b3fc-2c963f66afa6" } } ], "responses": { "200": { "description": "OK", "content": { "text/event-stream": { "schema": { "type": "string", "description": "Server-Sent Events stream with real-time run updates", "example": "event: connected\ndata: {\"message\":\"Connected to runs stream\",\"stageId\":\"...\"}\n\nevent: runs:new\ndata: {\"runs\":[...],\"count\":1,\"total\":5}\n\nevent: runs:updated\ndata: {\"runs\":[...],\"count\":1,\"total\":5}\n\nevent: runs:removed\ndata: {\"runIds\":[\"...\"],\"count\":1,\"total\":4}\n\n: keepalive at 2026-01-01T00:00:00Z\n\nevent: done\ndata: {\"message\":\"All runs have reached a terminal state\"}\n" } } } }, "400": { "description": "Bad Request", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "401": { "description": "Unauthorized", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "404": { "description": "Not Found", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "500": { "description": "INTERNAL SERVER ERROR", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } } }, "security": [ { "Bearer": [] } ] } }, "/stages/{stageID}/files": { "get": { "tags": [ "Stages" ], "summary": "Retrieve a list of files in a stage", "description": "Retrieves a paginated list of files stored in the specified stage. Filters can be applied based on file name and creation dates to refine the results.", "parameters": [ { "name": "page", "in": "query", "description": "The page number to retrieve in the paginated response (starting from 0).", "schema": { "type": "integer", "example": 0 } }, { "name": "size", "in": "query", "description": "The number of files to include in each page of results.", "schema": { "type": "integer", "example": 10 } }, { "name": "stageID", "in": "path", "description": "The unique identifier of the stage from which to list the files.", "required": true, "schema": { "type": "string", "example": "cb26237c-3beb-40f3-8e13-77d862f722b5" } }, { "name": "name", "in": "query", "description": "An optional filter to retrieve files that match the specified name.", "schema": { "type": "string", "example": "my_file.txt" } }, { "name": "createdAfter", "in": "query", "description": "An optional filter to retrieve files created after the specified date (in ISO 8601 format).", "schema": { "type": "string", "example": "2024-01-01" } }, { "name": "createdBefore", "in": "query", "description": "An optional filter to retrieve files created before the specified date (in ISO 8601 format).", "schema": { "type": "string", "example": "2024-12-31" } } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/FileListWithPagination" } } } }, "400": { "description": "Bad Request", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "401": { "description": "Unauthorized", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "404": { "description": "Not Found", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "500": { "description": "INTERNAL SERVER ERROR", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } } } }, "post": { "tags": [ "Stages" ], "summary": "Create a new file in a stage", "description": "Uploads a new file to the specified stage. The file's content and optional metadata (such as the file name) can be provided in the request body. When uploading a text or Markdown file to a stage with triggers, keep in mind that the file must be in UTF-8 encoding, with a LF line feed character.", "parameters": [ { "name": "stageID", "in": "path", "description": "The unique identifier of the stage where the file will be uploaded.", "required": true, "schema": { "type": "string", "example": "cb26237c-3beb-40f3-8e13-77d862f722b5" } } ], "requestBody": { "required": true, "content": { "multipart/form-data": { "schema": { "type": "object", "required": [ "sourceData" ], "properties": { "sourceData": { "type": "string", "description": "The binary content of the file being uploaded.", "format": "binary" }, "name": { "type": "string", "description": "The name to assign to the uploaded file (optional).", "example": "my_file.txt" }, "ingestionContext": { "$ref": "#/components/schemas/IngestionContext" } } } } } }, "responses": { "201": { "description": "Created", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/File" } } } }, "400": { "description": "Bad Request", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponseDataset" } } } }, "401": { "description": "Unauthorized", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponseDataset" } } } }, "404": { "description": "Not Found", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "413": { "description": "Request Entity Too Large", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "500": { "description": "INTERNAL SERVER ERROR", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } } } } }, "/stages/{stageID}/files/{fileID}": { "get": { "tags": [ "Stages" ], "summary": "Retrieve the content of a specific file", "description": "Fetches the content of a file identified by its unique file ID within the specified stage. This operation returns the raw data of the file.", "parameters": [ { "name": "stageID", "in": "path", "description": "The unique identifier of the stage where the file is stored.", "required": true, "schema": { "type": "string", "example": "cb26237c-3beb-40f3-8e13-77d862f722b5" } }, { "name": "fileID", "in": "path", "description": "The unique identifier of the file whose content is being retrieved.", "required": true, "schema": { "type": "string", "example": "3fa85f64-5717-4562-b3fc-2c963f66afa6" } } ], "responses": { "200": { "description": "OK", "content": { "application/octet-stream": { "schema": { "type": "string", "format": "binary" } } } }, "400": { "description": "Bad Request", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponseDataset" } } } }, "404": { "description": "Not Found", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponseDataset" } } } } } }, "put": { "tags": [ "Stages" ], "summary": "Update an existing file in a stage", "description": "Uploads a new version of the specified file identified by its unique file ID in the given stage. You can provide updated content and an optional new name for the file. Will be generated a new version of a file.", "parameters": [ { "name": "stageID", "in": "path", "description": "The unique identifier of the stage containing the file to be updated.", "required": true, "schema": { "type": "string", "example": "cb26237c-3beb-40f3-8e13-77d862f722b5" } }, { "name": "fileID", "in": "path", "description": "The unique identifier of the file to be updated.", "required": true, "schema": { "type": "string", "example": "3fa85f64-5717-4562-b3fc-2c963f66afa6" } } ], "requestBody": { "required": true, "content": { "multipart/form-data": { "schema": { "type": "object", "required": [ "sourceData" ], "properties": { "sourceData": { "type": "string", "description": "The binary content of the file being uploaded as an update.", "format": "binary" }, "name": { "type": "string", "description": "An optional new name to assign to the file (if you want to rename it).", "example": "updated_file.txt" } } } } } }, "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/File" } } } }, "400": { "description": "Bad Request", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponseDataset" } } } }, "401": { "description": "Unauthorized", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponseDataset" } } } }, "404": { "description": "Not Found", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponseDataset" } } } }, "500": { "description": "INTERNAL SERVER ERROR", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } } } }, "delete": { "tags": [ "Stages" ], "summary": "Permanently delete a specific file", "description": "Deletes the file identified by its unique file ID from the specified stage. This action cannot be undone, and all associated data with the file will be permanently removed.", "parameters": [ { "name": "stageID", "in": "path", "description": "The unique identifier of the stage from which the file will be deleted.", "required": true, "schema": { "type": "string", "example": "cb26237c-3beb-40f3-8e13-77d862f722b5" } }, { "name": "fileID", "in": "path", "description": "The unique identifier of the file to be deleted.", "required": true, "schema": { "type": "string", "example": "3fa85f64-5717-4562-b3fc-2c963f66afa6" } } ], "responses": { "204": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponseDataset" } } } }, "400": { "description": "Bad Request", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponseDataset" } } } }, "404": { "description": "Not Found", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } } } } }, "/stages/{stageID}/files/{fileID}/presigned-url": { "get": { "tags": [ "Stages" ], "summary": "Get a presigned URL for a file", "description": "Generate a presigned URL for secure, time-limited access to a file. The URL can be used to download the file directly from the storage provider without requiring authentication.", "parameters": [ { "name": "stageID", "in": "path", "description": "The ID of the stage containing the file.", "required": true, "schema": { "type": "string", "format": "uuid", "example": "cb26237c-3beb-40f3-8e13-77d862f722b5" } }, { "name": "fileID", "in": "path", "description": "The ID of the file to get the presigned URL for.", "required": true, "schema": { "type": "string", "format": "uuid", "example": "3fa85f64-5717-4562-b3fc-2c963f66afa6" } }, { "name": "ttl", "in": "query", "description": "The time to live of the presigned URL in seconds. Default is 1800 seconds (30 minutes), maximum is 3600 seconds (1 hour).", "required": false, "schema": { "type": "integer", "minimum": 60, "maximum": 3600, "default": 1800, "example": 1800 } } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/PresignedURL" }, "examples": { "success": { "summary": "Successful presigned URL generation", "value": { "success": true, "presignedUrl": "https://storage.example.com/presigned-url?signature=abc123&expires=1640995200", "ttlSeconds": 1800 } } } } } }, "400": { "description": "Bad Request", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" }, "examples": { "invalid_ttl": { "summary": "Invalid TTL parameter", "value": { "error": "Bad Request", "message": "TTL must be between 60 and 3600 seconds", "statusCode": 400 } } } } } }, "401": { "description": "Unauthorized", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "404": { "description": "Not Found", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" }, "examples": { "file_not_found": { "summary": "File not found", "value": { "error": "Not Found", "message": "File with ID 3fa85f64-5717-4562-b3fc-2c963f66afa6 not found in stage cb26237c-3beb-40f3-8e13-77d862f722b5", "statusCode": 404 } } } } } }, "500": { "description": "Internal Server Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" }, "examples": { "storage_error": { "summary": "Storage service error", "value": { "error": "Internal Server Error", "message": "Failed to generate presigned URL", "statusCode": 500 } } } } } } } } }, "/connectors": { "get": { "tags": [ "Connectors" ], "summary": "Get all connectors", "description": "Get all connectors", "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ConnectorList" } } } } }, "security": [ { "Bearer": [] } ] }, "post": { "tags": [ "Connectors" ], "summary": "Create a new connector", "description": "Create a new connector", "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ConnectorCreate" } } } }, "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Connector" } } } }, "400": { "description": "Bad Request", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" }, "examples": { "invalid_ttl": { "summary": "Invalid TTL parameter", "value": { "error": "Bad Request", "message": "TTL must be between 60 and 3600 seconds", "statusCode": 400 } } } } } }, "401": { "description": "Unauthorized", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "500": { "description": "Internal Server Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } } }, "security": [ { "Bearer": [] } ] } }, "/connectors/{connectorID}": { "get": { "tags": [ "Connectors" ], "summary": "Get a connector", "description": "Get a connector", "parameters": [ { "name": "connectorID", "in": "path", "description": "The ID of the connector to get.", "required": true, "schema": { "type": "string", "example": "3fa85f64-5717-4562-b3fc-2c963f66afa6" } } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Connector" } } } }, "404": { "description": "Not Found", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } } }, "security": [ { "Bearer": [] } ] }, "patch": { "tags": [ "Connectors" ], "summary": "Update a connector", "description": "Update a connector", "parameters": [ { "name": "connectorID", "in": "path", "description": "The ID of the connector to patch.", "required": true, "schema": { "type": "string", "example": "3fa85f64-5717-4562-b3fc-2c963f66afa6" } } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ConnectorUpdate" } } } }, "responses": { "204": { "description": "No Content" }, "500": { "description": "Internal Server Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } } }, "security": [ { "Bearer": [] } ] } }, "/connectors/{connectorID}/files": { "get": { "tags": [ "Connectors" ], "summary": "Get all files for a connector", "description": "Get all files for a connector", "parameters": [ { "name": "connectorID", "in": "path", "description": "The ID of the connector to get files for.", "required": true, "schema": { "type": "string", "example": "3fa85f64-5717-4562-b3fc-2c963f66afa6" } } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/FileListWithPagination" } } } }, "404": { "description": "Not Found", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "500": { "description": "Internal Server Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } } }, "security": [ { "Bearer": [] } ] } }, "/connectors/{connectorID}/runs": { "get": { "tags": [ "Connectors" ], "summary": "Get all runs for a connector", "description": "Get all runs for a connector", "parameters": [ { "name": "connectorID", "in": "path", "description": "The ID of the connector to get files for.", "required": true, "schema": { "type": "string", "example": "3fa85f64-5717-4562-b3fc-2c963f66afa6" } }, { "name": "status", "in": "query", "description": "status of run.", "required": false, "schema": { "type": "string", "example": "completed", "enum": [ "created", "running", "completed", "failed" ] } } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/RunListWithPagination" } } } }, "404": { "description": "Not Found", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } }, "500": { "description": "Internal Server Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HandleFailedResponse" } } } } }, "security": [ { "Bearer": [] } ] } }, "/search_stores": { "post": { "tags": [ "Search Store" ], "summary": "Create a new Search Store instance for indexing and retrieval.", "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/SearchStoreCreate" } } } }, "responses": { "201": { "description": "Successfully created a new search store. Returns the unique ID of\nthe created search store.\n", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/SearchStoreInformation" } } } }, "400": { "description": "The request was malformed and couldn't be processed." } }, "security": [ { "Bearer": [] } ] }, "get": { "tags": [ "Search Store" ], "summary": "Lists all search store.", "parameters": [ { "$ref": "#/components/parameters/PageNumber" }, { "$ref": "#/components/parameters/PageSize" } ], "responses": { "200": { "description": "A list of search stores.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/SearchStoreInformationPage" } } } }, "400": { "description": "The request was malformed and couldn't be processed." } }, "security": [ { "Bearer": [] } ] } }, "/search_stores/{searchStoreID}": { "get": { "tags": [ "Search Store" ], "summary": "Fetch details of a specific Search Store by its ID.", "parameters": [ { "$ref": "#/components/parameters/SearchStoreID" } ], "responses": { "200": { "description": "A search store.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/SearchStoreInformation" } } } }, "400": { "description": "The request was malformed and couldn't be processed." }, "404": { "description": "A search store with the specified ID was not found." } }, "security": [ { "Bearer": [] } ] }, "patch": { "tags": [ "Search Store" ], "summary": "Update metadata of an existing Search Store.", "parameters": [ { "$ref": "#/components/parameters/SearchStoreID" } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/SearchStoreUpdate" } } } }, "responses": { "200": { "description": "The search store has been updated successfully.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/SearchStoreUpdate" } } } }, "400": { "description": "The request was malformed and couldn't be processed." }, "404": { "description": "A search store with the specified ID was not found." } }, "security": [ { "Bearer": [] } ] }, "delete": { "tags": [ "Search Store" ], "summary": "Delete a Search Store and all its associated documents.", "parameters": [ { "$ref": "#/components/parameters/SearchStoreID" } ], "responses": { "202": { "description": "The search store and all its documents will be deleted." }, "400": { "description": "The request was malformed and couldn't be processed." }, "404": { "description": "A search store with the specified ID was not found." } }, "security": [ { "Bearer": [] } ] } }, "/search_stores/{searchStoreID}/transitioning": { "get": { "tags": [ "Search Store" ], "summary": "Get count of all documents in a Search Store that are currently being embedded or archived.", "parameters": [ { "$ref": "#/components/parameters/SearchStoreID" } ], "responses": { "200": { "description": "Number of documents currently being embedded or archived.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/transitioningResult" } } } }, "400": { "description": "The request was malformed and couldn't be processed." }, "404": { "description": "A search store with the specified ID was not found." } }, "security": [ { "Bearer": [] } ] } }, "/search_stores/{searchStoreID}/search": { "post": { "tags": [ "Search Store" ], "summary": "Perform a semantic search within the specified Search Store.", "parameters": [ { "$ref": "#/components/parameters/SearchStoreID" } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/SearchStoreSearchRequest" } } } }, "responses": { "200": { "description": "A list of ranked search results.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/SearchStoreQueryResult" } } } }, "400": { "description": "The request was malformed and couldn't be processed." }, "404": { "description": "A search store with the specified ID was not found." } }, "security": [ { "Bearer": [] } ] } }, "/search_stores/{searchStoreID}/documents": { "get": { "tags": [ "Document" ], "summary": "Lists the information of documents in a search store.", "parameters": [ { "$ref": "#/components/parameters/SearchStoreID" }, { "$ref": "#/components/parameters/PageNumber" }, { "$ref": "#/components/parameters/PageSize" }, { "in": "path", "name": "startsWith", "description": "Only return documents whose name start with the given prefix.", "schema": { "type": "string" }, "required": false } ], "responses": { "200": { "description": "A list of documents.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/DocumentInformationPage" } } } }, "400": { "description": "The request was malformed and couldn't be processed." }, "404": { "description": "A search store with the specified ID was not found." } }, "security": [ { "Bearer": [] } ] } }, "/search_stores/{searchStoreID}/documents/{documentName}": { "parameters": [ { "$ref": "#/components/parameters/SearchStoreID" }, { "$ref": "#/components/parameters/DocumentName" } ], "put": { "tags": [ "Document" ], "summary": "Upload or update a document in the specified Search Store.", "parameters": [ { "$ref": "#/components/parameters/SearchStoreID" } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "allOf": [ { "$ref": "#/components/schemas/DocumentContent" }, { "$ref": "#/components/schemas/DocumentMetadata" } ] } } } }, "responses": { "200": { "description": "The document has been added.", "content": { "application/json": { "schema": { "allOf": [ { "$ref": "#/components/schemas/DocumentInformation" }, { "$ref": "#/components/schemas/DocumentMetadata" } ] } } } }, "400": { "description": "The request was malformed and couldn't be processed." }, "404": { "description": "A search store with the specified ID was not found." } }, "security": [ { "Bearer": [] } ] }, "get": { "tags": [ "Document" ], "summary": "Retrieve a document by name.", "responses": { "200": { "description": "The document's metadata, or `null` if no metadata is available.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/DocumentInformation" } } } }, "400": { "description": "The request was malformed and couldn't be processed." }, "404": { "description": "A resource with the specified name was not found." } }, "security": [ { "Bearer": [] } ] }, "delete": { "tags": [ "Document" ], "summary": "Remove a document from the search store.", "responses": { "202": { "description": "The document will be deleted asynchronously." }, "400": { "description": "The request was malformed and couldn't be processed." }, "404": { "description": "A resource with the specified ID was not found." } }, "security": [ { "Bearer": [] } ] } }, "/search_stores/{searchStoreID}/documents/{documentName}/content": { "parameters": [ { "$ref": "#/components/parameters/SearchStoreID" }, { "$ref": "#/components/parameters/DocumentName" } ], "get": { "tags": [ "Document" ], "summary": "Get the content of a document by name.", "responses": { "200": { "description": "The content of the document.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Content" } } } }, "400": { "description": "The request was malformed and couldn't be processed." }, "404": { "description": "A resource with the specified ID was not found." } }, "security": [ { "Bearer": [] } ] } }, "/search_stores/{searchStoreID}/documents/{documentName}/metadata": { "parameters": [ { "$ref": "#/components/parameters/SearchStoreID" }, { "$ref": "#/components/parameters/DocumentName" } ], "put": { "tags": [ "Document" ], "summary": "Replaces a document's metadata.", "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/NestedMetadata" } } } }, "responses": { "200": { "description": "The metadata of the document has been replaced.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/NestedMetadata" } } } }, "400": { "description": "The request was malformed and couldn't be processed." }, "404": { "description": "A resource with the specified ID was not found." } }, "security": [ { "Bearer": [] } ] } }, "/search_stores/{searchStoreID}/documents/{documentName}/chunks": { "parameters": [ { "$ref": "#/components/parameters/SearchStoreID" }, { "$ref": "#/components/parameters/DocumentName" } ], "get": { "tags": [ "Document" ], "summary": "Get the chunks of a document.", "responses": { "200": { "description": "The list of chunks of the document.", "content": { "application/json": { "schema": { "type": "array", "items": { "$ref": "#/components/schemas/DocumentSection" } } } } }, "400": { "description": "The request was malformed and couldn't be processed." }, "404": { "description": "A resource with the specified ID was not found." } }, "security": [ { "Bearer": [] } ] } } }, "components": { "securitySchemes": { "Bearer": { "type": "http", "scheme": "bearer", "bearerFormat": "JWT" } }, "schemas": { "Dataset": { "type": "object", "properties": { "datasetId": { "type": "string", "description": "Unique identifier for the dataset.", "example": "3fa85f64-5717-4562-b3fc-2c963f66afa6" }, "repositoryId": { "type": "string", "description": "Unique identifier of the repository to which this dataset belongs.", "example": "cb26237c-3beb-40f3-8e13-77d862f722b5" }, "name": { "type": "string", "description": "The name assigned to the dataset for easy identification.", "example": "dataset_name" }, "labels": { "type": "array", "description": "List of labels associated with the dataset for categorization or tagging.", "example": [ "label" ], "items": { "type": "string" } }, "metadata": { "type": "string", "description": "Additional metadata for the dataset in a key-value format.", "example": { "key": "value" } }, "license": { "type": "string", "description": "The license of the uploaded dataset defines the permissions, restrictions, and obligations governing its use.", "example": { "id": "OAL-1.0", "url": "https://github.com/Aleph-Alpha/.github/blob/main/OAL-1.0.pdf" } }, "version": { "type": "string", "description": "The current version of the dataset, allowing version tracking.", "example": "55a62ef9-dd9c-4e5d-9b93-0361bae4db37" }, "totalDatapoints": { "type": "integer", "description": "The total number of data points within the dataset.", "example": 0 }, "updatedAt": { "type": "string", "description": "The timestamp of the last update made to the dataset in ISO 8601 format.", "example": "2024-07-04" }, "createdAt": { "type": "string", "description": "The timestamp when the dataset was initially created in ISO 8601 format.", "example": "2024-07-04" } } }, "DatasetListResponse": { "type": "object", "properties": { "datasets": { "type": "array", "description": "A list of files retrieved for the current page.", "items": { "$ref": "#/components/schemas/Dataset" } }, "page": { "type": "integer", "description": "The current page number in the paginated list.", "example": 0 }, "size": { "type": "integer", "description": "The number of datasets displayed per page.", "example": 1 }, "total": { "type": "integer", "description": "The total number of datasets available.", "example": 1 } } }, "DatasetMetadataUpdate": { "type": "object", "properties": { "labels": { "type": "array", "description": "List of labels associated with the dataset for categorization or tagging.", "example": [ "label" ], "items": { "type": "string" } }, "metadata": { "type": "string", "description": "Additional metadata for the dataset in a key-value format.", "example": { "key": "value" } }, "license": { "type": "string", "description": "The license of the uploaded dataset defines the permissions, restrictions, and obligations governing its use.", "example": { "id": "OAL-1.0", "url": "https://github.com/Aleph-Alpha/.github/blob/main/OAL-1.0.pdf" } }, "name": { "type": "string", "description": "The name assigned to the dataset for easy identification.", "example": "dataset_name" }, "totalDatapoints": { "type": "integer", "description": "The total number of data points within the dataset.", "example": 0 } } }, "Download": { "type": "object", "properties": { "downloadId": { "type": "string", "description": "Unique identifier for the download.", "example": "81233e70-3cce-4e7c-a1bb-a819607f072a" }, "datasetversion": { "type": "string", "description": "The current version of the dataset, allowing version tracking.", "example": "55a62ef9-dd9c-4e5d-9b93-0361bae4db37" }, "mediaType": { "type": "string", "description": "The media type of the data stored in the repository", "example": "application/x-ndjson", "enum": [ "application/x-ndjson", "application/json", "application/jsonlines", "jsonlines" ] }, "totalDatapoints": { "type": "integer", "description": "The total number of data points within the dataset.", "example": 0 }, "url": { "type": "string", "description": "URL of the generated file, formatted according to the specified media type. This URL can be used to download the file via a GET request.", "example": "/stages/3fa85f64-5717-4562-b3fc-2c963f66afa6/files/3fa85f64-5717-4562-b3fc-2c963f66afa6" }, "expireAt": { "type": "string", "description": "The timestamp when the download file will expired created in ISO 8601 format.", "example": "2021-07-01T00:00:00Z" }, "createdAt": { "type": "string", "description": "The timestamp when the download request was initially created in ISO 8601 format.", "example": "2021-07-01T00:00:00Z" } } }, "DownloadCreate": { "type": "object", "properties": { "datasetversion": { "type": "string", "description": "The current version of the dataset, allowing version tracking.", "example": "55a62ef9-dd9c-4e5d-9b93-0361bae4db37" } } }, "DownloadListResponse": { "type": "object", "properties": { "downloads": { "type": "array", "description": "A list of download retrieved for the current page.", "items": { "$ref": "#/components/schemas/Download" } }, "page": { "type": "integer", "description": "The current page number in the paginated list.", "example": 0 }, "size": { "type": "integer", "description": "The number of downloads displayed per page.", "example": 1 }, "total": { "type": "integer", "description": "The total number of downloads available." } } }, "File": { "type": "object", "properties": { "fileID": { "type": "string", "description": "Unique identifier for the file.", "example": "3fa85f64-5717-4562-b3fc-2c963f66afa6" }, "stageID": { "type": "string", "description": "Unique identifier of the stage where the file is stored.", "example": "3fa85f64-5717-4562-b3fc-2c963f66afa6" }, "mediaType": { "type": "string", "description": "The media type of the file, indicating its format (e.g., application/octet-stream).", "example": "application/octet-stream", "enum": [ "application/octet-stream" ] }, "name": { "type": "string", "description": "The name of the file, used for easy identification.", "example": "dataset_name" }, "size": { "type": "integer", "description": "The size of the file in bytes.", "example": 1024 }, "version": { "type": "string", "description": "The current version of the file, allowing version tracking.", "example": "55a62ef9-dd9c-4e5d-9b93-0361bae4db37" }, "updatedAt": { "type": "string", "description": "The timestamp of the last update made to the file in ISO 8601 format.", "example": "2024-07-04" }, "createdAt": { "type": "string", "description": "The timestamp when the file was initially created in ISO 8601 format.", "example": "2024-07-04" }, "expireAt": { "type": "string", "nullable": true, "description": "The timestamp when the file will expire in ISO 8601 format, after which it will be deleted.", "example": "2024-07-04" } } }, "FileListWithPagination": { "type": "object", "properties": { "files": { "type": "array", "description": "A list of files retrieved for the current page.", "items": { "$ref": "#/components/schemas/File" } }, "page": { "type": "integer", "description": "The current page number in the paginated list.", "example": 0 }, "size": { "type": "integer", "description": "The number of files displayed per page.", "example": 1 }, "total": { "type": "integer", "description": "The total number of files available in the data platform.", "example": 1 } } }, "HandleFailedResponse": { "type": "object", "properties": { "message": { "type": "string", "example": "A descriptive error message based on the specific status code encountered, detailing the issue and possible causes." } } }, "HandleFailedResponseDataset": { "type": "object", "properties": { "message": { "type": "string", "example": "A descriptive error message based on the specific status code encountered, detailing the issue and possible causes." } } }, "HandleFailedResponseDownload": { "type": "object", "properties": { "message": { "type": "string", "example": "A descriptive error message based on the specific status code encountered, detailing the issue and possible causes." } } }, "Repository": { "type": "object", "properties": { "repositoryId": { "type": "string", "description": "Unique identifier for the repository.", "example": "cb26237c-3beb-40f3-8e13-77d862f722b5" }, "name": { "type": "string", "description": "The name assigned to the repository for easy identification.", "example": "My Repository" }, "mediaType": { "type": "string", "description": "The media type of the data stored in the repository", "example": "application/x-ndjson", "enum": [ "application/x-ndjson", "application/json", "application/jsonlines", "jsonlines" ] }, "modality": { "type": "string", "description": "The format or type of data, indicating the way it is structured, such as text.", "example": "text", "enum": [ "text" ] }, "mutable": { "type": "boolean", "description": "Indicates if the repository data is mutable (modifiable) after creation.", "example": false }, "schema": { "type": "object", "description": "Defines the schema([AVRO](https://avro.apache.org/docs/1.11.1/specification/)) for data points in the dataset of a given repository, including field types and defaults. This is optional and can be null. The schema is not enforced when upload a dataset(comming feature)", "nullable": true, "example": { "type": "record", "name": "MySchema", "fields": [ { "name": "model", "type": "string", "default": "pharia" } ] } }, "createdAt": { "type": "string", "description": "The timestamp for when the repository was created in ISO 8601 format.", "example": "2021-08-25T15:04:05Z" }, "updatedAt": { "type": "string", "description": "The timestamp for when the repository was last updated in ISO 8601 format.", "example": "2021-08-25T15:04:05Z" } } }, "RepositoryCreate": { "type": "object", "properties": { "name": { "type": "string", "description": "The name assigned to the repository for easy identification.", "example": "My Repository", "nullable": false }, "mediaType": { "type": "string", "description": "The media type of the data stored in the repository", "example": "application/x-ndjson", "enum": [ "application/x-ndjson", "application/json", "application/jsonlines", "jsonlines" ], "nullable": false }, "modality": { "type": "string", "description": "The format or type of data, indicating the way it is structured, such as text.", "example": "text", "enum": [ "text" ], "nullable": false }, "schema": { "type": "object", "description": "Defines the [AVRO](https://avro.apache.org/docs/1.11.1/specification/) schema for data points within the dataset of a repository, specifying field types and default values. This schema is optional and can be null. When a schema is provided, all datasets in the repository must conform to the schema when being created.", "nullable": true, "example": { "type": "record", "name": "MySchema", "fields": [ { "name": "model", "type": "string", "default": "pharia" } ] } }, "mutable": { "type": "boolean", "nullable": true, "description": "Indicates if the repository data is mutable (modifiable) after creation. When schema is provided, it is not supported for mutable repositories." } } }, "RepositoryListWithPagination": { "type": "object", "properties": { "repositories": { "type": "array", "items": { "$ref": "#/components/schemas/Repository" } }, "page": { "type": "integer", "description": "The current page number in the paginated list.", "example": 0 }, "size": { "type": "integer", "description": "The number of repositories displayed per page.", "example": 1 }, "total": { "type": "integer", "description": "The total number of repositories available.", "example": 1 } } }, "Stage": { "type": "object", "properties": { "name": { "type": "string", "description": "The name assigned to the stage for easy identification.", "example": "My Stage" }, "stageId": { "type": "string", "description": "Unique identifier for the stage.", "example": "cb26237c-3beb-40f3-8e13-77d862f722b5" }, "createdAt": { "type": "string", "description": "The timestamp when the stage was created in ISO 8601 format.", "example": "2021-08-25T15:04:05Z" }, "updatedAt": { "type": "string", "description": "The timestamp of the last update made to the stage in ISO 8601 format.", "example": "2021-08-25T15:04:05Z" }, "triggers": { "type": "array", "description": "List of triggers associated with the stage for automated processing.", "items": { "type": "object", "properties": { "name": { "type": "string", "description": "The name of the trigger to be applied.", "example": "PDFTriggerMyStage" }, "transformationName": { "type": "string", "description": "The name of the transformation to be triggered.", "example": "PDFToText" }, "destinationType": { "type": "string", "description": "The type of destination where the output will be stored.", "example": "DataPlatform:Repository" }, "connectorType": { "type": "string", "description": "The type of connector to be used for the transformation run.", "example": "DocumentIndex:Collection" }, "repositoryId": { "type": "string", "format": "uuid", "nullable": true, "description": "The ID of the repository created for this trigger's output.", "example": "815f77f9-6a68-4c42-86a4-9bd603395a49" } } }, "nullable": true }, "retentionPolicy": { "type": "object", "nullable": true, "description": "Optional stage-level retention policy. If `null`, the stage retention policy is unrestricted (no automatic expiration).", "properties": { "retentionPeriod": { "type": "integer", "example": 30, "description": "Number of days in the retention period. Every file uploaded will be deleted after the specified number of days. Each time a file is updated, its retention period is extended, but the retention period for older versions of the file remains unchanged." } } }, "filesCount": { "type": "integer", "description": "The number of files currently stored in this stage.", "example": 42 }, "accessPolicy": { "type": "string", "nullable": true, "description": "The access policy for the stage. This field is only included in the response when the permission service is enabled.\n\n- `private`: Only the owner can read and write (default)\n\n- `public`: All authenticated users can read, only the owner can write. Can only be set by admin users during creation.\n\n- `shared`: The stage has been shared with specific users/groups", "enum": [ "private", "public", "shared" ], "example": "private" }, "ownerId": { "type": "string", "nullable": true, "description": "The unique identifier of the user who owns this stage. This field is only included in the response when the permission service is enabled." }, "searchStore": { "type": "object", "nullable": true, "description": "Minimal search store information returned for GET/LIST operations. Only includes the ID.", "properties": { "id": { "type": "string", "format": "uuid", "description": "Unique identifier for the search store.", "example": "41804fe6-34b2-4db6-a4b1-dcd50fa7cf6a" } } } } }, "StageCreatedResponse": { "type": "object", "description": "Response schema for stage creation, includes full search store configuration details.", "properties": { "name": { "type": "string", "description": "The name assigned to the stage for easy identification.", "example": "My Stage" }, "stageId": { "type": "string", "description": "Unique identifier for the stage.", "example": "cb26237c-3beb-40f3-8e13-77d862f722b5" }, "createdAt": { "type": "string", "description": "The timestamp when the stage was created in ISO 8601 format.", "example": "2021-08-25T15:04:05Z" }, "updatedAt": { "type": "string", "description": "The timestamp of the last update made to the stage in ISO 8601 format.", "example": "2021-08-25T15:04:05Z" }, "triggers": { "type": "array", "description": "List of triggers associated with the stage for automated processing.", "items": { "type": "object", "properties": { "name": { "type": "string", "description": "The name of the trigger to be applied.", "example": "PDFTriggerMyStage" }, "transformationName": { "type": "string", "description": "The name of the transformation to be triggered.", "example": "PDFToText" }, "destinationType": { "type": "string", "description": "The type of destination where the output will be stored.", "example": "DataPlatform:Repository" }, "connectorType": { "type": "string", "description": "The type of connector to be used for the transformation run.", "example": "DocumentIndex:Collection" }, "repositoryId": { "type": "string", "format": "uuid", "nullable": true, "description": "The ID of the repository created for this trigger's output.", "example": "815f77f9-6a68-4c42-86a4-9bd603395a49" } } }, "nullable": true }, "retentionPolicy": { "type": "object", "nullable": true, "description": "The retention policy of the stage is an optional field. It cannot be changed after the stage is created and applies to every file uploaded to the stage.", "properties": { "retentionPeriod": { "type": "integer", "example": 30, "description": "Number of days in the retention period. Every file uploaded will be deleted after the specified number of days. Each time a file is updated, its retention period is extended, but the retention period for older versions of the file remains unchanged." } } }, "searchStore": { "type": "object", "nullable": true, "description": "Full search store configuration returned on stage creation.", "$ref": "#/components/schemas/SearchStoreContext" }, "accessPolicy": { "type": "string", "nullable": true, "description": "The access policy for the stage. This field is only included in the response when the permission service is enabled.\\n\\n- `private`: Only the owner can read and write (default)\\n\\n- `public`: All authenticated users can read, only the owner can write. Can only be set by admin users during creation.", "enum": [ "private", "public" ], "example": "private" }, "filesCount": { "type": "integer", "description": "The total number of files stored in this stage. Always 0 when first created.", "example": 0 }, "ownerId": { "type": "string", "nullable": true, "description": "The unique identifier of the user who owns this stage. This field is only included in the response when the permission service is enabled.", "example": "316056799180494087" } } }, "StageCreate": { "type": "object", "properties": { "name": { "type": "string", "description": "The name to assign to the new stage.", "example": "My Stage" }, "triggers": { "type": "array", "description": "List of triggers associated with the stage for automated processing.", "items": { "type": "object", "properties": { "name": { "type": "string", "description": "The name of the trigger to be applied.", "example": "PDFTriggerMyStage" }, "transformationName": { "type": "string", "description": "The name of the transformation to be triggered.", "example": "PDFToText" }, "destinationType": { "type": "string", "description": "The type of destination where the output will be stored.", "example": "DataPlatform:Repository" }, "connectorType": { "type": "string", "description": "The type of connector to be used for the transformation run.", "example": "DocumentIndex:Collection" } } }, "nullable": true }, "retentionPolicy": { "type": "object", "nullable": true, "description": "Optional retention policy for the stage. Omit this field or set it to `null` to create an unrestricted stage (no automatic expiration). If provided, `retentionPeriod` must be >= 1.", "properties": { "retentionPeriod": { "type": "integer", "minimum": 1, "example": 30, "description": "Number of days in the retention period. Every file uploaded will be deleted after the specified number of days. Each time a file is updated, its retention period is extended, but the retention period for older versions of the file remains unchanged." } }, "required": [ "retentionPeriod" ] }, "accessPolicy": { "type": "string", "nullable": true, "description": "Access policy that controls who can access this stage and its files. Set to 'private' for owner-only access (default), or 'public' for read-only access by all users. Note: Only admin users can create public stages. Non-admin users will receive a 403 Forbidden error if attempting to create a public stage. Public stages grant read-only access to all users while maintaining write access exclusively for the owner.", "enum": [ "private", "public" ], "example": "private" }, "searchStore": { "type": "object", "nullable": true, "description": "Optional search store configuration for stages that include search functionality.", "$ref": "#/components/schemas/SearchStoreContext" } } }, "StageUpdate": { "type": "object", "properties": { "triggers": { "type": "array", "description": "Optional list of triggers to update for the stage. If provided, replaces all existing triggers.", "items": { "type": "object", "required": [ "name", "transformationName", "destinationType" ], "properties": { "name": { "type": "string", "description": "Unique name identifier for this trigger within the stage.", "example": "PDFTriggerMyStage" }, "transformationName": { "type": "string", "description": "The name of the transformation to be automatically applied.", "example": "PDFToText" }, "destinationType": { "type": "string", "description": "The type of destination where the transformation output will be stored.", "example": "DataPlatform:Repository" }, "connectorType": { "type": "string", "nullable": true, "description": "Optional connector type for sending transformation results to external services.", "example": "DocumentIndex:Collection" } } } }, "accessPolicy": { "type": "string", "nullable": true, "description": "Access policy that controls who can access this stage and its files. Set to 'private' for restricted access or 'public' for open access. When null or omitted, the current access policy is preserved.", "enum": [ "private", "public", "shared" ], "example": "private" }, "retentionPolicy": { "type": "object", "nullable": true, "description": "Optional retention policy update.\n\nSemantics:\n- If omitted: keep the current retention policy (no change).\n- If `null`: clear the retention policy (unrestricted / no automatic expiration).\n- If `{}` or `{ \"retentionPeriod\": 0 }`: clear the retention policy (unrestricted).\n- If `{ \"retentionPeriod\": N }` with N > 0: set retention to N days and recalculate expiration dates for existing and future files/runs in the stage.", "properties": { "retentionPeriod": { "type": "integer", "minimum": 0, "example": 30, "description": "Number of days in the retention period. Use `0` to clear the retention policy (unrestricted). Values must be >= 0." } } }, "searchStoreId": { "type": "string", "format": "uuid", "nullable": true, "description": "Optional search store ID to link to the stage. When provided, it links the stage to the specified search store. Constraints: The stage must not already be linked to any search store, and the search store must not be linked to any other stage (enforces 1-to-1 relationship).", "example": "cb26237c-3beb-40f3-8e13-77d862f722b5" } } }, "StageListWithPagination": { "type": "object", "properties": { "stages": { "type": "array", "description": "A list of stages retrieved for the current page.", "items": { "$ref": "#/components/schemas/Stage" } }, "page": { "type": "integer", "description": "The current page number in the paginated list of stages.", "example": 0 }, "size": { "type": "integer", "description": "The number of stages displayed per page.", "example": 1 }, "total": { "type": "integer", "description": "The total number of stages available in the data platform.", "example": 1 } } }, "Transformation": { "type": "object", "properties": { "name": { "type": "string", "description": "The name of the transformation.", "example": "PDFToText" }, "transformationId": { "type": "string", "description": "Unique identifier for the transformation.", "example": "cb26237c-3beb-40f3-8e13-77d862f722b5" }, "inputType": { "type": "string", "description": "The type of data objcet that can be recevied by the transformation.", "example": "DataPlatform:Stage:File" }, "supportedDestinations": { "type": "array", "description": "Supported destinations (in the Data Platform) where the output can be loaded.", "items": { "type": "string", "example": "DataPlatform:Repository", "enum": [ "DataPlatform:Repository", "DataPlatform:Stage" ] } }, "outputSchema": { "type": "object", "description": "Defines the schema([AVRO](https://avro.apache.org/docs/1.11.1/specification/)) for each item in the output generated by the transformation when applied to data object inputs.", "nullable": true, "example": { "type": "record", "name": "PDFToText", "fields": [ { "name": "text", "type": "string" }, { "name": "page", "type": "int" } ] } }, "supportedInputMediaTypes": { "type": "array", "description": "Defines the media type that the input data object must be in to be processed by the transformation. This requirement only applies when the input data object is of the type DataPlatform:Stage:File.", "items": { "type": "string", "example": "application/pdf" } }, "parameters": { "type": "array", "description": "A list of the parameters that can be passed on every request to trigger an execution of the transformation", "items": { "$ref": "#/components/schemas/TransformationParameter" } } } }, "TransformationParameter": { "type": "object", "properties": { "name": { "type": "string", "description": "The name of the transformation parameter.", "example": "parser" }, "allowedValues": { "type": "array", "description": "Allowed values for the parameters", "items": { "type": "string", "example": "pypdf" } }, "description": { "type": "string", "nullable": true, "description": "Description of the parameter.", "example": "Parameter to select the parser" } } }, "TransformationRunBatchCreate": { "type": "object", "properties": { "runs": { "type": "array", "description": "A list of transformations to be run.", "items": { "$ref": "#/components/schemas/TransformationRunCreate" } } } }, "TransformationRunBatch": { "type": "object", "properties": { "runs": { "type": "array", "description": "A list of transformations to be run.", "items": { "$ref": "#/components/schemas/TransformationRun" } } } }, "TransformationRunCreate": { "type": "object", "properties": { "input": { "type": "object", "description": "The input data object for the transformation run, specifying the data source and location.", "properties": { "type": { "type": "string", "description": "The type of data object that the transformation will receive as input.", "example": "DataPlatform:Stage:File", "enum": [ "DataPlatform:Stage:File" ] }, "fileID": { "type": "string", "description": "The unique identifier of the file to be processed by the transformation. If a file is text or Markdown, it must be in UTF-8 encoding, with a LF line feed character.", "example": "3fa85f64-5717-4562-b3fc-2c963f66afa6" }, "stageID": { "type": "string", "description": "The unique identifier of the stage containing the file to be processed.", "example": "3fa85f64-5717-4562-b3fc-2c963f66afa6" }, "version": { "type": "string", "description": "The version of the file to be processed by the transformation.", "example": null, "nullable": true } } }, "destination": { "type": "object", "description": "The destination for the output generated by the transformation run, specifying the target location and format.", "properties": { "type": { "type": "string", "description": "The type of destination where the output will be stored.", "example": "DataPlatform:Repository", "enum": [ "DataPlatform:Repository" ] }, "repositoryID": { "type": "string", "description": "The unique identifier of the repository where the output will be stored.", "example": "3fa85f64-5717-4562-b3fc-2c963f66afa6" }, "mapping": { "type": "object", "description": "A mapping of fields to be used when storing the output in the destination.", "example": null, "nullable": true } } }, "connector": { "type": "object", "description": "The connector to be used for the transformation run, specifying the external service to sync with.", "properties": { "type": { "type": "string", "description": "The type of connector to be used for the transformation run.", "example": "DocumentIndex:Collection", "enum": [ "DocumentIndex:Collection" ] }, "collection": { "type": "string", "description": "The collection name for the connector.", "example": "my-collection" }, "namespace": { "type": "string", "description": "The namespace for the connector.", "example": "my-namespace" } } }, "parameters": { "type": "object", "description": "Parameters of the transformation. Supported parameters for each transformation can be retrieved via the list transformation endpoint.", "properties": { "key": { "type": "string", "description": "Parameter", "example": "value", "nullable": false } } } } }, "TransformationRun": { "type": "object", "properties": { "transformationId": { "type": "string", "description": "Unique identifier for the transformation.", "example": "cb26237c-3beb-40f3-8e13-77d862f722b5" }, "runId": { "type": "string", "description": "Unique identifier for the transformation run.", "example": "3fa85f64-5717-4562-b3fc-2c963f66afa6" }, "transformationName": { "type": "string", "description": "The name of the transformation.", "example": "PDFToText" }, "status": { "type": "string", "description": "The current status of the transformation run, indicating if it is pending, in progress, or completed.", "example": "completed", "enum": [ "created", "running", "failed", "completed" ] }, "createdAt": { "type": "string", "description": "The timestamp when the transformation run was initially created in ISO 8601 format.", "example": "2021-08-25T15:04:05Z" }, "updatedAt": { "type": "string", "description": "The timestamp of the last update made to the transformation run in ISO 8601 format.", "example": "2021-08-25T15:04:05Z" }, "errors": { "type": "array", "description": "A list of error messages encountered during the transformation run.", "items": { "$ref": "#/components/schemas/RunError" } }, "parameters": { "type": "object", "description": "A mapping of parameters used for the transformation run, including input and output settings.", "example": { "key": "value" } }, "metadata": { "type": "object", "description": "Additional metadata for the transformation run in a key-value format.", "example": { "key": "value" } }, "input": { "type": "object", "description": "The input data object for the transformation run, specifying the data source and location.", "properties": { "type": { "type": "string", "description": "The type of data object that the transformation will receive as input.", "example": "DataPlatform:Stage:File", "enum": [ "DataPlatform:Stage:File" ] }, "fileID": { "type": "string", "description": "The unique identifier of the file to be processed by the transformation.", "example": "3fa85f64-5717-4562-b3fc-2c963f66afa6" }, "stageID": { "type": "string", "description": "The unique identifier of the stage containing the file to be processed.", "example": "3fa85f64-5717-4562-b3fc-2c963f66afa6" }, "version": { "type": "string", "description": "The version of the file to be processed by the transformation.", "example": null, "nullable": true } } }, "destination": { "type": "object", "description": "The destination for the output generated by the transformation run, specifying the target location and format.", "properties": { "type": { "type": "string", "description": "The type of destination where the output will be stored.", "example": "DataPlatform:Repository", "enum": [ "DataPlatform:Repository" ] }, "repositoryID": { "type": "string", "description": "The unique identifier of the repository where the output will be stored.", "example": "3fa85f64-5717-4562-b3fc-2c963f66afa6" }, "mapping": { "type": "object", "description": "A mapping of fields to be used when storing the output in the destination.", "example": null, "nullable": true } } }, "connector": { "type": "object", "description": "The connector to be used for the transformation run, specifying the external service to sync with.", "properties": { "type": { "type": "string", "description": "The type of connector to be used for the transformation run.", "example": "DocumentIndex:Collection", "enum": [ "DocumentIndex:Collection" ] }, "collection": { "type": "string", "description": "The collection name for the connector.", "example": "my-collection" }, "namespace": { "type": "string", "description": "The namespace for the connector.", "example": "my-namespace" } } }, "output": { "type": "object", "description": "The output data object for the transformation run, specifying the data source and location.", "properties": { "type": { "type": "string", "description": "The type of data object that the transformation will receive as input.", "example": "DataPlatform:Repository:Dataset", "enum": [ "DataPlatform:Repository:Dataset" ] }, "repositoryID": { "type": "string", "description": "The unique identifier of the file to be processed by the transformation.", "example": "3fa85f64-5717-4562-b3fc-2c963f66afa6" }, "datasetID": { "type": "string", "description": "The unique identifier of the stage containing the file to be processed.", "example": "3fa85f64-5717-4562-b3fc-2c963f66afa6" }, "version": { "type": "string", "description": "The version of the file to be processed by the transformation.", "example": null, "nullable": true } } } } }, "TransformationListWithPagination": { "type": "object", "properties": { "transformations": { "description": "A list of transformations retrieved for the current page.", "type": "array", "items": { "$ref": "#/components/schemas/Transformation" } }, "page": { "type": "integer", "description": "The current page number in the paginated list.", "example": 0 }, "size": { "type": "integer", "description": "The number of transformations displayed per page.", "example": 1 }, "total": { "type": "integer", "description": "The total number of transformations available.", "example": 1 } } }, "RunListWithPagination": { "type": "object", "properties": { "runs": { "description": "A list of runs retrieved for the current page given a transformation id and or other parameters.", "type": "array", "items": { "$ref": "#/components/schemas/TransformationRun" } }, "page": { "type": "integer", "description": "The current page number in the paginated list.", "example": 0 }, "size": { "type": "integer", "description": "The number of runs displayed per page.", "example": 1 }, "total": { "type": "integer", "description": "The total number of runs created.", "example": 1 } } }, "RunError": { "type": "object", "properties": { "type": { "type": "string", "description": "The type of error that occurred during the transformation run.", "enum": [ "UnexpectedError", "ExtractorError", "FileTooLargeError", "TooManyPagesError", "TransformerError", "LoaderError", "ConnectorError", "ConfigurationError", "InternalServiceError", "ServiceNotAvailableError", "TooManyTokensError" ], "example": "ExtractorError" }, "message": { "type": "string", "description": "A human-readable description of the error that occurred.", "example": "Error message" }, "httpCode": { "type": "integer", "description": "The HTTP status code associated with the error.", "example": 422 }, "dataCode": { "type": "string", "description": "A specific error code indicating the nature of the error: - DATA0000: Unexpected error - Code used for runs with legacy errors and errors that don't fit in the other categories - DATA0001: Extractor error - Something is wrong with the file (e.g., password protected file, parser unable to process) - DATA0002: File too large - File exceeds the size limit - DATA0003: Too many pages - File exceeds the page limit - DATA0004: Transformer error - Internal error in the transformation process - DATA0005: Loader error - Issue with dataset production (potential bug) - DATA0006: Connector error - Issue with sending to document index (e.g., request size limit reached) - DATA0007: Configuration error - Issue with document index configuration (e.g., non-existent namespace, collection) - DATA0008: Internal service error - Potential bug in the code - DATA0009: Service not available - Internal system (data API, document index) is unavailable - DATA0010: Too many tokens to index - The file contains too many tokens to be indexed and will not be processed", "enum": [ "DATA0000", "DATA0001", "DATA0002", "DATA0003", "DATA0004", "DATA0005", "DATA0006", "DATA0007", "DATA0008", "DATA0009", "DATA0010" ], "example": "DATA0001" } } }, "IngestionContext": { "type": "object", "properties": { "triggerName": { "type": "string", "description": "The name of a trigger registered to a Stage." }, "connectorContext": { "type": "object", "properties": { "collection": { "type": "string", "description": "The Document Index collection to be used for the transformation run." }, "namespace": { "type": "string", "description": "The Document Index namespace to be used for the transformation run." } } }, "destinationContext": { "type": "object", "properties": { "repositoryId": { "type": "string", "description": "The repository ID to be used as the target destination for the transformation run." } } }, "transformationContext": { "type": "object", "properties": { "parameters": { "type": "object", "description": "The parameters to be used for the transformation run." } } } } }, "ConnectorCreate": { "type": "object", "properties": { "id": { "type": "string", "description": "The ID of the connector." }, "connectionId": { "type": "string", "description": "The connection ID of the connector from OAuth-Gateway-API." }, "name": { "type": "string", "description": "The name of the connector. It is not unique." }, "connector_mode": { "type": "string", "enum": [ "non-synced", "synced" ], "description": "Whether the connector is enabled for sync." }, "stage_id": { "type": "string", "description": "The ID of the stage to be used to store the files. the triggers set in this stage will be used when files are fetched from." }, "source": { "type": "object", "description": "The data source configuration.", "properties": { "type": { "type": "string", "enum": [ "sharepoint", "google_drive" ] }, "configuration": { "type": "object", "properties": { "driveId": { "type": "string", "description": "The unique Microsoft Graph drive ID. No required with Google Drive." }, "folderId": { "type": "string", "description": "The folder Id from Sharepoint or Google Drive." }, "fileIds": { "type": "array", "items": { "type": "string" }, "description": "List of file Ids from from Shareporint or Google Drive." } } } } }, "destination": { "type": "object", "description": "The destination of the files ingested via the connector. It is not required if the stage was created with search store. Keep in mind the every time the parameter is provided, it will be validated againt the configuration of the satge.", "properties": { "type": { "type": "string", "enum": [ "DataPlatform:SearchStore", "DocumentIndex:SearchStore", "DocumentIndex:Collection" ], "example": "DataPlatform:SearchStore" }, "searchStore": { "type": "string", "description": "Id of the search store" }, "namespace": { "type": "string", "description": "Namesapce, requried when destination.type is DocumentIndex:Collection" }, "collection": { "type": "string", "description": "Name of rhe collection, requried when destination.type is DocumentIndex:Collection" } } }, "transformationContext": { "type": "object", "description": "The transformation context for the connector. Pass to the transformation applied to the files. In this case the first transformation set in the triggers of the stage.", "nullable": true, "properties": { "parameters": { "type": "object", "description": "The parameters to be used for the transformation run." } } }, "createdAt": { "type": "string", "description": "The date and time the connector was created." } } }, "Connector": { "type": "object", "properties": { "id": { "type": "string", "description": "The ID of the connector." }, "connectionId": { "type": "string", "description": "The connection ID of the connector from OAuth-Gateway-API." }, "name": { "type": "string", "description": "The name of the connector. It is not unique." }, "connector_mode": { "type": "string", "enum": [ "non-synced", "synced" ], "description": "Whether the connector is enabled for sync." }, "provider": { "type": "string", "enum": [ "sharepoint", "google_drive" ] }, "stage_id": { "type": "string", "description": "The ID of the stage to be used to store the files. the triggers set in this stage will be used when files are fetched from." }, "source": { "type": "object", "description": "The data source configuration.", "properties": { "type": { "type": "string", "enum": [ "sharepoint", "google_drive" ] }, "configuration": { "type": "object", "properties": { "driveId": { "type": "string", "description": "The unique Microsoft Graph drive ID. No required with Google Drive." }, "folderId": { "type": "string", "description": "The folder Id from Sharepoint or Google Drive." }, "fileIds": { "type": "array", "items": { "type": "string" }, "description": "List of file Ids from from Shareporint or Google Drive." } } } } }, "destination": { "type": "object", "description": "The destination of the files ingested via the connector. It is not required if the stage was created with search store. Keep in mind the every time the parameter is provided, it will be validated againt the configuration of the satge.", "properties": { "type": { "type": "string", "enum": [ "DataPlatform:SearchStore", "DocumentIndex:SearchStore", "DocumentIndex:Collection" ], "example": "DataPlatform:SearchStore" }, "configuration": { "type": "object", "properties": { "searchStore": { "type": "string", "description": "Id of the search store" }, "namespace": { "type": "string", "description": "Namesapce, requried when destination.type is DocumentIndex:Collection" }, "collection": { "type": "string", "description": "Name of rhe collection, requried when destination.type is DocumentIndex:Collection" } } } } }, "transformationContext": { "type": "object", "description": "The transformation context for the connector. Pass to the transformation applied to the files. In this case the first transformation set in the triggers of the stage.", "nullable": true, "properties": { "parameters": { "type": "object", "description": "The parameters to be used for the transformation run." } } }, "createdAt": { "type": "string", "description": "The date and time the connector was created." }, "status": { "type": "string", "description": "The status of the connector.", "example": "created", "enum": [ "initiated", "start_to_full_sync", "full_sync_in_progress", "full_sync_completed", "full_sync_failed", "start_to_incremental_sync", "incremental_sync_in_progress", "incremental_sync_completed", "deleted", "full_sync_partially_completed" ] } } }, "ConnectorList": { "type": "object", "properties": { "connectors": { "type": "array", "items": { "$ref": "#/components/schemas/Connector" } }, "page": { "type": "integer", "description": "The current page number in the paginated list.", "example": 0 }, "size": { "type": "integer", "description": "The number of connectors displayed per page.", "example": 1 } } }, "ConnectorUpdate": { "type": "object", "properties": { "connector_mode": { "type": "boolean", "description": "Whether the connector is enabled for sync." } } }, "PresignedURL": { "type": "object", "properties": { "success": { "type": "boolean", "description": "Indicates whether the presigned URL generation was successful.", "example": true }, "presignedUrl": { "type": "string", "description": "The presigned URL generated for the file.", "example": "https://storage.example.com/presigned-url?signature=abc123&expires=1640995200" }, "ttlSeconds": { "type": "integer", "description": "The time to live (TTL) of the presigned URL in seconds.", "example": 1800 } }, "required": [ "success", "presignedUrl" ] }, "SearchStoreContext": { "type": "object", "properties": { "id": { "type": "string", "description": "Unique identifier for the search store.", "example": "123e4567-e89b-12d3-a456-426614174000" }, "chunkingStrategy": { "type": "object", "nullable": true, "description": "Strategy for chunking documents in the search store.", "properties": { "maxChunkSizeTokens": { "type": "integer", "description": "Maximum number of tokens per chunk.", "example": 100 }, "chunkOverlapTokens": { "type": "integer", "description": "Number of tokens to overlap between chunks.", "example": 10 } } }, "embeddingStrategy": { "type": "object", "nullable": true, "description": "Strategy for embedding documents in the search store.", "properties": { "type": { "type": "string", "description": "Type of embedding strategy.", "enum": [ "instruct", "semantic" ], "example": "instruct" }, "config": { "type": "object", "description": "Configuration specific to the embedding strategy type. For 'instruct' type, contains model, instruction, and hybridIndex. For 'semantic' type, contains model, hybridIndex, and representation.", "oneOf": [ { "type": "object", "description": "Configuration for instruct embedding strategy", "properties": { "model": { "type": "string", "example": "luminous-base" }, "instruction": { "type": "object", "properties": { "document": { "type": "string", "example": "Represent this document for retrieval" }, "query": { "type": "string", "example": "Represent this query for searching" } } }, "hybridIndex": { "type": "string", "example": "bm25" } } }, { "type": "object", "description": "Configuration for semantic embedding strategy", "properties": { "model": { "type": "string", "example": "luminous-base" }, "hybridIndex": { "type": "string", "example": "bm25" }, "representation": { "type": "string", "example": "asymmetric" } } } ] } } }, "metadata": { "type": "object", "nullable": true, "description": "Additional metadata for the search store.", "additionalProperties": true, "example": { "description": "My Search Store", "tags": [ "personalDocument", "tags" ] } }, "metadataSchema": { "type": "object", "nullable": true, "description": "Schema defining additional custom metadata fields and their types for documents in the search store. A default metadata schema is always applied with standard fields (author, createdAt, fileName, fileSize, etc.). You can add new custom fields or specify default fields with their original types. However, you cannot change the type of default fields - attempting to do so will result in a validation error.", "additionalProperties": { "type": "string", "enum": [ "string", "integer", "float", "boolean", "date_time" ] }, "example": { "customField": "string", "customDate": "date_time", "customCount": "integer" } } } }, "SchemaVersionField": { "type": "string", "enum": [ "V1" ] }, "SearchStoreID": { "type": "string", "pattern": "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", "example": "568cf1c0-5251-4210-8b61-d1eff43011b2" }, "CreatedAt": { "type": "string", "pattern": "[0-9]{4}-[0-9]{2}-[0-9]{2}", "example": "2025-12-31" }, "ModalityText": { "type": "object", "properties": { "modality": { "type": "string", "enum": [ "text" ] }, "text": { "type": "string" } } }, "ModalityImage": { "type": "object", "properties": { "modality": { "type": "string", "enum": [ "image" ] }, "image": { "type": "string", "example": "" } } }, "DocumentMetadata": { "type": "object", "properties": { "metadata": { "$ref": "#/components/schemas/NestedMetadata" } } }, "NestedMetadata": { "type": "object", "description": "Metadata attached to a search store file. This can be useful for storing\nadditional structured information.\n\nKeys are strings with a maximum length of 64 characters.\n", "additionalProperties": { "anyOf": [ { "type": "string" }, { "type": "number" }, { "type": "boolean" }, { "type": "array", "items": {} }, { "type": "object" } ] }, "example": { "category": "research-paper", "wordCount": 6000, "peerReviewed": true } }, "ChunkingStrategy": { "type": "object", "properties": { "maxChunkSizeTokens": { "type": "integer", "minimum": 1 }, "chunkOverlapTokens": { "type": "integer" } } }, "EmbeddingStrategyInstruct": { "type": "object", "properties": { "model": { "type": "string" }, "instruction": { "type": "object", "properties": { "document": { "type": "string", "description": "Instruction to be used for embedding the document." }, "query": { "type": "string", "description": "Instruction to be used for embedding the query." } } }, "hybridIndex": { "type": "string", "enum": [ "bm25" ] } }, "required": [ "model", "instruction" ] }, "EmbeddingStrategySemantic": { "type": "object", "properties": { "model": { "type": "string" }, "representation": { "type": "string", "enum": [ "asymmetric", "symmetric" ], "description": "The type of embedding representation, which defines how text is\nencoded into a vector.\n\n* `symmetric`: embedding representation where the same encoding model\n and strategy are used to embed both queries and documents.\n \n PhariaSearch automatically selects the appropriate optimized\n representation based on the configured Index.\n \n Symmetric embeddings are suitable for tasks where both the query\n and the stored documents have a similar nature and should be directly\n comparable in the same vector space.\n\n* `asymmetric`: embedding representation where different encoding\n models and/or encoding strategies are used for queries and\n documents.\n\n PhariaSearch automatically selects the appropriate optimized\n representation based on the configured Index.\n \n Asymmetric embeddings are used when queries and documents differ in\n structure, intent, or length, requiring distinct optimizations for\n retrieval.\n" }, "hybridIndex": { "type": "string", "enum": [ "bm25" ] } }, "required": [ "model", "representation" ] }, "EmbeddingStrategyVllm": { "type": "object", "properties": { "model": { "type": "string", "description": "The VLLM model name to use for embeddings", "example": "qwen3-embedding-8b" }, "instruction": { "type": "object", "description": "Optional instruction templates for query and document embeddings. Both fields are optional - you can provide none, one, or both.\n\nIf a field is missing, the raw text is embedded as-is.\nIf provided and contains `{{text}}`, the placeholder is replaced with the actual text.\nIf provided without `{{text}}`, it is treated as a prefix.\n\nExamples:\n- query = \"query: {{text}}\" with text \"capital of germany\" \u2192 \"query: capital of germany\"\n- document = \"passage:\" with text \"Berlin is the capital\" \u2192 \"passage: Berlin is the capital\"", "properties": { "document": { "type": "string", "description": "Template used to format document text. Use `{{text}}` as placeholder (optional - without it, acts as prefix).", "example": "passage: {{text}}" }, "query": { "type": "string", "description": "Template used to format query text. Use `{{text}}` as placeholder (optional - without it, acts as prefix).", "example": "query: {{text}}" } } }, "encodingFormat": { "type": "string", "enum": [ "float", "base64" ], "description": "The format to return the embeddings in. Can be either `float` or `base64`.", "example": "float" }, "dimensions": { "type": "integer", "minimum": 1, "description": "The number of dimensions the resulting output embeddings should have. Not supported by all models.", "example": 1024 } }, "required": [ "model" ], "example": { "model": "qwen3-embedding-8b", "instruction": { "query": "query: {{text}}", "document": "passage: {{text}}" }, "encodingFormat": "float", "dimensions": 1024 } }, "SearchStoreMetadata": { "type": "object", "description": "Metadata attached to a search store.\n\nThis can be useful for storing additional structured information about\nthe search store itself or the contained files.\n\nKeys are strings with a maximum length of 64 characters. Values are either\nstrings with a maximum length of 512 characters, numbers or boolean values.\n", "additionalProperties": { "anyOf": [ { "type": "string", "maxLength": 512 }, { "type": "boolean" }, { "type": "number" } ] }, "example": { "category": "research-papers", "approved": true } }, "SearchStoreProperties": { "type": "object", "required": [ "chunkingStrategy", "embeddingStrategy" ], "properties": { "chunkingStrategy": { "$ref": "#/components/schemas/ChunkingStrategy" }, "embeddingStrategy": { "type": "object", "required": [ "type", "config" ], "properties": { "type": { "type": "string", "enum": [ "semantic", "instruct", "vllm" ] }, "config": { "oneOf": [ { "$ref": "#/components/schemas/EmbeddingStrategyVllm" }, { "$ref": "#/components/schemas/EmbeddingStrategySemantic" }, { "$ref": "#/components/schemas/EmbeddingStrategyInstruct" } ] } } }, "metadata": { "$ref": "#/components/schemas/SearchStoreMetadata" }, "metadataSchema": { "$ref": "#/components/schemas/MetadataSchema" } } }, "MetadataSchema": { "type": "object", "additionalProperties": { "type": "string", "enum": [ "string", "integer", "float", "boolean", "date_time" ] }, "description": "A dictionary of metadata fields on which to filter search results and their types.\n\nField names must only contain alphanumeric characters, dashes and underscores.\n\nNested fields can be specified using dot notation (e.g. 'a.b').\n\nArray-valued fields can either use a wildcard specifier (e.g. 'a[].b') or a specific index (e.g. 'a[1].b').\n\nThe maximum length of the field name is 1000 characters.\n", "example": { "approved": "boolean" } }, "SearchStoreCreate": { "allOf": [ { "$ref": "#/components/schemas/SearchStoreProperties" } ] }, "SearchStoreInformation": { "allOf": [ { "type": "object", "properties": { "id": { "$ref": "#/components/schemas/SearchStoreID" }, "createdAt": { "$ref": "#/components/schemas/CreatedAt" } } }, { "$ref": "#/components/schemas/SearchStoreProperties" } ] }, "SearchStoreInformationPage": { "type": "object", "properties": { "page": { "type": "number", "example": 1 }, "size": { "type": "number", "example": 1 }, "total": { "type": "number", "example": 10 }, "results": { "type": "array", "items": { "$ref": "#/components/schemas/SearchStoreInformation" } } } }, "SearchStoreUpdate": { "type": "object", "properties": { "metadata": { "$ref": "#/components/schemas/SearchStoreMetadata" } } }, "SearchStoreSearchRequest": { "type": "object", "properties": { "query": { "type": "array", "items": { "oneOf": [ { "$ref": "#/components/schemas/ModalityText" }, { "$ref": "#/components/schemas/ModalityImage" } ] } }, "filters": { "type": "array", "items": { "type": "object" } }, "minScore": { "type": "integer" }, "maxResults": { "type": "integer" } } }, "Content": { "type": "array", "items": { "oneOf": [ { "$ref": "#/components/schemas/ModalityText" }, { "$ref": "#/components/schemas/ModalityImage" } ] } }, "transitioningResult": { "description": "The result for a transitioning documents query.\n", "type": "object", "properties": { "currently_embedding": { "type": "integer" }, "currently_archiving": { "type": "integer" } }, "required": [ "currently_embedding", "currently_archiving" ] }, "SearchStoreQueryResult": { "description": "The result for semantic document search.\nAn array of document names and content of the found documents in the given collection.\n", "type": "array", "items": { "allOf": [ { "$ref": "#/components/schemas/DocumentSection" }, { "type": "object", "properties": { "score": { "type": "number", "format": "float", "description": "Search score of the found section, where a higher score indicates a closer match.\nWill be between -1 and 1. A score closer to -1 indicates the section opposes the\nquery. A score close 0 suggests the section is unrelated to the query. A score\nclose to 1 suggests the section is related to the query. The score depends on\nthe index configuration, e.g. the score of a section differs for hybrid and\nnon-hybrid indexes. For searches on hybrid indexes, the score can exceed the\nmin_score of the query as the min_score only applies to the semantic similarity\nsub-query.\n" } } } ] } }, "DocumentInformation": { "description": "Information about a document", "type": "object", "properties": { "name": { "type": "string", "description": "Name of the document", "example": "document-123" }, "createdAt": { "type": "string", "format": "date-time" }, "version": { "type": "integer", "minimum": 1 }, "metadata": { "$ref": "#/components/schemas/NestedMetadata" } } }, "DocumentInformationPage": { "type": "object", "properties": { "page": { "type": "number", "example": 1 }, "size": { "type": "number", "example": 1 }, "total": { "type": "number", "example": 10 }, "results": { "type": "array", "items": { "$ref": "#/components/schemas/DocumentInformation" } } } }, "DocumentPath": { "description": "The path to a document. A path uniquely identifies a document among all managed documents.", "type": "object", "properties": { "name": { "description": "Name of the document. The name is unique within a collection.", "type": "string", "maxLength": 512 }, "collection": { "description": "Collection containing documents. The name of the collection must be unique within a namespace.\n", "type": "string", "maxLength": 100 }, "namespace": { "description": "Namespace for collections of documents. Namespaces support preventing naming conflicts\nfor collections. A namespace typically corresponds to an organisation.\n", "type": "string", "maxLength": 100 } } }, "DocumentContent": { "description": "Represents a document. The actual content of the document is an array of text and image elements.\nThe version field allow to evolve the schema. The documentation here describes only the most recent version.\n", "type": "object", "properties": { "schemaVersion": { "type": "string", "description": "The schema version of the document", "example": "V1", "enum": [ "V1" ] }, "contents": { "type": "array", "items": { "oneOf": [ { "$ref": "#/components/schemas/Text" }, { "$ref": "#/components/schemas/Image" } ] } }, "metadata": { "description": "Store any metadata about the document. Accepts any valid JSON. You can filter search\nresults based on these metadata fields using filter indexes.\n" } } }, "DocumentSection": { "description": "A section of a document.\n", "type": "object", "properties": { "documentName": { "type": "string", "description": "The name of the document this chunk belongs to.", "example": "document-123" }, "section": { "type": "array", "items": { "oneOf": [ { "$ref": "#/components/schemas/Text" }, { "$ref": "#/components/schemas/Image" } ] } }, "start": { "oneOf": [ { "$ref": "#/components/schemas/TextCursor" }, { "$ref": "#/components/schemas/ImageCursor" } ] }, "end": { "description": "Inclusive cursor. The section of the document includes whatever is at this position", "oneOf": [ { "$ref": "#/components/schemas/TextCursor" }, { "$ref": "#/components/schemas/ImageCursor" } ] } } }, "Text": { "description": "Represent a single piece of continuous text. The modality must always be 'text'.", "type": "object", "properties": { "modality": { "type": "string", "enum": [ "text" ] }, "text": { "type": "string" } } }, "TextCursor": { "description": "A position within a document. The cursor is always inclusive of the current position, in both start and end positions.\n", "type": "object", "properties": { "modality": { "type": "string", "enum": [ "text" ] }, "item": { "type": "integer", "description": "Index of the item in the document" }, "position": { "type": "integer", "description": "The character position the cursor can be found at within the string." } } }, "Image": { "description": "Image data as base64-encoded bytes. Supported types are PNG, JPG, GIF, BMP, WEBP.", "type": "object", "properties": { "modality": { "type": "string", "enum": [ "image" ] }, "bytes": { "type": "string" } } }, "ImageCursor": { "description": "A position within a document. The cursor is always inclusive of the current position, in both start and end positions.\n", "type": "object", "properties": { "modality": { "type": "string", "enum": [ "image" ] }, "item": { "type": "integer", "description": "Index of the item in the document" } } } }, "parameters": { "SearchStoreID": { "in": "path", "name": "searchStoreID", "description": "The ID of the search store", "required": true, "schema": { "type": "string" }, "example": "7d78f98d-1c47-4c0b-8101-41b43b45e2cb" }, "DocumentName": { "in": "path", "name": "documentName", "description": "The name of the document", "required": true, "schema": { "type": "string" } }, "PageNumber": { "in": "query", "name": "page", "description": "The page number to retrieve.", "required": false, "schema": { "type": "integer", "minimum": 1, "default": 1, "example": 1 } }, "PageSize": { "in": "query", "name": "size", "description": "The number of items to include per page. A value between 1 and 100.", "required": false, "schema": { "type": "integer", "minimum": 1, "maximum": 100, "default": 50, "example": 10 } } } }, "x-tagGroups": [ { "name": "PhariaData API", "tags": [ "Repositories", "Datasets", "Stages", "Downloads", "Transformations", "Connectors" ] }, { "name": "Pharia Search API", "tags": [ "Search Store", "Document" ] } ], "servers": [ { "url": "{host}/v1/studio/data", "variables": { "host": { "default": "https://api.pharia.example.com" } } } ] }