{ "openapi": "3.0.3", "info": { "title": "Arroyo REST API", "description": "", "license": { "name": "MIT OR Apache-2.0" }, "version": "1.0.0" }, "servers": [ { "url": "/api/" } ], "paths": { "/v1/connection_profiles": { "get": { "tags": [ "connection_profiles" ], "summary": "List all connection profiles", "operationId": "get_connection_profiles", "responses": { "200": { "description": "Got connections collection", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ConnectionProfileCollection" } } } } } }, "post": { "tags": [ "connection_profiles" ], "summary": "Create connection profile", "operationId": "create_connection_profile", "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ConnectionProfilePost" } } }, "required": true }, "responses": { "200": { "description": "Created connection profile", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ConnectionProfile" } } } } } } }, "/v1/connection_profiles/test": { "post": { "tags": [ "connection_profiles" ], "summary": "Test connection profile", "operationId": "test_connection_profile", "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ConnectionProfilePost" } } }, "required": true }, "responses": { "200": { "description": "Result of testing connection profile", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/TestSourceMessage" } } } } } } }, "/v1/connection_profiles/{id}": { "delete": { "tags": [ "connection_profiles" ], "summary": "Delete a Connection Profile", "operationId": "delete_connection_profile", "parameters": [ { "name": "id", "in": "path", "description": "Connection Profile id", "required": true, "schema": { "type": "string" } } ], "responses": { "200": { "description": "Deleted connection profile" } } } }, "/v1/connection_profiles/{id}/autocomplete": { "get": { "tags": [ "connection_profiles" ], "summary": "Get autocomplete suggestions for a connection profile", "operationId": "get_connection_profile_autocomplete", "parameters": [ { "name": "id", "in": "path", "description": "Connection Profile id", "required": true, "schema": { "type": "string" } } ], "responses": { "200": { "description": "Autocomplete suggestions for connection profile", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ConnectionAutocompleteResp" } } } } } } }, "/v1/connection_tables": { "get": { "tags": [ "connection_tables" ], "summary": "List all connection tables", "operationId": "get_connection_tables", "parameters": [ { "name": "starting_after", "in": "query", "required": false, "schema": { "type": "string", "nullable": true } }, { "name": "limit", "in": "query", "required": false, "schema": { "type": "integer", "format": "int32", "nullable": true, "minimum": 0 } } ], "responses": { "200": { "description": "Got connection table collection", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ConnectionTableCollection" } } } } } }, "post": { "tags": [ "connection_tables" ], "summary": "Create a new connection table", "operationId": "create_connection_table", "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ConnectionTablePost" } } }, "required": true }, "responses": { "200": { "description": "Created connection table", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ConnectionTable" } } } } } } }, "/v1/connection_tables/schemas/test": { "post": { "tags": [ "connection_tables" ], "summary": "Test a Connection Schema", "operationId": "test_schema", "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ConnectionSchema" } } }, "required": true }, "responses": { "200": { "description": "Schema is valid" } } } }, "/v1/connection_tables/test": { "post": { "tags": [ "connection_tables" ], "summary": "Test a Connection Table", "operationId": "test_connection_table", "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ConnectionTablePost" } } }, "required": true }, "responses": { "200": { "description": "Job output as 'text/event-stream'" } } } }, "/v1/connection_tables/{id}": { "delete": { "tags": [ "connection_tables" ], "summary": "Delete a Connection Table", "operationId": "delete_connection_table", "parameters": [ { "name": "id", "in": "path", "description": "Connection Table id", "required": true, "schema": { "type": "string" } } ], "responses": { "200": { "description": "Deleted connection table" } } } }, "/v1/connectors": { "get": { "tags": [ "connectors" ], "summary": "List all connectors", "operationId": "get_connectors", "responses": { "200": { "description": "Got connectors collection", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ConnectorCollection" } } } } } } }, "/v1/jobs": { "get": { "tags": [ "jobs" ], "summary": "Get all jobs", "operationId": "get_jobs", "responses": { "200": { "description": "Get all jobs", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/JobCollection" } } } } } } }, "/v1/ping": { "get": { "tags": [ "ping" ], "summary": "Ping endpoint", "operationId": "ping", "responses": { "200": { "description": "Ping endpoint" } } } }, "/v1/pipelines": { "get": { "tags": [ "pipelines" ], "summary": "List all pipelines", "operationId": "get_pipelines", "parameters": [ { "name": "starting_after", "in": "query", "required": false, "schema": { "type": "string", "nullable": true } }, { "name": "limit", "in": "query", "required": false, "schema": { "type": "integer", "format": "int32", "nullable": true, "minimum": 0 } } ], "responses": { "200": { "description": "Got pipelines collection", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/PipelineCollection" } } } } } }, "post": { "tags": [ "pipelines" ], "summary": "Create a new pipeline", "description": "The API will create a single job for the pipeline.", "operationId": "create_pipeline", "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/PipelinePost" } } }, "required": true }, "responses": { "200": { "description": "Created pipeline and job", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Pipeline" } } } }, "400": { "description": "Bad request", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorResp" } } } } } } }, "/v1/pipelines/preview": { "post": { "tags": [ "pipelines" ], "summary": "Create a new preview pipeline", "operationId": "create_preview_pipeline", "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/PreviewPost" } } }, "required": true }, "responses": { "200": { "description": "Created pipeline and job", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Pipeline" } } } }, "400": { "description": "Bad request", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorResp" } } } } } } }, "/v1/pipelines/validate_query": { "post": { "tags": [ "pipelines" ], "summary": "Validate a query and return pipeline graph", "operationId": "validate_query", "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ValidateQueryPost" } } }, "required": true }, "responses": { "200": { "description": "Validated query", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/QueryValidationResult" } } } } } } }, "/v1/pipelines/{id}": { "get": { "tags": [ "pipelines" ], "summary": "Get a single pipeline", "operationId": "get_pipeline", "parameters": [ { "name": "id", "in": "path", "description": "Pipeline id", "required": true, "schema": { "type": "string" } } ], "responses": { "200": { "description": "Got pipeline", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Pipeline" } } } } } }, "delete": { "tags": [ "pipelines" ], "summary": "Delete a pipeline", "operationId": "delete_pipeline", "parameters": [ { "name": "id", "in": "path", "description": "Pipeline id", "required": true, "schema": { "type": "string" } } ], "responses": { "200": { "description": "Deleted pipeline" } } }, "patch": { "tags": [ "pipelines" ], "summary": "Update a pipeline", "operationId": "patch_pipeline", "parameters": [ { "name": "id", "in": "path", "description": "Pipeline id", "required": true, "schema": { "type": "string" } } ], "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/PipelinePatch" } } }, "required": true }, "responses": { "200": { "description": "Updated pipeline", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Pipeline" } } } } } } }, "/v1/pipelines/{id}/jobs": { "get": { "tags": [ "pipelines" ], "summary": "List a pipeline's jobs", "operationId": "get_pipeline_jobs", "parameters": [ { "name": "id", "in": "path", "description": "Pipeline id", "required": true, "schema": { "type": "string" } } ], "responses": { "200": { "description": "Got jobs collection", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/JobCollection" } } } } } } }, "/v1/pipelines/{id}/restart": { "post": { "tags": [ "pipelines" ], "summary": "Restart a pipeline", "operationId": "restart_pipeline", "parameters": [ { "name": "id", "in": "path", "description": "Pipeline id", "required": true, "schema": { "type": "string" } } ], "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/PipelineRestart" } } }, "required": true }, "responses": { "200": { "description": "Updated pipeline", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Pipeline" } } } } } } }, "/v1/pipelines/{pipeline_id}/jobs/{job_id}/checkpoints": { "get": { "tags": [ "jobs" ], "summary": "List a job's checkpoints", "operationId": "get_job_checkpoints", "parameters": [ { "name": "pipeline_id", "in": "path", "description": "Pipeline id", "required": true, "schema": { "type": "string" } }, { "name": "job_id", "in": "path", "description": "Job id", "required": true, "schema": { "type": "string" } } ], "responses": { "200": { "description": "Got job's checkpoints", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/CheckpointCollection" } } } } } } }, "/v1/pipelines/{pipeline_id}/jobs/{job_id}/checkpoints/{epoch}/operator_checkpoint_groups": { "get": { "tags": [ "jobs" ], "summary": "Get a checkpoint's details", "operationId": "get_checkpoint_details", "parameters": [ { "name": "pipeline_id", "in": "path", "description": "Pipeline id", "required": true, "schema": { "type": "string" } }, { "name": "job_id", "in": "path", "description": "Job id", "required": true, "schema": { "type": "string" } }, { "name": "epoch", "in": "path", "description": "Epoch", "required": true, "schema": { "type": "integer", "format": "int32", "minimum": 0 } } ], "responses": { "200": { "description": "Got checkpoint's details", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/OperatorCheckpointGroupCollection" } } } } } } }, "/v1/pipelines/{pipeline_id}/jobs/{job_id}/errors": { "get": { "tags": [ "jobs" ], "summary": "List a job's error messages", "operationId": "get_job_errors", "parameters": [ { "name": "pipeline_id", "in": "path", "description": "Pipeline id", "required": true, "schema": { "type": "string" } }, { "name": "job_id", "in": "path", "description": "Job id", "required": true, "schema": { "type": "string" } }, { "name": "starting_after", "in": "query", "description": "Starting after", "required": false, "schema": { "type": "string", "nullable": true } }, { "name": "limit", "in": "query", "description": "Limit", "required": false, "schema": { "type": "integer", "format": "int32", "nullable": true, "minimum": 0 } } ], "responses": { "200": { "description": "Got job's error messages", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/JobLogMessageCollection" } } } } } } }, "/v1/pipelines/{pipeline_id}/jobs/{job_id}/operator_metric_groups": { "get": { "tags": [ "jobs" ], "summary": "Get a job's metrics", "operationId": "get_operator_metric_groups", "parameters": [ { "name": "pipeline_id", "in": "path", "description": "Pipeline id", "required": true, "schema": { "type": "string" } }, { "name": "job_id", "in": "path", "description": "Job id", "required": true, "schema": { "type": "string" } } ], "responses": { "200": { "description": "Got metric groups", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/OperatorMetricGroupCollection" } } } } } } }, "/v1/pipelines/{pipeline_id}/jobs/{job_id}/output": { "get": { "tags": [ "jobs" ], "summary": "Subscribe to a job's output", "operationId": "get_job_output", "parameters": [ { "name": "pipeline_id", "in": "path", "description": "Pipeline id", "required": true, "schema": { "type": "string" } }, { "name": "job_id", "in": "path", "description": "Job id", "required": true, "schema": { "type": "string" } } ], "responses": { "200": { "description": "Job output as 'text/event-stream'" } } } }, "/v1/udfs": { "get": { "tags": [ "udfs" ], "summary": "Get Global UDFs", "operationId": "get_udfs", "responses": { "200": { "description": "List of UDFs", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/GlobalUdfCollection" } } } } } }, "post": { "tags": [ "udfs" ], "summary": "Create a global UDF", "operationId": "create_udf", "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/UdfPost" } } }, "required": true }, "responses": { "200": { "description": "Created UDF", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Udf" } } } } } } }, "/v1/udfs/validate": { "post": { "tags": [ "udfs" ], "summary": "Validate UDFs", "operationId": "validate_udf", "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ValidateUdfPost" } } }, "required": true }, "responses": { "200": { "description": "Validated query", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/UdfValidationResult" } } } } } } }, "/v1/udfs/{id}": { "delete": { "tags": [ "udfs" ], "summary": "Delete UDF", "operationId": "delete_udf", "parameters": [ { "name": "id", "in": "path", "description": "UDF id", "required": true, "schema": { "type": "string" } } ], "responses": { "200": { "description": "Deleted UDF" } } } } }, "components": { "schemas": { "AvroFormat": { "type": "object", "properties": { "confluent_schema_registry": { "type": "boolean" }, "into_unstructured_json": { "type": "boolean" }, "raw_datums": { "type": "boolean" }, "reader_schema": { "type": "string", "readOnly": true }, "schema_id": { "type": "integer", "format": "int32", "readOnly": true, "nullable": true, "minimum": 0 } } }, "BadData": { "oneOf": [ { "type": "object", "title": "Fail", "required": [ "behavior" ], "properties": { "behavior": { "type": "string", "enum": [ "fail" ] } } }, { "type": "object", "title": "Drop", "required": [ "behavior" ], "properties": { "behavior": { "type": "string", "enum": [ "drop" ] } } } ], "discriminator": { "propertyName": "behavior", "mapping": { "fail": "#/components/schemas/BadData_Fail", "drop": "#/components/schemas/BadData_Drop" } } }, "BadData_Drop": { "type": "object", "title": "Drop", "required": [ "behavior" ], "properties": { "behavior": { "type": "string", "enum": [ "drop" ] } } }, "BadData_Fail": { "type": "object", "title": "Fail", "required": [ "behavior" ], "properties": { "behavior": { "type": "string", "enum": [ "fail" ] } } }, "Checkpoint": { "type": "object", "required": [ "epoch", "backend", "start_time", "events" ], "properties": { "backend": { "type": "string" }, "epoch": { "type": "integer", "format": "int32", "minimum": 0 }, "events": { "type": "array", "items": { "$ref": "#/components/schemas/CheckpointEventSpan" } }, "finish_time": { "type": "integer", "format": "int64", "nullable": true, "minimum": 0 }, "start_time": { "type": "integer", "format": "int64", "minimum": 0 } } }, "CheckpointCollection": { "type": "object", "required": [ "data" ], "properties": { "data": { "type": "array", "items": { "$ref": "#/components/schemas/Checkpoint" } } } }, "CheckpointEventSpan": { "type": "object", "required": [ "start_time", "finish_time", "event", "description" ], "properties": { "description": { "type": "string" }, "event": { "type": "string" }, "finish_time": { "type": "integer", "format": "int64", "minimum": 0 }, "start_time": { "type": "integer", "format": "int64", "minimum": 0 } } }, "ConnectionAutocompleteResp": { "type": "object", "required": [ "values" ], "properties": { "values": { "type": "object", "additionalProperties": { "type": "array", "items": { "type": "string" } } } } }, "ConnectionProfile": { "type": "object", "required": [ "id", "name", "connector", "config", "description" ], "properties": { "config": {}, "connector": { "type": "string" }, "description": { "type": "string" }, "id": { "type": "string" }, "name": { "type": "string" } } }, "ConnectionProfileCollection": { "type": "object", "required": [ "data" ], "properties": { "data": { "type": "array", "items": { "$ref": "#/components/schemas/ConnectionProfile" } } } }, "ConnectionProfilePost": { "type": "object", "required": [ "name", "connector", "config" ], "properties": { "config": {}, "connector": { "type": "string" }, "name": { "type": "string" } } }, "ConnectionSchema": { "type": "object", "properties": { "bad_data": { "$ref": "#/components/schemas/BadData", "nullable": true }, "definition": { "$ref": "#/components/schemas/SchemaDefinition", "nullable": true }, "fields": { "type": "array", "items": { "$ref": "#/components/schemas/SourceField" } }, "format": { "$ref": "#/components/schemas/Format", "nullable": true }, "framing": { "$ref": "#/components/schemas/Framing", "nullable": true }, "inferred": { "type": "boolean", "nullable": true }, "primary_keys": { "type": "array", "items": { "type": "string" }, "uniqueItems": true } } }, "ConnectionTable": { "type": "object", "required": [ "id", "name", "created_at", "connector", "table_type", "config", "schema", "consumers" ], "properties": { "config": {}, "connection_profile": { "$ref": "#/components/schemas/ConnectionProfile", "nullable": true }, "connector": { "type": "string" }, "consumers": { "type": "integer", "format": "int32", "minimum": 0 }, "created_at": { "type": "integer", "format": "int64", "minimum": 0 }, "id": { "type": "string" }, "name": { "type": "string" }, "schema": { "$ref": "#/components/schemas/ConnectionSchema" }, "table_type": { "$ref": "#/components/schemas/ConnectionType" } } }, "ConnectionTableCollection": { "type": "object", "required": [ "data", "hasMore" ], "properties": { "data": { "type": "array", "items": { "$ref": "#/components/schemas/ConnectionTable" } }, "hasMore": { "type": "boolean" } } }, "ConnectionTablePost": { "type": "object", "required": [ "name", "connector", "config" ], "properties": { "config": {}, "connection_profile_id": { "type": "string", "nullable": true }, "connector": { "type": "string" }, "name": { "type": "string" }, "schema": { "$ref": "#/components/schemas/ConnectionSchema", "nullable": true } } }, "ConnectionType": { "type": "string", "enum": [ "source", "sink", "lookup" ] }, "Connector": { "type": "object", "required": [ "id", "name", "icon", "description", "table_config", "enabled", "source", "sink", "custom_schemas", "testing", "hidden" ], "properties": { "connection_config": { "type": "string", "nullable": true }, "custom_schemas": { "type": "boolean" }, "description": { "type": "string" }, "enabled": { "type": "boolean" }, "hidden": { "type": "boolean" }, "icon": { "type": "string" }, "id": { "type": "string" }, "name": { "type": "string" }, "sink": { "type": "boolean" }, "source": { "type": "boolean" }, "table_config": { "type": "string" }, "testing": { "type": "boolean" } } }, "ConnectorCollection": { "type": "object", "required": [ "data" ], "properties": { "data": { "type": "array", "items": { "$ref": "#/components/schemas/Connector" } } } }, "DecimalEncoding": { "type": "string", "enum": [ "number", "string", "bytes" ] }, "DecimalField": { "type": "object", "required": [ "precision", "scale" ], "properties": { "precision": { "type": "integer", "format": "int32", "minimum": 0 }, "scale": { "type": "integer", "format": "int32" } } }, "ErrorDomain": { "type": "string", "enum": [ "user", "external", "internal" ] }, "ErrorResp": { "type": "object", "required": [ "error" ], "properties": { "error": { "type": "string" } } }, "FailureReason": { "type": "object", "required": [ "error", "domain" ], "properties": { "domain": { "$ref": "#/components/schemas/ErrorDomain" }, "error": { "type": "string" } } }, "FieldType": { "oneOf": [ { "type": "object", "title": "Int32", "required": [ "type" ], "properties": { "type": { "type": "string", "enum": [ "int32" ] } } }, { "type": "object", "title": "Int64", "required": [ "type" ], "properties": { "type": { "type": "string", "enum": [ "int64" ] } } }, { "type": "object", "title": "UInt32", "required": [ "type" ], "properties": { "type": { "type": "string", "enum": [ "uint32" ] } } }, { "type": "object", "title": "UInt64", "required": [ "type" ], "properties": { "type": { "type": "string", "enum": [ "uint64" ] } } }, { "type": "object", "title": "Float32", "required": [ "type" ], "properties": { "type": { "type": "string", "enum": [ "float32" ] } } }, { "type": "object", "title": "Float64", "required": [ "type" ], "properties": { "type": { "type": "string", "enum": [ "float64" ] } } }, { "type": "object", "required": [ "precision", "scale", "type" ], "properties": { "precision": { "type": "integer", "format": "int32", "minimum": 0 }, "scale": { "type": "integer", "format": "int32" }, "type": { "type": "string", "enum": [ "decimal128" ] } }, "title": "Decimal128" }, { "type": "object", "title": "Bool", "required": [ "type" ], "properties": { "type": { "type": "string", "enum": [ "bool" ] } } }, { "type": "object", "title": "String", "required": [ "type" ], "properties": { "type": { "type": "string", "enum": [ "string" ] } } }, { "type": "object", "title": "Bytes", "required": [ "type" ], "properties": { "type": { "type": "string", "enum": [ "bytes" ] } } }, { "type": "object", "properties": { "unit": { "$ref": "#/components/schemas/TimestampUnit" }, "type": { "type": "string", "enum": [ "timestamp" ] } }, "required": [ "type" ], "title": "Timestamp" }, { "type": "object", "title": "Json", "required": [ "type" ], "properties": { "type": { "type": "string", "enum": [ "json" ] } } }, { "type": "object", "required": [ "fields", "type" ], "properties": { "fields": { "type": "array", "items": { "$ref": "#/components/schemas/SourceField" } }, "type": { "type": "string", "enum": [ "struct" ] } }, "title": "Struct" }, { "type": "object", "required": [ "items", "type" ], "properties": { "items": { "$ref": "#/components/schemas/ListFieldItem" }, "type": { "type": "string", "enum": [ "list" ] } }, "title": "List" } ], "discriminator": { "propertyName": "type", "mapping": { "int32": "#/components/schemas/FieldType_Int32", "int64": "#/components/schemas/FieldType_Int64", "uint32": "#/components/schemas/FieldType_UInt32", "uint64": "#/components/schemas/FieldType_UInt64", "float32": "#/components/schemas/FieldType_Float32", "float64": "#/components/schemas/FieldType_Float64", "decimal128": "#/components/schemas/FieldType_Decimal128", "bool": "#/components/schemas/FieldType_Bool", "string": "#/components/schemas/FieldType_String", "bytes": "#/components/schemas/FieldType_Bytes", "timestamp": "#/components/schemas/FieldType_Timestamp", "json": "#/components/schemas/FieldType_Json", "struct": "#/components/schemas/FieldType_Struct", "list": "#/components/schemas/FieldType_List" } } }, "FieldType_Bool": { "type": "object", "title": "Bool", "required": [ "type" ], "properties": { "type": { "type": "string", "enum": [ "bool" ] } } }, "FieldType_Bytes": { "type": "object", "title": "Bytes", "required": [ "type" ], "properties": { "type": { "type": "string", "enum": [ "bytes" ] } } }, "FieldType_Decimal128": { "type": "object", "required": [ "precision", "scale", "type" ], "properties": { "precision": { "type": "integer", "format": "int32", "minimum": 0 }, "scale": { "type": "integer", "format": "int32" }, "type": { "type": "string", "enum": [ "decimal128" ] } }, "title": "Decimal128" }, "FieldType_Float32": { "type": "object", "title": "Float32", "required": [ "type" ], "properties": { "type": { "type": "string", "enum": [ "float32" ] } } }, "FieldType_Float64": { "type": "object", "title": "Float64", "required": [ "type" ], "properties": { "type": { "type": "string", "enum": [ "float64" ] } } }, "FieldType_Int32": { "type": "object", "title": "Int32", "required": [ "type" ], "properties": { "type": { "type": "string", "enum": [ "int32" ] } } }, "FieldType_Int64": { "type": "object", "title": "Int64", "required": [ "type" ], "properties": { "type": { "type": "string", "enum": [ "int64" ] } } }, "FieldType_Json": { "type": "object", "title": "Json", "required": [ "type" ], "properties": { "type": { "type": "string", "enum": [ "json" ] } } }, "FieldType_List": { "type": "object", "required": [ "items", "type" ], "properties": { "items": { "$ref": "#/components/schemas/ListFieldItem" }, "type": { "type": "string", "enum": [ "list" ] } }, "title": "List" }, "FieldType_String": { "type": "object", "title": "String", "required": [ "type" ], "properties": { "type": { "type": "string", "enum": [ "string" ] } } }, "FieldType_Struct": { "type": "object", "required": [ "fields", "type" ], "properties": { "fields": { "type": "array", "items": { "$ref": "#/components/schemas/SourceField" } }, "type": { "type": "string", "enum": [ "struct" ] } }, "title": "Struct" }, "FieldType_Timestamp": { "type": "object", "properties": { "unit": { "$ref": "#/components/schemas/TimestampUnit" }, "type": { "type": "string", "enum": [ "timestamp" ] } }, "required": [ "type" ], "title": "Timestamp" }, "FieldType_UInt32": { "type": "object", "title": "UInt32", "required": [ "type" ], "properties": { "type": { "type": "string", "enum": [ "uint32" ] } } }, "FieldType_UInt64": { "type": "object", "title": "UInt64", "required": [ "type" ], "properties": { "type": { "type": "string", "enum": [ "uint64" ] } } }, "Format": { "oneOf": [ { "type": "object", "properties": { "confluent_schema_registry": { "type": "boolean" }, "debezium": { "type": "boolean" }, "decimal_encoding": { "$ref": "#/components/schemas/DecimalEncoding" }, "include_schema": { "type": "boolean" }, "schema_id": { "type": "integer", "format": "int32", "nullable": true, "minimum": 0 }, "timestamp_format": { "$ref": "#/components/schemas/TimestampFormat" }, "unstructured": { "type": "boolean" }, "type": { "type": "string", "enum": [ "json" ] } }, "required": [ "type" ], "title": "Json" }, { "type": "object", "properties": { "confluent_schema_registry": { "type": "boolean" }, "into_unstructured_json": { "type": "boolean" }, "raw_datums": { "type": "boolean" }, "reader_schema": { "type": "string", "readOnly": true }, "schema_id": { "type": "integer", "format": "int32", "readOnly": true, "nullable": true, "minimum": 0 }, "type": { "type": "string", "enum": [ "avro" ] } }, "required": [ "type" ], "title": "Avro" }, { "type": "object", "properties": { "compiled_schema": { "type": "string", "format": "binary", "nullable": true }, "confluent_schema_registry": { "type": "boolean" }, "into_unstructured_json": { "type": "boolean" }, "length_delimited": { "type": "boolean" }, "message_name": { "type": "string", "nullable": true }, "type": { "type": "string", "enum": [ "protobuf" ] } }, "required": [ "type" ], "title": "Protobuf" }, { "type": "object", "properties": { "compression": { "$ref": "#/components/schemas/ParquetCompression" }, "row_group_bytes": { "type": "integer", "format": "int64", "nullable": true, "minimum": 0 }, "type": { "type": "string", "enum": [ "parquet" ] } }, "required": [ "type" ], "title": "Parquet" }, { "type": "object", "required": [ "type" ], "properties": { "type": { "type": "string", "enum": [ "raw_string" ] } }, "title": "RawString" }, { "type": "object", "required": [ "type" ], "properties": { "type": { "type": "string", "enum": [ "raw_bytes" ] } }, "title": "RawBytes" } ], "discriminator": { "propertyName": "type", "mapping": { "json": "#/components/schemas/Format_Json", "avro": "#/components/schemas/Format_Avro", "protobuf": "#/components/schemas/Format_Protobuf", "parquet": "#/components/schemas/Format_Parquet", "raw_string": "#/components/schemas/Format_RawString", "raw_bytes": "#/components/schemas/Format_RawBytes" } } }, "Format_Avro": { "type": "object", "properties": { "confluent_schema_registry": { "type": "boolean" }, "into_unstructured_json": { "type": "boolean" }, "raw_datums": { "type": "boolean" }, "reader_schema": { "type": "string", "readOnly": true }, "schema_id": { "type": "integer", "format": "int32", "readOnly": true, "nullable": true, "minimum": 0 }, "type": { "type": "string", "enum": [ "avro" ] } }, "required": [ "type" ], "title": "Avro" }, "Format_Json": { "type": "object", "properties": { "confluent_schema_registry": { "type": "boolean" }, "debezium": { "type": "boolean" }, "decimal_encoding": { "$ref": "#/components/schemas/DecimalEncoding" }, "include_schema": { "type": "boolean" }, "schema_id": { "type": "integer", "format": "int32", "nullable": true, "minimum": 0 }, "timestamp_format": { "$ref": "#/components/schemas/TimestampFormat" }, "unstructured": { "type": "boolean" }, "type": { "type": "string", "enum": [ "json" ] } }, "required": [ "type" ], "title": "Json" }, "Format_Parquet": { "type": "object", "properties": { "compression": { "$ref": "#/components/schemas/ParquetCompression" }, "row_group_bytes": { "type": "integer", "format": "int64", "nullable": true, "minimum": 0 }, "type": { "type": "string", "enum": [ "parquet" ] } }, "required": [ "type" ], "title": "Parquet" }, "Format_Protobuf": { "type": "object", "properties": { "compiled_schema": { "type": "string", "format": "binary", "nullable": true }, "confluent_schema_registry": { "type": "boolean" }, "into_unstructured_json": { "type": "boolean" }, "length_delimited": { "type": "boolean" }, "message_name": { "type": "string", "nullable": true }, "type": { "type": "string", "enum": [ "protobuf" ] } }, "required": [ "type" ], "title": "Protobuf" }, "Format_RawBytes": { "type": "object", "required": [ "type" ], "properties": { "type": { "type": "string", "enum": [ "raw_bytes" ] } }, "title": "RawBytes" }, "Format_RawString": { "type": "object", "required": [ "type" ], "properties": { "type": { "type": "string", "enum": [ "raw_string" ] } }, "title": "RawString" }, "Framing": { "oneOf": [ { "type": "object", "properties": { "max_line_length": { "type": "integer", "format": "int64", "nullable": true, "minimum": 0 }, "method": { "type": "string", "enum": [ "newline" ] } }, "required": [ "method" ], "title": "Newline" } ], "discriminator": { "propertyName": "method", "mapping": { "newline": "#/components/schemas/Framing_Newline" } } }, "Framing_Newline": { "type": "object", "properties": { "max_line_length": { "type": "integer", "format": "int64", "nullable": true, "minimum": 0 }, "method": { "type": "string", "enum": [ "newline" ] } }, "required": [ "method" ], "title": "Newline" }, "GlobalUdf": { "type": "object", "required": [ "id", "prefix", "name", "language", "created_at", "updated_at", "definition" ], "properties": { "created_at": { "type": "integer", "format": "int64", "minimum": 0 }, "definition": { "type": "string" }, "description": { "type": "string", "nullable": true }, "dylib_url": { "type": "string", "nullable": true }, "id": { "type": "string" }, "language": { "$ref": "#/components/schemas/UdfLanguage" }, "name": { "type": "string" }, "prefix": { "type": "string" }, "updated_at": { "type": "integer", "format": "int64", "minimum": 0 } } }, "GlobalUdfCollection": { "type": "object", "required": [ "data" ], "properties": { "data": { "type": "array", "items": { "$ref": "#/components/schemas/GlobalUdf" } } } }, "Job": { "type": "object", "required": [ "id", "running_desired", "state", "run_id", "created_at" ], "properties": { "created_at": { "type": "integer", "format": "int64", "minimum": 0 }, "failure_reason": { "$ref": "#/components/schemas/FailureReason", "nullable": true }, "finish_time": { "type": "integer", "format": "int64", "nullable": true, "minimum": 0 }, "id": { "type": "string" }, "run_id": { "type": "integer", "format": "int64", "minimum": 0 }, "running_desired": { "type": "boolean" }, "start_time": { "type": "integer", "format": "int64", "nullable": true, "minimum": 0 }, "state": { "type": "string" }, "tasks": { "type": "integer", "format": "int64", "nullable": true, "minimum": 0 } } }, "JobCollection": { "type": "object", "required": [ "data" ], "properties": { "data": { "type": "array", "items": { "$ref": "#/components/schemas/Job" } } } }, "JobLogLevel": { "type": "string", "enum": [ "info", "warn", "error" ] }, "JobLogMessage": { "type": "object", "required": [ "id", "created_at", "level", "message", "details" ], "properties": { "created_at": { "type": "integer", "format": "int64", "minimum": 0 }, "details": { "type": "string" }, "error_domain": { "$ref": "#/components/schemas/ErrorDomain", "nullable": true }, "id": { "type": "string" }, "level": { "$ref": "#/components/schemas/JobLogLevel" }, "message": { "type": "string" }, "operator_id": { "type": "string", "nullable": true }, "task_index": { "type": "integer", "format": "int64", "nullable": true, "minimum": 0 } } }, "JobLogMessageCollection": { "type": "object", "required": [ "data", "hasMore" ], "properties": { "data": { "type": "array", "items": { "$ref": "#/components/schemas/JobLogMessage" } }, "hasMore": { "type": "boolean" } } }, "JsonFormat": { "type": "object", "properties": { "confluent_schema_registry": { "type": "boolean" }, "debezium": { "type": "boolean" }, "decimal_encoding": { "$ref": "#/components/schemas/DecimalEncoding" }, "include_schema": { "type": "boolean" }, "schema_id": { "type": "integer", "format": "int32", "nullable": true, "minimum": 0 }, "timestamp_format": { "$ref": "#/components/schemas/TimestampFormat" }, "unstructured": { "type": "boolean" } } }, "ListField": { "type": "object", "required": [ "items" ], "properties": { "items": { "$ref": "#/components/schemas/ListFieldItem" } } }, "ListFieldItem": { "allOf": [ { "$ref": "#/components/schemas/FieldType" }, { "type": "object", "properties": { "name": { "type": "string" }, "required": { "type": "boolean" }, "sql_name": { "type": "string", "readOnly": true, "nullable": true } } } ] }, "Metric": { "type": "object", "required": [ "time", "value" ], "properties": { "time": { "type": "integer", "format": "int64", "minimum": 0 }, "value": { "type": "number", "format": "double" } } }, "MetricGroup": { "type": "object", "required": [ "name", "subtasks" ], "properties": { "name": { "$ref": "#/components/schemas/MetricName" }, "subtasks": { "type": "array", "items": { "$ref": "#/components/schemas/SubtaskMetrics" } } } }, "MetricName": { "type": "string", "enum": [ "bytes_recv", "bytes_sent", "messages_recv", "messages_sent", "backpressure", "tx_queue_size", "tx_queue_rem" ] }, "NewlineDelimitedFraming": { "type": "object", "properties": { "max_line_length": { "type": "integer", "format": "int64", "nullable": true, "minimum": 0 } } }, "OperatorCheckpointGroup": { "type": "object", "required": [ "operator_id", "bytes", "subtasks" ], "properties": { "bytes": { "type": "integer", "format": "int64", "minimum": 0 }, "finish_time": { "type": "integer", "format": "int64", "nullable": true, "minimum": 0 }, "operator_id": { "type": "string" }, "started_metadata_write": { "type": "integer", "format": "int64", "nullable": true, "minimum": 0 }, "subtasks": { "type": "array", "items": { "$ref": "#/components/schemas/SubtaskCheckpointGroup" } } } }, "OperatorCheckpointGroupCollection": { "type": "object", "required": [ "data" ], "properties": { "data": { "type": "array", "items": { "$ref": "#/components/schemas/OperatorCheckpointGroup" } } } }, "OperatorMetricGroup": { "type": "object", "required": [ "node_id", "metric_groups" ], "properties": { "metric_groups": { "type": "array", "items": { "$ref": "#/components/schemas/MetricGroup" } }, "node_id": { "type": "integer", "format": "int32", "minimum": 0 } } }, "OperatorMetricGroupCollection": { "type": "object", "required": [ "data" ], "properties": { "data": { "type": "array", "items": { "$ref": "#/components/schemas/OperatorMetricGroup" } } } }, "OutputData": { "type": "object", "required": [ "operator_id", "subtask_idx", "timestamps", "start_id", "batch" ], "properties": { "batch": { "type": "string" }, "operator_id": { "type": "string" }, "start_id": { "type": "integer", "format": "int64", "minimum": 0 }, "subtask_idx": { "type": "integer", "format": "int32", "minimum": 0 }, "timestamps": { "type": "array", "items": { "type": "integer", "format": "int64", "minimum": 0 } } } }, "PaginationQueryParams": { "type": "object", "properties": { "limit": { "type": "integer", "format": "int32", "nullable": true, "minimum": 0 }, "starting_after": { "type": "string", "nullable": true } } }, "ParquetCompression": { "type": "string", "enum": [ "uncompressed", "snappy", "gzip", "zstd", "lz4", "lz4_raw" ] }, "ParquetFormat": { "type": "object", "properties": { "compression": { "$ref": "#/components/schemas/ParquetCompression" }, "row_group_bytes": { "type": "integer", "format": "int64", "nullable": true, "minimum": 0 } } }, "Pipeline": { "type": "object", "required": [ "id", "name", "query", "udfs", "checkpoint_interval_micros", "stop", "created_at", "action_text", "action_in_progress", "graph", "preview" ], "properties": { "action": { "$ref": "#/components/schemas/StopType", "nullable": true }, "action_in_progress": { "type": "boolean" }, "action_text": { "type": "string" }, "checkpoint_interval_micros": { "type": "integer", "format": "int64", "minimum": 0 }, "created_at": { "type": "integer", "format": "int64", "minimum": 0 }, "graph": { "$ref": "#/components/schemas/PipelineGraph" }, "id": { "type": "string" }, "name": { "type": "string" }, "preview": { "type": "boolean" }, "query": { "type": "string" }, "stop": { "$ref": "#/components/schemas/StopType" }, "udfs": { "type": "array", "items": { "$ref": "#/components/schemas/Udf" } } } }, "PipelineCollection": { "type": "object", "required": [ "data", "hasMore" ], "properties": { "data": { "type": "array", "items": { "$ref": "#/components/schemas/Pipeline" } }, "hasMore": { "type": "boolean" } } }, "PipelineEdge": { "type": "object", "required": [ "src_id", "dest_id", "key_type", "value_type", "edge_type" ], "properties": { "dest_id": { "type": "integer", "format": "int32", "minimum": 0 }, "edge_type": { "type": "string" }, "key_type": { "type": "string" }, "src_id": { "type": "integer", "format": "int32", "minimum": 0 }, "value_type": { "type": "string" } } }, "PipelineGraph": { "type": "object", "required": [ "nodes", "edges" ], "properties": { "edges": { "type": "array", "items": { "$ref": "#/components/schemas/PipelineEdge" } }, "nodes": { "type": "array", "items": { "$ref": "#/components/schemas/PipelineNode" } } } }, "PipelineNode": { "type": "object", "required": [ "node_id", "operator", "description", "parallelism" ], "properties": { "description": { "type": "string" }, "node_id": { "type": "integer", "format": "int32", "minimum": 0 }, "operator": { "type": "string" }, "parallelism": { "type": "integer", "format": "int32", "minimum": 0 } } }, "PipelinePatch": { "type": "object", "properties": { "checkpoint_interval_micros": { "type": "integer", "format": "int64", "nullable": true, "minimum": 0 }, "parallelism": { "type": "integer", "format": "int64", "nullable": true, "minimum": 0 }, "stop": { "$ref": "#/components/schemas/StopType", "nullable": true } } }, "PipelinePost": { "type": "object", "required": [ "name", "query", "parallelism" ], "properties": { "checkpoint_interval_micros": { "type": "integer", "format": "int64", "nullable": true, "minimum": 0 }, "name": { "type": "string" }, "parallelism": { "type": "integer", "format": "int64", "minimum": 0 }, "query": { "type": "string" }, "udfs": { "type": "array", "items": { "$ref": "#/components/schemas/Udf" }, "nullable": true } } }, "PipelineRestart": { "type": "object", "properties": { "force": { "type": "boolean", "nullable": true } } }, "PreviewPost": { "type": "object", "required": [ "query" ], "properties": { "enable_sinks": { "type": "boolean" }, "query": { "type": "string" }, "udfs": { "type": "array", "items": { "$ref": "#/components/schemas/Udf" }, "nullable": true } } }, "ProtobufFormat": { "type": "object", "properties": { "compiled_schema": { "type": "string", "format": "binary", "nullable": true }, "confluent_schema_registry": { "type": "boolean" }, "into_unstructured_json": { "type": "boolean" }, "length_delimited": { "type": "boolean" }, "message_name": { "type": "string", "nullable": true } } }, "QueryValidationResult": { "type": "object", "required": [ "errors" ], "properties": { "errors": { "type": "array", "items": { "type": "string" } }, "graph": { "$ref": "#/components/schemas/PipelineGraph", "nullable": true } } }, "RawBytesFormat": { "type": "object" }, "RawStringFormat": { "type": "object" }, "SchemaDefinition": { "oneOf": [ { "type": "object", "title": "JsonSchema", "required": [ "schema", "type" ], "properties": { "schema": { "type": "string" }, "type": { "type": "string", "enum": [ "json_schema" ] } } }, { "type": "object", "title": "Protobuf", "required": [ "schema", "type" ], "properties": { "dependencies": { "type": "object", "additionalProperties": { "type": "string" } }, "schema": { "type": "string" }, "type": { "type": "string", "enum": [ "protobuf_schema" ] } } }, { "type": "object", "title": "Avro", "required": [ "schema", "type" ], "properties": { "schema": { "type": "string" }, "type": { "type": "string", "enum": [ "avro_schema" ] } } } ], "discriminator": { "propertyName": "type", "mapping": { "json_schema": "#/components/schemas/SchemaDefinition_JsonSchema", "protobuf_schema": "#/components/schemas/SchemaDefinition_Protobuf", "avro_schema": "#/components/schemas/SchemaDefinition_Avro" } } }, "SchemaDefinition_Avro": { "type": "object", "title": "Avro", "required": [ "schema", "type" ], "properties": { "schema": { "type": "string" }, "type": { "type": "string", "enum": [ "avro_schema" ] } } }, "SchemaDefinition_JsonSchema": { "type": "object", "title": "JsonSchema", "required": [ "schema", "type" ], "properties": { "schema": { "type": "string" }, "type": { "type": "string", "enum": [ "json_schema" ] } } }, "SchemaDefinition_Protobuf": { "type": "object", "title": "Protobuf", "required": [ "schema", "type" ], "properties": { "dependencies": { "type": "object", "additionalProperties": { "type": "string" } }, "schema": { "type": "string" }, "type": { "type": "string", "enum": [ "protobuf_schema" ] } } }, "SourceField": { "allOf": [ { "$ref": "#/components/schemas/FieldType" }, { "type": "object", "required": [ "name" ], "properties": { "metadata_key": { "type": "string", "nullable": true }, "name": { "type": "string" }, "required": { "type": "boolean" }, "sql_name": { "type": "string", "readOnly": true, "nullable": true } } } ] }, "StopType": { "type": "string", "enum": [ "none", "checkpoint", "graceful", "immediate", "force" ] }, "StructField": { "type": "object", "required": [ "fields" ], "properties": { "fields": { "type": "array", "items": { "$ref": "#/components/schemas/SourceField" } } } }, "SubtaskCheckpointGroup": { "type": "object", "required": [ "index", "bytes", "event_spans" ], "properties": { "bytes": { "type": "integer", "format": "int64", "minimum": 0 }, "event_spans": { "type": "array", "items": { "$ref": "#/components/schemas/CheckpointEventSpan" } }, "index": { "type": "integer", "format": "int32", "minimum": 0 } } }, "SubtaskMetrics": { "type": "object", "required": [ "index", "metrics" ], "properties": { "index": { "type": "integer", "format": "int32", "minimum": 0 }, "metrics": { "type": "array", "items": { "$ref": "#/components/schemas/Metric" } } } }, "TestSourceMessage": { "type": "object", "required": [ "error", "done", "message" ], "properties": { "done": { "type": "boolean" }, "error": { "type": "boolean" }, "message": { "type": "string" } } }, "TimestampField": { "type": "object", "properties": { "unit": { "$ref": "#/components/schemas/TimestampUnit" } } }, "TimestampFormat": { "type": "string", "enum": [ "rfc3339", "unix_millis" ] }, "TimestampUnit": { "type": "string", "enum": [ "second", "millisecond", "microsecond", "nanosecond" ] }, "Udf": { "type": "object", "required": [ "definition" ], "properties": { "definition": { "type": "string" }, "language": { "$ref": "#/components/schemas/UdfLanguage" } } }, "UdfLanguage": { "type": "string", "enum": [ "python", "rust" ] }, "UdfPost": { "type": "object", "required": [ "prefix", "definition" ], "properties": { "definition": { "type": "string" }, "description": { "type": "string", "nullable": true }, "language": { "$ref": "#/components/schemas/UdfLanguage" }, "prefix": { "type": "string" } } }, "UdfValidationResult": { "type": "object", "required": [ "errors" ], "properties": { "errors": { "type": "array", "items": { "type": "string" } }, "udf_name": { "type": "string", "nullable": true } } }, "ValidateQueryPost": { "type": "object", "required": [ "query" ], "properties": { "query": { "type": "string" }, "udfs": { "type": "array", "items": { "$ref": "#/components/schemas/Udf" }, "nullable": true } } }, "ValidateUdfPost": { "type": "object", "required": [ "definition" ], "properties": { "definition": { "type": "string" }, "language": { "$ref": "#/components/schemas/UdfLanguage" } } } } }, "tags": [ { "name": "ping", "description": "Ping endpoint" }, { "name": "connection_profiles", "description": "Connection profiles management endpoints" }, { "name": "connection_tables", "description": "Connection tables management endpoints" }, { "name": "pipelines", "description": "Pipeline management endpoints" }, { "name": "jobs", "description": "Job management endpoints" }, { "name": "connectors", "description": "Connector management endpoints" } ] }