{ "openapi": "3.1.0", "info": { "title": "TT Media Server API (C++ Drogon)", "description": "High-performance C++ implementation of the TT Media Server using Drogon framework. Provides OpenAI-compatible chat completions API for benchmarking server overhead.", "version": "1.0.0", "contact": { "name": "Tenstorrent", "url": "https://tenstorrent.com" }, "license": { "name": "Apache-2.0", "url": "https://www.apache.org/licenses/LICENSE-2.0" } }, "servers": [ { "url": "/", "description": "Local server" } ], "tags": [ { "name": "Chat Completions", "description": "OpenAI-compatible chat completion endpoints" }, { "name": "Embeddings", "description": "OpenAI-compatible text embedding endpoints" }, { "name": "Images", "description": "OpenAI-compatible image generation, image-to-image, and edit endpoints. Only the route matching the active runner_type is registered at runtime." }, { "name": "Text to speech processing", "description": "Text-to-speech audio generation endpoints. Only registered when MODEL_SERVICE=tts." }, { "name": "Sessions", "description": "Session management for slot assignments" }, { "name": "Models", "description": "Model listing endpoints" }, { "name": "Health", "description": "Server health and liveness endpoints" }, { "name": "Monitoring", "description": "Prometheus metrics scrape endpoint" } ], "paths": { "/v1/chat/completions": { "post": { "tags": ["Chat Completions"], "summary": "Create chat completion", "description": "Creates a chat completion for the provided messages. OpenAI-compatible endpoint. Messages are converted to a prompt internally; responses use object \"chat.completion\" and choices[].message with role and content.", "operationId": "createChatCompletion", "security": [{ "BearerAuth": [] }], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ChatCompletionRequest" } } } }, "responses": { "200": { "description": "Successful chat completion (JSON) or streaming (text/event-stream)", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ChatCompletionResponse" } }, "text/event-stream": { "schema": { "type": "string", "description": "SSE stream; object \"chat.completion.chunk\", choices[].delta" } } } }, "400": { "description": "Invalid request (e.g. missing or empty messages)", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "401": { "description": "Missing or invalid authentication token", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "503": { "description": "Model not ready", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } } } }, "/v1/embeddings": { "post": { "tags": ["Embeddings"], "summary": "Create embeddings", "description": "Creates an embedding vector for the provided input text. OpenAI-compatible endpoint.", "operationId": "createEmbedding", "security": [{ "BearerAuth": [] }], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/EmbeddingRequest" } } } }, "responses": { "200": { "description": "Embedding created successfully", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/EmbeddingResponse" } } } }, "400": { "description": "Invalid request", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "503": { "description": "Model not ready", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } } } }, "/v1/audio/speech": { "post": { "tags": ["Text to speech processing"], "summary": "Create speech audio", "description": "Generates speech audio from text. Registered only when MODEL_SERVICE=tts. The initial C++ implementation streams audio/wav.", "operationId": "createSpeech", "security": [{ "BearerAuth": [] }], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/TtsSpeechRequest" } }, "multipart/form-data": { "schema": { "allOf": [ { "$ref": "#/components/schemas/TtsSpeechRequest" }, { "type": "object", "properties": { "voice_sample": { "type": "string", "format": "binary", "description": "Optional PCM16 WAV voice reference. The server normalizes it to 16 kHz mono PCM16 before scheduling." } } } ] } } } }, "responses": { "200": { "description": "Streaming WAV audio", "content": { "audio/wav": { "schema": { "type": "string", "format": "binary" } } } }, "400": { "description": "Invalid request", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "401": { "description": "Missing or invalid authentication token", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "429": { "description": "TTS request queue is full", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "503": { "description": "TTS backend is not ready", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } } } }, "/v1/images/generations": { "post": { "tags": ["Images"], "summary": "Create image (text-to-image)", "description": "Creates one or more images from a text prompt. Only registered when the active image runner_type is a text-to-image variant.", "operationId": "createImageGeneration", "security": [{ "BearerAuth": [] }], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ImageGenerateRequest" } } } }, "responses": { "200": { "description": "Image(s) generated successfully", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ImageResponse" } } } }, "400": { "description": "Invalid request (e.g. missing required field)", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "401": { "description": "Missing or invalid authentication token", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "503": { "description": "Model not ready", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } } } }, "/v1/images/image-to-image": { "post": { "tags": ["Images"], "summary": "Image-to-image", "description": "Generates a new image from a prompt and an input image. Only registered when the active image runner_type is an image-to-image variant.", "operationId": "createImageToImage", "security": [{ "BearerAuth": [] }], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ImageGenerateRequest" } } } }, "responses": { "200": { "description": "Image(s) generated successfully", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ImageResponse" } } } }, "400": { "description": "Invalid request (missing prompt or image)", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "401": { "description": "Missing or invalid authentication token", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "503": { "description": "Model not ready", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } } } }, "/v1/images/edits": { "post": { "tags": ["Images"], "summary": "Edit / inpaint image", "description": "Edits an input image using a prompt and a mask (inpainting). Only registered when the active image runner_type is an edit/inpaint variant.", "operationId": "createImageEdit", "security": [{ "BearerAuth": [] }], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ImageGenerateRequest" } } } }, "responses": { "200": { "description": "Image(s) edited successfully", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ImageResponse" } } } }, "400": { "description": "Invalid request (missing prompt, image, or mask)", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "401": { "description": "Missing or invalid authentication token", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "503": { "description": "Model not ready", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } } } }, "/health": { "get": { "tags": ["Health"], "summary": "Health check", "description": "Returns server health status.", "operationId": "healthCheck", "security": [], "responses": { "200": { "description": "Server is healthy", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HealthResponse" } } } } } } }, "/tt-liveness": { "get": { "tags": ["Health"], "summary": "Liveness check", "description": "Liveness probe (same as Python tt-liveness). Returns 200 with {\"status\": \"alive\", ...system status} when the process can respond. model_ready in the body reflects whether any worker has warmed up. Does not return 503 for model not ready; 500 only on unrecoverable failure.", "operationId": "livenessCheck", "security": [], "responses": { "200": { "description": "Process is alive; body includes status alive and system status (model_ready, workers, queue)", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ReadyResponse" } } } }, "500": { "description": "Unrecoverable failure (e.g. no service configured, exception)", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ReadyResponse" } } } } } } }, "/max-session-count": { "get": { "tags": ["Health"], "summary": "Get maximum session count", "description": "Returns the current maximum session count. This may be either the value set via the POST endpoint or the default from the MAX_SESSIONS_COUNT environment variable.", "operationId": "getMaxSessionCount", "security": [], "responses": { "200": { "description": "Successfully retrieved max session count", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/MaxSessionCountResponse" } } } } } }, "post": { "tags": ["Health"], "summary": "Set maximum session count", "description": "Sets the maximum session count at runtime. This overrides the MAX_SESSIONS_COUNT environment variable. Set to 0 to clear the override and revert to the environment variable value. Changes take effect immediately for new session allocation attempts.", "operationId": "setMaxSessionCount", "security": [], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/MaxSessionCountRequest" } } } }, "responses": { "200": { "description": "Successfully set max session count", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/MaxSessionCountSetResponse" } } } }, "400": { "description": "Invalid request (e.g. missing field or invalid value)", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "500": { "description": "Internal server error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } } } }, "/metrics": { "get": { "tags": ["Monitoring"], "summary": "Prometheus metrics", "description": "Exposes all server metrics in Prometheus text exposition format (version 0.0.4). No authentication required. Intended to be scraped by a Prometheus server every few seconds.", "operationId": "getMetrics", "security": [], "responses": { "200": { "description": "Prometheus text format metrics", "content": { "text/plain; version=0.0.4": { "schema": { "type": "string", "example": "# HELP tt_generation_tokens_total Total number of generation tokens produced\n# TYPE tt_generation_tokens_total counter\ntt_generation_tokens_total{model_name=\"llm\"} 42\n" } } } } } } }, "/info": { "get": { "tags": ["Health"], "summary": "Build identity", "description": "Returns the build identity of the running server: tt-inference-server version and commit, tt-llm-engine commit, tt-metal commit, and active model service. Missing values appear as the literal string \"unknown\". No authentication required.", "operationId": "getInfo", "security": [], "responses": { "200": { "description": "Build identity", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/InfoResponse" } } } } } } } }, "components": { "schemas": { "ChatCompletionRequest": { "type": "object", "required": ["messages"], "properties": { "model": { "type": "string", "description": "Model identifier", "example": "test-model" }, "messages": { "type": "array", "description": "Conversation messages (required, non-empty)", "items": { "type": "object", "required": ["role", "content"], "properties": { "role": { "type": "string", "description": "Message role: system, user, or assistant", "default": "user" }, "content": { "type": "string", "description": "Message content" } } } }, "max_tokens": { "type": "integer", "default": 16, "minimum": 1, "description": "Maximum number of tokens to generate" }, "stream": { "type": "boolean", "default": false, "description": "Whether to stream the response as SSE" }, "stream_options": { "$ref": "#/components/schemas/StreamOptions" }, "temperature": { "type": "number", "minimum": 0, "maximum": 2, "description": "Sampling temperature" }, "top_p": { "type": "number", "minimum": 0, "maximum": 1, "description": "Nucleus sampling probability" }, "stop": { "oneOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } } ], "description": "Stop sequence(s)" }, "presence_penalty": { "type": "number" }, "frequency_penalty": { "type": "number" }, "seed": { "type": "integer" }, "user": { "type": "string" }, "enable_reasoning": { "type": "boolean", "default": true, "description": "When false, reasoning models (e.g. DeepSeek-R1) skip chain-of-thought and respond directly. Has no effect on non-reasoning models." } } }, "ChatCompletionResponse": { "type": "object", "required": ["id", "object", "created", "choices"], "properties": { "id": { "type": "string", "description": "Unique chat completion identifier", "example": "chatcmpl-abc123def456" }, "object": { "type": "string", "enum": ["chat.completion"], "description": "Object type" }, "created": { "type": "integer", "description": "Unix timestamp of creation" }, "model": { "type": "string", "description": "Model used for completion" }, "choices": { "type": "array", "items": { "type": "object", "properties": { "index": { "type": "integer" }, "message": { "type": "object", "properties": { "role": { "type": "string" }, "content": { "type": "string" } } }, "finish_reason": { "type": "string" } } } }, "usage": { "$ref": "#/components/schemas/CompletionUsage" } } }, "StreamOptions": { "type": "object", "properties": { "include_usage": { "type": "boolean", "default": true, "description": "Include usage statistics in response" }, "continuous_usage_stats": { "type": "boolean", "default": false, "description": "Include usage stats in each streamed chunk" } } }, "CompletionUsage": { "type": "object", "properties": { "prompt_tokens": { "type": "integer", "description": "Number of tokens in the prompt" }, "completion_tokens": { "type": "integer", "description": "Number of tokens in the completion" }, "total_tokens": { "type": "integer", "description": "Total tokens used" }, "ttft_ms": { "type": "number", "description": "Time to first token in milliseconds", "nullable": true }, "tps": { "type": "number", "description": "Tokens per second (excluding first token)", "nullable": true } } }, "HealthResponse": { "type": "object", "properties": { "status": { "type": "string", "enum": ["healthy"], "description": "Server health status" }, "timestamp": { "type": "integer", "description": "Unix timestamp" } } }, "ReadyResponse": { "type": "object", "properties": { "model_ready": { "type": "boolean", "description": "Whether the model is ready for inference" }, "queue_size": { "type": "integer", "description": "Current number of requests in queue" }, "max_queue_size": { "type": "integer", "description": "Maximum queue capacity" }, "device": { "type": "string", "description": "Device type" }, "worker_info": { "type": "object", "description": "Worker status keyed by worker ID (Python-server-compatible format)", "additionalProperties": { "$ref": "#/components/schemas/WorkerInfo" } }, "runner_in_use": { "type": "string", "description": "Active runner identifier (image services only). Mirrors the Python ModelRunners enum (e.g. 'tt-sdxl-trace', 'tt-sdxl-image-to-image', 'tt-sdxl-edit'). Omitted when the active service does not expose a runner." } } }, "WorkerInfo": { "type": "object", "properties": { "worker_id": { "type": "string", "description": "Worker identifier" }, "is_ready": { "type": "boolean", "description": "Whether worker is ready" }, "is_alive": { "type": "boolean", "description": "Whether worker process is alive" }, "pid": { "type": "integer", "description": "Worker process ID (-1 for in-process runners)" } } }, "InfoResponse": { "type": "object", "required": ["tt_inference_server", "tt_llm_engine", "tt_metal", "model_service"], "properties": { "tt_inference_server": { "$ref": "#/components/schemas/ServerVersionInfo" }, "tt_llm_engine": { "$ref": "#/components/schemas/CommitInfo" }, "tt_metal": { "$ref": "#/components/schemas/CommitInfo" }, "model_service": { "type": "string", "enum": ["llm", "embedding", "image", "tts"], "description": "Active MODEL_SERVICE for route gating." } }, "example": { "tt_inference_server": { "version": "0.13.0", "commit": "b0f8adfc1234567890abcdef1234567890abcdef" }, "tt_llm_engine": { "commit": "abc123def456abc123def456abc123def4567890" }, "tt_metal": { "commit": "915d9e67ad572b97fbaf5339bd3e518db6dc05e3" }, "model_service": "tts" } }, "ServerVersionInfo": { "type": "object", "required": ["version", "commit"], "properties": { "version": { "type": "string", "description": "Release version from the top-level VERSION file. \"unknown\" if the file was not found at build time.", "example": "0.13.0" }, "commit": { "type": "string", "description": "Full git SHA of the inference-server tree at build time. \"unknown\" if .git was not available.", "example": "b0f8adfc1234567890abcdef1234567890abcdef" } } }, "CommitInfo": { "type": "object", "required": ["commit"], "properties": { "commit": { "type": "string", "description": "Full git SHA of the dependency at build time. \"unknown\" if the dependency's working tree was not present.", "example": "915d9e67ad572b97fbaf5339bd3e518db6dc05e3" } } }, "MaxSessionCountResponse": { "type": "object", "required": ["max_session_count"], "properties": { "max_session_count": { "type": "integer", "format": "int64", "minimum": 0, "description": "Current maximum session count", "example": 128 } } }, "MaxSessionCountRequest": { "type": "object", "required": ["max_session_count"], "properties": { "max_session_count": { "type": "integer", "format": "int64", "minimum": 0, "description": "New maximum session count. Set to 0 to clear override and use environment variable value.", "example": 256 } } }, "MaxSessionCountSetResponse": { "type": "object", "required": ["max_session_count", "status"], "properties": { "max_session_count": { "type": "integer", "format": "int64", "minimum": 0, "description": "The new maximum session count that was set", "example": 256 }, "status": { "type": "string", "enum": ["success"], "description": "Operation status" } } }, "EmbeddingRequest": { "type": "object", "required": ["model", "input"], "properties": { "model": { "type": "string", "description": "Model identifier to use for embedding", "example": "text-embedding-ada-002" }, "input": { "type": "string", "description": "Text to embed" }, "user": { "type": "string", "description": "Optional user identifier" } } }, "EmbeddingResponse": { "type": "object", "properties": { "object": { "type": "string", "enum": ["list"] }, "data": { "type": "array", "items": { "type": "object", "properties": { "object": { "type": "string", "enum": ["embedding"] }, "index": { "type": "integer" }, "embedding": { "type": "array", "items": { "type": "number" }, "description": "Embedding vector" } } } }, "model": { "type": "string" }, "usage": { "type": "object", "properties": { "prompt_tokens": { "type": "integer" }, "total_tokens": { "type": "integer" } } } } }, "TtsSpeechRequest": { "type": "object", "required": ["text"], "properties": { "text": { "type": "string", "description": "Text to synthesize into speech." }, "description": { "type": "string", "description": "Optional natural-language voice description." } } }, "ImageGenerateRequest": { "type": "object", "required": ["prompt"], "description": "Superset of fields for /v1/images/generations, /v1/images/image-to-image, and /v1/images/edits. The active route enforces its own subset (image-to-image requires `image`; edits also requires `mask`).", "properties": { "prompt": { "type": "string", "description": "Text prompt describing the desired output image" }, "prompt_2": { "type": "string", "description": "Optional secondary prompt routed to the second text encoder (SDXL)" }, "negative_prompt": { "type": "string", "description": "Negative prompt describing what to avoid" }, "negative_prompt_2": { "type": "string", "description": "Optional secondary negative prompt for the second text encoder" }, "num_inference_steps": { "type": "integer", "default": 20, "minimum": 1, "description": "Number of denoising steps" }, "guidance_scale": { "type": "number", "default": 5.0, "description": "Classifier-free guidance scale" }, "guidance_rescale": { "type": "number", "default": 0.0, "description": "Guidance rescale factor (mitigates over-saturation at high guidance)" }, "seed": { "type": "integer", "description": "Random seed for reproducible sampling" }, "number_of_images": { "type": "integer", "default": 1, "minimum": 1, "description": "Number of images to generate per request" }, "crop_coords_top_left": { "type": "array", "items": { "type": "integer" }, "minItems": 2, "maxItems": 2, "default": [0, 0], "description": "SDXL crop conditioning [top, left]" }, "timesteps": { "type": "array", "items": { "type": "number" }, "description": "Optional explicit list of timesteps for the scheduler" }, "sigmas": { "type": "array", "items": { "type": "number" }, "description": "Optional explicit list of sigmas for the scheduler" }, "lora_path": { "type": "string", "description": "Path to a LoRA weights file to load before generation" }, "lora_scale": { "type": "number", "default": 0.5, "description": "Scale applied to the LoRA" }, "image_return_format": { "type": "string", "enum": ["JPEG", "PNG"], "default": "JPEG", "description": "Encoding format for the returned base64 image bytes" }, "image_quality": { "type": "integer", "default": 85, "minimum": 50, "maximum": 100, "description": "Quality factor for JPEG encoding (ignored for PNG). Bounds match the Python service's Pydantic constraint (ge=50, le=100)." }, "image": { "type": "string", "description": "Base64-encoded input image. Required for image-to-image and edits." }, "mask": { "type": "string", "description": "Base64-encoded mask image. Required for edits/inpainting." }, "strength": { "type": "number", "description": "Image-to-image strength (0..1); higher values diverge further from the input image." } } }, "ImageResponse": { "type": "object", "required": ["images"], "properties": { "images": { "type": "array", "items": { "type": "string", "description": "Base64-encoded image bytes in the requested image_return_format" }, "description": "Generated images, base64-encoded" }, "generation_time": { "type": "number", "description": "Wall-clock generation time in seconds (omitted when zero)" } } }, "Error": { "type": "object", "properties": { "error": { "type": "object", "properties": { "message": { "type": "string" }, "type": { "type": "string" }, "code": { "type": "string" } } } } } }, "securitySchemes": { "BearerAuth": { "type": "http", "scheme": "bearer", "bearerFormat": "API Key", "description": "Bearer token authentication using OPENAI_API_KEY environment variable. If not set, defaults to 'your-secret-key'." } } } }