{ "openapi": "3.0.3", "info": { "title": "Text Embeddings Inference", "description": "Text Embedding Webserver", "contact": { "name": "Olivier Dehaene" }, "license": { "name": "HFOIL" }, "version": "1.1.0" }, "paths": { "/decode": { "post": { "tags": [ "Text Embeddings Inference" ], "summary": "Decode input ids", "description": "Decode input ids", "operationId": "decode", "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/DecodeRequest" } } }, "required": true }, "responses": { "200": { "description": "Decoded ids", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/DecodeResponse" } } } }, "422": { "description": "Tokenization error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorResponse" }, "example": { "message": "Tokenization error", "type": "tokenizer" } } } } } } }, "/embed": { "post": { "tags": [ "Text Embeddings Inference" ], "summary": "Get Embeddings. Returns a 424 status code if the model is not an embedding model.", "description": "Get Embeddings. Returns a 424 status code if the model is not an embedding model.", "operationId": "embed", "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/EmbedRequest" } } }, "required": true }, "responses": { "200": { "description": "Embeddings", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/EmbedResponse" } } } }, "413": { "description": "Batch size error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorResponse" }, "example": { "error": "Batch size error", "error_type": "validation" } } } }, "422": { "description": "Tokenization error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorResponse" }, "example": { "error": "Tokenization error", "error_type": "tokenizer" } } } }, "424": { "description": "Embedding Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorResponse" }, "example": { "error": "Inference failed", "error_type": "backend" } } } }, "429": { "description": "Model is overloaded", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorResponse" }, "example": { "error": "Model is overloaded", "error_type": "overloaded" } } } } } } }, "/embed_all": { "post": { "tags": [ "Text Embeddings Inference" ], "summary": "Get all Embeddings without Pooling.", "description": "Get all Embeddings without Pooling.\nReturns a 424 status code if the model is not an embedding model.", "operationId": "embed_all", "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/EmbedAllRequest" } } }, "required": true }, "responses": { "200": { "description": "Embeddings", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/EmbedAllResponse" } } } }, "413": { "description": "Batch size error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorResponse" }, "example": { "error": "Batch size error", "error_type": "validation" } } } }, "422": { "description": "Tokenization error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorResponse" }, "example": { "error": "Tokenization error", "error_type": "tokenizer" } } } }, "424": { "description": "Embedding Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorResponse" }, "example": { "error": "Inference failed", "error_type": "backend" } } } }, "429": { "description": "Model is overloaded", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorResponse" }, "example": { "error": "Model is overloaded", "error_type": "overloaded" } } } } } } }, "/embed_sparse": { "post": { "tags": [ "Text Embeddings Inference" ], "summary": "Get Sparse Embeddings. Returns a 424 status code if the model is not an embedding model with SPLADE pooling.", "description": "Get Sparse Embeddings. Returns a 424 status code if the model is not an embedding model with SPLADE pooling.", "operationId": "embed_sparse", "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/EmbedSparseRequest" } } }, "required": true }, "responses": { "200": { "description": "Embeddings", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/EmbedSparseResponse" } } } }, "413": { "description": "Batch size error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorResponse" }, "example": { "error": "Batch size error", "error_type": "validation" } } } }, "422": { "description": "Tokenization error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorResponse" }, "example": { "error": "Tokenization error", "error_type": "tokenizer" } } } }, "424": { "description": "Embedding Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorResponse" }, "example": { "error": "Inference failed", "error_type": "backend" } } } }, "429": { "description": "Model is overloaded", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorResponse" }, "example": { "error": "Model is overloaded", "error_type": "overloaded" } } } } } } }, "/embeddings": { "post": { "tags": [ "Text Embeddings Inference" ], "summary": "OpenAI compatible route. Returns a 424 status code if the model is not an embedding model.", "description": "OpenAI compatible route. Returns a 424 status code if the model is not an embedding model.", "operationId": "openai_embed", "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/OpenAICompatRequest" } } }, "required": true }, "responses": { "200": { "description": "Embeddings", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/OpenAICompatResponse" } } } }, "413": { "description": "Batch size error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/OpenAICompatErrorResponse" }, "example": { "message": "Batch size error", "type": "validation" } } } }, "422": { "description": "Tokenization error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/OpenAICompatErrorResponse" }, "example": { "message": "Tokenization error", "type": "tokenizer" } } } }, "424": { "description": "Embedding Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/OpenAICompatErrorResponse" }, "example": { "message": "Inference failed", "type": "backend" } } } }, "429": { "description": "Model is overloaded", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/OpenAICompatErrorResponse" }, "example": { "message": "Model is overloaded", "type": "overloaded" } } } } } } }, "/health": { "get": { "tags": [ "Text Embeddings Inference" ], "summary": "Health check method", "description": "Health check method", "operationId": "health", "responses": { "200": { "description": "Everything is working fine" }, "503": { "description": "Text embeddings Inference is down", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorResponse" }, "example": { "error": "unhealthy", "error_type": "unhealthy" } } } } } } }, "/info": { "get": { "tags": [ "Text Embeddings Inference" ], "summary": "Text Embeddings Inference endpoint info", "description": "Text Embeddings Inference endpoint info", "operationId": "get_model_info", "responses": { "200": { "description": "Served model info", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Info" } } } } } } }, "/metrics": { "get": { "tags": [ "Text Embeddings Inference" ], "summary": "Prometheus metrics scrape endpoint", "description": "Prometheus metrics scrape endpoint", "operationId": "metrics", "responses": { "200": { "description": "Prometheus Metrics", "content": { "text/plain": { "schema": { "type": "string" } } } } } } }, "/predict": { "post": { "tags": [ "Text Embeddings Inference" ], "summary": "Get Predictions. Returns a 424 status code if the model is not a Sequence Classification model", "description": "Get Predictions. Returns a 424 status code if the model is not a Sequence Classification model", "operationId": "predict", "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/PredictRequest" } } }, "required": true }, "responses": { "200": { "description": "Predictions", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/PredictResponse" } } } }, "413": { "description": "Batch size error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorResponse" }, "example": { "error": "Batch size error", "error_type": "validation" } } } }, "422": { "description": "Tokenization error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorResponse" }, "example": { "error": "Tokenization error", "error_type": "tokenizer" } } } }, "424": { "description": "Prediction Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorResponse" }, "example": { "error": "Inference failed", "error_type": "backend" } } } }, "429": { "description": "Model is overloaded", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorResponse" }, "example": { "error": "Model is overloaded", "error_type": "overloaded" } } } } } } }, "/rerank": { "post": { "tags": [ "Text Embeddings Inference" ], "summary": "Get Ranks. Returns a 424 status code if the model is not a Sequence Classification model with", "description": "Get Ranks. Returns a 424 status code if the model is not a Sequence Classification model with\na single class.", "operationId": "rerank", "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/RerankRequest" } } }, "required": true }, "responses": { "200": { "description": "Ranks", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/RerankResponse" } } } }, "413": { "description": "Batch size error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorResponse" }, "example": { "error": "Batch size error", "error_type": "validation" } } } }, "422": { "description": "Tokenization error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorResponse" }, "example": { "error": "Tokenization error", "error_type": "tokenizer" } } } }, "424": { "description": "Rerank Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorResponse" }, "example": { "error": "Inference failed", "error_type": "backend" } } } }, "429": { "description": "Model is overloaded", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorResponse" }, "example": { "error": "Model is overloaded", "error_type": "overloaded" } } } } } } }, "/tokenize": { "post": { "tags": [ "Text Embeddings Inference" ], "summary": "Tokenize inputs", "description": "Tokenize inputs", "operationId": "tokenize", "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/TokenizeRequest" } } }, "required": true }, "responses": { "200": { "description": "Tokenized ids", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/TokenizeResponse" } } } }, "422": { "description": "Tokenization error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorResponse" }, "example": { "message": "Tokenization error", "type": "tokenizer" } } } } } } }, "/vertex": { "post": { "tags": [ "Text Embeddings Inference" ], "summary": "Generate embeddings from a Vertex request", "description": "Generate embeddings from a Vertex request", "operationId": "vertex_compatibility", "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/VertexRequest" } } }, "required": true }, "responses": { "200": { "description": "Results" }, "413": { "description": "Batch size error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorResponse" }, "example": { "error": "Batch size error", "error_type": "validation" } } } }, "422": { "description": "Tokenization error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorResponse" }, "example": { "error": "Tokenization error", "error_type": "tokenizer" } } } }, "424": { "description": "Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorResponse" }, "example": { "error": "Inference failed", "error_type": "backend" } } } }, "429": { "description": "Model is overloaded", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorResponse" }, "example": { "error": "Model is overloaded", "error_type": "overloaded" } } } } } } } }, "components": { "schemas": { "ClassifierModel": { "type": "object", "required": [ "id2label", "label2id" ], "properties": { "id2label": { "type": "object", "additionalProperties": { "type": "string" }, "example": { "0": "LABEL" } }, "label2id": { "type": "object", "additionalProperties": { "type": "integer", "minimum": 0 }, "example": { "LABEL": 0 } } } }, "DecodeRequest": { "type": "object", "required": [ "ids" ], "properties": { "ids": { "$ref": "#/components/schemas/InputIds" }, "skip_special_tokens": { "type": "boolean", "default": "true", "example": "true" } } }, "DecodeResponse": { "type": "array", "items": { "type": "string" }, "example": [ "test" ] }, "EmbedAllRequest": { "type": "object", "required": [ "inputs" ], "properties": { "inputs": { "$ref": "#/components/schemas/Input" }, "truncate": { "type": "boolean", "default": "false", "example": "false" } } }, "EmbedAllResponse": { "type": "array", "items": { "type": "array", "items": { "type": "array", "items": { "type": "number", "format": "float" } } }, "example": [ [ [ 0.0, 1.0, 2.0 ] ] ] }, "EmbedRequest": { "type": "object", "required": [ "inputs" ], "properties": { "inputs": { "$ref": "#/components/schemas/Input" }, "normalize": { "type": "boolean", "default": "true", "example": "true" }, "truncate": { "type": "boolean", "default": "false", "example": "false" } } }, "EmbedResponse": { "type": "array", "items": { "type": "array", "items": { "type": "number", "format": "float" } }, "example": [ [ 0.0, 1.0, 2.0 ] ] }, "EmbedSparseRequest": { "type": "object", "required": [ "inputs" ], "properties": { "inputs": { "$ref": "#/components/schemas/Input" }, "truncate": { "type": "boolean", "default": "false", "example": "false" } } }, "EmbedSparseResponse": { "type": "array", "items": { "type": "array", "items": { "$ref": "#/components/schemas/SparseValue" } } }, "EmbeddingModel": { "type": "object", "required": [ "pooling" ], "properties": { "pooling": { "type": "string", "example": "cls" } } }, "ErrorResponse": { "type": "object", "required": [ "error", "error_type" ], "properties": { "error": { "type": "string" }, "error_type": { "$ref": "#/components/schemas/ErrorType" } } }, "ErrorType": { "type": "string", "enum": [ "Unhealthy", "Backend", "Overloaded", "Validation", "Tokenizer" ] }, "Info": { "type": "object", "required": [ "model_id", "model_dtype", "model_type", "max_concurrent_requests", "max_input_length", "max_batch_tokens", "max_client_batch_size", "tokenization_workers", "version" ], "properties": { "docker_label": { "type": "string", "example": "null", "nullable": true }, "max_batch_requests": { "type": "integer", "default": "null", "example": "null", "nullable": true, "minimum": 0 }, "max_batch_tokens": { "type": "integer", "example": "2048", "minimum": 0 }, "max_client_batch_size": { "type": "integer", "example": "32", "minimum": 0 }, "max_concurrent_requests": { "type": "integer", "description": "Router Parameters", "example": "128", "minimum": 0 }, "max_input_length": { "type": "integer", "example": "512", "minimum": 0 }, "model_dtype": { "type": "string", "example": "float16" }, "model_id": { "type": "string", "description": "Model info", "example": "thenlper/gte-base" }, "model_sha": { "type": "string", "example": "fca14538aa9956a46526bd1d0d11d69e19b5a101", "nullable": true }, "model_type": { "$ref": "#/components/schemas/ModelType" }, "sha": { "type": "string", "example": "null", "nullable": true }, "tokenization_workers": { "type": "integer", "example": "4", "minimum": 0 }, "version": { "type": "string", "description": "Router Info", "example": "0.5.0" } } }, "Input": { "oneOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } } ] }, "InputIds": { "oneOf": [ { "type": "array", "items": { "type": "integer", "format": "int32", "minimum": 0 } }, { "type": "array", "items": { "type": "array", "items": { "type": "integer", "format": "int32", "minimum": 0 } } } ] }, "ModelType": { "oneOf": [ { "type": "object", "required": [ "classifier" ], "properties": { "classifier": { "$ref": "#/components/schemas/ClassifierModel" } } }, { "type": "object", "required": [ "embedding" ], "properties": { "embedding": { "$ref": "#/components/schemas/EmbeddingModel" } } }, { "type": "object", "required": [ "reranker" ], "properties": { "reranker": { "$ref": "#/components/schemas/ClassifierModel" } } } ] }, "OpenAICompatEmbedding": { "type": "object", "required": [ "object", "embedding", "index" ], "properties": { "embedding": { "type": "array", "items": { "type": "number", "format": "float" }, "example": [ 0.0, 1.0, 2.0 ] }, "index": { "type": "integer", "example": "0", "minimum": 0 }, "object": { "type": "string", "example": "embedding" } } }, "OpenAICompatErrorResponse": { "type": "object", "required": [ "message", "code", "error_type" ], "properties": { "code": { "type": "integer", "format": "int32", "minimum": 0 }, "error_type": { "$ref": "#/components/schemas/ErrorType" }, "message": { "type": "string" } } }, "OpenAICompatRequest": { "type": "object", "required": [ "input" ], "properties": { "input": { "$ref": "#/components/schemas/Input" }, "model": { "type": "string", "example": "null", "nullable": true }, "user": { "type": "string", "example": "null", "nullable": true } } }, "OpenAICompatResponse": { "type": "object", "required": [ "object", "data", "model", "usage" ], "properties": { "data": { "type": "array", "items": { "$ref": "#/components/schemas/OpenAICompatEmbedding" } }, "model": { "type": "string", "example": "thenlper/gte-base" }, "object": { "type": "string", "example": "list" }, "usage": { "$ref": "#/components/schemas/OpenAICompatUsage" } } }, "OpenAICompatUsage": { "type": "object", "required": [ "prompt_tokens", "total_tokens" ], "properties": { "prompt_tokens": { "type": "integer", "example": "512", "minimum": 0 }, "total_tokens": { "type": "integer", "example": "512", "minimum": 0 } } }, "PredictInput": { "oneOf": [ { "type": "string", "description": "A single string" }, { "type": "array", "items": { "type": "string" }, "description": "A pair of strings", "maxItems": 2, "minItems": 2 }, { "type": "array", "items": { "oneOf": [ { "type": "array", "items": { "type": "string" }, "description": "A single string", "maxItems": 1, "minItems": 1 }, { "type": "array", "items": { "type": "string" }, "description": "A pair of strings", "maxItems": 2, "minItems": 2 } ] }, "description": "A batch" } ], "description": "Model input. Can be either a single string, a pair of strings or a batch of mixed single and pairs of strings.", "example": "What is Deep Learning?" }, "PredictRequest": { "type": "object", "required": [ "inputs" ], "properties": { "inputs": { "$ref": "#/components/schemas/PredictInput" }, "raw_scores": { "type": "boolean", "default": "false", "example": "false" }, "truncate": { "type": "boolean", "default": "false", "example": "false" } } }, "PredictResponse": { "oneOf": [ { "type": "array", "items": { "$ref": "#/components/schemas/Prediction" } }, { "type": "array", "items": { "type": "array", "items": { "$ref": "#/components/schemas/Prediction" } } } ] }, "Prediction": { "type": "object", "required": [ "score", "label" ], "properties": { "label": { "type": "string", "example": "admiration" }, "score": { "type": "number", "format": "float", "example": "0.5" } } }, "Rank": { "type": "object", "required": [ "index", "score" ], "properties": { "index": { "type": "integer", "example": "0", "minimum": 0 }, "score": { "type": "number", "format": "float", "example": "1.0" }, "text": { "type": "string", "default": "null", "example": "Deep Learning is ...", "nullable": true } } }, "RerankRequest": { "type": "object", "required": [ "query", "texts" ], "properties": { "query": { "type": "string", "example": "What is Deep Learning?" }, "raw_scores": { "type": "boolean", "default": "false", "example": "false" }, "return_text": { "type": "boolean", "default": "false", "example": "false" }, "texts": { "type": "array", "items": { "type": "string" }, "example": [ "Deep Learning is ..." ] }, "truncate": { "type": "boolean", "default": "false", "example": "false" } } }, "RerankResponse": { "type": "array", "items": { "$ref": "#/components/schemas/Rank" } }, "SimpleToken": { "type": "object", "required": [ "id", "text", "special" ], "properties": { "id": { "type": "integer", "format": "int32", "example": 0, "minimum": 0 }, "special": { "type": "boolean", "example": "false" }, "start": { "type": "integer", "example": 0, "nullable": true, "minimum": 0 }, "stop": { "type": "integer", "example": 2, "nullable": true, "minimum": 0 }, "text": { "type": "string", "example": "test" } } }, "SparseValue": { "type": "object", "required": [ "index", "value" ], "properties": { "index": { "type": "integer", "minimum": 0 }, "value": { "type": "number", "format": "float" } } }, "TokenizeRequest": { "type": "object", "required": [ "inputs" ], "properties": { "add_special_tokens": { "type": "boolean", "default": "true", "example": "true" }, "inputs": { "$ref": "#/components/schemas/Input" } } }, "TokenizeResponse": { "type": "array", "items": { "type": "array", "items": { "$ref": "#/components/schemas/SimpleToken" } }, "example": [ [ { "id": 0, "special": false, "start": 0, "stop": 2, "text": "test" } ] ] }, "VertexInstance": { "oneOf": [ { "allOf": [ { "$ref": "#/components/schemas/EmbedRequest" }, { "type": "object", "required": [ "type" ], "properties": { "type": { "type": "string", "enum": [ "embed" ] } } } ] }, { "allOf": [ { "$ref": "#/components/schemas/EmbedAllRequest" }, { "type": "object", "required": [ "type" ], "properties": { "type": { "type": "string", "enum": [ "embed_all" ] } } } ] }, { "allOf": [ { "$ref": "#/components/schemas/EmbedSparseRequest" }, { "type": "object", "required": [ "type" ], "properties": { "type": { "type": "string", "enum": [ "embed_sparse" ] } } } ] }, { "allOf": [ { "$ref": "#/components/schemas/PredictRequest" }, { "type": "object", "required": [ "type" ], "properties": { "type": { "type": "string", "enum": [ "predict" ] } } } ] }, { "allOf": [ { "$ref": "#/components/schemas/RerankRequest" }, { "type": "object", "required": [ "type" ], "properties": { "type": { "type": "string", "enum": [ "rerank" ] } } } ] }, { "allOf": [ { "$ref": "#/components/schemas/TokenizeRequest" }, { "type": "object", "required": [ "type" ], "properties": { "type": { "type": "string", "enum": [ "tokenize" ] } } } ] } ], "discriminator": { "propertyName": "type" } }, "VertexRequest": { "type": "object", "required": [ "instances" ], "properties": { "instances": { "type": "array", "items": { "$ref": "#/components/schemas/VertexInstance" } } } }, "VertexResponse": { "type": "array", "items": { "$ref": "#/components/schemas/VertexResponseInstance" } }, "VertexResponseInstance": { "oneOf": [ { "type": "object", "required": [ "type", "result" ], "properties": { "result": { "$ref": "#/components/schemas/EmbedResponse" }, "type": { "type": "string", "enum": [ "embed" ] } } }, { "type": "object", "required": [ "type", "result" ], "properties": { "result": { "$ref": "#/components/schemas/EmbedAllResponse" }, "type": { "type": "string", "enum": [ "embed_all" ] } } }, { "type": "object", "required": [ "type", "result" ], "properties": { "result": { "$ref": "#/components/schemas/EmbedSparseResponse" }, "type": { "type": "string", "enum": [ "embed_sparse" ] } } }, { "type": "object", "required": [ "type", "result" ], "properties": { "result": { "$ref": "#/components/schemas/PredictResponse" }, "type": { "type": "string", "enum": [ "predict" ] } } }, { "type": "object", "required": [ "type", "result" ], "properties": { "result": { "$ref": "#/components/schemas/RerankResponse" }, "type": { "type": "string", "enum": [ "rerank" ] } } }, { "type": "object", "required": [ "type", "result" ], "properties": { "result": { "$ref": "#/components/schemas/TokenizeResponse" }, "type": { "type": "string", "enum": [ "tokenize" ] } } } ], "discriminator": { "propertyName": "type" } } } }, "tags": [ { "name": "Text Embeddings Inference", "description": "Hugging Face Text Embeddings Inference API" } ] }