{ "openapi": "3.1.0", "info": { "title": "Chonkie API", "description": "🦛 Chonkie running on the Internet! 🌎 Get chunks via POST HTTP calls~", "version": "0.1.0" }, "servers": [ { "url": "https://api.chonkie.ai", "description": "Chonkie Cloud API" } ], "security": [ { "BearerAuth": [] } ], "paths": { "/v1/auth/validate": { "post": { "summary": "Validate API Key", "description": "Validates the provided API key. Use this endpoint to check if your API key is valid and active.", "operationId": "validate_auth_v1_auth_validate_post", "responses": { "200": { "description": "API key is valid and active", "content": { "application/json": { "schema": { "type": "object", "properties": { "message": { "type": "string", "example": "Token is valid!" }, "status": { "type": "integer", "example": 200 } } }, "example": { "message": "Token is valid!", "status": 200 } } } } } } }, "/v1/chunk/token": { "post": { "summary": "Token Chunker", "description": "Chunk the given text or file using the Token Chunker.", "operationId": "get_token_chunks_v1_chunk_token_post", "requestBody": { "required": true, "content": { "multipart/form-data": { "schema": { "type": "object", "properties": { "file": { "type": "string", "format": "binary", "description": "The file containing text to be chunked. Use this OR the 'text' field." }, "text": { "anyOf": [ { "type": "string" }, { "items": { "type": "string" }, "type": "array" } ], "title": "Text", "description": "The raw input text or list of texts to be chunked. Use this OR the 'file' field." }, "tokenizer": { "type": "string", "title": "Tokenizer", "default": "gpt2", "description": "Tokenizer to use. Can be a string identifier or a tokenizer instance" }, "chunk_size": { "type": "integer", "title": "Chunk Size", "default": 512, "description": "Maximum number of tokens per chunk" }, "chunk_overlap": { "type": "integer", "title": "Chunk Overlap", "default": 0, "description": "Number or percentage of overlapping tokens between chunks" }, "return_type": { "type": "string", "title": "Return Type", "default": "chunks", "enum": [ "texts", "chunks" ], "description": "Whether to return chunks as Chunk objects or plain text strings" } } } } } }, "responses": { "200": { "description": "Successful Response: A list of standard chunk objects.", "content": { "application/json": { "schema": { "title": "TokenChunkResponse", "type": "array", "items": { "$ref": "#/components/schemas/Chunk" }, "description": "A list containing standard chunk objects, each detailing a segment of the original text based on token count." } } } } } } }, "/v1/chunk/sentence": { "post": { "summary": "Sentence Chunker", "description": "Chunk the given text or file using the Sentence Chunker.", "operationId": "get_sentence_chunks_v1_chunk_sentence_post", "requestBody": { "required": true, "content": { "multipart/form-data": { "schema": { "type": "object", "properties": { "file": { "type": "string", "format": "binary", "description": "The file containing text to be chunked. Use this OR the 'text' field." }, "text": { "anyOf": [ { "type": "string" }, { "items": { "type": "string" }, "type": "array" } ], "title": "Text", "description": "The raw input text or list of texts to be chunked. Use this OR the 'file' field." }, "tokenizer_or_token_counter": { "type": "string", "title": "Tokenizer Or Token Counter", "default": "gpt2", "description": "Tokenizer to use. Can be a string identifier or a tokenizer instance" }, "chunk_size": { "type": "integer", "title": "Chunk Size", "default": 512, "description": "Maximum number of tokens per chunk" }, "chunk_overlap": { "type": "integer", "title": "Chunk Overlap", "default": 0, "description": "Number of overlapping tokens between chunks" }, "min_sentences_per_chunk": { "type": "integer", "title": "Min Sentences Per Chunk", "default": 1, "description": "Minimum number of sentences to include in each chunk" }, "min_characters_per_sentence": { "type": "integer", "title": "Min Characters Per Sentence", "default": 1, "description": "Minimum number of characters per sentence" }, "approximate": { "type": "boolean", "title": "Approximate", "default": true, "description": "Use approximate token counting for faster processing (deprecated)" }, "delim": { "anyOf": [ { "type": "string" }, { "items": { "type": "string" }, "type": "array" } ], "title": "Delim", "default": [ "\n", ".", "!", "?" ], "description": "Delimiters to split sentences on" }, "include_delim": { "anyOf": [ { "type": "string", "enum": [ "prev", "next" ] }, { "type": "null" } ], "title": "Include Delim", "default": "prev", "description": "Include delimiters in the chunk text. If so, specifies whether to include in the previous or next chunk" }, "return_type": { "type": "string", "title": "Return Type", "default": "chunks", "enum": [ "texts", "chunks" ], "description": "Whether to return chunks as text strings or as SentenceChunk objects" } } } } } }, "responses": { "200": { "description": "Successful Response: A list of sentence chunk objects.", "content": { "application/json": { "schema": { "title": "SentenceChunkResponse", "type": "array", "items": { "$ref": "#/components/schemas/SentenceChunkSchema" }, "description": "A list containing sentence chunk objects, each detailing a segment of the original text and the sentences within it." } } } } } } }, "/v1/chunk/recursive": { "post": { "summary": "Recursive Chunker", "description": "Chunk the given text or file using the Recursive Chunker.", "operationId": "get_recursive_chunks_v1_chunk_recursive_post", "requestBody": { "required": true, "content": { "multipart/form-data": { "schema": { "type": "object", "properties": { "file": { "type": "string", "format": "binary", "description": "The file containing text to be chunked. Use this OR the 'text' field." }, "text": { "anyOf": [ { "type": "string" }, { "items": { "type": "string" }, "type": "array" } ], "title": "Text", "description": "The raw input text or list of texts to be chunked. Use this OR the 'file' field." }, "tokenizer_or_token_counter": { "type": "string", "title": "Tokenizer Or Token Counter", "default": "gpt2", "description": "Tokenizer to use. Can be a string identifier or a tokenizer instance" }, "chunk_size": { "type": "integer", "title": "Chunk Size", "default": 512, "description": "Maximum number of tokens per chunk" }, "recipe": { "type": "string", "title": "Recipe", "default": "default", "description": "Rules to split text by. Find all recipes on our [Hugging Face](https://huggingface.co/datasets/chonkie-ai/recipes)." }, "lang": { "type": "string", "title": "Language", "default": "en", "description": "Language of the text. This must match the language of the recipe." }, "min_characters_per_chunk": { "type": "integer", "title": "Min Characters Per Chunk", "default": 1, "description": "Minimum number of characters per chunk" }, "return_type": { "type": "string", "title": "Return Type", "default": "chunks", "enum": [ "texts", "chunks" ], "description": "Whether to return chunks as text strings or as RecursiveChunk objects" } } } } } }, "responses": { "200": { "description": "Successful Response: A list of recursive chunk objects.", "content": { "application/json": { "schema": { "title": "RecursiveChunkResponse", "type": "array", "items": { "$ref": "#/components/schemas/RecursiveChunkSchema" }, "description": "A list containing recursive chunk objects, each detailing a segment of the original text and its recursion level." } } } } } } }, "/v1/chunk/semantic": { "post": { "summary": "Semantic Chunker", "description": "Chunk the given text or file using the Semantic Chunker.", "operationId": "get_semantic_chunks_v1_chunk_semantic_post", "requestBody": { "required": true, "content": { "multipart/form-data": { "schema": { "type": "object", "properties": { "file": { "type": "string", "format": "binary", "description": "The file containing text to be chunked. Use this OR the 'text' field." }, "text": { "anyOf": [ { "type": "string" }, { "items": { "type": "string" }, "type": "array" } ], "title": "Text", "description": "The raw input text or list of texts to be chunked. Use this OR the 'file' field." }, "embedding_model": { "type": "string", "title": "Embedding Model", "default": "minishlab/potion-base-8M", "description": "Model identifier or embedding model instance" }, "threshold": { "anyOf": [ { "type": "string", "const": "auto" }, { "type": "number" }, { "type": "integer" } ], "title": "Threshold", "default": "auto", "description": "When in the range [0,1], denotes the similarity threshold to consider sentences similar. When in the range (1,100], interprets the given value as a percentile threshold. When set to 'auto', the threshold is automatically calculated." }, "chunk_size": { "type": "integer", "title": "Chunk Size", "default": 512, "description": "Maximum tokens per chunk" }, "similarity_window": { "type": "integer", "title": "Similarity Window", "default": 1, "description": "Number of sentences to consider for similarity threshold calculation" }, "min_sentences": { "type": "integer", "title": "Min Sentences", "default": 1, "description": "Minimum number of sentences per chunk" }, "min_chunk_size": { "type": "integer", "title": "Min Chunk Size", "default": 2, "description": "Minimum tokens per chunk" }, "min_characters_per_sentence": { "type": "integer", "title": "Min Characters Per Sentence", "default": 1, "description": "Minimum number of characters per sentence" }, "threshold_step": { "type": "number", "title": "Threshold Step", "default": 0.01, "description": "Step size for similarity threshold calculation" }, "delim": { "anyOf": [ { "type": "string" }, { "items": { "type": "string" }, "type": "array" } ], "title": "Delim", "default": [ "\n", ".", "!", "?" ], "description": "Delimiters to split sentences on. Default is ['.', '!', '?', '\\n']" }, "include_delim": { "anyOf": [ { "type": "string", "enum": [ "prev", "next" ] }, { "type": "null" } ], "title": "Include Delim", "default": "prev", "description": "Include delimiters in the chunk text. If so, specifies whether to include in the previous or next chunk" }, "return_type": { "type": "string", "title": "Return Type", "default": "chunks", "enum": [ "texts", "chunks" ], "description": "Return type for chunking. If 'chunks', returns a list of SemanticChunk objects. If 'texts', returns a list of strings" } } } } } }, "responses": { "200": { "description": "Successful Response: A list of semantic chunk objects.", "content": { "application/json": { "schema": { "title": "SemanticChunkResponse", "type": "array", "items": { "$ref": "#/components/schemas/SemanticChunkSchema" }, "description": "A list containing semantic chunk objects, detailing segments and sentences with optional embeddings." } } } } } } }, "/v1/chunk/sdpm": { "post": { "summary": "SDPM Chunker", "description": "Chunk the given text or file using the SDPM Chunker.", "operationId": "get_sdpm_chunks_v1_chunk_sdpm_post", "requestBody": { "required": true, "content": { "multipart/form-data": { "schema": { "type": "object", "properties": { "file": { "type": "string", "format": "binary", "description": "The file containing text to be chunked. Use this OR the 'text' field." }, "text": { "anyOf": [ { "type": "string" }, { "items": { "type": "string" }, "type": "array" } ], "title": "Text", "description": "The raw input text or list of texts to be chunked. Use this OR the 'file' field." }, "embedding_model": { "type": "string", "title": "Embedding Model", "default": "minishlab/potion-base-8M", "description": "Model identifier or embedding model instance" }, "threshold": { "anyOf": [ { "type": "string", "const": "auto" }, { "type": "number" }, { "type": "integer" } ], "title": "Threshold", "default": "auto", "description": "When in the range [0,1], denotes the similarity threshold to consider sentences similar. When in the range (1,100], interprets the given value as a percentile threshold. When set to 'auto', the threshold is automatically calculated." }, "mode": { "type": "string", "title": "Mode", "default": "window", "description": "Mode for grouping sentences, either 'cumulative' or 'window'" }, "chunk_size": { "type": "integer", "title": "Chunk Size", "default": 512, "description": "Maximum tokens per chunk" }, "similarity_window": { "type": "integer", "title": "Similarity Window", "default": 1, "description": "Number of sentences to consider for similarity threshold calculation" }, "min_sentences": { "type": "integer", "title": "Min Sentences", "default": 1, "description": "Minimum number of sentences per chunk" }, "min_characters_per_sentence": { "type": "integer", "title": "Min Characters Per Sentence", "default": 1, "description": "Minimum number of characters per sentence" }, "threshold_step": { "type": "number", "title": "Threshold Step", "default": 0.01, "description": "Step size for threshold calculation" }, "delim": { "anyOf": [ { "type": "string" }, { "items": { "type": "string" }, "type": "array" } ], "title": "Delim", "default": [ "\n", ".", "!", "?" ], "description": "Delimiters to split sentences on" }, "skip_window": { "type": "integer", "title": "Skip Window", "default": 1, "description": "Number of chunks to skip when looking for similarities" }, "return_type": { "type": "string", "title": "Return Type", "default": "chunks", "enum": [ "texts", "chunks" ], "description": "Return type for the chunking process. If 'chunks', returns a list of SemanticChunk objects. If 'texts', returns a list of strings." } } } } } }, "responses": { "200": { "description": "Successful Response: A list of semantic chunk objects.", "content": { "application/json": { "schema": { "title": "SDPMChunkResponse", "type": "array", "items": { "$ref": "#/components/schemas/SemanticChunkSchema" }, "description": "A list containing semantic chunk objects (as SDPM is a type of semantic chunking), detailing segments and sentences with optional embeddings." } } } } } } }, "/v1/chunk/late": { "post": { "summary": "Late Chunker", "description": "Chunk the given text or file using the Late Chunker.", "operationId": "get_late_chunks_v1_chunk_late_post", "requestBody": { "required": true, "content": { "multipart/form-data": { "schema": { "type": "object", "properties": { "file": { "type": "string", "format": "binary", "description": "The file containing text to be chunked. Use this OR the 'text' field." }, "text": { "anyOf": [ { "type": "string" }, { "items": { "type": "string" }, "type": "array" } ], "title": "Text", "description": "The raw input text or list of texts to be chunked. Use this OR the 'file' field." }, "embedding_model": { "type": "string", "title": "Embedding Model", "default": "sentence-transformers/all-minilm-l6-v2", "description": "SentenceTransformer model to use for embedding" }, "chunk_size": { "type": "integer", "title": "Chunk Size", "default": 512, "description": "Maximum number of tokens per chunk" }, "recipe": { "type": "string", "title": "Recipe", "default": "default", "description": "Rules to split text by. Find all recipes on our [Hugging Face](https://huggingface.co/datasets/chonkie-ai/recipes)." }, "lang": { "type": "string", "title": "Language", "default": "en", "description": "Language of the text. This must match the language of the recipe." }, "min_characters_per_chunk": { "type": "integer", "title": "Min Characters Per Sentence", "default": 24, "description": "Minimum number of characters per sentence" } } } } } }, "responses": { "200": { "description": "Successful Response: A list of late chunk objects.", "content": { "application/json": { "schema": { "title": "LateChunkResponse", "type": "array", "items": { "$ref": "#/components/schemas/LateChunkSchema" }, "description": "A list containing late chunk objects, detailing segments, sentences, and an optional chunk-level embedding." } } } } } } } }, "components": { "securitySchemes": { "BearerAuth": { "type": "http", "scheme": "bearer", "description": "Your API Key from the [Chonkie Cloud dashboard](https://cloud.chonkie.ai/keys)" } }, "schemas": { "Chunk": { "title": "Chunk", "description": "Represents a single chunk of text with metadata.", "type": "object", "properties": { "text": { "type": "string", "title": "Text", "description": "The actual text content of the chunk." }, "start_index": { "type": "integer", "title": "Start Index", "description": "The starting character index of the chunk within the original input text." }, "end_index": { "type": "integer", "title": "End Index", "description": "The ending character index (exclusive) of the chunk within the original input text." }, "token_count": { "type": "integer", "title": "Token Count", "description": "The number of tokens in this specific chunk, according to the tokenizer used." } } }, "SentenceSchema": { "title": "Sentence", "description": "Represents a single sentence within a chunk.", "type": "object", "properties": { "text": { "type": "string", "title": "Text", "description": "The actual text content of the sentence." }, "start_index": { "type": "integer", "title": "Start Index", "description": "The starting character index of the sentence within the original input text." }, "end_index": { "type": "integer", "title": "End Index", "description": "The ending character index (exclusive) of the sentence within the original input text." }, "token_count": { "type": "integer", "title": "Token Count", "description": "The number of tokens in this specific sentence, according to the tokenizer used." } } }, "SemanticSentenceSchema": { "title": "SemanticSentence", "description": "Represents a single sentence within a semantic chunk, potentially including an embedding.", "type": "object", "properties": { "text": { "type": "string", "title": "Text", "description": "The actual text content of the sentence." }, "start_index": { "type": "integer", "title": "Start Index", "description": "The starting character index of the sentence within the original input text." }, "end_index": { "type": "integer", "title": "End Index", "description": "The ending character index (exclusive) of the sentence within the original input text." }, "token_count": { "type": "integer", "title": "Token Count", "description": "The number of tokens in this specific sentence, according to the tokenizer used." }, "embedding": { "type": [ "array", "null" ], "items": { "type": "number", "format": "float" }, "title": "Embedding", "description": "Optional embedding vector for the sentence (array of floats or null)." } } }, "SentenceChunkSchema": { "title": "SentenceChunk", "description": "Represents a chunk of text composed of sentences.", "type": "object", "properties": { "text": { "type": "string", "title": "Text", "description": "The actual text content of the chunk." }, "start_index": { "type": "integer", "title": "Start Index", "description": "The starting character index of the chunk within the original input text." }, "end_index": { "type": "integer", "title": "End Index", "description": "The ending character index (exclusive) of the chunk within the original input text." }, "token_count": { "type": "integer", "title": "Token Count", "description": "The number of tokens in this specific chunk, according to the tokenizer used." }, "sentences": { "type": "array", "items": { "$ref": "#/components/schemas/SentenceSchema" }, "title": "Sentences", "description": "List of sentences contained within this chunk." } } }, "RecursiveChunkSchema": { "title": "RecursiveChunk", "description": "Represents a chunk generated by the Recursive Chunker, including its level.", "type": "object", "properties": { "text": { "type": "string", "title": "Text", "description": "The actual text content of the chunk." }, "start_index": { "type": "integer", "title": "Start Index", "description": "The starting character index of the chunk within the original input text." }, "end_index": { "type": "integer", "title": "End Index", "description": "The ending character index (exclusive) of the chunk within the original input text." }, "token_count": { "type": "integer", "title": "Token Count", "description": "The number of tokens in this specific chunk, according to the tokenizer used." }, "level": { "type": "integer", "title": "Level", "description": "The level of this chunk in the recursive splitting process." } } }, "SemanticChunkSchema": { "title": "SemanticChunk", "description": "Represents a chunk generated by semantic chunking methods (Semantic, SDPM), containing sentences potentially with embeddings.", "type": "object", "properties": { "text": { "type": "string", "title": "Text", "description": "The actual text content of the chunk." }, "start_index": { "type": "integer", "title": "Start Index", "description": "The starting character index of the chunk within the original input text." }, "end_index": { "type": "integer", "title": "End Index", "description": "The ending character index (exclusive) of the chunk within the original input text." }, "token_count": { "type": "integer", "title": "Token Count", "description": "The number of tokens in this specific chunk, according to the tokenizer used." }, "sentences": { "type": "array", "items": { "$ref": "#/components/schemas/SemanticSentenceSchema" }, "title": "Sentences", "description": "List of semantic sentences contained within this chunk." } } }, "LateChunkSchema": { "title": "LateChunk", "description": "Represents a chunk generated by the Late Chunker, containing sentences and an optional chunk-level embedding.", "type": "object", "properties": { "text": { "type": "string", "title": "Text", "description": "The actual text content of the chunk." }, "start_index": { "type": "integer", "title": "Start Index", "description": "The starting character index of the chunk within the original input text." }, "end_index": { "type": "integer", "title": "End Index", "description": "The ending character index (exclusive) of the chunk within the original input text." }, "token_count": { "type": "integer", "title": "Token Count", "description": "The number of tokens in this specific chunk, according to the tokenizer used." }, "sentences": { "type": "array", "items": { "$ref": "#/components/schemas/SentenceSchema" }, "title": "Sentences", "description": "List of standard sentences contained within this chunk." }, "embedding": { "type": [ "array", "null" ], "items": { "type": "number", "format": "float" }, "title": "Embedding", "description": "Optional embedding vector for the entire chunk (array of floats or null)." } } }, "ErrorRES": { "properties": { "message": { "type": "string", "title": "Message" }, "status": { "type": "integer", "title": "Status", "default": 500 } }, "type": "object", "title": "ErrorRES", "description": "Error response for the Chonkie API." }, "HTTPValidationError": { "properties": { "detail": { "items": { "$ref": "#/components/schemas/ValidationError" }, "type": "array", "title": "Detail" } }, "type": "object", "title": "HTTPValidationError" }, "LateChunkerREQ": { "title": "LateChunkerREQ", "description": "Data to pass to the Late Chunker.", "type": "object", "required": [ "text" ], "properties": { "text": { "anyOf": [ { "type": "string" }, { "items": { "type": "string" }, "type": "array" } ], "title": "Text", "description": "The input text or list of texts to be chunked." }, "embedding_model": { "type": "string", "title": "Embedding Model", "default": "sentence-transformers/all-minilm-l6-v2", "description": "SentenceTransformer model to use for embedding" }, "chunk_size": { "type": "integer", "title": "Chunk Size", "default": 512, "description": "Maximum number of tokens per chunk" }, "recipe": { "type": "string", "title": "Recipe", "default": "default", "description": "Rules to split text by. Find all recipes on our [Hugging Face](https://huggingface.co/datasets/chonkie-ai/recipes)." }, "lang": { "type": "string", "title": "Language", "default": "en", "description": "Language of the text. This must match the language of the recipe." }, "min_characters_per_chunk": { "type": "integer", "title": "Min Characters Per Sentence", "default": 24, "description": "Minimum number of characters per sentence" } }, "additionalProperties": false }, "RecursiveChunkerREQ": { "title": "RecursiveChunkerREQ", "description": "Data to pass to the Recursive Character Text Splitter.", "type": "object", "required": [ "text" ], "properties": { "text": { "anyOf": [ { "type": "string" }, { "items": { "type": "string" }, "type": "array" } ], "title": "Text", "description": "The input text or list of texts to be chunked." }, "tokenizer_or_token_counter": { "type": "string", "title": "Tokenizer Or Token Counter", "default": "gpt2", "description": "Tokenizer to use. Can be a string identifier or a tokenizer instance" }, "chunk_size": { "type": "integer", "title": "Chunk Size", "default": 512, "description": "Maximum number of tokens per chunk" }, "recipe": { "type": "string", "title": "Recipe", "default": "default", "description": "Rules to split text by. Find all recipes on our [Hugging Face](https://huggingface.co/datasets/chonkie-ai/recipes)." }, "lang": { "type": "string", "title": "Language", "default": "en", "description": "Language of the text. This must match the language of the recipe." }, "min_characters_per_chunk": { "type": "integer", "title": "Min Characters Per Chunk", "default": 1, "description": "Minimum number of characters per chunk" }, "return_type": { "type": "string", "title": "Return Type", "default": "chunks", "enum": [ "texts", "chunks" ], "description": "Whether to return chunks as text strings or as RecursiveChunk objects" } }, "additionalProperties": false }, "SDPMChunkerREQ": { "title": "SDPMChunkerREQ", "description": "Data to pass to the SDPM Chunker.", "type": "object", "required": [ "text" ], "properties": { "text": { "anyOf": [ { "type": "string" }, { "items": { "type": "string" }, "type": "array" } ], "title": "Text", "description": "The input text or list of texts to be chunked." }, "embedding_model": { "type": "string", "title": "Embedding Model", "default": "minishlab/potion-base-8M", "description": "Model identifier or embedding model instance" }, "threshold": { "anyOf": [ { "type": "string", "const": "auto" }, { "type": "number" }, { "type": "integer" } ], "title": "Threshold", "default": "auto", "description": "When in the range [0,1], denotes the similarity threshold to consider sentences similar. When in the range (1,100], interprets the given value as a percentile threshold. When set to 'auto', the threshold is automatically calculated." }, "mode": { "type": "string", "title": "Mode", "default": "window", "description": "Mode for grouping sentences, either 'cumulative' or 'window'" }, "chunk_size": { "type": "integer", "title": "Chunk Size", "default": 512, "description": "Maximum tokens per chunk" }, "similarity_window": { "type": "integer", "title": "Similarity Window", "default": 1, "description": "Number of sentences to consider for similarity threshold calculation" }, "min_sentences": { "type": "integer", "title": "Min Sentences", "default": 1, "description": "Minimum number of sentences per chunk" }, "min_characters_per_sentence": { "type": "integer", "title": "Min Characters Per Sentence", "default": 1, "description": "Minimum number of characters per sentence" }, "threshold_step": { "type": "number", "title": "Threshold Step", "default": 0.01, "description": "Step size for threshold calculation" }, "delim": { "anyOf": [ { "type": "string" }, { "items": { "type": "string" }, "type": "array" } ], "title": "Delim", "default": [ "\n", ".", "!", "?" ], "description": "Delimiters to split sentences on" }, "skip_window": { "type": "integer", "title": "Skip Window", "default": 1, "description": "Number of chunks to skip when looking for similarities" }, "return_type": { "type": "string", "title": "Return Type", "default": "chunks", "enum": [ "texts", "chunks" ], "description": "Return type for the chunking process. If 'chunks', returns a list of SemanticChunk objects. If 'texts', returns a list of strings." } }, "additionalProperties": false }, "SemanticChunkerREQ": { "title": "SemanticChunkerREQ", "description": "Data to pass to the Semantic Chunker.", "type": "object", "required": [ "text" ], "properties": { "text": { "anyOf": [ { "type": "string" }, { "items": { "type": "string" }, "type": "array" } ], "title": "Text", "description": "The input text or list of texts to be chunked." }, "embedding_model": { "type": "string", "title": "Embedding Model", "default": "minishlab/potion-base-8M", "description": "Model identifier or embedding model instance" }, "threshold": { "anyOf": [ { "type": "string", "const": "auto" }, { "type": "number" }, { "type": "integer" } ], "title": "Threshold", "default": "auto", "description": "When in the range [0,1], denotes the similarity threshold to consider sentences similar. When in the range (1,100], interprets the given value as a percentile threshold. When set to 'auto', the threshold is automatically calculated." }, "chunk_size": { "type": "integer", "title": "Chunk Size", "default": 512, "description": "Maximum tokens per chunk" }, "similarity_window": { "type": "integer", "title": "Similarity Window", "default": 1, "description": "Number of sentences to consider for similarity threshold calculation" }, "min_sentences": { "type": "integer", "title": "Min Sentences", "default": 1, "description": "Minimum number of sentences per chunk" }, "min_chunk_size": { "type": "integer", "title": "Min Chunk Size", "default": 2, "description": "Minimum tokens per chunk" }, "min_characters_per_sentence": { "type": "integer", "title": "Min Characters Per Sentence", "default": 1, "description": "Minimum number of characters per sentence" }, "threshold_step": { "type": "number", "title": "Threshold Step", "default": 0.01, "description": "Step size for similarity threshold calculation" }, "delim": { "anyOf": [ { "type": "string" }, { "items": { "type": "string" }, "type": "array" } ], "title": "Delim", "default": [ "\n", ".", "!", "?" ], "description": "Delimiters to split sentences on. Default is ['.', '!', '?', '\\n']" }, "include_delim": { "anyOf": [ { "type": "string", "enum": [ "prev", "next" ] }, { "type": "null" } ], "title": "Include Delim", "default": "prev", "description": "Include delimiters in the chunk text. If so, specifies whether to include in the previous or next chunk" }, "return_type": { "type": "string", "title": "Return Type", "default": "chunks", "enum": [ "texts", "chunks" ], "description": "Return type for chunking. If 'chunks', returns a list of SemanticChunk objects. If 'texts', returns a list of strings" } }, "additionalProperties": false }, "SentenceChunkerREQ": { "title": "SentenceChunkerREQ", "description": "Data to pass to the Sentence Chunker.", "type": "object", "required": [ "text" ], "properties": { "text": { "anyOf": [ { "type": "string" }, { "items": { "type": "string" }, "type": "array" } ], "title": "Text", "description": "The input text or list of texts to be chunked." }, "tokenizer_or_token_counter": { "type": "string", "title": "Tokenizer Or Token Counter", "default": "gpt2", "description": "Tokenizer to use. Can be a string identifier or a tokenizer instance" }, "chunk_size": { "type": "integer", "title": "Chunk Size", "default": 512, "description": "Maximum number of tokens per chunk" }, "chunk_overlap": { "type": "integer", "title": "Chunk Overlap", "default": 0, "description": "Number of overlapping tokens between chunks" }, "min_sentences_per_chunk": { "type": "integer", "title": "Min Sentences Per Chunk", "default": 1, "description": "Minimum number of sentences to include in each chunk" }, "min_characters_per_sentence": { "type": "integer", "title": "Min Characters Per Sentence", "default": 1, "description": "Minimum number of characters per sentence" }, "approximate": { "type": "boolean", "title": "Approximate", "default": true, "description": "Use approximate token counting for faster processing (deprecated)" }, "delim": { "anyOf": [ { "type": "string" }, { "items": { "type": "string" }, "type": "array" } ], "title": "Delim", "default": [ "\n", ".", "!", "?" ], "description": "Delimiters to split sentences on" }, "include_delim": { "anyOf": [ { "type": "string", "enum": [ "prev", "next" ] }, { "type": "null" } ], "title": "Include Delim", "default": "prev", "description": "Include delimiters in the chunk text. If so, specifies whether to include in the previous or next chunk" }, "return_type": { "type": "string", "title": "Return Type", "default": "chunks", "enum": [ "texts", "chunks" ], "description": "Whether to return chunks as text strings or as SentenceChunk objects" } }, "additionalProperties": false }, "TokenChunkerREQ": { "title": "TokenChunkerREQ", "description": "Data to pass to the Token Chunker.", "type": "object", "required": [ "text" ], "properties": { "text": { "anyOf": [ { "type": "string" }, { "items": { "type": "string" }, "type": "array" } ], "title": "Text", "description": "The input text or list of texts to be chunked." }, "tokenizer": { "type": "string", "title": "Tokenizer", "default": "gpt2", "description": "Tokenizer to use. Can be a string identifier or a tokenizer instance" }, "chunk_size": { "type": "integer", "title": "Chunk Size", "default": 512, "description": "Maximum number of tokens per chunk" }, "chunk_overlap": { "type": "integer", "title": "Chunk Overlap", "default": 0, "description": "Number or percentage of overlapping tokens between chunks" }, "return_type": { "type": "string", "title": "Return Type", "default": "chunks", "enum": [ "texts", "chunks" ], "description": "Whether to return chunks as Chunk objects or plain text strings" } }, "additionalProperties": false }, "ValidationError": { "properties": { "loc": { "items": { "anyOf": [ { "type": "string" }, { "type": "integer" } ] }, "type": "array", "title": "Location" }, "msg": { "type": "string", "title": "Message" }, "type": { "type": "string", "title": "Error Type" } }, "type": "object", "title": "ValidationError" }, "WordChunkerREQ": { "title": "WordChunkerREQ", "description": "Data to pass to the Word Chunker.", "type": "object", "required": [ "text" ], "properties": { "text": { "anyOf": [ { "type": "string" }, { "items": { "type": "string" }, "type": "array" } ], "title": "Text", "description": "The input text or list of texts to be chunked." }, "tokenizer_or_token_counter": { "type": "string", "title": "Tokenizer Or Token Counter", "default": "gpt2" }, "chunk_size": { "type": "integer", "title": "Chunk Size", "default": 512 }, "chunk_overlap": { "type": "integer", "title": "Chunk Overlap", "default": 0 }, "return_type": { "type": "string", "title": "Return Type", "default": "chunks", "enum": [ "texts", "chunks" ] } }, "additionalProperties": false } } } }