// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 //! Ordered reasoning, text, and tool-call parsing through one state machine. //! //! Compatible configured families use this path by default; `DYN_PARSER_VERSION=1` //! selects the legacy parsers when available. //! //! # What this replaces //! //! The legacy Qwen3 route chains two independent parsers: the reasoning //! parser strips `...` across the whole stream into one assembled //! `reasoning_content`, and the tool-call jail then scans whatever is left. That chain //! cannot represent WHERE a thought happened. Given //! //! ```text //! Look it up.…Now answer.It's 18C. //! ``` //! //! it serves `reasoning("Look it up.Now answer.")`, then the call, then `text("It's 18C.")`: //! the second thought moved ahead of the call it followed and fused with the first. A //! client rendering thoughts inline puts them in the wrong place, and a client counting //! reasoning turns sees one where there were two. //! //! Ordering is not a field the split can add — it is lost at the seam between the two //! parsers. So when this path is enabled, ONE [`UnifiedParser`] owns the whole grammar //! and emits deltas in the order the model produced them, and this module maps those //! deltas onto the OpenAI streaming/batch wire shapes. //! //! # Why one chunk per delta //! //! [`ChatCompletionStreamResponseDelta`] carries `content`, `reasoning_content` and //! `tool_calls` side by side with no way to say which came first. Packing a whole //! `push` into one delta object would throw away exactly the ordering this path exists //! to preserve, so every [`UnifiedParserEvent`] becomes its own chunk. use std::collections::{HashMap, HashSet, VecDeque}; use std::sync::LazyLock; use async_stream::stream; use dynamo_parsers::tool_calling::ToolDefinition; use dynamo_parsers_v2::{ ArgumentFraming, InvalidGuidedPayloadPolicy, MalformedToolCallPolicy, Tool, ToolCallPolicy, UnifiedEvent, UnifiedParser, UnifiedParserContinuation, UnifiedParserEvent, UnifiedParserInit, UnifiedParserOutput, UnifiedParserStartingState, UnifiedToolOutputMode, create_unified_parser_for_family, }; use dynamo_protocols::types::{ ChatChoiceLogprobs, ChatChoiceStream, ChatCompletionMessageContent, ChatCompletionMessageToolCall, ChatCompletionMessageToolCallChunk, ChatCompletionStreamResponseDelta, FinishReason, FunctionCall, FunctionCallStream, FunctionType, }; use dynamo_runtime::error::{DynamoError, ErrorClass}; use dynamo_runtime::protocols::annotated::Annotated; use dynamo_runtime::protocols::maybe_error::MaybeError; use futures::{Stream, StreamExt}; use uuid::Uuid; use crate::protocols::openai::GuidedToolConstraint; use super::NvCreateChatCompletionStreamResponse; #[cfg(test)] use dynamo_protocols::types::ChatCompletionToolChoiceOption; /// The `dynamo-parsers-v2` unified family that serves Qwen3. /// /// `REGISTERED_UNIFIED_FAMILIES` accepts both `qwen3` and the `qwen3_coder` alias for /// the same XML grammar; `qwen3` is the canonical registry name and the one the /// conformance corpus uses, so it is what this module passes and logs. pub(crate) const QWEN3_UNIFIED_FAMILY: &str = "qwen3"; pub(crate) const KIMI_K2_UNIFIED_FAMILY: &str = "kimi_k2"; pub(crate) const KIMI_K3_UNIFIED_FAMILY: &str = "kimi_k3"; pub(crate) const DEEPSEEK_V41_UNIFIED_FAMILY: &str = "deepseek_v41"; pub(crate) const DEEPSEEK_V4_UNIFIED_FAMILY: &str = "deepseek_v4"; pub(crate) const GEMMA4_UNIFIED_FAMILY: &str = "gemma4"; /// The empty thought channel Gemma 4 opens with thinking off. pub(crate) const GEMMA4_EMPTY_THOUGHT_CHANNEL: &str = "<|channel>thought\n"; pub(crate) const GLM47_UNIFIED_FAMILY: &str = "glm47"; pub(crate) fn incomplete_tool_call_error() -> DynamoError { DynamoError::builder() .class(ErrorClass::BackendProtocol) .message("The model ended with an incomplete tool call.") .public_message("The model ended with an incomplete tool call.") .build() } fn parser_error_annotation( error: anyhow::Error, ) -> Annotated { Annotated::from_err(DynamoError::from( error.as_ref() as &(dyn std::error::Error + 'static) )) } fn incomplete_glm_terminal( family: &str, reason: Option, incomplete: impl FnOnce() -> bool, ) -> Option { (family == GLM47_UNIFIED_FAMILY && matches!(reason, Some(FinishReason::Stop | FinishReason::ToolCalls)) && incomplete()) .then(incomplete_tool_call_error) } pub(crate) const MUSE_UNIFIED_FAMILY: &str = "muse_glimmer"; /// The `dynamo-parsers-v2` unified family that serves Beam. Dynamo has no v1 Beam /// parser, so the `beam` + `beam` pair routes to the unified parser without /// `DYN_PARSER_VERSION=2`, like DeepSeek V4.1. pub(crate) const BEAM_UNIFIED_FAMILY: &str = "beam"; pub(crate) const MUSE_SPARK_UNIFIED_FAMILY: &str = "muse_spark"; /// Nemotron 3 (Ultra, Super, 3.5 Super): `nemotron_v3` forced reasoning over the /// Qwen3-Coder XML tool grammar. Model cards pair `--dyn-tool-call-parser qwen3_coder` /// with `--dyn-reasoning-parser nemotron_v3`. pub(crate) const NEMOTRON_V3_UNIFIED_FAMILY: &str = "nemotron_v3"; /// Dynamo's `--dyn-tool-call-parser` name that pairs into [`QWEN3_UNIFIED_FAMILY`]. const QWEN3_TOOL_CALL_PARSER: &str = "qwen3_coder"; fn uses_qwen_xml_grammar(family: &str) -> bool { matches!(family, QWEN3_UNIFIED_FAMILY | NEMOTRON_V3_UNIFIED_FAMILY) } /// A family that borrows another family's tool grammar names the owner through its /// reasoning half: `qwen3_coder` + `nemotron_v3` is Nemotron, not Qwen3, because the /// forced-reasoning default lives in the reasoning parser and the XML grammar is shared. fn shared_grammar_family(tool_family: &str, reasoning_family: &str) -> Option<&'static str> { match (tool_family, reasoning_family) { (QWEN3_UNIFIED_FAMILY, NEMOTRON_V3_UNIFIED_FAMILY) => Some(NEMOTRON_V3_UNIFIED_FAMILY), _ => None, } } const DYNAMO_FAMILY_ALIASES: &[(&str, &str)] = &[ ("deepseek-v4", DEEPSEEK_V4_UNIFIED_FAMILY), ("deepseekv4", DEEPSEEK_V4_UNIFIED_FAMILY), ("muse", MUSE_UNIFIED_FAMILY), ("gemma-4", GEMMA4_UNIFIED_FAMILY), ("glm45", GLM47_UNIFIED_FAMILY), ("kimi_k25", KIMI_K2_UNIFIED_FAMILY), ]; /// Canonical family names and aliases share one route for registration, stream, and batch. pub(crate) fn canonical_family(parser: &str) -> Option<&'static str> { let parser = DYNAMO_FAMILY_ALIASES .iter() .find_map(|(alias, family)| (*alias == parser).then_some(*family)) .unwrap_or(parser); dynamo_parsers_v2::canonical_unified_family(parser) } /// Aliases that still route but are scheduled for removal, paired with the family /// to configure instead. `muse` predates Muse Spark Open and now reads as "any Muse /// model" while it selects only Muse Glimmer; a Spark deployment configured with it /// starts without complaint and runs the wrong parser. /// TODO(next minor): remove these from `DYNAMO_FAMILY_ALIASES` and `FAMILY_NAMES` so /// the alias becomes a startup error. pub(crate) const DEPRECATED_FAMILY_ALIASES: &[(&str, &str)] = &[("muse", MUSE_UNIFIED_FAMILY)]; /// The startup warning for a deprecated alias, or `None` for any other selector. pub(crate) fn deprecated_alias_warning(parser: &str) -> Option { DEPRECATED_FAMILY_ALIASES .iter() .find(|(alias, _)| *alias == parser) .map(|(alias, family)| { format!( "parser `{alias}` is a deprecated alias of `{family}` and will be removed in a \ future release; configure `{family}` instead. Muse Spark Open is a different \ parser: `muse_spark_open`" ) }) } pub(crate) static FAMILY_NAMES: LazyLock> = LazyLock::new(|| { let mut names = dynamo_parsers_v2::REGISTERED_UNIFIED_FAMILIES.to_vec(); names.extend(DYNAMO_FAMILY_ALIASES.iter().map(|(alias, _)| *alias)); names.sort_unstable(); names.dedup(); names }); /// A configured half may select a unified family, but two configured halves must agree. pub(crate) fn configured_family( tool_call_parser: Option<&str>, reasoning_parser: Option<&str>, ) -> Option<&'static str> { match (tool_call_parser, reasoning_parser) { (Some(tool), Some(reasoning)) => { let tool_family = canonical_family(tool)?; let reasoning_family = canonical_family(reasoning)?; if tool_family == reasoning_family { return Some(tool_family); } shared_grammar_family(tool_family, reasoning_family) } (Some(parser), None) | (None, Some(parser)) => canonical_family(parser), (None, None) => None, } } pub(crate) fn is_v2_configured_family( tool_call_parser: Option<&str>, reasoning_parser: Option<&str>, ) -> bool { configured_family(tool_call_parser, reasoning_parser).is_some() } /// The unified family to actually use for this parser pair, or `None` to keep the /// existing split reasoning-parser + tool-call-jail path. pub(crate) fn selected_family( tool_call_parser: Option<&str>, reasoning_parser: Option<&str>, ) -> Option<&'static str> { let selection = super::tool_parser_v2::selected_version() .unwrap_or_else(|error| panic!("invalid parser routing configuration: {error:#}")); let selected = selected_family_for_selection(tool_call_parser, reasoning_parser, selection); tracing::debug!( target: "dynamo_unified", ?tool_call_parser, ?reasoning_parser, ?selection, ?selected, "unified parser path decision" ); selected } pub(crate) fn selected_family_for_selection( tool_call_parser: Option<&str>, reasoning_parser: Option<&str>, selection: dynamo_runtime::config::ParserVersion, ) -> Option<&'static str> { // Qwen3 thinking models can use Hermes syntax; reasoning-only Nemotron cards // do not identify their tool grammar. Auto keeps both on the legacy path. if selection == dynamo_runtime::config::ParserVersion::V1 || (selection == dynamo_runtime::config::ParserVersion::Auto && tool_call_parser.is_none() && matches!( reasoning_parser, Some("qwen3" | "nemotron_v3" | "nemotron3") )) { return None; } configured_family(tool_call_parser, reasoning_parser) } pub(crate) fn selected_content_decoder_family( tool_call_parser: Option<&str>, reasoning_parser: Option<&str>, ) -> Option<&'static str> { selected_family(tool_call_parser, reasoning_parser) } pub(crate) fn selected_batch_family( tool_call_parser: Option<&str>, reasoning_parser: Option<&str>, ) -> Option<&'static str> { selected_family(tool_call_parser, reasoning_parser) } pub(crate) fn selected_request_family( tool_call_parser: Option<&str>, reasoning_parser: Option<&str>, _tool_choice: Option<&dynamo_protocols::types::ChatCompletionToolChoiceOption>, _constraint: &GuidedToolConstraint, ) -> Option<&'static str> { selected_family(tool_call_parser, reasoning_parser) } /// Whether the operator explicitly selected V2 parsing. pub(crate) fn parsers_v2_selected() -> bool { super::tool_parser_v2::enabled() } /// Request facts for parser initialization and response interpretation. #[derive(Clone, Copy, Debug, Default)] pub(crate) struct UnifiedRequestPolicy { pub reasoning_disabled: bool, pub structured_response: bool, pub multiple_choices: bool, } impl UnifiedRequestPolicy { /// The policy as the installed grammar leaves it: a `response_format` the grammar /// holds inside the model's answer channel is native channel output, not the bare /// JSON document the structured-response probes expect. fn under(self, constraint: &GuidedToolConstraint) -> Self { Self { structured_response: self.structured_response && !constraint.frames_structured_answer(), ..self } } fn harmony_prefill_cache( self, family: &str, prefix: Option>, ) -> Option { if family == "gpt_oss" && self.multiple_choices { prefix.map(HarmonyPrefill::new) } else { None } } pub(crate) fn resolve_stream_prefill( self, family: &str, mut prefill: StreamPrefill, constraint: &GuidedToolConstraint, has_explicit_guided_decoding: bool, harmony_required_array: impl FnOnce() -> bool, ) -> StreamPrefill { if family == "gpt_oss" && (has_explicit_guided_decoding || constraint.installs_guided_json() && !harmony_required_array()) { return UnifiedParserStartingState::Response.into(); } prefill.starting_state = self.starting_state(family, prefill.starting_state); prefill } fn starting_state( self, family: &str, inferred: UnifiedParserStartingState, ) -> UnifiedParserStartingState { if self.reasoning_disabled && matches!(family, "gemma4" | "minimax_m3" | NEMOTRON_V3_UNIFIED_FAMILY) { // Gemma's Response state preserves literal thought delimiters and still parses tools. // Nemotron 3 defaults to an implicit open thought; `enable_thinking=false` renders // a closed `` pair instead, so generation starts in the answer. // MiniMax M3 reaches this only when the rendered prompt closed the thought with // `` or a grammar owns the output (the preprocessor's rendered-prompt // gate), so there is no reasoning channel left to parse. UnifiedParserStartingState::Response } else { inferred } } fn inspects_json_prefill(self, constraint: &GuidedToolConstraint) -> bool { self.structured_response || constraint.installs_guided_json() } fn batch_starting_state( self, family: &str, content: &str, ) -> anyhow::Result { let inferred = if self.structured_response && structured_json_response_prefill(family, content, true)? == Some(UnifiedParserStartingState::Response) { UnifiedParserStartingState::Response } else { detect_prefill(family, content)? }; Ok(self.starting_state(family, inferred)) } } /// Scalar and container roots can also begin genuine reasoning. Hold only that /// ambiguous prefix; process each incoming byte once and release it when a suffix /// disproves JSON. #[derive(Default)] struct JsonPrefillProbe { text: String, state: JsonPrefixState, starts_json: Option, inspected_bytes: usize, container_start_validated: bool, marker_probe: Option, marker_suffix: String, marker_decision: Option, marker_inspected_bytes: usize, } #[derive(Default)] enum JsonPrefixState { #[default] Start, ContainerStart { opener: u8, validated: bool, }, String { escaped: bool, unicode_left: u8, }, Literal { expected: &'static [u8], position: usize, }, Number(NumberPrefixState), Done, Invalid, } #[derive(Clone, Copy)] enum NumberPrefixState { Minus, Zero, Digits, Dot, Fraction, Exponent, ExponentSign, ExponentDigits, } impl JsonPrefillProbe { fn push(&mut self, text: &str) { self.text.push_str(text); for byte in text.bytes() { self.inspected_bytes += 1; let next_state = match &self.state { JsonPrefixState::Start => match byte { b' ' | b'\r' | b'\n' | b'\t' => JsonPrefixState::Start, b'{' | b'[' => JsonPrefixState::ContainerStart { opener: byte, validated: false, }, b'"' => JsonPrefixState::String { escaped: false, unicode_left: 0, }, b'-' => JsonPrefixState::Number(NumberPrefixState::Minus), b'0' => JsonPrefixState::Number(NumberPrefixState::Zero), b'1'..=b'9' => JsonPrefixState::Number(NumberPrefixState::Digits), b't' => JsonPrefixState::Literal { expected: b"true", position: 1, }, b'f' => JsonPrefixState::Literal { expected: b"false", position: 1, }, b'n' => JsonPrefixState::Literal { expected: b"null", position: 1, }, _ => JsonPrefixState::Invalid, }, JsonPrefixState::String { escaped, unicode_left, } => { if *unicode_left > 0 { if byte.is_ascii_hexdigit() { JsonPrefixState::String { escaped: false, unicode_left: unicode_left - 1, } } else { JsonPrefixState::Invalid } } else if *escaped { match byte { b'u' => JsonPrefixState::String { escaped: false, unicode_left: 4, }, b'"' | b'\\' | b'/' | b'b' | b'f' | b'n' | b'r' | b't' => { JsonPrefixState::String { escaped: false, unicode_left: 0, } } _ => JsonPrefixState::Invalid, } } else { match byte { b'"' => JsonPrefixState::Done, b'\\' => JsonPrefixState::String { escaped: true, unicode_left: 0, }, 0..=31 => JsonPrefixState::Invalid, _ => JsonPrefixState::String { escaped: false, unicode_left: 0, }, } } } JsonPrefixState::Literal { expected, position } => { if byte != expected[*position] { JsonPrefixState::Invalid } else if position + 1 == expected.len() { JsonPrefixState::Done } else { JsonPrefixState::Literal { expected, position: position + 1, } } } JsonPrefixState::Number(number) => { use NumberPrefixState::*; let next = match (*number, byte) { (Minus, b'0') => Some(Zero), (Minus, b'1'..=b'9') => Some(Digits), (Digits, b'0'..=b'9') => Some(Digits), (Zero | Digits, b'.') => Some(Dot), (Dot | Fraction, b'0'..=b'9') => Some(Fraction), (Zero | Digits | Fraction, b'e' | b'E') => Some(Exponent), (Exponent, b'+' | b'-') => Some(ExponentSign), (Exponent | ExponentSign | ExponentDigits, b'0'..=b'9') => { Some(ExponentDigits) } _ => None, }; if let Some(next) = next { JsonPrefixState::Number(next) } else if matches!(number, Zero | Digits | Fraction | ExponentDigits) && byte.is_ascii_whitespace() { JsonPrefixState::Done } else { JsonPrefixState::Invalid } } JsonPrefixState::Done => { if matches!(byte, b' ' | b'\r' | b'\n' | b'\t') { JsonPrefixState::Done } else { JsonPrefixState::Invalid } } JsonPrefixState::ContainerStart { opener, validated } => { if *validated { JsonPrefixState::ContainerStart { opener: *opener, validated: true, } } else if matches!(byte, b' ' | b'\r' | b'\n' | b'\t') { JsonPrefixState::ContainerStart { opener: *opener, validated: false, } } else { let valid_first_value = matches!( (*opener, byte), (b'{', b'"' | b'}') | ( b'[', b']' | b'{' | b'[' | b'"' | b'-' | b'0' ..=b'9' | b't' | b'f' | b'n', ) ); if valid_first_value { JsonPrefixState::ContainerStart { opener: *opener, validated: true, } } else { JsonPrefixState::Invalid } } } JsonPrefixState::Invalid => JsonPrefixState::Invalid, }; if self.starts_json.is_none() && !matches!(next_state, JsonPrefixState::Start) { self.starts_json = Some(!matches!(next_state, JsonPrefixState::Invalid)); } if matches!( &next_state, JsonPrefixState::ContainerStart { validated: true, .. } ) { self.container_start_validated = true; } self.state = next_state; } } fn unopened_channel_decision( &self, family: &str, terminal: bool, ) -> Option { // Muse Spark's prompt ends at `<|start|>assistant`, so every turn opens with // a ` to=` header whose `t` would otherwise read as JSON `true`. if family == MUSE_SPARK_UNIFIED_FAMILY { const HEADER: &str = "to="; let text = self.text.trim_start(); if text.starts_with(HEADER) { return Some(UnifiedParserStartingState::None); } if !terminal && !text.is_empty() && HEADER.starts_with(text) { return None; } } // JSON roots are unambiguous when the prompt has not opened reasoning; // waiting for a complete scalar would unnecessarily buffer its stream. match self.starts_json { Some(true) => Some(UnifiedParserStartingState::Response), Some(false) => Some(UnifiedParserStartingState::None), None if terminal => Some(UnifiedParserStartingState::Response), None => None, } } fn decision(&self, terminal: bool) -> Option { if self.container_start_validated { return Some(UnifiedParserStartingState::None); } match self.state { JsonPrefixState::ContainerStart { validated: true, .. } if terminal => Some(UnifiedParserStartingState::None), JsonPrefixState::ContainerStart { validated: true, .. } => None, JsonPrefixState::ContainerStart { .. } if terminal => { Some(UnifiedParserStartingState::None) } JsonPrefixState::ContainerStart { .. } => None, JsonPrefixState::Invalid => Some(UnifiedParserStartingState::Reasoning), JsonPrefixState::Done | JsonPrefixState::Number( NumberPrefixState::Zero | NumberPrefixState::Digits | NumberPrefixState::Fraction | NumberPrefixState::ExponentDigits, ) if terminal => Some(UnifiedParserStartingState::None), _ if terminal => Some(UnifiedParserStartingState::Reasoning), _ => None, } } fn has_control_marker(&mut self, family: &str, terminal: bool, latest_text: &str) -> bool { let first_push = self.marker_probe.is_none(); let shield = self .marker_probe .get_or_insert_with(|| QuotedControlMarkerProbe::new(family)); let incoming = if first_push { &self.text } else { latest_text }; let mut unquoted = shield.push(incoming); if terminal { unquoted.push_str(&shield.finish()); } self.marker_suffix.push_str(&unquoted); self.marker_inspected_bytes += self.marker_suffix.len(); let has_candidate = shield .markers .iter() .any(|marker| self.marker_suffix.contains(marker)); let suffix_limit = shield.markers.iter().map(String::len).max().unwrap_or(0); let mut keep_from = self.marker_suffix.len().saturating_sub(suffix_limit); while !self.marker_suffix.is_char_boundary(keep_from) { keep_from += 1; } self.marker_suffix.drain(..keep_from); has_candidate } fn decision_for_family( &mut self, family: &str, terminal: bool, latest_text: &str, ) -> anyhow::Result> { if let Some(decision) = self.marker_decision { return Ok(Some(decision)); } let has_candidate = self.has_control_marker(family, terminal, latest_text); // Only complete unquoted controls require raw-prefix classification; // rescanning growing JSON for every punctuation-bearing chunk is quadratic. if has_candidate && matches!(self.state, JsonPrefixState::ContainerStart { .. }) { self.marker_inspected_bytes += self.text.len(); if let Some(state) = reasoning_marker_prefill(family, &self.text)? { self.marker_decision = Some(state); return Ok(Some(state)); } } Ok(self.decision(terminal)) } } fn structured_json_prefill( family: &str, content: &str, terminal: bool, ) -> anyhow::Result> { let mut probe = JsonPrefillProbe::default(); probe.push(content); probe.decision_for_family(family, terminal, content) } fn structured_json_response_prefill( family: &str, content: &str, terminal: bool, ) -> anyhow::Result> { if let Some(state) = reasoning_marker_prefill(family, content)? { return Ok(Some(state)); } let json_state = structured_json_prefill(family, content, terminal)?; if json_state == Some(UnifiedParserStartingState::None) { return Ok(Some(UnifiedParserStartingState::Response)); } Ok(json_state) } fn reasoning_marker_prefill( family: &str, content: &str, ) -> anyhow::Result> { let has_marker = if let Some((opener, closer)) = reasoning_marker_pair(family) { contains_unquoted_marker(content, opener) || contains_unquoted_marker(content, closer) } else { match family { "kimi_k3" => { kimi_channel_marker_position(content, "<|open|>", "think").is_some() || kimi_channel_marker_position(content, "<|close|>", "think").is_some() } "muse_glimmer" | "muse_spark" => { contains_unquoted_marker(content, "<|start|>") || contains_unquoted_marker(content, "<|eom|>") } BEAM_UNIFIED_FAMILY => BEAM_OPENERS .iter() .chain(BEAM_CLOSERS.iter()) .any(|marker| content.contains(marker)), _ => false, } }; has_marker .then(|| detect_prefill(family, content)) .transpose() } fn starts_bare_json(content: &str) -> bool { matches!(content.trim_start().as_bytes().first(), Some(b'[' | b'{')) } /// Which channel the rendered generation prompt already opened, for the streaming path. /// /// `prompt_injected_reasoning` is the per-request fact the preprocessor already /// computes: the rendered prompt ended with the family's reasoning opener, so generated /// output starts inside a thought the model will close without ever emitting the /// opener. Absent that, the answer is per family. #[derive(Clone)] pub(crate) struct StreamPrefill { pub starting_state: UnifiedParserStartingState, pub prefix: Option>, } // The consumed seed belongs to the request; choices keep independent parser state. struct HarmonyPrefill { prefix: Option>, token_ids: Option>, continuation: Option>, } impl HarmonyPrefill { fn new(prefix: std::sync::Arc) -> Self { Self { prefix: Some(prefix), token_ids: None, continuation: None, } } fn matches_prefix(&self, prefix: Option<&std::sync::Arc>) -> bool { match (self.prefix.as_ref(), prefix) { (Some(cached), Some(current)) => std::sync::Arc::ptr_eq(cached, current), (None, None) => self.continuation.is_some(), _ => false, } } fn release_prefix(&mut self) -> bool { if self.continuation.is_none() { return false; } self.prefix = None; true } fn token_ids(&mut self) -> anyhow::Result<&[u32]> { if self.token_ids.is_none() { self.token_ids = Some( dynamo_parsers_v2::unified::harmony::encode_harmony( self.prefix .as_deref() .ok_or_else(|| anyhow::anyhow!("consumed prefix was released"))?, )? .into_boxed_slice(), ); } Ok(self.token_ids.as_deref().expect("encoded above")) } } fn replay_consumed_harmony( parser: &mut dyn UnifiedParser, tokens: &[u32], chunk_size: usize, ) -> anyhow::Result<()> { let mut discarded = UnifiedParserOutput::default(); for chunk in tokens.chunks(chunk_size) { parser.parse_token_ids_into(chunk, &mut discarded)?; // Only the parser state survives consumed input; generated deltas start fresh. discarded.events.clear(); } Ok(()) } impl From for StreamPrefill { fn from(starting_state: UnifiedParserStartingState) -> Self { Self { starting_state, prefix: None, } } } pub(crate) fn stream_prefill( family: &str, prompt_injected_reasoning: bool, ) -> UnifiedParserStartingState { if prompt_injected_reasoning { return UnifiedParserStartingState::Reasoning; } match family { // Qwen3's generation prompt ends at the assistant header with no channel open, // so the model emits `` itself when it thinks. QWEN3_UNIFIED_FAMILY => UnifiedParserStartingState::None, DEEPSEEK_V41_UNIFIED_FAMILY => UnifiedParserStartingState::None, // Beam's prompt ends at the `<|start|>assistant` header slot; the model opens // its own channel and the grammar owns that open slot. BEAM_UNIFIED_FAMILY => UnifiedParserStartingState::None, // Nemotron 3's template pre-opens `\n` whenever thinking is on, which the // prompt check above normally reports. When the rendered prompt is unavailable, // keep the v1 `nemotron_v3` contract: reasoning is on unless the request turned it // off, and `UnifiedRequestPolicy::starting_state` moves that case to `Response`. NEMOTRON_V3_UNIFIED_FAMILY => UnifiedParserStartingState::Reasoning, _ => UnifiedParserStartingState::None, } } /// Resolve a `Reasoning`-prefill choice's ACTUAL starting state when guided JSON is /// installed, from the shape of that choice's first content-bearing chunk. /// /// Guided decoding forbids the model from emitting the reasoning closer, so a request /// whose rendered prompt already opened reasoning (`prompt_injected_reasoning`) would, /// if started unconditionally at `Reasoning`, have its ENTIRE guided JSON payload /// swallowed as `reasoning_content` with zero tool calls ever extracted — the parser /// waits forever for a closer guided decoding will never produce. But committing to /// `None` unconditionally instead is equally wrong: a turn that genuinely reasons before /// its guided JSON (`reasoningJSON`) would then be misread as pure content, and /// the legitimate `reasoning_content`/`content` split would be lost. /// /// Mirrors `bypass_bare_guided_json`'s shape check in /// `Preprocessor::parse_reasoning_content_from_stream_inner` (preprocessor.rs), which /// makes the identical call for the non-unified reasoning-parser path: a payload whose /// first non-whitespace byte is `[` or `{` is bare guided JSON with no reasoning to /// split out (start at `None`, so the whole payload reaches the tool/response path /// untouched); anything else is ordinary text, meaning this really is a reasoning block /// that will close normally (start at `Reasoning`). /// /// Nonterminal empty and whitespace-only chunks are held by `apply_stream` until this /// check can classify the first non-whitespace content. A terminal empty stream keeps /// the prompt's `Reasoning` state because there is no later payload to classify. /// /// `None` is the family default, and Nemotron 3's default is the implicit thought, so /// bare JSON for that family starts in `Response` explicitly; `None` would hand the /// payload back to reasoning. fn bare_guided_json_prefill( family: &str, first_content: Option<&ChatCompletionMessageContent>, ) -> UnifiedParserStartingState { let Some(ChatCompletionMessageContent::Text(text)) = first_content else { return UnifiedParserStartingState::Reasoning; }; let trimmed = text.trim_start(); if trimmed.is_empty() { return UnifiedParserStartingState::Reasoning; } if starts_bare_json(trimmed) { bare_json_starting_state(family) } else { UnifiedParserStartingState::Reasoning } } /// The starting state that lets a bare guided JSON payload reach the tool parser. fn bare_json_starting_state(family: &str) -> UnifiedParserStartingState { if family == NEMOTRON_V3_UNIFIED_FAMILY { UnifiedParserStartingState::Response } else { UnifiedParserStartingState::None } } /// Which channel the prompt opened, inferred from complete output text. /// /// The batch path has no prompt in hand, only what the model produced, so the channel /// state is read back off the text: a `` opener means the model opened reasoning /// itself; a bare `` with no opener means the prompt had already opened it; and /// neither marker means reasoning never ran for this turn. fn detect_prefill(family: &str, content: &str) -> anyhow::Result { match family { "qwen3" | "deepseek_v4" | "deepseek_v41" | "glm47" | "kimi_k2" | "minimax_m3" | "nemotron_v3" | "hermes" => { let (opener_marker, closer_marker) = reasoning_marker_pair(family) .expect("think-marker families have a configured reasoning pair"); // Compare FIRST-occurrence positions, not mere presence: a prompt that // pre-opened reasoning produces a leading `` with no opener before // it, but a later `...` pair from the model can still follow // in the same output (e.g. after a tool-call gap). Testing "does an opener // exist anywhere" before "does a closer exist anywhere" would misclassify // that case as `None` instead of `Reasoning`. let opener = first_unquoted_marker_position(content, opener_marker); let closer = first_unquoted_marker_position(content, closer_marker); if family == "minimax_m3" && first_unquoted_native_tool_call_marker(content, family).is_some_and(|tool_at| { opener .into_iter() .chain(closer) .all(|marker_at| tool_at < marker_at) }) { // M3 parameters contain literal, unquoted text. A reasoning // closer inside an argument cannot imply a prompt-opened thought. // Let the parser own any channels after this first native call. return Ok(UnifiedParserStartingState::None); } Ok(match (opener, closer) { // No opener before it (or no opener at all): the prompt opened reasoning. (None, Some(_)) => UnifiedParserStartingState::Reasoning, (Some(open_at), Some(close_at)) if close_at < open_at => { UnifiedParserStartingState::Reasoning } // An opener exists (and any closer does not precede it): the model will // open reasoning itself later, currently visible as ordinary text. (Some(_), _) => UnifiedParserStartingState::None, // Marker-free text is the visible answer. This also holds for Nemotron // 3, whose implicit thought is split by the streaming path before the // aggregator re-reads a choice's text: starting that text in reasoning // would relabel an already-parsed answer. A raw thought cut off before // `` is indistinguishable here and is read the same way. (None, None) => UnifiedParserStartingState::Response, }) } "gemma4" => { let (opener, closer) = reasoning_marker_pair(family) .expect("Gemma 4 has a configured reasoning marker pair"); Ok(infer_marker_prefill(content, opener, closer)) } "kimi_k3" => Ok( match ( kimi_channel_marker_position(content, "<|open|>", "think"), kimi_channel_marker_position(content, "<|close|>", "think"), ) { (None, Some(_)) => UnifiedParserStartingState::Reasoning, (Some(open), Some(close)) if close < open => UnifiedParserStartingState::Reasoning, _ => UnifiedParserStartingState::None, }, ), "gpt_oss" => Ok(UnifiedParserStartingState::None), "muse_glimmer" | "muse_spark" => Ok(infer_marker_prefill(content, "<|start|>", "<|eom|>")), BEAM_UNIFIED_FAMILY => Ok(beam_prefill(content)), other => anyhow::bail!("no prefill detector for unified parser family '{other}'"), } } /// Beam's channel markers are special tokens the engine emits as such, so a literal /// never hides inside quotes: plain substring positions are the right reading. const BEAM_OPENERS: [&str; 2] = ["<|channel|>", "<|start|>"]; const BEAM_CLOSERS: [&str; 3] = ["<|end|>", "<|return|>", "<|call|>"]; /// Beam's channel state read back off complete output. /// /// The prompt ends at the `<|start|>assistant` header slot, so output normally opens /// its own channel (`<|channel|>…`, or a framed `<|start|>…` message): `None`. A closer /// before any opener means the prompt had opened a channel: `<|end|>` closes the /// analysis channel it opened (`Reasoning`), `<|return|>` the final one (`Response`). /// Output with no control marker at all is a final answer the prompt opened /// (`Response`), which is also what already-decoded text looks like when the /// aggregator re-reads it. fn beam_prefill(content: &str) -> UnifiedParserStartingState { let opener = BEAM_OPENERS .iter() .filter_map(|marker| content.find(marker)) .min(); let closer = BEAM_CLOSERS .iter() .filter_map(|marker| content.find(marker).map(|at| (at, *marker))) .min_by_key(|(at, _)| *at); match (opener, closer) { (Some(open), Some((close, _))) if open < close => UnifiedParserStartingState::None, (Some(_), None) => UnifiedParserStartingState::None, (_, Some((_, "<|end|>"))) => UnifiedParserStartingState::Reasoning, (_, Some(_)) | (None, None) => UnifiedParserStartingState::Response, } } fn reasoning_marker_pair(family: &str) -> Option<(&'static str, &'static str)> { match family { "minimax_m3" => Some(("", "")), "qwen3" | "deepseek_v4" | "deepseek_v41" | "glm47" | "kimi_k2" | "nemotron_v3" | "hermes" => Some(("", "")), "gemma4" => Some(("<|channel>", "")), _ => None, } } fn kimi_channel_marker_position(content: &str, prefix: &str, channel: &str) -> Option { content .match_indices(prefix) .filter_map(|(offset, _)| { let after = offset + prefix.len(); let (end, suffix_len) = if prefix == "<|close|>" { let end = after + content[after..] .find(['<', '[', '{']) .unwrap_or(content[after..].len()); let suffix_len = if content[end..].starts_with("<|sep|>") { "<|sep|>".len() } else { 0 }; (end, suffix_len) } else { (after + content[after..].find("<|sep|>")?, "<|sep|>".len()) }; if content[after..end].split_whitespace().next() != Some(channel) { return None; } let marker = &content[offset..end + suffix_len]; first_unquoted_marker_position(content, marker) }) .min() } fn infer_marker_prefill(content: &str, opener: &str, closer: &str) -> UnifiedParserStartingState { match ( first_unquoted_marker_position(content, opener), first_unquoted_marker_position(content, closer), ) { (None, Some(_)) => UnifiedParserStartingState::Reasoning, (Some(open), Some(close)) if close < open => UnifiedParserStartingState::Reasoning, _ => UnifiedParserStartingState::None, } } /// Marker-looking text quoted as prose is visible content, not parser control syntax. pub(crate) fn contains_unquoted_marker(content: &str, marker: &str) -> bool { first_unquoted_marker_position(content, marker).is_some() } /// The byte offset of the first unquoted occurrence of `marker` in `content`, or `None` /// if it never appears outside a quoted span. Prefill detection compares positions /// so a later opener cannot hide an earlier prompt-prefilled closer. pub(crate) fn first_unquoted_marker_position(content: &str, marker: &str) -> Option { first_unquoted_marker_position_after(content, marker, 0) } fn first_unquoted_marker_position_after( content: &str, marker: &str, minimum: usize, ) -> Option { let chars: Vec<(usize, char)> = content.char_indices().collect(); // Whether an unescaped `"`, `'`, or `` ` `` appears anywhere at or after each // position, precomputed once in O(n) so the scan below doesn't rescan the // remainder of the string for every candidate quote character (which would be // O(n^2) on adversarial text with many non-closing quote-like characters). let closes_later = later_unescaped_quote_closes(&chars); let mut quote = None; let mut escaped = false; for (position, &(offset, character)) in chars.iter().enumerate() { if let Some(active_quote) = quote { if escaped { escaped = false; } else if character == '\\' { escaped = true; } else if character == active_quote { let previous = content[..offset].chars().next_back(); let next = content[offset + character.len_utf8()..].chars().next(); if active_quote == '\'' && is_internal_apostrophe(previous, next) { continue; } quote = None; } continue; } let previous = content[..offset].chars().next_back(); let next = content[offset + character.len_utf8()..].chars().next(); let apostrophe = character == '\'' && (previous.is_some_and(char::is_alphanumeric) || next.is_some_and(char::is_alphanumeric) && !closes_later[position + 1][1]); // A quote that never closes before end-of-string was never really quoting // anything; treating it as an open span would blind the rest of the scan to // a real marker that follows (e.g. a stray `"` before a genuine ``). if let Some(slot) = quote_slot(character) && !apostrophe && closes_later[position + 1][slot] { quote = Some(character); continue; } if offset >= minimum && content[offset..].starts_with(marker) { return Some(offset); } } None } /// Return the first unquoted native tool-call marker configured for `parser`. /// /// Native markers quoted as prose remain visible text in both streaming and /// aggregated responses. pub(crate) fn first_unquoted_native_tool_call_marker(content: &str, parser: &str) -> Option { native_tool_call_start_tokens(parser)? .iter() .filter_map(|marker| first_unquoted_marker_position(content, marker)) .min() } /// Like [`first_unquoted_native_tool_call_marker`], but only for markers that cannot be /// ordinary prose. /// /// Truncating a response at a marker is safe only when the marker is control syntax. Some /// parsers open a call with a plain word — `phi4` uses `functools` — and an answer about /// the Python module of that name would lose everything from the word onward. A marker /// carrying a punctuation character (`<`, `|`, `[`) cannot be written by accident. pub(crate) fn first_unquoted_structural_tool_call_marker( content: &str, parser: &str, ) -> Option { native_tool_call_start_tokens(parser)? .iter() .filter(|marker| !marker.chars().all(char::is_alphanumeric)) .filter_map(|marker| first_unquoted_marker_position(content, marker)) .min() } /// Return the start of an unquoted native marker, including an unquoted /// partial marker suffix that must remain buffered for the next stream chunk. pub(crate) fn unquoted_native_tool_call_marker_or_prefix_start( content: &str, parser: &str, ) -> Option { let markers = native_tool_call_start_tokens(parser)?; if let Some(marker_start) = markers .iter() .filter_map(|marker| first_unquoted_marker_position(content, marker)) .min() { return Some(marker_start); } markers .iter() .flat_map(|marker| { marker .char_indices() .skip(1) .map(move |(end, _)| &marker[..end]) }) .filter(|prefix| { let before = content.strip_suffix(prefix).unwrap_or(content); first_unquoted_marker_position(before, prefix).is_none() }) .filter_map(|prefix| { let before = content.strip_suffix(prefix)?; let position = first_unquoted_marker_position(content, prefix)?; (position == before.len()).then_some(position) }) .min() } fn native_tool_call_start_tokens(parser: &str) -> Option> { let parser_key = if parser.is_empty() { "default" } else { parser }; Some( dynamo_parsers::tool_calling::parsers::get_tool_parser_map() .get(parser_key)? .parser_config .tool_call_start_tokens() .iter() .filter(|marker| !marker.is_empty()) .cloned() .collect(), ) } /// Index into the fixed-size "which quote characters close later" arrays used by /// [`later_unescaped_quote_closes`]. fn quote_slot(character: char) -> Option { match character { '"' => Some(0), '\'' => Some(1), '`' => Some(2), _ => None, } } fn is_internal_apostrophe(previous: Option, next: Option) -> bool { previous.is_some_and(char::is_alphanumeric) && next.is_some_and(char::is_alphanumeric) } /// For each position in `chars`, whether an unescaped `"`, `'`, or `` ` `` appears /// anywhere later in the string — `result[i][quote_slot(c)]` answers "does an /// unescaped `c` exist in `chars[i..]`". Built in one right-to-left O(n) pass: the /// answer at `i` depends only on `chars[i]` and the answer at `i + 1` (or `i + 2` for /// a backslash, which escapes exactly the next character), so this is an exact /// backward re-expression of the left-to-right escaped-flag scan that used to be /// re-run from scratch for every candidate quote character. fn later_unescaped_quote_closes(chars: &[(usize, char)]) -> Vec<[bool; 3]> { let n = chars.len(); let mut closes = vec![[false; 3]; n + 1]; let mut i = n; while i > 0 { i -= 1; let character = chars[i].1; closes[i] = if character == '\\' { closes.get(i + 2).copied().unwrap_or([false; 3]) } else { let mut next = closes[i + 1]; let internal_apostrophe = character == '\'' && is_internal_apostrophe( i.checked_sub(1) .and_then(|previous| chars.get(previous).map(|(_, c)| *c)), chars.get(i + 1).map(|(_, c)| *c), ); if let Some(slot) = quote_slot(character) && !internal_apostrophe { next[slot] = true; } next }; } closes } /// Map dynamo's v1 [`ToolDefinition`]s onto the v2 parser's [`Tool`] shape. pub(super) fn to_v2_tools(tools: Option<&[ToolDefinition]>) -> Vec { tools .unwrap_or(&[]) .iter() .map(|tool| Tool { name: tool.name.clone(), description: None, parameters: tool.parameters.clone().unwrap_or(serde_json::Value::Null), strict: tool.strict, }) .collect() } /// Map the request's `tool_choice` onto the wire format the backend will produce. /// /// A named or `required` choice is served by guided decoding, which constrains the /// model to bare JSON instead of Qwen's native `` XML: a named choice to /// that one tool's argument object, `required` to a call object or an array of them. /// Unset / `auto` / `none` leave the model in native markup. /// /// Callers must not consult this when a structural tag is active — see /// [`apply_stream`]'s `uses_tool_call_structural_tag`. fn tool_output_mode(constraint: &GuidedToolConstraint) -> UnifiedToolOutputMode { match constraint { GuidedToolConstraint::GuidedJsonNamed { tool_name } => UnifiedToolOutputMode::GuidedJson { named_tool: Some(tool_name.clone()), }, GuidedToolConstraint::GuidedJsonRequired => { UnifiedToolOutputMode::GuidedJson { named_tool: None } } GuidedToolConstraint::None | GuidedToolConstraint::StructuralTag | GuidedToolConstraint::BeamTurnGrammar { .. } => UnifiedToolOutputMode::Native, } } /// How the native parser frames argument values for this request: the Beam turn /// grammar writes every held tool's non-string parameters as JSON, so those slots own /// their strings and a closing tag inside one is data (see /// `dynamo_parsers_v2::ArgumentFraming::SchemaConstrained`); without a grammar a value /// runs to the first closing tag. fn argument_framing(constraint: &GuidedToolConstraint) -> ArgumentFraming { match constraint { GuidedToolConstraint::BeamTurnGrammar { .. } => ArgumentFraming::SchemaConstrained, GuidedToolConstraint::None | GuidedToolConstraint::StructuralTag | GuidedToolConstraint::GuidedJsonNamed { .. } | GuidedToolConstraint::GuidedJsonRequired => ArgumentFraming::Lenient, } } fn apply_argument_framing( family: &str, parser: &mut dyn UnifiedParser, argument_framing: ArgumentFraming, ) -> anyhow::Result<()> { if family == BEAM_UNIFIED_FAMILY || argument_framing != ArgumentFraming::default() { parser.set_tool_call_policy(ToolCallPolicy { malformed: if family == BEAM_UNIFIED_FAMILY && argument_framing == ArgumentFraming::SchemaConstrained { MalformedToolCallPolicy::Reject } else { Default::default() }, argument_framing, })?; } Ok(()) } /// The malformed-guided-payload contract for one request, selected by the /// guided tool-call streaming rollback lever. /// /// Only [`InvalidGuidedPayloadPolicy::StreamBestEffort`] may emit a call before its /// payload closes; the other two contracts buffer to completion (see the policy's /// own doc table). So this IS the streaming decision on the unified path, the same /// way `guided_streaming` is on the jail path — the caller passes down one already- /// made decision rather than re-deriving it here. /// /// Rolling back picks `RecoverAsText`, not `Reject`: the lever exists to stop early /// emission, not to convert a malformed payload from recovered text into a typed /// error, which would be a second, louder behaviour change riding along with it. fn invalid_guided_payload_policy(guided_streaming: bool) -> InvalidGuidedPayloadPolicy { if guided_streaming { InvalidGuidedPayloadPolicy::StreamBestEffort } else { InvalidGuidedPayloadPolicy::RecoverAsText } } /// Select the complete-output grammar from the bytes that actually arrived. /// /// A remote frontend can reconstruct a forced request as guided JSON without seeing /// that the worker installed a structural tag. Prefer the configured family's native /// marker over the reconstructed request policy; quoted markers remain payload. fn batch_tool_output_mode( family: &str, content: &str, constraint: &GuidedToolConstraint, ) -> UnifiedToolOutputMode { let tool_parser = if uses_qwen_xml_grammar(family) { QWEN3_TOOL_CALL_PARSER } else { family }; // Kimi K3's jail also owns reasoning and response boundaries. Those // envelopes can precede guided JSON and do not prove a native tool call. let native_call = if family == KIMI_K3_UNIFIED_FAMILY { ["call", "tools"] .into_iter() .any(|channel| kimi_channel_marker_position(content, "<|open|>", channel).is_some()) } else { first_unquoted_native_tool_call_marker(content, tool_parser).is_some() }; if constraint.installs_guided_json() && native_call { UnifiedToolOutputMode::Native } else { tool_output_mode(constraint) } } /// Ask the grammar owner which source bytes remain uncommitted. A literal tool /// closer inside an argument must not hide an unfinished sibling from recovery. pub(crate) fn glm_unfinished_call(input: &str) -> anyhow::Result> { let mut parser = create_unified_parser_for_family(GLM47_UNIFIED_FAMILY, &[])?; let mut output = UnifiedParserOutput::default(); parser.parse_into(input, &mut output)?; let pending = parser.reset(); let prefix = input .strip_suffix(&pending) .ok_or_else(|| anyhow::anyhow!("GLM pending input is not a source suffix"))?; // Only prose after the last completed call can quote a pending marker. // Argument quotes belong to the native grammar and cannot span sibling calls. let mut prose = String::new(); for event in output.events { match event { UnifiedParserEvent::ToolCall(call) if call.complete => prose.clear(), UnifiedParserEvent::Text(text) | UnifiedParserEvent::Reasoning(text) => { prose.push_str(&text) } _ => {} } } let pending_offset = prose.len(); prose.push_str(&pending); let markers = native_tool_call_start_tokens(GLM47_UNIFIED_FAMILY) .expect("registered GLM native grammar has start tokens"); Ok(markers .iter() .filter_map(|marker| first_unquoted_marker_position_after(&prose, marker, pending_offset)) .min() .map(|start| prefix.len() + start - pending_offset)) } /// Merge adjacent same-kind text/reasoning deltas so one `push` does not become three /// chunks that say the same thing. /// /// Tool-call deltas never merge: two fragments belonging to different `tool_index`es /// would fuse into one call, and even two fragments of the SAME call must keep their /// `name`-carrying first delta distinct from later argument-only ones. #[cfg(test)] fn contains_native_envelope(content: &str, family: &str) -> bool { unified_parser_control_markers(family) .iter() .any(|marker| contains_unquoted_marker(content, marker)) } fn unified_parser_control_markers(family: &str) -> Vec { let mut markers = reasoning_marker_pair(family) .map(|(opener, closer)| vec![opener.to_string(), closer.to_string()]) .unwrap_or_default(); if family == "kimi_k3" { for channel in ["think", "response", "tools", "call"] { markers.push(format!("<|open|>{channel}")); markers.push(format!("<|close|>{channel}")); } markers.push("<|sep|>".to_string()); } if matches!(family, "muse_glimmer" | "muse_spark") { markers.extend( ["<|start|>", "<|message|>", "<|eom|>", "<|eot|>"] .into_iter() .map(str::to_string), ); } if family == BEAM_UNIFIED_FAMILY { markers.extend( [ "<|start|>", "<|channel|>", "<|message|>", "<|constrain|>", "<|end|>", "<|return|>", "<|call|>", ] .into_iter() .map(str::to_string), ); } if let Some(marker) = native_family_marker(family) { markers.push(marker.to_string()); } if let Some(native_markers) = native_tool_call_start_tokens(family) { markers.extend(native_markers); } markers.sort_by_key(|marker| std::cmp::Reverse(marker.len())); markers.dedup(); markers } fn native_family_marker(family: &str) -> Option<&'static str> { match family { "minimax_m3" => Some("]<]minimax[>["), "qwen3" | "glm47" | "nemotron_v3" => Some(""), "deepseek_v4" => Some("<|DSML|tool_calls>"), "deepseek_v41" => Some("<|DSML| calls>"), "gemma4" => Some("<|tool_call>"), "kimi_k2" => Some("<|tool_call_begin|>"), "kimi_k3" => Some("<|open|>call"), "muse_glimmer" | "muse_spark" => Some(" Some("<|start|>assistant to="), _ => None, } } /// Merge adjacent same-kind text/reasoning deltas so one `push` does not become three /// chunks that say the same thing. /// /// Tool-call deltas never merge: two fragments belonging to different `tool_index`es /// would fuse into one call, and even two fragments of the SAME call must keep their /// `name`-carrying first delta distinct from later argument-only ones. fn coalesce(deltas: Vec) -> Vec { let mut out: Vec = Vec::with_capacity(deltas.len()); for delta in deltas { match (out.last_mut(), delta) { (Some(UnifiedParserEvent::Text(prev)), UnifiedParserEvent::Text(text)) => { prev.push_str(&text) } (Some(UnifiedParserEvent::Reasoning(prev)), UnifiedParserEvent::Reasoning(text)) => { prev.push_str(&text) } (_, delta) => out.push(delta), } } out } /// Pick the one fanout child that keeps the source chunk's token metrics. /// /// The reasoning-usage estimator runs after unified parsing and attributes the source /// chunk's tokens to reasoning when any emitted child carries `reasoning_content`. /// Keeping metrics on the last child unconditionally loses that classification whenever /// reasoning is followed by visible text, a tool call, or a later response choice. pub(crate) fn fanout_llm_metrics_position(choices: &[ChatChoiceStream]) -> Option { choices .iter() .position(|choice| choice.delta.reasoning_content.is_some()) .or_else(|| choices.len().checked_sub(1)) } /// An empty streaming choice for `index`, used as the base every emitted chunk is /// filled in from. /// /// It carries no `logprobs`: `choices_for` moves a source chunk's raw-token logprobs /// onto exactly one of the choices parsed from it. pub(crate) fn empty_choice(index: u32) -> ChatChoiceStream { #[allow(deprecated)] ChatChoiceStream { index, delta: ChatCompletionStreamResponseDelta { role: None, content: None, tool_calls: None, function_call: None, refusal: None, reasoning_content: None, }, finish_reason: None, logprobs: None, } } /// Additional adapter state, selected once for the native grammar. A shared /// `` spelling does not establish shared grammar semantics: reuse a /// variant only when its full behavior matches, otherwise document a new variant. enum NativeGrammarState { /// These adapter mechanisms are inactive; the selected parser remains active. None, /// Qwen's XML tool-call dialect, also used by Nemotron. Parameter delimiters /// and literal-marker rules require dialect-specific handling. QwenXml(QwenXmlStreamState), /// GLM's native EOF truncation check uses its grammar to distinguish unfinished /// calls from literal argument text. Glm(GlmNativeStreamState), } /// Adapter ordering state for the Qwen XML dialect; source responses live longer /// than a parser instance and remain in `ChoiceRecord`. #[derive(Default)] struct QwenXmlStreamState { lexical_state: QwenNativeLexicalState, /// Buffered text whose interpretation is unresolved, rather than a source position. pending_opener: String, /// Must follow recovered opener text on mismatch and precede completed call events. deferred_reasoning: Vec, /// Logprobs of chunks generated after the latest deferred reasoning, held until it /// is released. deferred_logprobs: Option, /// Entries in event/output lists, not XML character offsets: these preserve /// deferred reasoning's original response metadata through projection. deferred_reasoning_event_indices: HashSet, deferred_reasoning_choice_indices: HashSet, } #[derive(Default)] struct GlmNativeStreamState { input_for_truncation_check: String, } /// Per-choice streaming state: one parser instance plus the bookkeeping the OpenAI /// streaming tool-call contract needs. One instance parses exactly one choice of one /// request, which is what gives per-stream isolation by construction. #[derive(Clone, Copy, Default, PartialEq, Eq)] enum QwenNativeLexicalMode { #[default] OutsideBlock, InsideBlock, ParameterHeader, ParameterValue, } #[derive(Clone, Default)] struct QwenNativeLexicalState { mode: QwenNativeLexicalMode, marker_tail: String, } impl QwenNativeLexicalState { fn observe(&mut self, text: &str) { const BLOCK_START: &str = ""; const BLOCK_END: &str = ""; const PARAMETER_START: &str = " { if let Some(start) = input.find(BLOCK_START) { input.drain(..start + BLOCK_START.len()); self.mode = QwenNativeLexicalMode::InsideBlock; } else { self.retain_partial_suffix(&input, &[BLOCK_START]); return; } } QwenNativeLexicalMode::InsideBlock => { let parameter = input.find(PARAMETER_START); let block_end = input.find(BLOCK_END); match (parameter, block_end) { (Some(parameter), Some(block_end)) if block_end < parameter => { input.drain(..block_end + BLOCK_END.len()); self.mode = QwenNativeLexicalMode::OutsideBlock; } (Some(parameter), _) => { input.drain(..parameter + PARAMETER_START.len()); self.mode = QwenNativeLexicalMode::ParameterHeader; } (None, Some(block_end)) => { input.drain(..block_end + BLOCK_END.len()); self.mode = QwenNativeLexicalMode::OutsideBlock; } (None, None) => { self.retain_partial_suffix(&input, &[PARAMETER_START, BLOCK_END]); return; } } } QwenNativeLexicalMode::ParameterHeader => { let header_end = input.find('>'); if let Some(function_end) = input.find(FUNCTION_END) && header_end.is_none_or(|end| function_end < end) { input.drain(..function_end + FUNCTION_END.len()); self.mode = QwenNativeLexicalMode::InsideBlock; } else if let Some(header_end) = header_end { input.drain(..header_end + 1); self.mode = QwenNativeLexicalMode::ParameterValue; } else { self.retain_partial_suffix(&input, &[FUNCTION_END]); return; } } QwenNativeLexicalMode::ParameterValue => { // Native parameter values own literal function/block markers. // Only their own delimiter permits a detour to start another call. if let Some(at) = input.find(PARAMETER_END) { input.drain(..at + PARAMETER_END.len()); self.mode = QwenNativeLexicalMode::InsideBlock; } else { self.retain_partial_suffix(&input, &[PARAMETER_END]); return; } } } } } fn retain_partial_suffix(&mut self, input: &str, markers: &[&str]) { let suffix_length = markers .iter() .flat_map(|marker| (1..marker.len()).map(move |length| &marker[..length])) .filter(|prefix| input.ends_with(prefix)) .map(str::len) .max() .unwrap_or_default(); self.marker_tail = input[input.len() - suffix_length..].to_string(); } } fn qwen_opener_resolution(held: &str, text: &str) -> Option<(usize, bool)> { const BLOCK_START: &str = ""; for (start, character) in text.char_indices() { let end = start + character.len_utf8(); let candidate = format!("{held}{}", &text[..end]); if candidate == BLOCK_START { return Some((end, true)); } if !BLOCK_START.starts_with(&candidate) { return Some((end, false)); } } None } fn qwen_partial_opener_suffix_start( lexical_state: &QwenNativeLexicalState, text: &str, ) -> Option { const BLOCK_START: &str = ""; let start = (1..BLOCK_START.len()) .rev() .find(|length| text.ends_with(&BLOCK_START[..*length])) .map(|length| text.len() - length)?; let mut before_candidate = lexical_state.clone(); before_candidate.observe(&text[..start]); (before_candidate.mode == QwenNativeLexicalMode::OutsideBlock).then_some(start) } #[derive(Default)] struct QuotedControlMarkerProbe { markers: Vec, quote: Option, escaped: bool, pending_quote_candidate: bool, previous_character: Option, pending: String, pending_marker: Option, inspected_bytes: usize, } impl QuotedControlMarkerProbe { fn new(family: &str) -> Self { Self { markers: unified_parser_control_markers(family), ..Default::default() } } fn push(&mut self, text: &str) -> String { self.inspected_bytes += text.len(); let marker_strings = self.markers.clone(); let markers = marker_strings .iter() .map(String::as_str) .collect::>(); let mut ready = String::new(); let mut cursor = 0; if self.pending_quote_candidate { let Some(next) = text.chars().next() else { return ready; }; self.pending_quote_candidate = false; if !is_internal_apostrophe(self.previous_character, Some(next)) { self.quote = None; self.mask_pending_markers(&markers); ready.push_str(&std::mem::take(&mut self.pending)); self.pending_marker = None; } self.previous_character = Some('\''); } while !self.pending.is_empty() && self.pending_marker.is_none() { let Some(character) = text[cursor..].chars().next() else { return ready; }; let mut candidate = self.pending.clone(); candidate.push(character); if markers.iter().any(|marker| marker.starts_with(&candidate)) { let is_complete_marker = markers.iter().any(|marker| **marker == candidate); self.pending.push(character); cursor += character.len_utf8(); self.previous_character = Some(character); if is_complete_marker { // A configured marker can also prefix a longer marker. Stop on // the complete marker so the next byte cannot flush it unmasked. self.pending_marker = Some(candidate); } } else { ready.push_str(&std::mem::take(&mut self.pending)); } } while cursor < text.len() { if self.pending_marker.is_some() { let character = text[cursor..] .chars() .next() .expect("cursor is before the end of the input"); let previous = self.previous_character; self.pending.push(character); cursor += character.len_utf8(); let next = text[cursor..].chars().next(); let mut defer_previous_update = false; if self.escaped { self.escaped = false; } else if character == '\\' { self.escaped = true; } else if self.quote == Some(character) { if character == '\'' && is_internal_apostrophe(previous, next) { // Apostrophes inside words do not close a quoted span. } else if character == '\'' && next.is_none() && previous.is_some_and(char::is_alphanumeric) { self.pending_quote_candidate = true; defer_previous_update = true; } else { self.quote = None; self.mask_pending_markers(&markers); ready.push_str(&std::mem::take(&mut self.pending)); self.pending_marker = None; } } if !defer_previous_update { self.previous_character = Some(character); } continue; } let remaining = &text[cursor..]; if self.quote.is_some() { if let Some(marker) = markers .iter() .find(|marker| remaining.starts_with(**marker)) { self.pending.push_str(marker); self.pending_marker = Some((*marker).to_string()); cursor += marker.len(); self.previous_character = marker.chars().next_back(); continue; } if markers.iter().any(|marker| marker.starts_with(remaining)) { self.pending.push_str(remaining); self.previous_character = remaining.chars().next_back(); break; } } let character = remaining .chars() .next() .expect("cursor is before the end of the input"); let previous = self.previous_character; ready.push(character); cursor += character.len_utf8(); let next = text[cursor..].chars().next(); let mut defer_previous_update = false; if let Some(quote) = self.quote { if self.escaped { self.escaped = false; } else if character == '\\' { self.escaped = true; } else if character == quote { if quote == '\'' && is_internal_apostrophe(previous, next) { // Apostrophes inside words do not close a quoted span. } else if quote == '\'' && next.is_none() && previous.is_some_and(char::is_alphanumeric) { self.pending_quote_candidate = true; defer_previous_update = true; } else { self.quote = None; } } } else if character == '\'' { if !previous.is_some_and(char::is_alphanumeric) { self.quote = Some(character); } } else if matches!(character, '"' | '`') { self.quote = Some(character); } if !defer_previous_update { self.previous_character = Some(character); } } ready } fn mask_pending_markers(&mut self, markers: &[&str]) { let pending = std::mem::take(&mut self.pending); self.inspected_bytes += pending.len(); let mut cursor = 0; while cursor < pending.len() { if let Some(marker) = markers .iter() .find(|marker| pending[cursor..].starts_with(**marker)) { self.pending.extend(std::iter::repeat_n('\0', marker.len())); cursor += marker.len(); } else { let character = pending[cursor..] .chars() .next() .expect("cursor is before the end of pending text"); self.pending.push(character); cursor += character.len_utf8(); } } } fn finish(&mut self) -> String { if self.pending_quote_candidate { self.quote = None; self.pending_quote_candidate = false; } if self.quote.is_none() { let marker_strings = self.markers.clone(); let markers = marker_strings .iter() .map(String::as_str) .collect::>(); self.mask_pending_markers(&markers); } self.pending_marker = None; self.quote = None; self.escaped = false; std::mem::take(&mut self.pending) } } /// Per-choice streaming state: one parser instance plus the bookkeeping the OpenAI /// streaming tool-call contract needs. One instance parses exactly one choice of one /// request, which is what gives per-stream isolation by construction. pub(crate) struct ChoiceState { native_grammar_state: NativeGrammarState, incomplete_native_call: bool, family: String, message_boundaries: Vec>, parser: Box, tool_output_mode: UnifiedToolOutputMode, /// Tool indices that have emitted at least one `tool_calls` delta. opened_calls: HashSet, /// Upstream IDs and types may arrive after an index opens. Names remain fragments. tool_ids_emitted: HashSet, tool_types_emitted: HashSet, /// Stable externally visible indices for parser-local and pass-through calls. parser_tool_indices: HashMap, passthrough_tool_indices: HashMap, /// Offset applied to tool indices from a parser instance rebuilt after a detour. tool_index_offset: usize, /// Next externally visible tool index, retained across parser-instance detours. next_tool_index: usize, /// Tool indices whose UnifiedParser events have not emitted an explicit completion. pending_tool_calls: HashSet, parser_call_completion: HashMap, /// Pass-through OpenAI call argument fragments awaiting terminal or JSON completion. pending_parsed_tool_calls: HashMap, /// Whether any tool-call delta was emitted, including a streamed fragment. tool_emitted: bool, /// The parser errored. Later chunks pass through as plain text instead of failing /// the request — a parser bug must not turn a served answer into a 500. failed: bool, parse_error: Option, } impl ChoiceState { #[cfg(test)] pub(crate) fn new( family: &str, tools: &[Tool], prefill: impl Into, tool_output_mode: UnifiedToolOutputMode, guided_streaming: bool, argument_framing: ArgumentFraming, ) -> anyhow::Result { Self::new_with_prefill_cache( family, tools, prefill.into(), tool_output_mode, guided_streaming, argument_framing, None, ) } fn new_with_prefill_cache( family: &str, tools: &[Tool], prefill: StreamPrefill, tool_output_mode: UnifiedToolOutputMode, guided_streaming: bool, argument_framing: ArgumentFraming, mut cache: Option<&mut HarmonyPrefill>, ) -> anyhow::Result { // The built-in Harmony initializer resets its call counter. Replay consumed // tokens afterward so the open call and later calls share that counter. // Guided JSON inherits only the resolved channel, since its request constraint // owns the tool envelope. Vendor overrides retain their initialization contract. let harmony = family == "gpt_oss"; let native_harmony = harmony && matches!(&tool_output_mode, UnifiedToolOutputMode::Native); let vendor_harmony = harmony && dynamo_parsers_v2::vendor_unified_families() .iter() .any(|registered| registered == family); let builtin_native_harmony = native_harmony && !vendor_harmony; let continuation = cache .as_deref() .filter(|cache| builtin_native_harmony && cache.matches_prefix(prefill.prefix.as_ref())) .and_then(|cache| cache.continuation.as_ref()); let mut parser = if let Some(continuation) = continuation { let mut parser = continuation.fork()?; apply_argument_framing(family, parser.as_mut(), argument_framing)?; parser } else { let mut parser = create_unified_parser_for_family(family, tools)?; let replay_harmony = builtin_native_harmony && parser.input_capabilities().token_ids; let mut harmony_prefix_tokens = if native_harmony || vendor_harmony { prefill .prefix .as_deref() .or_else(|| { (native_harmony && prefill.starting_state == UnifiedParserStartingState::None) .then_some("<|start|>assistant") }) .map(|prefix| { if replay_harmony && let Some(cache) = cache.as_deref_mut() && cache.matches_prefix(prefill.prefix.as_ref()) { return cache.token_ids().map(std::borrow::Cow::Borrowed); } dynamo_parsers_v2::unified::harmony::encode_harmony(prefix) .map(std::borrow::Cow::Owned) }) .transpose()? } else { None }; let prompt_token_ids = if replay_harmony { Vec::new() } else { harmony_prefix_tokens .take() .map(std::borrow::Cow::into_owned) .unwrap_or_default() }; // Streaming guided calls commit per call once the name and argument object opener // are unambiguous. A committed fragment cannot later be rejected or recovered as // text, so this path opts into that contract explicitly — unless the request // rolled guided tool streaming back, in which case it buffers to completion. // `guided_streaming` is decided once per request by // `OpenAIPreprocessor::guided_tool_streaming_release` and threaded down here; // it is inert in `UnifiedToolOutputMode::Native`, where the parser builds no // guided state at all. parser.initialize_request(UnifiedParserInit { prompt_token_ids, starting_state: if prefill.prefix.is_some() && (!harmony || native_harmony || vendor_harmony) { UnifiedParserStartingState::None } else { prefill.starting_state }, tool_output_mode: tool_output_mode.clone(), invalid_guided_payload: invalid_guided_payload_policy(guided_streaming), })?; apply_argument_framing(family, parser.as_mut(), argument_framing)?; if let Some(tokens) = harmony_prefix_tokens { // Consumed prompt deltas stay private; token replay preserves literal // delimiter fragments without joining them to generated text. replay_consumed_harmony(parser.as_mut(), &tokens, 256)?; } if replay_harmony && let Some(cache) = cache && cache.matches_prefix(prefill.prefix.as_ref()) { cache.continuation = parser.continuation()?; if cache.continuation.is_some() { // The checkpoint owns consumed state. Token replay is no longer // needed for subsequent choices or delayed choice initialization. cache.token_ids = None; } } parser }; if family != "gpt_oss" && let Some(prefix) = prefill.prefix { // Replay only consumed structural bytes; the grammar retains ownership // of an optional channel label that generation has not completed yet. let mut output = UnifiedParserOutput::default(); parser.parse_into(&prefix, &mut output)?; anyhow::ensure!( output.events.is_empty(), "prompt prefix emitted visible output" ); } Ok(Self { native_grammar_state: if matches!(&tool_output_mode, UnifiedToolOutputMode::Native) { if uses_qwen_xml_grammar(family) { NativeGrammarState::QwenXml(QwenXmlStreamState::default()) } else if family == GLM47_UNIFIED_FAMILY { NativeGrammarState::Glm(GlmNativeStreamState::default()) } else { NativeGrammarState::None } } else { NativeGrammarState::None }, incomplete_native_call: false, family: family.to_string(), message_boundaries: Vec::new(), parser, tool_output_mode, opened_calls: HashSet::new(), tool_ids_emitted: HashSet::new(), tool_types_emitted: HashSet::new(), parser_tool_indices: HashMap::new(), passthrough_tool_indices: HashMap::new(), tool_index_offset: 0, next_tool_index: 0, pending_tool_calls: HashSet::new(), parser_call_completion: HashMap::new(), pending_parsed_tool_calls: HashMap::new(), tool_emitted: false, failed: false, parse_error: None, }) } /// Feed one decoded text delta through the parser. #[cfg(test)] pub(crate) fn new_with_constraint( family: &str, tools: &[Tool], prefill: UnifiedParserStartingState, constraint: &GuidedToolConstraint, guided_streaming: bool, ) -> anyhow::Result { Self::new( family, tools, prefill, tool_output_mode(constraint), guided_streaming, argument_framing(constraint), ) } #[cfg(test)] pub(crate) fn take_message_boundaries(&mut self) -> Vec> { std::mem::take(&mut self.message_boundaries) } pub(crate) fn preserve_message_phases(&mut self, enabled: bool) { self.parser.preserve_message_phases(enabled); } pub(crate) fn push(&mut self, text: &str) -> Vec { if self.failed { return text_delta(text.to_string()); } if let NativeGrammarState::QwenXml(qwen) = &mut self.native_grammar_state && qwen.lexical_state.mode == QwenNativeLexicalMode::OutsideBlock { let held = std::mem::take(&mut qwen.pending_opener); if !qwen.deferred_reasoning.is_empty() && !held.is_empty() { if let Some((_resolution_end, opener_completed)) = qwen_opener_resolution(&held, text) { let mut combined = held.clone(); combined.push_str(text); let mut events = self.parse_segment(&combined); if opener_completed { self.prepend_pending_qwen_reasoning(&mut events); } else { self.insert_pending_qwen_reasoning_after_recovered_opener( &held, &mut events, ); } return events; } qwen.pending_opener = held; qwen.pending_opener.push_str(text); return Vec::new(); } let mut combined = held; combined.push_str(text); if let Some(start) = qwen_partial_opener_suffix_start(&qwen.lexical_state, &combined) { let safe_text = combined[..start].to_string(); qwen.pending_opener = combined[start..].to_string(); return if safe_text.is_empty() { Vec::new() } else { self.parse_segment(&safe_text) }; } return self.parse_segment(&combined); } self.parse_segment(text) } fn qwen_resumed_marker_can_start_call(&self) -> bool { matches!(&self.native_grammar_state, NativeGrammarState::QwenXml(qwen) if qwen.lexical_state.mode != QwenNativeLexicalMode::ParameterValue) } fn has_pending_qwen_xml_opener(&self) -> bool { matches!(&self.native_grammar_state, NativeGrammarState::QwenXml(qwen) if qwen.lexical_state.mode == QwenNativeLexicalMode::OutsideBlock && !qwen.pending_opener.is_empty()) } fn is_deferred_qwen_xml_reasoning_choice(&self, index: usize) -> bool { matches!(&self.native_grammar_state, NativeGrammarState::QwenXml(qwen) if qwen.deferred_reasoning_choice_indices.contains(&index)) } fn parse_segment(&mut self, text: &str) -> Vec { match &mut self.native_grammar_state { NativeGrammarState::Glm(glm) => { // EOF recovery needs source context; scan once there, not on every push. glm.input_for_truncation_check.push_str(text); } NativeGrammarState::QwenXml(qwen) => qwen.lexical_state.observe(text), NativeGrammarState::None => {} } let mut output = UnifiedParserOutput::default(); match self.parser.parse_into(text, &mut output) { Ok(()) => output.events, Err(error) if self.family == BEAM_UNIFIED_FAMILY => { self.failed = true; self.parse_error = Some(error); Vec::new() } Err(error) => { tracing::warn!( error = %error, family = self.family, "unified parser push failed; falling back to plain text for this choice" ); output.events.extend(self.give_up(text)); output.events } } } /// Scores held since the previous queued reasoning were generated before this /// one, so they lead its source envelope's `logprobs`. fn queue_qwen_opener_reasoning( &mut self, reasoning: String, source_logprobs: &mut Option, ) { let NativeGrammarState::QwenXml(qwen) = &mut self.native_grammar_state else { return; }; qwen.deferred_reasoning.push(reasoning); if let Some(mut held) = qwen.deferred_logprobs.take() { if let Some(source) = source_logprobs.take() { extend_logprobs(&mut held, source); } *source_logprobs = Some(held); } } fn prepend_pending_qwen_reasoning(&mut self, events: &mut Vec) { let NativeGrammarState::QwenXml(qwen) = &mut self.native_grammar_state else { return; }; let mut reasoning = std::mem::take(&mut qwen.deferred_reasoning) .into_iter() .map(UnifiedParserEvent::Reasoning) .collect::>(); qwen.deferred_reasoning_event_indices .extend(0..reasoning.len()); reasoning.append(events); *events = reasoning; } fn append_pending_qwen_reasoning(&mut self, events: &mut Vec) { let NativeGrammarState::QwenXml(qwen) = &mut self.native_grammar_state else { return; }; qwen.deferred_reasoning_event_indices .extend(events.len()..events.len() + qwen.deferred_reasoning.len()); events.extend( std::mem::take(&mut qwen.deferred_reasoning) .into_iter() .map(UnifiedParserEvent::Reasoning), ); } fn insert_pending_qwen_reasoning_after_recovered_opener( &mut self, held: &str, events: &mut Vec, ) { let NativeGrammarState::QwenXml(qwen) = &mut self.native_grammar_state else { return; }; let pending = std::mem::take(&mut qwen.deferred_reasoning); let mut ordered = Vec::new(); let mut inserted = false; for event in std::mem::take(events) { if !inserted && let UnifiedParserEvent::Text(text) = &event && let Some(remainder) = text.strip_prefix(held) { if !held.is_empty() { ordered.push(UnifiedParserEvent::Text(held.to_string())); } qwen.deferred_reasoning_event_indices .extend(ordered.len()..ordered.len() + pending.len()); ordered.extend(pending.iter().cloned().map(UnifiedParserEvent::Reasoning)); if !remainder.is_empty() { ordered.push(UnifiedParserEvent::Text(remainder.to_string())); } inserted = true; continue; } ordered.push(event); } if !inserted { // TODO(frontend-crates): parser 0.7.17 malformed-call recovery still needs // a source-order barrier. Recovered text can span this thought at EOF; // matching a repeated opener cannot recover its original position. qwen.deferred_reasoning_event_indices .extend(ordered.len()..ordered.len() + pending.len()); ordered.extend(pending.into_iter().map(UnifiedParserEvent::Reasoning)); } *events = ordered; } /// Flush buffered partial state at end of stream. pub(crate) fn finish(&mut self) -> Vec { self.finish_with_eos(false) } fn finish_with_eos(&mut self, confirmed_eos: bool) -> Vec { if self.failed { return Vec::new(); } if let NativeGrammarState::Glm(glm) = &self.native_grammar_state { let input = &glm.input_for_truncation_check; match glm_unfinished_call(input) { Ok(Some(start)) => { self.incomplete_native_call = true; tracing::warn!( family = self.family, why = "truncated_native_tool_call_suppressed", suppressed_bytes = input.len() - start, "suppressing unfinished native tool call" ); self.parser.reset(); return Vec::new(); } Ok(None) => {} Err(error) => { tracing::warn!(%error, why = "glm_pending_input_failed", "using parser finish recovery") } } } let pending_qwen_opener = match &mut self.native_grammar_state { NativeGrammarState::QwenXml(qwen) => std::mem::take(&mut qwen.pending_opener), _ => String::new(), }; let mut events = if pending_qwen_opener.is_empty() { Vec::new() } else { self.parse_segment(&pending_qwen_opener) }; let finished = if confirmed_eos { self.parser.finish_at_eos() } else { self.parser.finish() }; events.extend(match finished { // `finish` now hands back the whole `UnifiedParserOutput`; this path only ever // wants the ordered events out of it. Ok(output) => output.events, Err(error) if self.family == BEAM_UNIFIED_FAMILY => { self.failed = true; self.parse_error = Some(error); Vec::new() } Err(error) => { tracing::warn!( error = %error, family = self.family, "unified parser finish failed; recovering buffered text" ); self.give_up("") } }); self.append_pending_qwen_reasoning(&mut events); events } /// Stop using the parser and surface whatever it was holding. /// /// `reset()` hands back the bytes the parser had buffered but not yet emitted. /// Dropping them would silently delete model output — the caller would see a /// truncated answer with no indication anything was lost — so they go out as /// visible text. When the parser had nothing buffered, the chunk that broke it /// does instead, so that chunk is not lost either. fn give_up(&mut self, fallback: &str) -> Vec { self.failed = true; let mut recovered = self.parser.reset(); if recovered.is_empty() { // TODO: Preserve the parser's consumed offset so a partial-commit error cannot replay input. recovered.push_str(fallback); } if let NativeGrammarState::QwenXml(qwen) = &mut self.native_grammar_state { recovered.push_str(&std::mem::take(&mut qwen.pending_opener)); } text_delta(recovered) } /// Convert one ordered delta into a streaming choice for `index`. fn delta_to_choice(&mut self, index: u32, delta: UnifiedParserEvent) -> ChatChoiceStream { let mut choice = empty_choice(index); let boundary = match &delta { UnifiedParserEvent::MessageStart(phase) => { Some(super::MessageBoundary::Start(match phase { dynamo_parsers_v2::unified::MessagePhase::Commentary => { dynamo_protocols::types::responses::MessagePhase::Commentary } dynamo_parsers_v2::unified::MessagePhase::FinalAnswer => { dynamo_protocols::types::responses::MessagePhase::FinalAnswer } })) } UnifiedParserEvent::MessageEnd => Some(super::MessageBoundary::End), _ => None, }; self.message_boundaries.push(boundary); match delta { UnifiedParserEvent::MessageStart(_) | UnifiedParserEvent::MessageEnd => {} UnifiedParserEvent::Text(text) => { choice.delta.content = Some(ChatCompletionMessageContent::Text(text)); } UnifiedParserEvent::Reasoning(text) => { choice.delta.reasoning_content = Some(text); } UnifiedParserEvent::ToolCall(call) => { let parser_index = call.tool_index + self.tool_index_offset; let tool_index = stable_tool_index( parser_index, &mut self.parser_tool_indices, &self.opened_calls, &mut self.next_tool_index, ); self.tool_emitted = true; let complete = self .parser_call_completion .entry(tool_index) .and_modify(|complete| *complete |= call.complete) .or_insert(call.complete); if *complete { self.pending_tool_calls.remove(&tool_index); self.pending_parsed_tool_calls.remove(&tool_index); } else { self.pending_tool_calls.insert(tool_index); } // The OpenAI streaming tool-call contract: the FIRST chunk for a tool // index carries id + type + name, later chunks carry only argument // fragments. Preserve model IDs when the native grammar supplies one. let first = self.opened_calls.insert(tool_index); if first { self.tool_ids_emitted.insert(tool_index); self.tool_types_emitted.insert(tool_index); } choice.delta.tool_calls = Some(vec![ChatCompletionMessageToolCallChunk { custom: None, index: tool_index as u32, id: first.then(|| { self.parser .tool_call_id(call.tool_index) .map(str::to_string) .unwrap_or_else(|| format!("call-{}", Uuid::new_v4())) }), r#type: first.then_some(FunctionType::Function), function: Some(FunctionCallStream { name: first.then_some(call.name).flatten(), arguments: Some(call.arguments), }), }]); } } choice } /// Convert an ordered delta run into the streaming choices it becomes. /// /// `role` / `refusal` ride on the first choice owned by the current source /// response; queued reasoning keeps its earlier source envelope. The terminating /// `finish_reason` stays after every emitted delta. The source chunk's `logprobs` /// move once to its last choice, waiting for any queued reasoning to go first: /// they score generated tokens, not parsed text. pub(crate) fn choices_for( &mut self, original: &ChatChoiceStream, deltas: Vec, emit_tool_calls: bool, finish_reason: Option, ) -> Vec { self.message_boundaries.clear(); let terminal_tool_call_delta = finish_reason_closes_tool_calls(finish_reason.as_ref()); // Queued reasoning belongs to its earlier source response. Coalescing it // with parser reasoning would erase the source envelope's event boundary. let source_positions = match &mut self.native_grammar_state { NativeGrammarState::QwenXml(qwen) => { let indices = std::mem::take(&mut qwen.deferred_reasoning_event_indices); qwen.deferred_reasoning_choice_indices.clear(); indices } _ => HashSet::new(), }; let mut run = Vec::new(); let mut projected = Vec::new(); for (position, delta) in deltas.into_iter().enumerate() { // Muse Spark boundaries feed the Responses API through the sidecar. // Other families must not create empty chunks for optional metadata. if self.family != MUSE_SPARK_UNIFIED_FAMILY && matches!( delta, UnifiedParserEvent::MessageStart(_) | UnifiedParserEvent::MessageEnd ) { continue; } if !emit_tool_calls && matches!(delta, UnifiedParserEvent::ToolCall(_)) { continue; } if source_positions.contains(&position) { projected.extend(coalesce(std::mem::take(&mut run))); if let NativeGrammarState::QwenXml(qwen) = &mut self.native_grammar_state { qwen.deferred_reasoning_choice_indices .insert(projected.len()); } projected.push(delta); } else { run.push(delta); } } projected.extend(coalesce(run)); let deltas = projected; let envelope_position = (0..deltas.len()) .find(|position| !self.is_deferred_qwen_xml_reasoning_choice(*position)); let mut original = original.clone(); if let Some(original_calls) = original.delta.tool_calls.as_mut() { for call in original_calls { let tool_index = normalize_passthrough_tool_call( call, &mut self.passthrough_tool_indices, &mut self.opened_calls, &mut self.tool_ids_emitted, &mut self.tool_types_emitted, &mut self.next_tool_index, ); append_parsed_tool_call_arguments( &mut self.pending_parsed_tool_calls, tool_index, call, ); self.tool_emitted = true; } } if terminal_tool_call_delta { resolve_complete_parsed_tool_calls_at_terminal( &mut self.pending_parsed_tool_calls, &self.family, finish_reason == Some(FinishReason::ToolCalls), ); } let index = original.index; let count = deltas.len(); let mut choices = Vec::with_capacity(count.max(1)); for (position, delta) in deltas.into_iter().enumerate() { let mut choice = self.delta_to_choice(index, delta); if Some(position) == envelope_position { choice.delta.role = original.delta.role; choice.delta.refusal = original.delta.refusal.clone(); choice.delta.function_call = original.delta.function_call.clone(); if let Some(reasoning) = original.delta.reasoning_content.as_deref() { choice .delta .reasoning_content .get_or_insert_default() .insert_str(0, reasoning); } if let Some(mut original_calls) = original.delta.tool_calls.clone() { if let Some(parsed_calls) = choice.delta.tool_calls.take() { original_calls.extend(parsed_calls); } choice.delta.tool_calls = Some(original_calls); } } if position + 1 == count && !self.is_deferred_qwen_xml_reasoning_choice(position) { choice.finish_reason = self.normalize_finish_reason(finish_reason); } choices.push(choice); } if !choices.is_empty() && (envelope_position.is_none() || (finish_reason.is_some() && self.is_deferred_qwen_xml_reasoning_choice(count - 1))) { let mut choice = empty_choice(index); if envelope_position.is_none() { choice.delta.role = original.delta.role; choice.delta.refusal = original.delta.refusal.clone(); choice.delta.reasoning_content = original.delta.reasoning_content.clone(); choice.delta.tool_calls = original.delta.tool_calls.clone(); choice.delta.function_call = original.delta.function_call.clone(); } choice.finish_reason = self.normalize_finish_reason(finish_reason); choices.push(choice); } // The parser produced nothing, but the chunk still carried envelope state that // has to reach the client (the opening role chunk, a refusal, or the terminal // finish_reason). if choices.is_empty() && (original.delta.role.is_some() || original.delta.refusal.is_some() || original.delta.reasoning_content.is_some() || original.delta.tool_calls.is_some() || original.delta.function_call.is_some() || finish_reason.is_some()) { let mut choice = empty_choice(index); choice.delta.role = original.delta.role; choice.delta.refusal = original.delta.refusal.clone(); choice.delta.reasoning_content = original.delta.reasoning_content.clone(); choice.delta.tool_calls = original.delta.tool_calls.clone(); choice.delta.function_call = original.delta.function_call.clone(); if choice .delta .tool_calls .as_ref() .is_some_and(|calls| !calls.is_empty()) { self.tool_emitted = true; } choice.finish_reason = self.normalize_finish_reason(finish_reason); choices.push(choice); } // Queued reasoning arrived before this chunk, so this chunk's logprobs must // not ride ahead of it: hold them while it is queued, then follow it. let mut logprobs = original.logprobs.take(); if let NativeGrammarState::QwenXml(qwen) = &mut self.native_grammar_state { if let Some(mut held) = qwen.deferred_logprobs.take() { if let Some(source) = logprobs.take() { extend_logprobs(&mut held, source); } logprobs = Some(held); } if !qwen.deferred_reasoning.is_empty() { qwen.deferred_logprobs = logprobs.take(); } } if let Some(logprobs) = logprobs { let last_is_queued = choices .len() .checked_sub(1) .is_some_and(|last| self.is_deferred_qwen_xml_reasoning_choice(last)); if choices.is_empty() || last_is_queued { choices.push(empty_choice(index)); } if let Some(choice) = choices.last_mut() { choice.logprobs = Some(logprobs); } } choices } pub(crate) fn unterminated_finish_reason(&self) -> Option { self.normalize_finish_reason(Some(FinishReason::Stop)) .filter(|reason| *reason == FinishReason::Length || *reason == FinishReason::ToolCalls) } /// Whether this choice has emitted a tool-call delta so far. pub(crate) fn tool_emitted(&self) -> bool { self.tool_emitted } fn resolve_complete_parsed_tool_calls_at_eof(&mut self) { resolve_complete_parsed_tool_calls_at_eof( &mut self.pending_parsed_tool_calls, &self.family, ); } fn should_preserve_reasoning_only_state(&self) -> bool { // A nonterminal out-of-band thought cannot finish pending native or guided input. !self.failed } /// Seed tool-call history from before an already-parsed detour into a freshly /// constructed parser instance for the same choice index. pub(crate) fn mark_tool_emitted(&mut self) { self.tool_emitted = true; } fn has_incomplete_tool_call(&self) -> bool { self.incomplete_native_call || !self.pending_tool_calls.is_empty() || !self.pending_parsed_tool_calls.is_empty() || has_incomplete_call_evidence(&self.parser_call_completion) } /// OpenAI streaming contract: `Stop` becomes `ToolCalls` only when every emitted /// call index is complete. `Length` / `ContentFilter` stay unchanged. fn normalize_finish_reason(&self, finish_reason: Option) -> Option { normalize_tool_finish_reason( finish_reason, self.tool_emitted, self.has_incomplete_tool_call(), ) } } // OpenAI tool-call chunks have no per-delta completion bit. Retain every argument // fragment so EOF can distinguish a complete JSON value from a truncated call. fn append_parsed_tool_call_arguments( pending: &mut HashMap, tool_index: usize, call: &ChatCompletionMessageToolCallChunk, ) { let arguments = call .function .as_ref() .and_then(|function| function.arguments.as_deref()) .unwrap_or_default(); pending.entry(tool_index).or_default().push_str(arguments); } fn resolve_complete_parsed_tool_calls_at_eof(pending: &mut HashMap, family: &str) { resolve_complete_parsed_tool_calls_at_terminal(pending, family, false); } fn resolve_complete_parsed_tool_calls_at_terminal( pending: &mut HashMap, family: &str, allow_empty_parameterless: bool, ) { // Default-on families preserve backend arguments at terminal/EOF with the experiment off. // Native pending calls and explicit parser completion evidence remain separate guards. if matches!(family, DEEPSEEK_V41_UNIFIED_FAMILY | MUSE_UNIFIED_FAMILY) && !parsers_v2_selected() { pending.clear(); return; } pending.retain(|_, arguments| { !(allow_empty_parameterless && arguments.is_empty()) && serde_json::from_str::(arguments).is_err() }); } fn finish_reason_closes_tool_calls(finish_reason: Option<&FinishReason>) -> bool { matches!( finish_reason, Some(FinishReason::Stop | FinishReason::ToolCalls) ) } fn stable_tool_index( source_index: usize, indices: &mut HashMap, opened_calls: &HashSet, next_tool_index: &mut usize, ) -> usize { if let Some(tool_index) = indices.get(&source_index) { return *tool_index; } let tool_index = if opened_calls.contains(&source_index) { let mut candidate = *next_tool_index; while opened_calls.contains(&candidate) { candidate += 1; } candidate } else { source_index }; indices.insert(source_index, tool_index); *next_tool_index = (*next_tool_index).max(tool_index + 1); tool_index } fn normalize_passthrough_tool_call( call: &mut ChatCompletionMessageToolCallChunk, indices: &mut HashMap, opened_calls: &mut HashSet, ids_emitted: &mut HashSet, types_emitted: &mut HashSet, next_tool_index: &mut usize, ) -> usize { let tool_index = stable_tool_index(call.index as usize, indices, opened_calls, next_tool_index); if call.id.is_some() && !ids_emitted.insert(tool_index) { call.id = None; } if call.r#type.is_some() && !types_emitted.insert(tool_index) { call.r#type = None; } call.index = tool_index as u32; opened_calls.insert(tool_index); tool_index } /// One text delta, or nothing at all when the text is empty — an empty content chunk /// carries no information and clients render it as a stray empty string. fn text_delta(text: String) -> Vec { if text.is_empty() { Vec::new() } else { vec![UnifiedParserEvent::Text(text)] } } /// The aggregated result of parsing one complete (non-streaming) output. #[derive(Debug)] pub(crate) struct CompleteOutput { pub incomplete_native_call: bool, pub text: String, pub reasoning: String, pub tool_calls: Vec, pub events: Vec, } /// Batch (non-streaming) path: run the whole output through the same parser lifecycle /// and fold the assembled events into the final message. /// /// Routing batch through `push`/`finish` is what makes stream/batch parity structural /// rather than a property two code paths have to agree on. /// /// Reasoning spans are concatenated because the non-streaming message schema has ONE /// `reasoning_content` string — the ordering the unified parser recovered survives only /// on the streaming path, which is where a client can act on it. #[cfg(test)] pub(crate) fn parse_complete( family: &str, content: &str, guided_tool_constraint: &GuidedToolConstraint, tool_definitions: &[ToolDefinition], ) -> anyhow::Result { parse_complete_with_policy( family, content, guided_tool_constraint, tool_definitions, UnifiedRequestPolicy::default(), ) } #[cfg(test)] pub(crate) fn parse_complete_with_policy( family: &str, content: &str, guided_tool_constraint: &GuidedToolConstraint, tool_definitions: &[ToolDefinition], policy: UnifiedRequestPolicy, ) -> anyhow::Result { parse_complete_with_eos( family, content, guided_tool_constraint, tool_definitions, policy, false, ) } pub(crate) fn parse_complete_with_eos( family: &str, content: &str, guided_tool_constraint: &GuidedToolConstraint, tool_definitions: &[ToolDefinition], policy: UnifiedRequestPolicy, model_eos: bool, ) -> anyhow::Result { let confirmed_eos = model_eos && family == BEAM_UNIFIED_FAMILY && matches!( guided_tool_constraint, GuidedToolConstraint::BeamTurnGrammar { .. } ); let policy = policy.under(guided_tool_constraint); // A remote frontend may reconstruct guided JSON even when the backend returned // native reasoning/content without a tool call. Keep that recovery under the // same grammar owner as ordinary native batch output. let mode = batch_tool_output_mode(family, content, guided_tool_constraint); match parse_complete_in_mode( family, content, guided_tool_constraint, tool_definitions, mode.clone(), policy, confirmed_eos, ) { Err(guided_error) if !matches!(mode, UnifiedToolOutputMode::Native) => { match parse_complete_in_mode( family, content, guided_tool_constraint, tool_definitions, UnifiedToolOutputMode::Native, policy, confirmed_eos, ) { Ok(parsed) if !parsed.tool_calls.is_empty() || !parsed.reasoning.is_empty() || parsed.text != content => { tracing::warn!( family, why = "guided_json_reconstructed_but_native_markup_observed", recovered_bytes = content.len(), "falling back to the configured unified parser" ); Ok(parsed) } Ok(_) => Err(guided_error), Err(native_error) => Err(native_error .context(format!("guided JSON parse also failed: {guided_error:#}"))), } } result => result, } } fn parse_complete_in_mode( family: &str, content: &str, guided_tool_constraint: &GuidedToolConstraint, tool_definitions: &[ToolDefinition], effective_mode: UnifiedToolOutputMode, policy: UnifiedRequestPolicy, confirmed_eos: bool, ) -> anyhow::Result { let tools = to_v2_tools(Some(tool_definitions)); let mut parser = create_unified_parser_for_family(family, &tools)?; parser.preserve_message_phases(family == MUSE_SPARK_UNIFIED_FAMILY); // `batch_tool_output_mode` drops to `Native` on observed markup even when the // constraint reconstructed a `GuidedJsonNamed` pin. That native fallback recovers // whatever tool name the markup happens to contain, so it must be checked against // the forced name below instead of trusted unfiltered — a malformed guided output // that embeds a *different* tool's markup must not be handed back as the pinned // tool's call. let forced_tool_name = match (guided_tool_constraint, &effective_mode) { (GuidedToolConstraint::GuidedJsonNamed { tool_name }, UnifiedToolOutputMode::Native) => { Some(tool_name.as_str()) } _ => None, }; let incomplete_start = (family == GLM47_UNIFIED_FAMILY && matches!(&effective_mode, UnifiedToolOutputMode::Native)) .then(|| glm_unfinished_call(content)) .transpose()? .flatten(); let content = &content[..incomplete_start.unwrap_or(content.len())]; parser.initialize_request(UnifiedParserInit { prompt_token_ids: if family == "gpt_oss" && matches!(&effective_mode, UnifiedToolOutputMode::Native) && content.trim_start().starts_with("to=") { dynamo_parsers_v2::unified::harmony::encode_harmony("<|start|>assistant")? } else { Vec::new() }, starting_state: policy.batch_starting_state(family, content)?, tool_output_mode: effective_mode, ..UnifiedParserInit::default() })?; apply_argument_framing( family, parser.as_mut(), argument_framing(guided_tool_constraint), )?; let mut text = String::new(); let mut reasoning = String::new(); let mut tool_calls = Vec::new(); let mut output = UnifiedParserOutput::default(); parser.parse_into(content, &mut output)?; output.events.extend( if confirmed_eos { parser.finish_at_eos()? } else { parser.finish()? } .events, ); let completed_indices: HashSet<_> = output .events .iter() .filter_map(|event| match event { UnifiedParserEvent::ToolCall(call) if call.complete => Some(call.tool_index), _ => None, }) .collect(); let mut seen = HashSet::new(); let mut completed_indices = output.events.iter().filter_map(|event| match event { UnifiedParserEvent::ToolCall(call) if completed_indices.contains(&call.tool_index) && seen.insert(call.tool_index) => { Some(call.tool_index) } _ => None, }); let events = dynamo_parsers_v2::assemble(&output.events); let mut ordered_events = Vec::new(); for event in events { match event { UnifiedEvent::MessageStart { phase } => ordered_events.push( super::ResponseOutputEvent::Boundary(super::MessageBoundary::Start(match phase { dynamo_parsers_v2::unified::MessagePhase::Commentary => { dynamo_protocols::types::responses::MessagePhase::Commentary } dynamo_parsers_v2::unified::MessagePhase::FinalAnswer => { dynamo_protocols::types::responses::MessagePhase::FinalAnswer } })), ), UnifiedEvent::MessageEnd => ordered_events.push(super::ResponseOutputEvent::Boundary( super::MessageBoundary::End, )), UnifiedEvent::Text { text: chunk } => { ordered_events.push(super::ResponseOutputEvent::Text(chunk.clone())); text.push_str(&chunk); } UnifiedEvent::Reasoning { text: chunk } => { if policy.reasoning_disabled { text.push_str(&chunk); } else { reasoning.push_str(&chunk); } } UnifiedEvent::ToolCall { name, arguments } => { let id = parser .tool_call_id( completed_indices .next() .expect("assembled call has completion evidence"), ) .map(str::to_string) .unwrap_or_else(|| format!("call-{}", Uuid::new_v4())); if forced_tool_name.is_some_and(|forced| forced != name) { tracing::warn!( forced_tool_name = forced_tool_name, recovered_tool_name = %name, "dropped native-fallback tool call whose name did not match the tool_choice-forced tool_name" ); continue; } ordered_events.push(super::ResponseOutputEvent::ToolCall(id.clone())); tool_calls.push(ChatCompletionMessageToolCall { custom: None, id, r#type: FunctionType::Function, // `assemble` already parsed the argument fragments into a typed // object, so this re-serializes rather than passing the model's // bytes through. Formatting is normalized as a result. function: FunctionCall { name, arguments: serde_json::to_string(&arguments)?, }, }); } } } Ok(CompleteOutput { events: ordered_events, incomplete_native_call: incomplete_start.is_some(), text, reasoning, tool_calls, }) } /// Per-choice bookkeeping that outlives any single `ChoiceState` instance for that /// index: whether it has emitted a tool call, which indices remain provisional, and whether its terminal chunk has /// already been sent. See the field comment at its use site in /// `apply_stream_with_constraint` for why this can't just live inside `ChoiceState`. #[derive(Default)] struct ChoiceRecord { incomplete_native_call: bool, opened_calls: HashSet, tool_ids_emitted: HashSet, tool_types_emitted: HashSet, parser_tool_indices: HashMap, passthrough_tool_indices: HashMap, next_tool_index: usize, /// Incomplete events from the live UnifiedParser instance. pending_tool_calls: HashSet, parser_call_completion: HashMap, /// Pass-through OpenAI call argument fragments awaiting terminal or JSON completion. pending_parsed_tool_calls: HashMap, /// Leading whitespace held until a Reasoning-prefill guided request reveals /// whether generation starts with reasoning prose or bare guided JSON. pending_guided_prefill_whitespace: String, /// Source envelopes for Qwen reasoning held until a split native opener resolves. deferred_qwen_xml_reasoning_responses: VecDeque, tool_emitted: bool, finished: bool, /// Whether an already-parsed chunk has EVER interrupted this choice. An /// already-parsed chunk implies the model has already emitted structured output /// (reasoning_content / tool_calls / function_call / `Parts` content), which per /// `already_parsed`'s own trigger set only happens after any reasoning phase would /// have concluded. So a raw run resuming after such a detour must rebuild its /// `ChoiceState` starting at `Response`, never at the outer request-level `prefill` /// (which only describes what the PROMPT opened, before this choice ever ran) — see /// the `Vacant` arm in the main loop below. detoured: bool, } struct DeferredQwenXmlReasoningResponse { text: String, choice: ChatChoiceStream, response: Annotated, } enum FinishedStreamChunk { Error(anyhow::Error), Choice(ChatChoiceStream), Boundary(ChatChoiceStream, super::MessageBoundary), Source(Annotated>), } fn extend_logprobs(held: &mut ChatChoiceLogprobs, later: ChatChoiceLogprobs) { if let Some(content) = later.content { held.content.get_or_insert_default().extend(content); } if let Some(refusal) = later.refusal { held.refusal.get_or_insert_default().extend(refusal); } } fn take_qwen_reasoning_response( record: &mut ChoiceRecord, ) -> Annotated> { let pending = record .deferred_qwen_xml_reasoning_responses .pop_front() .expect("queued reasoning event must have a source response"); let mut response = pending.response; let mut source_reasoning = pending.choice; source_reasoning.delta.reasoning_content = Some(pending.text); source_reasoning.finish_reason = None; response .data .as_mut() .expect("queued source response has data") .inner .choices = vec![source_reasoning]; response.map_data(|data| Ok(Box::new(data))) } fn stream_chunks_for_choices( state: &ChoiceState, record: &mut ChoiceRecord, choices: Vec, ) -> Vec { choices .into_iter() .enumerate() .map(|(position, choice)| { if state.is_deferred_qwen_xml_reasoning_choice(position) { FinishedStreamChunk::Source(take_qwen_reasoning_response(record)) } else if let Some(boundary) = state.message_boundaries.get(position).cloned().flatten() { FinishedStreamChunk::Boundary(choice, boundary) } else { FinishedStreamChunk::Choice(choice) } }) .collect() } impl ChoiceRecord { fn has_incomplete_tool_call(&self) -> bool { self.incomplete_native_call || !self.pending_tool_calls.is_empty() || !self.pending_parsed_tool_calls.is_empty() || has_incomplete_call_evidence(&self.parser_call_completion) } fn normalize_finish_reason(&self, finish_reason: Option) -> Option { normalize_tool_finish_reason( finish_reason, self.tool_emitted, self.has_incomplete_tool_call(), ) } fn remember_state(&mut self, state: &ChoiceState) { self.incomplete_native_call |= state.incomplete_native_call; self.opened_calls.extend(state.opened_calls.iter().copied()); self.tool_ids_emitted .extend(state.tool_ids_emitted.iter().copied()); self.tool_types_emitted .extend(state.tool_types_emitted.iter().copied()); self.parser_tool_indices .extend(state.parser_tool_indices.iter().map(|(k, v)| (*k, *v))); self.passthrough_tool_indices .extend(state.passthrough_tool_indices.iter().map(|(k, v)| (*k, *v))); self.next_tool_index = self.next_tool_index.max(state.next_tool_index); self.pending_tool_calls .clone_from(&state.pending_tool_calls); for (&index, &complete) in &state.parser_call_completion { self.parser_call_completion .entry(index) .and_modify(|remembered| *remembered |= complete) .or_insert(complete); } self.pending_parsed_tool_calls .clone_from(&state.pending_parsed_tool_calls); self.tool_emitted |= state.tool_emitted(); } } /// Finish every choice that never received a terminating chunk, in index order. /// /// A choice can reach this point two ways: still holding a live `ChoiceState` (the /// common case — nothing special happened, it just never got an explicit terminal), /// or "history-only" — its `ChoiceState` was removed by an already-parsed detour and /// never rebuilt, so all that remains is its `ChoiceRecord`. The second case has /// nothing left to flush, but a choice with a completed tool call still needs a synthesized /// `ToolCalls` terminal so a strict client doesn't hang waiting for one; a choice that /// never called a tool gets no synthetic chunk unless a suppressed GLM native /// call needs an error to expose the incomplete output. fn finish_unterminated_choices( states: &mut HashMap, records: &mut HashMap, family: &str, ) -> Vec { let mut indices: Vec = states .keys() .copied() .chain(records.keys().copied()) .collect(); indices.sort_unstable(); indices.dedup(); let mut chunks = Vec::new(); for index in indices { if records.get(&index).is_some_and(|record| record.finished) { continue; } let record = records.entry(index).or_default(); record.finished = true; let base = empty_choice(index); match states.get_mut(&index) { Some(state) => { let deltas = state.finish(); if let Some(error) = state.parse_error.take() { chunks.push(FinishedStreamChunk::Error(error)); continue; } // Native completion evidence remains authoritative; backend-only calls // use the family policy so default-on parsers retain their EOF contract. let mut flushed = state.choices_for(&base, deltas, true, None); state.resolve_complete_parsed_tool_calls_at_eof(); if let Some(error) = incomplete_glm_terminal(family, Some(FinishReason::Stop), || { state.has_incomplete_tool_call() }) { chunks.push(FinishedStreamChunk::Error(error.into())); break; } if let Some(reason) = state.unterminated_finish_reason() { if let Some(last_position) = flushed.len().checked_sub(1) { if state.is_deferred_qwen_xml_reasoning_choice(last_position) { let mut terminal = empty_choice(index); terminal.finish_reason = Some(reason); flushed.push(terminal); } else { flushed[last_position].finish_reason = Some(reason); } } else { flushed.extend(state.choices_for(&base, Vec::new(), true, Some(reason))); } } chunks.extend(stream_chunks_for_choices(state, record, flushed)); } None => { let pending_whitespace = std::mem::take(&mut record.pending_guided_prefill_whitespace); if !pending_whitespace.is_empty() { let mut choice = empty_choice(index); choice.delta.reasoning_content = Some(pending_whitespace); chunks.push(FinishedStreamChunk::Choice(choice)); } while !record.deferred_qwen_xml_reasoning_responses.is_empty() { chunks.push(FinishedStreamChunk::Source(take_qwen_reasoning_response( record, ))); } resolve_complete_parsed_tool_calls_at_eof( &mut record.pending_parsed_tool_calls, family, ); if let Some(error) = incomplete_glm_terminal(family, Some(FinishReason::Stop), || { record.has_incomplete_tool_call() }) { chunks.push(FinishedStreamChunk::Error(error.into())); break; } if let Some(reason @ (FinishReason::Length | FinishReason::ToolCalls)) = record.normalize_finish_reason(Some(FinishReason::Stop)) { let mut choice = base; choice.finish_reason = Some(reason); chunks.push(FinishedStreamChunk::Choice(choice)); } } } } chunks } // Live and history-only choices must expose the same unfinished-call outcome. fn normalize_tool_finish_reason( finish_reason: Option, tool_emitted: bool, incomplete: bool, ) -> Option { match finish_reason { Some(FinishReason::Stop | FinishReason::ToolCalls) if incomplete => { Some(FinishReason::Length) } Some(FinishReason::Stop) if tool_emitted => Some(FinishReason::ToolCalls), other => other, } } fn has_incomplete_call_evidence(completion: &HashMap) -> bool { completion.values().any(|complete| !complete) } fn merge_passthrough_completion( completion: &mut HashMap, indices: &HashMap, evidence: &[super::ToolCallCompletion], choice_index: u32, ) { for evidence in evidence .iter() .filter(|evidence| evidence.choice_index == choice_index) { if let Some(index) = indices.get(&(evidence.tool_index as usize)) { completion .entry(*index) .and_modify(|complete| *complete |= evidence.complete) .or_insert(evidence.complete); } } } fn normalize_completion_evidence( evidence: &mut super::ToolCallCompletion, states: &HashMap, records: &HashMap, ) { let index = evidence.tool_index as usize; let normalized = states .get(&evidence.choice_index) .and_then(|state| state.passthrough_tool_indices.get(&index)) .or_else(|| { records .get(&evidence.choice_index) .and_then(|record| record.passthrough_tool_indices.get(&index)) }); if let Some(index) = normalized { evidence.tool_index = *index as u32; } } fn normalized_source_response( response: Annotated>, states: &HashMap, records: &HashMap, ) -> Annotated { response.map_data(|data| { let mut data = *data; for evidence in &mut data.tool_call_completion { normalize_completion_evidence(evidence, states, records); } Ok(data) }) } fn attach_call_completion( data: &mut NvCreateChatCompletionStreamResponse, choice: &ChatChoiceStream, states: &HashMap, records: &HashMap, ) { // Source evidence may refer to a choice with no delta in this chunk. Preserve // it on one child rather than filtering it to the child's choice. for evidence in &mut data.tool_call_completion { normalize_completion_evidence(evidence, states, records); } let state = states.get(&choice.index); let record = records.get(&choice.index); let mut indices: HashSet = choice .delta .tool_calls .iter() .flatten() .map(|call| call.index as usize) .collect(); if choice.finish_reason.is_some() { indices.extend( state .into_iter() .flat_map(|state| state.parser_call_completion.keys().copied()), ); indices.extend( record .into_iter() .flat_map(|record| record.parser_call_completion.keys().copied()), ); } let mut indices: Vec<_> = indices.into_iter().collect(); indices.sort_unstable(); for index in indices { let complete = state .and_then(|state| state.parser_call_completion.get(&index)) .or_else(|| record.and_then(|record| record.parser_call_completion.get(&index))) .copied(); if let Some(complete) = complete { if let Some(evidence) = data.tool_call_completion.iter_mut().find(|evidence| { evidence.choice_index == choice.index && evidence.tool_index == index as u32 }) { evidence.complete |= complete; } else { data.tool_call_completion.push(super::ToolCallCompletion { choice_index: choice.index, tool_index: index as u32, complete, }); } } } } /// Wrap one rewritten choice in a response built from `template`. /// /// Usage, nvext and metrics are cleared: they belong to the chunk that carried them, /// and repeating them on a synthesized chunk would double-count. fn response_with_choice( template: &NvCreateChatCompletionStreamResponse, choice: ChatChoiceStream, states: &HashMap, records: &HashMap, ) -> Annotated { let mut response = Annotated::from_data(template.clone()); super::scrub_synthetic_chunk_metadata(&mut response); let data = response.data.as_mut().expect("template has response data"); attach_call_completion(data, &choice, states, records); data.inner.choices = vec![choice]; response } /// Whether a choice arrived already parsed by something upstream and must be passed /// through untouched rather than re-parsed. fn already_parsed(choice: &ChatChoiceStream) -> bool { let has_raw_text = matches!( choice.delta.content, Some(ChatCompletionMessageContent::Text(_)) ); matches!( choice.delta.content, Some(ChatCompletionMessageContent::Parts(_)) ) || (!has_raw_text && (choice.delta.tool_calls.is_some() || choice.delta.function_call.is_some() || choice.delta.reasoning_content.is_some())) } /// Streaming path: one unified parser per response choice, replacing both the reasoning /// parser and the tool-call jail for this request. /// /// `uses_tool_call_structural_tag` reports that the backend was given a structural tag /// constraining generation to the family's NATIVE grammar. When it is set the output is /// native markup no matter what `tool_choice` says, so `tool_choice` is not consulted; /// this path only parses guided JSON, it never builds the grammar that produces it. #[cfg(test)] pub(crate) fn apply_stream( stream_in: S, tool_definitions: Option>, tool_choice: Option, uses_tool_call_structural_tag: bool, prefill: UnifiedParserStartingState, family: &'static str, ) -> impl Stream> + Send where S: Stream> + Send + 'static, { let guided_tool_constraint = if uses_tool_call_structural_tag { GuidedToolConstraint::StructuralTag } else { match tool_choice { Some(ChatCompletionToolChoiceOption::Named(named)) => { GuidedToolConstraint::GuidedJsonNamed { tool_name: named.function.name, } } Some(ChatCompletionToolChoiceOption::Required) => { GuidedToolConstraint::GuidedJsonRequired } None | Some(ChatCompletionToolChoiceOption::Auto) | Some(ChatCompletionToolChoiceOption::None) => GuidedToolConstraint::None, } }; apply_stream_with_constraint( stream_in, tool_definitions, guided_tool_constraint, prefill, family, // Streaming released, matching the production default when the rollback lever // is unset. Tests that exercise the rolled-back contract call // `apply_stream_with_constraint` directly with `false`. true, ) } /// `guided_streaming` is the request's already-made guided tool-call streaming /// decision (`OpenAIPreprocessor::guided_tool_streaming_release`), not a second /// derivation of it: `true` releases each call as soon as its name and argument /// object opener are unambiguous, `false` buffers every call to completion. It is /// inert unless `guided_tool_constraint` installed guided JSON. fn resolve_guided_prefix( index: u32, probe: JsonPrefillProbe, policy: UnifiedRequestPolicy, prefill: UnifiedParserStartingState, ) -> Option { let decision = probe.decision(true); if probe.text.is_empty() { return None; } let mut choice = empty_choice(index); if policy.structured_response && (prefill == UnifiedParserStartingState::None || decision == Some(UnifiedParserStartingState::None)) { choice.delta.content = Some(ChatCompletionMessageContent::Text(probe.text)); } else { choice.delta.reasoning_content = Some(probe.text); } Some(choice) } fn flush_undecided_guided_prefixes( prefixes: &mut HashMap, policy: UnifiedRequestPolicy, prefill: UnifiedParserStartingState, ) -> Vec { prefixes .drain() .filter_map(|(index, probe)| resolve_guided_prefix(index, probe, policy, prefill)) .collect() } #[cfg(test)] pub(crate) fn apply_stream_with_constraint( stream_in: S, tool_definitions: Option>, guided_tool_constraint: GuidedToolConstraint, prefill: impl Into, family: &'static str, guided_streaming: bool, ) -> impl Stream> + Send where S: Stream> + Send + 'static, { apply_stream_with_policy( stream_in, tool_definitions, guided_tool_constraint, prefill, family, guided_streaming, UnifiedRequestPolicy::default(), ) } pub(crate) fn apply_stream_with_policy( stream_in: S, tool_definitions: Option>, guided_tool_constraint: GuidedToolConstraint, prefill: impl Into, family: &'static str, guided_streaming: bool, policy: UnifiedRequestPolicy, ) -> impl Stream> + Send where S: Stream> + Send + 'static, { let StreamPrefill { starting_state: prefill, prefix: prefill_prefix, } = prefill.into(); let policy = policy.under(&guided_tool_constraint); let prefill = policy.starting_state(family, prefill); let mut prefill_prefix = prefill_prefix .filter(|_| family == "gpt_oss" || prefill == UnifiedParserStartingState::Reasoning); let has_consumed_harmony_prefix = family == "gpt_oss" && prefill_prefix.is_some(); let argument_framing = argument_framing(&guided_tool_constraint); let mut harmony_prefill_cache = policy.harmony_prefill_cache(family, prefill_prefix.clone()); let tools = to_v2_tools(tool_definitions.as_deref()); stream! { let mut undecided_guided_prefix: HashMap = HashMap::new(); let mut states: HashMap = HashMap::new(); // Per-choice bookkeeping that must outlive a single `ChoiceState`: a // `ChoiceState` is dropped whenever an already-parsed chunk interrupts a // choice (its buffered parser state would be wrong to keep across the // detour), but whether that choice ever called a tool and whether its // terminal has already gone out must still be known afterward — by a // resumed raw run, an already-parsed terminal, or the EOF backstop below, // even when no `ChoiceState` for that index exists any more (or never did: // an index whose first-ever chunk is already-parsed never gets one either). let mut records: HashMap = HashMap::new(); // Last data response with its choices cleared, so an end-of-stream flush has an // envelope (id, model, created) to attach synthesized chunks to. let mut template: Option = None; tokio::pin!(stream_in); while let Some(mut response) = stream_in.next().await { if response.is_error() { yield response; return; } let response_id = response.id.clone(); let response_event = response.event.clone(); let response_comment = response.comment.clone(); let response_error = response.error.clone(); let Some(chat) = response.data.as_mut() else { // Non-data annotations (errors, comments) pass through untouched. yield response; continue; }; { let mut next = Annotated::from_data(chat.clone()); super::scrub_synthetic_chunk_metadata(&mut next); let mut next = next.data.expect("template has response data"); next.inner.choices.clear(); template = Some(next); } for evidence in &chat.tool_call_completion { if let Some(state) = states.get_mut(&evidence.choice_index) { merge_passthrough_completion(&mut state.parser_call_completion, &state.passthrough_tool_indices, std::slice::from_ref(evidence), evidence.choice_index); } if let Some(record) = records.get_mut(&evidence.choice_index) { merge_passthrough_completion(&mut record.parser_call_completion, &record.passthrough_tool_indices, std::slice::from_ref(evidence), evidence.choice_index); } } // A finished choice cannot reopen its parser; preserve envelope metadata below. chat.inner.choices.retain(|choice| { !records.get(&choice.index).is_some_and(|record| record.finished) }); if chat.inner.choices.is_empty() { // A metadata-only chunk. OpenAI stream ordering requires every choice's // terminal finish_reason to precede it, so flush first. if chat.inner.usage.is_some() && let Some(template) = &template { for choice in flush_undecided_guided_prefixes(&mut undecided_guided_prefix, policy, prefill) { yield response_with_choice(template, choice, &states, &records); } for output in finish_unterminated_choices(&mut states, &mut records, family) { match output { FinishedStreamChunk::Error(error) => { yield parser_error_annotation(error); return; } FinishedStreamChunk::Choice(choice) => { yield response_with_choice(template, choice, &states, &records); } FinishedStreamChunk::Boundary(choice, boundary) => { let mut response = response_with_choice(template, choice, &states, &records); response.data.as_mut().unwrap().message_boundary = Some(boundary); yield response; } FinishedStreamChunk::Source(response) => yield normalized_source_response(response, &states, &records), } } } for evidence in &mut chat.tool_call_completion { normalize_completion_evidence(evidence, &states, &records); } yield response; continue; } let originals = std::mem::take(&mut chat.inner.choices); let mut emitted: Vec = Vec::new(); let mut phase_boundaries = HashMap::new(); let mut deferred_qwen_responses = Vec::new(); let mut qwen_source_metadata_deferred = false; for mut original in originals { if already_parsed(&original) { if let Some(prefix) = undecided_guided_prefix.remove(&original.index) && let Some(choice) = resolve_guided_prefix(original.index, prefix, policy, prefill) { emitted.push(choice); } // Record this index exists even if no `ChoiceState` is ever // built for it (e.g. its first-ever chunk is already-parsed), // so it is not invisible to `finish_unterminated_choices`. let record = records.entry(original.index).or_default(); let pending_whitespace = std::mem::take(&mut record.pending_guided_prefill_whitespace); if !pending_whitespace.is_empty() { if let Some(reasoning) = original.delta.reasoning_content.as_mut() { reasoning.insert_str(0, &pending_whitespace); } else { let mut deferred_reasoning = empty_choice(original.index); deferred_reasoning.delta.reasoning_content = Some(pending_whitespace); emitted.push(deferred_reasoning); } } // Any raw run resuming this choice after this point must rebuild // starting at `Response`, not the outer request-level `prefill` — // see the `Vacant` arm below and the field doc on `ChoiceRecord`. record.detoured = true; if let Some(state) = states.get(&original.index) { record.remember_state(state); } let parsed_calls_are_terminal = finish_reason_closes_tool_calls(original.finish_reason.as_ref()); // An already-parsed chunk carries its tool calls verbatim in its // own delta rather than through a `ChoiceState`, so that history // has to be observed here directly — a state may never exist for // this index at all (its very first chunk can be already-parsed). if let Some(calls) = original .delta .tool_calls .as_mut() .filter(|calls| !calls.is_empty()) { record.tool_emitted = true; for call in calls { let tool_index = normalize_passthrough_tool_call( call, &mut record.passthrough_tool_indices, &mut record.opened_calls, &mut record.tool_ids_emitted, &mut record.tool_types_emitted, &mut record.next_tool_index, ); if parsed_calls_are_terminal { record.pending_tool_calls.remove(&tool_index); } append_parsed_tool_call_arguments( &mut record.pending_parsed_tool_calls, tool_index, call, ); } } merge_passthrough_completion( &mut record.parser_call_completion, &record.passthrough_tool_indices, &chat.tool_call_completion, original.index, ); if parsed_calls_are_terminal { resolve_complete_parsed_tool_calls_at_terminal( &mut record.pending_parsed_tool_calls, family, original.finish_reason == Some(FinishReason::ToolCalls), ); } let reported_tool_indices: HashSet = if parsed_calls_are_terminal { original .delta .tool_calls .as_ref() .into_iter() .flatten() .map(|call| call.index as usize) .collect() } else { HashSet::new() }; let preserve_reasoning_only_state = original.finish_reason.is_none() && original.delta.reasoning_content.is_some() && original.delta.content.is_none() && original.delta.tool_calls.is_none() && original.delta.function_call.is_none() && original.delta.refusal.is_none() && states .get(&original.index) .is_some_and(ChoiceState::should_preserve_reasoning_only_state); let queue_qwen_opener_reasoning = preserve_reasoning_only_state && original .delta .reasoning_content .as_ref() .is_some_and(|reasoning| !reasoning.is_empty()) && uses_qwen_xml_grammar(family) && states.get(&original.index).is_some_and(ChoiceState::has_pending_qwen_xml_opener); if preserve_reasoning_only_state { // Reasoning is an ordered event between generated bytes, not a // replacement parser input. Keep the parser state alive so a // later raw chunk can finish the same native call or guided // argument object. Kimi may still be buffering an open JSON // string before its first tool-call event. if queue_qwen_opener_reasoning { let reasoning = original .delta .reasoning_content .take() .expect("reasoning-only chunk has reasoning content"); states .get_mut(&original.index) .expect("preserved parser state must still exist") .queue_qwen_opener_reasoning( reasoning.clone(), &mut original.logprobs, ); let mut source_data = chat.clone(); source_data.inner.choices.clear(); let keep_source_metadata = !qwen_source_metadata_deferred; qwen_source_metadata_deferred = true; if !keep_source_metadata { source_data.tool_call_completion.clear(); source_data.inner.usage = None; source_data.nvext = None; source_data.llm_metrics = None; } record.deferred_qwen_xml_reasoning_responses.push_back( DeferredQwenXmlReasoningResponse { text: reasoning, choice: original.clone(), response: Annotated { data: Some(source_data), id: if keep_source_metadata { response_id.clone() } else { None }, event: if keep_source_metadata { response_event.clone() } else { None }, comment: if keep_source_metadata { response_comment.clone() } else { None }, error: if keep_source_metadata { response_error.clone() } else { None }, }, }, ); } } else if let Some(mut state) = states.remove(&original.index) { // A prior raw chunk on this choice may have emitted a tool // call before this already-parsed terminal replaced it; that // history lives only in the state being discarded here, so // fold it into the record before it's gone (also needed by a // later chunk or the EOF backstop after this `ChoiceState` is // gone). Normalize against `record.tool_emitted`, not the // state's own flag: this chunk's own `delta.tool_calls` (set // into `record` above) can carry a tool call the state itself // never saw, e.g. an already-parsed terminal chunk that is // itself the first and only tool-call signal for this choice. if state.tool_emitted() { record.tool_emitted = true; } record .pending_tool_calls .extend( state .pending_tool_calls .iter() .filter(|index| !reported_tool_indices.contains(*index)) .copied(), ); record.opened_calls.extend(state.opened_calls.iter().copied()); record .tool_ids_emitted .extend(state.tool_ids_emitted.iter().copied()); record .tool_types_emitted .extend(state.tool_types_emitted.iter().copied()); record .parser_tool_indices .extend(state.parser_tool_indices.iter().map(|(k, v)| (*k, *v))); record.passthrough_tool_indices.extend( state .passthrough_tool_indices .iter() .map(|(k, v)| (*k, *v)), ); record.next_tool_index = record.next_tool_index.max(state.next_tool_index); state .pending_tool_calls .retain(|index| !reported_tool_indices.contains(index)); state .pending_tool_calls .extend(record.pending_tool_calls.iter().copied()); state .pending_parsed_tool_calls .clone_from(&record.pending_parsed_tool_calls); let deltas = state.finish(); if let Some(error) = state.parse_error.take() { yield parser_error_annotation(error); return; } let flushed = state.choices_for( &empty_choice(original.index), deltas, true, None, ); for output in stream_chunks_for_choices(&state, record, flushed) { match output { FinishedStreamChunk::Error(error) => { yield parser_error_annotation(error); return; } FinishedStreamChunk::Source(response) => { deferred_qwen_responses.push((emitted.len(), response)); } FinishedStreamChunk::Boundary(choice, boundary) => { phase_boundaries.insert(emitted.len(), boundary); emitted.push(choice); } FinishedStreamChunk::Choice(choice) => emitted.push(choice), } } record.remember_state(&state); } if let Some(error) = incomplete_glm_terminal(family, original.finish_reason, || record.has_incomplete_tool_call()) { yield Annotated::from_err(error); return; } original.finish_reason = record.normalize_finish_reason(original.finish_reason); if original.finish_reason.is_some() { record.finished = true; } if !queue_qwen_opener_reasoning { emitted.push(original); } continue; } // Keep a provisional native call's parser across a reasoning-only // detour so its argument tail can finish. If the resumed bytes start // a fresh call instead, finish the old parser first; the record keeps // its emitted index pending while the next parser gets a new index. // Qwen's lexical state distinguishes a new block marker from the same // text inside a parameter value; JSON grammars use their own quoted // strings and may legally contain another parser marker there. let new_native_call_after_detour = records .get(&original.index) .is_some_and(|record| record.detoured) && uses_qwen_xml_grammar(family) && states .get(&original.index) .is_some_and(|state| { matches!(&state.tool_output_mode, UnifiedToolOutputMode::Native) && !state.pending_tool_calls.is_empty() && state.qwen_resumed_marker_can_start_call() }) && match original.delta.content.as_ref() { Some(ChatCompletionMessageContent::Text(text)) => { first_unquoted_structural_tool_call_marker( text, QWEN3_TOOL_CALL_PARSER, ) .is_some_and(|marker| text[..marker].trim().is_empty()) } _ => false, }; if new_native_call_after_detour && let Some(mut state) = states.remove(&original.index) { let deltas = state.finish(); if let Some(error) = state.parse_error.take() { yield parser_error_annotation(error); return; } let flushed = state.choices_for( &empty_choice(original.index), deltas, true, None, ); let record = records.entry(original.index).or_default(); for output in stream_chunks_for_choices(&state, record, flushed) { match output { FinishedStreamChunk::Error(error) => { yield parser_error_annotation(error); return; } FinishedStreamChunk::Source(response) => { deferred_qwen_responses.push((emitted.len(), response)); } FinishedStreamChunk::Boundary(choice, boundary) => { phase_boundaries.insert(emitted.len(), boundary); emitted.push(choice); } FinishedStreamChunk::Choice(choice) => emitted.push(choice), } } record.remember_state(&state); } let mut detected_prefill = prefill; if !states.contains_key(&original.index) && (prefill == UnifiedParserStartingState::Reasoning || policy.structured_response && prefill == UnifiedParserStartingState::None) && policy.inspects_json_prefill(&guided_tool_constraint) && !has_consumed_harmony_prefix && !records.get(&original.index).is_some_and(|record| record.detoured) { let text = match original.delta.content.as_ref() { Some(ChatCompletionMessageContent::Text(text)) => text.as_str(), _ => "", }; let mut probe = undecided_guided_prefix.remove(&original.index).unwrap_or_default(); probe.push(text); let probe_decision = if prefill == UnifiedParserStartingState::None { // Without a prompt-opened channel, non-JSON may contain an explicit // reasoning opener. Let the native parser recognize it. probe.unopened_channel_decision(family, original.finish_reason.is_some()) } else { match probe.decision_for_family(family, original.finish_reason.is_some(), text) { Ok(decision) => decision, Err(error) => { tracing::warn!( error = %error, family, choice = original.index, "structured response prefill detection failed; treating buffered output as reasoning" ); Some(UnifiedParserStartingState::Reasoning) } } }; let awaiting_reasoning_boundary = prefill == UnifiedParserStartingState::Reasoning && policy.structured_response && !policy.reasoning_disabled && probe_decision == Some(UnifiedParserStartingState::None) && original.finish_reason.is_none(); if (probe.text.trim().is_empty() && original.finish_reason.is_none()) || policy.structured_response && (probe_decision.is_none() || awaiting_reasoning_boundary) { undecided_guided_prefix.insert(original.index, probe); original.delta.content = None; emitted.push(original); continue; } if prefill == UnifiedParserStartingState::None && probe_decision == Some(UnifiedParserStartingState::Response) { detected_prefill = UnifiedParserStartingState::Response; } original.delta.content = Some(ChatCompletionMessageContent::Text(probe.text)); } let state = match states.entry(original.index) { std::collections::hash_map::Entry::Occupied(entry) => entry.into_mut(), std::collections::hash_map::Entry::Vacant(entry) => { let mode = tool_output_mode(&guided_tool_constraint); // `Vacant` covers both a brand-new choice AND one whose // `ChoiceState` was just dropped by an already-parsed detour. // Only the former may start at the request-level `prefill` // (what the PROMPT opened); a choice that has already been // through a detour has, by definition, already emitted // structured output, so its reasoning phase (if any) already // concluded and a resumed raw run must start at `Response`. let choice_prefill = if records .get(&original.index) .is_some_and(|record| record.detoured) { UnifiedParserStartingState::Response } else if prefill == UnifiedParserStartingState::Reasoning && policy.inspects_json_prefill(&guided_tool_constraint) && !has_consumed_harmony_prefix { if policy.structured_response { match original.delta.content.as_ref() { Some(ChatCompletionMessageContent::Text(text)) => match structured_json_response_prefill(family, text, original.finish_reason.is_some()) { Ok(Some(state)) => state, Ok(None) => UnifiedParserStartingState::Reasoning, Err(error) => { tracing::warn!( error = %error, family, choice = original.index, "structured response prefill detection failed; treating output as reasoning" ); UnifiedParserStartingState::Reasoning } }, _ => UnifiedParserStartingState::Reasoning, } } else { bare_guided_json_prefill(family, original.delta.content.as_ref()) } } else { detected_prefill }; let pending_whitespace = records .get_mut(&original.index) .map(|record| { std::mem::take( &mut record.pending_guided_prefill_whitespace, ) }) .unwrap_or_default(); if choice_prefill == UnifiedParserStartingState::Reasoning && !pending_whitespace.is_empty() { match original.delta.content.as_mut() { Some(ChatCompletionMessageContent::Text(text)) => { let mut combined = pending_whitespace; combined.push_str(text); *text = combined; } None => { original.delta.content = Some( ChatCompletionMessageContent::Text(pending_whitespace), ); } Some(ChatCompletionMessageContent::Parts(_)) => { unreachable!("already parsed parts bypass raw parser setup") } } } let uses_request_prefill = choice_prefill == prefill && !records.get(&original.index).is_some_and(|record| record.detoured); let choice_prefill = StreamPrefill { starting_state: choice_prefill, prefix: uses_request_prefill.then(|| prefill_prefix.clone()).flatten(), }; let cache = if uses_request_prefill { harmony_prefill_cache.as_mut() } else { None }; match ChoiceState::new_with_prefill_cache(family, &tools, choice_prefill, mode, guided_streaming, argument_framing, cache) { Ok(mut state) => { state.preserve_message_phases(family == MUSE_SPARK_UNIFIED_FAMILY); // Only a successful checkpoint replaces the request seed. // Detours use a fresh response parser and never this seed. if harmony_prefill_cache.as_mut().is_some_and(HarmonyPrefill::release_prefix) { prefill_prefix = None; } // A raw run resuming after an already-parsed detour // gets a brand-new parser instance; seed it with any // tool-call history from before the gap so a Stop at // the end of this run still normalizes correctly. if records .get(&original.index) .is_some_and(|record| record.tool_emitted) { state.mark_tool_emitted(); } if let Some(record) = records.get(&original.index) { state.incomplete_native_call = record.incomplete_native_call; state.tool_index_offset = record.next_tool_index; state.next_tool_index = record.next_tool_index; state.opened_calls.extend(record.opened_calls.iter().copied()); state .tool_ids_emitted .extend(record.tool_ids_emitted.iter().copied()); state .tool_types_emitted .extend(record.tool_types_emitted.iter().copied()); state .parser_tool_indices .extend(record.parser_tool_indices.iter().map(|(k, v)| (*k, *v))); state.passthrough_tool_indices.extend( record .passthrough_tool_indices .iter() .map(|(k, v)| (*k, *v)), ); state.parser_call_completion.clone_from(&record.parser_call_completion); state .pending_tool_calls .extend(record.pending_tool_calls.iter().copied()); state .pending_parsed_tool_calls .clone_from(&record.pending_parsed_tool_calls); } entry.insert(state) } Err(error) => { tracing::warn!( error = %error, family, choice = original.index, "unified parser construction failed; passing choice through" ); emitted.push(original); continue; } } } }; let mut deltas = Vec::new(); if let Some(ChatCompletionMessageContent::Text(text)) = original.delta.content.as_ref() { deltas.extend(state.push(text)); } let terminal = original.finish_reason; let already_finished = records .get(&original.index) .is_some_and(|record| record.finished); if terminal.is_some() { if !already_finished { let confirmed_eos = family == BEAM_UNIFIED_FAMILY && matches!(guided_tool_constraint, GuidedToolConstraint::BeamTurnGrammar { .. }) && terminal == Some(FinishReason::Stop) && chat.model_eos.contains(&original.index); deltas.extend(state.finish_with_eos(confirmed_eos)); } records.entry(original.index).or_default().finished = true; } if let Some(error) = state.parse_error.take() { yield parser_error_annotation(error); return; } let mut choices = state.choices_for(&original, deltas, true, terminal); merge_passthrough_completion( &mut state.parser_call_completion, &state.passthrough_tool_indices, &chat.tool_call_completion, original.index, ); if let Some(error) = incomplete_glm_terminal(family, terminal, || state.has_incomplete_tool_call()) { yield Annotated::from_err(error); return; } // Pass-through indices are assigned by choices_for, so completion // evidence must be remapped before accepting its terminal decision. for choice in &mut choices { choice.finish_reason = state.normalize_finish_reason(choice.finish_reason); } let mut parsed = Vec::new(); let record = records.entry(original.index).or_default(); for output in stream_chunks_for_choices(state, record, choices) { match output { FinishedStreamChunk::Error(error) => { yield parser_error_annotation(error); return; } FinishedStreamChunk::Source(response) => { deferred_qwen_responses.push((emitted.len() + parsed.len(), response)); } FinishedStreamChunk::Boundary(choice, boundary) => { phase_boundaries.insert(emitted.len() + parsed.len(), boundary); parsed.push(choice); } FinishedStreamChunk::Choice(choice) => parsed.push(choice), } } if parsed.is_empty() { // A marker-only chunk produced no deltas. Keep it as an empty // choice so the typed llm_metrics and annotation metadata it // carries still reach the client. parsed.push(empty_choice(original.index)); } emitted.extend(parsed); } if qwen_source_metadata_deferred { chat.tool_call_completion.clear(); chat.inner.usage = None; chat.nvext = None; chat.llm_metrics = None; response.id = None; response.event = None; response.comment = None; response.error = None; } if emitted.is_empty() && deferred_qwen_responses.is_empty() { continue; } // One upstream chunk can fan out into several. Usage, nvext and annotation // fields stay on the last child. Token metrics stay on one reasoning child // when present so the downstream reasoning-usage estimator preserves the // source chunk's classification without counting it twice. let last = emitted.len().checked_sub(1); let llm_metrics_position = fanout_llm_metrics_position(&emitted); let mut deferred_qwen_responses = deferred_qwen_responses.into_iter().peekable(); for position in 0..=emitted.len() { while deferred_qwen_responses .peek() .is_some_and(|(deferred_position, _)| *deferred_position == position) { let (_, response) = deferred_qwen_responses .next() .expect("peeked Qwen response should be present"); yield normalized_source_response(response, &states, &records); } let Some(choice) = emitted.get(position).cloned() else { continue; }; let is_last = Some(position) == last; let mut data = chat.clone(); data.message_boundary = phase_boundaries.get(&position).cloned(); if !is_last { data.tool_call_completion.clear(); } // Every child referencing a call can carry its final parser state, // even before its arguments. Completion is not a delivery barrier. attach_call_completion(&mut data, &choice, &states, &records); data.inner.choices = vec![choice]; if !is_last { data.inner.usage = None; data.nvext = None; } if Some(position) != llm_metrics_position { data.llm_metrics = None; } yield Annotated { data: Some(data), id: if is_last { response.id.take() } else { None }, event: if is_last { response.event.take() } else { None }, comment: if is_last { response.comment.take() } else { None }, error: if is_last { response.error.take() } else { None }, }; } } // Backstop: the stream ended without a terminating chunk for some choice. if let Some(template) = &template { for choice in flush_undecided_guided_prefixes(&mut undecided_guided_prefix, policy, prefill) { yield response_with_choice(template, choice, &states, &records); } for output in finish_unterminated_choices(&mut states, &mut records, family) { match output { FinishedStreamChunk::Error(error) => { yield parser_error_annotation(error); return; } FinishedStreamChunk::Choice(choice) => { yield response_with_choice(template, choice, &states, &records); } FinishedStreamChunk::Boundary(choice, boundary) => { let mut response = response_with_choice(template, choice, &states, &records); response.data.as_mut().unwrap().message_boundary = Some(boundary); yield response; } FinishedStreamChunk::Source(response) => yield normalized_source_response(response, &states, &records), } } } } } #[cfg(test)] pub(crate) mod tests { use super::*; use dynamo_parsers_v2::{ToolCallDelta, UnifiedParserExt}; use dynamo_protocols::types::{ ChatCompletionStreamResponseDeltaFunctionCall, CreateChatCompletionStreamResponse, Role, }; use futures::FutureExt; use futures::stream; struct PartialCommitParser { recovered: String, } impl UnifiedParser for PartialCommitParser { fn parse_into( &mut self, _delta: &str, output: &mut UnifiedParserOutput, ) -> anyhow::Result<()> { output .events .push(UnifiedParserEvent::Text("committed".to_string())); anyhow::bail!("intentional failure after a committed event") } fn finish(&mut self) -> anyhow::Result { Ok(UnifiedParserOutput::default()) } fn reset(&mut self) -> String { std::mem::take(&mut self.recovered) } } fn chunk(text: &str, finish: bool) -> Annotated { #[allow(deprecated)] let response = NvCreateChatCompletionStreamResponse { message_boundary: None, inner: CreateChatCompletionStreamResponse { id: "test".to_string(), choices: vec![ChatChoiceStream { index: 0, delta: ChatCompletionStreamResponseDelta { role: Some(Role::Assistant), content: Some(ChatCompletionMessageContent::Text(text.to_string())), tool_calls: None, function_call: None, refusal: None, reasoning_content: None, }, finish_reason: finish.then_some(FinishReason::Stop), logprobs: None, }], created: 0, model: "qwen3".to_string(), system_fingerprint: None, service_tier: None, object: "chat.completion.chunk".to_string(), usage: None, }, nvext: None, llm_metrics: None, prompt_logprobs: None, tool_call_completion: Vec::new(), model_eos: Vec::new(), }; Annotated::from_data(response) } fn weather_tools() -> Vec { vec![ToolDefinition { name: "get_weather".to_string(), parameters: Some(serde_json::json!({ "type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"] })), strict: None, }] } fn count_tools() -> Vec { vec![ToolDefinition { name: "count_items".to_string(), parameters: Some(serde_json::json!({ "type": "object", "properties": {"count": {"type": "integer"}}, "required": ["count"] })), strict: None, }] } fn collect_choices( responses: &[Annotated], ) -> Vec<&ChatChoiceStream> { responses .iter() .filter_map(|response| response.data.as_ref()) .flat_map(|data| data.inner.choices.iter()) .collect() } fn named_choice(name: &str) -> ChatCompletionToolChoiceOption { serde_json::from_value(serde_json::json!({ "type": "function", "function": {"name": name} })) .expect("named tool choice") } #[derive(Debug, PartialEq, Eq)] enum LogicalEvent { Reasoning(String), Text(String), Tool { index: u32, name: Option, arguments: String, }, } fn logical_events( responses: &[Annotated], ) -> Vec { let mut events = Vec::new(); for choice in collect_choices(responses) { if let Some(reasoning) = &choice.delta.reasoning_content { match events.last_mut() { Some(LogicalEvent::Reasoning(existing)) => existing.push_str(reasoning), _ => events.push(LogicalEvent::Reasoning(reasoning.clone())), } } if let Some(ChatCompletionMessageContent::Text(text)) = &choice.delta.content { match events.last_mut() { Some(LogicalEvent::Text(existing)) => existing.push_str(text), _ => events.push(LogicalEvent::Text(text.clone())), } } for call in choice.delta.tool_calls.iter().flatten() { let name = call.function.as_ref().and_then(|f| f.name.clone()); let arguments = call .function .as_ref() .and_then(|f| f.arguments.clone()) .unwrap_or_default(); match events.last_mut() { Some(LogicalEvent::Tool { index, name: existing_name, arguments: existing_arguments, }) if *index == call.index => { if existing_name.is_none() { *existing_name = name; } existing_arguments.push_str(&arguments); } _ => events.push(LogicalEvent::Tool { index: call.index, name, arguments, }), } } } events } async fn parse_at_split( input: &str, split: usize, tool_choice: Option, ) -> Vec { let (first, second) = input.split_at(split); let responses = apply_stream( stream::iter([chunk(first, false), chunk(second, true)]), Some(weather_tools()), tool_choice, false, UnifiedParserStartingState::None, QWEN3_UNIFIED_FAMILY, ) .collect::>() .await; logical_events(&responses) } // --- selected_family / configured_family ------------------------------------- #[test] fn deepseek_v41_uses_native_parsing_without_an_experimental_flag() { let pair = (Some("deepseek_v41"), Some("deepseek_v41")); assert_eq!(configured_family(pair.0, pair.1), Some("deepseek_v41")); assert_eq!( selected_family(pair.0, pair.1), (super::super::tool_parser_v2::selected_version().unwrap() != dynamo_runtime::config::ParserVersion::V1) .then_some("deepseek_v41") ); assert_eq!(selected_batch_family(pair.0, pair.1), Some("deepseek_v41")); assert_eq!(configured_family(Some("deepseek_v41"), Some("qwen3")), None); assert_eq!( configured_family(Some("deepseek_v41"), None), Some("deepseek_v41") ); assert_eq!( configured_family(None, Some("deepseek_v41")), Some("deepseek_v41") ); } #[tokio::test] async fn deepseek_v41_preserves_calls_across_utf8_chunk_boundaries() { let call = concat!( "<|DSML| calls><|DSML| invoke name=\"get_weather\">", "<|DSML| parameter name=\"city\" string=\"true\">東京", "" ); for (prefix, prefill, reasoning) in [ ("", UnifiedParserStartingState::Response, ""), ( "plan", UnifiedParserStartingState::Reasoning, "plan", ), ( "plan", UnifiedParserStartingState::None, "plan", ), ] { let input = format!("{prefix}{call}"); let batch = parse_complete( DEEPSEEK_V41_UNIFIED_FAMILY, &input, &GuidedToolConstraint::None, &[], ) .unwrap(); assert_eq!(batch.reasoning, reasoning); assert_eq!(batch.tool_calls.len(), 1); assert_eq!(batch.tool_calls[0].function.arguments, r#"{"city":"東京"}"#); for split in input.char_indices().map(|(index, _)| index) { let (first, second) = input.split_at(split); let responses = apply_stream( stream::iter([chunk(first, false), chunk(second, true)]), Some(weather_tools()), None, false, prefill, DEEPSEEK_V41_UNIFIED_FAMILY, ) .collect::>() .await; let mut expected = Vec::new(); if !reasoning.is_empty() { expected.push(LogicalEvent::Reasoning(reasoning.to_string())); } expected.push(LogicalEvent::Tool { index: 0, name: Some("get_weather".to_string()), arguments: r#"{"city":"東京"}"#.to_string(), }); let mut actual = logical_events(&responses); for event in &mut actual { if let LogicalEvent::Tool { arguments, .. } = event { *arguments = serde_json::from_str::(arguments) .unwrap() .to_string(); } } assert_eq!(actual, expected, "split={split}"); assert_eq!( collect_choices(&responses).last().unwrap().finish_reason, Some(FinishReason::ToolCalls) ); } } } #[tokio::test] async fn deepseek_v41_complete_call_survives_missing_outer_closer() { // Model the frontend hiding an EOS token that is also the outer DSML // closer. A complete invocation is committed at ``, so // removing only `` must not drop the call. let input = concat!( "<|DSML| calls><|DSML| invoke name=\"get_weather\">", "<|DSML| parameter name=\"city\" string=\"true\">Tokyo", "" ); let batch = parse_complete( DEEPSEEK_V41_UNIFIED_FAMILY, input, &GuidedToolConstraint::None, &weather_tools(), ) .unwrap(); assert_eq!(batch.tool_calls.len(), 1); assert_eq!(batch.tool_calls[0].function.name, "get_weather"); assert_eq!( batch.tool_calls[0].function.arguments, r#"{"city":"Tokyo"}"# ); let responses = apply_stream( stream::iter([chunk(input, true)]), Some(weather_tools()), None, false, UnifiedParserStartingState::Response, DEEPSEEK_V41_UNIFIED_FAMILY, ) .collect::>() .await; assert!(responses.iter().all(|response| !response.is_error())); assert_eq!( logical_events(&responses), vec![LogicalEvent::Tool { index: 0, name: Some("get_weather".to_string()), arguments: r#"{"city":"Tokyo"}"#.to_string(), }] ); assert_eq!( collect_choices(&responses).last().unwrap().finish_reason, Some(FinishReason::ToolCalls) ); } #[tokio::test] async fn deepseek_v41_recovers_malformed_closed_arguments_as_text() { let text = concat!( "<|DSML| calls><|DSML| invoke name=\"get_weather\">", "<|DSML| parameter name=\"count\" string=\"false\">not-json", "" ); let responses = apply_stream( stream::iter([chunk(text, true)]), Some(weather_tools()), None, false, UnifiedParserStartingState::Response, DEEPSEEK_V41_UNIFIED_FAMILY, ) .collect::>() .await; assert!(responses.iter().all(|response| !response.is_error())); assert_eq!( logical_events(&responses), vec![LogicalEvent::Text(text.to_string())] ); assert_eq!( collect_choices(&responses).last().unwrap().finish_reason, Some(FinishReason::Stop) ); } #[test] fn pairs_only_qwen3_coder_with_qwen3() { assert_eq!( configured_family(Some("qwen3_coder"), Some("qwen3")), Some(QWEN3_UNIFIED_FAMILY) ); // A single configured half selects the same family. assert_eq!(configured_family(Some("qwen3_coder"), None), Some("qwen3")); assert_eq!(configured_family(None, Some("qwen3")), Some("qwen3")); assert_eq!(configured_family(None, None), None); // Right tool parser, wrong reasoning family. assert_eq!( configured_family(Some("qwen3_coder"), Some("deepseek_r1")), None ); // Right reasoning parser, wrong tool family. assert_eq!(configured_family(Some("kimi_k2"), Some("qwen3")), None); // The canonical name and Dynamo tool alias select the same owner. assert_eq!( configured_family(Some("qwen3"), Some("qwen3")), Some("qwen3") ); } #[test] fn nemotron_pairs_qwen3_coder_with_nemotron_v3_reasoning() { // The released reasoning parser also answers to `nemotron3`; the canonical // spelling is covered by `eight_family_routing_all_modes`. assert_eq!( configured_family(Some("qwen3_coder"), Some("nemotron3")), Some(NEMOTRON_V3_UNIFIED_FAMILY) ); assert_eq!( configured_family(None, Some("nemotron3")), Some(NEMOTRON_V3_UNIFIED_FAMILY) ); // The reverse pairing (Nemotron tool half, Qwen reasoning) names no owner. assert_eq!(configured_family(Some("nemotron_v3"), Some("qwen3")), None); assert_eq!( configured_family(Some("kimi_k2"), Some("nemotron_v3")), None ); // The older Nemotron JSON tool grammar is not the shared XML one. assert_eq!( configured_family(Some("qwen3_coder"), Some("nemotron_nano")), None ); } #[test] fn nemotron_prefill_defaults_to_reasoning_unless_the_request_disables_it() { assert_eq!( stream_prefill(NEMOTRON_V3_UNIFIED_FAMILY, true), UnifiedParserStartingState::Reasoning ); assert_eq!( stream_prefill(NEMOTRON_V3_UNIFIED_FAMILY, false), UnifiedParserStartingState::Reasoning ); let disabled = UnifiedRequestPolicy { reasoning_disabled: true, ..Default::default() }; assert_eq!( disabled.starting_state( NEMOTRON_V3_UNIFIED_FAMILY, UnifiedParserStartingState::Reasoning ), UnifiedParserStartingState::Response ); assert_eq!( UnifiedRequestPolicy::default().starting_state( NEMOTRON_V3_UNIFIED_FAMILY, UnifiedParserStartingState::Reasoning ), UnifiedParserStartingState::Reasoning ); // Batch: a bare closer means the prompt pre-opened the thought; marker-free // text is the answer (the streaming path already split any thought off it). assert_eq!( detect_prefill(NEMOTRON_V3_UNIFIED_FAMILY, "reasonanswer").unwrap(), UnifiedParserStartingState::Reasoning ); assert_eq!( detect_prefill(NEMOTRON_V3_UNIFIED_FAMILY, "Hello!").unwrap(), UnifiedParserStartingState::Response ); assert_eq!( detect_prefill(NEMOTRON_V3_UNIFIED_FAMILY, "reasonanswer").unwrap(), UnifiedParserStartingState::None ); assert_eq!( disabled .batch_starting_state(NEMOTRON_V3_UNIFIED_FAMILY, "answer") .unwrap(), UnifiedParserStartingState::Response ); // Bare guided JSON must reach the tool parser: `None` is the family default, // which for Nemotron is the implicit thought, so the state is `Response`. let json = ChatCompletionMessageContent::Text(r#"{"city":"Paris"}"#.into()); assert_eq!( bare_guided_json_prefill(NEMOTRON_V3_UNIFIED_FAMILY, Some(&json)), UnifiedParserStartingState::Response ); assert_eq!( bare_guided_json_prefill(QWEN3_UNIFIED_FAMILY, Some(&json)), UnifiedParserStartingState::None ); let prose = ChatCompletionMessageContent::Text("thinking first".into()); assert_eq!( bare_guided_json_prefill(NEMOTRON_V3_UNIFIED_FAMILY, Some(&prose)), UnifiedParserStartingState::Reasoning ); } #[tokio::test] async fn nemotron_bare_guided_json_becomes_a_tool_call_in_stream_and_batch() { // The prompt pre-opened reasoning (`prefill = Reasoning`) and guided decoding // forbids the closer, so the payload's first byte is JSON. Both paths must // return the forced call instead of consuming the JSON as reasoning. let tools = weather_tools(); for (constraint, raw) in [ ( GuidedToolConstraint::GuidedJsonNamed { tool_name: "get_weather".into(), }, r#"{"city":"Paris"}"#, ), ( GuidedToolConstraint::GuidedJsonRequired, r#"[{"name":"get_weather","arguments":{"city":"Paris"}}]"#, ), ] { let batch = parse_complete_with_policy( NEMOTRON_V3_UNIFIED_FAMILY, raw, &constraint, &tools, UnifiedRequestPolicy::default(), ) .unwrap(); assert!(batch.reasoning.is_empty(), "batch {raw}"); assert!(batch.text.is_empty(), "batch {raw}"); assert_eq!(batch.tool_calls.len(), 1, "batch {raw}"); assert_eq!(batch.tool_calls[0].function.name, "get_weather"); assert_eq!( serde_json::from_str::(&batch.tool_calls[0].function.arguments) .unwrap(), serde_json::json!({"city": "Paris"}) ); for split in 0..=raw.len() { let outputs: Vec<_> = apply_stream_with_policy( futures::stream::iter([ chunk(&raw[..split], false), chunk(&raw[split..], true), ]), Some(tools.clone()), constraint.clone(), stream_prefill(NEMOTRON_V3_UNIFIED_FAMILY, true), NEMOTRON_V3_UNIFIED_FAMILY, true, UnifiedRequestPolicy::default(), ) .collect() .await; let choices = collect_choices(&outputs); assert!( choices .iter() .all(|choice| choice.delta.reasoning_content.is_none()), "split {split}: {raw}: {choices:?}" ); let names: Vec<_> = choices .iter() .filter_map(|choice| choice.delta.tool_calls.as_ref()) .flatten() .filter_map(|call| call.function.as_ref()?.name.as_deref()) .collect(); assert_eq!(names, ["get_weather"], "split {split}: {raw}"); let arguments: String = choices .iter() .filter_map(|choice| choice.delta.tool_calls.as_ref()) .flatten() .filter_map(|call| call.function.as_ref()?.arguments.as_deref()) .collect(); assert_eq!( serde_json::from_str::(&arguments).unwrap(), serde_json::json!({"city": "Paris"}), "split {split}: {raw}" ); } } } #[tokio::test] async fn nemotron_thinking_on_and_off_split_channels_at_every_split() { let call = "Montréal"; let tools = weather_tools(); let cases = [ // enable_thinking=true: the prompt opened ``, the model closes it. ( format!("Check Montréal.Sure.{call}"), UnifiedRequestPolicy::default(), "Check Montréal.", "Sure.", ), // enable_thinking=false: the prompt rendered ``; a literal // marker in the answer stays visible text. ( format!("Sure, is literal.{call}"), UnifiedRequestPolicy { reasoning_disabled: true, ..Default::default() }, "", "Sure, is literal.", ), ]; for (raw, policy, want_reasoning, want_text) in cases { let batch = parse_complete_with_policy( NEMOTRON_V3_UNIFIED_FAMILY, &raw, &GuidedToolConstraint::None, &tools, policy, ) .unwrap(); assert_eq!(batch.reasoning, want_reasoning, "batch {raw}"); assert_eq!(batch.text, want_text, "batch {raw}"); assert_eq!(batch.tool_calls.len(), 1, "batch {raw}"); assert_eq!(batch.tool_calls[0].function.name, "get_weather"); assert_eq!( serde_json::from_str::(&batch.tool_calls[0].function.arguments) .unwrap(), serde_json::json!({"city": "Montréal"}) ); for split in (0..=raw.len()).filter(|split| raw.is_char_boundary(*split)) { let outputs: Vec<_> = apply_stream_with_policy( futures::stream::iter([ chunk(&raw[..split], false), chunk(&raw[split..], true), ]), Some(tools.clone()), GuidedToolConstraint::None, stream_prefill(NEMOTRON_V3_UNIFIED_FAMILY, false), NEMOTRON_V3_UNIFIED_FAMILY, true, policy, ) .collect() .await; let choices = collect_choices(&outputs); let reasoning: String = choices .iter() .filter_map(|choice| choice.delta.reasoning_content.as_deref()) .collect(); let text: String = choices .iter() .filter_map(|choice| match &choice.delta.content { Some(ChatCompletionMessageContent::Text(text)) => Some(text.as_str()), _ => None, }) .collect(); assert_eq!(reasoning, want_reasoning, "stream split {split}: {raw}"); assert_eq!(text, want_text, "stream split {split}: {raw}"); let names: Vec<_> = choices .iter() .filter_map(|choice| choice.delta.tool_calls.as_ref()) .flatten() .filter_map(|call| call.function.as_ref()?.name.as_deref()) .collect(); assert_eq!(names, ["get_weather"], "stream split {split}: {raw}"); } } } #[test] fn batch_routing_uses_the_configured_pair_for_every_output_mode() { let pair = (Some("qwen3_coder"), Some("qwen3")); assert_eq!( configured_family(pair.0, pair.1), Some(QWEN3_UNIFIED_FAMILY), "the carried constraint selects native versus guided parsing after routing" ); } // --- tool_choice -> UnifiedToolOutputMode ------------------------------------ #[test] fn gpt_oss_aliases_select_the_unified_route_unless_v1() { use dynamo_runtime::config::ParserVersion; for (tool, reasoning) in [ (Some("harmony"), Some("gpt_oss")), (Some("harmony"), None), (None, Some("gpt_oss")), ] { let family = configured_family(tool, reasoning); assert_eq!(family, Some("gpt_oss")); for (version, expected) in [ (ParserVersion::Auto, Some("gpt_oss")), (ParserVersion::V1, None), (ParserVersion::V2, Some("gpt_oss")), ] { assert_eq!( selected_family_for_selection(tool, reasoning, version), expected ); } } } #[tokio::test] async fn gpt_oss_consumed_headers_and_guided_calls_match_batch_at_every_split() { for (input, constraint, reasoning) in [ ( " to=functions.get_weather<|channel|>analysis<|message|>{\"city\":\"Tokyo\"}<|call|>", GuidedToolConstraint::None, "", ), ( "<|channel|>analysis<|message|>plan<|end|><|start|>assistant to=functions.get_weather<|channel|>commentary<|message|>{\"city\":\"Tokyo\"}<|call|>", GuidedToolConstraint::StructuralTag, "plan", ), ( r#"[{"name":"get_weather","parameters":{"city":"Tokyo"}}]"#, GuidedToolConstraint::GuidedJsonRequired, "", ), ( "<|channel|>analysis<|message|>plan<|end|><|start|>assistant<|channel|>final<|message|>[{\"name\":\"get_weather\",\"parameters\":{\"city\":\"Tokyo\"}}]<|return|>", GuidedToolConstraint::GuidedJsonRequired, "plan", ), ( r#"{"city":"Tokyo"}"#, GuidedToolConstraint::GuidedJsonNamed { tool_name: "get_weather".to_string(), }, "", ), ] { let batch = parse_complete("gpt_oss", input, &constraint, &weather_tools()).unwrap(); assert!(batch.text.is_empty(), "{:?}", batch.text); assert_eq!(batch.reasoning, reasoning); assert_eq!(batch.tool_calls.len(), 1); assert_eq!(batch.tool_calls[0].function.name, "get_weather"); assert_eq!( serde_json::from_str::(&batch.tool_calls[0].function.arguments) .unwrap(), serde_json::json!({"city":"Tokyo"}) ); for split in 0..=input.len() { let output = apply_stream_with_constraint( stream::iter([chunk(&input[..split], false), chunk(&input[split..], true)]), Some(weather_tools()), constraint.clone(), UnifiedParserStartingState::None, "gpt_oss", true, ) .collect::>() .await; assert!(output.iter().all(|response| !response.is_error())); let events = logical_events(&output); assert_stream_calls_match_batch(&events, &batch, &format!("split={split}")); let streamed_reasoning: String = events .iter() .filter_map(|event| match event { LogicalEvent::Reasoning(text) => Some(text.as_str()), _ => None, }) .collect(); assert_eq!(streamed_reasoning, reasoning, "split={split}"); assert!( !events .iter() .any(|event| matches!(event, LogicalEvent::Text(text) if !text.is_empty())), "split={split}: {events:?}" ); assert_eq!( collect_choices(&output).last().unwrap().finish_reason, Some(FinishReason::ToolCalls) ); } } } #[tokio::test] async fn gpt_oss_stranded_final_reaches_stream_and_batch_at_stop_and_eof() { let input = "<|channel|>analysis<|message|>thought<|end|><|start|>assistant<|channel|>final*)__Hello!"; let batch = parse_complete("gpt_oss", input, &GuidedToolConstraint::None, &[]).unwrap(); assert_eq!(batch.text, "*)__Hello!"); assert_eq!(batch.reasoning, "thought"); assert!(batch.tool_calls.is_empty()); for terminal in [false, true] { for split in 0..=input.len() { let output = apply_stream_with_constraint( stream::iter([ chunk(&input[..split], false), chunk(&input[split..], terminal), ]), None, GuidedToolConstraint::None, UnifiedParserStartingState::None, "gpt_oss", true, ) .collect::>() .await; assert!(output.iter().all(|response| !response.is_error())); assert_eq!( logical_events(&output), vec![ LogicalEvent::Reasoning("thought".into()), LogicalEvent::Text("*)__Hello!".into()), ], "terminal={terminal} split={split}" ); assert_eq!( collect_choices(&output).last().unwrap().finish_reason, terminal.then_some(FinishReason::Stop) ); } } } #[test] fn gpt_oss_prefill_policy_preserves_guidance_envelope_ownership() { for (family, explicit, guided, required_array, bare) in [ ("gpt_oss", false, false, false, false), ("gpt_oss", true, false, false, true), ("gpt_oss", false, true, false, true), ("gpt_oss", false, true, true, false), ("gpt_oss", true, true, true, true), ("qwen3", true, true, true, false), ] { let constraint = if guided { GuidedToolConstraint::GuidedJsonRequired } else { GuidedToolConstraint::None }; for starting_state in [ UnifiedParserStartingState::None, UnifiedParserStartingState::Reasoning, UnifiedParserStartingState::Response, ] { let prefix = std::sync::Arc::::from("consumed prompt"); let mut array_checked = false; let actual = UnifiedRequestPolicy::default().resolve_stream_prefill( family, StreamPrefill { starting_state, prefix: Some(prefix.clone()), }, &constraint, explicit, || { array_checked = true; required_array }, ); assert_eq!(array_checked, family == "gpt_oss" && !explicit && guided); if bare { assert_eq!(actual.starting_state, UnifiedParserStartingState::Response); assert!(actual.prefix.is_none()); } else { assert_eq!(actual.starting_state, starting_state); assert!(std::sync::Arc::ptr_eq( actual.prefix.as_ref().unwrap(), &prefix )); } } } } // Consumed prompt replay and discarded output are adapter-owned, outside parser fixtures. #[test] fn gpt_oss_replay_discards_output_between_bounded_token_batches() { struct BoundedReplay { received: Vec, } impl UnifiedParser for BoundedReplay { fn parse_into(&mut self, _: &str, _: &mut UnifiedParserOutput) -> anyhow::Result<()> { anyhow::bail!("consumed prefix must remain token-native") } fn parse_token_ids_into( &mut self, ids: &[u32], output: &mut UnifiedParserOutput, ) -> anyhow::Result<()> { anyhow::ensure!(ids.len() <= 256, "whole-prefix batch"); anyhow::ensure!(output.events.is_empty(), "discarded history retained"); self.received.extend_from_slice(ids); output.push_text("consumed"); Ok(()) } fn finish(&mut self) -> anyhow::Result { Ok(UnifiedParserOutput::default()) } } let tokens: Vec = (0..1025).collect(); let mut parser = BoundedReplay { received: Vec::new(), }; assert!( parser .parse_token_ids_into(&tokens, &mut UnifiedParserOutput::default()) .is_err() ); let mut stale = UnifiedParserOutput::default(); stale.push_text("earlier consumed event"); assert!( parser .parse_token_ids_into(&tokens[..1], &mut stale) .is_err() ); replay_consumed_harmony(&mut parser, &tokens, 256).unwrap(); assert_eq!(parser.received, tokens); } #[test] fn gpt_oss_replay_chunking_preserves_continuation_state() { let large_body = "alpha ".repeat(512); let prefix = format!( "<|start|>assistant to=functions.lookup<|channel|>commentary<|message|>{{\"q\":\"{large_body}" ); let ids = dynamo_parsers_v2::unified::harmony::encode_harmony(&prefix).unwrap(); assert!( ids.len() > 256, "the consumed prefix must span replay batches" ); let generated = "tail\"}<|call|>"; let parse = |chunk_size| { let mut parser = create_unified_parser_for_family("gpt_oss", &[]).unwrap(); parser .initialize_request(UnifiedParserInit::default()) .unwrap(); replay_consumed_harmony(parser.as_mut(), &ids, chunk_size).unwrap(); let mut output = UnifiedParserOutput::default(); parser.parse_into(generated, &mut output).unwrap(); output.events.extend(parser.finish().unwrap().events); output.events }; assert_eq!( parse(256), parse(ids.len()), "production-sized replay must preserve continuation state" ); } #[test] fn gpt_oss_checkpoint_releases_raw_seed_and_forks_late_choices() { struct CountedContinuation { inner: Box, forks: std::sync::Arc, } impl UnifiedParserContinuation for CountedContinuation { fn fork(&self) -> anyhow::Result> { self.forks .fetch_add(1, std::sync::atomic::Ordering::Relaxed); self.inner.fork() } } let mut prefix: Option> = Some(std::sync::Arc::from( "<|start|>assistant to=functions.lookup<|message|>{\"q\":\"consumed", )); let observer = std::sync::Arc::downgrade(prefix.as_ref().unwrap()); let mut cache = HarmonyPrefill::new(prefix.as_ref().unwrap().clone()); assert!(!cache.release_prefix()); assert!(!cache.matches_prefix(None)); let first = ChoiceState::new_with_prefill_cache( "gpt_oss", &[], StreamPrefill { starting_state: UnifiedParserStartingState::Response, prefix: prefix.clone(), }, UnifiedToolOutputMode::Native, true, ArgumentFraming::default(), Some(&mut cache), ) .unwrap(); let forks = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); cache.continuation = Some(Box::new(CountedContinuation { inner: cache.continuation.take().unwrap(), forks: forks.clone(), })); assert!(cache.release_prefix()); drop(prefix.take()); assert!( observer.upgrade().is_none(), "raw seed remains owned after checkpoint" ); assert!(cache.matches_prefix(None)); let late = ChoiceState::new_with_prefill_cache( "gpt_oss", &[], StreamPrefill { starting_state: UnifiedParserStartingState::Response, prefix: None, }, UnifiedToolOutputMode::Native, true, ArgumentFraming::default(), Some(&mut cache), ) .unwrap(); assert_eq!(forks.load(std::sync::atomic::Ordering::Relaxed), 1); for (mut state, suffix) in [first, late].into_iter().zip(["first", "late"]) { let mut events = state.push(&format!("{suffix}\"}}<|call|>")); events.extend(state.finish()); let arguments = events .iter() .filter_map(|event| match event { UnifiedParserEvent::ToolCall(call) => Some(call.arguments.as_str()), _ => None, }) .collect::(); assert_eq!(arguments, format!("{suffix}\"}}")); } } #[tokio::test] async fn gpt_oss_released_seed_survives_late_choices_without_replaying_on_detours() { let mut parsed = chunk("", false); parsed.data.as_mut().unwrap().inner.choices[0].delta.content = None; parsed.data.as_mut().unwrap().inner.choices[0] .delta .reasoning_content = Some("parsed".into()); let mut late = chunk("late\"}<|call|>", true); late.data.as_mut().unwrap().inner.choices[0].index = 1; let responses = apply_stream_with_policy( stream::iter([ chunk("first\"}<|call|>", false), parsed, chunk("after", true), late, ]), None, GuidedToolConstraint::None, StreamPrefill { starting_state: UnifiedParserStartingState::Response, prefix: Some(std::sync::Arc::from( "<|start|>assistant to=functions.lookup<|message|>{\"q\":\"consumed", )), }, "gpt_oss", true, UnifiedRequestPolicy { multiple_choices: true, ..Default::default() }, ) .collect::>() .await; assert!(responses.iter().all(|response| !response.is_error())); let choices = collect_choices(&responses); for (index, expected, content) in [(0, "first\"}", "after"), (1, "late\"}", "")] { let selected = choices .iter() .filter(|choice| choice.index == index) .collect::>(); let arguments = selected .iter() .flat_map(|choice| choice.delta.tool_calls.iter().flatten()) .filter_map(|call| call.function.as_ref()?.arguments.as_deref()) .collect::(); let text = selected .iter() .filter_map(|choice| match &choice.delta.content { Some(ChatCompletionMessageContent::Text(text)) => Some(text.as_str()), _ => None, }) .collect::(); assert_eq!(arguments, expected); assert_eq!(text, content); assert_eq!( selected.last().unwrap().finish_reason, Some(FinishReason::ToolCalls) ); } } #[test] fn gpt_oss_choices_share_continuation_without_sharing_generated_state() { let prefix: std::sync::Arc = std::sync::Arc::from( "<|start|>assistant to=functions.lookup<|channel|>commentary<|message|>{\"q\":\"hé", ); assert!( UnifiedRequestPolicy::default() .harmony_prefill_cache("gpt_oss", Some(prefix.clone())) .is_none() ); let policy = UnifiedRequestPolicy { multiple_choices: true, ..Default::default() }; assert!( policy .harmony_prefill_cache("qwen3", Some(prefix.clone())) .is_none() ); assert!(policy.harmony_prefill_cache("gpt_oss", None).is_none()); let mut cache = policy .harmony_prefill_cache("gpt_oss", Some(prefix.clone())) .unwrap(); assert!(cache.token_ids.is_none()); let mut states = (0..2) .map(|_| { ChoiceState::new_with_prefill_cache( "gpt_oss", &[], StreamPrefill { starting_state: UnifiedParserStartingState::None, prefix: Some(prefix.clone()), }, UnifiedToolOutputMode::Native, true, ArgumentFraming::default(), Some(&mut cache), ) .unwrap() }) .collect::>(); assert!(cache.continuation.is_some()); assert!( cache.token_ids.is_none(), "checkpoint replaces replay tokens" ); for (state, tail) in states.iter_mut().zip(["first", "second"]) { let mut events = state.push(&format!("{tail}\"}}<|call|>")); events.extend(state.finish()); let calls: Vec<_> = events .into_iter() .filter_map(|e| { if let UnifiedParserEvent::ToolCall(c) = e { Some(c) } else { None } }) .collect(); assert_eq!( calls .iter() .map(|c| c.arguments.as_str()) .collect::(), format!("{tail}\"}}") ); assert!(calls.iter().all(|c| c.tool_index == 0)); assert!(calls.last().unwrap().complete); } let mut other_cache = HarmonyPrefill::new(prefix.clone()); let other_seed: std::sync::Arc = std::sync::Arc::from(prefix.as_ref()); ChoiceState::new_with_prefill_cache( "gpt_oss", &[], StreamPrefill { starting_state: UnifiedParserStartingState::None, prefix: Some(other_seed), }, UnifiedToolOutputMode::Native, true, ArgumentFraming::default(), Some(&mut other_cache), ) .unwrap(); assert!( other_cache.token_ids.is_none(), "a different seed allocation cannot use this request's cache" ); ChoiceState::new_with_prefill_cache( "gpt_oss", &[], StreamPrefill { starting_state: UnifiedParserStartingState::Reasoning, prefix: Some(prefix), }, UnifiedToolOutputMode::GuidedJson { named_tool: Some("lookup".into()), }, true, ArgumentFraming::default(), Some(&mut other_cache), ) .unwrap(); assert!( other_cache.token_ids.is_none(), "guided channel-only initialization does not encode the native seed" ); } #[test] fn gpt_oss_vendor_keeps_prompt_initialization() { if crate::test_utils::run_isolated( concat!( module_path!(), "::gpt_oss_vendor_keeps_prompt_initialization" ), &[], ) { return; } struct VendorHarmony { initialized: bool, token_input: bool, } impl UnifiedParser for VendorHarmony { fn input_capabilities(&self) -> dynamo_parsers_v2::UnifiedParserInputCapabilities { dynamo_parsers_v2::UnifiedParserInputCapabilities { text: true, token_ids: self.token_input, } } fn initialize_request(&mut self, init: UnifiedParserInit) -> anyhow::Result<()> { assert_eq!( dynamo_parsers_v2::unified::harmony::decode_harmony(&init.prompt_token_ids)?, "<|start|>assistant<|channel|>final<|message|>" ); assert_eq!(init.starting_state, UnifiedParserStartingState::None); self.initialized = true; Ok(()) } fn parse_into( &mut self, delta: &str, output: &mut UnifiedParserOutput, ) -> anyhow::Result<()> { assert!(self.initialized); output.push_text(delta); Ok(()) } fn parse_token_ids_into( &mut self, ids: &[u32], output: &mut UnifiedParserOutput, ) -> anyhow::Result<()> { anyhow::ensure!( self.token_input, "vendor does not support generated token ingestion" ); self.parse_into( &dynamo_parsers_v2::unified::harmony::decode_harmony(ids)?, output, ) } fn finish(&mut self) -> anyhow::Result { Ok(UnifiedParserOutput::default()) } } fn make_parser(token_input: bool) -> anyhow::Result> { Ok(Box::new(VendorHarmony { initialized: false, token_input, })) } fn text_factory(_: &[Tool]) -> anyhow::Result> { make_parser(false) } fn token_factory(_: &[Tool]) -> anyhow::Result> { make_parser(true) } let factories: [dynamo_parsers_v2::unified::UnifiedParserFactory; 2] = [text_factory, token_factory]; for factory in factories { let _ = dynamo_parsers_v2::register_unified_parser("gpt_oss", factory); for mode in [ UnifiedToolOutputMode::Native, UnifiedToolOutputMode::GuidedJson { named_tool: Some("lookup".into()), }, ] { let prefix: std::sync::Arc = std::sync::Arc::from("<|start|>assistant<|channel|>final<|message|>"); let mut cache = HarmonyPrefill::new(prefix.clone()); let mut state = ChoiceState::new_with_prefill_cache( "gpt_oss", &[], StreamPrefill { starting_state: UnifiedParserStartingState::Reasoning, prefix: Some(prefix), }, mode, true, ArgumentFraming::default(), Some(&mut cache), ) .unwrap(); assert_eq!( state.push("answer"), vec![UnifiedParserEvent::Text("answer".into())] ); assert!(state.finish().is_empty()); assert!(cache.token_ids.is_none(), "vendor owns initialization"); } assert!(dynamo_parsers_v2::unregister_unified_parser("gpt_oss").is_some()); } } // Adapter token replay and public tool indices are outside the parser-only corpus. #[test] fn gpt_oss_consumed_tool_body_keeps_later_call_indices_distinct() { let tools = to_v2_tools(Some(&weather_tools())); for (body, first_tail, first_arguments) in [ ("", "{\"city\":\"first\"}<|call|>", "{\"city\":\"first\"}"), ("{\"city\":\"fi", "rst\"}<|call|>", "rst\"}"), ("{\"city\":\"first\"}", "<|call|>", ""), ] { let prefix = format!( "<|start|>assistant to=functions.get_weather<|channel|>commentary<|message|>{body}" ); let input = format!( "{first_tail}<|start|>assistant to=functions.get_weather<|channel|>commentary<|message|>{{\"city\":\"second\"}}<|call|><|start|>assistant to=functions.get_weather<|channel|>commentary<|message|>{{\"city\":\"third\"}}<|call|>" ); for split in 0..=input.len() { let mut state = ChoiceState::new( "gpt_oss", &tools, StreamPrefill { starting_state: UnifiedParserStartingState::None, prefix: Some(std::sync::Arc::from(prefix.as_str())), }, UnifiedToolOutputMode::Native, true, ArgumentFraming::default(), ) .unwrap(); let events = state.push(&input[..split]); let mut choices = state.choices_for(&empty_choice(0), events, true, None); let mut events = state.push(&input[split..]); events.extend(state.finish()); choices.extend(state.choices_for( &empty_choice(0), events, true, Some(FinishReason::Stop), )); let mut calls = std::collections::BTreeMap::::new(); for call in choices .iter() .flat_map(|choice| choice.delta.tool_calls.iter().flatten()) { let entry = calls.entry(call.index).or_default(); entry.2 += usize::from(call.id.is_some()); if let Some(function) = &call.function { if let Some(name) = &function.name { entry.0.push_str(name); } if let Some(arguments) = &function.arguments { entry.1.push_str(arguments); } } } assert_eq!( calls.keys().copied().collect::>(), [0, 1, 2], "body={body} split={split}" ); assert_eq!(calls[&0].1, first_arguments); for (index, city) in [(1, "second"), (2, "third")] { assert_eq!(calls[&index].0, "get_weather"); assert_eq!( serde_json::from_str::(&calls[&index].1).unwrap(), serde_json::json!({"city":city}) ); } assert!(calls.values().all(|call| call.2 == 1)); assert_eq!( choices.last().unwrap().finish_reason, Some(FinishReason::ToolCalls) ); } } } #[test] fn gpt_oss_argument_progress_preserves_long_field_positions() { let tools = to_v2_tools(Some(&write_file_tools())); let value = format!( "é {} STAGE_TWO {} END", "first content ".repeat(64), "second content ".repeat(64) ); for position in 0..3 { let mut fields = vec![ ("path", serde_json::json!("a.rs")), ("count", serde_json::json!(2)), ]; fields.insert(position, ("content", serde_json::json!(value))); let arguments = ordered_arguments(&fields); let expected: serde_json::Value = serde_json::from_str(&arguments).unwrap(); for mode in [ UnifiedToolOutputMode::Native, UnifiedToolOutputMode::GuidedJson { named_tool: None }, UnifiedToolOutputMode::GuidedJson { named_tool: Some("write_file".into()), }, ] { let input = match &mode { UnifiedToolOutputMode::Native => format!( " to=functions.write_file<|channel|>commentary<|message|>{arguments}<|call|>" ), UnifiedToolOutputMode::GuidedJson { named_tool: None } => { format!("[{{\"name\":\"write_file\",\"parameters\":{arguments}}}]") } _ => arguments.clone(), }; let mut state = ChoiceState::new( "gpt_oss", &tools, UnifiedParserStartingState::None, mode.clone(), true, ArgumentFraming::default(), ) .unwrap(); let mut wire = Vec::new(); let mut start = 0; for (marker, fragment) in [(" STAGE_TWO ", "first content"), (" END", "second content")] { let end = input.find(marker).unwrap(); let events = state.push(&input[start..end]); wire.extend(state.choices_for(&empty_choice(0), events, true, None)); let arguments: String = wire .iter() .flat_map(|choice| choice.delta.tool_calls.iter().flatten()) .filter_map(|call| call.function.as_ref()?.arguments.as_deref()) .collect(); assert!( arguments.contains(fragment), "position={position} mode={mode:?} marker={marker}: {arguments:?}" ); start = end; } let mut events = state.push(&input[start..]); events.extend(state.finish()); wire.extend(state.choices_for( &empty_choice(0), events, true, Some(FinishReason::Stop), )); let calls: Vec<_> = wire .iter() .flat_map(|choice| choice.delta.tool_calls.iter().flatten()) .collect(); assert!(calls.iter().all(|call| call.index == 0)); assert_eq!(calls.iter().filter(|call| call.id.is_some()).count(), 1); assert_eq!( calls .iter() .filter_map(|call| call.function.as_ref()?.name.as_deref()) .collect::(), "write_file" ); let arguments: String = calls .iter() .filter_map(|call| call.function.as_ref()?.arguments.as_deref()) .collect(); assert_eq!( serde_json::from_str::(&arguments).unwrap(), expected ); assert_eq!( wire.last().unwrap().finish_reason, Some(FinishReason::ToolCalls) ); } } } #[test] fn maps_tool_choice_onto_the_output_mode() { assert_eq!( tool_output_mode(&GuidedToolConstraint::GuidedJsonNamed { tool_name: "get_weather".to_string(), }), UnifiedToolOutputMode::GuidedJson { named_tool: Some("get_weather".to_string()) } ); assert_eq!( tool_output_mode(&GuidedToolConstraint::GuidedJsonRequired), UnifiedToolOutputMode::GuidedJson { named_tool: None } ); for native in [ GuidedToolConstraint::None, GuidedToolConstraint::StructuralTag, GuidedToolConstraint::BeamTurnGrammar { frames_answer: false, }, GuidedToolConstraint::BeamTurnGrammar { frames_answer: true, }, ] { assert_eq!( tool_output_mode(&native), UnifiedToolOutputMode::Native, "{native:?} must stay on native markup" ); } } #[test] fn the_beam_turn_grammar_frames_arguments_and_owns_a_structured_answer() { // Only the grammar that typed the arguments frames them. for (constraint, framing) in [ (GuidedToolConstraint::None, ArgumentFraming::Lenient), ( GuidedToolConstraint::StructuralTag, ArgumentFraming::Lenient, ), ( GuidedToolConstraint::GuidedJsonRequired, ArgumentFraming::Lenient, ), ( GuidedToolConstraint::BeamTurnGrammar { frames_answer: false, }, ArgumentFraming::SchemaConstrained, ), ] { assert_eq!(argument_framing(&constraint), framing, "{constraint:?}"); } // A `response_format` the grammar holds in the final channel is native output. let structured = UnifiedRequestPolicy { reasoning_disabled: false, structured_response: true, ..Default::default() }; assert!( structured .under(&GuidedToolConstraint::BeamTurnGrammar { frames_answer: false, }) .structured_response ); assert!( !structured .under(&GuidedToolConstraint::BeamTurnGrammar { frames_answer: true, }) .structured_response ); assert!( structured .under(&GuidedToolConstraint::None) .structured_response ); } #[test] fn batch_native_marker_overrides_a_reconstructed_guided_constraint() { for constraint in [ GuidedToolConstraint::GuidedJsonRequired, GuidedToolConstraint::GuidedJsonNamed { tool_name: "get_weather".to_string(), }, ] { assert_eq!( batch_tool_output_mode( QWEN3_UNIFIED_FAMILY, "\n\n\n", &constraint, ), UnifiedToolOutputMode::Native, ); } assert_eq!( batch_tool_output_mode( QWEN3_UNIFIED_FAMILY, r#"[{"name":"get_weather","parameters":{"literal":""}}]"#, &GuidedToolConstraint::GuidedJsonRequired, ), UnifiedToolOutputMode::GuidedJson { named_tool: None }, "a marker inside JSON arguments remains payload data", ); } #[tokio::test] async fn every_valid_utf8_split_matches_whole_input_across_output_modes() { let cases = [ ( concat!( "理由は東京です。", "\n\n", "東京\n\n" ), Some(ChatCompletionToolChoiceOption::Auto), ), ( r#"{"city":"東京"}"#, Some(named_choice("get_weather")), ), ( r#"[{"name":"get_weather","parameters":{"city":"東京"}}]"#, Some(ChatCompletionToolChoiceOption::Required), ), ]; for (input, tool_choice) in cases { let expected = parse_at_split(input, 0, tool_choice.clone()).await; for split in (0..=input.len()).filter(|split| input.is_char_boundary(*split)) { assert_eq!( parse_at_split(input, split, tool_choice.clone()).await, expected, "output changed at UTF-8 split {split} for {input:?}" ); } } } #[tokio::test] async fn structural_tag_keeps_required_on_native_markup() { // A structural tag constrains generation to Qwen's XML, so `required` must NOT // put the parser into guided-JSON mode; doing so would surface the whole call // as text. let output = concat!( "reason", "\n\n", "Tokyo\n\n" ); let responses = apply_stream( stream::iter([chunk(output, true)]), Some(weather_tools()), Some(ChatCompletionToolChoiceOption::Required), true, UnifiedParserStartingState::None, QWEN3_UNIFIED_FAMILY, ) .collect::>() .await; let choices = collect_choices(&responses); let call = choices .iter() .filter_map(|choice| choice.delta.tool_calls.as_ref()) .flatten() .next() .expect("a native tool call"); let function = call.function.as_ref().expect("function"); assert_eq!(function.name.as_deref(), Some("get_weather")); assert_eq!(function.arguments.as_deref(), Some(r#"{"city":"Tokyo"}"#)); } // --- prefill ----------------------------------------------------------------- #[test] fn prompt_injected_reasoning_selects_the_reasoning_prefill() { assert_eq!( stream_prefill(QWEN3_UNIFIED_FAMILY, true), UnifiedParserStartingState::Reasoning ); // Qwen3's template pre-opens no channel, so the model emits `` itself. assert_eq!( stream_prefill(QWEN3_UNIFIED_FAMILY, false), UnifiedParserStartingState::None ); } #[test] fn detects_batch_prefill_from_complete_output() { assert_eq!( detect_prefill(QWEN3_UNIFIED_FAMILY, "reasonanswer").unwrap(), UnifiedParserStartingState::None ); assert_eq!( detect_prefill(QWEN3_UNIFIED_FAMILY, "reasonanswer").unwrap(), UnifiedParserStartingState::Reasoning ); assert_eq!( detect_prefill(QWEN3_UNIFIED_FAMILY, "answer").unwrap(), UnifiedParserStartingState::Response ); assert_eq!( detect_prefill(QWEN3_UNIFIED_FAMILY, "It's hiddenvisible").unwrap(), UnifiedParserStartingState::Reasoning, "an apostrophe in prose must not hide a later control marker" ); assert!(detect_prefill("unknown", "answer").is_err()); } #[test] fn detect_prefill_compares_first_marker_positions_not_mere_presence() { // A prompt-opened reasoning span that the model closes, followed by the model // opening (and closing) a SECOND thought later in the same output. The leading // `` has no opener before it, so the prompt pre-opened reasoning: this // must return `Reasoning`, not `None`. Presence-only testing ("does // exist anywhere") wrongly answers `None` because a `` exists somewhere // in the string, even though it comes AFTER the first ``. assert_eq!( detect_prefill( QWEN3_UNIFIED_FAMILY, "secretanswersecond thoughttail" ) .unwrap(), UnifiedParserStartingState::Reasoning, "a leading closer with no prior opener means the prompt pre-opened reasoning, \ regardless of a later model-opened thought" ); } #[test] fn minimax_m3_prefill_does_not_treat_argument_markers_as_prompt_state() { let (_, native) = all_family_native_calls() .into_iter() .find(|(family, _)| *family == "minimax_m3") .unwrap(); for body in [native.to_string(), native.replace("Tokyo", "")] { let input = format!("{body}Done."); let parsed = parse_complete( "minimax_m3", &input, &GuidedToolConstraint::None, &weather_tools(), ) .unwrap(); assert!(parsed.reasoning.is_empty(), "{:?}", parsed.reasoning); assert_eq!(parsed.text, "Done."); assert_eq!(parsed.tool_calls.len(), 1); } let parsed = parse_complete( "minimax_m3", &format!("{native}Check.Done."), &GuidedToolConstraint::None, &weather_tools(), ) .unwrap(); assert_eq!(parsed.reasoning, "Check."); assert_eq!(parsed.text, "Done."); assert_eq!( detect_prefill("minimax_m3", &format!("Check.{native}")).unwrap(), UnifiedParserStartingState::Reasoning ); } #[tokio::test] async fn minimax_m3_prompt_seed_preserves_reasoning_at_every_split() { for (input, expected_text, expected_reasoning) in [ ("Zürich 🙂", "", "Zürich 🙂"), ("Plan.Zürich 🙂", "Zürich 🙂", "Plan."), ("[plan]Paris", "Paris", "[plan]"), ("[plan]", "", "[plan]"), (r#"{"city":"Paris"}"#, "", r#"{"city":"Paris"}"#), ("unfinished >() .await; assert!(output.iter().all(|item| !item.is_error())); let choices = collect_choices(&output); let text: String = choices .iter() .filter_map(|choice| match &choice.delta.content { Some(ChatCompletionMessageContent::Text(text)) => Some(text.as_str()), _ => None, }) .collect(); let reasoning: String = choices .iter() .filter_map(|choice| choice.delta.reasoning_content.as_deref()) .collect(); assert_eq!( (text.as_str(), reasoning.as_str()), (expected_text, expected_reasoning), "input={input:?}, split={split}, finish={finish:?}" ); let finishes: Vec<_> = choices .iter() .filter_map(|choice| choice.finish_reason) .collect(); assert_eq!(finishes, finish.into_iter().collect::>()); } } } } #[test] fn apostrophe_exemption_needs_only_one_alphanumeric_side() { // A possessive apostrophe that ends a word (`James'`) has non-alphanumeric on // the RIGHT. Requiring alphanumeric on both sides treats it as a real // quote-open, and the later contraction apostrophe in `isn't` then closes that // phantom span, hiding the real `` marker in between. assert_eq!( detect_prefill( QWEN3_UNIFIED_FAMILY, "James' secret answer isn't final" ) .unwrap(), UnifiedParserStartingState::Reasoning, "a trailing possessive apostrophe must not be treated as a real quote-open" ); // Mirror case: a leading contraction (`'twas`) has non-alphanumeric on the LEFT // and alphanumeric only on the right — the OR-based fix must exempt this too. assert!( contains_unquoted_marker("'twas hidden", ""), "a leading-apostrophe contraction must not be treated as a real quote-open" ); } // --- delta -> chunk conversion ----------------------------------------------- fn call_delta(tool_index: usize, name: Option<&str>, arguments: &str) -> UnifiedParserEvent { UnifiedParserEvent::ToolCall(ToolCallDelta { tool_index, name: name.map(str::to_string), arguments: arguments.to_string(), complete: true, }) } fn partial_call_delta( tool_index: usize, name: Option<&str>, arguments: &str, ) -> UnifiedParserEvent { UnifiedParserEvent::ToolCall(ToolCallDelta { tool_index, name: name.map(str::to_string), arguments: arguments.to_string(), complete: false, }) } fn test_state() -> ChoiceState { ChoiceState::new( QWEN3_UNIFIED_FAMILY, &[], UnifiedParserStartingState::None, UnifiedToolOutputMode::Native, true, ArgumentFraming::default(), ) .expect("qwen3 unified parser") } #[test] fn native_grammar_state_matches_family_and_output_mode() { for family in [ QWEN3_UNIFIED_FAMILY, NEMOTRON_V3_UNIFIED_FAMILY, GLM47_UNIFIED_FAMILY, KIMI_K2_UNIFIED_FAMILY, "gpt_oss", DEEPSEEK_V4_UNIFIED_FAMILY, MUSE_SPARK_UNIFIED_FAMILY, ] { for mode in [ UnifiedToolOutputMode::Native, UnifiedToolOutputMode::GuidedJson { named_tool: None }, UnifiedToolOutputMode::GuidedJson { named_tool: Some("get_weather".to_string()), }, ] { let state = ChoiceState::new( family, &[], UnifiedParserStartingState::None, mode.clone(), true, ArgumentFraming::default(), ) .unwrap(); match (&state.native_grammar_state, &mode, family) { ( NativeGrammarState::QwenXml(qwen), UnifiedToolOutputMode::Native, QWEN3_UNIFIED_FAMILY | NEMOTRON_V3_UNIFIED_FAMILY, ) => { assert!(qwen.lexical_state.mode == QwenNativeLexicalMode::OutsideBlock); assert!(qwen.pending_opener.is_empty()); assert!(qwen.deferred_reasoning.is_empty()); assert!(qwen.deferred_reasoning_event_indices.is_empty()); assert!(qwen.deferred_reasoning_choice_indices.is_empty()); } ( NativeGrammarState::Glm(glm), UnifiedToolOutputMode::Native, GLM47_UNIFIED_FAMILY, ) => assert!(glm.input_for_truncation_check.is_empty()), (NativeGrammarState::None, UnifiedToolOutputMode::GuidedJson { .. }, _) | ( NativeGrammarState::None, UnifiedToolOutputMode::Native, KIMI_K2_UNIFIED_FAMILY | "gpt_oss" | DEEPSEEK_V4_UNIFIED_FAMILY | MUSE_SPARK_UNIFIED_FAMILY, ) => {} _ => panic!("wrong adapter grammar state for {family} / {mode:?}"), } assert!(!state.is_deferred_qwen_xml_reasoning_choice(0)); } } } #[test] fn each_delta_becomes_its_own_chunk_in_order() { // The whole point of this path: a thought that followed a call stays after it. let mut state = test_state(); let base = empty_choice(0); let choices = state.choices_for( &base, vec![ UnifiedParserEvent::Reasoning("look it up".into()), call_delta(0, Some("get_weather"), r#"{"city":"Tokyo"}"#), UnifiedParserEvent::Reasoning("now answer".into()), UnifiedParserEvent::Text("It's 18C.".into()), ], true, Some(FinishReason::Stop), ); assert_eq!(choices.len(), 4, "one chunk per delta: {choices:?}"); assert_eq!( choices[0].delta.reasoning_content.as_deref(), Some("look it up") ); assert!(choices[1].delta.tool_calls.is_some()); assert_eq!( choices[2].delta.reasoning_content.as_deref(), Some("now answer"), "the second thought must stay AFTER the call, not fuse with the first" ); assert_eq!( choices[3].delta.content, Some(ChatCompletionMessageContent::Text("It's 18C.".into())) ); // Only the last chunk terminates, and Stop became ToolCalls. assert_eq!(choices[3].finish_reason, Some(FinishReason::ToolCalls)); assert!(choices[..3].iter().all(|c| c.finish_reason.is_none())); } #[test] fn mixed_raw_content_and_upstream_tool_call_do_not_reuse_tool_index() { let mut state = test_state(); let mut original = empty_choice(0); original.delta.content = Some(ChatCompletionMessageContent::Text( "raw parser input".to_string(), )); original.delta.tool_calls = Some(vec![ChatCompletionMessageToolCallChunk { custom: None, index: 0, id: Some("upstream-call".to_string()), r#type: Some(FunctionType::Function), function: Some(FunctionCallStream { name: Some("upstream".to_string()), arguments: Some("{}".to_string()), }), }]); let choices = state.choices_for( &original, vec![call_delta(0, Some("parsed"), "{}")], true, None, ); let mut indices: Vec<_> = choices .iter() .filter_map(|choice| choice.delta.tool_calls.as_ref()) .flatten() .map(|call| call.index) .collect(); indices.sort_unstable(); assert_eq!(indices, vec![0, 1]); let mut upstream_continuation = empty_choice(0); upstream_continuation.delta.tool_calls = Some(vec![ChatCompletionMessageToolCallChunk { custom: None, index: 0, id: None, r#type: None, function: Some(FunctionCallStream { name: None, arguments: Some("more".to_string()), }), }]); let choices = state.choices_for(&upstream_continuation, Vec::new(), true, None); assert_eq!( choices[0].delta.tool_calls.as_ref().unwrap()[0].index, 0, "the pass-through call keeps its remapped index across chunks" ); let choices = state.choices_for( &empty_choice(0), vec![call_delta(0, None, "more")], true, None, ); assert_eq!( choices[0].delta.tool_calls.as_ref().unwrap()[0].index, 1, "the parser call keeps its index while the pass-through call continues" ); } #[test] fn first_tool_chunk_opens_the_call_and_later_ones_only_add_arguments() { let mut state = test_state(); let base = empty_choice(7); let choices = state.choices_for( &base, vec![ call_delta(0, Some("get_weather"), r#"{"city":"#), call_delta(0, None, r#""Tokyo"}"#), ], true, None, ); assert_eq!(choices.len(), 2); let first = &choices[0].delta.tool_calls.as_ref().unwrap()[0]; assert_eq!(choices[0].index, 7, "the choice index is preserved"); assert_eq!(first.index, 0, "the tool index is preserved"); assert!(first.id.is_some(), "the opening chunk mints an id"); assert_eq!(first.r#type, Some(FunctionType::Function)); assert_eq!( first.function.as_ref().unwrap().name.as_deref(), Some("get_weather") ); let second = &choices[1].delta.tool_calls.as_ref().unwrap()[0]; assert!(second.id.is_none(), "only the first chunk carries an id"); assert!(second.r#type.is_none()); assert!(second.function.as_ref().unwrap().name.is_none()); assert_eq!( second.function.as_ref().unwrap().arguments.as_deref(), Some(r#""Tokyo"}"#) ); } #[test] fn two_calls_get_distinct_ids_and_keep_their_indices() { let mut state = test_state(); let base = empty_choice(0); let choices = state.choices_for( &base, vec![ call_delta(0, Some("a"), "{}"), call_delta(1, Some("b"), "{}"), ], true, None, ); let ids: Vec<_> = choices .iter() .map(|choice| { let call = &choice.delta.tool_calls.as_ref().unwrap()[0]; (call.index, call.id.clone().expect("id")) }) .collect(); assert_eq!(ids[0].0, 0); assert_eq!(ids[1].0, 1); assert_ne!(ids[0].1, ids[1].1, "each call gets its own id"); } #[tokio::test] async fn every_family_api_tool_ids_follow_protocol_across_turns() { use dynamo_parsers_v2::{REGISTERED_UNIFIED_FAMILIES, canonical_unified_family}; use std::collections::{BTreeMap, BTreeSet}; let inputs: BTreeMap = serde_json::from_str(include_str!(concat!( env!("CARGO_MANIFEST_DIR"), "/tests/data/unified_tool_call_ids.json" ))) .unwrap(); let registered: BTreeSet<_> = REGISTERED_UNIFIED_FAMILIES .iter() .map(|family| canonical_unified_family(family).unwrap()) .collect(); assert_eq!( inputs.keys().map(String::as_str).collect::>(), registered ); for (family, input) in inputs { let family = canonical_unified_family(&family).unwrap(); let mut conversation_ids = BTreeSet::new(); for character_chunks in [false, true] { for turn in 0..2 { let pieces = if character_chunks { input.chars().map(|ch| ch.to_string()).collect::>() } else { vec![input.clone()] }; let chunks = pieces .iter() .map(|text| chunk(text, false)) .chain([chunk("", true)]) .collect::>(); let responses = apply_stream_with_constraint( stream::iter(chunks), Some(weather_tools()), GuidedToolConstraint::None, UnifiedParserStartingState::None, family, true, ) .collect::>() .await; assert!( responses.iter().all(|response| !response.is_error()), "{family}" ); let mut calls = BTreeMap::::new(); for choice in collect_choices(&responses) { let Some(deltas) = &choice.delta.tool_calls else { continue; }; assert_eq!(deltas.len(), 1, "each API event carries one call: {family}"); for delta in deltas { let function = delta.function.as_ref().unwrap(); if let Some((_, _, arguments)) = calls.get_mut(&delta.index) { assert!( delta.id.is_none(), "ID repeated after first chunk: {family}" ); assert!(function.name.is_none()); arguments .push_str(function.arguments.as_deref().unwrap_or_default()); } else { calls.insert( delta.index, ( delta.id.clone().expect("first API chunk must carry an ID"), function .name .clone() .expect("first API chunk must carry a name"), function.arguments.clone().unwrap_or_default(), ), ); } } } assert_eq!( calls.keys().copied().collect::>(), [0, 1], "{family}" ); for (index, (id, name, arguments)) in calls { assert_eq!(name, "get_weather"); assert_eq!( serde_json::from_str::(&arguments).unwrap(), serde_json::json!({"city":"Paris"}) ); if family == "kimi_k2" { assert_eq!(id, format!("functions.get_weather:{index}")); } else { assert!( conversation_ids.insert(id), "API ID reused: {family}, turn={turn}, char_chunks={character_chunks}" ); } } let batch = parse_complete( family, &input, &GuidedToolConstraint::None, &weather_tools(), ) .unwrap(); assert_eq!(batch.tool_calls.len(), 2, "{family}"); for (index, call) in batch.tool_calls.into_iter().enumerate() { assert_eq!(call.function.name, "get_weather"); if family == "kimi_k2" { assert_eq!(call.id, format!("functions.get_weather:{index}")); } else { assert!( conversation_ids.insert(call.id), "batch reused an API ID: {family}" ); } } } } } } #[test] fn optional_message_phases_preserve_muse_boundaries() { let mut plain = test_state(); let mut phased = test_state(); let original = empty_choice(0); let expected = plain.choices_for( &original, vec![UnifiedParserEvent::Text("answer".into())], true, Some(FinishReason::Stop), ); let deltas = vec![ UnifiedParserEvent::MessageStart(dynamo_parsers_v2::unified::MessagePhase::FinalAnswer), UnifiedParserEvent::Text("answer".into()), UnifiedParserEvent::MessageEnd, ]; let actual = phased.choices_for(&original, deltas.clone(), true, Some(FinishReason::Stop)); assert_eq!(actual, expected); assert_eq!(phased.take_message_boundaries(), vec![None]); phased.family = MUSE_SPARK_UNIFIED_FAMILY.to_owned(); let actual = phased.choices_for(&original, deltas, true, Some(FinishReason::Stop)); assert_eq!(actual.len(), 3); assert_eq!(actual[1].delta.content, expected[0].delta.content); assert_eq!( phased.take_message_boundaries(), vec![ Some(super::super::MessageBoundary::Start( dynamo_protocols::types::responses::MessagePhase::FinalAnswer, )), None, Some(super::super::MessageBoundary::End), ], ); } #[test] fn adjacent_same_kind_deltas_coalesce_but_calls_never_do() { let merged = coalesce(vec![ UnifiedParserEvent::Text("he".into()), UnifiedParserEvent::Text("llo".into()), call_delta(0, Some("f"), "{"), call_delta(0, None, "}"), UnifiedParserEvent::Reasoning("a".into()), UnifiedParserEvent::Reasoning("b".into()), ]); assert_eq!(merged.len(), 4); assert_eq!(merged[0], UnifiedParserEvent::Text("hello".into())); assert_eq!(merged[1], call_delta(0, Some("f"), "{")); assert_eq!(merged[2], call_delta(0, None, "}")); assert_eq!(merged[3], UnifiedParserEvent::Reasoning("ab".into())); } #[test] fn role_rides_the_first_chunk_and_finish_reason_the_last() { let mut state = test_state(); let mut base = empty_choice(0); base.delta.role = Some(Role::Assistant); let choices = state.choices_for( &base, vec![ UnifiedParserEvent::Text("a".into()), call_delta(0, Some("f"), "{}"), ], true, Some(FinishReason::Stop), ); assert_eq!(choices[0].delta.role, Some(Role::Assistant)); assert!(choices[1].delta.role.is_none()); assert!(choices[0].finish_reason.is_none()); assert_eq!(choices[1].finish_reason, Some(FinishReason::ToolCalls)); } #[test] fn incomplete_tool_call_fragments_do_not_mark_the_choice_terminal() { let mut state = test_state(); let partial = state.choices_for( &empty_choice(0), vec![partial_call_delta( 0, Some("get_weather"), r#"{"city":"Tok"#, )], true, Some(FinishReason::Stop), ); assert_eq!(partial.len(), 1); assert!(partial[0].delta.tool_calls.is_some()); assert_eq!( partial[0].finish_reason, Some(FinishReason::Length), "an unfinished parser fragment must expose truncation" ); assert!(state.tool_emitted()); assert_eq!( state.unterminated_finish_reason(), Some(FinishReason::Length) ); let completed = state.choices_for( &empty_choice(0), vec![call_delta(0, None, "yo\"}")], true, Some(FinishReason::Stop), ); assert_eq!(completed[0].finish_reason, Some(FinishReason::ToolCalls)); assert!(state.tool_emitted()); assert_eq!( state.unterminated_finish_reason(), Some(FinishReason::ToolCalls) ); } #[test] fn completed_call_does_not_hide_another_incomplete_call() { let mut state = test_state(); let partial = state.choices_for( &empty_choice(0), vec![partial_call_delta(0, Some("first"), r#"{"city":"Tok"#)], true, None, ); assert_eq!(partial.len(), 1); let completed = state.choices_for( &empty_choice(0), vec![call_delta(1, Some("second"), r#"{"city":"Paris"}"#)], true, Some(FinishReason::Stop), ); assert_eq!(completed[0].finish_reason, Some(FinishReason::Length)); assert_eq!( state.unterminated_finish_reason(), Some(FinishReason::Length) ); let final_fragment = state.choices_for( &empty_choice(0), vec![call_delta(0, None, "yo\"}")], true, Some(FinishReason::Stop), ); assert_eq!( final_fragment[0].finish_reason, Some(FinishReason::ToolCalls) ); assert_eq!( state.unterminated_finish_reason(), Some(FinishReason::ToolCalls) ); } #[test] fn separate_terminal_finish_closes_all_parsed_call_fragments() { let mut state = test_state(); let mut parsed_fragments = empty_choice(0); parsed_fragments.delta.tool_calls = Some(vec![ ChatCompletionMessageToolCallChunk { custom: None, index: 0, id: Some("call-first".to_string()), r#type: Some(FunctionType::Function), function: Some(FunctionCallStream { name: Some("first".to_string()), arguments: Some(r#"{"city":"Paris"}"#.to_string()), }), }, ChatCompletionMessageToolCallChunk { custom: None, index: 1, id: Some("call-second".to_string()), r#type: Some(FunctionType::Function), function: Some(FunctionCallStream { name: Some("second".to_string()), arguments: Some(r#"{"city":"Tokyo"}"#.to_string()), }), }, ]); let partials = state.choices_for(&parsed_fragments, Vec::new(), true, None); assert_eq!(partials.len(), 1); assert_eq!( state.unterminated_finish_reason(), Some(FinishReason::Length) ); let terminal = state.choices_for(&empty_choice(0), Vec::new(), true, Some(FinishReason::Stop)); assert_eq!(terminal.len(), 1); assert_eq!(terminal[0].finish_reason, Some(FinishReason::ToolCalls)); assert_eq!( state.unterminated_finish_reason(), Some(FinishReason::ToolCalls) ); } #[test] fn an_empty_run_still_carries_the_terminating_envelope() { let mut state = test_state(); let mut base = empty_choice(0); base.delta.role = Some(Role::Assistant); let choices = state.choices_for(&base, Vec::new(), true, Some(FinishReason::Length)); assert_eq!(choices.len(), 1); assert_eq!(choices[0].delta.role, Some(Role::Assistant)); assert_eq!( choices[0].finish_reason, Some(FinishReason::Length), "Length is not rewritten" ); } #[test] fn an_empty_run_with_no_envelope_emits_nothing() { let mut state = test_state(); let base = empty_choice(0); assert!(state.choices_for(&base, Vec::new(), true, None).is_empty()); } // --- end-to-end streaming ---------------------------------------------------- #[tokio::test] async fn streams_ordered_reasoning_text_and_tool_calls() { let output = concat!( "reasonanswer ", "\n\n", "Tokyo\n\n" ); // Split into small chunks so markers straddle push() boundaries. let mut chunks: Vec<_> = output .as_bytes() .chunks(7) .map(|bytes| chunk(std::str::from_utf8(bytes).unwrap(), false)) .collect(); chunks.push(chunk("", true)); let responses = apply_stream( stream::iter(chunks), Some(weather_tools()), None, false, UnifiedParserStartingState::None, QWEN3_UNIFIED_FAMILY, ) .collect::>() .await; let choices = collect_choices(&responses); let reasoning: String = choices .iter() .filter_map(|choice| choice.delta.reasoning_content.as_deref()) .collect(); let content: String = choices .iter() .filter_map(|choice| match &choice.delta.content { Some(ChatCompletionMessageContent::Text(text)) => Some(text.as_str()), _ => None, }) .collect(); let arguments: String = choices .iter() .filter_map(|choice| choice.delta.tool_calls.as_ref()) .flatten() .filter_map(|call| call.function.as_ref()?.arguments.as_deref()) .collect(); assert_eq!(reasoning, "reason"); assert_eq!(content, "answer "); assert_eq!( serde_json::from_str::(&arguments).unwrap()["city"], "Tokyo" ); for marker in ["", "", "reasonanswer", false); source.data.as_mut().unwrap().llm_metrics = Some(crate::protocols::common::metrics::LLMMetricAnnotation { chunk_tokens: 4, ..Default::default() }); let responses = apply_stream( stream::iter([source]), None, None, false, UnifiedParserStartingState::None, QWEN3_UNIFIED_FAMILY, ) .collect::>() .await; let metric_choices: Vec<_> = responses .iter() .filter_map(|response| response.data.as_ref()) .filter(|data| data.llm_metrics.is_some()) .flat_map(|data| data.inner.choices.iter()) .collect(); assert_eq!( metric_choices.len(), 1, "source metrics must be emitted once" ); assert_eq!( metric_choices[0].delta.reasoning_content.as_deref(), Some("reason"), "the metrics owner must retain the source chunk's reasoning classification" ); } #[tokio::test] async fn qwen_delayed_reasoning_keeps_source_envelope_and_continuation_metadata() { for (opening, continuation_prefix) in [ ("before "), ("before <", "raw"), ] { for reasoning_text in ["", "held thought"] { let mut reasoning = chunk("", false); let choice = &mut reasoning.data.as_mut().unwrap().inner.choices[0]; choice.delta.content = None; choice.delta.reasoning_content = Some(reasoning_text.to_string()); let source = reasoning.data.as_mut().unwrap(); source.nvext = Some(serde_json::json!({"source": "reasoning"})); source.llm_metrics = Some(crate::protocols::common::metrics::LLMMetricAnnotation { chunk_tokens: 7, ..Default::default() }); reasoning.id = Some("reasoning-source-id".to_string()); reasoning.event = Some("reasoning-source-event".to_string()); reasoning.comment = Some(vec!["reasoning-source-comment".to_string()]); let continuation_text = format!( "{continuation_prefix}\n\nParis\n\nanswer" ); let mut continuation = chunk(&continuation_text, true); let continuation_data = continuation.data.as_mut().unwrap(); continuation_data.nvext = Some(serde_json::json!({"source": "continuation"})); continuation.id = Some("continuation-id".to_string()); continuation.event = Some("continuation-event".to_string()); continuation.comment = Some(vec!["continuation-comment".to_string()]); let responses = apply_stream( stream::iter([chunk(opening, false), reasoning, continuation]), Some(weather_tools()), None, false, UnifiedParserStartingState::None, QWEN3_UNIFIED_FAMILY, ) .collect::>() .await; let delayed_reasoning = responses .iter() .find(|response| { response.data.as_ref().is_some_and(|data| { data.inner.choices.iter().any(|choice| { choice.delta.reasoning_content.as_deref() == Some(reasoning_text) }) }) }) .expect("queued reasoning should be released when the opener resolves"); let delayed_data = delayed_reasoning.data.as_ref().unwrap(); assert_eq!(delayed_reasoning.id.as_deref(), Some("reasoning-source-id")); assert_eq!( delayed_reasoning.event.as_deref(), Some("reasoning-source-event") ); assert_eq!( delayed_reasoning.comment.as_deref(), Some(["reasoning-source-comment".to_string()].as_slice()) ); assert_eq!( delayed_data.nvext, Some(serde_json::json!({"source": "reasoning"})) ); assert_eq!(delayed_data.llm_metrics.as_ref().unwrap().chunk_tokens, 7); assert_eq!(delayed_data.inner.choices.len(), 1); assert!(delayed_data.inner.choices[0].delta.tool_calls.is_none()); let continuation_answer = responses .iter() .find(|response| { response.data.as_ref().is_some_and(|data| { data.inner.choices.iter().any(|choice| { matches!(&choice.delta.content, Some(ChatCompletionMessageContent::Text(text)) if text == "answer") }) }) }) .expect("continuation answer should keep the resolving chunk envelope"); let continuation_data = continuation_answer.data.as_ref().unwrap(); assert_eq!(continuation_answer.id.as_deref(), Some("continuation-id")); assert_eq!( continuation_answer.event.as_deref(), Some("continuation-event") ); assert_eq!( continuation_answer.comment.as_deref(), Some(["continuation-comment".to_string()].as_slice()) ); assert_eq!( continuation_data.nvext, Some(serde_json::json!({"source": "continuation"})) ); let content_chunks: Vec<_> = responses .iter() .flat_map(|response| response.data.iter()) .flat_map(|data| data.inner.choices.iter()) .filter_map(|choice| match &choice.delta.content { Some(ChatCompletionMessageContent::Text(text)) => Some(text.as_str()), _ => None, }) .collect(); assert!( content_chunks.iter().all(|text| !text.contains("rawanswer", finish); let data = continuation.data.as_mut().unwrap(); data.inner.choices[0].delta.reasoning_content = Some("mixed".to_string()); data.nvext = Some(serde_json::json!({"source": "continuation"})); continuation.id = Some("continuation".to_string()); inputs.push(continuation); let responses = apply_stream( stream::iter(inputs), Some(weather_tools()), None, false, UnifiedParserStartingState::None, QWEN3_UNIFIED_FAMILY, ) .collect::>() .await; for (source_id, tokens) in [("first-source", 3), ("second-source", 5)] { let sources: Vec<_> = responses .iter() .filter(|response| response.id.as_deref() == Some(source_id)) .collect(); assert_eq!(sources.len(), 1); let data = sources[0].data.as_ref().unwrap(); assert_eq!( data.inner.choices[0].delta.reasoning_content.as_deref(), Some("same") ); assert!(data.inner.choices[0].delta.content.is_none()); assert_eq!(data.llm_metrics.as_ref().unwrap().chunk_tokens, tokens); assert_eq!(data.nvext, Some(serde_json::json!({"source": source_id}))); } let reasoning: Vec<_> = collect_choices(&responses) .into_iter() .filter_map(|choice| choice.delta.reasoning_content.clone()) .collect(); assert_eq!(reasoning, ["mixed", "same", "same", "raw"]); assert_eq!( responses .iter() .filter(|response| response.id.as_deref() == Some("continuation")) .count(), 1, ); assert_eq!( collect_choices(&responses) .iter() .filter(|choice| choice.finish_reason.is_some()) .count(), usize::from(finish), ); } } #[tokio::test] async fn qwen_eof_flushes_held_opener_before_source_annotated_reasoning() { let mut reasoning = chunk("", false); let choice = &mut reasoning.data.as_mut().unwrap().inner.choices[0]; choice.delta.content = None; choice.delta.reasoning_content = Some("eof thought".to_string()); let source = reasoning.data.as_mut().unwrap(); source.nvext = Some(serde_json::json!({"source": "reasoning"})); source.llm_metrics = Some(crate::protocols::common::metrics::LLMMetricAnnotation { chunk_tokens: 9, ..Default::default() }); reasoning.id = Some("eof-reasoning-id".to_string()); reasoning.event = Some("eof-reasoning-event".to_string()); reasoning.comment = Some(vec!["eof-reasoning-comment".to_string()]); let responses = apply_stream( stream::iter([chunk("before >() .await; let ordered: Vec<_> = responses .iter() .filter_map(|response| { let choice = response.data.as_ref()?.inner.choices.first()?; if let Some(ChatCompletionMessageContent::Text(text)) = &choice.delta.content { Some(("text", text.as_str())) } else { choice .delta .reasoning_content .as_deref() .map(|text| ("reasoning", text)) } }) .collect(); assert_eq!( ordered, vec![ ("text", "before "), ("text", ">().await); let ordered: Vec<_> = responses .iter() .filter_map(|response| { let choice = response.data.as_ref()?.inner.choices.first()?; if let Some(ChatCompletionMessageContent::Text(text)) = &choice.delta.content { Some(("text", text.as_str())) } else { choice .delta .reasoning_content .as_deref() .map(|text| ("reasoning", text)) } }) .collect(); assert_eq!( ordered, vec![ ("text", "before "), ("text", ">() .await; let choices = collect_choices(&responses); let calls: Vec<_> = choices .iter() .filter_map(|choice| choice.delta.tool_calls.as_ref()) .flatten() .collect(); assert_eq!( calls .iter() .find_map(|call| call.function.as_ref()?.name.as_deref()), Some("get_weather") ); let arguments: String = calls .iter() .filter_map(|call| call.function.as_ref()?.arguments.as_deref()) .collect(); assert_eq!(arguments, r#"{"city":"Tokyo"}"#); assert!( choices .iter() .all(|choice| choice.delta.reasoning_content.is_none()) ); } } #[tokio::test] async fn deferred_prefill_whitespace_is_reasoning_when_prose_arrives_first() { let responses = apply_stream( stream::iter([ chunk(" \n", false), chunk("reason{\"city\": ", false), chunk("\"Tokyo\"}", true), ]), Some(weather_tools()), Some(named_choice("get_weather")), false, UnifiedParserStartingState::Reasoning, QWEN3_UNIFIED_FAMILY, ) .collect::>() .await; let choices = collect_choices(&responses); let reasoning: String = choices .iter() .filter_map(|choice| choice.delta.reasoning_content.as_deref()) .collect(); assert_eq!(reasoning, " \nreason"); let arguments: String = choices .iter() .filter_map(|choice| choice.delta.tool_calls.as_ref()) .flatten() .filter_map(|call| call.function.as_ref()?.arguments.as_deref()) .collect(); assert_eq!(arguments, r#"{"city": "Tokyo"}"#); } #[tokio::test] async fn named_guided_json_becomes_a_tool_call() { let responses = apply_stream( stream::iter([ chunk("reason{\"city\": ", false), chunk("\"Tokyo\"}", true), ]), Some(weather_tools()), Some(named_choice("get_weather")), false, UnifiedParserStartingState::Reasoning, QWEN3_UNIFIED_FAMILY, ) .collect::>() .await; let choices = collect_choices(&responses); assert_eq!( choices[0].delta.reasoning_content.as_deref(), Some("reason") ); // A named choice now STREAMS: the name rides the first delta and the argument // bytes follow, so collect across deltas exactly as the required case does. let calls: Vec<_> = choices .iter() .filter_map(|choice| choice.delta.tool_calls.as_ref()) .flatten() .collect(); assert_eq!( calls .iter() .find_map(|call| call.function.as_ref()?.name.as_deref()), Some("get_weather") ); let arguments: String = calls .iter() .filter_map(|call| call.function.as_ref()?.arguments.as_deref()) .collect(); assert_eq!( arguments, "{\"city\": \"Tokyo\"}", "a named choice passes the model's argument bytes through verbatim" ); // The argument bytes must arrive as they are generated, not in one terminal // burst: the caller split the payload across two chunks, so a client that // renders each delta sees the call grow instead of waiting for the whole // object. One frame here means the wiring buffered what the parser streamed. let frames = calls .iter() .filter(|call| { call.function .as_ref() .and_then(|f| f.arguments.as_deref()) .is_some_and(|a| !a.is_empty()) }) .count(); assert!( frames >= 2, "arguments arrived in {frames} frame(s) - that is a burst, not a stream" ); assert!( choices .iter() .any(|c| c.finish_reason == Some(FinishReason::ToolCalls)), "the stream must terminate with ToolCalls" ); } #[tokio::test] async fn truncated_named_guided_json_stays_pending_at_stop() { let responses = apply_stream( stream::iter([ chunk("reason{\"city\": ", false), chunk("\"Tokyo\"", true), ]), Some(weather_tools()), Some(named_choice("get_weather")), false, UnifiedParserStartingState::Reasoning, QWEN3_UNIFIED_FAMILY, ) .collect::>() .await; let choices = collect_choices(&responses); assert!( choices .iter() .any(|choice| choice.delta.tool_calls.is_some()), "the guided parser may have already streamed the provisional call" ); assert_eq!( choices .iter() .filter_map(|choice| choice.finish_reason) .collect::>(), vec![FinishReason::Length], "an incomplete argument object must not be reported as a completed tool call" ); } #[tokio::test] async fn truncated_named_guided_json_downgrades_tool_calls_terminal() { let mut terminal = chunk("\"Tokyo\"", true); terminal.data.as_mut().unwrap().inner.choices[0].finish_reason = Some(FinishReason::ToolCalls); let responses = apply_stream( stream::iter([chunk("reason{\"city\": ", false), terminal]), Some(weather_tools()), Some(named_choice("get_weather")), false, UnifiedParserStartingState::Reasoning, QWEN3_UNIFIED_FAMILY, ) .collect::>() .await; let choices = collect_choices(&responses); assert!( choices .iter() .any(|choice| choice.delta.tool_calls.is_some()) ); assert_eq!( choices .iter() .filter_map(|choice| choice.finish_reason) .collect::>(), vec![FinishReason::Length] ); } #[tokio::test] async fn named_guided_json_continues_across_reasoning_detour() { let mut reasoning_detour = chunk("", false); let choice = &mut reasoning_detour.data.as_mut().unwrap().inner.choices[0]; choice.delta.content = None; choice.delta.reasoning_content = Some("interleaved thought".to_string()); let responses = apply_stream( stream::iter([ chunk("reason{\"city\": ", false), reasoning_detour, chunk("\"Tokyo\"}", true), ]), Some(weather_tools()), Some(named_choice("get_weather")), false, UnifiedParserStartingState::Reasoning, QWEN3_UNIFIED_FAMILY, ) .collect::>() .await; let choices = collect_choices(&responses); let arguments: String = choices .iter() .filter_map(|choice| choice.delta.tool_calls.as_ref()) .flatten() .filter_map(|call| call.function.as_ref()?.arguments.as_deref()) .collect(); assert_eq!(arguments, "{\"city\": \"Tokyo\"}"); assert!( choices.iter().any(|choice| { choice.delta.reasoning_content.as_deref() == Some("interleaved thought") }), "the reasoning detour must remain visible in its original position" ); assert!( choices .iter() .any(|choice| choice.finish_reason == Some(FinishReason::ToolCalls)), "the resumed argument object must complete the same guided tool call" ); } #[tokio::test] async fn native_qwen_call_continues_across_reasoning_detour() { let mut reasoning_detour = chunk("", false); let choice = &mut reasoning_detour.data.as_mut().unwrap().inner.choices[0]; choice.delta.content = None; choice.delta.reasoning_content = Some("interleaved thought".to_string()); let responses = apply_stream( stream::iter([ chunk("NY", false), reasoning_detour, chunk("C", true), ]), Some(weather_tools()), None, false, UnifiedParserStartingState::None, QWEN3_UNIFIED_FAMILY, ) .collect::>() .await; let choices = collect_choices(&responses); let calls: Vec<_> = choices .iter() .filter_map(|choice| choice.delta.tool_calls.as_ref()) .flatten() .collect(); let arguments: String = calls .iter() .filter_map(|call| call.function.as_ref()?.arguments.as_deref()) .collect(); assert_eq!(arguments, r#"{"city":"NYC"}"#); assert!(calls.iter().any(|call| call.index == 0)); assert_eq!( choices .iter() .filter_map(|choice| choice.finish_reason) .collect::>(), vec![FinishReason::ToolCalls] ); let reasoning_position = choices .iter() .position(|choice| choice.delta.reasoning_content.is_some()) .expect("the already-parsed reasoning delta must remain visible"); assert!( choices[..reasoning_position] .iter() .any(|choice| choice.delta.tool_calls.is_some()), "the provisional native call must be released before the reasoning detour" ); assert!( choices[reasoning_position + 1..] .iter() .any(|choice| choice.delta.tool_calls.is_some()), "the resumed native call fragment must follow the reasoning detour" ); } #[tokio::test] async fn native_qwen_partial_call_opener_survives_reasoning_detour_at_every_split() { let opener = ""; let body_and_answer = concat!( "\n\n", "Paris\n", "\nanswer" ); for split in 1..opener.len() { let mut reasoning_detour = chunk("", false); let choice = &mut reasoning_detour.data.as_mut().unwrap().inner.choices[0]; choice.delta.content = None; choice.delta.reasoning_content = Some("interleaved thought".to_string()); let continuation = format!("{}{}", &opener[split..], body_and_answer); let responses = apply_stream( stream::iter([ chunk(&opener[..split], false), reasoning_detour, chunk(&continuation, true), ]), Some(weather_tools()), None, false, UnifiedParserStartingState::None, QWEN3_UNIFIED_FAMILY, ) .collect::>() .await; let choices = collect_choices(&responses); let calls: Vec<_> = choices .iter() .flat_map(|choice| choice.delta.tool_calls.iter().flatten()) .collect(); let arguments: String = calls .iter() .filter_map(|call| call.function.as_ref()?.arguments.as_deref()) .collect(); assert_eq!(arguments, r#"{"city":"Paris"}"#, "split {split}"); assert_eq!( calls .iter() .filter_map(|call| call.function.as_ref()?.name.as_deref()) .collect::(), "get_weather", "split {split}" ); let mut indices: Vec<_> = calls.iter().map(|call| call.index).collect(); indices.sort_unstable(); indices.dedup(); assert_eq!(indices, vec![0], "split {split}"); assert_eq!( calls .iter() .filter_map(|call| call.id.as_deref()) .collect::>() .len(), 1, "split {split} must keep one call ID" ); assert!(choices.iter().any(|choice| { choice.delta.reasoning_content.as_deref() == Some("interleaved thought") })); let content: String = choices .iter() .filter_map(|choice| match &choice.delta.content { Some(ChatCompletionMessageContent::Text(text)) => Some(text.as_str()), _ => None, }) .collect(); assert_eq!(content, "answer", "split {split}"); assert_eq!( choices .iter() .filter_map(|choice| choice.finish_reason) .collect::>(), vec![FinishReason::ToolCalls], "split {split}" ); let reasoning_position = choices .iter() .position(|choice| choice.delta.reasoning_content.is_some()) .expect("reasoning detour must be emitted"); let call_position = choices .iter() .position(|choice| choice.delta.tool_calls.is_some()) .expect("tool call must follow the detour"); let answer_position = choices .iter() .position(|choice| choice.delta.content.is_some()) .expect("trailing answer must be emitted"); assert!( reasoning_position < call_position && call_position < answer_position, "raw/parsed/raw order changed at split {split}" ); } } #[tokio::test] async fn native_qwen_mismatched_partial_opener_orders_recovery_before_reasoning_at_every_split() { let opener = ""; for split in 1..opener.len() { let mismatch_position = opener.len() - 1; let continuation = format!("{}Xpost", &opener[split..mismatch_position]); let mut reasoning_detour = chunk("", false); let choice = &mut reasoning_detour.data.as_mut().unwrap().inner.choices[0]; choice.delta.content = None; choice.delta.reasoning_content = Some("interleaved thought".to_string()); let responses = apply_stream( stream::iter([ chunk(&opener[..split], false), reasoning_detour, chunk(&continuation, true), ]), Some(weather_tools()), None, false, UnifiedParserStartingState::None, QWEN3_UNIFIED_FAMILY, ) .collect::>() .await; let choices = collect_choices(&responses); let ordered: Vec<_> = choices .iter() .filter_map(|choice| { if let Some(ChatCompletionMessageContent::Text(text)) = &choice.delta.content { Some(("content", text.as_str())) } else { choice .delta .reasoning_content .as_deref() .map(|text| ("reasoning", text)) } }) .collect(); assert_eq!( ordered, vec![ ("content", &opener[..split]), ("reasoning", "interleaved thought"), ("content", &continuation[..]), ], "mismatch at opener split {split}" ); } } #[tokio::test] async fn native_qwen_partial_opener_eof_flushes_recovered_text_before_reasoning() { let mut reasoning_detour = chunk("", false); let choice = &mut reasoning_detour.data.as_mut().unwrap().inner.choices[0]; choice.delta.content = None; choice.delta.reasoning_content = Some("interleaved thought".to_string()); let responses = apply_stream( stream::iter([chunk("before >() .await; let choices = collect_choices(&responses); let ordered: Vec<_> = choices .iter() .filter_map(|choice| { if let Some(ChatCompletionMessageContent::Text(text)) = &choice.delta.content { Some(("content", text.as_str())) } else { choice .delta .reasoning_content .as_deref() .map(|text| ("reasoning", text)) } }) .collect(); assert_eq!( ordered, vec![ ("content", "before "), ("content", "1", false), reasoning_detour, chunk("2", true), ]), Some(count_tools()), None, false, UnifiedParserStartingState::None, QWEN3_UNIFIED_FAMILY, ) .collect::>() .await; let choices = collect_choices(&responses); let calls: Vec<_> = choices .iter() .flat_map(|choice| choice.delta.tool_calls.iter().flatten()) .collect(); assert_eq!(calls.len(), 1); assert_eq!( calls[0] .function .as_ref() .and_then(|function| function.name.as_deref()), Some("count_items") ); assert_eq!( calls[0] .function .as_ref() .and_then(|function| function.arguments.as_deref()), Some(r#"{"count":12}"#) ); assert!(choices.iter().any(|choice| { choice.delta.reasoning_content.as_deref() == Some("interleaved thought") })); assert_eq!( choices .iter() .filter_map(|choice| choice.finish_reason) .collect::>(), vec![FinishReason::ToolCalls] ); } #[tokio::test] async fn truncated_named_guided_json_stays_pending_across_reasoning_detour_at_eof() { let mut reasoning_detour = chunk("", false); let choice = &mut reasoning_detour.data.as_mut().unwrap().inner.choices[0]; choice.delta.content = None; choice.delta.reasoning_content = Some("interleaved thought".to_string()); let responses = apply_stream( stream::iter([chunk("reason{\"city\": ", false), reasoning_detour]), Some(weather_tools()), Some(named_choice("get_weather")), false, UnifiedParserStartingState::Reasoning, QWEN3_UNIFIED_FAMILY, ) .collect::>() .await; let choices = collect_choices(&responses); assert!( choices .iter() .any(|choice| choice.delta.tool_calls.is_some()), "the cursor may have streamed provisional name and argument fragments" ); assert!( choices .iter() .all(|choice| choice.finish_reason != Some(FinishReason::ToolCalls)), "EOF must not complete an invalid or truncated argument object" ); } #[tokio::test] async fn bare_guided_json_under_prompt_injected_reasoning_still_becomes_a_tool_call() { // Regression for the guided-JSON-swallowed-as-reasoning bug: a Qwen3-Thinking // style template appends `` to the generation prompt // (`prompt_injected_reasoning`, modeled here by starting the parser at // `Reasoning`), and guided decoding forbids the model from ever emitting // ``. Unconditionally honoring the `Reasoning` prefill would make the // parser wait forever for a closer that will never arrive, classifying the // ENTIRE guided JSON payload as `reasoning_content` with zero tool calls // extracted. The payload here has no `reason` prefix at all — its first // byte is `[` — so `bare_guided_json_prefill` must reclassify this choice's // starting state to `None` and let the array parse as a tool call instead. let responses = apply_stream( stream::iter([chunk( r#"[{"name":"get_weather","parameters":{"city":"Paris"}}]"#, true, )]), Some(weather_tools()), Some(ChatCompletionToolChoiceOption::Required), false, UnifiedParserStartingState::Reasoning, QWEN3_UNIFIED_FAMILY, ) .collect::>() .await; let choices = collect_choices(&responses); assert!( choices.iter().all(|c| c.delta.reasoning_content.is_none()), "a bare guided JSON payload with no reasoning prefix must not be classified \ as reasoning_content: {choices:?}" ); let calls: Vec<_> = choices .iter() .filter_map(|choice| choice.delta.tool_calls.as_ref()) .flatten() .collect(); assert_eq!( calls .iter() .find_map(|call| call.function.as_ref()?.name.as_deref()), Some("get_weather"), "the guided JSON payload must still be recovered as a tool call: {choices:?}" ); let arguments: String = calls .iter() .filter_map(|call| call.function.as_ref()?.arguments.as_deref()) .collect(); assert_eq!(arguments, r#"{"city":"Paris"}"#); } #[tokio::test] async fn required_guided_json_becomes_a_tool_call() { let responses = apply_stream( stream::iter([chunk( r#"reason[{"name":"get_weather","parameters":{"city":"Tokyo"}}]"#, true, )]), Some(weather_tools()), Some(ChatCompletionToolChoiceOption::Required), false, UnifiedParserStartingState::Reasoning, QWEN3_UNIFIED_FAMILY, ) .collect::>() .await; let choices = collect_choices(&responses); let calls: Vec<_> = choices .iter() .filter_map(|choice| choice.delta.tool_calls.as_ref()) .flatten() .collect(); assert_eq!( calls .iter() .find_map(|call| call.function.as_ref()?.name.as_deref()), Some("get_weather") ); let arguments: String = calls .iter() .filter_map(|call| call.function.as_ref()?.arguments.as_deref()) .collect(); assert_eq!(arguments, r#"{"city":"Tokyo"}"#); } #[tokio::test] async fn required_guided_emits_name_before_polling_call_completion() { let prefix = r#"reason[{"name":"get_weather","arguments":{"#; let upstream = stream::iter([chunk(prefix, false)]).chain(stream::poll_fn(|_| { panic!("apply_stream polled for call-completing bytes before emitting the tool name") })); let responses = apply_stream( upstream, Some(weather_tools()), Some(ChatCompletionToolChoiceOption::Required), false, UnifiedParserStartingState::Reasoning, QWEN3_UNIFIED_FAMILY, ); tokio::pin!(responses); let reasoning = responses.next().await.expect("reasoning delta"); assert_eq!( collect_choices(&[reasoning])[0] .delta .reasoning_content .as_deref(), Some("reason") ); let tool = responses.next().await.expect("tool-name delta"); let tool_responses = [tool]; let tool_choices = collect_choices(&tool_responses); let call = &tool_choices[0] .delta .tool_calls .as_ref() .expect("tool call before completion")[0]; assert_eq!( call.function .as_ref() .and_then(|function| function.name.as_deref()), Some("get_weather") ); } /// Drive one guided-`required` stream whose payload closes only in the LAST /// upstream chunk, and report how many emitted responses carried a tool-call /// delta plus the assembled logical events. /// /// Anything counted above one is an argument fragment that went out before the /// payload closed, which is exactly the difference the guided tool-call /// streaming rollback lever is supposed to make. Nothing here touches process /// env, so it is safe under the default parallel test runner. async fn guided_required_tool_chunk_count( guided_streaming: bool, ) -> (usize, Vec) { let head = r#"reason[{"name":"get_weather","arguments":{"city":"Tok"#; let tail = r#"yo"}}]"#; let responses = apply_stream_with_constraint( stream::iter([chunk(head, false), chunk(tail, true)]), Some(weather_tools()), GuidedToolConstraint::GuidedJsonRequired, UnifiedParserStartingState::Reasoning, QWEN3_UNIFIED_FAMILY, guided_streaming, ) .collect::>() .await; let tool_chunks = responses .iter() .filter(|response| { response.data.as_ref().is_some_and(|data| { data.inner.choices.iter().any(|choice| { choice .delta .tool_calls .as_ref() .is_some_and(|calls| !calls.is_empty()) }) }) }) .count(); (tool_chunks, logical_events(&responses)) } #[tokio::test] async fn guided_streaming_rollback_buffers_the_call_to_completion() { let expected_call = LogicalEvent::Tool { index: 0, name: Some("get_weather".to_string()), arguments: r#"{"city":"Tokyo"}"#.to_string(), }; // Released (production default when DYN_ENABLE_GUIDED_TOOL_STREAMING is unset): // the call goes out in pieces as the argument object streams in. let (released_chunks, released_events) = guided_required_tool_chunk_count(true).await; assert!( released_chunks > 1, "guided streaming must emit argument fragments before the payload closes, \ got {released_chunks} tool-call chunk(s)" ); // Rolled back (DYN_ENABLE_GUIDED_TOOL_STREAMING=0): nothing may go out until the // payload closes, so the whole call arrives as one chunk. let (rolled_back_chunks, rolled_back_events) = guided_required_tool_chunk_count(false).await; assert_eq!( rolled_back_chunks, 1, "DYN_ENABLE_GUIDED_TOOL_STREAMING=0 must buffer the guided call to completion \ on the unified path, but {rolled_back_chunks} tool-call chunk(s) went out" ); // The lever changes WHEN the call is emitted, never WHAT is emitted. assert!( released_events.contains(&expected_call), "released events lost the call: {released_events:?}" ); assert!( rolled_back_events.contains(&expected_call), "rolled-back events lost the call: {rolled_back_events:?}" ); } #[test] fn only_stream_best_effort_may_emit_before_the_payload_closes() { // The two buffering contracts are interchangeable for WHEN, but not for WHAT a // malformed payload becomes, so the rollback must pick `RecoverAsText` (text) // and not `Reject` (typed error). assert_eq!( invalid_guided_payload_policy(true), InvalidGuidedPayloadPolicy::StreamBestEffort ); assert_eq!( invalid_guided_payload_policy(false), InvalidGuidedPayloadPolicy::RecoverAsText ); } #[tokio::test] async fn already_parsed_choices_pass_through_untouched() { // A chunk that already carries reasoning_content was parsed upstream; running // it through the parser again would double it. let mut pre_parsed = chunk("", false); let choice = &mut pre_parsed.data.as_mut().unwrap().inner.choices[0]; choice.delta.content = None; choice.delta.reasoning_content = Some("already".to_string()); let responses = apply_stream( stream::iter([pre_parsed]), None, None, false, UnifiedParserStartingState::None, QWEN3_UNIFIED_FAMILY, ) .collect::>() .await; let choices = collect_choices(&responses); assert_eq!(choices.len(), 1); assert_eq!( choices[0].delta.reasoning_content.as_deref(), Some("already") ); } #[tokio::test] async fn resumed_raw_tool_call_gets_a_new_index_after_an_already_parsed_detour() { let first_partial = concat!( "\n\n", "Paris" ); let already_parsed = { let mut value = chunk("", false); let choice = &mut value.data.as_mut().unwrap().inner.choices[0]; choice.delta.content = None; choice.delta.reasoning_content = Some("already parsed".to_string()); value }; let second_call = concat!( "\n\n", "Tokyo\n\n" ); let responses = apply_stream( stream::iter([ chunk(first_partial, false), already_parsed, chunk(second_call, true), ]), Some(weather_tools()), None, false, UnifiedParserStartingState::None, QWEN3_UNIFIED_FAMILY, ) .collect::>() .await; let choices = collect_choices(&responses); let calls: Vec<_> = choices .iter() .filter_map(|choice| choice.delta.tool_calls.as_ref()) .flatten() .collect(); assert!(calls.iter().any(|call| call.index == 0)); assert!(calls.iter().any(|call| call.index == 1)); let first_ids: HashSet<_> = calls .iter() .filter(|call| call.index == 0) .filter_map(|call| call.id.as_deref()) .collect(); let second_ids: HashSet<_> = calls .iter() .filter(|call| call.index == 1) .filter_map(|call| call.id.as_deref()) .collect(); assert_eq!( first_ids.len(), 1, "the first call identity must stay stable" ); assert_eq!(second_ids.len(), 1, "the resumed call needs one stable ID"); assert_ne!(first_ids, second_ids, "separate calls must not reuse an ID"); let names: Vec<_> = calls .iter() .filter_map(|call| call.function.as_ref()?.name.as_deref()) .collect(); assert_eq!(names, vec!["get_weather", "get_weather"]); } #[tokio::test] async fn mixed_preparsed_reasoning_still_parses_raw_content() { let output = concat!( "\n\n", "Tokyo\n\n" ); let mut mixed = chunk(output, true); mixed.data.as_mut().unwrap().inner.choices[0] .delta .reasoning_content = Some("already parsed".to_string()); let responses = apply_stream( stream::iter([mixed]), Some(weather_tools()), None, false, UnifiedParserStartingState::None, QWEN3_UNIFIED_FAMILY, ) .collect::>() .await; let choices = collect_choices(&responses); let visible: String = choices .iter() .filter_map(|choice| match &choice.delta.content { Some(ChatCompletionMessageContent::Text(text)) => Some(text.as_str()), _ => None, }) .collect(); assert_eq!( choices .iter() .filter_map(|choice| choice.delta.reasoning_content.as_deref()) .collect::(), "already parsed" ); assert!( choices .iter() .any(|choice| choice.delta.tool_calls.is_some()), "raw native tool markup must still become a structured call" ); assert!( !visible.contains(""), "raw markup leaked: {visible:?}" ); } #[tokio::test] async fn mixed_preparsed_tool_call_still_parses_raw_content() { let output = concat!( "\n\n", "Tokyo\n\n" ); let mut mixed = chunk(output, true); let choice = &mut mixed.data.as_mut().unwrap().inner.choices[0]; choice.delta.tool_calls = Some(vec![ChatCompletionMessageToolCallChunk { custom: None, index: 7, id: Some("existing-call".to_string()), r#type: Some(FunctionType::Function), function: Some(FunctionCallStream { name: Some("already_parsed".to_string()), arguments: Some("{}".to_string()), }), }]); let responses = apply_stream( stream::iter([mixed]), Some(weather_tools()), None, false, UnifiedParserStartingState::None, QWEN3_UNIFIED_FAMILY, ) .collect::>() .await; let choices = collect_choices(&responses); let visible: String = choices .iter() .filter_map(|choice| match &choice.delta.content { Some(ChatCompletionMessageContent::Text(text)) => Some(text.as_str()), _ => None, }) .collect(); let names: Vec<_> = choices .iter() .filter_map(|choice| choice.delta.tool_calls.as_ref()) .flatten() .filter_map(|call| call.function.as_ref()?.name.as_deref()) .collect(); assert!(names.contains(&"already_parsed")); assert!(names.contains(&"get_weather")); assert!( !visible.contains(""), "raw markup leaked: {visible:?}" ); } #[tokio::test] async fn already_parsed_detour_allocates_index_after_parser_index_collision() { let raw_call = concat!( "\n\n", "Tokyo\n\n" ); let mut upstream_call = chunk("", false); let choice = &mut upstream_call.data.as_mut().unwrap().inner.choices[0]; choice.delta.content = None; choice.delta.tool_calls = Some(vec![ChatCompletionMessageToolCallChunk { custom: None, index: 0, id: Some("upstream-call".to_string()), r#type: Some(FunctionType::Function), function: Some(FunctionCallStream { name: Some("upstream".to_string()), arguments: Some("{\"city\":\"Paris\"}".to_string()), }), }]); let mut continuation = chunk("", false); let choice = &mut continuation.data.as_mut().unwrap().inner.choices[0]; choice.delta.content = None; choice.delta.tool_calls = Some(vec![ChatCompletionMessageToolCallChunk { custom: None, index: 0, id: None, r#type: None, function: Some(FunctionCallStream { name: None, arguments: Some("more".to_string()), }), }]); let responses = apply_stream( stream::iter([chunk(raw_call, false), upstream_call, continuation]), Some(weather_tools()), None, false, UnifiedParserStartingState::None, QWEN3_UNIFIED_FAMILY, ) .collect::>() .await; let choices = collect_choices(&responses); let calls: Vec<_> = choices .iter() .filter_map(|choice| choice.delta.tool_calls.as_ref()) .flatten() .collect(); let parser_call = calls .iter() .find(|call| { call.function .as_ref() .and_then(|function| function.name.as_deref()) == Some("get_weather") }) .expect("raw parser call must be emitted"); let upstream_open = calls .iter() .find(|call| { call.function .as_ref() .and_then(|function| function.name.as_deref()) == Some("upstream") }) .expect("the first upstream call chunk must keep its name"); assert_eq!(parser_call.index, 0); assert_eq!(upstream_open.index, 1); assert_eq!(upstream_open.id.as_deref(), Some("upstream-call")); let passthrough_continuation = choices .iter() .filter_map(|choice| choice.delta.tool_calls.as_ref()) .flatten() .find(|call| { call.function .as_ref() .and_then(|function| function.arguments.as_deref()) == Some("more") }) .expect("upstream continuation must be emitted"); assert_eq!( passthrough_continuation.index, 1, "continuation must retain the upstream call's remapped index" ); assert_eq!(passthrough_continuation.id, None); assert_eq!(passthrough_continuation.r#type, None); assert_eq!( passthrough_continuation .function .as_ref() .and_then(|function| function.name.as_deref()), None ); } #[tokio::test] async fn preparsed_terminal_choice_does_not_drop_buffered_bytes() { let mut terminal = chunk("", true); let choice = &mut terminal.data.as_mut().unwrap().inner.choices[0]; choice.delta.content = None; choice.delta.reasoning_content = Some("already".to_string()); let responses = apply_stream( stream::iter([chunk("hello >() .await; let choices = collect_choices(&responses); let visible: String = choices .iter() .filter_map(|choice| match &choice.delta.content { Some(ChatCompletionMessageContent::Text(text)) => Some(text.as_str()), _ => None, }) .collect(); assert_eq!(visible, "hello \n\n", "Tokyo\n\n" ); let mut terminal = chunk("", true); let choice = &mut terminal.data.as_mut().unwrap().inner.choices[0]; choice.delta.content = None; choice.delta.reasoning_content = Some("already parsed".to_string()); let responses = apply_stream( stream::iter([chunk(tool_call, false), terminal]), Some(weather_tools()), None, false, UnifiedParserStartingState::None, QWEN3_UNIFIED_FAMILY, ) .collect::>() .await; let choices = collect_choices(&responses); assert!( choices .iter() .any(|choice| choice.delta.tool_calls.is_some()), "the raw first chunk must emit a tool call" ); assert_eq!( choices .iter() .filter_map(|choice| choice.finish_reason) .next_back(), Some(FinishReason::ToolCalls), "an already-parsed terminal Stop must retain the earlier tool-call state" ); } #[tokio::test] async fn raw_preparsed_raw_sequence_preserves_order_and_buffered_bytes() { let mut pre_parsed = chunk("", false); let choice = &mut pre_parsed.data.as_mut().unwrap().inner.choices[0]; choice.delta.content = None; choice.delta.reasoning_content = Some("middle".to_string()); let responses = apply_stream( stream::iter([ chunk("before >() .await; let choices = collect_choices(&responses); let ordered: Vec<_> = choices .iter() .filter_map(|choice| { if let Some(ChatCompletionMessageContent::Text(text)) = &choice.delta.content { Some(("content", text.as_str())) } else { choice .delta .reasoning_content .as_deref() .map(|reasoning| ("reasoning", reasoning)) } }) .collect(); assert_eq!( ordered, vec![ ("content", "before "), ("content", "visible-before-gap ", false), pre_parsed, chunk("visible-after-gap", true), ]), None, None, false, UnifiedParserStartingState::Reasoning, QWEN3_UNIFIED_FAMILY, ) .collect::>() .await; let choices = collect_choices(&responses); let reasoning: String = choices .iter() .filter_map(|choice| choice.delta.reasoning_content.as_deref()) .collect(); let visible: String = choices .iter() .filter_map(|choice| match &choice.delta.content { Some(ChatCompletionMessageContent::Text(text)) => Some(text.as_str()), _ => None, }) .collect(); assert_eq!( visible, "visible-before-gap visible-after-gap", "text after the already-parsed gap must resume as content, not reasoning" ); assert_eq!(reasoning, "secretalready parsed"); } #[tokio::test] async fn gpt_oss_consumed_analysis_prefix_is_not_replayed_after_parsed_gap() { let mut parsed = chunk("", false); let choice = &mut parsed.data.as_mut().unwrap().inner.choices[0]; choice.delta.content = None; choice.delta.reasoning_content = Some("already parsed".into()); let responses = apply_stream_with_policy( stream::iter([ chunk( "secret<|end|><|start|>assistant<|channel|>final<|message|>before ", false, ), parsed, chunk("after", true), ]), None, GuidedToolConstraint::None, StreamPrefill { starting_state: UnifiedParserStartingState::Reasoning, prefix: Some(std::sync::Arc::from( "<|start|>assistant<|channel|>analysis<|message|>consumed", )), }, "gpt_oss", true, UnifiedRequestPolicy::default(), ) .collect::>() .await; assert!(responses.iter().all(|response| !response.is_error())); assert_eq!( logical_events(&responses), vec![ LogicalEvent::Reasoning("secret".into()), LogicalEvent::Text("before ".into()), LogicalEvent::Reasoning("already parsed".into()), LogicalEvent::Text("after".into()), ] ); assert_eq!( collect_choices(&responses).last().unwrap().finish_reason, Some(FinishReason::Stop) ); } #[tokio::test] async fn raw_preparsed_raw_terminal_normalizes_stop_after_tool_call_before_gap() { let tool_call = concat!( "\n\n", "Tokyo\n\n" ); let mut pre_parsed = chunk("", false); let choice = &mut pre_parsed.data.as_mut().unwrap().inner.choices[0]; choice.delta.content = None; choice.delta.reasoning_content = Some("already parsed".to_string()); let responses = apply_stream( stream::iter([ chunk(tool_call, false), pre_parsed, chunk(" trailing text", true), ]), Some(weather_tools()), None, false, UnifiedParserStartingState::None, QWEN3_UNIFIED_FAMILY, ) .collect::>() .await; let choices = collect_choices(&responses); assert!( choices .iter() .any(|choice| choice.delta.tool_calls.is_some()), "the raw first chunk must emit a tool call" ); let finish_reasons: Vec<_> = choices .iter() .filter_map(|choice| choice.finish_reason) .collect(); assert_eq!( finish_reasons, vec![FinishReason::ToolCalls], "a fresh parser instance for the resumed raw run must not lose the \ tool-call history from before the already-parsed gap, and exactly \ one terminal must be emitted" ); } #[tokio::test] async fn two_consecutive_already_parsed_chunks_after_tool_call_normalize_once() { let tool_call = concat!( "\n\n", "Tokyo\n\n" ); let already_parsed_chunk = |reasoning: &str, finish: bool| { let mut c = chunk("", finish); let choice = &mut c.data.as_mut().unwrap().inner.choices[0]; choice.delta.content = None; choice.delta.reasoning_content = Some(reasoning.to_string()); c }; let responses = apply_stream( stream::iter([ chunk(tool_call, false), already_parsed_chunk("first already-parsed", false), already_parsed_chunk("second already-parsed", false), already_parsed_chunk("third already-parsed (terminal)", true), ]), Some(weather_tools()), None, false, UnifiedParserStartingState::None, QWEN3_UNIFIED_FAMILY, ) .collect::>() .await; let choices = collect_choices(&responses); assert!( choices .iter() .any(|choice| choice.delta.tool_calls.is_some()), "the raw first chunk must emit a tool call" ); let finish_reasons: Vec<_> = choices .iter() .filter_map(|choice| choice.finish_reason) .collect(); assert_eq!( finish_reasons, vec![FinishReason::ToolCalls], "two already-parsed chunks in a row after the tool call (the second \ with no live ChoiceState left to remove) must still normalize to \ exactly one ToolCalls terminal, not Stop and not more than one terminal" ); } #[tokio::test] async fn tool_history_is_isolated_per_choice_index() { fn two_choice_response( index0: (&str, bool), index1: (&str, bool), ) -> Annotated { let make = |index: u32, text: &str, finish: bool| ChatChoiceStream { index, delta: ChatCompletionStreamResponseDelta { role: Some(Role::Assistant), content: Some(ChatCompletionMessageContent::Text(text.to_string())), tool_calls: None, function_call: None, refusal: None, reasoning_content: None, }, finish_reason: finish.then_some(FinishReason::Stop), logprobs: None, }; #[allow(deprecated)] let response = NvCreateChatCompletionStreamResponse { message_boundary: None, inner: CreateChatCompletionStreamResponse { id: "test".to_string(), choices: vec![make(0, index0.0, index0.1), make(1, index1.0, index1.1)], created: 0, model: "qwen3".to_string(), system_fingerprint: None, service_tier: None, object: "chat.completion.chunk".to_string(), usage: None, }, nvext: None, llm_metrics: None, prompt_logprobs: None, tool_call_completion: Vec::new(), model_eos: Vec::new(), }; Annotated::from_data(response) } let already_parsed_pair = || { let mut response = two_choice_response(("", false), ("", false)); for choice in &mut response.data.as_mut().unwrap().inner.choices { choice.delta.content = None; choice.delta.reasoning_content = Some("already parsed".to_string()); } response }; let tool_call = concat!( "\n\n", "Tokyo\n\n" ); let responses = apply_stream( stream::iter([ two_choice_response((tool_call, false), ("hello", false)), already_parsed_pair(), two_choice_response((" trailing0", true), (" trailing1", true)), ]), Some(weather_tools()), None, false, UnifiedParserStartingState::None, QWEN3_UNIFIED_FAMILY, ) .collect::>() .await; let choices = collect_choices(&responses); let finish_for = |index: u32| { choices .iter() .filter(|choice| choice.index == index) .filter_map(|choice| choice.finish_reason) .collect::>() }; assert_eq!( finish_for(0), vec![FinishReason::ToolCalls], "choice 0 emitted a tool call before the gap and must normalize to ToolCalls" ); assert_eq!( finish_for(1), vec![FinishReason::Stop], "choice 1 never emitted a tool call and must not be contaminated by \ choice 0's tool history" ); } #[tokio::test] #[allow(deprecated)] async fn legacy_function_call_passes_through_untouched() { let mut legacy = chunk("", true); let choice = &mut legacy.data.as_mut().unwrap().inner.choices[0]; choice.delta.content = None; choice.delta.function_call = Some(ChatCompletionStreamResponseDeltaFunctionCall { name: Some("get_weather".to_string()), arguments: Some(r#"{"city":"Tokyo"}"#.to_string()), }); let responses = apply_stream( stream::iter([legacy]), None, None, false, UnifiedParserStartingState::None, QWEN3_UNIFIED_FAMILY, ) .collect::>() .await; let choices = collect_choices(&responses); assert_eq!(choices.len(), 1); assert_eq!( choices[0] .delta .function_call .as_ref() .and_then(|call| call.name.as_deref()), Some("get_weather") ); } #[tokio::test] async fn a_stream_without_a_finish_reason_still_terminates_a_tool_call() { let output = concat!( "\n\n", "Tokyo\n\n" ); // No terminating chunk at all — the stream just ends. let responses = apply_stream( stream::iter([chunk(output, false)]), Some(weather_tools()), None, false, UnifiedParserStartingState::None, QWEN3_UNIFIED_FAMILY, ) .collect::>() .await; let choices = collect_choices(&responses); assert_eq!( choices .iter() .filter_map(|choice| choice.finish_reason) .next_back(), Some(FinishReason::ToolCalls), "the backstop must synthesize a terminal reason or a strict client hangs" ); } #[tokio::test] async fn already_parsed_gap_then_immediate_eof_still_terminates_a_tool_call() { let tool_call = concat!( "\n\n", "Tokyo\n\n" ); let mut already_parsed = chunk("", false); let choice = &mut already_parsed.data.as_mut().unwrap().inner.choices[0]; choice.delta.content = None; choice.delta.reasoning_content = Some("already parsed".to_string()); // Raw tool call, then an already-parsed chunk that removes the live // ChoiceState (no finish_reason on it), then the stream just ends — no // further chunks, no usage-only chunk, no explicit terminal at all. let responses = apply_stream( stream::iter([chunk(tool_call, false), already_parsed]), Some(weather_tools()), None, false, UnifiedParserStartingState::None, QWEN3_UNIFIED_FAMILY, ) .collect::>() .await; let choices = collect_choices(&responses); assert!( choices .iter() .any(|choice| choice.delta.tool_calls.is_some()), "the raw first chunk must emit a tool call" ); let finish_reasons: Vec<_> = choices .iter() .filter_map(|choice| choice.finish_reason) .collect(); assert_eq!( finish_reasons, vec![FinishReason::ToolCalls], "a choice whose live state was removed by an already-parsed gap must \ still get exactly one terminal at end-of-stream, not zero — the EOF \ backstop must not silently drop it" ); } #[tokio::test] async fn complete_passthrough_tool_call_without_terminal_gets_eof_terminal() { let mut parsed = chunk("", false); let choice = &mut parsed.data.as_mut().unwrap().inner.choices[0]; choice.delta.content = None; choice.delta.tool_calls = Some(vec![ChatCompletionMessageToolCallChunk { custom: None, index: 0, id: Some("call-complete".to_string()), r#type: Some(FunctionType::Function), function: Some(FunctionCallStream { name: Some("get_weather".to_string()), arguments: Some("{}".to_string()), }), }]); let responses = apply_stream( stream::iter([parsed]), None, None, false, UnifiedParserStartingState::None, QWEN3_UNIFIED_FAMILY, ) .collect::>() .await; let choices = collect_choices(&responses); let finish_reasons: Vec<_> = choices .iter() .filter_map(|choice| choice.finish_reason) .collect(); assert_eq!( finish_reasons, vec![FinishReason::ToolCalls], "a complete already-parsed call must receive a terminal reason at EOF" ); } #[tokio::test] async fn passthrough_argument_fragments_are_combined_before_eof_validation() { let mut first = chunk("", false); let first_choice = &mut first.data.as_mut().unwrap().inner.choices[0]; first_choice.delta.content = None; first_choice.delta.tool_calls = Some(vec![ChatCompletionMessageToolCallChunk { custom: None, index: 0, id: Some("call-fragmented".to_string()), r#type: Some(FunctionType::Function), function: Some(FunctionCallStream { name: Some("get_weather".to_string()), arguments: Some(r#"{"city":"#.to_string()), }), }]); let mut second = chunk("", false); let second_choice = &mut second.data.as_mut().unwrap().inner.choices[0]; second_choice.delta.content = None; second_choice.delta.tool_calls = Some(vec![ChatCompletionMessageToolCallChunk { custom: None, index: 0, id: None, r#type: None, function: Some(FunctionCallStream { name: None, arguments: Some(r#""Tokyo"}"#.to_string()), }), }]); let responses = apply_stream( stream::iter([first, second]), None, None, false, UnifiedParserStartingState::None, QWEN3_UNIFIED_FAMILY, ) .collect::>() .await; let choices = collect_choices(&responses); let arguments = choices .iter() .flat_map(|choice| choice.delta.tool_calls.iter().flatten()) .filter_map(|call| call.function.as_ref()?.arguments.as_deref()) .fold(String::new(), |mut all, fragment| { all.push_str(fragment); all }); let finish_reasons: Vec<_> = choices .iter() .filter_map(|choice| choice.finish_reason) .collect(); assert_eq!(arguments, r#"{"city":"Tokyo"}"#); assert_eq!(finish_reasons, vec![FinishReason::ToolCalls]); } #[tokio::test] async fn valid_json_prefix_with_invalid_suffix_stays_provisional_at_eof() { let parsed_fragment = |arguments: &str, id: Option<&str>| { let mut parsed = chunk("", false); let choice = &mut parsed.data.as_mut().unwrap().inner.choices[0]; choice.delta.content = None; choice.delta.tool_calls = Some(vec![ChatCompletionMessageToolCallChunk { custom: None, index: 0, id: id.map(str::to_string), r#type: id.map(|_| FunctionType::Function), function: Some(FunctionCallStream { name: id.map(|_| "get_weather".to_string()), arguments: Some(arguments.to_string()), }), }]); parsed }; let responses = apply_stream( stream::iter([ parsed_fragment("{}", Some("call-suffix")), parsed_fragment(" trailing", None), ]), None, None, false, UnifiedParserStartingState::None, QWEN3_UNIFIED_FAMILY, ) .collect::>() .await; let choices = collect_choices(&responses); assert!( choices .iter() .filter_map(|choice| choice.finish_reason) .collect::>() == vec![FinishReason::Length], "a valid JSON prefix followed by malformed bytes must remain provisional" ); } #[tokio::test] async fn already_parsed_gap_terminal_precedes_a_usage_only_chunk() { let tool_call = concat!( "\n\n", "Tokyo\n\n" ); let mut already_parsed = chunk("", false); let choice = &mut already_parsed.data.as_mut().unwrap().inner.choices[0]; choice.delta.content = None; choice.delta.reasoning_content = Some("already parsed".to_string()); let mut usage_only = chunk("unused", false); usage_only.data.as_mut().unwrap().inner.choices.clear(); usage_only.data.as_mut().unwrap().inner.usage = Some( serde_json::from_value( serde_json::json!({"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}), ) .unwrap(), ); let responses = apply_stream( stream::iter([chunk(tool_call, false), already_parsed, usage_only]), Some(weather_tools()), None, false, UnifiedParserStartingState::None, QWEN3_UNIFIED_FAMILY, ) .collect::>() .await; let usage_position = responses .iter() .position(|response| { response .data .as_ref() .is_some_and(|data| data.inner.choices.is_empty()) }) .expect("the usage-only chunk must still be forwarded"); let terminal_position = responses .iter() .position(|response| { response.data.as_ref().is_some_and(|data| { data.inner .choices .iter() .any(|choice| choice.finish_reason == Some(FinishReason::ToolCalls)) }) }) .expect("a history-only choice must still get exactly one ToolCalls terminal"); assert!( terminal_position < usage_position, "the terminal must precede the usage-only chunk, per OpenAI stream ordering" ); } #[tokio::test] async fn non_tool_choice_gets_no_synthetic_terminal_after_a_gap() { let mut already_parsed = chunk("", false); let choice = &mut already_parsed.data.as_mut().unwrap().inner.choices[0]; choice.delta.content = None; choice.delta.reasoning_content = Some("already parsed".to_string()); // No tool call anywhere: raw plain text, then a gap, then immediate EOF. let responses = apply_stream( stream::iter([chunk("plain text, no tool call", false), already_parsed]), None, None, false, UnifiedParserStartingState::None, QWEN3_UNIFIED_FAMILY, ) .collect::>() .await; let choices = collect_choices(&responses); assert!( choices.iter().all(|choice| choice.finish_reason.is_none()), "a choice that never called a tool must not get a synthesized terminal \ after a gap, matching the text-only, no-signal contract a live \ ChoiceState already has" ); } #[tokio::test] async fn late_tool_metadata_and_fragmented_name_survive_index_open() { fn call_chunk( id: Option<&str>, call_type: Option, name: Option<&str>, arguments: Option<&str>, finish: Option, ) -> Annotated { let mut response = chunk("", false); let choice = &mut response.data.as_mut().unwrap().inner.choices[0]; choice.delta.content = None; choice.delta.tool_calls = Some(vec![ChatCompletionMessageToolCallChunk { custom: None, index: 0, id: id.map(str::to_string), r#type: call_type, function: Some(FunctionCallStream { name: name.map(str::to_string), arguments: arguments.map(str::to_string), }), }]); choice.finish_reason = finish; response } let responses = apply_stream( stream::iter([ call_chunk(None, None, None, Some(""), None), call_chunk( Some("call-late"), Some(FunctionType::Function), Some("get_"), None, None, ), call_chunk( None, None, Some("weather"), Some("{}"), Some(FinishReason::ToolCalls), ), ]), None, None, false, UnifiedParserStartingState::None, QWEN3_UNIFIED_FAMILY, ) .collect::>() .await; let choices = collect_choices(&responses); let calls: Vec<_> = choices .iter() .flat_map(|choice| choice.delta.tool_calls.iter().flatten()) .collect(); assert_eq!( calls .iter() .filter_map(|call| call.id.as_deref()) .collect::>(), vec!["call-late"], "an ID supplied after its tool index opens must be forwarded once" ); assert_eq!( calls .iter() .filter_map(|call| call.r#type.clone()) .collect::>(), vec![FunctionType::Function], "a type supplied after its tool index opens must be forwarded once" ); assert_eq!( calls .iter() .filter_map(|call| call.function.as_ref()?.name.as_deref()) .collect::>(), vec!["get_", "weather"], "name fragments must remain separate deltas so clients can reassemble them" ); assert_eq!( calls .iter() .filter_map(|call| call.function.as_ref()?.arguments.as_deref()) .collect::>(), vec!["", "{}"] ); } #[tokio::test] async fn late_name_fragments_survive_raw_content_and_reasoning_detour() { let mut first = chunk("visible", false); first.data.as_mut().unwrap().inner.choices[0] .delta .tool_calls = Some(vec![ChatCompletionMessageToolCallChunk { custom: None, index: 0, id: Some("call-once".to_string()), r#type: Some(FunctionType::Function), function: Some(FunctionCallStream { name: Some("get_".to_string()), arguments: Some(r#"{"city":"Pa"#.to_string()), }), }]); let mut gap = chunk("", false); let gap_choice = &mut gap.data.as_mut().unwrap().inner.choices[0]; gap_choice.delta.content = None; gap_choice.delta.reasoning_content = Some("reasoning gap".to_string()); let mut resumed = chunk("", true); let resumed_choice = &mut resumed.data.as_mut().unwrap().inner.choices[0]; resumed_choice.delta.content = None; resumed_choice.delta.tool_calls = Some(vec![ChatCompletionMessageToolCallChunk { custom: None, index: 0, id: Some("duplicate-id".to_string()), r#type: Some(FunctionType::Function), function: Some(FunctionCallStream { name: Some("weather".to_string()), arguments: Some("ris\"}".to_string()), }), }]); resumed_choice.finish_reason = Some(FinishReason::ToolCalls); let responses = apply_stream( stream::iter([first, gap, resumed]), None, None, false, UnifiedParserStartingState::None, QWEN3_UNIFIED_FAMILY, ) .collect::>() .await; let choices = collect_choices(&responses); let calls: Vec<_> = choices .iter() .flat_map(|choice| choice.delta.tool_calls.iter().flatten()) .collect(); assert_eq!( calls .iter() .filter_map(|call| call.id.as_deref()) .collect::>(), vec!["call-once"], "an already emitted ID must not be repeated after a detour" ); assert_eq!( calls .iter() .filter_map(|call| call.r#type.clone()) .collect::>(), vec![FunctionType::Function], "an already emitted type must not be repeated after a detour" ); assert_eq!( calls .iter() .filter_map(|call| call.function.as_ref()?.name.as_deref()) .collect::>(), vec!["get_", "weather"], "a later name fragment must survive the parsed detour" ); assert_eq!( choices .iter() .filter_map(|choice| choice.finish_reason) .collect::>(), vec![FinishReason::ToolCalls] ); } #[tokio::test] async fn incomplete_upstream_call_stays_pending_across_reasoning_detour_at_eof() { let mut first = chunk("visible", false); first.data.as_mut().unwrap().inner.choices[0] .delta .tool_calls = Some(vec![ChatCompletionMessageToolCallChunk { custom: None, index: 0, id: Some("call-pending".to_string()), r#type: Some(FunctionType::Function), function: Some(FunctionCallStream { name: Some("get_weather".to_string()), arguments: Some(r#"{"city":"Pa"#.to_string()), }), }]); let mut gap = chunk("", false); let gap_choice = &mut gap.data.as_mut().unwrap().inner.choices[0]; gap_choice.delta.content = None; gap_choice.delta.reasoning_content = Some("reasoning gap".to_string()); let responses = apply_stream( stream::iter([first, gap]), None, None, false, UnifiedParserStartingState::None, QWEN3_UNIFIED_FAMILY, ) .collect::>() .await; let choices = collect_choices(&responses); assert!( choices.iter().any(|choice| choice .delta .tool_calls .as_ref() .is_some_and(|calls| !calls.is_empty())), "the partial upstream tool-call delta must remain visible" ); assert!( choices .iter() .filter_map(|choice| choice.finish_reason) .collect::>() == vec![FinishReason::Length], "EOF must not claim tool-call completion when upstream arguments remain partial" ); } #[tokio::test] #[allow(deprecated)] async fn legacy_function_call_choice_gets_no_synthetic_terminal_after_a_gap() { // The legacy `function_call` field is a distinct signal from `tool_calls` // (it has its own `FinishReason::FunctionCall`, never `ToolCalls`) and, like // plain text, is never treated as a completed tool call anywhere in this file // (only complete parser events and terminal parsed deltas are). An // already-parsed chunk that only carries `function_call` must therefore get // the same "no synthetic terminal" treatment after a gap as plain text does, // matching the pre-existing live-`ChoiceState` precedent, not a new gap. let mut already_parsed = chunk("", false); let choice = &mut already_parsed.data.as_mut().unwrap().inner.choices[0]; choice.delta.content = None; choice.delta.function_call = Some(ChatCompletionStreamResponseDeltaFunctionCall { name: Some("get_weather".to_string()), arguments: Some(r#"{"city": "Tokyo"}"#.to_string()), }); let responses = apply_stream( stream::iter([chunk("plain text, no tool call", false), already_parsed]), None, None, false, UnifiedParserStartingState::None, QWEN3_UNIFIED_FAMILY, ) .collect::>() .await; let choices = collect_choices(&responses); assert!( choices.iter().all(|choice| choice.finish_reason.is_none()), "a choice whose only already-parsed signal is the legacy function_call \ field must not get a synthesized ToolCalls terminal after a gap" ); } #[tokio::test] async fn already_parsed_terminal_carrying_its_own_tool_call_normalizes_stop() { // The live ChoiceState never emits a tool call itself (raw reasoning-only // content), but the already-parsed TERMINAL chunk that follows carries its // own `delta.tool_calls` together with `finish_reason: Stop` in the same // chunk. The state's own completed-call state is false, but the record must still // pick up this chunk's own tool call and normalize Stop -> ToolCalls. let mut terminal = chunk("", true); let choice = &mut terminal.data.as_mut().unwrap().inner.choices[0]; choice.delta.content = None; choice.delta.tool_calls = Some(vec![ChatCompletionMessageToolCallChunk { custom: None, index: 0, id: Some("call-terminal".to_string()), r#type: Some(FunctionType::Function), function: Some(FunctionCallStream { name: Some("get_weather".to_string()), arguments: Some("{}".to_string()), }), }]); let responses = apply_stream( stream::iter([chunk("reasoning only, no tool call", false), terminal]), Some(weather_tools()), None, false, UnifiedParserStartingState::None, QWEN3_UNIFIED_FAMILY, ) .collect::>() .await; let choices = collect_choices(&responses); assert_eq!( choices .iter() .filter_map(|choice| choice.finish_reason) .collect::>(), vec![FinishReason::ToolCalls], "an already-parsed terminal chunk that itself carries a tool call must \ normalize Stop to ToolCalls even when the live state never saw one" ); } #[tokio::test] async fn terminal_already_parsed_call_keeps_index_separate_from_raw_call() { let raw_call = concat!( "\n\n", "Tokyo\n\n" ); let mut terminal = chunk("", true); let choice = &mut terminal.data.as_mut().unwrap().inner.choices[0]; choice.delta.content = None; choice.delta.tool_calls = Some(vec![ChatCompletionMessageToolCallChunk { custom: None, index: 0, id: Some("call-complete".to_string()), r#type: Some(FunctionType::Function), function: Some(FunctionCallStream { name: Some("upstream".to_string()), arguments: Some("{\"city\":\"Paris\"}".to_string()), }), }]); let responses = apply_stream( stream::iter([chunk(raw_call, false), terminal]), Some(weather_tools()), None, false, UnifiedParserStartingState::None, QWEN3_UNIFIED_FAMILY, ) .collect::>() .await; let choices = collect_choices(&responses); let tool_call_chunks: Vec<_> = choices .iter() .filter_map(|choice| choice.delta.tool_calls.as_ref()) .flatten() .collect(); assert_eq!(tool_call_chunks.len(), 2); assert_eq!(tool_call_chunks[0].index, 0); assert!(tool_call_chunks[0].id.is_some()); assert_eq!(tool_call_chunks[1].index, 1); assert_eq!(tool_call_chunks[1].id.as_deref(), Some("call-complete")); assert_eq!( tool_call_chunks[1] .function .as_ref() .and_then(|function| function.name.as_deref()), Some("upstream") ); let finish_reasons: Vec<_> = choices .iter() .filter_map(|choice| choice.finish_reason) .collect(); assert_eq!( finish_reasons, vec![FinishReason::ToolCalls], "the already-parsed completion must clear the matching raw provisional call" ); } #[tokio::test] async fn finish_recovery_during_reasoning_detour_clears_pending_tool_call() { let complete_json_without_call_end = concat!( "<|tool_calls_section_begin|><|tool_call_begin|>", "functions.get_weather:0<|tool_call_argument_begin|>", r#"{"city":"Paris"}"# ); for terminal in [false, true] { let mut parsed_reasoning = chunk("", terminal); let choice = &mut parsed_reasoning.data.as_mut().unwrap().inner.choices[0]; choice.delta.content = None; choice.delta.reasoning_content = Some("already parsed reasoning".to_string()); let responses = apply_stream( stream::iter([ chunk(complete_json_without_call_end, false), parsed_reasoning, ]), Some(weather_tools()), None, false, UnifiedParserStartingState::None, "kimi_k2", ) .collect::>() .await; let choices = collect_choices(&responses); assert!( choices .iter() .any(|choice| choice.delta.tool_calls.is_some()), "finish must recover the valid JSON body when Kimi omits the call-end marker" ); assert_eq!( choices .iter() .filter_map(|choice| choice.finish_reason) .collect::>(), vec![FinishReason::ToolCalls], "finish-recovered calls must clear the pending index before EOF (terminal={terminal})" ); } } #[tokio::test] async fn raw_provisional_call_downgrades_already_parsed_tool_calls_terminal() { let mut parsed_terminal = chunk("", true); let choice = &mut parsed_terminal.data.as_mut().unwrap().inner.choices[0]; choice.delta.content = None; choice.delta.reasoning_content = Some("already parsed reasoning".to_string()); choice.finish_reason = Some(FinishReason::ToolCalls); let responses = apply_stream( stream::iter([ chunk( "Paris", false, ), parsed_terminal, ]), Some(weather_tools()), None, false, UnifiedParserStartingState::Response, QWEN3_UNIFIED_FAMILY, ) .collect::>() .await; let choices = collect_choices(&responses); assert!( choices .iter() .any(|choice| choice.delta.tool_calls.is_some()) ); assert_eq!( choices .iter() .filter_map(|choice| choice.finish_reason) .collect::>(), vec![FinishReason::Length] ); } #[tokio::test] async fn qwen_call_start_marker_inside_parameter_value_survives_reasoning_detour_at_every_split() { async fn run( split: usize, continuation: &str, ) -> ( String, String, bool, String, Vec, Vec, usize, bool, ) { let mut reasoning_detour = chunk("", false); let choice = &mut reasoning_detour.data.as_mut().unwrap().inner.choices[0]; choice.delta.content = None; choice.delta.reasoning_content = Some("interleaved thought".to_string()); let responses = apply_stream( stream::iter([ chunk("Pa", false), reasoning_detour, chunk(&continuation[..split], false), chunk(&continuation[split..], true), ]), Some(weather_tools()), None, false, UnifiedParserStartingState::None, QWEN3_UNIFIED_FAMILY, ) .collect::>() .await; let choices = collect_choices(&responses); let calls: Vec<_> = choices .iter() .flat_map(|choice| choice.delta.tool_calls.iter().flatten()) .collect(); let arguments = calls .iter() .filter_map(|call| call.function.as_ref()?.arguments.as_deref()) .collect(); let name = calls .iter() .filter_map(|call| call.function.as_ref()?.name.as_deref()) .collect(); let reasoning = choices.iter().any(|choice| { choice.delta.reasoning_content.as_deref() == Some("interleaved thought") }); let content = choices .iter() .filter_map(|choice| match &choice.delta.content { Some(ChatCompletionMessageContent::Text(text)) => Some(text.as_str()), _ => None, }) .collect(); let finish_reasons = choices .iter() .filter_map(|choice| choice.finish_reason) .collect(); let mut indices: Vec<_> = calls.iter().map(|call| call.index).collect(); indices.sort_unstable(); indices.dedup(); let call_ids: HashSet<_> = calls.iter().filter_map(|call| call.id.as_deref()).collect(); let reasoning_position = choices .iter() .position(|choice| choice.delta.reasoning_content.is_some()) .expect("the reasoning detour must be emitted"); let streamed_before_reasoning = choices[..reasoning_position].iter().any(|choice| { choice.delta.tool_calls.as_ref().is_some_and(|calls| { calls.iter().any(|call| { call.function .as_ref() .and_then(|function| function.arguments.as_deref()) .is_some_and(|arguments| !arguments.is_empty()) }) }) }); ( arguments, name, reasoning, content, finish_reasons, indices, call_ids.len(), streamed_before_reasoning, ) } let continuation = "after"; let whole = run(continuation.len(), continuation).await; assert_eq!(whole.0, r#"{"city":"Paafter"}"#); assert_eq!(whole.1, "get_weather"); assert!(whole.2, "the reasoning detour must stay visible"); assert_eq!(whole.3, "", "native markup must not leak into content"); assert_eq!(whole.4, vec![FinishReason::ToolCalls]); assert_eq!(whole.5, vec![0]); assert_eq!(whole.6, 1, "exactly one call ID must be emitted"); assert!(whole.7, "argument bytes must stream before the detour"); for split in 0..=continuation.len() { assert_eq!( run(split, continuation).await, whole, "splitting the continuation at byte {split} changed the result" ); } } #[tokio::test] async fn kimi_call_start_marker_inside_json_string_survives_reasoning_detour_at_every_split() { async fn run( split: usize, continuation: &str, ) -> (String, bool, String, Vec) { let mut reasoning_detour = chunk("", false); let choice = &mut reasoning_detour.data.as_mut().unwrap().inner.choices[0]; choice.delta.content = None; choice.delta.reasoning_content = Some("interleaved thought".to_string()); let responses = apply_stream( stream::iter([ chunk( "<|tool_calls_section_begin|><|tool_call_begin|>functions.get_weather:0<|tool_call_argument_begin|>{\"city\":\"Pa", false, ), reasoning_detour, chunk(&continuation[..split], false), chunk(&continuation[split..], true), ]), Some(weather_tools()), None, false, UnifiedParserStartingState::None, "kimi_k2", ) .collect::>() .await; let choices = collect_choices(&responses); let arguments = choices .iter() .filter_map(|choice| choice.delta.tool_calls.as_ref()) .flatten() .filter_map(|call| call.function.as_ref()?.arguments.as_deref()) .collect(); let reasoning = choices.iter().any(|choice| { choice.delta.reasoning_content.as_deref() == Some("interleaved thought") }); let content = choices .iter() .filter_map(|choice| match &choice.delta.content { Some(ChatCompletionMessageContent::Text(text)) => Some(text.as_str()), _ => None, }) .collect(); let finish_reasons = choices .iter() .filter_map(|choice| choice.finish_reason) .collect(); (arguments, reasoning, content, finish_reasons) } let continuation = "<|tool_call_begin|>after\"}<|tool_call_end|><|tool_calls_section_end|>"; let whole = run(continuation.len(), continuation).await; assert_eq!(whole.0, "{\"city\":\"Pa<|tool_call_begin|>after\"}"); assert!(whole.1, "the reasoning detour must stay visible"); assert_eq!(whole.2, "", "no native markup may leak into content"); assert_eq!(whole.3, vec![FinishReason::ToolCalls]); for split in 0..=continuation.len() { assert_eq!( run(split, continuation).await, whole, "splitting the continuation at byte {split} changed the result" ); } } #[tokio::test] async fn nonterminal_already_parsed_tool_fragment_does_not_complete_pending_call() { let first_partial = concat!( "\n\n", "Paris" ); let mut parsed_fragment = chunk("", false); let choice = &mut parsed_fragment.data.as_mut().unwrap().inner.choices[0]; choice.delta.content = None; choice.delta.tool_calls = Some(vec![ChatCompletionMessageToolCallChunk { custom: None, index: 0, id: Some("call-parsed".to_string()), r#type: Some(FunctionType::Function), function: Some(FunctionCallStream { name: Some("get_weather".to_string()), arguments: Some(r#"{"city":"Paris""#.to_string()), }), }]); let responses = apply_stream( stream::iter([chunk(first_partial, false), parsed_fragment]), Some(weather_tools()), None, false, UnifiedParserStartingState::None, QWEN3_UNIFIED_FAMILY, ) .collect::>() .await; let choices = collect_choices(&responses); assert!( choices .iter() .any(|choice| choice.delta.tool_calls.is_some()), "both the raw and already-parsed partial fragments should be emitted" ); assert!( choices .iter() .filter_map(|choice| choice.finish_reason) .collect::>() == vec![FinishReason::Length], "nonterminal parsed argument fragments must not synthesize a completed-call terminal" ); } #[tokio::test] async fn qwen_queued_reasoning_is_emitted_once_when_parsed_tool_call_interrupts() { #[derive(Debug, PartialEq)] enum OutputEvent { Content(String), Reasoning(String), ToolCall { index: u32, id: Option, name: Option, arguments: Option, }, Finish(FinishReason), } for terminal in [false, true] { let mut reasoning = chunk("", false); let choice = &mut reasoning.data.as_mut().unwrap().inner.choices[0]; choice.delta.content = None; choice.delta.reasoning_content = Some("held thought".to_string()); let source = reasoning.data.as_mut().unwrap(); source.nvext = Some(serde_json::json!({"source": "reasoning"})); source.llm_metrics = Some(crate::protocols::common::metrics::LLMMetricAnnotation { chunk_tokens: 7, ..Default::default() }); reasoning.id = Some("reasoning-source-id".to_string()); reasoning.event = Some("reasoning-source-event".to_string()); reasoning.comment = Some(vec!["reasoning-source-comment".to_string()]); let mut parsed_call = chunk("", terminal); let choice = &mut parsed_call.data.as_mut().unwrap().inner.choices[0]; choice.delta.content = None; choice.delta.tool_calls = Some(vec![ChatCompletionMessageToolCallChunk { custom: None, index: 0, id: Some("parsed-call".to_string()), r#type: Some(FunctionType::Function), function: Some(FunctionCallStream { name: Some("get_weather".to_string()), arguments: Some(r#"{"city":"NY"}"#.to_string()), }), }]); let responses = apply_stream( stream::iter([chunk("before >() .await; let choices = collect_choices(&responses); let observed: Vec<_> = choices .iter() .flat_map(|choice| { let mut events = Vec::new(); if let Some(ChatCompletionMessageContent::Text(text)) = &choice.delta.content { events.push(OutputEvent::Content(text.clone())); } if let Some(reasoning) = &choice.delta.reasoning_content { events.push(OutputEvent::Reasoning(reasoning.clone())); } if let Some(calls) = &choice.delta.tool_calls { events.extend(calls.iter().map(|call| { let function = call.function.as_ref(); OutputEvent::ToolCall { index: call.index, id: call.id.clone(), name: function.and_then(|function| function.name.clone()), arguments: function.and_then(|function| function.arguments.clone()), } })); } if let Some(reason) = &choice.finish_reason { events.push(OutputEvent::Finish(*reason)); } events }) .collect(); let mut expected = vec![ OutputEvent::Content("before ".to_string()), OutputEvent::Content(" = responses .iter() .filter(|response| { response.data.as_ref().is_some_and(|data| { data.inner.choices.iter().any(|choice| { choice.delta.reasoning_content.as_deref() == Some("held thought") }) }) }) .collect(); assert_eq!(source_reasoning.len(), 1); assert_eq!( source_reasoning[0].id.as_deref(), Some("reasoning-source-id") ); assert_eq!( source_reasoning[0].event.as_deref(), Some("reasoning-source-event") ); assert_eq!( source_reasoning[0].comment.as_deref(), Some(["reasoning-source-comment".to_string()].as_slice()) ); let source_data = source_reasoning[0].data.as_ref().unwrap(); assert_eq!( source_data.nvext, Some(serde_json::json!({"source": "reasoning"})) ); assert_eq!(source_data.llm_metrics.as_ref().unwrap().chunk_tokens, 7); } } #[tokio::test] async fn qwen_multi_choice_queued_reasoning_emits_source_metadata_once() { fn choice(index: u32, content: Option<&str>, reasoning: Option<&str>) -> ChatChoiceStream { ChatChoiceStream { index, delta: ChatCompletionStreamResponseDelta { role: Some(Role::Assistant), content: content .map(|text| ChatCompletionMessageContent::Text(text.to_string())), tool_calls: None, function_call: None, refusal: None, reasoning_content: reasoning.map(str::to_string), }, finish_reason: None, logprobs: None, } } for (reasoning_zero, reasoning_one) in [ ("thought zero", "thought one"), ("", ""), ("", "thought one"), ("thought zero", ""), ] { let mut opening = chunk("", false); opening.data.as_mut().unwrap().inner.choices = vec![ choice(0, Some("before \n\nNY\n\n", ), None, ), choice( 1, Some( "all>\n\nSF\n\n", ), None, ), ]; let responses = apply_stream( stream::iter([opening, reasoning, continuation]), Some(weather_tools()), None, false, UnifiedParserStartingState::None, QWEN3_UNIFIED_FAMILY, ) .collect::>() .await; let choices = collect_choices(&responses); for (index, expected) in [(0, reasoning_zero), (1, reasoning_one)] { assert_eq!( choices .iter() .filter(|choice| { choice.index == index && choice.delta.reasoning_content.as_deref() == Some(expected) }) .count(), 1, "choice {index} reasoning must remain attached to the correct choice once" ); } assert_eq!( responses .iter() .filter(|response| response.id.as_deref() == Some("shared-reasoning-id")) .count(), 1, "the source response id must be emitted once across the two queued choices" ); assert_eq!( responses .iter() .filter(|response| response.event.as_deref() == Some("shared-reasoning-event")) .count(), 1 ); assert_eq!( responses .iter() .filter(|response| response.comment.is_some()) .count(), 1 ); assert_eq!( responses .iter() .filter_map(|response| response.data.as_ref()) .filter(|data| data.nvext.is_some()) .count(), 1 ); assert_eq!( responses .iter() .filter_map(|response| response.data.as_ref()) .filter(|data| data.llm_metrics.is_some()) .count(), 1 ); } } #[tokio::test] async fn separate_terminal_chunk_keeps_malformed_parsed_fragments_provisional() { for tool_calls_finish in [false, true] { let mut parsed_fragments = chunk("", false); let choice = &mut parsed_fragments.data.as_mut().unwrap().inner.choices[0]; choice.delta.content = None; choice.delta.tool_calls = Some(vec![ ChatCompletionMessageToolCallChunk { custom: None, index: 0, id: Some("call-first".to_string()), r#type: Some(FunctionType::Function), function: Some(FunctionCallStream { name: Some("get_weather".to_string()), arguments: Some(r#"{"city":"Par"#.to_string()), }), }, ChatCompletionMessageToolCallChunk { custom: None, index: 1, id: Some("call-second".to_string()), r#type: Some(FunctionType::Function), function: Some(FunctionCallStream { name: Some("get_weather".to_string()), arguments: Some(r#"{"city":"Tok"#.to_string()), }), }, ]); let mut terminal = chunk("", false); let choice = &mut terminal.data.as_mut().unwrap().inner.choices[0]; choice.delta.content = None; choice.delta.reasoning_content = Some("terminal reasoning".to_string()); choice.finish_reason = Some(if tool_calls_finish { FinishReason::ToolCalls } else { FinishReason::Stop }); let responses = apply_stream( stream::iter([parsed_fragments, terminal]), Some(weather_tools()), None, false, UnifiedParserStartingState::None, QWEN3_UNIFIED_FAMILY, ) .collect::>() .await; let choices = collect_choices(&responses); assert_eq!( choices .iter() .filter_map(|choice| choice.finish_reason) .collect::>(), vec![FinishReason::Length], "a terminal reason must not complete malformed parsed fragments (tool_calls_finish={tool_calls_finish})" ); let emitted_arguments: Vec<_> = choices .iter() .flat_map(|choice| choice.delta.tool_calls.iter().flatten()) .filter_map(|call| call.function.as_ref()?.arguments.as_deref()) .collect(); assert_eq!( emitted_arguments, vec![r#"{"city":"Par"#, r#"{"city":"Tok"#], "malformed fragments remain identifiable but receive no successful terminal" ); } } #[tokio::test] async fn terminal_argument_fragments_are_validated_after_assembly_per_choice() { let fragments = chunk("raw prelude", false); let mut partials = chunk("", false); let first = &mut partials.data.as_mut().unwrap().inner.choices[0]; first.delta.content = None; first.delta.tool_calls = Some(vec![ ChatCompletionMessageToolCallChunk { custom: None, index: 0, id: Some("call-invalid".to_string()), r#type: Some(FunctionType::Function), function: Some(FunctionCallStream { name: Some("get_weather".to_string()), arguments: Some(r#"{"city":"#.to_string()), }), }, ChatCompletionMessageToolCallChunk { custom: None, index: 1, id: Some("call-valid".to_string()), r#type: Some(FunctionType::Function), function: Some(FunctionCallStream { name: Some("get_weather".to_string()), arguments: Some(r#"{"city":"#.to_string()), }), }, ]); let mut second = first.clone(); second.index = 1; second.delta.tool_calls = Some(vec![ChatCompletionMessageToolCallChunk { custom: None, index: 0, id: Some("call-parameterless".to_string()), r#type: Some(FunctionType::Function), function: Some(FunctionCallStream { name: Some("get_weather".to_string()), arguments: Some(String::new()), }), }]); partials.data.as_mut().unwrap().inner.choices.push(second); let mut terminal = chunk("", false); let first = &mut terminal.data.as_mut().unwrap().inner.choices[0]; first.delta.content = None; first.finish_reason = Some(FinishReason::ToolCalls); first.delta.tool_calls = Some(vec![ ChatCompletionMessageToolCallChunk { custom: None, index: 0, id: None, r#type: None, function: Some(FunctionCallStream { name: None, arguments: Some("not-json".to_string()), }), }, ChatCompletionMessageToolCallChunk { custom: None, index: 1, id: None, r#type: None, function: Some(FunctionCallStream { name: None, arguments: Some(r#""Tokyo"}"#.to_string()), }), }, ]); let mut second = first.clone(); second.index = 1; second.delta.tool_calls = Some(vec![ChatCompletionMessageToolCallChunk { custom: None, index: 0, id: None, r#type: None, function: Some(FunctionCallStream { name: None, arguments: Some(String::new()), }), }]); terminal.data.as_mut().unwrap().inner.choices.push(second); let responses = apply_stream( stream::iter([fragments, partials, terminal]), None, None, false, UnifiedParserStartingState::None, QWEN3_UNIFIED_FAMILY, ) .collect::>() .await; let choices = collect_choices(&responses); let finishes: HashMap<_, _> = choices .iter() .filter_map(|choice| choice.finish_reason.map(|reason| (choice.index, reason))) .collect(); assert_eq!(finishes.get(&0), Some(&FinishReason::Length)); assert_eq!(finishes.get(&1), Some(&FinishReason::ToolCalls)); let calls: HashMap<_, _> = choices .iter() .flat_map(|choice| { choice .delta .tool_calls .iter() .flatten() .filter_map(move |call| { call.id .as_deref() .map(|id| ((choice.index, call.index as usize), id)) }) }) .collect(); assert_eq!(calls.get(&(0, 0)), Some(&"call-invalid")); assert_eq!(calls.get(&(0, 1)), Some(&"call-valid")); assert_eq!(calls.get(&(1, 0)), Some(&"call-parameterless")); } #[tokio::test] async fn already_parsed_first_terminal_chunk_normalizes_stop() { let mut already_parsed = chunk("", true); let choice = &mut already_parsed.data.as_mut().unwrap().inner.choices[0]; choice.delta.content = None; choice.delta.tool_calls = Some(vec![ChatCompletionMessageToolCallChunk { custom: None, index: 0, id: Some("call-preexisting".to_string()), r#type: Some(FunctionType::Function), function: Some(FunctionCallStream { name: Some("get_weather".to_string()), arguments: Some("{}".to_string()), }), }]); // This choice index never has a raw chunk at all — no ChoiceState is ever // constructed for it — but the parsed chunk carries the terminal Stop signal. let responses = apply_stream( stream::iter([already_parsed]), Some(weather_tools()), None, false, UnifiedParserStartingState::None, QWEN3_UNIFIED_FAMILY, ) .collect::>() .await; let choices = collect_choices(&responses); assert_eq!( choices .iter() .filter_map(|choice| choice.finish_reason) .collect::>(), vec![FinishReason::ToolCalls], "an already-parsed tool call on the terminal chunk must normalize Stop to ToolCalls" ); } #[tokio::test] async fn multi_choice_interleave_finishes_each_unfinished_index_once() { fn choice(index: u32, text: &str, finish: bool) -> ChatChoiceStream { ChatChoiceStream { index, delta: ChatCompletionStreamResponseDelta { role: Some(Role::Assistant), content: Some(ChatCompletionMessageContent::Text(text.to_string())), tool_calls: None, function_call: None, refusal: None, reasoning_content: None, }, finish_reason: finish.then_some(FinishReason::Stop), logprobs: None, } } fn response( choices: Vec, ) -> Annotated { #[allow(deprecated)] let response = NvCreateChatCompletionStreamResponse { message_boundary: None, inner: CreateChatCompletionStreamResponse { id: "test".to_string(), choices, created: 0, model: "qwen3".to_string(), system_fingerprint: None, service_tier: None, object: "chat.completion.chunk".to_string(), usage: None, }, nvext: None, llm_metrics: None, prompt_logprobs: None, tool_call_completion: Vec::new(), model_eos: Vec::new(), }; Annotated::from_data(response) } let tool_call = concat!( "\n\n", "Tokyo\n\n" ); // Index 0: live the whole time, never removed, never explicitly finished — // the ordinary backstop case. Index 1: raw tool call, then a gap that removes // its state and is never touched again — history-only at EOF. Index 2: raw // tool call, explicitly finished right away, never touched again. let first = response(vec![ choice(0, tool_call, false), choice(1, tool_call, false), choice(2, tool_call, false), ]); let mut second = response(vec![choice(0, " more", false), choice(2, " done", true)]); { let mut gap = choice(1, "", false); gap.delta.content = None; gap.delta.reasoning_content = Some("already parsed".to_string()); second.data.as_mut().unwrap().inner.choices.push(gap); } // Index 1 gets no further chunks after this — it must stay history-only // through to EOF for this to actually exercise the bug. let responses = apply_stream( stream::iter([first, second]), Some(weather_tools()), None, false, UnifiedParserStartingState::None, QWEN3_UNIFIED_FAMILY, ) .collect::>() .await; let choices = collect_choices(&responses); let finish_for = |index: u32| { choices .iter() .filter(|choice| choice.index == index) .filter_map(|choice| choice.finish_reason) .collect::>() }; assert_eq!( finish_for(0), vec![FinishReason::ToolCalls], "the ordinary live choice must get exactly one terminal via the backstop" ); assert_eq!( finish_for(1), vec![FinishReason::ToolCalls], "the history-only choice must get exactly one terminal, not zero" ); assert_eq!( finish_for(2), vec![FinishReason::ToolCalls], "the already-explicitly-finished choice must not be finished a second \ time by the EOF backstop" ); } #[tokio::test] async fn finished_choices_keep_other_choices_and_late_metadata() { use crate::protocols::common::metrics::LLMMetricAnnotation; let mut first = chunk("A", true); let mut active = empty_choice(1); active.delta.content = Some(ChatCompletionMessageContent::Text("B".into())); first.data.as_mut().unwrap().inner.choices.push(active); let mut mixed = chunk("late", false); let mut terminal = empty_choice(1); terminal.delta.content = Some(ChatCompletionMessageContent::Text("C".into())); terminal.delta.reasoning_content = Some("r".into()); terminal.finish_reason = Some(FinishReason::Stop); mixed.data.as_mut().unwrap().inner.choices.push(terminal); let mut late = chunk("late raw payload", false); late.id = Some("late-metadata".into()); let data = late.data.as_mut().unwrap(); let mut parsed = empty_choice(1); parsed.delta.reasoning_content = Some("late parsed payload".into()); parsed.finish_reason = Some(FinishReason::Stop); data.inner.choices.push(parsed); data.inner.usage = Some( serde_json::from_value(serde_json::json!({ "prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3 })) .unwrap(), ); data.nvext = Some(serde_json::json!({"completion_token_ids": [42]})); data.llm_metrics = Some(LLMMetricAnnotation { chunk_tokens: 2, ..Default::default() }); let responses: Vec<_> = apply_stream( stream::iter([first, mixed, late]), None, None, false, UnifiedParserStartingState::None, QWEN3_UNIFIED_FAMILY, ) .collect() .await; let payloads: Vec<_> = collect_choices(&responses) .into_iter() .map(|choice| { let text = match &choice.delta.content { Some(ChatCompletionMessageContent::Text(text)) => Some(text.as_str()), _ => None, }; ( choice.index, text, choice.delta.reasoning_content.as_deref(), choice.finish_reason, ) }) .collect(); assert_eq!( payloads, [ (0, Some("A"), None, Some(FinishReason::Stop)), (1, Some("B"), None, None), (1, Some("C"), Some("r"), Some(FinishReason::Stop)), ] ); assert_eq!( responses .iter() .filter_map(|response| response.data.as_ref()) .filter(|data| data.inner.usage.is_some()) .count(), 1 ); let tail = responses.last().unwrap(); assert_eq!(tail.id.as_deref(), Some("late-metadata")); let metadata = tail.data.as_ref().unwrap(); assert!(metadata.inner.choices.is_empty()); assert_eq!(metadata.inner.usage.as_ref().unwrap().total_tokens, 3); assert_eq!( metadata.nvext, Some(serde_json::json!({"completion_token_ids": [42]})) ); assert_eq!(metadata.llm_metrics.as_ref().unwrap().chunk_tokens, 2); } #[tokio::test] async fn three_consecutive_already_parsed_gaps_then_eof_finishes_once() { let tool_call = concat!( "\n\n", "Tokyo\n\n" ); let already_parsed_chunk = |reasoning: &str| { let mut c = chunk("", false); let choice = &mut c.data.as_mut().unwrap().inner.choices[0]; choice.delta.content = None; choice.delta.reasoning_content = Some(reasoning.to_string()); c }; // Three already-parsed chunks in a row, no raw chunk between them, no // explicit terminal ever, then the stream just ends. let responses = apply_stream( stream::iter([ chunk(tool_call, false), already_parsed_chunk("first"), already_parsed_chunk("second"), already_parsed_chunk("third"), ]), Some(weather_tools()), None, false, UnifiedParserStartingState::None, QWEN3_UNIFIED_FAMILY, ) .collect::>() .await; let choices = collect_choices(&responses); assert_eq!( choices .iter() .filter_map(|choice| choice.finish_reason) .collect::>(), vec![FinishReason::ToolCalls], "N consecutive already-parsed gaps with no explicit terminal must still \ produce exactly one ToolCalls terminal at EOF, not zero and not N" ); } #[tokio::test] async fn raw_gap_resume_second_gap_then_eof_finishes_once_in_order() { // Distinct from `three_consecutive_already_parsed_gaps_then_eof_finishes_once` // (which never resumes with a raw chunk in between): this drives raw -> gap -> // raw-resumed (a fresh `ChoiceState`, completed-call state reseeded from the // `ChoiceRecord`) -> a SECOND gap on that resumed instance -> EOF, so the // fold-into-record step at removal runs twice on two different `ChoiceState` // instances for the same choice index. let tool_call = concat!( "\n\n", "Tokyo\n\n" ); let already_parsed_chunk = |reasoning: &str| { let mut c = chunk("", false); let choice = &mut c.data.as_mut().unwrap().inner.choices[0]; choice.delta.content = None; choice.delta.reasoning_content = Some(reasoning.to_string()); c }; let responses = apply_stream( stream::iter([ chunk(tool_call, false), already_parsed_chunk("first gap"), chunk(" resumed text", false), already_parsed_chunk("second gap"), ]), Some(weather_tools()), None, false, UnifiedParserStartingState::None, QWEN3_UNIFIED_FAMILY, ) .collect::>() .await; let choices = collect_choices(&responses); let ordered: Vec<_> = choices .iter() .filter_map(|choice| { if choice.delta.tool_calls.is_some() { Some("tool_call") } else if let Some(ChatCompletionMessageContent::Text(text)) = &choice.delta.content { if text.is_empty() { None } else { Some("content") } } else { choice .delta .reasoning_content .as_deref() .map(|_| "reasoning") } }) .collect(); assert_eq!( ordered, vec!["tool_call", "reasoning", "content", "reasoning"], "delivery order across the raw/gap/resume/gap sequence must be preserved: \ tool call, first gap's reasoning, resumed content, second gap's reasoning" ); assert_eq!( choices .iter() .filter_map(|choice| choice.finish_reason) .collect::>(), vec![FinishReason::ToolCalls], "a second gap on a resumed (fresh) ChoiceState must still fold into the \ same choice's record and produce exactly one ToolCalls terminal at EOF, \ not zero and not two" ); } #[tokio::test] async fn text_only_stream_gets_no_synthetic_finish_reason() { let responses = apply_stream( stream::iter([chunk("hello world", false)]), None, None, false, UnifiedParserStartingState::None, QWEN3_UNIFIED_FAMILY, ) .collect::>() .await; let choices = collect_choices(&responses); assert!( choices.iter().all(|choice| choice.finish_reason.is_none()), "there is no signal to synthesize a finish_reason from" ); } // --- error recovery ---------------------------------------------------------- #[test] fn a_failed_parser_surfaces_buffered_text_and_stops_parsing() { let mut state = test_state(); // The parser releases the settled text immediately and holds back only the // bytes that could still turn out to be a `` opener. assert_eq!( state.push("hello ".to_string(), }), tool_output_mode: UnifiedToolOutputMode::Native, opened_calls: HashSet::new(), tool_ids_emitted: HashSet::new(), tool_types_emitted: HashSet::new(), parser_tool_indices: HashMap::new(), passthrough_tool_indices: HashMap::new(), tool_index_offset: 0, next_tool_index: 0, pending_tool_calls: HashSet::new(), parser_call_completion: HashMap::new(), pending_parsed_tool_calls: HashMap::new(), tool_emitted: false, failed: false, parse_error: None, }; assert_eq!( state.push("ignored"), vec![ UnifiedParserEvent::Text("committed".to_string()), UnifiedParserEvent::Text("".to_string()), ] ); } #[tokio::test] async fn terminal_error_suppresses_eof_recovery() { let responses = apply_stream( stream::iter([ chunk("hello >() .await; let error_position = responses .iter() .position(Annotated::is_error) .expect("terminal error"); assert!( responses[error_position + 1..] .iter() .all(|response| response.data.is_none()), "no parser data may be emitted after a terminal error" ); } #[test] fn give_up_falls_back_to_the_chunk_when_nothing_was_buffered() { let mut state = test_state(); assert_eq!( state.give_up("the chunk that broke it"), vec![UnifiedParserEvent::Text("the chunk that broke it".into())] ); } // --- batch ------------------------------------------------------------------- #[test] fn parses_complete_output_with_reasoning_and_a_tool_call() { let parsed = parse_complete( QWEN3_UNIFIED_FAMILY, concat!( "reasonanswer ", "\n\n", "Tokyo\n\n" ), &GuidedToolConstraint::None, &[], ) .unwrap(); assert_eq!(parsed.reasoning, "reason"); assert_eq!(parsed.text, "answer "); assert_eq!(parsed.tool_calls.len(), 1); assert_eq!(parsed.tool_calls[0].function.name, "get_weather"); assert_eq!( parsed.tool_calls[0].function.arguments, r#"{"city":"Tokyo"}"# ); assert!(parsed.tool_calls[0].id.starts_with("call-")); } #[test] fn parses_complete_output_with_a_reasoning_prefill() { let parsed = parse_complete( QWEN3_UNIFIED_FAMILY, "hiddenvisible", &GuidedToolConstraint::None, &[], ) .unwrap(); assert_eq!(parsed.reasoning, "hidden"); assert_eq!(parsed.text, "visible"); assert!(parsed.tool_calls.is_empty()); } #[test] fn unmatched_quote_before_real_reasoning_close_does_not_expose_reasoning() { let parsed = parse_complete( QWEN3_UNIFIED_FAMILY, r#"hidden "unfinishedvisible"#, &GuidedToolConstraint::None, &[], ) .unwrap(); assert_eq!(parsed.reasoning, r#"hidden "unfinished"#); assert_eq!(parsed.text, "visible"); } #[test] fn unmatched_single_quote_before_real_reasoning_close_does_not_expose_reasoning() { // Leading with a space keeps this a genuine quote-open candidate rather // than the in-word apostrophe exception (e.g. "it's"). let parsed = parse_complete( QWEN3_UNIFIED_FAMILY, "hidden 'unfinishedvisible", &GuidedToolConstraint::None, &[], ) .unwrap(); assert_eq!(parsed.reasoning, "hidden 'unfinished"); assert_eq!(parsed.text, "visible"); } #[test] fn unmatched_backtick_before_real_reasoning_close_does_not_expose_reasoning() { let parsed = parse_complete( QWEN3_UNIFIED_FAMILY, "hidden `unfinishedvisible", &GuidedToolConstraint::None, &[], ) .unwrap(); assert_eq!(parsed.reasoning, "hidden `unfinished"); assert_eq!(parsed.text, "visible"); } #[tokio::test] async fn unmatched_quote_does_not_hide_real_reasoning_close_at_any_stream_split() { let raw = r#"I considered "Paris.The answer is Paris."#; let batch = parse_complete(QWEN3_UNIFIED_FAMILY, raw, &GuidedToolConstraint::None, &[]).unwrap(); assert_eq!(batch.reasoning, r#"I considered "Paris."#); assert_eq!(batch.text, "The answer is Paris."); for split in 0..=raw.len() { let responses = apply_stream( stream::iter([chunk(&raw[..split], false), chunk(&raw[split..], true)]), None, None, false, UnifiedParserStartingState::None, QWEN3_UNIFIED_FAMILY, ) .collect::>() .await; let choices = collect_choices(&responses); let reasoning = choices .iter() .filter_map(|choice| choice.delta.reasoning_content.as_deref()) .collect::(); let text = choices .iter() .filter_map(|choice| choice.delta.content.as_ref()) .filter_map(|content| match content { ChatCompletionMessageContent::Text(text) => Some(text.as_str()), _ => None, }) .collect::(); assert_eq!(reasoning, batch.reasoning, "split {split}"); assert_eq!(text, batch.text, "split {split}"); } } #[tokio::test] async fn structured_json_then_reasoning_close_matches_batch_and_streaming() { let policy = UnifiedRequestPolicy { structured_response: true, ..Default::default() }; let raw = r#"{"plan":"check"}{"answer":42}"#; let batch = parse_complete_with_policy( QWEN3_UNIFIED_FAMILY, raw, &GuidedToolConstraint::None, &[], policy, ) .unwrap(); assert_eq!(batch.reasoning, r#"{"plan":"check"}"#); assert_eq!(batch.text, r#"{"answer":42}"#); for split in 0..=raw.len() { let responses = apply_stream_with_policy( stream::iter([chunk(&raw[..split], false), chunk(&raw[split..], true)]), None, GuidedToolConstraint::None, UnifiedParserStartingState::Reasoning, QWEN3_UNIFIED_FAMILY, false, policy, ) .collect::>() .await; let choices = collect_choices(&responses); let reasoning = choices .iter() .filter_map(|choice| choice.delta.reasoning_content.as_deref()) .collect::(); let text = choices .iter() .filter_map(|choice| choice.delta.content.as_ref()) .filter_map(|content| match content { ChatCompletionMessageContent::Text(text) => Some(text.as_str()), _ => None, }) .collect::(); assert_eq!(reasoning, batch.reasoning, "split {split}"); assert_eq!(text, batch.text, "split {split}"); } } // TODO: Extend early-progress checks to literal native tool-call opener cases. #[test] fn quoted_marker_in_tool_arguments_releases_following_content_before_close() { assert_quoted_marker_tool_arguments(true); } #[test] fn quoted_marker_tool_arguments_preserve_completed_output_at_every_split() { assert_quoted_marker_tool_arguments(false); } fn assert_quoted_marker_tool_arguments(require_early_progress: bool) { let tools = vec![Tool { name: "write_file".to_string(), description: None, parameters: serde_json::json!({ "type": "object", "properties": { "path": {"type": "string"}, "content": {"type": "string"} }, "required": ["path", "content"] }), strict: None, }]; let body = "a".repeat(8192); for (mode, prefix, suffix) in [ ( UnifiedToolOutputMode::GuidedJson { named_tool: Some("write_file".to_string()), }, r#"{"path":"x","content":"prefix "#, r#" suffix"}"#, ), ( UnifiedToolOutputMode::GuidedJson { named_tool: None }, r#"[{"name":"write_file","arguments":{"path":"x","content":"prefix "#, r#" suffix"}}]"#, ), ( UnifiedToolOutputMode::Native, r#"x"prefix "#, r#" suffix""#, ), ] { let mut state = ChoiceState::new( QWEN3_UNIFIED_FAMILY, &tools, UnifiedParserStartingState::Response, mode.clone(), true, ArgumentFraming::default(), ) .unwrap(); let mut events = state.push(prefix); let progress = state.push(&body); let arguments: String = progress .iter() .filter_map(|event| match event { UnifiedParserEvent::ToolCall(call) => Some(call.arguments.as_str()), _ => None, }) .collect(); if require_early_progress { assert!( arguments.contains(&body), "{mode:?}: argument body must stream before its quote closes" ); } events.extend(progress); events.extend(state.push(suffix)); events.extend(state.finish()); let mut batch = create_unified_parser_for_family(QWEN3_UNIFIED_FAMILY, &tools).unwrap(); let init = UnifiedParserInit { starting_state: UnifiedParserStartingState::Response, tool_output_mode: mode.clone(), invalid_guided_payload: InvalidGuidedPayloadPolicy::StreamBestEffort, ..Default::default() }; batch.initialize_request(init.clone()).unwrap(); let raw = format!("{prefix}{body}{suffix}"); let completed = dynamo_parsers_v2::assemble(&events); assert_eq!( completed, vec![UnifiedEvent::ToolCall { name: "write_file".to_string(), arguments: serde_json::json!({ "path": "x", "content": if matches!(mode, UnifiedToolOutputMode::Native) { // XML parameter quotes are payload; JSON string quotes are syntax. format!("\"prefix {body} suffix\"") } else { format!("prefix {body} suffix") }, }), }], "{mode:?}: completed arguments must preserve every body byte" ); assert_eq!(completed, batch.parse_complete(&raw).unwrap()); let short_raw = format!(r#"{prefix}é \"quoted\" {suffix}"#); batch.reset(); batch.initialize_request(init).unwrap(); let expected = batch.parse_complete(&short_raw).unwrap(); for split in (0..=short_raw.len()).filter(|split| short_raw.is_char_boundary(*split)) { let mut split_state = ChoiceState::new( QWEN3_UNIFIED_FAMILY, &tools, UnifiedParserStartingState::Response, mode.clone(), true, ArgumentFraming::default(), ) .unwrap(); let mut split_events = split_state.push(&short_raw[..split]); split_events.extend(split_state.push(&short_raw[split..])); split_events.extend(split_state.finish()); assert_eq!( dynamo_parsers_v2::assemble(&split_events), expected, "{mode:?} split {split}" ); } } } #[test] fn repeated_quoted_markers_are_excluded_from_prefill_classification() { let literal = "The literal \"private\" stays visible. ".repeat(1024); let mut shield = QuotedControlMarkerProbe::new(QWEN3_UNIFIED_FAMILY); let mut masked = shield.push(&literal); masked.push_str(&shield.finish()); assert!(!masked.contains("")); assert_eq!(masked.len(), literal.len()); } #[test] fn contains_unquoted_marker_finds_real_marker_past_many_unmatched_escaped_quotes() { // Every `"` here is immediately preceded by an escaping backslash, so none of // them closes any other: each is individually a "does an unescaped close exist // later" candidate. `later_unescaped_quote_closes` answers all of these in one // O(n) pass; a naive per-candidate rescan (the pre-fix `has_unescaped_close` // called fresh per candidate) would instead be O(n^2) on input shaped like // this. This case is a deterministic correctness check, not a timing // assertion — see the stage evidence ledger for measured old-vs-new behavior. let unit = "a".repeat(50) + "\\\""; let filler: String = unit.repeat(4000); let content = format!("{filler}visible"); assert!( contains_unquoted_marker(&content, ""), "the real marker after {} chars of adversarial unmatched-escaped-quote \ filler must still be found", content.len() ); } async fn assert_quoted_literal_streams_as_text(family: &'static str, literal: &str) { for split in literal .char_indices() .map(|(offset, _)| offset) .chain(std::iter::once(literal.len())) { let responses = apply_stream( stream::iter([ chunk(&literal[..split], false), chunk(&literal[split..], false), ]), None, None, false, UnifiedParserStartingState::None, family, ) .collect::>() .await; let choices = collect_choices(&responses); let streamed = choices .iter() .filter_map(|choice| choice.delta.content.as_ref()) .filter_map(|content| match content { ChatCompletionMessageContent::Text(text) => Some(text.as_str()), _ => None, }) .collect::(); let reasoning = choices .iter() .filter_map(|choice| choice.delta.reasoning_content.as_deref()) .collect::(); assert_eq!(streamed, literal, "{family} split {split}"); assert!( reasoning.is_empty(), "{family} split {split}: {reasoning:?}" ); assert!( choices .iter() .all(|choice| choice.delta.tool_calls.is_none()), "{family} split {split}: {choices:?}" ); } } // TODO: Cover literal native tool-call opener text separately from reasoning controls. #[tokio::test] async fn quoted_control_markers_stay_visible_in_batch_and_streaming() { assert_quoted_control_markers(true, true).await; } #[tokio::test] async fn quoted_control_markers_stay_visible_in_batch() { assert_quoted_control_markers(false, false).await; } // TODO: Cover literal native tool-call opener text in native-channel batch responses. #[tokio::test] async fn native_channel_quoted_control_markers_stay_visible_in_batch() { assert_quoted_control_markers(false, true).await; } async fn assert_quoted_control_markers(check_streaming: bool, include_deferred: bool) { for family in [ "deepseek_v4", "deepseek_v41", "qwen3", "glm47", "kimi_k2", "gemma4", "kimi_k3", "muse_glimmer", ] { if matches!(family, "gemma4" | "kimi_k3" | "muse_glimmer") && !include_deferred { continue; } let (opener, closer) = match family { "kimi_k3" => ("<|open|>think<|sep|>", "<|close|>think<|sep|>"), "muse_glimmer" => ("<|start|>assistant to=self<|message|>", "<|eom|>"), _ => reasoning_marker_pair(family).unwrap(), }; for quote in ['"', '\'', '`'] { let literal = format!("{quote}The literal {opener}private{closer}{quote} stays visible."); let parsed = parse_complete(family, &literal, &GuidedToolConstraint::None, &[]).unwrap(); assert_eq!(parsed.text, literal, "{family} quote {quote}"); assert!( parsed.reasoning.is_empty(), "{family} quote {quote}: {:?}", parsed.reasoning ); if check_streaming { assert_quoted_literal_streams_as_text(family, &literal).await; } } } for quote in ['\'', '`'] { let literal = format!("The literal {quote}private{quote} stays visible."); let parsed = parse_complete( QWEN3_UNIFIED_FAMILY, &literal, &GuidedToolConstraint::None, &[], ) .unwrap(); assert_eq!(parsed.text, literal, "quote {quote}"); assert!(parsed.reasoning.is_empty(), "quote {quote}"); if check_streaming { assert_quoted_literal_streams_as_text(QWEN3_UNIFIED_FAMILY, &literal).await; } } } #[test] fn marker_looking_visible_text_stays_visible_in_batch() { let literal = "The literal \"\" closes reasoning."; let parsed = parse_complete( QWEN3_UNIFIED_FAMILY, literal, &GuidedToolConstraint::None, &[], ) .unwrap(); assert_eq!(parsed.text, literal); assert!(parsed.reasoning.is_empty()); let embedded = "The string \"a token\" stays visible."; let parsed = parse_complete( QWEN3_UNIFIED_FAMILY, embedded, &GuidedToolConstraint::None, &[], ) .unwrap(); assert_eq!(parsed.text, embedded); assert!(parsed.reasoning.is_empty()); } #[test] fn plain_text_passes_through_the_batch_path_unchanged() { let parsed = parse_complete( QWEN3_UNIFIED_FAMILY, "just an answer", &GuidedToolConstraint::None, &[], ) .unwrap(); assert_eq!(parsed.text, "just an answer"); assert!(parsed.reasoning.is_empty()); assert!(parsed.tool_calls.is_empty()); } // Regression for a batch/streaming argument-type divergence: `parse_complete` // used to hardcode an empty tool slice, so it had no schema to type-coerce // arguments against and always emitted them as JSON strings. The streaming // path (`apply_stream_with_constraint`) builds its parser from the request's // real `tool_definitions` and correctly coerces `count` to an integer for the // same input. Passing the real tool schema through here must match that. #[test] fn batch_path_coerces_argument_types_from_the_real_tool_schema() { let tool_definitions = vec![ToolDefinition { name: "set_count".to_string(), parameters: Some(serde_json::json!({ "type": "object", "properties": {"count": {"type": "integer"}}, "required": ["count"], })), strict: None, }]; let parsed = parse_complete( QWEN3_UNIFIED_FAMILY, concat!( "\n\n", "42\n\n" ), &GuidedToolConstraint::None, &tool_definitions, ) .unwrap(); assert_eq!( parsed.tool_calls[0].function.arguments, r#"{"count":42}"#, "batch path must type-coerce `count` to an integer using the real tool \ schema, matching what the streaming path already produces for the same \ input" ); } #[tokio::test] async fn qwen_reasoning_detours_preserve_native_grammar_at_every_utf8_split() { async fn observe( input: Vec>, tools: Vec, ) -> ( String, Vec<(u32, String, String)>, Vec, String, ) { let responses = apply_stream( stream::iter(input), Some(tools), None, false, UnifiedParserStartingState::None, QWEN3_UNIFIED_FAMILY, ) .collect::>() .await; let choices = collect_choices(&responses); let mut calls = std::collections::BTreeMap::::new(); let mut text = String::new(); let mut reasoning = String::new(); let mut finishes = Vec::new(); for choice in choices { if let Some(ChatCompletionMessageContent::Text(value)) = &choice.delta.content { text.push_str(value); } if let Some(value) = &choice.delta.reasoning_content { reasoning.push_str(value); } if let Some(value) = choice.finish_reason { finishes.push(value); } for call in choice.delta.tool_calls.iter().flatten() { let entry = calls.entry(call.index).or_default(); if let Some(function) = &call.function { if let Some(name) = &function.name { entry.0.push_str(name); } if let Some(arguments) = &function.arguments { entry.1.push_str(arguments); } } } } ( text, calls .into_iter() .map(|(index, (name, arguments))| (index, name, arguments)) .collect(), finishes, reasoning, ) } let cases = [ ( serde_json::json!({"type":"string"}), "prefix literal", ), ( serde_json::json!({"type":"string"}), "quote \\\" é literal", ), ( serde_json::json!({"type":"object"}), r#"{"a":"é literal"}"#, ), ( serde_json::json!({"type":"array"}), r#"["é ", {"a":true}]"#, ), (serde_json::json!({"type":"integer"}), "42"), (serde_json::json!({"$ref":"#/$defs/value"}), r#"{"a":"é"}"#), ( serde_json::json!({"type":["object","null"]}), r#"{"a":"é"}"#, ), (serde_json::json!({}), r#"{"a":"é"}"#), // Qwen closes parameters literally even inside JSON-looking text. // Pin its whole-input recovery rather than inventing a JSON boundary. ( serde_json::json!({"type":"object"}), r#"{"a":"literal"}"#, ), ]; for (schema, value) in cases { let tools = vec![ToolDefinition { name: "f".into(), parameters: Some( serde_json::json!({"type":"object","properties":{"x":schema},"$defs":{"value":{"type":"object"}}}), ), strict: None, }]; let input = format!( "{value}" ); let whole = observe(vec![chunk(&input, true)], tools.clone()).await; assert_eq!( whole.1.len(), 1, "whole-input control must emit f: {value:?}" ); assert_eq!(whole.1[0].1, "f"); if value.contains("") { assert_eq!(whole.1[0].2, r#"{"x":"{\"a\":\""}"#); } for split in (1..input.len()).filter(|at| input.is_char_boundary(*at)) { let mut detour = chunk("", false); let delta = &mut detour.data.as_mut().unwrap().inner.choices[0].delta; delta.content = None; delta.reasoning_content = Some("detour".into()); let actual = observe( vec![ chunk(&input[..split], false), detour, chunk(&input[split..], true), ], tools.clone(), ) .await; assert_eq!( (&actual.0, &actual.1, &actual.2), (&whole.0, &whole.1, &whole.2), "value={value:?} split={split}" ); assert_eq!( actual.3, "detour", "reasoning must survive once at split={split}" ); } } } #[tokio::test] async fn qwen_queued_reasoning_preserves_choice_envelope_on_resolution_and_eof() { for suffix in [ Some( "all>Paris", ), Some("at literal"), None, ] { for refusal in [None, Some("declined")] { let mut opening = chunk(">() .await; let choices = collect_choices(&responses); let reasoning: Vec<_> = choices .iter() .filter(|choice| choice.delta.reasoning_content.as_deref() == Some("detour")) .collect(); assert_eq!(reasoning.len(), 1, "suffix={suffix:?} refusal={refusal:?}"); assert_eq!( *reasoning[0], &expected, "queued or immediate reasoning must retain its untouched choice envelope" ); assert_eq!( choices .iter() .filter(|choice| choice.delta.role.is_some()) .count(), 1 ); assert_eq!( choices .iter() .filter(|choice| choice.logprobs.is_some()) .count(), 1 ); } } } /// Scores stay in generation order while raw text and reasoning alternate behind a /// split Qwen opener. #[tokio::test] async fn qwen_queued_reasoning_keeps_source_logprob_order() { fn scored( mut response: Annotated, token: &str, ) -> Annotated { response.data.as_mut().unwrap().inner.choices[0].logprobs = Some( serde_json::from_value(serde_json::json!({ "content": [{"token": token, "logprob": -0.5, "bytes": [], "top_logprobs": []}], "refusal": null })) .unwrap(), ); response } fn reasoning(text: &str, token: &str) -> Annotated { let mut source = scored(chunk("", false), token); let choice = &mut source.data.as_mut().unwrap().inner.choices[0]; choice.delta.content = None; choice.delta.reasoning_content = Some(text.to_string()); source } let completed_call = concat!( "ll>Paris", "" ); let responses = apply_stream( stream::iter(vec![ scored(chunk(">() .await; let tokens: Vec<_> = collect_choices(&responses) .into_iter() .filter_map(|choice| choice.logprobs.as_ref()?.content.as_ref()) .flatten() .map(|entry| entry.token.as_str()) .collect(); assert_eq!(tokens, ["A", "B", "C", "D", "E"]); } #[tokio::test] #[ignore = "Parser 0.7.17 missing-parameter-close recovery delays text across external reasoning; run explicitly to measure ordering"] async fn qwen_recovered_parameter_close_restores_opener_ordering_at_every_split() { for first in [ "Paris", "", ] { for split in 1..first.len() { let mut reasoning = chunk("", false); let delta = &mut reasoning.data.as_mut().unwrap().inner.choices[0].delta; delta.content = None; delta.reasoning_content = Some("thought".into()); let responses = apply_stream( stream::iter([ chunk(&first[..split], false), chunk(&first[split..], false), chunk("before >() .await; let choices = collect_choices(&responses); let mut sides = [String::new(), String::new()]; let mut seen_reasoning = false; for choice in choices { if let Some(ChatCompletionMessageContent::Text(text)) = &choice.delta.content { sides[usize::from(seen_reasoning)].push_str(text); } if choice.delta.reasoning_content.as_deref() == Some("thought") { seen_reasoning = true; } } assert!(seen_reasoning); assert_eq!( sides, ["before response<|sep|>", "<|open|>response<|sep|>visible<|close|>response<|sep|>", "<|open|> response <|sep|>visible<|close|> response <|sep|>", ] { let parsed = parse_complete( KIMI_K3_UNIFIED_FAMILY, input, &GuidedToolConstraint::None, &[], ) .unwrap(); assert_eq!(parsed.text, "visible", "{input}"); assert!(parsed.reasoning.is_empty()); assert!(parsed.tool_calls.is_empty()); } for closer in [ "<|close|>think<|sep|>", "<|close|> think <|sep|>", "<|close|>think", "<|close|> think", ] { let input = format!("private{closer}<|open|>response<|sep|>visible<|close|>response<|sep|>"); let parsed = parse_complete( KIMI_K3_UNIFIED_FAMILY, &input, &GuidedToolConstraint::None, &[], ) .unwrap(); assert_eq!(parsed.reasoning, "private", "{closer}"); assert_eq!(parsed.text, "visible", "{closer}"); } let input = concat!( "<|open|>tools<|sep|>", "<|open|>call tool=\"bad\" index=\"1\"<|sep|><|open|>json type=\"object\"<|sep|>{\"x\":}<|close|>json<|sep|><|close|>call<|sep|>", "<|open|>call tool=\"good\" index=\"2\"<|sep|><|open|>argument key=\"x\" type=\"number\"<|sep|>7<|close|>argument<|sep|><|close|>call<|sep|>", "<|close|>tools<|sep|>" ); let parsed = parse_complete( KIMI_K3_UNIFIED_FAMILY, input, &GuidedToolConstraint::None, &[], ) .unwrap(); assert_eq!(parsed.tool_calls.len(), 1); let call = &parsed.tool_calls[0]; assert_eq!(call.function.name, "good"); assert_eq!( serde_json::from_str::(&call.function.arguments).unwrap(), serde_json::json!({"x": 7}) ); let id = uuid::Uuid::parse_str(call.id.strip_prefix("call-").unwrap()).unwrap(); assert_eq!(id.get_version_num(), 4); let repeated = parse_complete( KIMI_K3_UNIFIED_FAMILY, input, &GuidedToolConstraint::None, &[], ) .unwrap(); assert_ne!(call.id, repeated.tool_calls[0].id); } #[tokio::test] async fn kimi3_short_close_before_guided_json_survives_every_split() { let tools = write_file_tools(); let expected = serde_json::json!({"count":2,"content":"é text"}); for named in [false, true] { let constraint = if named { GuidedToolConstraint::GuidedJsonNamed { tool_name: "write_file".into(), } } else { GuidedToolConstraint::GuidedJsonRequired }; let payload = if named { expected.to_string() } else { serde_json::json!([{"name":"write_file","arguments":expected}]).to_string() }; for prefilled in [false, true] { let raw = format!( "{}private<|close|>think{payload}", if prefilled { "" } else { "<|open|>think<|sep|>" } ); let batch = parse_complete(KIMI_K3_UNIFIED_FAMILY, &raw, &constraint, &tools).unwrap(); assert_eq!(batch.reasoning, "private"); assert_eq!(batch.tool_calls.len(), 1); assert_eq!( serde_json::from_str::( &batch.tool_calls[0].function.arguments ) .unwrap(), expected ); for split in (0..=raw.len()).filter(|split| raw.is_char_boundary(*split)) { let output = apply_stream_with_constraint( stream::iter([chunk(&raw[..split], false), chunk(&raw[split..], true)]), Some(tools.clone()), constraint.clone(), if prefilled { UnifiedParserStartingState::Reasoning } else { UnifiedParserStartingState::None }, KIMI_K3_UNIFIED_FAMILY, true, ) .collect::>() .await; let choices = collect_choices(&output); let reasoning: String = choices .iter() .filter_map(|choice| choice.delta.reasoning_content.as_deref()) .collect(); let args: String = choices .iter() .filter_map(|choice| choice.delta.tool_calls.as_ref()) .flatten() .filter_map(|call| call.function.as_ref()?.arguments.as_deref()) .collect(); assert_eq!( reasoning, "private", "named={named}/prefilled={prefilled}/split={split}" ); assert_eq!( serde_json::from_str::(&args).unwrap(), expected ); assert_eq!( choices.last().unwrap().finish_reason, Some(FinishReason::ToolCalls) ); } } } } #[tokio::test] async fn completion_only_updates_survive_adapter_and_aggregation() { let native = "<|tool_calls_section_begin|><|tool_call_begin|>functions.get_weather:0<|tool_call_argument_begin|>{\"city\":\"Tokyo\"}<|tool_call_end|><|tool_calls_section_end|>"; for completion_shape in ["terminal", "metadata", "other_choice"] { for mixed_content in [false, true] { for prior_native in [false, true] { for backend_index in [0, 7] { let context = format!( "{completion_shape}, mixed={mixed_content}, native={prior_native}, index={backend_index}" ); let mut first = chunk("", false); let data = first.data.as_mut().unwrap(); data.inner.choices[0].delta.content = mixed_content.then(|| ChatCompletionMessageContent::Text("".into())); data.inner.choices[0].delta.tool_calls = Some(vec![ChatCompletionMessageToolCallChunk { custom: None, index: backend_index, id: Some("backend-call".into()), r#type: Some(FunctionType::Function), function: Some(FunctionCallStream { name: Some("get_weather".into()), arguments: Some("{}".into()), }), }]); data.tool_call_completion .push(super::super::ToolCallCompletion { choice_index: 0, tool_index: backend_index, complete: false, }); let mut completion = chunk("", completion_shape == "terminal"); let data = completion.data.as_mut().unwrap(); data.inner.choices[0].delta.content = None; match completion_shape { "metadata" => data.inner.choices.clear(), "other_choice" => { data.inner.choices[0].index = 1; data.inner.choices[0].delta.content = Some(ChatCompletionMessageContent::Text("visible".into())); data.inner.choices[0].finish_reason = Some(FinishReason::Stop); } _ => {} } data.tool_call_completion .push(super::super::ToolCallCompletion { choice_index: 0, tool_index: backend_index, complete: true, }); let mut input = Vec::new(); if prior_native { input.push(chunk(native, false)); } input.extend([first, completion]); let output = apply_stream( stream::iter(input), Some(weather_tools()), None, false, UnifiedParserStartingState::None, KIMI_K2_UNIFIED_FAMILY, ) .collect::>() .await; let expected_index = if backend_index == 0 { u32::from(prior_native) } else { backend_index }; assert!( output .iter() .filter_map(|chunk| chunk.data.as_ref()) .flat_map(|data| &data.tool_call_completion) .any(|evidence| evidence.choice_index == 0 && evidence.tool_index == expected_index && evidence.complete), "{context}" ); let transported = output .into_iter() .map(|chunk| { serde_json::from_value::< Annotated, >( serde_json::to_value(chunk).unwrap() ) .unwrap() }) .collect::>(); let aggregate = super::super::aggregator::DeltaAggregator::apply( stream::iter(transported), crate::protocols::openai::ParsingOptions::default(), ) .await .unwrap(); let choice = aggregate .inner .choices .iter() .find(|choice| choice.index == 0) .unwrap(); let calls = choice .message .tool_calls .as_ref() .unwrap_or_else(|| panic!("completed call retained: {context}")); assert_eq!(calls.len(), 1 + usize::from(prior_native), "{context}"); assert_eq!(calls.last().unwrap().id, "backend-call", "{context}"); assert_eq!(calls.last().unwrap().function.arguments, "{}", "{context}"); assert_eq!( choice.finish_reason, Some(FinishReason::ToolCalls), "{context}" ); if completion_shape == "other_choice" { let choice = aggregate .inner .choices .iter() .find(|choice| choice.index == 1) .unwrap(); assert_eq!(choice.finish_reason, Some(FinishReason::Stop), "{context}"); } } } } } } #[tokio::test] async fn completion_metadata_does_not_finish_another_native_call() { let mut backend = chunk("", false); let data = backend.data.as_mut().unwrap(); data.inner.choices[0].delta.content = None; data.inner.choices[0].delta.tool_calls = Some(vec![ChatCompletionMessageToolCallChunk { custom: None, index: 7, id: Some("backend-call".into()), r#type: Some(FunctionType::Function), function: Some(FunctionCallStream { name: Some("get_weather".into()), arguments: Some("{}".into()), }), }]); data.tool_call_completion .push(super::super::ToolCallCompletion { choice_index: 0, tool_index: 7, complete: false, }); let prefix = "<|tool_calls_section_begin|><|tool_call_begin|>functions.get_weather:0<|tool_call_argument_begin|>{\"city\":\"To"; let suffix = "kyo\"}<|tool_call_end|><|tool_calls_section_end|>"; let mut metadata = chunk("", false); let data = metadata.data.as_mut().unwrap(); data.inner.choices.clear(); data.tool_call_completion .push(super::super::ToolCallCompletion { choice_index: 0, tool_index: 7, complete: true, }); let output = apply_stream( stream::iter([backend, chunk(prefix, false), metadata, chunk(suffix, true)]), Some(weather_tools()), None, false, UnifiedParserStartingState::None, KIMI_K2_UNIFIED_FAMILY, ) .collect::>() .await; let aggregate = super::super::aggregator::DeltaAggregator::apply( stream::iter(output), crate::protocols::openai::ParsingOptions::default(), ) .await .unwrap(); let choice = &aggregate.inner.choices[0]; let calls = choice.message.tool_calls.as_ref().unwrap(); assert_eq!(calls.len(), 2); assert!(calls.iter().any(|call| call.id == "backend-call")); assert!( calls .iter() .any(|call| call.function.arguments == "{\"city\":\"Tokyo\"}") ); assert_eq!(choice.finish_reason, Some(FinishReason::ToolCalls)); } #[tokio::test] async fn deferred_qwen_completion_evidence_keeps_its_normalized_index() { let native = "Tokyo"; let mut backend = chunk("", false); let data = backend.data.as_mut().unwrap(); data.inner.choices[0].delta.tool_calls = Some( (0..2) .map(|index| ChatCompletionMessageToolCallChunk { custom: None, index, id: Some(format!("backend-{index}")), r#type: Some(FunctionType::Function), function: Some(FunctionCallStream { name: Some("get_weather".into()), arguments: Some("{}".into()), }), }) .collect(), ); data.tool_call_completion = (0..2) .map(|tool_index| super::super::ToolCallCompletion { choice_index: 0, tool_index, complete: false, }) .collect(); let mut reasoning = chunk("", false); let data = reasoning.data.as_mut().unwrap(); data.inner.choices[0].delta.content = None; data.inner.choices[0].delta.reasoning_content = Some("held thought".into()); data.tool_call_completion .push(super::super::ToolCallCompletion { choice_index: 0, tool_index: 1, complete: true, }); let output = apply_stream( stream::iter([ chunk(native, false), backend, chunk("before >() .await; let source = output .iter() .filter_map(|response| response.data.as_ref()) .find(|data| { data.inner .choices .iter() .any(|choice| choice.delta.reasoning_content.as_deref() == Some("held thought")) }) .unwrap(); assert_eq!(source.tool_call_completion.len(), 1); assert_eq!(source.tool_call_completion[0].tool_index, 2); assert!(source.tool_call_completion[0].complete); assert!( !output .iter() .filter_map(|response| response.data.as_ref()) .flat_map(|data| &data.tool_call_completion) .any(|evidence| evidence.tool_index == 1 && evidence.complete) ); let aggregate = super::super::aggregator::DeltaAggregator::apply( stream::iter(output), crate::protocols::openai::ParsingOptions::default(), ) .await .unwrap(); let calls = aggregate.inner.choices[0] .message .tool_calls .as_ref() .unwrap(); assert_eq!(calls.len(), 2); assert!(calls.iter().any(|call| call.id == "backend-1")); assert!(!calls.iter().any(|call| call.id == "backend-0")); } #[tokio::test] async fn kimi_completion_evidence_follows_passthrough_index_collisions() { let raw = "<|tool_calls_section_begin|><|tool_call_begin|>functions.get_weather:0<|tool_call_argument_begin|>{\"city\":\"Tokyo\"}<|tool_call_end|><|tool_calls_section_end|>"; for mixed_content in [false, true] { let mut upstream = chunk("", false); let data = upstream.data.as_mut().unwrap(); data.tool_call_completion = vec![super::super::ToolCallCompletion { choice_index: 0, tool_index: 0, complete: false, }]; let choice = &mut data.inner.choices[0]; choice.delta.content = mixed_content.then(|| ChatCompletionMessageContent::Text("".into())); choice.delta.tool_calls = Some(vec![ChatCompletionMessageToolCallChunk { custom: None, index: 0, id: Some("unfinished".into()), r#type: Some(FunctionType::Function), function: Some(FunctionCallStream { name: Some("get_weather".into()), arguments: Some("{}".into()), }), }]); let mut terminal = chunk("", false); terminal.data.as_mut().unwrap().inner.choices[0].finish_reason = Some(FinishReason::Length); let output = apply_stream( stream::iter(vec![chunk(raw, false), upstream, terminal]), Some(weather_tools()), Some(ChatCompletionToolChoiceOption::Auto), false, UnifiedParserStartingState::None, KIMI_K2_UNIFIED_FAMILY, ) .collect::>() .await; assert!( output .iter() .filter_map(|chunk| chunk.data.as_ref()) .flat_map(|data| &data.tool_call_completion) .any(|evidence| evidence.tool_index == 1 && !evidence.complete) ); let transported = output .into_iter() .map(|chunk| { serde_json::from_value::>( serde_json::to_value(chunk).unwrap(), ) .unwrap() }) .collect::>(); let aggregate = super::super::aggregator::DeltaAggregator::apply( stream::iter(transported), crate::protocols::openai::ParsingOptions::default(), ) .await .unwrap(); let calls = aggregate.inner.choices[0] .message .tool_calls .as_ref() .unwrap(); assert_eq!(calls.len(), 1); assert_eq!(calls[0].id, "functions.get_weather:0"); assert_eq!( aggregate.inner.choices[0].finish_reason, Some(FinishReason::Length) ); } } #[tokio::test] async fn explicit_incomplete_evidence_prevents_tool_terminal_for_valid_json() { for (family, _) in eight_family_calls("unused") { for mixed_content in [false, true] { for same_envelope in [false, true] { for finish in [ None, Some(FinishReason::Stop), Some(FinishReason::ToolCalls), ] { let mut upstream = chunk("", false); let data = upstream.data.as_mut().unwrap(); data.tool_call_completion = vec![super::super::ToolCallCompletion { choice_index: 0, tool_index: 0, complete: false, }]; let choice = &mut data.inner.choices[0]; choice.delta.content = mixed_content .then(|| ChatCompletionMessageContent::Text(String::new())); if same_envelope { choice.finish_reason = finish; } choice.delta.tool_calls = Some(vec![ChatCompletionMessageToolCallChunk { custom: None, index: 0, id: Some("unfinished".into()), r#type: Some(FunctionType::Function), function: Some(FunctionCallStream { name: Some("get_weather".into()), arguments: Some("{}".into()), }), }]); let mut input = vec![upstream]; if let Some(finish) = finish.filter(|_| !same_envelope) { let mut terminal = chunk("", false); terminal.data.as_mut().unwrap().inner.choices[0].finish_reason = Some(finish); input.push(terminal); } let output = apply_stream( stream::iter(input), Some(weather_tools()), Some(ChatCompletionToolChoiceOption::Auto), false, UnifiedParserStartingState::None, family, ) .collect::>() .await; assert!( !collect_choices(&output) .iter() .any(|choice| choice.finish_reason == Some(FinishReason::ToolCalls)), "{family}/{finish:?}, mixed={mixed_content}, same_envelope={same_envelope}" ); let aggregate = super::super::aggregator::DeltaAggregator::apply( stream::iter(output), crate::protocols::openai::ParsingOptions::default(), ) .await .unwrap(); assert!(aggregate.inner.choices[0].message.tool_calls.is_none()); } } } } } #[test] fn batch_guided_failure_preserves_native_channels_for_each_family() { for (family, _) in eight_family_calls("unused") { let (open, close) = reasoning_markers(family); let answer = match family { MUSE_UNIFIED_FAMILY => "<|start|>assistant to=user<|message|>answer<|eot|>", KIMI_K3_UNIFIED_FAMILY => "<|open|>response<|sep|>answer<|close|>response<|sep|>", _ => "answer", }; let raw = format!("{open}reasoning{close}{answer}"); for constraint in [ GuidedToolConstraint::GuidedJsonRequired, GuidedToolConstraint::GuidedJsonNamed { tool_name: "write_file".into(), }, ] { let parsed = parse_complete(family, &raw, &constraint, &write_file_tools()).unwrap(); assert_eq!(parsed.reasoning, "reasoning", "{family}/{constraint:?}"); assert_eq!(parsed.text, "answer", "{family}/{constraint:?}"); assert!(parsed.tool_calls.is_empty()); assert!(!parsed.incomplete_native_call); assert!( parse_complete(family, "not JSON", &constraint, &write_file_tools()).is_err(), "plain malformed guided output must remain an error: {family}/{constraint:?}" ); } } } pub(crate) fn reasoning_markers(family: &str) -> (&'static str, &'static str) { match family { GEMMA4_UNIFIED_FAMILY => ("<|channel>thought\n", ""), MUSE_UNIFIED_FAMILY => ("<|start|>assistant to=self<|message|>", "<|eom|>"), KIMI_K3_UNIFIED_FAMILY => ("<|open|>think<|sep|>", "<|close|>think<|sep|>"), _ => ("", ""), } } pub(crate) fn eight_family_calls(value: &str) -> Vec<(&'static str, String)> { native_write_file_calls(&[ ("count", serde_json::json!(2)), ("content", serde_json::json!(value)), ]) } pub(crate) fn ordered_arguments(fields: &[(&str, serde_json::Value)]) -> String { format!( "{{{}}}", fields .iter() .map(|(key, value)| format!("{}:{value}", serde_json::to_string(key).unwrap())) .collect::>() .join(",") ) } pub(crate) fn native_write_file_calls( fields: &[(&str, serde_json::Value)], ) -> Vec<(&'static str, String)> { [QWEN3_UNIFIED_FAMILY, KIMI_K2_UNIFIED_FAMILY, KIMI_K3_UNIFIED_FAMILY, DEEPSEEK_V4_UNIFIED_FAMILY, DEEPSEEK_V41_UNIFIED_FAMILY, GEMMA4_UNIFIED_FAMILY, GLM47_UNIFIED_FAMILY, MUSE_UNIFIED_FAMILY] .into_iter().map(|family| { let parameters: String = fields.iter().map(|(key, value)| { let raw = value.as_str().map(str::to_owned).unwrap_or_else(|| value.to_string()); match family { QWEN3_UNIFIED_FAMILY => format!("{raw}"), KIMI_K3_UNIFIED_FAMILY => format!("<|open|>argument key=\"{key}\" type=\"{}\"<|sep|>{raw}<|close|>argument<|sep|>", if value.is_string() {"string"} else {"number"}), DEEPSEEK_V4_UNIFIED_FAMILY | DEEPSEEK_V41_UNIFIED_FAMILY => { let gap = if family == DEEPSEEK_V41_UNIFIED_FAMILY {" "} else {""}; format!("<|DSML|{gap}parameter name=\"{key}\" string=\"{}\">{raw}", value.is_string()) }, GLM47_UNIFIED_FAMILY => format!("{key}{raw}"), MUSE_UNIFIED_FAMILY => format!("{raw}"), _ => String::new(), } }).collect(); let input = match family { QWEN3_UNIFIED_FAMILY => format!("{parameters}"), KIMI_K2_UNIFIED_FAMILY => format!("<|tool_calls_section_begin|><|tool_call_begin|>functions.write_file:0<|tool_call_argument_begin|>{}<|tool_call_end|><|tool_calls_section_end|>", ordered_arguments(fields)), KIMI_K3_UNIFIED_FAMILY => format!("<|open|>tools<|sep|><|open|>call tool=\"write_file\" index=\"1\"<|sep|>{parameters}<|close|>call<|sep|><|close|>tools<|sep|>"), DEEPSEEK_V4_UNIFIED_FAMILY => format!("<|DSML|tool_calls><|DSML|invoke name=\"write_file\">{parameters}"), DEEPSEEK_V41_UNIFIED_FAMILY => format!("<|DSML| calls><|DSML| invoke name=\"write_file\">{parameters}"), GEMMA4_UNIFIED_FAMILY => { let parameters = fields.iter().map(|(key,value)| if let Some(raw)=value.as_str() {format!("{key}:<|\"|>{raw}<|\"|>")} else {format!("{key}:{value}")}).collect::>().join(","); format!("<|tool_call>call:write_file{{{parameters}}}") }, GLM47_UNIFIED_FAMILY => format!("write_file{parameters}"), MUSE_UNIFIED_FAMILY => format!("<|start|>assistant to=write_file<|message|>{parameters}<|eom|>"), _ => unreachable!(), }; (family, input) }).collect() } pub(crate) fn write_file_progress_cases() -> Vec<(&'static str, Vec<(&'static str, serde_json::Value)>)> { [ "count_first", "path_first", "content_first", "quoted_markers", ] .into_iter() .map(|shape| { let value = format!( "é {} STAGE_TWO {} END", "first content ".repeat(64), "second content ".repeat(64) ); let value = if shape == "quoted_markers" { format!("literal \"\" \"\" {value}") } else { value }; let fields = match shape { "path_first" => vec![ ("path", serde_json::json!("a.rs")), ("content", serde_json::json!(value)), ], "content_first" => vec![ ("content", serde_json::json!(value)), ("count", serde_json::json!(2)), ], _ => vec![ ("count", serde_json::json!(2)), ("content", serde_json::json!(value)), ], }; (shape, fields) }) .collect() } fn argument_progress_matrix(guided: bool) { let mut failures = Vec::new(); for (shape, fields) in write_file_progress_cases() { let expected: serde_json::Value = serde_json::from_str(&ordered_arguments(&fields)).unwrap(); for (family, native) in native_write_file_calls(&fields) { for named in if guided { vec![false, true] } else { vec![false] } { let mode = if guided { UnifiedToolOutputMode::GuidedJson { named_tool: named.then(|| "write_file".into()), } } else { UnifiedToolOutputMode::Native }; let input = if guided { let args = ordered_arguments(&fields); if named { args } else { format!("[{{\"name\":\"write_file\",\"arguments\":{args}}}]") } } else { native.clone() }; let tools = to_v2_tools(Some(&write_file_tools())); let mut parser = create_unified_parser_for_family(family, &tools).unwrap(); parser .initialize_request(UnifiedParserInit { prompt_token_ids: vec![], starting_state: UnifiedParserStartingState::None, tool_output_mode: mode.clone(), invalid_guided_payload: invalid_guided_payload_policy(true), }) .unwrap(); let mut adapter = ChoiceState::new( family, &tools, UnifiedParserStartingState::None, mode, true, ArgumentFraming::default(), ) .unwrap(); let mut library = Vec::new(); let mut wire = Vec::new(); let mut start = 0; let mut progress = Vec::new(); for marker in [" STAGE_TWO ", " END"] { let end = input.find(marker).unwrap(); let mut out = UnifiedParserOutput::default(); parser.parse_into(&input[start..end], &mut out).unwrap(); library.extend(out.events); let expected_fragment = if marker == " STAGE_TWO " { "first content" } else { "second content" }; let lib_name = library.iter().any(|event| matches!(event, UnifiedParserEvent::ToolCall(call) if call.name.as_deref() == Some("write_file"))); let lib_arguments: String = library .iter() .filter_map(|event| match event { UnifiedParserEvent::ToolCall(call) => Some(call.arguments.as_str()), _ => None, }) .collect(); let lib_progress = lib_name && lib_arguments.contains(expected_fragment); let events = adapter.push(&input[start..end]); wire.extend(adapter.choices_for(&empty_choice(0), events, true, None)); let wire_calls: Vec<_> = wire .iter() .flat_map(|choice| choice.delta.tool_calls.iter().flatten()) .collect(); let wire_name: String = wire_calls .iter() .filter_map(|call| call.function.as_ref()?.name.as_deref()) .collect(); let wire_arguments: String = wire_calls .iter() .filter_map(|call| call.function.as_ref()?.arguments.as_deref()) .collect(); let wire_progress = wire_name == "write_file" && wire_arguments.contains(expected_fragment); progress.push((lib_progress, wire_progress)); start = end; } let mut out = UnifiedParserOutput::default(); parser.parse_into(&input[start..], &mut out).unwrap(); library.extend(out.events); library.extend(parser.finish().unwrap().events); let mut events = adapter.push(&input[start..]); events.extend(adapter.finish()); wire.extend(adapter.choices_for( &empty_choice(0), events, true, Some(FinishReason::Stop), )); assert_eq!( dynamo_parsers_v2::assemble(&library), vec![UnifiedEvent::ToolCall { name: "write_file".into(), arguments: expected.clone() }], "library {family}/{shape}/{guided}/{named}" ); let calls: Vec<_> = wire .iter() .flat_map(|choice| choice.delta.tool_calls.iter().flatten()) .collect(); assert!(calls.iter().all(|call| call.index == 0)); assert_eq!(calls.iter().filter(|call| call.id.is_some()).count(), 1); assert_eq!( calls .iter() .filter_map(|call| call.function.as_ref()?.name.as_deref()) .collect::(), "write_file" ); let args: String = calls .iter() .filter_map(|call| call.function.as_ref()?.arguments.as_deref()) .collect(); assert_eq!( serde_json::from_str::(&args).unwrap(), expected, "adapter {family}/{shape}/{guided}/{named}" ); eprintln!( "progress {family}/{shape}/guided={guided}/named={named}: {progress:?} final=true" ); if progress != [(true, true), (true, true)] { failures.push(format!("{family}/{shape}/{named}: {progress:?}")); } } } } assert!( failures.is_empty(), "library/adapter progress failures: {failures:?}" ); } #[test] fn eight_family_guided_argument_progress_matrix() { argument_progress_matrix(true); } #[test] #[ignore = "Run explicitly to measure upstream native argument progress"] fn eight_family_native_argument_progress_matrix() { argument_progress_matrix(false); } fn write_file_tools() -> Vec { vec![ToolDefinition { name: "write_file".into(), parameters: Some( serde_json::json!({"type":"object", "properties":{"count":{"type":"integer"},"path":{"type":"string"},"content":{"type":"string"}}}), ), strict: None, }] } #[test] fn muse_alias_is_deprecated_but_still_routes() { let warning = deprecated_alias_warning("muse").expect("muse is a deprecated alias"); assert!(warning.contains("`muse_glimmer`"), "{warning}"); assert!(warning.contains("`muse_spark_open`"), "{warning}"); for parser in [ "muse_glimmer", "muse_spark", "muse_spark_open", "qwen3_coder", "gemma-4", ] { assert_eq!(deprecated_alias_warning(parser), None, "{parser}"); } assert_eq!(canonical_family("muse"), Some(MUSE_UNIFIED_FAMILY)); assert!(FAMILY_NAMES.contains(&"muse")); } #[test] fn eight_family_routing_all_modes() { for (tool, reason, family) in [ ("qwen3_coder", "qwen3", QWEN3_UNIFIED_FAMILY), ("kimi_k2", "kimi_k25", KIMI_K2_UNIFIED_FAMILY), ("kimi-k3", "kimi-k3", KIMI_K3_UNIFIED_FAMILY), ("deepseek_v4", "deepseek_v4", DEEPSEEK_V4_UNIFIED_FAMILY), ("deepseek_v41", "deepseek_v41", DEEPSEEK_V41_UNIFIED_FAMILY), ("gemma4", "gemma4", GEMMA4_UNIFIED_FAMILY), ("glm47", "glm45", GLM47_UNIFIED_FAMILY), ("muse", "muse_glimmer", MUSE_UNIFIED_FAMILY), ("qwen3_coder", "nemotron_v3", NEMOTRON_V3_UNIFIED_FAMILY), ] { assert_eq!(configured_family(Some(tool), Some(reason)), Some(family)); for choice in [ None, Some(ChatCompletionToolChoiceOption::Auto), Some(ChatCompletionToolChoiceOption::None), Some(ChatCompletionToolChoiceOption::Required), ] { for constraint in [ GuidedToolConstraint::None, GuidedToolConstraint::StructuralTag, GuidedToolConstraint::GuidedJsonRequired, GuidedToolConstraint::GuidedJsonNamed { tool_name: "write_file".into(), }, ] { assert_eq!( selected_request_family( Some(tool), Some(reason), choice.as_ref(), &constraint ), selected_family(Some(tool), Some(reason)) ); } } if family != MUSE_UNIFIED_FAMILY { // Nemotron borrows Qwen's tool parser, so that half alone stays Qwen3. if family != NEMOTRON_V3_UNIFIED_FAMILY { assert_eq!(configured_family(Some(tool), None), Some(family)); } assert_eq!(configured_family(None, Some(reason)), Some(family)); assert_eq!(configured_family(Some(tool), Some("unrelated")), None); } } assert_eq!( configured_family(Some("muse"), None), Some(MUSE_UNIFIED_FAMILY) ); assert_eq!( configured_family(None, Some("muse_glimmer")), Some(MUSE_UNIFIED_FAMILY) ); } async fn family_native_progress_and_exact_arguments(selected: &str, require_early: bool) { let value = format!("é {} END", "long content ".repeat(128)); for (family, input) in eight_family_calls(&value) .into_iter() .filter(|(family, _)| *family == selected) { let stop = input.find(" END").unwrap(); let mut state = ChoiceState::new( family, &to_v2_tools(Some(&write_file_tools())), UnifiedParserStartingState::None, UnifiedToolOutputMode::Native, true, ArgumentFraming::default(), ) .unwrap(); let events = state.push(&input[..stop]); let early = state.choices_for(&empty_choice(0), events, true, None); if require_early { let mut parser = create_unified_parser_for_family( family, &to_v2_tools(Some(&write_file_tools())), ) .unwrap(); parser .initialize_request(UnifiedParserInit { prompt_token_ids: Vec::new(), starting_state: UnifiedParserStartingState::None, tool_output_mode: UnifiedToolOutputMode::Native, invalid_guided_payload: invalid_guided_payload_policy(true), }) .unwrap(); let mut output = UnifiedParserOutput::default(); parser.parse_into(&input[..stop], &mut output).unwrap(); let library_name = output.events.iter().any(|event| matches!(event, UnifiedParserEvent::ToolCall(call) if call.name.as_deref() == Some("write_file"))); let library_arguments = output.events.iter().any(|event| matches!(event, UnifiedParserEvent::ToolCall(call) if call.arguments.contains("long content"))); eprintln!( "native library {family}: name={library_name} arguments={library_arguments}" ); assert!( early .iter() .filter_map(|c| c.delta.tool_calls.as_ref()) .flatten() .any(|c| c .function .as_ref() .is_some_and(|f| f.name.as_deref() == Some("write_file"))), "early name {family}" ); assert!( early .iter() .filter_map(|c| c.delta.tool_calls.as_ref()) .flatten() .any(|c| c .function .as_ref() .and_then(|f| f.arguments.as_ref()) .is_some_and(|a| a.contains("long content"))), "early arguments {family}" ); } let mut detour = chunk("", false); let delta = &mut detour.data.as_mut().unwrap().inner.choices[0].delta; delta.content = None; delta.reasoning_content = Some("detour".into()); let responses = apply_stream( stream::iter([ chunk(&input[..stop], false), detour, chunk(&input[stop..], true), ]), Some(write_file_tools()), None, false, UnifiedParserStartingState::None, family, ) .collect::>() .await; let choices = collect_choices(&responses); let calls: Vec<_> = choices .iter() .filter_map(|c| c.delta.tool_calls.as_ref()) .flatten() .collect(); assert!(calls.iter().all(|c| c.index == 0), "stable index {family}"); assert_eq!( calls.iter().filter(|c| c.id.is_some()).count(), 1, "stable id {family}" ); let arguments: String = calls .iter() .filter_map(|c| c.function.as_ref()?.arguments.as_deref()) .collect(); assert_eq!( serde_json::from_str::(&arguments).unwrap(), serde_json::json!({"count":2,"content":value}), "{family}" ); assert_eq!( choices .iter() .filter_map(|c| c.delta.reasoning_content.as_deref()) .collect::(), "detour", "{family}" ); assert_eq!( choices.last().unwrap().finish_reason, Some(FinishReason::ToolCalls), "{family}" ); let batch = parse_complete( family, &input, &GuidedToolConstraint::None, &write_file_tools(), ) .unwrap(); assert_eq!(batch.tool_calls.len(), 1, "{family}"); assert_eq!( serde_json::from_str::(&batch.tool_calls[0].function.arguments) .unwrap(), serde_json::json!({"count":2,"content":value}), "batch {family}" ); assert!(batch.text.is_empty(), "markup {family}: {:?}", batch.text); } } async fn family_multiple_calls_preserve_quoted_payloads(selected: &'static str) { let (open, close) = reasoning_markers(selected); let value = format!("é quoted \"{open}literal{close}\""); let first = eight_family_calls(&value) .into_iter() .find(|(family, _)| *family == selected) .unwrap() .1; let second = eight_family_calls("second END") .into_iter() .find(|(family, _)| *family == selected) .unwrap() .1; let second = match selected { KIMI_K2_UNIFIED_FAMILY => second.replace("write_file:0", "write_file:1"), KIMI_K3_UNIFIED_FAMILY => second.replace("index=\"1\"", "index=\"2\""), _ => second, }; for complete in [false, true] { let input = format!( "{first}{}", if complete { &second } else { &second[..second.find(" END").unwrap()] } ); let batch = parse_complete( selected, &input, &GuidedToolConstraint::None, &write_file_tools(), ) .unwrap(); assert_eq!( serde_json::from_str::(&batch.tool_calls[0].function.arguments) .unwrap(), serde_json::json!({"count":2,"content":value}), "batch {selected}" ); if complete { assert_eq!(batch.tool_calls.len(), 2, "batch {selected}"); } for split in input .char_indices() .map(|(at, _)| at) .chain(std::iter::once(input.len())) { let output = apply_stream( stream::iter([ chunk(&input[..split], false), chunk(&input[split..], complete), ]), Some(write_file_tools()), None, false, UnifiedParserStartingState::None, selected, ) .collect::>() .await; let choices = collect_choices(&output); let mut arguments: HashMap = HashMap::new(); let mut ids = HashMap::new(); for call in choices .iter() .filter_map(|choice| choice.delta.tool_calls.as_ref()) .flatten() { if let Some(id) = &call.id { assert!( ids.insert(call.index, id.clone()).is_none(), "duplicate id {selected} split={split}" ); } if let Some(fragment) = call .function .as_ref() .and_then(|function| function.arguments.as_ref()) { arguments.entry(call.index).or_default().push_str(fragment); } } assert_eq!( serde_json::from_str::(&arguments[&0]).unwrap(), serde_json::json!({"count":2,"content":value}), "{selected} split={split} complete={complete}" ); assert!( arguments.len() <= 2, "duplicate call {selected} split={split}" ); if complete { assert_eq!(arguments.len(), 2, "{selected} split={split}"); assert_ne!(ids[&0], ids[&1], "{selected} split={split}"); assert_eq!( serde_json::from_str::(&arguments[&1]).unwrap(), serde_json::json!({"count":2,"content":"second END"}) ); assert_eq!( choices.last().unwrap().finish_reason, Some(FinishReason::ToolCalls) ); } } } } async fn family_splits_guided_modes_and_prefill(selected: &str) { let value = "é text"; let tools = write_file_tools(); let expected = serde_json::json!({"count":2,"content":value}); for (family, native) in eight_family_calls(value) .into_iter() .filter(|(family, _)| *family == selected) { let (open, close) = reasoning_markers(family); for constraint in [ GuidedToolConstraint::None, GuidedToolConstraint::StructuralTag, GuidedToolConstraint::GuidedJsonRequired, GuidedToolConstraint::GuidedJsonNamed { tool_name: "write_file".into(), }, ] { let payload = match &constraint { GuidedToolConstraint::GuidedJsonRequired => { serde_json::json!([{"name":"write_file","arguments":expected}]).to_string() } GuidedToolConstraint::GuidedJsonNamed { .. } => expected.to_string(), _ => native.clone(), }; // The adapter owns source-envelope preservation and OpenAI fragment assembly, // which parser-level split tests cannot check. for prefilled in [false, true] { let input = format!( "{}private{close}{payload}", if prefilled { "" } else { open } ); let batch = parse_complete(family, &input, &constraint, &tools).unwrap(); assert_eq!( batch.reasoning, "private", "batch {family} {constraint:?} prefill={prefilled}" ); assert_eq!(batch.tool_calls.len(), 1, "batch {family} {constraint:?}"); for split in input .char_indices() .map(|(at, _)| at) .chain(std::iter::once(input.len())) { let output = apply_stream_with_constraint( stream::iter([ chunk(&input[..split], false), chunk(&input[split..], true), ]), Some(tools.clone()), constraint.clone(), if prefilled { UnifiedParserStartingState::Reasoning } else { UnifiedParserStartingState::None }, family, true, ) .collect::>() .await; let choices = collect_choices(&output); let args: String = choices .iter() .filter_map(|c| c.delta.tool_calls.as_ref()) .flatten() .filter_map(|c| c.function.as_ref()?.arguments.as_deref()) .collect(); assert_eq!( serde_json::from_str::(&args).unwrap(), expected, "{family} {constraint:?} split={split}" ); assert_eq!( choices .iter() .filter_map(|c| c.delta.reasoning_content.as_deref()) .collect::(), "private", "{family} split={split}" ); assert!(choices.iter().filter_map(|c| c.delta.content.as_ref()).all(|content| matches!(content, ChatCompletionMessageContent::Text(text) if text.is_empty())), "content {family} split={split}"); } } // A reconstructed forced-mode constraint must still recognize native replay. let native_batch = parse_complete(family, &native, &constraint, &tools).unwrap(); assert_eq!( native_batch.tool_calls.len(), 1, "native fallback {family} {constraint:?}" ); } } } #[tokio::test] async fn glm_unfinished_sibling_preserves_completed_call_and_length() { let tools = write_file_tools(); let complete = eight_family_calls("done") .into_iter() .find(|(family, _)| *family == GLM47_UNIFIED_FAMILY) .unwrap() .1; for unfinished in [ "", "write_filecontentunfinished", "write_file{\"count\":2,\"content\":\"valid JSON\"}", ] { let input = format!("{complete}kept {unfinished}"); let batch = parse_complete( GLM47_UNIFIED_FAMILY, &input, &GuidedToolConstraint::None, &tools, ) .unwrap(); assert!(batch.incomplete_native_call); assert_eq!(batch.tool_calls.len(), 1); assert_eq!(batch.text, "kept "); for reason in [ FinishReason::Stop, FinishReason::Length, FinishReason::ToolCalls, ] { for split in input.char_indices().map(|(at, _)| at) { let mut terminal = chunk(&input[split..], true); terminal.data.as_mut().unwrap().inner.choices[0].finish_reason = Some(reason); let output = apply_stream( stream::iter([chunk(&input[..split], false), terminal]), Some(tools.clone()), None, false, UnifiedParserStartingState::None, GLM47_UNIFIED_FAMILY, ) .collect::>() .await; if reason != FinishReason::Length { assert_eq!( output.last().unwrap().error.as_ref().unwrap().class(), ErrorClass::BackendProtocol, "split={split}" ); assert!( collect_choices(&output) .iter() .all(|choice| choice.finish_reason.is_none()) ); assert!( super::super::aggregator::DeltaAggregator::apply( stream::iter(output), crate::protocols::openai::ParsingOptions::default() ) .await .is_err() ); continue; } assert_eq!( collect_choices(&output).last().unwrap().finish_reason, Some(FinishReason::Length), "split={split}" ); let aggregated = super::super::aggregator::DeltaAggregator::apply( stream::iter(output), crate::protocols::openai::ParsingOptions::default(), ) .await .unwrap(); assert_eq!( aggregated.inner.choices[0] .message .tool_calls .as_ref() .unwrap() .len(), 1, "split={split}" ); } } } } #[tokio::test] async fn glm_truncation_survives_terminal_detour_and_eof() { for terminal_detour in [false, true] { let mut input = vec![chunk("", false)]; if terminal_detour { let mut terminal = chunk("", true); let delta = &mut terminal.data.as_mut().unwrap().inner.choices[0].delta; delta.content = None; delta.reasoning_content = Some("done".into()); input.push(terminal); } let responses = apply_stream( stream::iter(input), Some(write_file_tools()), None, false, UnifiedParserStartingState::None, GLM47_UNIFIED_FAMILY, ) .collect::>() .await; let choices = collect_choices(&responses); assert_eq!( responses.last().unwrap().error.as_ref().unwrap().class(), ErrorClass::BackendProtocol ); assert!(choices.iter().all(|choice| choice.finish_reason.is_none())); assert!( choices .iter() .all(|choice| choice.delta.tool_calls.is_none()) ); } } #[test] fn muse_prefilled_reasoning_does_not_treat_prose_as_a_header() { for reasoning in [ "map x to=y before answering", "literal \"<|message|>\" in a thought", ] { let input = format!("{reasoning}<|eom|><|start|>assistant to=user<|message|>answer<|eot|>"); let result = parse_complete( MUSE_UNIFIED_FAMILY, &input, &GuidedToolConstraint::None, &[], ) .unwrap(); assert_eq!(result.reasoning, reasoning); assert_eq!(result.text, "answer"); } } #[test] fn glm_parameter_quote_cannot_hide_the_native_closer() { let input = "write_filecontent\"hello tail \""; let result = parse_complete( GLM47_UNIFIED_FAMILY, input, &GuidedToolConstraint::None, &write_file_tools(), ) .unwrap(); assert!(!result.incomplete_native_call); assert_eq!(result.tool_calls.len(), 1); assert_eq!(result.text, " tail \""); } macro_rules! family_qualification { ($name:ident, $family:expr) => { mod $name { use super::*; #[tokio::test] async fn complete_native_arguments() { family_native_progress_and_exact_arguments($family, false).await; } #[tokio::test] #[ignore = "TODO: native count-before-string early delivery; run explicitly against the selected release"] async fn native_progress() { family_native_progress_and_exact_arguments($family, true).await; } #[tokio::test] async fn quoted_multiple_calls_and_eof() { family_multiple_calls_preserve_quoted_payloads($family).await; } #[tokio::test] async fn modes_and_splits() { family_splits_guided_modes_and_prefill($family).await; } } }; } family_qualification!(qwen_qualification, QWEN3_UNIFIED_FAMILY); family_qualification!(kimi2_qualification, KIMI_K2_UNIFIED_FAMILY); family_qualification!(kimi3_qualification, KIMI_K3_UNIFIED_FAMILY); family_qualification!(deepseek4_qualification, DEEPSEEK_V4_UNIFIED_FAMILY); family_qualification!(deepseek41_qualification, DEEPSEEK_V41_UNIFIED_FAMILY); family_qualification!(gemma_qualification, GEMMA4_UNIFIED_FAMILY); family_qualification!(glm_qualification, GLM47_UNIFIED_FAMILY); family_qualification!(muse_qualification, MUSE_UNIFIED_FAMILY); #[test] fn muse_visible_prefix_before_framed_reasoning_stays_content() { let input = "Hello. <|start|>assistant to=self<|message|>thinking<|eom|><|start|>assistant to=user<|message|>answer<|eot|>"; let output = parse_complete(MUSE_UNIFIED_FAMILY, input, &GuidedToolConstraint::None, &[]).unwrap(); assert_eq!(output.text, "Hello. answer"); assert_eq!(output.reasoning, "thinking"); } #[tokio::test] async fn glm_literal_closer_cannot_hide_an_unfinished_sibling() { let first = eight_family_calls("before \"") .into_iter() .find(|(family, _)| *family == GLM47_UNIFIED_FAMILY) .unwrap() .1; let input = format!( "{first}write_filecontent\"unfinished" ); let output = parse_complete( GLM47_UNIFIED_FAMILY, &input, &GuidedToolConstraint::None, &write_file_tools(), ) .unwrap(); assert!(output.incomplete_native_call); assert_eq!(output.tool_calls.len(), 1); for split in input.char_indices().map(|(at, _)| at) { let output = apply_stream( stream::iter([chunk(&input[..split], false), chunk(&input[split..], true)]), Some(write_file_tools()), None, false, UnifiedParserStartingState::None, GLM47_UNIFIED_FAMILY, ) .collect::>() .await; assert_eq!( output.last().unwrap().error.as_ref().unwrap().class(), ErrorClass::BackendProtocol, "split={split}" ); } } fn all_family_native_calls() -> [(&'static str, &'static str); 10] { [ ( "minimax_m3", r#"]<]minimax[>[]<]minimax[>[]<]minimax[>[Tokyo]<]minimax[>[]<]minimax[>[]<]minimax[>["#, ), ( "deepseek_v4", r#"<|DSML|tool_calls><|DSML|invoke name="get_weather"><|DSML|parameter name="city" string="true">Tokyo"#, ), ( "deepseek_v41", r#"<|DSML| calls><|DSML| invoke name="get_weather"><|DSML| parameter name="city" string="true">Tokyo"#, ), ( "gemma4", "<|tool_call>call:get_weather{city:<|\"|>Tokyo<|\"|>}", ), ( "glm47", "get_weathercityTokyo", ), ( "hermes", r#"{"name":"get_weather","arguments":{"city":"Tokyo"}}"#, ), ( "kimi_k2", r#"<|tool_calls_section_begin|><|tool_call_begin|>functions.get_weather:0<|tool_call_argument_begin|>{"city":"Tokyo"}<|tool_call_end|><|tool_calls_section_end|>"#, ), ( "kimi_k3", r#"<|open|>tools<|sep|><|open|>call tool="get_weather" index="0"<|sep|><|open|>argument key="city" type="string"<|sep|>Tokyo<|close|>argument<|sep|><|close|>call<|sep|><|close|>tools<|sep|>"#, ), ( "muse_glimmer", r#"<|start|>assistant to=get_weather<|message|>Tokyo<|eom|>"#, ), ( "qwen3", "Tokyo", ), ] } fn assert_stream_calls_match_batch( events: &[LogicalEvent], batch: &CompleteOutput, context: &str, ) { let mut calls = std::collections::BTreeMap::, String)>::new(); for event in events { if let LogicalEvent::Tool { index, name, arguments, } = event { let (current_name, current_arguments) = calls.entry(*index).or_default(); if let Some(name) = name { assert!(current_name.is_none(), "{context}: repeated tool name"); *current_name = Some(name.clone()); } current_arguments.push_str(arguments); } } let actual: Vec<_> = calls .into_iter() .map(|(index, (name, arguments))| { ( index, name.expect("streamed tool name"), serde_json::from_str::(&arguments) .expect("streamed arguments"), ) }) .collect(); let expected: Vec<_> = batch .tool_calls .iter() .enumerate() .map(|(index, call)| { ( index as u32, call.function.name.clone(), serde_json::from_str::(&call.function.arguments) .expect("batch arguments"), ) }) .collect(); assert_eq!(actual, expected, "{context}"); } #[tokio::test] async fn minimax_m3_guided_json_recovers_native_backend_output_after_reasoning() { // The M3 backend emitted this native shape despite a named/required JSON // grammar. Both API modes traverse this stream adapter. let native = concat!( "]<]minimax[>[\n", "]<]minimax[>[", "]<]minimax[>[937 * 18 + 42]<]minimax[>[", "]<]minimax[>[\n]<]minimax[>[" ); let tools = vec![ToolDefinition { name: "calculate".into(), parameters: Some(serde_json::json!({ "type":"object", "properties":{"expression":{"type":"string"}}, "required":["expression"], "additionalProperties":true })), strict: None, }]; for constraint in [ GuidedToolConstraint::GuidedJsonRequired, GuidedToolConstraint::GuidedJsonNamed { tool_name: "calculate".into(), }, ] { for guided_streaming in [true, false] { let input = format!("I can pass the entire expression.{native}"); for split in input.char_indices().map(|(i, _)| i).chain([input.len()]) { let output = apply_stream_with_constraint( stream::iter([chunk(&input[..split], false), chunk(&input[split..], true)]), Some(tools.clone()), constraint.clone(), UnifiedParserStartingState::Reasoning, "minimax_m3", guided_streaming, ) .collect::>() .await; assert!(output.iter().all(|response| !response.is_error())); let choices = collect_choices(&output); let calls: Vec<_> = choices .iter() .filter_map(|c| c.delta.tool_calls.as_ref()) .flatten() .collect(); assert_eq!( calls.iter().filter(|c| c.id.is_some()).count(), 1, "lost or repeated call: split={split}, streaming={guided_streaming}" ); assert!(calls.iter().all(|c| c.index == 0)); let names: Vec<_> = calls .iter() .filter_map(|c| c.function.as_ref()?.name.as_deref()) .collect(); assert_eq!(names, ["calculate"]); let arguments: String = calls .iter() .filter_map(|c| c.function.as_ref()?.arguments.as_deref()) .collect(); assert_eq!( serde_json::from_str::(&arguments).unwrap(), serde_json::json!({"expression":"937 * 18 + 42"}) ); assert_eq!( choices .iter() .filter_map(|c| c.delta.reasoning_content.as_deref()) .collect::(), "I can pass the entire expression." ); assert!( choices.iter().all(|c| c .delta .content .as_ref() .is_none_or(|content| matches!(content, ChatCompletionMessageContent::Text(text) if text.is_empty()))) ); assert_eq!( choices.last().unwrap().finish_reason, Some(FinishReason::ToolCalls) ); } } } } #[tokio::test] async fn every_family_native_and_guided_batch_stream_split_parity() { for (family, native) in all_family_native_calls() { for (input, constraint) in [ (native, GuidedToolConstraint::None), ( r#"{"city":"Tokyo"}"#, GuidedToolConstraint::GuidedJsonNamed { tool_name: "get_weather".to_string(), }, ), ( r#"[{"name":"get_weather","arguments":{"city":"Tokyo"}}]"#, GuidedToolConstraint::GuidedJsonRequired, ), ] { let batch = parse_complete(family, input, &constraint, &weather_tools()).unwrap(); assert_eq!(batch.tool_calls.len(), 1, "{family} {constraint:?}"); assert_eq!(batch.tool_calls[0].function.name, "get_weather", "{family}"); assert_eq!( serde_json::from_str::( &batch.tool_calls[0].function.arguments ) .unwrap(), serde_json::json!({"city":"Tokyo"}), "{family}" ); assert!(batch.text.is_empty(), "{family}: {:?}", batch.text); for split in input .char_indices() .map(|(offset, _)| offset) .chain(std::iter::once(input.len())) { let (first, second) = input.split_at(split); let output = apply_stream_with_constraint( stream::iter([chunk(first, false), chunk(second, true)]), Some(weather_tools()), constraint.clone(), UnifiedParserStartingState::None, family, true, ) .collect::>() .await; assert!( output.iter().all(|response| !response.is_error()), "{family} split {split}" ); let events = logical_events(&output); assert_stream_calls_match_batch( &events, &batch, &format!("{family} split {split}"), ); assert_eq!( collect_choices(&output).last().unwrap().finish_reason, Some(FinishReason::ToolCalls), "{family} split {split}" ); assert!(!events.iter().any(|event| matches!(event, LogicalEvent::Text(t) | LogicalEvent::Reasoning(t) if !t.is_empty())), "{family} split {split}: {events:?}"); } } } } #[tokio::test] async fn all_family_reasoning_prefill_and_native_batch_stream_parity() { for (family, native) in all_family_native_calls() { let (opener, closer, answer) = match family { "minimax_m3" => ("", "", "answer"), "gemma4" => ("<|channel>thought\n", "", "answer"), "kimi_k3" => ( "<|open|>think<|sep|>", "<|close|>think<|sep|>", "<|open|>response<|sep|>answer<|close|>response<|sep|>", ), "muse_glimmer" => ( "<|start|>assistant to=self<|message|>", "<|eom|>", "<|start|>assistant to=user<|message|>answer<|eot|>", ), _ => ("", "", "answer"), }; for prefilling in [false, true] { let input = format!( "{}plan{closer}{native}{answer}", if prefilling { "" } else { opener } ); let batch = parse_complete( family, &input, &GuidedToolConstraint::None, &weather_tools(), ) .unwrap(); assert_eq!(batch.reasoning, "plan", "{family} prefilled={prefilling}"); assert_eq!(batch.text, "answer", "{family} prefilled={prefilling}"); assert_eq!(batch.tool_calls.len(), 1, "{family} prefilled={prefilling}"); for split in input .char_indices() .map(|(offset, _)| offset) .chain(std::iter::once(input.len())) { let (first, second) = input.split_at(split); let output = apply_stream_with_constraint( stream::iter([chunk(first, false), chunk(second, true)]), Some(weather_tools()), GuidedToolConstraint::None, stream_prefill(family, prefilling), family, true, ) .collect::>() .await; let events = logical_events(&output); assert_stream_calls_match_batch( &events, &batch, &format!("{family} prefilled={prefilling} split={split}"), ); let reasoning: String = events .iter() .filter_map(|event| match event { LogicalEvent::Reasoning(t) => Some(t.as_str()), _ => None, }) .collect(); let text: String = events .iter() .filter_map(|event| match event { LogicalEvent::Text(t) => Some(t.as_str()), _ => None, }) .collect(); assert_eq!( reasoning, batch.reasoning, "{family} prefilled={prefilling} split={split}" ); assert_eq!( text, batch.text, "{family} prefilled={prefilling} split={split}" ); } } } } #[tokio::test] async fn all_family_guided_streaming_rollback_preserves_complete_calls() { for (family, _) in all_family_native_calls() { let constraint = GuidedToolConstraint::GuidedJsonRequired; let input = r#"[{"name":"get_weather","arguments":{"city":"Tokyo"}}]"#; let released = apply_stream_with_constraint( stream::iter([chunk(&input[..35], false), chunk(&input[35..], true)]), Some(weather_tools()), constraint.clone(), UnifiedParserStartingState::None, family, true, ) .collect::>() .await; let buffered = apply_stream_with_constraint( stream::iter([chunk(&input[..35], false), chunk(&input[35..], true)]), Some(weather_tools()), constraint, UnifiedParserStartingState::None, family, false, ) .collect::>() .await; assert_eq!( logical_events(&released), logical_events(&buffered), "{family}" ); } } #[tokio::test] async fn kimi_named_guided_call_does_not_leak_native_end_marker() { let tools = vec![ToolDefinition { name: "send_email".to_string(), parameters: Some( serde_json::json!({"type":"object", "properties":{"to":{"type":"string"},"subject":{"type":"string"},"body":{"type":"string"}}}), ), strict: None, }]; let input = r#"{"to":"user@example.com","subject":"Hello","body":"Hi"}<|tool_call_end|>"#; for by_character in [false, true] { for finish_reason in [FinishReason::Stop, FinishReason::Length] { let mut chunks: Vec<_> = if by_character { input .char_indices() .map(|(start, c)| chunk(&input[start..start + c.len_utf8()], false)) .collect() } else { vec![chunk(input, false)] }; let mut terminal = chunk("", true); terminal.data.as_mut().unwrap().inner.choices[0].finish_reason = Some(finish_reason); chunks.push(terminal); let output = apply_stream_with_constraint( stream::iter(chunks), Some(tools.clone()), GuidedToolConstraint::GuidedJsonNamed { tool_name: "send_email".to_string(), }, UnifiedParserStartingState::None, "kimi_k2", true, ) .collect::>() .await; let events = logical_events(&output); let text: String = events .iter() .filter_map(|event| match event { LogicalEvent::Text(t) => Some(t.as_str()), _ => None, }) .collect(); let names: Vec<_> = events .iter() .filter_map(|event| match event { LogicalEvent::Tool { name: Some(n), .. } => Some(n.as_str()), _ => None, }) .collect(); assert_eq!( names, vec!["send_email"], "{by_character} {finish_reason:?}" ); assert!( text.is_empty(), "{by_character} {finish_reason:?}: {text:?}" ); } } } #[test] fn invalid_parser_version_cannot_enable_unified_routing() { if crate::test_utils::run_isolated( concat!( module_path!(), "::invalid_parser_version_cannot_enable_unified_routing" ), &[( dynamo_runtime::config::environment_names::llm::DYN_PARSER_VERSION, "v3", )], ) { return; } assert!(std::panic::catch_unwind(|| selected_family(Some("qwen3"), None)).is_err()); } #[test] fn explicit_v1_rolls_back_unified_routes_in_an_isolated_process() { if crate::test_utils::run_isolated( concat!( module_path!(), "::explicit_v1_rolls_back_unified_routes_in_an_isolated_process" ), &[( dynamo_runtime::config::environment_names::llm::DYN_PARSER_VERSION, "1", )], ) { return; } for (family, _) in all_family_native_calls() { assert_eq!( selected_family(Some(family), Some(family)), None, "{family}" ); assert_eq!( selected_batch_family(Some(family), Some(family)), None, "{family}" ); } } #[test] fn spaced_kimi_think_prefill_and_response_closer_are_distinct() { assert_eq!(detect_prefill("kimi_k3", "hidden<|close|> think <|sep|><|open|> response <|sep|>answer<|close|> response <|sep|>").unwrap(), UnifiedParserStartingState::Reasoning); assert_eq!( detect_prefill("kimi_k3", "answer<|close|> response <|sep|>").unwrap(), UnifiedParserStartingState::None ); } #[test] fn every_family_batch_native_markup_overrides_reconstructed_guided_json() { for (family, input) in all_family_native_calls() { for constraint in [ GuidedToolConstraint::GuidedJsonNamed { tool_name: "get_weather".to_string(), }, GuidedToolConstraint::GuidedJsonRequired, ] { let parsed = parse_complete(family, input, &constraint, &weather_tools()).unwrap(); assert_eq!(parsed.tool_calls.len(), 1, "{family}"); assert_eq!( parsed.tool_calls[0].function.name, "get_weather", "{family}" ); assert_eq!( serde_json::from_str::( &parsed.tool_calls[0].function.arguments ) .unwrap(), serde_json::json!({"city":"Tokyo"}), "{family}" ); } } let (_, input) = all_family_native_calls() .into_iter() .find(|(family, _)| *family == "kimi_k3") .unwrap(); let spaced = input.replace("<|open|>call", "<|open|> call"); let parsed = parse_complete( "kimi_k3", &spaced, &GuidedToolConstraint::GuidedJsonRequired, &weather_tools(), ) .unwrap(); assert_eq!(parsed.tool_calls.len(), 1); } #[tokio::test] async fn guided_prefill_whitespace_is_decided_before_json_and_flushed_before_detours() { let constraint = GuidedToolConstraint::GuidedJsonNamed { tool_name: "get_weather".to_string(), }; for (family, _) in all_family_native_calls() { let output = apply_stream_with_constraint( stream::iter([ chunk("", false), chunk(" ", false), chunk(r#"{"city":"Tokyo"}"#, true), ]), Some(weather_tools()), constraint.clone(), UnifiedParserStartingState::Reasoning, family, true, ) .collect::>() .await; assert!(logical_events(&output).iter().any(|event| matches!(event, LogicalEvent::Tool { name: Some(name), .. } if name == "get_weather")), "{family}"); } let mut terminal = chunk("", true); terminal.data.as_mut().unwrap().inner.choices[0] .delta .content = None; terminal.data.as_mut().unwrap().inner.choices[0] .delta .reasoning_content = Some("done".to_string()); let output = apply_stream_with_constraint( stream::iter([chunk(" ", false), terminal]), Some(weather_tools()), constraint, UnifiedParserStartingState::Reasoning, "qwen3", true, ) .collect::>() .await; assert_eq!( logical_events(&output), vec![LogicalEvent::Reasoning(" done".to_string())] ); let terminal_position = output .iter() .position(|r| { r.data .as_ref() .is_some_and(|d| d.inner.choices.iter().any(|c| c.finish_reason.is_some())) }) .unwrap(); assert!( !output[terminal_position + 1..] .iter() .any(|r| r.data.as_ref().is_some_and(|d| d .inner .choices .iter() .any(|c| c.delta.reasoning_content.is_some()))) ); } #[test] fn all_registered_families_route_to_one_unified_owner() { for (family, _) in all_family_native_calls() { assert_eq!(configured_family(Some(family), Some(family)), Some(family)); assert!( create_unified_parser_for_family(family, &[]) .unwrap() .preserve_special_tokens() ); } assert_eq!( configured_family(Some("kimi_k2"), Some("kimi_k25")), Some("kimi_k2") ); assert_eq!(configured_family(Some("qwen3_coder"), Some("glm47")), None); } #[test] fn default_v2_routing_in_isolated_processes() { use dynamo_runtime::config::environment_names::llm::{ DYN_ENABLE_EXPERIMENTAL_PARSERS_V2, DYN_PARSER_VERSION, }; const CHILD: &str = "DYNAMO_DEFAULT_V2_ROUTING_CHILD"; let Ok(selection) = std::env::var(CHILD) else { for selection in ["unset", "auto", "1", "2", "deprecated_false"] { let mut command = crate::test_utils::isolated_command( "protocols::openai::chat_completions::unified_parser::tests::default_v2_routing_in_isolated_processes", ); command.env(CHILD, selection); match selection { "unset" => {} "deprecated_false" => { command.env(DYN_ENABLE_EXPERIMENTAL_PARSERS_V2, "false"); } value => { command.env(DYN_PARSER_VERSION, value); } } let output = command.output().expect("run isolated routing regression"); crate::test_utils::assert_isolated_success(&output); } return; }; // Qwen3 was previously left on V1 by both unset and auto. let expected = (selection != "1").then_some(QWEN3_UNIFIED_FAMILY); assert_eq!( selected_family(Some("qwen3_coder"), Some("qwen3")), expected ); assert_eq!( selected_family(None, Some("qwen3")), (selection == "2").then_some(QWEN3_UNIFIED_FAMILY) ); // Nemotron 3 shares Qwen's tool grammar: the explicit pair routes like Qwen3, // and a reasoning-only card stays legacy unless V2 is selected explicitly. assert_eq!( selected_family(Some("qwen3_coder"), Some("nemotron_v3")), (selection != "1").then_some(NEMOTRON_V3_UNIFIED_FAMILY) ); assert_eq!( selected_family(None, Some("nemotron_v3")), (selection == "2").then_some(NEMOTRON_V3_UNIFIED_FAMILY) ); for &parser in FAMILY_NAMES.iter() { let family = canonical_family(parser).unwrap(); for (tool, reasoning) in [ (Some(parser), Some(family)), (Some(parser), None), (None, Some(parser)), ] { let reasoning_only_shared_grammar = tool.is_none() && matches!(reasoning, Some("qwen3" | "nemotron_v3" | "nemotron3")); let expected = (selection != "1" && (selection == "2" || !reasoning_only_shared_grammar)) .then_some(family); assert_eq!( selected_family(tool, reasoning), expected, "{selection} {tool:?} {reasoning:?}" ); assert_eq!(selected_batch_family(tool, reasoning), expected); assert_eq!(selected_content_decoder_family(tool, reasoning), expected); } } for (tool, reasoning) in [ (Some("qwen3_coder"), Some("glm47")), (Some("qwen3_coder"), Some("deepseek_r1")), (Some("unknown_parser"), None), (None, Some("deepseek_r1")), (None, None), ] { assert_eq!(selected_family(tool, reasoning), None); assert_eq!(selected_batch_family(tool, reasoning), None); assert_eq!(selected_content_decoder_family(tool, reasoning), None); } } #[test] fn apostrophe_handling_preserves_quotes_contractions_and_possessives() { // A possessive apostrophe that ends a word (`James'`) has non-alphanumeric on // the RIGHT. Requiring alphanumeric on both sides treats it as a real // quote-open, and the later contraction apostrophe in `isn't` then closes that // phantom span, hiding the real `` marker in between. assert_eq!( detect_prefill( QWEN3_UNIFIED_FAMILY, "James' secret answer isn't final" ) .unwrap(), UnifiedParserStartingState::Reasoning, "a trailing possessive apostrophe must not be treated as a real quote-open" ); // Mirror case: a leading contraction (`'twas`) has non-alphanumeric on the LEFT // and alphanumeric only on the right — the OR-based fix must exempt this too. assert!( contains_unquoted_marker("'twas hidden", ""), "a leading-apostrophe contraction must not be treated as a real quote-open" ); assert!( contains_unquoted_marker("'twas didn't hidden", ""), "an apostrophe inside a later contraction must not close a leading contraction" ); assert!( !contains_unquoted_marker("'The literal marker' stays visible", ""), "a leading quote before a word must shield markers through its closing quote" ); assert!( !contains_unquoted_marker("'The word doesn't contain as syntax'", ""), "an internal contraction must not close a quoted span" ); } #[tokio::test] async fn already_parsed_first_chunk_with_no_prior_raw_still_terminates_correctly() { let mut already_parsed = chunk("", false); let choice = &mut already_parsed.data.as_mut().unwrap().inner.choices[0]; choice.delta.content = None; choice.delta.tool_calls = Some(vec![ChatCompletionMessageToolCallChunk { custom: None, index: 0, id: Some("call-preexisting".to_string()), r#type: Some(FunctionType::Function), function: Some(FunctionCallStream { name: Some("get_weather".to_string()), arguments: Some("{}".to_string()), }), }]); // This choice index never has a raw chunk at all — no ChoiceState is ever // constructed for it — then the stream ends immediately. let responses = apply_stream( stream::iter([already_parsed]), Some(weather_tools()), None, false, UnifiedParserStartingState::None, QWEN3_UNIFIED_FAMILY, ) .collect::>() .await; let choices = collect_choices(&responses); assert_eq!( choices .iter() .filter_map(|choice| choice.finish_reason) .collect::>(), vec![FinishReason::ToolCalls], "a choice whose only-ever chunk was already-parsed and carried a tool \ call must still get exactly one ToolCalls terminal at EOF" ); } #[test] fn quoted_control_marker_probe_masks_closed_quotes_and_flushes_unclosed_quotes() { for family in [ "deepseek_v4", "deepseek_v41", "gemma4", "glm47", "kimi_k2", "kimi_k3", "muse_glimmer", "qwen3", ] { let marker = unified_parser_control_markers(family) .into_iter() .next() .unwrap(); for quote in ['"', '\'', '`'] { let literal = format!("The literal {quote}{marker}{quote} stays visible."); for split in literal .char_indices() .map(|(offset, _)| offset) .chain(std::iter::once(literal.len())) { let mut shield = QuotedControlMarkerProbe::new(family); let mut masked = shield.push(&literal[..split]); masked.push_str(&shield.push(&literal[split..])); masked.push_str(&shield.finish()); assert!( !masked.contains(&marker), "{family} quote {quote} split {split}" ); assert_eq!(masked.len(), literal.len(), "{family} split {split}"); } } let literal = format!("The literal \"{marker}"); let mut unfinished = QuotedControlMarkerProbe::new(family); let mut masked = unfinished.push(&literal); masked.push_str(&unfinished.finish()); assert!(masked.contains(&marker), "{family} EOF"); assert_eq!(masked, literal, "{family} EOF"); } let literal = r#"The literal "escaped \" closer " remains visible."#; for split in literal .char_indices() .map(|(offset, _)| offset) .chain(std::iter::once(literal.len())) { let mut shield = QuotedControlMarkerProbe::new(QWEN3_UNIFIED_FAMILY); let mut masked = shield.push(&literal[..split]); masked.push_str(&shield.push(&literal[split..])); masked.push_str(&shield.finish()); assert!(!masked.contains(""), "split {split}"); assert_eq!(masked.len(), literal.len(), "split {split}"); } } #[test] fn single_quoted_prose_shields_a_marker_after_its_opening_word() { for family in [ "deepseek_v4", "deepseek_v41", "gemma4", "glm47", "kimi_k2", "kimi_k3", "muse_glimmer", "qwen3", ] { for marker in unified_parser_control_markers(family) { for (literal, should_be_shielded) in [ ( format!("'The literal {marker} marker' stays visible."), true, ), ( format!("'The word doesn't contain {marker} as syntax'"), true, ), (format!("'twas {marker} must remain unshielded"), false), ] { for split in literal .char_indices() .map(|(offset, _)| offset) .chain(std::iter::once(literal.len())) { let mut shield = QuotedControlMarkerProbe::new(family); let mut masked = shield.push(&literal[..split]); let second_output = shield.push(&literal[split..]); masked.push_str(&second_output); masked.push_str(&shield.finish()); assert_eq!( masked.contains(&marker), !should_be_shielded, "{family} marker {marker:?} split {split}, literal {literal:?}, masked {masked:?}" ); assert_eq!(masked.len(), literal.len(), "{family} split {split}"); } } } } } fn engine_like_chunks(raw: &str) -> Vec { // Engine-like chunks: special tokens alone, other text in short pieces. let mut out = Vec::new(); let mut rest = raw; while !rest.is_empty() { if rest.starts_with("<|") { let end = rest.find("|>").unwrap() + 2; out.push(rest[..end].to_string()); rest = &rest[end..]; } else { let next = rest.find("<|").unwrap_or(rest.len()); let mut piece_end = next.min(3); while !rest.is_char_boundary(piece_end) { piece_end += 1; } out.push(rest[..piece_end].to_string()); rest = &rest[piece_end..]; } } out } async fn muse_spark_structured_stream( raw_chunks: &[String], policy: UnifiedRequestPolicy, ) -> (String, String) { let last = raw_chunks.len() - 1; let outputs: Vec<_> = apply_stream_with_policy( futures::stream::iter( raw_chunks .iter() .enumerate() .map(|(i, c)| chunk(c, i == last)) .collect::>(), ), None, GuidedToolConstraint::None, UnifiedParserStartingState::None, "muse_spark", true, policy, ) .collect() .await; let mut text = String::new(); let mut reasoning = String::new(); for choice in collect_choices(&outputs) { if let Some(ChatCompletionMessageContent::Text(t)) = &choice.delta.content { text.push_str(t); } if let Some(r) = &choice.delta.reasoning_content { reasoning.push_str(r); } } (text, reasoning) } #[tokio::test] async fn muse_spark_structured_response_bare_and_final_json_stay_content() { let policy = UnifiedRequestPolicy { structured_response: true, ..Default::default() }; let cases = [ ("{\"name\":\"John Doe\"}", "{\"name\":\"John Doe\"}", ""), ("true", "true", ""), ( " to=user<|message|>{\"name\":\"John Doe\"}<|eot|>", "{\"name\":\"John Doe\"}", "", ), ]; let mut failures = Vec::new(); for (raw, want_text, want_reasoning) in cases { for (name, chunks) in [ ("whole", vec![raw.to_string()]), ("tokenish", engine_like_chunks(raw)), ( "per-char", raw.chars().map(|c| c.to_string()).collect::>(), ), ] { let (text, reasoning) = muse_spark_structured_stream(&chunks, policy).await; if text != want_text || reasoning != want_reasoning { failures.push(format!( "{name} {raw:?}: text={text:?} reasoning={reasoning:?}" )); } } } assert!(failures.is_empty(), "{failures:#?}"); } #[tokio::test] async fn muse_spark_structured_response_separates_recipient_header_reasoning() { let raws = [ " to=self<|message|>Need the name.<|eom|><|start|>assistant to=user<|message|>{\"name\":\"John Doe\"}<|eot|>", " to=self<|message|>Need the name.<|eom|>{\"name\":\"John Doe\"}<|eot|>", ]; let mut failures = Vec::new(); for raw in raws { for structured_response in [false, true] { let policy = UnifiedRequestPolicy { structured_response, ..Default::default() }; let batch = parse_complete_with_policy( "muse_spark", raw, &GuidedToolConstraint::None, &[], policy, ) .unwrap(); assert_eq!(batch.text, "{\"name\":\"John Doe\"}"); let chunkings = [ ("whole", vec![raw.to_string()]), ("tokenish", engine_like_chunks(raw)), ("per-char", raw.chars().map(|c| c.to_string()).collect()), ]; for (name, chunks) in chunkings { let (text, reasoning) = muse_spark_structured_stream(&chunks, policy).await; if text != "{\"name\":\"John Doe\"}" || reasoning != "Need the name." { failures.push(format!( "structured_response={structured_response} {name} raw={raw:?}: text={text:?} reasoning={reasoning:?}" )); } } } } assert!(failures.is_empty(), "{failures:#?}"); } #[tokio::test] async fn structured_response_json_stays_content_at_every_split_for_all_families_and_choices() { let policy = UnifiedRequestPolicy { structured_response: true, ..Default::default() }; for prefill in [ UnifiedParserStartingState::None, UnifiedParserStartingState::Reasoning, ] { for family in dynamo_parsers_v2::REGISTERED_UNIFIED_FAMILIES { for raw in [ " \n{\"country\":\"France\"}", "[{\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}]", "true", "false", "null", "42", "-1.25e2", "\"literal text\"", ] { let batch = parse_complete_with_policy( family, raw, &GuidedToolConstraint::None, &weather_tools(), policy, ) .unwrap(); assert_eq!(batch.text, raw, "{family} batch"); assert!(batch.reasoning.is_empty()); assert!(batch.tool_calls.is_empty()); for split in 0..=raw.len() { let mut other_prefix = chunk(" \n", false); other_prefix.data.as_mut().unwrap().inner.choices[0].index = 1; let mut other_body = chunk(raw, true); other_body.data.as_mut().unwrap().inner.choices[0].index = 1; let outputs: Vec<_> = apply_stream_with_policy( futures::stream::iter([ chunk(&raw[..split], false), other_prefix, chunk(&raw[split..], true), other_body, ]), Some(weather_tools()), GuidedToolConstraint::None, prefill, family, true, policy, ) .collect() .await; for index in [0, 1] { let choices: Vec<_> = collect_choices(&outputs) .into_iter() .filter(|choice| choice.index == index) .collect(); let text: String = choices .iter() .filter_map(|choice| match &choice.delta.content { Some(ChatCompletionMessageContent::Text(text)) => { Some(text.as_str()) } _ => None, }) .collect(); assert_eq!( text, if index == 0 { raw.to_string() } else { format!(" \n{raw}") }, "{family} split {split} choice {index}" ); assert!( choices.iter().all(|choice| choice .delta .reasoning_content .is_none() && choice.delta.tool_calls.is_none()), "{family} split {split} choice {index}" ); } } let outputs: Vec<_> = apply_stream_with_policy( futures::stream::iter( raw.chars() .map(|c| chunk(&c.to_string(), false)) .collect::>(), ), None, GuidedToolConstraint::None, prefill, family, true, policy, ) .collect() .await; let text: String = collect_choices(&outputs) .iter() .filter_map(|choice| match &choice.delta.content { Some(ChatCompletionMessageContent::Text(text)) => Some(text.as_str()), _ => None, }) .collect(); assert_eq!(text, raw, "{family} EOF without finish"); assert!( collect_choices(&outputs) .iter() .all(|choice| choice.delta.reasoning_content.is_none()) ); } } } } #[tokio::test] async fn structured_unopened_channel_prefix_flush_and_progress() { let policy = UnifiedRequestPolicy { structured_response: true, ..Default::default() }; for raw in [" \n", "\"unterminated ", "{", "[", "42"] { for ending in 0..4 { let mut input = vec![chunk(raw, false)]; match ending { 0 => {} // Raw EOF. 1 => input.push(chunk("", true)), 2 => { let mut usage = chunk("", false); usage.data.as_mut().unwrap().inner.choices.clear(); input.push(usage); } _ => { let mut parsed = chunk("", true); let delta = &mut parsed.data.as_mut().unwrap().inner.choices[0].delta; delta.content = None; delta.reasoning_content = Some("upstream".into()); input.push(parsed); } } let output = apply_stream_with_policy( futures::stream::iter(input), None, GuidedToolConstraint::None, UnifiedParserStartingState::None, "qwen3", true, policy, ) .collect::>() .await; let choices = collect_choices(&output); let text: String = choices .iter() .filter_map(|c| match &c.delta.content { Some(ChatCompletionMessageContent::Text(text)) => Some(text.as_str()), _ => None, }) .collect(); let reasoning: String = choices .iter() .filter_map(|c| c.delta.reasoning_content.as_deref()) .collect(); assert_eq!(text, raw, "ending {ending}"); assert_eq!(reasoning, if ending == 3 { "upstream" } else { "" }); } } for raw in ["{", "[", "\"partial", "4", "t", "n", "f"] { let output = apply_stream_with_policy( futures::stream::iter([chunk(raw, false), chunk("", true)]), None, GuidedToolConstraint::None, UnifiedParserStartingState::None, "qwen3", true, policy, ) .take(1) .collect::>() .await; let text: String = collect_choices(&output) .iter() .filter_map(|c| match &c.delta.content { Some(ChatCompletionMessageContent::Text(text)) => Some(text.as_str()), _ => None, }) .collect(); assert_eq!(text, raw, "JSON prefix must progress before terminal"); } } #[test] fn scalar_probe_inspects_each_byte_once_and_resolves_all_release_paths() { let raw = format!("\"{}\"", "x".repeat(65536)); let mut probe = JsonPrefillProbe::default(); for byte in raw.bytes() { probe.push(std::str::from_utf8(&[byte]).unwrap()); assert!(probe.decision(false).is_none()); } assert_eq!(probe.inspected_bytes, raw.len()); assert_eq!(probe.decision(true), Some(UnifiedParserStartingState::None)); let policy = UnifiedRequestPolicy { structured_response: true, ..Default::default() }; let resolved = resolve_guided_prefix(7, probe, policy, UnifiedParserStartingState::Reasoning).unwrap(); assert_eq!( resolved.delta.content, Some(ChatCompletionMessageContent::Text(raw)) ); assert!(resolved.delta.reasoning_content.is_none()); for raw in ["42", "true", "null", "\"answer\""] { let mut probe = JsonPrefillProbe::default(); probe.push(raw); assert!(probe.decision(false).is_none()); let resolved = resolve_guided_prefix(0, probe, policy, UnifiedParserStartingState::Reasoning) .unwrap(); assert_eq!( resolved.delta.content, Some(ChatCompletionMessageContent::Text(raw.into())) ); } let mut thought = JsonPrefillProbe::default(); thought.push("42"); assert!(thought.decision(false).is_none()); thought.push(" reasons"); assert_eq!( thought.decision(false), Some(UnifiedParserStartingState::Reasoning) ); assert_eq!(thought.inspected_bytes, 10); } #[test] fn json_prefill_marker_detection_has_linear_work_for_quoted_controls() { for payload in ["|", "<", ">", "", ""] { let raw = format!("[\"{}\"]", payload.repeat(4096)); let mut probe = JsonPrefillProbe::default(); for character in raw.chars() { let text = character.to_string(); probe.push(&text); assert_ne!( probe .decision_for_family(QWEN3_UNIFIED_FAMILY, false, &text) .unwrap(), Some(UnifiedParserStartingState::Reasoning), ); } assert_eq!( probe .decision_for_family(QWEN3_UNIFIED_FAMILY, true, "") .unwrap(), Some(UnifiedParserStartingState::None), ); let shield_work = probe.marker_probe.as_ref().unwrap().inspected_bytes; assert!( probe.inspected_bytes + probe.marker_inspected_bytes + shield_work <= raw.len() * 64, "payload {payload:?}: JSON {}, marker {}, shield {}, input {}", probe.inspected_bytes, probe.marker_inspected_bytes, shield_work, raw.len(), ); } } #[test] fn structured_scalar_prefix_keeps_real_reasoning_before_json() { for thought in ["42 reasons", "\"a thought\"", "true", "null hypothesis"] { let full = format!("{thought}\"answer\""); for split in 0..=full.len() { let prefix = &full[..split]; if prefix.contains("") { assert_eq!( structured_json_prefill(QWEN3_UNIFIED_FAMILY, prefix, false).unwrap(), Some(UnifiedParserStartingState::Reasoning), "split {split}" ); } } assert_eq!( structured_json_prefill(QWEN3_UNIFIED_FAMILY, &full, true).unwrap(), Some(UnifiedParserStartingState::Reasoning) ); } } #[tokio::test] async fn structured_response_literal_native_markers_match_batch_and_stream() { let policy = UnifiedRequestPolicy { structured_response: true, ..Default::default() }; let tools = weather_tools(); let literal_marker_json = r#"{"note":"literal"}"#; let batch = parse_complete_with_policy( "qwen3", literal_marker_json, &GuidedToolConstraint::None, &tools, policy, ) .unwrap(); assert_eq!(batch.text, literal_marker_json); assert!(batch.reasoning.is_empty()); let streamed = apply_stream_with_policy( futures::stream::iter(vec![chunk(literal_marker_json, true)]), Some(tools.clone()), GuidedToolConstraint::None, stream_prefill("qwen3", false), "qwen3", false, policy, ) .collect::>() .await; let streamed_choices = collect_choices(&streamed); let streamed_text: String = streamed_choices .iter() .filter_map(|choice| match &choice.delta.content { Some(ChatCompletionMessageContent::Text(text)) => Some(text.as_str()), _ => None, }) .collect(); let streamed_reasoning: String = streamed_choices .iter() .filter_map(|choice| choice.delta.reasoning_content.as_deref()) .collect(); assert_eq!(streamed_text, literal_marker_json); assert!(streamed_reasoning.is_empty()); } #[tokio::test] async fn structured_response_reasoning_prefill_starting_with_container_matches_batch_and_stream() { let policy = UnifiedRequestPolicy { structured_response: true, ..Default::default() }; let tools = weather_tools(); let reasoning_then_json = "{private thought}{\"answer\":42}"; let batch = parse_complete_with_policy( "qwen3", reasoning_then_json, &GuidedToolConstraint::None, &tools, policy, ) .unwrap(); assert_eq!(batch.reasoning, "{private thought}"); assert_eq!(batch.text, "{\"answer\":42}"); let streamed = apply_stream_with_policy( futures::stream::iter(vec![chunk(reasoning_then_json, true)]), Some(tools.clone()), GuidedToolConstraint::None, stream_prefill("qwen3", true), "qwen3", false, policy, ) .collect::>() .await; let streamed_choices = collect_choices(&streamed); let streamed_text: String = streamed_choices .iter() .filter_map(|choice| match &choice.delta.content { Some(ChatCompletionMessageContent::Text(text)) => Some(text.as_str()), _ => None, }) .collect(); let streamed_reasoning: String = streamed_choices .iter() .filter_map(|choice| choice.delta.reasoning_content.as_deref()) .collect(); assert_eq!( streamed_reasoning, "{private thought}", "streamed content was {streamed_text:?}" ); assert_eq!(streamed_text, "{\"answer\":42}"); } #[test] fn malformed_guided_zero_call_native_channels_recover_for_every_family() { for (family, raw) in [ ("deepseek_v4", "privateanswer"), ("deepseek_v41", "privateanswer"), ("qwen3", "privateanswer"), ("glm47", "privateanswer"), ("kimi_k2", "privateanswer"), ("gemma4", "<|channel>thought\nprivateanswer"), ( "kimi_k3", "<|open|>think<|sep|>private<|close|>think<|sep|><|open|>response<|sep|>answer<|close|>response<|sep|>", ), ( "muse_glimmer", "<|start|>assistant to=self<|message|>private<|eom|><|start|>assistant to=user<|message|>answer<|eot|>", ), ] { for constraint in [ GuidedToolConstraint::GuidedJsonRequired, GuidedToolConstraint::GuidedJsonNamed { tool_name: "get_weather".to_string(), }, ] { let batch = parse_complete_with_policy( family, raw, &constraint, &weather_tools(), UnifiedRequestPolicy::default(), ) .unwrap(); assert_eq!(batch.reasoning, "private", "{family} {constraint:?}"); assert_eq!(batch.text, "answer", "{family} {constraint:?}"); assert!(batch.tool_calls.is_empty()); let quoted = serde_json::to_string(raw).unwrap(); assert!( !contains_native_envelope("ed, family), "{family} quoted control" ); assert!( parse_complete_with_policy( family, "ed, &constraint, &weather_tools(), UnifiedRequestPolicy::default() ) .is_err(), "{family} malformed JSON string must keep its error" ); } } } #[tokio::test] async fn disabled_gemma_preserves_thought_delimiters_around_native_calls_at_every_split() { let prefix = "<|channel>thought\nbefore"; let suffix = "after<|channel>thought\nlaterdone"; let raw = format!( "{prefix}<|tool_call>call:get_weather{{city:<|\"|>Paris<|\"|>}}{suffix}" ); let policy = UnifiedRequestPolicy { reasoning_disabled: true, ..Default::default() }; let batch = parse_complete_with_policy( "gemma4", &raw, &GuidedToolConstraint::None, &weather_tools(), policy, ) .unwrap(); assert_eq!(batch.text, format!("{prefix}{suffix}")); assert!(batch.reasoning.is_empty()); assert_eq!(batch.tool_calls.len(), 1); assert_eq!(batch.tool_calls[0].function.name, "get_weather"); assert_eq!( serde_json::from_str::(&batch.tool_calls[0].function.arguments) .unwrap(), serde_json::json!({"city":"Paris"}) ); for split in 0..=raw.len() { let outputs: Vec<_> = apply_stream_with_policy( futures::stream::iter([chunk(&raw[..split], false), chunk(&raw[split..], true)]), Some(weather_tools()), GuidedToolConstraint::None, UnifiedParserStartingState::None, "gemma4", true, policy, ) .collect() .await; let choices = collect_choices(&outputs); let text: String = choices .iter() .filter_map(|choice| match &choice.delta.content { Some(ChatCompletionMessageContent::Text(text)) => Some(text.as_str()), _ => None, }) .collect(); assert_eq!(text, format!("{prefix}{suffix}"), "split {split}"); assert!( choices .iter() .all(|choice| choice.delta.reasoning_content.is_none()), "split {split}" ); assert_eq!( choices .iter() .flat_map(|choice| choice.delta.tool_calls.iter().flatten()) .filter(|call| call.id.is_some()) .count(), 1, "split {split}" ); let first_call = choices .iter() .position(|choice| choice.delta.tool_calls.is_some()) .unwrap(); let before_call: String = choices[..first_call] .iter() .filter_map(|choice| match &choice.delta.content { Some(ChatCompletionMessageContent::Text(text)) => Some(text.as_str()), _ => None, }) .collect(); assert_eq!(before_call, prefix, "split {split}: call ordering"); } } #[test] fn review_completion_is_monotonic_after_whitespace_delta() { let mut state = test_state(); state.delta_to_choice(0, call_delta(0, Some("get_weather"), "{}")); state.delta_to_choice(0, partial_call_delta(0, None, " ")); assert_eq!(state.parser_call_completion.get(&0), Some(&true)); assert!(state.pending_tool_calls.is_empty()); let mut record = ChoiceRecord::default(); record.remember_state(&state); assert_eq!(record.parser_call_completion.get(&0), Some(&true)); assert_eq!( record.normalize_finish_reason(Some(FinishReason::Stop)), Some(FinishReason::ToolCalls) ); assert_eq!( state.normalize_finish_reason(Some(FinishReason::Stop)), Some(FinishReason::ToolCalls) ); } #[tokio::test] async fn review_named_guided_whitespace_preserves_completion_at_every_split() { let raw = r#"{"city":"Tokyo"}"#; for (family, _) in all_family_native_calls() { for split in 0..=raw.len() { let output = apply_stream_with_constraint( stream::iter([ chunk(&raw[..split], false), chunk(&raw[split..], false), chunk(" ", false), chunk("\n", true), ]), Some(weather_tools()), GuidedToolConstraint::GuidedJsonNamed { tool_name: "get_weather".into(), }, UnifiedParserStartingState::None, family, true, ) .collect::>() .await; let choices = collect_choices(&output); assert_eq!( choices.last().unwrap().finish_reason, Some(FinishReason::ToolCalls), "{family} split {split}" ); let arguments = choices .iter() .flat_map(|choice| choice.delta.tool_calls.iter().flatten()) .filter_map(|call| call.function.as_ref()?.arguments.as_deref()) .collect::(); assert_eq!( serde_json::from_str::(&arguments).unwrap(), serde_json::json!({"city":"Tokyo"}), "{family} split {split}" ); } } } #[test] fn review_provisional_sibling_exposes_truncation_in_live_and_history_states() { for completed_sibling in [false, true] { let mut state = test_state(); if completed_sibling { state.delta_to_choice(0, call_delta(0, Some("first"), "{}")); } state.delta_to_choice(0, partial_call_delta(1, Some("unfinished"), "{")); let mut record = ChoiceRecord::default(); record.remember_state(&state); for finish in [ None, Some(FinishReason::Stop), Some(FinishReason::ToolCalls), Some(FinishReason::Length), Some(FinishReason::ContentFilter), ] { let expected = match finish { Some(FinishReason::Stop | FinishReason::ToolCalls) => { Some(FinishReason::Length) } other => other, }; assert_eq!(state.normalize_finish_reason(finish), expected); assert_eq!(record.normalize_finish_reason(finish), expected); } assert_eq!( state.unterminated_finish_reason(), Some(FinishReason::Length) ); let mut records = HashMap::from([(0, record)]); let chunks = finish_unterminated_choices( &mut HashMap::new(), &mut records, QWEN3_UNIFIED_FAMILY, ); assert!( matches!(&chunks[..], [FinishedStreamChunk::Choice(choice)] if choice.finish_reason == Some(FinishReason::Length)) ); } } #[tokio::test] async fn review_default_on_backend_malformed_arguments_keep_flag_off_finish() { for family in [DEEPSEEK_V41_UNIFIED_FAMILY, MUSE_UNIFIED_FAMILY] { for arguments in ["", "{"] { for terminal in [false, true] { let mut input = chunk("", terminal); let delta = &mut input.data.as_mut().unwrap().inner.choices[0].delta; delta.content = None; delta.tool_calls = Some(vec![ChatCompletionMessageToolCallChunk { custom: None, index: 0, id: Some("backend-call".into()), r#type: Some(FunctionType::Function), function: Some(FunctionCallStream { name: Some("get_weather".into()), arguments: Some(arguments.into()), }), }]); let output = apply_stream( stream::iter([input]), Some(weather_tools()), None, false, UnifiedParserStartingState::None, family, ) .collect::>() .await; let choices = collect_choices(&output); let expected = if parsers_v2_selected() { FinishReason::Length } else { FinishReason::ToolCalls }; assert_eq!( choices.last().unwrap().finish_reason, Some(expected), "{family} args={arguments:?} terminal={terminal}" ); } } } } #[test] fn review_glm_long_unfinished_argument_retains_truncation() { let (family, input) = eight_family_calls(&"x".repeat(8000)) .into_iter() .find(|(family, _)| *family == GLM47_UNIFIED_FAMILY) .unwrap(); let end = input.rfind('x').unwrap() + 1; let mut state = ChoiceState::new( family, &to_v2_tools(Some(&write_file_tools())), UnifiedParserStartingState::None, UnifiedToolOutputMode::Native, true, ArgumentFraming::default(), ) .unwrap(); let started = std::time::Instant::now(); for character in input[..end].chars() { state.push(&character.to_string()); } eprintln!( "GLM 8000-byte unfinished call, character cadence: {:?}", started.elapsed() ); let events = state.finish(); state.choices_for(&empty_choice(0), events, true, None); assert_eq!( state.unterminated_finish_reason(), Some(FinishReason::Length) ); } #[test] fn review_glm_pending_markers_keep_prose_quotes_and_native_payload_ownership() { let complete = r#"write_filecontentsay "hi"#; let incomplete = r#"write_filecontent"unfinished"#; for input in [ format!("{complete}{incomplete}"), format!("{complete} The literal \"\" stays visible. {incomplete}"), "write_filecontentliteral text" .to_string(), ] { let expected_start = if input.starts_with(complete) { input.rfind(incomplete).unwrap() } else { 0 }; assert_eq!(glm_unfinished_call(&input).unwrap(), Some(expected_start)); for split in input .char_indices() .map(|(at, _)| at) .chain(std::iter::once(input.len())) { let mut state = ChoiceState::new( GLM47_UNIFIED_FAMILY, &to_v2_tools(Some(&write_file_tools())), UnifiedParserStartingState::None, UnifiedToolOutputMode::Native, true, ArgumentFraming::default(), ) .unwrap(); state.push(&input[..split]); state.push(&input[split..]); state.finish(); assert!(state.incomplete_native_call, "split {split}: {input}"); } } for input in [ r#"The literal "" stays visible."#.to_string(), format!("{complete} The literal \"\" stays visible."), ] { assert_eq!(glm_unfinished_call(&input).unwrap(), None); } } } #[cfg(test)] mod beam_routing_tests { use super::*; use dynamo_runtime::config::ParserVersion; #[tokio::test] async fn beam_finalization_respects_grammar_and_confirmed_eos() { let tools = vec![ToolDefinition { name: "weather".into(), parameters: Some( serde_json::json!({"type": "object", "properties": {"city": {"type": "string"}}}), ), strict: None, }]; let body = "<|start|>assistant to=weather<|channel|>analysis <|constrain|>xml<|message|>Paris"; for (suffix, finish, eos, held, succeeds) in [ ("", Some(FinishReason::Stop), true, true, true), ("<|return|>", Some(FinishReason::Stop), true, true, true), ("", Some(FinishReason::Stop), false, true, false), ("", Some(FinishReason::Stop), true, false, true), ("", Some(FinishReason::Stop), false, false, true), ("", Some(FinishReason::Length), true, true, false), ("", None, false, true, false), ("<|call|>", Some(FinishReason::Stop), false, false, true), ("<|call|>", Some(FinishReason::Length), false, true, true), ( "", Some(FinishReason::Stop), true, true, false, ), ] { let constraint = if held { GuidedToolConstraint::BeamTurnGrammar { frames_answer: false, } } else { GuidedToolConstraint::None }; let input = format!("{body}{suffix}"); let raw: NvCreateChatCompletionStreamResponse = serde_json::from_value(serde_json::json!({ "id": "test", "object": "chat.completion.chunk", "model": "beam", "created": 0, "choices": [{"index": 0, "delta": {"content": input}, "finish_reason": finish}], "model_eos": if eos { vec![0] } else { vec![] }, })) .unwrap(); let output: Vec<_> = apply_stream_with_constraint( futures::stream::iter([Annotated::from_data(raw)]), Some(tools.clone()), constraint.clone(), UnifiedParserStartingState::None, BEAM_UNIFIED_FAMILY, true, ) .collect() .await; assert_eq!( !output.iter().any(|item| item.is_error()), succeeds, "{suffix:?}, {finish:?}, eos={eos}, held={held}" ); if succeeds { let choices: Vec<_> = output .iter() .flat_map(|item| item.data.iter().flat_map(|data| &data.inner.choices)) .collect(); let arguments: String = choices .iter() .flat_map(|choice| choice.delta.tool_calls.iter().flatten()) .filter_map(|call| call.function.as_ref()?.arguments.as_deref()) .collect(); assert_eq!( serde_json::from_str::(&arguments).unwrap(), serde_json::json!({"city": "Paris"}) ); assert_eq!( choices.last().unwrap().finish_reason, Some(if finish == Some(FinishReason::Length) { FinishReason::Length } else { FinishReason::ToolCalls }) ); } // Length and EOF without a suffix duplicate Stop without EOS in unary parsing. if finish == Some(FinishReason::Stop) || !suffix.is_empty() { let batch = parse_complete_with_eos( BEAM_UNIFIED_FAMILY, &input, &constraint, &tools, UnifiedRequestPolicy::default(), eos && finish == Some(FinishReason::Stop), ); assert_eq!(batch.is_ok(), succeeds); } } } /// The turn grammar writes a held tool's non-string parameters as JSON, so a closing /// tag inside one of their strings is data; and it holds a `response_format` answer /// in the final channel behind the analysis, so that output is a native turn. #[test] fn beam_turn_grammar_frames_held_json_arguments_and_the_structured_answer() { let tools = vec![ToolDefinition { name: "render".into(), parameters: Some(serde_json::json!({ "type": "object", "properties": { "doc": {"type": "object", "properties": {"html": {"type": "string"}}}, "title": {"type": "string"} }, "required": ["doc", "title"] })), strict: None, }]; let content = concat!( "<|channel|>analysis<|message|>Rendering.<|end|>", "<|start|>assistant to=render<|channel|>analysis <|constrain|>xml<|message|>", "{\"html\":\"x inside\"}", "t<|call|>" ); let expected = serde_json::json!({ "doc": {"html": "x inside"}, "title": "t" }); let held = GuidedToolConstraint::BeamTurnGrammar { frames_answer: false, }; let batch = parse_complete_with_policy( BEAM_UNIFIED_FAMILY, content, &held, &tools, UnifiedRequestPolicy::default(), ) .unwrap(); assert_eq!(batch.reasoning, "Rendering."); assert_eq!(batch.tool_calls.len(), 1, "{batch:?}"); assert_eq!(batch.tool_calls[0].function.name, "render"); assert_eq!( serde_json::from_str::(&batch.tool_calls[0].function.arguments) .unwrap(), expected ); // Without the grammar a value runs to the first closing tag. let lenient = parse_complete_with_policy( BEAM_UNIFIED_FAMILY, content, &GuidedToolConstraint::None, &tools, UnifiedRequestPolicy::default(), ) .unwrap(); assert!( lenient.tool_calls.first().is_none_or(|call| { serde_json::from_str::(&call.function.arguments).ok() != Some(expected.clone()) }), "{lenient:?}" ); // Streaming frames the same way at every split. let v2_tools = to_v2_tools(Some(&tools)); for split in content .char_indices() .map(|(at, _)| at) .chain(std::iter::once(content.len())) { let mut state = ChoiceState::new( BEAM_UNIFIED_FAMILY, &v2_tools, UnifiedParserStartingState::None, UnifiedToolOutputMode::Native, true, argument_framing(&held), ) .unwrap(); let mut wire = Vec::new(); for events in [ state.push(&content[..split]), state.push(&content[split..]), state.finish(), ] { wire.extend(state.choices_for(&empty_choice(0), events, true, None)); } let calls: Vec<_> = wire .iter() .flat_map(|choice| choice.delta.tool_calls.iter().flatten()) .collect(); let arguments: String = calls .iter() .filter_map(|call| call.function.as_ref()?.arguments.as_deref()) .collect(); assert_eq!( serde_json::from_str::(&arguments).unwrap(), expected, "split {split}" ); let reasoning: String = wire .iter() .filter_map(|choice| choice.delta.reasoning_content.as_deref()) .collect(); assert_eq!(reasoning, "Rendering.", "split {split}"); } // A `response_format` answer the grammar framed keeps the reasoning and the // channel structure, and is not probed as a bare JSON document. let answer = concat!( "<|channel|>analysis<|message|>Thinking.<|end|>", "<|start|>assistant<|channel|>final<|message|>{\"ok\":true}<|return|>" ); let structured = UnifiedRequestPolicy { reasoning_disabled: false, structured_response: true, ..Default::default() }; let framed = parse_complete_with_policy( BEAM_UNIFIED_FAMILY, answer, &GuidedToolConstraint::BeamTurnGrammar { frames_answer: true, }, &[], structured, ) .unwrap(); assert_eq!(framed.reasoning, "Thinking."); assert_eq!(framed.text, "{\"ok\":true}"); assert!(framed.tool_calls.is_empty()); } #[test] fn beam_pair_routes_to_the_unified_parser_without_an_explicit_generation() { assert_eq!( configured_family(Some("beam"), Some("beam")), Some(BEAM_UNIFIED_FAMILY) ); assert_eq!( selected_family_for_selection( Some(BEAM_UNIFIED_FAMILY), Some(BEAM_UNIFIED_FAMILY), ParserVersion::Auto, ), Some(BEAM_UNIFIED_FAMILY) ); assert_eq!( selected_family_for_selection( Some(BEAM_UNIFIED_FAMILY), Some(BEAM_UNIFIED_FAMILY), ParserVersion::V2, ), Some(BEAM_UNIFIED_FAMILY) ); // There is no v1 Beam parser to fall back to: an explicit `1` selects nothing // here (and `validate_parser_version` rejects it), and half a pair under `auto` // is rejected by the same validation before this choice is made. assert_eq!( selected_family_for_selection( Some(BEAM_UNIFIED_FAMILY), Some(BEAM_UNIFIED_FAMILY), ParserVersion::V1, ), None ); assert!(FAMILY_NAMES.contains(&BEAM_UNIFIED_FAMILY)); } #[test] fn beam_prefill_reads_the_channel_the_prompt_opened() { assert_eq!( beam_prefill( "<|channel|>analysis<|message|>t<|end|><|start|>assistant<|channel|>final<|message|>a<|return|>" ), UnifiedParserStartingState::None ); assert_eq!( beam_prefill( "<|start|>assistant to=get_weather<|channel|>analysis <|constrain|>xml<|message|><|call|>" ), UnifiedParserStartingState::None ); assert_eq!( beam_prefill("thought<|end|><|start|>assistant<|channel|>final<|message|>a<|return|>"), UnifiedParserStartingState::Reasoning ); assert_eq!( beam_prefill("answer<|return|>"), UnifiedParserStartingState::Response ); assert_eq!( beam_prefill("plain decoded answer"), UnifiedParserStartingState::Response ); assert_eq!( detect_prefill("beam", "plain").unwrap(), UnifiedParserStartingState::Response ); } #[test] fn beam_control_markers_cover_the_whole_grammar() { let markers = unified_parser_control_markers("beam"); for marker in [ "<|start|>", "<|channel|>", "<|message|>", "<|constrain|>", "<|end|>", "<|return|>", "<|call|>", ] { assert!(markers.iter().any(|m| m == marker), "{marker}"); } } }