// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
//! Ordered reasoning, text, and tool-call parsing through one state machine.
//!
//! Compatible configured families use this path by default; `DYN_PARSER_VERSION=1`
//! selects the legacy parsers when available.
//!
//! # What this replaces
//!
//! The legacy Qwen3 route chains two independent parsers: the reasoning
//! parser strips `...` across the whole stream into one assembled
//! `reasoning_content`, and the tool-call jail then scans whatever is left. That chain
//! cannot represent WHERE a thought happened. Given
//!
//! ```text
//! Look it up.…Now answer.It's 18C.
//! ```
//!
//! it serves `reasoning("Look it up.Now answer.")`, then the call, then `text("It's 18C.")`:
//! the second thought moved ahead of the call it followed and fused with the first. A
//! client rendering thoughts inline puts them in the wrong place, and a client counting
//! reasoning turns sees one where there were two.
//!
//! Ordering is not a field the split can add — it is lost at the seam between the two
//! parsers. So when this path is enabled, ONE [`UnifiedParser`] owns the whole grammar
//! and emits deltas in the order the model produced them, and this module maps those
//! deltas onto the OpenAI streaming/batch wire shapes.
//!
//! # Why one chunk per delta
//!
//! [`ChatCompletionStreamResponseDelta`] carries `content`, `reasoning_content` and
//! `tool_calls` side by side with no way to say which came first. Packing a whole
//! `push` into one delta object would throw away exactly the ordering this path exists
//! to preserve, so every [`UnifiedParserEvent`] becomes its own chunk.
use std::collections::{HashMap, HashSet, VecDeque};
use std::sync::LazyLock;
use async_stream::stream;
use dynamo_parsers::tool_calling::ToolDefinition;
use dynamo_parsers_v2::{
ArgumentFraming, InvalidGuidedPayloadPolicy, MalformedToolCallPolicy, Tool, ToolCallPolicy,
UnifiedEvent, UnifiedParser, UnifiedParserContinuation, UnifiedParserEvent, UnifiedParserInit,
UnifiedParserOutput, UnifiedParserStartingState, UnifiedToolOutputMode,
create_unified_parser_for_family,
};
use dynamo_protocols::types::{
ChatChoiceLogprobs, ChatChoiceStream, ChatCompletionMessageContent,
ChatCompletionMessageToolCall, ChatCompletionMessageToolCallChunk,
ChatCompletionStreamResponseDelta, FinishReason, FunctionCall, FunctionCallStream,
FunctionType,
};
use dynamo_runtime::error::{DynamoError, ErrorClass};
use dynamo_runtime::protocols::annotated::Annotated;
use dynamo_runtime::protocols::maybe_error::MaybeError;
use futures::{Stream, StreamExt};
use uuid::Uuid;
use crate::protocols::openai::GuidedToolConstraint;
use super::NvCreateChatCompletionStreamResponse;
#[cfg(test)]
use dynamo_protocols::types::ChatCompletionToolChoiceOption;
/// The `dynamo-parsers-v2` unified family that serves Qwen3.
///
/// `REGISTERED_UNIFIED_FAMILIES` accepts both `qwen3` and the `qwen3_coder` alias for
/// the same XML grammar; `qwen3` is the canonical registry name and the one the
/// conformance corpus uses, so it is what this module passes and logs.
pub(crate) const QWEN3_UNIFIED_FAMILY: &str = "qwen3";
pub(crate) const KIMI_K2_UNIFIED_FAMILY: &str = "kimi_k2";
pub(crate) const KIMI_K3_UNIFIED_FAMILY: &str = "kimi_k3";
pub(crate) const DEEPSEEK_V41_UNIFIED_FAMILY: &str = "deepseek_v41";
pub(crate) const DEEPSEEK_V4_UNIFIED_FAMILY: &str = "deepseek_v4";
pub(crate) const GEMMA4_UNIFIED_FAMILY: &str = "gemma4";
/// The empty thought channel Gemma 4 opens with thinking off.
pub(crate) const GEMMA4_EMPTY_THOUGHT_CHANNEL: &str = "<|channel>thought\n";
pub(crate) const GLM47_UNIFIED_FAMILY: &str = "glm47";
pub(crate) fn incomplete_tool_call_error() -> DynamoError {
DynamoError::builder()
.class(ErrorClass::BackendProtocol)
.message("The model ended with an incomplete tool call.")
.public_message("The model ended with an incomplete tool call.")
.build()
}
fn parser_error_annotation(
error: anyhow::Error,
) -> Annotated {
Annotated::from_err(DynamoError::from(
error.as_ref() as &(dyn std::error::Error + 'static)
))
}
fn incomplete_glm_terminal(
family: &str,
reason: Option,
incomplete: impl FnOnce() -> bool,
) -> Option {
(family == GLM47_UNIFIED_FAMILY
&& matches!(reason, Some(FinishReason::Stop | FinishReason::ToolCalls))
&& incomplete())
.then(incomplete_tool_call_error)
}
pub(crate) const MUSE_UNIFIED_FAMILY: &str = "muse_glimmer";
/// The `dynamo-parsers-v2` unified family that serves Beam. Dynamo has no v1 Beam
/// parser, so the `beam` + `beam` pair routes to the unified parser without
/// `DYN_PARSER_VERSION=2`, like DeepSeek V4.1.
pub(crate) const BEAM_UNIFIED_FAMILY: &str = "beam";
pub(crate) const MUSE_SPARK_UNIFIED_FAMILY: &str = "muse_spark";
/// Nemotron 3 (Ultra, Super, 3.5 Super): `nemotron_v3` forced reasoning over the
/// Qwen3-Coder XML tool grammar. Model cards pair `--dyn-tool-call-parser qwen3_coder`
/// with `--dyn-reasoning-parser nemotron_v3`.
pub(crate) const NEMOTRON_V3_UNIFIED_FAMILY: &str = "nemotron_v3";
/// Dynamo's `--dyn-tool-call-parser` name that pairs into [`QWEN3_UNIFIED_FAMILY`].
const QWEN3_TOOL_CALL_PARSER: &str = "qwen3_coder";
fn uses_qwen_xml_grammar(family: &str) -> bool {
matches!(family, QWEN3_UNIFIED_FAMILY | NEMOTRON_V3_UNIFIED_FAMILY)
}
/// A family that borrows another family's tool grammar names the owner through its
/// reasoning half: `qwen3_coder` + `nemotron_v3` is Nemotron, not Qwen3, because the
/// forced-reasoning default lives in the reasoning parser and the XML grammar is shared.
fn shared_grammar_family(tool_family: &str, reasoning_family: &str) -> Option<&'static str> {
match (tool_family, reasoning_family) {
(QWEN3_UNIFIED_FAMILY, NEMOTRON_V3_UNIFIED_FAMILY) => Some(NEMOTRON_V3_UNIFIED_FAMILY),
_ => None,
}
}
const DYNAMO_FAMILY_ALIASES: &[(&str, &str)] = &[
("deepseek-v4", DEEPSEEK_V4_UNIFIED_FAMILY),
("deepseekv4", DEEPSEEK_V4_UNIFIED_FAMILY),
("muse", MUSE_UNIFIED_FAMILY),
("gemma-4", GEMMA4_UNIFIED_FAMILY),
("glm45", GLM47_UNIFIED_FAMILY),
("kimi_k25", KIMI_K2_UNIFIED_FAMILY),
];
/// Canonical family names and aliases share one route for registration, stream, and batch.
pub(crate) fn canonical_family(parser: &str) -> Option<&'static str> {
let parser = DYNAMO_FAMILY_ALIASES
.iter()
.find_map(|(alias, family)| (*alias == parser).then_some(*family))
.unwrap_or(parser);
dynamo_parsers_v2::canonical_unified_family(parser)
}
/// Aliases that still route but are scheduled for removal, paired with the family
/// to configure instead. `muse` predates Muse Spark Open and now reads as "any Muse
/// model" while it selects only Muse Glimmer; a Spark deployment configured with it
/// starts without complaint and runs the wrong parser.
/// TODO(next minor): remove these from `DYNAMO_FAMILY_ALIASES` and `FAMILY_NAMES` so
/// the alias becomes a startup error.
pub(crate) const DEPRECATED_FAMILY_ALIASES: &[(&str, &str)] = &[("muse", MUSE_UNIFIED_FAMILY)];
/// The startup warning for a deprecated alias, or `None` for any other selector.
pub(crate) fn deprecated_alias_warning(parser: &str) -> Option {
DEPRECATED_FAMILY_ALIASES
.iter()
.find(|(alias, _)| *alias == parser)
.map(|(alias, family)| {
format!(
"parser `{alias}` is a deprecated alias of `{family}` and will be removed in a \
future release; configure `{family}` instead. Muse Spark Open is a different \
parser: `muse_spark_open`"
)
})
}
pub(crate) static FAMILY_NAMES: LazyLock> = LazyLock::new(|| {
let mut names = dynamo_parsers_v2::REGISTERED_UNIFIED_FAMILIES.to_vec();
names.extend(DYNAMO_FAMILY_ALIASES.iter().map(|(alias, _)| *alias));
names.sort_unstable();
names.dedup();
names
});
/// A configured half may select a unified family, but two configured halves must agree.
pub(crate) fn configured_family(
tool_call_parser: Option<&str>,
reasoning_parser: Option<&str>,
) -> Option<&'static str> {
match (tool_call_parser, reasoning_parser) {
(Some(tool), Some(reasoning)) => {
let tool_family = canonical_family(tool)?;
let reasoning_family = canonical_family(reasoning)?;
if tool_family == reasoning_family {
return Some(tool_family);
}
shared_grammar_family(tool_family, reasoning_family)
}
(Some(parser), None) | (None, Some(parser)) => canonical_family(parser),
(None, None) => None,
}
}
pub(crate) fn is_v2_configured_family(
tool_call_parser: Option<&str>,
reasoning_parser: Option<&str>,
) -> bool {
configured_family(tool_call_parser, reasoning_parser).is_some()
}
/// The unified family to actually use for this parser pair, or `None` to keep the
/// existing split reasoning-parser + tool-call-jail path.
pub(crate) fn selected_family(
tool_call_parser: Option<&str>,
reasoning_parser: Option<&str>,
) -> Option<&'static str> {
let selection = super::tool_parser_v2::selected_version()
.unwrap_or_else(|error| panic!("invalid parser routing configuration: {error:#}"));
let selected = selected_family_for_selection(tool_call_parser, reasoning_parser, selection);
tracing::debug!(
target: "dynamo_unified",
?tool_call_parser,
?reasoning_parser,
?selection,
?selected,
"unified parser path decision"
);
selected
}
pub(crate) fn selected_family_for_selection(
tool_call_parser: Option<&str>,
reasoning_parser: Option<&str>,
selection: dynamo_runtime::config::ParserVersion,
) -> Option<&'static str> {
// Qwen3 thinking models can use Hermes syntax; reasoning-only Nemotron cards
// do not identify their tool grammar. Auto keeps both on the legacy path.
if selection == dynamo_runtime::config::ParserVersion::V1
|| (selection == dynamo_runtime::config::ParserVersion::Auto
&& tool_call_parser.is_none()
&& matches!(
reasoning_parser,
Some("qwen3" | "nemotron_v3" | "nemotron3")
))
{
return None;
}
configured_family(tool_call_parser, reasoning_parser)
}
pub(crate) fn selected_content_decoder_family(
tool_call_parser: Option<&str>,
reasoning_parser: Option<&str>,
) -> Option<&'static str> {
selected_family(tool_call_parser, reasoning_parser)
}
pub(crate) fn selected_batch_family(
tool_call_parser: Option<&str>,
reasoning_parser: Option<&str>,
) -> Option<&'static str> {
selected_family(tool_call_parser, reasoning_parser)
}
pub(crate) fn selected_request_family(
tool_call_parser: Option<&str>,
reasoning_parser: Option<&str>,
_tool_choice: Option<&dynamo_protocols::types::ChatCompletionToolChoiceOption>,
_constraint: &GuidedToolConstraint,
) -> Option<&'static str> {
selected_family(tool_call_parser, reasoning_parser)
}
/// Whether the operator explicitly selected V2 parsing.
pub(crate) fn parsers_v2_selected() -> bool {
super::tool_parser_v2::enabled()
}
/// Request facts for parser initialization and response interpretation.
#[derive(Clone, Copy, Debug, Default)]
pub(crate) struct UnifiedRequestPolicy {
pub reasoning_disabled: bool,
pub structured_response: bool,
pub multiple_choices: bool,
}
impl UnifiedRequestPolicy {
/// The policy as the installed grammar leaves it: a `response_format` the grammar
/// holds inside the model's answer channel is native channel output, not the bare
/// JSON document the structured-response probes expect.
fn under(self, constraint: &GuidedToolConstraint) -> Self {
Self {
structured_response: self.structured_response && !constraint.frames_structured_answer(),
..self
}
}
fn harmony_prefill_cache(
self,
family: &str,
prefix: Option>,
) -> Option {
if family == "gpt_oss" && self.multiple_choices {
prefix.map(HarmonyPrefill::new)
} else {
None
}
}
pub(crate) fn resolve_stream_prefill(
self,
family: &str,
mut prefill: StreamPrefill,
constraint: &GuidedToolConstraint,
has_explicit_guided_decoding: bool,
harmony_required_array: impl FnOnce() -> bool,
) -> StreamPrefill {
if family == "gpt_oss"
&& (has_explicit_guided_decoding
|| constraint.installs_guided_json() && !harmony_required_array())
{
return UnifiedParserStartingState::Response.into();
}
prefill.starting_state = self.starting_state(family, prefill.starting_state);
prefill
}
fn starting_state(
self,
family: &str,
inferred: UnifiedParserStartingState,
) -> UnifiedParserStartingState {
if self.reasoning_disabled
&& matches!(family, "gemma4" | "minimax_m3" | NEMOTRON_V3_UNIFIED_FAMILY)
{
// Gemma's Response state preserves literal thought delimiters and still parses tools.
// Nemotron 3 defaults to an implicit open thought; `enable_thinking=false` renders
// a closed `` pair instead, so generation starts in the answer.
// MiniMax M3 reaches this only when the rendered prompt closed the thought with
// `` or a grammar owns the output (the preprocessor's rendered-prompt
// gate), so there is no reasoning channel left to parse.
UnifiedParserStartingState::Response
} else {
inferred
}
}
fn inspects_json_prefill(self, constraint: &GuidedToolConstraint) -> bool {
self.structured_response || constraint.installs_guided_json()
}
fn batch_starting_state(
self,
family: &str,
content: &str,
) -> anyhow::Result {
let inferred = if self.structured_response
&& structured_json_response_prefill(family, content, true)?
== Some(UnifiedParserStartingState::Response)
{
UnifiedParserStartingState::Response
} else {
detect_prefill(family, content)?
};
Ok(self.starting_state(family, inferred))
}
}
/// Scalar and container roots can also begin genuine reasoning. Hold only that
/// ambiguous prefix; process each incoming byte once and release it when a suffix
/// disproves JSON.
#[derive(Default)]
struct JsonPrefillProbe {
text: String,
state: JsonPrefixState,
starts_json: Option,
inspected_bytes: usize,
container_start_validated: bool,
marker_probe: Option,
marker_suffix: String,
marker_decision: Option,
marker_inspected_bytes: usize,
}
#[derive(Default)]
enum JsonPrefixState {
#[default]
Start,
ContainerStart {
opener: u8,
validated: bool,
},
String {
escaped: bool,
unicode_left: u8,
},
Literal {
expected: &'static [u8],
position: usize,
},
Number(NumberPrefixState),
Done,
Invalid,
}
#[derive(Clone, Copy)]
enum NumberPrefixState {
Minus,
Zero,
Digits,
Dot,
Fraction,
Exponent,
ExponentSign,
ExponentDigits,
}
impl JsonPrefillProbe {
fn push(&mut self, text: &str) {
self.text.push_str(text);
for byte in text.bytes() {
self.inspected_bytes += 1;
let next_state = match &self.state {
JsonPrefixState::Start => match byte {
b' ' | b'\r' | b'\n' | b'\t' => JsonPrefixState::Start,
b'{' | b'[' => JsonPrefixState::ContainerStart {
opener: byte,
validated: false,
},
b'"' => JsonPrefixState::String {
escaped: false,
unicode_left: 0,
},
b'-' => JsonPrefixState::Number(NumberPrefixState::Minus),
b'0' => JsonPrefixState::Number(NumberPrefixState::Zero),
b'1'..=b'9' => JsonPrefixState::Number(NumberPrefixState::Digits),
b't' => JsonPrefixState::Literal {
expected: b"true",
position: 1,
},
b'f' => JsonPrefixState::Literal {
expected: b"false",
position: 1,
},
b'n' => JsonPrefixState::Literal {
expected: b"null",
position: 1,
},
_ => JsonPrefixState::Invalid,
},
JsonPrefixState::String {
escaped,
unicode_left,
} => {
if *unicode_left > 0 {
if byte.is_ascii_hexdigit() {
JsonPrefixState::String {
escaped: false,
unicode_left: unicode_left - 1,
}
} else {
JsonPrefixState::Invalid
}
} else if *escaped {
match byte {
b'u' => JsonPrefixState::String {
escaped: false,
unicode_left: 4,
},
b'"' | b'\\' | b'/' | b'b' | b'f' | b'n' | b'r' | b't' => {
JsonPrefixState::String {
escaped: false,
unicode_left: 0,
}
}
_ => JsonPrefixState::Invalid,
}
} else {
match byte {
b'"' => JsonPrefixState::Done,
b'\\' => JsonPrefixState::String {
escaped: true,
unicode_left: 0,
},
0..=31 => JsonPrefixState::Invalid,
_ => JsonPrefixState::String {
escaped: false,
unicode_left: 0,
},
}
}
}
JsonPrefixState::Literal { expected, position } => {
if byte != expected[*position] {
JsonPrefixState::Invalid
} else if position + 1 == expected.len() {
JsonPrefixState::Done
} else {
JsonPrefixState::Literal {
expected,
position: position + 1,
}
}
}
JsonPrefixState::Number(number) => {
use NumberPrefixState::*;
let next = match (*number, byte) {
(Minus, b'0') => Some(Zero),
(Minus, b'1'..=b'9') => Some(Digits),
(Digits, b'0'..=b'9') => Some(Digits),
(Zero | Digits, b'.') => Some(Dot),
(Dot | Fraction, b'0'..=b'9') => Some(Fraction),
(Zero | Digits | Fraction, b'e' | b'E') => Some(Exponent),
(Exponent, b'+' | b'-') => Some(ExponentSign),
(Exponent | ExponentSign | ExponentDigits, b'0'..=b'9') => {
Some(ExponentDigits)
}
_ => None,
};
if let Some(next) = next {
JsonPrefixState::Number(next)
} else if matches!(number, Zero | Digits | Fraction | ExponentDigits)
&& byte.is_ascii_whitespace()
{
JsonPrefixState::Done
} else {
JsonPrefixState::Invalid
}
}
JsonPrefixState::Done => {
if matches!(byte, b' ' | b'\r' | b'\n' | b'\t') {
JsonPrefixState::Done
} else {
JsonPrefixState::Invalid
}
}
JsonPrefixState::ContainerStart { opener, validated } => {
if *validated {
JsonPrefixState::ContainerStart {
opener: *opener,
validated: true,
}
} else if matches!(byte, b' ' | b'\r' | b'\n' | b'\t') {
JsonPrefixState::ContainerStart {
opener: *opener,
validated: false,
}
} else {
let valid_first_value = matches!(
(*opener, byte),
(b'{', b'"' | b'}')
| (
b'[',
b']' | b'{' | b'[' | b'"' | b'-' | b'0'
..=b'9' | b't' | b'f' | b'n',
)
);
if valid_first_value {
JsonPrefixState::ContainerStart {
opener: *opener,
validated: true,
}
} else {
JsonPrefixState::Invalid
}
}
}
JsonPrefixState::Invalid => JsonPrefixState::Invalid,
};
if self.starts_json.is_none() && !matches!(next_state, JsonPrefixState::Start) {
self.starts_json = Some(!matches!(next_state, JsonPrefixState::Invalid));
}
if matches!(
&next_state,
JsonPrefixState::ContainerStart {
validated: true,
..
}
) {
self.container_start_validated = true;
}
self.state = next_state;
}
}
fn unopened_channel_decision(
&self,
family: &str,
terminal: bool,
) -> Option {
// Muse Spark's prompt ends at `<|start|>assistant`, so every turn opens with
// a ` to=` header whose `t` would otherwise read as JSON `true`.
if family == MUSE_SPARK_UNIFIED_FAMILY {
const HEADER: &str = "to=";
let text = self.text.trim_start();
if text.starts_with(HEADER) {
return Some(UnifiedParserStartingState::None);
}
if !terminal && !text.is_empty() && HEADER.starts_with(text) {
return None;
}
}
// JSON roots are unambiguous when the prompt has not opened reasoning;
// waiting for a complete scalar would unnecessarily buffer its stream.
match self.starts_json {
Some(true) => Some(UnifiedParserStartingState::Response),
Some(false) => Some(UnifiedParserStartingState::None),
None if terminal => Some(UnifiedParserStartingState::Response),
None => None,
}
}
fn decision(&self, terminal: bool) -> Option {
if self.container_start_validated {
return Some(UnifiedParserStartingState::None);
}
match self.state {
JsonPrefixState::ContainerStart {
validated: true, ..
} if terminal => Some(UnifiedParserStartingState::None),
JsonPrefixState::ContainerStart {
validated: true, ..
} => None,
JsonPrefixState::ContainerStart { .. } if terminal => {
Some(UnifiedParserStartingState::None)
}
JsonPrefixState::ContainerStart { .. } => None,
JsonPrefixState::Invalid => Some(UnifiedParserStartingState::Reasoning),
JsonPrefixState::Done
| JsonPrefixState::Number(
NumberPrefixState::Zero
| NumberPrefixState::Digits
| NumberPrefixState::Fraction
| NumberPrefixState::ExponentDigits,
) if terminal => Some(UnifiedParserStartingState::None),
_ if terminal => Some(UnifiedParserStartingState::Reasoning),
_ => None,
}
}
fn has_control_marker(&mut self, family: &str, terminal: bool, latest_text: &str) -> bool {
let first_push = self.marker_probe.is_none();
let shield = self
.marker_probe
.get_or_insert_with(|| QuotedControlMarkerProbe::new(family));
let incoming = if first_push { &self.text } else { latest_text };
let mut unquoted = shield.push(incoming);
if terminal {
unquoted.push_str(&shield.finish());
}
self.marker_suffix.push_str(&unquoted);
self.marker_inspected_bytes += self.marker_suffix.len();
let has_candidate = shield
.markers
.iter()
.any(|marker| self.marker_suffix.contains(marker));
let suffix_limit = shield.markers.iter().map(String::len).max().unwrap_or(0);
let mut keep_from = self.marker_suffix.len().saturating_sub(suffix_limit);
while !self.marker_suffix.is_char_boundary(keep_from) {
keep_from += 1;
}
self.marker_suffix.drain(..keep_from);
has_candidate
}
fn decision_for_family(
&mut self,
family: &str,
terminal: bool,
latest_text: &str,
) -> anyhow::Result