# Copyright (c) 2025-2026, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Sampler orchestration: the entry points the executor drives each step. Holds the :class:`Sampler` ABC and its ``SampleState`` types, the trivial early-stop samplers (:class:`EarlyStopSampler` for non-generation models, :class:`EarlyStopWithMMResult` for the multimodal-encoder-only engine), and :class:`TorchSampler` -- the PyTorch sampling path. ``TorchSampler`` owns no sampling logic of its own beyond batching and orchestration: each feature (beam search, penalties, token bans, top-p decay, finish reasons, log-probs, seeds) lives in its own module and is held here as a handler, driven through ``setup_sampler_step`` / ``sample_async`` / ``update_requests``. """ import sys from abc import ABC, abstractmethod from collections.abc import Hashable, Iterable from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, Dict, Generic, List, Optional, TypeAlias, TypeVar, cast import torch from tensorrt_llm._torch.flashinfer_utils import IS_FLASHINFER_AVAILABLE from tensorrt_llm._utils import nvtx_range, prefer_pinned from tensorrt_llm.bindings.executor import FinishReason from tensorrt_llm.bindings.internal.batch_manager import add_new_tokens_to_requests from tensorrt_llm.executor.result import Logprob from tensorrt_llm.executor.utils import context_length_exceeded_message from tensorrt_llm.logger import logger from ...utils import torch_multi_arange from ..llm_request import LlmRequest, LlmRequestState, get_draft_token_length from ..resource_manager import ResourceManager from ..scheduler import ScheduledRequests from .beam_search import ( BeamHistory, BeamHistoryBuilder, BeamSearchHandler, BeamSearchParams, BeamSearchStep, BeamSearchStore, PendingBeamHistories, finalize_beam, prepare_beam_search, ) from .finish_reasons import FinishReasonsHandler from .logprobs import LogProbsHandler, LogProbsState, LogProbsStateList, LogProbsStore from .ops.custom import fused_sample_from_logits, fused_sample_from_logits_with_probs from .ops.vanilla import greedy_search_sampling_batch from .penalties import PenaltyHandler, has_occurrence_penalty from .sampler_common import ( DEFAULT_BEAM_IDX, DEFAULT_STEP_IDX, FinishReasonsList, SampleType, _BatchedSamplingResult, _get_beam_width_out, add_token, int_tensor, ) from .sampler_features import ( AsyncWorkerMixin, EmbeddingBiasCache, SamplerEvent, _PackedStepIndexer, _UnpackedStepIndexer, check_stop_words_length, fast_greedy_sample_kernel, meet_stop_token_criteria, scatter_new_tokens, ) from .sampling_resolution import SamplingMode, SlotSamplingParams, request_row_params from .seed_manager import _SeedManager from .token_ban import OverlappedTokenBanHandler, SynchronousTokenBanHandler, TokenBanHandler from .top_p_decay import TopPDecayHandler from .watermark import WatermarkHandler if TYPE_CHECKING: from tensorrt_llm.watermark.config import WatermarkConfig if sys.version_info[:2] >= (3, 12): from typing import override else: from typing_extensions import override T = TypeVar("T") # Seed of every unseeded request, the same on every rank. _GLOBAL_SEED = 42 @dataclass(kw_only=True) class SampleStateTensors: new_tokens: torch.Tensor log_probs: torch.Tensor | None = None GenericSampleStateTensorsHost = TypeVar("GenericSampleStateTensorsHost", bound=SampleStateTensors) GenericSampleStateTensorsDevice = TypeVar( "GenericSampleStateTensorsDevice", bound=SampleStateTensors ) @dataclass(kw_only=True) class SampleState(Generic[GenericSampleStateTensorsHost, GenericSampleStateTensorsDevice]): requests: list[LlmRequest] device: Optional[GenericSampleStateTensorsDevice] = None host: Optional[GenericSampleStateTensorsHost] = None sampler_event: Optional[SamplerEvent] = None runtime_draft_len: Optional[int] = None # Generic bounds not supported, https://github.com/python/typing/issues/548 GenericSampleState = TypeVar("GenericSampleState", bound=SampleState) # type: ignore # Model output keys of a forward that samples greedily in place of its LM head: per-row greedy token ids, and the # head's input rows so the logits can still be computed for a batch that needs them. Such a forward returns no # logits. GREEDY_TOKENS_KEY = "greedy_tokens" GREEDY_HIDDEN_STATES_KEY = "greedy_hidden_states" def greedy_tokens_usable(request: LlmRequest) -> bool: """Whether a generation request is served by the forward's greedy token alone, without logits. The request conditions of TorchSampler's stable greedy path, plus every other reader of the logits. A dummy request is not usable: the stable greedy path takes no batch that carries one. """ return not request.is_dummy and ( get_draft_token_length(request) == 0 and request.py_embedding_bias is None and not getattr(request, "py_bad_words", None) and not getattr(request, "py_no_repeat_ngram_size", None) and not has_occurrence_penalty(request) and not request.py_min_length and not request.py_return_log_probs and not request.py_return_generation_logits and not request.py_logits_post_processors and request.py_guided_decoding_params is None and not check_stop_words_length(request) and request_row_params(request).mode is SamplingMode.GREEDY ) @dataclass(frozen=True, kw_only=True) class _FusedRowParamsBuffers: """Per-row fused-kernel params the in-graph fast tier samples from. A captured graph holds these addresses, so they are allocated once for the widest batch and only their leading rows are written. ``floats`` stacks temperature, top_p and min_p, in the layout of ``SlotSamplingParams.floats``. """ floats: torch.Tensor top_k: torch.Tensor seed: torch.Tensor offset: torch.Tensor @classmethod def allocate(cls, capacity: int, device: torch.device) -> "_FusedRowParamsBuffers": """Initial values are neutral (temperature 1, top_k 0 = keep all, top_p 1, min_p 0), so rows past the staged batch filter nothing even if a replay reads them.""" floats = torch.zeros((3, capacity), dtype=torch.float32, device=device) floats[:2] = 1.0 return cls( floats=floats, top_k=torch.zeros(capacity, dtype=torch.int32, device=device), seed=torch.zeros(capacity, dtype=torch.int64, device=device), offset=torch.zeros(capacity, dtype=torch.int64, device=device), ) @property def capacity(self) -> int: return self.top_k.numel() @property def temperature(self) -> torch.Tensor: return self.floats[0] @property def top_p(self) -> torch.Tensor: return self.floats[1] @property def min_p(self) -> torch.Tensor: return self.floats[2] @dataclass(frozen=True, kw_only=True) class _StagedBatch: """What ``stage_in_graph_sampling`` staged for a batch. Kept after the step so that the same batch on the next step costs neither restaging nor re-deriving what ``_process_requests`` needs to read the in-graph samples back. """ # (tier, request ids, slots) of every staged row, padding included. key: tuple[Any, ...] # The staged rows, padding included, and their live (non-dummy) prefix. requests: list[LlmRequest] live_requests: list[LlmRequest] live_request_ids: list[int] # No live request needs the finish-reason kernel, so stopping reduces to the # host-side per-token checks of _update_requests_single_beam_single_step. single_step_greedy: bool live_seq_slots_host: torch.Tensor @dataclass(kw_only=True) class _SamplingGroup: """Requests of one batch sampled together, by ``TorchSampler._group_requests``.""" mode: SamplingMode needs_probs: bool req_indices: list[int] = field(default_factory=list) """Indices into the batch's requests, ascending.""" beam_params: list[BeamSearchParams] = field(default_factory=list) """Per request of a beam-search group; empty otherwise.""" class Sampler(ABC, Generic[GenericSampleState]): def setup_sampler_step(self, scheduled_requests: ScheduledRequests) -> None: pass def release_request(self, request: LlmRequest) -> None: """Release sampler-owned state before the request's slot is returned.""" def get_cache_indirection(self) -> torch.Tensor | None: return None @abstractmethod def sample_async( self, scheduled_requests: ScheduledRequests, model_outputs: dict[str, Any], num_context_logits_prefix_sum: list[int], resource_manager: Optional[ResourceManager] = None, ) -> GenericSampleState: raise NotImplementedError def sample_in_graph( self, model_outputs: dict[str, Any], sample_type: SampleType, ) -> None: """Sample inside the model-forward CUDA graph. Called while the forward graph is being captured/replayed, after the model has written its logits, so that sampling for graph-capturable tiers is recorded as the tail of that graph instead of costing a separate launch. Implementations must stay capture-safe: device work only, no host synchronization, no allocation, and no dependence on tensor shapes that vary across replays. Any host-side preparation (index computation, parameter staging) belongs outside the graph, staged into persistent buffers that this method reads. The base implementation is a no-op, which keeps samplers that do not opt in unaffected: they leave the graph without sampling, and ``sample_async`` performs the whole sampling step eagerly as before. """ @abstractmethod def update_requests( self, state: GenericSampleState, resource_manager: Optional[ResourceManager] = None, ) -> None: raise NotImplementedError @staticmethod def beam_width(requests: Iterable[LlmRequest]) -> int: for req in requests: return req.py_beam_width return 0 @abstractmethod def is_generation_model(self) -> bool: raise NotImplementedError def validate_request(self, request: LlmRequest) -> None: """Validate that the request can be processed by the sampler. If the request is not supported by the sampler, this should raise an appropriate exception. Args: request: The request to validate Returns: None if request is valid Raises: Appropriate exception if request is not supported by sampler. """ def _validate_single_forward_request(request: LlmRequest, max_num_tokens: Optional[int]) -> None: """Reject a prompt that does not fit the one forward a non-generation request runs in.""" if max_num_tokens is not None and request.prompt_len > max_num_tokens: raise ValueError( context_length_exceeded_message( max_context_length=max_num_tokens, num_prompt_tokens=request.prompt_len ) ) class EarlyStopSampler(Sampler[SampleState[SampleStateTensors, SampleStateTensors]]): """ Use for skipping decoding step for non generation model, such as encoder-only model (e.g., BERT) or reward models that only need context phase. Args: max_num_tokens: The engine's token budget. A request runs whole in one forward, so a prompt longer than this is rejected; None skips the check. """ SampleState: TypeAlias = SampleState[SampleStateTensors, SampleStateTensors] def __init__(self, max_num_tokens: Optional[int] = None) -> None: self.max_num_tokens = max_num_tokens @override def validate_request(self, request: LlmRequest) -> None: _validate_single_forward_request(request, self.max_num_tokens) @override def sample_async( self, scheduled_requests: ScheduledRequests, model_outputs: dict[str, Any], num_context_logits_prefix_sum: list[int], resource_manager: Optional[ResourceManager] = None, ) -> SampleState: host = SampleStateTensors(new_tokens=torch.empty(0)) assert not scheduled_requests.generation_requests return self.SampleState(requests=scheduled_requests.context_requests, host=host) @override def update_requests( self, state: SampleState, resource_manager: Optional[ResourceManager] = None, ) -> None: assert isinstance(state, SampleState) requests = state.requests for idx, request in enumerate(requests): request.state = LlmRequestState.GENERATION_COMPLETE # NOTE: This is a hack: set finish reason manually and set the beam 0 request.set_finished_reason(FinishReason.LENGTH, 0) @override def is_generation_model(self) -> bool: return False @dataclass(kw_only=True) class MultimodalResult: mm_embeddings: List[torch.Tensor] # needed to torch.split the mm_embeddings into item-wise chunks mm_embedding_lengths: List[List[int]] # needed when requests mix text-only and multimodal ones mm_embedding_request_indices: List[int] # number of context requests in the batch num_context_requests: int # Can be used to include e.g. `mrope_position_ids`, etc. extra_data: Optional[Dict[str, Any]] = None def __post_init__(self) -> None: num_embeddings = len(self.mm_embeddings) metadata = (self.extra_data or {}).get("mm_embedding_metadata") if metadata is not None and len(metadata) != num_embeddings: raise ValueError("mm_embedding_metadata batch size does not match mm_embeddings") num_lengths = len(self.mm_embedding_lengths) if num_lengths != num_embeddings: raise ValueError( "mm_embedding_lengths batch size does not match mm_embeddings: " f"{num_lengths} != {num_embeddings}" ) num_request_indices = len(self.mm_embedding_request_indices) if num_request_indices != num_embeddings: raise ValueError( "mm_embedding_request_indices batch size does not match " f"mm_embeddings: {num_request_indices} != {num_embeddings}" ) for result_index, (mm_embedding, mm_embedding_lengths) in enumerate( zip(self.mm_embeddings, self.mm_embedding_lengths, strict=True) ): actual_rows = len(mm_embedding) expected_rows = sum(mm_embedding_lengths) if actual_rows != expected_rows: raise ValueError( f"mm_embedding shape mismatch for result {result_index}: " f"{actual_rows} != {expected_rows}" ) for request_index in self.mm_embedding_request_indices: if request_index < 0 or request_index >= self.num_context_requests: raise ValueError( "mm_embedding_request_indices contains an invalid request " f"index: {request_index} not in [0, {self.num_context_requests})" ) @classmethod def from_model_outputs( cls, model_outputs: Dict[str, Any], num_context_requests: int ) -> "MultimodalResult": result_keys = { "mm_embeddings", "mm_embedding_lengths", "mm_embedding_request_indices", } return cls( mm_embeddings=model_outputs["mm_embeddings"], mm_embedding_lengths=model_outputs["mm_embedding_lengths"], mm_embedding_request_indices=model_outputs["mm_embedding_request_indices"], num_context_requests=num_context_requests, extra_data={ key: value for key, value in model_outputs.items() if key not in result_keys }, ) @dataclass(kw_only=True) class SampleStateWithMMResult(SampleState[SampleStateTensors, SampleStateTensors]): data: MultimodalResult class EarlyStopWithMMResult(Sampler[SampleStateWithMMResult]): """ Use for skipping decoding step for non generation model, and return the batch_output (such as mm_embeddings) Args: return_mm_results: Whether to attach the multimodal outputs to the requests' results. Disable it on ranks whose responses never reach the frontend: the results are exported as CUDA IPC handles, and PyTorch keeps an exported allocation alive until a consumer opens and releases its handle. max_num_tokens: The engine's token budget. A request runs whole in one forward, so a prompt longer than this is rejected; None skips the check. """ SampleState: TypeAlias = SampleStateWithMMResult def __init__( self, return_mm_results: bool = True, max_num_tokens: Optional[int] = None ) -> None: self.return_mm_results = return_mm_results self.max_num_tokens = max_num_tokens @override def validate_request(self, request: LlmRequest) -> None: _validate_single_forward_request(request, self.max_num_tokens) @override def sample_async( self, scheduled_requests: ScheduledRequests, model_outputs: dict[str, Any], num_context_logits_prefix_sum: list[int], resource_manager: Optional[ResourceManager] = None, ) -> SampleState: # from model_outputs to MultimodalResult assert not scheduled_requests.generation_requests data = MultimodalResult.from_model_outputs( model_outputs, scheduled_requests.num_context_requests ) return self.SampleState(requests=scheduled_requests.context_requests, data=data) @override def update_requests( self, state: SampleState, resource_manager: Optional[ResourceManager] = None, ) -> None: # resource_manager will not be used in this function, just for interface consistency. assert isinstance(state, SampleState) requests = state.requests mm_embeddings = state.data.mm_embeddings extra_data = state.data.extra_data or {} mrope_position_ids = extra_data.get("mrope_position_ids", None) mrope_position_deltas = extra_data.get("mrope_position_deltas", None) for request in requests: request.state = LlmRequestState.GENERATION_COMPLETE # NOTE: This is a hack: set finish reason manually and set the beam 0 request.set_finished_reason(FinishReason.LENGTH, 0) if not self.return_mm_results: return request_indices = state.data.mm_embedding_request_indices for result_index, (request_index, mm_embedding) in enumerate( zip(request_indices, mm_embeddings, strict=True) ): request = requests[request_index] mm_embedding_lengths = state.data.mm_embedding_lengths[result_index] embedding_metadata = extra_data.get("mm_embedding_metadata") if embedding_metadata is None: request.py_result.append_mm_embeddings(mm_embedding, mm_embedding_lengths) else: request.py_result.append_mm_embeddings( mm_embedding, mm_embedding_lengths, embedding_metadata[result_index] ) # Store mrope data if available if mrope_position_ids is not None and mrope_position_deltas is not None: mrope_index = ( request_index if len(mrope_position_ids) == len(requests) else result_index ) request.py_result.set_mrope_position( mrope_position_ids[mrope_index], mrope_position_deltas[mrope_index], ) @override def is_generation_model(self) -> bool: return False @dataclass(kw_only=True) class SamplingRequestsMetadata: """Metadata for the sampling requests.""" req_num_generated_tokens: torch.Tensor """The number of generated tokens for each sampling request. In beam search, this uses the incoming beam width.""" req_num_generated_tokens_output: torch.Tensor """The number of generated tokens for each sampling request. In beam search, this uses the outgoing beam width.""" req_num_beams: torch.Tensor """The number of beams for each sampling request.""" req_num_steps: torch.Tensor """The number of generation steps for each sampling request.""" req_offsets: torch.Tensor """The start offsets of the sampling requests in the raw logits.""" @dataclass(kw_only=True) class SampleStateTensorsHostTorch(SampleStateTensors): finish_reasons: torch.Tensor | None first_finish_reasons: torch.Tensor | None logprobs_state: LogProbsState | None = None single_step_greedy: bool = False """Whether `new_tokens` uses the compact `(num_requests,)` layout instead of `[step, slot, beam]`. Describes these host tensors, so it must live here rather than on `SampleStateTorch`: under pipeline parallelism only this object crosses the ring hand-off, and a receiving rank would otherwise pair the compact buffer with an outer flag left at its default.""" def finish_reasons_list(self) -> FinishReasonsList: """`(num_seq_slots, num_steps)`""" # step, slot, beam => slot, step, beam finish_reasons = self.finish_reasons if finish_reasons is None: return [] else: return finish_reasons.permute(1, 0, 2).tolist() @dataclass(kw_only=True) class SampleStateTorch(SampleState[SampleStateTensorsHostTorch, SampleStateTensors]): beam_history_builders: list[BeamHistoryBuilder | None] | None = None sample_type: SampleType = SampleType.FULL """Tier this batch was sampled at. Carried on the state rather than passed through ``sample_async`` / ``update_requests`` so the base-class signatures -- and therefore every other Sampler implementation -- stay untouched. ``TorchSampler`` decides it from the batch and is the only reader. """ class TorchSampler(Sampler[SampleStateTorch], AsyncWorkerMixin): SampleState = SampleStateTorch # Text-watermark state; set by __init__ when the engine configures a watermark. _watermark: WatermarkHandler | None = None @override def get_cache_indirection(self) -> torch.Tensor | None: if (beam_search_store := self.store.beam_search_store) is not None: return beam_search_store.cache_indirection return None def start_beam_histories(self, requests: list[LlmRequest]) -> PendingBeamHistories: """Start copying the generated tokens of each live beam of each beam-search request. See ``BeamSearchHandler.start_beam_histories``; used by the model engine to give logits post-processors each beam's real hypothesis. """ return self._beam_search.start_beam_histories(requests) @override def is_generation_model(self) -> bool: return True @dataclass(kw_only=True) @dataclass(kw_only=True) class Store: new_tokens: torch.Tensor """Device tensor containing latest sampled tokens. Shape: ``NEW_TOKENS_SHAPE`` -- (max_tokens, max_num_sequences + 1, max_beam_width); the trailing row is ``dummy_slot_row`` scratch, not a sequence slot. """ beam_search_store: "BeamSearchStore | None" = None """Holds data related to beam search.""" log_probs_store: LogProbsStore """Holds data related to log-probs handling.""" def _create_store(self) -> Store: # Tensors necessary for all sampling methods new_tokens = int_tensor(self.NEW_TOKENS_SHAPE) # Only used for logprobs processing or beam search sampled_log_probs = torch.empty(self.LOGPROBS_SHAPE, device="cuda", dtype=torch.float32) # Only used for logprobs processing sampled_log_prob_indices = torch.empty( self.LOGPROBS_SHAPE, device="cuda", dtype=torch.int32 ) sampled_log_prob_ranks = torch.empty(self.LOGPROBS_SHAPE, device="cuda", dtype=torch.int32) # These are 0 sized tensors, if topk-logprobs are not used topk_indices = torch.empty( self._log_probs.TOPK_LOGPROBS_SHAPE, device="cuda", dtype=torch.int32 ) topk_vals = torch.empty( self._log_probs.TOPK_LOGPROBS_SHAPE, device="cuda", dtype=torch.float32 ) log_probs_store = LogProbsStore( sampled_log_prob_indices=sampled_log_prob_indices, sampled_log_probs=sampled_log_probs, sampled_log_prob_ranks=sampled_log_prob_ranks, topk_indices=topk_indices, topk_vals=topk_vals, ) beam_search_store = None if self._use_beam_search: beam_search_store = BeamSearchStore.create( cache_indirection_shape=self.CACHE_INDIRECTION_SHAPE, max_num_sequences=self.max_num_sequences, max_beam_width=self.max_beam_width, ) return self.Store( new_tokens=new_tokens, log_probs_store=log_probs_store, beam_search_store=beam_search_store, ) @dataclass(frozen=True, kw_only=True) class Args: max_seq_len: int max_draft_len: int max_num_sequences: int max_beam_width: int max_total_draft_tokens: int disable_overlap_scheduler: bool = False enable_async_worker: bool = False enable_speculative_beam_history_d2h: bool = False watermark_config: "Optional[WatermarkConfig]" = None def __init__(self, args: Args): self.max_seq_len = args.max_seq_len self.max_tokens = args.max_total_draft_tokens + 1 self.max_beam_width = args.max_beam_width # Snapshot of `not self._use_beam_search` so the update_requests # fast-path avoids a property call per iteration. self._batch_fastpath_eligible: bool = self.max_beam_width == 1 if args.max_total_draft_tokens > 0 and args.max_beam_width > 1: raise ValueError("TorchSampler does not support beam search with speculative decoding") if args.watermark_config is not None and args.max_total_draft_tokens > 0: raise ValueError( "watermark_config is not supported with speculative decoding that verifies " "draft tokens in TorchSampler (NGram or user-provided drafts). Disable " "speculative decoding or remove watermark_config." ) self.max_num_sequences = args.max_num_sequences # With the overlap scheduler, sample_async for step i runs before # update_requests for step i-1, so the host-side token lists lag the # device state. Track, per seq slot, how many sampled steps have not # been folded back into the request yet; bad-words handling uses this # to decide whether the newest token must be read device-side. self._track_pending_steps = not args.disable_overlap_scheduler self._pending_steps = [0] * self.max_num_sequences # One row past the real sequence slots. CUDA graph padding rows own no # slot, but the captured graph scatters a token for every row it # replays, so they need a destination that belongs to nobody: every # index in range(max_num_sequences) is owned by some request, and the # overlap scheduler reads a slot's row back as that request's next input # token. Mirrors the spec path's dummy_slot_row. self.dummy_slot_row = self.max_num_sequences self.NEW_TOKENS_SHAPE = ( self.max_tokens, self.max_num_sequences + 1, self.max_beam_width, ) self.CACHE_INDIRECTION_SHAPE = ( self.max_num_sequences, self.max_beam_width, self.max_seq_len, ) self.LOGPROBS_SHAPE = (self.max_num_sequences, self.max_beam_width, self.max_tokens) # Owns max_topk_logprobs / batch_max_topk_logprobs / TOPK_LOGPROBS_SHAPE; # constructed before _create_store, which reads the shape. self._log_probs = LogProbsHandler(self) self._embedding_bias_cache: EmbeddingBiasCache | None = None # The Torch sampler hard-depends on flashinfer. Enforce it once here, at # construction, so the check stays out of the CUDA-graph-captured # sampling loop. if not IS_FLASHINFER_AVAILABLE: raise ImportError( "flashinfer is not available, please install the version pinned " "in requirements.txt." ) # Per-slot Top-P Decay runtime state. Allocated for all # sampler instances; only decay-admitted slots are ever read. self._top_p_decay = TopPDecayHandler(self.max_num_sequences) # Token-ban handling (bad words, no-repeat ngram). The overlap-aware # variant is selected once here from whether the overlap scheduler is # enabled; only it produces the conditional (stale-host) bans. self._token_ban_handler: TokenBanHandler = ( OverlappedTokenBanHandler() if self._track_pending_steps else SynchronousTokenBanHandler() ) # State the in-graph step reads, staged by the host before the forward. # Every buffer below is allocated once, sized for the widest batch, and # updated in place: a captured graph bakes in the ADDRESSES it reads and # scatters through, so a fresh tensor each step would leave it pointing # at freed memory. _in_graph_staged is False when no in-graph step is # prepared, which keeps sample_in_graph a no-op. self._in_graph_dest_indices: torch.Tensor | None = None self._in_graph_staged: bool = False self._in_graph_rows: int = 0 # Tier resolved for the batch being staged, read back by the in-graph # callback (which the engine invokes without the batch). self._current_sample_type: SampleType = SampleType.FULL self._in_graph_request_ids: list[int] = [] self._in_graph_live_rows: int = 0 # Per-row sampling params for the fast tier. self._fast_params: _FusedRowParamsBuffers | None = None # Per-row "carries a watermark" flags the fast tier reads (see _fast_sample_in_graph). self._fast_watermark_enabled: torch.Tensor | None = None self._fast_watermark_flags: tuple[bool, ...] | None = None # What was staged for the batch the buffers above hold, so a repeated # batch is not restaged; see stage_in_graph_sampling. self._staged_batch: _StagedBatch | None = None # The sampler can be created in inference mode, which disallows # in-place mutation of new_tokens. Temporarily exit inference mode. with torch.inference_mode(False): self.store = self._create_store() # One row per sequence slot plus the scratch row (dummy_slot_row). self._slot_params = SlotSamplingParams(self.max_num_sequences + 1, device="cuda") self._watermark: WatermarkHandler | None = ( WatermarkHandler( args.watermark_config, max_num_sequences=self.max_num_sequences, max_seq_len=self.max_seq_len, device="cuda", ) if args.watermark_config is not None else None ) self._finish_reasons_handler = FinishReasonsHandler( max_num_sequences=self.max_num_sequences, max_beam_width=self.max_beam_width, max_tokens=self.max_tokens, max_seq_len=self.max_seq_len, ) # Both are indexed by sequence slot, so they must agree on every # dimension except that new_tokens carries one extra slot row as # scratch for CUDA graph padding (see dummy_slot_row). Nothing # indexes either buffer by that row. new_tokens_shape = self.store.new_tokens.shape finish_reasons_shape = self._finish_reasons_handler.store.finish_reasons_cuda.shape assert ( new_tokens_shape[0] == finish_reasons_shape[0] and new_tokens_shape[1] == finish_reasons_shape[1] + 1 and new_tokens_shape[2] == finish_reasons_shape[2] ), f"{new_tokens_shape} is not {finish_reasons_shape} plus a scratch slot row" self._penalty_handler = PenaltyHandler( max_num_sequences=self.max_num_sequences, max_beam_width=self.max_beam_width, device="cuda", ) # Per-request RNG state backing SamplingParams.seed. Kept per sequence # slot so a seeded request's stream depends only on its own step count, # not on batch composition. The global seed (for unseeded requests) is # fixed for multi-GPU consistency. self._seed_manager = _SeedManager( max_num_sequences=self.max_num_sequences, global_seed=_GLOBAL_SEED, ) # Force number of accepted tokens for speculative decoding testing. # Imported here (not at module level) to keep sampler.sampler off the # import cycle sampler.sampler -> speculative -> (draft_target/mtp) -> # pyexecutor.sampler that this package's lazy __init__ exists to avoid. from ...speculative.interface import get_force_num_accepted_tokens self._force_num_accepted_tokens = get_force_num_accepted_tokens() self._async_worker_init(args.enable_async_worker) # The speculative path bypasses _copy_to_host, so it cannot coexist # with the async worker. LlmArgs validation rejects the explicit # conflict with sampler_force_async_worker. Confidential compute may # also enable the async worker at runtime; if so, disable the path # here with a warning. self._use_speculative_beam_history_d2h: bool = args.enable_speculative_beam_history_d2h if self._use_speculative_beam_history_d2h and self.async_worker_enabled(): logger.warning( "enable_speculative_beam_history_d2h is incompatible with the " "sampler async worker (likely auto-enabled by confidential " "compute); disabling the speculative beam-history D2H path." ) self._use_speculative_beam_history_d2h = False self._stable_greedy_request_ids: list[int] = [] self._stable_greedy_seq_slots: list[int] = [] self._stable_greedy_seq_slots_host: Optional[torch.Tensor] = None self._stable_greedy_seq_slots_cuda: Optional[torch.Tensor] = None # BeamSearchHandler owns the lagged first_finish_reasons snapshots the # speculative predictor reads, so no separate host mirror is kept here. self._beam_search = BeamSearchHandler( store=self.store.beam_search_store, max_seq_len=self.max_seq_len, max_num_sequences=self.max_num_sequences, use_speculative_d2h=self._use_speculative_beam_history_d2h, has_multi_token_stop_words=check_stop_words_length, copy_to_host=self._copy_to_host, make_side_stream_copier=self._make_side_stream_copier, ) @property def _use_beam_search(self) -> bool: return self.max_beam_width > 1 @staticmethod def _request_rewrites_logits(request: LlmRequest) -> bool: """Whether a feature rewrites the logits before sampling. These have to run before the token is picked, and none of them run in-graph, so a request using one cannot be sampled there at all. """ return ( request.py_embedding_bias is not None or bool(getattr(request, "py_bad_words", None)) or bool(getattr(request, "py_no_repeat_ngram_size", None)) or has_occurrence_penalty(request) # min_length is enforced prospectively, by banning end_id out of the # logits until the request is long enough (see # TokenBanHandler._add_min_length_bans), not by a finish-reason # check. Sampling in-graph would skip that ban and let EOS through. or bool(request.py_min_length) ) @staticmethod def _request_needs_finish_reason_kernel(request: LlmRequest) -> bool: """Whether stopping needs the finish-reason kernel rather than host checks. Min length compares against a running count the host path does not track, and a multi-token stop word needs history matching. Everything else reduces to the per-token checks ``_update_requests_single_beam_single_step`` performs. """ return bool(request.py_min_length) or check_stop_words_length(request) def _watermark_row_is_neutral(self, request: LlmRequest) -> bool: """Whether the request filters nothing but its temperature (see WatermarkHandler).""" slot = request.py_seq_slot if slot is not None and self._top_p_decay.uses_decay(slot): return False # A request's sampling params never change, and this is asked for every request on # every step by the fast-path check: resolve them once. neutral = getattr(request, "py_watermark_neutral", None) if neutral is None: params = request_row_params(request) neutral = ( params.mode is SamplingMode.SAMPLING and params.top_k == 0 and params.top_p >= 1.0 and params.min_p <= 0.0 ) request.py_watermark_neutral = neutral return neutral def _request_fast_path_ok(self, req: LlmRequest) -> bool: """Whether one request is plain temperature / top_k / top_p / min_p sampling.""" slot = req.py_seq_slot assert slot is not None self._slot_params.assert_admitted(req, slot) # Greedy is left out on purpose: greedy batches stay on the eager # path. Capturing argmax saves about as much launch overhead as # staging for it costs (on Qwen2-0.5B the two cancelled out), the # eager stable-greedy path already handles them, and expressing # greedy rows as top_k=1 would drag them onto the seeded sampling # path for no gain. if self._slot_params.modes[slot] is not SamplingMode.SAMPLING: return False if self._top_p_decay.uses_decay(slot): return False # The fast tier emits tokens only and samples the raw logits, so # anything that rewrites them first is excluded. Stop words are # allowed: they only decide when a request finishes, which still # runs after the forward. return not (req.py_return_log_probs or self._request_rewrites_logits(req)) def _can_use_fast_path(self, requests: list[LlmRequest]) -> bool: """Whether every request is plain temperature / top_k / top_p / min_p sampling. Rejects anything the fast-tier kernel cannot express per row: beam search, top-p decay (its top_p is per-step state, not a fixed per-row value), and the feature handlers that rewrite logits or need extra outputs -- penalties, token bans, embedding bias, logprobs and speculation, none of which the fast path runs. The answer is fixed for a request's lifetime (its sampling parameters, slot and features do not change), so it is computed once per request and kept on it; a batch then costs one attribute read per request. """ if self._use_beam_search: return False for req in requests: ok = getattr(req, "py_fast_path_ok", None) if ok is None: ok = req.py_fast_path_ok = self._request_fast_path_ok(req) if not ok: return False # In-graph the keyed draw runs on the raw logits, which only a row that filters # nothing but its temperature can do. if ( self._watermark is not None and self._watermark.requested(req) and not self._watermark_row_is_neutral(req) ): return False return True @staticmethod def _request_bypasses_in_graph_sampling(request: LlmRequest) -> bool: """Whether sampling this request in-graph would skip work it depends on. The hook runs at the tail of the forward, before anything the executor does to the logits afterwards: the guided-decoding bitmask applied by ``guided_decoder.execute()``, and the user's own logits processors run by ``_execute_logit_post_processors``. Both happen once ``_forward_step`` has returned, so sampling in-graph would read unmasked logits and emit a token they were meant to forbid, while their output is discarded. Any future post-forward logits mutator belongs here too. """ return request.py_guided_decoding_params is not None or bool( getattr(request, "py_logits_post_processors", None) ) def get_sample_type(self, requests: list[LlmRequest]) -> SampleType: """Pick the simplest tier that satisfies every request in the batch. A batch runs at the capability of its most demanding request, so a tier is chosen only when *every* request in the batch fits it. """ if any(self._request_bypasses_in_graph_sampling(request) for request in requests): return SampleType.FULL if self._can_use_fast_path(requests): return SampleType.FAST return SampleType.FULL @property def current_sample_type(self) -> SampleType: """Tier staged for the current batch; FULL when nothing was staged.""" return self._current_sample_type def invalidate_in_graph_sampling(self) -> None: """Drop any staged in-graph sampling state.""" self._consume_in_graph_staging() self._current_sample_type = SampleType.FULL def _consume_in_graph_staging(self) -> None: """Drop the staged buffers, keeping the tier this step ran at. Used once the staged tokens have been read: the buffers must not be matched against another batch, but ``current_sample_type`` still describes the step in progress and is recorded on its SampleState. """ self._in_graph_staged = False self._in_graph_request_ids = [] self._in_graph_live_rows = 0 def resolve_in_graph_sample_type( self, scheduled_requests: ScheduledRequests, pinned_sample_type: SampleType | None = None, ) -> SampleType: """Pick the tier for this batch, without staging anything. Runs while the CUDA graph key is being built, so the tier can become part of that key. Staging is deliberately left to ``stage_in_graph_sampling``, which runs once the engine has committed to a batch -- see its docstring. Restricted to the plain generation-only step the graphs are captured for: one token per request, beam width 1, no context requests, no drafts. """ if scheduled_requests.num_context_requests != 0 or self.max_beam_width != 1: return SampleType.FULL requests = scheduled_requests.generation_requests if not requests: return SampleType.FULL live_requests = [] dummy_seen = False for request in requests: if get_draft_token_length(request) != 0: return SampleType.FULL if request.is_dummy: dummy_seen = True elif dummy_seen: # Dummies are expected to be a trailing run; the live rows are taken # to be the leading prefix when the results are read back. return SampleType.FULL else: live_requests.append(request) if not live_requests: # An all-dummy batch is a capture pass: the pinned tier decides, # since the warmup requests carry no sampling params of their own. return pinned_sample_type or SampleType.FULL return pinned_sample_type or self.get_sample_type(live_requests) def stage_in_graph_sampling( self, scheduled_requests: ScheduledRequests, sample_type: SampleType, ) -> None: """Stage the buffers ``sample_in_graph`` reads for this batch. Called after the engine has settled which batch the forward runs on, so the staged width matches the width the graph replays at. Staging while the key is built would instead capture the padded batch even on steps that then fall back to eager on the unpadded one, leaving the scatter indices wider than the logits. FULL is a no-op: it covers both "this batch needs the eager sampler" and "the engine dropped the graph for this step". """ self.invalidate_in_graph_sampling() if sample_type is SampleType.FULL: return requests = scheduled_requests.generation_requests if not requests: return # The graph samples and scatters every padded row on every replay, so # stage the whole padded batch to keep the staged width equal to the # width the graph runs at. Padding rows own no sequence slot, so they go # to the scratch row, which no request reads back. seq_slots = [ cast(int, request.py_seq_slot) if request.py_seq_slot is not None else self.dummy_slot_row for request in requests ] request_ids = [request.py_request_id for request in requests] num_rows = len(seq_slots) # In a steady decode batch every step restages the same requests on the # same slots. Their destinations and sampling params cannot have # changed (a slot gets new params only when a new request is admitted # to it, which changes the ids), and the buffers still hold them # because only the staging calls below write the buffers. Only the # seeds, which advance every step, need to be written again. batch_key = (sample_type, request_ids, seq_slots) staged = self._staged_batch if staged is not None and staged.key == batch_key: if sample_type is SampleType.FAST: self._stage_fast_sampling_seeds(seq_slots[: len(staged.live_requests)]) else: self._staged_batch = None live_requests = [request for request in requests if not request.is_dummy] live_rows = len(live_requests) # Write through the persistent buffer: a captured graph scatters # through a fixed address, so this is sized once for the widest # batch and only its leading rows are used. device = self.store.new_tokens.device if ( self._in_graph_dest_indices is None or self._in_graph_dest_indices.numel() < num_rows ): self._in_graph_dest_indices = torch.zeros( max(num_rows, self.max_num_sequences), dtype=torch.int64, device=device ) seq_slots_host = torch.tensor(seq_slots, dtype=torch.int64, pin_memory=prefer_pinned()) self._in_graph_dest_indices[:num_rows].copy_(seq_slots_host, non_blocking=True) if sample_type is SampleType.FAST: self.stage_fast_sampling_params( requests, slots_per_row=seq_slots, live_rows=live_rows, ) staged = self._staged_batch = _StagedBatch( key=batch_key, requests=list(requests), live_requests=live_requests, live_request_ids=[request.py_request_id for request in live_requests], single_step_greedy=not any( self._request_needs_finish_reason_kernel(request) for request in live_requests ), live_seq_slots_host=torch.tensor( [cast(int, request.py_seq_slot) for request in live_requests], dtype=torch.int32, pin_memory=prefer_pinned(), ), ) live_rows = len(staged.live_requests) self._in_graph_live_rows = live_rows self._in_graph_rows = num_rows self._in_graph_request_ids = staged.live_request_ids self._in_graph_staged = True self._current_sample_type = sample_type def _ensure_fast_buffers(self, num_rows: int) -> _FusedRowParamsBuffers: """The fast-tier per-row param buffers, allocated once for the widest batch. Never reallocated for a batch that already fits, since a captured graph holds these addresses. """ if self._fast_params is None or self._fast_params.capacity < num_rows: self._fast_params = _FusedRowParamsBuffers.allocate( max(num_rows, self.max_num_sequences * self.max_tokens), self.store.new_tokens.device, ) return self._fast_params def stage_fast_sampling_params( self, requests: list[LlmRequest], slots_per_row: list[int], live_rows: int, ) -> None: """Stage this batch's per-row sampling params for the in-graph fast-tier step. Runs *before* the forward graph, because picking each row's params and seeds is host work that cannot happen during capture. ``requests`` owns one logits row each, and ``slots_per_row`` gives the sequence slot of each row; the first ``live_rows`` rows are live, the rest padding. """ num_rows = len(slots_per_row) # The buffers are about to hold a batch that stage_in_graph_sampling # may not know about. self._staged_batch = None params = self._ensure_fast_buffers(num_rows) # Live rows read their slot's params. Padding rows are sampled only to # be discarded, so they read the scratch row's neutral params. param_slots = slots_per_row[:live_rows] + [self.dummy_slot_row] * (num_rows - live_rows) param_slots_cuda = torch.tensor( param_slots, dtype=torch.int64, pin_memory=prefer_pinned() ).to(params.floats.device, non_blocking=True) slot_params = self._slot_params params.floats[:, :num_rows].copy_(slot_params.floats.index_select(1, param_slots_cuda)) params.top_k[:num_rows].copy_(slot_params.top_k.index_select(0, param_slots_cuda)) # Philox (seed, offset) per row, from the sampler's own seed manager so # a request's stream stays tied to how far it has decoded. self._seed_manager.observe(requests) self._stage_fast_sampling_seeds(slots_per_row[:live_rows]) watermark = self._watermark if watermark is not None: watermark.observe( [request for request in requests if not request.is_dummy], is_neutral=self._watermark_row_is_neutral, ) flags = [watermark.is_enabled(slot) for slot in slots_per_row[:live_rows]] flags += [False] * (num_rows - live_rows) if ( self._fast_watermark_enabled is None or self._fast_watermark_enabled.numel() < num_rows ): self._fast_watermark_flags = None self._fast_watermark_enabled = torch.zeros( max(num_rows, self.max_num_sequences * self.max_tokens), dtype=torch.bool, device=params.seed.device, ) # A steady decode batch repeats its flags step after step: copy them only on change. flags_key = tuple(flags) if flags_key != self._fast_watermark_flags: self._fast_watermark_flags = flags_key self._fast_watermark_enabled[:num_rows].copy_( torch.tensor(flags, dtype=torch.bool, pin_memory=prefer_pinned()), non_blocking=True, ) def _stage_fast_sampling_seeds(self, live_slots_per_row: list[int]) -> None: """Write the live rows' next Philox (seed, offset) into the staged buffers. _SeedManager is indexed by sequence slot and has no entry for the scratch row, so only the live rows are staged. The padded rows keep whatever the buffers already hold; their tokens are discarded. Only the live rows consume RNG stream. Padding rows borrow a slot that may belong to an active request this batch simply did not schedule; advancing it would move that request's Philox offset by an amount that depends on concurrent load, breaking the guarantee _SeedManager exists for -- that a seeded request's stream depends only on its own draws. """ params = self._fast_params assert params is not None row_seeds = self._seed_manager.take_row_seeds(live_slots_per_row, device=params.seed.device) live_rows = len(live_slots_per_row) params.seed[:live_rows].copy_(row_seeds.seed, non_blocking=True) params.offset[:live_rows].copy_(row_seeds.offset, non_blocking=True) @override def sample_in_graph( self, model_outputs: dict[str, Any], sample_type: SampleType, ) -> None: """Sample the fast tier as the tail of the forward graph. ``FULL`` is eager by definition and a no-op here; those batches are sampled by ``sample_async`` after the forward instead. The destination indices are not computed here -- they depend on batch composition, which is host state unreadable during capture -- but staged into ``_in_graph_dest_indices`` before the graph is launched. """ if sample_type is SampleType.FULL: return dest_indices = self._in_graph_dest_indices if not self._in_graph_staged or dest_indices is None: # Nothing staged: leave sampling to sample_async rather than # sampling against stale buffers. return logits_cuda = model_outputs["logits"] # Staging runs after the batch is settled, so the staged width has to # equal the rows the forward produced. A mismatch means an unexpected # path reached here; skipping silently would be the worst outcome, # because the staged state stays valid and every later replay of this # key would report new_tokens rows nothing ever wrote. Drop the staged # state first so the eager path takes over even when assertions are # compiled out, then fail loudly. if logits_cuda.shape[0] != self._in_graph_rows: staged_rows = self._in_graph_rows self.invalidate_in_graph_sampling() raise AssertionError( f"in-graph sampling staged {staged_rows} rows but the forward " f"produced {logits_cuda.shape[0]}" ) dest_indices = dest_indices[: self._in_graph_rows] self._fast_sample_in_graph(logits_cuda, dest_indices) def _fast_sample_in_graph( self, logits_cuda: torch.Tensor, dest_indices: torch.Tensor, ) -> None: """Sample the fast tier from the staged per-row params. Uses the fused kernel the one-model speculation path uses: per-row temperature / top_k / top_p with explicit Philox (seed, offset) and no host synchronization, so it is capture-safe. The row index is kept out of the RNG so a row's draws depend only on its own (seed, offset). """ params = self._fast_params assert params is not None, ( "fast-tier sampling params were not staged; call stage_fast_sampling_params" ) # Rows staged for this batch, padding included; the buffers are wider. num_rows = dest_indices.shape[0] logits_cuda = logits_cuda[:num_rows] sampling_args = ( logits_cuda, params.temperature[:num_rows], params.top_k[:num_rows], params.top_p[:num_rows], params.min_p[:num_rows], ) watermark = self._watermark next_tokens = fused_sample_from_logits( *sampling_args, seed=params.seed[:num_rows], offset=params.offset[:num_rows] ) if watermark is not None: # Rows that carry a watermark take the keyed draw over the raw logits and advance # their slot, in one launch. The rows' slots are the scatter destinations; padding # rows use the scratch slot and are not flagged, so they leave every slot alone. assert self._fast_watermark_enabled is not None next_tokens = watermark.sample_step( logits_cuda, params.temperature[:num_rows], dest_indices, self._fast_watermark_enabled[:num_rows], next_tokens, from_probs=False, ) scatter_new_tokens(next_tokens, self.store.new_tokens, dest_indices, self.max_beam_width) def _can_use_fast_greedy_path(self, requests: list[LlmRequest]) -> bool: """ Check if we can use the fast argmax path for greedy sampling. """ # Check if all requests use greedy sampling and don't require features # that the fast path skips modes = self._slot_params.modes for req in requests: if modes[cast(int, req.py_seq_slot)] is not SamplingMode.GREEDY: return False # Fast path skips logprobs handling if req.py_return_log_probs: return False return True @staticmethod def _finished_beam_prefix_lengths(finish_reasons: torch.Tensor) -> list[int]: """Count the leading finished beams of every slot in one batched reduction. A request is complete once all of the beams it actually uses have a finish reason, i.e. once its leading ``py_beam_width`` entries are all set. Rather than reducing each request's row separately, reduce the whole tensor once and return, per slot, how many leading beams have finished. The per-request check then degenerates to ``prefix_length >= beam_width``, which needs no tensor work and stays correct for mixed beam widths regardless of what the columns past a request's width hold. Args: finish_reasons: Shape ``(max_batch_size, max_beam_width)``. The finish reasons of every beam of every slot. Returns: Per slot, the number of leading beams whose finish reason is set. """ unfinished = finish_reasons == FinishReason.NOT_FINISHED.value # A beam belongs to the finished prefix iff no unfinished beam precedes it # and it is finished itself, i.e. iff the running count of unfinished beams # up to and including it is still zero. Counting those positions yields the # prefix length directly, and needs no special case for a fully finished # row (every position counts) or a row finishing at beam 0 (none do). return (unfinished.cumsum(dim=1) == 0).sum(dim=1).tolist() def _handle_first_finish_reasons( self, request: LlmRequest, finished_beam_prefix_lengths: list[int], finish_reasons_list: list[list[int]], ) -> bool: """Check if all beams of a request have finished and set the request state accordingly Args: request: LlmRequest. The request to check. finished_beam_prefix_lengths: Per slot, the number of leading beams that have finished, as returned by ``_finished_beam_prefix_lengths``. finish_reasons_list: list[list[int]]. The finish reasons for each beam. Returns: True if all beams have finished, False otherwise. """ assert request.py_seq_slot is not None beam_width = request.py_beam_width if finished_beam_prefix_lengths[request.py_seq_slot] < beam_width: return False request.state = LlmRequestState.GENERATION_COMPLETE request_finish_reasons = finish_reasons_list[request.py_seq_slot] for beam_idx in range(beam_width): request.set_finished_reason( FinishReason(request_finish_reasons[beam_idx]), beam_idx, ) return True @staticmethod @nvtx_range("update_original_tokens") def _update_original_tokens( original_tokens: torch.Tensor, seq_slots: torch.Tensor, seq_lens: torch.Tensor, new_tokens: torch.Tensor, ) -> None: """Update the original tokens storage for the request with the newly sampled tokens When using streaming a requests tokens may be altered, leading to wrong results when called multiple times. Store the original tokens in a separate buffer to use them as a consistent basis when updating the tokens in a request.""" assert new_tokens.device == original_tokens.device, ( "new_tokens and original_tokens must be on the same device" ) original_tokens[seq_slots, :, seq_lens] = new_tokens[0, seq_slots, :] def finish_if_reason( self, request: LlmRequest, finish_reasons: FinishReasonsList, *, step: int, beam_idx: int ) -> bool: assert request.py_seq_slot is not None reason = FinishReason(finish_reasons[request.py_seq_slot][step][beam_idx]) valid_reasons = {FinishReason.END_ID, FinishReason.LENGTH, FinishReason.STOP_WORDS} if reason in valid_reasons: request.finish_by(reason, beam_idx) return True return False def _process_draft_tokens( self, request: LlmRequest, new_tokens_list: list[list[list[int]]], finish_reasons: FinishReasonsList, ) -> int: """Add the target's tokens up to the first rejected draft token. Only drafter-based speculation (NGram, user-provided) attaches draft tokens to requests sampled here; a draft token is accepted while it matches the token the target sampled at the same position. When TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS is set, up to that many draft tokens are accepted without comparing them. Returns the number of accepted draft tokens. """ new_token = add_token(request, new_tokens_list, beam_idx=DEFAULT_BEAM_IDX) if ( self.finish_if_reason(request, finish_reasons, step=0, beam_idx=DEFAULT_BEAM_IDX) or get_draft_token_length(request) == 0 ): return 0 force_accept = self._force_num_accepted_tokens != 0 draft_tokens = ( request.py_draft_tokens[: self._force_num_accepted_tokens] if force_accept else request.py_draft_tokens ) num_accepted = 0 for draft_token in draft_tokens: if not force_accept and draft_token != new_token: break num_accepted += 1 new_token = add_token( request, new_tokens_list, beam_idx=DEFAULT_BEAM_IDX, step=num_accepted ) if self.finish_if_reason( request, finish_reasons, step=num_accepted, beam_idx=DEFAULT_BEAM_IDX ): break return num_accepted @classmethod def _collect_new_requests_for_setup( cls, scheduled_requests: ScheduledRequests ) -> list[LlmRequest]: # ADP can inject generation-phase dummy requests after request activation. # Those still need sampler-side slot initialization before grouping/sampling. # The two source lists are disjoint by construction. context_new_requests = [ request for request in scheduled_requests.context_requests_last_chunk if not request.is_finished ] adp_dummy_generation_requests = [ request for request in scheduled_requests.generation_requests if request.is_attention_dp_dummy ] return context_new_requests + adp_dummy_generation_requests @override def validate_request(self, request: LlmRequest) -> None: # Reject unsupported top-p-decay combinations at admission, so only the offending # request fails (raising later, inside setup_sampler_step or sampling, would abort # the whole executor step). Occurrence penalties have no unsupported combination # left: beam search is supported, and beam search with speculative decoding is # rejected for the whole sampler in __init__. self._top_p_decay.validate_request(request) if self._use_beam_search: if request.py_return_log_probs: if request.py_num_logprobs > 1: raise ValueError( "Beam search does not support returning multiple logprobs per request" ) if request.py_num_logprobs != 0: raise ValueError( "Beam search only supports returning the sampled logprob per token" ) # Every early_stopping mode is served by the candidate-beams-array # path (see beam_search_sampling_batch_cba); the mode only selects # the done verdict computed there. @override def release_request(self, request: LlmRequest) -> None: self._finish_reasons_handler.release_request(request) # The staged batch is reused only by the same requests on a later step, which a released # request never joins again; dropping it keeps it from holding the request (and the # outputs it owns, such as generation logits) after its output was returned. staged = self._staged_batch if staged is not None and request.py_request_id in staged.live_request_ids: self._staged_batch = None @override @nvtx_range("setup_sampler_step") def setup_sampler_step(self, scheduled_requests: ScheduledRequests) -> None: """Setup the sampler step for the requests Args: scheduled_requests: The scheduled requests to set up the sampler step for. """ new_requests = self._collect_new_requests_for_setup(scheduled_requests) if not new_requests: return # Used for all store updates seq_slots: list[int] = [] # Used for beam search updates max_prompt_len: int = 0 # Prepare finish reasons handler self._finish_reasons_handler.setup_new_request_handling() for request in new_requests: slot = request.py_seq_slot assert slot is not None seq_slots.append(slot) # update temp_data with this requests data self._finish_reasons_handler.prepare_for_new_request(request) if self._use_beam_search: assert not (request.py_return_log_probs and request.py_num_logprobs > 1), ( "Beam search does not support returning multiple logprobs per request" ) max_prompt_len = max(max_prompt_len, request.py_prompt_len) if self._use_speculative_beam_history_d2h: # Drop stale predictor state from any prior occupant of this slot. self._beam_search.clear_slot(slot) self._penalty_handler.prepare_for_new_request(request, slot) max_lens = self._finish_reasons_handler.new_max_lens end_ids = self._finish_reasons_handler.new_end_ids prompt_lens = [request.py_prompt_len for request in new_requests] beam_caps = [request.py_beam_width for request in new_requests] # Perform updates to the stores full_list = [seq_slots, max_lens, end_ids, prompt_lens, beam_caps] # perform only a single copy full_list_tensor_host = torch.tensor( full_list, device="cpu", dtype=torch.int32, pin_memory=prefer_pinned() ) full_list_tensor_cuda = full_list_tensor_host.to(device="cuda", non_blocking=True) seq_slots_tensor_host = full_list_tensor_host[0] seq_slots_tensor_cuda = full_list_tensor_cuda[0] max_lens_tensor_cuda = full_list_tensor_cuda[1] end_ids_tensor_cuda = full_list_tensor_cuda[2] prompt_lens_tensor_cuda = full_list_tensor_cuda[3] beam_caps_tensor_cuda = full_list_tensor_cuda[4] # Cast to int64 once for downstream ``index_copy_`` / ``index_fill_`` calls. seq_slots_tensor_cuda_long = seq_slots_tensor_cuda.long() self._finish_reasons_handler.update_for_new_request( seq_slots_cuda_long=seq_slots_tensor_cuda_long, max_lengths_cuda=max_lens_tensor_cuda, end_ids_cuda=end_ids_tensor_cuda, seq_slots_host=seq_slots_tensor_host, ) self._slot_params.admit(new_requests, seq_slots_tensor_cuda_long) self._top_p_decay.setup_for_new_requests( new_requests, new_seq_slots_cuda_long=seq_slots_tensor_cuda_long ) self._penalty_handler.update_for_new_requests( new_seq_slots_cuda_long=seq_slots_tensor_cuda_long ) if self._use_beam_search: beam_search_store = self.store.beam_search_store assert beam_search_store is not None # Allocate the CBA tensors on the first beam-search request: every # early_stopping mode runs on that path. beam_search_store.ensure_cba() prepare_beam_search( beam_search_store, self.store.log_probs_store, seq_slots_long=seq_slots_tensor_cuda_long, max_prompt_len=max_prompt_len, prompt_lens_cuda=prompt_lens_tensor_cuda, beam_caps_cuda=beam_caps_tensor_cuda, ) @override @nvtx_range("update_requests") @torch.inference_mode() def update_requests( self, state: SampleStateTorch, resource_manager: Optional[ResourceManager] = None, ) -> None: if state.sampler_event: state.sampler_event.synchronize() if not state.requests: return if self._track_pending_steps: for req in state.requests: slot = req.py_seq_slot if slot is not None and self._pending_steps[slot] > 0: self._pending_steps[slot] -= 1 assert state.host is not None # Reuse sample_async's qualification instead of rechecking every # request after the asynchronous sample completes. if state.host.single_step_greedy: self._update_requests_single_beam_single_step(state) return new_tokens = state.host.new_tokens first_finish_reasons_host = state.host.first_finish_reasons if first_finish_reasons_host is not None: first_finish_reasons = first_finish_reasons_host.tolist() # Reduce every slot at once; the per-request loop below only reads the # result and updates the request objects. finished_beam_prefix_lengths = self._finished_beam_prefix_lengths( first_finish_reasons_host ) else: first_finish_reasons = [] finished_beam_prefix_lengths = [] new_tokens_list = new_tokens.tolist() logprobs_state_list: LogProbsStateList | None = None if state.host.logprobs_state is not None: logprobs_state_list = LogProbsStateList.from_logprobs_state(state.host.logprobs_state) # Beam-search requests read their finish reasons from # first_finish_reasons; only single-beam requests (and the batched # fast path below) consume the per-step finish_reasons list, whose # conversion scales with max_num_sequences x max_beam_width. finish_reasons: FinishReasonsList = [] if (self._batch_fastpath_eligible and logprobs_state_list is None) or any( req.py_beam_width == 1 for req in state.requests ): finish_reasons = state.host.finish_reasons_list() beam_history_builders = state.beam_history_builders assert (beam_history_builders is not None) == self._use_beam_search def _maybe_build_beam_history(req_idx: int) -> BeamHistory | None: if ( beam_history_builders is not None and (beam_history_builder := beam_history_builders[req_idx]) is not None ): return beam_history_builder() else: return None finalized_token_updates: list[tuple[int, list[int]]] = [] # Fast-path (batched pybind): when the batch is greedy with no beam # search, no logprobs, no draft tokens and no stop-words, collapse # per-request pybind chatter into one batched # add_new_tokens_to_requests call. Single-pass eligibility check with # early-break; falls through when any invariant breaks. if self._batch_fastpath_eligible and logprobs_state_list is None: alive_reqs: list[LlmRequest] = [] tokens_flat: list[int] = [] fastpath_ok = True new_tokens_step0 = new_tokens_list[0] for req in state.requests: if req.state == LlmRequestState.GENERATION_COMPLETE: continue if get_draft_token_length(req) != 0 or req.py_stop_words_list: fastpath_ok = False break assert req.py_seq_slot is not None alive_reqs.append(req) tokens_flat.append(new_tokens_step0[req.py_seq_slot][DEFAULT_BEAM_IDX]) if fastpath_ok and alive_reqs: add_new_tokens_to_requests(alive_reqs, tokens_flat, DEFAULT_BEAM_IDX) _valid_finish_reasons = { FinishReason.END_ID, FinishReason.LENGTH, FinishReason.STOP_WORDS, } for req in alive_reqs: assert req.py_seq_slot is not None reason_val = finish_reasons[req.py_seq_slot][0][DEFAULT_BEAM_IDX] if reason_val != 0: reason = FinishReason(reason_val) if reason in _valid_finish_reasons: req.finish_by(reason, DEFAULT_BEAM_IDX) req.py_num_accepted_draft_tokens = 0 req.py_num_draft_tokens_verified = 0 req.py_rewind_len = 0 req.py_decoding_iter += 1 return for req_idx, req in enumerate(state.requests): if req.state == LlmRequestState.GENERATION_COMPLETE: self._top_p_decay.retire_slot(req) continue if req.py_beam_width > 1: # Context-only (disaggregated prefill) requests reach the # sampler already flagged as finished -- the context server is # done with them -- but they are migrating, not completing: # the generation server decodes the rest. Finalizing here # would rewrite the beam rows from the beam history, and after # a single step only beam 0 has history; every other beam is # all BEAM_SEARCH_PAD_TOKEN, so set_generated_tokens would # give it an empty generated sequence. The handoff reads the # per-beam first token back out via getTokens().back() # (llmRequest.cpp), which would then hand the generation # server the prompt tail instead of that beam's token. Append # this step's tokens instead and let the generation side own # finalization. # The builder only produces a history when every beam has # finished, checking it here first keeps the builder off # for every step that does not end the request. assert req.py_seq_slot is not None request_finishing = bool(finished_beam_prefix_lengths) and ( finished_beam_prefix_lengths[req.py_seq_slot] >= req.py_beam_width ) if ( not req.is_context_only_request and request_finishing and (beam_history := _maybe_build_beam_history(req_idx)) is not None ): finalize_beam(req, beam_history) else: # Only the leading beam_width_out columns hold real tokens; # the op pads the rest of the store-width row with # BEAM_SEARCH_PAD_TOKEN. Appending those would put the # sentinel into the request's token history, which is # visible to streaming consumers and to anything reading # get_tokens() mid-flight (e.g. the token-ban suffix # matching) even though finalization later rewrites it. beam_width_out = _get_beam_width_out(req) if beam_width_out == req.py_beam_width: # One binding call appends this step's token to every # beam; the C++ side requires exactly beam_width tokens. assert req.py_seq_slot is not None req.add_new_tokens(new_tokens_list[0][req.py_seq_slot][:beam_width_out]) else: for beam_idx in range(beam_width_out): # Beam search does not support speculative decoding. add_token(req, new_tokens_list, beam_idx=beam_idx) self._log_probs.handle_logprobs( req, logprobs_state_list=logprobs_state_list, count=1 ) assert first_finish_reasons_host is not None self._handle_first_finish_reasons( req, finished_beam_prefix_lengths, first_finish_reasons ) if self._use_speculative_beam_history_d2h: # Snapshot for the next step's predictor. assert req.py_seq_slot is not None self._beam_search.record_first_finish_reasons( req.py_seq_slot, first_finish_reasons_host[req.py_seq_slot] ) if req.is_context_only_request: beam_search_store = self.store.beam_search_store assert beam_search_store is not None assert req.py_seq_slot is not None beam_width = req.py_beam_width first_gen_scores = ( beam_search_store.cum_log_probs[req.py_seq_slot, :beam_width] .detach() .cpu() .tolist() ) first_gen_tokens = [ new_tokens_list[0][req.py_seq_slot][beam_idx] for beam_idx in range(beam_width) ] first_gen_log_probs = [ {token_id: Logprob(logprob=log_prob, rank=None)} for token_id, log_prob in zip(first_gen_tokens, first_gen_scores) ] req.py_result.set_first_gen_log_probs(first_gen_log_probs) req.py_num_accepted_draft_tokens = 0 req.py_num_draft_tokens_verified = 0 req.py_rewind_len = 0 else: processed = 1 num_tokens_before = req.get_num_tokens(DEFAULT_BEAM_IDX) num_accepted = self._process_draft_tokens( req, new_tokens_list=new_tokens_list, finish_reasons=finish_reasons, ) if (actual_draft_len := get_draft_token_length(req)) > 0: req.py_num_accepted_draft_tokens = num_accepted # Pair the acceptance count with the real proposal count # of the same step: py_draft_tokens is padded to the # static max for CUDA graphs, so use the pre-padding # length the drafter recorded. py_rewind_len keeps the # padded length (padding occupies KV cache). effective_len = req.py_draft_tokens_effective_len req.py_num_draft_tokens_verified = ( min(effective_len, actual_draft_len) if effective_len is not None else actual_draft_len ) req.py_rewind_len = actual_draft_len - num_accepted else: req.py_num_accepted_draft_tokens = 0 req.py_num_draft_tokens_verified = 0 req.py_rewind_len = 0 processed += num_accepted if actual_draft_len > 0: num_new_tokens = req.get_num_tokens(DEFAULT_BEAM_IDX) - num_tokens_before if num_new_tokens > 0: assert req.py_seq_slot is not None confirmed_tokens = req.get_tokens(DEFAULT_BEAM_IDX)[-num_new_tokens:] finalized_token_updates.append((req.py_seq_slot, confirmed_tokens)) self._log_probs.handle_logprobs( req, logprobs_state_list=logprobs_state_list, count=processed ) req.py_decoding_iter += 1 # Check None or empty list if req.py_stop_words_list: self._finish_reasons_handler.store.num_accepted_draft_tokens_host[ req.py_seq_slot ] = req.py_num_accepted_draft_tokens # req.state can become GENERATION_COMPLETE within this loop iteration # (e.g. process_draft_tokens -> _handle_stop_criteria -> finish_by), # so the comparison is valid at runtime; the type checker narrowed it away via the # `continue` at the top of the loop and cannot see the mutating calls. if req.state == LlmRequestState.GENERATION_COMPLETE: # type: ignore[comparison-overlap] self._top_p_decay.retire_slot(req) self._penalty_handler.update_token_counts(finalized_token_updates) @nvtx_range("_update_requests_single_beam_single_step") def _update_requests_single_beam_single_step(self, state: SampleStateTorch) -> None: """Update the common greedy, single-token case without draft machinery.""" assert state.host is not None requests = [ request for request in state.requests if request.state != LlmRequestState.GENERATION_COMPLETE ] if not requests: return all_new_tokens = state.host.new_tokens.tolist() if len(requests) == len(state.requests): new_tokens = all_new_tokens else: new_tokens = [ new_token for request, new_token in zip(state.requests, all_new_tokens) if request.state != LlmRequestState.GENERATION_COMPLETE ] add_new_tokens_to_requests(requests, new_tokens, DEFAULT_BEAM_IDX) # sample_async deliberately omits the device finish-reason tensor for # this qualified path; completion is derived from compact host tokens. assert state.host.finish_reasons is None for request, new_token in zip(requests, new_tokens): # Only single-token stop words reach this path (multi-token ones # need history matching, so they take the finish-reason kernel). # Order matches handle_stop_criteria: END_ID, then LENGTH, then # STOP_WORDS, so a terminal EOS at the token limit is reported as # END_ID. if new_token == request.py_end_id: request.finish_by(FinishReason.END_ID, DEFAULT_BEAM_IDX) elif ( request.max_beam_num_tokens - request.py_orig_prompt_len >= request.py_max_new_tokens or request.max_beam_num_tokens >= self.max_seq_len ): request.finish_by(FinishReason.LENGTH, DEFAULT_BEAM_IDX) elif meet_stop_token_criteria(request, new_token, DEFAULT_BEAM_IDX): request.finish_by(FinishReason.STOP_WORDS, DEFAULT_BEAM_IDX) request.py_num_accepted_draft_tokens = 0 request.py_rewind_len = 0 request.py_decoding_iter += 1 @override @torch.inference_mode() @nvtx_range("sample_async") def sample_async( self, scheduled_requests: ScheduledRequests, model_outputs: dict[str, Any], num_context_logits_prefix_sum: list[int], resource_manager: Optional[ResourceManager] = None, ) -> SampleStateTorch: # NB: There are 1 + get_draft_token_length(request) tokens per request. self.setup_sampler_step(scheduled_requests) new_tokens = self.store.new_tokens if self._track_pending_steps: # A context request claims a (possibly reused) slot: clear any # counter leaked by a prior occupant that never got its final # update_requests. Must happen before _process_requests, which # reads the counters for bad-words staleness. for r in scheduled_requests.context_requests: assert r.py_seq_slot is not None self._pending_steps[r.py_seq_slot] = 0 # seq_slots_cuda / seq_lens_cuda are cast once inside # _process_requests and shared with the beam-search metadata builder. ( requests, seq_slots_host, seq_lens_host, seq_slots_cuda, seq_lens_cuda, new_tokens_host, single_step_greedy, ) = self._process_requests( scheduled_requests, model_outputs, new_tokens, num_context_logits_prefix_sum, ) if self._track_pending_steps and requests: for r in requests: assert r.py_seq_slot is not None self._pending_steps[r.py_seq_slot] += 1 finish_reasons_host: torch.Tensor | None = None first_finish_reasons_host: torch.Tensor | None = None beam_history_builders: list[BeamHistoryBuilder | None] | None = None # Forwarded to _record_sampler_event so SamplerEvent.synchronize # awaits any side-stream D2H copies host-side. side_stream_event: torch.cuda.Event | None = None if requests: beam_search_store = self.store.beam_search_store assert self._use_beam_search == (beam_search_store is not None) if not single_step_greedy: assert seq_lens_host is not None assert seq_lens_cuda is not None finish_reasons_device = self._finish_reasons_handler.write_finish_reasons( seq_slots_host=seq_slots_host, seq_slots_cuda=seq_slots_cuda, seq_lens_cuda=seq_lens_cuda, new_tokens_cuda=new_tokens, first_finish_reasons_cuda=( beam_search_store.first_finish_reasons if beam_search_store is not None else None ), pending_harvest_cuda=( beam_search_store.pending_harvest if beam_search_store is not None else None ), ) finish_reasons_host = self._copy_to_host(finish_reasons_device) if self._use_beam_search: assert beam_search_store is not None assert seq_lens_cuda is not None first_finish_reasons = beam_search_store.first_finish_reasons first_finish_reasons_host = self._copy_to_host(first_finish_reasons) self._update_original_tokens( beam_search_store.original_tokens, seq_slots_cuda, seq_lens_cuda, new_tokens ) beam_history_builders, side_stream_event = self._beam_search.prepare_beam_histories( requests, finish_reasons=first_finish_reasons ) # copy logprobs to host logprobs_state: LogProbsState | None = None if self._log_probs._return_log_probs(requests): log_probs_store = self.store.log_probs_store host_topk_vals = self._copy_to_host( log_probs_store.topk_vals[..., : self._log_probs.batch_max_topk_logprobs] ) host_topk_indices = self._copy_to_host( log_probs_store.topk_indices[..., : self._log_probs.batch_max_topk_logprobs] ) host_sampled_vals = self._copy_to_host(log_probs_store.sampled_log_probs) host_sampled_indices = self._copy_to_host(log_probs_store.sampled_log_prob_indices) host_sampled_rank = self._copy_to_host(log_probs_store.sampled_log_prob_ranks) logprobs_state = LogProbsState( topk_vals=host_topk_vals, topk_indices=host_topk_indices, sampled_vals=host_sampled_vals, sampled_indices=host_sampled_indices, sampled_rank=host_sampled_rank, ) sampler_event = self._record_sampler_event(side_stream_event=side_stream_event) return SampleStateTorch( requests=requests, device=SampleStateTensors(new_tokens=new_tokens), host=SampleStateTensorsHostTorch( new_tokens=new_tokens_host, finish_reasons=finish_reasons_host, first_finish_reasons=first_finish_reasons_host, logprobs_state=logprobs_state, single_step_greedy=single_step_greedy, ), sampler_event=sampler_event, beam_history_builders=beam_history_builders, sample_type=self._current_sample_type, ) def _group_requests( self, requests: list[LlmRequest], seq_slots: list[int] ) -> list[_SamplingGroup]: """Split the batch into the groups sampled by one call each. Requests are grouped by sampling mode and by whether they need probs. Beam-search requests are further split by what one beam-search step must share (``BeamSearchParams.group_key``). Request indices within a group are ascending. """ slot_params = self._slot_params modes = slot_params.modes need_processed_logprobs = slot_params.need_processed_logprobs groups: dict[Hashable, _SamplingGroup] = {} for req_idx, (request, slot) in enumerate(zip(requests, seq_slots)): slot_params.assert_admitted(request, slot) mode = modes[slot] needs_probs = need_processed_logprobs[slot] beam_params = None key: Hashable if mode is SamplingMode.BEAM: beam_params = BeamSearchParams.from_request(request) key = (mode, needs_probs, beam_params.group_key) else: key = (mode, needs_probs) group = groups.get(key) if group is None: group = groups[key] = _SamplingGroup(mode=mode, needs_probs=needs_probs) group.req_indices.append(req_idx) if beam_params is not None: group.beam_params.append(beam_params) return list(groups.values()) def _sample_fused( self, logits_cuda: torch.Tensor, slots_per_row: list[int], *, return_probs: bool, ) -> tuple[torch.Tensor, torch.Tensor | None, torch.Tensor]: """Sample rows with the fused kernel, from their slots' params and seeds. Returns the tokens, the probs when ``return_probs``, and the per-row temperature. """ device = logits_cuda.device seeds = self._seed_manager.take_row_seeds(slots_per_row, device=device) slots_cuda = torch.tensor(slots_per_row, dtype=torch.int64, pin_memory=prefer_pinned()).to( device, non_blocking=True ) temperature, top_k, top_p, min_p = self._slot_params.gather(slots_cuda) top_p = self._top_p_decay.override_top_p(top_p, slots_cuda) # A device-side assert rather than a host-side check, which would force a # device sync on every call. torch._assert_async(~torch.any(torch.isnan(logits_cuda))) params = (logits_cuda, temperature, top_k, top_p, min_p) watermark = self._watermark step = watermark.step_inputs(slots_per_row) if watermark is not None else None if watermark is not None and step is not None: # Rows that carry a watermark replace the uniform draw by the keyed Gumbel-max # draw over the same filtered distribution; the rest keep their ordinary draw. wm_slots, wm_enabled, all_enabled, all_neutral = step if all_neutral and not return_probs: # No filter beyond the temperature: draw on the logits, no probabilities needed. fallback = None if not all_enabled or watermark.needs_fallback_token: fallback = fused_sample_from_logits( *params, seed=seeds.seed, offset=seeds.offset ) tokens = watermark.sample_step( logits_cuda, temperature, wm_slots, wm_enabled, fallback, from_probs=False ) return tokens, None, temperature tokens, probs = fused_sample_from_logits_with_probs( *params, seed=seeds.seed, offset=seeds.offset ) if not return_probs and not all_enabled: # Opted-out rows keep the native tokens-only draw. Marked rows # retain the with-probs token as their repeated-context fallback. stock_tokens = fused_sample_from_logits( *params, seed=seeds.seed, offset=seeds.offset ) tokens = torch.where(wm_enabled, tokens, stock_tokens) tokens = watermark.sample_step( probs, None, wm_slots, wm_enabled, tokens, from_probs=True ) return tokens, (probs if return_probs else None), temperature if return_probs: tokens, probs = fused_sample_from_logits_with_probs( *params, seed=seeds.seed, offset=seeds.offset ) return tokens, probs, temperature return ( fused_sample_from_logits(*params, seed=seeds.seed, offset=seeds.offset), None, temperature, ) @nvtx_range("sample_batched_by_strategy") @torch.inference_mode() def _sample_batched_by_strategy( self, logits_cuda: torch.Tensor, requests: list[LlmRequest], *, logits_cuda_indexer: _PackedStepIndexer, req_num_generated_tokens: torch.Tensor, req_num_steps: torch.Tensor, req_offsets: torch.Tensor, seq_slots: torch.Tensor, seq_lens: Optional[torch.Tensor] = None, seq_slots_cuda: torch.Tensor, seq_lens_cuda: torch.Tensor, token_dtype: torch.dtype, return_log_probs: bool, ) -> _BatchedSamplingResult: cuda_device = logits_cuda.device self._seed_manager.observe(requests) if self._watermark is not None: self._watermark.observe(requests, is_neutral=self._watermark_row_is_neutral) seq_slots_list = seq_slots.tolist() groups = self._group_requests(requests, seq_slots_list) # Tensors for collecting sampling results (in batch ordering) batch_req_indices = torch.empty((len(requests),), dtype=torch.int32) batch_next_tokens_cuda_int = torch.empty( (logits_cuda.size(0), self.max_beam_width), device=cuda_device, dtype=token_dtype ) # Only processed logprobs need a full-vocabulary buffer. Raw logprobs # are selected directly from logits; beam search uses its own history. reqs_indices_needing_raw_logprobs: list[int] = [] reqs_indices_needing_raw_logits: list[int] = [] num_logprobs_rows = 0 if return_log_probs: need_raw_logprobs = self._slot_params.need_raw_logprobs need_processed_logprobs = self._slot_params.need_processed_logprobs for req_idx, (request, slot, num_generated_tokens) in enumerate( zip(requests, seq_slots_list, req_num_generated_tokens.tolist()) ): if need_raw_logprobs[slot]: reqs_indices_needing_raw_logprobs.append(req_idx) if self._use_beam_search and request.py_beam_width != 1: continue if need_raw_logprobs[slot]: reqs_indices_needing_raw_logits.append(req_idx) if need_processed_logprobs[slot]: num_logprobs_rows += num_generated_tokens batch_processed_logprobs_cuda = ( torch.empty( (num_logprobs_rows, logits_cuda.size(1)), device=cuda_device, dtype=torch.float32 ) if return_log_probs else None ) reqs_indices_needing_processed_logprobs: list[int] = [] batch_processed_logprobs_offset = 0 batch_req_idx_offset_start = 0 batch_next_tokens_offset_start = 0 for group in groups: group_req_indices = torch.tensor(group.req_indices, dtype=torch.int32) batch_req_idx_offset_end = batch_req_idx_offset_start + len(group.req_indices) batch_req_indices[batch_req_idx_offset_start:batch_req_idx_offset_end] = ( group_req_indices ) # The group's logits rows, one per step of each request. group_req_1st_index = group.req_indices[0] group_req_last_index = group.req_indices[-1] if group_req_last_index - group_req_1st_index + 1 == len(group.req_indices): # Avoid data movement if indices are contiguous group_logits_cuda = logits_cuda[ req_offsets[group_req_1st_index] : req_offsets[group_req_last_index] + req_num_generated_tokens[group_req_last_index] ] else: group_logits_cuda = logits_cuda.index_select( 0, logits_cuda_indexer[group_req_indices].to( device=cuda_device, non_blocking=True ), ) group_temperature_cuda: torch.Tensor | None = None if group.mode is SamplingMode.GREEDY: group_next_tokens_cuda, group_softmax_cuda = greedy_search_sampling_batch( group_logits_cuda, return_probs=group.needs_probs ) elif group.mode is SamplingMode.SAMPLING: group_slots_per_step = [ seq_slots_list[req_idx] for req_idx, steps in zip( group.req_indices, req_num_steps[group_req_indices].tolist() ) for _ in range(steps) ] group_next_tokens_cuda, group_softmax_cuda, group_temperature_cuda = ( self._sample_fused( group_logits_cuda, group_slots_per_step, return_probs=group.needs_probs ) ) else: stop_words_metadata_cuda: torch.Tensor | None = None if self._finish_reasons_handler.has_stop_word_history: self._finish_reasons_handler.record_stop_word_stream() stop_words_metadata_cuda = ( self._finish_reasons_handler.store.stop_words_metadata_cuda ) beam_step = BeamSearchStep.from_params( group.beam_params, cuda_device, return_probs=group.needs_probs ) group_next_tokens_cuda, group_softmax_cuda = beam_step.sample( group_logits_cuda, metadata=self._beam_search.build_metadata( requests=requests, group_req_indices=group_req_indices, seq_slots=seq_slots, seq_lens=seq_lens, seq_slots_cuda=seq_slots_cuda, seq_lens_cuda=seq_lens_cuda, num_requests=len(requests), new_log_probs=self.store.log_probs_store.sampled_log_probs[ ..., DEFAULT_STEP_IDX ], end_ids_cuda=self._finish_reasons_handler.store.end_ids_cuda, stop_words_metadata_cuda=stop_words_metadata_cuda, ), ) group_temperature_cuda = beam_step.temperature batch_next_tokens_offset_end = ( batch_next_tokens_offset_start + group_next_tokens_cuda.size(0) ) # if no beam search is used, the shape is (batch_size,), so we need to unsqueeze it to (batch_size, 1) if group_next_tokens_cuda.dim() == 1: group_next_tokens_cuda = group_next_tokens_cuda.unsqueeze(1) batch_next_tokens_cuda_int[ batch_next_tokens_offset_start:batch_next_tokens_offset_end ].copy_(group_next_tokens_cuda, non_blocking=True) # --- Processed logprobs: every request of a group with needs_probs wants them --- if group.needs_probs: assert batch_processed_logprobs_cuda is not None assert group_softmax_cuda is not None if logits_cuda.dtype != batch_processed_logprobs_cuda.dtype: # NB: tensorrt_llm._torch.modules.logits_processor.LogitsProcessor forces logits # to float32, so this warning is not expected to get triggered. To support, e.g., # bfloat16, unittest/_torch/sampler/test_logits_logprobs.py::TestLogsprobsInBatchedSampling # should be parametrized accordingly and dtype handling in TorchSampler needs # to be cleaned up to ensure consistent dtypes for logits, temperature, softmax, etc. logger.warning_once( "Processed logprobs calculation is only tested with float32 logits. Results for lower " "precision logits may be inaccurate.", key="WARN_INACCURATE_PROCESSED_LOGPROBS", ) reqs_indices_needing_processed_logprobs += group.req_indices # Beam-search requests get theirs from the beam history instead, # so gather logprobs only for the single-beam ones. gather_processed_logprobs = ( [requests[req_idx].py_beam_width == 1 for req_idx in group.req_indices] if self._use_beam_search else None ) proc_lp_temperature_cuda: torch.Tensor | None if gather_processed_logprobs is None or all(gather_processed_logprobs): proc_lp_logits_cuda = group_logits_cuda proc_lp_softmax_cuda = group_softmax_cuda proc_lp_temperature_cuda = group_temperature_cuda if proc_lp_temperature_cuda is not None: proc_lp_temperature_cuda = proc_lp_temperature_cuda.unsqueeze(-1) else: # Masked with an output size computed on the host, so that # selecting the rows does not sync with the device. proc_lp_group_steps = req_num_generated_tokens[group_req_indices] proc_lp_req_mask = torch.tensor(gather_processed_logprobs, dtype=torch.bool) proc_lp_step_mask_cuda = torch.repeat_interleave( proc_lp_req_mask.to(device=cuda_device, non_blocking=True), proc_lp_group_steps.to(device=cuda_device, non_blocking=True), output_size=cast(int, proc_lp_group_steps.sum().item()), ).unsqueeze(-1) proc_lp_steps_num_selected = cast( int, (proc_lp_req_mask.to(dtype=proc_lp_group_steps.dtype) * proc_lp_group_steps) .sum() .item(), ) def _select_steps(t: torch.Tensor, mask: torch.Tensor) -> torch.Tensor: selected = t.new_empty((proc_lp_steps_num_selected, *t.shape[1:])) torch.masked_select(t, mask, out=selected.view(-1)) return selected proc_lp_logits_cuda = _select_steps(group_logits_cuda, proc_lp_step_mask_cuda) proc_lp_softmax_cuda = _select_steps(group_softmax_cuda, proc_lp_step_mask_cuda) proc_lp_temperature_cuda = None if group_temperature_cuda is not None: proc_lp_temperature_cuda = _select_steps( group_temperature_cuda, proc_lp_step_mask_cuda.squeeze(-1) ).unsqueeze(-1) # Apply temperature if proc_lp_temperature_cuda is not None: if proc_lp_logits_cuda is group_logits_cuda: proc_lp_logits_cuda = proc_lp_logits_cuda / proc_lp_temperature_cuda else: proc_lp_logits_cuda /= proc_lp_temperature_cuda # Reconstruct processed logprobs from softmax and logits proc_lp_max_logits_cuda, proc_lp_max_logit_indices = proc_lp_logits_cuda.max(dim=-1) # NB: The operations below could be fused using torch.compile (cf. Fusions) for performance. proc_lp_renorm_offset_cuda = ( proc_lp_softmax_cuda.gather( dim=1, index=proc_lp_max_logit_indices.unsqueeze(-1) ) .log() .squeeze(-1) - proc_lp_max_logits_cuda ) batch_processed_logprobs_offset_next = ( batch_processed_logprobs_offset + proc_lp_max_logits_cuda.size(0) ) proc_lp_dest_cuda_view = batch_processed_logprobs_cuda[ batch_processed_logprobs_offset:batch_processed_logprobs_offset_next ] batch_processed_logprobs_offset = batch_processed_logprobs_offset_next torch.add( proc_lp_logits_cuda, proc_lp_renorm_offset_cuda.unsqueeze(-1), out=proc_lp_dest_cuda_view, ) # NB: In principle. copying and masking whole-vocab tensors could be avoided by storing: # - the number of unmasked tokens 'nu' # - r := log(max(probs)) - max(logits) # Later, processed logprobs can be reconstructed from logits _after_ applying # top-k: Add 'r' and mask smallest entries so that only min(k, nu) tokens remain. # Currently, this approach is not viable because various tie-breaking behaviors are # unspecified for several of the kernels involved. proc_lp_dest_cuda_view.masked_fill_(proc_lp_softmax_cuda == 0, float("-inf")) batch_next_tokens_offset_start = batch_next_tokens_offset_end batch_req_idx_offset_start = batch_req_idx_offset_end # Prepare logit indices for raw logprobs; beam-search requests get theirs # from the beam history instead. batch_raw_logprob_indices_cuda = None if reqs_indices_needing_raw_logits: batch_raw_logprob_indices_cuda = logits_cuda_indexer[ torch.tensor(reqs_indices_needing_raw_logits, dtype=torch.int64) ].to(device=cuda_device, non_blocking=True) return _BatchedSamplingResult( req_indices=batch_req_indices, next_tokens_cuda_int=batch_next_tokens_cuda_int, raw_logprobs_reqs_indices=reqs_indices_needing_raw_logprobs, raw_logprobs_logit_indices_cuda=batch_raw_logprob_indices_cuda, processed_logprobs_reqs_indices=reqs_indices_needing_processed_logprobs, processed_logprobs_end=batch_processed_logprobs_offset, logprobs_cuda=batch_processed_logprobs_cuda, ) def _unbatch_sampling_results( self, batched_sampling_result: _BatchedSamplingResult, *, new_tokens_cuda: torch.Tensor, req_num_generated_tokens: torch.Tensor, seq_slots: torch.Tensor, seq_slots_cuda: torch.Tensor, ) -> torch.Tensor: batch_req_indices = batched_sampling_result.req_indices batch_next_tokens_cuda_int = batched_sampling_result.next_tokens_cuda_int def _dims_canonically_ordered(t: torch.Tensor) -> bool: return len(t.dim_order(ambiguity_check=[torch.contiguous_format])) == t.ndim # Assert destination tensor dimensions are canonically ordered ("row"-major); this # matters for element ordering in the .view(...).scatter_(...) calls below. assert _dims_canonically_ordered(new_tokens_cuda) # Construct index mapping from slice indices of computed tensors # (packed request_idx and step dimensions) to linearized indices # in (steps, seq_slot). batch_destination_cuda_indexer = _UnpackedStepIndexer( seq_slots=seq_slots[batch_req_indices], num_steps=req_num_generated_tokens[batch_req_indices], steps_dim_size=new_tokens_cuda.size(0), slots_dim_size=new_tokens_cuda.size(1), dim_order=_UnpackedStepIndexer.DimOrder.STEP_MAJOR, index_dtype=torch.int64, # enforced by Tensor.scatter_ ) # Batch update output tensors batch_dest_indices_1d_cuda = ( batch_destination_cuda_indexer[:] .to(new_tokens_cuda.device, non_blocking=True) .unsqueeze(1) .expand(-1, self.max_beam_width) ) new_tokens_cuda.view(-1, *new_tokens_cuda.shape[2:]).scatter_( 0, batch_dest_indices_1d_cuda, batch_next_tokens_cuda_int ) # Post-sample: decay the runtime top-p for any decay-active slots that were # sampled this iteration (must run after tokens land in new_tokens_cuda). # batch_req_indices is a permutation of all sampled requests, so the set of # sampled slots is exactly seq_slots (the kernel updates each slot # independently; order is irrelevant) -- pass the resident device copy # instead of gathering seq_slots[batch_req_indices] on host and copying it. self._top_p_decay.update_after_sample( step_tokens=new_tokens_cuda[DEFAULT_STEP_IDX, :, DEFAULT_BEAM_IDX], sampled_slots_cuda=seq_slots_cuda, ) return self._copy_to_host(new_tokens_cuda) def _compute_pending_steps(self, requests: list[LlmRequest]) -> list[int] | None: """Per-request count of tokens sampled but not yet written back, or None. With the overlap scheduler ``sample_async`` for step ``i`` runs before ``update_requests`` for step ``i - 1``, so the host token list — and hence ``get_num_tokens()`` — lags the true sequence by this many tokens. Length-based bans (min_length) add it back to recover the real generated length. Unlike the suffix-rule staleness this needs no device-side lookup, so it is not restricted to the single-step / single-beam case. Returns None when the overlap scheduler is off or when nothing is pending, so callers can skip the correction entirely. """ if not self._track_pending_steps: return None pending = [ self._pending_steps[r.py_seq_slot] if r.py_seq_slot is not None else 0 for r in requests ] return pending if any(pending) else None def _compute_stale_by_one(self, requests: list[LlmRequest]) -> list[bool] | None: """Per-request overlap-scheduler stale flags, or None when not applicable. Returns a list where entry ``i`` is True when request ``i``'s host token history lags the device state by exactly one token (the previous step's token was sampled but not yet written back). Only the single-step, single-beam overlap case is reconstructible on the device side; under speculative decoding or beam search the missing history cannot be recovered, so bans are matched against the lagging host history and may be enforced one step late (warned once). Returns None when the overlap scheduler is off or when nothing is pending. """ if not self._track_pending_steps: return None pending = [ self._pending_steps[r.py_seq_slot] if r.py_seq_slot is not None else 0 for r in requests ] if not any(pending): return None if self.max_tokens == 1 and self.max_beam_width == 1 and max(pending) == 1: return [p > 0 for p in pending] logger.warning_once( "bad_words / no_repeat_ngram_size with the overlap scheduler and " "speculative decoding or beam search: bans are matched against a " "host token history that lags the device state and may be enforced " "inexactly.", key="bad_words_stale_overlap", ) return None @staticmethod def _select_generated_logits( scheduled_requests: ScheduledRequests, raw_logits_cuda: torch.Tensor, *, num_context_logits_prefix_sum: list[int], ) -> tuple[list[LlmRequest], SamplingRequestsMetadata, torch.Tensor]: """Select the sampling requests and the corresponding logits from the raw logits. Args: scheduled_requests: The scheduled requests. Sampling requests will be selected from this list. raw_logits_cuda: The raw logits corresponding to the scheduled requests. num_context_logits_prefix_sum: The prefix sum of the number of logits for each context request. Returns: A tuple containing the following: - sampling requests: The requests that are selected for sampling. - sampling requests metadata: The metadata for the sampling requests. - logits: The logits for the sampling requests. """ finished_context_requests = scheduled_requests.context_requests_last_chunk sampling_requests = finished_context_requests + scheduled_requests.generation_requests req_num_generation_steps_list = [ 1 + get_draft_token_length(req) for req in sampling_requests ] req_num_generation_steps = torch.tensor( req_num_generation_steps_list, dtype=torch.int32, pin_memory=prefer_pinned() ) # Rows in the logits tensor, i.e. how many rows each request occupies. # ModelEngine lays out generation requests at the *static* admission # width (py_beam_width), not the per-iteration width: with a variable # beam width array the two differ, and offsetting by the narrower # per-iteration width would make every request after the first read # another request's rows. logits.view() succeeds for any shape whose # element count divides, so that is silent corruption rather than an # error. Match the layout here and slice down to the per-iteration # width where the beams are actually consumed. # NB: context requests do not have multiple beams yet, hence the 1s. req_num_beams_list = [1] * len(finished_context_requests) + [ req.py_beam_width for req in scheduled_requests.generation_requests ] req_num_beams = torch.tensor( req_num_beams_list, dtype=torch.int32, pin_memory=prefer_pinned() ) # context requests do not have multiple beams yet, so beam width may differ after sampling req_num_output_beams_list = [req.get_beam_width_by_iter(True) for req in sampling_requests] req_num_beams_output = torch.tensor( req_num_output_beams_list, dtype=torch.int32, pin_memory=prefer_pinned() ) req_num_generated_tokens = req_num_generation_steps * req_num_beams req_num_generated_tokens_output = req_num_generation_steps * req_num_beams_output # NB: These offsets consider generated tokens _only_ (draft and target, but not context). # Filter out the context tokens below. req_offsets, sum_num_generated_tokens = _PackedStepIndexer.calculate_request_offsets( req_num_generated_tokens, pin_memory=prefer_pinned() ) generation_requests_total_steps = ( # NB: requests == finished_context_requests + scheduled_requests.generation_requests sum_num_generated_tokens - cast(int, req_offsets[len(finished_context_requests)].item()) if scheduled_requests.generation_requests else 0 ) sampling_requests_metadata = SamplingRequestsMetadata( req_num_generated_tokens=req_num_generated_tokens, req_num_generated_tokens_output=req_num_generated_tokens_output, req_num_beams=req_num_beams, req_num_steps=req_num_generation_steps, req_offsets=req_offsets, ) # logits_cuda should contain only the generated logits for the sampling requests. # If return context logits is requested, select only the generated logits of the context requests. # # NB: context_requests_chunking precede finished_context_requests, which precede generation requests. # context_requests_chunking do not sample new tokens, so they should be skipped. # sampling_requests == finished_context_requests + scheduled_requests.generation_requests num_skipped_requests = len(scheduled_requests.context_requests_chunking) if any(r.py_return_context_logits for r in finished_context_requests): assert len(num_context_logits_prefix_sum) == scheduled_requests.num_context_requests + 1 logits_end_offsets = num_context_logits_prefix_sum[num_skipped_requests + 1 :] if scheduled_requests.generation_requests: # Since logits for generation requests are densely packed, add them all as one contiguous block logits_end_offsets.append( num_context_logits_prefix_sum[-1] + generation_requests_total_steps ) num_logits = req_num_generated_tokens[: len(finished_context_requests) + 1].clone() num_logits[-1] = generation_requests_total_steps else: num_logits = req_num_generated_tokens[: len(finished_context_requests)] logits_end_offsets_cuda = torch.tensor( logits_end_offsets, dtype=torch.int32, pin_memory=prefer_pinned() ).to(device=raw_logits_cuda.device, non_blocking=True) num_logits_cuda = num_logits.to(raw_logits_cuda.device, non_blocking=True) num_logits_to_keep = sum_num_generated_tokens # Now, the generated tokens for context request i are at indices # range(logits_end_offsets_cuda[i] - num_logits_cuda[i], # logits_end_offsets_cuda[i]) # And if generation requests are present, those tensors each include a trailing entry selecting # all tokens/logits generated by all generation requests. indices_to_keep_cuda = torch_multi_arange( starts=(logits_end_offsets_cuda - num_logits_cuda), ends=logits_end_offsets_cuda, output_length=num_logits_to_keep, ) logits_cuda = raw_logits_cuda[indices_to_keep_cuda] else: logits_begin_offset = num_context_logits_prefix_sum[num_skipped_requests] logits_cuda = raw_logits_cuda[logits_begin_offset:] return sampling_requests, sampling_requests_metadata, logits_cuda @nvtx_range("_process_requests") def _process_requests( self, scheduled_requests: ScheduledRequests, model_outputs: dict[str, Any], new_tokens_cuda: torch.Tensor, num_context_logits_prefix_sum: list[int], ) -> tuple[ list[LlmRequest], torch.Tensor, Optional[torch.Tensor], torch.Tensor, Optional[torch.Tensor], torch.Tensor, bool, ]: # A forward that sampled greedy tokens in place of the logits (GREEDY_TOKENS_KEY) may return no logits. raw_logits_cuda: Optional[torch.Tensor] = model_outputs.get("logits") greedy_tokens: Optional[torch.Tensor] = model_outputs.get(GREEDY_TOKENS_KEY) if greedy_tokens is not None: rows_cuda = greedy_tokens elif raw_logits_cuda is not None: rows_cuda = raw_logits_cuda else: raise RuntimeError("the forward returned neither logits nor greedy tokens") generation_requests = scheduled_requests.generation_requests request_ids = [request.py_request_id for request in generation_requests] maybe_seq_slots = [request.py_seq_slot for request in generation_requests] has_stable_greedy_batch = ( self._stable_greedy_request_ids == request_ids and self._stable_greedy_seq_slots == maybe_seq_slots ) can_use_stable_greedy_path = ( bool(generation_requests) and self.max_beam_width == 1 and scheduled_requests.num_context_requests == 0 and len(generation_requests) <= rows_cuda.shape[0] and all( not request.is_dummy and get_draft_token_length(request) == 0 for request in generation_requests ) and ( has_stable_greedy_batch or all( request.py_embedding_bias is None and not getattr(request, "py_bad_words", None) and not getattr(request, "py_no_repeat_ngram_size", None) and not has_occurrence_penalty(request) and not request.py_min_length and not request.py_return_log_probs # Single-token stop words are checked on the host by # _update_requests_single_beam_single_step, like py_end_id; # only multi-token ones need the finish-reason kernel. and not check_stop_words_length(request) and self._slot_params.modes[cast(int, request.py_seq_slot)] is SamplingMode.GREEDY for request in generation_requests ) ) ) # Sampled inside the forward graph: the tokens are already in # new_tokens_cuda and only the D2H copy update_requests reads is left. # Re-sampling would waste a launch and, for the fast tier, draw different # tokens (this step's Philox offsets were already consumed). # # Staging happened before the forward, so verify the staged buffers # really describe this batch: a forward that was skipped, repeated or # replaced (context-only steps, eager fallbacks) would # otherwise be sampled against another batch's destinations. if self._current_sample_type is not SampleType.FULL: staged = self._staged_batch if ( staged is not None and self._in_graph_staged # A forward that sampled greedily in place of its LM head skipped the # in-graph sampling, so the staged buffers were never written. and greedy_tokens is None and self._in_graph_dest_indices is not None and ( generation_requests == staged.requests or generation_requests == staged.live_requests ) ): # Exactly the rows staging saw: what it derived for them holds. live_requests = list(staged.live_requests) staged_matches_batch = True seq_slots_host = staged.live_seq_slots_host single_step_greedy = staged.single_step_greedy else: live_requests = [request for request in generation_requests if not request.is_dummy] staged_matches_batch = bool( self._in_graph_staged and greedy_tokens is None and self._in_graph_dest_indices is not None and [request.py_request_id for request in live_requests] == self._in_graph_request_ids ) if staged_matches_batch: seq_slots_host = torch.tensor( [cast(int, request.py_seq_slot) for request in live_requests], dtype=torch.int32, pin_memory=prefer_pinned(), ) single_step_greedy = not any( self._request_needs_finish_reason_kernel(request) for request in live_requests ) if staged_matches_batch: # The staged destinations span the padded batch the graph runs # at; only the leading live rows are reported back, the padded # ones scattered into the scratch row and are dropped here. assert self._in_graph_dest_indices is not None seq_slots_cuda = self._in_graph_dest_indices[: self._in_graph_live_rows] # Stopping still has to be decided after the forward. It reduces # to the per-token EOS and length checks # _update_requests_single_beam_single_step runs on the host only # when no request needs the finish-reason kernel; otherwise the # caller must take the regular path and compute finish reasons, # or a stop word would never be detected. if not single_step_greedy: # write_finish_reasons shifts stop-word history by the # accepted-draft count, which update_requests only refreshes # after this step. Nothing was speculated here -- the tier # requires zero draft tokens -- so a count left over from an # earlier speculative step would shift the comparison and # could miss the stop word. accepted = self._finish_reasons_handler.store.num_accepted_draft_tokens_host for request in live_requests: if request.py_stop_words_list: accepted[request.py_seq_slot] = 0 seq_lens_host: torch.Tensor | None = None seq_lens_cuda: torch.Tensor | None = None if not single_step_greedy: # write_finish_reasons needs the per-request lengths the # regular path would have built for it. seq_lens_host = torch.tensor( [request.max_beam_num_tokens for request in live_requests], dtype=torch.int32, pin_memory=prefer_pinned(), ) seq_lens_cuda = seq_lens_host.to(device="cuda", non_blocking=True) # The two consumers want different layouts: # _update_requests_single_beam_single_step zips a flat # per-request tensor against the requests, while the regular # path indexes the whole (steps, slots, beams) store by slot. new_tokens_host = self._copy_to_host( new_tokens_cuda[DEFAULT_STEP_IDX, seq_slots_cuda, DEFAULT_BEAM_IDX] if single_step_greedy else new_tokens_cuda ) # Consumption is one-shot; the next forward restages. self._consume_in_graph_staging() return ( live_requests, seq_slots_host, seq_lens_host, seq_slots_cuda, seq_lens_cuda, new_tokens_host, single_step_greedy, ) # Staged for a different batch: fall through and sample eagerly. self.invalidate_in_graph_sampling() if can_use_stable_greedy_path: if has_stable_greedy_batch: assert self._stable_greedy_seq_slots_host is not None assert self._stable_greedy_seq_slots_cuda is not None seq_slots_host = self._stable_greedy_seq_slots_host seq_slots_cuda = self._stable_greedy_seq_slots_cuda else: assert all(seq_slot is not None for seq_slot in maybe_seq_slots) seq_slots = [cast(int, seq_slot) for seq_slot in maybe_seq_slots] seq_slots_host = torch.tensor( seq_slots, dtype=torch.int32, pin_memory=prefer_pinned() ) seq_slots_cuda = seq_slots_host.to( device="cuda", dtype=torch.int64, non_blocking=True ) self._stable_greedy_request_ids = request_ids self._stable_greedy_seq_slots = seq_slots self._stable_greedy_seq_slots_host = seq_slots_host self._stable_greedy_seq_slots_cuda = seq_slots_cuda if greedy_tokens is not None: next_tokens = greedy_tokens[: len(generation_requests)].to(new_tokens_cuda.dtype) scatter_new_tokens( next_tokens, new_tokens_cuda, seq_slots_cuda, self.max_beam_width ) else: # Without greedy tokens, rows_cuda is the logits. assert raw_logits_cuda is not None next_tokens = fast_greedy_sample_kernel( raw_logits_cuda[: len(generation_requests)], new_tokens_cuda, seq_slots_cuda, self.max_beam_width, ) new_tokens_host = self._copy_to_host(next_tokens) return ( generation_requests, seq_slots_host, None, seq_slots_cuda, None, new_tokens_host, True, ) self._stable_greedy_request_ids = [] self._stable_greedy_seq_slots = [] if raw_logits_cuda is None: raise RuntimeError( "the forward returned greedy tokens without logits for a batch that needs logits" ) sampling_requests, sampling_requests_metadata, logits_cuda = self._select_generated_logits( scheduled_requests, raw_logits_cuda, num_context_logits_prefix_sum=num_context_logits_prefix_sum, ) return_log_probs = self._log_probs._return_log_probs(sampling_requests) if return_log_probs: self._log_probs._prepare_log_probs(sampling_requests) seq_slots_host = torch.tensor( [r.py_seq_slot for r in sampling_requests], dtype=torch.int32, pin_memory=prefer_pinned(), ) # necessary for beam search and max_length checks seq_lens_host = torch.tensor( [r.max_beam_num_tokens for r in sampling_requests], dtype=torch.int32, pin_memory=prefer_pinned(), ) # Cast seq_slots / seq_lens to CUDA exactly once; consumed by both # the per-group beam-search metadata builder and the finish-reasons # handler in sample_async. int64 is required for the index_*_ ops # downstream. seq_slots_cuda = seq_slots_host.to(device="cuda", dtype=torch.int64, non_blocking=True) seq_lens_cuda = seq_lens_host.to(device="cuda", non_blocking=True) # Handle embedding bias if self._embedding_bias_cache is None: # Always initializing to ensure correct KV cache sizing self._embedding_bias_cache = EmbeddingBiasCache( logit_dtype=logits_cuda.dtype, vocab_size=logits_cuda.size(-1), max_seq_slots=self.max_num_sequences, device=logits_cuda.device, ) self._embedding_bias_cache.apply_bias( logits_cuda, sampling_requests, sampling_requests_metadata.req_num_steps, ) # Apply repetition/presence/frequency penalties in place, before the greedy fast # path, so both greedy and grouped-sampling logits are penalized. With beam search # this also re-parents the per-beam counts, which reads the predecessor map the # step's sampling is about to overwrite -- so it has to stay ahead of sampling. beam_search_store = self.store.beam_search_store self._penalty_handler.apply( logits_cuda, sampling_requests, new_tokens=new_tokens_cuda, seq_slots=seq_slots_cuda, request_offsets=sampling_requests_metadata.req_offsets, request_num_steps=sampling_requests_metadata.req_num_steps, request_num_beams=sampling_requests_metadata.req_num_beams, predecessor_beams=( beam_search_store.predecessor_beams if beam_search_store is not None else None ), ) has_min_length = any(getattr(r, "py_min_length", None) for r in sampling_requests) has_bad_words = any(getattr(r, "py_bad_words", None) for r in sampling_requests) # Normalized in executor_request_to_llm_request: a positive int, or # None when the restriction is disabled for the request. ngram_sizes = [getattr(r, "py_no_repeat_ngram_size", None) for r in sampling_requests] has_no_repeat_ngram = any(size is not None for size in ngram_sizes) if has_min_length or has_bad_words or has_no_repeat_ngram: # Overlap-scheduler stale flags (per request): True when the host # token history lags the device by one token. Only the overlap # handler consumes them; computed here as it needs sampler state. # Only the suffix-matching bans (bad words, no-repeat ngram) care: # min_length bans EOS from a length count, never from token values, # so a lagging history cannot mismatch it. Skipping the call for # min-length-only batches also avoids emitting the stale-history # warning, which names features such a batch does not use. stale_by_one = ( self._compute_stale_by_one(sampling_requests) if (has_bad_words or has_no_repeat_ngram) else None ) # min_length compares against get_num_tokens(), which counts only # the host history; add back the tokens still pending write-back so # the generated length is exact under the overlap scheduler. pending_steps = ( self._compute_pending_steps(sampling_requests) if has_min_length else None ) bans = self._token_ban_handler.generate_ban_list( sampling_requests, sampling_requests_metadata.req_num_steps.tolist(), sampling_requests_metadata.req_num_beams.tolist(), ngram_sizes, stale_by_one=stale_by_one, pending_steps=pending_steps, ) self._token_ban_handler.apply_ban_list( logits_cuda, bans, new_tokens_cuda=new_tokens_cuda ) # Fast path for greedy sampling if self._can_use_fast_greedy_path(sampling_requests): # Compute destination indices on CPU (same pattern as _unbatch_sampling_results) batch_destination_indexer = _UnpackedStepIndexer( seq_slots=seq_slots_host, num_steps=sampling_requests_metadata.req_num_generated_tokens, steps_dim_size=new_tokens_cuda.size(0), slots_dim_size=new_tokens_cuda.size(1), dim_order=_UnpackedStepIndexer.DimOrder.STEP_MAJOR, index_dtype=torch.int64, ) batch_dest_indices_cuda = batch_destination_indexer[:].to( new_tokens_cuda.device, non_blocking=True ) # Run compiled kernel for argmax and scatter fast_greedy_sample_kernel( logits_cuda, new_tokens_cuda, batch_dest_indices_cuda, self.max_beam_width, ) new_tokens_host = self._copy_to_host(new_tokens_cuda) return ( sampling_requests, seq_slots_host, seq_lens_host, seq_slots_cuda, seq_lens_cuda, new_tokens_host, False, ) # Indexer for accessing tokens in 'logits_cuda', corresponding to the # requests in 'requests'. steps_dim_size = new_tokens_cuda.size(0) logits_cuda_indexer = _PackedStepIndexer( num_steps=sampling_requests_metadata.req_num_generated_tokens, max_steps=steps_dim_size * self.max_beam_width, req_offsets=sampling_requests_metadata.req_offsets, ) # Perform sampling in batches batched_sampling_result = self._sample_batched_by_strategy( logits_cuda, sampling_requests, logits_cuda_indexer=logits_cuda_indexer, req_offsets=sampling_requests_metadata.req_offsets, seq_slots=seq_slots_host, seq_lens=seq_lens_host, seq_slots_cuda=seq_slots_cuda, seq_lens_cuda=seq_lens_cuda, req_num_generated_tokens=sampling_requests_metadata.req_num_generated_tokens, req_num_steps=sampling_requests_metadata.req_num_steps, token_dtype=new_tokens_cuda.dtype, return_log_probs=return_log_probs, ) # Fill results into output buffers new_tokens_host = self._unbatch_sampling_results( batched_sampling_result, new_tokens_cuda=new_tokens_cuda, req_num_generated_tokens=sampling_requests_metadata.req_num_generated_tokens, seq_slots=seq_slots_host, seq_slots_cuda=seq_slots_cuda, ) if return_log_probs: self._log_probs._process_logprobs( batched_sampling_result, logits_cuda=logits_cuda, # Drop the trailing scratch slot row (see dummy_slot_row): it # belongs to no request, and the logprobs stores are sized by # max_num_sequences, which also sets the flat gather stride # there. contiguous() because slicing the slot dimension leaves # a strided view and _process_logprobs calls view() on it. new_tokens_cuda=new_tokens_cuda[:, : self.max_num_sequences].contiguous(), seq_slots=seq_slots_host, requests=sampling_requests, req_num_generated_tokens=sampling_requests_metadata.req_num_generated_tokens_output, ) # NB: update_requests syncs w/ device computation and async D2H copies return ( sampling_requests, seq_slots_host, seq_lens_host, seq_slots_cuda, seq_lens_cuda, new_tokens_host, False, )