# Copyright 2024 AllenAI. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import logging import os import threading import time from collections.abc import Callable, Iterable, Iterator from concurrent.futures import ThreadPoolExecutor from dataclasses import asdict, dataclass, field from pathlib import Path from queue import Empty from typing import TYPE_CHECKING, Any, Literal import numpy as np import ray import torch from datasets import Dataset from olmo_core.data import data_loader from ray.util import queue as ray_queue from tqdm import tqdm from transformers import PreTrainedTokenizer if TYPE_CHECKING: import vllm from open_instruct import data_types, padding_free_collator, utils from open_instruct.data_types import EnvConfig, EnvConfigEntry from open_instruct.dataset_transformation import ( ENV_CONFIG_KEY, GROUND_TRUTHS_KEY, INPUT_IDS_PROMPT_KEY, RAW_PROMPT_KEY, TOOLS_COLUMN_KEY, VERIFIER_SOURCE_KEY, ) from open_instruct.environments.tools.utils import EnvStatistics from open_instruct.model_utils import Batch from open_instruct.rl_utils import PackedSequences, pack_sequences, save_rollout_metadata, save_rollouts_to_disk from open_instruct.rubrics import RubricManager from open_instruct.utils import combine_reward_metrics logger = logging.getLogger(__name__) DATA_PREP_ACTOR_NAME = "data_prep_singleton" def to_device(batch: dict[str, Any], device: torch.device | None) -> dict[str, Any]: """Move all tensors in a batch dictionary to the specified device. Args: batch: Dictionary potentially containing torch.Tensor values. device: Target device. If None, tensors are not moved. Returns: Dictionary with the same keys, but tensor values moved to the target device. """ if device is None: return batch return {k: v.to(device, non_blocking=True) if isinstance(v, torch.Tensor) else v for k, v in batch.items()} class HFDataLoader(data_loader.DataLoaderBase): """A DataLoader that wraps a HuggingFace Dataset for use with olmo_core's Trainer. This class implements the DataLoaderBase interface, providing iteration over a HuggingFace Dataset with support for sharding across distributed workers, shuffling, checkpointing, and optional collation. """ def __init__( self, dataset: Dataset, batch_size: int, seed: int, dp_rank: int, dp_world_size: int, work_dir: str, automatic_reshuffle: bool = False, collator: Callable[[list[dict[str, Any]]], dict[str, Any]] | None = None, device: torch.device | None = None, drop_last: bool = True, fs_local_rank: int | None = None, max_seq_length: int = 1, ) -> None: """Initialize the HFDataLoader. Args: dataset: The HuggingFace Dataset to load data from. Must have an 'index' column. batch_size: The global batch size (in sequences). seed: Random seed for shuffling. dp_rank: The rank of the current process in the distributed setup. dp_world_size: Total number of data-parallel processes in the distributed setup. work_dir: Working directory for the data loader (required by DataLoaderBase). automatic_reshuffle: If True, automatically reshuffle at epoch boundaries. collator: Optional collation function for batching examples. If None, batches will be dictionaries of the form `{'examples': [example_1, example_2, ...]}`. device: Device to move tensors to. drop_last: If True, drop the last incomplete batch. If False, pad the last batch with repeated indices to fill a complete batch. fs_local_rank: File system local rank. Defaults to dp_rank when None. max_seq_length: Maximum sequence length. Used to report global_batch_size in tokens to the trainer for batch-size validation. Note: The dataset must have an 'index' column for tracking samples across epochs. This is automatically added by get_cached_dataset_tulu(). For custom datasets, add it with: dataset.add_column('index', range(len(dataset))) """ # OLMo-core's trainer expects global_batch_size in tokens, not sequences. super().__init__( work_dir=work_dir, global_batch_size=batch_size * max_seq_length, dp_world_size=dp_world_size, dp_rank=dp_rank, fs_local_rank=fs_local_rank if fs_local_rank is not None else dp_rank, ) if "index" not in dataset.column_names: raise ValueError( "Dataset must have an 'index' column. This is typically added by get_cached_dataset_tulu(). " "If using a custom dataset, add it with: dataset.add_column('index', range(len(dataset)))" ) self._full_dataset = dataset self.seed = seed self._batch_size = batch_size if batch_size < dp_world_size: raise ValueError( f"Global batch size ({batch_size}) must be >= world size ({dp_world_size}). " f"Each rank needs at least one example per batch." ) if batch_size % dp_world_size != 0: logger.warning( f"Global batch size {batch_size} is not divisible by world size {dp_world_size}. " f"The effective global batch size will be {batch_size // dp_world_size * dp_world_size}." ) self._per_rank_batch_size = batch_size // dp_world_size self._collator = collator if collator is not None else (lambda x: {"examples": x}) self._automatic_reshuffle = automatic_reshuffle self._drop_last = drop_last self._excluded_indices: set[int] = set() self._overflow: list[dict[str, Any]] = [] self._precomputed_batch_sizes: list[int] | None = None self._num_padding_batches: int = 0 self._epoch: int = 0 self._current_iter: Iterator[dict[str, Any]] | None = None self._device = device self._reshard(epoch=0) def __next__(self) -> dict[str, Any]: if self._current_iter is None: self._current_iter = iter(self) try: return next(self._current_iter) except StopIteration: self._current_iter = None if self._automatic_reshuffle: self.reshuffle() if self.effective_size == 0: raise RuntimeError("All dataset examples have been excluded. Cannot continue iteration.") from None self._current_iter = iter(self) return next(self._current_iter) self._epoch += 1 self.batches_processed = 0 raise def _iter_batches(self) -> Iterable[dict[str, Any]]: """Return an iterable over all batches in the epoch.""" # World-aware packing: batch boundaries were precomputed by # _reshard_with_packing so that every rank has the same number of # batches. Each entry in _precomputed_batch_sizes is the number of # examples in that batch (variable due to packing). if self._precomputed_batch_sizes is not None: num_real = len(self._precomputed_batch_sizes) - self._num_padding_batches offset = 0 for batch_idx, batch_size in enumerate(self._precomputed_batch_sizes): if batch_idx < self.batches_processed: offset += batch_size continue examples = [] for i in range(offset, offset + batch_size): example = self.dataset[i] examples.append(example | {"prompt_id": f"{self._epoch}_{example['index']}"}) batch = to_device(self._collator(examples), self._device) | {"is_padding": batch_idx >= num_real} offset += batch_size yield batch return start_example = self.batches_processed * self._per_rank_batch_size batch_examples: list[dict[str, Any]] = [] for i in range(start_example, self.effective_size): example = self.dataset[i] batch_examples.append(example | {"prompt_id": f"{self._epoch}_{example['index']}"}) if len(batch_examples) == self._per_rank_batch_size: all_examples = self._overflow + batch_examples batch = to_device(self._collator(all_examples), self._device) self._overflow = all_examples[len(batch["index"]) :] yield batch batch_examples = [] while self._overflow: batch = to_device(self._collator(self._overflow), self._device) assert len(batch["index"]) > 0, ( f"Collator consumed 0 examples from {len(self._overflow)} overflow examples" ) self._overflow = self._overflow[len(batch["index"]) :] yield batch @property def total_batches(self) -> int: """Return the total number of batches in an epoch.""" if self._precomputed_batch_sizes is not None: return len(self._precomputed_batch_sizes) return self.effective_size // self._per_rank_batch_size def state_dict(self) -> dict[str, Any]: """Return a state dictionary for checkpointing.""" return { "epoch": self._epoch, "batches_processed": self.batches_processed, "excluded_indices": list(self._excluded_indices), } def load_state_dict(self, state_dict: dict[str, Any]) -> None: """Load a state dictionary to restore the data loader's state.""" self._excluded_indices = set(state_dict.get("excluded_indices", [])) # Set epoch to one less than target since reshuffle() increments it self._epoch = state_dict["epoch"] - 1 self.reshuffle() assert self._epoch == state_dict["epoch"] self.batches_processed = state_dict["batches_processed"] self._current_iter = None def exclude_index(self, index: int) -> None: """Exclude a dataset index from future iterations. Args: index: The index to exclude. """ self._excluded_indices.add(index) def reshuffle(self, epoch: int | None = None, **kwargs: Any) -> None: """Reshuffle and reshard the dataset for a new epoch. Args: epoch: The epoch number to use for shuffling seed. If None, increments internal counter. **kwargs: Additional keyword arguments (unused, for API compatibility). """ self._epoch = self._epoch + 1 if epoch is None else epoch self.batches_processed = 0 self._reshard(self._epoch) def _reshard(self, epoch: int) -> None: """Reshard the dataset for a given epoch. Uses index-based shuffling to avoid copying the dataset. """ generator = torch.Generator() generator.manual_seed(self.seed + epoch) dataset_len = len(self._full_dataset) all_indices = torch.randperm(dataset_len, generator=generator).numpy() if self._excluded_indices: mask = np.isin(all_indices, list(self._excluded_indices), invert=True) all_indices = all_indices[mask] packing_enabled = hasattr(self._collator, "max_seq_length") and self._collator.max_seq_length is not None if packing_enabled: self._reshard_with_packing(all_indices) return self._precomputed_batch_sizes = None self._num_padding_batches = 0 self._overflow = [] global_size = len(all_indices) total_batches = global_size // self._batch_size usable_size = total_batches * self._batch_size if not self._drop_last and usable_size < global_size: remainder = global_size - usable_size pad_indices = all_indices[: self._batch_size - remainder] all_indices = np.concatenate([all_indices, pad_indices]) total_batches += 1 usable_size = total_batches * self._batch_size # Distribute examples from global batches to ranks. This is a form of strided sampling where each # rank gets a subset of examples from each global batch, ensuring a diverse set of examples. rank_indices = all_indices[:usable_size].reshape(total_batches, self._batch_size) rank_indices = rank_indices[:, self.dp_rank :: self.dp_world_size].flatten() self.effective_size = len(rank_indices) self.dataset = self._full_dataset.select(rank_indices.tolist()) def _reshard_with_packing(self, all_indices: np.ndarray) -> None: """Reshard with world-aware packing so all ranks get the same batch count. Instead of distributing examples to ranks and letting each rank pack independently (which can produce different batch counts due to variable overflow), this packs globally first and then distributes packed batches round-robin to ranks. """ max_seq_length = self._collator.max_seq_length column_names = self._full_dataset.column_names subset = self._full_dataset.select(all_indices.tolist()) if "chosen_input_ids" in column_names: lengths = [[len(c), len(r)] for c, r in zip(subset["chosen_input_ids"], subset["rejected_input_ids"])] else: lengths = [[len(x)] for x in subset["input_ids"]] num_streams = len(lengths[0]) batches: list[list[int]] = [] current_batch: list[int] = [] running_totals = [0] * num_streams for i in range(len(all_indices)): new_totals = [running_totals[s] + lengths[i][s] for s in range(num_streams)] would_exceed = len(current_batch) > 0 and any(t > max_seq_length for t in new_totals) at_max_samples = len(current_batch) >= self._per_rank_batch_size if would_exceed or at_max_samples: batches.append(current_batch) current_batch = [i] running_totals = list(lengths[i]) else: current_batch.append(i) running_totals = new_totals if current_batch: batches.append(current_batch) num_batches = len(batches) padding_start = num_batches if self._drop_last: num_batches = (num_batches // self.dp_world_size) * self.dp_world_size batches = batches[:num_batches] else: if (remainder := num_batches % self.dp_world_size) > 0: for _ in range(self.dp_world_size - remainder): batches.append(batches[-1]) rank_global_indices = list(range(self.dp_rank, len(batches), self.dp_world_size)) self._num_padding_batches = sum(1 for gi in rank_global_indices if gi >= padding_start) rank_batches = batches[self.dp_rank :: self.dp_world_size] rank_indices: list[int] = [] self._precomputed_batch_sizes = [] for batch in rank_batches: for pos in batch: rank_indices.append(int(all_indices[pos])) self._precomputed_batch_sizes.append(len(batch)) self.effective_size = len(rank_indices) self.dataset = self._full_dataset.select(rank_indices) def get_mock_batch(self) -> dict[str, Any]: """Return a batch with arbitrary data for dry-run testing. Used by the trainer to do a dry-run of the forward and backward pass before training officially starts. """ num_examples = min(self._per_rank_batch_size, len(self.dataset)) examples = [self.dataset[i] for i in range(num_examples)] return to_device(self._collator(examples), self._device) def global_num_tokens_in_batch(self, batch: dict[str, Any]) -> int: """Return the total number of tokens in the batch across all ranks. Counts tokens from all keys containing 'input_ids' that are torch tensors. Args: batch: A batch dictionary containing input tensors. Returns: Total number of tokens across all ranks. Raises: ValueError: If no input_ids tensors are found in the batch. """ num_tokens = padding_free_collator.get_num_tokens(batch) return num_tokens * self.dp_world_size @dataclass class VLLMConfig: vllm_num_engines: int = 1 vllm_tensor_parallel_size: int = 1 vllm_enforce_eager: bool = False vllm_attention_backend: str | None = None vllm_sync_backend: str = "nccl" vllm_gpu_memory_utilization: float = 0.9 vllm_enable_prefix_caching: bool = False vllm_top_p: float = 1.0 @dataclass class StreamingDataLoaderConfig: # Data loading/packing max_prompt_token_length: int = 256 response_length: int = 256 pack_length: int = 512 # Batching async_steps: int = 8 num_samples_per_prompt_rollout: int = 4 num_unique_prompts_rollout: int = 16 # GRPO sampling/filtering active_sampling: bool = False filter_zero_std_samples: bool = True no_resampling_pass_rate: float | None = None advantage_normalization_type: str = "centered" mask_truncated_completions: bool = False mask_tool_use: bool = True # Dataset dataset_mixer_list: list[str] = field(default_factory=lambda: ["ai2-adapt-dev/rlvr_gsm8k_zs", "1.0"]) dataset_mixer_eval_list: list[str] = field(default_factory=list) dataset_mixer_list_splits: list[str] = field(default_factory=lambda: ["train"]) dataset_mixer_eval_list_splits: list[str] = field(default_factory=lambda: ["test"]) dataset_transform_fn: list[str] = field(default_factory=lambda: ["rlvr_tokenize_v1", "rlvr_max_length_filter_v1"]) dataset_cache_mode: Literal["hf", "local"] = "local" dataset_local_cache_dir: str = "local_dataset_cache" dataset_config_hash: str | None = None dataset_config_eval_hash: str | None = None dataset_skip_cache: bool = False system_prompt_override_file: str | None = None # Generation temperature: float = 1.0 stop_strings: list[str] | None = None inflight_updates: bool = True eval_response_length: int | None = None """Local eval max tokens in GRPO `grpo_fast`. Defaults to `response_length` (see `__post_init__`).""" # Reward - R1 style format reward apply_r1_style_format_reward: bool = False r1_style_format_reward: float = 1.0 additive_format_reward: bool = False # Reward - Verifiable reward apply_verifiable_reward: bool = True verification_reward: float = 10.0 remap_verifier: str | None = None # Reward aggregation reward_aggregator: Literal["last", "sum"] = "last" """How to combine per-turn rewards: 'last' (use last turn reward) or 'sum' (sum all rewards across turns).""" # LLM judge verifier llm_judge_model: str = "azure/gpt-4o-mini-standard" llm_judge_max_tokens: int = 2048 llm_judge_max_context_length: int = 8192 llm_judge_temperature: float = 1.0 llm_judge_timeout: int = 60 # Code verifier code_api_url: str = field( default_factory=lambda: os.environ.get("CODE_API_URL", "http://localhost:1234") + "/test_program" ) code_max_execution_time: float = 1.0 code_pass_rate_reward_threshold: float = 0.0 code_apply_perf_penalty: bool = False # Max length verifier max_length_verifier_max_length: int = 32768 # Non stop penalty non_stop_penalty: bool = False non_stop_penalty_value: float = 0.0 # Evolving rubric reward apply_evolving_rubric_reward: bool = False """Whether to generate and apply evolving rubrics for reward computation. When enabled, a rubric buffer is automatically maintained across training steps.""" max_active_rubrics: int = 5 """Maximum number of active evolving rubrics per query.""" cache_evolving_rubric_data_dir: str | None = None """Directory to cache evolving rubric generation data for debugging/analysis. If set, rubric data will be saved.""" # Rollout saving save_traces: bool = False rollouts_save_path: str = "/weka/oe-adapt-default/allennlp/deletable_rollouts/" # Computed at post_init max_possible_score: float = 1.0 def __post_init__(self): if self.eval_response_length is None: self.eval_response_length = self.response_length assert self.pack_length >= self.max_prompt_token_length + self.response_length, ( "The `pack_length` needs to be greater than the sum of `max_prompt_token_length` and `response_length`!" ) assert self.num_samples_per_prompt_rollout > 0, "Number of samples per prompt must be greater than 0!" if self.num_samples_per_prompt_rollout == 1: logger.warning("num_samples_per_prompt_rollout is 1. This reduces GRPO to REINFORCE.") if self.active_sampling: assert self.async_steps > 1, ( "With active_sampling, you should set async_steps > 1 to account for filtering of the first batch. " "Otherwise, your generator only generates only one batch worth of prompts and a single filtered " "prompt will cause the trainer to stall waiting for more data . " ) assert self.filter_zero_std_samples, ( "filter_zero_std_samples must be True when active_sampling is True. " "Active sampling requires filtering to work correctly." ) if self.num_samples_per_prompt_rollout == 1 and self.filter_zero_std_samples: raise ValueError( "`filter_zero_std_samples` cannot be True when `num_samples_per_prompt_rollout` is 1, " "as the reward standard deviation will always be 0, causing all samples to be filtered." ) if self.async_steps < 1: raise ValueError("`async_steps` must be greater than 0. Fully synchronous training is not supported.") assert ( self.apply_verifiable_reward or self.apply_r1_style_format_reward or self.non_stop_penalty or self.apply_evolving_rubric_reward ), "At least one reward must be applied!" if self.stop_strings is None: self.stop_strings = [] self.max_possible_score = 0.0 if self.apply_verifiable_reward: self.max_possible_score += self.verification_reward if self.apply_r1_style_format_reward and self.additive_format_reward: self.max_possible_score += self.r1_style_format_reward if self.save_traces and not self.rollouts_save_path: raise ValueError("`rollouts_save_path` must be provided when `save_traces` is True.") def build_dataloader( self, tokenizer: PreTrainedTokenizer, dp_rank: int, fs_local_rank: int, num_training_steps: int, work_dir: Path | str, dp_world_size: int, ) -> "StreamingDataLoader": """Build a thin wrapper dataloader that pulls from the DataPreparationActor singleton.""" return StreamingDataLoader( tokenizer=tokenizer, work_dir=work_dir, global_batch_size=self.num_unique_prompts_rollout, num_training_steps=num_training_steps, dp_world_size=dp_world_size, dp_rank=dp_rank, fs_local_rank=fs_local_rank, ) class StreamingDataLoader(data_loader.DataLoaderBase): """Thin wrapper dataloader that pulls pre-prepared data from the DataPreparationActor singleton.""" def __init__( self, *, tokenizer: PreTrainedTokenizer, work_dir: Path | str, global_batch_size: int, num_training_steps: int = 0, dp_world_size: int = 1, dp_rank: int = 0, fs_local_rank: int = 0, ): super().__init__( work_dir=work_dir, global_batch_size=global_batch_size, dp_world_size=dp_world_size, dp_rank=dp_rank, fs_local_rank=fs_local_rank, ) self.data_prep_actor = ray.get_actor(DATA_PREP_ACTOR_NAME) self.tokenizer = tokenizer self.num_training_steps = num_training_steps self.training_step = 0 self.current_epoch = 0 @property def total_batches(self) -> int | None: return self.num_training_steps def state_dict(self) -> dict[str, Any]: return {"training_step": self.training_step, "current_epoch": self.current_epoch} def load_state_dict(self, state_dict: dict[str, Any]): self.training_step = state_dict["training_step"] self.current_epoch = state_dict.get("current_epoch", 0) def reshuffle(self, epoch: int | None = None, **kwargs): if epoch is not None: self.current_epoch = epoch def get_mock_batch(self) -> dict[str, Any]: dummy_qr = torch.tensor([[self.tokenizer.pad_token_id, self.tokenizer.eos_token_id]], dtype=torch.long) dummy_attention = torch.tensor([[1, 1]], dtype=torch.long) dummy_position_ids = torch.arange(dummy_qr.shape[-1], dtype=torch.long).unsqueeze(0) dummy_response_mask = torch.tensor([[False, True]], dtype=torch.bool) dummy_advantage = torch.tensor([[0.0, 1.0]], dtype=torch.float) batch = data_types.CollatedBatchData( query_responses=[dummy_qr], attention_masks=[dummy_attention], position_ids=[dummy_position_ids], advantages=[dummy_advantage], response_masks=[dummy_response_mask], vllm_logprobs=[torch.zeros_like(dummy_qr, dtype=torch.float)], ) return {"batch": batch, "metrics": {}} def _iter_batches(self) -> Iterable[dict[str, Any]]: for step in range(self.training_step, self.num_training_steps): wait_start_time = time.perf_counter() batch_data = ray.get(self.data_prep_actor.get_data.remote(rank=self.dp_rank, step=step)) trainer_idle_wait_time = time.perf_counter() - wait_start_time batch_data.setdefault("metrics", {})["time/trainer_waiting_for_data"] = trainer_idle_wait_time self.training_step = step + 1 yield batch_data def collate_fn(tensors_list: list[torch.Tensor], pad_token_id: int, pin_memory: bool = True) -> torch.Tensor: padded_tensor = torch.nn.utils.rnn.pad_sequence(tensors_list, batch_first=True, padding_value=pad_token_id) padded_tensor = torch.atleast_2d(padded_tensor) if pin_memory and torch.cuda.is_available(): padded_tensor = padded_tensor.pin_memory() return padded_tensor @dataclass class BatchStatistics: prompt_lengths: list[int] response_lengths: list[int] filtered_prompts: int filtered_prompts_zero: int filtered_prompts_solved: int filtered_prompts_nonzero: int filtered_prompts_pct: float percent_solved_mean: float percent_solved_hist: np.ndarray no_resampled_prompts: int total_prompts: int per_group_generation_times: np.ndarray = field(default_factory=lambda: np.array([], dtype=float)) def single_example_collator(examples: list[dict[str, Any]]) -> dict[str, Any]: assert len(examples) == 1, f"Expected 1 example, got {len(examples)}" example = examples[0] return example | {"index": torch.tensor([example["index"]])} def _merge_env_config(base_env_config: EnvConfig, sample_env_config: dict[str, Any] | None) -> EnvConfig: """Merge base and sample env config into canonical payload. Sample env_config overrides any base env_configs with the same name. """ if sample_env_config is None: return base_env_config max_steps = sample_env_config.get("max_steps", base_env_config.max_steps) merged = dict(base_env_config.env_configs) for sample_entry in sample_env_config.get("env_configs", []): env_name = sample_entry["env_name"] base = merged.get(env_name) is_text_env = sample_entry.get("is_text_env", base.is_text_env if base else False) extra = {k: v for k, v in sample_entry.items() if k not in ("env_name", "is_text_env")} merged_kwargs = {**(base.kwargs if base else {}), **extra} merged[env_name] = EnvConfigEntry(env_name=env_name, is_text_env=is_text_env, kwargs=merged_kwargs) return EnvConfig(max_steps=max_steps, env_configs=merged) def _aggregate_env_metrics(rollout_states: list[dict]) -> dict[str, float]: env_metrics: dict[str, dict[str, list[float]]] = {} for rs in rollout_states: info = rs.get("info", {}) multi_env_metrics = info.get("env_metrics") if multi_env_metrics is not None: for ename, per_env in multi_env_metrics.items(): bucket = env_metrics.setdefault(ename, {}) for k, v in per_env.items(): bucket.setdefault(k, []).append(float(v)) continue ename = info.get("env_name", "unknown") bucket = env_metrics.setdefault(ename, {}) for k, v in info.items(): if k != "env_name" and isinstance(v, (int, float)): bucket.setdefault(k, []).append(float(v)) return { f"env/{ename}/{k}": float(np.mean(vals)) for ename, metrics in env_metrics.items() for k, vals in metrics.items() } def add_prompt_to_generator( example: dict[str, Any], epoch_number: int, param_prompt_Q: ray_queue.Queue, generation_config, is_eval: bool, base_env_config: EnvConfig, ground_truth_overrides: dict[int, Any] | None = None, ) -> None: index = int(example["index"]) sample_env_config = example.get(ENV_CONFIG_KEY) env_config = _merge_env_config(base_env_config, sample_env_config) ground_truth = ground_truth_overrides.get(index) if ground_truth_overrides else None param_prompt_Q.put( data_types.PromptRequest( prompt=example[INPUT_IDS_PROMPT_KEY], generation_config=generation_config, index=index, prompt_id=f"{epoch_number}_{index}", is_eval=is_eval, active_tools=example.get(TOOLS_COLUMN_KEY), env_config=env_config, ground_truth=ground_truth, ) ) @dataclass class Group: """All samples for one GRPO group: a single prompt with n generations.""" result: data_types.GenerationResult query: list[int] ground_truth: list[int] dataset: str raw_query: str active_tools: list[str] | None index: int decoded_responses: list[str] reward_scores: list[float] reward_metrics: dict[str, Any] percent_solved: float def process_group( result: data_types.GenerationResult, generation_config: "vllm.SamplingParams", tokenizer: PreTrainedTokenizer, dataset: Dataset, max_possible_score: float, no_resampling_pass_rate: float | None, iter_dataloader: HFDataLoader | None, filter_zero_std_samples: bool, replenish_prompts: bool, param_prompt_Q: ray_queue.Queue | None, base_env_config: EnvConfig, ground_truth_overrides: dict[int, Any] | None = None, ) -> Group | None: assert result.index is not None assert result.reward_scores is not None assert result.token_statistics is not None assert len(result.responses) == generation_config.n, ( f"Mismatch: individual prompt result has {len(result.responses)} responses " f"but expected {generation_config.n} samples per prompt. " f"Index: {result.index}, Prompt ID: {result.prompt_id}" ) example = dataset[result.index] query = example[INPUT_IDS_PROMPT_KEY] ground_truth = example[GROUND_TRUTHS_KEY] dataset_name = example[VERIFIER_SOURCE_KEY] raw_query = example[RAW_PROMPT_KEY] sample_active_tools = example.get(TOOLS_COLUMN_KEY) if replenish_prompts: assert iter_dataloader is not None and param_prompt_Q is not None example = next(iter_dataloader) add_prompt_to_generator( example, iter_dataloader._epoch, param_prompt_Q, generation_config, is_eval=False, base_env_config=base_env_config, ground_truth_overrides=ground_truth_overrides, ) for i in range(len(result.finish_reasons)): if result.finish_reasons[i] == "stop" and len(result.responses[i]) == 0: result.responses[i].append(tokenizer.eos_token_id) result.masks[i].append(1) result.logprobs[i].append(float("nan")) decoded_responses = tokenizer.batch_decode(result.responses, skip_special_tokens=False) percent_solved = np.mean(result.reward_scores).item() / max_possible_score if no_resampling_pass_rate is not None and percent_solved >= no_resampling_pass_rate: assert iter_dataloader is not None iter_dataloader.exclude_index(result.index) if filter_zero_std_samples and np.std(result.reward_scores) == 0: return None return Group( result=result, query=query, ground_truth=ground_truth, dataset=dataset_name, raw_query=raw_query, active_tools=sample_active_tools, index=result.index, decoded_responses=decoded_responses, reward_scores=result.reward_scores, reward_metrics=result.reward_metrics or {}, percent_solved=percent_solved, ) def make_batch_from_groups( groups: list[Group], generation_config: "vllm.SamplingParams", training_step: int, actor_manager=None, filtered_prompts: int = 0, filtered_prompts_zero: int = 0, filtered_prompts_solved: int = 0, filtered_prompts_nonzero: int = 0, no_resampled_prompts: int = 0, stale_results_dropped: int = 0, ) -> tuple[data_types.GenerationResult, Batch, dict, BatchStatistics]: assert len(groups) > 0, "make_batch_from_groups requires at least one group" all_queries = [] all_ground_truths = [] all_datasets = [] all_raw_queries = [] all_decoded_responses = [] all_reward_metrics = [] all_active_tools = [] all_scores = [] all_indices = [] all_percent_solved = [] all_model_steps = [] combined_responses = [] combined_finish_reasons = [] combined_masks = [] combined_num_calls = [] combined_timeouts = [] combined_tool_errors = [] combined_tool_outputs = [] combined_tool_runtimes = [] combined_tool_calleds = [] combined_tool_call_stats = [] combined_rollout_states = [] combined_logprobs = [] earliest_start_time = float("inf") prompt_lengths = [] response_lengths = [] total_prompt_tokens = 0 total_response_tokens = 0 max_generation_time = 0 per_group_generation_times: list[float] = [] for group in groups: result = group.result all_queries.extend([group.query] * generation_config.n) all_ground_truths.extend([group.ground_truth] * generation_config.n) all_datasets.extend([group.dataset] * generation_config.n) all_raw_queries.extend([group.raw_query] * generation_config.n) all_active_tools.extend([group.active_tools] * generation_config.n) all_indices.extend([group.index] * generation_config.n) all_decoded_responses.extend(group.decoded_responses) all_scores.extend(group.reward_scores) all_reward_metrics.append(group.reward_metrics) all_percent_solved.append(group.percent_solved) all_model_steps.extend([result.model_step] * len(result.responses)) combined_responses.extend(result.responses) combined_finish_reasons.extend(result.finish_reasons) combined_masks.extend(result.masks) combined_num_calls.extend(result.request_info.num_calls) combined_timeouts.extend(result.request_info.timeouts) combined_tool_errors.extend(result.request_info.tool_errors) combined_tool_outputs.extend(result.request_info.tool_outputs) combined_tool_runtimes.extend(result.request_info.tool_runtimes) combined_tool_calleds.extend(result.request_info.tool_calleds) combined_tool_call_stats.extend(result.request_info.tool_call_stats) combined_rollout_states.extend(result.request_info.rollout_states) combined_logprobs.extend(result.logprobs) earliest_start_time = min(earliest_start_time, result.start_time) prompt_lengths.append(len(group.query)) for response in result.responses: response_lengths.append(len(response)) total_prompt_tokens += result.token_statistics.num_prompt_tokens total_response_tokens += result.token_statistics.num_response_tokens max_generation_time = max(max_generation_time, result.token_statistics.generation_time) per_group_generation_times.append(result.token_statistics.generation_time) accumulated_stats = data_types.TokenStatistics( num_prompt_tokens=total_prompt_tokens, num_response_tokens=total_response_tokens, generation_time=max_generation_time, earliest_start_time=earliest_start_time, ) combined_request_info = data_types.RequestInfo( num_calls=combined_num_calls, timeouts=combined_timeouts, tool_errors=combined_tool_errors, tool_outputs=combined_tool_outputs, tool_runtimes=combined_tool_runtimes, tool_calleds=combined_tool_calleds, tool_call_stats=combined_tool_call_stats, rollout_states=combined_rollout_states, ) combined_result = data_types.GenerationResult( responses=combined_responses, finish_reasons=combined_finish_reasons, masks=combined_masks, request_info=combined_request_info, index=None, prompt_id=groups[0].result.prompt_id, token_statistics=accumulated_stats, logprobs=combined_logprobs, ) if actor_manager is not None: ray.get(actor_manager.report_token_statistics.remote(accumulated_stats)) batch = Batch( queries=all_queries, ground_truths=all_ground_truths, datasets=all_datasets, raw_queries=all_raw_queries, decoded_responses=all_decoded_responses, indices=all_indices, scores=all_scores, active_tools=all_active_tools if any(all_active_tools) else None, model_steps=all_model_steps, ) combined_reward_metrics = combine_reward_metrics(all_reward_metrics) model_steps_array = np.array(all_model_steps, dtype=float) combined_reward_metrics["model_step_min"] = float(model_steps_array.min()) combined_reward_metrics["model_step_max"] = float(model_steps_array.max()) combined_reward_metrics["model_step_mean"] = float(model_steps_array.mean()) combined_reward_metrics["num_steps_off_policy"] = float(training_step - model_steps_array.mean()) combined_reward_metrics["stale_results_dropped"] = float(stale_results_dropped) percent_solved_mean = np.mean(all_percent_solved) if all_percent_solved else 0.0 total_prompts = len(groups) total_seen = filtered_prompts + total_prompts filtered_prompts_pct = filtered_prompts / total_seen if total_seen > 0 else 0.0 batch_stats = BatchStatistics( prompt_lengths=prompt_lengths, response_lengths=response_lengths, filtered_prompts=filtered_prompts, filtered_prompts_zero=filtered_prompts_zero, filtered_prompts_solved=filtered_prompts_solved, filtered_prompts_nonzero=filtered_prompts_nonzero, filtered_prompts_pct=filtered_prompts_pct, percent_solved_mean=percent_solved_mean, percent_solved_hist=np.array(all_percent_solved), no_resampled_prompts=no_resampled_prompts, total_prompts=total_prompts, per_group_generation_times=np.array(per_group_generation_times, dtype=float), ) return combined_result, batch, combined_reward_metrics, batch_stats def result_is_stale(model_step: int | None, training_step: int | None, max_result_age_steps: int | None) -> bool: """Whether an async rollout result is too far behind the current training step. Returns False (never stale) when staleness checking is disabled or the inputs are unavailable. A result is stale when ``training_step - model_step`` exceeds ``max_result_age_steps``. """ if max_result_age_steps is None or training_step is None or model_step is None: return False return training_step - model_step > max_result_age_steps def accumulate_inference_batches( inference_results_Q: ray_queue.Queue, generation_config: "vllm.SamplingParams", num_prompts: int, model_dims: utils.ModelDims, tokenizer: PreTrainedTokenizer, dataset: Dataset, base_env_config: EnvConfig, training_step: int, actor_manager=None, timeout: float | None = None, active_sampling: bool = False, filter_zero_std_samples: bool = False, replenish_prompts: bool = False, no_resampling_pass_rate: float | None = None, iter_dataloader: HFDataLoader | None = None, param_prompt_Q: ray_queue.Queue | None = None, verbose: bool = False, max_possible_score: float = 1.0, requeue_on_timeout: bool = True, ground_truth_overrides: dict[int, Any] | None = None, max_result_age_steps: int | None = None, ) -> ( tuple[data_types.GenerationResult, Batch, dict, BatchStatistics] | tuple[data_types.ShutdownSentinel | None, None, None, None] ): if no_resampling_pass_rate is not None: assert iter_dataloader is not None, "no_resampling requires the iter_dataloader passed" if replenish_prompts: assert param_prompt_Q is not None and iter_dataloader is not None and dataset is not None, ( "replenish_prompts requires param_prompt_Q and iter_dataloader and dataset" ) if max_result_age_steps is not None and not replenish_prompts: # Dropping stale results without replenishing would steadily drain the in-flight # prompt pool and eventually hang the accumulator waiting for results that never come. raise ValueError("max_result_age_steps requires replenish_prompts=True to avoid draining the prompt pipeline.") groups: list[Group] = [] total_filtered_prompts = 0 filtered_prompt_zero = 0 filtered_prompt_solved = 0 filtered_prompt_nonzero = 0 total_no_resampled = 0 stale_results_dropped = 0 progress_bar = tqdm( total=num_prompts, desc=f"Accumulating Responses and Rewarding {num_prompts} prompts", bar_format="{l_bar}{bar}{r_bar}\n", disable=not verbose, ) logger.info( f"[accumulate_inference_batches] Starting to accumulate {num_prompts} prompts, training_step={training_step}" ) num_prompts_sampled = 0 collected_results = [] while num_prompts_sampled < num_prompts: logger.info( f"[accumulate_inference_batches] Waiting for result {num_prompts_sampled + 1}/{num_prompts} from inference_results_Q" ) try: result = inference_results_Q.get(timeout=timeout) except Empty: if requeue_on_timeout and collected_results: logger.info( f"[accumulate_inference_batches] Timeout with {len(collected_results)}/{num_prompts} results, requeuing" ) for r in collected_results: inference_results_Q.put(r) raise logger.info( f"[accumulate_inference_batches] Got result {num_prompts_sampled + 1}/{num_prompts}, type: {type(result).__name__}" ) if isinstance(result, data_types.ShutdownSentinel): return result, None, None, None # Drop rollouts generated by a policy that is now too far behind the trainer # (async off-policy lag). Replenish a fresh prompt so the generator keeps up, and # don't keep the stale result around for requeue-on-timeout. if result_is_stale(result.model_step, training_step, max_result_age_steps): stale_results_dropped += 1 logger.warning( "[accumulate_inference_batches] Dropping stale result for index=%s at training_step=%s: " "model_step=%s lag=%s max_result_age_steps=%s", result.index, training_step, result.model_step, training_step - result.model_step, max_result_age_steps, ) # replenish_prompts is guaranteed True here (validated above), so always # replenish a fresh prompt to keep the in-flight prompt pool from draining. assert iter_dataloader is not None and param_prompt_Q is not None add_prompt_to_generator( next(iter_dataloader), iter_dataloader._epoch, param_prompt_Q, generation_config, is_eval=False, base_env_config=base_env_config, ground_truth_overrides=ground_truth_overrides, ) continue collected_results.append(result) group = process_group( result=result, generation_config=generation_config, tokenizer=tokenizer, dataset=dataset, max_possible_score=max_possible_score, no_resampling_pass_rate=no_resampling_pass_rate, iter_dataloader=iter_dataloader, filter_zero_std_samples=filter_zero_std_samples, replenish_prompts=replenish_prompts, param_prompt_Q=param_prompt_Q, base_env_config=base_env_config, ground_truth_overrides=ground_truth_overrides, ) if group is None: if not active_sampling: num_prompts_sampled += 1 progress_bar.update(1) total_filtered_prompts += 1 if result.reward_scores[0] == 0: filtered_prompt_zero += 1 elif result.reward_scores[0] == max_possible_score: filtered_prompt_solved += 1 else: filtered_prompt_nonzero += 1 logger.info( f"[accumulate_inference_batches] training_step={training_step} " f"sampled={num_prompts_sampled}/{num_prompts} " f"filtered={total_filtered_prompts} " f"(all_zero={filtered_prompt_zero}, all_solved={filtered_prompt_solved}, nonzero_std0={filtered_prompt_nonzero})" ) continue if no_resampling_pass_rate is not None and group.percent_solved >= no_resampling_pass_rate: total_no_resampled += 1 logging.debug( f"[Data Preparation Thread] Prompt solved at {group.percent_solved}, " f"will be excluded from resampling, total no resampled: {total_no_resampled}" ) num_prompts_sampled += 1 progress_bar.update(1) groups.append(group) if stale_results_dropped > 0: logger.info( f"[accumulate_inference_batches] training_step={training_step}: dropped {stale_results_dropped} " f"stale result(s) this batch (max_result_age_steps={max_result_age_steps})" ) if len(groups) == 0: logging.warning( "[Data Preparation Thread] All prompts were filtered during accumulation. " f"Filtered: {total_filtered_prompts} (zero std: {filtered_prompt_zero}, " f"solved: {filtered_prompt_solved}, nonzero: {filtered_prompt_nonzero})" ) return None, None, None, None return make_batch_from_groups( groups, generation_config, training_step, actor_manager, filtered_prompts=total_filtered_prompts, filtered_prompts_zero=filtered_prompt_zero, filtered_prompts_solved=filtered_prompt_solved, filtered_prompts_nonzero=filtered_prompt_nonzero, no_resampled_prompts=total_no_resampled, stale_results_dropped=stale_results_dropped, ) def compute_group_advantages(scores: np.ndarray, num_samples_per_prompt: int, normalization_type: str) -> np.ndarray: """Normalize each score against the other samples for the same prompt. `scores` must hold complete groups: `num_samples_per_prompt` consecutive samples per prompt. """ scores_per_prompt = scores.reshape(-1, num_samples_per_prompt) mean_grouped_rewards = np.repeat(scores_per_prompt.mean(axis=-1), num_samples_per_prompt, axis=0) std_grouped_rewards = np.repeat(scores_per_prompt.std(axis=-1), num_samples_per_prompt, axis=0) if normalization_type == "standard": return (scores - mean_grouped_rewards) / (std_grouped_rewards + 1e-8) if normalization_type == "centered": return scores - mean_grouped_rewards raise ValueError(f"Invalid advantage normalization type: {normalization_type}") def maybe_mask_truncated_completions( result: data_types.GenerationResult, batch: Batch, advantages: np.ndarray, enabled: bool ) -> tuple[Batch, np.ndarray]: """If enabled, drop rollouts that didn't finish with 'stop' from result (in place), batch, and advantages. Call this after `compute_group_advantages`: dropping single rollouts leaves groups of uneven size. """ if not enabled: return batch, advantages stop_idxes = [i for i, fr in enumerate(result.finish_reasons) if fr == "stop"] num_truncated = len(result.finish_reasons) - len(stop_idxes) if num_truncated > 0: logger.info( f"[DataPreparationActor] Filtered {num_truncated} responses that didn't finish with 'stop'. " f"Retention rate: {len(stop_idxes) / len(result.finish_reasons):.2%}" ) result.responses = [result.responses[i] for i in stop_idxes] result.masks = [result.masks[i] for i in stop_idxes] result.finish_reasons = [result.finish_reasons[i] for i in stop_idxes] result.logprobs = [result.logprobs[i] for i in stop_idxes] return batch[stop_idxes], advantages[stop_idxes] def prepare_collated_data_for_workers( packed_sequences: PackedSequences, dp_world_size: int, per_device_train_batch_size: int, pad_token_id: int, pin_memory: bool = True, ) -> list[data_types.CollatedBatchData]: """Distributes and collates packed sequences for distributed training. Splits packed sequences across workers, randomly shuffles each worker's data, and collates into micro-batches for training. Args: packed_sequences: Packed training sequences containing query responses, attention masks, position IDs, advantages, response masks, and vllm logprobs. dp_world_size: Number of distributed workers. per_device_train_batch_size: Batch size for each device's micro-batch. pad_token_id: Token ID used for padding sequences. pin_memory: Whether to pin memory for faster data transfer to GPU. Returns: List of CollatedBatchData, one per worker, each containing collated tensors for query_responses, attention_masks, position_ids, advantages, response_masks, and vllm_logprobs. """ total_sequences = len(packed_sequences.query_responses) if total_sequences % dp_world_size != 0: new_total = (total_sequences // dp_world_size) * dp_world_size logger.warning( f"Total packed sequences ({total_sequences}) is not evenly divisible by dp_world_size ({dp_world_size}). " f"Truncating to {new_total} sequences (dropping {total_sequences - new_total})." ) B = total_sequences // dp_world_size collated_data = [] assert packed_sequences.position_ids is not None assert packed_sequences.advantages is not None assert packed_sequences.vllm_logprobs is not None for i in range(dp_world_size): per_device_packed_query_responses = packed_sequences.query_responses[B * i : B * (i + 1)] per_device_packed_attention_masks = packed_sequences.attention_masks[B * i : B * (i + 1)] per_device_packed_position_ids = packed_sequences.position_ids[B * i : B * (i + 1)] per_device_packed_advantages = packed_sequences.advantages[B * i : B * (i + 1)] per_device_packed_response_masks = packed_sequences.response_masks[B * i : B * (i + 1)] per_device_packed_vllm_logprobs = packed_sequences.vllm_logprobs[B * i : B * (i + 1)] # Shuffle the batch and collate the data b_inds = np.random.permutation(len(per_device_packed_query_responses)) collated_query_responses = [] collated_attention_masks = [] collated_position_ids = [] collated_response_masks = [] collated_advantages = [] collated_vllm_logprobs = [] for j in range(0, len(per_device_packed_query_responses), per_device_train_batch_size): micro_range = b_inds[j : j + per_device_train_batch_size] collated_query_responses.append( collate_fn([per_device_packed_query_responses[idx] for idx in micro_range], pad_token_id, pin_memory) ) collated_attention_masks.append( collate_fn([per_device_packed_attention_masks[idx] for idx in micro_range], 0, pin_memory) ) collated_position_ids.append( collate_fn([per_device_packed_position_ids[idx] for idx in micro_range], 0, pin_memory) ) collated_response_masks.append( collate_fn([per_device_packed_response_masks[idx] for idx in micro_range], 0, pin_memory) ) collated_advantages.append( collate_fn([per_device_packed_advantages[idx] for idx in micro_range], 0, pin_memory) ) collated_vllm_logprobs.append( collate_fn([per_device_packed_vllm_logprobs[idx] for idx in micro_range], 0, pin_memory) ) collated_data.append( data_types.CollatedBatchData( query_responses=collated_query_responses, attention_masks=collated_attention_masks, position_ids=collated_position_ids, advantages=collated_advantages, response_masks=collated_response_masks, vllm_logprobs=collated_vllm_logprobs, ) ) return collated_data @ray.remote class DataPreparationActor: """Ray actor singleton that handles centralized data preparation for all ranks. This actor runs a background thread that continuously prepares training data, ensuring all ranks receive the same number of micro-batches (preventing deadlock from uneven filtering). """ def __init__( self, dataset: Dataset, inference_results_Q: ray_queue.Queue, param_prompt_Q: ray_queue.Queue, tokenizer: PreTrainedTokenizer, config: StreamingDataLoaderConfig, generation_config, num_training_steps: int, seed: int, per_device_train_batch_size: int, global_batch_size: int, dp_world_size: int, max_possible_score: float, actor_manager, model_dims: utils.ModelDims, verbose: bool, work_dir: str, tool_names: list[str], run_name: str, model_name: str | None, base_env_config: EnvConfig, initial_state: dict | None = None, ): self.inference_results_Q = inference_results_Q self.param_prompt_Q = param_prompt_Q self.tokenizer = tokenizer self.config = config self.config.max_possible_score = max_possible_score self.generation_config = generation_config self.num_training_steps = num_training_steps self.per_device_train_batch_size = per_device_train_batch_size self.global_batch_size = global_batch_size self.dp_world_size = dp_world_size self.actor_manager = actor_manager self.model_dims = model_dims self.verbose = verbose self.dataset = dataset self.tool_names = tool_names self.run_name = run_name self.model_name = model_name self.base_env_config = base_env_config self.iter_dataloader = HFDataLoader( dataset=dataset, batch_size=1, seed=seed, dp_rank=0, dp_world_size=1, work_dir=work_dir, automatic_reshuffle=True, collator=single_example_collator, ) self.prepared_data: dict[int, list[data_types.CollatedBatchData]] = {} self.metrics: dict[int, dict] = {} self.current_prepared_step = -1 self._last_consumed_step = -1 self.lock = threading.Lock() self.training_step = 0 self.total_samples_written = 0 self.metadata_saved = False self._executor: ThreadPoolExecutor | None = None self._prep_future = None self.rubric_manager: RubricManager | None = None self.ground_truth_overrides: dict[int, Any] = {} if self.config.apply_evolving_rubric_reward: self.rubric_manager = RubricManager(self.config, dataset[GROUND_TRUTHS_KEY]) if initial_state is not None: logger.info("[DataPreparationActor] Given initial state, setting state and starting preparation loop") self.set_state(initial_state) self.start() def start(self): if self._prep_future is not None: return self._executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="DataPrepActor") self._prep_future = self._executor.submit(self._data_preparation_loop) logger.info(f"[DataPreparationActor] Started preparation loop from training_step={self.training_step}") def _data_preparation_loop(self): logger.info("[DataPreparationActor] Starting _data_preparation_loop") if self.config.save_traces and self.config.rollouts_save_path and not self.metadata_saved: save_rollout_metadata(self.config.rollouts_save_path, self.run_name, self.model_name) self.metadata_saved = True num_initial_prompts = self.config.async_steps * self.global_batch_size logger.info(f"[DataPreparationActor] Pushing {num_initial_prompts} initial prompts to param_prompt_Q") for _ in range(num_initial_prompts): add_prompt_to_generator( next(self.iter_dataloader), self.iter_dataloader._epoch, self.param_prompt_Q, self.generation_config, is_eval=False, base_env_config=self.base_env_config, ground_truth_overrides=self.ground_truth_overrides, ) while self.training_step < self.num_training_steps: generation_wait_start_time = time.perf_counter() while self.training_step - self._last_consumed_step > self.config.async_steps: logger.info( f"[DataPreparationActor] Step {self.training_step}: waiting for step {self._last_consumed_step + self.config.async_steps} to be consumed. Consider increasing training compute." ) time.sleep(0.1) generation_wait_time = time.perf_counter() - generation_wait_start_time logger.info( f"[DataPreparationActor] Step {self.training_step}: calling accumulate_inference_batches for {self.global_batch_size} prompts" ) result, batch, reward_metrics, batch_stats = accumulate_inference_batches( self.inference_results_Q, self.generation_config, num_prompts=self.global_batch_size, model_dims=self.model_dims, tokenizer=self.tokenizer, dataset=self.dataset, actor_manager=self.actor_manager, active_sampling=self.config.active_sampling, filter_zero_std_samples=self.config.filter_zero_std_samples, replenish_prompts=True, no_resampling_pass_rate=self.config.no_resampling_pass_rate, iter_dataloader=self.iter_dataloader, param_prompt_Q=self.param_prompt_Q, training_step=self.training_step, verbose=self.verbose, max_possible_score=self.config.max_possible_score, base_env_config=self.base_env_config, ground_truth_overrides=self.ground_truth_overrides, max_result_age_steps=self.config.async_steps, ) logger.info( f"[DataPreparationActor] Step {self.training_step}: accumulate_inference_batches returned, result type: {type(result).__name__}" ) if isinstance(result, data_types.ShutdownSentinel): return if result is None: logger.warning( f"[DataPreparationActor] 🤡 Step {self.training_step}: all groups filtered (zero-std rewards); " "resampling without advancing step counter" ) continue assert batch is not None assert batch_stats is not None # Advantages are normalized per prompt group, so compute them (and save traces, which index # samples by group) before the truncation filter drops individual samples. advantages = compute_group_advantages( np.array(batch.scores), self.config.num_samples_per_prompt_rollout, self.config.advantage_normalization_type, ) if self.config.save_traces and self.config.rollouts_save_path: save_rollouts_to_disk( self.config.rollouts_save_path, self.run_name, self.training_step, batch, result, advantages, self.config.num_samples_per_prompt_rollout, self.total_samples_written, ) self.total_samples_written += len(batch.queries) batch, advantages = maybe_mask_truncated_completions( result, batch, advantages, self.config.mask_truncated_completions ) if len(result.responses) == 0: logger.warning( f"[DataPreparationActor] 🤡 Step {self.training_step}: no trainable responses after truncation filter; " "resampling without advancing step counter" ) continue if self.rubric_manager and batch.decoded_responses: rubric_metrics, new_overrides = self.rubric_manager.run_step( decoded_responses=batch.decoded_responses, ground_truths=batch.ground_truths, indices=batch.indices, step=self.training_step, ) reward_metrics.update(rubric_metrics) self.ground_truth_overrides.update(new_overrides) scores = np.array(batch.scores) packed_sequences = pack_sequences( queries=batch.queries, responses=result.responses, masks=result.masks, pack_length=self.config.pack_length, pad_token_id=self.tokenizer.pad_token_id, vllm_logprobs=result.logprobs, mask_tool_use=self.config.mask_tool_use, min_num_batches=self.dp_world_size, ) lookup_advantages = np.zeros(len(advantages) + 1, dtype=np.float32) lookup_advantages[1:] = advantages packed_advantages = [ torch.tensor(lookup_advantages[packed_mask], dtype=torch.float32) for packed_mask in packed_sequences.response_masks ] packed_sequences.advantages = packed_advantages # `pack_sequences` returns int64 doc-id-valued masks (0 for query/pad, i+1 for tokens of # sample i) because the gather above uses them as integer indices into `lookup_advantages`. # Now that the gather is done, downcast to bool so all downstream consumers see a single # contract. packed_sequences.response_masks = [mask.bool() for mask in packed_sequences.response_masks] collated_data = prepare_collated_data_for_workers( packed_sequences, self.dp_world_size, self.per_device_train_batch_size, self.tokenizer.pad_token_id ) real_num_responses = len(result.responses) expected_num_responses = self.config.num_samples_per_prompt_rollout * self.global_batch_size unsolved_num_responses = (scores < self.config.max_possible_score).sum() sequence_lengths = np.array([len(response) for response in result.responses]) sequence_length_solved = ( np.array([]) if np.all(scores == 0) else np.array(sequence_lengths[scores == self.config.max_possible_score]) ) sequence_length_unsolved = ( np.array([]) if np.all(scores == self.config.max_possible_score) else np.array(sequence_lengths[scores == 0]) ) stop_rate = sum(int(fr == "stop") for fr in result.finish_reasons) / len(result.finish_reasons) batch_metrics_dict = asdict(batch_stats) batch_metrics_prefixed = {f"batch/{k}": v for k, v in batch_metrics_dict.items()} step_metrics = { "time/generation_waiting_for_trainer": generation_wait_time, "scores": scores.mean(), "real_batch_size_ratio": real_num_responses / expected_num_responses, "unsolved_batch_size_ratio": unsolved_num_responses / real_num_responses, "packed_ratio": len(packed_sequences.query_responses) / real_num_responses, "val/solve_rate_hist": batch_stats.percent_solved_hist, "val/total_reward_groups": real_num_responses / self.config.num_samples_per_prompt_rollout, "val/sequence_lengths": sequence_lengths.mean(), "val/sequence_lengths_min": sequence_lengths.min(), "val/sequence_lengths_max": sequence_lengths.max(), "val/sequence_lengths_unsolved": ( 0 if len(sequence_length_unsolved) == 0 else sequence_length_unsolved.mean() ), "val/sequence_lengths_solved": ( 0 if len(sequence_length_solved) == 0 else sequence_length_solved.mean() ), "val/sequence_lengths_unsolved_hist": sequence_length_unsolved, "val/sequence_lengths_solved_hist": sequence_length_solved, "val/stop_rate": stop_rate, "val/advantages_mean": advantages.mean(), "val/advantages_min": advantages.min(), "val/advantages_max": advantages.max(), "val/advantages_hist": advantages, **reward_metrics, **batch_metrics_prefixed, } tool_stats = EnvStatistics(tool_names=self.tool_names) for rollout_stats in result.request_info.tool_call_stats: tool_stats.add_rollout(rollout_stats) step_metrics.update(tool_stats.compute_metrics()) step_metrics.update(_aggregate_env_metrics(result.request_info.rollout_states)) assert result.token_statistics is not None total_tokens = result.token_statistics.num_prompt_tokens + result.token_statistics.num_response_tokens step_metrics["val/actor_tokens_per_second"] = total_tokens / result.token_statistics.generation_time group_times = batch_stats.per_group_generation_times step_metrics["time/group_generation_mean"] = float(group_times.mean()) step_metrics["time/group_generation_max"] = float(group_times.max()) step_metrics["time/group_generation_min"] = float(group_times.min()) with self.lock: self.prepared_data[self.training_step] = collated_data self.metrics[self.training_step] = step_metrics self.current_prepared_step = self.training_step self.training_step += 1 def get_data(self, rank: int, step: int) -> dict: """Called by each rank's StreamingDataLoader. Blocks until data ready.""" if self._prep_future is None: self.start() logger.info( f"[DataPreparationActor.get_data] rank={rank} requesting step={step}, current_prepared_step={self.current_prepared_step}" ) wait_count = 0 while True: if self._prep_future.done(): self._prep_future.result() with self.lock: if step <= self.current_prepared_step: batch_data = self.prepared_data[step][rank] result = {"batch": batch_data, "metrics": self.metrics[step]} self._last_consumed_step = max(self._last_consumed_step, step) self._cleanup_old_steps(step) logger.info( f"[DataPreparationActor.get_data] rank={rank} got data for step={step} after {wait_count} waits" ) return result wait_count += 1 if wait_count % 1000 == 0: logger.info( f"[DataPreparationActor.get_data] rank={rank} still waiting for step={step}, current_prepared_step={self.current_prepared_step}, wait_count={wait_count}" ) time.sleep(0.01) def _cleanup_old_steps(self, current_step: int): """Remove old step data to prevent memory leak.""" steps_to_remove = [s for s in self.prepared_data if s < current_step - 1] for s in steps_to_remove: del self.prepared_data[s] if s in self.metrics: del self.metrics[s] def get_state(self) -> dict: return { "training_step": self.training_step, "last_consumed_step": self._last_consumed_step, "iter_dataloader_state": self.iter_dataloader.state_dict(), } def set_state(self, state: dict): if self._prep_future is not None: raise RuntimeError("Cannot update DataPreparationActor state after preparation has started") self.iter_dataloader.load_state_dict(state["iter_dataloader_state"]) self._last_consumed_step = state.get("last_consumed_step", state["training_step"] - 1) self.training_step = self._last_consumed_step + 1 logger.info( f"[DataPreparationActor] Restored state: training_step={self.training_step}, last_consumed_step={self._last_consumed_step}" )