//! Minimal end-to-end training loop for [`FoundationModel`](super::runtime::FoundationModel). //! //! This is intentionally one training regime rather than another experiment ladder: //! random initialization (or optional same-architecture warm start), one optimizer, //! TRAIN-only normalization, Validation-only early stopping, and one checkpoint format. use super::auxiliary_supervision::{ chemistry_summary_targets, conformation_proxy_targets_v0520, masked_bce_with_logits, masked_mse, pair_interaction_targets, }; use super::causal::foundation_causal_sequence_mean_nlls; use super::collate::{FoundationCollator, FoundationCollatorConfig, FoundationCorruptionConfig}; use super::corpus::{load_foundation_corpus, FoundationCorpusConfig}; use super::data::{FoundationTrainingRecord, RetentionTimeObjective}; use super::diffusion::FOUNDATION_OPEN_PTM_MASS_SCALE_DA; use super::experiment::{FoundationBenchmarkManifest, FoundationPartition}; use super::loss::{ contrastive_info_nce_loss, multi_task_loss_with_ms2_config, FoundationLossWeights, FoundationMs2LossConfig, FoundationTargets, }; use super::mobility_consensus::{ bruker_ccs_factor_from_values, build_train_consensus_supervision, deterministic_example_order, finite_mobility_ccs_indices, MobilityConsensusExample, MobilityConsensusSupervision, }; use super::normalization::{ FoundationRegressionNormalization, FoundationRegressionNormalizationStrategy, FoundationTargetNormalizationConfig, }; use super::optimizer::{FoundationAdamW, FoundationAdamWConfig}; use super::runtime::{FoundationCheckpointMetadata, FoundationModel, FoundationModelConfig}; use super::spectrum::FoundationSpectrum; use anyhow::{Context, Result}; use candle_core::{Device, Tensor}; use serde::{Deserialize, Serialize}; use std::fs::{self, OpenOptions}; use std::io::Write; use std::path::{Path, PathBuf}; /// Production training controller. `Joint` preserves the cleanup-era behavior; /// `ResearchCurriculum` trains one random-init model with deterministic staged /// property, representation, mobility, and inverse updates inspired by the /// successful v0.35-v0.52 research program. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum FoundationTrainingStrategy { #[default] Joint, ResearchCurriculum, } fn legacy_training_strategy() -> FoundationTrainingStrategy { FoundationTrainingStrategy::Joint } fn legacy_warmup_steps() -> usize { 0 } fn legacy_min_lr_ratio() -> f64 { 1.0 } fn legacy_max_gradient_norm() -> Option { None } /// Hyperparameters for the one supported end-to-end foundation training regime. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(default)] pub struct FoundationTrainingConfig { pub model: FoundationModelConfig, pub batch_size: usize, pub learning_rate: f64, /// Update schedule. Historical configs deserialize as `joint`. #[serde(default = "legacy_training_strategy")] pub strategy: FoundationTrainingStrategy, /// Linear warmup updates before cosine decay. Historical configs use zero. #[serde(default = "legacy_warmup_steps")] pub warmup_steps: usize, /// Final/base LR ratio after cosine decay. Historical configs use one (constant LR). #[serde(default = "legacy_min_lr_ratio")] pub min_learning_rate_ratio: f64, /// Optional global gradient norm clip. Historical configs leave clipping disabled. #[serde(default = "legacy_max_gradient_norm")] pub max_gradient_norm: Option, /// Enable TRAIN-only source-aware Bruker mobility consensus in curriculum stage D. /// Missing field in historical JSON remains disabled (old direct-CCS stage). #[serde(default)] pub mobility_consensus_supervision: bool, /// v0.50 six-class residue-pair geometry/composition proxy. #[serde(default)] pub pair_auxiliary_weight: f64, /// v0.50 eight-component chemistry summary. #[serde(default)] pub chemistry_summary_weight: f64, /// v0.52 fourteen-component coarse conformation/charge proxy. #[serde(default)] pub conformation_proxy_weight: f64, pub epochs: usize, pub early_stopping_patience: usize, pub seed: u64, /// Opt-in CUDA device RNG seed, set once before model creation. It changes weight /// initialization and all later device-side random draws. `None` keeps Candle's /// fixed default (all historical runs), so only `seed` (data order) varied between them. #[serde(default)] pub init_seed: Option, /// Opt-in: also save `epoch_NNN` checkpoints after every epoch (learning curves). #[serde(default)] pub save_epoch_checkpoints: bool, /// Opt-in resume of an interrupted run from its `epoch_NNN` checkpoint (passed as the /// initial checkpoint, whose metadata must record `completed_epochs == N`). Epochs /// 1..=N are skipped: their data order is replayed only to rebuild the exposure report, /// and the LR schedule continues at the exact global update. AdamW moments are NOT /// restored (checkpoints do not store them), so they restart from zero with bias /// correction. `None` = normal training (all historical runs). #[serde(default)] pub resume_from_epoch: Option, /// Opt-in diagnostic: every N curriculum cycles, also compute one gradient per weighted /// loss term (extra backward passes, no parameter change) and log its global norm and /// cosine with the total gradient. `None` = off (all historical runs). #[serde(default)] pub term_gradient_probe_every: Option, /// Optional bounded smoke/debug mode; `None` trains the full partition. pub max_train_batches_per_epoch: Option, /// Opt-in v0.51/v0.52 training: two disjoint property batches per curriculum /// cycle. `None` retains the historical unified-trainer batch reuse. #[serde(default)] pub property_batch_size: Option, /// Opt-in research-curriculum Mobility-stage batch size (source-aware CCS consensus /// examples per cycle). `None` keeps the historical value `batch_size` (all earlier runs). #[serde(default)] pub mobility_batch_size: Option, /// Optional bounded validation mode; `None` evaluates the full partition. pub max_validation_batches: Option, /// Maximum number of spectrum-bearing Validation records used for teacher-forced inverse and retrieval metrics per epoch. /// Set to zero to disable inverse evaluation metrics without changing the training objective. pub inverse_evaluation_records: usize, /// Maximum number of spectrum-bearing Validation records decoded greedily per epoch. /// Set to zero to disable free-generation metrics while retaining teacher-forced inverse metrics. pub generation_evaluation_records: usize, pub forward_loss_weights: FoundationLossWeights, pub ms2_loss: FoundationMs2LossConfig, pub peptide_contrastive_weight: f64, pub inverse_weight: f64, pub inverse_ptm_mass_weight: f64, pub cross_modal_alignment_weight: f64, pub contrastive_temperature: f64, pub corruption: FoundationCorruptionConfig, } impl Default for FoundationTrainingConfig { fn default() -> Self { Self { model: FoundationModelConfig::default(), batch_size: 32, learning_rate: 2.0e-5, strategy: FoundationTrainingStrategy::ResearchCurriculum, warmup_steps: 500, min_learning_rate_ratio: 0.10, max_gradient_norm: Some(1.0), mobility_consensus_supervision: true, pair_auxiliary_weight: 0.10, chemistry_summary_weight: 0.10, conformation_proxy_weight: 0.10, epochs: 20, early_stopping_patience: 4, seed: 20_261_006, init_seed: None, save_epoch_checkpoints: false, resume_from_epoch: None, term_gradient_probe_every: None, max_train_batches_per_epoch: None, property_batch_size: None, mobility_batch_size: None, max_validation_batches: None, inverse_evaluation_records: 256, generation_evaluation_records: 16, forward_loss_weights: FoundationLossWeights { rt: 1.0, ccs: 1.0, ms2: 1.0, masked_residue: 0.10, chemistry: 0.10, contrastive: 0.0, }, ms2_loss: FoundationMs2LossConfig { pointwise_weight: 1.0, cosine_weight: 0.25, pearson_weight: 0.25, cosine_epsilon: 1.0e-8, }, peptide_contrastive_weight: 0.10, inverse_weight: 1.0, inverse_ptm_mass_weight: 0.10, cross_modal_alignment_weight: 0.10, contrastive_temperature: 0.10, corruption: FoundationCorruptionConfig::default(), } } } impl FoundationTrainingConfig { pub fn validate(&self) -> Result<()> { if self.batch_size == 0 || self.epochs == 0 || self.early_stopping_patience == 0 { anyhow::bail!( "foundation batch_size, epochs, and early_stopping_patience must be positive" ); } if !(self.learning_rate > 0.0 && self.learning_rate.is_finite()) { anyhow::bail!("foundation learning_rate must be positive and finite"); } if !(self.min_learning_rate_ratio > 0.0 && self.min_learning_rate_ratio <= 1.0 && self.min_learning_rate_ratio.is_finite()) { anyhow::bail!("foundation min_learning_rate_ratio must be finite and in (0, 1]"); } if let Some(max_norm) = self.max_gradient_norm { if !(max_norm > 0.0 && max_norm.is_finite()) { anyhow::bail!("foundation max_gradient_norm must be positive and finite"); } } if self.term_gradient_probe_every == Some(0) { anyhow::bail!("foundation term_gradient_probe_every must be positive when set"); } if let Some(resume) = self.resume_from_epoch { if resume == 0 || resume >= self.epochs { anyhow::bail!( "foundation resume_from_epoch must be in 1..epochs ({}); got {resume}", self.epochs ); } } for (name, value) in [ ( "max_train_batches_per_epoch", self.max_train_batches_per_epoch, ), ("max_validation_batches", self.max_validation_batches), ("property_batch_size", self.property_batch_size), ("mobility_batch_size", self.mobility_batch_size), ] { if matches!(value, Some(0)) { anyhow::bail!("foundation {name} must be positive when set"); } } if self.strategy == FoundationTrainingStrategy::ResearchCurriculum && self .property_batch_size .is_some_and(|size| size < self.batch_size) { anyhow::bail!("research property_batch_size must be at least batch_size"); } self.model.validate()?; if self.strategy == FoundationTrainingStrategy::ResearchCurriculum && (self.pair_auxiliary_weight > 0.0 || self.chemistry_summary_weight > 0.0 || self.conformation_proxy_weight > 0.0) && (!self.model.auxiliary_supervision_heads || self.model.peptide_backbone != super::runtime::FoundationPeptideBackbone::ResiduePairTaskTokens) { anyhow::bail!("research auxiliary losses require a residue-pair backbone with auxiliary supervision heads"); } if self.strategy == FoundationTrainingStrategy::ResearchCurriculum && self.conformation_proxy_weight > 0.0 && (!self.model.specialists.enabled || !self.model.specialists.mobility_ccs) { anyhow::bail!("conformation auxiliary requires the native mobility specialist"); } for (name, value) in [ ("rt", self.forward_loss_weights.rt), ("ccs", self.forward_loss_weights.ccs), ("ms2", self.forward_loss_weights.ms2), ("masked_residue", self.forward_loss_weights.masked_residue), ("chemistry", self.forward_loss_weights.chemistry), ] { if !(value >= 0.0 && value.is_finite()) { anyhow::bail!( "foundation forward loss weight {name} must be finite and non-negative" ); } } for (name, value) in [ ( "residue_mask_probability", self.corruption.residue_mask_probability, ), ( "chemistry_mask_probability", self.corruption.chemistry_mask_probability, ), ] { if !(0.0..=1.0).contains(&value) { anyhow::bail!("foundation corruption {name} must be in [0, 1]"); } } for (name, value) in [ ("pair_auxiliary_weight", self.pair_auxiliary_weight), ("chemistry_summary_weight", self.chemistry_summary_weight), ("conformation_proxy_weight", self.conformation_proxy_weight), ( "peptide_contrastive_weight", self.peptide_contrastive_weight, ), ("inverse_weight", self.inverse_weight), ("inverse_ptm_mass_weight", self.inverse_ptm_mass_weight), ( "cross_modal_alignment_weight", self.cross_modal_alignment_weight, ), ] { if !(value >= 0.0 && value.is_finite()) { anyhow::bail!("foundation {name} must be finite and non-negative"); } } if !(self.contrastive_temperature > 0.0 && self.contrastive_temperature.is_finite()) { anyhow::bail!("foundation contrastive_temperature must be positive and finite"); } self.ms2_loss.validate()?; Ok(()) } } /// Minimal subset of the historical prepared-run YAML needed by production training. #[derive(Debug, Clone, Deserialize)] struct FoundationTrainingSource { corpus: FoundationCorpusConfig, benchmark_manifest: PathBuf, } /// Load materialized foundation records from the same prepared-run YAML used by training. pub fn load_foundation_records_from_run( run_yaml: impl AsRef, ) -> Result> { let (_, records) = load_foundation_run(run_yaml)?; Ok(records) } /// Load only one benchmark partition from a prepared run. /// /// This keeps engineering inference/evaluation from silently consuming protected /// partitions simply because they share the same materialized corpus. pub fn load_foundation_records_from_run_partition( run_yaml: impl AsRef, partition: FoundationPartition, ) -> Result> { let (benchmark, records) = load_foundation_run(run_yaml)?; let indices = benchmark.partition_indices(partition); if indices.is_empty() { anyhow::bail!("foundation requested partition contains no records"); } indices .into_iter() .map(|index| { records.get(index).cloned().ok_or_else(|| { anyhow::anyhow!("foundation benchmark record index {index} is out of range") }) }) .collect() } fn load_foundation_run( run_yaml: impl AsRef, ) -> Result<(FoundationBenchmarkManifest, Vec)> { let run_yaml = run_yaml.as_ref(); let source: FoundationTrainingSource = serde_yaml::from_str( &fs::read_to_string(run_yaml) .with_context(|| format!("read foundation run YAML {}", run_yaml.display()))?, ) .with_context(|| format!("parse foundation run YAML {}", run_yaml.display()))?; let corpus = load_foundation_corpus(&source.corpus)?; let benchmark = FoundationBenchmarkManifest::read_tsv(&source.benchmark_manifest) .with_context(|| format!("read benchmark {}", source.benchmark_manifest.display()))?; benchmark.validate_against_records(&corpus.records)?; Ok((benchmark, corpus.records)) } /// One epoch's aggregate optimization/evaluation losses. #[derive(Debug, Clone, Copy, Default)] pub struct FoundationEpochLosses { pub total: f64, pub forward: f64, pub inverse: f64, pub inverse_ptm_mass: f64, pub peptide_contrastive: f64, pub cross_modal_alignment: f64, pub batches: usize, } impl FoundationEpochLosses { fn add(&mut self, value: BatchLossScalars) { self.total += f64::from(value.total); self.forward += f64::from(value.forward); self.inverse += f64::from(value.inverse.unwrap_or(0.0)); self.inverse_ptm_mass += f64::from(value.inverse_ptm_mass.unwrap_or(0.0)); self.peptide_contrastive += f64::from(value.peptide_contrastive.unwrap_or(0.0)); self.cross_modal_alignment += f64::from(value.cross_modal_alignment.unwrap_or(0.0)); self.batches += 1; } fn means(mut self) -> Self { if self.batches > 0 { let n = self.batches as f64; self.total /= n; self.forward /= n; self.inverse /= n; self.inverse_ptm_mass /= n; self.peptide_contrastive /= n; self.cross_modal_alignment /= n; } self } } #[derive(Debug, Clone)] pub struct FoundationTrainingSummary { pub completed_epochs: usize, pub best_validation_loss: f64, pub checkpoint: PathBuf, pub validation_metrics: PathBuf, } /// Native-unit and inverse-task diagnostics for one Validation pass. #[derive(Debug, Clone, Copy, Default)] pub struct FoundationEvaluationMetrics { pub records: usize, pub rt_count: usize, pub rt_mae_native: Option, pub rt_rmse_native: Option, pub ccs_count: usize, pub ccs_mae: Option, pub ccs_rmse: Option, pub ms2_point_count: usize, pub ms2_rmse: Option, pub ms2_spectrum_count: usize, pub ms2_cosine_similarity: Option, pub ms2_spectral_angle: Option, pub ms2_pearson: Option, pub inverse_sequence_count: usize, pub inverse_mean_nll: Option, pub inverse_ptm_site_count: usize, pub inverse_ptm_mass_rmse_da: Option, pub retrieval_queries: usize, pub retrieval_top1_accuracy: Option, pub retrieval_mrr: Option, pub generation_attempts: usize, pub generation_success_rate: Option, pub generation_sequence_exact_rate: Option, pub generation_sequence_il_equivalent_rate: Option, pub generation_mean_length: Option, } struct BatchLoss { total: Tensor, scalars: BatchLossScalars, /// Weighted loss terms whose sum is `total`. They are tensor handles into the same /// graph, so collecting them adds no compute. Used only by the opt-in gradient probe. terms: Vec<(&'static str, Tensor)>, } #[derive(Debug, Clone, Copy, Default)] struct BatchLossScalars { total: f32, forward: f32, inverse: Option, inverse_ptm_mass: Option, peptide_contrastive: Option, cross_modal_alignment: Option, } /// Train a foundation model from a prepared run YAML. /// /// `initial_checkpoint` is model-only warm start for the same stable architecture. /// Omit it for true random initialization. pub fn train_foundation_model( run_yaml: impl AsRef, output_dir: impl AsRef, training: FoundationTrainingConfig, initial_checkpoint: Option<&Path>, device: Device, ) -> Result { training.validate()?; let run_yaml = run_yaml.as_ref(); let output_dir = output_dir.as_ref(); if output_dir.exists() && fs::read_dir(output_dir) .with_context(|| format!("read foundation output {}", output_dir.display()))? .next() .is_some() { anyhow::bail!( "foundation training requires a fresh output directory: {}", output_dir.display() ); } let source: FoundationTrainingSource = serde_yaml::from_str( &fs::read_to_string(run_yaml) .with_context(|| format!("read foundation run YAML {}", run_yaml.display()))?, ) .with_context(|| format!("parse foundation run YAML {}", run_yaml.display()))?; let corpus = load_foundation_corpus(&source.corpus)?; let benchmark = FoundationBenchmarkManifest::read_tsv(&source.benchmark_manifest) .with_context(|| format!("read benchmark {}", source.benchmark_manifest.display()))?; benchmark.validate_against_records(&corpus.records)?; let train_indices = partition_indices(&benchmark, FoundationPartition::Train); let validation_indices = partition_indices(&benchmark, FoundationPartition::Validation); if train_indices.is_empty() || validation_indices.is_empty() { anyhow::bail!( "foundation training requires non-empty TRAIN and Validation partitions; observed train={} validation={}", train_indices.len(), validation_indices.len() ); } let mut normalization = FoundationTargetNormalizationConfig { rt: FoundationRegressionNormalization { strategy: FoundationRegressionNormalizationStrategy::TrainStandardize, ..FoundationRegressionNormalization::default() }, ccs: FoundationRegressionNormalization { strategy: FoundationRegressionNormalizationStrategy::TrainStandardize, ..FoundationRegressionNormalization::default() }, }; let rt_objective = training.model.rt_objective; normalization.resolve_from_training_partition(&corpus.records, &train_indices, rt_objective)?; let mut model_config = training.model.clone(); model_config.peptide.instrument_vocab_size = source.corpus.instrument_vocab_size; if let Some(init_seed) = training.init_seed { if !device.is_cuda() { anyhow::bail!("foundation init_seed is supported on CUDA only"); } device.set_seed(init_seed)?; } let mut model = FoundationModel::new( model_config.clone(), normalization, corpus.instrument_names.clone(), device.clone(), )?; if let Some(checkpoint) = initial_checkpoint { model.load_weights(checkpoint)?; } let start_epoch = match training.resume_from_epoch { None => 0, Some(resume) => { let checkpoint = initial_checkpoint.context( "foundation resume_from_epoch requires the epoch checkpoint as checkpoint_file", )?; let path = checkpoint.join("metadata.yaml"); let metadata: super::runtime::FoundationCheckpointMetadata = serde_yaml::from_str(&fs::read_to_string(&path).with_context(|| { format!("read resume metadata {}", path.display()) })?)?; if metadata.completed_epochs != resume { anyhow::bail!( "foundation resume_from_epoch={resume} but checkpoint records completed_epochs={}", metadata.completed_epochs ); } resume } }; // Fit the source affine/reliability model using TRAIN identities only. // Never refit on Validation; historical TEST and TRAIN-HOLDOUT stay closed. let mobility_consensus = if training.strategy == FoundationTrainingStrategy::ResearchCurriculum && training.mobility_consensus_supervision { let eligible = finite_mobility_ccs_indices(&corpus.records, &train_indices); if eligible.is_empty() { anyhow::bail!("TRAIN contains no valid mobility/charge/mz records for curriculum mobility consensus"); } let fitted = build_train_consensus_supervision(&corpus.records, &corpus.provenance, &eligible)?; if fitted.examples.is_empty() { anyhow::bail!("TRAIN produced no source-aware mobility consensus examples"); } Some(fitted) } else { None }; let train_collator = FoundationCollator::new( model_config.peptide.clone(), FoundationCollatorConfig { retention_time_objective: rt_objective, corruption: training.corruption, }, )?; let optimizer_config = FoundationAdamWConfig { learning_rate: training.learning_rate, ..FoundationAdamWConfig::default() }; let mut joint_optimizer = match training.strategy { FoundationTrainingStrategy::Joint => { Some(FoundationAdamW::new(model.varmap(), optimizer_config)?) } FoundationTrainingStrategy::ResearchCurriculum => None, }; let mut peptide_optimizer = match training.strategy { FoundationTrainingStrategy::ResearchCurriculum => Some( FoundationAdamW::new_for_prefixes(model.varmap(), optimizer_config, &["peptide."])?, ), FoundationTrainingStrategy::Joint => None, }; let mut inverse_optimizer = match training.strategy { FoundationTrainingStrategy::ResearchCurriculum => Some(FoundationAdamW::new_for_prefixes( model.varmap(), optimizer_config, &["inverse.", "alignment."], )?), FoundationTrainingStrategy::Joint => None, }; let train_batches_per_epoch = match training.strategy { FoundationTrainingStrategy::Joint => train_indices.len().div_ceil(training.batch_size), FoundationTrainingStrategy::ResearchCurriculum => { research_curriculum_cycles(train_indices.len(), &training) } } .min(training.max_train_batches_per_epoch.unwrap_or(usize::MAX)); if train_batches_per_epoch == 0 { anyhow::bail!("TRAIN is too small for one research curriculum cycle with the configured property_batch_size"); } let updates_per_cycle = match training.strategy { FoundationTrainingStrategy::Joint => 1usize, FoundationTrainingStrategy::ResearchCurriculum => 5usize, }; let planned_updates = training .epochs .saturating_mul(train_batches_per_epoch) .saturating_mul(updates_per_cycle) .max(1); let mut global_update = 0usize; // A byte per corpus record is cheaper than a set and reports exact across-epoch // coverage without materializing another copy of multi-million-record indices. let mut seen_property_records = if training.strategy == FoundationTrainingStrategy::ResearchCurriculum { vec![false; corpus.records.len()] } else { Vec::new() }; fs::create_dir_all(output_dir) .with_context(|| format!("create foundation output {}", output_dir.display()))?; if let Some(consensus) = mobility_consensus.as_ref() { write_mobility_consensus_provenance(output_dir, consensus)?; } let mut resolved_training = training.clone(); resolved_training.model = model_config.clone(); fs::write( output_dir.join("training.yaml"), serde_yaml::to_string(&resolved_training)?, ) .with_context(|| { format!( "write foundation training config to {}", output_dir.display() ) })?; let history = output_dir.join("training_history.tsv"); let exposure_path = output_dir.join("training_exposure.tsv"); if training.strategy == FoundationTrainingStrategy::ResearchCurriculum { fs::write(&exposure_path, "epoch\tcycles\tproperty_updates\tproperty_batch_size\tproperty_records\tnew_property_records\tcumulative_unique_property_records\ttrain_records\trt_labelled\tms2_labelled\tccs_labelled\tinverse_spectrum_records\n")?; } let validation_metrics_path = output_dir.join("validation_metrics.tsv"); fs::write( &history, "epoch\ttrain_total\ttrain_forward\ttrain_inverse\ttrain_inverse_ptm_mass\ttrain_peptide_contrastive\ttrain_cross_modal_alignment\tvalidation_total\tvalidation_forward\tvalidation_inverse\tvalidation_inverse_ptm_mass\tvalidation_peptide_contrastive\tvalidation_cross_modal_alignment\n", )?; fs::write( &validation_metrics_path, "epoch\trecords\trt_count\trt_mae_native\trt_rmse_native\tccs_count\tccs_mae\tccs_rmse\tms2_point_count\tms2_rmse\tms2_spectrum_count\tms2_cosine_similarity\tms2_spectral_angle\tms2_pearson\tinverse_sequence_count\tinverse_mean_nll\tinverse_ptm_site_count\tinverse_ptm_mass_rmse_da\tretrieval_queries\tretrieval_top1_accuracy\tretrieval_mrr\tgeneration_attempts\tgeneration_success_rate\tgeneration_sequence_exact_rate\tgeneration_sequence_il_equivalent_rate\tgeneration_mean_length\n", )?; let mut best_validation_loss = f64::INFINITY; let mut best_forward_score = f64::INFINITY; let mut best_inverse_nll = f64::INFINITY; let mut epochs_without_improvement = 0usize; let mut completed_epochs = 0usize; // Resume: replay the skipped epochs' data order (pure function of seed ^ epoch) so the // exposure report and seen-record set match an uninterrupted run, and continue the LR // schedule at the exact update index. No training happens for these epochs. for epoch in 0..start_epoch { if training.strategy == FoundationTrainingStrategy::ResearchCurriculum { let mut order = train_indices.clone(); shuffle_indices(&mut order, training.seed ^ epoch as u64); write_research_exposure( &exposure_path, epoch + 1, &order, &corpus.records, &training, train_batches_per_epoch, train_indices.len(), &mut seen_property_records, )?; } global_update += train_batches_per_epoch * updates_per_cycle; completed_epochs = epoch + 1; } if start_epoch > 0 { eprintln!( "foundation_resume from_epoch={start_epoch} global_update={global_update} planned_updates={planned_updates} lr={:.9e} adam_moments=reset", scheduled_learning_rate(&training, global_update, planned_updates) ); } for epoch in start_epoch..training.epochs { let mut order = train_indices.clone(); shuffle_indices(&mut order, training.seed ^ epoch as u64); let train_losses = match training.strategy { FoundationTrainingStrategy::Joint => run_epoch_joint( &model, &corpus.records, &order, &train_collator, &training, joint_optimizer.as_mut(), training.max_train_batches_per_epoch, epoch as u64, &mut global_update, planned_updates, &device, )?, FoundationTrainingStrategy::ResearchCurriculum => run_epoch_research_curriculum( &model, &corpus.records, &order, &train_collator, &training, mobility_consensus.as_ref(), peptide_optimizer .as_mut() .expect("research peptide optimizer"), inverse_optimizer .as_mut() .expect("research inverse optimizer"), training.max_train_batches_per_epoch, epoch as u64, &mut global_update, planned_updates, &device, )?, }; if training.strategy == FoundationTrainingStrategy::ResearchCurriculum { write_research_exposure( &exposure_path, epoch + 1, &order, &corpus.records, &training, train_batches_per_epoch, train_indices.len(), &mut seen_property_records, )?; } let validation_losses = run_epoch_joint( &model, &corpus.records, &validation_indices, &train_collator, &training, None, training.max_validation_batches, training.seed ^ 0xa5a5_5a5a_1234_5678, &mut global_update, planned_updates, &device, )?; if !train_losses.total.is_finite() || !validation_losses.total.is_finite() { anyhow::bail!( "foundation training produced non-finite loss at epoch {}: train={} validation={}", epoch + 1, train_losses.total, validation_losses.total ); } completed_epochs = epoch + 1; append_history(&history, completed_epochs, train_losses, validation_losses)?; let validation_metrics = evaluate_partition( &model, &corpus.records, &validation_indices, training.batch_size, training.max_validation_batches, training.inverse_evaluation_records, training.generation_evaluation_records, training.ms2_loss.cosine_epsilon, &device, )?; append_validation_metrics( &validation_metrics_path, completed_epochs, validation_metrics, )?; if training.save_epoch_checkpoints { let metadata = checkpoint_metadata( model_config.clone(), normalization, &corpus, &benchmark, completed_epochs, validation_losses.total, ); model.save_checkpoint( output_dir.join(format!("epoch_{completed_epochs:03}")), &metadata, )?; } if let Some(score) = forward_balanced_score(validation_metrics) { if score < best_forward_score { best_forward_score = score; let metadata = checkpoint_metadata( model_config.clone(), normalization, &corpus, &benchmark, completed_epochs, validation_losses.total, ); model.save_checkpoint(output_dir.join("best_forward"), &metadata)?; } } if let Some(inverse_nll) = validation_metrics.inverse_mean_nll { if inverse_nll < best_inverse_nll { best_inverse_nll = inverse_nll; let metadata = checkpoint_metadata( model_config.clone(), normalization, &corpus, &benchmark, completed_epochs, validation_losses.total, ); model.save_checkpoint(output_dir.join("best_inverse"), &metadata)?; } } eprintln!( "foundation epoch={} train_loss={:.6} validation_loss={:.6} rt_mae={} ccs_mae={} ms2_cosine={} inverse_nll={} retrieval_mrr={} generation_exact={}", completed_epochs, train_losses.total, validation_losses.total, optional_metric(validation_metrics.rt_mae_native), optional_metric(validation_metrics.ccs_mae), optional_metric(validation_metrics.ms2_cosine_similarity), optional_metric(validation_metrics.inverse_mean_nll), optional_metric(validation_metrics.retrieval_mrr), optional_metric(validation_metrics.generation_sequence_exact_rate), ); if validation_losses.total < best_validation_loss { best_validation_loss = validation_losses.total; epochs_without_improvement = 0; let metadata = checkpoint_metadata( model_config.clone(), normalization, &corpus, &benchmark, completed_epochs, best_validation_loss, ); model.save_checkpoint(output_dir, &metadata)?; } else { epochs_without_improvement += 1; if epochs_without_improvement >= training.early_stopping_patience { break; } } } Ok(FoundationTrainingSummary { completed_epochs, best_validation_loss, checkpoint: output_dir.to_path_buf(), validation_metrics: validation_metrics_path, }) } fn checkpoint_metadata( model_config: FoundationModelConfig, normalization: FoundationTargetNormalizationConfig, corpus: &super::corpus::FoundationCorpus, benchmark: &FoundationBenchmarkManifest, completed_epochs: usize, validation_loss: f64, ) -> FoundationCheckpointMetadata { let mut metadata = FoundationCheckpointMetadata::new( model_config, normalization, corpus.instrument_names.clone(), ); metadata.completed_epochs = completed_epochs; metadata.best_validation_loss = Some(validation_loss); metadata.corpus_fingerprint = Some(format!("fnv1a64:{:016x}", corpus.corpus_fingerprint)); metadata.benchmark_fingerprint = Some(format!("fnv1a64:{:016x}", benchmark.manifest_fingerprint())); metadata } fn forward_balanced_score(metrics: FoundationEvaluationMetrics) -> Option { let rt = metrics.rt_mae_native? / 4.528375; let ccs = metrics.ccs_mae? / 8.80611929; let cosine = (1.0 - metrics.ms2_cosine_similarity?) / (1.0 - 0.904061); let spectral = (1.0 - metrics.ms2_spectral_angle?) / (1.0 - 0.747700); let pearson = (1.0 - metrics.ms2_pearson?) / (1.0 - 0.684548); Some(0.25 * rt + 0.40 * ccs + 0.12 * cosine + 0.12 * spectral + 0.11 * pearson) } fn run_epoch_joint( model: &FoundationModel, records: &[FoundationTrainingRecord], indices: &[usize], collator: &FoundationCollator, config: &FoundationTrainingConfig, mut optimizer: Option<&mut FoundationAdamW>, max_batches: Option, seed: u64, global_update: &mut usize, planned_updates: usize, device: &Device, ) -> Result { let mut aggregate = FoundationEpochLosses::default(); for (batch_index, chunk) in indices .chunks(config.batch_size) .take(max_batches.unwrap_or(usize::MAX)) .enumerate() { let owned = chunk .iter() .map(|&index| { records.get(index).cloned().ok_or_else(|| { anyhow::anyhow!("foundation training record index {index} is out of bounds") }) }) .collect::>>()?; if owned.is_empty() { continue; } let batch_seed = seed ^ (batch_index as u64).wrapping_mul(0x9e37_79b9_7f4a_7c15); let loss = batch_loss( model, &owned, collator, config, batch_seed, optimizer.is_some(), device, )?; if let Some(opt) = optimizer.as_mut() { let lr = scheduled_learning_rate(config, *global_update, planned_updates); (*opt).set_learning_rate(lr)?; (*opt).backward_step(&loss.total, config.max_gradient_norm)?; *global_update = (*global_update).saturating_add(1); } aggregate.add(loss.scalars); } if aggregate.batches == 0 { anyhow::bail!("foundation epoch produced no batches"); } Ok(aggregate.means()) } #[derive(Debug, Clone, Copy)] enum FoundationCurriculumStage { Property, Representation, Mobility, Inverse, } // The opt-in research schedule consumes two *different* property batches per // cycle, as v0.51/v0.52 did. Missing config preserves the unified trainer's // original same-batch schedule for existing experiments and small fixtures. fn research_curriculum_cycles(records: usize, config: &FoundationTrainingConfig) -> usize { match config.property_batch_size { Some(size) => records / size.saturating_mul(2).max(1), None => records.div_ceil(config.batch_size), } } fn research_property_batches<'a>( indices: &'a [usize], cycle: usize, config: &FoundationTrainingConfig, ) -> (&'a [usize], &'a [usize]) { match config.property_batch_size { Some(size) => { let start = cycle * size * 2; ( &indices[start..start + size], &indices[start + size..start + size * 2], ) } None => { let start = cycle * config.batch_size; let batch = &indices[start..(start + config.batch_size).min(indices.len())]; (batch, batch) } } } fn write_research_exposure( path: &Path, epoch: usize, indices: &[usize], records: &[FoundationTrainingRecord], config: &FoundationTrainingConfig, cycles: usize, train_records: usize, seen: &mut [bool], ) -> Result<()> { let mut selected = 0usize; let mut fresh = 0usize; let mut rt = 0usize; let mut ms2 = 0usize; let mut ccs = 0usize; let mut inverse = 0usize; for cycle in 0..cycles { let (a, b) = research_property_batches(indices, cycle, config); for &index in a.iter().chain(b.iter()) { let record = &records[index]; selected += 1; if !seen[index] { seen[index] = true; fresh += 1; } if record.retention_time.normalized.is_some_and(f32::is_finite) || record.retention_time.harmonized.is_some_and(f32::is_finite) || record .retention_time .observed_seconds .is_some_and(f32::is_finite) { rt += 1; } if !record.fragments.is_empty() { ms2 += 1; } if record.ccs.is_some_and(f32::is_finite) { ccs += 1; } } // Inverse uses only the first `batch_size` rows of property batch B. for &index in b.iter().take(config.batch_size) { let record = &records[index]; if !record.observed_spectrum_peaks.is_empty() || !record.fragments.is_empty() { inverse += 1; } } } let cumulative_unique = seen.iter().filter(|&&value| value).count(); let property_batch_size = config.property_batch_size.unwrap_or(config.batch_size); let line = format!("{epoch}\t{cycles}\t{}\t{property_batch_size}\t{selected}\t{fresh}\t{cumulative_unique}\t{}\t{rt}\t{ms2}\t{ccs}\t{inverse}\n", cycles * 2, train_records); OpenOptions::new() .append(true) .open(path)? .write_all(line.as_bytes())?; println!("foundation_train_exposure\t{}", line.trim_end()); Ok(()) } fn run_epoch_research_curriculum( model: &FoundationModel, records: &[FoundationTrainingRecord], indices: &[usize], collator: &FoundationCollator, config: &FoundationTrainingConfig, mobility_consensus: Option<&MobilityConsensusSupervision>, peptide_optimizer: &mut FoundationAdamW, inverse_optimizer: &mut FoundationAdamW, max_batches: Option, seed: u64, global_update: &mut usize, planned_updates: usize, device: &Device, ) -> Result { let mut aggregate = FoundationEpochLosses::default(); let mut stage_loss_sums = [0.0f64; 5]; // Log-only gradient diagnostics per stage: sum of pre-clip norms, max norm, clipped count. let mut stage_grad = [(0.0f64, 0.0f64, 0usize); 5]; let cycles = research_curriculum_cycles(indices.len(), config).min(max_batches.unwrap_or(usize::MAX)); let consensus_order = mobility_consensus.map(|supervision| { deterministic_example_order( &supervision.examples, cycles.saturating_mul(config.mobility_batch_size.unwrap_or(config.batch_size)), seed, config.seed, ) }); for batch_index in 0..cycles { let (first_indices, second_indices) = research_property_batches(indices, batch_index, config); let property_batches = [first_indices, second_indices] .map(|chunk| { chunk .iter() .map(|&index| { records.get(index).cloned().ok_or_else(|| { anyhow::anyhow!( "foundation training record index {index} is out of bounds" ) }) }) .collect::>>() }) .into_iter() .collect::>>()?; let cycle_seed = seed ^ (batch_index as u64).wrapping_mul(0x9e37_79b9_7f4a_7c15); for (stage_index, stage) in [ FoundationCurriculumStage::Property, FoundationCurriculumStage::Property, FoundationCurriculumStage::Representation, FoundationCurriculumStage::Mobility, FoundationCurriculumStage::Inverse, ] .into_iter() .enumerate() { let stage_seed = cycle_seed ^ (stage_index as u64).wrapping_mul(0xd1b5_4a32_d192_ed03); let staged = staged_training_config(config, stage); let stage_collator = match stage { FoundationCurriculumStage::Representation => collator, _ => model.peptide_collator(), }; let loss = if matches!(stage, FoundationCurriculumStage::Mobility) && mobility_consensus.is_some() { let supervision = mobility_consensus.expect("checked presence"); let order = consensus_order.as_ref().expect("consensus order exists"); let mobility_batch = config.mobility_batch_size.unwrap_or(config.batch_size); let start = batch_index.saturating_mul(mobility_batch); let end = (start + mobility_batch).min(order.len()); let examples = order[start..end] .iter() .map(|&index| supervision.examples[index].clone()) .collect::>(); mobility_consensus_batch_loss( model, records, &examples, staged.forward_loss_weights.ccs, staged.conformation_proxy_weight, stage_seed, device, )? } else { let selected = match stage { FoundationCurriculumStage::Property => &property_batches[stage_index], FoundationCurriculumStage::Representation => &property_batches[0], FoundationCurriculumStage::Inverse => &property_batches[1], FoundationCurriculumStage::Mobility => &property_batches[0], }; // Historical v0.52 used distinct property batches but smaller // representation batches; use a bounded inverse subset as well. let selected = if matches!(stage, FoundationCurriculumStage::Property) { selected.as_slice() } else { &selected[..selected.len().min(config.batch_size)] }; batch_loss( model, selected, stage_collator, &staged, stage_seed, true, device, )? }; let lr = scheduled_learning_rate(config, *global_update, planned_updates); if config .term_gradient_probe_every .is_some_and(|every| batch_index % every == 0) && loss.terms.len() > 1 { let opt: &FoundationAdamW = match stage { FoundationCurriculumStage::Inverse => inverse_optimizer, _ => peptide_optimizer, }; probe_term_gradients(opt, &loss, stage_index, batch_index, *global_update)?; } let step = match stage { FoundationCurriculumStage::Inverse => { inverse_optimizer.set_learning_rate(lr)?; inverse_optimizer.backward_step(&loss.total, config.max_gradient_norm)? } _ => { peptide_optimizer.set_learning_rate(lr)?; peptide_optimizer.backward_step(&loss.total, config.max_gradient_norm)? } }; *global_update = (*global_update).saturating_add(1); let grad = &mut stage_grad[stage_index]; grad.0 += step.gradient_norm; grad.1 = grad.1.max(step.gradient_norm); grad.2 += usize::from(step.gradient_scale < 1.0); stage_loss_sums[stage_index] += f64::from(loss.scalars.total); aggregate.add(loss.scalars); } } if aggregate.batches == 0 { anyhow::bail!("foundation research-curriculum epoch produced no updates"); } let n = cycles as f64; println!("foundation_stage_losses\tproperty_a={:.6}\tproperty_b={:.6}\trepresentation={:.6}\tmobility={:.6}\tinverse={:.6}", stage_loss_sums[0] / n, stage_loss_sums[1] / n, stage_loss_sums[2] / n, stage_loss_sums[3] / n, stage_loss_sums[4] / n); let mut line = String::from("foundation_stage_gradients"); for (name, (sum, max, clipped)) in [ "property_a", "property_b", "representation", "mobility", "inverse", ] .iter() .zip(stage_grad) { line.push_str(&format!( "\t{name}_mean_norm={:.6}\t{name}_max_norm={max:.6}\t{name}_clip_fraction={:.6}", sum / n, clipped as f64 / n )); } println!("{line}"); Ok(aggregate.means()) } /// Opt-in diagnostic. One extra backward pass per weighted loss term; parameters and /// optimizer state are not changed. The norms are over the stage optimizer's variables, /// before clipping. The cosine is with the gradient of the stage total. fn probe_term_gradients( optimizer: &FoundationAdamW, loss: &BatchLoss, stage_index: usize, cycle: usize, update: usize, ) -> Result<()> { let total = loss.total.backward()?; let total_norm = optimizer.gradient_norm(&total)?; let total_value = loss.total.to_scalar::()?; let mut term_sum = 0.0f32; for (_, term) in &loss.terms { term_sum += term.to_scalar::()?; } // Must be about 0; otherwise the term list misses part of the stage loss. let sum_residual = total_value - term_sum; let mut line = format!( "foundation_term_gradients\tstage={stage_index}\tcycle={cycle}\tupdate={update}\ttotal_value={total_value:.6}\tsum_residual={sum_residual:.3e}\ttotal_norm={total_norm:.6}" ); for (name, term) in &loss.terms { let grads = term.backward()?; let norm = optimizer.gradient_norm(&grads)?; let cosine = optimizer .gradient_cosine(&grads, &total)? .map_or_else(|| "NA".to_string(), |c| format!("{c:.6}")); line.push_str(&format!( "\t{name}_value={:.6}\t{name}_norm={norm:.6}\t{name}_cos_total={cosine}", term.to_scalar::()? )); } println!("{line}"); Ok(()) } fn mobility_consensus_batch_loss( model: &FoundationModel, records: &[FoundationTrainingRecord], examples: &[MobilityConsensusExample], ccs_weight: f64, conformation_proxy_weight: f64, seed: u64, device: &Device, ) -> Result { if examples.is_empty() { anyhow::bail!("mobility consensus batch has no examples"); } let mut owned = Vec::with_capacity(examples.len()); let mut ccs_values = Vec::with_capacity(examples.len()); let mut weights = Vec::with_capacity(examples.len()); for example in examples { let record = records .get(example.representative_index) .ok_or_else(|| anyhow::anyhow!("consensus representative outside corpus"))?; let charge = record .context .charge .ok_or_else(|| anyhow::anyhow!("consensus representative missing charge"))?; let mz = record .context .precursor_mz .ok_or_else(|| anyhow::anyhow!("consensus representative missing precursor m/z"))?; let factor = bruker_ccs_factor_from_values(charge, mz) .ok_or_else(|| anyhow::anyhow!("invalid Bruker CCS conversion physics"))?; let ccs = f64::from(example.target_mobility) * factor; if !ccs.is_finite() || ccs <= 0.0 || !example.weight.is_finite() || example.weight <= 0.0 { anyhow::bail!("invalid mobility consensus target/weight"); } ccs_values.push(ccs as f32); weights.push(example.weight); owned.push(record.clone()); } let input = model.peptide_collator().collate(&owned, device, seed)?; let (output, auxiliaries) = model.forward_properties_with_auxiliaries_t(&input.input, &input.context, true)?; // The production specialist currently predicts standardized CCS, not native mobility. // Convert source-aware mobility consensus to native CCS *before* TRAIN-fitted // CCS standardization. This preserves a physically meaningful supervision target. let native_ccs = Tensor::from_vec(ccs_values, (examples.len(), 1), device)?; let target = model .target_normalization() .ccs .normalize_tensor(&native_ccs)?; let weight = Tensor::from_vec(weights, (examples.len(), 1), device)?; let delta = (&output.ccs - &target)?; let weight_sum = weight.sum_all()?.clamp(1.0e-6, f64::INFINITY)?; let mse = delta .sqr()? .broadcast_mul(&weight)? .sum_all()? .broadcast_div(&weight_sum)?; // Historical-style normalized pseudo-Huber alongside a lower-weight MSE. let robust = delta .affine(2.0, 0.0)? .sqr()? .affine(1.0, 1.0)? .sqrt()? .affine(0.25, -0.25)?; let pseudo_huber = robust .broadcast_mul(&weight)? .sum_all()? .broadcast_div(&weight_sum)?; let mut combined = (pseudo_huber + mse.affine(0.25, 0.0)?)?.affine(ccs_weight, 0.0)?; let mut terms = vec![("ccs", combined.clone())]; if conformation_proxy_weight > 0.0 { let proxy = auxiliaries .as_ref() .and_then(|aux| aux.conformation_proxy.as_ref()) .ok_or_else(|| { anyhow::anyhow!("conformation auxiliary requested without enabled mobility head") })?; let target = conformation_proxy_targets_v0520( &owned, model.config().peptide.max_sequence_len, device, )?; let mask = Tensor::ones(target.dims().to_vec(), candle_core::DType::F32, device)?; let loss = masked_mse(proxy, &target, &mask)?; let weighted = loss.affine(conformation_proxy_weight, 0.0)?; terms.push(("conformation", weighted.clone())); combined = (combined + weighted)?; } let scalar = combined.to_scalar::()?; Ok(BatchLoss { total: combined, scalars: BatchLossScalars { total: scalar, forward: scalar, ..BatchLossScalars::default() }, terms, }) } fn write_mobility_consensus_provenance( output_dir: &Path, supervision: &MobilityConsensusSupervision, ) -> Result<()> { let mut detail = String::from("source\traw_records\tshared_identities\taffine_intercept\taffine_slope\tresidual_mae\treliability\n"); for (source, entry) in &supervision.source_supervision { use std::fmt::Write as _; let (intercept, slope) = entry.affine_parameters(); writeln!( &mut detail, "{}\t{}\t{}\t{:.8}\t{:.8}\t{:.8}\t{:.8}", source, entry.raw_records, entry.shared_identities, intercept, slope, entry.residual_mae, entry.reliability )?; } fs::write(output_dir.join("mobility_source_reliability.tsv"), detail)?; fs::write(output_dir.join("mobility_consensus_summary.tsv"), format!( "metric\tvalue\ntrain_valid_mobility_records\t{}\ntrain_consensus_examples\t{}\nmultisource_examples\t{}\nsingleton_examples\t{}\nmean_weight\t{:.8}\nmean_adjusted_delta\t{:.8}\nfingerprint\tfnv1a64:{:016x}\n", supervision.raw_records, supervision.examples.len(), supervision.multisource_examples, supervision.singleton_examples, supervision.mean_weight, supervision.mean_abs_adjusted_delta_to_consensus, supervision.fingerprint, ))?; Ok(()) } fn staged_training_config( base: &FoundationTrainingConfig, stage: FoundationCurriculumStage, ) -> FoundationTrainingConfig { let mut config = base.clone(); config.forward_loss_weights = FoundationLossWeights { rt: 0.0, ccs: 0.0, ms2: 0.0, masked_residue: 0.0, chemistry: 0.0, contrastive: 0.0, }; config.peptide_contrastive_weight = 0.0; config.inverse_weight = 0.0; config.inverse_ptm_mass_weight = 0.0; config.cross_modal_alignment_weight = 0.0; config.pair_auxiliary_weight = 0.0; config.chemistry_summary_weight = 0.0; config.conformation_proxy_weight = 0.0; match stage { FoundationCurriculumStage::Property => { config.forward_loss_weights.rt = base.forward_loss_weights.rt; config.forward_loss_weights.ms2 = base.forward_loss_weights.ms2; config.pair_auxiliary_weight = base.pair_auxiliary_weight; config.chemistry_summary_weight = base.chemistry_summary_weight; } FoundationCurriculumStage::Representation => { config.forward_loss_weights.masked_residue = base.forward_loss_weights.masked_residue.max(0.15); config.forward_loss_weights.chemistry = base.forward_loss_weights.chemistry.max(0.10); config.peptide_contrastive_weight = base.peptide_contrastive_weight.max(0.05); config.pair_auxiliary_weight = base.pair_auxiliary_weight; config.chemistry_summary_weight = base.chemistry_summary_weight; } FoundationCurriculumStage::Mobility => { config.forward_loss_weights.ccs = base.forward_loss_weights.ccs; config.conformation_proxy_weight = base.conformation_proxy_weight; } FoundationCurriculumStage::Inverse => { config.inverse_weight = base.inverse_weight; config.inverse_ptm_mass_weight = base.inverse_ptm_mass_weight; config.cross_modal_alignment_weight = base.cross_modal_alignment_weight; } } config } fn scheduled_learning_rate( config: &FoundationTrainingConfig, update: usize, planned_updates: usize, ) -> f64 { if config.strategy == FoundationTrainingStrategy::Joint && config.warmup_steps == 0 && (config.min_learning_rate_ratio - 1.0).abs() <= f64::EPSILON { return config.learning_rate; } if config.warmup_steps > 0 && update < config.warmup_steps { return config.learning_rate * ((update + 1) as f64 / config.warmup_steps as f64); } let decay_start = config.warmup_steps.min(planned_updates); let decay_steps = planned_updates.saturating_sub(decay_start).max(1); let progress = update.saturating_sub(decay_start).min(decay_steps) as f64 / decay_steps as f64; let cosine = 0.5 * (1.0 + (std::f64::consts::PI * progress).cos()); let ratio = config.min_learning_rate_ratio + (1.0 - config.min_learning_rate_ratio) * cosine; config.learning_rate * ratio } fn batch_loss( model: &FoundationModel, records: &[FoundationTrainingRecord], collator: &FoundationCollator, config: &FoundationTrainingConfig, seed: u64, train: bool, device: &Device, ) -> Result { let views = collator.collate_views(records, device, seed)?; let use_auxiliary = train && (config.pair_auxiliary_weight > 0.0 || config.chemistry_summary_weight > 0.0 || config.conformation_proxy_weight > 0.0); let (first, auxiliaries) = if use_auxiliary { model.forward_properties_with_auxiliaries_t( &views.first.input, &views.first.context, train, )? } else { ( model.forward_properties_t(&views.first.input, &views.first.context, train)?, None, ) }; let targets = normalized_targets(&views.first.targets, model.target_normalization())?; let forward = multi_task_loss_with_ms2_config( &first, &targets, config.forward_loss_weights, config.ms2_loss, )?; let mut total = forward.total.clone(); // Weighted terms in the same order they are summed. Forward terms use the configured // task weights, exactly as in multi_task_loss_with_ms2_config. let mut terms: Vec<(&'static str, Tensor)> = Vec::new(); let w = config.forward_loss_weights; for (name, value, weight) in [ ("rt", &forward.rt, w.rt), ("ccs", &forward.ccs, w.ccs), ("ms2", &forward.ms2, w.ms2), ("masked_residue", &forward.masked_residue, w.masked_residue), ("chemistry", &forward.chemistry, w.chemistry), ] { if let Some(value) = value { if weight != 0.0 { terms.push((name, value.affine(weight, 0.0)?)); } } } if use_auxiliary { let aux = auxiliaries.as_ref().ok_or_else(|| { anyhow::anyhow!("auxiliary losses configured but auxiliary model heads are disabled") })?; let sequence_len = aux.pair_logits.dims4()?.1; if config.pair_auxiliary_weight > 0.0 { let (target, mask) = pair_interaction_targets(records, sequence_len, device)?; let loss = masked_bce_with_logits(&aux.pair_logits, &target, &mask)?; let weighted = loss.affine(config.pair_auxiliary_weight, 0.0)?; terms.push(("pair_aux", weighted.clone())); total = (total + weighted)?; } if config.chemistry_summary_weight > 0.0 { let target = chemistry_summary_targets(records, sequence_len, device)?; let mask = Tensor::ones(target.dims().to_vec(), candle_core::DType::F32, device)?; let loss = masked_mse(&aux.chemistry_summary, &target, &mask)?; let weighted = loss.affine(config.chemistry_summary_weight, 0.0)?; terms.push(("chem_summary", weighted.clone())); total = (total + weighted)?; } if config.conformation_proxy_weight > 0.0 { let prediction = aux.conformation_proxy.as_ref().ok_or_else(|| { anyhow::anyhow!( "conformation auxiliary enabled without the native mobility specialist" ) })?; let target = conformation_proxy_targets_v0520(records, sequence_len, device)?; let mask = Tensor::ones(target.dims().to_vec(), candle_core::DType::F32, device)?; let loss = masked_mse(prediction, &target, &mask)?; let weighted = loss.affine(config.conformation_proxy_weight, 0.0)?; terms.push(("conformation", weighted.clone())); total = (total + weighted)?; } } let peptide_contrastive = if records.len() > 1 && config.peptide_contrastive_weight > 0.0 { let second = model.forward_properties_t(&views.second.input, &views.second.context, train)?; let loss = contrastive_info_nce_loss( &first.contrastive_projection, &second.contrastive_projection, config.contrastive_temperature, )?; let weighted = loss.affine(config.peptide_contrastive_weight, 0.0)?; terms.push(("contrastive", weighted.clone())); total = (total + weighted)?; Some(loss) } else { None }; let inverse_records = records .iter() .filter(|record| FoundationSpectrum::from_training_record(record).is_some()) .cloned() .collect::>(); let mut inverse_loss = None; let mut inverse_ptm_mass_loss = None; let mut cross_modal_alignment = None; if !inverse_records.is_empty() && config.inverse_weight > 0.0 { let spectra = inverse_records .iter() .map(|record| FoundationSpectrum::from_training_record(record).unwrap()) .collect::>(); let spectrum_batch = model.spectrum_collator().collate(&spectra, device)?; let peptides = inverse_records .iter() .map(|record| record.peptidoform.clone()) .collect::>(); let causal_batch = model.causal_collator().collate(&peptides, device)?; let clean = model .peptide_collator() .collate(&inverse_records, device, 0)?; let inverse_output = model.inverse_forward_t(&causal_batch, &spectrum_batch, &clean.context, train)?; let token_loss = super::causal::foundation_causal_next_token_loss(&inverse_output, &causal_batch)?; let weighted = token_loss.affine(config.inverse_weight, 0.0)?; terms.push(("inverse", weighted.clone())); total = (total + weighted)?; inverse_loss = Some(token_loss); if config.inverse_ptm_mass_weight > 0.0 { let mass_loss = model.inverse_mass_loss(&inverse_output, &causal_batch)?; let weighted = mass_loss.affine(config.inverse_ptm_mass_weight, 0.0)?; terms.push(("inverse_ptm_mass", weighted.clone())); total = (total + weighted)?; inverse_ptm_mass_loss = Some(mass_loss); } if inverse_records.len() > 1 && config.cross_modal_alignment_weight > 0.0 { let peptide_output = model.forward_properties_t(&clean.input, &clean.context, train)?; let spectrum_projection = model.project_spectrum(&inverse_output.spectrum_embedding)?; let alignment = contrastive_info_nce_loss( &peptide_output.contrastive_projection, &spectrum_projection, config.contrastive_temperature, )?; let weighted = alignment.affine(config.cross_modal_alignment_weight, 0.0)?; terms.push(("alignment", weighted.clone())); total = (total + weighted)?; cross_modal_alignment = Some(alignment); } } let scalars = BatchLossScalars { total: total.to_scalar::()?, forward: forward.total.to_scalar::()?, inverse: optional_scalar(inverse_loss.as_ref())?, inverse_ptm_mass: optional_scalar(inverse_ptm_mass_loss.as_ref())?, peptide_contrastive: optional_scalar(peptide_contrastive.as_ref())?, cross_modal_alignment: optional_scalar(cross_modal_alignment.as_ref())?, }; Ok(BatchLoss { total, scalars, terms, }) } fn normalized_targets( targets: &FoundationTargets, normalization: FoundationTargetNormalizationConfig, ) -> Result { let mut normalized = targets.clone(); normalized.rt = targets .rt .as_ref() .map(|value| normalization.rt.normalize_tensor(value)) .transpose()?; normalized.ccs = targets .ccs .as_ref() .map(|value| normalization.ccs.normalize_tensor(value)) .transpose()?; Ok(normalized) } fn optional_scalar(value: Option<&Tensor>) -> Result> { Ok(value.map(|tensor| tensor.to_scalar::()).transpose()?) } fn partition_indices( benchmark: &FoundationBenchmarkManifest, partition: FoundationPartition, ) -> Vec { benchmark .entries .iter() .filter(|entry| entry.partition == partition) .map(|entry| entry.record_index) .collect() } fn shuffle_indices(values: &mut [usize], seed: u64) { if values.len() < 2 { return; } let mut state = seed.max(1); for index in (1..values.len()).rev() { state ^= state << 13; state ^= state >> 7; state ^= state << 17; let swap = (state as usize) % (index + 1); values.swap(index, swap); } } #[derive(Debug, Default)] struct EvaluationAccumulator { records: usize, rt_abs_error_sum: f64, rt_squared_error_sum: f64, rt_count: usize, ccs_abs_error_sum: f64, ccs_squared_error_sum: f64, ccs_count: usize, ms2_squared_error_sum: f64, ms2_point_count: usize, ms2_cosine_sum: f64, ms2_spectral_angle_sum: f64, ms2_pearson_sum: f64, ms2_pearson_count: usize, ms2_spectrum_count: usize, inverse_nll_sum: f64, inverse_sequence_count: usize, inverse_ptm_scaled_squared_error_sum: f64, inverse_ptm_site_count: usize, retrieval_top1_count: usize, retrieval_reciprocal_rank_sum: f64, retrieval_queries: usize, generation_attempts: usize, generation_successes: usize, generation_exact: usize, generation_il_equivalent: usize, generation_length_sum: usize, } impl EvaluationAccumulator { fn finish(self) -> FoundationEvaluationMetrics { FoundationEvaluationMetrics { records: self.records, rt_count: self.rt_count, rt_mae_native: ratio(self.rt_abs_error_sum, self.rt_count), rt_rmse_native: ratio(self.rt_squared_error_sum, self.rt_count).map(f64::sqrt), ccs_count: self.ccs_count, ccs_mae: ratio(self.ccs_abs_error_sum, self.ccs_count), ccs_rmse: ratio(self.ccs_squared_error_sum, self.ccs_count).map(f64::sqrt), ms2_point_count: self.ms2_point_count, ms2_rmse: ratio(self.ms2_squared_error_sum, self.ms2_point_count).map(f64::sqrt), ms2_spectrum_count: self.ms2_spectrum_count, ms2_cosine_similarity: ratio(self.ms2_cosine_sum, self.ms2_spectrum_count), ms2_spectral_angle: ratio(self.ms2_spectral_angle_sum, self.ms2_spectrum_count), ms2_pearson: ratio(self.ms2_pearson_sum, self.ms2_pearson_count), inverse_sequence_count: self.inverse_sequence_count, inverse_mean_nll: ratio(self.inverse_nll_sum, self.inverse_sequence_count), inverse_ptm_site_count: self.inverse_ptm_site_count, inverse_ptm_mass_rmse_da: ratio( self.inverse_ptm_scaled_squared_error_sum, self.inverse_ptm_site_count, ) .map(|mse| mse.sqrt() * f64::from(FOUNDATION_OPEN_PTM_MASS_SCALE_DA)), retrieval_queries: self.retrieval_queries, retrieval_top1_accuracy: ratio( self.retrieval_top1_count as f64, self.retrieval_queries, ), retrieval_mrr: ratio(self.retrieval_reciprocal_rank_sum, self.retrieval_queries), generation_attempts: self.generation_attempts, generation_success_rate: ratio( self.generation_successes as f64, self.generation_attempts, ), generation_sequence_exact_rate: ratio( self.generation_exact as f64, self.generation_attempts, ), generation_sequence_il_equivalent_rate: ratio( self.generation_il_equivalent as f64, self.generation_attempts, ), generation_mean_length: ratio( self.generation_length_sum as f64, self.generation_successes, ), } } } fn evaluate_partition( model: &FoundationModel, records: &[FoundationTrainingRecord], indices: &[usize], batch_size: usize, max_batches: Option, inverse_limit: usize, generation_limit: usize, ms2_cosine_epsilon: f64, device: &Device, ) -> Result { let mut aggregate = EvaluationAccumulator::default(); // Forward metrics use the same bounded Validation slice as early-stopping loss, // but with clean (uncorrupted) peptide inputs and native-unit predictions. for chunk in indices .chunks(batch_size) .take(max_batches.unwrap_or(usize::MAX)) { let owned = collect_records(records, chunk, "evaluation")?; if owned.is_empty() { continue; } aggregate.records += owned.len(); let predictions = model.predict_records(&owned)?; for (record, prediction) in owned.iter().zip(&predictions) { accumulate_forward_metrics( &mut aggregate, record, prediction, ms2_cosine_epsilon, model.config().rt_objective, ); } } if aggregate.records == 0 { anyhow::bail!("foundation evaluation produced no records"); } // Inverse diagnostics deliberately scan the whole Validation assignment for // spectrum-bearing rows before applying their own cap. This prevents a // property-heavy prefix of Validation from silently producing zero inverse // metrics, while still keeping the engineering/scientific evaluation bounded. let inverse_indices = indices .iter() .copied() .filter(|&index| { records .get(index) .and_then(FoundationSpectrum::from_training_record) .is_some() }) .take(inverse_limit) .collect::>(); for chunk in inverse_indices.chunks(batch_size) { let inverse_records = collect_records(records, chunk, "inverse evaluation")?; if inverse_records.is_empty() { continue; } let spectra = inverse_records .iter() .map(|record| FoundationSpectrum::from_training_record(record).unwrap()) .collect::>(); let spectrum_batch = model.spectrum_collator().collate(&spectra, device)?; let peptides = inverse_records .iter() .map(|record| record.peptidoform.clone()) .collect::>(); let causal_batch = model.causal_collator().collate(&peptides, device)?; let clean = model .peptide_collator() .collate(&inverse_records, device, 0)?; let inverse_output = model.inverse_forward_t(&causal_batch, &spectrum_batch, &clean.context, false)?; let sequence_nlls = foundation_causal_sequence_mean_nlls(&inverse_output, &causal_batch)? .to_vec1::()?; for value in sequence_nlls { if value.is_finite() { aggregate.inverse_nll_sum += f64::from(value); aggregate.inverse_sequence_count += 1; } } let ptm_sites = causal_batch .target_modification_mask .sum_all()? .to_scalar::()?; if ptm_sites > 0.0 && ptm_sites.is_finite() { let scaled_mse = model .inverse_mass_loss(&inverse_output, &causal_batch)? .to_scalar::()?; if scaled_mse.is_finite() { aggregate.inverse_ptm_scaled_squared_error_sum += f64::from(scaled_mse) * f64::from(ptm_sites); aggregate.inverse_ptm_site_count += ptm_sites.round() as usize; } } if inverse_records.len() > 1 { let peptide_output = model.forward_properties_t(&clean.input, &clean.context, false)?; let spectrum_projection = model.project_spectrum(&inverse_output.spectrum_embedding)?; accumulate_retrieval_metrics( &mut aggregate, &peptide_output.contrastive_projection, &spectrum_projection, )?; } for record in &inverse_records { if aggregate.generation_attempts >= generation_limit { break; } aggregate.generation_attempts += 1; if let Ok(generated) = model.generate_peptide(record) { aggregate.generation_successes += 1; aggregate.generation_length_sum += generated.sequence.chars().count(); if generated.sequence == record.peptidoform.sequence { aggregate.generation_exact += 1; } if il_equivalent(&generated.sequence, &record.peptidoform.sequence) { aggregate.generation_il_equivalent += 1; } } } } Ok(aggregate.finish()) } fn collect_records( records: &[FoundationTrainingRecord], indices: &[usize], context: &str, ) -> Result> { indices .iter() .map(|&index| { records.get(index).cloned().ok_or_else(|| { anyhow::anyhow!("foundation {context} record index {index} is out of bounds") }) }) .collect() } fn accumulate_forward_metrics( aggregate: &mut EvaluationAccumulator, record: &FoundationTrainingRecord, prediction: &crate::models::model_interface::PredictionOutput, cosine_epsilon: f64, rt_objective: RetentionTimeObjective, ) { if let (Some(target), Some(predicted)) = ( rt_objective .target(&record.retention_time) .filter(|value| value.is_finite()), prediction.rt.filter(|value| value.is_finite()), ) { let error = f64::from(predicted - target); aggregate.rt_abs_error_sum += error.abs(); aggregate.rt_squared_error_sum += error * error; aggregate.rt_count += 1; } if let (Some(target), Some(predicted)) = ( record.ccs.filter(|value| value.is_finite()), prediction.ccs.filter(|value| value.is_finite()), ) { let error = f64::from(predicted - target); aggregate.ccs_abs_error_sum += error.abs(); aggregate.ccs_squared_error_sum += error * error; aggregate.ccs_count += 1; } let Some(predicted) = prediction.ms2.as_ref() else { return; }; let Some(channels) = predicted.first().map(Vec::len) else { return; }; if channels == 0 { return; } let cleavage = predicted.len(); let mut target = vec![vec![None::; channels]; cleavage]; for fragment in &record.fragments { if fragment.cleavage_index < cleavage && fragment.channel < channels && fragment.intensity.is_finite() { target[fragment.cleavage_index][fragment.channel] = Some(fragment.intensity); } } let mut dot = 0.0f64; let mut prediction_norm = 0.0f64; let mut target_norm = 0.0f64; let mut points = 0usize; let mut pred_values = Vec::new(); let mut target_values = Vec::new(); for cleavage_index in 0..cleavage { for channel in 0..channels { let Some(target_value) = target[cleavage_index][channel] else { continue; }; let predicted_value = predicted[cleavage_index][channel]; if !predicted_value.is_finite() { continue; } let p = f64::from(predicted_value); let t = f64::from(target_value); let error = p - t; aggregate.ms2_squared_error_sum += error * error; aggregate.ms2_point_count += 1; dot += p * t; prediction_norm += p * p; target_norm += t * t; pred_values.push(p); target_values.push(t); points += 1; } } if points > 0 { let denominator = (prediction_norm + cosine_epsilon).sqrt() * (target_norm + cosine_epsilon).sqrt(); if denominator > 0.0 && denominator.is_finite() { let cosine = (dot / denominator).clamp(-1.0, 1.0); if cosine.is_finite() { aggregate.ms2_cosine_sum += cosine; aggregate.ms2_spectral_angle_sum += 1.0 - (2.0 / std::f64::consts::PI) * cosine.acos(); aggregate.ms2_spectrum_count += 1; if let Some(pearson) = pearson_correlation(&pred_values, &target_values) { aggregate.ms2_pearson_sum += pearson; aggregate.ms2_pearson_count += 1; } } } } } fn pearson_correlation(left: &[f64], right: &[f64]) -> Option { if left.len() != right.len() || left.len() < 2 { return None; } let n = left.len() as f64; let left_mean = left.iter().sum::() / n; let right_mean = right.iter().sum::() / n; let mut covariance = 0.0; let mut left_variance = 0.0; let mut right_variance = 0.0; for (&a, &b) in left.iter().zip(right) { let da = a - left_mean; let db = b - right_mean; covariance += da * db; left_variance += da * da; right_variance += db * db; } let denominator = (left_variance * right_variance).sqrt(); (denominator > 0.0 && denominator.is_finite()) .then(|| (covariance / denominator).clamp(-1.0, 1.0)) } fn accumulate_retrieval_metrics( aggregate: &mut EvaluationAccumulator, peptide_projection: &Tensor, spectrum_projection: &Tensor, ) -> Result<()> { let peptides = peptide_projection.to_vec2::()?; let spectra = spectrum_projection.to_vec2::()?; if peptides.len() != spectra.len() || peptides.len() < 2 { return Ok(()); } if peptides.first().map(Vec::len) != spectra.first().map(Vec::len) { anyhow::bail!("foundation cross-modal retrieval projection dimensions differ"); } let peptide_norm = l2_normalize_rows(&peptides); let spectrum_norm = l2_normalize_rows(&spectra); if peptide_norm.iter().any(Option::is_none) || spectrum_norm.iter().any(Option::is_none) { return Ok(()); } let peptide_norm = peptide_norm.into_iter().flatten().collect::>(); let spectrum_norm = spectrum_norm.into_iter().flatten().collect::>(); accumulate_retrieval_direction(aggregate, &peptide_norm, &spectrum_norm); accumulate_retrieval_direction(aggregate, &spectrum_norm, &peptide_norm); Ok(()) } fn accumulate_retrieval_direction( aggregate: &mut EvaluationAccumulator, queries: &[Vec], candidates: &[Vec], ) { for (index, query) in queries.iter().enumerate() { let positive = dot_product(query, &candidates[index]); let mut better = 0usize; for (candidate_index, candidate) in candidates.iter().enumerate() { if candidate_index != index && dot_product(query, candidate) > positive { better += 1; } } let rank = better + 1; aggregate.retrieval_queries += 1; if rank == 1 { aggregate.retrieval_top1_count += 1; } aggregate.retrieval_reciprocal_rank_sum += 1.0 / rank as f64; } } fn l2_normalize_rows(rows: &[Vec]) -> Vec>> { rows.iter() .map(|row| { let norm = row .iter() .map(|value| f64::from(*value).powi(2)) .sum::() .sqrt(); if !(norm > 0.0 && norm.is_finite()) { return None; } Some(row.iter().map(|value| f64::from(*value) / norm).collect()) }) .collect() } fn dot_product(left: &[f64], right: &[f64]) -> f64 { left.iter().zip(right).map(|(a, b)| a * b).sum() } fn il_equivalent(left: &str, right: &str) -> bool { left.chars().count() == right.chars().count() && left .chars() .zip(right.chars()) .all(|(a, b)| canonical_il(a) == canonical_il(b)) } fn canonical_il(residue: char) -> char { if matches!(residue, 'I' | 'L') { 'J' } else { residue } } fn ratio(sum: f64, count: usize) -> Option { if count > 0 { Some(sum / count as f64) } else { None } } fn optional_metric(value: Option) -> String { value .map(|value| format!("{value:.6}")) .unwrap_or_else(|| "NA".to_string()) } fn append_validation_metrics( path: &Path, epoch: usize, metrics: FoundationEvaluationMetrics, ) -> Result<()> { let mut file = OpenOptions::new().append(true).open(path)?; writeln!( file, "{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}", epoch, metrics.records, metrics.rt_count, tsv_metric(metrics.rt_mae_native), tsv_metric(metrics.rt_rmse_native), metrics.ccs_count, tsv_metric(metrics.ccs_mae), tsv_metric(metrics.ccs_rmse), metrics.ms2_point_count, tsv_metric(metrics.ms2_rmse), metrics.ms2_spectrum_count, tsv_metric(metrics.ms2_cosine_similarity), tsv_metric(metrics.ms2_spectral_angle), tsv_metric(metrics.ms2_pearson), metrics.inverse_sequence_count, tsv_metric(metrics.inverse_mean_nll), metrics.inverse_ptm_site_count, tsv_metric(metrics.inverse_ptm_mass_rmse_da), metrics.retrieval_queries, tsv_metric(metrics.retrieval_top1_accuracy), tsv_metric(metrics.retrieval_mrr), metrics.generation_attempts, tsv_metric(metrics.generation_success_rate), tsv_metric(metrics.generation_sequence_exact_rate), tsv_metric(metrics.generation_sequence_il_equivalent_rate), tsv_metric(metrics.generation_mean_length), )?; Ok(()) } fn tsv_metric(value: Option) -> String { value.map(|value| format!("{value:.8}")).unwrap_or_default() } fn append_history( path: &Path, epoch: usize, train: FoundationEpochLosses, validation: FoundationEpochLosses, ) -> Result<()> { let mut file = OpenOptions::new().append(true).open(path)?; writeln!( file, "{}\t{:.8}\t{:.8}\t{:.8}\t{:.8}\t{:.8}\t{:.8}\t{:.8}\t{:.8}\t{:.8}\t{:.8}\t{:.8}\t{:.8}", epoch, train.total, train.forward, train.inverse, train.inverse_ptm_mass, train.peptide_contrastive, train.cross_modal_alignment, validation.total, validation.forward, validation.inverse, validation.inverse_ptm_mass, validation.peptide_contrastive, validation.cross_modal_alignment, )?; Ok(()) } #[cfg(test)] mod tests { use super::*; #[test] fn default_training_config_enables_forward_inverse_and_alignment() { let config = FoundationTrainingConfig::default(); config.validate().unwrap(); assert!(config.inverse_weight > 0.0); assert!(config.cross_modal_alignment_weight > 0.0); assert!(config.peptide_contrastive_weight > 0.0); assert_eq!(config.inverse_evaluation_records, 256); assert_eq!(config.generation_evaluation_records, 16); } #[test] fn unified_defaults_use_research_curriculum_controls() { let config = FoundationTrainingConfig::default(); assert_eq!( config.strategy, FoundationTrainingStrategy::ResearchCurriculum ); assert_eq!(config.warmup_steps, 500); assert!((config.min_learning_rate_ratio - 0.10).abs() < 1e-12); assert_eq!(config.max_gradient_norm, Some(1.0)); assert!(config.mobility_consensus_supervision); assert!((scheduled_learning_rate(&config, 0, 10_000) - 4.0e-8).abs() < 1e-12); let final_lr = scheduled_learning_rate(&config, 9_999, 10_000); assert!(final_lr >= config.learning_rate * config.min_learning_rate_ratio); assert!(final_lr < config.learning_rate * 0.101); } #[test] fn historical_training_config_without_curriculum_fields_stays_joint_constant_lr() { let mut value = serde_yaml::to_value(FoundationTrainingConfig::default()).unwrap(); for key in [ "strategy", "warmup_steps", "min_learning_rate_ratio", "max_gradient_norm", "mobility_consensus_supervision", ] { value .as_mapping_mut() .unwrap() .remove(&serde_yaml::Value::String(key.into())); } let restored: FoundationTrainingConfig = serde_yaml::from_value(value).unwrap(); assert_eq!(restored.strategy, FoundationTrainingStrategy::Joint); assert_eq!(restored.warmup_steps, 0); assert_eq!(restored.min_learning_rate_ratio, 1.0); assert_eq!(restored.max_gradient_norm, None); assert!(!restored.mobility_consensus_supervision); } #[test] fn property_stage_terms_sum_to_total_value_and_gradient() { use super::super::config::FoundationConfig; use super::super::data::{FragmentTarget, RetentionTimeLabels, TrainingContext}; use super::super::featurize::PeptidoformInput; use super::super::normalization::FoundationRegressionNormalization; let mut model_config = FoundationModelConfig::default(); model_config.peptide = FoundationConfig { max_sequence_len: 16, graph_hidden_dim: 16, graph_layers: 1, model_dim: 32, num_attention_heads: 4, transformer_ff_dim: 64, transformer_layers: 1, contrastive_dim: 16, dropout: 0.0, ..model_config.peptide.clone() }; model_config.inverse.model_dim = 32; model_config.inverse.num_attention_heads = 4; model_config.inverse.feed_forward_dim = 64; model_config.inverse.spectrum_layers = 1; model_config.inverse.decoder_layers = 1; model_config.inverse.dropout = 0.0; let standardize = |mean: f64, sd: f64| FoundationRegressionNormalization { strategy: FoundationRegressionNormalizationStrategy::TrainStandardize, mean: Some(mean), standard_deviation: Some(sd), label_count: 10, ..FoundationRegressionNormalization::default() }; let normalization = FoundationTargetNormalizationConfig { rt: standardize(30.0, 10.0), ccs: standardize(450.0, 90.0), }; let model = FoundationModel::new(model_config, normalization, Vec::new(), Device::Cpu).unwrap(); let record = |sequence: &str, rt: f32| FoundationTrainingRecord { peptidoform: PeptidoformInput::unmodified(sequence), retention_time: RetentionTimeLabels { normalized: Some(rt), harmonized: None, observed_seconds: None, }, ccs: None, fragments: vec![ FragmentTarget { cleavage_index: 1, channel: 0, intensity: 1.0, product_mz: None, }, FragmentTarget { cleavage_index: 2, channel: 1, intensity: 0.4, product_mz: None, }, ], observed_spectrum_peaks: Vec::new(), context: TrainingContext { charge: Some(2), nce: Some(27.0), ..TrainingContext::default() }, run_id: None, }; let records = vec![record("PEPTIDEK", 25.0), record("ACDEFGHK", 41.0)]; let config = FoundationTrainingConfig::default(); let staged = staged_training_config(&config, FoundationCurriculumStage::Property); let loss = batch_loss( &model, &records, model.peptide_collator(), &staged, 7, true, &Device::Cpu, ) .unwrap(); let names: Vec<_> = loss.terms.iter().map(|(n, _)| *n).collect(); assert_eq!(names, ["rt", "ms2", "pair_aux", "chem_summary"]); let total = loss.total.to_scalar::().unwrap(); let sum: f32 = loss .terms .iter() .map(|(_, t)| t.to_scalar::().unwrap()) .sum(); assert!( (total - sum).abs() <= 1e-5 * total.abs().max(1.0), "{total} vs {sum}" ); // Gradients are linear: sum of per-term gradients equals the total gradient. let opt = FoundationAdamW::new_for_prefixes( model.varmap(), FoundationAdamWConfig::default(), &["peptide."], ) .unwrap(); let total_grads = loss.total.backward().unwrap(); let mut dot_sum = 0.0; for (_, term) in &loss.terms { dot_sum += opt .gradient_dot(&term.backward().unwrap(), &total_grads) .unwrap(); } let total_sq = opt.gradient_norm(&total_grads).unwrap().powi(2); assert!(total_sq > 0.0); assert!( (dot_sum - total_sq).abs() <= 1e-3 * total_sq, "sum of {dot_sum} vs |g_total|^2 {total_sq}" ); probe_term_gradients(&opt, &loss, 0, 0, 0).unwrap(); } #[test] fn historical_training_config_without_seed_and_epoch_fields_keeps_defaults() { let mut value = serde_yaml::to_value(FoundationTrainingConfig::default()).unwrap(); value .as_mapping_mut() .unwrap() .remove(&serde_yaml::Value::String("init_seed".into())); value .as_mapping_mut() .unwrap() .remove(&serde_yaml::Value::String("save_epoch_checkpoints".into())); value .as_mapping_mut() .unwrap() .remove(&serde_yaml::Value::String( "term_gradient_probe_every".into(), )); let restored: FoundationTrainingConfig = serde_yaml::from_value(value).unwrap(); assert_eq!(restored.init_seed, None); assert!(!restored.save_epoch_checkpoints); assert_eq!(restored.term_gradient_probe_every, None); } #[test] fn mobility_batch_size_defaults_off_and_rejects_zero() { let mut value = serde_yaml::to_value(FoundationTrainingConfig::default()).unwrap(); value .as_mapping_mut() .unwrap() .remove(&serde_yaml::Value::String("mobility_batch_size".into())); let restored: FoundationTrainingConfig = serde_yaml::from_value(value).unwrap(); assert_eq!(restored.mobility_batch_size, None); let mut config = FoundationTrainingConfig::default(); config.mobility_batch_size = Some(0); assert!(config.validate().is_err()); config.mobility_batch_size = Some(32); config.validate().unwrap(); } #[test] fn resume_from_epoch_defaults_off_validates_range_and_continues_lr_schedule() { let mut value = serde_yaml::to_value(FoundationTrainingConfig::default()).unwrap(); value .as_mapping_mut() .unwrap() .remove(&serde_yaml::Value::String("resume_from_epoch".into())); let restored: FoundationTrainingConfig = serde_yaml::from_value(value).unwrap(); assert_eq!(restored.resume_from_epoch, None); let mut config = FoundationTrainingConfig::default(); config.epochs = 60; for bad in [0, 60, 61] { config.resume_from_epoch = Some(bad); assert!(config.validate().is_err(), "resume {bad} must be rejected"); } config.resume_from_epoch = Some(40); config.validate().unwrap(); // The resumed run starts at update 40 * cycles * 5. The LR there is exactly the LR an // uninterrupted run uses at its first update of epoch 41 (same function, same index). let (cycles, planned) = (1024usize, 60 * 1024 * 5); let resumed_first = 40 * cycles * 5; let lr = scheduled_learning_rate(&config, resumed_first, planned); assert!(lr < config.learning_rate && lr > config.learning_rate * config.min_learning_rate_ratio); assert!(scheduled_learning_rate(&config, resumed_first - 1, planned) > lr); } #[test] fn historical_training_config_without_auxiliary_weights_stays_zero_weight() { let mut value = serde_yaml::to_value(FoundationTrainingConfig::default()).unwrap(); for field in [ "pair_auxiliary_weight", "chemistry_summary_weight", "conformation_proxy_weight", ] { value .as_mapping_mut() .unwrap() .remove(&serde_yaml::Value::String(field.into())); } let restored: FoundationTrainingConfig = serde_yaml::from_value(value).unwrap(); assert_eq!(restored.pair_auxiliary_weight, 0.0); assert_eq!(restored.chemistry_summary_weight, 0.0); assert_eq!(restored.conformation_proxy_weight, 0.0); } #[test] fn research_auxiliary_heads_must_be_enabled_for_nonzero_weights() { let mut config = FoundationTrainingConfig::default(); assert_eq!(config.pair_auxiliary_weight, 0.10); assert_eq!(config.chemistry_summary_weight, 0.10); assert_eq!(config.conformation_proxy_weight, 0.10); config.model.auxiliary_supervision_heads = false; assert!(config.validate().is_err()); } #[test] fn il_equivalence_collapses_only_isoleucine_and_leucine() { assert!(il_equivalent("PEPTIDE", "PEPTLDE")); assert!(il_equivalent("LLLL", "IIII")); assert!(!il_equivalent("PEPTIDE", "PEPTVDE")); assert!(!il_equivalent("PEPTIDE", "PEPTIDES")); } #[test] fn retrieval_metrics_reward_diagonal_pairs() { let peptide = Tensor::from_vec(vec![1.0f32, 0.0, 0.0, 1.0], (2, 2), &Device::Cpu).unwrap(); let spectrum = Tensor::from_vec(vec![0.9f32, 0.1, 0.1, 0.9], (2, 2), &Device::Cpu).unwrap(); let mut metrics = EvaluationAccumulator::default(); accumulate_retrieval_metrics(&mut metrics, &peptide, &spectrum).unwrap(); assert_eq!(metrics.retrieval_queries, 4); assert_eq!(metrics.retrieval_top1_count, 4); assert!((metrics.retrieval_reciprocal_rank_sum - 4.0).abs() < 1e-9); } #[test] fn research_property_batches_are_distinct_and_cover_expected_records() { let mut config = FoundationTrainingConfig::default(); config.batch_size = 8; config.property_batch_size = Some(16); config.validate().unwrap(); let indices: Vec<_> = (0..68).collect(); assert_eq!(research_curriculum_cycles(indices.len(), &config), 2); let (a, b) = research_property_batches(&indices, 1, &config); assert_eq!(a, &(32..48).collect::>()); assert_eq!(b, &(48..64).collect::>()); assert!(a.iter().all(|index| !b.contains(index))); assert_eq!(a.len(), 16); assert_eq!(b.len(), 16); config.property_batch_size = None; let (a, b) = research_property_batches(&indices, 1, &config); assert_eq!(a, b); // Old serialized configs keep the legacy path. assert_eq!(research_curriculum_cycles(indices.len(), &config), 9); config.property_batch_size = Some(4); assert!(config.validate().is_err()); let mut encoded = serde_yaml::to_value(FoundationTrainingConfig::default()).unwrap(); encoded .as_mapping_mut() .unwrap() .remove(&serde_yaml::Value::String("property_batch_size".into())); let legacy: FoundationTrainingConfig = serde_yaml::from_value(encoded).unwrap(); assert_eq!(legacy.property_batch_size, None); } #[test] fn training_shuffle_is_seeded_and_permutation_preserving() { let mut first = (0..16).collect::>(); let mut second = first.clone(); shuffle_indices(&mut first, 42); shuffle_indices(&mut second, 42); assert_eq!(first, second); first.sort_unstable(); assert_eq!(first, (0..16).collect::>()); } }