# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # http://www.apache.org/licenses/LICENSE-2.0 # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from ..device_type import DEVICE_TYPE_TORCH, clean_gpu_cache from ..fsdp2_launch import fsdp_version_from_env import hashlib import importlib import os import torch import re import tempfile import types import shutil import contextlib import contextvars import copy import threading as _threading from time import monotonic as _time_monotonic import functools import inspect from typing import Union from .mapper import ( INT_TO_FLOAT_MAPPER, FLOAT_TO_INT_MAPPER, MAP_TO_UNSLOTH_16bit, FLOAT_TO_FP8_BLOCK_MAPPER, FLOAT_TO_FP8_ROW_MAPPER, build_mappers, _add_with_lower, _add_lower_only, ) # The alias helpers a fetched mapper.py may call, resolved to the INSTALLED implementations: _get_new_mapper reads such calls as data, taking the table and the two literal strings out of the AST, so the fetched text never supplies behaviour. _MAPPER_HELPERS = { "_add_with_lower": _add_with_lower, "_add_lower_only": _add_lower_only, } from transformers import __version__ as transformers_version from unsloth.models._utils import TorchAOConfig from unsloth_zoo.utils import Version, get_quant_type from .packed_expert_loader import quantize_moe_experts_if_requested # noqa: F401 (re-exported) import gc import traceback as _traceback transformers_version = Version(transformers_version) SUPPORTS_FOURBIT = transformers_version >= Version("4.37") LOCAL_RANK_KEYS = ("LOCAL_RANK", "RANK") WORLD_SIZE_KEYS = ("WORLD_SIZE",) # Only the node-local one. torchrun documents RANK as "The global rank" and LOCAL_RANK as "The local rank", and torch.distributed.get_rank() is global too, so neither of those may index a device. LOCAL_RANK_ONLY_KEYS = ("LOCAL_RANK",) BAD_MAPPINGS = { "unsloth/Qwen3-32B-unsloth-bnb-4bit".lower(): "unsloth/Qwen3-32B-bnb-4bit", # 32B dynamic quant is way too big "unsloth/Qwen3-30B-A3B-unsloth-bnb-4bit".lower(): "unsloth/Qwen3-30B-A3B", # HF loads MoEs too slowly "unsloth/Qwen3-30B-A3B-bnb-4bit".lower(): "unsloth/Qwen3-30B-A3B", # We rather do it on the fly "unsloth/Qwen3-30B-A3B-Base-unsloth-bnb-4bit".lower(): "unsloth/Qwen3-30B-A3B-Base", # HF loads MoEs too slowly "unsloth/Qwen3-30B-A3B-Base-bnb-4bit".lower(): "unsloth/Qwen3-30B-A3B-Base", # We rather do it on the fly "unsloth/solar-10.7b-bnb-4bit".lower(): "upstage/SOLAR-10.7B-v1.0", "unsloth/Hermes-3-Llama-3.1-405B-bnb-4bit".lower(): "NousResearch/Hermes-3-Llama-3.1-405B", "unsloth/Pixtral-12B-2409-Base-bnb-4bit".lower(): "unsloth/Pixtral-12B-Base-2409-bnb-4bit", "unsloth/Llama-3.1-Tulu-3-8B-bnb-4bit".lower(): "unsloth/Llama-3.1-Tulu-3-8B", "unsloth/Llama-3.1-Tulu-3-70B-bnb-4bit".lower(): "allenai/Llama-3.1-Tulu-3-70B", "unsloth/OpenThinker-7B-unsloth-bnb-4bit".lower(): "open-thoughts/OpenThinker-7B", } def _get_torchao_fp8_config(fp8_mode): from unsloth_zoo.vllm_utils import _get_torchao_fp8_config as _impl return _impl(fp8_mode) def _get_env_int(keys): for key in keys: value = os.environ.get(key) if value is None: continue try: return int(value) except ValueError: continue return None def _infer_distributed_ranks(): if ( torch.distributed.is_available() and getattr(torch.distributed, "is_initialized", lambda: False)() ): try: return torch.distributed.get_rank(), torch.distributed.get_world_size() except Exception: pass return _get_env_int(LOCAL_RANK_KEYS), _get_env_int(WORLD_SIZE_KEYS) def is_distributed(): rank, world_size = _infer_distributed_ranks() return (world_size or 1) > 1 or (rank is not None and rank > 0) def _visible_device_count(): """Devices this process can actually address, 0 when the backend has none.""" try: module = getattr(torch, DEVICE_TYPE_TORCH, None) counter = getattr(module, "device_count", None) return int(counter()) if callable(counter) else 0 except Exception: return 0 def _infer_local_rank(rank): """The device index this rank owns *on this host*. `rank` is global: `torch.distributed.get_rank()` is "a unique identifier assigned to each process within a distributed process group ... 0 to world_size", and torchrun's RANK is documented as the global rank. Indexing a device with it names cuda:8 on the second node of a 2 x 8 job, which is an invalid ordinal, so read LOCAL_RANK first -- torchrun, accelerate launch and deepspeed all set it. The modulo is the last resort for a launcher that sets neither, and for a per-rank CUDA_VISIBLE_DEVICES where only one card is visible and the local rank still counts from the launcher's point of view. """ local_rank = _get_env_int(LOCAL_RANK_ONLY_KEYS) if local_rank is None: local_rank = 0 if rank is None else rank if local_rank < 0: return 0 count = _visible_device_count() if count > 0 and local_rank >= count: local_rank = local_rank % count return local_rank _TRUTHY = ("1", "true", "yes", "on") def fsdp_version_requested(): """The FSDP version this process trains with: 0 for none, else 1 or 2. A built Accelerator is authoritative; before one exists (model load, patch_peft_model) the launcher env is the only signal (`fsdp_version_from_env`: `accelerate launch` with an FSDP config, FSDP_VERSION by hand, or UNSLOTH_FSDP2=1 under torchrun). Every rank of a job sees the same env and the same Accelerator config, so every rank gets the same answer. `TrainingArguments(fsdp = ...)` alone is invisible until the Trainer builds its Accelerator; the Trainer re-checks then (`_unsloth_trainer_init`). """ try: from accelerate.state import AcceleratorState shared = AcceleratorState._shared_state plugin = shared.get("fsdp_plugin", None) if plugin is not None: return int(getattr(plugin, "fsdp_version", 1) or 1) distributed_type = shared.get("distributed_type", None) if distributed_type is not None and "FSDP" in str(distributed_type).upper(): return fsdp_version_from_env() or 1 except Exception: pass return fsdp_version_from_env() def fsdp_will_wrap(): """True when this process will hand the model to FSDP (unsloth#409). patch_peft_model runs before the Trainer builds its Accelerator, so the launcher env (`accelerate launch` exports ACCELERATE_USE_FSDP + FSDP_VERSION for FSDP configs, neither for DDP) is usually the only signal. `UNSLOTH_FORCE_FUSED_LORA=1` opts back in. """ if os.environ.get("UNSLOTH_FORCE_FUSED_LORA", "0") == "1": return False return fsdp_version_requested() > 0 def fsdp2_requested(): """True when this process runs under FSDP2: `accelerate launch` exports FSDP_VERSION=2, UNSLOTH_FSDP2=1 under torchrun, or an Accelerator built already records its fsdp_plugin.""" return fsdp_version_requested() == 2 def fsdp2_keeps_fused_lora(): """Load-time half of the fused LoRA gate under FSDP. FSDP1 hands the fused kernels flat shard views (unsloth#409), so they are declined. FSDP2 unshards a block's parameters in its pre-forward hook and the kernels run inside the block's forward, so with one FSDP unit per decoder block they read whole weights; the wrap accelerate actually builds is checked once it exists (`unsloth.models.fsdp2`), which declines them for a root-only or finer wrap. `UNSLOTH_FSDP2_FUSED_LORA=0` declines them under FSDP2 as well. """ if str(os.environ.get("UNSLOTH_FSDP2_FUSED_LORA", "1")).strip().lower() in ( "0", "false", "no", "off", ): return False return fsdp2_requested() def _zoo_gathers_fsdp2_lora(): """unsloth_zoo hands vLLM whole adapters from FSDP2 DTensor shards (older ones pass the shards).""" try: from unsloth_zoo import vllm_utils except Exception: return False return hasattr(vllm_utils, "_lora_state_dict") def raise_if_fast_inference_under_fsdp2(fast_inference): """Refuse vLLM under accelerate's FSDP2 early (unsloth#3551); native FSDP2 keeps it (one engine per rank).""" if not fast_inference or not fsdp2_requested(): return from ..fsdp2_launch import unsloth_fsdp2_native_requested if unsloth_fsdp2_native_requested() and _zoo_gathers_fsdp2_lora(): return raise NotImplementedError( "Unsloth: `fast_inference = True` (vLLM) is not supported with accelerate's FSDP2, or with an " "unsloth_zoo that cannot gather FSDP2 adapters for vLLM (update unsloth_zoo).\n" "Load the model with `fast_inference = False` to train with FSDP2, launch with " "`UNSLOTH_FSDP2=1 torchrun ...` (Unsloth's native FSDP2 keeps vLLM), " "or launch without FSDP2 to use vLLM." ) def prepare_device_map(): rank, world_size = _infer_distributed_ranks() distributed = (world_size or 1) > 1 or (rank is not None and rank > 0) if not distributed: return None, False local_rank = _infer_local_rank(rank) device_map = {"": f"{DEVICE_TYPE_TORCH}:{local_rank}"} try: if DEVICE_TYPE_TORCH == "cuda": torch.cuda.set_device(local_rank) elif DEVICE_TYPE_TORCH == "xpu" and hasattr(torch, "xpu"): torch.xpu.set_device(local_rank) elif DEVICE_TYPE_TORCH == "npu" and hasattr(torch, "npu"): torch.npu.set_device(local_rank) except Exception: pass return device_map, True def deepspeed_zero3_device_map(device_map): """None under DeepSpeed ZeRO-3 init: transformers refuses any device_map there, since ZeRO-3 places the weights itself (#225). Only a placement left to us is dropped; a map the caller chose still raises.""" try: from transformers.integrations import is_deepspeed_zero3_enabled zero3 = is_deepspeed_zero3_enabled() except Exception: zero3 = False if not zero3: return device_map return None if is_automatic_device_map(device_map) else device_map def deepspeed_zero_stage(): """DeepSpeed ZeRO stage, else None. Unsloth loads before TrainingArguments, so `accelerate launch` env counts.""" try: from transformers.integrations.deepspeed import deepspeed_config config = deepspeed_config() except Exception: config = None if config is None: try: from accelerate.state import AcceleratorState plugins = AcceleratorState._shared_state.get("deepspeed_plugins", None) or {} if not isinstance(plugins, dict): plugins = {"": plugins} for plugin in plugins.values(): if ( getattr(plugin, "selected", True) and getattr(plugin, "zero_stage", None) is not None ): return int(plugin.zero_stage) except Exception: pass if str(os.environ.get("ACCELERATE_USE_DEEPSPEED", "")).strip().lower() not in _TRUTHY: return None stage = str(os.environ.get("ACCELERATE_DEEPSPEED_ZERO_STAGE", "")).strip() if stage.isdigit(): return int(stage) path = os.environ.get("ACCELERATE_DEEPSPEED_CONFIG_FILE", "") if not path: return None try: import json with open(path, "r", encoding = "utf-8") as f: config = json.load(f) except Exception: return None try: return int(config.get("zero_optimization", {}).get("stage")) except Exception: return None def deepspeed_zero3_requested(): return deepspeed_zero_stage() == 3 def sync_deepspeed_mixed_precision(args): """Resync an already-filled DeepSpeed config's fp16 / bf16 after Unsloth changed them, else transformers stops with "ds fp16.enabled=True vs hf fp16|fp16_full_eval=False".""" want = { "fp16": bool(getattr(args, "fp16", False) or getattr(args, "fp16_full_eval", False)), "bf16": bool(getattr(args, "bf16", False) or getattr(args, "bf16_full_eval", False)), } configs = [] for holder, attr in ( (getattr(args, "hf_deepspeed_config", None), "config"), (getattr(args, "deepspeed_plugin", None), "deepspeed_config"), ): config = getattr(holder, attr, None) if holder is not None else None if isinstance(config, dict) and all(config is not c for c in configs): configs.append(config) changed = [] for config in configs: for key, enabled in want.items(): section = config.get(key) if not isinstance(section, dict) or section.get("enabled", False) == "auto": continue if bool(section.get("enabled", False)) != enabled: section["enabled"] = enabled changed.append(key) hf_config = getattr(args, "hf_deepspeed_config", None) if hf_config is not None and isinstance(getattr(hf_config, "config", None), dict): # trainer_config_process already queued mismatches for the old flags; the config now agrees. mismatches = getattr(hf_config, "mismatches", None) if isinstance(mismatches, list): mismatches[:] = [ m for m in mismatches if not m.startswith(("- ds fp16.enabled=", "- ds bf16.enabled=")) ] if hasattr(hf_config, "_dtype"): # From the flags: accelerate adds absent fp16 / bf16 sections later without refreshing _dtype. hf_config._dtype = ( torch.bfloat16 if want["bf16"] else torch.float16 if want["fp16"] else torch.float32 ) return changed def vlm_gradient_checkpointing_reentrant(): """Only DDP needs non-reentrant (#3713); under ZeRO-3 non-reentrant fails torch's saved-tensor check.""" return not is_distributed() or fsdp2_requested() or deepspeed_zero_stage() is not None def _is_moe_block(module): children = dict(module.named_children()) return "experts" in children and ("gate" in children or "router" in children) def set_deepspeed_moe_leaf_modules(model): """Mark MoE blocks ZeRO-3 leaves (before `deepspeed.initialize`): per-expert hooks give mismatched all-gathers across ranks.""" if model is None or not deepspeed_zero3_requested(): return () try: from deepspeed.utils import set_z3_leaf_modules except Exception: return () classes = [] for module in model.modules(): if type(module) not in classes and _is_moe_block(module): classes.append(type(module)) if not classes: return () try: set_z3_leaf_modules(model, classes) except Exception as exc: print(f"Unsloth: could not mark MoE blocks as DeepSpeed ZeRO-3 leaf modules ({exc}).") return () return tuple(cls.__name__ for cls in classes) def set_deepspeed_fused_lora_leaf_modules(layers): """Fused LoRA kernels read weights without calling their modules, so ZeRO-3's prefetch trace breaks unless layers are leaves.""" layers = list(layers) for layer in layers: layer._unsloth_fused_lora_layer = True if not deepspeed_zero3_requested(): return 0 for layer in layers: layer._z3_leaf = True return len(layers) def apply_deepspeed_zero3_model_setup(model): """Redo load-time ZeRO-3 setup at trainer time, for models loaded before DeepSpeed was visible.""" if not hasattr(model, "modules") or not deepspeed_zero3_requested(): return try: set_deepspeed_moe_leaf_modules(model) set_deepspeed_fused_lora_leaf_modules( m for m in model.modules() if getattr(m, "_unsloth_fused_lora_layer", False) ) except Exception as exc: print(f"Unsloth: could not apply the DeepSpeed ZeRO-3 model setup ({exc}).") if getattr(model, "_unsloth_vlm_gc_non_reentrant", False) and not getattr( model, "_unsloth_warned_zero3_gc", False ): model._unsloth_warned_zero3_gc = True print( "Unsloth: this model was loaded without DeepSpeed detected, so it uses non-reentrant gradient " "checkpointing, which can fail under ZeRO-3. Load the model under the DeepSpeed launch instead." ) UNSLOTH_DEVICE_MAP = "unsloth" # Same planner, different answer when it declines: every veto path ends in "sequential", which fills the first device to its whole free budget, wrong for a caller that picked several cards. UNSLOTH_BALANCED_DEVICE_MAP = "unsloth_balanced" _PLANNED_DEVICE_MAPS = {UNSLOTH_DEVICE_MAP: "sequential", UNSLOTH_BALANCED_DEVICE_MAP: "balanced"} # A string, not None, so an explicit False stays distinguishable from "unset". OFFLOAD_EMBEDDING_AUTO = "auto" class _DefaultDeviceMap(str): """`"sequential"`, marked as the value nobody asked for. The env opt-in must upgrade the default and leave an explicit "sequential" alone, and a plain comparison cannot tell them apart. A `str` subclass keeps the distinction without changing the value, in `inspect.signature` too, which a sentinel object would not.""" __slots__ = () DEFAULT_DEVICE_MAP = _DefaultDeviceMap("sequential") def planner_hub_kwargs(loader_kwargs): """The Hub options the planner's own `AutoConfig` lookup needs. It resolves the config a second time, from `model_name`, and without these the lookup can go to the network behind a `local_files_only` request, or miss a cache-only model and turn a plan the load needed into a "sequential" fallback.""" loader_kwargs = loader_kwargs or {} hub = {} if loader_kwargs.get("cache_dir") is not None: hub["cache_dir"] = loader_kwargs["cache_dir"] if _get_effective_local_files_only(loader_kwargs): hub["local_files_only"] = True # Resolving a remote class is a third lookup, and the planner honours code_revision for it; left out, the plan is built from the default revision's code and can name a tree the model lacks. if loader_kwargs.get("code_revision") is not None: hub["code_revision"] = loader_kwargs["code_revision"] return hub def planner_config_overrides(loader_kwargs): """Config fields the caller overrides on the load, which the planner has to size for. The planner rebuilds the repo's config from a name, so an override living only in kwargs is invisible to it. `max_position_embeddings` is the one that changes weight sizes: raise it on learned position embeddings and the planned tensors come out smaller than the materialized ones, so a map that fitted on paper OOMs. `plan_device_map_for_pretrained` hands leftover kwargs to `AutoConfig.from_pretrained`, which applies them the way the load does.""" value = (loader_kwargs or {}).get("max_position_embeddings") return {} if value is None else {"max_position_embeddings": value} def planner_kwargs_with_max_memory(planner_kwargs, loader_kwargs): """The caller's transformers `max_memory` has to reach the planner as well. Once a plan is returned the load gets an explicit dict, and transformers consults `max_memory` only for a string `device_map` (`_get_device_map` gates the whole `infer_auto_device_map` branch on `isinstance(device_map, str)`), so a budget that used to bound placement would be dropped in silence and the plan could exceed their caps or use a card they withheld. An explicit `device_map_planner_kwargs["max_memory"]` wins.""" budget = (loader_kwargs or {}).get("max_memory") if budget is None: return planner_kwargs merged = dict(planner_kwargs or {}) merged.setdefault("max_memory", budget) return merged # transformers' bitsandbytes quantizers (4-bit and 8-bit) raise this when the automatic map spills past the GPU. _BNB_CPU_SPILL_PREFIX = "Some modules are dispatched on the CPU or the disk" def raise_if_bnb_cpu_spill( error, model_name, offload_layers = None, device_map = None, load_in_8bit = False, quantization_config = None, max_memory = None, ): """Replace transformers' bitsandbytes CPU-spill error with what to do in Unsloth (#1629). Its advice, `llm_int8_enable_fp32_cpu_offload`, keeps the spilled weights unquantized in fp32 on the CPU, which is no route to training; `offload_layers = "auto"` is. Returns for any other error, and for a caller's own dict or CPU map (that placement is a choice, and transformers' advice is the relevant one), so the caller re-raises it unchanged. `offload_layers`, `quantization_config` and `max_memory` are what the caller passed, before the loader added its own.""" if not isinstance(error, ValueError) or not str(error).startswith(_BNB_CPU_SPILL_PREFIX): return # On transformers 4.x an all-CPU map raises this too unless bitsandbytes' multi-backend is on. if isinstance(device_map, dict) or str(device_map).split(":")[0] in ("cpu", "disk"): return free = "" try: # The current card only: probing others would open a CUDA context on cards the caller may have withheld. # A caller's max_memory may withhold this card; then the figure would be about the wrong one. if DEVICE_TYPE_TORCH == "cuda" and torch.cuda.is_available() and not max_memory: device = torch.cuda.current_device() free = ( f" (cuda:{device} has {torch.cuda.mem_get_info(device)[0] / 1024**3:.2f} GB free)" ) except Exception: free = "" if load_in_8bit or getattr(quantization_config, "load_in_8bit", False): # offload_layers supports 16-bit and 4-bit loads only. hint = "Load in 4-bit (load_in_4bit = True) to halve the weights, or load a smaller model." elif quantization_config is not None: # offload_layers refuses a quantization_config. hint = ( 'Pass load_in_4bit = True with `offload_layers = "auto"` instead of a ' "quantization_config to stream the layers the GPU cannot hold from host RAM, " "or load a smaller model." ) elif offload_layers == "auto": hint = "Lower max_seq_length or the batch size, load a smaller model, or add a GPU." elif offload_layers: hint = 'Raise offload_layers or pass `offload_layers = "auto"`, load a smaller model, or add a GPU.' else: hint = ( 'Pass `offload_layers = "auto"` to from_pretrained to keep the decoder layers the GPU ' "cannot hold in host RAM and stream them in during training (slower, and it needs that " "much free system RAM), " "load a smaller model, or free GPU memory held by other programs." ) raise ValueError( f"Unsloth: {model_name} does not fit in GPU memory{free}, so transformers placed " f"part of it on the CPU, which bitsandbytes cannot quantize. {hint}" ) from error def unmarked_device_map(device_map): """The default with its marker removed; anything else exactly as it came in. For a nested load that must not re-read the value as "nobody chose this". A bare `str()` would also flatten a caller's `{"": 0}` into text transformers reads as a device name.""" return str(device_map) if isinstance(device_map, _DefaultDeviceMap) else device_map def requested_device_map(device_map): """Head-aware planning is what a caller who chose nothing gets. Only the untouched default is upgraded: a dict, "auto", or a "sequential" the caller typed is a placement someone chose, and greedy fill is a different execution model. `UNSLOTH_AUTO_DEVICE_MAP=0` turns it off process-wide.""" if device_map is DEFAULT_DEVICE_MAP and os.environ.get("UNSLOTH_AUTO_DEVICE_MAP", "1") == "1": return UNSLOTH_DEVICE_MAP return device_map # transformers routes exactly these four through infer_auto_device_map; every other string becomes `{"": torch.device(value)}` (modeling_utils.from_pretrained), i.e. a device the caller named. TRANSFORMERS_PLACEMENT_STRATEGIES = frozenset({"auto", "balanced", "balanced_low_0", "sequential"}) AUTOMATIC_DEVICE_MAPS = TRANSFORMERS_PLACEMENT_STRATEGIES | { UNSLOTH_DEVICE_MAP, UNSLOTH_BALANCED_DEVICE_MAP, } def is_automatic_device_map(device_map): """True when the placement is still ours to pick, so a distributed launch may pin it to the rank's own card. A strategy name spreads one model over whatever devices it finds, which is exactly what every rank must not do (#3459). A device the caller named -- "cpu", "cuda:2", "mps" -- is a choice, and rewriting it loads the model on hardware they deliberately avoided. A bare accelerator type is the in-between: they named the type, never the index, so filling in the rank's index honours it. """ if isinstance(device_map, _DefaultDeviceMap): return True if not isinstance(device_map, str): return False if device_map in AUTOMATIC_DEVICE_MAPS: return True return DEVICE_TYPE_TORCH != "cpu" and device_map == DEVICE_TYPE_TORCH def planner_quantization_kwargs( load_in_4bit = False, load_in_8bit = False, quantization_config = None, extra_skip_modules = None, rewritten_quantization_config = None, ): """The quantization the planner must size for, as the load will really apply it. The config or the flags, never both, since transformers refuses both and loader.py clears the flags whenever it forwards a config; bare flags would describe a full-precision load and raise `DeviceMapInfeasible` on one that would have fit. The skip list travels with the flags: SKIP_QUANTIZATION_MODULES stays in compute dtype as `modules_to_not_convert`, and sizing it at 4bit understates the head device by GiBs on a large-vocab VLM. A pre-quantized checkpoint carries its own list in config.json.""" rewritten = ( {} if rewritten_quantization_config is None else {"rewritten_quantization_config": rewritten_quantization_config} ) if quantization_config is not None: return {"quantization_config": quantization_config, **rewritten} kwargs = {"load_in_4bit": load_in_4bit, "load_in_8bit": load_in_8bit, **rewritten} if load_in_4bit or load_in_8bit: try: from unsloth_zoo.peft_utils import SKIP_QUANTIZATION_MODULES except Exception: # Built on every quantized load, planning or not, so an older unsloth_zoo must not turn a 4bit load into an ImportError. One without the shared list predates the planner that consumes it, so this plan was going to decline anyway. return kwargs kwargs["llm_int8_skip_modules"] = SKIP_QUANTIZATION_MODULES + list(extra_skip_modules or []) return kwargs def _single_device_index(device_map): """The one CUDA device a string/int device map resolves to on this host, else None.""" if isinstance(device_map, bool): return None if isinstance(device_map, int): return device_map if isinstance(device_map, torch.device): if device_map.type != "cuda": return None # An unindexed torch.device means the current device (set_device(local_rank)), not 0. return device_map.index if device_map.index is not None else torch.cuda.current_device() if not isinstance(device_map, str): return None if device_map.startswith("cuda:"): try: return int(device_map.split(":", 1)[1]) except ValueError: return None if device_map == "cuda": # transformers maps a bare "cuda" to cuda:{LOCAL_RANK}, one device on any host. try: return int(os.environ.get("LOCAL_RANK", 0)) except ValueError: return None if ( device_map in ("auto", "sequential", "balanced", "balanced_low_0") or isinstance(device_map, _DefaultDeviceMap) or device_map in AUTOMATIC_DEVICE_MAPS ): try: if DEVICE_TYPE_TORCH == "cuda" and torch.cuda.device_count() == 1: return 0 except Exception: return None return None def no_placement_matcher(model, names = None): """Predicate: is this tensor name one of `model._no_placement_params`? transformers reads the names relative to a `_no_split_modules` module: a bare name ("weight", DeepSeek-V4.1's engram table) matches only a direct tensor of such a module (a suffix match would take every weight in the model); a dotted one (Qwen4Exp's "ple.ple_embedding.ngram_embedding.weight") keeps the suffix match.""" names = list(getattr(model, "_no_placement_params", None) or [] if names is None else names) dotted = [n for n in names if "." in n] bare = {n for n in names if "." not in n} roots = set() if bare: no_split = set(getattr(model, "_no_split_modules", None) or []) roots = { path for path, module in model.named_modules() if type(module).__name__ in no_split } def match(name): if any(name == n or name.endswith("." + n) for n in dotted): return True owner, _, leaf = name.rpartition(".") return leaf in bare and owner in roots return match def no_placement_tensor_names(model): """Every tensor name under a module owning one of the model's `_no_placement_params`.""" names = getattr(model, "_no_placement_params", None) if not names: return set() tensors = list(model.named_parameters(remove_duplicate = False)) + list( model.named_buffers(remove_duplicate = False) ) match = no_placement_matcher(model, names) owners = {name.rsplit(".", 1)[0] for name, _ in tensors if match(name)} return {name for name, _ in tensors if any(name.startswith(o + ".") for o in owners)} def exclude_no_placement_params(device_map, model_class, config): """Keep `_no_placement_params` off the device map (on CPU): transformers' handling sent all of Qwen4Exp to CPU on one GPU. `UNSLOTH_PLACE_NO_PLACEMENT_PARAMS=1` restores it.""" names = getattr(model_class, "_no_placement_params", None) if model_class is not None else None if not names or os.environ.get("UNSLOTH_PLACE_NO_PLACEMENT_PARAMS", "0") == "1": return device_map if isinstance(device_map, dict): base = dict(device_map) else: index = _single_device_index(device_map) if index is None: print( f"Unsloth: {model_class.__name__} keeps {', '.join(names)} off the device map, but " f"device_map = {device_map!r} spans several devices; leaving the placement to transformers." ) return device_map base = {"": index} try: from accelerate import init_empty_weights with init_empty_weights(): meta = model_class._from_config(config) except Exception as error: print( f"Unsloth: could not build {model_class.__name__} on meta to place {names} ({error})." ) return device_map match = no_placement_matcher(meta, names) matched = { name for name, _ in list(meta.named_parameters()) + list(meta.named_buffers()) if match(name) } if not matched: return device_map # Whole owning module: FP8Embedding multiplies by a sibling weight_scale on the same device. excluded_modules = sorted({name.rsplit(".", 1)[0] for name in matched}) excluded = { name for name, _ in list(meta.named_parameters()) + list(meta.named_buffers()) if any(name.startswith(module + ".") for module in excluded_modules) } def owner(path): best = None for key in out: if key == "" or path == key or path.startswith(key + "."): if best is None or len(key) > len(best): best = key return best out = dict(base) for path in excluded_modules: while (key := owner(path)) is not None: device = out[key] # Already off the GPU: keep the module key so accelerate's offload hooks still cover it. if str(device).split(":")[0] in ("cpu", "disk", "meta"): break out.pop(key) if key == path: continue module = meta.get_submodule(key) if key else meta prefix = key parts = path[len(key) + 1 :].split(".") if key else path.split(".") for part in parts: for child_name, _ in module.named_children(): if child_name != part: out.setdefault(f"{prefix}.{child_name}" if prefix else child_name, device) for tensor_name, _ in list(module.named_parameters(recurse = False)) + list( module.named_buffers(recurse = False) ): if tensor_name != part: out.setdefault(f"{prefix}.{tensor_name}" if prefix else tensor_name, device) module = getattr(module, part) prefix = f"{prefix}.{part}" if prefix else part billions = ( sum( t.numel() for n, t in list(meta.named_parameters()) + list(meta.named_buffers()) if n in excluded ) / 1e9 ) print( f"Unsloth: keeping {', '.join(excluded_modules)} ({billions:.1f}B parameters, frozen) on CPU; " f"set UNSLOTH_PLACE_NO_PLACEMENT_PARAMS=1 to place it on the GPU instead." ) del meta return out def compressed_tensors_planner_bits(model_config, load_in_4bit, load_in_8bit): """(load_in_4bit, load_in_8bit) to size an armed compressed-tensors load at: the packed route keeps INT8 at 8 bits.""" if not load_in_4bit or compressed_tensors_prepared_config(model_config) is None: return load_in_4bit, load_in_8bit try: from .compressed_tensors_bnb import UNSLOTH_COMPRESSED_TENSORS_ATTR from .compressed_tensors_int4 import int4_packed_route_enabled, int4_packed_supported_plan plan = getattr(model_config, UNSLOTH_COMPRESSED_TENSORS_ATTR) if int4_packed_route_enabled() and int4_packed_supported_plan(plan): if max(int(g["weights"]["num_bits"]) for g in plan["config_groups"].values()) > 4: return False, True except Exception: pass return load_in_4bit, load_in_8bit def compressed_tensors_planner_quantization( model_config, load_in_4bit, load_in_8bit, quantization_config ): """Planner flags + config: a packed INT8 load is sized at 8 bits even under an explicit bnb 4-bit config.""" wants_4bit = bool(load_in_4bit) or ( quantization_config is not None and quantization_config_selects_bnb_4bit(quantization_config) ) if ( compressed_tensors_planner_bits(model_config, wants_4bit, load_in_8bit) == (False, True) and wants_4bit ): return dict(load_in_4bit = False, load_in_8bit = True, quantization_config = None) return dict( load_in_4bit = load_in_4bit, load_in_8bit = load_in_8bit, quantization_config = quantization_config, ) def compressed_tensors_prepared_config(model_config): """`model_config` if armed for packed compressed-tensors re-quantization (plan must size from it), else None.""" if model_config is None: return None try: from .compressed_tensors_bnb import UNSLOTH_COMPRESSED_TENSORS_ATTR except Exception: return None return ( model_config if getattr(model_config, UNSLOTH_COMPRESSED_TENSORS_ATTR, None) is not None else None ) def planner_model_class(config, trust_remote_code = False): """The model class the planner's own rules pick for `config`, or None if unknown. The planner never sees the auto class the load chose: `config` is whatever the caller passed, while the planner rebuilds the repo's from `model_name`, and the two can disagree.""" try: from unsloth_zoo.device_map_planner import _auto_class_for from ._utils import resolve_model_class auto_class = _auto_class_for(config, trust_remote_code = trust_remote_code) return resolve_model_class(auto_class, config, trust_remote_code = trust_remote_code) except Exception: # Unknown, not mismatched: an unsloth_zoo without this has no planner to feed. return None def planner_class_mismatch_reason(loaded_class, planned_class): """Why the planner's model differs from the one being loaded, else None. Overriding the config's own choice gets a map for a module tree the model does not have: `num_labels` swaps in a `score` head where `lm_head` was planned, and dispatch refuses with "does not give any device for ... score.weight". Compared as model classes, since two distinct auto classes can resolve to the same VLM.""" if loaded_class is None or planned_class is None or loaded_class is planned_class: return None return f"the load builds {loaded_class.__name__}, not the planned {planned_class.__name__}" # accelerate's max_memory spellings, in the order it tries them: GiB/MiB/KiB are binary, GB/MB/KB decimal, and a lowercase trailing `b` on a decimal unit means bits. _SIZE_UNITS = ( ("GIB", 2**30, False), ("MIB", 2**20, False), ("KIB", 2**10, False), ("GB", 10**9, True), ("MB", 10**6, True), ("KB", 10**3, True), ) def _as_bytes(size): """A `max_memory` budget in bytes, or None if it cannot be read as one. accelerate takes `"10GiB"` as readily as an int, so a caller writes what the load would have taken. The rules are reproduced rather than imported because this runs before anything has established accelerate is installed, and an ImportError would read every budget as unparseable and drop the cap in silence; `test_the_local_size_parser_agrees_with_accelerate` holds the copy in step wherever accelerate is present. None leaves the measured free memory in place rather than dropping the device. A bool is unreadable here where accelerate would take its int value: `{0: True}` is a typo, and a one byte budget reads as "unusable" to the planner.""" if isinstance(size, bool): return None if isinstance(size, int): return size if size >= 0 else None if not isinstance(size, str): return None upper = size.upper() for unit, scale, has_bit_form in _SIZE_UNITS: if not upper.endswith(unit): continue try: amount = int(float(size[: -len(unit)]) * scale) except ValueError: return None # Bits, if they spelled the unit "Gb" rather than "GB"; binary units have no such form. if has_bit_form and size.endswith("b"): amount //= 8 return amount if amount >= 0 else None return None def resolve_unsloth_device_map( device_map, model_name, *, fast_inference = False, full_finetuning = False, planner_kwargs = None, skip_reason = None, planner_config = None, planner_config_reason = None, **config_kwargs, ): """Plan a head-aware multi-GPU map for `device_map = "unsloth"`, else return as-is. Opt-in only, so nothing an existing caller passes changes meaning, and the plan is built on the meta device: no GPU memory, no weight download. Falls back to "sequential" wherever a plan cannot apply, since a model that loads the old way beats one that refuses to load at all; `DeviceMapInfeasible` is the exception, raised rather than spilling a bitsandbytes model to CPU, and swallowing it would hand the user an OOM instead of a diagnosis. `skip_reason` is the caller's veto, for when only the caller can tell the planner would describe a different model than the load builds.""" # isinstance first: a caller's explicit dict is unhashable, so `in` alone raises. if not isinstance(device_map, str) or device_map not in _PLANNED_DEVICE_MAPS: return device_map _declined = _PLANNED_DEVICE_MAPS[device_map] def _fallback(reason): print(f"Unsloth: Not planning a device map; {reason}. Using `{_declined}`.") return _declined def _plan_fits_first_device( plan, explicit_reserve, activation_share = 0.2, ): """First allowed card if weights + headroom + load transient leave `activation_share` (about Gemma 3n E4B fp32 on a T4) and any passed reserve free; None for a planner without sizes.""" try: budgets = plan.raw_budgets first = min(budgets) budget = int(budgets[first]) transient = max((plan.load_transient_by_device or {}).values(), default = 0) need = int(plan.total_weight_bytes) + int(plan.headroom_bytes) + int(transient) free = budget * activation_share if explicit_reserve: # A passed reserve is a hard constraint to the planner. free = max(free, int(plan.activation_reserve_by_device.get(first, 0))) except Exception: return None if budget > 0 and need + free <= budget: return first return None if skip_reason is not None: return _fallback(skip_reason) if fast_inference: return _fallback("vLLM places its own weights") if full_finetuning: return _fallback("full finetuning does not use the quantized planner") if is_distributed(): # Every rank already owns the whole model on its own card; splitting on top of that puts every rank on every card, a different execution model. return _fallback("each rank of a distributed launch owns its own device") if DEVICE_TYPE_TORCH != "cuda": return _fallback(f"the planner has no memory budgets for {DEVICE_TYPE_TORCH}") try: device_count = torch.cuda.device_count() except Exception as error: return _fallback(f"the devices could not be counted ({error})") # Popped, not forwarded: max_memory is a named parameter of the planner, so a copy left in planner_kwargs raises TypeError, which the handler below turns into a silent "sequential". A caller's mapping replaces the measured one rather than editing it: its keys are the devices they will let the load use, as accelerate reads it too, so {0: ..., 1: ...} on a four-GPU host means GPUs 2 and 3 are somebody else's. planner_kwargs = dict(planner_kwargs or {}) requested_memory = planner_kwargs.pop("max_memory", None) requested_memory = dict(requested_memory) if requested_memory else None # Read before probing: mem_get_info initialises a CUDA context on each device it touches, and a withheld card is likely busy with the workload it was withheld for. One that refuses (ECC error, MIG parent, Exclusive_Process) would also drop the plan to "sequential" over a device this load was never going to use. if requested_memory is None: probe = list(range(device_count)) else: probe = [ device for device in requested_memory if isinstance(device, int) and not isinstance(device, bool) and 0 <= device < device_count ] if len(probe) < 2: return _declined try: from unsloth_zoo.device_map_planner import plan_device_map_for_pretrained except Exception as error: return _fallback(f"the planner is unavailable ({error})") # Older planners without `config` would forward it to AutoConfig and plan the full repo model. if planner_config is not None: try: _planner_params = inspect.signature(plan_device_map_for_pretrained).parameters except (TypeError, ValueError): _planner_params = {} if "config" not in _planner_params: return _fallback( planner_config_reason or "this unsloth_zoo cannot plan from a resolved config" ) config_kwargs["config"] = planner_config # Two `config` kwargs raise TypeError, silently falling back to "sequential". if "config" in config_kwargs: planner_kwargs.pop("config", None) # Free, not total: this process's context and anything else resident make total an overcommit. Guarded, because a card can still refuse mid-probe. try: max_memory = {index: torch.cuda.mem_get_info(index)[0] for index in probe} except Exception as error: return _fallback(f"free memory could not be read ({error})") if requested_memory is not None: budgets = {} for device, written in requested_memory.items(): measured = max_memory.get(device) budget = _as_bytes(written) if budget is None: # Nothing to compare against: what we measured, or for a device we never measured (cpu, disk) their value untouched for the planner to read. budgets[device] = measured if measured is not None else written else: # Under what is actually free: they may know of reservations we cannot measure, but planning above free is how a plan OOMs on dispatch. budgets[device] = budget if measured is None else min(measured, budget) max_memory = budgets try: plan = plan_device_map_for_pretrained( model_name, max_memory = max_memory, **planner_kwargs, **config_kwargs ) except Exception as error: if type(error).__name__ == "DeviceMapInfeasible": raise return _fallback(f"planning failed ({error})") if plan is None: return _declined single_device = ( _plan_fits_first_device(plan, planner_kwargs.get("activation_reserve_bytes") is not None) if device_map == UNSLOTH_DEVICE_MAP else None ) if single_device is not None: # Splitting a model one card holds only adds cross-device bugs (Kaggle T4x2). print( f"Unsloth: Not splitting across GPUs; the model fits on cuda:{single_device} with room " f'to train. Pass device_map = "{UNSLOTH_BALANCED_DEVICE_MAP}" to split it anyway.' ) return {"": single_device} print(plan.describe()) return plan.device_map def _sharded_data_parallel_launch(): """ "FSDP" / "DeepSpeed" when this process will hand the model to one, else None.""" try: from accelerate.state import AcceleratorState distributed_type = str(AcceleratorState._shared_state.get("distributed_type", "")).upper() except Exception: distributed_type = "" if "FSDP" in distributed_type: return "FSDP" if "DEEPSPEED" in distributed_type: return "DeepSpeed" if str(os.environ.get("ACCELERATE_USE_FSDP", "")).strip().lower() in _TRUTHY or str( os.environ.get("FSDP_VERSION", "") ).strip() not in ("", "0"): return "FSDP" if str(os.environ.get("ACCELERATE_USE_DEEPSPEED", "")).strip().lower() in _TRUTHY: return "DeepSpeed" return None def _single_card_device_map(device_map): if isinstance(device_map, dict): if set(device_map) != {""}: return False device_map = device_map[""] if isinstance(device_map, int) and not isinstance(device_map, bool): return True if isinstance(device_map, torch.device): return device_map.type == "cuda" return isinstance(device_map, str) and (device_map == "cuda" or device_map.startswith("cuda:")) def ddp_auto_block_swap_launch(device_map): return ( is_distributed() and _sharded_data_parallel_launch() is None and _single_card_device_map(device_map) ) def auto_offload_for_launch(offload_layers): """No "auto" swap on a launch that already shows FSDP / DeepSpeed.""" if ( offload_layers == "auto" and is_distributed() and _sharded_data_parallel_launch() is not None ): return 0 return offload_layers _BLOCK_SWAP_PLANNER_KEYS = ( "batch_size", "lora_rank", "reserve_bytes", "prefetch_depth", "rows_per_chunk", "retained_rows", "headroom_bytes", "safety_bytes", "free_space_policy", "no_split_module_classes", ) def resolve_auto_block_swap( device_map, model_name, *, max_seq_length, offload_embedding = False, planner_kwargs = None, skip_reason = None, placement = "tail", **config_kwargs, ): """`from_pretrained(offload_layers = "auto")`: `(layers, device_map, embedding)`, the trailing decoder layers to build in host RAM so the rest plus a training step's reserve fits, the map to load the rest with, and whether to move the input embedding to host RAM first (one GPU, when `offload_embedding` allows; None when nothing was planned). 0, the map unchanged and no move when everything fits, so nothing slows down. `device_map = "unsloth"` / `"unsloth_balanced"` sizes every card through the multi-GPU planner; anything else sizes the one card the load uses.""" def _none(reason): print(f"Unsloth: offload_layers = 'auto' loads every layer onto the GPU: {reason}.") return 0, device_map, None if skip_reason is not None: return _none(skip_reason) if DEVICE_TYPE_TORCH != "cuda" or not torch.cuda.is_available(): return _none("block swap needs a CUDA or ROCm GPU") distributed = is_distributed() if distributed: sharded = _sharded_data_parallel_launch() if sharded is not None: return _none(f"{sharded} shards the weights across the ranks itself") if not _single_card_device_map(device_map): return _none("each rank of a distributed launch owns its own device") # DDP refuses a CPU parameter. offload_embedding = False try: from unsloth_zoo.device_map_planner import plan_block_swap except ImportError: return _none("this unsloth_zoo cannot plan block swap") planner_kwargs = dict(planner_kwargs or {}) requested_memory = planner_kwargs.pop("max_memory", None) devices = [] if isinstance(device_map, str) and device_map in _PLANNED_DEVICE_MAPS: devices = ( [d for d in requested_memory if isinstance(d, int) and not isinstance(d, bool)] if requested_memory else list(range(torch.cuda.device_count())) ) multi = len(devices) > 1 if multi: target = None elif devices: target = devices[0] else: if isinstance(device_map, dict) and set(device_map) == {""}: target = torch.device(device_map[""]) if target.type != "cuda": return _none(f"the load places the model on {target}") target = target.index if target.index is not None else torch.cuda.current_device() elif ( isinstance(device_map, (int, torch.device)) and _single_device_index(device_map) is not None ): target = _single_device_index(device_map) elif device_map is None or ( isinstance(device_map, str) and (device_map in TRANSFORMERS_PLACEMENT_STRATEGIES or device_map.startswith("cuda")) ): target = ( torch.device(device_map).index if isinstance(device_map, str) and device_map.startswith("cuda:") else torch.cuda.current_device() ) else: return _none( "it sizes one GPU or the `device_map = 'unsloth'` planner, not an explicit map" ) devices = [target] from ._utils import usable_cuda_bytes max_memory = {} for d in devices: free = usable_cuda_bytes(d) cap = _as_bytes((requested_memory or {}).get(d)) if requested_memory else None max_memory[d] = free if cap is None else min(free, cap) options = {k: planner_kwargs[k] for k in _BLOCK_SWAP_PLANNER_KEYS if k in planner_kwargs} if "prefetch_depth" in options: from ._utils import auto_plan_depth options["prefetch_depth"] = auto_plan_depth(options["prefetch_depth"]) try: planner_parameters = inspect.signature(plan_block_swap).parameters except (TypeError, ValueError): planner_parameters = {} if offload_embedding and not multi and "offload_embedding" in planner_parameters: options["offload_embedding"] = True if placement != "tail" and not multi: if "placement" not in planner_parameters: return _none("this unsloth_zoo plans only trailing layers") options["placement"] = placement plan = plan_block_swap( model_name, max_memory = max_memory, seq_len = max_seq_length, **options, **config_kwargs, ) print( "Unsloth: " + plan.describe().splitlines()[0].replace("block swap:", "offload_layers = 'auto':") ) embedding = bool(getattr(plan, "offload_embedding", False)) if distributed: # No collective: a loader only some ranks run would hang. Plans may differ per rank. layers = int(plan.layers or 0) print(f"Unsloth: offload_layers = 'auto' plans {layers} layers on this rank.") return layers, ({"": target} if layers else device_map), False if not plan.layers: return 0, device_map, embedding if multi and plan.device_plan is not None: print(plan.device_plan.describe()) return plan.layers, plan.device_plan.device_map, embedding return plan.layers, {"": target}, embedding def __get_model_name( model_name, load_in_4bit = True, INT_TO_FLOAT_MAPPER = None, FLOAT_TO_INT_MAPPER = None, MAP_TO_UNSLOTH_16bit = None, load_in_fp8 = False, FLOAT_TO_FP8_BLOCK_MAPPER = None, FLOAT_TO_FP8_ROW_MAPPER = None, ): model_name = str(model_name) lower_model_name = model_name.lower() assert load_in_fp8 in (True, False, "block") if load_in_fp8 != False: if load_in_fp8 == True and (os.environ.get("UNSLOTH_HAS_FBGEMM", "0") == "1"): if lower_model_name in FLOAT_TO_FP8_ROW_MAPPER: # Faster row scaling only works if FBGEMM works; otherwise use the slower blockwise type. return FLOAT_TO_FP8_ROW_MAPPER[lower_model_name] elif lower_model_name in FLOAT_TO_FP8_BLOCK_MAPPER: return FLOAT_TO_FP8_BLOCK_MAPPER[lower_model_name] else: if lower_model_name in FLOAT_TO_FP8_BLOCK_MAPPER: return FLOAT_TO_FP8_BLOCK_MAPPER[lower_model_name] # No pre-quantized model found. vllm >= 0.12.0 quantizes to FP8 on the fly (returning the original name); older vllm falls through to offline quant. if importlib.util.find_spec("vllm") is not None: import vllm if Version(vllm.__version__) >= Version("0.12.0"): return model_name return None elif not SUPPORTS_FOURBIT and lower_model_name in INT_TO_FLOAT_MAPPER: model_name = INT_TO_FLOAT_MAPPER[lower_model_name] print( f"Unsloth: Your transformers version of {transformers_version} does not support native " f"4bit loading.\nThe minimum required version is 4.37.\n" f'Try `pip install --upgrade "transformers>=4.37"`\n' f"to obtain the latest transformers build, then restart this session.\n" f"For now, we shall load `{model_name}` instead (still 4bit, just slower downloading)." ) return model_name elif not load_in_4bit and lower_model_name in INT_TO_FLOAT_MAPPER: new_model_name = INT_TO_FLOAT_MAPPER[lower_model_name] return new_model_name elif not load_in_4bit and lower_model_name in MAP_TO_UNSLOTH_16bit: new_model_name = MAP_TO_UNSLOTH_16bit[lower_model_name] return new_model_name elif load_in_4bit and SUPPORTS_FOURBIT and lower_model_name in FLOAT_TO_INT_MAPPER: # Keep an explicit -bnb-4bit name; otherwise map to the dynamic version. if lower_model_name.endswith("-bnb-4bit"): return model_name new_model_name = FLOAT_TO_INT_MAPPER[lower_model_name] return new_model_name return None def _get_new_mapper(): try: import requests import time new_mapper = ( "https://raw.githubusercontent.com/unslothai/unsloth/main/unsloth/models/mapper.py" ) # Capped WHILE reading, since requests.get buffers the whole body first, and the deadline is total because timeout is per-read. byte_cap = 1_000_000 deadline = time.monotonic() + 10 chunks, total = [], 0 # Redirects by hand: requests drains each intermediate body inside get. url = new_mapper for _ in range(5): response = requests.get(url, timeout = 3, stream = True, allow_redirects = False) with response: location = ( response.headers.get("location") if (300 <= response.status_code < 400) else None ) if location is not None: if time.monotonic() > deadline: return {}, {}, {}, {}, {} else: # read1: the timeout is per SOCKET READ, so a trickling peer resets it forever and the deadline is never reached. requests only decompresses inside iter_content. raw = response.raw try: raw.decode_content = True except AttributeError: pass read_once = getattr(raw, "read1", None) while True: if time.monotonic() > deadline: return {}, {}, {}, {}, {} if read_once is not None: chunk = read_once(65_536) else: # Older urllib3 has no read1; one byte returns as promptly. chunk = raw.read(1) if not chunk: break chunks.append(chunk) total += len(chunk) if total > byte_cap: return {}, {}, {}, {}, {} encoding = response.encoding or "utf-8" if location is None: break url = requests.compat.urljoin(url, location) else: return {}, {}, {}, {}, {} new_mapper = b"".join(chunks).decode(encoding, errors = "replace") # Never exec the response: that is arbitrary code execution inside every from_pretrained that hits an unmapped name. import ast # ast.parse builds the whole tree before any literal-only check runs, so this is no defence against exhaustion (python/cpython#95588); the byte cap above is. tree = ast.parse(new_mapper) # Every module-level literal dict, in source order; chosen below. literal_bindings = [] for index, node in enumerate(tree.body): # AnnAssign too, or an annotation upstream turns the probe off silently. if isinstance(node, ast.AnnAssign): targets = [node.target.id] if isinstance(node.target, ast.Name) else [] value = node.value elif isinstance(node, ast.Assign): targets = [t.id for t in node.targets if isinstance(t, ast.Name)] value = node.value else: continue if value is None or not targets: continue try: literal = ast.literal_eval(value) except Exception: continue if not isinstance(literal, dict): continue for name in targets: literal_bindings.append((index, name, literal)) def _binding_at(name, before): """What `name` holds when the statement at index `before` runs: the last assignment BEFORE that point, so a rebind after the builder ran is not read back over what the builder actually saw.""" found = None for index, bound, literal in literal_bindings: if bound == name and (before is None or index <= before): found = literal return found # Statements that really RUN at import: ast.walk reaches into function bodies and dead branches, which would fabricate a mapping. Local, because the tests run this body in a bare namespace. def _constant_truth(test): """True/False for a statically decidable condition, else None. `not True` is a UnaryOp, so it fell through and its body was read as run.""" if isinstance(test, ast.Constant): return bool(test.value) if isinstance(test, ast.UnaryOp) and isinstance(test.op, ast.Not): inner = _constant_truth(test.operand) return None if inner is None else not inner return None def _iterates_nothing(iterable): if isinstance(iterable, (ast.Tuple, ast.List, ast.Set)): return not iterable.elts if isinstance(iterable, ast.Dict): return not iterable.keys if isinstance(iterable, ast.Constant): # An empty literal iterates nothing; a nonempty one keeps its body. return isinstance(iterable.value, (str, bytes)) and not iterable.value return False def _class_bound(statement): # A class binding makes every LATER mutation a class attribute. targets = [] if isinstance(statement, ast.Assign): targets = statement.targets elif isinstance(statement, ast.AnnAssign): targets = [statement.target] return {target.id for target in targets if isinstance(target, ast.Name)} def _executed_nodes( body, shadowed = frozenset(), class_body = False, ): # What ENDED the suite: "return" leaves the function, True the suite, and only the statements ABOVE it still see the module global. for statement in body: if class_body: shadowed = shadowed | _class_bound(statement) if isinstance(statement, (ast.FunctionDef, ast.AsyncFunctionDef)): continue if isinstance(statement, (ast.Break, ast.Continue, ast.Return)): return "return" if isinstance(statement, ast.Return) else True if isinstance(statement, ast.ClassDef): # A class body, unlike a function body, RUNS at import. yield from _executed_nodes(statement.body, shadowed, class_body = True) continue if isinstance(statement, ast.If) and _constant_truth(statement.test) is not None: branch = statement.body if _constant_truth(statement.test) else statement.orelse ended = yield from _executed_nodes(branch, shadowed) if ended: return ended continue if isinstance(statement, ast.While) and _constant_truth(statement.test) is not None: if _constant_truth(statement.test): ended = yield from _executed_nodes(statement.body, shadowed) if ended == "return": return ended else: ended = yield from _executed_nodes(statement.orelse, shadowed) if ended: return ended continue if isinstance(statement, ast.For) and _iterates_nothing(statement.iter): ended = yield from _executed_nodes(statement.orelse, shadowed) if ended: return ended continue for field in ("body", "orelse", "finalbody"): children = getattr(statement, field, None) if isinstance(children, list): yield from _executed_nodes(children, shadowed) for handler in getattr(statement, "handlers", []) or []: yield from _executed_nodes(handler.body, shadowed) # Only the non-suite parts; a Lambda is excluded at the seed too. pending = [ child for child in ast.iter_child_nodes(statement) if not isinstance( child, ( ast.stmt, ast.excepthandler, ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef, ast.Lambda, ), ) ] yield statement, shadowed while pending: current = pending.pop() yield current, shadowed for child in ast.iter_child_nodes(current): if isinstance( child, ( ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef, ast.Lambda, ), ): continue if isinstance(child, (ast.stmt, ast.excepthandler)): continue pending.append(child) # The aliases a newer mapper.py adds live inside build_mappers, which the installed builder cannot know. def _calls_the_builder(node): """Whether this executed node IS the `build_mappers(...)` call. The node itself, not `ast.walk` over it: walking the parent statement descended back into the deferred children the yield had excluded, so `unused = lambda: build_mappers(...)` counted as a call.""" return ( isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id == "build_mappers" ) builder_body = [] # From the EXECUTED nodes, with the statement each runs in: a source table rebound after the call is not what the module exports. exported_names = ( "INT_TO_FLOAT_MAPPER", "FLOAT_TO_INT_MAPPER", "MAP_TO_UNSLOTH_16bit", "FLOAT_TO_FP8_BLOCK_MAPPER", "FLOAT_TO_FP8_ROW_MAPPER", ) def _binds_the_exports(statement): """Whether this statement assigns one of the exported tables.""" if isinstance(statement, ast.AnnAssign): targets = [statement.target] elif isinstance(statement, ast.Assign): targets = statement.targets else: return False bound = set() for target in targets: if isinstance(target, (ast.Tuple, ast.List)): bound |= { element.id for element in target.elts if isinstance(element, ast.Name) } elif isinstance(target, ast.Name): bound.add(target.id) return bool(bound.intersection(exported_names)) builder_call = None builder_index = None first_call = None # A call that ASSIGNS the exports replaces all five; the fallback binds nothing. builder_rebinds = False for index, statement in enumerate(tree.body): calls = [ node for node, _shadowed in _executed_nodes([statement]) if _calls_the_builder(node) ] if not calls: continue if first_call is None: first_call = (calls[0], index) # The LAST call that BINDS the exports; a validation call may run first. if _binds_the_exports(statement): builder_call, builder_index = calls[0], index builder_rebinds = True if builder_call is None and first_call is not None: builder_call, builder_index = first_call builder_called = builder_call is not None if builder_called: for statement in tree.body: if isinstance(statement, ast.FunctionDef) and statement.name == "build_mappers": builder_body = statement.body break # The table the builder was HANDED, read where the call runs. source_name = "__INT_TO_FLOAT_MAPPER" source_table = None if builder_call is not None: argument = None if builder_call.args: argument = builder_call.args[0] else: for keyword in builder_call.keywords: if keyword.arg is not None: argument = keyword.value break if isinstance(argument, ast.Name): source_name = argument.id elif argument is not None: try: literal = ast.literal_eval(argument) except Exception: literal = None if isinstance(literal, dict): source_table = literal def _source_before_the_builder(name): """The source table as the builder receives it, mutations included. `.update({...})` and subscript assignment are ordinary ways to extend it before it is handed over; the mutation pass below reads only the EXPORTED tables. In execution order, and only up to the call.""" current = None for node, _shadowed in _executed_nodes(tree.body): # The SELECTED call, by identity, so two alike calls are told apart. if node is builder_call or (builder_call is None and _calls_the_builder(node)): break if isinstance(node, (ast.Assign, ast.AnnAssign)): if node.value is None: continue targets = [node.target] if isinstance(node, ast.AnnAssign) else node.targets for target in targets: if isinstance(target, ast.Name) and target.id == name: try: literal = ast.literal_eval(node.value) except Exception: continue if isinstance(literal, dict): current = dict(literal) elif ( isinstance(target, ast.Subscript) and isinstance(target.value, ast.Name) and target.value.id == name and current is not None ): try: current[ast.literal_eval(target.slice)] = ast.literal_eval( node.value ) except Exception: continue elif isinstance(node, ast.Delete): for target in node.targets: if isinstance(target, ast.Name) and target.id == name: # Empty, not None: unbound must not fall back. current = {} elif ( isinstance(target, ast.Subscript) and isinstance(target.value, ast.Name) and target.value.id == name and current is not None ): try: current.pop(ast.literal_eval(target.slice), None) except Exception: continue elif ( isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) and node.func.attr == "update" and isinstance(node.func.value, ast.Name) and node.func.value.id == name and current is not None and len(node.args) == 1 and not node.keywords ): try: additions = ast.literal_eval(node.args[0]) except Exception: continue if isinstance(additions, dict): current.update(additions) return current if source_table is None and builder_call is not None: source_table = _source_before_the_builder(source_name) if source_table is None: source_table = _binding_at(source_name, builder_index) # Not the end of the probe: a row-only FP8 repo cannot be expressed through the source table at all, so a body that adds nothing is reported so at the end. empty_base = not source_table tables = build_mappers(source_table or {}) # Restored at the call below, which REPLACES all five tables. built = [dict(table) for table in tables] by_name = { "INT_TO_FLOAT_MAPPER": tables[0], "FLOAT_TO_INT_MAPPER": tables[1], "MAP_TO_UNSLOTH_16bit": tables[2], "FLOAT_TO_FP8_BLOCK_MAPPER": tables[3], "FLOAT_TO_FP8_ROW_MAPPER": tables[4], } # Only the helper CALLS: the builder binds its own INT_TO_FLOAT_MAPPER = {}, which the whole-name rule below would read as a clear. builder_additions = [ (node, shadowed) for node, shadowed in _executed_nodes(builder_body) if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id in _MAPPER_HELPERS ] # In EXECUTION order, so a rebind after the call still empties the table; `rebuilt` stands where the build assigns the exports, matched by identity. rebuilt = object() ordered = [] for node, shadowed in _executed_nodes(tree.body): ordered.append((node, shadowed)) if node is builder_call or (builder_call is None and _calls_the_builder(node)): if builder_rebinds: ordered.append((rebuilt, frozenset())) ordered.extend(builder_additions) builder_additions = [] ordered.extend(builder_additions) for node, shadowed in ordered: if node is rebuilt: for table, original in zip(tables, built): table.clear() table.update(original) elif isinstance(node, (ast.Assign, ast.AnnAssign)): # An annotated subscript assigns as the plain form does; a bare one binds nothing. if isinstance(node, ast.AnnAssign): if node.value is None: continue targets = [node.target] else: targets = node.targets for target in targets: if isinstance(target, ast.Name): # A whole-name assignment REPLACES the exported table. table = by_name.get(target.id) if table is None or target.id in shadowed: continue try: replacement = ast.literal_eval(node.value) except ValueError: continue if isinstance(replacement, dict): if not replacement and not builder_called: # An INITIALISER, not a clear: a mapper.py with no build_mappers writes X = {} and fills all five from a module-scope loop the installed builder reproduces, so reading it as a clear emptied every table and stopped the upgrade notice firing. continue table.clear() table.update(replacement) continue if not isinstance(target, ast.Subscript): continue if not isinstance(target.value, ast.Name): continue if target.value.id in shadowed: continue table = by_name.get(target.value.id) if table is None: continue try: table[ast.literal_eval(target.slice)] = ast.literal_eval(node.value) except ValueError: continue elif isinstance(node, ast.Delete): for target in node.targets: if isinstance(target, ast.Name): table = by_name.get(target.id) if table is not None and target.id not in shadowed: table.clear() continue if not isinstance(target, ast.Subscript): continue if not isinstance(target.value, ast.Name): continue if target.value.id in shadowed: continue table = by_name.get(target.value.id) if table is None: continue try: table.pop(ast.literal_eval(target.slice), None) except ValueError: continue elif ( isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) and node.func.attr == "update" ): # Only a named receiver and a literal mapping; the keyword form cannot express keys with a slash. if not isinstance(node.func.value, ast.Name): continue if node.func.value.id in shadowed: continue table = by_name.get(node.func.value.id) if table is None or len(node.args) != 1 or node.keywords: continue try: additions = ast.literal_eval(node.args[0]) except ValueError: continue if isinstance(additions, dict): table.update(additions) elif isinstance(node, ast.Call) and isinstance(node.func, ast.Name): # An alias not derivable from the source table, applied with the INSTALLED helper from literal arguments only. helper = _MAPPER_HELPERS.get(node.func.id) if helper is None: continue supplied = dict(zip(("mapper", "key", "value"), node.args)) for keyword in node.keywords: if keyword.arg not in ("mapper", "key", "value"): supplied = None break if keyword.arg in supplied: supplied = None break supplied[keyword.arg] = keyword.value if supplied is None or set(supplied) != {"mapper", "key", "value"}: continue destination = supplied["mapper"] key, value = supplied["key"], supplied["value"] if not isinstance(destination, ast.Name): continue if destination.id in shadowed: continue table = by_name.get(destination.id) if table is None: continue try: helper(table, ast.literal_eval(key), ast.literal_eval(value)) except ValueError: continue pass pass if empty_base and tables == build_mappers({}): # Nothing on top of the empty base, so the body carried no mapping at all. return {}, {}, {}, {}, {} return tables except: return {}, {}, {}, {}, {} def _resolve_with_mappers( model_name, load_in_4bit, load_in_fp8, int_to_float, float_to_int, map_to_unsloth_16bit, fp8_block = None, fp8_row = None, ): # The probe passes the FETCHED fp8 tables, without rebinding the installed ones. return __get_model_name( model_name = model_name, load_in_4bit = load_in_4bit, INT_TO_FLOAT_MAPPER = int_to_float, FLOAT_TO_INT_MAPPER = float_to_int, MAP_TO_UNSLOTH_16bit = map_to_unsloth_16bit, load_in_fp8 = load_in_fp8, FLOAT_TO_FP8_BLOCK_MAPPER = FLOAT_TO_FP8_BLOCK_MAPPER if fp8_block is None else fp8_block, FLOAT_TO_FP8_ROW_MAPPER = FLOAT_TO_FP8_ROW_MAPPER if fp8_row is None else fp8_row, ) def _prefer_legacy_lowercase_cache( repo_id, local_files_only = False, cache_dir = None, revision = None, ): # The mapper returned lowercased ids before #2506, so an offline cache may only hold that spelling. if not (local_files_only or _env_says_offline()) or not isinstance(repo_id, str): return repo_id legacy = repo_id.lower() if legacy == repo_id: return repo_id try: from huggingface_hub import try_to_load_from_cache if cache_dir is None: # transformers 4.x still honours TRANSFORMERS_CACHE, which can differ from HF_HUB_CACHE. from transformers.utils import hub as _tf_hub cache_dir = getattr(_tf_hub, "TRANSFORMERS_CACHE", None) def cached(repo, files): return any( isinstance( try_to_load_from_cache(repo, f, cache_dir = cache_dir, revision = revision), str ) for f in files ) # A config-only canonical snapshot must not hide a legacy one that also has weights. weights = ( "model.safetensors", "model.safetensors.index.json", "pytorch_model.bin", "pytorch_model.bin.index.json", ) for files in (weights, ("config.json",)): if cached(repo_id, files) and cached(repo_id, ("config.json",)): return repo_id if cached(legacy, files) and cached(legacy, ("config.json",)): return legacy except Exception: pass return repo_id _LOAD_PROBE_MEMO = contextvars.ContextVar("_unsloth_load_probe_memo", default = None) def _load_probe_memo_scope(fn): @functools.wraps(fn) def _wrapper(*args, **kwargs): if _LOAD_PROBE_MEMO.get() is not None: return fn(*args, **kwargs) scope = _LOAD_PROBE_MEMO.set({}) try: return fn(*args, **kwargs) finally: _LOAD_PROBE_MEMO.reset(scope) return _wrapper # Failed fetches retry after a short pause, not per load: a firewall would cost each load the timeout. _NEW_MAPPER_TTL_S = 3600.0 _NEW_MAPPER_RETRY_S = 300.0 _NEW_MAPPER_CACHE = {"entry": None} # (tables, fetched_at): one slot, never a torn pair def _new_mapper_tables(): memo = _LOAD_PROBE_MEMO.get() if memo is not None and "new_mapper" in memo: return tuple(dict(table) for table in memo["new_mapper"]) entry = _NEW_MAPPER_CACHE["entry"] tables = None if entry is not None: cached, at = entry age = _time_monotonic() - at if 0 <= age < (_NEW_MAPPER_TTL_S if any(cached) else _NEW_MAPPER_RETRY_S): tables = tuple(dict(table) for table in cached) if tables is None: # No lock around the fetch: a fork while another thread is fetching must not leave the child waiting. tables = _get_new_mapper() _NEW_MAPPER_CACHE["entry"] = (tuple(dict(table) for table in tables), _time_monotonic()) if memo is not None and any(tables): memo["new_mapper"] = tuple(dict(table) for table in tables) return tables def _copy_error(error): try: clone = copy.copy(error) except Exception: return None if type(clone) is not type(error) or str(clone) != str(error): return None # copy.copy shares exception notes. notes = getattr(error, "__notes__", None) if isinstance(notes, list): clone.__notes__ = list(notes) clone.__cause__ = error.__cause__ clone.__context__ = error.__context__ clone.__suppress_context__ = error.__suppress_context__ clone.__traceback__ = None return clone def _memo_config_probe(loader, model_name, **kwargs): memo = _LOAD_PROBE_MEMO.get() if memo is None or kwargs.get("trust_remote_code") or not isinstance(model_name, str): return loader(model_name, **kwargs) if kwargs.get("cache_dir", 0) is None: kwargs.pop("cache_dir") try: key = (loader, model_name, tuple(sorted(kwargs.items())), _env_says_offline()) hash(key) except TypeError: return loader(model_name, **kwargs) hit = memo.get(key) if hit is not None: result, error = hit if error is not None: raise _copy_error(error) return copy.deepcopy(result) try: result = loader(model_name, **kwargs) except ImportError: raise except Exception as error: if not _is_offline_related_error(error): stored = _copy_error(error) if stored is not None and _copy_error(stored) is not None: memo[key] = (None, stored) raise try: memo[key] = (copy.deepcopy(result), None) except Exception: pass return result def get_model_name( model_name, load_in_4bit = True, load_in_fp8 = False, token = None, trust_remote_code = False, local_files_only = False, cache_dir = None, revision = None, ): assert load_in_fp8 in (True, False, "block") new_model_name = _resolve_with_mappers( model_name = model_name, load_in_4bit = load_in_4bit, load_in_fp8 = load_in_fp8, int_to_float = INT_TO_FLOAT_MAPPER, float_to_int = FLOAT_TO_INT_MAPPER, map_to_unsloth_16bit = MAP_TO_UNSLOTH_16bit, ) if ( new_model_name is not None and type(new_model_name) is str and new_model_name.lower() in BAD_MAPPINGS ): new_model_name = BAD_MAPPINGS[new_model_name.lower()] elif new_model_name is None and model_name.lower() in BAD_MAPPINGS: # Some bad names (the -unsloth-bnb-4bit dynamic quants) are keys of the mappers, not values, so the resolver returns None and the remap above is skipped; remap the input name directly. new_model_name = BAD_MAPPINGS[model_name.lower()] if ( new_model_name is None and model_name.count("/") == 1 and model_name[0].isalnum() and not _env_says_offline() # offline: skip the remote (raw GitHub) mapper refresh ): ( NEW_INT_TO_FLOAT_MAPPER, NEW_FLOAT_TO_INT_MAPPER, NEW_MAP_TO_UNSLOTH_16bit, NEW_FP8_BLOCK_MAPPER, NEW_FP8_ROW_MAPPER, ) = _new_mapper_tables() upgraded_model_name = _resolve_with_mappers( model_name = model_name, load_in_4bit = load_in_4bit, load_in_fp8 = load_in_fp8, int_to_float = NEW_INT_TO_FLOAT_MAPPER, float_to_int = NEW_FLOAT_TO_INT_MAPPER, map_to_unsloth_16bit = NEW_MAP_TO_UNSLOTH_16bit, # The fp8 probe has to look at the FETCHED tables too, or a new fp8 repo would miss both here and in the installed tables and skip the upgrade message. fp8_block = NEW_FP8_BLOCK_MAPPER, fp8_row = NEW_FP8_ROW_MAPPER, ) if upgraded_model_name is not None: raise NotImplementedError( f"Unsloth: {model_name} is not supported in your current Unsloth version! Please update Unsloth via:\n\n" "pip uninstall unsloth unsloth_zoo -y\n" 'pip install --upgrade --no-cache-dir "unsloth[colab-new] @ git+https://github.com/unslothai/unsloth.git"\n' 'pip install --upgrade --no-cache-dir "git+https://github.com/unslothai/unsloth-zoo.git"\n' ) if new_model_name is None: new_model_name = model_name else: # Also when equal to the input: main returned it lowercased. A real remap drops the revision. same_repo = new_model_name.lower() == str(model_name).lower() new_model_name = _prefer_legacy_lowercase_cache( new_model_name, local_files_only, cache_dir, revision if same_repo else None ) return new_model_name def _refuse_unless_private(path, what): """On POSIX, refuse a path that is a symlink, not ours, or writable by group/other.""" import stat if not hasattr(os, "getuid"): return info = os.lstat(path) if stat.S_ISLNK(info.st_mode) or info.st_uid != os.getuid(): raise RuntimeError( f"Unsloth: refusing {what} that is a symlink or owned by another user: {path}" ) if stat.S_IMODE(info.st_mode) & 0o022: raise RuntimeError( f"Unsloth: refusing a group- or world-writable {what} " f"(mode {stat.S_IMODE(info.st_mode):04o}): {path}" ) def _private_fp8_cache_dir(): """Per-user 0700 root for fp8 artifacts. The shared /tmp lets another account pre-create a predictable artifact name and have it loaded (with trust_remote_code) as this user.""" if hasattr(os, "getuid"): directory = os.path.join(tempfile.gettempdir(), "unsloth-fp8-%d" % os.getuid()) else: # No owner check on Windows, and TEMP can be shared (services, build agents), so use the # profile's LocalAppData, which Windows ACLs to this account. base = os.environ.get("LOCALAPPDATA") or os.path.join( os.path.expanduser("~"), "AppData", "Local" ) directory = os.path.join(base, "unsloth", "fp8-cache") os.makedirs(directory, mode = 0o700, exist_ok = True) if hasattr(os, "getuid"): info = os.lstat(directory) # exist_ok does not apply mode, so tighten one of ours; anyone else's is refused below. if not os.path.islink(directory) and info.st_uid == os.getuid() and info.st_mode & 0o077: try: os.chmod(directory, 0o700) except OSError: pass _refuse_unless_private(directory, "an fp8 cache directory") return directory def _offline_quantize_to_fp8( model_name: str, fp8_mode: str, *, text_only: bool = False, revision: str = None, token = None, trust_remote_code: bool = False, ) -> str: """Quantize the model to fp8 via torchao, save to a temp dir, return its path. For vllm >= 0.12.0, prefer dynamic quantization in vllm instead (via hf_overrides={"quantization_config_file": "torchao_config.json"}). The caller's revision has to reach the source loads, and the cache name has to name it too: the returned path replaces model_name, so the revision gate downstream drops the pin, and two refs of one repo would otherwise share a single artifact.""" from transformers import ( AutoModelForCausalLM, AutoModelForImageTextToText, AutoTokenizer, AutoProcessor, TorchAoConfig, AutoConfig, ) config = AutoConfig.from_pretrained( model_name, revision = revision, token = token, trust_remote_code = trust_remote_code ) is_vlm = any( x.endswith(("ForConditionalGeneration", "ForVisionText2Text")) for x in (getattr(config, "architectures", None) or []) ) is_vlm = is_vlm or hasattr(config, "vision_config") # Decide text-only before the cache name so the fp8 artifact and its path stay in sync (#5816). text_config = None if text_only and hasattr(config, "vision_config"): from ._utils import ( _get_text_only_config, resolve_model_class, _is_family_text_decoder, ) candidate = _get_text_only_config(config, model_name) text_class = resolve_model_class(AutoModelForCausalLM, candidate) if text_class is not None and _is_family_text_decoder( getattr(config, "model_type", ""), getattr(candidate, "model_type", ""), ): text_config = candidate is_vlm = False temp_dir = _private_fp8_cache_dir() # Cache text-only and full-VLM artifacts separately so neither reuses the other (#5816). cache_name = model_name.split("/")[-1] + "-fp8-" + fp8_mode # The basename alone would let org-a/model and org-b/model share one artifact. source = os.path.abspath(model_name) if os.path.isdir(model_name) else model_name cache_name += "-" + hashlib.sha256(source.encode("utf-8")).hexdigest()[:12] if revision is not None: # Sanitizing is lossy (release/v1 and release.v1 collapse), so a digest of the raw ref rides along and two refs never share an artifact. digest = hashlib.sha256(revision.encode("utf-8")).hexdigest()[:12] readable = re.sub(r"[^0-9A-Za-z_-]", "_", revision)[:40] cache_name += "-rev-" + readable + "-" + digest if text_config is not None: cache_name += "-text-only" new_model_name = os.path.join(temp_dir, cache_name) print(f"Unsloth: Quantizing '{model_name}' to fp8, using model_name='{new_model_name}' instead") if os.path.lexists(new_model_name): _refuse_unless_private(new_model_name, "a cached fp8 artifact") if not os.path.isdir(new_model_name): from ._utils import _apply_text_only_key_mapping qconfig = _get_torchao_fp8_config(fp8_mode) qconfig = TorchAoConfig(qconfig) # Under torchrun every rank converts: keep each on its own GPU, not all of them on "auto". local_rank = os.environ.get("LOCAL_RANK") device_map = ( {"": int(local_rank)} if local_rank is not None and int(os.environ.get("WORLD_SIZE", "1")) > 1 else "auto" ) load_kwargs = dict( torch_dtype = "auto", device_map = device_map, quantization_config = qconfig, trust_remote_code = trust_remote_code, ) if text_config is not None: _apply_text_only_key_mapping(load_kwargs, config, text_config) config = text_config auto_model = AutoModelForImageTextToText if is_vlm else AutoModelForCausalLM auto_processor = AutoProcessor if is_vlm else AutoTokenizer with sync_load_when_quantizing(qconfig, config): model = auto_model.from_pretrained( model_name, config = config, revision = revision, token = token, **load_kwargs, ) tokenizer = auto_processor.from_pretrained( model_name, revision = revision, token = token, trust_remote_code = trust_remote_code, ) # Atomic publish: the isdir check above must never see another rank's half-written artifact. partial = tempfile.mkdtemp(prefix = cache_name + ".partial-", dir = temp_dir) try: model.save_pretrained(partial, safe_serialization = False) del model for _ in range(2): clean_gpu_cache() gc.collect() tokenizer.save_pretrained(partial) try: os.rename(partial, new_model_name) except OSError: if not os.path.isdir(new_model_name): raise finally: shutil.rmtree(partial, ignore_errors = True) return new_model_name def _fp8_needs_offline_quantize(resolved_name, requested_name, fast_inference): """True when no pre-quantized FP8 repo was mapped and vLLM won't quantize it on the fly.""" if resolved_name is not None and str(resolved_name).lower() != str(requested_name).lower(): return False if not fast_inference or importlib.util.find_spec("vllm") is None: return True import vllm return Version(vllm.__version__) < Version("0.12.0") def _drop_fp8_dequantize_conversions(model): """Drop reverse FP8 conversions that would save unmarked FP8 weights as a 16-bit checkpoint.""" conversions = getattr(model, "_weight_conversions", None) if not conversions: return 0 try: from transformers.integrations.finegrained_fp8 import Fp8Dequantize except Exception: return 0 def _targets(conversion): patterns = getattr(conversion, "_original_target_patterns", None) or [] return tuple(p[:-1] if p.endswith("$") else p for p in patterns) def _sources(patterns): patterns = [p[:-1] if p.endswith("$") else p for p in patterns or []] return tuple( sorted( p for p in patterns if not p.endswith((".weight_scale_inv", ".activation_scale")) ) ) def _source_forms(conversion): return { _sources(getattr(conversion, "_original_source_patterns", None)), _sources(getattr(conversion, "source_patterns", None)), } def _ops(conversion): operations = getattr(conversion, "operations", None) or [] return tuple(type(op).__name__ for op in operations if not isinstance(op, Fp8Dequantize)) def _signature(conversion): return ( type(conversion), getattr(conversion, "scope_prefix", None), min(_source_forms(conversion)), _targets(conversion), _ops(conversion), ) def _has_dequant(conversion): return any( isinstance(op, Fp8Dequantize) for op in getattr(conversion, "operations", None) or [] ) plain = None kept, changed = [], 0 retained = {_signature(c) for c in conversions if not _has_dequant(c)} for conversion in conversions: operations = getattr(conversion, "operations", None) or [] if not _has_dequant(conversion): kept.append(conversion) continue if getattr(getattr(model, "config", None), "quantization_config", None) is None: # transformers dropped the fp8 config: 16bit weights, so no bitsandbytes stamp either. model._unsloth_fp8_dequantized = True if all(isinstance(op, Fp8Dequantize) for op in operations): changed += 1 continue if plain is None: try: from transformers.conversion_mapping import get_model_conversion_mapping plain = get_model_conversion_mapping(model, None, None) except Exception: plain = [] # Dequantization can lose scope_prefix; restore every matching scoped counterpart. scope = getattr(conversion, "scope_prefix", None) sources = _source_forms(conversion) matches = [ c for c in plain if type(c) is type(conversion) and _targets(c) == _targets(conversion) and _source_forms(c) & sources and _ops(c) == _ops(conversion) and (scope is None or getattr(c, "scope_prefix", None) == scope) ] if not matches: print( "Unsloth: an fp8 load conversion has no 16bit counterpart; save_pretrained may write fp8 " f"weights for {list(_targets(conversion))}." ) kept.append(conversion) continue for match in matches: signature = _signature(match) if signature not in retained: retained.add(signature) kept.append(match) changed += 1 if changed: model._weight_conversions = kept return changed def _loaded_quant_method(config): """quant_method of `config.quantization_config` as a lower-case string, dict or config object, else None.""" qc = getattr(config, "quantization_config", None) if qc is None: return None method = qc.get("quant_method") if isinstance(qc, dict) else getattr(qc, "quant_method", None) if method is None: return None return str(getattr(method, "value", method)).lower() def _fp8_dequantize_quantization_config(model_config): """The checkpoint's fine-grained fp8 config with `dequantize = True` for an explicit 16bit load, else None. On FP8-capable GPUs transformers otherwise keeps FP8Linear layers; FastModel passes the same flag.""" quantization_config = getattr(model_config, "quantization_config", None) if quantization_config is None: return None as_dict = ( quantization_config if isinstance(quantization_config, dict) else getattr(quantization_config, "to_dict", lambda: {})() ) quant_method = as_dict.get("quant_method", "") # A config object's to_dict() can keep the QuantizationMethod enum, whose str() is not its value. if str(getattr(quant_method, "value", quant_method)).lower() != "fp8": return None try: from transformers import FineGrainedFP8Config except Exception: return None if "dequantize" not in inspect.signature(FineGrainedFP8Config).parameters: return None try: return FineGrainedFP8Config.from_dict(dict(as_dict), dequantize = True) except Exception: return None def _fp8_load_as_is( model_name, revision = None, token = None, trust_remote_code = False, ): """Only a readable, unquantized model config takes the offline fp8 path and its H100 gate: an already quantized checkpoint, an adapter (resolved to its base later) or an unreadable config loads as before.""" from transformers import AutoConfig try: from peft import PeftConfig # The loader treats a repo holding both config.json and adapter_config.json as an adapter too. PeftConfig.from_pretrained(model_name, revision = revision, token = token) return True except Exception: pass try: config = AutoConfig.from_pretrained( model_name, revision = revision, token = token, trust_remote_code = trust_remote_code ) except Exception: # Unreadable here (a Mistral-format checkpoint, say): leave it to the loader's own fallbacks. return True text_config = getattr(config, "text_config", None) return bool( getattr(config, "quantization_config", None) or getattr(text_config, "quantization_config", None) ) class _Float8TensorLinear(torch.autograd.Function): """torchao's Float8Tensor linear has no backward, so a LoRA layer Unsloth's kernels do not take (a biased Qwen2 q_proj, FastModel's PEFT path) failed in backward.""" @staticmethod def forward(ctx, X, weight, bias): ctx.weight = weight ctx.bias_needs_grad = bias is not None and bias.requires_grad return torch.nn.functional.linear(X, weight, bias) @staticmethod def backward(ctx, dY): dX = dY @ ctx.weight.dequantize().to(dY.dtype) dB = dY.reshape(-1, dY.shape[-1]).sum(0) if ctx.bias_needs_grad else None return dX, None, dB def _float8_tensor_linear_forward(self, input): # A trainable fp8 weight keeps torchao's own (backward-less) path, so it errors instead of silently getting no gradient. if torch.is_grad_enabled() and input.requires_grad and not self.weight.requires_grad: return _Float8TensorLinear.apply(input, self.weight, self.bias) return torch.nn.functional.linear(input, self.weight, self.bias) def _enable_float8_tensor_backward(model: torch.nn.Module): from ..kernels.utils import Float8Tensor for module in model.modules(): weight = getattr(module, "weight", None) if type(module) is torch.nn.Linear and isinstance( getattr(weight, "data", weight), Float8Tensor ): module.forward = types.MethodType(_float8_tensor_linear_forward, module) def _tag_model_with_fp8_torchao_config(model: torch.nn.Module, fp8_mode: str): """Tag a model with a `TorchAOConfig` so downstream callers know how to handle it.""" try: _enable_float8_tensor_backward(model) except Exception: pass try: base_config = _get_torchao_fp8_config(fp8_mode) model.torchao_config = TorchAOConfig( qat_scheme = None, base_config_and_filter_fns = [(base_config, None)], ) except: pass _FP8_DTYPES = tuple( dtype for dtype in (getattr(torch, "float8_e4m3fn", None), getattr(torch, "float8_e5m2", None)) if dtype is not None ) def _fp8_block_size_from_config(model): """Return the [block_out, block_in] block size of an fp8 checkpoint, or None if not block-fp8.""" config = getattr(model, "config", None) quant = getattr(config, "quantization_config", None) if quant is None: return None if hasattr(quant, "to_dict"): quant = quant.to_dict() if not isinstance(quant, dict): return None if quant.get("quant_method") != "fp8": return None block = quant.get("weight_block_size") if not block: return None if isinstance(block, (int, float)): block = [block, block] elif isinstance(block, (list, tuple)): if len(block) == 1: block = [block[0], block[0]] elif len(block) < 2: return None else: return None return [int(block[0]), int(block[1])] def backfill_fp8_modules_to_convert( config, model_name, token = None, revision = None, local_files_only = False, subfolder = None, cache_dir = None, variant = None, ): """List per-tensor fp8 tables (`.weight_scale`) a config leaves out of `modules_to_convert` (Qwen3.8-Flash-Next-FP8).""" quant = getattr(config, "quantization_config", None) if ( not isinstance(quant, dict) or quant.get("quant_method") != "fp8" or quant.get("modules_to_convert") ): return None try: weight_map = _load_fp8_weight_map( model_name, local_files_only, token, revision, subfolder, cache_dir, variant = variant, index_only = True, # runs before the guarded weight download: never fetch the weights here ) except Exception: return None skip = set(quant.get("modules_to_not_convert") or []) modules = {k[: -len(".weight_scale")] for k in weight_map or {} if k.endswith(".weight_scale")} # Linear blocks carry weight_scale_inv; only embeddings match, and the quantizer converts nn.Embedding only. # Keep the layer index: the quantizer matches by suffix, so a bare name would also convert another layer's dense table. found = sorted({re.sub(r"^.*\.(?=layers\.\d+\.)", "", m) for m in modules - skip}) if found: quant["modules_to_convert"] = found print( f"Unsloth: {model_name} ships fp8 scales for {found} without listing them in modules_to_convert; added." ) return found or None def _load_fp8_weight_map( model_name, local_files_only, token, revision = None, subfolder = None, cache_dir = None, variant = None, index_only = False, ): """Tensor->file map from the load's snapshot; sharded index, else single `model.safetensors`. `index_only` never downloads a remote single file just to list its keys.""" def _local_path(filename): return ( os.path.join(model_name, subfolder, filename) if subfolder else os.path.join(model_name, filename) ) def _remote_path(filename): from huggingface_hub import hf_hub_download return hf_hub_download( model_name, filename, revision = revision, subfolder = subfolder, cache_dir = cache_dir, local_files_only = local_files_only, token = token, ) index_file = ( f"model.safetensors.index.{variant}.json" if variant else "model.safetensors.index.json" ) single_file = f"model.{variant}.safetensors" if variant else "model.safetensors" is_local = os.path.isdir(model_name) if is_local and os.path.exists(_local_path(index_file)): index_path = _local_path(index_file) elif not is_local: # Errors after this point are per-tensor: warn and continue, never abort or hide them. try: index_path = _remote_path(index_file) except Exception: index_path = None else: index_path = None if index_path is not None: import json with open(index_path, "r", encoding = "utf-8") as f: return json.load(f).get("weight_map", None) try: if is_local and os.path.exists(_local_path(single_file)): single_path = _local_path(single_file) elif not is_local and not index_only: single_path = _remote_path(single_file) else: return None from safetensors import safe_open with safe_open(single_path, framework = "pt") as f: return {key: single_file for key in f.keys()} except Exception: return None def _resolve_fp8_shard( model_name, shard, local_files_only, token, revision = None, subfolder = None, cache_dir = None, ): """Resolve a checkpoint shard filename to a local path (repo id or local dir).""" if os.path.isdir(model_name): return ( os.path.join(model_name, subfolder, shard) if subfolder else os.path.join(model_name, shard) ) from huggingface_hub import hf_hub_download return hf_hub_download( model_name, shard, revision = revision, subfolder = subfolder, cache_dir = cache_dir, local_files_only = local_files_only, token = token, ) def _match_fp8_module(module_by_name, base): """Resolve a checkpoint module name to a live module, allowing for VLM key remappings. VLM loads can name the text tower differently from the checkpoint keys: `text_only=True` strips the `language_model.` wrapper, and full VLM loads may expose `model.language_model.*` while the checkpoint stores `language_model.model.*`. Try the raw key first, then a few safe remappings.""" if base in module_by_name: return module_by_name[base] candidates = [] if "language_model." in base: candidates.append(base.replace("language_model.", "", 1)) # text-only: drop wrapper if "language_model.model." in base: candidates.append(base.replace("language_model.model.", "model.language_model.", 1)) if base.startswith("language_model."): candidates.append("model." + base) for candidate in candidates: if candidate in module_by_name: return module_by_name[candidate] return None _CT_FP8_STRATEGIES = ("tensor", "channel", "block") def _compressed_tensors_fp8_block_size(module, weights): """(out, in) block of a routable compressed-tensors FP8 weight, else None.""" if getattr(weights, "type", None) != "float" or getattr(weights, "num_bits", None) != 8: return None strategy = getattr(weights, "strategy", None) strategy = str(getattr(strategy, "value", strategy)) # Block FP8 is opt-in: fp8_block_quant_linear is far slower than the decompressed bf16 model. if ( strategy == "block" and os.environ.get("UNSLOTH_COMPRESSED_TENSORS_FP8_BLOCK_KERNELS", "0") != "1" ): return None if ( strategy not in _CT_FP8_STRATEGIES or getattr(weights, "dynamic", False) or getattr(weights, "actorder", None) ): return None weight = getattr(module, "weight", None) scale = getattr(module, "weight_scale", None) if weight is None or scale is None or weight.dtype != torch.float8_e4m3fn or weight.dim() != 2: return None zero_point = getattr(module, "weight_zero_point", None) if zero_point is not None and bool(torch.any(zero_point != 0)): return None out_features, in_features = weight.shape if strategy == "tensor": # Routed as per-channel (scale broadcast per row): the rowwise paths beat the block kernel. return [1, in_features] if scale.numel() == 1 else None if strategy == "channel": return ( [1, in_features] if tuple(scale.shape) in ((out_features, 1), (out_features,)) else None ) block = list(getattr(weights, "block_structure", None) or ()) if len(block) != 2 or scale.dim() != 2: return None if tuple(scale.shape) != (-(-out_features // block[0]), -(-in_features // block[1])): return None return block def _save_compressed_tensors_scale_shape(module, state_dict, prefix, local_metadata): # Saved checkpoints keep the config's scale shape; a per-tensor scale was broadcast per row. key = prefix + "weight_scale" shape = getattr(module, "_unsloth_ct_scale_shape", None) scale = state_dict.get(key) if shape is None or scale is None or tuple(scale.shape) == shape: return state_dict numel = 1 for n in shape: numel *= n state_dict[key] = scale.reshape(-1)[:1].reshape(shape) if numel == 1 else scale.reshape(shape) return state_dict def _load_compressed_tensors_scale_shape(module, state_dict, prefix, *args): key = prefix + "weight_scale" scale = state_dict.get(key) current = getattr(module, "weight_scale", None) if scale is None or current is None or scale.shape == current.shape: return if scale.numel() == 1: state_dict[key] = scale.reshape(1, 1).expand(current.shape).contiguous() elif scale.numel() == current.numel(): state_dict[key] = scale.reshape(current.shape) def _unsloth_compressed_tensors_fp8_forward(self, input): from unsloth.kernels.fp8 import can_use_fp8_rowwise_gemv, fp8_linear, fp8_rowwise_gemv if can_use_fp8_rowwise_gemv(input, self.weight, self.weight_scale): out = fp8_rowwise_gemv(input, self.weight, self.weight_scale) else: out = fp8_linear(input, self.weight, self.weight_scale) if self.bias is not None: out = out + self.bias.to(out.dtype) return out @functools.lru_cache(maxsize = 1) def _zoo_peft_forward_keeps_fp8_inputs(): """Older zoo compiled LoRA forwards cast `x` to the FP8 weight dtype; route only when it does not.""" try: import inspect from unsloth_zoo import compiler source = inspect.getsource(compiler.patch_lora_forwards) except Exception: return False return "x = x.to(self.base_layer.weight.dtype)" not in source or ( "self.base_layer.weight.dtype in (torch.float16, torch.bfloat16, torch.float32)" in source ) def _route_compressed_tensors_fp8_to_unsloth( model, skip = (), default = "0", ): # Opt-in for FP8 checkpoints (on-the-fly dequant trains slower than bf16); default beside NVFP4. # decompressed FP8 layers would keep compressed-tensors' graph-breaking fake-quant forward. if os.environ.get("UNSLOTH_COMPRESSED_TENSORS_FP8_KERNELS", default) != "1": return 0 if getattr(getattr(model, "config", None), "quantization_config", None) is None: return 0 if not _zoo_peft_forward_keeps_fp8_inputs(): return 0 # The fused CE loss multiplies lm_head.weight directly, so a quantized lm_head is decompressed, never routed. get_output_embeddings = getattr(model, "get_output_embeddings", None) lm_head = get_output_embeddings() if callable(get_output_embeddings) else None routable, decompress = [], [] for module in model.modules(): # Modules another route already owns (NVFP4) do not count against the all-or-nothing check. if module in skip: continue scheme = getattr(module, "quantization_scheme", None) if scheme is None or getattr(scheme, "weights", None) is None: continue if module is lm_head: decompress.append(module) continue block = None weight = getattr(module, "weight", None) on_gpu = isinstance(weight, torch.Tensor) and weight.device.type in ("cuda", "xpu") if ( on_gpu and isinstance(module, torch.nn.Linear) and getattr(scheme, "output_activations", None) is None ): block = _compressed_tensors_fp8_block_size(module, scheme.weights) if block is None: return 0 routable.append((module, block)) if not routable: return 0 if decompress: try: with torch.inference_mode(False), torch.no_grad(): for module in decompress: _decompress_like_a_full_load(module) _remove_same_device_compressed_tensors_offload(module) except Exception: return 0 converted = 0 compute_dtype = _fp8_dequant_target_dtype(model) for module, block in routable: scale = module.weight_scale scale_shape = tuple(scale.shape) if scale.numel() == 1: scale.data = scale.data.reshape(1, 1).expand(module.weight.shape[0], 1).contiguous() elif scale.dim() == 1: scale.data = scale.data.view(-1, 1) if tuple(scale.shape) != scale_shape: module._unsloth_ct_scale_shape = scale_shape module._register_state_dict_hook(_save_compressed_tensors_scale_shape) module._register_load_state_dict_pre_hook( _load_compressed_tensors_scale_shape, with_module = True ) module.weight.requires_grad_(False) scale.requires_grad_(False) if block != [1, module.weight.shape[1]]: module.weight.block_size = block scale.block_size = block module.block_size = block module.forward = _unsloth_compressed_tensors_fp8_forward.__get__(module) module._unsloth_compressed_tensors_fp8 = True module._unsloth_ct_compute_dtype = compute_dtype converted += 1 if converted: _remove_compressed_tensors_decompress_hook(model) _patch_peft_for_routed_compressed_tensors() model._unsloth_compressed_tensors_fp8 = converted return converted def _remove_compressed_tensors_decompress_hook(model): for owner in (model, getattr(model, "model", None), getattr(model, "base_model", None)): hook = getattr(owner, "ct_decompress_hook", None) if hook is not None: hook.remove() try: delattr(owner, "ct_decompress_hook") except AttributeError: pass def _nvfp4_scheme_supported(scheme): weights = getattr(scheme, "weights", None) if weights is None or getattr(scheme, "output_activations", None) is not None: return False enum = lambda v: str(getattr(v, "value", v)) if v is not None else None fmt = enum(getattr(scheme, "format", None)) return ( str(enum(getattr(weights, "type", None))) == "float" and getattr(weights, "num_bits", None) == 4 and getattr(weights, "group_size", None) == 16 and enum(getattr(weights, "strategy", None)) == "tensor_group" and not getattr(weights, "dynamic", False) # "static" is an alias of "weight": calibration order only, columns stored in place (no g_idx). and enum(getattr(weights, "actorder", None)) in (None, "weight", "static") and fmt in (None, "nvfp4-pack-quantized") ) def _nvfp4_module_routable(module): packed = getattr(module, "weight_packed", None) scale = getattr(module, "weight_scale", None) global_scale = getattr(module, "weight_global_scale", None) return ( isinstance(module, torch.nn.Linear) and isinstance(packed, torch.Tensor) and packed.dtype == torch.uint8 and packed.dim() == 2 and packed.device.type in ("cuda", "xpu") and isinstance(scale, torch.Tensor) and scale.dtype == torch.float8_e4m3fn and tuple(scale.shape) == (packed.shape[0], packed.shape[1] * 2 // 16) and isinstance(global_scale, torch.Tensor) and global_scale.numel() == 1 and getattr(module, "weight_zero_point", None) is None and "weight" not in module._parameters and type(module) is torch.nn.Linear ) def _unsloth_compressed_tensors_nvfp4_forward(self, input): from unsloth.kernels.nvfp4 import nvfp4_linear # W4A16: the checkpoint's NVFP4 activation quantization is not applied. out = nvfp4_linear(input, self.weight_packed, self.weight_scale, self.weight_global_scale) if self.bias is not None: out = out + self.bias.to(out.dtype) return out class _UnslothNVFP4Linear(torch.nn.Linear): """nn.Linear whose `weight` is the packed NVFP4 tensor carrying an NVFP4QuantState. PEFT and get_lora_parameters read `.weight`; the state_dict keeps only weight_packed, and the uint8 dtype keeps compiled LoRA forwards from casting activations to it. Built on each access so `.to()` / device moves stay valid. """ @property def weight(self): from unsloth.kernels.nvfp4 import NVFP4QuantState packed = self.weight_packed packed.quant_state = NVFP4QuantState( self.weight_scale, self.weight_global_scale, (packed.shape[0], packed.shape[1] * 2), getattr(self, "_unsloth_nvfp4_dtype", torch.bfloat16), ) return packed def _apply( self, fn, recurse = True, ): # model.to(dtype) must never cast the packed weight or scales: an fp16 global scale overflows. quant = {} for name in ("weight_packed", "weight_scale", "weight_global_scale", "input_global_scale"): param = self._parameters.get(name) if param is not None: quant[name] = self._parameters.pop(name) try: super()._apply(fn, recurse) finally: for name, param in quant.items(): device = fn(param.data[:0]).device if param.device != device: param = torch.nn.Parameter(param.data.to(device), requires_grad = False) self._parameters[name] = param # The compute dtype follows the cast (model.half() dequantizes to fp16 from then on). dtype = getattr(self, "_unsloth_nvfp4_dtype", torch.bfloat16) self._unsloth_nvfp4_dtype = fn(torch.empty(0, dtype = dtype)).dtype return self def dequantize_(self): """Turn this layer into a plain nn.Linear holding the dense weight (PEFT merges add into `.weight`).""" self.__dict__.pop("forward", None) self.__class__ = torch.nn.Linear self._unsloth_compressed_tensors_nvfp4 = False _decompress_like_a_full_load(self) return self _CT_QUANT_PARAMS = ( "weight_scale", "weight_global_scale", "weight_zero_point", "input_global_scale", "input_scale", ) def _decompress_like_a_full_load(module): """compressed-tensors' own per-module decompress, then a plain dense layer. The config still says compressed for the rest, so the layer joins its ignore list: save_pretrained then writes a checkpoint that reloads.""" try: from compressed_tensors.compressors import decompress_module except ImportError: decompress_module = None # compressed-tensors < 0.13 with torch.inference_mode(False), torch.no_grad(): if decompress_module is not None: decompress_module(module) else: _dequantize_in_place_without_compressed_tensors(module) for name in _CT_QUANT_PARAMS: module._parameters.pop(name, None) for attr in ("quantization_scheme", "quantization_status"): module.__dict__.pop(attr, None) for p in module.parameters(recurse = False): p.requires_grad_(False) ct_config = getattr(getattr(module, "_unsloth_ct_config", None), "quantization_config", None) name = getattr(module, "_unsloth_ct_name", None) if ct_config is None or name is None: return if getattr(ct_config, "ignore", None) is None: ct_config.ignore = [] if name not in ct_config.ignore: ct_config.ignore.append(name) def _dequantize_in_place_without_compressed_tensors(module): """Same dense weight from Unsloth's kernels (bit-identical to compressed-tensors for NVFP4).""" dtype = getattr(module, "_unsloth_nvfp4_dtype", None) or getattr( module, "_unsloth_ct_compute_dtype", torch.bfloat16 ) if isinstance(getattr(module, "weight_packed", None), torch.Tensor): from unsloth.kernels.nvfp4 import nvfp4_dequantize W = nvfp4_dequantize( module.weight_packed, module.weight_scale, module.weight_global_scale, dtype ) module._parameters.pop("weight_packed", None) else: from unsloth.kernels.fp8 import weight_dequant W = weight_dequant(module.weight, module.weight_scale, dtype) module.weight = torch.nn.Parameter(W, requires_grad = False) def _tag_compressed_tensors_modules(model): config = getattr(getattr(model, "config", None), "quantization_config", None) for name, module in model.named_modules(): if getattr(module, "quantization_scheme", None) is not None: module._unsloth_ct_name = name module._unsloth_ct_config = config def _routed_dense_weight(module): """Dense weight of an NVFP4- or FP8-routed linear, else None.""" if isinstance(module, _UnslothNVFP4Linear): from unsloth.kernels.nvfp4 import nvfp4_dequantize return nvfp4_dequantize( module.weight_packed, module.weight_scale, module.weight_global_scale, getattr(module, "_unsloth_nvfp4_dtype", torch.bfloat16), ) if getattr(module, "_unsloth_compressed_tensors_fp8", False): from unsloth.kernels.fp8 import weight_dequant return weight_dequant( module.weight, module.weight_scale, getattr(module, "_unsloth_ct_compute_dtype", torch.bfloat16), ) return None def _dequantize_routed_linear_(module): """Turn a routed linear into a plain nn.Linear with its dense weight, for PEFT merges.""" if isinstance(module, _UnslothNVFP4Linear): return module.dequantize_() if getattr(module, "_unsloth_compressed_tensors_fp8", False): module.__dict__.pop("forward", None) module._unsloth_compressed_tensors_fp8 = False _decompress_like_a_full_load(module) return module def _patch_peft_for_routed_compressed_tensors(): """PEFT merges add a dense delta into base_layer.weight and DoRA reads it: give both the dense weight.""" try: from peft.tuners.lora import layer as lora_layer except Exception: return merge = lora_layer.Linear.merge if not getattr(merge, "_unsloth_routed", False): @functools.wraps(merge) def patched_merge(self, *args, **kwargs): _dequantize_routed_linear_(self.get_base_layer()) return merge(self, *args, **kwargs) patched_merge._unsloth_routed = True lora_layer.Linear.merge = patched_merge # PiSSA / OLoRA / CorDA / LoftQ / orthogonal / LoRA-GA read or rewrite the dense base weight (or its dtype). for init in ( "pissa_init", "olora_init", "corda_init", "loftq_init", "orthogonal_init", "lora_ga_init", ): original = getattr(lora_layer.LoraLayer, init, None) if original is None or getattr(original, "_unsloth_routed", False): continue def patched_init( self, *args, _original = original, **kwargs, ): _dequantize_routed_linear_(self.get_base_layer()) return _original(self, *args, **kwargs) patched_init = functools.wraps(original)(patched_init) patched_init._unsloth_routed = True setattr(lora_layer.LoraLayer, init, patched_init) # LoRA-GA / CorDA preprocessing reads each target's weight (or enables its grad) before get_peft_model. for module_name, attr in (("loraga", "get_target_modules"), ("corda", "target_modules")): try: owner = importlib.import_module("peft.tuners.lora." + module_name) except Exception: continue targets = getattr(owner, attr, None) if targets is None or getattr(targets, "_unsloth_routed", False): continue def patched_targets( *args, _targets = targets, **kwargs, ): for name, module in _targets(*args, **kwargs): yield name, _dequantize_routed_linear_(module) patched_targets = functools.wraps(targets)(patched_targets) patched_targets._unsloth_routed = True setattr(owner, attr, patched_targets) try: from peft.tuners.lora import dora except Exception: return dequantize = getattr(dora, "dequantize_module_weight", None) if dequantize is not None and not getattr(dequantize, "_unsloth_routed", False): @functools.wraps(dequantize) def patched_dequantize(module): W = _routed_dense_weight(module) return dequantize(module) if W is None else W patched_dequantize._unsloth_routed = True dora.dequantize_module_weight = patched_dequantize update_layer = dora.DoraLinearLayer.update_layer if not getattr(update_layer, "_unsloth_routed", False): @functools.wraps(update_layer) def patched_update_layer(self, *, base_layer, lora_A, lora_B, **kwargs): # PEFT has cast new adapters to an FP8 base's float8 dtype; DoRA multiplies them right away. for p in (lora_A, lora_B): if isinstance(p, torch.Tensor) and p.dtype in _FP8_DTYPES: p.data = p.data.to(torch.float32) return update_layer(self, base_layer = base_layer, lora_A = lora_A, lora_B = lora_B, **kwargs) patched_update_layer._unsloth_routed = True dora.DoraLinearLayer.update_layer = patched_update_layer def _route_compressed_tensors_nvfp4_to_unsloth(model): """Keep NVFP4 (nvfp4-pack-quantized) weights packed and run them W4A16; decompress or FP8-route the rest.""" if os.environ.get("UNSLOTH_COMPRESSED_TENSORS_NVFP4_KERNELS", "1") == "0": return 0 if getattr(getattr(model, "config", None), "quantization_config", None) is None: return 0 try: from compressed_tensors.compressors import decompress_module except Exception: decompress_module = None # The fused CE loss and decode read lm_head.weight directly, so an NVFP4 lm_head is decompressed too. get_output_embeddings = getattr(model, "get_output_embeddings", None) lm_head = get_output_embeddings() if callable(get_output_embeddings) else None nvfp4, others = [], [] for module in model.modules(): scheme = getattr(module, "quantization_scheme", None) if scheme is None or getattr(scheme, "weights", None) is None: continue if str(getattr(getattr(module, "quantization_status", None), "value", "")) != "compressed": continue if ( module is not lm_head and _nvfp4_scheme_supported(scheme) and _nvfp4_module_routable(module) ): nvfp4.append(module) else: others.append(module) if not nvfp4: return 0 if others and decompress_module is None: print( "Unsloth: this compressed-tensors has no per-module decompress; decompressing the NVFP4 layers too." ) return 0 compute_dtype = _fp8_dequant_target_dtype(model) # The FP8 kernels dequantize to bf16, so an fp16 model keeps its FP8 layers decompressed by default. fp8_routed = ( _route_compressed_tensors_fp8_to_unsloth( model, skip = set(nvfp4), default = "1" if compute_dtype == torch.bfloat16 else "0" ) if others else 0 ) if others and not fp8_routed: try: with torch.inference_mode(False), torch.no_grad(): for module in others: _decompress_like_a_full_load(module) _remove_same_device_compressed_tensors_offload(module) except Exception as e: print(f"Unsloth: could not decompress the non-NVFP4 compressed-tensors layers ({e}).") return 0 for module in nvfp4: for name in ("weight_packed", "weight_scale", "weight_global_scale", "input_global_scale"): param = getattr(module, name, None) if isinstance(param, torch.Tensor): param.requires_grad_(False) module.__class__ = _UnslothNVFP4Linear module._unsloth_nvfp4_dtype = compute_dtype module.forward = _unsloth_compressed_tensors_nvfp4_forward.__get__(module) module._unsloth_compressed_tensors_nvfp4 = True _remove_compressed_tensors_decompress_hook(model) _patch_peft_for_routed_compressed_tensors() model._unsloth_compressed_tensors_nvfp4 = len(nvfp4) return len(nvfp4) _FP8_DEQUANT_DTYPES = (torch.bfloat16, torch.float16, torch.float32) def _has_fp8_scale_attr(module): return any( isinstance(getattr(module, name, None), torch.Tensor) for name in ("weight_scale_inv", "weight_scale") ) def _fp8_dequant_target_dtype(model, dtype = None): candidates = [dtype] config = getattr(model, "config", None) candidates += [getattr(config, "dtype", None), getattr(config, "torch_dtype", None)] try: candidates.append(model.get_input_embeddings().weight.dtype) except Exception: pass for candidate in candidates: if isinstance(candidate, str): candidate = getattr(torch, candidate.replace("torch.", ""), None) if candidate in _FP8_DEQUANT_DTYPES: return candidate return torch.bfloat16 def _holds_raw_fp8_values(tensor, chunk_elements = 1 << 24): """True if every value lies on the e4m3/e5m2 grid; a folded-in scale falls off it, so never scale twice.""" if tensor.dtype in _FP8_DTYPES: return True if tensor.dtype not in _FP8_DEQUANT_DTYPES: return False flat = tensor.detach().reshape(-1) for fp8 in _FP8_DTYPES: for start in range(0, flat.numel(), chunk_elements): chunk = flat[start : start + chunk_elements] if not torch.equal(chunk.to(fp8).to(chunk.dtype), chunk): break else: return True return False def _apply_fp8_block_scale(weight, scale, bs0, bs1): out_features, in_features = weight.shape out_blocks = (out_features + bs0 - 1) // bs0 in_blocks = (in_features + bs1 - 1) // bs1 if out_features % bs0 == 0 and in_features % bs1 == 0: # In place on block views: a full expanded fp32 scale + copy can OOM. weight.view(out_blocks, bs0, in_blocks, bs1).mul_(scale[:, None, :, None]) return weight scale_expanded = scale.repeat_interleave(bs0, dim = 0).repeat_interleave(bs1, dim = 1)[ :out_features, :in_features ] return (weight.to(torch.float32) * scale_expanded).to(weight.dtype) def _accelerate_offload_store(model): for module in model.modules(): hook = getattr(module, "_hf_hook", None) if hook is None: continue for h in getattr(hook, "hooks", None) or (hook,): weights_map = getattr(h, "weights_map", None) if weights_map is None: continue return getattr(weights_map, "dataset", weights_map) return None def _offload_store_has(store, key): # `key in OffloadedWeightsLoader` would read the disk tensor. keys = getattr(store, "all_keys", None) if keys is not None: return key in keys try: return key in store except Exception: return False def _offload_store_set(store, key, value): state_dict = getattr(store, "state_dict", None) index = getattr(store, "index", None) save_folder = getattr(store, "save_folder", None) if ( save_folder and isinstance(index, dict) and key in index and not (isinstance(state_dict, dict) and key in state_dict) ): # Disk-offloaded: write a new .dat (the index entry may point at the read-only checkpoint), never RAM. from accelerate.utils import offload_weight offload_weight(value.detach().cpu(), key, save_folder, index = index) return # OffloadedWeightsLoader checks state_dict before the disk index, so this overrides cpu tensors. if isinstance(state_dict, dict): state_dict[key] = value elif isinstance(store, dict): store[key] = value else: raise TypeError(f"Unsloth: cannot write offloaded tensor into {type(store).__name__}") def _dequantize_multihead_attention_out_proj(model): """nn.MultiheadAttention passes out_proj.weight straight to F.linear, so a bitsandbytes 4-bit out_proj (SigLIP pooling head, granite-vision, #2672; transformers 4.x quantizes it) fails with "Half and Byte". Swap each for a float Linear holding the dequantized weight, as transformers 5 loads it. Returns the count.""" try: import bitsandbytes as bnb except Exception: return 0 n = 0 for module in model.modules(): if not isinstance(module, torch.nn.MultiheadAttention): continue out_proj = module.out_proj quant_state = getattr(getattr(out_proj, "weight", None), "quant_state", None) if not isinstance(out_proj, bnb.nn.Linear4bit) or quant_state is None: continue dtype = ( module.in_proj_weight.dtype if module.in_proj_weight is not None else quant_state.dtype ) weight = bnb.functional.dequantize_4bit(out_proj.weight.data, quant_state).to(dtype) new = torch.nn.modules.linear.NonDynamicallyQuantizableLinear( out_proj.in_features, out_proj.out_features, bias = out_proj.bias is not None, device = "meta", ) new.weight = torch.nn.Parameter(weight, requires_grad = False) if out_proj.bias is not None: new.bias = out_proj.bias if hasattr(out_proj, "_hf_hook"): from accelerate.hooks import add_hook_to_module add_hook_to_module(new, out_proj._hf_hook) module.out_proj = new n += 1 return n def _restore_dropped_fp8_scales( model, model_name, *, local_files_only = False, token = None, revision = None, subfolder = None, cache_dir = None, variant = None, dtype = None, ): """Re-apply block-fp8 `weight_scale_inv` tensors that transformers dropped on load. On some block-scale fp8 checkpoints (e.g. Qwen3.6-27B-FP8, issue #6200) transformers fails to convert a Linear such as `mlp.gate_proj` to an fp8 module, loading the raw quantized values into a plain bf16 weight and discarding its `weight_scale_inv` as an unexpected key, so the weight is used un-scaled and the model is garbage. For every checkpoint scale whose live weight is not fp8, dequantize the orphaned weight in place; correctly converted modules keep an fp8 weight and are skipped, so a healthy checkpoint is a no-op. A text_only-renamed key can instead leave raw fp8 values with no scale attribute; those are dequantized into `dtype`. Returns (restored, skipped).""" try: block = _fp8_block_size_from_config(model) if block is None or not _FP8_DTYPES: return (0, 0) # A variant load reads variant-named files; skip to avoid applying default scales to them. if variant: return (0, 0) # No fp8 params means the checkpoint was dequantized on purpose (load_in_16bit), and re-applying a scale would corrupt those already-correct 16bit weights. if not any(p.dtype in _FP8_DTYPES for p in model.parameters()): return (0, 0) weight_map = _load_fp8_weight_map( model_name, local_files_only, token, revision, subfolder, cache_dir ) if not weight_map: return (0, 0) scale_keys = {k: v for k, v in weight_map.items() if k.endswith(".weight_scale_inv")} if not scale_keys: return (0, 0) module_by_name = dict(model.named_modules()) name_by_module = {id(m): n for n, m in module_by_name.items()} _unset = object() offload_store = _unset bs0, bs1 = block restored = 0 skipped = 0 failed = 0 offloaded = 0 shard_cache = {} for scale_key, shard in scale_keys.items(): base = scale_key[: -len(".weight_scale_inv")] module = _match_fp8_module(module_by_name, base) if module is None: continue weight = getattr(module, "weight", None) if not isinstance(weight, torch.Tensor) or weight.ndim != 2: continue if weight.dtype in _FP8_DTYPES and _has_fp8_scale_attr(module): skipped += 1 continue store_key = None if weight.device.type == "meta": if offload_store is _unset: offload_store = _accelerate_offload_store(model) store_key = f"{name_by_module.get(id(module), '')}.weight" if offload_store is None or not _offload_store_has(offload_store, store_key): offloaded += 1 continue orphan_fp8 = weight.dtype in _FP8_DTYPES # Errors after this point are per-tensor: warn and continue, never abort or hide them. try: if shard not in shard_cache: from safetensors import safe_open shard_path = _resolve_fp8_shard( model_name, shard, local_files_only, token, revision, subfolder, cache_dir, ) shard_cache[shard] = safe_open(shard_path, framework = "pt") scale = shard_cache[shard].get_tensor(scale_key).to(torch.float32) out_features, in_features = weight.shape out_blocks = (out_features + bs0 - 1) // bs0 in_blocks = (in_features + bs1 - 1) // bs1 if tuple(scale.shape) == (out_blocks, in_blocks): pass elif tuple(scale.shape) == (in_blocks, out_blocks) and out_blocks != in_blocks: # Transposed block layout: same handling as the fp8 forward path. scale = scale.t().contiguous() else: # Shape does not match the block grid: skip rather than apply a wrong scale. continue stored = None if store_key is not None: stored = offload_store[store_key] if tuple(stored.shape) != tuple(weight.shape): continue orphan_fp8 = stored.dtype in _FP8_DTYPES source = stored if stored is not None else weight if source.is_floating_point() and not _holds_raw_fp8_values(source): # Scale already folded in: never double-scale. skipped += 1 continue target = _fp8_dequant_target_dtype(model, dtype) if orphan_fp8 else weight.dtype with torch.no_grad(): if store_key is not None: # Copy: the stored tensor can be a read-only disk memmap. value = stored.to(dtype = target, copy = True) value = _apply_fp8_block_scale(value, scale.to(value.device), bs0, bs1) _offload_store_set(offload_store, store_key, value) if weight.dtype != target: # accelerate casts the stored value to the placeholder dtype. module.weight.data = torch.empty_like(weight, dtype = target) else: # Swap in once so an OOM leaves the weight untouched, not widened but unscaled. value = weight.data.to(target) if orphan_fp8 else weight.data module.weight.data = _apply_fp8_block_scale( value, scale.to(weight.device), bs0, bs1 ) restored += 1 except Exception: failed += 1 continue if restored > 0: print(f"Unsloth: Restored {restored} dropped FP8 weight_scale_inv tensor(s) on load") if failed > 0: print(f"Unsloth: {failed} dropped FP8 weight_scale_inv tensor(s) could not be restored") if offloaded > 0: print( f"Unsloth: {offloaded} dropped FP8 weight_scale_inv tensor(s) skipped because the " "layer is offloaded outside accelerate's weights map; load without offload so the scales can be restored" ) return (restored, skipped) except Exception: return (0, 0) def _forward_calls_checkpointing(cls): """True when a backbone's own `forward` hands its layers to a checkpoint function.""" import inspect try: source = inspect.getsource(cls.forward) except Exception: return True # cannot read it: assume it does and change nothing return "_gradient_checkpointing_func" in source or "checkpoint(" in source def _checkpointed_layer_forward(original): @functools.wraps(original) def forward(self, *args, **kwargs): holder = self.__dict__.get("_unsloth_gradient_checkpointing_holder") holder = holder() if holder is not None else None if ( not self.training or holder is None or not getattr(holder, "gradient_checkpointing", False) # gradient_checkpointing_enable(every_n_layers = N) marks the layers it leaves out. or self.__dict__.get("_unsloth_every_n_skip", False) or not torch.is_grad_enabled() # A cache would be written a second time by the recompute. or kwargs.get("past_key_value") is not None or kwargs.get("past_key_values") is not None ): return original(self, *args, **kwargs) if not args and "hidden_states" in kwargs: # A reentrant checkpoint only tracks gradients through positional tensors. args = (kwargs.pop("hidden_states"),) # Grad-requiring kwargs (Kimi-K3 block_residual) go positional: a closure ties recompute to earlier layers. grad_keys = [k for k, v in kwargs.items() if torch.is_tensor(v) and v.requires_grad] n_args = len(args) grad_values = tuple(kwargs.pop(k) for k in grad_keys) def run(*inputs): return original( self, *inputs[:n_args], **kwargs, **dict(zip(grad_keys, inputs[n_args:])) ) # gradient_checkpointing_enable()'s function: keeps offloading and the caller's use_reentrant. checkpoint = getattr(holder, "_gradient_checkpointing_func", None) if checkpoint is None: return torch.utils.checkpoint.checkpoint(run, *args, *grad_values, use_reentrant = False) return checkpoint(run, *args, *grad_values) forward._unsloth_manual_checkpoint = True return forward def install_remote_gradient_checkpointing(model, verbose = True): """Wrap decoder layers of remote-code backbones whose loop ignores `self.gradient_checkpointing`.""" import types import weakref try: from transformers.modeling_layers import GradientCheckpointingLayer except Exception: GradientCheckpointingLayer = () wrapped = [] for _, holder in model.named_modules(): holder_cls = type(holder) if "transformers_modules" not in (getattr(holder_cls, "__module__", "") or ""): continue if not hasattr(holder, "gradient_checkpointing"): continue layers = getattr(holder, "layers", None) if not isinstance(layers, torch.nn.ModuleList) or len(layers) == 0: continue if _forward_calls_checkpointing(holder_cls): continue for layer in layers: cls = type(layer) if GradientCheckpointingLayer and isinstance(layer, GradientCheckpointingLayer): continue layer.__dict__["_unsloth_gradient_checkpointing_holder"] = weakref.ref(holder) if not getattr(cls.__dict__.get("forward"), "_unsloth_manual_checkpoint", False): cls.forward = _checkpointed_layer_forward(cls.forward) wrapped.append(cls.__name__) # device_map hooks keep the bound original in `_old_forward`. if "_old_forward" in vars(layer): layer._old_forward = types.MethodType(cls.forward, layer) wrapped = sorted(set(wrapped)) if wrapped and verbose: print( "Unsloth: the remote modeling code never calls a checkpoint function; wrapping " + ", ".join(wrapped) + " so gradient checkpointing actually saves activations." ) return wrapped def enable_composite_gradient_checkpointing(model, verbose = True): """Mark a remote-code outer class checkpointable when an inner PreTrainedModel is (Kimi-K2.7).""" try: from transformers import PreTrainedModel except Exception: return False if not isinstance(model, PreTrainedModel): return False # Always install: the flag is what transformers checks, the wrapper makes it act. install_remote_gradient_checkpointing(model, verbose = verbose) if getattr(type(model), "supports_gradient_checkpointing", False): return False inner = [ type(m).__name__ for name, m in model.named_modules() if name and isinstance(m, PreTrainedModel) and getattr(type(m), "supports_gradient_checkpointing", False) ] if not inner: return False type(model).supports_gradient_checkpointing = True if verbose: print( f"Unsloth: {type(model).__name__} did not declare gradient checkpointing support but its " f"{inner[0]} does; enabling it on the outer model." ) return True def _move_compressed_tensors_inputs(value, device): if isinstance(value, torch.Tensor): return value.to(device = device) if isinstance(value, list): return [_move_compressed_tensors_inputs(item, device) for item in value] if isinstance(value, tuple): return tuple(_move_compressed_tensors_inputs(item, device) for item in value) from dataclasses import fields, is_dataclass if is_dataclass(value): return type(value)( **{ field.name: _move_compressed_tensors_inputs(getattr(value, field.name), device) for field in fields(value) } ) if isinstance(value, dict): return {key: _move_compressed_tensors_inputs(item, device) for key, item in value.items()} return value def _align_compressed_tensors_inputs(module, args, kwargs): reference = next((p for p in module._parameters.values() if p is not None), None) if reference is None: reference = next((b for b in module._buffers.values() if b is not None), None) if reference is None: reference = next(module.parameters(), None) if reference is None: reference = next(module.buffers(), None) if reference is None: return args, kwargs if len(args) == 1 and not kwargs and isinstance(args[0], torch.Tensor): if args[0].device == reference.device: return None return (args[0].to(device = reference.device),), kwargs return _move_compressed_tensors_inputs((args, kwargs), reference.device) def _compressed_tensors_input_forward(original_forward, module): from functools import wraps mapping_name, tensor_name = next( ( (mapping, name) for mapping in ("_parameters", "_buffers") for name, value in getattr(module, mapping).items() if value is not None ), (None, None), ) @wraps(original_forward) def forward(module, *args, **kwargs): if ( mapping_name is not None and len(args) == 1 and not kwargs and isinstance(args[0], torch.Tensor) ): reference = getattr(module, mapping_name).get(tensor_name) if reference is not None: value = args[0] if value.device != reference.device: value = value.to(device = reference.device) return original_forward(module, value) aligned = _align_compressed_tensors_inputs(module, args, kwargs) if aligned is not None: args, kwargs = aligned return original_forward(module, *args, **kwargs) return forward def _remove_same_device_compressed_tensors_offload(model): try: from compressed_tensors.offload.cache import OffloadCache from compressed_tensors.offload.module import remove_module_offload except Exception: return 0 same_device = [] onload_devices = set() for module in model.modules(): cache = module._parameters if not isinstance(cache, OffloadCache): continue onload = getattr(cache, "onload_device", None) offload = getattr(cache, "offload_device", None) if onload is not None: onload_devices.add(torch.device(onload)) if onload is None or offload is None or torch.device(onload) != torch.device(offload): continue same_device.append(module) split = len(onload_devices) > 1 for module in same_device: remove_module_offload(module) if split: # The original mover disables autograd and compares data pointers. module.forward = _compressed_tensors_input_forward( module.forward.__func__, module ).__get__(module) return len(same_device) def _decompress_compressed_tensors_model(model): """Decompress at load: PEFT skips the first-forward hook, and in `generate` it yields inference tensors.""" if getattr(model, "_unsloth_compressed_tensors_fp8", 0) or getattr( model, "_unsloth_compressed_tensors_nvfp4", 0 ): return False quant_config = getattr(getattr(model, "config", None), "quantization_config", None) if isinstance(quant_config, dict): method = quant_config.get("quant_method", None) else: method = getattr(quant_config, "quant_method", None) if getattr(method, "value", method) != "compressed-tensors": return False # MXFP4 kept packed on purpose (all-or-nothing plan): decompress_model would drop weight_packed. if any(getattr(module, "_unsloth_mxfp4_packed_linear", False) for module in model.modules()): return False if not any( str(getattr(getattr(module, "quantization_status", None), "value", "")) == "compressed" for module in model.modules() ): return False compressor = getattr(getattr(model, "hf_quantizer", None), "compressor", None) if compressor is None or not hasattr(compressor, "decompress_model"): return False def _decompress(module, *args): with torch.inference_mode(False), torch.no_grad(): compressor.decompress_model(module) _remove_same_device_compressed_tensors_offload(module) # Some compressed-tensors releases (0.19.0) leave the hook in place: drop it, also after a retry. hook = getattr(model, "ct_decompress_hook", None) if hook is not None: hook.remove() try: delattr(model, "ct_decompress_hook") except AttributeError: pass # Swap the hook first: decompress_model removes it by name. hook = getattr(model, "ct_decompress_hook", None) if hook is not None: hook.remove() model.ct_decompress_hook = model.register_forward_pre_hook(_decompress) try: _decompress(model) except Exception as e: print(f"Unsloth: could not decompress the compressed-tensors checkpoint after load: {e}") return False return True def _dequantize_bitsandbytes_for_full_finetuning( model, dtype = None, model_name = "", ): """Full finetuning marks every weight trainable, which the uint8 / int8 weights of a pre-quantized bitsandbytes checkpoint (a local `-bnb-4bit` folder, which the name mapper cannot redirect) reject (#2613).""" quantizer = getattr(model, "hf_quantizer", None) method = getattr(getattr(quantizer, "quantization_config", None), "quant_method", None) # pre_quantized False = an explicit on-the-fly config; rounding then restoring would train a lossy copy. if str(getattr(method, "value", method)).lower() != "bitsandbytes" or not getattr( quantizer, "pre_quantized", True ): return False print( f"Unsloth: `{model_name}` is a pre-quantized bitsandbytes checkpoint, so full finetuning " "dequantizes it to 16bit. For the best accuracy, full finetune the original 16bit model instead." ) # transformers < 5 deletes these unguarded, and a composite's extracted text core lacks some of them. for owner, attribute in ( (model, "quantization_method"), (model.config, "quantization_config"), (model.config, "_pre_quantization_dtype"), ): if not hasattr(owner, attribute): setattr(owner, attribute, None) # transformers < 5 has no dtype argument; prepare_model_for_training casts the weights anyway. if "dtype" in inspect.signature(model.dequantize).parameters: model.dequantize(dtype = dtype) else: model.dequantize() # Left True by dequantize; the trainer's DataParallel gate and PEFT still read them. for attribute in ("is_loaded_in_4bit", "is_loaded_in_8bit"): if getattr(model, attribute, False): setattr(model, attribute, False) # transformers 5 keeps the load's bnb deserialize converter, whose missing reverse op makes save_pretrained raise NotImplementedError. conversions = getattr(model, "_weight_conversions", None) if isinstance(conversions, list): model._weight_conversions = [ conversion for conversion in conversions if not any( getattr(op, "hf_quantizer", None) is quantizer for op in getattr(conversion, "operations", None) or () ) ] return True def _compressed_tensors_format(quant_config): if isinstance(quant_config, dict): fmt = quant_config.get("format", None) else: fmt = getattr(quant_config, "format", None) inner = getattr(quant_config, "quantization_config", None) if inner is not None: fmt = getattr(inner, "format", None) or fmt return str(getattr(fmt, "value", fmt) or "unknown format") def _prepare_compressed_tensors_full_finetuning(model): """Full finetuning marks every parameter trainable; integer leftovers of a compressed-tensors checkpoint reject that.""" quant_config = getattr(getattr(model, "config", None), "quantization_config", None) if isinstance(quant_config, dict): method = quant_config.get("quant_method", None) else: method = getattr(quant_config, "quant_method", None) if str(getattr(method, "value", method)).lower() != "compressed-tensors": return 0 name = getattr(getattr(model, "config", None), "_name_or_path", "") or "this model" fmt = _compressed_tensors_format(quant_config) moved = 0 for module_name, module in model.named_modules(): for param_name, param in list(module._parameters.items()): if param is None or param.is_floating_point() or param.is_complex(): continue full_name = f"{module_name}.{param_name}" if module_name else param_name # A weight still packed to integers was not decompressed and cannot be trained. if param_name in ("weight", "weight_packed"): raise RuntimeError( f"Unsloth: `{name}` is a compressed-tensors checkpoint ({fmt}) and `{full_name}` " f"is still stored as {param.dtype}, so full finetuning cannot train it. " "Use `full_finetuning = False` to train LoRA on top of it, or full finetune " "the original 16bit model instead." ) # Kept by decompress_model (e.g. int64 `weight_shape`) so the module can be recompressed on save: # a persistent buffer keeps the state_dict key but leaves the trainable parameter list. delattr(module, param_name) module.register_buffer(param_name, param.data, persistent = True) moved += 1 print( f"Unsloth: `{name}` is a compressed-tensors checkpoint ({fmt}), so full finetuning trains " "its decompressed 16bit weights. For the best accuracy, full finetune the original 16bit model instead." ) return moved def _prepare_compressed_tensors_model(model, full_finetuning = False): # Routed FP8 / NVFP4 weights are frozen, so full finetuning always takes the decompressed bf16 weights. if full_finetuning: _decompress_compressed_tensors_model(model) _prepare_compressed_tensors_full_finetuning(model) return _tag_compressed_tensors_modules(model) if _route_compressed_tensors_nvfp4_to_unsloth(model): return if not _route_compressed_tensors_fp8_to_unsloth(model): _decompress_compressed_tensors_model(model) _FP8_SCALE_SUFFIXES = (".weight_scale_inv", "_scale_inv", ".weight_scale", "_scale") _FP8_LEFTOVER_MAX_CHUNK = 1 << 26 def _empty_device_cache(device): try: backend = getattr(torch, getattr(device, "type", "cuda"), None) empty = getattr(backend, "empty_cache", None) if callable(empty): empty() except Exception: pass def _orient_block_scale(scale, rows, cols, block_size): """Fix a grid stored `(in_blocks, out_blocks)`; block size decides when both orientations tile.""" if block_size is None or scale.ndim < 2: return scale try: bm, bn = int(block_size[0]), int(block_size[1]) except Exception: return scale if bm <= 0 or bn <= 0 or rows % bm or cols % bn: return scale canonical = (rows // bm, cols // bn) grid = tuple(scale.shape[-2:]) if grid == canonical: return scale if grid == canonical[::-1] and canonical[0] != canonical[1]: return scale.transpose(-1, -2).contiguous() return scale def _fp8_scale_grid_dequant( quantized, scale, out_dtype, block_size = None, ): """Returns None when the scale grid does not tile the weight.""" # MXFP8 stores E8M0 exponents as uint8: the scale is 2 ** (byte - 127). if scale.dtype == torch.uint8: scale = (scale.to(torch.float32) - 127.0).exp2() else: scale = scale.to(torch.float32) q_shape = tuple(quantized.shape) if scale.numel() == 1: if quantized.ndim == 3: scale = scale.reshape(()).expand(q_shape[0], 1, 1) else: return (quantized.to(torch.float32) * scale.reshape(())).to(out_dtype) if quantized.ndim == 2: if scale.ndim != 2: if scale.ndim == 1 and scale.numel() == q_shape[0]: scale = scale.view(-1, 1) else: return None rows, cols = q_shape scale = _orient_block_scale(scale, rows, cols, block_size) p, q = scale.shape if rows % p or cols % q: return None bm, bn = rows // p, cols // q out = quantized.to(torch.float32).view(p, bm, q, bn) * scale[:, None, :, None] return out.reshape(rows, cols).to(out_dtype) if quantized.ndim == 3: E, rows, cols = q_shape if scale.ndim == 1 and scale.numel() == E: scale = scale.view(E, 1, 1) elif scale.ndim == 2 and scale.shape[0] == E and scale.shape[1] == 1: scale = scale.view(E, 1, 1) if scale.ndim != 3 or scale.shape[0] != E: return None scale = _orient_block_scale(scale, rows, cols, block_size) p, q = scale.shape[1], scale.shape[2] if rows % p or cols % q: return None bm, bn = rows // p, cols // q out = torch.empty((E, rows, cols), dtype = out_dtype, device = quantized.device) step = max(1, _FP8_LEFTOVER_MAX_CHUNK // max(1, rows * cols)) for start in range(0, E, step): stop = min(E, start + step) chunk = quantized[start:stop].to(torch.float32).view(stop - start, p, bm, q, bn) out[start:stop] = ( (chunk * scale[start:stop, :, None, :, None]) .reshape(stop - start, rows, cols) .to(out_dtype) ) return out return None class FP8LeftoverOffloadedError(RuntimeError): """A leftover fp8 tensor is disk-offloaded, so the 16bit load cannot be finished in place.""" def _restore_parked_fp8(module, attr, device): try: stranded = module._parameters.get(attr) if ( isinstance(stranded, torch.Tensor) and stranded.device != device and stranded.device.type == "cpu" ): module._parameters[attr] = torch.nn.Parameter( stranded.data.to(device), requires_grad = bool(stranded.requires_grad) ) except Exception: pass def _dequantize_leftover_fp8_params( model, model_name, dtype, *, local_files_only = False, token = None, revision = None, subfolder = None, cache_dir = None, variant = None, ): """Dequantize fp8 params transformers left on a 16bit load. Returns (dequantized, skipped).""" try: if not _FP8_DTYPES: return (0, 0) leftover = [ name for name, param in model.named_parameters() if param.dtype in _FP8_DTYPES and param.ndim in (2, 3) ] block_size = None try: _qc = getattr(getattr(model, "config", None), "quantization_config", None) _bs = ( _qc.get("weight_block_size") if isinstance(_qc, dict) else getattr(_qc, "weight_block_size", None) ) if _bs is not None and len(_bs) == 2: block_size = (int(_bs[0]), int(_bs[1])) except Exception: block_size = None _grid_kwargs = {"block_size": block_size} if block_size is not None else {} if not leftover: return (0, 0) weight_map = _load_fp8_weight_map( model_name, local_files_only, token, revision, subfolder, cache_dir, variant = variant, ) if not weight_map: return (0, 0) scale_by_weight_key = {} for key, shard in weight_map.items(): for suffix in _FP8_SCALE_SUFFIXES: if key.endswith(suffix): weight_key = key[: -len(suffix)] # `layer.weight_scale_inv` belongs to `layer.weight`, not to `layer`. if suffix.startswith(".weight"): weight_key += ".weight" scale_by_weight_key[weight_key] = (key, shard) break if not scale_by_weight_key: return (0, 0) module_by_name = dict(model.named_modules()) # Names only: holding the fp8 Parameter would defeat the OOM fallback. params_by_module = {} for name in leftover: module_name, _, attr = name.rpartition(".") params_by_module.setdefault(module_name, []).append(attr) target_of_ckpt = {} for weight_key in scale_by_weight_key: module_part, _, attr = weight_key.rpartition(".") if attr == "weight": ckpt_module, ckpt_attr = module_part, "weight" else: ckpt_module, ckpt_attr = module_part, attr module = _match_fp8_module(module_by_name, ckpt_module) if module is None: continue live_name = None for candidate, mod in module_by_name.items(): if mod is module: live_name = candidate break if live_name is None or live_name not in params_by_module: continue target_of_ckpt[weight_key] = (module, live_name, ckpt_attr) dequantized = 0 skipped = 0 failed = 0 last_error = None shard_cache = {} def _scale_for(weight_key, device): scale_key, shard = scale_by_weight_key[weight_key] if shard not in shard_cache: from safetensors import safe_open shard_path = _resolve_fp8_shard( model_name, shard, local_files_only, token, revision, subfolder, cache_dir ) shard_cache[shard] = safe_open(shard_path, framework = "pt") return shard_cache[shard].get_tensor(scale_key).to(device) def _is_oom(error): # Some backends raise a plain RuntimeError for OOM. return ( hasattr(torch, "OutOfMemoryError") and isinstance(error, torch.OutOfMemoryError) ) or (isinstance(error, RuntimeError) and "out of memory" in str(error).lower()) deferred = [] converted = set() for weight_key, (module, live_name, attr) in target_of_ckpt.items(): param = getattr(module, attr, None) if not isinstance(param, torch.Tensor) or param.dtype not in _FP8_DTYPES: continue trainable = bool(getattr(param, "requires_grad", False)) # A module with its own scale runs the fp8 forward. if any( isinstance(getattr(module, s, None), torch.Tensor) for s in (attr + "_scale_inv", attr + "_scale", "weight_scale_inv", "weight_scale") ): skipped += 1 continue if param.device.type == "meta": # Disk-offloaded: the hook would restore raw fp8 bytes at forward time. raise FP8LeftoverOffloadedError( f"Unsloth: `{weight_key}` is an fp8 tensor transformers left quantized on a 16bit " "load, and it is offloaded to disk, so it cannot be dequantized in place. Load with " "enough GPU or CPU memory to keep the model resident (no disk offload), or use " "`load_in_4bit = True`." ) try: scale = _scale_for(weight_key, param.device) with torch.no_grad(): out = _fp8_scale_grid_dequant(param.data, scale, dtype, **_grid_kwargs) if out is None: failed += 1 last_error = f"{weight_key}: scale {tuple(scale.shape)} does not tile {tuple(param.shape)}" continue module._parameters[attr] = torch.nn.Parameter(out, requires_grad = trainable) converted.add((module, attr)) dequantized += 1 except Exception as e: if _is_oom(e): device = param.device quantized = param.data.to("cpu") module._parameters[attr] = torch.nn.Parameter( quantized, requires_grad = trainable ) del param deferred.append((weight_key, module, attr, device)) _empty_device_cache(device) continue failed += 1 last_error = f"{weight_key}: {type(e).__name__}: {e}" continue for weight_key, module, attr, device in deferred: try: param = getattr(module, attr) trainable = bool(getattr(param, "requires_grad", False)) scale = _scale_for(weight_key, "cpu") with torch.no_grad(): out = _fp8_scale_grid_dequant(param.data, scale, dtype, **_grid_kwargs) if out is None: failed += 1 last_error = f"{weight_key}: scale {tuple(scale.shape)} does not tile {tuple(param.shape)}" _restore_parked_fp8(module, attr, device) continue _empty_device_cache(device) module._parameters[attr] = torch.nn.Parameter( out.to(device), requires_grad = trainable ) del param, out converted.add((module, attr)) dequantized += 1 except Exception as e: failed += 1 last_error = f"{weight_key}: {type(e).__name__}: {e}" _restore_parked_fp8(module, attr, device) continue for shard_file in shard_cache.values(): close = getattr(shard_file, "__exit__", None) if close is not None: try: close(None, None, None) except Exception: pass if dequantized > 0: by_module = {} for module, attr in converted: by_module.setdefault(module, set()).add(attr) for module, attrs in by_module.items(): still_fp8 = [ n for n, p in module._parameters.items() if isinstance(p, torch.Tensor) and p.dtype in _FP8_DTYPES ] for stale in [ n for n in list(module._parameters) + list(module._buffers) if n.endswith("activation_scale") ]: owner = next( (a for a in list(attrs) + still_fp8 if stale.startswith(a + "_")), None ) if owner is None and still_fp8: continue if owner is not None and owner not in attrs: continue if stale in module._parameters: del module._parameters[stale] else: del module._buffers[stale] print( f"Unsloth: Dequantized {dequantized} FP8 tensor(s) transformers left quantized on a 16bit load." ) if failed > 0: print( f"Unsloth: {failed} FP8 tensor(s) could not be dequantized on the 16bit load and stay fp8 " f"(last: {last_error})." ) return (dequantized, skipped) except FP8LeftoverOffloadedError: raise except Exception: return (0, 0) _BNB_QUANTIZED_TYPES = ("Params4bit", "Int8Params", "Linear4bit", "Linear8bitLt") def _bnb_bits_requested(quantization_config): """4 or 8 if a bitsandbytes config object or dict asks to quantize, else None.""" if quantization_config is None: return None if isinstance(quantization_config, dict): get = quantization_config.get else: get = lambda key, default = None: getattr(quantization_config, key, default) method = get("quant_method", "") or "" method = str(getattr(method, "value", method)).lower() # The dict shorthand {"load_in_4bit": True} has no quant_method but is still bitsandbytes. if not method and isinstance(quantization_config, dict): method = "bitsandbytes" if "bitsandbytes" not in method: return None if get("load_in_4bit", False): return 4 if get("load_in_8bit", False): return 8 return None _ASYNC_LOAD_ENV = "HF_DEACTIVATE_ASYNC_LOAD" @contextlib.contextmanager def sync_load_when_quantizing(quantization_config, model_config): """Sync-load on-the-fly quantization: transformers 5.0-5.3 worker threads put full-precision tensors on the card faster than they are quantized (5.4+ already loads these synchronously).""" if ( quantization_config is None or getattr(model_config, "quantization_config", None) is not None or _ASYNC_LOAD_ENV in os.environ ): yield return os.environ[_ASYNC_LOAD_ENV] = "1" try: yield finally: os.environ.pop(_ASYNC_LOAD_ENV, None) def gptq_trainable_quantization_config(model_config, user_quantization_config): """GPTQConfig / AwqConfig asking gptqmodel for a trainable kernel on a GPTQ or AWQ checkpoint, else None. gptqmodel's default kernels (Marlin / ExLlama) raise NotImplementedError on model.train(); only `backend` is a loading attribute, so the checkpoint's own bits / group_size still apply. """ if user_quantization_config is not None: return None qc = getattr(model_config, "quantization_config", None) if qc is not None and not isinstance(qc, dict): qc = qc.to_dict() quant_method = str((qc or {}).get("quant_method", "")).lower() if quant_method not in ("gptq", "awq"): return None # transformers 4.x wrote "autoawq" / "llm-awq" into AWQ configs (Qwen3-8B-AWQ); AwqConfig reads both as auto. default_backends = ( (None, "auto", "autoawq", "llm-awq") if quant_method == "awq" else (None, "auto") ) if qc.get("backend") not in default_backends: return None try: from transformers import AwqConfig, GPTQConfig from transformers.utils import is_gptqmodel_available except ImportError: return None if not is_gptqmodel_available() or "backend" not in inspect.signature(GPTQConfig).parameters: return None if quant_method == "awq": # transformers < 5 AwqConfig has no auto_trainable backend (autoawq era). try: from transformers.utils.quantization_config import AwqBackend except ImportError: return None if not hasattr(AwqBackend, "AUTO_TRAINABLE"): return None return AwqConfig(bits = qc.get("bits", 4), backend = "auto_trainable") return GPTQConfig(bits = qc["bits"], backend = "auto_trainable") def refuse_fast_inference_load_in_8bit( load_in_8bit, model_config = None, quantization_config = None, ): """Refuse bnb 8-bit (flag, caller config or checkpoint config) under fast_inference: no 8-bit vLLM path.""" checkpoint_config = getattr(model_config, "quantization_config", None) if not ( load_in_8bit or _bnb_bits_requested(quantization_config) == 8 or _bnb_bits_requested(checkpoint_config) == 8 ): return raise NotImplementedError( "Unsloth: fast_inference = True does not support bitsandbytes 8-bit " "(load_in_8bit = True or an 8-bit bitsandbytes checkpoint): vLLM would " "not run in 8-bit. Use load_in_4bit = True or 16-bit " "(load_in_4bit = False, load_in_8bit = False) with fast_inference, or set " "fast_inference = False to keep load_in_8bit." ) def warn_if_bitsandbytes_quantized_nothing( model, quantization_config, model_name = "", ): """Warn and return True when a bitsandbytes load quantized no weight (e.g. every Linear skipped).""" bits = _bnb_bits_requested(quantization_config) if bits is None or model is None: return False try: for module in model.modules(): if type(module).__name__ in _BNB_QUANTIZED_TYPES: return False # compressed-tensors INT4 kept packed (compressed_tensors_int4.py) is 4-bit too. if getattr(module, "_unsloth_int4_packed_linear", False): return False for param in module.parameters(recurse = False): if type(param).__name__ in _BNB_QUANTIZED_TYPES: return False except Exception: return False print( f"Unsloth: WARNING: {bits}bit loading was on, but no weight of `{model_name}` was quantized, " f"so the model is in 16bit and needs far more VRAM than a {bits}bit load." ) return True def check_and_disable_bitsandbytes_loading( model_config, load_in_4bit = True, load_in_8bit = False, verbose = True, requantize_packed = True, rewrite_modelopt = True, token = None, model_name = None, revision = None, hub_kwargs = None, allow_fp8_to_nf4 = True, ): """Disable bnb flags for non-bnb quantized checkpoints; returns ``(load_in_4bit, load_in_8bit, quant_method)``. ``requantize_packed = False`` under fast_inference / full_finetuning: nothing would consume the plan.""" quant_method = get_quant_type(model_config) if quant_method is None: # Also under vLLM: it reads the file itself, but the bitsandbytes flags must still drop. from .modelopt_fp8 import attach_hf_quant_config if attach_hf_quant_config( model_config, token = token, model_name = model_name, revision = revision, hub_kwargs = hub_kwargs, ): quant_method = get_quant_type(model_config) if quant_method is None or quant_method == "bitsandbytes": return load_in_4bit, load_in_8bit, quant_method if str(quant_method).lower() in ("compressed-tensors", "compressed_tensors", "sparseml"): from .mxfp4_compressed_linear import install_compressed_tensors_keep_packed install_compressed_tensors_keep_packed() from .packed_expert_loader import install_packed_expert_loading install_packed_expert_loading() # Packed compressed-tensors: drop its quant config here and keep load_in_4bit for on-the-fly bnb re-quantization. if ( requantize_packed and load_in_4bit and not load_in_8bit and str(quant_method).lower() in ("compressed-tensors", "compressed_tensors", "sparseml") ): from .compressed_tensors_bnb import arm_compressed_tensors_bnb_loading if arm_compressed_tensors_bnb_loading(model_config, verbose = verbose) is not None: return True, False, None if str(quant_method).lower() == "modelopt": # Whoever loads the weights (vLLM included), a merged_16bit save must dequantize them. from .modelopt_fp8 import enable_modelopt_merged_save enable_modelopt_merged_save(model_config) if rewrite_modelopt and str(quant_method).lower() == "modelopt": from .modelopt_nvfp4 import arm_modelopt_nvfp4_loading if arm_modelopt_nvfp4_loading(model_config, verbose = verbose) is not None: quant_method = "compressed-tensors" if rewrite_modelopt and str(quant_method).lower() == "modelopt": from .modelopt_fp8 import arm_modelopt_fp8_loading if arm_modelopt_fp8_loading(model_config, verbose = verbose) is not None: quant_method = "fp8" # An explicit 4bit request on a block-fp8 checkpoint: dequantize each fp8 tensor and quantize it to NF4 while loading. if allow_fp8_to_nf4 and str(quant_method).lower() == "fp8": from .fp8_to_nf4 import maybe_arm_fp8_to_nf4 if maybe_arm_fp8_to_nf4(model_config, load_in_4bit, load_in_8bit, verbose = verbose): return load_in_4bit, load_in_8bit, None # A non-bitsandbytes quantization config (compressed-tensors, gptq, awq) means BOTH bitsandbytes loading flags must be disabled to avoid config conflicts. if load_in_4bit or load_in_8bit: if verbose: print( f"Unsloth: Model already quantized with {quant_method}. " f"Disabling `load_in_4bit` and `load_in_8bit` to avoid quantization config conflict." ) if load_in_4bit and not load_in_8bit and str(quant_method).lower() == "fp8": from .fp8_to_nf4 import ( explicit_4bit_requested, fp8_block_quantization_config, fp8_to_nf4_disabled, ) if ( not explicit_4bit_requested() and not fp8_to_nf4_disabled() and fp8_block_quantization_config(model_config) is not None ): print( "Unsloth: Pass `load_in_4bit = True` explicitly (or set UNSLOTH_FP8_TO_NF4=1) to " "quantize this fp8 checkpoint to 4bit while it loads." ) load_in_4bit = False load_in_8bit = False return load_in_4bit, load_in_8bit, quant_method def quantization_config_selects_bnb_4bit(quantization_config): """True when ``quantization_config`` keeps the load on bnb 4-bit; other quantizers must keep the checkpoint config.""" if quantization_config is None: return True if isinstance(quantization_config, dict): get = quantization_config.get else: get = lambda key, default = None: getattr(quantization_config, key, default) method = get("quant_method", "") or "" # BitsAndBytesConfig stores a QuantizationMethod enum, whose str() is the member name on some Pythons. method = str(getattr(method, "value", method)).lower() # The dict shorthand {"load_in_4bit": True} has no quant_method; transformers reads it as bitsandbytes. if not method and isinstance(quantization_config, dict): method = "bitsandbytes" if "bitsandbytes" not in method: return False return bool(get("load_in_4bit", False)) and not bool(get("load_in_8bit", False)) _FUSED_EXPERT_SUFFIX = ".experts.gate_up_proj" def _safetensors_dtypes(path, names = None): from safetensors import safe_open with safe_open(path, framework = "pt") as f: keys = f.keys() if names is None else names return {k: f.get_slice(k).get_dtype() for k in keys} class GptOssUnknownExpertLayoutError(RuntimeError): """A bitsandbytes gpt-oss checkpoint whose expert layout cannot be read without loading its weights.""" class _UnreadableCheckpointNames(Exception): """The checkpoint is a lone pytorch_model.bin whose tensor names cannot be read without loading it.""" def _with_variant(filename, variant): # Mirrors transformers' _add_variant. if not variant: return filename parts = filename.split(".") return ".".join(parts[:-1] + [variant, parts[-1]]) def _index_weight_map(path): import json with open(path, "r", encoding = "utf-8") as f: return json.load(f).get("weight_map", {}) or {} def _bin_dtypes(path): # mmap + weights_only reads the pickle and storage metadata, not the tensor bytes. state_dict = torch.load(path, map_location = "cpu", mmap = True, weights_only = True) return {k: str(v.dtype) for k, v in state_dict.items() if isinstance(v, torch.Tensor)} def _hub_safetensors_header_dtypes( repo_id, filename, revision = None, token = None, ): """Range-reads one safetensors header from the Hub (never the tensor bytes).""" import json import struct from huggingface_hub import HfFileSystem fs = HfFileSystem(token = token) with fs.open(f"{repo_id}/{filename}", "rb", revision = revision, block_size = 0) as f: (size,) = struct.unpack(" str: """Validate `load_in_fp8` settings/environment and return the fp8 mode ("row" or "block"). Requires H100+, torchao 0.15.0+, torch 2.9.0+, and fbgemm_gpu_genai 1.4.1+ if installed.""" assert load_in_fp8 is not False if load_in_fp8 is True: fp8_mode = "row" # default else: fp8_mode = load_in_fp8 if fp8_mode not in ["row", "block"]: raise ValueError(f"Unsloth: `load_in_fp8` can only be 'row' or 'block', got '{fp8_mode}'") if full_finetuning: raise ValueError("Unsloth: `load_in_fp8` is not compatible with full finetuning") if load_in_4bit or load_in_8bit or load_in_16bit: raise ValueError( "Unsloth: `load_in_fp8` is not compatible with `load_in_4bit`, `load_in_8bit` or `load_in_16bit`", ) if not ( torch.cuda.is_available() and torch.version.cuda and torch.cuda.get_device_capability() >= (9, 0) ): raise ValueError( "Unsloth: On the fly `load_in_fp8` requires H100 GPUs or after. Try `unsloth/Qwen3-8B` instead." ) if Version(torch.__version__) < Version("2.9.0"): raise ValueError( "Unsloth: On the fly `load_in_fp8` requires torch 2.9.0+. Try `unsloth/Qwen3-8B` instead." ) # Check if torchao has pytorch/ao#3158, released in 0.15.0. if importlib.util.find_spec("torchao") is None: raise ValueError( "Unsloth: Please install torchao for on the fly float8 to work! Try `unsloth/Qwen3-8B` instead." ) import torchao error_message = ( "Unsloth: `load_in_fp8` requires torchao 0.15.0+ (or nightly).\n" f"You have torchao version={torchao.__version__}\n" "Use `pip install --upgrade --force-reinstall torchao`" ) if Version(torchao.__version__) < Version("0.15.0"): raise ValueError(error_message) # If fbgemm_gpu_genai is installed and old, disable FBGEMM and use Triton kernels instead. if ( importlib.util.find_spec("fbgemm_gpu") is not None and importlib.util.find_spec("fbgemm_gpu.experimental") is not None ): import fbgemm_gpu.experimental.gen_ai if Version(fbgemm_gpu.__version__) < Version("1.4.1"): os.environ["UNSLOTH_HAS_FBGEMM"] = "0" from unsloth_zoo.log import logger logger.info( f"Unsloth: fbgemm_gpu_genai=={fbgemm_gpu.__version__} is old for FP8 loading. " f"Using Triton kernels instead." ) return fp8_mode # Rotary inv_freq buffers are deliberately kept on CPU: Unsloth pre-builds a cos/sin cache per GPU instead (LlamaRotaryEmbedding.multi_gpu_cos_cached). DistributedDataParallel ignores device when it broadcasts buffers across ranks, so a CPU buffer crashes NCCL's _broadcast_coalesced with "No backend type associated with device type cpu"; telling DDP to skip these buffers avoids that without moving inv_freq to GPU. Re-run after wrapping with PEFT, since the fully qualified names change under a PeftModel (#6656). _ROTARY_INV_FREQ_BUFFER_NAMES = ("inv_freq", "short_inv_freq", "long_inv_freq") def _is_tensor_subclass_weight(t): """A wrapper tensor subclass weight such as torchao's Float8Tensor (load_in_fp8). DDP's start-up broadcast flattens every parameter (reshape to [-1]), which Float8Tensor does not implement. torchao.float8's training weights (WeightWithDynamicFloat8CastTensor) implement it and DDP trains them, so they are not counted, as in fsdp2_native's _float8_params.""" if type(t).__module__.startswith("torchao.float8"): return False if any(c.__module__.split(".")[0] == "torchao" for c in type(t).__mro__): return True try: from torch.distributed.tensor import DTensor if isinstance(t, DTensor): return False except Exception: pass try: from torch.utils._python_dispatch import is_traceable_wrapper_subclass return is_traceable_wrapper_subclass(t) except Exception: return False def _holds_float8(t, depth = 0): """True when a tensor, or a tensor subclass's inner tensors (__tensor_flatten__), stores float8.""" if "float8" in str(t.dtype): return True if depth > 3 or not hasattr(t, "__tensor_flatten__"): return False try: names = t.__tensor_flatten__()[0] except Exception: return False for name in names: inner = getattr(t, name, None) if isinstance(inner, torch.Tensor) and _holds_float8(inner, depth + 1): return True return False def _ddp_can_flatten(t): """DDP's start-up broadcast flattens every parameter (_flatten_dense_tensors); probe that on the weight.""" try: torch._utils._flatten_dense_tensors([t.detach()]) return True except Exception: return False def _frozen_tensor_subclass_names_for_ddp(model, refuse_trainable = False): """Frozen tensor subclass weights (load_in_fp8's Float8Tensor) stay out of DDP: every rank loads and quantizes the same checkpoint, so there is nothing to broadcast, and a frozen weight has no gradient to reduce. A trainable one would need both, which DDP cannot do for it: refused when asked.""" frozen, trainable = [], [] for name, p in model.named_parameters(): if not _is_tensor_subclass_weight(p): continue if not p.requires_grad: frozen.append(name) elif _holds_float8(p) or not _ddp_can_flatten(p): # FP8 storage, and weights DDP's broadcast cannot flatten (Int8Tensor, AffineQuantizedTensor), # are refused: trainable subclasses that flatten (torchao's int8 mixed precision training # weight, NF4Tensor) train as before. trainable.append(name) if trainable and refuse_trainable: raise NotImplementedError( f"Unsloth: DDP cannot train the {type(model.get_parameter(trainable[0])).__name__} weight " f"{trainable[0]} ({len(trainable)} such weights are trainable); it cannot be broadcast or have " "its gradient all-reduced. Train LoRA on the frozen FP8 base (get_peft_model), or load in 16-bit." ) return frozen def _exclude_rope_inv_freq_from_ddp(model): ignored = list(getattr(model, "_ddp_params_and_buffers_to_ignore", None) or []) for module_name, module in model.named_modules(): for buffer_name, _ in module.named_buffers(recurse = False): if buffer_name in _ROTARY_INV_FREQ_BUFFER_NAMES: fqn = f"{module_name}.{buffer_name}" if module_name else buffer_name if fqn not in ignored: ignored.append(fqn) try: for fqn in _frozen_tensor_subclass_names_for_ddp(model): if fqn not in ignored: ignored.append(fqn) except Exception: pass if ignored: try: from torch.nn.parallel import DistributedDataParallel DistributedDataParallel._set_params_and_buffers_to_ignore_for_model(model, ignored) except Exception: # Private PyTorch API: fall back to setting the attribute DDP reads directly if it ever moves or changes signature. model._ddp_params_and_buffers_to_ignore = ignored return model # Offline loading, shared by vision.py, loader.py and the Unsloth exporter: decide offline ONCE at the load boundary and force it ONCE around the whole load, so nested HF calls inherit it. _OFFLINE_ENV_VALUES = {"1", "true", "yes", "on"} _OFFLINE_ENV_KEYS = ("HF_HUB_OFFLINE", "TRANSFORMERS_OFFLINE") def _env_says_offline(): """True if an HF offline env var is set to a truthy value.""" return any( os.environ.get(_k, "").strip().lower() in _OFFLINE_ENV_VALUES for _k in _OFFLINE_ENV_KEYS ) def _get_effective_local_files_only(kwargs): """Offline if local_files_only is truthy or an HF offline env var is set. Read-only.""" if kwargs.get("local_files_only", None): return True return _env_says_offline() # Stamped on a tokenizer/processor loaded local-only so a later save still knows: transformers takes local_files_only as an explicit from_pretrained parameter and never copies it into tokenizer.init_kwargs (#7481). The load's cache_dir and ref travel with it, since saving otherwise derives a cache from HF_HUB_CACHE and drops the branch from name_or_path. _LOCAL_FILES_ONLY_ATTR = "_unsloth_local_files_only" _LOADED_CACHE_DIR_ATTR = "_unsloth_loaded_cache_dir" _LOADED_REVISION_ATTR = "_unsloth_loaded_revision" def _mark_loaded_revision( result, revision, cache_dir = None, load_kwargs = None, ): from ..save import _capture_source_tokenizer_class for obj in result if isinstance(result, (tuple, list)) else (result,): try: targets = (obj, getattr(obj, "tokenizer", None)) except Exception: targets = (obj,) for target in targets: if target is None: continue try: if revision is not None: setattr(target, _LOADED_REVISION_ATTR, str(revision)) if cache_dir and not getattr(target, _LOADED_CACHE_DIR_ATTR, None): setattr(target, _LOADED_CACHE_DIR_ATTR, str(cache_dir)) source_tokenizer = getattr(target, "tokenizer", target) context = load_kwargs or {} _capture_source_tokenizer_class( target, source = getattr(source_tokenizer, "name_or_path", None) if load_kwargs is not None else None, cache_dir = _tokenizer_cache_dir(source_tokenizer), revision = _tokenizer_revision(source_tokenizer) or context.get("tokenizer_revision") or context.get("revision"), subfolder = context.get("subfolder") or "", ) except Exception: pass return result def _tokenizer_revision(tokenizer): """The ref this tokenizer was loaded at, or None for the default branch.""" tokenizer = tokenizer.tokenizer if hasattr(tokenizer, "tokenizer") else tokenizer return getattr(tokenizer, _LOADED_REVISION_ATTR, None) def _mark_loaded_local_files_only(result, cache_dir = None): """Stamp a load's local-only mode and cache_dir onto the returned objects.""" for obj in result if isinstance(result, (tuple, list)) else (result,): try: # A processor keeps the tokenizer that _has_tokenizer_model unwraps to, so stamp both; a wrapped model can raise from its own __getattr__. targets = (obj, getattr(obj, "tokenizer", None)) except Exception: targets = (obj,) for target in targets: if target is None: continue try: setattr(target, _LOCAL_FILES_ONLY_ATTR, True) if cache_dir: setattr(target, _LOADED_CACHE_DIR_ATTR, str(cache_dir)) except Exception: pass return result def _tokenizer_cache_dir(tokenizer): """The cache_dir the load used, when it was not the environment's.""" tokenizer = tokenizer.tokenizer if hasattr(tokenizer, "tokenizer") else tokenizer return getattr(tokenizer, _LOADED_CACHE_DIR_ATTR, None) def _tokenizer_wants_local_only(tokenizer): """True when Hub metadata probes should be skipped for this tokenizer.""" if _env_says_offline(): return True if getattr(tokenizer, _LOCAL_FILES_ONLY_ATTR, False): return True init_kwargs = getattr(tokenizer, "init_kwargs", None) or {} return bool(init_kwargs.get("local_files_only")) def _is_offline_related_error(exc): """True if exc (or its cause/context chain) is a lost-connection error, not a missing file. Plain FileNotFoundError propagates; LocalEntryNotFoundError is offline.""" import socket import ssl import urllib.error # Match network failures by type (locale independent), not just message wording. _net_types = [ConnectionError, TimeoutError, socket.gaierror, urllib.error.URLError] _offline_fnf_types = () # FileNotFoundError subclasses that count as offline # urllib HTTPError is a URLError subclass, so judge by status (5xx offline, 4xx propagates). TLS/cert failures are security-sensitive (MITM, expired CA) and are never offline-retried. _http_types = (urllib.error.HTTPError,) _ssl_types = [ssl.SSLError] try: import requests _net_types += [requests.exceptions.ConnectionError, requests.exceptions.Timeout] _http_types += (requests.exceptions.HTTPError,) _ssl_types.append(requests.exceptions.SSLError) except Exception: pass try: from huggingface_hub.errors import ( OfflineModeIsEnabled, HfHubHTTPError, LocalEntryNotFoundError, ) _net_types += [OfflineModeIsEnabled, LocalEntryNotFoundError] _offline_fnf_types = (LocalEntryNotFoundError,) _http_types += (HfHubHTTPError,) except Exception: pass _net_types = tuple(_net_types) _ssl_types = tuple(_ssl_types) def _http_status(e): resp = getattr(e, "response", None) code = getattr(resp, "status_code", None) if code is None: code = getattr(e, "status_code", None) if code is None: code = getattr(e, "code", None) # urllib.error.HTTPError uses .code try: return int(code) except (TypeError, ValueError): return None _wording = ( "couldn't connect", "could not connect", "connection error", "connectionerror", "max retries", "offline", "timed out", "timeout", "couldn't reach", "could not reach", "failed to resolve", "getaddrinfo", "name resolution", "no address associated", "network is unreachable", "connection refused", "we couldn't connect to", "proxyerror", # Raw socket.gaierror DNS wording (Linux / macOS). "name or service not known", "temporary failure in name resolution", "nodename nor servname provided", ) seen = set() cur = exc while cur is not None and id(cur) not in seen: seen.add(id(cur)) # TLS/cert failure (corporate MITM, expired CA): security-sensitive, never retry from cache. Skip this node; a deeper cause in the chain may still be a genuine outage. if isinstance(cur, _ssl_types) or isinstance(getattr(cur, "reason", None), _ssl_types): cur = cur.__cause__ or cur.__context__ continue is_fnf = isinstance(cur, FileNotFoundError) and not isinstance(cur, _offline_fnf_types) # urllib HTTPError is a URLError (net type) but must be judged by status code below, unlike LocalEntryNotFoundError, an HfHubHTTPError that is always offline. if ( isinstance(cur, _net_types) and not is_fnf and not isinstance(cur, urllib.error.HTTPError) ): return True if isinstance(cur, _http_types): code = _http_status(cur) if code is not None and 500 <= code < 600: return True # No status, so fall back to wording; a coded 4xx was already decided above. if code is None and not is_fnf and any(w in str(cur).lower() for w in _wording): return True # OSError wording fallback; the HTTP status was already decided above. elif isinstance(cur, OSError) and not is_fnf: if any(w in str(cur).lower() for w in _wording): return True cur = cur.__cause__ or cur.__context__ return False # Process-wide HF offline state; the depth counter lets nested windows share one flip (first entrant saves originals, last exit restores). Lock guards flip/restore. _force_offline_lock = _threading.RLock() _force_offline_depth = 0 _force_offline_saved = [] _force_offline_saved_env = {} def _reset_hf_sessions(): """Clear hub's per-thread cached Sessions so the next rebuilds against the current offline flag. On hub 0.x the offline adapter is baked in at Session creation. Best-effort.""" # Snapshot in-process constants BEFORE forcing the env: a module first imported here would otherwise initialize its constant from the just-set "1" and we would save (then restore) True, pinning the process offline after the window. try: from huggingface_hub.utils._http import reset_sessions except Exception: try: from huggingface_hub.utils import reset_sessions except Exception: return try: reset_sessions() except Exception: pass @contextlib.contextmanager def _force_hf_offline(): """Force HF offline for the window. local_files_only alone is not enough (transformers < 5 still pings /api/models), so set BOTH the env vars (covering subprocesses and raw urllib/requests) AND the in-process hub/transformers constants. Process-global; the refcount keeps restore correct under nesting.""" global _force_offline_depth, _force_offline_saved, _force_offline_saved_env with _force_offline_lock: if _force_offline_depth == 0: saved = [] saved_env = {} try: import huggingface_hub.constants as _hfc if hasattr(_hfc, "HF_HUB_OFFLINE"): saved.append((_hfc, "HF_HUB_OFFLINE", _hfc.HF_HUB_OFFLINE)) except Exception: pass try: import transformers.utils.hub as _tuh for _attr in ("_is_offline_mode", "OFFLINE"): if hasattr(_tuh, _attr): saved.append((_tuh, _attr, getattr(_tuh, _attr))) except Exception: pass for _k in _OFFLINE_ENV_KEYS: saved_env[_k] = os.environ.get(_k) os.environ[_k] = "1" for _obj, _attr, _ in saved: try: setattr(_obj, _attr, True) except Exception: pass _force_offline_saved = saved _force_offline_saved_env = saved_env # Drop offline-mounted sessions so later online calls rebuild for the network. _reset_hf_sessions() _force_offline_depth += 1 try: yield finally: with _force_offline_lock: _force_offline_depth -= 1 if _force_offline_depth == 0: for obj, attr, val in _force_offline_saved: try: setattr(obj, attr, val) except Exception: pass _force_offline_saved = [] for _k, _v in _force_offline_saved_env.items(): if _v is None: os.environ.pop(_k, None) else: os.environ[_k] = _v _force_offline_saved_env = {} _reset_hf_sessions() def _progress_bars_were_disabled(): """Snapshot HF progress-bar state (None if unknown); pairs with _restore_progress_bars.""" try: from huggingface_hub.utils import are_progress_bars_disabled return are_progress_bars_disabled() except Exception: return None def _restore_progress_bars(were_disabled): """Re-enable HF progress bars only if a failed attempt left them disabled after they were enabled (a loader disables them around config probes and skips re-enabling on error). No-op if the user had them disabled or the state is unknown.""" if were_disabled is False: try: from huggingface_hub.utils import enable_progress_bars enable_progress_bars() except Exception: pass # Every way a cache miss reaches the caller once offline mode has skipped Transformers' own "does not appear to have a file named" raise: the resolved path stays None and the next line dereferences it, so the message names the None and never the cache. Same set the Unsloth training worker matches (studio/backend/core/training/worker.py, #7845): weights come out as `endswith`, tokenizers/processors as any of the other four. _EMPTY_CACHE_ARTIFACTS = ( "'nonetype' object has no attribute 'endswith'", "'nonetype' object has no attribute 'readlines'", "argument should be a str or an os.pathlike object", "expected str, bytes or os.pathlike object", "stat: path should be string, bytes, os.pathlike or integer", "can't find a vocabulary file at path 'none'", ) def _empty_cache_artifact(exc): """True if exc, or something it wraps, is offline mode's empty-cache artifact, the one family of retry failure that says nothing useful. Named positively rather than asking "is this an OOM": every other retry failure is real news and must reach the user, and enumerating the ways an accelerator spells OOM cannot be complete (accelerate re-raises it as a bare RuntimeError, XPU has its own class), while asking for the artifact is complete by construction.""" seen = set() while exc is not None and id(exc) not in seen: seen.add(id(exc)) text = str(exc).lower() # Gate on the None: the same TypeError wording about a real path (`not 'int'`) is a caller bug, not an empty cache. if ("nonetype" in text or "path 'none'" in text) and any( artifact in text for artifact in _EMPTY_CACHE_ARTIFACTS ): return True exc = exc.__cause__ or exc.__context__ return False def _release_traceback_locals(error): """Drop the frame locals along an exception chain, keeping file and line. An exception kept past its handler keeps its frames alive, and a failed load's frames still own whatever the attempt had already built, so a retained error can pin a model's GPU tensors for as long as the caller holds it. Clearing the locals beats dropping the traceback: the memory goes either way, but the origin stays printable, which for a network failure inside `trust_remote_code` is the only clue there is.""" seen = set() while error is not None and id(error) not in seen: seen.add(id(error)) # The frame we are running in cannot be cleared; clear_frames skips it for us. _traceback.clear_frames(error.__traceback__) error = error.__cause__ or error.__context__ def _note_offline_retry(error, retry_error): """Record the failed cache retry on the online error we are about to surface. A note is the only place it can go that survives: the online error usually already has a cause, and Python prints a cause INSTEAD of a context, so chaining the retry on would hide it and would cost the chain that makes the online error classifiable. Notes are 3.11+; below that this is a no-op and the attribute is all there is.""" text = f"Unsloth: retrying from the local cache also failed: {type(retry_error).__name__}: {retry_error}" try: error._unsloth_offline_retry_error = retry_error except Exception: pass add_note = getattr(error, "add_note", None) if add_note is None: return try: add_note(text) except Exception: pass # Set per load (family branches in FastModel.from_pretrained, the compiler's norm check) and read only during it. Left set, a later load of another family gets float32 norms beside 16 bit projections and fails with "float != BFloat16". LOAD_SCOPED_ENV_VARS = ("UNSLOTH_HIGH_PRECISION_LAYERNORM",) def _restore_load_scoped_env(fn): @functools.wraps(fn) def _wrapper(*args, **kwargs): saved = {name: os.environ.get(name) for name in LOAD_SCOPED_ENV_VARS} try: return fn(*args, **kwargs) finally: for name, value in saved.items(): if value is None: os.environ.pop(name, None) else: os.environ[name] = value return _wrapper def _offline_aware_load(fn): """Decide offline ONCE (local_files_only kwarg or env) and force it around the whole load. If we started online and hit a network error, retry once forced-offline. The network-up online path is unchanged: no window, no retry.""" @functools.wraps(fn) def _wrapper(*args, **kwargs): def finish(result): return _mark_loaded_revision( result, None, cache_dir = kwargs.get("cache_dir"), load_kwargs = kwargs ) if _get_effective_local_files_only(kwargs): kwargs["local_files_only"] = True with _force_hf_offline(): # Stamp inside the window: the env vars are restored on exit, so the request has to travel on the objects themselves to reach saving. return finish( _mark_loaded_local_files_only(fn(*args, **kwargs), kwargs.get("cache_dir")) ) _pb_were_disabled = _progress_bars_were_disabled() try: return finish(fn(*args, **kwargs)) except Exception as e: # Skip if not network-related, or already retried by a nested decorator (else outer layers reload the whole model again). if not _is_offline_related_error(e) or getattr(e, "_unsloth_offline_retried", False): raise # Holding `e` holds its frames, and those frames hold the half-built model, so the collect below could not free it and a large VLM OOMed on the reload the retry exists to make. A wrapper's cause/context carries tracebacks over the SAME frames, so the whole chain has to be released, not just the top. online_error = e _release_traceback_locals(online_error) # Retry OUTSIDE the except so the failed attempt's traceback (a partial model) is freed before reallocating, else a large VLM can OOM on the second load. try: gc.collect() clean_gpu_cache() except Exception: pass # A failed attempt may have left HF progress bars disabled; restore before retry. _restore_progress_bars(_pb_were_disabled) kwargs["local_files_only"] = True try: with _force_hf_offline(): return finish(fn(*args, **kwargs)) except Exception as e: # A real retry failure (corrupt checkpoint, OOM) is news and goes out as itself; only the empty-cache artifact is worth replacing. if not _empty_cache_artifact(e): # Tag so an enclosing _offline_aware_load skips its own redundant retry. try: e._unsloth_offline_retried = True except Exception: pass raise # The retry can load a whole cached model and only then trip over a missing tokenizer file, and this error outlives the call on the online one, so its frames would keep that model resident for as long as the caller holds it. _release_traceback_locals(e) retry_error = e # Report the ONLINE error: this retry only runs because of it, and its own failure names an empty cache badly (offline mode skips Transformers' "does not appear to have a file named" raise, so the user saw `AttributeError: 'NoneType' ... 'endswith'`). try: online_error._unsloth_offline_retried = True except Exception: pass # Raise OUTSIDE the handler above: inside it, Python would overwrite `__context__` with the cache miss, and that chain is often the only thing that still makes the online error classifiable as network-related. if online_error.__cause__ is None and online_error.__context__ is None: # No chain to lose, so chain the retry on where it also prints. raise online_error from retry_error _note_offline_retry(online_error, retry_error) raise online_error return _wrapper def _has_local_tokenizer_files(path): """True if a local dir has a loadable tokenizer (BPE vocab.json needs merges.txt; special_tokens_map.json is not required).""" return ( os.path.exists(os.path.join(path, "tokenizer.json")) or os.path.exists(os.path.join(path, "tokenizer.model")) or ( os.path.exists(os.path.join(path, "vocab.json")) and os.path.exists(os.path.join(path, "merges.txt")) ) or os.path.exists(os.path.join(path, "vocab.txt")) or os.path.exists(os.path.join(path, "spiece.model")) ) def _has_local_processor_files(path): """True if a local dir ships a processor/image-processor config (a VLM needs this to build AutoProcessor; tokenizer files alone are not enough).""" return os.path.exists(os.path.join(path, "processor_config.json")) or os.path.exists( os.path.join(path, "preprocessor_config.json") ) def _resolve_hub_repo_local_dir( repo_id, *, token = None, cache_dir = None, revision = None, # Default closed: a "resolve local dir" helper must not download. False here means five filenames each retried with backoff before it gives up. local_files_only = True, filenames = ( "tokenizer_config.json", "config.json", "tokenizer.json", "preprocessor_config.json", "processor_config.json", ), ): """A local snapshot directory for a Hub repo id when files are cached. On transformers 4.57.2 through 5.5.4, ``PreTrainedTokenizerFast.from_pretrained`` on a repo id can still call ``model_info()`` when ``local_files_only=True`` and no offline env var is set; loading from the resolved snapshot dir avoids that Hub probe. Fixed upstream in transformers 5.6.0 (huggingface/transformers#43603), so this can go once the floor is past that.""" if not isinstance(repo_id, str) or not repo_id: return None if os.path.isdir(repo_id): return repo_id if cache_dir is None: cache_dir = os.environ.get("HF_HUB_CACHE") from huggingface_hub import hf_hub_download for filename in filenames: try: path = hf_hub_download( repo_id = repo_id, filename = filename, token = token, cache_dir = cache_dir, local_files_only = local_files_only, revision = revision, ) if path and os.path.isfile(path): return os.path.dirname(path) except Exception: continue return None def _resolve_hub_repo_cached_file( repo_id, filename, *, token = None, cache_dir = None, local_files_only = True, revision = None, ): """Return a cached file path under a Hub snapshot, or None if absent.""" local_dir = _resolve_hub_repo_local_dir( repo_id, token = token, cache_dir = cache_dir, local_files_only = local_files_only, revision = revision, filenames = (filename,), ) if local_dir is None: return None path = os.path.join(local_dir, filename) return path if os.path.isfile(path) else None def _hub_repo_or_local_path( repo_id, *, token = None, cache_dir = None, local_files_only = False, filenames = None, revision = None, ): """Prefer a cached snapshot path over a Hub repo id when offline or ``local_files_only``.""" if isinstance(repo_id, str) and os.path.isdir(repo_id): return repo_id lfo = bool(local_files_only) or _env_says_offline() if not lfo: return repo_id local_dir = _resolve_hub_repo_local_dir( repo_id, token = token, cache_dir = cache_dir, local_files_only = True, revision = revision, filenames = filenames or ( "tokenizer_config.json", "config.json", "tokenizer.json", "preprocessor_config.json", "processor_config.json", ), ) return local_dir if local_dir is not None else repo_id def _load_pretrained_tokenizer_fast( tokenizer_name, *, padding_side = "left", token = None, trust_remote_code = False, cache_dir = None, local_files_only = False, revision = None, ): """Load ``PreTrainedTokenizerFast`` without Hub metadata probes when cached/offline. Needed on transformers 4.57.2-5.5.4; redundant once the floor is past 5.6.0.""" from transformers import PreTrainedTokenizerFast lfo = bool(local_files_only) or _env_says_offline() load_path = _hub_repo_or_local_path( tokenizer_name, token = token, cache_dir = cache_dir, local_files_only = lfo, revision = revision, filenames = ( "tokenizer_config.json", "tokenizer.json", "tokenizer.model", ), ) return PreTrainedTokenizerFast.from_pretrained( load_path, padding_side = padding_side, token = token, trust_remote_code = trust_remote_code, cache_dir = cache_dir, local_files_only = lfo, revision = revision, ) def _resolve_checkpoint_tokenizer_name( old_model_name, kwargs, require_processor = False, ): """tokenizer_name for a PEFT/checkpoint load: caller override, else the local checkpoint dir if self-sufficient, else None (base repo). Always popped from kwargs (also passed explicitly downstream). For a VLM (require_processor), the dir must also ship processor files; otherwise fall back to the base repo whose cached processor still loads.""" explicit = kwargs.pop("tokenizer_name", None) if explicit is not None: return explicit has_config = os.path.exists(os.path.join(old_model_name, "tokenizer_config.json")) if not (has_config and _has_local_tokenizer_files(old_model_name)): return None if require_processor and not _has_local_processor_files(old_model_name): return None return old_model_name