[ { "page": "Home", "title": "From source model to deployment", "subtitle": "Choose a preparation path, then save, reload, evaluate, and deploy.", "lanes": [ { "label": "Prepare a quantized model \u2014 calibration-based methods", "steps": [ { "title": "Load source + config", "detail": "Select a supported model, method, format, and device." }, { "title": "Prepare calibration", "detail": "Use representative inputs for the chosen task and model." }, { "title": "Quantize", "detail": "Capture, solve, replay, and pack the selected modules." } ] }, { "label": "Prepare a quantized model \u2014 calibration-free methods", "steps": [ { "title": "Load source + config", "detail": "Select RTN, GGUF, FP8, or BitsAndBytes as supported." }, { "title": "Read source weights", "detail": "No calibration forwards or activation statistics." }, { "title": "Convert and pack", "detail": "Apply the method's weight-only conversion contract." } ] }, { "label": "After either preparation path", "steps": [ { "title": "Save", "detail": "Export model tensors, config, and required auxiliary state." }, { "title": "Reload", "detail": "Validate the saved layout against the target runtime." }, { "title": "Evaluate", "detail": "Compare quality to a dense baseline with matched prompts." }, { "title": "Generate or serve", "detail": "Run locally or choose a compatible serving engine." } ] } ], "note": "The first two lanes are alternatives; both continue into the final lane. Existing quantized models start at Reload. Resume checkpoints are separate from inference exports.", "sources": [ "gptqmodel/models/auto.py", "gptqmodel/models/base.py", "gptqmodel/models/writer.py" ] }, { "page": "What-Is-Quantization", "title": "Storage and arithmetic are separate", "subtitle": "Low-bit weight storage does not determine activation precision.", "lanes": [ { "label": "Weight representation", "steps": [ { "title": "Dense weights", "detail": "Start from floating-point model weights." }, { "title": "Quantization method", "detail": "Choose low-bit codes and family-specific reconstruction data." }, { "title": "Compact model state", "detail": "Store codes with required scales, zeros or codebooks." } ] }, { "label": "Typical GPTQ or AWQ", "steps": [ { "title": "Floating-point inputs", "detail": "Calibration does not itself quantize inference activations." }, { "title": "Packed-weight compute", "detail": "Use a compatible kernel with floating-point activations." }, { "title": "Floating-point outputs", "detail": "Keep persistent weights packed, not expanded between calls." } ] }, { "label": "Explicit GPTQ W4A policy", "steps": [ { "title": "Floating-point inputs", "detail": "Use the strict INT4 GPTQ contract and a supported backend." }, { "title": "Low-bit operands", "detail": "Encode activation operands as FP8 or NVFP4 per the policy." }, { "title": "Specialized compute", "detail": "Execute the selected W4A path; saved weights remain INT4." } ] } ], "note": "Weight representation is an independent concern; the two activation lanes are alternatives, not sequential stages.", "sources": [ "gptqmodel/quantization/config.py", "gptqmodel/quantization/quantizer.py", "gptqmodel/nn_modules/qlinear/torch.py", "gptqmodel/nn_modules/qlinear/__init__.py" ] }, { "page": "Installation-and-Quickstart", "title": "Choose the starting lifecycle", "subtitle": "Calibration, calibration-free quantization and inference are alternative entry paths.", "lanes": [ { "label": "GPTQ calibration", "steps": [ { "title": "Load dense model", "detail": "GPTQModel.load with GPTQConfig and a suitable device." }, { "title": "Quantize with data", "detail": "Collect activations and quantize using representative inputs." }, { "title": "Save model artifact", "detail": "model.save writes the packed model and its configuration." }, { "title": "Reload and generate", "detail": "Load the saved directory with a compatible runtime." } ] }, { "label": "Calibration-free RTN", "steps": [ { "title": "Load dense model", "detail": "GPTQModel.load with RTNConfig and a supported layout." }, { "title": "Quantize weights", "detail": "model.quantize() uses the calibration-free lifecycle." }, { "title": "Save model artifact", "detail": "model.save writes the chosen output layout and metadata." }, { "title": "Reload and generate", "detail": "Load the saved directory with a compatible runtime." } ] }, { "label": "Existing quantized model", "steps": [ { "title": "Choose a checkpoint", "detail": "Match its method and layout to available hardware support." }, { "title": "GPTQModel.load", "detail": "Read quantization metadata and initialize the runtime." }, { "title": "model.generate", "detail": "Run inference without starting another quantization job." } ] } ], "note": "These lanes are alternatives. RTN is one calibration-free example. A saved artifact is not a CheckpointConfig resume checkpoint.", "sources": [ "gptqmodel/models/auto.py", "gptqmodel/models/base.py", "gptqmodel/models/writer.py", "gptqmodel/quantization/config.py", "gptqmodel/utils/calibration.py" ] }, { "page": "QuantizeConfig-Reference", "title": "Configuration normalization and dispatch", "subtitle": "Different entry points converge on method-specific configuration objects.", "lanes": [ { "label": "QuantizeConfig factory", "steps": [ { "title": "Constructor keywords", "detail": "Supply method, format, bits and relevant method options." }, { "title": "Normalize aliases", "detail": "Canonical method and format keywords take precedence." }, { "title": "Dispatch and filter", "detail": "Select a concrete class and retain its initializer fields." }, { "title": "Run class validation", "detail": "Resolve defaults, sub-configs and method constraints." } ] }, { "label": "Direct concrete class", "steps": [ { "title": "Choose a config class", "detail": "For example, GPTQConfig, RTNConfig or GGUFConfig." }, { "title": "Normalize aliases", "detail": "Use that class's initializer; unknown keywords raise." }, { "title": "Run class validation", "detail": "Apply its post-init rules without factory class selection." } ] }, { "label": "Checkpoint config import", "steps": [ { "title": "Read saved fields", "detail": "from_pretrained finds standalone or nested config metadata." }, { "title": "Import aliases and meta", "detail": "from_quant_config applies JSON-specific compatibility rules." }, { "title": "Dispatch and filter", "detail": "Resolve the target class and its accepted initializer fields." }, { "title": "Run class validation", "detail": "Build the normalized method-specific configuration." } ] } ], "note": "Entry lanes are alternatives. JSON and constructor alias precedence differ. Config acceptance does not guarantee kernel compatibility.", "sources": [ "gptqmodel/quantization/config.py", "gptqmodel/utils/importer.py" ] }, { "page": "Checkpoint-Formats-and-Compatibility", "title": "Portable artifacts and runtime layouts", "subtitle": "A format label, tensor layout and executable backend must agree.", "lanes": [ { "label": "Load for inference", "steps": [ { "title": "Read metadata", "detail": "Resolve method, format, bits, grouping and model definition." }, { "title": "Validate a backend", "detail": "Check device, dtype, shape and dependency constraints." }, { "title": "Load packed tensors", "detail": "Honor layout axes, zero convention and auxiliary state." }, { "title": "Initialize and run", "detail": "Repack only where needed; execute compatible kernels." } ] }, { "label": "Save for distribution", "steps": [ { "title": "Checkpoint-facing model", "detail": "Reconstruct loaded layouts when required by the writer." }, { "title": "Write tensors and config", "detail": "Save packed tensors with matching quantization metadata." }, { "title": "Compatible reader", "detail": "The consumer must recognize both metadata and tensor layout." } ] } ], "note": "Loading and saving are independent concerns. Runtime repacking does not relabel a checkpoint or universally convert method families.", "sources": [ "gptqmodel/models/loader.py", "gptqmodel/models/writer.py", "gptqmodel/utils/importer.py", "gptqmodel/utils/packing.py", "gptqmodel/quantization/config.py" ] }, { "page": "GPTQ", "title": "Three routes to rounded weights", "subtitle": "Calibration and algorithm choice are separate from checkpoint layout.", "lanes": [ { "label": "GPTQ", "steps": [ { "title": "Capture calibration", "detail": "Replay each layer and subset to observe module inputs" }, { "title": "Build inverse factor", "detail": "Accumulate Gram statistics, damp, and factor the Hessian" }, { "title": "Round and compensate", "detail": "Quantize columns and propagate errors within and across blocks" }, { "title": "Pack weights", "detail": "Encode rounded weights, scales, zeros, and group metadata" } ] }, { "label": "GPTAQ", "steps": [ { "title": "Capture native inputs", "detail": "NativeProcessor retains the floating-point input reference" }, { "title": "Compare replayed inputs", "detail": "Build Hessian and native-versus-replayed cross statistics" }, { "title": "Apply GPTAQ correction", "detail": "Combine rounding compensation with the input-mismatch term" }, { "title": "Pack weights", "detail": "Use a compatible GPTQ-family layout, not a new algorithm tag" } ] }, { "label": "RTN: no calibration", "steps": [ { "title": "Read module weights", "detail": "RTNConfig selects the separate weight-only lifecycle" }, { "title": "Fit group ranges", "detail": "Derive scales and zeros; optional smoothing adjusts ranges" }, { "title": "Round directly", "detail": "No Hessian accumulation or inter-column error compensation" }, { "title": "Pack compatible layout", "detail": "Export GPTQ or AWQ storage without running those algorithms" } ] } ], "note": "Alternative paths, not consecutive stages. GPTQ/GPTAQ repeat per layer and subset; GPTQ_V2 selects zero-point storage, not GPTAQ.", "sources": [ "gptqmodel/quantization/config.py", "gptqmodel/models/base.py", "gptqmodel/looper/module_looper.py", "gptqmodel/looper/gptq_processor.py", "gptqmodel/looper/native_processor.py", "gptqmodel/quantization/gptq.py", "gptqmodel/quantization/gptaq.py", "gptqmodel/quantization/rtn.py" ] }, { "page": "AWQ", "title": "Scale search versus sparse-coverage fallback", "subtitle": "AWQ chooses a per-module route inside a calibrated layer pass.", "lanes": [ { "label": "Normal calibration coverage", "steps": [ { "title": "Capture scaling groups", "detail": "Collect input features for model-defined groups of linears" }, { "title": "Search 20 ratios", "detail": "Replay candidate scales and retain the lowest output MSE" }, { "title": "Fold scales", "detail": "Apply selected scales; optionally search clipping bounds" }, { "title": "Quantize and pack", "detail": "Pseudo-quantize weights, then use the selected AWQ layout" } ] }, { "label": "Fallback selected", "steps": [ { "title": "Check coverage", "detail": "Identify modules with insufficient calibration evidence" }, { "title": "Skip scale and clip", "detail": "Exclude fallback modules from normal activation-aware search" }, { "title": "Quantize and pack", "detail": "apply_quant uses an empty scales_list for these modules" } ] } ], "note": "These are alternative module paths within AWQ, not calibration-free RTN. Clipping defaults on; fallback does not invoke the generic RTN processor.", "sources": [ "gptqmodel/looper/awq_processor.py", "gptqmodel/quantization/awq/quantize/scale.py", "gptqmodel/quantization/awq/utils/packing_utils.py", "gptqmodel/models/base.py" ] }, { "page": "QQQ", "title": "Calibrated W4 weights and dynamic A8 execution", "subtitle": "Weight preparation and native inference have different responsibilities.", "lanes": [ { "label": "Weight preparation", "steps": [ { "title": "Collect Gram statistics", "detail": "Calibration supplies the per-module input Hessian" }, { "title": "Damp and factor", "detail": "Prepare the inverse-Hessian factor for error compensation" }, { "title": "Quantize 4-bit weights", "detail": "Round in 128-column blocks with Hessian error updates" }, { "title": "Pack QQQ state", "detail": "Store B, s_channel, and s_group when groups are enabled" } ] }, { "label": "Native QQQ forward", "steps": [ { "title": "Use FP16 working input", "detail": "Remember the original dtype; flatten activation rows" }, { "title": "Quantize rows to INT8", "detail": "Use a per-row absmax divided by 127 for activation scales" }, { "title": "Run packed W4A8 GEMM", "detail": "qqq_gemm consumes INT8 inputs, packed B, and scales" }, { "title": "Restore output contract", "detail": "Apply bias or adapter when present; restore shape and dtype" } ] } ], "note": "Independent preparation and execution concerns; neither uses model rotation. QQQ_TORCH and MLX have separate execution paths.", "sources": [ "gptqmodel/quantization/qqq.py", "gptqmodel/looper/qqq_processor.py", "gptqmodel/nn_modules/qlinear/qqq.py", "gptqmodel/utils/qqq.py" ] }, { "page": "VPTQ", "title": "One codebook stream or residual refinement", "subtitle": "Residual settings change the vector recipe, not the calibration requirement.", "lanes": [ { "label": "Residual disabled", "steps": [ { "title": "Learn main codebooks", "detail": "Fit output-axis vectors within each input-channel group" }, { "title": "Assign and compensate", "detail": "Choose centroids while propagating errors across columns" }, { "title": "Pack main indices", "detail": "Store codebooks, one index stream, and reconstruction metadata" } ] }, { "label": "Residual enabled: default", "steps": [ { "title": "Learn main codebooks", "detail": "Fit the same grouped vectors before residual refinement" }, { "title": "Run first pass", "detail": "Use main centroids and retain the compensated-pass errors" }, { "title": "Fit residual codebooks", "detail": "Cluster error vectors for the main input-channel groups" }, { "title": "Rerun and pack", "detail": "Assign main plus residual centroids; store both index streams" } ] } ], "note": "Alternative recipes using calibration. Residuals are disabled with num_res_centroids=[-1, -1]; outlier residual codebooks are unsupported.", "sources": [ "gptqmodel/quantization/config.py", "gptqmodel/quantization/vptq.py", "gptqmodel/looper/vptq_processor.py", "gptqmodel/nn_modules/qlinear/vptq_torch.py" ] }, { "page": "ParoQuant", "title": "Learned coordinates and packed inference", "subtitle": "ParoQuant stores transformed weights together with required rotation metadata.", "lanes": [ { "label": "Module-scope optimization", "steps": [ { "title": "Capture activation rows", "detail": "Form floating-point targets and training/validation selections" }, { "title": "Learn angles and scales", "detail": "Rotation stage updates theta and bounded channel scales" }, { "title": "Fine-tune quantization", "detail": "Initialize the quantizer; fit weights and quantizer parameters" }, { "title": "Pack runtime state", "detail": "Keep INT4 weights, pairs, theta, and reciprocal channel scales" } ] }, { "label": "Packed inference", "steps": [ { "title": "Scale input channels", "detail": "Multiply by saved reciprocal channel_scales" }, { "title": "Apply learned rotations", "detail": "Use pairs and theta for the staged Givens transforms" }, { "title": "Evaluate packed weights", "detail": "Project transformed inputs using a compatible ParoQuant backend" }, { "title": "Restore output shape", "detail": "Bias is included; an optional adapter uses the original input" } ] } ], "note": "Independent optimization and execution concerns. Grouped scopes use layer replay objectives; zero epochs skip updates, not export. Packing is not a resume checkpoint.", "sources": [ "gptqmodel/quantization/paroquant/optimization.py", "gptqmodel/looper/paroquant_processor.py", "gptqmodel/nn_modules/qlinear/paroquant.py", "gptqmodel/utils/paroquant.py" ] }, { "page": "EXL3", "title": "From captured inputs to EXL3 trellises", "subtitle": "Calibration replay and checkpoint loading are separate concerns.", "lanes": [ { "label": "Per-layer, per-subset solve", "steps": [ { "title": "Capture selected modules", "detail": "Collect activation batches and Hessians with GPTQ capture tasks" }, { "title": "Solve EXL3 trellises", "detail": "Use module-specific integer K, codebook and output-scale policy" }, { "title": "Replay solved weights", "detail": "Later subsets observe reconstructed earlier projections" }, { "title": "Finalize EXL3 modules", "detail": "Replace dense modules with trellis and sign-scale tensors" } ] }, { "label": "Load an EXL3 artifact", "steps": [ { "title": "Read EXL3 configuration", "detail": "Require tensor_storage metadata and select a supported backend" }, { "title": "Rebuild packed modules", "detail": "Create EXL3 placeholders and load their declared tensors" }, { "title": "Run the selected runtime", "detail": "Use extension, Torch or explicit MLX inference as supported" } ] } ], "note": "The public quantization entry point requires CUDA/HIP and calibration. Torch/MLX inference does not enable CPU/MLX quantization through that entry point.", "sources": [ "gptqmodel/models/base.py", "gptqmodel/models/loader.py", "gptqmodel/looper/exllamav3_processor.py", "gptqmodel/looper/module_looper.py", "gptqmodel/utils/exllamav3.py" ] }, { "page": "FP8-and-NVFP4", "title": "Choose the floating-point quantization path", "subtitle": "Weight format, source decoding and activation policy are different choices.", "lanes": [ { "label": "Weight-only FP8 export", "steps": [ { "title": "Configure FP8 weights", "detail": "FP8Config chooses the weight dtype and scale layout" }, { "title": "Pack original weights", "detail": "Fit FP32 inverse scales without activation calibration" }, { "title": "Save FP8 checkpoint", "detail": "Store FP8 weight tensors and weight_scale_inv" } ] }, { "label": "GPTQ with a W4A policy", "steps": [ { "title": "Set activation policy", "detail": "GPTQConfig selects w4afp8 or w4a_nvfp4" }, { "title": "Calibrate and solve", "detail": "Run GPTQ capture and policy-aware activation replay" }, { "title": "Save INT4 weights", "detail": "Keep GPTQ packing plus activation-policy metadata" }, { "title": "Run encoded activations", "detail": "Use the matching W4A backend on supported hardware" } ] }, { "label": "Native source inference", "steps": [ { "title": "Inspect source metadata", "detail": "Recognize supported FP8 or NVFP4 weight and scale declarations" }, { "title": "Map source tensors", "detail": "Validate layouts and normalize source scale conventions" }, { "title": "Use the source runtime", "detail": "Execute the supported weight format without a new GPTQ solve" } ] }, { "label": "Native FP8 re-quantization", "steps": [ { "title": "Choose a fresh target", "detail": "Supply a supported GPTQ or AWQ quantization configuration" }, { "title": "Decode module weights", "detail": "Reconstruct source weights locally for the new solver" }, { "title": "Calibrate target weights", "detail": "Run the target method; do not recover pre-source precision" } ] } ], "note": "These lanes are alternatives, not a conversion chain. W4A stores INT4 weights, not FP8/NVFP4 weights. Native-source and hardware support remain layout-specific.", "sources": [ "gptqmodel/quantization/config.py", "gptqmodel/models/loader.py", "gptqmodel/looper/weight_only_processor.py", "gptqmodel/nn_modules/qlinear/fp8.py", "gptqmodel/quantization/dtype.py", "gptqmodel/looper/gptq_processor.py", "gptqmodel/nn_modules/qlinear/w4a_llama_stream.py" ] }, { "page": "GGUF", "title": "GGUF blocks are not a GGUF file export", "subtitle": "Separate weight-only checkpoint creation from model-file import.", "lanes": [ { "label": "Create a GPTQModel checkpoint", "steps": [ { "title": "Choose a GGUF subtype", "detail": "Normalize bits and format into a GGUFBits specification" }, { "title": "Pack source matrices", "detail": "Pad and quantize original weights into GGML block bytes" }, { "title": "Save checkpoint directory", "detail": "Store packed tensors and config, not a standalone .gguf file" } ] }, { "label": "Eligible packed .gguf import", "steps": [ { "title": "Check import eligibility", "detail": "Require Qwen3 and a supported common projection tensor type" }, { "title": "Install packed blocks", "detail": "Map byte rows into quantized linears and validate shapes" }, { "title": "Execute block kernels", "detail": "Use a backend allowed by the subtype and import route" } ] }, { "label": "Dense HF .gguf import", "steps": [ { "title": "Resolve the model file", "detail": "Use a supported local or repository .gguf model source" }, { "title": "Decode GGUF tensors", "detail": "Hugging Face import reconstructs floating-point weights" }, { "title": "Run the dense model", "detail": "This route is not packed GGUF inference" } ] } ], "note": "Lanes are separate routes. Packing needs no calibration; .gguf import depends on architecture, tensor type, processor and profile. Q1_0 auto/fast chooses dense import.", "sources": [ "gptqmodel/quantization/config.py", "gptqmodel/nn_modules/qlinear/gguf.py", "gptqmodel/utils/internal_gguf.py", "gptqmodel/models/loader.py", "gptqmodel/models/writer.py" ] }, { "page": "BitsAndBytes", "title": "Persist the state that each BnB format needs", "subtitle": "Four-bit codebooks and rowwise INT8 have different checkpoint contracts.", "lanes": [ { "label": "FP4 or NF4 packing", "steps": [ { "title": "Choose codebook and block", "detail": "Set FP4/NF4, block_size and optional compressed statistics" }, { "title": "Quantize weight blocks", "detail": "Encode values with block absmax and the selected codebook" }, { "title": "Persist four-bit state", "detail": "Save uint8 weight with scales, codebook and weight_quant_state" } ] }, { "label": "INT8 packing", "steps": [ { "title": "Select rowwise INT8", "detail": "Use bits=8 and format=int8 rather than a four-bit codebook" }, { "title": "Scale and round rows", "detail": "Quantize weights with one scale per output row" }, { "title": "Persist INT8 state", "detail": "Save int8 weight with weight_scb row-scale metadata" } ] }, { "label": "Checkpoint loading", "steps": [ { "title": "Validate format metadata", "detail": "Normalize BnB aliases and require the matching state tensors" }, { "title": "Restore packed payload", "detail": "Rebuild QuantState or the INT8 weight/SCB contract" }, { "title": "Select a valid runtime", "detail": "Use BITSANDBYTES or explicit MLX with its support checks" } ] } ], "note": "The packing lanes are alternatives and do not consume calibration data. block_size is not GPTQ group_size; INT8 does not use FP4/NF4 codebook state.", "sources": [ "gptqmodel/quantization/config.py", "gptqmodel/nn_modules/qlinear/bitsandbytes.py", "gptqmodel/nn_modules/qlinear/mlx_bitsandbytes.py", "gptqmodel/looper/weight_only_processor.py", "gptqmodel/models/loader.py" ] }, { "page": "EoRA", "title": "Fit a correction without changing the packed base", "subtitle": "Activation-weighted factorization produces a separate low-rank adapter.", "lanes": [ { "label": "Training-free adapter generation", "steps": [ { "title": "Capture activation stats", "detail": "Accumulate and merge input second moments from calibration" }, { "title": "Weight the residual", "detail": "Scale original-minus-quantized weights in retained eigenspace" }, { "title": "Factor low-rank A and B", "detail": "Use reduced SVD; the correction is B times A" }, { "title": "Save a separate adapter", "detail": "Export factors and rank metadata beside a compatible base" } ] }, { "label": "Adapter-aware inference", "steps": [ { "title": "Load the quantized base", "detail": "Select a backend that supports Lora for the base format" }, { "title": "Load matching factors", "detail": "Validate adapter paths, target modules, shapes and ranks" }, { "title": "Add the low-rank output", "detail": "Apply the correction separately from the packed base matmul" } ] } ], "note": "Generation requires calibration and matching original weights. The post-hoc API is limited to the GPTQ TorchLinear path. Training-free does not mean calibration-free or merged weights.", "sources": [ "gptqmodel/eora/eora.py", "gptqmodel/looper/eora_processor.py", "gptqmodel/looper/dequantize_processor.py", "gptqmodel/adapter/adapter.py", "gptqmodel/models/writer.py" ] }, { "page": "Rotation-Smoothing-and-Norm-Tuning", "title": "Choose where to intervene in quantization", "subtitle": "Basis changes, weight-range fitting and norm training are independent options.", "lanes": [ { "label": "Rotation before GPTQ", "steps": [ { "title": "Check rotation support", "detail": "Use supported Llama/Qwen2 and a rotation-aware GPTQ backend" }, { "title": "Fuse norms and rotate", "detail": "Change weight basis and install matching online Hadamard" }, { "title": "Quantize and replay", "detail": "Keep the down-projection activation transform during capture" }, { "title": "Reload with rotation", "detail": "GPTQ/GPTQ_V2 metadata restores online Hadamard handling" } ] }, { "label": "Optional weight smoothing", "steps": [ { "title": "Choose a supported path", "detail": "Use RTN, a GPTQ/QQQ fallback or a supported direct packer" }, { "title": "Apply the weight policy", "detail": "Use the selected clipping or fitting method and its gates" }, { "title": "Quantize weight blocks", "detail": "Reapply compensation only where the consumer supports it" } ] }, { "label": "Post-load norm tuning", "steps": [ { "title": "Reload the saved student", "detail": "Use materialized tensors and an autograd-capable backend" }, { "title": "Select tagged norms", "detail": "Choose eligible gains and biases through NormConfig" }, { "title": "Distill teacher logits", "detail": "Update norm parameters; keep packed weights unchanged" }, { "title": "Save norm replacements", "detail": "Preserve other tensors from the source safetensors shards" } ] } ], "note": "These lanes are independent choices, not a mandatory stack. Shared model rotation is not supported for QQQ/EXL3 checkpoints. Smoothing is not activation-based SmoothQuant.", "sources": [ "gptqmodel/models/base.py", "gptqmodel/models/loader.py", "gptqmodel/quantization/rotation/rotation.py", "gptqmodel/quantization/fallback_smooth.py", "gptqmodel/quantization/norm_tuning.py", "gptqmodel/models/writer.py" ] }, { "page": "Quantization-Lifecycle", "title": "Two quantization lifecycles", "subtitle": "The configuration selects calibration-based or weight-only processing.", "lanes": [ { "label": "Calibration-based", "steps": [ { "title": "Prepare calibration", "detail": "Normalize supported samples into calibration batches" }, { "title": "Capture first inputs", "detail": "Stop at the first language layer and cache its inputs" }, { "title": "Quantize layer subsets", "detail": "Collect statistics, solve, and replay as the processor requires" }, { "title": "Finalize model", "detail": "Drain packing work and mark the in-memory model quantized" } ] }, { "label": "Weight-only", "steps": [ { "title": "Select conversion", "detail": "Use a configuration that selects the weight-only lifecycle" }, { "title": "Walk weight targets", "detail": "Materialize eligible modules without calibration forwards" }, { "title": "Convert and pack", "detail": "Convert each target directly from its source weights" }, { "title": "Finalize model", "detail": "Complete the selected format without activation statistics" } ] } ], "note": "The lanes are alternatives. A later save() exports the model; it is not a quantization resume checkpoint.", "sources": [ "gptqmodel/models/base.py", "gptqmodel/quantization/config.py", "gptqmodel/looper/module_looper.py", "gptqmodel/looper/stage_inputs_capture.py", "gptqmodel/looper/weight_only_looper.py", "gptqmodel/looper/weight_only_processor.py" ] }, { "page": "Looper-Processors", "title": "Repeat layers and subsets", "subtitle": "ModuleLooper schedules work while each processor defines its operations.", "lanes": [ { "label": "Outer scope: each layer", "steps": [ { "title": "Stage layer", "detail": "Materialize the layer and prepare its cached inputs" }, { "title": "Run processor subsets", "detail": "Execute each processor's ordered subset plans" }, { "title": "Propagate outputs", "detail": "Carry replay outputs into the next layer's input caches" }, { "title": "Finalize layer work", "detail": "Queue packing and cleanup; drain when safety requires it" } ] }, { "label": "Inner scope: each subset", "steps": [ { "title": "Preprocess targets", "detail": "Create algorithm tasks for the eligible named modules" }, { "title": "Capture if required", "detail": "Attach hooks and forward the cached calibration batches" }, { "title": "Process targets", "detail": "Solve or transform using the processor's collected state" }, { "title": "Replay if required", "detail": "Forward reconstructed weights according to execution flags" } ] } ], "note": "These are nested scopes, not consecutive lanes: repeat subsets inside each processor and layer. Native capture merges subsets; processor flags can skip capture or replay.", "sources": [ "gptqmodel/looper/module_looper.py", "gptqmodel/looper/stage_layer.py", "gptqmodel/looper/stage_subset.py", "gptqmodel/looper/loop_processor.py", "gptqmodel/looper/native_processor.py" ] }, { "page": "Model-Definitions-and-Module-Tree", "title": "From definition to execution plans", "subtitle": "A selected model definition supplies both target and capture views.", "lanes": [ { "label": "Quantization target view", "steps": [ { "title": "Select definition", "detail": "MODEL_MAP dispatches on the checkpoint's model_type" }, { "title": "Resolve effective tree", "detail": "Apply a method override or eligible autodetection if needed" }, { "title": "Build simple plan", "detail": "Expand experts; remove dense, capture-only, and excluded leaves" }, { "title": "Process target subsets", "detail": "Resolve layer-relative paths in the declared execution order" } ] }, { "label": "Capture and planning view", "steps": [ { "title": "Use the same tree", "detail": "Read the selected definition's effective_module_tree" }, { "title": "Build full plan", "detail": "Retain dense entries; request capture-only parents when needed" }, { "title": "Prepare capture", "detail": "Use model hooks and the ordered plan for replay and scaling" } ] } ], "note": "The lanes are separate views of one definition. A full-plan entry is not automatically a quantization target; skipped weights remain dense.", "sources": [ "gptqmodel/models/auto.py", "gptqmodel/models/base.py", "gptqmodel/utils/model.py", "gptqmodel/looper/stage_inputs_capture.py", "gptqmodel/looper/stage_subset.py" ] }, { "page": "Memory-Offload-and-Multi-GPU", "title": "Separate memory and device controls", "subtitle": "Memory controls govern separate parts of the quantization workload.", "lanes": [ { "label": "Source weights: lazy path", "steps": [ { "title": "Index checkpoint shards", "detail": "LazyTurtle retains a checkpoint tensor map for the shell" }, { "title": "Read requested tensors", "detail": "Resolve aliases and load the needed source subtree" }, { "title": "Materialize active work", "detail": "Populate active modules without a second full dense model" } ] }, { "label": "Calibration cache", "steps": [ { "title": "Capture input batches", "detail": "Retain layer inputs, masks, and replay kwargs" }, { "title": "Choose cache storage", "detail": "Choose local, explicit, or balanced input-cache placement" }, { "title": "Replay cached batches", "detail": "Move inputs to the device executing each forward" } ] }, { "label": "Compute placement", "steps": [ { "title": "Find eligible devices", "detail": "Apply visibility, compute filtering, and method restrictions" }, { "title": "Resolve device pools", "detail": "Validate dense and MoE pools against eligible devices" }, { "title": "Place module groups", "detail": "Balance eligible groups while keeping expert families together" } ] }, { "label": "Completed-module offload", "steps": [ { "title": "Finalize module", "detail": "Finish its packed representation before offload" }, { "title": "Write eligible payload", "detail": "Persist completed tensors when disk offload is enabled" }, { "title": "Release tensor storage", "detail": "Replace offloaded tensor storage with meta placeholders" } ] } ], "note": "The lanes are independent concerns, not consecutive stages. Cache storage is calibration-only; lazy loading and disk offload are conditional. Pools do not enable tensor parallelism.", "sources": [ "gptqmodel/models/loader.py", "gptqmodel/utils/structure.py", "gptqmodel/looper/stage_inputs_capture.py", "gptqmodel/looper/module_looper.py", "gptqmodel/looper/stage_subset.py", "gptqmodel/utils/offload.py" ] }, { "page": "Multimodal-and-MoE-Models", "title": "Architecture-specific calibration", "subtitle": "Model definitions control media preparation, decoder targets, and expert coverage.", "lanes": [ { "label": "Multimodal input capture", "steps": [ { "title": "Select supported inputs", "detail": "Check the definition's accepted sample and media formats" }, { "title": "Prepare model inputs", "detail": "Use that model's processor or custom preparation" }, { "title": "Capture language inputs", "detail": "Run model-specific hooks up to the first language layer" }, { "title": "Apply decoder plan", "detail": "Quantize declared projections while excluded modules stay dense" } ] }, { "label": "Optional MoE coverage", "steps": [ { "title": "Choose routing policy", "detail": "Select a supported top-k override or expert replay bypass" }, { "title": "Collect expert inputs", "detail": "Use model lifecycle hooks to broaden calibration coverage" }, { "title": "Restore normal routing", "detail": "Restore patched forwards or top-k when the context exits" } ] } ], "note": "The lanes are independent calibration concerns, not a required combination. Media support varies by definition; temporary routing changes do not set inference policy.", "sources": [ "gptqmodel/models/base.py", "gptqmodel/models/definitions/base_qwen2_vl.py", "gptqmodel/models/definitions/base_qwen3_vl.py", "gptqmodel/models/definitions/mllama.py", "gptqmodel/models/moe_lifecycle.py", "gptqmodel/looper/module_looper.py", "gptqmodel/quantization/config.py" ] }, { "page": "Loading-and-Inference", "title": "Loading intents and dispatch", "subtitle": "Checkpoint detection distinguishes dense, source, and saved quantized models.", "lanes": [ { "label": "Dense inference", "steps": [ { "title": "Dense checkpoint", "detail": "Start with a non-quantized source model" }, { "title": "No QuantizeConfig", "detail": "GPTQModel.load with quantize_config=None" }, { "title": "HF model placement", "detail": "from_pretrained uses the requested device_map" }, { "title": "Native generation", "detail": "The wrapper generates without quantization preparation" } ] }, { "label": "Prepare quantization", "steps": [ { "title": "Source checkpoint", "detail": "Start with a supported source model" }, { "title": "Supply QuantizeConfig", "detail": "Choose the intended quantization method and settings" }, { "title": "from_pretrained", "detail": "Prepare source weights and quantization placement" }, { "title": "Ready to quantize", "detail": "model.quantize is a separate, method-specific operation" } ] }, { "label": "Saved quantized model", "steps": [ { "title": "Recognized metadata", "detail": "Supported quantization metadata or standalone config" }, { "title": "from_quantized", "detail": "Read the saved method, format, and quantization contract" }, { "title": "Initialize runtime", "detail": "Validate and load the selected native backend or engine" }, { "title": "Runtime generation", "detail": "Use generation arguments supported by that runtime" } ] } ], "note": "Alternative intents, not sequential phases. Detected quantized inputs normally take from_quantized even when QuantizeConfig is supplied; native-float source exceptions are documented in the page.", "sources": [ "gptqmodel/models/auto.py", "gptqmodel/models/loader.py", "gptqmodel/models/base.py" ] }, { "page": "Inference-Backends", "title": "Backend selection boundaries", "subtitle": "Local selection and external engines enforce different contracts.", "lanes": [ { "label": "Local AUTO", "steps": [ { "title": "Method and format", "detail": "Use the checkpoint contract and requested device targets" }, { "title": "Rank declarations", "detail": "Positive SUPPORTS_FORMATS priorities define candidate order" }, { "title": "Validate candidates", "detail": "Check bits, groups, dtype, devices, and runtime requirements" }, { "title": "Select a valid class", "detail": "Normal single-select returns the first compatible kernel" } ] }, { "label": "Explicit local backend", "steps": [ { "title": "Named BACKEND", "detail": "Normalize the backend name for the quantization method" }, { "title": "Validate selection", "detail": "Apply the requested backend's format and hardware checks" }, { "title": "Initialize or reject", "detail": "Incompatible selection raises instead of promising AUTO" } ] }, { "label": "vLLM or SGLang", "steps": [ { "title": "Engine requested", "detail": "Choose VLLM or SGLANG rather than a local qlinear kernel" }, { "title": "Check engine format", "detail": "The loader applies the engine-specific format allowlist" }, { "title": "Engine initialization", "detail": "Delegate model loading to the installed engine" }, { "title": "Engine generation", "detail": "The wrapper uses the engine adapter, not local qlinear" } ] } ], "note": "Alternative routes. Format-specific loader branches can precede local selection; AUTO is priority-based, not benchmarking. MLX has its own conversion route.", "sources": [ "gptqmodel/utils/backend.py", "gptqmodel/utils/importer.py", "gptqmodel/models/loader.py" ] }, { "page": "Kernel-Backends-Reference", "title": "Packed-layer state lifecycle", "subtitle": "Raw packing, repacking, and state changes have different initialization needs.", "lanes": [ { "label": "Raw packing", "steps": [ { "title": "Quantizer outputs", "detail": "Original layer, scales, zeros, and method-specific metadata" }, { "title": "hf_pack_layer", "detail": "Encode into a compatible raw-packing kernel module" }, { "title": "Initialize if required", "detail": "post_init may need scratch or other runtime arguments" }, { "title": "Layer forward", "detail": "Execute with the kernel's packed runtime representation" } ] }, { "label": "Existing packed source", "steps": [ { "title": "Checkpoint-layout layer", "detail": "Preserve source codes, zero conventions, and scale layout" }, { "title": "hf_repack_layer", "detail": "Use a supported repack hook or compatible tensor transfer" }, { "title": "Initialize target", "detail": "post_init defaults on; defer it when scratch is needed" }, { "title": "Layer forward", "detail": "Execute the target kernel without relabeling the checkpoint" } ] }, { "label": "Changed packed state", "steps": [ { "title": "State changes", "detail": "Weights mutate, a state dict loads, or placement changes" }, { "title": "Invalidate owned state", "detail": "Use the owning class's cache and handle invalidation rules" }, { "title": "Reinitialize as needed", "detail": "Rebuild required packed ABI state from current weights" }, { "title": "Next forward", "detail": "Consume current packed weights, not stale cached state" } ] } ], "note": "Independent entry routes. Initialization is kernel-specific; scratch is not universal. Dense or expanded weight caches may not survive their consuming operation.", "sources": [ "gptqmodel/utils/packing.py", "gptqmodel/nn_modules/qlinear/__init__.py", "gptqmodel/nn_modules/qlinear/_runtime_cache.py", "gptqmodel/nn_modules/qlinear/exllamav2.py" ] }, { "page": "MLX-Apple-Silicon", "title": "MLX artifacts versus runtimes", "subtitle": "Native model quantization and packed-checkpoint conversion are separate routes.", "lanes": [ { "label": "Native MLX-LM quantization", "steps": [ { "title": "Load in MLX-LM", "detail": "quantize_mlx loads the source model lazily" }, { "title": "Calibration tokens", "detail": "Use supplied rank-2 tokens or MLX-LM load_data" }, { "title": "GPTQ or supported AWQ", "detail": "Run native GPTQ or the model-specific MLX-LM AWQ path" }, { "title": "Save MLX-LM artifact", "detail": "Save model, tokenizer, and config; return output_path" } ] }, { "label": "Packed checkpoint transfer", "steps": [ { "title": "Saved quantized model", "detail": "GPTQModel.load selects backend mlx" }, { "title": "Validate source state", "detail": "Check holder format, shapes, zeros, and runtime metadata" }, { "title": "Transcode packed state", "detail": "Transfer compatible codes and metadata without requantizing" }, { "title": "Construct MLX runtime", "detail": "Install native or custom layers and generate with prompt" } ] } ], "note": "Alternative routes. Failed transfer may requantize ordinary GPTQ/AWQ or reject special formats. A custom converted runtime is not automatically a stock MLX-LM reloadable export.", "sources": [ "gptqmodel/models/auto.py", "gptqmodel/quantization/mlx_native.py", "gptqmodel/utils/mlx.py", "gptqmodel/nn_modules/qlinear/mlx.py", "gptqmodel/models/loader.py" ] }, { "page": "HuggingFace-Integration", "title": "Hugging Face integration boundaries", "subtitle": "Transformers loading and Hub publication use different entry points.", "lanes": [ { "label": "Transformers loading", "steps": [ { "title": "Import gptqmodel", "detail": "Apply available process-local compatibility patches" }, { "title": "Read checkpoint config", "detail": "AutoConfig supplies quantization metadata" }, { "title": "Choose method backend", "detail": "Use a backend compatible with the saved method and format" }, { "title": "AutoModel loading", "detail": "Transformers and its quantizer integration load the model" } ] }, { "label": "Publish a local artifact", "steps": [ { "title": "GPTQModel writer", "detail": "model.save serializes the quantized wrapper locally" }, { "title": "Check artifact contents", "detail": "Include configs, weights, tokenizer, and required auxiliaries" }, { "title": "Reload for verification", "detail": "Check the saved artifact with its intended consumer" }, { "title": "Upload with Hub client", "detail": "Publish that directory using the external Hub API or CLI" } ] } ], "note": "Independent concerns, not consecutive lanes. Forwarded Transformers push_to_hub is not GPTQModel's quantized writer; Hub hosting does not guarantee runtime compatibility.", "sources": [ "gptqmodel/__init__.py", "gptqmodel/utils/importer.py", "gptqmodel/models/base.py", "gptqmodel/models/writer.py", "gptqmodel/utils/hub.py" ] }, { "page": "Serving", "title": "Three separate serving paths", "subtitle": "Choose the local HTTP shim or an in-process engine adapter.", "lanes": [ { "label": "Built-in HTTP shim", "steps": [ { "title": "Receive chat request", "detail": "POST /v1/chat/completions on the loaded model." }, { "title": "Render and tokenize", "detail": "Apply the chat template and place input IDs on the device." }, { "title": "Native generation", "detail": "Prompt length plus max_tokens sets max_length." }, { "title": "Return text choices", "detail": "Decode token IDs into choices.text, not chat messages." } ] }, { "label": "vLLM adapter", "steps": [ { "title": "Normalize prompts", "detail": "Accept text or token IDs; filter masks for token inputs." }, { "title": "Build SamplingParams", "detail": "Convert HF kwargs or use explicit native sampling parameters." }, { "title": "Call LLM.generate", "detail": "Run the in-process engine, not an HTTP server." }, { "title": "Return CPU token rows", "detail": "Combine prompt and generated IDs, then right-pad rows." } ] }, { "label": "SGLang Engine adapter", "steps": [ { "title": "Validate inputs", "detail": "Text or token IDs; token masks may remove only edge padding." }, { "title": "Normalize sampling", "detail": "Map token limits and merge recognized top-level settings." }, { "title": "Call Engine.generate", "detail": "Pass prompts or IDs plus sampling and request options." }, { "title": "Extract generated text", "detail": "Return the engine's string or list structure, not token IDs." } ] } ], "note": "Lanes are alternatives. The HTTP shim is not an engine frontend; legacy SGLang Runtime accepts text only.", "sources": [ "gptqmodel/models/base.py", "gptqmodel/models/loader.py", "gptqmodel/utils/openai_server.py", "gptqmodel/utils/vllm.py", "gptqmodel/utils/sglang.py" ] }, { "page": "Evaluation", "title": "Task scores versus perplexity", "subtitle": "These answer different quality questions and need matched protocols.", "lanes": [ { "label": "Evalution task scoring", "steps": [ { "title": "Fix the protocol", "detail": "Choose task, dataset, prompts, engine, and batch policy." }, { "title": "Build the session", "detail": "Use a model path or supported live model with its tokenizer." }, { "title": "Score each task", "detail": "Run suites; ARC compares continuation loglikelihoods." }, { "title": "Close and report", "detail": "Always close the session; return RunResult on success." } ] }, { "label": "Compact perplexity", "steps": [ { "title": "Tokenize held-out text", "detail": "Use the WikiText-2 test split and recorded token budget." }, { "title": "Slice disjoint windows", "detail": "Use eval_seq_len; skip windows shorter than two tokens." }, { "title": "Sum next-token loss", "detail": "Accumulate FP32 cross entropy and scored token counts." }, { "title": "Compute perplexity", "detail": "Exponentiate mean loss; compare only matched protocols." } ] } ], "note": "Independent measurements. CLI JSON contains aggregated metrics, not RunResult; disjoint windows omit cross-window transitions.", "sources": [ "tests/eval.py", "scripts/eval_model.py", "scripts/benchmark_vptq.py" ] }, { "page": "Troubleshooting-and-FAQ", "title": "Diagnose the failing boundary", "subtitle": "Follow the lane matching the failure before changing data or metadata.", "lanes": [ { "label": "Import or JIT failure", "steps": [ { "title": "Record the environment", "detail": "Capture package, Torch, CUDA or HIP, and device versions." }, { "title": "Identify the extension", "detail": "Inspect the requested operator and its recorded load error." }, { "title": "Fix the first failure", "detail": "Match compiler, headers, ABI, and target architecture." }, { "title": "Retry the affected path", "detail": "Recheck that extension without rebuilding unrelated kernels." } ] }, { "label": "Kernel selection failure", "steps": [ { "title": "Read saved metadata", "detail": "Keep METHOD, FORMAT, bits, groups, and symmetry distinct." }, { "title": "Resolve the device", "detail": "Check logical ordinals, visibility, dtype, and layer shapes." }, { "title": "Check kernel contracts", "detail": "Compare SUPPORTS declarations and validation failures." }, { "title": "Choose a valid backend", "detail": "Use AUTO or a compatible explicit backend; do not relabel." } ] }, { "label": "Quantization OOM", "steps": [ { "title": "Locate the stage", "detail": "Distinguish capture, solver, replay, packing, and offload." }, { "title": "Inspect live memory", "detail": "Account for weights, activations, statistics, and replicas." }, { "title": "Bound relevant work", "detail": "Use method-specific chunks, workspace budgets, or offload." }, { "title": "Retry and validate", "detail": "Record protocol changes and recheck quality and peak memory." } ] }, { "label": "Interrupted quantization", "steps": [ { "title": "Reload the source", "detail": "Start from the original local safetensors model." }, { "title": "Verify resume identity", "detail": "Match source, config, calibration, packing, and execution." }, { "title": "Restore committed state", "detail": "Resume a supported chain from its durable layer boundary." }, { "title": "Finish, then export", "detail": "Save the inference model only after normal completion." } ] } ], "note": "Lanes are independent diagnoses. Workspace budgets are not total-memory caps; an inference export is not a resume checkpoint.", "sources": [ "gptqmodel/extension.py", "gptqmodel/utils/cpp.py", "gptqmodel/utils/importer.py", "gptqmodel/quantization/config.py", "gptqmodel/looper/stage_layer.py", "gptqmodel/looper/gptq_checkpoint.py", "gptqmodel/looper/checkpoint.py", "gptqmodel/looper/checkpoint_store.py" ] }, { "page": "Developer-Guide", "title": "Contribution paths and contracts", "subtitle": "Choose the extension boundary and validate the contracts it changes.", "lanes": [ { "label": "Quantization method", "steps": [ { "title": "Normalize configuration", "detail": "Define method fields, valid formats, and serialization." }, { "title": "Implement the processor", "detail": "Choose capture and replay flags; weight-only needs neither." }, { "title": "Pack and finalize", "detail": "Emit the declared representation and release temporary state." }, { "title": "Validate the round trip", "detail": "Check independent math, saved tensors, reload, and quality." } ] }, { "label": "Inference backend", "steps": [ { "title": "Declare support", "detail": "Specify methods, formats, dtypes, devices, and shapes." }, { "title": "Implement packed compute", "detail": "Consume packed weights; release temporary dense values." }, { "title": "Check independent oracle", "detail": "Test device and dtype paths, including FP16 and BF16 on MLX." }, { "title": "Audit persistent state", "detail": "Test invalidation and ensure no expanded weights survive." } ] }, { "label": "Model architecture", "steps": [ { "title": "Describe the module tree", "detail": "Map real paths, subsets, shared inputs, and modalities." }, { "title": "Register the definition", "detail": "Map the exact model_type and required loader behavior." }, { "title": "Exercise the layer flow", "detail": "Test repeated layers and subsets for supported methods." }, { "title": "Compare to the baseline", "detail": "Verify save/reload and quality with identical prompt IDs." } ] } ], "note": "Lanes are independent contribution concerns; a change may span several. Calibration is method-dependent, not universal.", "sources": [ "gptqmodel/quantization/config.py", "gptqmodel/looper/loop_processor.py", "gptqmodel/looper/gptq_processor.py", "gptqmodel/looper/weight_only_processor.py", "gptqmodel/utils/importer.py", "gptqmodel/nn_modules/qlinear/__init__.py", "gptqmodel/models/auto.py", "gptqmodel/models/definitions/llama.py", "tests/models/model_test.py", "AGENTS.md", ".agents/skills/gptqmodel-torch-oracle-numerics/SKILL.md" ] } ]