[build-system] requires = ["setuptools >= 77.0.3"] build-backend = "setuptools.build_meta" [project] name = "trl" description = "Train transformer language models with reinforcement learning." authors = [ { name = "Leandro von Werra", email = "leandro.vonwerra@gmail.com" } ] readme = { file = "README.md", content-type = "text/markdown" } license = "Apache-2.0" license-files = ["LICENSE"] keywords = [ "transformers", "huggingface", "language modeling", "post-training", "rlhf", "sft", "dpo", "grpo" ] classifiers = [ "Development Status :: 2 - Pre-Alpha", "Intended Audience :: Developers", "Intended Audience :: Science/Research", "Natural Language :: English", "Operating System :: OS Independent", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.14" ] requires-python = ">=3.10" dependencies = [ "accelerate>=1.4.0", "datasets>=4.7.0", # Support Json type and on_mixed_types="use_json" "jinja2", "packaging>20.0", "transformers>=4.56.2", ] dynamic = ["version"] [project.urls] Homepage = "https://github.com/huggingface/trl" [project.scripts] trl = "trl.cli:main" [project.optional-dependencies] bco = [ "scikit-learn", "joblib" ] deepspeed = [ "deepspeed>=0.14.4", "transformers!=5.1.0", # see transformers#43780 ] kernels = [ # transformers renamed the "hub-kernels" extra to "kernels" in 5.1.0 "transformers[kernels]", # transformers >= 5.1.0 "transformers[hub-kernels]", # transformers < 5.1.0 ] liger = [ "liger-kernel>=0.8.2" ] peft = [ "peft>=0.8.0" ] quality = [ "pre-commit", "hf-doc-builder" ] quantization = [ "bitsandbytes" ] scikit = [ "scikit-learn" ] test = [ "pytest-cov", "pytest-datadir>=1.7.0", # lazy datadirs "pytest-reportlog", # test report consumed by scripts/log_reports.py "pytest-rerunfailures==15.1", "pytest-xdist", "pytest", "tabulate" # test report rendering in scripts/log_reports.py ] vllm = [ "vllm>=0.19.1,<=0.27.1", "aiohttp>=3.13.3", "requests", ] vlm = [ "Pillow", "torchvision", "num2words==0.5.14" ] math_verify = [ "math-verify>=0.5.2", ] openreward = [ "openreward>=0.1.109; python_version >= '3.11'", # openreward requires Python 3.11+ ] harbor = [ "harbor>=0.13.0; python_version >= '3.12'", # harbor requires Python 3.12+ (pulls its sandbox backends) ] dev = [ # bco "scikit-learn", "joblib", # deepspeed "deepspeed>=0.14.4", # kernels: transformers renamed the "hub-kernels" extra to "kernels" in 5.1.0 "transformers[kernels]", # transformers >= 5.1.0 "transformers[hub-kernels]", # transformers < 5.1.0 # liger "liger-kernel>=0.8.2", # openreward (requires Python 3.11+) "openreward>=0.1.109; python_version >= '3.11'", # peft "peft>=0.8.0", # quality "pre-commit", "hf-doc-builder", # quantization "bitsandbytes", # scikit: included in bco # test "pytest-cov", "pytest-datadir>=1.7.0", # lazy datadirs "pytest-reportlog", # test report consumed by scripts/log_reports.py "pytest-rerunfailures==15.1", "pytest-xdist", "pytest", "tabulate", # test report rendering in scripts/log_reports.py # vllm: not included in dev by default due to CUDA error; see GH-4228 # vlm "Pillow", "torchvision", "num2words==0.5.14", # for response parsing (required for training with tools) "jmespath", ] [tool.setuptools] package-dir = {"trl" = "trl"} [tool.setuptools.dynamic] version = { file = "VERSION" } [tool.coverage.run] branch = true [tool.ruff] target-version = "py310" line-length = 119 src = ["trl"] [tool.ruff.lint] ignore = [ "B028", # warning without explicit stacklevel "C408", # dict() calls (stylistic) "C901", # function complexity "E501", ] # `select`, not `extend-select`: the rule set is explicit, so a ruff release that expands its default # selection (0.16 raised the defaults from 59 to 413 rules) does not silently enable new rules. select = ["E", "F", "I", "W", "UP", "B", "T", "C"] [tool.ruff.lint.per-file-ignores] # Allow prints in auxiliary scripts "examples/**.py" = ["T201"] "scripts/**.py" = ["T201"] "trl/cli/**.py" = ["T201"] # Ignore import violations in all `__init__.py` files. "__init__.py" = ["F401"] [tool.ruff.lint.isort] lines-after-imports = 2 known-first-party = ["trl"] [tool.pytest.ini_options] markers = [ "slow: marks tests as slow (deselect with '-m \"not slow\"')", "low_priority: marks tests as low priority (deselect with '-m \"not low_priority\"')", "invariant: training-invariant checks (run with '-m invariant')", ] norecursedirs = [ "tests/experimental", "tests/invariant", ] filterwarnings = [ # SWIG deprecations from SWIG-generated C/C++ extensions: sentencepiece # Upstream issue: https://github.com/google/sentencepiece/issues/1150 # Remove once: no supported sentencepiece version emits it "ignore:builtin type SwigPyPacked has no __module__ attribute:DeprecationWarning", "ignore:builtin type SwigPyObject has no __module__ attribute:DeprecationWarning", "ignore:builtin type swigvarlink has no __module__ attribute:DeprecationWarning", # PyTorch JIT deprecations (upstream, not actionable in TRL) # Upstream issue: https://github.com/deepspeedai/DeepSpeed/issues/7835 # Upstream PR: https://github.com/deepspeedai/DeepSpeed/pull/7840 # Upstream fix released in deepspeed v0.18.6: https://github.com/deepspeedai/DeepSpeed/releases/tag/v0.18.6 # Remove once: deepspeed >= 0.18.6 is required "ignore:`torch.jit.script_method` is deprecated:DeprecationWarning", "ignore:`torch.jit.script` is deprecated:DeprecationWarning", # On Python 3.14+ the same deprecation is reworded to "is not supported in Python 3.14+ and may break" "ignore:`torch.jit.script_method` is not supported:DeprecationWarning", # PyTorch DataLoader pin_memory device argument deprecations # Triggered internally by torch.utils.data, not by our code # Upstream issue: https://github.com/pytorch/pytorch/issues/174546 # Remove once: no supported torch version emits it "ignore:The argument 'device' of Tensor.pin_memory:DeprecationWarning", "ignore:The argument 'device' of Tensor.is_pinned:DeprecationWarning", # bitsandbytes calls the deprecated torch._check_is_size in its CUDA quant ops (upstream, not actionable in TRL) # Triggered indirectly by loading bitsandbytes-quantized models in the PEFT + quantization tests # Tracking issue: https://github.com/huggingface/trl/issues/6447 # Upstream fix released in bitsandbytes 0.50.0: https://github.com/bitsandbytes-foundation/bitsandbytes/pull/1940 # Remove once: bitsandbytes >= 0.50.0 is required "ignore:_check_is_size will be removed in a future PyTorch release:FutureWarning", # triton builds an ast.AnnAssign node without the required `simple` field during kernel compilation (upstream, # not actionable in TRL); a DeprecationWarning on Python 3.14 that becomes an error on 3.15 # Tracking issue: https://github.com/huggingface/trl/issues/6466 # Upstream issue: https://github.com/triton-lang/triton/issues/10981 # Upstream fix (merged, unreleased as of triton 3.7.1): https://github.com/triton-lang/triton/pull/10986 # Remove once: no supported torch resolves a triton without the fix "ignore:AnnAssign.__init__ missing 1 required positional argument:DeprecationWarning", # kernels-community/mamba-ssm (a Hub-hosted build of state-spaces/mamba) calls the deprecated # torch.get_autocast_gpu_dtype() in its Triton ssd_combined.py kernel (upstream, not actionable in TRL) # Triggered by the NemotronH (Nemotron 3) Mamba2 mixer fast path during training under CUDA autocast # Tracking issue: https://github.com/huggingface/trl/issues/6555 # Remove once: the resolved Hub kernel build no longer calls it "ignore:torch.get_autocast_gpu_dtype\\(\\) is deprecated:DeprecationWarning", # bitsandbytes warns, on every forward of a 4-bit layer whose inner dimension is not a multiple of the quantization # blocksize, that it falls back to the slower dequantize + linear path (upstream, not actionable in TRL) # Triggered by the tiny models in the PEFT + quantization tests: their hidden_size (8) and intermediate_size (32) # are not multiples of the 64 blocksize, and transformers exposes no option to change it # Tracking issue: https://github.com/huggingface/trl/issues/6580 # Upstream issue: https://github.com/bitsandbytes-foundation/bitsandbytes/issues/2027 # Remove once: no supported bitsandbytes version emits it "ignore:inner dimension \\(\\d+\\) is not aligned for fast kernel:UserWarning", # transformers passes the `num_patches` torch tensor to np.cumsum in the InternVL processor, so numpy falls back to # the legacy Tensor.__array_wrap__ path, whose signature predates NumPy 2.0 (upstream, not actionable in TRL) # Triggered by the InternVL cases of the GRPO, RLOO and Distillation VLM tests # Tracking issue: https://github.com/huggingface/trl/issues/6848 # Upstream issue: https://github.com/huggingface/transformers/issues/48177 # Upstream issue (torch, for the outdated signature): https://github.com/pytorch/pytorch/issues/180657 # Remove once: no supported transformers version passes a tensor to np.cumsum there "ignore:__array_wrap__ must accept context and return_scalar arguments:DeprecationWarning", # torch inductor advises enabling TF32 on every compile when the float32 matmul precision is left at its default # (upstream, not actionable in TRL: the precision is a process-wide numerics setting only the user should change) # Triggered by the DPO and KTO Liger tests, the only ones that compile, since the GPU CI moved from T4 to L40S: # the advisory is guarded on compute capability >= (8, 0), unreachable on T4 # Tracking issue: https://github.com/huggingface/trl/issues/6849 # Upstream issue: https://github.com/pytorch/pytorch/issues/175484 # Remove once: no supported torch version emits it "ignore:TensorFloat32 tensor cores for float32 matrix multiplication available but not enabled:UserWarning", # torch's DataParallel warns that it is gathering the per-replica scalar losses into a vector, since scalars have # to be unsqueezed first (upstream, not actionable in TRL: the gather happens entirely inside nn.DataParallel) # Triggered by the SFT slow tests since the multi-GPU CI job moved to a runner with two visible devices: # transformers' Trainer wraps the model in nn.DataParallel whenever n_gpu > 1 and no distributed launcher is used # Tracking issue: https://github.com/huggingface/trl/issues/7018 # Remove once: the multi-GPU CI no longer exercises nn.DataParallel (https://github.com/huggingface/trl/issues/6836) "ignore:Was asked to gather along dimension 0, but all input tensors were scalars:UserWarning", ]