[build-system]
requires = ["setuptools>=42", "wheel"]
build-backend = "setuptools.build_meta"

[project]
name = "mteb"
version = "2.26.28"
description = "Massive Text Embedding Benchmark"
readme = "README.md"
authors = [
    { name = "MTEB Contributors", email = "niklas@huggingface.co" },
    { name = "Kenneth Enevoldsen", email = "kenneth.enevoldsen@cas.au.dk" },
    { name = "Nouamane Tazi", email = "nouamane@huggingface.co" },
    { name = "Nils Reimers", email = "info@nils-reimers.de" },
]
maintainers = [
    { name = "Kenneth Enevoldsen", email = "kenneth.enevoldsen@cas.au.dk" },
    { name = "Roman Solomatin", email = "risolomatin@gmail.com" },
    { name = "Isaac Chung", email = "chungisaac1217@gmail.com" },
]
license = "Apache-2.0"
license-files = ["LICENSE"]
keywords = ["deep learning", "text embeddings", "embeddings", "multimodal", "benchmark", "retrieval", "information retrieval"]
classifiers = [
    "Development Status :: 5 - Production/Stable",
    "Environment :: Console",
    "Intended Audience :: Developers",
    "Intended Audience :: Information Technology",
    "Operating System :: OS Independent",
    "Programming Language :: Python",
]
requires-python = ">=3.11,<3.15"
dependencies = [
    "datasets>=2.19.0",
    "numpy>=1.23.0,<3.0.0",
    "requests>=2.26.0",
    "scikit-learn>=1.4.0",
    "scipy>=1.14.0",
    "sentence_transformers>=3.0.0",
    "transformers>=4.40.0,<6.0.0",
    "typing-extensions>=4.5.0",
    "torch>=2.0.0",
    "tqdm>1.0.0",
    "rich>=13.0.0",
    "pytrec-eval-terrier>=0.5.6",
    "pydantic>=2.11.0",
    "polars>=1.0.0",
]


[project.urls]
Homepage = "https://github.com/embeddings-benchmark/mteb"
Documentation = "https://embeddings-benchmark.github.io/mteb/"
Repository = "https://github.com/embeddings-benchmark/mteb"
"Hugging Face Organization" = "https://huggingface.co/mteb"

[project.scripts]
mteb = "mteb.cli:main"

[project.optional-dependencies]
image = [
    "pillow>=10.0.1",
    "torchvision>0.2.1",
    "transformers[vision]",
]
audio = [
    "torchaudio",
    # datsaets==4 requires transformers>=5, because of https://github.com/huggingface/transformers/issues/42103
    "datasets[audio]",
]
codecarbon = ["codecarbon>=2.0.0,<3.0.0"]
leaderboard = [
    "gradio==6.0.1",
    "plotly>=5.24.0,<6.0.0",
    "cachetools>=5.2.0",
    "matplotlib>=3.9.4",
    "pymongo>=4.15.5",
]
api = [
    "fastapi>=0.110",
    "pydantic-settings>=2.0",
    "uvicorn[standard]>=0.27",
    "prometheus-client>=0.20",
    "opentelemetry-sdk>=1.27",
    "opentelemetry-exporter-otlp-proto-http>=1.27",
    "opentelemetry-instrumentation-fastapi>=0.48b0",
]
# Open Graph hero-card generation. Pulled in only by the Dockerfile's
# builder stage — the runtime image strips this back out so it doesn't
# carry Playwright or a Chromium install.
og = [
    "playwright>=1.49.0",
]
video = ["datasets>=4.0.0", "torchcodec"]
github = ["PyGithub>=1.60"]

# model specific optional dependencies:
peft = ["peft>=0.11.0"]
flagembedding = ["FlagEmbedding==1.3.4"]
jina = ["einops>=0.8.0", "sentence-transformers>=3.1.0,<5.4.0", "transformers<5.0.0"]  # ST>=5.4 double-passes cache_dir into custom_st.py
jina-clip = ["peft>=0.15.2", "einops>=0.8.0", "transformers>=4.52.0,<5.0.0", "torchvision>=0.22.1", "timm>=1.0.15,<1.1.0"]
jina-v4 = ["peft>=0.15.2", "transformers>=4.52.0,<5.0.0", "torchvision>=0.22.1"]
flash_attention = ["flash-attn>=2.6.3"]
internvideo2 = ["peft>=0.13.0", "timm>=1.0.9", "einops>=0.8.0", "sentencepiece>=0.2.0", "protobuf>=3.0.0", "transformers<5", "open_clip_torch>=2.24.0", "torchaudio>=2.0.0"]
omniretriever = ["peft>=0.11.0", "transformers>=4.51.0,<4.52.0"]
colvec1_1 = ["transformers>=5.14.1,<6.0.0"]
topk-embed = ["sentence-transformers>=6.0.1", "transformers>=5.9.0,<5.13.0", "flash-linear-attention>=0.5.1", "kernels>=0.14.1,<0.15"]
openai = ["openai>=1.41.0", "tiktoken>=0.8.0"]
model2vec = ["model2vec>=0.3.0"]
pylate = ["pylate>=1.3.1"]
colnanovdr = ["sentence-transformers>=6.0.0", "transformers>=5.0.0"]
pplx-embed-v2 = ["sentence-transformers>=6.0.0", "transformers>=5.4.0"]
msclap = ["msclap>=1.3.4", "soundfile>=0.13.1"]
bm25s = ["bm25s>=0.2.6", "PyStemmer>=2.2.0.3"]
boto3 = ["boto3>=1.34.0"]
av = ["av>=17.0.0"]
gritlm = ["gritlm>=1.0.2"]
xformers = ["xformers>=0.0.29"]
arctic-embed-m-v2 = ["xformers>=0.0.29", "transformers<5.0.0"]
blip2 = ["salesforce-lavis>=1.0.2"]
voyageai = ["voyageai>0.3.0,<2.0.0"]
voyage_v = ["voyageai>0.3.0,<2.0.0", "tenacity>9.0.0"]
cohere = ["cohere==5.14.0"]
vertexai = ["vertexai==1.71.1"]
google_genai = ["google-genai>=1.72.0"]
octen = ["octen>=0.4.0"]
llm2vec = ["llm2vec>=0.2.3,<0.3.0"]
timm = ["timm>=1.0.15,<1.1.0"]
radio = ["timm>=1.0.15,<1.1.0", "einops>=0.8.0", "transformers>=4.57.0,<5.0.0"]
open_clip_torch = ["open_clip_torch==2.31.0"]
nomic = ["einops>=0.8.1"]
ark = ["volcengine-python-sdk[ark]==3.0.2", "tiktoken>=0.8.0"]
colpali_engine = ["colpali_engine>=0.3.12"]
evie = ["colpali_engine>=0.3.18", "transformers>=5.13.1,<6.0.0", "torchvision>=0.22.1"]
dasheng = ["einops>=0.8.0"]
colqwen3 = ["transformers>=4.57.0", "torchvision>=0.22.1"]
visrag-ret = ["transformers>=4.49.0,<4.53", "timm>=1.0.15,<1.1.0", "torchvision>=0.16.0", "sentencepiece>=0.1.99", "accelerate>=0.26.0"]
sauerkrautlm-colpali = ["sauerkrautlm-colpali>=0.1.0; python_full_version < '3.14'"]
xet = ["huggingface_hub>=0.32.0"]
youtu = ["tencentcloud-sdk-python-common>=3.0.1454", "tencentcloud-sdk-python-lkeap>=3.0.1451"]
llama-embed-nemotron = ["transformers==4.51.0"]
llama-nemotron-colembed-vl = ["transformers[torch]==4.49.0", "torchvision>=0.22.0", "flash-attn>=2.6.3", "accelerate"]
nemotron-colembed-vl-v2 = ["transformers[torch]==5.0.0", "torchvision>=0.22.0", "flash-attn>=2.6.3", "accelerate"]
nemotron-3-5-vl = [
    # PyTorch #187684 fixes SDPA mask-offset overflow for large image batches.
    "torch>=2.13.0",
    "sentence-transformers>=6.1.0",
    "transformers[torch]>=5.12.1,<5.17.0",
    "torchvision>=0.22.0",
    "accelerate>=0.26.0",
]
llama-nemotron-embed-vl-1b-v2 = ["transformers[torch]>=4.56.0", "torchvision>=0.22.0", "flash-attn>=2.6.3", "accelerate"]
nemotron-3-embed = ["sentence-transformers>=5.4.1", "transformers[torch]>=5.2.0"]
multimodal-sbert = ["sentence-transformers>=5.4.0", "transformers>=5.3.0", "qwen-vl-utils>=0.0.14"]  # qwen-vl-utils: Qwen3-VL video frame sizes
faiss-cpu = ["faiss-cpu>=1.12.0"]
eager_embed = ["qwen_vl_utils>=0.0.14"]
speechbrain = ["speechbrain>=0.5.12"]
muq = ["muq==0.1.0"]
wav2clip = ["wav2clip==0.1.0", "accelerate>=0.26.0"]
fusion-embedding = ["librosa>=0.10.0", "soundfile>=0.13.1", "torchvision>=0.22.1", "pillow>=10.0.1", "accelerate>=0.26.0"]
torch-vggish-yamnet = ["torch-vggish-yamnet==0.2.1"]
vllm = ["vllm>=0.11.1"]
moca = ["transformers>=4.49.0,<4.53"]
transformers-v4 = ["transformers<5"]
transformers-v5 = ["transformers>=5"]
e5-v = ["transformers<=4.44.2"] # https://github.com/embeddings-benchmark/mteb/issues/1647
# NV-Embed previously needs sentence-transformers==2.7.0, which cannot be expressed here:
# mteb itself requires sentence-transformers>=3.0.0, so the extra would be unsatisfiable.
nvembed = ["transformers==4.42.4"]
nemotron-rerank = ["transformers==4.47.1"]
modernbert = ["transformers>=4.48.0"]
sparse-encoder = ["sentence-transformers>=5.0.0"]
cosmos-embed1 = ["transformers<5", "einops>=0.8.0", "torchvision>=0.16.0"]
siglip = ["sentencepiece>=0.2.0","protobuf>=3.0.0"]
tipsv2 = ["sentencepiece>=0.2.0", "protobuf>=3.0.0"]
ps3 = ["ps3-torch>=0.1.3", "timm==1.0.15", "torchvision>=0.22.1"] # timm 1.0.16+ changed the ViT Block signature that ps3 subclasses
qwen-vl = ["transformers>=4.57.0", "qwen-vl-utils>=0.0.14"]
wemm = [
    "transformers>=5.2.0",
    "sentence-transformers>=5.7.0",
    "qwen-vl-utils>=0.0.14",
    "accelerate>=1.1.0",
]
uembed = ["transformers>=5.4.0,<6.0.0", "qwen-vl-utils>=0.0.14"]
videoprism = ["transformers>=5.13.0"]  # first release with VideoPrism
# omnivinci = ["transformers==4.46.0", "torch==2.8.*", "einops>=0.8.0", "openai-whisper>=20240930", "soundfile>=0.13.1", "opencv-python-headless>=4.0.0", "kaldiio>=2.18.0", "decord>=0.6.0", "librosa>=0.10.0", "beartype>=0.18.0", "accelerate>=1.0.0", "s2wrapper @ git+https://github.com/bfshi/scaling_on_scales", "deepspeed>=0.9.0", "torchvision>=0.15.0", "torchaudio>=2.0.0", "torchcodec==0.7.0", "av>=10.0.0", "numpy>=1.23.0,<2.0.0", "flash-attn>=2.6.3"]
qwen_omni_utils = ["qwen_omni_utils"]
embeddinggemma = ["transformers>=4.56.0"]
# ebind = ["ebind @ git+https://github.com/encord-team/ebind@7909701e9372353ca678b9515f9f61cf87c83c71", "soundfile>=0.13.1", "transformers>=4.57.1,!=5.0.0"]
languagebind = [
    "languagebind[video,audio]==0.1.0",
    "transformers>=4.40.0,<4.46",
    "torchcodec>=0.1.0 ; sys_platform == 'linux'",
    "librosa>=0.10.0",
    "datasets>=3.3.0,<5.0.0",
]
gme = ["transformers<4.52.0"]
imagebind = ["imagebind-unofficial>=0.1.0", "soundfile>=0.13.1", "einops>=0.8.0", "setuptools<81"]  # imagebind imports pkg_resources
speecht5 = ["sentencepiece>=0.2.0"]
seamless-m4t = ["sentencepiece>=0.2.0", "protobuf>=3.0.0"]
audio-flamingo-3 = ["accelerate>=0.26.0"]
bidirlm-omni = ["librosa>=0.10.0"]
# ViCLIP remote code; setuptools<81 for its `from pkg_resources import packaging`
viclip = ["einops>=0.8.0", "timm>=1.0.15,<1.1.0", "ftfy>=6.0", "regex", "fvcore", "setuptools<81", "transformers<5"]

[dependency-groups]
lint = [
    "ruff==0.16.0", # locked so we don't get PRs which fail only due to a lint update
    "pre-commit>=4.1.0",
    "typos>1",
    "bibtexparser>=2.0.0,<3.0.0", # used for tests/test_citation_formatting.py
]
test = [
    "pytest>=8.3.4,<8.4.0",
    "pytest-xdist>=3.6.1,<3.7.0",
    "pytest-coverage>=0.0",
    "pytest-rerunfailures>=15.0,<16.0",
    "iso639>=0.1.4", # used for tests/scripts/test_generate_model_meta.py
    "GitPython>=3.0.0",
    "tabulate>=0.9.0",
    "bibtexparser>=2.0.0,<3.0.0", # used for tests/test_citation_formatting.py
    "refaudit>=0.4.9", # used for tests/test_validate_metadata/check_citations.py
    "av>=17.0.0",
]
docs = [
    "zensical==0.0.68",
    "mkdocstrings-python>=1.18.2",
    "tabulate>=0.9.0",
]
typing = [
    "mypy==2.1.0",
    "types-cachetools>=6.2.0.20250827",
    "types-pysocks>=1.7.1.20250828",
    "types-pyyaml>=6.0.12.20250822",
    "types-requests>=2.32.4.20250913",
    "types-simplejson>=3.20.0.20250822",
    "types-tqdm>=4.67.0.20250809",
    "types-tensorflow>=2.18.0.20250809",
    "types-pygments>=2.19.0.20251121",
    "types-colorama>=0.4.15.20250801",
    "types-gevent>=25.9.0.20251102",
    "types-networkx>=3.6.1.20251220",
    "types-openpyxl>=3.1.5.20250919",
    "types-psutil>=7.1.3.20251211",
    "types-python-dateutil>=2.9.0.20251115",
    "types-pywin32>=311.0.0.20251008",
    "types-regex>=2025.11.3.20251106",
    "types-setuptools>=80.9.0.20251221",
    "types-tabulate>=0.9.0.20241207",
    "types-xlrd>=2.0.0.20251020",
    "pandas-stubs>=2.3.2.250926",
    "scipy-stubs>=1.15.3.0",
    "types-defusedxml>=0.7.0.20250822",
    "pillow>=12.0.0",
]
dev = [
    {include-group = "lint"},
    {include-group = "test"},
    {include-group = "typing"},
    {include-group = "docs"},
]


[tool.coverage.report]
omit = ["tests/*", "mteb/tasks/**/*", "scripts"]

# Regexes for lines to exclude from consideration
exclude_also = [
    # Don't complain about missing debug-only code:
    "def __repr__",
    "if self\\.debug",

    # Don't complain if tests don't hit defensive assertion code:
    "raise AssertionError",
    "raise NotImplementedError",

    # Don't complain if non-runnable code isn't run:
    "if 0:",
    "if __name__ == .__main__.:",

    # Don't complain about abstract methods, they aren't run:
    "@(abc\\.)?abstractmethod",
]

[tool.setuptools.packages.find]
where = ["."]
include = ["mteb", "mteb.*"]
namespaces = false

[tool.setuptools.package-data]
"mteb" = [
    "languages/*.json",  # for languages
    "descriptive_stats/*/*.json",  # for descriptive_stats image
    "descriptive_stats/Image/*/*.json",  # for descriptive_stats image
]
"mteb.api" = ["static/*"]  # favicon served from /favicon.ico
"mteb.abstasks" = ["dataset_card_template.md"]
"mteb.benchmarks" = ["benchmark_card_template.md"]
"mteb.tasks.zeroshot_classification.eng.templates" = ["*.txt"]

[tool.ruff]
target-version = "py311"
exclude = [
    "tests/create_meta/*",  # don't format test md model cards
]


[tool.ruff.lint]
select = [
    "F",  # pyflakes rules,
    "I",  # sorting for imports
    "E",  # formatting for docs
    "W",  # pycodestyle warnings
    "D",  # formatting for docs
    "UP", # upgrade to latest syntax if possible
    "FA", # Future annotations
    "C4", # cleaner comprehensions
    "NPY", # numpy
    "N",       # PEP8 naming convention
    "RUF009",  # function-call-in-dataclass-default-argument
    "RUF008",  # mutable-dataclass-default
    "RUF013",  # implicit-optional
    "RUF012",  # mutable-class-default
    "RUF015",  # unnecessary-iterable-allocation-for-first-element
    "RUF016",  # invalid-index-type
    "RUF017",  # quadratic-list-summation
    "RUF018",  # assignment-in-assert
    "RUF019",  # unnecessary-key-check
    "RUF020",  # never-union
    "RUF021",  # parenthesize-chained-operators
    "RUF022",  # unsorted-dunder-all
    "RUF024",  # mutable-fromkeys-value
    "RUF026",  # default-factory-kwarg
    "RUF033",  # post-init-default
    "RUF034",  # useless-if-else
    "RUF041",  # unnecessary-nested-literal
    "RUF040",  # invalid-assert-message-literal-argument
    "RUF046",  # unnecessary-cast-to-int
    "RUF051",  # if-key-in-dict-del
    "RUF100",  # unused-noqa
    "RUF101",  # redirected-noqa
    "RUF200",  # invalid-pyproject-toml
    "PTH",     # use pathlib
    "TID",     # tidy-imports
    "D",       # pydocstyle
    "PGH",     # pygrep-hooks Use specific rule codes when ignoring type issues
    "LOG",     # logging
    "TC",      # type-checking
    "T20",     # don't allow print
    "PL",      # pylint
    "S",       # flake8-bandit
    "FAST",       # fastapi
    "ICN",     # import conventions, e.g. `import numpy as np`
    "PIE",     # misc. lints, e.g. unnecessary `pass`
    "PD",      # pandas-vet
    "DTZ",     # require timezone-aware datetimes
    "SLOT",    # __slots__ on str/tuple/namedtuple subclasses
    "RSE",     # no unnecessary parens on raised exceptions
    "YTT",     # correct sys.version checks
    "EXE",     # shebang/executable-bit consistency
    "INT",     # gettext f-string misuse
    "ASYNC",   # blocking calls in async functions
    "ISC",     # implicit string concatenation
    "B",       # flake8-bugbear
    "PT",      # flake8-pytest-style
    "A",       # flake8-builtins, don't shadow builtins
    "RET",     # flake8-return
    "SIM",     # flake8-simplify
    "ANN",     # flake8-annotations
]

ignore = [
    "E501",   # line too long
    "E741",     # ambiguous variable name
    "D100",     # Missing docstring in public module
    "D105",     # Missing docstring in magic method
    "D104",     # Missing docstring in public package
    "D107",     # Missing docstring in __init__
    "D415",     # First line should end with a period
    "C408",     # don't use unecc. collection call, e.g. dict over {}
    "S404",    # subprocess import insecure
    "S311",     # standard pseudo-random generators are fine for non-crypto use
    "S104",     # allow binding to all interfaces in tests
    "S607",     # start a process with partial path
    "S603",   # allow subprocess calls with shell=True
    "PLR2004",   # magic value comparison
    "PLC0415",   # import outside of top-level
    "PLR0912",  # too many branches
    "PLR0915",  # too many statements
    "PLW0717",  # too-many-statements-in-try-clause
    "ISC001",   # single-line implicit concat; conflicts with `ruff format`
    "PT018",    # composite assertions; splitting `assert a and b` is not always clearer
    "RET504",   # unnecessary-assign; named intermediate results often aid readability
    "SIM108",   # ternary over if/else is a style preference, and displaces inline comments
]
future-annotations = true  # For TC rules
typing-modules = ["typing_extensions", "mteb.types", "collections.abc"]
preview = true


[tool.ruff.lint.per-file-ignores]
# every script except scripts/tools, which is used in CI and is linted
"scripts/[!t]*" = ["ALL"]
"scripts/t[!o]*" = ["ALL"]
"scripts/tools/*" = [
    "T201",  # scripts report on stdout
    "D1",  # docstrings
    "PLR0913",
    "PLR0914",
    "PLR0917",
    "PLR1702",
    "PLW0603",
    "ASYNC240",
]
# a script rather than a test: it reports what it checked on stdout
"tests/test_validate_metadata/check_citations.py" = ["T201"]
"docs/*" = [
    "D",
    "DOC",
    "S101"
]
"tests/*" = [
    "RUF012",
    "D",
    "DOC",
    "TC",
    "S101",  # allow asserts in tests
    "S602",  # allow shell=True in tests
    "S603",  # allow subprocess calls in tests
    "PLC2701",  # Private name import
    "PLR6301",  # Method could be a function - pytest test methods are instance methods
    # tests are not type-checked (`make mypy` covers `mteb` only), so annotations
    # here would go unverified.
    "ANN001",
    "ANN201",
    "ANN202",
]
"mteb/mocks/*" = [
    "RUF012",
    "D",
]
"mteb/tasks/*__init__.py" = ["F403"]  # undefined import `from .lang import *`
"mteb/tasks/*" = [
    "D",
    "DOC",
]
"mteb/models/model_implementations/*" = [
    "D",
    "DOC",
]
"mteb/benchmarks/benchmark.py" = [
    # pydantic would try to validate fields in Benchmark class
    "TC",
]
"mteb/leaderboard/app.py" = [
    # gradio would try to validate typehints
    "TC002",
    "TC001",
    # nested `with` blocks mirror the gradio layout hierarchy; collapsing them hides it
    "SIM117",
]
"mteb/types/statistics.py" = [
    # pydantic / FastAPI resolve TypedDict annotations at runtime when building
    # the OpenAPI schema for /v1/tasks/{name}/descriptive-stats; ``NotRequired``
    # and ``HFSubset`` must be importable outside TYPE_CHECKING.
    "TC001",
    "TC002",
]


[tool.ruff.lint.pep8-naming]
ignore-names = [
    "F",  # torch.nn.functional
    "X",
    "X_train",
    "X_test",
]

[tool.ruff.lint.pydocstyle]
convention = "google"

[tool.ruff.lint.flake8-annotations]
mypy-init-return = true
suppress-none-returning = true
# ANN003 requires `**kwargs` to be annotated; ANN401 rejects `Any`. `**kwargs` here is a
# pass-through for arbitrary model/task options, so `Any` is the only correct annotation.
allow-star-arg-any = true

[tool.ruff.lint.flake8-type-checking]
strict = true
runtime-evaluated-base-classes = [
    "pydantic.BaseModel",
    "mteb.abstasks.task_metadata.TaskMetadata",  # https://github.com/astral-sh/ruff/issues/7866
]

[tool.ruff.lint.pylint]
max-args = 9

[tool.semantic_release]
branch = "main"
version_toml = ["pyproject.toml:project.version"]
# builds `mteb` and `mteb-core` (the same code without torch) into dist/, which the release workflow publishes
build_command = "python -m pip install build packaging tomli && python -m build && python scripts/build_core_package.py"
tag_format = "{version}"

[tool.semantic_release.commit_parser_options]
minor_tags = ["minor", "feat", "benchmark"]
patch_tags = [
    "patch",
    "fix",
    "perf",
    "model",
    "dataset",
]

[tool.pytest.ini_options]
addopts = """
    -ra
    --reruns 3
    --only-rerun requests.exceptions.ReadTimeout
    --only-rerun huggingface_hub.errors.HfHubHTTPError
    --only-rerun huggingface_hub.errors.LocalEntryNotFoundError
    --only-rerun FileNotFoundError
    --durations=5
    --reruns-delay 10
"""
# -ra -> # show extra test summary info for skipped, failed, etc.
# --reruns 3 -> # Retry failed tests 3 times
# requests.exceptions.ReadTimeout -> # HF Read timed out -> https://github.com/embeddings-benchmark/mteb/actions/runs/13275350693/job/37093688544
# huggingface_hub.errors.HfHubHTTPError -> # HF is unavailable, e.g. seen here: https://github.com/embeddings-benchmark/mteb/actions/runs/13275350693/job/37093688544
# huggingface_hub.errors.LocalEntryNotFoundError -> # Gateway Time-out from HF, e.g. seen here: https://github.com/embeddings-benchmark/mteb/actions/runs/13275350693/job/37093688544
# FileNotFoundError -> HF Cache is broken:  https://github.com/embeddings-benchmark/mteb/actions/runs/13302915091/job/37147507251?pr=2029
# --durations=5 -> Show the 5 slowest tests
# --reruns-delay 10 -> Delay between reruns in seconds to avoid running into the same issue again

markers = [
    "core: marks tests that must pass on a minimal `mteb-core` installation, without torch (`make test-core`)",
    "test_datasets: marks tests that tests dataset availability",
    "test_reference_models: marks tests that verify reference model result coverage (requires downloading results repo)",
]

[tool.uv]
required-version = ">=0.10.0"
default-groups = ["test", "docs", "typing", "lint"]
override-dependencies = [
    "salesforce-lavis", # salesforce-lavis is not valid with sentence transformers >=3.0.0
]
conflicts = [
    [
        { extra = "timm" },
        { extra = "radio" }, # pins timm, same conflict as the timm extra
        { extra = "blip2" },
        { extra = "llm2vec" },
        { extra = "model2vec" },
        { extra = "pylate" }, # conflicting versions of transformers
        { extra = "llama-embed-nemotron" }, # conflicting versions of transformers
        { extra = "llama-nemotron-colembed-vl" }, # conflicting versions of transformers
        { extra = "nemotron-colembed-vl-v2" }, # conflicting versions of transformers
        { extra = "llama-nemotron-embed-vl-1b-v2" }, # conflicting versions of transformers
        { extra = "nemotron-3-embed" }, # conflicting versions of transformers
        { extra = "nemotron-3-5-vl" }, # conflicting versions of transformers
        { extra = "multimodal-sbert" }, # conflicting version of sentence-transformers
        { extra = "colpali_engine" },
        { extra = "evie" }, # transformers>=5.13.1 conflicts with visrag-ret etc.
        { extra = "colqwen3" },
        { extra = "colvec1-1" },
        { extra = "topk-embed" }, # needs sentence-transformers>=6.0.1 and transformers>=5.9.0,<5.13.0 (remote code reads Qwen3_5DecoderLayer.layer_type); kernels>=0.15 breaks transformers<5.13 imports
        { extra = "jina-v4" },
        { extra = "sauerkrautlm-colpali" },
        { extra = "vllm" },
        { extra = "jina-clip" },
        { extra = "msclap" },
        { extra = "muq" },
        { extra = "moca" }, # needs transformers with _update_causal_mask
        { extra = "omniretriever" },
        { extra = "transformers-v4" },
        { extra = "internvideo2" }, # conflicting versions of transformers
        { extra = "transformers-v5" },
        { extra = "cosmos-embed1" }, # remote code needs transformers<5
        { extra = "viclip" }, # remote code needs transformers<5
        { extra = "qwen-vl" },
        { extra = "wemm" },
        { extra = "uembed" }, # needs transformers>=5.4.0
        { extra = "videoprism" }, # needs transformers>=5.13.0
        { extra = "ps3" }, # ps3-torch pins transformers<=4.49.0
        # { extra = "omnivinci" },
        { extra = "embeddinggemma" },
        # { extra = "ebind" },
        { extra = "visrag-ret" }, # conflicting version of timm with blip2
        { extra = "languagebind" }, # conflicting versions of transformers
        { extra = "e5-v" }, # conflicting versions of transformers
        { extra = "nvembed" }, # conflicting versions of transformers
        { extra = "flagembedding" }, # FlagEmbedding pins transformers>=4.44.2
        { extra = "nemotron-rerank" }, # conflicting versions of transformers
        { extra = "modernbert" }, # conflicting versions of transformers
        { extra = "sparse-encoder" }, # conflicting version of sentence-transformers
        { extra = "gme" }, # requires transformers<4.52.0
        { extra = "jina" }, # remote code needs transformers<5
        { extra = "arctic-embed-m-v2" }, # remote code (GTE) needs transformers<5
        { extra = "colnanovdr" },
        { extra = "pplx-embed-v2" }, # needs transformers>=5.4.0
    ],
]

[tool.uv.extra-build-dependencies]
flash-attn = [{ requirement = "torch", match-runtime = true }]

[tool.mypy]
plugins = ['pydantic.mypy']
strict = true

[[tool.mypy.overrides]]
# these modules not typed and don't have stubs
module = [
    "datasets",
    "datasets.exceptions",
    "sklearn",
    "sklearn.*",
    "github",
    "github.*",
    "faiss",
    "vllm",
    "torchaudio",
    "torchvision.*",
]
ignore_missing_imports = true

[[tool.mypy.overrides]]
# don't typecheck these modules (too many issues)
module = [
    "mteb.models.model_implementations.*",
    "mteb.tasks.*",
    "mteb.leaderboard.*",
    "mteb.mocks.*",
]
ignore_errors = true

[[tool.mypy.overrides]]
# mypy can't resolve dataset dict
module = ["mteb.abstasks.*"]
disable_error_code = ["index"]

[tool.typos.default]
binary = false
locale = "en"
extend-ignore-words-re = [
    "Classfication", # misspelled task name
    "nd", # license
    # authors
    "Stap",
    "Thant",
    "Asai",
    "Tung",
    "Gool",
    "Winn",
    "STIL",
    "Sherif",
    "Mor",
    "Ond",
    "Mehta",
    "Parth",
    # languages/bibtext artifacts
    "SHW",
    "SHS",  # SecondHandSongs / SHS100K dataset
    "shs",
    "ba",
    "creen",
    "Accademia",
    "dementieva",
    "Expresso",
    "expresso",
    "Iz",
    "Allo",
    "Soler",
    # language keys:
    "tha",
    "yor",
    "als",
    "bre",
    "thi",
    # ISO 15924 script codes
    "Beng",  # Bengali script
    # dataset names
    "Matix",  # Docmatix (HF dataset)

]
extend-ignore-re = [
    "author.*",  # ignore authors in citations
    "editor.*",  # ignore authors in citations
    "FIne-Grained Music retrievAl",  # stylised paper title (FIGMA)
    "@\\w+\\{[^,\n]+,",  # bibtex entry keys are arbitrary identifiers
    '"[a-z]{3}": \["[a-z]{3}-[A-Z][a-z]{3}"\]',  # ISO 639-3 language-map entries are codes, not prose
]

[tool.typos.files]
extend-exclude = [
    "*.json",
    "*.jsonl",
    "*.cff",
    "*.tex",  # latex files with language codes
    "*.bib",  # bibliography files with author names and journal abbreviations
    "*.txt",  # zero shot labels
    "docs/mmteb",
    "scripts/",
    "mteb/models/model_implementations/gme_v_models.py",  # video_grid_thw `thw`
    "mteb/models/model_implementations/vista_models.py",  # self.normlized: in visual bge
    "mteb/models/model_implementations/salesforce_models.py", # multiligual in paper title
    "tests/mock_tasks.py",  #  "denne her matche ikke den ovenstående",
    "mteb/models/model_implementations/kalm_models.py",  # prompt:  classify ist topic",
    "mteb/tasks/reranking/eng/built_bench_reranking.py",  # prompt:   descriptions from buit asset
    "mteb/tasks/retrieval/eng/built_bench_retrieval.py",
    "mteb/tasks/retrieval/eng/llava_it2t_retrieval.py",
    # dutch prompts
    "mteb/tasks/classification/nld/",
    "mteb/tasks/clustering/nld/",
    "mteb/tasks/multilabel_classification/nld/",
    "mteb/tasks/pair_classification/nld/",
    "mteb/tasks/sts/nld/",
    "mteb/tasks/retrieval/nld/",
    "mteb/descriptive_stats/",
    # Beltagy, Iz
    "mteb/tasks/retrieval/eng/lembqm_sum_retrieval.py",
    "mteb/tasks/classification/jpn/wrime_classification.py",
    "mteb/tasks/reranking/eng/sci_docs_reranking.py",
]

[tool.typos.type.py]
extend-ignore-re = [
    # ignore lang identifiers
    "...-....",
    "..._....",
]

[tool.typos.type.py.extend-identifiers]
BulgarianStoreReviewSentimentClassfication = "BulgarianStoreReviewSentimentClassfication"
SciRepEvalFoSClassification = "SciRepEvalFoSClassification"
FoS = "FoS"
LaBSE = "LaBSE"

[tool.typos.type.py.extend-words]
wdth = "wdth"
Allo = "Allo"
som = "som"
yor = "yor"
Beng = "Beng"
als = "als"
bse = "bse"
tha = "tha"
tham = "tham"
thi = "thi"
bre = "bre"
mak = "mak"
Maka = "Maka"
Mape = "Mape"
Mian = "Mian"
Ono = "Ono"
Yau = "Yau"
Altas = "Altas"
matche = "matche"
expresso = "expresso"
Expresso = "Expresso"
tema = "tema" # Spanish word used in task prompts
NIFE = "NIFE"
thw = "thw"
ist = "ist" # spelling error in prompt for harrier-oss-v1
shw = "SHW"
SME = "SME"
SHS = "SHS"  # SecondHandSongs / SHS100K dataset
shs = "shs"
nd = "nd"