[tool.ruff]
line-length = 240
extend-exclude = ["*.ipynb", "*.md"]

[tool.ruff.lint]
# Pycodestyle, Pyflakes, import sorting, builtin shadowing, and async correctness.
select = ["E4", "E7", "E9", "F", "I", "A", "ASYNC"]

[tool.ruff.lint.isort]
known-first-party = ["lmms_eval"]

[build-system]
requires = ["setuptools>=42", "wheel"]
build-backend = "setuptools.build_meta"

[project]
name = "lmms_eval"
version = "0.7.3"
authors = [
    { name = "LMMMs-Lab Evaluation Team", email = "lmms-lab@outlook.com" },
]
description = "A framework for evaluating large multi-modality language models"
readme = "README.md"
license = "MIT"
license-files = ["LICENSE"]
classifiers = [
    "Programming Language :: Python :: 3",
    "Operating System :: OS Independent",
]
requires-python = ">=3.10"
dependencies = [
    "chess>=1.11.2,<2", # MET-Bench board-state scoring
    "accelerate>=0.29.1",
    "ruff>=0.16.4",
    "datasets>=2.19.0",
    "evaluate>=0.4.0",
    "httpx>=0.23.3",
    "jsonlines",
    "aiohttp",
    "numexpr",
    "numpy>=1.26.4",
    "peft>=0.2.0",
    "pybind11>=2.6.2",
    "pytablewriter",
    "sacrebleu>=1.5.0",
    "scikit-learn>=0.24.1",
    "torch>=2.1.0", # to enable sdpa mode for running 34B model on one 80GB GPU
    "torchvision>=0.16.0",
    "timm",
    "einops",
    "ftfy",
    "openai",
    "opencv-python-headless",
    "av>=17.1.0,<18.0.0", # PyAV 18 requires Python 3.11+
    "hf_transfer",
    "nltk",
    "sentencepiece",
    "yt-dlp",
    "pycocoevalcap",
    "tqdm-multiprocess",
    "transformers>=4.39.2",
    "transformers-stream-generator",
    "zstandard",
    "pillow",
    "portalocker>=2.8",
    "pyyaml",
    "sympy",
    "latex2sympy2-extended[antlr4_13_2]>=1.11.0,<1.12.0",
    "mpmath",
    "Jinja2",
    "openpyxl",
    "loguru",
    "hf_transfer",
    "tenacity>=8.3.0",
    "wandb==0.25.0",
    "tiktoken",
    "pre-commit>=4.6.2,<5",
    "pydantic",
    "packaging",
    "zss",
    "protobuf",
    "sentence_transformers",
    "python-dotenv",
    "qwen-vl-utils>=0.0.14",
    "math-verify>=0.9.0,<0.10.0",
    "word2number>=1.1,<2.0",
    "jieba>=0.42.1",
    "distance>=0.1.3",
    "editdistance>=0.8.1",
    "Levenshtein>=0.27.4",
    "apted>=1.0.3",
]

[project.optional-dependencies]
core = [
    "lmms_eval[server]",
    "lmms_eval[audio]",
    "lmms_eval[metrics]",
]
server = [
    "fastapi",
    "uvicorn",
]
mcp = [
    "mcp[cli]>=1.20.0",
]
audio = [
    "more-itertools",
    "editdistance",
    "zhconv",
    "librosa",
    "soundfile"
]
metrics = [
    "pywsd",
    "spacy",
    "anls",
    "rouge",
    "capture_metric",
    "Levenshtein",
]
gemini = [
    "google-genai>=1.0.0",
]
litellm = [
    "litellm",
]
swanlab = [
    "swanlab",
]
reka = [
    "httpx>=0.23.3",
    "reka-api",
]
qwen = [
    "qwen_vl_utils",
]
mmsearch = [
    "playwright",
    "requests",
    "matplotlib",
    "duckduckgo_search",
    "langchain",
    "langchain-community",
    "beautifulsoup4",
    "FlagEmbedding",
    "rouge",
]
prismmbench = [
    "pdf2image>=1.17.0",
    "pdfplumber>=0.11.9"
]
tui = [
    "fastapi>=0.100.0",
    "uvicorn>=0.20.0",
]
video = [
    "torchcodec>=0.3.0",
]
video-legacy = [
    "decord; platform_system != 'Darwin'",
    "eva-decord; platform_system == 'Darwin' and python_version < '3.12'",
]
all = [
    "lmms_eval[core]",
    "lmms_eval[mcp]",
    "lmms_eval[gemini]",
    "lmms_eval[reka]",
    "lmms_eval[qwen]",
    "lmms_eval[mmsearch]",
    "lmms_eval[tui]",
    "lmms_eval[video]",
]

[tool.setuptools.package-data]
"lmms_eval" = ["**/*.yaml", "**/*.yml", "**/*.json", "**/*.txt", "tasks/**/*"]
"lmms_eval.tui.web.dist" = ["**/*"]
"lmms_eval.tui.web.dist.assets" = ["**/*"]

[tool.setuptools.packages.find]
include = ["lmms_eval*"]
exclude = [
    "assets*",
    "benchmark*",
    "docs",
    "dist*",
    "playground*",
    "scripts*",
    "tests*",
    "checkpoints*",
    "project_checkpoints*",
    "debug_checkpoints*",
    "mlx_configs*",
    "wandb*",
    "notebooks*",
    "logs*",
]

[tool.wheel]
exclude = [
    "assets*",
    "benchmark*",
    "docs",
    "dist*",
    "playground*",
    "scripts*",
    "tests*",
    "checkpoints*",
    "project_checkpoints*",
    "debug_checkpoints*",
    "mlx_configs*",
    "wandb*",
    "notebooks*",
    "logs*",
]

[project.scripts]
lmms-eval = "lmms_eval.cli.dispatch:main"
lmms-eval-mcp = "lmms_eval.mcp:main"
lmms-eval-ui = "lmms_eval.tui.cli:main"

[project.urls]
Homepage = "https://www.lmms-lab.com/"
Repository = "https://github.com/EvolvingLMMs-Lab/lmms-eval"

[dependency-groups]
dev = [
    "pytest>=8.0",
    "textual>=7.2.0",
    "textual-dev>=1.8.0",
]

[tool.pytest.ini_options]
testpaths = ["test"]
python_files = "test_*.py"
python_classes = "Test*"
python_functions = "test_*"
markers = [
    "gpu: requires GPU hardware (deselect with '-m not gpu')",
    "api: requires external API keys (deselect with '-m not api')",
    "slow: slow integration tests (deselect with '-m not slow')",
]
filterwarnings = [
    "ignore::DeprecationWarning",
]