[project] name = "py-data-juicer" dynamic = ["version"] description = "Data Processing for and with Foundation Models." authors = [ {name = "SysML Team of Alibaba Tongyi Lab"} ] readme = "README.md" license = {text = "Apache-2.0"} requires-python = ">=3.10,<4" urls = {repository = "https://github.com/datajuicer/data-juicer"} classifiers = [ "License :: OSI Approved :: Apache Software License", "Programming Language :: Python :: 3", "Operating System :: OS Independent" ] # Core dependencies dependencies = [ "datasets>=4.7.0", # core data loading "fsspec>=2026.1.0", # file system operations; >=2026.1.0 for ArrowFile.size compat with pyarrow>=17 "pandas", # data manipulation "numpy>=1.26.4", # numerical operations "loguru", # logging "tqdm", # progress bars "jsonargparse[signatures]", # configuration "jsonlines", # JSONL handling "zstandard", # compression "lz4", # compression "multiprocess==0.70.16", # parallel processing "dill==0.3.8", # serialization "psutil", # system monitoring "pydantic>=2.0", # data validation "uv", "httpx", "emoji==2.2.0", # emoji handling "tabulate", "beautifulsoup4", "requests", "wget", "streamlit", "Pillow", "mwparserfromhell", "regex", # regular expressions "tomli", "tomli-w", "gitpython", "dep-logic", ] [project.optional-dependencies] # Generic ML & DL generic = [ "torch==2.8.0", # PyTorch "transformers==4.57.1", # Hugging Face Transformers "einops", # Tensor operations "accelerate", # Model acceleration "vllm==0.11.0", # LLM serving "uvloop==0.21.0", # avoid async error before it's fixed in uvloop "onnxruntime", # ONNX runtime "cudf-cu12==25.10.0; sys_platform == 'linux'", # dataframe on GPU (Linux only) ] # Computer Vision vision = [ "av==13.1.0", # video/audio handling "opencv-contrib-python", "imagededup", # Image deduplication "diffusers>=0.33.0", # Diffusion models "simple-aesthetics-predictor", # Image aesthetics "scenedetect", # Scene detection "ultralytics", # YOLO models "rembg", # Background removal "decord; (sys_platform == 'linux' and platform_machine == 'x86_64') or (sys_platform == 'win32' and platform_machine == 'AMD64')", # video/audio handling (no aarch64/macOS py3.10+ wheel) "qwen-vl-utils==0.0.14", # for Qwen-VL "ptlflow==0.4.1", # optical flow models collection "timm==1.0.22", # avoid importing issue in the latest v1.0.23 # --- HumanVBench (CVPR 2026) face/video ops --- # These ops additionally need ffmpeg/ffprobe binaries on PATH (VideoLLaMA3 # video loading) and a flash-attn build matching the local torch/CUDA; the # matching prebuilt flash-attn wheel should be installed manually since a # plain `pip install flash-attn` triggers a slow source build. "deepface==0.0.100", # face age/gender/race demographic op "tf-keras==2.21.0", # required by deepface on TensorFlow>=2.16 "imageio>=2.37", # required by VideoLLaMA3 remote code "imageio-ffmpeg>=0.6", # ffmpeg backend for imageio "scipy", # signal/interpolation/wavfile for face-track ASD ops ] # Natural Language Processing nlp = [ "nltk==3.10.3", # NLP toolkit "easyocr==1.7.1", # OCR "fasttext-predict", # FastText (no numpy dependency, numpy 2.x safe) "kenlm", # Language modeling "sentencepiece", # Tokenization "ftfy", # Text fixing "simhash-pybind", # Text similarity "selectolax", # HTML parsing "nlpaug", # Text augmentation "nlpcda", # Chinese text augmentation "tiktoken", # Token counting "opencc==1.1.9", # Chinese conversion "spacy-pkuseg", # Chinese segmentation "rouge", # Text evaluation "pdfplumber", # For PDF input and processing "spacy==3.8.7", "python-docx", # For docx input and processing ] # Audio Processing audio = [ "librosa>=0.10", "resampy", "samplerate==0.1.0", "torchaudio", # Audio processing with PyTorch "torchcodec==0.7; (sys_platform == 'linux' and platform_machine == 'x86_64') or (sys_platform == 'darwin' and platform_machine == 'arm64') or (sys_platform == 'win32' and platform_machine == 'AMD64')", # only platforms with 0.7 wheels "soundfile", # audio handling "ffmpeg-python", "audiomentations", "funasr==1.3.14", # SenseVoice-based ASR / speech-emotion ops (HumanVBench) ] # Distributed Computing distributed = [ "ray[default]>=2.51.0", # distributed computing "pandas<3", # Ray Data versions supported by Data-Juicer are not compatible with pandas 3 "uvloop==0.21.0", # avoid async error before it's fixed in uvloop "pyspark==3.5.5", # distributed data processing "s3fs", # S3 filesystem support for cloud storage "boto3", # AWS SDK for S3 operations "bitarray", # efficient arrays of booleans "pylance", # to support I/O for lance data in ray mode ] # Development & Tools dev = [ "coverage", "pre-commit", # pre-commit hooks "sphinx", "sphinx-autobuild", "sphinx_copybutton", "furo", "myst_parser", "linkify-it-py", "recommonmark", "wandb>=0.19.11,<0.22", # >=0.19.11 lifts protobuf cap to <7, compatible with tensorflow 2.21 (protobuf>=6.31); old <=0.19.0 pin (added in #512 to dodge a 0.19.x bug) forced protobuf<6 and clashed with TF "fire", "click", "toml", # for yapf configuration "pytest", # testing framework "pytest-cov", # coverage reporting "yacs>=0.1.8", # configuration nodes used by HaWoR CPU tests "build", # package building "black>=25.1.0", "flake8-black", "docstring-parser>=0.16", ] # AI Services & APIs ai_services = [ "dashscope", # Alibaba Cloud AI "openai", # OpenAI API "litellm>=1.80.0,<2", # Allows OpenAI 1.x compatibility with label-studio "label-studio>=1.23.0", # Data labeling ] # tools, including analysis, mcp, ... tools = [ "matplotlib", # For drawing analysis results "wordcloud==1.9.4", # For drawing word clouds "mcp[cli]>=1.7.0", # For mcp service "rank-bm25>=0.2.2", # For OP search "fastapi>=0.110", # For using data-juicer in a service way ] # All dependencies (default + all optional) all = [ "py-data-juicer[generic]", "py-data-juicer[vision]", "py-data-juicer[nlp]", "py-data-juicer[audio]", "py-data-juicer[distributed]", "py-data-juicer[dev]", "py-data-juicer[ai_services]", "py-data-juicer[tools]", ] [project.scripts] dj-process = "data_juicer.tools.process_data:main" dj-analyze = "data_juicer.tools.analyze_data:main" dj-install = "data_juicer.tools.dj_install:main" dj-mcp = "data_juicer.tools.mcp_server:main" [build-system] requires = [ "hatchling", "uv>=0.1.0", "Cython>=0.29", "pybind11>=2.6", "setuptools>=64", ] build-backend = "hatchling.build" [tool.hatch.version] path = "data_juicer/__init__.py" [tool.hatch.build.targets.wheel] packages = ["data_juicer", "tools"] include = ["uv.lock", "pyproject.toml"] [tool.hatch.build.targets.wheel.hooks.custom] path = "hatch_build.py" [tool.hatch.build] include = ["uv.lock", "pyproject.toml"] [tool.hatch.build.targets.wheel.shared-data] "uv.lock" = "uv.lock" "pyproject.toml" = "pyproject.toml" [tool.flake8] per-file-ignores = [ "*/__init__.py: F401" ] max-line-length = 120 extend-ignore = [ "E203", # whitespace before ':' (black handles this) "E501", # line too long (black handles this) "BLK100", # black would make changes (black handles this) "F541", # f-string is missing placeholders ] [tool.black] line-length = 120 target-version = ['py310'] [tool.isort] profile = "black" [tool.pytest.ini_options] testpaths = ["tests"] python_files = ["test_*.py"] python_classes = ["*Test", "Test*"] python_functions = ["test_*"] addopts = "-v" filterwarnings = [ "ignore::DeprecationWarning", "ignore::PendingDeprecationWarning", ] [tool.uv] override-dependencies = [ "opencv-python; sys_platform == 'never'", "opencv-python-headless; sys_platform == 'never'", "opencv-contrib-python-headless; sys_platform == 'never'", # never install other versions of opencv-python ]