[project]
name = "dolma"
version = "1.2.1"
description = "Toolkit for pre-processing LLM training data."
license = { text = "Apache-2.0" }
readme = "README.md"
requires-python = ">=3.10,<3.13"
dependencies = [
    "anyascii>=0.3.2",
    "blingfire==0.1.8",
    # "boto3>=1.28",
    "boto3",
    # "cached-path>=1.5.1", # no longer needed
    # "fasttext==0.9.2",    # broken with new version of setuptools; using fasttext-wheel instead
    "fasttext-wheel==0.9.2",
    "fsspec>=2023.6.0",
    "jsonpath-ng",
    "jq",
    "msgspec>=0.14.2",
    "nltk>=3.9.1",
    "omegaconf>=2.3.0",
    # "pycld2==0.41",
    # "pycld3==0.22",       # does not install correctly
    "platformdirs>=4.2.0",
    "pyyaml",
    "python-dotenv>=0.19.0",
    "requests",
    "rich",
    "s3fs==2023.6.0",
    "smart-open>=7.0.4",
    "tokenizers>=0.15.0,<=0.19.1",
    "tqdm",
    "uniseg",
    "numpy<2",
    "necessary>=0.4.3",
    "charset-normalizer>=3.2.0",
    "zstandard>=0.23.0",
]
classifiers = [
    "Development Status :: 5 - Production/Stable",
    "Intended Audience :: Science/Research",
    "License :: OSI Approved :: Apache Software License",
    "Programming Language :: Python :: Implementation :: CPython",
    "Programming Language :: Rust",
    "Topic :: Scientific/Engineering :: Artificial Intelligence",
    "Topic :: Text Processing",
    "Typing :: Typed",
]

[[project.authors]]
name = "Allen Institute for Artificial Intelligence"
email = "contact@allenai.org"

[[project.authors]]
name = "Luca Soldaini"
email = "luca@soldaini.net"

[[project.authors]]
name = "Kyle Lo"
email = "kylel@allenai.org"

[[project.authors]]
name = "Rodney Kinney"
email = "rodneyk@allenai.org"

[[project.authors]]
name = "Aakanksha Naik"
email = "aakankshan@allenai.org"

[[project.authors]]
name = "Abhilasha Ravichander"
email = "abhilashar@allenai.org"

[[project.authors]]
name = "Akshita Bhagia"
email = "akshitab@allenai.org"

[[project.authors]]
name = "Dirk Groeneveld"
email = "dirkg@allenai.org"

[[project.authors]]
name = "Dustin Schwenk"
email = "dustins@allenai.org"

[[project.authors]]
name = "Ian Magnusson"
email = "ianm@allenai.org"

[[project.authors]]
name = "Khyathi Chandu"
email = "khyathic@allenai.org"

[[project.maintainers]]
name = "Allen Institute for Artificial Intelligence"
email = "contact@allenai.org"

[project.urls]
Homepage = "https://github.com/allenai/dolma"


[project.scripts]
dolma = "dolma.cli.__main__:main"

[project.optional-dependencies]
dev = [
    "Flake8-pyproject>=1.1.0",
    "black>=22.6.0",
    "flake8-pyi>=22.8.1",
    "flake8>=5.0",
    "ipdb>=0.13.0",
    "ipython>=8.4.0",
    "isort>=5.10.1",
    "mypy==0.971",
    "pytest>=5.2",
    "types-PyYAML",
    "types-dateparser",
]
# extension to process code
code = ["detect-secrets==1.4.0", "beautifulsoup4>=4", "pygments", "regex"]
# extension to detect PIIs using presidio
pii = [
    # "presidio_analyzer==2.2.32", # presidio causes too many issues with installation, asking users to install it manually
    "regex",
]

# language detection; by default, we use fastttext, everything else is optional
lang = [
    "fasttext-wheel==0.9.2",
    # "LTpycld2==0.42",                  # LTpycld2/pycld2 all so buggy; recommending user install them on their own
    "pycld2",
    "lingua-language-detector>=2.0.0",
    # "langdetect>=1.0.9",
]

# extension to parse warc files
warc = ["fastwarc", "w3lib", "url-normalize"]
trafilatura = [
    # must include warc dependencies
    "dolma[warc]",
    # core package
    "trafilatura>=1.6.1",
    # following are all for speeding up trafilatura
    "brotli",
    "cchardet >= 2.1.7; python_version < '3.11'",         # build issue
    "faust-cchardet >= 2.1.18; python_version >= '3.11'", # fix for build
    "htmldate[speed] >= 1.4.3",
    "py3langid >= 0.2.2",
]

resiliparse = ["dolma[warc]", "resiliparse"]

# all extensions
all = [
    "dolma[dev]",
    "dolma[code]",
    "dolma[pii]",
    "dolma[trafilatura]",
    "dolma[resiliparse]",
    "dolma[lang]",
]

[build-system]
requires = [
    "setuptools >= 61.0.0",
    "wheel",
    "maturin[patchelf]>=1.1,<2.0 ; platform_system=='Linux'",
    "maturin>=1.1,<2.0 ; platform_system!='Linux'",
]
build-backend = "maturin"


[tool.maturin]
python-source = "python"
features = ["pyo3/extension-module"]


[tool.setuptools.packages.find]
where = ["src"]

[tool.setuptools.package-data]
dolma = ["py.typed", "data/*"]

[tool.black]
line-length = 115
include = '\.pyi?$'
exclude = '''
(
      __pycache__
    | \.git
    | \.mypy_cache
    | \.pytest_cache
    | \.vscode
    | \.venv
    | \bdist\b
    | \bdoc\b
    | \.cargo
    | configs
    | docs
    | scripts
    | sources
    | src
    | target
    | tests/config
    | tests/data
    | tests/work
)
'''
target-version = ["py39", "py310", "py311", "py312"]


[tool.isort]
profile = "black"
py_version = 39
known_first_party = ["dolma"]
known_local_folder = ["tests", "python"]
extend_skip_glob = [
    "configs/*",
    "docs/*",
    "scripts/*",
    "sources/*",
    "src/*",
    "tests/config/*",
    "tests/data/*",
    "tests/work/*",
]

[tool.autopep8]
max_line_length = 115
in-place = true
recursive = true
aggressive = 3

[tool.mypy]
python_version = "3.10"
ignore_missing_imports = true
no_site_packages = true
allow_redefinition = false
warn_unused_configs = true
warn_unused_ignores = true
warn_no_return = true
warn_return_any = false
warn_unreachable = true
show_error_codes = true
pretty = true
plugins = ["numpy.typing.mypy_plugin"]

[tool.mypy-tests]
strict_optional = false