[build-system] requires = ["setuptools>=77.0.0", "wheel"] build-backend = "setuptools.build_meta" [project] name = "docpull" version = "6.5.5" dynamic = [] description = "Declare, sync, diff, and lock context dependencies for AI agents" readme = {file = "README.md", content-type = "text/markdown"} requires-python = ">=3.10" license = "MIT" license-files = ["LICENSE"] authors = [ {name = "Zachary Roth", email = "support@raintree.technology"} ] maintainers = [ {name = "Raintree Technology", email = "support@raintree.technology"} ] keywords = [ "python", "markdown", "documentation", "web", "website", "web-extraction", "web-to-markdown", "source-ingestion", "provenance", "context-ci", "content-extraction", "developer-tools", "cli", "rag", "llm", "mcp", "ai-agents", "context-engineering", "context-dependencies", "context-packs", "source-packs", "research", "claude", "ai-training-data", ] classifiers = [ # Development Status "Development Status :: 5 - Production/Stable", # Intended Audience "Intended Audience :: Developers", "Intended Audience :: Information Technology", "Intended Audience :: Science/Research", "Intended Audience :: Education", # Environment "Environment :: Console", # Topic "Topic :: Internet :: WWW/HTTP :: Indexing/Search", "Topic :: Text Processing :: Markup :: HTML", "Topic :: Text Processing :: Markup :: Markdown", "Topic :: Utilities", "Topic :: Scientific/Engineering :: Artificial Intelligence", # Natural Language "Natural Language :: English", # Operating System "Operating System :: OS Independent", # Programming Language "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.14", "Programming Language :: Python :: 3 :: Only", # Typing "Typing :: Typed", ] dependencies = [ "beautifulsoup4>=4.12.0", "html2text>=2020.1.16", "defusedxml>=0.7.1", "extruct>=0.15.0", "aiohttp>=3.14.3", # Security floor for the 2026 request-parsing advisories "idna>=3.15", # Security floor for transitive URL/IDNA handling; Renovate raises as needed "regex>=2024.11.6", "rich>=13.0.0", "pyyaml>=6.0", "pydantic>=2.0", "urllib3>=2.7.0", # Security floor for transitive HTTP clients; Renovate raises as needed ] [project.optional-dependencies] proxy = [ "aiohttp-socks>=0.11.0", ] normalize = [ "url-normalize>=1.4.0", ] trafilatura = [ "trafilatura>=1.12.0", ] tokens = [ "tiktoken>=0.7.0", ] mcp = [ "cryptography>=50.0.0", "mcp>=1.28.1,<2", "pydantic-settings>=2.14.2", "pyjwt>=2.13.0", "python-multipart>=0.0.27", "starlette>=1.3.1", ] serve = [ "uvicorn>=0.30.0", ] parquet = [ "pyarrow>=16.0.0", ] markitdown = [ "pypdf>=6.15.0", "markitdown[docx,pdf]>=0.1.0", ] unstructured = [ "pypdf>=6.15.0", "unstructured[all-docs]>=0.16.0", ] pdf = [ "pypdf>=6.15.0", ] parse = [ "pypdf>=6.15.0", "markitdown[docx,pdf]>=0.1.0", "unstructured[all-docs]>=0.16.0", ] presidio = [ "presidio-analyzer>=2.2.0", ] provenance = [ "cryptography>=50.0.0", ] e2b = [ "e2b>=2.0.0", ] llm = [ "tiktoken>=0.7.0", ] all = [ "aiohttp-socks>=0.11.0", "url-normalize>=1.4.0", "trafilatura>=1.12.0", "tiktoken>=0.7.0", "cryptography>=50.0.0", "mcp>=1.28.1,<2", "pydantic-settings>=2.14.2", "pyjwt>=2.13.0", "python-multipart>=0.0.27", "starlette>=1.3.1", "uvicorn>=0.30.0", "pyarrow>=16.0.0", "pypdf>=6.15.0", "markitdown[docx,pdf]>=0.1.0", "unstructured[all-docs]>=0.16.0", "presidio-analyzer>=2.2.0", "e2b>=2.0.0", ] dev = [ "pytest>=7.0.0", "pytest-cov>=4.0.0", "pytest-asyncio>=0.21.0", "mypy>=1.0.0", "ruff>=0.1.0", "bandit>=1.7.0", "pip-audit>=2.0.0", "msgpack>=1.2.1", "pre-commit>=3.0.0", "types-requests>=2.31.0", "types-beautifulsoup4>=4.12.0", "types-defusedxml>=0.7.0", "types-pyyaml>=6.0.0", ] [project.scripts] docpull = "docpull.cli:main" [project.urls] Homepage = "https://github.com/raintree-technology/docpull" Documentation = "https://github.com/raintree-technology/docpull#readme" Repository = "https://github.com/raintree-technology/docpull" "Source Code" = "https://github.com/raintree-technology/docpull" "Bug Tracker" = "https://github.com/raintree-technology/docpull/issues" "Releases" = "https://github.com/raintree-technology/docpull/releases" "MCP Plugin" = "https://github.com/raintree-technology/docpull/tree/main/plugin" "Comparison Guide" = "https://github.com/raintree-technology/docpull/blob/main/docs/alternatives.md" "Download Stats" = "https://pepy.tech/project/docpull" "Changelog" = "https://github.com/raintree-technology/docpull/blob/main/docs/CHANGELOG.md" [tool.setuptools.packages.find] where = ["src"] include = ["docpull*"] [tool.setuptools.package-data] docpull = ["py.typed", "fixtures/*.json", "schemas/*.schema.json"] [tool.uv] constraint-dependencies = ["h2>=4.4.1"] [tool.ruff] line-length = 110 target-version = "py310" [tool.ruff.lint] select = ["E", "F", "W", "I", "N", "UP", "B", "A", "C4", "SIM"] ignore = ["A003"] # Allow 'type' and 'format' as field names in data models [tool.mypy] python_version = "3.10" warn_return_any = true warn_unused_configs = true disallow_untyped_defs = true disallow_any_unimported = true no_implicit_optional = true strict_equality = true warn_redundant_casts = true ignore_missing_imports = true exclude = ["tests/"] plugins = ["pydantic.mypy"] [tool.pydantic-mypy] init_forbid_extra = true init_typed = true warn_required_dynamic_aliases = true [[tool.mypy.overrides]] module = "extruct.*" ignore_missing_imports = true [[tool.mypy.overrides]] module = "url_normalize" ignore_missing_imports = true [[tool.mypy.overrides]] module = "docpull.models.*" disallow_any_unimported = false warn_return_any = false [tool.bandit] # Policy: every entry in `skips` MUST have a one-line justification # above it explaining what bandit found, why it's a false positive # *for this codebase*, and (if narrow) why a `# nosec BXXX # reason` # annotation at the call site would have been worse. Bandit skips # silence findings repo-wide, so the bar to add one is higher than # silencing a single line. If a new skip is unavoidable, add it here # in PR review, not as a drive-by. # # B101 (assert_used) — flags every `assert x is not None` we use for # type narrowing. Bandit's concern is that assertions vanish under # `python -O`. docpull is a CLI / SDK, never invoked with -O, and the # narrowing asserts are not load-bearing safety checks. Skipping the # rule globally keeps the existing idiom without 8+ inline `# nosec` # annotations in fetcher.py / pipeline/steps/convert.py. skips = ["B101"] [tool.pytest.ini_options] minversion = "7.0" testpaths = ["tests"] python_files = ["test_*.py"] python_classes = ["Test*"] python_functions = ["test_*"] asyncio_mode = "auto" addopts = [ "-ra", "--showlocals", ] markers = [ "integration: marks tests as integration tests (deselect with '-m \"not integration\"')", "internal_legacy: marks tests for private legacy modules outside the public release surface", ] [tool.coverage.run] source = ["src/docpull"] omit = ["tests/*", "*/test_*.py"] patch = ["subprocess"] [tool.coverage.report] exclude_lines = [ "pragma: no cover", "def __repr__", "raise AssertionError", "raise NotImplementedError", "if __name__ == .__main__.:", "if TYPE_CHECKING:", ]