[build-system] requires = ["hatchling"] build-backend = "hatchling.build" [project] name = "tokenmizer" version = "0.6.1" description = "Graph-backed checkpoint and resume for LLM sessions. An OpenAI-compatible proxy that remembers decisions across context limits." readme = "README.md" requires-python = ">=3.10" license = { text = "MIT" } authors = [{ name = "Shweta Mishra", url = "https://github.com/Shweta-Mishra-ai" }] keywords = [ "llm", "context", "memory", "checkpoint", "knowledge-graph", "ai", "anthropic", "openai", "proxy", "mcp", "rag", ] classifiers = [ "Development Status :: 4 - Beta", "Intended Audience :: Developers", "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Topic :: Software Development :: Libraries", "Topic :: Scientific/Engineering :: Artificial Intelligence", "Typing :: Typed", ] dependencies = [ "fastapi>=0.111.0", "uvicorn[standard]>=0.30.0", "pydantic>=2.7.0", "pydantic-settings>=2.3.0", "httpx>=0.27.0", "typer>=0.12.0", "rich>=13.7.0", "tiktoken>=0.7.0", "pyyaml>=6.0.1", ] [project.optional-dependencies] anthropic = ["anthropic>=0.30.0"] openai = ["openai>=1.35.0"] gemini = ["google-genai>=1.0.0"] cohere = ["cohere>=5.5.0"] # Semantic cache (optional but recommended) cache = ["sentence-transformers>=3.0.0", "numpy>=1.26.0"] # File intelligence extras (better PDF/Excel support) files = ["pypdf>=4.0.0", "openpyxl>=3.1.0"] # ML compression (much better than heuristic-only) compression = ["llmlingua>=0.2.0"] # Redis client. NOTE: nothing in TokenMizer talks to Redis. The module # that was going to (tokenmizer/state/backend.py) was deleted for having # no callers; shared cross-worker state is SQLite, like everything else # durable here. Kept as an extra only so an existing install does not # break on the name. redis = ["redis>=5.0.0"] # Observability observability = ["prometheus-client>=0.20.0", "opentelemetry-sdk>=1.25.0", "opentelemetry-api>=1.25.0"] # Framework adapters under examples/ (langchain_memory.py, # langgraph_checkpointer.py). Not a runtime dependency of the proxy or of # tokenmizer.agents.Memory itself — only install this to run those two # example files or their tests. adapters = ["langchain-core>=0.3.0", "langgraph>=0.2.0"] # Development dev = [ "pytest>=8.0.0", "pytest-asyncio>=0.23.0", "pytest-cov>=5.0.0", "httpx>=0.27.0", "ruff>=0.5.0", "anthropic>=0.30.0", "openai>=1.35.0", "google-genai>=1.0.0", ] # Install everything all = [ "tokenmizer[anthropic,openai,gemini,cohere,cache,files,compression,redis,observability]" ] [project.scripts] tokenmizer = "tokenmizer.cli:app" tokenmizer-mcp = "tokenmizer.mcp.server:run_stdio_server" [project.urls] Homepage = "https://github.com/Shweta-Mishra-ai/tokenmizer" Repository = "https://github.com/Shweta-Mishra-ai/tokenmizer" Issues = "https://github.com/Shweta-Mishra-ai/tokenmizer/issues" Changelog = "https://github.com/Shweta-Mishra-ai/tokenmizer/blob/main/CHANGELOG.md" Security = "https://github.com/Shweta-Mishra-ai/tokenmizer/blob/main/SECURITY.md" [tool.hatch.build.targets.wheel] packages = ["tokenmizer"] # py.typed marks the package as typed so mypy/pyright in a CONSUMER's # project actually reads our annotations instead of treating the whole # package as Any (PEP 561). Without it in the wheel, the annotations we # ship are invisible downstream. artifacts = ["tokenmizer/py.typed"] [tool.hatch.build.targets.sdist] # Ship what a consumer needs to verify the package, and nothing else: # the source tree, tests, benchmarks and the docs they reference. include = [ "/tokenmizer", "/tests", "/benchmarks", "/README.md", "/CHANGELOG.md", "/LICENSE", "/SECURITY.md", "/CONTRIBUTING.md", "/tokenmizer.yaml", "/pyproject.toml", ] [tool.pytest.ini_options] asyncio_mode = "auto" testpaths = ["tests"] addopts = "-v --tb=short" [tool.ruff] line-length = 100 target-version = "py310" [tool.ruff.lint] select = ["E", "F", "W", "I"] ignore = ["E501"] [tool.ruff.lint.per-file-ignores] # A self-contained diagnostic script that deliberately interleaves # imports with progress output so each section runs and reports even if # a later import is unavailable. Restructuring it would defeat the point. "scripts/run_stdlib_tests.py" = ["E402"] [tool.coverage.run] source = ["tokenmizer"] # Only NON-PRODUCT code is omitted. Do not add product modules here to # inflate the number — a gamed coverage gate is worse than a low honest one. omit = [ "tests/*", "benchmarks/*", "examples/*", ] [tool.coverage.report] # Floor over ALL product code — nothing is omitted to flatter the number. # Set below the measured figure (~77%) on purpose, so an incidental dip # does not red the build while a real collapse still does. Single source # of truth: CI reads this, and must not override it on the command line. fail_under = 50