[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"

[project]
name = "marin-root"
version = "0.1.0"
description = "Marin workspace root and experiments"
license = { file = "LICENSE" }
requires-python = ">=3.12"
dependencies = [
    # workspaces
    "marin-iris",
    "marin-fray",
    "marin-haliax",
    "marin-levanter",
    "marin-core",
    # [secrets] so `iris cluster up` on a Kubernetes cluster can resolve the
    # `gcp-secret://` signing key in the operator's shell: the controller pod holds no
    # cloud credentials and reads the key from a projected Secret instead.
    "marin-rigging[secrets]",
    "marin-zephyr",
    # marin-finelog is the pure-Python finelog client + deploy/CLI tooling, a
    # workspace member (editable source) so deploy/CLI fixes land without a wheel
    # republish. Its native companion marin-finelog-server (the in-process
    # log-server ext, module `finelog_server`, maturin lib/finelog/rust) is NOT a
    # member: it ships as a pre-built wheel and reaches us transitively via
    # marin-iris, which depends on it explicitly. `scripts/rust_mode.py dev` swaps
    # marin-finelog-server to a local source build.
    "marin-finelog",
    "marin-deploy",
    # marin-ducky is the ad-hoc DuckDB SQL service (dashboard + runner + deploy),
    # a workspace member so service fixes land without a wheel republish.
    "marin-ducky",
    # top level deps. try to keep these small.
    "watchdog",
    # marin-dupekit is the pure-Python text-dedup surface, a workspace member
    # (editable source). Its native companion marin-dupekit-native (the Rust
    # kernels, module `dupekit_native`, maturin lib/dupekit/rust) is NOT a
    # member: it ships as a pre-built platform wheel and reaches us transitively
    # as marin-dupekit's runtime dependency. `scripts/rust_mode.py dev` swaps
    # marin-dupekit-native to a local source build.
    "marin-dupekit",
]

[tool.uv]
fork-strategy = "fewest"
# "explicit": allow pre-releases only for packages whose declared specifier
# names one, so our intentional pre-releases resolve while everything else stays
# on its latest stable release. (A global "allow" would instead promote any
# transitive dependency's newest beta/rc into the lock.) Opted in: the
# marin-dupekit-native nightly (floor in lib/dupekit/pyproject.toml); the
# omegaconf dev pin in override-dependencies
# below; and the prerelease-only transitive deps in constraint-dependencies
# (tfp-nightly and the OpenTelemetry instrumentation packages publish no stable
# release).
prerelease = "explicit"
constraint-dependencies = [
    # Enable pre-releases for transitive deps that publish no stable release at
    # all. A constraint intersects (rather than replaces) the upstream
    # specifier, so the resolved version still follows whatever requires them;
    # the pre-release component here only flips uv's per-package opt-in.
    "tfp-nightly>=0.1.dev0",
    "opentelemetry-instrumentation>=0.41b0",
    "opentelemetry-instrumentation-grpc>=0.41b0",
    "opentelemetry-semantic-conventions>=0.1b0",
]
override-dependencies = [
    "omegaconf>=2.4.0.dev4",
    "aiofiles>=24.1.0",
    "anthropic>=0.71.0",
    "antlr4-python3-runtime==4.11",
    "python-multipart>=0.0.27",
    "wheel>=0.46.2",
    "datasets>=3.1.0,<5.0.0",
    "equinox>=0.11.10",  # Override vizier's pin for modern JAX compatibility
    "authlib>=1.7.2",
    "cbor2>=6.0.1",
    "click>=8.3.1",
    "gitpython>=3.1.50",
    "gradio>=6.14.0",
    "jupyter-server>=2.18.2",
    "jupyterlab>=4.5.7",
    "litellm>=1.83.14",
    "lxml>=6.1.0",
    "mistune>=3.2.1",
    "multipart>=1.3.1",
    "nltk>=3.9.4",
    "notebook>=7.5.6",
    "orjson>=3.11.9",
    "pillow>=12.2.0",
    "pyasn1>=0.6.3",
    "python-dotenv>=1.1.1",
    "ray>=2.55.1",
    "rich>=14.1.0",
    "setuptools>=82.0.1",
    "tiktoken>=0.12.0",
    "tornado>=6.5.5",
]
conflicts = [
    # GPU extras still carry CUDA/Linux-only solver constraints. Keep them out
    # of non-GPU cross-package splits while allowing the TPU stack to resolve
    # together.
    [
        { package = "marin-levanter", extra = "gpu" },
        { package = "marin-core", extra = "cpu" },
    ],
    [
        { package = "marin-levanter", extra = "gpu" },
        { package = "marin-core", extra = "tpu" },
    ],
    [
        { package = "marin-core", extra = "gpu" },
        { package = "marin-levanter", extra = "tpu" },
    ],
    [
        { package = "marin-core", extra = "gpu" },
        { package = "marin-fray", group = "fray_tpu_test" },
    ],
    [
        { package = "marin-levanter", extra = "gpu" },
        { package = "marin-fray", group = "fray_tpu_test" },
    ],
]

[tool.uv.workspace]
members = [
    "lib/iris",
    "lib/fray",
    "lib/finelog",
    "lib/dupekit",
    "lib/ducky",
    "lib/haliax",
    "lib/levanter",
    "lib/marin",
    "lib/rigging",
    "lib/zephyr",
    "lib/finestore",
    "infra/pulumi",
    "infra/buckets",
    "infra/deploy",
]
# Native maturin packages are excluded from uv workspace discovery because each
# owns its own build backend and package metadata. The path sources below build
# them from this checkout for development; published packages resolve their
# companion wheels from PyPI.
#   - lib/finelog/rust  -> marin-finelog-server (module `finelog_server`)
#   - lib/dupekit/rust  -> marin-dupekit-native  (module `dupekit_native`)
#   - lib/iris/rust     -> marin-iris-native     (module `iris_native`)
exclude = ["lib/finelog/rust", "lib/dupekit/rust", "lib/iris/rust", "config/external/*"]

[tool.uv.sources]
marin-iris = { workspace = true }
marin-fray = { workspace = true }
marin-finelog = { workspace = true }
marin-deploy = { workspace = true }
marin-dupekit = { workspace = true }
marin-ducky = { workspace = true }
marin-haliax = { workspace = true }
marin-levanter = { workspace = true }
marin-core = { workspace = true }
marin-rigging = { workspace = true }
marin-zephyr = { workspace = true }
marin-finestore = { workspace = true }
marin-iac = { workspace = true }
nvidia-cutlass-dsl = { index = "nvidia" }
nvidia-cutlass-dsl-libs-base = { index = "nvidia" }
nvidia-cutlass-dsl-libs-cu13 = { index = "nvidia" }
# ### BEGIN RUST-DEV SOURCES ###
# ### END RUST-DEV SOURCES ###

[[tool.uv.index]]
name = "nvidia"
url = "https://pypi.nvidia.com"
explicit = true


[tool.black]
line-length = 121
target-version = ["py312"]
preview = true

# Note :: Grow more strict over time!
extend-exclude = """
(
    scripts/
)
"""

[tool.ruff]
line-length = 121
target-version = "py312"

# Note :: Grow more strict over time!
# Most of `scripts/**` is excluded until cleaned up, but `scripts/ci/**` and
# `scripts/cost_manager/**` hold scripts invoked directly by `.github/workflows`
# and stay under ruff.
extend-exclude = ["scripts/"]
extend-include = ["scripts/ci/**/*.py", "scripts/cost_manager/**/*.py"]

[tool.ruff.lint]
# A,B,E,F,I,NPY,RUF,UP,W are the established families. The individually-listed
# codes below mechanically enforce CodeQL "Code quality" maintainability/reliability
# checks locally so they are caught at pre-commit time rather than only post-merge:
#   N805    -> "First parameter of a method is not named 'self'"
#   PLR0124 -> "Comparison of identical values" / "Redundant comparison"
#   PLR1722 -> "Use of exit() or quit()" (use sys.exit)
#   RET502  -> "Explicit returns mixed with implicit returns" (bare `return` value path)
#   RET503  -> "Explicit returns mixed with implicit returns" (missing explicit return)
#   SIM115  -> "File is not always closed" (open() without a context manager)
#   PLC0415 -> "import should be at the top-level of a file" (no function-local
#              imports except to break a circular import or guard an optional dep)
# B018 (already via "B") and F401/F841/F811 (via "F") cover the rest of the
# actionable CodeQL maintainability set; B018 correctly ignores `...`/docstrings.
select = [
    "A", "B", "E", "F", "I", "NPY", "RUF", "UP", "W",
    "N805", "PLR0124", "PLR1722", "RET502", "RET503", "SIM115", "PLC0415",
    # Local-import hygiene (issue #6219):
    #   TID252 -> "relative-imports" (prefer absolute over parent-relative imports)
    #   ICN001 -> "unconventional-import-alias" (e.g. `import numpy as np`)
    # lib/haliax and lib/levanter carry their own [tool.ruff.lint] configs; these
    # two rules are mirrored there via extend-select so they apply repo-wide.
    "TID252", "ICN001",
]
# Keep UP046/UP047 ignored for now: migrating generics to PEP 695 syntax is
# out of scope for the Python 3.12 floor bump.
ignore = ["F722", "B008", "UP015", "UP046", "UP047", "A005", "E741"]

[tool.ruff.lint.per-file-ignores]
"__init__.py" = ["E402", "F401"]
# Notebook cells legitimately interleave imports with computation as they are
# executed top-to-bottom; PLC0415's top-of-file rule does not apply to cells.
"*.ipynb" = ["PLC0415"]
# Bloom filter tests deliberately compare an object with itself to exercise its
# reflexive comparison operators (==, <=, >=, ...), so PLR0124 is expected here.
"lib/dupekit/tests/test_bloom.py" = ["PLR0124"]
# The decon report renderer embeds CSS/JS/HTML as long string literals; wrapping
# those lines hurts readability far more than the length limit helps.
"experiments/datakit/decontam/viewer/report.py" = ["E501"]

[tool.ruff.lint.isort]
known-first-party = ["experiments"]
# "config" is the local module in infra/tpu-ci/{setup,vm_manager}.py. The repo-root
# config/ directory (cluster YAMLs) would otherwise make isort treat ``import config``
# as first-party and reorder it, so pin it to the third-party group it already sits in.
known-third-party = ["wandb", "config"]


# Pyrefly type checker configuration
[tool.pyrefly]
python-version = "3.12"
# Worktrees often live under hidden parent directories (e.g. `.codex/worktrees/*`).
# Disable pyrefly's built-in exclude heuristics so hidden ancestors do not exclude
# the entire project when running from those paths.
disable-project-excludes-heuristics = true

project-includes = [
    "lib/marin/src/**/*.py",
    "lib/levanter/src/**/*.py",
    "lib/haliax/src/**/*.py",
    "lib/fray/src/**/*.py",
    "lib/iris/src/**/*.py",
    "lib/iris/scripts/**/*.py",
    "lib/ducky/src/**/*.py",
    "lib/rigging/src/**/*.py",
    "lib/zephyr/src/**/*.py",
    "lib/finestore/src/**/*.py",
    "infra/evaldash/src/**/*.py",
    "infra/deploy/src/**/*.py",
    "scripts/ci",
    "scripts/cost_manager",
]

# Explicitly tell Pyrefly where our editable packages live so it resolves imports
# against the library sources instead of the top-level `src` directory, which only
# contains a handful of legacy modules.
search-path = [
    "lib/marin/src",
    "lib/levanter/src",
    "lib/haliax/src",
    "lib/rigging/src",
    "lib/zephyr/src",
    "lib/finestore/src",
    "lib/fray/src",
    "lib/iris/src",
    "lib/ducky/src",
    "infra/deploy/src",
]
disable-search-path-heuristics = true

# Type-check against real installed third-party types. CI's marin-lint job syncs
# this venv (uv sync --all-packages --extra=cpu --no-default-groups); locally it is
# your worktree .venv. Relative path keeps it worktree-local.
python-interpreter-path = ".venv/bin/python"
use-ignore-files = false

# Parse jaxtyping/native tensor shape annotations (e.g. Float[Array, "batch seq"])
# as shape specifications instead of forward-reference type expressions. Without this,
# Pyrefly reads the dimension tokens inside the shape string ("B", "N", "groups", ...)
# as undefined forward references and emits ~141 false `unknown-name` errors across
# levanter/grug and levanter/kernels. See docs/plans/pyrefly-tensor-shapes.md.
tensor-shapes = true

# Modules treated as Any when absent on disk. Pyrefly still uses real stubs when
# the dep is installed (so a deleted/misspelled internal module still trips
# missing-import). Covers deps no lint profile installs (vllm/lm_eval, resiliparse)
# and compute deps a local venv commonly skips (torch, sympy). List both `pkg` and
# `pkg.*`.
ignore-missing-imports = [
    # The evaldash server's own deps: its image installs them, no lint profile does.
    "anthropic",
    "anthropic.*",
    "google.cloud.compute_v1",
    "google.cloud.compute_v1.*",
    "google.cloud.sql.connector",
    "google.cloud.sql.connector.*",
    # Optional extra marin-rigging[secrets]; only rigging.secrets imports it, lazily.
    "google.cloud.secretmanager",
    "google.cloud.secretmanager.*",
    # QuACK and CUTLASS are Linux/CUDA-only optional dependencies.
    "cutlass",
    "cutlass.*",
    "jax_triton",
    "jax_triton.*",
    "jaxlib",
    "jaxlib.*",
    "kubernetes",
    "kubernetes.*",
    "librosa",
    "librosa.*",
    "lm_eval",
    "lm_eval.*",
    "memray",
    "memray.*",
    "prompt_toolkit",
    "prompt_toolkit.*",
    "quack",
    "quack.*",
    "resiliparse",
    "resiliparse.*",
    "sympy",
    "sympy.*",
    "tensorboardX",
    "tensorboardX.*",
    "torch",
    "torch.*",
    "tpu_inference",
    "tpu_inference.*",
    "trackio",
    "trackio.*",
    "triton",
    "triton.*",
    "vllm",
    "vllm.*",
    "xprof",
    "xprof.*",
]

# Exclude non-production code from type checking
project-excludes = [
    "experiments/**",
    "scripts/debug/**",
    "scripts/ferries/**",
    "scripts/gpu_eval/**",
    "scripts/iris/**",
    "scripts/ops/**",
    "scripts/pm/**",
    "scripts/speedrun/**",
    "scripts/training/**",
    "scripts/*.py",
    "tests/**",
    "examples/**",           # Example code doesn't need strict typing
    "lib/**/crawl/**",       # Crawl scripts have library typing issues with smart_open
    "lib/iris/src/iris/rpc/*_pb2*",  # Generated protobuf files
    "lib/iris/src/iris/rpc/*_connect.py",  # Generated Connect RPC files
    # Keep key excludes that pyrefly normally adds heuristically.
    "**/node_modules",
    "**/__pycache__",
    "**/venv/**",
]

# Disable specific error codes that are primarily noise from missing type stubs.
# Enable gradually as the codebase improves.
[tool.pyrefly.errors]
# Untyped-library attribute access (jax/flax/equinox); sampling found ~no real bugs.
missing-attribute = false

# Real undefined-name bugs. The former jaxtyping shape-string false positives are now
# handled natively by tensor-shapes = true above; residual false positives are
# suppressed in .pyrefly-baseline.json.
unknown-name = true

# Complex overload resolution (stdlib/library overloads); sampling found ~no real bugs.
no-matching-overload = false

# Mostly third-party stub noise (smart_open, Flax, jax/numpy arrays), with a handful of
# real callsite mismatches buried in the volume.
bad-argument-type = false

# Real "code that can be better written" bugs mixed with control-flow false positives
# pyrefly cannot prove safe.
unbound-name = false

# Kept enabled (high signal): bad-override, bad-assignment, bad-return,
# unsupported-operation, not-callable. Pre-existing instances are suppressed via
# .pyrefly-baseline.json.

# jit_scheduler.py annotates eqx.Module fields with single-axis haxtyping shapes,
# e.g. `ht.i32[NamedArray, "position"]`. tensor-shapes = true parses multi-token
# shape strings ("seq page") correctly, but reads a lone bare-identifier axis
# ("position", "seq") as a forward reference, yielding false unknown-name errors.
# Scope the disable to this one file rather than scattering 17 inline ignores.
[[tool.pyrefly.sub_config]]
matches = "lib/levanter/src/levanter/inference/jit_scheduler.py"
[tool.pyrefly.sub_config.errors]
unknown-name = false

# Turn bad-argument-type ON for these controller-driving scripts (it is off
# repo-wide) so a wrong-type call into the controller API fails type-checking.
[[tool.pyrefly.sub_config]]
matches = "lib/iris/scripts/**"
[tool.pyrefly.sub_config.errors]
bad-argument-type = true

[tool.hatch.metadata]
allow-direct-references = true

[tool.hatch.build.targets.wheel]
packages = ["experiments"]

[tool.hatch.build.targets.sdist]
packages = ["experiments"]

[tool.pytest.ini_options]
# pytest-timeout uses SIGALRM on POSIX; the timeout guard hard-kills the process if a test suppresses that failure.
timeout = 60
filterwarnings = ["ignore::DeprecationWarning"]
log_format = "%(asctime)s %(levelname)s %(message)s"
log_date_format = "%Y-%m-%d %H:%M:%S"
markers = [
    "slow: mark tests as slow for CI - use -m 'not slow'",
    "integration: mark tests as integration tests that require cluster infrastructure",
    "data_integration: mark tests that require external data fixtures (e.g. HuggingFace datasets)",
    "cluster: mark smokes that submit a real job to a standing Marin cluster (run directly with explicit credentials)",
    "requires_cluster: mark Iris tests that start or connect to a cluster",
    "docker: mark tests that require a Docker daemon",
    "manual: mark tests that are run by hand on demand, never by any CI workflow",
]
testpaths = ["tests", "experiments"]
pythonpath = ["."]

# Don't run slow, integration, data_integration, cluster-bound, Docker, or manual tests by default.
# Integration tests require external infrastructure (Iris cluster, GCS,
# gated HF repos, etc.); data_integration tests fetch fixtures from HF;
# cluster and requires_cluster tests submit jobs or start cluster infrastructure;
# Docker tests require a local daemon;
# manual tests are run by hand (e.g. live TPU smokes in a custom container).
# All but manual are run from dedicated CI workflows. Tests that need a TPU carry
# @skip_if_no_tpu (tests/test_utils.py) and skip themselves when none is present;
# the Iris markers also apply when lib/iris test paths are passed from the root.
addopts = "-p marin.pytest_timeout_guard --session-timeout=600 -n auto --dist=worksteal -m 'not slow and not integration and not data_integration and not cluster and not requires_cluster and not docker and not manual' --import-mode=importlib"