[project] name = "fabric-emulator-python" version = "0.1.0" description = "Locked Python tooling and end-to-end witnesses for fabric-emulator" requires-python = ">=3.12" dependencies = [] [dependency-groups] test = [ "fabric-emulator-notebookutils", "fabric-target", "pytest>=8.3,<10", # The Go side has enforced a coverage floor since the SQLite double was # dropped; the python side had 28 tests and no number at all, so a checker # could lose its tests without anything saying so. "pytest-cov>=5,<8", # test_govern_column_schema.py validates the payloads govern_ingest builds # against OpenMetadata's OWN vendored schema (third_party/openmetadata-schema). # Draft-07, so `referencing` comes with it for the local $ref registry. "jsonschema>=4.21,<5", # test_spark_agent_packaging.py reads the compose files to prove the agent # ships INSIDE its image rather than over a bind mount. Declared here # because a developer machine usually has pyyaml lying around and CI does # not — which is exactly how it passed locally and broke on the runner. "pyyaml>=6,<7", # test_example_sql_endpoint.py imports the EXAMPLES' resolver # (examples/contoso-fixtures/common.py) to test the SQL-endpoint discovery # its real branch takes, which no local e2e can reach. That module imports # requests, and the same trap as pyyaml above caught this one: it passed in a # developer .venv carrying requests from another group and failed in a clean # environment. Verified with UV_PROJECT_ENVIRONMENT pointed at an empty dir. "requests>=2.32,<3", # The MERGE/CDF/DESCRIBE tests that write a real `_delta_log` used to skip # on the notebookutils CI job (`--group test` alone). Skipping them meant # CI never ran the proof those cells claim, and the coverage floor hid the # gap the moment the Connect wraps landed. Same pins as the `delta-rs` # group; pandas is what `read_change_feed` materialises through. "deltalake>=0.23,<2", "pyarrow>=23.0.1,<26", "pandas>=2,<3", "pyjks>=20,<22", # 49.0.0 is the first release that closes the three open wheel and # path-building advisories; held below 50 so a major cannot land unreviewed. "cryptography>=42,<51", ] lint = [ "ruff>=0.14,<0.17", # Astral's type checker. Pre-1.0, so pinned tight: a minor bump can change # which inferences it makes, and a type checker that changes its mind # between runs is a broken build nobody caused. "ty>=0.0.1a14", ] adls-sdk = [ "azure-storage-blob>=12.24,<13", "pyarrow>=23.0.1,<26", ] # The third-party witness for the Purview Data Map row. pyapacheatlas speaks # Apache Atlas v2, which is what the Data Map IS — so this is a real client # rather than a client shaped like our implementation. purview-datamap = [ "pyapacheatlas>=0.16,<1", ] # Official Python MCP SDK — the unmodified Streamable HTTP client that # witnesses Fabric Core MCP Server (POST /v1/mcp/core). Pinned below v2: v2 # renamed streamablehttp_client and dropped the headers= kwarg. mcp = [ # >=2, not >=1.12: mcp 2.0 renamed `streamablehttp_client` to # `streamable_http_client`, which e2e/mcp-core/driver.py imports. A # floor that still admitted 1.x would resolve a client that import # cannot find, and the suite would fail on ImportError rather than on # anything about the emulator. "mcp>=2,<3", ] delta-rs = [ "deltalake>=0.23,<2", "pyarrow>=23.0.1,<26", ] duckdb = [ "deltalake>=0.23,<2", "duckdb>=1.1,<2", "pyarrow>=23.0.1,<26", ] s3 = [ "boto3>=1.35,<2", "requests>=2.32,<3", ] fabric-cicd = ["fabric-cicd>=1.3"] fabric-target = [ "fabric-target[real,sessions]", # The conformance suite's runMultiple cases drive the notebookutils shim on # both legs, so it belongs in this group's closure. Without it the emulator # leg dies at import on a clean checkout — which it did, while passing on a # laptop whose venv had other groups installed. "fabric-emulator-notebookutils", "pytest>=8.3,<10", ] governance = [ "deltalake>=0.23,<2", "pyarrow>=23.0.1,<26", "requests>=2.32,<3", # govern_ingest reads the ODCS contracts, so a cataloged table carries the # meaning a human wrote rather than only the shape the emulator can infer. "pyyaml>=6,<7", ] # docs/demo/flow.py: the REST driver plus a real Delta writer, which is all the # flow.gif recording needs — the medallion it films is moved by the emulator's # own Copy executor rather than by an engine. demo = [ "deltalake>=0.23,<2", "pyarrow>=23.0.1,<26", "requests>=2.32,<3", ] great-expectations = [ "great-expectations>=0.18,<2", "pandas>=2.2,<4", ] runtime = [] # pyspark-client is pinned to what pysail is built and tested against — read # BOTH versions below as one number, and move them together or not at all. # # The reason is a real outage, not tidiness. pyspark-client 4.2.0 added # `runner_conf`/`eval_conf` to `pyspark.worker.read_udfs`; Sail 0.6.x called the # 3-argument form, so pairing 0.6.6 with 4.2.0 killed every Python UDF with # TypeError: read_udfs() missing 2 required positional arguments # The UDF runs inside the Sail server's embedded CPython, so the *server* image # is what must match. Client and server are pinned together so they cannot drift. # # Sail 0.7.0 moved to Spark 4.2 (lakehq/sail#2289) and its own `test` extra now # pins pyspark-client==4.2.0, which is why both numbers change together. # # MEASURED, because the obvious symmetry argument is wrong. Three combinations # were run locally against a real Sail server before this landed: # 0.7.0 + 4.2.0 UDF round-trips <- what we ship # 0.7.0 + 4.1.1 UDF ALSO round-trips <- 0.7.0 tolerates the old client # 0.6.6 + 4.2.0 dies in `createDataFrame` <- ValueError: invalid literal # for int() with base 10: '3GB' # So the constraint is NOT symmetric, and it is worth knowing which way it runs: # the OLD server cannot take the NEW client, while the new server takes either. # Note also that the 0.6.6 + 4.2.0 break reproduces as a config-parse failure # well before any UDF runs — the `read_udfs` TypeError above is a second, # later incompatibility on the same pairing, reachable only once you avoid # `createDataFrame` (which is why e2e/sail/driver.py builds rows with VALUES). # # Practical rule: the client may lag the server, but must never lead it. Pin # them together anyway — the `test` extra of each pysail release states the # client it was built against, and matching it is free. # # The client version therefore lives HERE, once. It used to be copied into five # groups, which meant "one number" was a claim maintained by hand across five # edit sites; the groups below include this one instead. It is also the leanest # thing that makes `ty` able to see pyspark — no compiled extras — which is why # the lint job installs exactly this and nothing more. spark-client = [ "pyspark-client==4.2.0", ] sail = [ "pysail==0.7.1", { include-group = "spark-client" }, ] # The Sail statement agent also runs Delta maintenance (OPTIMIZE/VACUUM/CDF) # through delta-rs, since Sail's planner has no such commands — so the runtime # that hosts the agent needs both the Connect client and deltalake. This IS the # spark-agent image's group; the engine matrix's "Sail + delta-rs" column builds # from it too, so that column measures the shipped runtime rather than a # lookalike. requests for the same reason spark-connect carries it: user # notebook code calls HTTP APIs, and the agent is where that code runs. sail-delta = [ { include-group = "spark-client" }, "deltalake>=0.23,<2", "pandas>=2,<3", "pyarrow>=23.0.1,<26", "requests>=2.32,<3", # OSS format("kafka") on Sail: the agent consumes and createDataFrames # Kafka-schema rows into the engine. JVM keeps spark-sql-kafka. "kafka-python>=2.0.2,<4", # JKS/P12 → PEM so kafka-python can honour Spark's Java truststore options. "pyjks>=20,<22", "cryptography>=42,<51", # GSSAPI handshake: kafka-python imports this at connect time. "gssapi>=1.8,<2", ] # dbt on the statement agent, for databricks-emulator's `dbt_task`. # # A SEPARATE GROUP, not folded into sail-delta, because sail-delta also builds # the engine matrix's "Sail + delta-rs" probe and that column measures the # shipped runtime: adding an adapter it never calls would make the probe a # lookalike of the agent rather than the agent. # # Why a DATABRICKS adapter ships from this repository: the agent image is the # family's, not Fabric's. databricks-emulator terminates `dbt_task` and hands # the project here, because dbt is an ordinary warehouse client and running it # as a job changes who invokes it, not what it connects to. Without this the # task fails naming this image, which is honest and still unusable. spark-agent-dbt = [ { include-group = "sail-delta" }, "dbt-databricks==1.12.5", # The engine, in the AGENT image. Fabric starts a Spark session per notebook # and shares one only within a single-user boundary, so the agent needs to # be able to start an engine per user rather than route every caller through # one shared server (docs/54). Same pin as the `sail` group: two Sails of # different versions in one stack would make a parity result depend on which # one answered. "pysail==0.7.1", ] # requests: user notebook code (and libraries it imports) routinely calls HTTP # APIs, and the statement agent is where that code executes. # JupyterLab for the `jupyter` compose profile: a REAL notebook editor shipped # rather than one built into the portal (docs/44). pyspark-client is pinned to # the SAME version the agent uses (see the `sail` group above, which explains # why that number is really one number shared with the server) — a kernel on a # different client than the engine is exactly the drift this profile exists to # avoid. Stated as "the same as the agent" rather than repeating the digits, # because a restated version is one someone has to remember to update twice. jupyter = [ "jupyterlab>=4.2,<5", { include-group = "spark-client" }, "deltalake>=0.23,<2", "pyarrow>=23.0.1,<26", "requests>=2.32,<3", ] spark-connect = [ { include-group = "spark-client" }, "requests>=2.32,<3", "kafka-python>=2.0.2,<4", # JKS/P12 → PEM so kafka-python can honour Spark's Java truststore options. "pyjks>=20,<22", "cryptography>=42,<51", # GSSAPI handshake: kafka-python imports this at connect time. "gssapi>=1.8,<2", ] # The official Databricks SDK, as the third-party witness for the family chain: # fabric submits a Databricks activity to databricks-emulator, and this reads # back from the workspace side that a job and a run really exist. A real client # with its own expectations, not a generic HTTP call — if our submission shape # were wrong, the SDK itself would fail to parse the workspace's answer. databricks-sdk = [ "databricks-sdk>=0.130,<1", "requests>=2.32,<3", ] dbt-fabric = ["dbt-fabric>=1.11"] dbt-fabricspark = ["dbt-fabricspark>=1.13"] dbt-duckdb = [ "dbt-duckdb>=1.9,<2", "deltalake>=0.23,<2", "pyarrow>=23.0.1,<26", ] # FLOOR AT 3.15, not 3.1. `>=3.1` let a "bump the python group" PR RESOLVE # DOWNWARDS on this runtime: main locked one mlflow 3.15.1, and the update # split it into 3.2.0 for python < 3.14 and 3.15.1 above, so the 3.12 image # quietly took a thirteen-minor-version downgrade. It surfaced as # `mlflow server: No such option '--disable-security-middleware'`, a flag # e2e/data-science-loop passes, so the container never started. # # A range whose lower bound is far below what we run is not a range, it is a # licence for a resolver to move backwards, and a dependency bump is the last # place anyone looks for a downgrade. mlflow = ["mlflow>=3.15,<4"] rti = [ "azure-kusto-data>=5,<7", "requests>=2.32,<3", ] data-science-loop = [ "dbt-duckdb>=1.9,<2", "deltalake>=0.23,<2", "mlflow>=3.15,<4", # see the `mlflow` group above for why the floor is 3.15 "pyarrow>=23.0.1,<26", { include-group = "spark-client" }, ] # NOTE: examples/ deliberately do NOT appear here. Each example owns its # pyproject.toml + uv.lock so it can be copied out and run standalone, and so # its dependencies never enter the emulator's own graph. See examples/medallion. [tool.uv] package = false # sqlparse reaches this lock ONLY through dbt-core, which pins `sqlparse<0.6.0` # in every release it has published — 1.11.13 through 1.12.2, checked against # PyPI, so there is no version of dbt to upgrade INTO that takes the patched # sqlparse. Four advisories (three DoS, one escaping) name 0.6.0 as the first # patched version, so the choice is this override or living with them. # # Overriding a maintainer's upper bound is a deliberate deviation, so it was # measured rather than assumed. dbt-core touches sqlparse in exactly one place # (`dbt/compilation.py`: `parse`, `sql.Token`, `tokens`, `lexer.Lexer`, and the # `engine.grouping.MAX_GROUPING_*` caps). Running dbt's own `inject_ctes_into_sql` # over four SQL shapes under 0.5.5 and 0.6.0 returns BYTE-IDENTICAL output, and # both grouping caps still exist in 0.6.0 — so the pin is upstream caution # about a minor bump, not a known incompatibility with the surface dbt uses. # # Re-check when dbt-core relaxes the bound: this override should then be # DELETED rather than left to pin the ecosystem quietly. override-dependencies = ["sqlparse>=0.6.0"] conflicts = [ [ { group = "dbt-fabric" }, { group = "dbt-fabricspark" }, ], # dbt-databricks 1.12.4 pins databricks-sdk<0.118; this repository's own # databricks-sdk group wants >=0.130 for the Databricks-activity e2e. Both # are correct and they are never installed together: the agent image takes # spark-agent-dbt, that e2e takes databricks-sdk. Declared rather than # resolved by loosening one, which would silently move the version a # witness runs against. [ { group = "spark-agent-dbt" }, { group = "databricks-sdk" }, ], ] [tool.uv.sources] fabric-emulator-notebookutils = { workspace = true } fabric-target = { workspace = true } [tool.uv.workspace] members = ["python", "python/fabric-target"] [tool.pytest.ini_options] testpaths = ["python/tests", "python/fabric-target/tests"] markers = [ "target: dual-target conformance runs against fabric-emulator or real Fabric per FABRIC_TARGET", "cases(suite): parametrize the test's `case` argument from cases/.json (python/tests/conftest.py)", ] # Coverage flags are deliberately NOT in addopts. addopts applies to EVERY # pytest invocation in the repo, including the ones e2e harnesses run inside # containers that have pytest but not pytest-cov — where the flags are simply # unrecognised arguments and the suite dies with exit 4. The unit suite passes # them explicitly instead; see docs/10-testing.md. # Coverage is scoped to the code THIS suite owns, not to the whole tree. # # The modules omitted below are not untested — they are witnessed by the e2e # jobs in CI, which run them inside the Spark image or against a live stack and # cannot report statement coverage back here. Including them would average a # real number against a structural zero, and a floor computed from that gate # nothing: the four invariant checkers could lose every test and the total would # barely move. docs/witnesses.json is where their proof is recorded. [tool.coverage.run] # notebookutils is SHIPPED code — the shim a consumer's notebook imports — and # it was outside this list entirely, so the module with the most user-facing # surface in the repo contributed nothing to the gate. Its low-coverage members # (fs, _http) are witnessed by e2e rather than unit tests, the same way the # omitted spark_agent files are. # e2e/conformance carries the NORMATIVE logic of the conformance kit — # probes.py is where every contract in docs/38 is actually decided, and it was # outside this list, so the file the whole matrix rests on contributed nothing # to the gate. Same reasoning that brought notebookutils in. source = ["scripts", "python/spark_agent", "python/notebookutils", "docker/sail", "e2e/conformance"] omit = [ # Run inside the Spark image, witnessed by e2e/notebook-run and e2e/sail. # # `input_file.py` used to be here and came OFF when it acquired tests. It is # not process glue whose behaviour is only visible end to end: it is a # semantic shim that decides what a notebook SEES, and it shipped a defect # in three releases that no gate could have caught while it was omitted. # `files_mount.py` left for the same reason: write-back and the conflicting # bind are the notebook's filesystem, not HTTP glue. "python/spark_agent/agent.py", # CI harnesses and release tooling: containerized or one-shot, and each # already named as the witness for the claim it proves. "scripts/govern_ingest.py", "scripts/spark_check.py", "scripts/build_fixture_wheels.py", # An operator tool: it runs only from `workflow_dispatch` against a real # tenant, so its HTTP shell (token, workspace and item resolution, # getDefinition) cannot be exercised here. Its one load-bearing part — the # redaction that keeps a private tenant's contents out of a PUBLIC Actions # log — is NOT unwitnessed: scripts/check_capture_redaction.py asserts it in # `make check`, and python/tests/test_check_capture_redaction.py drives the # same cases plus the two mutations (a leaky redactor, an over-zealous one). # Fetch-and-write with a truncation guard, and nothing else: the ADF # schema is one file at one commit, so there is no parsing to get wrong — # unlike vendor_fabric_activity_types.py, whose HTML extraction IS the # load-bearing part and is now tested (test_vendor_fabric_activity_types.py) # rather than exempted. Omitted for what it is, not for convenience. # Needs pysail AND a running Sail server, so it cannot execute in the unit # suite at all — it is a MEASUREMENT tool, run by hand on a dependency bump # to re-test the claim delta_ops.py's MERGE intercept rests on. Its output # is a verdict for a human, not a code path anything imports. "scripts/probe_sail_merge_premise.py", "scripts/vendor_adf_pipeline_schema.py", "scripts/capture_definition_shape.py", # The conformance harness itself: it publishes notebooks, drives # RunNotebook and reads OneLake, so it cannot run without the compose # stack it exists to drive. Its DECISIONS are not here — probes.py grades # every contract and run.py's verdict parsing was extracted precisely so # both are testable without docker. What is omitted is the plumbing # between them. "e2e/conformance/live.py", "*/tests/*", ] [tool.coverage.report] show_missing = true skip_covered = false # THE FLOOR MUST MEAN WHAT IT PRINTS. Without a precision, coverage compares the # ROUNDED percentage against --cov-fail-under, so 89.88% satisfied a floor of 90 # while pytest-cov printed # # FAIL Required test coverage of 90% not reached. Total coverage: 89.88% # # on a build that exited 0. A FAIL line that does not fail is how people learn # to skim CI output — and the effective floor was 89.5, not 90. Two decimals # makes the comparison exact and the message truthful. precision = 2 # ONE number, here. It used to live in two workflow steps as # `--cov-fail-under=`, which is two things to move for one change and the way a # ratchet quietly stops ratcheting. 92 against a measured 93.03 leaves headroom # for a change that lands genuinely-hard-to-cover code, and no more. fail_under = 92 # --- ruff ------------------------------------------------------------------ # Line length matches what this repo's prose and comments already wrap to. [tool.ruff] line-length = 100 target-version = "py312" # third_party is somebody else's artifact, pinned. Linting it would report # style in code we must not touch — the sha256 in each PROVENANCE.md describes # the file as upstream ships it, and a ruff --fix would break that on the one # field whose whole job is to be exact. extend-exclude = ["e2e/*/target", "website", "third_party"] [tool.ruff.lint] # Deliberately not the kitchen sink. Each family here has caught something in # this codebase or guards a shape it already relies on; a rule that only # produces churn is worse than no rule, because the noise trains people to skim # the output. select = [ "E", "W", # pycodestyle "F", # pyflakes: undefined names, unused imports "I", # import order "UP", # pyupgrade "B", # bugbear: mutable defaults, loop-variable capture "SIM", # simplify "RUF", # ruff's own ] ignore = [ "E501", # line length is enforced by the formatter, not twice "SIM105", # contextlib.suppress reads worse than try/except pass in shims "B008", # call-in-default is how argparse/Path defaults are written here # Blind `except Exception:` is a DELIBERATE pattern here: a best-effort # mount, a cleanup that must not fail a run, an optional import. Each site # already carries a comment saying why, and several carry `# noqa: BLE001` # from before this config existed. Enabling the rule would flag 22 places # that are correct; RUF100 is off for the same reason, so those surviving # directives keep documenting intent instead of becoming lint of their own. "RUF100", ] [tool.ruff.lint.per-file-ignores] # Fabric notebook content. `spark`, `mssparkutils` and `notebookutils` are # injected into the session by the runtime and are deliberately not imported — # that IS the contract this emulator provides (docs/38-framework-conformance). # Flagging them as undefined would be flagging Fabric's own programming model. "**/notebook-content.py" = ["F821"] "e2e/**/*.py" = ["F821"] # Test files import a module by path before asserting on it, so the import is # not at the top and does not need to be. "python/tests/*" = ["E402"] # `python/spark_agent/` IS MOUNTED INTO THE JVM SPARK IMAGE, which ships # **Python 3.8** (`apache/spark:3.5.5-...-python3-ubuntu`). ruff targets py312 # and UP017 rewrites `timezone.utc` to `datetime.UTC` — a 3.11+ alias that # raises ImportError there. That is not hypothetical: it broke the eventstream # JVM suite on main, and only a cron-only workflow could see it. "python/spark_agent/*" = ["UP017"] # The notebookutils shim mirrors Microsoft's API, which is camelCase and takes # arguments this code does not use. Renaming to satisfy a linter would break # the thing it exists to imitate. "python/notebookutils/*" = ["N802", "N803", "ARG001"] # Harness and example code, witnessed by CI actually RUNNING it. These rules # are about long-term maintainability of library code; a probe script that # opens a file without a context manager or reuses a loop variable is read # once, run in one container, and proven by its own e2e. Enforcing them here # produces churn in code whose correctness is established a better way. "e2e/**" = ["B007", "B904", "B905", "E402", "E741", "RUF005", "RUF015", "RUF059", "SIM115", "RUF001"] "examples/**" = ["B007", "B904", "B905", "E402", "E741", "RUF005", "RUF015", "RUF059", "SIM115", "RUF001"] "docker/**" = ["B007", "SIM115", "RUF005"] [tool.ty.src] # third_party is somebody else's artifact, pinned, and it does not type-check: # Microsoft's own notebookutils stub annotates a parameter `parameters: {}`, # which is not a valid type expression. That is a fact about their package, not # a defect in ours, and the sha256 in its PROVENANCE.md describes the file as # they ship it — so it is excluded rather than corrected. ruff excludes it for # the same reason (see [tool.ruff]). exclude = ["third_party"] # PEP 723 uv-script drivers carry their own dependency block and are run with # `uv run --script`. Checking them against the lint/test venv reports # unresolved imports for packages that exist only in that block (this PR: # azure-mgmt-fabric). CI witnesses them by running the harness, not by ty. exclude-scripts = true [tool.ty.environment] root = ["scripts", "python"] [tool.ty.rules] # Unresolved by CONSTRUCTION, not by mistake: # * the checkers and their tests load one another by path via importlib, so # no static checker can follow the module identity; # * notebook content reads `spark`/`mssparkutils` from injected globals, which # is the contract this emulator exists to provide (docs/38); # * the agent's remaining engine-side imports (kafka, gssapi) live only in the # Spark image, because pulling them into a lint venv means compiling a # Kerberos stack (docker/spark-agent/Dockerfile). # # `pyspark` USED to be the headline reason here, and is not any more: the lint # job installs `--group spark-client`, so pyspark resolves and the agent is # actually type-checked. That mattered — while pyspark was unresolved, every # type in ~1800 statements of agent was Unknown, so this ignore was not # narrowing the check, it was standing in for the absence of one. unresolved-import = "ignore" unresolved-attribute = "ignore" unresolved-reference = "ignore" # `BaseHTTPRequestHandler.log_message(self, format, *args)` is typed loosely in # typeshed, and every handler in this repo overrides it to silence request # logging. The override is correct; the signature comparison is not useful. invalid-method-override = "ignore" [[tool.ty.overrides]] # e2e harnesses and examples run against a live stack with pyspark present and # notebook globals injected. They are witnessed by CI actually RUNNING them, # which is a stronger check than inference over code the checker cannot import. # Monkeypatching socket.getaddrinfo to pin a host, for instance, is the point of # the harness and can never match the stdlib signature. # Overrides replace the global rule table rather than merging with it, so the # unresolved-* ignores from [tool.ty.rules] are restated here. include = ["e2e/**", "examples/**"] [tool.ty.overrides.rules] unresolved-import = "ignore" unresolved-attribute = "ignore" unresolved-reference = "ignore" invalid-argument-type = "ignore" unsupported-operator = "ignore" not-subscriptable = "ignore" possibly-missing-submodule = "ignore" invalid-assignment = "ignore" unused-type-ignore-comment = "ignore" [[tool.ty.overrides]] # The checkers parse hand-maintained markdown into loosely-typed dicts, so an # element's type is genuinely unknown until it is used. Narrowing every access # would add casts that say less than the code does. include = ["scripts/**"] [tool.ty.overrides.rules] unsupported-operator = "ignore" invalid-argument-type = "ignore" call-non-callable = "ignore"