# Auto-loaded alongside docker-compose.yml by `docker compose up` (Compose # merges *.override.yml automatically when present — no -f flag needed): adds # REAL engines to the default experience — LakeSail's Sail (Rust Spark-Connect, # no JVM) behind the Livy statement agent — native Livy sessions + notebook # cell execution on a real engine, and a SQL Server sidecar for # the T-SQL/TDS warehouse surface (Warehouse, Lakehouse SQL endpoint, Fabric SQL # Database) — so `docker compose up` gives you the emulator with real compute # attached, not just the contract. See docs/14-real-compute.md. # # Opt out — the lite, contract-only pair (no heavy sidecars, honest 501s on the # Spark/SQL surfaces), by naming the base file explicitly so Compose skips this # override: # docker compose -f docker-compose.yml up # # Local dev-only credentials below (not for anything internet-facing). services: # One long-lived SparkSession, a REPL over HTTP — the emulator terminates the # Livy REST protocol itself and drives this agent (no Apache Livy server; it's # retired). See python/spark_agent/agent.py. # LakeSail's Sail: a Rust Spark-Connect server — the engine behind the agent. # No JVM anywhere; PySpark clients can also connect directly at # sc://localhost:50051. See docs/20-lakesail-engine.md. sail: # image + build: `docker compose pull` fetches the published sidecar (so a # pinned SAIL_VERSION is reproducible), while `docker compose build` still # works from source for local development. Defaults to latest, like the rest. image: ghcr.io/calvinchengx/emulator-sail:${SAIL_VERSION:-latest} build: context: . dockerfile: docker/sail/Dockerfile depends_on: fabric-emulator: condition: service_healthy ports: - "50051:50051" # Spark Connect (plain h2c gRPC; no auth — local use only) environment: # The launcher mints AZURE_STORAGE_TOKEN from entra before exec'ing sail # (object_store reads credentials from process env at startup only). ENTRA_TOKEN_URL: "https://entra-emulator:8443/6f89cf12-978b-4d23-ac18-9ef0c127cf87/oauth2/v2.0/token" ENTRA_CLIENT_ID: "00d88624-f0d7-46f6-a641-6232c2608928" ENTRA_CLIENT_SECRET: "daemon-app-secret" # az://{workspace}/{item}/… → the emulator's account-prefixed Blob path. AZURE_STORAGE_ACCOUNT_NAME: "onelake" AZURE_STORAGE_ENDPOINT: "https://fabric-emulator:9443/onelake" AZURE_ALLOW_INVALID_CERTIFICATES: "true" # emulator's self-signed TLS SAIL_SPARK__SESSION_TIMEOUT_SECS: "3600" RUST_LOG: "info" # Without this the container reports `running` forever whether or not the # engine ever bound its port, so nothing downstream can wait on it. Plain # TCP because the surface is h2c gRPC and the image carries no gRPC client; # proving the port listens is the honest scope of a healthcheck. The deeper # "does Spark compute" question is `scripts/status.sh --spark`. healthcheck: test: ["CMD", "python", "-c", "import socket; socket.create_connection(('127.0.0.1', 50051), 2).close()"] interval: 5s timeout: 3s retries: 20 spark-agent: image: ghcr.io/calvinchengx/emulator-spark-agent:${SPARK_AGENT_VERSION:-latest} build: context: . dockerfile: docker/spark-agent/Dockerfile depends_on: sail: # Now that sail reports health, wait for the engine to actually listen # rather than merely be started: the agent holds a Spark Connect client. condition: service_healthy # NO BIND MOUNT. The agent is baked into the image (python/spark_agent/), # so this file configures the container rather than completing it — which # is what a consumer of the published tag gets too. environment: # THE SHIM NEEDS ENDPOINTS. Its default is 127.0.0.1:19080, which inside # this container is nothing, so `notebookutils.fs` raised `Connection # refused` from a RunNotebook cell while working fine in a Jupyter one — # the jupyter service in docker-compose.yml has carried this wiring all # along and the agent never did, though that service's own comment claims # "a cell here and a cell in a RunNotebook job execute identically". # # Found by the framework-conformance contract-1 probe, which was the first # thing that ever called notebookutils from inside a notebook job. # # NOT workspace/lakehouse ids: those are the environment fallback, and a # runtime that answers through it can hide two broken control-plane links # (docs/38 §1). The agent binds the real context per statement. NOTEBOOKUTILS_FABRIC_URL: "http://fabric-emulator" NOTEBOOKUTILS_ENTRA_URL: "http://entra-emulator:8443" NOTEBOOKUTILS_INSECURE: "1" SPARK_REMOTE: "sc://sail:50051" # Spark Connect client — no JVM # The delta-rs path talks to OneLake directly, so it needs its own # credentials — a Connect client cannot read back the bearer sail holds. # Same issuer, same Storage audience, so it acts as the same principal. ENTRA_TOKEN_URL: "https://entra-emulator:8443/6f89cf12-978b-4d23-ac18-9ef0c127cf87/oauth2/v2.0/token" # Notebook attribution for the delta-rs path: Rust object_store cannot set # request headers, so the agent mints a bearer carrying fabric_job_id / # fabric_cell_index as extra claims and the emulator reads them off the # validated principal. See internal/onelake/observe.go. ENTRA_FORGE_URL: "https://entra-emulator:8443/admin/api/tokens" ENTRA_CLIENT_ID: "00d88624-f0d7-46f6-a641-6232c2608928" ENTRA_CLIENT_SECRET: "daemon-app-secret" # Eventstream adapter: mint a Fabric-audience token and resolve / consume. FABRIC_API_URL: "https://fabric-emulator:9443" ENTRA_FABRIC_SCOPE: "https://api.fabric.microsoft.com/.default" AZURE_STORAGE_ACCOUNT_NAME: "onelake" AZURE_STORAGE_ENDPOINT: "https://fabric-emulator:9443/onelake" AZURE_ALLOW_INVALID_CERTIFICATES: "true" # emulator's self-signed TLS # No `command:` either — the image's CMD is the agent. # # The agent already serves /health (python/spark_agent/agent.py); it just # was not wired up, so `running` was never evidence the statement executor # could answer. healthcheck: test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8099/health', timeout=3)"] interval: 5s timeout: 5s retries: 20 sqlserver: # Digest-pinned for the same reason as kustainer: this sidecar IS the warehouse # engine, so an unannounced change moves T-SQL behaviour under the tests. Refresh: # docker buildx imagetools inspect mcr.microsoft.com/mssql/server:2022-latest image: mcr.microsoft.com/mssql/server:2022-latest@sha256:ba4c8329f48fb8f02e1416be6a930ebfd71268caee78aa985f3af4315e457c89 # amd64 only — Microsoft publishes no arm64 manifest for mssql/server. On an # Apple-silicon host the engine translates this one container (Rosetta); # stating the platform beats relying on per-engine fallback, which can also # fail outright with "no matching manifest for linux/arm64/v8". Everything # else in this stack has a native arm64 build, so nothing else pays for it. platform: linux/amd64 environment: ACCEPT_EULA: "Y" MSSQL_SA_PASSWORD: "Str0ng!Passw0rd" # Bound the engine's memory. UNCAPPED, SQL Server sizes its buffer pool from # what it can see — 12.8 GB of a 16 GB host — and grew to ~1.8 GB RSS under a # medallion run; on a bigger dev box it takes proportionally more. This caps # what it believes it has, which is the knob that matters: # # uncapped physical 12832 MB, committed target 1490 MB, ~1.8 GB busy # 2048 physical 2048 MB, committed target ~1.2 GB, ~0.9 GB busy # # 2048 and not lower because the engine FLOORS it there: MSSQL_MEMORY_LIMIT_MB=1024 # still reports 2000 MB, so a smaller number would only mislead the next reader. # Note this does NOT move `max server memory` in sys.configurations, which stays # at its 2147483647 sentinel — checking that setting makes the cap look inert. MSSQL_MEMORY_LIMIT_MB: "2048" healthcheck: test: - CMD - /opt/mssql-tools18/bin/sqlcmd - -S - localhost - -U - sa - -P - "Str0ng!Passw0rd" - -C - -Q - "SELECT 1" interval: 5s timeout: 5s retries: 40 start_period: 20s fabric-emulator: environment: # Native Livy: terminate the protocol and drive the agent above — real # interactive sessions, notebook cell execution, high-concurrency REPLs. FABRIC_SPARK_AGENT_URL: "http://spark-agent:8099" # The T-SQL/TDS warehouse surface, relaying to the SQL Server sidecar # above. Generous dial/connection timeouts: the backend dials lazily on # first query, and the sidecar (esp. an amd64 image emulated on arm64) # can take a while to finish booting. FABRIC_SQL_TDS_ADDR: ":1433" FABRIC_WAREHOUSE_SQL_URL: "server=sqlserver;user id=sa;password=Str0ng!Passw0rd;encrypt=disable;dial timeout=90;connection timeout=90" # Restate the base port mapping alongside the new one. Compose APPENDS # `ports` across files and de-duplicates identical entries — it does not # replace the list. Restating 9443:9443 is therefore harmless (the # duplicate collapses) and the merged result is exactly the two below, # which is what this file wants. # # The distinction matters when you want a DIFFERENT host port rather than # an extra one. Listing "19543:9443" here would not move the published # port; it would publish BOTH, and the original still collides with # whatever already holds it. Replacing the list needs Compose's explicit # merge control: # # ports: !override ["19543:9443"] # # Measured against these files, not assumed: base alone resolves to # [9443->9443], base+this to [9443->9443, 1433->1433]. ports: - "9443:9443" # api.fabric + onelake.dfs + portal (host-routed) - "1433:1433" # warehouse T-SQL/TDS (FedAuth; sqlcmd/pyodbc/dbt-fabric point here)