x-app-defaults: host_persistence_path: &hostPersistencePath ${PWD}/olake-data amoro_log_persistence_path: &amoroLogPersistencePath ${PWD}/amoro-logs worker_config_volume_details: &workerConfigVolumeDetails type: bind source: *hostPersistencePath target: /tmp/olake-config amoro_log_volume_details: &amoroLogVolumeDetails type: bind source: *amoroLogPersistencePath target: /mnt/amoro-logs x-signup-defaults: username: &defaultUsername "admin" password: &defaultPassword "password" email: &defaultEmail "admin@example.com" x-encryption: key: &encryptionKey "" x-images: curl: &curlImage ${CONTAINER_REGISTRY_BASE:-registry-1.docker.io}/curlimages/curl:8.21.0 x-envs: shared: &sharedEnvs CONTAINER_REGISTRY_BASE: ${CONTAINER_REGISTRY_BASE:-registry-1.docker.io} OLAKE_SECRET_KEY: *encryptionKey PERSISTENT_DIR: *hostPersistencePath ENABLE_OPTIMIZATION: ${ENABLE_OPTIMIZATION:-false} # S3 log file storage — uncomment and fill values below to enable # OLAKE_STORAGE_MODE: s3 # OLAKE_S3_BUCKET: "" # OLAKE_S3_REGION: "" # OLAKE_S3_PREFIX: "" # optional key prefix # OLAKE_S3_ENDPOINT: "" # required for non-AWS S3 (MinIO, GCS S3 API, etc.) # OLAKE_S3_ACCESS_KEY_ID: "" # OLAKE_S3_SECRET_ACCESS_KEY: "" services: olake-ui: image: ${CONTAINER_REGISTRY_BASE:-registry-1.docker.io}/olakego/ui:latest pull_policy: always container_name: olake-ui environment: <<: *sharedEnvs # Request body size limits (64 MB in bytes) MAX_MEMORY: 67108864 MAX_UPLOAD_SIZE: 67108864 HTTP_READ_HEADER_TIMEOUT: 10s HTTP_READ_TIMEOUT: 600s HTTP_WRITE_TIMEOUT: 600s HTTP_IDLE_TIMEOUT: 600s USERNAME: *defaultUsername PASSWORD: *defaultPassword OPTIMIZATION_BASE_URL: http://fusion.olake.internal:1630 OPTIMIZATION_GROUP: spark-container ports: - "8000:8000" # Single port: Go backend serves both API and frontend volumes: # Mount Docker socket to enable black box environment by managing local containers/images - /var/run/docker.sock:/var/run/docker.sock - <<: *workerConfigVolumeDetails networks: - olake-network depends_on: postgresql: condition: service_healthy # Wait for postgres to be healthy temporal: condition: service_healthy temporal-create-namespace: condition: service_completed_successfully fusion-init: condition: service_completed_successfully # Wait for initial Fusion setup to complete required: false restart: unless-stopped healthcheck: # Updated healthcheck for olake-ui test: ["CMD-SHELL", "nc -z localhost 8000"] # Check if port 8000 is listening interval: 15s timeout: 5s retries: 5 start_period: 30s # Give it time to start up before first health check signup-init: image: *curlImage container_name: olake-signup-init networks: - olake-network depends_on: olake-ui: condition: service_healthy # Wait for olake-ui to be healthy environment: USERNAME: *defaultUsername PASSWORD: *defaultPassword EMAIL: *defaultEmail OLAKE_APP_URL: "http://olake-ui:8000/signup" command: > sh -c " echo 'signup-init: Initializing user setup...' # The depends_on condition: service_healthy should handle the waiting for olake-ui. JSON_PAYLOAD=$$(printf '{\"username\":\"%s\",\"password\":\"%s\",\"email\":\"%s\"}' \"$${USERNAME}\" \"$${PASSWORD}\" \"$${EMAIL}\") echo \"signup-init: Attempting to create user '$${USERNAME}' via $${OLAKE_APP_URL}\" HTTP_RESPONSE_CODE=$$(/usr/bin/curl -s -o /dev/stderr -w '%{http_code}' -X POST -H 'Content-Type: application/json' -d \"$${JSON_PAYLOAD}\" \"$${OLAKE_APP_URL}\") # The actual response body from olake-ui will be printed to stderr by the '-o /dev/stderr' curl option. # A newline after stderr output from curl can make logs cleaner. echo '' if ! [ \"$${HTTP_RESPONSE_CODE}\" -eq \"$${HTTP_RESPONSE_CODE}\" ] 2>/dev/null; then echo \"signup-init: ERROR - HTTP_RESPONSE_CODE is not a number: '$${HTTP_RESPONSE_CODE}'\" exit 1; fi if [ \"$${HTTP_RESPONSE_CODE}\" -ge 200 ] && [ \"$${HTTP_RESPONSE_CODE}\" -lt 300 ]; then echo \"signup-init: User '$${USERNAME}' creation request successful (HTTP $${HTTP_RESPONSE_CODE}).\"; else echo \"signup-init: User '$${USERNAME}' creation request FAILED (HTTP $${HTTP_RESPONSE_CODE}). Server response body above.\"; exit 1; # Exit with error if signup failed fi echo 'signup-init: User setup process complete.'; " restart: "no" temporal-worker: image: ${CONTAINER_REGISTRY_BASE:-registry-1.docker.io}/olakego/ui-worker:latest pull_policy: always container_name: olake-temporal-worker networks: - olake-network volumes: - /var/run/docker.sock:/var/run/docker.sock - <<: *workerConfigVolumeDetails environment: <<: *sharedEnvs OLAKE_CALLBACK_URL: "http://olake-ui:8000/internal/worker/callback" # worker bind-mounts host ./olake-data/index-db at this path in the driver container OLAKE_INDEX_DB_DIR: /index-db OLAKE_INDEX_DB_CACHE_SIZE: 512 # in mb OLAKE_INDEX_DB_MAX_OPEN_FILES: 1000 # max open files depends_on: temporal: condition: service_healthy temporal-create-namespace: condition: service_completed_successfully olake-ui: condition: service_healthy restart: unless-stopped postgresql: container_name: temporal-postgresql image: ${CONTAINER_REGISTRY_BASE:-registry-1.docker.io}/library/postgres:13 environment: POSTGRES_USER: temporal POSTGRES_PASSWORD: temporal networks: - olake-network volumes: - temporal-postgresql-data:/var/lib/postgresql/data restart: unless-stopped healthcheck: test: ["CMD-SHELL", "pg_isready -U temporal -h localhost -p 5432"] # Checks if server is accepting connections interval: 10s timeout: 5s retries: 5 temporal: container_name: temporal image: ${CONTAINER_REGISTRY_BASE:-registry-1.docker.io}/temporalio/server:1.32.0 depends_on: temporal-schema-migrate: condition: service_completed_successfully environment: - DB=postgres12 - DB_PORT=5432 - POSTGRES_USER=temporal - POSTGRES_PWD=temporal - POSTGRES_SEEDS=postgresql - BIND_ON_IP=0.0.0.0 configs: - source: temporal_dynamicconfig target: /etc/temporal/config/dynamicconfig/docker.yaml mode: 0444 networks: - olake-network restart: unless-stopped healthcheck: test: ["CMD-SHELL", "nc -z 127.0.0.1 7233"] interval: 5s timeout: 3s retries: 30 start_period: 20s temporal-schema-migrate: container_name: temporal-schema-migrate image: ${CONTAINER_REGISTRY_BASE:-registry-1.docker.io}/temporalio/admin-tools:1.32.0 depends_on: postgresql: condition: service_healthy networks: - olake-network environment: - DB=postgres12 - DB_PORT=5432 - POSTGRES_USER=temporal - POSTGRES_PWD=temporal - POSTGRES_SEEDS=postgresql - SQL_PASSWORD=temporal configs: - source: temporal_migrate_script target: /scripts/setup-postgres.sh mode: 0755 entrypoint: ["/bin/sh"] command: /scripts/setup-postgres.sh restart: on-failure:6 temporal-create-namespace: container_name: temporal-create-namespace image: ${CONTAINER_REGISTRY_BASE:-registry-1.docker.io}/temporalio/admin-tools:1.32.0 depends_on: temporal: condition: service_healthy networks: - olake-network configs: - source: temporal_create_namespace_script target: /scripts/create-namespace.sh mode: 0755 entrypoint: ["/bin/sh"] command: /scripts/create-namespace.sh restart: on-failure:5 temporal-ui: container_name: temporal-ui image: ${CONTAINER_REGISTRY_BASE:-registry-1.docker.io}/temporalio/ui:2.54.1 profiles: - debug depends_on: temporal: condition: service_healthy environment: - TEMPORAL_ADDRESS=temporal:7233 networks: - olake-network ports: - "8081:8080" restart: unless-stopped fusion-db-init: image: ${CONTAINER_REGISTRY_BASE:-registry-1.docker.io}/library/postgres:13 container_name: fusion-db-init profiles: - fusion depends_on: postgresql: condition: service_healthy networks: - olake-network environment: <<: *sharedEnvs PGPASSWORD: temporal volumes: - fusion-config:/fusion-config entrypoint: - /bin/sh - -c - | set -e echo "==> Creating Fusion database (idempotent)..." psql -h postgresql -U temporal -tc \ "SELECT 1 FROM pg_database WHERE datname = 'fusion'" \ | grep -q 1 \ || psql -h postgresql -U temporal -c \ "CREATE DATABASE fusion;" inject_pod_envs() { src="$1" dst="$2" awk ' BEGIN { for (k in ENVIRON) { v = ENVIRON[k] gsub(/"/, "\\\"", v) env_lines = env_lines sprintf(" spark-conf.spark.kubernetes.driverEnv.%s: \"%s\"\n", k, v) env_lines = env_lines sprintf(" spark-conf.spark.executorEnv.%s: \"%s\"\n", k, v) } } /SPARK_POD_ENVS_PLACEHOLDER/ { printf "%s", env_lines; next } { print } ' "$$src" > "$$dst" } echo "==> Installing curl..." apt-get update -qq && apt-get install -y -qq curl ca-certificates --no-install-recommends >/dev/null 2>&1 echo "==> Fetching Fusion config files from GitHub..." BASE="https://raw.githubusercontent.com/datazip-inc/olake-ui/master/optimization" curl -fsSL "$${BASE}/config.yaml" -o /tmp/config.yaml curl -fsSL "$${BASE}/kind-config.yaml" -o /fusion-config/kind-config.yaml curl -fsSL "$${BASE}/spark-rbac.yaml" -o /fusion-config/spark-rbac.yaml inject_pod_envs /tmp/config.yaml /fusion-config/config.yaml echo "==> fusion-db-init complete." restart: "no" fusion: image: olakego/fusion:latest container_name: olake-fusion profiles: - fusion depends_on: fusion-db-init: condition: service_completed_successfully kind-setup: condition: service_completed_successfully spark-copy: condition: service_completed_successfully networks: olake-network: aliases: - fusion.olake.internal ports: - "1630:1630" # Web UI + REST API - "1260:1260" # Thrift table service - "1261:1261" # Thrift optimizing service environment: <<: *sharedEnvs AMORO_LOG_DIR: /mnt/amoro-logs JVM_XMS: "1024" JVM_XMX: "2048" KUBECONFIG: /root/.kube/config LOG_DIR: /mnt/amoro-logs/compaction volumes: - fusion-config:/fusion-config:ro - kind-kubeconfig:/root/.kube:ro - spark-home:/opt/spark:ro - <<: *amoroLogVolumeDetails entrypoint: - /bin/sh - -c - | cp /fusion-config/config.yaml /usr/local/amoro/conf/config.yaml exec /entrypoint.sh ams tty: true stdin_open: true fusion-init: image: *curlImage container_name: fusion-init profiles: - fusion depends_on: fusion: condition: service_started kind-load-image: condition: service_completed_successfully networks: - olake-network entrypoint: - /bin/sh - -c - | echo "Waiting for Fusion to be ready..." until curl -sf http://fusion:1630/ >/dev/null 2>&1; do sleep 2; done sleep 10 echo "Logging in..." curl -sf -c /tmp/cookies.txt \ -X POST http://fusion:1630/api/ams/v1/login \ -H "Content-Type: application/json" \ -H "X-Request-Source: Web" \ -d '{"user":"admin","password":"password"}' echo "" # ---- Optimizer Group (Spark on Kubernetes) ---- echo "Checking optimizer group..." GROUPS=$$(curl -s -b /tmp/cookies.txt \ -H "X-Request-Source: Web" \ http://fusion:1630/api/ams/v1/optimize/resourceGroups) if echo "$$GROUPS" | grep -q '"name":"spark-container"'; then echo " Optimizer group 'spark-container' already exists — skipping." else echo " Creating optimizer group 'spark-container' (Spark on K8s)..." curl -s -b /tmp/cookies.txt \ -X POST http://fusion:1630/api/ams/v1/optimize/resourceGroups \ -H "Content-Type: application/json" \ -H "X-Request-Source: Web" \ -d '{"name":"spark-container","container":"sparkContainer","properties":{}}' echo "" sleep 5 fi # ---- Optimizer (always ensure at least 1 is running) ---- echo "Checking optimizers..." OPTIMIZERS=$$(curl -s -b /tmp/cookies.txt \ -H "X-Request-Source: Web" \ "http://fusion:1630/api/ams/v1/optimize/optimizerGroups/spark-container/optimizers?page=1&pageSize=10") if echo "$$OPTIMIZERS" | grep -q '"jobStatus":"RUNNING"'; then echo " Spark optimizer already running — skipping." else echo " Scaling out Spark optimizer (parallelism=1)..." curl -s -b /tmp/cookies.txt \ -X POST http://fusion:1630/api/ams/v1/optimize/optimizerGroups/spark-container/optimizers \ -H "Content-Type: application/json" \ -H "X-Request-Source: Web" \ -d '{"parallelism":1}' echo "" sleep 5 fi echo "" echo "============================================" echo " Fusion is ready!" echo " Web UI : http://localhost:1630" echo " Login : admin / password" echo "============================================" kind-setup: image: docker:27-cli container_name: fusion-kind-setup profiles: - fusion depends_on: fusion-db-init: condition: service_completed_successfully networks: - olake-network environment: AMORO_LOG_DIR: *amoroLogPersistencePath volumes: - /var/run/docker.sock:/var/run/docker.sock - kind-kubeconfig:/output - fusion-config:/fusion-config:ro entrypoint: - /bin/sh - -c - | set -e LOG_DIR="$${AMORO_LOG_DIR}" echo "==> Log directory: $$LOG_DIR" echo "==> Generating kind cluster config..." sed "s|AMORO_LOG_PLACEHOLDER|$$LOG_DIR|g" /fusion-config/kind-config.yaml > /tmp/kind-config-gen.yaml echo "==> Installing kind..." apk add --no-cache curl >/dev/null 2>&1 curl -sLo /usr/local/bin/kind https://kind.sigs.k8s.io/dl/v0.23.0/kind-linux-amd64 chmod +x /usr/local/bin/kind echo "==> Creating Kind cluster (fusion-cluster)..." if kind get clusters 2>/dev/null | grep -q '^fusion-cluster$$'; then echo " Cluster already exists, skipping creation." else kind create cluster --config /tmp/kind-config-gen.yaml --wait 120s fi echo "==> Connecting Kind nodes to olake-network..." docker network connect olake-network fusion-cluster-control-plane 2>/dev/null || true for w in $$(docker ps --filter "name=fusion-cluster-worker" --format "{{.Names}}"); do docker network connect olake-network "$$w" 2>/dev/null || true done echo "==> Applying Spark RBAC (namespace + service account)..." cat /fusion-config/spark-rbac.yaml | docker exec -i fusion-cluster-control-plane \ kubectl --kubeconfig /etc/kubernetes/admin.conf apply -f - echo "==> Generating kubeconfig for Fusion..." kind get kubeconfig --name fusion-cluster \ | sed 's|server: .*|server: https://fusion-cluster-control-plane:6443|' \ > /output/config chmod 644 /output/config echo "==> Kind setup complete!" kind-load-image: image: docker:27-cli container_name: fusion-kind-load-image profiles: - fusion depends_on: kind-setup: condition: service_completed_successfully volumes: - /var/run/docker.sock:/var/run/docker.sock tmpfs: - /tmp:size=3G entrypoint: - /bin/sh - -c - | set -e IMAGE="olakego/fusion-spark:latest" IMAGE_REF="docker.io/olakego/fusion-spark:latest" if ! docker image inspect $$IMAGE >/dev/null 2>&1; then echo "==> Image $$IMAGE not found locally — skipping." exit 0 fi # ---- Collect worker nodes ---- NODES=$$(docker ps --filter "name=fusion-cluster-worker" --format "{{.Names}}") if [ -z "$$NODES" ]; then echo "==> No worker nodes found — skipping." exit 0 fi # ---- Always import to refresh mutable tags like ":latest" ---- # This prevents stale runtime images (e.g. old Java version) from being reused. echo "==> Saving $$IMAGE to tarball (tmpfs)..." docker save $$IMAGE -o /tmp/spark-optimizer.tar echo " Saved ($$(du -h /tmp/spark-optimizer.tar | cut -f1))." # ---- Parallel import into all workers ---- echo "==> Importing into Kind worker nodes (parallel, force refresh)..." PIDS="" for NODE in $$NODES; do echo " Starting import into $$NODE..." (cat /tmp/spark-optimizer.tar | docker exec -i $$NODE ctr --namespace=k8s.io images import - && \ echo " $$NODE: done.") & PIDS="$$PIDS $$!" done FAIL=0 for PID in $$PIDS; do if ! wait $$PID; then FAIL=1 fi done rm -f /tmp/spark-optimizer.tar if [ $$FAIL -ne 0 ]; then echo "==> WARNING: Some imports failed!" exit 1 fi echo "==> Done — image available on all workers." spark-copy: image: olakego/fusion-spark:latest container_name: fusion-spark-copy profiles: - fusion entrypoint: - /bin/sh - -c - | if [ -f /spark-vol/.done ]; then echo "Spark already copied — skipping." else echo "Copying Spark to shared volume..." cp -a /opt/spark/. /spark-vol/ touch /spark-vol/.done echo "Spark copy complete." fi volumes: - spark-home:/spark-vol networks: olake-network: driver: bridge name: olake-network volumes: temporal-postgresql-data: driver: local olake-config-data: driver: local kind-kubeconfig: spark-home: fusion-config: configs: temporal_dynamicconfig: content: | # empty dynamic config temporal_migrate_script: content: | #!/bin/sh set -eu : "$${POSTGRES_SEEDS:?ERROR: POSTGRES_SEEDS environment variable is required}" : "$${POSTGRES_USER:?ERROR: POSTGRES_USER environment variable is required}" sqltool() { temporal-sql-tool \ --plugin postgres12 \ --ep "$${POSTGRES_SEEDS}" \ -u "$${POSTGRES_USER}" \ -p "$${DB_PORT:-5432}" \ "$@" } echo 'Starting PostgreSQL schema setup...' echo 'Waiting for PostgreSQL port to be available...' nc -z -w 10 "$${POSTGRES_SEEDS}" "$${DB_PORT:-5432}" echo 'PostgreSQL port is available' # create and setup-schema error if the DB/schema already exists (existing OLake volumes). # update-schema applies pending versions and must succeed. echo 'Setting up temporal database...' sqltool --db temporal create \ || echo ' (create skipped: database already exists)' sqltool --db temporal setup-schema -v 0.0 \ || echo ' (setup-schema skipped: already initialized)' sqltool --db temporal update-schema \ -d /etc/temporal/schema/postgresql/v12/temporal/versioned echo 'Setting up temporal_visibility database...' sqltool --db temporal_visibility create \ || echo ' (create skipped: database already exists)' sqltool --db temporal_visibility setup-schema -v 0.0 \ || echo ' (setup-schema skipped: already initialized)' sqltool --db temporal_visibility update-schema \ -d /etc/temporal/schema/postgresql/v12/visibility/versioned echo 'PostgreSQL schema setup complete' temporal_create_namespace_script: content: | #!/bin/sh set -eu NAMESPACE=$${DEFAULT_NAMESPACE:-default} TEMPORAL_ADDRESS=$${TEMPORAL_ADDRESS:-temporal:7233} MAX_ATTEMPTS=$${TEMPORAL_HEALTH_CHECK_MAX_ATTEMPTS:-30} SLEEP_SECONDS=$${TEMPORAL_HEALTH_CHECK_SLEEP_SECONDS:-5} echo "Creating namespace '$$NAMESPACE'..." attempt=1 while :; do if temporal operator namespace describe -n "$$NAMESPACE" --address "$$TEMPORAL_ADDRESS" >/dev/null 2>&1; then echo "Namespace '$$NAMESPACE' already exists" break fi if temporal operator namespace create -n "$$NAMESPACE" --address "$$TEMPORAL_ADDRESS" >/dev/null 2>&1; then echo "Namespace '$$NAMESPACE' created" break fi if [ "$$attempt" -ge "$$MAX_ATTEMPTS" ]; then echo "Failed to create namespace '$$NAMESPACE' after $$MAX_ATTEMPTS attempts" exit 1 fi echo "Namespace operation not ready yet, waiting... (attempt $$attempt/$$MAX_ATTEMPTS)" attempt=$$((attempt + 1)) sleep "$$SLEEP_SECONDS" done