#!/usr/bin/env bash # ============================================================================== # fusionAIze Grid - Advanced Universal Orchestrator # Version 1.10.1 # ============================================================================== set -euo pipefail # --- Colors & Styling --- RED='\033[0;31m' GREEN='\033[0;32m' BLUE='\033[0;34m' YELLOW='\033[1;33m' CYAN='\033[0;36m' MAGENTA='\033[0;35m' NC='\033[0m' BOLD='\033[1m' DIM='\033[2m' echo -e "\033[38;2;0;82;204m▐▘ ▘ \033[38;2;196;217;0m▄▖▄▖\033[38;2;0;82;204m \033[0m \033[38;2;38;67;123m▄▖ ▘ ▌\033[0m" echo -e "\033[38;2;0;82;204m▜▘▌▌▛▘▌▛▌▛▌\033[38;2;196;217;0m▌▌▐ \033[38;2;0;82;204m▀▌█▌\033[0m \033[38;2;38;67;123m▌ ▛▘▌▛▌\033[0m" echo -e "\033[38;2;0;82;204m▐ ▙▌▄▌▌▙▌▌▌\033[38;2;196;217;0m▛▌▟▖\033[38;2;0;82;204m▙▖▙▖\033[0m \033[38;2;38;67;123m▙▌▌ ▌▙▌\033[0m ${DIM}v1.10.1${NC}" echo "" echo -e "${DIM} Sovereign AI Infrastructure — Advanced Universal Orchestrator${NC}" echo "" # --- Helpers --- info() { echo -e " ${BLUE}ℹ${NC} $1"; } success() { echo -e " ${GREEN}✔${NC} $1"; } warning() { echo -e " ${YELLOW}⚠${NC} $1"; } error() { echo -e " ${RED}✘${NC} $1"; exit 1; } divider() { echo -e " ${DIM}──────────────────────────────────────────────────────${NC}"; } prompt() { read -r -p "$(echo -e " ${CYAN}▸${NC} $1")" "$2" || printf -v "$2" "%s" "" } prompt_hidden() { read -r -s -p "$(echo -e " ${CYAN}▸${NC} $1")" "$2" echo "" } _quit() { echo "" divider success "fusionAIze Grid Orchestration Complete! 🚀" divider echo "" exit 0 } # Single canonical state directory (legacy ~/.grid-state is deprecated) STATE_DIR="${HOME}/.config/faigrid/registry" STATE_FILE="${STATE_DIR}/state.env" # Repository root for relative path resolution REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" TOPOLOGY_FILE="${HOME}/.config/faigrid/.env.topology" # User-scoped registry — stable regardless of install method (git clone vs Homebrew) LOCAL_REGISTRY="${STATE_DIR}" # --- 1.3.0 Legacy Migration Hook --- if [[ -f "${REPO_ROOT}/core/heart/scripts/migrate_1.3.sh" ]]; then bash "${REPO_ROOT}/core/heart/scripts/migrate_1.3.sh" fi # --- Registry location migration (repo-relative → user-scoped) --- _old_registry="${REPO_ROOT}/.faigrid/state" if [[ -d "$_old_registry" ]]; then mkdir -p "$LOCAL_REGISTRY" for _f in "$_old_registry"/*.state; do [[ -f "$_f" ]] || continue _base="$(basename "$_f")" if [[ ! -f "$LOCAL_REGISTRY/$_base" ]]; then cp "$_f" "$LOCAL_REGISTRY/$_base" fi done fi unset _old_registry _f _base CURRENT_ROLE="none" CURRENT_VERSION="none" # Ensure user config dirs exist mkdir -p "$LOCAL_REGISTRY" mkdir -p "$(dirname "$TOPOLOGY_FILE")" # --- Agent-Native CLI Commands --- _cmd_help() { echo "" echo -e " ${BOLD}Usage:${NC}" echo -e " faigrid [OPTIONS]" echo -e " faigrid --status [--json]" echo "" echo -e " ${BOLD}Options:${NC}" echo -e " --mode local|remote Execution mode (default: interactive)" echo -e " --target SSH_TARGET Remote SSH target for remote mode" echo -e " --role ROLE Node role: core|edge|worker|backup|external|runner" echo -e " --action ACTION Action: install|verify|update|control|workbench|uninstall" echo -e " --component NAME Target a specific component" echo -e " --vnc Enable VNC for core role" echo -e " --yes Auto-confirm non-destructive prompts" echo -e " --force Bypass destructive prompts (except delete-all-data)" echo -e " --status [--json] Print node health status and exit" echo -e " --help Show this help and exit" echo "" echo -e " ${BOLD}Agent interface:${NC}" echo -e " faigrid --status --json Machine-readable JSON health output" echo -e " faigrid-workbench status Workbench plugin registry" echo -e " fgm health grid-messenger HTTP status" echo -e " fgm notify \"text\" Send Telegram notification" echo "" exit 0 } _port_reachable() { # -s silent, no -f so any HTTP response (incl. 404) counts as "port open" curl --connect-timeout 1 -so /dev/null "http://127.0.0.1:${1}/" 2>/dev/null } _cmd_status() { local output_json="${1:-false}" local version version="$(cat "${REPO_ROOT}/VERSION" 2>/dev/null || echo "unknown")" local node node="$(hostname -s 2>/dev/null || hostname 2>/dev/null || echo "unknown")" local os_name os_name="$(uname -s | tr '[:upper:]' '[:lower:]')" local ts ts="$(date -u +%Y-%m-%dT%H:%M:%SZ)" local svc_names=("messenger" "n8n" "faigate" "openclaw") local svc_ports=("9119" "5678" "8090" "18789") local svc_stati=() local i=0 while [[ $i -lt 4 ]]; do if _port_reachable "${svc_ports[$i]}" 2>/dev/null; then svc_stati+=("running") else svc_stati+=("stopped") fi i=$((i + 1)) done if [[ "$output_json" == "true" ]]; then printf '{\n' printf ' "version": "%s",\n' "$version" printf ' "node": "%s",\n' "$node" printf ' "os": "%s",\n' "$os_name" printf ' "services": {\n' local j=0 while [[ $j -lt 4 ]]; do local comma="," [[ $j -eq 3 ]] && comma="" printf ' "%s": {"port": %s, "status": "%s"}%s\n' \ "${svc_names[$j]}" "${svc_ports[$j]}" "${svc_stati[$j]}" "$comma" j=$((j + 1)) done printf ' },\n' printf ' "at": "%s"\n' "$ts" printf '}\n' else divider echo -e " ${BOLD}Node Status${NC} ${DIM}v${version} · ${node} · ${os_name}${NC}" divider printf " %-14s %-6s %s\n" "SERVICE" "PORT" "STATUS" echo -e " ${DIM}──────────────────────────────────────${NC}" local k=0 while [[ $k -lt 4 ]]; do local st="${svc_stati[$k]}" printf " %-14s %-6s " "${svc_names[$k]}" "${svc_ports[$k]}" if [[ "$st" == "running" ]]; then echo -e "${GREEN}✔ running${NC}" else echo -e "${DIM}✘ stopped${NC}" fi k=$((k + 1)) done divider echo "" fi exit 0 } # --- CLI Arguments --- AUTO_YES="false" FORCE="false" VNC_CHOICE="" ROLE_CHOICE="" ACTION_NAME="" MODE_CHOICE="" EXEC_MODE="" SSH_TARGET="" ROLE_NAME="" BOOTSTRAP_MODE="false" COMPONENT_NAME="" OUTPUT_JSON="false" RUN_STATUS="false" while [[ $# -gt 0 ]]; do case $1 in --mode) MODE_CHOICE="$2" case "$MODE_CHOICE" in local|remote) EXEC_MODE="$MODE_CHOICE" ;; *) error "Invalid --mode value: '${MODE_CHOICE}' (expected local|remote)" ;; esac shift 2 ;; --target) SSH_TARGET="$2"; shift 2 ;; --role) ROLE_NAME="$2"; shift 2 ;; --action) ACTION_NAME="$2"; shift 2 ;; --component) COMPONENT_NAME="$2"; shift 2 ;; --vnc) VNC_CHOICE="y"; shift ;; --yes) AUTO_YES="true"; shift ;; --force) FORCE="true"; shift ;; --bootstrap) BOOTSTRAP_MODE="true"; shift ;; --json) OUTPUT_JSON="true"; shift ;; --status) RUN_STATUS="true"; shift ;; --help|-h) _cmd_help ;; *) error "Unknown parameter passed: $1" ;; esac done [[ "$RUN_STATUS" == "true" ]] && _cmd_status "$OUTPUT_JSON" # --- Hardware Check Functions --- check_hardware() { local target_role=$1 local mode=$2 local ssh_target=$3 info "Performing hardware pre-flight checks for role: ${target_role}..." local ncpu local mem_kb local mem_gb if [[ "$mode" == "local" ]]; then if [[ "$(uname -s)" == "Darwin" ]]; then ncpu=$(sysctl -n hw.ncpu) mem_kb=$(( $(sysctl -n hw.memsize) / 1024 )) else ncpu=$(nproc) mem_kb=$(awk '/MemTotal/ {print $2}' /proc/meminfo) fi else if ssh -q "$ssh_target" "uname -s" | grep -q 'Darwin'; then ncpu=$(ssh -q "$ssh_target" "sysctl -n hw.ncpu") mem_kb=$(ssh -q "$ssh_target" "sysctl -n hw.memsize") mem_kb=$(( mem_kb / 1024 )) else ncpu=$(ssh -q "$ssh_target" "nproc") mem_kb=$(ssh -q "$ssh_target" "awk '/MemTotal/ {print \$2}' /proc/meminfo || echo 2000000") fi fi mem_gb=$(( mem_kb / 1024 / 1024 )) info "Detected CPU Cores: ${ncpu}" info "Detected RAM: ${mem_gb} GB" if [[ "$target_role" == "worker" ]]; then if (( mem_gb < 16 )); then warning "Role 'worker' usually requires >16GB RAM for local LLMs. You have ${mem_gb}GB." fi if (( ncpu < 4 )); then warning "Role 'worker' needs >=4 CPU Cores for efficient prompt processing." fi elif [[ "$target_role" == "core" ]]; then if (( mem_gb < 4 )); then warning "Role 'core' runs n8n, OpenClaw, and PostgreSQL. Recommended >=4GB RAM. You have ${mem_gb}GB." fi fi success "Hardware check completed." } # --- State Inspection --- inspect_state() { local mode=$1 local ssh_target=$2 if [ "$mode" = "local" ]; then if [ -f "$STATE_FILE" ]; then # shellcheck disable=SC1090 source "$STATE_FILE" || true CURRENT_ROLE=${GRID_ROLE:-none} CURRENT_VERSION=${GRID_VERSION:-none} fi elif [ "$mode" = "remote" ]; then # Check remote state safely: canonical registry first, legacy read-only fallback local remote_state="" local canonical="\$HOME/.config/faigrid/registry/state.env" local legacy="\$HOME/.grid-state" if ssh -q -o ConnectTimeout=3 "$ssh_target" "[ -f \"$canonical\" ]" 2>/dev/null; then remote_state=$(ssh -q -o ConnectTimeout=3 "$ssh_target" "cat \"$canonical\"" || echo "") elif ssh -q -o ConnectTimeout=3 "$ssh_target" "[ -f \"$legacy\" ]" 2>/dev/null; then remote_state=$(ssh -q -o ConnectTimeout=3 "$ssh_target" "cat \"$legacy\"" || echo "") warning "Legacy state file ~/.grid-state found on ${ssh_target}; canonical registry is preferred." fi if [ -n "$remote_state" ]; then CURRENT_ROLE=$(echo "$remote_state" | grep "GRID_ROLE" | cut -d'=' -f2 || echo "none") CURRENT_VERSION=$(echo "$remote_state" | grep "GRID_VERSION" | cut -d'=' -f2 || echo "none") fi fi } # --- Write State --- write_state() { local mode=$1 local ssh_target=$2 local role=$3 # Save to local registry first mkdir -p "$LOCAL_REGISTRY" { echo "GRID_ROLE=$role" echo "GRID_VERSION=latest" echo "INSTALL_DATE=\"$(date)\"" echo "EXEC_MODE=$mode" if [ "$mode" = "remote" ]; then echo "SSH_TARGET=$ssh_target" fi } > "$LOCAL_REGISTRY/${role}.state" if [ "$mode" = "local" ]; then { echo "GRID_ROLE=$role" echo "GRID_VERSION=latest" echo "INSTALL_DATE=\"$(date)\"" } > "$STATE_FILE" success "Saved state to ${STATE_FILE} (and local registry)" else ssh -q "$ssh_target" "mkdir -p \"\$HOME/.config/faigrid/registry\"; \ echo 'GRID_ROLE=$role' > \"\$HOME/.config/faigrid/registry/state.env\"; \ echo 'GRID_VERSION=latest' >> \"\$HOME/.config/faigrid/registry/state.env\"; \ echo 'INSTALL_DATE=\"$(date)\"' >> \"\$HOME/.config/faigrid/registry/state.env\"" || true success "Saved remote state to ${ssh_target}:~/.config/faigrid/registry/state.env (and local registry)" fi } # --- Role Resolver --- resolve_role_dir() { local role=$1 case "$role" in "edge") echo "edge/pi" ;; "core") echo "core/heart" ;; "worker") echo "worker" ;; "backup") echo "backup" ;; "external") echo "external" ;; "runner") echo "core/runners" ;; "supervisor") echo "core/supervisor" ;; *) error "Unknown role: $role" ;; esac } # --- Grid Status Probe --- # Uses individual variables (Bash 3.2 compatible — no associative arrays) GRID_STATUS_core="○" GRID_STATUS_edge="○" GRID_STATUS_worker="○" GRID_STATUS_backup="○" GRID_STATUS_external="○" GRID_STATUS_runner="○" GRID_STATUS_supervisor="○" probe_grid_status() { # Check local registry only (instant) local roles=("core" "edge" "worker" "backup" "external" "runner" "supervisor") for role in "${roles[@]}"; do if [[ -f "$LOCAL_REGISTRY/${role}.state" ]]; then # Bash 3.2 lacks associative arrays; role comes from the hardcoded array # above, never from operator/network input, so this eval is safe (V7). eval "GRID_STATUS_${role}=\"✔\"" fi done } # --- Load Local State --- # Retrieves target info (EXEC_MODE, SSH_TARGET) from registry load_local_state() { local role=$1 local state_file="$LOCAL_REGISTRY/${role}.state" if [ -f "$state_file" ]; then # Safer parsing for Bash 3.2 + set -e + pipefail while IFS='=' read -r key val; do if [ -z "$key" ] || echo "$key" | grep -q "^#"; then continue fi case "$key" in EXEC_MODE) EXEC_MODE="$val" ;; SSH_TARGET) SSH_TARGET="$val" ;; esac done < "$state_file" if [ -n "${EXEC_MODE:-}" ]; then MODE_CHOICE="$EXEC_MODE" fi fi return 0 } # --- Service Discovery --- DISCOVERED_SERVICES=() discover_services() { local mode=$1 local ssh_target=$2 local run_cmd if [[ "$mode" == "local" ]]; then run_cmd="bash -c" else run_cmd="ssh -q $ssh_target" fi info "Scanning target for existing services..." echo "" # Service probe list: name, check command local -a probes=( "Caddy:command -v caddy || pgrep -x caddy" "Pi-hole:command -v pihole || [ -d /etc/pihole ]" "Docker:command -v docker" "Ollama:command -v ollama || pgrep -x ollama" "LM Studio:command -v lms" "n8n:docker ps --format '{{.Names}}' 2>/dev/null | grep -q n8n" "PostgreSQL:command -v psql || docker ps --format '{{.Names}}' 2>/dev/null | grep -q postgres" "Redis:command -v redis-cli || docker ps --format '{{.Names}}' 2>/dev/null | grep -q redis" "Restic:command -v restic" "Tailscale:command -v tailscale" "Nginx:command -v nginx || pgrep -x nginx" ) for probe in "${probes[@]}"; do local svc_name="${probe%%:*}" local svc_cmd="${probe#*:}" if $run_cmd "$svc_cmd" &>/dev/null; then DISCOVERED_SERVICES+=("$svc_name") echo -e " ${GREEN}●${NC} ${svc_name} detected" fi done if [[ ${#DISCOVERED_SERVICES[@]} -eq 0 ]]; then echo -e " ${DIM}No known services detected (bare system).${NC}" fi echo "" } # ============================================================================== # STEP 1: GRID OVERVIEW # ============================================================================== divider echo -e " ${BOLD}Step 1 │ Your fusionAIze Grid${NC}" divider echo "" echo -e " The Grid ${BOLD}4+1 Architecture${NC} consists of these roles:" echo "" # Start of interactive wizard set +e # Probe known nodes from .faigrid/state for registered icons probe_grid_status _grid_icon() { local role=$1 local status case "$role" in core) status="$GRID_STATUS_core" ;; edge) status="$GRID_STATUS_edge" ;; worker) status="$GRID_STATUS_worker" ;; backup) status="$GRID_STATUS_backup" ;; external) status="$GRID_STATUS_external" ;; runner) status="$GRID_STATUS_runner" ;; supervisor) status="$GRID_STATUS_supervisor" ;; *) status="○" ;; esac if [[ "$status" == "✔" ]]; then echo -e "${GREEN}✔${NC}" else echo -e "${DIM}○${NC}" fi } echo -e " $(_grid_icon core) ${BOLD}grid-core${NC} ${DIM}AI Workbench — n8n, OpenClaw, Postgres, Redis${NC} ${YELLOW}[required]${NC}" echo -e " $(_grid_icon edge) ${BOLD}grid-edge${NC} ${DIM}TLS ingress, reverse proxy, firewall, DNS${NC} ${DIM}[optional]${NC}" echo -e " $(_grid_icon worker) ${BOLD}grid-worker${NC} ${DIM}Local LLM inference — Ollama, LM Studio${NC} ${DIM}[optional]${NC}" echo -e " $(_grid_icon backup) ${BOLD}grid-backup${NC} ${DIM}Offsite storage vault — Synology, S3, USB${NC} ${DIM}[optional]${NC}" echo -e " $(_grid_icon external) ${BOLD}grid-external${NC} ${DIM}Cloud extension — ext. n8n, Plane PM${NC} ${DIM}[optional]${NC}" echo -e " $(_grid_icon runner) ${BOLD}grid-runner${NC} ${DIM}Isolated shell/browser execution environments${NC} ${DIM}[optional]${NC}" echo -e " $(_grid_icon supervisor) ${BOLD}grid-supervisor${NC} ${DIM}Control API — Scheduler, Runner Pool, Recovery${NC} ${DIM}[optional]${NC}" echo "" echo -e " ${DIM}Tip: Start with grid-core. Add other nodes as your grid grows.${NC}" # ============================================================================== # STEP 2: WHICH NODE TO PROVISION NOW # ============================================================================== echo "" divider echo -e " ${BOLD}Step 2 │ Select Node to Provision${NC}" divider if [[ -z "$ROLE_NAME" ]]; then echo "" echo " Which node do you want to work on right now?" echo "" echo -e " ${BOLD}1)${NC} grid-core ${DIM}The brain — always provision this first${NC}" echo -e " ${BOLD}2)${NC} grid-edge ${DIM}Secure ingress gateway (Caddy, DNS)${NC}" echo -e " ${BOLD}3)${NC} grid-worker ${DIM}Local LLM execution (MacBook, GPU server)${NC}" echo -e " ${BOLD}4)${NC} grid-backup ${DIM}Restic vault (NAS, USB disk, S3 bucket)${NC}" echo -e " ${BOLD}5)${NC} grid-external ${DIM}Public cloud node (VPS, agency server)${NC}" echo -e " ${BOLD}6)${NC} grid-runner ${DIM}Isolated secure execution runners${NC}" echo -e " ${BOLD}7)${NC} grid-supervisor ${DIM}Control API for Scheduler, Runner Pool, Recovery${NC}" echo "" echo -e " ${BOLD}q)${NC} Quit" echo "" prompt "Enter role (1-6 / q): " ROLE_CHOICE case "$ROLE_CHOICE" in 1) ROLE_NAME="core" ;; 2) ROLE_NAME="edge" ;; 3) ROLE_NAME="worker" ;; 4) ROLE_NAME="backup" ;; 5) ROLE_NAME="external" ;; 6) ROLE_NAME="runner" ;; 7) ROLE_NAME="supervisor" ;; [Qq]|"") _quit ;; *) warning "Invalid choice. Please enter 1-6 or q." ;; esac # Smart suggestion for backup co-location if [[ "$ROLE_NAME" == "backup" ]]; then echo "" echo -e " ${CYAN}💡 Tip:${NC} You don't need a dedicated backup node." echo -e " Restic can run directly on ${BOLD}grid-core${NC} or ${BOLD}grid-external${NC}," echo -e " backing up to a Synology NAS, S3 bucket, or USB drive." echo "" if [[ "$AUTO_YES" != "true" ]]; then prompt "Continue with a dedicated backup node anyway? (Y/n): " BACKUP_CONFIRM if [[ "${BACKUP_CONFIRM:-Y}" =~ ^[Nn]$ ]]; then info "Returning to node selection..." exec bash "$0" fi fi fi fi ROLE_DIR="$(resolve_role_dir "$ROLE_NAME")" success "Selected: ${BOLD}${ROLE_NAME}${NC}" # Check for cached target info load_local_state "$ROLE_NAME" # ============================================================================== # STEP 3: DEPLOY MODE # ============================================================================== echo "" divider echo -e " ${BOLD}Step 3 │ Deploy Mode${NC}" divider if [[ -z "$MODE_CHOICE" ]]; then if [[ -n "${EXEC_MODE:-}" ]]; then # Known target found in registry target_desc="Local Node" if [ "$EXEC_MODE" = "remote" ]; then target_desc="SSH: ${SSH_TARGET:-unknown}" fi echo "" echo -e " ${CYAN}💡${NC} Known target found for ${BOLD}${ROLE_NAME}${NC}:" echo "" echo -e " ${BOLD}1)${NC} Use existing ${DIM}(${target_desc})${NC}" echo -e " ${BOLD}2)${NC} Change target ${DIM}(Switch to local/remote/new IP)${NC}" echo "" prompt "Choice (1/2): " USE_EXISTING if [ "$USE_EXISTING" = "2" ]; then EXEC_MODE="" MODE_CHOICE="" SSH_TARGET="" fi fi fi # If no cached mode (or user chose to change), ask if [[ -z "$MODE_CHOICE" ]]; then echo "" echo -e " How are you deploying ${BOLD}${ROLE_NAME}${NC}?" echo "" echo -e " ${BOLD}1)${NC} On-Node ${DIM}— I am logged into the target node right now${NC}" echo -e " ${BOLD}2)${NC} Remote Push ${DIM}— Push to a remote node via SSH from this workstation${NC}" echo "" prompt "Select mode (1/2): " MODE_CHOICE # Numeric mapping is the interactive form; the words are accepted too. case "$MODE_CHOICE" in 1|local) EXEC_MODE="local" ;; 2|remote) EXEC_MODE="remote" ;; esac fi if [[ "${EXEC_MODE:-}" == "remote" ]]; then if [[ -z "$SSH_TARGET" ]]; then default_ip="192.168.178.10" case "$ROLE_NAME" in core) default_ip="192.168.178.20" ;; worker) default_ip="192.168.178.30" ;; backup) default_ip="192.168.178.40" ;; supervisor) default_ip="192.168.178.20" ;; esac echo "" prompt "SSH target (e.g. grid@${default_ip}): " SSH_TARGET fi info "Testing SSH connectivity to ${SSH_TARGET}..." if ! ssh -q "$SSH_TARGET" exit; then error "SSH connectivity to ${SSH_TARGET} failed. Ensure keys or passwords are correct." fi # Pre-flight: Check for rsync on remote target if ! ssh -q "$SSH_TARGET" "command -v rsync" > /dev/null; then warning "rsync is missing on ${SSH_TARGET}." prompt "Would you like to try installing rsync automatically via sudo? (y/n): " INSTALL_RSYNC if [[ "$INSTALL_RSYNC" == "y" ]]; then info "Attempting to install rsync on ${SSH_TARGET} (may prompt for password)..." # Use ssh -t to allow interactive sudo password prompt ssh -t "$SSH_TARGET" "sudo apt-get update && sudo apt-get install -y rsync" if ssh -q "$SSH_TARGET" "command -v rsync" > /dev/null; then success "rsync installed successfully." else error "rsync installation failed. Please install it manually (e.g., 'sudo apt install rsync') and try again." fi else error "rsync is required for remote deployment. Please install it manually and try again." fi fi success "Connected to ${SSH_TARGET} (rsync verified)." else EXEC_MODE="local" success "On-Node mode: this machine is the target." fi # Detect existing state inspect_state "$EXEC_MODE" "$SSH_TARGET" echo "" if [[ "$CURRENT_ROLE" != "none" ]]; then success "Detected existing node: Role [${BOLD}${CURRENT_ROLE}${NC}], Version [${CURRENT_VERSION}]" else info "No state registry found. Scanning target for existing services..." echo "" discover_services "$EXEC_MODE" "$SSH_TARGET" # If services were discovered, suggest bootstrapping if [[ ${#DISCOVERED_SERVICES[@]} -gt 0 ]]; then echo -e " ${CYAN}💡${NC} Services are already running, but this node is ${BOLD}not registered${NC}" echo -e " with fusionAIze Grid. You can:" echo "" echo -e " ${BOLD}a)${NC} ${GREEN}Adopt${NC} ${DIM}— Register this node as ${BOLD}${ROLE_NAME}${NC}${DIM} (keeps everything as-is)${NC}" echo -e " ${BOLD}b)${NC} Continue ${DIM}— Proceed to action selection without registering${NC}" echo "" prompt "Adopt or continue? (a/b): " ADOPT_CHOICE if [[ "${ADOPT_CHOICE:-a}" =~ ^[Aa]$ ]]; then write_state "$EXEC_MODE" "$SSH_TARGET" "$ROLE_NAME" CURRENT_ROLE="$ROLE_NAME" echo "" success "Node adopted as ${BOLD}${ROLE_NAME}${NC}. Future runs will recognize this node." fi fi fi # ============================================================================== # BOOTSTRAP MODE: Register an existing node without running install scripts # ============================================================================== if [[ "$BOOTSTRAP_MODE" == "true" ]]; then echo "" write_state "$EXEC_MODE" "$SSH_TARGET" "$ROLE_NAME" echo "" success "Node bootstrapped as [${BOLD}${ROLE_NAME}${NC}]. Future runs will detect this role automatically." exit 0 fi # If the target already has a different role, warn if [[ "$CURRENT_ROLE" != "none" && "$CURRENT_ROLE" != "$ROLE_NAME" ]]; then echo "" warning "This target is already registered as ${BOLD}${CURRENT_ROLE}${NC}, but you selected ${BOLD}${ROLE_NAME}${NC}." if [[ "$AUTO_YES" == "true" ]]; then ROLE_NAME="$CURRENT_ROLE" ROLE_DIR="$(resolve_role_dir "$ROLE_NAME")" info "Keeping existing role: ${ROLE_NAME} (--yes: non-destructive auto-confirm)" else prompt "Change this node's role? (y/N): " CHANGE_ROLE if [[ ! "${CHANGE_ROLE:-N}" =~ ^[Yy]$ ]]; then ROLE_NAME="$CURRENT_ROLE" ROLE_DIR="$(resolve_role_dir "$ROLE_NAME")" info "Keeping existing role: ${ROLE_NAME}" fi fi fi # ============================================================================== # STEP 4: ACTION SELECTION # ============================================================================== echo "" divider echo -e " ${BOLD}Step 4 │ Action${NC}" divider if [[ -z "$ACTION_NAME" ]]; then echo "" echo -e " What do you want to do with this ${BOLD}${ROLE_NAME}${NC} node?" echo "" if [[ "$CURRENT_ROLE" != "none" ]]; then # Existing registered node — safe actions first, destructive at bottom echo -e " ${BOLD}1)${NC} ${GREEN}Verify${NC} ${DIM}Read-only: check services, ports, disk (changes nothing)${NC}" echo -e " ${BOLD}2)${NC} Update ${DIM}System packages only (apt upgrade). Your configs stay untouched${NC}" echo -e " ${BOLD}3)${NC} Control ${DIM}Start / Stop / Restart managed services${NC}" if [[ "$ROLE_NAME" == "core" ]]; then echo -e " ${BOLD}4)${NC} ${MAGENTA}Workbench${NC} ${DIM}AI Tooling — CLIs, Agents, Routers${NC}" fi echo "" if [[ "$ROLE_NAME" == "core" ]]; then echo -e " ${BOLD}5)${NC} ${YELLOW}Reinstall${NC} ${DIM}⚠ Wipe and re-provision from scratch${NC}" echo -e " ${BOLD}6)${NC} ${YELLOW}Uninstall${NC} ${DIM}⚠ Remove this node role entirely${NC}" echo -e " ${BOLD}7)${NC} ${CYAN}Backup${NC} ${DIM}Dump Postgres + n8n volume to /var/backups/grid-core-heart${NC}" else echo -e " ${BOLD}4)${NC} ${YELLOW}Reinstall${NC} ${DIM}⚠ Wipe and re-provision from scratch${NC}" echo -e " ${BOLD}5)${NC} ${YELLOW}Uninstall${NC} ${DIM}⚠ Remove this node role entirely${NC}" fi echo "" echo -e " ${BOLD}h)${NC} ${BOLD}Help / Connect${NC} ${DIM}How to access services from your workstation${NC}" echo -e " ${BOLD}s)${NC} Switch node ${DIM}Go back to node selection (Step 2)${NC}" echo -e " ${BOLD}q)${NC} Quit" echo "" echo -e " ${DIM}Tip: Start with Verify to see the current state of this node.${NC}" echo "" if [[ "$ROLE_NAME" == "core" ]]; then prompt "Select action (1-7 / h / s / q): " ACTION_CHOICE else prompt "Select action (1-5 / h / s / q): " ACTION_CHOICE fi case "$ACTION_CHOICE" in 1) ACTION_NAME="verify" ;; 2) ACTION_NAME="update" ;; 3) ACTION_NAME="control" ;; 4) if [[ "$ROLE_NAME" == "core" ]]; then ACTION_NAME="workbench"; else ACTION_NAME="install"; fi ;; 5) if [[ "$ROLE_NAME" == "core" ]]; then ACTION_NAME="install"; else ACTION_NAME="uninstall"; fi ;; 6) if [[ "$ROLE_NAME" == "core" ]]; then ACTION_NAME="uninstall"; else warning "Invalid choice. Please enter 1-5, h, s, or q."; fi ;; 7) if [[ "$ROLE_NAME" == "core" ]]; then ACTION_NAME="backup"; else warning "Invalid choice. Please enter 1-5, h, s, or q."; fi ;; [Hh]) _show_help "$EXEC_MODE" "$SSH_TARGET" "$ROLE_NAME" ; exec bash "$0" ;; [Ss]) exec bash "$0" ;; [Qq]|"") _quit ;; *) warning "Invalid choice." ;; esac else # Fresh target — install is the natural first action echo -e " ${BOLD}1)${NC} Install ${DIM}Set up this node with the ${ROLE_NAME} role${NC}" echo -e " ${BOLD}2)${NC} Verify ${DIM}Check connectivity and environment (read-only)${NC}" echo "" echo -e " ${BOLD}s)${NC} Switch node ${DIM}Go back to node selection (Step 2)${NC}" echo -e " ${BOLD}q)${NC} Quit" echo "" echo -e " ${DIM}Tip: Start with Verify to check connectivity first.${NC}" echo "" prompt "Select action (1-2 / s / q): " ACTION_CHOICE case "$ACTION_CHOICE" in 1) ACTION_NAME="install" ;; 2) ACTION_NAME="verify" ;; [Ss]) exec bash "$0" ;; [Qq]|"") _quit ;; *) warning "Invalid choice. Please enter 1-2, s, or q." ;; esac fi fi # ============================================================================== # STEP 5: SAFETY & CONFIRMATION # ============================================================================== if [[ "$ACTION_NAME" == "install" && "$CURRENT_ROLE" != "none" ]]; then echo "" divider echo -e " ${BOLD}Step 5 │ Safety Check${NC}" divider echo "" warning "This target already has a ${BOLD}${CURRENT_ROLE}${NC} installation." warning "A fresh install may overwrite critical services." if [[ "$FORCE" == "false" ]]; then prompt "Type 'overwrite' to confirm destructive reinstall: " CONFIRM_OVERWRITE if [[ "${CONFIRM_OVERWRITE:-}" != "overwrite" ]]; then error "Overwrite not confirmed. Aborting." fi else info "--force flag detected. Bypassing overwrite confirmation." fi fi # Uninstall safety gate — warn before any data is touched if [[ "$ACTION_NAME" == "uninstall" ]]; then echo "" divider echo -e " ${BOLD}Step 5 │ Uninstall Safety Check${NC}" divider echo "" warning "You are about to UNINSTALL the ${BOLD}${ROLE_NAME}${NC} node." if [[ "$ROLE_NAME" == "core" ]]; then echo "" echo -e " ${RED}✖${NC} ${BOLD}ALL Docker volumes will be permanently deleted:${NC}" echo -e " ${DIM}postgres_data${NC} — n8n workflows, credentials, execution history" echo -e " ${DIM}n8n_data${NC} — n8n settings, installed nodes, encryption key" echo -e " ${DIM}redis_data${NC} — queue state" echo "" echo -e " ${YELLOW}Tip: Run${NC} ${BOLD}Backup${NC} ${YELLOW}first to preserve your data.${NC}" fi echo "" prompt "Type 'delete-all-data' to confirm permanent data loss: " CONFIRM_UNINSTALL if [[ "${CONFIRM_UNINSTALL:-}" != "delete-all-data" ]]; then error "Uninstall not confirmed. Aborting." exit 1 fi fi # Hardware Checks (only for fresh installs) if [[ "$ACTION_NAME" == "install" ]]; then check_hardware "$ROLE_NAME" "$EXEC_MODE" "$SSH_TARGET" fi # --- Role-Specific Options --- if [[ "$ROLE_NAME" == "core" && "$ACTION_NAME" == "install" ]]; then if [[ -z "$VNC_CHOICE" ]]; then echo "" if [[ "$AUTO_YES" == "true" ]]; then VNC_CHOICE="N" info "Disabling VNC (--yes: non-destructive auto-confirm)" else prompt "Enable VNC GUI for the AI Workbench? (y/N): " VNC_CHOICE fi fi fi # ============================================================================== # TOPOLOGY WRITING # ============================================================================== echo "# Generated by Universal Orchestrator v2.0" > "$TOPOLOGY_FILE" echo "SERVER_NAME=${ROLE_NAME}-node" >> "$TOPOLOGY_FILE" echo "ROLE=$ROLE_NAME" >> "$TOPOLOGY_FILE" if [[ "$ROLE_NAME" == "core" ]]; then if [[ "${VNC_CHOICE:-}" =~ ^[Yy]$ ]]; then echo "ENABLE_VNC=true" >> "$TOPOLOGY_FILE" else echo "ENABLE_VNC=false" >> "$TOPOLOGY_FILE" fi fi # ============================================================================== # DEPLOYMENT PHASE — with post-action loop # ============================================================================== _run_action() { local mode=$1 local ssh_target=$2 local role=$3 local action=$4 local role_dir=$5 local cmd_arg="${COMPONENT_NAME:-all}" # Workbench — map to 'control workbench', skip the interactive control submenu if [ "$action" = "workbench" ]; then action="control" cmd_arg="workbench" fi # Handle control sub-commands interactively (skipped for workbench above) if [ "$action" = "control" ] && [ "$cmd_arg" != "workbench" ]; then echo "" echo -e " ${BOLD}Control Interface${NC} — Select command for ${BOLD}${role}${NC}:" echo "" echo -e " ${BOLD}1)${NC} ${GREEN}Start${NC}" echo -e " ${BOLD}2)${NC} ${RED}Stop${NC}" echo -e " ${BOLD}3)${NC} ${YELLOW}Restart${NC}" echo -e " ${BOLD}4)${NC} Status" echo -e " ${BOLD}5)${NC} Reload" echo -e " ${BOLD}6)${NC} Logs ${DIM}last 60 lines — all services${NC}" echo "" echo -e " ${BOLD}a)${NC} Action selection ${DIM}(Back to Step 4)${NC}" echo "" prompt "Select command (1-6 / a / s / q): " CONTROL_CHOICE case "$CONTROL_CHOICE" in 1) cmd_arg="start" ;; 2) cmd_arg="stop" ;; 3) cmd_arg="restart" ;; 4) cmd_arg="status" ;; 5) cmd_arg="reload" ;; 6) cmd_arg="logs" ;; [Aa]) return 0 ;; [Ss]) exec bash "$0" ;; [Qq]|"") _quit ;; *) warning "Invalid choice. Defaulting to 'status'." ; cmd_arg="status" ;; esac fi echo "" divider echo -e " ${BOLD}Deploying${NC} ${role} → ${action} (${cmd_arg})" divider echo "" if [ "$mode" = "local" ]; then info "Initiating Local Pipeline [${action}] for ${role}..." if [ ! -d "$role_dir" ]; then warning "Directory $role_dir not populated in repo. Skipping module execution." else TARGET_SCRIPT="${role_dir}/scripts/${action}.sh" if [ -f "$TARGET_SCRIPT" ]; then echo "" bash "$TARGET_SCRIPT" "$cmd_arg" else warning "Script not found: ${TARGET_SCRIPT}. Action skipped." fi fi if [ "$action" = "install" ]; then write_state "$mode" "$ssh_target" "$role" fi elif [ "$mode" = "remote" ]; then info "Initiating Remote Pipeline [${action}] to ${ssh_target}..." ssh -q "$ssh_target" "mkdir -p /tmp/grid-install" info "Transferring payload to target..." rsync -az --exclude='.git' --exclude='node_modules' "${REPO_ROOT}/" "$ssh_target:/tmp/grid-install/" > /dev/null scp "${TOPOLOGY_FILE}" "${ssh_target}:/tmp/grid-install/" > /dev/null info "Executing remote [${action}] payload..." echo "" # Calculate relative path from REPO_ROOT to target script REL_ROLE_DIR="${role_dir#$REPO_ROOT/}" TARGET_SCRIPT="${REL_ROLE_DIR}/scripts/${action}.sh" ssh -t "$ssh_target" "cd /tmp/grid-install || exit 1; bash \"$TARGET_SCRIPT\" \"$cmd_arg\"" if [ "$action" = "install" ]; then write_state "$mode" "$ssh_target" "$role" fi fi } # Show tailored connection help based on role and target _show_help() { local mode=$1 local ssh_target=$2 local role=$3 local ip="localhost" if [[ "$mode" == "remote" ]]; then # Extract IP from user@ip [[ "$ssh_target" == *"@"* ]] && ip=$(echo "$ssh_target" | cut -d'@' -f2) || ip="$ssh_target" fi echo "" divider echo -e " ${BOLD}Access & Connection Guide │ ${role}${NC}" divider echo "" case "$role" in core) echo -e " ${BOLD}n8n Automation Workbench:${NC}" # Check edge status local edge_status="○" [[ -f "$LOCAL_REGISTRY/edge.state" ]] && edge_status="✔" if [[ "$edge_status" == "✔" ]]; then echo -e " Public URL : https://n8n.your-domain.com ${DIM}(via grid-edge Caddy Proxy)${NC}" echo -e " ${DIM}(Make sure your edge DNS is configured to point here)${NC}" else echo -e " Local URL : http://${ip}:5678" if [[ "$mode" == "remote" ]]; then echo -e " SSH Tunnel : ${CYAN}ssh -L 5678:localhost:5678 ${ssh_target}${NC}" echo -e " ${DIM}(Run this on your workstation to access n8n in your local browser)${NC}" fi fi echo "" echo -e " ${BOLD}OpenClaw Gateway:${NC}" echo -e " Port : 18789 (Internal)" echo -e " Status : Check via 'Verify' or 'Control'" ;; edge) echo -e " ${BOLD}Pi-hole Admin:${NC}" echo -e " URL : http://${ip}/admin" echo "" echo -e " ${BOLD}Caddy Ingress:${NC}" echo -e " Ports : 80 / 443" echo -e " Config : /etc/caddy/Caddyfile" ;; worker) echo -e " ${BOLD}Ollama API:${NC}" echo -e " Local URL : http://${ip}:11434" if [[ "$mode" == "remote" ]]; then echo -e " SSH Tunnel : ${CYAN}ssh -L 11434:localhost:11434 ${ssh_target}${NC}" fi ;; supervisor) echo -e " ${BOLD}Supervisor Control API:${NC}" echo -e " URL : http://${ip}:5000" echo -e " Endpoints : /v1/supervisor/register, /v1/supervisor/poll," echo -e " /v1/supervisor/recovery_history/{job_id}, /v1/supervisor/max_workers" if [[ "$mode" == "remote" ]]; then echo -e " SSH Tunnel : ${CYAN}ssh -L 5000:localhost:5000 ${ssh_target}${NC}" fi ;; *) echo -e " No specific connection help for this role yet." ;; esac echo "" echo -e " ${BOLD}Documentation:${NC}" echo -e " Runbooks : docs/runbooks/" echo -e " Architecture: docs/architecture.md" echo "" divider } # Destructive actions exit immediately; safe actions loop back _is_loopable() { [[ "$1" == "verify" || "$1" == "update" || "$1" == "control" || "$1" == "workbench" || "$1" == "backup" ]] } # Execute the requested action _run_action "$EXEC_MODE" "$SSH_TARGET" "$ROLE_NAME" "$ACTION_NAME" "$ROLE_DIR" if _is_loopable "$ACTION_NAME"; then while true; do echo "" divider inspect_state "$EXEC_MODE" "$SSH_TARGET" if [[ "$CURRENT_ROLE" != "none" ]]; then echo -e " ${GREEN}✔${NC} ${BOLD}${ROLE_NAME}${NC} is registered ${DIM}(version: ${CURRENT_VERSION})${NC}" else echo -e " ${DIM}○${NC} ${BOLD}${ROLE_NAME}${NC} is ${YELLOW}not registered${NC} — run Install or Adopt to register" fi divider echo "" echo -e " ${BOLD}What next?${NC}" echo "" echo -e " ${BOLD}1)${NC} ${GREEN}Verify${NC} ${DIM}Read-only: check services, ports, disk${NC}" echo -e " ${BOLD}2)${NC} Update ${DIM}System packages only (apt upgrade). Configs stay untouched${NC}" echo -e " ${BOLD}3)${NC} Control ${DIM}Start / Stop / Restart managed services${NC}" if [[ "$ROLE_NAME" == "core" ]]; then echo -e " ${BOLD}4)${NC} ${MAGENTA}Workbench${NC} ${DIM}AI Tooling — CLIs, Agents, Routers${NC}" fi echo "" if [[ "$ROLE_NAME" == "core" ]]; then echo -e " ${BOLD}5)${NC} ${YELLOW}Reinstall${NC} ${DIM}⚠ Wipe and re-provision from scratch${NC}" echo -e " ${BOLD}6)${NC} ${YELLOW}Uninstall${NC} ${DIM}⚠ Remove this node role entirely${NC}" echo -e " ${BOLD}7)${NC} ${CYAN}Backup${NC} ${DIM}Dump Postgres + n8n volume to /var/backups/grid-core-heart${NC}" else echo -e " ${BOLD}4)${NC} ${YELLOW}Reinstall${NC} ${DIM}⚠ Wipe and re-provision from scratch${NC}" echo -e " ${BOLD}5)${NC} ${YELLOW}Uninstall${NC} ${DIM}⚠ Remove this node role entirely${NC}" fi echo "" echo -e " ${BOLD}h)${NC} ${BOLD}Help / Connect${NC}" echo -e " ${BOLD}s)${NC} Switch node ${DIM}Go back to node selection (Step 2)${NC}" echo -e " ${BOLD}q)${NC} Quit" echo "" if [[ "$ROLE_NAME" == "core" ]]; then prompt "Select action (1-7 / h / s / q): " NEXT_CHOICE else prompt "Select action (1-5 / h / s / q): " NEXT_CHOICE fi case "$NEXT_CHOICE" in 1) _run_action "$EXEC_MODE" "$SSH_TARGET" "$ROLE_NAME" "verify" "$ROLE_DIR" || true ;; 2) _run_action "$EXEC_MODE" "$SSH_TARGET" "$ROLE_NAME" "update" "$ROLE_DIR" || true ;; 3) _run_action "$EXEC_MODE" "$SSH_TARGET" "$ROLE_NAME" "control" "$ROLE_DIR" || true ;; 4) if [[ "$ROLE_NAME" == "core" ]]; then _run_action "$EXEC_MODE" "$SSH_TARGET" "$ROLE_NAME" "workbench" "$ROLE_DIR" || true else warning "This will wipe and re-provision ${BOLD}${ROLE_NAME}${NC}. Are you sure?" prompt "Type YES to confirm: " CONFIRM if [[ "$CONFIRM" == "YES" ]]; then _run_action "$EXEC_MODE" "$SSH_TARGET" "$ROLE_NAME" "install" "$ROLE_DIR" break else info "Reinstall cancelled." fi fi ;; 5) if [[ "$ROLE_NAME" == "core" ]]; then warning "This will wipe and re-provision ${BOLD}${ROLE_NAME}${NC}. Are you sure?" prompt "Type YES to confirm: " CONFIRM if [[ "$CONFIRM" == "YES" ]]; then _run_action "$EXEC_MODE" "$SSH_TARGET" "$ROLE_NAME" "install" "$ROLE_DIR" break else info "Reinstall cancelled." fi else warning "This will remove the ${BOLD}${ROLE_NAME}${NC} role. Are you sure?" prompt "Type YES to confirm: " CONFIRM if [[ "$CONFIRM" == "YES" ]]; then _run_action "$EXEC_MODE" "$SSH_TARGET" "$ROLE_NAME" "uninstall" "$ROLE_DIR" break else info "Uninstall cancelled." fi fi ;; 6) if [[ "$ROLE_NAME" == "core" ]]; then echo "" warning "DESTRUCTIVE: This will permanently delete ALL Docker volumes!" echo -e " ${RED}✖${NC} ${DIM}postgres_data${NC} — n8n workflows, credentials, execution history" echo -e " ${RED}✖${NC} ${DIM}n8n_data${NC} — settings, installed nodes, encryption key" echo -e " ${RED}✖${NC} ${DIM}redis_data${NC} — queue state" echo "" echo -e " ${YELLOW}Tip: Run${NC} ${BOLD}Backup (7)${NC} ${YELLOW}first to preserve your data.${NC}" echo "" prompt "Type 'delete-all-data' to confirm permanent data loss: " CONFIRM if [[ "$CONFIRM" == "delete-all-data" ]]; then _run_action "$EXEC_MODE" "$SSH_TARGET" "$ROLE_NAME" "uninstall" "$ROLE_DIR" break else info "Uninstall cancelled." fi else warning "Invalid choice." fi ;; 7) if [[ "$ROLE_NAME" == "core" ]]; then _run_action "$EXEC_MODE" "$SSH_TARGET" "$ROLE_NAME" "backup" "$ROLE_DIR" || true else warning "Invalid choice." fi ;; [Hh]) _show_help "$EXEC_MODE" "$SSH_TARGET" "$ROLE_NAME" ;; [Ss]) exec bash "$0" ;; [Qq]|"") break ;; *) warning "Invalid choice." ;; esac done fi echo "" divider success "fusionAIze Grid Orchestration Complete! 🚀" divider echo ""