#!/bin/bash # Script Metadata #name=Mover Status Script #description=This script monitors the progress of the "Mover" process and posts updates to Discord, Telegram, Pushover, Apprise, and/or native Unraid notifications. #backgroundOnly=true #arrayStarted=true # --------------------------------------------------------- # Mover Status Script # --------------------------------------------------------- # Monitors Unraid's mover process and posts progress updates # to Discord, Telegram, Pushover, Apprise, and/or native Unraid notifications. # # Dependencies: bash, curl, jq, du, pgrep, date # Optional: apprise CLI when APPRISE_MODE=cli # reachable Apprise API when APPRISE_MODE=api # Runs as a backgroundOnly Unraid user script. # --------------------------------------------------------- # Simple timestamp for logs log() { echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" } # Log the starting message log "Starting Mover Status Monitor..." # ------------------------------------------- # Script Settings: Edit these! # ------------------------------------------- # Configure basic settings and webhook details USE_TELEGRAM=false # Enable notifications to Telegram USE_DISCORD=false # Enable notifications to Discord USE_PUSHOVER=false # Enable notifications to Pushover USE_APPRISE=false # Enable notifications through Apprise USE_UNRAID=false # Enable native Unraid notifications/toasts TELEGRAM_BOT_TOKEN="xxxx" # Telegram bot token TELEGRAM_CHAT_ID="xxxx" # Telegram chat ID DISCORD_WEBHOOK_URL="https://discord.com/api/webhooks/xxxx/xxxx" # Discord webhook URL DISCORD_NAME_OVERRIDE="Mover Bot" # Display name for Discord notifications PUSHOVER_APP_TOKEN="xxxx" # Pushover application/API token PUSHOVER_USER_KEY="xxxx" # Pushover user/group key PUSHOVER_TITLE="Mover Status" # Notification title for Pushover APPRISE_MODE="cli" # Apprise transport mode: cli | api APPRISE_BIN="/usr/bin/apprise" # Apprise CLI executable APPRISE_API_URL="http://127.0.0.1:8000" # Apprise API base URL when mode=api APPRISE_TITLE="Mover Status" # Notification title for Apprise UNRAID_NOTIFY_BIN="/usr/local/emhttp/webGui/scripts/notify" # Native Unraid notify executable UNRAID_EVENT="Mover Status" # Event name shown by Unraid UNRAID_TITLE="Mover Status" # Notification subject shown by Unraid APPRISE_TARGETS=( # "pover://USER_KEY@APP_TOKEN" ) NOTIFICATION_INCREMENT=25 # Notification frequency in percentage increments DRY_RUN=false # Enable this to test the notifications without actual monitoring ENABLE_DEBUG=false # Set to true to enable debug logging DU_POLL_INTERVAL=30 # Seconds between disk usage recalculations (higher = less I/O load) CACHE_PATH="/mnt/cache" # Path to cache directory to monitor ENABLE_FILE_INFO=false # Show file count and current file in notifications (requires Mover Tuning plugin) # ------------------------------------------- # Webhook Messages: Edit these if you want # ------------------------------------------- # Custom messages for each notification point TELEGRAM_MOVING_MESSAGE="Moving data from SSD Cache to HDD Array. Progress: {percent}% complete. Remaining data: {remaining_data}. Estimated completion time: {etc}. Note: Services like Plex may run slow or be unavailable during the move." DISCORD_MOVING_MESSAGE="Moving data from SSD Cache to HDD Array.\nProgress: **{percent}%** complete.\nRemaining data: {remaining_data}.\nEstimated completion time: {etc}.\n\nNote: Services like Plex may run slow or be unavailable during the move." PUSHOVER_MOVING_MESSAGE=$'Moving data from SSD Cache to HDD Array.\nProgress: {percent}% complete.\nRemaining data: {remaining_data}.\nEstimated completion time: {etc}.\n\nNote: Services like Plex may run slow or be unavailable during the move.' APPRISE_MOVING_MESSAGE=$'Moving data from SSD Cache to HDD Array.\nProgress: {percent}% complete.\nRemaining data: {remaining_data}.\nEstimated completion time: {etc}.\n\nNote: Services like Plex may run slow or be unavailable during the move.' UNRAID_MOVING_MESSAGE="Moving data from SSD Cache to HDD Array. Progress: {percent}% complete. Remaining data: {remaining_data}. Estimated completion time: {etc}." PREPARING_MESSAGE="Mover has started. Mover Tuning is preparing/scanning cache data. Progress will be available when transfer begins." COMPLETION_MESSAGE="Moving has been completed!" DISCORD_COMPLETION_MESSAGE="" # If empty, falls back to COMPLETION_MESSAGE TELEGRAM_COMPLETION_MESSAGE="" # If empty, falls back to COMPLETION_MESSAGE PUSHOVER_COMPLETION_MESSAGE="" # If empty, falls back to COMPLETION_MESSAGE APPRISE_COMPLETION_MESSAGE="" # If empty, falls back to COMPLETION_MESSAGE UNRAID_COMPLETION_MESSAGE="" # If empty, falls back to COMPLETION_MESSAGE # --------------------------------------- # Exclusion Folders: Define paths to exclude # --------------------------------------- # Set EXCLUDE_PATH_XX to directories you want to exclude from being monitored. # Leave EXCLUDE_PATH_XX variables empty if no exclusions are needed. # This will result in monitoring the entire directory specified in the script (/mnt/cache). # # Example usage: # EXCLUDE_PATH_01="/mnt/cache/your/excluded/folder" # EXCLUDE_PATH_02="/mnt/cache/another/excluded/folder" # EXCLUDE_PATH_03="/mnt/cache/maybe/a/.hidden/folder" # Add more EXCLUDE_PATH_XX as needed. # shellcheck disable=SC2034 EXCLUDE_PATH_01="" # shellcheck disable=SC2034 EXCLUDE_PATH_02="" # --------------------------------- # Do Not Modify: Script essentials # --------------------------------- # Script versioning - check for updates CURRENT_VERSION="0.1.0" # Function to check the latest version check_latest_version() { LATEST_VERSION=$(curl -fsSL --connect-timeout 5 --max-time 10 "https://api.github.com/repos/edbfi/mover-status/releases" | jq -r .[0].tag_name) || LATEST_VERSION="" } # Initialize to -1 to ensure 0% notification LAST_NOTIFIED=-1 # --------------------------------------------------------- # Do Not Modify: Variable checking! # --------------------------------------------------------- # Check if at least one notification method is enabled if ! $USE_TELEGRAM && ! $USE_DISCORD && ! $USE_PUSHOVER && ! $USE_APPRISE && ! $USE_UNRAID; then log "Error: All notification methods are disabled. At least one must be enabled." exit 1 fi # Check webhook configurations conditionally if [[ $USE_TELEGRAM == true ]]; then if [ -z "$TELEGRAM_BOT_TOKEN" ] || [ -z "$TELEGRAM_CHAT_ID" ]; then log "Error: Telegram settings not configured correctly." exit 1 fi fi if [[ $USE_DISCORD == true ]]; then if ! [[ $DISCORD_WEBHOOK_URL =~ ^https://(discord\.com|discordapp\.com)/api/webhooks/ ]]; then log "Error: Invalid Discord webhook URL." exit 1 fi fi if [[ $USE_PUSHOVER == true ]]; then if [ -z "$PUSHOVER_APP_TOKEN" ] || [ "$PUSHOVER_APP_TOKEN" = "xxxx" ] || [ -z "$PUSHOVER_USER_KEY" ] || [ "$PUSHOVER_USER_KEY" = "xxxx" ]; then log "Error: Pushover settings not configured correctly." exit 1 fi fi if [[ $USE_UNRAID == true ]]; then if [ ! -x "$UNRAID_NOTIFY_BIN" ]; then log "Error: Unraid notify executable is not available at: $UNRAID_NOTIFY_BIN" exit 1 fi fi if [[ $USE_APPRISE == true ]]; then case "$APPRISE_MODE" in cli) if [ ! -x "$APPRISE_BIN" ]; then log "Error: Apprise CLI is not executable at: $APPRISE_BIN" exit 1 fi ;; api) if ! [[ "$APPRISE_API_URL" =~ ^https?://[^[:space:]]+$ ]]; then log "Error: APPRISE_API_URL must be a valid http:// or https:// URL." exit 1 fi ;; *) log "Error: Unsupported APPRISE_MODE '$APPRISE_MODE'. Supported modes: cli, api." exit 1 ;; esac if [ "${#APPRISE_TARGETS[@]}" -eq 0 ]; then log "Error: USE_APPRISE is true but APPRISE_TARGETS is empty." exit 1 fi for apprise_target in "${APPRISE_TARGETS[@]}"; do if [ -z "$apprise_target" ]; then log "Error: APPRISE_TARGETS contains an empty target." exit 1 fi done fi # Send a Pushover notification and fail on transport/API errors send_pushover() { local message="$1" local response if ! response=$(/usr/bin/curl -fsS \ --connect-timeout 10 \ --max-time 30 \ --form-string "token=$PUSHOVER_APP_TOKEN" \ --form-string "user=$PUSHOVER_USER_KEY" \ --form-string "title=$PUSHOVER_TITLE" \ --form-string "message=$message" \ "https://api.pushover.net/1/messages.json" 2>&1); then log "Error: Failed to send Pushover notification: $response" return 1 fi if ! printf '%s' "$response" | jq -e '.status == 1' > /dev/null 2>&1; then log "Error: Pushover returned an unsuccessful response: $response" return 1 fi if $ENABLE_DEBUG; then log "Pushover response: $response" fi return 0 } # Send one Apprise target. Target URLs are deliberately never written to logs. send_apprise_target() { local target_index=$1 local title=$2 local message=$3 local notification_type=${4:-info} local rc case "$APPRISE_MODE" in cli) if "$APPRISE_BIN" \ --title "$title" \ --body "$message" \ --notification-type "$notification_type" \ --input-format text \ "${APPRISE_TARGETS[$target_index]}" > /dev/null 2>&1; then if $ENABLE_DEBUG; then log "Apprise target $((target_index + 1)) delivered successfully via CLI." fi return 0 else rc=$? log "Error: Apprise target $((target_index + 1)) failed via CLI (exit code $rc)." return 1 fi ;; api) local payload local api_endpoint="${APPRISE_API_URL%/}/notify" if ! payload=$(jq -cn \ --arg url "${APPRISE_TARGETS[$target_index]}" \ --arg title "$title" \ --arg body "$message" \ --arg type "$notification_type" \ '{urls: [$url], title: $title, body: $body, type: $type, format: "text"}'); then log "Error: Failed to build Apprise API request for target $((target_index + 1))." return 1 fi local http_code if http_code=$(printf '%s' "$payload" | /usr/bin/curl -fsS \ --connect-timeout 10 \ --max-time 30 \ -H "Content-Type: application/json" \ --data-binary @- \ -o /dev/null \ -w "%{http_code}" \ "$api_endpoint" 2>/dev/null); then if [[ "$http_code" =~ ^2[0-9]{2}$ ]]; then if $ENABLE_DEBUG; then log "Apprise target $((target_index + 1)) delivered successfully via API." fi return 0 fi log "Error: Apprise target $((target_index + 1)) failed via API (unexpected HTTP status $http_code)." return 1 else rc=$? log "Error: Apprise target $((target_index + 1)) failed via API (curl exit code $rc)." return 1 fi ;; esac } # True when at least one built-in direct notification channel is enabled. direct_notifications_enabled() { $USE_TELEGRAM || $USE_DISCORD || $USE_PUSHOVER || $USE_UNRAID } # Validate DU_POLL_INTERVAL is a positive integer if ! [[ "$DU_POLL_INTERVAL" =~ ^[0-9]+$ ]] || [ "$DU_POLL_INTERVAL" -eq 0 ]; then log "Error: DU_POLL_INTERVAL must be a positive integer. Got: '$DU_POLL_INTERVAL'" exit 1 fi # Validate CACHE_PATH exists if [ ! -d "$CACHE_PATH" ]; then log "Error: CACHE_PATH directory does not exist: '$CACHE_PATH'" exit 1 fi # Check latest version once at startup (after validation so we don't hit the API on misconfiguration) LATEST_VERSION="" check_latest_version # --------------------------------------------------------- # Do Not Modify: Dry-run check # --------------------------------------------------------- if $DRY_RUN; then log "Running in dry-run mode. No real monitoring will be performed." # Detect data source for informational purposes if [ -f "/usr/local/emhttp/state/mover.ini" ] && grep -q "TotalToSecondary" "/usr/local/emhttp/state/mover.ini" 2>/dev/null; then log "Dry-run: mover.ini detected (Mover Tuning plugin available)" dry_run_data_source="mover_ini" else log "Dry-run: Using du polling mode (no mover.ini found)" dry_run_data_source="du_polling" fi # Simulate data for notification dry_run_percent=50 # Arbitrary progress percentage for testing dry_run_remaining_data="500 GB" # Arbitrary remaining data amount for testing dry_run_datetime=$(date +"%B %d (%Y) - %H:%M:%S") dry_run_etc_discord="" dry_run_etc_telegram="01/01/2099, 12pm" dry_run_etc_pushover="01/01/2099, 12pm" dry_run_etc_apprise="01/01/2099, 12pm" dry_run_etc_unraid="01/01/2099, 12pm" # Simulate file info if available dry_run_file_count="" dry_run_current_file="" if $ENABLE_FILE_INFO && [ "$dry_run_data_source" = "mover_ini" ]; then dry_run_file_count="898/1796 files" dry_run_current_file="/mnt/cache/share/example_file.txt" fi # Determine color based on percentage if [ "$dry_run_percent" -le 34 ]; then dry_run_color=16744576 # Light Red elif [ "$dry_run_percent" -le 65 ]; then dry_run_color=16753920 # Light Orange else dry_run_color=9498256 # Light Green fi # Footer text with version checking footer_text="Version: v${CURRENT_VERSION}" if [[ -n "${LATEST_VERSION}" && "${LATEST_VERSION}" != "${CURRENT_VERSION}" ]]; then footer_text+=" (update available)" fi # Prepare messages with placeholders dry_run_value_message_discord="${DISCORD_MOVING_MESSAGE//\{percent\}/$dry_run_percent}" dry_run_value_message_discord="${dry_run_value_message_discord//\{remaining_data\}/$dry_run_remaining_data}" dry_run_value_message_discord="${dry_run_value_message_discord//\{etc\}/$dry_run_etc_discord}" dry_run_value_message_discord="${dry_run_value_message_discord//\{file_count\}/$dry_run_file_count}" dry_run_value_message_discord="${dry_run_value_message_discord//\{current_file\}/$dry_run_current_file}" dry_run_value_message_telegram="${TELEGRAM_MOVING_MESSAGE//\{percent\}/$dry_run_percent}" dry_run_value_message_telegram="${dry_run_value_message_telegram//\{remaining_data\}/$dry_run_remaining_data}" dry_run_value_message_telegram="${dry_run_value_message_telegram//\{etc\}/$dry_run_etc_telegram}" dry_run_value_message_telegram="${dry_run_value_message_telegram//\{file_count\}/$dry_run_file_count}" dry_run_value_message_telegram="${dry_run_value_message_telegram//\{current_file\}/$dry_run_current_file}" dry_run_value_message_telegram+=" ${footer_text}" dry_run_value_message_pushover="${PUSHOVER_MOVING_MESSAGE//\{percent\}/$dry_run_percent}" dry_run_value_message_pushover="${dry_run_value_message_pushover//\{remaining_data\}/$dry_run_remaining_data}" dry_run_value_message_pushover="${dry_run_value_message_pushover//\{etc\}/$dry_run_etc_pushover}" dry_run_value_message_pushover="${dry_run_value_message_pushover//\{file_count\}/$dry_run_file_count}" dry_run_value_message_pushover="${dry_run_value_message_pushover//\{current_file\}/$dry_run_current_file}" dry_run_value_message_pushover+=$'\n\n'"${footer_text}" dry_run_value_message_apprise="${APPRISE_MOVING_MESSAGE//\{percent\}/$dry_run_percent}" dry_run_value_message_apprise="${dry_run_value_message_apprise//\{remaining_data\}/$dry_run_remaining_data}" dry_run_value_message_apprise="${dry_run_value_message_apprise//\{etc\}/$dry_run_etc_apprise}" dry_run_value_message_apprise="${dry_run_value_message_apprise//\{file_count\}/$dry_run_file_count}" dry_run_value_message_apprise="${dry_run_value_message_apprise//\{current_file\}/$dry_run_current_file}" dry_run_value_message_apprise+=$'\n\n'"${footer_text}" dry_run_value_message_unraid="${UNRAID_MOVING_MESSAGE//\{percent\}/$dry_run_percent}" dry_run_value_message_unraid="${dry_run_value_message_unraid//\{remaining_data\}/$dry_run_remaining_data}" dry_run_value_message_unraid="${dry_run_value_message_unraid//\{etc\}/$dry_run_etc_unraid}" dry_run_value_message_unraid="${dry_run_value_message_unraid//\{file_count\}/$dry_run_file_count}" dry_run_value_message_unraid="${dry_run_value_message_unraid//\{current_file\}/$dry_run_current_file}" # Send test notifications if $USE_TELEGRAM; then log "Sending test notification to Telegram..." dry_run_json_payload=$(jq -n \ --arg chat_id "$TELEGRAM_CHAT_ID" \ --arg text "$dry_run_value_message_telegram" \ '{chat_id: $chat_id, text: $text, disable_notification: "false", parse_mode: "HTML"}') /usr/bin/curl -s -o /dev/null -H "Content-Type: application/json" -X POST -d "$dry_run_json_payload" "https://api.telegram.org/bot$TELEGRAM_BOT_TOKEN/sendMessage" fi if $USE_PUSHOVER; then log "Sending test notification to Pushover..." if ! send_pushover "$dry_run_value_message_pushover"; then log "Error: Pushover dry-run notification was not delivered." exit 1 fi fi if $USE_UNRAID; then log "Sending test notification through native Unraid notifications..." if ! "$UNRAID_NOTIFY_BIN" \ -e "$UNRAID_EVENT" \ -s "$UNRAID_TITLE" \ -d "$dry_run_value_message_unraid" \ -i normal; then log "Error: Native Unraid dry-run notification could not be submitted." exit 1 fi fi if $USE_APPRISE; then log "Sending test notification through Apprise to ${#APPRISE_TARGETS[@]} target(s)..." apprise_dry_run_failed=false for apprise_target_index in "${!APPRISE_TARGETS[@]}"; do if ! send_apprise_target "$apprise_target_index" "$APPRISE_TITLE" "$dry_run_value_message_apprise" "info"; then apprise_dry_run_failed=true fi done if $apprise_dry_run_failed; then log "Error: One or more Apprise dry-run notifications were not delivered." exit 1 fi fi if $USE_DISCORD; then log "Sending test notification to Discord..." dry_run_notification_data='{ "username": "'"$DISCORD_NAME_OVERRIDE"'", "content": null, "embeds": [ { "title": "Mover: Moving Data", "description": "This is a test message from dry-run mode (data source: '"$dry_run_data_source"').", "color": '"$dry_run_color"', "fields": [ { "name": "'"$dry_run_datetime"'", "value": "'"${dry_run_value_message_discord}"'" } ], "footer": { "text": "'"$footer_text"'" } } ] }' /usr/bin/curl -s -o /dev/null -H "Content-Type: application/json" -X POST -d "$dry_run_notification_data" "$DISCORD_WEBHOOK_URL" fi log "Dry-run complete. Exiting script." exit 0 fi # --------------------------------------------------------- # Mover Status Script - Do Not Edit! # --------------------------------------------------------- # Prepare exclusion paths declare -a exclude_paths for var_name in "${!EXCLUDE_PATH_@}"; do if [ -n "${!var_name}" ]; then if [ ! -d "${!var_name}" ]; then log "Error: Exclusion path '${!var_name}' (${var_name}) does not exist." exit 1 fi exclude_paths+=("${!var_name}") fi done # Calculate total size of all excluded directories get_excluded_size() { local total=0 local path size for path in "${exclude_paths[@]}"; do if [ -d "$path" ]; then size=$(du -sb "$path" 2>/dev/null | cut -f1) total=$((total + ${size:-0})) fi done echo "$total" } # Check if any mover-related process is running (supports Unraid v7+ and Mover Tuning plugin) is_mover_running() { pgrep -x "mover" > /dev/null 2>&1 && return 0 pgrep -x "age_mover" > /dev/null 2>&1 && return 0 pgrep -f "^/usr/libexec/unraid/move" > /dev/null 2>&1 && return 0 return 1 } # Mover.ini path (written by Mover Tuning plugin's age_mover) MOVER_INI_PATH="/usr/local/emhttp/state/mover.ini" # State persistence paths STATE_DIR="/tmp/mover-status" STATE_FILE="${STATE_DIR}/state" LAST_RUN_FILE="${STATE_DIR}/last-run" # Global data source identifier: "preparing", "mover_ini", or "du_polling" DATA_SOURCE="" # Progress globals (set by get_progress) PROGRESS_PERCENT=0 PROGRESS_REMAINING_BYTES=0 PROGRESS_MOVED_BYTES=0 PROGRESS_TOTAL_BYTES=0 PROGRESS_FILE_COUNT="" PROGRESS_REMAIN_FILES="" PROGRESS_CURRENT_FILE="" # Tracks bytes already moved when script started monitoring (for accurate late-join ETA) monitoring_start_bytes=0 # INI globals (set by read_mover_ini) INI_TOTAL_TO_SECONDARY=0 INI_REMAIN_TO_SECONDARY=0 INI_TOTAL_FILES=0 INI_REMAIN_FILES=0 INI_CURRENT_FILE="" # Load one stable, complete mover.ini snapshot into INI_* globals. # Mover Tuning rewrites this file in place, so progress must never be # calculated from a file that changed while it was being read. load_mover_ini_snapshot() { [ -f "$MOVER_INI_PATH" ] || return 1 [ -n "$mover_start_time" ] || return 1 local before after mod_time snapshot before=$(stat -c '%i:%y:%s' "$MOVER_INI_PATH" 2>/dev/null) || return 1 mod_time=$(stat -c %Y "$MOVER_INI_PATH" 2>/dev/null) || return 1 [ "$mod_time" -ge "$mover_start_time" ] || return 1 # Capture the whole small file using Bash itself, then verify that the # inode/mtime/size did not change while it was being read. snapshot=$(<"$MOVER_INI_PATH") after=$(stat -c '%i:%y:%s' "$MOVER_INI_PATH" 2>/dev/null) || return 1 [ "$before" = "$after" ] || return 1 local total="" remain="" total_files="" remain_files="" local current_file="" key value while IFS='=' read -r key value; do value="${value%\"}" value="${value#\"}" case "$key" in TotalToSecondary) total="$value" ;; RemainToSecondary) remain="$value" ;; TotalFilesToSecondary) total_files="$value" ;; RemainFilesToSecondary) remain_files="$value" ;; File) current_file="$value" ;; esac done <<< "$snapshot" # Byte counters are mandatory for trustworthy progress. [[ "$total" =~ ^[0-9]+$ ]] || return 1 [[ "$remain" =~ ^[0-9]+$ ]] || return 1 [ "$remain" -le "$total" ] || return 1 # File counters are optional, but if either appears require a complete, # sane pair from the same stable snapshot. if [ -n "$total_files" ] || [ -n "$remain_files" ]; then [[ "$total_files" =~ ^[0-9]+$ ]] || return 1 [[ "$remain_files" =~ ^[0-9]+$ ]] || return 1 [ "$remain_files" -le "$total_files" ] || return 1 fi # Publish only after the snapshot has passed every validation check. INI_TOTAL_TO_SECONDARY="$total" INI_REMAIN_TO_SECONDARY="$remain" INI_TOTAL_FILES="$total_files" INI_REMAIN_FILES="$remain_files" INI_CURRENT_FILE="$current_file" } # True only when mover.ini belongs to the currently running mover operation # and contains one stable, complete snapshot. mover_ini_is_current() { load_mover_ini_snapshot } # Detect whether current-run mover.ini data is available. # Mover Tuning may spend significant time preparing/scanning before it writes # progress for the current run; do not fall back to an expensive second du scan. detect_data_source() { if mover_ini_is_current; then DATA_SOURCE="mover_ini" log "Data source: mover.ini (current Mover Tuning run)" elif pgrep -x "age_mover" > /dev/null 2>&1; then DATA_SOURCE="preparing" if [ -f "$MOVER_INI_PATH" ]; then local ini_mtime ini_mtime=$(stat -c %Y "$MOVER_INI_PATH" 2>/dev/null || true) if [ -n "$ini_mtime" ] && [ -n "$mover_start_time" ] && [ "$ini_mtime" -lt "$mover_start_time" ]; then log "Ignoring stale mover.ini from $(date -d "@$ini_mtime" '+%Y-%m-%d %H:%M:%S'); current mover started at $(date -d "@$mover_start_time" '+%Y-%m-%d %H:%M:%S')." fi fi log "Mover Tuning is preparing/scanning; waiting for current-run progress data." else DATA_SOURCE="du_polling" log "Data source: du polling (standard mover)" fi } # Parse one validated mover.ini snapshot into INI_* globals. read_mover_ini() { if ! load_mover_ini_snapshot; then log "Warning: mover.ini is stale, incomplete, or changed while being read; keeping the last valid progress snapshot." return 1 fi # Staleness check: warn if file hasn't been modified in >3x DU_POLL_INTERVAL local mod_time current_time age stale_threshold mod_time=$(stat -c %Y "$MOVER_INI_PATH" 2>/dev/null) || return 0 current_time=$(date +%s) age=$((current_time - mod_time)) stale_threshold=$((DU_POLL_INTERVAL * 3)) if [ "$age" -gt "$stale_threshold" ]; then if is_mover_running; then # Mover is alive — large file transfer likely; only log in debug mode if $ENABLE_DEBUG; then log "mover.ini unchanged for ${age}s — mover still running (likely processing a large file: ${INI_CURRENT_FILE##*/})" fi else log "Warning: mover.ini hasn't been updated in ${age}s (threshold: ${stale_threshold}s) and mover process not found — plugin may have stalled" fi fi return 0 } # Unified progress reader — sets PROGRESS_* globals from either data source get_progress() { if [ "$DATA_SOURCE" = "preparing" ]; then PROGRESS_PERCENT=0 PROGRESS_REMAINING_BYTES=0 PROGRESS_MOVED_BYTES=0 PROGRESS_TOTAL_BYTES=0 PROGRESS_FILE_COUNT="" PROGRESS_REMAIN_FILES="" PROGRESS_CURRENT_FILE="" return 0 elif [ "$DATA_SOURCE" = "mover_ini" ]; then read_mover_ini || return 1 PROGRESS_TOTAL_BYTES="$INI_TOTAL_TO_SECONDARY" PROGRESS_REMAINING_BYTES="$INI_REMAIN_TO_SECONDARY" PROGRESS_MOVED_BYTES=$((INI_TOTAL_TO_SECONDARY - INI_REMAIN_TO_SECONDARY)) if [ "$PROGRESS_MOVED_BYTES" -lt 0 ]; then PROGRESS_MOVED_BYTES=0 fi if [ "$PROGRESS_TOTAL_BYTES" -gt 0 ]; then PROGRESS_PERCENT=$((PROGRESS_MOVED_BYTES * 100 / PROGRESS_TOTAL_BYTES)) if [ "$PROGRESS_PERCENT" -gt 99 ]; then PROGRESS_PERCENT=99 fi else PROGRESS_PERCENT=0 fi # Apply exclusion adjustment (uses initial snapshot — excluded dirs are static during mover run) if [ ${#exclude_paths[@]} -gt 0 ]; then PROGRESS_REMAINING_BYTES=$((PROGRESS_REMAINING_BYTES - initial_excluded_size)) if [ "$PROGRESS_REMAINING_BYTES" -lt 0 ]; then PROGRESS_REMAINING_BYTES=0 fi PROGRESS_TOTAL_BYTES=$((PROGRESS_TOTAL_BYTES - initial_excluded_size)) if [ "$PROGRESS_TOTAL_BYTES" -lt 0 ]; then PROGRESS_TOTAL_BYTES=0 fi PROGRESS_MOVED_BYTES=$((PROGRESS_TOTAL_BYTES - PROGRESS_REMAINING_BYTES)) if [ "$PROGRESS_MOVED_BYTES" -lt 0 ]; then PROGRESS_MOVED_BYTES=0 fi # Recalculate percent if [ "$PROGRESS_TOTAL_BYTES" -gt 0 ]; then PROGRESS_PERCENT=$((PROGRESS_MOVED_BYTES * 100 / PROGRESS_TOTAL_BYTES)) if [ "$PROGRESS_PERCENT" -gt 99 ]; then PROGRESS_PERCENT=99 fi else PROGRESS_PERCENT=0 fi fi PROGRESS_FILE_COUNT="$INI_TOTAL_FILES" PROGRESS_REMAIN_FILES="$INI_REMAIN_FILES" PROGRESS_CURRENT_FILE="$INI_CURRENT_FILE" else # du_polling mode — uses current_size and initial_size (set in main loop) local current_du current_du=$(du -sb "$CACHE_PATH" | cut -f1) if [ ${#exclude_paths[@]} -gt 0 ]; then current_du=$((current_du - initial_excluded_size)) if [ "$current_du" -lt 0 ]; then current_du=0 fi fi PROGRESS_REMAINING_BYTES="$current_du" PROGRESS_TOTAL_BYTES="$initial_size" PROGRESS_MOVED_BYTES=$((initial_size - current_du)) if [ "$PROGRESS_MOVED_BYTES" -lt 0 ]; then PROGRESS_MOVED_BYTES=0 fi if [ "$initial_size" -gt 0 ]; then PROGRESS_PERCENT=$((PROGRESS_MOVED_BYTES * 100 / initial_size)) if [ "$PROGRESS_PERCENT" -lt 0 ]; then PROGRESS_PERCENT=0 elif [ "$PROGRESS_PERCENT" -gt 99 ]; then PROGRESS_PERCENT=99 fi else PROGRESS_PERCENT=0 fi PROGRESS_FILE_COUNT="" PROGRESS_REMAIN_FILES="" PROGRESS_CURRENT_FILE="" fi return 0 } # Get mover PID from pid file or pgrep get_mover_pid() { local pid if [ -f "/var/run/mover.pid" ]; then pid=$(cat /var/run/mover.pid 2>/dev/null) if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then echo "$pid" return 0 fi fi # Fallback to pgrep pid=$(pgrep -x "mover" 2>/dev/null || pgrep -x "age_mover" 2>/dev/null || pgrep -f "^/usr/libexec/unraid/move" 2>/dev/null) if [ -n "$pid" ]; then echo "$pid" return 0 fi return 1 } # Get mover process start time as epoch seconds get_mover_start_time() { local pid=$1 if [ -z "$pid" ]; then return 1 fi # Use stat on /proc/ to get process start time (Linux-specific) local start_time start_time=$(stat -c %Y "/proc/${pid}" 2>/dev/null) if [ -n "$start_time" ]; then echo "$start_time" return 0 fi return 1 } # Create state directory if missing init_state_dir() { if [ ! -d "$STATE_DIR" ]; then mkdir -p "$STATE_DIR" if $ENABLE_DEBUG; then log "Created state directory: $STATE_DIR" fi fi } # Atomically write tracking state to state file save_state() { local tmp_file="${STATE_FILE}.tmp" cat > "$tmp_file" < "$LAST_RUN_FILE" <" elif [[ $platform == "telegram" || $platform == "pushover" || $platform == "apprise" || $platform == "unraid" ]]; then date -d "@${completion_time_estimate}" +"%H:%M on %b %d (%Z)" fi else echo "Calculating..." fi } build_apprise_progress_message() { local percent=$1 local remaining_data=$2 local etc_apprise etc_apprise=$(calculate_etc "$percent" "apprise") local file_count_str="" local current_file_str="" if $ENABLE_FILE_INFO && [ -n "$PROGRESS_FILE_COUNT" ] && [ "$PROGRESS_FILE_COUNT" != "0" ]; then local files_moved=$((PROGRESS_FILE_COUNT - ${PROGRESS_REMAIN_FILES:-0})) file_count_str="${files_moved}/${PROGRESS_FILE_COUNT} files" if [ -n "$PROGRESS_CURRENT_FILE" ]; then current_file_str="$PROGRESS_CURRENT_FILE" fi fi APPRISE_CURRENT_MESSAGE="${APPRISE_MOVING_MESSAGE//\{percent\}/$percent}" APPRISE_CURRENT_MESSAGE="${APPRISE_CURRENT_MESSAGE//\{remaining_data\}/$remaining_data}" APPRISE_CURRENT_MESSAGE="${APPRISE_CURRENT_MESSAGE//\{etc\}/$etc_apprise}" APPRISE_CURRENT_MESSAGE="${APPRISE_CURRENT_MESSAGE//\{file_count\}/$file_count_str}" APPRISE_CURRENT_MESSAGE="${APPRISE_CURRENT_MESSAGE//\{current_file\}/$current_file_str}" local footer_text="Version: v${CURRENT_VERSION}" if [[ -n "${LATEST_VERSION}" && "${LATEST_VERSION}" != "${CURRENT_VERSION}" ]]; then footer_text+=" (update available)" fi APPRISE_CURRENT_MESSAGE+=$'\n\n'"${footer_text}" } init_apprise_retry_state() { apprise_progress_pending=() apprise_completion_pending=() apprise_pending_percent="" apprise_pending_remaining="" apprise_completion_started=false apprise_completion_message="" local i for i in "${!APPRISE_TARGETS[@]}"; do apprise_progress_pending[i]=false apprise_completion_pending[i]=false done } apprise_progress_has_pending() { local i for i in "${!APPRISE_TARGETS[@]}"; do if [[ "${apprise_progress_pending[$i]:-false}" == true ]]; then return 0 fi done return 1 } apprise_completion_has_pending() { local i for i in "${!APPRISE_TARGETS[@]}"; do if [[ "${apprise_completion_pending[$i]:-false}" == true ]]; then return 0 fi done return 1 } send_apprise_progress() { local percent=$1 local remaining_data=$2 local i build_apprise_progress_message "$percent" "$remaining_data" for i in "${!APPRISE_TARGETS[@]}"; do if [[ "${apprise_progress_pending[$i]:-false}" == true ]]; then continue fi if send_apprise_target "$i" "$APPRISE_TITLE" "$APPRISE_CURRENT_MESSAGE" "info"; then log "Apprise target $((i + 1)) notification sent for ${percent}% completion." else apprise_progress_pending[i]=true log "Warning: Apprise target $((i + 1)) will be retried." fi done if apprise_progress_has_pending; then apprise_pending_percent="$percent" apprise_pending_remaining="$remaining_data" return 1 fi apprise_pending_percent="" apprise_pending_remaining="" return 0 } retry_apprise_progress() { local i if ! apprise_progress_has_pending; then return 0 fi build_apprise_progress_message "$apprise_pending_percent" "$apprise_pending_remaining" for i in "${!APPRISE_TARGETS[@]}"; do if [[ "${apprise_progress_pending[$i]:-false}" != true ]]; then continue fi if send_apprise_target "$i" "$APPRISE_TITLE" "$APPRISE_CURRENT_MESSAGE" "info"; then apprise_progress_pending[i]=false log "Apprise target $((i + 1)) retry succeeded for ${apprise_pending_percent}% completion." else log "Warning: Apprise target $((i + 1)) retry failed; will retry again." fi done if apprise_progress_has_pending; then return 1 fi apprise_pending_percent="" apprise_pending_remaining="" return 0 } start_apprise_completion() { local i build_completion_summary apprise_completion_message="$COMPLETION_SUMMARY_APPRISE" for i in "${!APPRISE_TARGETS[@]}"; do if send_apprise_target "$i" "$APPRISE_TITLE" "$apprise_completion_message" "success"; then apprise_completion_pending[i]=false log "Final Apprise notification sent to target $((i + 1))." else apprise_completion_pending[i]=true log "Warning: Final Apprise notification to target $((i + 1)) will be retried." fi done ! apprise_completion_has_pending } retry_apprise_completion() { local i for i in "${!APPRISE_TARGETS[@]}"; do if [[ "${apprise_completion_pending[$i]:-false}" != true ]]; then continue fi if send_apprise_target "$i" "$APPRISE_TITLE" "$apprise_completion_message" "success"; then apprise_completion_pending[i]=false log "Final Apprise notification retry succeeded for target $((i + 1))." else log "Warning: Final Apprise notification retry failed for target $((i + 1)); will retry again." fi done ! apprise_completion_has_pending } send_notification() { local percent=$1 local remaining_data=$2 local pushover_only=${3:-false} local skip_pushover=${4:-false} local message_override=${5:-} local datetime datetime=$(date +"%B %d (%Y) - %H:%M:%S") local etc_discord etc_discord=$(calculate_etc "$percent" "discord") local etc_telegram etc_telegram=$(calculate_etc "$percent" "telegram") local etc_pushover etc_pushover=$(calculate_etc "$percent" "pushover") local etc_unraid etc_unraid=$(calculate_etc "$percent" "unraid") # Prepare file info placeholders local file_count_str="" local current_file_str="" if $ENABLE_FILE_INFO && [ -n "$PROGRESS_FILE_COUNT" ] && [ "$PROGRESS_FILE_COUNT" != "0" ]; then local files_moved=$((PROGRESS_FILE_COUNT - ${PROGRESS_REMAIN_FILES:-0})) file_count_str="${files_moved}/${PROGRESS_FILE_COUNT} files" if [ -n "$PROGRESS_CURRENT_FILE" ]; then current_file_str="$PROGRESS_CURRENT_FILE" fi fi # Prepare the messages using the predefined templates local value_message_discord="${DISCORD_MOVING_MESSAGE//\{percent\}/$percent}" value_message_discord="${value_message_discord//\{remaining_data\}/$remaining_data}" value_message_discord="${value_message_discord//\{etc\}/$etc_discord}" value_message_discord="${value_message_discord//\{file_count\}/$file_count_str}" value_message_discord="${value_message_discord//\{current_file\}/$current_file_str}" local value_message_telegram="${TELEGRAM_MOVING_MESSAGE//\{percent\}/$percent}" value_message_telegram="${value_message_telegram//\{remaining_data\}/$remaining_data}" value_message_telegram="${value_message_telegram//\{etc\}/$etc_telegram}" value_message_telegram="${value_message_telegram//\{file_count\}/$file_count_str}" value_message_telegram="${value_message_telegram//\{current_file\}/$current_file_str}" local value_message_pushover="${PUSHOVER_MOVING_MESSAGE//\{percent\}/$percent}" value_message_pushover="${value_message_pushover//\{remaining_data\}/$remaining_data}" value_message_pushover="${value_message_pushover//\{etc\}/$etc_pushover}" value_message_pushover="${value_message_pushover//\{file_count\}/$file_count_str}" value_message_pushover="${value_message_pushover//\{current_file\}/$current_file_str}" local value_message_unraid="${UNRAID_MOVING_MESSAGE//\{percent\}/$percent}" value_message_unraid="${value_message_unraid//\{remaining_data\}/$remaining_data}" value_message_unraid="${value_message_unraid//\{etc\}/$etc_unraid}" value_message_unraid="${value_message_unraid//\{file_count\}/$file_count_str}" value_message_unraid="${value_message_unraid//\{current_file\}/$current_file_str}" # A preparing/start notification uses the normal transport plumbing but # deliberately does not pretend that percentage data exists yet. if [ -n "$message_override" ]; then value_message_discord="$message_override" value_message_telegram="$message_override" value_message_pushover="$message_override" value_message_unraid="$message_override" fi local footer_text="Version: v${CURRENT_VERSION}" if [[ -n "${LATEST_VERSION}" && "${LATEST_VERSION}" != "${CURRENT_VERSION}" ]]; then footer_text+=" (update available)" fi value_message_telegram+=" ${footer_text}" value_message_pushover+=$'\n\n'"${footer_text}" # Determine the color based on completion and percentage local color if [ "$percent" -ge 100 ] || ! is_mover_running; then build_completion_summary value_message_discord="$COMPLETION_SUMMARY_DISCORD" value_message_telegram="$COMPLETION_SUMMARY_TELEGRAM" value_message_pushover="$COMPLETION_SUMMARY_PUSHOVER" value_message_unraid="$COMPLETION_SUMMARY_UNRAID" color=65280 # Green for completion else if [ "$percent" -le 34 ]; then color=16744576 # Light Red elif [ "$percent" -le 65 ]; then color=16753920 # Light Orange else color=9498256 # Light Green fi fi # Send the notifications log "Sending notification..." local notification_failed=false PUSHOVER_DELIVERY_FAILED=false if ! $pushover_only && $USE_TELEGRAM; then local json_payload json_payload=$(jq -n \ --arg chat_id "$TELEGRAM_CHAT_ID" \ --arg text "$value_message_telegram" \ '{chat_id: $chat_id, text: $text, disable_notification: "false", parse_mode: "HTML"}') if $ENABLE_DEBUG; then log "Preparing to send to Telegram: $json_payload" fi local response response=$(curl -s -H "Content-Type: application/json" -X POST -d "$json_payload" "https://api.telegram.org/bot$TELEGRAM_BOT_TOKEN/sendMessage") if $ENABLE_DEBUG; then log "Telegram response: $response" fi fi if ! $pushover_only && $USE_UNRAID; then if $ENABLE_DEBUG; then log "Submitting native Unraid notification: event='$UNRAID_EVENT', subject='$UNRAID_TITLE', description='$value_message_unraid'" fi if ! "$UNRAID_NOTIFY_BIN" \ -e "$UNRAID_EVENT" \ -s "$UNRAID_TITLE" \ -d "$value_message_unraid" \ -i normal; then # Native notify is a local handoff. Surface a failed handoff to the # caller, but do not incorrectly route it through Pushover retries. log "Warning: Native Unraid notification could not be submitted." notification_failed=true fi fi if ! $skip_pushover && $USE_PUSHOVER; then if $ENABLE_DEBUG; then log "Preparing to send to Pushover: title='$PUSHOVER_TITLE', message='$value_message_pushover'" fi if ! send_pushover "$value_message_pushover"; then notification_failed=true PUSHOVER_DELIVERY_FAILED=true fi fi if ! $pushover_only && $USE_DISCORD; then # Existing Discord templates use literal \n sequences because the old # payload was hand-built JSON. Convert only that legacy newline marker; # jq handles all JSON escaping. An override is already plain text. local discord_field_value="$value_message_discord" if [ -z "$message_override" ]; then discord_field_value=${discord_field_value//\\n/$'\n'} fi local notification_data notification_data=$(jq -cn \ --arg username "$DISCORD_NAME_OVERRIDE" \ --arg field_name "$datetime" \ --arg field_value "$discord_field_value" \ --arg footer "$footer_text" \ --argjson color "$color" \ '{ username: $username, content: null, embeds: [{ title: "Mover: Moving Data", color: $color, fields: [{ name: $field_name, value: $field_value }], footer: { text: $footer } }] }') if $ENABLE_DEBUG; then log "Preparing to send to Discord: $notification_data" fi local response response=$(curl -s -H "Content-Type: application/json" -X POST -d "$notification_data" "$DISCORD_WEBHOOK_URL" -w "\nHTTP status: %{http_code}\nCurl Error: %{errormsg}") if $ENABLE_DEBUG; then log "Discord response: $response" fi fi if $notification_failed; then return 1 fi return 0 } # Send a one-time start notification while Mover Tuning is preparing. # This is informational and best-effort; normal progress/completion delivery # remains authoritative if a transport is temporarily unavailable. send_preparing_notifications() { local notification_failed=false if direct_notifications_enabled; then if send_notification 0 "Calculating..." false false "$PREPARING_MESSAGE"; then log "Mover preparing notification submitted to direct channels." else notification_failed=true log "Warning: Mover preparing notification had a direct-channel delivery failure." fi fi if $USE_APPRISE; then local i for i in "${!APPRISE_TARGETS[@]}"; do if send_apprise_target "$i" "$APPRISE_TITLE" "$PREPARING_MESSAGE" "info"; then log "Mover preparing notification sent to Apprise target $((i + 1))." else notification_failed=true log "Warning: Mover preparing notification failed for Apprise target $((i + 1))." fi done fi ! $notification_failed } # Send the first percentage notification once trustworthy progress exists. send_initial_notifications() { local percent=$1 local remaining_readable=$2 if direct_notifications_enabled; then if send_notification "$percent" "$remaining_readable"; then log "Initial direct notification attempt completed at ${percent}%." else log "Warning: Initial direct notification had a delivery failure." if [[ "$PUSHOVER_DELIVERY_FAILED" == true ]]; then log "Pushover will be retried." pushover_retry_pending=true pending_percent="$percent" pending_remaining="$remaining_readable" fi fi fi if $USE_APPRISE; then if ! send_apprise_progress "$percent" "$remaining_readable"; then log "Warning: One or more initial Apprise targets are pending retry." fi fi LAST_NOTIFIED=$((percent / NOTIFICATION_INCREMENT * NOTIFICATION_INCREMENT)) } # Initialize state directory for crash recovery init_state_dir # Main Script Execution Loop while true; do log "Monitoring new mover process..." # Wait for the mover process to start log "Mover process not found, waiting to start monitoring..." while ! is_mover_running; do sleep 10 done log "Mover process found, starting monitoring..." # Get mover identity before choosing a progress source so stale mover.ini # from a previous Mover Tuning run can be rejected. mover_pid=$(get_mover_pid) || mover_pid="" mover_start_time=$(get_mover_start_time "$mover_pid") || mover_start_time="" # Detect data source (preparing vs current mover.ini vs du polling) detect_data_source # Capture initial excluded size (used for consistent total adjustment) initial_excluded_size=0 if [ ${#exclude_paths[@]} -gt 0 ]; then initial_excluded_size=$(get_excluded_size) if [ "$initial_excluded_size" -gt 0 ]; then log "Excluding $(human_readable "$initial_excluded_size") from ${#exclude_paths[@]} path(s)" fi fi # Notification retry state pushover_retry_pending=false pending_percent="" pending_remaining="" completion_retry_pending=false standard_completion_done=false init_apprise_retry_state # Try to resume from saved state (crash recovery) if load_state; then log "Resuming monitoring from saved state — skipping 0% notification" # Get fresh progress data get_progress remaining_readable=$(human_readable "$PROGRESS_REMAINING_BYTES") percent="$PROGRESS_PERCENT" else # Fresh start — determine initial size if [ "$DATA_SOURCE" = "mover_ini" ]; then get_progress initial_size="$PROGRESS_TOTAL_BYTES" elif [ "$DATA_SOURCE" = "preparing" ]; then initial_size=0 else initial_size=$(du -sb "$CACHE_PATH" | cut -f1) initial_size=$((initial_size - initial_excluded_size)) if [ "$initial_size" -lt 0 ]; then initial_size=0 fi fi if [ "$DATA_SOURCE" = "preparing" ]; then initial_readable="Calculating..." log "Initial total size unavailable while Mover Tuning is preparing." else initial_readable=$(human_readable "$initial_size") log "Initial total size of data: $initial_readable" fi start_time=$(date +%s) log "Monitoring started at: $(date -d "@$start_time" '+%Y-%m-%d %H:%M:%S')" # Check for late-join (mover already running before script started) if [ "$DATA_SOURCE" = "preparing" ]; then monitoring_start_bytes=0 percent=0 remaining_readable="Calculating..." elif [ "$DATA_SOURCE" = "mover_ini" ] && [ "$PROGRESS_MOVED_BYTES" -gt 0 ]; then monitoring_start_bytes="$PROGRESS_MOVED_BYTES" log "Late join detected — mover already $(( PROGRESS_PERCENT ))% complete (using mover.ini data, baseline: $(human_readable "$monitoring_start_bytes"))" percent="$PROGRESS_PERCENT" remaining_readable=$(human_readable "$PROGRESS_REMAINING_BYTES") elif [ "$DATA_SOURCE" = "du_polling" ] && [ -n "$mover_start_time" ]; then monitoring_start_bytes=0 script_time=$(date +%s) if [ $((script_time - mover_start_time)) -gt 60 ]; then log "Late join detected — progress relative to cache size at script start" fi percent=0 remaining_readable="$initial_readable" else monitoring_start_bytes=0 percent=0 remaining_readable="$initial_readable" fi LAST_NOTIFIED=-1 # While Mover Tuning is preparing, announce the run without inventing # a percentage. Normal percentage notifications begin with fresh data. if [ "$DATA_SOURCE" = "preparing" ]; then send_preparing_notifications || true else send_initial_notifications "$percent" "$remaining_readable" fi fi # Monitor the progress last_du_time=0 while true; do current_time=$(date +%s) # Switch to mover.ini as soon as Mover Tuning writes current-run data. if [ "$DATA_SOURCE" = "preparing" ] && is_mover_running && mover_ini_is_current; then DATA_SOURCE="mover_ini" log "Fresh mover.ini detected; switching to Mover Tuning progress tracking." get_progress initial_size="$PROGRESS_TOTAL_BYTES" remaining_readable=$(human_readable "$PROGRESS_REMAINING_BYTES") percent="$PROGRESS_PERCENT" monitoring_start_bytes="$PROGRESS_MOVED_BYTES" if [ "$monitoring_start_bytes" -gt 0 ]; then log "Mover already ${percent}% complete when current-run progress became available." fi if [ "$percent" -gt 0 ]; then send_initial_notifications "$percent" "$remaining_readable" else log "Fresh mover.ini reports 0%; preparing notification already announced this run, suppressing duplicate 0% notification." LAST_NOTIFIED=0 fi save_state last_du_time=$current_time fi # Only recalculate progress when DU_POLL_INTERVAL has passed if [ $((current_time - last_du_time)) -ge "$DU_POLL_INTERVAL" ]; then get_progress remaining_readable=$(human_readable "$PROGRESS_REMAINING_BYTES") percent="$PROGRESS_PERCENT" last_du_time=$current_time # Save state for crash recovery save_state if $ENABLE_DEBUG; then log "Progress poll [${DATA_SOURCE}]: percent=$percent, moved=${PROGRESS_MOVED_BYTES}, remaining=${PROGRESS_REMAINING_BYTES}, total=${PROGRESS_TOTAL_BYTES}" if [ -n "$PROGRESS_FILE_COUNT" ] && [ "$PROGRESS_FILE_COUNT" != "0" ]; then log " Files: ${PROGRESS_REMAIN_FILES}/${PROGRESS_FILE_COUNT} remaining" fi fi fi # Check if the mover process is still running if ! is_mover_running; then log "Mover process is no longer running." # If Mover Tuning produced current-run data just as the wrapper exited, # use it for final stats without emitting a late progress notification. if [ "$DATA_SOURCE" = "preparing" ] && mover_ini_is_current; then DATA_SOURCE="mover_ini" log "Current-run mover.ini became available at completion; using it for final stats." fi # Final progress read — captures mover.ini final state before it goes stale get_progress remaining_readable=$(human_readable "$PROGRESS_REMAINING_BYTES") log "Total data moved: ${PROGRESS_MOVED_BYTES} bytes, Total: ${PROGRESS_TOTAL_BYTES} bytes." if [[ "$standard_completion_done" != true ]]; then if ! direct_notifications_enabled; then standard_completion_done=true elif [[ "$completion_retry_pending" == true ]]; then if send_notification 100 "$remaining_readable" true; then log "Final Pushover notification sent after retry." completion_retry_pending=false standard_completion_done=true else log "Warning: Final Pushover notification delivery failed; retry remains pending." fi elif send_notification 100 "$remaining_readable"; then log "Final direct notification attempt completed." standard_completion_done=true else log "Warning: Final direct notification had a delivery failure." if [[ "$PUSHOVER_DELIVERY_FAILED" == true ]]; then log "Final Pushover notification will be retried." completion_retry_pending=true else # Native Unraid submission is a one-shot local handoff. # The failure has been surfaced; avoid duplicating healthy # direct channels by resending the whole completion message. standard_completion_done=true fi fi fi if $USE_APPRISE; then if [[ "$apprise_completion_started" != true ]]; then apprise_completion_started=true if ! start_apprise_completion; then log "Warning: One or more final Apprise targets are pending retry." fi elif apprise_completion_has_pending; then if ! retry_apprise_completion; then log "Warning: One or more final Apprise targets remain pending." fi fi fi apprise_completion_done=true if $USE_APPRISE && apprise_completion_has_pending; then apprise_completion_done=false fi if [[ "$standard_completion_done" == true && "$apprise_completion_done" == true ]]; then save_last_run LAST_NOTIFIED=-1 log "Final notification processing complete; monitoring loop exiting." break fi log "Completion notification delivery still pending; retrying in 5 seconds." sleep 5 continue fi # Send notifications based on increment. A pending Pushover retry must not # stall healthy Telegram/Discord channels. if [ "$((percent / NOTIFICATION_INCREMENT * NOTIFICATION_INCREMENT))" -ge $((LAST_NOTIFIED + NOTIFICATION_INCREMENT)) ]; then log "Condition met for sending update: Current percent $percent (rounded down to nearest increment: $((percent / NOTIFICATION_INCREMENT * NOTIFICATION_INCREMENT))) >= Last notified $LAST_NOTIFIED + Increment $NOTIFICATION_INCREMENT" if direct_notifications_enabled; then if [[ "$pushover_retry_pending" == true ]]; then # Pushover is already pending, so send only to the healthy direct channels. if $USE_TELEGRAM || $USE_DISCORD || $USE_UNRAID; then if send_notification "$percent" "$remaining_readable" false true; then log "Direct notification attempt completed for $percent% on non-Pushover channels." else log "Warning: A non-Pushover direct notification failed for $percent% completion." fi fi # Coalesce the pending Pushover retry to the newest progress update. pending_percent="$percent" pending_remaining="$remaining_readable" log "Pending Pushover retry updated to ${percent}% completion." elif send_notification "$percent" "$remaining_readable"; then log "Direct notification attempt completed for $percent% completion." else log "Warning: Direct notification for $percent% had a delivery failure." if [[ "$PUSHOVER_DELIVERY_FAILED" == true ]]; then log "Pushover will be retried." pushover_retry_pending=true pending_percent="$percent" pending_remaining="$remaining_readable" fi fi fi if $USE_APPRISE; then if ! send_apprise_progress "$percent" "$remaining_readable"; then log "Warning: One or more Apprise targets are pending retry for ${percent}% completion." fi fi # Healthy-channel scheduling is independent of transport delivery state. LAST_NOTIFIED="$((percent / NOTIFICATION_INCREMENT * NOTIFICATION_INCREMENT))" fi # Retry only Pushover. This runs after the increment branch so any pending # message is first coalesced to the newest progress value. if [[ "$pushover_retry_pending" == true ]]; then if send_notification "$pending_percent" "$pending_remaining" true; then log "Pushover notification retry succeeded for ${pending_percent}% completion." pushover_retry_pending=false pending_percent="" pending_remaining="" else log "Warning: Pushover notification retry failed; will retry again." fi fi if $USE_APPRISE && apprise_progress_has_pending; then retry_apprise_progress || true fi sleep 5 # Check every 5 seconds; progress only recalculated per DU_POLL_INTERVAL done # Delay before restarting monitoring log "Restarting monitoring after completion..." sleep 10 done # Mover Status Script # # This script monitors the progress of the "Mover" process and posts updates to Discord, Telegram, Pushover, Apprise, and/or native Unraid notifications. # Copyright (C) 2024 - engels74 # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published # by the Free Software Foundation, either version 3 of the License, or any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Affero General Public License for more details. # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . # # Contact: https://github.com/edbfi/mover-status/issues