#!/usr/bin/env bash # P9 preflight — runbook §7 (T-3d + T-1d operator checklists) automated. # # Catches operator mistakes BEFORE the cutover window opens. Every check # either passes (OK), warns (recoverable, still proceeds), or fails (must # fix before the next gate). Exit code reflects the FAIL count. # # Usage: # ./scripts/p9-preflight.sh # run everything (default) # ./scripts/p9-preflight.sh --t-3d # only T-3d checks (1-13) # ./scripts/p9-preflight.sh --t-1d # only T-1d checks (14-23) # ./scripts/p9-preflight.sh --all # explicit all # # Env (T-1d checks require these; T-3d ignores them): # CF_API_TOKEN Cloudflare API token (or --cf-token=...) # CF_ZONE_ID Cloudflare zone ID (or --cf-zone-id=...) # CF_RECORD_ID A-record ID to flip (or --cf-record-id=...) # CI_AUTO=1 skip interactive prompts (auto-yes on confirmations) # # Exit codes: # 0 — all selected checks PASS (warnings allowed) # 1 — at least one FAIL — fix before next gate # 2 — bad arguments / unable to start set -euo pipefail # --- repo root -------------------------------------------------------- SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" # --- log file --------------------------------------------------------- TS="$(date +%Y%m%d-%H%M%S)" LOG_FILE="/tmp/p9-preflight-${TS}.log" # --- color helpers (TTY-aware) --------------------------------------- if [ -t 1 ] && command -v tput >/dev/null 2>&1 && [ "$(tput colors 2>/dev/null || echo 0)" -ge 8 ]; then C_GREEN="$(tput setaf 2)" C_RED="$(tput setaf 1)" C_YELLOW="$(tput setaf 3)" C_BOLD="$(tput bold)" C_RESET="$(tput sgr0)" else C_GREEN=""; C_RED=""; C_YELLOW=""; C_BOLD=""; C_RESET="" fi ok() { printf "%s[OK]%s %s\n" "$C_GREEN" "$C_RESET" "$1"; PASS_COUNT=$((PASS_COUNT + 1)); } warn() { printf "%s[WARN]%s %s\n" "$C_YELLOW" "$C_RESET" "$1"; WARN_COUNT=$((WARN_COUNT + 1)); } fail() { printf "%s[FAIL]%s %s\n" "$C_RED" "$C_RESET" "$1"; FAIL_COUNT=$((FAIL_COUNT + 1)); } info() { printf "%s[..]%s %s\n" "$C_BOLD" "$C_RESET" "$1"; } # Print a command before running it (mirrors p9-dry-run.sh style). # Writes to stderr so callers can still capture command stdout via $(...). show_cmd() { printf "%s[CMD]%s %s\n" "$C_BOLD" "$C_RESET" "$*" >&2; } # Section header section() { printf "\n%s========== %s ==========%s\n" "$C_BOLD" "$1" "$C_RESET" } PASS_COUNT=0 WARN_COUNT=0 FAIL_COUNT=0 # Tee both stdout AND stderr through the log AFTER the colors are set. # show_cmd writes to stderr (so $(...) captures stay clean), so we have # to tee both streams independently to land everything in the log. exec > >(tee -a "$LOG_FILE") exec 2> >(tee -a "$LOG_FILE" >&2) info "logging to $LOG_FILE" # --- args ------------------------------------------------------------- MODE="all" CF_TOKEN="${CF_API_TOKEN:-}" CF_ZONE="${CF_ZONE_ID:-}" CF_RECORD="${CF_RECORD_ID:-}" OLD_IP="${OLD_VPS_IP:-}" NEW_IP="${NEW_VPS_IP:-}" # SSH via ~/.ssh/config aliases. Old and new differ in BOTH port and key # (old = /id_rsa, new = /), so the previous # single --ssh-port= default silently broke every old-VPS check. # Verified + fixed 2026-05-29. OLD_HOST="animego-old" NEW_HOST="animego-new" for arg in "$@"; do case "$arg" in --t-3d) MODE="t-3d" ;; --t-1d) MODE="t-1d" ;; --all) MODE="all" ;; --cf-token=*) CF_TOKEN="${arg#--cf-token=}" ;; --cf-zone-id=*) CF_ZONE="${arg#--cf-zone-id=}" ;; --cf-record-id=*) CF_RECORD="${arg#--cf-record-id=}" ;; --old-vps-ip=*) OLD_IP="${arg#--old-vps-ip=}" ;; --new-vps-ip=*) NEW_IP="${arg#--new-vps-ip=}" ;; --old-vps-host=*) OLD_HOST="${arg#--old-vps-host=}" ;; --new-vps-host=*) NEW_HOST="${arg#--new-vps-host=}" ;; --help|-h) cat < Cloudflare API token (env: CF_API_TOKEN) --cf-zone-id= Cloudflare zone ID (env: CF_ZONE_ID) --cf-record-id= A-record ID to flip (env: CF_RECORD_ID) --old-vps-ip= default: (CF A-record compare only) --new-vps-ip= default: --old-vps-host= default: animego-old (~/.ssh/config: /id_rsa) --new-vps-host= default: animego-new (~/.ssh/config: /) env: CI_AUTO=1 skip interactive confirmations (auto-yes) exit codes: 0 all selected checks PASS (warnings allowed) 1 at least one FAIL — fix before next gate 2 bad arguments / unable to start EOF exit 0 ;; *) fail "unknown arg: $arg"; exit 2 ;; esac done info "mode: $MODE" # --- SSH helper (centralised flags to keep checks short) ------------- ssh_old() { show_cmd "ssh $OLD_HOST '$*'" ssh -o ConnectTimeout=10 "$OLD_HOST" "$@" } ssh_new() { show_cmd "ssh $NEW_HOST '$*'" ssh -o ConnectTimeout=10 "$NEW_HOST" "$@" } # --- confirm helper (CI_AUTO=1 → auto-yes) --------------------------- confirm() { local prompt="$1" if [ "${CI_AUTO:-0}" = "1" ]; then info "CI_AUTO=1 — auto-yes for: $prompt" return 0 fi local reply read -p "$prompt (y/N) " -r reply [[ "$reply" =~ ^[Yy]$ ]] } # ====================================================================== # T-3d preflight (checks 1-13) # ====================================================================== run_t_3d() { section "T-3d preflight (checks 1-13)" # ---- 1. Local docker daemon up ---- info "CHECK 1/13 — local docker daemon" show_cmd "docker info >/dev/null" if docker info >/dev/null 2>&1; then ok "docker daemon reachable" else fail "docker daemon not reachable — start Docker Desktop / colima" fi # ---- 2. .env.production exists + POSTGRES_PASSWORD set ---- info "CHECK 2/13 — .env.production + POSTGRES_PASSWORD" local env_file="$REPO_ROOT/.env.production" if [ ! -f "$env_file" ]; then fail ".env.production not found at $env_file" else local pg_pass pg_pass="$(grep -E '^POSTGRES_PASSWORD=' "$env_file" | head -1 | cut -d= -f2- | tr -d '"' | tr -d "'" || true)" if [ -z "$pg_pass" ]; then fail "POSTGRES_PASSWORD missing or empty in .env.production" else ok ".env.production present + POSTGRES_PASSWORD set (length=${#pg_pass})" fi fi # ---- 3. SSH to old VPS ---- info "CHECK 3/13 — SSH to old VPS ($OLD_IP)" local who if who="$(ssh_old 'whoami' 2>/dev/null)" && [ "$who" = "root" ]; then ok "old VPS SSH OK (whoami=root)" else fail "old VPS SSH failed (got: '$who')" fi # ---- 4. SSH to new VPS ---- info "CHECK 4/13 — SSH to new VPS ($NEW_IP)" if who="$(ssh_new 'whoami' 2>/dev/null)" && [ "$who" = "root" ]; then ok "new VPS SSH OK (whoami=root)" else fail "new VPS SSH failed (got: '$who')" fi # ---- 5. Old VPS has /opt/animego + docker-compose.yml + mongodb running ---- info "CHECK 5/13 — old VPS /opt/animego stack present + mongodb running" local old_ps if old_ps="$(ssh_old 'cd /opt/animego && ls docker-compose.yml >/dev/null 2>&1 && docker compose ps mongodb' 2>&1)"; then if echo "$old_ps" | grep -qE '(Up|running|healthy)'; then ok "old VPS mongodb container is Up" else fail "old VPS mongodb container NOT Up — output: $(echo "$old_ps" | tail -1)" fi else fail "old VPS /opt/animego missing or compose failed" fi # ---- 6. Mongo container responds to ping ---- info "CHECK 6/13 — old VPS mongo ping" local ping_resp ping_resp="$(ssh_old 'cd /opt/animego && docker compose exec -T mongodb mongosh --quiet --eval "db.runCommand({ping:1})"' 2>&1 || true)" if echo "$ping_resp" | grep -qE '"?ok"?\s*:\s*1'; then ok "mongo ping returned ok:1" else fail "mongo ping did not return ok:1 — got: $(echo "$ping_resp" | tail -3 | tr '\n' ' ')" fi # ---- 7. Mongo dataSize estimate (timing helper, never fails) ---- info "CHECK 7/13 — estimate mongo dataSize (timing helper)" local stats stats="$(ssh_old 'cd /opt/animego && docker compose exec -T mongodb mongosh --quiet --eval "JSON.stringify(db.stats())"' 2>&1 || true)" # mongosh prefixes/suffixes occasionally; extract the JSON line. local json json="$(printf '%s\n' "$stats" | grep -oE '\{.*"dataSize".*\}' | head -1 || true)" if [ -n "$json" ]; then # python is the most portable JSON parser on macOS/Linux without jq. local mb mb="$(printf '%s' "$json" | python3 -c 'import json,sys; d=json.loads(sys.stdin.read()); print(round(d.get("dataSize",0)/1024/1024,1))' 2>/dev/null || echo "?")" ok "mongo dataSize ≈ ${mb} MB (allow ~1 min dump per 100 MB)" else warn "could not parse db.stats() — manual check: ssh + mongosh db.stats()" fi # ---- 8. New VPS disk free > 5GB ---- info "CHECK 8/13 — new VPS disk free" local new_avail new_avail="$(ssh_new 'df -BG --output=avail / | tail -1' 2>/dev/null | tr -d ' G' || echo "0")" if [ -n "$new_avail" ] && [ "$new_avail" -gt 5 ] 2>/dev/null; then ok "new VPS / has ${new_avail}G free" else fail "new VPS / has only ${new_avail}G free (need > 5G)" fi # ---- 9. Old VPS disk free > 5GB ---- info "CHECK 9/13 — old VPS disk free" local old_avail old_avail="$(ssh_old 'df -BG --output=avail / | tail -1' 2>/dev/null | tr -d ' G' || echo "0")" if [ -n "$old_avail" ] && [ "$old_avail" -gt 5 ] 2>/dev/null; then ok "old VPS / has ${old_avail}G free" else fail "old VPS / has only ${old_avail}G free (need > 5G for dump)" fi # ---- 10. default.maintenance.conf exists + matches HEAD ---- info "CHECK 10/13 — nginx/default.maintenance.conf vs HEAD" local mc="$REPO_ROOT/nginx/default.maintenance.conf" if [ ! -f "$mc" ]; then fail "$mc missing" else local disk_sha head_sha disk_sha="$(shasum -a 256 "$mc" | awk '{print $1}')" head_sha="$(cd "$REPO_ROOT" && git show HEAD:nginx/default.maintenance.conf 2>/dev/null | shasum -a 256 | awk '{print $1}' || echo "")" if [ -z "$head_sha" ] || [ "$head_sha" = "$(printf '' | shasum -a 256 | awk '{print $1}')" ]; then fail "nginx/default.maintenance.conf not committed to HEAD" elif [ "$disk_sha" = "$head_sha" ]; then ok "default.maintenance.conf matches HEAD" else fail "default.maintenance.conf differs from HEAD — commit or revert (disk=${disk_sha:0:8} head=${head_sha:0:8})" fi fi # ---- 11. default.legacy.conf on disk + committed ---- info "CHECK 11/13 — nginx/default.legacy.conf present + committed" local lc="$REPO_ROOT/nginx/default.legacy.conf" if [ ! -f "$lc" ]; then fail "$lc missing on disk" elif ! (cd "$REPO_ROOT" && git cat-file -e "HEAD:nginx/default.legacy.conf" 2>/dev/null); then fail "nginx/default.legacy.conf not committed to HEAD" else ok "default.legacy.conf present + committed" fi # ---- 12. smoke-p8.1.sh exists + executable ---- info "CHECK 12/13 — scripts/smoke-p8.1.sh exists + executable" local smoke="$REPO_ROOT/scripts/smoke-p8.1.sh" if [ ! -f "$smoke" ]; then fail "$smoke missing" elif [ ! -x "$smoke" ]; then fail "$smoke not executable (chmod +x)" else ok "smoke-p8.1.sh present + executable" fi # ---- 13. p9-{dry-run,smoke,rollback}.sh exist + executable ---- info "CHECK 13/13 — p9-dry-run.sh / p9-smoke.sh / p9-rollback.sh exist + executable" local missing=0 for s in p9-dry-run.sh p9-smoke.sh p9-rollback.sh; do local p="$REPO_ROOT/scripts/$s" if [ ! -f "$p" ]; then warn "$s missing" missing=1 elif [ ! -x "$p" ]; then warn "$s not executable" missing=1 fi done if [ "$missing" -eq 0 ]; then ok "all three p9 scripts present + executable" else fail "one or more p9 scripts missing/not-executable (see warns above)" fi } # ====================================================================== # T-1d preflight (checks 14-23) # ====================================================================== run_t_1d() { section "T-1d preflight (checks 14-23)" # ---- 14. Dry-run gate passed (log file < 7d, or operator confirms) ---- info "CHECK 14/23 — dry-run gate passed" local recent_log # Find any p9-dry-run-*.log modified in the last 7 days that contains "GATE: PASS". recent_log="$(find /tmp -maxdepth 1 -name 'p9-dry-run-*.log' -mtime -7 -print 2>/dev/null \ | xargs grep -l 'DRY-RUN GATE: PASS' 2>/dev/null | head -1 || true)" if [ -n "$recent_log" ]; then ok "found PASS log: $recent_log" else warn "no PASS log in /tmp from the last 7 days" if confirm "did the T-3d dry-run pass and you have the log?"; then ok "operator confirms dry-run gate passed" else fail "dry-run gate NOT confirmed — run scripts/p9-dry-run.sh first" fi fi # ---- 15. CF API token works ---- info "CHECK 15/23 — Cloudflare token verify" if [ -z "$CF_TOKEN" ]; then fail "CF_API_TOKEN missing (set env or --cf-token=)" else local tok_body tok_code tok_body="/tmp/p9-preflight-cf-tok-${TS}.json" show_cmd "curl -sH 'Authorization: Bearer ***' https://api.cloudflare.com/client/v4/user/tokens/verify" tok_code="$(curl -s -o "$tok_body" -w "%{http_code}" \ -H "Authorization: Bearer ${CF_TOKEN}" \ https://api.cloudflare.com/client/v4/user/tokens/verify || echo "000")" if [ "$tok_code" = "200" ] && grep -q '"status":"active"' "$tok_body"; then ok "CF token active" else fail "CF token verify failed (HTTP $tok_code): $(head -c 200 "$tok_body")" fi fi # ---- 16. CF zone ID resolves ---- info "CHECK 16/23 — Cloudflare zone resolves" if [ -z "$CF_TOKEN" ] || [ -z "$CF_ZONE" ]; then fail "CF_API_TOKEN + CF_ZONE_ID required" else local z_body z_code z_body="/tmp/p9-preflight-cf-zone-${TS}.json" show_cmd "curl -sH 'Authorization: Bearer ***' https://api.cloudflare.com/client/v4/zones/${CF_ZONE}" z_code="$(curl -s -o "$z_body" -w "%{http_code}" \ -H "Authorization: Bearer ${CF_TOKEN}" \ "https://api.cloudflare.com/client/v4/zones/${CF_ZONE}" || echo "000")" if [ "$z_code" = "200" ] && grep -q '"success":true' "$z_body"; then ok "CF zone reachable" else fail "CF zone GET failed (HTTP $z_code): $(head -c 200 "$z_body")" fi fi # ---- 17. CF DNS record exists + currently points at old VPS ---- info "CHECK 17/23 — Cloudflare A-record points at old VPS" if [ -z "$CF_TOKEN" ] || [ -z "$CF_ZONE" ] || [ -z "$CF_RECORD" ]; then fail "CF_API_TOKEN + CF_ZONE_ID + CF_RECORD_ID all required" else local r_body r_code r_body="/tmp/p9-preflight-cf-rec-${TS}.json" show_cmd "curl -sH 'Authorization: Bearer ***' https://api.cloudflare.com/client/v4/zones/${CF_ZONE}/dns_records/${CF_RECORD}" r_code="$(curl -s -o "$r_body" -w "%{http_code}" \ -H "Authorization: Bearer ${CF_TOKEN}" \ "https://api.cloudflare.com/client/v4/zones/${CF_ZONE}/dns_records/${CF_RECORD}" || echo "000")" if [ "$r_code" = "200" ] && grep -q '"success":true' "$r_body"; then local content content="$(python3 -c 'import json,sys; print(json.load(sys.stdin)["result"]["content"])' < "$r_body" 2>/dev/null || echo "")" if [ "$content" = "$OLD_IP" ]; then ok "A-record currently points at old VPS ($content)" else fail "A-record points at '$content', expected '$OLD_IP'" fi else fail "CF DNS record GET failed (HTTP $r_code)" fi fi # ---- 18. New VPS git on origin/feat/go-backend HEAD ---- info "CHECK 18/23 — new VPS git on origin/feat/go-backend HEAD" local local_sha remote_sha local_sha="$(cd "$REPO_ROOT" && git rev-parse HEAD 2>/dev/null || echo "")" remote_sha="$(ssh_new 'cd /opt/animego && git fetch origin feat/go-backend --quiet 2>/dev/null; git rev-parse HEAD' 2>/dev/null || echo "")" if [ -z "$local_sha" ] || [ -z "$remote_sha" ]; then fail "could not read git HEAD (local=$local_sha remote=$remote_sha)" elif [ "$local_sha" = "$remote_sha" ]; then ok "new VPS HEAD matches local ($local_sha)" else fail "new VPS HEAD ($remote_sha) differs from local ($local_sha) — push + pull on VPS" fi # ---- 19. New VPS docker images recent (warn-only) ---- info "CHECK 19/23 — new VPS animego-go-api image age" local image_age image_age="$(ssh_new "docker images animego-go-api --format '{{.CreatedSince}}' | head -1" 2>/dev/null || echo "")" if [ -z "$image_age" ]; then fail "no animego-go-api image on new VPS — run docker compose build" elif echo "$image_age" | grep -qE 'years?|months?|weeks? ago' && ! echo "$image_age" | grep -qE '^1 weeks? ago$'; then warn "animego-go-api image is OLD: $image_age (rebuild recommended)" else ok "animego-go-api image age: $image_age" fi # ---- 20. New VPS postgres healthy ---- info "CHECK 20/23 — new VPS postgres healthy" local pg_ps pg_ps="$(ssh_new 'cd /opt/animego && docker compose ps postgres' 2>&1 || true)" if echo "$pg_ps" | grep -qi 'healthy'; then ok "new VPS postgres is healthy" elif echo "$pg_ps" | grep -qE '(Up|running)'; then warn "new VPS postgres is Up but no healthy marker — may have no healthcheck" else fail "new VPS postgres NOT Up — last line: $(echo "$pg_ps" | tail -1)" fi # ---- 21. New VPS PG schema migrations applied (>= 15 tables) ---- info "CHECK 21/23 — new VPS PG migrations applied" # Expected app tables (14 from go-api/migrations/0001+; system River tables add 3 more). # We assert >= 15 to keep a margin while making sure the schema is real, not empty. local expected_tables="users subscriptions follows danmakus episode_comments episode_windows anime_cache anime_genres anime_studios anime_characters anime_staff anime_relations anime_recommendations anime_episode_titles" local tbl_out tbl_count tbl_out="$(ssh_new 'cd /opt/animego && docker compose exec -T postgres psql -U animego -d animego -c "\\dt" -At' 2>&1 || true)" tbl_count="$(printf '%s\n' "$tbl_out" | grep -cE '^\s*public\|' || true)" if [ "${tbl_count:-0}" -ge 15 ] 2>/dev/null; then ok "PG has ${tbl_count} tables (>= 15)" # Spot-check a few must-have app tables; warn (not fail) on any miss. local missing_tbls="" for t in $expected_tables; do if ! printf '%s\n' "$tbl_out" | grep -qE "^\s*public\|${t}\b"; then missing_tbls="${missing_tbls} $t" fi done if [ -n "$missing_tbls" ]; then warn "expected app table(s) NOT found:${missing_tbls}" fi else fail "PG has only ${tbl_count} tables (need >= 15) — run migrations" fi # ---- 22. REGISTER_DISABLED present in old VPS .env.production ---- info "CHECK 22/23 — REGISTER_DISABLED on old VPS" if ssh_old 'grep -E "^REGISTER_DISABLED=" /opt/animego/.env.production' >/dev/null 2>&1; then ok "REGISTER_DISABLED set on old VPS .env.production" else fail "REGISTER_DISABLED missing — append and restart app (runbook §3 T-1d step 2)" fi # ---- 23. Maintenance banner deployed (curl + operator confirm) ---- info "CHECK 23/23 — site banner '维护' visible" local banner_body banner_code banner_body="/tmp/p9-preflight-banner-${TS}.html" show_cmd "curl -ks -o $banner_body -w '%{http_code}' https://animegoclub.com/" banner_code="$(curl -ks -o "$banner_body" -w '%{http_code}' --max-time 15 https://animegoclub.com/ || echo "000")" if [ "$banner_code" = "200" ] && grep -q '维护' "$banner_body" 2>/dev/null; then ok "banner '维护' found in homepage HTML" else warn "banner '维护' not found in homepage HTML (HTTP $banner_code) — string may be wrapped in JS chunk" if confirm "have you visually confirmed the maintenance banner is live?"; then ok "operator confirms banner deployed" else fail "banner not confirmed — deploy banner before cutover" fi fi } # ====================================================================== # main # ====================================================================== case "$MODE" in t-3d) run_t_3d ;; t-1d) run_t_1d ;; all) run_t_3d; run_t_1d ;; *) fail "unknown mode: $MODE"; exit 2 ;; esac # ====================================================================== # SUMMARY # ====================================================================== section "SUMMARY (mode=$MODE)" printf "passed: %d\n" "$PASS_COUNT" printf "warnings: %d\n" "$WARN_COUNT" printf "failed: %d\n" "$FAIL_COUNT" info "log: $LOG_FILE" if [ "$FAIL_COUNT" -eq 0 ]; then ok "P9 PREFLIGHT ($MODE): PASS" exit 0 else fail "P9 PREFLIGHT ($MODE): FAIL — $FAIL_COUNT check(s) failed, do NOT proceed to next gate" exit 1 fi