#!/usr/bin/env bash # P9 rollback — runbook §5 decision-tree execution. # # Three minutes from "we're rolling back" to "DNS points at old VPS, # new VPS is in maintenance mode, old stack is healthy". # # Usage: # ./scripts/p9-rollback.sh --reason="login broken 5%/min" # # Optional (defaults shown): # --cf-token=$CF_API_TOKEN # --cf-zone-id=$CF_ZONE_ID # --cf-record-id=$CF_RECORD_ID # --old-vps-ip= (CF A-record rollback target) # --old-vps-host=animego-old (~/.ssh/config alias: /id_rsa) # --new-vps-ip= # --new-vps-host=animego-new (~/.ssh/config alias: /) # # Env: # CI_AUTO=1 skip y/N confirmation (use carefully) # # Exit codes: # 0 — DNS flipped + old VPS healthy + new VPS in maintenance # 1 — any step failed; partial state described in log # 2 — bad args / preflight set -euo pipefail # --- log files -------------------------------------------------------- TS="$(date +%Y%m%d-%H%M%S)" LOG_FILE="/tmp/p9-rollback-${TS}.log" # Audit log — runbook §5 calls this out explicitly so each rollback # leaves a permanent breadcrumb beyond the per-run /tmp logs. AUDIT_LOG="/tmp/p9-rollback.log" # --- colors ----------------------------------------------------------- if [ -t 1 ] && command -v tput >/dev/null 2>&1 && [ "$(tput colors 2>/dev/null || echo 0)" -ge 8 ]; then C_GREEN="$(tput setaf 2)" C_RED="$(tput setaf 1)" C_YELLOW="$(tput setaf 3)" C_BG_RED="$(tput setab 1)" C_WHITE="$(tput setaf 7)" C_BOLD="$(tput bold)" C_RESET="$(tput sgr0)" else C_GREEN=""; C_RED=""; C_YELLOW=""; C_BG_RED=""; C_WHITE=""; C_BOLD=""; C_RESET="" fi ok() { printf "%s[OK]%s %s\n" "$C_GREEN" "$C_RESET" "$1"; } warn() { printf "%s[WARN]%s %s\n" "$C_YELLOW" "$C_RESET" "$1"; } fail() { printf "%s[FAIL]%s %s\n" "$C_RED" "$C_RESET" "$1"; } info() { printf "%s[..]%s %s\n" "$C_BOLD" "$C_RESET" "$1"; } exec > >(tee -a "$LOG_FILE") 2>&1 info "logging to $LOG_FILE" info "audit log: $AUDIT_LOG" # --- args -------------------------------------------------------------- REASON="" CF_TOKEN="${CF_API_TOKEN:-}" CF_ZONE="${CF_ZONE_ID:-}" CF_RECORD="${CF_RECORD_ID:-}" OLD_IP="${OLD_VPS_IP:-}" # CF A-record rollback target (DNS flips back here) NEW_IP="${NEW_VPS_IP:-}" # display only # SSH via ~/.ssh/config aliases. old=/id_rsa, new=/. # Single OLD_PORT= default was wrong for the old VPS — fixed 2026-05-29. OLD_HOST="animego-old" NEW_HOST="animego-new" for arg in "$@"; do case "$arg" in --reason=*) REASON="${arg#--reason=}" ;; --cf-token=*) CF_TOKEN="${arg#--cf-token=}" ;; --cf-zone-id=*) CF_ZONE="${arg#--cf-zone-id=}" ;; --cf-record-id=*) CF_RECORD="${arg#--cf-record-id=}" ;; --old-vps-ip=*) OLD_IP="${arg#--old-vps-ip=}" ;; --old-vps-host=*) OLD_HOST="${arg#--old-vps-host=}" ;; --new-vps-ip=*) NEW_IP="${arg#--new-vps-ip=}" ;; --new-vps-host=*) NEW_HOST="${arg#--new-vps-host=}" ;; --help|-h) printf "usage: %s --reason=\"\" [--cf-token=... --cf-zone-id=... --cf-record-id=...] [--old-vps-ip=...] [--new-vps-ip=...]\n" "$0" exit 0 ;; *) fail "unknown arg: $arg"; exit 2 ;; esac done if [ -z "$REASON" ]; then fail "--reason=\"\" is required (logged for post-mortem)" exit 2 fi if [ -z "$CF_TOKEN" ] || [ -z "$CF_ZONE" ] || [ -z "$CF_RECORD" ]; then fail "Cloudflare creds missing — need --cf-token + --cf-zone-id + --cf-record-id (or CF_API_TOKEN/CF_ZONE_ID/CF_RECORD_ID env)" exit 2 fi # ====================================================================== # BANNER # ====================================================================== printf "\n" printf "%s%s%s %s\n" "$C_BG_RED" "$C_WHITE" "$C_BOLD" "$C_RESET" printf "%s%s%s P9 ROLLBACK INITIATED %s\n" "$C_BG_RED" "$C_WHITE" "$C_BOLD" "$C_RESET" printf "%s%s%s reason: %-55s%s\n" "$C_BG_RED" "$C_WHITE" "$C_BOLD" "$REASON" "$C_RESET" printf "%s%s%s timestamp: %s %s\n" "$C_BG_RED" "$C_WHITE" "$C_BOLD" "$TS" "$C_RESET" printf "%s%s%s %s\n" "$C_BG_RED" "$C_WHITE" "$C_BOLD" "$C_RESET" printf "\n" # --- audit log entry (permanent) -------------------------------------- printf "%s\trollback initiated\treason=%s\n" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$REASON" >> "$AUDIT_LOG" # ====================================================================== # CONFIRMATION # ====================================================================== confirm() { local action="$1" printf "%s[ABOUT TO]%s %s\n" "$C_YELLOW" "$C_RESET" "$action" if [ "${CI_AUTO:-0}" = "1" ]; then info "CI_AUTO=1 — auto-continuing" return 0 fi sleep 3 local reply read -p "Continue? (y/N) " -r reply if [[ ! "$reply" =~ ^[Yy]$ ]]; then fail "user aborted rollback at: $action" printf "%s\trollback aborted at: %s\n" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$action" >> "$AUDIT_LOG" exit 1 fi } confirm "execute full rollback — flip DNS, restart nginx on both VPSes" ANY_FAIL=0 # ====================================================================== # STEP 1 — Cloudflare API: PATCH A record back to old VPS # ====================================================================== info "STEP 1/4 — Cloudflare PATCH A record -> $OLD_IP" CF_URL="https://api.cloudflare.com/client/v4/zones/${CF_ZONE}/dns_records/${CF_RECORD}" CF_PAYLOAD="$(printf '{"content":"%s","ttl":60}' "$OLD_IP")" # Print the command we're about to run (with token masked so the operator # can copy-paste if the script aborts). printf "%s[CMD]%s curl -X PATCH '%s' -H 'Authorization: Bearer ***' -H 'Content-Type: application/json' --data '%s'\n" \ "$C_BOLD" "$C_RESET" "$CF_URL" "$CF_PAYLOAD" CF_RESP_FILE="/tmp/p9-rollback-cf-${TS}.json" CF_CODE="$(curl -s \ -o "$CF_RESP_FILE" \ -w "%{http_code}" \ -X PATCH "$CF_URL" \ -H "Authorization: Bearer ${CF_TOKEN}" \ -H "Content-Type: application/json" \ --data "$CF_PAYLOAD" || echo "000")" info "CF API response (HTTP $CF_CODE):" cat "$CF_RESP_FILE" || true printf "\n" if [ "$CF_CODE" = "200" ]; then # Verify CF says success=true if grep -q '"success":true' "$CF_RESP_FILE" 2>/dev/null; then ok "DNS A record flipped to $OLD_IP (TTL 60s)" else fail "CF returned 200 but success!=true — check the JSON above" ANY_FAIL=1 fi else fail "CF API returned HTTP $CF_CODE" warn "MANUAL FALLBACK: run the curl above with the real token, or use the Cloudflare dashboard" ANY_FAIL=1 fi printf "%s\tCF PATCH\tHTTP=%s\tip=%s\n" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$CF_CODE" "$OLD_IP" >> "$AUDIT_LOG" # ====================================================================== # STEP 2 — verify old VPS stack is healthy # ====================================================================== info "STEP 2/4 — verify old VPS ($OLD_IP) stack is healthy" OLD_SSH=(ssh -o ConnectTimeout=10 "$OLD_HOST") printf "%s[CMD]%s %s 'cd /opt/animego && docker compose ps'\n" "$C_BOLD" "$C_RESET" "${OLD_SSH[*]}" OLD_PS="/tmp/p9-rollback-old-ps-${TS}.txt" if "${OLD_SSH[@]}" 'cd /opt/animego && docker compose ps' > "$OLD_PS" 2>&1; then cat "$OLD_PS" # Look for at least app + mongodb in "Up"/"running" state. APP_UP="$(grep -E '^(app|next-app)' "$OLD_PS" | grep -cE '(Up|running|healthy)' || true)" MONGO_UP="$(grep -E '^mongodb' "$OLD_PS" | grep -cE '(Up|running|healthy)' || true)" if [ "${APP_UP:-0}" -ge 1 ] && [ "${MONGO_UP:-0}" -ge 1 ]; then ok "old VPS: app + mongodb running" else fail "old VPS: app or mongodb NOT in Up state — DNS now points at a broken stack" warn "MANUAL: SSH in and start the old stack:" warn " ${OLD_SSH[*]} 'cd /opt/animego && docker compose up -d'" ANY_FAIL=1 fi else fail "ssh to old VPS failed — could not verify health" warn "MANUAL: ${OLD_SSH[*]} 'cd /opt/animego && docker compose ps'" ANY_FAIL=1 fi # Per runbook §5 step 2 (and decision matrix), if the old VPS is still # in maintenance mode from the cutover dry-run, swap it back to legacy. # We don't know the current state remotely, so we attempt the swap and # tolerate the "already legacy" case (it's a plain cp + restart). info "ensuring old VPS nginx is on default.legacy.conf" printf "%s[CMD]%s %s 'cd /opt/animego && cp nginx/default.legacy.conf nginx/default.conf && docker compose restart nginx'\n" \ "$C_BOLD" "$C_RESET" "${OLD_SSH[*]}" if "${OLD_SSH[@]}" 'cd /opt/animego && cp nginx/default.legacy.conf nginx/default.conf && docker compose restart nginx'; then ok "old VPS nginx restored to legacy config" else fail "old VPS nginx restart failed — check legacy.conf exists or whether nginx is even running" ANY_FAIL=1 fi printf "%s\told VPS verify\tapp_up=%s\tmongo_up=%s\n" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "${APP_UP:-?}" "${MONGO_UP:-?}" >> "$AUDIT_LOG" # ====================================================================== # STEP 3 — new VPS: put nginx in maintenance mode # ====================================================================== info "STEP 3/4 — new VPS ($NEW_IP) into maintenance mode" NEW_SSH=(ssh -o ConnectTimeout=10 "$NEW_HOST") NEW_CMD='cd /opt/animego && cp nginx/default.maintenance.conf nginx/default.conf && docker compose restart nginx' printf "%s[CMD]%s %s '%s'\n" "$C_BOLD" "$C_RESET" "${NEW_SSH[*]}" "$NEW_CMD" confirm "swap new VPS nginx to maintenance.conf and restart nginx" if "${NEW_SSH[@]}" "$NEW_CMD"; then ok "new VPS nginx now serving maintenance.html (503 for ISP-cached clients)" else fail "new VPS maintenance swap failed" warn "MANUAL: ${NEW_SSH[*]} '$NEW_CMD'" ANY_FAIL=1 fi printf "%s\tnew VPS maintenance\n" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "$AUDIT_LOG" # ====================================================================== # STEP 4 — data loss warning + post-rollback checklist # ====================================================================== printf "\n" printf "%s%s========== DATA LOSS WARNING ==========%s\n" "$C_RED" "$C_BOLD" "$C_RESET" printf "%sAll writes to the NEW stack between T+0 DNS flip and now are LOST.%s\n" "$C_YELLOW" "$C_RESET" printf "%sReason: new PG data cannot flow back into the old Mongo schema.%s\n" "$C_YELLOW" "$C_RESET" printf "%sRunbook §5 estimate: <100 writes lost if rollback within 30 min of cutover.%s\n" "$C_YELLOW" "$C_RESET" printf "\n" printf "%s========== POST-ROLLBACK CHECKLIST ==========%s\n" "$C_BOLD" "$C_RESET" cat <> "$AUDIT_LOG" exit 0 else fail "P9 ROLLBACK: PARTIAL — at least one step needs manual follow-up (see warnings above)" info "log: $LOG_FILE" info "audit: $AUDIT_LOG" printf "%s\trollback complete\tstatus=partial\n" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "$AUDIT_LOG" exit 1 fi