#!/usr/bin/env bash
# ADR-0001 Phase 3 Cutover — rollback.sh
#
# STANDALONE and callable independently AT ANY POINT — not just as
# cutover.sh's failure handler. Detects current state and takes the
# minimum action to restore the pre-cutover configuration.
#
# Run DIRECTLY on docker-2 via SSH:
#   ssh root@192.168.4.32
#   bash /volume1/docker/compose-files/postgresql/cutover/rollback.sh ["reason text"]
#
# Idempotent: safe to run multiple times, safe to run if nothing is
# actually mid-cutover (it will detect that and exit cleanly).
#
# ⚠️  IMPORTANT ASYMMETRY — READ BEFORE RUNNING MANUALLY LONG AFTER A
#     CUTOVER YOU BELIEVED SUCCEEDED:
# Once legacy has been stopped (cutover.sh Phase 10) AND the new leader
# (patroni-1) has been genuinely serving writes for a meaningful period,
# "rolling back" is NOT a free flag-flip anymore — it means demoting a
# leader that may hold real production writes newer than legacy's last
# known state, and restoring legacy would risk DISCARDING those writes.
# This script defends against that specific danger with a grace-window
# check (see "Case E" below): if legacy is stopped and the new leader has
# been up longer than GRACE_WINDOW_SECONDS, this script REFUSES to
# automatically restore legacy and instead prints manual guidance. This
# is deliberate — do not bypass it by hand without actually reconciling
# data first (e.g. exporting recent writes from patroni-1 before touching
# anything).
set -uo pipefail

REASON="${1:-<no reason given — invoked directly>}"
CUTOVER_DIR="/volume1/docker/compose-files/postgresql/cutover"
HA_STACK="postgresqlha"
LEGACY_STACK="postgresql"
LEGACY_SERVICE="postgresql_postgresql"
NETWORK_NAME="postgresql_db-backend"
GRACE_WINDOW_SECONDS=300   # 5 minutes — matches the kind of near-immediate
                           # failure cutover.sh's own Phase 10 auto-rollback
                           # path would trigger. Anything older than this is
                           # treated as "cutover had already stuck" and
                           # requires explicit manual handling, not blind
                           # automation.

# Consumer checks reused from cutover.sh's Phase 9 list, kept in sync
# manually — see RUNBOOK.md section 1 dependency map if this list changes.
declare -A CONSUMER_HEALTH_URLS=(
  ["auth_authentik-server"]="https://auth.bryanmail.net/-/health/live/"
  ["git_gitea-server"]="https://git.bryanmail.net/api/healthz"
  ["woodpecker_woodpecker-server"]="https://woodpecker.bryanmail.net/healthz"
)

log()  { echo "[rollback $(date +%H:%M:%S)] $*"; }
warn() { echo "[rollback WARN $(date +%H:%M:%S)] $*" >&2; }
die()  { echo "[rollback FATAL $(date +%H:%M:%S)] $*" >&2; exit 1; }

log "=== rollback.sh invoked ==="
log "Reason: ${REASON}"

legacy_cid() { docker ps -q --filter "name=${LEGACY_SERVICE}" | head -1; }

http_get_via_probe() {
  # Uses a disposable probe container rather than assuming legacy is up —
  # rollback.sh must work even when legacy has already been stopped.
  local host="$1" port="$2" path="$3"
  docker run --rm --network "${NETWORK_NAME}" alpine:3 sh -c "
    apk add -q --no-cache bash >/dev/null 2>&1
    exec 3<>/dev/tcp/${host}/${port} || exit 99
    printf 'GET ${path} HTTP/1.0\r\nHost: ${host}\r\nConnection: close\r\n\r\n' >&3
    cat <&3
  " 2>/dev/null
}

# ── Step 0: detect current state ────────────────────────────────────────
log "--- Detecting current state ---"

HA_STACK_EXISTS=0
if docker stack ls --format '{{.Name}}' 2>/dev/null | grep -qx "${HA_STACK}"; then
  HA_STACK_EXISTS=1
fi
log "postgresqlha stack exists: ${HA_STACK_EXISTS}"

LEGACY_REPLICAS=$(docker service inspect "${LEGACY_SERVICE}" --format '{{.Spec.Mode.Replicated.Replicas}}' 2>/dev/null || echo "unknown")
log "Legacy service (${LEGACY_SERVICE}) desired replicas: ${LEGACY_REPLICAS}"

LEGACY_READONLY="unknown"
LEGACY_CID="$(legacy_cid)"
if [ -n "$LEGACY_CID" ]; then
  LEGACY_READONLY=$(docker exec "$LEGACY_CID" bash -c 'psql -U "$POSTGRES_USER" -tAc "SHOW default_transaction_read_only;"' 2>/dev/null | tr -d '[:space:]')
fi
log "Legacy read-only state: ${LEGACY_READONLY}"

PATRONI_LEADER_ROLE="unreachable"
PATRONI_LEADER_SINCE=""
if [ "$HA_STACK_EXISTS" -eq 1 ]; then
  RESP=$(http_get_via_probe patroni-1 8008 "/patroni")
  if echo "$RESP" | grep -q '"role"'; then
    PATRONI_LEADER_ROLE=$(echo "$RESP" | grep -o '"role"[^,}]*' | sed 's/.*: *"//;s/"$//')
  fi
fi
log "patroni-1 role (if reachable): ${PATRONI_LEADER_ROLE}"

ALIAS_LIVE=0
if [ -n "$LEGACY_CID" ]; then
  RESOLVE_OUT=$(docker exec "$LEGACY_CID" bash -c "getent hosts postgresql" 2>/dev/null)
elif [ "$HA_STACK_EXISTS" -eq 1 ]; then
  RESOLVE_OUT=$(docker run --rm --network "${NETWORK_NAME}" alpine:3 sh -c "getent hosts postgresql" 2>/dev/null)
else
  RESOLVE_OUT=""
fi
NUM_IPS=$(echo "$RESOLVE_OUT" | awk 'NF{print $1}' | sort -u | wc -l)
log "'postgresql' currently resolves to ${NUM_IPS} distinct IP(s): ${RESOLVE_OUT}"
if [ "$NUM_IPS" -ge 2 ] || { [ "$LEGACY_REPLICAS" = "0" ] && [ "$HA_STACK_EXISTS" -eq 1 ]; }; then
  ALIAS_LIVE=1
fi

# ── Step 1: classify and act ─────────────────────────────────────────────

if [ "$HA_STACK_EXISTS" -eq 0 ]; then
  log "Case A: no postgresqlha stack exists. Nothing to roll back."
  if [ "$LEGACY_REPLICAS" != "1" ]; then
    warn "Legacy replicas = ${LEGACY_REPLICAS}, expected 1 even with no HA stack present. Restoring."
    docker service scale "${LEGACY_SERVICE}=1"
  fi
  if [ "$LEGACY_READONLY" = "on" ]; then
    warn "Legacy is read-only with no HA stack present — reversing (this should not normally happen)."
    docker exec "$(legacy_cid)" bash -c 'psql -U "$POSTGRES_USER" -c "ALTER SYSTEM SET default_transaction_read_only = off; SELECT pg_reload_conf();"'
  fi
  log "=== Rollback complete: nothing was in progress. Pre-cutover state confirmed/restored. ==="
  exit 0
fi

if [ "$LEGACY_REPLICAS" = "0" ]; then
  # Legacy has been stopped — this is Case E territory. Must determine
  # whether we're in the "just happened, cutover.sh's own auto-rollback"
  # window, or a genuinely-completed-and-aged cutover.
  log "Legacy is scaled to 0 — checking how long patroni-1 has held the leader role before deciding how to proceed..."
  SINCE_INFO=$(http_get_via_probe patroni-1 8008 "/patroni")
  # Patroni's /patroni response includes a timestamp; if we can't parse
  # one confidently, fail SAFE (treat as "aged", require manual review)
  # rather than guessing young.
  LEADER_TIMESTAMP=$(echo "$SINCE_INFO" | grep -o '"timestamp"[^,}]*' | sed 's/.*: *"//;s/"$//')
  AGE_SECONDS=999999
  if [ -n "$LEADER_TIMESTAMP" ]; then
    LEADER_EPOCH=$(date -d "$LEADER_TIMESTAMP" +%s 2>/dev/null || echo "")
    NOW_EPOCH=$(date +%s)
    if [ -n "$LEADER_EPOCH" ]; then
      AGE_SECONDS=$((NOW_EPOCH - LEADER_EPOCH))
    fi
  fi
  log "Estimated age signal: ${AGE_SECONDS}s (grace window: ${GRACE_WINDOW_SECONDS}s)"

  if [ "$AGE_SECONDS" -gt "$GRACE_WINDOW_SECONDS" ]; then
    die "
════════════════════════════════════════════════════════════════════════
REFUSING to automatically roll back: legacy is stopped and the new
leader (patroni-1) appears to have been running longer than the
${GRACE_WINDOW_SECONDS}s grace window. Automatically restoring legacy
now risks DISCARDING real production writes that landed on patroni-1
since promotion.

This requires MANUAL handling:
  1. Confirm whether patroni-1 genuinely has newer data than legacy's
     last synced point (check application data directly, or compare
     row counts/timestamps on tables you know are actively written).
  2. If patroni-1's data is authoritative (i.e. the cutover basically
     succeeded and this is a LATER problem, not a failed cutover):
       - Do NOT run this rollback path. Treat this as 'fix forward' —
         investigate why rollback.sh was invoked and address that
         specific problem instead (e.g. a single consumer's connection
         string, not the whole DB layer).
  3. If you genuinely need to revert to legacy despite this:
       - Take a fresh pg_dumpall from patroni-1 FIRST (same method as
         preflight.sh), so no data is lost even if you proceed.
       - Manually restore that dump into legacy before scaling it back
         up, OR manually apply just the delta if you can identify it.
       - Then re-run this script, or reproduce the case-D steps below
         by hand once legacy has legacy's data superseded correctly.

Reason this rollback.sh run was invoked: ${REASON}
════════════════════════════════════════════════════════════════════════
"
  fi

  log "Case E (within grace window — treating as an immediate post-Phase-10 failure, matches cutover.sh's own auto-rollback trigger point). Proceeding automatically."
  log "Restoring legacy service to 1 replica..."
  docker service scale "${LEGACY_SERVICE}=1"
  log "Waiting for legacy to become healthy..."
  for i in $(seq 1 24); do
    STATE=$(docker service ps "${LEGACY_SERVICE}" --filter "desired-state=running" --format '{{.CurrentState}}' 2>/dev/null | head -1)
    log "  legacy state: ${STATE}"
    echo "$STATE" | grep -q "^Running" && break
    sleep 5
  done
  LEGACY_CID="$(legacy_cid)"
  if [ -z "$LEGACY_CID" ]; then
    die "Legacy service was scaled back to 1 but no running container found after waiting. MANUAL INTERVENTION REQUIRED — check 'docker service logs ${LEGACY_SERVICE} --tail 100' and 'docker stack ps ${LEGACY_STACK} --no-trunc' directly."
  fi
  log "Legacy container back: ${LEGACY_CID}"
  # fall through to alias-removal and HA-stack-teardown below, shared with case D
fi

if [ "$LEGACY_READONLY" = "on" ]; then
  log "Reversing legacy read-only flag..."
  LEGACY_CID="$(legacy_cid)"
  docker exec "$LEGACY_CID" bash -c 'psql -U "$POSTGRES_USER" -c "ALTER SYSTEM SET default_transaction_read_only = off; SELECT pg_reload_conf();"'
  VERIFY=$(docker exec "$LEGACY_CID" bash -c 'psql -U "$POSTGRES_USER" -tAc "SHOW default_transaction_read_only;"' 2>/dev/null | tr -d '[:space:]')
  if [ "$VERIFY" != "off" ]; then
    die "Attempted to reverse read-only flag but SHOW still reports '${VERIFY}'. MANUAL INTERVENTION REQUIRED — do not leave legacy read-only if consumers are pointed at it."
  fi
  log "Confirmed: legacy is read-write again."
fi

if [ "$ALIAS_LIVE" -eq 1 ]; then
  log "postgresql/db aliases are currently held by haproxy — removing by tearing down the HA stack entirely below (simplest reliable removal; redeploying staging-without-alias was considered but full teardown is more certain and this stack has no data worth preserving once we're rolling back)."
fi

log "--- Tearing down postgresqlha stack ---"
docker stack rm "${HA_STACK}"
log "Waiting for tasks to actually finish shutting down (docker stack rm returns immediately, tasks take several seconds)..."
for i in $(seq 1 24); do
  REMAINING=$(docker stack ps "${HA_STACK}" 2>/dev/null | grep -c . || true)
  if [ "$REMAINING" -eq 0 ]; then
    break
  fi
  sleep 5
done
REMAINING=$(docker stack ps "${HA_STACK}" 2>/dev/null | grep -c . || true)
if [ "$REMAINING" -gt 0 ]; then
  warn "postgresqlha still shows ${REMAINING} task(s) after 120s wait. Proceeding to wipe data dirs anyway is UNSAFE while tasks are still shutting down — stopping here."
  die "MANUAL INTERVENTION REQUIRED: check 'docker stack ps ${HA_STACK} --no-trunc' before wiping any data dirs by hand."
fi
log "Confirmed: postgresqlha stack fully removed."

log "--- Wiping postgresqlha data dirs ---"
for d in etcd-1-data etcd-2-data etcd-3-data patroni-0-data patroni-1-data; do
  DIR="/volume1/docker/PostgreSQL/${d}"
  if [ -d "$DIR" ]; then
    rm -rf "$DIR"
    mkdir -p "$DIR"
    log "  wiped and recreated: ${DIR}"
  fi
done
log "Data dirs wiped (mirrors the pgha-test dry-run teardown precedent — parent dirs recreated with mkdir -p since Swarm bind mounts don't auto-create missing host parent dirs)."

log "--- Re-verifying consumers against restored legacy ---"
CONSUMER_FAIL=0
for svc in "${!CONSUMER_HEALTH_URLS[@]}"; do
  url="${CONSUMER_HEALTH_URLS[$svc]}"
  CODE=$(curl -sk -o /dev/null -w '%{http_code}' --max-time 10 "$url")
  log "  ${svc} -> ${url} -> HTTP ${CODE}"
  [ "$CODE" != "200" ] && { warn "  ${svc} did not return 200"; CONSUMER_FAIL=1; }
done
if [ "$CONSUMER_FAIL" -ne 0 ]; then
  warn "One or more consumers did not verify healthy post-rollback. This is now a DIFFERENT problem than the original cutover attempt — legacy itself may need direct attention. Check 'docker service logs <service> --tail 100' for each flagged consumer."
else
  log "All checked consumers verified healthy against restored legacy."
fi

FINAL_RESOLVE=$(docker exec "$(legacy_cid)" bash -c "getent hosts postgresql" 2>/dev/null)
log "Final 'postgresql' resolution: ${FINAL_RESOLVE}"

log "=== ROLLBACK COMPLETE ==="
log "Summary:"
log "  - postgresqlha stack: removed, data dirs wiped and recreated empty"
log "  - legacy service: running, read-write, sole answer for 'postgresql'/'db'"
log "  - consumer verification: $([ "$CONSUMER_FAIL" -eq 0 ] && echo 'all passed' || echo 'SOME FAILED — see warnings above, needs manual follow-up')"
log "  - original invocation reason: ${REASON}"
log "Legacy's data directory itself (/volume1/docker/PostgreSQL/data-17) was"
log "never touched by this script — only postgresqlha's own dirs were wiped."
exit 0