ADR-0001 Phase 3: add rollback.sh (standalone, idempotent, grace-window guard against post-cutover data loss)
This commit is contained in:
@@ -0,0 +1,270 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# ADR-0001 Phase 3 Cutover — rollback.sh
|
||||||
|
#
|
||||||
|
# STANDALONE and callable independently AT ANY POINT — not just as
|
||||||
|
# cutover.sh's failure handler. Detects current state and takes the
|
||||||
|
# minimum action to restore the pre-cutover configuration.
|
||||||
|
#
|
||||||
|
# Run DIRECTLY on docker-2 via SSH:
|
||||||
|
# ssh root@192.168.4.32
|
||||||
|
# bash /volume1/docker/compose-files/postgresql/cutover/rollback.sh ["reason text"]
|
||||||
|
#
|
||||||
|
# Idempotent: safe to run multiple times, safe to run if nothing is
|
||||||
|
# actually mid-cutover (it will detect that and exit cleanly).
|
||||||
|
#
|
||||||
|
# ⚠️ IMPORTANT ASYMMETRY — READ BEFORE RUNNING MANUALLY LONG AFTER A
|
||||||
|
# CUTOVER YOU BELIEVED SUCCEEDED:
|
||||||
|
# Once legacy has been stopped (cutover.sh Phase 10) AND the new leader
|
||||||
|
# (patroni-1) has been genuinely serving writes for a meaningful period,
|
||||||
|
# "rolling back" is NOT a free flag-flip anymore — it means demoting a
|
||||||
|
# leader that may hold real production writes newer than legacy's last
|
||||||
|
# known state, and restoring legacy would risk DISCARDING those writes.
|
||||||
|
# This script defends against that specific danger with a grace-window
|
||||||
|
# check (see "Case E" below): if legacy is stopped and the new leader has
|
||||||
|
# been up longer than GRACE_WINDOW_SECONDS, this script REFUSES to
|
||||||
|
# automatically restore legacy and instead prints manual guidance. This
|
||||||
|
# is deliberate — do not bypass it by hand without actually reconciling
|
||||||
|
# data first (e.g. exporting recent writes from patroni-1 before touching
|
||||||
|
# anything).
|
||||||
|
set -uo pipefail
|
||||||
|
|
||||||
|
REASON="${1:-<no reason given — invoked directly>}"
|
||||||
|
CUTOVER_DIR="/volume1/docker/compose-files/postgresql/cutover"
|
||||||
|
HA_STACK="postgresqlha"
|
||||||
|
LEGACY_STACK="postgresql"
|
||||||
|
LEGACY_SERVICE="postgresql_postgresql"
|
||||||
|
NETWORK_NAME="postgresql_db-backend"
|
||||||
|
GRACE_WINDOW_SECONDS=300 # 5 minutes — matches the kind of near-immediate
|
||||||
|
# failure cutover.sh's own Phase 10 auto-rollback
|
||||||
|
# path would trigger. Anything older than this is
|
||||||
|
# treated as "cutover had already stuck" and
|
||||||
|
# requires explicit manual handling, not blind
|
||||||
|
# automation.
|
||||||
|
|
||||||
|
# Consumer checks reused from cutover.sh's Phase 9 list, kept in sync
|
||||||
|
# manually — see RUNBOOK.md section 1 dependency map if this list changes.
|
||||||
|
declare -A CONSUMER_HEALTH_URLS=(
|
||||||
|
["auth_authentik-server"]="https://auth.bryanmail.net/-/health/live/"
|
||||||
|
["git_gitea-server"]="https://git.bryanmail.net/api/healthz"
|
||||||
|
["woodpecker_woodpecker-server"]="https://woodpecker.bryanmail.net/healthz"
|
||||||
|
)
|
||||||
|
|
||||||
|
log() { echo "[rollback $(date +%H:%M:%S)] $*"; }
|
||||||
|
warn() { echo "[rollback WARN $(date +%H:%M:%S)] $*" >&2; }
|
||||||
|
die() { echo "[rollback FATAL $(date +%H:%M:%S)] $*" >&2; exit 1; }
|
||||||
|
|
||||||
|
log "=== rollback.sh invoked ==="
|
||||||
|
log "Reason: ${REASON}"
|
||||||
|
|
||||||
|
legacy_cid() { docker ps -q --filter "name=${LEGACY_SERVICE}" | head -1; }
|
||||||
|
|
||||||
|
http_get_via_probe() {
|
||||||
|
# Uses a disposable probe container rather than assuming legacy is up —
|
||||||
|
# rollback.sh must work even when legacy has already been stopped.
|
||||||
|
local host="$1" port="$2" path="$3"
|
||||||
|
docker run --rm --network "${NETWORK_NAME}" alpine:3 sh -c "
|
||||||
|
apk add -q --no-cache bash >/dev/null 2>&1
|
||||||
|
exec 3<>/dev/tcp/${host}/${port} || exit 99
|
||||||
|
printf 'GET ${path} HTTP/1.0\r\nHost: ${host}\r\nConnection: close\r\n\r\n' >&3
|
||||||
|
cat <&3
|
||||||
|
" 2>/dev/null
|
||||||
|
}
|
||||||
|
|
||||||
|
# ── Step 0: detect current state ────────────────────────────────────────
|
||||||
|
log "--- Detecting current state ---"
|
||||||
|
|
||||||
|
HA_STACK_EXISTS=0
|
||||||
|
if docker stack ls --format '{{.Name}}' 2>/dev/null | grep -qx "${HA_STACK}"; then
|
||||||
|
HA_STACK_EXISTS=1
|
||||||
|
fi
|
||||||
|
log "postgresqlha stack exists: ${HA_STACK_EXISTS}"
|
||||||
|
|
||||||
|
LEGACY_REPLICAS=$(docker service inspect "${LEGACY_SERVICE}" --format '{{.Spec.Mode.Replicated.Replicas}}' 2>/dev/null || echo "unknown")
|
||||||
|
log "Legacy service (${LEGACY_SERVICE}) desired replicas: ${LEGACY_REPLICAS}"
|
||||||
|
|
||||||
|
LEGACY_READONLY="unknown"
|
||||||
|
LEGACY_CID="$(legacy_cid)"
|
||||||
|
if [ -n "$LEGACY_CID" ]; then
|
||||||
|
LEGACY_READONLY=$(docker exec "$LEGACY_CID" bash -c 'psql -U "$POSTGRES_USER" -tAc "SHOW default_transaction_read_only;"' 2>/dev/null | tr -d '[:space:]')
|
||||||
|
fi
|
||||||
|
log "Legacy read-only state: ${LEGACY_READONLY}"
|
||||||
|
|
||||||
|
PATRONI_LEADER_ROLE="unreachable"
|
||||||
|
PATRONI_LEADER_SINCE=""
|
||||||
|
if [ "$HA_STACK_EXISTS" -eq 1 ]; then
|
||||||
|
RESP=$(http_get_via_probe patroni-1 8008 "/patroni")
|
||||||
|
if echo "$RESP" | grep -q '"role"'; then
|
||||||
|
PATRONI_LEADER_ROLE=$(echo "$RESP" | grep -o '"role"[^,}]*' | sed 's/.*: *"//;s/"$//')
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
log "patroni-1 role (if reachable): ${PATRONI_LEADER_ROLE}"
|
||||||
|
|
||||||
|
ALIAS_LIVE=0
|
||||||
|
if [ -n "$LEGACY_CID" ]; then
|
||||||
|
RESOLVE_OUT=$(docker exec "$LEGACY_CID" bash -c "getent hosts postgresql" 2>/dev/null)
|
||||||
|
elif [ "$HA_STACK_EXISTS" -eq 1 ]; then
|
||||||
|
RESOLVE_OUT=$(docker run --rm --network "${NETWORK_NAME}" alpine:3 sh -c "getent hosts postgresql" 2>/dev/null)
|
||||||
|
else
|
||||||
|
RESOLVE_OUT=""
|
||||||
|
fi
|
||||||
|
NUM_IPS=$(echo "$RESOLVE_OUT" | awk 'NF{print $1}' | sort -u | wc -l)
|
||||||
|
log "'postgresql' currently resolves to ${NUM_IPS} distinct IP(s): ${RESOLVE_OUT}"
|
||||||
|
if [ "$NUM_IPS" -ge 2 ] || { [ "$LEGACY_REPLICAS" = "0" ] && [ "$HA_STACK_EXISTS" -eq 1 ]; }; then
|
||||||
|
ALIAS_LIVE=1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ── Step 1: classify and act ─────────────────────────────────────────────
|
||||||
|
|
||||||
|
if [ "$HA_STACK_EXISTS" -eq 0 ]; then
|
||||||
|
log "Case A: no postgresqlha stack exists. Nothing to roll back."
|
||||||
|
if [ "$LEGACY_REPLICAS" != "1" ]; then
|
||||||
|
warn "Legacy replicas = ${LEGACY_REPLICAS}, expected 1 even with no HA stack present. Restoring."
|
||||||
|
docker service scale "${LEGACY_SERVICE}=1"
|
||||||
|
fi
|
||||||
|
if [ "$LEGACY_READONLY" = "on" ]; then
|
||||||
|
warn "Legacy is read-only with no HA stack present — reversing (this should not normally happen)."
|
||||||
|
docker exec "$(legacy_cid)" bash -c 'psql -U "$POSTGRES_USER" -c "ALTER SYSTEM SET default_transaction_read_only = off; SELECT pg_reload_conf();"'
|
||||||
|
fi
|
||||||
|
log "=== Rollback complete: nothing was in progress. Pre-cutover state confirmed/restored. ==="
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$LEGACY_REPLICAS" = "0" ]; then
|
||||||
|
# Legacy has been stopped — this is Case E territory. Must determine
|
||||||
|
# whether we're in the "just happened, cutover.sh's own auto-rollback"
|
||||||
|
# window, or a genuinely-completed-and-aged cutover.
|
||||||
|
log "Legacy is scaled to 0 — checking how long patroni-1 has held the leader role before deciding how to proceed..."
|
||||||
|
SINCE_INFO=$(http_get_via_probe patroni-1 8008 "/patroni")
|
||||||
|
# Patroni's /patroni response includes a timestamp; if we can't parse
|
||||||
|
# one confidently, fail SAFE (treat as "aged", require manual review)
|
||||||
|
# rather than guessing young.
|
||||||
|
LEADER_TIMESTAMP=$(echo "$SINCE_INFO" | grep -o '"timestamp"[^,}]*' | sed 's/.*: *"//;s/"$//')
|
||||||
|
AGE_SECONDS=999999
|
||||||
|
if [ -n "$LEADER_TIMESTAMP" ]; then
|
||||||
|
LEADER_EPOCH=$(date -d "$LEADER_TIMESTAMP" +%s 2>/dev/null || echo "")
|
||||||
|
NOW_EPOCH=$(date +%s)
|
||||||
|
if [ -n "$LEADER_EPOCH" ]; then
|
||||||
|
AGE_SECONDS=$((NOW_EPOCH - LEADER_EPOCH))
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
log "Estimated age signal: ${AGE_SECONDS}s (grace window: ${GRACE_WINDOW_SECONDS}s)"
|
||||||
|
|
||||||
|
if [ "$AGE_SECONDS" -gt "$GRACE_WINDOW_SECONDS" ]; then
|
||||||
|
die "
|
||||||
|
════════════════════════════════════════════════════════════════════════
|
||||||
|
REFUSING to automatically roll back: legacy is stopped and the new
|
||||||
|
leader (patroni-1) appears to have been running longer than the
|
||||||
|
${GRACE_WINDOW_SECONDS}s grace window. Automatically restoring legacy
|
||||||
|
now risks DISCARDING real production writes that landed on patroni-1
|
||||||
|
since promotion.
|
||||||
|
|
||||||
|
This requires MANUAL handling:
|
||||||
|
1. Confirm whether patroni-1 genuinely has newer data than legacy's
|
||||||
|
last synced point (check application data directly, or compare
|
||||||
|
row counts/timestamps on tables you know are actively written).
|
||||||
|
2. If patroni-1's data is authoritative (i.e. the cutover basically
|
||||||
|
succeeded and this is a LATER problem, not a failed cutover):
|
||||||
|
- Do NOT run this rollback path. Treat this as 'fix forward' —
|
||||||
|
investigate why rollback.sh was invoked and address that
|
||||||
|
specific problem instead (e.g. a single consumer's connection
|
||||||
|
string, not the whole DB layer).
|
||||||
|
3. If you genuinely need to revert to legacy despite this:
|
||||||
|
- Take a fresh pg_dumpall from patroni-1 FIRST (same method as
|
||||||
|
preflight.sh), so no data is lost even if you proceed.
|
||||||
|
- Manually restore that dump into legacy before scaling it back
|
||||||
|
up, OR manually apply just the delta if you can identify it.
|
||||||
|
- Then re-run this script, or reproduce the case-D steps below
|
||||||
|
by hand once legacy has legacy's data superseded correctly.
|
||||||
|
|
||||||
|
Reason this rollback.sh run was invoked: ${REASON}
|
||||||
|
════════════════════════════════════════════════════════════════════════
|
||||||
|
"
|
||||||
|
fi
|
||||||
|
|
||||||
|
log "Case E (within grace window — treating as an immediate post-Phase-10 failure, matches cutover.sh's own auto-rollback trigger point). Proceeding automatically."
|
||||||
|
log "Restoring legacy service to 1 replica..."
|
||||||
|
docker service scale "${LEGACY_SERVICE}=1"
|
||||||
|
log "Waiting for legacy to become healthy..."
|
||||||
|
for i in $(seq 1 24); do
|
||||||
|
STATE=$(docker service ps "${LEGACY_SERVICE}" --filter "desired-state=running" --format '{{.CurrentState}}' 2>/dev/null | head -1)
|
||||||
|
log " legacy state: ${STATE}"
|
||||||
|
echo "$STATE" | grep -q "^Running" && break
|
||||||
|
sleep 5
|
||||||
|
done
|
||||||
|
LEGACY_CID="$(legacy_cid)"
|
||||||
|
if [ -z "$LEGACY_CID" ]; then
|
||||||
|
die "Legacy service was scaled back to 1 but no running container found after waiting. MANUAL INTERVENTION REQUIRED — check 'docker service logs ${LEGACY_SERVICE} --tail 100' and 'docker stack ps ${LEGACY_STACK} --no-trunc' directly."
|
||||||
|
fi
|
||||||
|
log "Legacy container back: ${LEGACY_CID}"
|
||||||
|
# fall through to alias-removal and HA-stack-teardown below, shared with case D
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$LEGACY_READONLY" = "on" ]; then
|
||||||
|
log "Reversing legacy read-only flag..."
|
||||||
|
LEGACY_CID="$(legacy_cid)"
|
||||||
|
docker exec "$LEGACY_CID" bash -c 'psql -U "$POSTGRES_USER" -c "ALTER SYSTEM SET default_transaction_read_only = off; SELECT pg_reload_conf();"'
|
||||||
|
VERIFY=$(docker exec "$LEGACY_CID" bash -c 'psql -U "$POSTGRES_USER" -tAc "SHOW default_transaction_read_only;"' 2>/dev/null | tr -d '[:space:]')
|
||||||
|
if [ "$VERIFY" != "off" ]; then
|
||||||
|
die "Attempted to reverse read-only flag but SHOW still reports '${VERIFY}'. MANUAL INTERVENTION REQUIRED — do not leave legacy read-only if consumers are pointed at it."
|
||||||
|
fi
|
||||||
|
log "Confirmed: legacy is read-write again."
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$ALIAS_LIVE" -eq 1 ]; then
|
||||||
|
log "postgresql/db aliases are currently held by haproxy — removing by tearing down the HA stack entirely below (simplest reliable removal; redeploying staging-without-alias was considered but full teardown is more certain and this stack has no data worth preserving once we're rolling back)."
|
||||||
|
fi
|
||||||
|
|
||||||
|
log "--- Tearing down postgresqlha stack ---"
|
||||||
|
docker stack rm "${HA_STACK}"
|
||||||
|
log "Waiting for tasks to actually finish shutting down (docker stack rm returns immediately, tasks take several seconds)..."
|
||||||
|
for i in $(seq 1 24); do
|
||||||
|
REMAINING=$(docker stack ps "${HA_STACK}" 2>/dev/null | grep -c . || true)
|
||||||
|
if [ "$REMAINING" -eq 0 ]; then
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
sleep 5
|
||||||
|
done
|
||||||
|
REMAINING=$(docker stack ps "${HA_STACK}" 2>/dev/null | grep -c . || true)
|
||||||
|
if [ "$REMAINING" -gt 0 ]; then
|
||||||
|
warn "postgresqlha still shows ${REMAINING} task(s) after 120s wait. Proceeding to wipe data dirs anyway is UNSAFE while tasks are still shutting down — stopping here."
|
||||||
|
die "MANUAL INTERVENTION REQUIRED: check 'docker stack ps ${HA_STACK} --no-trunc' before wiping any data dirs by hand."
|
||||||
|
fi
|
||||||
|
log "Confirmed: postgresqlha stack fully removed."
|
||||||
|
|
||||||
|
log "--- Wiping postgresqlha data dirs ---"
|
||||||
|
for d in etcd-1-data etcd-2-data etcd-3-data patroni-0-data patroni-1-data; do
|
||||||
|
DIR="/volume1/docker/PostgreSQL/${d}"
|
||||||
|
if [ -d "$DIR" ]; then
|
||||||
|
rm -rf "$DIR"
|
||||||
|
mkdir -p "$DIR"
|
||||||
|
log " wiped and recreated: ${DIR}"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
log "Data dirs wiped (mirrors the pgha-test dry-run teardown precedent — parent dirs recreated with mkdir -p since Swarm bind mounts don't auto-create missing host parent dirs)."
|
||||||
|
|
||||||
|
log "--- Re-verifying consumers against restored legacy ---"
|
||||||
|
CONSUMER_FAIL=0
|
||||||
|
for svc in "${!CONSUMER_HEALTH_URLS[@]}"; do
|
||||||
|
url="${CONSUMER_HEALTH_URLS[$svc]}"
|
||||||
|
CODE=$(curl -sk -o /dev/null -w '%{http_code}' --max-time 10 "$url")
|
||||||
|
log " ${svc} -> ${url} -> HTTP ${CODE}"
|
||||||
|
[ "$CODE" != "200" ] && { warn " ${svc} did not return 200"; CONSUMER_FAIL=1; }
|
||||||
|
done
|
||||||
|
if [ "$CONSUMER_FAIL" -ne 0 ]; then
|
||||||
|
warn "One or more consumers did not verify healthy post-rollback. This is now a DIFFERENT problem than the original cutover attempt — legacy itself may need direct attention. Check 'docker service logs <service> --tail 100' for each flagged consumer."
|
||||||
|
else
|
||||||
|
log "All checked consumers verified healthy against restored legacy."
|
||||||
|
fi
|
||||||
|
|
||||||
|
FINAL_RESOLVE=$(docker exec "$(legacy_cid)" bash -c "getent hosts postgresql" 2>/dev/null)
|
||||||
|
log "Final 'postgresql' resolution: ${FINAL_RESOLVE}"
|
||||||
|
|
||||||
|
log "=== ROLLBACK COMPLETE ==="
|
||||||
|
log "Summary:"
|
||||||
|
log " - postgresqlha stack: removed, data dirs wiped and recreated empty"
|
||||||
|
log " - legacy service: running, read-write, sole answer for 'postgresql'/'db'"
|
||||||
|
log " - consumer verification: $([ "$CONSUMER_FAIL" -eq 0 ] && echo 'all passed' || echo 'SOME FAILED — see warnings above, needs manual follow-up')"
|
||||||
|
log " - original invocation reason: ${REASON}"
|
||||||
|
log "Legacy's data directory itself (/volume1/docker/PostgreSQL/data-17) was"
|
||||||
|
log "never touched by this script — only postgresqlha's own dirs were wiped."
|
||||||
|
exit 0
|
||||||
Reference in New Issue
Block a user