From 066cbaf97fcf35d6125c0a56b40ffc81ce3f95fa Mon Sep 17 00:00:00 2001 From: admin Date: Sun, 2 Aug 2026 17:38:50 -0700 Subject: [PATCH] ADR-0001 Phase 3: add rollback.sh (standalone, idempotent, grace-window guard against post-cutover data loss) --- postgresql/cutover/rollback.sh | 270 +++++++++++++++++++++++++++++++++ 1 file changed, 270 insertions(+) create mode 100644 postgresql/cutover/rollback.sh diff --git a/postgresql/cutover/rollback.sh b/postgresql/cutover/rollback.sh new file mode 100644 index 0000000..d0e7d26 --- /dev/null +++ b/postgresql/cutover/rollback.sh @@ -0,0 +1,270 @@ +#!/usr/bin/env bash +# ADR-0001 Phase 3 Cutover — rollback.sh +# +# STANDALONE and callable independently AT ANY POINT — not just as +# cutover.sh's failure handler. Detects current state and takes the +# minimum action to restore the pre-cutover configuration. +# +# Run DIRECTLY on docker-2 via SSH: +# ssh root@192.168.4.32 +# bash /volume1/docker/compose-files/postgresql/cutover/rollback.sh ["reason text"] +# +# Idempotent: safe to run multiple times, safe to run if nothing is +# actually mid-cutover (it will detect that and exit cleanly). +# +# ⚠️ IMPORTANT ASYMMETRY — READ BEFORE RUNNING MANUALLY LONG AFTER A +# CUTOVER YOU BELIEVED SUCCEEDED: +# Once legacy has been stopped (cutover.sh Phase 10) AND the new leader +# (patroni-1) has been genuinely serving writes for a meaningful period, +# "rolling back" is NOT a free flag-flip anymore — it means demoting a +# leader that may hold real production writes newer than legacy's last +# known state, and restoring legacy would risk DISCARDING those writes. +# This script defends against that specific danger with a grace-window +# check (see "Case E" below): if legacy is stopped and the new leader has +# been up longer than GRACE_WINDOW_SECONDS, this script REFUSES to +# automatically restore legacy and instead prints manual guidance. This +# is deliberate — do not bypass it by hand without actually reconciling +# data first (e.g. exporting recent writes from patroni-1 before touching +# anything). +set -uo pipefail + +REASON="${1:-}" +CUTOVER_DIR="/volume1/docker/compose-files/postgresql/cutover" +HA_STACK="postgresqlha" +LEGACY_STACK="postgresql" +LEGACY_SERVICE="postgresql_postgresql" +NETWORK_NAME="postgresql_db-backend" +GRACE_WINDOW_SECONDS=300 # 5 minutes — matches the kind of near-immediate + # failure cutover.sh's own Phase 10 auto-rollback + # path would trigger. Anything older than this is + # treated as "cutover had already stuck" and + # requires explicit manual handling, not blind + # automation. + +# Consumer checks reused from cutover.sh's Phase 9 list, kept in sync +# manually — see RUNBOOK.md section 1 dependency map if this list changes. +declare -A CONSUMER_HEALTH_URLS=( + ["auth_authentik-server"]="https://auth.bryanmail.net/-/health/live/" + ["git_gitea-server"]="https://git.bryanmail.net/api/healthz" + ["woodpecker_woodpecker-server"]="https://woodpecker.bryanmail.net/healthz" +) + +log() { echo "[rollback $(date +%H:%M:%S)] $*"; } +warn() { echo "[rollback WARN $(date +%H:%M:%S)] $*" >&2; } +die() { echo "[rollback FATAL $(date +%H:%M:%S)] $*" >&2; exit 1; } + +log "=== rollback.sh invoked ===" +log "Reason: ${REASON}" + +legacy_cid() { docker ps -q --filter "name=${LEGACY_SERVICE}" | head -1; } + +http_get_via_probe() { + # Uses a disposable probe container rather than assuming legacy is up — + # rollback.sh must work even when legacy has already been stopped. + local host="$1" port="$2" path="$3" + docker run --rm --network "${NETWORK_NAME}" alpine:3 sh -c " + apk add -q --no-cache bash >/dev/null 2>&1 + exec 3<>/dev/tcp/${host}/${port} || exit 99 + printf 'GET ${path} HTTP/1.0\r\nHost: ${host}\r\nConnection: close\r\n\r\n' >&3 + cat <&3 + " 2>/dev/null +} + +# ── Step 0: detect current state ──────────────────────────────────────── +log "--- Detecting current state ---" + +HA_STACK_EXISTS=0 +if docker stack ls --format '{{.Name}}' 2>/dev/null | grep -qx "${HA_STACK}"; then + HA_STACK_EXISTS=1 +fi +log "postgresqlha stack exists: ${HA_STACK_EXISTS}" + +LEGACY_REPLICAS=$(docker service inspect "${LEGACY_SERVICE}" --format '{{.Spec.Mode.Replicated.Replicas}}' 2>/dev/null || echo "unknown") +log "Legacy service (${LEGACY_SERVICE}) desired replicas: ${LEGACY_REPLICAS}" + +LEGACY_READONLY="unknown" +LEGACY_CID="$(legacy_cid)" +if [ -n "$LEGACY_CID" ]; then + LEGACY_READONLY=$(docker exec "$LEGACY_CID" bash -c 'psql -U "$POSTGRES_USER" -tAc "SHOW default_transaction_read_only;"' 2>/dev/null | tr -d '[:space:]') +fi +log "Legacy read-only state: ${LEGACY_READONLY}" + +PATRONI_LEADER_ROLE="unreachable" +PATRONI_LEADER_SINCE="" +if [ "$HA_STACK_EXISTS" -eq 1 ]; then + RESP=$(http_get_via_probe patroni-1 8008 "/patroni") + if echo "$RESP" | grep -q '"role"'; then + PATRONI_LEADER_ROLE=$(echo "$RESP" | grep -o '"role"[^,}]*' | sed 's/.*: *"//;s/"$//') + fi +fi +log "patroni-1 role (if reachable): ${PATRONI_LEADER_ROLE}" + +ALIAS_LIVE=0 +if [ -n "$LEGACY_CID" ]; then + RESOLVE_OUT=$(docker exec "$LEGACY_CID" bash -c "getent hosts postgresql" 2>/dev/null) +elif [ "$HA_STACK_EXISTS" -eq 1 ]; then + RESOLVE_OUT=$(docker run --rm --network "${NETWORK_NAME}" alpine:3 sh -c "getent hosts postgresql" 2>/dev/null) +else + RESOLVE_OUT="" +fi +NUM_IPS=$(echo "$RESOLVE_OUT" | awk 'NF{print $1}' | sort -u | wc -l) +log "'postgresql' currently resolves to ${NUM_IPS} distinct IP(s): ${RESOLVE_OUT}" +if [ "$NUM_IPS" -ge 2 ] || { [ "$LEGACY_REPLICAS" = "0" ] && [ "$HA_STACK_EXISTS" -eq 1 ]; }; then + ALIAS_LIVE=1 +fi + +# ── Step 1: classify and act ───────────────────────────────────────────── + +if [ "$HA_STACK_EXISTS" -eq 0 ]; then + log "Case A: no postgresqlha stack exists. Nothing to roll back." + if [ "$LEGACY_REPLICAS" != "1" ]; then + warn "Legacy replicas = ${LEGACY_REPLICAS}, expected 1 even with no HA stack present. Restoring." + docker service scale "${LEGACY_SERVICE}=1" + fi + if [ "$LEGACY_READONLY" = "on" ]; then + warn "Legacy is read-only with no HA stack present — reversing (this should not normally happen)." + docker exec "$(legacy_cid)" bash -c 'psql -U "$POSTGRES_USER" -c "ALTER SYSTEM SET default_transaction_read_only = off; SELECT pg_reload_conf();"' + fi + log "=== Rollback complete: nothing was in progress. Pre-cutover state confirmed/restored. ===" + exit 0 +fi + +if [ "$LEGACY_REPLICAS" = "0" ]; then + # Legacy has been stopped — this is Case E territory. Must determine + # whether we're in the "just happened, cutover.sh's own auto-rollback" + # window, or a genuinely-completed-and-aged cutover. + log "Legacy is scaled to 0 — checking how long patroni-1 has held the leader role before deciding how to proceed..." + SINCE_INFO=$(http_get_via_probe patroni-1 8008 "/patroni") + # Patroni's /patroni response includes a timestamp; if we can't parse + # one confidently, fail SAFE (treat as "aged", require manual review) + # rather than guessing young. + LEADER_TIMESTAMP=$(echo "$SINCE_INFO" | grep -o '"timestamp"[^,}]*' | sed 's/.*: *"//;s/"$//') + AGE_SECONDS=999999 + if [ -n "$LEADER_TIMESTAMP" ]; then + LEADER_EPOCH=$(date -d "$LEADER_TIMESTAMP" +%s 2>/dev/null || echo "") + NOW_EPOCH=$(date +%s) + if [ -n "$LEADER_EPOCH" ]; then + AGE_SECONDS=$((NOW_EPOCH - LEADER_EPOCH)) + fi + fi + log "Estimated age signal: ${AGE_SECONDS}s (grace window: ${GRACE_WINDOW_SECONDS}s)" + + if [ "$AGE_SECONDS" -gt "$GRACE_WINDOW_SECONDS" ]; then + die " +════════════════════════════════════════════════════════════════════════ +REFUSING to automatically roll back: legacy is stopped and the new +leader (patroni-1) appears to have been running longer than the +${GRACE_WINDOW_SECONDS}s grace window. Automatically restoring legacy +now risks DISCARDING real production writes that landed on patroni-1 +since promotion. + +This requires MANUAL handling: + 1. Confirm whether patroni-1 genuinely has newer data than legacy's + last synced point (check application data directly, or compare + row counts/timestamps on tables you know are actively written). + 2. If patroni-1's data is authoritative (i.e. the cutover basically + succeeded and this is a LATER problem, not a failed cutover): + - Do NOT run this rollback path. Treat this as 'fix forward' — + investigate why rollback.sh was invoked and address that + specific problem instead (e.g. a single consumer's connection + string, not the whole DB layer). + 3. If you genuinely need to revert to legacy despite this: + - Take a fresh pg_dumpall from patroni-1 FIRST (same method as + preflight.sh), so no data is lost even if you proceed. + - Manually restore that dump into legacy before scaling it back + up, OR manually apply just the delta if you can identify it. + - Then re-run this script, or reproduce the case-D steps below + by hand once legacy has legacy's data superseded correctly. + +Reason this rollback.sh run was invoked: ${REASON} +════════════════════════════════════════════════════════════════════════ +" + fi + + log "Case E (within grace window — treating as an immediate post-Phase-10 failure, matches cutover.sh's own auto-rollback trigger point). Proceeding automatically." + log "Restoring legacy service to 1 replica..." + docker service scale "${LEGACY_SERVICE}=1" + log "Waiting for legacy to become healthy..." + for i in $(seq 1 24); do + STATE=$(docker service ps "${LEGACY_SERVICE}" --filter "desired-state=running" --format '{{.CurrentState}}' 2>/dev/null | head -1) + log " legacy state: ${STATE}" + echo "$STATE" | grep -q "^Running" && break + sleep 5 + done + LEGACY_CID="$(legacy_cid)" + if [ -z "$LEGACY_CID" ]; then + die "Legacy service was scaled back to 1 but no running container found after waiting. MANUAL INTERVENTION REQUIRED — check 'docker service logs ${LEGACY_SERVICE} --tail 100' and 'docker stack ps ${LEGACY_STACK} --no-trunc' directly." + fi + log "Legacy container back: ${LEGACY_CID}" + # fall through to alias-removal and HA-stack-teardown below, shared with case D +fi + +if [ "$LEGACY_READONLY" = "on" ]; then + log "Reversing legacy read-only flag..." + LEGACY_CID="$(legacy_cid)" + docker exec "$LEGACY_CID" bash -c 'psql -U "$POSTGRES_USER" -c "ALTER SYSTEM SET default_transaction_read_only = off; SELECT pg_reload_conf();"' + VERIFY=$(docker exec "$LEGACY_CID" bash -c 'psql -U "$POSTGRES_USER" -tAc "SHOW default_transaction_read_only;"' 2>/dev/null | tr -d '[:space:]') + if [ "$VERIFY" != "off" ]; then + die "Attempted to reverse read-only flag but SHOW still reports '${VERIFY}'. MANUAL INTERVENTION REQUIRED — do not leave legacy read-only if consumers are pointed at it." + fi + log "Confirmed: legacy is read-write again." +fi + +if [ "$ALIAS_LIVE" -eq 1 ]; then + log "postgresql/db aliases are currently held by haproxy — removing by tearing down the HA stack entirely below (simplest reliable removal; redeploying staging-without-alias was considered but full teardown is more certain and this stack has no data worth preserving once we're rolling back)." +fi + +log "--- Tearing down postgresqlha stack ---" +docker stack rm "${HA_STACK}" +log "Waiting for tasks to actually finish shutting down (docker stack rm returns immediately, tasks take several seconds)..." +for i in $(seq 1 24); do + REMAINING=$(docker stack ps "${HA_STACK}" 2>/dev/null | grep -c . || true) + if [ "$REMAINING" -eq 0 ]; then + break + fi + sleep 5 +done +REMAINING=$(docker stack ps "${HA_STACK}" 2>/dev/null | grep -c . || true) +if [ "$REMAINING" -gt 0 ]; then + warn "postgresqlha still shows ${REMAINING} task(s) after 120s wait. Proceeding to wipe data dirs anyway is UNSAFE while tasks are still shutting down — stopping here." + die "MANUAL INTERVENTION REQUIRED: check 'docker stack ps ${HA_STACK} --no-trunc' before wiping any data dirs by hand." +fi +log "Confirmed: postgresqlha stack fully removed." + +log "--- Wiping postgresqlha data dirs ---" +for d in etcd-1-data etcd-2-data etcd-3-data patroni-0-data patroni-1-data; do + DIR="/volume1/docker/PostgreSQL/${d}" + if [ -d "$DIR" ]; then + rm -rf "$DIR" + mkdir -p "$DIR" + log " wiped and recreated: ${DIR}" + fi +done +log "Data dirs wiped (mirrors the pgha-test dry-run teardown precedent — parent dirs recreated with mkdir -p since Swarm bind mounts don't auto-create missing host parent dirs)." + +log "--- Re-verifying consumers against restored legacy ---" +CONSUMER_FAIL=0 +for svc in "${!CONSUMER_HEALTH_URLS[@]}"; do + url="${CONSUMER_HEALTH_URLS[$svc]}" + CODE=$(curl -sk -o /dev/null -w '%{http_code}' --max-time 10 "$url") + log " ${svc} -> ${url} -> HTTP ${CODE}" + [ "$CODE" != "200" ] && { warn " ${svc} did not return 200"; CONSUMER_FAIL=1; } +done +if [ "$CONSUMER_FAIL" -ne 0 ]; then + warn "One or more consumers did not verify healthy post-rollback. This is now a DIFFERENT problem than the original cutover attempt — legacy itself may need direct attention. Check 'docker service logs --tail 100' for each flagged consumer." +else + log "All checked consumers verified healthy against restored legacy." +fi + +FINAL_RESOLVE=$(docker exec "$(legacy_cid)" bash -c "getent hosts postgresql" 2>/dev/null) +log "Final 'postgresql' resolution: ${FINAL_RESOLVE}" + +log "=== ROLLBACK COMPLETE ===" +log "Summary:" +log " - postgresqlha stack: removed, data dirs wiped and recreated empty" +log " - legacy service: running, read-write, sole answer for 'postgresql'/'db'" +log " - consumer verification: $([ "$CONSUMER_FAIL" -eq 0 ] && echo 'all passed' || echo 'SOME FAILED — see warnings above, needs manual follow-up')" +log " - original invocation reason: ${REASON}" +log "Legacy's data directory itself (/volume1/docker/PostgreSQL/data-17) was" +log "never touched by this script — only postgresqlha's own dirs were wiped." +exit 0