From 1203630bb513b43d910b345e36d5c25471e2947d Mon Sep 17 00:00:00 2001 From: admin Date: Sun, 2 Aug 2026 17:32:42 -0700 Subject: [PATCH] ADR-0001 Phase 3: add cutover.sh (11-phase scripted cutover with auto-rollback) --- postgresql/cutover/cutover.sh | 452 ++++++++++++++++++++++++++++++++++ 1 file changed, 452 insertions(+) create mode 100644 postgresql/cutover/cutover.sh diff --git a/postgresql/cutover/cutover.sh b/postgresql/cutover/cutover.sh new file mode 100644 index 0000000..425cbfd --- /dev/null +++ b/postgresql/cutover/cutover.sh @@ -0,0 +1,452 @@ +#!/usr/bin/env bash +# ADR-0001 Phase 3 Cutover — cutover.sh +# +# Run DIRECTLY on docker-2 via SSH (NOT via the LXC-exec MCP tool — several +# phases here run well past its ~30s timeout): +# +# ssh root@192.168.4.32 +# bash /volume1/docker/compose-files/postgresql/cutover/cutover.sh +# +# Implements RUNBOOK.md's 11 phases. Any failed check triggers immediate, +# automatic invocation of rollback.sh and the script exits nonzero — no +# manual intervention should be required to reach a safe state. Designed +# to be run unattended, but given the blast radius (production DB, this +# chat's own dependency on it, ~13 consumers) watching it live the first +# time is strongly recommended anyway. +# +# ── Design notes on HOW checks are implemented (read before editing) ────── +# +# Node locality: plain `docker exec` only works for containers whose task +# is actually scheduled on the node you run it from — Swarm does NOT proxy +# exec across nodes. patroni-0/1/haproxy/etcd-1/3 may land on docker-1 or +# docker-3, not docker-2. Two techniques are used throughout to work +# around this, both already validated in the ADR-0001 dry run: +# 1. Legacy's own container IS pinned to docker-2 (see postgresql.yaml: +# node.hostname == docker-2) and shares postgresql_db-backend with +# every HA-stack container — so `docker exec ...` is +# always available locally and can reach any HA container by its +# Swarm-DNS hostname (patroni-0, patroni-1, haproxy, etcd-2, etc.). +# This is the SAME proxy pattern documented in the ADR-0001 note +# ("docker exec psql -h patroni-1 -p 5432 ..."). +# 2. `docker service ps ` and `docker service logs ` +# work cluster-wide from any manager regardless of which node the +# task landed on — used for stability/health signal checks that +# don't need to reach inside the container. +# Patroni REST calls (port 8008) are made via bash's `/dev/tcp` raw-socket +# trick executed inside the legacy container, NOT via curl — this avoids +# assuming curl is present in the (Debian-slim) postgres:17 legacy image. +# +# ⚠️ DEVIATION FLAGGED FOR OPERATOR REVIEW — Phase 5 (promotion): +# The ADR-0001 dry run promoted by removing the standby_cluster key +# directly via etcdctl (patronictl had an unrelated config-loader quirk +# in that environment). This script instead uses Patroni's own official, +# documented REST API mechanism: `PATCH /config` with `{"standby_cluster": +# null}`, which is Patroni's sanctioned way to remove a dynamic-config +# key (see Patroni docs, REST API section). This avoids needing jq/python +# inside the minimal etcd image to hand-edit YAML, and avoids the +# patronictl quirk entirely — but it was NOT the exact mechanism proven +# in the dry run. A manual fallback (raw etcdctl, matching the dry-run +# method) is documented below and in RUNBOOK.md in case this PATCH +# doesn't behave as expected. Recommend a low-stakes smoke test (e.g. a +# GET /config call, done automatically below before the real PATCH) and +# operator attention specifically during this phase. +# +# Manual fallback for Phase 5 if the PATCH approach ever needs bypassing: +# docker exec etcdctl --endpoints=http://etcd-1:2379,http://etcd-2:2379,http://etcd-3:2379 \ +# get /service/postgres-ha/config --print-value-only +# (hand-edit the returned YAML to remove the standby_cluster block, then) +# docker exec etcdctl --endpoints=... put /service/postgres-ha/config -- "" +# etcd-2 is guaranteed to be on docker-2 (node.hostname == docker-2 +# constraint in both staging/final yaml), so this exec always works +# locally if needed. + +set -uo pipefail # NOTE: deliberately not -e — every phase below checks + # its own command's exit status explicitly so we can + # invoke rollback with a specific, accurate reason + # rather than an opaque `set -e` abort mid-phase. + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="/volume1/docker/compose-files" +CUTOVER_DIR="${REPO_ROOT}/postgresql/cutover" +HA_STACK="postgresqlha" +LEGACY_STACK="postgresql" +LEGACY_SERVICE="postgresql_postgresql" +NETWORK_NAME="postgresql_db-backend" +PATRONI_SCOPE="postgres-ha" +BACKUP_DIR="/volume1/docker/PostgreSQL/cutover-backups" + +# Consumer services checked in Phase 9 — (service_name, external health URL +# or empty if internal-only). Derived from the dependency map in +# RUNBOOK.md section 1. Update this list if the dependency map changes. +declare -A CONSUMER_HEALTH_URLS=( + ["auth_authentik-server"]="https://auth.bryanmail.net/-/health/live/" + ["git_gitea-server"]="https://git.bryanmail.net/api/healthz" + ["woodpecker_woodpecker-server"]="https://woodpecker.bryanmail.net/healthz" +) +# Services checked via Swarm task-stability only (no forwardAuth-free +# external endpoint known / internal-only) — open-webui and litellm sit +# behind Authentik forwardAuth on their public routes, so an external curl +# would just hit a login redirect, not real app health. +CONSUMER_STABILITY_ONLY=( + "ai_open-webui" + "ai_litellm" + "auth_authentik-worker" +) + +log() { echo "[cutover $(date +%H:%M:%S)] $*"; } +warn() { echo "[cutover WARN $(date +%H:%M:%S)] $*" >&2; } + +# Any hard failure calls this — invokes rollback.sh, then exits nonzero. +# No manual step should be required after this runs. +trigger_rollback() { + local reason="$1" + echo "" >&2 + echo "[cutover FAIL $(date +%H:%M:%S)] ${reason}" >&2 + echo "[cutover] Invoking rollback.sh automatically — no manual action should be required." >&2 + bash "${CUTOVER_DIR}/rollback.sh" "${reason}" + local rc=$? + if [ $rc -ne 0 ]; then + echo "" >&2 + echo "[cutover] ⚠️ rollback.sh ITSELF returned nonzero (${rc}). This is the one" >&2 + echo "[cutover] scenario requiring manual intervention — see RUNBOOK.md section 4," >&2 + echo "[cutover] 'If rollback.sh itself fails partway'. Do not blindly re-run scripts." >&2 + fi + exit 1 +} + +# Get the container ID for the (always docker-2-local) legacy container. +legacy_cid() { + docker ps -q --filter "name=${LEGACY_SERVICE}" | head -1 +} + +# Raw-socket HTTP GET via /dev/tcp, executed inside the legacy container. +# Avoids assuming curl exists in the legacy postgres:17 image. +# Args: host port path +http_get_via_legacy() { + local host="$1" port="$2" path="$3" cid + cid="$(legacy_cid)" + docker exec "$cid" bash -c " + exec 3<>/dev/tcp/${host}/${port} || exit 99 + printf 'GET ${path} HTTP/1.0\r\nHost: ${host}\r\nConnection: close\r\n\r\n' >&3 + cat <&3 + " 2>/dev/null +} + +# Raw-socket HTTP PATCH with basic auth + JSON body, via /dev/tcp inside +# the legacy container. Args: host port path username password json_body +http_patch_via_legacy() { + local host="$1" port="$2" path="$3" user="$4" pass="$5" body="$6" cid auth + cid="$(legacy_cid)" + auth="$(docker exec "$cid" bash -c "printf '%s' '${user}:${pass}' | base64 -w0")" + local body_len=${#body} + docker exec "$cid" bash -c " + exec 3<>/dev/tcp/${host}/${port} || exit 99 + printf 'PATCH ${path} HTTP/1.0\r\nHost: ${host}\r\nAuthorization: Basic ${auth}\r\nContent-Type: application/json\r\nContent-Length: ${body_len}\r\nConnection: close\r\n\r\n${body}' >&3 + cat <&3 + " 2>/dev/null +} + +# Poll a Patroni node's /patroni endpoint for its reported role. +# Args: hostname (e.g. patroni-1) -> prints role string (leader/replica/standby_leader/etc) +patroni_role() { + local host="$1" + http_get_via_legacy "$host" 8008 "/patroni" | grep -o '"role"[^,}]*' | sed 's/.*: *"//;s/"$//' +} + +# Poll a Patroni node's /patroni endpoint for streaming lag against +# whichever host it's replicating from. Returns the numeric lag if found, +# empty otherwise (some responses omit it when lag is genuinely 0/absent). +patroni_lag() { + local host="$1" + http_get_via_legacy "$host" 8008 "/patroni" | grep -o '"replication_state"[^,}]*\|"lag"[^,}]*' | head -1 +} + +wait_for_stack_running() { + local stack="$1" expected_count="$2" max_wait="${3:-120}" + local waited=0 + while [ "$waited" -lt "$max_wait" ]; do + local running + running=$(docker stack ps "$stack" --filter "desired-state=running" --format '{{.CurrentState}}' 2>/dev/null | grep -c '^Running') + if [ "$running" -ge "$expected_count" ]; then + return 0 + fi + sleep 5 + waited=$((waited + 5)) + done + return 1 +} + +# ═══════════════════════════════════════════════════════════════════════ +log "=== ADR-0001 Phase 3 real cutover starting ===" +log "Chat dependency reminder: this chat (ai_open-webui/litellm) is itself" +log "a consumer of the DB being migrated. If this script exits nonzero" +log "before completing rollback, follow RUNBOOK.md section 4 manually via" +log "direct SSH — do not assume this chat will remain usable to help." + +# ── Phase 0: preflight ──────────────────────────────────────────────── +log "--- Phase 0: preflight ---" +if ! bash "${CUTOVER_DIR}/preflight.sh"; then + echo "[cutover] preflight.sh failed. Nothing has been touched — safe to stop here, no rollback needed." >&2 + exit 1 +fi +log "PASS: Phase 0 preflight" + +# ── Phase 1: deploy postgresqlha staging stack ───────────────────────── +log "--- Phase 1: deploy postgresqlha (staging) ---" + +PRIMARY_LABELED=$(docker node ls --format '{{.Hostname}}' | xargs -I{} docker node inspect {} --format '{{.Spec.Labels.pg_role}}{{.Spec.Labels.pg-role}}' 2>/dev/null | grep -c primary || true) +REPLICA_LABELED=$(docker node ls --format '{{.Hostname}}' | xargs -I{} docker node inspect {} --format '{{.Spec.Labels.pg_role}}{{.Spec.Labels.pg-role}}' 2>/dev/null | grep -c replica || true) +if [ "$PRIMARY_LABELED" -lt 1 ] || [ "$REPLICA_LABELED" -lt 1 ]; then + trigger_rollback "No node(s) found with pg-role=primary/replica labels — patroni-0/1 placement constraints cannot schedule. Set via: docker node update --label-add pg-role=primary (and =replica on another). Nothing deployed yet, rollback is a no-op safety call." +fi + +docker stack deploy -c "${CUTOVER_DIR}/postgresql-ha-staging.yaml" "${HA_STACK}" +if [ $? -ne 0 ]; then + trigger_rollback "docker stack deploy of postgresql-ha-staging.yaml failed." +fi + +log "Waiting for postgresqlha services to be running (etcd-1/2/3, patroni-0/1, haproxy = 6)..." +if ! wait_for_stack_running "${HA_STACK}" 6 180; then + trigger_rollback "postgresqlha stack did not reach 6 running tasks within 180s." +fi + +log "Confirming patroni-1 reaches standby_leader..." +ROLE="" +for i in $(seq 1 24); do + ROLE="$(patroni_role patroni-1)" + [ "$ROLE" = "standby_leader" ] && break + sleep 10 +done +if [ "$ROLE" != "standby_leader" ]; then + trigger_rollback "patroni-1 did not reach standby_leader role within 240s (last observed: '${ROLE}')." +fi +log "PASS: Phase 1 — patroni-1 is standby_leader" + +# ── Phase 2: wait for streaming lag = 0 ──────────────────────────────── +log "--- Phase 2: confirm streaming lag = 0 ---" +CONSECUTIVE_ZERO=0 +for i in $(seq 1 30); do + P0_ROLE="$(patroni_role patroni-0)" + LAG_INFO="$(patroni_lag patroni-0)" + log "patroni-0 role=${P0_ROLE} lag_info=${LAG_INFO}" + if [ "$P0_ROLE" = "replica" ] && { [ -z "$LAG_INFO" ] || echo "$LAG_INFO" | grep -q '"lag": *0\b\|streaming'; }; then + CONSECUTIVE_ZERO=$((CONSECUTIVE_ZERO + 1)) + else + CONSECUTIVE_ZERO=0 + fi + [ "$CONSECUTIVE_ZERO" -ge 3 ] && break + sleep 10 +done +if [ "$CONSECUTIVE_ZERO" -lt 3 ]; then + trigger_rollback "Streaming lag never reached 3 consecutive zero/streaming polls on patroni-0. Last state: role=${P0_ROLE} lag_info=${LAG_INFO}" +fi +log "PASS: Phase 2 — streaming confirmed, lag=0 for 3 consecutive polls" + +# ── Phase 3: pre-promotion canary write/propagation check ───────────── +log "--- Phase 3: pre-promotion canary (legacy -> patroni-0 cascade replica) ---" +LEGACY_CID="$(legacy_cid)" +if [ -z "$LEGACY_CID" ]; then + trigger_rollback "Could not find legacy container for canary write." +fi +CANARY_VAL="cutover-$(date +%s)" +docker exec "$LEGACY_CID" bash -c "psql -U \"\$POSTGRES_USER\" -c \"CREATE TABLE IF NOT EXISTS _cutover_canary (val text, ts timestamptz default now()); INSERT INTO _cutover_canary(val) VALUES ('${CANARY_VAL}');\"" >/dev/null 2>&1 +if [ $? -ne 0 ]; then + trigger_rollback "Failed to write pre-promotion canary row to legacy." +fi +FOUND=0 +for i in $(seq 1 5); do + RESULT=$(docker exec "$LEGACY_CID" bash -c "psql -h patroni-0 -U \"\$POSTGRES_USER\" -tAc \"SELECT 1 FROM _cutover_canary WHERE val = '${CANARY_VAL}';\"" 2>/dev/null) + if [ "$RESULT" = "1" ]; then + FOUND=1 + break + fi + sleep 1 +done +if [ "$FOUND" -ne 1 ]; then + trigger_rollback "Canary row did not propagate to patroni-0 within 5s. standby_cluster streaming is not working as expected." +fi +log "PASS: Phase 3 — canary write propagated to patroni-0 within 5s" + +# ── Phase 4: flip legacy to read-only ─────────────────────────────────── +log "--- Phase 4: flip legacy read-only ---" +docker exec "$LEGACY_CID" bash -c 'psql -U "$POSTGRES_USER" -c "ALTER SYSTEM SET default_transaction_read_only = on; SELECT pg_reload_conf();"' >/dev/null 2>&1 +if [ $? -ne 0 ]; then + trigger_rollback "Failed to set default_transaction_read_only=on on legacy." +fi +RO_STATE=$(docker exec "$LEGACY_CID" bash -c 'psql -U "$POSTGRES_USER" -tAc "SHOW default_transaction_read_only;"' 2>/dev/null | tr -d '[:space:]') +if [ "$RO_STATE" != "on" ]; then + trigger_rollback "Verified read-only state is '${RO_STATE}', expected 'on'. Reversing immediately." +fi +log "Legacy confirmed read-only. Draining one more lag poll to confirm quiescence..." +sleep 5 +LAG_INFO="$(patroni_lag patroni-0)" +log "Post-read-only lag check: ${LAG_INFO}" +log "PASS: Phase 4 — legacy is read-only, no further writes possible there" + +# ── Phase 5: promote Patroni ───────────────────────────────────────── +log "--- Phase 5: promote patroni-1 (remove standby_cluster from DCS config) ---" +log "⚠️ Highest-scrutiny phase — see script header for the documented deviation from the dry-run's raw-etcdctl method (using Patroni's official REST PATCH /config instead). Manual etcdctl fallback documented above and in RUNBOOK.md if needed." + +PATRONI_PASS=$(docker exec "$LEGACY_CID" cat /run/secrets/postgresql_patroni_password 2>/dev/null) +if [ -z "$PATRONI_PASS" ]; then + # legacy container may not have this secret mounted — fall back to reading via patroni-1 itself if reachable, else hard-fail before touching anything + trigger_rollback "Could not read postgresql_patroni_password secret needed for the promotion PATCH call. Nothing promoted yet." +fi + +log "Smoke-test: GET /config on patroni-1 before attempting the PATCH..." +CONFIG_CHECK=$(http_get_via_legacy patroni-1 8008 "/config") +if ! echo "$CONFIG_CHECK" | grep -qi "standby_cluster"; then + trigger_rollback "GET /config on patroni-1 did not show a standby_cluster key as expected before promotion — refusing to PATCH against an unexpected state. Raw response logged above for review." +fi +log "Smoke-test passed — standby_cluster key confirmed present pre-promotion." + +PATCH_RESPONSE=$(http_patch_via_legacy patroni-1 8008 "/config" "patroni" "$PATRONI_PASS" '{"standby_cluster": null}') +log "PATCH /config response: ${PATCH_RESPONSE}" + +log "Waiting for patroni-1 to transition standby_leader -> leader (expect ~15-30s per dry-run drills)..." +PROMOTED=0 +for i in $(seq 1 12); do + ROLE="$(patroni_role patroni-1)" + log "patroni-1 role: ${ROLE}" + if [ "$ROLE" = "leader" ]; then + PROMOTED=1 + break + fi + sleep 5 +done +if [ "$PROMOTED" -ne 1 ]; then + trigger_rollback "patroni-1 did not reach 'leader' role within 60s of the promotion PATCH. Last observed role: '${ROLE}'. Manual etcdctl fallback is documented in this script's header / RUNBOOK.md if the operator wants to investigate before accepting the automatic rollback." +fi + +IN_RECOVERY=$(docker exec "$LEGACY_CID" bash -c 'psql -h patroni-1 -U "$POSTGRES_USER" -tAc "SELECT pg_is_in_recovery();"' 2>/dev/null | tr -d '[:space:]') +if [ "$IN_RECOVERY" != "f" ]; then + trigger_rollback "patroni-1 reports role=leader but pg_is_in_recovery() = '${IN_RECOVERY}' (expected 'f'). Not trusting this as a genuine promotion." +fi +log "PASS: Phase 5 — patroni-1 is genuine writable leader, pg_is_in_recovery=f" + +# ── Phase 6: confirm HAProxy would see a healthy backend ────────────── +log "--- Phase 6: confirm healthy backend before touching DNS ---" +PRIMARY_CHECK=$(http_get_via_legacy patroni-1 8008 "/primary") +if ! echo "$PRIMARY_CHECK" | grep -q "200 OK"; then + trigger_rollback "patroni-1 does not answer 200 on GET /primary post-promotion. Raw response: ${PRIMARY_CHECK}" +fi +log "PASS: Phase 6 — patroni-1 answers 200 on /primary, HAProxy's healthcheck will pass" + +# ── Phase 7: deploy final HAProxy with postgresql/db aliases ────────── +log "--- Phase 7: deploy postgresql-ha-final.yaml (adds postgresql/db aliases) ---" +docker stack deploy -c "${CUTOVER_DIR}/postgresql-ha-final.yaml" "${HA_STACK}" +if [ $? -ne 0 ]; then + trigger_rollback "docker stack deploy of postgresql-ha-final.yaml failed." +fi +log "Waiting for haproxy service update to roll out..." +sleep 15 +if ! wait_for_stack_running "${HA_STACK}" 6 90; then + trigger_rollback "postgresqlha stack did not return to 6 running tasks after haproxy alias update." +fi + +log "Confirming DNS resolution of 'postgresql' now shows BOTH legacy and haproxy..." +RESOLVE_OUTPUT=$(docker exec "$LEGACY_CID" bash -c "getent hosts postgresql" 2>/dev/null) +log "Resolution: ${RESOLVE_OUTPUT}" +NUM_IPS=$(echo "$RESOLVE_OUTPUT" | awk '{print $1}' | sort -u | wc -l) +if [ "$NUM_IPS" -lt 2 ]; then + trigger_rollback "Expected 2 distinct IPs for 'postgresql' (legacy + haproxy) after alias deploy, found ${NUM_IPS}. Resolution output: ${RESOLVE_OUTPUT}" +fi +log "PASS: Phase 7 — 'postgresql' resolves to ${NUM_IPS} IPs (legacy + haproxy), alias handoff live" + +# ── Phase 8: canary write through the alias (retry-tolerant of legacy read-only hits) ── +log "--- Phase 8: canary write/propagation THROUGH the shared alias ---" +log "Note: during this window 'postgresql' round-robins between legacy (read-only)" +log "and haproxy (writable). Write attempts landing on legacy will correctly fail" +log "with a read-only error — that is EXPECTED, not a hard failure by itself." +CANARY2_VAL="cutover-alias-$(date +%s)" +ALIAS_WRITE_OK=0 +for i in $(seq 1 10); do + docker exec "$LEGACY_CID" bash -c "psql -h postgresql -U \"\$POSTGRES_USER\" -c \"INSERT INTO _cutover_canary(val) VALUES ('${CANARY2_VAL}');\"" >/tmp/cutover_phase8_attempt_$i.log 2>&1 + if [ $? -eq 0 ]; then + ALIAS_WRITE_OK=1 + break + fi + sleep 2 +done +if [ "$ALIAS_WRITE_OK" -ne 1 ]; then + trigger_rollback "Never succeeded writing through the 'postgresql' alias after 10 attempts (~20s). See /tmp/cutover_phase8_attempt_*.log on docker-2 for the read-only/connection errors encountered." +fi +log "Alias write succeeded on attempt ${i}. Confirming visibility on BOTH patroni-0 and patroni-1..." +V0=$(docker exec "$LEGACY_CID" bash -c "psql -h patroni-0 -U \"\$POSTGRES_USER\" -tAc \"SELECT 1 FROM _cutover_canary WHERE val = '${CANARY2_VAL}';\"" 2>/dev/null) +V1=$(docker exec "$LEGACY_CID" bash -c "psql -h patroni-1 -U \"\$POSTGRES_USER\" -tAc \"SELECT 1 FROM _cutover_canary WHERE val = '${CANARY2_VAL}';\"" 2>/dev/null) +if [ "$V0" != "1" ] || [ "$V1" != "1" ]; then + trigger_rollback "Alias-written canary row not visible on both nodes (patroni-0='${V0}' patroni-1='${V1}')." +fi +log "PASS: Phase 8 — real client-path write through the alias succeeded and is visible on both nodes" + +# ── Phase 9: verify all known consumers ──────────────────────────────── +log "--- Phase 9: verify known consumers reconnect/function ---" +CONSUMER_FAIL=0 +for svc in "${!CONSUMER_HEALTH_URLS[@]}"; do + url="${CONSUMER_HEALTH_URLS[$svc]}" + CODE=$(curl -sk -o /dev/null -w '%{http_code}' --max-time 10 "$url") + log " ${svc} -> ${url} -> HTTP ${CODE}" + if [ "$CODE" != "200" ]; then + warn " ${svc} did not return 200 (got ${CODE})" + CONSUMER_FAIL=1 + fi +done +for svc in "${CONSUMER_STABILITY_ONLY[@]}"; do + RESTARTS=$(docker service ps "$svc" --filter "desired-state=running" --format '{{.CurrentState}}' 2>/dev/null | grep -c 'seconds ago\|Starting') + log " ${svc} -> recent-restart signal count: ${RESTARTS} (0 expected if stable)" + if [ "$RESTARTS" -gt 0 ]; then + warn " ${svc} shows a recent restart — may just be catching up post-cutover, but flagging." + CONSUMER_FAIL=1 + fi +done +if [ "$CONSUMER_FAIL" -ne 0 ]; then + trigger_rollback "One or more consumers failed verification in Phase 9. See warnings above. Treating any consumer failure here as a cutover failure, not something to patch around live." +fi +log "PASS: Phase 9 — all known consumers verified" + +# ── Phase 10: stop legacy ────────────────────────────────────────────── +log "--- Phase 10: stop legacy service ---" +docker service scale "${LEGACY_SERVICE}=0" +if [ $? -ne 0 ]; then + trigger_rollback "Failed to scale ${LEGACY_SERVICE} to 0." +fi +sleep 10 +log "Confirming HAProxy backend still healthy post-legacy-stop..." +PRIMARY_CHECK2=$(http_get_via_legacy patroni-1 8008 "/primary" 2>/dev/null || true) +# Note: legacy is now stopped, so http_get_via_legacy (which proxies via +# the legacy container) will start failing itself shortly — this is +# EXPECTED and is why this is the last check that uses that proxy method. +log "Last legacy-proxied check (legacy is now down, this method retires after this call): ${PRIMARY_CHECK2}" + +log "Confirming 'postgresql' now resolves ONLY to haproxy..." +# Use a disposable probe container instead of the (now-stopping) legacy container. +PROBE_OUT=$(docker run --rm --network "${NETWORK_NAME}" alpine:3 sh -c "getent hosts postgresql" 2>/dev/null) +log "Resolution post-legacy-stop: ${PROBE_OUT}" +NUM_IPS2=$(echo "$PROBE_OUT" | awk '{print $1}' | sort -u | wc -l) +if [ "$NUM_IPS2" -ne 1 ]; then + warn "Expected exactly 1 IP for 'postgresql' now that legacy is stopped, found ${NUM_IPS2}. This may just be DNS cache settling — re-check manually before treating as fatal, but NOT auto-rolling-back here since legacy has already been cleanly stopped (scale=1 would just resume it)." +fi + +log "Re-running Phase 9 consumer checks post-legacy-stop..." +CONSUMER_FAIL2=0 +for svc in "${!CONSUMER_HEALTH_URLS[@]}"; do + url="${CONSUMER_HEALTH_URLS[$svc]}" + CODE=$(curl -sk -o /dev/null -w '%{http_code}' --max-time 10 "$url") + log " ${svc} -> ${url} -> HTTP ${CODE}" + [ "$CODE" != "200" ] && CONSUMER_FAIL2=1 +done +if [ "$CONSUMER_FAIL2" -ne 0 ]; then + trigger_rollback "Consumer check failed AFTER stopping legacy. Per RUNBOOK.md, rollback past this point is asymmetric (restores legacy as primary of record, not just a flag flip) — invoking rollback.sh now rather than leaving this in a broken state." +fi +log "PASS: Phase 10 — legacy stopped, consumers still healthy" + +# ── Phase 11: final verification ──────────────────────────────────────── +log "--- Phase 11: final verification pass ---" +docker stack ps "${HA_STACK}" --no-trunc | tail -n +1 +AVAIL_GB=$(df --output=avail -BG /volume1/docker/PostgreSQL 2>/dev/null | tail -1 | tr -dc '0-9') +log "Disk space remaining at /volume1/docker/PostgreSQL: ${AVAIL_GB}GB" +log "=== CUTOVER COMPLETE ===" +log "Backup taken this run: $(cat ${BACKUP_DIR}/LATEST 2>/dev/null || echo 'unknown')" +log "See RUNBOOK.md section 7 for manual post-cutover follow-up (legacy" +log "decommission, external port 5430 / Traefik SNI handoff, burn-in period)." +exit 0