rollback: add full-session logging (writes to same dir as backup)
Persists this run's entire stdout/stderr transcript to BACKUP_DIR (/volume1/SMB-docker/backup) — the same directory the pg_dumpall backup lands in. Appends to an already-open CUTOVER_SESSION_LOG if invoked as a child of cutover.sh (merging into that session's single transcript); opens its own rollback-standalone-<TS>.log if run standalone (including a manual run long after the fact, per this script's own asymmetry warning). This is the single highest-value place for a durable transcript in the whole suite, since rollback.sh failing partway is the one scenario RUNBOOK.md flags as requiring manual intervention. Also logs the transcript path in the grace-window refusal message so it's not lost even in that failure mode. Console/SSH output unchanged (tee mirrors to both). See ADR-0001 note, Session Update 9.
This commit is contained in:
@@ -33,6 +33,22 @@
|
|||||||
# returned HTTP 204 — a legitimate "healthy, no content" response, not a
|
# returned HTTP 204 — a legitimate "healthy, no content" response, not a
|
||||||
# failure. FIXED: health checks now accept any 2xx status code, matching
|
# failure. FIXED: health checks now accept any 2xx status code, matching
|
||||||
# the same fix applied to cutover.sh.
|
# the same fix applied to cutover.sh.
|
||||||
|
#
|
||||||
|
# Later addition (no new incident, proactive) — SESSION LOGGING. This
|
||||||
|
# script is explicitly the ONE piece of the whole procedure RUNBOOK.md
|
||||||
|
# flags as needing manual intervention if it itself fails partway (see
|
||||||
|
# the asymmetry warning above and cutover.sh's trigger_rollback()) — which
|
||||||
|
# makes it the single highest-value place in this entire cutover/rollback
|
||||||
|
# suite to have a durable transcript, since it's the exact scenario where
|
||||||
|
# nobody may be watching a live terminal (rollback.sh is invoked
|
||||||
|
# AUTOMATICALLY by cutover.sh on any failed phase) and where getting the
|
||||||
|
# post-incident story right matters most. FIXED: every run now persists
|
||||||
|
# its full stdout/stderr transcript to BACKUP_DIR (the same directory the
|
||||||
|
# pg_dumpall backup lands in) — either appending to an already-open
|
||||||
|
# CUTOVER_SESSION_LOG if invoked as a child of cutover.sh (one merged
|
||||||
|
# transcript covering the whole session), or opening its own timestamped
|
||||||
|
# log if run standalone (e.g. by the operator, long after the fact, per
|
||||||
|
# the asymmetry warning above). See ADR-0001 note, Session Update 9.
|
||||||
set -uo pipefail
|
set -uo pipefail
|
||||||
|
|
||||||
REASON="${1:-<no reason given — invoked directly>}"
|
REASON="${1:-<no reason given — invoked directly>}"
|
||||||
@@ -41,6 +57,7 @@ HA_STACK="postgresqlha"
|
|||||||
LEGACY_STACK="postgresql"
|
LEGACY_STACK="postgresql"
|
||||||
LEGACY_SERVICE="postgresql_postgresql"
|
LEGACY_SERVICE="postgresql_postgresql"
|
||||||
NETWORK_NAME="postgresql_db-backend"
|
NETWORK_NAME="postgresql_db-backend"
|
||||||
|
BACKUP_DIR="/volume1/SMB-docker/backup"
|
||||||
GRACE_WINDOW_SECONDS=300 # 5 minutes — matches the kind of near-immediate
|
GRACE_WINDOW_SECONDS=300 # 5 minutes — matches the kind of near-immediate
|
||||||
# failure cutover.sh's own Phase 10 auto-rollback
|
# failure cutover.sh's own Phase 10 auto-rollback
|
||||||
# path would trigger. Anything older than this is
|
# path would trigger. Anything older than this is
|
||||||
@@ -48,6 +65,20 @@ GRACE_WINDOW_SECONDS=300 # 5 minutes — matches the kind of near-immediate
|
|||||||
# requires explicit manual handling, not blind
|
# requires explicit manual handling, not blind
|
||||||
# automation.
|
# automation.
|
||||||
|
|
||||||
|
# ── SESSION LOGGING (added — see ADR-0001 note, Session Update 9) ────────
|
||||||
|
# Same location as preflight.sh's backup and cutover.sh's session log —
|
||||||
|
# BACKUP_DIR. If a parent cutover.sh already exported CUTOVER_SESSION_LOG,
|
||||||
|
# append to that single merged transcript; otherwise (standalone
|
||||||
|
# invocation, including a long-after-the-fact manual run per the
|
||||||
|
# asymmetry warning above) open our own timestamped log. tee mirrors to
|
||||||
|
# the real stdout/stderr too, so console/SSH-visible output is unchanged.
|
||||||
|
mkdir -p "$BACKUP_DIR"
|
||||||
|
if [ -z "${CUTOVER_SESSION_LOG:-}" ]; then
|
||||||
|
CUTOVER_SESSION_LOG="${BACKUP_DIR}/rollback-standalone-$(date +%Y%m%d-%H%M%S).log"
|
||||||
|
export CUTOVER_SESSION_LOG
|
||||||
|
fi
|
||||||
|
exec > >(tee -a "$CUTOVER_SESSION_LOG") 2>&1
|
||||||
|
|
||||||
# Consumer checks reused from cutover.sh's Phase 9 list, kept in sync
|
# Consumer checks reused from cutover.sh's Phase 9 list, kept in sync
|
||||||
# manually — see RUNBOOK.md section 1 dependency map if this list changes.
|
# manually — see RUNBOOK.md section 1 dependency map if this list changes.
|
||||||
declare -A CONSUMER_HEALTH_URLS=(
|
declare -A CONSUMER_HEALTH_URLS=(
|
||||||
@@ -65,6 +96,7 @@ is_2xx() { [[ "$1" =~ ^2[0-9][0-9]$ ]]; }
|
|||||||
|
|
||||||
log "=== rollback.sh invoked ==="
|
log "=== rollback.sh invoked ==="
|
||||||
log "Reason: ${REASON}"
|
log "Reason: ${REASON}"
|
||||||
|
log "Full session transcript: ${CUTOVER_SESSION_LOG}"
|
||||||
|
|
||||||
legacy_cid() { docker ps -q --filter "name=${LEGACY_SERVICE}" | head -1; }
|
legacy_cid() { docker ps -q --filter "name=${LEGACY_SERVICE}" | head -1; }
|
||||||
|
|
||||||
@@ -187,6 +219,7 @@ This requires MANUAL handling:
|
|||||||
by hand once legacy has legacy's data superseded correctly.
|
by hand once legacy has legacy's data superseded correctly.
|
||||||
|
|
||||||
Reason this rollback.sh run was invoked: ${REASON}
|
Reason this rollback.sh run was invoked: ${REASON}
|
||||||
|
Full session transcript up to this point: ${CUTOVER_SESSION_LOG}
|
||||||
════════════════════════════════════════════════════════════════════════
|
════════════════════════════════════════════════════════════════════════
|
||||||
"
|
"
|
||||||
fi
|
fi
|
||||||
@@ -278,6 +311,7 @@ log " - postgresqlha stack: removed, data dirs wiped and recreated empty"
|
|||||||
log " - legacy service: running, read-write, sole answer for 'postgresql'/'db'"
|
log " - legacy service: running, read-write, sole answer for 'postgresql'/'db'"
|
||||||
log " - consumer verification: $([ "$CONSUMER_FAIL" -eq 0 ] && echo 'all passed' || echo 'SOME FAILED — see warnings above, needs manual follow-up')"
|
log " - consumer verification: $([ "$CONSUMER_FAIL" -eq 0 ] && echo 'all passed' || echo 'SOME FAILED — see warnings above, needs manual follow-up')"
|
||||||
log " - original invocation reason: ${REASON}"
|
log " - original invocation reason: ${REASON}"
|
||||||
|
log " - full session transcript: ${CUTOVER_SESSION_LOG}"
|
||||||
log "Legacy's data directory itself (/volume1/docker/PostgreSQL/data-17) was"
|
log "Legacy's data directory itself (/volume1/docker/PostgreSQL/data-17) was"
|
||||||
log "never touched by this script — only postgresqlha's own dirs were wiped."
|
log "never touched by this script — only postgresqlha's own dirs were wiped."
|
||||||
exit 0
|
exit 0
|
||||||
|
|||||||
Reference in New Issue
Block a user