rollback: add full-session logging (writes to same dir as backup)
Persists this run's entire stdout/stderr transcript to BACKUP_DIR (/volume1/SMB-docker/backup) — the same directory the pg_dumpall backup lands in. Appends to an already-open CUTOVER_SESSION_LOG if invoked as a child of cutover.sh (merging into that session's single transcript); opens its own rollback-standalone-<TS>.log if run standalone (including a manual run long after the fact, per this script's own asymmetry warning). This is the single highest-value place for a durable transcript in the whole suite, since rollback.sh failing partway is the one scenario RUNBOOK.md flags as requiring manual intervention. Also logs the transcript path in the grace-window refusal message so it's not lost even in that failure mode. Console/SSH output unchanged (tee mirrors to both). See ADR-0001 note, Session Update 9.
This commit is contained in:
@@ -33,6 +33,22 @@
|
||||
# returned HTTP 204 — a legitimate "healthy, no content" response, not a
|
||||
# failure. FIXED: health checks now accept any 2xx status code, matching
|
||||
# the same fix applied to cutover.sh.
|
||||
#
|
||||
# Later addition (no new incident, proactive) — SESSION LOGGING. This
|
||||
# script is explicitly the ONE piece of the whole procedure RUNBOOK.md
|
||||
# flags as needing manual intervention if it itself fails partway (see
|
||||
# the asymmetry warning above and cutover.sh's trigger_rollback()) — which
|
||||
# makes it the single highest-value place in this entire cutover/rollback
|
||||
# suite to have a durable transcript, since it's the exact scenario where
|
||||
# nobody may be watching a live terminal (rollback.sh is invoked
|
||||
# AUTOMATICALLY by cutover.sh on any failed phase) and where getting the
|
||||
# post-incident story right matters most. FIXED: every run now persists
|
||||
# its full stdout/stderr transcript to BACKUP_DIR (the same directory the
|
||||
# pg_dumpall backup lands in) — either appending to an already-open
|
||||
# CUTOVER_SESSION_LOG if invoked as a child of cutover.sh (one merged
|
||||
# transcript covering the whole session), or opening its own timestamped
|
||||
# log if run standalone (e.g. by the operator, long after the fact, per
|
||||
# the asymmetry warning above). See ADR-0001 note, Session Update 9.
|
||||
set -uo pipefail
|
||||
|
||||
REASON="${1:-<no reason given — invoked directly>}"
|
||||
@@ -41,6 +57,7 @@ HA_STACK="postgresqlha"
|
||||
LEGACY_STACK="postgresql"
|
||||
LEGACY_SERVICE="postgresql_postgresql"
|
||||
NETWORK_NAME="postgresql_db-backend"
|
||||
BACKUP_DIR="/volume1/SMB-docker/backup"
|
||||
GRACE_WINDOW_SECONDS=300 # 5 minutes — matches the kind of near-immediate
|
||||
# failure cutover.sh's own Phase 10 auto-rollback
|
||||
# path would trigger. Anything older than this is
|
||||
@@ -48,6 +65,20 @@ GRACE_WINDOW_SECONDS=300 # 5 minutes — matches the kind of near-immediate
|
||||
# requires explicit manual handling, not blind
|
||||
# automation.
|
||||
|
||||
# ── SESSION LOGGING (added — see ADR-0001 note, Session Update 9) ────────
|
||||
# Same location as preflight.sh's backup and cutover.sh's session log —
|
||||
# BACKUP_DIR. If a parent cutover.sh already exported CUTOVER_SESSION_LOG,
|
||||
# append to that single merged transcript; otherwise (standalone
|
||||
# invocation, including a long-after-the-fact manual run per the
|
||||
# asymmetry warning above) open our own timestamped log. tee mirrors to
|
||||
# the real stdout/stderr too, so console/SSH-visible output is unchanged.
|
||||
mkdir -p "$BACKUP_DIR"
|
||||
if [ -z "${CUTOVER_SESSION_LOG:-}" ]; then
|
||||
CUTOVER_SESSION_LOG="${BACKUP_DIR}/rollback-standalone-$(date +%Y%m%d-%H%M%S).log"
|
||||
export CUTOVER_SESSION_LOG
|
||||
fi
|
||||
exec > >(tee -a "$CUTOVER_SESSION_LOG") 2>&1
|
||||
|
||||
# Consumer checks reused from cutover.sh's Phase 9 list, kept in sync
|
||||
# manually — see RUNBOOK.md section 1 dependency map if this list changes.
|
||||
declare -A CONSUMER_HEALTH_URLS=(
|
||||
@@ -65,6 +96,7 @@ is_2xx() { [[ "$1" =~ ^2[0-9][0-9]$ ]]; }
|
||||
|
||||
log "=== rollback.sh invoked ==="
|
||||
log "Reason: ${REASON}"
|
||||
log "Full session transcript: ${CUTOVER_SESSION_LOG}"
|
||||
|
||||
legacy_cid() { docker ps -q --filter "name=${LEGACY_SERVICE}" | head -1; }
|
||||
|
||||
@@ -187,6 +219,7 @@ This requires MANUAL handling:
|
||||
by hand once legacy has legacy's data superseded correctly.
|
||||
|
||||
Reason this rollback.sh run was invoked: ${REASON}
|
||||
Full session transcript up to this point: ${CUTOVER_SESSION_LOG}
|
||||
════════════════════════════════════════════════════════════════════════
|
||||
"
|
||||
fi
|
||||
@@ -278,6 +311,7 @@ log " - postgresqlha stack: removed, data dirs wiped and recreated empty"
|
||||
log " - legacy service: running, read-write, sole answer for 'postgresql'/'db'"
|
||||
log " - consumer verification: $([ "$CONSUMER_FAIL" -eq 0 ] && echo 'all passed' || echo 'SOME FAILED — see warnings above, needs manual follow-up')"
|
||||
log " - original invocation reason: ${REASON}"
|
||||
log " - full session transcript: ${CUTOVER_SESSION_LOG}"
|
||||
log "Legacy's data directory itself (/volume1/docker/PostgreSQL/data-17) was"
|
||||
log "never touched by this script — only postgresqlha's own dirs were wiped."
|
||||
exit 0
|
||||
|
||||
Reference in New Issue
Block a user