From 2ed406db822cc8a2209066a296cf1c0a1a09dfa5 Mon Sep 17 00:00:00 2001 From: admin Date: Wed, 5 Aug 2026 15:08:47 -0700 Subject: [PATCH] rollback: add full-session logging (writes to same dir as backup) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Persists this run's entire stdout/stderr transcript to BACKUP_DIR (/volume1/SMB-docker/backup) — the same directory the pg_dumpall backup lands in. Appends to an already-open CUTOVER_SESSION_LOG if invoked as a child of cutover.sh (merging into that session's single transcript); opens its own rollback-standalone-.log if run standalone (including a manual run long after the fact, per this script's own asymmetry warning). This is the single highest-value place for a durable transcript in the whole suite, since rollback.sh failing partway is the one scenario RUNBOOK.md flags as requiring manual intervention. Also logs the transcript path in the grace-window refusal message so it's not lost even in that failure mode. Console/SSH output unchanged (tee mirrors to both). See ADR-0001 note, Session Update 9. --- postgresql/cutover/rollback.sh | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/postgresql/cutover/rollback.sh b/postgresql/cutover/rollback.sh index e4f4114..e7d83c4 100644 --- a/postgresql/cutover/rollback.sh +++ b/postgresql/cutover/rollback.sh @@ -33,6 +33,22 @@ # returned HTTP 204 — a legitimate "healthy, no content" response, not a # failure. FIXED: health checks now accept any 2xx status code, matching # the same fix applied to cutover.sh. +# +# Later addition (no new incident, proactive) — SESSION LOGGING. This +# script is explicitly the ONE piece of the whole procedure RUNBOOK.md +# flags as needing manual intervention if it itself fails partway (see +# the asymmetry warning above and cutover.sh's trigger_rollback()) — which +# makes it the single highest-value place in this entire cutover/rollback +# suite to have a durable transcript, since it's the exact scenario where +# nobody may be watching a live terminal (rollback.sh is invoked +# AUTOMATICALLY by cutover.sh on any failed phase) and where getting the +# post-incident story right matters most. FIXED: every run now persists +# its full stdout/stderr transcript to BACKUP_DIR (the same directory the +# pg_dumpall backup lands in) — either appending to an already-open +# CUTOVER_SESSION_LOG if invoked as a child of cutover.sh (one merged +# transcript covering the whole session), or opening its own timestamped +# log if run standalone (e.g. by the operator, long after the fact, per +# the asymmetry warning above). See ADR-0001 note, Session Update 9. set -uo pipefail REASON="${1:-}" @@ -41,6 +57,7 @@ HA_STACK="postgresqlha" LEGACY_STACK="postgresql" LEGACY_SERVICE="postgresql_postgresql" NETWORK_NAME="postgresql_db-backend" +BACKUP_DIR="/volume1/SMB-docker/backup" GRACE_WINDOW_SECONDS=300 # 5 minutes — matches the kind of near-immediate # failure cutover.sh's own Phase 10 auto-rollback # path would trigger. Anything older than this is @@ -48,6 +65,20 @@ GRACE_WINDOW_SECONDS=300 # 5 minutes — matches the kind of near-immediate # requires explicit manual handling, not blind # automation. +# ── SESSION LOGGING (added — see ADR-0001 note, Session Update 9) ──────── +# Same location as preflight.sh's backup and cutover.sh's session log — +# BACKUP_DIR. If a parent cutover.sh already exported CUTOVER_SESSION_LOG, +# append to that single merged transcript; otherwise (standalone +# invocation, including a long-after-the-fact manual run per the +# asymmetry warning above) open our own timestamped log. tee mirrors to +# the real stdout/stderr too, so console/SSH-visible output is unchanged. +mkdir -p "$BACKUP_DIR" +if [ -z "${CUTOVER_SESSION_LOG:-}" ]; then + CUTOVER_SESSION_LOG="${BACKUP_DIR}/rollback-standalone-$(date +%Y%m%d-%H%M%S).log" + export CUTOVER_SESSION_LOG +fi +exec > >(tee -a "$CUTOVER_SESSION_LOG") 2>&1 + # Consumer checks reused from cutover.sh's Phase 9 list, kept in sync # manually — see RUNBOOK.md section 1 dependency map if this list changes. declare -A CONSUMER_HEALTH_URLS=( @@ -65,6 +96,7 @@ is_2xx() { [[ "$1" =~ ^2[0-9][0-9]$ ]]; } log "=== rollback.sh invoked ===" log "Reason: ${REASON}" +log "Full session transcript: ${CUTOVER_SESSION_LOG}" legacy_cid() { docker ps -q --filter "name=${LEGACY_SERVICE}" | head -1; } @@ -187,6 +219,7 @@ This requires MANUAL handling: by hand once legacy has legacy's data superseded correctly. Reason this rollback.sh run was invoked: ${REASON} +Full session transcript up to this point: ${CUTOVER_SESSION_LOG} ════════════════════════════════════════════════════════════════════════ " fi @@ -278,6 +311,7 @@ log " - postgresqlha stack: removed, data dirs wiped and recreated empty" log " - legacy service: running, read-write, sole answer for 'postgresql'/'db'" log " - consumer verification: $([ "$CONSUMER_FAIL" -eq 0 ] && echo 'all passed' || echo 'SOME FAILED — see warnings above, needs manual follow-up')" log " - original invocation reason: ${REASON}" +log " - full session transcript: ${CUTOVER_SESSION_LOG}" log "Legacy's data directory itself (/volume1/docker/PostgreSQL/data-17) was" log "never touched by this script — only postgresqlha's own dirs were wiped." exit 0