From 653f3086234169a8a96884ef17b4cf11eb57fc95 Mon Sep 17 00:00:00 2001
From: admin <alec@bryanmail.net>
Date: Wed, 5 Aug 2026 15:07:22 -0700
Subject: [PATCH] cutover: add full-session logging (writes to same dir as
 backup)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Persists the ENTIRE multi-phase transcript (Pre-Phase-0 through Phase
11, including Phase 0's preflight.sh output and any automatically
triggered rollback.sh output) to a single timestamped
cutover-session-<TS>.log in BACKUP_DIR (/volume1/SMB-docker/backup) —
the same directory the pg_dumpall backup lands in. Exports
CUTOVER_SESSION_LOG so child preflight.sh/rollback.sh invocations
append to the same file instead of opening their own. Adds an EXIT
trap that always announces final exit code + log path, specifically
so the one scenario RUNBOOK.md flags as needing manual intervention
(rollback.sh itself failing partway) is still fully investigable
after the fact even without a live terminal. Console/SSH output is
unchanged (tee mirrors to both).

See ADR-0001 note, Session Update 9.
---
 postgresql/cutover/cutover.sh | 61 +++++++++++++++++++++++++++++++++++
 1 file changed, 61 insertions(+)

diff --git a/postgresql/cutover/cutover.sh b/postgresql/cutover/cutover.sh
index 0573072..f631cf6 100644
--- a/postgresql/cutover/cutover.sh
+++ b/postgresql/cutover/cutover.sh
@@ -176,6 +176,24 @@
 # for the exact detection logic, the prompt itself, and the
 # AUTO_CLEANUP=yes escape hatch for unattended re-runs where the operator
 # has already decided cleanup is always wanted.
+#
+# 2026 run #6 (addition, no new run yet) — SESSION LOGGING added. Every
+# prior investigation into a failed run depended entirely on someone
+# having a live terminal open and scrolled back far enough, or on the
+# operator's own memory. This is fragile, especially for the one
+# genuinely dangerous scenario every phase above is built to avoid ever
+# reaching manually: rollback.sh itself failing partway (see
+# trigger_rollback() below — this is the one case RUNBOOK.md section 4
+# says requires manual intervention). Without a persisted transcript,
+# investigating THAT scenario after the fact means reconstructing what
+# happened from partial memory of a scrollback buffer that may already be
+# gone. FIXED: this script now writes its ENTIRE stdout/stderr transcript
+# (all 11 phases, plus Pre-Phase-0 and Phase 0's preflight.sh output, plus
+# any triggered rollback.sh output) to a single timestamped file in
+# BACKUP_DIR — the SAME directory the pg_dumpall backup itself lands in,
+# so the backup and the transcript of the run that produced/needed it are
+# always sitting right next to each other. See "SESSION LOGGING" comment
+# below for implementation detail. See ADR-0001 note, Session Update 9.
 
 set -uo pipefail   # NOTE: deliberately not -e — every phase below checks
                    # its own command's exit status explicitly so we can
@@ -192,6 +210,41 @@ NETWORK_NAME="postgresql_db-backend"
 PATRONI_SCOPE="postgres-ha"
 BACKUP_DIR="/volume1/SMB-docker/backup"
 
+# ── SESSION LOGGING (added — see ADR-0001 note, Session Update 9) ────────
+# Persists this run's ENTIRE multi-phase stdout/stderr transcript to a
+# single timestamped file in BACKUP_DIR — the SAME location preflight.sh
+# writes the pg_dumpall backup to, per explicit operator request: the
+# backup and the transcript of the run that needed it always end up
+# sitting right next to each other. CUTOVER_SESSION_LOG is exported so
+# preflight.sh and rollback.sh (both invoked below as plain
+# `bash <script>` child processes, which inherit exported env vars)
+# automatically detect it's already open and APPEND to this same file
+# instead of opening their own — one merged transcript per session,
+# covering Pre-Phase-0 through Phase 11 AND any automatically-triggered
+# rollback.sh output, not three separate fragments. `tee` mirrors
+# everything to the real stdout/stderr too, so live console/SSH-visible
+# output is completely unchanged — this is purely additive persistence.
+mkdir -p "$BACKUP_DIR"
+CUTOVER_SESSION_TS="$(date +%Y%m%d-%H%M%S)"
+CUTOVER_SESSION_LOG="${BACKUP_DIR}/cutover-session-${CUTOVER_SESSION_TS}.log"
+export CUTOVER_SESSION_LOG
+exec > >(tee -a "$CUTOVER_SESSION_LOG") 2>&1
+
+# Always print exactly where the transcript lives and what the final exit
+# code was, even on an unexpected/unhandled exit (e.g. a bug in this
+# script itself, not one of the deliberate trigger_rollback() paths) —
+# the exit code and log location should never be a mystery to whoever
+# investigates later.
+_cutover_exit_trap() {
+  local rc=$?
+  echo "" >&2
+  echo "[cutover $(date +%H:%M:%S)] Session ending, exit code ${rc}. Full transcript: ${CUTOVER_SESSION_LOG}" >&2
+  if [ "$rc" -ne 0 ]; then
+    echo "[cutover $(date +%H:%M:%S)] Non-zero exit — if this did NOT go through a clean trigger_rollback() path (check the log above for a matching FAIL line), treat current state as unverified and consult RUNBOOK.md section 4 before assuming it's safe." >&2
+  fi
+}
+trap _cutover_exit_trap EXIT
+
 # Consumer services checked in Phase 9 — (service_name, external health URL
 # or empty if internal-only). Derived from the dependency map in
 # RUNBOOK.md section 1. Update this list if the dependency map changes.
@@ -230,6 +283,8 @@ trigger_rollback() {
     echo "[cutover] ⚠️  rollback.sh ITSELF returned nonzero (${rc}). This is the one" >&2
     echo "[cutover] scenario requiring manual intervention — see RUNBOOK.md section 4," >&2
     echo "[cutover] 'If rollback.sh itself fails partway'. Do not blindly re-run scripts." >&2
+    echo "[cutover] Full transcript of everything up to this point (including rollback.sh's" >&2
+    echo "[cutover] own attempted output above) is preserved at: ${CUTOVER_SESSION_LOG}" >&2
   fi
   exit 1
 }
@@ -333,6 +388,11 @@ wait_for_stack_running() {
 
 # ═══════════════════════════════════════════════════════════════════════
 log "=== ADR-0001 Phase 3 real cutover starting ==="
+log "Full session transcript (this run, all phases + any triggered rollback.sh output): ${CUTOVER_SESSION_LOG}"
+if [ -d "${REPO_ROOT}/.git" ]; then
+  GIT_HEAD="$(git -C "$REPO_ROOT" rev-parse HEAD 2>/dev/null || echo unknown)"
+  log "Repo HEAD at ${REPO_ROOT}: ${GIT_HEAD} (confirm this matches origin/main — see ADR-0001 note's local-checkout-drift lesson, Session Update 8, if unsure)"
+fi
 log "Chat dependency reminder: this chat (ai_open-webui/litellm) is itself"
 log "a consumer of the DB being migrated. If this script exits nonzero"
 log "before completing rollback, follow RUNBOOK.md section 4 manually via"
@@ -742,6 +802,7 @@ AVAIL_GB=$(df --output=avail -BG /volume1/docker/PostgreSQL 2>/dev/null | tail -
 log "Disk space remaining at /volume1/docker/PostgreSQL: ${AVAIL_GB}GB"
 log "=== CUTOVER COMPLETE ==="
 log "Backup taken this run: $(cat ${BACKUP_DIR}/LATEST 2>/dev/null || echo 'unknown')"
+log "Full session transcript: ${CUTOVER_SESSION_LOG}"
 log "See RUNBOOK.md section 7 for manual post-cutover follow-up (legacy"
 log "decommission, external port 5430 / Traefik SNI handoff, burn-in period)."
 exit 0