compose-files/postgresql/cutover/preflight.sh

#!/usr/bin/env bash
# ADR-0001 Phase 3 Cutover — preflight.sh
#
# Run DIRECTLY on docker-2 via SSH (NOT via the LXC-exec MCP tool — that
# has a ~30s timeout and pg_dumpall against a 42GB instance will run far
# longer than that; this must run as a real, unbounded shell process).
#
#   ssh root@192.168.4.32
#   bash /volume1/docker/compose-files/postgresql/cutover/preflight.sh
#
# Exit code 0  = all hard gates passed, safe to proceed to cutover.sh
# Exit nonzero = hard stop; see output for which gate failed. Nothing is
#                modified by this script beyond writing a backup file —
#                it never touches the live legacy service, HA stack, or
#                any consumer.
#
# Hard gates (halt everything on failure):
#   1. Disk space  — need at least MIN_FREE_GB free before backup starts
#   2. Backup       — fresh pg_dumpall every run, unconditional, must
#                      produce a non-empty file with no reported errors
#
# Informational checks (logged, do NOT halt the script):
#   3. Catalog sanity   — enumerate live databases
#   4. Consumer enumeration — live network + static repo cross-check
#   5. Reconciliation  — match catalog dbs against known consumers /
#                          known no-consumer exceptions, flag anything
#                          truly unrecognized for human review
#
# Rationale for treating 3-5 as informational, not hard gates: database
# name <-> Swarm service/container name mapping is not reliably 1:1
# (e.g. the `authentik` database vs the `authentik-server` container), so
# fully automating a hard pass/fail per-database would be fragile and
# risks either false-positive halts or false-confidence passes. This
# check exists to surface anomalies for review, not to silently gate
# cutover on an imperfect heuristic.
#
# ── SESSION LOGGING (added — see ADR-0001 note, Session Update 9) ─────────
# Every run's full stdout/stderr transcript is persisted to a timestamped
# .log file in BACKUP_DIR — the SAME directory the pg_dumpall backup file
# itself lands in — so a failed run (or, worse, a failed automatic
# rollback.sh invoked afterward by cutover.sh) can still be investigated
# later even if nobody was watching a live terminal at the time. This
# script either appends to an already-open session log
# (CUTOVER_SESSION_LOG, exported by a parent cutover.sh so all phases land
# in ONE merged transcript) or opens its own timestamped log if run
# standalone. Console/SSH output is unchanged either way — tee mirrors to
# both the file and the real stdout/stderr.

set -euo pipefail

LEGACY_CONTAINER_FILTER="postgresql_postgresql"
PG_DATA_ROOT="/volume1/docker/PostgreSQL"
BACKUP_DIR="/volume1/SMB-docker/backup"
TS="$(date +%Y%m%d-%H%M%S)"
BACKUP_FILE="${BACKUP_DIR}/pre-cutover-${TS}.sql"
MIN_FREE_GB=50
REPO_ROOT="/volume1/docker/compose-files"
NETWORK_NAME="postgresql_db-backend"

# Known database(s) that exist in the shared legacy Postgres instance but
# have NO live Swarm consumer container attached to postgresql_db-backend
# (e.g. SparkyFitness — confirmed not deployed in prod as a running
# stack). These are NOT excluded from anything:
#   - pg_dumpall below captures the WHOLE instance regardless, so this
#     data is in the backup either way.
#   - Patroni's standby_cluster streams the entire physical cluster, not
#     per-database, so this data is replicated into the new HA primary
#     automatically with zero special handling.
# This list exists purely so step 5 doesn't flag a KNOWN, already-
# accepted case as if it were a surprise. Add to this list only for
# genuinely confirmed no-consumer databases — do not use it to silence
# a check you haven't actually investigated.
KNOWN_NO_CONSUMER_DBS=("sparkyfitness")

log()  { echo "[preflight $(date +%H:%M:%S)] $*"; }
warn() { echo "[preflight WARN $(date +%H:%M:%S)] $*" >&2; }
fail() { echo "[preflight FAIL $(date +%H:%M:%S)] $*" >&2; exit 1; }

# ── Session logging setup — MUST happen before any other output so the
# very first log lines are captured too. Deliberately uses the SAME
# BACKUP_DIR as the pg_dumpall backup file itself (not a separate log
# location), per explicit operator request: whoever goes looking for "the
# backup from that failed run" finds the matching transcript sitting
# right next to it.
mkdir -p "$BACKUP_DIR"
if [ -z "${CUTOVER_SESSION_LOG:-}" ]; then
  # No parent cutover.sh session log already open — standalone invocation,
  # open our own.
  CUTOVER_SESSION_LOG="${BACKUP_DIR}/preflight-standalone-${TS}.log"
  export CUTOVER_SESSION_LOG
fi
# Append (not truncate) — if a parent cutover.sh already wrote earlier
# output to this same file, we must not clobber it. tee mirrors to the
# real stdout/stderr too, so console/SSH-visible output is unchanged.
exec > >(tee -a "$CUTOVER_SESSION_LOG") 2>&1

log "=== ADR-0001 Phase 3 cutover preflight starting ==="
log "Full session transcript: ${CUTOVER_SESSION_LOG}"
if [ -d "${REPO_ROOT}/.git" ]; then
  GIT_HEAD="$(git -C "$REPO_ROOT" rev-parse HEAD 2>/dev/null || echo unknown)"
  log "Repo HEAD at ${REPO_ROOT}: ${GIT_HEAD} (confirm this matches origin/main — see ADR-0001 note's local-checkout-drift lesson, Session Update 8, if unsure)"
fi

# ── Gate 1: Disk space (checked BEFORE backup; halts before anything is written) ──
log "Checking free space on ${PG_DATA_ROOT} (resolves through symlinks to real backing filesystem)..."
if [ ! -d "$PG_DATA_ROOT" ]; then
  fail "${PG_DATA_ROOT} does not exist on this host — wrong node? This must run on docker-2."
fi
AVAIL_GB=$(df --output=avail -BG "$PG_DATA_ROOT" 2>/dev/null | tail -1 | tr -dc '0-9')
if [ -z "$AVAIL_GB" ]; then
  fail "Could not determine available disk space at ${PG_DATA_ROOT}."
fi
log "Available space: ${AVAIL_GB}GB (minimum required: ${MIN_FREE_GB}GB)"
if [ "$AVAIL_GB" -lt "$MIN_FREE_GB" ]; then
  fail "Only ${AVAIL_GB}GB free at ${PG_DATA_ROOT}, need at least ${MIN_FREE_GB}GB. Halting BEFORE backup — nothing has been written or touched."
fi
log "PASS: disk space gate (${AVAIL_GB}GB >= ${MIN_FREE_GB}GB)"

mkdir -p "$BACKUP_DIR"

# ── Gate 2: Fresh backup (unconditional every run) ────────────────────────
log "Taking fresh pg_dumpall backup of legacy instance -> ${BACKUP_FILE}"
LEGACY_CID=$(docker ps -q --filter "name=${LEGACY_CONTAINER_FILTER}" | head -1)
if [ -z "$LEGACY_CID" ]; then
  fail "Could not find a running legacy container matching '${LEGACY_CONTAINER_FILTER}'. Is the postgresql stack up?"
fi
log "Legacy container: ${LEGACY_CID}"

# pg_dumpall dumps ALL databases + roles/globals in the instance in one
# pass, including any database with no live Swarm consumer (e.g.
# sparkyfitness) — nothing is excluded from this backup regardless of how
# it's classified in the catalog-sanity step below.
if ! docker exec "$LEGACY_CID" bash -c 'pg_dumpall -U "$POSTGRES_USER"' > "$BACKUP_FILE" 2> "${BACKUP_FILE}.stderr"; then
  fail "pg_dumpall exited nonzero. See ${BACKUP_FILE}.stderr for detail. Backup NOT trusted, halting."
fi
if [ ! -s "$BACKUP_FILE" ]; then
  fail "Backup file is empty — pg_dumpall produced no output. See ${BACKUP_FILE}.stderr"
fi
if grep -qi "error" "${BACKUP_FILE}.stderr"; then
  fail "pg_dumpall reported errors on stderr — see ${BACKUP_FILE}.stderr. Backup NOT trusted, halting."
fi
BACKUP_SIZE=$(du -h "$BACKUP_FILE" | cut -f1)
log "PASS: backup gate — ${BACKUP_FILE} (${BACKUP_SIZE})"
rm -f "${BACKUP_FILE}.stderr"

# Record the backup path so cutover.sh/rollback.sh can reference the most
# recent one without re-deriving the timestamp.
echo "$BACKUP_FILE" > "${BACKUP_DIR}/LATEST"

# ── Informational 3: Catalog sanity — enumerate live databases ───────────
log "Enumerating live databases in legacy instance..."
LIVE_DBS=$(docker exec "$LEGACY_CID" bash -c 'psql -U "$POSTGRES_USER" -tAc "SELECT datname FROM pg_database WHERE datistemplate = false AND datname NOT IN ('"'"'postgres'"'"');"')
log "Databases found in catalog:"
echo "$LIVE_DBS" | sed 's/^/    - /'

# ── Informational 4: Authoritative consumer enumeration ───────────────────
log "Enumerating live containers attached to ${NETWORK_NAME}..."
LIVE_CONSUMERS=$(docker network inspect "$NETWORK_NAME" --format '{{range .Containers}}{{.Name}}{{"\n"}}{{end}}' 2>/dev/null || true)
if [ -z "$LIVE_CONSUMERS" ]; then
  warn "Could not inspect network ${NETWORK_NAME}, or it has no attached containers. This is unusual — review before proceeding."
else
  log "Live containers on ${NETWORK_NAME}:"
  echo "$LIVE_CONSUMERS" | sed 's/^/    - /'
fi

log "Cross-checking against repo (static grep for DB references, catches anything not currently running)..."
if [ -d "$REPO_ROOT" ]; then
  GREP_HITS=$(grep -rl -E "postgresql_db-backend|DATABASE_URL|POSTGRES_(HOST|USER|PASSWORD)|AUTHENTIK_POSTGRESQL" "$REPO_ROOT" --include="*.yaml" --include="*.yml" 2>/dev/null || true)
  log "Compose files referencing the DB network/env vars:"
  echo "$GREP_HITS" | sed 's/^/    - /'
else
  warn "Repo root ${REPO_ROOT} not found on this host — skipping static cross-check. Live network enumeration above is still authoritative for 'attached right now', but anything not currently running would be missed."
fi

# ── Informational 5: Reconcile catalog vs consumers / known exceptions ────
log "Reconciling catalog databases against known no-consumer exceptions..."
while IFS= read -r db; do
  [ -z "$db" ] && continue
  is_known_exception=false
  for known in "${KNOWN_NO_CONSUMER_DBS[@]}"; do
    if [ "$db" == "$known" ]; then
      is_known_exception=true
      break
    fi
  done
  if [ "$is_known_exception" = true ]; then
    log "  '${db}': known no-consumer database (accepted exception, not in Prod as a deployed app). Fully covered by the pg_dumpall backup above and will be fully covered by standby_cluster's whole-instance physical replication — no exclusion, no special handling, just flagged for visibility."
  else
    log "  '${db}': present in catalog — cross-reference against the consumer/compose lists above. If this is a database you don't recognize, STOP and investigate before running cutover.sh."
  fi
done <<< "$LIVE_DBS"

log "PASS (informational): catalog/consumer enumeration complete — review the lists above."
log "=== Preflight complete. Backup: ${BACKUP_FILE} (${BACKUP_SIZE}) ==="
log "=== Hard gates (disk space, backup) both passed. Proceed to cutover.sh after reviewing the catalog/consumer lists above. ==="