From 01768d14f58bb0a524c63c62ad6d4cbcf040dbc9 Mon Sep 17 00:00:00 2001
From: admin <alec@bryanmail.net>
Date: Sun, 2 Aug 2026 16:45:44 -0700
Subject: [PATCH] ADR-0001 Phase 3: add preflight.sh (disk/backup hard gates,
 catalog+consumer enumeration)

---
 postgresql/cutover/preflight.sh | 160 ++++++++++++++++++++++++++++++++
 1 file changed, 160 insertions(+)
 create mode 100644 postgresql/cutover/preflight.sh

diff --git a/postgresql/cutover/preflight.sh b/postgresql/cutover/preflight.sh
new file mode 100644
index 0000000..fd3658d
--- /dev/null
+++ b/postgresql/cutover/preflight.sh
@@ -0,0 +1,160 @@
+#!/usr/bin/env bash
+# ADR-0001 Phase 3 Cutover — preflight.sh
+#
+# Run DIRECTLY on docker-2 via SSH (NOT via the LXC-exec MCP tool — that
+# has a ~30s timeout and pg_dumpall against a 42GB instance will run far
+# longer than that; this must run as a real, unbounded shell process).
+#
+#   ssh root@192.168.4.32
+#   bash /volume1/docker/compose-files/postgresql/cutover/preflight.sh
+#
+# Exit code 0  = all hard gates passed, safe to proceed to cutover.sh
+# Exit nonzero = hard stop; see output for which gate failed. Nothing is
+#                modified by this script beyond writing a backup file —
+#                it never touches the live legacy service, HA stack, or
+#                any consumer.
+#
+# Hard gates (halt everything on failure):
+#   1. Disk space  — need at least MIN_FREE_GB free before backup starts
+#   2. Backup       — fresh pg_dumpall every run, unconditional, must
+#                      produce a non-empty file with no reported errors
+#
+# Informational checks (logged, do NOT halt the script):
+#   3. Catalog sanity   — enumerate live databases
+#   4. Consumer enumeration — live network + static repo cross-check
+#   5. Reconciliation  — match catalog dbs against known consumers /
+#                          known no-consumer exceptions, flag anything
+#                          truly unrecognized for human review
+#
+# Rationale for treating 3-5 as informational, not hard gates: database
+# name <-> Swarm service/container name mapping is not reliably 1:1
+# (e.g. the `authentik` database vs the `authentik-server` container), so
+# fully automating a hard pass/fail per-database would be fragile and
+# risks either false-positive halts or false-confidence passes. This
+# check exists to surface anomalies for review, not to silently gate
+# cutover on an imperfect heuristic.
+
+set -euo pipefail
+
+LEGACY_CONTAINER_FILTER="postgresql_postgresql"
+PG_DATA_ROOT="/volume1/docker/PostgreSQL"
+BACKUP_DIR="${PG_DATA_ROOT}/cutover-backups"
+TS="$(date +%Y%m%d-%H%M%S)"
+BACKUP_FILE="${BACKUP_DIR}/pre-cutover-${TS}.sql"
+MIN_FREE_GB=50
+REPO_ROOT="/volume1/docker/compose-files"
+NETWORK_NAME="postgresql_db-backend"
+
+# Known database(s) that exist in the shared legacy Postgres instance but
+# have NO live Swarm consumer container attached to postgresql_db-backend
+# (e.g. SparkyFitness — confirmed not deployed in prod as a running
+# stack). These are NOT excluded from anything:
+#   - pg_dumpall below captures the WHOLE instance regardless, so this
+#     data is in the backup either way.
+#   - Patroni's standby_cluster streams the entire physical cluster, not
+#     per-database, so this data is replicated into the new HA primary
+#     automatically with zero special handling.
+# This list exists purely so step 5 doesn't flag a KNOWN, already-
+# accepted case as if it were a surprise. Add to this list only for
+# genuinely confirmed no-consumer databases — do not use it to silence
+# a check you haven't actually investigated.
+KNOWN_NO_CONSUMER_DBS=("sparkyfitness")
+
+log()  { echo "[preflight $(date +%H:%M:%S)] $*"; }
+warn() { echo "[preflight WARN $(date +%H:%M:%S)] $*" >&2; }
+fail() { echo "[preflight FAIL $(date +%H:%M:%S)] $*" >&2; exit 1; }
+
+log "=== ADR-0001 Phase 3 cutover preflight starting ==="
+
+# ── Gate 1: Disk space (checked BEFORE backup; halts before anything is written) ──
+log "Checking free space on ${PG_DATA_ROOT} (resolves through symlinks to real backing filesystem)..."
+if [ ! -d "$PG_DATA_ROOT" ]; then
+  fail "${PG_DATA_ROOT} does not exist on this host — wrong node? This must run on docker-2."
+fi
+AVAIL_GB=$(df --output=avail -BG "$PG_DATA_ROOT" 2>/dev/null | tail -1 | tr -dc '0-9')
+if [ -z "$AVAIL_GB" ]; then
+  fail "Could not determine available disk space at ${PG_DATA_ROOT}."
+fi
+log "Available space: ${AVAIL_GB}GB (minimum required: ${MIN_FREE_GB}GB)"
+if [ "$AVAIL_GB" -lt "$MIN_FREE_GB" ]; then
+  fail "Only ${AVAIL_GB}GB free at ${PG_DATA_ROOT}, need at least ${MIN_FREE_GB}GB. Halting BEFORE backup — nothing has been written or touched."
+fi
+log "PASS: disk space gate (${AVAIL_GB}GB >= ${MIN_FREE_GB}GB)"
+
+mkdir -p "$BACKUP_DIR"
+
+# ── Gate 2: Fresh backup (unconditional every run) ────────────────────────
+log "Taking fresh pg_dumpall backup of legacy instance -> ${BACKUP_FILE}"
+LEGACY_CID=$(docker ps -q --filter "name=${LEGACY_CONTAINER_FILTER}" | head -1)
+if [ -z "$LEGACY_CID" ]; then
+  fail "Could not find a running legacy container matching '${LEGACY_CONTAINER_FILTER}'. Is the postgresql stack up?"
+fi
+log "Legacy container: ${LEGACY_CID}"
+
+# pg_dumpall dumps ALL databases + roles/globals in the instance in one
+# pass, including any database with no live Swarm consumer (e.g.
+# sparkyfitness) — nothing is excluded from this backup regardless of how
+# it's classified in the catalog-sanity step below.
+if ! docker exec "$LEGACY_CID" bash -c 'pg_dumpall -U "$POSTGRES_USER"' > "$BACKUP_FILE" 2> "${BACKUP_FILE}.stderr"; then
+  fail "pg_dumpall exited nonzero. See ${BACKUP_FILE}.stderr for detail. Backup NOT trusted, halting."
+fi
+if [ ! -s "$BACKUP_FILE" ]; then
+  fail "Backup file is empty — pg_dumpall produced no output. See ${BACKUP_FILE}.stderr"
+fi
+if grep -qi "error" "${BACKUP_FILE}.stderr"; then
+  fail "pg_dumpall reported errors on stderr — see ${BACKUP_FILE}.stderr. Backup NOT trusted, halting."
+fi
+BACKUP_SIZE=$(du -h "$BACKUP_FILE" | cut -f1)
+log "PASS: backup gate — ${BACKUP_FILE} (${BACKUP_SIZE})"
+rm -f "${BACKUP_FILE}.stderr"
+
+# Record the backup path so cutover.sh/rollback.sh can reference the most
+# recent one without re-deriving the timestamp.
+echo "$BACKUP_FILE" > "${BACKUP_DIR}/LATEST"
+
+# ── Informational 3: Catalog sanity — enumerate live databases ───────────
+log "Enumerating live databases in legacy instance..."
+LIVE_DBS=$(docker exec "$LEGACY_CID" bash -c 'psql -U "$POSTGRES_USER" -tAc "SELECT datname FROM pg_database WHERE datistemplate = false AND datname NOT IN ('"'"'postgres'"'"');"')
+log "Databases found in catalog:"
+echo "$LIVE_DBS" | sed 's/^/    - /'
+
+# ── Informational 4: Authoritative consumer enumeration ───────────────────
+log "Enumerating live containers attached to ${NETWORK_NAME}..."
+LIVE_CONSUMERS=$(docker network inspect "$NETWORK_NAME" --format '{{range .Containers}}{{.Name}}{{"\n"}}{{end}}' 2>/dev/null || true)
+if [ -z "$LIVE_CONSUMERS" ]; then
+  warn "Could not inspect network ${NETWORK_NAME}, or it has no attached containers. This is unusual — review before proceeding."
+else
+  log "Live containers on ${NETWORK_NAME}:"
+  echo "$LIVE_CONSUMERS" | sed 's/^/    - /'
+fi
+
+log "Cross-checking against repo (static grep for DB references, catches anything not currently running)..."
+if [ -d "$REPO_ROOT" ]; then
+  GREP_HITS=$(grep -rl -E "postgresql_db-backend|DATABASE_URL|POSTGRES_(HOST|USER|PASSWORD)|AUTHENTIK_POSTGRESQL" "$REPO_ROOT" --include="*.yaml" --include="*.yml" 2>/dev/null || true)
+  log "Compose files referencing the DB network/env vars:"
+  echo "$GREP_HITS" | sed 's/^/    - /'
+else
+  warn "Repo root ${REPO_ROOT} not found on this host — skipping static cross-check. Live network enumeration above is still authoritative for 'attached right now', but anything not currently running would be missed."
+fi
+
+# ── Informational 5: Reconcile catalog vs consumers / known exceptions ────
+log "Reconciling catalog databases against known no-consumer exceptions..."
+while IFS= read -r db; do
+  [ -z "$db" ] && continue
+  is_known_exception=false
+  for known in "${KNOWN_NO_CONSUMER_DBS[@]}"; do
+    if [ "$db" == "$known" ]; then
+      is_known_exception=true
+      break
+    fi
+  done
+  if [ "$is_known_exception" = true ]; then
+    log "  '${db}': known no-consumer database (accepted exception, not in Prod as a deployed app). Fully covered by the pg_dumpall backup above and will be fully covered by standby_cluster's whole-instance physical replication — no exclusion, no special handling, just flagged for visibility."
+  else
+    log "  '${db}': present in catalog — cross-reference against the consumer/compose lists above. If this is a database you don't recognize, STOP and investigate before running cutover.sh."
+  fi
+done <<< "$LIVE_DBS"
+
+log "PASS (informational): catalog/consumer enumeration complete — review the lists above."
+log "=== Preflight complete. Backup: ${BACKUP_FILE} (${BACKUP_SIZE}) ==="
+log "=== Hard gates (disk space, backup) both passed. Proceed to cutover.sh after reviewing the catalog/consumer lists above. ==="