ADR-0001 Phase 3: add preflight.sh (disk/backup hard gates, catalog+consumer enumeration)
ci/woodpecker/push/deploy Pipeline was successful
ci/woodpecker/push/deploy Pipeline was successful
This commit is contained in:
@@ -0,0 +1,160 @@
|
||||
#!/usr/bin/env bash
|
||||
# ADR-0001 Phase 3 Cutover — preflight.sh
|
||||
#
|
||||
# Run DIRECTLY on docker-2 via SSH (NOT via the LXC-exec MCP tool — that
|
||||
# has a ~30s timeout and pg_dumpall against a 42GB instance will run far
|
||||
# longer than that; this must run as a real, unbounded shell process).
|
||||
#
|
||||
# ssh root@192.168.4.32
|
||||
# bash /volume1/docker/compose-files/postgresql/cutover/preflight.sh
|
||||
#
|
||||
# Exit code 0 = all hard gates passed, safe to proceed to cutover.sh
|
||||
# Exit nonzero = hard stop; see output for which gate failed. Nothing is
|
||||
# modified by this script beyond writing a backup file —
|
||||
# it never touches the live legacy service, HA stack, or
|
||||
# any consumer.
|
||||
#
|
||||
# Hard gates (halt everything on failure):
|
||||
# 1. Disk space — need at least MIN_FREE_GB free before backup starts
|
||||
# 2. Backup — fresh pg_dumpall every run, unconditional, must
|
||||
# produce a non-empty file with no reported errors
|
||||
#
|
||||
# Informational checks (logged, do NOT halt the script):
|
||||
# 3. Catalog sanity — enumerate live databases
|
||||
# 4. Consumer enumeration — live network + static repo cross-check
|
||||
# 5. Reconciliation — match catalog dbs against known consumers /
|
||||
# known no-consumer exceptions, flag anything
|
||||
# truly unrecognized for human review
|
||||
#
|
||||
# Rationale for treating 3-5 as informational, not hard gates: database
|
||||
# name <-> Swarm service/container name mapping is not reliably 1:1
|
||||
# (e.g. the `authentik` database vs the `authentik-server` container), so
|
||||
# fully automating a hard pass/fail per-database would be fragile and
|
||||
# risks either false-positive halts or false-confidence passes. This
|
||||
# check exists to surface anomalies for review, not to silently gate
|
||||
# cutover on an imperfect heuristic.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
LEGACY_CONTAINER_FILTER="postgresql_postgresql"
|
||||
PG_DATA_ROOT="/volume1/docker/PostgreSQL"
|
||||
BACKUP_DIR="${PG_DATA_ROOT}/cutover-backups"
|
||||
TS="$(date +%Y%m%d-%H%M%S)"
|
||||
BACKUP_FILE="${BACKUP_DIR}/pre-cutover-${TS}.sql"
|
||||
MIN_FREE_GB=50
|
||||
REPO_ROOT="/volume1/docker/compose-files"
|
||||
NETWORK_NAME="postgresql_db-backend"
|
||||
|
||||
# Known database(s) that exist in the shared legacy Postgres instance but
|
||||
# have NO live Swarm consumer container attached to postgresql_db-backend
|
||||
# (e.g. SparkyFitness — confirmed not deployed in prod as a running
|
||||
# stack). These are NOT excluded from anything:
|
||||
# - pg_dumpall below captures the WHOLE instance regardless, so this
|
||||
# data is in the backup either way.
|
||||
# - Patroni's standby_cluster streams the entire physical cluster, not
|
||||
# per-database, so this data is replicated into the new HA primary
|
||||
# automatically with zero special handling.
|
||||
# This list exists purely so step 5 doesn't flag a KNOWN, already-
|
||||
# accepted case as if it were a surprise. Add to this list only for
|
||||
# genuinely confirmed no-consumer databases — do not use it to silence
|
||||
# a check you haven't actually investigated.
|
||||
KNOWN_NO_CONSUMER_DBS=("sparkyfitness")
|
||||
|
||||
log() { echo "[preflight $(date +%H:%M:%S)] $*"; }
|
||||
warn() { echo "[preflight WARN $(date +%H:%M:%S)] $*" >&2; }
|
||||
fail() { echo "[preflight FAIL $(date +%H:%M:%S)] $*" >&2; exit 1; }
|
||||
|
||||
log "=== ADR-0001 Phase 3 cutover preflight starting ==="
|
||||
|
||||
# ── Gate 1: Disk space (checked BEFORE backup; halts before anything is written) ──
|
||||
log "Checking free space on ${PG_DATA_ROOT} (resolves through symlinks to real backing filesystem)..."
|
||||
if [ ! -d "$PG_DATA_ROOT" ]; then
|
||||
fail "${PG_DATA_ROOT} does not exist on this host — wrong node? This must run on docker-2."
|
||||
fi
|
||||
AVAIL_GB=$(df --output=avail -BG "$PG_DATA_ROOT" 2>/dev/null | tail -1 | tr -dc '0-9')
|
||||
if [ -z "$AVAIL_GB" ]; then
|
||||
fail "Could not determine available disk space at ${PG_DATA_ROOT}."
|
||||
fi
|
||||
log "Available space: ${AVAIL_GB}GB (minimum required: ${MIN_FREE_GB}GB)"
|
||||
if [ "$AVAIL_GB" -lt "$MIN_FREE_GB" ]; then
|
||||
fail "Only ${AVAIL_GB}GB free at ${PG_DATA_ROOT}, need at least ${MIN_FREE_GB}GB. Halting BEFORE backup — nothing has been written or touched."
|
||||
fi
|
||||
log "PASS: disk space gate (${AVAIL_GB}GB >= ${MIN_FREE_GB}GB)"
|
||||
|
||||
mkdir -p "$BACKUP_DIR"
|
||||
|
||||
# ── Gate 2: Fresh backup (unconditional every run) ────────────────────────
|
||||
log "Taking fresh pg_dumpall backup of legacy instance -> ${BACKUP_FILE}"
|
||||
LEGACY_CID=$(docker ps -q --filter "name=${LEGACY_CONTAINER_FILTER}" | head -1)
|
||||
if [ -z "$LEGACY_CID" ]; then
|
||||
fail "Could not find a running legacy container matching '${LEGACY_CONTAINER_FILTER}'. Is the postgresql stack up?"
|
||||
fi
|
||||
log "Legacy container: ${LEGACY_CID}"
|
||||
|
||||
# pg_dumpall dumps ALL databases + roles/globals in the instance in one
|
||||
# pass, including any database with no live Swarm consumer (e.g.
|
||||
# sparkyfitness) — nothing is excluded from this backup regardless of how
|
||||
# it's classified in the catalog-sanity step below.
|
||||
if ! docker exec "$LEGACY_CID" bash -c 'pg_dumpall -U "$POSTGRES_USER"' > "$BACKUP_FILE" 2> "${BACKUP_FILE}.stderr"; then
|
||||
fail "pg_dumpall exited nonzero. See ${BACKUP_FILE}.stderr for detail. Backup NOT trusted, halting."
|
||||
fi
|
||||
if [ ! -s "$BACKUP_FILE" ]; then
|
||||
fail "Backup file is empty — pg_dumpall produced no output. See ${BACKUP_FILE}.stderr"
|
||||
fi
|
||||
if grep -qi "error" "${BACKUP_FILE}.stderr"; then
|
||||
fail "pg_dumpall reported errors on stderr — see ${BACKUP_FILE}.stderr. Backup NOT trusted, halting."
|
||||
fi
|
||||
BACKUP_SIZE=$(du -h "$BACKUP_FILE" | cut -f1)
|
||||
log "PASS: backup gate — ${BACKUP_FILE} (${BACKUP_SIZE})"
|
||||
rm -f "${BACKUP_FILE}.stderr"
|
||||
|
||||
# Record the backup path so cutover.sh/rollback.sh can reference the most
|
||||
# recent one without re-deriving the timestamp.
|
||||
echo "$BACKUP_FILE" > "${BACKUP_DIR}/LATEST"
|
||||
|
||||
# ── Informational 3: Catalog sanity — enumerate live databases ───────────
|
||||
log "Enumerating live databases in legacy instance..."
|
||||
LIVE_DBS=$(docker exec "$LEGACY_CID" bash -c 'psql -U "$POSTGRES_USER" -tAc "SELECT datname FROM pg_database WHERE datistemplate = false AND datname NOT IN ('"'"'postgres'"'"');"')
|
||||
log "Databases found in catalog:"
|
||||
echo "$LIVE_DBS" | sed 's/^/ - /'
|
||||
|
||||
# ── Informational 4: Authoritative consumer enumeration ───────────────────
|
||||
log "Enumerating live containers attached to ${NETWORK_NAME}..."
|
||||
LIVE_CONSUMERS=$(docker network inspect "$NETWORK_NAME" --format '{{range .Containers}}{{.Name}}{{"\n"}}{{end}}' 2>/dev/null || true)
|
||||
if [ -z "$LIVE_CONSUMERS" ]; then
|
||||
warn "Could not inspect network ${NETWORK_NAME}, or it has no attached containers. This is unusual — review before proceeding."
|
||||
else
|
||||
log "Live containers on ${NETWORK_NAME}:"
|
||||
echo "$LIVE_CONSUMERS" | sed 's/^/ - /'
|
||||
fi
|
||||
|
||||
log "Cross-checking against repo (static grep for DB references, catches anything not currently running)..."
|
||||
if [ -d "$REPO_ROOT" ]; then
|
||||
GREP_HITS=$(grep -rl -E "postgresql_db-backend|DATABASE_URL|POSTGRES_(HOST|USER|PASSWORD)|AUTHENTIK_POSTGRESQL" "$REPO_ROOT" --include="*.yaml" --include="*.yml" 2>/dev/null || true)
|
||||
log "Compose files referencing the DB network/env vars:"
|
||||
echo "$GREP_HITS" | sed 's/^/ - /'
|
||||
else
|
||||
warn "Repo root ${REPO_ROOT} not found on this host — skipping static cross-check. Live network enumeration above is still authoritative for 'attached right now', but anything not currently running would be missed."
|
||||
fi
|
||||
|
||||
# ── Informational 5: Reconcile catalog vs consumers / known exceptions ────
|
||||
log "Reconciling catalog databases against known no-consumer exceptions..."
|
||||
while IFS= read -r db; do
|
||||
[ -z "$db" ] && continue
|
||||
is_known_exception=false
|
||||
for known in "${KNOWN_NO_CONSUMER_DBS[@]}"; do
|
||||
if [ "$db" == "$known" ]; then
|
||||
is_known_exception=true
|
||||
break
|
||||
fi
|
||||
done
|
||||
if [ "$is_known_exception" = true ]; then
|
||||
log " '${db}': known no-consumer database (accepted exception, not in Prod as a deployed app). Fully covered by the pg_dumpall backup above and will be fully covered by standby_cluster's whole-instance physical replication — no exclusion, no special handling, just flagged for visibility."
|
||||
else
|
||||
log " '${db}': present in catalog — cross-reference against the consumer/compose lists above. If this is a database you don't recognize, STOP and investigate before running cutover.sh."
|
||||
fi
|
||||
done <<< "$LIVE_DBS"
|
||||
|
||||
log "PASS (informational): catalog/consumer enumeration complete — review the lists above."
|
||||
log "=== Preflight complete. Backup: ${BACKUP_FILE} (${BACKUP_SIZE}) ==="
|
||||
log "=== Hard gates (disk space, backup) both passed. Proceed to cutover.sh after reviewing the catalog/consumer lists above. ==="
|
||||
Reference in New Issue
Block a user