Add Proxmox backup hook script

2026-06-18 22:44:40 -07:00
parent 696a481ccf
commit 514d1f73cb
1 changed files with 132 additions and 0 deletions
@@ -0,0 +1,132 @@
+#!/bin/bash
+# vzdump-docker-hook.sh — Proxmox backup hook
+# Gracefully stops/restarts Docker inside docker LXCs around snapshot backups
+# to prevent Swarm overlay network corruption.
+#
+# Proxmox calls this script with: $1=phase $2=vmid
+# We act on: backup-start (stop Docker), post-restart + backup-abort (start Docker)
+#
+# Install: copy to /usr/local/bin/vzdump-docker-hook.sh on every Proxmox node
+# then add  script: /usr/local/bin/vzdump-docker-hook.sh  to each backup job.
+
+set -euo pipefail
+
+PHASE="$1"
+VMID="${2:-}"
+
+# Only act on our docker LXC IDs — edit this list if you add more docker hosts
+DOCKER_VMIDS="4031 4032 4033"
+
+# Teams webhook for notifications (re-uses same webhook as watchdog)
+TEAMS_WEBHOOK_URL="${TEAMS_WEBHOOK_URL:-}"
+TEAMS_ENV="/etc/docker-prune-watchdog.env"
+[ -f "${TEAMS_ENV}" ] && source "${TEAMS_ENV}" || true
+
+# How long to wait for Docker to stop gracefully before giving up (seconds)
+DOCKER_STOP_TIMEOUT="${DOCKER_STOP_TIMEOUT:-60}"
+
+# Log file on the Proxmox node (not inside the LXC)
+LOG_FILE="/var/log/vzdump-docker-hook.log"
+
+log() {
+    echo "[$(date '+%Y-%m-%d %H:%M:%S')] [HOOK] [vmid=${VMID}] [phase=${PHASE}] $*" \
+        | tee -a "${LOG_FILE}" >&2
+}
+
+notify() {
+    local title="$1" msg="$2" color="${3:-0078d4}"
+    [ -z "${TEAMS_WEBHOOK_URL:-}" ] && return 0
+    curl -s -X POST -H 'Content-Type: application/json' \
+        -d "{\"title\":\"${title}\",\"text\":\"${msg}\",\"themeColor\":\"${color}\"}" \
+        "${TEAMS_WEBHOOK_URL}" >/dev/null 2>&1 || true
+}
+
+is_docker_vmid() {
+    for id in ${DOCKER_VMIDS}; do
+        [ "${id}" = "${VMID}" ] && return 0
+    done
+    return 1
+}
+
+lxc_exec() {
+    pct exec "${VMID}" -- "$@"
+}
+
+docker_is_running() {
+    lxc_exec systemctl is-active docker >/dev/null 2>&1
+}
+
+
+stop_docker() {
+    local h; h=$(lxc_exec hostname 2>/dev/null || echo "vmid-${VMID}")
+    log "Stopping Docker on ${h}..."
+    if lxc_exec docker info 2>/dev/null | grep -q "Swarm: active"; then
+        local nid; nid=$(lxc_exec docker info --format '{{.Swarm.NodeID}}' 2>/dev/null || true)
+        if [ -n "${nid}" ]; then
+            log "Draining Swarm node ${nid} on ${h}..."
+            lxc_exec docker node update --availability drain "${nid}" >/dev/null 2>&1 || true
+            sleep 10
+        fi
+    fi
+    lxc_exec systemctl stop docker || true
+    local w=0
+    while docker_is_running && [ "${w}" -lt "${DOCKER_STOP_TIMEOUT}" ]; do
+        sleep 2; w=$((w+2))
+    done
+    if docker_is_running; then
+        log "WARNING: Docker still running after ${DOCKER_STOP_TIMEOUT}s on ${h}"
+        notify "Backup Warning - ${h}" "vmid ${VMID}: Docker did not stop before snapshot." "ffaa00"
+    else
+        log "Docker stopped cleanly on ${h} in ${w}s"
+        notify "Backup: Docker Stopped - ${h}" "vmid ${VMID}: Stopped OK. Snapshot starting." "0078d4"
+    fi
+}
+
+start_docker() {
+    local h; h=$(lxc_exec hostname 2>/dev/null || echo "vmid-${VMID}")
+    log "Restarting Docker on ${h} after snapshot..."
+    lxc_exec systemctl start docker || true
+    local w=0
+    while ! docker_is_running && [ "${w}" -lt "${DOCKER_STOP_TIMEOUT}" ]; do
+        sleep 2; w=$((w+2))
+    done
+    if ! docker_is_running; then
+        log "ERROR: Docker failed to start on ${h} after ${w}s"
+        notify "CRITICAL: Docker Failed - ${h}" "vmid ${VMID}: Docker did not restart after snapshot! Manual fix needed." "ff0000"
+        return 1
+    fi
+    log "Docker started on ${h} in ${w}s"
+    if lxc_exec docker info 2>/dev/null | grep -q "Swarm: active"; then
+        local nid; nid=$(lxc_exec docker info --format '{{.Swarm.NodeID}}' 2>/dev/null || true)
+        if [ -n "${nid}" ]; then
+            log "Restoring Swarm node ${nid} to active on ${h}..."
+            lxc_exec docker node update --availability active "${nid}" >/dev/null 2>&1 || true
+            log "Swarm node ${nid} restored to active"
+        fi
+    fi
+    notify "Backup: Docker Restarted - ${h}" "vmid ${VMID}: Docker restarted. Swarm node active." "00aa00"
+}
+
+# ── Main phase dispatch ──────────────────────────────────────────────────────
+log "Hook called: phase=${PHASE} vmid=${VMID}"
+
+if ! is_docker_vmid; then
+    log "vmid ${VMID} not in DOCKER_VMIDS list — skipping"
+    exit 0
+fi
+
+case "${PHASE}" in
+    backup-start)  stop_docker ;;
+    post-restart)  start_docker ;;
+    backup-abort)
+        log "Backup aborted — restarting Docker to restore services..."
+        notify "Backup Aborted - vmid ${VMID}" "Backup failed. Restarting Docker to restore services." "ffaa00"
+        start_docker || true
+        ;;
+    job-init|job-start|job-end|job-abort|log-end|pre-stop|pre-restart)
+        log "Phase '${PHASE}' — no action needed" ;;
+    *)
+        log "Unhandled phase '${PHASE}' — skipping" ;;
+esac
+
+exit 0