Archived
Remove backup scripts - moving to homelab-scripts repo
This commit is contained in:
@@ -1,132 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
# vzdump-docker-hook.sh — Proxmox backup hook
|
|
||||||
# Gracefully stops/restarts Docker inside docker LXCs around snapshot backups
|
|
||||||
# to prevent Swarm overlay network corruption.
|
|
||||||
#
|
|
||||||
# Proxmox calls this script with: $1=phase $2=vmid
|
|
||||||
# We act on: backup-start (stop Docker), post-restart + backup-abort (start Docker)
|
|
||||||
#
|
|
||||||
# Install: copy to /usr/local/bin/vzdump-docker-hook.sh on every Proxmox node
|
|
||||||
# then add script: /usr/local/bin/vzdump-docker-hook.sh to each backup job.
|
|
||||||
|
|
||||||
set -euo pipefail
|
|
||||||
|
|
||||||
PHASE="$1"
|
|
||||||
VMID="${2:-}"
|
|
||||||
|
|
||||||
# Only act on our docker LXC IDs — edit this list if you add more docker hosts
|
|
||||||
DOCKER_VMIDS="4031 4032 4033"
|
|
||||||
|
|
||||||
# Teams webhook for notifications (re-uses same webhook as watchdog)
|
|
||||||
TEAMS_WEBHOOK_URL="${TEAMS_WEBHOOK_URL:-}"
|
|
||||||
TEAMS_ENV="/etc/docker-prune-watchdog.env"
|
|
||||||
[ -f "${TEAMS_ENV}" ] && source "${TEAMS_ENV}" || true
|
|
||||||
|
|
||||||
# How long to wait for Docker to stop gracefully before giving up (seconds)
|
|
||||||
DOCKER_STOP_TIMEOUT="${DOCKER_STOP_TIMEOUT:-60}"
|
|
||||||
|
|
||||||
# Log file on the Proxmox node (not inside the LXC)
|
|
||||||
LOG_FILE="/var/log/vzdump-docker-hook.log"
|
|
||||||
|
|
||||||
log() {
|
|
||||||
echo "[$(date '+%Y-%m-%d %H:%M:%S')] [HOOK] [vmid=${VMID}] [phase=${PHASE}] $*" \
|
|
||||||
| tee -a "${LOG_FILE}" >&2
|
|
||||||
}
|
|
||||||
|
|
||||||
notify() {
|
|
||||||
local title="$1" msg="$2" color="${3:-0078d4}"
|
|
||||||
[ -z "${TEAMS_WEBHOOK_URL:-}" ] && return 0
|
|
||||||
curl -s -X POST -H 'Content-Type: application/json' \
|
|
||||||
-d "{\"title\":\"${title}\",\"text\":\"${msg}\",\"themeColor\":\"${color}\"}" \
|
|
||||||
"${TEAMS_WEBHOOK_URL}" >/dev/null 2>&1 || true
|
|
||||||
}
|
|
||||||
|
|
||||||
is_docker_vmid() {
|
|
||||||
for id in ${DOCKER_VMIDS}; do
|
|
||||||
[ "${id}" = "${VMID}" ] && return 0
|
|
||||||
done
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
lxc_exec() {
|
|
||||||
pct exec "${VMID}" -- "$@"
|
|
||||||
}
|
|
||||||
|
|
||||||
docker_is_running() {
|
|
||||||
lxc_exec systemctl is-active docker >/dev/null 2>&1
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
stop_docker() {
|
|
||||||
local h; h=$(lxc_exec hostname 2>/dev/null || echo "vmid-${VMID}")
|
|
||||||
log "Stopping Docker on ${h}..."
|
|
||||||
if lxc_exec docker info 2>/dev/null | grep -q "Swarm: active"; then
|
|
||||||
local nid; nid=$(lxc_exec docker info --format '{{.Swarm.NodeID}}' 2>/dev/null || true)
|
|
||||||
if [ -n "${nid}" ]; then
|
|
||||||
log "Draining Swarm node ${nid} on ${h}..."
|
|
||||||
lxc_exec docker node update --availability drain "${nid}" >/dev/null 2>&1 || true
|
|
||||||
sleep 10
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
lxc_exec systemctl stop docker || true
|
|
||||||
local w=0
|
|
||||||
while docker_is_running && [ "${w}" -lt "${DOCKER_STOP_TIMEOUT}" ]; do
|
|
||||||
sleep 2; w=$((w+2))
|
|
||||||
done
|
|
||||||
if docker_is_running; then
|
|
||||||
log "WARNING: Docker still running after ${DOCKER_STOP_TIMEOUT}s on ${h}"
|
|
||||||
notify "Backup Warning - ${h}" "vmid ${VMID}: Docker did not stop before snapshot." "ffaa00"
|
|
||||||
else
|
|
||||||
log "Docker stopped cleanly on ${h} in ${w}s"
|
|
||||||
notify "Backup: Docker Stopped - ${h}" "vmid ${VMID}: Stopped OK. Snapshot starting." "0078d4"
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
start_docker() {
|
|
||||||
local h; h=$(lxc_exec hostname 2>/dev/null || echo "vmid-${VMID}")
|
|
||||||
log "Restarting Docker on ${h} after snapshot..."
|
|
||||||
lxc_exec systemctl start docker || true
|
|
||||||
local w=0
|
|
||||||
while ! docker_is_running && [ "${w}" -lt "${DOCKER_STOP_TIMEOUT}" ]; do
|
|
||||||
sleep 2; w=$((w+2))
|
|
||||||
done
|
|
||||||
if ! docker_is_running; then
|
|
||||||
log "ERROR: Docker failed to start on ${h} after ${w}s"
|
|
||||||
notify "CRITICAL: Docker Failed - ${h}" "vmid ${VMID}: Docker did not restart after snapshot! Manual fix needed." "ff0000"
|
|
||||||
return 1
|
|
||||||
fi
|
|
||||||
log "Docker started on ${h} in ${w}s"
|
|
||||||
if lxc_exec docker info 2>/dev/null | grep -q "Swarm: active"; then
|
|
||||||
local nid; nid=$(lxc_exec docker info --format '{{.Swarm.NodeID}}' 2>/dev/null || true)
|
|
||||||
if [ -n "${nid}" ]; then
|
|
||||||
log "Restoring Swarm node ${nid} to active on ${h}..."
|
|
||||||
lxc_exec docker node update --availability active "${nid}" >/dev/null 2>&1 || true
|
|
||||||
log "Swarm node ${nid} restored to active"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
notify "Backup: Docker Restarted - ${h}" "vmid ${VMID}: Docker restarted. Swarm node active." "00aa00"
|
|
||||||
}
|
|
||||||
|
|
||||||
# ── Main phase dispatch ──────────────────────────────────────────────────────
|
|
||||||
log "Hook called: phase=${PHASE} vmid=${VMID}"
|
|
||||||
|
|
||||||
if ! is_docker_vmid; then
|
|
||||||
log "vmid ${VMID} not in DOCKER_VMIDS list — skipping"
|
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
|
|
||||||
case "${PHASE}" in
|
|
||||||
backup-start) stop_docker ;;
|
|
||||||
post-restart) start_docker ;;
|
|
||||||
backup-abort)
|
|
||||||
log "Backup aborted — restarting Docker to restore services..."
|
|
||||||
notify "Backup Aborted - vmid ${VMID}" "Backup failed. Restarting Docker to restore services." "ffaa00"
|
|
||||||
start_docker || true
|
|
||||||
;;
|
|
||||||
job-init|job-start|job-end|job-abort|log-end|pre-stop|pre-restart)
|
|
||||||
log "Phase '${PHASE}' — no action needed" ;;
|
|
||||||
*)
|
|
||||||
log "Unhandled phase '${PHASE}' — skipping" ;;
|
|
||||||
esac
|
|
||||||
|
|
||||||
exit 0
|
|
||||||
Reference in New Issue
Block a user