Archived
Add Docker Prune Watchdog script
This commit is contained in:
@@ -0,0 +1,202 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
################################################################################
|
||||||
|
# Docker System Prune Watchdog
|
||||||
|
#
|
||||||
|
# Monitors disk usage and automatically runs 'docker system prune -f' when
|
||||||
|
# disk exceeds 85%. Sends notifications to Microsoft Teams via webhook.
|
||||||
|
# Logs to RAM (tmpfs) to avoid disk space consumption.
|
||||||
|
################################################################################
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
# ============================================================================
|
||||||
|
# CONFIGURATION
|
||||||
|
# ============================================================================
|
||||||
|
|
||||||
|
# Use environment variables (sourced from config), with defaults as fallback
|
||||||
|
DISK_THRESHOLD="${DISK_THRESHOLD:-85}"
|
||||||
|
CHECK_INTERVAL="${CHECK_INTERVAL:-60}"
|
||||||
|
TEAMS_WEBHOOK_URL="${TEAMS_WEBHOOK_URL:-}"
|
||||||
|
NOTIFICATION_MODE="${NOTIFICATION_MODE:-both}"
|
||||||
|
|
||||||
|
# Log file (in RAM via tmpfs)
|
||||||
|
LOG_DIR="/dev/shm"
|
||||||
|
LOG_FILE="${LOG_DIR}/docker-prune-watchdog.log"
|
||||||
|
|
||||||
|
# ============================================================================
|
||||||
|
# FUNCTIONS
|
||||||
|
# ============================================================================
|
||||||
|
|
||||||
|
log() {
|
||||||
|
local level="$1"
|
||||||
|
shift
|
||||||
|
local message="$@"
|
||||||
|
local timestamp=$(date '+%Y-%m-%d %H:%M:%S')
|
||||||
|
echo "[${timestamp}] [${level}] ${message}" >> "${LOG_FILE}"
|
||||||
|
}
|
||||||
|
|
||||||
|
get_disk_usage() {
|
||||||
|
# Returns disk usage percentage for root filesystem
|
||||||
|
df /volume1/docker-root | awk 'NR==2 {print $5}' | sed 's/%//'
|
||||||
|
}
|
||||||
|
|
||||||
|
send_teams_notification() {
|
||||||
|
local title="$1"
|
||||||
|
local message="$2"
|
||||||
|
local severity="${3:-informational}" # informational, warning, danger
|
||||||
|
|
||||||
|
if [ -z "${TEAMS_WEBHOOK_URL}" ]; then
|
||||||
|
log "WARN" "Teams webhook URL not configured, skipping notification"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Determine color based on severity
|
||||||
|
local color
|
||||||
|
case "${severity}" in
|
||||||
|
danger) color="ff0000" ;; # Red
|
||||||
|
warning) color="ffaa00" ;; # Orange
|
||||||
|
*) color="0078d4" ;; # Blue (Microsoft Teams color)
|
||||||
|
esac
|
||||||
|
|
||||||
|
# Create JSON payload (simple format)
|
||||||
|
local json_payload="{\"title\":\"${title}\",\"text\":\"${message}\",\"themeColor\":\"${color}\"}"
|
||||||
|
|
||||||
|
# Send to Teams
|
||||||
|
if curl -s -X POST \
|
||||||
|
-H 'Content-Type: application/json' \
|
||||||
|
-d "${json_payload}" \
|
||||||
|
"${TEAMS_WEBHOOK_URL}" > /dev/null 2>&1; then
|
||||||
|
log "INFO" "Teams notification sent: ${title}"
|
||||||
|
return 0
|
||||||
|
else
|
||||||
|
log "ERROR" "Failed to send Teams notification: ${title}"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
run_docker_prune() {
|
||||||
|
local disk_usage_before=$1
|
||||||
|
local hostname=$(hostname)
|
||||||
|
|
||||||
|
log "INFO" "Disk usage at ${disk_usage_before}% (threshold: ${DISK_THRESHOLD}%). Running docker system prune..."
|
||||||
|
|
||||||
|
local prune_output
|
||||||
|
local prune_exit_code=0
|
||||||
|
|
||||||
|
# Run docker system prune and capture output
|
||||||
|
prune_output=$(docker system prune -f 2>&1) || prune_exit_code=$?
|
||||||
|
|
||||||
|
# Get disk usage after prune
|
||||||
|
local disk_usage_after=$(get_disk_usage)
|
||||||
|
local disk_freed=$((disk_usage_before - disk_usage_after))
|
||||||
|
|
||||||
|
if [ ${prune_exit_code} -eq 0 ]; then
|
||||||
|
log "INFO" "Docker prune completed successfully"
|
||||||
|
log "INFO" "Disk usage: before=${disk_usage_before}%, after=${disk_usage_after}%, freed=${disk_freed}%"
|
||||||
|
log "DEBUG" "Prune output: ${prune_output}"
|
||||||
|
|
||||||
|
# Send notification based on mode
|
||||||
|
if [[ "${NOTIFICATION_MODE}" == "always" || "${NOTIFICATION_MODE}" == "both" ]]; then
|
||||||
|
local message="Host: ${hostname}\n\nDisk Usage: ${disk_usage_before}% → ${disk_usage_after}% (${disk_freed}% freed)\n\nDocker system prune completed successfully."
|
||||||
|
send_teams_notification \
|
||||||
|
"✅ Docker Prune Executed" \
|
||||||
|
"${message}" \
|
||||||
|
"informational"
|
||||||
|
fi
|
||||||
|
|
||||||
|
return 0
|
||||||
|
else
|
||||||
|
log "ERROR" "Docker prune failed with exit code ${prune_exit_code}"
|
||||||
|
log "ERROR" "Prune error output: ${prune_output}"
|
||||||
|
|
||||||
|
# Send error notification
|
||||||
|
if [[ "${NOTIFICATION_MODE}" == "errors_only" || "${NOTIFICATION_MODE}" == "both" ]]; then
|
||||||
|
local message="Host: ${hostname}\n\nDisk Usage Before: ${disk_usage_before}%\n\nError: ${prune_output}"
|
||||||
|
send_teams_notification \
|
||||||
|
"❌ Docker Prune Failed" \
|
||||||
|
"${message}" \
|
||||||
|
"danger"
|
||||||
|
fi
|
||||||
|
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
show_logs() {
|
||||||
|
if [ -f "${LOG_FILE}" ]; then
|
||||||
|
echo "=== Docker Prune Watchdog Logs ==="
|
||||||
|
tail -100 "${LOG_FILE}"
|
||||||
|
else
|
||||||
|
echo "No logs found at ${LOG_FILE}"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
show_usage() {
|
||||||
|
cat <<EOF
|
||||||
|
Usage: $0 [COMMAND]
|
||||||
|
|
||||||
|
Commands:
|
||||||
|
start Start the watchdog (default)
|
||||||
|
logs Display recent logs
|
||||||
|
help Show this message
|
||||||
|
|
||||||
|
Configuration:
|
||||||
|
Set environment variables before running:
|
||||||
|
- TEAMS_WEBHOOK_URL: Microsoft Teams webhook URL (required for notifications)
|
||||||
|
- NOTIFICATION_MODE: "always", "errors_only", or "both" (default: both)
|
||||||
|
- DISK_THRESHOLD: Disk usage percentage threshold (default: 85)
|
||||||
|
- CHECK_INTERVAL: Check interval in seconds (default: 60)
|
||||||
|
|
||||||
|
Example:
|
||||||
|
export TEAMS_WEBHOOK_URL="https://bryanfamily.webhook.office.com/..."
|
||||||
|
export NOTIFICATION_MODE="both"
|
||||||
|
$0 start
|
||||||
|
|
||||||
|
EOF
|
||||||
|
}
|
||||||
|
|
||||||
|
# ============================================================================
|
||||||
|
# MAIN
|
||||||
|
# ============================================================================
|
||||||
|
|
||||||
|
main() {
|
||||||
|
local command="${1:-start}"
|
||||||
|
|
||||||
|
case "${command}" in
|
||||||
|
start)
|
||||||
|
# Initialize log file
|
||||||
|
touch "${LOG_FILE}" 2>/dev/null || {
|
||||||
|
echo "ERROR: Cannot write to ${LOG_FILE}. Ensure /dev/shm is writable."
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
log "INFO" "Docker Prune Watchdog started"
|
||||||
|
log "INFO" "Configuration: Threshold=${DISK_THRESHOLD}%, Interval=${CHECK_INTERVAL}s, Mode=${NOTIFICATION_MODE}"
|
||||||
|
|
||||||
|
# Main watchdog loop
|
||||||
|
while true; do
|
||||||
|
disk_usage=$(get_disk_usage)
|
||||||
|
|
||||||
|
if [ "${disk_usage}" -gt "${DISK_THRESHOLD}" ]; then
|
||||||
|
run_docker_prune "${disk_usage}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
sleep "${CHECK_INTERVAL}"
|
||||||
|
done
|
||||||
|
;;
|
||||||
|
logs)
|
||||||
|
show_logs
|
||||||
|
;;
|
||||||
|
help|--help|-h)
|
||||||
|
show_usage
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo "Unknown command: ${command}"
|
||||||
|
show_usage
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
main "$@"
|
||||||
Reference in New Issue
Block a user