Archived
Add Docker Prune Watchdog script
This commit is contained in:
@@ -0,0 +1,202 @@
|
||||
#!/bin/bash
|
||||
|
||||
################################################################################
|
||||
# Docker System Prune Watchdog
|
||||
#
|
||||
# Monitors disk usage and automatically runs 'docker system prune -f' when
|
||||
# disk exceeds 85%. Sends notifications to Microsoft Teams via webhook.
|
||||
# Logs to RAM (tmpfs) to avoid disk space consumption.
|
||||
################################################################################
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# ============================================================================
|
||||
# CONFIGURATION
|
||||
# ============================================================================
|
||||
|
||||
# Use environment variables (sourced from config), with defaults as fallback
|
||||
DISK_THRESHOLD="${DISK_THRESHOLD:-85}"
|
||||
CHECK_INTERVAL="${CHECK_INTERVAL:-60}"
|
||||
TEAMS_WEBHOOK_URL="${TEAMS_WEBHOOK_URL:-}"
|
||||
NOTIFICATION_MODE="${NOTIFICATION_MODE:-both}"
|
||||
|
||||
# Log file (in RAM via tmpfs)
|
||||
LOG_DIR="/dev/shm"
|
||||
LOG_FILE="${LOG_DIR}/docker-prune-watchdog.log"
|
||||
|
||||
# ============================================================================
|
||||
# FUNCTIONS
|
||||
# ============================================================================
|
||||
|
||||
log() {
|
||||
local level="$1"
|
||||
shift
|
||||
local message="$@"
|
||||
local timestamp=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
echo "[${timestamp}] [${level}] ${message}" >> "${LOG_FILE}"
|
||||
}
|
||||
|
||||
get_disk_usage() {
|
||||
# Returns disk usage percentage for root filesystem
|
||||
df /volume1/docker-root | awk 'NR==2 {print $5}' | sed 's/%//'
|
||||
}
|
||||
|
||||
send_teams_notification() {
|
||||
local title="$1"
|
||||
local message="$2"
|
||||
local severity="${3:-informational}" # informational, warning, danger
|
||||
|
||||
if [ -z "${TEAMS_WEBHOOK_URL}" ]; then
|
||||
log "WARN" "Teams webhook URL not configured, skipping notification"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Determine color based on severity
|
||||
local color
|
||||
case "${severity}" in
|
||||
danger) color="ff0000" ;; # Red
|
||||
warning) color="ffaa00" ;; # Orange
|
||||
*) color="0078d4" ;; # Blue (Microsoft Teams color)
|
||||
esac
|
||||
|
||||
# Create JSON payload (simple format)
|
||||
local json_payload="{\"title\":\"${title}\",\"text\":\"${message}\",\"themeColor\":\"${color}\"}"
|
||||
|
||||
# Send to Teams
|
||||
if curl -s -X POST \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d "${json_payload}" \
|
||||
"${TEAMS_WEBHOOK_URL}" > /dev/null 2>&1; then
|
||||
log "INFO" "Teams notification sent: ${title}"
|
||||
return 0
|
||||
else
|
||||
log "ERROR" "Failed to send Teams notification: ${title}"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
run_docker_prune() {
|
||||
local disk_usage_before=$1
|
||||
local hostname=$(hostname)
|
||||
|
||||
log "INFO" "Disk usage at ${disk_usage_before}% (threshold: ${DISK_THRESHOLD}%). Running docker system prune..."
|
||||
|
||||
local prune_output
|
||||
local prune_exit_code=0
|
||||
|
||||
# Run docker system prune and capture output
|
||||
prune_output=$(docker system prune -f 2>&1) || prune_exit_code=$?
|
||||
|
||||
# Get disk usage after prune
|
||||
local disk_usage_after=$(get_disk_usage)
|
||||
local disk_freed=$((disk_usage_before - disk_usage_after))
|
||||
|
||||
if [ ${prune_exit_code} -eq 0 ]; then
|
||||
log "INFO" "Docker prune completed successfully"
|
||||
log "INFO" "Disk usage: before=${disk_usage_before}%, after=${disk_usage_after}%, freed=${disk_freed}%"
|
||||
log "DEBUG" "Prune output: ${prune_output}"
|
||||
|
||||
# Send notification based on mode
|
||||
if [[ "${NOTIFICATION_MODE}" == "always" || "${NOTIFICATION_MODE}" == "both" ]]; then
|
||||
local message="Host: ${hostname}\n\nDisk Usage: ${disk_usage_before}% → ${disk_usage_after}% (${disk_freed}% freed)\n\nDocker system prune completed successfully."
|
||||
send_teams_notification \
|
||||
"✅ Docker Prune Executed" \
|
||||
"${message}" \
|
||||
"informational"
|
||||
fi
|
||||
|
||||
return 0
|
||||
else
|
||||
log "ERROR" "Docker prune failed with exit code ${prune_exit_code}"
|
||||
log "ERROR" "Prune error output: ${prune_output}"
|
||||
|
||||
# Send error notification
|
||||
if [[ "${NOTIFICATION_MODE}" == "errors_only" || "${NOTIFICATION_MODE}" == "both" ]]; then
|
||||
local message="Host: ${hostname}\n\nDisk Usage Before: ${disk_usage_before}%\n\nError: ${prune_output}"
|
||||
send_teams_notification \
|
||||
"❌ Docker Prune Failed" \
|
||||
"${message}" \
|
||||
"danger"
|
||||
fi
|
||||
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
show_logs() {
|
||||
if [ -f "${LOG_FILE}" ]; then
|
||||
echo "=== Docker Prune Watchdog Logs ==="
|
||||
tail -100 "${LOG_FILE}"
|
||||
else
|
||||
echo "No logs found at ${LOG_FILE}"
|
||||
fi
|
||||
}
|
||||
|
||||
show_usage() {
|
||||
cat <<EOF
|
||||
Usage: $0 [COMMAND]
|
||||
|
||||
Commands:
|
||||
start Start the watchdog (default)
|
||||
logs Display recent logs
|
||||
help Show this message
|
||||
|
||||
Configuration:
|
||||
Set environment variables before running:
|
||||
- TEAMS_WEBHOOK_URL: Microsoft Teams webhook URL (required for notifications)
|
||||
- NOTIFICATION_MODE: "always", "errors_only", or "both" (default: both)
|
||||
- DISK_THRESHOLD: Disk usage percentage threshold (default: 85)
|
||||
- CHECK_INTERVAL: Check interval in seconds (default: 60)
|
||||
|
||||
Example:
|
||||
export TEAMS_WEBHOOK_URL="https://bryanfamily.webhook.office.com/..."
|
||||
export NOTIFICATION_MODE="both"
|
||||
$0 start
|
||||
|
||||
EOF
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# MAIN
|
||||
# ============================================================================
|
||||
|
||||
main() {
|
||||
local command="${1:-start}"
|
||||
|
||||
case "${command}" in
|
||||
start)
|
||||
# Initialize log file
|
||||
touch "${LOG_FILE}" 2>/dev/null || {
|
||||
echo "ERROR: Cannot write to ${LOG_FILE}. Ensure /dev/shm is writable."
|
||||
exit 1
|
||||
}
|
||||
|
||||
log "INFO" "Docker Prune Watchdog started"
|
||||
log "INFO" "Configuration: Threshold=${DISK_THRESHOLD}%, Interval=${CHECK_INTERVAL}s, Mode=${NOTIFICATION_MODE}"
|
||||
|
||||
# Main watchdog loop
|
||||
while true; do
|
||||
disk_usage=$(get_disk_usage)
|
||||
|
||||
if [ "${disk_usage}" -gt "${DISK_THRESHOLD}" ]; then
|
||||
run_docker_prune "${disk_usage}"
|
||||
fi
|
||||
|
||||
sleep "${CHECK_INTERVAL}"
|
||||
done
|
||||
;;
|
||||
logs)
|
||||
show_logs
|
||||
;;
|
||||
help|--help|-h)
|
||||
show_usage
|
||||
;;
|
||||
*)
|
||||
echo "Unknown command: ${command}"
|
||||
show_usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Reference in New Issue
Block a user