diff --git a/scripts/monitoring/docker-prune-watchdog.sh b/scripts/monitoring/docker-prune-watchdog.sh new file mode 100644 index 0000000..9849704 --- /dev/null +++ b/scripts/monitoring/docker-prune-watchdog.sh @@ -0,0 +1,202 @@ +#!/bin/bash + +################################################################################ +# Docker System Prune Watchdog +# +# Monitors disk usage and automatically runs 'docker system prune -f' when +# disk exceeds 85%. Sends notifications to Microsoft Teams via webhook. +# Logs to RAM (tmpfs) to avoid disk space consumption. +################################################################################ + +set -euo pipefail + +# ============================================================================ +# CONFIGURATION +# ============================================================================ + +# Use environment variables (sourced from config), with defaults as fallback +DISK_THRESHOLD="${DISK_THRESHOLD:-85}" +CHECK_INTERVAL="${CHECK_INTERVAL:-60}" +TEAMS_WEBHOOK_URL="${TEAMS_WEBHOOK_URL:-}" +NOTIFICATION_MODE="${NOTIFICATION_MODE:-both}" + +# Log file (in RAM via tmpfs) +LOG_DIR="/dev/shm" +LOG_FILE="${LOG_DIR}/docker-prune-watchdog.log" + +# ============================================================================ +# FUNCTIONS +# ============================================================================ + +log() { + local level="$1" + shift + local message="$@" + local timestamp=$(date '+%Y-%m-%d %H:%M:%S') + echo "[${timestamp}] [${level}] ${message}" >> "${LOG_FILE}" +} + +get_disk_usage() { + # Returns disk usage percentage for root filesystem + df /volume1/docker-root | awk 'NR==2 {print $5}' | sed 's/%//' +} + +send_teams_notification() { + local title="$1" + local message="$2" + local severity="${3:-informational}" # informational, warning, danger + + if [ -z "${TEAMS_WEBHOOK_URL}" ]; then + log "WARN" "Teams webhook URL not configured, skipping notification" + return 1 + fi + + # Determine color based on severity + local color + case "${severity}" in + danger) color="ff0000" ;; # Red + warning) color="ffaa00" ;; # Orange + *) color="0078d4" ;; # Blue (Microsoft Teams color) + esac + + # Create JSON payload (simple format) + local json_payload="{\"title\":\"${title}\",\"text\":\"${message}\",\"themeColor\":\"${color}\"}" + + # Send to Teams + if curl -s -X POST \ + -H 'Content-Type: application/json' \ + -d "${json_payload}" \ + "${TEAMS_WEBHOOK_URL}" > /dev/null 2>&1; then + log "INFO" "Teams notification sent: ${title}" + return 0 + else + log "ERROR" "Failed to send Teams notification: ${title}" + return 1 + fi +} + +run_docker_prune() { + local disk_usage_before=$1 + local hostname=$(hostname) + + log "INFO" "Disk usage at ${disk_usage_before}% (threshold: ${DISK_THRESHOLD}%). Running docker system prune..." + + local prune_output + local prune_exit_code=0 + + # Run docker system prune and capture output + prune_output=$(docker system prune -f 2>&1) || prune_exit_code=$? + + # Get disk usage after prune + local disk_usage_after=$(get_disk_usage) + local disk_freed=$((disk_usage_before - disk_usage_after)) + + if [ ${prune_exit_code} -eq 0 ]; then + log "INFO" "Docker prune completed successfully" + log "INFO" "Disk usage: before=${disk_usage_before}%, after=${disk_usage_after}%, freed=${disk_freed}%" + log "DEBUG" "Prune output: ${prune_output}" + + # Send notification based on mode + if [[ "${NOTIFICATION_MODE}" == "always" || "${NOTIFICATION_MODE}" == "both" ]]; then + local message="Host: ${hostname}\n\nDisk Usage: ${disk_usage_before}% → ${disk_usage_after}% (${disk_freed}% freed)\n\nDocker system prune completed successfully." + send_teams_notification \ + "✅ Docker Prune Executed" \ + "${message}" \ + "informational" + fi + + return 0 + else + log "ERROR" "Docker prune failed with exit code ${prune_exit_code}" + log "ERROR" "Prune error output: ${prune_output}" + + # Send error notification + if [[ "${NOTIFICATION_MODE}" == "errors_only" || "${NOTIFICATION_MODE}" == "both" ]]; then + local message="Host: ${hostname}\n\nDisk Usage Before: ${disk_usage_before}%\n\nError: ${prune_output}" + send_teams_notification \ + "❌ Docker Prune Failed" \ + "${message}" \ + "danger" + fi + + return 1 + fi +} + +show_logs() { + if [ -f "${LOG_FILE}" ]; then + echo "=== Docker Prune Watchdog Logs ===" + tail -100 "${LOG_FILE}" + else + echo "No logs found at ${LOG_FILE}" + fi +} + +show_usage() { + cat </dev/null || { + echo "ERROR: Cannot write to ${LOG_FILE}. Ensure /dev/shm is writable." + exit 1 + } + + log "INFO" "Docker Prune Watchdog started" + log "INFO" "Configuration: Threshold=${DISK_THRESHOLD}%, Interval=${CHECK_INTERVAL}s, Mode=${NOTIFICATION_MODE}" + + # Main watchdog loop + while true; do + disk_usage=$(get_disk_usage) + + if [ "${disk_usage}" -gt "${DISK_THRESHOLD}" ]; then + run_docker_prune "${disk_usage}" + fi + + sleep "${CHECK_INTERVAL}" + done + ;; + logs) + show_logs + ;; + help|--help|-h) + show_usage + ;; + *) + echo "Unknown command: ${command}" + show_usage + exit 1 + ;; + esac +} + +main "$@"