diff --git a/scripts/monitoring/docker-prune-watchdog.sh b/scripts/monitoring/docker-prune-watchdog.sh deleted file mode 100644 index 9849704..0000000 --- a/scripts/monitoring/docker-prune-watchdog.sh +++ /dev/null @@ -1,202 +0,0 @@ -#!/bin/bash - -################################################################################ -# Docker System Prune Watchdog -# -# Monitors disk usage and automatically runs 'docker system prune -f' when -# disk exceeds 85%. Sends notifications to Microsoft Teams via webhook. -# Logs to RAM (tmpfs) to avoid disk space consumption. -################################################################################ - -set -euo pipefail - -# ============================================================================ -# CONFIGURATION -# ============================================================================ - -# Use environment variables (sourced from config), with defaults as fallback -DISK_THRESHOLD="${DISK_THRESHOLD:-85}" -CHECK_INTERVAL="${CHECK_INTERVAL:-60}" -TEAMS_WEBHOOK_URL="${TEAMS_WEBHOOK_URL:-}" -NOTIFICATION_MODE="${NOTIFICATION_MODE:-both}" - -# Log file (in RAM via tmpfs) -LOG_DIR="/dev/shm" -LOG_FILE="${LOG_DIR}/docker-prune-watchdog.log" - -# ============================================================================ -# FUNCTIONS -# ============================================================================ - -log() { - local level="$1" - shift - local message="$@" - local timestamp=$(date '+%Y-%m-%d %H:%M:%S') - echo "[${timestamp}] [${level}] ${message}" >> "${LOG_FILE}" -} - -get_disk_usage() { - # Returns disk usage percentage for root filesystem - df /volume1/docker-root | awk 'NR==2 {print $5}' | sed 's/%//' -} - -send_teams_notification() { - local title="$1" - local message="$2" - local severity="${3:-informational}" # informational, warning, danger - - if [ -z "${TEAMS_WEBHOOK_URL}" ]; then - log "WARN" "Teams webhook URL not configured, skipping notification" - return 1 - fi - - # Determine color based on severity - local color - case "${severity}" in - danger) color="ff0000" ;; # Red - warning) color="ffaa00" ;; # Orange - *) color="0078d4" ;; # Blue (Microsoft Teams color) - esac - - # Create JSON payload (simple format) - local json_payload="{\"title\":\"${title}\",\"text\":\"${message}\",\"themeColor\":\"${color}\"}" - - # Send to Teams - if curl -s -X POST \ - -H 'Content-Type: application/json' \ - -d "${json_payload}" \ - "${TEAMS_WEBHOOK_URL}" > /dev/null 2>&1; then - log "INFO" "Teams notification sent: ${title}" - return 0 - else - log "ERROR" "Failed to send Teams notification: ${title}" - return 1 - fi -} - -run_docker_prune() { - local disk_usage_before=$1 - local hostname=$(hostname) - - log "INFO" "Disk usage at ${disk_usage_before}% (threshold: ${DISK_THRESHOLD}%). Running docker system prune..." - - local prune_output - local prune_exit_code=0 - - # Run docker system prune and capture output - prune_output=$(docker system prune -f 2>&1) || prune_exit_code=$? - - # Get disk usage after prune - local disk_usage_after=$(get_disk_usage) - local disk_freed=$((disk_usage_before - disk_usage_after)) - - if [ ${prune_exit_code} -eq 0 ]; then - log "INFO" "Docker prune completed successfully" - log "INFO" "Disk usage: before=${disk_usage_before}%, after=${disk_usage_after}%, freed=${disk_freed}%" - log "DEBUG" "Prune output: ${prune_output}" - - # Send notification based on mode - if [[ "${NOTIFICATION_MODE}" == "always" || "${NOTIFICATION_MODE}" == "both" ]]; then - local message="Host: ${hostname}\n\nDisk Usage: ${disk_usage_before}% → ${disk_usage_after}% (${disk_freed}% freed)\n\nDocker system prune completed successfully." - send_teams_notification \ - "✅ Docker Prune Executed" \ - "${message}" \ - "informational" - fi - - return 0 - else - log "ERROR" "Docker prune failed with exit code ${prune_exit_code}" - log "ERROR" "Prune error output: ${prune_output}" - - # Send error notification - if [[ "${NOTIFICATION_MODE}" == "errors_only" || "${NOTIFICATION_MODE}" == "both" ]]; then - local message="Host: ${hostname}\n\nDisk Usage Before: ${disk_usage_before}%\n\nError: ${prune_output}" - send_teams_notification \ - "❌ Docker Prune Failed" \ - "${message}" \ - "danger" - fi - - return 1 - fi -} - -show_logs() { - if [ -f "${LOG_FILE}" ]; then - echo "=== Docker Prune Watchdog Logs ===" - tail -100 "${LOG_FILE}" - else - echo "No logs found at ${LOG_FILE}" - fi -} - -show_usage() { - cat </dev/null || { - echo "ERROR: Cannot write to ${LOG_FILE}. Ensure /dev/shm is writable." - exit 1 - } - - log "INFO" "Docker Prune Watchdog started" - log "INFO" "Configuration: Threshold=${DISK_THRESHOLD}%, Interval=${CHECK_INTERVAL}s, Mode=${NOTIFICATION_MODE}" - - # Main watchdog loop - while true; do - disk_usage=$(get_disk_usage) - - if [ "${disk_usage}" -gt "${DISK_THRESHOLD}" ]; then - run_docker_prune "${disk_usage}" - fi - - sleep "${CHECK_INTERVAL}" - done - ;; - logs) - show_logs - ;; - help|--help|-h) - show_usage - ;; - *) - echo "Unknown command: ${command}" - show_usage - exit 1 - ;; - esac -} - -main "$@"