compose-files/scripts/monitoring/docker-prune-watchdog.sh

#!/bin/bash

################################################################################
# Docker System Prune Watchdog
# 
# Monitors disk usage and automatically runs 'docker system prune -f' when
# disk exceeds 85%. Sends notifications to Microsoft Teams via webhook.
# Logs to RAM (tmpfs) to avoid disk space consumption.
################################################################################

set -euo pipefail

# ============================================================================
# CONFIGURATION
# ============================================================================

# Use environment variables (sourced from config), with defaults as fallback
DISK_THRESHOLD="${DISK_THRESHOLD:-85}"
CHECK_INTERVAL="${CHECK_INTERVAL:-60}"
TEAMS_WEBHOOK_URL="${TEAMS_WEBHOOK_URL:-}"
NOTIFICATION_MODE="${NOTIFICATION_MODE:-both}"

# Log file (in RAM via tmpfs)
LOG_DIR="/dev/shm"
LOG_FILE="${LOG_DIR}/docker-prune-watchdog.log"

# ============================================================================
# FUNCTIONS
# ============================================================================

log() {
    local level="$1"
    shift
    local message="$@"
    local timestamp=$(date '+%Y-%m-%d %H:%M:%S')
    echo "[${timestamp}] [${level}] ${message}" >> "${LOG_FILE}"
}

get_disk_usage() {
    # Returns disk usage percentage for root filesystem
    df /volume1/docker-root | awk 'NR==2 {print $5}' | sed 's/%//'
}

send_teams_notification() {
    local title="$1"
    local message="$2"
    local severity="${3:-informational}"  # informational, warning, danger
    
    if [ -z "${TEAMS_WEBHOOK_URL}" ]; then
        log "WARN" "Teams webhook URL not configured, skipping notification"
        return 1
    fi
    
    # Determine color based on severity
    local color
    case "${severity}" in
        danger)  color="ff0000" ;;      # Red
        warning) color="ffaa00" ;;      # Orange
        *)       color="0078d4" ;;      # Blue (Microsoft Teams color)
    esac
    
    # Create JSON payload (simple format)
    local json_payload="{\"title\":\"${title}\",\"text\":\"${message}\",\"themeColor\":\"${color}\"}"
    
    # Send to Teams
    if curl -s -X POST \
        -H 'Content-Type: application/json' \
        -d "${json_payload}" \
        "${TEAMS_WEBHOOK_URL}" > /dev/null 2>&1; then
        log "INFO" "Teams notification sent: ${title}"
        return 0
    else
        log "ERROR" "Failed to send Teams notification: ${title}"
        return 1
    fi
}

run_docker_prune() {
    local disk_usage_before=$1
    local hostname=$(hostname)
    
    log "INFO" "Disk usage at ${disk_usage_before}% (threshold: ${DISK_THRESHOLD}%). Running docker system prune..."
    
    local prune_output
    local prune_exit_code=0
    
    # Run docker system prune and capture output
    prune_output=$(docker system prune -f 2>&1) || prune_exit_code=$?
    
    # Get disk usage after prune
    local disk_usage_after=$(get_disk_usage)
    local disk_freed=$((disk_usage_before - disk_usage_after))
    
    if [ ${prune_exit_code} -eq 0 ]; then
        log "INFO" "Docker prune completed successfully"
        log "INFO" "Disk usage: before=${disk_usage_before}%, after=${disk_usage_after}%, freed=${disk_freed}%"
        log "DEBUG" "Prune output: ${prune_output}"
        
        # Send notification based on mode
        if [[ "${NOTIFICATION_MODE}" == "always" || "${NOTIFICATION_MODE}" == "both" ]]; then
            local message="Host: ${hostname}\n\nDisk Usage: ${disk_usage_before}% → ${disk_usage_after}% (${disk_freed}% freed)\n\nDocker system prune completed successfully."
            send_teams_notification \
                "✅ Docker Prune Executed" \
                "${message}" \
                "informational"
        fi
        
        return 0
    else
        log "ERROR" "Docker prune failed with exit code ${prune_exit_code}"
        log "ERROR" "Prune error output: ${prune_output}"
        
        # Send error notification
        if [[ "${NOTIFICATION_MODE}" == "errors_only" || "${NOTIFICATION_MODE}" == "both" ]]; then
            local message="Host: ${hostname}\n\nDisk Usage Before: ${disk_usage_before}%\n\nError: ${prune_output}"
            send_teams_notification \
                "❌ Docker Prune Failed" \
                "${message}" \
                "danger"
        fi
        
        return 1
    fi
}

show_logs() {
    if [ -f "${LOG_FILE}" ]; then
        echo "=== Docker Prune Watchdog Logs ==="
        tail -100 "${LOG_FILE}"
    else
        echo "No logs found at ${LOG_FILE}"
    fi
}

show_usage() {
    cat <<EOF
Usage: $0 [COMMAND]

Commands:
    start       Start the watchdog (default)
    logs        Display recent logs
    help        Show this message

Configuration:
    Set environment variables before running:
    - TEAMS_WEBHOOK_URL: Microsoft Teams webhook URL (required for notifications)
    - NOTIFICATION_MODE: "always", "errors_only", or "both" (default: both)
    - DISK_THRESHOLD: Disk usage percentage threshold (default: 85)
    - CHECK_INTERVAL: Check interval in seconds (default: 60)

Example:
    export TEAMS_WEBHOOK_URL="https://bryanfamily.webhook.office.com/..."
    export NOTIFICATION_MODE="both"
    $0 start

EOF
}

# ============================================================================
# MAIN
# ============================================================================

main() {
    local command="${1:-start}"
    
    case "${command}" in
        start)
            # Initialize log file
            touch "${LOG_FILE}" 2>/dev/null || {
                echo "ERROR: Cannot write to ${LOG_FILE}. Ensure /dev/shm is writable."
                exit 1
            }
            
            log "INFO" "Docker Prune Watchdog started"
            log "INFO" "Configuration: Threshold=${DISK_THRESHOLD}%, Interval=${CHECK_INTERVAL}s, Mode=${NOTIFICATION_MODE}"
            
            # Main watchdog loop
            while true; do
                disk_usage=$(get_disk_usage)
                
                if [ "${disk_usage}" -gt "${DISK_THRESHOLD}" ]; then
                    run_docker_prune "${disk_usage}"
                fi
                
                sleep "${CHECK_INTERVAL}"
            done
            ;;
        logs)
            show_logs
            ;;
        help|--help|-h)
            show_usage
            ;;
        *)
            echo "Unknown command: ${command}"
            show_usage
            exit 1
            ;;
    esac
}

main "$@"