#!/bin/bash # ============================================================= # PLEX32 — Daily Server Health Report # KingDezigns Infrastructure # Schedule: Daily at 7:05 AM (run after update check) # Location: /usr/scripts/plex32/plex32_health_report.sh # ============================================================= SMTP_SERVER="smtppro.zoho.com" SMTP_PORT="465" SMTP_USER="rufus.king@kingdezigns.com" SMTP_PASS="Valerie8545" EMAIL_TO="rufus.king@kingdezigns.com" EMAIL_FROM="rufus.king@kingdezigns.com" TIMESTAMP=$(date "+%A, %B %d, %Y at %I:%M %p") DATE_SHORT=$(date "+%Y-%m-%d") LOG_DIR="/var/log/plex32-health" LOG_FILE="$LOG_DIR/health-$DATE_SHORT.log" mkdir -p "$LOG_DIR" exec > >(tee -a "$LOG_FILE") 2>&1 echo "[$TIMESTAMP] Starting health check..." # ============================================================= # DISK USAGE — eMMC + NFS MOUNTS # ============================================================= DISK_WARN=85 DISK_CRIT=95 DISK_ISSUES="" DISK_STATUS="OK" check_disk() { local label="$1" local mount="$2" local pct=$(df -h "$mount" 2>/dev/null | awk 'NR==2{gsub(/%/,"",$5); print $5}') local size=$(df -h "$mount" 2>/dev/null | awk 'NR==2{print $2}') local used=$(df -h "$mount" 2>/dev/null | awk 'NR==2{print $3}') local avail=$(df -h "$mount" 2>/dev/null | awk 'NR==2{print $4}') if [ -z "$pct" ]; then echo "UNAVAILABLE|$label|--|--|--|--" return fi local status="OK" if [ "$pct" -ge "$DISK_CRIT" ]; then status="CRITICAL" elif [ "$pct" -ge "$DISK_WARN" ]; then status="WARNING" fi echo "$status|$label|$size|$used|$avail|$pct%" } ROOT_DISK=$(check_disk "eMMC — OS (/)." "/") BOOT_DISK=$(check_disk "eMMC — /boot" "/boot") NFS_CONFIG=$(check_disk "NAS08 — Plex Config" "/mnt/plex/config") NFS_MEDIA=$(check_disk "NAS08 — Media" "/mnt/plex/media") for disk in "$ROOT_DISK" "$BOOT_DISK" "$NFS_CONFIG" "$NFS_MEDIA"; do status=$(echo "$disk" | cut -d'|' -f1) label=$(echo "$disk" | cut -d'|' -f2) if [ "$status" = "CRITICAL" ]; then DISK_STATUS="CRITICAL" DISK_ISSUES+="CRITICAL: $label is nearly full."$'\n' elif [ "$status" = "WARNING" ] && [ "$DISK_STATUS" != "CRITICAL" ]; then DISK_STATUS="WARNING" DISK_ISSUES+="WARNING: $label is getting full."$'\n' elif [ "$status" = "UNAVAILABLE" ]; then [ "$DISK_STATUS" = "OK" ] && DISK_STATUS="WARNING" DISK_ISSUES+="WARNING: $label is unavailable."$'\n' fi done echo "[INFO] Disk status: $DISK_STATUS" # ============================================================= # NFS MOUNT HEALTH # ============================================================= NFS_STATUS="OK" NFS_ISSUES="" check_nfs() { local label="$1" local mount="$2" if mountpoint -q "$mount" 2>/dev/null && timeout 5 ls "$mount" > /dev/null 2>&1; then echo "OK|$label" else echo "DOWN|$label" fi } NFS_CONFIG_STATUS=$(check_nfs "Plex Config" "/mnt/plex/config") NFS_TRANSCODE_STATUS=$(check_nfs "Plex Transcode" "/mnt/plex/transcode") NFS_MEDIA_STATUS=$(check_nfs "Media Library" "/mnt/plex/media") for nfs in "$NFS_CONFIG_STATUS" "$NFS_TRANSCODE_STATUS" "$NFS_MEDIA_STATUS"; do status=$(echo "$nfs" | cut -d'|' -f1) label=$(echo "$nfs" | cut -d'|' -f2) if [ "$status" = "DOWN" ]; then NFS_STATUS="CRITICAL" NFS_ISSUES+="CRITICAL: NFS mount $label is down."$'\n' fi done echo "[INFO] NFS status: $NFS_STATUS" # ============================================================= # MEMORY # ============================================================= MEM_TOTAL=$(free -m | awk 'NR==2{print $2}') MEM_USED=$(free -m | awk 'NR==2{print $3}') MEM_AVAIL=$(free -m | awk 'NR==2{print $7}') MEM_PCT=$(awk "BEGIN{printf \"%d\", ($MEM_USED/$MEM_TOTAL)*100}") MEM_TOTAL_H=$(free -h | awk 'NR==2{print $2}') MEM_USED_H=$(free -h | awk 'NR==2{print $3}') MEM_AVAIL_H=$(free -h | awk 'NR==2{print $7}') MEM_STATUS="OK" MEM_ISSUES="" if [ "$MEM_PCT" -ge 95 ]; then MEM_STATUS="CRITICAL" MEM_ISSUES="CRITICAL: Memory usage at ${MEM_PCT}% — system may be under pressure." elif [ "$MEM_PCT" -ge 85 ]; then MEM_STATUS="WARNING" MEM_ISSUES="WARNING: Memory usage at ${MEM_PCT}%." fi echo "[INFO] Memory: ${MEM_PCT}% used" # ============================================================= # CPU LOAD # ============================================================= LOAD_1=$(cat /proc/loadavg | awk '{print $1}') LOAD_5=$(cat /proc/loadavg | awk '{print $2}') LOAD_15=$(cat /proc/loadavg | awk '{print $3}') CPU_CORES=$(nproc) LOAD_PCT=$(awk "BEGIN{printf \"%d\", ($LOAD_5/$CPU_CORES)*100}") CPU_STATUS="OK" CPU_ISSUES="" if [ "$LOAD_PCT" -ge 90 ]; then CPU_STATUS="CRITICAL" CPU_ISSUES="CRITICAL: CPU load average is very high (${LOAD_5} on ${CPU_CORES} cores)." elif [ "$LOAD_PCT" -ge 70 ]; then CPU_STATUS="WARNING" CPU_ISSUES="WARNING: CPU load average is elevated (${LOAD_5} on ${CPU_CORES} cores)." fi echo "[INFO] CPU load: $LOAD_5 (5min avg), $CPU_CORES cores" # ============================================================= # UPTIME & REBOOT CHECK # ============================================================= UPTIME_SECONDS=$(awk '{print int($1)}' /proc/uptime) UPTIME_DAYS=$((UPTIME_SECONDS / 86400)) UPTIME_HOURS=$(( (UPTIME_SECONDS % 86400) / 3600 )) UPTIME_MINS=$(( (UPTIME_SECONDS % 3600) / 60 )) UPTIME_DISPLAY="${UPTIME_DAYS}d ${UPTIME_HOURS}h ${UPTIME_MINS}m" REBOOT_STATUS="OK" REBOOT_ISSUES="" if [ -f /var/run/reboot-required ]; then REBOOT_STATUS="WARNING" REBOOT_ISSUES="Reboot required to apply kernel or system updates." fi echo "[INFO] Uptime: $UPTIME_DISPLAY | Reboot required: $REBOOT_STATUS" # ============================================================= # PLEX CONTAINER STATUS # ============================================================= PLEX_CONTAINER_STATUS="OK" PLEX_CONTAINER_ISSUES="" PLEX_RUNNING=$(docker ps --filter "name=plex" --filter "status=running" --format "{{.Names}}" 2>/dev/null) PLEX_UPTIME=$(docker ps --filter "name=plex" --format "{{.Status}}" 2>/dev/null) if [ -z "$PLEX_RUNNING" ]; then PLEX_CONTAINER_STATUS="CRITICAL" PLEX_CONTAINER_ISSUES="CRITICAL: Plex container is not running." fi echo "[INFO] Plex container: $PLEX_CONTAINER_STATUS — $PLEX_UPTIME" # ============================================================= # FAILED SYSTEMD UNITS # ============================================================= SYSTEMD_STATUS="OK" SYSTEMD_ISSUES="" FAILED_UNITS=$(systemctl --failed --no-pager --plain 2>/dev/null | grep "●" | awk '{print $2}') if [ -n "$FAILED_UNITS" ]; then SYSTEMD_STATUS="WARNING" SYSTEMD_ISSUES="Failed units: $FAILED_UNITS" fi echo "[INFO] Systemd failed units: $([ -z "$FAILED_UNITS" ] && echo 'none' || echo "$FAILED_UNITS")" # ============================================================= # RECENT JOURNAL ERRORS (last 24h, filter known-harmless) # ============================================================= JOURNAL_ERRORS=$(journalctl -p err -b --no-pager 2>/dev/null \ | grep -v "SGX disabled by BIOS" \ | grep -v "^-- " \ | tail -10) JOURNAL_COUNT=$(echo "$JOURNAL_ERRORS" | grep -c "." || true) [ "$JOURNAL_COUNT" -le 1 ] && JOURNAL_COUNT=0 JOURNAL_STATUS="OK" [ "$JOURNAL_COUNT" -gt 0 ] && JOURNAL_STATUS="WARNING" echo "[INFO] Journal errors: $JOURNAL_COUNT" # ============================================================= # FAIL2BAN STATUS # ============================================================= F2B_RUNNING=$(systemctl is-active fail2ban 2>/dev/null) F2B_BANS=$(fail2ban-client status sshd 2>/dev/null | grep "Total banned" | awk '{print $NF}' || echo "0") F2B_CURRENTLY_BANNED=$(fail2ban-client status sshd 2>/dev/null | grep "Currently banned" | awk '{print $NF}' || echo "0") F2B_HEALTH_STATUS="OK" F2B_HEALTH_ISSUES="" if [ "$F2B_RUNNING" != "active" ]; then F2B_HEALTH_STATUS="CRITICAL" F2B_HEALTH_ISSUES="CRITICAL: Fail2Ban is not running." fi echo "[INFO] Fail2Ban: $F2B_RUNNING | Total bans: $F2B_BANS | Current: $F2B_CURRENTLY_BANNED" # ============================================================= # OVERALL STATUS # ============================================================= OVERALL="OK" for s in "$DISK_STATUS" "$NFS_STATUS" "$MEM_STATUS" "$CPU_STATUS" "$REBOOT_STATUS" "$PLEX_CONTAINER_STATUS" "$SYSTEMD_STATUS" "$F2B_HEALTH_STATUS"; do if [ "$s" = "CRITICAL" ]; then OVERALL="CRITICAL"; break; fi if [ "$s" = "WARNING" ] && [ "$OVERALL" != "CRITICAL" ]; then OVERALL="WARNING"; fi done if [ "$OVERALL" = "CRITICAL" ]; then STATUS_COLOR="#b91c1c" STATUS_BADGE="🚨 ACTION REQUIRED" STATUS_BANNER_MSG="🚨  One or more critical issues detected — immediate attention required" SUBJECT="🚨 PLEX32 — Critical Health Alert — $DATE_SHORT" elif [ "$OVERALL" = "WARNING" ]; then STATUS_COLOR="#b45309" STATUS_BADGE="⚠ ATTENTION NEEDED" STATUS_BANNER_MSG="⚠  One or more items need your attention" SUBJECT="⚠ PLEX32 — Health Warning — $DATE_SHORT" else STATUS_COLOR="#1a7f4b" STATUS_BADGE="✅ HEALTHY" STATUS_BANNER_MSG="✅  All systems are healthy — no action required" SUBJECT="✅ PLEX32 — Daily Health Report — $DATE_SHORT" fi echo "[INFO] Overall status: $OVERALL" # ============================================================= # BUILD DISK TABLE HTML # ============================================================= build_disk_row() { local row="$1" local status=$(echo "$row" | cut -d'|' -f1) local label=$(echo "$row" | cut -d'|' -f2) local size=$(echo "$row" | cut -d'|' -f3) local used=$(echo "$row" | cut -d'|' -f4) local avail=$(echo "$row" | cut -d'|' -f5) local pct=$(echo "$row" | cut -d'|' -f6) local icon="✅" local row_bg="white" local pct_color="#1a7f4b" [ "$status" = "WARNING" ] && icon="⚠" && row_bg="#fffbeb" && pct_color="#b45309" [ "$status" = "CRITICAL" ] && icon="🚨" && row_bg="#fff1f2" && pct_color="#b91c1c" [ "$status" = "UNAVAILABLE" ] && icon="❌" && row_bg="#fff1f2" && pct_color="#b91c1c" echo " $icon $label $size $used $avail $pct " } DISK_ROWS="" DISK_ROWS+=$(build_disk_row "$ROOT_DISK") DISK_ROWS+=$(build_disk_row "$BOOT_DISK") DISK_ROWS+=$(build_disk_row "$NFS_CONFIG") DISK_ROWS+=$(build_disk_row "$NFS_MEDIA") # ============================================================= # BUILD NFS TABLE HTML # ============================================================= build_nfs_row() { local row="$1" local status=$(echo "$row" | cut -d'|' -f1) local label=$(echo "$row" | cut -d'|' -f2) local icon="✅" local row_bg="white" local status_text="Mounted & Accessible" local status_color="#1a7f4b" if [ "$status" = "DOWN" ]; then icon="🚨" row_bg="#fff1f2" status_text="DOWN — Not accessible" status_color="#b91c1c" fi echo " $icon $label $status_text " } NFS_ROWS="" NFS_ROWS+=$(build_nfs_row "$NFS_CONFIG_STATUS") NFS_ROWS+=$(build_nfs_row "$NFS_TRANSCODE_STATUS") NFS_ROWS+=$(build_nfs_row "$NFS_MEDIA_STATUS") # ============================================================= # ISSUES SUMMARY HTML # ============================================================= ALL_ISSUES="" [ -n "$DISK_ISSUES" ] && ALL_ISSUES+="$DISK_ISSUES" [ -n "$NFS_ISSUES" ] && ALL_ISSUES+="$NFS_ISSUES" [ -n "$MEM_ISSUES" ] && ALL_ISSUES+="$MEM_ISSUES"$'\n' [ -n "$CPU_ISSUES" ] && ALL_ISSUES+="$CPU_ISSUES"$'\n' [ -n "$REBOOT_ISSUES" ] && ALL_ISSUES+="$REBOOT_ISSUES"$'\n' [ -n "$PLEX_CONTAINER_ISSUES" ] && ALL_ISSUES+="$PLEX_CONTAINER_ISSUES"$'\n' [ -n "$SYSTEMD_ISSUES" ] && ALL_ISSUES+="$SYSTEMD_ISSUES"$'\n' [ -n "$F2B_HEALTH_ISSUES" ] && ALL_ISSUES+="$F2B_HEALTH_ISSUES"$'\n' ISSUES_HTML="" if [ -n "$ALL_ISSUES" ]; then ISSUES_HTML="
⚠ Issues Detected
" while IFS= read -r issue; do [ -z "$issue" ] && continue BG="#fffbeb"; BORDER="#b45309"; COL="#92400e" echo "$issue" | grep -q "CRITICAL" && BG="#fff1f2" && BORDER="#b91c1c" && COL="#7f1d1d" ISSUES_HTML+="
$issue
" done <<< "$ALL_ISSUES" ISSUES_HTML+="
" fi # ============================================================= # REBOOT REQUIRED HTML # ============================================================= REBOOT_HTML="" if [ "$REBOOT_STATUS" = "WARNING" ]; then REBOOT_HTML="
⚠ Reboot Required
A system reboot is required to complete pending updates.
sudo reboot
👆 Click command to select — then Ctrl+C
" fi # ============================================================= # JOURNAL ERRORS HTML # ============================================================= JOURNAL_HTML="" if [ "$JOURNAL_COUNT" -gt 0 ]; then JOURNAL_ESCAPED=$(echo "$JOURNAL_ERRORS" | sed 's/&/\&/g; s//\>/g') JOURNAL_HTML="
📋 Recent System Errors
$JOURNAL_ESCAPED
" fi # ============================================================= # STATUS ICON HELPER # ============================================================= status_icon() { case "$1" in OK) echo "✅" ;; WARNING) echo "⚠" ;; CRITICAL) echo "🚨" ;; *) echo "❓" ;; esac } STATUS_ICON_DISK=$(status_icon "$DISK_STATUS") STATUS_ICON_NFS=$(status_icon "$NFS_STATUS") STATUS_ICON_MEM=$(status_icon "$MEM_STATUS") STATUS_ICON_CPU=$(status_icon "$CPU_STATUS") STATUS_ICON_REBOOT=$(status_icon "$REBOOT_STATUS") STATUS_ICON_PLEX=$(status_icon "$PLEX_CONTAINER_STATUS") STATUS_ICON_SYSTEMD=$(status_icon "$SYSTEMD_STATUS") STATUS_ICON_F2B=$(status_icon "$F2B_HEALTH_STATUS") # ============================================================= # BUILD & SEND EMAIL # ============================================================= python3 << PYEOF import smtplib, ssl from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText timestamp = "$TIMESTAMP" status_color = "$STATUS_COLOR" status_badge = "$STATUS_BADGE" status_banner = "$STATUS_BANNER_MSG" subject = "$SUBJECT" from_email = "$EMAIL_FROM" uptime_display = "$UPTIME_DISPLAY" load_1 = "$LOAD_1" load_5 = "$LOAD_5" load_15 = "$LOAD_15" cpu_cores = "$CPU_CORES" mem_total = "$MEM_TOTAL_H" mem_used = "$MEM_USED_H" mem_avail = "$MEM_AVAIL_H" mem_pct = "$MEM_PCT" plex_uptime = "$PLEX_UPTIME" f2b_running = "$F2B_RUNNING" f2b_bans = "$F2B_BANS" f2b_current = "$F2B_CURRENTLY_BANNED" disk_rows = """$DISK_ROWS""" nfs_rows = """$NFS_ROWS""" issues_html = """$ISSUES_HTML""" reboot_html = """$REBOOT_HTML""" journal_html = """$JOURNAL_HTML""" icon_disk = "$STATUS_ICON_DISK" icon_nfs = "$STATUS_ICON_NFS" icon_mem = "$STATUS_ICON_MEM" icon_cpu = "$STATUS_ICON_CPU" icon_reboot = "$STATUS_ICON_REBOOT" icon_plex = "$STATUS_ICON_PLEX" icon_systemd = "$STATUS_ICON_SYSTEMD" icon_f2b = "$STATUS_ICON_F2B" mem_color = "#b91c1c" if int(mem_pct) >= 95 else "#b45309" if int(mem_pct) >= 85 else "#1a7f4b" load_color = "#b91c1c" if float(load_5) >= float(cpu_cores)*0.9 else "#b45309" if float(load_5) >= float(cpu_cores)*0.7 else "#1a7f4b" html = f"""
Health Report
🖥 PLEX32
{timestamp} | VLAN 50 — 192.168.150.45
{status_badge}
{status_banner}
Uptime
{uptime_display}
CPU Load (5m)
{load_5}
Memory Used
{mem_pct}%
F2B Bans Today
{f2b_bans}
{issues_html} {reboot_html}
🔍 System Overview
System Status Detail
{icon_plex} Plex Container Running {plex_uptime}
{icon_f2b} Fail2Ban {f2b_running} Currently banned: {f2b_current}
{icon_mem} Memory {mem_used} / {mem_total} used {mem_avail} available
{icon_cpu} CPU Load {load_1} / {load_5} / {load_15} {cpu_cores} cores (1m / 5m / 15m avg)
{icon_systemd} Systemd Units No failed units All services healthy
{icon_reboot} Reboot Required {'⚠ Yes' if '$REBOOT_STATUS' == 'WARNING' else 'No'} {'Pending kernel/system updates' if '$REBOOT_STATUS' == 'WARNING' else 'System is current'}
💾 Disk Usage
{disk_rows}
Filesystem Size Used Avail Use%
📁 NFS Mounts (NAS08)
{nfs_rows}
Mount Status
{journal_html}
Health Report  |  PLEX32  |  KingDezigns Infrastructure
""" msg = MIMEMultipart("alternative") msg["Subject"] = subject msg["From"] = f"KingDezigns PLEX32 <{from_email}>" msg["To"] = "$EMAIL_TO" msg.attach(MIMEText(html, "html")) context = ssl.create_default_context() try: with smtplib.SMTP_SSL("$SMTP_SERVER", $SMTP_PORT, context=context, timeout=15) as server: server.login("$SMTP_USER", "$SMTP_PASS") server.sendmail("$EMAIL_FROM", "$EMAIL_TO", msg.as_string()) print("[$TIMESTAMP] Health report email sent successfully.") except Exception as e: print(f"[$TIMESTAMP] Email send failed: {e}") exit(1) PYEOF echo "[$TIMESTAMP] Health check complete."