598 lines
27 KiB
Bash
598 lines
27 KiB
Bash
#!/bin/bash
|
|
# =============================================================
|
|
# PLEX32 — Daily Server Health Report
|
|
# KingDezigns Infrastructure
|
|
# Schedule: Daily at 7:05 AM (run after update check)
|
|
# Location: /usr/scripts/plex32/plex32_health_report.sh
|
|
# =============================================================
|
|
|
|
SMTP_SERVER="smtppro.zoho.com"
|
|
SMTP_PORT="465"
|
|
SMTP_USER="rufus.king@kingdezigns.com"
|
|
SMTP_PASS="Valerie8545"
|
|
EMAIL_TO="rufus.king@kingdezigns.com"
|
|
EMAIL_FROM="rufus.king@kingdezigns.com"
|
|
TIMESTAMP=$(date "+%A, %B %d, %Y at %I:%M %p")
|
|
DATE_SHORT=$(date "+%Y-%m-%d")
|
|
LOG_DIR="/var/log/plex32-health"
|
|
LOG_FILE="$LOG_DIR/health-$DATE_SHORT.log"
|
|
|
|
mkdir -p "$LOG_DIR"
|
|
exec > >(tee -a "$LOG_FILE") 2>&1
|
|
echo "[$TIMESTAMP] Starting health check..."
|
|
|
|
# =============================================================
|
|
# DISK USAGE — eMMC + NFS MOUNTS
|
|
# =============================================================
|
|
DISK_WARN=85
|
|
DISK_CRIT=95
|
|
DISK_ISSUES=""
|
|
DISK_STATUS="OK"
|
|
|
|
check_disk() {
|
|
local label="$1"
|
|
local mount="$2"
|
|
local pct=$(df -h "$mount" 2>/dev/null | awk 'NR==2{gsub(/%/,"",$5); print $5}')
|
|
local size=$(df -h "$mount" 2>/dev/null | awk 'NR==2{print $2}')
|
|
local used=$(df -h "$mount" 2>/dev/null | awk 'NR==2{print $3}')
|
|
local avail=$(df -h "$mount" 2>/dev/null | awk 'NR==2{print $4}')
|
|
|
|
if [ -z "$pct" ]; then
|
|
echo "UNAVAILABLE|$label|--|--|--|--"
|
|
return
|
|
fi
|
|
|
|
local status="OK"
|
|
if [ "$pct" -ge "$DISK_CRIT" ]; then
|
|
status="CRITICAL"
|
|
elif [ "$pct" -ge "$DISK_WARN" ]; then
|
|
status="WARNING"
|
|
fi
|
|
echo "$status|$label|$size|$used|$avail|$pct%"
|
|
}
|
|
|
|
ROOT_DISK=$(check_disk "eMMC — OS (/)." "/")
|
|
BOOT_DISK=$(check_disk "eMMC — /boot" "/boot")
|
|
NFS_CONFIG=$(check_disk "NAS08 — Plex Config" "/mnt/plex/config")
|
|
NFS_MEDIA=$(check_disk "NAS08 — Media" "/mnt/plex/media")
|
|
|
|
for disk in "$ROOT_DISK" "$BOOT_DISK" "$NFS_CONFIG" "$NFS_MEDIA"; do
|
|
status=$(echo "$disk" | cut -d'|' -f1)
|
|
label=$(echo "$disk" | cut -d'|' -f2)
|
|
if [ "$status" = "CRITICAL" ]; then
|
|
DISK_STATUS="CRITICAL"
|
|
DISK_ISSUES+="CRITICAL: $label is nearly full."$'\n'
|
|
elif [ "$status" = "WARNING" ] && [ "$DISK_STATUS" != "CRITICAL" ]; then
|
|
DISK_STATUS="WARNING"
|
|
DISK_ISSUES+="WARNING: $label is getting full."$'\n'
|
|
elif [ "$status" = "UNAVAILABLE" ]; then
|
|
[ "$DISK_STATUS" = "OK" ] && DISK_STATUS="WARNING"
|
|
DISK_ISSUES+="WARNING: $label is unavailable."$'\n'
|
|
fi
|
|
done
|
|
|
|
echo "[INFO] Disk status: $DISK_STATUS"
|
|
|
|
# =============================================================
|
|
# NFS MOUNT HEALTH
|
|
# =============================================================
|
|
NFS_STATUS="OK"
|
|
NFS_ISSUES=""
|
|
|
|
check_nfs() {
|
|
local label="$1"
|
|
local mount="$2"
|
|
if mountpoint -q "$mount" 2>/dev/null && timeout 5 ls "$mount" > /dev/null 2>&1; then
|
|
echo "OK|$label"
|
|
else
|
|
echo "DOWN|$label"
|
|
fi
|
|
}
|
|
|
|
NFS_CONFIG_STATUS=$(check_nfs "Plex Config" "/mnt/plex/config")
|
|
NFS_TRANSCODE_STATUS=$(check_nfs "Plex Transcode" "/mnt/plex/transcode")
|
|
NFS_MEDIA_STATUS=$(check_nfs "Media Library" "/mnt/plex/media")
|
|
|
|
for nfs in "$NFS_CONFIG_STATUS" "$NFS_TRANSCODE_STATUS" "$NFS_MEDIA_STATUS"; do
|
|
status=$(echo "$nfs" | cut -d'|' -f1)
|
|
label=$(echo "$nfs" | cut -d'|' -f2)
|
|
if [ "$status" = "DOWN" ]; then
|
|
NFS_STATUS="CRITICAL"
|
|
NFS_ISSUES+="CRITICAL: NFS mount $label is down."$'\n'
|
|
fi
|
|
done
|
|
|
|
echo "[INFO] NFS status: $NFS_STATUS"
|
|
|
|
# =============================================================
|
|
# MEMORY
|
|
# =============================================================
|
|
MEM_TOTAL=$(free -m | awk 'NR==2{print $2}')
|
|
MEM_USED=$(free -m | awk 'NR==2{print $3}')
|
|
MEM_AVAIL=$(free -m | awk 'NR==2{print $7}')
|
|
MEM_PCT=$(awk "BEGIN{printf \"%d\", ($MEM_USED/$MEM_TOTAL)*100}")
|
|
MEM_TOTAL_H=$(free -h | awk 'NR==2{print $2}')
|
|
MEM_USED_H=$(free -h | awk 'NR==2{print $3}')
|
|
MEM_AVAIL_H=$(free -h | awk 'NR==2{print $7}')
|
|
|
|
MEM_STATUS="OK"
|
|
MEM_ISSUES=""
|
|
if [ "$MEM_PCT" -ge 95 ]; then
|
|
MEM_STATUS="CRITICAL"
|
|
MEM_ISSUES="CRITICAL: Memory usage at ${MEM_PCT}% — system may be under pressure."
|
|
elif [ "$MEM_PCT" -ge 85 ]; then
|
|
MEM_STATUS="WARNING"
|
|
MEM_ISSUES="WARNING: Memory usage at ${MEM_PCT}%."
|
|
fi
|
|
|
|
echo "[INFO] Memory: ${MEM_PCT}% used"
|
|
|
|
# =============================================================
|
|
# CPU LOAD
|
|
# =============================================================
|
|
LOAD_1=$(cat /proc/loadavg | awk '{print $1}')
|
|
LOAD_5=$(cat /proc/loadavg | awk '{print $2}')
|
|
LOAD_15=$(cat /proc/loadavg | awk '{print $3}')
|
|
CPU_CORES=$(nproc)
|
|
LOAD_PCT=$(awk "BEGIN{printf \"%d\", ($LOAD_5/$CPU_CORES)*100}")
|
|
|
|
CPU_STATUS="OK"
|
|
CPU_ISSUES=""
|
|
if [ "$LOAD_PCT" -ge 90 ]; then
|
|
CPU_STATUS="CRITICAL"
|
|
CPU_ISSUES="CRITICAL: CPU load average is very high (${LOAD_5} on ${CPU_CORES} cores)."
|
|
elif [ "$LOAD_PCT" -ge 70 ]; then
|
|
CPU_STATUS="WARNING"
|
|
CPU_ISSUES="WARNING: CPU load average is elevated (${LOAD_5} on ${CPU_CORES} cores)."
|
|
fi
|
|
|
|
echo "[INFO] CPU load: $LOAD_5 (5min avg), $CPU_CORES cores"
|
|
|
|
# =============================================================
|
|
# UPTIME & REBOOT CHECK
|
|
# =============================================================
|
|
UPTIME_SECONDS=$(awk '{print int($1)}' /proc/uptime)
|
|
UPTIME_DAYS=$((UPTIME_SECONDS / 86400))
|
|
UPTIME_HOURS=$(( (UPTIME_SECONDS % 86400) / 3600 ))
|
|
UPTIME_MINS=$(( (UPTIME_SECONDS % 3600) / 60 ))
|
|
UPTIME_DISPLAY="${UPTIME_DAYS}d ${UPTIME_HOURS}h ${UPTIME_MINS}m"
|
|
|
|
REBOOT_STATUS="OK"
|
|
REBOOT_ISSUES=""
|
|
if [ -f /var/run/reboot-required ]; then
|
|
REBOOT_STATUS="WARNING"
|
|
REBOOT_ISSUES="Reboot required to apply kernel or system updates."
|
|
fi
|
|
|
|
echo "[INFO] Uptime: $UPTIME_DISPLAY | Reboot required: $REBOOT_STATUS"
|
|
|
|
# =============================================================
|
|
# PLEX CONTAINER STATUS
|
|
# =============================================================
|
|
PLEX_CONTAINER_STATUS="OK"
|
|
PLEX_CONTAINER_ISSUES=""
|
|
PLEX_RUNNING=$(docker ps --filter "name=plex" --filter "status=running" --format "{{.Names}}" 2>/dev/null)
|
|
PLEX_UPTIME=$(docker ps --filter "name=plex" --format "{{.Status}}" 2>/dev/null)
|
|
|
|
if [ -z "$PLEX_RUNNING" ]; then
|
|
PLEX_CONTAINER_STATUS="CRITICAL"
|
|
PLEX_CONTAINER_ISSUES="CRITICAL: Plex container is not running."
|
|
fi
|
|
|
|
echo "[INFO] Plex container: $PLEX_CONTAINER_STATUS — $PLEX_UPTIME"
|
|
|
|
# =============================================================
|
|
# FAILED SYSTEMD UNITS
|
|
# =============================================================
|
|
SYSTEMD_STATUS="OK"
|
|
SYSTEMD_ISSUES=""
|
|
FAILED_UNITS=$(systemctl --failed --no-pager --plain 2>/dev/null | grep "●" | awk '{print $2}')
|
|
|
|
if [ -n "$FAILED_UNITS" ]; then
|
|
SYSTEMD_STATUS="WARNING"
|
|
SYSTEMD_ISSUES="Failed units: $FAILED_UNITS"
|
|
fi
|
|
|
|
echo "[INFO] Systemd failed units: $([ -z "$FAILED_UNITS" ] && echo 'none' || echo "$FAILED_UNITS")"
|
|
|
|
# =============================================================
|
|
# RECENT JOURNAL ERRORS (last 24h, filter known-harmless)
|
|
# =============================================================
|
|
JOURNAL_ERRORS=$(journalctl -p err -b --no-pager 2>/dev/null \
|
|
| grep -v "SGX disabled by BIOS" \
|
|
| grep -v "^-- " \
|
|
| tail -10)
|
|
|
|
JOURNAL_COUNT=$(echo "$JOURNAL_ERRORS" | grep -c "." || true)
|
|
[ "$JOURNAL_COUNT" -le 1 ] && JOURNAL_COUNT=0
|
|
|
|
JOURNAL_STATUS="OK"
|
|
[ "$JOURNAL_COUNT" -gt 0 ] && JOURNAL_STATUS="WARNING"
|
|
|
|
echo "[INFO] Journal errors: $JOURNAL_COUNT"
|
|
|
|
# =============================================================
|
|
# FAIL2BAN STATUS
|
|
# =============================================================
|
|
F2B_RUNNING=$(systemctl is-active fail2ban 2>/dev/null)
|
|
F2B_BANS=$(fail2ban-client status sshd 2>/dev/null | grep "Total banned" | awk '{print $NF}' || echo "0")
|
|
F2B_CURRENTLY_BANNED=$(fail2ban-client status sshd 2>/dev/null | grep "Currently banned" | awk '{print $NF}' || echo "0")
|
|
|
|
F2B_HEALTH_STATUS="OK"
|
|
F2B_HEALTH_ISSUES=""
|
|
if [ "$F2B_RUNNING" != "active" ]; then
|
|
F2B_HEALTH_STATUS="CRITICAL"
|
|
F2B_HEALTH_ISSUES="CRITICAL: Fail2Ban is not running."
|
|
fi
|
|
|
|
echo "[INFO] Fail2Ban: $F2B_RUNNING | Total bans: $F2B_BANS | Current: $F2B_CURRENTLY_BANNED"
|
|
|
|
# =============================================================
|
|
# OVERALL STATUS
|
|
# =============================================================
|
|
OVERALL="OK"
|
|
for s in "$DISK_STATUS" "$NFS_STATUS" "$MEM_STATUS" "$CPU_STATUS" "$REBOOT_STATUS" "$PLEX_CONTAINER_STATUS" "$SYSTEMD_STATUS" "$F2B_HEALTH_STATUS"; do
|
|
if [ "$s" = "CRITICAL" ]; then OVERALL="CRITICAL"; break; fi
|
|
if [ "$s" = "WARNING" ] && [ "$OVERALL" != "CRITICAL" ]; then OVERALL="WARNING"; fi
|
|
done
|
|
|
|
if [ "$OVERALL" = "CRITICAL" ]; then
|
|
STATUS_COLOR="#b91c1c"
|
|
STATUS_BADGE="🚨 ACTION REQUIRED"
|
|
STATUS_BANNER_MSG="🚨 One or more critical issues detected — immediate attention required"
|
|
SUBJECT="🚨 PLEX32 — Critical Health Alert — $DATE_SHORT"
|
|
elif [ "$OVERALL" = "WARNING" ]; then
|
|
STATUS_COLOR="#b45309"
|
|
STATUS_BADGE="⚠ ATTENTION NEEDED"
|
|
STATUS_BANNER_MSG="⚠ One or more items need your attention"
|
|
SUBJECT="⚠ PLEX32 — Health Warning — $DATE_SHORT"
|
|
else
|
|
STATUS_COLOR="#1a7f4b"
|
|
STATUS_BADGE="✅ HEALTHY"
|
|
STATUS_BANNER_MSG="✅ All systems are healthy — no action required"
|
|
SUBJECT="✅ PLEX32 — Daily Health Report — $DATE_SHORT"
|
|
fi
|
|
|
|
echo "[INFO] Overall status: $OVERALL"
|
|
|
|
# =============================================================
|
|
# BUILD DISK TABLE HTML
|
|
# =============================================================
|
|
build_disk_row() {
|
|
local row="$1"
|
|
local status=$(echo "$row" | cut -d'|' -f1)
|
|
local label=$(echo "$row" | cut -d'|' -f2)
|
|
local size=$(echo "$row" | cut -d'|' -f3)
|
|
local used=$(echo "$row" | cut -d'|' -f4)
|
|
local avail=$(echo "$row" | cut -d'|' -f5)
|
|
local pct=$(echo "$row" | cut -d'|' -f6)
|
|
|
|
local icon="✅"
|
|
local row_bg="white"
|
|
local pct_color="#1a7f4b"
|
|
[ "$status" = "WARNING" ] && icon="⚠" && row_bg="#fffbeb" && pct_color="#b45309"
|
|
[ "$status" = "CRITICAL" ] && icon="🚨" && row_bg="#fff1f2" && pct_color="#b91c1c"
|
|
[ "$status" = "UNAVAILABLE" ] && icon="❌" && row_bg="#fff1f2" && pct_color="#b91c1c"
|
|
|
|
echo "<tr style='background:$row_bg;'>
|
|
<td style='padding:9px 14px;font-size:13px;color:#374151;border-bottom:1px solid #f3f4f6;'>$icon $label</td>
|
|
<td style='padding:9px 14px;font-size:13px;color:#374151;border-bottom:1px solid #f3f4f6;text-align:center;'>$size</td>
|
|
<td style='padding:9px 14px;font-size:13px;color:#374151;border-bottom:1px solid #f3f4f6;text-align:center;'>$used</td>
|
|
<td style='padding:9px 14px;font-size:13px;color:#374151;border-bottom:1px solid #f3f4f6;text-align:center;'>$avail</td>
|
|
<td style='padding:9px 14px;font-size:13px;font-weight:700;color:$pct_color;border-bottom:1px solid #f3f4f6;text-align:center;'>$pct</td>
|
|
</tr>"
|
|
}
|
|
|
|
DISK_ROWS=""
|
|
DISK_ROWS+=$(build_disk_row "$ROOT_DISK")
|
|
DISK_ROWS+=$(build_disk_row "$BOOT_DISK")
|
|
DISK_ROWS+=$(build_disk_row "$NFS_CONFIG")
|
|
DISK_ROWS+=$(build_disk_row "$NFS_MEDIA")
|
|
|
|
# =============================================================
|
|
# BUILD NFS TABLE HTML
|
|
# =============================================================
|
|
build_nfs_row() {
|
|
local row="$1"
|
|
local status=$(echo "$row" | cut -d'|' -f1)
|
|
local label=$(echo "$row" | cut -d'|' -f2)
|
|
local icon="✅"
|
|
local row_bg="white"
|
|
local status_text="Mounted & Accessible"
|
|
local status_color="#1a7f4b"
|
|
if [ "$status" = "DOWN" ]; then
|
|
icon="🚨"
|
|
row_bg="#fff1f2"
|
|
status_text="DOWN — Not accessible"
|
|
status_color="#b91c1c"
|
|
fi
|
|
echo "<tr style='background:$row_bg;'>
|
|
<td style='padding:9px 14px;font-size:13px;color:#374151;border-bottom:1px solid #f3f4f6;'>$icon $label</td>
|
|
<td style='padding:9px 14px;font-size:13px;font-weight:600;color:$status_color;border-bottom:1px solid #f3f4f6;'>$status_text</td>
|
|
</tr>"
|
|
}
|
|
|
|
NFS_ROWS=""
|
|
NFS_ROWS+=$(build_nfs_row "$NFS_CONFIG_STATUS")
|
|
NFS_ROWS+=$(build_nfs_row "$NFS_TRANSCODE_STATUS")
|
|
NFS_ROWS+=$(build_nfs_row "$NFS_MEDIA_STATUS")
|
|
|
|
# =============================================================
|
|
# ISSUES SUMMARY HTML
|
|
# =============================================================
|
|
ALL_ISSUES=""
|
|
[ -n "$DISK_ISSUES" ] && ALL_ISSUES+="$DISK_ISSUES"
|
|
[ -n "$NFS_ISSUES" ] && ALL_ISSUES+="$NFS_ISSUES"
|
|
[ -n "$MEM_ISSUES" ] && ALL_ISSUES+="$MEM_ISSUES"$'\n'
|
|
[ -n "$CPU_ISSUES" ] && ALL_ISSUES+="$CPU_ISSUES"$'\n'
|
|
[ -n "$REBOOT_ISSUES" ] && ALL_ISSUES+="$REBOOT_ISSUES"$'\n'
|
|
[ -n "$PLEX_CONTAINER_ISSUES" ] && ALL_ISSUES+="$PLEX_CONTAINER_ISSUES"$'\n'
|
|
[ -n "$SYSTEMD_ISSUES" ] && ALL_ISSUES+="$SYSTEMD_ISSUES"$'\n'
|
|
[ -n "$F2B_HEALTH_ISSUES" ] && ALL_ISSUES+="$F2B_HEALTH_ISSUES"$'\n'
|
|
|
|
ISSUES_HTML=""
|
|
if [ -n "$ALL_ISSUES" ]; then
|
|
ISSUES_HTML="<div style='margin-top:24px;'>
|
|
<div style='font-size:13px;font-weight:700;color:#374151;margin-bottom:12px;text-transform:uppercase;letter-spacing:.5px;'>⚠ Issues Detected</div>"
|
|
while IFS= read -r issue; do
|
|
[ -z "$issue" ] && continue
|
|
BG="#fffbeb"; BORDER="#b45309"; COL="#92400e"
|
|
echo "$issue" | grep -q "CRITICAL" && BG="#fff1f2" && BORDER="#b91c1c" && COL="#7f1d1d"
|
|
ISSUES_HTML+="<div style='margin-bottom:8px;padding:10px 14px;background:$BG;border-left:4px solid $BORDER;border-radius:4px;font-size:13px;color:$COL;'>$issue</div>"
|
|
done <<< "$ALL_ISSUES"
|
|
ISSUES_HTML+="</div>"
|
|
fi
|
|
|
|
# =============================================================
|
|
# REBOOT REQUIRED HTML
|
|
# =============================================================
|
|
REBOOT_HTML=""
|
|
if [ "$REBOOT_STATUS" = "WARNING" ]; then
|
|
REBOOT_HTML="<div style='margin-top:16px;padding:12px 14px;background:#fffbeb;border-left:4px solid #b45309;border-radius:4px;'>
|
|
<div style='font-size:11px;font-weight:700;color:#b45309;text-transform:uppercase;letter-spacing:.5px;margin-bottom:6px;'>⚠ Reboot Required</div>
|
|
<div style='font-size:13px;color:#92400e;'>A system reboot is required to complete pending updates.</div>
|
|
<pre style='margin-top:8px;background:#1e293b;color:#e2e8f0;padding:10px;border-radius:4px;font-size:12px;user-select:all;-webkit-user-select:all;cursor:pointer;'>sudo reboot</pre>
|
|
<div style='font-size:11px;color:#b45309;margin-top:4px;'>👆 Click command to select — then Ctrl+C</div>
|
|
</div>"
|
|
fi
|
|
|
|
# =============================================================
|
|
# JOURNAL ERRORS HTML
|
|
# =============================================================
|
|
JOURNAL_HTML=""
|
|
if [ "$JOURNAL_COUNT" -gt 0 ]; then
|
|
JOURNAL_ESCAPED=$(echo "$JOURNAL_ERRORS" | sed 's/&/\&/g; s/</\</g; s/>/\>/g')
|
|
JOURNAL_HTML="<div style='margin-top:24px;'>
|
|
<div style='font-size:13px;font-weight:700;color:#374151;margin-bottom:8px;text-transform:uppercase;letter-spacing:.5px;'>📋 Recent System Errors</div>
|
|
<pre style='margin:0;background:#1e293b;color:#e2e8f0;padding:14px;border-radius:6px;font-size:11px;line-height:1.6;white-space:pre-wrap;word-break:break-all;'>$JOURNAL_ESCAPED</pre>
|
|
</div>"
|
|
fi
|
|
|
|
# =============================================================
|
|
# STATUS ICON HELPER
|
|
# =============================================================
|
|
status_icon() {
|
|
case "$1" in
|
|
OK) echo "✅" ;;
|
|
WARNING) echo "⚠" ;;
|
|
CRITICAL) echo "🚨" ;;
|
|
*) echo "❓" ;;
|
|
esac
|
|
}
|
|
|
|
STATUS_ICON_DISK=$(status_icon "$DISK_STATUS")
|
|
STATUS_ICON_NFS=$(status_icon "$NFS_STATUS")
|
|
STATUS_ICON_MEM=$(status_icon "$MEM_STATUS")
|
|
STATUS_ICON_CPU=$(status_icon "$CPU_STATUS")
|
|
STATUS_ICON_REBOOT=$(status_icon "$REBOOT_STATUS")
|
|
STATUS_ICON_PLEX=$(status_icon "$PLEX_CONTAINER_STATUS")
|
|
STATUS_ICON_SYSTEMD=$(status_icon "$SYSTEMD_STATUS")
|
|
STATUS_ICON_F2B=$(status_icon "$F2B_HEALTH_STATUS")
|
|
|
|
# =============================================================
|
|
# BUILD & SEND EMAIL
|
|
# =============================================================
|
|
python3 << PYEOF
|
|
import smtplib, ssl
|
|
from email.mime.multipart import MIMEMultipart
|
|
from email.mime.text import MIMEText
|
|
|
|
timestamp = "$TIMESTAMP"
|
|
status_color = "$STATUS_COLOR"
|
|
status_badge = "$STATUS_BADGE"
|
|
status_banner = "$STATUS_BANNER_MSG"
|
|
subject = "$SUBJECT"
|
|
from_email = "$EMAIL_FROM"
|
|
|
|
uptime_display = "$UPTIME_DISPLAY"
|
|
load_1 = "$LOAD_1"
|
|
load_5 = "$LOAD_5"
|
|
load_15 = "$LOAD_15"
|
|
cpu_cores = "$CPU_CORES"
|
|
mem_total = "$MEM_TOTAL_H"
|
|
mem_used = "$MEM_USED_H"
|
|
mem_avail = "$MEM_AVAIL_H"
|
|
mem_pct = "$MEM_PCT"
|
|
plex_uptime = "$PLEX_UPTIME"
|
|
|
|
f2b_running = "$F2B_RUNNING"
|
|
f2b_bans = "$F2B_BANS"
|
|
f2b_current = "$F2B_CURRENTLY_BANNED"
|
|
|
|
disk_rows = """$DISK_ROWS"""
|
|
nfs_rows = """$NFS_ROWS"""
|
|
issues_html = """$ISSUES_HTML"""
|
|
reboot_html = """$REBOOT_HTML"""
|
|
journal_html = """$JOURNAL_HTML"""
|
|
|
|
icon_disk = "$STATUS_ICON_DISK"
|
|
icon_nfs = "$STATUS_ICON_NFS"
|
|
icon_mem = "$STATUS_ICON_MEM"
|
|
icon_cpu = "$STATUS_ICON_CPU"
|
|
icon_reboot = "$STATUS_ICON_REBOOT"
|
|
icon_plex = "$STATUS_ICON_PLEX"
|
|
icon_systemd = "$STATUS_ICON_SYSTEMD"
|
|
icon_f2b = "$STATUS_ICON_F2B"
|
|
|
|
mem_color = "#b91c1c" if int(mem_pct) >= 95 else "#b45309" if int(mem_pct) >= 85 else "#1a7f4b"
|
|
load_color = "#b91c1c" if float(load_5) >= float(cpu_cores)*0.9 else "#b45309" if float(load_5) >= float(cpu_cores)*0.7 else "#1a7f4b"
|
|
|
|
html = f"""<!DOCTYPE html>
|
|
<html><head><meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1.0"></head>
|
|
<body style="margin:0;padding:0;background:#f1f5f9;font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,sans-serif;">
|
|
<table width="100%" cellspacing="0" cellpadding="0" style="background:#f1f5f9;padding:32px 16px;">
|
|
<tr><td align="center">
|
|
<table width="100%" cellspacing="0" cellpadding="0" style="max-width:720px;background:white;border-radius:10px;box-shadow:0 1px 3px rgba(0,0,0,.1);border:1px solid #e5e7eb;overflow:hidden;">
|
|
|
|
<tr><td style="background:#0f172a;border-radius:10px 10px 0 0;padding:28px 32px;">
|
|
<table width="100%" cellspacing="0" cellpadding="0"><tr>
|
|
<td>
|
|
<div style="font-size:11px;color:#94a3b8;letter-spacing:1px;text-transform:uppercase;">Health Report</div>
|
|
<div style="font-size:26px;font-weight:800;color:white;margin-top:4px;">🖥 PLEX32</div>
|
|
<div style="font-size:13px;color:#94a3b8;margin-top:4px;">{timestamp} | VLAN 50 — 192.168.150.45</div>
|
|
</td>
|
|
<td align="right">
|
|
<div style="background:{status_color};color:white;padding:10px 20px;border-radius:8px;font-size:14px;font-weight:700;">{status_badge}</div>
|
|
</td>
|
|
</tr></table>
|
|
</td></tr>
|
|
|
|
<tr><td style="background:{status_color};padding:12px 32px;">
|
|
<span style="color:white;font-size:14px;font-weight:600;">{status_banner}</span>
|
|
</td></tr>
|
|
|
|
<tr><td style="padding:28px 32px;">
|
|
|
|
<!-- STAT CARDS -->
|
|
<table width="100%" cellspacing="0" cellpadding="0">
|
|
<tr>
|
|
<td style="width:24%;padding:8px 6px 8px 0;">
|
|
<div style="background:#f9fafb;border-radius:6px;padding:12px;text-align:center;border:1px solid #e5e7eb;">
|
|
<div style="font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;">Uptime</div>
|
|
<div style="font-size:18px;font-weight:700;color:#111827;margin-top:4px;">{uptime_display}</div>
|
|
</div>
|
|
</td>
|
|
<td style="width:24%;padding:8px 6px;">
|
|
<div style="background:#f9fafb;border-radius:6px;padding:12px;text-align:center;border:1px solid #e5e7eb;">
|
|
<div style="font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;">CPU Load (5m)</div>
|
|
<div style="font-size:28px;font-weight:700;color:{load_color};margin-top:4px;">{load_5}</div>
|
|
</div>
|
|
</td>
|
|
<td style="width:24%;padding:8px 6px;">
|
|
<div style="background:#f9fafb;border-radius:6px;padding:12px;text-align:center;border:1px solid #e5e7eb;">
|
|
<div style="font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;">Memory Used</div>
|
|
<div style="font-size:28px;font-weight:700;color:{mem_color};margin-top:4px;">{mem_pct}%</div>
|
|
</div>
|
|
</td>
|
|
<td style="width:24%;padding:8px 0 8px 6px;">
|
|
<div style="background:#f9fafb;border-radius:6px;padding:12px;text-align:center;border:1px solid #e5e7eb;">
|
|
<div style="font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;">F2B Bans Today</div>
|
|
<div style="font-size:28px;font-weight:700;color:#111827;margin-top:4px;">{f2b_bans}</div>
|
|
</div>
|
|
</td>
|
|
</tr>
|
|
</table>
|
|
|
|
{issues_html}
|
|
{reboot_html}
|
|
|
|
<!-- SYSTEM OVERVIEW TABLE -->
|
|
<div style="margin-top:24px;">
|
|
<div style="font-size:13px;font-weight:700;color:#374151;margin-bottom:12px;text-transform:uppercase;letter-spacing:.5px;">🔍 System Overview</div>
|
|
<table width="100%" cellspacing="0" cellpadding="0" style="background:white;border-radius:8px;border:1px solid #e5e7eb;overflow:hidden;">
|
|
<tr style="background:#f8fafc;">
|
|
<th style="text-align:left;padding:8px 14px;font-size:11px;color:#6b7280;text-transform:uppercase;letter-spacing:.5px;border-bottom:1px solid #e5e7eb;">System</th>
|
|
<th style="text-align:left;padding:8px 14px;font-size:11px;color:#6b7280;text-transform:uppercase;letter-spacing:.5px;border-bottom:1px solid #e5e7eb;">Status</th>
|
|
<th style="text-align:left;padding:8px 14px;font-size:11px;color:#6b7280;text-transform:uppercase;letter-spacing:.5px;border-bottom:1px solid #e5e7eb;">Detail</th>
|
|
</tr>
|
|
<tr>
|
|
<td style="padding:9px 14px;font-size:13px;color:#374151;font-weight:600;border-bottom:1px solid #f3f4f6;">{icon_plex} Plex Container</td>
|
|
<td style="padding:9px 14px;font-size:13px;color:#374151;border-bottom:1px solid #f3f4f6;">Running</td>
|
|
<td style="padding:9px 14px;font-size:13px;color:#6b7280;border-bottom:1px solid #f3f4f6;">{plex_uptime}</td>
|
|
</tr>
|
|
<tr>
|
|
<td style="padding:9px 14px;font-size:13px;color:#374151;font-weight:600;border-bottom:1px solid #f3f4f6;">{icon_f2b} Fail2Ban</td>
|
|
<td style="padding:9px 14px;font-size:13px;color:#374151;border-bottom:1px solid #f3f4f6;">{f2b_running}</td>
|
|
<td style="padding:9px 14px;font-size:13px;color:#6b7280;border-bottom:1px solid #f3f4f6;">Currently banned: {f2b_current}</td>
|
|
</tr>
|
|
<tr>
|
|
<td style="padding:9px 14px;font-size:13px;color:#374151;font-weight:600;border-bottom:1px solid #f3f4f6;">{icon_mem} Memory</td>
|
|
<td style="padding:9px 14px;font-size:13px;color:#374151;border-bottom:1px solid #f3f4f6;">{mem_used} / {mem_total} used</td>
|
|
<td style="padding:9px 14px;font-size:13px;color:#6b7280;border-bottom:1px solid #f3f4f6;">{mem_avail} available</td>
|
|
</tr>
|
|
<tr>
|
|
<td style="padding:9px 14px;font-size:13px;color:#374151;font-weight:600;border-bottom:1px solid #f3f4f6;">{icon_cpu} CPU Load</td>
|
|
<td style="padding:9px 14px;font-size:13px;color:#374151;border-bottom:1px solid #f3f4f6;">{load_1} / {load_5} / {load_15}</td>
|
|
<td style="padding:9px 14px;font-size:13px;color:#6b7280;border-bottom:1px solid #f3f4f6;">{cpu_cores} cores (1m / 5m / 15m avg)</td>
|
|
</tr>
|
|
<tr>
|
|
<td style="padding:9px 14px;font-size:13px;color:#374151;font-weight:600;border-bottom:1px solid #f3f4f6;">{icon_systemd} Systemd Units</td>
|
|
<td style="padding:9px 14px;font-size:13px;color:#374151;border-bottom:1px solid #f3f4f6;">No failed units</td>
|
|
<td style="padding:9px 14px;font-size:13px;color:#6b7280;border-bottom:1px solid #f3f4f6;">All services healthy</td>
|
|
</tr>
|
|
<tr>
|
|
<td style="padding:9px 14px;font-size:13px;color:#374151;font-weight:600;">{icon_reboot} Reboot Required</td>
|
|
<td style="padding:9px 14px;font-size:13px;color:#374151;">{'⚠ Yes' if '$REBOOT_STATUS' == 'WARNING' else 'No'}</td>
|
|
<td style="padding:9px 14px;font-size:13px;color:#6b7280;">{'Pending kernel/system updates' if '$REBOOT_STATUS' == 'WARNING' else 'System is current'}</td>
|
|
</tr>
|
|
</table>
|
|
</div>
|
|
|
|
<!-- DISK USAGE -->
|
|
<div style="margin-top:24px;">
|
|
<div style="font-size:13px;font-weight:700;color:#374151;margin-bottom:12px;text-transform:uppercase;letter-spacing:.5px;">💾 Disk Usage</div>
|
|
<table width="100%" cellspacing="0" cellpadding="0" style="background:white;border-radius:8px;border:1px solid #e5e7eb;overflow:hidden;">
|
|
<tr style="background:#f8fafc;">
|
|
<th style="text-align:left;padding:8px 14px;font-size:11px;color:#6b7280;text-transform:uppercase;letter-spacing:.5px;border-bottom:1px solid #e5e7eb;">Filesystem</th>
|
|
<th style="text-align:center;padding:8px 14px;font-size:11px;color:#6b7280;text-transform:uppercase;letter-spacing:.5px;border-bottom:1px solid #e5e7eb;">Size</th>
|
|
<th style="text-align:center;padding:8px 14px;font-size:11px;color:#6b7280;text-transform:uppercase;letter-spacing:.5px;border-bottom:1px solid #e5e7eb;">Used</th>
|
|
<th style="text-align:center;padding:8px 14px;font-size:11px;color:#6b7280;text-transform:uppercase;letter-spacing:.5px;border-bottom:1px solid #e5e7eb;">Avail</th>
|
|
<th style="text-align:center;padding:8px 14px;font-size:11px;color:#6b7280;text-transform:uppercase;letter-spacing:.5px;border-bottom:1px solid #e5e7eb;">Use%</th>
|
|
</tr>
|
|
{disk_rows}
|
|
</table>
|
|
</div>
|
|
|
|
<!-- NFS MOUNTS -->
|
|
<div style="margin-top:24px;">
|
|
<div style="font-size:13px;font-weight:700;color:#374151;margin-bottom:12px;text-transform:uppercase;letter-spacing:.5px;">📁 NFS Mounts (NAS08)</div>
|
|
<table width="100%" cellspacing="0" cellpadding="0" style="background:white;border-radius:8px;border:1px solid #e5e7eb;overflow:hidden;">
|
|
<tr style="background:#f8fafc;">
|
|
<th style="text-align:left;padding:8px 14px;font-size:11px;color:#6b7280;text-transform:uppercase;letter-spacing:.5px;border-bottom:1px solid #e5e7eb;">Mount</th>
|
|
<th style="text-align:left;padding:8px 14px;font-size:11px;color:#6b7280;text-transform:uppercase;letter-spacing:.5px;border-bottom:1px solid #e5e7eb;">Status</th>
|
|
</tr>
|
|
{nfs_rows}
|
|
</table>
|
|
</div>
|
|
|
|
{journal_html}
|
|
|
|
</td></tr>
|
|
|
|
<tr><td style="background:#0f172a;border-radius:0 0 10px 10px;padding:16px 32px;text-align:center;">
|
|
<span style="font-size:11px;color:#64748b;">Health Report | PLEX32 | KingDezigns Infrastructure</span>
|
|
</td></tr>
|
|
|
|
</table>
|
|
</td></tr>
|
|
</table>
|
|
</body></html>"""
|
|
|
|
msg = MIMEMultipart("alternative")
|
|
msg["Subject"] = subject
|
|
msg["From"] = f"KingDezigns PLEX32 <{from_email}>"
|
|
msg["To"] = "$EMAIL_TO"
|
|
msg.attach(MIMEText(html, "html"))
|
|
|
|
context = ssl.create_default_context()
|
|
try:
|
|
with smtplib.SMTP_SSL("$SMTP_SERVER", $SMTP_PORT, context=context, timeout=15) as server:
|
|
server.login("$SMTP_USER", "$SMTP_PASS")
|
|
server.sendmail("$EMAIL_FROM", "$EMAIL_TO", msg.as_string())
|
|
print("[$TIMESTAMP] Health report email sent successfully.")
|
|
except Exception as e:
|
|
print(f"[$TIMESTAMP] Email send failed: {e}")
|
|
exit(1)
|
|
PYEOF
|
|
|
|
echo "[$TIMESTAMP] Health check complete."
|