295 lines
15 KiB
Bash
295 lines
15 KiB
Bash
#!/bin/bash
|
|
# =============================================================================
|
|
# ZFS Pool Scrub & Email Report — NAS16
|
|
# Location: /usr/scripts/zfs/nas16_zfs_scrub.sh
|
|
# Schedule via OMV Scheduler (cron)
|
|
# =============================================================================
|
|
|
|
# ── Configuration ─────────────────────────────────────────────────────────────
|
|
HOSTNAME="NAS16"
|
|
REPORT_TO="rufus.king@kingdezigns.com"
|
|
REPORT_FROM="zfs-monitor@nas16.local" # Display only — Postfix uses your Gmail relay
|
|
SCRUB_WAIT_SECONDS=28800 # Max wait per pool scrub (8 hrs for large pools)
|
|
CAPACITY_WARN_THRESHOLD=80 # % capacity to flag as WARNING
|
|
SCRUB_AGE_WARN_DAYS=8 # Days since last scrub before flagging stale
|
|
LOG_FILE="/var/log/zfs_scrub_nas16.log"
|
|
# Note: Email is sent via OMV-configured Postfix (Gmail relay). No SMTP config needed here.
|
|
# ──────────────────────────────────────────────────────────────────────────────
|
|
|
|
REPORT_TIME=$(date "+%Y-%m-%d %H:%M:%S")
|
|
POOLS=$(zpool list -H -o name)
|
|
POOL_COUNT=$(echo "$POOLS" | wc -l)
|
|
|
|
log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a "$LOG_FILE"; }
|
|
|
|
log "===== ZFS Scrub started on $HOSTNAME ====="
|
|
|
|
# ── Run scrubs sequentially ───────────────────────────────────────────────────
|
|
for pool in $POOLS; do
|
|
log "Starting scrub on pool: $pool"
|
|
zpool scrub "$pool"
|
|
|
|
# Wait for scrub to finish
|
|
ELAPSED=0
|
|
while true; do
|
|
SCRUB_STATE=$(zpool status "$pool" | grep -E "scan:" | awk '{print $2}')
|
|
[[ "$SCRUB_STATE" == "scrub" ]] || break
|
|
sleep 30
|
|
ELAPSED=$((ELAPSED + 30))
|
|
if [[ $ELAPSED -ge $SCRUB_WAIT_SECONDS ]]; then
|
|
log "WARNING: Scrub on $pool exceeded wait time. Moving on."
|
|
break
|
|
fi
|
|
done
|
|
log "Scrub complete (or timed out) for pool: $pool"
|
|
done
|
|
|
|
# ── Gather data & build HTML report ──────────────────────────────────────────
|
|
|
|
OVERALL_STATUS="HEALTHY" # HEALTHY | WARNING | CRITICAL
|
|
POOLS_WITH_ISSUES=0
|
|
POOL_CARDS_HTML=""
|
|
|
|
for pool in $POOLS; do
|
|
STATUS_RAW=$(zpool list -H -o health "$pool")
|
|
SIZE=$(zpool list -H -o size "$pool")
|
|
USED=$(zpool list -H -o alloc "$pool")
|
|
FREE=$(zpool list -H -o free "$pool")
|
|
CAP=$(zpool list -H -o cap "$pool" | tr -d '%')
|
|
FRAG=$(zpool list -H -o frag "$pool")
|
|
SCRUB_LINE=$(zpool status "$pool" | grep -E "scan:")
|
|
ERRORS_LINE=$(zpool status "$pool" | grep "errors:")
|
|
CONFIG_BLOCK=$(zpool status "$pool" | awk '/config:/,/errors:/' | head -n -1)
|
|
FAULTED_LINES=$(echo "$CONFIG_BLOCK" | grep -E "FAULTED|DEGRADED|UNAVAIL|REMOVED" | grep -v "^$")
|
|
|
|
# Determine per-pool severity
|
|
POOL_SEVERITY="HEALTHY"
|
|
ISSUE_DETAIL=""
|
|
|
|
if [[ "$STATUS_RAW" == "DEGRADED" || "$STATUS_RAW" == "FAULTED" || "$STATUS_RAW" == "UNAVAIL" ]]; then
|
|
POOL_SEVERITY="CRITICAL"
|
|
OVERALL_STATUS="CRITICAL"
|
|
ISSUE_DETAIL="$FAULTED_LINES"
|
|
elif [[ "$CAP" -ge "$CAPACITY_WARN_THRESHOLD" ]]; then
|
|
POOL_SEVERITY="WARNING"
|
|
[[ "$OVERALL_STATUS" != "CRITICAL" ]] && OVERALL_STATUS="WARNING"
|
|
fi
|
|
|
|
# Scrub error count from status
|
|
SCRUB_ERRORS=$(echo "$SCRUB_LINE" | grep -oP '\d+ errors' | head -1)
|
|
[[ -z "$SCRUB_ERRORS" ]] && SCRUB_ERRORS="0 errors"
|
|
|
|
# Scrub date
|
|
SCRUB_DATE=$(echo "$SCRUB_LINE" | grep -oP '[A-Z][a-z]{2} \d+ \d+:\d+:\d+ \d{4}' | head -1)
|
|
[[ -z "$SCRUB_DATE" ]] && SCRUB_DATE="No scrub data"
|
|
|
|
# Scrub age warning
|
|
SCRUB_AGE_NOTE=""
|
|
if [[ -n "$SCRUB_DATE" && "$SCRUB_DATE" != "No scrub data" ]]; then
|
|
SCRUB_EPOCH=$(date -d "$SCRUB_DATE" +%s 2>/dev/null)
|
|
NOW_EPOCH=$(date +%s)
|
|
DIFF_DAYS=$(( (NOW_EPOCH - SCRUB_EPOCH) / 86400 ))
|
|
if [[ $DIFF_DAYS -ge $SCRUB_AGE_WARN_DAYS ]]; then
|
|
SCRUB_AGE_NOTE=" — ⚠️ Last scrub was ${DIFF_DAYS} days ago"
|
|
[[ "$POOL_SEVERITY" == "HEALTHY" ]] && POOL_SEVERITY="WARNING"
|
|
[[ "$OVERALL_STATUS" != "CRITICAL" ]] && OVERALL_STATUS="WARNING"
|
|
fi
|
|
fi
|
|
|
|
[[ "$POOL_SEVERITY" != "HEALTHY" ]] && POOLS_WITH_ISSUES=$((POOLS_WITH_ISSUES + 1))
|
|
|
|
# Health value color
|
|
case "$STATUS_RAW" in
|
|
ONLINE) HEALTH_COLOR="#1a7f4b" ;;
|
|
DEGRADED) HEALTH_COLOR="#b91c1c" ;;
|
|
FAULTED) HEALTH_COLOR="#b91c1c" ;;
|
|
*) HEALTH_COLOR="#b45309" ;;
|
|
esac
|
|
|
|
# Pool card header color
|
|
case "$POOL_SEVERITY" in
|
|
CRITICAL) CARD_BG="#fff1f2"; BADGE_BG="#b91c1c"; BADGE_LABEL="$STATUS_RAW" ;;
|
|
WARNING) CARD_BG="#fffbeb"; BADGE_BG="#b45309"; BADGE_LABEL="WARNING" ;;
|
|
*) CARD_BG="#f0fdf4"; BADGE_BG="#1a7f4b"; BADGE_LABEL="HEALTHY" ;;
|
|
esac
|
|
|
|
# Capacity color
|
|
CAP_COLOR="#111827"
|
|
[[ "$CAP" -ge "$CAPACITY_WARN_THRESHOLD" ]] && CAP_COLOR="#b45309"
|
|
[[ "$CAP" -ge 90 ]] && CAP_COLOR="#b91c1c"
|
|
|
|
# Issue block (only shown when there's a problem)
|
|
ISSUE_BLOCK=""
|
|
if [[ -n "$ISSUE_DETAIL" ]]; then
|
|
ESCAPED_DETAIL=$(echo "$ISSUE_DETAIL" | sed 's/</\</g; s/>/\>/g')
|
|
ISSUE_BLOCK="<div style=\"margin-top:10px;padding:10px 14px;background:#fff1f2;border-left:4px solid #b91c1c;border-radius:4px;font-family:monospace;font-size:13px;color:#7f1d1d;white-space:pre;\">$ESCAPED_DETAIL</div>"
|
|
fi
|
|
|
|
# Config block
|
|
ESCAPED_CONFIG=$(echo "$CONFIG_BLOCK" | sed 's/</\</g; s/>/\>/g')
|
|
|
|
POOL_CARDS_HTML+="
|
|
<div style=\"background:white;border-radius:10px;box-shadow:0 1px 6px rgba(0,0,0,.10);margin-bottom:28px;overflow:hidden;border:1px solid #e5e7eb;\">
|
|
<div style=\"background:$CARD_BG;padding:16px 22px;display:flex;align-items:center;justify-content:space-between;border-bottom:1px solid #e5e7eb;\">
|
|
<div>
|
|
<span style=\"font-size:18px;font-weight:700;color:#111827;\">🗄️ $pool</span>
|
|
<span style=\"margin-left:8px;font-size:12px;color:#6b7280;\">pool</span>
|
|
</div>
|
|
<span style=\"background:$BADGE_BG;color:white;padding:5px 14px;border-radius:20px;font-size:12px;font-weight:700;letter-spacing:.5px;\">$BADGE_LABEL</span>
|
|
</div>
|
|
<div style=\"padding:20px 22px;\">
|
|
<table width=\"100%\" cellspacing=\"0\" cellpadding=\"0\">
|
|
<tr>
|
|
<td style=\"width:16%;padding:8px 12px;background:#f9fafb;border-radius:6px;text-align:center;\">
|
|
<div style=\"font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;\">Total Size</div>
|
|
<div style=\"font-size:20px;font-weight:700;color:#111827;margin-top:2px;\">$SIZE</div>
|
|
</td>
|
|
<td style=\"width:4%;\"></td>
|
|
<td style=\"width:16%;padding:8px 12px;background:#f9fafb;border-radius:6px;text-align:center;\">
|
|
<div style=\"font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;\">Used</div>
|
|
<div style=\"font-size:20px;font-weight:700;color:#111827;margin-top:2px;\">$USED</div>
|
|
</td>
|
|
<td style=\"width:4%;\"></td>
|
|
<td style=\"width:16%;padding:8px 12px;background:#f9fafb;border-radius:6px;text-align:center;\">
|
|
<div style=\"font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;\">Free</div>
|
|
<div style=\"font-size:20px;font-weight:700;color:#111827;margin-top:2px;\">$FREE</div>
|
|
</td>
|
|
<td style=\"width:4%;\"></td>
|
|
<td style=\"width:16%;padding:8px 12px;background:#f9fafb;border-radius:6px;text-align:center;\">
|
|
<div style=\"font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;\">Capacity</div>
|
|
<div style=\"font-size:20px;font-weight:700;color:$CAP_COLOR;margin-top:2px;\">${CAP}%</div>
|
|
</td>
|
|
<td style=\"width:4%;\"></td>
|
|
<td style=\"width:16%;padding:8px 12px;background:#f9fafb;border-radius:6px;text-align:center;\">
|
|
<div style=\"font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;\">Fragmentation</div>
|
|
<div style=\"font-size:20px;font-weight:700;color:#111827;margin-top:2px;\">$FRAG</div>
|
|
</td>
|
|
<td style=\"width:4%;\"></td>
|
|
<td style=\"width:16%;padding:8px 12px;background:#f9fafb;border-radius:6px;text-align:center;\">
|
|
<div style=\"font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;\">Health</div>
|
|
<div style=\"font-size:20px;font-weight:700;color:$HEALTH_COLOR;margin-top:2px;\">$STATUS_RAW</div>
|
|
</td>
|
|
</tr>
|
|
</table>
|
|
<div style=\"margin-top:16px;padding:12px 14px;background:#f8fafc;border-radius:6px;border:1px solid #e5e7eb;\">
|
|
<span style=\"font-size:12px;font-weight:600;color:#374151;text-transform:uppercase;letter-spacing:.5px;\">🔍 Last Scrub</span>
|
|
<span style=\"margin-left:10px;font-size:13px;color:#374151;\">Completed | $SCRUB_DATE | $SCRUB_ERRORS$SCRUB_AGE_NOTE</span>
|
|
</div>
|
|
$ISSUE_BLOCK
|
|
<details style=\"margin-top:14px;\">
|
|
<summary style=\"cursor:pointer;font-size:12px;color:#6b7280;font-weight:600;letter-spacing:.3px;\">▶ vdev / drive configuration</summary>
|
|
<pre style=\"margin-top:8px;background:#1e293b;color:#e2e8f0;padding:14px;border-radius:6px;font-size:12px;line-height:1.6;overflow-x:auto;\">$ESCAPED_CONFIG</pre>
|
|
</details>
|
|
</div>
|
|
</div>"
|
|
done
|
|
|
|
# ── Overall banner ────────────────────────────────────────────────────────────
|
|
case "$OVERALL_STATUS" in
|
|
CRITICAL)
|
|
BADGE_HTML="<div style=\"background:#b91c1c;color:white;padding:10px 20px;border-radius:8px;font-size:14px;font-weight:700;\">🚨 CRITICAL</div>"
|
|
BANNER_HTML="<tr><td style=\"background:#b91c1c;padding:12px 32px;\"><span style=\"color:white;font-size:14px;font-weight:600;\">🚨 CRITICAL issue detected — immediate action required!</span></td></tr>"
|
|
;;
|
|
WARNING)
|
|
BADGE_HTML="<div style=\"background:#b45309;color:white;padding:10px 20px;border-radius:8px;font-size:14px;font-weight:700;\">⚠️ WARNING</div>"
|
|
BANNER_HTML="<tr><td style=\"background:#b45309;padding:12px 32px;\"><span style=\"color:white;font-size:14px;font-weight:600;\">⚠️ Warning condition detected — review recommended.</span></td></tr>"
|
|
;;
|
|
*)
|
|
BADGE_HTML="<div style=\"background:#1a7f4b;color:white;padding:10px 20px;border-radius:8px;font-size:14px;font-weight:700;\">✅ ALL HEALTHY</div>"
|
|
BANNER_HTML="<tr><td style=\"background:#1a7f4b;padding:12px 32px;\"><span style=\"color:white;font-size:14px;font-weight:600;\">✅ All pools are healthy — no action required.</span></td></tr>"
|
|
;;
|
|
esac
|
|
|
|
# ── Compose full HTML email ───────────────────────────────────────────────────
|
|
HTML_BODY=$(cat <<EOF
|
|
<!DOCTYPE html>
|
|
<html lang="en">
|
|
<head><meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1"></head>
|
|
<body style="margin:0;padding:0;background:#f1f5f9;font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,sans-serif;">
|
|
<table width="100%" cellspacing="0" cellpadding="0" style="background:#f1f5f9;padding:30px 0;">
|
|
<tr><td align="center">
|
|
<table width="680" cellspacing="0" cellpadding="0" style="max-width:680px;width:100%;">
|
|
|
|
<tr><td style="background:#0f172a;border-radius:10px 10px 0 0;padding:28px 32px;">
|
|
<table width="100%" cellspacing="0" cellpadding="0">
|
|
<tr>
|
|
<td>
|
|
<div style="font-size:11px;color:#94a3b8;letter-spacing:1px;text-transform:uppercase;">ZFS Pool Health Monitor</div>
|
|
<div style="font-size:26px;font-weight:800;color:white;margin-top:4px;">🔌 $HOSTNAME</div>
|
|
<div style="font-size:13px;color:#94a3b8;margin-top:4px;">$REPORT_TIME | $POOL_COUNT pool(s) monitored</div>
|
|
</td>
|
|
<td align="right">$BADGE_HTML</td>
|
|
</tr>
|
|
</table>
|
|
</td></tr>
|
|
|
|
$BANNER_HTML
|
|
|
|
<tr><td style="padding:28px 24px 8px;">
|
|
$POOL_CARDS_HTML
|
|
</td></tr>
|
|
|
|
<tr><td style="padding:0 24px 28px;">
|
|
<div style="background:white;border-radius:10px;border:1px solid #e5e7eb;padding:18px 22px;">
|
|
<div style="font-size:13px;font-weight:700;color:#374151;margin-bottom:10px;text-transform:uppercase;letter-spacing:.5px;">📋 Report Summary</div>
|
|
<table width="100%" cellspacing="0" cellpadding="0">
|
|
<tr>
|
|
<td style="font-size:13px;color:#6b7280;">Host</td>
|
|
<td style="font-size:13px;color:#111827;font-weight:600;">$HOSTNAME</td>
|
|
</tr>
|
|
<tr><td colspan="2" style="height:6px;"></td></tr>
|
|
<tr>
|
|
<td style="font-size:13px;color:#6b7280;">Report Time</td>
|
|
<td style="font-size:13px;color:#111827;font-weight:600;">$REPORT_TIME</td>
|
|
</tr>
|
|
<tr><td colspan="2" style="height:6px;"></td></tr>
|
|
<tr>
|
|
<td style="font-size:13px;color:#6b7280;">Pools Checked</td>
|
|
<td style="font-size:13px;color:#111827;font-weight:600;">$POOL_COUNT</td>
|
|
</tr>
|
|
<tr><td colspan="2" style="height:6px;"></td></tr>
|
|
<tr>
|
|
<td style="font-size:13px;color:#6b7280;">Pools with Issues</td>
|
|
<td style="font-size:13px;font-weight:700;color:#b91c1c;">$POOLS_WITH_ISSUES</td>
|
|
</tr>
|
|
<tr><td colspan="2" style="height:6px;"></td></tr>
|
|
<tr>
|
|
<td style="font-size:13px;color:#6b7280;">Scrub Age Warning Threshold</td>
|
|
<td style="font-size:13px;color:#111827;font-weight:600;">$SCRUB_AGE_WARN_DAYS days</td>
|
|
</tr>
|
|
<tr><td colspan="2" style="height:6px;"></td></tr>
|
|
<tr>
|
|
<td style="font-size:13px;color:#6b7280;">Capacity Warning Threshold</td>
|
|
<td style="font-size:13px;color:#111827;font-weight:600;">${CAPACITY_WARN_THRESHOLD}%</td>
|
|
</tr>
|
|
</table>
|
|
</div>
|
|
</td></tr>
|
|
|
|
<tr><td style="background:#0f172a;border-radius:0 0 10px 10px;padding:16px 32px;text-align:center;">
|
|
<span style="font-size:11px;color:#64748b;">Automated ZFS Monitor | $HOSTNAME | KingDezigns Infrastructure</span>
|
|
</td></tr>
|
|
|
|
</table>
|
|
</td></tr>
|
|
</table>
|
|
</body>
|
|
</html>
|
|
EOF
|
|
)
|
|
|
|
# ── Send email via OMV Postfix (Gmail relay) ──────────────────────────────────
|
|
SUBJECT="[ZFS] $HOSTNAME — $OVERALL_STATUS | $REPORT_TIME"
|
|
|
|
{
|
|
echo "To: $REPORT_TO"
|
|
echo "From: $REPORT_FROM"
|
|
echo "Subject: $SUBJECT"
|
|
echo "MIME-Version: 1.0"
|
|
echo "Content-Type: text/html; charset=UTF-8"
|
|
echo ""
|
|
echo "$HTML_BODY"
|
|
} | sendmail -t
|
|
|
|
log "Email sent to $REPORT_TO — Overall status: $OVERALL_STATUS"
|
|
log "===== ZFS Scrub complete on $HOSTNAME ====="
|