nas08-scripts/zfs/nas08_zfs_report.sh

352 lines
21 KiB
Bash
Raw Permalink Normal View History

2026-07-26 21:34:22 -04:00
#!/usr/bin/env bash
# =============================================================================
# ZFS Pool Health Report — NAS08
# Sends an HTML status email to rufus.king@kingdezigns.com
# Cron example: 0 7 * * * /opt/scripts/zfs_report_nas08.sh
#
# Commands executed per pool:
# zpool status -v <pool> — full status + verbose per-file error listing
# zpool list <pool> — pool-level size / capacity / health
# zfs list -r <pool> — per-dataset breakdown (used, avail, refer, mount)
# =============================================================================
set -euo pipefail
# ── Configuration ─────────────────────────────────────────────────────────────
TARGET_HOST="NAS08"
REPORT_TO="rufus.king@kingdezigns.com"
REPORT_FROM="zfs-monitor@${TARGET_HOST,,}.local"
SCRUB_MAX_AGE_DAYS=8 # Warn if last scrub is older than this
CAP_WARN=80 # Capacity % that triggers WARNING
CAP_CRIT=90 # Capacity % that triggers CRITICAL
# ──────────────────────────────────────────────────────────────────────────────
NOW=$(date '+%Y-%m-%d %H:%M:%S')
HOSTNAME_REAL=$(hostname -s 2>/dev/null || echo "$TARGET_HOST")
# ── HTML escape helper ────────────────────────────────────────────────────────
html_escape() { sed 's/&/\&amp;/g; s/</\&lt;/g; s/>/\&gt;/g; s/"/\&quot;/g'; }
# ── Collect pool list ─────────────────────────────────────────────────────────
POOL_LIST=$(zpool list -H -o name 2>/dev/null) || { echo "ERROR: zpool not found"; exit 1; }
OVERALL_STATUS="HEALTHY"
POOL_BLOCKS=""
POOL_COUNT=0
PROBLEM_COUNT=0
# ── Per-pool loop ─────────────────────────────────────────────────────────────
while IFS= read -r POOL; do
[[ -z "$POOL" ]] && continue
POOL_COUNT=$((POOL_COUNT + 1))
# 1. zpool status -v ── verbose: includes per-file permanent error listing
STATUS_RAW=$(zpool status -v "$POOL" 2>&1)
# 2. zpool list ── raw (-p) for math, human for display
ZPOOL_RAW=$(zpool list -H -p -o name,size,alloc,free,frag,cap,health "$POOL" 2>&1)
ZPOOL_HUM=$(zpool list -H -o name,size,alloc,free,frag,cap,health "$POOL" 2>&1)
HEALTH=$(echo "$ZPOOL_RAW" | awk '{print $7}')
CAP_RAW=$(echo "$ZPOOL_RAW" | awk '{print $6}')
SIZE_H=$(echo "$ZPOOL_HUM" | awk '{print $2}')
ALLOC_H=$(echo "$ZPOOL_HUM" | awk '{print $3}')
FREE_H=$(echo "$ZPOOL_HUM" | awk '{print $4}')
FRAG_H=$(echo "$ZPOOL_HUM" | awk '{print $5}')
CAP_H=$(echo "$ZPOOL_HUM" | awk '{print $6}')
# 3. zfs list -r ── per-dataset: name, used, avail, refer, mountpoint
ZFS_LIST_RAW=$(zfs list -r -H -o name,used,avail,refer,mountpoint "$POOL" 2>&1)
# Build dataset table rows
DATASET_ROWS=""
DATASET_COUNT=0
while IFS=$'\t' read -r DS_NAME DS_USED DS_AVAIL DS_REFER DS_MOUNT; do
[[ -z "$DS_NAME" ]] && continue
DATASET_COUNT=$((DATASET_COUNT + 1))
(( DATASET_COUNT % 2 == 0 )) && ROW_SHADE="#f9fafb" || ROW_SHADE="white"
DS_NAME_ESC=$(echo "$DS_NAME" | html_escape)
DS_MOUNT_ESC=$(echo "$DS_MOUNT" | html_escape)
DATASET_ROWS+="
<tr style='background:${ROW_SHADE};'>
<td style='padding:7px 10px;font-size:12px;color:#111827;font-family:monospace;border-bottom:1px solid #f3f4f6;'>${DS_NAME_ESC}</td>
<td style='padding:7px 10px;font-size:12px;color:#374151;text-align:right;border-bottom:1px solid #f3f4f6;'>${DS_USED}</td>
<td style='padding:7px 10px;font-size:12px;color:#374151;text-align:right;border-bottom:1px solid #f3f4f6;'>${DS_AVAIL}</td>
<td style='padding:7px 10px;font-size:12px;color:#374151;text-align:right;border-bottom:1px solid #f3f4f6;'>${DS_REFER}</td>
<td style='padding:7px 10px;font-size:12px;color:#6b7280;font-family:monospace;border-bottom:1px solid #f3f4f6;'>${DS_MOUNT_ESC}</td>
</tr>"
done <<< "$ZFS_LIST_RAW"
DATASET_TABLE="
<details style='margin-top:14px;'>
<summary style='cursor:pointer;font-size:12px;color:#6b7280;font-weight:600;letter-spacing:.3px;'>&#9654; Datasets / Filesystems — zfs list -r &nbsp;(${DATASET_COUNT} entries)</summary>
<div style='margin-top:8px;overflow-x:auto;border-radius:6px;border:1px solid #e5e7eb;'>
<table width='100%' cellspacing='0' cellpadding='0' style='border-collapse:collapse;'>
<thead>
<tr style='background:#0f172a;'>
<th style='padding:8px 10px;font-size:11px;color:#94a3b8;text-align:left;text-transform:uppercase;letter-spacing:.5px;'>Dataset</th>
<th style='padding:8px 10px;font-size:11px;color:#94a3b8;text-align:right;text-transform:uppercase;letter-spacing:.5px;'>Used</th>
<th style='padding:8px 10px;font-size:11px;color:#94a3b8;text-align:right;text-transform:uppercase;letter-spacing:.5px;'>Available</th>
<th style='padding:8px 10px;font-size:11px;color:#94a3b8;text-align:right;text-transform:uppercase;letter-spacing:.5px;'>Referenced</th>
<th style='padding:8px 10px;font-size:11px;color:#94a3b8;text-align:left;text-transform:uppercase;letter-spacing:.5px;'>Mountpoint</th>
</tr>
</thead>
<tbody>${DATASET_ROWS}
</tbody>
</table>
</div>
</details>"
# ── Scrub parse ────────────────────────────────────────────────────────────
SCRUB_LINE=$(echo "$STATUS_RAW" | grep -E "scan:|scrub" | head -1 || true)
if echo "$SCRUB_LINE" | grep -q "scrub repaired"; then
LAST_SCRUB=$(echo "$SCRUB_LINE" | grep -oP '\w{3} \w{3} +\d+ \d+:\d+:\d+ \d{4}' | head -1 || echo "unknown")
ERRORS_SCRUB=$(echo "$SCRUB_LINE" | grep -oP '\d+ errors' | head -1 || echo "0 errors")
SCRUB_STATUS="Completed"
elif echo "$SCRUB_LINE" | grep -q "in progress"; then
LAST_SCRUB="In Progress"; SCRUB_STATUS="Running"; ERRORS_SCRUB="—"
elif echo "$SCRUB_LINE" | grep -q "none requested"; then
LAST_SCRUB="Never"; SCRUB_STATUS="Never Run"; ERRORS_SCRUB="—"
else
LAST_SCRUB=$(echo "$SCRUB_LINE" | sed 's/.*on //' | sed 's/ with.*//' || echo "unknown")
SCRUB_STATUS="Completed"
ERRORS_SCRUB=$(echo "$SCRUB_LINE" | grep -oP '\d+ errors' | head -1 || echo "0 errors")
fi
# ── Severity logic ─────────────────────────────────────────────────────────
POOL_SEVERITY="ok"
POOL_LABEL="HEALTHY"
if [[ "$HEALTH" != "ONLINE" ]]; then
POOL_SEVERITY="critical"; POOL_LABEL="$HEALTH"
OVERALL_STATUS="CRITICAL"; PROBLEM_COUNT=$((PROBLEM_COUNT + 1))
fi
CAP_NUM=${CAP_RAW//%/}; CAP_NUM=${CAP_NUM%%.*}
if [[ "$CAP_NUM" =~ ^[0-9]+$ ]]; then
if (( CAP_NUM >= CAP_CRIT )); then
POOL_SEVERITY="critical"; POOL_LABEL="CRITICAL — CAPACITY ${CAP_H}"
OVERALL_STATUS="CRITICAL"
[[ "$HEALTH" == "ONLINE" ]] && PROBLEM_COUNT=$((PROBLEM_COUNT + 1))
elif (( CAP_NUM >= CAP_WARN )); then
[[ "$POOL_SEVERITY" == "ok" ]] && POOL_SEVERITY="warning"
[[ "$POOL_LABEL" == "HEALTHY" ]] && POOL_LABEL="WARNING — CAPACITY ${CAP_H}"
[[ "$OVERALL_STATUS" == "HEALTHY" ]] && OVERALL_STATUS="WARNING"
fi
fi
if [[ "$LAST_SCRUB" == "Never" ]]; then
[[ "$POOL_SEVERITY" == "ok" ]] && POOL_SEVERITY="warning"
[[ "$POOL_LABEL" == "HEALTHY" ]] && POOL_LABEL="WARNING — NO SCRUB"
[[ "$OVERALL_STATUS" == "HEALTHY" ]] && OVERALL_STATUS="WARNING"
fi
case "$POOL_SEVERITY" in
ok) BADGE_BG="#1a7f4b"; BADGE_TEXT="white"; ROW_BG="#f0fdf4" ;;
warning) BADGE_BG="#b45309"; BADGE_TEXT="white"; ROW_BG="#fffbeb" ;;
critical) BADGE_BG="#b91c1c"; BADGE_TEXT="white"; ROW_BG="#fff1f2" ;;
esac
# ── Error blocks ───────────────────────────────────────────────────────────
# a) Pool/vdev state errors
ERROR_LINES=$(echo "$STATUS_RAW" \
| grep -E "(DEGRADED|FAULTED|OFFLINE|REMOVED|UNAVAIL|errors:)" \
| grep -v "errors: No known data errors" || true)
# b) Permanent per-file errors exposed by -v flag
VERBOSE_ERRORS=$(echo "$STATUS_RAW" \
| awk '/Permanent errors have been detected/,0' \
| grep -v "^$" || true)
ERROR_HTML=""
if [[ -n "$ERROR_LINES" ]]; then
ERROR_LINES_ESC=$(echo "$ERROR_LINES" | head -30 | html_escape)
ERROR_HTML+="
<div style='margin-top:10px;padding:10px 14px;background:#fff1f2;border-left:4px solid #b91c1c;border-radius:4px;'>
<div style='font-size:11px;font-weight:700;color:#b91c1c;text-transform:uppercase;letter-spacing:.5px;margin-bottom:6px;'>&#9888; Pool / vdev Errors</div>
<pre style='margin:0;font-family:monospace;font-size:12px;color:#7f1d1d;white-space:pre-wrap;'>${ERROR_LINES_ESC}</pre>
</div>"
fi
if [[ -n "$VERBOSE_ERRORS" ]]; then
VERBOSE_ESC=$(echo "$VERBOSE_ERRORS" | head -50 | html_escape)
ERROR_HTML+="
<div style='margin-top:10px;padding:10px 14px;background:#fef3c7;border-left:4px solid #d97706;border-radius:4px;'>
<div style='font-size:11px;font-weight:700;color:#92400e;text-transform:uppercase;letter-spacing:.5px;margin-bottom:6px;'>&#128196; Affected Files — zpool status -v</div>
<pre style='margin:0;font-family:monospace;font-size:12px;color:#78350f;white-space:pre-wrap;'>${VERBOSE_ESC}</pre>
</div>"
fi
# ── vdev config tree ───────────────────────────────────────────────────────
VDEV_TREE=$(echo "$STATUS_RAW" \
| sed -n '/config:/,/errors:/p' \
| grep -v "^$" | head -60 | html_escape || true)
# ── Assemble pool card ─────────────────────────────────────────────────────
POOL_BLOCKS+="
<div style='background:white;border-radius:10px;box-shadow:0 1px 6px rgba(0,0,0,.10);margin-bottom:28px;overflow:hidden;border:1px solid #e5e7eb;'>
<div style='background:${ROW_BG};padding:16px 22px;border-bottom:1px solid #e5e7eb;'>
<table width='100%' cellspacing='0' cellpadding='0'><tr>
<td><span style='font-size:18px;font-weight:700;color:#111827;'>&#128201; ${POOL}</span>
<span style='margin-left:8px;font-size:12px;color:#6b7280;'>pool</span></td>
<td align='right'><span style='background:${BADGE_BG};color:${BADGE_TEXT};padding:5px 14px;border-radius:20px;font-size:12px;font-weight:700;letter-spacing:.5px;'>${POOL_LABEL}</span></td>
</tr></table>
</div>
<div style='padding:20px 22px;'>
<table width='100%' cellspacing='0' cellpadding='0'>
<tr>
<td style='width:16%;padding:8px 12px;background:#f9fafb;border-radius:6px;text-align:center;'>
<div style='font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;'>Total Size</div>
<div style='font-size:20px;font-weight:700;color:#111827;margin-top:2px;'>${SIZE_H}</div>
</td><td style='width:4%;'></td>
<td style='width:16%;padding:8px 12px;background:#f9fafb;border-radius:6px;text-align:center;'>
<div style='font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;'>Used</div>
<div style='font-size:20px;font-weight:700;color:#111827;margin-top:2px;'>${ALLOC_H}</div>
</td><td style='width:4%;'></td>
<td style='width:16%;padding:8px 12px;background:#f9fafb;border-radius:6px;text-align:center;'>
<div style='font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;'>Free</div>
<div style='font-size:20px;font-weight:700;color:#111827;margin-top:2px;'>${FREE_H}</div>
</td><td style='width:4%;'></td>
<td style='width:16%;padding:8px 12px;background:#f9fafb;border-radius:6px;text-align:center;'>
<div style='font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;'>Capacity</div>
<div style='font-size:20px;font-weight:700;color:#111827;margin-top:2px;'>${CAP_H}</div>
</td><td style='width:4%;'></td>
<td style='width:16%;padding:8px 12px;background:#f9fafb;border-radius:6px;text-align:center;'>
<div style='font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;'>Fragmentation</div>
<div style='font-size:20px;font-weight:700;color:#111827;margin-top:2px;'>${FRAG_H}</div>
</td><td style='width:4%;'></td>
<td style='width:16%;padding:8px 12px;background:#f9fafb;border-radius:6px;text-align:center;'>
<div style='font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;'>Health</div>
<div style='font-size:20px;font-weight:700;color:#111827;margin-top:2px;'>${HEALTH}</div>
</td>
</tr>
</table>
<div style='margin-top:16px;padding:12px 14px;background:#f8fafc;border-radius:6px;border:1px solid #e5e7eb;'>
<span style='font-size:12px;font-weight:600;color:#374151;text-transform:uppercase;letter-spacing:.5px;'>&#128269; Last Scrub</span>
<span style='margin-left:10px;font-size:13px;color:#374151;'>${SCRUB_STATUS} &nbsp;|&nbsp; ${LAST_SCRUB} &nbsp;|&nbsp; Errors: ${ERRORS_SCRUB}</span>
</div>
${ERROR_HTML}
<details style='margin-top:14px;'>
<summary style='cursor:pointer;font-size:12px;color:#6b7280;font-weight:600;letter-spacing:.3px;'>&#9654; vdev / Drive Configuration — zpool status -v</summary>
<pre style='margin-top:8px;background:#1e293b;color:#e2e8f0;padding:14px;border-radius:6px;font-size:12px;line-height:1.6;overflow-x:auto;white-space:pre;'>${VDEV_TREE}</pre>
</details>
${DATASET_TABLE}
</div>
</div>"
done <<< "$POOL_LIST"
# ── Overall banner ─────────────────────────────────────────────────────────────
case "$OVERALL_STATUS" in
HEALTHY) BANNER_BG="#1a7f4b"; BANNER_ICON="&#9989;"; BANNER_MSG="All pools are healthy — no action required." ;;
WARNING) BANNER_BG="#b45309"; BANNER_ICON="&#9888;&#65039;"; BANNER_MSG="One or more pools require your attention." ;;
CRITICAL) BANNER_BG="#b91c1c"; BANNER_ICON="&#128680;"; BANNER_MSG="CRITICAL issue detected — immediate action required!" ;;
esac
SUBJECT="[ZFS] ${TARGET_HOST}${OVERALL_STATUS}${NOW}"
# ── Build HTML email ───────────────────────────────────────────────────────────
HTML=$(cat <<EOF
<!DOCTYPE html>
<html lang="en">
<head><meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1"></head>
<body style="margin:0;padding:0;background:#f1f5f9;font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,sans-serif;">
<table width="100%" cellspacing="0" cellpadding="0" style="background:#f1f5f9;padding:30px 0;">
<tr><td align="center">
<table width="720" cellspacing="0" cellpadding="0" style="max-width:720px;width:100%;">
<tr><td style="background:#0f172a;border-radius:10px 10px 0 0;padding:28px 32px;">
<table width="100%" cellspacing="0" cellpadding="0"><tr>
<td>
<div style="font-size:11px;color:#94a3b8;letter-spacing:1px;text-transform:uppercase;">ZFS Pool Health Monitor</div>
<div style="font-size:26px;font-weight:800;color:white;margin-top:4px;">&#128268; ${TARGET_HOST}</div>
<div style="font-size:13px;color:#94a3b8;margin-top:4px;">${NOW} &nbsp;|&nbsp; ${POOL_COUNT} pool(s) monitored</div>
</td>
<td align="right">
<div style="background:${BANNER_BG};color:white;padding:10px 20px;border-radius:8px;font-size:14px;font-weight:700;">${BANNER_ICON} ${OVERALL_STATUS}</div>
</td>
</tr></table>
</td></tr>
<tr><td style="background:${BANNER_BG};padding:12px 32px;">
<span style="color:white;font-size:14px;font-weight:600;">${BANNER_ICON} &nbsp;${BANNER_MSG}</span>
</td></tr>
<tr><td style="padding:28px 24px 8px;">${POOL_BLOCKS}</td></tr>
<tr><td style="padding:0 24px 28px;">
<div style="background:white;border-radius:10px;border:1px solid #e5e7eb;padding:18px 22px;">
<div style="font-size:13px;font-weight:700;color:#374151;margin-bottom:12px;text-transform:uppercase;letter-spacing:.5px;">&#128203; Report Summary</div>
<table width="100%" cellspacing="0" cellpadding="0">
<tr><td style="font-size:13px;color:#6b7280;padding-bottom:6px;">Host</td>
<td style="font-size:13px;color:#111827;font-weight:600;padding-bottom:6px;">${TARGET_HOST} (${HOSTNAME_REAL})</td></tr>
<tr><td style="font-size:13px;color:#6b7280;padding-bottom:6px;">Report Time</td>
<td style="font-size:13px;color:#111827;font-weight:600;padding-bottom:6px;">${NOW}</td></tr>
<tr><td style="font-size:13px;color:#6b7280;padding-bottom:6px;">Pools Checked</td>
<td style="font-size:13px;color:#111827;font-weight:600;padding-bottom:6px;">${POOL_COUNT}</td></tr>
<tr><td style="font-size:13px;color:#6b7280;padding-bottom:6px;">Pools with Issues</td>
<td style="font-size:13px;font-weight:700;color:$([ "$PROBLEM_COUNT" -gt 0 ] && echo '#b91c1c' || echo '#1a7f4b');padding-bottom:6px;">${PROBLEM_COUNT}</td></tr>
<tr><td style="font-size:13px;color:#6b7280;padding-bottom:6px;">Capacity Warning Threshold</td>
<td style="font-size:13px;color:#111827;font-weight:600;padding-bottom:6px;">&ge; ${CAP_WARN}%</td></tr>
<tr><td style="font-size:13px;color:#6b7280;padding-bottom:6px;">Capacity Critical Threshold</td>
<td style="font-size:13px;color:#111827;font-weight:600;padding-bottom:6px;">&ge; ${CAP_CRIT}%</td></tr>
<tr><td style="font-size:13px;color:#6b7280;">Scrub Age Warning</td>
<td style="font-size:13px;color:#111827;font-weight:600;">&gt; ${SCRUB_MAX_AGE_DAYS} days</td></tr>
</table>
<div style="margin-top:14px;padding:10px 12px;background:#f8fafc;border-radius:6px;border:1px solid #e5e7eb;">
<div style="font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;margin-bottom:4px;">Commands Executed</div>
<code style="font-size:12px;color:#374151;line-height:2;">
zpool status -v &lt;pool&gt;<br>
zpool list -H -o name,size,alloc,free,frag,cap,health &lt;pool&gt;<br>
zfs list -r -H -o name,used,avail,refer,mountpoint &lt;pool&gt;
</code>
</div>
</div>
</td></tr>
<tr><td style="background:#0f172a;border-radius:0 0 10px 10px;padding:16px 32px;text-align:center;">
<span style="font-size:11px;color:#64748b;">Automated ZFS Monitor &nbsp;|&nbsp; ${TARGET_HOST} &nbsp;|&nbsp; KingDezigns Infrastructure</span>
</td></tr>
</table>
</td></tr>
</table>
</body>
</html>
EOF
)
# ── Send email ─────────────────────────────────────────────────────────────────
send_email() {
if command -v sendmail &>/dev/null; then
{ echo "To: ${REPORT_TO}"; echo "From: ${REPORT_FROM}"; echo "Subject: ${SUBJECT}"
echo "MIME-Version: 1.0"; echo "Content-Type: text/html; charset=UTF-8"; echo ""
echo "$HTML"; } | sendmail -t
echo "Report sent via sendmail to ${REPORT_TO}"
elif command -v mailx &>/dev/null; then
echo "$HTML" | mailx -a "Content-Type: text/html" -s "$SUBJECT" "$REPORT_TO"
echo "Report sent via mailx to ${REPORT_TO}"
elif command -v mail &>/dev/null; then
echo "$HTML" | mail -a "Content-Type: text/html; charset=UTF-8" -s "$SUBJECT" "$REPORT_TO"
echo "Report sent via mail to ${REPORT_TO}"
else
echo "ERROR: No mail transport found. Install sendmail, mailx, or msmtp."
FALLBACK="/tmp/zfs_report_${TARGET_HOST}_$(date +%Y%m%d_%H%M%S).html"
echo "$HTML" > "$FALLBACK"
echo "HTML report saved to: $FALLBACK"
exit 1
fi
}
send_email
echo "[$(date '+%Y-%m-%d %H:%M:%S')] ZFS report for ${TARGET_HOST} — Status: ${OVERALL_STATUS} — Pools: ${POOL_COUNT} — Problems: ${PROBLEM_COUNT}"