#!/bin/bash
# =============================================================================
# ZFS Pool Scrub & Email Report — NAS08
# Location: /usr/scripts/zfs/nas08_zfs_scrub.sh
# Schedule via OMV Scheduler (cron)
# =============================================================================
# ── Configuration ─────────────────────────────────────────────────────────────
HOSTNAME="NAS08"
REPORT_TO="rufus.king@kingdezigns.com"
REPORT_FROM="zfs-monitor@nas08.local" # Display only — Postfix uses your Gmail relay
SCRUB_WAIT_SECONDS=28800 # Max wait per pool scrub (8 hrs for large pools)
CAPACITY_WARN_THRESHOLD=80 # % capacity to flag as WARNING
SCRUB_AGE_WARN_DAYS=8 # Days since last scrub before flagging stale
LOG_FILE="/var/log/zfs_scrub_nas08.log"
# Note: Email is sent via OMV-configured Postfix (Gmail relay). No SMTP config needed here.
# ──────────────────────────────────────────────────────────────────────────────
REPORT_TIME=$(date "+%Y-%m-%d %H:%M:%S")
POOLS=$(zpool list -H -o name)
POOL_COUNT=$(echo "$POOLS" | wc -l)
log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a "$LOG_FILE"; }
log "===== ZFS Scrub started on $HOSTNAME ====="
# ── Run scrubs sequentially ───────────────────────────────────────────────────
for pool in $POOLS; do
log "Starting scrub on pool: $pool"
zpool scrub "$pool"
# Wait for scrub to finish
ELAPSED=0
while true; do
SCRUB_STATE=$(zpool status "$pool" | grep -E "scan:" | awk '{print $2}')
[[ "$SCRUB_STATE" == "scrub" ]] || break
sleep 30
ELAPSED=$((ELAPSED + 30))
if [[ $ELAPSED -ge $SCRUB_WAIT_SECONDS ]]; then
log "WARNING: Scrub on $pool exceeded wait time. Moving on."
break
fi
done
log "Scrub complete (or timed out) for pool: $pool"
done
# ── Gather data & build HTML report ──────────────────────────────────────────
OVERALL_STATUS="HEALTHY" # HEALTHY | WARNING | CRITICAL
POOLS_WITH_ISSUES=0
POOL_CARDS_HTML=""
for pool in $POOLS; do
STATUS_RAW=$(zpool list -H -o health "$pool")
SIZE=$(zpool list -H -o size "$pool")
USED=$(zpool list -H -o alloc "$pool")
FREE=$(zpool list -H -o free "$pool")
CAP=$(zpool list -H -o cap "$pool" | tr -d '%')
FRAG=$(zpool list -H -o frag "$pool")
SCRUB_LINE=$(zpool status "$pool" | grep -E "scan:")
ERRORS_LINE=$(zpool status "$pool" | grep "errors:")
CONFIG_BLOCK=$(zpool status "$pool" | awk '/config:/,/errors:/' | head -n -1)
FAULTED_LINES=$(echo "$CONFIG_BLOCK" | grep -E "FAULTED|DEGRADED|UNAVAIL|REMOVED" | grep -v "^$")
# Determine per-pool severity
POOL_SEVERITY="HEALTHY"
ISSUE_DETAIL=""
if [[ "$STATUS_RAW" == "DEGRADED" || "$STATUS_RAW" == "FAULTED" || "$STATUS_RAW" == "UNAVAIL" ]]; then
POOL_SEVERITY="CRITICAL"
OVERALL_STATUS="CRITICAL"
ISSUE_DETAIL="$FAULTED_LINES"
elif [[ "$CAP" -ge "$CAPACITY_WARN_THRESHOLD" ]]; then
POOL_SEVERITY="WARNING"
[[ "$OVERALL_STATUS" != "CRITICAL" ]] && OVERALL_STATUS="WARNING"
fi
# Scrub error count from status
SCRUB_ERRORS=$(echo "$SCRUB_LINE" | grep -oP '\d+ errors' | head -1)
[[ -z "$SCRUB_ERRORS" ]] && SCRUB_ERRORS="0 errors"
# Scrub date
SCRUB_DATE=$(echo "$SCRUB_LINE" | grep -oP '[A-Z][a-z]{2} \d+ \d+:\d+:\d+ \d{4}' | head -1)
[[ -z "$SCRUB_DATE" ]] && SCRUB_DATE="No scrub data"
# Scrub age warning
SCRUB_AGE_NOTE=""
if [[ -n "$SCRUB_DATE" && "$SCRUB_DATE" != "No scrub data" ]]; then
SCRUB_EPOCH=$(date -d "$SCRUB_DATE" +%s 2>/dev/null)
NOW_EPOCH=$(date +%s)
DIFF_DAYS=$(( (NOW_EPOCH - SCRUB_EPOCH) / 86400 ))
if [[ $DIFF_DAYS -ge $SCRUB_AGE_WARN_DAYS ]]; then
SCRUB_AGE_NOTE=" — ⚠️ Last scrub was ${DIFF_DAYS} days ago"
[[ "$POOL_SEVERITY" == "HEALTHY" ]] && POOL_SEVERITY="WARNING"
[[ "$OVERALL_STATUS" != "CRITICAL" ]] && OVERALL_STATUS="WARNING"
fi
fi
[[ "$POOL_SEVERITY" != "HEALTHY" ]] && POOLS_WITH_ISSUES=$((POOLS_WITH_ISSUES + 1))
# Health value color
case "$STATUS_RAW" in
ONLINE) HEALTH_COLOR="#1a7f4b" ;;
DEGRADED) HEALTH_COLOR="#b91c1c" ;;
FAULTED) HEALTH_COLOR="#b91c1c" ;;
*) HEALTH_COLOR="#b45309" ;;
esac
# Pool card header color
case "$POOL_SEVERITY" in
CRITICAL) CARD_BG="#fff1f2"; BADGE_BG="#b91c1c"; BADGE_LABEL="$STATUS_RAW" ;;
WARNING) CARD_BG="#fffbeb"; BADGE_BG="#b45309"; BADGE_LABEL="WARNING" ;;
*) CARD_BG="#f0fdf4"; BADGE_BG="#1a7f4b"; BADGE_LABEL="HEALTHY" ;;
esac
# Capacity color
CAP_COLOR="#111827"
[[ "$CAP" -ge "$CAPACITY_WARN_THRESHOLD" ]] && CAP_COLOR="#b45309"
[[ "$CAP" -ge 90 ]] && CAP_COLOR="#b91c1c"
# Issue block (only shown when there's a problem)
ISSUE_BLOCK=""
if [[ -n "$ISSUE_DETAIL" ]]; then
ESCAPED_DETAIL=$(echo "$ISSUE_DETAIL" | sed 's//\>/g')
ISSUE_BLOCK="$ESCAPED_DETAIL
"
fi
# Config block
ESCAPED_CONFIG=$(echo "$CONFIG_BLOCK" | sed 's//\>/g')
POOL_CARDS_HTML+="
🗄️ $pool
pool
$BADGE_LABEL
|
Total Size
$SIZE
|
|
Used
$USED
|
|
Free
$FREE
|
|
Capacity
${CAP}%
|
|
Fragmentation
$FRAG
|
|
Health
$STATUS_RAW
|
🔍 Last Scrub
Completed | $SCRUB_DATE | $SCRUB_ERRORS$SCRUB_AGE_NOTE
$ISSUE_BLOCK
▶ vdev / drive configuration
$ESCAPED_CONFIG
"
done
# ── Overall banner ────────────────────────────────────────────────────────────
case "$OVERALL_STATUS" in
CRITICAL)
BADGE_HTML="🚨 CRITICAL
"
BANNER_HTML="| 🚨 CRITICAL issue detected — immediate action required! |
"
;;
WARNING)
BADGE_HTML="⚠️ WARNING
"
BANNER_HTML="| ⚠️ Warning condition detected — review recommended. |
"
;;
*)
BADGE_HTML="✅ ALL HEALTHY
"
BANNER_HTML="| ✅ All pools are healthy — no action required. |
"
;;
esac
# ── Compose full HTML email ───────────────────────────────────────────────────
HTML_BODY=$(cat <
|
ZFS Pool Health Monitor
🔌 $HOSTNAME
$REPORT_TIME | $POOL_COUNT pool(s) monitored
|
$BADGE_HTML |
|
$BANNER_HTML
|
$POOL_CARDS_HTML
|
📋 Report Summary
| Host |
$HOSTNAME |
|
| Report Time |
$REPORT_TIME |
|
| Pools Checked |
$POOL_COUNT |
|
| Pools with Issues |
$POOLS_WITH_ISSUES |
|
| Scrub Age Warning Threshold |
$SCRUB_AGE_WARN_DAYS days |
|
| Capacity Warning Threshold |
${CAPACITY_WARN_THRESHOLD}% |
|
|
Automated ZFS Monitor | $HOSTNAME | KingDezigns Infrastructure
|
|
EOF
)
# ── Send email via OMV Postfix (Gmail relay) ──────────────────────────────────
SUBJECT="[ZFS] $HOSTNAME — $OVERALL_STATUS | $REPORT_TIME"
{
echo "To: $REPORT_TO"
echo "From: $REPORT_FROM"
echo "Subject: $SUBJECT"
echo "MIME-Version: 1.0"
echo "Content-Type: text/html; charset=UTF-8"
echo ""
echo "$HTML_BODY"
} | sendmail -t
log "Email sent to $REPORT_TO — Overall status: $OVERALL_STATUS"
log "===== ZFS Scrub complete on $HOSTNAME ====="