NAS16/nas16-scripts
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Initial import

Browse source
This commit is contained in:
Rufus King 2026-07-26 21:21:03 -04:00
commit 16746fc2b8
17 changed files with 7360 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

527
CrowdSec/crowdsec_digest_summary.html Normal file
View file

@ -0,0 +1,527 @@
<!DOCTYPE html>
<!--
=============================================================
KINGDEZIGNS — CONVERSATION SUMMARY TEMPLATE
=============================================================
HOW TO USE:
Give this file to Claude at the end of any troubleshooting
session and say:
"Use my summary template to document this conversation."
Claude will fill in all sections below based on what
actually happened in the conversation. Remove or add
sections as needed per session. Sections marked OPTIONAL
can be deleted if not applicable.
=============================================================
-->
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>KingDezigns — CrowdSec Local Alert Digest Summary</title>
<link rel="preconnect" href="https://fonts.googleapis.com">
<link href="https://fonts.googleapis.com/css2?family=IBM+Plex+Mono:wght@400;500&family=IBM+Plex+Sans:wght@400;500&display=swap" rel="stylesheet">
<style>
*, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
:root {
--bg: #f7f6f2;
--surface: #ffffff;
--surface-alt: #f1f0eb;
--border: #e2e0d8;
--border-strong: #c8c5ba;
--text-primary: #1a1917;
--text-secondary: #6b6960;
--text-muted: #9b9890;
--green-bg: #eaf3de;
--green-border: #97c459;
--green-text: #2d5a0e;
--red-bg: #fcebeb;
--red-border: #f09595;
--red-text: #7a1f1f;
--amber-bg: #faeeda;
--amber-border: #fac775;
--amber-text: #633806;
--coral-bg: #faece7;
--coral-border: #f0997b;
--coral-text: #4a1b0c;
--mono: 'IBM Plex Mono', monospace;
--sans: 'IBM Plex Sans', sans-serif;
}
body {
font-family: var(--sans);
font-size: 14px;
background: var(--bg);
color: var(--text-primary);
line-height: 1.7;
padding: 2.5rem 1.5rem;
}
.page { max-width: 860px; margin: 0 auto; }
header {
border-bottom: 1px solid var(--border-strong);
padding-bottom: 1.25rem;
margin-bottom: 2rem;
}
header h1 {
font-family: var(--mono);
font-size: 18px;
font-weight: 500;
letter-spacing: -0.02em;
margin-bottom: 6px;
}
.meta {
font-size: 12px;
color: var(--text-muted);
font-family: var(--mono);
display: flex;
gap: 16px;
flex-wrap: wrap;
}
.section { margin-bottom: 2.25rem; }
.section-title {
font-family: var(--mono);
font-size: 11px;
font-weight: 500;
text-transform: uppercase;
letter-spacing: 0.08em;
color: var(--text-muted);
margin-bottom: 1rem;
display: flex;
align-items: center;
gap: 10px;
}
.section-title::before {
content: attr(data-num);
background: var(--text-primary);
color: var(--bg);
font-size: 10px;
width: 18px;
height: 18px;
border-radius: 50%;
display: inline-flex;
align-items: center;
justify-content: center;
flex-shrink: 0;
}
.section-title::after {
content: '';
flex: 1;
height: 1px;
background: var(--border);
}
p { margin-bottom: 0.75rem; color: var(--text-primary); }
p:last-child { margin-bottom: 0; }
.badge-row { display: flex; gap: 8px; flex-wrap: wrap; margin-bottom: 1rem; }
.badge {
font-family: var(--mono);
font-size: 11px;
padding: 3px 10px;
border-radius: 3px;
border: 1px solid var(--border);
background: var(--surface);
color: var(--text-secondary);
}
.badge.red { background: var(--red-bg); color: var(--red-text); border-color: var(--red-border); }
.badge.green { background: var(--green-bg); color: var(--green-text); border-color: var(--green-border); }
.badge.amber { background: var(--amber-bg); color: var(--amber-text); border-color: var(--amber-border); }
.step {
background: var(--surface);
border: 1px solid var(--border);
border-radius: 6px;
padding: 1rem 1.25rem;
margin-bottom: 0.6rem;
}
.step-header {
font-family: var(--mono);
font-size: 12px;
font-weight: 500;
color: var(--text-secondary);
margin-bottom: 0.6rem;
display: flex;
align-items: center;
gap: 8px;
}
.step-num {
background: var(--surface-alt);
border: 1px solid var(--border);
border-radius: 50%;
width: 20px;
height: 20px;
display: inline-flex;
align-items: center;
justify-content: center;
font-size: 10px;
flex-shrink: 0;
color: var(--text-muted);
}
pre {
background: var(--surface-alt);
border: 1px solid var(--border);
border-radius: 4px;
padding: 10px 12px;
font-family: var(--mono);
font-size: 12px;
overflow-x: auto;
margin: 0.5rem 0;
white-space: pre;
line-height: 1.6;
}
code {
font-family: var(--mono);
font-size: 12px;
background: var(--surface-alt);
padding: 1px 5px;
border-radius: 3px;
border: 1px solid var(--border);
}
.why {
font-size: 12px;
color: var(--text-muted);
margin-top: 0.4rem;
padding-left: 2px;
}
.why::before { content: '↳ why: '; font-family: var(--mono); }
.result { font-size: 12px; margin-top: 0.35rem; padding-left: 2px; }
.result.pass { color: var(--green-text); }
.result.pass::before { content: '✓ '; font-family: var(--mono); }
.result.fail { color: var(--red-text); }
.result.fail::before { content: '✗ '; font-family: var(--mono); }
.result.neutral { color: var(--text-secondary); }
.result.neutral::before { content: '→ '; font-family: var(--mono); }
.callout { border-radius: 6px; padding: 1rem 1.25rem; margin-bottom: 0.75rem; }
.callout.coral { background: var(--coral-bg); border: 1px solid var(--coral-border); }
.callout.coral p { color: var(--coral-text); }
.callout.coral pre { background: #fdf0eb; border-color: var(--coral-border); }
.callout.amber { background: var(--amber-bg); border: 1px solid var(--amber-border); }
.callout.amber p { color: var(--amber-text); }
.callout.amber pre { background: #fdf5e6; border-color: var(--amber-border); }
.callout.amber code { background: #fdf5e6; border-color: var(--amber-border); color: var(--amber-text); }
.callout.green { background: var(--green-bg); border: 1px solid var(--green-border); }
.callout.green p { color: var(--green-text); }
.callout.green pre { background: #d8efc0; border-color: var(--green-border); }
.clue-label {
font-family: var(--mono);
font-size: 10px;
font-weight: 500;
text-transform: uppercase;
letter-spacing: 0.07em;
color: var(--amber-text);
margin-bottom: 8px;
}
.script-box { background: var(--surface); border: 1px solid var(--border); border-radius: 6px; overflow: hidden; }
.script-row { padding: 0.75rem 1.25rem; border-bottom: 1px solid var(--border); }
.script-row:last-child { border-bottom: none; }
.script-label { font-family: var(--mono); font-size: 10px; text-transform: uppercase; letter-spacing: 0.07em; color: var(--text-muted); margin-bottom: 6px; }
.script-row pre { margin: 0; }
table { width: 100%; border-collapse: collapse; font-size: 13px; background: var(--surface); border: 1px solid var(--border); border-radius: 6px; overflow: hidden; }
th { text-align: left; padding: 8px 14px; font-family: var(--mono); font-size: 10px; text-transform: uppercase; letter-spacing: 0.06em; color: var(--text-muted); background: var(--surface-alt); border-bottom: 1px solid var(--border); font-weight: 500; }
td { padding: 9px 14px; border-bottom: 1px solid var(--border); vertical-align: top; line-height: 1.5; }
tr:last-child td { border-bottom: none; }
.footer {
font-family: var(--mono);
font-size: 11px;
color: var(--text-muted);
margin-top: 2.5rem;
padding-top: 1rem;
border-top: 1px solid var(--border);
display: flex;
justify-content: space-between;
flex-wrap: wrap;
gap: 8px;
}
/* TEMPLATE ONLY — highlights placeholder text in browser, invisible in final output */
.placeholder {
background: #fff3cd;
border: 1px dashed #ffc107;
border-radius: 3px;
padding: 1px 6px;
font-family: var(--mono);
font-size: 12px;
color: #856404;
}
</style>
</head>
<body>
<div class="page">
<!-- =============================================
HEADER
Fill in: topic title, date, system/device info
============================================= -->
<header>
<h1>KingDezigns — CrowdSec Local Alert Digest summary</h1>
<div class="meta">
<span>📅 2026-06-09</span>
<span>🖥 NAS16 (CrowdSec notifier) / HAS (CrowdSec LAPI)</span>
<span>⚙️ OpenMediaVault / OMV Scheduled Jobs</span>
<span>🐳 Bash + Python3 + SMTP_SSL</span>
<span>☁️ CrowdSec LAPI stream</span>
</div>
</header>
<!-- =============================================
SECTION 1 — THE PROBLEM
Describe what broke, what the symptoms were,
and what changed before it broke.
============================================= -->
<div class="section">
<div class="section-title" data-num="1">The problem</div>
<p>The CrowdSec local notifier (<code>cs_notifier.sh</code>) sent an immediate, fully-styled HTML email for every single locally-detected attack ban (origin = <code>crowdsec</code>). During active scanning/probing periods this produced dozens to hundreds of emails per week, which was far more volume than expected and made the inbox unmanageable.</p>
<div class="badge-row">
<span class="badge amber">"CrowdSec LIVE ATTACK Detected on HAS - $IP" (one email per ban)</span>
<span class="badge amber">Hundreds of emails received weekly</span>
</div>
<p>No infrastructure change caused this — it was the original by-design behavior of the notifier (immediate per-decision alerting), which became unmanageable at the observed local attack volume.</p>
</div>
<!-- =============================================
SECTION 2 — DIAGNOSTIC COMMANDS
One .step block per command or command group.
Only include commands that contributed to
the diagnosis. Remove dead ends.
.result classes:
pass = confirmed something working (green)
fail = confirmed something broken (red)
neutral = ruled something out (gray)
============================================= -->
<div class="section">
<div class="section-title" data-num="2">Build &amp; test steps</div>
<div class="step">
<div class="step-header"><span class="step-num">1</span> Reviewed original cs_notifier.sh</div>
<pre>Reviewed full per-IP email-sending loop in cs_notifier.sh</pre>
<p class="why">Needed to understand existing LAPI polling, origin routing, and email format before changing behavior</p>
<p class="result neutral">Confirmed: every "crowdsec" origin decision triggered an immediate styled HTML email; CAPI decisions were already batched into a midnight digest</p>
</div>
<div class="step">
<div class="step-header"><span class="step-num">2</span> Split notifier into polling + digest</div>
<pre>cs_notifier.sh: append local "crowdsec" decisions to local_pending.json (no email)
cs_local_digest.sh (new): read local_pending.json, send one batched email, clear log</pre>
<p class="why">Decouples fast polling (every 5 min) from email sending (every 4 hours) so volume drops without losing any ban data</p>
<p class="result pass">cs_notifier.sh simplified — CAPI routing logic unchanged, local routing now just appends to a pending file</p>
</div>
<div class="step">
<div class="step-header"><span class="step-num">3</span> Manual test — inject fake ban</div>
<pre>sudo bash -c 'echo '"'"'{"value":"1.2.3.4","scenario":"crowdsecurity/http-probing","duration":"4h","type":"ban","scope":"Ip","origin":"crowdsec","start_at":"2026-06-09T14:00:00Z"}'"'"' >> /var/log/crowdsec-notifier/local_pending.json'
sudo bash /usr/scripts/crowdsec/cs_local_digest.sh
sudo tail -20 /var/log/crowdsec-notifier/notifier.log</pre>
<p class="why">Verify digest builds, sends, and clears the pending log without waiting for a real attack</p>
<p class="result fail">First attempts: email format didn't match expectations, and the "Banned IPs — Past 4 Hours" table rendered with an empty body — IP rows were missing entirely</p>
</div>
<div class="step">
<div class="step-header"><span class="step-num">4</span> Isolated row-generation bug</div>
<pre>echo "$SNAPSHOT" | python3 - &lt;&lt; 'PYEOF'
... (reads sys.stdin) ...
PYEOF</pre>
<p class="why">Table rows were empty even though local_pending.json had data — needed to isolate whether the bug was in JSON parsing or in HTML assembly</p>
<p class="result fail">Confirmed: this pattern produced zero output. A heredoc attached to <code>python3 -</code> becomes stdin itself, silently overriding the piped data — <code>sys.stdin</code> in Python was reading the heredoc body, not $SNAPSHOT</p>
</div>
<div class="step">
<div class="step-header"><span class="step-num">5</span> Fixed with inline variable substitution</div>
<pre>python3 &lt;&lt; PYEOF
snapshot = """$SNAPSHOT"""
for line in snapshot.strip().splitlines():
...
PYEOF</pre>
<p class="why">Pass $SNAPSHOT directly into the heredoc as a shell-expanded Python string instead of piping</p>
<p class="result pass">Re-ran full test — table populated correctly with IP, scenario, type, duration, scope, detected time, and CTI/WHOIS/AbuseIPDB links; log confirmed send + local_pending.json cleared</p>
</div>
</div>
<!-- =============================================
SECTION 3 — ROOT CAUSE
Use the coral callout for the cause statement.
Use the amber callout for any key clue or
missed signal.
Use the green callout for the resolution.
============================================= -->
<div class="section">
<div class="section-title" data-num="3">Root cause</div>
<div class="callout coral">
<p><strong>Notification volume:</strong> the original notifier sent one full HTML email per locally-detected ban with no batching, which scales linearly with attack frequency and quickly becomes unmanageable.</p>
</div>
<div class="callout amber">
<div class="clue-label">⚠ Key clue (table-rendering bug during testing)</div>
<p style="margin-bottom:6px;">The "Banned IPs — Past 4 Hours" table rendered with headers but an empty &lt;tbody&gt;. The script logged a non-zero pending count, so data existed — the failure was in row generation, not data collection.</p>
<pre>echo "$SNAPSHOT" | python3 - &lt;&lt; 'PYEOF'
import sys, json
for line in sys.stdin: # <- reads the HEREDOC body, not $SNAPSHOT
...
PYEOF</pre>
<p style="margin-top:8px;">When a heredoc is attached directly to <code>python3 -</code>, the heredoc body becomes the process's stdin, completely overriding any piped input. <code>sys.stdin</code> received the (empty) heredoc content instead of $SNAPSHOT, so the loop never executed.</p>
</div>
<div class="callout green" style="margin-top:0.75rem;">
<p><strong>Resolution:</strong> Replaced immediate per-ban emails with a 5-minute polling script that appends local bans to <code>local_pending.json</code>, plus a new <code>cs_local_digest.sh</code> that runs every 4 hours, builds one batched email (stat cards + full IP table), sends it, and clears the log — and fixed the row-generation bug by passing <code>$SNAPSHOT</code> as an inline Python variable inside the heredoc instead of piping it.</p>
</div>
</div>
<!-- Section 4 (Secondary issues) not applicable for this session -->
<!-- =============================================
SECTION 5 — SCRIPTS / FILES CREATED (OPTIONAL)
Use this section if any scripts, config files,
or tools were created during the session.
============================================= -->
<div class="section">
<div class="section-title" data-num="5">Scripts / files created</div>
<div class="script-box">
<div class="script-row">
<div class="script-label">Replace (updated)</div>
<pre>/usr/scripts/crowdsec/cs_notifier.sh</pre>
</div>
<div class="script-row">
<div class="script-label">Add (new)</div>
<pre>/usr/scripts/crowdsec/cs_local_digest.sh</pre>
</div>
<div class="script-row">
<div class="script-label">Setup</div>
<pre>chmod +x /usr/scripts/crowdsec/cs_notifier.sh
chmod +x /usr/scripts/crowdsec/cs_local_digest.sh
mkdir -p /var/log/crowdsec-notifier</pre>
</div>
<div class="script-row">
<div class="script-label">Run — cs_notifier.sh (existing OMV job, unchanged schedule)</div>
<pre>Every 5 minutes:
bash /usr/scripts/crowdsec/cs_notifier.sh</pre>
</div>
<div class="script-row">
<div class="script-label">Run — cs_local_digest.sh (new OMV scheduled job)</div>
<pre>Every 4 hours (cron: 0 0,4,8,12,16,20 * * *):
bash /usr/scripts/crowdsec/cs_local_digest.sh</pre>
</div>
<div class="script-row">
<div class="script-label">Manual test</div>
<pre>sudo bash -c 'echo '"'"'{"value":"1.2.3.4","scenario":"crowdsecurity/http-probing","duration":"4h","type":"ban","scope":"Ip","origin":"crowdsec","start_at":"2026-06-09T14:00:00Z"}'"'"' >> /var/log/crowdsec-notifier/local_pending.json'
sudo bash /usr/scripts/crowdsec/cs_local_digest.sh
sudo tail -20 /var/log/crowdsec-notifier/notifier.log</pre>
</div>
</div>
</div>
<!-- =============================================
SECTION 6 — LESSONS LEARNED
Keep entries short — one lesson per row.
============================================= -->
<div class="section">
<div class="section-title" data-num="6">Lessons learned</div>
<table>
<thead>
<tr><th>Lesson</th><th>Detail</th></tr>
</thead>
<tbody>
<tr>
<td>Batch high-frequency alerts</td>
<td>Per-event email alerting doesn't scale with attack frequency. A 4-hour batched digest preserves full visibility (every IP, scenario, type, duration, scope, and threat-intel links) while cutting email volume from hundreds/week to a handful/day.</td>
</tr>
<tr>
<td>Skip-if-empty for digests</td>
<td>cs_local_digest.sh exits silently with no email if local_pending.json is empty — avoids inbox clutter during quiet periods.</td>
</tr>
<tr>
<td>Snapshot-before-clear pattern</td>
<td>The pending log is only cleared (<code>&gt; local_pending.json</code>) after a confirmed successful SMTP send. If the send fails, the data stays intact and is retried on the next 4-hour run.</td>
</tr>
<tr>
<td>python3 - with a heredoc ignores piped stdin</td>
<td><code>echo "$DATA" | python3 - &lt;&lt; EOF ... EOF</code> does NOT work as expected — the heredoc body becomes stdin and the pipe is discarded. Pass shell data into Python via inline variable interpolation inside the heredoc instead (<code>var = """$DATA"""</code>).</td>
</tr>
<tr>
<td>Iterating on email templates</td>
<td>Several format iterations were needed to match the desired layout exactly. Keeping the rendered HTML output from a real test send was the fastest way to confirm/correct structure.</td>
</tr>
</tbody>
</table>
</div>
<!-- =============================================
SECTION 7 — NETWORK MAP / DOCS TO UPDATE (OPTIONAL)
List anything that changed and needs to be
reflected in your architecture map or docs.
============================================= -->
<div class="section">
<div class="section-title" data-num="7">Network map — items to update</div>
<table>
<thead>
<tr><th>Item</th><th>Old value</th><th>New value</th></tr>
</thead>
<tbody>
<tr>
<td>cs_notifier.sh behavior</td>
<td><code>Sends immediate per-IP email on each local ban</code></td>
<td><code>Appends local bans to local_pending.json (no email)</code></td>
</tr>
<tr>
<td>New scheduled job</td>
<td><code>(none)</code></td>
<td><code>cs_local_digest.sh every 4 hours — cron: 0 0,4,8,12,16,20 * * *</code></td>
</tr>
<tr>
<td>New log file</td>
<td><code>(none)</code></td>
<td><code>/var/log/crowdsec-notifier/local_pending.json</code></td>
</tr>
<tr>
<td>server_nas16.md</td>
<td><code>2-script architecture (cs_notifier.sh, cs_digest.sh)</code></td>
<td><code>3-script architecture — updated (cs_notifier.sh, cs_local_digest.sh, cs_digest.sh)</code></td>
</tr>
</tbody>
</table>
</div>
<!-- FOOTER — date and network name auto-filled by Claude -->
<div class="footer">
<span>KingDezigns Home Network</span>
<span>Generated 2026-06-09</span>
</div>
</div>
</body>
</html>

237
CrowdSec/cs_local_digest.sh Normal file
View file

@ -0,0 +1,237 @@
#!/bin/bash
# =============================================================
# CrowdSec Local Attack — 4-Hour Digest Mailer
# Host: NAS16 (192.168.150.40)
# Schedule: Every 4 hours via OMV Scheduled Jobs
# Recommended cron: 0 0,4,8,12,16,20 * * *
#
# Reads: /var/log/crowdsec-notifier/local_pending.json
# Sends: One batched HTML email:
# - Dark header + red status banner
# - 3 stat cards: Local Bans / Origin / CAPI Today
# - Single table: IP Address / Scenario / Type /
# Duration / Scope / Detected At / Links
# - Amber CAPI footer note
# - Dark footer
# Clears: local_pending.json after a successful send.
# Skips: Send entirely if there are zero pending bans.
# =============================================================
SMTP_HOST="smtppro.zoho.com"
SMTP_PORT="465"
SMTP_USER="rufus.king@kingdezigns.com"
SMTP_PASS="Valerie8545"
FROM_EMAIL="rufus.king@kingdezigns.com"
TO_EMAIL="rufus.king@kingdezigns.com"
LOCAL_PENDING="/var/log/crowdsec-notifier/local_pending.json"
CAPI_DIGEST="/var/log/crowdsec-notifier/capi_digest.json"
LOG_FILE="/var/log/crowdsec-notifier/notifier.log"
mkdir -p /var/log/crowdsec-notifier
touch "$LOCAL_PENDING"
touch "$CAPI_DIGEST"
log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" >> "$LOG_FILE"; }
log "--- cs_local_digest run start ---"
# --- Count pending bans ---
PENDING_COUNT=$(grep -c . "$LOCAL_PENDING" 2>/dev/null || echo 0)
PENDING_COUNT=$(echo "$PENDING_COUNT" | tr -d ' ')
if [[ "$PENDING_COUNT" -eq 0 ]]; then
log "No local bans in this window. Skipping email."
log "--- cs_local_digest run complete (no bans) ---"
exit 0
fi
log "Building digest for $PENDING_COUNT local ban(s)."
NOW=$(date '+%Y-%m-%d %H:%M:%S UTC')
CAPI_TODAY=$(wc -l < "$CAPI_DIGEST" 2>/dev/null | tr -d ' ')
CAPI_TODAY=${CAPI_TODAY:-0}
# Snapshot the pending file before clearing
SNAPSHOT=$(cat "$LOCAL_PENDING")
# --- Build table rows ---
# NOTE: $SNAPSHOT is expanded inline inside the Python heredoc.
# Do NOT pipe into python3 - with a heredoc: the heredoc becomes stdin,
# silently replacing the pipe. Pass data as an inline variable instead.
TABLE_ROWS=$(python3 << PYEOF
import json
snapshot = """$SNAPSHOT"""
rows = []
for line in snapshot.strip().splitlines():
line = line.strip()
if not line:
continue
try:
d = json.loads(line)
except Exception:
continue
ip = d.get("value", "unknown")
scenario = d.get("scenario", "unknown")
duration = d.get("duration", "unknown")
ban_type = d.get("type", "ban").upper()
scope = d.get("scope", "Ip").upper()
ban_time = d.get("start_at", "")
if ban_time:
ban_time = ban_time[:19].replace("T", " ") + " UTC"
else:
ban_time = "unknown"
rows.append(f""" <tr style="background:#fff1f2;">
<td style="padding:10px 14px;font-size:13px;font-weight:700;color:#111827;white-space:nowrap;">&#127760; {ip}</td>
<td style="padding:10px 14px;font-size:12px;color:#374151;font-family:monospace;">{scenario}</td>
<td style="padding:10px 14px;font-size:12px;color:#6b7280;text-align:center;">{ban_type}</td>
<td style="padding:10px 14px;font-size:12px;color:#6b7280;text-align:center;">{duration}</td>
<td style="padding:10px 14px;font-size:12px;color:#6b7280;text-align:center;">{scope}</td>
<td style="padding:10px 14px;font-size:12px;color:#6b7280;white-space:nowrap;">{ban_time}</td>
<td style="padding:10px 14px;font-size:11px;white-space:nowrap;">
<a href="https://app.crowdsec.net/cti/{ip}" style="margin-right:6px;padding:3px 8px;background:#0f172a;color:white;border-radius:4px;text-decoration:none;font-size:11px;font-weight:600;">CTI</a>
<a href="https://www.whois.com/whois/{ip}" style="margin-right:6px;padding:3px 8px;background:#374151;color:white;border-radius:4px;text-decoration:none;font-size:11px;font-weight:600;">WHOIS</a>
<a href="https://www.abuseipdb.com/check/{ip}" style="padding:3px 8px;background:#374151;color:white;border-radius:4px;text-decoration:none;font-size:11px;font-weight:600;">Abuse</a>
</td>
</tr>""")
print("\n".join(rows))
PYEOF
)
log "Generated ${PENDING_COUNT} table row(s)."
# --- Write the full email HTML ---
HTML_FILE=$(mktemp /tmp/cs_local_digest_XXXXXX.html)
cat > "$HTML_FILE" << HTMLEOF
<!DOCTYPE html>
<html lang="en">
<head><meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1"></head>
<body style="margin:0;padding:0;background:#f1f5f9;font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,sans-serif;">
<table width="100%" cellspacing="0" cellpadding="0" style="background:#f1f5f9;padding:30px 0;">
<tr><td align="center">
<table width="720" cellspacing="0" cellpadding="0" style="max-width:720px;width:100%;">
<!-- HEADER -->
<tr><td style="background:#0f172a;border-radius:10px 10px 0 0;padding:28px 32px;">
<table width="100%" cellspacing="0" cellpadding="0"><tr>
<td>
<div style="font-size:11px;color:#94a3b8;letter-spacing:1px;text-transform:uppercase;">CrowdSec Intrusion Prevention</div>
<div style="font-size:26px;font-weight:800;color:white;margin-top:4px;">&#128737; HAS &mdash; homeassistant</div>
<div style="font-size:13px;color:#94a3b8;margin-top:4px;">$NOW &nbsp;|&nbsp; Machine: homeassistant</div>
</td>
<td align="right">
<div style="background:#b91c1c;color:white;padding:10px 20px;border-radius:8px;font-size:14px;font-weight:700;">&#128680; LOCAL BANS &mdash; 4H DIGEST</div>
</td>
</tr></table>
</td></tr>
<!-- STATUS BANNER -->
<tr><td style="background:#b91c1c;padding:12px 32px;">
<span style="color:white;font-size:14px;font-weight:600;">&#128680; &nbsp;$PENDING_COUNT locally-detected attack(s) banned in the past 4 hours.</span>
</td></tr>
<!-- STAT CARDS -->
<tr><td style="padding:24px 24px 8px;">
<table width="100%" cellspacing="0" cellpadding="0"><tr>
<td style="width:30%;padding:10px 14px;background:white;border-radius:8px;border:1px solid #e5e7eb;text-align:center;">
<div style="font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;">Local Bans</div>
<div style="font-size:32px;font-weight:800;color:#b91c1c;margin-top:4px;">$PENDING_COUNT</div>
<div style="font-size:11px;color:#6b7280;margin-top:2px;">this window</div>
</td>
<td style="width:5%;"></td>
<td style="width:30%;padding:10px 14px;background:white;border-radius:8px;border:1px solid #e5e7eb;text-align:center;">
<div style="font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;">Origin</div>
<div style="font-size:20px;font-weight:800;color:#b91c1c;margin-top:8px;">&#9889; LOCAL</div>
<div style="font-size:11px;color:#6b7280;margin-top:2px;">crowdsec detection</div>
</td>
<td style="width:5%;"></td>
<td style="width:30%;padding:10px 14px;background:#fffbeb;border-radius:8px;border:1px solid #fac775;text-align:center;">
<div style="font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;">CAPI Today</div>
<div style="font-size:32px;font-weight:800;color:#b45309;margin-top:4px;">$CAPI_TODAY</div>
<div style="font-size:11px;color:#6b7280;margin-top:2px;">cloud blocklist bans</div>
</td>
</tr></table>
</td></tr>
<!-- BAN TABLE -->
<tr><td style="padding:16px 24px 8px;">
<div style="background:white;border-radius:10px;border:1px solid #e5e7eb;overflow:hidden;">
<div style="padding:14px 18px;border-bottom:1px solid #e5e7eb;background:#f9fafb;">
<span style="font-size:13px;font-weight:700;color:#374151;text-transform:uppercase;letter-spacing:.5px;">&#128204; Banned IPs &mdash; Past 4 Hours</span>
</div>
<table width="100%" cellspacing="0" cellpadding="0">
<thead>
<tr style="background:#f9fafb;">
<th style="padding:8px 14px;font-size:10px;text-transform:uppercase;letter-spacing:.6px;color:#6b7280;font-weight:600;text-align:left;border-bottom:1px solid #e5e7eb;">IP Address</th>
<th style="padding:8px 14px;font-size:10px;text-transform:uppercase;letter-spacing:.6px;color:#6b7280;font-weight:600;text-align:left;border-bottom:1px solid #e5e7eb;">Scenario</th>
<th style="padding:8px 14px;font-size:10px;text-transform:uppercase;letter-spacing:.6px;color:#6b7280;font-weight:600;text-align:center;border-bottom:1px solid #e5e7eb;">Type</th>
<th style="padding:8px 14px;font-size:10px;text-transform:uppercase;letter-spacing:.6px;color:#6b7280;font-weight:600;text-align:center;border-bottom:1px solid #e5e7eb;">Duration</th>
<th style="padding:8px 14px;font-size:10px;text-transform:uppercase;letter-spacing:.6px;color:#6b7280;font-weight:600;text-align:center;border-bottom:1px solid #e5e7eb;">Scope</th>
<th style="padding:8px 14px;font-size:10px;text-transform:uppercase;letter-spacing:.6px;color:#6b7280;font-weight:600;text-align:left;border-bottom:1px solid #e5e7eb;">Detected At</th>
<th style="padding:8px 14px;font-size:10px;text-transform:uppercase;letter-spacing:.6px;color:#6b7280;font-weight:600;text-align:left;border-bottom:1px solid #e5e7eb;">Links</th>
</tr>
</thead>
<tbody>
$TABLE_ROWS
</tbody>
</table>
</div>
</td></tr>
<!-- CAPI NOTE -->
<tr><td style="padding:12px 24px 28px;">
<div style="background:#fffbeb;border-radius:10px;border:1px solid #fac775;padding:14px 22px;">
<span style="font-size:12px;color:#92400e;">&#9925; &nbsp;<strong>Cloud blocklist (CAPI):</strong> &nbsp;$CAPI_TODAY pre-emptive ban(s) recorded today. Full CAPI digest delivered at midnight by cs_digest.sh.</span>
</div>
</td></tr>
<!-- FOOTER -->
<tr><td style="background:#0f172a;border-radius:0 0 10px 10px;padding:16px 32px;text-align:center;">
<span style="font-size:11px;color:#64748b;">CrowdSec Intrusion Prevention &nbsp;|&nbsp; homeassistant (HAS) &nbsp;|&nbsp; KingDezigns Infrastructure</span>
</td></tr>
</table>
</td></tr>
</table>
</body>
</html>
HTMLEOF
# --- Send the email ---
python3 << PYEOF >> "$LOG_FILE" 2>&1
import smtplib, ssl
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
with open("$HTML_FILE") as f:
html_body = f.read()
msg = MIMEMultipart("alternative")
msg["Subject"] = "CrowdSec 4H Local Digest — $PENDING_COUNT Ban(s) on HAS"
msg["From"] = "CrowdSec KingDezigns <$FROM_EMAIL>"
msg["To"] = "$TO_EMAIL"
msg.attach(MIMEText(html_body, "html"))
ctx = ssl.create_default_context()
with smtplib.SMTP_SSL("$SMTP_HOST", $SMTP_PORT, context=ctx, timeout=15) as s:
s.login("$SMTP_USER", "$SMTP_PASS")
s.sendmail("$FROM_EMAIL", ["$TO_EMAIL"], msg.as_string())
print("sent")
PYEOF
SEND_STATUS=$?
rm -f "$HTML_FILE"
if [[ $SEND_STATUS -eq 0 ]]; then
log "4-hour digest sent. $PENDING_COUNT ban(s) reported. Clearing local_pending.json."
> "$LOCAL_PENDING"
else
log "ERROR: Digest email failed. local_pending.json NOT cleared — will retry next run."
fi
log "--- cs_local_digest run complete ---"

100
CrowdSec/cs_notifier.sh Normal file
View file

@ -0,0 +1,100 @@
#!/bin/bash
# =============================================================
# CrowdSec Local Attack Notifier
# Host: NAS16 (192.168.150.40)
# Target: CrowdSec LAPI on HAS (192.168.150.30:8080)
# Schedule: Every 5 minutes via OMV Scheduled Jobs
#
# Local (crowdsec origin) decisions are appended to:
# /var/log/crowdsec-notifier/local_pending.json
# and batched into a 4-hour digest by cs_local_digest.sh.
#
# CAPI decisions continue to be appended to capi_digest.json
# and sent as a midnight summary by cs_digest.sh (unchanged).
# =============================================================
LAPI_URL="http://192.168.150.30:8080"
API_KEY="K1nZ5TYeyl18adqw7+dop/po9gHZNQvsuLz78lla/34"
STARTUP_FLAG="/var/log/crowdsec-notifier/stream_initialized.flag"
LOCAL_PENDING="/var/log/crowdsec-notifier/local_pending.json"
CAPI_DIGEST="/var/log/crowdsec-notifier/capi_digest.json"
LOG_FILE="/var/log/crowdsec-notifier/notifier.log"
mkdir -p /var/log/crowdsec-notifier
touch "$CAPI_DIGEST"
touch "$LOCAL_PENDING"
log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" >> "$LOG_FILE"; }
log "--- cs_notifier run start ---"
# --- First run: initialize stream cursor silently ---
if [[ ! -f "$STARTUP_FLAG" ]]; then
log "First run — initializing stream cursor. No emails will be sent."
INIT_RESPONSE=$(curl -s --max-time 60 \
-H "X-Api-Key: $API_KEY" \
"${LAPI_URL}/v1/decisions/stream?startup=true")
if [[ -z "$INIT_RESPONSE" ]]; then
log "ERROR: Empty response from LAPI during startup. Will retry next run."
exit 1
fi
NEW_COUNT=$(echo "$INIT_RESPONSE" | python3 -c "
import sys, json
data = json.load(sys.stdin)
print(len(data.get('new') or []))
" 2>/dev/null)
log "Stream cursor initialized. Skipped $NEW_COUNT existing decisions. Ready for live monitoring."
touch "$STARTUP_FLAG"
log "--- cs_notifier run complete (init) ---"
exit 0
fi
# --- Normal run: fetch only new decisions since last call ---
RESPONSE=$(curl -s --max-time 30 \
-H "X-Api-Key: $API_KEY" \
"${LAPI_URL}/v1/decisions/stream?startup=false")
HTTP_CHECK=$(echo "$RESPONSE" | python3 -c "import sys,json; json.load(sys.stdin); print('ok')" 2>/dev/null)
if [[ "$HTTP_CHECK" != "ok" ]]; then
log "ERROR: Invalid response from LAPI stream endpoint"
exit 1
fi
# --- Parse and route new decisions ---
NEW_LOCAL_COUNT=0
NEW_CAPI_COUNT=0
while IFS= read -r decision; do
[[ -z "$decision" ]] && continue
ORIGIN=$(echo "$decision" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('origin',''))" 2>/dev/null)
if [[ "$ORIGIN" == "crowdsec" ]]; then
# Append to pending local digest — cs_local_digest.sh will send every 4 hours
echo "$decision" >> "$LOCAL_PENDING"
NEW_LOCAL_COUNT=$((NEW_LOCAL_COUNT + 1))
elif [[ "$ORIGIN" == "CAPI" ]]; then
# Append to CAPI digest — cs_digest.sh will send at midnight
echo "$decision" >> "$CAPI_DIGEST"
NEW_CAPI_COUNT=$((NEW_CAPI_COUNT + 1))
fi
done < <(echo "$RESPONSE" | python3 -c "
import sys, json
data = json.load(sys.stdin)
new = data.get('new') or []
for item in new:
print(json.dumps(item))
")
CAPI_TODAY=$(wc -l < "$CAPI_DIGEST" 2>/dev/null | tr -d ' ')
CAPI_TODAY=${CAPI_TODAY:-0}
LOCAL_PENDING_COUNT=$(wc -l < "$LOCAL_PENDING" 2>/dev/null | tr -d ' ')
LOCAL_PENDING_COUNT=${LOCAL_PENDING_COUNT:-0}
log "New local: $NEW_LOCAL_COUNT (pending total: $LOCAL_PENDING_COUNT) | New CAPI: $NEW_CAPI_COUNT | CAPI today: $CAPI_TODAY"
log "--- cs_notifier run complete ---"

406
ddns/ddns_summary.html Normal file
View file

@ -0,0 +1,406 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>KingDezigns — DDNS Updater Summary</title>
<link rel="preconnect" href="https://fonts.googleapis.com">
<link href="https://fonts.googleapis.com/css2?family=IBM+Plex+Mono:wght@400;500&family=IBM+Plex+Sans:wght@400;500&display=swap" rel="stylesheet">
<style>
*, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
:root {
--bg: #f7f6f2; --surface: #ffffff; --surface-alt: #f1f0eb;
--border: #e2e0d8; --border-strong: #c8c5ba;
--text-primary: #1a1917; --text-secondary: #6b6960; --text-muted: #9b9890;
--green-bg: #eaf3de; --green-border: #97c459; --green-text: #2d5a0e;
--red-bg: #fcebeb; --red-border: #f09595; --red-text: #7a1f1f;
--amber-bg: #faeeda; --amber-border: #fac775; --amber-text: #633806;
--coral-bg: #faece7; --coral-border: #f0997b; --coral-text: #4a1b0c;
--mono: 'IBM Plex Mono', monospace; --sans: 'IBM Plex Sans', sans-serif;
}
body { font-family: var(--sans); font-size: 14px; background: var(--bg); color: var(--text-primary); line-height: 1.7; padding: 2.5rem 1.5rem; }
.page { max-width: 860px; margin: 0 auto; }
header { border-bottom: 1px solid var(--border-strong); padding-bottom: 1.25rem; margin-bottom: 2rem; }
header h1 { font-family: var(--mono); font-size: 18px; font-weight: 500; letter-spacing: -0.02em; margin-bottom: 6px; }
.meta { font-size: 12px; color: var(--text-muted); font-family: var(--mono); display: flex; gap: 16px; flex-wrap: wrap; }
.section { margin-bottom: 2.25rem; }
.section-title { font-family: var(--mono); font-size: 11px; font-weight: 500; text-transform: uppercase; letter-spacing: 0.08em; color: var(--text-muted); margin-bottom: 1rem; display: flex; align-items: center; gap: 10px; }
.section-title::before { content: attr(data-num); background: var(--text-primary); color: var(--bg); font-size: 10px; width: 18px; height: 18px; border-radius: 50%; display: inline-flex; align-items: center; justify-content: center; flex-shrink: 0; }
.section-title::after { content: ''; flex: 1; height: 1px; background: var(--border); }
p { margin-bottom: 0.75rem; color: var(--text-primary); }
p:last-child { margin-bottom: 0; }
.badge { font-family: var(--mono); font-size: 11px; padding: 3px 10px; border-radius: 3px; border: 1px solid var(--border); background: var(--surface); color: var(--text-secondary); display: inline-block; }
.badge.red { background: var(--red-bg); color: var(--red-text); border-color: var(--red-border); }
.badge.green { background: var(--green-bg); color: var(--green-text); border-color: var(--green-border); }
.badge.amber { background: var(--amber-bg); color: var(--amber-text); border-color: var(--amber-border); }
.badge-row { display: flex; gap: 8px; flex-wrap: wrap; margin-bottom: 1rem; }
.step { background: var(--surface); border: 1px solid var(--border); border-radius: 6px; padding: 1rem 1.25rem; margin-bottom: 0.6rem; }
.step-header { font-family: var(--mono); font-size: 12px; font-weight: 500; color: var(--text-secondary); margin-bottom: 0.6rem; display: flex; align-items: center; gap: 8px; }
.step-num { background: var(--surface-alt); border: 1px solid var(--border); border-radius: 50%; width: 20px; height: 20px; display: inline-flex; align-items: center; justify-content: center; font-size: 10px; flex-shrink: 0; color: var(--text-muted); }
pre { background: var(--surface-alt); border: 1px solid var(--border); border-radius: 4px; padding: 10px 12px; font-family: var(--mono); font-size: 12px; overflow-x: auto; margin: 0.5rem 0; white-space: pre; line-height: 1.6; }
code { font-family: var(--mono); font-size: 12px; background: var(--surface-alt); padding: 1px 5px; border-radius: 3px; border: 1px solid var(--border); }
.why { font-size: 12px; color: var(--text-muted); margin-top: 0.4rem; padding-left: 2px; }
.why::before { content: '↳ note: '; font-family: var(--mono); }
.result { font-size: 12px; margin-top: 0.35rem; padding-left: 2px; }
.result.pass { color: var(--green-text); }
.result.pass::before { content: '✓ '; font-family: var(--mono); }
.result.neutral { color: var(--text-secondary); }
.result.neutral::before { content: '→ '; font-family: var(--mono); }
.callout { border-radius: 6px; padding: 1rem 1.25rem; margin-bottom: 0.75rem; }
.callout.coral { background: var(--coral-bg); border: 1px solid var(--coral-border); }
.callout.coral p { color: var(--coral-text); }
.callout.amber { background: var(--amber-bg); border: 1px solid var(--amber-border); }
.callout.amber p { color: var(--amber-text); }
.callout.amber pre { background: #fdf5e6; border-color: var(--amber-border); }
.callout.amber code { background: #fdf5e6; border-color: var(--amber-border); color: var(--amber-text); }
.callout.green { background: var(--green-bg); border: 1px solid var(--green-border); }
.callout.green p { color: var(--green-text); }
.callout.green pre { background: #d8efc0; border-color: var(--green-border); }
.clue-label { font-family: var(--mono); font-size: 10px; font-weight: 500; text-transform: uppercase; letter-spacing: 0.07em; color: var(--amber-text); margin-bottom: 8px; }
.script-box { background: var(--surface); border: 1px solid var(--border); border-radius: 6px; overflow: hidden; }
.script-row { padding: 0.75rem 1.25rem; border-bottom: 1px solid var(--border); }
.script-row:last-child { border-bottom: none; }
.script-label { font-family: var(--mono); font-size: 10px; text-transform: uppercase; letter-spacing: 0.07em; color: var(--text-muted); margin-bottom: 6px; }
.script-row pre { margin: 0; }
table { width: 100%; border-collapse: collapse; font-size: 13px; background: var(--surface); border: 1px solid var(--border); border-radius: 6px; overflow: hidden; }
th { text-align: left; padding: 8px 14px; font-family: var(--mono); font-size: 10px; text-transform: uppercase; letter-spacing: 0.06em; color: var(--text-muted); background: var(--surface-alt); border-bottom: 1px solid var(--border); font-weight: 500; }
td { padding: 9px 14px; border-bottom: 1px solid var(--border); vertical-align: top; line-height: 1.5; }
tr:last-child td { border-bottom: none; }
.flow { display: flex; flex-direction: column; gap: 0; }
.flow-row { display: flex; align-items: stretch; gap: 0; }
.flow-node { background: var(--surface); border: 1px solid var(--border); border-radius: 6px; padding: 0.65rem 1rem; font-family: var(--mono); font-size: 12px; flex-shrink: 0; display: flex; align-items: center; gap: 8px; }
.flow-node.trigger { border-color: var(--border-strong); background: var(--surface-alt); font-weight: 500; }
.flow-node.action { border-color: var(--green-border); background: var(--green-bg); color: var(--green-text); }
.flow-node.warn { border-color: var(--amber-border); background: var(--amber-bg); color: var(--amber-text); }
.flow-node.fail { border-color: var(--red-border); background: var(--red-bg); color: var(--red-text); }
.flow-node.silent { border-color: var(--border); background: var(--surface-alt); color: var(--text-muted); }
.flow-node.daily { border-color: #a89ccc; background: #f0edf8; color: #3d2d6b; }
.flow-arrow { display: flex; flex-direction: column; align-items: center; padding: 3px 0; color: var(--text-muted); font-family: var(--mono); font-size: 11px; gap: 1px; }
.flow-branch { display: grid; grid-template-columns: 1fr 1fr; gap: 0.6rem; margin: 0.3rem 0; }
.flow-branch-col { display: flex; flex-direction: column; gap: 0.3rem; }
.branch-label { font-family: var(--mono); font-size: 10px; text-transform: uppercase; letter-spacing: 0.06em; color: var(--text-muted); padding: 0 2px; margin-bottom: 2px; }
.footer { font-family: var(--mono); font-size: 11px; color: var(--text-muted); margin-top: 2.5rem; padding-top: 1rem; border-top: 1px solid var(--border); display: flex; justify-content: space-between; flex-wrap: wrap; gap: 8px; }
</style>
</head>
<body>
<div class="page">
<header>
<h1>KingDezigns — DDNS Updater Summary</h1>
<div class="meta">
<span>📅 May 19, 2025</span>
<span>🖥 NAS16 (Raspberry Pi)</span>
<span>⚙️ OpenMediaVault</span>
<span>☁️ DreamHost DNS API</span>
<span>🌐 20 A Records · 4 Domains</span>
</div>
</header>
<!-- ===== SECTION 1 — WHAT WAS BUILT ===================================== -->
<div class="section">
<div class="section-title" data-num="1">What was built</div>
<p>A self-contained Bash script that keeps all KingDezigns DreamHost A records synchronized with the current home WAN IP. It runs every 15 minutes via OMV Scheduled Tasks. It only calls the DreamHost API when the IP actually changes (or during a nightly sanity check), so there is zero risk of being banned for spamming.</p>
<div class="badge-row">
<span class="badge green">20 A records managed</span>
<span class="badge green">4 domains</span>
<span class="badge green">Max 2 API calls per event</span>
<span class="badge green">HTML email notifications</span>
<span class="badge">Every 15 min via cron</span>
<span class="badge">Daily sanity check 00:00–00:14</span>
</div>
</div>
<!-- ===== SECTION 2 — LOGIC FLOW MAP ===================================== -->
<div class="section">
<div class="section-title" data-num="2">Logic flow map</div>
<div class="flow">
<div class="flow-node trigger">⏱ Script runs (every 15 min via OMV cron)</div>
<div class="flow-arrow">↓</div>
<div class="flow-node">🌐 Get current WAN IP from api.ipify.org / ifconfig.me / icanhazip.com</div>
<div class="flow-arrow">↓</div>
<div class="flow-node">📄 Read cached previous IP from <code>/var/log/ddns/last_wan_ip.txt</code></div>
<div class="flow-arrow">↓</div>
<div class="flow-node">🕛 Is the time between 00:00 and 00:14? (daily sanity-check window)</div>
<div class="flow-arrow">↓</div>
<!-- Branch: IP changed vs same -->
<div class="flow-branch">
<div class="flow-branch-col">
<div class="branch-label">IP changed OR daily window</div>
<div class="flow-node action">▶ Query DreamHost — check each A record</div>
<div class="flow-node action">📡 Call dns-remove_record + dns-add_record for each mismatched record</div>
<div class="flow-node">⏳ Wait 3–5 minutes (random)</div>
<div class="flow-node">🔍 Re-query DreamHost — verify each record</div>
<!-- Sub-branch: all ok vs still failing -->
<div class="flow-branch" style="margin-top:0.3rem;">
<div class="flow-branch-col">
<div class="branch-label">Still wrong</div>
<div class="flow-node warn">🔁 Attempt 2 of 2 — update remaining records</div>
<div class="flow-node">⏳ Wait 3–5 min · Re-verify</div>
<div class="flow-branch">
<div class="flow-branch-col">
<div class="branch-label">Still failing</div>
<div class="flow-node fail">✗ STOP (max attempts reached)</div>
<div class="flow-node fail">📧 Email: partial failure + manual fix instructions + WAN IP</div>
</div>
<div class="flow-branch-col">
<div class="branch-label">Now correct</div>
<div class="flow-node action">✓ Mark success</div>
<div class="flow-node action">📧 Email: all records updated</div>
</div>
</div>
</div>
<div class="flow-branch-col">
<div class="branch-label">All correct</div>
<div class="flow-node action">✓ All records verified</div>
<div class="flow-node action">📧 Email: success — all records updated</div>
</div>
</div>
<div class="flow-node">💾 Write new WAN IP to cache file</div>
</div>
<div class="flow-branch-col">
<div class="branch-label">IP unchanged + not daily window</div>
<div class="flow-node silent">💾 Overwrite cache with same IP (refreshes timestamp)</div>
<div class="flow-node silent">🔇 Exit silently — no API calls, no email</div>
</div>
</div>
</div>
<div class="callout amber" style="margin-top:1rem;">
<div class="clue-label">⚠ DreamHost API — how updates work</div>
<p>DreamHost has no "update" DNS command. To change an A record you must <strong>remove the old value then add the new one</strong>. The script always fetches the live DreamHost value immediately before each remove/add pair so it never tries to remove a value that isn't there.</p>
</div>
</div>
<!-- ===== SECTION 3 — DOMAINS & RECORDS ================================== -->
<div class="section">
<div class="section-title" data-num="3">Domains &amp; records managed</div>
<table>
<thead><tr><th>Record (DreamHost "record" field)</th><th>Zone / Domain</th><th>Type</th></tr></thead>
<tbody>
<tr><td><code>albranch.org</code></td><td>albranch.org</td><td>A (root / @)</td></tr>
<tr><td><code>www.albranch.org</code></td><td>albranch.org</td><td>A</td></tr>
<tr><td><code>indigorhayne.com</code></td><td>indigorhayne.com</td><td>A (root / @)</td></tr>
<tr><td><code>www.indigorhayne.com</code></td><td>indigorhayne.com</td><td>A</td></tr>
<tr><td><code>rufusking.com</code></td><td>rufusking.com</td><td>A (root / @)</td></tr>
<tr><td><code>www.rufusking.com</code></td><td>rufusking.com</td><td>A</td></tr>
<tr><td><code>kingdezigns.com</code></td><td>kingdezigns.com</td><td>A (root / @)</td></tr>
<tr><td><code>www.kingdezigns.com</code></td><td>kingdezigns.com</td><td>A</td></tr>
<tr><td><code>*.web.kingdezigns.com</code></td><td>kingdezigns.com</td><td>A (wildcard)</td></tr>
<tr><td><code>cloud.kingdezigns.com</code></td><td>kingdezigns.com</td><td>A</td></tr>
<tr><td><code>files08.kingdezigns.com</code></td><td>kingdezigns.com</td><td>A</td></tr>
<tr><td><code>files16.kingdezigns.com</code></td><td>kingdezigns.com</td><td>A</td></tr>
<tr><td><code>has.kingdezigns.com</code></td><td>kingdezigns.com</td><td>A</td></tr>
<tr><td><code>nginx.kingdezigns.com</code></td><td>kingdezigns.com</td><td>A</td></tr>
<tr><td><code>omv08.kingdezigns.com</code></td><td>kingdezigns.com</td><td>A</td></tr>
<tr><td><code>omv16.kingdezigns.com</code></td><td>kingdezigns.com</td><td>A</td></tr>
<tr><td><code>pihole.kingdezigns.com</code></td><td>kingdezigns.com</td><td>A</td></tr>
<tr><td><code>plex.kingdezigns.com</code></td><td>kingdezigns.com</td><td>A</td></tr>
<tr><td><code>vault.kingdezigns.com</code></td><td>kingdezigns.com</td><td>A</td></tr>
<tr><td><code>webmin.kingdezigns.com</code></td><td>kingdezigns.com</td><td>A</td></tr>
</tbody>
</table>
</div>
<!-- ===== SECTION 4 — DEPLOYMENT STEPS ================================== -->
<div class="section">
<div class="section-title" data-num="4">Deployment — step by step</div>
<div class="step">
<div class="step-header"><span class="step-num">1</span> Install dependencies</div>
<pre>sudo apt-get update
sudo apt-get install -y curl python3 mailutils</pre>
<p class="why"><code>curl</code> fetches WAN IP and calls the DreamHost API. <code>python3</code> parses the JSON responses. <code>mailutils</code> provides the <code>mail</code> command for sending HTML notifications.</p>
</div>
<div class="step">
<div class="step-header"><span class="step-num">2</span> Copy the script to its permanent location</div>
<pre>sudo cp ddns_update.sh /usr/local/sbin/ddns_update.sh</pre>
<p class="result neutral">This puts it in a root-only system bin directory alongside other admin utilities.</p>
</div>
<div class="step">
<div class="step-header"><span class="step-num">3</span> Set ownership and permissions</div>
<pre>sudo chown root:root /usr/local/sbin/ddns_update.sh
sudo chmod 700 /usr/local/sbin/ddns_update.sh</pre>
<p class="why">The script reads/writes to <code>/var/log/ddns/</code> and calls sendmail — it must run as root. Mode 700 prevents other users from reading the embedded API key.</p>
</div>
<div class="step">
<div class="step-header"><span class="step-num">4</span> Create the log directory</div>
<pre>sudo mkdir -p /var/log/ddns
sudo chmod 700 /var/log/ddns</pre>
<p class="why">The script will create this automatically on first run, but creating it manually with 700 ensures no other user can read the cached IP or logs before then.</p>
</div>
<div class="step">
<div class="step-header"><span class="step-num">5</span> Test a manual run</div>
<pre>sudo /usr/local/sbin/ddns_update.sh</pre>
<p class="why">Run once by hand to verify API connectivity, confirm all 20 records are found, and check that the email arrives at rufus.king@kingdezigns.com. Watch the output — it prints everything it does.</p>
<p class="result neutral">On the very first run there is no cached IP, so it will treat every record as new and perform a full update cycle.</p>
</div>
<div class="step">
<div class="step-header"><span class="step-num">6</span> Verify the log file after test run</div>
<pre>sudo cat /var/log/ddns/ddns_run.log</pre>
<p class="result neutral">You should see the WAN IP detected, DreamHost responses for each record, and "DDNS run complete" at the bottom.</p>
</div>
<div class="step">
<div class="step-header"><span class="step-num">7</span> Add OMV Scheduled Task</div>
<pre>OMV → System → Scheduled Tasks → + (Add)
Enabled: ✓
Minute: */15
Hour: *
Day: *
Month: *
Weekday: *
User: root
Command: /usr/local/sbin/ddns_update.sh</pre>
<p class="why">This fires the script every 15 minutes around the clock. OMV uses standard cron under the hood — you can verify with <code>sudo crontab -l</code> after saving.</p>
<p class="result pass">Script is now live and fully automated.</p>
</div>
</div>
<!-- ===== SECTION 5 — SCRIPT & FILES ===================================== -->
<div class="section">
<div class="section-title" data-num="5">Script &amp; files created</div>
<div class="script-box">
<div class="script-row">
<div class="script-label">Script location</div>
<pre>/usr/local/sbin/ddns_update.sh</pre>
</div>
<div class="script-row">
<div class="script-label">IP cache (previous WAN IP)</div>
<pre>/var/log/ddns/last_wan_ip.txt</pre>
</div>
<div class="script-row">
<div class="script-label">Run log (last 5000 lines, auto-trimmed)</div>
<pre>/var/log/ddns/ddns_run.log</pre>
</div>
<div class="script-row">
<div class="script-label">Notification email</div>
<pre>rufus.king@kingdezigns.com</pre>
</div>
<div class="script-row">
<div class="script-label">Manual run command</div>
<pre>sudo /usr/local/sbin/ddns_update.sh</pre>
</div>
<div class="script-row">
<div class="script-label">Force a fresh update (clears cache)</div>
<pre>sudo rm /var/log/ddns/last_wan_ip.txt &amp;&amp; sudo /usr/local/sbin/ddns_update.sh</pre>
</div>
<div class="script-row">
<div class="script-label">Watch the live log</div>
<pre>sudo tail -f /var/log/ddns/ddns_run.log</pre>
</div>
</div>
</div>
<!-- ===== SECTION 6 — EMAIL BEHAVIOR ===================================== -->
<div class="section">
<div class="section-title" data-num="6">Email notification behavior</div>
<table>
<thead><tr><th>Scenario</th><th>Email sent?</th><th>Subject line</th></tr></thead>
<tbody>
<tr>
<td>IP unchanged, not daily window</td>
<td><span class="badge">No</span></td>
<td>Silent exit — no email</td>
</tr>
<tr>
<td>IP changed, all records updated successfully</td>
<td><span class="badge green">Yes</span></td>
<td>[DDNS] ✅ All Records Updated — &lt;new IP&gt;</td>
</tr>
<tr>
<td>IP changed, some records failed after 2 attempts</td>
<td><span class="badge red">Yes — action needed</span></td>
<td>[DDNS] ⚠️ PARTIAL FAILURE — Manual action needed — &lt;new IP&gt;</td>
</tr>
<tr>
<td>Daily sanity check (00:00–00:14), all records match</td>
<td><span class="badge green">Yes</span></td>
<td>[DDNS] ✅ All Records Updated — &lt;IP&gt;</td>
</tr>
<tr>
<td>Daily sanity check, mismatch found &amp; fixed</td>
<td><span class="badge green">Yes</span></td>
<td>[DDNS] ✅ All Records Updated — &lt;new IP&gt;</td>
</tr>
<tr>
<td>WAN IP discovery fails (all 3 sources down)</td>
<td><span class="badge">No</span></td>
<td>Script aborts — will retry next 15-min run</td>
</tr>
</tbody>
</table>
<div class="callout green" style="margin-top:0.75rem;">
<p>All emails are sent as <strong>HTML</strong> using the KingDezigns branded template — same typography and color system as this document. They include a record-by-record results table and, on failure, a direct link to the DreamHost DNS panel with the correct IP pre-noted for manual entry.</p>
</div>
</div>
<!-- ===== SECTION 7 — IMPORTANT NOTES =================================== -->
<div class="section">
<div class="section-title" data-num="7">Important notes &amp; gotchas</div>
<table>
<thead><tr><th>Item</th><th>Detail</th></tr></thead>
<tbody>
<tr>
<td>API key is embedded in the script</td>
<td>The script is chmod 700 / root only. Do not copy or share the file without scrubbing the key first.</td>
</tr>
<tr>
<td>DreamHost has no "update" API command</td>
<td>Every update is a remove + add pair. The script always reads the live DreamHost value first so it never removes the wrong IP.</td>
</tr>
<tr>
<td>Wildcard record <code>*.web.kingdezigns.com</code></td>
<td>DreamHost supports wildcard A records via API. The script passes the literal string <code>*.web.kingdezigns.com</code> to <code>dns-list_records</code> and it will match correctly.</td>
</tr>
<tr>
<td>Mail must be configured on NAS16</td>
<td>Run <code>sudo apt-get install mailutils</code> and ensure your MTA (Postfix or ssmtp) is configured to relay outbound email. Test with: <code>echo "test" | mail -s "test" rufus.king@kingdezigns.com</code></td>
</tr>
<tr>
<td>First run behavior</td>
<td>On the very first run there is no cached IP. The script treats every record as needing an update and performs a full cycle. This is expected and correct.</td>
</tr>
<tr>
<td>Daily window fires once per day</td>
<td>The 00:00–00:14 window aligns with the every-15-min schedule — only one run per day falls in that window. No extra logic needed.</td>
</tr>
<tr>
<td>Log rotation</td>
<td>The run log auto-trims itself to 5000 lines on every execution. No logrotate config needed.</td>
</tr>
</tbody>
</table>
</div>
<div class="footer">
<span>KingDezigns Home Network</span>
<span>Generated May 19, 2025 — DDNS Updater build session</span>
</div>
</div>
</body>
</html>

662
ddns/ddns_update.sh Normal file
View file

@ -0,0 +1,662 @@
#!/bin/bash
# =============================================================================
# ddns_update.sh — DreamHost Dynamic DNS Updater (v2 — hardened 2026-07-23)
# KingDezigns Home Network — NAS16
#
# Schedule every 15 minutes via OMV Scheduled Tasks:
# Minute: */15 Hour/Day/Month/Weekday: *
# Command: /usr/scripts/ddns/ddns_update.sh
#
# Manages A records for:
# albranch.org, indigorhayne.com, rufusking.com, kingdezigns.com (+subs)
#
# v2 CHANGES (root cause: 2026-07-23 DreamHost API rate-limit outage — see
# KingDezigns troubleshooting summary for full diagnosis):
# 1. Zone is fetched ONCE per run (1 API call) instead of once per record
# (was 23 calls just to check status). All checks are done locally
# against that snapshot.
# 2. Records are ADDED before the old value is REMOVED (was remove-then-add,
# which left a domain with NO A record at all if the add step failed).
# 3. A sleep is inserted between each record's mutating calls to stay well
# under DreamHost's documented add/remove throttling.
# 4. A max number of records are mutated per run (BATCH_LIMIT). Anything
# left over is written to a pending file and picked up automatically on
# the next 15-min run — no need to wait for the old remove-then-add loop
# to hammer all 23 records in one shot.
# 5. If several records in a row come back with an API error, the run
# aborts immediately (fail-fast) instead of grinding through the rest —
# that's the signature of a rate limit, and keeps retrying makes it worse.
# 6. The real DreamHost error reason (the "data" field) is captured and
# logged/emailed, not just the generic "error" result.
# 7. curl now uses --data-urlencode for every parameter instead of manual
# string concatenation (safer for records like *.web.kingdezigns.com).
#
# Behavior:
# Every 15-min run, an update pass happens if ANY of the following is true:
# - WAN IP changed since last cached value
# - Daily sanity-check window (00:00–00:14)
# - There are records left over (pending) from a previous run's batch
# limit or a rate-limit abort
# Each pass re-fetches the live zone once and compares it locally — so a
# "pending" record that was fixed manually (or by a later successful
# attempt) will simply drop out on its own.
# =============================================================================
# ── Configuration ─────────────────────────────────────────────────────────────
HOSTNAME_LABEL=$(hostname -s 2>/dev/null || echo "NAS16")
REPORT_TO="rufus.king@kingdezigns.com"
REPORT_FROM="ddns-monitor@nas16.local" # Display only — Postfix uses your Gmail relay
API_KEY="4ACNN8PRYNQCMCWW"
LOG_FILE="/var/log/ddns/ddns_run.log"
IP_CACHE="/var/log/ddns/last_wan_ip.txt"
PENDING_FILE="/var/log/ddns/pending_records.txt" # record<TAB>failed_attempt_count
# Rate-limit hardening knobs — tuned to stay well under DreamHost's documented
# dns-add_record throttle (~10 successful calls / 5 min, ~50 / hour).
BATCH_LIMIT=8 # max records mutated (add+remove) per script run
SLEEP_BETWEEN_RECORDS=25 # seconds between each record's add/remove pair
FAIL_FAST_THRESHOLD=3 # abort the run after this many consecutive API errors
STALE_THRESHOLD=6 # attempts before a pending record is flagged "stale" in the email
# Note: Email sent via OMV-configured Postfix (Gmail relay). No SMTP config needed here.
# ──────────────────────────────────────────────────────────────────────────────
RECORDS=(
"albranch.org"
"www.albranch.org"
"indigorhayne.com"
"www.indigorhayne.com"
"rufusking.com"
"www.rufusking.com"
"kingdezigns.com"
"www.kingdezigns.com"
"*.web.kingdezigns.com"
"cloud.kingdezigns.com"
"files08.kingdezigns.com"
"files16.kingdezigns.com"
"has.kingdezigns.com"
"nginx.kingdezigns.com"
"omv08.kingdezigns.com"
"omv16.kingdezigns.com"
"pihole.kingdezigns.com"
"plex.kingdezigns.com"
"vault.kingdezigns.com"
"webmin.kingdezigns.com"
"office.kingdezigns.com"
"stocks.kingdezigns.com"
"collabora.kingdezigns.com"
)
DH_API="https://api.dreamhost.com/"
REPORT_TIME=$(date "+%Y-%m-%d %H:%M:%S")
mkdir -p /var/log/ddns
# Trim log to last 5000 lines
if [[ -f "$LOG_FILE" ]]; then
tail -n 5000 "$LOG_FILE" > "${LOG_FILE}.tmp" && mv "${LOG_FILE}.tmp" "$LOG_FILE"
fi
log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a "$LOG_FILE"; }
log "===== DDNS Update started on $HOSTNAME_LABEL (v2 hardened) ====="
# ── Helpers ───────────────────────────────────────────────────────────────────
unique_id() {
cat /proc/sys/kernel/random/uuid 2>/dev/null \
|| python3 -c "import uuid; print(uuid.uuid4())" 2>/dev/null \
|| echo "$(date +%s%N)-$$"
}
get_wan_ip() {
local ip
for src in "https://api.ipify.org" "https://ifconfig.me/ip" "https://icanhazip.com"; do
ip=$(curl -s --max-time 8 "$src" 2>/dev/null | tr -d '[:space:]')
if [[ "$ip" =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ ]]; then
echo "$ip"; return 0
fi
done
return 1
}
# Fetches the ENTIRE DreamHost DNS zone in a single API call and prints one
# "record<TAB>value" line per A record on stdout. On any failure (including a
# throttled/erroring API), prints a reason to stderr and returns non-zero —
# callers must NOT treat that as "record doesn't exist".
fetch_zone_snapshot() {
curl -s --max-time 20 -G "$DH_API" \
--data-urlencode "key=${API_KEY}" \
--data-urlencode "cmd=dns-list_records" \
--data-urlencode "format=json" \
--data-urlencode "unique_id=$(unique_id)" \
| python3 -c "
import sys, json
try:
data = json.load(sys.stdin)
if data.get('result') != 'success':
sys.stderr.write('LIST_ERROR: ' + str(data.get('data', 'unknown')) + '\n')
sys.exit(2)
for e in data.get('data', []):
if e.get('type', '') == 'A':
sys.stdout.write(e.get('record', '') + '\t' + e.get('value', '') + '\n')
except Exception as ex:
sys.stderr.write('LIST_EXCEPTION: ' + str(ex) + '\n')
sys.exit(2)
"
}
# Both print "result<TAB>data" on stdout — always something printable, so a
# JSON/network failure shows up as an explicit "exception" row in the log
# instead of silently looking like an empty/unknown result.
dh_add_record() {
local record="$1" ip="$2"
curl -s --max-time 15 -G "$DH_API" \
--data-urlencode "key=${API_KEY}" \
--data-urlencode "cmd=dns-add_record" \
--data-urlencode "format=json" \
--data-urlencode "unique_id=$(unique_id)" \
--data-urlencode "record=${record}" \
--data-urlencode "type=A" \
--data-urlencode "value=${ip}" \
| python3 -c "
import sys, json
try:
d = json.load(sys.stdin)
print(str(d.get('result', 'unknown')) + '\t' + str(d.get('data', '')))
except Exception as ex:
print('exception\t' + str(ex))
" 2>/dev/null
}
dh_remove_record() {
local record="$1" ip="$2"
curl -s --max-time 15 -G "$DH_API" \
--data-urlencode "key=${API_KEY}" \
--data-urlencode "cmd=dns-remove_record" \
--data-urlencode "format=json" \
--data-urlencode "unique_id=$(unique_id)" \
--data-urlencode "record=${record}" \
--data-urlencode "type=A" \
--data-urlencode "value=${ip}" \
| python3 -c "
import sys, json
try:
d = json.load(sys.stdin)
print(str(d.get('result', 'unknown')) + '\t' + str(d.get('data', '')))
except Exception as ex:
print('exception\t' + str(ex))
" 2>/dev/null
}
# ── Get WAN IP ────────────────────────────────────────────────────────────────
log "Checking current WAN IP..."
if ! WAN_IP=$(get_wan_ip); then
log "ERROR: All WAN IP sources failed. Aborting — will retry next run."
exit 1
fi
log "Current WAN IP: $WAN_IP"
PREV_IP=""
[[ -f "$IP_CACHE" ]] && PREV_IP=$(tr -d '[:space:]' < "$IP_CACHE")
log "Cached previous IP: ${PREV_IP:-<none>}"
# ── Load pending records left over from a previous run ────────────────────────
declare -A PENDING_ATTEMPTS
PENDING_COUNT=0
if [[ -f "$PENDING_FILE" ]]; then
while IFS=$'\t' read -r p_rec p_attempts; do
[[ -z "$p_rec" ]] && continue
PENDING_ATTEMPTS["${p_rec,,}"]="${p_attempts:-0}"
(( PENDING_COUNT++ ))
done < "$PENDING_FILE"
fi
[[ $PENDING_COUNT -gt 0 ]] && log "Found $PENDING_COUNT record(s) pending from a previous run."
# ── Daily window check (00:00–00:14) ─────────────────────────────────────────
HOUR=$(date '+%H')
MINUTE=$(date '+%M')
IN_DAILY_WINDOW=false
if [[ "$HOUR" == "00" && 10#$MINUTE -lt 15 ]]; then
IN_DAILY_WINDOW=true
log "Daily sanity-check window active (00:00–00:14)."
fi
# ── Decide whether to run ─────────────────────────────────────────────────────
DO_UPDATE=false
TRIGGER_REASON=""
if [[ "$WAN_IP" != "$PREV_IP" ]]; then
DO_UPDATE=true
TRIGGER_REASON="WAN IP changed (${PREV_IP:-none} → ${WAN_IP})"
elif [[ "$IN_DAILY_WINDOW" == "true" ]]; then
DO_UPDATE=true
TRIGGER_REASON="Daily sanity-check (IP unchanged at ${WAN_IP})"
elif [[ $PENDING_COUNT -gt 0 ]]; then
DO_UPDATE=true
TRIGGER_REASON="Retrying $PENDING_COUNT pending record(s) from a previous run"
fi
if [[ "$DO_UPDATE" == "false" ]]; then
log "IP unchanged ($WAN_IP) and no pending records. No action needed — exiting silently."
echo "$WAN_IP" > "$IP_CACHE"
log "===== DDNS Update complete (no change) on $HOSTNAME_LABEL ====="
exit 0
fi
log "Trigger: $TRIGGER_REASON"
# ── Fetch the full zone ONCE ──────────────────────────────────────────────────
log "Fetching current DNS zone from DreamHost (single API call)..."
ZONE_ERR_FILE=$(mktemp)
ZONE_RAW=$(fetch_zone_snapshot 2> "$ZONE_ERR_FILE")
ZONE_RC=$?
if [[ $ZONE_RC -ne 0 ]]; then
ZONE_ERR=$(cat "$ZONE_ERR_FILE")
rm -f "$ZONE_ERR_FILE"
log "ERROR: could not fetch DNS zone snapshot ($ZONE_ERR) — likely DreamHost API throttling/outage."
log "Not touching pending state or the IP cache — will retry in full on the next scheduled run."
log "===== DDNS Update aborted on $HOSTNAME_LABEL ====="
# Lightweight failure alert — deliberately not the full report template.
{
echo "To: $REPORT_TO"
echo "From: $REPORT_FROM"
echo "Subject: [DDNS] $HOSTNAME_LABEL — API UNREACHABLE | $REPORT_TIME"
echo "Content-Type: text/plain; charset=UTF-8"
echo ""
echo "DDNS update on $HOSTNAME_LABEL could not even fetch the DNS zone from DreamHost."
echo "Reason: $ZONE_ERR"
echo ""
echo "This usually means the DreamHost API is rate-limiting this key right now."
echo "No changes were attempted. The script will retry automatically on the next"
echo "15-minute run — no action needed unless this repeats for more than an hour."
} | sendmail -t
exit 1
fi
rm -f "$ZONE_ERR_FILE"
declare -A ZONE
ZONE_RECORD_COUNT=0
while IFS=$'\t' read -r z_rec z_val; do
[[ -z "$z_rec" ]] && continue
ZONE["${z_rec,,}"]="$z_val"
(( ZONE_RECORD_COUNT++ ))
done <<< "$ZONE_RAW"
log "Zone snapshot retrieved — $ZONE_RECORD_COUNT A record(s) found on DreamHost."
# ── Determine which records need updating (all local — no API calls) ─────────
RECORDS_OK=()
NEEDS_UPDATE=()
for rec in "${RECORDS[@]}"; do
cur_ip="${ZONE[${rec,,}]:-}"
if [[ "$cur_ip" == "$WAN_IP" ]]; then
RECORDS_OK+=("$rec")
else
log " [$rec] needs update — DreamHost shows '${cur_ip:-<not found>}'"
NEEDS_UPDATE+=("$rec")
fi
done
log "Checked all ${#RECORDS[@]} records against target IP $WAN_IP — ${#RECORDS_OK[@]} already correct, ${#NEEDS_UPDATE[@]} need updating."
RECORDS_FAILED=()
RECORDS_DEFERRED=()
ATTEMPTED_COUNT=0
RATE_LIMIT_ABORTED=false
if [[ ${#NEEDS_UPDATE[@]} -eq 0 ]]; then
log "Nothing to update."
rm -f "$PENDING_FILE"
else
# Split into this run's batch vs. overflow (batch-limit deferral)
THIS_RUN_BATCH=("${NEEDS_UPDATE[@]:0:$BATCH_LIMIT}")
OVERFLOW=("${NEEDS_UPDATE[@]:$BATCH_LIMIT}")
if [[ ${#OVERFLOW[@]} -gt 0 ]]; then
log "Batch limit is $BATCH_LIMIT records/run — deferring ${#OVERFLOW[@]} record(s) to the next run: ${OVERFLOW[*]}"
RECORDS_DEFERRED+=("${OVERFLOW[@]}")
fi
CONSECUTIVE_ERRORS=0
FIRST=true
for rec in "${THIS_RUN_BATCH[@]}"; do
if [[ "$RATE_LIMIT_ABORTED" == "true" ]]; then
RECORDS_DEFERRED+=("$rec")
continue
fi
if [[ "$FIRST" == "false" ]]; then
sleep "$SLEEP_BETWEEN_RECORDS"
fi
FIRST=false
old_ip="${ZONE[${rec,,}]:-}"
log " Updating [$rec] ${old_ip:-<not found>} → $WAN_IP"
IFS=$'\t' read -r add_result add_data <<< "$(dh_add_record "$rec" "$WAN_IP")"
log " add: $add_result${add_data:+ ($add_data)}"
(( ATTEMPTED_COUNT++ ))
if [[ "$add_result" == "success" ]]; then
RECORDS_OK+=("$rec")
CONSECUTIVE_ERRORS=0
# Clean up the stale old value now that the new one is live.
if [[ -n "$old_ip" && "$old_ip" != "$WAN_IP" ]]; then
IFS=$'\t' read -r rm_result rm_data <<< "$(dh_remove_record "$rec" "$old_ip")"
log " remove: $rm_result${rm_data:+ ($rm_data)} (cleanup of old value $old_ip)"
if [[ "$rm_result" != "success" ]]; then
log " NOTE: old A record ($old_ip) for [$rec] could not be removed — DNS may briefly round-robin between old and new IP until a future cleanup pass."
fi
fi
else
log " WARNING: update failed for [$rec] — $add_data"
RECORDS_FAILED+=("${rec}::${add_data}")
(( CONSECUTIVE_ERRORS++ ))
if [[ $CONSECUTIVE_ERRORS -ge $FAIL_FAST_THRESHOLD ]]; then
log "WARNING: $FAIL_FAST_THRESHOLD consecutive API errors — this looks like DreamHost rate-limiting. Aborting remaining updates this run rather than making it worse."
RATE_LIMIT_ABORTED=true
fi
fi
done
fi
# ── Persist pending state for next run ────────────────────────────────────────
declare -A NEW_PENDING
for rec in "${RECORDS_DEFERRED[@]}"; do
key="${rec,,}"
NEW_PENDING["$key"]="${PENDING_ATTEMPTS[$key]:-0}" # not actually attempted — don't increment
done
for entry in "${RECORDS_FAILED[@]}"; do
rec="${entry%%::*}"
key="${rec,,}"
prev="${PENDING_ATTEMPTS[$key]:-0}"
NEW_PENDING["$key"]=$(( prev + 1 ))
done
STALE_RECORDS=()
if [[ ${#NEW_PENDING[@]} -gt 0 ]]; then
: > "$PENDING_FILE"
for key in "${!NEW_PENDING[@]}"; do
echo -e "${key}\t${NEW_PENDING[$key]}" >> "$PENDING_FILE"
if [[ ${NEW_PENDING[$key]} -ge $STALE_THRESHOLD ]]; then
STALE_RECORDS+=("$key (${NEW_PENDING[$key]} failed attempts)")
fi
done
else
rm -f "$PENDING_FILE"
fi
# ── Persist WAN IP cache ───────────────────────────────────────────────────────
echo "$WAN_IP" > "$IP_CACHE"
log "IP cache updated to $WAN_IP"
log "Update complete — OK: ${#RECORDS_OK[@]} Failed: ${#RECORDS_FAILED[@]} Deferred: ${#RECORDS_DEFERRED[@]} API calls used: $(( ZONE_RECORD_COUNT >= 0 ? 1 : 1 ))+$(( ATTEMPTED_COUNT * 2 ))"
# ── Skip email entirely if there was truly nothing to report ─────────────────
if [[ ${#NEEDS_UPDATE[@]} -eq 0 ]]; then
log "===== DDNS Update complete (all records already correct) on $HOSTNAME_LABEL ====="
exit 0
fi
# ── Determine overall status ──────────────────────────────────────────────────
if [[ ${#RECORDS_FAILED[@]} -eq 0 && ${#RECORDS_DEFERRED[@]} -eq 0 ]]; then
OVERALL_STATUS="SUCCESS"
elif [[ ${#RECORDS_FAILED[@]} -eq 0 && ${#RECORDS_DEFERRED[@]} -gt 0 ]]; then
OVERALL_STATUS="DEFERRED"
elif [[ ${#RECORDS_FAILED[@]} -gt 0 && ( $ATTEMPTED_COUNT -gt ${#RECORDS_FAILED[@]} || ${#RECORDS_DEFERRED[@]} -gt 0 ) ]]; then
OVERALL_STATUS="PARTIAL"
else
OVERALL_STATUS="FAILED"
fi
# ── Build HTML email — matching KingDezigns email style guide ────────────────
case "$OVERALL_STATUS" in
SUCCESS)
BADGE_HTML="<div style=\"background:#1a7f4b;color:white;padding:10px 20px;border-radius:8px;font-size:14px;font-weight:700;\">✅ ALL UPDATED</div>"
BANNER_HTML="<tr><td style=\"background:#1a7f4b;padding:12px 32px;\"><span style=\"color:white;font-size:14px;font-weight:600;\">✅ &nbsp;All A records updated successfully — no action required.</span></td></tr>"
;;
DEFERRED)
BADGE_HTML="<div style=\"background:#b45309;color:white;padding:10px 20px;border-radius:8px;font-size:14px;font-weight:700;\">🕓 BATCHED</div>"
BANNER_HTML="<tr><td style=\"background:#b45309;padding:12px 32px;\"><span style=\"color:white;font-size:14px;font-weight:600;\">🕓 &nbsp;Some records queued for the next run (rate-limit batching) — no errors, no action required.</span></td></tr>"
;;
PARTIAL)
BADGE_HTML="<div style=\"background:#b45309;color:white;padding:10px 20px;border-radius:8px;font-size:14px;font-weight:700;\">⚠️ PARTIAL</div>"
BANNER_HTML="<tr><td style=\"background:#b45309;padding:12px 32px;\"><span style=\"color:white;font-size:14px;font-weight:600;\">⚠️ &nbsp;Some records failed to update — they will be retried automatically.</span></td></tr>"
;;
FAILED)
BADGE_HTML="<div style=\"background:#b91c1c;color:white;padding:10px 20px;border-radius:8px;font-size:14px;font-weight:700;\">🚨 FAILED</div>"
BANNER_HTML="<tr><td style=\"background:#b91c1c;padding:12px 32px;\"><span style=\"color:white;font-size:14px;font-weight:600;\">🚨 &nbsp;Updates failed — likely DreamHost rate-limiting. Will retry automatically; check if this persists past an hour.</span></td></tr>"
;;
esac
# Record rows: OK
RECORD_ROWS_HTML=""
for rec in "${RECORDS_OK[@]}"; do
RECORD_ROWS_HTML+="
<tr>
<td style=\"padding:9px 14px;border-bottom:1px solid #f3f4f6;font-family:monospace;font-size:13px;color:#111827;\">$rec</td>
<td style=\"padding:9px 14px;border-bottom:1px solid #f3f4f6;\"><span style=\"background:#d1fae5;color:#065f46;padding:3px 10px;border-radius:20px;font-size:11px;font-weight:700;\">✓ UPDATED</span></td>
<td style=\"padding:9px 14px;border-bottom:1px solid #f3f4f6;font-family:monospace;font-size:13px;color:#1a7f4b;font-weight:600;\">$WAN_IP</td>
</tr>"
done
# Record rows: FAILED (with real DreamHost reason)
for entry in "${RECORDS_FAILED[@]}"; do
rec="${entry%%::*}"
reason="${entry#*::}"
RECORD_ROWS_HTML+="
<tr>
<td style=\"padding:9px 14px;border-bottom:1px solid #f3f4f6;font-family:monospace;font-size:13px;color:#111827;\">$rec</td>
<td style=\"padding:9px 14px;border-bottom:1px solid #f3f4f6;\"><span style=\"background:#fee2e2;color:#991b1b;padding:3px 10px;border-radius:20px;font-size:11px;font-weight:700;\">✗ FAILED</span></td>
<td style=\"padding:9px 14px;border-bottom:1px solid #f3f4f6;font-family:monospace;font-size:12px;color:#b91c1c;font-weight:600;\">$reason</td>
</tr>"
done
# Record rows: DEFERRED (queued for next run — not a failure)
for rec in "${RECORDS_DEFERRED[@]}"; do
RECORD_ROWS_HTML+="
<tr>
<td style=\"padding:9px 14px;border-bottom:1px solid #f3f4f6;font-family:monospace;font-size:13px;color:#111827;\">$rec</td>
<td style=\"padding:9px 14px;border-bottom:1px solid #f3f4f6;\"><span style=\"background:#fef3c7;color:#92400e;padding:3px 10px;border-radius:20px;font-size:11px;font-weight:700;\">🕓 QUEUED</span></td>
<td style=\"padding:9px 14px;border-bottom:1px solid #f3f4f6;font-family:monospace;font-size:12px;color:#92400e;\">retrying next run</td>
</tr>"
done
# Manual action block — only shown when there are real (non-deferred) failures
MANUAL_BLOCK=""
if [[ ${#RECORDS_FAILED[@]} -gt 0 ]]; then
FAILED_LIST_HTML=""
for entry in "${RECORDS_FAILED[@]}"; do
rec="${entry%%::*}"
reason="${entry#*::}"
FAILED_LIST_HTML+="<div style=\"font-family:monospace;font-size:13px;padding:3px 0;\">• $rec <span style=\"color:#9a5b1f;\">($reason)</span></div>"
done
MANUAL_BLOCK="
<tr><td style=\"padding:0 24px 24px;\">
<div style=\"background:#fff7ed;border:1px solid #fed7aa;border-radius:10px;padding:20px 22px;\">
<div style=\"font-size:13px;font-weight:700;color:#9a3412;margin-bottom:12px;text-transform:uppercase;letter-spacing:.5px;\">⚠️ Failed Records — Will Auto-Retry</div>
<p style=\"font-size:13px;color:#7c3516;margin:0 0 10px;\">These records did not update this run. They stay queued and will be retried automatically on the next 15-minute run — no manual action needed unless they keep failing for more than an hour.</p>
<div style=\"background:#fff;border:1px solid #fed7aa;border-radius:6px;padding:12px 16px;margin-bottom:12px;\">
$FAILED_LIST_HTML
</div>
<table cellspacing=\"0\" cellpadding=\"0\" width=\"100%\">
<tr>
<td style=\"font-size:13px;color:#7c3516;\">Target A record value</td>
<td style=\"font-family:monospace;font-size:15px;font-weight:700;color:#b91c1c;\">$WAN_IP</td>
</tr>
<tr><td colspan=\"2\" style=\"height:8px;\"></td></tr>
<tr>
<td style=\"font-size:13px;color:#7c3516;\">DreamHost DNS Panel (manual fallback)</td>
<td><a href=\"https://panel.dreamhost.com/index.cgi?tree=domain.dashboard\" style=\"color:#b45309;font-size:13px;\">panel.dreamhost.com → Manage Websites → DNS</a></td>
</tr>
</table>
</div>
</td></tr>"
fi
# Stale block — records that have failed repeatedly across multiple runs
STALE_BLOCK=""
if [[ ${#STALE_RECORDS[@]} -gt 0 ]]; then
STALE_LIST_HTML=""
for s in "${STALE_RECORDS[@]}"; do
STALE_LIST_HTML+="<div style=\"font-family:monospace;font-size:13px;padding:3px 0;\">• $s</div>"
done
STALE_BLOCK="
<tr><td style=\"padding:0 24px 24px;\">
<div style=\"background:#fff1f2;border:1px solid #f09595;border-radius:10px;padding:20px 22px;\">
<div style=\"font-size:13px;font-weight:700;color:#7a1f1f;margin-bottom:12px;text-transform:uppercase;letter-spacing:.5px;\">🔴 Stale — Failing Across Multiple Runs</div>
<p style=\"font-size:13px;color:#7a1f1f;margin:0 0 10px;\">These records have failed $STALE_THRESHOLD+ times across separate runs — this is no longer likely to be a transient rate limit. Worth checking manually.</p>
<div style=\"background:#fff;border:1px solid #f09595;border-radius:6px;padding:12px 16px;\">
$STALE_LIST_HTML
</div>
</div>
</td></tr>"
fi
# ── Compose full HTML email ───────────────────────────────────────────────────
HTML_BODY=$(cat <<EOF
<!DOCTYPE html>
<html lang="en">
<head><meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1"></head>
<body style="margin:0;padding:0;background:#f1f5f9;font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,sans-serif;">
<table width="100%" cellspacing="0" cellpadding="0" style="background:#f1f5f9;padding:30px 0;">
<tr><td align="center">
<table width="680" cellspacing="0" cellpadding="0" style="max-width:680px;width:100%;">
<!-- Header -->
<tr><td style="background:#0f172a;border-radius:10px 10px 0 0;padding:28px 32px;">
<table width="100%" cellspacing="0" cellpadding="0">
<tr>
<td>
<div style="font-size:11px;color:#94a3b8;letter-spacing:1px;text-transform:uppercase;">DreamHost Dynamic DNS Monitor</div>
<div style="font-size:26px;font-weight:800;color:white;margin-top:4px;">🌐 $HOSTNAME_LABEL</div>
<div style="font-size:13px;color:#94a3b8;margin-top:4px;">$REPORT_TIME &nbsp;|&nbsp; ${#RECORDS[@]} records monitored &nbsp;|&nbsp; Trigger: $TRIGGER_REASON</div>
</td>
<td align="right">$BADGE_HTML</td>
</tr>
</table>
</td></tr>
<!-- Status banner -->
$BANNER_HTML
<!-- Record results table -->
<tr><td style="padding:28px 24px 8px;">
<div style="background:white;border-radius:10px;box-shadow:0 1px 6px rgba(0,0,0,.10);overflow:hidden;border:1px solid #e5e7eb;">
<div style="background:#f8fafc;padding:14px 22px;border-bottom:1px solid #e5e7eb;">
<span style="font-size:13px;font-weight:700;color:#374151;text-transform:uppercase;letter-spacing:.5px;">📋 A Record Update Results</span>
<span style="margin-left:12px;font-size:12px;color:#6b7280;">${#RECORDS_OK[@]} correct &nbsp;·&nbsp; ${#RECORDS_FAILED[@]} failed &nbsp;·&nbsp; ${#RECORDS_DEFERRED[@]} queued</span>
</div>
<table width="100%" cellspacing="0" cellpadding="0">
<tr style="background:#f9fafb;">
<th style="text-align:left;padding:8px 14px;font-size:10px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;font-weight:600;border-bottom:1px solid #e5e7eb;">Record</th>
<th style="text-align:left;padding:8px 14px;font-size:10px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;font-weight:600;border-bottom:1px solid #e5e7eb;">Status</th>
<th style="text-align:left;padding:8px 14px;font-size:10px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;font-weight:600;border-bottom:1px solid #e5e7eb;">Detail</th>
</tr>
$RECORD_ROWS_HTML
</table>
</div>
</td></tr>
<!-- Failed block (only on real failures) -->
$MANUAL_BLOCK
<!-- Stale block (only on repeated multi-run failures) -->
$STALE_BLOCK
<!-- Summary card -->
<tr><td style="padding:16px 24px 28px;">
<div style="background:white;border-radius:10px;border:1px solid #e5e7eb;padding:18px 22px;">
<div style="font-size:13px;font-weight:700;color:#374151;margin-bottom:10px;text-transform:uppercase;letter-spacing:.5px;">📊 Run Summary</div>
<table width="100%" cellspacing="0" cellpadding="0">
<tr>
<td style="font-size:13px;color:#6b7280;">Host</td>
<td style="font-size:13px;color:#111827;font-weight:600;">$HOSTNAME_LABEL</td>
</tr>
<tr><td colspan="2" style="height:6px;"></td></tr>
<tr>
<td style="font-size:13px;color:#6b7280;">Run Time</td>
<td style="font-size:13px;color:#111827;font-weight:600;">$REPORT_TIME</td>
</tr>
<tr><td colspan="2" style="height:6px;"></td></tr>
<tr>
<td style="font-size:13px;color:#6b7280;">Trigger</td>
<td style="font-size:13px;color:#111827;font-weight:600;">$TRIGGER_REASON</td>
</tr>
<tr><td colspan="2" style="height:6px;"></td></tr>
<tr>
<td style="font-size:13px;color:#6b7280;">Target WAN IP</td>
<td style="font-family:monospace;font-size:14px;color:#111827;font-weight:700;">$WAN_IP</td>
</tr>
<tr><td colspan="2" style="height:6px;"></td></tr>
<tr>
<td style="font-size:13px;color:#6b7280;">Previous IP</td>
<td style="font-family:monospace;font-size:13px;color:#6b7280;">${PREV_IP:-<none cached>}</td>
</tr>
<tr><td colspan="2" style="height:6px;"></td></tr>
<tr>
<td style="font-size:13px;color:#6b7280;">Records Correct</td>
<td style="font-size:13px;color:#1a7f4b;font-weight:700;">${#RECORDS_OK[@]} / ${#RECORDS[@]}</td>
</tr>
<tr><td colspan="2" style="height:6px;"></td></tr>
<tr>
<td style="font-size:13px;color:#6b7280;">API Calls This Run</td>
<td style="font-size:13px;color:#111827;font-weight:600;">1 zone fetch + $(( ATTEMPTED_COUNT * 2 )) mutation call(s)</td>
</tr>
<tr><td colspan="2" style="height:6px;"></td></tr>
<tr>
<td style="font-size:13px;color:#6b7280;">Rate-Limit Abort Triggered</td>
<td style="font-size:13px;color:#111827;font-weight:600;">$([ "$RATE_LIMIT_ABORTED" == "true" ] && echo "Yes — stopped early" || echo "No")</td>
</tr>
<tr><td colspan="2" style="height:6px;"></td></tr>
<tr>
<td style="font-size:13px;color:#6b7280;">Overall Status</td>
<td style="font-size:13px;font-weight:700;color:$(
case "$OVERALL_STATUS" in
SUCCESS) echo "#1a7f4b" ;;
DEFERRED|PARTIAL) echo "#b45309" ;;
*) echo "#b91c1c" ;;
esac
);">$OVERALL_STATUS</td>
</tr>
</table>
</div>
</td></tr>
<!-- Footer -->
<tr><td style="background:#0f172a;border-radius:0 0 10px 10px;padding:16px 32px;text-align:center;">
<span style="font-size:11px;color:#64748b;">Automated DDNS Monitor &nbsp;|&nbsp; $HOSTNAME_LABEL &nbsp;|&nbsp; KingDezigns Infrastructure</span>
</td></tr>
</table>
</td></tr>
</table>
</body>
</html>
EOF
)
# ── Send email via OMV Postfix (Gmail relay) ──────────────────────────────────
SUBJECT="[DDNS] $HOSTNAME_LABEL — $OVERALL_STATUS | $REPORT_TIME"
{
echo "To: $REPORT_TO"
echo "From: $REPORT_FROM"
echo "Subject: $SUBJECT"
echo "MIME-Version: 1.0"
echo "Content-Type: text/html; charset=UTF-8"
echo ""
echo "$HTML_BODY"
} | sendmail -t
log "Email sent to $REPORT_TO — Overall status: $OVERALL_STATUS"
log "===== DDNS Update complete on $HOSTNAME_LABEL ====="

530
nas/NAS08-to-NAS16-Sync-Summary.html Normal file
View file

@ -0,0 +1,530 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>KingDezigns — NAS08 → NAS16 Sync Setup Summary</title>
<link rel="preconnect" href="https://fonts.googleapis.com">
<link href="https://fonts.googleapis.com/css2?family=IBM+Plex+Mono:wght@400;500&family=IBM+Plex+Sans:wght@400;500&display=swap" rel="stylesheet">
<style>
*, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
:root {
--bg: #f7f6f2;
--surface: #ffffff;
--surface-alt: #f1f0eb;
--border: #e2e0d8;
--border-strong: #c8c5ba;
--text-primary: #1a1917;
--text-secondary: #6b6960;
--text-muted: #9b9890;
--green-bg: #eaf3de;
--green-border: #97c459;
--green-text: #2d5a0e;
--red-bg: #fcebeb;
--red-border: #f09595;
--red-text: #7a1f1f;
--amber-bg: #faeeda;
--amber-border: #fac775;
--amber-text: #633806;
--coral-bg: #faece7;
--coral-border: #f0997b;
--coral-text: #4a1b0c;
--mono: 'IBM Plex Mono', monospace;
--sans: 'IBM Plex Sans', sans-serif;
}
body { font-family: var(--sans); font-size: 14px; background: var(--bg); color: var(--text-primary); line-height: 1.7; padding: 2.5rem 1.5rem; }
.page { max-width: 860px; margin: 0 auto; }
header { border-bottom: 1px solid var(--border-strong); padding-bottom: 1.25rem; margin-bottom: 2rem; }
header h1 { font-family: var(--mono); font-size: 18px; font-weight: 500; letter-spacing: -0.02em; margin-bottom: 6px; }
.meta { font-size: 12px; color: var(--text-muted); font-family: var(--mono); display: flex; gap: 16px; flex-wrap: wrap; }
.section { margin-bottom: 2.25rem; }
.section-title { font-family: var(--mono); font-size: 11px; font-weight: 500; text-transform: uppercase; letter-spacing: 0.08em; color: var(--text-muted); margin-bottom: 1rem; display: flex; align-items: center; gap: 10px; }
.section-title::before { content: attr(data-num); background: var(--text-primary); color: var(--bg); font-size: 10px; width: 18px; height: 18px; border-radius: 50%; display: inline-flex; align-items: center; justify-content: center; flex-shrink: 0; }
.section-title::after { content: ''; flex: 1; height: 1px; background: var(--border); }
p { margin-bottom: 0.75rem; color: var(--text-primary); }
p:last-child { margin-bottom: 0; }
.badge-row { display: flex; gap: 8px; flex-wrap: wrap; margin-bottom: 1rem; }
.badge { font-family: var(--mono); font-size: 11px; padding: 3px 10px; border-radius: 3px; border: 1px solid var(--border); background: var(--surface); color: var(--text-secondary); }
.badge.red { background: var(--red-bg); color: var(--red-text); border-color: var(--red-border); }
.badge.green { background: var(--green-bg); color: var(--green-text); border-color: var(--green-border); }
.badge.amber { background: var(--amber-bg); color: var(--amber-text); border-color: var(--amber-border); }
.step { background: var(--surface); border: 1px solid var(--border); border-radius: 6px; padding: 1rem 1.25rem; margin-bottom: 0.6rem; }
.step-header { font-family: var(--mono); font-size: 12px; font-weight: 500; color: var(--text-secondary); margin-bottom: 0.6rem; display: flex; align-items: center; gap: 8px; }
.step-num { background: var(--surface-alt); border: 1px solid var(--border); border-radius: 50%; width: 20px; height: 20px; display: inline-flex; align-items: center; justify-content: center; font-size: 10px; flex-shrink: 0; color: var(--text-muted); }
pre { background: var(--surface-alt); border: 1px solid var(--border); border-radius: 4px; padding: 10px 12px; font-family: var(--mono); font-size: 12px; overflow-x: auto; margin: 0.5rem 0; white-space: pre; line-height: 1.6; }
code { font-family: var(--mono); font-size: 12px; background: var(--surface-alt); padding: 1px 5px; border-radius: 3px; border: 1px solid var(--border); }
.why { font-size: 12px; color: var(--text-muted); margin-top: 0.4rem; padding-left: 2px; }
.why::before { content: '↳ why: '; font-family: var(--mono); }
.result { font-size: 12px; margin-top: 0.35rem; padding-left: 2px; }
.result.pass { color: var(--green-text); }
.result.pass::before { content: '✓ '; font-family: var(--mono); }
.result.fail { color: var(--red-text); }
.result.fail::before { content: '✗ '; font-family: var(--mono); }
.result.neutral { color: var(--text-secondary); }
.result.neutral::before { content: '→ '; font-family: var(--mono); }
.callout { border-radius: 6px; padding: 1rem 1.25rem; margin-bottom: 0.75rem; }
.callout.coral { background: var(--coral-bg); border: 1px solid var(--coral-border); }
.callout.coral p { color: var(--coral-text); }
.callout.amber { background: var(--amber-bg); border: 1px solid var(--amber-border); }
.callout.amber p { color: var(--amber-text); }
.callout.amber pre { background: #fdf5e6; border-color: var(--amber-border); }
.callout.amber code { background: #fdf5e6; border-color: var(--amber-border); color: var(--amber-text); }
.callout.green { background: var(--green-bg); border: 1px solid var(--green-border); }
.callout.green p { color: var(--green-text); }
.callout.red { background: var(--red-bg); border: 1px solid var(--red-border); }
.callout.red p { color: var(--red-text); }
.clue-label { font-family: var(--mono); font-size: 10px; font-weight: 500; text-transform: uppercase; letter-spacing: 0.07em; color: var(--amber-text); margin-bottom: 8px; }
.clue-label.red { color: var(--red-text); }
.clue-label.green { color: var(--green-text); }
.script-box { background: var(--surface); border: 1px solid var(--border); border-radius: 6px; overflow: hidden; }
.script-row { padding: 0.75rem 1.25rem; border-bottom: 1px solid var(--border); }
.script-row:last-child { border-bottom: none; }
.script-label { font-family: var(--mono); font-size: 10px; text-transform: uppercase; letter-spacing: 0.07em; color: var(--text-muted); margin-bottom: 6px; }
.script-row pre { margin: 0; }
table { width: 100%; border-collapse: collapse; font-size: 13px; background: var(--surface); border: 1px solid var(--border); border-radius: 6px; overflow: hidden; }
th { text-align: left; padding: 8px 14px; font-family: var(--mono); font-size: 10px; text-transform: uppercase; letter-spacing: 0.06em; color: var(--text-muted); background: var(--surface-alt); border-bottom: 1px solid var(--border); font-weight: 500; }
td { padding: 9px 14px; border-bottom: 1px solid var(--border); vertical-align: top; line-height: 1.5; }
tr:last-child td { border-bottom: none; }
.footer { font-family: var(--mono); font-size: 11px; color: var(--text-muted); margin-top: 2.5rem; padding-top: 1rem; border-top: 1px solid var(--border); display: flex; justify-content: space-between; flex-wrap: wrap; gap: 8px; }
.version-tag { font-family: var(--mono); font-size: 10px; background: var(--text-primary); color: var(--bg); padding: 2px 8px; border-radius: 3px; margin-left: 8px; vertical-align: middle; }
</style>
</head>
<body>
<div class="page">
<header>
<h1>KingDezigns — NAS08 → NAS16 Sync Setup Summary <span class="version-tag">v2.0</span></h1>
<div class="meta">
<span>📅 Updated May 18, 2026</span>
<span>🖥 NAS08 (192.168.150.35) → NAS16 (192.168.150.40)</span>
<span>⚙️ OpenMediaVault / Raspberry Pi OS</span>
<span>🔧 rsync + NFSv4 + HTML Email Report</span>
</div>
</header>
<!-- SECTION 1 — THE GOAL -->
<div class="section">
<div class="section-title" data-num="1">The goal</div>
<p>Mirror the <code>Public</code> folder from NAS08 to NAS16, copying only files that have changed. The sync must be lightweight enough for Raspberry Pi hardware, non-destructive (no deletes on NAS16), and scheduled via the OMV Task Scheduler. After every run, an HTML email report is sent summarizing what transferred, what folders were affected, and any errors.</p>
<div class="badge-row">
<span class="badge green">Source: NAS08 — 192.168.150.35</span>
<span class="badge green">Destination: NAS16 — 192.168.150.40</span>
<span class="badge amber">~225,000 files / ~983 GB</span>
<span class="badge">NFSv4 file share mount</span>
<span class="badge">HTML email report</span>
</div>
<p>NAS16 pulls from NAS08 (rather than NAS08 pushing) so the heavier rsync process runs on the machine with more RAM (16GB vs 8GB), leaving NAS08 as a passive file server during the sync.</p>
</div>
<!-- SECTION 2 — FOLDER PATHS -->
<div class="section">
<div class="section-title" data-num="2">Folder paths</div>
<div class="step">
<div class="step-header"><span class="step-num">S</span> Source — NAS08</div>
<pre>/mnt/kingdezigns08/Public/</pre>
<p class="result neutral">NFS share mounted on NAS16 via fstab at boot. Mount verified as non-empty before rsync runs.</p>
</div>
<div class="step">
<div class="step-header"><span class="step-num">D</span> Destination — NAS16</div>
<pre>/export/kingdezignsnas-16/Public/</pre>
<p class="result neutral">Local path on NAS16. Script verifies directory exists before writing.</p>
</div>
<div class="step">
<div class="step-header"><span class="step-num">⚠</span> NAS08-only items (copied on first run)</div>
<pre>16 - Documents/
Savings Card Forms.pdf
kingdezigns08.nas</pre>
<p class="result pass">All three copied successfully on first sync. kingdezigns08.nas included intentionally.</p>
</div>
<div class="step">
<div class="step-header"><span class="step-num">✕</span> Permanently excluded</div>
<pre>/mnt/kingdezigns08/Public/.Trash-1000/</pre>
<p class="why">System trash folder generated by NAS08's desktop environment. Never needed in backup. Excluded via <code>--exclude='.Trash-1000/'</code> in rsync flags.</p>
<p class="result pass">No longer appears in logs or emails.</p>
</div>
</div>
<!-- SECTION 3 — KEY DECISIONS -->
<div class="section">
<div class="section-title" data-num="3">Key decisions</div>
<div class="callout amber">
<div class="clue-label">Pull vs Push</div>
<p>NAS16 pulls from NAS08 via the mounted NFS share. rsync runs entirely on NAS16 to keep NAS08 load minimal — NAS08 is passive during the sync window.</p>
</div>
<div class="callout amber">
<div class="clue-label">NFSv4 File Share vs SSH</div>
<p>NFS share used instead of SSH. Drive already mounted via fstab on NAS16 — no SSH keys or remote execution needed. NFS maps users by UID. Both machines have <code>rufusking</code> as UID 1000 so permissions resolve correctly over the mount.</p>
</div>
<div class="callout amber">
<div class="clue-label">No --delete flag</div>
<p>Files on NAS16 that do not exist on NAS08 are left untouched. NAS16 is a backup, not a strict mirror. This protects NAS16-only files from accidental deletion.</p>
</div>
<div class="callout amber">
<div class="clue-label">--update flag (not --checksum)</div>
<p>rsync skips files where the NAS16 copy is <em>newer</em> than NAS08. This is lightweight (uses timestamps, not file hashing) and protects any edits made directly on NAS16.</p>
</div>
<div class="callout green">
<p><strong>Schedule:</strong> Every other day at 2:30 AM via OMV Task Scheduler. NAS08 runs its own <code>nas16-backup.sh</code> at 2:00 AM (finishes ~2:05 AM). The 25-minute buffer prevents overlap on heavy nights.</p>
</div>
</div>
<!-- SECTION 4 — SETUP STEPS -->
<div class="section">
<div class="section-title" data-num="4">Setup steps performed</div>
<div class="step">
<div class="step-header"><span class="step-num">1</span> Created script directory on NAS16</div>
<pre>sudo mkdir /usr/scripts/nas
sudo chown rufusking:users /usr/scripts/nas</pre>
<p class="why">/usr/scripts already existed. /nas subfolder created manually on NAS16. chown required after sudo mkdir — ownership defaults to root otherwise.</p>
<p class="result pass">Directory created and ownership set successfully.</p>
</div>
<div class="step">
<div class="step-header"><span class="step-num">2</span> Copied script from Linux PC to NAS16</div>
<pre>scp /path/to/nas08_to_nas16_sync.sh rufusking@192.168.150.40:/usr/scripts/nas/</pre>
<p class="result pass">File transferred successfully.</p>
</div>
<div class="step">
<div class="step-header"><span class="step-num">3</span> Set executable permission on NAS16</div>
<pre>sudo chmod +x /usr/scripts/nas/nas08_to_nas16_sync.sh</pre>
<p class="why">Run directly on NAS16 (not via remote SSH). Makes the script executable by the OMV scheduler.</p>
<p class="result pass">Permission set. Script ready to run.</p>
</div>
<div class="step">
<div class="step-header"><span class="step-num">4</span> NFS export updated on NAS08 — subtree_check → no_subtree_check</div>
<pre># Changed in OMV UI → Services → NFS → Shares (all four shares)
# Then reloaded on NAS08:
sudo exportfs -ra
# Then remounted on NAS16 to pick up the new export options:
sudo umount /mnt/kingdezigns08
sudo mount /mnt/kingdezigns08</pre>
<p class="why">subtree_check was causing intermittent Permission Denied errors on 02 - Photos, 08 - Taxes, and 16 - Documents during scheduled overnight runs. no_subtree_check is the modern recommended default for private NFS setups.</p>
<p class="result pass">All three folders now accessible. Permission errors resolved.</p>
</div>
<div class="step">
<div class="step-header"><span class="step-num">5</span> Fixed log directory ownership on NAS16</div>
<pre>sudo chown -R rufusking:users /var/log/nas-sync/</pre>
<p class="why">Log directory was created by root during the first automated OMV run. Script now self-corrects ownership at startup so this never needs to be done manually again.</p>
<p class="result pass">Log writing works cleanly. Self-healing ownership check added to script.</p>
</div>
<div class="step">
<div class="step-header"><span class="step-num">6</span> OMV Scheduler entry added on NAS16</div>
<pre>Command : /usr/scripts/nas/nas08_to_nas16_sync.sh
Schedule: 30 2 */2 * * (2:30 AM every other day)</pre>
<p class="result pass">Scheduled and confirmed running.</p>
</div>
</div>
<!-- SECTION 5 — SCRIPT DETAILS -->
<div class="section">
<div class="section-title" data-num="5">Script — final state</div>
<div class="script-box">
<div class="script-row">
<div class="script-label">Location on NAS16</div>
<pre>/usr/scripts/nas/nas08_to_nas16_sync.sh</pre>
</div>
<div class="script-row">
<div class="script-label">Log output</div>
<pre>/var/log/nas-sync/nas08_to_nas16_YYYY-MM-DD.log</pre>
</div>
<div class="script-row">
<div class="script-label">Log rotation</div>
<pre>Logs older than 30 days deleted automatically</pre>
</div>
<div class="script-row">
<div class="script-label">rsync flags</div>
<pre>--archive Preserve permissions, timestamps, symlinks, owner/group
--update Skip files where NAS16 copy is newer (protects NAS16 edits)
--human-readable Sizes in KB/MB/GB
--itemize-changes One structured line per transferred file — used for folder activity parsing
--stats Summary totals at end of each run
--exclude Excludes .Trash-1000/ from all operations</pre>
</div>
<div class="script-row">
<div class="script-label">Safety features</div>
<pre>NFS mount check Waits up to 30s for NFS share to be fully responsive before starting
Source empty check Aborts if NAS08 share appears empty — prevents accidental wipe of destination
Dest check Confirms NAS16 folder exists before writing
Smart lock file Stores PID in lock file; auto-clears stale locks if process is no longer running
Log ownership fix Self-corrects /var/log/nas-sync/ ownership on every startup
Exit code Returns rsync exit code so OMV scheduler can detect failures</pre>
</div>
<div class="script-row">
<div class="script-label">Email report sections</div>
<pre>Header/banner SUCCESS / WARNING / FAILED status with color coding
Stats tiles Files scanned, transferred, new files, data moved, duration
Source/dest info IPs, paths, transfer rate, total source size
Folder Activity Collapsed by default — folders with changes show file count,
folders with errors show the error message, no-activity runs
show "all files up to date"
Full rsync log Collapsed by default — complete raw output available on demand</pre>
</div>
<div class="script-row">
<div class="script-label">Email recipient</div>
<pre>rufus.king@kingdezigns.com</pre>
</div>
</div>
</div>
<!-- SECTION 6 — TROUBLESHOOTING LOG -->
<div class="section">
<div class="section-title" data-num="6">Troubleshooting log</div>
<div class="step">
<div class="step-header"><span class="step-num">T1</span> Permission Denied on 02 - Photos, 08 - Taxes, 16 - Documents</div>
<div class="callout red" style="margin-top:8px;">
<div class="clue-label red">Error</div>
<p>rsync: [sender] opendir "/mnt/kingdezigns08/Public/02 - Photos" failed: Permission denied (13)</p>
</div>
<p><strong>Investigation:</strong> ACLs confirmed correct on NAS08 (<code>rufusking:rwx</code> present on all affected folders and subfolders). UIDs matched on both machines (UID 1000). Folders readable manually via NAS16 terminal. Error only appeared during scheduled overnight runs.</p>
<p><strong>Root cause:</strong> <code>subtree_check</code> in the NFS export config on NAS08 was causing intermittent access failures for folders with ACL-based permissions and mixed ownership (some owned by <code>www-data</code> via Nextcloud). subtree_check forces NFS to validate every file access against the export tree — this check fails intermittently on first access after an idle period, especially overnight.</p>
<p><strong>Fix:</strong> Changed all four NFS shares on NAS08 from <code>subtree_check</code> to <code>no_subtree_check</code> via OMV UI. Reloaded exports with <code>sudo exportfs -ra</code>. Remounted share on NAS16. All three folders accessible on next run.</p>
<p class="result pass">Resolved — no Permission Denied errors since fix applied.</p>
</div>
<div class="step">
<div class="step-header"><span class="step-num">T2</span> tee: Permission Denied on log file</div>
<div class="callout red" style="margin-top:8px;">
<div class="clue-label red">Error</div>
<p>tee: /var/log/nas-sync/nas08_to_nas16_2026-05-17.log: Permission denied</p>
</div>
<p><strong>Root cause:</strong> /var/log/nas-sync/ was created by root during the first automated OMV run. Running the script manually as <code>rufusking</code> couldn't write to a root-owned directory.</p>
<p><strong>Fix:</strong> <code>sudo chown -R rufusking:users /var/log/nas-sync/</code>. Script updated to self-correct ownership on every startup — checks if the log directory owner matches the running user and auto-chowns if not.</p>
<p class="result pass">Resolved — script is now self-healing for this condition.</p>
</div>
<div class="step">
<div class="step-header"><span class="step-num">T3</span> Stale lock file blocking manual runs</div>
<div class="callout red" style="margin-top:8px;">
<div class="clue-label red">Error</div>
<p>ERROR: Lock file exists at /tmp/nas08_to_nas16.lock. Another sync may still be running. Exiting.</p>
</div>
<p><strong>Root cause:</strong> Script was killed mid-run during testing, leaving the lock file behind. Original lock logic just checked for file existence with no way to distinguish a live run from a stale one.</p>
<p><strong>Fix:</strong> Lock file now stores the script's PID. On startup, if a lock file exists the script checks whether that PID is still running. If not, it removes the stale lock and continues with a warning in the log.</p>
<p class="result pass">Resolved — stale locks are now self-clearing.</p>
</div>
<div class="step">
<div class="step-header"><span class="step-num">T4</span> Folder Activity section showing 0 updated despite files transferring</div>
<p style="margin-top:8px;"><strong>Root cause:</strong> Original folder parser relied on <code>xfr#</code> markers from rsync's <code>--progress</code> flag. When <code>--progress</code> was removed (to clean up captured output), those markers disappeared and the parser never incremented folder counts. Switching to <code>--verbose</code> didn't help because verbose output only shows directory names, not individual files.</p>
<p><strong>Fix:</strong> Replaced <code>--verbose</code> with <code>--itemize-changes</code>, which outputs a structured prefix line for every transferred file regardless of size (e.g. <code>&gt;f+++++++++ 02 - Photos/2020/12/photo.jpg</code>). Parser updated to detect lines starting with <code>&gt;f</code> and extract the top-level folder name.</p>
<p class="result pass">Resolved — folder activity now correctly shows file counts per folder.</p>
</div>
</div>
<!-- SECTION 7 — NFS EXPORT REFERENCE -->
<div class="section">
<div class="section-title" data-num="7">NAS08 NFS export reference</div>
<p>All four shares updated to <code>no_subtree_check</code> on May 18, 2026. The <code>/export</code> root entries are OMV auto-generated read-only entries — leave them as-is.</p>
<table>
<thead>
<tr><th>Share</th><th>Networks</th><th>Key Options</th></tr>
</thead>
<tbody>
<tr>
<td><code>/export/kingdezigns-all</code></td>
<td>.100 .110 .120 .150</td>
<td><code>rw, no_subtree_check, insecure, crossmnt</code></td>
</tr>
<tr>
<td><code>/export/kingdezigns08-secure</code></td>
<td>.100 .110 .120 .150</td>
<td><code>rw, no_subtree_check, insecure, crossmnt</code></td>
</tr>
<tr>
<td><code>/export/kingdezigns-public</code></td>
<td>.100 .110 .120 .150</td>
<td><code>rw, no_subtree_check, insecure, crossmnt</code></td>
</tr>
<tr>
<td><code>/export/HomeAssistantBackups</code></td>
<td>.150 only</td>
<td><code>rw, no_subtree_check, insecure, crossmnt</code></td>
</tr>
<tr>
<td><code>/export</code> (root)</td>
<td>.100 .110 .120 .150</td>
<td><code>ro, root_squash, subtree_check</code> — OMV auto-generated, do not edit</td>
</tr>
</tbody>
</table>
<p style="margin-top:0.75rem;font-size:12px;color:var(--text-muted);">Note: duplicate 192.168.100.0/24 entry for /export root exists in OMV config — pre-existing, harmless, clean up in OMV UI when convenient.</p>
</div>
<!-- SECTION 8 — NIGHTLY TASK TIMELINE -->
<div class="section">
<div class="section-title" data-num="8">NAS08 nightly task timeline</div>
<table>
<thead>
<tr><th>Time</th><th>Device</th><th>Script</th><th>Notes</th></tr>
</thead>
<tbody>
<tr>
<td>12:00 AM</td>
<td>NAS08</td>
<td><code>/usr/scripts/zfs/nas08_zfs_report.sh</code></td>
<td>Daily ZFS health report email.</td>
</tr>
<tr>
<td>2:00 AM</td>
<td>NAS08 &amp; NAS16</td>
<td><code>/usr/scripts/omv/nas16-backup.sh</code></td>
<td>Pre-existing backup script. NAS16 finishes ~1 min, NAS08 ~5 min. Runs every 3 days.</td>
</tr>
<tr>
<td>2:30 AM</td>
<td>NAS16</td>
<td><code>/usr/scripts/nas/nas08_to_nas16_sync.sh</code></td>
<td>NAS08 → NAS16 Public folder sync. 25-min buffer after NAS08 backup finishes.</td>
</tr>
<tr>
<td>3:00 AM (Sun)</td>
<td>NAS08</td>
<td><code>/usr/scripts/zfs/nas08_zfs_scrub.sh</code></td>
<td>Weekly ZFS scrub. Sundays only.</td>
</tr>
<tr>
<td>TBD</td>
<td>TBD</td>
<td>Future maintenance tasks</td>
<td>Drive health checks, log cleanups, etc. — to be added later.</td>
</tr>
</tbody>
</table>
</div>
<!-- SECTION 9 — LESSONS LEARNED -->
<div class="section">
<div class="section-title" data-num="9">Lessons learned</div>
<table>
<thead>
<tr><th>Lesson</th><th>Detail</th></tr>
</thead>
<tbody>
<tr>
<td>subtree_check causes intermittent NFS permission failures</td>
<td>On folders with ACL-based or mixed ownership, subtree_check can fail the export tree validation intermittently — especially after idle periods overnight. no_subtree_check is the safe modern default for private home networks.</td>
</tr>
<tr>
<td>NFS client cache must be cleared after export changes</td>
<td>Running exportfs -ra on the server isn't enough. The NFS client (NAS16) must unmount and remount to pick up the new export options. Without this, the old subtree_check behavior persists.</td>
</tr>
<tr>
<td>ACL permissions and NFS UIDs are separate concerns</td>
<td>Files can have correct ACLs (rufusking:rwx) but still be blocked over NFS if the UID mapping fails. Always verify UIDs match on both machines with id username before assuming ACL issues.</td>
</tr>
<tr>
<td>sudo operations reset directory ownership</td>
<td>Any sudo mkdir or sudo mount can leave directories owned by root. Scripts that write to those directories will fail with Permission Denied when run as a regular user. Always chown after sudo directory operations.</td>
</tr>
<tr>
<td>Lock files should store PIDs, not just exist</td>
<td>A bare touch lock file can't distinguish a live run from a stale one left by a killed process. Storing the PID and checking if it's still running makes lock files self-healing.</td>
</tr>
<tr>
<td>--itemize-changes is the right flag for parsing file transfers</td>
<td>--progress generates xfr# markers but is noisy. --verbose only lists directories. --itemize-changes gives one clean structured line per transferred file (&gt;f prefix) that's easy to parse reliably regardless of file size.</td>
</tr>
<tr>
<td>Pull is safer than push for backups</td>
<td>Running rsync on the destination keeps the source device (NAS08) passive and low-load during the sync window.</td>
</tr>
<tr>
<td>Schedule buffer matters</td>
<td>Running the sync immediately after another backup script risks overlap on heavy nights. A 25-30 min buffer between jobs keeps things clean.</td>
</tr>
<tr>
<td>File count is larger than expected when blocked folders open up</td>
<td>Initial estimate was ~16,000 files. Actual count after Photos/Taxes/Documents became accessible was ~225,000 files across ~983 GB. rsync handled this without issue.</td>
</tr>
</tbody>
</table>
</div>
<!-- SECTION 10 — NETWORK MAP -->
<div class="section">
<div class="section-title" data-num="10">Network map — items to update</div>
<table>
<thead>
<tr><th>Item</th><th>Detail</th></tr>
</thead>
<tbody>
<tr>
<td>New script on NAS16</td>
<td><code>/usr/scripts/nas/nas08_to_nas16_sync.sh</code> — document in NAS16 script inventory</td>
</tr>
<tr>
<td>New log location on NAS16</td>
<td><code>/var/log/nas-sync/</code> — add to log monitoring notes. 30-day auto-rotation.</td>
</tr>
<tr>
<td>OMV Scheduler — NAS16</td>
<td>Task added: <code>30 2 */2 * *</code> running the sync script</td>
</tr>
<tr>
<td>NFS exports changed on NAS08</td>
<td>All four shares updated to <code>no_subtree_check</code> in OMV. Document in NAS08 config notes.</td>
</tr>
<tr>
<td>NAS08 → NAS16 data flow</td>
<td>NAS08 Public folder now syncing to NAS16 Public every other day at 2:30 AM. 225,000 files / 983 GB in scope.</td>
</tr>
<tr>
<td>OMV duplicate export entry (cleanup pending)</td>
<td>192.168.100.0/24 appears twice for the /export root in NAS08's exports. Harmless but should be cleaned up in OMV UI when convenient.</td>
</tr>
</tbody>
</table>
</div>
<div class="footer">
<span>KingDezigns Home Network</span>
<span>v1.0 — May 13, 2026 &nbsp;|&nbsp; v2.0 — May 18, 2026</span>
</div>
</div>
</body>
</html>

476
nas/nas08_to_nas16_sync.sh Normal file
View file

@ -0,0 +1,476 @@
#!/bin/bash
# =============================================================================
# NAS08 → NAS16 Sync Script
# Source : NAS08 (192.168.150.35) /mnt/kingdezigns08/Public
# Dest : NAS16 (192.168.150.40) /export/kingdezignsnas-16/Public
#
# Behavior:
# - Copies only changed/new files from NAS08 to NAS16
# - Does NOT delete files on NAS16 that are missing from NAS08
# - Preserves file timestamps and symlinks
# - Skips permission syncing (NFS all_squash incompatible with chmod)
# - Skips directory timestamp syncing (NFS all_squash incompatible with utimes)
# - Auto-corrects destination ownership/permissions before sync if needed
# - Sends an HTML status email after every run
# - Logs every run with timestamps
#
# Schedule via OMV Task Scheduler: 30 2 */2 * *
# =============================================================================
# --- Configuration -----------------------------------------------------------
SOURCE="/mnt/kingdezigns08/Public/"
DEST="/export/kingdezignsnas-16/Public/"
LOG_DIR="/var/log/nas-sync"
LOG_FILE="${LOG_DIR}/nas08_to_nas16_$(date +%Y-%m-%d).log"
LOCK_FILE="/tmp/nas08_to_nas16.lock"
REPORT_TO="rufus.king@kingdezigns.com"
REPORT_FROM="nas-sync@kingdezigns16.local"
NOW=$(date '+%Y-%m-%d %H:%M:%S')
NFS_MOUNT="/mnt/kingdezigns08"
DEST_EXPECTED_OWNER="www-data"
DEST_EXPECTED_GROUP="ncshare"
# --- HTML escape helper ------------------------------------------------------
html_escape() { sed 's/&/\&amp;/g; s/</\&lt;/g; s/>/\&gt;/g; s/"/\&quot;/g'; }
# --- Safety Checks -----------------------------------------------------------
# Ensure log directory exists and is owned by the running user.
mkdir -p "$LOG_DIR"
if [ "$(stat -c '%U' "$LOG_DIR")" != "$(whoami)" ]; then
sudo chown -R "$(whoami)":users "$LOG_DIR" 2>/dev/null || true
fi
# Prevent overlapping runs — but self-heal stale lock files.
if [ -f "$LOCK_FILE" ]; then
LOCK_PID=$(cat "$LOCK_FILE" 2>/dev/null)
if [ -n "$LOCK_PID" ] && kill -0 "$LOCK_PID" 2>/dev/null; then
echo "[$(date '+%Y-%m-%d %H:%M:%S')] ERROR: Sync already running as PID $LOCK_PID. Exiting." | tee -a "$LOG_FILE"
exit 1
else
echo "[$(date '+%Y-%m-%d %H:%M:%S')] WARNING: Stale lock file found (PID $LOCK_PID no longer running). Removing and continuing." | tee -a "$LOG_FILE"
rm -f "$LOCK_FILE"
fi
fi
echo $$ > "$LOCK_FILE"
# Wait for NFS mount to be fully ready (up to 30 seconds)
echo "[$(date '+%Y-%m-%d %H:%M:%S')] Checking NFS mount readiness..." | tee -a "$LOG_FILE"
WAIT=0
until mountpoint -q "$NFS_MOUNT"; do
sleep 5
WAIT=$((WAIT + 5))
if [ $WAIT -ge 30 ]; then
echo "[$(date '+%Y-%m-%d %H:%M:%S')] ERROR: NFS mount $NFS_MOUNT not ready after 30 seconds. Aborting." | tee -a "$LOG_FILE"
rm -f "$LOCK_FILE"
exit 1
fi
echo "[$(date '+%Y-%m-%d %H:%M:%S')] Waiting for NFS mount... (${WAIT}s)" | tee -a "$LOG_FILE"
done
echo "[$(date '+%Y-%m-%d %H:%M:%S')] NFS mount ready." | tee -a "$LOG_FILE"
# Verify source is mounted and not empty
if [ ! -d "$SOURCE" ] || [ -z "$(ls -A "$SOURCE" 2>/dev/null)" ]; then
echo "[$(date '+%Y-%m-%d %H:%M:%S')] ERROR: Source '$SOURCE' is not mounted or is empty. Aborting to prevent data loss." | tee -a "$LOG_FILE"
rm -f "$LOCK_FILE"
exit 1
fi
# Verify destination exists
if [ ! -d "$DEST" ]; then
echo "[$(date '+%Y-%m-%d %H:%M:%S')] ERROR: Destination '$DEST' does not exist. Aborting." | tee -a "$LOG_FILE"
rm -f "$LOCK_FILE"
exit 1
fi
# --- Destination Permission Precheck -----------------------------------------
# NFS all_squash maps all writes to www-data:ncshare (anonuid=33, anongid=1001).
# If the destination tree is not owned by www-data:ncshare, rsync will fail to
# set permissions and timestamps. Self-correct before the sync runs so this
# never requires manual intervention.
PERM_CORRECTED=false
echo "[$(date '+%Y-%m-%d %H:%M:%S')] Verifying destination ownership and permissions..." | tee -a "$LOG_FILE"
DEST_OWNER=$(stat -c '%U' "$DEST" 2>/dev/null)
DEST_GROUP=$(stat -c '%G' "$DEST" 2>/dev/null)
if [ "$DEST_OWNER" != "$DEST_EXPECTED_OWNER" ] || [ "$DEST_GROUP" != "$DEST_EXPECTED_GROUP" ]; then
echo "[$(date '+%Y-%m-%d %H:%M:%S')] WARNING: Destination ownership is ${DEST_OWNER}:${DEST_GROUP} — expected ${DEST_EXPECTED_OWNER}:${DEST_EXPECTED_GROUP}. Correcting now..." | tee -a "$LOG_FILE"
sudo chown -R "${DEST_EXPECTED_OWNER}:${DEST_EXPECTED_GROUP}" "$DEST" 2>&1 | tee -a "$LOG_FILE"
sudo chmod -R 2770 "$DEST" 2>&1 | tee -a "$LOG_FILE"
sudo setfacl -R -m "g:${DEST_EXPECTED_GROUP}:rwx" "$DEST" 2>&1 | tee -a "$LOG_FILE"
sudo setfacl -R -d -m "g:${DEST_EXPECTED_GROUP}:rwx" "$DEST" 2>&1 | tee -a "$LOG_FILE"
PERM_CORRECTED=true
echo "[$(date '+%Y-%m-%d %H:%M:%S')] Destination permissions corrected successfully." | tee -a "$LOG_FILE"
else
echo "[$(date '+%Y-%m-%d %H:%M:%S')] Destination ownership OK (${DEST_OWNER}:${DEST_GROUP})." | tee -a "$LOG_FILE"
fi
# --- Run Sync ----------------------------------------------------------------
echo "[$(date '+%Y-%m-%d %H:%M:%S')] ===== NAS08 → NAS16 Sync Started =====" | tee -a "$LOG_FILE"
echo "[$(date '+%Y-%m-%d %H:%M:%S')] Source : $SOURCE" | tee -a "$LOG_FILE"
echo "[$(date '+%Y-%m-%d %H:%M:%S')] Dest : $DEST" | tee -a "$LOG_FILE"
START_TIME=$(date +%s)
# --archive : recursive, preserve symlinks, owner, group, devices
# --no-perms : skip chmod — NFS all_squash prevents non-owner chmod
# --omit-dir-times : skip directory utimes — NFS all_squash prevents non-owner utimes on dirs
# File timestamps are still preserved exactly.
# --update : skip files that are newer on destination
RSYNC_OUTPUT=$(rsync \
--archive \
--no-perms \
--omit-dir-times \
--update \
--human-readable \
--itemize-changes \
--stats \
--exclude='.Trash-1000/' \
"$SOURCE" \
"$DEST" 2>&1)
RSYNC_EXIT=$?
END_TIME=$(date +%s)
ELAPSED=$(( END_TIME - START_TIME ))
ELAPSED_FMT=$(printf '%dm %ds' $((ELAPSED/60)) $((ELAPSED%60)))
# Log full rsync output
echo "$RSYNC_OUTPUT" >> "$LOG_FILE"
# --- Parse rsync stats -------------------------------------------------------
FILES_TOTAL=$(echo "$RSYNC_OUTPUT" | grep "Number of files:" | grep -oP '[\d,]+' | head -1 | tr -d ',')
FILES_TRANSFERRED=$(echo "$RSYNC_OUTPUT" | grep "Number of regular files transferred:" | grep -oP '[\d,]+' | head -1 | tr -d ',')
FILES_CREATED=$(echo "$RSYNC_OUTPUT" | grep "Number of created files:" | grep -oP '[\d,]+' | head -1 | tr -d ',')
TOTAL_SIZE=$(echo "$RSYNC_OUTPUT" | grep "Total file size:" | grep -oP '[0-9.,]+\s*[KMGT]?' | head -1)
TRANSFERRED_SIZE=$(echo "$RSYNC_OUTPUT" | grep "Total transferred file size:" | grep -oP '[0-9.,]+\s*[KMGT]?' | head -1)
TRANSFER_RATE=$(echo "$RSYNC_OUTPUT" | grep "sent.*bytes.*received" | grep -oP '[\d.,]+\s*[KMGT]?bytes/sec' | head -1)
FILES_TOTAL=${FILES_TOTAL:-0}
FILES_TRANSFERRED=${FILES_TRANSFERRED:-0}
FILES_CREATED=${FILES_CREATED:-0}
TOTAL_SIZE=${TOTAL_SIZE:-"unknown"}
TRANSFERRED_SIZE=${TRANSFERRED_SIZE:-"0"}
TRANSFER_RATE=${TRANSFER_RATE:-"unknown"}
# --- Parse per-folder activity -----------------------------------------------
declare -A FOLDER_COUNTS
declare -A FOLDER_ERRORS
while IFS= read -r LINE; do
if echo "$LINE" | grep -qE '^>f'; then
FILEPATH=$(echo "$LINE" | sed 's/^[^ ]* //')
TOP=$(echo "$FILEPATH" | cut -d'/' -f1)
if [ -n "$TOP" ]; then
FOLDER_COUNTS["$TOP"]=$(( ${FOLDER_COUNTS["$TOP"]:-0} + 1 ))
fi
fi
if echo "$LINE" | grep -qE "failed:|Permission denied|No such file|error:"; then
ERR_FOLDER=$(echo "$LINE" | grep -oP '"[^"]*"' | head -1 | tr -d '"' | xargs basename 2>/dev/null || echo "unknown")
FOLDER_ERRORS["$ERR_FOLDER"]=$(echo "$LINE" | html_escape)
fi
done <<< "$RSYNC_OUTPUT"
# Build folder activity rows
FOLDER_ROWS=""
FOLDERS_WITH_CHANGES=0
FOLDERS_WITH_ERRORS=0
for FOLDER in "${!FOLDER_COUNTS[@]}"; do
COUNT=${FOLDER_COUNTS[$FOLDER]}
FOLDERS_WITH_CHANGES=$((FOLDERS_WITH_CHANGES + 1))
FOLDER_ESC=$(echo "$FOLDER" | html_escape)
FOLDER_ROWS+="
<tr style='background:#f0fdf4;'>
<td style='padding:7px 12px;font-size:12px;font-family:monospace;color:#111827;border-bottom:1px solid #e5e7eb;'>&#128193; ${FOLDER_ESC}</td>
<td style='padding:7px 12px;font-size:12px;text-align:center;border-bottom:1px solid #e5e7eb;'>
<span style='background:#1a7f4b;color:white;padding:2px 10px;border-radius:12px;font-size:11px;font-weight:700;'>${COUNT} file(s) updated</span>
</td>
</tr>"
done
for FOLDER in "${!FOLDER_ERRORS[@]}"; do
FOLDERS_WITH_ERRORS=$((FOLDERS_WITH_ERRORS + 1))
FOLDER_ESC=$(echo "$FOLDER" | html_escape)
ERR_ESC=${FOLDER_ERRORS[$FOLDER]}
FOLDER_ROWS+="
<tr style='background:#fff1f2;'>
<td style='padding:7px 12px;font-size:12px;font-family:monospace;color:#7f1d1d;border-bottom:1px solid #fecdd3;'>&#9888; ${FOLDER_ESC}</td>
<td style='padding:7px 12px;font-size:11px;color:#b91c1c;border-bottom:1px solid #fecdd3;'>${ERR_ESC}</td>
</tr>"
done
if [ -z "$FOLDER_ROWS" ]; then
FOLDER_ROWS="
<tr>
<td colspan='2' style='padding:12px;font-size:12px;color:#6b7280;text-align:center;border-bottom:1px solid #e5e7eb;'>No folder activity — all files are already up to date.</td>
</tr>"
fi
# --- Determine overall status ------------------------------------------------
if [ $RSYNC_EXIT -eq 0 ] && [ "$FOLDERS_WITH_ERRORS" -eq 0 ]; then
OVERALL_STATUS="SUCCESS"
BANNER_BG="#1a7f4b"
BANNER_ICON="&#9989;"
BANNER_MSG="Sync completed successfully — no errors detected."
STATUS_BADGE_BG="#1a7f4b"
CARD_HEADER_BG="#f0fdf4"
elif [ "$FOLDERS_WITH_ERRORS" -gt 0 ]; then
OVERALL_STATUS="WARNING"
BANNER_BG="#b45309"
BANNER_ICON="&#9888;&#65039;"
BANNER_MSG="Sync completed with warnings — some folders had issues."
STATUS_BADGE_BG="#b45309"
CARD_HEADER_BG="#fffbeb"
else
OVERALL_STATUS="FAILED"
BANNER_BG="#b91c1c"
BANNER_ICON="&#128680;"
BANNER_MSG="Sync failed — check the log for details."
STATUS_BADGE_BG="#b91c1c"
CARD_HEADER_BG="#fff1f2"
fi
# Include permission correction notice in subject if it ran
PERM_NOTE=""
if [ "$PERM_CORRECTED" = true ]; then
PERM_NOTE=" [permissions auto-corrected]"
fi
SUBJECT="[NAS Sync] NAS08 → NAS16 — ${OVERALL_STATUS}${PERM_NOTE} — ${NOW}"
# --- Log result --------------------------------------------------------------
if [ $RSYNC_EXIT -eq 0 ]; then
echo "[$(date '+%Y-%m-%d %H:%M:%S')] ===== Sync Completed Successfully =====" | tee -a "$LOG_FILE"
else
echo "[$(date '+%Y-%m-%d %H:%M:%S')] ERROR: rsync exited with code $RSYNC_EXIT. Check log: $LOG_FILE" | tee -a "$LOG_FILE"
fi
# --- Build HTML email --------------------------------------------------------
RAW_LOG_ESC=$(echo "$RSYNC_OUTPUT" | html_escape)
# Permission correction notice block (only shown if correction ran)
if [ "$PERM_CORRECTED" = true ]; then
PERM_NOTICE_BLOCK="
<div style='margin-bottom:20px;padding:12px 16px;background:#fffbeb;border:1px solid #fac775;border-radius:6px;'>
<div style='font-size:11px;font-weight:700;color:#633806;text-transform:uppercase;letter-spacing:.5px;margin-bottom:4px;'>&#9888; Destination Permissions Auto-Corrected</div>
<div style='font-size:12px;color:#633806;'>Destination ownership was not <code>www-data:ncshare</code>. chown, chmod, and setfacl were applied automatically before the sync ran.</div>
</div>"
else
PERM_NOTICE_BLOCK=""
fi
HTML=$(cat <<HTMLEOF
<!DOCTYPE html>
<html lang="en">
<head><meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1"></head>
<body style="margin:0;padding:0;background:#f1f5f9;font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,sans-serif;">
<table width="100%" cellspacing="0" cellpadding="0" style="background:#f1f5f9;padding:30px 0;">
<tr><td align="center">
<table width="720" cellspacing="0" cellpadding="0" style="max-width:720px;width:100%;">
<!-- Header -->
<tr><td style="background:#0f172a;border-radius:10px 10px 0 0;padding:28px 32px;">
<table width="100%" cellspacing="0" cellpadding="0"><tr>
<td>
<div style="font-size:11px;color:#94a3b8;letter-spacing:1px;text-transform:uppercase;">NAS File Sync Monitor</div>
<div style="font-size:26px;font-weight:800;color:white;margin-top:4px;">&#128260; NAS08 &rarr; NAS16</div>
<div style="font-size:13px;color:#94a3b8;margin-top:4px;">${NOW} &nbsp;|&nbsp; Schedule: every other day at 2:30 AM</div>
</td>
<td align="right">
<div style="background:${BANNER_BG};color:white;padding:10px 20px;border-radius:8px;font-size:14px;font-weight:700;">${BANNER_ICON} ${OVERALL_STATUS}</div>
</td>
</tr></table>
</td></tr>
<!-- Status Banner -->
<tr><td style="background:${BANNER_BG};padding:12px 32px;">
<span style="color:white;font-size:14px;font-weight:600;">${BANNER_ICON} &nbsp;${BANNER_MSG}</span>
</td></tr>
<!-- Sync Result Card -->
<tr><td style="padding:28px 24px 8px;">
<div style="background:white;border-radius:10px;box-shadow:0 1px 6px rgba(0,0,0,.10);margin-bottom:28px;overflow:hidden;border:1px solid #e5e7eb;">
<div style="background:${CARD_HEADER_BG};padding:16px 22px;border-bottom:1px solid #e5e7eb;">
<table width="100%" cellspacing="0" cellpadding="0"><tr>
<td>
<span style="font-size:18px;font-weight:700;color:#111827;">&#128196; Sync Results</span>
<span style="margin-left:8px;font-size:12px;color:#6b7280;">NAS08 Public &rarr; NAS16 Public</span>
</td>
<td align="right">
<span style="background:${STATUS_BADGE_BG};color:white;padding:5px 14px;border-radius:20px;font-size:12px;font-weight:700;letter-spacing:.5px;">${OVERALL_STATUS}</span>
</td>
</tr></table>
</div>
<div style="padding:20px 22px;">
${PERM_NOTICE_BLOCK}
<!-- Stats tiles -->
<table width="100%" cellspacing="0" cellpadding="0">
<tr>
<td style="width:18%;padding:8px 10px;background:#f9fafb;border-radius:6px;text-align:center;">
<div style="font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;">Scanned</div>
<div style="font-size:20px;font-weight:700;color:#111827;margin-top:2px;">${FILES_TOTAL}</div>
</td><td style="width:2%;"></td>
<td style="width:18%;padding:8px 10px;background:#f9fafb;border-radius:6px;text-align:center;">
<div style="font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;">Transferred</div>
<div style="font-size:20px;font-weight:700;color:#111827;margin-top:2px;">${FILES_TRANSFERRED}</div>
</td><td style="width:2%;"></td>
<td style="width:18%;padding:8px 10px;background:#f9fafb;border-radius:6px;text-align:center;">
<div style="font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;">New Files</div>
<div style="font-size:20px;font-weight:700;color:#111827;margin-top:2px;">${FILES_CREATED}</div>
</td><td style="width:2%;"></td>
<td style="width:18%;padding:8px 10px;background:#f9fafb;border-radius:6px;text-align:center;">
<div style="font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;">Data Moved</div>
<div style="font-size:20px;font-weight:700;color:#111827;margin-top:2px;">${TRANSFERRED_SIZE}</div>
</td><td style="width:2%;"></td>
<td style="width:18%;padding:8px 10px;background:#f9fafb;border-radius:6px;text-align:center;">
<div style="font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;">Duration</div>
<div style="font-size:20px;font-weight:700;color:#111827;margin-top:2px;">${ELAPSED_FMT}</div>
</td>
</tr>
</table>
<!-- Source / Dest details -->
<div style="margin-top:16px;padding:12px 14px;background:#f8fafc;border-radius:6px;border:1px solid #e5e7eb;">
<table width="100%" cellspacing="0" cellpadding="0">
<tr>
<td style="font-size:12px;color:#6b7280;width:90px;padding-bottom:5px;">Source</td>
<td style="font-size:12px;font-family:monospace;color:#111827;padding-bottom:5px;">NAS08 (192.168.150.35) &rarr; ${SOURCE}</td>
</tr>
<tr>
<td style="font-size:12px;color:#6b7280;padding-bottom:5px;">Destination</td>
<td style="font-size:12px;font-family:monospace;color:#111827;padding-bottom:5px;">NAS16 (192.168.150.40) &rarr; ${DEST}</td>
</tr>
<tr>
<td style="font-size:12px;color:#6b7280;padding-bottom:5px;">Transfer Rate</td>
<td style="font-size:12px;font-family:monospace;color:#111827;padding-bottom:5px;">${TRANSFER_RATE}</td>
</tr>
<tr>
<td style="font-size:12px;color:#6b7280;">Total Source</td>
<td style="font-size:12px;font-family:monospace;color:#111827;">${TOTAL_SIZE}</td>
</tr>
</table>
</div>
<!-- Folder Activity — collapsible -->
<details style="margin-top:14px;">
<summary style="cursor:pointer;font-size:12px;color:#6b7280;font-weight:600;letter-spacing:.3px;">&#9654; Folder Activity &nbsp;&mdash;&nbsp; ${FOLDERS_WITH_CHANGES} updated &nbsp;|&nbsp; ${FOLDERS_WITH_ERRORS} error(s)</summary>
<div style="margin-top:8px;overflow-x:auto;border-radius:6px;border:1px solid #e5e7eb;">
<table width="100%" cellspacing="0" cellpadding="0" style="border-collapse:collapse;">
<thead>
<tr style="background:#0f172a;">
<th style="padding:8px 12px;font-size:11px;color:#94a3b8;text-align:left;text-transform:uppercase;letter-spacing:.5px;">Folder</th>
<th style="padding:8px 12px;font-size:11px;color:#94a3b8;text-align:center;text-transform:uppercase;letter-spacing:.5px;width:200px;">Status</th>
</tr>
</thead>
<tbody>${FOLDER_ROWS}</tbody>
</table>
</div>
</details>
<!-- Full rsync log — collapsible -->
<details style="margin-top:10px;">
<summary style="cursor:pointer;font-size:12px;color:#6b7280;font-weight:600;letter-spacing:.3px;">&#9654; Full rsync Log &nbsp;&mdash;&nbsp; click to expand</summary>
<pre style="margin-top:8px;background:#1e293b;color:#e2e8f0;padding:14px;border-radius:6px;font-size:11px;line-height:1.6;overflow-x:auto;white-space:pre-wrap;">${RAW_LOG_ESC}</pre>
</details>
</div>
</div>
</td></tr>
<!-- Summary -->
<tr><td style="padding:0 24px 28px;">
<div style="background:white;border-radius:10px;border:1px solid #e5e7eb;padding:18px 22px;">
<div style="font-size:13px;font-weight:700;color:#374151;margin-bottom:12px;text-transform:uppercase;letter-spacing:.5px;">&#128203; Report Summary</div>
<table width="100%" cellspacing="0" cellpadding="0">
<tr>
<td style="font-size:13px;color:#6b7280;padding-bottom:6px;">Host</td>
<td style="font-size:13px;color:#111827;font-weight:600;padding-bottom:6px;">NAS16 (kingdezigns16)</td>
</tr>
<tr>
<td style="font-size:13px;color:#6b7280;padding-bottom:6px;">Run Time</td>
<td style="font-size:13px;color:#111827;font-weight:600;padding-bottom:6px;">${NOW}</td>
</tr>
<tr>
<td style="font-size:13px;color:#6b7280;padding-bottom:6px;">rsync Exit Code</td>
<td style="font-size:13px;font-weight:700;padding-bottom:6px;">${RSYNC_EXIT}</td>
</tr>
<tr>
<td style="font-size:13px;color:#6b7280;padding-bottom:6px;">Permissions Auto-Corrected</td>
<td style="font-size:13px;font-weight:700;padding-bottom:6px;">${PERM_CORRECTED}</td>
</tr>
<tr>
<td style="font-size:13px;color:#6b7280;padding-bottom:6px;">Folders Updated</td>
<td style="font-size:13px;color:#111827;font-weight:600;padding-bottom:6px;">${FOLDERS_WITH_CHANGES}</td>
</tr>
<tr>
<td style="font-size:13px;color:#6b7280;padding-bottom:6px;">Folder Errors</td>
<td style="font-size:13px;font-weight:700;padding-bottom:6px;">${FOLDERS_WITH_ERRORS}</td>
</tr>
<tr>
<td style="font-size:13px;color:#6b7280;">Log File</td>
<td style="font-size:13px;font-family:monospace;color:#111827;">${LOG_FILE}</td>
</tr>
</table>
</div>
</td></tr>
<!-- Footer -->
<tr><td style="background:#0f172a;border-radius:0 0 10px 10px;padding:16px 32px;text-align:center;">
<span style="font-size:11px;color:#64748b;">Automated NAS Sync Monitor &nbsp;|&nbsp; NAS16 &nbsp;|&nbsp; KingDezigns Infrastructure</span>
</td></tr>
</table>
</td></tr>
</table>
</body>
</html>
HTMLEOF
)
# --- Send email --------------------------------------------------------------
send_email() {
if command -v sendmail &>/dev/null; then
{ echo "To: ${REPORT_TO}"; echo "From: ${REPORT_FROM}"; echo "Subject: ${SUBJECT}"
echo "MIME-Version: 1.0"; echo "Content-Type: text/html; charset=UTF-8"; echo ""
echo "$HTML"; } | sendmail -t
echo "[$(date '+%Y-%m-%d %H:%M:%S')] Report sent via sendmail to ${REPORT_TO}" | tee -a "$LOG_FILE"
elif command -v mailx &>/dev/null; then
echo "$HTML" | mailx -a "Content-Type: text/html" -s "$SUBJECT" "$REPORT_TO"
echo "[$(date '+%Y-%m-%d %H:%M:%S')] Report sent via mailx to ${REPORT_TO}" | tee -a "$LOG_FILE"
elif command -v mail &>/dev/null; then
echo "$HTML" | mail -a "Content-Type: text/html; charset=UTF-8" -s "$SUBJECT" "$REPORT_TO"
echo "[$(date '+%Y-%m-%d %H:%M:%S')] Report sent via mail to ${REPORT_TO}" | tee -a "$LOG_FILE"
else
echo "[$(date '+%Y-%m-%d %H:%M:%S')] ERROR: No mail transport found." | tee -a "$LOG_FILE"
FALLBACK="/tmp/nas_sync_report_$(date +%Y%m%d_%H%M%S).html"
echo "$HTML" > "$FALLBACK"
echo "[$(date '+%Y-%m-%d %H:%M:%S')] HTML report saved to: $FALLBACK" | tee -a "$LOG_FILE"
fi
}
send_email
# --- Cleanup -----------------------------------------------------------------
rm -f "$LOCK_FILE"
# Rotate logs older than 30 days
find "$LOG_DIR" -name "nas08_to_nas16_*.log" -mtime +30 -delete
echo "[$(date '+%Y-%m-%d %H:%M:%S')] Done — Status: ${OVERALL_STATUS} | Transferred: ${FILES_TRANSFERRED} file(s) | Duration: ${ELAPSED_FMT}" | tee -a "$LOG_FILE"
exit $RSYNC_EXIT

1003
omv/NAS16-Backup-Summary.html Normal file
View file

File diff suppressed because it is too large Load diff

521
omv/SSD-fstrim-Summary.html Normal file
View file

@ -0,0 +1,521 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>KingDezigns — SSD Hardening Summary</title>
<link rel="preconnect" href="https://fonts.googleapis.com">
<link href="https://fonts.googleapis.com/css2?family=IBM+Plex+Mono:wght@400;500&family=IBM+Plex+Sans:wght@400;500&display=swap" rel="stylesheet">
<style>
*, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
:root {
--bg: #f7f6f2;
--surface: #ffffff;
--surface-alt: #f1f0eb;
--border: #e2e0d8;
--border-strong: #c8c5ba;
--text-primary: #1a1917;
--text-secondary: #6b6960;
--text-muted: #9b9890;
--green-bg: #eaf3de;
--green-border: #97c459;
--green-text: #2d5a0e;
--red-bg: #fcebeb;
--red-border: #f09595;
--red-text: #7a1f1f;
--amber-bg: #faeeda;
--amber-border: #fac775;
--amber-text: #633806;
--coral-bg: #faece7;
--coral-border: #f0997b;
--coral-text: #4a1b0c;
--mono: 'IBM Plex Mono', monospace;
--sans: 'IBM Plex Sans', sans-serif;
}
body {
font-family: var(--sans);
font-size: 14px;
background: var(--bg);
color: var(--text-primary);
line-height: 1.7;
padding: 2.5rem 1.5rem;
}
.page { max-width: 860px; margin: 0 auto; }
header {
border-bottom: 1px solid var(--border-strong);
padding-bottom: 1.25rem;
margin-bottom: 2rem;
}
header h1 {
font-family: var(--mono);
font-size: 18px;
font-weight: 500;
letter-spacing: -0.02em;
margin-bottom: 6px;
}
.meta {
font-size: 12px;
color: var(--text-muted);
font-family: var(--mono);
display: flex;
gap: 16px;
flex-wrap: wrap;
}
.section { margin-bottom: 2.25rem; }
.section-title {
font-family: var(--mono);
font-size: 11px;
font-weight: 500;
text-transform: uppercase;
letter-spacing: 0.08em;
color: var(--text-muted);
margin-bottom: 1rem;
display: flex;
align-items: center;
gap: 10px;
}
.section-title::before {
content: attr(data-num);
background: var(--text-primary);
color: var(--bg);
font-size: 10px;
width: 18px;
height: 18px;
border-radius: 50%;
display: inline-flex;
align-items: center;
justify-content: center;
flex-shrink: 0;
}
.section-title::after {
content: '';
flex: 1;
height: 1px;
background: var(--border);
}
p { margin-bottom: 0.75rem; color: var(--text-primary); }
p:last-child { margin-bottom: 0; }
.badge-row { display: flex; gap: 8px; flex-wrap: wrap; margin-bottom: 1rem; }
.badge {
font-family: var(--mono);
font-size: 11px;
padding: 3px 10px;
border-radius: 3px;
border: 1px solid var(--border);
background: var(--surface);
color: var(--text-secondary);
}
.badge.red { background: var(--red-bg); color: var(--red-text); border-color: var(--red-border); }
.badge.green { background: var(--green-bg); color: var(--green-text); border-color: var(--green-border); }
.badge.amber { background: var(--amber-bg); color: var(--amber-text); border-color: var(--amber-border); }
.step {
background: var(--surface);
border: 1px solid var(--border);
border-radius: 6px;
padding: 1rem 1.25rem;
margin-bottom: 0.6rem;
}
.step-header {
font-family: var(--mono);
font-size: 12px;
font-weight: 500;
color: var(--text-secondary);
margin-bottom: 0.6rem;
display: flex;
align-items: center;
gap: 8px;
}
.step-num {
background: var(--surface-alt);
border: 1px solid var(--border);
border-radius: 50%;
width: 20px;
height: 20px;
display: inline-flex;
align-items: center;
justify-content: center;
font-size: 10px;
flex-shrink: 0;
color: var(--text-muted);
}
pre {
background: var(--surface-alt);
border: 1px solid var(--border);
border-radius: 4px;
padding: 10px 12px;
font-family: var(--mono);
font-size: 12px;
overflow-x: auto;
margin: 0.5rem 0;
white-space: pre;
line-height: 1.6;
}
code {
font-family: var(--mono);
font-size: 12px;
background: var(--surface-alt);
padding: 1px 5px;
border-radius: 3px;
border: 1px solid var(--border);
}
.why {
font-size: 12px;
color: var(--text-muted);
margin-top: 0.4rem;
padding-left: 2px;
}
.why::before { content: '↳ why: '; font-family: var(--mono); }
.result { font-size: 12px; margin-top: 0.35rem; padding-left: 2px; }
.result.pass { color: var(--green-text); }
.result.pass::before { content: '✓ '; font-family: var(--mono); }
.result.fail { color: var(--red-text); }
.result.fail::before { content: '✗ '; font-family: var(--mono); }
.result.neutral { color: var(--text-secondary); }
.result.neutral::before { content: '→ '; font-family: var(--mono); }
.callout { border-radius: 6px; padding: 1rem 1.25rem; margin-bottom: 0.75rem; }
.callout.coral { background: var(--coral-bg); border: 1px solid var(--coral-border); }
.callout.coral p { color: var(--coral-text); }
.callout.coral pre { background: #fdf0eb; border-color: var(--coral-border); }
.callout.amber { background: var(--amber-bg); border: 1px solid var(--amber-border); }
.callout.amber p { color: var(--amber-text); }
.callout.amber pre { background: #fdf5e6; border-color: var(--amber-border); }
.callout.amber code { background: #fdf5e6; border-color: var(--amber-border); color: var(--amber-text); }
.callout.green { background: var(--green-bg); border: 1px solid var(--green-border); }
.callout.green p { color: var(--green-text); }
.callout.green pre { background: #d8efc0; border-color: var(--green-border); }
.clue-label {
font-family: var(--mono);
font-size: 10px;
font-weight: 500;
text-transform: uppercase;
letter-spacing: 0.07em;
color: var(--amber-text);
margin-bottom: 8px;
}
.script-box { background: var(--surface); border: 1px solid var(--border); border-radius: 6px; overflow: hidden; }
.script-row { padding: 0.75rem 1.25rem; border-bottom: 1px solid var(--border); }
.script-row:last-child { border-bottom: none; }
.script-label { font-family: var(--mono); font-size: 10px; text-transform: uppercase; letter-spacing: 0.07em; color: var(--text-muted); margin-bottom: 6px; }
.script-row pre { margin: 0; }
table { width: 100%; border-collapse: collapse; font-size: 13px; background: var(--surface); border: 1px solid var(--border); border-radius: 6px; overflow: hidden; }
th { text-align: left; padding: 8px 14px; font-family: var(--mono); font-size: 10px; text-transform: uppercase; letter-spacing: 0.06em; color: var(--text-muted); background: var(--surface-alt); border-bottom: 1px solid var(--border); font-weight: 500; }
td { padding: 9px 14px; border-bottom: 1px solid var(--border); vertical-align: top; line-height: 1.5; }
tr:last-child td { border-bottom: none; }
.footer {
font-family: var(--mono);
font-size: 11px;
color: var(--text-muted);
margin-top: 2.5rem;
padding-top: 1rem;
border-top: 1px solid var(--border);
display: flex;
justify-content: space-between;
flex-wrap: wrap;
gap: 8px;
}
</style>
</head>
<body>
<div class="page">
<header>
<h1>KingDezigns — SSD Hardening (TRIM + noatime) summary</h1>
<div class="meta">
<span>📅 2026-05-23</span>
<span>🖥 NAS08 &amp; NAS16</span>
<span>⚙️ OpenMediaVault</span>
<span>🔧 Raspberry Pi 5 / Raspberry Pi</span>
</div>
</header>
<!-- SECTION 1 — OBJECTIVE -->
<div class="section">
<div class="section-title" data-num="1">Objective</div>
<p>Harden both NAS08 and NAS16 to extend SSD lifespan by ensuring TRIM is running correctly, noatime is active, and SMART health is being monitored — with all scheduled jobs visible in OMV's scheduler for easy management.</p>
<div class="badge-row">
<span class="badge green">NAS08 — 5 drives (1x PNY CS900 + 4x Crucial BX500)</span>
<span class="badge green">NAS16 — 1 drive (unknown model, standard SMART attrs)</span>
</div>
</div>
<!-- SECTION 2 — INVESTIGATION STEPS -->
<div class="section">
<div class="section-title" data-num="2">Investigation steps — both units</div>
<div class="step">
<div class="step-header"><span class="step-num">1</span> Verify TRIM support on all drives</div>
<pre>lsblk --discard /dev/sda</pre>
<p class="why">Confirm DISC-GRAN and DISC-MAX are non-zero before enabling TRIM.</p>
<p class="result pass">Both units: DISC-GRAN 4K / DISC-MAX 4G — TRIM fully supported on all drives.</p>
</div>
<div class="step">
<div class="step-header"><span class="step-num">2</span> Check if systemd fstrim.timer was already active</div>
<pre>systemctl status fstrim.timer</pre>
<p class="why">Determine whether a TRIM schedule already existed before creating a new one.</p>
<p class="result neutral">Both units: timer active and enabled, running weekly. NAS08 next run Monday 12:33 AM — NAS16 next run Monday 1:37 AM.</p>
</div>
<div class="step">
<div class="step-header"><span class="step-num">3</span> Confirm which volumes were being trimmed</div>
<pre>journalctl -u fstrim.service --no-pager</pre>
<p class="why">Verify the timer was actually running and catching all mounted volumes.</p>
<p class="result pass">Both units: / and /boot/firmware trimmed successfully on May 11 and May 18. NAS08 ~400 GiB freed per run. NAS16 ~228 GiB on first run, 3.3 GiB on second (less free space).</p>
</div>
<div class="step">
<div class="step-header"><span class="step-num">4</span> Check noatime mount status</div>
<pre>findmnt -t ext4,xfs,btrfs -o TARGET,SOURCE,OPTIONS</pre>
<p class="why">Confirm noatime was already set on all filesystems to prevent reads generating writes.</p>
<p class="result pass">Both units: every mount point already showing noatime — handled automatically by the openmediavault-flashmemory plugin. No action required.</p>
</div>
<div class="step">
<div class="step-header"><span class="step-num">5</span> Inspect SMART attributes per drive</div>
<pre>sudo smartctl -A /dev/sda
sudo smartctl -A /dev/sdb # NAS08 data drives</pre>
<p class="why">Identify vendor-specific SMART attribute names before writing the monitoring script — attribute names differ significantly between PNY and Crucial.</p>
<p class="result neutral">NAS08 PNY CS900: uses SSD_Life_Left (attr 231), Bad_Blk_Ct_Lat/Erl (attr 170), no pending/uncorrectable attrs. NAS08 Crucial BX500: uses Percent_Lifetime_Remain (attr 202), Reallocate_NAND_Blk_Cnt (attr 5), Reported_Uncorrect. NAS16 drive: uses standard attribute names — no vendor mapping required.</p>
</div>
</div>
<!-- SECTION 3 — WHAT WAS ALREADY DONE -->
<div class="section">
<div class="section-title" data-num="3">What was already handled by OMV</div>
<div class="callout green">
<p><strong>noatime:</strong> Already active on all filesystems on both units via the <code>openmediavault-flashmemory</code> plugin. No manual configuration required.</p>
</div>
<div class="callout green">
<p><strong>Swap disabled, logs in tmpfs:</strong> Also handled automatically by the flashmemory plugin — swap off, /var/log and /tmp in RAM, reducing write amplification significantly.</p>
</div>
<div class="callout amber">
<div class="clue-label">⚠ Note</div>
<p>The systemd <code>fstrim.timer</code> was already running weekly on both units but was invisible in OMV's scheduler. It was replaced with an OMV Scheduled Job so all jobs are visible and manageable in one place.</p>
</div>
</div>
<!-- SECTION 4 — TRIM REPORT SCRIPT -->
<div class="section">
<div class="section-title" data-num="4">TRIM report script — created and deployed</div>
<p>A custom script was written to replace the bare <code>fstrim</code> call with a full monitoring run that emails an HTML report matching the KingDezigns backup monitor style (dark header, badge/banner, section rows, cards, footer).</p>
<p>The email report includes: TRIM results per volume with total bytes freed, SSD SMART health per drive (health, temperature, wear, power-on hours, sector counts), filesystem usage with colour-coded warnings at 75% and 90%, and system info (OMV version, kernel, IP, uptime, load, memory, next run time).</p>
<div class="script-box">
<div class="script-row">
<div class="script-label">Script location</div>
<pre>/usr/scripts/omv/fstrim-report.sh</pre>
</div>
<div class="script-row">
<div class="script-label">Log location</div>
<pre>/var/log/fstrim/fstrim-YYYY-MM-DD.log (90-day retention)</pre>
</div>
<div class="script-row">
<div class="script-label">Deploy (preferred method — sudo nano)</div>
<pre>sudo nano /usr/scripts/omv/fstrim-report.sh
# paste script content, save with Ctrl+O, exit with Ctrl+X
sudo chmod +x /usr/scripts/omv/fstrim-report.sh</pre>
</div>
<div class="script-row">
<div class="script-label">Test run</div>
<pre>sudo systemctl start fstrim.service
sudo journalctl -u fstrim.service -f</pre>
</div>
<div class="script-row">
<div class="script-label">Email delivery</div>
<pre>sendmail -t (via OMV's configured SMTP relay)</pre>
</div>
</div>
<div class="callout amber" style="margin-top:0.75rem;">
<div class="clue-label">⚠ SMART attribute fixes required</div>
<p style="margin-bottom:6px;">Initial script used generic attribute names. Two rounds of fixes were needed after testing:</p>
<pre>Round 1 — Crucial BX500 fix:
Reallocated: Reallocated_Sector → Reallocate_NAND_Blk_Cnt
Pending: Current_Pending_Sector → Current_Pending_ECC_Cnt
Uncorrect: Offline_Uncorrectable → Reported_Uncorrect
Temperature: $10 → $10+0 (strips "32 (Min/Max 25/56)" format)
Round 2 — PNY CS900 fix:
Wear: Wear_Leveling_Count → SSD_Life_Left
Reallocated: Reallocate_NAND_Blk_Cnt → Bad_Blk_Ct_Lat/Erl (first value before /)</pre>
<p style="margin-top:8px;">Each pattern now tries vendor-specific names first, then falls back to generic names — script works correctly on any brand.</p>
</div>
</div>
<!-- SECTION 5 — SYSTEMD → OMV MIGRATION -->
<div class="section">
<div class="section-title" data-num="5">Migrating from systemd timer to OMV scheduler</div>
<p>The systemd <code>fstrim.timer</code> was disabled on both units and replaced with OMV Scheduled Jobs so all scheduled tasks are visible in one place.</p>
<div class="step">
<div class="step-header"><span class="step-num">1</span> Disable systemd timer (both units)</div>
<pre>sudo systemctl disable fstrim.timer
sudo systemctl stop fstrim.timer</pre>
<p class="why">Prevent the systemd timer from running alongside the new OMV job — avoids double execution.</p>
<p class="result pass">Timer stopped and disabled on both NAS08 and NAS16.</p>
</div>
<div class="step">
<div class="step-header"><span class="step-num">2</span> Remove systemd override (both units)</div>
<pre>sudo rm /etc/systemd/system/fstrim.service.d/override.conf
sudo systemctl daemon-reload</pre>
<p class="why">Clean up the override that pointed fstrim.service to the script — no longer needed since OMV cron calls the script directly.</p>
<p class="result pass">Override removed, systemd reloaded cleanly on both units.</p>
</div>
<div class="step">
<div class="step-header"><span class="step-num">3</span> Add OMV Scheduled Job (each unit)</div>
<pre>System → Scheduled Jobs → Add
Command: /usr/scripts/omv/fstrim-report.sh
User: root
NAS08: Wednesday at 1:00 AM
NAS16: Wednesday at 1:30 AM</pre>
<p class="why">Wednesday was chosen as it sits midweek, away from Sunday ZFS scrubs and the rolling every-3-day backup jobs.</p>
<p class="result pass">Both units now show TRIM Report in OMV scheduler alongside all other scheduled jobs.</p>
</div>
</div>
<!-- SECTION 6 — SCHEDULED JOB REFERENCE -->
<div class="section">
<div class="section-title" data-num="6">Final OMV scheduled jobs — both units</div>
<p style="margin-bottom:0.75rem;"><strong>NAS08</strong></p>
<table>
<thead>
<tr><th>Schedule</th><th>Script</th><th>Tag</th></tr>
</thead>
<tbody>
<tr><td>Every 3 days at 2:00 AM</td><td><code>/usr/scripts/omv/nas08-backup.sh</code></td><td>NAS08 Backup</td></tr>
<tr><td>Daily at 12:00 AM</td><td><code>/usr/scripts/zfs/nas08_zfs_report.sh</code></td><td>ZFS Status Report</td></tr>
<tr><td>Sunday at 3:00 AM</td><td><code>/usr/scripts/zfs/nas08_zfs_scrub.sh</code></td><td>ZFS Scrub</td></tr>
<tr><td>Wednesday at 1:00 AM</td><td><code>/usr/scripts/omv/fstrim-report.sh</code></td><td>TRIM Report</td></tr>
</tbody>
</table>
<p style="margin:1.25rem 0 0.75rem;"><strong>NAS16</strong></p>
<table>
<thead>
<tr><th>Schedule</th><th>Script</th><th>Tag</th></tr>
</thead>
<tbody>
<tr><td>Every 3 days at 2:00 AM</td><td><code>/usr/scripts/omv/nas16-backup.sh</code></td><td>NAS16 Backup</td></tr>
<tr><td>Every 2 days at 2:30 AM</td><td><code>bash /usr/scripts/nas/nas08_to_nas16_sync.sh</code></td><td>NAS Drive Backup</td></tr>
<tr><td>Daily at 12:00 AM</td><td><code>/usr/scripts/zfs/nas16_zfs_report.sh</code></td><td>ZFS Status Report</td></tr>
<tr><td>Sunday at 3:00 AM</td><td><code>/usr/scripts/zfs/nas16_zfs_scrub.sh</code></td><td>ZFS Scrub</td></tr>
<tr><td>Every 15 minutes</td><td><code>/usr/scripts/ddns/ddns_update.sh</code></td><td>DDNS Update</td></tr>
<tr><td>Wednesday at 1:30 AM</td><td><code>/usr/scripts/omv/fstrim-report.sh</code></td><td>TRIM Report</td></tr>
</tbody>
</table>
</div>
<!-- SECTION 7 — LESSONS LEARNED -->
<div class="section">
<div class="section-title" data-num="7">Lessons learned</div>
<table>
<thead>
<tr><th>Lesson</th><th>Detail</th></tr>
</thead>
<tbody>
<tr>
<td>Always investigate before configuring</td>
<td>Both noatime and fstrim were already handled by OMV's flashmemory plugin — checking first avoided duplicating work.</td>
</tr>
<tr>
<td>SMART attribute names are vendor-specific</td>
<td>PNY and Crucial use completely different attribute names for wear, reallocated sectors, and temperature format. Always run <code>smartctl -A /dev/sdX</code> first to map the real names.</td>
</tr>
<tr>
<td>systemd timers are invisible in OMV scheduler</td>
<td>The default fstrim.timer runs correctly but doesn't appear in OMV's job list — migrate to OMV Scheduled Jobs for full visibility alongside ZFS scrubs and backups.</td>
</tr>
<tr>
<td>Never run script without sudo for testing</td>
<td>Running fstrim-report.sh without sudo causes permission denied on /var/log/fstrim/ and returns N/A for all SMART data — always test with sudo or via systemctl start.</td>
</tr>
<tr>
<td>Use sudo nano over cp for file edits</td>
<td>Preferred method for placing and editing scripts on the NAS — faster navigation to the correct path, avoids confusion about source file location.</td>
</tr>
<tr>
<td>Inline TRIM (discard mount option) is not recommended</td>
<td>Real-time per-delete TRIM adds latency under heavy load. Periodic batch TRIM via fstrim is the correct approach for NAS workloads.</td>
</tr>
<tr>
<td>PNY CS900 has no pending/uncorrectable attributes</td>
<td>N/A for those fields is correct and expected — not a script bug. The drive simply does not expose those attributes.</td>
</tr>
</tbody>
</table>
</div>
<!-- SECTION 8 — DOCS UPDATED -->
<div class="section">
<div class="section-title" data-num="8">Documentation updated</div>
<table>
<thead>
<tr><th>File</th><th>What changed</th></tr>
</thead>
<tbody>
<tr>
<td><code>server_nas08.md</code></td>
<td>Added drive inventory table, full SSD Hardening &amp; Maintenance section (TRIM, noatime, flashmemory plugin, SMART attribute mapping), and complete OMV Scheduled Jobs table.</td>
</tr>
<tr>
<td><code>server_nas16.md</code></td>
<td>Added drive inventory table with Power_Cycle_Count monitoring note, full SSD Hardening &amp; Maintenance section, and complete OMV Scheduled Jobs table including NAS drive backup sync job.</td>
</tr>
</tbody>
</table>
</div>
<div class="footer">
<span>KingDezigns Home Network</span>
<span>Generated 2026-05-23</span>
</div>
</div>
</body>
</html>

4
omv/fstrim-override.conf Normal file
View file

@ -0,0 +1,4 @@
[Service]
# Clear the default ExecStart then replace with our script
ExecStart=
ExecStart=/usr/scripts/omv/fstrim-report.sh

471
omv/fstrim-report.sh Normal file
View file

@ -0,0 +1,471 @@
#!/bin/bash
# =============================================================================
# NAS08 Weekly TRIM Report Script
# =============================================================================
# Runs fstrim on all mounted filesystems, captures results, gathers SMART
# health data for each SSD, and sends an HTML email report matching the
# KingDezigns backup monitor style.
#
# Triggered by: systemd fstrim.service (via drop-in override)
# Schedule: Weekly (Monday ~12:33 AM — managed by fstrim.timer)
#
# Script location: /usr/scripts/omv/fstrim-report.sh
# Log location: /var/log/fstrim/
# =============================================================================
set -uo pipefail
# -----------------------------------------------------------------------------
# CONFIGURATION
# -----------------------------------------------------------------------------
HOSTNAME_LABEL=$(hostname -s 2>/dev/null || echo "NAS08")
REPORT_TO="rufus.king@kingdezigns.com"
REPORT_FROM="trim-monitor@nas08.local" # Display only — OMV uses your configured SMTP relay
LOG_DIR="/var/log/fstrim"
KEEP_LOGS=90 # days to retain log files
DATE=$(date +%Y-%m-%d)
REPORT_TIME=$(date "+%Y-%m-%d %H:%M:%S")
LOG_FILE="${LOG_DIR}/fstrim-${DATE}.log"
# -----------------------------------------------------------------------------
# STATUS TRACKING
# OVERALL_STATUS: SUCCESS | WARNING | FAILED
# -----------------------------------------------------------------------------
OVERALL_STATUS="SUCCESS"
SECTION_ROWS_HTML=""
WARNING_COUNT=0
ERROR_COUNT=0
# Append a result row to the section table
# Usage: add_section_row "Section name" "STATUS" "Detail message"
# STATUS: OK | WARNING | ERROR | SKIPPED
add_section_row() {
local section="$1" status="$2" detail="$3"
case "$status" in
OK)
BADGE="<span style=\"background:#d1fae5;color:#065f46;padding:3px 10px;border-radius:20px;font-size:11px;font-weight:700;\">✓ OK</span>"
;;
WARNING)
BADGE="<span style=\"background:#fef3c7;color:#92400e;padding:3px 10px;border-radius:20px;font-size:11px;font-weight:700;\">⚠ WARNING</span>"
WARNING_COUNT=$((WARNING_COUNT + 1))
[[ "$OVERALL_STATUS" == "SUCCESS" ]] && OVERALL_STATUS="WARNING"
;;
ERROR)
BADGE="<span style=\"background:#fee2e2;color:#991b1b;padding:3px 10px;border-radius:20px;font-size:11px;font-weight:700;\">✗ ERROR</span>"
ERROR_COUNT=$((ERROR_COUNT + 1))
OVERALL_STATUS="FAILED"
;;
SKIPPED)
BADGE="<span style=\"background:#f3f4f6;color:#6b7280;padding:3px 10px;border-radius:20px;font-size:11px;font-weight:700;\">— SKIPPED</span>"
;;
esac
SECTION_ROWS_HTML+="
<tr>
<td style=\"padding:9px 14px;border-bottom:1px solid #f3f4f6;font-size:13px;color:#111827;font-weight:600;\">$section</td>
<td style=\"padding:9px 14px;border-bottom:1px solid #f3f4f6;\">$BADGE</td>
<td style=\"padding:9px 14px;border-bottom:1px solid #f3f4f6;font-size:12px;color:#6b7280;\">$detail</td>
</tr>"
}
# -----------------------------------------------------------------------------
# LOGGING
# -----------------------------------------------------------------------------
mkdir -p "$LOG_DIR"
log() {
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG_FILE"
}
log_section() {
echo "" | tee -a "$LOG_FILE"
echo "------------------------------------------------------------" | tee -a "$LOG_FILE"
log "$1"
echo "------------------------------------------------------------" | tee -a "$LOG_FILE"
}
# -----------------------------------------------------------------------------
# SECTION 1: RUN FSTRIM
# -----------------------------------------------------------------------------
log_section "Running fstrim on all mounted filesystems"
TRIM_RAW=$(fstrim -av 2>&1)
TRIM_EXIT=$?
TRIM_ROWS_HTML=""
TOTAL_BYTES=0
VOLUME_COUNT=0
while IFS= read -r line; do
if [[ "$line" =~ ^(/.+):[[:space:]]+(.+)[[:space:]]+\(([0-9]+)[[:space:]]bytes\) ]]; then
MOUNT="${BASH_REMATCH[1]}"
HUMAN="${BASH_REMATCH[2]}"
BYTES="${BASH_REMATCH[3]}"
TOTAL_BYTES=$((TOTAL_BYTES + BYTES))
VOLUME_COUNT=$((VOLUME_COUNT + 1))
log "Trimmed ${MOUNT}: ${HUMAN}"
TRIM_ROWS_HTML+="
<tr>
<td style=\"padding:8px 14px;border-bottom:1px solid #f3f4f6;font-size:13px;font-family:monospace;color:#111827;\">$MOUNT</td>
<td style=\"padding:8px 14px;border-bottom:1px solid #f3f4f6;font-size:13px;color:#374151;font-weight:600;\">$HUMAN freed</td>
</tr>"
fi
done <<< "$TRIM_RAW"
# Convert total bytes to human readable
if [ $TOTAL_BYTES -ge 1073741824 ]; then
TOTAL_HUMAN=$(awk "BEGIN {printf \"%.1f GiB\", $TOTAL_BYTES/1073741824}")
elif [ $TOTAL_BYTES -ge 1048576 ]; then
TOTAL_HUMAN=$(awk "BEGIN {printf \"%.1f MiB\", $TOTAL_BYTES/1048576}")
else
TOTAL_HUMAN="${TOTAL_BYTES} bytes"
fi
if [ $TRIM_EXIT -eq 0 ]; then
log "fstrim completed — $VOLUME_COUNT volume(s) · $TOTAL_HUMAN total freed"
add_section_row "TRIM Execution" "OK" "$VOLUME_COUNT volume(s) processed · $TOTAL_HUMAN total freed"
else
log "ERROR: fstrim exited with code $TRIM_EXIT"
add_section_row "TRIM Execution" "ERROR" "fstrim exited with code $TRIM_EXIT"
fi
# -----------------------------------------------------------------------------
# SECTION 2: SSD SMART HEALTH
# -----------------------------------------------------------------------------
log_section "Gathering SSD SMART health data"
SMART_ROWS_HTML=""
SMART_WARN=0
SMART_ERR=0
for DISK in $(lsblk -dpno NAME | grep -E '^/dev/sd|^/dev/nvme'); do
MODEL=$(smartctl -i "$DISK" 2>/dev/null | awk -F': ' '/Device Model|Model Number/{gsub(/^[ \t]+|[ \t]+$/, "", $2); print $2; exit}')
SERIAL=$(smartctl -i "$DISK" 2>/dev/null | awk -F': ' '/Serial Number/{gsub(/^[ \t]+|[ \t]+$/, "", $2); print $2; exit}')
CAPACITY=$(lsblk -dno SIZE "$DISK" 2>/dev/null)
HEALTH=$(smartctl -H "$DISK" 2>/dev/null | awk '/overall-health|test result/{print $NF}')
# Temperature: +0 forces numeric conversion, handling "32 (Min/Max 25/56)" Crucial format
TEMP=$(smartctl -A "$DISK" 2>/dev/null | awk '/Temperature_Celsius|Temperature_Internal/{print $10+0; exit}')
POWER_ON=$(smartctl -A "$DISK" 2>/dev/null | awk '/Power_On_Hours/{print $10; exit}')
# Wear: PNY=SSD_Life_Left, Crucial=Percent_Lifetime_Remain, Samsung/WD=Wear_Leveling_Count, Intel=Media_Wearout_Indicator
WEAR=$(smartctl -A "$DISK" 2>/dev/null | awk '/SSD_Life_Left|Percent_Lifetime_Remain|Wear_Leveling_Count|Media_Wearout_Indicator/{print $10; exit}')
# Reallocated: PNY=Bad_Blk_Ct_Lat/Erl (RAW_VALUE is "0/88" — take first number), Crucial=Reallocate_NAND_Blk_Cnt, others=Reallocated_Sector_Ct
REALLOCATED=$(smartctl -A "$DISK" 2>/dev/null | awk '/Bad_Blk_Ct_Lat\/Erl/{split($10,a,"/"); print a[1]; exit} /Reallocate_NAND_Blk_Cnt|Reallocated_Sector_Ct|Reallocated_Sector/{print $10; exit}')
# Pending: Crucial=Current_Pending_ECC_Cnt, others=Current_Pending_Sector (PNY has no equivalent — will show N/A)
PENDING=$(smartctl -A "$DISK" 2>/dev/null | awk '/Current_Pending_ECC_Cnt|Current_Pending_Sector/{print $10; exit}')
# Uncorrectable: Crucial=Reported_Uncorrect, others=Offline_Uncorrectable (PNY has no equivalent — will show N/A)
UNCORRECTABLE=$(smartctl -A "$DISK" 2>/dev/null | awk '/Reported_Uncorrect|Offline_Uncorrectable/{print $10; exit}')
[ -z "$MODEL" ] && MODEL="Unknown"
[ -z "$SERIAL" ] && SERIAL="Unknown"
[ -z "$HEALTH" ] && HEALTH="Unknown"
[ -z "$TEMP" ] && TEMP="N/A"
[ -z "$POWER_ON" ] && POWER_ON="N/A"
[ -z "$WEAR" ] && WEAR="N/A"
[ -z "$REALLOCATED" ] && REALLOCATED="N/A"
[ -z "$PENDING" ] && PENDING="N/A"
[ -z "$UNCORRECTABLE" ] && UNCORRECTABLE="N/A"
# Health colour
if [ "$HEALTH" = "PASSED" ]; then
HEALTH_COLOUR="#065f46"
HEALTH_BG="#d1fae5"
DISK_STATUS="OK"
else
HEALTH_COLOUR="#991b1b"
HEALTH_BG="#fee2e2"
DISK_STATUS="WARNING"
SMART_WARN=$((SMART_WARN + 1))
fi
# Flag non-zero reallocated/pending/uncorrectable sectors
SECTOR_NOTE=""
if [ "$REALLOCATED" != "N/A" ] && [ "$REALLOCATED" -gt 0 ] 2>/dev/null; then
SECTOR_NOTE+=" ⚠ Reallocated: $REALLOCATED"
SMART_WARN=$((SMART_WARN + 1))
[[ "$OVERALL_STATUS" == "SUCCESS" ]] && OVERALL_STATUS="WARNING"
fi
if [ "$PENDING" != "N/A" ] && [ "$PENDING" -gt 0 ] 2>/dev/null; then
SECTOR_NOTE+=" ⚠ Pending: $PENDING"
SMART_WARN=$((SMART_WARN + 1))
[[ "$OVERALL_STATUS" == "SUCCESS" ]] && OVERALL_STATUS="WARNING"
fi
if [ "$UNCORRECTABLE" != "N/A" ] && [ "$UNCORRECTABLE" -gt 0 ] 2>/dev/null; then
SECTOR_NOTE+=" 🚨 Uncorrectable: $UNCORRECTABLE"
SMART_ERR=$((SMART_ERR + 1))
OVERALL_STATUS="FAILED"
fi
log "Drive $DISK ($MODEL) — Health: $HEALTH | Temp: ${TEMP}°C | Wear: $WEAR | Power-on: ${POWER_ON}h${SECTOR_NOTE}"
SMART_ROWS_HTML+="
<tr>
<td colspan=\"2\" style=\"padding:10px 14px 4px;font-size:13px;font-weight:700;color:#111827;border-top:2px solid #e5e7eb;\">
$DISK &nbsp;<span style=\"font-weight:400;color:#6b7280;\">$MODEL &nbsp;·&nbsp; $CAPACITY &nbsp;·&nbsp; S/N: $SERIAL</span>
</td>
</tr>
<tr>
<td style=\"padding:4px 14px 4px 24px;font-size:12px;color:#6b7280;width:30%;\">Overall health</td>
<td style=\"padding:4px 14px;\"><span style=\"background:${HEALTH_BG};color:${HEALTH_COLOUR};padding:2px 10px;border-radius:20px;font-size:11px;font-weight:700;\">$HEALTH</span>${SECTOR_NOTE:+<span style=\"font-size:11px;color:#b45309;margin-left:8px;\">$SECTOR_NOTE</span>}</td>
</tr>
<tr>
<td style=\"padding:4px 14px 4px 24px;font-size:12px;color:#6b7280;\">Temperature</td>
<td style=\"padding:4px 14px;font-size:12px;color:#374151;font-weight:600;\">${TEMP}°C</td>
</tr>
<tr>
<td style=\"padding:4px 14px 4px 24px;font-size:12px;color:#6b7280;\">Power-on hours</td>
<td style=\"padding:4px 14px;font-size:12px;color:#374151;font-weight:600;\">${POWER_ON} hrs</td>
</tr>
<tr>
<td style=\"padding:4px 14px 4px 24px;font-size:12px;color:#6b7280;\">Wear leveling</td>
<td style=\"padding:4px 14px;font-size:12px;color:#374151;font-weight:600;\">$WEAR</td>
</tr>
<tr>
<td style=\"padding:4px 14px 8px 24px;font-size:12px;color:#6b7280;\">Sectors (realloc / pending / uncorrectable)</td>
<td style=\"padding:4px 14px 8px;font-size:12px;color:#374151;font-weight:600;\">$REALLOCATED &nbsp;/&nbsp; $PENDING &nbsp;/&nbsp; $UNCORRECTABLE</td>
</tr>"
done
if [ $SMART_ERR -gt 0 ]; then
add_section_row "SSD SMART Health" "ERROR" "$SMART_ERR drive(s) with uncorrectable sectors — immediate attention required"
elif [ $SMART_WARN -gt 0 ]; then
add_section_row "SSD SMART Health" "WARNING" "$SMART_WARN warning(s) — review drive details below"
else
add_section_row "SSD SMART Health" "OK" "All drives PASSED · no sector errors detected"
fi
# -----------------------------------------------------------------------------
# SECTION 3: FILESYSTEM USAGE
# -----------------------------------------------------------------------------
log_section "Filesystem usage"
FS_ROWS_HTML=""
FS_WARN=0
while IFS= read -r line; do
USE_PCT=$(echo "$line" | awk '{print $5}' | tr -d '%')
MOUNT_PT=$(echo "$line" | awk '{print $6}')
USED=$(echo "$line" | awk '{print $3}')
SIZE=$(echo "$line" | awk '{print $2}')
if [ "$USE_PCT" -ge 90 ] 2>/dev/null; then
ROW_COLOUR="#fee2e2"; PCT_COLOUR="#991b1b"
FS_WARN=$((FS_WARN + 1))
[[ "$OVERALL_STATUS" == "SUCCESS" ]] && OVERALL_STATUS="WARNING"
elif [ "$USE_PCT" -ge 75 ] 2>/dev/null; then
ROW_COLOUR="#fef3c7"; PCT_COLOUR="#92400e"
FS_WARN=$((FS_WARN + 1))
[[ "$OVERALL_STATUS" == "SUCCESS" ]] && OVERALL_STATUS="WARNING"
else
ROW_COLOUR="transparent"; PCT_COLOUR="#065f46"
fi
log "Filesystem $MOUNT_PT: ${USED} used of ${SIZE} (${USE_PCT}%)"
FS_ROWS_HTML+="
<tr style=\"background:${ROW_COLOUR};\">
<td style=\"padding:8px 14px;font-size:13px;font-family:monospace;color:#111827;\">$MOUNT_PT</td>
<td style=\"padding:8px 14px;font-size:13px;color:#374151;\">$USED used of $SIZE</td>
<td style=\"padding:8px 14px;font-size:13px;font-weight:700;color:${PCT_COLOUR};\">${USE_PCT}%</td>
</tr>"
done < <(df -h | grep -E '^/dev/')
if [ $FS_WARN -gt 0 ]; then
add_section_row "Filesystem Usage" "WARNING" "$FS_WARN volume(s) above 75% — review recommended"
else
add_section_row "Filesystem Usage" "OK" "All volumes within normal range"
fi
# -----------------------------------------------------------------------------
# ROTATE OLD LOGS
# -----------------------------------------------------------------------------
find "$LOG_DIR" -name "fstrim-*.log" -mtime +${KEEP_LOGS} -delete
# -----------------------------------------------------------------------------
# SYSTEM CONTEXT
# -----------------------------------------------------------------------------
OMV_VER=$(dpkg -l openmediavault 2>/dev/null | awk '/^ii/{print $3}' || echo "Unknown")
KERNEL_VER=$(uname -r)
HOST_IP=$(hostname -I | awk '{print $1}')
UPTIME_STR=$(uptime -p)
LOAD_STR=$(uptime | awk -F'load average:' '{print $2}' | xargs)
MEM_USED=$(free -h | awk '/^Mem:/{print $3}')
MEM_TOTAL=$(free -h | awk '/^Mem:/{print $2}')
NEXT_RUN=$(systemctl show fstrim.timer --property=NextElapseUSecRealtime 2>/dev/null \
| awk -F= '{print $2}' || echo "See: systemctl status fstrim.timer")
log_section "Report complete — Overall: $OVERALL_STATUS"
# -----------------------------------------------------------------------------
# BUILD HTML EMAIL (style: KingDezigns monitor)
# -----------------------------------------------------------------------------
# Overall banner & badge
case "$OVERALL_STATUS" in
SUCCESS)
BADGE_HTML="<div style=\"background:#1a7f4b;color:white;padding:10px 20px;border-radius:8px;font-size:14px;font-weight:700;\">✅ SUCCESS</div>"
BANNER_HTML="<tr><td style=\"background:#1a7f4b;padding:12px 32px;\"><span style=\"color:white;font-size:14px;font-weight:600;\">✅ &nbsp;TRIM completed successfully — no action required.</span></td></tr>"
;;
WARNING)
BADGE_HTML="<div style=\"background:#b45309;color:white;padding:10px 20px;border-radius:8px;font-size:14px;font-weight:700;\">⚠️ WARNING</div>"
BANNER_HTML="<tr><td style=\"background:#b45309;padding:12px 32px;\"><span style=\"color:white;font-size:14px;font-weight:600;\">⚠️ &nbsp;TRIM completed with warnings — review recommended.</span></td></tr>"
;;
FAILED)
BADGE_HTML="<div style=\"background:#b91c1c;color:white;padding:10px 20px;border-radius:8px;font-size:14px;font-weight:700;\">🚨 FAILED</div>"
BANNER_HTML="<tr><td style=\"background:#b91c1c;padding:12px 32px;\"><span style=\"color:white;font-size:14px;font-weight:600;\">🚨 &nbsp;TRIM run failed — immediate attention required!</span></td></tr>"
;;
esac
HTML_BODY=$(cat <<EOF
<!DOCTYPE html>
<html lang="en">
<head><meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1"></head>
<body style="margin:0;padding:0;background:#f1f5f9;font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,sans-serif;">
<table width="100%" cellspacing="0" cellpadding="0" style="background:#f1f5f9;padding:30px 0;">
<tr><td align="center">
<table width="680" cellspacing="0" cellpadding="0" style="max-width:680px;width:100%;">
<!-- Header -->
<tr><td style="background:#0f172a;border-radius:10px 10px 0 0;padding:28px 32px;">
<table width="100%" cellspacing="0" cellpadding="0">
<tr>
<td>
<div style="font-size:11px;color:#94a3b8;letter-spacing:1px;text-transform:uppercase;">SSD Maintenance Monitor</div>
<div style="font-size:26px;font-weight:800;color:white;margin-top:4px;">💽 $HOSTNAME_LABEL</div>
<div style="font-size:13px;color:#94a3b8;margin-top:4px;">$REPORT_TIME &nbsp;|&nbsp; Weekly TRIM · $WARNING_COUNT warning(s) · $ERROR_COUNT error(s)</div>
</td>
<td align="right">$BADGE_HTML</td>
</tr>
</table>
</td></tr>
<!-- Status banner -->
$BANNER_HTML
<!-- Section results table -->
<tr><td style="padding:28px 24px 8px;">
<div style="background:white;border-radius:10px;box-shadow:0 1px 6px rgba(0,0,0,.10);overflow:hidden;border:1px solid #e5e7eb;">
<div style="background:#f8fafc;padding:14px 22px;border-bottom:1px solid #e5e7eb;">
<span style="font-size:13px;font-weight:700;color:#374151;text-transform:uppercase;letter-spacing:.5px;">📋 Run Summary</span>
</div>
<table width="100%" cellspacing="0" cellpadding="0">
<tr style="background:#f9fafb;">
<th style="text-align:left;padding:8px 14px;font-size:10px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;font-weight:600;border-bottom:1px solid #e5e7eb;width:30%;">Section</th>
<th style="text-align:left;padding:8px 14px;font-size:10px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;font-weight:600;border-bottom:1px solid #e5e7eb;width:15%;">Status</th>
<th style="text-align:left;padding:8px 14px;font-size:10px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;font-weight:600;border-bottom:1px solid #e5e7eb;">Detail</th>
</tr>
$SECTION_ROWS_HTML
</table>
</div>
</td></tr>
<!-- TRIM volumes card -->
<tr><td style="padding:0 24px 8px;">
<div style="background:white;border-radius:10px;border:1px solid #e5e7eb;padding:18px 22px;">
<div style="font-size:13px;font-weight:700;color:#374151;margin-bottom:12px;text-transform:uppercase;letter-spacing:.5px;">✂️ TRIM Results — $TOTAL_HUMAN Total Freed</div>
<table width="100%" cellspacing="0" cellpadding="0">
<tr style="background:#f9fafb;">
<th style="text-align:left;padding:8px 14px;font-size:10px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;font-weight:600;border-bottom:1px solid #e5e7eb;">Mount point</th>
<th style="text-align:left;padding:8px 14px;font-size:10px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;font-weight:600;border-bottom:1px solid #e5e7eb;">Space freed</th>
</tr>
$TRIM_ROWS_HTML
</table>
</div>
</td></tr>
<!-- SMART health card -->
<tr><td style="padding:0 24px 8px;">
<div style="background:white;border-radius:10px;border:1px solid #e5e7eb;padding:18px 22px;">
<div style="font-size:13px;font-weight:700;color:#374151;margin-bottom:12px;text-transform:uppercase;letter-spacing:.5px;">🔬 SSD SMART Health</div>
<table width="100%" cellspacing="0" cellpadding="0">
$SMART_ROWS_HTML
</table>
</div>
</td></tr>
<!-- Filesystem usage card -->
<tr><td style="padding:0 24px 8px;">
<div style="background:white;border-radius:10px;border:1px solid #e5e7eb;padding:18px 22px;">
<div style="font-size:13px;font-weight:700;color:#374151;margin-bottom:12px;text-transform:uppercase;letter-spacing:.5px;">💾 Filesystem Usage</div>
<table width="100%" cellspacing="0" cellpadding="0">
<tr style="background:#f9fafb;">
<th style="text-align:left;padding:8px 14px;font-size:10px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;font-weight:600;border-bottom:1px solid #e5e7eb;width:35%;">Mount point</th>
<th style="text-align:left;padding:8px 14px;font-size:10px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;font-weight:600;border-bottom:1px solid #e5e7eb;">Usage</th>
<th style="text-align:left;padding:8px 14px;font-size:10px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;font-weight:600;border-bottom:1px solid #e5e7eb;width:15%;">Used %</th>
</tr>
$FS_ROWS_HTML
</table>
</div>
</td></tr>
<!-- System info card -->
<tr><td style="padding:0 24px 28px;">
<div style="background:white;border-radius:10px;border:1px solid #e5e7eb;padding:18px 22px;">
<div style="font-size:13px;font-weight:700;color:#374151;margin-bottom:10px;text-transform:uppercase;letter-spacing:.5px;">🖥️ System Info</div>
<table width="100%" cellspacing="0" cellpadding="0">
<tr>
<td style="font-size:13px;color:#6b7280;padding:4px 0;width:30%;">OMV</td>
<td style="font-size:13px;color:#111827;font-weight:600;">${OMV_VER}</td>
<td style="font-size:13px;color:#6b7280;padding:4px 0;width:30%;">Kernel</td>
<td style="font-size:13px;color:#111827;font-weight:600;">${KERNEL_VER}</td>
</tr>
<tr><td colspan="4" style="height:6px;"></td></tr>
<tr>
<td style="font-size:13px;color:#6b7280;padding:4px 0;">IP Address</td>
<td style="font-size:13px;color:#111827;font-weight:600;font-family:monospace;">${HOST_IP}</td>
<td style="font-size:13px;color:#6b7280;padding:4px 0;">Uptime</td>
<td style="font-size:13px;color:#111827;font-weight:600;">${UPTIME_STR}</td>
</tr>
<tr><td colspan="4" style="height:6px;"></td></tr>
<tr>
<td style="font-size:13px;color:#6b7280;padding:4px 0;">Load average</td>
<td style="font-size:13px;color:#111827;font-weight:600;">${LOAD_STR}</td>
<td style="font-size:13px;color:#6b7280;padding:4px 0;">Memory</td>
<td style="font-size:13px;color:#111827;font-weight:600;">${MEM_USED} / ${MEM_TOTAL}</td>
</tr>
<tr><td colspan="4" style="height:6px;"></td></tr>
<tr>
<td style="font-size:13px;color:#6b7280;padding:4px 0;">Log file</td>
<td colspan="3" style="font-size:13px;color:#111827;font-weight:600;font-family:monospace;">${LOG_FILE}</td>
</tr>
<tr><td colspan="4" style="height:6px;"></td></tr>
<tr>
<td style="font-size:13px;color:#6b7280;padding:4px 0;">Next TRIM run</td>
<td colspan="3" style="font-size:13px;color:#111827;font-weight:600;">${NEXT_RUN}</td>
</tr>
</table>
</div>
</td></tr>
<!-- Footer -->
<tr><td style="background:#0f172a;border-radius:0 0 10px 10px;padding:16px 32px;text-align:center;">
<span style="font-size:11px;color:#64748b;">SSD Maintenance Monitor &nbsp;|&nbsp; $HOSTNAME_LABEL &nbsp;|&nbsp; KingDezigns Infrastructure</span>
</td></tr>
</table>
</td></tr>
</table>
</body>
</html>
EOF
)
# -----------------------------------------------------------------------------
# SEND EMAIL
# -----------------------------------------------------------------------------
SUBJECT="[TRIM] $HOSTNAME_LABEL — $OVERALL_STATUS | $REPORT_TIME"
{
echo "To: $REPORT_TO"
echo "From: $REPORT_FROM"
echo "Subject: $SUBJECT"
echo "MIME-Version: 1.0"
echo "Content-Type: text/html; charset=UTF-8"
echo ""
echo "$HTML_BODY"
} | sendmail -t
log "Email sent to $REPORT_TO — Overall status: $OVERALL_STATUS"
log "===== fstrim-report.sh finished ====="

695
omv/nas16-backup.sh Normal file
View file

@ -0,0 +1,695 @@
#!/bin/bash
# =============================================================================
# NAS16 System Backup Script
# =============================================================================
# Backs up OMV config, system /etc, web server configs (Apache/PHP),
# MariaDB/MySQL databases, Webmin config, and scheduled jobs.
# Writes a single dated .tar.gz archive to the NAS data drive.
# Retains the last 10 backups (~30 days), auto-removes older ones.
# Sends an HTML email report on completion (via OMV Postfix / Gmail relay).
#
# NOTE: Website files stored on NAS drives are NOT backed up here —
# they are covered separately via NAS-to-NAS redundancy.
#
# Schedule via OMV Scheduled Jobs (cron):
# 0 2 */3 * * (2:00 AM every 3 days)
#
# Backup destination: /export/kingdezignsnas-16/Backups/NAS16/
# Script location: /usr/scripts/omv/nas16-backup.sh
# =============================================================================
set -uo pipefail
# -----------------------------------------------------------------------------
# CONFIGURATION
# -----------------------------------------------------------------------------
HOSTNAME_LABEL=$(hostname -s 2>/dev/null || echo "NAS16")
REPORT_TO="rufus.king@kingdezigns.com"
REPORT_FROM="backup-monitor@nas16.local" # Display only — Postfix uses your Gmail relay
BACKUP_ROOT="/export/kingdezignsnas-16/Backups/NAS16"
TMP_BASE="/export/kingdezignsnas-16/Backups/.tmp"
DATE=$(date +%Y-%m-%d)
TIMESTAMP=$(date +%Y-%m-%d_%H-%M-%S)
REPORT_TIME=$(date "+%Y-%m-%d %H:%M:%S")
ARCHIVE_NAME="nas16-backup-${DATE}.tar.gz"
TMP_DIR="${TMP_BASE}/nas16-backup-${TIMESTAMP}"
RETAIN_COUNT=10
LOG_FILE="${BACKUP_ROOT}/backup.log"
# MariaDB/MySQL connection — uses OS root auth (no password needed when run as root)
DB_BACKUP_DIR_NAME="databases"
# -----------------------------------------------------------------------------
# STATUS TRACKING
# Each section appends a row to SECTION_ROWS_HTML and sets OVERALL_STATUS.
# OVERALL_STATUS: SUCCESS | WARNING | FAILED
# -----------------------------------------------------------------------------
OVERALL_STATUS="SUCCESS"
SECTION_ROWS_HTML=""
WARNING_COUNT=0
ERROR_COUNT=0
# Append a result row to the section table
# Usage: add_section_row "Section name" "STATUS" "Detail message"
# STATUS: OK | WARNING | ERROR | SKIPPED
add_section_row() {
local section="$1" status="$2" detail="$3"
case "$status" in
OK)
BADGE="<span style=\"background:#d1fae5;color:#065f46;padding:3px 10px;border-radius:20px;font-size:11px;font-weight:700;\">✓ OK</span>"
;;
WARNING)
BADGE="<span style=\"background:#fef3c7;color:#92400e;padding:3px 10px;border-radius:20px;font-size:11px;font-weight:700;\">⚠ WARNING</span>"
WARNING_COUNT=$((WARNING_COUNT + 1))
[[ "$OVERALL_STATUS" == "SUCCESS" ]] && OVERALL_STATUS="WARNING"
;;
ERROR)
BADGE="<span style=\"background:#fee2e2;color:#991b1b;padding:3px 10px;border-radius:20px;font-size:11px;font-weight:700;\">✗ ERROR</span>"
ERROR_COUNT=$((ERROR_COUNT + 1))
OVERALL_STATUS="FAILED"
;;
SKIPPED)
BADGE="<span style=\"background:#f3f4f6;color:#6b7280;padding:3px 10px;border-radius:20px;font-size:11px;font-weight:700;\">— SKIPPED</span>"
;;
esac
SECTION_ROWS_HTML+="
<tr>
<td style=\"padding:9px 14px;border-bottom:1px solid #f3f4f6;font-size:13px;color:#111827;font-weight:600;\">$section</td>
<td style=\"padding:9px 14px;border-bottom:1px solid #f3f4f6;\">$BADGE</td>
<td style=\"padding:9px 14px;border-bottom:1px solid #f3f4f6;font-size:12px;color:#6b7280;\">$detail</td>
</tr>"
}
# -----------------------------------------------------------------------------
# LOGGING
# -----------------------------------------------------------------------------
log() {
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG_FILE"
}
log_section() {
echo "" | tee -a "$LOG_FILE"
echo "------------------------------------------------------------" | tee -a "$LOG_FILE"
log "$1"
echo "------------------------------------------------------------" | tee -a "$LOG_FILE"
}
# -----------------------------------------------------------------------------
# PRE-FLIGHT CHECKS
# -----------------------------------------------------------------------------
log_section "NAS16 Backup Starting"
if [ ! -d "/export/kingdezignsnas-16" ]; then
log "ERROR: /export/kingdezignsnas-16 is not mounted or accessible. Aborting."
add_section_row "Pre-flight check" "ERROR" "/export/kingdezignsnas-16 not mounted or accessible"
# Send failure email immediately and exit
OVERALL_STATUS="FAILED"
# (email block at bottom handles this via trap — see END section)
exit 1
fi
mkdir -p "$BACKUP_ROOT"
mkdir -p "$TMP_DIR"
log "Temporary working directory: $TMP_DIR"
# -----------------------------------------------------------------------------
# SECTION 1: OMV CONFIGURATION
# -----------------------------------------------------------------------------
log_section "Backing up OMV configuration"
OMV_DIR="${TMP_DIR}/omv-config"
mkdir -p "$OMV_DIR"
OMV_STATUS="OK"
OMV_DETAIL=""
# omv-confbak is optional — if present, export the structured XML as a bonus.
# The raw config.xml copy below is the primary backup and is fully sufficient on its own.
if command -v omv-confbak &>/dev/null; then
if omv-confbak "${OMV_DIR}/omv-config.xml" 2>/dev/null; then
log "omv-confbak completed successfully"
OMV_DETAIL="omv-confbak exported + raw config.xml copied"
else
log "WARNING: omv-confbak failed or returned non-zero — raw config.xml still captured"
OMV_DETAIL="omv-confbak failed (non-fatal) + raw config.xml copied"
fi
else
log "omv-confbak not present — using raw config.xml (this is normal)"
fi
if [ -f /etc/openmediavault/config.xml ]; then
cp /etc/openmediavault/config.xml "${OMV_DIR}/config.xml.raw"
log "Raw OMV config.xml copied"
[[ -z "$OMV_DETAIL" ]] && OMV_DETAIL="raw config.xml copied"
else
log "WARNING: /etc/openmediavault/config.xml not found — OMV config not backed up"
OMV_STATUS="WARNING"
OMV_DETAIL="config.xml not found at /etc/openmediavault/config.xml"
fi
add_section_row "OMV Configuration" "$OMV_STATUS" "$OMV_DETAIL"
# -----------------------------------------------------------------------------
# SECTION 2: SYSTEM /etc
# -----------------------------------------------------------------------------
log_section "Backing up /etc (system configuration)"
ETC_DIR="${TMP_DIR}/etc"
mkdir -p "$ETC_DIR"
if rsync -a --quiet \
--exclude='/etc/mtab' \
--exclude='/etc/fstab.d' \
--exclude='/etc/blkid.tab' \
/etc/ "${ETC_DIR}/" 2>/dev/null; then
log "/etc backed up successfully"
add_section_row "System /etc" "OK" "rsync completed successfully"
else
log "WARNING: /etc rsync encountered errors"
add_section_row "System /etc" "WARNING" "rsync completed with errors"
fi
# -----------------------------------------------------------------------------
# SECTION 3: APACHE CONFIGURATION
# -----------------------------------------------------------------------------
log_section "Backing up Apache configuration"
APACHE_DIR="${TMP_DIR}/apache"
mkdir -p "$APACHE_DIR"
APACHE_WARN=0
for DIR in /etc/apache2/sites-available /etc/apache2/sites-enabled \
/etc/apache2/mods-enabled /etc/apache2/conf-available \
/etc/apache2/conf-enabled /etc/apache2; do
if [ -d "$DIR" ]; then
rsync -a --quiet "$DIR/" "${APACHE_DIR}/$(basename "$DIR")/" \
&& log "Apache $(basename "$DIR") backed up" \
|| { log "WARNING: Apache $(basename "$DIR") rsync encountered errors"; APACHE_WARN=$((APACHE_WARN+1)); }
else
log "WARNING: $DIR not found — skipping"
APACHE_WARN=$((APACHE_WARN+1))
fi
done
if [ -f /etc/apache2/apache2.conf ]; then
cp /etc/apache2/apache2.conf "${APACHE_DIR}/apache2.conf"
log "apache2.conf copied"
fi
if [ "$APACHE_WARN" -eq 0 ]; then
add_section_row "Apache Configuration" "OK" "All config directories backed up"
else
add_section_row "Apache Configuration" "WARNING" "$APACHE_WARN director(ies) missing or failed"
fi
# -----------------------------------------------------------------------------
# SECTION 4: PHP CONFIGURATION
# -----------------------------------------------------------------------------
log_section "Backing up PHP configuration"
PHP_DIR="${TMP_DIR}/php"
mkdir -p "$PHP_DIR"
PHP_FOUND=0
PHP_VERSIONS=""
PHP_WARN=0
for PHP_VER_DIR in /etc/php/*/; do
if [ -d "$PHP_VER_DIR" ]; then
PHP_VER=$(basename "$PHP_VER_DIR")
rsync -a --quiet "$PHP_VER_DIR" "${PHP_DIR}/${PHP_VER}/" \
&& log "PHP ${PHP_VER} config backed up" \
|| { log "WARNING: PHP ${PHP_VER} rsync encountered errors"; PHP_WARN=$((PHP_WARN+1)); }
PHP_FOUND=1
PHP_VERSIONS="${PHP_VERSIONS} ${PHP_VER}"
fi
done
if [ "$PHP_FOUND" -eq 0 ]; then
log "WARNING: No PHP config directories found under /etc/php/ — skipping"
add_section_row "PHP Configuration" "SKIPPED" "No PHP installations found under /etc/php/"
elif [ "$PHP_WARN" -gt 0 ]; then
add_section_row "PHP Configuration" "WARNING" "Versions found:${PHP_VERSIONS} — $PHP_WARN version(s) had rsync errors"
else
add_section_row "PHP Configuration" "OK" "Versions backed up:${PHP_VERSIONS}"
fi
# -----------------------------------------------------------------------------
# SECTION 5: MARIADB / MYSQL DATABASES
# -----------------------------------------------------------------------------
log_section "Backing up MariaDB/MySQL databases"
DB_DIR="${TMP_DIR}/${DB_BACKUP_DIR_NAME}"
mkdir -p "$DB_DIR"
DB_COUNT=0
DB_FAIL=0
if command -v mysqldump &>/dev/null; then
DB_CLIENT="mysqldump"
elif command -v mariadb-dump &>/dev/null; then
DB_CLIENT="mariadb-dump"
else
log "WARNING: mysqldump/mariadb-dump not found — skipping database backup"
DB_CLIENT=""
fi
if [ -n "$DB_CLIENT" ]; then
DB_LIST=$(mysql --defaults-file=/etc/mysql/debian.cnf \
-e "SHOW DATABASES;" 2>/dev/null \
| grep -Ev "^(Database|information_schema|performance_schema|sys|mysql)$" \
|| true)
if [ -z "$DB_LIST" ]; then
log "No user databases found — nothing to dump"
add_section_row "MariaDB / MySQL" "SKIPPED" "No user databases found"
else
while IFS= read -r DB_NAME; do
[ -z "$DB_NAME" ] && continue
$DB_CLIENT \
--defaults-file=/etc/mysql/debian.cnf \
--single-transaction \
--routines \
--triggers \
--events \
"$DB_NAME" > "${DB_DIR}/${DB_NAME}.sql" \
&& { log "Database dumped: ${DB_NAME}.sql"; DB_COUNT=$((DB_COUNT+1)); } \
|| { log "WARNING: dump failed for database: ${DB_NAME}"; DB_FAIL=$((DB_FAIL+1)); }
done <<< "$DB_LIST"
log "Total databases dumped: ${DB_COUNT}"
gzip "${DB_DIR}"/*.sql 2>/dev/null && log "SQL dumps compressed" || true
if [ "$DB_FAIL" -eq 0 ]; then
add_section_row "MariaDB / MySQL" "OK" "$DB_COUNT database(s) dumped and compressed"
else
add_section_row "MariaDB / MySQL" "WARNING" "$DB_COUNT dumped · $DB_FAIL failed"
fi
fi
else
add_section_row "MariaDB / MySQL" "SKIPPED" "mysqldump / mariadb-dump not found"
fi
# -----------------------------------------------------------------------------
# SECTION 6: WEBMIN CONFIGURATION
# -----------------------------------------------------------------------------
log_section "Backing up Webmin configuration"
WEBMIN_DIR="${TMP_DIR}/webmin"
mkdir -p "$WEBMIN_DIR"
if [ -d /etc/webmin ]; then
if rsync -a --quiet /etc/webmin/ "${WEBMIN_DIR}/" 2>/dev/null; then
log "Webmin config backed up"
add_section_row "Webmin Configuration" "OK" "rsync from /etc/webmin completed"
else
log "WARNING: Webmin rsync encountered errors"
add_section_row "Webmin Configuration" "WARNING" "rsync completed with errors"
fi
else
log "WARNING: /etc/webmin not found — skipping (Webmin may not be installed)"
add_section_row "Webmin Configuration" "SKIPPED" "/etc/webmin not found — Webmin may not be installed"
fi
# -----------------------------------------------------------------------------
# SECTION 7: CRONTAB & SCHEDULED JOBS
# -----------------------------------------------------------------------------
log_section "Backing up crontab and scheduled jobs"
CRON_DIR="${TMP_DIR}/cron"
mkdir -p "$CRON_DIR"
CRON_WARN=0
crontab -l > "${CRON_DIR}/root-crontab.txt" 2>/dev/null \
&& log "Root crontab saved" \
|| { log "No root crontab found"; CRON_WARN=$((CRON_WARN+1)); }
if [ -d /var/spool/cron/crontabs ]; then
cp -r /var/spool/cron/crontabs/ "${CRON_DIR}/crontabs/" \
&& log "All crontabs copied" \
|| { log "WARNING: crontabs copy failed"; CRON_WARN=$((CRON_WARN+1)); }
fi
if [ -d /etc/cron.d ]; then
cp -r /etc/cron.d/ "${CRON_DIR}/cron.d/" \
&& log "cron.d copied" \
|| { log "WARNING: cron.d copy failed"; CRON_WARN=$((CRON_WARN+1)); }
fi
if [ "$CRON_WARN" -eq 0 ]; then
add_section_row "Crontab / Scheduled Jobs" "OK" "root crontab, crontabs/, cron.d/ saved"
else
add_section_row "Crontab / Scheduled Jobs" "WARNING" "$CRON_WARN item(s) missing or failed to copy"
fi
# -----------------------------------------------------------------------------
# SECTION 8: BACKUP MANIFEST
# -----------------------------------------------------------------------------
log_section "Generating backup manifest"
APACHE_VER=$(apache2 -v 2>/dev/null | grep "Server version" | awk '{print $3}' || echo "Unknown")
PHP_VER_RUNTIME=$(php -r 'echo PHP_VERSION;' 2>/dev/null || echo "Unknown")
DB_VER=$(mysql --defaults-file=/etc/mysql/debian.cnf \
-e "SELECT VERSION();" 2>/dev/null | tail -1 || echo "Unknown")
WEBMIN_VER=$(dpkg -l webmin 2>/dev/null | awk '/^ii/{print $3}' || echo "Unknown")
OMV_VER=$(dpkg -l openmediavault 2>/dev/null | awk '/^ii/{print $3}' || echo "Unknown")
KERNEL_VER=$(uname -r)
HOST_IP=$(hostname -I | awk '{print $1}')
MANIFEST="${TMP_DIR}/MANIFEST.txt"
cat > "$MANIFEST" <<MANIFEST_EOF
=============================================================
NAS16 BACKUP MANIFEST
=============================================================
Hostname: $(hostname)
Date: $(date)
Kernel: ${KERNEL_VER}
OMV Version: ${OMV_VER}
Apache: ${APACHE_VER}
PHP: ${PHP_VER_RUNTIME}
MariaDB/MySQL: ${DB_VER}
Webmin: ${WEBMIN_VER}
IP Address: ${HOST_IP}
CONTENTS:
omv-config/ - OMV exported config + raw config.xml
etc/ - Full /etc system configuration
apache/ - Apache2 virtual hosts, mods, site configs
php/ - PHP ini files and FPM pool configs (all versions)
databases/ - mysqldump of all user databases (.sql.gz per DB)
webmin/ - Webmin configuration (/etc/webmin)
cron/ - Root crontab and scheduled jobs
MANIFEST.txt - This file
NOT INCLUDED (by design):
Website files on NAS drives — covered by NAS-to-NAS redundancy
RESTORE NOTES:
1. Flash fresh Raspberry Pi OS Lite (64-bit) to SD card
2. SSH in and run OS update: sudo apt-get update && sudo apt-get upgrade -y
3. Install OMV via installScript
4. Enable PCIe in /boot/firmware/config.txt (after arm_boost=1):
dtparam=pciex1
dtparam=pciex1_gen=3
Then reboot.
5. Install ZFS: sudo apt-get install -y zfsutils-linux
6. Import ZFS pool: sudo zpool import -f kingdezignsnas-16
7. Extract backup archive to /tmp/nas16-restore/
8. Restore /etc: sudo rsync -a etc/ /etc/
9. Restore OMV config and run omv-salt deploy run all
10. Reboot — verify OMV loads correctly
11. Install Apache, PHP, MariaDB, Webmin, Adminer
12. Restore Apache config from apache/
13. Restore PHP config from php/
14. Restore all databases from databases/*.sql.gz
15. Restore Webmin config from webmin/
16. Restore crontab: sudo crontab cron/root-crontab.txt
17. Restart services: apache2, php-fpm, mariadb, webmin
18. Verify all websites load and databases are accessible
=============================================================
MANIFEST_EOF
log "Manifest written"
add_section_row "Backup Manifest" "OK" "MANIFEST.txt written to archive"
# -----------------------------------------------------------------------------
# SECTION 9: CREATE ARCHIVE
# -----------------------------------------------------------------------------
log_section "Creating compressed archive"
cd "$TMP_BASE"
if tar -czf "${BACKUP_ROOT}/${ARCHIVE_NAME}" -C "$TMP_BASE" "$(basename "$TMP_DIR")" 2>/dev/null; then
log "Archive created: ${BACKUP_ROOT}/${ARCHIVE_NAME}"
ARCHIVE_SIZE=$(du -sh "${BACKUP_ROOT}/${ARCHIVE_NAME}" | cut -f1)
log "Archive size: ${ARCHIVE_SIZE}"
add_section_row "Archive Creation" "OK" "${ARCHIVE_NAME} · ${ARCHIVE_SIZE}"
else
log "ERROR: Archive creation failed"
ARCHIVE_SIZE="N/A"
add_section_row "Archive Creation" "ERROR" "tar failed — archive not created"
rm -rf "$TMP_DIR"
OVERALL_STATUS="FAILED"
fi
# -----------------------------------------------------------------------------
# SECTION 10: CLEANUP TEMP FILES
# -----------------------------------------------------------------------------
log_section "Cleaning up temporary files"
if rm -rf "$TMP_DIR" 2>/dev/null; then
log "Temporary directory removed"
else
log "WARNING: Failed to remove temp directory: $TMP_DIR"
fi
rmdir "$TMP_BASE" 2>/dev/null && log ".tmp directory removed (empty)" || true
# -----------------------------------------------------------------------------
# SECTION 11: ROTATE OLD BACKUPS
# -----------------------------------------------------------------------------
log_section "Rotating old backups (keeping last ${RETAIN_COUNT})"
BACKUP_LIST=$(ls -1t "${BACKUP_ROOT}"/nas16-backup-*.tar.gz 2>/dev/null || true)
BACKUP_COUNT=$(echo "$BACKUP_LIST" | grep -c . || true)
DELETED_COUNT=0
if [ "$BACKUP_COUNT" -gt "$RETAIN_COUNT" ]; then
DELETE_LIST=$(echo "$BACKUP_LIST" | tail -n +$((RETAIN_COUNT + 1)))
while IFS= read -r OLD_BACKUP; do
rm -f "$OLD_BACKUP"
log "Removed old backup: $(basename "$OLD_BACKUP")"
DELETED_COUNT=$((DELETED_COUNT+1))
done <<< "$DELETE_LIST"
add_section_row "Backup Rotation" "OK" "Kept $RETAIN_COUNT most recent · removed $DELETED_COUNT old archive(s)"
else
log "Backup count (${BACKUP_COUNT}) within retention limit — no rotation needed"
add_section_row "Backup Rotation" "OK" "$BACKUP_COUNT / $RETAIN_COUNT slots used — no rotation needed"
fi
# -----------------------------------------------------------------------------
# DONE — log summary
# -----------------------------------------------------------------------------
log_section "Backup Complete"
log "Archive: ${BACKUP_ROOT}/${ARCHIVE_NAME}"
log "Size: ${ARCHIVE_SIZE:-N/A}"
log "Status: ${OVERALL_STATUS}"
log ""
# -----------------------------------------------------------------------------
# BUILD & SEND HTML EMAIL (pattern: nas08_zfs_scrub.sh)
# -----------------------------------------------------------------------------
# Overall banner & badge
case "$OVERALL_STATUS" in
SUCCESS)
BADGE_HTML="<div style=\"background:#1a7f4b;color:white;padding:10px 20px;border-radius:8px;font-size:14px;font-weight:700;\">✅ SUCCESS</div>"
BANNER_HTML="<tr><td style=\"background:#1a7f4b;padding:12px 32px;\"><span style=\"color:white;font-size:14px;font-weight:600;\">✅ &nbsp;Backup completed successfully — no action required.</span></td></tr>"
;;
WARNING)
BADGE_HTML="<div style=\"background:#b45309;color:white;padding:10px 20px;border-radius:8px;font-size:14px;font-weight:700;\">⚠️ WARNING</div>"
BANNER_HTML="<tr><td style=\"background:#b45309;padding:12px 32px;\"><span style=\"color:white;font-size:14px;font-weight:600;\">⚠️ &nbsp;Backup completed with warnings — review recommended.</span></td></tr>"
;;
FAILED)
BADGE_HTML="<div style=\"background:#b91c1c;color:white;padding:10px 20px;border-radius:8px;font-size:14px;font-weight:700;\">🚨 FAILED</div>"
BANNER_HTML="<tr><td style=\"background:#b91c1c;padding:12px 32px;\"><span style=\"color:white;font-size:14px;font-weight:600;\">🚨 &nbsp;Backup failed — immediate attention required!</span></td></tr>"
;;
esac
# Archive info card — shown only when archive was created
if [[ "${ARCHIVE_SIZE:-N/A}" != "N/A" ]]; then
ARCHIVE_CARD="
<tr><td style=\"padding:0 24px 8px;\">
<div style=\"background:white;border-radius:10px;border:1px solid #e5e7eb;padding:18px 22px;\">
<div style=\"font-size:13px;font-weight:700;color:#374151;margin-bottom:12px;text-transform:uppercase;letter-spacing:.5px;\">🗜️ Archive</div>
<table width=\"100%\" cellspacing=\"0\" cellpadding=\"0\">
<tr>
<td style=\"width:50%;padding:8px 12px;background:#f9fafb;border-radius:6px;text-align:center;\">
<div style=\"font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;\">Filename</div>
<div style=\"font-family:monospace;font-size:13px;font-weight:600;color:#111827;margin-top:4px;\">$ARCHIVE_NAME</div>
</td>
<td style=\"width:4%;\"></td>
<td style=\"width:20%;padding:8px 12px;background:#f9fafb;border-radius:6px;text-align:center;\">
<div style=\"font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;\">Size</div>
<div style=\"font-size:20px;font-weight:700;color:#111827;margin-top:2px;\">$ARCHIVE_SIZE</div>
</td>
<td style=\"width:4%;\"></td>
<td style=\"width:20%;padding:8px 12px;background:#f9fafb;border-radius:6px;text-align:center;\">
<div style=\"font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;\">Retained</div>
<div style=\"font-size:20px;font-weight:700;color:#111827;margin-top:2px;\">$BACKUP_COUNT / $RETAIN_COUNT</div>
</td>
</tr>
</table>
</div>
</td></tr>"
else
ARCHIVE_CARD=""
fi
# Software versions card
VERSIONS_CARD="
<tr><td style=\"padding:0 24px 8px;\">
<div style=\"background:white;border-radius:10px;border:1px solid #e5e7eb;padding:18px 22px;\">
<div style=\"font-size:13px;font-weight:700;color:#374151;margin-bottom:12px;text-transform:uppercase;letter-spacing:.5px;\">🖥️ System Versions</div>
<table width=\"100%\" cellspacing=\"0\" cellpadding=\"0\">
<tr>
<td style=\"font-size:13px;color:#6b7280;padding:4px 0;\">OMV</td>
<td style=\"font-size:13px;color:#111827;font-weight:600;\">${OMV_VER}</td>
<td style=\"font-size:13px;color:#6b7280;padding:4px 0;\">Kernel</td>
<td style=\"font-size:13px;color:#111827;font-weight:600;\">${KERNEL_VER}</td>
</tr>
<tr><td colspan=\"4\" style=\"height:4px;\"></td></tr>
<tr>
<td style=\"font-size:13px;color:#6b7280;padding:4px 0;\">Apache</td>
<td style=\"font-size:13px;color:#111827;font-weight:600;\">${APACHE_VER}</td>
<td style=\"font-size:13px;color:#6b7280;padding:4px 0;\">PHP</td>
<td style=\"font-size:13px;color:#111827;font-weight:600;\">${PHP_VER_RUNTIME}</td>
</tr>
<tr><td colspan=\"4\" style=\"height:4px;\"></td></tr>
<tr>
<td style=\"font-size:13px;color:#6b7280;padding:4px 0;\">MariaDB / MySQL</td>
<td style=\"font-size:13px;color:#111827;font-weight:600;\">${DB_VER}</td>
<td style=\"font-size:13px;color:#6b7280;padding:4px 0;\">Webmin</td>
<td style=\"font-size:13px;color:#111827;font-weight:600;\">${WEBMIN_VER}</td>
</tr>
<tr><td colspan=\"4\" style=\"height:4px;\"></td></tr>
<tr>
<td style=\"font-size:13px;color:#6b7280;padding:4px 0;\">IP Address</td>
<td style=\"font-size:13px;color:#111827;font-weight:600;font-family:monospace;\">${HOST_IP}</td>
<td></td><td></td>
</tr>
</table>
</div>
</td></tr>"
# Compose full HTML email
HTML_BODY=$(cat <<EOF
<!DOCTYPE html>
<html lang="en">
<head><meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1"></head>
<body style="margin:0;padding:0;background:#f1f5f9;font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,sans-serif;">
<table width="100%" cellspacing="0" cellpadding="0" style="background:#f1f5f9;padding:30px 0;">
<tr><td align="center">
<table width="680" cellspacing="0" cellpadding="0" style="max-width:680px;width:100%;">
<!-- Header -->
<tr><td style="background:#0f172a;border-radius:10px 10px 0 0;padding:28px 32px;">
<table width="100%" cellspacing="0" cellpadding="0">
<tr>
<td>
<div style="font-size:11px;color:#94a3b8;letter-spacing:1px;text-transform:uppercase;">System Backup Monitor</div>
<div style="font-size:26px;font-weight:800;color:white;margin-top:4px;">💾 $HOSTNAME_LABEL</div>
<div style="font-size:13px;color:#94a3b8;margin-top:4px;">$REPORT_TIME &nbsp;|&nbsp; 7 sections · $WARNING_COUNT warning(s) · $ERROR_COUNT error(s)</div>
</td>
<td align="right">$BADGE_HTML</td>
</tr>
</table>
</td></tr>
<!-- Status banner -->
$BANNER_HTML
<!-- Section results table -->
<tr><td style="padding:28px 24px 8px;">
<div style="background:white;border-radius:10px;box-shadow:0 1px 6px rgba(0,0,0,.10);overflow:hidden;border:1px solid #e5e7eb;">
<div style="background:#f8fafc;padding:14px 22px;border-bottom:1px solid #e5e7eb;">
<span style="font-size:13px;font-weight:700;color:#374151;text-transform:uppercase;letter-spacing:.5px;">📋 Backup Section Results</span>
</div>
<table width="100%" cellspacing="0" cellpadding="0">
<tr style="background:#f9fafb;">
<th style="text-align:left;padding:8px 14px;font-size:10px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;font-weight:600;border-bottom:1px solid #e5e7eb;width:30%;">Section</th>
<th style="text-align:left;padding:8px 14px;font-size:10px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;font-weight:600;border-bottom:1px solid #e5e7eb;width:15%;">Status</th>
<th style="text-align:left;padding:8px 14px;font-size:10px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;font-weight:600;border-bottom:1px solid #e5e7eb;">Detail</th>
</tr>
$SECTION_ROWS_HTML
</table>
</div>
</td></tr>
<!-- Archive info card -->
$ARCHIVE_CARD
<!-- System versions card -->
$VERSIONS_CARD
<!-- Run summary card -->
<tr><td style="padding:0 24px 28px;">
<div style="background:white;border-radius:10px;border:1px solid #e5e7eb;padding:18px 22px;">
<div style="font-size:13px;font-weight:700;color:#374151;margin-bottom:10px;text-transform:uppercase;letter-spacing:.5px;">📊 Run Summary</div>
<table width="100%" cellspacing="0" cellpadding="0">
<tr>
<td style="font-size:13px;color:#6b7280;">Host</td>
<td style="font-size:13px;color:#111827;font-weight:600;">$HOSTNAME_LABEL</td>
</tr>
<tr><td colspan="2" style="height:6px;"></td></tr>
<tr>
<td style="font-size:13px;color:#6b7280;">Run Time</td>
<td style="font-size:13px;color:#111827;font-weight:600;">$REPORT_TIME</td>
</tr>
<tr><td colspan="2" style="height:6px;"></td></tr>
<tr>
<td style="font-size:13px;color:#6b7280;">Archive</td>
<td style="font-family:monospace;font-size:13px;color:#111827;font-weight:600;">${ARCHIVE_NAME}</td>
</tr>
<tr><td colspan="2" style="height:6px;"></td></tr>
<tr>
<td style="font-size:13px;color:#6b7280;">Archive Size</td>
<td style="font-size:13px;color:#111827;font-weight:600;">${ARCHIVE_SIZE:-N/A}</td>
</tr>
<tr><td colspan="2" style="height:6px;"></td></tr>
<tr>
<td style="font-size:13px;color:#6b7280;">Backups on Disk</td>
<td style="font-size:13px;color:#111827;font-weight:600;">$BACKUP_COUNT of $RETAIN_COUNT max retained</td>
</tr>
<tr><td colspan="2" style="height:6px;"></td></tr>
<tr>
<td style="font-size:13px;color:#6b7280;">Warnings</td>
<td style="font-size:13px;font-weight:700;color:$([ "$WARNING_COUNT" -gt 0 ] && echo "#b45309" || echo "#1a7f4b");">$WARNING_COUNT</td>
</tr>
<tr><td colspan="2" style="height:6px;"></td></tr>
<tr>
<td style="font-size:13px;color:#6b7280;">Errors</td>
<td style="font-size:13px;font-weight:700;color:$([ "$ERROR_COUNT" -gt 0 ] && echo "#b91c1c" || echo "#1a7f4b");">$ERROR_COUNT</td>
</tr>
<tr><td colspan="2" style="height:6px;"></td></tr>
<tr>
<td style="font-size:13px;color:#6b7280;">Overall Status</td>
<td style="font-size:13px;font-weight:700;color:$(
case "$OVERALL_STATUS" in
SUCCESS) echo "#1a7f4b" ;;
WARNING) echo "#b45309" ;;
*) echo "#b91c1c" ;;
esac
);">$OVERALL_STATUS</td>
</tr>
</table>
</div>
</td></tr>
<!-- Footer -->
<tr><td style="background:#0f172a;border-radius:0 0 10px 10px;padding:16px 32px;text-align:center;">
<span style="font-size:11px;color:#64748b;">Automated Backup Monitor &nbsp;|&nbsp; $HOSTNAME_LABEL &nbsp;|&nbsp; KingDezigns Infrastructure</span>
</td></tr>
</table>
</td></tr>
</table>
</body>
</html>
EOF
)
# Send email via OMV Postfix (Gmail relay)
SUBJECT="[Backup] $HOSTNAME_LABEL — $OVERALL_STATUS | $REPORT_TIME"
{
echo "To: $REPORT_TO"
echo "From: $REPORT_FROM"
echo "Subject: $SUBJECT"
echo "MIME-Version: 1.0"
echo "Content-Type: text/html; charset=UTF-8"
echo ""
echo "$HTML_BODY"
} | sendmail -t
log "Email sent to $REPORT_TO — Overall status: $OVERALL_STATUS"
log "===== NAS16 Backup script finished ====="

510
zfs/NAS16-zfs_pools.html Normal file
View file

@ -0,0 +1,510 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>KingDezigns — NAS16 ZFS Pool Maintenance Scripts</title>
<link rel="preconnect" href="https://fonts.googleapis.com">
<link href="https://fonts.googleapis.com/css2?family=IBM+Plex+Mono:wght@400;500&family=IBM+Plex+Sans:wght@400;500&display=swap" rel="stylesheet">
<style>
*, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
:root {
--bg: #f7f6f2;
--surface: #ffffff;
--surface-alt: #f1f0eb;
--border: #e2e0d8;
--border-strong: #c8c5ba;
--text-primary: #1a1917;
--text-secondary: #6b6960;
--text-muted: #9b9890;
--green-bg: #eaf3de;
--green-border: #97c459;
--green-text: #2d5a0e;
--red-bg: #fcebeb;
--red-border: #f09595;
--red-text: #7a1f1f;
--amber-bg: #faeeda;
--amber-border: #fac775;
--amber-text: #633806;
--coral-bg: #faece7;
--coral-border: #f0997b;
--coral-text: #4a1b0c;
--mono: 'IBM Plex Mono', monospace;
--sans: 'IBM Plex Sans', sans-serif;
}
body {
font-family: var(--sans);
font-size: 14px;
background: var(--bg);
color: var(--text-primary);
line-height: 1.7;
padding: 2.5rem 1.5rem;
}
.page { max-width: 860px; margin: 0 auto; }
header {
border-bottom: 1px solid var(--border-strong);
padding-bottom: 1.25rem;
margin-bottom: 2rem;
}
header h1 {
font-family: var(--mono);
font-size: 18px;
font-weight: 500;
letter-spacing: -0.02em;
margin-bottom: 6px;
}
.meta {
font-size: 12px;
color: var(--text-muted);
font-family: var(--mono);
display: flex;
gap: 16px;
flex-wrap: wrap;
}
.section { margin-bottom: 2.25rem; }
.section-title {
font-family: var(--mono);
font-size: 11px;
font-weight: 500;
text-transform: uppercase;
letter-spacing: 0.08em;
color: var(--text-muted);
margin-bottom: 1rem;
display: flex;
align-items: center;
gap: 10px;
}
.section-title::before {
content: attr(data-num);
background: var(--text-primary);
color: var(--bg);
font-size: 10px;
width: 18px;
height: 18px;
border-radius: 50%;
display: inline-flex;
align-items: center;
justify-content: center;
flex-shrink: 0;
}
.section-title::after {
content: '';
flex: 1;
height: 1px;
background: var(--border);
}
p { margin-bottom: 0.75rem; color: var(--text-primary); }
p:last-child { margin-bottom: 0; }
.badge-row { display: flex; gap: 8px; flex-wrap: wrap; margin-bottom: 1rem; }
.badge {
font-family: var(--mono);
font-size: 11px;
padding: 3px 10px;
border-radius: 3px;
border: 1px solid var(--border);
background: var(--surface);
color: var(--text-secondary);
}
.badge.red { background: var(--red-bg); color: var(--red-text); border-color: var(--red-border); }
.badge.green { background: var(--green-bg); color: var(--green-text); border-color: var(--green-border); }
.badge.amber { background: var(--amber-bg); color: var(--amber-text); border-color: var(--amber-border); }
.step {
background: var(--surface);
border: 1px solid var(--border);
border-radius: 6px;
padding: 1rem 1.25rem;
margin-bottom: 0.6rem;
}
.step-header {
font-family: var(--mono);
font-size: 12px;
font-weight: 500;
color: var(--text-secondary);
margin-bottom: 0.6rem;
display: flex;
align-items: center;
gap: 8px;
}
.step-num {
background: var(--surface-alt);
border: 1px solid var(--border);
border-radius: 50%;
width: 20px;
height: 20px;
display: inline-flex;
align-items: center;
justify-content: center;
font-size: 10px;
flex-shrink: 0;
color: var(--text-muted);
}
pre {
background: var(--surface-alt);
border: 1px solid var(--border);
border-radius: 4px;
padding: 10px 12px;
font-family: var(--mono);
font-size: 12px;
overflow-x: auto;
margin: 0.5rem 0;
white-space: pre;
line-height: 1.6;
}
code {
font-family: var(--mono);
font-size: 12px;
background: var(--surface-alt);
padding: 1px 5px;
border-radius: 3px;
border: 1px solid var(--border);
}
.why {
font-size: 12px;
color: var(--text-muted);
margin-top: 0.4rem;
padding-left: 2px;
}
.why::before { content: '↳ why: '; font-family: var(--mono); }
.note {
font-size: 12px;
color: var(--text-secondary);
margin-top: 0.5rem;
padding-left: 2px;
}
.note::before { content: '📌 '; }
.result { font-size: 12px; margin-top: 0.35rem; padding-left: 2px; }
.result.pass { color: var(--green-text); }
.result.pass::before { content: '✓ '; font-family: var(--mono); }
.result.fail { color: var(--red-text); }
.result.fail::before { content: '✗ '; font-family: var(--mono); }
.result.neutral { color: var(--text-secondary); }
.result.neutral::before { content: '→ '; font-family: var(--mono); }
.callout { border-radius: 6px; padding: 1rem 1.25rem; margin-bottom: 0.75rem; }
.callout.coral { background: var(--coral-bg); border: 1px solid var(--coral-border); }
.callout.coral p { color: var(--coral-text); }
.callout.coral pre { background: #fdf0eb; border-color: var(--coral-border); }
.callout.amber { background: var(--amber-bg); border: 1px solid var(--amber-border); }
.callout.amber p { color: var(--amber-text); }
.callout.amber pre { background: #fdf5e6; border-color: var(--amber-border); }
.callout.amber code { background: #fdf5e6; border-color: var(--amber-border); color: var(--amber-text); }
.callout.green { background: var(--green-bg); border: 1px solid var(--green-border); }
.callout.green p { color: var(--green-text); }
.callout.green pre { background: #d8efc0; border-color: var(--green-border); }
.clue-label {
font-family: var(--mono);
font-size: 10px;
font-weight: 500;
text-transform: uppercase;
letter-spacing: 0.07em;
color: var(--amber-text);
margin-bottom: 8px;
}
.phase-label {
font-family: var(--mono);
font-size: 10px;
font-weight: 500;
text-transform: uppercase;
letter-spacing: 0.08em;
color: var(--text-primary);
background: var(--surface-alt);
border: 1px solid var(--border-strong);
border-radius: 4px;
padding: 5px 12px;
margin: 1.5rem 0 0.6rem 0;
display: inline-block;
}
.script-box { background: var(--surface); border: 1px solid var(--border); border-radius: 6px; overflow: hidden; margin-bottom: 1rem; }
.script-row { padding: 0.75rem 1.25rem; border-bottom: 1px solid var(--border); }
.script-row:last-child { border-bottom: none; }
.script-label { font-family: var(--mono); font-size: 10px; text-transform: uppercase; letter-spacing: 0.07em; color: var(--text-muted); margin-bottom: 6px; }
.script-row pre { margin: 0; }
table { width: 100%; border-collapse: collapse; font-size: 13px; background: var(--surface); border: 1px solid var(--border); border-radius: 6px; overflow: hidden; }
th { text-align: left; padding: 8px 14px; font-family: var(--mono); font-size: 10px; text-transform: uppercase; letter-spacing: 0.06em; color: var(--text-muted); background: var(--surface-alt); border-bottom: 1px solid var(--border); font-weight: 500; }
td { padding: 9px 14px; border-bottom: 1px solid var(--border); vertical-align: top; line-height: 1.5; }
tr:last-child td { border-bottom: none; }
.footer {
font-family: var(--mono);
font-size: 11px;
color: var(--text-muted);
margin-top: 2.5rem;
padding-top: 1rem;
border-top: 1px solid var(--border);
display: flex;
justify-content: space-between;
flex-wrap: wrap;
gap: 8px;
}
</style>
</head>
<body>
<div class="page">
<header>
<h1>KingDezigns — NAS16 ZFS Pool Maintenance Scripts</h1>
<div class="meta">
<span>📅 2026-05-12</span>
<span>🖥 NAS16 / Raspberry Pi 5</span>
<span>⚙️ OpenMediaVault (OMV)</span>
<span>🌐 Apache · PHP · MariaDB · Webmin</span>
<span>💾 ZFS RAIDZ2 — Penta SATA HAT</span>
</div>
</header>
<!-- SECTION 1 — THE GOAL -->
<div class="section">
<div class="section-title" data-num="1">The goal</div>
<p>NAS16 serves live websites via Apache/PHP and hosts their databases in MariaDB. A software or hardware failure — corrupt SD card, bad OMV update, runaway script — would take down all hosted sites and lose database state with no documented recovery path and no saved configuration.</p>
<div class="badge-row">
<span class="badge amber">ZFS Scrubbing</span>
<span class="badge amber">ZFS Reporting</span>
</div>
<p>The goal is to create regular maintenance to the ZFS pools for this NAS. This is done to protect and keep the pools healthy.</p>
<div class="callout amber" style="margin-top:1rem;">
<div class="clue-label">⚠ Critical — store this document off NAS16</div>
<p>Keep a copy of this document on your workstation, a USB drive, or in print. If NAS16 is down you cannot read files stored on it.</p>
</div>
</div>
<!-- SECTION 2 — Script to Create ZFS report -->
<div class="section">
<div class="section-title" data-num="2">Script to Create ZFS report</div>
<div class="step">
<div class="step-header"><span class="step-num">1</span> Create the folders on NAS16</div>
<p class="note">This script gets the current ZFS pool status and reports it back to the administrator via email. Use these commands to setup the scripts on the NAS.</p>
<pre>ssh rufusking@192.168.150.40
sudo mkdir /usr/scripts/zfs
sudo nano nas16_zfs_report.sh</pre>
</div>
<div class="step">
<div class="step-header"><span class="step-num">2</span> Copy the content of the entire nas16_zfs_report.sh script and paste it in the terminal window. Save (CTRL + X) and commit (Y) the changes</div>
<pre><a href="nas16_zfs_report.sh">nas16_zfs_report.sh</a></pre>
</div>
<div class="step">
<div class="step-header"><span class="step-num">3</span> Make the script executable — NAS16</div>
<pre>chmod +x /usr/scripts/zfs/nas16_zfs_report.sh</pre>
</div>
<div class="step">
<div class="step-header"><span class="step-num">4</span> Run a test report — NAS16</div>
<pre>sudo /usr/scripts/zfs/nas16_zfs_report.sh</pre>
<p>Watch for the report in your inbox</p>
</div>
</div>
<!-- SECTION 3 — Script to perform a ZFS pool scrub -->
<div class="section">
<div class="section-title" data-num="3">Script to perform a ZFS pool scrub</div>
<div class="step">
<div class="step-header"><span class="step-num">1</span> Create the folders on NAS16</div>
<p class="note">This script initiates a ZFS pool scrub and reports back the status to the administrator via email. Use these commands to setup this script on the NAS.</p>
<pre>ssh rufusking@192.168.150.40
sudo mkdir /usr/scripts/zfs
sudo nano nas16_zfs_scrub.sh</pre>
</div>
<div class="step">
<div class="step-header"><span class="step-num">2</span> Copy the content of the entire nas16_zfs_scrub.sh script and paste it in the terminal window. Save (CTRL + X) and commit (Y) the changes</div>
<pre><a href="nas16_zfs_scrub.sh">nas16_zfs_scrub.sh</a></pre>
</div>
<div class="step">
<div class="step-header"><span class="step-num">3</span> Make the script executable — NAS16</div>
<pre>chmod +x /usr/scripts/zfs/nas16_zfs_scrub.sh</pre>
</div>
<div class="step">
<div class="step-header"><span class="step-num">4</span> Run a test report — NAS16</div>
<pre>sudo /usr/scripts/zfs/nas16_zfs_report.sh</pre>
<p>Watch for the report in your inbox</p>
</div>
</div>
<!-- SECTION 4 — KEY DIFFERENCES FROM NAS08 -->
<div class="section">
<div class="section-title" data-num="4">Key differences from NAS08</div>
<table>
<thead>
<tr><th>Area</th><th>NAS08</th><th>NAS16</th></tr>
</thead>
<tbody>
<tr>
<td>Primary role</td>
<td>Docker containers (Pi-hole, Plex, Vaultwarden, Nextcloud, NPM)</td>
<td>Web server (Apache/PHP), databases (MariaDB), Webmin admin</td>
</tr>
<tr>
<td>What's backed up beyond /etc + OMV</td>
<td>Docker Compose files, Pi-hole data, Plex config/metadata</td>
<td>Apache vhosts, PHP config, MariaDB dumps, Webmin config</td>
</tr>
<tr>
<td>Website files</td>
<td>N/A</td>
<td>Intentionally excluded — stored on NAS drives, covered by redundancy</td>
</tr>
<tr>
<td>Database backup</td>
<td>None (no databases)</td>
<td>Full mysqldump of all user databases, .sql.gz per database</td>
</tr>
<tr>
<td>Backup destination</td>
<td>/export/kingdezigns-all/Backups/NAS08/</td>
<td>/export/kingdezignsnas-16/Backups/NAS16/</td>
</tr>
<tr>
<td>ZFS pool name</td>
<td>kingdezignsnas</td>
<td>kingdezignsnas-16 <em>(assumed — verify with <code>sudo zpool list</code>)</em></td>
</tr>
<tr>
<td>Recovery phases</td>
<td>9 phases, 34 steps</td>
<td>10 phases, 40 steps</td>
</tr>
</tbody>
</table>
</div>
<!-- SECTION 5 — NOTES & ASSUMPTIONS -->
<div class="section">
<div class="section-title" data-num="5">Notes &amp; assumptions to verify</div>
<div class="callout amber">
<div class="clue-label">⚠ Verify these before first run</div>
<p>The script was written based on information provided and follows Raspberry Pi OS / Debian conventions. Verify the following on NAS16 before treating any backup as production-ready.</p>
</div>
<table>
<thead>
<tr><th>Item</th><th>Assumed value</th><th>How to verify</th></tr>
</thead>
<tbody>
<tr>
<td>ZFS pool name</td>
<td><code>kingdezignsnas-16</code></td>
<td><code>sudo zpool list</code></td>
</tr>
<tr>
<td>Backup destination path</td>
<td><code>/export/kingdezignsnas-16/Backups/NAS16/</code></td>
<td><code>ls /export/</code> — confirm share name matches</td>
</tr>
<tr>
<td>MariaDB auth method</td>
<td>Uses <code>/etc/mysql/debian.cnf</code> (maintenance account, no password when run as root)</td>
<td>Run: <code>sudo mysql --defaults-file=/etc/mysql/debian.cnf -e "SHOW DATABASES;"</code> — should work without password</td>
</tr>
<tr>
<td>PHP version(s)</td>
<td>Auto-detected from <code>/etc/php/*/</code></td>
<td><code>php -v</code> and <code>ls /etc/php/</code></td>
</tr>
<tr>
<td>Apache config location</td>
<td><code>/etc/apache2/</code> (standard Debian)</td>
<td><code>apache2 -V</code> — confirms config path</td>
</tr>
<tr>
<td>Webmin config location</td>
<td><code>/etc/webmin/</code></td>
<td><code>ls /etc/webmin/</code></td>
</tr>
</tbody>
</table>
</div>
<!-- SECTION 6 — NETWORK MAP UPDATES -->
<div class="section">
<div class="section-title" data-num="6">Network map — items to update</div>
<table>
<thead>
<tr><th>Item</th><th>Old value</th><th>New value</th></tr>
</thead>
<tbody>
<tr>
<td>NAS16 — Backup status</td>
<td><code>None</code></td>
<td><code>Automated — every 3 days — 30-day retention</code></td>
</tr>
<tr>
<td>NAS16 — Backup destination</td>
<td><code>None</code></td>
<td><code>/export/kingdezignsnas-16/Backups/NAS16/</code></td>
</tr>
<tr>
<td>NAS16 — Backup script</td>
<td><code>None</code></td>
<td><code>/usr/scripts/omv/nas16-backup.sh</code></td>
</tr>
<tr>
<td>NAS16 — PCIe requirement</td>
<td><code>not documented</code></td>
<td><code>dtparam=pciex1 + dtparam=pciex1_gen=3 required in /boot/firmware/config.txt on fresh OS</code></td>
</tr>
<tr>
<td>NAS16 — Web stack</td>
<td><code>not documented</code></td>
<td><code>Apache · PHP · MariaDB · Webmin · Adminer</code></td>
</tr>
<tr>
<td>NAS16 — Recovery plan</td>
<td><code>None</code></td>
<td><code>NAS16-Backup-Summary.html — 40 steps across 10 phases + ZFS troubleshooting</code></td>
</tr>
</tbody>
</table>
</div>
<div class="footer">
<span>KingDezigns Home Network</span>
<span>Generated 2026-05-12</span>
</div>
</div>
</body>
</html>

351
zfs/nas16_zfs_report.sh Normal file
View file

@ -0,0 +1,351 @@
#!/usr/bin/env bash
# =============================================================================
# ZFS Pool Health Report — NAS16
# Sends an HTML status email to rufus.king@kingdezigns.com
# Cron example: 0 7 * * * /opt/scripts/zfs_report_nas16.sh
#
# Commands executed per pool:
# zpool status -v <pool> — full status + verbose per-file error listing
# zpool list <pool> — pool-level size / capacity / health
# zfs list -r <pool> — per-dataset breakdown (used, avail, refer, mount)
# =============================================================================
set -euo pipefail
# ── Configuration ─────────────────────────────────────────────────────────────
TARGET_HOST="NAS16"
REPORT_TO="rufus.king@kingdezigns.com"
REPORT_FROM="zfs-monitor@${TARGET_HOST,,}.local"
SCRUB_MAX_AGE_DAYS=8 # Warn if last scrub is older than this
CAP_WARN=80 # Capacity % that triggers WARNING
CAP_CRIT=90 # Capacity % that triggers CRITICAL
# ──────────────────────────────────────────────────────────────────────────────
NOW=$(date '+%Y-%m-%d %H:%M:%S')
HOSTNAME_REAL=$(hostname -s 2>/dev/null || echo "$TARGET_HOST")
# ── HTML escape helper ────────────────────────────────────────────────────────
html_escape() { sed 's/&/\&amp;/g; s/</\&lt;/g; s/>/\&gt;/g; s/"/\&quot;/g'; }
# ── Collect pool list ─────────────────────────────────────────────────────────
POOL_LIST=$(zpool list -H -o name 2>/dev/null) || { echo "ERROR: zpool not found"; exit 1; }
OVERALL_STATUS="HEALTHY"
POOL_BLOCKS=""
POOL_COUNT=0
PROBLEM_COUNT=0
# ── Per-pool loop ─────────────────────────────────────────────────────────────
while IFS= read -r POOL; do
[[ -z "$POOL" ]] && continue
POOL_COUNT=$((POOL_COUNT + 1))
# 1. zpool status -v ── verbose: includes per-file permanent error listing
STATUS_RAW=$(zpool status -v "$POOL" 2>&1)
# 2. zpool list ── raw (-p) for math, human for display
ZPOOL_RAW=$(zpool list -H -p -o name,size,alloc,free,frag,cap,health "$POOL" 2>&1)
ZPOOL_HUM=$(zpool list -H -o name,size,alloc,free,frag,cap,health "$POOL" 2>&1)
HEALTH=$(echo "$ZPOOL_RAW" | awk '{print $7}')
CAP_RAW=$(echo "$ZPOOL_RAW" | awk '{print $6}')
SIZE_H=$(echo "$ZPOOL_HUM" | awk '{print $2}')
ALLOC_H=$(echo "$ZPOOL_HUM" | awk '{print $3}')
FREE_H=$(echo "$ZPOOL_HUM" | awk '{print $4}')
FRAG_H=$(echo "$ZPOOL_HUM" | awk '{print $5}')
CAP_H=$(echo "$ZPOOL_HUM" | awk '{print $6}')
# 3. zfs list -r ── per-dataset: name, used, avail, refer, mountpoint
ZFS_LIST_RAW=$(zfs list -r -H -o name,used,avail,refer,mountpoint "$POOL" 2>&1)
# Build dataset table rows
DATASET_ROWS=""
DATASET_COUNT=0
while IFS=$'\t' read -r DS_NAME DS_USED DS_AVAIL DS_REFER DS_MOUNT; do
[[ -z "$DS_NAME" ]] && continue
DATASET_COUNT=$((DATASET_COUNT + 1))
(( DATASET_COUNT % 2 == 0 )) && ROW_SHADE="#f9fafb" || ROW_SHADE="white"
DS_NAME_ESC=$(echo "$DS_NAME" | html_escape)
DS_MOUNT_ESC=$(echo "$DS_MOUNT" | html_escape)
DATASET_ROWS+="
<tr style='background:${ROW_SHADE};'>
<td style='padding:7px 10px;font-size:12px;color:#111827;font-family:monospace;border-bottom:1px solid #f3f4f6;'>${DS_NAME_ESC}</td>
<td style='padding:7px 10px;font-size:12px;color:#374151;text-align:right;border-bottom:1px solid #f3f4f6;'>${DS_USED}</td>
<td style='padding:7px 10px;font-size:12px;color:#374151;text-align:right;border-bottom:1px solid #f3f4f6;'>${DS_AVAIL}</td>
<td style='padding:7px 10px;font-size:12px;color:#374151;text-align:right;border-bottom:1px solid #f3f4f6;'>${DS_REFER}</td>
<td style='padding:7px 10px;font-size:12px;color:#6b7280;font-family:monospace;border-bottom:1px solid #f3f4f6;'>${DS_MOUNT_ESC}</td>
</tr>"
done <<< "$ZFS_LIST_RAW"
DATASET_TABLE="
<details style='margin-top:14px;'>
<summary style='cursor:pointer;font-size:12px;color:#6b7280;font-weight:600;letter-spacing:.3px;'>&#9654; Datasets / Filesystems — zfs list -r &nbsp;(${DATASET_COUNT} entries)</summary>
<div style='margin-top:8px;overflow-x:auto;border-radius:6px;border:1px solid #e5e7eb;'>
<table width='100%' cellspacing='0' cellpadding='0' style='border-collapse:collapse;'>
<thead>
<tr style='background:#0f172a;'>
<th style='padding:8px 10px;font-size:11px;color:#94a3b8;text-align:left;text-transform:uppercase;letter-spacing:.5px;'>Dataset</th>
<th style='padding:8px 10px;font-size:11px;color:#94a3b8;text-align:right;text-transform:uppercase;letter-spacing:.5px;'>Used</th>
<th style='padding:8px 10px;font-size:11px;color:#94a3b8;text-align:right;text-transform:uppercase;letter-spacing:.5px;'>Available</th>
<th style='padding:8px 10px;font-size:11px;color:#94a3b8;text-align:right;text-transform:uppercase;letter-spacing:.5px;'>Referenced</th>
<th style='padding:8px 10px;font-size:11px;color:#94a3b8;text-align:left;text-transform:uppercase;letter-spacing:.5px;'>Mountpoint</th>
</tr>
</thead>
<tbody>${DATASET_ROWS}
</tbody>
</table>
</div>
</details>"
# ── Scrub parse ────────────────────────────────────────────────────────────
SCRUB_LINE=$(echo "$STATUS_RAW" | grep -E "scan:|scrub" | head -1 || true)
if echo "$SCRUB_LINE" | grep -q "scrub repaired"; then
LAST_SCRUB=$(echo "$SCRUB_LINE" | grep -oP '\w{3} \w{3} +\d+ \d+:\d+:\d+ \d{4}' | head -1 || echo "unknown")
ERRORS_SCRUB=$(echo "$SCRUB_LINE" | grep -oP '\d+ errors' | head -1 || echo "0 errors")
SCRUB_STATUS="Completed"
elif echo "$SCRUB_LINE" | grep -q "in progress"; then
LAST_SCRUB="In Progress"; SCRUB_STATUS="Running"; ERRORS_SCRUB="—"
elif echo "$SCRUB_LINE" | grep -q "none requested"; then
LAST_SCRUB="Never"; SCRUB_STATUS="Never Run"; ERRORS_SCRUB="—"
else
LAST_SCRUB=$(echo "$SCRUB_LINE" | sed 's/.*on //' | sed 's/ with.*//' || echo "unknown")
SCRUB_STATUS="Completed"
ERRORS_SCRUB=$(echo "$SCRUB_LINE" | grep -oP '\d+ errors' | head -1 || echo "0 errors")
fi
# ── Severity logic ─────────────────────────────────────────────────────────
POOL_SEVERITY="ok"
POOL_LABEL="HEALTHY"
if [[ "$HEALTH" != "ONLINE" ]]; then
POOL_SEVERITY="critical"; POOL_LABEL="$HEALTH"
OVERALL_STATUS="CRITICAL"; PROBLEM_COUNT=$((PROBLEM_COUNT + 1))
fi
CAP_NUM=${CAP_RAW//%/}; CAP_NUM=${CAP_NUM%%.*}
if [[ "$CAP_NUM" =~ ^[0-9]+$ ]]; then
if (( CAP_NUM >= CAP_CRIT )); then
POOL_SEVERITY="critical"; POOL_LABEL="CRITICAL — CAPACITY ${CAP_H}"
OVERALL_STATUS="CRITICAL"
[[ "$HEALTH" == "ONLINE" ]] && PROBLEM_COUNT=$((PROBLEM_COUNT + 1))
elif (( CAP_NUM >= CAP_WARN )); then
[[ "$POOL_SEVERITY" == "ok" ]] && POOL_SEVERITY="warning"
[[ "$POOL_LABEL" == "HEALTHY" ]] && POOL_LABEL="WARNING — CAPACITY ${CAP_H}"
[[ "$OVERALL_STATUS" == "HEALTHY" ]] && OVERALL_STATUS="WARNING"
fi
fi
if [[ "$LAST_SCRUB" == "Never" ]]; then
[[ "$POOL_SEVERITY" == "ok" ]] && POOL_SEVERITY="warning"
[[ "$POOL_LABEL" == "HEALTHY" ]] && POOL_LABEL="WARNING — NO SCRUB"
[[ "$OVERALL_STATUS" == "HEALTHY" ]] && OVERALL_STATUS="WARNING"
fi
case "$POOL_SEVERITY" in
ok) BADGE_BG="#1a7f4b"; BADGE_TEXT="white"; ROW_BG="#f0fdf4" ;;
warning) BADGE_BG="#b45309"; BADGE_TEXT="white"; ROW_BG="#fffbeb" ;;
critical) BADGE_BG="#b91c1c"; BADGE_TEXT="white"; ROW_BG="#fff1f2" ;;
esac
# ── Error blocks ───────────────────────────────────────────────────────────
# a) Pool/vdev state errors
ERROR_LINES=$(echo "$STATUS_RAW" \
| grep -E "(DEGRADED|FAULTED|OFFLINE|REMOVED|UNAVAIL|errors:)" \
| grep -v "errors: No known data errors" || true)
# b) Permanent per-file errors exposed by -v flag
VERBOSE_ERRORS=$(echo "$STATUS_RAW" \
| awk '/Permanent errors have been detected/,0' \
| grep -v "^$" || true)
ERROR_HTML=""
if [[ -n "$ERROR_LINES" ]]; then
ERROR_LINES_ESC=$(echo "$ERROR_LINES" | head -30 | html_escape)
ERROR_HTML+="
<div style='margin-top:10px;padding:10px 14px;background:#fff1f2;border-left:4px solid #b91c1c;border-radius:4px;'>
<div style='font-size:11px;font-weight:700;color:#b91c1c;text-transform:uppercase;letter-spacing:.5px;margin-bottom:6px;'>&#9888; Pool / vdev Errors</div>
<pre style='margin:0;font-family:monospace;font-size:12px;color:#7f1d1d;white-space:pre-wrap;'>${ERROR_LINES_ESC}</pre>
</div>"
fi
if [[ -n "$VERBOSE_ERRORS" ]]; then
VERBOSE_ESC=$(echo "$VERBOSE_ERRORS" | head -50 | html_escape)
ERROR_HTML+="
<div style='margin-top:10px;padding:10px 14px;background:#fef3c7;border-left:4px solid #d97706;border-radius:4px;'>
<div style='font-size:11px;font-weight:700;color:#92400e;text-transform:uppercase;letter-spacing:.5px;margin-bottom:6px;'>&#128196; Affected Files — zpool status -v</div>
<pre style='margin:0;font-family:monospace;font-size:12px;color:#78350f;white-space:pre-wrap;'>${VERBOSE_ESC}</pre>
</div>"
fi
# ── vdev config tree ───────────────────────────────────────────────────────
VDEV_TREE=$(echo "$STATUS_RAW" \
| sed -n '/config:/,/errors:/p' \
| grep -v "^$" | head -60 | html_escape || true)
# ── Assemble pool card ─────────────────────────────────────────────────────
POOL_BLOCKS+="
<div style='background:white;border-radius:10px;box-shadow:0 1px 6px rgba(0,0,0,.10);margin-bottom:28px;overflow:hidden;border:1px solid #e5e7eb;'>
<div style='background:${ROW_BG};padding:16px 22px;border-bottom:1px solid #e5e7eb;'>
<table width='100%' cellspacing='0' cellpadding='0'><tr>
<td><span style='font-size:18px;font-weight:700;color:#111827;'>&#128201; ${POOL}</span>
<span style='margin-left:8px;font-size:12px;color:#6b7280;'>pool</span></td>
<td align='right'><span style='background:${BADGE_BG};color:${BADGE_TEXT};padding:5px 14px;border-radius:20px;font-size:12px;font-weight:700;letter-spacing:.5px;'>${POOL_LABEL}</span></td>
</tr></table>
</div>
<div style='padding:20px 22px;'>
<table width='100%' cellspacing='0' cellpadding='0'>
<tr>
<td style='width:16%;padding:8px 12px;background:#f9fafb;border-radius:6px;text-align:center;'>
<div style='font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;'>Total Size</div>
<div style='font-size:20px;font-weight:700;color:#111827;margin-top:2px;'>${SIZE_H}</div>
</td><td style='width:4%;'></td>
<td style='width:16%;padding:8px 12px;background:#f9fafb;border-radius:6px;text-align:center;'>
<div style='font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;'>Used</div>
<div style='font-size:20px;font-weight:700;color:#111827;margin-top:2px;'>${ALLOC_H}</div>
</td><td style='width:4%;'></td>
<td style='width:16%;padding:8px 12px;background:#f9fafb;border-radius:6px;text-align:center;'>
<div style='font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;'>Free</div>
<div style='font-size:20px;font-weight:700;color:#111827;margin-top:2px;'>${FREE_H}</div>
</td><td style='width:4%;'></td>
<td style='width:16%;padding:8px 12px;background:#f9fafb;border-radius:6px;text-align:center;'>
<div style='font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;'>Capacity</div>
<div style='font-size:20px;font-weight:700;color:#111827;margin-top:2px;'>${CAP_H}</div>
</td><td style='width:4%;'></td>
<td style='width:16%;padding:8px 12px;background:#f9fafb;border-radius:6px;text-align:center;'>
<div style='font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;'>Fragmentation</div>
<div style='font-size:20px;font-weight:700;color:#111827;margin-top:2px;'>${FRAG_H}</div>
</td><td style='width:4%;'></td>
<td style='width:16%;padding:8px 12px;background:#f9fafb;border-radius:6px;text-align:center;'>
<div style='font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;'>Health</div>
<div style='font-size:20px;font-weight:700;color:#111827;margin-top:2px;'>${HEALTH}</div>
</td>
</tr>
</table>
<div style='margin-top:16px;padding:12px 14px;background:#f8fafc;border-radius:6px;border:1px solid #e5e7eb;'>
<span style='font-size:12px;font-weight:600;color:#374151;text-transform:uppercase;letter-spacing:.5px;'>&#128269; Last Scrub</span>
<span style='margin-left:10px;font-size:13px;color:#374151;'>${SCRUB_STATUS} &nbsp;|&nbsp; ${LAST_SCRUB} &nbsp;|&nbsp; Errors: ${ERRORS_SCRUB}</span>
</div>
${ERROR_HTML}
<details style='margin-top:14px;'>
<summary style='cursor:pointer;font-size:12px;color:#6b7280;font-weight:600;letter-spacing:.3px;'>&#9654; vdev / Drive Configuration — zpool status -v</summary>
<pre style='margin-top:8px;background:#1e293b;color:#e2e8f0;padding:14px;border-radius:6px;font-size:12px;line-height:1.6;overflow-x:auto;white-space:pre;'>${VDEV_TREE}</pre>
</details>
${DATASET_TABLE}
</div>
</div>"
done <<< "$POOL_LIST"
# ── Overall banner ─────────────────────────────────────────────────────────────
case "$OVERALL_STATUS" in
HEALTHY) BANNER_BG="#1a7f4b"; BANNER_ICON="&#9989;"; BANNER_MSG="All pools are healthy — no action required." ;;
WARNING) BANNER_BG="#b45309"; BANNER_ICON="&#9888;&#65039;"; BANNER_MSG="One or more pools require your attention." ;;
CRITICAL) BANNER_BG="#b91c1c"; BANNER_ICON="&#128680;"; BANNER_MSG="CRITICAL issue detected — immediate action required!" ;;
esac
SUBJECT="[ZFS] ${TARGET_HOST} — ${OVERALL_STATUS} — ${NOW}"
# ── Build HTML email ───────────────────────────────────────────────────────────
HTML=$(cat <<EOF
<!DOCTYPE html>
<html lang="en">
<head><meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1"></head>
<body style="margin:0;padding:0;background:#f1f5f9;font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,sans-serif;">
<table width="100%" cellspacing="0" cellpadding="0" style="background:#f1f5f9;padding:30px 0;">
<tr><td align="center">
<table width="720" cellspacing="0" cellpadding="0" style="max-width:720px;width:100%;">
<tr><td style="background:#0f172a;border-radius:10px 10px 0 0;padding:28px 32px;">
<table width="100%" cellspacing="0" cellpadding="0"><tr>
<td>
<div style="font-size:11px;color:#94a3b8;letter-spacing:1px;text-transform:uppercase;">ZFS Pool Health Monitor</div>
<div style="font-size:26px;font-weight:800;color:white;margin-top:4px;">&#128268; ${TARGET_HOST}</div>
<div style="font-size:13px;color:#94a3b8;margin-top:4px;">${NOW} &nbsp;|&nbsp; ${POOL_COUNT} pool(s) monitored</div>
</td>
<td align="right">
<div style="background:${BANNER_BG};color:white;padding:10px 20px;border-radius:8px;font-size:14px;font-weight:700;">${BANNER_ICON} ${OVERALL_STATUS}</div>
</td>
</tr></table>
</td></tr>
<tr><td style="background:${BANNER_BG};padding:12px 32px;">
<span style="color:white;font-size:14px;font-weight:600;">${BANNER_ICON} &nbsp;${BANNER_MSG}</span>
</td></tr>
<tr><td style="padding:28px 24px 8px;">${POOL_BLOCKS}</td></tr>
<tr><td style="padding:0 24px 28px;">
<div style="background:white;border-radius:10px;border:1px solid #e5e7eb;padding:18px 22px;">
<div style="font-size:13px;font-weight:700;color:#374151;margin-bottom:12px;text-transform:uppercase;letter-spacing:.5px;">&#128203; Report Summary</div>
<table width="100%" cellspacing="0" cellpadding="0">
<tr><td style="font-size:13px;color:#6b7280;padding-bottom:6px;">Host</td>
<td style="font-size:13px;color:#111827;font-weight:600;padding-bottom:6px;">${TARGET_HOST} (${HOSTNAME_REAL})</td></tr>
<tr><td style="font-size:13px;color:#6b7280;padding-bottom:6px;">Report Time</td>
<td style="font-size:13px;color:#111827;font-weight:600;padding-bottom:6px;">${NOW}</td></tr>
<tr><td style="font-size:13px;color:#6b7280;padding-bottom:6px;">Pools Checked</td>
<td style="font-size:13px;color:#111827;font-weight:600;padding-bottom:6px;">${POOL_COUNT}</td></tr>
<tr><td style="font-size:13px;color:#6b7280;padding-bottom:6px;">Pools with Issues</td>
<td style="font-size:13px;font-weight:700;color:$([ "$PROBLEM_COUNT" -gt 0 ] && echo '#b91c1c' || echo '#1a7f4b');padding-bottom:6px;">${PROBLEM_COUNT}</td></tr>
<tr><td style="font-size:13px;color:#6b7280;padding-bottom:6px;">Capacity Warning Threshold</td>
<td style="font-size:13px;color:#111827;font-weight:600;padding-bottom:6px;">&ge; ${CAP_WARN}%</td></tr>
<tr><td style="font-size:13px;color:#6b7280;padding-bottom:6px;">Capacity Critical Threshold</td>
<td style="font-size:13px;color:#111827;font-weight:600;padding-bottom:6px;">&ge; ${CAP_CRIT}%</td></tr>
<tr><td style="font-size:13px;color:#6b7280;">Scrub Age Warning</td>
<td style="font-size:13px;color:#111827;font-weight:600;">&gt; ${SCRUB_MAX_AGE_DAYS} days</td></tr>
</table>
<div style="margin-top:14px;padding:10px 12px;background:#f8fafc;border-radius:6px;border:1px solid #e5e7eb;">
<div style="font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;margin-bottom:4px;">Commands Executed</div>
<code style="font-size:12px;color:#374151;line-height:2;">
zpool status -v &lt;pool&gt;<br>
zpool list -H -o name,size,alloc,free,frag,cap,health &lt;pool&gt;<br>
zfs list -r -H -o name,used,avail,refer,mountpoint &lt;pool&gt;
</code>
</div>
</div>
</td></tr>
<tr><td style="background:#0f172a;border-radius:0 0 10px 10px;padding:16px 32px;text-align:center;">
<span style="font-size:11px;color:#64748b;">Automated ZFS Monitor &nbsp;|&nbsp; ${TARGET_HOST} &nbsp;|&nbsp; KingDezigns Infrastructure</span>
</td></tr>
</table>
</td></tr>
</table>
</body>
</html>
EOF
)
# ── Send email ─────────────────────────────────────────────────────────────────
send_email() {
if command -v sendmail &>/dev/null; then
{ echo "To: ${REPORT_TO}"; echo "From: ${REPORT_FROM}"; echo "Subject: ${SUBJECT}"
echo "MIME-Version: 1.0"; echo "Content-Type: text/html; charset=UTF-8"; echo ""
echo "$HTML"; } | sendmail -t
echo "Report sent via sendmail to ${REPORT_TO}"
elif command -v mailx &>/dev/null; then
echo "$HTML" | mailx -a "Content-Type: text/html" -s "$SUBJECT" "$REPORT_TO"
echo "Report sent via mailx to ${REPORT_TO}"
elif command -v mail &>/dev/null; then
echo "$HTML" | mail -a "Content-Type: text/html; charset=UTF-8" -s "$SUBJECT" "$REPORT_TO"
echo "Report sent via mail to ${REPORT_TO}"
else
echo "ERROR: No mail transport found. Install sendmail, mailx, or msmtp."
FALLBACK="/tmp/zfs_report_${TARGET_HOST}_$(date +%Y%m%d_%H%M%S).html"
echo "$HTML" > "$FALLBACK"
echo "HTML report saved to: $FALLBACK"
exit 1
fi
}
send_email
echo "[$(date '+%Y-%m-%d %H:%M:%S')] ZFS report for ${TARGET_HOST} — Status: ${OVERALL_STATUS} — Pools: ${POOL_COUNT} — Problems: ${PROBLEM_COUNT}"

322
zfs/nas16_zfs_scrub-1.sh Normal file
View file

@ -0,0 +1,322 @@
#!/bin/bash
# =============================================================================
# ZFS Pool Scrub & Email Report — NAS16
# Location: /usr/scripts/zfs/nas16_zfs_scrub.sh
# Schedule via OMV Scheduler (cron)
# =============================================================================
# ── Configuration ─────────────────────────────────────────────────────────────
HOSTNAME="NAS16"
REPORT_TO="rufus.king@kingdezigns.com"
REPORT_FROM="zfs-monitor@nas16.local" # Display only — Postfix uses your Gmail relay
SCRUB_WAIT_SECONDS=28800 # Max wait per pool scrub (8 hrs for large pools)
CAPACITY_WARN_THRESHOLD=80 # % capacity to flag as WARNING
SCRUB_AGE_WARN_DAYS=8 # Days since last scrub before flagging stale
LOG_FILE="/var/log/zfs_scrub_nas16.log"
# Note: Email is sent via OMV-configured Postfix (Gmail relay). No SMTP config needed here.
# ──────────────────────────────────────────────────────────────────────────────
REPORT_TIME=$(date "+%Y-%m-%d %H:%M:%S")
POOLS=$(zpool list -H -o name)
POOL_COUNT=$(echo "$POOLS" | wc -l)
log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a "$LOG_FILE"; }
log "===== ZFS Scrub started on $HOSTNAME ====="
# ── Run scrubs sequentially ───────────────────────────────────────────────────
for pool in $POOLS; do
log "Starting scrub on pool: $pool"
zpool scrub "$pool"
ELAPSED=0
while true; do
SCRUB_STATE=$(zpool status "$pool" | grep -E "scan:" | awk '{print $2}')
[[ "$SCRUB_STATE" == "scrub" ]] || break
sleep 30
ELAPSED=$((ELAPSED + 30))
if [[ $ELAPSED -ge $SCRUB_WAIT_SECONDS ]]; then
log "WARNING: Scrub on $pool exceeded wait time. Moving on."
break
fi
done
log "Scrub complete (or timed out) for pool: $pool"
done
# ── Gather data & build pool cards ───────────────────────────────────────────
OVERALL_STATUS="HEALTHY" # HEALTHY | WARNING | CRITICAL
POOLS_WITH_ISSUES=0
POOL_CARDS_HTML=""
for pool in $POOLS; do
STATUS_RAW=$(zpool list -H -o health "$pool")
SIZE=$(zpool list -H -o size "$pool")
USED=$(zpool list -H -o alloc "$pool")
FREE=$(zpool list -H -o free "$pool")
CAP=$(zpool list -H -o cap "$pool" | tr -d '%')
FRAG=$(zpool list -H -o frag "$pool")
SCRUB_LINE=$(zpool status "$pool" | grep -E "scan:")
ERRORS_LINE=$(zpool status "$pool" | grep "errors:")
CONFIG_BLOCK=$(zpool status "$pool" | awk '/config:/,/errors:/' | head -n -1)
FAULTED_LINES=$(echo "$CONFIG_BLOCK" | grep -E "FAULTED|DEGRADED|UNAVAIL|REMOVED" | grep -v "^$")
# Per-pool severity
POOL_SEVERITY="HEALTHY"
ISSUE_DETAIL=""
if [[ "$STATUS_RAW" == "DEGRADED" || "$STATUS_RAW" == "FAULTED" || "$STATUS_RAW" == "UNAVAIL" ]]; then
POOL_SEVERITY="CRITICAL"
OVERALL_STATUS="CRITICAL"
ISSUE_DETAIL="$FAULTED_LINES"
elif [[ "$CAP" -ge "$CAPACITY_WARN_THRESHOLD" ]]; then
POOL_SEVERITY="WARNING"
[[ "$OVERALL_STATUS" != "CRITICAL" ]] && OVERALL_STATUS="WARNING"
fi
# Scrub error count
SCRUB_ERRORS=$(echo "$SCRUB_LINE" | grep -oP '\d+ errors' | head -1)
[[ -z "$SCRUB_ERRORS" ]] && SCRUB_ERRORS="0 errors"
# Scrub date
SCRUB_DATE=$(echo "$SCRUB_LINE" | grep -oP '[A-Z][a-z]{2} \d+ \d+:\d+:\d+ \d{4}' | head -1)
[[ -z "$SCRUB_DATE" ]] && SCRUB_DATE="No scrub data"
# Scrub age warning
SCRUB_AGE_NOTE=""
if [[ -n "$SCRUB_DATE" && "$SCRUB_DATE" != "No scrub data" ]]; then
SCRUB_EPOCH=$(date -d "$SCRUB_DATE" +%s 2>/dev/null)
NOW_EPOCH=$(date +%s)
DIFF_DAYS=$(( (NOW_EPOCH - SCRUB_EPOCH) / 86400 ))
if [[ $DIFF_DAYS -ge $SCRUB_AGE_WARN_DAYS ]]; then
SCRUB_AGE_NOTE=" — ⚠️ Last scrub was ${DIFF_DAYS} days ago"
[[ "$POOL_SEVERITY" == "HEALTHY" ]] && POOL_SEVERITY="WARNING"
[[ "$OVERALL_STATUS" != "CRITICAL" ]] && OVERALL_STATUS="WARNING"
fi
fi
[[ "$POOL_SEVERITY" != "HEALTHY" ]] && POOLS_WITH_ISSUES=$((POOLS_WITH_ISSUES + 1))
# Colors
case "$STATUS_RAW" in
ONLINE) HEALTH_COLOR="#1a7f4b" ;;
DEGRADED) HEALTH_COLOR="#b91c1c" ;;
FAULTED) HEALTH_COLOR="#b91c1c" ;;
*) HEALTH_COLOR="#b45309" ;;
esac
case "$POOL_SEVERITY" in
CRITICAL) CARD_BORDER="#fca5a5"; CARD_HEADER_BG="#fff1f2"; POOL_BADGE_BG="#b91c1c"; POOL_BADGE_LABEL="$STATUS_RAW" ;;
WARNING) CARD_BORDER="#fcd34d"; CARD_HEADER_BG="#fffbeb"; POOL_BADGE_BG="#b45309"; POOL_BADGE_LABEL="WARNING" ;;
*) CARD_BORDER="#86efac"; CARD_HEADER_BG="#f0fdf4"; POOL_BADGE_BG="#1a7f4b"; POOL_BADGE_LABEL="HEALTHY" ;;
esac
CAP_COLOR="#111827"
[[ "$CAP" -ge "$CAPACITY_WARN_THRESHOLD" ]] && CAP_COLOR="#b45309"
[[ "$CAP" -ge 90 ]] && CAP_COLOR="#b91c1c"
# Issue block
ISSUE_BLOCK=""
if [[ -n "$ISSUE_DETAIL" ]]; then
ESCAPED_DETAIL=$(echo "$ISSUE_DETAIL" | sed 's/</\&lt;/g; s/>/\&gt;/g')
ISSUE_BLOCK="
<tr><td colspan=\"7\" style=\"padding:10px 0 0;\">
<div style=\"padding:10px 14px;background:#fff1f2;border-left:4px solid #b91c1c;border-radius:4px;font-family:monospace;font-size:12px;color:#7f1d1d;white-space:pre;\">$ESCAPED_DETAIL</div>
</td></tr>"
fi
# Config block
ESCAPED_CONFIG=$(echo "$CONFIG_BLOCK" | sed 's/</\&lt;/g; s/>/\&gt;/g')
POOL_CARDS_HTML+="
<div style=\"background:white;border-radius:10px;box-shadow:0 1px 6px rgba(0,0,0,.08);margin-bottom:20px;overflow:hidden;border:1px solid ${CARD_BORDER};\">
<!-- Pool card header -->
<div style=\"background:${CARD_HEADER_BG};padding:14px 20px;border-bottom:1px solid ${CARD_BORDER};\">
<table width=\"100%\" cellspacing=\"0\" cellpadding=\"0\">
<tr>
<td>
<span style=\"font-size:16px;font-weight:700;color:#111827;\">🗄️ $pool</span>
<span style=\"margin-left:8px;font-size:12px;color:#6b7280;\">pool</span>
</td>
<td align=\"right\">
<span style=\"background:${POOL_BADGE_BG};color:white;padding:4px 14px;border-radius:20px;font-size:11px;font-weight:700;letter-spacing:.5px;\">$POOL_BADGE_LABEL</span>
</td>
</tr>
</table>
</div>
<!-- Pool stats -->
<div style=\"padding:18px 20px;\">
<table width=\"100%\" cellspacing=\"0\" cellpadding=\"0\">
<tr>
<td style=\"width:14%;padding:8px 10px;background:#f9fafb;border-radius:6px;text-align:center;\">
<div style=\"font-size:10px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;\">Total</div>
<div style=\"font-size:18px;font-weight:700;color:#111827;margin-top:2px;\">$SIZE</div>
</td>
<td style=\"width:3%;\"></td>
<td style=\"width:14%;padding:8px 10px;background:#f9fafb;border-radius:6px;text-align:center;\">
<div style=\"font-size:10px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;\">Used</div>
<div style=\"font-size:18px;font-weight:700;color:#111827;margin-top:2px;\">$USED</div>
</td>
<td style=\"width:3%;\"></td>
<td style=\"width:14%;padding:8px 10px;background:#f9fafb;border-radius:6px;text-align:center;\">
<div style=\"font-size:10px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;\">Free</div>
<div style=\"font-size:18px;font-weight:700;color:#111827;margin-top:2px;\">$FREE</div>
</td>
<td style=\"width:3%;\"></td>
<td style=\"width:14%;padding:8px 10px;background:#f9fafb;border-radius:6px;text-align:center;\">
<div style=\"font-size:10px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;\">Capacity</div>
<div style=\"font-size:18px;font-weight:700;color:${CAP_COLOR};margin-top:2px;\">${CAP}%</div>
</td>
<td style=\"width:3%;\"></td>
<td style=\"width:14%;padding:8px 10px;background:#f9fafb;border-radius:6px;text-align:center;\">
<div style=\"font-size:10px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;\">Frag</div>
<div style=\"font-size:18px;font-weight:700;color:#111827;margin-top:2px;\">$FRAG</div>
</td>
<td style=\"width:3%;\"></td>
<td style=\"width:14%;padding:8px 10px;background:#f9fafb;border-radius:6px;text-align:center;\">
<div style=\"font-size:10px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;\">Health</div>
<div style=\"font-size:18px;font-weight:700;color:${HEALTH_COLOR};margin-top:2px;\">$STATUS_RAW</div>
</td>
</tr>
$ISSUE_BLOCK
</table>
<!-- Last scrub row -->
<div style=\"margin-top:14px;padding:10px 14px;background:#f8fafc;border-radius:6px;border:1px solid #e5e7eb;\">
<span style=\"font-size:11px;font-weight:700;color:#374151;text-transform:uppercase;letter-spacing:.5px;\">🔍 Last Scrub</span>
<span style=\"margin-left:10px;font-size:13px;color:#374151;\">Completed &nbsp;|&nbsp; $SCRUB_DATE &nbsp;|&nbsp; $SCRUB_ERRORS${SCRUB_AGE_NOTE}</span>
</div>
<!-- vdev config collapsible -->
<details style=\"margin-top:12px;\">
<summary style=\"cursor:pointer;font-size:12px;color:#6b7280;font-weight:600;letter-spacing:.3px;padding:6px 0;\">▶ vdev / drive configuration</summary>
<pre style=\"margin-top:8px;background:#1e293b;color:#e2e8f0;padding:14px;border-radius:6px;font-size:12px;line-height:1.6;overflow-x:auto;\">$ESCAPED_CONFIG</pre>
</details>
</div>
</div>"
done
# ── Overall banner & badge ────────────────────────────────────────────────────
case "$OVERALL_STATUS" in
CRITICAL)
BADGE_HTML="<div style=\"background:#b91c1c;color:white;padding:10px 20px;border-radius:8px;font-size:14px;font-weight:700;\">🚨 CRITICAL</div>"
BANNER_HTML="<tr><td style=\"background:#b91c1c;padding:12px 32px;\"><span style=\"color:white;font-size:14px;font-weight:600;\">🚨 &nbsp;CRITICAL issue detected — immediate action required!</span></td></tr>"
ISSUES_COLOR="#b91c1c"
;;
WARNING)
BADGE_HTML="<div style=\"background:#b45309;color:white;padding:10px 20px;border-radius:8px;font-size:14px;font-weight:700;\">⚠️ WARNING</div>"
BANNER_HTML="<tr><td style=\"background:#b45309;padding:12px 32px;\"><span style=\"color:white;font-size:14px;font-weight:600;\">⚠️ &nbsp;Warning condition detected — review recommended.</span></td></tr>"
ISSUES_COLOR="#b45309"
;;
*)
BADGE_HTML="<div style=\"background:#1a7f4b;color:white;padding:10px 20px;border-radius:8px;font-size:14px;font-weight:700;\">✅ ALL HEALTHY</div>"
BANNER_HTML="<tr><td style=\"background:#1a7f4b;padding:12px 32px;\"><span style=\"color:white;font-size:14px;font-weight:600;\">✅ &nbsp;All pools are healthy — no action required.</span></td></tr>"
ISSUES_COLOR="#1a7f4b"
;;
esac
# ── Compose full HTML email ───────────────────────────────────────────────────
HTML_BODY=$(cat <<EOF
<!DOCTYPE html>
<html lang="en">
<head><meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1"></head>
<body style="margin:0;padding:0;background:#f1f5f9;font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,sans-serif;">
<table width="100%" cellspacing="0" cellpadding="0" style="background:#f1f5f9;padding:30px 0;">
<tr><td align="center">
<table width="680" cellspacing="0" cellpadding="0" style="max-width:680px;width:100%;">
<!-- Header -->
<tr><td style="background:#0f172a;border-radius:10px 10px 0 0;padding:28px 32px;">
<table width="100%" cellspacing="0" cellpadding="0">
<tr>
<td>
<div style="font-size:11px;color:#94a3b8;letter-spacing:1px;text-transform:uppercase;">ZFS Pool Health Monitor</div>
<div style="font-size:26px;font-weight:800;color:white;margin-top:4px;">🔌 $HOSTNAME</div>
<div style="font-size:13px;color:#94a3b8;margin-top:4px;">$REPORT_TIME &nbsp;|&nbsp; $POOL_COUNT pool(s) monitored</div>
</td>
<td align="right">$BADGE_HTML</td>
</tr>
</table>
</td></tr>
<!-- Status banner -->
$BANNER_HTML
<!-- Pool cards -->
<tr><td style="padding:28px 24px 8px;">
$POOL_CARDS_HTML
</td></tr>
<!-- Report summary card -->
<tr><td style="padding:0 24px 28px;">
<div style="background:white;border-radius:10px;border:1px solid #e5e7eb;padding:18px 22px;">
<div style="font-size:13px;font-weight:700;color:#374151;margin-bottom:10px;text-transform:uppercase;letter-spacing:.5px;">📊 Report Summary</div>
<table width="100%" cellspacing="0" cellpadding="0">
<tr>
<td style="font-size:13px;color:#6b7280;">Host</td>
<td style="font-size:13px;color:#111827;font-weight:600;">$HOSTNAME</td>
</tr>
<tr><td colspan="2" style="height:6px;"></td></tr>
<tr>
<td style="font-size:13px;color:#6b7280;">Report Time</td>
<td style="font-size:13px;color:#111827;font-weight:600;">$REPORT_TIME</td>
</tr>
<tr><td colspan="2" style="height:6px;"></td></tr>
<tr>
<td style="font-size:13px;color:#6b7280;">Pools Checked</td>
<td style="font-size:13px;color:#111827;font-weight:600;">$POOL_COUNT</td>
</tr>
<tr><td colspan="2" style="height:6px;"></td></tr>
<tr>
<td style="font-size:13px;color:#6b7280;">Pools with Issues</td>
<td style="font-size:13px;font-weight:700;color:${ISSUES_COLOR};">$POOLS_WITH_ISSUES</td>
</tr>
<tr><td colspan="2" style="height:6px;"></td></tr>
<tr>
<td style="font-size:13px;color:#6b7280;">Scrub Age Warning Threshold</td>
<td style="font-size:13px;color:#111827;font-weight:600;">$SCRUB_AGE_WARN_DAYS days</td>
</tr>
<tr><td colspan="2" style="height:6px;"></td></tr>
<tr>
<td style="font-size:13px;color:#6b7280;">Capacity Warning Threshold</td>
<td style="font-size:13px;color:#111827;font-weight:600;">${CAPACITY_WARN_THRESHOLD}%</td>
</tr>
<tr><td colspan="2" style="height:6px;"></td></tr>
<tr>
<td style="font-size:13px;color:#6b7280;">Overall Status</td>
<td style="font-size:13px;font-weight:700;color:${ISSUES_COLOR};">$OVERALL_STATUS</td>
</tr>
</table>
</div>
</td></tr>
<!-- Footer -->
<tr><td style="background:#0f172a;border-radius:0 0 10px 10px;padding:16px 32px;text-align:center;">
<span style="font-size:11px;color:#64748b;">Automated ZFS Monitor &nbsp;|&nbsp; $HOSTNAME &nbsp;|&nbsp; KingDezigns Infrastructure</span>
</td></tr>
</table>
</td></tr>
</table>
</body>
</html>
EOF
)
# ── Send email via OMV Postfix (Gmail relay) ──────────────────────────────────
SUBJECT="[ZFS] $HOSTNAME — $OVERALL_STATUS | $REPORT_TIME"
{
echo "To: $REPORT_TO"
echo "From: $REPORT_FROM"
echo "Subject: $SUBJECT"
echo "MIME-Version: 1.0"
echo "Content-Type: text/html; charset=UTF-8"
echo ""
echo "$HTML_BODY"
} | sendmail -t
log "Email sent to $REPORT_TO — Overall status: $OVERALL_STATUS"
log "===== ZFS Scrub complete on $HOSTNAME ====="

295
zfs/nas16_zfs_scrub.sh Normal file
View file

@ -0,0 +1,295 @@
#!/bin/bash
# =============================================================================
# ZFS Pool Scrub & Email Report — NAS16
# Location: /usr/scripts/zfs/nas16_zfs_scrub.sh
# Schedule via OMV Scheduler (cron)
# =============================================================================
# ── Configuration ─────────────────────────────────────────────────────────────
HOSTNAME="NAS16"
REPORT_TO="rufus.king@kingdezigns.com"
REPORT_FROM="zfs-monitor@nas16.local" # Display only — Postfix uses your Gmail relay
SCRUB_WAIT_SECONDS=28800 # Max wait per pool scrub (8 hrs for large pools)
CAPACITY_WARN_THRESHOLD=80 # % capacity to flag as WARNING
SCRUB_AGE_WARN_DAYS=8 # Days since last scrub before flagging stale
LOG_FILE="/var/log/zfs_scrub_nas16.log"
# Note: Email is sent via OMV-configured Postfix (Gmail relay). No SMTP config needed here.
# ──────────────────────────────────────────────────────────────────────────────
REPORT_TIME=$(date "+%Y-%m-%d %H:%M:%S")
POOLS=$(zpool list -H -o name)
POOL_COUNT=$(echo "$POOLS" | wc -l)
log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a "$LOG_FILE"; }
log "===== ZFS Scrub started on $HOSTNAME ====="
# ── Run scrubs sequentially ───────────────────────────────────────────────────
for pool in $POOLS; do
log "Starting scrub on pool: $pool"
zpool scrub "$pool"
# Wait for scrub to finish
ELAPSED=0
while true; do
SCRUB_STATE=$(zpool status "$pool" | grep -E "scan:" | awk '{print $2}')
[[ "$SCRUB_STATE" == "scrub" ]] || break
sleep 30
ELAPSED=$((ELAPSED + 30))
if [[ $ELAPSED -ge $SCRUB_WAIT_SECONDS ]]; then
log "WARNING: Scrub on $pool exceeded wait time. Moving on."
break
fi
done
log "Scrub complete (or timed out) for pool: $pool"
done
# ── Gather data & build HTML report ──────────────────────────────────────────
OVERALL_STATUS="HEALTHY" # HEALTHY | WARNING | CRITICAL
POOLS_WITH_ISSUES=0
POOL_CARDS_HTML=""
for pool in $POOLS; do
STATUS_RAW=$(zpool list -H -o health "$pool")
SIZE=$(zpool list -H -o size "$pool")
USED=$(zpool list -H -o alloc "$pool")
FREE=$(zpool list -H -o free "$pool")
CAP=$(zpool list -H -o cap "$pool" | tr -d '%')
FRAG=$(zpool list -H -o frag "$pool")
SCRUB_LINE=$(zpool status "$pool" | grep -E "scan:")
ERRORS_LINE=$(zpool status "$pool" | grep "errors:")
CONFIG_BLOCK=$(zpool status "$pool" | awk '/config:/,/errors:/' | head -n -1)
FAULTED_LINES=$(echo "$CONFIG_BLOCK" | grep -E "FAULTED|DEGRADED|UNAVAIL|REMOVED" | grep -v "^$")
# Determine per-pool severity
POOL_SEVERITY="HEALTHY"
ISSUE_DETAIL=""
if [[ "$STATUS_RAW" == "DEGRADED" || "$STATUS_RAW" == "FAULTED" || "$STATUS_RAW" == "UNAVAIL" ]]; then
POOL_SEVERITY="CRITICAL"
OVERALL_STATUS="CRITICAL"
ISSUE_DETAIL="$FAULTED_LINES"
elif [[ "$CAP" -ge "$CAPACITY_WARN_THRESHOLD" ]]; then
POOL_SEVERITY="WARNING"
[[ "$OVERALL_STATUS" != "CRITICAL" ]] && OVERALL_STATUS="WARNING"
fi
# Scrub error count from status
SCRUB_ERRORS=$(echo "$SCRUB_LINE" | grep -oP '\d+ errors' | head -1)
[[ -z "$SCRUB_ERRORS" ]] && SCRUB_ERRORS="0 errors"
# Scrub date
SCRUB_DATE=$(echo "$SCRUB_LINE" | grep -oP '[A-Z][a-z]{2} \d+ \d+:\d+:\d+ \d{4}' | head -1)
[[ -z "$SCRUB_DATE" ]] && SCRUB_DATE="No scrub data"
# Scrub age warning
SCRUB_AGE_NOTE=""
if [[ -n "$SCRUB_DATE" && "$SCRUB_DATE" != "No scrub data" ]]; then
SCRUB_EPOCH=$(date -d "$SCRUB_DATE" +%s 2>/dev/null)
NOW_EPOCH=$(date +%s)
DIFF_DAYS=$(( (NOW_EPOCH - SCRUB_EPOCH) / 86400 ))
if [[ $DIFF_DAYS -ge $SCRUB_AGE_WARN_DAYS ]]; then
SCRUB_AGE_NOTE=" — ⚠️ Last scrub was ${DIFF_DAYS} days ago"
[[ "$POOL_SEVERITY" == "HEALTHY" ]] && POOL_SEVERITY="WARNING"
[[ "$OVERALL_STATUS" != "CRITICAL" ]] && OVERALL_STATUS="WARNING"
fi
fi
[[ "$POOL_SEVERITY" != "HEALTHY" ]] && POOLS_WITH_ISSUES=$((POOLS_WITH_ISSUES + 1))
# Health value color
case "$STATUS_RAW" in
ONLINE) HEALTH_COLOR="#1a7f4b" ;;
DEGRADED) HEALTH_COLOR="#b91c1c" ;;
FAULTED) HEALTH_COLOR="#b91c1c" ;;
*) HEALTH_COLOR="#b45309" ;;
esac
# Pool card header color
case "$POOL_SEVERITY" in
CRITICAL) CARD_BG="#fff1f2"; BADGE_BG="#b91c1c"; BADGE_LABEL="$STATUS_RAW" ;;
WARNING) CARD_BG="#fffbeb"; BADGE_BG="#b45309"; BADGE_LABEL="WARNING" ;;
*) CARD_BG="#f0fdf4"; BADGE_BG="#1a7f4b"; BADGE_LABEL="HEALTHY" ;;
esac
# Capacity color
CAP_COLOR="#111827"
[[ "$CAP" -ge "$CAPACITY_WARN_THRESHOLD" ]] && CAP_COLOR="#b45309"
[[ "$CAP" -ge 90 ]] && CAP_COLOR="#b91c1c"
# Issue block (only shown when there's a problem)
ISSUE_BLOCK=""
if [[ -n "$ISSUE_DETAIL" ]]; then
ESCAPED_DETAIL=$(echo "$ISSUE_DETAIL" | sed 's/</\&lt;/g; s/>/\&gt;/g')
ISSUE_BLOCK="<div style=\"margin-top:10px;padding:10px 14px;background:#fff1f2;border-left:4px solid #b91c1c;border-radius:4px;font-family:monospace;font-size:13px;color:#7f1d1d;white-space:pre;\">$ESCAPED_DETAIL</div>"
fi
# Config block
ESCAPED_CONFIG=$(echo "$CONFIG_BLOCK" | sed 's/</\&lt;/g; s/>/\&gt;/g')
POOL_CARDS_HTML+="
<div style=\"background:white;border-radius:10px;box-shadow:0 1px 6px rgba(0,0,0,.10);margin-bottom:28px;overflow:hidden;border:1px solid #e5e7eb;\">
<div style=\"background:$CARD_BG;padding:16px 22px;display:flex;align-items:center;justify-content:space-between;border-bottom:1px solid #e5e7eb;\">
<div>
<span style=\"font-size:18px;font-weight:700;color:#111827;\">🗄️ $pool</span>
<span style=\"margin-left:8px;font-size:12px;color:#6b7280;\">pool</span>
</div>
<span style=\"background:$BADGE_BG;color:white;padding:5px 14px;border-radius:20px;font-size:12px;font-weight:700;letter-spacing:.5px;\">$BADGE_LABEL</span>
</div>
<div style=\"padding:20px 22px;\">
<table width=\"100%\" cellspacing=\"0\" cellpadding=\"0\">
<tr>
<td style=\"width:16%;padding:8px 12px;background:#f9fafb;border-radius:6px;text-align:center;\">
<div style=\"font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;\">Total Size</div>
<div style=\"font-size:20px;font-weight:700;color:#111827;margin-top:2px;\">$SIZE</div>
</td>
<td style=\"width:4%;\"></td>
<td style=\"width:16%;padding:8px 12px;background:#f9fafb;border-radius:6px;text-align:center;\">
<div style=\"font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;\">Used</div>
<div style=\"font-size:20px;font-weight:700;color:#111827;margin-top:2px;\">$USED</div>
</td>
<td style=\"width:4%;\"></td>
<td style=\"width:16%;padding:8px 12px;background:#f9fafb;border-radius:6px;text-align:center;\">
<div style=\"font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;\">Free</div>
<div style=\"font-size:20px;font-weight:700;color:#111827;margin-top:2px;\">$FREE</div>
</td>
<td style=\"width:4%;\"></td>
<td style=\"width:16%;padding:8px 12px;background:#f9fafb;border-radius:6px;text-align:center;\">
<div style=\"font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;\">Capacity</div>
<div style=\"font-size:20px;font-weight:700;color:$CAP_COLOR;margin-top:2px;\">${CAP}%</div>
</td>
<td style=\"width:4%;\"></td>
<td style=\"width:16%;padding:8px 12px;background:#f9fafb;border-radius:6px;text-align:center;\">
<div style=\"font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;\">Fragmentation</div>
<div style=\"font-size:20px;font-weight:700;color:#111827;margin-top:2px;\">$FRAG</div>
</td>
<td style=\"width:4%;\"></td>
<td style=\"width:16%;padding:8px 12px;background:#f9fafb;border-radius:6px;text-align:center;\">
<div style=\"font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;\">Health</div>
<div style=\"font-size:20px;font-weight:700;color:$HEALTH_COLOR;margin-top:2px;\">$STATUS_RAW</div>
</td>
</tr>
</table>
<div style=\"margin-top:16px;padding:12px 14px;background:#f8fafc;border-radius:6px;border:1px solid #e5e7eb;\">
<span style=\"font-size:12px;font-weight:600;color:#374151;text-transform:uppercase;letter-spacing:.5px;\">🔍 Last Scrub</span>
<span style=\"margin-left:10px;font-size:13px;color:#374151;\">Completed &nbsp;|&nbsp; $SCRUB_DATE &nbsp;|&nbsp; $SCRUB_ERRORS$SCRUB_AGE_NOTE</span>
</div>
$ISSUE_BLOCK
<details style=\"margin-top:14px;\">
<summary style=\"cursor:pointer;font-size:12px;color:#6b7280;font-weight:600;letter-spacing:.3px;\">▶ vdev / drive configuration</summary>
<pre style=\"margin-top:8px;background:#1e293b;color:#e2e8f0;padding:14px;border-radius:6px;font-size:12px;line-height:1.6;overflow-x:auto;\">$ESCAPED_CONFIG</pre>
</details>
</div>
</div>"
done
# ── Overall banner ────────────────────────────────────────────────────────────
case "$OVERALL_STATUS" in
CRITICAL)
BADGE_HTML="<div style=\"background:#b91c1c;color:white;padding:10px 20px;border-radius:8px;font-size:14px;font-weight:700;\">🚨 CRITICAL</div>"
BANNER_HTML="<tr><td style=\"background:#b91c1c;padding:12px 32px;\"><span style=\"color:white;font-size:14px;font-weight:600;\">🚨 &nbsp;CRITICAL issue detected — immediate action required!</span></td></tr>"
;;
WARNING)
BADGE_HTML="<div style=\"background:#b45309;color:white;padding:10px 20px;border-radius:8px;font-size:14px;font-weight:700;\">⚠️ WARNING</div>"
BANNER_HTML="<tr><td style=\"background:#b45309;padding:12px 32px;\"><span style=\"color:white;font-size:14px;font-weight:600;\">⚠️ &nbsp;Warning condition detected — review recommended.</span></td></tr>"
;;
*)
BADGE_HTML="<div style=\"background:#1a7f4b;color:white;padding:10px 20px;border-radius:8px;font-size:14px;font-weight:700;\">✅ ALL HEALTHY</div>"
BANNER_HTML="<tr><td style=\"background:#1a7f4b;padding:12px 32px;\"><span style=\"color:white;font-size:14px;font-weight:600;\">✅ &nbsp;All pools are healthy — no action required.</span></td></tr>"
;;
esac
# ── Compose full HTML email ───────────────────────────────────────────────────
HTML_BODY=$(cat <<EOF
<!DOCTYPE html>
<html lang="en">
<head><meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1"></head>
<body style="margin:0;padding:0;background:#f1f5f9;font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,sans-serif;">
<table width="100%" cellspacing="0" cellpadding="0" style="background:#f1f5f9;padding:30px 0;">
<tr><td align="center">
<table width="680" cellspacing="0" cellpadding="0" style="max-width:680px;width:100%;">
<tr><td style="background:#0f172a;border-radius:10px 10px 0 0;padding:28px 32px;">
<table width="100%" cellspacing="0" cellpadding="0">
<tr>
<td>
<div style="font-size:11px;color:#94a3b8;letter-spacing:1px;text-transform:uppercase;">ZFS Pool Health Monitor</div>
<div style="font-size:26px;font-weight:800;color:white;margin-top:4px;">🔌 $HOSTNAME</div>
<div style="font-size:13px;color:#94a3b8;margin-top:4px;">$REPORT_TIME &nbsp;|&nbsp; $POOL_COUNT pool(s) monitored</div>
</td>
<td align="right">$BADGE_HTML</td>
</tr>
</table>
</td></tr>
$BANNER_HTML
<tr><td style="padding:28px 24px 8px;">
$POOL_CARDS_HTML
</td></tr>
<tr><td style="padding:0 24px 28px;">
<div style="background:white;border-radius:10px;border:1px solid #e5e7eb;padding:18px 22px;">
<div style="font-size:13px;font-weight:700;color:#374151;margin-bottom:10px;text-transform:uppercase;letter-spacing:.5px;">📋 Report Summary</div>
<table width="100%" cellspacing="0" cellpadding="0">
<tr>
<td style="font-size:13px;color:#6b7280;">Host</td>
<td style="font-size:13px;color:#111827;font-weight:600;">$HOSTNAME</td>
</tr>
<tr><td colspan="2" style="height:6px;"></td></tr>
<tr>
<td style="font-size:13px;color:#6b7280;">Report Time</td>
<td style="font-size:13px;color:#111827;font-weight:600;">$REPORT_TIME</td>
</tr>
<tr><td colspan="2" style="height:6px;"></td></tr>
<tr>
<td style="font-size:13px;color:#6b7280;">Pools Checked</td>
<td style="font-size:13px;color:#111827;font-weight:600;">$POOL_COUNT</td>
</tr>
<tr><td colspan="2" style="height:6px;"></td></tr>
<tr>
<td style="font-size:13px;color:#6b7280;">Pools with Issues</td>
<td style="font-size:13px;font-weight:700;color:#b91c1c;">$POOLS_WITH_ISSUES</td>
</tr>
<tr><td colspan="2" style="height:6px;"></td></tr>
<tr>
<td style="font-size:13px;color:#6b7280;">Scrub Age Warning Threshold</td>
<td style="font-size:13px;color:#111827;font-weight:600;">$SCRUB_AGE_WARN_DAYS days</td>
</tr>
<tr><td colspan="2" style="height:6px;"></td></tr>
<tr>
<td style="font-size:13px;color:#6b7280;">Capacity Warning Threshold</td>
<td style="font-size:13px;color:#111827;font-weight:600;">${CAPACITY_WARN_THRESHOLD}%</td>
</tr>
</table>
</div>
</td></tr>
<tr><td style="background:#0f172a;border-radius:0 0 10px 10px;padding:16px 32px;text-align:center;">
<span style="font-size:11px;color:#64748b;">Automated ZFS Monitor &nbsp;|&nbsp; $HOSTNAME &nbsp;|&nbsp; KingDezigns Infrastructure</span>
</td></tr>
</table>
</td></tr>
</table>
</body>
</html>
EOF
)
# ── Send email via OMV Postfix (Gmail relay) ──────────────────────────────────
SUBJECT="[ZFS] $HOSTNAME — $OVERALL_STATUS | $REPORT_TIME"
{
echo "To: $REPORT_TO"
echo "From: $REPORT_FROM"
echo "Subject: $SUBJECT"
echo "MIME-Version: 1.0"
echo "Content-Type: text/html; charset=UTF-8"
echo ""
echo "$HTML_BODY"
} | sendmail -t
log "Email sent to $REPORT_TO — Overall status: $OVERALL_STATUS"
log "===== ZFS Scrub complete on $HOSTNAME ====="

250
zfs/zfs_report_preview.html Normal file
View file

@ -0,0 +1,250 @@
<!DOCTYPE html>
<html lang="en">
<head><meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1">
<title>ZFS Report Preview</title>
</head>
<body style="margin:0;padding:0;background:#f1f5f9;font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,sans-serif;">
<!-- Preview label -->
<div style="background:#1e40af;color:white;text-align:center;padding:10px;font-size:13px;font-weight:600;letter-spacing:.5px;">
📧 EMAIL REPORT PREVIEW — This is how the email will look in your inbox
</div>
<table width="100%" cellspacing="0" cellpadding="0" style="background:#f1f5f9;padding:30px 0;">
<tr><td align="center">
<table width="680" cellspacing="0" cellpadding="0" style="max-width:680px;width:100%;">
<!-- Header -->
<tr><td style="background:#0f172a;border-radius:10px 10px 0 0;padding:28px 32px;">
<table width="100%" cellspacing="0" cellpadding="0">
<tr>
<td>
<div style="font-size:11px;color:#94a3b8;letter-spacing:1px;text-transform:uppercase;">ZFS Pool Health Monitor</div>
<div style="font-size:26px;font-weight:800;color:white;margin-top:4px;">🔌 NAS08</div>
<div style="font-size:13px;color:#94a3b8;margin-top:4px;">2025-06-10 07:00:01 &nbsp;|&nbsp; 3 pool(s) monitored</div>
</td>
<td align="right">
<div style="background:#b91c1c;color:white;padding:10px 20px;border-radius:8px;font-size:14px;font-weight:700;">
🚨 CRITICAL
</div>
</td>
</tr>
</table>
</td></tr>
<!-- Status Banner -->
<tr><td style="background:#b91c1c;padding:12px 32px;">
<span style="color:white;font-size:14px;font-weight:600;">🚨 &nbsp;CRITICAL issue detected — immediate action required!</span>
</td></tr>
<!-- Body -->
<tr><td style="padding:28px 24px 8px;">
<!-- POOL 1 — CRITICAL -->
<div style="background:white;border-radius:10px;box-shadow:0 1px 6px rgba(0,0,0,.10);margin-bottom:28px;overflow:hidden;border:1px solid #e5e7eb;">
<div style="background:#fff1f2;padding:16px 22px;display:flex;align-items:center;justify-content:space-between;border-bottom:1px solid #e5e7eb;">
<div>
<span style="font-size:18px;font-weight:700;color:#111827;">📉 tank0</span>
<span style="margin-left:8px;font-size:12px;color:#6b7280;">pool</span>
</div>
<span style="background:#b91c1c;color:white;padding:5px 14px;border-radius:20px;font-size:12px;font-weight:700;letter-spacing:.5px;">DEGRADED</span>
</div>
<div style="padding:20px 22px;">
<table width="100%" cellspacing="0" cellpadding="0">
<tr>
<td style="width:16%;padding:8px 12px;background:#f9fafb;border-radius:6px;text-align:center;">
<div style="font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;">Total Size</div>
<div style="font-size:20px;font-weight:700;color:#111827;margin-top:2px;">18.2T</div>
</td>
<td style="width:4%;"></td>
<td style="width:16%;padding:8px 12px;background:#f9fafb;border-radius:6px;text-align:center;">
<div style="font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;">Used</div>
<div style="font-size:20px;font-weight:700;color:#111827;margin-top:2px;">14.8T</div>
</td>
<td style="width:4%;"></td>
<td style="width:16%;padding:8px 12px;background:#f9fafb;border-radius:6px;text-align:center;">
<div style="font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;">Free</div>
<div style="font-size:20px;font-weight:700;color:#111827;margin-top:2px;">3.4T</div>
</td>
<td style="width:4%;"></td>
<td style="width:16%;padding:8px 12px;background:#f9fafb;border-radius:6px;text-align:center;">
<div style="font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;">Capacity</div>
<div style="font-size:20px;font-weight:700;color:#111827;margin-top:2px;">81%</div>
</td>
<td style="width:4%;"></td>
<td style="width:16%;padding:8px 12px;background:#f9fafb;border-radius:6px;text-align:center;">
<div style="font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;">Fragmentation</div>
<div style="font-size:20px;font-weight:700;color:#111827;margin-top:2px;">12%</div>
</td>
<td style="width:4%;"></td>
<td style="width:16%;padding:8px 12px;background:#f9fafb;border-radius:6px;text-align:center;">
<div style="font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;">Health</div>
<div style="font-size:20px;font-weight:700;color:#b91c1c;margin-top:2px;">DEGRADED</div>
</td>
</tr>
</table>
<div style="margin-top:16px;padding:12px 14px;background:#f8fafc;border-radius:6px;border:1px solid #e5e7eb;">
<span style="font-size:12px;font-weight:600;color:#374151;text-transform:uppercase;letter-spacing:.5px;">🔍 Last Scrub</span>
<span style="margin-left:10px;font-size:13px;color:#374151;">Completed &nbsp;|&nbsp; Jun 3 07:01:44 2025 &nbsp;|&nbsp; Errors: 0 errors</span>
</div>
<div style="margin-top:10px;padding:10px 14px;background:#fff1f2;border-left:4px solid #b91c1c;border-radius:4px;font-family:monospace;font-size:13px;color:#7f1d1d;white-space:pre;"> sdb DEGRADED 0 0 0
sdb1 FAULTED 5 8 0 too many errors</div>
<details style="margin-top:14px;">
<summary style="cursor:pointer;font-size:12px;color:#6b7280;font-weight:600;letter-spacing:.3px;">▶ vdev / drive configuration</summary>
<pre style="margin-top:8px;background:#1e293b;color:#e2e8f0;padding:14px;border-radius:6px;font-size:12px;line-height:1.6;overflow-x:auto;">config:
NAME STATE READ WRITE CKSUM
tank0 DEGRADED 0 0 0
raidz2-0 DEGRADED 0 0 0
sda ONLINE 0 0 0
sdb DEGRADED 5 8 0
sdc ONLINE 0 0 0
sdd ONLINE 0 0 0</pre>
</details>
</div>
</div>
<!-- POOL 2 — WARNING -->
<div style="background:white;border-radius:10px;box-shadow:0 1px 6px rgba(0,0,0,.10);margin-bottom:28px;overflow:hidden;border:1px solid #e5e7eb;">
<div style="background:#fffbeb;padding:16px 22px;display:flex;align-items:center;justify-content:space-between;border-bottom:1px solid #e5e7eb;">
<div>
<span style="font-size:18px;font-weight:700;color:#111827;">📉 backup</span>
<span style="margin-left:8px;font-size:12px;color:#6b7280;">pool</span>
</div>
<span style="background:#b45309;color:white;padding:5px 14px;border-radius:20px;font-size:12px;font-weight:700;letter-spacing:.5px;">WARNING — CAPACITY</span>
</div>
<div style="padding:20px 22px;">
<table width="100%" cellspacing="0" cellpadding="0">
<tr>
<td style="width:16%;padding:8px 12px;background:#f9fafb;border-radius:6px;text-align:center;">
<div style="font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;">Total Size</div>
<div style="font-size:20px;font-weight:700;color:#111827;margin-top:2px;">7.3T</div>
</td>
<td style="width:4%;"></td>
<td style="width:16%;padding:8px 12px;background:#f9fafb;border-radius:6px;text-align:center;">
<div style="font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;">Used</div>
<div style="font-size:20px;font-weight:700;color:#111827;margin-top:2px;">6.1T</div>
</td>
<td style="width:4%;"></td>
<td style="width:16%;padding:8px 12px;background:#f9fafb;border-radius:6px;text-align:center;">
<div style="font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;">Free</div>
<div style="font-size:20px;font-weight:700;color:#111827;margin-top:2px;">1.2T</div>
</td>
<td style="width:4%;"></td>
<td style="width:16%;padding:8px 12px;background:#f9fafb;border-radius:6px;text-align:center;">
<div style="font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;">Capacity</div>
<div style="font-size:20px;font-weight:700;color:#b45309;margin-top:2px;">84%</div>
</td>
<td style="width:4%;"></td>
<td style="width:16%;padding:8px 12px;background:#f9fafb;border-radius:6px;text-align:center;">
<div style="font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;">Fragmentation</div>
<div style="font-size:20px;font-weight:700;color:#111827;margin-top:2px;">7%</div>
</td>
<td style="width:4%;"></td>
<td style="width:16%;padding:8px 12px;background:#f9fafb;border-radius:6px;text-align:center;">
<div style="font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;">Health</div>
<div style="font-size:20px;font-weight:700;color:#111827;margin-top:2px;">ONLINE</div>
</td>
</tr>
</table>
<div style="margin-top:16px;padding:12px 14px;background:#f8fafc;border-radius:6px;border:1px solid #e5e7eb;">
<span style="font-size:12px;font-weight:600;color:#374151;text-transform:uppercase;letter-spacing:.5px;">🔍 Last Scrub</span>
<span style="margin-left:10px;font-size:13px;color:#374151;">Completed &nbsp;|&nbsp; Jun 3 07:01:44 2025 &nbsp;|&nbsp; Errors: 0 errors</span>
</div>
</div>
</div>
<!-- POOL 3 — HEALTHY -->
<div style="background:white;border-radius:10px;box-shadow:0 1px 6px rgba(0,0,0,.10);margin-bottom:28px;overflow:hidden;border:1px solid #e5e7eb;">
<div style="background:#f0fdf4;padding:16px 22px;display:flex;align-items:center;justify-content:space-between;border-bottom:1px solid #e5e7eb;">
<div>
<span style="font-size:18px;font-weight:700;color:#111827;">📉 media</span>
<span style="margin-left:8px;font-size:12px;color:#6b7280;">pool</span>
</div>
<span style="background:#1a7f4b;color:white;padding:5px 14px;border-radius:20px;font-size:12px;font-weight:700;letter-spacing:.5px;">HEALTHY</span>
</div>
<div style="padding:20px 22px;">
<table width="100%" cellspacing="0" cellpadding="0">
<tr>
<td style="width:16%;padding:8px 12px;background:#f9fafb;border-radius:6px;text-align:center;">
<div style="font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;">Total Size</div>
<div style="font-size:20px;font-weight:700;color:#111827;margin-top:2px;">21.8T</div>
</td>
<td style="width:4%;"></td>
<td style="width:16%;padding:8px 12px;background:#f9fafb;border-radius:6px;text-align:center;">
<div style="font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;">Used</div>
<div style="font-size:20px;font-weight:700;color:#111827;margin-top:2px;">9.2T</div>
</td>
<td style="width:4%;"></td>
<td style="width:16%;padding:8px 12px;background:#f9fafb;border-radius:6px;text-align:center;">
<div style="font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;">Free</div>
<div style="font-size:20px;font-weight:700;color:#111827;margin-top:2px;">12.6T</div>
</td>
<td style="width:4%;"></td>
<td style="width:16%;padding:8px 12px;background:#f9fafb;border-radius:6px;text-align:center;">
<div style="font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;">Capacity</div>
<div style="font-size:20px;font-weight:700;color:#111827;margin-top:2px;">42%</div>
</td>
<td style="width:4%;"></td>
<td style="width:16%;padding:8px 12px;background:#f9fafb;border-radius:6px;text-align:center;">
<div style="font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;">Fragmentation</div>
<div style="font-size:20px;font-weight:700;color:#111827;margin-top:2px;">3%</div>
</td>
<td style="width:4%;"></td>
<td style="width:16%;padding:8px 12px;background:#f9fafb;border-radius:6px;text-align:center;">
<div style="font-size:11px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;">Health</div>
<div style="font-size:20px;font-weight:700;color:#1a7f4b;margin-top:2px;">ONLINE</div>
</td>
</tr>
</table>
<div style="margin-top:16px;padding:12px 14px;background:#f8fafc;border-radius:6px;border:1px solid #e5e7eb;">
<span style="font-size:12px;font-weight:600;color:#374151;text-transform:uppercase;letter-spacing:.5px;">🔍 Last Scrub</span>
<span style="margin-left:10px;font-size:13px;color:#374151;">Completed &nbsp;|&nbsp; Jun 3 07:02:11 2025 &nbsp;|&nbsp; Errors: 0 errors</span>
</div>
</div>
</div>
</td></tr>
<!-- Summary -->
<tr><td style="padding:0 24px 28px;">
<div style="background:white;border-radius:10px;border:1px solid #e5e7eb;padding:18px 22px;">
<div style="font-size:13px;font-weight:700;color:#374151;margin-bottom:10px;text-transform:uppercase;letter-spacing:.5px;">📋 Report Summary</div>
<table width="100%" cellspacing="0" cellpadding="0">
<tr>
<td style="font-size:13px;color:#6b7280;">Host</td>
<td style="font-size:13px;color:#111827;font-weight:600;">NAS08</td>
</tr>
<tr><td colspan="2" style="height:6px;"></td></tr>
<tr>
<td style="font-size:13px;color:#6b7280;">Report Time</td>
<td style="font-size:13px;color:#111827;font-weight:600;">2025-06-10 07:00:01</td>
</tr>
<tr><td colspan="2" style="height:6px;"></td></tr>
<tr>
<td style="font-size:13px;color:#6b7280;">Pools Checked</td>
<td style="font-size:13px;color:#111827;font-weight:600;">3</td>
</tr>
<tr><td colspan="2" style="height:6px;"></td></tr>
<tr>
<td style="font-size:13px;color:#6b7280;">Pools with Issues</td>
<td style="font-size:13px;font-weight:700;color:#b91c1c;">2</td>
</tr>
<tr><td colspan="2" style="height:6px;"></td></tr>
<tr>
<td style="font-size:13px;color:#6b7280;">Scrub Warning Threshold</td>
<td style="font-size:13px;color:#111827;font-weight:600;">8 days</td>
</tr>
</table>
</div>
</td></tr>
<!-- Footer -->
<tr><td style="background:#0f172a;border-radius:0 0 10px 10px;padding:16px 32px;text-align:center;">
<span style="font-size:11px;color:#64748b;">Automated ZFS Monitor &nbsp;|&nbsp; NAS08 &nbsp;|&nbsp; KingDezigns Infrastructure</span>
</td></tr>
</table>
</td></tr>
</table>
</body>
</html>