nas16-scripts/ddns/ddns_update.sh

#!/bin/bash
# =============================================================================
#  ddns_update.sh  —  DreamHost Dynamic DNS Updater  (v2 — hardened 2026-07-23)
#  KingDezigns Home Network — NAS16
#
#  Schedule every 15 minutes via OMV Scheduled Tasks:
#    Minute: */15   Hour/Day/Month/Weekday: *
#    Command: /usr/scripts/ddns/ddns_update.sh
#
#  Manages A records for:
#    albranch.org, indigorhayne.com, rufusking.com, kingdezigns.com (+subs)
#
#  v2 CHANGES (root cause: 2026-07-23 DreamHost API rate-limit outage — see
#  KingDezigns troubleshooting summary for full diagnosis):
#    1. Zone is fetched ONCE per run (1 API call) instead of once per record
#       (was 23 calls just to check status). All checks are done locally
#       against that snapshot.
#    2. Records are ADDED before the old value is REMOVED (was remove-then-add,
#       which left a domain with NO A record at all if the add step failed).
#    3. A sleep is inserted between each record's mutating calls to stay well
#       under DreamHost's documented add/remove throttling.
#    4. A max number of records are mutated per run (BATCH_LIMIT). Anything
#       left over is written to a pending file and picked up automatically on
#       the next 15-min run — no need to wait for the old remove-then-add loop
#       to hammer all 23 records in one shot.
#    5. If several records in a row come back with an API error, the run
#       aborts immediately (fail-fast) instead of grinding through the rest —
#       that's the signature of a rate limit, and keeps retrying makes it worse.
#    6. The real DreamHost error reason (the "data" field) is captured and
#       logged/emailed, not just the generic "error" result.
#    7. curl now uses --data-urlencode for every parameter instead of manual
#       string concatenation (safer for records like *.web.kingdezigns.com).
#
#  Behavior:
#    Every 15-min run, an update pass happens if ANY of the following is true:
#      - WAN IP changed since last cached value
#      - Daily sanity-check window (00:00–00:14)
#      - There are records left over (pending) from a previous run's batch
#        limit or a rate-limit abort
#    Each pass re-fetches the live zone once and compares it locally — so a
#    "pending" record that was fixed manually (or by a later successful
#    attempt) will simply drop out on its own.
# =============================================================================

# ── Configuration ─────────────────────────────────────────────────────────────
HOSTNAME_LABEL=$(hostname -s 2>/dev/null || echo "NAS16")
REPORT_TO="rufus.king@kingdezigns.com"
REPORT_FROM="ddns-monitor@nas16.local"     # Display only — Postfix uses your Gmail relay
API_KEY="4ACNN8PRYNQCMCWW"
LOG_FILE="/var/log/ddns/ddns_run.log"
IP_CACHE="/var/log/ddns/last_wan_ip.txt"
PENDING_FILE="/var/log/ddns/pending_records.txt"   # record<TAB>failed_attempt_count

# Rate-limit hardening knobs — tuned to stay well under DreamHost's documented
# dns-add_record throttle (~10 successful calls / 5 min, ~50 / hour).
BATCH_LIMIT=8                 # max records mutated (add+remove) per script run
SLEEP_BETWEEN_RECORDS=25      # seconds between each record's add/remove pair
FAIL_FAST_THRESHOLD=3         # abort the run after this many consecutive API errors
STALE_THRESHOLD=6             # attempts before a pending record is flagged "stale" in the email

# Note: Email sent via OMV-configured Postfix (Gmail relay). No SMTP config needed here.
# ──────────────────────────────────────────────────────────────────────────────

RECORDS=(
    "albranch.org"
    "www.albranch.org"
    "indigorhayne.com"
    "www.indigorhayne.com"
    "rufusking.com"
    "www.rufusking.com"
    "kingdezigns.com"
    "www.kingdezigns.com"
    "*.web.kingdezigns.com"
    "cloud.kingdezigns.com"
    "files08.kingdezigns.com"
    "files16.kingdezigns.com"
    "has.kingdezigns.com"
    "nginx.kingdezigns.com"
    "omv08.kingdezigns.com"
    "omv16.kingdezigns.com"
    "pihole.kingdezigns.com"
    "plex.kingdezigns.com"
    "vault.kingdezigns.com"
    "webmin.kingdezigns.com"
    "office.kingdezigns.com"
    "stocks.kingdezigns.com"
    "collabora.kingdezigns.com"
)

DH_API="https://api.dreamhost.com/"
REPORT_TIME=$(date "+%Y-%m-%d %H:%M:%S")

mkdir -p /var/log/ddns

# Trim log to last 5000 lines
if [[ -f "$LOG_FILE" ]]; then
    tail -n 5000 "$LOG_FILE" > "${LOG_FILE}.tmp" && mv "${LOG_FILE}.tmp" "$LOG_FILE"
fi

log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a "$LOG_FILE"; }

log "===== DDNS Update started on $HOSTNAME_LABEL (v2 hardened) ====="

# ── Helpers ───────────────────────────────────────────────────────────────────

unique_id() {
    cat /proc/sys/kernel/random/uuid 2>/dev/null \
        || python3 -c "import uuid; print(uuid.uuid4())" 2>/dev/null \
        || echo "$(date +%s%N)-$$"
}

get_wan_ip() {
    local ip
    for src in "https://api.ipify.org" "https://ifconfig.me/ip" "https://icanhazip.com"; do
        ip=$(curl -s --max-time 8 "$src" 2>/dev/null | tr -d '[:space:]')
        if [[ "$ip" =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ ]]; then
            echo "$ip"; return 0
        fi
    done
    return 1
}

# Fetches the ENTIRE DreamHost DNS zone in a single API call and prints one
# "record<TAB>value" line per A record on stdout. On any failure (including a
# throttled/erroring API), prints a reason to stderr and returns non-zero —
# callers must NOT treat that as "record doesn't exist".
fetch_zone_snapshot() {
    curl -s --max-time 20 -G "$DH_API" \
        --data-urlencode "key=${API_KEY}" \
        --data-urlencode "cmd=dns-list_records" \
        --data-urlencode "format=json" \
        --data-urlencode "unique_id=$(unique_id)" \
    | python3 -c "
import sys, json
try:
    data = json.load(sys.stdin)
    if data.get('result') != 'success':
        sys.stderr.write('LIST_ERROR: ' + str(data.get('data', 'unknown')) + '\n')
        sys.exit(2)
    for e in data.get('data', []):
        if e.get('type', '') == 'A':
            sys.stdout.write(e.get('record', '') + '\t' + e.get('value', '') + '\n')
except Exception as ex:
    sys.stderr.write('LIST_EXCEPTION: ' + str(ex) + '\n')
    sys.exit(2)
"
}

# Both print "result<TAB>data" on stdout — always something printable, so a
# JSON/network failure shows up as an explicit "exception" row in the log
# instead of silently looking like an empty/unknown result.
dh_add_record() {
    local record="$1" ip="$2"
    curl -s --max-time 15 -G "$DH_API" \
        --data-urlencode "key=${API_KEY}" \
        --data-urlencode "cmd=dns-add_record" \
        --data-urlencode "format=json" \
        --data-urlencode "unique_id=$(unique_id)" \
        --data-urlencode "record=${record}" \
        --data-urlencode "type=A" \
        --data-urlencode "value=${ip}" \
    | python3 -c "
import sys, json
try:
    d = json.load(sys.stdin)
    print(str(d.get('result', 'unknown')) + '\t' + str(d.get('data', '')))
except Exception as ex:
    print('exception\t' + str(ex))
" 2>/dev/null
}

dh_remove_record() {
    local record="$1" ip="$2"
    curl -s --max-time 15 -G "$DH_API" \
        --data-urlencode "key=${API_KEY}" \
        --data-urlencode "cmd=dns-remove_record" \
        --data-urlencode "format=json" \
        --data-urlencode "unique_id=$(unique_id)" \
        --data-urlencode "record=${record}" \
        --data-urlencode "type=A" \
        --data-urlencode "value=${ip}" \
    | python3 -c "
import sys, json
try:
    d = json.load(sys.stdin)
    print(str(d.get('result', 'unknown')) + '\t' + str(d.get('data', '')))
except Exception as ex:
    print('exception\t' + str(ex))
" 2>/dev/null
}

# ── Get WAN IP ────────────────────────────────────────────────────────────────

log "Checking current WAN IP..."
if ! WAN_IP=$(get_wan_ip); then
    log "ERROR: All WAN IP sources failed. Aborting — will retry next run."
    exit 1
fi
log "Current WAN IP: $WAN_IP"

PREV_IP=""
[[ -f "$IP_CACHE" ]] && PREV_IP=$(tr -d '[:space:]' < "$IP_CACHE")
log "Cached previous IP: ${PREV_IP:-<none>}"

# ── Load pending records left over from a previous run ────────────────────────

declare -A PENDING_ATTEMPTS
PENDING_COUNT=0
if [[ -f "$PENDING_FILE" ]]; then
    while IFS=$'\t' read -r p_rec p_attempts; do
        [[ -z "$p_rec" ]] && continue
        PENDING_ATTEMPTS["${p_rec,,}"]="${p_attempts:-0}"
        (( PENDING_COUNT++ ))
    done < "$PENDING_FILE"
fi
[[ $PENDING_COUNT -gt 0 ]] && log "Found $PENDING_COUNT record(s) pending from a previous run."

# ── Daily window check (00:00–00:14) ─────────────────────────────────────────

HOUR=$(date '+%H')
MINUTE=$(date '+%M')
IN_DAILY_WINDOW=false
if [[ "$HOUR" == "00" && 10#$MINUTE -lt 15 ]]; then
    IN_DAILY_WINDOW=true
    log "Daily sanity-check window active (00:00–00:14)."
fi

# ── Decide whether to run ─────────────────────────────────────────────────────

DO_UPDATE=false
TRIGGER_REASON=""

if [[ "$WAN_IP" != "$PREV_IP" ]]; then
    DO_UPDATE=true
    TRIGGER_REASON="WAN IP changed (${PREV_IP:-none} → ${WAN_IP})"
elif [[ "$IN_DAILY_WINDOW" == "true" ]]; then
    DO_UPDATE=true
    TRIGGER_REASON="Daily sanity-check (IP unchanged at ${WAN_IP})"
elif [[ $PENDING_COUNT -gt 0 ]]; then
    DO_UPDATE=true
    TRIGGER_REASON="Retrying $PENDING_COUNT pending record(s) from a previous run"
fi

if [[ "$DO_UPDATE" == "false" ]]; then
    log "IP unchanged ($WAN_IP) and no pending records. No action needed — exiting silently."
    echo "$WAN_IP" > "$IP_CACHE"
    log "===== DDNS Update complete (no change) on $HOSTNAME_LABEL ====="
    exit 0
fi

log "Trigger: $TRIGGER_REASON"

# ── Fetch the full zone ONCE ──────────────────────────────────────────────────

log "Fetching current DNS zone from DreamHost (single API call)..."
ZONE_ERR_FILE=$(mktemp)
ZONE_RAW=$(fetch_zone_snapshot 2> "$ZONE_ERR_FILE")
ZONE_RC=$?
if [[ $ZONE_RC -ne 0 ]]; then
    ZONE_ERR=$(cat "$ZONE_ERR_FILE")
    rm -f "$ZONE_ERR_FILE"
    log "ERROR: could not fetch DNS zone snapshot ($ZONE_ERR) — likely DreamHost API throttling/outage."
    log "Not touching pending state or the IP cache — will retry in full on the next scheduled run."
    log "===== DDNS Update aborted on $HOSTNAME_LABEL ====="
    # Lightweight failure alert — deliberately not the full report template.
    {
        echo "To: $REPORT_TO"
        echo "From: $REPORT_FROM"
        echo "Subject: [DDNS] $HOSTNAME_LABEL — API UNREACHABLE | $REPORT_TIME"
        echo "Content-Type: text/plain; charset=UTF-8"
        echo ""
        echo "DDNS update on $HOSTNAME_LABEL could not even fetch the DNS zone from DreamHost."
        echo "Reason: $ZONE_ERR"
        echo ""
        echo "This usually means the DreamHost API is rate-limiting this key right now."
        echo "No changes were attempted. The script will retry automatically on the next"
        echo "15-minute run — no action needed unless this repeats for more than an hour."
    } | sendmail -t
    exit 1
fi
rm -f "$ZONE_ERR_FILE"

declare -A ZONE
ZONE_RECORD_COUNT=0
while IFS=$'\t' read -r z_rec z_val; do
    [[ -z "$z_rec" ]] && continue
    ZONE["${z_rec,,}"]="$z_val"
    (( ZONE_RECORD_COUNT++ ))
done <<< "$ZONE_RAW"
log "Zone snapshot retrieved — $ZONE_RECORD_COUNT A record(s) found on DreamHost."

# ── Determine which records need updating (all local — no API calls) ─────────

RECORDS_OK=()
NEEDS_UPDATE=()
for rec in "${RECORDS[@]}"; do
    cur_ip="${ZONE[${rec,,}]:-}"
    if [[ "$cur_ip" == "$WAN_IP" ]]; then
        RECORDS_OK+=("$rec")
    else
        log "  [$rec] needs update — DreamHost shows '${cur_ip:-<not found>}'"
        NEEDS_UPDATE+=("$rec")
    fi
done

log "Checked all ${#RECORDS[@]} records against target IP $WAN_IP — ${#RECORDS_OK[@]} already correct, ${#NEEDS_UPDATE[@]} need updating."

RECORDS_FAILED=()
RECORDS_DEFERRED=()
ATTEMPTED_COUNT=0
RATE_LIMIT_ABORTED=false

if [[ ${#NEEDS_UPDATE[@]} -eq 0 ]]; then
    log "Nothing to update."
    rm -f "$PENDING_FILE"
else
    # Split into this run's batch vs. overflow (batch-limit deferral)
    THIS_RUN_BATCH=("${NEEDS_UPDATE[@]:0:$BATCH_LIMIT}")
    OVERFLOW=("${NEEDS_UPDATE[@]:$BATCH_LIMIT}")

    if [[ ${#OVERFLOW[@]} -gt 0 ]]; then
        log "Batch limit is $BATCH_LIMIT records/run — deferring ${#OVERFLOW[@]} record(s) to the next run: ${OVERFLOW[*]}"
        RECORDS_DEFERRED+=("${OVERFLOW[@]}")
    fi

    CONSECUTIVE_ERRORS=0
    FIRST=true
    for rec in "${THIS_RUN_BATCH[@]}"; do
        if [[ "$RATE_LIMIT_ABORTED" == "true" ]]; then
            RECORDS_DEFERRED+=("$rec")
            continue
        fi

        if [[ "$FIRST" == "false" ]]; then
            sleep "$SLEEP_BETWEEN_RECORDS"
        fi
        FIRST=false

        old_ip="${ZONE[${rec,,}]:-}"
        log "  Updating [$rec]  ${old_ip:-<not found>} → $WAN_IP"

        IFS=$'\t' read -r add_result add_data <<< "$(dh_add_record "$rec" "$WAN_IP")"
        log "    add:    $add_result${add_data:+ ($add_data)}"
        (( ATTEMPTED_COUNT++ ))

        if [[ "$add_result" == "success" ]]; then
            RECORDS_OK+=("$rec")
            CONSECUTIVE_ERRORS=0

            # Clean up the stale old value now that the new one is live.
            if [[ -n "$old_ip" && "$old_ip" != "$WAN_IP" ]]; then
                IFS=$'\t' read -r rm_result rm_data <<< "$(dh_remove_record "$rec" "$old_ip")"
                log "    remove: $rm_result${rm_data:+ ($rm_data)}  (cleanup of old value $old_ip)"
                if [[ "$rm_result" != "success" ]]; then
                    log "    NOTE: old A record ($old_ip) for [$rec] could not be removed — DNS may briefly round-robin between old and new IP until a future cleanup pass."
                fi
            fi
        else
            log "  WARNING: update failed for [$rec] — $add_data"
            RECORDS_FAILED+=("${rec}::${add_data}")
            (( CONSECUTIVE_ERRORS++ ))
            if [[ $CONSECUTIVE_ERRORS -ge $FAIL_FAST_THRESHOLD ]]; then
                log "WARNING: $FAIL_FAST_THRESHOLD consecutive API errors — this looks like DreamHost rate-limiting. Aborting remaining updates this run rather than making it worse."
                RATE_LIMIT_ABORTED=true
            fi
        fi
    done
fi

# ── Persist pending state for next run ────────────────────────────────────────

declare -A NEW_PENDING
for rec in "${RECORDS_DEFERRED[@]}"; do
    key="${rec,,}"
    NEW_PENDING["$key"]="${PENDING_ATTEMPTS[$key]:-0}"   # not actually attempted — don't increment
done
for entry in "${RECORDS_FAILED[@]}"; do
    rec="${entry%%::*}"
    key="${rec,,}"
    prev="${PENDING_ATTEMPTS[$key]:-0}"
    NEW_PENDING["$key"]=$(( prev + 1 ))
done

STALE_RECORDS=()
if [[ ${#NEW_PENDING[@]} -gt 0 ]]; then
    : > "$PENDING_FILE"
    for key in "${!NEW_PENDING[@]}"; do
        echo -e "${key}\t${NEW_PENDING[$key]}" >> "$PENDING_FILE"
        if [[ ${NEW_PENDING[$key]} -ge $STALE_THRESHOLD ]]; then
            STALE_RECORDS+=("$key (${NEW_PENDING[$key]} failed attempts)")
        fi
    done
else
    rm -f "$PENDING_FILE"
fi

# ── Persist WAN IP cache ───────────────────────────────────────────────────────
echo "$WAN_IP" > "$IP_CACHE"
log "IP cache updated to $WAN_IP"

log "Update complete — OK: ${#RECORDS_OK[@]}  Failed: ${#RECORDS_FAILED[@]}  Deferred: ${#RECORDS_DEFERRED[@]}  API calls used: $(( ZONE_RECORD_COUNT >= 0 ? 1 : 1 ))+$(( ATTEMPTED_COUNT * 2 ))"

# ── Skip email entirely if there was truly nothing to report ─────────────────
if [[ ${#NEEDS_UPDATE[@]} -eq 0 ]]; then
    log "===== DDNS Update complete (all records already correct) on $HOSTNAME_LABEL ====="
    exit 0
fi

# ── Determine overall status ──────────────────────────────────────────────────
if [[ ${#RECORDS_FAILED[@]} -eq 0 && ${#RECORDS_DEFERRED[@]} -eq 0 ]]; then
    OVERALL_STATUS="SUCCESS"
elif [[ ${#RECORDS_FAILED[@]} -eq 0 && ${#RECORDS_DEFERRED[@]} -gt 0 ]]; then
    OVERALL_STATUS="DEFERRED"
elif [[ ${#RECORDS_FAILED[@]} -gt 0 && ( $ATTEMPTED_COUNT -gt ${#RECORDS_FAILED[@]} || ${#RECORDS_DEFERRED[@]} -gt 0 ) ]]; then
    OVERALL_STATUS="PARTIAL"
else
    OVERALL_STATUS="FAILED"
fi

# ── Build HTML email — matching KingDezigns email style guide ────────────────

case "$OVERALL_STATUS" in
    SUCCESS)
        BADGE_HTML="<div style=\"background:#1a7f4b;color:white;padding:10px 20px;border-radius:8px;font-size:14px;font-weight:700;\">✅ ALL UPDATED</div>"
        BANNER_HTML="<tr><td style=\"background:#1a7f4b;padding:12px 32px;\"><span style=\"color:white;font-size:14px;font-weight:600;\">✅ &nbsp;All A records updated successfully — no action required.</span></td></tr>"
        ;;
    DEFERRED)
        BADGE_HTML="<div style=\"background:#b45309;color:white;padding:10px 20px;border-radius:8px;font-size:14px;font-weight:700;\">🕓 BATCHED</div>"
        BANNER_HTML="<tr><td style=\"background:#b45309;padding:12px 32px;\"><span style=\"color:white;font-size:14px;font-weight:600;\">🕓 &nbsp;Some records queued for the next run (rate-limit batching) — no errors, no action required.</span></td></tr>"
        ;;
    PARTIAL)
        BADGE_HTML="<div style=\"background:#b45309;color:white;padding:10px 20px;border-radius:8px;font-size:14px;font-weight:700;\">⚠️ PARTIAL</div>"
        BANNER_HTML="<tr><td style=\"background:#b45309;padding:12px 32px;\"><span style=\"color:white;font-size:14px;font-weight:600;\">⚠️ &nbsp;Some records failed to update — they will be retried automatically.</span></td></tr>"
        ;;
    FAILED)
        BADGE_HTML="<div style=\"background:#b91c1c;color:white;padding:10px 20px;border-radius:8px;font-size:14px;font-weight:700;\">🚨 FAILED</div>"
        BANNER_HTML="<tr><td style=\"background:#b91c1c;padding:12px 32px;\"><span style=\"color:white;font-size:14px;font-weight:600;\">🚨 &nbsp;Updates failed — likely DreamHost rate-limiting. Will retry automatically; check if this persists past an hour.</span></td></tr>"
        ;;
esac

# Record rows: OK
RECORD_ROWS_HTML=""
for rec in "${RECORDS_OK[@]}"; do
    RECORD_ROWS_HTML+="
        <tr>
          <td style=\"padding:9px 14px;border-bottom:1px solid #f3f4f6;font-family:monospace;font-size:13px;color:#111827;\">$rec</td>
          <td style=\"padding:9px 14px;border-bottom:1px solid #f3f4f6;\"><span style=\"background:#d1fae5;color:#065f46;padding:3px 10px;border-radius:20px;font-size:11px;font-weight:700;\">✓ UPDATED</span></td>
          <td style=\"padding:9px 14px;border-bottom:1px solid #f3f4f6;font-family:monospace;font-size:13px;color:#1a7f4b;font-weight:600;\">$WAN_IP</td>
        </tr>"
done

# Record rows: FAILED (with real DreamHost reason)
for entry in "${RECORDS_FAILED[@]}"; do
    rec="${entry%%::*}"
    reason="${entry#*::}"
    RECORD_ROWS_HTML+="
        <tr>
          <td style=\"padding:9px 14px;border-bottom:1px solid #f3f4f6;font-family:monospace;font-size:13px;color:#111827;\">$rec</td>
          <td style=\"padding:9px 14px;border-bottom:1px solid #f3f4f6;\"><span style=\"background:#fee2e2;color:#991b1b;padding:3px 10px;border-radius:20px;font-size:11px;font-weight:700;\">✗ FAILED</span></td>
          <td style=\"padding:9px 14px;border-bottom:1px solid #f3f4f6;font-family:monospace;font-size:12px;color:#b91c1c;font-weight:600;\">$reason</td>
        </tr>"
done

# Record rows: DEFERRED (queued for next run — not a failure)
for rec in "${RECORDS_DEFERRED[@]}"; do
    RECORD_ROWS_HTML+="
        <tr>
          <td style=\"padding:9px 14px;border-bottom:1px solid #f3f4f6;font-family:monospace;font-size:13px;color:#111827;\">$rec</td>
          <td style=\"padding:9px 14px;border-bottom:1px solid #f3f4f6;\"><span style=\"background:#fef3c7;color:#92400e;padding:3px 10px;border-radius:20px;font-size:11px;font-weight:700;\">🕓 QUEUED</span></td>
          <td style=\"padding:9px 14px;border-bottom:1px solid #f3f4f6;font-family:monospace;font-size:12px;color:#92400e;\">retrying next run</td>
        </tr>"
done

# Manual action block — only shown when there are real (non-deferred) failures
MANUAL_BLOCK=""
if [[ ${#RECORDS_FAILED[@]} -gt 0 ]]; then
    FAILED_LIST_HTML=""
    for entry in "${RECORDS_FAILED[@]}"; do
        rec="${entry%%::*}"
        reason="${entry#*::}"
        FAILED_LIST_HTML+="<div style=\"font-family:monospace;font-size:13px;padding:3px 0;\">• $rec <span style=\"color:#9a5b1f;\">($reason)</span></div>"
    done
    MANUAL_BLOCK="
      <tr><td style=\"padding:0 24px 24px;\">
        <div style=\"background:#fff7ed;border:1px solid #fed7aa;border-radius:10px;padding:20px 22px;\">
          <div style=\"font-size:13px;font-weight:700;color:#9a3412;margin-bottom:12px;text-transform:uppercase;letter-spacing:.5px;\">⚠️ Failed Records — Will Auto-Retry</div>
          <p style=\"font-size:13px;color:#7c3516;margin:0 0 10px;\">These records did not update this run. They stay queued and will be retried automatically on the next 15-minute run — no manual action needed unless they keep failing for more than an hour.</p>
          <div style=\"background:#fff;border:1px solid #fed7aa;border-radius:6px;padding:12px 16px;margin-bottom:12px;\">
            $FAILED_LIST_HTML
          </div>
          <table cellspacing=\"0\" cellpadding=\"0\" width=\"100%\">
            <tr>
              <td style=\"font-size:13px;color:#7c3516;\">Target A record value</td>
              <td style=\"font-family:monospace;font-size:15px;font-weight:700;color:#b91c1c;\">$WAN_IP</td>
            </tr>
            <tr><td colspan=\"2\" style=\"height:8px;\"></td></tr>
            <tr>
              <td style=\"font-size:13px;color:#7c3516;\">DreamHost DNS Panel (manual fallback)</td>
              <td><a href=\"https://panel.dreamhost.com/index.cgi?tree=domain.dashboard\" style=\"color:#b45309;font-size:13px;\">panel.dreamhost.com → Manage Websites → DNS</a></td>
            </tr>
          </table>
        </div>
      </td></tr>"
fi

# Stale block — records that have failed repeatedly across multiple runs
STALE_BLOCK=""
if [[ ${#STALE_RECORDS[@]} -gt 0 ]]; then
    STALE_LIST_HTML=""
    for s in "${STALE_RECORDS[@]}"; do
        STALE_LIST_HTML+="<div style=\"font-family:monospace;font-size:13px;padding:3px 0;\">• $s</div>"
    done
    STALE_BLOCK="
      <tr><td style=\"padding:0 24px 24px;\">
        <div style=\"background:#fff1f2;border:1px solid #f09595;border-radius:10px;padding:20px 22px;\">
          <div style=\"font-size:13px;font-weight:700;color:#7a1f1f;margin-bottom:12px;text-transform:uppercase;letter-spacing:.5px;\">🔴 Stale — Failing Across Multiple Runs</div>
          <p style=\"font-size:13px;color:#7a1f1f;margin:0 0 10px;\">These records have failed $STALE_THRESHOLD+ times across separate runs — this is no longer likely to be a transient rate limit. Worth checking manually.</p>
          <div style=\"background:#fff;border:1px solid #f09595;border-radius:6px;padding:12px 16px;\">
            $STALE_LIST_HTML
          </div>
        </div>
      </td></tr>"
fi

# ── Compose full HTML email ───────────────────────────────────────────────────
HTML_BODY=$(cat <<EOF
<!DOCTYPE html>
<html lang="en">
<head><meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1"></head>
<body style="margin:0;padding:0;background:#f1f5f9;font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,sans-serif;">
<table width="100%" cellspacing="0" cellpadding="0" style="background:#f1f5f9;padding:30px 0;">
  <tr><td align="center">
  <table width="680" cellspacing="0" cellpadding="0" style="max-width:680px;width:100%;">

    <!-- Header -->
    <tr><td style="background:#0f172a;border-radius:10px 10px 0 0;padding:28px 32px;">
      <table width="100%" cellspacing="0" cellpadding="0">
        <tr>
          <td>
            <div style="font-size:11px;color:#94a3b8;letter-spacing:1px;text-transform:uppercase;">DreamHost Dynamic DNS Monitor</div>
            <div style="font-size:26px;font-weight:800;color:white;margin-top:4px;">🌐 $HOSTNAME_LABEL</div>
            <div style="font-size:13px;color:#94a3b8;margin-top:4px;">$REPORT_TIME &nbsp;|&nbsp; ${#RECORDS[@]} records monitored &nbsp;|&nbsp; Trigger: $TRIGGER_REASON</div>
          </td>
          <td align="right">$BADGE_HTML</td>
        </tr>
      </table>
    </td></tr>

    <!-- Status banner -->
    $BANNER_HTML

    <!-- Record results table -->
    <tr><td style="padding:28px 24px 8px;">
      <div style="background:white;border-radius:10px;box-shadow:0 1px 6px rgba(0,0,0,.10);overflow:hidden;border:1px solid #e5e7eb;">
        <div style="background:#f8fafc;padding:14px 22px;border-bottom:1px solid #e5e7eb;">
          <span style="font-size:13px;font-weight:700;color:#374151;text-transform:uppercase;letter-spacing:.5px;">📋 A Record Update Results</span>
          <span style="margin-left:12px;font-size:12px;color:#6b7280;">${#RECORDS_OK[@]} correct &nbsp;·&nbsp; ${#RECORDS_FAILED[@]} failed &nbsp;·&nbsp; ${#RECORDS_DEFERRED[@]} queued</span>
        </div>
        <table width="100%" cellspacing="0" cellpadding="0">
          <tr style="background:#f9fafb;">
            <th style="text-align:left;padding:8px 14px;font-size:10px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;font-weight:600;border-bottom:1px solid #e5e7eb;">Record</th>
            <th style="text-align:left;padding:8px 14px;font-size:10px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;font-weight:600;border-bottom:1px solid #e5e7eb;">Status</th>
            <th style="text-align:left;padding:8px 14px;font-size:10px;color:#9ca3af;text-transform:uppercase;letter-spacing:.5px;font-weight:600;border-bottom:1px solid #e5e7eb;">Detail</th>
          </tr>
          $RECORD_ROWS_HTML
        </table>
      </div>
    </td></tr>

    <!-- Failed block (only on real failures) -->
    $MANUAL_BLOCK

    <!-- Stale block (only on repeated multi-run failures) -->
    $STALE_BLOCK

    <!-- Summary card -->
    <tr><td style="padding:16px 24px 28px;">
      <div style="background:white;border-radius:10px;border:1px solid #e5e7eb;padding:18px 22px;">
        <div style="font-size:13px;font-weight:700;color:#374151;margin-bottom:10px;text-transform:uppercase;letter-spacing:.5px;">📊 Run Summary</div>
        <table width="100%" cellspacing="0" cellpadding="0">
          <tr>
            <td style="font-size:13px;color:#6b7280;">Host</td>
            <td style="font-size:13px;color:#111827;font-weight:600;">$HOSTNAME_LABEL</td>
          </tr>
          <tr><td colspan="2" style="height:6px;"></td></tr>
          <tr>
            <td style="font-size:13px;color:#6b7280;">Run Time</td>
            <td style="font-size:13px;color:#111827;font-weight:600;">$REPORT_TIME</td>
          </tr>
          <tr><td colspan="2" style="height:6px;"></td></tr>
          <tr>
            <td style="font-size:13px;color:#6b7280;">Trigger</td>
            <td style="font-size:13px;color:#111827;font-weight:600;">$TRIGGER_REASON</td>
          </tr>
          <tr><td colspan="2" style="height:6px;"></td></tr>
          <tr>
            <td style="font-size:13px;color:#6b7280;">Target WAN IP</td>
            <td style="font-family:monospace;font-size:14px;color:#111827;font-weight:700;">$WAN_IP</td>
          </tr>
          <tr><td colspan="2" style="height:6px;"></td></tr>
          <tr>
            <td style="font-size:13px;color:#6b7280;">Previous IP</td>
            <td style="font-family:monospace;font-size:13px;color:#6b7280;">${PREV_IP:-<none cached>}</td>
          </tr>
          <tr><td colspan="2" style="height:6px;"></td></tr>
          <tr>
            <td style="font-size:13px;color:#6b7280;">Records Correct</td>
            <td style="font-size:13px;color:#1a7f4b;font-weight:700;">${#RECORDS_OK[@]} / ${#RECORDS[@]}</td>
          </tr>
          <tr><td colspan="2" style="height:6px;"></td></tr>
          <tr>
            <td style="font-size:13px;color:#6b7280;">API Calls This Run</td>
            <td style="font-size:13px;color:#111827;font-weight:600;">1 zone fetch + $(( ATTEMPTED_COUNT * 2 )) mutation call(s)</td>
          </tr>
          <tr><td colspan="2" style="height:6px;"></td></tr>
          <tr>
            <td style="font-size:13px;color:#6b7280;">Rate-Limit Abort Triggered</td>
            <td style="font-size:13px;color:#111827;font-weight:600;">$([ "$RATE_LIMIT_ABORTED" == "true" ] && echo "Yes — stopped early" || echo "No")</td>
          </tr>
          <tr><td colspan="2" style="height:6px;"></td></tr>
          <tr>
            <td style="font-size:13px;color:#6b7280;">Overall Status</td>
            <td style="font-size:13px;font-weight:700;color:$(
                case "$OVERALL_STATUS" in
                    SUCCESS) echo "#1a7f4b" ;;
                    DEFERRED|PARTIAL) echo "#b45309" ;;
                    *)       echo "#b91c1c" ;;
                esac
            );">$OVERALL_STATUS</td>
          </tr>
        </table>
      </div>
    </td></tr>

    <!-- Footer -->
    <tr><td style="background:#0f172a;border-radius:0 0 10px 10px;padding:16px 32px;text-align:center;">
      <span style="font-size:11px;color:#64748b;">Automated DDNS Monitor &nbsp;|&nbsp; $HOSTNAME_LABEL &nbsp;|&nbsp; KingDezigns Infrastructure</span>
    </td></tr>

  </table>
  </td></tr>
</table>
</body>
</html>
EOF
)

# ── Send email via OMV Postfix (Gmail relay) ──────────────────────────────────
SUBJECT="[DDNS] $HOSTNAME_LABEL — $OVERALL_STATUS | $REPORT_TIME"

{
    echo "To: $REPORT_TO"
    echo "From: $REPORT_FROM"
    echo "Subject: $SUBJECT"
    echo "MIME-Version: 1.0"
    echo "Content-Type: text/html; charset=UTF-8"
    echo ""
    echo "$HTML_BODY"
} | sendmail -t

log "Email sent to $REPORT_TO — Overall status: $OVERALL_STATUS"
log "===== DDNS Update complete on $HOSTNAME_LABEL ====="