#!/bin/bash
#
# nd-ha - NDOS active-passive high availability.
#
# Two sensors on parallel wire paths. Only the active node has its capture
# ports (IPS pairs or the IDS port) up; the standby keeps them
# administratively down, so it never forwards or transmits. keepalived
# (VRRP over a dedicated HA cable) decides which node is active; this tool
# renders its configuration, is its health check (track_script) and handles
# its state changes (notify FIFO).
#
# States in ha.conf (HA_STATE):
#   created   node 1 after "Create cluster", waiting for node 2's token.
#             Capture ports are NOT touched (the box keeps forwarding).
#   pending   node 2 after "Join cluster", waiting for node 1. Ports held
#             down, keepalived off, peer channel up (receives the first sync).
#   enabled   cluster member, keepalived running.
#   detached  left the cluster; ports held down until "Release capture ports".
#
# See ndos-ha-active-passive-design.md (NDOS project) for the full design.

# "A && B || fail" is used on purpose: fail when either test is false.
# shellcheck disable=SC2015
set -u
umask 022

LIB="${ND_HA_LIB:-/usr/libexec/nd-ha/nd-ha-common.sh}"
# shellcheck source=nd-ha-common.sh
. "$LIB" || { echo "nd-ha: cannot load $LIB" >&2; exit 1; }

SELF="${ND_HA_SELF:-/usr/sbin/nd-ha}"
CHECK_FILE="$RUN/check"

# --------------------------------------------------
# Helpers
# --------------------------------------------------
need_root() {
    [ -n "$R" ] && return 0
    [ "$(id -u)" -eq 0 ] || ha_die "must run as root"
}

transition_lock() {
    # Serialises promotion, demotion and the health check (fd 8).
    exec 8>"$RUN/transition.lock"
    flock -w "${1:-15}" 8 || ha_log "transition lock busy for ${1:-15} s - continuing"
}

controlled() {
    # Capture ports are under nd-ha control
    case "${HA_STATE:-}" in pending|enabled|detached) return 0 ;; esac
    return 1
}

unit_running() {
    systemctl is-active --quiet "$1" 2>/dev/null
}

check_result_set() {
    # Record the health-check result; log only when it changes.
    local new="$1" old=""
    old=$(cat "$CHECK_FILE" 2>/dev/null)
    if [ "$new" != "$old" ]; then
        printf '%s\n' "$new" > "$CHECK_FILE.tmp" && mv -f "$CHECK_FILE.tmp" "$CHECK_FILE"
        ha_log "check: $new"
    fi
}

holddown_active() {
    local until
    until=$(cat "$HOLDDOWN_FILE" 2>/dev/null) || return 1
    case "$until" in ''|*[!0-9]*) return 1 ;; esac
    [ "$(now)" -lt "$until" ]
}

zeek_start() {
    [ -n "$R" ] && return 0
    [ -e "$NSM_OPERATOR_STOP" ] && return 0
    [ -e "$NSM_CONFIGURED" ] || return 0
    systemctl reset-failed zeek.service >/dev/null 2>&1
    systemctl start --no-block zeek.service >/dev/null 2>&1
}

zeek_stop() {
    [ -n "$R" ] && return 0
    systemctl stop --no-block zeek.service >/dev/null 2>&1
}

i_yield() {
    # Both nodes forwarding: the one that started later yields; node 2 on a tie.
    if [ "$S_SINCE" -gt "$P_SINCE" ]; then return 0; fi
    if [ "$S_SINCE" -eq "$P_SINCE" ] && [ "$HA_NODE" = 2 ]; then return 0; fi
    return 1
}

mgmt_ip() {
    # First IPv4 address of interface $1
    ip -4 -o addr show dev "$1" 2>/dev/null |
        awk '{ split($4, a, "/"); print a[1]; exit }'
}

# --------------------------------------------------
# Rendering (everything HA needs at runtime lives in /run)
# --------------------------------------------------
render_dirs() {
    install -d -m 0755 "$RUN" "$R/data/nd"
    install -d -m 0711 "$HA_DIR"
    install -d -m 0700 "$HA_APPLY"
    install -d -m 0755 "$HA_SYNC_DIR"
    install -d -m 0700 "$HA_SPOOL"
    if [ -z "$R" ] && id "$HA_USER" >/dev/null 2>&1; then
        chown "$HA_USER:$HA_USER" "$HA_SPOOL"
        install -d -m 0750 -o "$HA_USER" -g "$HA_USER" /var/lib/nd-ha
    fi
}

fix_ha_account() {
    # sshd (UsePAM no) refuses a locked account ("!..."). systemd-sysusers
    # creates accounts locked, so turn a locked field into "*" (no password,
    # key login only).
    [ -n "$R" ] && return 0
    local f
    f=$(awk -F: -v u="$HA_USER" '$1 == u { print $2 }' /etc/shadow 2>/dev/null)
    case "$f" in
        '!'*) usermod -p '*' "$HA_USER" >/dev/null 2>&1 &&
              ha_log "account $HA_USER: unlocked for key login" ;;
    esac
}

render_network() {
    local f base d
    install -d -m 0755 "$RUN_NET_DIR"
    # Remove what an earlier run left (only our own drop-ins)
    for d in "$RUN_NET_DIR/$CAPTURE_PREFIX"*.network.d; do
        [ -d "$d" ] || continue
        rm -f "$d/50-nd-ha.conf"
        rmdir "$d" 2>/dev/null
    done
    if controlled; then
        capture_files | while IFS= read -r f; do
            base=$(basename "$f")
            install -d -m 0755 "$RUN_NET_DIR/$base.d"
            cat > "$RUN_NET_DIR/$base.d/50-nd-ha.conf" <<'EOF'
# nd-ha: on HA nodes only nd-ha raises capture ports (on the active node).
[Link]
ActivationPolicy=manual
EOF
        done
    fi
    case "${HA_STATE:-}" in
        pending|enabled)
            cat > "$HA_LINK_NETFILE.tmp" <<EOF
# nd-ha: dedicated HA cable (written at boot from /data/nd/ha/ha.conf)
[Match]
Name=$HA_IFACE

[Link]
RequiredForOnline=no

[Network]
Address=$HA_LOCAL_IP/$HA_NET_LEN
DHCP=no
LinkLocalAddressing=no
IPv6AcceptRA=no
LLDP=no
EmitLLDP=no
ConfigureWithoutCarrier=yes
EOF
            mv -f "$HA_LINK_NETFILE.tmp" "$HA_LINK_NETFILE"
            ;;
        *)
            rm -f "$HA_LINK_NETFILE"
            ;;
    esac
}

networkd_reload() {
    [ -n "$R" ] && return 0
    unit_running systemd-networkd || return 0
    networkctl reload >/dev/null 2>&1
}

render_suricata_env() {
    case "${HA_STATE:-}" in
        pending|enabled)
            # Sessions that were open at failover reach the new active node
            # mid-stream. Without this, IPS mode drops them (midstream-policy
            # resolves to drop-flow).
            printf '%s\n' \
                "# nd-ha: mid-stream pick-up so sessions survive a failover" \
                "ND_SURICATA_HA_ARGS=--set stream.midstream=true --set stream.midstream-policy=ignore" \
                > "$SURICATA_ENV.tmp" && mv -f "$SURICATA_ENV.tmp" "$SURICATA_ENV"
            ;;
        *)
            rm -f "$SURICATA_ENV"
            ;;
    esac
}

render_keepalived() {
    local tmp vip_block
    if [ -n "$HA_VIP" ]; then
        vip_block="        $HA_VIP dev $HA_MGMT_IFACE"
    else
        vip_block=""
    fi
    tmp=$(mktemp "$RUN/.keepalived.XXXXXX") || return 1
    cat > "$tmp" <<EOF
# Written by nd-ha from /data/nd/ha/ha.conf - do not edit.
global_defs {
    router_id ndos-ha-node$HA_NODE
    script_user root
    enable_script_security
    vrrp_garp_master_delay 1
    vrrp_garp_master_repeat 3
    vrrp_notify_fifo $RUN/notify.fifo
    vrrp_notify_fifo_script "$SELF notify-fifo"
}

vrrp_script nd_ha_check {
    script "$SELF check"
    interval 2
    timeout 5
    fall 2
    rise 2
    weight 0
    # Unhealthy until the first check passed (no MASTER before Suricata is up)
    init_fail
}

vrrp_instance $HA_INSTANCE {
    state BACKUP
    nopreempt
    interface $HA_IFACE
    virtual_router_id $HA_VRID
    priority $HA_PRIORITY
    advert_int 1
    unicast_src_ip $HA_LOCAL_IP
    unicast_peer {
        $HA_PEER_IP
    }
    authentication {
        auth_type PASS
        auth_pass $HA_AUTH
    }
    virtual_ipaddress {
        $HA_LINK_VIP/32 dev $HA_IFACE
$vip_block
    }
    track_script {
        nd_ha_check
    }
}
EOF
    chmod 0600 "$tmp" && mv -f "$tmp" "$KA_CONF"
}

render_sshd() {
    local tmp
    tmp=$(mktemp "$RUN/.sshd.XXXXXX") || return 1
    cat > "$tmp" <<EOF
# nd-ha peer channel. Only the peer node's sync key, only the forced command.
Port $HA_SSH_PORT
AddressFamily inet
ListenAddress 0.0.0.0
HostKey /etc/ssh/ssh_host_ed25519_key
PidFile $RUN/sshd.pid
AllowUsers $HA_USER
AuthenticationMethods publickey
PubkeyAuthentication yes
PasswordAuthentication no
KbdInteractiveAuthentication no
PermitEmptyPasswords no
PermitRootLogin no
UsePAM no
AuthorizedKeysFile /data/nd/ha/authorized_keys
StrictModes yes
ForceCommand /usr/libexec/nd-ha/nd-ha-remote
AllowTcpForwarding no
AllowStreamLocalForwarding no
AllowAgentForwarding no
GatewayPorts no
X11Forwarding no
PermitTunnel no
PermitTTY no
PermitUserRC no
PermitUserEnvironment no
PrintMotd no
ClientAliveInterval 15
ClientAliveCountMax 3
LoginGraceTime 10
MaxAuthTries 2
MaxSessions 10
MaxStartups 10:30:20
EOF
    chmod 0644 "$tmp" && mv -f "$tmp" "$SSHD_CONF"
}

cmd_render() {
    # Boot (nd-ha-render.service, before networkd) and after config changes.
    need_root
    if ! conf_load; then
        cleanup_runtime
        return 0
    fi
    render_dirs
    fix_ha_account
    [ -f "$STATE_FILE" ] || state_write INIT 0 "starting"
    render_network
    render_suricata_env
    if [ -z "$R" ] && unit_running iptables; then
        fw_apply
    fi
    networkd_reload
    if controlled; then
        state_read
        [ "$S_FORWARDING" = 1 ] || ports_set down
    fi
    return 0
}

cleanup_runtime() {
    local d
    for d in "$RUN_NET_DIR/$CAPTURE_PREFIX"*.network.d; do
        [ -d "$d" ] || continue
        rm -f "$d/50-nd-ha.conf"
        rmdir "$d" 2>/dev/null
    done
    rm -f "$HA_LINK_NETFILE" "$SURICATA_ENV" "$KA_CONF" "$SSHD_CONF" "$STATE_FILE" \
          "$PEER_FILE" "$CHECK_FILE" "$RELEASE_FLAG" "$HOLDDOWN_FILE"
}

cmd_render_keepalived() {
    need_root
    conf_load && [ "$HA_STATE" = enabled ] || ha_die "HA is not enabled"
    render_dirs
    render_keepalived || ha_die "cannot write $KA_CONF"
    # Fail safe: whatever happened before, a starting keepalived is not active
    ports_set down
    state_write INIT 0 "keepalived starting"
    rm -f "$PEER_FILE"
    # unicast_src_ip must exist before VRRP starts; networkd sets it
    local i=0
    while [ -z "$R" ] && [ $i -lt 20 ]; do
        ip -4 -o addr show dev "$HA_IFACE" 2>/dev/null | grep -q " $HA_LOCAL_IP/" && break
        sleep 1
        i=$((i + 1))
    done
    return 0
}

cmd_render_sshd() {
    need_root
    conf_load || ha_die "HA is not configured"
    render_dirs
    fix_ha_account
    # Privilege-separation directory, shared with the main sshd
    [ -n "$R" ] || install -d -m 0755 /run/sshd
    render_sshd || ha_die "cannot write $SSHD_CONF"
}

cmd_net_policy() {
    # /etc/systemd/network changed (menu, restore, migrate, sync)
    need_root
    if ! conf_load; then
        cleanup_runtime
        networkd_reload
        return 0
    fi
    render_network
    networkd_reload
    if controlled; then
        state_read
        if [ "$S_FORWARDING" != 1 ]; then
            ports_set down
        fi
    fi
}

# --------------------------------------------------
# Health check (keepalived track_script, every 2 s, as root)
# --------------------------------------------------
peer_refresh_async() {
    # Ask the peer in the background, so a dead peer never slows the check
    local t=0
    if [ -f "$PEER_FILE" ]; then
        t=$(sed -n 's/^TIME=//p' "$PEER_FILE" | head -n1)
        case "$t" in ''|*[!0-9]*) t=0 ;; esac
    fi
    [ $(( $(now) - t )) -ge "$PEER_REFRESH" ] || return 0
    if [ -n "$R" ]; then
        return 0
    fi
    setsid -f "$SELF" peer-refresh </dev/null >/dev/null 2>&1 8>&-
}

cmd_peer_refresh() {
    conf_load || return 1
    exec 9>"$RUN/peer.lock"
    flock -n 9 || return 0
    peer_query
}

cmd_check() {
    local reason="" p last
    conf_load || exit 1
    [ "$HA_STATE" = enabled ] || { check_result_set "FAIL HA not enabled"; exit 1; }
    install -d -m 0755 "$RUN"
    # A state change is in progress: repeat the last result instead of
    # judging a half-changed node.
    exec 8>"$RUN/transition.lock"
    if ! flock -n 8; then
        last=$(cat "$CHECK_FILE" 2>/dev/null)
        [ "$last" = OK ] && exit 0
        exit 1
    fi

    fw_ok || { fw_apply; ha_log "firewall: ND_HA chain restored"; }
    state_read

    if [ -e "$HA_MAINT" ]; then
        reason="maintenance mode"
    elif [ -e "$RELEASE_FLAG" ]; then
        reason="switch-over to peer"
    elif holddown_active; then
        reason="hold-down after link loss"
    elif [ ! -e "$NSM_CONFIGURED" ]; then
        reason="NSM not configured"
    elif [ -e "$NSM_OPERATOR_STOP" ]; then
        reason="NSM stopped by operator"
    elif [ -z "$(capture_ports)" ]; then
        reason="no capture ports"
    elif [ -z "$R" ] && ! unit_running suricata; then
        reason="IPS engine not running"
    fi

    # Link-state propagation: the active node yields if it loses a port
    if [ -z "$reason" ] && [ "$S_FORWARDING" = 1 ] &&
       [ $(( $(now) - S_SINCE )) -gt "$CARRIER_GRACE" ]; then
        for p in $(capture_ports); do
            if ! link_has_carrier "$p"; then
                reason="no link on $p"
                echo $(( $(now) + HOLDDOWN_SECS )) > "$HOLDDOWN_FILE"
                break
            fi
        done
    fi

    # A node that is not forwarding never has a capture port up
    if [ "$S_FORWARDING" != 1 ] && p=$(ports_any_up); then
        ports_set down
        ha_log "ENFORCE: capture port $p was up on a node that is not active - set down"
    fi

    # Split-brain guard
    if [ -z "$reason" ]; then
        case "$S_VRRP" in
            MASTER|FAULT) peer_refresh_async ;;
            *) [ "$S_FORWARDING" = 1 ] && peer_refresh_async ;;
        esac
        if peer_cached; then
            if [ "$S_FORWARDING" = 1 ] && [ "$P_FORWARDING" = 1 ]; then
                i_yield && reason="split brain: peer is also forwarding"
            elif [ "$S_FORWARDING" != 1 ] && [ "$P_FORWARDING" = 1 ] &&
                 { [ "$S_VRRP" = MASTER ] || [ "$S_VRRP" = FAULT ]; }; then
                # (BACKUP: VRRP adverts arrive, so the HA link works)
                if [ "$P_VIA" = mgmt ]; then
                    reason="split-brain guard: HA link down, peer active"
                elif [ "$S_VRRP" = MASTER ]; then
                    reason="peer active"
                fi
            fi
        fi
    fi

    if [ -n "$reason" ]; then
        check_result_set "FAIL $reason"
        exit 1
    fi
    check_result_set "OK"
    # MASTER that held back (peer was forwarding) and may now forward
    if [ "$S_VRRP" = MASTER ] && [ "$S_FORWARDING" != 1 ] && [ -z "$R" ]; then
        setsid -f "$SELF" reactivate </dev/null >/dev/null 2>&1 8>&-
    fi
    exit 0
}

cmd_reactivate() {
    need_root
    conf_load && [ "$HA_STATE" = enabled ] || return 0
    transition_lock 15
    state_read
    # Only if keepalived still says MASTER (the FIFO handler records every change)
    [ "$S_VRRP" = MASTER ] && [ "$S_FORWARDING" != 1 ] || return 0
    ha_log "MASTER re-evaluates after holding back"
    become_active
}

# --------------------------------------------------
# State changes (keepalived notify FIFO)
# --------------------------------------------------
become_active() {
    # Guard: never raise ports while the peer forwards
    peer_query
    if peer_cached && [ "$P_FORWARDING" = 1 ]; then
        state_write MASTER 0 "held: peer is forwarding"
        ha_log "MASTER but peer is forwarding (via $P_VIA) - capture ports stay down"
        return 0
    fi
    # State first: the health check never sees raised ports on a node that
    # is not recorded as forwarding.
    state_write MASTER 1 "active"
    if ! ports_set up; then
        ha_log "some capture ports could not be raised"
    fi
    ha_log "became ACTIVE: capture ports up"
    zeek_start
    if [ -z "$R" ] && [ -n "$(ips_pairs)" ]; then
        systemd-run --no-block --collect --quiet --unit="nd-ha-verify-$(now)" \
            "$SELF" verify-forwarding >/dev/null 2>&1
    fi
}

become_inactive() {
    # $1 = VRRP state. Ports first, then everything else.
    local why
    ports_set down
    why=$(sed 's/^FAIL //' "$CHECK_FILE" 2>/dev/null)
    case "$1" in
        BACKUP) why="standby" ;;
        STOP)   why="stopped" ;;
    esac
    state_write "$1" 0 "${why:-$1}"
    ha_log "became $1: capture ports down${why:+ ($why)}"
    zeek_stop
}

notify_state() {
    conf_load || return 0
    controlled || return 0
    case "$1" in
        MASTER)
            if [ "$HA_STATE" = enabled ]; then
                become_active
            else
                become_inactive FAULT
            fi
            ;;
        BACKUP|FAULT|STOP)
            become_inactive "$1"
            ;;
    esac
}

cmd_notify_fifo() {
    # keepalived starts this once and passes the FIFO as the last argument.
    # Lines: INSTANCE "NDOS_HA" MASTER 150
    local fifo="${!#}" line typ name st
    [ -p "$fifo" ] || ha_die "not a FIFO: $fifo"
    while :; do
        while IFS= read -r line; do
            read -r typ name st _ <<< "$line"
            [ "$typ" = INSTANCE ] || continue
            name="${name//\"/}"
            [ "$name" = "$HA_INSTANCE" ] || continue
            ( transition_lock 15; notify_state "$st" )
        done < "$fifo"
        [ -p "$fifo" ] || exit 0
        sleep 0.2
    done
}

cmd_ports_down() {
    # keepalived stopped or crashed (ExecStopPost): fail closed.
    need_root
    conf_load || return 0
    controlled || return 0
    install -d -m 0755 "$RUN"
    transition_lock 15
    ports_set down
    state_write STOP 0 "${1:-stopped}"
    zeek_stop
    ha_log "capture ports down (${1:-stopped})"
}

cmd_verify_forwarding() {
    # One-shot after promotion (IPS): Suricata must reattach to the raised
    # ports. A port that receives traffic while its partner transmits
    # nothing means the engine is not forwarding: restart it once.
    local pairs a b ra0 tb0 rb0 ta0 ra1 tb1 rb1 ta1 bad=""
    sleep 20
    conf_load || return 0
    state_read
    [ "$S_FORWARDING" = 1 ] || return 0
    pairs=$(ips_pairs)
    [ -n "$pairs" ] || return 0
    rd() { cat "$SYS_NET/$1/statistics/$2" 2>/dev/null || echo 0; }
    declare -A s0
    while read -r a b; do
        s0["$a.rx"]=$(rd "$a" rx_packets); s0["$b.tx"]=$(rd "$b" tx_packets)
        s0["$b.rx"]=$(rd "$b" rx_packets); s0["$a.tx"]=$(rd "$a" tx_packets)
    done <<< "$pairs"
    sleep 10
    while read -r a b; do
        ra0=${s0["$a.rx"]}; tb0=${s0["$b.tx"]}; rb0=${s0["$b.rx"]}; ta0=${s0["$a.tx"]}
        ra1=$(rd "$a" rx_packets); tb1=$(rd "$b" tx_packets)
        rb1=$(rd "$b" rx_packets); ta1=$(rd "$a" tx_packets)
        if { [ $((ra1 - ra0)) -gt 100 ] && [ $((tb1 - tb0)) -eq 0 ]; } ||
           { [ $((rb1 - rb0)) -gt 100 ] && [ $((ta1 - ta0)) -eq 0 ]; }; then
            bad="$bad $a/$b"
        fi
    done <<< "$pairs"
    if [ -n "$bad" ]; then
        ha_log "IPS engine not forwarding on$bad after takeover - restarting it"
        systemctl restart suricata.service
    fi
}

# --------------------------------------------------
# Gates used by systemd units
# --------------------------------------------------
cmd_allow_start() {
    # ExecCondition for zeek.service: 0 = start, 1 = skip
    [ "${1:-}" = zeek ] || exit 0
    conf_load || exit 0
    controlled || exit 0
    state_read
    [ "$S_FORWARDING" = 1 ] && exit 0
    exit 1
}

cmd_unit_allowed() {
    conf_load || exit 1
    case "${1:-}:$HA_STATE" in
        keepalived:enabled) exit 0 ;;
        sshd:pending|sshd:enabled) exit 0 ;;
        sync:enabled) exit 0 ;;
        apply:pending|apply:enabled) exit 0 ;;
        netwatch:pending|netwatch:enabled|netwatch:detached) exit 0 ;;
    esac
    exit 1
}

cmd_is_enabled() {
    conf_load && [ "$HA_STATE" = enabled ]
}

cmd_is_passive() {
    # Menu guard: cluster member that is not forwarding
    conf_load || return 1
    case "$HA_STATE" in pending|enabled) ;; *) return 1 ;; esac
    state_read
    [ "$S_FORWARDING" != 1 ]
}

# --------------------------------------------------
# Status
# --------------------------------------------------
role_text() {
    case "$HA_STATE" in
        created)  echo "PAIRING (node 1, waiting for node 2's token)"; return ;;
        pending)  echo "PAIRING (node 2, waiting for node 1)"; return ;;
        detached) echo "DETACHED (capture ports held down)"; return ;;
    esac
    state_read
    if [ -e "$HA_MAINT" ]; then
        echo "MAINTENANCE"
    elif [ "$S_FORWARDING" = 1 ]; then
        echo "ACTIVE"
    else
        case "$S_VRRP" in
            BACKUP) echo "STANDBY" ;;
            MASTER) echo "HELD (${S_REASON:-peer forwarding})" ;;
            FAULT)  echo "FAULT (${S_REASON:-unknown})" ;;
            INIT)   echo "STARTING" ;;
            *)      echo "STOPPED" ;;
        esac
    fi
}

peer_text() {
    if peer_cached; then
        if [ "$P_FORWARDING" = 1 ]; then echo "peer ACTIVE"
        else
            case "$P_VRRP" in
                BACKUP) echo "peer STANDBY" ;;
                *) echo "peer $P_VRRP" ;;
            esac
        fi
    else
        state_read
        if [ "$S_VRRP" = BACKUP ]; then echo "peer ACTIVE (VRRP)"; else echo "peer unknown"; fi
    fi
}

sync_text() {
    local l
    if l=$(cat "$HA_LAST_PUSH" 2>/dev/null) && [ -n "$l" ]; then
        printf 'last sync %s\n' "$l"
    elif l=$(sed -n 's/^TIME=//p' "$HA_APPLIED" 2>/dev/null) && [ -n "$l" ]; then
        printf 'config received %s\n' "$l"
    else
        echo "no sync yet"
    fi
}

cmd_status() {
    if ! conf_load; then
        [ "${1:-}" = --line ] && return 0
        echo "High availability is not configured."
        return 0
    fi
    if [ "${1:-}" = --line ]; then
        printf '%s, node %s' "$(role_text)" "$HA_NODE"
        [ "$HA_STATE" = enabled ] && printf ', %s, %s' "$(peer_text)" "$(sync_text)"
        echo
        return 0
    fi
    [ "$HA_STATE" = enabled ] && peer_query >/dev/null 2>&1
    state_read
    echo "High availability"
    echo "  This node      : node $HA_NODE - $(role_text)"
    echo "  Cluster        : $HA_CLUSTER_ID (VRID $HA_VRID)"
    echo "  HA link        : $HA_IFACE $HA_LOCAL_IP -> peer $HA_PEER_IP"
    echo "  Management     : $HA_MGMT_IFACE, peer ${HA_PEER_MGMT_IP:-?}, VIP ${HA_VIP:-none}"
    echo "  Capture ports  : $(for p in $(capture_ports); do
                                 if link_is_up "$p"; then s=up; else s=down; fi
                                 printf '%s(%s) ' "$p" "$s"; done)"
    echo "  VRRP state     : $S_VRRP, forwarding=$S_FORWARDING${S_REASON:+ ($S_REASON)}"
    echo "  Health check   : $(cat "$CHECK_FILE" 2>/dev/null || echo n/a)"
    if [ "$HA_STATE" = enabled ]; then
        if peer_cached; then
            echo "  Peer           : VRRP $P_VRRP, forwarding=$P_FORWARDING, version ${P_VERSION:-?} (via $P_VIA)"
            [ -n "$P_VERSION" ] && [ "$P_VERSION" != "$(os_version)" ] &&
                echo "                   VERSION MISMATCH - configuration sync is paused"
        else
            echo "  Peer           : not reachable on the HA link or management network"
        fi
    fi
    echo "  Config sync    : $(sync_text)"
    [ -e "$HA_MAINT" ] && echo "  Maintenance    : ON (this node stays standby)"
    echo "  Events         : $HA_EVENTS"
    return 0
}

# --------------------------------------------------
# Operations
# --------------------------------------------------
cmd_release() {
    # Switch over: make the peer active, this node standby.
    need_root
    conf_load && [ "$HA_STATE" = enabled ] || ha_die "HA is not enabled"
    state_read
    [ "$S_FORWARDING" = 1 ] || ha_die "this node is not the active node"
    peer_query
    if ! peer_cached || [ "$P_VRRP" != BACKUP ]; then
        ha_die "peer is not ready to take over (state: ${P_VRRP:-unreachable})"
    fi
    [ -z "$P_VERSION" ] || [ "$P_VERSION" = "$(os_version)" ] ||
        echo "Note: peer runs $P_VERSION, this node $(os_version)."
    : > "$RELEASE_FLAG"
    ha_log "switch-over requested"
    echo "Switching over to the peer..."
    local i=0
    while [ $i -lt 25 ]; do
        sleep 1
        i=$((i + 1))
        peer_query >/dev/null 2>&1
        if peer_cached && [ "$P_FORWARDING" = 1 ]; then
            rm -f "$RELEASE_FLAG"
            ha_log "switch-over complete: peer is active"
            echo "Done: the peer is active, this node is standby."
            return 0
        fi
    done
    rm -f "$RELEASE_FLAG"
    ha_log "switch-over FAILED: peer did not take over"
    echo "The peer did not take over within 25 s; this node resumes."
    return 1
}

cmd_maintenance() {
    need_root
    conf_load || ha_die "HA is not configured"
    case "${1:-}" in
        on)  : > "$HA_MAINT"; ha_log "maintenance ON"; echo "Maintenance on: this node stays standby." ;;
        off) rm -f "$HA_MAINT"; ha_log "maintenance OFF"; echo "Maintenance off." ;;
        *)   ha_die "usage: nd-ha maintenance on|off" ;;
    esac
}

cmd_after_change() {
    # Called by the menu after options 2, 10, 11 and 13.
    conf_load || return 0
    [ -n "$R" ] && return 0
    fw_ok || fw_apply
    [ "$HA_STATE" = enabled ] || return 0
    if [ "${1:-}" = net ] && unit_running nd-ha-keepalived; then
        # networkd may have dropped the VIP; keepalived re-adds it on reload
        systemctl reload nd-ha-keepalived >/dev/null 2>&1
    fi
    state_read
    [ "$S_FORWARDING" = 1 ] && systemctl start --no-block nd-ha-sync.service >/dev/null 2>&1
    return 0
}

# --------------------------------------------------
# Pairing
# --------------------------------------------------
rand_hex() {
    od -An -tx1 -N"$1" /dev/urandom | tr -d ' \n'
}

rand_alnum() {
    local s=""
    while [ "${#s}" -lt "$1" ]; do
        s="$s$(head -c 64 /dev/urandom | tr -dc 'A-Za-z0-9')"
    done
    printf '%s' "${s:0:$1}"
}

ensure_hostkey() {
    [ -f "$HOSTKEY.pub" ] && return 0
    ssh-keygen -q -t ed25519 -N '' -f "$HOSTKEY" >/dev/null 2>&1
}

new_sync_key() {
    rm -f "$HA_KEY" "$HA_KEY.pub"
    ssh-keygen -q -t ed25519 -N '' -C "nd-ha-node$HA_NODE" -f "$HA_KEY" >/dev/null 2>&1 ||
        ha_die "cannot create the sync key"
    chmod 0600 "$HA_KEY"
}

pubkey_of() {
    # "ssh-ed25519 AAAA..." without comment
    awk '{ print $1, $2; exit }' "$1" 2>/dev/null
}

token_make() {
    # stdin: KEY=VALUE lines. $1 = prefix
    printf '%s.%s\n' "$1" "$(base64 -w0)"
}

token_parse() {
    # $1 token, $2 expected prefix. Sets T_* variables.
    local tok raw line key val
    T_CLUSTER="" T_VRID="" T_AUTH="" T_VIP="" T_MGMT="" T_KEY="" T_HOST=""
    tok=$(printf '%s' "$1" | tr -d '[:space:]')
    case "$tok" in "$2".*) ;; *) echo "This is not a $2 token."; return 1 ;; esac
    [ "${#tok}" -le 2048 ] || { echo "Token too long."; return 1; }
    raw=$(printf '%s' "${tok#"$2".}" | base64 -d 2>/dev/null) || { echo "Token is damaged."; return 1; }
    while IFS= read -r line; do
        key="${line%%=*}" val="${line#*=}"
        case "$key" in
            CLUSTER) is_hex "$val" 16 && T_CLUSTER="$val" ;;
            VRID)    case "$val" in ''|*[!0-9]*|0*) ;; *) [ "$val" -le 255 ] && T_VRID="$val" ;; esac ;;
            AUTH)    printf '%s' "$val" | grep -Eq '^[A-Za-z0-9]{8}$' && T_AUTH="$val" ;;
            VIP)     { [ -z "$val" ] || is_cidr4 "$val"; } && T_VIP="$val" ;;
            MGMT)    is_ipv4 "$val" && T_MGMT="$val" ;;
            KEY)     is_pubkey "$val" && T_KEY="$val" ;;
            HOST)    is_pubkey "$val" && T_HOST="$val" ;;
        esac
    done <<< "$raw"
    [ -n "$T_CLUSTER" ] && [ -n "$T_MGMT" ] && [ -n "$T_KEY" ] && [ -n "$T_HOST" ] ||
        { echo "Token is incomplete."; return 1; }
    if [ "$2" = NDHA1 ]; then
        [ -n "$T_VRID" ] && [ -n "$T_AUTH" ] || { echo "Token is incomplete."; return 1; }
    fi
    return 0
}

trust_peer() {
    # $1 peer pubkey, $2 peer host key, $3 peer mgmt IP
    local peer_ha="$HA_NET_PREFIX.$HA_PEER_NODE"
    printf 'from="%s,%s",restrict %s nd-ha-node%s\n' "$peer_ha" "$3" "$1" "$HA_PEER_NODE" \
        > "$HA_AUTHKEYS.tmp" && chmod 0644 "$HA_AUTHKEYS.tmp" && mv -f "$HA_AUTHKEYS.tmp" "$HA_AUTHKEYS"
    printf '[%s]:%s,[%s]:%s %s\n' "$peer_ha" "$HA_SSH_PORT" "$3" "$HA_SSH_PORT" "$2" \
        > "$HA_KNOWN.tmp" && chmod 0644 "$HA_KNOWN.tmp" && mv -f "$HA_KNOWN.tmp" "$HA_KNOWN"
}

load_input_helpers() {
    local f="${ND_INPUT:-/etc/nd-input.sh}"
    # shellcheck source=/dev/null
    . "$f" 2>/dev/null || ha_die "cannot load $f"
}

ask() {
    # ask VAR "Prompt" [default]
    local _a
    if [ -n "${3:-}" ]; then printf '%s [%s]: ' "$2" "$3"; else printf '%s: ' "$2"; fi
    nd_read _a
    _a=$(nd_trim "$_a")
    [ -n "$_a" ] || _a="${3:-}"
    printf -v "$1" '%s' "$_a"
}

confirm() {
    local _y
    printf '%s [y/N]: ' "$1"
    nd_read _y
    case "$_y" in y|Y|yes|YES) return 0 ;; esac
    return 1
}

list_ifaces() {
    local i
    for i in "$SYS_NET"/*; do
        i=$(basename "$i")
        case "$i" in lo) continue ;; esac
        [ -e "$SYS_NET/$i/device" ] || [ -n "$R" ] || continue
        printf '%s\n' "$i"
    done
}

pick_ifaces() {
    # Asks for the HA and management interfaces. Sets HA_IFACE HA_MGMT_IFACE.
    local caps all def_mgmt i
    caps=$(capture_ports | tr '\n' ' ')
    all=$(list_ifaces | tr '\n' ' ')
    def_mgmt=$(ip -4 route show default 2>/dev/null |
        awk '{ for (i = 1; i < NF; i++) if ($i == "dev") { print $(i + 1); exit } }')
    echo
    echo "Interfaces      : $all"
    echo "Capture ports   : ${caps:-none (configure NSM, option 11, first)}"
    while :; do
        ask HA_MGMT_IFACE "Management interface" "$def_mgmt"
        is_ifname "$HA_MGMT_IFACE" && [ -e "$SYS_NET/$HA_MGMT_IFACE" ] && break
        echo "No such interface."
    done
    while :; do
        ask HA_IFACE "Dedicated HA interface (direct cable to the other node)"
        if ! is_ifname "$HA_IFACE" || [ ! -e "$SYS_NET/$HA_IFACE" ]; then
            echo "No such interface."; continue
        fi
        [ "$HA_IFACE" != "$HA_MGMT_IFACE" ] || { echo "Must not be the management interface."; continue; }
        case " $caps " in *" $HA_IFACE "*) echo "Must not be a capture port."; continue ;; esac
        break
    done
    case " $caps " in
        *" $HA_MGMT_IFACE "*) ha_die "the management interface is a capture port - HA needs separate ports" ;;
    esac
    for i in $caps; do
        [ -e "$NET_DIR/$i.network" ] && echo "Note: $NET_DIR/$i.network exists; the capture-port file takes precedence."
    done
    [ -e "$NET_DIR/$HA_IFACE.network" ] &&
        echo "Note: $HA_IFACE has an address from option 2; the HA link address replaces it."
    return 0
}

cmd_create() {
    need_root
    load_input_helpers
    if conf_load; then
        echo "HA is already configured on this node (state $HA_STATE). Disable it first."
        return 1
    fi
    echo "Create an HA cluster. This node becomes node 1 (preferred active node)."
    echo "Both nodes need the same NIC layout and interface names."
    [ -e "$NSM_CONFIGURED" ] || echo "Warning: NSM is not configured yet; the cluster stays inactive until it is."
    HA_NODE=1
    conf_derive
    pick_ifaces
    while :; do
        ask HA_PEER_MGMT_IP "Management IPv4 address of node 2"
        is_ipv4 "$HA_PEER_MGMT_IP" && break
        echo "Not a valid IPv4 address."
    done
    while :; do
        ask HA_VIP "Cluster management VIP (address/prefix, empty for none)"
        [ -z "$HA_VIP" ] || is_cidr4 "$HA_VIP" && break
        echo "Use the form 192.0.2.10/24."
    done
    HA_STATE=created
    HA_CLUSTER_ID=$(rand_hex 8)
    HA_VRID=$(( 0x${HA_CLUSTER_ID:0:2} % 254 + 1 ))
    HA_AUTH=$(rand_alnum 8)
    render_dirs
    conf_save || ha_die "cannot save $HA_CONF"
    new_sync_key
    ensure_hostkey || ha_die "no SSH host key"
    ha_log "cluster $HA_CLUSTER_ID created (node 1)"
    local me
    me=$(mgmt_ip "$HA_MGMT_IFACE")
    [ -n "$me" ] || ha_die "no IPv4 address on $HA_MGMT_IFACE"
    echo
    echo "On node 2 choose 'Join cluster' and paste this token (one line):"
    echo
    printf 'CLUSTER=%s\nVRID=%s\nAUTH=%s\nVIP=%s\nMGMT=%s\nKEY=%s\nHOST=%s\n' \
        "$HA_CLUSTER_ID" "$HA_VRID" "$HA_AUTH" "$HA_VIP" "$me" \
        "$(pubkey_of "$HA_KEY.pub")" "$(pubkey_of "$HOSTKEY.pub")" | token_make NDHA1
    echo
    echo "Then come back here and choose 'Finish pairing' with node 2's token."
    echo "This node keeps forwarding until then."
}

cmd_join() {
    need_root
    load_input_helpers
    local tok me
    if conf_load; then
        echo "HA is already configured on this node (state $HA_STATE). Disable it first."
        return 1
    fi
    echo "Join an HA cluster as node 2 (standby)."
    echo "Warning: this node's capture ports go DOWN now and stay down while node 1 is active."
    confirm "Continue?" || return 1
    printf 'Paste the token from node 1: '
    nd_read tok
    token_parse "$tok" NDHA1 || return 1
    HA_NODE=2
    conf_derive
    pick_ifaces
    HA_STATE=pending
    HA_CLUSTER_ID="$T_CLUSTER" HA_VRID="$T_VRID" HA_AUTH="$T_AUTH"
    HA_VIP="$T_VIP" HA_PEER_MGMT_IP="$T_MGMT"
    render_dirs
    conf_save || ha_die "cannot save $HA_CONF"
    new_sync_key
    ensure_hostkey || ha_die "no SSH host key"
    trust_peer "$T_KEY" "$T_HOST" "$T_MGMT"
    cmd_render
    if [ -z "$R" ]; then
        systemctl start nd-ha-sshd.service nd-ha-netwatch.path nd-ha-sync-apply.path >/dev/null 2>&1 ||
            echo "Warning: could not start the HA peer channel."
    fi
    ha_log "joined cluster $HA_CLUSTER_ID as node 2 (pending)"
    me=$(mgmt_ip "$HA_MGMT_IFACE")
    [ -n "$me" ] || ha_die "no IPv4 address on $HA_MGMT_IFACE"
    echo
    echo "On node 1 choose 'Finish pairing' and paste this token (one line):"
    echo
    printf 'CLUSTER=%s\nMGMT=%s\nKEY=%s\nHOST=%s\n' \
        "$HA_CLUSTER_ID" "$me" "$(pubkey_of "$HA_KEY.pub")" "$(pubkey_of "$HOSTKEY.pub")" |
        token_make NDHA2
    echo
    echo "Node 1 then sends its configuration here and this node becomes standby."
}

cmd_finish() {
    need_root
    load_input_helpers
    local tok i
    conf_load && [ "$HA_STATE" = created ] && [ "$HA_NODE" = 1 ] ||
        { echo "Finish pairing runs on node 1 after 'Create cluster'."; return 1; }
    printf 'Paste the token from node 2: '
    nd_read tok
    token_parse "$tok" NDHA2 || return 1
    [ "$T_CLUSTER" = "$HA_CLUSTER_ID" ] || { echo "That token belongs to another cluster."; return 1; }
    if [ "$T_MGMT" != "$HA_PEER_MGMT_IP" ]; then
        echo "Node 2 reports management address $T_MGMT (you entered $HA_PEER_MGMT_IP); using $T_MGMT."
        HA_PEER_MGMT_IP="$T_MGMT"
    fi
    echo
    echo "Enabling HA stops traffic through this node for a few seconds while it"
    echo "becomes the active node. Do this in a maintenance window."
    confirm "Enable HA now?" || return 1
    trust_peer "$T_KEY" "$T_HOST" "$T_MGMT"
    HA_STATE=enabled
    conf_save || ha_die "cannot save $HA_CONF"
    cmd_render
    ha_log "pairing finished, HA enabled (node 1)"
    [ -n "$R" ] && return 0
    systemctl start nd-ha-sshd.service nd-ha-netwatch.path nd-ha-sync-apply.path nd-ha-sync.timer >/dev/null 2>&1
    systemctl restart nd-ha-keepalived.service || ha_die "keepalived did not start (journalctl -u nd-ha-keepalived)"
    printf 'Waiting for this node to become active'
    i=0
    while [ $i -lt 20 ]; do
        state_read
        [ "$S_FORWARDING" = 1 ] && break
        printf '.'
        sleep 1
        i=$((i + 1))
    done
    echo
    state_read
    if [ "$S_FORWARDING" != 1 ]; then
        echo "This node is not active yet: $(cat "$CHECK_FILE" 2>/dev/null)"
        echo "Fix the cause; node 2 is activated by the first sync once this node is active"
        echo "(menu: 'Sync configuration to peer now')."
        return 1
    fi
    echo "This node is active. Sending the configuration to node 2..."
    if /usr/sbin/nd-ha-sync push --activate; then
        echo "Node 2 is configured and becomes standby."
    else
        echo "Sync to node 2 failed; retry with 'Sync configuration to peer now'."
    fi
    echo
    echo "Mid-stream session pick-up (needed so sessions survive a failover) takes"
    echo "effect when the IPS engine restarts. Restarting it now stops traffic"
    echo "through this node until it has loaded its rules (typically 30-90 s)."
    if confirm "Restart the IPS engine now?"; then
        systemctl restart suricata.service
    fi
}

cmd_disable() {
    need_root
    load_input_helpers
    conf_load || { echo "HA is not configured."; return 0; }
    case "$HA_STATE" in
        detached) cmd_release_ports; return ;;
        created)
            confirm "Discard the unfinished cluster?" || return 1
            wipe_ha
            echo "HA configuration removed."
            return 0 ;;
    esac
    local peer_alive=0
    if [ "$HA_STATE" = enabled ]; then
        peer_query >/dev/null 2>&1 && peer_cached && peer_alive=1
    fi
    echo "Disabling HA removes this node from the cluster."
    if [ $peer_alive = 1 ]; then
        echo "The peer is running; it stays (or becomes) the active node."
        echo "This node's capture ports stay DOWN until you release them, so the two"
        echo "nodes never forward on the same wire."
    else
        echo "The peer is not reachable."
    fi
    confirm "Disable HA on this node?" || return 1
    if [ -z "$R" ]; then
        systemctl stop nd-ha-keepalived.service nd-ha-sync.timer nd-ha-sync-apply.path nd-ha-sshd.service >/dev/null 2>&1
    fi
    ports_set down
    HA_STATE=detached
    conf_save || ha_die "cannot save $HA_CONF - HA state unchanged on disk"
    rm -f "$HA_AUTHKEYS" "$HA_KNOWN" "$HA_KEY" "$HA_KEY.pub" "$HA_MAINT"
    cmd_render
    state_write STOP 0 "detached"
    ha_log "HA disabled: node detached, capture ports held down"
    echo "This node has left the cluster. Capture ports are held down."
    if [ $peer_alive = 0 ] && confirm "Peer not reachable. Release the capture ports now (run standalone)?"; then
        cmd_release_ports yes
    else
        echo "Use 'Release capture ports' when this node no longer shares the wire with the other node."
    fi
}

cmd_release_ports() {
    # Detached -> standalone: capture ports back to always-up.
    need_root
    conf_load || return 0
    [ "$HA_STATE" = detached ] || { echo "Only for a detached node."; return 1; }
    if [ "${1:-}" != yes ]; then
        load_input_helpers
        echo "Only do this when the other sensor no longer forwards on the same wire."
        confirm "Release the capture ports and run standalone?" || return 1
    fi
    wipe_ha
    if [ -z "$R" ]; then
        networkctl reload >/dev/null 2>&1
        for p in $(capture_ports); do
            networkctl reconfigure "$p" >/dev/null 2>&1
        done
        systemctl start --no-block zeek.service >/dev/null 2>&1
    fi
    echo "Standalone: capture ports are up again."
    echo "Mid-stream pick-up is switched off at the next IPS engine restart."
}

wipe_ha() {
    ha_log "HA configuration removed"
    [ -f "$HA_EVENTS" ] && mv -f "$HA_EVENTS" "$R/data/nd/ha-events.last.log" 2>/dev/null
    rm -rf "$HA_DIR"
    cleanup_runtime
    [ -n "$R" ] || fw_apply
}

# --------------------------------------------------
# Console submenu (rc.initial option 17)
# --------------------------------------------------
cmd_menu() {
    need_root
    load_input_helpers
    local opt
    trap '' INT TSTP
    while :; do
        echo
        echo "========================================"
        echo " High Availability"
        echo "========================================"
        if conf_load; then echo " State: $(cmd_status --line)"; else echo " State: not configured"; fi
        echo
        echo " 1) Show status"
        echo " 2) Create cluster (this node = node 1)"
        echo " 3) Join cluster (this node = node 2)"
        echo " 4) Finish pairing (on node 1)"
        echo " 5) Switch over to peer"
        echo " 6) Maintenance mode on/off"
        echo " 7) Sync configuration to peer now"
        echo " 8) Disable HA / release capture ports"
        echo " 9) Show recent HA events"
        echo " 0) Back"
        echo
        printf 'Enter an option: '
        nd_read opt
        echo
        case "$opt" in
            1) cmd_status ;;
            2) cmd_create ;;
            3) cmd_join ;;
            4) cmd_finish ;;
            5) cmd_release ;;
            6) if [ -e "$HA_MAINT" ]; then cmd_maintenance off
               else
                   echo "Maintenance keeps this node standby (also across reboots)."
                   confirm "Turn maintenance on?" && cmd_maintenance on
               fi ;;
            7) /usr/sbin/nd-ha-sync push --force ;;
            8) cmd_disable ;;
            9) tail -n 30 "$HA_EVENTS" 2>/dev/null || echo "No events." ;;
            0|"") return 0 ;;
            *) echo "Unknown option." ;;
        esac
    done
}

# --------------------------------------------------
usage() {
    cat <<'EOF'
usage: nd-ha COMMAND
  status [--line]        show the HA state
  menu                   console submenu (rc.initial option 17)
  create | join | finish pairing steps (interactive)
  disable                leave the cluster (ports held down)
  release-ports          detached node -> standalone
  release                switch over to the peer
  maintenance on|off     keep this node standby
  render                 write runtime configuration (boot)
  net-policy             re-apply the capture-port policy
  firewall               (re)build the ND_HA firewall chain
  after-change [net]     menu hook after configuration changes
Internal: check, notify-fifo, notify, reactivate, peer-refresh, ports-down, verify-forwarding,
  render-keepalived, render-sshd, allow-start, unit-allowed, is-enabled, is-passive
EOF
}

cmd="${1:-}"
[ $# -gt 0 ] && shift
case "$cmd" in
    render)            cmd_render ;;
    render-keepalived) cmd_render_keepalived ;;
    render-sshd)       cmd_render_sshd ;;
    net-policy)        cmd_net_policy ;;
    firewall)          need_root; fw_apply ;;
    check)             cmd_check ;;
    notify-fifo)       cmd_notify_fifo "$@" ;;
    notify)            need_root; install -d -m 0755 "$RUN"; transition_lock 15; notify_state "${1:-}" ;;
    reactivate)        cmd_reactivate ;;
    peer-refresh)      cmd_peer_refresh ;;
    ports-down)        cmd_ports_down "${1:-}" ;;
    verify-forwarding) cmd_verify_forwarding ;;
    allow-start)       cmd_allow_start "${1:-}" ;;
    unit-allowed)      cmd_unit_allowed "${1:-}" ;;
    is-enabled)        cmd_is_enabled ;;
    is-passive)        cmd_is_passive ;;
    status)            cmd_status "${1:-}" ;;
    release)           cmd_release ;;
    maintenance)       cmd_maintenance "${1:-}" ;;
    after-change)      cmd_after_change "${1:-}" ;;
    create)            cmd_create ;;
    join)              cmd_join ;;
    finish)            cmd_finish ;;
    disable)           cmd_disable ;;
    release-ports)     cmd_release_ports ;;
    menu)              cmd_menu ;;
    ""|-h|--help|help) usage ;;
    *)                 usage >&2; exit 2 ;;
esac
