#!/bin/bash
#
# nd-ha-sync - NSM configuration sync from the active to the standby node.
#
#   push [--force] [--activate] [--quiet]
#        On the active node: bundle the NSM site configuration and send it
#        to the peer if it differs from what the peer applied last.
#        --activate is used once by "Finish pairing": the bundle also
#        switches the pending node 2 to enabled.
#   apply-spool
#        On the standby (root, nd-ha-sync-apply.service): validate and apply
#        bundles that the peer handed to nd-ha-remote.
#   manifest     print the local manifest hash
#   status       last push / last applied
#
# Synced: the same files the upgrader treats as NSM site configuration.
# Never synced: addresses, accounts, keys, licenses, HA settings.

# "A && B || fail" is used on purpose: fail when either test is false.
# shellcheck disable=SC2015
set -u
umask 077

LIB="${ND_HA_LIB:-/usr/libexec/nd-ha/nd-ha-common.sh}"
# shellcheck source=nd-ha-common.sh
. "$LIB" || { echo "nd-ha-sync: cannot load $LIB" >&2; exit 1; }

FORMAT=1
MAX_BYTES=134217728
LOCK="$RUN/sync.lock"
QUIET=0

# Single files (relative to /). Replaced when present in a bundle; never
# deleted because a bundle lacks them.
SYNC_FILES="
etc/suricata/suricata.yaml
etc/suricata/enable.conf
etc/suricata/disable.conf
etc/suricata/drop.conf
etc/suricata/modify.conf
etc/suricata/rules/local.rules
etc/zeek/node.cfg
etc/zeek/networks.cfg
"
# Markers: presence is mirrored. The operator-stop marker is NOT synced:
# "Stop NSM Engine" takes only that node out of service (its peer carries
# on); stopping both nodes stops the cluster.
SYNC_MARKERS="
var/lib/nsm/.nsm_configured
"
# Mirrored directory (regular files directly inside it)
RULES_DIR="var/lib/suricata/rules"
# Mirrored capture-port files: etc/systemd/network/00-nd-capture-*.network
NET_REL="etc/systemd/network"

say() {
    [ "$QUIET" = 1 ] || echo "$*"
}

safe_name() {
    # File names inside mirrored directories
    [ -n "$1" ] && [ "${#1}" -le 128 ] || return 1
    case "$1" in
        .*|*[!A-Za-z0-9._+-]*) return 1 ;;
    esac
    return 0
}

in_whitelist() {
    # $1 relative path of a regular file in a bundle
    local p="$1" n
    case "$p" in
        .nd-ha-meta) return 0 ;;
    esac
    case "
$SYNC_FILES
$SYNC_MARKERS
" in
        *"
$p
"*) return 0 ;;
    esac
    case "$p" in
        "$RULES_DIR"/*)
            n="${p#"$RULES_DIR"/}"
            safe_name "$n" ;;
        "$NET_REL/$CAPTURE_PREFIX"*.network)
            n="${p#"$NET_REL"/}"
            safe_name "$n" ;;
        *) return 1 ;;
    esac
}

dir_allowed() {
    case "$1" in
        etc|etc/suricata|etc/suricata/rules|etc/zeek|etc/systemd|"$NET_REL"|\
        var|var/lib|var/lib/nsm|var/lib/suricata|"$RULES_DIR") return 0 ;;
    esac
    return 1
}

# --------------------------------------------------
# Building a bundle (active node)
# --------------------------------------------------
collect_list() {
    # Relative paths of every file to send, one per line
    local f
    for f in $SYNC_FILES $SYNC_MARKERS; do
        [ -f "$R/$f" ] && [ ! -L "$R/$f" ] && printf '%s\n' "$f"
    done
    for f in "$R/$RULES_DIR"/*; do
        [ -f "$f" ] && [ ! -L "$f" ] && safe_name "$(basename "$f")" &&
            printf '%s\n' "$RULES_DIR/$(basename "$f")"
    done
    for f in "$R/$NET_REL/$CAPTURE_PREFIX"*.network; do
        [ -f "$f" ] && [ ! -L "$f" ] && safe_name "$(basename "$f")" &&
            printf '%s\n' "$NET_REL/$(basename "$f")"
    done
}

manifest_of() {
    # $1 tree root. Hash over "sha256  path" of every whitelisted file.
    local root="$1"
    ( cd "$root" && find . -type f ! -name .nd-ha-meta | sed 's|^\./||' | LC_ALL=C sort |
        while IFS= read -r f; do sha256sum "$f"; done ) | sha256sum | awk '{ print $1 }'
}

build_bundle() {
    # $1 work dir, $2 activate 0|1. Prints the bundle path.
    local work="$1" stage="$1/stage" f
    install -d -m 0700 "$stage"
    collect_list | while IFS= read -r f; do
        install -d -m 0755 "$stage/$(dirname "$f")"
        cp -p "$R/$f" "$stage/$f"
    done
    MANIFEST=$(manifest_of "$stage")
    {
        echo "FORMAT=$FORMAT"
        echo "CLUSTER=$HA_CLUSTER_ID"
        echo "NODE=$HA_NODE"
        echo "VERSION=$(os_version)"
        echo "MANIFEST=$MANIFEST"
        echo "ACTIVATE=$2"
        echo "CREATED=$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
    } > "$stage/.nd-ha-meta"
    tar -czf "$work/bundle.tgz" --owner=0 --group=0 --numeric-owner -C "$stage" . || return 1
    printf '%s\n' "$work/bundle.tgz"
}

cmd_push() {
    local force=0 activate=0 work rc
    while [ $# -gt 0 ]; do
        case "$1" in
            --force) force=1 ;;
            --activate) activate=1; force=1 ;;
            --quiet) QUIET=1 ;;
        esac
        shift
    done
    conf_load || { say "HA is not configured."; return 0; }
    [ "$HA_STATE" = enabled ] || { say "HA is not enabled."; return 0; }
    state_read
    if [ "$S_FORWARDING" != 1 ]; then
        say "This node is not the active node; nothing to send."
        return 0
    fi
    install -d -m 0755 "$RUN"
    exec 9>"$LOCK"
    flock -w 120 9 || { say "Another sync is running."; return 1; }

    work=$(mktemp -d "$HA_DIR/.out.XXXXXX") || return 1
    push_bundle "$work" "$force" "$activate"
    rc=$?
    rm -rf "$work"
    return $rc
}

push_bundle() {
    local work="$1" force="$2" activate="$3" bundle peer_m res rc
    bundle=$(build_bundle "$work" "$activate") || { ha_log "sync: cannot build bundle"; return 1; }
    # build_bundle ran in a subshell; recompute the manifest here
    MANIFEST=$(manifest_of "$work/stage")

    if [ "$force" = 0 ]; then
        peer_m=$(peer_ssh "$HA_PEER_IP" manifest 2>/dev/null | head -n1)
        if [ "$peer_m" = "$MANIFEST" ]; then
            say "Peer is up to date."
            return 0
        fi
    fi

    res=$(peer_ssh_long "$HA_PEER_IP" push < "$bundle" 2>&1 | tail -n1)
    rc=$?
    [ -n "$res" ] || res="ERROR no answer from peer"
    case "$res" in
        OK*) rc=0 ;;
        *) rc=1 ;;
    esac
    printf '%s %s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$res" > "$HA_LAST_PUSH.tmp" &&
        chmod 0644 "$HA_LAST_PUSH.tmp" && mv -f "$HA_LAST_PUSH.tmp" "$HA_LAST_PUSH"
    ha_log "sync push: $res"
    say "Peer: $res"
    return $rc
}

# --------------------------------------------------
# Applying a bundle (standby node, root)
# --------------------------------------------------
result() {
    # $1 request id, rest = result text
    local id="$1"; shift
    install -d -m 0755 "$RUN"
    {
        tail -n 19 "$SYNC_RESULTS" 2>/dev/null
        printf '%s %s\n' "$id" "$*"
    } > "$SYNC_RESULTS.tmp" && chmod 0644 "$SYNC_RESULTS.tmp" && mv -f "$SYNC_RESULTS.tmp" "$SYNC_RESULTS"
    ha_log "sync apply $id: $*"
}

archive_check() {
    # $1 bundle. Every member: regular file or directory, safe relative
    # path inside the whitelist. Prints the reason on failure.
    local b="$1" line name typ
    if ! tar -tzf "$b" >/dev/null 2>&1; then
        echo "not a valid archive"; return 1
    fi
    while IFS= read -r line; do
        typ="${line:0:1}"
        case "$typ" in
            -|d) ;;
            *) echo "member type '$typ' not allowed"; return 1 ;;
        esac
        case "${line:0:10}" in
            ???[sS]*|??????[sS]*) echo "setuid/setgid member"; return 1 ;;
        esac
    done < <(tar -tvzf "$b" 2>/dev/null)
    while IFS= read -r name; do
        name="${name#./}"
        name="${name%/}"
        [ -n "$name" ] && [ "$name" != . ] || continue
        case "$name" in
            /*|*..*|*\\*|*' '*) echo "unsafe path: $name"; return 1 ;;
        esac
        printf '%s' "$name" | grep -q '^[A-Za-z0-9._/+-]*$' || { echo "unsafe path: $name"; return 1; }
    done < <(tar -tzf "$b" 2>/dev/null)
    return 0
}

stage_check() {
    # $1 extracted tree. Prints the reason on failure.
    local st="$1" rel
    if [ -n "$(find "$st" ! -type f ! -type d -print -quit)" ]; then
        echo "special file in bundle"; return 1
    fi
    if [ -n "$(find "$st" -perm /6000 -print -quit)" ]; then
        echo "setuid/setgid file in bundle"; return 1
    fi
    while IFS= read -r rel; do
        rel="${rel#./}"
        [ -n "$rel" ] && [ "$rel" != . ] || continue
        dir_allowed "$rel" || { echo "directory not allowed: $rel"; return 1; }
    done < <(cd "$st" && find . -mindepth 1 -type d)
    while IFS= read -r rel; do
        rel="${rel#./}"
        in_whitelist "$rel" || { echo "file not allowed: $rel"; return 1; }
    done < <(cd "$st" && find . -type f)
    return 0
}

meta_read() {
    # $1 meta file. Sets M_* or fails.
    local line key val
    M_FORMAT="" M_CLUSTER="" M_NODE="" M_VERSION="" M_MANIFEST="" M_ACTIVATE=0
    [ -f "$1" ] || return 1
    [ "$(wc -c < "$1")" -le 4096 ] || return 1
    while IFS= read -r line; do
        key="${line%%=*}" val="${line#*=}"
        case "$key" in
            FORMAT)   case "$val" in 1) M_FORMAT="$val" ;; esac ;;
            CLUSTER)  is_hex "$val" 16 && M_CLUSTER="$val" ;;
            NODE)     case "$val" in 1|2) M_NODE="$val" ;; esac ;;
            VERSION)  printf '%s' "$val" | grep -Eq '^[A-Za-z0-9._-]{1,40}$' && M_VERSION="$val" ;;
            MANIFEST) is_hex "$val" 64 && M_MANIFEST="$val" ;;
            ACTIVATE) case "$val" in 0|1) M_ACTIVATE="$val" ;; esac ;;
        esac
    done < "$1"
    [ -n "$M_FORMAT" ] && [ -n "$M_CLUSTER" ] && [ -n "$M_NODE" ] &&
        [ -n "$M_VERSION" ] && [ -n "$M_MANIFEST" ]
}

install_file() {
    # $1 source, $2 destination. Keeps the destination's owner and mode;
    # a new file gets the parent directory's owner and 0644.
    local src="$1" dst="$2" dir tmp own mode
    dir=$(dirname "$dst")
    [ -d "$dir" ] || install -d -m 0755 "$dir" || return 1
    if [ -e "$dst" ] && [ ! -L "$dst" ]; then
        own=$(stat -c '%u:%g' "$dst") mode=$(stat -c '%a' "$dst")
    else
        own=$(stat -c '%u:%g' "$dir") mode=644
    fi
    tmp=$(mktemp "$dir/.nd-ha.XXXXXX") || return 1
    if cat "$src" > "$tmp" && chmod "$mode" "$tmp"; then
        chown "$own" "$tmp" 2>/dev/null
        mv -f "$tmp" "$dst" && return 0
    fi
    rm -f "$tmp"
    return 1
}

same_file() {
    [ -f "$1" ] && [ -f "$2" ] && cmp -s "$1" "$2"
}

apply_tree() {
    # $1 extracted tree. Sets CH_NET CH_YAML CH_RULES CH_MARKERS.
    local st="$1" f rel n
    CH_NET=0 CH_YAML=0 CH_RULES=0 CH_MARKERS=0
    for rel in $SYNC_FILES; do
        [ -f "$st/$rel" ] || continue
        same_file "$st/$rel" "$R/$rel" && continue
        install_file "$st/$rel" "$R/$rel" || return 1
        case "$rel" in
            etc/suricata/suricata.yaml) CH_YAML=1 ;;
            etc/suricata/*) CH_RULES=1 ;;
        esac
    done
    for rel in $SYNC_MARKERS; do
        if [ -f "$st/$rel" ]; then
            [ -e "$R/$rel" ] && continue
            install -d -m 0755 "$(dirname "$R/$rel")"
            : > "$R/$rel"
            CH_MARKERS=1
        elif [ -e "$R/$rel" ]; then
            rm -f "$R/$rel"
            CH_MARKERS=1
        fi
    done
    # Rules directory: mirror
    for f in "$st/$RULES_DIR"/*; do
        [ -f "$f" ] || continue
        n=$(basename "$f")
        same_file "$f" "$R/$RULES_DIR/$n" && continue
        install_file "$f" "$R/$RULES_DIR/$n" || return 1
        CH_RULES=1
    done
    # Delete only if the bundle carries a rule set at all (an active node
    # without rules never empties the standby's)
    [ -d "$st/$RULES_DIR" ] && for f in "$R/$RULES_DIR"/*; do
        [ -f "$f" ] || continue
        n=$(basename "$f")
        safe_name "$n" || continue
        [ -f "$st/$RULES_DIR/$n" ] && continue
        rm -f "$f"
        CH_RULES=1
    done
    # Capture-port files: mirror
    for f in "$st/$NET_REL/$CAPTURE_PREFIX"*.network; do
        [ -f "$f" ] || continue
        n=$(basename "$f")
        same_file "$f" "$R/$NET_REL/$n" && continue
        install_file "$f" "$R/$NET_REL/$n" || return 1
        CH_NET=1
    done
    for f in "$R/$NET_REL/$CAPTURE_PREFIX"*.network; do
        [ -f "$f" ] || continue
        n=$(basename "$f")
        [ -f "$st/$NET_REL/$n" ] && continue
        rm -f "$f"
        CH_NET=1
    done
    return 0
}

bundle_ports_ok() {
    # Capture ports named in the bundle must exist here and must not be
    # the HA or management interface.
    local st="$1" f n
    for f in "$st/$NET_REL/$CAPTURE_PREFIX"*.network; do
        [ -f "$f" ] || continue
        n=$(sed -n 's/^[[:space:]]*Name=[[:space:]]*//p' "$f" | head -n1 | tr -d '[:space:]')
        is_ifname "$n" || { echo "invalid capture port name"; return 1; }
        [ -e "$SYS_NET/$n" ] || { echo "capture port $n does not exist on this node"; return 1; }
        [ "$n" != "$HA_IFACE" ] && [ "$n" != "$HA_MGMT_IFACE" ] ||
            { echo "capture port $n is this node's HA or management interface"; return 1; }
    done
    return 0
}

post_apply() {
    [ -n "$R" ] && return 0
    if [ "$CH_NET" = 1 ]; then
        /usr/sbin/nd-ha net-policy >/dev/null 2>&1
    fi
    if [ -e "$NSM_CONFIGURED" ]; then
        # Stopped on this node by the operator: files are updated, engines
        # stay stopped
        [ -e "$NSM_OPERATOR_STOP" ] && return 0
        systemctl enable suricata.service zeek.service >/dev/null 2>&1
        if [ "$CH_YAML" = 1 ] || [ "$CH_NET" = 1 ] || ! systemctl is-active --quiet suricata.service; then
            systemctl restart suricata.service >/dev/null 2>&1 ||
                ha_log "sync apply: IPS engine failed to start"
        elif [ "$CH_RULES" = 1 ]; then
            systemctl reload suricata.service >/dev/null 2>&1
        fi
    fi
}

apply_one() {
    # $1 bundle (already moved into the root-only apply directory)
    local b="$1" id st why size
    id=$(basename "$b" .bundle); id="${id#req-}"
    size=$(wc -c < "$b")
    [ "$size" -le "$MAX_BYTES" ] || { result "$id" "REFUSED too large"; return 1; }
    if ! why=$(archive_check "$b"); then
        result "$id" "REFUSED $why"; return 1
    fi
    st=$(mktemp -d "$HA_APPLY/.st.XXXXXX") || { result "$id" "ERROR no space"; return 1; }
    apply_stage "$b" "$id" "$st"
    local rc=$?
    rm -rf "$st"
    return $rc
}

apply_stage() {
    local b="$1" id="$2" st="$3" why
    if ! tar -xzf "$b" -C "$st" --no-same-owner --no-same-permissions --no-overwrite-dir 2>/dev/null; then
        result "$id" "REFUSED cannot extract"; return 1
    fi
    if ! why=$(stage_check "$st"); then
        result "$id" "REFUSED $why"; return 1
    fi
    meta_read "$st/.nd-ha-meta" || { result "$id" "REFUSED bad metadata"; return 1; }
    [ "$M_CLUSTER" = "$HA_CLUSTER_ID" ] || { result "$id" "REFUSED other cluster"; return 1; }
    [ "$M_NODE" = "$HA_PEER_NODE" ] || { result "$id" "REFUSED not from the peer node"; return 1; }
    if [ "$M_VERSION" != "$(os_version)" ]; then
        result "$id" "REFUSED version mismatch (peer $M_VERSION, here $(os_version))"; return 1
    fi
    rm -f "$st/.nd-ha-meta"
    if [ "$(manifest_of "$st")" != "$M_MANIFEST" ]; then
        result "$id" "REFUSED manifest mismatch"; return 1
    fi
    if ! why=$(bundle_ports_ok "$st"); then
        result "$id" "REFUSED $why"; return 1
    fi
    # Never on a node that forwards: the active node is the source
    state_read
    if [ "$S_FORWARDING" = 1 ]; then
        result "$id" "REFUSED this node is active"; return 1
    fi
    if ! apply_tree "$st"; then
        result "$id" "ERROR applying files"; return 1
    fi
    {
        echo "MANIFEST=$M_MANIFEST"
        echo "TIME=$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
        echo "VERSION=$M_VERSION"
    } > "$HA_APPLIED.tmp" && chmod 0644 "$HA_APPLIED.tmp" && mv -f "$HA_APPLIED.tmp" "$HA_APPLIED"
    post_apply
    if [ "$M_ACTIVATE" = 1 ] && [ "$HA_STATE" = pending ]; then
        HA_STATE=enabled
        conf_save
        ha_log "activated by node $M_NODE: HA enabled"
        if [ -z "$R" ]; then
            systemctl start --no-block nd-ha-keepalived.service nd-ha-sync.timer >/dev/null 2>&1
        fi
    fi
    result "$id" "OK $M_MANIFEST (net=$CH_NET yaml=$CH_YAML rules=$CH_RULES markers=$CH_MARKERS)"
    return 0
}

cmd_apply_spool() {
    local f dst
    conf_load || return 0
    case "$HA_STATE" in pending|enabled) ;; *) return 0 ;; esac
    install -d -m 0700 "$HA_APPLY"
    install -d -m 0755 "$RUN"
    exec 9>"$LOCK"
    flock -w 300 9 || return 1
    for f in "$HA_SPOOL"/req-*.bundle; do
        [ -e "$f" ] || [ -L "$f" ] || continue
        dst="$HA_APPLY/$(basename "$f")"
        case "$(basename "$f")" in
            req-[0-9a-f]*.bundle) ;;
            *) rm -f "$f"; continue ;;
        esac
        # Same filesystem: a rename, never a copy through a link
        if [ -L "$f" ] || [ ! -f "$f" ]; then
            rm -f "$f"
            result "$(basename "$f" .bundle | sed 's/^req-//')" "REFUSED not a regular file"
            continue
        fi
        mv -f "$f" "$dst" || continue
        if [ -L "$dst" ] || [ ! -f "$dst" ]; then
            rm -f "$dst"; continue
        fi
        apply_one "$dst"
        rm -f "$dst"
    done
    return 0
}

cmd_status() {
    echo "Last push : $(cat "$HA_LAST_PUSH" 2>/dev/null || echo never)"
    echo "Applied   : $(tr '\n' ' ' < "$HA_APPLIED" 2>/dev/null || echo never)"
}

cmd_manifest() {
    local work
    conf_load || return 1
    work=$(mktemp -d) || return 1
    build_bundle "$work" 0 >/dev/null && manifest_of "$work/stage"
    rm -rf "$work"
}

cmd="${1:-}"
[ $# -gt 0 ] && shift
case "$cmd" in
    push)        cmd_push "$@" ;;
    apply-spool) cmd_apply_spool ;;
    status)      cmd_status ;;
    manifest)    cmd_manifest ;;
    *) echo "usage: nd-ha-sync push [--force] [--activate] [--quiet] | apply-spool | status | manifest" >&2; exit 2 ;;
esac
