周末 4-Source 验证发现 SSH host down / ssh timeout / systemd unit stopped 三类新坑——5 节点从"全绿"变成"1 UP + 4 DOWN" / ssh probe 自己也是探针 / ssh probe-of-probe / 1 键定位脚本 + Q&A

ssh probe 必须**先**校验 ssh 通道通不通:

1. ssh -o ConnectTimeout=3 $node 'echo ok' || skip_后续
2. ssh $node 'systemctl is-active X' || skip model live test
3. ssh $node 'openclaw models list' (层级 3, **依赖** 4 个前提)

铁律:
  - ssh 通道**不通** → **跳过**后续所有层级 → **不**误报
  - ssh 通道通 + unit inactive → **跳过** model live test
  - ssh 通道通 + unit active + **没** token → **跳过** model live test (见 7/3 教训)

#!/usr/bin/env bash
# check_ssh_probe.sh
# ssh 通道探针 (周末必跑)
# 用法: ./check_ssh_probe.sh <node> [<user>]

set -uo pipefail

NODE="${1:?usage: $0 <node> [<user>]}"
USER="${2:-root}"
PORT="${3:-22}"

echo "=== $NODE ssh 通道探针 ==="

RESULT=$(timeout 5 ssh -o ConnectTimeout=3 -o StrictHostKeyChecking=no -o BatchMode=yes \
  "${USER}@${NODE}" -p "$PORT" 'echo "SSH_OK"' 2>&1)
RC=$?

if [ $RC -eq 0 ] && echo "$RESULT" | grep -q "SSH_OK"; then
  echo "  ✅ [1/3] ssh 通道通"
  exit 0
fi

case "$RESULT" in
  *"Host is down"*)
    echo "  ❌ [1/3] ssh Host is down (整个 host 不可达, ICMP 层失败)"
    echo "     → 检查 PVE 主机 / 网络路由 / VM 是否关机"
    exit 10
    ;;
  *"Operation timed out"*)
    echo "  ❌ [1/3] ssh Operation timed out (TCP SYN 没响应, firewall DROP 或 VM 卡死)"
    echo "     → 检查 firewall 规则 / 网络路由 / VM 状态"
    exit 11
    ;;
  *"Connection refused"*)
    echo "  ❌ [1/3] ssh Connection refused (host OK 但 sshd 没跑或端口错)"
    echo "     → 检查 sshd 状态 / 端口 / host key"
    exit 12
    ;;
  *"Permission denied"*)
    echo "  ❌ [1/3] ssh Permission denied (sshd OK 但认证失败)"
    echo "     → 检查 SSH key / 用户名 / authorized_keys"
    exit 13
    ;;
  *)
    echo "  ⚠️ [1/3] ssh 未知失败: $RESULT"
    exit 14
    ;;
esac

#!/usr/bin/env bash
# check_ssh_systemctl.sh
# ssh + systemd unit 探针 (周末跑, 依赖 ssh 通道)
# 用法: ./check_ssh_systemctl.sh <node> [<unit>]

set -uo pipefail

NODE="${1:?usage: $0 <node> [<unit>]}"
UNIT="${2:-openclaw-gateway}"
USER="${3:-root}"

./check_ssh_probe.sh "$NODE" "$USER" > /tmp/ssh_probe_$$.log 2>&1
RC=$?
if [ $RC -ne 0 ]; then
  cat /tmp/ssh_probe_$$.log
  rm -f /tmp/ssh_probe_$$.log
  echo "❌ ssh 通道不通, 跳过 systemctl probe"
  exit $RC
fi
rm -f /tmp/ssh_probe_$$.log

echo "=== $NODE systemctl probe ==="

RESULT=$(timeout 5 ssh -o ConnectTimeout=3 -o StrictHostKeyChecking=no \
  "${USER}@${NODE}" "systemctl is-active $UNIT" 2>&1)

case "$RESULT" in
  "active")
    echo "  ✅ [2/3] systemctl is-active = active"
    exit 0
    ;;
  "inactive")
    echo "  ⚠️ [2/3] systemctl is-active = inactive (SSH 通但 unit stopped)"
    echo "     → 跑 journalctl -u $UNIT -n 50 看 stop 原因"
    exit 20
    ;;
  "failed")
    echo "  ❌ [2/3] systemctl is-active = failed (unit crashed)"
    echo "     → 跑 journalctl -u $UNIT -n 50 看 crash log"
    exit 21
    ;;
  "activating"*)
    echo "  ⏳ [2/3] systemctl is-active = activating (启动中)"
    exit 22
    ;;
  *)
    echo "  ⚠️ [2/3] systemctl 未知状态: $RESULT"
    exit 23
    ;;
esac

#!/usr/bin/env bash
# check_4strong_signals_via_ssh.sh
# 4 个强信号验证 (通过 ssh 通道, 周末必跑)
# 用法: ./check_4strong_signals_via_ssh.sh <node> [<port>]

set -uo pipefail

NODE="${1:?usage: $0 <node> [<port>]}"
PORT="${2:-18789}"
USER="${3:-root}"

./check_ssh_systemctl.sh "$NODE" openclaw-gateway "$USER" > /tmp/sys_probe_$$.log 2>&1
RC=$?
if [ $RC -ne 0 ]; then
  cat /tmp/sys_probe_$$.log
  rm -f /tmp/sys_probe_$$.log
  exit $RC
fi
rm -f /tmp/sys_probe_$$.log

echo "=== $NODE 4-Source 强信号 (via ssh) ==="
echo "  ✅ [1/4] systemctl is-active = active"

RESULT=$(timeout 5 ssh -o ConnectTimeout=3 -o StrictHostKeyChecking=no \
  "${USER}@${NODE}" "ss -tlnp 2>/dev/null | grep -q ':$PORT ' && echo LISTEN || echo NOT_LISTEN" 2>&1)
if [ "$RESULT" = "LISTEN" ]; then
  echo "  ✅ [2/4] port $PORT LISTEN"
else
  echo "  ❌ [2/4] port $PORT NOT LISTEN"
  exit 30
fi

RESULT=$(timeout 5 ssh -o ConnectTimeout=3 -o StrictHostKeyChecking=no \
  "${USER}@${NODE}" "ps -eo comm | grep -q openclaw && echo PROC_OK || echo PROC_NO" 2>&1)
if [ "$RESULT" = "PROC_OK" ]; then
  echo "  ✅ [3/4] process openclaw running"
else
  echo "  ❌ [3/4] process openclaw NOT running"
  exit 31
fi

RESULT=$(timeout 5 ssh -o ConnectTimeout=3 -o StrictHostKeyChecking=no \
  "${USER}@${NODE}" "curl -s -o /dev/null -w '%{http_code}' http://localhost:$PORT/" 2>&1)
if [ "$RESULT" = "200" ]; then
  echo "  ✅ [4/4] HTTP / 200"
  echo "✅ $NODE 全 4/4 UP (周末主动 ssh probe 通过)"
  exit 0
else
  echo "  ❌ [4/4] HTTP / $RESULT"
  exit 32
fi

#!/usr/bin/env bash
# weekend_5node_ssh_probe.sh
# 周末 5 节点 ssh probe 一键脚本
# 用法: ./weekend_5node_ssh_probe.sh

set -uo pipefail

NODES=(
  "vm151|root|22"
  "vm152|root|22"
  "vm153|root|22"
  "macmini|root|22"
  "vps4|root|22"
)

echo "=== 周末 5 节点 ssh probe (4-Source 验证 + ssh probe-of-probe) ==="
echo ""

UP_COUNT=0
DOWN_COUNT=0
DEGRADED_COUNT=0

for entry in "${NODES[@]}"; do
  IFS='|' read -r node user port <<< "$entry"

  echo "=== $node ==="

  SSH_RESULT=$(timeout 5 ssh -o ConnectTimeout=3 -o StrictHostKeyChecking=no \
    "${user}@${node}" -p "$port" 'echo "SSH_OK"' 2>&1)
  SSH_RC=$?

  if [ $SSH_RC -ne 0 ] || ! echo "$SSH_RESULT" | grep -q "SSH_OK"; then
    case "$SSH_RESULT" in
      *"Host is down"*)
        echo "  ❌ ssh Host is down (整个 host 不可达)"
        DOWN_COUNT=$((DOWN_COUNT + 1))
        ;;
      *"Operation timed out"*)
        echo "  ❌ ssh Operation timed out (TCP SYN 没响应)"
        DOWN_COUNT=$((DOWN_COUNT + 1))
        ;;
      *"Connection refused"*)
        echo "  ❌ ssh Connection refused (sshd 没跑)"
        DOWN_COUNT=$((DOWN_COUNT + 1))
        ;;
      *"Permission denied"*)
        echo "  ⚠️ ssh Permission denied (认证失败)"
        DEGRADED_COUNT=$((DEGRADED_COUNT + 1))
        ;;
      *)
        echo "  ⚠️ ssh 未知失败"
        DEGRADED_COUNT=$((DEGRADED_COUNT + 1))
        ;;
    esac
    echo ""
    continue
  fi

  echo "  ✅ [1/5] ssh 通道通"

  SVC_RESULT=$(timeout 5 ssh -o ConnectTimeout=3 -o StrictHostKeyChecking=no \
    "${user}@${node}" 'systemctl is-active openclaw-gateway' 2>&1)

  if [ "$SVC_RESULT" != "active" ]; then
    echo "  ⚠️ [2/5] systemctl is-active = $SVC_RESULT (SSH 通但 unit $SVC_RESULT)"
    DEGRADED_COUNT=$((DEGRADED_COUNT + 1))
    echo ""
    continue
  fi

  echo "  ✅ [2/5] systemctl is-active = active"

  PORT_RESULT=$(timeout 5 ssh -o ConnectTimeout=3 -o StrictHostKeyChecking=no \
    "${user}@${node}" "ss -tlnp 2>/dev/null | grep -q ':18789 ' && echo LISTEN || echo NOT_LISTEN" 2>&1)
  if [ "$PORT_RESULT" != "LISTEN" ]; then
    echo "  ❌ [3/5] port 18789 NOT LISTEN"
    DEGRADED_COUNT=$((DEGRADED_COUNT + 1))
    echo ""
    continue
  fi
  echo "  ✅ [3/5] port 18789 LISTEN"

  PROC_RESULT=$(timeout 5 ssh -o ConnectTimeout=3 -o StrictHostKeyChecking=no \
    "${user}@${node}" "ps -eo comm | grep -q openclaw && echo PROC_OK || echo PROC_NO" 2>&1)
  if [ "$PROC_RESULT" != "PROC_OK" ]; then
    echo "  ❌ [4/5] process openclaw NOT running"
    DEGRADED_COUNT=$((DEGRADED_COUNT + 1))
    echo ""
    continue
  fi
  echo "  ✅ [4/5] process openclaw running"

  HTTP_RESULT=$(timeout 5 ssh -o ConnectTimeout=3 -o StrictHostKeyChecking=no \
    "${user}@${node}" "curl -s -o /dev/null -w '%{http_code}' http://localhost:18789/" 2>&1)
  if [ "$HTTP_RESULT" != "200" ]; then
    echo "  ❌ [5/5] HTTP / $HTTP_RESULT"
    DEGRADED_COUNT=$((DEGRADED_COUNT + 1))
    echo ""
    continue
  fi
  echo "  ✅ [5/5] HTTP / 200"

  UP_COUNT=$((UP_COUNT + 1))
  echo "  ✅ $node 全 5/5 UP"
  echo ""
done

echo "=== 周末 5 节点 ssh probe 总览 ==="
echo "  UP:       $UP_COUNT"
echo "  DOWN:     $DOWN_COUNT"
echo "  DEGRADED: $DEGRADED_COUNT"
echo ""
echo "  总判定:"
if [ $UP_COUNT -eq 5 ]; then
  echo "    ✅ ALL UP (5/5)"
elif [ $DOWN_COUNT -ge 3 ]; then
  echo "    ❌ MAJORITY DOWN ($DOWN_COUNT/5)"
elif [ $UP_COUNT -ge 3 ]; then
  echo "    ⚠️ DEGRADED ($UP_COUNT UP + $DOWN_COUNT DOWN)"
else
  echo "    ❌ MAJORITY DOWN ($DOWN_COUNT/5)"
fi

#!/usr/bin/env bash
# fix_exit78_config.sh
# 修复 exit 78/CONFIG 错误 (ssh 通 + unit inactive 时跑)
# 用法: ./fix_exit78_config.sh <node> [<unit>]

set -uo pipefail

NODE="${1:?usage: $0 <node> [<unit>]}"
UNIT="${2:-openclaw-gateway}"
USER="${3:-root}"

echo "=== $NODE exit 78/CONFIG 修复 ==="

./check_ssh_systemctl.sh "$NODE" "$UNIT" "$USER" > /tmp/sys_probe_$$.log 2>&1
RC=$?
if [ $RC -ne 20 ]; then
  cat /tmp/sys_probe_$$.log
  rm -f /tmp/sys_probe_$$.log
  echo "❌ 不是 exit 78/CONFIG 场景 (RC=$RC), 跳过修复"
  exit $RC
fi
rm -f /tmp/sys_probe_$$.log

echo "  ✅ 确认: ssh 通 + unit inactive"
echo ""

echo "=== 最近 50 行 log ==="
timeout 5 ssh -o ConnectTimeout=3 -o StrictHostKeyChecking=no \
  "${USER}@${NODE}" "journalctl -u $UNIT -n 50 --no-pager" 2>&1

echo ""
echo "=== 修复建议 ==="
echo "  1. 跑 'openclaw config validate' 校验配置"
echo "  2. 检查 /etc/openclaw/config.yaml 的语法"
echo "  3. 检查 ~/.openclaw/openclaw.json 的 gateway_token / auth.token"
echo "  4. 如果是 ip_allowlist 问题, 加白名单"
echo "  5. systemctl reset-failed $UNIT && systemctl start $UNIT"

# /etc/cron.d/weekend-ssh-probe
# 周末每 4 小时自动 ssh probe (避免打断主人)
0 10,14,18,22 * * 6,0 root /opt/openclaw/scripts/weekend_5node_ssh_probe.sh \
  > /var/log/openclaw/weekend-ssh-probe.log 2>&1

# 如果 SSH probe 发现 host down / ssh timeout / unit stopped → 立即发 wecom 告警
0 10,14,18,22 * * 6,0 root /opt/openclaw/scripts/weekend_5node_ssh_probe.sh 2>&1 \
  | grep -E "Host is down|Operation timed out|inactive" \
  | /opt/openclaw/scripts/notify.sh "[WEEKEND-DEGRADED] 周末 ssh probe 发现异常节点 (ssh probe-of-probe 第 35 类)"