OpenClaw 健康检查中如何正确判定"manual 进程 + systemd inactive"节点的健康状态：一份可复用的判断矩阵

$ systemctl is-active openclaw-gateway
inactive

$ ps -ef | grep openclaw | grep -v grep
root  338431  ...  /usr/local/bin/openclaw-gateway --config /etc/openclaw/gateway.yaml

$ curl -s http://127.0.0.1:18789/readyz
{"status":"ready","uptime":"29h12m"}

$ systemctl is-active openclaw-gateway
inactive  ← OpenClaw unit 不知道 Hermes 进程

$ ps -ef | grep hermes
hermes  338431  ...  /usr/local/bin/hermes-gateway

$ systemctl is-active openclaw-gateway
inactive

$ systemctl is-enabled openclaw-gateway
not-found  ← unit 文件不存在

$ ps -ef | grep openclaw
root  802705  ...  /usr/local/bin/openclaw-gateway

$ cat /etc/systemd/system/hermes.service
# 注释掉的 unit 文件
# 故意不开 systemd — 详见 Hermes 部署文档

$ systemctl is-active openclaw-gateway
inactive  ← 真的 inactive

$ ps -ef | grep openclaw
(空)  ← 没有 manual 进程

$ curl -s http://127.0.0.1:18789/readyz
curl: (7) Failed to connect to 127.0.0.1 port 18789

# ❌ 错误逻辑
NRestarts=$(systemctl show openclaw-gateway --property=NRestarts --value)
if [ "$NRestarts" -gt 100 ]; then
  echo "🔴 不健康"
  exit 1
fi

# ✅ 改进
# 1. 看 ActiveEnterTimestamp 是多久前
ActiveEnterTimestamp=$(systemctl show openclaw-gateway --property=ActiveEnterTimestamp --value)
SecondsSinceStart=$(date -d "$ActiveEnterTimestamp" +%s 2>/dev/null)

# 2. 如果进程已经稳定运行 1h+，即使 NRestarts=2000 也是健康的
if [ "$SecondsSinceStart" -gt 3600 ]; then
  echo "✅ 进程已稳定 1h+ — 历史 NRestarts 不影响当前状态"
fi

# 进程二进制名 = openclaw-gateway  → OpenClaw 栈
# 进程二进制名 = hermes-gateway     → Hermes 栈
# 进程二进制名 = node ... hermes     → Hermes 跑的 Node 进程

PROCESS_BIN=$(ps -ef | grep -E "openclaw|hermes" | grep -v grep | awk '{print $8}' | head -1)
case "$PROCESS_BIN" in
  *openclaw*)  echo "OpenClaw 栈";;
  *hermes*)    echo "Hermes 栈";;
  *node*hermes*) echo "Hermes (Node) 栈";;
esac

# manual 启动的进程 — PPID 一般不是 1
# systemd 启动的进程 — PPID 一定是 1 (PID 1 = systemd)

PPID=$(ps -ef | grep -E "openclaw|hermes" | grep -v grep | awk '{print $3}' | head -1)
if [ "$PPID" = "1" ]; then
  echo "systemd 启动"
else
  echo "manual 启动 (PPID=$PPID)"
fi

# systemd 报的 NRestarts 是 unit 重启次数
# manual 进程的 etime 是它当前实例运行了多久

# 如果 manual 进程 etime > 24h
#   → 这是"长期 manual 续命"，不是"刚被踢出来"
#   → 灰色健康，不动

# 如果 manual 进程 etime < 5min
#   → 这是"刚被踢出来又拉起"
#   → 灰色健康 + 警告（有人在反复重启）

#!/bin/bash
# diagnose-manual-systemd.sh
# 用途：诊断 "systemd inactive + manual 进程在跑" 节点的健康状态
# 原则：只读，不重启、不 kill、不动 systemd

set -e

NODE_NAME="${1:-$(hostname)}"
GATEWAY_PORT="${2:-18789}"
HERMES_PORT="${3:-9119}"

echo "============================================"
echo "  节点健康判定矩阵 — $NODE_NAME"
echo "============================================"
echo

# === 0. 节点角色判断 ===
echo "=== 0. 节点角色判断 ==="
PROCESS_LIST=$(ps -ef | grep -E "openclaw|hermes" | grep -v grep || true)
if [ -z "$PROCESS_LIST" ]; then
  echo "❌ 没有 openclaw / hermes 进程在跑"
  echo "   建议：手动启动 gateway"
  exit 1
fi

PROCESS_BIN=$(echo "$PROCESS_LIST" | awk '{print $8}' | head -1)
PROCESS_PID=$(echo "$PROCESS_LIST" | awk '{print $2}' | head -1)
PROCESS_PPID=$(echo "$PROCESS_LIST" | awk '{print $3}' | head -1)
PROCESS_ETIME=$(ps -p "$PROCESS_PID" -o etime= 2>/dev/null | xargs)

case "$PROCESS_BIN" in
  *openclaw*)
    STACK="OpenClaw"
    READYZ_URL="http://127.0.0.1:$GATEWAY_PORT/readyz"
    ;;
  *hermes*)
    STACK="Hermes"
    READYZ_URL="http://127.0.0.1:$HERMES_PORT/api/status"
    ;;
  *node*)
    STACK="Hermes (Node)"
    READYZ_URL="http://127.0.0.1:$HERMES_PORT/api/status"
    ;;
  *)
    STACK="Unknown ($PROCESS_BIN)"
    READYZ_URL=""
    ;;
esac

echo "栈:      $STACK"
echo "PID:     $PROCESS_PID"
echo "PPID:    $PROCESS_PPID (1 = systemd 启动, 其他 = manual/supervisor)"
echo "etime:   $PROCESS_ETIME"
echo "readyz:  $READYZ_URL"
echo

# === 1. D1: systemd 状态 ===
echo "=== 1. D1: systemd 状态 ==="
SYSTEMD_ACTIVE=$(systemctl is-active openclaw-gateway 2>/dev/null || echo "unknown")
SYSTEMD_ENABLED=$(systemctl is-enabled openclaw-gateway 2>/dev/null || echo "unknown")
NRESTARTS=$(systemctl show openclaw-gateway --property=NRestarts --value 2>/dev/null || echo "0")
ACTIVE_TS=$(systemctl show openclaw-gateway --property=ActiveEnterTimestamp --value 2>/dev/null || echo "n/a")

echo "is-active:    $SYSTEMD_ACTIVE"
echo "is-enabled:   $SYSTEMD_ENABLED"
echo "NRestarts:    $NRESTARTS (历史累计)"
echo "ActiveEnter:  $ACTIVE_TS"
echo

# === 2. D2: 进程存在性 ===
echo "=== 2. D2: 进程存在性 ==="
if [ -n "$PROCESS_PID" ]; then
  echo "✅ $STACK 进程在跑 (PID $PROCESS_PID, etime $PROCESS_ETIME)"
else
  echo "❌ 没有 $STACK 进程"
fi
echo

# === 3. D3: readyz 端点 ===
echo "=== 3. D3: readyz 端点 ==="
if [ -n "$READYZ_URL" ]; then
  HTTP_CODE=$(curl -sS -o /tmp/readyz_resp.json -w "%{http_code}" \
    --max-time 5 "$READYZ_URL" 2>/dev/null || echo "000")
  if [ "$HTTP_CODE" = "200" ]; then
    echo "✅ $READYZ_URL → 200"
    cat /tmp/readyz_resp.json | jq -r '.status // ."uptime // . // empty' 2>/dev/null
  elif [ "$HTTP_CODE" = "000" ]; then
    echo "❌ $READYZ_URL → 无法连接"
  else
    echo "⚠️  $READYZ_URL → $HTTP_CODE"
  fi
else
  echo "⚠️  未知栈，跳过 readyz 检查"
fi
echo

# === 4. D4: 上游连接 ===
echo "=== 4. D4: 上游连接 ==="
ESTAB_COUNT=$(ss -tan 2>/dev/null | grep -E "ESTAB.*:443" | wc -l | xargs)
echo "443 ESTAB 连接数: $ESTAB_COUNT"
if [ "$ESTAB_COUNT" -gt 0 ]; then
  echo "✅ 有上游连接（飞书/钉钉/wecom 至少一个连着）"
  ss -tan 2>/dev/null | grep -E "ESTAB.*:443" | awk '{print $5}' | head -3
else
  echo "⚠️  没有 443 ESTAB 连接 — 上游可能没连上"
fi
echo

# === 5. 综合判定 ===
echo "=== 5. 综合判定 ==="
if [ "$SYSTEMD_ACTIVE" = "active" ] && [ "$HTTP_CODE" = "200" ] && [ "$ESTAB_COUNT" -gt 0 ]; then
  echo "✅ 完美健康 (systemd active + 进程在跑 + readyz 200 + 上游连着)"
  EXIT_CODE=0
elif [ "$SYSTEMD_ACTIVE" != "active" ] && [ "$HTTP_CODE" = "200" ] && [ "$ESTAB_COUNT" -gt 0 ]; then
  echo "⚠️  灰色健康 (systemd inactive + 进程在跑 + readyz 200 + 上游连着)"
  echo "   说明: systemd 单元不健康但 gateway 自身正常"
  case "$STACK" in
    Hermes*)        echo "   原因: Hermes 栈不依赖 OpenClaw systemd 单元 — 预期状态";;
    OpenClaw*)
      if [ "$SYSTEMD_ENABLED" = "not-found" ]; then
        echo "   原因: systemd unit 文件丢失，manual 进程在续命"
        echo "   建议: 重写 systemd unit 让它接管 (但不强制)"
      else
        echo "   原因: systemd unit 存在但 inactive，进程是 manual 启动"
        echo "   建议: 看 journalctl 找 systemd 为何没拉起"
      fi
      ;;
  esac
  EXIT_CODE=0
elif [ "$HTTP_CODE" != "200" ] || [ "$ESTAB_COUNT" -eq 0 ]; then
  echo "❌ 准故障 (gateway 自身或上游有问题)"
  echo "   建议: 看 journalctl / gateway 日志找原因"
  EXIT_CODE=1
elif [ -z "$PROCESS_PID" ]; then
  echo "❌ 真故障 (进程不在)"
  echo "   建议: 手动启动 gateway"
  EXIT_CODE=1
else
  echo "⚠️  状态未明 — 请人工排查"
  EXIT_CODE=1
fi
echo
echo "============================================"
echo "  诊断完成 (exit code: $EXIT_CODE)"
echo "============================================"

exit $EXIT_CODE

chmod +x diagnose-manual-systemd.sh

# OpenClaw 节点
./diagnose-manual-systemd.sh VM151

# Hermes 节点
./diagnose-manual-systemd.sh VM152 18789 9119

$ ps -p $PID -o etime=
1-06:23:45  ← 1 天 6 小时 23 分 45 秒（长期续命）
05:23       ← 5 分 23 秒（刚被踢出来）