健康检查"清单之外"四类反常稳定——fallback 类 / 角色误判类 / 单位混淆类 / stale model 类 一键检测脚本 v6 + 探针再升级

# 1) health-check-cron.sh 加节点角色矩阵
declare -A NODE_ROLE
NODE_ROLE[VM152]="hermes"
NODE_ROLE[VM154]="hermes"
NODE_ROLE[VM151]="openclaw"
NODE_ROLE[VM153]="openclaw"
NODE_ROLE[Macmini]="openclaw"
NODE_ROLE[VPS4]="openclaw"

# 2) 根据 role 跑不同检查
for node in "${!NODE_ROLE[@]}"; do
  role="${NODE_ROLE[$node]}"
  case "$role" in
    openclaw) check_openclaw_health "$node" ;;
    hermes)   check_hermes_health "$node" ;;
  esac
done

$ curl -s http://localhost:18789/readyz | jq .
{
  "ready": true,
  "uptimeMs": 520935776,    ← 毫秒 (ms)
  "version": "0.13.0",
  "providers_count": 4
}

$ echo $((520935776 / 1000))    # 毫秒 → 秒
520935
$ echo $((520935776 / 86400000))  # 毫秒 → 天
6

# 探针自动换算单位
uptime_ms=$(curl -s http://localhost:18789/readyz | jq -r '.uptimeMs')
uptime_days=$((uptime_ms / 86400000))
uptime_hours=$((uptime_ms / 3600000))
echo "uptime: ${uptime_days}d ${uptime_hours}h (raw=${uptime_ms}ms)"

# 输出示例:
# uptime: 6d 144h (raw=520935776ms)

# 1) 找出 stale model
stale_models=$(curl -s http://localhost:18789/readyz | jq -r '.providers[] | select(.status=="stale") | .name')
echo "stale models: $stale_models"

# 2) 删除 stale model（不删 provider，只删该 provider 下的 stale model）
python3 -c "
import json
d = json.load(open('/root/.openclaw/openclaw.json'))
d['models']['providers']['openai-custom']['models'] = [
    m for m in d['models']['providers']['openai-custom']['models']
    if m['id'] != 'google_gemma-4-E4B-it-Q4_K_M.gguf'
]
json.dump(d, open('/root/.openclaw/openclaw.json', 'w'), indent=2)
"

# 1) 节点角色声明
declare -A NODE_ROLE=(
  [VM151]="openclaw"
  [VM152]="hermes"
  [VM153]="openclaw"
  [VM154]="hermes"
  [Macmini]="openclaw"
  [VPS4]="openclaw"
)

# 2) 按角色跑不同检查
check_by_role() {
  local node=$1 role=$2
  case "$role" in
    openclaw)
      check_openclaw_readyz "$node"
      check_openclaw_process "$node"
      check_openclaw_channels "$node"   # 第 13 类: 探 fallback
      check_openclaw_models "$node"     # 第 16 类: 探 stale
      check_uptime_ms "$node"           # 第 15 类: 单位换算
      ;;
    hermes)
      check_hermes_9119 "$node"
      check_hermes_platforms "$node"
      ;;
  esac
}

#!/bin/bash
# health-check-cron-v6.sh
# 覆盖 6/8-6/13 的 12 类 + 6/14 的 4 类（fallback / 角色 / 单位 / stale model）
# 用法: ./health-check-cron-v6.sh

set -uo pipefail

# ---------- 0. 节点角色矩阵（修复第 14 类） ----------
declare -A NODE_ROLE=(
  ["VM151"]="openclaw"
  ["VM152"]="hermes"
  ["VM153"]="openclaw"
  ["VM154"]="hermes"
  ["Macmini"]="openclaw"
  ["VPS4"]="openclaw"
)

declare -A NODE_HOST=(
  ["VM151"]="[email protected]"
  ["VM152"]="[email protected]"
  ["VM153"]="[email protected]"
  ["VM154"]="[email protected]"
  ["Macmini"]="[email protected]"
  ["VPS4"]="[email protected]"
)

check_openclaw_node() {
  local node=$1 host=$2
  echo "=== $node ($host) [openclaw] ==="

  # 1) readyz + uptimeMs（修复第 15 类单位混淆）
  local readyz uptime_ms uptime_days
  readyz=$(ssh "$host" 'curl -s -m 3 http://localhost:18789/readyz' 2>/dev/null)
  if [ -z "$readyz" ]; then
    echo "  ❌ readyz unreachable"
    return
  fi
  uptime_ms=$(echo "$readyz" | jq -r '.uptimeMs // 0')
  uptime_days=$((uptime_ms / 86400000))
  echo "  ✅ ready: $(echo "$readyz" | jq -r '.ready')"
  echo "  ⏱  uptime: ${uptime_days}d (raw=${uptime_ms}ms)"

  # 2) 进程数
  local proc
  proc=$(ssh "$host" 'pgrep -af openclaw-gateway | wc -l' 2>/dev/null)
  echo "  🔧 openclaw-gateway processes: $proc"
  if [ "$proc" -gt 1 ]; then
    echo "  ⚠️ 多个进程 (manual + systemd 共存?)"
  fi

  # 3) 通道状态（修复第 12 类: running vs connected, 第 13 类: fallback）
  local channels
  channels=$(ssh "$host" 'openclaw channels status 2>/dev/null' 2>/dev/null)
  echo "$channels" | jq -r '.channels // {} | to_entries[] | "  📡 \(.key): status=\(.value.status) fallback=\(.value.fallback // "none")"' 2>/dev/null

  # 探 fallback (第 13 类)
  local fallback_count
  fallback_count=$(echo "$channels" | jq -r '[.channels // {} | to_entries[] | select(.value.status == "degraded" or .value.fallback != null and .value.fallback != "none")] | length' 2>/dev/null)
  if [ "$fallback_count" -gt 0 ]; then
    echo "  ❌ 第 13 类: $fallback_count channel 处于 fallback 状态"
  fi

  # 探 connected vs running (第 12 类)
  local not_connected
  not_connected=$(echo "$channels" | jq -r '[.channels // {} | to_entries[] | select(.value.status == "running" and (.value.connected // false) == false)] | length' 2>/dev/null)
  if [ "$not_connected" -gt 0 ]; then
    echo "  ❌ 第 12 类: $not_connected channel running 但未 connected"
  fi

  # 4) Model probe（修复第 16 类: stale model）
  local models_status
  models_status=$(ssh "$host" 'python3 -c "
import json, urllib.request, urllib.error
d = json.load(open(\"/root/.openclaw/openclaw.json\"))
for k, v in d[\"models\"][\"providers\"].items():
    base = v[\"baseUrl\"].rstrip(\"/\")
    for m in v.get(\"models\", []):
        try:
            req = urllib.request.Request(base + \"/chat/completions\", method=\"POST\",
                data=json.dumps({\"model\": m[\"id\"], \"max_tokens\": 5, \"messages\": [{\"role\": \"user\", \"content\": \"hi\"}]}).encode(),
                headers={\"Content-Type\": \"application/json\", \"Authorization\": f\"Bearer {v[\\\"apiKey\\\"]}\"})
            resp = urllib.request.urlopen(req, timeout=10)
            print(f\"  🧠 {k}/{m[\\\"id\\\"]}: {resp.status}\")
        except urllib.error.HTTPError as e:
            print(f\"  ❌ 第 16 类: {k}/{m[\\\"id\\\"]}: {e.code} {e.reason}\")
        except Exception as e:
            print(f\"  ⚠️  {k}/{m[\\\"id\\\"]}: {type(e).__name__}\")
"' 2>/dev/null)
  echo "$models_status"

  # 5) systemd 状态（已知第 11 类: 状态机脱节）
  local systemd_status
  systemd_status=$(ssh "$host" 'systemctl --user is-active openclaw-gateway 2>/dev/null || echo "unknown"' 2>/dev/null)
  echo "  ⚙️  systemd: $systemd_status (若 inactive 但进程在跑 = 已知第 11 类)"

  # 6) Chrome 9222
  local chrome
  chrome=$(ssh "$host" 'pgrep -af chrome | wc -l' 2>/dev/null)
  echo "  🌐 chrome processes: $chrome"
}

check_hermes_node() {
  local node=$1 host=$2
  echo "=== $node ($host) [hermes] ==="

  # 1) Hermes 9119 状态
  local status
  status=$(ssh "$host" 'curl -s -m 3 http://localhost:9119/api/status' 2>/dev/null)
  if [ -z "$status" ]; then
    echo "  ❌ Hermes 9119 unreachable"
    return
  fi
  echo "  ✅ service: $(echo "$status" | jq -r '.service')"
  echo "  ✅ version: $(echo "$status" | jq -r '.version')"
  echo "  ✅ gateway_running: $(echo "$status" | jq -r '.gateway_running')"

  # 2) platforms 状态
  echo "$status" | jq -r '.platforms // {} | to_entries[] | "  📡 \(.key): status=\(.value.status) updated=\(.value.updated_at)"' 2>/dev/null

  # 3) uptime
  local uptime_s uptime_d
  uptime_s=$(echo "$status" | jq -r '.uptime_seconds // 0')
  uptime_d=$((uptime_s / 86400))
  echo "  ⏱  uptime: ${uptime_d}d (raw=${uptime_s}s)"
}

# ---------- 主循环 ----------
for node in VM151 VM152 VM153 VM154 Macmini VPS4; do
  host="${NODE_HOST[$node]}"
  role="${NODE_ROLE[$node]}"
  case "$role" in
    openclaw) check_openclaw_node "$node" "$host" ;;
    hermes)   check_hermes_node "$node" "$host" ;;
  esac
  echo ""
done

declare -A NODE_ROLE=(
  [VM151]="openclaw"
  [VM152]="hermes"   # ← 关键：标记为 hermes
  [VM153]="openclaw"
  [VM154]="hermes"
  [Macmini]="openclaw"
  [VPS4]="openclaw"
)

uptime_ms=520935776
uptime_days=$((uptime_ms / 86400000))   # 6
uptime_hours=$((uptime_ms / 3600000))   # 144
echo "${uptime_days}d ${uptime_hours}h"  # "6d 144h"

# 1) 找出 stale model
stale=$(openclaw channels status 2>/dev/null | jq -r '.channels // {} | to_entries[] | select(.value.status=="stale") | .key')
echo "stale channels: $stale"

# 2) 备份配置
cp /root/.openclaw/openclaw.json /root/.openclaw/openclaw.json.bak.$(date +%s)

# 3) 删除 stale model（不删整个 provider）
python3 -c "
import json
d = json.load(open('/root/.openclaw/openclaw.json'))
for k in d['models']['providers']:
    d['models']['providers'][k]['models'] = [
        m for m in d['models']['providers'][k].get('models', [])
        if m.get('status') != 'stale'
    ]
json.dump(d, open('/root/.openclaw/openclaw.json', 'w'), indent=2)
"

# 4) 重启 OpenClaw 让新配置生效
systemctl --user restart openclaw-gateway  # 或手动重启