OpenClaw 健康检查超时与恢复机制详解：从原理到实践

# 检查端口是否可达
nc -zv 192.168.102.xxx 18789
# 或
telnet 192.168.102.xxx 18789

curl -s http://192.168.102.xxx:18789/health
# 正常响应：{"status":"ok","uptime":123456}
# 异常响应：超时或错误

# 检查 RPC 端口
ss -tlnp | grep 18789

# 1. 检查进程状态
ps aux | grep openclaw-gateway

# 2. 查看进程树
pstree -p $(pgrep openclaw-gateway)

# 3. 检查线程状态
top -Hp $(pgrep openclaw-gateway)

# 4. 查看 GC 日志
grep "GC" /var/log/openclaw/gc.log

# 5. 检查内存使用
free -h

# 1. 查看重启历史
journalctl -u openclaw-gateway -n 50

# 2. 检查网络稳定性
ping -c 100 192.168.102.xxx

# 3. 查看历史告警
grep "timeout" /var/log/openclaw/app.log | tail -20

# 1. 检查 systemd 服务的启动时间
systemctl status openclaw-gateway

# 2. 检查进程的实际运行时间
ps -p $(pgrep openclaw-gateway) -o etime=

# 3. 检查进程的父进程
ps -ef | grep $(pgrep openclaw-gateway)

# 4. 手动触发健康检查
curl -s --connect-timeout 5 http://localhost:18789/health

#!/bin/bash
# /opt/scripts/health-check-and-recover.sh

GATEWAY_PORT=18789
MAX_RETRIES=3
RETRY_INTERVAL=30

check_gateway() {
    # 检查进程是否存在
    if ! pgrep -f "openclaw-gateway" > /dev/null; then
        echo "[$(date)] 进程不存在，尝试启动..."
        systemctl start openclaw-gateway
        return 1
    fi
    
    # 检查端口是否可达
    if ! nc -z localhost $GATEWAY_PORT 2>/dev/null; then
        echo "[$(date)] 端口不可达，重启服务..."
        systemctl restart openclaw-gateway
        return 1
    fi
    
    # 检查健康端点
    if ! curl -s --connect-timeout 5 http://localhost:$GATEWAY_PORT/health > /dev/null 2>&1; then
        echo "[$(date)] 健康检查失败，重启服务..."
        systemctl restart openclaw-gateway
        return 1
    fi
    
    echo "[$(date)] 健康检查通过"
    return 0
}

# 连续检查 MAX_RETRIES 次
FAILED_COUNT=0
while [ $FAILED_COUNT -lt $MAX_RETRIES ]; do
    if check_gateway; then
        FAILED_COUNT=0
    else
        FAILED_COUNT=$((FAILED_COUNT + 1))
        echo "[$(date)] 健康检查失败，当前失败次数：$FAILED_COUNT/$MAX_RETRIES"
    fi
    sleep $RETRY_INTERVAL
done

if [ $FAILED_COUNT -eq $MAX_RETRIES ]; then
    echo "[$(date)] 连续 $MAX_RETRIES 次检查失败，执行强制重启"
    systemctl restart openclaw-gateway
fi

#!/bin/bash
# health-check-diagnosis.sh

echo "=== OpenClaw 健康检查诊断脚本 ==="
echo ""

# 配置参数
GATEWAY_PORT=${1:-18789}
GATEWAY_HOST=${2:-localhost}

# 1. 检查进程状态
echo "1. 检查进程状态..."
if pgrep -f "openclaw-gateway" > /dev/null; then
    echo "   ✓ 进程存在"
    ps -p $(pgrep -f "openclaw-gateway") -o pid,etime,cmd --no-headers
else
    echo "   ✗ 进程不存在"
fi
echo ""

# 2. 检查 systemd 服务状态
echo "2. 检查 systemd 服务状态..."
systemctl is-active openclaw-gateway 2>/dev/null && echo "   ✓ 服务状态: active" || echo "   ✗ 服务状态: inactive"
echo ""

# 3. 检查端口监听
echo "3. 检查端口监听..."
if ss -tlnp | grep ":${GATEWAY_PORT} " > /dev/null; then
    echo "   ✓ 端口 ${GATEWAY_PORT} 正在监听"
    ss -tlnp | grep ":${GATEWAY_PORT} "
else
    echo "   ✗ 端口 ${GATEWAY_PORT} 未监听"
fi
echo ""

# 4. TCP 连接测试
echo "4. TCP 连接测试..."
if nc -zv $GATEWAY_HOST $GATEWAY_PORT 2>&1 | grep -q "succeeded"; then
    echo "   ✓ TCP 连接成功"
else
    echo "   ✗ TCP 连接失败"
fi
echo ""

# 5. HTTP 健康端点测试
echo "5. HTTP 健康端点测试..."
HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --connect-timeout 5 http://${GATEWAY_HOST}:${GATEWAY_PORT}/health 2>/dev/null)
if [ "$HTTP_CODE" = "200" ]; then
    echo "   ✓ 健康端点响应正常"
    curl -s http://${GATEWAY_HOST}:${GATEWAY_PORT}/health | jq .
else
    echo "   ✗ 健康端点响应异常 (HTTP $HTTP_CODE)"
fi
echo ""

# 6. 系统资源检查
echo "6. 系统资源检查..."
MEM_USAGE=$(free | awk '/Mem:/ {printf "%.1f", $3/$2 * 100}')
DISK_USAGE=$(df -h / | awk 'NR==2 {print $5}' | sed 's/%//')
echo "   内存使用率: ${MEM_USAGE}%"
echo "   磁盘使用率: ${DISK_USAGE}%"
echo ""

# 7. 日志检查
echo "7. 最近错误日志..."
journalctl -u openclaw-gateway -n 10 --no-pager 2>/dev/null | grep -i "error\|fail\|timeout" || echo "   无最近错误日志"
echo ""

# 8. 提供修复建议
echo "=== 修复建议 ==="
echo "如果进程存在但端口未监听："
echo "   kill \$(pgrep -f openclaw-gateway)"
echo "   systemctl restart openclaw-gateway"
echo ""
echo "如果服务无响应但进程存在："
echo "   查看日志：journalctl -u openclaw-gateway -n 50"
echo "   考虑手动重启：systemctl restart openclaw-gateway"
echo ""

echo "=== 诊断完成 ==="

# 告警规则示例：连续3次健康检查失败
- alert: GatewayUnhealthy
  expr: openclaw_health_check_failures > 3
  for: 1m
  labels:
    severity: critical
  annotations:
    summary: "Gateway 健康检查连续失败"

# 添加到 crontab
0 9 * * * /opt/scripts/health-check-diagnosis.sh >> /var/log/openclaw/daily-check.log 2>&1