记一次 Gateway RPC 超时问题的完整排查：从心跳告警到根因定位

# SSH 登录到问题节点
ssh [email protected]

# 检查进程状态
ps aux | grep openclaw-gateway | grep -v grep

# 检查 systemd 服务状态
systemctl --user status openclaw-gateway

# 查看进程启动时间
ps -eo pid,etime,cmd | grep openclaw-gateway

# 查看 Gateway 监听端口
ss -tlnp | grep 18789

# 或者
netstat -tlnp | grep 18789

# 测试本地 RPC 接口
curl -X POST http://127.0.0.1:18789/api/rpc \
  -H "Content-Type: application/json" \
  -d '{"method":"status","params":{}}'

# 从 VM151 测试调用 VM152 的 RPC
curl -X POST http://192.168.102.xxx:18789/api/rpc \
  --connect-timeout 5 \
  -H "Content-Type: application/json" \
  -d '{"method":"status","params":{}}'

# 或者使用 telnet 测试连通性
telnet 192.168.102.xxx 18789

# 测试基础连通性
ping -c 3 192.168.102.xxx

# 路由追踪
traceroute 192.168.102.xxx

# 查看网络接口状态
ip link show
ip addr show

# 检查防火墙规则
iptables -L -n --line-numbers

# 查看 CPU 使用率
top -bn1 | head -20

# 查看内存使用
free -h

# 查看磁盘 IO
iostat -x 1 3

# 查看进程列表（按 CPU/内存排序）
ps aux --sort=-%cpu | head -10
ps aux --sort=-%mem | head -10

# 查看最近的 Gateway 日志
journalctl --user-unit openclaw-gateway -n 100 --no-pager

# 或者查看 openclaw 日志文件
tail -n 200 /var/log/openclaw/openclaw.log | grep -E "(ERROR|WARN|RPC|timeout)"

# 查看 Gateway 配置
openclaw config get

# 修改 Gateway RPC 超时阈值（从 30 秒增加到 60 秒）
openclaw config set gateway.rpc.timeout 60000

# 或者使用环境变量
echo "OPENCLAW_GATEWAY_RPC_TIMEOUT=60000" >> /etc/environment

# 重启 Gateway 服务使配置生效
systemctl --user restart openclaw-gateway

# 验证配置是否生效
openclaw config get | grep timeout

# 方法一：配置 docker system prune（推荐）
# 编辑 crontab
crontab -e

# 添加每天凌晨 2 点自动清理
0 2 * * * /usr/bin/docker system prune -f --filter "until=24h"

# 方法二：使用 systemd timer 替代 cron
cat > ~/.config/systemd/user/docker-cleanup.timer << 'EOF'
[Unit]
Description=Docker cleanup timer

[Timer]
OnCalendar=daily
Persistent=true

[Install]
WantedBy=timers.target
EOF

cat > ~/.config/systemd/user/docker-cleanup.service << 'EOF'
[Unit]
Description=Docker cleanup service

[Service]
Type=oneshot
ExecStart=/usr/bin/docker system prune -f --filter "until=24h"
StandardOutput=journal
StandardError=journal
EOF

systemctl --user enable --now docker-cleanup.timer

# 增加 Gateway 并发处理能力
openclaw config set gateway.worker.maxConcurrency 10

# 增加 Gateway 内存限制
openclaw config set gateway.worker.maxMemory 512mb

# 调整心跳检测间隔
openclaw config set gateway.healthCheck.interval 30

# 调整最大重试次数
openclaw config set gateway.healthCheck.maxRetries 3

# 重启使配置生效
systemctl --user restart openclaw-gateway

#!/bin/bash

# Gateway RPC 超时问题快速排查脚本

TARGET_HOST=$1
TARGET_PORT=18789

if [ -z "$TARGET_HOST" ]; then
    echo "用法: $0 <目标主机IP>"
    exit 1
fi

echo "=== Gateway RPC 超时问题排查 ==="
echo "目标: $TARGET_HOST:$TARGET_PORT"
echo ""

echo "1. 检查 Gateway 进程..."
ssh root@$TARGET_HOST "ps aux | grep openclaw-gateway | grep -v grep"
echo ""

echo "2. 检查端口监听..."
ssh root@$TARGET_HOST "ss -tlnp | grep $TARGET_PORT"
echo ""

echo "3. 检查本地 RPC 调用..."
ssh root@$TARGET_HOST "curl -s -X POST http://127.0.0.1:$TARGET_PORT/api/rpc -H 'Content-Type: application/json' -d '{\"method\":\"status\",\"params\":{}}' | head -5"
echo ""

echo "4. 测试远程 RPC 调用..."
curl -X POST http://$TARGET_HOST:$TARGET_PORT/api/rpc \
  --connect-timeout 10 \
  -H "Content-Type: application/json" \
  -d '{"method":"status","params":{}}'
echo ""

echo "5. 检查系统资源..."
ssh root@$TARGET_HOST "free -h | grep Mem; docker system df | head -5"
echo ""

echo "6. 检查最近日志..."
ssh root@$TARGET_HOST "journalctl --user-unit openclaw-gateway -n 20 --no-pager | grep -E '(ERROR|WARN|timeout)'"
echo ""

echo "7. 检查网络连通性..."
ping -c 3 $TARGET_HOST
traceroute $TARGET_HOST 2>&1 | head -10
echo ""

echo "=== 排查完成 ==="