# 查看健康检查服务的 systemd 日志
journalctl --user -u openclaw-health-monitor.service -n 100 --no-pager

# 过滤错误和重启相关事件
journalctl --user -u openclaw-health-monitor.service --no-pager | grep -E "restart|RPC|failed|error" | tail -30

# 查看近一周内存使用趋势（通过 /proc/meminfo）
cat /proc/meminfo | grep -E "MemTotal|MemFree|MemAvailable|Cached|SReclaimable"

# 查看 Docker 资源占用
docker system df

# 查看具体容器内存使用
docker stats --no-stream --format "table {{.Name}}\t{{.MemUsage}}\t{{.MemPerc}}"

# 查看内存消耗前 10 的进程
ps aux --sort=-%mem | head -10

# 查看 Docker 详细内存占用
docker stats --no-stream

# 查看磁盘使用（文件缓存相关）
df -h
du -sh /var/*

# 查看系统 dmesg 中是否有 OOM Killer 的记录
dmesg | grep -i "oom\|killed\|memory" | tail -20

# 查看 auth.log 中与 systemd restart 相关的事件
journalctl --system | grep -E "oom\|restart|memory" | tail -10

# 查看 openclaw-gateway 的 systemd 日志（重启相关）
journalctl --user -u openclaw-gateway.service | grep -E "Main PID|Started|Stopped|killed" | tail -20

# 查看所有镜像（包含未使用的）
docker images -a

# 删除悬挂镜像（<none>）
docker image prune -a -f

# 查看并删除指定废弃镜像
docker rmi openclaw-gateway:old-v1 openclaw-gateway:old-v2 2>/dev/null || true

# 验证清理结果
docker system df

# 查看超过 500MB 的日志文件
find /var/log -type f -size +500M -exec ls -lh {} \;

# 截断超大日志（保留最后 10000 行）
> /var/log/some-service.log

# 如果日志是被某个进程持续写入，先重启该进程
systemctl --user restart some-service

sudo cat > /etc/logrotate.d/some-service << 'EOF'
/var/log/some-service.log {
    daily              # 每天轮转一次
    rotate 7           # 保留 7 个历史归档
    compress           # 压缩历史归档
    delaycompress      # 延迟压缩（保留最近一个不压缩，方便调试）
    missingok          # 日志不存在也不报错
    notifempty         # 空日志不轮转
    create 644 root root  # 新建日志文件权限
    postrotate
        systemctl --user restart some-service > /dev/null 2>&1 || true
    endscript
}
EOF

# 手动测试配置
logrotate -d /etc/logrotate.d/some-service

# 立即执行一次轮转（可选）
logrotate -f /etc/logrotate.d/some-service

#!/bin/bash
# openclaw-health-check.sh - 增强版健康检查

set -e

GATEWAY_HOST="127.0.0.1"
GATEWAY_PORT="18789"
LOG_FILE="/var/log/openclaw-health.log"
MAX_MEMORY_PERCENT=85  # 新增：内存告警阈值

log() {
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a "$LOG_FILE"
}

# 新增：内存检查
check_memory() {
    local mem_usage=$(free | awk '/Mem:/ {printf "%.0f", $3/$2 * 100}')
    if [ "$mem_usage" -gt "$MAX_MEMORY_PERCENT" ]; then
        log "WARNING: Memory usage is ${mem_usage}% (threshold: ${MAX_MEMORY_PERCENT}%)"
        return 1
    fi
    log "Memory usage: ${mem_usage}% (OK)"
    return 0
}

# RPC 健康检查
check_gateway_rpc() {
    if curl -s --connect-timeout 5 --max-time 10 "http://${GATEWAY_HOST}:${GATEWAY_PORT}/api/status" > /dev/null 2>&1; then
        log "Gateway RPC probe: ok"
        return 0
    else
        log "Gateway RPC probe: failed"
        return 1
    fi
}

# 主流程
main() {
    log "=== Health Check Started ==="

    # 先检查内存
    if ! check_memory; then
        log "Memory check failed. Attempting to free memory..."
        sync
        echo 3 > /proc/sys/vm/drop_caches 2>/dev/null || true
        docker system prune -a -f > /dev/null 2>&1 || true
    fi

    # 再检查 Gateway RPC
    if ! check_gateway_rpc; then
        log "Gateway unhealthy. Attempting restart..."
        systemctl --user restart openclaw-gateway
        sleep 5
        if check_gateway_rpc; then
            log "Gateway restarted successfully"
        else
            log "ERROR: Gateway restart failed"
        fi
    fi

    log "=== Health Check Completed ==="
}

main

#!/bin/bash
# fix-gateway-memory.sh - Gateway 内存问题一键修复脚本

set -e

echo "========== Gateway 内存问题修复 =========="
echo "执行时间：$(date '+%Y-%m-%d %H:%M:%S')"
echo ""

# 1. 清理 Docker 废弃镜像和缓存
echo "[1/5] 清理 Docker 资源..."
docker system df
docker system prune -a -f --volumes 2>/dev/null || true
docker builder prune -a -f 2>/dev/null || true
echo "清理完成。当前 Docker 资源："
docker system df

# 2. 清理日志缓存
echo ""
echo "[2/5] 清理系统缓存..."
sync
echo 3 > /proc/sys/vm/drop_caches 2>/dev/null || true

# 3. 清理大日志文件（超过 500MB 的）
echo ""
echo "[3/5] 查找并处理超大日志文件..."
find /var/log -type f -size +500M 2>/dev/null | while read logfile; do
    echo "  发现大日志: $logfile ($(du -h "$logfile" | cut -f1))"
    # 截断日志（保留最后 10000 行）
    if [ -s "$logfile" ]; then
        tail -n 10000 "$logfile" > "${logfile}.tmp" && mv "${logfile}.tmp" "$logfile"
        echo "  已截断: $logfile"
    fi
done

# 4. 检查并重启相关服务
echo ""
echo "[4/5] 重启 Gateway 服务..."
systemctl --user restart openclaw-gateway
sleep 3
systemctl --user status openclaw-gateway.service --no-pager -l

# 5. 验证
echo ""
echo "[5/5] 验证修复结果..."
sleep 2
MEM_USAGE=$(free | awk '/Mem:/ {printf "%.0f", $3/$2 * 100}')
echo "当前内存使用率: ${MEM_USAGE}%"

RPC_STATUS=$(curl -s --connect-timeout 5 "http://127.0.0.1:18789/api/status" | grep -o '"status":"live"' || echo "failed")
echo "Gateway RPC 状态: $RPC_STATUS"

echo ""
echo "========== 修复完成 =========="
echo "建议后续操作:"
echo "  1. 配置 logrotate（见上述方案二）"
echo "  2. 优化 Prometheus 告警规则（见上述方案三）"
echo "  3. 部署增强版健康检查脚本（见上述方案四）"