排查一次诡异的健康检查假阳性：论监控系统的边界问题

# 查看 Prometheus 抓取的目标
curl http://localhost:9090/api/v1/targets | jq '.data.activeTargets[] | {job: .labels.job, endpoint: .scrapeUrl}'

# 查看当前抓取的健康检查指标
curl -G 'http://localhost:9090/api/v1/query' \
  --data-urlencode 'query=up{job="openclaw-gateway"}'

curl http://localhost:18789/health
# 返回：{"status":"UP"}

#!/bin/bash

# 测试脚本 - 模拟真实用户请求
BASE_URL="http://localhost:18789"
TEST_COUNT=100

echo "开始测试，共 $TEST_COUNT 次请求..."
echo "========================================"

success=0
timeout=0
total_time=0

for i in $(seq 1 $TEST_COUNT); do
  start_time=$(date +%s%3N)
  response=$(curl -s -w "\n%{http_code}\n%{time_total}" \
    -o /dev/null \
    --max-time 10 \
    "$BASE_URL/api/v1/status")
  
  end_time=$(date +%s%3N)
  elapsed=$((end_time - start_time))
  
  http_code=$(echo "$response" | tail -1)
  
  if [ "$http_code" = "000" ]; then
    timeout=$((timeout + 1))
    echo "[超时] 请求 #$i"
  elif [ "$http_code" = "200" ]; then
    success=$((success + 1))
    total_time=$((total_time + elapsed))
    echo "[成功] 请求 #$i - ${elapsed}ms"
  else
    echo "[失败] 请求 #$i - HTTP $http_code"
  fi
done

echo "========================================"
echo "测试完成：成功 $success 次，超时 $timeout 次"
if [ $success -gt 0 ]; then
  echo "平均响应时间：$((total_time / success))ms"
fi

# 查看当前连接池使用情况
mysql -h localhost -u root -p -e "
SELECT 
  id AS connection_id,
  user,
  db,
  command AS current_command,
  time AS duration_seconds,
  state
FROM information_schema.processlist
WHERE command != 'Sleep'
ORDER BY time DESC;
"

# 查看最大连接数配置
mysql -h localhost -u root -p -e "SHOW VARIABLES LIKE 'max_connections';"

# 开启慢查询日志
mysql -h localhost -u root -p -e "SET GLOBAL slow_query_log = 'ON';"
mysql -h localhost -u root -p -e "SET GLOBAL long_query_time = 0.5;"  # 超过500ms的查询都记录

# 查看慢查询
mysql -h localhost -u root -p -e "
SELECT 
  start_time,
  query_time,
  lock_time,
  rows_sent,
  rows_examined,
  db,
  LEFT(query, 100) AS query_preview
FROM mysql.slow_log
ORDER BY start_time DESC
LIMIT 10;
"

# 查看最近的应用错误日志
tail -100 /var/log/openclaw/error.log | grep -E "(pool|timeout|connection)"

# 典型错误日志：
# [2026-04-16T10:23:45.123Z] ERROR - Connection pool exhausted, waiting 5000ms for connection
# [2026-04-16T10:24:12.456Z] ERROR - Request timeout after 10000ms
# [2026-04-16T10:25:33.789Z] WARN - Slow query detected: 2341ms

#!/bin/bash
# 文件名：health-check-deeper.sh
# 用途：深度健康检查脚本

echo "=========================================="
echo "深度健康检查开始..."
echo "时间：$(date '+%Y-%m-%d %H:%M:%S')"
echo "=========================================="
echo ""

# 1. 测试基础连通性
echo "[1/8] 测试基础连通性..."
if curl -s --max-time 5 http://localhost:18789/health > /dev/null; then
  echo "  ✓ 基础连通性正常"
else
  echo "  ✗ 基础连通性失败"
fi

# 2. 测试真实业务端点
echo "[2/8] 测试真实业务端点..."
start=$(date +%s%3N)
response=$(curl -s --max-time 10 http://localhost:18789/api/v1/status)
elapsed=$(($(date +%s%3N) - start))
echo "  响应时间：${elapsed}ms"
if [ $elapsed -gt 1000 ]; then
  echo "  ⚠ 警告：响应时间超过1秒"
fi

# 3. 测试数据库连接
echo "[3/8] 测试数据库连接..."
if mysql -h localhost -u root -p"${MYSQL_PASSWORD}" -e "SELECT 1" > /dev/null 2>&1; then
  echo "  ✓ 数据库连接正常"
else
  echo "  ✗ 数据库连接失败"
fi

# 4. 检查数据库连接池
echo "[4/8] 检查数据库连接池..."
pool_usage=$(mysql -h localhost -u root -p"${MYSQL_PASSWORD}" -N -e "
SELECT COUNT(*) 
FROM information_schema.processlist 
WHERE command != 'Sleep';
")
echo "  当前活动连接数：$pool_usage"
if [ $pool_usage -gt 80 ]; then
  echo "  ⚠ 警告：连接数过高"
fi

# 5. 测试缓存
echo "[5/8] 测试缓存..."
if redis-cli ping > /dev/null 2>&1; then
  echo "  ✓ 缓存正常"
else
  echo "  ✗ 缓存失败"
fi

# 6. 检查磁盘空间
echo "[6/8] 检查磁盘空间..."
disk_usage=$(df / | tail -1 | awk '{print $5}' | sed 's/%//')
echo "  磁盘使用率：${disk_usage}%"
if [ $disk_usage -gt 80 ]; then
  echo "  ⚠ 警告：磁盘使用率过高"
fi

# 7. 检查内存使用
echo "[7/8] 检查内存使用..."
mem_usage=$(free | grep Mem | awk '{printf "%.0f", $3/$2 * 100}')
echo "  内存使用率：${mem_usage}%"
if [ $mem_usage -gt 85 ]; then
  echo "  ⚠ 警告：内存使用率过高"
fi

# 8. 检查进程状态
echo "[8/8] 检查关键进程..."
if pgrep -f "openclaw-gateway" > /dev/null; then
  echo "  ✓ Gateway 进程运行中"
else
  echo "  ✗ Gateway 进程未运行"
fi

echo ""
echo "=========================================="
echo "深度健康检查完成"
echo "=========================================="