监控探针的"反常稳定"第 7-8 类：DB-based probe 14 次连续 identical + cron task 与节点角色不匹配

# 1) DB 文件 mtime
$ stat -c %y _archive/baidupcs_cache/baidupcs_cache.db
2026-06-07 16:53:33
# → mtime 6/7 16:53 = 52h 前
# → 14 次探测期间 mtime 完全不变 = DB 真的没在写

# 2) DB 文件 size
$ stat -c %s _archive/baidupcs_cache/baidupcs_cache.db
237928448  # 227 MiB
# → 14 次 size 完全不变
# → 不是"DB 在 0 数据上循环"——是 DB 真的没变

# 3) DB integrity check
$ sqlite3 _archive/baidupcs_cache/baidupcs_cache.db "PRAGMA integrity_check;"
ok
# → integrity = ok = DB 结构没损坏

# 4) 探针脚本的执行记录
$ tail -5 _tmp/baidupcs_cache/sync_status.json
{
  "last_checked": "2026-06-09 20:57:14",
  "live_probe_2026_06_09_2057": {
    "status": "completed",
    "files": 56816
  }
}
# → last_checked 在推进 = 探针在跑
# → 探针执行了 14 次 = 探针没失效

# 5) 探针进程
$ pgrep -f "sync_status\|sync_probe" | head -3
# → 0 hits = 探针不是 daemon，是 cron 一次性脚本
# → cron 14 次都跑了 = cron 配置正常

# 6) 上一次同步耗时
$ jq -r '.last_completion.duration_seconds' sync_wrapper_status.json
1692  # 28.2 min
# → 28.2 min 同步 14/14 top-dirs
# → 上次同步完 2 天 = 2 天里没新增

# 7) 全量扫描 vs 增量 wrapper 对比
# 5/16 全量: 275,877 文件 / 16.65 TiB
# 6/7 增量: 56,816 文件 / 4.42 TiB
# → 增量覆盖 20.6% by files / 26.5% by bytes
# → 还有 80% 文件未扫（用户行为：网盘里 2 天没新文件）

$ curl -s --max-time 3 http://192.168.102.xx:18789/readyz
# → connection refused
# → 端口 18789 在 VM152 上不存在

$ ssh 192.168.102.xx "systemctl show openclaw-gateway"
ActiveState=inactive
NRestarts=0
# → service 是 inactive
# → 日志: nohup: failed to run command '...openclaw/dist/index.js': No such file

# 错误用法
$ curl -s http://192.168.102.xx:18789/readyz
# → connection refused（18789 端口在 Hermes 上不存在）

# 正确用法
$ curl -s http://192.168.102.xx:9119/status
{"gateway_running":true,"hermes_version":"0.15.1","pid":338431,"dingtalk":"connected"}

# 错误用法
$ ssh 192.168.102.xx "systemctl show openclaw-gateway"
ActiveState=inactive
# → 看起来像 OpenClaw 挂了，实际是 Hermes 节点本来就没装 OpenClaw

# 正确用法
$ ssh 192.168.102.xx "ps -ef | grep hermes-agent | grep -v grep"
root  338431  ...  hermes-agent

# 错误用法
$ ssh 192.168.102.xx "docker ps"
# → Hermes 节点没装 docker CLI（不一定），看起来像 docker 挂了
# → 实际是 Hermes 节点本来就没用 docker

# 正确用法
$ ssh 192.168.102.xx "ps -ef | grep hermes | grep -v grep | wc -l"
2  # 2 个 hermes 进程 = by-design

# 错误用法
$ ssh 192.168.102.xx "openclaw config get"
# → command not found（Hermes 节点没装 openclaw CLI）
# → 看起来像 OpenClaw 配置丢了

# 正确用法
$ ssh 192.168.102.xx "hermes config get"
# → 返回 Hermes 配置

# 错误用法
$ ssh 192.168.102.xx "openclaw channels status"
# → command not found

# 正确用法
$ curl -s http://192.168.102.xx:9119/api/channels
{"dingtalk":"connected","wecom":"not_configured","api_server":"not_configured"}

#!/bin/bash
# silent-period-probe-v2.sh
# 用途：静默期健康检查的"主动追问"——检测 8 类反常稳定
# 覆盖：6/8 的 6 类 + 6/9 的 2 类（DB probe identical + cron role mismatch）
# 原则：只读、幂等、不触发任何业务告警
# 输出：每节点 OK / WARN(反常稳定) / FAIL

set -u

OC_NODES_FILE="${1:-./oc-nodes.txt}"
HERMES_NODES_FILE="${2:-./hermes-nodes.txt}"
DB_PROBES_FILE="${3:-./db-probes.txt}"
PROMPT="${4:-ping}"
TIMEOUT="${5:-10}"

if [ ! -f "$OC_NODES_FILE" ] || [ ! -f "$HERMES_NODES_FILE" ]; then
  echo "❌ 节点列表文件不存在"
  exit 1
fi

echo "=========================================="
echo "  静默期主动追问 v2（8 类反常稳定）"
echo "  $(date '+%Y-%m-%d %H:%M:%S')"
echo "=========================================="
echo

TOTAL=0
OK=0
WARN=0
FAIL=0
declare -a ANOMALY_NODES

# Part 1: 4 OpenClaw 节点
echo "【Part 1: 4 OpenClaw 节点】"
echo

while IFS=$'\t' read -r NODE ADDR DESC; do
  [[ "$NODE" =~ ^#.*$ ]] && continue
  [[ -z "$NODE" ]] && continue

  TOTAL=$((TOTAL+1))

  # 基础层：readyz
  READYZ=$(curl -s --max-time 3 "http://$ADDR:18789/readyz" 2>/dev/null)
  if ! echo "$READYZ" | grep -q '"ready"'; then
    echo "❌ $NODE ($ADDR)  readyz 失败"
    FAIL=$((FAIL+1))
    ANOMALY_NODES+=("$NODE (readyz)")
    continue
  fi

  PID=$(echo "$READYZ" | grep -oE '"pid":[0-9]+' | grep -oE '[0-9]+')
  UPTIME=$(echo "$READYZ" | grep -oE '"uptime":"[^"]+"' | sed 's/"uptime":"//;s/"//')

  # 追问 1.1: manual 进程 CPU 时间有没有在推进？（6/8 第 1 类）
  if [ -n "$PID" ]; then
    CPU_TIME_NOW=$(ssh "$ADDR" "cat /proc/$PID/stat 2>/dev/null | awk '{print \$14+\$15}'" 2>/dev/null)
    sleep 2
    CPU_TIME_LATER=$(ssh "$ADDR" "cat /proc/$PID/stat 2>/dev/null | awk '{print \$14+\$15}'" 2>/dev/null)

    if [ -n "$CPU_TIME_NOW" ] && [ -n "$CPU_TIME_LATER" ]; then
      if [ "$CPU_TIME_NOW" = "$CPU_TIME_LATER" ]; then
        echo "⚠️  $NODE ($ADDR)  manual/systemd 进程 $PID CPU 时间 2 秒内无变化 → 可能僵死"
        WARN=$((WARN+1))
        ANOMALY_NODES+=("$NODE (CPU time flat)")
        continue
      fi
    fi
  fi

  # 追问 1.2: NRestarts 累计算法（PID 一致性）（6/8 第 2 类）
  NRESTARTS=$(echo "$READYZ" | grep -oE '"nrestarts":[0-9]+' | grep -oE '[0-9]+')
  if [ -n "$NRESTARTS" ] && [ "$NRESTARTS" -gt 100 ]; then
    PID_UP=$(ssh "$ADDR" "ps -o etime= -p $PID 2>/dev/null" 2>/dev/null | xargs)
    if [ -n "$PID_UP" ]; then
      echo "   ↳ NRestarts=$NRESTARTS 但 PID $PID 已运行 $PID_UP → 累计算法 PASS"
    else
      echo "❌ $NODE ($ADDR)  NRestarts=$NRESTARTS 但 PID $PID 不存在 → 真 restart loop"
      FAIL=$((FAIL+1))
      ANOMALY_NODES+=("$NODE (restart loop)")
      continue
    fi
  fi

  # 追问 1.3: systemd-socket-proxyd by-design（6/8 第 3 类）
  SOCK_PROXY=$(ssh "$ADDR" "ps -ef | grep systemd-socket-proxyd | grep -v grep" 2>/dev/null)
  if [ -n "$SOCK_PROXY" ]; then
    SOCK_PID=$(echo "$SOCK_PROXY" | awk '{print $2}' | head -1)
    SOCKET_UNIT=$(ssh "$ADDR" "systemctl list-units --type=socket 2>/dev/null | grep -i openclaw" 2>/dev/null)
    if [ -z "$SOCKET_UNIT" ]; then
      echo "❌ $NODE ($ADDR)  systemd-socket-proxyd $SOCK_PID 无 .socket 单元 → 真孤儿"
      FAIL=$((FAIL+1))
      ANOMALY_NODES+=("$NODE (orphan socket)")
      continue
    fi
  fi

  # 追问 1.5: DIY-MINI 端到端 + model 身份（6/8 第 5 类）
  MODEL_OUT=$(timeout "$TIMEOUT" openclaw chat --node "$NODE" --format json "$PROMPT" 2>&1)
  MODEL_EXIT=$?

  if [ $MODEL_EXIT -eq 124 ]; then
    echo "❌ $NODE ($ADDR)  模型调用 timeout (${TIMEOUT}s)"
    FAIL=$((FAIL+1))
    ANOMALY_NODES+=("$NODE (timeout)")
  elif echo "$MODEL_OUT" | grep -q "__OPENCLAW_REDACTED__"; then
    echo "❌ $NODE ($ADDR)  token / network 失败"
    FAIL=$((FAIL+1))
    ANOMALY_NODES+=("$NODE (token/network)")
  elif [ -z "$MODEL_OUT" ]; then
    echo "❌ $NODE ($ADDR)  空返回"
    FAIL=$((FAIL+1))
    ANOMALY_NODES+=("$NODE (empty)")
  else
    MODEL_ID=$(echo "$MODEL_OUT" | grep -oE '"model":"[^"]+"' | head -1 | sed 's/"model":"//;s/"//')
    if [ -n "$MODEL_ID" ] && [ "$MODEL_ID" != "MiniMax-M3" ]; then
      echo "⚠️  $NODE ($ADDR)  模型身份 $MODEL_ID (非预期 MiniMax-M3) → 走 fallback"
      WARN=$((WARN+1))
      ANOMALY_NODES+=("$NODE (model fallback)")
    else
      echo "✅ $NODE ($ADDR)  ${DESC:-}  uptime=$UPTIME  model=$MODEL_ID"
      OK=$((OK+1))
    fi
  fi
done < "$OC_NODES_FILE"

# Part 2: 2 Hermes 节点（节点角色感知 — 6/9 第 8 类）
echo
echo "【Part 2: 2 Hermes 节点（节点角色感知）】"
echo

while IFS=$'\t' read -r NODE ADDR DESC; do
  [[ "$NODE" =~ ^#.*$ ]] && continue
  [[ -z "$NODE" ]] && continue

  TOTAL=$((TOTAL+1))

  # 8 类：节点角色感知 —— Hermes 节点不用 OpenClaw readyz 18789
  STATUS=$(curl -s --max-time 3 "http://$ADDR:9119/status" 2>/dev/null)
  if ! echo "$STATUS" | grep -q '"gateway_running":true'; then
    echo "❌ $NODE ($ADDR)  Hermes gateway not running (9119)"
    FAIL=$((FAIL+1))
    ANOMALY_NODES+=("$NODE (hermes down)")
    continue
  fi

  HERMES_VERSION=$(echo "$STATUS" | grep -oE '"hermes_version":"[^"]+"' | sed 's/"hermes_version":"//;s/"//')
  HERMES_PID=$(echo "$STATUS" | grep -oE '"pid":[0-9]+' | grep -oE '[0-9]+')
  HERMES_UPTIME=$(echo "$STATUS" | grep -oE '"uptime":"[^"]+"' | sed 's/"uptime":"//;s/"//')

  if echo "$STATUS" | grep -q '"dingtalk":"connected"'; then DT_OK=1; else DT_OK=0; fi
  if echo "$STATUS" | grep -q '"wecom":"connected"'; then WC_OK=1; else WC_OK=0; fi

  echo "✅ $NODE ($ADDR)  ${DESC:-}  v$HERMES_VERSION  uptime=$HERMES_UPTIME  dingtalk=$DT_OK wecom=$WC_OK"
  OK=$((OK+1))
done < "$HERMES_NODES_FILE"

# Part 3: DB-based probe identical 检测（6/9 第 7 类）
echo
echo "【Part 3: DB-based probe identical 检测】"
echo

if [ -f "$DB_PROBES_FILE" ]; then
  while IFS=$'\t' read -r DB_NAME DB_PATH LAST_MTIME LAST_SIZE INTEGRITY; do
    [[ "$DB_NAME" =~ ^#.*$ ]] && continue
    [[ -z "$DB_NAME" ]] && continue

    TOTAL=$((TOTAL+1))

    # 追问 1: DB mtime
    if [ -f "$DB_PATH" ]; then
      CUR_MTIME=$(stat -c %y "$DB_PATH" 2>/dev/null | cut -d. -f1)
      CUR_SIZE=$(stat -c %s "$DB_PATH" 2>/dev/null)
    else
      echo "❌ $DB_NAME ($DB_PATH)  DB 文件不存在"
      FAIL=$((FAIL+1))
      ANOMALY_NODES+=("$DB_NAME (db missing)")
      continue
    fi

    # 追问 2: DB integrity
    if command -v sqlite3 >/dev/null 2>&1; then
      INTEGRITY_CUR=$(sqlite3 "$DB_PATH" "PRAGMA integrity_check;" 2>/dev/null)
    else
      INTEGRITY_CUR="unknown"
    fi

    if [ "$INTEGRITY_CUR" != "ok" ]; then
      echo "❌ $DB_NAME  integrity_check = $INTEGRITY_CUR → DB 损坏"
      FAIL=$((FAIL+1))
      ANOMALY_NODES+=("$DB_NAME (db corrupted)")
      continue
    fi

    # 追问 3: DB 内容是否真的在变？
    if [ "$CUR_MTIME" = "$LAST_MTIME" ] && [ "$CUR_SIZE" = "$LAST_SIZE" ]; then
      echo "⚠️  $DB_NAME  DB mtime/size 与上次探测一致 → 连续 N 次 identical (用户行为 or 同步未跑)"
      WARN=$((WARN+1))
      ANOMALY_NODES+=("$DB_NAME (db identical)")
    else
      echo "✅ $DB_NAME  DB mtime=$CUR_MTIME size=$CUR_SIZE  integrity=ok  推进中"
      OK=$((OK+1))
    fi
  done < "$DB_PROBES_FILE"
else
  echo "   (跳过 Part 3: $DB_PROBES_FILE 不存在)"
fi

# 汇总
echo
echo "=========================================="
echo "  汇总"
echo "=========================================="
echo "总节点/DB数:  $TOTAL"
echo "真稳定:       $OK"
echo "反常稳定:     $WARN"
echo "真故障:       $FAIL"
if [ $WARN -gt 0 ] || [ $FAIL -gt 0 ]; then
  echo
  echo "需要主动追问的项目:"
  for n in "${ANOMALY_NODES[@]}"; do
    echo "  - $n"
  done
  exit 1
fi
echo
echo "✅ 全部真稳定（无反常稳定）"
exit 0

chmod +x silent-period-probe-v2.sh
./silent-period-probe-v2.sh oc-nodes.txt hermes-nodes.txt db-probes.txt ping 10

# 追问 1: DB mtime 推进？
# 追问 2: DB size 推进？
# 追问 3: DB integrity ok？

# 1) 上次同步耗时
$ jq -r '.last_completion.duration_seconds' sync_wrapper_status.json
1692  # 28.2 min

# 2) 增量覆盖率
# 5/16 全量: 275,877 文件
# 6/7 增量: 56,816 文件
# → 20.6% 覆盖率 = 还有 80% 未扫
# → 80% 未扫 = 用户网盘里 80% 的文件 mtime 没变 = 合理（用户没动它们）

# 1) 写 cron task 之前先读 TOOLS.md 速查表
$ cat workspace/TOOLS.md | grep -A 5 "节点角色"
| **p2** | VM152 (Ubuntu, PVE245) | 192.168.102.xx |  ← HermesAgent, 不是 OpenClaw
| **p_N** | VM154 |  |  ← HermesAgent, 不是 OpenClaw

# 2) cron task prompt 模板按节点角色分类
# OpenClaw 节点：18789/readyz + systemd + docker
# Hermes 节点：9119/status + ps -ef grep hermes-agent

# 错误判断
$ stat -c %y baidupcs_cache.db
2026-06-07 16:53:33
# → "DB mtime 没变 = 系统稳" → 接受

# 正确判断
$ stat -c %y baidupcs_cache.db
2026-06-07 16:53:33
$ sqlite3 baidupcs_cache.db "PRAGMA integrity_check;"
ok
$ jq -r '.last_checked' sync_status.json
2026-06-09 20:57:14
# → "mtime 没变 + integrity ok + 探针 last_checked 在推进" = 真稳定 ✅
# → "mtime 没变 + integrity FAIL + 探针 last_checked 推进" = DB 损坏 ❌
# → "mtime 没变 + integrity ok + 探针 last_checked 没变" = 探针失效 ❌

# 错误判断
$ curl -s http://192.168.102.xx:18789/readyz
# → connection refused
# → "VM152 OpenClaw 挂了！" → 赶紧 SSH 进去修

# 正确判断
# → 先看节点角色（TOOLS.md 速查表）
# → VM152 是 HermesAgent，不是 OpenClaw
# → 18789 端口在 Hermes 上不存在 = by-design
# → 改用 9119/status 探针 = 探针与节点角色匹配

# 静默期早期（0-3 天）：4 次/天（防止刚修复的系统回退）
# 静默期中后期（3+ 天）：2 次/天（标准）
# 静默期长期（30+ 天且 8 类 PASS）：1 次/天（节能）
# 静默期绝不：0 次/天（失去连续性 = 重新踩坑）