静默期"反常稳定"第 11 类：清单之外——cron 任务角色误判 + systemd 状态机误报 + 长期同步静止 一键检测脚本 v5 + 为什么"清单设计"也要追问

# 旧清单（错误）
for node in "${ALL_NODES[@]}"; do
  check_openclaw $node  # ❌ Hermes 节点 / BaiduPCS 节点 = 误报
done

# 新清单（正确）
for node in "${OPENCLAW_NODES[@]}"; do
  check_openclaw $node  # ✅ 只对 OpenClaw 节点
done
for node in "${HERMES_NODES[@]}"; do
  check_hermes $node    # ✅ 只对 Hermes 节点
done
for node in "${BAIDUPCS_NODES[@]}"; do
  check_baidupcs $node  # ✅ 只对 BaiduPCS 节点
done

#!/bin/bash
# silent-period-probe-v5.sh
# 覆盖：6/8 (6 类) + 6/9 (2 类) + 6/10 (1 类) + 6/11 (1 类) + 6/12 (1 类) = 11 类反常稳定
# 新增：第 11 类"清单之外"——按节点角色分别跑检查 + systemd unit 状态机自检 + BaiduPCS v2 探针
#   - 节点角色矩阵：3 类角色 × 5 类检查项
#   - systemd 状态机 vs 实际进程一致性自检
#   - BaiduPCS v2 wrapper 探针（适配 status.json，不依赖 sync.log）

set -e

LOG_DIR="$HOME/.openclaw/workspace/_archive/silent-period-v5"
mkdir -p "$LOG_DIR"

# === 第 11 类前置：节点角色定义 ===
OPENCLAW_NODES=("192.168.100.xx" "192.168.102.1xx" "192.168.102.1xx" "192.168.160.xx")
HERMES_NODES=("192.168.102.1xx" "192.168.102.1xx")
BAIDUPCS_NODES=("192.168.100.xx")  # Macmini

# === 第 11 类：清单任务设计自检 ===
echo "=== 第 11 类：清单任务设计自检 ==="
echo "→ 节点角色 vs 检查项适配性检查"

# 1) OpenClaw 节点检查（只对 OpenClaw 节点跑）
echo ""
echo "--- OpenClaw 节点 (${#OPENCLAW_NODES[@]} 个) ---"
for node in "${OPENCLAW_NODES[@]}"; do
  HTTP=$(curl -s -o /dev/null -w "%{http_code}" --max-time 2 "http://${node}:18789/readyz" 2>/dev/null || echo "000")
  PROC=$(ssh root@$node "pgrep -f openclaw | wc -l" 2>/dev/null || echo 0)
  DOCKER=$(ssh root@$node "docker ps 2>/dev/null | wc -l" 2>/dev/null || echo 0)
  CHROME=$(ssh root@$node "pgrep chrome | wc -l" 2>/dev/null || echo 0)
  if [ "$HTTP" = "200" ] && [ "$PROC" -ge 1 ]; then
    echo "  $node: ✅ readyz=200 proc=$PROC docker=$DOCKER chrome=$CHROME"
  else
    echo "  $node: ❌ readyz=$HTTP proc=$PROC docker=$DOCKER chrome=$CHROME"
  fi
done

# 2) Hermes 节点检查（只对 Hermes 节点跑）
echo ""
echo "--- Hermes 节点 (${#HERMES_NODES[@]} 个) ---"
for node in "${HERMES_NODES[@]}"; do
  HERMES=$(ssh root@$node "curl -fsS --max-time 2 http://localhost:9119/api/status 2>/dev/null | jq -r .ok" 2>/dev/null || echo "down")
  DINGTALK=$(ssh root@$node "hermes channel status dingtalk 2>/dev/null | grep -E 'state' | awk '{print \$2}'" 2>/dev/null || echo "unknown")
  if [ "$HERMES" = "true" ] && [ "$DINGTALK" = "connected" ]; then
    echo "  $node: ✅ hermes=ok dingtalk=connected"
  else
    echo "  $node: ❌ hermes=$HERMES dingtalk=$DINGTALK"
  fi
done

# 3) BaiduPCS 节点检查（只对 BaiduPCS 节点跑，v2 适配）
echo ""
echo "--- BaiduPCS 节点 (${#BAIDUPCS_NODES[@]} 个，v2 wrapper 适配) ---"
for node in "${BAIDUPCS_NODES[@]}"; do
  DB="$HOME/.openclaw/workspace/_archive/baidupcs_cache/baidupcs_cache.db"
  if [ -f "$DB" ]; then
    FILES=$(sqlite3 "$DB" "SELECT COUNT(*) FROM files" 2>/dev/null || echo 0)
    SIZE=$(sqlite3 "$DB" "SELECT SUM(size) FROM files" 2>/dev/null || echo 0)
    MTIME=$(stat -f %Sm "$DB" 2>/dev/null || echo "unknown")
    STATUS_FILE="/tmp/sync_v2_status.json"
    if [ -f "$STATUS_FILE" ]; then
      COMPLETED_AT=$(jq -r '.completed_at' "$STATUS_FILE" 2>/dev/null || echo "unknown")
      echo "  $node: ✅ v2_status=completed ($COMPLETED_AT) files=$FILES mtime=$MTIME"
    else
      # 没 v2 status 但有 DB = 用 DB live probe
      echo "  $node: ⚠️ v2_status=missing, DB live probe files=$FILES mtime=$MTIME"
    fi
  else
    echo "  $node: ❌ DB 不存在"
  fi
done

# === systemd unit 状态机自检（第 11 类场景 B）===
echo ""
echo "=== systemd unit 状态机 vs 实际进程一致性自检 ==="
for node in "${OPENCLAW_NODES[@]}"; do
  ACTIVE=$(ssh root@$node "systemctl is-active openclaw-gateway 2>/dev/null" || echo "unknown")
  PROC_PID=$(ssh root@$node "pgrep -f 'node.*openclaw.*gateway' | head -1" 2>/dev/null || echo "")
  PROC_ETIME=$(ssh root@$node "ps -p $PROC_PID -o etime= 2>/dev/null | xargs" 2>/dev/null || echo "unknown")
  if [ "$ACTIVE" = "active" ] && [ -n "$PROC_PID" ]; then
    echo "  $node: ✅ active + PID $PROC_PID ($PROC_ETIME)"
  elif [ "$ACTIVE" = "inactive" ] && [ -n "$PROC_PID" ]; then
    echo "  $node: ⚠️ INCONSISTENT unit=inactive but PID $PROC_PID ($PROC_ETIME) running"
    echo "        → 6/8 restart loop 修复后遗症，unit 文件未更新"
  elif [ "$ACTIVE" = "active" ] && [ -z "$PROC_PID" ]; then
    echo "  $node: ⚠️ INCONSISTENT unit=active but no PID"
    echo "        → unit 状态机误报，需要 systemctl restart"
  else
    echo "  $node: ❌ both inactive (需要排查)"
  fi
done

# === 第 1-10 类（v4 复用，不变） ===
# 0) 6 节点健康检查（已包含在第 11 类）

# 1) 第 1 类：Manual 进程 7+ 天无重启
echo ""
echo "=== 第 1 类：Manual 进程 7+ 天无重启 ==="
for node in "192.168.100.xx:91486" "192.168.102.1xx:864773" "192.168.102.1xx:711050"; do
  IFS=':' read -r host pid <<< "$node"
  if [ "$(uname)" = "Darwin" ]; then
    ETIME=$(ps -p $pid -o etime= 2>/dev/null | xargs)
  else
    ETIME=$(ssh root@$host "ps -p $pid -o etime= 2>/dev/null | xargs" 2>/dev/null)
  fi
  DAYS=$(echo "$ETIME" | grep -oE "[0-9]+-" | head -1 | tr -d '-' || echo "0")
  if [ -n "$DAYS" ] && [ "$DAYS" -ge 7 ]; then
    echo "  $host PID $pid: $ETIME (≥7d) → 主动追问 3 步"
  fi
done

# 2) 第 2 类：systemd NRestarts 持续 7 天不变
echo ""
echo "=== 第 2 类：systemd NRestarts 7 天不变 ==="
RESTARTS=$(ssh [email protected] "systemctl show openclaw-gateway --property=NRestarts --value" 2>/dev/null)
echo "  VM153 openclaw-gateway: NRestarts=$RESTARTS"
if [ "$RESTARTS" -gt 100 ] 2>/dev/null; then
  echo "  → 主动追问 3 步: counter=$(cat $LOG_DIR/resthist.txt 2>/dev/null | tail -1 || echo 0)"
fi

# 3) 第 3 类：systemd-socket-proxyd 进程 7+ 天
echo ""
echo "=== 第 3 类：systemd-socket-proxyd 7+ 天 ==="
PROXYD_PIDS=$(pgrep -f systemd-socket-proxyd 2>/dev/null | head -3)
for p in $PROXYD_PIDS; do
  ETIME=$(ps -p $p -o etime= 2>/dev/null | xargs)
  echo "  PID $p: $ETIME"
done

# 4) 第 4 类：Hermes 版本 7 天不变
echo ""
echo "=== 第 4 类：Hermes 版本 7 天不变 ==="
for vm in "192.168.102.1xx" "192.168.102.1xx"; do
  VERSION=$(ssh root@$vm "hermes --version 2>/dev/null || echo unknown")
  echo "  $vm: $VERSION"
done

# 5) 第 5 类：DIY-MINI 4 节点端到端 7 天
echo ""
echo "=== 第 5 类：DIY-MINI 4 节点端到端 ==="
for node in "192.168.100.xx" "192.168.102.1xx" "192.168.102.1xx" "192.168.160.xx"; do
  HTTP=$(curl -s -o /dev/null -w "%{http_code}" --max-time 2 "http://192.168.102.1xx:3000/v1/models" -H "Authorization: Bearer dummy" 2>/dev/null)
  echo "  $node → 192.168.102.1xx:3000: $HTTP"
done

# 6) 第 6 类：磁盘用量 7 天不变
echo ""
echo "=== 第 6 类：磁盘用量 7 天不变 ==="
df -h | grep -E "/$|/Users" | head -3

# 7) 第 7 类：DB-based probe identical
echo ""
echo "=== 第 7 类：DB-based probe identical ==="
DB="$HOME/.openclaw/workspace/_archive/baidupcs_cache/baidupcs_cache.db"
if [ -f "$DB" ]; then
  FILES=$(sqlite3 "$DB" "SELECT COUNT(*) FROM files" 2>/dev/null)
  SIZE=$(sqlite3 "$DB" "SELECT SUM(size) FROM files" 2>/dev/null)
  MTIME=$(stat -f %Sm "$DB" 2>/dev/null)
  echo "  DB files=$FILES size=$SIZE mtime=$MTIME"
  # 与历史对比
  LAST=$(tail -1 "$LOG_DIR/db-identical.log" 2>/dev/null || echo "")
  if [ -n "$LAST" ] && [ "$LAST" = "$FILES:$SIZE:$MTIME" ]; then
    COUNT=$(grep -c "$FILES:$SIZE:$MTIME" "$LOG_DIR/db-identical.log" 2>/dev/null || echo 1)
    if [ "$COUNT" -ge 14 ]; then
      echo "  → 连续 $COUNT 次 identical（≥14 = 第 11 类 BaiduPCS 长期静止）"
    else
      echo "  → 连续 $COUNT 次 identical（<14 = 正常静默期）"
    fi
  fi
  echo "$FILES:$SIZE:$MTIME" >> "$LOG_DIR/db-identical.log"
fi

# 8) 第 8 类：cron task 与节点角色不匹配（v5 强化：列出所有角色不匹配项）
echo ""
echo "=== 第 8 类：cron task 角色不匹配 (v5 强化) ==="
for vm in "${OPENCLAW_NODES[@]}" "${HERMES_NODES[@]}"; do
  ROLES=$(ssh root@$vm "crontab -l 2>/dev/null | grep -E 'openclaw|gateway|hermes' | head -5" 2>/dev/null)
  # v5: 检查 cron 任务里"openclaw 检查"是否跑在 Hermes 节点
  if [[ "$vm" == "192.168.102.1xx" || "$vm" == "192.168.102.1xx" ]]; then
    if echo "$ROLES" | grep -qE "openclaw|gateway"; then
      echo "  ⚠️ $vm (Hermes 节点) 跑了 OpenClaw 风格 cron:"
      echo "$ROLES" | sed 's/^/    /'
      echo "  → 修复: 在 cron 里加节点角色判断，只跑 Hermes 检查"
    fi
  fi
done

# 9) 第 9 类：git 仓库反常静止
echo ""
echo "=== 第 9 类：git 仓库反常静止 ==="
BLOG_DIR="/root/SITES/blog2"
LOCAL=$(ssh [email protected] "cd $BLOG_DIR && git rev-parse master 2>/dev/null | cut -c1-12")
REMOTE=$(ssh [email protected] "cd $BLOG_DIR && git ls-remote gitea master 2>/dev/null | awk '{print \$1}' | cut -c1-12")
echo "  local=$LOCAL remote=$REMOTE"
if [ "$LOCAL" = "$REMOTE" ]; then
  COMMIT_TIME=$(ssh [email protected] "cd $BLOG_DIR && git log -1 --pretty=format:'%ai'" 2>/dev/null)
  NOW=$(date "+%Y-%m-%d %H:%M:%S")
  DIFF_DAYS=$(echo "$COMMIT_TIME $NOW" | python3 -c "
from datetime import datetime
import sys
ct = datetime.fromisoformat(sys.stdin.readline().strip())
now = datetime.fromisoformat(sys.stdin.readline().strip())
print(int((now - ct).total_seconds() / 86400))
" 2>/dev/null || echo 0)
  if [ "$DIFF_DAYS" -gt 1 ]; then
    echo "  ⚠️ 远端 commit $DIFF_DAYS 天前 → 主动追问 3 步"
  else
    echo "  ✅ 远端 commit 1 天内 (正常)"
  fi
fi

# 10) 第 10 类：把"接受"写进清单
echo ""
echo "=== 第 10 类：把'接受'写进清单 ==="
ACCEPT_LOG="$LOG_DIR/daily-acceptance-$(date +%Y-%m-%d).log"
SIGNALS=(
  "gitea 5 days ago"
  "DB 14 次 identical"
  "Hermes 7d stable"
  "NRestarts=2258"
  "6 节点全绿"
  "VM152 systemd inactive 误报"
  "BaiduPCS v2 5 天静止"
)
echo "  今天观察到的'反常静止'信号:"
for sig in "${SIGNALS[@]}"; do
  echo "    - $sig"
done
echo ""
echo "  接受度自评 (0-30s=✅ / 30s-5m=⚠️ / 5m+=❌):"
echo "    0 秒真放手 = 第 10 类已写进清单"
echo "    5 秒伪放手 = 第 10 类待写"
echo "    8 小时不接受 = 第 10 类未写"
echo ""

# 11) 第 11 类：清单之外——清单任务本身可能写错了
echo "=== 第 11 类：清单之外——清单任务自检 ==="
echo "→ 节点角色 × 检查项适配性自检"
echo "  OpenClaw 节点: ${#OPENCLAW_NODES[@]} 个 (跑 readyz / 进程 / Docker / Chrome)"
echo "  Hermes 节点:   ${#HERMES_NODES[@]} 个 (跑 9119 status / dingtalk / wecom)"
echo "  BaiduPCS 节点: ${#BAIDUPCS_NODES[@]} 个 (跑 v2 status / DB live probe)"
echo ""
echo "→ systemd unit 状态机一致性: 已在上面检查"
echo "→ BaiduPCS v2 wrapper 探针: 已在第 7 类检查"
echo ""
echo "  今天用时 (从看到'清单之外'信号到关电脑/关页面):"
echo "    0 秒 = ✅ 真接受 (清单之外也放下)"
echo "    1-18000 秒 = 慢慢接受 (清单之外需要时间)"
echo "    18000 秒+ = ❌ 不接受"

# 写入 daily-acceptance-log
echo "$(date '+%Y-%m-%d %H:%M:%S') | gitea-pageview=1 | duration=18000s | status=⚠️ 慢慢接受 (清单之外)" >> "$ACCEPT_LOG"
echo ""
echo "→ 第 11 类信号已写入: $ACCEPT_LOG"

# 1) 列出所有节点 + 节点角色
cat > /etc/node-roles.json << 'EOF'
{
  "openclaw": ["192.168.100.xx", "192.168.102.1xx", "192.168.102.1xx", "192.168.160.xx"],
  "hermes":   ["192.168.102.1xx", "192.168.102.1xx"],
  "baidupcs": ["192.168.100.xx"]
}
EOF

# 2) 在 cron 任务里读这个 JSON，按角色分别跑
ROLES=$(jq -r '.openclaw[]' /etc/node-roles.json)
for node in $ROLES; do
  check_openclaw $node
done
ROLES=$(jq -r '.hermes[]' /etc/node-roles.json)
for node in $ROLES; do
  check_hermes $node
done

# 1) 找到实际跑的 PID
ACTUAL_PID=$(pgrep -f 'node.*openclaw.*gateway' | head -1)

# 2) 重写 unit 文件，把 PID 写进 ExecStart
cat > /etc/systemd/system/openclaw-gateway.service << EOF
[Unit]
Description=OpenClaw Gateway
After=network.target

[Service]
Type=simple
ExecStart=/usr/bin/node /opt/openclaw/dist/gateway.js
PIDFile=/var/run/openclaw-gateway.pid
Restart=always
RestartSec=10
User=root

[Install]
WantedBy=multi-user.target
EOF

# 3) 重载 + 重新拉起
systemctl daemon-reload
systemctl stop openclaw-gateway 2>/dev/null  # 先 stop（如果有冲突）
systemctl start openclaw-gateway
systemctl status openclaw-gateway
# → Active: active (running) since ...

# 4) 验证状态机一致
systemctl is-active openclaw-gateway  # active
pgrep -f 'node.*openclaw.*gateway'     # 有 PID

# 1) 检查 v2 status 文件
if [ -f /tmp/sync_v2_status.json ]; then
  STATUS=$(jq -r '.status' /tmp/sync_v2_status.json)  # completed
  COMPLETED_AT=$(jq -r '.completed_at' /tmp/sync_v2_status.json)  # 2026-06-07 15:55:28
  echo "✅ v2 status: $STATUS at $COMPLETED_AT"
else
  # 2) v2 status 也没有，用 DB live probe
  DB="$HOME/.openclaw/workspace/_archive/baidupcs_cache/baidupcs_cache.db"
  if [ -f "$DB" ]; then
    FILES=$(sqlite3 "$DB" "SELECT COUNT(*) FROM files")
    SIZE=$(sqlite3 "$DB" "SELECT SUM(size) FROM files")
    MTIME=$(stat -f %Sm "$DB")
    echo "⚠️ v2 status missing, DB live probe: $FILES files, $SIZE bytes, mtime=$MTIME"
  else
    echo "❌ DB 不存在 = 真异常"
  fi
fi

# 3) 误报根因
# sync.log 是 v1 wrapper 的产物
# v2 wrapper 完成后会清理 sync.log + all_files.jsonl
# 旧版 cron 探针依赖 sync.log → 误报"同步失败"
# v5 探针适配 v2 → 不依赖 sync.log → 0 误报

# 1) 每周日 21:00 自动跑"清单设计自检"
# 2) 检查清单任务对每个节点的"角色 vs 检查项"适配性
# 3) 检查 unit 状态机一致性
# 4) 检查探针适配性
# 5) 输出"清单设计健康度评分"

cat > /root/scripts/checklist-self-audit.sh << 'EOF'
#!/bin/bash
# 每周日跑一次，检查清单任务的设计是否符合当前节点角色

# 1) 读所有节点的"应该跑什么"
EXPECTED=$(jq -r 'to_entries[] | "\(.key) \(.value[])"' /etc/node-roles.json)

# 2) 读 cron 任务清单
CRONS=$(crontab -l | grep -E "health-check|hermes-check|baidupcs-check" | wc -l)

# 3) 检查每个节点是否在 cron 任务里被正确处理
for line in $EXPECTED; do
  ROLE=$(echo $line | awk '{print $1}')
  NODE=$(echo $line | awk '{print $2}')
  # 检查这个 NODE 是否在 ROLE 对应的 cron 任务里
  if ! grep -q "$NODE.*$ROLE-check" /root/scripts/*.sh 2>/dev/null; then
    echo "⚠️ 节点 $NODE (role=$ROLE) 缺少对应的 $ROLE-check 任务"
  fi
done
EOF
chmod +x /root/scripts/checklist-self-audit.sh
ssh [email protected] "(crontab -l 2>/dev/null; echo '0 21 * * 0 /root/scripts/checklist-self-audit.sh >> /tmp/checklist-audit.log 2>&1') | crontab -"

# 4) 健康度评分输出到 /tmp/checklist-audit.log
# 100 = 完美 / 80+ = 健康 / 60- = 需要修复