tiktok-scripts/_probe/crawl.sh

#!/system/bin/sh
# 全量抓取 tikwm 顶级评论（串行，含假 hasMore / 空页容错），落盘到 /sdcard/Download/_pages/
PX="http://127.0.0.1:2080"
VID="7552202593888603410"
VURL="https://www.tiktok.com/video/$VID"
DIR=/sdcard/Download/_pages
LOG=/sdcard/Download/_crawl.log

mkdir -p $DIR
echo "START $(date)" > $LOG

cursor=0
emptyStreak=0
pages=0
got=0

while [ $cursor -le 11200 ]; do
  f="$DIR/p_$cursor.json"
  pages=$((pages+1))

  # 已存在且有效则跳过（支持断点续传）
  if [ -s "$f" ] && grep -q '"code":0' "$f" && [ "$(grep -o '"video_id"' $f | wc -l)" -gt 0 ]; then
    n=$(grep -o '"video_id"' $f | wc -l)
    got=$((got+n))
    echo "P$pages cursor=$cursor SKIP(已有) n=$n 累计=$got" >> $LOG
    cursor=$((cursor+50)); emptyStreak=0; continue
  fi

  attempt=1
  n=0
  while [ $attempt -le 3 ]; do
    curl -s --compressed -m 40 -x "$PX" "https://www.tikwm.com/api/comment/list?url=$VURL&count=50&cursor=$cursor" -o "$f"
    if grep -q '"code":0' "$f" 2>/dev/null; then
      n=$(grep -o '"video_id"' $f | wc -l)
      [ "$n" -gt 0 ] && break
    fi
    rm -f "$f"
    attempt=$((attempt+1))
    sleep 2
  done

  if [ "$n" -gt 0 ]; then
    emptyStreak=0
    got=$((got+n))
    # 短路标志：返回条数偏少或 hasMore=false，仅记录
    flag=$(grep -oE '"hasMore":(true|false)' "$f" | head -1)
    echo "P$pages cursor=$cursor n=$n 累计=$got $flag" >> $LOG
  else
    emptyStreak=$((emptyStreak+1))
    echo "P$pages cursor=$cursor EMPTY (streak=$emptyStreak)" >> $LOG
    if [ $emptyStreak -ge 3 ]; then
      echo "STOP 连续3个空cursor" >> $LOG
      break
    fi
  fi

  cursor=$((cursor+50))
  sleep 0.4
done

echo "DONE cursor=$cursor pages=$pages 累计条目=$got $(date)" >> $LOG
echo "DONE" > /sdcard/Download/_crawl.done