动态网页抓取

cd /home/axelhu/.openclaw/workspace
npm install playwright

mkdir -p ~/bin
cat > ~/bin/google-chrome << 'EOF'
#!/bin/bash
exec /usr/bin/google-chrome --remote-debugging-port=9222 "$@"
EOF
chmod +x ~/bin/google-chrome
echo 'export PATH="$HOME/bin:$PATH"' >> ~/.bashrc
export PATH="$HOME/bin:$PATH"

# 必须加 DISPLAY=:0，否则 exec 会话中 Chrome 无法找到显示器
DISPLAY=:0 google-chrome \
  --remote-debugging-port=9222 \
  --user-data-dir=$HOME/.config/google-chrome/Default \
  --new-window \
  --no-sandbox \
  > /tmp/chrome-debug.log 2>&1 &

# 验证启动成功
sleep 3 && curl -s http://localhost:9222/json/version | head -c 50

bash /home/axelhu/.openclaw/skills/axelhu-playwright-scrape/scripts/start-chrome-debug.sh

node skills/axelhu-playwright-scrape/scripts/playwright-scrape.js <URL> [mode]

const { chromium } = require('/home/axelhu/.openclaw/workspace/node_modules/playwright');
const browser = await chromium.connectOverCDP('http://localhost:9222');
const page = (await browser.contexts())[0].pages()[0]; // 复用已有标签页

const ctx = await browser.newContext();
const page = await ctx.newPage();
await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 15000 });
await page.waitForTimeout(4000); // 等待 JS 渲染

// 获取当前登录用户的 SESSDATA cookie
const cookies = await ctx.cookies(['https://bilibili.com']);
const sessdata = cookies.find(c => c.name === 'SESSDATA')?.value;

// 调用 B站 API
const resp = await fetch('https://api.bilibili.com/x/relation/followings?pn=1&ps=20&vmid=UID', {
  headers: { 'Cookie': 'SESSDATA=' + sessdata }
});
const data = await resp.json();

{
  "url": "https://...",
  "mode": "gui|headless|stealth",
  "title": "页面标题",
  "content": "正文内容（前15000字）",
  "images": ["图片URL列表"],
  "links": [{"text": "链接文字", "href": "链接地址"}],
  "loadTime": "1.23s"
}

await page.waitForTimeout(6000); // 默认 4000