新增 wall-audit.js:反爬墙现场审计(Cookie/网络/登录态/IP 快照+跨天对比)

skills/news-harvester/scripts/wall-audit.js

+#!/usr/bin/env node
+/**
+ * wall-audit.js - 反爬墙现场审计:回答"墙出现是会话丢失还是服务端重新风控"
+ *
+ * 每次运行采集一份快照(JSON):
+ *   - Cookie 全量清单(名称/域/到期时间/httpOnly/session,重点 datadome)
+ *   - 页面状态(是否撞墙/是否登录/最终 URL/标题)
+ *   - Network 证据(主文档及 xhr 请求的状态码、响应头、跳转,重点 403/429/set-cookie/x-dd-*)
+ *   - 出口 IP(api.ipify.org)
+ *
+ * 用法:
+ *   CDP_PORT_OVERRIDE=9408 CDP_PROFILE_OVERRIDE=~/.news-harvester/profiles/reuters \
+ *   NH_AUDIT_DIR=~/NewsHarvester/logs/reuters.wall-audit node scripts/wall-audit.js            # 抓快照
+ *   ... node scripts/wall-audit.js --compare                                                   # 与上一份对比
+ *
+ * 快照: $NH_AUDIT_DIR/<时间戳>.json,并更新 latest.json
+ * 退出码:0 无墙 / 3 有墙(compare 模式按当前状态)
+ */
+const fs = require('fs');
+const path = require('path');
+const http = require('http');
+const https = require('https');
+const { spawnSync } = require('child_process');
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+
+const PORT = parseInt(process.env.CDP_PORT_OVERRIDE || '9408', 10);
+const URL_TO_VISIT = process.env.NH_KEEPALIVE_URL || 'https://www.reuters.com/';
+const AUDIT_DIR = process.env.NH_AUDIT_DIR || '/tmp/wall-audit';
+const COMPARE = process.argv.includes('--compare');
+
+const cdp = (p, method = 'GET') => new Promise((res, rej) => {
+  const r = http.request({ host: '127.0.0.1', port: PORT, path: p, method }, (resp) => {
+    let d = ''; resp.on('data', (c) => (d += c));
+    resp.on('end', () => { try { res(JSON.parse(d)); } catch { res(d); } });
+  });
+  r.on('error', rej); r.end();
+});
+const fetchText = (url) => new Promise((res) => {
+  https.get(url, { timeout: 8000 }, (r) => { let d = ''; r.on('data', (c) => (d += c)); r.on('end', () => res(d.trim())); })
+    .on('error', () => res(''));
+});
+
+async function main() {
+  fs.mkdirSync(AUDIT_DIR, { recursive: true });
+  const startedByUs = !(await (async () => { try { await cdp('/json/version'); return true; } catch { return false; } })());
+  if (startedByUs) {
+    const r = spawnSync('node', [path.join(__dirname, 'ensure-chromium.js')], { stdio: 'ignore', env: process.env });
+    if (r.status !== 0) { console.error('wall-audit: 浏览器实例拉起失败'); process.exit(2); }
+  }
+
+  const tab = await cdp('/json/new', 'PUT');
+  const ws = new WebSocket((await cdp('/json')).find((t) => t.id === tab.id).webSocketDebuggerUrl);
+  await new Promise((r, j) => { ws.onopen = r; ws.onerror = j; });
+  let seq = 0; const pending = new Map();
+  const send = (method, params = {}) => new Promise((res) => {
+    const s = ++seq; pending.set(s, res); ws.send(JSON.stringify({ id: s, method, params }));
+  });
+  const events = [];
+  ws.onmessage = (m) => {
+    const j = JSON.parse(m.data);
+    if (pending.has(j.id)) { pending.get(j.id)(j.result); pending.delete(j.id); return; }
+    if (j.method === 'Network.responseReceived') {
+      const r = j.params.response;
+      events.push({
+        url: (r.url || '').slice(0, 120), status: r.status,
+        location: r.headers?.Location || r.headers?.location || '',
+        server: r.headers?.Server || r.headers?.server || '',
+        ddHeaders: Object.fromEntries(Object.entries(r.headers || {}).filter(([k]) => /dd|datadome/i.test(k))),
+        setCookie: (r.headers?.['Set-Cookie'] || r.headers?.['set-cookie'] || '').slice(0, 200),
+      });
+    }
+  };
+
+  await send('Page.enable');
+  await send('Network.enable');
+  await send('Page.navigate', { url: URL_TO_VISIT });
+  await sleep(15000);
+
+  const page = await send('Runtime.evaluate', {
+    returnByValue: true,
+    expression: `JSON.stringify({
+      url: location.href, title: document.title.slice(0, 80),
+      ddIframe: !!document.querySelector('iframe[title*=datadome i], iframe[src*=datadome]'),
+      ddWidget: !!document.querySelector('#ddv1-captcha, [class*=ddcaptcha]'),
+      captchaWord: /unusual traffic|verify you|滑动|验证/i.test(document.body.innerText.slice(0, 4000)),
+      accountBtn: !!document.querySelector('[data-testid=\\"AccountButton\\"]'),
+      signIn: [...document.querySelectorAll('a,button')].some(e => /sign in to reuters/i.test(e.innerText||''))
+    })`,
+  });
+  const pageInfo = JSON.parse(page.result.value || '{}');
+  pageInfo.wall = !!(pageInfo.ddIframe || pageInfo.ddWidget || pageInfo.captchaWord);
+  pageInfo.loggedIn = !!pageInfo.accountBtn;
+
+  const cookiesRaw = (await send('Network.getCookies', { urls: [URL_TO_VISIT] })).cookies || [];
+  const cookies = cookiesRaw
+    .map((c) => ({
+      name: c.name, domain: c.domain,
+      expires: c.session ? 'session' : new Date(c.expires * 1000).toISOString(),
+      httpOnly: !!c.httpOnly, secure: !!c.secure,
+      maxAgeDays: c.session ? null : Math.round((c.expires - Date.now() / 1000) / 86400),
+    }))
+    .sort((a, b) => a.name.localeCompare(b.name));
+  ws.close();
+  await cdp(`/json/close/${tab.id}`).catch(() => {});
+
+  const snap = {
+    ts: new Date().toISOString(), visit: URL_TO_VISIT, port: PORT,
+    egressIP: await fetchText('https://api.ipify.org'),
+    page: pageInfo, cookies,
+    ddCookie: cookies.find((c) => c.name === 'datadome') || null,
+    network: events.filter((e) => e.status >= 300 || Object.keys(e.ddHeaders).length).slice(0, 30),
+  };
+
+  const file = path.join(AUDIT_DIR, snap.ts.replace(/[:.]/g, '-') + '.json');
+  fs.writeFileSync(file, JSON.stringify(snap, null, 2));
+  fs.writeFileSync(path.join(AUDIT_DIR, 'latest.json'), JSON.stringify(snap, null, 2));
+
+  if (startedByUs) {
+    spawnSync('node', [path.join(__dirname, 'instance-quit.js')], { stdio: 'ignore', env: process.env });
+  }
+
+  // 摘要输出
+  console.log(`wall-audit: ${snap.ts} wall=${pageInfo.wall} loggedIn=${pageInfo.loggedIn} egressIP=${snap.egressIP || '?'}`);
+  console.log(`  datadome cookie: ${snap.ddCookie ? `expires=${snap.ddCookie.expires} (剩 ${snap.ddCookie.maxAgeDays} 天)` : '不存在'}`);
+  console.log(`  cookies=${cookies.length} 条;关键拦截/跳转请求=${snap.network.length} 条;快照=${file}`);
+
+  if (COMPARE) {
+    const prevs = fs.readdirSync(AUDIT_DIR).filter((f) => f.endsWith('.json') && f !== 'latest.json').sort();
+    if (prevs.length >= 2) {
+      const prev = JSON.parse(fs.readFileSync(path.join(AUDIT_DIR, prevs[prevs.length - 2]), 'utf-8'));
+      const key = (c) => `${c.name}@${c.domain}`;
+      const prevMap = new Map(prev.cookies.map((c) => [key(c), c]));
+      const curMap = new Map(cookies.map((c) => [key(c), c]));
+      const added = [...curMap.keys()].filter((k) => !prevMap.has(k));
+      const removed = [...prevMap.keys()].filter((k) => !curMap.has(k));
+      const changed = [...curMap.keys()].filter((k) => prevMap.has(k) && prevMap.get(k).expires !== curMap.get(k).expires);
+      console.log('  ── 对比上一份(' + prev.ts + ')──');
+      console.log(`  状态: wall ${prev.page.wall}→${pageInfo.wall} | 登录 ${prev.page.loggedIn}→${pageInfo.loggedIn} | IP ${prev.egressIP}→${snap.egressIP}`);
+      if (prev.ddCookie && snap.ddCookie) console.log(`  datadome 到期: ${prev.ddCookie.expires} → ${snap.ddCookie.expires}`);
+      if (added.length) console.log('  新增 cookie: ' + added.join(', '));
+      if (removed.length) console.log('  消失 cookie: ' + removed.join(', '));
+      if (changed.length) console.log('  到期变化: ' + changed.join(', '));
+      if (!added.length && !removed.length && !changed.length) console.log('  cookie 无变化');
+      console.log('  解读:cookie 在而墙出现=服务端按 IP/行为重新风控;cookie 消失/过期=会话状态丢失');
+    } else {
+      console.log('  (只有一份快照,再采一份后 --compare 才能对比)');
+    }
+  }
+  process.exit(pageInfo.wall ? 3 : 0);
+}
+
+main().catch((e) => { console.error(`wall-audit: ${e.message}`); process.exit(2); });