Skip to content

N
News Hub Skills
Commit 6a313771 authored by 谢宇轩's avatar 谢宇轩
Browse files
Options

feat: edit cdp and block check script

parent a891ad4e
Hide whitespace changes
Inline Side-by-side
Showing with 324 additions and 162 deletions
skills/news-harvester/bundles/bloomberg-technology.nhsource.json 0 → 100644
View file @ 6a313771
{
"format": "news-harvester-source",
"version": 1,
"exportedAt": "2026-08-13T06:55:12.088Z",
"source": {
"id": "bloomberg-technology",
"name": "彭博社科技",
"homepageUrl": "https://www.bloomberg.com/technology",
"method": "browser",
"vaultFolder": "彭博社/科技"
},
"files": {
"helpers/bloomberg-technology.md": "---\nsource: bloomberg-technology\n\n# ── 发现阶段 ──\n# Bloomberg Technology 首页文章分散在多个 LineupContent* 卡片容器 +\n# styles_itemTextContainer 列表中，用复合 listSelector 一次性圈中。\nlistSelector: '[class*=\"LineupContent\"], [class*=\"itemTextContainer\"]'\nlinkSelector: 'a[href]'\n\n# 文章 URL 形如 /news/articles/YYYY-MM-DD/<slug> 或 /news/features/YYYY-MM-DD/<slug>\n# 视频 /news/videos/ 用 excludeUrlPattern 排除。\nurlPattern: 'bloomberg\\.com/news/(articles|features)/.+'\nexcludeUrlPattern: 'bloomberg\\.com/news/videos/'\n\nscrollSteps: 3\nscrollWaitMs: 2000\nwaitMs: 15000\nmaxLinks: 30\n\n# ── 提取阶段 ──\n# Bloomberg 正文容器候选（按优先级）。\nbodySelectors:\n - '[data-testid=\"paragraph\"]'\n - '[class*=\"articleBodyContent\"] p'\n - '[class*=\"body-content\"] p'\n - '[class*=\"ArticleBody\"] p'\n - '[class*=\"article-body\"] p'\n - 'article p'\nbodyMinParagraphs: 3\n\ndateSelector: 'time[datetime]'\nauthorSelector: '[rel=\"author\"], [class*=\"author\"] a'\n\nimageSelector: 'img'\nimageMinWidth: 200\nexcludeImage:\n - logo\n - icon\n - avatar\n - profile\n\n# Bloomberg 文末常见署名/订阅推广噪声。\nnoisePrefixes:\n - 'Sign up for'\n - 'Subscribe to'\n - 'Read more:'\n - 'Most Read from Bloomberg'\n - 'This article was produced'\n - '©2026 Bloomberg'\n\ntitleStripSuffix: ' - Bloomberg$'\nmaxBodyChars: 15000\n---\n# bloomberg-technology 抓取备忘\n\n- 首页 `https://www.bloomberg.com/technology` 文章卡片分布在多个 `LineupContent*`\n 容器（4Up/3Up/2Up/Basic）以及 `styles_itemTextContainer` 列表中，inspect 实测\n 首屏约 22 篇文章链接 + 4 个视频。用复合 `listSelector` 圈中所有卡片。\n- `urlPattern` 限定 `/news/(articles|features)/.+`，天然排除 `/technology/ai` 等\n section 页与 `/markets` 等导航；`excludeUrlPattern` 排除 `/news/videos/`。\n- 反爬/付费墙：Bloomberg 对 headless 有反爬，正文可能因未登录被截断。\n 如 preview 出现 charCount<500 或 CAPTCHA（exit 70），需\n `node scripts/ensure-chromium.js --login --url https://www.bloomberg.com/technology`\n 在窗口浏览器登录后退出，持久化 cookie 到 userDataDir。\n- 验证: node scripts/preview.js --url https://www.bloomberg.com/technology \\\n --recipe helpers/bloomberg-technology.md --count 3\n",
"helpers/bloomberg-technology.fixtures.json": "{\n \"articles\": [\n \"https://www.bloomberg.com/news/articles/2026-08-13/trump-enlists-private-sector-to-boost-cyber-offensive-arsenal\",\n \"https://www.bloomberg.com/news/articles/2026-08-13/anthropic-said-in-talks-to-buy-ai-startup-decart-for-6-billion\",\n \"https://www.bloomberg.com/news/features/2026-08-12/thousands-of-india-workers-are-helping-ai-firms-train-robots-to-replace-them\"\n ],\n \"sections\": [\n \"https://www.bloomberg.com/technology/ai\",\n \"https://www.bloomberg.com/technology/big-tech\"\n ],\n \"minLinks\": 10\n}\n"
}
}
\ No newline at end of file
skills/news-harvester/scripts/block-check.js
View file @ 6a313771
......@@ -4,10 +4,16 @@
*
* When a block is detected, a HumanInterventionError is thrown so harvest.js can
* exit immediately with a friendly message instead of silently archiving empty
* or garbage articles. The check is intentionally conservative: it only fires on
* high-confidence signals (challenge iframes/titles, auth-host redirects, or a
* login-wall CTA paired with a near-empty body) to avoid false positives on
* legitimate articles.
* or garbage articles.
*
* Detection philosophy — multi-signal + body-length gating:
* A real challenge/anti-bot page has almost no body text (the challenge
* widget replaces content). A real article page may embed passive risk-scoring
* iframes (invisible reCAPTCHA Enterprise, PerimeterX) or cookie-consent
* banners whose copy overlaps challenge phrases — but it also has thousands of
* chars of real article text. So weak signals (iframe, body-text) are only
* treated as blocking when the body is short; strong signals (challenge page
* title, dual weak signals) block regardless of body length.
*
* Two entry points:
* • Browser (CDP) sources — evaluateBrowserBlock(ws) runs an in-page probe via
......@@ -19,18 +25,30 @@
// Sent to the page as a Runtime.evaluate expression. Serialized from a real
// function via .toString() so regex backslashes survive without double-escaping.
// IMPORTANT: must be self-contained (no closure vars, no arrow/const/let) so
// .toString() produces valid JS for Runtime.evaluate.
function _browserBlockProbe() {
var url = location.href, host = location.hostname;
var title = (document.title || '').trim();
var bodyText = (document.body && document.body.innerText) ? document.body.innerText.substring(0, 4000) : '';
var bodyLen = bodyText.length;
// 1. CAPTCHA / anti-bot challenge — iframes from known challenge providers.
// Note: recaptcha/api2/aframe is a passive background iframe used by
// reCAPTCHA v3 for ad-fraud prevention — it does NOT block the user and is
// present on many news sites (Reuters, Nikkei, etc.). Only match the active
// challenge iframes (anchor/bframe/enterprise) that constitute a visible
// verification widget.
// 正文长度门控阈值。超过此值说明页面有实质内容，被动风控元素
// (invisible reCAPTCHA、cookie 横幅中的挑战文案) 不应判定为阻断性挑战。
// 真正的挑战/反爬页正文极短，不会超过这个阈值。
var BODY_GATE = 1500;
// 1. CAPTCHA / anti-bot challenge.
//
// challengeTitle — 挑战页标题 (Cloudflare "Just a moment..."、AWS "Access
// Denied" 等)。高置信度信号，直接阻断，不受正文门控影响。注意 "verify"
// 已收紧为 "verify you are human"，避免误判以 "Verify" 开头的文章标题。
var challengeTitle = /^(just a moment|attention required|verify you are human|checking your browser|are you human)/i.test(title)
|| /access denied|请输入验证码|安全验证/i.test(title);
// challengeIframe — 已知挑战提供商的 iframe。不再对 invisible reCAPTCHA
// 做逐 iframe 特殊判断：正文门控统一处理被动风控 iframe。
// recaptcha/api2/aframe 是 reCAPTCHA v3 的被动后台 iframe，不在匹配列表中。
var challengeIframe = Array.from(document.querySelectorAll('iframe[src]')).some(function (f) {
var s = f.src.toLowerCase();
return s.indexOf('recaptcha/api2/anchor') !== -1
......@@ -41,12 +59,17 @@ function _browserBlockProbe() {
|| s.indexOf('turnstile') !== -1
|| s.indexOf('arkoselabs') !== -1 || s.indexOf('funcaptcha') !== -1;
});
// Challenge page titles (Cloudflare "Just a moment...", "Access Denied", etc.).
var challengeTitle = /^(just a moment|attention required|verify|checking your browser|are you human)/i.test(title)
|| /access denied|请输入验证码|安全验证/i.test(title);
// Challenge body copy.
// challengeText — 挑战页正文文案。
var challengeText = /please verify (that )?you are human|verify you are human|unusual traffic from your computer|checking your browser before accessing|enable javascript and cookies to continue|our system has detected unusual traffic|请完成安全验证|进行人机验证/i.test(bodyText);
if (challengeIframe || challengeTitle || challengeText) {
// 多信号置信度模型:
// challengeTitle 命中 → 高置信度，直接阻断
// challengeIframe + challengeText 同时命中 → 双弱信号叠加为高置信度，直接阻断
// 单弱信号 (iframe 或 text) + 短正文 (< BODY_GATE) → 阻断 (挑战页无实质内容)
// 单弱信号 + 长正文 (≥ BODY_GATE) → 不阻断 (被动风控 iframe / cookie 横幅)
if (challengeTitle || (challengeIframe && challengeText)
|| ((challengeIframe || challengeText) && bodyLen < BODY_GATE)) {
return JSON.stringify({ blocked: true, kind: 'captcha', reason: '人机验证/反爬挑战页面', url: url, title: title });
}
......@@ -54,7 +77,11 @@ function _browserBlockProbe() {
// path. A real article page is never served from accounts.* / login.* or a
// /signin path, so this is a high-confidence signal regardless of body length.
var authHost = /^(accounts\.|login\.|signin\.|auth\.|sso\.|chkpt\.)/i.test(host);
var authPath = /\/(signin|sign-in|login|auth|account\/login|subscribe|sso)\b/i.test(location.pathname + location.search);
// authPath: 只匹配完整路径段 (前后为 / 或字符串边界)，避免误判
// /technology/sso-integration-guide 等含关键词子串的正常路径。
// 只检测 pathname，不检测 search —— query 参数中的 login/subscribe
// 不是重定向信号，由 path 3 (paywall CTA) 兜底。
var authPath = /(^|\/)(signin|sign-in|login|auth|account\/login|subscribe|sso)(\/|$)/i.test(location.pathname);
if (authHost || authPath) {
return JSON.stringify({ blocked: true, kind: 'login', reason: '页面被重定向到登录/认证页面（登录状态可能已失效）', url: url, title: title });
}
......@@ -106,24 +133,33 @@ async function evaluateBrowserBlock(ws) {
// String scan of raw HTML for challenge/login-wall signals. No DOM, so the
// signals are a subset of the browser probe — enough to catch a Cloudflare
// interstitial or a login redirect on an open site.
//
// Same body-gating principle as the browser probe: challenge title is a
// high-confidence signal (block immediately); challenge body text is gated by
// <p> content length so a cookie-banner phrase on a real article doesn't
// trigger a false positive.
const DIRECT_CHALLENGE_RE = /please verify (that )?you are human|verify you are human|unusual traffic from your computer|checking your browser before accessing|enable javascript and cookies to continue|our system has detected unusual traffic|请完成安全验证|进行人机验证/i;
const DIRECT_CHALLENGE_TITLE_RE = /<title[^>]*>(just a moment|attention required|verify|checking your browser|are you human|access denied|请输入验证码|安全验证)/i;
const DIRECT_CHALLENGE_TITLE_RE = /<title[^>]*>(just a moment|attention required|verify you are human|checking your browser|are you human|access denied|请输入验证码|安全验证)/i;
const DIRECT_LOGIN_CTA_RE = /sign in to (continue|read)|log in to (continue|read)|subscribe to (continue|read|keep)|以继续阅读|登录后查看|订阅后查看|登录以继续|登录后可继续阅读/i;
function checkDirectBlock(html, url) {
if (!html) return;
const head = html.substring(0, 8000);
if (DIRECT_CHALLENGE_TITLE_RE.test(head) || DIRECT_CHALLENGE_RE.test(head)) {
// 计算 <p> 文本总长度作为正文门控 (与 login-wall 共用)。
const paras = head.match(/<p[^>]*>([\s\S]*?)<\/p>/gi) || [];
const pTextLen = paras.reduce((n, p) => n + p.replace(/<[^>]+>/g, '').trim().length, 0);
// 挑战页标题 → 高置信度，直接阻断。
if (DIRECT_CHALLENGE_TITLE_RE.test(head)) {
throw new HumanInterventionError('人机验证/反爬挑战页面', { kind: 'captcha', url });
}
// 挑战正文文案 + 短 <p> 内容 → 挑战页。
// 长 <p> 内容 + 挑战文案 → 可能是真实文章中的 cookie 横幅文案，不阻断。
if (DIRECT_CHALLENGE_RE.test(head) && pTextLen < 400) {
throw new HumanInterventionError('人机验证/反爬挑战页面', { kind: 'captcha', url });
}
// Login wall: CTA present AND the extracted article body would be tiny. We
// approximate "tiny body" by counting <p> text length in the head slice.
if (DIRECT_LOGIN_CTA_RE.test(head)) {
const paras = head.match(/<p[^>]*>([\s\S]*?)<\/p>/gi) || [];
const textLen = paras.reduce((n, p) => n + p.replace(/<[^>]+>/g, '').trim().length, 0);
if (textLen < 400) {
throw new HumanInterventionError('检测到登录/订阅墙（正文极短且出现登录/订阅提示）', { kind: 'login', url });
}
// Login wall: CTA present AND the extracted article body would be tiny.
if (DIRECT_LOGIN_CTA_RE.test(head) && pTextLen < 400) {
throw new HumanInterventionError('检测到登录/订阅墙（正文极短且出现登录/订阅提示）', { kind: 'login', url });
}
}
......
skills/news-harvester/scripts/cdp-client.js
View file @ 6a313771
......@@ -246,6 +246,19 @@ async function openSession() {
ws.once('open', resolve);
ws.once('error', reject);
});
// Stealth: hide CDP automation artifacts before any navigation.
// Anti-bot systems (PerimeterX, Cloudflare Turnstile, etc.) check
// navigator.webdriver on page load. Page.addScriptToEvaluateOnNewDocument
// injects before any page script runs, so the override is in place by the
// time the anti-bot checker reads it. Non-fatal on older Chromium.
try {
await cdpSend(ws, 'Page.enable');
await cdpSend(ws, 'Page.addScriptToEvaluateOnNewDocument', {
source: 'Object.defineProperty(navigator,"webdriver",{get:()=>undefined});'
});
} catch (e) { /* non-fatal — older Chromium may not support it */ }
const closed = { value: false };
const close = async () => {
if (closed.value) return;
......
skills/news-harvester/scripts/ensure-chromium.js
View file @ 6a313771
......@@ -189,6 +189,10 @@ function spawnBrowser(bin, opts) {
'--no-default-browser-check',
'--disable-extensions',
'--disable-gpu',
// Hide the navigator.webdriver=true flag that CDP sets by default.
// Anti-bot systems (PerimeterX, Cloudflare) check this on page load.
// Combined with the stealth injection in cdp-client.js openSession().
'--disable-blink-features=AutomationControlled',
// Let Chromium auto-detect Wayland vs X11 on Linux. Without this it
// defaults to the X11 backend and crashes with "Missing X server or
// $DISPLAY" on Wayland-only sessions (common on Arch/GNOME/Sway). Harmless
......
skills/news-harvester/scripts/fetcher-helper.js
View file @ 6a313771
......@@ -260,8 +260,15 @@ function buildExtractExpr(spec) {
// requested count of *new* articles can still be satisfied — instead of being
// capped by the fixed `scrollSteps` lazy-load pass. The slice cap is widened
// to `max(maxLinks, desiredCount)` so the extra candidates aren't discarded.
async function fetchHomeLinks(url, spec, desiredCount) {
const { ws, close } = await openSession();
//
// Optional `session` (analyzer scripts): when provided, reuse an existing
// { ws, close } session instead of opening/closing a new tab per call. This
// avoids tab churn in serial workflows like preview.js / recipe-test.js where
// many pages are fetched one after another. When omitted, a new session is
// created and closed in the finally block (harvester behavior).
async function fetchHomeLinks(url, spec, desiredCount, session) {
const ownSession = !session;
const { ws, close } = session || await openSession();
try {
await navigateAndWait(ws, url, spec.waitMs);
await evaluateBrowserBlock(ws);
......@@ -282,21 +289,24 @@ async function fetchHomeLinks(url, spec, desiredCount) {
: (spec.maxLinks || 30);
return items.map(i => i.url).slice(0, cap);
} finally {
await close();
if (ownSession) await close();
}
}
// Extract a single article's content. Returns { title, date, authors, imgs, body }
// or null if evaluation yielded nothing.
async function fetchPage(url, spec) {
const { ws, close } = await openSession();
//
// Optional `session`: see fetchHomeLinks above.
async function fetchPage(url, spec, session) {
const ownSession = !session;
const { ws, close } = session || await openSession();
try {
await navigateAndWait(ws, url, spec.waitMs);
await evaluateBrowserBlock(ws);
const result = await cdpEval(ws, buildExtractExpr(spec));
return result ? JSON.parse(result) : null;
} finally {
await close();
if (ownSession) await close();
}
}
......
skills/news-source-analyzer/scripts/block-check.js
View file @ 6a313771
......@@ -4,10 +4,16 @@
*
* When a block is detected, a HumanInterventionError is thrown so harvest.js can
* exit immediately with a friendly message instead of silently archiving empty
* or garbage articles. The check is intentionally conservative: it only fires on
* high-confidence signals (challenge iframes/titles, auth-host redirects, or a
* login-wall CTA paired with a near-empty body) to avoid false positives on
* legitimate articles.
* or garbage articles.
*
* Detection philosophy — multi-signal + body-length gating:
* A real challenge/anti-bot page has almost no body text (the challenge
* widget replaces content). A real article page may embed passive risk-scoring
* iframes (invisible reCAPTCHA Enterprise, PerimeterX) or cookie-consent
* banners whose copy overlaps challenge phrases — but it also has thousands of
* chars of real article text. So weak signals (iframe, body-text) are only
* treated as blocking when the body is short; strong signals (challenge page
* title, dual weak signals) block regardless of body length.
*
* Two entry points:
* • Browser (CDP) sources — evaluateBrowserBlock(ws) runs an in-page probe via
......@@ -19,18 +25,30 @@
// Sent to the page as a Runtime.evaluate expression. Serialized from a real
// function via .toString() so regex backslashes survive without double-escaping.
// IMPORTANT: must be self-contained (no closure vars, no arrow/const/let) so
// .toString() produces valid JS for Runtime.evaluate.
function _browserBlockProbe() {
var url = location.href, host = location.hostname;
var title = (document.title || '').trim();
var bodyText = (document.body && document.body.innerText) ? document.body.innerText.substring(0, 4000) : '';
var bodyLen = bodyText.length;
// 1. CAPTCHA / anti-bot challenge — iframes from known challenge providers.
// Note: recaptcha/api2/aframe is a passive background iframe used by
// reCAPTCHA v3 for ad-fraud prevention — it does NOT block the user and is
// present on many news sites (Reuters, Nikkei, etc.). Only match the active
// challenge iframes (anchor/bframe/enterprise) that constitute a visible
// verification widget.
// 正文长度门控阈值。超过此值说明页面有实质内容，被动风控元素
// (invisible reCAPTCHA、cookie 横幅中的挑战文案) 不应判定为阻断性挑战。
// 真正的挑战/反爬页正文极短，不会超过这个阈值。
var BODY_GATE = 1500;
// 1. CAPTCHA / anti-bot challenge.
//
// challengeTitle — 挑战页标题 (Cloudflare "Just a moment..."、AWS "Access
// Denied" 等)。高置信度信号，直接阻断，不受正文门控影响。注意 "verify"
// 已收紧为 "verify you are human"，避免误判以 "Verify" 开头的文章标题。
var challengeTitle = /^(just a moment|attention required|verify you are human|checking your browser|are you human)/i.test(title)
|| /access denied|请输入验证码|安全验证/i.test(title);
// challengeIframe — 已知挑战提供商的 iframe。不再对 invisible reCAPTCHA
// 做逐 iframe 特殊判断：正文门控统一处理被动风控 iframe。
// recaptcha/api2/aframe 是 reCAPTCHA v3 的被动后台 iframe，不在匹配列表中。
var challengeIframe = Array.from(document.querySelectorAll('iframe[src]')).some(function (f) {
var s = f.src.toLowerCase();
return s.indexOf('recaptcha/api2/anchor') !== -1
......@@ -41,12 +59,17 @@ function _browserBlockProbe() {
|| s.indexOf('turnstile') !== -1
|| s.indexOf('arkoselabs') !== -1 || s.indexOf('funcaptcha') !== -1;
});
// Challenge page titles (Cloudflare "Just a moment...", "Access Denied", etc.).
var challengeTitle = /^(just a moment|attention required|verify|checking your browser|are you human)/i.test(title)
|| /access denied|请输入验证码|安全验证/i.test(title);
// Challenge body copy.
// challengeText — 挑战页正文文案。
var challengeText = /please verify (that )?you are human|verify you are human|unusual traffic from your computer|checking your browser before accessing|enable javascript and cookies to continue|our system has detected unusual traffic|请完成安全验证|进行人机验证/i.test(bodyText);
if (challengeIframe || challengeTitle || challengeText) {
// 多信号置信度模型:
// challengeTitle 命中 → 高置信度，直接阻断
// challengeIframe + challengeText 同时命中 → 双弱信号叠加为高置信度，直接阻断
// 单弱信号 (iframe 或 text) + 短正文 (< BODY_GATE) → 阻断 (挑战页无实质内容)
// 单弱信号 + 长正文 (≥ BODY_GATE) → 不阻断 (被动风控 iframe / cookie 横幅)
if (challengeTitle || (challengeIframe && challengeText)
|| ((challengeIframe || challengeText) && bodyLen < BODY_GATE)) {
return JSON.stringify({ blocked: true, kind: 'captcha', reason: '人机验证/反爬挑战页面', url: url, title: title });
}
......@@ -54,7 +77,11 @@ function _browserBlockProbe() {
// path. A real article page is never served from accounts.* / login.* or a
// /signin path, so this is a high-confidence signal regardless of body length.
var authHost = /^(accounts\.|login\.|signin\.|auth\.|sso\.|chkpt\.)/i.test(host);
var authPath = /\/(signin|sign-in|login|auth|account\/login|subscribe|sso)\b/i.test(location.pathname + location.search);
// authPath: 只匹配完整路径段 (前后为 / 或字符串边界)，避免误判
// /technology/sso-integration-guide 等含关键词子串的正常路径。
// 只检测 pathname，不检测 search —— query 参数中的 login/subscribe
// 不是重定向信号，由 path 3 (paywall CTA) 兜底。
var authPath = /(^|\/)(signin|sign-in|login|auth|account\/login|subscribe|sso)(\/|$)/i.test(location.pathname);
if (authHost || authPath) {
return JSON.stringify({ blocked: true, kind: 'login', reason: '页面被重定向到登录/认证页面（登录状态可能已失效）', url: url, title: title });
}
......@@ -106,24 +133,33 @@ async function evaluateBrowserBlock(ws) {
// String scan of raw HTML for challenge/login-wall signals. No DOM, so the
// signals are a subset of the browser probe — enough to catch a Cloudflare
// interstitial or a login redirect on an open site.
//
// Same body-gating principle as the browser probe: challenge title is a
// high-confidence signal (block immediately); challenge body text is gated by
// <p> content length so a cookie-banner phrase on a real article doesn't
// trigger a false positive.
const DIRECT_CHALLENGE_RE = /please verify (that )?you are human|verify you are human|unusual traffic from your computer|checking your browser before accessing|enable javascript and cookies to continue|our system has detected unusual traffic|请完成安全验证|进行人机验证/i;
const DIRECT_CHALLENGE_TITLE_RE = /<title[^>]*>(just a moment|attention required|verify|checking your browser|are you human|access denied|请输入验证码|安全验证)/i;
const DIRECT_CHALLENGE_TITLE_RE = /<title[^>]*>(just a moment|attention required|verify you are human|checking your browser|are you human|access denied|请输入验证码|安全验证)/i;
const DIRECT_LOGIN_CTA_RE = /sign in to (continue|read)|log in to (continue|read)|subscribe to (continue|read|keep)|以继续阅读|登录后查看|订阅后查看|登录以继续|登录后可继续阅读/i;
function checkDirectBlock(html, url) {
if (!html) return;
const head = html.substring(0, 8000);
if (DIRECT_CHALLENGE_TITLE_RE.test(head) || DIRECT_CHALLENGE_RE.test(head)) {
// 计算 <p> 文本总长度作为正文门控 (与 login-wall 共用)。
const paras = head.match(/<p[^>]*>([\s\S]*?)<\/p>/gi) || [];
const pTextLen = paras.reduce((n, p) => n + p.replace(/<[^>]+>/g, '').trim().length, 0);
// 挑战页标题 → 高置信度，直接阻断。
if (DIRECT_CHALLENGE_TITLE_RE.test(head)) {
throw new HumanInterventionError('人机验证/反爬挑战页面', { kind: 'captcha', url });
}
// 挑战正文文案 + 短 <p> 内容 → 挑战页。
// 长 <p> 内容 + 挑战文案 → 可能是真实文章中的 cookie 横幅文案，不阻断。
if (DIRECT_CHALLENGE_RE.test(head) && pTextLen < 400) {
throw new HumanInterventionError('人机验证/反爬挑战页面', { kind: 'captcha', url });
}
// Login wall: CTA present AND the extracted article body would be tiny. We
// approximate "tiny body" by counting <p> text length in the head slice.
if (DIRECT_LOGIN_CTA_RE.test(head)) {
const paras = head.match(/<p[^>]*>([\s\S]*?)<\/p>/gi) || [];
const textLen = paras.reduce((n, p) => n + p.replace(/<[^>]+>/g, '').trim().length, 0);
if (textLen < 400) {
throw new HumanInterventionError('检测到登录/订阅墙（正文极短且出现登录/订阅提示）', { kind: 'login', url });
}
// Login wall: CTA present AND the extracted article body would be tiny.
if (DIRECT_LOGIN_CTA_RE.test(head) && pTextLen < 400) {
throw new HumanInterventionError('检测到登录/订阅墙（正文极短且出现登录/订阅提示）', { kind: 'login', url });
}
}
......
skills/news-source-analyzer/scripts/cdp-client.js
View file @ 6a313771
......@@ -246,6 +246,19 @@ async function openSession() {
ws.once('open', resolve);
ws.once('error', reject);
});
// Stealth: hide CDP automation artifacts before any navigation.
// Anti-bot systems (PerimeterX, Cloudflare Turnstile, etc.) check
// navigator.webdriver on page load. Page.addScriptToEvaluateOnNewDocument
// injects before any page script runs, so the override is in place by the
// time the anti-bot checker reads it. Non-fatal on older Chromium.
try {
await cdpSend(ws, 'Page.enable');
await cdpSend(ws, 'Page.addScriptToEvaluateOnNewDocument', {
source: 'Object.defineProperty(navigator,"webdriver",{get:()=>undefined});'
});
} catch (e) { /* non-fatal — older Chromium may not support it */ }
const closed = { value: false };
const close = async () => {
if (closed.value) return;
......
skills/news-source-analyzer/scripts/ensure-chromium.js
View file @ 6a313771
......@@ -189,6 +189,10 @@ function spawnBrowser(bin, opts) {
'--no-default-browser-check',
'--disable-extensions',
'--disable-gpu',
// Hide the navigator.webdriver=true flag that CDP sets by default.
// Anti-bot systems (PerimeterX, Cloudflare) check this on page load.
// Combined with the stealth injection in cdp-client.js openSession().
'--disable-blink-features=AutomationControlled',
// Let Chromium auto-detect Wayland vs X11 on Linux. Without this it
// defaults to the X11 backend and crashes with "Missing X server or
// $DISPLAY" on Wayland-only sessions (common on Arch/GNOME/Sway). Harmless
......
skills/news-source-analyzer/scripts/fetcher-helper.js
View file @ 6a313771
......@@ -260,8 +260,15 @@ function buildExtractExpr(spec) {
// requested count of *new* articles can still be satisfied — instead of being
// capped by the fixed `scrollSteps` lazy-load pass. The slice cap is widened
// to `max(maxLinks, desiredCount)` so the extra candidates aren't discarded.
async function fetchHomeLinks(url, spec, desiredCount) {
const { ws, close } = await openSession();
//
// Optional `session` (analyzer scripts): when provided, reuse an existing
// { ws, close } session instead of opening/closing a new tab per call. This
// avoids tab churn in serial workflows like preview.js / recipe-test.js where
// many pages are fetched one after another. When omitted, a new session is
// created and closed in the finally block (harvester behavior).
async function fetchHomeLinks(url, spec, desiredCount, session) {
const ownSession = !session;
const { ws, close } = session || await openSession();
try {
await navigateAndWait(ws, url, spec.waitMs);
await evaluateBrowserBlock(ws);
......@@ -282,21 +289,24 @@ async function fetchHomeLinks(url, spec, desiredCount) {
: (spec.maxLinks || 30);
return items.map(i => i.url).slice(0, cap);
} finally {
await close();
if (ownSession) await close();
}
}
// Extract a single article's content. Returns { title, date, authors, imgs, body }
// or null if evaluation yielded nothing.
async function fetchPage(url, spec) {
const { ws, close } = await openSession();
//
// Optional `session`: see fetchHomeLinks above.
async function fetchPage(url, spec, session) {
const ownSession = !session;
const { ws, close } = session || await openSession();
try {
await navigateAndWait(ws, url, spec.waitMs);
await evaluateBrowserBlock(ws);
const result = await cdpEval(ws, buildExtractExpr(spec));
return result ? JSON.parse(result) : null;
} finally {
await close();
if (ownSession) await close();
}
}
......
skills/news-source-analyzer/scripts/preview.js
View file @ 6a313771
......@@ -35,6 +35,7 @@ function loadConfig() {
}
const { loadSpec, fetchPage, fetchHomeLinks } = require('./fetcher-helper');
const { openSession } = require('./cdp-client');
const { HumanInterventionError } = require('./block-check');
// Body preview length (matches harvest.js's PREVIEW_LEN)
......@@ -120,68 +121,78 @@ async function main() {
process.exit(1);
}
// Fetch homepage links
console.error(`\n📡 Fetching homepage: ${args.url}`);
let links;
// Open one shared session for the entire preview run (homepage + all articles).
// This avoids creating/closing a new tab per fetch — the analyzer is inherently
// serial, so one tab is enough and much less noisy than the old per-fetch churn.
const session = await openSession();
console.error(` Tab opened (shared session for all fetches)`);
try {
links = await fetchHomeLinks(args.url, spec);
} catch (e) {
if (e instanceof HumanInterventionError) {
console.error(`\n🚫 采集中止 — 需要人工介入: ${e.message}`);
process.exit(70);
}
console.error('Failed to fetch homepage:', e.message);
process.exit(1);
}
console.error(` Collected ${links.length} candidate links`);
const normalizedLinks = [...new Set(links.map(u => normalizeUrl(u)))];
console.error(` Unique: ${normalizedLinks.length} candidates`);
// Fetch each article (read-only)
const out = [];
for (let i = 0; i < normalizedLinks.length; i++) {
if (out.length >= args.count) break;
const url = normalizedLinks[i];
console.error(`\n[${i+1}/${normalizedLinks.length}] ${url}`);
let articleData;
// Fetch homepage links
console.error(`\n📡 Fetching homepage: ${args.url}`);
let links;
try {
articleData = await fetchPage(url, spec);
links = await fetchHomeLinks(args.url, spec, null, session);
} catch (e) {
if (e instanceof HumanInterventionError) {
console.error(`\n🚫 采集中止 — 需要人工介入: ${e.message}`);
process.exit(70);
}
console.error(` ✗ Fetch failed: ${e.message}`);
continue;
console.error('Failed to fetch homepage:', e.message);
process.exit(1);
}
if (!articleData || !articleData.title) { console.error(' ✗ No title, skip'); continue; }
if (articleData.isArticle === false) { console.error(' ✗ Not an article (og:type/body), skip'); continue; }
const category = getCategory(url, articleData.title);
const entry = {
title: articleData.title,
date: articleData.date,
authors: articleData.authors,
url,
category,
charCount: (articleData.body || '').length,
imgCount: (articleData.imgs || []).length,
imgs: (articleData.imgs || []).map(im => ({ alt: im.alt || '', src: im.src }))
};
if (args.full) {
entry.body = articleData.body || '';
} else {
entry.bodyPreview = (articleData.body || '').substring(0, PREVIEW_LEN);
console.error(` Collected ${links.length} candidate links`);
const normalizedLinks = [...new Set(links.map(u => normalizeUrl(u)))];
console.error(` Unique: ${normalizedLinks.length} candidates`);
// Fetch each article (read-only)
const out = [];
for (let i = 0; i < normalizedLinks.length; i++) {
if (out.length >= args.count) break;
const url = normalizedLinks[i];
console.error(`\n[${i+1}/${normalizedLinks.length}] ${url}`);
let articleData;
try {
articleData = await fetchPage(url, spec, session);
} catch (e) {
if (e instanceof HumanInterventionError) {
console.error(`\n🚫 采集中止 — 需要人工介入: ${e.message}`);
process.exit(70);
}
console.error(` ✗ Fetch failed: ${e.message}`);
continue;
}
if (!articleData || !articleData.title) { console.error(' ✗ No title, skip'); continue; }
if (articleData.isArticle === false) { console.error(' ✗ Not an article (og:type/body), skip'); continue; }
const category = getCategory(url, articleData.title);
const entry = {
title: articleData.title,
date: articleData.date,
authors: articleData.authors,
url,
category,
charCount: (articleData.body || '').length,
imgCount: (articleData.imgs || []).length,
imgs: (articleData.imgs || []).map(im => ({ alt: im.alt || '', src: im.src }))
};
if (args.full) {
entry.body = articleData.body || '';
} else {
entry.bodyPreview = (articleData.body || '').substring(0, PREVIEW_LEN);
}
out.push(entry);
console.error(` ✓ ${articleData.title} (${entry.charCount} chars, ${entry.imgCount} imgs)`);
}
out.push(entry);
console.error(` ✓ ${articleData.title} (${entry.charCount} chars, ${entry.imgCount} imgs)`);
}
process.stdout.write(JSON.stringify({
url: args.url, recipe: recipePath, preview: true, full: args.full,
count: out.length, articles: out
}, null, 2));
console.error(`\n✅ Preview: ${out.length} articles (no files written)`);
process.stdout.write(JSON.stringify({
url: args.url, recipe: recipePath, preview: true, full: args.full,
count: out.length, articles: out
}, null, 2));
console.error(`\n✅ Preview: ${out.length} articles (no files written)`);
} finally {
await session.close();
}
}
main().catch(e => { console.error('Fatal:', e.message); process.exit(1); });
skills/news-source-analyzer/scripts/recipe-test.js
View file @ 6a313771
......@@ -33,6 +33,7 @@ function loadConfig() {
}
const { loadSpec, fetchPage, fetchHomeLinks } = require('./fetcher-helper');
const { openSession } = require('./cdp-client');
// ── Assertions ───────────────────────────────────────────────────────────────
......@@ -99,61 +100,69 @@ async function main() {
process.exit(1);
}
// ── 1. Article URLs must validate as articles ─────────────────────────────
console.error(`\n📄 Article fixtures (must be isArticle=true, body ≥ ${spec.bodyMinChars || 500} chars):`);
for (const url of (fixtures.articles || [])) {
try {
const data = await fetchPage(url, spec);
if (!data) { assert(false, `article: ${url}`, 'fetchPage returned null'); continue; }
assert(data.isArticle !== false, `article: ${data.title ? data.title.slice(0,60) : url}`,
`isArticle=${data.isArticle} og=${data.ogType} ld=${data.ldType} body=${data.bodyChars}c/${data.bodyParagraphs}p`);
} catch (e) {
assert(false, `article: ${url}`, e.message);
}
}
// Open one shared session for the entire test run (all article/section/homepage
// fetches). Avoids creating/closing a new tab per fetch.
const session = await openSession();
// ── 2. Section/category URLs must NOT validate as articles ────────────────
// Only meaningful when the recipe uses validateArticle; otherwise this group
// is skipped (no isArticle signal to assert against).
if (spec.validateArticle) {
console.error(`\n🗂️ Section fixtures (must be isArticle=false):`);
for (const url of (fixtures.sections || [])) {
try {
// ── 1. Article URLs must validate as articles ─────────────────────────────
console.error(`\n📄 Article fixtures (must be isArticle=true, body ≥ ${spec.bodyMinChars || 500} chars):`);
for (const url of (fixtures.articles || [])) {
try {
const data = await fetchPage(url, spec);
if (!data) { assert(false, `section: ${url}`, 'fetchPage returned null'); continue; }
assert(data.isArticle === false, `section: ${url}`,
const data = await fetchPage(url, spec, session);
if (!data) { assert(false, `article: ${url}`, 'fetchPage returned null'); continue; }
assert(data.isArticle !== false, `article: ${data.title ? data.title.slice(0,60) : url}`,
`isArticle=${data.isArticle} og=${data.ogType} ld=${data.ldType} body=${data.bodyChars}c/${data.bodyParagraphs}p`);
} catch (e) {
assert(false, `section: ${url}`, e.message);
assert(false, `article: ${url}`, e.message);
}
}
}
// ── 3. Homepage link collection: count + no excluded URLs ─────────────────
console.error(`\n🔗 Homepage links (≥ ${fixtures.minLinks || 10}, none matching excludeUrlPattern):`);
try {
const links = await fetchHomeLinks(homepageUrl, spec);
assert(links.length >= (fixtures.minLinks || 10),
`homepage yields ≥ ${fixtures.minLinks || 10} links`, `got ${links.length}`);
if (spec.excludeUrlPattern) {
const exRe = new RegExp(spec.excludeUrlPattern);
const leaked = links.filter(u => exRe.test(u));
assert(leaked.length === 0, `no links match excludeUrlPattern (${spec.excludeUrlPattern})`,
leaked.length ? leaked.slice(0,3).join(' ; ') : '');
// ── 2. Section/category URLs must NOT validate as articles ────────────────
// Only meaningful when the recipe uses validateArticle; otherwise this group
// is skipped (no isArticle signal to assert against).
if (spec.validateArticle) {
console.error(`\n🗂️ Section fixtures (must be isArticle=false):`);
for (const url of (fixtures.sections || [])) {
try {
const data = await fetchPage(url, spec, session);
if (!data) { assert(false, `section: ${url}`, 'fetchPage returned null'); continue; }
assert(data.isArticle === false, `section: ${url}`,
`isArticle=${data.isArticle} og=${data.ogType} ld=${data.ldType} body=${data.bodyChars}c/${data.bodyParagraphs}p`);
} catch (e) {
assert(false, `section: ${url}`, e.message);
}
}
}
// ── 3. Homepage link collection: count + no excluded URLs ─────────────────
console.error(`\n🔗 Homepage links (≥ ${fixtures.minLinks || 10}, none matching excludeUrlPattern):`);
try {
const links = await fetchHomeLinks(homepageUrl, spec, null, session);
assert(links.length >= (fixtures.minLinks || 10),
`homepage yields ≥ ${fixtures.minLinks || 10} links`, `got ${links.length}`);
if (spec.excludeUrlPattern) {
const exRe = new RegExp(spec.excludeUrlPattern);
const leaked = links.filter(u => exRe.test(u));
assert(leaked.length === 0, `no links match excludeUrlPattern (${spec.excludeUrlPattern})`,
leaked.length ? leaked.slice(0,3).join(' ; ') : '');
}
// Also verify every link matches urlPattern (sanity: the recipe isn't
// letting unrelated hrefs through).
const urlRe = new RegExp(spec.urlPattern);
const offPattern = links.filter(u => !urlRe.test(u));
assert(offPattern.length === 0, 'all links match urlPattern',
offPattern.length ? offPattern.slice(0,3).join(' ; ') : '');
} catch (e) {
assert(false, 'homepage link collection', e.message);
}
// Also verify every link matches urlPattern (sanity: the recipe isn't
// letting unrelated hrefs through).
const urlRe = new RegExp(spec.urlPattern);
const offPattern = links.filter(u => !urlRe.test(u));
assert(offPattern.length === 0, 'all links match urlPattern',
offPattern.length ? offPattern.slice(0,3).join(' ; ') : '');
} catch (e) {
assert(false, 'homepage link collection', e.message);
}
// ── Summary ───────────────────────────────────────────────────────────────
console.error(`\n${failed === 0 ? '✅' : '❌'} ${passed} passed, ${failed} failed`);
process.exit(failed === 0 ? 0 : 1);
// ── Summary ───────────────────────────────────────────────────────────────
console.error(`\n${failed === 0 ? '✅' : '❌'} ${passed} passed, ${failed} failed`);
process.exit(failed === 0 ? 0 : 1);
} finally {
await session.close();
}
}
main().catch(e => { console.error('Fatal:', e.message); process.exit(1); });
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment