feat: support manage tab in cdp mode

.gitignore

@@ -19,3 +19,4 @@ helpers/
 
 # IDE / agent workspace files
 .zcode/
+AGENTS.md

skills/news-harvester/SKILL.md

@@ -89,6 +89,8 @@ If this returns article data, setup is complete.
 | `chromium.headless` | Optional | Headless (`true`, default) or windowed (`false`). |
 | `chromium.executablePath` | Optional | Full path to browser binary. `null` = auto-discover. |
 | `chromium.userDataDir` | Optional | Browser profile dir for login state. |
+| `chromium.maxConcurrentTabs` | Optional | Global max concurrent browser tabs across all harvest processes (default `5`). Enforced via CDP `Target.getTargets` browser self-check. |
+| `sources[].concurrency` | Optional | Per-source concurrent article fetch count (default `1` = serial). Capped by `chromium.maxConcurrentTabs`. See [Source Management](references/source-management.md#concurrency). |
 
 #### Dedicated profile (login state)
 

skills/news-harvester/references/config-template.json

@@ -15,7 +15,8 @@
     "cdpPort": 9222,
     "headless": true,
     "executablePath": null,
-    "userDataDir": null
+    "userDataDir": null,
+    "maxConcurrentTabs": 5
   },
   "sources": []
 }

skills/news-harvester/references/source-management.md

@@ -12,7 +12,7 @@ reuters-world  路透社世界         browser   ✅        ✅ enabled
 theconversation The Conversation  direct    —         ❌ disabled
 ```
 
-RECIPE ✅ = `helpers/<id>.md` exists (spec-driven fetcher); — = legacy fetcher.
+RECIPE ✅ = `helpers/<id>.md` exists (spec-driven fetcher-helper); — = no recipe (direct sources only, uses fetcher-direct).
 
 ## Edit Source
 
@@ -20,7 +20,17 @@ RECIPE ✅ = `helpers/<id>.md` exists (spec-driven fetcher); — = legacy fetche
 node scripts/source-manage.js edit <id> --name "New Name" --url https://... --method browser
 ```
 
-Editable fields: `name`, `homepageUrl`, `method`, `articleUrlPattern`, `vaultFolder`, `enabled`.
+Editable fields: `name`, `homepageUrl`, `method`, `articleUrlPattern`, `vaultFolder`, `enabled`, `concurrency`.
+
+### Concurrency
+
+Each source has a `concurrency` field (default `1` = serial). Set it to `2` or higher to fetch multiple articles in parallel using separate browser tabs:
+
+```bash
+node scripts/source-manage.js edit <id> --concurrency 3
+```
+
+The global tab limit (`chromium.maxConcurrentTabs`, default `5`) is enforced by the browser itself via CDP `Target.getTargets` — even when multiple harvest processes run simultaneously, the total open tabs never stays above the limit. A source's `concurrency` is silently capped to the global limit.
 
 ## Enable / Disable Source
 
@@ -50,7 +60,7 @@ A `.nhsource.json` is plain JSON:
 | `format` | `"news-harvester-source"` (install validates this) |
 | `version` | `1` |
 | `exportedAt` | ISO timestamp |
-| `source` | `{ id, name, homepageUrl, method, vaultFolder }` — the config entry, minus `vaultPath`, `enabled`, and `articleUrlPattern` |
+| `source` | `{ id, name, homepageUrl, method, vaultFolder }` — the config entry, minus `vaultPath`, `enabled`, `articleUrlPattern`, and `concurrency` (all set by install) |
 | `files` | Map of relative path → file content. Includes `helpers/<id>.md` (the recipe) and `helpers/<id>.fixtures.json` (if present). |
 
 ### Install behavior

skills/news-harvester/scripts/block-check.js

@@ -11,7 +11,7 @@
  *
  * Two entry points:
  *   • Browser (CDP) sources — evaluateBrowserBlock(ws) runs an in-page probe via
- *     cdpEval and throws if blocked. Used by fetcher-helper.js / fetcher-browser.js.
+ *     cdpEval and throws if blocked. Used by fetcher-helper.js.
  *   • Direct HTTP sources — checkDirectBlock(html, url) does a string scan of the
  *     raw HTML. Used by fetcher-direct.js (open sites rarely need login, but a
  *     Cloudflare-style challenge can still appear).
@@ -26,9 +26,17 @@ function _browserBlockProbe() {
   var bodyLen = bodyText.length;
 
   // 1. CAPTCHA / anti-bot challenge — iframes from known challenge providers.
+  // Note: recaptcha/api2/aframe is a passive background iframe used by
+  // reCAPTCHA v3 for ad-fraud prevention — it does NOT block the user and is
+  // present on many news sites (Reuters, Nikkei, etc.). Only match the active
+  // challenge iframes (anchor/bframe/enterprise) that constitute a visible
+  // verification widget.
   var challengeIframe = Array.from(document.querySelectorAll('iframe[src]')).some(function (f) {
     var s = f.src.toLowerCase();
-    return s.indexOf('recaptcha') !== -1 || s.indexOf('hcaptcha') !== -1
+    return s.indexOf('recaptcha/api2/anchor') !== -1
+      || s.indexOf('recaptcha/api2/bframe') !== -1
+      || s.indexOf('recaptcha/enterprise') !== -1
+      || s.indexOf('hcaptcha') !== -1
       || s.indexOf('challenges.cloudflare.com') !== -1
       || s.indexOf('turnstile') !== -1
       || s.indexOf('arkoselabs') !== -1 || s.indexOf('funcaptcha') !== -1;

skills/news-harvester/scripts/cdp-client.js

 /**
  * cdp-client.js — Shared Chrome DevTools Protocol primitives.
  *
- * Previously every CDP caller (fetcher-browser.js) hand-rolled the same
- * WebSocket boilerplate: open connection, send a command with an id, attach a
- * message listener matching that id, parse the result. That pattern was
- * duplicated 4× and the Page.enable call didn't even check the id (resolving
- * on the first message of any kind). This module is the single source of truth
- * for those operations so fetcher-helper.js (and a future fetcher-browser
- * refactor) can build on top.
+ * This module is the single source of truth for CDP operations used by
+ * fetcher-helper.js and the analyzer scripts (inspect-source / preview /
+ * recipe-test). It provides two layers:
+ *
+ *   1. Browser-level target management — a long-lived WebSocket to the
+ *      browser's own devtools endpoint (webSocketDebuggerUrl from /json/version),
+ *      used for Target.createTarget / Target.closeTarget / Target.getTargets.
+ *      This is what enables multi-tab concurrency: every openSession() call
+ *      creates a fresh page target instead of reusing the browser's first tab,
+ *      so concurrent fetchers navigate independent tabs without clobbering
+ *      each other.
+ *
+ *   2. Per-tab CDP commands — cdpSend / cdpEval / navigateAndWait /
+ *      scrollAndCollect operate on a specific tab's WebSocket (returned by
+ *      openSession). These are unchanged from the original single-tab design.
+ *
+ * Global concurrency control: openSession() checks Target.getTargets before
+ * creating a new tab. If the number of page-type targets is at or above the
+ * configured limit (chromium.maxConcurrentTabs, default 5), it polls every
+ * 200 ms until a slot frees up. This is a "browser self-check" strategy — all
+ * processes sharing the same Chromium instance see the same target list, so
+ * the limit is enforced globally across concurrent harvest runs. A brief
+ * TOCTOU over-limit of 1-2 tabs is possible and harmless.
  *
  * Lifecycle (browser launch/discovery) stays in ensure-chromium.js — this
- * module only handles per-tab CDP commands, assuming a browser is already up.
+ * module only handles CDP commands, assuming a browser is already up.
  */
 
 const http = require('http');
@@ -26,6 +42,20 @@ const TEMPLATE_PATH = path.join(SKILL_DIR, 'references', 'config-template.json')
 
 function sleep(ms) { return new Promise(r => setTimeout(r, ms)); }
 
+// Small HTTP GET helper returning parsed JSON. Used for /json/version.
+function httpGetJson(url) {
+  return new Promise((resolve, reject) => {
+    http.get(url, (res) => {
+      let data = '';
+      res.on('data', d => data += d);
+      res.on('end', () => {
+        try { resolve(JSON.parse(data)); }
+        catch (e) { reject(new Error(`Bad JSON from ${url}: ${e.message}`)); }
+      });
+    }).on('error', reject);
+  });
+}
+
 // Monotonic id for CDP commands. Per-tab sessions are sequential, so a simple
 // counter is fine and avoids the random-id collisions the old code risked.
 let _msgId = 0;
@@ -33,7 +63,6 @@ function nextId() { return ++_msgId; }
 
 // ── Port resolution ──────────────────────────────────────────────────────────
 // Precedence: env override > config.chromium.cdpPort > default 9222.
-// (Mirrors the old private resolveCdpPort in fetcher-browser.js.)
 function resolveCdpPort() {
   if (process.env.CDP_PORT_OVERRIDE) return parseInt(process.env.CDP_PORT_OVERRIDE, 10);
   try {
@@ -82,16 +111,156 @@ function getWsUrl() {
   });
 }
 
+// ── Browser-level target management ──────────────────────────────────────────
+//
+// A long-lived WebSocket to the browser's own devtools endpoint (not a page
+// tab). Used for Target.createTarget / Target.closeTarget / Target.getTargets
+// so that each openSession() call gets its own independent tab — the basis for
+// multi-tab concurrency. Multiple Node processes may each hold their own
+// browser-level connection; CDP supports multiple clients on the same browser
+// target, each with an independent message-id space.
+
+let _browserWs = null;
+let _browserWsPromise = null;
+let _browserWsRefs = 0;   // active sessions holding a reference to the browser WS
+
+async function getBrowserWs() {
+  if (_browserWs && _browserWs.readyState === WebSocket.OPEN) return _browserWs;
+  if (_browserWsPromise) return _browserWsPromise;
+  _browserWsPromise = (async () => {
+    const port = resolveCdpPort();
+    const data = await httpGetJson(`http://localhost:${port}/json/version`);
+    const wsUrl = data.webSocketDebuggerUrl;
+    if (!wsUrl) throw new Error('CDP /json/version returned no webSocketDebuggerUrl');
+    const ws = new WebSocket(wsUrl);
+    await new Promise((res, rej) => {
+      ws.once('open', res);
+      ws.once('error', rej);
+    });
+    // Auto-recovery: if the socket drops, clear the cache so the next call
+    // reconnects.
+    ws.on('close', () => { _browserWs = null; });
+    ws.on('error', () => { _browserWs = null; });
+    _browserWs = ws;
+    _browserWsPromise = null;
+    return ws;
+  })();
+  return _browserWsPromise;
+}
+
+// Reference counting for the browser-level WebSocket.
+//
+// The browser WS is a long-lived cached connection used for Target.* commands
+// (createTab / closeTab / countPageTabs). If it stays open after all sessions
+// are closed, its TCP socket keeps the Node.js event loop alive — the process
+// hangs forever after completing its work.
+//
+// openSession() calls refBrowserWs() to increment the counter; close() calls
+// unrefBrowserWs() to decrement. When the counter reaches zero, we proactively
+// close the browser WS, removing the last handle so the process can exit
+// naturally. If a new session opens later, getBrowserWs() reconnects.
+//
+// Both the browser WS and per-tab WS stay fully ref'd (Node.js default) during
+// active use — this is essential because cdpSend() needs the event loop alive
+// to receive CDP responses. We only close the browser WS when NO sessions are
+// active, so there's never a risk of closing it mid-command.
+function refBrowserWs() { _browserWsRefs++; }
+
+function unrefBrowserWs() {
+  _browserWsRefs = Math.max(0, _browserWsRefs - 1);
+  if (_browserWsRefs === 0 && _browserWs) {
+    try { _browserWs.close(); } catch (e) { /* best-effort */ }
+    _browserWs = null;
+  }
+}
+
+// ── Concurrency control (global, browser self-check) ─────────────────────────
+
+const MAX_TABS_DEFAULT = 5;
+const TAB_POLL_INTERVAL = 200;   // ms between Target.getTargets polls
+
+function getMaxConcurrentTabs() {
+  try {
+    const cfgPath = fs.existsSync(CONFIG_PATH) ? CONFIG_PATH : TEMPLATE_PATH;
+    const cfg = JSON.parse(fs.readFileSync(cfgPath, 'utf8'));
+    return (cfg.chromium && cfg.chromium.maxConcurrentTabs) || MAX_TABS_DEFAULT;
+  } catch (e) { return MAX_TABS_DEFAULT; }
+}
+
+// Count current page-type targets via the browser-level WS.
+async function countPageTabs() {
+  const ws = await getBrowserWs();
+  const result = await cdpSend(ws, 'Target.getTargets');
+  return (result.targetInfos || []).filter(t => t.type === 'page').length;
+}
+
+// Block until the number of open page tabs is below the configured limit.
+// TOCTOU note: the check and the subsequent createTab are not atomic, so under
+// heavy multi-process contention the tab count may briefly exceed the limit by
+// 1-2. This is harmless — Chromium handles a few extra tabs, and they drain as
+// closeTab calls land.
+async function waitForTabSlot() {
+  const maxTabs = getMaxConcurrentTabs();
+  for (;;) {
+    if (await countPageTabs() < maxTabs) return;
+    await sleep(TAB_POLL_INTERVAL);
+  }
+}
+
+// Create a new page tab (about:blank) and return its targetId + wsUrl.
+async function createTab() {
+  const ws = await getBrowserWs();
+  const result = await cdpSend(ws, 'Target.createTarget', { url: 'about:blank' });
+  const targetId = result.targetId;
+  const port = resolveCdpPort();
+  return { targetId, wsUrl: `ws://localhost:${port}/devtools/page/${targetId}` };
+}
+
+// Close a page tab by targetId. Best-effort — errors are swallowed because the
+// tab may already be gone (e.g. site-triggered navigation closed it).
+async function closeTab(targetId) {
+  try {
+    const ws = await getBrowserWs();
+    await cdpSend(ws, 'Target.closeTarget', { targetId });
+  } catch (e) { /* tab may already be closed */ }
+}
+
 // ── Session management ───────────────────────────────────────────────────────
 
-// Open a WebSocket to the page target and resolve once connected.
-// Returns { ws, close }. The caller owns the lifecycle (close when done).
-function openSession() {
-  return getWsUrl().then(wsUrl => new Promise((resolve, reject) => {
+// Open a new browser tab, connect to it, and return { ws, close, targetId }.
+// The caller owns the lifecycle — it must call close() when done (which closes
+// both the WebSocket and the tab itself). This is the basis for multi-tab
+// concurrency: each call gets a fresh, isolated tab, so concurrent fetchers
+// never clobber each other's navigation.
+//
+// Before creating a tab, waitForTabSlot() enforces the global concurrency limit
+// (chromium.maxConcurrentTabs) via a Target.getTargets browser self-check. This
+// makes the limit global across all processes sharing the Chromium instance.
+async function openSession() {
+  await waitForTabSlot();
+  refBrowserWs();   // keep browser WS alive for the lifetime of this session
+  try {
+    const { targetId, wsUrl } = await createTab();
     const ws = new WebSocket(wsUrl);
-    ws.on('open', () => resolve({ ws, close: () => ws.close() }));
-    ws.on('error', reject);
-  }));
+    await new Promise((resolve, reject) => {
+      ws.once('open', resolve);
+      ws.once('error', reject);
+    });
+    const closed = { value: false };
+    const close = async () => {
+      if (closed.value) return;
+      closed.value = true;
+      ws.close();
+      await closeTab(targetId);   // must await so process.exit doesn't kill the tab
+      unrefBrowserWs();           // may close browser WS if this was the last session
+    };
+    return { ws, close, targetId };
+  } catch (e) {
+    // If session creation failed (e.g. tab creation error), release the ref
+    // so the browser WS can be cleaned up.
+    unrefBrowserWs();
+    throw e;
+  }
 }
 
 // ── Command sending ──────────────────────────────────────────────────────────
@@ -229,5 +398,12 @@ module.exports = {
   cdpSend,
   cdpEval,
   navigateAndWait,
-  scrollAndCollect
+  scrollAndCollect,
+  // Browser-level target management (multi-tab concurrency)
+  getBrowserWs,
+  countPageTabs,
+  waitForTabSlot,
+  createTab,
+  closeTab,
+  getMaxConcurrentTabs
 };

skills/news-harvester/scripts/ensure-chromium.js

@@ -256,6 +256,41 @@ function looksLikeDisplayFailure(log) {
   return /Missing X server|\$DISPLAY|platform failed to initialize|ozone/i.test(log);
 }
 
+// ── Startup-tab cleanup ─────────────────────────────────────────────────────
+// Chromium opens an initial tab (about:blank or chrome://newtab) on launch.
+// With the multi-tab architecture, openSession() creates its own tabs, so this
+// startup tab is an orphan that wastes a tab slot and inflates Target.getTargets
+// counts. Close it via the HTTP /json/close/<id> endpoint (no WebSocket needed).
+// Only closes about:blank / chrome://newtab — never touches real harvest tabs.
+const STARTUP_TAB_URLS = new Set(['about:blank', 'chrome://newtab/', 'chrome://newtab']);
+
+function closeStartupTabs(port) {
+  return new Promise((resolve) => {
+    // agent: false → don't pool sockets in the global HTTP agent. Without this
+    // the keep-alive sockets linger and keep the Node event loop alive, hanging
+    // ensure-chromium.js (and any process that calls ensureChromium()).
+    http.get(`http://localhost:${port}/json`, { agent: false }, (res) => {
+      let data = '';
+      res.on('data', (d) => (data += d));
+      res.on('end', () => {
+        try {
+          const tabs = JSON.parse(data);
+          const orphans = tabs.filter(t => t.type === 'page' && STARTUP_TAB_URLS.has(t.url));
+          if (orphans.length === 0) return resolve(0);
+          let remaining = orphans.length;
+          for (const tab of orphans) {
+            http.get(`http://localhost:${port}/json/close/${tab.id}`, { agent: false }, () => {
+              if (--remaining === 0) resolve(orphans.length);
+            }).on('error', () => {
+              if (--remaining === 0) resolve(orphans.length);
+            });
+          }
+        } catch (e) { resolve(0); }
+      });
+    }).on('error', () => resolve(0));
+  });
+}
+
 // ── Main ────────────────────────────────────────────────────────────────────
 
 async function ensureChromium(config, portOverride) {
@@ -264,6 +299,7 @@ async function ensureChromium(config, portOverride) {
   // 1. Already up? (idempotent — safe to call every run)
   try {
     const v = await cdpVersion(opts.cdpPort, 1500);
+    await closeStartupTabs(opts.cdpPort);   // clean orphan tabs from prior runs
     return { port: opts.cdpPort, alreadyRunning: true, browser: v.Browser };
   } catch (e) { /* not up yet, fall through to launch */ }
 
@@ -280,6 +316,7 @@ async function ensureChromium(config, portOverride) {
   // 4. Wait for readiness
   try {
     const v = await waitForCdp(opts.cdpPort, 30000, 500);
+    await closeStartupTabs(opts.cdpPort);   // close the initial about:blank tab
     return { port: opts.cdpPort, pid, browser: v.Browser, bin, logPath };
   } catch (e) {
     const logTail = tailFile(logPath);
@@ -527,6 +564,7 @@ if (require.main === module) {
         console.log(`   binary: ${r.bin}`);
         console.log(`   log:    ${r.logPath}`);
       }
+      process.exit(0);
     } catch (e) {
       console.error(`❌ ${e.message}`);
       process.exit(1);

skills/news-harvester/scripts/fetcher-browser.js

-/**
- * fetcher-browser.js
- * Fetch article content using Chromium CDP (for JS-heavy sites like Reuters)
- * Usage: node fetcher-browser.js <url> [waitMs]
- */
-
-const http = require('http');
-const fs = require('fs');
-const path = require('path');
-const WebSocket = require('ws');
-const { evaluateBrowserBlock } = require('./block-check');
-
-const SKILL_DIR = path.dirname(__dirname);
-const CONFIG_PATH = path.join(SKILL_DIR, 'config.json');
-const TEMPLATE_PATH = path.join(SKILL_DIR, 'references', 'config-template.json');
-
-// Resolve the CDP port lazily so config.json changes are picked up per run.
-// Precedence: env override > config.chromium.cdpPort > default 9222.
-function resolveCdpPort() {
-  if (process.env.CDP_PORT_OVERRIDE) return parseInt(process.env.CDP_PORT_OVERRIDE, 10);
-  try {
-    const cfgPath = fs.existsSync(CONFIG_PATH) ? CONFIG_PATH : TEMPLATE_PATH;
-    const cfg = JSON.parse(fs.readFileSync(cfgPath, 'utf8'));
-    if (cfg.chromium && cfg.chromium.cdpPort) return parseInt(cfg.chromium.cdpPort, 10);
-  } catch (e) { /* fall through to default */ }
-  return 9222;
-}
-
-// Retry the /json endpoint a few times: right after ensure-chromium spawns the
-// browser, the port may accept connections a beat before /json responds.
-function getWsUrl() {
-  const port = resolveCdpPort();
-  const MAX_ATTEMPTS = 6;
-  const STEP_MS = 500;
-  return new Promise((resolve, reject) => {
-    let attempt = 0;
-    const tryOnce = () => {
-      attempt++;
-      http.get(`http://localhost:${port}/json`, (res) => {
-        let data = '';
-        res.on('data', d => data += d);
-        res.on('end', () => {
-          try {
-            const tabs = JSON.parse(data);
-            // 优先找 page 类型的 tab，避免拿到 iframe/service_worker
-            const tab = tabs.find(t => t.type === 'page') || tabs[0];
-            if (!tab || !tab.webSocketDebuggerUrl) {
-              return reject(new Error(`CDP /json returned no page tab on :${port}`));
-            }
-            resolve(tab.webSocketDebuggerUrl);
-          } catch (e) {
-            if (attempt < MAX_ATTEMPTS) return setTimeout(tryOnce, STEP_MS);
-            reject(e);
-          }
-        });
-      }).on('error', (e) => {
-        if (attempt < MAX_ATTEMPTS) return setTimeout(tryOnce, STEP_MS);
-        reject(new Error(`Cannot reach CDP on :${port} after ${MAX_ATTEMPTS} tries (${e.message}). Run: node scripts/ensure-chromium.js`));
-      });
-    };
-    tryOnce();
-  });
-}
-
-async function sleep(ms) { return new Promise(r => setTimeout(r, ms)); }
-
-async function fetchPage(url, waitMs = 12000) {
-  const wsUrl = await getWsUrl();
-  const ws = new WebSocket(wsUrl);
-  await new Promise((resolve, reject) => { ws.on('open', resolve); ws.on('error', reject); });
-
-  // Enable Page events
-  await new Promise(resolve => {
-    ws.send(JSON.stringify({ id: 1, method: 'Page.enable' }));
-    ws.once('message', () => resolve());
-  });
-
-  // Navigate + wait for load
-  let loaded = false;
-  const loadPromise = new Promise(resolve => {
-    const handler = (raw) => {
-      const msg = JSON.parse(raw.toString());
-      if (msg.method === 'Page.loadEventFired' && !loaded) {
-        loaded = true;
-        ws.removeListener('message', handler);
-        resolve();
-      }
-    };
-    ws.on('message', handler);
-  });
-  ws.send(JSON.stringify({ id: 2, method: 'Page.navigate', params: { url } }));
-  await Promise.race([loadPromise, sleep(25000)]);
-  await sleep(waitMs); // wait for React/JS to fully render
-
-  // Human-verification / login-state-loss guard. Throws HumanInterventionError
-  // (propagates to harvest.js, which exits with a friendly message) before we
-  // try to extract content from what might be a challenge or login page.
-  await evaluateBrowserBlock(ws);
-
-  // Extract content
-  const result = await new Promise((resolve) => {
-    const id = Math.floor(Math.random() * 100000) + 90000;
-    const handler = (raw) => {
-      const msg = JSON.parse(raw.toString());
-      if (msg.id === id) {
-        ws.removeListener('message', handler);
-        resolve(msg.result && msg.result.result && msg.result.result.value);
-      }
-    };
-    ws.on('message', handler);
-    ws.send(JSON.stringify({ id, method: 'Runtime.evaluate', params: {
-      expression: `(function(){
-        var title = document.title.replace(/ \\| Reuters| \\| The Conversation| - Bloomberg$/g,'').trim();
-        var dateEl = document.querySelector('time[datetime]');
-        var date = dateEl ? dateEl.getAttribute('datetime') : '';
-        var authors = Array.from(document.querySelectorAll('[rel="author"], [class*="author"] a'))
-          .map(a => a.innerText.trim()).filter(Boolean)
-          .filter((v,i,a) => a.indexOf(v)===i).join(', ');
-        var imgs = Array.from(document.querySelectorAll('img')).map(img => ({
-          src: img.src,
-          alt: img.alt || '',
-          width: img.naturalWidth || img.width || 0
-        })).filter(i => i.src && i.src.startsWith('http') && i.width > 200
-          && !i.src.includes('logo') && !i.src.includes('icon') && !i.src.includes('avatar')
-          && !i.src.includes('profile')).slice(0, 6);
-        var body = '';
-        var selectors = [
-          '[data-testid^="paragraph"]',
-          '[class*="articleBodyContent"] p',
-          '[class*="article-body"] p',
-          '[class*="ArticleBody"] p',
-          '[class*="story-body"] p',
-          'article p'
-        ];
-        for (var sel of selectors) {
-          var els = document.querySelectorAll(sel);
-          if (els.length >= 3) {
-            body = Array.from(els).map(e => e.innerText.trim()).filter(t => t.length > 40).join('\\n\\n');
-            break;
-          }
-        }
-        if (!body) {
-          var noise = ['Reporting by','Editing by','Access unmatched','Browse an unrivalled',
-            'Screen for heightened','Sign up here','Reuters, the news','All quotes delayed','See here for'];
-          body = Array.from(document.querySelectorAll('p'))
-            .map(e => e.innerText.trim())
-            .filter(t => t.length > 60 && !noise.some(n => t.startsWith(n)))
-            .join('\\n\\n');
-        }
-        return JSON.stringify({ title, date, authors, imgs, body: body.substring(0, 15000) });
-      })()`
-    }}));
-  });
-
-  ws.close();
-  return result ? JSON.parse(result) : null;
-}
-
-// Extract article links from homepage
-async function fetchHomeLinks(url, pattern, waitMs = 15000) {
-  const wsUrl = await getWsUrl();
-  const ws = new WebSocket(wsUrl);
-  await new Promise((resolve, reject) => { ws.on('open', resolve); ws.on('error', reject); });
-
-  await new Promise(resolve => {
-    ws.send(JSON.stringify({ id: 1, method: 'Page.enable' }));
-    ws.once('message', () => resolve());
-  });
-
-  let loaded = false;
-  const loadPromise = new Promise(resolve => {
-    const handler = (raw) => {
-      const msg = JSON.parse(raw.toString());
-      if (msg.method === 'Page.loadEventFired' && !loaded) {
-        loaded = true;
-        ws.removeListener('message', handler);
-        resolve();
-      }
-    };
-    ws.on('message', handler);
-  });
-  ws.send(JSON.stringify({ id: 2, method: 'Page.navigate', params: { url } }));
-  await Promise.race([loadPromise, sleep(25000)]);
-  await sleep(waitMs);
-
-  // Human-verification / login-state-loss guard (see fetchPage above).
-  await evaluateBrowserBlock(ws);
-
-  const result = await new Promise((resolve) => {
-    const id = Math.floor(Math.random() * 100000) + 80000;
-    const patternStr = pattern.toString();
-    const handler = (raw) => {
-      const msg = JSON.parse(raw.toString());
-      if (msg.id === id) {
-        ws.removeListener('message', handler);
-        resolve(msg.result && msg.result.result && msg.result.result.value);
-      }
-    };
-    ws.on('message', handler);
-    ws.send(JSON.stringify({ id, method: 'Runtime.evaluate', params: {
-      expression: `JSON.stringify(
-        Array.from(document.querySelectorAll('a[href]'))
-          .map(a => a.href)
-          .filter(h => h && new RegExp(${JSON.stringify(pattern)}).test(h))
-          .filter((h, i, arr) => arr.indexOf(h) === i)
-          .slice(0, 30)
-      )`
-    }}));
-  });
-
-  ws.close();
-  return result ? JSON.parse(result) : [];
-}
-
-module.exports = { fetchPage, fetchHomeLinks };
-
-// CLI usage
-if (require.main === module) {
-  const url = process.argv[2];
-  const waitMs = parseInt(process.argv[3] || '12000');
-  if (!url) { console.error('Usage: node fetcher-browser.js <url> [waitMs]'); process.exit(1); }
-  fetchPage(url, waitMs).then(data => {
-    process.stdout.write(JSON.stringify(data, null, 2));
-  }).catch(e => { console.error('Error:', e.message); process.exit(1); });
-}

skills/news-harvester/scripts/fetcher-helper.js

@@ -25,9 +25,8 @@ const { openSession, navigateAndWait, scrollAndCollect, cdpEval } = require('./c
 const { evaluateBrowserBlock } = require('./block-check');
 
 // ── Spec defaults ────────────────────────────────────────────────────────────
-// Defaults reproduce the existing hard-coded behaviour of fetcher-browser.js,
-// so a minimal helper.md (just urlPattern + listSelector) already improves on
-// legacy without forcing the author to redeclare every field.
+// Sensible defaults so a minimal helper.md (just urlPattern + listSelector)
+// already works well without forcing the author to redeclare every field.
 
 const DEFAULTS = {
   // ── Discovery ──
@@ -283,7 +282,7 @@ async function fetchHomeLinks(url, spec, desiredCount) {
       : (spec.maxLinks || 30);
     return items.map(i => i.url).slice(0, cap);
   } finally {
-    close();
+    await close();
   }
 }
 
@@ -297,7 +296,7 @@ async function fetchPage(url, spec) {
     const result = await cdpEval(ws, buildExtractExpr(spec));
     return result ? JSON.parse(result) : null;
   } finally {
-    close();
+    await close();
   }
 }
 

skills/news-harvester/scripts/harvest.js

@@ -395,6 +395,51 @@ function exitForHumanIntervention(err, context) {
   process.exit(70);  // EX_SOFTWARE-ish: distinct from network/parse errors
 }
 
+// ── Concurrency helpers ──────────────────────────────────────────────────────
+
+// Resolve the per-source concurrency, capped by the global browser tab limit.
+// source.concurrency defaults to 1 (serial, backward-compatible). Only when the
+// user explicitly sets it to 2+ does concurrent fetching kick in. The global
+// cap (chromium.maxConcurrentTabs) is enforced inside cdp-client.openSession()
+// via a browser self-check, so even if multiple harvest processes each request
+// their full concurrency, the total open tabs never stays above the limit.
+function resolveConcurrency(source, config) {
+  const sourceConcurrency = source.concurrency || 1;
+  const globalMax = (config.chromium && config.chromium.maxConcurrentTabs) || 5;
+  return Math.min(sourceConcurrency, globalMax);
+}
+
+// Run an async mapper over items with a bounded concurrency. When concurrency
+// <= 1, runs serially with an optional intervalMs between iterations (preserving
+// the original sleep(1000) pacing for backward compatibility). When > 1, runs
+// items through a worker pool of the given size with no inter-item delay.
+//
+// Results are returned in the same order as items, regardless of completion
+// order. Errors are NOT thrown — the caller's fn should catch and return a
+// sentinel (e.g. { data: null, error }) so that HumanInterventionError can
+// propagate by re-throwing from fn.
+async function mapWithConcurrency(items, concurrency, fn, opts = {}) {
+  const { intervalMs = 0 } = opts;
+  if (concurrency <= 1) {
+    const results = [];
+    for (let i = 0; i < items.length; i++) {
+      results.push(await fn(items[i], i));
+      if (intervalMs > 0 && i < items.length - 1) await sleep(intervalMs);
+    }
+    return results;
+  }
+  const results = new Array(items.length);
+  let index = 0;
+  const workers = Array.from({ length: concurrency }, async () => {
+    while (index < items.length) {
+      const i = index++;
+      results[i] = await fn(items[i], i);
+    }
+  });
+  await Promise.all(workers);
+  return results;
+}
+
 // ── Main ──────────────────────────────────────────────────────────────────────
 
 async function main() {
@@ -459,10 +504,10 @@ Examples:
   console.error(`   Method: ${source.method}`);
 
   // Strategy dispatch: if a helpers/<id>.md recipe exists for this source, use
-  // the spec-driven fetcher-helper; otherwise fall back to the legacy fetcher
-  // for its method. Existing sources keep working unchanged; a source opts in
-  // simply by gaining a helper.md file. See references/source-management.md
-  // (Helper recipes) for the schema.
+  // the spec-driven fetcher-helper. For direct (HTTP) sources without a recipe,
+  // fall back to fetcher-direct. Browser sources without a recipe are no longer
+  // supported — fetcher-browser has been removed. All browser-type sources must
+  // have a helper recipe (every bundle-installed source gets one automatically).
   const helperPath = path.join(SKILL_DIR, 'helpers', `${source.id}.md`);
   let fetcher, spec = null;
   if (fs.existsSync(helperPath)) {
@@ -470,10 +515,13 @@ Examples:
     spec = fetcherHelper.loadSpec(helperPath);
     fetcher = fetcherHelper;
     console.error(`   Recipe: helpers/${source.id}.md (listSelector=${spec.listSelector || 'whole page'}, scroll=${spec.scrollSteps})`);
+  } else if (source.method === 'direct') {
+    fetcher = require('./fetcher-direct');
   } else {
-    fetcher = source.method === 'browser'
-      ? require('./fetcher-browser')
-      : require('./fetcher-direct');
+    console.error(`❌ 数据源 "${source.id}" (method=browser) 缺少 helpers/${source.id}.md`);
+    console.error('   fetcher-browser 已废弃，所有 browser 类型数据源必须通过 helper recipe 工作。');
+    console.error('   请使用 news-source-analyzer 生成 recipe，或安装 .nhsource.json bundle。');
+    process.exit(1);
   }
 
   // CDP-based fetchers (browser method, or any source with a helper recipe)
@@ -556,44 +604,82 @@ Examples:
 
   // ── Preview mode: fetch only, print compact list, write nothing ───────────
   if (preview) {
+    const concurrency = resolveConcurrency(source, config);
+    if (concurrency > 1) console.error(`   Concurrency: ${concurrency} tabs`);
     const out = [];
-    for (let i = 0; i < newLinks.length; i++) {
-      if (out.length >= count) break;   // cap on successful articles, not candidates
-      const url = newLinks[i];
-      console.error(`\n[${i+1}/${newLinks.length}] ${url}`);
-      let articleData;
-      try {
-        articleData = await getPage(url);
-      } catch (e) {
-        if (e instanceof HumanInterventionError) exitForHumanIntervention(e, `文章抓取（第 ${i+1} 篇）`);
-        console.error(`  ✗ Fetch failed: ${e.message}`);
-        continue;
+
+    if (concurrency > 1) {
+      // ── Concurrent mode: batch fetch, then process results ─────────────────
+      // Cap candidates to count + a small buffer (some will be non-article/
+      // low-content) since all are fetched in parallel and early-exit isn't
+      // possible mid-batch.
+      const previewCandidates = newLinks.slice(0, Math.min(newLinks.length, count + 5));
+
+      const fetchResults = await mapWithConcurrency(
+        previewCandidates,
+        concurrency,
+        async (url, i) => {
+          console.error(`\n[${i+1}/${previewCandidates.length}] ${url}`);
+          try {
+            const data = await getPage(url);
+            return { url, data };
+          } catch (e) {
+            if (e instanceof HumanInterventionError) throw e;
+            return { url, data: null, error: e };
+          }
+        }
+      ).catch(e => {
+        if (e instanceof HumanInterventionError) exitForHumanIntervention(e, '文章抓取');
+        throw e;
+      });
+
+      for (const { url, data, error } of fetchResults) {
+        if (out.length >= count) break;
+        if (error) { console.error(`  ✗ Fetch failed: ${error.message}`); continue; }
+        if (!data || !data.title) { console.error('  ✗ No title, skip'); continue; }
+        if (data.isArticle === false) { console.error('  ✗ Not an article (og:type/body), skip'); continue; }
+        const category = getCategory(url, data.title);
+        const entry = {
+          title: data.title, date: data.date, authors: data.authors, url, category,
+          charCount: (data.body || '').length,
+          imgCount: (data.imgs || []).length,
+          imgs: (data.imgs || []).map(im => ({ alt: im.alt || '', src: im.src }))
+        };
+        if (full) entry.body = data.body || '';
+        else entry.bodyPreview = (data.body || '').substring(0, PREVIEW_LEN);
+        out.push(entry);
+        console.error(`  ✓ ${data.title} (${entry.charCount} chars, ${entry.imgCount} imgs)`);
       }
-      if (!articleData || !articleData.title) { console.error('  ✗ No title, skip'); continue; }
-      // Recipe article-validation: a broad urlPattern may let section/category
-      // pages through as candidate links; fetchPage marks them isArticle=false
-      // (via og:type / JSON-LD / body-length probes). Non-validating recipes
-      // always return isArticle=true, so this is a no-op for them.
-      if (articleData.isArticle === false) { console.error('  ✗ Not an article (og:type/body), skip'); continue; }
-      const category = getCategory(url, articleData.title);
-      const entry = {
-        title: articleData.title,
-        date: articleData.date,
-        authors: articleData.authors,
-        url,
-        category,
-        charCount: (articleData.body || '').length,
-        imgCount: (articleData.imgs || []).length,
-        imgs: (articleData.imgs || []).map(im => ({ alt: im.alt || '', src: im.src }))
-      };
-      if (full) {
-        entry.body = articleData.body || '';
-      } else {
-        entry.bodyPreview = (articleData.body || '').substring(0, PREVIEW_LEN);
+    } else {
+      // ── Serial mode: original early-exit loop (fetch → check → break) ──────
+      for (let i = 0; i < newLinks.length; i++) {
+        if (out.length >= count) break;
+        const url = newLinks[i];
+        console.error(`\n[${i+1}/${newLinks.length}] ${url}`);
+        let articleData;
+        try {
+          articleData = await getPage(url);
+        } catch (e) {
+          if (e instanceof HumanInterventionError) exitForHumanIntervention(e, `文章抓取（第 ${i+1} 篇）`);
+          console.error(`  ✗ Fetch failed: ${e.message}`);
+          continue;
+        }
+        if (!articleData || !articleData.title) { console.error('  ✗ No title, skip'); continue; }
+        if (articleData.isArticle === false) { console.error('  ✗ Not an article (og:type/body), skip'); continue; }
+        const category = getCategory(url, articleData.title);
+        const entry = {
+          title: articleData.title, date: articleData.date, authors: articleData.authors, url, category,
+          charCount: (articleData.body || '').length,
+          imgCount: (articleData.imgs || []).length,
+          imgs: (articleData.imgs || []).map(im => ({ alt: im.alt || '', src: im.src }))
+        };
+        if (full) entry.body = articleData.body || '';
+        else entry.bodyPreview = (articleData.body || '').substring(0, PREVIEW_LEN);
+        out.push(entry);
+        console.error(`  ✓ ${articleData.title} (${entry.charCount} chars, ${entry.imgCount} imgs)`);
       }
-      out.push(entry);
-      console.error(`  ✓ ${articleData.title} (${entry.charCount} chars, ${entry.imgCount} imgs)`);
     }
+
     process.stdout.write(JSON.stringify({
       sourceId, sourceName: source.name, preview: true, full, count: out.length, articles: out
     }, null, 2));
@@ -618,38 +704,25 @@ Examples:
 
   const processed = [];
 
-  for (let i = 0; i < newLinks.length; i++) {
-    if (processed.length >= count) break;   // cap on successful articles, not candidates
-    const url = newLinks[i];
-    console.error(`\n[${i+1}/${newLinks.length}] ${url}`);
-
-    let articleData;
-    try {
-      articleData = await getPage(url);
-    } catch (e) {
-      if (e instanceof HumanInterventionError) exitForHumanIntervention(e, `文章抓取（第 ${i+1} 篇）`);
-      console.error(`  ✗ Fetch failed: ${e.message}`);
-      continue;
+  const concurrency = resolveConcurrency(source, config);
+  if (concurrency > 1) console.error(`   Concurrency: ${concurrency} tabs`);
+
+  // Helper: validate + write a single article to disk. Returns the articleInfo
+  // on success, or null if the article should be skipped.
+  async function processArticle(url, data) {
+    if (!data || !data.title) { console.error('  ✗ No title, skip'); return null; }
+    if (data.isArticle === false) { console.error('  ✗ Not an article (og:type/body), skip'); return null; }
+    if ((data.body || '').length < 100) {
+      console.error(`  ✗ Insufficient content (body: ${(data?.body||'').length} chars)`);
+      return null;
     }
-
-    if (!articleData || !articleData.title) { console.error('  ✗ No title, skip'); continue; }
-    // Recipe article-validation: reject section/category pages that slipped
-    // through a broad urlPattern (fetchPage set isArticle=false via og:type /
-    // JSON-LD / body-length probes). No-op for non-validating recipes.
-    if (articleData.isArticle === false) { console.error('  ✗ Not an article (og:type/body), skip'); continue; }
-    if ((articleData.body || '').length < 100) {
-      console.error(`  ✗ Insufficient content (body: ${(articleData?.body||'').length} chars)`);
-      continue;
-    }
-
-    const { title: titleEn, date, authors, imgs = [], body } = articleData;
+    const { title: titleEn, date, authors, imgs = [], body } = data;
     const slug = slugFromUrl(url);
     const category = getCategory(url, titleEn);
-    const tags = [source.name];   // placeholder; finalize overwrites with agent-extracted tags
-    const titleZh = sentinelFor(slug);          // patched by finalize.js
+    const tags = [source.name];
+    const titleZh = sentinelFor(slug);
     const origName = originalFilename(titleEn, source.name);
 
-    // Download images
     const imgFiles = [];
     for (let j = 0; j < imgs.length; j++) {
       const imgSrc = imgs[j].src;
@@ -666,21 +739,62 @@ Examples:
       originalFilename: origName, sourceName: source.name
     };
 
-    // Write original doc (full body on disk, never sent to the agent)
     fs.writeFileSync(path.join(dayDir, origName), generateOriginalDoc(articleInfo, source.name, imgFiles));
-
-    // Append to registry (dedup record, written immediately so a crash keeps it)
     appendRegistry(config, source, today, {
       url, titleZh, titleEn, date, processedAt: new Date().toISOString(),
       category, tags, slug, originalFilename: origName
     });
-
-    // Refresh daily index from the running set (sentinel titles until finalize)
     updateIndex(dayDir, [...processed, articleInfo], source.name);
-
-    processed.push(articleInfo);
     console.error(`  ✓ ${titleEn} (${imgFiles.length} imgs, ${body.length} chars) → ${origName}`);
-    await sleep(1000);
+    return articleInfo;
+  }
+
+  if (concurrency > 1) {
+    // ── Concurrent mode: batch fetch, then write serially ───────────────────
+    const fetchResults = await mapWithConcurrency(
+      newLinks,
+      concurrency,
+      async (url, i) => {
+        console.error(`\n[${i+1}/${newLinks.length}] ${url}`);
+        try {
+          const data = await getPage(url);
+          return { url, data };
+        } catch (e) {
+          if (e instanceof HumanInterventionError) throw e;
+          return { url, data: null, error: e };
+        }
+      }
+    ).catch(e => {
+      if (e instanceof HumanInterventionError) exitForHumanIntervention(e, '文章抓取');
+      throw e;
+    });
+
+    for (const { url, data, error } of fetchResults) {
+      if (processed.length >= count) break;
+      if (error) { console.error(`  ✗ Fetch failed: ${error.message}`); continue; }
+      const articleInfo = await processArticle(url, data);
+      if (articleInfo) processed.push(articleInfo);
+    }
+  } else {
+    // ── Serial mode: original early-exit loop (fetch → write → sleep → break)
+    for (let i = 0; i < newLinks.length; i++) {
+      if (processed.length >= count) break;
+      const url = newLinks[i];
+      console.error(`\n[${i+1}/${newLinks.length}] ${url}`);
+      let articleData;
+      try {
+        articleData = await getPage(url);
+      } catch (e) {
+        if (e instanceof HumanInterventionError) exitForHumanIntervention(e, `文章抓取（第 ${i+1} 篇）`);
+        console.error(`  ✗ Fetch failed: ${e.message}`);
+        continue;
+      }
+      const articleInfo = await processArticle(url, articleData);
+      if (articleInfo) {
+        processed.push(articleInfo);
+        await sleep(1000);
+      }
+    }
   }
 
   if (!preview && processed.length < count) {

skills/news-harvester/scripts/source-manage.js

@@ -168,7 +168,8 @@ switch (command) {
       homepageUrl: src.homepageUrl || '',
       method: src.method || 'browser',
       vaultFolder: src.vaultFolder || src.name || id,
-      enabled: true
+      enabled: true,
+      concurrency: 1   // serial by default; user can raise to enable multi-tab fetching
     };
     if (existing) {
       Object.assign(existing, entry);

skills/news-source-analyzer/references/recipe-guide.md

@@ -2,7 +2,7 @@
 
 ## How recipes work
 
-A recipe (`helpers/<id>.md`) is a YAML frontmatter file that declares the selectors, scroll behavior, and URL filters for a news source. When the harvester's `harvest.js` finds a recipe, it uses `fetcher-helper.js` (which reads the recipe's spec) instead of the legacy `fetcher-browser.js`/`fetcher-direct.js`.
+A recipe (`helpers/<id>.md`) is a YAML frontmatter file that declares the selectors, scroll behavior, and URL filters for a news source. When the harvester's `harvest.js` finds a recipe, it uses `fetcher-helper.js` (which reads the recipe's spec) to drive Chromium via CDP. Browser-type sources without a recipe are no longer supported (`fetcher-browser.js` has been removed); direct-type sources fall back to `fetcher-direct.js`.
 
 > **Precedence**: when a recipe exists, its `urlPattern`/selectors are the source of truth and the config's `articleUrlPattern` is **ignored**. A recipe also forces Chromium use regardless of `method` (since `fetcher-helper.js` drives the browser).
 

skills/news-source-analyzer/scripts/block-check.js

@@ -11,7 +11,7 @@
  *
  * Two entry points:
  *   • Browser (CDP) sources — evaluateBrowserBlock(ws) runs an in-page probe via
- *     cdpEval and throws if blocked. Used by fetcher-helper.js / fetcher-browser.js.
+ *     cdpEval and throws if blocked. Used by fetcher-helper.js.
  *   • Direct HTTP sources — checkDirectBlock(html, url) does a string scan of the
  *     raw HTML. Used by fetcher-direct.js (open sites rarely need login, but a
  *     Cloudflare-style challenge can still appear).
@@ -26,9 +26,17 @@ function _browserBlockProbe() {
   var bodyLen = bodyText.length;
 
   // 1. CAPTCHA / anti-bot challenge — iframes from known challenge providers.
+  // Note: recaptcha/api2/aframe is a passive background iframe used by
+  // reCAPTCHA v3 for ad-fraud prevention — it does NOT block the user and is
+  // present on many news sites (Reuters, Nikkei, etc.). Only match the active
+  // challenge iframes (anchor/bframe/enterprise) that constitute a visible
+  // verification widget.
   var challengeIframe = Array.from(document.querySelectorAll('iframe[src]')).some(function (f) {
     var s = f.src.toLowerCase();
-    return s.indexOf('recaptcha') !== -1 || s.indexOf('hcaptcha') !== -1
+    return s.indexOf('recaptcha/api2/anchor') !== -1
+      || s.indexOf('recaptcha/api2/bframe') !== -1
+      || s.indexOf('recaptcha/enterprise') !== -1
+      || s.indexOf('hcaptcha') !== -1
       || s.indexOf('challenges.cloudflare.com') !== -1
       || s.indexOf('turnstile') !== -1
       || s.indexOf('arkoselabs') !== -1 || s.indexOf('funcaptcha') !== -1;

skills/news-source-analyzer/scripts/cdp-client.js

 /**
  * cdp-client.js — Shared Chrome DevTools Protocol primitives.
  *
- * Previously every CDP caller (fetcher-browser.js) hand-rolled the same
- * WebSocket boilerplate: open connection, send a command with an id, attach a
- * message listener matching that id, parse the result. That pattern was
- * duplicated 4× and the Page.enable call didn't even check the id (resolving
- * on the first message of any kind). This module is the single source of truth
- * for those operations so fetcher-helper.js (and a future fetcher-browser
- * refactor) can build on top.
+ * This module is the single source of truth for CDP operations used by
+ * fetcher-helper.js and the analyzer scripts (inspect-source / preview /
+ * recipe-test). It provides two layers:
+ *
+ *   1. Browser-level target management — a long-lived WebSocket to the
+ *      browser's own devtools endpoint (webSocketDebuggerUrl from /json/version),
+ *      used for Target.createTarget / Target.closeTarget / Target.getTargets.
+ *      This is what enables multi-tab concurrency: every openSession() call
+ *      creates a fresh page target instead of reusing the browser's first tab,
+ *      so concurrent fetchers navigate independent tabs without clobbering
+ *      each other.
+ *
+ *   2. Per-tab CDP commands — cdpSend / cdpEval / navigateAndWait /
+ *      scrollAndCollect operate on a specific tab's WebSocket (returned by
+ *      openSession). These are unchanged from the original single-tab design.
+ *
+ * Global concurrency control: openSession() checks Target.getTargets before
+ * creating a new tab. If the number of page-type targets is at or above the
+ * configured limit (chromium.maxConcurrentTabs, default 5), it polls every
+ * 200 ms until a slot frees up. This is a "browser self-check" strategy — all
+ * processes sharing the same Chromium instance see the same target list, so
+ * the limit is enforced globally across concurrent harvest runs. A brief
+ * TOCTOU over-limit of 1-2 tabs is possible and harmless.
  *
  * Lifecycle (browser launch/discovery) stays in ensure-chromium.js — this
- * module only handles per-tab CDP commands, assuming a browser is already up.
+ * module only handles CDP commands, assuming a browser is already up.
  */
 
 const http = require('http');
@@ -26,6 +42,20 @@ const TEMPLATE_PATH = path.join(SKILL_DIR, 'references', 'config-template.json')
 
 function sleep(ms) { return new Promise(r => setTimeout(r, ms)); }
 
+// Small HTTP GET helper returning parsed JSON. Used for /json/version.
+function httpGetJson(url) {
+  return new Promise((resolve, reject) => {
+    http.get(url, (res) => {
+      let data = '';
+      res.on('data', d => data += d);
+      res.on('end', () => {
+        try { resolve(JSON.parse(data)); }
+        catch (e) { reject(new Error(`Bad JSON from ${url}: ${e.message}`)); }
+      });
+    }).on('error', reject);
+  });
+}
+
 // Monotonic id for CDP commands. Per-tab sessions are sequential, so a simple
 // counter is fine and avoids the random-id collisions the old code risked.
 let _msgId = 0;
@@ -33,7 +63,6 @@ function nextId() { return ++_msgId; }
 
 // ── Port resolution ──────────────────────────────────────────────────────────
 // Precedence: env override > config.chromium.cdpPort > default 9222.
-// (Mirrors the old private resolveCdpPort in fetcher-browser.js.)
 function resolveCdpPort() {
   if (process.env.CDP_PORT_OVERRIDE) return parseInt(process.env.CDP_PORT_OVERRIDE, 10);
   try {
@@ -82,16 +111,156 @@ function getWsUrl() {
   });
 }
 
+// ── Browser-level target management ──────────────────────────────────────────
+//
+// A long-lived WebSocket to the browser's own devtools endpoint (not a page
+// tab). Used for Target.createTarget / Target.closeTarget / Target.getTargets
+// so that each openSession() call gets its own independent tab — the basis for
+// multi-tab concurrency. Multiple Node processes may each hold their own
+// browser-level connection; CDP supports multiple clients on the same browser
+// target, each with an independent message-id space.
+
+let _browserWs = null;
+let _browserWsPromise = null;
+let _browserWsRefs = 0;   // active sessions holding a reference to the browser WS
+
+async function getBrowserWs() {
+  if (_browserWs && _browserWs.readyState === WebSocket.OPEN) return _browserWs;
+  if (_browserWsPromise) return _browserWsPromise;
+  _browserWsPromise = (async () => {
+    const port = resolveCdpPort();
+    const data = await httpGetJson(`http://localhost:${port}/json/version`);
+    const wsUrl = data.webSocketDebuggerUrl;
+    if (!wsUrl) throw new Error('CDP /json/version returned no webSocketDebuggerUrl');
+    const ws = new WebSocket(wsUrl);
+    await new Promise((res, rej) => {
+      ws.once('open', res);
+      ws.once('error', rej);
+    });
+    // Auto-recovery: if the socket drops, clear the cache so the next call
+    // reconnects.
+    ws.on('close', () => { _browserWs = null; });
+    ws.on('error', () => { _browserWs = null; });
+    _browserWs = ws;
+    _browserWsPromise = null;
+    return ws;
+  })();
+  return _browserWsPromise;
+}
+
+// Reference counting for the browser-level WebSocket.
+//
+// The browser WS is a long-lived cached connection used for Target.* commands
+// (createTab / closeTab / countPageTabs). If it stays open after all sessions
+// are closed, its TCP socket keeps the Node.js event loop alive — the process
+// hangs forever after completing its work.
+//
+// openSession() calls refBrowserWs() to increment the counter; close() calls
+// unrefBrowserWs() to decrement. When the counter reaches zero, we proactively
+// close the browser WS, removing the last handle so the process can exit
+// naturally. If a new session opens later, getBrowserWs() reconnects.
+//
+// Both the browser WS and per-tab WS stay fully ref'd (Node.js default) during
+// active use — this is essential because cdpSend() needs the event loop alive
+// to receive CDP responses. We only close the browser WS when NO sessions are
+// active, so there's never a risk of closing it mid-command.
+function refBrowserWs() { _browserWsRefs++; }
+
+function unrefBrowserWs() {
+  _browserWsRefs = Math.max(0, _browserWsRefs - 1);
+  if (_browserWsRefs === 0 && _browserWs) {
+    try { _browserWs.close(); } catch (e) { /* best-effort */ }
+    _browserWs = null;
+  }
+}
+
+// ── Concurrency control (global, browser self-check) ─────────────────────────
+
+const MAX_TABS_DEFAULT = 5;
+const TAB_POLL_INTERVAL = 200;   // ms between Target.getTargets polls
+
+function getMaxConcurrentTabs() {
+  try {
+    const cfgPath = fs.existsSync(CONFIG_PATH) ? CONFIG_PATH : TEMPLATE_PATH;
+    const cfg = JSON.parse(fs.readFileSync(cfgPath, 'utf8'));
+    return (cfg.chromium && cfg.chromium.maxConcurrentTabs) || MAX_TABS_DEFAULT;
+  } catch (e) { return MAX_TABS_DEFAULT; }
+}
+
+// Count current page-type targets via the browser-level WS.
+async function countPageTabs() {
+  const ws = await getBrowserWs();
+  const result = await cdpSend(ws, 'Target.getTargets');
+  return (result.targetInfos || []).filter(t => t.type === 'page').length;
+}
+
+// Block until the number of open page tabs is below the configured limit.
+// TOCTOU note: the check and the subsequent createTab are not atomic, so under
+// heavy multi-process contention the tab count may briefly exceed the limit by
+// 1-2. This is harmless — Chromium handles a few extra tabs, and they drain as
+// closeTab calls land.
+async function waitForTabSlot() {
+  const maxTabs = getMaxConcurrentTabs();
+  for (;;) {
+    if (await countPageTabs() < maxTabs) return;
+    await sleep(TAB_POLL_INTERVAL);
+  }
+}
+
+// Create a new page tab (about:blank) and return its targetId + wsUrl.
+async function createTab() {
+  const ws = await getBrowserWs();
+  const result = await cdpSend(ws, 'Target.createTarget', { url: 'about:blank' });
+  const targetId = result.targetId;
+  const port = resolveCdpPort();
+  return { targetId, wsUrl: `ws://localhost:${port}/devtools/page/${targetId}` };
+}
+
+// Close a page tab by targetId. Best-effort — errors are swallowed because the
+// tab may already be gone (e.g. site-triggered navigation closed it).
+async function closeTab(targetId) {
+  try {
+    const ws = await getBrowserWs();
+    await cdpSend(ws, 'Target.closeTarget', { targetId });
+  } catch (e) { /* tab may already be closed */ }
+}
+
 // ── Session management ───────────────────────────────────────────────────────
 
-// Open a WebSocket to the page target and resolve once connected.
-// Returns { ws, close }. The caller owns the lifecycle (close when done).
-function openSession() {
-  return getWsUrl().then(wsUrl => new Promise((resolve, reject) => {
+// Open a new browser tab, connect to it, and return { ws, close, targetId }.
+// The caller owns the lifecycle — it must call close() when done (which closes
+// both the WebSocket and the tab itself). This is the basis for multi-tab
+// concurrency: each call gets a fresh, isolated tab, so concurrent fetchers
+// never clobber each other's navigation.
+//
+// Before creating a tab, waitForTabSlot() enforces the global concurrency limit
+// (chromium.maxConcurrentTabs) via a Target.getTargets browser self-check. This
+// makes the limit global across all processes sharing the Chromium instance.
+async function openSession() {
+  await waitForTabSlot();
+  refBrowserWs();   // keep browser WS alive for the lifetime of this session
+  try {
+    const { targetId, wsUrl } = await createTab();
     const ws = new WebSocket(wsUrl);
-    ws.on('open', () => resolve({ ws, close: () => ws.close() }));
-    ws.on('error', reject);
-  }));
+    await new Promise((resolve, reject) => {
+      ws.once('open', resolve);
+      ws.once('error', reject);
+    });
+    const closed = { value: false };
+    const close = async () => {
+      if (closed.value) return;
+      closed.value = true;
+      ws.close();
+      await closeTab(targetId);   // must await so process.exit doesn't kill the tab
+      unrefBrowserWs();           // may close browser WS if this was the last session
+    };
+    return { ws, close, targetId };
+  } catch (e) {
+    // If session creation failed (e.g. tab creation error), release the ref
+    // so the browser WS can be cleaned up.
+    unrefBrowserWs();
+    throw e;
+  }
 }
 
 // ── Command sending ──────────────────────────────────────────────────────────
@@ -229,5 +398,12 @@ module.exports = {
   cdpSend,
   cdpEval,
   navigateAndWait,
-  scrollAndCollect
+  scrollAndCollect,
+  // Browser-level target management (multi-tab concurrency)
+  getBrowserWs,
+  countPageTabs,
+  waitForTabSlot,
+  createTab,
+  closeTab,
+  getMaxConcurrentTabs
 };

skills/news-source-analyzer/scripts/ensure-chromium.js

@@ -256,6 +256,41 @@ function looksLikeDisplayFailure(log) {
   return /Missing X server|\$DISPLAY|platform failed to initialize|ozone/i.test(log);
 }
 
+// ── Startup-tab cleanup ─────────────────────────────────────────────────────
+// Chromium opens an initial tab (about:blank or chrome://newtab) on launch.
+// With the multi-tab architecture, openSession() creates its own tabs, so this
+// startup tab is an orphan that wastes a tab slot and inflates Target.getTargets
+// counts. Close it via the HTTP /json/close/<id> endpoint (no WebSocket needed).
+// Only closes about:blank / chrome://newtab — never touches real harvest tabs.
+const STARTUP_TAB_URLS = new Set(['about:blank', 'chrome://newtab/', 'chrome://newtab']);
+
+function closeStartupTabs(port) {
+  return new Promise((resolve) => {
+    // agent: false → don't pool sockets in the global HTTP agent. Without this
+    // the keep-alive sockets linger and keep the Node event loop alive, hanging
+    // ensure-chromium.js (and any process that calls ensureChromium()).
+    http.get(`http://localhost:${port}/json`, { agent: false }, (res) => {
+      let data = '';
+      res.on('data', (d) => (data += d));
+      res.on('end', () => {
+        try {
+          const tabs = JSON.parse(data);
+          const orphans = tabs.filter(t => t.type === 'page' && STARTUP_TAB_URLS.has(t.url));
+          if (orphans.length === 0) return resolve(0);
+          let remaining = orphans.length;
+          for (const tab of orphans) {
+            http.get(`http://localhost:${port}/json/close/${tab.id}`, { agent: false }, () => {
+              if (--remaining === 0) resolve(orphans.length);
+            }).on('error', () => {
+              if (--remaining === 0) resolve(orphans.length);
+            });
+          }
+        } catch (e) { resolve(0); }
+      });
+    }).on('error', () => resolve(0));
+  });
+}
+
 // ── Main ────────────────────────────────────────────────────────────────────
 
 async function ensureChromium(config, portOverride) {
@@ -264,6 +299,7 @@ async function ensureChromium(config, portOverride) {
   // 1. Already up? (idempotent — safe to call every run)
   try {
     const v = await cdpVersion(opts.cdpPort, 1500);
+    await closeStartupTabs(opts.cdpPort);   // clean orphan tabs from prior runs
     return { port: opts.cdpPort, alreadyRunning: true, browser: v.Browser };
   } catch (e) { /* not up yet, fall through to launch */ }
 
@@ -280,6 +316,7 @@ async function ensureChromium(config, portOverride) {
   // 4. Wait for readiness
   try {
     const v = await waitForCdp(opts.cdpPort, 30000, 500);
+    await closeStartupTabs(opts.cdpPort);   // close the initial about:blank tab
     return { port: opts.cdpPort, pid, browser: v.Browser, bin, logPath };
   } catch (e) {
     const logTail = tailFile(logPath);
@@ -527,6 +564,7 @@ if (require.main === module) {
         console.log(`   binary: ${r.bin}`);
         console.log(`   log:    ${r.logPath}`);
       }
+      process.exit(0);
     } catch (e) {
       console.error(`❌ ${e.message}`);
       process.exit(1);

skills/news-source-analyzer/scripts/fetcher-helper.js

@@ -25,9 +25,8 @@ const { openSession, navigateAndWait, scrollAndCollect, cdpEval } = require('./c
 const { evaluateBrowserBlock } = require('./block-check');
 
 // ── Spec defaults ────────────────────────────────────────────────────────────
-// Defaults reproduce the existing hard-coded behaviour of fetcher-browser.js,
-// so a minimal helper.md (just urlPattern + listSelector) already improves on
-// legacy without forcing the author to redeclare every field.
+// Sensible defaults so a minimal helper.md (just urlPattern + listSelector)
+// already works well without forcing the author to redeclare every field.
 
 const DEFAULTS = {
   // ── Discovery ──
@@ -283,7 +282,7 @@ async function fetchHomeLinks(url, spec, desiredCount) {
       : (spec.maxLinks || 30);
     return items.map(i => i.url).slice(0, cap);
   } finally {
-    close();
+    await close();
   }
 }
 
@@ -297,7 +296,7 @@ async function fetchPage(url, spec) {
     const result = await cdpEval(ws, buildExtractExpr(spec));
     return result ? JSON.parse(result) : null;
   } finally {
-    close();
+    await close();
   }
 }
 

skills/news-source-analyzer/scripts/inspect-source.js

@@ -305,7 +305,7 @@ async function main() {
       console.log(`\n   Save to helpers/<source-id>.md (or re-run with --write <id>), then test: node scripts/preview.js --url <url> --recipe helpers/<source-id>.md --count 3`);
     }
   } finally {
-    close();
+    await close();
   }
 }