feat(news-harvester): sitemap candidate discovery + dateline body fallback

Reuters /technology/ renders a fixed ~20-article list with no lazy-load
or load-more — on high-volume days the window fully churns between two
harvest runs, making gap articles structurally un-capturable from the
DOM. Two recipe-gated additions to fetcher-helper:

- sitemap discovery (sitemapUrl/sitemapFromStep/sitemapPages/sitemap-
  Include): archive-mode link collection reads Google-News sitemap pages
  via same-origin fetch() inside the site session (DataDome cookies
  apply), newest-first, section-prefixed so a site-wide sitemap cannot
  swallow sibling channels' articles. Any failure falls back to the DOM
  scan. Enabled for all four reuters channel recipes (machine-local).
- datelineFallback: wire-service bodies (Reuters) render in wildly
  varying DOM structures, so selector lists go stale and articles look
  paywalled while the full text sits in innerText. When selectors
  under-produce (< bodyMinChars), slice innerText from the dateline
  lead ("AMSTERDAM, Nov 12 (Reuters) -") through the byline
  ("Reporting by"), stripping boilerplate lines — immune to markup
  churn. Also adds cdpEvalAsync (awaitPromise) for in-page fetch().

skills/news-harvester/scripts/fetcher-helper.js

@@ -21,9 +21,23 @@ const fs = require('fs');
 const path = require('path');
 const yaml = require('js-yaml');
 
-const { openSession, navigateAndWait, scrollAndCollect, clickAndWait, cdpEval } = require('./cdp-client');
+const { openSession, navigateAndWait, scrollAndCollect, clickAndWait, cdpEval, cdpSend } = require('./cdp-client');
 const { evaluateBrowserBlock } = require('./block-check');
 
+// Runtime.evaluate WITH awaitPromise — needed for in-page fetch() of sitemaps
+// (cdpEval returns immediately and would hand back an unawaited Promise object).
+async function cdpEvalAsync(ws, expression) {
+  const result = await cdpSend(ws, 'Runtime.evaluate', {
+    expression, returnByValue: true, awaitPromise: true,
+  });
+  const ex = result && result.exceptionDetails;
+  if (ex) {
+    const desc = (ex.exception && ex.exception.description) || ex.text || 'page eval failed';
+    throw new Error(desc.split('\n')[0].slice(0, 200));
+  }
+  return result && result.result && result.result.value;
+}
+
 // ── Spec defaults ────────────────────────────────────────────────────────────
 // Sensible defaults so a minimal helper.md (just urlPattern + listSelector)
 // already works well without forcing the author to redeclare every field.
@@ -162,6 +176,7 @@ function buildDiscoveryExpr(spec) {
 function buildExtractExpr(spec) {
   const bodySelectors = JSON.stringify(spec.bodySelectors);
   const bodyMinParagraphs = spec.bodyMinParagraphs;
+  const datelineFallback = !!spec.datelineFallback;
   const dateSelector = JSON.stringify(spec.dateSelector);
   const authorSelector = JSON.stringify(spec.authorSelector);
   const imageMinWidth = spec.imageMinWidth;
@@ -260,6 +275,36 @@ function buildExtractExpr(spec) {
       bodyParagraphs = fallback.length;
       body = fallback.join('\\n\\n');
     }
+    // Dateline fallback (recipe flag datelineFallback, e.g. Reuters): wire
+    // services print the body as a contiguous block in document order — a
+    // dateline lead ("AMSTERDAM, Nov 12 (Reuters) - …") through the byline
+    // ("Reporting by …") — but render it in wildly varying DOM structures
+    // (paragraph DIVs, a single Body <p>, template A/B variants), so selector
+    // lists keep going stale. Anchoring on the dateline/byline TEXT instead
+    // of the DOM is immune to markup churn. Only used when the selectors
+    // under-produced (< bodyMinChars); junk lines (newsletter promo, footer)
+    // between the anchors are dropped.
+    if (${datelineFallback ? 'true' : 'false'} && body.length < ${bodyMinChars}) {
+      var months = 'Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec';
+      var dlRe = new RegExp(',\\\\s*(' + months + ')\\\\s+\\\\d{1,2}\\\\s+\\\\(Reuters\\\\)\\\\s*-');
+      var lines = document.body.innerText.split('\\n')
+        .map(function(l){ return l.replace(/[\\u200b-\\u200d\\u2060\\ufeff]/g, '').trim(); });
+      var start = -1;
+      for (var li = 0; li < lines.length; li++) {
+        if (dlRe.test(lines[li])) { start = li; break; }
+      }
+      if (start >= 0) {
+        var out = [];
+        for (var lj = start; lj < lines.length; lj++) {
+          if (/^Reporting by/.test(lines[lj])) break;
+          if (!lines[lj]) continue;
+          if (/^(Learn about the latest|Our Standards|opens new tab|, opens new tab|The Thomson Reuters Trust Principles)/.test(lines[lj])) continue;
+          out.push(lines[lj]);
+        }
+        var joined = out.join('\\n\\n');
+        if (joined.length > body.length) { body = joined; bodyParagraphs = out.length; }
+      }
+    }
     // Article-validation signal: og:type + JSON-LD @type + body length. A
     // section/category page typically has og:type=website, @type=Thing, and
     // zero body paragraphs, so this cleanly separates real articles from the
@@ -311,14 +356,73 @@ function buildExtractExpr(spec) {
 // many pages are fetched one after another. When omitted, a new session is
 // created and closed in the finally block (harvester behavior).
 //
-// Optional `paginate` (harvest.js): when true AND desiredCount > 0 AND the
-// recipe configures a pagination mode (nextSelector or pageUrlTemplate),
+// ── Sitemap discovery ───────────────────────────────────────────────────────
+// Some section pages (Reuters /technology/) render a fixed ~20-article list
+// with neither lazy-load nor a "load more" control — on high-volume days the
+// window fully churns between two harvest runs, so articles published in the
+// gap become structurally un-capturable from the DOM. Google-News-style
+// sitemaps list every recently published article instead. When the recipe
+// configures `sitemapUrl`, archive-mode discovery reads sitemap pages via a
+// same-origin fetch() inside the already-navigated site session (DataDome /
+// login cookies apply), instead of scanning the visible list.
+//
+// Recipe fields:
+//   sitemapUrl        absolute URL of sitemap page 1 (page N>1 appends &from=N*step)
+//   sitemapFromStep   `from=` increment per page (Reuters Arc: 100)
+//   sitemapPages      max pages to read (default 3, cap 10)
+//   sitemapInclude    substring a URL must contain (e.g. '/technology/') so a
+//                     site-wide sitemap only yields THIS channel's articles —
+//                     without it a site-wide sweep would swallow other channels'
+//                     articles into the wrong vault folder (cross-channel dedup
+//                     would then block the rightful channel from archiving).
+async function fetchSitemapLinks(ws, spec, desiredCount) {
+  const pages = Math.min(spec.sitemapPages || 3, 10);
+  const step = spec.sitemapFromStep || 100;
+  const include = spec.sitemapInclude || '';
+  const urlRe = new RegExp(spec.urlPattern);
+  const exRe = spec.excludeUrlPattern ? new RegExp(spec.excludeUrlPattern) : null;
+  const collected = new Map();  // url -> publication_date (ISO string)
+  for (let p = 0; p < pages; p++) {
+    const pageUrl = p === 0 ? spec.sitemapUrl
+      : `${spec.sitemapUrl}${spec.sitemapUrl.includes('?') ? '&' : '?'}from=${p * step}`;
+    const xml = String(await cdpEvalAsync(ws,
+      `fetch(${JSON.stringify(pageUrl)}, { credentials: 'include' })` +
+      `.then(r => { if (!r.ok) throw new Error('HTTP ' + r.status); return r.text(); })`));
+    if (!xml.includes('<loc>')) throw new Error(`sitemap ${pageUrl} has no <loc> entries`);
+    let added = 0;
+    for (const m of xml.matchAll(/<url>([\s\S]*?)<\/url>/g)) {
+      const entry = m[1];
+      const loc = (entry.match(/<loc>([^<]+)<\/loc>/) || [])[1];
+      if (!loc) continue;
+      if (include && !loc.includes(include)) continue;
+      if (!urlRe.test(loc)) continue;
+      if (exRe && exRe.test(loc)) continue;
+      const pub = (entry.match(/<news:publication_date>([^<]+)<\/news:publication_date>/) || [])[1] || '';
+      if (!collected.has(loc)) { collected.set(loc, pub); added++; }
+    }
+    console.error(`   🗺️  sitemap 第 ${p + 1} 页:+${added} 条候选(累计 ${collected.size})`);
+    if (collected.size >= desiredCount) break;
+  }
+  // Newest first: news sitemaps are reverse-chronological, but sort anyway so
+  // the slice cap can never drop a newer article in favor of an older one.
+  const urls = [...collected.entries()]
+    .sort((a, b) => (b[1] || '').localeCompare(a[1] || ''))
+    .map(e => e[0]);
+  return urls;
+}
+
+
 // follow multi-page article lists — scrolling each page with fixed scrollSteps,
 // then advancing to the next page, cross-page deduping until desiredCount is
 // met, maxPages is exhausted, or consecutive pages yield no new candidates.
 // Two-level switch: config.json `paginate` gates the feature per source; the
 // recipe fields describe *how* to paginate. A warning is printed when one
 // level is on but the other is missing.
+//
+// Sitemap mode (archive mode only): when the recipe sets `sitemapUrl`,
+// discovery reads news-sitemap pages (see fetchSitemapLinks) instead of the
+// DOM list; any sitemap fetch/parse failure falls back to the DOM path so a
+// sitemap outage degrades to previous behaviour instead of breaking the run.
 async function fetchHomeLinks(url, spec, desiredCount, session, paginate) {
   const ownSession = !session;
   const { ws, close } = session || await openSession();
@@ -330,6 +434,19 @@ async function fetchHomeLinks(url, spec, desiredCount, session, paginate) {
     const archiveMode = desiredCount && desiredCount > 0;
     const enablePaginate = archiveMode && paginate && recipePaginate;
 
+    if (archiveMode && spec.sitemapUrl) {
+      try {
+        const links = await fetchSitemapLinks(ws, spec, desiredCount);
+        if (links.length > 0) {
+          const cap = Math.max(spec.maxLinks || 30, desiredCount);
+          return links.slice(0, cap);
+        }
+        console.error('   ⚠️  sitemap 模式未取得候选,回退页面扫描');
+      } catch (e) {
+        console.error(`   ⚠️  sitemap 抓取失败(${e.message}),回退页面扫描`);
+      }
+    }
+
     // Two-level switch diagnostics: warn when only one level is configured.
     if (archiveMode && paginate && !recipePaginate) {
       console.error('   ⚠️  config 已启用 paginate 但 recipe 未配置翻页字段（nextSelector / pageUrlTemplate），按单页采集。');