feat: better harvest action

SKILL.md

@@ -211,7 +211,7 @@ No files are written. stdout is a compact JSON list (one call, no re-fetching /
 
 ### Guardrails (token hygiene)
 
-- **Don't re-run `harvest.js`** for the same source/day just to "get more" — it deduplicates via the registry, so a second run only fetches genuinely new articles (and merges them into the pending `.manifest.json`, which is safe). If you need more, increase `count` on the first run.
+- **`count` is the target number of NEW articles**, not raw links. `harvest.js` already accounts for registry dedup: it scrolls to collect extra candidate links (`count + seen + buffer`, capped) so the *new* count can still reach `count` even when most of the homepage was already archived. If the run returns **fewer than `count`**, the stderr prints `⚠️ … stream may be exhausted` — that's a genuine source limit (no fresh articles to fetch), **not** a bug; re-running won't help. Wait for new articles or broaden the source instead. Don't re-run `harvest.js` for the same source/day just to "get more" — it deduplicates via the registry, so a second run only fetches genuinely new articles (and merges them into the pending `.manifest.json`, which is safe).
 - **Never `Read` the 原文.md files** to write summaries — the `bodyPreview` in the manifest is sufficient. Targeted slice reads are OK if truly needed.
 - **One `finalize.js` call** after writing `.summaries.json`. Do not hand-write file I/O — the scripts handle all original/registry/index/summary writes.
 

helpers/reuters-business.md

+---
+source: reuters-business
+listSelector: null
+linkSelector: 'a[href]'
+urlPattern: 'reuters\.com/business/(?:[^/]+/)*[\w-]+-\d{4}-\d{2}-\d{2}/'
+excludeUrlPattern: null
+scrollSteps: 3
+scrollWaitMs: 1500
+waitMs: 12000
+bodySelectors:
+  - '[data-testid^="paragraph"]'
+  - '[class*="article-body"] p'
+  - 'article p'
+bodyMinParagraphs: 3
+dateSelector: 'time[datetime]'
+authorSelector: '[rel="author"], [class*="author"] a'
+imageSelector: 'img'
+imageMinWidth: 200
+excludeImage:
+  - logo
+  - icon
+  - avatar
+  - profile
+titleStripSuffix: ''
+maxBodyChars: 15000
+---
+# Reuters Business 抓取备忘
+
+- Reuters Business 首页的文章链接位于页面主体 `div[data-testid="Title"]`，但使用全页扫描配合严格 URL 过滤可避免依赖动态 CSS 类名。
+- 文章 URL 形如 `/business/<可选分区>/<slug>-YYYY-MM-DD/`；日期后缀用于排除导航、专题页和分区页。
+- `scrollSteps: 3` 触发首页懒加载，正文优先使用段落测试选择器。

helpers/reuters-markets.md

+---
+source: reuters-markets
+listSelector: "div.media-story-card-module__placement-container__1ZSB1"
+linkSelector: 'a[href]'
+urlPattern: 'reuters\.com/.+-\d{4}-\d{2}-\d{2}'
+excludeUrlPattern: null
+scrollSteps: 3
+scrollWaitMs: 2000
+waitMs: 15000
+bodySelectors:
+  - '[data-testid^="paragraph"]'
+  - '[class*="articleBodyContent"] p'
+  - '[class*="article-body"] p'
+  - 'article p'
+bodyMinParagraphs: 3
+dateSelector: 'time[datetime]'
+authorSelector: '[rel="author"], [class*="author"] a'
+imageSelector: 'img'
+imageMinWidth: 200
+excludeImage:
+  - logo
+  - icon
+  - avatar
+  - profile
+titleStripSuffix: ' \| Reuters$'
+maxBodyChars: 15000
+---
+# Reuters Markets 抓取备忘
+
+- 主文章流容器 `div.media-story-card-module__placement-container__1ZSB1` 实测 6 个候选链接（inspect 自动生成）。
+- urlPattern `reuters\.com/.+-\d{4}-\d{2}-\d{2}` 由样本 URL 推断，如 preview 漏抓/误抓请手编收紧。
+- 首屏可能不全，scrollSteps=0 触发懒加载；反爬严重时调高 waitMs。
+- 验证: node scripts/harvest.js reuters-markets 3 --preview（链接为真实文章且 charCount>500 即通过）
\ No newline at end of file

helpers/reuters-world.md

+---
+source: reuters-world
+listSelector: 'div[data-testid="Title"]'
+linkSelector: 'a[href]'
+urlPattern: 'www\.reuters\.com/(world|business|markets|technology|sustainability)/.+-\d{4}-\d{2}-\d{2}/'
+excludeUrlPattern: null
+scrollSteps: 3
+scrollWaitMs: 2000
+waitMs: 15000
+bodySelectors:
+  - '[data-testid^="paragraph"]'
+  - '[class*="articleBodyContent"] p'
+  - '[class*="article-body"] p'
+  - 'article p'
+bodyMinParagraphs: 3
+dateSelector: 'time[datetime]'
+authorSelector: '[rel="author"], [class*="author"] a'
+imageSelector: 'img'
+imageMinWidth: 200
+excludeImage:
+  - logo
+  - icon
+  - avatar
+  - profile
+titleStripSuffix: ''
+maxBodyChars: 15000
+---
+# 路透社 World 抓取备忘
+
+- 主文章流容器 `li.link-group-module__item__Za3t6` 实测 30 个候选链接（inspect 自动生成）。
+- urlPattern `www\.reuters\.com/sitemap/.+` 由样本 URL 推断，如 preview 漏抓/误抓请手编收紧。
+- 首屏可能不全，scrollSteps=3 触发懒加载；反爬严重时调高 waitMs。
+- 验证: node scripts/harvest.js reuters-world 3 --preview（链接为真实文章且 charCount>500 即通过）
\ No newline at end of file

scripts/cdp-client.js

@@ -163,7 +163,21 @@ async function navigateAndWait(ws, url, waitMs = 12000) {
 // Returns the merged array (of parsed objects). This is what lets SPA list
 // pages that lazy-load article cards yield their full set instead of just the
 // first screenful.
-async function scrollAndCollect(ws, { steps = 0, waitMs = 1500, collectExpr, dedupKey }) {
+//
+// Two loop modes (mutually exclusive):
+//   • Fixed steps (default): exactly `steps` scroll iterations, no early exit.
+//     Used by preview mode and any caller that just wants a fixed lazy-load
+//     pass. Backward compatible — no `targetCount` ⇒ this mode.
+//   • Target-count: when `targetCount` is set, scroll until `merged.length >=
+//     targetCount` OR `maxSteps` iterations OR `maxStaleSteps` consecutive
+//     scrolls add nothing (stream exhausted). `steps` is ignored in this mode.
+//     Used by archive harvest so the fetcher can keep loading a lazy stream
+//     until enough candidates exist to satisfy the requested count *after*
+//     dedup, instead of giving up at a fixed `scrollSteps` cap.
+async function scrollAndCollect(ws, {
+  steps = 0, waitMs = 1500, collectExpr, dedupKey,
+  targetCount = null, maxSteps = 10, maxStaleSteps = 3
+}) {
   const keyFn = dedupKey || (item => JSON.stringify(item));
   const seen = new Set();
   const merged = [];
@@ -181,6 +195,23 @@ async function scrollAndCollect(ws, { steps = 0, waitMs = 1500, collectExpr, ded
 
   absorb(await cdpEval(ws, collectExpr));
 
+  // Target-count mode: bounded by maxSteps, exits early on enough items or
+  // on a stale streak (consecutive scrolls adding zero new items ⇒ the lazy
+  // stream has run dry, so more scrolling won't help).
+  if (targetCount && merged.length < targetCount) {
+    let stale = 0;
+    for (let i = 0; i < maxSteps && merged.length < targetCount; i++) {
+      const before = merged.length;
+      await cdpEval(ws, 'window.scrollTo(0, document.body.scrollHeight)');
+      await sleep(waitMs);
+      absorb(await cdpEval(ws, collectExpr));
+      stale = (merged.length === before) ? stale + 1 : 0;
+      if (stale >= maxStaleSteps) break;
+    }
+    return merged;
+  }
+
+  // Fixed-steps mode (default / preview): backward compatible.
   for (let i = 0; i < steps; i++) {
     await cdpEval(ws, 'window.scrollTo(0, document.body.scrollHeight)');
     await sleep(waitMs);

scripts/fetcher-helper.js

@@ -34,7 +34,9 @@ const DEFAULTS = {
   linkSelector: 'a[href]',     // candidate links inside each container
   urlPattern: null,            // REQUIRED — regex source; article URL validation
   excludeUrlPattern: null,     // regex source; URLs to drop even if they match urlPattern
-  scrollSteps: 0,              // bottom-scroll iterations to trigger lazy-load
+  scrollSteps: 0,              // bottom-scroll iterations to trigger lazy-load (fixed-steps mode)
+  maxScrollSteps: 12,          // scroll ceiling in target-count mode (archive harvest tops up
+                               // candidates until dedup can still satisfy the requested count)
   scrollWaitMs: 1500,          // pause after each scroll
   waitMs: 12000,               // initial render wait after load
   maxLinks: 30,
@@ -186,17 +188,34 @@ function buildExtractExpr(spec) {
 // Discover article links on a listing page. Returns absolute URL strings
 // (deduped, query/hash stripped, capped at spec.maxLinks) — the same shape
 // legacy fetchHomeLinks returns so harvest.js needs no changes downstream.
-async function fetchHomeLinks(url, spec) {
+//
+// Optional `desiredCount` (archive harvest): when > 0, switch scrollAndCollect
+// into target-count mode so a lazy stream keeps loading until ~desiredCount
+// candidates exist (or the stream goes stale / hits maxScrollSteps). This lets
+// the harvest top up candidates enough that, after registry dedup, the
+// requested count of *new* articles can still be satisfied — instead of being
+// capped by the fixed `scrollSteps` lazy-load pass. The slice cap is widened
+// to `max(maxLinks, desiredCount)` so the extra candidates aren't discarded.
+async function fetchHomeLinks(url, spec, desiredCount) {
   const { ws, close } = await openSession();
   try {
     await navigateAndWait(ws, url, spec.waitMs);
-    const items = await scrollAndCollect(ws, {
-      steps: spec.scrollSteps,
+    const opts = {
       waitMs: spec.scrollWaitMs,
       collectExpr: buildDiscoveryExpr(spec),
       dedupKey: item => item.url
-    });
-    return items.map(i => i.url).slice(0, spec.maxLinks);
+    };
+    if (desiredCount && desiredCount > 0) {
+      opts.targetCount = desiredCount;
+      opts.maxSteps = spec.maxScrollSteps;
+    } else {
+      opts.steps = spec.scrollSteps;
+    }
+    const items = await scrollAndCollect(ws, opts);
+    const cap = (desiredCount && desiredCount > 0)
+      ? Math.max(spec.maxLinks || 30, desiredCount)
+      : (spec.maxLinks || 30);
+    return items.map(i => i.url).slice(0, cap);
   } finally {
     close();
   }

scripts/harvest.js

@@ -26,6 +26,14 @@ const TEMPLATE_PATH = path.join(SKILL_DIR, 'references', 'config-template.json')
 // Body preview length sent to the agent (full body stays on disk, never in context)
 const PREVIEW_LEN = 4000;
 
+// Upper bound on candidate links to request from a fetcher when topping up.
+// The harvest asks for `count + seen.size + buffer` candidates so registry
+// dedup can still leave `count` *new* articles; this caps that figure so a
+// large 7-day registry can't trigger hundreds of scroll iterations. The
+// fetcher's stale-exit stops early when a lazy stream runs dry, so a high
+// ask on a shallow stream is still cheap.
+const MAX_CANDIDATE_LINKS = 100;
+
 // ── Helpers ─────────────────────────────────────────────────────────────────
 
 function loadConfig() {
@@ -80,7 +88,8 @@ function sentinelFor(slug) {
 }
 
 function originalFilename(titleEn, sourceName) {
-  return `${titleEn} - ${sourceName}原文.md`;
+  const safeTitle = titleEn.replace(/[\\/:*?"<>|]/g, '-').replace(/-+/g, '-').trim();
+  return `${safeTitle} - ${sourceName}原文.md`;
 }
 
 // ── Registry ─────────────────────────────────────────────────────────────────
@@ -179,7 +188,7 @@ ${data.summaryZh || '（摘要生成中）'}
 }
 
 function generateOriginalDoc(data, sourceName, imgFiles) {
-  const { titleEn, titleZh, date, authors, source: url, body } = data;
+  const { titleEn, titleZh, date, authors, url, body } = data;
   const tags = (data.tags || []).map(t => `  - ${t}`).join('\n');
   // Empty-alt guard: only render the caption line when alt text exists, so we
   // never emit a bare `**` (empty bold) for avatar/tracking images.
@@ -396,8 +405,15 @@ Examples:
   // while fetcher-helper takes (url, spec). The closures hide that difference
   // so the main flow below is identical for both paths. NB: spec must NOT be
   // passed to legacy fetchPage (it would be coerced to waitMs=NaN).
-  const getLinks = () => spec
-    ? fetcher.fetchHomeLinks(source.homepageUrl, spec)
+  // Call-site adapters: legacy fetchers take (url, pattern) / (url[, waitMs]),
+  // while fetcher-helper takes (url, spec[, desiredCount]). The closures hide
+  // that difference so the main flow below is identical for both paths.
+  // NB: spec must NOT be passed to legacy fetchPage (it would be coerced to
+  // waitMs=NaN). desiredCount is only threaded into the spec path — legacy
+  // sources do a single-page fetch with no scroll/pagination, so they can't
+  // top up candidates (that's a separate, larger change).
+  const getLinks = (desiredCount) => spec
+    ? fetcher.fetchHomeLinks(source.homepageUrl, spec, desiredCount)
     : fetcher.fetchHomeLinks(source.homepageUrl, source.articleUrlPattern);
   const getPage = (url) => spec
     ? fetcher.fetchPage(url, spec)
@@ -407,16 +423,30 @@ Examples:
   const seen = preview ? new Set() : buildDedupSet(config, sourceId);
   console.error(`   Registry: ${seen.size} URLs in last ${config.registryWindowDays || 7} days`);
 
+  // Ask the fetcher for enough candidate links that, after registry dedup
+  // eats the already-archived ones, `count` NEW articles can still remain.
+  // `count + seen.size` covers the worst case (every seen URL reappears in the
+  // candidates); `+ 10` is a buffer for candidates that fetch but are then
+  // dropped as low-content. Capped so a large registry can't trigger hundreds
+  // of scroll iterations — the fetcher's stale-exit stops a shallow stream
+  // early regardless. Preview stays null (fixed scrollSteps, speed-first).
+  const desiredCount = preview
+    ? null
+    : Math.min(count + seen.size + 10, MAX_CANDIDATE_LINKS);
+  if (!preview && desiredCount > count) {
+    console.error(`   Targeting ${desiredCount} candidates to net ${count} new after dedup`);
+  }
+
   // Fetch homepage links
   console.error(`\n📡 Fetching homepage: ${source.homepageUrl}`);
   let links;
   try {
-    links = await getLinks();
+    links = await getLinks(desiredCount);
   } catch (e) {
     console.error('Failed to fetch homepage:', e.message);
     process.exit(1);
   }
-  console.error(`   Found ${links.length} article links`);
+  console.error(`   Collected ${links.length} candidate links`);
 
   // Normalize URLs (strip query params for dedup)
   const normalizeUrl = (u) => { try { const p = new URL(u); p.search = ''; return p.toString(); } catch(e) { return u; } };
@@ -424,6 +454,9 @@ Examples:
   const newLinks = normalizedLinks.filter(u => !seen.has(u)).slice(0, count);
   const skipped = normalizedLinks.filter(u => seen.has(u)).length;
   console.error(`   New: ${newLinks.length} | Already seen: ${skipped}`);
+  if (!preview && newLinks.length < count) {
+    console.error(`   ⚠️  Only ${newLinks.length} new of ${count} requested — stream may be exhausted (fewer fresh articles than asked). Re-running won't help; wait for new articles or broaden the source.`);
+  }
 
   // ── Preview mode: fetch only, print compact list, write nothing ───────────
   if (preview) {