feat(news-harvester): single-article refetch by URL

- refetch.js: fetch one article via its source recipe on the site-shared
  browser instance; refresh 原文.md in place for already-archived articles
  (registry ownership-checked, index rebuilt from registry) and re-push
  via push.js --url; new URLs archive to today's dir pending summaries.
  Busy site locks are respected; only self-launched instances are recycled.
- push.js: --url <articleUrl> re-pushes a single article (query/hash
  normalized match, ledger ignored for it); recordResult now appends a
  ledger entry when the article postdates the ledger instead of silently
  dropping a successful push result (refetch-into-finalized-day case).
- harvest.js: export downloadImage/normalizeUrl for refetch.

skills/news-harvester/scripts/harvest.js

@@ -791,6 +791,19 @@ Examples:
         console.error(`  ✗ Fetch failed: ${e.message}`);
         continue;
       }
+      if (articleData && articleData.isArticle === false) {
+        // Lazy-rendered pages often miss og:type/body on first paint — wait 5s
+        // and reload once before judging it a non-article page.
+        console.error('  ⏳ 首载未过文章校验 — 等 5 秒重载一次再判');
+        await sleep(5000);
+        try {
+          const reloaded = await getPage(url);
+          if (reloaded) articleData = reloaded;
+        } catch (e) {
+          if (e instanceof HumanInterventionError) throw e;
+          console.error(`  ⚠️ 重载失败（保留首载结果再判）: ${e.message}`);
+        }
+      }
       const articleInfo = await processArticle(url, articleData);
       if (articleInfo) {
         processed.push(articleInfo);
@@ -881,5 +894,7 @@ module.exports = {
   slugFromUrl,
   sentinelFor,
   originalFilename,
-  dateShort
+  dateShort,
+  downloadImage,
+  normalizeUrl
 };

skills/news-harvester/scripts/push.js

@@ -69,6 +69,13 @@ function loadJson(p, label) {
 
 function sleep(ms) { return new Promise(r => setTimeout(r, ms)); }
 
+// Canonical URL key for matching a registry article regardless of tracking
+// params: strip query string and hash (same normalization harvest.js applies).
+function normKey(u) {
+  try { const p = new URL(u); p.search = ''; p.hash = ''; return p.toString(); }
+  catch (e) { return u; }
+}
+
 function uuid() {
   return crypto.randomUUID();
 }
@@ -418,8 +425,15 @@ function newSyncState(registry) {
 
 // Mark a url's result in the ledger.
 function recordResult(state, url, result) {
-  const entry = state.articles.find(a => a.url === url);
-  if (!entry) return;
+  let entry = state.articles.find(a => a.url === url);
+  if (!entry) {
+    // Articles appended to the registry AFTER the ledger was created (e.g.
+    // refetch.js adding one to an already-synced day dir) have no entry yet.
+    // Create it here instead of dropping the result — otherwise a successful
+    // push is never recorded and the article re-pushes on every later run.
+    entry = { url, slug: null, status: 'pending', articleId: null, version: null, error: null };
+    state.articles.push(entry);
+  }
   if (result.status === 'error') {
     entry.status = 'error';
     entry.error = result.error || null;
@@ -478,10 +492,26 @@ async function pushDayDir(dayDir, cfg, opts) {
     state = newSyncState(registry);
     console.error('   --force: ledger reset, re-pushing all articles.');
   }
-  const pending = articles.filter(a => {
-    const e = state.articles.find(x => x.url === a.url);
-    return !e || e.status !== 'synced';
-  });
+  let pending;
+  if (opts.url) {
+    // --url: single-article re-push (refetch flow). Ignores the ledger status
+    // of that one article (it is being re-pushed deliberately); the rest of
+    // the day's ledger stays untouched. Comparison strips query/hash — the
+    // same normalization harvest.js applies to registry keys.
+    const key = normKey(opts.url);
+    pending = articles.filter(a => normKey(a.url) === key);
+    if (pending.length === 0) {
+      console.error(`✗ --url: no registry article matches ${opts.url}`);
+      console.error('  (registry URLs are compared with query/hash stripped)');
+      process.exit(1);
+    }
+    console.error(`   --url: re-pushing ${pending.length} matching article (ledger ignored for it)`);
+  } else {
+    pending = articles.filter(a => {
+      const e = state.articles.find(x => x.url === a.url);
+      return !e || e.status !== 'synced';
+    });
+  }
   if (pending.length === 0) {
     console.error('   ✅ All articles already synced — nothing to do.');
     console.error('   (use --force to re-push)');
@@ -697,6 +727,10 @@ function printHelp() {
 Usage:
   node scripts/push.js <dayDir>              Push a day directory (incremental)
   node scripts/push.js <dayDir> --force      Ignore ledger, re-push all articles
+  node scripts/push.js <dayDir> --url <articleUrl>
+                          Re-push a single article (matched with query/hash
+                          stripped); its ledger status is ignored, the rest of
+                          the day's ledger is untouched (refetch flow)
   node scripts/push.js <dayDir> --bump-processed-at
                           Override processedAt to now() so the backend's LWW
                           accepts the update (use when re-pushing corrected
@@ -740,14 +774,23 @@ function main() {
     process.exit(0);
   }
 
-  const positional = args.filter(a => !a.startsWith('-'));
-  const flags = args.filter(a => a.startsWith('-'));
+  // --url <url>: its value is a positional-looking string (starts with http),
+  // so pull the pair out BEFORE splitting positional/flags.
+  const urlIdx = args.indexOf('--url');
+  const urlFilter = urlIdx >= 0 ? (args[urlIdx + 1] || null) : null;
+  const skipIdx = urlIdx >= 0 ? urlIdx + 1 : -1;
+  const positional = args.filter((a, i) => !a.startsWith('-') && i !== skipIdx);
+  const flags = args.filter((a, i) => a.startsWith('-') && i !== skipIdx);
 
   if (positional.length === 0) {
-    console.error('Usage: node scripts/push.js <dayDir> [--force] [--bump-processed-at] [--status]');
+    console.error('Usage: node scripts/push.js <dayDir> [--force] [--bump-processed-at] [--status] [--url <articleUrl>]');
     console.error('Run "node scripts/push.js --help" for details.');
     process.exit(1);
   }
+  if (urlIdx >= 0 && !urlFilter) {
+    console.error('✗ --url requires a value: --url https://...');
+    process.exit(1);
+  }
 
   const dayDir = positional[0];
   if (!fs.existsSync(dayDir) || !fs.statSync(dayDir).isDirectory()) {
@@ -776,6 +819,7 @@ function main() {
   pushDayDir(dayDir, cfg, {
     force: flags.includes('--force'),
     bumpProcessedAt: flags.includes('--bump-processed-at'),
+    url: urlFilter,
   }).catch(e => {
     console.error(`Fatal: ${e.message}`);
     process.exit(1);

skills/news-harvester/scripts/refetch.js

+#!/usr/bin/env node
+/**
+ * refetch.js - 单篇重新采集(按 URL)
+ *
+ * 用途:某篇文章抓取不完整/被付费墙截断/图片缺失时,粘贴原文链接重新抓一遍。
+ *
+ *   node scripts/refetch.js <article-url>
+ *
+ * 流程:
+ *   1. URL 定位归属:先在近 30 天各源 registry.json 里按 URL 找已归档记录
+ *      (query/hash 归一化后比较);找不到再按 hostname 匹配 config 源。
+ *   2. 浏览器源:按站点共享实例规则(端口=9300+hash(site)%200,profile=
+ *      ~/.news-harvester/profiles/<site>)拉起/复用 Chromium 再抓;direct 源
+ *      直接 HTTP 抓。站点有采集任务持锁时拒绝执行(避免与采集抢实例)。
+ *   3. 已在库 → 覆盖该日目录的 原文.md(重下图到 assets/)、原位更新 registry
+ *      条目(保留中文标题/分类/标签)、按 registry 重建 index.md,然后
+ *      push.js --url 单篇重推(后端按 sourceId+url upsert,版本 +1)。
+ *      不在库 → 归档到今天的日目录(sentinel 标题,待摘要),不推送。
+ *
+ * 退出码:0 成功;2 抓取失败;3 非文章/内容过短;4 无法匹配源;5 站点忙;
+ *         70 需要人工介入(人机验证/登录失效,同 harvest.js 语义)。
+ */
+const fs = require('fs');
+const path = require('path');
+const os = require('os');
+const { spawnSync } = require('child_process');
+const { HumanInterventionError } = require('./block-check');
+
+const SKILL_DIR = path.dirname(__dirname);
+const CONFIG_PATH = path.join(SKILL_DIR, 'config.json');
+
+// Registry 搜索窗口(比去重窗口宽:重采的往往是几天前的文章)
+const SEARCH_WINDOW_DAYS = 30;
+
+function loadConfig() {
+  return JSON.parse(fs.readFileSync(CONFIG_PATH, 'utf8'));
+}
+
+// URL 归一化键:去 query、去 hash(harvest.js normalizeUrl 只去 query;
+// fetcher 发现链接时 query+hash 都去,这里取更宽的口径保证两侧都能命中)
+function normKey(u) {
+  try { const p = new URL(u); p.search = ''; p.hash = ''; return p.toString(); }
+  catch (e) { return u; }
+}
+
+// 与 platform.sh nh_cdp_port 完全一致的站点端口散列(9300 + hash%200)
+function sitePort(site) {
+  let s = 0;
+  for (const ch of site) s = (s * 31 + ch.codePointAt(0)) % 200;
+  return 9300 + s;
+}
+
+function siteOf(sourceId) { return sourceId.split('-')[0]; }
+
+function fail(code, msg) {
+  console.error(msg);
+  process.exit(code);
+}
+
+// ── 定位:先查 registry,再按 hostname 匹配 ─────────────────────────────────
+
+// 遍历一个源(vaultFolder 含 "/" 时是其主目录下所有栏目子目录)近 N 天的
+// registry.json,返回 { source, dayDir, reg, idx } 或 null。
+function findInRegistry(config, url) {
+  const key = normKey(url);
+  const today = new Date();
+  const days = [];
+  for (let i = 0; i < SEARCH_WINDOW_DAYS; i++) {
+    const d = new Date(today); d.setDate(today.getDate() - i);
+    days.push(`${d.getFullYear()}${String(d.getMonth() + 1).padStart(2, '0')}${String(d.getDate()).padStart(2, '0')}`);
+  }
+  for (const source of config.sources) {
+    const parts = source.vaultFolder.split('/');
+    let roots;
+    if (parts.length > 1) {
+      const primary = path.join(config.vaultPath, parts[0]);
+      if (!fs.existsSync(primary)) continue;
+      roots = fs.readdirSync(primary)
+        .filter(sub => fs.statSync(path.join(primary, sub)).isDirectory())
+        .map(sub => path.join(primary, sub));
+    } else {
+      const folder = path.join(config.vaultPath, source.vaultFolder);
+      if (!fs.existsSync(folder)) continue;
+      roots = [folder];
+    }
+    for (const root of roots) {
+      for (const day of days) {
+        const regPath = path.join(root, day, 'registry.json');
+        if (!fs.existsSync(regPath)) continue;
+        try {
+          const reg = JSON.parse(fs.readFileSync(regPath, 'utf8'));
+          // Ownership check: multi-channel folders (路透社/科技 etc.) are scanned
+          // by EVERY channel of that site (vaultFolder primary traversal) —
+          // without this check the first sibling channel in config order
+          // (e.g. reuters-world) claims articles archived by another channel
+          // (reuters-technology), writing wrong publisher/filenames.
+          if (reg.source && reg.source !== source.id) continue;
+          const idx = (reg.articles || []).findIndex(a => normKey(a.url) === key);
+          if (idx >= 0) return { source, dayDir: path.join(root, day), reg, regPath, idx };
+        } catch (e) { /* 坏 registry 跳过 */ }
+      }
+    }
+  }
+  return null;
+}
+
+// hostname 匹配:同站多频道(reuters-*)时,优先 URL 路径与频道 homepageUrl
+// 首段路径一致的(如 /technology/... 归 reuters-technology,不落 markets 目录)
+function matchByHost(config, url) {
+  let u;
+  try { u = new URL(url); } catch (e) { return null; }
+  const host = u.hostname.replace(/^www\./, '');
+  let best = null, bestScore = -1;
+  for (const s of config.sources) {
+    let h = null, primaryPath = '';
+    try {
+      const hp = new URL(s.homepageUrl);
+      h = hp.hostname.replace(/^www\./, '');
+      primaryPath = (hp.pathname.split('/').filter(Boolean)[0] || '').toLowerCase();
+    } catch (e) {}
+    if (!h || h !== host) continue;
+    const hasRecipe = fs.existsSync(path.join(SKILL_DIR, 'helpers', `${s.id}.md`));
+    const pathAffinity = primaryPath && u.pathname.toLowerCase().includes(`/${primaryPath}/`) ? 2 : 0;
+    const score = (s.enabled ? 2 : 0) + (hasRecipe ? 1 : 0) + pathAffinity;
+    if (score > bestScore) { best = s; bestScore = score; }
+  }
+  return best;
+}
+
+// ── 主流程 ───────────────────────────────────────────────────────────────────
+
+async function main() {
+  const url = process.argv[2];
+  if (!url || url.startsWith('-')) {
+    console.error('Usage: node scripts/refetch.js <article-url>');
+    console.error('Example: node scripts/refetch.js https://www.reuters.com/technology/some-article-2026-11-13/');
+    process.exit(2);
+  }
+  const config = loadConfig();
+
+  // 1. 归属定位
+  const existing = findInRegistry(config, url);
+  const source = existing ? existing.source : matchByHost(config, url);
+  if (!source) fail(4, `❌ 无法匹配数据源:${url}\n   (先在近 ${SEARCH_WINDOW_DAYS} 天归档记录里查,再按域名对 config.json 各源 homepageUrl 匹配,均未命中)`);
+
+  const {
+    generateOriginalDoc, updateIndex, appendRegistry, getCategory,
+    slugFromUrl, sentinelFor, originalFilename, downloadImage,
+  } = require('./harvest.js');
+
+  // 2. 站点实例与锁(浏览器源)
+  const helperPath = path.join(SKILL_DIR, 'helpers', `${source.id}.md`);
+  const useBrowser = source.method === 'browser' || fs.existsSync(helperPath);
+  if (useBrowser) {
+    const site = siteOf(source.id);
+    const port = sitePort(site);
+    const profile = path.join(os.homedir(), '.news-harvester', 'profiles', site);
+
+    // 站点正被采集占用:拒绝(采集本身就是大量访问,插单反而互相拖慢)
+    const lockDir = process.env.NH_LOCK_DIR;
+    if (lockDir && fs.existsSync(lockDir)) {
+      const busy = fs.readdirSync(lockDir).some(f => f.startsWith(`port-${port}.`));
+      if (busy) fail(5, `⏸ 站点 ${site}(端口 ${port})有采集任务正在进行,请稍后重试。\n   (可在面板「最近抓取」页查看进度)`);
+    }
+
+    process.env.CDP_PORT_OVERRIDE = String(port);
+    process.env.CDP_PROFILE_OVERRIDE = profile;
+    const { ensureChromium, cdpVersion } = require('./ensure-chromium');
+    console.error(`🔁 重新采集 (源 ${source.id} · 站点实例 :${port})`);
+    try { await cdpVersion(port, 1500); console.error('   CDP 实例已在线,复用'); }
+    catch (e) {
+      console.error('   拉起站点共享实例...');
+      const r = await ensureChromium(config);
+      _startedByUs = true;
+      console.error(`   CDP ready on :${r.port} (pid ${r.pid})`);
+    }
+  } else {
+    console.error(`🔁 重新采集 (源 ${source.id} · direct HTTP)`);
+  }
+
+  // 3. 抓单篇
+  let data;
+  const fetchOne = async () => {
+    if (fs.existsSync(helperPath)) {
+      const fetcherHelper = require('./fetcher-helper');
+      const spec = fetcherHelper.loadSpec(helperPath);
+      return fetcherHelper.fetchPage(url, spec);
+    }
+    const fetcher = require('./fetcher-direct');
+    return fetcher.fetchPage(url);
+  };
+  try {
+    data = await fetchOne();
+  } catch (e) {
+    if (e instanceof HumanInterventionError) {
+      recycleInstance();
+      fail(70, `🚫 ${e.message}\n   建议:面板「站点登录」→ 人工登录/过一次验证后重试。`);
+    }
+    recycleInstance();
+    fail(2, `❌ 抓取失败:${e.message}`);
+  }
+  recycleInstance();
+
+  if (!data || !data.title) fail(3, '❌ 页面无标题,可能不是文章页');
+  if (data.isArticle === false) {
+    // 重采是用户显式指令:校验不过(如付费墙截短正文)也尽量收录,仅提示
+    console.error(`   ⚠️  页面校验未过(isArticle=false,正文 ${((data.body || '').length)} 字符),按重采指令继续收录`);
+  }
+  if ((data.body || '').length < 100) fail(3, `❌ 正文过短(${(data.body || '').length} 字符),疑似被付费墙/反爬截断`);
+
+  // 4. 落盘
+  const titleEn = data.title;
+  const origName = originalFilename(titleEn, source.name);
+  let dayDir, regPath, reg, idx;
+
+  if (existing) {
+    dayDir = existing.dayDir; regPath = existing.regPath;
+    reg = existing.reg; idx = existing.idx;
+    console.error(`   已归档于 ${path.basename(dayDir)} — 覆盖原文并重推`);
+  } else {
+    const today = `${new Date().getFullYear()}${String(new Date().getMonth() + 1).padStart(2, '0')}${String(new Date().getDate()).padStart(2, '0')}`;
+    dayDir = path.join(config.vaultPath, source.vaultFolder, today);
+    regPath = path.join(dayDir, 'registry.json');
+    reg = { date: today, source: source.id, articles: [] };
+    if (fs.existsSync(regPath)) { try { reg = JSON.parse(fs.readFileSync(regPath, 'utf8')); } catch (e) {} }
+    idx = -1;
+    console.error(`   未归档过 — 作为新文章写入今天目录 ${today}(待摘要,不推送)`);
+  }
+
+  const assetsDir = path.join(dayDir, 'assets');
+  fs.mkdirSync(assetsDir, { recursive: true });
+
+  const entry = idx >= 0 ? reg.articles[idx] : null;
+  const slug = (entry && entry.slug) || slugFromUrl(url);
+  const titleZh = (entry && entry.titleZh) || sentinelFor(slug);
+
+  const imgs = data.imgs || [];
+  const imgFiles = [];
+  for (let j = 0; j < imgs.length; j++) {
+    const filename = `${slug}-${j + 1}.jpg`;
+    const ok = await downloadImage(imgs[j].src, path.join(assetsDir, filename));
+    if (ok) imgFiles.push({ filename, alt: imgs[j].alt, src: imgs[j].src });
+  }
+
+  const articleInfo = {
+    url, titleEn, titleZh,
+    date: data.date || (entry && entry.date) || '',
+    authors: data.authors || '',
+    body: data.body,
+    imgs: imgFiles,
+    category: (entry && entry.category) || getCategory(url, titleEn),
+    tags: (entry && entry.tags) || [source.name],
+    slug, originalFilename: origName, sourceName: source.name,
+  };
+  fs.writeFileSync(path.join(dayDir, origName), generateOriginalDoc(articleInfo, source.name, imgFiles));
+
+  // 标题变了 → 旧原文文件清掉,避免同一天目录里新旧两份
+  const oldName = entry && entry.originalFilename;
+  if (oldName && oldName !== origName) {
+    try { fs.unlinkSync(path.join(dayDir, oldName)); } catch (e) {}
+  }
+
+  const record = {
+    url, titleZh,
+    titleEn,
+    date: articleInfo.date,
+    processedAt: new Date().toISOString(),
+    category: articleInfo.category, tags: articleInfo.tags, slug,
+    originalFilename: origName,
+  };
+  if (idx >= 0) reg.articles[idx] = record;
+  else reg.articles.push(record);
+  fs.writeFileSync(regPath, JSON.stringify(reg, null, 2));
+
+  // index.md 是全量重建:用(更新后的)整日 registry 列表喂给它
+  updateIndex(dayDir, reg.articles.map(e => ({
+    titleZh: e.titleZh, titleEn: e.titleEn, category: e.category,
+    tags: e.tags, date: e.date,
+  })), source.name);
+
+  console.error(`  ✓ ${titleEn}`);
+  console.error(`    正文 ${data.body.length} 字符 | 图片 ${imgFiles.length}/${imgs.length} 张 → ${origName}`);
+
+  // 5. 重推(仅已归档过的文章;新文章等摘要完成后再走正常 finalize/push)
+  if (existing) {
+    console.error('\n📤 单篇重推 News Hub...');
+    const r = spawnSync('node', [path.join(SKILL_DIR, 'scripts', 'push.js'), dayDir,
+      '--url', url, '--bump-processed-at'], { stdio: 'inherit' });
+    if (r.status !== 0) {
+      console.error(`⚠️ 重推退出码 ${r.status} — 原文已更新,推送未完成;可稍后手动:`);
+      console.error(`   node scripts/push.js "${dayDir}" --url "${url}" --bump-processed-at`);
+      process.exit(2);
+    }
+    console.error(`\n✅ 重新采集完成:原文已覆盖、registry/index 已更新、已重推 News Hub`);
+  } else {
+    console.error(`\n✅ 重新采集完成:原文已归档(新文章)`);
+    console.error(`   ⏳ 待中文摘要:该文章尚未生成摘要,日目录在补齐摘要前不会整体推送。`);
+    console.error(`      补摘要流程与日常采集一致(summary-add → finalize)。`);
+  }
+}
+
+// 回收自己拉起的站点实例(复用中的实例不动;有采集任务接管时跳过,
+// 同 keepalive 语义)。只对 startedByUs 的实例生效。
+let _startedByUs = false;
+function recycleInstance() {
+  if (!_startedByUs) return;
+  const port = process.env.CDP_PORT_OVERRIDE;
+  if (!port) return;
+  const lockDir = process.env.NH_LOCK_DIR;
+  if (lockDir && fs.existsSync(lockDir)) {
+    const busy = fs.readdirSync(lockDir).some(f => f.startsWith(`port-${port}.`));
+    if (busy) { console.error('   采集任务已接管实例,跳过回收'); return; }
+  }
+  const r = spawnSync('node', [path.join(__dirname, 'instance-quit.js')], { stdio: 'ignore' });
+  if (r.status === 0) console.error('   站点实例已回收');
+}
+
+main().catch(e => { recycleInstance(); console.error('Fatal:', e); process.exit(2); });