Commit c1b3c711 authored by 郭阔's avatar 郭阔

feat(news-harvester): single-article refetch by URL

- refetch.js: fetch one article via its source recipe on the site-shared
  browser instance; refresh 原文.md in place for already-archived articles
  (registry ownership-checked, index rebuilt from registry) and re-push
  via push.js --url; new URLs archive to today's dir pending summaries.
  Busy site locks are respected; only self-launched instances are recycled.
- push.js: --url <articleUrl> re-pushes a single article (query/hash
  normalized match, ledger ignored for it); recordResult now appends a
  ledger entry when the article postdates the ledger instead of silently
  dropping a successful push result (refetch-into-finalized-day case).
- harvest.js: export downloadImage/normalizeUrl for refetch.
parent 3a9b3ea2
......@@ -791,6 +791,19 @@ Examples:
console.error(` ✗ Fetch failed: ${e.message}`);
continue;
}
if (articleData && articleData.isArticle === false) {
// Lazy-rendered pages often miss og:type/body on first paint — wait 5s
// and reload once before judging it a non-article page.
console.error(' ⏳ 首载未过文章校验 — 等 5 秒重载一次再判');
await sleep(5000);
try {
const reloaded = await getPage(url);
if (reloaded) articleData = reloaded;
} catch (e) {
if (e instanceof HumanInterventionError) throw e;
console.error(` ⚠️ 重载失败(保留首载结果再判): ${e.message}`);
}
}
const articleInfo = await processArticle(url, articleData);
if (articleInfo) {
processed.push(articleInfo);
......@@ -881,5 +894,7 @@ module.exports = {
slugFromUrl,
sentinelFor,
originalFilename,
dateShort
dateShort,
downloadImage,
normalizeUrl
};
......@@ -69,6 +69,13 @@ function loadJson(p, label) {
function sleep(ms) { return new Promise(r => setTimeout(r, ms)); }
// Canonical URL key for matching a registry article regardless of tracking
// params: strip query string and hash (same normalization harvest.js applies).
function normKey(u) {
try { const p = new URL(u); p.search = ''; p.hash = ''; return p.toString(); }
catch (e) { return u; }
}
function uuid() {
return crypto.randomUUID();
}
......@@ -418,8 +425,15 @@ function newSyncState(registry) {
// Mark a url's result in the ledger.
function recordResult(state, url, result) {
const entry = state.articles.find(a => a.url === url);
if (!entry) return;
let entry = state.articles.find(a => a.url === url);
if (!entry) {
// Articles appended to the registry AFTER the ledger was created (e.g.
// refetch.js adding one to an already-synced day dir) have no entry yet.
// Create it here instead of dropping the result — otherwise a successful
// push is never recorded and the article re-pushes on every later run.
entry = { url, slug: null, status: 'pending', articleId: null, version: null, error: null };
state.articles.push(entry);
}
if (result.status === 'error') {
entry.status = 'error';
entry.error = result.error || null;
......@@ -478,10 +492,26 @@ async function pushDayDir(dayDir, cfg, opts) {
state = newSyncState(registry);
console.error(' --force: ledger reset, re-pushing all articles.');
}
const pending = articles.filter(a => {
let pending;
if (opts.url) {
// --url: single-article re-push (refetch flow). Ignores the ledger status
// of that one article (it is being re-pushed deliberately); the rest of
// the day's ledger stays untouched. Comparison strips query/hash — the
// same normalization harvest.js applies to registry keys.
const key = normKey(opts.url);
pending = articles.filter(a => normKey(a.url) === key);
if (pending.length === 0) {
console.error(`✗ --url: no registry article matches ${opts.url}`);
console.error(' (registry URLs are compared with query/hash stripped)');
process.exit(1);
}
console.error(` --url: re-pushing ${pending.length} matching article (ledger ignored for it)`);
} else {
pending = articles.filter(a => {
const e = state.articles.find(x => x.url === a.url);
return !e || e.status !== 'synced';
});
}
if (pending.length === 0) {
console.error(' ✅ All articles already synced — nothing to do.');
console.error(' (use --force to re-push)');
......@@ -697,6 +727,10 @@ function printHelp() {
Usage:
node scripts/push.js <dayDir> Push a day directory (incremental)
node scripts/push.js <dayDir> --force Ignore ledger, re-push all articles
node scripts/push.js <dayDir> --url <articleUrl>
Re-push a single article (matched with query/hash
stripped); its ledger status is ignored, the rest of
the day's ledger is untouched (refetch flow)
node scripts/push.js <dayDir> --bump-processed-at
Override processedAt to now() so the backend's LWW
accepts the update (use when re-pushing corrected
......@@ -740,14 +774,23 @@ function main() {
process.exit(0);
}
const positional = args.filter(a => !a.startsWith('-'));
const flags = args.filter(a => a.startsWith('-'));
// --url <url>: its value is a positional-looking string (starts with http),
// so pull the pair out BEFORE splitting positional/flags.
const urlIdx = args.indexOf('--url');
const urlFilter = urlIdx >= 0 ? (args[urlIdx + 1] || null) : null;
const skipIdx = urlIdx >= 0 ? urlIdx + 1 : -1;
const positional = args.filter((a, i) => !a.startsWith('-') && i !== skipIdx);
const flags = args.filter((a, i) => a.startsWith('-') && i !== skipIdx);
if (positional.length === 0) {
console.error('Usage: node scripts/push.js <dayDir> [--force] [--bump-processed-at] [--status]');
console.error('Usage: node scripts/push.js <dayDir> [--force] [--bump-processed-at] [--status] [--url <articleUrl>]');
console.error('Run "node scripts/push.js --help" for details.');
process.exit(1);
}
if (urlIdx >= 0 && !urlFilter) {
console.error('✗ --url requires a value: --url https://...');
process.exit(1);
}
const dayDir = positional[0];
if (!fs.existsSync(dayDir) || !fs.statSync(dayDir).isDirectory()) {
......@@ -776,6 +819,7 @@ function main() {
pushDayDir(dayDir, cfg, {
force: flags.includes('--force'),
bumpProcessedAt: flags.includes('--bump-processed-at'),
url: urlFilter,
}).catch(e => {
console.error(`Fatal: ${e.message}`);
process.exit(1);
......
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment