Commit 30c3b794 authored by 郭阔's avatar 郭阔

feat(news-harvester): panel 重新采集 (refetch by URL)

- platform.sh nh_refetch <url>: run scripts/refetch.js in the skill dir
  with NH_LOCK_DIR wired; refetch resolves the owning source itself.
- web /refetch page (GET form + POST background thread, 5s auto-refresh
  while running, last-run output shown); nav link added.
parent c9c5c5cf
......@@ -445,6 +445,18 @@ nh_quit_instance() { # <source> 回收该源所属站点的共享浏览器实
return 0
}
nh_refetch() { # <url> 单篇重新采集:按 URL 匹配源→站点实例抓取→覆盖原文→单篇重推
local url="$1" tools_dir
[ -n "$url" ] || { echo "用法: nh_refetch <文章URL>"; return 2; }
tools_dir="$(nh_skill_dir)"
if [ ! -f "$tools_dir/scripts/refetch.js" ]; then
echo "❌ 技能缺少 scripts/refetch.js(请更新 news-hub-skills 技能仓)" >&2
return 1
fi
# NH_LOCK_DIR 供脚本判断站点采集锁;PATH 补齐 launchd 环境(node 在 homebrew)
( cd "$tools_dir" && NH_LOCK_DIR="$WORK_DIR/runlocks" node scripts/refetch.js "$url" )
}
nh_login_preflight() { # <source>
local source="$1" creds tools_dir
creds="$(nh_login_creds_file "$source" || true)"
......@@ -454,8 +466,7 @@ nh_login_preflight() { # <source>
echo "⚠️ 已配置 $source 的登录凭据,但技能缺少 login-tools.js,跳过预检" >&2
return 0
fi
nh_ensure_instance "$source"
export NH_LOGIN_USER="$(jq -r '.user // empty' "$creds" 2>/dev/null)"
nh_ensure_instance "$source" export NH_LOGIN_USER="$(jq -r '.user // empty' "$creds" 2>/dev/null)"
export NH_LOGIN_PASS="$(jq -r '.password // empty' "$creds" 2>/dev/null)"
if (cd "$tools_dir" && node scripts/login-tools.js check "$source") >/dev/null 2>&1; then
echo "ℹ️ 登录预检:$source 已登录(端口 ${CDP_PORT_OVERRIDE:-默认})"
......
......@@ -1059,6 +1059,46 @@ def toggle_source(name):
return redirect(url_for("index"))
# ── 单篇重新采集(按 URL:覆盖原文 + 单篇重推) ──────────────────
_refetch_lock = threading.Lock()
_refetch_state = {} # {"running": bool, "done": bool, "rc": int, "url": str, "output": str, "start": str}
@app.route("/refetch", methods=["GET", "POST"])
@require_auth
def refetch_page():
if request.method == "POST":
url = request.form.get("url", "").strip()
if not re.match(r"^https?://\S+$", url):
flash("请输入合法的文章链接(http:// 或 https:// 开头)", "error")
return redirect(url_for("refetch_page"))
with _refetch_lock:
if _refetch_state.get("running"):
flash("已有一次重新采集在进行中,请等它结束(本页自动刷新进度)", "error")
return redirect(url_for("refetch_page"))
_refetch_state.clear()
_refetch_state.update({
"running": True, "done": False, "url": url,
"start": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
})
def _run():
rc, out, err = pl.call("nh_refetch", url, timeout=600)
with _refetch_lock:
_refetch_state.update({
"running": False, "done": True, "rc": rc,
"output": ((out or "") + (("\n" + err) if err else ""))[-4000:],
})
threading.Thread(target=_run, daemon=True).start()
flash("重新采集已启动(约 0.5-2 分钟,页面自动刷新显示结果)", "success")
return redirect(url_for("refetch_page"))
with _refetch_lock:
state = dict(_refetch_state)
return render_template("refetch.html", state=state)
# ── 站点登录凭据管理(站点 = 源名第一段,同站多频道共享一份凭据/配方) ──
_login_test_lock = threading.Lock()
......
......@@ -224,6 +224,7 @@
<a href="{{ url_for('guide') }}">接入指南</a>
<a href="{{ url_for('provider_list') }}">Provider</a>
<a href="{{ url_for('login_list') }}">站点登录</a>
<a href="{{ url_for('refetch_page') }}">重新采集</a>
<a href="{{ url_for('latest_content') }}">最近抓取</a>
</nav>
<div class="container">
......
{% extends "base.html" %}
{% block title %}重新采集 — News Harvester{% endblock %}
{% block extra_head %}
{% if state and state.running %}
<meta http-equiv="refresh" content="5">
{% endif %}
{% endblock %}
{% block content %}
<h1>重新采集(按链接)</h1>
<p style="color:#666;font-size:13px">
粘贴一篇文章的原文链接,系统会<strong>重新抓取该篇正文与配图</strong>:已归档过的文章覆盖原
<code>原文.md</code>(保留中文摘要)、更新 registry/index,并<strong>单篇重推 News Hub</strong>
(后端按 sourceId+url 幂等更新,版本 +1);未归档过的链接作为新文章写入今天目录
(待补中文摘要后随日目录推送)。浏览器源走该站共享实例,站点正在采集时会拒绝执行,稍等重试即可。
</p>
{% with messages = get_flashed_messages() %}
{% if messages %}{% for m in messages %}<div class="flash">{{ m }}</div>{% endfor %}{% endif %}
{% endwith %}
<div class="card" style="max-width:860px">
<form method="POST" action="{{ url_for('refetch_page') }}" style="display:flex;gap:8px;flex-wrap:wrap">
<input type="text" name="url" placeholder="https://www.reuters.com/technology/…" required
style="flex:1;min-width:340px" value="{{ state.url or '' }}">
<button type="submit">🔁 重新采集</button>
</form>
</div>
{% if state %}
<div class="card" style="max-width:860px">
<h2 style="margin-top:0">上次执行</h2>
<p>
<strong>链接:</strong> {{ state.url }}<br>
<strong>开始:</strong> {{ state.start or '—' }}
{% if state.running %} · <span style="color:#e6a23c">⏳ 进行中…(每 5 秒自动刷新)</span>
{% elif state.done %}
{% if state.rc == 0 %} · <span style="color:#67c23a">✅ 成功</span>
{% else %} · <span style="color:#f56c6c">✗ 失败(退出码 {{ state.rc }})</span>{% endif %}
{% endif %}
</p>
{% if state.output %}
<pre style="white-space:pre-wrap;background:#1e1e1e;color:#d4d4d4;padding:12px;border-radius:6px;font-size:12px;max-height:420px;overflow:auto">{{ state.output }}</pre>
{% endif %}
</div>
{% endif %}
{% endblock %}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment