Commit a17c0314 authored by 郭阔's avatar 郭阔

feat: site-level login credentials shared across channels

- platform.sh: credential lookup falls back site-level (logins/<site>.json,
  site = first segment of source name); nikkei-world/markets/tech share one
  nikkei account, single file to maintain
- platform_layer: login_sites()/login_rows() site view (coverage, probe
  source, recipe status) for the panel
- web: /logins page now site-based — save/delete/discover/test operate per
  site; discovery probes via the site's representative source
parent acac322d
...@@ -378,9 +378,23 @@ nh_skill_bootstrap() { # clone(可选)+ 依赖安装 + 全量 bundle 导入 + v ...@@ -378,9 +378,23 @@ nh_skill_bootstrap() { # clone(可选)+ 依赖安装 + 全量 bundle 导入 + v
} }
# --- 登录预检(配置了凭据的浏览器源,采集前自动检查/登录) --- # --- 登录预检(配置了凭据的浏览器源,采集前自动检查/登录) ---
# 凭据文件:$CONF_DIR/logins/<source>.json({"url":…,"user":…,"password":…},建议 600)。 # 凭据文件:$CONF_DIR/logins/<site>.json({"url":…,"user":…,"password":…},建议 600)。
# 站点 = 源名第一段(nikkei-world/markets/tech → nikkei;wsj/ft 即自身),
# 同一站点的所有频道共享一份凭据;找不到站点文件再回退源级文件。
# 配方与浏览器自动化在技能仓 login-tools.js;本层只做编排,登录失败不中断采集 # 配方与浏览器自动化在技能仓 login-tools.js;本层只做编排,登录失败不中断采集
# (结果契约会如实标记付费墙)。 # (结果契约会如实标记付费墙)。
nh_login_site() { # <source> → 输出站点名
echo "${1%%-*}"
}
nh_login_creds_file() { # <source> → 输出凭据文件路径(站点级优先,源级兜底)
local source="$1" site f
site="$(nh_login_site "$source")"
for f in "$CONF_DIR/logins/${site}.json" "$CONF_DIR/logins/${source}.json"; do
[ -f "$f" ] && { echo "$f"; return 0; }
done
return 1
}
nh_ensure_instance() { # <source> 确保该源专属浏览器实例在线(已在线为无操作) nh_ensure_instance() { # <source> 确保该源专属浏览器实例在线(已在线为无操作)
local source="$1" local source="$1"
if nh_is_darwin && [ -z "${CDP_PORT_OVERRIDE:-}" ]; then if nh_is_darwin && [ -z "${CDP_PORT_OVERRIDE:-}" ]; then
...@@ -394,7 +408,7 @@ nh_ensure_instance() { # <source> 确保该源专属浏览器实例在线(已 ...@@ -394,7 +408,7 @@ nh_ensure_instance() { # <source> 确保该源专属浏览器实例在线(已
nh_login_preflight() { # <source> nh_login_preflight() { # <source>
local source="$1" creds tools_dir local source="$1" creds tools_dir
creds="$CONF_DIR/logins/${source}.json" creds="$(nh_login_creds_file "$source" || true)"
[ -f "$creds" ] || return 0 [ -f "$creds" ] || return 0
tools_dir="$(nh_skill_dir)" tools_dir="$(nh_skill_dir)"
if [ ! -f "$tools_dir/scripts/login-tools.js" ]; then if [ ! -f "$tools_dir/scripts/login-tools.js" ]; then
...@@ -419,8 +433,8 @@ nh_login_preflight() { # <source> ...@@ -419,8 +433,8 @@ nh_login_preflight() { # <source>
nh_login_autodiscover() { # <source> 自动发现登录流程生成配方,并立即实登验证 nh_login_autodiscover() { # <source> 自动发现登录流程生成配方,并立即实登验证
local source="$1" creds tools_dir local source="$1" creds tools_dir
creds="$CONF_DIR/logins/${source}.json" creds="$(nh_login_creds_file "$source" || true)"
[ -f "$creds" ] || { echo "❌ 未配置 $source 的凭据(看板「站点登录」页维护)" >&2; return 1; } [ -n "$creds" ] || { echo "❌ 未配置 $(nh_login_site "$source") 的凭据(看板「站点登录」页维护)" >&2; return 1; }
tools_dir="$(nh_skill_dir)" tools_dir="$(nh_skill_dir)"
[ -f "$tools_dir/scripts/login-tools.js" ] || { echo "❌ 技能缺少 login-tools.js" >&2; return 1; } [ -f "$tools_dir/scripts/login-tools.js" ] || { echo "❌ 技能缺少 login-tools.js" >&2; return 1; }
nh_ensure_instance "$source" nh_ensure_instance "$source"
......
...@@ -194,11 +194,29 @@ def skill_set_enabled(name, enabled): ...@@ -194,11 +194,29 @@ def skill_set_enabled(name, enabled):
return 1, "", str(e) return 1, "", str(e)
# ── 站点登录凭据($CONF_DIR/logins/<source>.json,600) ────── # ── 站点登录凭据($CONF_DIR/logins/<site>.json,600) ──────
# 站点 = 源名第一段:nikkei-world/markets/tech 同属 nikkei,共享一份凭据与配方;
# 单段源(wsj/ft)站点即自身。同一账号的多频道只维护一次。
def login_site(source):
return str(source).split("-")[0]
def login_conf_path(source): def login_conf_path(source):
return Path(os.environ.get("CONF_DIR", default_conf_dir(default_work_dir()))) / "logins" / f"{source}.json" return Path(os.environ.get("CONF_DIR", default_conf_dir(default_work_dir()))) / "logins" / f"{source}.json"
def login_sites():
"""站点 → 该站点覆盖的已接入源列表(来自技能源 ∪ 调度)。"""
sites = {}
try:
names = set(skill_sources()) | set(list_schedules())
except Exception: # noqa: BLE001
names = set()
for name in sorted(names):
sites.setdefault(login_site(name), []).append(name)
return sites
def get_logins(): def get_logins():
"""列出已配置的登录凭据(密码永不回显)。""" """列出已配置的登录凭据(密码永不回显)。"""
result = {} result = {}
...@@ -242,13 +260,39 @@ def delete_login(source): ...@@ -242,13 +260,39 @@ def delete_login(source):
def login_recipes(): def login_recipes():
"""技能仓里已有登录配方的源集合(logins/<source>.json)。""" """技能仓里已有登录配方的站点/源集合(logins/<name>.json)。"""
recipes = skills_root() / "skills" / "news-harvester" / "logins" recipes = skills_root() / "skills" / "news-harvester" / "logins"
if not recipes.is_dir(): if not recipes.is_dir():
return set() return set()
return {p.stem for p in recipes.glob("*.json")} return {p.stem for p in recipes.glob("*.json")}
def login_rows():
"""站点登录视图(面板「站点登录」页数据源)。
每行:{site, url, user, configured, sources, probe_source, has_recipe}
probe_source = 该站点任一已接入源(自动配方的浏览器探测入口);
has_recipe = 站点级或任一源级配方存在。
"""
creds = get_logins()
sites = login_sites()
recipes = login_recipes()
rows = []
for site in sorted(set(creds) | set(sites)):
sources = sites.get(site, [])
c = creds.get(site, {})
rows.append({
"site": site,
"url": c.get("url", ""),
"user": c.get("user", ""),
"configured": bool(c.get("configured")),
"sources": sources,
"probe_source": sources[0] if sources else "",
"has_recipe": site in recipes or any(s in recipes for s in sources),
})
return rows
# ── 组合视图 ──────────────────────────────────────────────── # ── 组合视图 ────────────────────────────────────────────────
def source_rows(log_dir=None): def source_rows(log_dir=None):
"""面板源列表全集:已装技能源 ∪ 已有调度。 """面板源列表全集:已装技能源 ∪ 已有调度。
......
...@@ -966,26 +966,24 @@ def toggle_source(name): ...@@ -966,26 +966,24 @@ def toggle_source(name):
return redirect(url_for("index")) return redirect(url_for("index"))
# ── 站点登录凭据管理 ──────────────────────────────────────── # ── 站点登录凭据管理(站点 = 源名第一段,同站多频道共享一份凭据/配方) ──
_login_test_lock = threading.Lock() _login_test_lock = threading.Lock()
_login_test = {} # {source: {"running": bool, "done": bool, "rc": int, "output": str, "start": str}} _login_test = {} # {site: {"running": bool, "done": bool, "rc": int, "output": str, "start": str}}
def _site_probe_source(site):
"""该站点的代表源(自动配方/登录测试的浏览器入口);未接入源返回 ''。"""
for row in pl.login_rows():
if row["site"] == site:
return row["probe_source"]
return ""
@app.route("/logins") @app.route("/logins")
@require_auth @require_auth
def login_list(): def login_list():
logins = pl.get_logins() rows = pl.login_rows()
recipes = pl.login_recipes()
rows = []
for source in sorted(set(logins) | recipes):
rows.append({
"source": source,
"user": logins.get(source, {}).get("user", ""),
"url": logins.get(source, {}).get("url", ""),
"configured": logins.get(source, {}).get("configured", False),
"has_recipe": source in recipes,
})
test_state = dict(_login_test) test_state = dict(_login_test)
return render_template("logins.html", rows=rows, test_state=test_state) return render_template("logins.html", rows=rows, test_state=test_state)
...@@ -993,90 +991,98 @@ def login_list(): ...@@ -993,90 +991,98 @@ def login_list():
@app.route("/login-save", methods=["POST"]) @app.route("/login-save", methods=["POST"])
@require_auth @require_auth
def login_save(): def login_save():
source = request.form.get("source", "").strip() site = request.form.get("site", "").strip().lower()
url = request.form.get("url", "").strip() url = request.form.get("url", "").strip()
user = request.form.get("user", "").strip() user = request.form.get("user", "").strip()
password = request.form.get("password", "").strip() password = request.form.get("password", "").strip()
if not re.match(r"^[a-zA-Z0-9_-]+$", source): if not re.match(r"^[a-zA-Z0-9_-]+$", site):
flash("源名只能含字母、数字、下划线、连字符", "error") flash("站点名只能含字母、数字、下划线、连字符(如 nikkei,对全部 nikkei-* 频道生效)", "error")
return redirect(url_for("login_list")) return redirect(url_for("login_list"))
if not user: if not user:
flash("账号不能为空", "error") flash("账号不能为空", "error")
return redirect(url_for("login_list")) return redirect(url_for("login_list"))
try: try:
pl.save_login(source, url, user, password) pl.save_login(site, url, user, password)
flash(f"{source} 凭据已保存(密码以 600 权限落盘,不再回显)", "success") flash(f"站点 {site} 凭据已保存(对该站点全部频道生效;密码以 600 权限落盘,不再回显)", "success")
except Exception as e: # noqa: BLE001 except Exception as e: # noqa: BLE001
flash(f"保存失败: {e}", "error") flash(f"保存失败: {e}", "error")
return redirect(url_for("login_list")) return redirect(url_for("login_list"))
@app.route("/login-delete/<source>", methods=["POST"]) @app.route("/login-delete/<site>", methods=["POST"])
@require_auth @require_auth
def login_delete(source): def login_delete(site):
if not re.match(r"^[a-zA-Z0-9_-]+$", source): if not re.match(r"^[a-zA-Z0-9_-]+$", site):
abort(404) abort(404)
if pl.delete_login(source): if pl.delete_login(site):
flash(f"{source} 凭据已删除", "success") flash(f"站点 {site} 凭据已删除", "success")
else: else:
flash(f"{source} 无凭据", "error") flash(f"站点 {site} 无凭据", "error")
return redirect(url_for("login_list")) return redirect(url_for("login_list"))
@app.route("/login-discover/<source>", methods=["POST"]) @app.route("/login-discover/<site>", methods=["POST"])
@require_auth @require_auth
def login_discover(source): def login_discover(site):
"""自动发现登录流程生成配方,并立即实登验证(后台线程,页面轮询)。""" """自动发现登录流程生成站点级配方,并立即实登验证(后台线程,页面轮询)。"""
if not re.match(r"^[a-zA-Z0-9_-]+$", source): if not re.match(r"^[a-zA-Z0-9_-]+$", site):
abort(404) abort(404)
if not pl.get_logins().get(source, {}).get("configured"): if not pl.get_logins().get(site, {}).get("configured"):
flash(f"{source} 未配置凭据,请先在下方表单保存", "error") flash(f"站点 {site} 未配置凭据,请先在下方表单保存", "error")
return redirect(url_for("login_list"))
probe = _site_probe_source(site)
if not probe:
flash(f"站点 {site} 尚未接入任何数据源,先在「栏目管理」添加 {site}-* 源后再自动配置配方", "error")
return redirect(url_for("login_list")) return redirect(url_for("login_list"))
with _login_test_lock: with _login_test_lock:
if _login_test.get(source, {}).get("running"): if _login_test.get(site, {}).get("running"):
flash(f"{source} 已有登录测试在运行", "error") flash(f"站点 {site} 已有登录测试在运行", "error")
return redirect(url_for("login_list")) return redirect(url_for("login_list"))
_login_test[source] = {"running": True, "done": False, "start": _login_test[site] = {"running": True, "done": False, "start":
datetime.now().strftime("%Y-%m-%d %H:%M:%S")} datetime.now().strftime("%Y-%m-%d %H:%M:%S")}
def _run(): def _run():
rc, out, err = call_platform("nh_login_autodiscover", source) rc, out, err = call_platform("nh_login_autodiscover", probe)
with _login_test_lock: with _login_test_lock:
_login_test[source] = { _login_test[site] = {
"running": False, "done": True, "rc": rc, "running": False, "done": True, "rc": rc,
"output": ((out or "") + (("\n" + err) if err else ""))[-2000:], "output": ((out or "") + (("\n" + err) if err else ""))[-2000:],
"start": _login_test[source]["start"], "start": _login_test[site]["start"],
} }
threading.Thread(target=_run, daemon=True).start() threading.Thread(target=_run, daemon=True).start()
flash(f"{source} 自动配置配方并测试已启动(约 30-90 秒,页面自动刷新)", "success") flash(f"站点 {site} 自动配置配方并测试已启动(经 {probe} 探测,约 30-90 秒,页面自动刷新)", "success")
return redirect(url_for("login_list")) return redirect(url_for("login_list"))
@app.route("/login-test/<source>", methods=["POST"]) @app.route("/login-test/<site>", methods=["POST"])
@require_auth @require_auth
def login_test(source): def login_test(site):
"""立即执行登录预检(检查+未登录则自动登录),后台线程运行,页面轮询。""" """立即执行登录预检(检查+未登录则自动登录),后台线程运行,页面轮询。"""
if not re.match(r"^[a-zA-Z0-9_-]+$", source): if not re.match(r"^[a-zA-Z0-9_-]+$", site):
abort(404) abort(404)
probe = _site_probe_source(site)
if not probe:
flash(f"站点 {site} 尚未接入任何数据源,无法测试", "error")
return redirect(url_for("login_list"))
with _login_test_lock: with _login_test_lock:
if _login_test.get(source, {}).get("running"): if _login_test.get(site, {}).get("running"):
flash(f"{source} 登录测试已在运行", "error") flash(f"站点 {site} 登录测试已在运行", "error")
return redirect(url_for("login_list")) return redirect(url_for("login_list"))
_login_test[source] = {"running": True, "done": False, "start": _login_test[site] = {"running": True, "done": False, "start":
datetime.now().strftime("%Y-%m-%d %H:%M:%S")} datetime.now().strftime("%Y-%m-%d %H:%M:%S")}
def _run(): def _run():
rc, out, err = call_platform("nh_login_preflight", source) rc, out, err = call_platform("nh_login_preflight", probe)
with _login_test_lock: with _login_test_lock:
_login_test[source] = { _login_test[site] = {
"running": False, "done": True, "rc": rc, "running": False, "done": True, "rc": rc,
"output": ((out or "") + (("\n" + err) if err else ""))[-2000:], "output": ((out or "") + (("\n" + err) if err else ""))[-2000:],
"start": _login_test[source]["start"], "start": _login_test[site]["start"],
} }
threading.Thread(target=_run, daemon=True).start() threading.Thread(target=_run, daemon=True).start()
flash(f"{source} 登录测试已启动(约 10-60 秒,页面自动刷新显示结果)", "success") flash(f"站点 {site} 登录测试已启动(经 {probe} 频道,约 10-60 秒,页面自动刷新显示结果)", "success")
return redirect(url_for("login_list")) return redirect(url_for("login_list"))
......
...@@ -9,8 +9,10 @@ ...@@ -9,8 +9,10 @@
<h1>站点登录</h1> <h1>站点登录</h1>
<p style="color:#666;font-size:13px"> <p style="color:#666;font-size:13px">
为付费/需登录的数据源维护凭据。保存后以 600 权限落盘(<code>etc/news-harvester/logins/</code>),密码不再回显; 按<strong>站点</strong>维护凭据:站点 = 源名第一段,同一账号的多个频道共享一份凭据与配方
每次采集前自动预检,未登录会自动登录。配方(页面选择器)在技能仓 <code>logins/</code> 下维护。 (如 <code>nikkei</code> 一份凭据覆盖 <code>nikkei-world / nikkei-markets / nikkei-tech</code> 全部频道)。
保存后以 600 权限落盘(<code>etc/news-harvester/logins/</code>),密码不再回显;
每个频道采集前自动预检,未登录会自动登录。配方在技能仓 <code>logins/</code> 下维护(同样按站点)。
</p> </p>
{% with messages = get_flashed_messages() %} {% with messages = get_flashed_messages() %}
...@@ -20,24 +22,31 @@ ...@@ -20,24 +22,31 @@
<div class="table-wrap"> <div class="table-wrap">
<table> <table>
<thead> <thead>
<tr><th>数据源</th><th>登录账号</th><th>站点</th><th>凭据</th><th>自动登录配方</th><th>上次测试</th><th>操作</th></tr> <tr><th>站点</th><th>登录账号</th><th>首页</th><th>覆盖频道</th><th>凭据</th><th>自动登录配方</th><th>上次测试</th><th>操作</th></tr>
</thead> </thead>
<tbody> <tbody>
{% for r in rows %} {% for r in rows %}
<tr> <tr>
<td><code>{{ r.source }}</code></td> <td><code>{{ r.site }}</code></td>
<td>{{ r.user or '—' }}</td> <td>{{ r.user or '—' }}</td>
<td>{{ r.url or '—' }}</td> <td>{{ r.url or '—' }}</td>
<td>
{% if r.sources %}
{% for s in r.sources %}<code style="margin-right:4px">{{ s }}</code>{% endfor %}
{% else %}
<span class="badge unknown">未接入源(先在栏目管理添加 {{ r.site }}-* 源)</span>
{% endif %}
</td>
<td> <td>
{% if r.configured %}<span class="badge ok">已配置</span> {% if r.configured %}<span class="badge ok">已配置</span>
{% else %}<span class="badge unknown">未配置</span>{% endif %} {% else %}<span class="badge unknown">未配置</span>{% endif %}
</td> </td>
<td> <td>
{% if r.has_recipe %}<span class="badge ok">支持</span> {% if r.has_recipe %}<span class="badge ok">支持</span>
{% else %}<span class="badge unknown">无配方(点右侧按钮自动生成)</span>{% endif %} {% else %}<span class="badge unknown">无配方(配置凭据后点右侧按钮自动生成)</span>{% endif %}
</td> </td>
{% set st = test_state.get(r.site) %}
<td> <td>
{% set st = test_state.get(r.source) %}
{% if st and st.running %} {% if st and st.running %}
<span class="badge active">测试中…(启动于 {{ st.start }})</span> <span class="badge active">测试中…(启动于 {{ st.start }})</span>
{% elif st and st.done %} {% elif st and st.done %}
...@@ -53,20 +62,22 @@ ...@@ -53,20 +62,22 @@
</td> </td>
<td> <td>
{% if r.has_recipe %} {% if r.has_recipe %}
<form method="POST" action="{{ url_for('login_test', source=r.source) }}" style="display:inline"> <form method="POST" action="{{ url_for('login_test', site=r.site) }}" style="display:inline">
<button type="submit" class="btn btn-sm" {% if st and st.running %}disabled{% endif %}>测试登录</button> <button type="submit" class="btn btn-sm" {% if st and st.running %}disabled{% endif %}>测试登录</button>
</form> </form>
{% elif r.configured %} {% elif r.configured and r.probe_source %}
<form method="POST" action="{{ url_for('login_discover', source=r.source) }}" style="display:inline" <form method="POST" action="{{ url_for('login_discover', site=r.site) }}" style="display:inline"
onsubmit="return confirm('自动识别 {{ r.source }} 的登录流程并生成配方,随后立即用凭据实登验证?')"> onsubmit="return confirm('自动识别 {{ r.site }} 的登录流程并生成站点级配方,随后立即用凭据实登验证?')">
<button type="submit" class="btn btn-sm btn-primary" {% if st and st.running %}disabled{% endif %}>自动配置配方并测试</button> <button type="submit" class="btn btn-sm btn-primary" {% if st and st.running %}disabled{% endif %}>自动配置配方并测试</button>
</form> </form>
{% endif %} {% endif %}
<button class="btn btn-sm" onclick="fillForm('{{ r.source }}', '{{ r.url }}', '{{ r.user }}')">编辑</button> <button class="btn btn-sm" onclick="fillForm('{{ r.site }}', '{{ r.url }}', '{{ r.user }}')">编辑</button>
<form method="POST" action="{{ url_for('login_delete', source=r.source) }}" style="display:inline" {% if r.configured %}
onsubmit="return confirm('删除 {{ r.source }} 的登录凭据?')"> <form method="POST" action="{{ url_for('login_delete', site=r.site) }}" style="display:inline"
onsubmit="return confirm('删除站点 {{ r.site }} 的登录凭据(对该站点全部频道生效)?')">
<button type="submit" class="btn btn-sm btn-danger">删除</button> <button type="submit" class="btn btn-sm btn-danger">删除</button>
</form> </form>
{% endif %}
</td> </td>
</tr> </tr>
{% endfor %} {% endfor %}
...@@ -74,10 +85,10 @@ ...@@ -74,10 +85,10 @@
</table> </table>
</div> </div>
<h2>{{ '编辑凭据' if edit_hint else '添加 / 更新凭据' }}</h2> <h2>{{ '编辑站点凭据' if edit_hint else '添加 / 更新站点凭据' }}</h2>
<form method="POST" action="{{ url_for('login_save') }}" class="card" style="max-width:520px;padding:16px;display:grid;gap:10px"> <form method="POST" action="{{ url_for('login_save') }}" class="card" style="max-width:520px;padding:16px;display:grid;gap:10px">
<label>数据源 ID(须与采集源一致,如 digitimes-feed) <label>站点 ID(源名第一段,如 nikkei / digitimes / wsj;单段源即自身)
<input name="source" id="f-source" required pattern="[a-zA-Z0-9_-]+" <input name="site" id="f-site" required pattern="[a-zA-Z0-9_-]+"
style="width:100%;background:#1a1d27;border:1px solid #2a2d37;border-radius:4px;color:#d4d4d4;padding:6px 8px"> style="width:100%;background:#1a1d27;border:1px solid #2a2d37;border-radius:4px;color:#d4d4d4;padding:6px 8px">
</label> </label>
<label>站点首页(信息性,便于识别) <label>站点首页(信息性,便于识别)
...@@ -98,8 +109,8 @@ ...@@ -98,8 +109,8 @@
</form> </form>
<script> <script>
function fillForm(source, url, user) { function fillForm(site, url, user) {
document.getElementById('f-source').value = source; document.getElementById('f-site').value = site;
document.getElementById('f-url').value = url; document.getElementById('f-url').value = url;
document.getElementById('f-user').value = user; document.getElementById('f-user').value = user;
document.getElementById('f-password').value = ''; document.getElementById('f-password').value = '';
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment