Commit 35feb5c2 authored by 郭阔's avatar 郭阔

feat(harvest): 排队从全局锁改为按站点分锁,不同站点并行采集

- 队列目录 .queue → .queue.port-<port>:同站(同浏览器同账号)互斥,
  异站并行(如 nikkei 9315 与 reuters 9408 同时跑,日常即两条流水线)
- 站点端口在排队锁之前解析,锁粒度与 runlock 一致
- reuters-world 09:27 实跑验证重试链:attempt1 交付17篇+结构化交付失败
  → attempt2 续会话确认 all_seen → 合并17篇并推送成功
parent 0aed7fc0
...@@ -60,13 +60,14 @@ cd "$WORK_DIR" ...@@ -60,13 +60,14 @@ cd "$WORK_DIR"
source "$SCRIPT_DIR/result.sh" source "$SCRIPT_DIR/result.sh"
prepare_result_contract || exit 1 prepare_result_contract || exit 1
# 运行锁先建(排队中也算"运行中",面板可见);全局排队:同一时刻仅一个任务使用浏览器, # 运行锁先建(排队中也算"运行中",面板可见);排队按站点分锁:同站(同端口=同浏览器
# 到点时若忙则等待(排队中),前一任务结束自动开始。陈旧队列锁(PID 已死)自动窃取。 # 同账号)互斥排队,不同站点并行采集(如 nikkei 与 reuters 可同时跑)。
# 到点时若同站忙则等待(排队中),前一任务结束自动开始。陈旧队列锁(PID 已死)自动窃取。
_nh_lock_dir="$WORK_DIR/runlocks" _nh_lock_dir="$WORK_DIR/runlocks"
mkdir -p "$_nh_lock_dir" 2>/dev/null || true mkdir -p "$_nh_lock_dir" 2>/dev/null || true
_nh_port="${CDP_PORT_OVERRIDE:-$(nh_cdp_port "$(nh_login_site "$SOURCE")" 2>/dev/null)}" _nh_port="${CDP_PORT_OVERRIDE:-$(nh_cdp_port "$(nh_login_site "$SOURCE")" 2>/dev/null)}"
[ -n "$_nh_port" ] && touch "$_nh_lock_dir/port-${_nh_port}.$$.lock" [ -n "$_nh_port" ] && touch "$_nh_lock_dir/port-${_nh_port}.$$.lock"
_nh_queue_dir="$_nh_lock_dir/.queue" _nh_queue_dir="$_nh_lock_dir/.queue.port-${_nh_port:-nosite}"
QUEUE_WAIT_MAX_MIN="${QUEUE_WAIT_MAX_MIN:-180}" QUEUE_WAIT_MAX_MIN="${QUEUE_WAIT_MAX_MIN:-180}"
_nh_quit_instance() { _nh_quit_instance() {
local skill_dir f base pid local skill_dir f base pid
...@@ -106,7 +107,7 @@ until mkdir "$_nh_queue_dir" 2>/dev/null; do ...@@ -106,7 +107,7 @@ until mkdir "$_nh_queue_dir" 2>/dev/null; do
continue continue
fi fi
if [ $((SECONDS - _queue_t0)) -gt $((QUEUE_WAIT_MAX_MIN * 60)) ]; then if [ $((SECONDS - _queue_t0)) -gt $((QUEUE_WAIT_MAX_MIN * 60)) ]; then
echo "[queue] 排队超过 ${QUEUE_WAIT_MAX_MIN} 分钟,放弃等待直接启动(可能并发)" >&2 echo "[queue] 同站排队超过 ${QUEUE_WAIT_MAX_MIN} 分钟,放弃等待直接启动(可能同站并发)" >&2
break break
fi fi
sleep 20 sleep 20
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment