Commit d179c21f authored by 郭阔's avatar 郭阔

feat(harvest-result): exhausted + all-invalid-candidates counts as success

Two validation refinements so 'nothing new but junk candidates' stops
alarming as FAILED (reuters-markets case: 18 candidates, 16 deduped, the
only 2 new ones rejected as non-article pages):
- majority-failure rule downgrades to a warning when source_exhausted and
  every failure is a non-article rejection (og:type/body), same standing
  as all_seen
- preloaded per-URL INVALID_OUTPUT failures no longer set task-failed
  (task-level no-URL failures and RUNTIME/PERMISSION still fatal); the
  per-article signal already lives in counts and warnings
parent 1a53ae75
...@@ -277,8 +277,10 @@ def normalize(envelope, source, count, process_rc, work_dir=None, vault_root=Non ...@@ -277,8 +277,10 @@ def normalize(envelope, source, count, process_rc, work_dir=None, vault_root=Non
raise ValueError("source/requested_count 与本次调用不一致") raise ValueError("source/requested_count 与本次调用不一致")
result = copy.deepcopy(candidate) result = copy.deepcopy(candidate)
for failure in result["failures"]: for failure in result["failures"]:
# 任务级失败(无 URL)或运行/权限类才致命;带 URL 的逐篇失败(如非文章页
# 校验拒绝)已体现在文章计数与警告中,不再重复判任务失败
if (not failure["url"] or if (not failure["url"] or
failure["type"] in {"RUNTIME_ERROR", "PERMISSION_DENIED", "INVALID_OUTPUT"}): failure["type"] in {"RUNTIME_ERROR", "PERMISSION_DENIED"}):
task_failed = True task_failed = True
except ValueError as exc: except ValueError as exc:
fail("INVALID_OUTPUT", str(exc)) fail("INVALID_OUTPUT", str(exc))
...@@ -334,8 +336,21 @@ def normalize(envelope, source, count, process_rc, work_dir=None, vault_root=Non ...@@ -334,8 +336,21 @@ def normalize(envelope, source, count, process_rc, work_dir=None, vault_root=Non
result["failed_count"] = result["attempted_count"] - result["succeeded_count"] result["failed_count"] = result["attempted_count"] - result["succeeded_count"]
if (result["attempted_count"] and if (result["attempted_count"] and
result["failed_count"] * 2 >= result["attempted_count"]): result["failed_count"] * 2 >= result["attempted_count"]):
fail("FETCH_FAILED", message = (f"{result['attempted_count']} 篇文章中有 {result['failed_count']} 篇失败,未取得多数成功")
f"{result['attempted_count']} 篇文章中有 {result['failed_count']} 篇失败,未取得多数成功") # 取尽 + 失败全部为"非文章页"类(og:type/body 校验拒绝)→ 与 all_seen 同等,
# 视为"无新内容"降级为警告;其余多数失败仍判失败
non_article_only = result["source_exhausted"] and all(
(a.get("failure_type") in ("CONTENT_INCOMPLETE", "INVALID_OUTPUT")
and ("not an article" in (a.get("failure_reason") or "")
or "og:type" in (a.get("failure_reason") or "")
or "非文章" in (a.get("failure_reason") or "")))
for a in result["articles"] if a["status"] == "FAILED")
if non_article_only:
warning = f"仅有的新候选均为非文章页(校验拒绝)且已确认取尽,视为无新内容:{message}"
if warning not in result["warnings"]:
result["warnings"].append(warning)
else:
fail("FETCH_FAILED", message)
readable = [a for a in result["articles"] if a["archive"]["loaded"]] readable = [a for a in result["articles"] if a["archive"]["loaded"]]
short_count = sum(a["archive"]["body_length"] < SHORT_BODY_LIMIT for a in readable) short_count = sum(a["archive"]["body_length"] < SHORT_BODY_LIMIT for a in readable)
if readable and short_count * 2 >= len(readable): if readable and short_count * 2 >= len(readable):
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment