liqing/holy-python
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

完善验收采集调度与截图管理

Browse Source
This commit is contained in:
李清
2026-09-01 16:18:00 +08:00
parent af340206a5
commit 292bd3a126
7 changed files with 401 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

276
acceptance_service.py
View File

@@ -745,6 +745,8 @@ def _poll_screenshot_results(public_id: str) -> None:
deadline = time.monotonic() + max(1.0, Config.XAPI_RESULT_MAX_WAIT_HOURS) * 3600
while True:
if _acceptance_task_is_cancelled(public_id):
return
poll_errors = []
for batch_id, (keyword_run_id, _, _) in batch_map.items():
try:
@@ -756,22 +758,31 @@ def _poll_screenshot_results(public_id: str) -> None:
poll_errors.append(f"{batch_id}: {exc}")
with connection() as conn:
current_task = conn.execute(
"SELECT status FROM acceptance_tasks WHERE id=?", (task["id"],),
).fetchone()
if not current_task or current_task["status"] == "cancelled":
return
rows = conn.execute(
"SELECT * FROM acceptance_xapi_runs WHERE task_id=?", (task["id"],),
).fetchall()
tracked_rows = [row for row in rows if row["batch_id"] in batch_map]
expected_total = sum(item[2] for item in batch_map.values())
terminal_count = sum(
str(row["status"] or "").lower() in terminal_statuses for row in rows
str(row["status"] or "").lower() in terminal_statuses for row in tracked_rows
)
feishu_ready = all(
str(row["status"] or "").lower() != "completed"
or str(row["feishu_sync_status"] or "").lower() == "succeeded"
for row in rows
for row in tracked_rows
)
received = min(len(rows), expected_total)
received = min(len(tracked_rows), expected_total)
progress = min(99, 70 + round(29 * received / max(1, expected_total)))
for keyword_run in keyword_runs:
related = [row for row in rows if row["keyword_run_id"] == keyword_run["id"]]
related = [
row for row in tracked_rows
if row["keyword_run_id"] == keyword_run["id"]
]
related_batch_ids = [
value.strip() for value in str(keyword_run["crawler_task_id"] or "").split(",")
if value.strip()
@@ -792,8 +803,13 @@ def _poll_screenshot_results(public_id: str) -> None:
all_terminal = expected_total > 0 and terminal_count >= expected_total
timed_out = time.monotonic() >= deadline
if all_terminal and feishu_ready:
latest_rows: dict[tuple[str, str], Any] = {}
for row in rows:
key = (str(row["keyword"] or ""), str(row["xhs_account_id"] or ""))
latest_rows[key] = row
failed_count = sum(
str(row["status"] or "").lower() != "completed" for row in rows
str(row["status"] or "").lower() != "completed"
for row in latest_rows.values()
)
final_status = "partial" if failed_count else "completed"
conn.execute(
@@ -820,6 +836,94 @@ def _poll_screenshot_results(public_id: str) -> None:
time.sleep(max(5.0, Config.XAPI_RESULT_POLL_INTERVAL_SECONDS))
def retry_failed_acceptance_devices(public_id: str) -> dict[str, Any]:
"""仅对截图失败的“关键词 × 设备”组合重新立即下发。"""
with connection() as conn:
task = conn.execute(
"SELECT * FROM acceptance_tasks WHERE public_task_id=?", (public_id,),
).fetchone()
if not task:
raise TaskError("ACCEPTANCE_TASK_NOT_FOUND", "验收任务不存在", 404)
if task["acceptance_method"] not in {"screenshot", "combined"}:
raise TaskError("SCREENSHOT_RETRY_NOT_SUPPORTED", "该任务不包含截图采集", 409)
rows = conn.execute(
"""SELECT * FROM acceptance_xapi_runs
WHERE task_id=? ORDER BY id""",
(task["id"],),
).fetchall()
keyword_runs = {
row["keyword"]: row["id"] for row in conn.execute(
"""SELECT id,keyword FROM acceptance_keyword_runs
WHERE task_id=? AND run_type='screenshot'""",
(task["id"],),
).fetchall()
}
latest: dict[tuple[str, str], Any] = {}
for row in rows:
keyword = str(row["keyword"] or "").strip()
account_id = str(row["xhs_account_id"] or "").strip()
if keyword and account_id:
latest[(keyword, account_id)] = row
failed_by_keyword: dict[str, list[str]] = {}
terminal_failures = {"failed", "cancelled", "canceled", "error"}
for (keyword, account_id), row in latest.items():
status = str(row["status"] or "").lower()
artifacts = _decode_json(row["artifacts_json"], []) or []
has_screenshot = any(
str(item.get("image_url") or item.get("url") or "").strip()
for item in artifacts if isinstance(item, dict)
)
if status in terminal_failures or (status == "completed" and not has_screenshot):
failed_by_keyword.setdefault(keyword, []).append(account_id)
if not failed_by_keyword:
raise TaskError("NO_FAILED_SCREENSHOT_DEVICES", "当前没有可重新采集的失败设备", 409)
dispatched_devices: set[str] = set()
dispatched_pairs = 0
errors: list[str] = []
for keyword, account_ids in failed_by_keyword.items():
run_id = keyword_runs.get(keyword)
if not run_id:
errors.append(f"{keyword}: 缺少关键词运行记录")
continue
try:
result = schedule_xapi_screenshots(
[keyword], trigger_source="acceptance", xhs_account_ids=account_ids,
)
batches = result.get("batches") or []
batch_ids = [
str(batch.get("batch_id") or "").strip()
for batch in batches if str(batch.get("batch_id") or "").strip()
]
if not batch_ids:
raise RuntimeError("Framework 未返回可回查的 batch_id")
_update_run(
run_id, status="waiting", result={**result, "batches": batches},
crawler_task_id=",".join(batch_ids), error=None,
)
dispatched_devices.update(account_ids)
dispatched_pairs += len(account_ids)
except Exception as exc:
errors.append(f"{keyword}: {exc}")
if not dispatched_pairs:
raise TaskError("FAILED_DEVICE_RETRY_DISPATCH_FAILED", "；".join(errors)[:2000], 502)
with connection() as conn:
conn.execute(
"""UPDATE acceptance_tasks SET status='running',progress_percent=70,
error_message=?,updated_at=? WHERE id=?""",
("；".join(errors)[:2000] if errors else None, now_iso(), task["id"]),
)
threading.Thread(
target=_poll_screenshot_results, args=(public_id,), daemon=True,
name=f"acceptance-retry-failed-{public_id}",
).start()
result_task = get_acceptance_task(public_id)
result_task["retry_dispatched_device_count"] = len(dispatched_devices)
result_task["retry_dispatched_pair_count"] = dispatched_pairs
result_task["retry_dispatch_errors"] = errors
return result_task
def create_acceptance_task(data: dict[str, Any]) -> dict[str, Any]:
links = list(dict.fromkeys(_validate_url(item) for item in data.get("feishu_links", [])))
method = str(data.get("acceptance_method") or "")
@@ -918,24 +1022,56 @@ def create_acceptance_task(data: dict[str, Any]) -> dict[str, Any]:
return get_acceptance_task(public_id)
def _next_weekly_run(weekday: int, *, after: datetime | None = None) -> str:
current = after or datetime.now().astimezone()
days = (weekday - current.isoweekday()) % 7
candidate = (current + timedelta(days=days)).replace(
hour=0, minute=0, second=0, microsecond=0,
)
if candidate <= current:
candidate += timedelta(days=7)
return candidate.isoformat(timespec="seconds")
def update_acceptance_schedule(public_id: str, weekday: int) -> dict[str, Any]:
def _normalize_schedule_weekdays(value: Any) -> list[int]:
raw = value if isinstance(value, (list, tuple, set)) else [value]
try:
weekday = int(weekday)
weekdays = sorted(set(int(item) for item in raw))
except (TypeError, ValueError) as exc:
raise TaskError("INVALID_WEEKDAY", "采集周期必须选择周一至周日", 422) from exc
if weekday not in WEEKDAYS:
raise TaskError("INVALID_WEEKDAY", "采集周期必须选择周一至周日", 422)
if not weekdays or any(item not in WEEKDAYS for item in weekdays):
raise TaskError("INVALID_WEEKDAY", "采集周期至少选择一个周一至周日", 422)
return weekdays
def _normalize_schedule_time(value: Any, field_name: str) -> str:
text = str(value or "").strip()
try:
parsed = datetime.strptime(text, "%H:%M")
except ValueError as exc:
raise TaskError("INVALID_SCHEDULE_TIME", f"{field_name}必须为 HH:MM 格式", 422) from exc
return parsed.strftime("%H:%M")
def _next_weekly_run(
weekdays: list[int] | tuple[int, ...] | set[int] | int,
start_time: str = "00:00",
*,
after: datetime | None = None,
) -> str:
normalized_weekdays = _normalize_schedule_weekdays(weekdays)
normalized_start = _normalize_schedule_time(start_time, "开始时间")
current = after or datetime.now().astimezone()
hour, minute = (int(item) for item in normalized_start.split(":"))
for offset in range(0, 8):
candidate = (current + timedelta(days=offset)).replace(
hour=hour, minute=minute, second=0, microsecond=0,
)
if candidate.isoweekday() in normalized_weekdays and candidate > current:
return candidate.isoformat(timespec="seconds")
raise RuntimeError("无法计算下一次周期采集时间")
def update_acceptance_schedule(
public_id: str,
weekdays: Any,
start_time: Any = "00:00",
end_time: Any = "23:59",
) -> dict[str, Any]:
normalized_weekdays = _normalize_schedule_weekdays(weekdays)
normalized_start = _normalize_schedule_time(start_time, "开始时间")
normalized_end = _normalize_schedule_time(end_time, "结束时间")
if normalized_start > normalized_end:
raise TaskError("INVALID_SCHEDULE_RANGE", "开始时间不能晚于结束时间", 422)
with connection() as conn:
task = conn.execute(
"SELECT id FROM acceptance_tasks WHERE public_task_id=?", (public_id,),
@@ -944,8 +1080,14 @@ def update_acceptance_schedule(public_id: str, weekday: int) -> dict[str, Any]:
raise TaskError("ACCEPTANCE_TASK_NOT_FOUND", "验收任务不存在", 404)
conn.execute(
"""UPDATE acceptance_tasks SET schedule_enabled=1,schedule_weekday=?,
schedule_weekdays=?,schedule_start_time=?,schedule_end_time=?,
next_run_at=?,schedule_error=NULL,updated_at=? WHERE id=?""",
(weekday, _next_weekly_run(weekday), now_iso(), task["id"]),
(
normalized_weekdays[0], json.dumps(normalized_weekdays),
normalized_start, normalized_end,
_next_weekly_run(normalized_weekdays, normalized_start),
now_iso(), task["id"],
),
)
return get_acceptance_task(public_id)
@@ -962,6 +1104,40 @@ def stop_acceptance_schedule(public_id: str) -> dict[str, Any]:
return get_acceptance_task(public_id)
def _acceptance_task_is_cancelled(public_id: str) -> bool:
with connection() as conn:
row = conn.execute(
"SELECT status FROM acceptance_tasks WHERE public_task_id=?", (public_id,),
).fetchone()
return not row or row["status"] == "cancelled"
def cancel_acceptance_task(public_id: str, reason: str = "") -> dict[str, Any]:
"""只停止当前批次的处理；该项目已有的周期调度保持不变。"""
current_time = now_iso()
with connection() as conn:
task = conn.execute(
"SELECT id,status FROM acceptance_tasks WHERE public_task_id=?", (public_id,),
).fetchone()
if not task:
raise TaskError("ACCEPTANCE_TASK_NOT_FOUND", "验收任务不存在", 404)
if task["status"] == "cancelled":
return get_acceptance_task(public_id)
if task["status"] not in {"waiting", "running"}:
raise TaskError("ACCEPTANCE_TASK_NOT_CANCELLABLE", "任务状态已变化，无法停止", 409)
conn.execute(
"""UPDATE acceptance_tasks SET status='cancelled',
error_message=?,updated_at=? WHERE id=?""",
((reason or "用户主动停止验收任务")[:500], current_time, task["id"]),
)
conn.execute(
"""UPDATE acceptance_keyword_runs SET status='cancelled',error_message=?,
completed_at=?,updated_at=? WHERE task_id=? AND status IN ('waiting','running')""",
((reason or "用户主动停止验收任务")[:500], current_time, current_time, task["id"]),
)
return get_acceptance_task(public_id)
def _copy_acceptance_task(public_id: str, *, trigger_source: str = "acceptance") -> dict[str, Any]:
with connection() as conn:
task = conn.execute(
@@ -1012,21 +1188,39 @@ def collect_acceptance_now(public_id: str) -> dict[str, Any]:
def run_due_acceptance_schedules() -> int:
now = now_iso()
current = datetime.now().astimezone()
now = current.isoformat(timespec="seconds")
with connection() as conn:
due = conn.execute(
"""SELECT public_task_id,schedule_weekday FROM acceptance_tasks
"""SELECT public_task_id,schedule_weekday,schedule_weekdays,
schedule_start_time,schedule_end_time,next_run_at
FROM acceptance_tasks
WHERE schedule_enabled=1 AND next_run_at IS NOT NULL AND next_run_at<=?
ORDER BY next_run_at LIMIT 20""",
(now,),
).fetchall()
runnable = []
for row in due:
weekdays = _decode_json(row["schedule_weekdays"], None) or [row["schedule_weekday"]]
start_time = row["schedule_start_time"] or "00:00"
end_time = row["schedule_end_time"] or "23:59"
due_at = datetime.fromisoformat(row["next_run_at"])
end_hour, end_minute = (int(item) for item in end_time.split(":"))
window_end = due_at.replace(hour=end_hour, minute=end_minute, second=59)
missed = current > window_end
conn.execute(
"""UPDATE acceptance_tasks SET next_run_at=?,last_run_at=?,
schedule_error=NULL,updated_at=? WHERE public_task_id=?""",
(_next_weekly_run(row["schedule_weekday"]), now, now, row["public_task_id"]),
schedule_error=?,updated_at=? WHERE public_task_id=?""",
(
_next_weekly_run(weekdays, start_time, after=current),
None if missed else now,
"本次调度已超过设定结束时间，自动顺延" if missed else None,
now, row["public_task_id"],
),
)
for row in due:
if not missed:
runnable.append(row)
for row in runnable:
try:
_copy_acceptance_task(row["public_task_id"], trigger_source="acceptance-weekly")
except Exception as exc:
@@ -1035,7 +1229,7 @@ def run_due_acceptance_schedules() -> int:
"UPDATE acceptance_tasks SET schedule_error=?,updated_at=? WHERE public_task_id=?",
(str(exc)[:2000], now_iso(), row["public_task_id"]),
)
return len(due)
return len(runnable)
def start_acceptance_schedule_worker() -> None:
@@ -1065,6 +1259,8 @@ def _fetch_sources(public_id: str) -> None:
"SELECT keyword FROM acceptance_keywords WHERE task_id=? ORDER BY keyword_order",
(task["id"],),
).fetchall()]
if task["status"] == "cancelled":
return
conn.execute("UPDATE acceptance_tasks SET status='running',progress_percent=5,updated_at=? WHERE id=?", (now_iso(), task["id"]))
# 数据分析时，表单中已知的关键词无需等待飞书解析完成，立即并行采集。
analysis_pool = None
@@ -1080,6 +1276,8 @@ def _fetch_sources(public_id: str) -> None:
note_failures = 0
discovered_keywords: list[str] = []
for index, source in enumerate(sources):
if _acceptance_task_is_cancelled(public_id):
return
try:
response = requests.get(source["source_url"], timeout=25, allow_redirects=True, headers={
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/124 Safari/537.36",
@@ -1152,6 +1350,8 @@ def _fetch_sources(public_id: str) -> None:
conn.execute("UPDATE acceptance_tasks SET progress_percent=?,updated_at=? WHERE public_task_id=?", (progress, now_iso(), public_id))
run_failures = 0
run_error = None
if _acceptance_task_is_cancelled(public_id):
return
if not sources or failures < len(sources):
try:
keywords = _sync_acceptance_keywords(task["id"], discovered_keywords)
@@ -1189,6 +1389,8 @@ def _fetch_sources(public_id: str) -> None:
if analysis_pool is not None:
analysis_pool.shutdown(wait=False)
if task["acceptance_method"] in {"screenshot", "combined"} and not run_error and not run_failures:
if _acceptance_task_is_cancelled(public_id):
return
with connection() as conn:
conn.execute(
"""UPDATE acceptance_tasks SET status='running',progress_percent=70,
@@ -1209,7 +1411,8 @@ def _fetch_sources(public_id: str) -> None:
])) or None
with connection() as conn:
conn.execute(
"UPDATE acceptance_tasks SET status=?,progress_percent=100,error_message=?,updated_at=? WHERE public_task_id=?",
"""UPDATE acceptance_tasks SET status=?,progress_percent=100,error_message=?,updated_at=?
WHERE public_task_id=? AND status!='cancelled'""",
(status, error, now_iso(), public_id),
)
@@ -1282,6 +1485,8 @@ def _resume_data_analysis_worker(public_id: str) -> None:
).fetchall()
failures = 0
for run in runs:
if _acceptance_task_is_cancelled(public_id):
return
try:
if not _run_keyword_with_cookie_failover(
task["id"], run["id"], run["keyword"], task["device_count"],
@@ -1303,7 +1508,7 @@ def _resume_data_analysis_worker(public_id: str) -> None:
error = None if status == "completed" else "Cookie 补采后仍有部分数据未完成"
conn.execute(
"""UPDATE acceptance_tasks SET status=?,progress_percent=100,
error_message=?,updated_at=? WHERE id=?""",
error_message=?,updated_at=? WHERE id=? AND status!='cancelled'""",
(status, error, now_iso(), task["id"]),
)
@@ -1370,9 +1575,13 @@ def _task_dict(conn, task) -> dict[str, Any]:
resource_progress = []
for resource in task_resources:
if resource["resource_type"] == "device":
related_runs = [
all_related_runs = [
row for row in xapi_rows if row["xhs_account_id"] == resource["resource_id"]
]
latest_by_keyword: dict[str, Any] = {}
for row in all_related_runs:
latest_by_keyword[str(row["keyword"] or "")] = row
related_runs = list(latest_by_keyword.values())
completed_keywords = len({
row["keyword"] for row in related_runs
if str(row["status"] or "").lower() == "completed"
@@ -1403,7 +1612,7 @@ def _task_dict(conn, task) -> dict[str, Any]:
"keyword_failed": failed_keywords,
"keyword_running": running_keywords,
"progress_percent": round(terminal_keywords / max(1, total_keywords) * 100),
"attempt_count": len(related_runs),
"attempt_count": len(all_related_runs),
"error_message": next(
(row["feishu_sync_error"] for row in reversed(related_runs) if row["feishu_sync_error"]),
None,
@@ -1456,6 +1665,11 @@ def _task_dict(conn, task) -> dict[str, Any]:
"cookie_count": task["cookie_count"],
"schedule_enabled": bool(task["schedule_enabled"]),
"schedule_weekday": task["schedule_weekday"],
"schedule_weekdays": _decode_json(task["schedule_weekdays"], None) or (
[task["schedule_weekday"]] if task["schedule_weekday"] else []
),
"schedule_start_time": task["schedule_start_time"] or "00:00",
"schedule_end_time": task["schedule_end_time"] or "23:59",
"next_run_at": task["next_run_at"], "last_run_at": task["last_run_at"],
"scheduled_from_task_id": task["scheduled_from_task_id"],
"schedule_error": task["schedule_error"],