feat: 验收任务周期采集与设备资源接口

- 新增 acceptance-tasks 周期采集、立即采集、断点续采与批次管理
- acceptance-resources 返回 xhs_device_accounts 真实设备清单
- 数据看板服务调整及测试更新
This commit is contained in:
2026-08-28 13:43:53 +08:00
parent a021f2fe60
commit e188c50fab
8 changed files with 819 additions and 47 deletions

View File

@@ -1,5 +1,6 @@
import json import json
import csv import csv
import hashlib
import io import io
import platform import platform
import re import re
@@ -8,9 +9,10 @@ import string
import threading import threading
import time import time
from concurrent.futures import ThreadPoolExecutor, as_completed from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime from datetime import datetime, timedelta
from html import unescape from html import unescape
from html.parser import HTMLParser from html.parser import HTMLParser
from http.cookies import SimpleCookie
from typing import Any from typing import Any
from urllib.parse import urlparse from urllib.parse import urlparse
@@ -22,8 +24,10 @@ from crawler_client import (
collect_keyword_sources, collect_keyword_sources,
fetch_content_detail, fetch_content_detail,
get_wen_resources, get_wen_resources,
list_xhs_wen_xapi_devices,
match_feishu_notes, match_feishu_notes,
query_collection, query_collection,
query_xapi_batch_runs,
schedule_xapi_screenshots, schedule_xapi_screenshots,
) )
from database import connection, now_iso, upsert_feishu_note from database import connection, now_iso, upsert_feishu_note
@@ -31,9 +35,11 @@ from task_service import TaskError
ALLOWED_HOST_SUFFIXES = (".feishu.cn", ".larksuite.com") ALLOWED_HOST_SUFFIXES = (".feishu.cn", ".larksuite.com")
METHODS = {"screenshot", "data-analysis"} METHODS = {"screenshot", "data-analysis", "combined"}
WEEKDAYS = set(range(1, 8))
_schedule_started = False
XHS_URL_PATTERN = re.compile( XHS_URL_PATTERN = re.compile(
r"https?://(?:www\.)?(?:xiaohongshu\.com|xhslink\.com)/[^\s\t\r\n<>\"']+", r"https?://(?:www\.)?(?:xiaohongshu\.com|xhslink\.(?:com|cn))/[^\s\t\r\n<>\"']+",
re.IGNORECASE, re.IGNORECASE,
) )
@@ -78,6 +84,34 @@ def _public_id() -> str:
return f"HV-{datetime.now():%Y%m%d}-{''.join(secrets.choice(chars) for _ in range(5))}" return f"HV-{datetime.now():%Y%m%d}-{''.join(secrets.choice(chars) for _ in range(5))}"
def _normalize_custom_cookie(value: Any) -> str:
cookie = str(value or "").strip()
if cookie.lower().startswith("cookie:"):
cookie = cookie.split(":", 1)[1].strip()
if not cookie:
return ""
if len(cookie) > 16384:
raise TaskError("CUSTOM_COOKIE_TOO_LONG", "自定义 Cookie 不能超过 16384 个字符", 422)
parsed = SimpleCookie()
try:
parsed.load(cookie)
except Exception as exc:
raise TaskError("INVALID_CUSTOM_COOKIE", "自定义 Cookie 格式无效", 422) from exc
if not parsed:
raise TaskError("INVALID_CUSTOM_COOKIE", "自定义 Cookie 格式无效", 422)
return cookie
def _custom_cookie_for_task(task_id: int) -> str:
with connection() as conn:
row = conn.execute(
"""SELECT secret_value FROM acceptance_task_secrets
WHERE task_id=? AND secret_type='custom_cookie'""",
(task_id,),
).fetchone()
return str(row["secret_value"] if row else "")
def _validate_url(value: Any) -> str: def _validate_url(value: Any) -> str:
url = str(value or "").strip() url = str(value or "").strip()
parsed = urlparse(url) parsed = urlparse(url)
@@ -343,11 +377,17 @@ def _update_run(run_id: int, *, status: str, result: dict[str, Any] | None = Non
) )
def get_acceptance_resources() -> dict[str, int]: def get_acceptance_resources() -> dict[str, Any]:
resources = get_wen_resources() resources = get_wen_resources()
try:
devices = list_xhs_wen_xapi_devices()
except Exception:
devices = []
return { return {
"cookie_count": max(0, int(resources.get("cookie_count") or 0)), "cookie_count": max(0, int(resources.get("cookie_count") or 0)),
"device_count": max(0, int(resources.get("device_count") or 0)), "device_count": max(0, int(resources.get("device_count") or 0)),
"ocr_questions": resources.get("ocr_questions") or [],
"devices": devices,
} }
@@ -418,11 +458,20 @@ def _run_keyword_with_cookie_failover(
task_id: int, run_id: int, keyword: str, target_count: int, task_id: int, run_id: int, keyword: str, target_count: int,
) -> bool: ) -> bool:
"""补足目标成功数;成功 Cookie 不重跑,失败时自动尝试池中其他 Cookie。""" """补足目标成功数;成功 Cookie 不重跑,失败时自动尝试池中其他 Cookie。"""
resources = get_wen_resources() custom_cookie = _custom_cookie_for_task(task_id)
available_ids = list(dict.fromkeys( custom_credential_id = (
f"custom-{hashlib.sha256(custom_cookie.encode('utf-8')).hexdigest()[:10]}"
if custom_cookie else ""
)
resources = get_wen_resources() if not custom_cookie else {}
available_ids = (
[custom_credential_id]
if custom_cookie
else list(dict.fromkeys(
str(item).strip() for item in resources.get("credential_ids", []) str(item).strip() for item in resources.get("credential_ids", [])
if str(item).strip() if str(item).strip()
)) ))
)
with connection() as conn: with connection() as conn:
now = now_iso() now = now_iso()
for index, credential_id in enumerate(available_ids[:target_count]): for index, credential_id in enumerate(available_ids[:target_count]):
@@ -459,7 +508,11 @@ def _run_keyword_with_cookie_failover(
) )
try: try:
result, crawler_id, remote_status = _wait_collection_result( result, crawler_id, remote_status = _wait_collection_result(
collect_keyword_sources(keyword, credential_id) collect_keyword_sources(
keyword,
None if custom_cookie else credential_id,
custom_cookie or None,
)
) )
if crawler_id: if crawler_id:
crawler_ids.append(crawler_id) crawler_ids.append(crawler_id)
@@ -510,12 +563,53 @@ def _execute_acceptance_method(task_id: int, method: str, keywords: list[str], r
failures = 0 failures = 0
if method == "screenshot": if method == "screenshot":
try: try:
with connection() as conn:
trigger_row = conn.execute(
"SELECT trigger_source FROM acceptance_tasks WHERE id=?", (task_id,),
).fetchone()
trigger_source = str(trigger_row["trigger_source"] if trigger_row else "acceptance")
for run_id in run_ids: for run_id in run_ids:
_update_run(run_id, status="running") _update_run(run_id, status="running")
result = schedule_xapi_screenshots(keywords, resource_count) result = schedule_xapi_screenshots(
crawler_id = str(result.get("task_id") or result.get("schedule_id") or "") or None keywords, resource_count, trigger_source=trigger_source,
for run_id in run_ids: )
_update_run(run_id, status="completed", result=result, crawler_task_id=crawler_id, completed=True) if result.get("execution_mode") == "immediate":
plans = {keyword: {"batches": result.get("batches") or []} for keyword in keywords}
else:
plans = {
str(plan.get("search_query") or ""): plan
for plan in result.get("plans") or [] if isinstance(plan, dict)
}
plan_rows = []
for run_id, keyword in zip(run_ids, keywords):
plan = plans.get(keyword) or {}
batch_ids = [
str(batch.get("batch_id") or "").strip()
for batch in plan.get("batches") or []
if str(batch.get("batch_id") or "").strip()
]
_update_run(
run_id, status="waiting", result=plan or result,
crawler_task_id=",".join(batch_ids) or None,
)
plan_rows.append(plan)
with connection() as conn:
resource_order = 0
seen_resources = set()
for plan in plan_rows:
for batch in plan.get("batches") or []:
for item in batch.get("items") or []:
account_id = str(item.get("xhs_account_id") or "").strip()
if not account_id or account_id in seen_resources:
continue
seen_resources.add(account_id)
conn.execute(
"""INSERT OR IGNORE INTO acceptance_task_resources
(task_id,resource_type,resource_id,resource_order,created_at)
VALUES (?,'device',?,?,?)""",
(task_id, account_id, resource_order, now_iso()),
)
resource_order += 1
except Exception as exc: except Exception as exc:
failures = len(run_ids) failures = len(run_ids)
for run_id in run_ids: for run_id in run_ids:
@@ -551,25 +645,218 @@ def _execute_acceptance_method(task_id: int, method: str, keywords: list[str], r
return failures return failures
def _decode_json(value: Any, fallback: Any) -> Any:
if not value:
return fallback
if not isinstance(value, str):
return value
try:
return json.loads(value)
except (TypeError, json.JSONDecodeError):
return fallback
def _upsert_acceptance_xapi_runs(
task_id: int, keyword_run_id: int | None, batch_id: str,
runs: list[dict[str, Any]],
) -> None:
now = now_iso()
with connection() as conn:
for run in runs:
xapi_task_id = str(run.get("task_id") or run.get("id") or "").strip()
if not xapi_task_id:
continue
artifacts = run.get("artifacts")
if artifacts is None:
artifacts = _decode_json(run.get("artifacts_json"), [])
feishu_record = run.get("feishu_record")
if feishu_record is None:
feishu_record = _decode_json(run.get("feishu_record_json"), None)
conn.execute(
"""INSERT INTO acceptance_xapi_runs
(task_id,keyword_run_id,batch_id,xapi_task_id,keyword,xhs_account_id,
device_serial,device_label,status,writeback_status,source_base_url,
base_record_id,artifacts_json,feishu_result_text,feishu_record_json,
feishu_sync_status,feishu_sync_error,raw_json,created_at,updated_at)
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
ON CONFLICT(xapi_task_id) DO UPDATE SET
keyword_run_id=excluded.keyword_run_id,batch_id=excluded.batch_id,
keyword=excluded.keyword,xhs_account_id=excluded.xhs_account_id,
device_serial=excluded.device_serial,device_label=excluded.device_label,
status=excluded.status,writeback_status=excluded.writeback_status,
source_base_url=excluded.source_base_url,base_record_id=excluded.base_record_id,
artifacts_json=excluded.artifacts_json,
feishu_result_text=excluded.feishu_result_text,
feishu_record_json=excluded.feishu_record_json,
feishu_sync_status=excluded.feishu_sync_status,
feishu_sync_error=excluded.feishu_sync_error,
raw_json=excluded.raw_json,updated_at=excluded.updated_at""",
(
task_id, keyword_run_id, batch_id, xapi_task_id,
run.get("search_query"), run.get("xhs_account_id"),
run.get("device_serial"), run.get("device_label"), run.get("status"),
run.get("writeback_status"), run.get("source_base_url"),
run.get("base_record_id"), json.dumps(artifacts or [], ensure_ascii=False),
run.get("feishu_result_text"),
json.dumps(feishu_record, ensure_ascii=False) if feishu_record is not None else None,
run.get("feishu_sync_status"), run.get("feishu_sync_error"),
json.dumps(run, ensure_ascii=False, default=str), now, now,
),
)
def _poll_screenshot_results(public_id: str) -> None:
"""轮询已计划批次,持久化截图和飞书结果,并更新前端任务进度。"""
terminal_statuses = {"completed", "failed", "cancelled", "canceled"}
with connection() as conn:
task = conn.execute(
"SELECT * FROM acceptance_tasks WHERE public_task_id=?", (public_id,),
).fetchone()
if not task or task["acceptance_method"] not in {"screenshot", "combined"}:
return
keyword_runs = conn.execute(
"""SELECT id,keyword,crawler_task_id,result_json FROM acceptance_keyword_runs
WHERE task_id=? AND run_type='screenshot' ORDER BY run_order""",
(task["id"],),
).fetchall()
batch_map: dict[str, tuple[int, str, int]] = {}
for keyword_run in keyword_runs:
plan = _decode_json(keyword_run["result_json"], {}) or {}
batches = plan.get("batches") or []
expected_by_id = {
str(batch.get("batch_id") or ""): len(batch.get("items") or [])
for batch in batches if str(batch.get("batch_id") or "").strip()
}
for batch_id in str(keyword_run["crawler_task_id"] or "").split(","):
normalized = batch_id.strip()
if normalized:
batch_map[normalized] = (
keyword_run["id"], keyword_run["keyword"],
max(1, expected_by_id.get(normalized, 1)),
)
if not batch_map:
with connection() as conn:
conn.execute(
"""UPDATE acceptance_tasks SET status='failed',progress_percent=100,
error_message=?,updated_at=? WHERE id=?""",
("截图调度没有返回可回查的 batch_id", now_iso(), task["id"]),
)
return
deadline = time.monotonic() + max(1.0, Config.XAPI_RESULT_MAX_WAIT_HOURS) * 3600
while True:
poll_errors = []
for batch_id, (keyword_run_id, _, _) in batch_map.items():
try:
result = query_xapi_batch_runs(batch_id, refresh=True)
_upsert_acceptance_xapi_runs(
task["id"], keyword_run_id, batch_id, result.get("runs") or [],
)
except Exception as exc:
poll_errors.append(f"{batch_id}: {exc}")
with connection() as conn:
rows = conn.execute(
"SELECT * FROM acceptance_xapi_runs WHERE task_id=?", (task["id"],),
).fetchall()
expected_total = sum(item[2] for item in batch_map.values())
terminal_count = sum(
str(row["status"] or "").lower() in terminal_statuses for row in rows
)
feishu_ready = all(
str(row["status"] or "").lower() != "completed"
or str(row["feishu_sync_status"] or "").lower() == "succeeded"
for row in rows
)
received = min(len(rows), expected_total)
progress = min(99, 70 + round(29 * received / max(1, expected_total)))
for keyword_run in keyword_runs:
related = [row for row in rows if row["keyword_run_id"] == keyword_run["id"]]
related_batch_ids = [
value.strip() for value in str(keyword_run["crawler_task_id"] or "").split(",")
if value.strip()
]
keyword_expected = sum(batch_map[item][2] for item in related_batch_ids)
keyword_terminal = sum(
str(row["status"] or "").lower() in terminal_statuses for row in related
)
if keyword_expected and keyword_terminal >= keyword_expected:
failed = any(str(row["status"] or "").lower() != "completed" for row in related)
conn.execute(
"""UPDATE acceptance_keyword_runs SET status=?,error_message=?,
completed_at=?,updated_at=? WHERE id=?""",
("failed" if failed else "completed",
"部分设备截图失败" if failed else None,
now_iso(), now_iso(), keyword_run["id"]),
)
all_terminal = expected_total > 0 and terminal_count >= expected_total
timed_out = time.monotonic() >= deadline
if all_terminal and feishu_ready:
failed_count = sum(
str(row["status"] or "").lower() != "completed" for row in rows
)
final_status = "partial" if failed_count else "completed"
conn.execute(
"""UPDATE acceptance_tasks SET status=?,progress_percent=100,
error_message=?,updated_at=? WHERE id=?""",
(final_status, "部分设备截图失败" if failed_count else None,
now_iso(), task["id"]),
)
return
if timed_out:
reason = "截图或飞书结果等待超时"
if poll_errors:
reason += ";" + ";".join(poll_errors[:3])
conn.execute(
"""UPDATE acceptance_tasks SET status='partial',progress_percent=100,
error_message=?,updated_at=? WHERE id=?""",
(reason[:2000], now_iso(), task["id"]),
)
return
conn.execute(
"UPDATE acceptance_tasks SET status='running',progress_percent=?,updated_at=? WHERE id=?",
(progress, now_iso(), task["id"]),
)
time.sleep(max(5.0, Config.XAPI_RESULT_POLL_INTERVAL_SECONDS))
def create_acceptance_task(data: dict[str, Any]) -> dict[str, Any]: def create_acceptance_task(data: dict[str, Any]) -> dict[str, Any]:
links = list(dict.fromkeys(_validate_url(item) for item in data.get("feishu_links", []))) links = list(dict.fromkeys(_validate_url(item) for item in data.get("feishu_links", [])))
if not links:
raise TaskError("FEISHU_LINK_REQUIRED", "至少需要一个公开飞书链接", 422)
method = str(data.get("acceptance_method") or "") method = str(data.get("acceptance_method") or "")
if method not in METHODS: if method not in METHODS:
raise TaskError("INVALID_ACCEPTANCE_METHOD", "验收方式无效", 422) raise TaskError("INVALID_ACCEPTANCE_METHOD", "验收方式无效", 422)
keywords = list(dict.fromkeys(str(item).strip() for item in data.get("keywords", []) if str(item).strip())) custom_cookie = _normalize_custom_cookie(data.get("custom_cookie"))
if custom_cookie and method not in {"data-analysis", "combined"}:
raise TaskError(
"CUSTOM_COOKIE_NOT_SUPPORTED",
"自定义 Cookie 仅支持数据分析验收",
422,
)
raw_resources = get_wen_resources() raw_resources = get_wen_resources()
keywords = list(dict.fromkeys(
str(item).strip() for item in data.get("keywords", []) if str(item).strip()
))
match_keywords = list(dict.fromkeys(
str(item).strip() for item in data.get("match_keywords", []) if str(item).strip()
))
if len(keywords) > 20:
raise TaskError("KEYWORD_LIMIT_EXCEEDED", "单次最多支持 20 个搜索词", 422)
if method in {"screenshot", "combined"} and not keywords:
raise TaskError("KEYWORD_REQUIRED", "截图取证请至少输入一个搜索词", 422)
if method in {"data-analysis", "combined"} and not links:
raise TaskError("FEISHU_LINK_REQUIRED", "笔记收录请至少输入一个公开飞书链接", 422)
resources = { resources = {
"cookie_count": max(0, int(raw_resources.get("cookie_count") or 0)), "cookie_count": max(0, int(raw_resources.get("cookie_count") or 0)),
"device_count": max(0, int(raw_resources.get("device_count") or 0)), "device_count": max(0, int(raw_resources.get("device_count") or 0)),
} }
device_count = ( cookie_count = 1 if custom_cookie else resources["cookie_count"]
resources["cookie_count"] if method == "data-analysis" device_count = resources["device_count"] if method == "combined" else (
else int(data.get("device_count") or 1) cookie_count if method == "data-analysis" else int(data.get("device_count") or 1)
) )
available_count = resources["device_count" if method == "screenshot" else "cookie_count"] available_count = resources["device_count"] if method in {"screenshot", "combined"} else cookie_count
resource_name = "设备" if method == "screenshot" else "Cookie" resource_name = "设备" if method in {"screenshot", "combined"} else "Cookie"
if method == "combined" and cookie_count < 1:
raise TaskError("INVALID_RESOURCE_COUNT", "当前没有可用的数据分析 Cookie", 422)
if not 1 <= device_count <= available_count: if not 1 <= device_count <= available_count:
raise TaskError( raise TaskError(
"INVALID_RESOURCE_COUNT", "INVALID_RESOURCE_COUNT",
@@ -583,22 +870,38 @@ def create_acceptance_task(data: dict[str, Any]) -> dict[str, Any]:
cursor = conn.execute( cursor = conn.execute(
"""INSERT INTO acceptance_tasks """INSERT INTO acceptance_tasks
(public_task_id,name,acceptance_method,status,progress_percent,device_count, (public_task_id,name,acceptance_method,status,progress_percent,device_count,
created_by_id,created_by_name,created_at,updated_at) created_by_id,created_by_name,created_at,updated_at,trigger_source,cookie_count)
VALUES (?,?,?,'waiting',0,?,?,?,?,?)""", VALUES (?,?,?,'waiting',0,?,?,?,?,?,?,?)""",
(public_id, name, method, device_count, str(data.get("created_by_id") or "local-dev"), (public_id, name, method, device_count, str(data.get("created_by_id") or "local-dev"),
str(data.get("created_by_name") or "本地开发用户"), now, now), str(data.get("created_by_name") or "本地开发用户"), now, now,
str(data.get("trigger_source") or "acceptance"), cookie_count),
) )
task_id = cursor.lastrowid task_id = cursor.lastrowid
conn.executemany( conn.executemany(
"INSERT INTO acceptance_keywords(task_id,keyword_order,keyword,created_at) VALUES (?,?,?,?)", "INSERT INTO acceptance_keywords(task_id,keyword_order,keyword,created_at) VALUES (?,?,?,?)",
[(task_id, index, keyword, now) for index, keyword in enumerate(keywords)], [(task_id, index, keyword, now) for index, keyword in enumerate(keywords)],
) )
if method == "data-analysis": conn.executemany(
"INSERT INTO acceptance_match_keywords(task_id,keyword_order,keyword,created_at) VALUES (?,?,?,?)",
[(task_id, index, keyword, now) for index, keyword in enumerate(match_keywords)],
)
if method in {"data-analysis", "combined"}:
if custom_cookie:
credential_ids = [
f"custom-{hashlib.sha256(custom_cookie.encode('utf-8')).hexdigest()[:10]}"
]
conn.execute(
"""INSERT INTO acceptance_task_secrets
(task_id,secret_type,secret_value,created_at)
VALUES (?,'custom_cookie',?,?)""",
(task_id, custom_cookie, now),
)
else:
credential_ids = [ credential_ids = [
str(item).strip() str(item).strip()
for item in raw_resources.get("credential_ids", []) for item in raw_resources.get("credential_ids", [])
if str(item).strip() if str(item).strip()
][:device_count] ][:cookie_count]
conn.executemany( conn.executemany(
"""INSERT OR IGNORE INTO acceptance_task_resources """INSERT OR IGNORE INTO acceptance_task_resources
(task_id,resource_type,resource_id,resource_order,created_at) (task_id,resource_type,resource_id,resource_order,created_at)
@@ -615,6 +918,143 @@ def create_acceptance_task(data: dict[str, Any]) -> dict[str, Any]:
return get_acceptance_task(public_id) return get_acceptance_task(public_id)
def _next_weekly_run(weekday: int, *, after: datetime | None = None) -> str:
current = after or datetime.now().astimezone()
days = (weekday - current.isoweekday()) % 7
candidate = (current + timedelta(days=days)).replace(
hour=0, minute=0, second=0, microsecond=0,
)
if candidate <= current:
candidate += timedelta(days=7)
return candidate.isoformat(timespec="seconds")
def update_acceptance_schedule(public_id: str, weekday: int) -> dict[str, Any]:
try:
weekday = int(weekday)
except (TypeError, ValueError) as exc:
raise TaskError("INVALID_WEEKDAY", "采集周期必须选择周一至周日", 422) from exc
if weekday not in WEEKDAYS:
raise TaskError("INVALID_WEEKDAY", "采集周期必须选择周一至周日", 422)
with connection() as conn:
task = conn.execute(
"SELECT id FROM acceptance_tasks WHERE public_task_id=?", (public_id,),
).fetchone()
if not task:
raise TaskError("ACCEPTANCE_TASK_NOT_FOUND", "验收任务不存在", 404)
conn.execute(
"""UPDATE acceptance_tasks SET schedule_enabled=1,schedule_weekday=?,
next_run_at=?,schedule_error=NULL,updated_at=? WHERE id=?""",
(weekday, _next_weekly_run(weekday), now_iso(), task["id"]),
)
return get_acceptance_task(public_id)
def stop_acceptance_schedule(public_id: str) -> dict[str, Any]:
with connection() as conn:
result = conn.execute(
"""UPDATE acceptance_tasks SET schedule_enabled=0,next_run_at=NULL,
schedule_error=NULL,updated_at=? WHERE public_task_id=?""",
(now_iso(), public_id),
)
if not result.rowcount:
raise TaskError("ACCEPTANCE_TASK_NOT_FOUND", "验收任务不存在", 404)
return get_acceptance_task(public_id)
def _copy_acceptance_task(public_id: str, *, trigger_source: str = "acceptance") -> dict[str, Any]:
with connection() as conn:
task = conn.execute(
"SELECT * FROM acceptance_tasks WHERE public_task_id=?", (public_id,),
).fetchone()
if not task:
raise TaskError("ACCEPTANCE_TASK_NOT_FOUND", "验收任务不存在", 404)
keywords = [row["keyword"] for row in conn.execute(
"SELECT keyword FROM acceptance_keywords WHERE task_id=? ORDER BY keyword_order",
(task["id"],),
).fetchall()]
match_keywords = [row["keyword"] for row in conn.execute(
"SELECT keyword FROM acceptance_match_keywords WHERE task_id=? ORDER BY keyword_order",
(task["id"],),
).fetchall()]
links = [row["source_url"] for row in conn.execute(
"SELECT source_url FROM acceptance_feishu_sources WHERE task_id=? ORDER BY source_order",
(task["id"],),
).fetchall()]
secret = conn.execute(
"""SELECT secret_value FROM acceptance_task_secrets
WHERE task_id=? AND secret_type='custom_cookie'""",
(task["id"],),
).fetchone()
child = create_acceptance_task({
"name": task["name"],
"acceptance_method": task["acceptance_method"],
"feishu_links": links,
"keywords": keywords,
"match_keywords": match_keywords,
"device_count": task["device_count"],
"created_by_id": task["created_by_id"],
"created_by_name": task["created_by_name"],
"custom_cookie": secret["secret_value"] if secret else None,
"trigger_source": trigger_source,
})
with connection() as conn:
conn.execute(
"UPDATE acceptance_tasks SET scheduled_from_task_id=? WHERE public_task_id=?",
(task["id"], child["task_id"]),
)
return get_acceptance_task(child["task_id"])
def collect_acceptance_now(public_id: str) -> dict[str, Any]:
"""复用任务配置创建一次独立采集,截图任务会立即向 Framework 下发。"""
return _copy_acceptance_task(public_id, trigger_source="acceptance")
def run_due_acceptance_schedules() -> int:
now = now_iso()
with connection() as conn:
due = conn.execute(
"""SELECT public_task_id,schedule_weekday FROM acceptance_tasks
WHERE schedule_enabled=1 AND next_run_at IS NOT NULL AND next_run_at<=?
ORDER BY next_run_at LIMIT 20""",
(now,),
).fetchall()
for row in due:
conn.execute(
"""UPDATE acceptance_tasks SET next_run_at=?,last_run_at=?,
schedule_error=NULL,updated_at=? WHERE public_task_id=?""",
(_next_weekly_run(row["schedule_weekday"]), now, now, row["public_task_id"]),
)
for row in due:
try:
_copy_acceptance_task(row["public_task_id"], trigger_source="acceptance-weekly")
except Exception as exc:
with connection() as conn:
conn.execute(
"UPDATE acceptance_tasks SET schedule_error=?,updated_at=? WHERE public_task_id=?",
(str(exc)[:2000], now_iso(), row["public_task_id"]),
)
return len(due)
def start_acceptance_schedule_worker() -> None:
global _schedule_started
if _schedule_started:
return
_schedule_started = True
def worker() -> None:
while True:
try:
run_due_acceptance_schedules()
except Exception:
pass
time.sleep(30)
threading.Thread(target=worker, daemon=True, name="acceptance-weekly-scheduler").start()
def _fetch_sources(public_id: str) -> None: def _fetch_sources(public_id: str) -> None:
with connection() as conn: with connection() as conn:
task = conn.execute("SELECT * FROM acceptance_tasks WHERE public_task_id=?", (public_id,)).fetchone() task = conn.execute("SELECT * FROM acceptance_tasks WHERE public_task_id=?", (public_id,)).fetchone()
@@ -629,11 +1069,12 @@ def _fetch_sources(public_id: str) -> None:
# 数据分析时,表单中已知的关键词无需等待飞书解析完成,立即并行采集。 # 数据分析时,表单中已知的关键词无需等待飞书解析完成,立即并行采集。
analysis_pool = None analysis_pool = None
initial_analysis_future = None initial_analysis_future = None
if task["acceptance_method"] == "data-analysis" and initial_keywords: if task["acceptance_method"] in {"data-analysis", "combined"} and initial_keywords:
analysis_pool = ThreadPoolExecutor(max_workers=1, thread_name_prefix=f"analysis-{public_id}") analysis_pool = ThreadPoolExecutor(max_workers=1, thread_name_prefix=f"analysis-{public_id}")
initial_analysis_future = analysis_pool.submit( initial_analysis_future = analysis_pool.submit(
_execute_acceptance_method, _execute_acceptance_method,
task["id"], task["acceptance_method"], initial_keywords, task["device_count"], task["id"], "data-analysis", initial_keywords,
task["cookie_count"] if task["acceptance_method"] == "combined" else task["device_count"],
) )
failures = 0 failures = 0
note_failures = 0 note_failures = 0
@@ -711,7 +1152,7 @@ def _fetch_sources(public_id: str) -> None:
conn.execute("UPDATE acceptance_tasks SET progress_percent=?,updated_at=? WHERE public_task_id=?", (progress, now_iso(), public_id)) conn.execute("UPDATE acceptance_tasks SET progress_percent=?,updated_at=? WHERE public_task_id=?", (progress, now_iso(), public_id))
run_failures = 0 run_failures = 0
run_error = None run_error = None
if failures < len(sources): if not sources or failures < len(sources):
try: try:
keywords = _sync_acceptance_keywords(task["id"], discovered_keywords) keywords = _sync_acceptance_keywords(task["id"], discovered_keywords)
with connection() as conn: with connection() as conn:
@@ -721,7 +1162,12 @@ def _fetch_sources(public_id: str) -> None:
new_keywords = [keyword for keyword in keywords if keyword not in initial_keywords] new_keywords = [keyword for keyword in keywords if keyword not in initial_keywords]
if new_keywords: if new_keywords:
run_failures += _execute_acceptance_method( run_failures += _execute_acceptance_method(
task["id"], task["acceptance_method"], new_keywords, task["device_count"] task["id"], "data-analysis", new_keywords,
task["cookie_count"] if task["acceptance_method"] == "combined" else task["device_count"],
)
if task["acceptance_method"] == "combined":
run_failures += _execute_acceptance_method(
task["id"], "screenshot", keywords, task["device_count"],
) )
else: else:
run_failures = _execute_acceptance_method( run_failures = _execute_acceptance_method(
@@ -742,6 +1188,19 @@ def _fetch_sources(public_id: str) -> None:
finally: finally:
if analysis_pool is not None: if analysis_pool is not None:
analysis_pool.shutdown(wait=False) analysis_pool.shutdown(wait=False)
if task["acceptance_method"] in {"screenshot", "combined"} and not run_error and not run_failures:
with connection() as conn:
conn.execute(
"""UPDATE acceptance_tasks SET status='running',progress_percent=70,
error_message=?,updated_at=? WHERE public_task_id=?""",
("部分飞书资料或小红书笔记解析失败" if failures or note_failures else None,
now_iso(), public_id),
)
threading.Thread(
target=_poll_screenshot_results,
args=(public_id,), daemon=True, name=f"acceptance-xapi-{public_id}",
).start()
return
status = "failed" if failures == len(sources) or run_error else ("partial" if failures or note_failures or run_failures else "completed") status = "failed" if failures == len(sources) or run_error else ("partial" if failures or note_failures or run_failures else "completed")
error = ";".join(filter(None, [ error = ";".join(filter(None, [
"部分飞书资料或小红书笔记解析失败" if failures or note_failures else None, "部分飞书资料或小红书笔记解析失败" if failures or note_failures else None,
@@ -782,13 +1241,23 @@ def resume_acceptance_task(public_id: str) -> dict[str, Any]:
def resume_unfinished_acceptance_tasks() -> None: def resume_unfinished_acceptance_tasks() -> None:
"""服务重启后恢复等待中或运行中的验收任务。""" """服务重启后恢复等待中或运行中的验收任务。"""
with connection() as conn: with connection() as conn:
task_ids = [row["public_task_id"] for row in conn.execute( tasks = conn.execute(
"""SELECT public_task_id FROM acceptance_tasks """SELECT t.public_task_id,t.acceptance_method,
EXISTS(SELECT 1 FROM acceptance_keyword_runs r
WHERE r.task_id=t.id AND r.run_type='screenshot'
AND r.crawler_task_id IS NOT NULL) AS has_screenshot_batches
FROM acceptance_tasks t
WHERE status IN ('waiting','running') ORDER BY created_at""" WHERE status IN ('waiting','running') ORDER BY created_at"""
).fetchall()] ).fetchall()
for public_id in task_ids: for task in tasks:
public_id = task["public_task_id"]
target = (
_poll_screenshot_results
if task["acceptance_method"] in {"screenshot", "combined"} and task["has_screenshot_batches"]
else _fetch_sources
)
threading.Thread( threading.Thread(
target=_fetch_sources, args=(public_id,), daemon=True, target=target, args=(public_id,), daemon=True,
name=f"acceptance-recover-{public_id}", name=f"acceptance-recover-{public_id}",
).start() ).start()
@@ -843,6 +1312,17 @@ def _task_dict(conn, task) -> dict[str, Any]:
keywords = [row["keyword"] for row in conn.execute( keywords = [row["keyword"] for row in conn.execute(
"SELECT keyword FROM acceptance_keywords WHERE task_id=? ORDER BY keyword_order", (task["id"],) "SELECT keyword FROM acceptance_keywords WHERE task_id=? ORDER BY keyword_order", (task["id"],)
).fetchall()] ).fetchall()]
match_keywords = [row["keyword"] for row in conn.execute(
"SELECT keyword FROM acceptance_match_keywords WHERE task_id=? ORDER BY keyword_order",
(task["id"],),
).fetchall()]
root_id = task["scheduled_from_task_id"] or task["id"]
batches = [dict(row) for row in conn.execute(
"""SELECT public_task_id,status,progress_percent,created_at,updated_at
FROM acceptance_tasks WHERE id=? OR scheduled_from_task_id=?
ORDER BY created_at DESC""",
(root_id, root_id),
).fetchall()]
sources = [dict(row) for row in conn.execute( sources = [dict(row) for row in conn.execute(
"""SELECT source_url,final_url,source_type,title,content_text,fetch_status,http_status, """SELECT source_url,final_url,source_type,title,content_text,fetch_status,http_status,
error_message,fetched_at, error_message,fetched_at,
@@ -879,8 +1359,57 @@ def _task_dict(conn, task) -> dict[str, Any]:
FROM acceptance_cookie_attempts WHERE task_id=? ORDER BY id""", FROM acceptance_cookie_attempts WHERE task_id=? ORDER BY id""",
(task["id"],), (task["id"],),
).fetchall() ).fetchall()
xapi_rows = conn.execute(
"""SELECT batch_id,xapi_task_id,keyword,xhs_account_id,device_serial,
device_label,status,writeback_status,source_base_url,base_record_id,
artifacts_json,feishu_result_text,feishu_record_json,
feishu_sync_status,feishu_sync_error,updated_at
FROM acceptance_xapi_runs WHERE task_id=? ORDER BY id""",
(task["id"],),
).fetchall()
resource_progress = [] resource_progress = []
for resource in task_resources: for resource in task_resources:
if resource["resource_type"] == "device":
related_runs = [
row for row in xapi_rows if row["xhs_account_id"] == resource["resource_id"]
]
completed_keywords = len({
row["keyword"] for row in related_runs
if str(row["status"] or "").lower() == "completed"
and str(row["feishu_sync_status"] or "").lower() == "succeeded"
})
failed_keywords = len({
row["keyword"] for row in related_runs
if str(row["status"] or "").lower() in {"failed", "cancelled", "canceled"}
})
running_keywords = len({
row["keyword"] for row in related_runs
if str(row["status"] or "").lower() not in {"completed", "failed", "cancelled", "canceled"}
})
total_keywords = len(keywords)
terminal_keywords = completed_keywords + failed_keywords
resource_progress.append({
"resource_type": "device",
"resource_id": resource["resource_id"],
"resource_order": resource["resource_order"],
"status": (
"failed" if failed_keywords
else "completed" if total_keywords and completed_keywords >= total_keywords
else "running" if related_runs
else "waiting"
),
"keyword_total": total_keywords,
"keyword_completed": completed_keywords,
"keyword_failed": failed_keywords,
"keyword_running": running_keywords,
"progress_percent": round(terminal_keywords / max(1, total_keywords) * 100),
"attempt_count": len(related_runs),
"error_message": next(
(row["feishu_sync_error"] for row in reversed(related_runs) if row["feishu_sync_error"]),
None,
),
})
continue
related = [row for row in cookie_attempts if row["credential_id"] == resource["resource_id"]] related = [row for row in cookie_attempts if row["credential_id"] == resource["resource_id"]]
completed_keywords = failed_keywords = running_keywords = 0 completed_keywords = failed_keywords = running_keywords = 0
for keyword in keywords: for keyword in keywords:
@@ -914,14 +1443,29 @@ def _task_dict(conn, task) -> dict[str, Any]:
"attempt_count": len(related), "attempt_count": len(related),
"error_message": next((row["error_message"] for row in reversed(related) if row["error_message"]), None), "error_message": next((row["error_message"] for row in reversed(related) if row["error_message"]), None),
}) })
xapi_runs = []
for row in xapi_rows:
item = dict(row)
item["artifacts"] = _decode_json(item.pop("artifacts_json"), [])
item["feishu_record"] = _decode_json(item.pop("feishu_record_json"), None)
xapi_runs.append(item)
return { return {
"task_id": task["public_task_id"], "name": task["name"], "task_id": task["public_task_id"], "name": task["name"],
"acceptance_method": task["acceptance_method"], "status": task["status"], "acceptance_method": task["acceptance_method"], "status": task["status"],
"progress_percent": task["progress_percent"], "device_count": task["device_count"], "progress_percent": task["progress_percent"], "device_count": task["device_count"],
"keywords": keywords, "feishu_links": [item["source_url"] for item in sources], "cookie_count": task["cookie_count"],
"schedule_enabled": bool(task["schedule_enabled"]),
"schedule_weekday": task["schedule_weekday"],
"next_run_at": task["next_run_at"], "last_run_at": task["last_run_at"],
"scheduled_from_task_id": task["scheduled_from_task_id"],
"schedule_error": task["schedule_error"],
"trigger_source": task["trigger_source"],
"keywords": keywords, "match_keywords": match_keywords,
"batches": batches, "feishu_links": [item["source_url"] for item in sources],
"feishu_sources": sources, "created_by": {"id": task["created_by_id"], "name": task["created_by_name"]}, "feishu_sources": sources, "created_by": {"id": task["created_by_id"], "name": task["created_by_name"]},
"note_stats": {"total": note_stats["total"] or 0, "completed": note_stats["completed"] or 0, "failed": note_stats["failed"] or 0}, "note_stats": {"total": note_stats["total"] or 0, "completed": note_stats["completed"] or 0, "failed": note_stats["failed"] or 0},
"keyword_runs": keyword_runs, "keyword_runs": keyword_runs,
"xapi_runs": xapi_runs,
"resource_progress": resource_progress, "resource_progress": resource_progress,
"error_message": task["error_message"], "created_at": task["created_at"], "updated_at": task["updated_at"], "error_message": task["error_message"], "created_at": task["created_at"], "updated_at": task["updated_at"],
} }

View File

@@ -12,6 +12,53 @@
} }
], ],
"paths": { "paths": {
"/api/v1/acceptance-tasks": {
"post": {
"summary": "创建项目验收任务",
"description": "关键词截图取证必须填写 keywords,无需飞书链接;笔记收录必须填写 feishu_links,可选传入仅供本任务使用的 custom_cookie。",
"tags": ["项目验收"],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {"$ref": "#/components/schemas/CreateAcceptanceTask"}
}
}
},
"responses": {"201": {"description": "验收任务创建成功"}}
}
},
"/api/v1/acceptance-tasks/{task_id}/collect-now": {
"post": {
"summary": "立即采集并下发指令",
"description": "复用原任务配置创建独立的新一轮采集。截图取证会立即请求 Framework 下发设备指令。",
"tags": ["项目验收"],
"parameters": [{"$ref": "#/components/parameters/TaskId"}],
"responses": {"201": {"description": "新一轮验收任务已创建"}}
}
},
"/api/v1/acceptance-tasks/{task_id}/schedule": {
"put": {
"summary": "设置每周采集周期",
"description": "按 Asia/Shanghai 时区,在指定星期的 00:00 创建新一轮采集。",
"tags": ["项目验收"],
"parameters": [{"$ref": "#/components/parameters/TaskId"}],
"requestBody": {
"required": true,
"content": {"application/json": {"schema": {"type": "object", "required": ["weekday"], "properties": {"weekday": {"type": "integer", "minimum": 1, "maximum": 7, "description": "1=周一,7=周日"}}}}}
},
"responses": {"200": {"description": "周期设置成功"}}
}
},
"/api/v1/acceptance-tasks/{task_id}/schedule/stop": {
"post": {
"summary": "停止周期采集",
"description": "关闭该任务后续的每周执行,不影响已经开始的采集任务。",
"tags": ["项目验收"],
"parameters": [{"$ref": "#/components/parameters/TaskId"}],
"responses": {"200": {"description": "周期采集已停止"}}
}
},
"/api/v1/tasks/{task_id}/cancel": { "/api/v1/tasks/{task_id}/cancel": {
"post": { "post": {
"summary": "停止任务", "summary": "停止任务",
@@ -58,6 +105,36 @@
} }
}, },
"components": { "components": {
"schemas": {
"CreateAcceptanceTask": {
"type": "object",
"required": ["acceptance_method"],
"properties": {
"name": {"type": "string"},
"acceptance_method": {"type": "string", "enum": ["screenshot", "data-analysis"]},
"feishu_links": {
"type": "array",
"items": {"type": "string", "format": "uri"},
"description": "笔记收录必填;关键词截图取证可为空。"
},
"keywords": {
"type": "array",
"maxItems": 20,
"items": {"type": "string"},
"description": "关键词截图取证至少填写 1 个;笔记收录选填。"
},
"device_count": {"type": "integer", "minimum": 1},
"custom_cookie": {
"type": "string",
"maxLength": 16384,
"writeOnly": true,
"description": "仅用于笔记收录。填写后仅使用该 Cookie;留空时使用 Cookie 池全部可用凭据。不会在任务详情中返回。"
},
"created_by_id": {"type": "string"},
"created_by_name": {"type": "string"}
}
}
},
"parameters": { "parameters": {
"TaskId": { "TaskId": {
"name": "task_id", "name": "task_id",

29
app.py
View File

@@ -17,6 +17,7 @@ from auth_service import (
from config import Config from config import Config
from database import init_database from database import init_database
from ocr_dashboard_service import ( from ocr_dashboard_service import (
get_comparison as get_ocr_dashboard_comparison,
get_result_matches as get_ocr_result_matches, get_result_matches as get_ocr_result_matches,
get_summary as get_ocr_dashboard_summary, get_summary as get_ocr_dashboard_summary,
list_batches as list_ocr_batches, list_batches as list_ocr_batches,
@@ -27,6 +28,7 @@ from ocr_dashboard_service import (
) )
from acceptance_service import ( from acceptance_service import (
create_acceptance_task, create_acceptance_task,
collect_acceptance_now,
get_acceptance_resources, get_acceptance_resources,
get_acceptance_link_ai_data, get_acceptance_link_ai_data,
get_acceptance_task, get_acceptance_task,
@@ -34,6 +36,9 @@ from acceptance_service import (
list_acceptance_tasks, list_acceptance_tasks,
resume_acceptance_task, resume_acceptance_task,
resume_unfinished_acceptance_tasks, resume_unfinished_acceptance_tasks,
start_acceptance_schedule_worker,
stop_acceptance_schedule,
update_acceptance_schedule,
) )
from rule_service import ( from rule_service import (
delete_rule_document, delete_rule_document,
@@ -308,6 +313,22 @@ def create_app() -> Flask:
def resume_project_acceptance_task(task_id: str): def resume_project_acceptance_task(task_id: str):
return jsonify({"data": resume_acceptance_task(task_id), "request_id": g.request_id}) return jsonify({"data": resume_acceptance_task(task_id), "request_id": g.request_id})
@app.route("/api/v1/acceptance-tasks/<task_id>/collect-now", methods=["POST"])
def collect_project_acceptance_now(task_id: str):
return jsonify({"data": collect_acceptance_now(task_id), "request_id": g.request_id}), 201
@app.route("/api/v1/acceptance-tasks/<task_id>/schedule", methods=["PUT"])
def configure_project_acceptance_schedule(task_id: str):
payload = request.get_json(silent=True) or {}
return jsonify({
"data": update_acceptance_schedule(task_id, payload.get("weekday")),
"request_id": g.request_id,
})
@app.route("/api/v1/acceptance-tasks/<task_id>/schedule/stop", methods=["POST"])
def stop_project_acceptance_schedule(task_id: str):
return jsonify({"data": stop_acceptance_schedule(task_id), "request_id": g.request_id})
@app.route("/api/v1/data-dashboard/ocr/batches", methods=["GET"]) @app.route("/api/v1/data-dashboard/ocr/batches", methods=["GET"])
def query_ocr_dashboard_batches(): def query_ocr_dashboard_batches():
return jsonify({"data": list_ocr_batches(), "request_id": g.request_id}) return jsonify({"data": list_ocr_batches(), "request_id": g.request_id})
@@ -323,6 +344,13 @@ def create_app() -> Flask:
request.args.get("collection_key", "").strip(), request.args.get("collection_key", "").strip(),
), "request_id": g.request_id}) ), "request_id": g.request_id})
@app.route("/api/v1/data-dashboard/ocr/comparison", methods=["GET"])
def query_ocr_dashboard_comparison():
return jsonify({"data": get_ocr_dashboard_comparison(
request.args.get("batch_key", "").strip(),
request.args.get("collection_key", "").strip(),
), "request_id": g.request_id})
@app.route("/api/v1/data-dashboard/ocr/results", methods=["GET"]) @app.route("/api/v1/data-dashboard/ocr/results", methods=["GET"])
def query_ocr_dashboard_results(): def query_ocr_dashboard_results():
return jsonify({"data": list_ocr_results(request.args.to_dict()), "request_id": g.request_id}) return jsonify({"data": list_ocr_results(request.args.to_dict()), "request_id": g.request_id})
@@ -437,6 +465,7 @@ def create_app() -> Flask:
resume_unfinished_tasks() resume_unfinished_tasks()
resume_unfinished_acceptance_tasks() resume_unfinished_acceptance_tasks()
start_acceptance_schedule_worker()
return app return app

View File

@@ -23,6 +23,8 @@ class Config:
CRAWLER_RETRY_TIMES = int(os.getenv("CRAWLER_RETRY_TIMES", "3")) CRAWLER_RETRY_TIMES = int(os.getenv("CRAWLER_RETRY_TIMES", "3"))
CRAWLER_RETRY_DELAY_SECONDS = float(os.getenv("CRAWLER_RETRY_DELAY_SECONDS", "2")) CRAWLER_RETRY_DELAY_SECONDS = float(os.getenv("CRAWLER_RETRY_DELAY_SECONDS", "2"))
CRAWLER_POLL_INTERVAL_SECONDS = float(os.getenv("CRAWLER_POLL_INTERVAL_SECONDS", "5")) CRAWLER_POLL_INTERVAL_SECONDS = float(os.getenv("CRAWLER_POLL_INTERVAL_SECONDS", "5"))
XAPI_RESULT_POLL_INTERVAL_SECONDS = float(os.getenv("XAPI_RESULT_POLL_INTERVAL_SECONDS", "300"))
XAPI_RESULT_MAX_WAIT_HOURS = float(os.getenv("XAPI_RESULT_MAX_WAIT_HOURS", "48"))
ACCEPTANCE_BROWSER_TIMEOUT_SECONDS = int(os.getenv("ACCEPTANCE_BROWSER_TIMEOUT_SECONDS", "60")) ACCEPTANCE_BROWSER_TIMEOUT_SECONDS = int(os.getenv("ACCEPTANCE_BROWSER_TIMEOUT_SECONDS", "60"))
ACCEPTANCE_NOTE_WORKERS = int(os.getenv("ACCEPTANCE_NOTE_WORKERS", "4")) ACCEPTANCE_NOTE_WORKERS = int(os.getenv("ACCEPTANCE_NOTE_WORKERS", "4"))
SEARCH_QA_API_BASE_URL = os.getenv( SEARCH_QA_API_BASE_URL = os.getenv(

View File

@@ -306,6 +306,17 @@ def get_wen_resources() -> dict[str, Any]:
return _get("/api/v1/xhs/wen/resources") return _get("/api/v1/xhs/wen/resources")
def list_xhs_wen_xapi_devices() -> list[dict[str, Any]]:
"""查询 Framework `xhs_device_accounts` 中启用的设备清单。
通过 Framework 的调度配置接口读取,返回每台设备的账号、设备编码与
可用时间窗口等调度字段,供前端「截图设备列表」直接展示真实设备。
"""
data = _get("/api/v1/xhs/wen/xapi/devices/settings")
items = data.get("items")
return items if isinstance(items, list) else []
def fetch_content_detail(note_url: str) -> dict[str, Any]: def fetch_content_detail(note_url: str) -> dict[str, Any]:
"""按小红书笔记链接读取完整笔记详情。""" """按小红书笔记链接读取完整笔记详情。"""
return _post( return _post(
@@ -315,21 +326,43 @@ def fetch_content_detail(note_url: str) -> dict[str, Any]:
) )
def schedule_xapi_screenshots(search_queries: list[str], device_count: int) -> dict[str, Any]: def schedule_xapi_screenshots(
"""向手机集群下发关键词搜索与截图任务。""" search_queries: list[str], device_count: int, *, trigger_source: str = "acceptance",
) -> dict[str, Any]:
"""向手机集群立即下发关键词搜索与截图任务。"""
queries = normalize_collection_keywords(search_queries) queries = normalize_collection_keywords(search_queries)
return _post( return _post(
"/api/v1/xhs/wen/xapi/schedule", "/api/v1/xhs/wen/xapi/schedule",
{"keywords": queries, "device_count": device_count}, {
"keywords": queries,
"device_count": device_count,
"execute_now": True,
"trigger_source": trigger_source,
},
retry_transient=False, retry_transient=False,
) )
def collect_keyword_sources(keyword: str, credential_id: str | None = None) -> dict[str, Any]: def query_xapi_batch_runs(batch_id: str, *, refresh: bool = True) -> dict[str, Any]:
"""读取某个 Framework 调度批次下的设备任务、截图与飞书结果。"""
normalized = str(batch_id or "").strip()
if not normalized:
raise CrawlerError("batch_id 不能为空")
return _get(
"/api/v1/xhs/wen/xapi/runs"
f"?batch_id={normalized}&refresh={'true' if refresh else 'false'}"
)
def collect_keyword_sources(
keyword: str, credential_id: str | None = None, cookie: str | None = None,
) -> dict[str, Any]:
"""使用指定 Cookie 采集单个关键词。""" """使用指定 Cookie 采集单个关键词。"""
normalized = normalize_collection_keywords([keyword])[0] normalized = normalize_collection_keywords([keyword])[0]
body: dict[str, Any] = {"keyword": normalized} body: dict[str, Any] = {"keyword": normalized}
if credential_id: if cookie:
body["cookie"] = cookie
elif credential_id:
body["credential_id"] = credential_id body["credential_id"] = credential_id
return _post( return _post(
"/api/v1/xhs/wen/sources", "/api/v1/xhs/wen/sources",

View File

@@ -193,6 +193,17 @@ def init_database() -> None:
FOREIGN KEY(task_id) REFERENCES acceptance_tasks(id) ON DELETE CASCADE FOREIGN KEY(task_id) REFERENCES acceptance_tasks(id) ON DELETE CASCADE
); );
CREATE TABLE IF NOT EXISTS acceptance_match_keywords (
id INTEGER PRIMARY KEY AUTOINCREMENT,
task_id INTEGER NOT NULL,
keyword_order INTEGER NOT NULL,
keyword TEXT NOT NULL,
created_at TEXT NOT NULL,
UNIQUE(task_id, keyword_order),
UNIQUE(task_id, keyword),
FOREIGN KEY(task_id) REFERENCES acceptance_tasks(id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS acceptance_feishu_sources ( CREATE TABLE IF NOT EXISTS acceptance_feishu_sources (
id INTEGER PRIMARY KEY AUTOINCREMENT, id INTEGER PRIMARY KEY AUTOINCREMENT,
task_id INTEGER NOT NULL, task_id INTEGER NOT NULL,
@@ -288,6 +299,34 @@ def init_database() -> None:
CREATE INDEX IF NOT EXISTS idx_acceptance_keyword_runs_task CREATE INDEX IF NOT EXISTS idx_acceptance_keyword_runs_task
ON acceptance_keyword_runs(task_id, status); ON acceptance_keyword_runs(task_id, status);
CREATE TABLE IF NOT EXISTS acceptance_xapi_runs (
id INTEGER PRIMARY KEY AUTOINCREMENT,
task_id INTEGER NOT NULL,
keyword_run_id INTEGER,
batch_id TEXT NOT NULL,
xapi_task_id TEXT NOT NULL UNIQUE,
keyword TEXT,
xhs_account_id TEXT,
device_serial TEXT,
device_label TEXT,
status TEXT,
writeback_status TEXT,
source_base_url TEXT,
base_record_id TEXT,
artifacts_json TEXT,
feishu_result_text TEXT,
feishu_record_json TEXT,
feishu_sync_status TEXT,
feishu_sync_error TEXT,
raw_json TEXT NOT NULL,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL,
FOREIGN KEY(task_id) REFERENCES acceptance_tasks(id) ON DELETE CASCADE,
FOREIGN KEY(keyword_run_id) REFERENCES acceptance_keyword_runs(id) ON DELETE SET NULL
);
CREATE INDEX IF NOT EXISTS idx_acceptance_xapi_runs_task
ON acceptance_xapi_runs(task_id, status, batch_id);
CREATE TABLE IF NOT EXISTS acceptance_cookie_attempts ( CREATE TABLE IF NOT EXISTS acceptance_cookie_attempts (
id INTEGER PRIMARY KEY AUTOINCREMENT, id INTEGER PRIMARY KEY AUTOINCREMENT,
task_id INTEGER NOT NULL, task_id INTEGER NOT NULL,
@@ -321,8 +360,33 @@ def init_database() -> None:
); );
CREATE INDEX IF NOT EXISTS idx_acceptance_task_resources CREATE INDEX IF NOT EXISTS idx_acceptance_task_resources
ON acceptance_task_resources(task_id, resource_order); ON acceptance_task_resources(task_id, resource_order);
CREATE TABLE IF NOT EXISTS acceptance_task_secrets (
task_id INTEGER NOT NULL,
secret_type TEXT NOT NULL,
secret_value TEXT NOT NULL,
created_at TEXT NOT NULL,
PRIMARY KEY(task_id, secret_type),
FOREIGN KEY(task_id) REFERENCES acceptance_tasks(id) ON DELETE CASCADE
);
""" """
) )
acceptance_columns = {
row["name"] for row in conn.execute("PRAGMA table_info(acceptance_tasks)").fetchall()
}
acceptance_migrations = {
"schedule_enabled": "INTEGER NOT NULL DEFAULT 0",
"schedule_weekday": "INTEGER",
"next_run_at": "TEXT",
"last_run_at": "TEXT",
"scheduled_from_task_id": "INTEGER",
"schedule_error": "TEXT",
"trigger_source": "TEXT NOT NULL DEFAULT 'acceptance'",
"cookie_count": "INTEGER NOT NULL DEFAULT 0",
}
for column, definition in acceptance_migrations.items():
if column not in acceptance_columns:
conn.execute(f"ALTER TABLE acceptance_tasks ADD COLUMN {column} {definition}")
# 将旧版本保存在 acceptance_xhs_notes.parsed_json 中的成功结果迁移到 # 将旧版本保存在 acceptance_xhs_notes.parsed_json 中的成功结果迁移到
# 全局去重的笔记业务表;任务关联仍由 acceptance_xhs_notes 保留。 # 全局去重的笔记业务表;任务关联仍由 acceptance_xhs_notes 保留。
legacy_rows = conn.execute( legacy_rows = conn.execute(

View File

@@ -52,6 +52,12 @@ def get_summary(batch_key: str = "", collection_key: str = "") -> dict[str, Any]
}) or {} }) or {}
def get_comparison(batch_key: str = "", collection_key: str = "") -> dict[str, Any]:
return _get("/xhs/ocr/dashboard/comparison", {
"batch_key": batch_key, "collection_key": collection_key,
}) or {}
def list_results(params: dict[str, Any]) -> dict[str, Any]: def list_results(params: dict[str, Any]) -> dict[str, Any]:
allowed = { allowed = {
"batch_key", "collection_key", "question_code", "search_keyword", "city", "batch_key", "collection_key", "question_code", "search_keyword", "city",

View File

@@ -1,7 +1,7 @@
import unittest import unittest
from unittest.mock import Mock, patch from unittest.mock import Mock, patch
from ocr_dashboard_service import export_evidences, get_summary, list_batches, list_results from ocr_dashboard_service import export_evidences, get_comparison, get_summary, list_batches, list_results
from task_service import TaskError from task_service import TaskError
@@ -21,6 +21,23 @@ class OcrDashboardServiceTest(unittest.TestCase):
get_summary("b1", "c1") get_summary("b1", "c1")
self.assertEqual(get.call_args.kwargs["params"], {"batch_key": "b1", "collection_key": "c1"}) self.assertEqual(get.call_args.kwargs["params"], {"batch_key": "b1", "collection_key": "c1"})
@patch("ocr_dashboard_service.SESSION.get")
def test_comparison_forwards_scope(self, get):
response = Mock()
response.json.return_value = {
"success": True, "code": 200,
"data": {"current": {}, "previous": {}},
}
get.return_value = response
result = get_comparison("b1", "c1")
self.assertIn("current", result)
self.assertEqual(
get.call_args.kwargs["params"],
{"batch_key": "b1", "collection_key": "c1"},
)
def test_results_rejects_oversized_page(self): def test_results_rejects_oversized_page(self):
with self.assertRaises(TaskError): with self.assertRaises(TaskError):
list_results({"page_size": "201"}) list_results({"page_size": "201"})