验收爬虫与截图按自然日归批
This commit is contained in:
@@ -1183,8 +1183,9 @@ def _copy_acceptance_task(public_id: str, *, trigger_source: str = "acceptance")
|
||||
with connection() as conn:
|
||||
conn.execute(
|
||||
"""UPDATE acceptance_tasks SET scheduled_from_task_id=?,dashboard_batch_key=?,
|
||||
use_all_devices=? WHERE public_task_id=?""",
|
||||
(task["id"], task["dashboard_batch_key"], task["use_all_devices"], child["task_id"]),
|
||||
use_all_devices=?,batch_date=? WHERE public_task_id=?""",
|
||||
(task["id"], task["dashboard_batch_key"], task["use_all_devices"],
|
||||
datetime.now().astimezone().date().isoformat(), child["task_id"]),
|
||||
)
|
||||
return get_acceptance_task(child["task_id"])
|
||||
|
||||
@@ -1195,7 +1196,7 @@ def collect_acceptance_now(public_id: str) -> dict[str, Any]:
|
||||
|
||||
|
||||
def collect_acceptance_crawler_now(public_id: str) -> dict[str, Any]:
|
||||
"""按项目搜索词启动独立的 Cookie 爬虫采集批次,不下发截图设备。"""
|
||||
"""按项目搜索词同时启动 Cookie 爬虫和截图设备,按自然日归入批次。"""
|
||||
with connection() as conn:
|
||||
task = conn.execute("SELECT * FROM acceptance_tasks WHERE public_task_id=?", (public_id,)).fetchone()
|
||||
if not task:
|
||||
@@ -1205,8 +1206,8 @@ def collect_acceptance_crawler_now(public_id: str) -> dict[str, Any]:
|
||||
keywords = [row["keyword"] for row in conn.execute("SELECT keyword FROM acceptance_keywords WHERE task_id=? ORDER BY keyword_order", (root_id,)).fetchall()]
|
||||
matches = [row["keyword"] for row in conn.execute("SELECT keyword FROM acceptance_match_keywords WHERE task_id=? ORDER BY keyword_order", (root_id,)).fetchall()]
|
||||
child = create_acceptance_task({
|
||||
"name": f"{root['name']}|爬虫数据采集",
|
||||
"acceptance_method": "data-analysis",
|
||||
"name": f"{root['name']}|立即完整采集",
|
||||
"acceptance_method": "combined",
|
||||
"feishu_links": [],
|
||||
"keywords": keywords,
|
||||
"match_keywords": matches,
|
||||
@@ -1216,9 +1217,10 @@ def collect_acceptance_crawler_now(public_id: str) -> dict[str, Any]:
|
||||
})
|
||||
with connection() as conn:
|
||||
conn.execute(
|
||||
"""UPDATE acceptance_tasks SET scheduled_from_task_id=?,dashboard_batch_key=?
|
||||
"""UPDATE acceptance_tasks SET scheduled_from_task_id=?,dashboard_batch_key=?,
|
||||
use_all_devices=1,batch_date=?
|
||||
WHERE public_task_id=?""",
|
||||
(root_id, root["dashboard_batch_key"], child["task_id"]),
|
||||
(root_id, root["dashboard_batch_key"], datetime.now().astimezone().date().isoformat(), child["task_id"]),
|
||||
)
|
||||
return get_acceptance_task(child["task_id"])
|
||||
|
||||
@@ -1559,7 +1561,7 @@ def _task_dict(conn, task) -> dict[str, Any]:
|
||||
).fetchall()]
|
||||
root_id = task["scheduled_from_task_id"] or task["id"]
|
||||
batches = [dict(row) for row in conn.execute(
|
||||
"""SELECT public_task_id,status,progress_percent,created_at,updated_at
|
||||
"""SELECT public_task_id,status,progress_percent,batch_date,created_at,updated_at
|
||||
FROM acceptance_tasks WHERE id=? OR scheduled_from_task_id=?
|
||||
ORDER BY created_at DESC""",
|
||||
(root_id, root_id),
|
||||
@@ -1701,6 +1703,7 @@ def _task_dict(conn, task) -> dict[str, Any]:
|
||||
"cookie_count": task["cookie_count"],
|
||||
"dashboard_batch_key": task["dashboard_batch_key"],
|
||||
"use_all_devices": bool(task["use_all_devices"]),
|
||||
"batch_date": task["batch_date"],
|
||||
"schedule_enabled": bool(task["schedule_enabled"]),
|
||||
"schedule_weekday": task["schedule_weekday"],
|
||||
"schedule_weekdays": _decode_json(task["schedule_weekdays"], None) or (
|
||||
@@ -1733,7 +1736,9 @@ def get_acceptance_task(public_id: str) -> dict[str, Any]:
|
||||
|
||||
def list_acceptance_tasks() -> dict[str, Any]:
|
||||
with connection() as conn:
|
||||
rows = conn.execute("SELECT * FROM acceptance_tasks ORDER BY created_at DESC").fetchall()
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM acceptance_tasks WHERE scheduled_from_task_id IS NULL ORDER BY created_at DESC"
|
||||
).fetchall()
|
||||
items = [_task_dict(conn, row) for row in rows]
|
||||
return {"items": items, "total": len(items)}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user