feat: add project acceptance collection service
This commit is contained in:
235
database.py
235
database.py
@@ -133,8 +133,243 @@ def init_database() -> None:
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_auth_sessions_user
|
||||
ON auth_sessions(user_id, expires_at);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS acceptance_tasks (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
public_task_id TEXT NOT NULL UNIQUE,
|
||||
name TEXT NOT NULL,
|
||||
acceptance_method TEXT NOT NULL,
|
||||
status TEXT NOT NULL DEFAULT 'waiting',
|
||||
progress_percent INTEGER NOT NULL DEFAULT 0,
|
||||
device_count INTEGER NOT NULL DEFAULT 1,
|
||||
created_by_id TEXT NOT NULL,
|
||||
created_by_name TEXT NOT NULL,
|
||||
error_message TEXT,
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_acceptance_tasks_created
|
||||
ON acceptance_tasks(created_at DESC);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS acceptance_keywords (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
task_id INTEGER NOT NULL,
|
||||
keyword_order INTEGER NOT NULL,
|
||||
keyword TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL,
|
||||
UNIQUE(task_id, keyword_order),
|
||||
FOREIGN KEY(task_id) REFERENCES acceptance_tasks(id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS acceptance_feishu_sources (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
task_id INTEGER NOT NULL,
|
||||
source_order INTEGER NOT NULL,
|
||||
source_url TEXT NOT NULL,
|
||||
final_url TEXT,
|
||||
source_type TEXT,
|
||||
title TEXT,
|
||||
content_text TEXT,
|
||||
content_json TEXT,
|
||||
fetch_status TEXT NOT NULL DEFAULT 'waiting',
|
||||
http_status INTEGER,
|
||||
error_message TEXT,
|
||||
fetched_at TEXT,
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL,
|
||||
UNIQUE(task_id, source_order),
|
||||
UNIQUE(task_id, source_url),
|
||||
FOREIGN KEY(task_id) REFERENCES acceptance_tasks(id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS acceptance_feishu_records (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
source_id INTEGER NOT NULL,
|
||||
row_number INTEGER NOT NULL,
|
||||
row_json TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL,
|
||||
UNIQUE(source_id, row_number),
|
||||
FOREIGN KEY(source_id) REFERENCES acceptance_feishu_sources(id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS acceptance_xhs_notes (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
task_id INTEGER NOT NULL,
|
||||
source_id INTEGER NOT NULL,
|
||||
source_row_number INTEGER,
|
||||
source_column_name TEXT,
|
||||
note_url TEXT NOT NULL,
|
||||
normalized_url TEXT NOT NULL,
|
||||
note_id TEXT,
|
||||
parse_status TEXT NOT NULL DEFAULT 'waiting',
|
||||
parsed_json TEXT,
|
||||
error_message TEXT,
|
||||
parsed_at TEXT,
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL,
|
||||
UNIQUE(task_id, normalized_url),
|
||||
FOREIGN KEY(task_id) REFERENCES acceptance_tasks(id) ON DELETE CASCADE,
|
||||
FOREIGN KEY(source_id) REFERENCES acceptance_feishu_sources(id) ON DELETE CASCADE
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_acceptance_notes_task_status
|
||||
ON acceptance_xhs_notes(task_id, parse_status);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS feishu_notes (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
note_url TEXT NOT NULL,
|
||||
normalized_url TEXT NOT NULL UNIQUE,
|
||||
note_id TEXT UNIQUE,
|
||||
title TEXT,
|
||||
body TEXT,
|
||||
author_nickname TEXT,
|
||||
author_avatar TEXT,
|
||||
likes INTEGER,
|
||||
collects INTEGER,
|
||||
comments INTEGER,
|
||||
images_json TEXT,
|
||||
tags_json TEXT,
|
||||
raw_json TEXT NOT NULL,
|
||||
parsed_at TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_feishu_notes_note_id
|
||||
ON feishu_notes(note_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS acceptance_keyword_runs (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
task_id INTEGER NOT NULL,
|
||||
keyword TEXT NOT NULL,
|
||||
run_type TEXT NOT NULL,
|
||||
run_order INTEGER NOT NULL,
|
||||
status TEXT NOT NULL DEFAULT 'waiting',
|
||||
crawler_task_id TEXT,
|
||||
result_json TEXT,
|
||||
error_message TEXT,
|
||||
started_at TEXT,
|
||||
completed_at TEXT,
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL,
|
||||
UNIQUE(task_id, run_type, keyword),
|
||||
FOREIGN KEY(task_id) REFERENCES acceptance_tasks(id) ON DELETE CASCADE
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_acceptance_keyword_runs_task
|
||||
ON acceptance_keyword_runs(task_id, status);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS acceptance_cookie_attempts (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
task_id INTEGER NOT NULL,
|
||||
keyword_run_id INTEGER NOT NULL,
|
||||
keyword TEXT NOT NULL,
|
||||
credential_id TEXT NOT NULL,
|
||||
attempt_number INTEGER NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
crawler_task_id TEXT,
|
||||
result_json TEXT,
|
||||
error_message TEXT,
|
||||
started_at TEXT NOT NULL,
|
||||
completed_at TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL,
|
||||
UNIQUE(keyword_run_id, credential_id, attempt_number),
|
||||
FOREIGN KEY(task_id) REFERENCES acceptance_tasks(id) ON DELETE CASCADE,
|
||||
FOREIGN KEY(keyword_run_id) REFERENCES acceptance_keyword_runs(id) ON DELETE CASCADE
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_cookie_attempts_run_status
|
||||
ON acceptance_cookie_attempts(keyword_run_id, status);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS acceptance_task_resources (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
task_id INTEGER NOT NULL,
|
||||
resource_type TEXT NOT NULL,
|
||||
resource_id TEXT NOT NULL,
|
||||
resource_order INTEGER NOT NULL,
|
||||
created_at TEXT NOT NULL,
|
||||
UNIQUE(task_id, resource_type, resource_id),
|
||||
FOREIGN KEY(task_id) REFERENCES acceptance_tasks(id) ON DELETE CASCADE
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_acceptance_task_resources
|
||||
ON acceptance_task_resources(task_id, resource_order);
|
||||
"""
|
||||
)
|
||||
# 将旧版本保存在 acceptance_xhs_notes.parsed_json 中的成功结果迁移到
|
||||
# 全局去重的笔记业务表；任务关联仍由 acceptance_xhs_notes 保留。
|
||||
legacy_rows = conn.execute(
|
||||
"""SELECT n.* FROM acceptance_xhs_notes n
|
||||
LEFT JOIN feishu_notes f ON f.normalized_url=n.normalized_url
|
||||
WHERE n.parse_status='completed' AND n.parsed_json IS NOT NULL AND f.id IS NULL"""
|
||||
).fetchall()
|
||||
for row in legacy_rows:
|
||||
try:
|
||||
payload = json.loads(row["parsed_json"])
|
||||
except (TypeError, json.JSONDecodeError):
|
||||
continue
|
||||
upsert_feishu_note(conn, row, payload)
|
||||
# 为升级前已有的 Cookie 尝试补建任务资源记录，详情页可立即展示设备进度。
|
||||
legacy_resources = conn.execute(
|
||||
"""SELECT task_id,credential_id,MIN(id) AS first_id
|
||||
FROM acceptance_cookie_attempts
|
||||
GROUP BY task_id,credential_id ORDER BY task_id,first_id"""
|
||||
).fetchall()
|
||||
resource_orders: dict[int, int] = {}
|
||||
for row in legacy_resources:
|
||||
order = resource_orders.get(row["task_id"], 0)
|
||||
conn.execute(
|
||||
"""INSERT OR IGNORE INTO acceptance_task_resources
|
||||
(task_id,resource_type,resource_id,resource_order,created_at)
|
||||
VALUES (?,'cookie',?,?,?)""",
|
||||
(row["task_id"], row["credential_id"], order, now_iso()),
|
||||
)
|
||||
resource_orders[row["task_id"]] = order + 1
|
||||
|
||||
|
||||
def _integer_or_none(value: Any) -> int | None:
|
||||
if value is None or value == "":
|
||||
return None
|
||||
try:
|
||||
return int(str(value).replace(",", ""))
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def upsert_feishu_note(conn: sqlite3.Connection, note_row: sqlite3.Row, payload: dict[str, Any]) -> None:
|
||||
"""按规范化链接更新或新增笔记；任务关联保存在 acceptance_xhs_notes。"""
|
||||
|
||||
now = now_iso()
|
||||
note_id = payload.get("noteId") or payload.get("note_id") or note_row["note_id"]
|
||||
existing = conn.execute(
|
||||
"""SELECT id FROM feishu_notes
|
||||
WHERE normalized_url=? OR (? IS NOT NULL AND note_id=?)
|
||||
ORDER BY CASE WHEN note_id=? THEN 0 ELSE 1 END LIMIT 1""",
|
||||
(note_row["normalized_url"], note_id, note_id, note_id),
|
||||
).fetchone()
|
||||
values = (
|
||||
note_row["note_url"], note_row["normalized_url"], note_id,
|
||||
payload.get("title"), payload.get("body") or payload.get("content"),
|
||||
payload.get("nickname") or payload.get("author_nickname"),
|
||||
payload.get("avatar") or payload.get("author_avatar"),
|
||||
_integer_or_none(payload.get("likes")), _integer_or_none(payload.get("collects")),
|
||||
_integer_or_none(payload.get("comments")),
|
||||
json.dumps(payload.get("images") or payload.get("images_list") or [], ensure_ascii=False),
|
||||
json.dumps(payload.get("tags") or [], ensure_ascii=False),
|
||||
json.dumps(payload, ensure_ascii=False), now, now,
|
||||
)
|
||||
if existing:
|
||||
conn.execute(
|
||||
"""UPDATE feishu_notes SET
|
||||
note_url=?,normalized_url=?,note_id=?,title=?,body=?,author_nickname=?,author_avatar=?,
|
||||
likes=?,collects=?,comments=?,images_json=?,tags_json=?,raw_json=?,parsed_at=?,updated_at=?
|
||||
WHERE id=?""",
|
||||
(*values, existing["id"]),
|
||||
)
|
||||
else:
|
||||
conn.execute(
|
||||
"""INSERT INTO feishu_notes
|
||||
(note_url,normalized_url,note_id,title,body,author_nickname,author_avatar,
|
||||
likes,collects,comments,images_json,tags_json,raw_json,parsed_at,created_at,updated_at)
|
||||
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
|
||||
""",
|
||||
(*values, now),
|
||||
)
|
||||
|
||||
|
||||
def json_dumps(value: Any) -> str:
|
||||
|
||||
Reference in New Issue
Block a user