import json import sqlite3 from contextlib import contextmanager from datetime import datetime from typing import Any, Iterator from config import Config def now_iso() -> str: return datetime.now().astimezone().isoformat(timespec="seconds") @contextmanager def connection() -> Iterator[sqlite3.Connection]: conn = sqlite3.connect(Config.DATABASE_PATH, timeout=30) conn.row_factory = sqlite3.Row conn.execute("PRAGMA foreign_keys = ON") try: yield conn conn.commit() finally: conn.close() def init_database() -> None: """仅在服务启动时创建表,接口调用过程中不会重复执行建表。""" Config.DATABASE_PATH.parent.mkdir(parents=True, exist_ok=True) Config.FILE_STORAGE_PATH.mkdir(parents=True, exist_ok=True) with connection() as conn: conn.executescript( """ CREATE TABLE IF NOT EXISTS research_tasks ( id INTEGER PRIMARY KEY AUTOINCREMENT, public_task_id TEXT NOT NULL UNIQUE, name TEXT NOT NULL, platform TEXT NOT NULL, mode TEXT NOT NULL, status TEXT NOT NULL, execution_stage TEXT NOT NULL DEFAULT 'queued', progress_percent INTEGER NOT NULL DEFAULT 0 CHECK(progress_percent BETWEEN 0 AND 100), created_by_id TEXT NOT NULL, created_by_name TEXT NOT NULL, retry_of_task_id INTEGER, crawler_task_id TEXT, error_code TEXT, error_message TEXT, result_json TEXT, cancel_requested_at TEXT, started_at TEXT, completed_at TEXT, created_at TEXT NOT NULL, updated_at TEXT NOT NULL, version INTEGER NOT NULL DEFAULT 0, FOREIGN KEY(retry_of_task_id) REFERENCES research_tasks(id) ); CREATE INDEX IF NOT EXISTS idx_tasks_status_created ON research_tasks(status, created_at DESC); CREATE INDEX IF NOT EXISTS idx_tasks_creator_created ON research_tasks(created_by_id, created_at DESC); CREATE TABLE IF NOT EXISTS task_keywords ( id INTEGER PRIMARY KEY AUTOINCREMENT, task_id INTEGER NOT NULL, keyword_order INTEGER NOT NULL, keyword TEXT NOT NULL, normalized_keyword TEXT NOT NULL, created_at TEXT NOT NULL, UNIQUE(task_id, keyword_order), UNIQUE(task_id, normalized_keyword), FOREIGN KEY(task_id) REFERENCES research_tasks(id) ON DELETE CASCADE ); CREATE TABLE IF NOT EXISTS task_events ( id INTEGER PRIMARY KEY AUTOINCREMENT, task_id INTEGER NOT NULL, event_type TEXT NOT NULL, stage TEXT, progress_percent INTEGER, actor_type TEXT NOT NULL, actor_id TEXT, payload_json TEXT, occurred_at TEXT NOT NULL, FOREIGN KEY(task_id) REFERENCES research_tasks(id) ON DELETE CASCADE ); CREATE TABLE IF NOT EXISTS task_keyword_runs ( id INTEGER PRIMARY KEY AUTOINCREMENT, task_id INTEGER NOT NULL, keyword_order INTEGER NOT NULL, keyword TEXT NOT NULL, status TEXT NOT NULL DEFAULT 'queued', progress_percent INTEGER NOT NULL DEFAULT 0, crawler_task_id TEXT, result_json TEXT, error_message TEXT, started_at TEXT, completed_at TEXT, updated_at TEXT NOT NULL, UNIQUE(task_id, keyword_order), FOREIGN KEY(task_id) REFERENCES research_tasks(id) ON DELETE CASCADE ); CREATE INDEX IF NOT EXISTS idx_keyword_runs_task_status ON task_keyword_runs(task_id, status, keyword_order); CREATE TABLE IF NOT EXISTS task_files ( id INTEGER PRIMARY KEY AUTOINCREMENT, public_file_id TEXT NOT NULL UNIQUE, task_id INTEGER NOT NULL, name TEXT NOT NULL, stage TEXT NOT NULL, category TEXT NOT NULL, format TEXT NOT NULL, record_count INTEGER NOT NULL DEFAULT 0, size_bytes INTEGER NOT NULL DEFAULT 0, storage_key TEXT NOT NULL, sha256 TEXT NOT NULL, schema_version TEXT NOT NULL DEFAULT '1.0', generated_at TEXT NOT NULL, available INTEGER NOT NULL DEFAULT 1, UNIQUE(task_id, name), FOREIGN KEY(task_id) REFERENCES research_tasks(id) ON DELETE CASCADE ); CREATE TABLE IF NOT EXISTS auth_users ( id INTEGER PRIMARY KEY AUTOINCREMENT, ding_user_id TEXT NOT NULL UNIQUE, union_id TEXT, open_id TEXT, name TEXT NOT NULL, avatar TEXT, mobile TEXT, email TEXT, raw_json TEXT, last_login_at TEXT NOT NULL, created_at TEXT NOT NULL, updated_at TEXT NOT NULL ); CREATE INDEX IF NOT EXISTS idx_auth_users_name ON auth_users(name); CREATE TABLE IF NOT EXISTS auth_sessions ( id INTEGER PRIMARY KEY AUTOINCREMENT, session_hash TEXT NOT NULL UNIQUE, user_id INTEGER NOT NULL, expires_at TEXT NOT NULL, created_at TEXT NOT NULL, last_seen_at TEXT NOT NULL, revoked_at TEXT, FOREIGN KEY(user_id) REFERENCES auth_users(id) ON DELETE CASCADE ); CREATE INDEX IF NOT EXISTS idx_auth_sessions_user ON auth_sessions(user_id, expires_at); CREATE TABLE IF NOT EXISTS ai_access_tokens ( id INTEGER PRIMARY KEY AUTOINCREMENT, token_hash TEXT NOT NULL UNIQUE, user_id INTEGER NOT NULL, expires_at TEXT NOT NULL, created_at TEXT NOT NULL, last_seen_at TEXT NOT NULL, revoked_at TEXT, FOREIGN KEY(user_id) REFERENCES auth_users(id) ON DELETE CASCADE ); CREATE INDEX IF NOT EXISTS idx_ai_access_tokens_user ON ai_access_tokens(user_id, expires_at); CREATE TABLE IF NOT EXISTS acceptance_tasks ( id INTEGER PRIMARY KEY AUTOINCREMENT, public_task_id TEXT NOT NULL UNIQUE, name TEXT NOT NULL, acceptance_method TEXT NOT NULL, status TEXT NOT NULL DEFAULT 'waiting', progress_percent INTEGER NOT NULL DEFAULT 0, device_count INTEGER NOT NULL DEFAULT 1, created_by_id TEXT NOT NULL, created_by_name TEXT NOT NULL, error_message TEXT, created_at TEXT NOT NULL, updated_at TEXT NOT NULL ); CREATE INDEX IF NOT EXISTS idx_acceptance_tasks_created ON acceptance_tasks(created_at DESC); CREATE TABLE IF NOT EXISTS acceptance_keywords ( id INTEGER PRIMARY KEY AUTOINCREMENT, task_id INTEGER NOT NULL, keyword_order INTEGER NOT NULL, keyword TEXT NOT NULL, created_at TEXT NOT NULL, UNIQUE(task_id, keyword_order), FOREIGN KEY(task_id) REFERENCES acceptance_tasks(id) ON DELETE CASCADE ); CREATE TABLE IF NOT EXISTS acceptance_feishu_sources ( id INTEGER PRIMARY KEY AUTOINCREMENT, task_id INTEGER NOT NULL, source_order INTEGER NOT NULL, source_url TEXT NOT NULL, final_url TEXT, source_type TEXT, title TEXT, content_text TEXT, content_json TEXT, fetch_status TEXT NOT NULL DEFAULT 'waiting', http_status INTEGER, error_message TEXT, fetched_at TEXT, created_at TEXT NOT NULL, updated_at TEXT NOT NULL, UNIQUE(task_id, source_order), UNIQUE(task_id, source_url), FOREIGN KEY(task_id) REFERENCES acceptance_tasks(id) ON DELETE CASCADE ); CREATE TABLE IF NOT EXISTS acceptance_feishu_records ( id INTEGER PRIMARY KEY AUTOINCREMENT, source_id INTEGER NOT NULL, row_number INTEGER NOT NULL, row_json TEXT NOT NULL, created_at TEXT NOT NULL, UNIQUE(source_id, row_number), FOREIGN KEY(source_id) REFERENCES acceptance_feishu_sources(id) ON DELETE CASCADE ); CREATE TABLE IF NOT EXISTS acceptance_xhs_notes ( id INTEGER PRIMARY KEY AUTOINCREMENT, task_id INTEGER NOT NULL, source_id INTEGER NOT NULL, source_row_number INTEGER, source_column_name TEXT, note_url TEXT NOT NULL, normalized_url TEXT NOT NULL, note_id TEXT, parse_status TEXT NOT NULL DEFAULT 'waiting', parsed_json TEXT, error_message TEXT, parsed_at TEXT, created_at TEXT NOT NULL, updated_at TEXT NOT NULL, UNIQUE(task_id, normalized_url), FOREIGN KEY(task_id) REFERENCES acceptance_tasks(id) ON DELETE CASCADE, FOREIGN KEY(source_id) REFERENCES acceptance_feishu_sources(id) ON DELETE CASCADE ); CREATE INDEX IF NOT EXISTS idx_acceptance_notes_task_status ON acceptance_xhs_notes(task_id, parse_status); CREATE TABLE IF NOT EXISTS feishu_notes ( id INTEGER PRIMARY KEY AUTOINCREMENT, note_url TEXT NOT NULL, normalized_url TEXT NOT NULL UNIQUE, note_id TEXT UNIQUE, title TEXT, body TEXT, author_nickname TEXT, author_avatar TEXT, likes INTEGER, collects INTEGER, comments INTEGER, images_json TEXT, tags_json TEXT, raw_json TEXT NOT NULL, parsed_at TEXT NOT NULL, created_at TEXT NOT NULL, updated_at TEXT NOT NULL ); CREATE INDEX IF NOT EXISTS idx_feishu_notes_note_id ON feishu_notes(note_id); CREATE TABLE IF NOT EXISTS acceptance_keyword_runs ( id INTEGER PRIMARY KEY AUTOINCREMENT, task_id INTEGER NOT NULL, keyword TEXT NOT NULL, run_type TEXT NOT NULL, run_order INTEGER NOT NULL, status TEXT NOT NULL DEFAULT 'waiting', crawler_task_id TEXT, result_json TEXT, error_message TEXT, started_at TEXT, completed_at TEXT, created_at TEXT NOT NULL, updated_at TEXT NOT NULL, UNIQUE(task_id, run_type, keyword), FOREIGN KEY(task_id) REFERENCES acceptance_tasks(id) ON DELETE CASCADE ); CREATE INDEX IF NOT EXISTS idx_acceptance_keyword_runs_task ON acceptance_keyword_runs(task_id, status); CREATE TABLE IF NOT EXISTS acceptance_cookie_attempts ( id INTEGER PRIMARY KEY AUTOINCREMENT, task_id INTEGER NOT NULL, keyword_run_id INTEGER NOT NULL, keyword TEXT NOT NULL, credential_id TEXT NOT NULL, attempt_number INTEGER NOT NULL, status TEXT NOT NULL, crawler_task_id TEXT, result_json TEXT, error_message TEXT, started_at TEXT NOT NULL, completed_at TEXT NOT NULL, created_at TEXT NOT NULL, UNIQUE(keyword_run_id, credential_id, attempt_number), FOREIGN KEY(task_id) REFERENCES acceptance_tasks(id) ON DELETE CASCADE, FOREIGN KEY(keyword_run_id) REFERENCES acceptance_keyword_runs(id) ON DELETE CASCADE ); CREATE INDEX IF NOT EXISTS idx_cookie_attempts_run_status ON acceptance_cookie_attempts(keyword_run_id, status); CREATE TABLE IF NOT EXISTS acceptance_task_resources ( id INTEGER PRIMARY KEY AUTOINCREMENT, task_id INTEGER NOT NULL, resource_type TEXT NOT NULL, resource_id TEXT NOT NULL, resource_order INTEGER NOT NULL, created_at TEXT NOT NULL, UNIQUE(task_id, resource_type, resource_id), FOREIGN KEY(task_id) REFERENCES acceptance_tasks(id) ON DELETE CASCADE ); CREATE INDEX IF NOT EXISTS idx_acceptance_task_resources ON acceptance_task_resources(task_id, resource_order); """ ) # 将旧版本保存在 acceptance_xhs_notes.parsed_json 中的成功结果迁移到 # 全局去重的笔记业务表;任务关联仍由 acceptance_xhs_notes 保留。 legacy_rows = conn.execute( """SELECT n.* FROM acceptance_xhs_notes n LEFT JOIN feishu_notes f ON f.normalized_url=n.normalized_url WHERE n.parse_status='completed' AND n.parsed_json IS NOT NULL AND f.id IS NULL""" ).fetchall() for row in legacy_rows: try: payload = json.loads(row["parsed_json"]) except (TypeError, json.JSONDecodeError): continue upsert_feishu_note(conn, row, payload) # 为升级前已有的 Cookie 尝试补建任务资源记录,详情页可立即展示设备进度。 legacy_resources = conn.execute( """SELECT task_id,credential_id,MIN(id) AS first_id FROM acceptance_cookie_attempts GROUP BY task_id,credential_id ORDER BY task_id,first_id""" ).fetchall() resource_orders: dict[int, int] = {} for row in legacy_resources: order = resource_orders.get(row["task_id"], 0) conn.execute( """INSERT OR IGNORE INTO acceptance_task_resources (task_id,resource_type,resource_id,resource_order,created_at) VALUES (?,'cookie',?,?,?)""", (row["task_id"], row["credential_id"], order, now_iso()), ) resource_orders[row["task_id"]] = order + 1 def _integer_or_none(value: Any) -> int | None: if value is None or value == "": return None try: return int(str(value).replace(",", "")) except (TypeError, ValueError): return None def upsert_feishu_note(conn: sqlite3.Connection, note_row: sqlite3.Row, payload: dict[str, Any]) -> None: """按规范化链接更新或新增笔记;任务关联保存在 acceptance_xhs_notes。""" now = now_iso() note_id = payload.get("noteId") or payload.get("note_id") or note_row["note_id"] existing = conn.execute( """SELECT id FROM feishu_notes WHERE normalized_url=? OR (? IS NOT NULL AND note_id=?) ORDER BY CASE WHEN note_id=? THEN 0 ELSE 1 END LIMIT 1""", (note_row["normalized_url"], note_id, note_id, note_id), ).fetchone() values = ( note_row["note_url"], note_row["normalized_url"], note_id, payload.get("title"), payload.get("body") or payload.get("content"), payload.get("nickname") or payload.get("author_nickname"), payload.get("avatar") or payload.get("author_avatar"), _integer_or_none(payload.get("likes")), _integer_or_none(payload.get("collects")), _integer_or_none(payload.get("comments")), json.dumps(payload.get("images") or payload.get("images_list") or [], ensure_ascii=False), json.dumps(payload.get("tags") or [], ensure_ascii=False), json.dumps(payload, ensure_ascii=False), now, now, ) if existing: conn.execute( """UPDATE feishu_notes SET note_url=?,normalized_url=?,note_id=?,title=?,body=?,author_nickname=?,author_avatar=?, likes=?,collects=?,comments=?,images_json=?,tags_json=?,raw_json=?,parsed_at=?,updated_at=? WHERE id=?""", (*values, existing["id"]), ) else: conn.execute( """INSERT INTO feishu_notes (note_url,normalized_url,note_id,title,body,author_nickname,author_avatar, likes,collects,comments,images_json,tags_json,raw_json,parsed_at,created_at,updated_at) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) """, (*values, now), ) def json_dumps(value: Any) -> str: return json.dumps(value, ensure_ascii=False, separators=(",", ":")) def json_loads(value: str | None) -> Any: if not value: return None return json.loads(value)