fix: preserve production task and auth integrations

app.py

@@ -6,7 +6,9 @@ from flask import Flask, g, jsonify, redirect, request, send_file
 
 from auth_service import (
     build_dingtalk_login_url,
+    create_agent_access_token,
     exchange_dingtalk_user,
+    get_user_by_agent_token,
     get_user_by_session_token,
     revoke_session,
     upsert_user_and_session,
@@ -32,10 +34,12 @@ from task_service import (
     TaskError,
     cancel_task,
     create_task,
+    delete_task,
     get_file,
     get_task,
     get_task_extracted,
     list_tasks,
+    merge_tasks,
     recollect_task,
     resume_task,
     resume_unfinished_tasks,
@@ -50,12 +54,8 @@ def create_app() -> Flask:
         if not Config.LOCAL_AUTH_BYPASS or request.remote_addr not in {"127.0.0.1", "::1"}:
             return None
         return {
-            "id": "local-dev",
+            "id": "local-dev", "name": "本地开发用户", "account": "local-dev",
-            "name": "本地开发用户",
+            "avatar": "", "union_id": "", "open_id": "",
-            "account": "local-dev",
-            "avatar": "",
-            "union_id": "",
-            "open_id": "",
         }
 
     @app.before_request
@@ -67,9 +67,16 @@ def create_app() -> Flask:
         if request.path == "/health" or any(request.path.startswith(path) for path in open_paths):
             return None
         user = local_dev_user() or get_user_by_session_token(request.cookies.get(Config.SESSION_COOKIE_NAME))
+        if user is None:
+            authorization = request.headers.get("Authorization", "")
+            if authorization.lower().startswith("bearer "):
+                user = get_user_by_agent_token(authorization[7:].strip())
         if user is None and request.path.startswith("/api/"):
             return jsonify({
-                "error": {"code": "UNAUTHORIZED", "message": "请先通过钉钉授权登录"},
+                "error": {
+                    "code": "UNAUTHORIZED",
+                    "message": "登录或 AI 访问 Token 已失效，请重新登录后复制 Link AI",
+                },
                 "request_id": g.request_id,
             }), 401
         g.current_user = user
@@ -104,10 +111,14 @@ def create_app() -> Flask:
 
     @app.route("/api/v1/auth/dingtalk/login", methods=["GET"])
     def dingtalk_login():
-        return jsonify({
+        response = jsonify({
             "data": build_dingtalk_login_url(request.args.get("redirect", "/tasks")),
             "request_id": g.request_id,
         })
+        # OAuth URL 带一次性 state，且 scope/corpId 配置可能更新，禁止缓存。
+        response.headers["Cache-Control"] = "no-store, no-cache, must-revalidate"
+        response.headers["Pragma"] = "no-cache"
+        return response
 
     @app.route("/api/v1/auth/dingtalk/callback", methods=["GET"])
     def dingtalk_callback():
@@ -118,8 +129,13 @@ def create_app() -> Flask:
             frontend_redirect = state.split(":", 1)[1] or frontend_redirect
         if not code:
             return redirect(f"{Config.FRONTEND_BASE_URL}/login?error={quote('钉钉授权缺少 code')}")
+        try:
             user_payload = exchange_dingtalk_user(code)
             user, token, expires_at = upsert_user_and_session(user_payload)
+        except TaskError as exc:
+            return redirect(
+                f"{Config.FRONTEND_BASE_URL}/login?error={quote(exc.message)}"
+            )
         response = redirect(f"{Config.FRONTEND_BASE_URL}{frontend_redirect}")
         response.set_cookie(
             Config.SESSION_COOKIE_NAME,
@@ -130,17 +146,6 @@ def create_app() -> Flask:
             samesite=Config.SESSION_COOKIE_SAMESITE,
             path="/",
         )
-        # 给前端“Link AI”复制提示词使用。后端鉴权仍读取 hc_session；
-        # 前端只读取这个辅助 Cookie 的 value，并在提示词中拼成 hc_session=<value>。
-        response.set_cookie(
-            "hc_session_agent",
-            token,
-            expires=expires_at,
-            httponly=False,
-            secure=Config.SESSION_COOKIE_SECURE,
-            samesite=Config.SESSION_COOKIE_SAMESITE,
-            path="/",
-        )
         app.logger.info("钉钉用户登录成功 user_id=%s name=%s", user["id"], user["name"])
         return response
 
@@ -216,11 +221,34 @@ def create_app() -> Flask:
             raise TaskError("UNAUTHORIZED", "请先通过钉钉授权登录", 401)
         return jsonify({"data": {"user": user}, "request_id": g.request_id})
 
+    @app.route("/api/v1/auth/agent-token", methods=["POST"])
+    def auth_agent_token():
+        issued = create_agent_access_token(
+            request.cookies.get(Config.SESSION_COOKIE_NAME)
+        )
+        if not issued:
+            raise TaskError(
+                "UNAUTHORIZED",
+                "登录已失效，请重新完成钉钉登录后再复制 Link AI",
+                401,
+            )
+        token, expires_at = issued
+        response = jsonify({
+            "data": {
+                "token": token,
+                "expires_at": expires_at.isoformat(timespec="seconds"),
+            },
+            "request_id": g.request_id,
+        })
+        response.headers["Cache-Control"] = "no-store"
+        return response
+
     @app.route("/api/v1/auth/logout", methods=["POST"])
     def auth_logout():
         revoke_session(request.cookies.get(Config.SESSION_COOKIE_NAME))
         response = jsonify({"data": {"ok": True}, "request_id": g.request_id})
         response.delete_cookie(Config.SESSION_COOKIE_NAME, path="/")
+        # 清理旧版本曾设置的可读辅助 Cookie。
         response.delete_cookie("hc_session_agent", path="/")
         return response
 
@@ -272,12 +300,27 @@ def create_app() -> Flask:
         )
         return jsonify({"data": result, "request_id": g.request_id})
 
+    @app.route("/api/v1/tasks/merge", methods=["POST"])
+    def merge_research_tasks():
+        payload = request.get_json(silent=True) or {}
+        task_ids = payload.get("task_ids")
+        if not isinstance(task_ids, list):
+            raise TaskError("MERGE_TASKS_REQUIRED", "task_ids 必须是任务 ID 数组", 422)
+        return jsonify({
+            "data": merge_tasks(task_ids),
+            "request_id": g.request_id,
+        })
+
     @app.route("/api/v1/tasks/<task_id>", methods=["GET"])
     def get_research_task(task_id: str):
         if request.args.get("view") in {"extracted", "cleaned", "parsed"}:
             return jsonify({"data": get_task_extracted(task_id), "request_id": g.request_id})
         return jsonify({"data": get_task(task_id), "request_id": g.request_id})
 
+    @app.route("/api/v1/tasks/<task_id>", methods=["DELETE"])
+    def delete_research_task(task_id: str):
+        return jsonify({"data": delete_task(task_id), "request_id": g.request_id})
+
     @app.route("/api/v1/tasks/<task_id>/extracted", methods=["GET"])
     def get_research_task_extracted(task_id: str):
         return jsonify({"data": get_task_extracted(task_id), "request_id": g.request_id})
@@ -297,8 +340,13 @@ def create_app() -> Flask:
 
     @app.route("/api/v1/tasks/<task_id>/recollect", methods=["POST"])
     def recollect_research_task(task_id: str):
+        current_user = getattr(g, "current_user", None) or {}
         return jsonify({
-            "data": recollect_task(task_id),
+            "data": recollect_task(
+                task_id,
+                created_by_id=current_user.get("id"),
+                created_by_name=current_user.get("name"),
+            ),
             "request_id": g.request_id,
         }), 201
 

config.py

@@ -35,10 +35,19 @@ class Config:
     LOCAL_AUTH_BYPASS = os.getenv("LOCAL_AUTH_BYPASS", "false").lower() in ("1", "true", "yes", "on")
     DINGTALK_CLIENT_ID = os.getenv("DINGTALK_CLIENT_ID", os.getenv("DINGTALK_APP_KEY", ""))
     DINGTALK_CLIENT_SECRET = os.getenv("DINGTALK_CLIENT_SECRET", os.getenv("DINGTALK_APP_SECRET", ""))
+    DINGTALK_ALLOWED_CORP_ID = os.getenv(
+        "DINGTALK_ALLOWED_CORP_ID",
+        "dingbc8e63f8aa03221d35c2f4657eb6378f",
+    )
     DINGTALK_REDIRECT_URI = os.getenv(
         "DINGTALK_REDIRECT_URI",
         f"{FRONTEND_BASE_URL}/api/v1/auth/dingtalk/callback",
     )
+    CRAWLER_DATABASE_HOST = os.getenv("CRAWLER_DATABASE_HOST", "")
+    CRAWLER_DATABASE_PORT = int(os.getenv("CRAWLER_DATABASE_PORT", "3306"))
+    CRAWLER_DATABASE_USER = os.getenv("CRAWLER_DATABASE_USER", "")
+    CRAWLER_DATABASE_PASSWORD = os.getenv("CRAWLER_DATABASE_PASSWORD", "")
+    CRAWLER_DATABASE_NAME = os.getenv("CRAWLER_DATABASE_NAME", "")
     DINGTALK_AUTH_URL = os.getenv("DINGTALK_AUTH_URL", "https://login.dingtalk.com/oauth2/auth")
     DINGTALK_TOKEN_URL = os.getenv("DINGTALK_TOKEN_URL", "https://api.dingtalk.com/v1.0/oauth2/userAccessToken")
     DINGTALK_USER_URL = os.getenv("DINGTALK_USER_URL", "https://api.dingtalk.com/v1.0/contact/users/me")

crawler_client.py

@@ -1,8 +1,10 @@
 import re
+import secrets
 import time
 from typing import Any
 
 import requests
+import json
 
 from config import Config
 
@@ -11,6 +13,10 @@ class CrawlerError(RuntimeError):
     pass
 
 
+class NoAiCardError(CrawlerError):
+    """搜索请求成功，但当前结果中没有问一问 AI 卡片。"""
+
+
 CRAWLER_SESSION = requests.Session()
 # 爬虫服务为明确配置的直连地址，不应继承 macOS 系统代理。
 # 否则 requests 会经由 127.0.0.1:7890，约 30 秒后被代理返回 502。
@@ -30,6 +36,58 @@ def normalize_collection_keywords(keywords: list[str]) -> list[str]:
     return normalized
 
 
+def compact_search_keyword(keyword: str) -> str:
+    """生成问一问卡片匹配的无空白回退词。
+
+    小红书会把 ``ai 生成 ppt 哪个好用`` 和 ``ai生成ppt哪个好用`` 当成
+    不同搜索词，前者可能不返回 AI 卡片。仅在原词明确无卡片时使用该
+    回退，不改变任务对外展示的原始关键词。
+    """
+
+    return re.sub(r"\s+", "", keyword)
+
+
+def fallback_search_keywords(keyword: str) -> list[str]:
+    """Generate conservative query variants when the original query misses the AI card."""
+
+    variants: list[str] = []
+
+    def add(value: str) -> None:
+        value = re.sub(r"\s+", " ", value).strip()
+        if value and value != keyword and value not in variants:
+            variants.append(value)
+
+    add(compact_search_keyword(keyword))
+
+    # “aippt自动生成哪个好”在搜索端可能无法正确分词，而等价的自然分词
+    # “ai 生成 ppt 哪个好用”可以命中相同意图下的问一问 AI 卡片。
+    spaced = re.sub(
+        r"^ai\s*ppt\s*自动生成",
+        "ai 生成 ppt ",
+        keyword,
+        flags=re.IGNORECASE,
+    )
+    spaced = re.sub(r"哪个好$", "哪个好用", spaced)
+    add(spaced)
+    return variants
+
+
+def create_search_context_id() -> str:
+    """生成小红书搜索链路使用的 36 进制上下文 ID。
+
+    与 Web 端 createSearchId 的算法保持一致：
+    ``(Date.now() << 64) + random(1..2147483646)``。
+    """
+
+    value = (int(time.time() * 1000) << 64) + secrets.randbelow(2147483646) + 1
+    alphabet = "0123456789abcdefghijklmnopqrstuvwxyz"
+    encoded = ""
+    while value:
+        value, remainder = divmod(value, 36)
+        encoded = alphabet[remainder] + encoded
+    return encoded or "0"
+
+
 def _post(
     path: str,
     body: dict[str, Any],
@@ -71,13 +129,160 @@ def _post(
         raise CrawlerError(f"爬虫服务请求失败（已重试 {retry_times} 次）")
 
     if payload.get("code") != 200 or not payload.get("success", False):
-        raise CrawlerError(payload.get("msg") or "爬虫服务返回失败")
+        message = str(payload.get("msg") or "爬虫服务返回失败")
+        no_ai_card_signals = (
+            "没有返回问一问 AI 卡片",
+            "NoneType' object has no attribute 'get'",
+            "complex/detail 未返回 AI 数据",
+        )
+        if any(signal in message for signal in no_ai_card_signals):
+            raise NoAiCardError(message)
+        raise CrawlerError(message)
     data = payload.get("data")
     if not isinstance(data, dict):
         raise CrawlerError("爬虫服务响应缺少 data")
     return data
 
 
+def start_collection(mode: str, keywords: list[str]) -> dict[str, Any]:
+    """深度任务走 sources，常规任务走 overview。"""
+
+    collection_keywords = normalize_collection_keywords(keywords)
+    path = "/api/v1/xhs/wen/sources" if mode == "deep" else "/api/v1/xhs/wen/overview"
+    # 常规任务允许批量关键词；数组由远程 overview 创建异步批量任务。
+    keyword: str | list[str] = collection_keywords if len(collection_keywords) > 1 else collection_keywords[0]
+    # 轻度 overview 在首次计算超过网关时限后，远程通常已经写好缓存，
+    # 允许受控重试以取得结果；深度 sources 会启动长任务，禁止盲目重试，
+    # 避免远程请求虽断开但后台继续执行时产生重复采集任务。
+    # 采集服务的 search/notes 需要搜索上下文，但业务调用方只应传 keyword。
+    # 在 Holy蟹内部为每个新任务生成一次，并由同一个远程采集任务全程复用。
+    def submit(value: str | list[str]) -> dict[str, Any]:
+        return _post(
+            path,
+            {
+                "keyword": value,
+                "search_id": create_search_context_id(),
+                "session_id": create_search_context_id(),
+            },
+            retry_transient=(mode != "deep"),
+        )
+
+    try:
+        return submit(keyword)
+    except NoAiCardError:
+        # 搜索端分词差异可能让真实存在的问一问卡片暂时无法命中。
+        # 依次尝试保守的等价写法；全部失败后才交给上层按空结果完成。
+        if isinstance(keyword, str):
+            for fallback_keyword in fallback_search_keywords(keyword):
+                try:
+                    result = submit(fallback_keyword)
+                except NoAiCardError:
+                    continue
+                result["requested_keyword"] = keyword
+                result["collection_keyword"] = fallback_keyword
+                return result
+        raise
+
+
+def query_collection(crawler_task_id: str) -> dict[str, Any]:
+    return _post("/api/v1/xhs/wen/tasks/query", {"task_id": crawler_task_id})
+
+
+def recover_source_note_details(keyword: str) -> dict[str, Any]:
+    """从采集器已落库的 complex/source 记录恢复可分析的来源 notes。"""
+
+    if not all((
+        Config.CRAWLER_DATABASE_HOST,
+        Config.CRAWLER_DATABASE_USER,
+        Config.CRAWLER_DATABASE_NAME,
+    )):
+        return {}
+    import pymysql
+
+    db = pymysql.connect(
+        host=Config.CRAWLER_DATABASE_HOST,
+        port=Config.CRAWLER_DATABASE_PORT,
+        user=Config.CRAWLER_DATABASE_USER,
+        password=Config.CRAWLER_DATABASE_PASSWORD,
+        database=Config.CRAWLER_DATABASE_NAME,
+        charset="utf8mb4",
+        cursorclass=pymysql.cursors.DictCursor,
+    )
+    try:
+        with db.cursor() as cursor:
+            cursor.execute(
+                "SELECT searchid,card_id FROM xhs_wen_ai_answers "
+                "WHERE keyword=%s ORDER BY updated_at DESC,id DESC LIMIT 1",
+                (keyword,),
+            )
+            linkage = cursor.fetchone()
+            if not linkage:
+                return {}
+            cursor.execute(
+                "SELECT n.raw_json,n.tag_name,t.title AS product_name "
+                "FROM xhs_wen_tag_notes n LEFT JOIN xhs_wen_ai_tags t "
+                "ON t.keyword=n.keyword AND t.searchid=n.searchid "
+                "AND t.card_id=n.card_id AND t.entity_id=n.entity_id "
+                "AND t.tag_name=n.tag_name "
+                "WHERE n.keyword=%s AND n.searchid=%s AND n.card_id=%s "
+                "ORDER BY n.id",
+                (keyword, linkage["searchid"], linkage["card_id"]),
+            )
+            rows = cursor.fetchall()
+    finally:
+        db.close()
+
+    grouped: dict[str, dict[str, list[dict[str, Any]]]] = {}
+    for row in rows:
+        try:
+            source = json.loads(row.get("raw_json") or "{}")
+        except (TypeError, ValueError):
+            continue
+        note = source.get("note_info") if isinstance(source.get("note_info"), dict) else {}
+        if not note:
+            continue
+        note = dict(note)
+        note.setdefault("type", source.get("type") or note.get("type"))
+        note.setdefault(
+            "wechat_share_desc",
+            source.get("wechat_share_desc")
+            or note.get("paragraph_text")
+            or note.get("relevant_text"),
+        )
+        product = str(row.get("product_name") or "")
+        tag = str(row.get("tag_name") or "")
+        grouped.setdefault(product, {}).setdefault(tag, []).append(note)
+
+    products = [
+        {
+            "product_name": product,
+            "content_tags": [
+                {"content_tag": tag, "count": len(notes), "notes": notes}
+                for tag, notes in tags.items()
+            ],
+        }
+        for product, tags in grouped.items()
+    ]
+    count = sum(
+        len(tag["notes"])
+        for product in products for tag in product["content_tags"]
+    )
+    return {
+        "requested_count": count,
+        "success_count": count,
+        "error_count": 0,
+        "errors": [],
+        "products": products,
+        "recovered_from": "xhs_wen_tag_notes",
+    }
+
+
+def cancel_collection(crawler_task_id: str) -> dict[str, Any]:
+    """通知爬虫任务在下一个安全检查点停止并保存部分结果。"""
+
+    return _post(f"/api/v1/xhs/wen/tasks/{crawler_task_id}/cancel", {})
+
+
 def _get(path: str) -> dict[str, Any]:
     url = f"{Config.CRAWLER_BASE_URL}{path}"
     try:
@@ -101,46 +306,8 @@ def get_wen_resources() -> dict[str, Any]:
     return _get("/api/v1/xhs/wen/resources")
 
 
-def start_collection(mode: str, keywords: list[str]) -> dict[str, Any]:
-    """深度任务走 sources，轻度/常规任务走 overview。"""
-
-    collection_keywords = normalize_collection_keywords(keywords)
-    path = "/api/v1/xhs/wen/sources" if mode == "deep" else "/api/v1/xhs/wen/overview"
-    # 远程 overview 只有接收数组时才创建异步任务并立即返回 task_id。
-    # 因此轻度任务即使只有一个关键词也必须传数组，避免单关键词走同步
-    # overview 并因采集耗时超过网关时限而返回 502。
-    keyword: str | list[str] = (
-        collection_keywords if mode == "mild" else collection_keywords[0]
-    )
-    # 轻度 overview 在首次计算超过网关时限后，远程通常已经写好缓存，
-    # 允许受控重试以取得结果；深度 sources 会启动长任务，禁止盲目重试，
-    # 避免远程请求虽断开但后台继续执行时产生重复采集任务。
-    return _post(
-        path,
-        {"keyword": keyword},
-        retry_transient=(mode != "deep"),
-    )
-
-
-def query_collection(crawler_task_id: str) -> dict[str, Any]:
-    return _post("/api/v1/xhs/wen/tasks/query", {"task_id": crawler_task_id})
-
-
-def cancel_collection(crawler_task_id: str) -> dict[str, Any]:
-    """通知爬虫任务在下一个安全检查点停止并保存部分结果。"""
-
-    return _post(f"/api/v1/xhs/wen/tasks/{crawler_task_id}/cancel", {})
-
-
-def resume_collection(crawler_task_id: str) -> dict[str, Any]:
-    """沿用原爬虫任务 ID，从 framework 保存的检查点继续采集。"""
-
-    return _post(f"/api/v1/xhs/wen/tasks/{crawler_task_id}/resume", {})
-
-
 def fetch_content_detail(note_url: str) -> dict[str, Any]:
     """按小红书笔记链接读取完整笔记详情。"""
-
     return _post(
         "/api/v1/fetch_content_detail",
         {"plant": "xhs", "link": note_url},
@@ -149,8 +316,7 @@ def fetch_content_detail(note_url: str) -> dict[str, Any]:
 
 
 def schedule_xapi_screenshots(search_queries: list[str], device_count: int) -> dict[str, Any]:
-    """向手机集群下发问一问关键词搜索与截图任务。"""
+    """向手机集群下发关键词搜索与截图任务。"""
-
     queries = normalize_collection_keywords(search_queries)
     return _post(
         "/api/v1/xhs/wen/xapi/schedule",
@@ -160,8 +326,7 @@ def schedule_xapi_screenshots(search_queries: list[str], device_count: int) -> d
 
 
 def collect_keyword_sources(keyword: str, credential_id: str | None = None) -> dict[str, Any]:
-    """使用 sources 的完整 Cookie 池采集单个关键词。"""
+    """使用指定 Cookie 采集单个关键词。"""
-
     normalized = normalize_collection_keywords([keyword])[0]
     body: dict[str, Any] = {"keyword": normalized}
     if credential_id:
@@ -171,3 +336,9 @@ def collect_keyword_sources(keyword: str, credential_id: str | None = None) -> d
         body,
         retry_transient=False,
     )
+
+
+def resume_collection(crawler_task_id: str) -> dict[str, Any]:
+    """沿用原爬虫任务 ID，从 framework 保存的检查点继续采集。"""
+
+    return _post(f"/api/v1/xhs/wen/tasks/{crawler_task_id}/resume", {})

database.py

@@ -85,6 +85,25 @@ def init_database() -> None:
                 FOREIGN KEY(task_id) REFERENCES research_tasks(id) ON DELETE CASCADE
             );
 
+            CREATE TABLE IF NOT EXISTS task_keyword_runs (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                task_id INTEGER NOT NULL,
+                keyword_order INTEGER NOT NULL,
+                keyword TEXT NOT NULL,
+                status TEXT NOT NULL DEFAULT 'queued',
+                progress_percent INTEGER NOT NULL DEFAULT 0,
+                crawler_task_id TEXT,
+                result_json TEXT,
+                error_message TEXT,
+                started_at TEXT,
+                completed_at TEXT,
+                updated_at TEXT NOT NULL,
+                UNIQUE(task_id, keyword_order),
+                FOREIGN KEY(task_id) REFERENCES research_tasks(id) ON DELETE CASCADE
+            );
+            CREATE INDEX IF NOT EXISTS idx_keyword_runs_task_status
+                ON task_keyword_runs(task_id, status, keyword_order);
+
             CREATE TABLE IF NOT EXISTS task_files (
                 id INTEGER PRIMARY KEY AUTOINCREMENT,
                 public_file_id TEXT NOT NULL UNIQUE,
@@ -134,6 +153,19 @@ def init_database() -> None:
             CREATE INDEX IF NOT EXISTS idx_auth_sessions_user
                 ON auth_sessions(user_id, expires_at);
 
+            CREATE TABLE IF NOT EXISTS ai_access_tokens (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                token_hash TEXT NOT NULL UNIQUE,
+                user_id INTEGER NOT NULL,
+                expires_at TEXT NOT NULL,
+                created_at TEXT NOT NULL,
+                last_seen_at TEXT NOT NULL,
+                revoked_at TEXT,
+                FOREIGN KEY(user_id) REFERENCES auth_users(id) ON DELETE CASCADE
+            );
+            CREATE INDEX IF NOT EXISTS idx_ai_access_tokens_user
+                ON ai_access_tokens(user_id, expires_at);
+
             CREATE TABLE IF NOT EXISTS acceptance_tasks (
                 id INTEGER PRIMARY KEY AUTOINCREMENT,
                 public_task_id TEXT NOT NULL UNIQUE,

requirements.txt

@@ -2,4 +2,5 @@ Flask==3.1.1
 requests==2.32.4
 python-dotenv==1.1.1
 gunicorn==23.0.0
+PyMySQL==1.1.1
 playwright==1.58.0