holy后端

crawler_client.py

@@ -0,0 +1,115 @@
+import re
+import time
+from typing import Any
+
+import requests
+
+from config import Config
+
+
+class CrawlerError(RuntimeError):
+    pass
+
+
+CRAWLER_SESSION = requests.Session()
+# 爬虫服务为明确配置的直连地址，不应继承 macOS 系统代理。
+# 否则 requests 会经由 127.0.0.1:7890，约 30 秒后被代理返回 502。
+CRAWLER_SESSION.trust_env = False
+
+
+def normalize_collection_keywords(keywords: list[str]) -> list[str]:
+    """清理影响小红书问一问卡片匹配的末尾标点，并保持关键词顺序。"""
+
+    normalized: list[str] = []
+    for keyword in keywords:
+        value = re.sub(r"[?？!！。．.]+$", "", str(keyword).strip()).strip()
+        if value and value not in normalized:
+            normalized.append(value)
+    if not normalized:
+        raise CrawlerError("关键词不能为空")
+    return normalized
+
+
+def _post(
+    path: str,
+    body: dict[str, Any],
+    *,
+    retry_transient: bool = True,
+) -> dict[str, Any]:
+    url = f"{Config.CRAWLER_BASE_URL}{path}"
+    retry_times = max(1, Config.CRAWLER_RETRY_TIMES) if retry_transient else 1
+    response: requests.Response | None = None
+
+    for attempt in range(1, retry_times + 1):
+        try:
+            response = CRAWLER_SESSION.post(
+                url,
+                json=body,
+                timeout=Config.CRAWLER_TIMEOUT_SECONDS,
+            )
+            # 网关临时异常时自动重试，避免直接将业务任务标记为失败。
+            if response.status_code in {502, 503, 504} and attempt < retry_times:
+                time.sleep(Config.CRAWLER_RETRY_DELAY_SECONDS * attempt)
+                continue
+            response.raise_for_status()
+            payload = response.json()
+            break
+        except (requests.ConnectionError, requests.Timeout) as exc:
+            if attempt < retry_times:
+                time.sleep(Config.CRAWLER_RETRY_DELAY_SECONDS * attempt)
+                continue
+            raise CrawlerError(f"爬虫服务请求失败（已重试 {retry_times} 次）：{exc}") from exc
+        except requests.RequestException as exc:
+            detail = response.text[:500] if response is not None else ""
+            message = f"爬虫服务请求失败：{exc}"
+            if detail:
+                message += f"，响应：{detail}"
+            raise CrawlerError(message) from exc
+        except ValueError as exc:
+            raise CrawlerError("爬虫服务返回的不是有效 JSON") from exc
+    else:
+        raise CrawlerError(f"爬虫服务请求失败（已重试 {retry_times} 次）")
+
+    if payload.get("code") != 200 or not payload.get("success", False):
+        raise CrawlerError(payload.get("msg") or "爬虫服务返回失败")
+    data = payload.get("data")
+    if not isinstance(data, dict):
+        raise CrawlerError("爬虫服务响应缺少 data")
+    return data
+
+
+def start_collection(mode: str, keywords: list[str]) -> dict[str, Any]:
+    """深度任务走 sources，轻度/常规任务走 overview。"""
+
+    collection_keywords = normalize_collection_keywords(keywords)
+    path = "/api/v1/xhs/wen/sources" if mode == "deep" else "/api/v1/xhs/wen/overview"
+    # 远程 overview 只有接收数组时才创建异步任务并立即返回 task_id。
+    # 因此轻度任务即使只有一个关键词也必须传数组，避免单关键词走同步
+    # overview 并因采集耗时超过网关时限而返回 502。
+    keyword: str | list[str] = (
+        collection_keywords if mode == "mild" else collection_keywords[0]
+    )
+    # 轻度 overview 在首次计算超过网关时限后，远程通常已经写好缓存，
+    # 允许受控重试以取得结果；深度 sources 会启动长任务，禁止盲目重试，
+    # 避免远程请求虽断开但后台继续执行时产生重复采集任务。
+    return _post(
+        path,
+        {"keyword": keyword},
+        retry_transient=(mode != "deep"),
+    )
+
+
+def query_collection(crawler_task_id: str) -> dict[str, Any]:
+    return _post("/api/v1/xhs/wen/tasks/query", {"task_id": crawler_task_id})
+
+
+def cancel_collection(crawler_task_id: str) -> dict[str, Any]:
+    """通知爬虫任务在下一个安全检查点停止并保存部分结果。"""
+
+    return _post(f"/api/v1/xhs/wen/tasks/{crawler_task_id}/cancel", {})
+
+
+def resume_collection(crawler_task_id: str) -> dict[str, Any]:
+    """沿用原爬虫任务 ID，从 framework 保存的检查点继续采集。"""
+
+    return _post(f"/api/v1/xhs/wen/tasks/{crawler_task_id}/resume", {})