export const DEFAULT_COLLECTION_MCP_URL = "https://middle-aitool.gbotai.cn/mcp"; const MAX_MCP_ATTEMPTS = 10; export type CollectionMcpBindings = { AI_TOOL_CENTER_MCP_URL?: string; AI_TOOL_CENTER_MCP_KEY?: string; }; export type CollectionMcpConfig = { endpoint: string; key?: string; timeoutMs?: number; }; export type XhsPublicMetrics = { likes: number; comments: number; collects: number; }; export type XhsAccountProfile = { platformUid: string; nickname: string; profileUrl: string; redId: string; ipLocation: string; followers: number | null; }; type JsonRpcEnvelope = { error?: { code?: number; message?: string; }; result?: { isError?: boolean; content?: Array<{ type?: string; text?: string; }>; serverInfo?: { name?: string; version?: string; }; }; }; type ToolResult = { isError: boolean; payload: unknown; }; class McpSessionLostError extends Error {} function isRetryableTransportError(error: unknown) { return ( error instanceof McpSessionLostError || (error instanceof Error && ["AbortError", "TimeoutError"].includes(error.name)) || error instanceof TypeError ); } function asRecord(value: unknown): Record | null { return value !== null && typeof value === "object" && !Array.isArray(value) ? (value as Record) : null; } function safeMessage(value: unknown, fallback: string) { const message = String(value ?? "").trim(); if (!message) return fallback; return message .replace(/https?:\/\/[^\s"'<>]+/gi, "[链接]") .replace(/eyJ[A-Za-z0-9._-]+/g, "[密钥]") .slice(0, 240); } function metricValue(value: unknown, label: string) { if (typeof value === "number" && Number.isFinite(value) && value >= 0) { return Math.round(value); } if (typeof value !== "string") { throw new Error(`采集结果缺少${label}`); } const normalized = value.trim().toLowerCase().replace(/[,\s]/g, ""); const match = normalized.match(/^(\d+(?:\.\d+)?)(万|w|千|k)?$/); if (!match) throw new Error(`采集结果中的${label}格式异常`); const multiplier = match[2] === "万" || match[2] === "w" ? 10_000 : match[2] === "千" || match[2] === "k" ? 1_000 : 1; return Math.round(Number(match[1]) * multiplier); } function optionalCountValue(value: unknown) { if (value === null || value === undefined || value === "") return null; const normalized = typeof value === "string" ? value.trim().replace(/\+$/, "") : value; try { return metricValue(normalized, "粉丝数"); } catch { return null; } } function parseJsonRpc(raw: string) { const messages = raw .split(/\r?\n/) .filter((line) => line.startsWith("data:")) .map((line) => line.slice(5).trim()) .filter(Boolean); const candidate = messages.at(-1) ?? raw.trim(); if (!candidate) throw new Error("MCP采集服务返回空响应"); try { return JSON.parse(candidate) as JsonRpcEnvelope; } catch { throw new Error("MCP采集服务返回了无法解析的数据"); } } function buildMcpUrl(config: CollectionMcpConfig) { let endpoint: URL; try { endpoint = new URL(config.endpoint); } catch { throw new Error("MCP采集服务地址无效"); } if (!["http:", "https:"].includes(endpoint.protocol)) { throw new Error("MCP采集服务地址无效"); } if (config.key?.trim()) { endpoint.searchParams.set("key", config.key.trim()); } if (!endpoint.searchParams.get("key")) { throw new Error("MCP采集密钥未配置"); } return endpoint.toString(); } function requestHeaders(sessionId?: string) { const headers = new Headers({ "Content-Type": "application/json", Accept: "application/json, text/event-stream", }); if (sessionId) headers.set("Mcp-Session-Id", sessionId); return headers; } async function postMcp( fetchImpl: typeof fetch, endpoint: string, body: Record, timeoutMs: number, sessionId?: string, allowEmpty = false, ) { const response = await fetchImpl(endpoint, { method: "POST", headers: requestHeaders(sessionId), body: JSON.stringify(body), signal: AbortSignal.timeout(timeoutMs), }); const text = await response.text(); if (!response.ok) { if (/session not found|missing session id/i.test(text)) { throw new McpSessionLostError("MCP采集会话已失效"); } let providerMessage: unknown = text; try { providerMessage = (JSON.parse(text) as JsonRpcEnvelope).error?.message ?? text; } catch { // Keep the original body when the upstream response is not JSON. } throw new Error( safeMessage( providerMessage, `MCP采集服务请求失败(HTTP ${response.status})`, ), ); } if (allowEmpty && !text.trim()) { return { response, envelope: null }; } const envelope = parseJsonRpc(text); if (envelope.error) { const message = safeMessage( envelope.error.message, "MCP采集服务调用失败", ); if (/session not found|missing session id/i.test(message)) { throw new McpSessionLostError("MCP采集会话已失效"); } throw new Error(message); } return { response, envelope }; } async function createMcpSession( fetchImpl: typeof fetch, endpoint: string, timeoutMs: number, ) { const initialize = await postMcp( fetchImpl, endpoint, { jsonrpc: "2.0", id: crypto.randomUUID(), method: "initialize", params: { protocolVersion: "2025-03-26", capabilities: {}, clientInfo: { name: "koc-loop-collector", version: "1.0.0", }, }, }, timeoutMs, ); const sessionId = initialize.response.headers.get("mcp-session-id"); if (!sessionId) throw new Error("MCP采集服务未返回会话标识"); await postMcp( fetchImpl, endpoint, { jsonrpc: "2.0", method: "notifications/initialized", params: {}, }, timeoutMs, sessionId, true, ); return sessionId; } async function callMcpTool( fetchImpl: typeof fetch, endpoint: string, sessionId: string, timeoutMs: number, name: string, args: Record, ): Promise { const result = await postMcp( fetchImpl, endpoint, { jsonrpc: "2.0", id: crypto.randomUUID(), method: "tools/call", params: { name, arguments: args, }, }, timeoutMs, sessionId, ); const content = result.envelope?.result?.content ?? []; const text = content.find((block) => block.type === "text")?.text; if (!text) throw new Error("MCP采集工具未返回数据"); let payload: unknown; try { payload = JSON.parse(text); } catch { throw new Error("MCP采集工具返回了无法解析的数据"); } return { isError: result.envelope?.result?.isError === true, payload, }; } function metricsFromToolResult(result: ToolResult): XhsPublicMetrics { const root = asRecord(result.payload); const response = asRecord(root?.response) ?? root; const data = asRecord(response?.data) ?? asRecord(root?.data); const code = Number(response?.code); const success = response?.success; if ( result.isError || success === false || (Number.isFinite(code) && code >= 400) ) { throw new Error( safeMessage( response?.msg ?? response?.message ?? root?.message, "公开数据采集失败", ), ); } if (!data) throw new Error("采集结果缺少互动数据"); return { likes: metricValue(data.likes, "点赞数"), comments: metricValue(data.comments, "评论数"), collects: metricValue( data.collects ?? data.favorites ?? data.favourites, "收藏数", ), }; } function successfulToolData(result: ToolResult, fallbackMessage: string) { const root = asRecord(result.payload); const response = asRecord(root?.response) ?? root; const data = asRecord(response?.data) ?? asRecord(root?.data); const code = Number(response?.code); if ( result.isError || response?.success === false || (Number.isFinite(code) && code >= 400) ) { throw new Error( safeMessage( response?.msg ?? response?.message ?? root?.message, fallbackMessage, ), ); } if (!data) throw new Error(fallbackMessage); return data; } function stringValue(value: unknown) { return typeof value === "string" ? value.trim() : ""; } function findRecord( value: unknown, predicate: (record: Record) => boolean, depth = 0, ): Record | null { if (depth > 12) return null; if (Array.isArray(value)) { for (const item of value) { const found = findRecord(item, predicate, depth + 1); if (found) return found; } return null; } const record = asRecord(value); if (!record) return null; if (predicate(record)) return record; for (const child of Object.values(record)) { const found = findRecord(child, predicate, depth + 1); if (found) return found; } return null; } function findStringByKey(value: unknown, key: string, depth = 0): string { if (depth > 12) return ""; if (Array.isArray(value)) { for (const item of value) { const found = findStringByKey(item, key, depth + 1); if (found) return found; } return ""; } const record = asRecord(value); if (!record) return ""; const direct = stringValue(record[key]); if (direct) return direct; for (const child of Object.values(record)) { const found = findStringByKey(child, key, depth + 1); if (found) return found; } return ""; } function findValueByKeys( value: unknown, keys: ReadonlySet, depth = 0, ): unknown { if (depth > 12) return undefined; if (Array.isArray(value)) { for (const item of value) { const found = findValueByKeys(item, keys, depth + 1); if (found !== undefined) return found; } return undefined; } const record = asRecord(value); if (!record) return undefined; for (const [key, child] of Object.entries(record)) { if (keys.has(key) && child !== null && child !== undefined && child !== "") { return child; } } for (const child of Object.values(record)) { const found = findValueByKeys(child, keys, depth + 1); if (found !== undefined) return found; } return undefined; } const FOLLOWER_KEYS = new Set([ "fans", "fans_count", "fansCount", "follower", "followers", "follower_count", "followerCount", ]); function followerCountFromPayload(value: unknown, depth = 0): number | null { const direct = optionalCountValue(findValueByKeys(value, FOLLOWER_KEYS)); if (direct !== null) return direct; if (typeof value === "string") { return optionalCountValue( value.match(/(?:粉丝数?|followers?|fans)\s*[::]\s*([\d,.万千wkWKW+]+)/i)?.[1], ); } if (depth > 12) return null; if (Array.isArray(value)) { for (const item of value) { const found = followerCountFromPayload(item, depth + 1); if (found !== null) return found; } return null; } const record = asRecord(value); if (!record) return null; const label = stringValue( record.name ?? record.label ?? record.title ?? record.type, ).toLowerCase(); if (["粉丝", "粉丝数", "fans", "followers"].includes(label)) { const labeled = optionalCountValue( record.count ?? record.value ?? record.i18nCount, ); if (labeled !== null) return labeled; } for (const child of Object.values(record)) { const found = followerCountFromPayload(child, depth + 1); if (found !== null) return found; } return null; } export function xhsNoteIdFromUrl(input: string) { try { const url = new URL(input); if ( !( url.hostname === "xiaohongshu.com" || url.hostname.endsWith(".xiaohongshu.com") ) ) { return ""; } return ( url.pathname.match( /\/(?:discovery\/item|explore)\/([A-Za-z0-9_-]{8,80})/, )?.[1] ?? "" ); } catch { return ""; } } async function xhsNoteIdFromShortLink( input: string, fallbackNickname: string, fetchImpl: typeof fetch, timeoutMs: number, ) { let url: URL; try { url = new URL(input); } catch { return { noteId: "", profile: null }; } if ( url.protocol !== "http:" && url.protocol !== "https:" ) { return { noteId: "", profile: null }; } if ( url.hostname !== "xhslink.cn" && !url.hostname.endsWith(".xhslink.cn") ) { return { noteId: "", profile: null }; } const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), timeoutMs); try { const response = await fetchImpl(url.toString(), { method: "GET", redirect: "follow", signal: controller.signal, headers: { "user-agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 18_0 like Mac OS X) AppleWebKit/605.1.15 Mobile/15E148", }, }); const location = response.headers.get("location"); const resolvedUrl = location ? new URL(location, url).toString() : response.url; const noteId = xhsNoteIdFromUrl(resolvedUrl); const html = response.ok && response.headers.get("content-type")?.includes("text/html") ? await response.text() : ""; return { noteId, profile: accountProfileFromPublicPage( html, fallbackNickname, ), }; } catch { return { noteId: "", profile: null }; } finally { clearTimeout(timeout); } } function accountProfileFromPublicPage( html: string, fallbackNickname: string, ): XhsAccountProfile | null { if (!html) return null; const decoded = html .replaceAll(""", '"') .replaceAll(""", '"') .replaceAll("&", "&") .replaceAll("'", "'") .replaceAll("<", "<") .replaceAll(">", ">"); const noteData = decoded.match( /"noteData":\{[\s\S]{0,30000}?"user":\{([\s\S]{0,1800}?)\}/, )?.[1]; if (!noteData) return null; const platformUid = noteData.match(/"userId":"([A-Za-z0-9_-]{8,80})"/)?.[1] ?? ""; if (!platformUid) return null; const nickname = noteData.match(/"(?:nickName|nickname)":"([^"]+)"/)?.[1] ?? fallbackNickname.trim(); const redId = noteData.match(/"(?:redId|red_id)":"([^"]+)"/)?.[1] ?? decoded.match(/"(?:redId|red_id)":"([^"]+)"/)?.[1] ?? decoded.match(/小红书号[::]\s*([^<"\s]+)/)?.[1] ?? ""; return { platformUid, nickname, profileUrl: `https://www.xiaohongshu.com/user/profile/${encodeURIComponent( platformUid, )}`, redId, ipLocation: "待识别", followers: null, }; } export async function resolveXhsPublicAccountDetails( profileUrl: string, fetchImpl: typeof fetch = fetch, ) { let parsed: URL; try { parsed = new URL(profileUrl); } catch { return { redId: "", followers: null, ipLocation: "" }; } if ( parsed.protocol !== "https:" || !( parsed.hostname === "xiaohongshu.com" || parsed.hostname.endsWith(".xiaohongshu.com") ) || !parsed.pathname.startsWith("/user/profile/") ) { return { redId: "", followers: null, ipLocation: "" }; } try { const response = await fetchImpl(parsed.toString(), { method: "GET", redirect: "follow", signal: AbortSignal.timeout(15_000), headers: { "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/138.0 Safari/537.36", }, }); if (!response.ok) { return { redId: "", followers: null, ipLocation: "" }; } const html = await response.text(); return { redId: html.match(/"(?:redId|red_id)":"([^"]+)"/)?.[1] ?? html.match(/小红书号[::]\s*([^<"\s]+)/)?.[1] ?? "", followers: optionalCountValue( html.match( /"name"\s*:\s*"粉丝"\s*,\s*"count"\s*:\s*"([^"]+)"/, )?.[1], ) ?? followerCountFromPayload(html), ipLocation: "", }; } catch { return { redId: "", followers: null, ipLocation: "" }; } } export async function resolveXhsPublicAccountId( profileUrl: string, fetchImpl: typeof fetch = fetch, ) { return (await resolveXhsPublicAccountDetails(profileUrl, fetchImpl)).redId; } function profileDetailsFromToolResult(result: ToolResult) { const root = asRecord(result.payload); const response = asRecord(root?.response) ?? root; const code = Number(response?.code); if ( result.isError || response?.success === false || (Number.isFinite(code) && code >= 400) ) { throw new Error( safeMessage( response?.msg ?? response?.message ?? root?.message, "账号主页数据采集失败", ), ); } const payload = asRecord(response?.data) ?? asRecord(root?.data) ?? result.payload; const followers = followerCountFromPayload(payload); const redId = findStringByKey(payload, "red_id") || findStringByKey(payload, "redId") || findStringByKey(payload, "userId") || findStringByKey(payload, "user_id"); const ipLocation = findStringByKey(payload, "ip_location") || findStringByKey(payload, "ipLocation"); if (followers === null && !redId && !ipLocation) { throw new Error("账号主页采集结果缺少可用字段"); } return { followers, redId, ipLocation }; } function accountProfileFromToolResult( result: ToolResult, fallbackNickname: string, ): XhsAccountProfile { const data = successfulToolData(result, "账号主页识别失败"); const user = findRecord( data, (record) => Boolean( stringValue(record.user_id ?? record.userid) && (stringValue(record.profile_url) || stringValue(record.nickname ?? record.name)), ), ); if (!user) throw new Error("账号主页识别结果缺少作者信息"); const platformUid = stringValue(user.user_id ?? user.userid); const candidateProfileUrl = stringValue(user.profile_url); let profileUrl = ""; if (candidateProfileUrl) { try { const parsed = new URL(candidateProfileUrl); if ( parsed.protocol === "https:" && (parsed.hostname === "xiaohongshu.com" || parsed.hostname.endsWith(".xiaohongshu.com")) && parsed.pathname.startsWith("/user/profile/") ) { profileUrl = parsed.toString(); } } catch { // Fall back to the verified user id below. } } if (!profileUrl) { profileUrl = `https://www.xiaohongshu.com/user/profile/${encodeURIComponent( platformUid, )}`; } return { platformUid, nickname: stringValue(user.nickname ?? user.name) || fallbackNickname.trim(), profileUrl, redId: stringValue(user.red_id), ipLocation: findStringByKey(data, "ip_location") || "待识别", followers: followerCountFromPayload(data), }; } async function completeAccountProfile( profile: XhsAccountProfile, fetchImpl: typeof fetch, endpoint: string, sessionId: string, timeoutMs: number, ) { let completed = profile; for (const [name, args] of [ [ "parse_xhs_user_summary", { url: profile.profileUrl, use_proxy: true }, ], [ "fetch_user_detail", { link: profile.profileUrl, plant: "xhs" }, ], ] as const) { if ( completed.followers !== null && completed.redId && completed.ipLocation !== "待识别" ) { break; } try { const result = await callMcpTool( fetchImpl, endpoint, sessionId, timeoutMs, name, args, ); const details = profileDetailsFromToolResult(result); completed = { ...completed, followers: details.followers ?? completed.followers, redId: details.redId || completed.redId, ipLocation: details.ipLocation && details.ipLocation !== "待识别" ? details.ipLocation : completed.ipLocation, }; } catch (error) { if (error instanceof McpSessionLostError) throw error; } } if (completed.followers === null || !completed.redId) { const details = await resolveXhsPublicAccountDetails( completed.profileUrl, fetchImpl, ); completed = { ...completed, followers: details.followers ?? completed.followers, redId: details.redId || completed.redId, }; } return completed; } async function resolveProfileDetailsInSession( profileUrl: string, config: CollectionMcpConfig, fetchImpl: typeof fetch, ) { const endpoint = buildMcpUrl(config); const timeoutMs = Math.max(5_000, config.timeoutMs ?? 30_000); const sessionId = await createMcpSession(fetchImpl, endpoint, timeoutMs); const result = await callMcpTool( fetchImpl, endpoint, sessionId, timeoutMs, "parse_xhs_user_summary", { url: profileUrl, use_proxy: true, }, ); return profileDetailsFromToolResult(result); } async function resolveAccountInSession( publishUrl: string, fallbackNickname: string, config: CollectionMcpConfig, fetchImpl: typeof fetch, ) { const endpoint = buildMcpUrl(config); const timeoutMs = Math.max(5_000, config.timeoutMs ?? 30_000); let noteId = xhsNoteIdFromUrl(publishUrl); let publicPageProfile: XhsAccountProfile | null = null; if (!noteId) { const shortLink = await xhsNoteIdFromShortLink( publishUrl, fallbackNickname, fetchImpl, timeoutMs, ); noteId = shortLink.noteId; publicPageProfile = shortLink.profile; } try { const sessionId = await createMcpSession( fetchImpl, endpoint, timeoutMs, ); if (!noteId) { const noteResult = await callMcpTool( fetchImpl, endpoint, sessionId, timeoutMs, "fetch_content_detail", { link: publishUrl, plant: "xhs", include_comments: false, auto_cookie: true, }, ); const noteData = successfulToolData( noteResult, "无法识别小红书笔记", ); noteId = stringValue(noteData.noteId ?? noteData.note_id) || findStringByKey(noteData, "noteId") || findStringByKey(noteData, "note_id"); } if (!noteId) throw new Error("无法从发布链接识别小红书笔记ID"); const authorResult = await callMcpTool( fetchImpl, endpoint, sessionId, timeoutMs, "collect_xhs_wen_note_detail", { note_id: noteId, need_desc: false, include_raw: false, }, ); const profile = accountProfileFromToolResult( authorResult, fallbackNickname, ); return completeAccountProfile( profile, fetchImpl, endpoint, sessionId, timeoutMs, ); } catch (error) { if (error instanceof McpSessionLostError) throw error; if (publicPageProfile) { const details = await resolveXhsPublicAccountDetails( publicPageProfile.profileUrl, fetchImpl, ); return { ...publicPageProfile, redId: details.redId || publicPageProfile.redId, followers: details.followers, }; } throw error; } } async function collectInSession( publishUrl: string, config: CollectionMcpConfig, fetchImpl: typeof fetch, ) { const endpoint = buildMcpUrl(config); const timeoutMs = Math.max(5_000, config.timeoutMs ?? 30_000); const sessionId = await createMcpSession(fetchImpl, endpoint, timeoutMs); const primary = await callMcpTool( fetchImpl, endpoint, sessionId, timeoutMs, "fetch_content_detail", { link: publishUrl, plant: "xhs", include_comments: false, auto_cookie: true, }, ); try { return metricsFromToolResult(primary); } catch { const fallback = await callMcpTool( fetchImpl, endpoint, sessionId, timeoutMs, "parse_xhs_note", { url: publishUrl, include_comments: false, auto_cookie: true, }, ); return metricsFromToolResult(fallback); } } export function resolveCollectionMcpConfig( bindings: CollectionMcpBindings, ): CollectionMcpConfig { return { endpoint: bindings.AI_TOOL_CENTER_MCP_URL?.trim() || DEFAULT_COLLECTION_MCP_URL, key: bindings.AI_TOOL_CENTER_MCP_KEY?.trim(), }; } export async function collectXhsMetricsFromMcp( publishUrl: string, config: CollectionMcpConfig, fetchImpl: typeof fetch = fetch, ) { let parsed: URL; try { parsed = new URL(publishUrl); } catch { throw new Error("发布链接无效"); } if (!["http:", "https:"].includes(parsed.protocol)) { throw new Error("发布链接无效"); } let lastError: unknown; for (let attempt = 0; attempt < MAX_MCP_ATTEMPTS; attempt += 1) { try { return await collectInSession(parsed.toString(), config, fetchImpl); } catch (error) { lastError = error; if (!isRetryableTransportError(error)) throw error; } } if ( lastError instanceof Error && ["AbortError", "TimeoutError"].includes(lastError.name) ) { throw new Error("MCP采集服务响应超时,请稍后重试"); } throw lastError instanceof Error ? lastError : new Error("MCP采集服务暂时不可用"); } export async function resolveXhsAccountProfileFromMcp( publishUrl: string, fallbackNickname: string, config: CollectionMcpConfig, fetchImpl: typeof fetch = fetch, ) { let parsed: URL; try { parsed = new URL(publishUrl); } catch { throw new Error("发布链接无效"); } if (!["http:", "https:"].includes(parsed.protocol)) { throw new Error("发布链接无效"); } let lastError: unknown; for (let attempt = 0; attempt < MAX_MCP_ATTEMPTS; attempt += 1) { try { return await resolveAccountInSession( parsed.toString(), fallbackNickname, config, fetchImpl, ); } catch (error) { lastError = error; if (!isRetryableTransportError(error)) throw error; } } throw lastError instanceof Error ? lastError : new Error("账号主页识别服务暂时不可用"); } export async function resolveXhsProfileDetailsFromMcp( profileUrl: string, config: CollectionMcpConfig, fetchImpl: typeof fetch = fetch, ) { let parsed: URL; try { parsed = new URL(profileUrl); } catch { throw new Error("账号主页链接无效"); } if ( parsed.protocol !== "https:" || !( parsed.hostname === "xiaohongshu.com" || parsed.hostname.endsWith(".xiaohongshu.com") ) || !parsed.pathname.startsWith("/user/profile/") ) { throw new Error("账号主页链接无效"); } let lastError: unknown; for (let attempt = 0; attempt < MAX_MCP_ATTEMPTS; attempt += 1) { try { return await resolveProfileDetailsInSession( parsed.toString(), config, fetchImpl, ); } catch (error) { lastError = error; if (!isRetryableTransportError(error)) throw error; } } throw lastError instanceof Error ? lastError : new Error("账号主页数据采集服务暂时不可用"); }