1054 lines
27 KiB
TypeScript
1054 lines
27 KiB
TypeScript
|
export const DEFAULT_COLLECTION_MCP_URL =
|
|||
|
|
"https://middle-aitool.gbotai.cn/mcp";
|
|||
|
|
const MAX_MCP_ATTEMPTS = 10;
|
|||
|
|
|
|||
|
|
export type CollectionMcpBindings = {
|
|||
|
|
AI_TOOL_CENTER_MCP_URL?: string;
|
|||
|
|
AI_TOOL_CENTER_MCP_KEY?: string;
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
export type CollectionMcpConfig = {
|
|||
|
|
endpoint: string;
|
|||
|
|
key?: string;
|
|||
|
|
timeoutMs?: number;
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
export type XhsPublicMetrics = {
|
|||
|
|
likes: number;
|
|||
|
|
comments: number;
|
|||
|
|
collects: number;
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
export type XhsAccountProfile = {
|
|||
|
|
platformUid: string;
|
|||
|
|
nickname: string;
|
|||
|
|
profileUrl: string;
|
|||
|
|
redId: string;
|
|||
|
|
ipLocation: string;
|
|||
|
|
followers: number | null;
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
type JsonRpcEnvelope = {
|
|||
|
|
error?: {
|
|||
|
|
code?: number;
|
|||
|
|
message?: string;
|
|||
|
|
};
|
|||
|
|
result?: {
|
|||
|
|
isError?: boolean;
|
|||
|
|
content?: Array<{
|
|||
|
|
type?: string;
|
|||
|
|
text?: string;
|
|||
|
|
}>;
|
|||
|
|
serverInfo?: {
|
|||
|
|
name?: string;
|
|||
|
|
version?: string;
|
|||
|
|
};
|
|||
|
|
};
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
type ToolResult = {
|
|||
|
|
isError: boolean;
|
|||
|
|
payload: unknown;
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
class McpSessionLostError extends Error {}
|
|||
|
|
|
|||
|
|
function isRetryableTransportError(error: unknown) {
|
|||
|
|
return (
|
|||
|
|
error instanceof McpSessionLostError ||
|
|||
|
|
(error instanceof Error &&
|
|||
|
|
["AbortError", "TimeoutError"].includes(error.name)) ||
|
|||
|
|
error instanceof TypeError
|
|||
|
|
);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function asRecord(value: unknown): Record<string, unknown> | null {
|
|||
|
|
return value !== null && typeof value === "object" && !Array.isArray(value)
|
|||
|
|
? (value as Record<string, unknown>)
|
|||
|
|
: null;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function safeMessage(value: unknown, fallback: string) {
|
|||
|
|
const message = String(value ?? "").trim();
|
|||
|
|
if (!message) return fallback;
|
|||
|
|
return message
|
|||
|
|
.replace(/https?:\/\/[^\s"'<>]+/gi, "[链接]")
|
|||
|
|
.replace(/eyJ[A-Za-z0-9._-]+/g, "[密钥]")
|
|||
|
|
.slice(0, 240);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function metricValue(value: unknown, label: string) {
|
|||
|
|
if (typeof value === "number" && Number.isFinite(value) && value >= 0) {
|
|||
|
|
return Math.round(value);
|
|||
|
|
}
|
|||
|
|
if (typeof value !== "string") {
|
|||
|
|
throw new Error(`采集结果缺少${label}`);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
const normalized = value.trim().toLowerCase().replace(/[,\s]/g, "");
|
|||
|
|
const match = normalized.match(/^(\d+(?:\.\d+)?)(万|w|千|k)?$/);
|
|||
|
|
if (!match) throw new Error(`采集结果中的${label}格式异常`);
|
|||
|
|
const multiplier =
|
|||
|
|
match[2] === "万" || match[2] === "w"
|
|||
|
|
? 10_000
|
|||
|
|
: match[2] === "千" || match[2] === "k"
|
|||
|
|
? 1_000
|
|||
|
|
: 1;
|
|||
|
|
return Math.round(Number(match[1]) * multiplier);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function optionalCountValue(value: unknown) {
|
|||
|
|
if (value === null || value === undefined || value === "") return null;
|
|||
|
|
const normalized =
|
|||
|
|
typeof value === "string" ? value.trim().replace(/\+$/, "") : value;
|
|||
|
|
try {
|
|||
|
|
return metricValue(normalized, "粉丝数");
|
|||
|
|
} catch {
|
|||
|
|
return null;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function parseJsonRpc(raw: string) {
|
|||
|
|
const messages = raw
|
|||
|
|
.split(/\r?\n/)
|
|||
|
|
.filter((line) => line.startsWith("data:"))
|
|||
|
|
.map((line) => line.slice(5).trim())
|
|||
|
|
.filter(Boolean);
|
|||
|
|
const candidate = messages.at(-1) ?? raw.trim();
|
|||
|
|
if (!candidate) throw new Error("MCP采集服务返回空响应");
|
|||
|
|
try {
|
|||
|
|
return JSON.parse(candidate) as JsonRpcEnvelope;
|
|||
|
|
} catch {
|
|||
|
|
throw new Error("MCP采集服务返回了无法解析的数据");
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function buildMcpUrl(config: CollectionMcpConfig) {
|
|||
|
|
let endpoint: URL;
|
|||
|
|
try {
|
|||
|
|
endpoint = new URL(config.endpoint);
|
|||
|
|
} catch {
|
|||
|
|
throw new Error("MCP采集服务地址无效");
|
|||
|
|
}
|
|||
|
|
if (!["http:", "https:"].includes(endpoint.protocol)) {
|
|||
|
|
throw new Error("MCP采集服务地址无效");
|
|||
|
|
}
|
|||
|
|
if (config.key?.trim()) {
|
|||
|
|
endpoint.searchParams.set("key", config.key.trim());
|
|||
|
|
}
|
|||
|
|
if (!endpoint.searchParams.get("key")) {
|
|||
|
|
throw new Error("MCP采集密钥未配置");
|
|||
|
|
}
|
|||
|
|
return endpoint.toString();
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function requestHeaders(sessionId?: string) {
|
|||
|
|
const headers = new Headers({
|
|||
|
|
"Content-Type": "application/json",
|
|||
|
|
Accept: "application/json, text/event-stream",
|
|||
|
|
});
|
|||
|
|
if (sessionId) headers.set("Mcp-Session-Id", sessionId);
|
|||
|
|
return headers;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
async function postMcp(
|
|||
|
|
fetchImpl: typeof fetch,
|
|||
|
|
endpoint: string,
|
|||
|
|
body: Record<string, unknown>,
|
|||
|
|
timeoutMs: number,
|
|||
|
|
sessionId?: string,
|
|||
|
|
allowEmpty = false,
|
|||
|
|
) {
|
|||
|
|
const response = await fetchImpl(endpoint, {
|
|||
|
|
method: "POST",
|
|||
|
|
headers: requestHeaders(sessionId),
|
|||
|
|
body: JSON.stringify(body),
|
|||
|
|
signal: AbortSignal.timeout(timeoutMs),
|
|||
|
|
});
|
|||
|
|
const text = await response.text();
|
|||
|
|
if (!response.ok) {
|
|||
|
|
if (/session not found|missing session id/i.test(text)) {
|
|||
|
|
throw new McpSessionLostError("MCP采集会话已失效");
|
|||
|
|
}
|
|||
|
|
let providerMessage: unknown = text;
|
|||
|
|
try {
|
|||
|
|
providerMessage =
|
|||
|
|
(JSON.parse(text) as JsonRpcEnvelope).error?.message ?? text;
|
|||
|
|
} catch {
|
|||
|
|
// Keep the original body when the upstream response is not JSON.
|
|||
|
|
}
|
|||
|
|
throw new Error(
|
|||
|
|
safeMessage(
|
|||
|
|
providerMessage,
|
|||
|
|
`MCP采集服务请求失败（HTTP ${response.status}）`,
|
|||
|
|
),
|
|||
|
|
);
|
|||
|
|
}
|
|||
|
|
if (allowEmpty && !text.trim()) {
|
|||
|
|
return { response, envelope: null };
|
|||
|
|
}
|
|||
|
|
const envelope = parseJsonRpc(text);
|
|||
|
|
if (envelope.error) {
|
|||
|
|
const message = safeMessage(
|
|||
|
|
envelope.error.message,
|
|||
|
|
"MCP采集服务调用失败",
|
|||
|
|
);
|
|||
|
|
if (/session not found|missing session id/i.test(message)) {
|
|||
|
|
throw new McpSessionLostError("MCP采集会话已失效");
|
|||
|
|
}
|
|||
|
|
throw new Error(message);
|
|||
|
|
}
|
|||
|
|
return { response, envelope };
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
async function createMcpSession(
|
|||
|
|
fetchImpl: typeof fetch,
|
|||
|
|
endpoint: string,
|
|||
|
|
timeoutMs: number,
|
|||
|
|
) {
|
|||
|
|
const initialize = await postMcp(
|
|||
|
|
fetchImpl,
|
|||
|
|
endpoint,
|
|||
|
|
{
|
|||
|
|
jsonrpc: "2.0",
|
|||
|
|
id: crypto.randomUUID(),
|
|||
|
|
method: "initialize",
|
|||
|
|
params: {
|
|||
|
|
protocolVersion: "2025-03-26",
|
|||
|
|
capabilities: {},
|
|||
|
|
clientInfo: {
|
|||
|
|
name: "koc-loop-collector",
|
|||
|
|
version: "1.0.0",
|
|||
|
|
},
|
|||
|
|
},
|
|||
|
|
},
|
|||
|
|
timeoutMs,
|
|||
|
|
);
|
|||
|
|
const sessionId = initialize.response.headers.get("mcp-session-id");
|
|||
|
|
if (!sessionId) throw new Error("MCP采集服务未返回会话标识");
|
|||
|
|
|
|||
|
|
await postMcp(
|
|||
|
|
fetchImpl,
|
|||
|
|
endpoint,
|
|||
|
|
{
|
|||
|
|
jsonrpc: "2.0",
|
|||
|
|
method: "notifications/initialized",
|
|||
|
|
params: {},
|
|||
|
|
},
|
|||
|
|
timeoutMs,
|
|||
|
|
sessionId,
|
|||
|
|
true,
|
|||
|
|
);
|
|||
|
|
return sessionId;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
async function callMcpTool(
|
|||
|
|
fetchImpl: typeof fetch,
|
|||
|
|
endpoint: string,
|
|||
|
|
sessionId: string,
|
|||
|
|
timeoutMs: number,
|
|||
|
|
name: string,
|
|||
|
|
args: Record<string, unknown>,
|
|||
|
|
): Promise<ToolResult> {
|
|||
|
|
const result = await postMcp(
|
|||
|
|
fetchImpl,
|
|||
|
|
endpoint,
|
|||
|
|
{
|
|||
|
|
jsonrpc: "2.0",
|
|||
|
|
id: crypto.randomUUID(),
|
|||
|
|
method: "tools/call",
|
|||
|
|
params: {
|
|||
|
|
name,
|
|||
|
|
arguments: args,
|
|||
|
|
},
|
|||
|
|
},
|
|||
|
|
timeoutMs,
|
|||
|
|
sessionId,
|
|||
|
|
);
|
|||
|
|
const content = result.envelope?.result?.content ?? [];
|
|||
|
|
const text = content.find((block) => block.type === "text")?.text;
|
|||
|
|
if (!text) throw new Error("MCP采集工具未返回数据");
|
|||
|
|
let payload: unknown;
|
|||
|
|
try {
|
|||
|
|
payload = JSON.parse(text);
|
|||
|
|
} catch {
|
|||
|
|
throw new Error("MCP采集工具返回了无法解析的数据");
|
|||
|
|
}
|
|||
|
|
return {
|
|||
|
|
isError: result.envelope?.result?.isError === true,
|
|||
|
|
payload,
|
|||
|
|
};
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function metricsFromToolResult(result: ToolResult): XhsPublicMetrics {
|
|||
|
|
const root = asRecord(result.payload);
|
|||
|
|
const response = asRecord(root?.response) ?? root;
|
|||
|
|
const data = asRecord(response?.data) ?? asRecord(root?.data);
|
|||
|
|
const code = Number(response?.code);
|
|||
|
|
const success = response?.success;
|
|||
|
|
|
|||
|
|
if (
|
|||
|
|
result.isError ||
|
|||
|
|
success === false ||
|
|||
|
|
(Number.isFinite(code) && code >= 400)
|
|||
|
|
) {
|
|||
|
|
throw new Error(
|
|||
|
|
safeMessage(
|
|||
|
|
response?.msg ?? response?.message ?? root?.message,
|
|||
|
|
"公开数据采集失败",
|
|||
|
|
),
|
|||
|
|
);
|
|||
|
|
}
|
|||
|
|
if (!data) throw new Error("采集结果缺少互动数据");
|
|||
|
|
|
|||
|
|
return {
|
|||
|
|
likes: metricValue(data.likes, "点赞数"),
|
|||
|
|
comments: metricValue(data.comments, "评论数"),
|
|||
|
|
collects: metricValue(
|
|||
|
|
data.collects ?? data.favorites ?? data.favourites,
|
|||
|
|
"收藏数",
|
|||
|
|
),
|
|||
|
|
};
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function successfulToolData(result: ToolResult, fallbackMessage: string) {
|
|||
|
|
const root = asRecord(result.payload);
|
|||
|
|
const response = asRecord(root?.response) ?? root;
|
|||
|
|
const data = asRecord(response?.data) ?? asRecord(root?.data);
|
|||
|
|
const code = Number(response?.code);
|
|||
|
|
if (
|
|||
|
|
result.isError ||
|
|||
|
|
response?.success === false ||
|
|||
|
|
(Number.isFinite(code) && code >= 400)
|
|||
|
|
) {
|
|||
|
|
throw new Error(
|
|||
|
|
safeMessage(
|
|||
|
|
response?.msg ?? response?.message ?? root?.message,
|
|||
|
|
fallbackMessage,
|
|||
|
|
),
|
|||
|
|
);
|
|||
|
|
}
|
|||
|
|
if (!data) throw new Error(fallbackMessage);
|
|||
|
|
return data;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function stringValue(value: unknown) {
|
|||
|
|
return typeof value === "string" ? value.trim() : "";
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function findRecord(
|
|||
|
|
value: unknown,
|
|||
|
|
predicate: (record: Record<string, unknown>) => boolean,
|
|||
|
|
depth = 0,
|
|||
|
|
): Record<string, unknown> | null {
|
|||
|
|
if (depth > 12) return null;
|
|||
|
|
if (Array.isArray(value)) {
|
|||
|
|
for (const item of value) {
|
|||
|
|
const found = findRecord(item, predicate, depth + 1);
|
|||
|
|
if (found) return found;
|
|||
|
|
}
|
|||
|
|
return null;
|
|||
|
|
}
|
|||
|
|
const record = asRecord(value);
|
|||
|
|
if (!record) return null;
|
|||
|
|
if (predicate(record)) return record;
|
|||
|
|
for (const child of Object.values(record)) {
|
|||
|
|
const found = findRecord(child, predicate, depth + 1);
|
|||
|
|
if (found) return found;
|
|||
|
|
}
|
|||
|
|
return null;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function findStringByKey(value: unknown, key: string, depth = 0): string {
|
|||
|
|
if (depth > 12) return "";
|
|||
|
|
if (Array.isArray(value)) {
|
|||
|
|
for (const item of value) {
|
|||
|
|
const found = findStringByKey(item, key, depth + 1);
|
|||
|
|
if (found) return found;
|
|||
|
|
}
|
|||
|
|
return "";
|
|||
|
|
}
|
|||
|
|
const record = asRecord(value);
|
|||
|
|
if (!record) return "";
|
|||
|
|
const direct = stringValue(record[key]);
|
|||
|
|
if (direct) return direct;
|
|||
|
|
for (const child of Object.values(record)) {
|
|||
|
|
const found = findStringByKey(child, key, depth + 1);
|
|||
|
|
if (found) return found;
|
|||
|
|
}
|
|||
|
|
return "";
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function findValueByKeys(
|
|||
|
|
value: unknown,
|
|||
|
|
keys: ReadonlySet<string>,
|
|||
|
|
depth = 0,
|
|||
|
|
): unknown {
|
|||
|
|
if (depth > 12) return undefined;
|
|||
|
|
if (Array.isArray(value)) {
|
|||
|
|
for (const item of value) {
|
|||
|
|
const found = findValueByKeys(item, keys, depth + 1);
|
|||
|
|
if (found !== undefined) return found;
|
|||
|
|
}
|
|||
|
|
return undefined;
|
|||
|
|
}
|
|||
|
|
const record = asRecord(value);
|
|||
|
|
if (!record) return undefined;
|
|||
|
|
for (const [key, child] of Object.entries(record)) {
|
|||
|
|
if (keys.has(key) && child !== null && child !== undefined && child !== "") {
|
|||
|
|
return child;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
for (const child of Object.values(record)) {
|
|||
|
|
const found = findValueByKeys(child, keys, depth + 1);
|
|||
|
|
if (found !== undefined) return found;
|
|||
|
|
}
|
|||
|
|
return undefined;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
const FOLLOWER_KEYS = new Set([
|
|||
|
|
"fans",
|
|||
|
|
"fans_count",
|
|||
|
|
"fansCount",
|
|||
|
|
"follower",
|
|||
|
|
"followers",
|
|||
|
|
"follower_count",
|
|||
|
|
"followerCount",
|
|||
|
|
]);
|
|||
|
|
|
|||
|
|
function followerCountFromPayload(value: unknown, depth = 0): number | null {
|
|||
|
|
const direct = optionalCountValue(findValueByKeys(value, FOLLOWER_KEYS));
|
|||
|
|
if (direct !== null) return direct;
|
|||
|
|
if (typeof value === "string") {
|
|||
|
|
return optionalCountValue(
|
|||
|
|
value.match(/(?:粉丝数?|followers?|fans)\s*[：:]\s*([\d,.万千wkWＫＷ+]+)/i)?.[1],
|
|||
|
|
);
|
|||
|
|
}
|
|||
|
|
if (depth > 12) return null;
|
|||
|
|
if (Array.isArray(value)) {
|
|||
|
|
for (const item of value) {
|
|||
|
|
const found = followerCountFromPayload(item, depth + 1);
|
|||
|
|
if (found !== null) return found;
|
|||
|
|
}
|
|||
|
|
return null;
|
|||
|
|
}
|
|||
|
|
const record = asRecord(value);
|
|||
|
|
if (!record) return null;
|
|||
|
|
const label = stringValue(
|
|||
|
|
record.name ?? record.label ?? record.title ?? record.type,
|
|||
|
|
).toLowerCase();
|
|||
|
|
if (["粉丝", "粉丝数", "fans", "followers"].includes(label)) {
|
|||
|
|
const labeled = optionalCountValue(
|
|||
|
|
record.count ?? record.value ?? record.i18nCount,
|
|||
|
|
);
|
|||
|
|
if (labeled !== null) return labeled;
|
|||
|
|
}
|
|||
|
|
for (const child of Object.values(record)) {
|
|||
|
|
const found = followerCountFromPayload(child, depth + 1);
|
|||
|
|
if (found !== null) return found;
|
|||
|
|
}
|
|||
|
|
return null;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
export function xhsNoteIdFromUrl(input: string) {
|
|||
|
|
try {
|
|||
|
|
const url = new URL(input);
|
|||
|
|
if (
|
|||
|
|
!(
|
|||
|
|
url.hostname === "xiaohongshu.com" ||
|
|||
|
|
url.hostname.endsWith(".xiaohongshu.com")
|
|||
|
|
)
|
|||
|
|
) {
|
|||
|
|
return "";
|
|||
|
|
}
|
|||
|
|
return (
|
|||
|
|
url.pathname.match(
|
|||
|
|
/\/(?:discovery\/item|explore)\/([A-Za-z0-9_-]{8,80})/,
|
|||
|
|
)?.[1] ?? ""
|
|||
|
|
);
|
|||
|
|
} catch {
|
|||
|
|
return "";
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
async function xhsNoteIdFromShortLink(
|
|||
|
|
input: string,
|
|||
|
|
fallbackNickname: string,
|
|||
|
|
fetchImpl: typeof fetch,
|
|||
|
|
timeoutMs: number,
|
|||
|
|
) {
|
|||
|
|
let url: URL;
|
|||
|
|
try {
|
|||
|
|
url = new URL(input);
|
|||
|
|
} catch {
|
|||
|
|
return { noteId: "", profile: null };
|
|||
|
|
}
|
|||
|
|
if (
|
|||
|
|
url.protocol !== "http:" &&
|
|||
|
|
url.protocol !== "https:"
|
|||
|
|
) {
|
|||
|
|
return { noteId: "", profile: null };
|
|||
|
|
}
|
|||
|
|
if (
|
|||
|
|
url.hostname !== "xhslink.cn" &&
|
|||
|
|
!url.hostname.endsWith(".xhslink.cn")
|
|||
|
|
) {
|
|||
|
|
return { noteId: "", profile: null };
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
const controller = new AbortController();
|
|||
|
|
const timeout = setTimeout(() => controller.abort(), timeoutMs);
|
|||
|
|
try {
|
|||
|
|
const response = await fetchImpl(url.toString(), {
|
|||
|
|
method: "GET",
|
|||
|
|
redirect: "follow",
|
|||
|
|
signal: controller.signal,
|
|||
|
|
headers: {
|
|||
|
|
"user-agent":
|
|||
|
|
"Mozilla/5.0 (iPhone; CPU iPhone OS 18_0 like Mac OS X) AppleWebKit/605.1.15 Mobile/15E148",
|
|||
|
|
},
|
|||
|
|
});
|
|||
|
|
const location = response.headers.get("location");
|
|||
|
|
const resolvedUrl = location
|
|||
|
|
? new URL(location, url).toString()
|
|||
|
|
: response.url;
|
|||
|
|
const noteId = xhsNoteIdFromUrl(resolvedUrl);
|
|||
|
|
const html =
|
|||
|
|
response.ok &&
|
|||
|
|
response.headers.get("content-type")?.includes("text/html")
|
|||
|
|
? await response.text()
|
|||
|
|
: "";
|
|||
|
|
return {
|
|||
|
|
noteId,
|
|||
|
|
profile: accountProfileFromPublicPage(
|
|||
|
|
html,
|
|||
|
|
fallbackNickname,
|
|||
|
|
),
|
|||
|
|
};
|
|||
|
|
} catch {
|
|||
|
|
return { noteId: "", profile: null };
|
|||
|
|
} finally {
|
|||
|
|
clearTimeout(timeout);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function accountProfileFromPublicPage(
|
|||
|
|
html: string,
|
|||
|
|
fallbackNickname: string,
|
|||
|
|
): XhsAccountProfile | null {
|
|||
|
|
if (!html) return null;
|
|||
|
|
const decoded = html
|
|||
|
|
.replaceAll(""", '"')
|
|||
|
|
.replaceAll(""", '"')
|
|||
|
|
.replaceAll("&", "&")
|
|||
|
|
.replaceAll("'", "'")
|
|||
|
|
.replaceAll("<", "<")
|
|||
|
|
.replaceAll(">", ">");
|
|||
|
|
const noteData = decoded.match(
|
|||
|
|
/"noteData":\{[\s\S]{0,30000}?"user":\{([\s\S]{0,1800}?)\}/,
|
|||
|
|
)?.[1];
|
|||
|
|
if (!noteData) return null;
|
|||
|
|
const platformUid =
|
|||
|
|
noteData.match(/"userId":"([A-Za-z0-9_-]{8,80})"/)?.[1] ?? "";
|
|||
|
|
if (!platformUid) return null;
|
|||
|
|
const nickname =
|
|||
|
|
noteData.match(/"(?:nickName|nickname)":"([^"]+)"/)?.[1] ??
|
|||
|
|
fallbackNickname.trim();
|
|||
|
|
const redId =
|
|||
|
|
noteData.match(/"(?:redId|red_id)":"([^"]+)"/)?.[1] ??
|
|||
|
|
decoded.match(/"(?:redId|red_id)":"([^"]+)"/)?.[1] ??
|
|||
|
|
decoded.match(/小红书号[：:]\s*([^<"\s]+)/)?.[1] ??
|
|||
|
|
"";
|
|||
|
|
return {
|
|||
|
|
platformUid,
|
|||
|
|
nickname,
|
|||
|
|
profileUrl: `https://www.xiaohongshu.com/user/profile/${encodeURIComponent(
|
|||
|
|
platformUid,
|
|||
|
|
)}`,
|
|||
|
|
redId,
|
|||
|
|
ipLocation: "待识别",
|
|||
|
|
followers: null,
|
|||
|
|
};
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
export async function resolveXhsPublicAccountDetails(
|
|||
|
|
profileUrl: string,
|
|||
|
|
fetchImpl: typeof fetch = fetch,
|
|||
|
|
) {
|
|||
|
|
let parsed: URL;
|
|||
|
|
try {
|
|||
|
|
parsed = new URL(profileUrl);
|
|||
|
|
} catch {
|
|||
|
|
return { redId: "", followers: null, ipLocation: "" };
|
|||
|
|
}
|
|||
|
|
if (
|
|||
|
|
parsed.protocol !== "https:" ||
|
|||
|
|
!(
|
|||
|
|
parsed.hostname === "xiaohongshu.com" ||
|
|||
|
|
parsed.hostname.endsWith(".xiaohongshu.com")
|
|||
|
|
) ||
|
|||
|
|
!parsed.pathname.startsWith("/user/profile/")
|
|||
|
|
) {
|
|||
|
|
return { redId: "", followers: null, ipLocation: "" };
|
|||
|
|
}
|
|||
|
|
try {
|
|||
|
|
const response = await fetchImpl(parsed.toString(), {
|
|||
|
|
method: "GET",
|
|||
|
|
redirect: "follow",
|
|||
|
|
signal: AbortSignal.timeout(15_000),
|
|||
|
|
headers: {
|
|||
|
|
"user-agent":
|
|||
|
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/138.0 Safari/537.36",
|
|||
|
|
},
|
|||
|
|
});
|
|||
|
|
if (!response.ok) {
|
|||
|
|
return { redId: "", followers: null, ipLocation: "" };
|
|||
|
|
}
|
|||
|
|
const html = await response.text();
|
|||
|
|
return {
|
|||
|
|
redId:
|
|||
|
|
html.match(/"(?:redId|red_id)":"([^"]+)"/)?.[1] ??
|
|||
|
|
html.match(/小红书号[：:]\s*([^<"\s]+)/)?.[1] ??
|
|||
|
|
"",
|
|||
|
|
followers:
|
|||
|
|
optionalCountValue(
|
|||
|
|
html.match(
|
|||
|
|
/"name"\s*:\s*"粉丝"\s*,\s*"count"\s*:\s*"([^"]+)"/,
|
|||
|
|
)?.[1],
|
|||
|
|
) ??
|
|||
|
|
followerCountFromPayload(html),
|
|||
|
|
ipLocation: "",
|
|||
|
|
};
|
|||
|
|
} catch {
|
|||
|
|
return { redId: "", followers: null, ipLocation: "" };
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
export async function resolveXhsPublicAccountId(
|
|||
|
|
profileUrl: string,
|
|||
|
|
fetchImpl: typeof fetch = fetch,
|
|||
|
|
) {
|
|||
|
|
return (await resolveXhsPublicAccountDetails(profileUrl, fetchImpl)).redId;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function profileDetailsFromToolResult(result: ToolResult) {
|
|||
|
|
const root = asRecord(result.payload);
|
|||
|
|
const response = asRecord(root?.response) ?? root;
|
|||
|
|
const code = Number(response?.code);
|
|||
|
|
if (
|
|||
|
|
result.isError ||
|
|||
|
|
response?.success === false ||
|
|||
|
|
(Number.isFinite(code) && code >= 400)
|
|||
|
|
) {
|
|||
|
|
throw new Error(
|
|||
|
|
safeMessage(
|
|||
|
|
response?.msg ?? response?.message ?? root?.message,
|
|||
|
|
"账号主页数据采集失败",
|
|||
|
|
),
|
|||
|
|
);
|
|||
|
|
}
|
|||
|
|
const payload =
|
|||
|
|
asRecord(response?.data) ?? asRecord(root?.data) ?? result.payload;
|
|||
|
|
const followers = followerCountFromPayload(payload);
|
|||
|
|
const redId =
|
|||
|
|
findStringByKey(payload, "red_id") ||
|
|||
|
|
findStringByKey(payload, "redId") ||
|
|||
|
|
findStringByKey(payload, "userId") ||
|
|||
|
|
findStringByKey(payload, "user_id");
|
|||
|
|
const ipLocation =
|
|||
|
|
findStringByKey(payload, "ip_location") ||
|
|||
|
|
findStringByKey(payload, "ipLocation");
|
|||
|
|
if (followers === null && !redId && !ipLocation) {
|
|||
|
|
throw new Error("账号主页采集结果缺少可用字段");
|
|||
|
|
}
|
|||
|
|
return { followers, redId, ipLocation };
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function accountProfileFromToolResult(
|
|||
|
|
result: ToolResult,
|
|||
|
|
fallbackNickname: string,
|
|||
|
|
): XhsAccountProfile {
|
|||
|
|
const data = successfulToolData(result, "账号主页识别失败");
|
|||
|
|
const user = findRecord(
|
|||
|
|
data,
|
|||
|
|
(record) =>
|
|||
|
|
Boolean(
|
|||
|
|
stringValue(record.user_id ?? record.userid) &&
|
|||
|
|
(stringValue(record.profile_url) ||
|
|||
|
|
stringValue(record.nickname ?? record.name)),
|
|||
|
|
),
|
|||
|
|
);
|
|||
|
|
if (!user) throw new Error("账号主页识别结果缺少作者信息");
|
|||
|
|
const platformUid = stringValue(user.user_id ?? user.userid);
|
|||
|
|
const candidateProfileUrl = stringValue(user.profile_url);
|
|||
|
|
let profileUrl = "";
|
|||
|
|
if (candidateProfileUrl) {
|
|||
|
|
try {
|
|||
|
|
const parsed = new URL(candidateProfileUrl);
|
|||
|
|
if (
|
|||
|
|
parsed.protocol === "https:" &&
|
|||
|
|
(parsed.hostname === "xiaohongshu.com" ||
|
|||
|
|
parsed.hostname.endsWith(".xiaohongshu.com")) &&
|
|||
|
|
parsed.pathname.startsWith("/user/profile/")
|
|||
|
|
) {
|
|||
|
|
profileUrl = parsed.toString();
|
|||
|
|
}
|
|||
|
|
} catch {
|
|||
|
|
// Fall back to the verified user id below.
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
if (!profileUrl) {
|
|||
|
|
profileUrl = `https://www.xiaohongshu.com/user/profile/${encodeURIComponent(
|
|||
|
|
platformUid,
|
|||
|
|
)}`;
|
|||
|
|
}
|
|||
|
|
return {
|
|||
|
|
platformUid,
|
|||
|
|
nickname:
|
|||
|
|
stringValue(user.nickname ?? user.name) || fallbackNickname.trim(),
|
|||
|
|
profileUrl,
|
|||
|
|
redId: stringValue(user.red_id),
|
|||
|
|
ipLocation: findStringByKey(data, "ip_location") || "待识别",
|
|||
|
|
followers: followerCountFromPayload(data),
|
|||
|
|
};
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
async function completeAccountProfile(
|
|||
|
|
profile: XhsAccountProfile,
|
|||
|
|
fetchImpl: typeof fetch,
|
|||
|
|
endpoint: string,
|
|||
|
|
sessionId: string,
|
|||
|
|
timeoutMs: number,
|
|||
|
|
) {
|
|||
|
|
let completed = profile;
|
|||
|
|
for (const [name, args] of [
|
|||
|
|
[
|
|||
|
|
"parse_xhs_user_summary",
|
|||
|
|
{ url: profile.profileUrl, use_proxy: true },
|
|||
|
|
],
|
|||
|
|
[
|
|||
|
|
"fetch_user_detail",
|
|||
|
|
{ link: profile.profileUrl, plant: "xhs" },
|
|||
|
|
],
|
|||
|
|
] as const) {
|
|||
|
|
if (
|
|||
|
|
completed.followers !== null &&
|
|||
|
|
completed.redId &&
|
|||
|
|
completed.ipLocation !== "待识别"
|
|||
|
|
) {
|
|||
|
|
break;
|
|||
|
|
}
|
|||
|
|
try {
|
|||
|
|
const result = await callMcpTool(
|
|||
|
|
fetchImpl,
|
|||
|
|
endpoint,
|
|||
|
|
sessionId,
|
|||
|
|
timeoutMs,
|
|||
|
|
name,
|
|||
|
|
args,
|
|||
|
|
);
|
|||
|
|
const details = profileDetailsFromToolResult(result);
|
|||
|
|
completed = {
|
|||
|
|
...completed,
|
|||
|
|
followers: details.followers ?? completed.followers,
|
|||
|
|
redId: details.redId || completed.redId,
|
|||
|
|
ipLocation:
|
|||
|
|
details.ipLocation && details.ipLocation !== "待识别"
|
|||
|
|
? details.ipLocation
|
|||
|
|
: completed.ipLocation,
|
|||
|
|
};
|
|||
|
|
} catch (error) {
|
|||
|
|
if (error instanceof McpSessionLostError) throw error;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if (completed.followers === null || !completed.redId) {
|
|||
|
|
const details = await resolveXhsPublicAccountDetails(
|
|||
|
|
completed.profileUrl,
|
|||
|
|
fetchImpl,
|
|||
|
|
);
|
|||
|
|
completed = {
|
|||
|
|
...completed,
|
|||
|
|
followers: details.followers ?? completed.followers,
|
|||
|
|
redId: details.redId || completed.redId,
|
|||
|
|
};
|
|||
|
|
}
|
|||
|
|
return completed;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
async function resolveProfileDetailsInSession(
|
|||
|
|
profileUrl: string,
|
|||
|
|
config: CollectionMcpConfig,
|
|||
|
|
fetchImpl: typeof fetch,
|
|||
|
|
) {
|
|||
|
|
const endpoint = buildMcpUrl(config);
|
|||
|
|
const timeoutMs = Math.max(5_000, config.timeoutMs ?? 30_000);
|
|||
|
|
const sessionId = await createMcpSession(fetchImpl, endpoint, timeoutMs);
|
|||
|
|
const result = await callMcpTool(
|
|||
|
|
fetchImpl,
|
|||
|
|
endpoint,
|
|||
|
|
sessionId,
|
|||
|
|
timeoutMs,
|
|||
|
|
"parse_xhs_user_summary",
|
|||
|
|
{
|
|||
|
|
url: profileUrl,
|
|||
|
|
use_proxy: true,
|
|||
|
|
},
|
|||
|
|
);
|
|||
|
|
return profileDetailsFromToolResult(result);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
async function resolveAccountInSession(
|
|||
|
|
publishUrl: string,
|
|||
|
|
fallbackNickname: string,
|
|||
|
|
config: CollectionMcpConfig,
|
|||
|
|
fetchImpl: typeof fetch,
|
|||
|
|
) {
|
|||
|
|
const endpoint = buildMcpUrl(config);
|
|||
|
|
const timeoutMs = Math.max(5_000, config.timeoutMs ?? 30_000);
|
|||
|
|
let noteId = xhsNoteIdFromUrl(publishUrl);
|
|||
|
|
let publicPageProfile: XhsAccountProfile | null = null;
|
|||
|
|
if (!noteId) {
|
|||
|
|
const shortLink = await xhsNoteIdFromShortLink(
|
|||
|
|
publishUrl,
|
|||
|
|
fallbackNickname,
|
|||
|
|
fetchImpl,
|
|||
|
|
timeoutMs,
|
|||
|
|
);
|
|||
|
|
noteId = shortLink.noteId;
|
|||
|
|
publicPageProfile = shortLink.profile;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
try {
|
|||
|
|
const sessionId = await createMcpSession(
|
|||
|
|
fetchImpl,
|
|||
|
|
endpoint,
|
|||
|
|
timeoutMs,
|
|||
|
|
);
|
|||
|
|
if (!noteId) {
|
|||
|
|
const noteResult = await callMcpTool(
|
|||
|
|
fetchImpl,
|
|||
|
|
endpoint,
|
|||
|
|
sessionId,
|
|||
|
|
timeoutMs,
|
|||
|
|
"fetch_content_detail",
|
|||
|
|
{
|
|||
|
|
link: publishUrl,
|
|||
|
|
plant: "xhs",
|
|||
|
|
include_comments: false,
|
|||
|
|
auto_cookie: true,
|
|||
|
|
},
|
|||
|
|
);
|
|||
|
|
const noteData = successfulToolData(
|
|||
|
|
noteResult,
|
|||
|
|
"无法识别小红书笔记",
|
|||
|
|
);
|
|||
|
|
noteId =
|
|||
|
|
stringValue(noteData.noteId ?? noteData.note_id) ||
|
|||
|
|
findStringByKey(noteData, "noteId") ||
|
|||
|
|
findStringByKey(noteData, "note_id");
|
|||
|
|
}
|
|||
|
|
if (!noteId) throw new Error("无法从发布链接识别小红书笔记ID");
|
|||
|
|
|
|||
|
|
const authorResult = await callMcpTool(
|
|||
|
|
fetchImpl,
|
|||
|
|
endpoint,
|
|||
|
|
sessionId,
|
|||
|
|
timeoutMs,
|
|||
|
|
"collect_xhs_wen_note_detail",
|
|||
|
|
{
|
|||
|
|
note_id: noteId,
|
|||
|
|
need_desc: false,
|
|||
|
|
include_raw: false,
|
|||
|
|
},
|
|||
|
|
);
|
|||
|
|
const profile = accountProfileFromToolResult(
|
|||
|
|
authorResult,
|
|||
|
|
fallbackNickname,
|
|||
|
|
);
|
|||
|
|
return completeAccountProfile(
|
|||
|
|
profile,
|
|||
|
|
fetchImpl,
|
|||
|
|
endpoint,
|
|||
|
|
sessionId,
|
|||
|
|
timeoutMs,
|
|||
|
|
);
|
|||
|
|
} catch (error) {
|
|||
|
|
if (error instanceof McpSessionLostError) throw error;
|
|||
|
|
if (publicPageProfile) {
|
|||
|
|
const details = await resolveXhsPublicAccountDetails(
|
|||
|
|
publicPageProfile.profileUrl,
|
|||
|
|
fetchImpl,
|
|||
|
|
);
|
|||
|
|
return {
|
|||
|
|
...publicPageProfile,
|
|||
|
|
redId: details.redId || publicPageProfile.redId,
|
|||
|
|
followers: details.followers,
|
|||
|
|
};
|
|||
|
|
}
|
|||
|
|
throw error;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
async function collectInSession(
|
|||
|
|
publishUrl: string,
|
|||
|
|
config: CollectionMcpConfig,
|
|||
|
|
fetchImpl: typeof fetch,
|
|||
|
|
) {
|
|||
|
|
const endpoint = buildMcpUrl(config);
|
|||
|
|
const timeoutMs = Math.max(5_000, config.timeoutMs ?? 30_000);
|
|||
|
|
const sessionId = await createMcpSession(fetchImpl, endpoint, timeoutMs);
|
|||
|
|
|
|||
|
|
const primary = await callMcpTool(
|
|||
|
|
fetchImpl,
|
|||
|
|
endpoint,
|
|||
|
|
sessionId,
|
|||
|
|
timeoutMs,
|
|||
|
|
"fetch_content_detail",
|
|||
|
|
{
|
|||
|
|
link: publishUrl,
|
|||
|
|
plant: "xhs",
|
|||
|
|
include_comments: false,
|
|||
|
|
auto_cookie: true,
|
|||
|
|
},
|
|||
|
|
);
|
|||
|
|
try {
|
|||
|
|
return metricsFromToolResult(primary);
|
|||
|
|
} catch {
|
|||
|
|
const fallback = await callMcpTool(
|
|||
|
|
fetchImpl,
|
|||
|
|
endpoint,
|
|||
|
|
sessionId,
|
|||
|
|
timeoutMs,
|
|||
|
|
"parse_xhs_note",
|
|||
|
|
{
|
|||
|
|
url: publishUrl,
|
|||
|
|
include_comments: false,
|
|||
|
|
auto_cookie: true,
|
|||
|
|
},
|
|||
|
|
);
|
|||
|
|
return metricsFromToolResult(fallback);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
export function resolveCollectionMcpConfig(
|
|||
|
|
bindings: CollectionMcpBindings,
|
|||
|
|
): CollectionMcpConfig {
|
|||
|
|
return {
|
|||
|
|
endpoint:
|
|||
|
|
bindings.AI_TOOL_CENTER_MCP_URL?.trim() ||
|
|||
|
|
DEFAULT_COLLECTION_MCP_URL,
|
|||
|
|
key: bindings.AI_TOOL_CENTER_MCP_KEY?.trim(),
|
|||
|
|
};
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
export async function collectXhsMetricsFromMcp(
|
|||
|
|
publishUrl: string,
|
|||
|
|
config: CollectionMcpConfig,
|
|||
|
|
fetchImpl: typeof fetch = fetch,
|
|||
|
|
) {
|
|||
|
|
let parsed: URL;
|
|||
|
|
try {
|
|||
|
|
parsed = new URL(publishUrl);
|
|||
|
|
} catch {
|
|||
|
|
throw new Error("发布链接无效");
|
|||
|
|
}
|
|||
|
|
if (!["http:", "https:"].includes(parsed.protocol)) {
|
|||
|
|
throw new Error("发布链接无效");
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
let lastError: unknown;
|
|||
|
|
for (let attempt = 0; attempt < MAX_MCP_ATTEMPTS; attempt += 1) {
|
|||
|
|
try {
|
|||
|
|
return await collectInSession(parsed.toString(), config, fetchImpl);
|
|||
|
|
} catch (error) {
|
|||
|
|
lastError = error;
|
|||
|
|
if (!isRetryableTransportError(error)) throw error;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
if (
|
|||
|
|
lastError instanceof Error &&
|
|||
|
|
["AbortError", "TimeoutError"].includes(lastError.name)
|
|||
|
|
) {
|
|||
|
|
throw new Error("MCP采集服务响应超时，请稍后重试");
|
|||
|
|
}
|
|||
|
|
throw lastError instanceof Error
|
|||
|
|
? lastError
|
|||
|
|
: new Error("MCP采集服务暂时不可用");
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
export async function resolveXhsAccountProfileFromMcp(
|
|||
|
|
publishUrl: string,
|
|||
|
|
fallbackNickname: string,
|
|||
|
|
config: CollectionMcpConfig,
|
|||
|
|
fetchImpl: typeof fetch = fetch,
|
|||
|
|
) {
|
|||
|
|
let parsed: URL;
|
|||
|
|
try {
|
|||
|
|
parsed = new URL(publishUrl);
|
|||
|
|
} catch {
|
|||
|
|
throw new Error("发布链接无效");
|
|||
|
|
}
|
|||
|
|
if (!["http:", "https:"].includes(parsed.protocol)) {
|
|||
|
|
throw new Error("发布链接无效");
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
let lastError: unknown;
|
|||
|
|
for (let attempt = 0; attempt < MAX_MCP_ATTEMPTS; attempt += 1) {
|
|||
|
|
try {
|
|||
|
|
return await resolveAccountInSession(
|
|||
|
|
parsed.toString(),
|
|||
|
|
fallbackNickname,
|
|||
|
|
config,
|
|||
|
|
fetchImpl,
|
|||
|
|
);
|
|||
|
|
} catch (error) {
|
|||
|
|
lastError = error;
|
|||
|
|
if (!isRetryableTransportError(error)) throw error;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
throw lastError instanceof Error
|
|||
|
|
? lastError
|
|||
|
|
: new Error("账号主页识别服务暂时不可用");
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
export async function resolveXhsProfileDetailsFromMcp(
|
|||
|
|
profileUrl: string,
|
|||
|
|
config: CollectionMcpConfig,
|
|||
|
|
fetchImpl: typeof fetch = fetch,
|
|||
|
|
) {
|
|||
|
|
let parsed: URL;
|
|||
|
|
try {
|
|||
|
|
parsed = new URL(profileUrl);
|
|||
|
|
} catch {
|
|||
|
|
throw new Error("账号主页链接无效");
|
|||
|
|
}
|
|||
|
|
if (
|
|||
|
|
parsed.protocol !== "https:" ||
|
|||
|
|
!(
|
|||
|
|
parsed.hostname === "xiaohongshu.com" ||
|
|||
|
|
parsed.hostname.endsWith(".xiaohongshu.com")
|
|||
|
|
) ||
|
|||
|
|
!parsed.pathname.startsWith("/user/profile/")
|
|||
|
|
) {
|
|||
|
|
throw new Error("账号主页链接无效");
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
let lastError: unknown;
|
|||
|
|
for (let attempt = 0; attempt < MAX_MCP_ATTEMPTS; attempt += 1) {
|
|||
|
|
try {
|
|||
|
|
return await resolveProfileDetailsInSession(
|
|||
|
|
parsed.toString(),
|
|||
|
|
config,
|
|||
|
|
fetchImpl,
|
|||
|
|
);
|
|||
|
|
} catch (error) {
|
|||
|
|
lastError = error;
|
|||
|
|
if (!isRetryableTransportError(error)) throw error;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
throw lastError instanceof Error
|
|||
|
|
? lastError
|
|||
|
|
: new Error("账号主页数据采集服务暂时不可用");
|
|||
|
|
}
|