wufengping/koc-loop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 完善视频任务与 KOC 资源库

Browse Source
This commit is contained in:
巫凤萍
2026-08-15 03:53:09 +08:00
parent ad3dbdcc86
commit f37d05dd88
66 changed files with 6633 additions and 558 deletions
Download Patch File Download Diff File Expand all files Collapse all files

562
lib/mcp-collection-client.ts
View File

@@ -17,6 +17,7 @@ export type XhsPublicMetrics = {
likes: number;
comments: number;
collects: number;
shares: number;
};
export type XhsAccountProfile = {
@@ -26,6 +27,10 @@ export type XhsAccountProfile = {
redId: string;
ipLocation: string;
followers: number | null;
gender: "" | "男" | "女";
bio: string;
recentNoteTitles: string[];
providerTags: string[];
};
type JsonRpcEnvelope = {
@@ -341,16 +346,118 @@ function metricsFromToolResult(result: ToolResult): XhsPublicMetrics {
}
if (!data) throw new Error("采集结果缺少互动数据");
const count = (value: unknown, label: string) =>
value === null || value === undefined || value === ""
? 0
: metricValue(value, label);
return {
likes: metricValue(data.likes, "点赞数"),
comments: metricValue(data.comments, "评论数"),
collects: metricValue(
data.collects ?? data.favorites ?? data.favourites,
likes: count(
data.likes ??
data.liked_count ??
data.likedCount ??
data.like_count ??
data.likeCount ??
data.digg_count ??
data.diggCount,
"点赞数",
),
comments: count(
data.comments ?? data.comment_count ?? data.commentCount,
"评论数",
),
collects: count(
data.collects ??
data.collected_count ??
data.collectedCount ??
data.favorites ??
data.favourites ??
data.collect_count ??
data.collectCount,
"收藏数",
),
shares: count(
data.shares ??
data.share_count ??
data.shareCount ??
data.forwards ??
data.forward_count ??
data.forwardCount,
"转发数",
),
};
}
function usableDouyinSecUid(value: unknown) {
const candidate = stringValue(value);
return candidate && !/^\d+$/.test(candidate) && /^[A-Za-z0-9_-]{20,220}$/.test(candidate)
? candidate
: "";
}
function verifiedDouyinProfileUrl(value: unknown) {
const candidate = stringValue(value);
if (!candidate) return "";
try {
const parsed = new URL(candidate);
const secUid = decodeURIComponent(parsed.pathname.match(/^\/user\/([^/?#]+)/)?.[1] ?? "");
if (
parsed.protocol === "https:" &&
(parsed.hostname === "douyin.com" || parsed.hostname.endsWith(".douyin.com")) &&
usableDouyinSecUid(secUid)
) {
return parsed.toString();
}
} catch {
// The public redirect fallback below can still recover the profile URL.
}
return "";
}
async function douyinProfileFromPublicRedirect(
publishUrl: string,
fetchImpl: typeof fetch,
timeoutMs: number,
) {
let parsed: URL;
try {
parsed = new URL(publishUrl);
} catch {
return null;
}
if (
parsed.protocol !== "https:" ||
!(parsed.hostname === "douyin.com" || parsed.hostname.endsWith(".douyin.com"))
) {
return null;
}
try {
const response = await fetchImpl(parsed.toString(), {
method: "GET",
redirect: "manual",
signal: AbortSignal.timeout(Math.min(timeoutMs, 15_000)),
headers: {
"user-agent":
"Mozilla/5.0 (iPhone; CPU iPhone OS 18_0 like Mac OS X) AppleWebKit/605.1.15 Mobile/15E148",
},
});
const location = response.headers.get("location");
if (!location) return null;
const redirectUrl = new URL(location, parsed);
const secUid = usableDouyinSecUid(
redirectUrl.searchParams.get("sec_uid") ??
redirectUrl.searchParams.get("sec_user_id"),
);
return secUid
? {
platformUid: secUid,
profileUrl: `https://www.douyin.com/user/${encodeURIComponent(secUid)}`,
}
: null;
} catch {
return null;
}
}
function successfulToolData(result: ToolResult, fallbackMessage: string) {
const root = asRecord(result.payload);
const response = asRecord(root?.response) ?? root;
@@ -446,6 +553,64 @@ function findValueByKeys(
return undefined;
}
function profileGender(value: unknown): "" | "男" | "女" {
if (value === 1) return "男";
if (value === 2) return "女";
const normalized = String(value ?? "").trim().toLocaleLowerCase("zh-CN");
if (["男", "男性", "male", "m", "1"].includes(normalized)) return "男";
if (["女", "女性", "female", "f", "2"].includes(normalized)) return "女";
return "";
}
function recentNoteTitlesFromPayload(value: unknown) {
const titles: string[] = [];
const visit = (current: unknown, depth = 0) => {
if (depth > 12 || titles.length >= 20) return;
if (Array.isArray(current)) {
current.forEach((item) => visit(item, depth + 1));
return;
}
const record = asRecord(current);
if (!record) return;
const title = stringValue(record.title ?? record.note_title ?? record.noteTitle);
if (
title &&
(record.note_id || record.noteId || record.url || record.cover) &&
!titles.includes(title)
) {
titles.push(title);
}
Object.values(record).forEach((child) => visit(child, depth + 1));
};
visit(value);
return titles;
}
function providerTagsFromUser(value: unknown) {
const user = findRecord(value, (record) =>
Boolean(
record.gender !== undefined ||
record.desc !== undefined ||
record.signature !== undefined ||
record.fansCount !== undefined ||
record.fans_count !== undefined,
),
);
const raw = user?.tags;
if (!Array.isArray(raw)) return [];
return [
...new Set(
raw
.map((item) =>
typeof item === "string"
? item.trim()
: stringValue(asRecord(item)?.name ?? asRecord(item)?.title),
)
.filter(Boolean),
),
].slice(0, 5);
}
const FOLLOWER_KEYS = new Set([
"fans",
"fans_count",
@@ -529,10 +694,13 @@ async function xhsNoteIdFromShortLink(
) {
return { noteId: "", profile: null };
}
if (
url.hostname !== "xhslink.cn" &&
!url.hostname.endsWith(".xhslink.cn")
) {
const isShortLink =
url.hostname === "xhslink.cn" ||
url.hostname.endsWith(".xhslink.cn");
const isXhsPage =
url.hostname === "xiaohongshu.com" ||
url.hostname.endsWith(".xiaohongshu.com");
if (!isShortLink && !isXhsPage) {
return { noteId: "", profile: null };
}
@@ -608,6 +776,10 @@ function accountProfileFromPublicPage(
redId,
ipLocation: "待识别",
followers: null,
gender: "",
bio: "",
recentNoteTitles: [],
providerTags: [],
};
}
@@ -698,15 +870,45 @@ function profileDetailsFromToolResult(result: ToolResult) {
const redId =
findStringByKey(payload, "red_id") ||
findStringByKey(payload, "redId") ||
findStringByKey(payload, "unique_id") ||
findStringByKey(payload, "uniqueId") ||
findStringByKey(payload, "short_id") ||
findStringByKey(payload, "shortId") ||
findStringByKey(payload, "douyin_id") ||
findStringByKey(payload, "userId") ||
findStringByKey(payload, "user_id");
const ipLocation =
findStringByKey(payload, "ip_location") ||
findStringByKey(payload, "ipLocation");
if (followers === null && !nickname && !redId && !ipLocation) {
const gender = profileGender(findValueByKeys(payload, new Set(["gender", "sex"])));
const bio =
findStringByKey(payload, "desc") ||
findStringByKey(payload, "description") ||
findStringByKey(payload, "signature") ||
findStringByKey(payload, "bio");
const recentNoteTitles = recentNoteTitlesFromPayload(payload);
const providerTags = providerTagsFromUser(payload);
if (
followers === null &&
!nickname &&
!redId &&
!ipLocation &&
!gender &&
!bio &&
recentNoteTitles.length === 0
) {
throw new Error("账号主页采集结果缺少可用字段");
}
return { nickname, followers, redId, ipLocation };
return {
nickname,
followers,
redId,
ipLocation,
gender,
bio,
recentNoteTitles,
providerTags,
};
}
function accountProfileFromToolResult(
@@ -718,14 +920,18 @@ function accountProfileFromToolResult(
data,
(record) =>
Boolean(
stringValue(record.user_id ?? record.userid) &&
stringValue(record.user_id ?? record.userid ?? record.userId) &&
(stringValue(record.profile_url) ||
stringValue(record.nickname ?? record.name)),
),
);
if (!user) throw new Error("账号主页识别结果缺少作者信息");
const platformUid = stringValue(user.user_id ?? user.userid);
const candidateProfileUrl = stringValue(user.profile_url);
const platformUid = stringValue(
user.user_id ?? user.userid ?? user.userId,
);
const candidateProfileUrl = stringValue(
user.profile_url ?? user.profileUrl,
);
let profileUrl = "";
if (candidateProfileUrl) {
try {
@@ -755,6 +961,10 @@ function accountProfileFromToolResult(
redId: stringValue(user.red_id),
ipLocation: findStringByKey(data, "ip_location") || "待识别",
followers: followerCountFromPayload(data),
gender: "",
bio: "",
recentNoteTitles: [],
providerTags: [],
};
}
@@ -771,18 +981,7 @@ async function completeAccountProfile(
"parse_xhs_user_summary",
{ url: profile.profileUrl, use_proxy: true },
],
[
"fetch_user_detail",
{ link: profile.profileUrl, plant: "xhs" },
],
] as const) {
if (
completed.followers !== null &&
completed.redId &&
completed.ipLocation !== "待识别"
) {
break;
}
try {
const result = await callMcpTool(
fetchImpl,
@@ -801,6 +1000,16 @@ async function completeAccountProfile(
details.ipLocation && details.ipLocation !== "待识别"
? details.ipLocation
: completed.ipLocation,
gender: details.gender || completed.gender,
bio: details.bio || completed.bio,
recentNoteTitles:
details.recentNoteTitles.length > 0
? details.recentNoteTitles
: completed.recentNoteTitles,
providerTags:
details.providerTags.length > 0
? details.providerTags
: completed.providerTags,
};
} catch (error) {
if (error instanceof McpSessionLostError) throw error;
@@ -852,17 +1061,14 @@ async function resolveAccountInSession(
const endpoint = buildMcpUrl(config);
const timeoutMs = Math.max(5_000, config.timeoutMs ?? 30_000);
let noteId = xhsNoteIdFromUrl(publishUrl);
let publicPageProfile: XhsAccountProfile | null = null;
if (!noteId) {
const shortLink = await xhsNoteIdFromShortLink(
publishUrl,
fallbackNickname,
fetchImpl,
timeoutMs,
);
noteId = shortLink.noteId;
publicPageProfile = shortLink.profile;
}
const linkPage = await xhsNoteIdFromShortLink(
publishUrl,
fallbackNickname,
fetchImpl,
timeoutMs,
);
noteId = noteId || linkPage.noteId;
const publicPageProfile = linkPage.profile;
try {
const sessionId = await createMcpSession(
@@ -870,47 +1076,39 @@ async function resolveAccountInSession(
endpoint,
timeoutMs,
);
if (!noteId) {
const noteResult = await callMcpTool(
fetchImpl,
endpoint,
sessionId,
timeoutMs,
"fetch_content_detail",
{
link: publishUrl,
plant: "xhs",
include_comments: false,
auto_cookie: true,
},
);
const noteData = successfulToolData(
noteResult,
"无法识别小红书笔记",
);
noteId =
stringValue(noteData.noteId ?? noteData.note_id) ||
findStringByKey(noteData, "noteId") ||
findStringByKey(noteData, "note_id");
}
if (!noteId) throw new Error("无法从发布链接识别小红书笔记ID");
const authorResult = await callMcpTool(
const noteResult = await callMcpTool(
fetchImpl,
endpoint,
sessionId,
timeoutMs,
"collect_xhs_wen_note_detail",
"fetch_content_detail",
{
note_id: noteId,
need_desc: false,
include_raw: false,
link: publishUrl,
plant: "xhs",
include_comments: false,
auto_cookie: true,
},
);
const profile = accountProfileFromToolResult(
authorResult,
fallbackNickname,
const noteData = successfulToolData(
noteResult,
"无法识别小红书笔记",
);
noteId =
noteId ||
stringValue(noteData.noteId ?? noteData.note_id) ||
findStringByKey(noteData, "noteId") ||
findStringByKey(noteData, "note_id");
if (!noteId) throw new Error("无法从发布链接识别小红书笔记ID");
let profile: XhsAccountProfile;
try {
profile = accountProfileFromToolResult(
noteResult,
fallbackNickname,
);
} catch {
if (!publicPageProfile) throw new Error("笔记数据缺少公开作者主页");
profile = publicPageProfile;
}
return completeAccountProfile(
profile,
fetchImpl,
@@ -937,6 +1135,7 @@ async function resolveAccountInSession(
async function collectInSession(
publishUrl: string,
platform: "小红书" | "抖音",
config: CollectionMcpConfig,
fetchImpl: typeof fetch,
) {
@@ -952,28 +1151,12 @@ async function collectInSession(
"fetch_content_detail",
{
link: publishUrl,
plant: "xhs",
plant: platform === "抖音" ? "dy" : "xhs",
include_comments: false,
auto_cookie: true,
},
);
try {
return metricsFromToolResult(primary);
} catch {
const fallback = await callMcpTool(
fetchImpl,
endpoint,
sessionId,
timeoutMs,
"parse_xhs_note",
{
url: publishUrl,
include_comments: false,
auto_cookie: true,
},
);
return metricsFromToolResult(fallback);
}
return metricsFromToolResult(primary);
}
export function resolveCollectionMcpConfig(
@@ -1005,7 +1188,7 @@ export async function collectXhsMetricsFromMcp(
let lastError: unknown;
for (let attempt = 0; attempt < MAX_MCP_ATTEMPTS; attempt += 1) {
try {
return await collectInSession(parsed.toString(), config, fetchImpl);
return await collectInSession(parsed.toString(), "小红书", config, fetchImpl);
} catch (error) {
lastError = error;
if (!isRetryableTransportError(error)) throw error;
@@ -1022,6 +1205,211 @@ export async function collectXhsMetricsFromMcp(
: new Error("MCP采集服务暂时不可用");
}
export async function collectMetricsFromMcp(
publishUrl: string,
platform: "小红书" | "抖音",
config: CollectionMcpConfig,
fetchImpl: typeof fetch = fetch,
) {
if (platform === "小红书") {
return collectXhsMetricsFromMcp(publishUrl, config, fetchImpl);
}
let parsed: URL;
try {
parsed = new URL(publishUrl);
} catch {
throw new Error("发布链接无效");
}
if (!["http:", "https:"].includes(parsed.protocol)) {
throw new Error("发布链接无效");
}
let lastError: unknown;
for (let attempt = 0; attempt < MAX_MCP_ATTEMPTS; attempt += 1) {
try {
return await collectInSession(parsed.toString(), platform, config, fetchImpl);
} catch (error) {
lastError = error;
if (!isRetryableTransportError(error)) throw error;
}
}
throw lastError instanceof Error
? lastError
: new Error("MCP采集服务暂时不可用");
}
function douyinProfileFromToolResult(
result: ToolResult,
fallbackNickname: string,
): XhsAccountProfile {
const data = successfulToolData(result, "无法识别抖音作品");
const author = findRecord(data, (record) =>
Boolean(
stringValue(
record.sec_uid ?? record.secUid ?? record.uid ?? record.user_id ?? record.userId,
) && stringValue(record.nickname ?? record.name ?? record.unique_id ?? record.uniqueId),
),
);
if (!author) throw new Error("抖音作品数据缺少作者信息");
const verifiedSecUid = usableDouyinSecUid(author.sec_uid ?? author.secUid);
const fallbackUid = stringValue(author.uid ?? author.user_id ?? author.userId);
const platformUid = verifiedSecUid || fallbackUid;
const publicId = stringValue(
author.unique_id ?? author.uniqueId ?? author.short_id ?? author.shortId ?? author.douyin_id,
);
const candidateProfileUrl = verifiedDouyinProfileUrl(
author.profile_url ?? author.profileUrl,
);
const profileUrl = candidateProfileUrl ||
(verifiedSecUid
? `https://www.douyin.com/user/${encodeURIComponent(verifiedSecUid)}`
: "");
return {
platformUid,
nickname: stringValue(author.nickname ?? author.name) || fallbackNickname.trim(),
profileUrl,
redId: publicId,
ipLocation:
stringValue(author.ip_location ?? author.ipLocation) ||
findStringByKey(data, "ip_location") ||
findStringByKey(data, "ipLocation") ||
"待识别",
followers: followerCountFromPayload(author) ?? followerCountFromPayload(data),
gender: profileGender(author.gender ?? author.sex),
bio: stringValue(author.desc ?? author.description ?? author.signature ?? author.bio),
recentNoteTitles: recentNoteTitlesFromPayload(data),
providerTags: providerTagsFromUser(data),
};
}
async function resolveDouyinAccountInSession(
publishUrl: string,
fallbackNickname: string,
config: CollectionMcpConfig,
fetchImpl: typeof fetch,
) {
const endpoint = buildMcpUrl(config);
const timeoutMs = Math.max(5_000, config.timeoutMs ?? 30_000);
const sessionId = await createMcpSession(fetchImpl, endpoint, timeoutMs);
const result = await callMcpTool(
fetchImpl,
endpoint,
sessionId,
timeoutMs,
"fetch_content_detail",
{
link: publishUrl,
plant: "dy",
include_comments: false,
auto_cookie: true,
},
);
let profile = douyinProfileFromToolResult(result, fallbackNickname);
if (!verifiedDouyinProfileUrl(profile.profileUrl)) {
const resolved = await douyinProfileFromPublicRedirect(
publishUrl,
fetchImpl,
timeoutMs,
);
if (resolved) {
profile = {
...profile,
platformUid: resolved.platformUid,
profileUrl: resolved.profileUrl,
};
}
}
if (!verifiedDouyinProfileUrl(profile.profileUrl)) return profile;
try {
const detailsResult = await callMcpTool(
fetchImpl,
endpoint,
sessionId,
timeoutMs,
"parse_dy_user_summary",
{ url: profile.profileUrl },
);
const details = profileDetailsFromToolResult(detailsResult);
profile = {
...profile,
nickname: details.nickname || profile.nickname,
redId: details.redId || profile.redId,
followers: details.followers ?? profile.followers,
ipLocation:
details.ipLocation && details.ipLocation !== "待识别"
? details.ipLocation
: profile.ipLocation,
gender: details.gender || profile.gender,
bio: details.bio || profile.bio,
recentNoteTitles:
details.recentNoteTitles.length > 0
? details.recentNoteTitles
: profile.recentNoteTitles,
providerTags:
details.providerTags.length > 0
? details.providerTags
: profile.providerTags,
};
} catch (error) {
if (error instanceof McpSessionLostError) throw error;
}
return profile;
}
export async function resolveAccountProfileFromMcp(
publishUrl: string,
fallbackNickname: string,
platform: "小红书" | "抖音",
config: CollectionMcpConfig,
fetchImpl: typeof fetch = fetch,
) {
if (platform === "小红书") {
return resolveXhsAccountProfileFromMcp(
publishUrl,
fallbackNickname,
config,
fetchImpl,
);
}
let lastError: unknown;
for (let attempt = 0; attempt < MAX_MCP_ATTEMPTS; attempt += 1) {
try {
return await resolveDouyinAccountInSession(
publishUrl,
fallbackNickname,
config,
fetchImpl,
);
} catch (error) {
lastError = error;
if (!isRetryableTransportError(error)) throw error;
}
}
throw lastError instanceof Error ? lastError : new Error("抖音账号识别失败");
}
export async function resolveProfileDetailsFromMcp(
profileUrl: string,
platform: "小红书" | "抖音",
config: CollectionMcpConfig,
fetchImpl: typeof fetch = fetch,
) {
if (platform === "小红书") {
return resolveXhsProfileDetailsFromMcp(profileUrl, config, fetchImpl);
}
const endpoint = buildMcpUrl(config);
const timeoutMs = Math.max(5_000, config.timeoutMs ?? 30_000);
const sessionId = await createMcpSession(fetchImpl, endpoint, timeoutMs);
const result = await callMcpTool(
fetchImpl,
endpoint,
sessionId,
timeoutMs,
"parse_dy_user_summary",
{ url: profileUrl },
);
return profileDetailsFromToolResult(result);
}
export async function resolveXhsAccountProfileFromMcp(
publishUrl: string,
fallbackNickname: string,