feat: KOC 资源导入后自动爬取主页资料并调用 LLM 打标签
- 新增 structured_text_process 打标客户端(文本构造、标签解析清洗、MCP 调用) - 写库后后台打标(tagUntaggedAccounts),单条新增 / 批量导入 / 每日补全三个入口统一 - 标签只补空,UPDATE 带 tags 为空守卫,不覆盖已有标签 - 新增 AI_TAGGING_TOOL 配置项(留空关闭),更新配置示例与文档 - 新增 16 个打标测试用例,全量 131 个测试通过
This commit is contained in:
@@ -8,6 +8,9 @@ KOC_MCP_API_KEY=replace-with-a-separate-long-random-secret
|
|||||||
AI_TOOL_CENTER_MCP_URL=https://middle-aitool.gbotai.cn/mcp
|
AI_TOOL_CENTER_MCP_URL=https://middle-aitool.gbotai.cn/mcp
|
||||||
AI_TOOL_CENTER_MCP_KEY=replace-with-mcp-key
|
AI_TOOL_CENTER_MCP_KEY=replace-with-mcp-key
|
||||||
|
|
||||||
|
# 账号 LLM 自动打标(可选):工具中心 structured_text_process 工具,留空 = 关闭自动打标
|
||||||
|
AI_TAGGING_TOOL=
|
||||||
|
|
||||||
# Feishu custom app credentials. Keep the secret out of source control.
|
# Feishu custom app credentials. Keep the secret out of source control.
|
||||||
FEISHU_APP_ID=cli_xxxxxxxxxxxxxxxxx
|
FEISHU_APP_ID=cli_xxxxxxxxxxxxxxxxx
|
||||||
FEISHU_APP_SECRET=replace-with-feishu-app-secret
|
FEISHU_APP_SECRET=replace-with-feishu-app-secret
|
||||||
|
|||||||
@@ -24,6 +24,9 @@ FEISHU_APP_SECRET=
|
|||||||
AI_TOOL_CENTER_MCP_URL=
|
AI_TOOL_CENTER_MCP_URL=
|
||||||
AI_TOOL_CENTER_MCP_KEY=
|
AI_TOOL_CENTER_MCP_KEY=
|
||||||
|
|
||||||
|
# 账号 LLM 自动打标(可选):AI_TOOL_CENTER 的 structured_text_process 工具,留空 = 关闭自动打标。
|
||||||
|
AI_TAGGING_TOOL=
|
||||||
|
|
||||||
# 企业微信通知(临期催办 + 管理员汇总)。群机器人只需 webhook;KOC 侧催办还需 corp/agent/secret,
|
# 企业微信通知(临期催办 + 管理员汇总)。群机器人只需 webhook;KOC 侧催办还需 corp/agent/secret,
|
||||||
# 并在后台 partners 编辑里把 wecom_external_user_id 填好。
|
# 并在后台 partners 编辑里把 wecom_external_user_id 填好。
|
||||||
WECOM_ROBOT_WEBHOOK=
|
WECOM_ROBOT_WEBHOOK=
|
||||||
|
|||||||
@@ -84,6 +84,7 @@ ADMIN_ALLOWED_EMAIL — ChatGPT email allowed for admin access
|
|||||||
ADMIN_INTERNAL_TOKEN — Shared secret for API-to-API auth
|
ADMIN_INTERNAL_TOKEN — Shared secret for API-to-API auth
|
||||||
AI_TOOL_CENTER_MCP_URL — MCP endpoint for XHS data collection
|
AI_TOOL_CENTER_MCP_URL — MCP endpoint for XHS data collection
|
||||||
AI_TOOL_CENTER_MCP_KEY — MCP API key
|
AI_TOOL_CENTER_MCP_KEY — MCP API key
|
||||||
|
AI_TAGGING_TOOL — 账号 LLM 自动打标:工具中心 structured_text_process,留空关闭(可选)
|
||||||
FEISHU_APP_ID — Feishu custom app credentials
|
FEISHU_APP_ID — Feishu custom app credentials
|
||||||
FEISHU_APP_SECRET — Feishu app secret
|
FEISHU_APP_SECRET — Feishu app secret
|
||||||
WECOM_CORP_ID — 企业微信企业 ID(自建应用消息推送用,可选)
|
WECOM_CORP_ID — 企业微信企业 ID(自建应用消息推送用,可选)
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ import {
|
|||||||
parseResourceImportFile,
|
parseResourceImportFile,
|
||||||
RESOURCE_IMPORT_MAX_BYTES,
|
RESOURCE_IMPORT_MAX_BYTES,
|
||||||
RESOURCE_IMPORT_MAX_ROWS,
|
RESOURCE_IMPORT_MAX_ROWS,
|
||||||
resourceImportMissingFields,
|
needsResourceEnrichment,
|
||||||
type ResourceImportRow,
|
type ResourceImportRow,
|
||||||
} from "../../../lib/resource-import";
|
} from "../../../lib/resource-import";
|
||||||
import {
|
import {
|
||||||
@@ -12,6 +12,7 @@ import {
|
|||||||
enrichRows,
|
enrichRows,
|
||||||
loadAccounts,
|
loadAccounts,
|
||||||
mergeExistingFields,
|
mergeExistingFields,
|
||||||
|
tagUntaggedAccounts,
|
||||||
writeAnalyzedRows,
|
writeAnalyzedRows,
|
||||||
RESOURCE_IMPORT_SYNC_ENRICH_ROWS,
|
RESOURCE_IMPORT_SYNC_ENRICH_ROWS,
|
||||||
type AnalyzedRow,
|
type AnalyzedRow,
|
||||||
@@ -49,7 +50,7 @@ function deferredEnrichmentCount(rows: ResourceImportRow[]) {
|
|||||||
(row) =>
|
(row) =>
|
||||||
row.errors.length === 0 &&
|
row.errors.length === 0 &&
|
||||||
row.profileUrl &&
|
row.profileUrl &&
|
||||||
resourceImportMissingFields(row).length > 0,
|
needsResourceEnrichment(row),
|
||||||
).length;
|
).length;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -60,7 +61,7 @@ async function enrichImportedRowsInBackground(rows: ResourceImportRow[]) {
|
|||||||
(row) =>
|
(row) =>
|
||||||
row.errors.length === 0 &&
|
row.errors.length === 0 &&
|
||||||
row.profileUrl &&
|
row.profileUrl &&
|
||||||
resourceImportMissingFields(row).length > 0,
|
needsResourceEnrichment(row),
|
||||||
);
|
);
|
||||||
if (missingRows.length === 0) return;
|
if (missingRows.length === 0) return;
|
||||||
const enriched = await enrichRows(missingRows, accounts.results);
|
const enriched = await enrichRows(missingRows, accounts.results);
|
||||||
@@ -132,6 +133,10 @@ export async function POST(request: Request) {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
await writeAnalyzedRows(importableRows);
|
await writeAnalyzedRows(importableRows);
|
||||||
|
runInBackground(
|
||||||
|
tagUntaggedAccounts(importableRows),
|
||||||
|
"resource account tagging",
|
||||||
|
);
|
||||||
if (deferredEnrichment > 0) {
|
if (deferredEnrichment > 0) {
|
||||||
const importableSourceRows = rows.filter((row) =>
|
const importableSourceRows = rows.filter((row) =>
|
||||||
importableRowNumbers.has(row.rowNumber),
|
importableRowNumbers.has(row.rowNumber),
|
||||||
|
|||||||
@@ -1,9 +1,11 @@
|
|||||||
import { isManagerRequest, managerForbidden } from "../../../lib/user-auth";
|
import { isManagerRequest, managerForbidden } from "../../../lib/user-auth";
|
||||||
|
import { runInBackground } from "../../../lib/background";
|
||||||
import { buildManualRow, type ManualResourceInput } from "../../../lib/resource-import";
|
import { buildManualRow, type ManualResourceInput } from "../../../lib/resource-import";
|
||||||
import {
|
import {
|
||||||
analyzeRows,
|
analyzeRows,
|
||||||
enrichRows,
|
enrichRows,
|
||||||
loadAccounts,
|
loadAccounts,
|
||||||
|
tagUntaggedAccounts,
|
||||||
writeAnalyzedRows,
|
writeAnalyzedRows,
|
||||||
} from "../../../lib/resource-write";
|
} from "../../../lib/resource-write";
|
||||||
|
|
||||||
@@ -48,6 +50,10 @@ export async function POST(request: Request) {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
await writeAnalyzedRows(analyzed);
|
await writeAnalyzedRows(analyzed);
|
||||||
|
runInBackground(
|
||||||
|
tagUntaggedAccounts(analyzed),
|
||||||
|
"resource account tagging",
|
||||||
|
);
|
||||||
const result = analyzed[0];
|
const result = analyzed[0];
|
||||||
const outcome: InsertResult = {
|
const outcome: InsertResult = {
|
||||||
action: result.action === "update" ? "update" : "create",
|
action: result.action === "update" ? "update" : "create",
|
||||||
|
|||||||
9
examples/koc-resources-sample.csv
Normal file
9
examples/koc-resources-sample.csv
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
账号链接,账号昵称,小红书号,IP属地,粉丝数,性别,简介,标签,合作来源
|
||||||
|
https://www.xiaohongshu.com/user/profile/5ff0e6410000000001008401,小红薯爱分享,xhs_88012345,上海,1.2万,女,爱美食爱生活的上海小姐姐,合作私信~,"美食,探店,生活方式",媒介推荐
|
||||||
|
https://www.xiaohongshu.com/user/profile/5ff0e6410000000001008402,阿杰的穿搭日记,xhs_7712ab,广东,8600,男,男生穿搭 | 平价好物 | 每周更新,"穿搭,男装,平价好物",历史合作
|
||||||
|
https://www.xiaohongshu.com/user/profile/5ff0e6410000000001008403,琳琳妈妈育儿记,xhs_6633cd,浙江,3.5万,女,二胎妈妈,分享育儿干货和母婴好物,"母婴,育儿,好物分享",蒲公英
|
||||||
|
https://www.xiaohongshu.com/user/profile/5ff0e6410000000001008404,健身教练大凯,xhs_5544ef,北京,10万+,男,NSCA认证教练,运动补剂科普,"健身,运动,补剂",自荐
|
||||||
|
https://www.xiaohongshu.com/user/profile/5ff0e6410000000001008405,甜品研究社,xhs_4455gh,四川,2.1万,女,烘焙教程 | 甜品测评 | 成都探店,"烘焙,甜品,探店",媒介推荐
|
||||||
|
https://www.douyin.com/user/ms4y9x873kq1,数码测评君,dy_3321ij,江苏,5.8万,男,手机数码深度测评,避坑指南,"数码,测评,科技",星图
|
||||||
|
https://www.xiaohongshu.com/user/profile/5ff0e6410000000001008407,旅行摄影的小舟,xhs_2266mn,云南,6400,女,风光摄影 | 旅行vlog | 云南旅拍,"旅行,摄影",历史合作
|
||||||
|
https://www.xiaohongshu.com/user/profile/5ff0e6410000000001008408,美妆小课堂Amy,xhs_1177op,上海,15万,女,化妆品成分党,专注国货美妆测评,"美妆,护肤,国货",蒲公英
|
||||||
|
8
examples/koc-resources-sample.jsonl
Normal file
8
examples/koc-resources-sample.jsonl
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
{"profileUrl": "https://www.xiaohongshu.com/user/profile/5ff0e6410000000001008401", "nickname": "小红薯爱分享", "publicAccountId": "xhs_88012345", "ipLocation": "上海", "followers": "1.2万", "gender": "女", "bio": "爱美食爱生活的上海小姐姐,合作私信~", "tags": "美食,探店,生活方式", "cooperationSource": "媒介推荐"}
|
||||||
|
{"profileUrl": "https://www.xiaohongshu.com/user/profile/5ff0e6410000000001008402", "nickname": "阿杰的穿搭日记", "publicAccountId": "xhs_7712ab", "ipLocation": "广东", "followers": "8600", "gender": "男", "bio": "男生穿搭 | 平价好物 | 每周更新", "tags": "穿搭,男装,平价好物", "cooperationSource": "历史合作"}
|
||||||
|
{"profileUrl": "https://www.xiaohongshu.com/user/profile/5ff0e6410000000001008403", "nickname": "琳琳妈妈育儿记", "publicAccountId": "xhs_6633cd", "ipLocation": "浙江", "followers": "3.5万", "gender": "女", "bio": "二胎妈妈,分享育儿干货和母婴好物", "tags": "母婴,育儿,好物分享", "cooperationSource": "蒲公英"}
|
||||||
|
{"profileUrl": "https://www.xiaohongshu.com/user/profile/5ff0e6410000000001008404", "nickname": "健身教练大凯", "publicAccountId": "xhs_5544ef", "ipLocation": "北京", "followers": "10万+", "gender": "男", "bio": "NSCA认证教练,运动补剂科普", "tags": "健身,运动,补剂", "cooperationSource": "自荐"}
|
||||||
|
{"profileUrl": "https://www.xiaohongshu.com/user/profile/5ff0e6410000000001008405", "nickname": "甜品研究社", "publicAccountId": "xhs_4455gh", "ipLocation": "四川", "followers": "2.1万", "gender": "女", "bio": "烘焙教程 | 甜品测评 | 成都探店", "tags": "烘焙,甜品,探店", "cooperationSource": "媒介推荐"}
|
||||||
|
{"profileUrl": "https://www.douyin.com/user/ms4y9x873kq1", "nickname": "数码测评君", "publicAccountId": "dy_3321ij", "ipLocation": "江苏", "followers": "5.8万", "gender": "男", "bio": "手机数码深度测评,避坑指南", "tags": "数码,测评,科技", "cooperationSource": "星图"}
|
||||||
|
{"profileUrl": "https://www.xiaohongshu.com/user/profile/5ff0e6410000000001008407", "nickname": "旅行摄影的小舟", "publicAccountId": "xhs_2266mn", "ipLocation": "云南", "followers": "6400", "gender": "女", "bio": "风光摄影 | 旅行vlog | 云南旅拍", "tags": "旅行,摄影", "cooperationSource": "历史合作"}
|
||||||
|
{"profileUrl": "https://www.xiaohongshu.com/user/profile/5ff0e6410000000001008408", "nickname": "美妆小课堂Amy", "publicAccountId": "xhs_1177op", "ipLocation": "上海", "followers": "15万", "gender": "女", "bio": "化妆品成分党,专注国货美妆测评", "tags": "美妆,护肤,国货", "cooperationSource": "蒲公英"}
|
||||||
@@ -1,9 +1,12 @@
|
|||||||
import {
|
import {
|
||||||
|
generateAccountTagsFromMcp,
|
||||||
resolveXhsPublicAccountDetails,
|
resolveXhsPublicAccountDetails,
|
||||||
resolveAccountProfileFromMcp,
|
resolveAccountProfileFromMcp,
|
||||||
resolveProfileDetailsFromMcp,
|
resolveProfileDetailsFromMcp,
|
||||||
|
type AccountTaggingEnv,
|
||||||
type CollectionMcpConfig,
|
type CollectionMcpConfig,
|
||||||
} from "./mcp-collection-client";
|
} from "./mcp-collection-client";
|
||||||
|
import { getRuntimeEnv } from "./runtime-env";
|
||||||
import { hashText } from "./mvp-db";
|
import { hashText } from "./mvp-db";
|
||||||
import type { DatabaseClient } from "./database";
|
import type { DatabaseClient } from "./database";
|
||||||
|
|
||||||
@@ -260,6 +263,7 @@ export async function backfillAccountProfiles(
|
|||||||
mcpConfig: CollectionMcpConfig,
|
mcpConfig: CollectionMcpConfig,
|
||||||
limit = 10,
|
limit = 10,
|
||||||
) {
|
) {
|
||||||
|
const taggingEnv: AccountTaggingEnv = getRuntimeEnv();
|
||||||
const rows = await db
|
const rows = await db
|
||||||
.prepare(
|
.prepare(
|
||||||
`SELECT
|
`SELECT
|
||||||
@@ -334,6 +338,7 @@ export async function backfillAccountProfiles(
|
|||||||
const isDemoAccount = row.platform_uid?.startsWith("xhs-");
|
const isDemoAccount = row.platform_uid?.startsWith("xhs-");
|
||||||
let attemptedThisRow = false;
|
let attemptedThisRow = false;
|
||||||
let publicProfileUpdated = false;
|
let publicProfileUpdated = false;
|
||||||
|
let tagsUpdated = false;
|
||||||
if (
|
if (
|
||||||
row.platform === "小红书" &&
|
row.platform === "小红书" &&
|
||||||
!isDemoAccount &&
|
!isDemoAccount &&
|
||||||
@@ -341,7 +346,8 @@ export async function backfillAccountProfiles(
|
|||||||
(!row.public_account_id ||
|
(!row.public_account_id ||
|
||||||
Number(row.followers ?? 0) === 0 ||
|
Number(row.followers ?? 0) === 0 ||
|
||||||
!row.gender ||
|
!row.gender ||
|
||||||
!row.bio)
|
!row.bio ||
|
||||||
|
!row.tags?.trim())
|
||||||
) {
|
) {
|
||||||
attempted += 1;
|
attempted += 1;
|
||||||
attemptedThisRow = true;
|
attemptedThisRow = true;
|
||||||
@@ -402,6 +408,39 @@ export async function backfillAccountProfiles(
|
|||||||
.run();
|
.run();
|
||||||
publicProfileUpdated = true;
|
publicProfileUpdated = true;
|
||||||
}
|
}
|
||||||
|
if (
|
||||||
|
row.account_id &&
|
||||||
|
!row.tags?.trim() &&
|
||||||
|
(details.bio ||
|
||||||
|
details.providerTags.length > 0 ||
|
||||||
|
details.recentNoteTitles.length > 0)
|
||||||
|
) {
|
||||||
|
const tags = await generateAccountTagsFromMcp(
|
||||||
|
{
|
||||||
|
platform: "小红书",
|
||||||
|
nickname: row.nickname || details.nickname || "",
|
||||||
|
bio: details.bio,
|
||||||
|
gender: details.gender,
|
||||||
|
ipLocation: details.ipLocation ?? "",
|
||||||
|
followers: details.followers,
|
||||||
|
providerTags: details.providerTags,
|
||||||
|
recentNoteTitles: details.recentNoteTitles,
|
||||||
|
},
|
||||||
|
mcpConfig,
|
||||||
|
taggingEnv,
|
||||||
|
).catch(() => []);
|
||||||
|
if (tags.length > 0) {
|
||||||
|
await db
|
||||||
|
.prepare(
|
||||||
|
`UPDATE accounts
|
||||||
|
SET tags = CASE WHEN ? != '' THEN ? ELSE tags END
|
||||||
|
WHERE id = ?`,
|
||||||
|
)
|
||||||
|
.bind(tags.join(","), tags.join(","), row.account_id)
|
||||||
|
.run();
|
||||||
|
tagsUpdated = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
if (
|
if (
|
||||||
(details.redId || row.public_account_id) &&
|
(details.redId || row.public_account_id) &&
|
||||||
Number(details.followers ?? row.followers ?? 0) > 0
|
Number(details.followers ?? row.followers ?? 0) > 0
|
||||||
@@ -410,7 +449,7 @@ export async function backfillAccountProfiles(
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if (attempted >= Math.max(1, Math.min(25, limit))) {
|
if (attempted >= Math.max(1, Math.min(25, limit))) {
|
||||||
if (publicProfileUpdated) updated += 1;
|
if (publicProfileUpdated || tagsUpdated) updated += 1;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -430,7 +469,7 @@ export async function backfillAccountProfiles(
|
|||||||
row.platform_uid?.startsWith("pending-") ||
|
row.platform_uid?.startsWith("pending-") ||
|
||||||
Boolean(noteId && row.platform_uid === noteId));
|
Boolean(noteId && row.platform_uid === noteId));
|
||||||
if (!needsEnrichment || !row.publish_url) {
|
if (!needsEnrichment || !row.publish_url) {
|
||||||
if (publicProfileUpdated) updated += 1;
|
if (publicProfileUpdated || tagsUpdated) updated += 1;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -443,9 +482,9 @@ export async function backfillAccountProfiles(
|
|||||||
row.nickname || "待识别账号",
|
row.nickname || "待识别账号",
|
||||||
mcpConfig,
|
mcpConfig,
|
||||||
);
|
);
|
||||||
if (result.updated || publicProfileUpdated) updated += 1;
|
if (result.updated || publicProfileUpdated || tagsUpdated) updated += 1;
|
||||||
} catch {
|
} catch {
|
||||||
if (publicProfileUpdated) updated += 1;
|
if (publicProfileUpdated || tagsUpdated) updated += 1;
|
||||||
else failed += 1;
|
else failed += 1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1484,3 +1484,154 @@ export async function resolveXhsProfileDetailsFromMcp(
|
|||||||
? lastError
|
? lastError
|
||||||
: new Error("账号主页数据采集服务暂时不可用");
|
: new Error("账号主页数据采集服务暂时不可用");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export type AccountTaggingInput = {
|
||||||
|
platform: string;
|
||||||
|
nickname: string;
|
||||||
|
bio: string;
|
||||||
|
gender: string;
|
||||||
|
ipLocation: string;
|
||||||
|
followers: number | null;
|
||||||
|
providerTags: string[];
|
||||||
|
recentNoteTitles: string[];
|
||||||
|
};
|
||||||
|
|
||||||
|
export type AccountTaggingEnv = {
|
||||||
|
AI_TAGGING_TOOL?: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export function buildAccountTagText(input: AccountTaggingInput): string {
|
||||||
|
const facts: string[] = [`平台:${input.platform.trim() || "小红书"}`];
|
||||||
|
if (input.nickname.trim()) facts.push(`昵称:${input.nickname.trim()}`);
|
||||||
|
if (input.bio.trim()) facts.push(`简介:${input.bio.trim()}`);
|
||||||
|
if (input.gender.trim()) facts.push(`性别:${input.gender.trim()}`);
|
||||||
|
if (input.ipLocation.trim()) {
|
||||||
|
facts.push(`IP属地:${input.ipLocation.trim()}`);
|
||||||
|
}
|
||||||
|
if (input.followers !== null && input.followers > 0) {
|
||||||
|
facts.push(`粉丝数:${input.followers}`);
|
||||||
|
}
|
||||||
|
if (input.providerTags.length > 0) {
|
||||||
|
facts.push(`平台标签:${input.providerTags.slice(0, 5).join("、")}`);
|
||||||
|
}
|
||||||
|
if (input.recentNoteTitles.length > 0) {
|
||||||
|
const titles = input.recentNoteTitles
|
||||||
|
.slice(0, 20)
|
||||||
|
.map((title) => `《${title}》`)
|
||||||
|
.join("");
|
||||||
|
facts.push(`最近笔记标题:${titles}`);
|
||||||
|
}
|
||||||
|
return facts.join("\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
function sanitizeTagList(items: unknown[]): string[] {
|
||||||
|
const tags: string[] = [];
|
||||||
|
for (const item of items) {
|
||||||
|
if (typeof item !== "string") continue;
|
||||||
|
const tag = item.trim().replace(/^#+|#+$/g, "").trim();
|
||||||
|
if (!tag || tag.length > 12) continue;
|
||||||
|
if (!tags.includes(tag)) tags.push(tag);
|
||||||
|
}
|
||||||
|
return tags.slice(0, 5);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function parseAccountTags(text: string): string[] {
|
||||||
|
const stripped = text
|
||||||
|
.trim()
|
||||||
|
.replace(/^```(?:json)?\s*/i, "")
|
||||||
|
.replace(/\s*```\s*$/, "");
|
||||||
|
const candidates = [stripped];
|
||||||
|
const arrayMatch = stripped.match(/\[[\s\S]*\]/);
|
||||||
|
if (arrayMatch) candidates.push(arrayMatch[0]);
|
||||||
|
for (const candidate of candidates) {
|
||||||
|
let parsed: unknown;
|
||||||
|
try {
|
||||||
|
parsed = JSON.parse(candidate);
|
||||||
|
} catch {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (!Array.isArray(parsed)) continue;
|
||||||
|
const tags = sanitizeTagList(parsed);
|
||||||
|
if (tags.length > 0) return tags;
|
||||||
|
}
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseTagsFromToolPayload(payload: unknown, depth = 0): string[] {
|
||||||
|
if (typeof payload === "string") return parseAccountTags(payload);
|
||||||
|
if (Array.isArray(payload)) {
|
||||||
|
const direct = sanitizeTagList(payload);
|
||||||
|
if (direct.length > 0) return direct;
|
||||||
|
if (depth < 2) {
|
||||||
|
for (const item of payload) {
|
||||||
|
const tags = parseTagsFromToolPayload(item, depth + 1);
|
||||||
|
if (tags.length > 0) return tags;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
const record = asRecord(payload);
|
||||||
|
if (record && depth < 2) {
|
||||||
|
for (const key of [
|
||||||
|
"tags",
|
||||||
|
"labels",
|
||||||
|
"result",
|
||||||
|
"data",
|
||||||
|
"response",
|
||||||
|
"output",
|
||||||
|
"text",
|
||||||
|
"content",
|
||||||
|
"message",
|
||||||
|
]) {
|
||||||
|
if (key in record) {
|
||||||
|
const tags = parseTagsFromToolPayload(record[key], depth + 1);
|
||||||
|
if (tags.length > 0) return tags;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function generateAccountTagsFromMcp(
|
||||||
|
input: AccountTaggingInput,
|
||||||
|
config: CollectionMcpConfig,
|
||||||
|
taggingEnv: AccountTaggingEnv,
|
||||||
|
fetchImpl: typeof fetch = fetch,
|
||||||
|
): Promise<string[]> {
|
||||||
|
const toolName = String(taggingEnv.AI_TAGGING_TOOL ?? "").trim();
|
||||||
|
if (!toolName || !config.endpoint.trim()) return [];
|
||||||
|
try {
|
||||||
|
const endpoint = buildMcpUrl(config);
|
||||||
|
const timeoutMs = Math.max(5_000, config.timeoutMs ?? 60_000);
|
||||||
|
const sessionId = await createMcpSession(fetchImpl, endpoint, timeoutMs);
|
||||||
|
const result = await invokeMcpTool(
|
||||||
|
fetchImpl,
|
||||||
|
endpoint,
|
||||||
|
sessionId,
|
||||||
|
timeoutMs,
|
||||||
|
toolName,
|
||||||
|
{
|
||||||
|
objective: "为KOC账号生成运营标签",
|
||||||
|
text: buildAccountTagText(input),
|
||||||
|
rules: [
|
||||||
|
"最多生成 5 个标签",
|
||||||
|
"每个标签不超过 8 个字",
|
||||||
|
"从内容领域、人设风格、受众画像等维度概括",
|
||||||
|
"只依据文本中给到的资料,不要编造",
|
||||||
|
],
|
||||||
|
output_schema: {
|
||||||
|
type: "object",
|
||||||
|
properties: {
|
||||||
|
tags: { type: "array", items: { type: "string" } },
|
||||||
|
},
|
||||||
|
required: ["tags"],
|
||||||
|
additionalProperties: false,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
);
|
||||||
|
if (result.isError) return [];
|
||||||
|
return parseTagsFromToolPayload(result.payload);
|
||||||
|
} catch {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -279,6 +279,21 @@ export function resourceImportMissingFields(
|
|||||||
return missing;
|
return missing;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function needsResourceEnrichment(
|
||||||
|
row: Pick<
|
||||||
|
ResourceImportRow,
|
||||||
|
| "nickname"
|
||||||
|
| "publicAccountId"
|
||||||
|
| "ipLocation"
|
||||||
|
| "followersResolved"
|
||||||
|
| "gender"
|
||||||
|
| "bio"
|
||||||
|
| "tags"
|
||||||
|
>,
|
||||||
|
) {
|
||||||
|
return resourceImportMissingFields(row).length > 0 || row.tags.length === 0;
|
||||||
|
}
|
||||||
|
|
||||||
function normalizeRows(rows: string[][]) {
|
function normalizeRows(rows: string[][]) {
|
||||||
const header = findHeader(rows);
|
const header = findHeader(rows);
|
||||||
if (!header) {
|
if (!header) {
|
||||||
|
|||||||
@@ -1,8 +1,10 @@
|
|||||||
import { ensureSchema, getRawDb } from "./mvp-db";
|
import { ensureSchema, getRawDb } from "./mvp-db";
|
||||||
import {
|
import {
|
||||||
|
generateAccountTagsFromMcp,
|
||||||
resolveCollectionMcpConfig,
|
resolveCollectionMcpConfig,
|
||||||
resolveProfileDetailsFromMcp,
|
resolveProfileDetailsFromMcp,
|
||||||
resolveXhsPublicAccountDetails,
|
resolveXhsPublicAccountDetails,
|
||||||
|
type AccountTaggingEnv,
|
||||||
type CollectionMcpBindings,
|
type CollectionMcpBindings,
|
||||||
} from "./mcp-collection-client";
|
} from "./mcp-collection-client";
|
||||||
import { getRuntimeEnv } from "./runtime-env";
|
import { getRuntimeEnv } from "./runtime-env";
|
||||||
@@ -135,9 +137,8 @@ export async function enrichRows(
|
|||||||
rows: ResourceImportRow[],
|
rows: ResourceImportRow[],
|
||||||
accounts: AccountRow[],
|
accounts: AccountRow[],
|
||||||
) {
|
) {
|
||||||
const mcpConfig = resolveCollectionMcpConfig(
|
const env = getRuntimeEnv() as unknown as CollectionMcpBindings;
|
||||||
getRuntimeEnv() as unknown as CollectionMcpBindings,
|
const mcpConfig = resolveCollectionMcpConfig(env);
|
||||||
);
|
|
||||||
const baselineRows = mergeExistingFields(rows, accounts);
|
const baselineRows = mergeExistingFields(rows, accounts);
|
||||||
return mapConcurrent(baselineRows, 4, async (baseline) => {
|
return mapConcurrent(baselineRows, 4, async (baseline) => {
|
||||||
const row = baseline;
|
const row = baseline;
|
||||||
@@ -216,6 +217,89 @@ export async function enrichRows(
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type UntaggedAccountRow = {
|
||||||
|
id: string;
|
||||||
|
platform: string;
|
||||||
|
platform_uid: string;
|
||||||
|
profile_url: string;
|
||||||
|
nickname: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export async function tagUntaggedAccounts(
|
||||||
|
rows: Array<Pick<AnalyzedRow, "platform" | "platformUid">>,
|
||||||
|
) {
|
||||||
|
const env = getRuntimeEnv() as unknown as CollectionMcpBindings &
|
||||||
|
AccountTaggingEnv;
|
||||||
|
const toolName = String(env.AI_TAGGING_TOOL ?? "").trim();
|
||||||
|
const refs = new Set(
|
||||||
|
rows
|
||||||
|
.filter((row) => row.platformUid)
|
||||||
|
.map((row) => `${row.platform}:${row.platformUid}`),
|
||||||
|
);
|
||||||
|
if (!toolName || refs.size === 0) return;
|
||||||
|
|
||||||
|
const db = getRawDb();
|
||||||
|
const uids = [...new Set([...refs].map((key) => key.slice(key.indexOf(":") + 1)))];
|
||||||
|
const candidates: UntaggedAccountRow[] = [];
|
||||||
|
for (let index = 0; index < uids.length; index += 100) {
|
||||||
|
const chunk = uids.slice(index, index + 100);
|
||||||
|
const placeholders = chunk.map(() => "?").join(", ");
|
||||||
|
const result = await db
|
||||||
|
.prepare(
|
||||||
|
`SELECT id, platform, platform_uid, profile_url, nickname
|
||||||
|
FROM accounts
|
||||||
|
WHERE (tags = '' OR tags IS NULL)
|
||||||
|
AND profile_url IS NOT NULL AND profile_url != ''
|
||||||
|
AND platform_uid IN (${placeholders})`,
|
||||||
|
)
|
||||||
|
.bind(...chunk)
|
||||||
|
.all<UntaggedAccountRow>();
|
||||||
|
candidates.push(...result.results);
|
||||||
|
}
|
||||||
|
const targets = candidates.filter((account) =>
|
||||||
|
refs.has(`${account.platform}:${account.platform_uid}`),
|
||||||
|
);
|
||||||
|
if (targets.length === 0) return;
|
||||||
|
|
||||||
|
const mcpConfig = resolveCollectionMcpConfig(env);
|
||||||
|
await mapConcurrent(targets, 4, async (account) => {
|
||||||
|
const details = await resolveProfileDetailsFromMcp(
|
||||||
|
account.profile_url,
|
||||||
|
account.platform === "抖音" ? "抖音" : "小红书",
|
||||||
|
mcpConfig,
|
||||||
|
).catch(() => null);
|
||||||
|
if (
|
||||||
|
!details ||
|
||||||
|
(!details.bio &&
|
||||||
|
details.providerTags.length === 0 &&
|
||||||
|
details.recentNoteTitles.length === 0)
|
||||||
|
) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const tags = await generateAccountTagsFromMcp(
|
||||||
|
{
|
||||||
|
platform: account.platform,
|
||||||
|
nickname: details.nickname?.trim() || account.nickname || "",
|
||||||
|
bio: details.bio,
|
||||||
|
gender: details.gender,
|
||||||
|
ipLocation: details.ipLocation ?? "",
|
||||||
|
followers: details.followers,
|
||||||
|
providerTags: details.providerTags,
|
||||||
|
recentNoteTitles: details.recentNoteTitles,
|
||||||
|
},
|
||||||
|
mcpConfig,
|
||||||
|
{ AI_TAGGING_TOOL: toolName },
|
||||||
|
).catch(() => []);
|
||||||
|
if (tags.length === 0) return;
|
||||||
|
await db
|
||||||
|
.prepare(
|
||||||
|
`UPDATE accounts SET tags = ? WHERE id = ? AND (tags = '' OR tags IS NULL)`,
|
||||||
|
)
|
||||||
|
.bind(tags.join(","), account.id)
|
||||||
|
.run();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
export function analyzeRows(
|
export function analyzeRows(
|
||||||
rows: ResourceImportRow[],
|
rows: ResourceImportRow[],
|
||||||
accountRows: AccountRow[],
|
accountRows: AccountRow[],
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ export type RuntimeEnv = {
|
|||||||
FEISHU_APP_SECRET?: string;
|
FEISHU_APP_SECRET?: string;
|
||||||
AI_TOOL_CENTER_MCP_URL?: string;
|
AI_TOOL_CENTER_MCP_URL?: string;
|
||||||
AI_TOOL_CENTER_MCP_KEY?: string;
|
AI_TOOL_CENTER_MCP_KEY?: string;
|
||||||
|
AI_TAGGING_TOOL?: string;
|
||||||
COLLECTION_MCP_URL?: string;
|
COLLECTION_MCP_URL?: string;
|
||||||
COLLECTION_MCP_KEY?: string;
|
COLLECTION_MCP_KEY?: string;
|
||||||
WECOM_CORP_ID?: string;
|
WECOM_CORP_ID?: string;
|
||||||
|
|||||||
221
tests/account-tagging.test.mjs
Normal file
221
tests/account-tagging.test.mjs
Normal file
@@ -0,0 +1,221 @@
|
|||||||
|
import assert from "node:assert/strict";
|
||||||
|
import test from "node:test";
|
||||||
|
import {
|
||||||
|
buildAccountTagText,
|
||||||
|
generateAccountTagsFromMcp,
|
||||||
|
parseAccountTags,
|
||||||
|
} from "../lib/mcp-collection-client.ts";
|
||||||
|
import { needsResourceEnrichment } from "../lib/resource-import.ts";
|
||||||
|
|
||||||
|
function sse(payload, headers = {}) {
|
||||||
|
return new Response(`event: message\ndata: ${JSON.stringify(payload)}\n\n`, {
|
||||||
|
status: 200,
|
||||||
|
headers: {
|
||||||
|
"content-type": "text/event-stream",
|
||||||
|
...headers,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function toolEnvelope(payload, isError = false) {
|
||||||
|
return {
|
||||||
|
jsonrpc: "2.0",
|
||||||
|
id: "tool-call",
|
||||||
|
result: {
|
||||||
|
isError,
|
||||||
|
content: [{ type: "text", text: JSON.stringify(payload) }],
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function createFakeTaggingMcp(toolResult, isError = false) {
|
||||||
|
const calls = [];
|
||||||
|
const fetchImpl = async (url, init) => {
|
||||||
|
const body = JSON.parse(init.body);
|
||||||
|
calls.push({ url: String(url), body });
|
||||||
|
if (body.method === "initialize") {
|
||||||
|
return sse(
|
||||||
|
{
|
||||||
|
jsonrpc: "2.0",
|
||||||
|
id: body.id,
|
||||||
|
result: {
|
||||||
|
protocolVersion: "2025-03-26",
|
||||||
|
capabilities: {},
|
||||||
|
serverInfo: { name: "ai-tool-center", version: "test" },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{ "mcp-session-id": "session-test" },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (body.method === "notifications/initialized") {
|
||||||
|
return new Response("", { status: 202 });
|
||||||
|
}
|
||||||
|
if (body.method === "tools/call") {
|
||||||
|
return sse(toolEnvelope(toolResult, isError));
|
||||||
|
}
|
||||||
|
throw new Error(`Unexpected MCP method: ${body.method}`);
|
||||||
|
};
|
||||||
|
return { fetchImpl, calls };
|
||||||
|
}
|
||||||
|
|
||||||
|
const config = { endpoint: "https://mcp.test/mcp", key: "test-key" };
|
||||||
|
const taggingEnv = { AI_TAGGING_TOOL: "generate_tags" };
|
||||||
|
const input = {
|
||||||
|
platform: "小红书",
|
||||||
|
nickname: "小美的日常",
|
||||||
|
bio: "分享护肤和生活",
|
||||||
|
gender: "女",
|
||||||
|
ipLocation: "上海",
|
||||||
|
followers: 1200,
|
||||||
|
providerTags: ["美妆"],
|
||||||
|
recentNoteTitles: ["平价护肤分享"],
|
||||||
|
};
|
||||||
|
|
||||||
|
test("parseAccountTags 解析纯 JSON 数组", () => {
|
||||||
|
assert.deepEqual(parseAccountTags('["美妆","学生党"]'), ["美妆", "学生党"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("parseAccountTags 剥离代码块围栏", () => {
|
||||||
|
assert.deepEqual(parseAccountTags('```json\n["美食"]\n```'), ["美食"]);
|
||||||
|
assert.deepEqual(parseAccountTags('```\n["旅行"]\n```'), ["旅行"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("parseAccountTags 从夹杂文字中提取数组", () => {
|
||||||
|
const text = '好的,以下是标签:["旅行","摄影"] 希望有帮助';
|
||||||
|
assert.deepEqual(parseAccountTags(text), ["旅行", "摄影"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("parseAccountTags 清洗与过滤", () => {
|
||||||
|
assert.deepEqual(parseAccountTags('["#美食","探店#"]'), ["美食", "探店"]);
|
||||||
|
assert.deepEqual(parseAccountTags('["美妆","美妆","穿搭"]'), [
|
||||||
|
"美妆",
|
||||||
|
"穿搭",
|
||||||
|
]);
|
||||||
|
assert.deepEqual(
|
||||||
|
parseAccountTags('["这个标签实在太长了超过十二个字","护肤"]'),
|
||||||
|
["护肤"],
|
||||||
|
);
|
||||||
|
assert.deepEqual(
|
||||||
|
parseAccountTags('[1, null, "母婴", true]'),
|
||||||
|
["母婴"],
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("parseAccountTags 超过 5 个截断", () => {
|
||||||
|
const text = '["一","二","三","四","五","六"]';
|
||||||
|
assert.deepEqual(parseAccountTags(text), ["一", "二", "三", "四", "五"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("parseAccountTags 无法解析时返回空数组", () => {
|
||||||
|
assert.deepEqual(parseAccountTags("无法生成标签"), []);
|
||||||
|
assert.deepEqual(parseAccountTags(""), []);
|
||||||
|
assert.deepEqual(parseAccountTags('{"tags":"美妆"}'), []);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("buildAccountTagText 包含关键字段", () => {
|
||||||
|
const text = buildAccountTagText(input);
|
||||||
|
assert.ok(text.includes("小美的日常"));
|
||||||
|
assert.ok(text.includes("分享护肤和生活"));
|
||||||
|
assert.ok(text.includes("粉丝数:1200"));
|
||||||
|
assert.ok(text.includes("美妆"));
|
||||||
|
assert.ok(text.includes("平价护肤分享"));
|
||||||
|
});
|
||||||
|
|
||||||
|
test("buildAccountTagText 粉丝数为 0 或空时省略", () => {
|
||||||
|
assert.ok(!buildAccountTagText({ ...input, followers: 0 }).includes("粉丝数"));
|
||||||
|
assert.ok(
|
||||||
|
!buildAccountTagText({ ...input, followers: null }).includes("粉丝数"),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("generateAccountTagsFromMcp 工具名为空时不发起请求", async () => {
|
||||||
|
const fetchImpl = async () => {
|
||||||
|
throw new Error("should not be called");
|
||||||
|
};
|
||||||
|
const tags = await generateAccountTagsFromMcp(
|
||||||
|
input,
|
||||||
|
config,
|
||||||
|
{},
|
||||||
|
fetchImpl,
|
||||||
|
);
|
||||||
|
assert.deepEqual(tags, []);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("generateAccountTagsFromMcp 正常返回标签数组", async () => {
|
||||||
|
const { fetchImpl, calls } = createFakeTaggingMcp(["美妆", "探店", "好物分享"]);
|
||||||
|
const tags = await generateAccountTagsFromMcp(
|
||||||
|
input,
|
||||||
|
config,
|
||||||
|
taggingEnv,
|
||||||
|
fetchImpl,
|
||||||
|
);
|
||||||
|
assert.deepEqual(tags, ["美妆", "探店", "好物分享"]);
|
||||||
|
const toolCall = calls.find((call) => call.body.method === "tools/call");
|
||||||
|
assert.equal(toolCall.body.params.name, "generate_tags");
|
||||||
|
const args = toolCall.body.params.arguments;
|
||||||
|
assert.equal(args.objective, "为KOC账号生成运营标签");
|
||||||
|
assert.ok(args.text.includes("小美的日常"));
|
||||||
|
assert.ok(Array.isArray(args.rules) && args.rules.length > 0);
|
||||||
|
assert.equal(args.output_schema.properties.tags.type, "array");
|
||||||
|
assert.ok(String(toolCall.url).includes("key=test-key"));
|
||||||
|
});
|
||||||
|
|
||||||
|
test("generateAccountTagsFromMcp 支持对象包裹的返回", async () => {
|
||||||
|
const { fetchImpl } = createFakeTaggingMcp({ tags: ["母婴", "育儿"] });
|
||||||
|
const tags = await generateAccountTagsFromMcp(
|
||||||
|
input,
|
||||||
|
config,
|
||||||
|
taggingEnv,
|
||||||
|
fetchImpl,
|
||||||
|
);
|
||||||
|
assert.deepEqual(tags, ["母婴", "育儿"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("generateAccountTagsFromMcp 工具报错时返回空数组", async () => {
|
||||||
|
const { fetchImpl } = createFakeTaggingMcp({ message: "boom" }, true);
|
||||||
|
const tags = await generateAccountTagsFromMcp(
|
||||||
|
input,
|
||||||
|
config,
|
||||||
|
taggingEnv,
|
||||||
|
fetchImpl,
|
||||||
|
);
|
||||||
|
assert.deepEqual(tags, []);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("generateAccountTagsFromMcp 网络异常时返回空数组", async () => {
|
||||||
|
const fetchImpl = async () => {
|
||||||
|
throw new Error("network down");
|
||||||
|
};
|
||||||
|
const tags = await generateAccountTagsFromMcp(
|
||||||
|
input,
|
||||||
|
config,
|
||||||
|
taggingEnv,
|
||||||
|
fetchImpl,
|
||||||
|
);
|
||||||
|
assert.deepEqual(tags, []);
|
||||||
|
});
|
||||||
|
|
||||||
|
const completeRow = {
|
||||||
|
nickname: "小美",
|
||||||
|
publicAccountId: "xhs-123",
|
||||||
|
ipLocation: "上海",
|
||||||
|
followersResolved: true,
|
||||||
|
gender: "女",
|
||||||
|
bio: "分享生活",
|
||||||
|
tags: ["美妆"],
|
||||||
|
};
|
||||||
|
|
||||||
|
test("needsResourceEnrichment 资料齐全且有标签时为 false", () => {
|
||||||
|
assert.equal(needsResourceEnrichment(completeRow), false);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("needsResourceEnrichment 资料齐全但无标签时为 true", () => {
|
||||||
|
assert.equal(needsResourceEnrichment({ ...completeRow, tags: [] }), true);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("needsResourceEnrichment 缺资料且有标签时为 true", () => {
|
||||||
|
assert.equal(
|
||||||
|
needsResourceEnrichment({ ...completeRow, bio: "" }),
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
});
|
||||||
Reference in New Issue
Block a user