feat: KOC 资源导入后自动爬取主页资料并调用 LLM 打标签

- 新增 structured_text_process 打标客户端(文本构造、标签解析清洗、MCP 调用)
- 写库后后台打标(tagUntaggedAccounts),单条新增 / 批量导入 / 每日补全三个入口统一
- 标签只补空,UPDATE 带 tags 为空守卫,不覆盖已有标签
- 新增 AI_TAGGING_TOOL 配置项(留空关闭),更新配置示例与文档
- 新增 16 个打标测试用例,全量 131 个测试通过
This commit is contained in:
ABAPPLO
2026-08-26 15:21:28 +08:00
parent 9697b5890d
commit 4a64feee02
13 changed files with 557 additions and 11 deletions

View File

@@ -1,8 +1,10 @@
import { ensureSchema, getRawDb } from "./mvp-db";
import {
generateAccountTagsFromMcp,
resolveCollectionMcpConfig,
resolveProfileDetailsFromMcp,
resolveXhsPublicAccountDetails,
type AccountTaggingEnv,
type CollectionMcpBindings,
} from "./mcp-collection-client";
import { getRuntimeEnv } from "./runtime-env";
@@ -135,9 +137,8 @@ export async function enrichRows(
rows: ResourceImportRow[],
accounts: AccountRow[],
) {
const mcpConfig = resolveCollectionMcpConfig(
getRuntimeEnv() as unknown as CollectionMcpBindings,
);
const env = getRuntimeEnv() as unknown as CollectionMcpBindings;
const mcpConfig = resolveCollectionMcpConfig(env);
const baselineRows = mergeExistingFields(rows, accounts);
return mapConcurrent(baselineRows, 4, async (baseline) => {
const row = baseline;
@@ -216,6 +217,89 @@ export async function enrichRows(
});
}
type UntaggedAccountRow = {
id: string;
platform: string;
platform_uid: string;
profile_url: string;
nickname: string;
};
export async function tagUntaggedAccounts(
rows: Array<Pick<AnalyzedRow, "platform" | "platformUid">>,
) {
const env = getRuntimeEnv() as unknown as CollectionMcpBindings &
AccountTaggingEnv;
const toolName = String(env.AI_TAGGING_TOOL ?? "").trim();
const refs = new Set(
rows
.filter((row) => row.platformUid)
.map((row) => `${row.platform}:${row.platformUid}`),
);
if (!toolName || refs.size === 0) return;
const db = getRawDb();
const uids = [...new Set([...refs].map((key) => key.slice(key.indexOf(":") + 1)))];
const candidates: UntaggedAccountRow[] = [];
for (let index = 0; index < uids.length; index += 100) {
const chunk = uids.slice(index, index + 100);
const placeholders = chunk.map(() => "?").join(", ");
const result = await db
.prepare(
`SELECT id, platform, platform_uid, profile_url, nickname
FROM accounts
WHERE (tags = '' OR tags IS NULL)
AND profile_url IS NOT NULL AND profile_url != ''
AND platform_uid IN (${placeholders})`,
)
.bind(...chunk)
.all<UntaggedAccountRow>();
candidates.push(...result.results);
}
const targets = candidates.filter((account) =>
refs.has(`${account.platform}:${account.platform_uid}`),
);
if (targets.length === 0) return;
const mcpConfig = resolveCollectionMcpConfig(env);
await mapConcurrent(targets, 4, async (account) => {
const details = await resolveProfileDetailsFromMcp(
account.profile_url,
account.platform === "抖音" ? "抖音" : "小红书",
mcpConfig,
).catch(() => null);
if (
!details ||
(!details.bio &&
details.providerTags.length === 0 &&
details.recentNoteTitles.length === 0)
) {
return;
}
const tags = await generateAccountTagsFromMcp(
{
platform: account.platform,
nickname: details.nickname?.trim() || account.nickname || "",
bio: details.bio,
gender: details.gender,
ipLocation: details.ipLocation ?? "",
followers: details.followers,
providerTags: details.providerTags,
recentNoteTitles: details.recentNoteTitles,
},
mcpConfig,
{ AI_TAGGING_TOOL: toolName },
).catch(() => []);
if (tags.length === 0) return;
await db
.prepare(
`UPDATE accounts SET tags = ? WHERE id = ? AND (tags = '' OR tags IS NULL)`,
)
.bind(tags.join(","), account.id)
.run();
});
}
export function analyzeRows(
rows: ResourceImportRow[],
accountRows: AccountRow[],