feat: KOC 资源导入后自动爬取主页资料并调用 LLM 打标签
- 新增 structured_text_process 打标客户端(文本构造、标签解析清洗、MCP 调用) - 写库后后台打标(tagUntaggedAccounts),单条新增 / 批量导入 / 每日补全三个入口统一 - 标签只补空,UPDATE 带 tags 为空守卫,不覆盖已有标签 - 新增 AI_TAGGING_TOOL 配置项(留空关闭),更新配置示例与文档 - 新增 16 个打标测试用例,全量 131 个测试通过
This commit is contained in:
@@ -1,8 +1,10 @@
|
||||
import { ensureSchema, getRawDb } from "./mvp-db";
|
||||
import {
|
||||
generateAccountTagsFromMcp,
|
||||
resolveCollectionMcpConfig,
|
||||
resolveProfileDetailsFromMcp,
|
||||
resolveXhsPublicAccountDetails,
|
||||
type AccountTaggingEnv,
|
||||
type CollectionMcpBindings,
|
||||
} from "./mcp-collection-client";
|
||||
import { getRuntimeEnv } from "./runtime-env";
|
||||
@@ -135,9 +137,8 @@ export async function enrichRows(
|
||||
rows: ResourceImportRow[],
|
||||
accounts: AccountRow[],
|
||||
) {
|
||||
const mcpConfig = resolveCollectionMcpConfig(
|
||||
getRuntimeEnv() as unknown as CollectionMcpBindings,
|
||||
);
|
||||
const env = getRuntimeEnv() as unknown as CollectionMcpBindings;
|
||||
const mcpConfig = resolveCollectionMcpConfig(env);
|
||||
const baselineRows = mergeExistingFields(rows, accounts);
|
||||
return mapConcurrent(baselineRows, 4, async (baseline) => {
|
||||
const row = baseline;
|
||||
@@ -216,6 +217,89 @@ export async function enrichRows(
|
||||
});
|
||||
}
|
||||
|
||||
type UntaggedAccountRow = {
|
||||
id: string;
|
||||
platform: string;
|
||||
platform_uid: string;
|
||||
profile_url: string;
|
||||
nickname: string;
|
||||
};
|
||||
|
||||
export async function tagUntaggedAccounts(
|
||||
rows: Array<Pick<AnalyzedRow, "platform" | "platformUid">>,
|
||||
) {
|
||||
const env = getRuntimeEnv() as unknown as CollectionMcpBindings &
|
||||
AccountTaggingEnv;
|
||||
const toolName = String(env.AI_TAGGING_TOOL ?? "").trim();
|
||||
const refs = new Set(
|
||||
rows
|
||||
.filter((row) => row.platformUid)
|
||||
.map((row) => `${row.platform}:${row.platformUid}`),
|
||||
);
|
||||
if (!toolName || refs.size === 0) return;
|
||||
|
||||
const db = getRawDb();
|
||||
const uids = [...new Set([...refs].map((key) => key.slice(key.indexOf(":") + 1)))];
|
||||
const candidates: UntaggedAccountRow[] = [];
|
||||
for (let index = 0; index < uids.length; index += 100) {
|
||||
const chunk = uids.slice(index, index + 100);
|
||||
const placeholders = chunk.map(() => "?").join(", ");
|
||||
const result = await db
|
||||
.prepare(
|
||||
`SELECT id, platform, platform_uid, profile_url, nickname
|
||||
FROM accounts
|
||||
WHERE (tags = '' OR tags IS NULL)
|
||||
AND profile_url IS NOT NULL AND profile_url != ''
|
||||
AND platform_uid IN (${placeholders})`,
|
||||
)
|
||||
.bind(...chunk)
|
||||
.all<UntaggedAccountRow>();
|
||||
candidates.push(...result.results);
|
||||
}
|
||||
const targets = candidates.filter((account) =>
|
||||
refs.has(`${account.platform}:${account.platform_uid}`),
|
||||
);
|
||||
if (targets.length === 0) return;
|
||||
|
||||
const mcpConfig = resolveCollectionMcpConfig(env);
|
||||
await mapConcurrent(targets, 4, async (account) => {
|
||||
const details = await resolveProfileDetailsFromMcp(
|
||||
account.profile_url,
|
||||
account.platform === "抖音" ? "抖音" : "小红书",
|
||||
mcpConfig,
|
||||
).catch(() => null);
|
||||
if (
|
||||
!details ||
|
||||
(!details.bio &&
|
||||
details.providerTags.length === 0 &&
|
||||
details.recentNoteTitles.length === 0)
|
||||
) {
|
||||
return;
|
||||
}
|
||||
const tags = await generateAccountTagsFromMcp(
|
||||
{
|
||||
platform: account.platform,
|
||||
nickname: details.nickname?.trim() || account.nickname || "",
|
||||
bio: details.bio,
|
||||
gender: details.gender,
|
||||
ipLocation: details.ipLocation ?? "",
|
||||
followers: details.followers,
|
||||
providerTags: details.providerTags,
|
||||
recentNoteTitles: details.recentNoteTitles,
|
||||
},
|
||||
mcpConfig,
|
||||
{ AI_TAGGING_TOOL: toolName },
|
||||
).catch(() => []);
|
||||
if (tags.length === 0) return;
|
||||
await db
|
||||
.prepare(
|
||||
`UPDATE accounts SET tags = ? WHERE id = ? AND (tags = '' OR tags IS NULL)`,
|
||||
)
|
||||
.bind(tags.join(","), account.id)
|
||||
.run();
|
||||
});
|
||||
}
|
||||
|
||||
export function analyzeRows(
|
||||
rows: ResourceImportRow[],
|
||||
accountRows: AccountRow[],
|
||||
|
||||
Reference in New Issue
Block a user