feat: KOC 资源导入后自动爬取主页资料并调用 LLM 打标签

- 新增 structured_text_process 打标客户端（文本构造、标签解析清洗、MCP 调用） - 写库后后台打标（tagUntaggedAccounts），单条新增 / 批量导入 / 每日补全三个入口统一 - 标签只补空，UPDATE 带 tags 为空守卫，不覆盖已有标签 - 新增 AI_TAGGING_TOOL 配置项（留空关闭），更新配置示例与文档 - 新增 16 个打标测试用例，全量 131 个测试通过
2026-08-26 15:21:28 +08:00
parent 9697b5890d
commit 4a64feee02
13 changed files with 557 additions and 11 deletions
--- a/lib/account-enrichment-service.ts
+++ b/lib/account-enrichment-service.ts
@@ -1,9 +1,12 @@
 import {
+  generateAccountTagsFromMcp,
  resolveXhsPublicAccountDetails,
  resolveAccountProfileFromMcp,
  resolveProfileDetailsFromMcp,
+  type AccountTaggingEnv,
  type CollectionMcpConfig,
 } from "./mcp-collection-client";
+import { getRuntimeEnv } from "./runtime-env";
 import { hashText } from "./mvp-db";
 import type { DatabaseClient } from "./database";

@@ -260,6 +263,7 @@ export async function backfillAccountProfiles(
  mcpConfig: CollectionMcpConfig,
  limit = 10,
 ) {
+  const taggingEnv: AccountTaggingEnv = getRuntimeEnv();
  const rows = await db
    .prepare(
      `SELECT
@@ -334,6 +338,7 @@ export async function backfillAccountProfiles(
    const isDemoAccount = row.platform_uid?.startsWith("xhs-");
    let attemptedThisRow = false;
    let publicProfileUpdated = false;
+    let tagsUpdated = false;
    if (
      row.platform === "小红书" &&
      !isDemoAccount &&
@@ -341,7 +346,8 @@ export async function backfillAccountProfiles(
      (!row.public_account_id ||
        Number(row.followers ?? 0) === 0 ||
        !row.gender ||
-        !row.bio)
+        !row.bio ||
+        !row.tags?.trim())
    ) {
      attempted += 1;
      attemptedThisRow = true;
@@ -402,6 +408,39 @@ export async function backfillAccountProfiles(
          .run();
        publicProfileUpdated = true;
      }
+      if (
+        row.account_id &&
+        !row.tags?.trim() &&
+        (details.bio ||
+          details.providerTags.length > 0 ||
+          details.recentNoteTitles.length > 0)
+      ) {
+        const tags = await generateAccountTagsFromMcp(
+          {
+            platform: "小红书",
+            nickname: row.nickname || details.nickname || "",
+            bio: details.bio,
+            gender: details.gender,
+            ipLocation: details.ipLocation ?? "",
+            followers: details.followers,
+            providerTags: details.providerTags,
+            recentNoteTitles: details.recentNoteTitles,
+          },
+          mcpConfig,
+          taggingEnv,
+        ).catch(() => []);
+        if (tags.length > 0) {
+          await db
+            .prepare(
+              `UPDATE accounts
+               SET tags = CASE WHEN ? != '' THEN ? ELSE tags END
+               WHERE id = ?`,
+            )
+            .bind(tags.join(","), tags.join(","), row.account_id)
+            .run();
+          tagsUpdated = true;
+        }
+      }
      if (
        (details.redId || row.public_account_id) &&
        Number(details.followers ?? row.followers ?? 0) > 0
@@ -410,7 +449,7 @@ export async function backfillAccountProfiles(
        continue;
      }
      if (attempted >= Math.max(1, Math.min(25, limit))) {
-        if (publicProfileUpdated) updated += 1;
+        if (publicProfileUpdated || tagsUpdated) updated += 1;
        continue;
      }
    }
@@ -430,7 +469,7 @@ export async function backfillAccountProfiles(
      row.platform_uid?.startsWith("pending-") ||
      Boolean(noteId && row.platform_uid === noteId));
    if (!needsEnrichment || !row.publish_url) {
-      if (publicProfileUpdated) updated += 1;
+      if (publicProfileUpdated || tagsUpdated) updated += 1;
      continue;
    }

@@ -443,9 +482,9 @@ export async function backfillAccountProfiles(
        row.nickname || "待识别账号",
        mcpConfig,
      );
-      if (result.updated || publicProfileUpdated) updated += 1;
+      if (result.updated || publicProfileUpdated || tagsUpdated) updated += 1;
    } catch {
-      if (publicProfileUpdated) updated += 1;
+      if (publicProfileUpdated || tagsUpdated) updated += 1;
      else failed += 1;
    }
  }
--- a/lib/mcp-collection-client.ts
+++ b/lib/mcp-collection-client.ts
@@ -1484,3 +1484,154 @@ export async function resolveXhsProfileDetailsFromMcp(
    ? lastError
    : new Error("账号主页数据采集服务暂时不可用");
 }
+
+export type AccountTaggingInput = {
+  platform: string;
+  nickname: string;
+  bio: string;
+  gender: string;
+  ipLocation: string;
+  followers: number | null;
+  providerTags: string[];
+  recentNoteTitles: string[];
+};
+
+export type AccountTaggingEnv = {
+  AI_TAGGING_TOOL?: string;
+};
+
+export function buildAccountTagText(input: AccountTaggingInput): string {
+  const facts: string[] = [`平台：${input.platform.trim() || "小红书"}`];
+  if (input.nickname.trim()) facts.push(`昵称：${input.nickname.trim()}`);
+  if (input.bio.trim()) facts.push(`简介：${input.bio.trim()}`);
+  if (input.gender.trim()) facts.push(`性别：${input.gender.trim()}`);
+  if (input.ipLocation.trim()) {
+    facts.push(`IP属地：${input.ipLocation.trim()}`);
+  }
+  if (input.followers !== null && input.followers > 0) {
+    facts.push(`粉丝数：${input.followers}`);
+  }
+  if (input.providerTags.length > 0) {
+    facts.push(`平台标签：${input.providerTags.slice(0, 5).join("、")}`);
+  }
+  if (input.recentNoteTitles.length > 0) {
+    const titles = input.recentNoteTitles
+      .slice(0, 20)
+      .map((title) => `《${title}》`)
+      .join("");
+    facts.push(`最近笔记标题：${titles}`);
+  }
+  return facts.join("\n");
+}
+
+function sanitizeTagList(items: unknown[]): string[] {
+  const tags: string[] = [];
+  for (const item of items) {
+    if (typeof item !== "string") continue;
+    const tag = item.trim().replace(/^#+|#+$/g, "").trim();
+    if (!tag || tag.length > 12) continue;
+    if (!tags.includes(tag)) tags.push(tag);
+  }
+  return tags.slice(0, 5);
+}
+
+export function parseAccountTags(text: string): string[] {
+  const stripped = text
+    .trim()
+    .replace(/^```(?:json)?\s*/i, "")
+    .replace(/\s*```\s*$/, "");
+  const candidates = [stripped];
+  const arrayMatch = stripped.match(/\[[\s\S]*\]/);
+  if (arrayMatch) candidates.push(arrayMatch[0]);
+  for (const candidate of candidates) {
+    let parsed: unknown;
+    try {
+      parsed = JSON.parse(candidate);
+    } catch {
+      continue;
+    }
+    if (!Array.isArray(parsed)) continue;
+    const tags = sanitizeTagList(parsed);
+    if (tags.length > 0) return tags;
+  }
+  return [];
+}
+
+function parseTagsFromToolPayload(payload: unknown, depth = 0): string[] {
+  if (typeof payload === "string") return parseAccountTags(payload);
+  if (Array.isArray(payload)) {
+    const direct = sanitizeTagList(payload);
+    if (direct.length > 0) return direct;
+    if (depth < 2) {
+      for (const item of payload) {
+        const tags = parseTagsFromToolPayload(item, depth + 1);
+        if (tags.length > 0) return tags;
+      }
+    }
+    return [];
+  }
+  const record = asRecord(payload);
+  if (record && depth < 2) {
+    for (const key of [
+      "tags",
+      "labels",
+      "result",
+      "data",
+      "response",
+      "output",
+      "text",
+      "content",
+      "message",
+    ]) {
+      if (key in record) {
+        const tags = parseTagsFromToolPayload(record[key], depth + 1);
+        if (tags.length > 0) return tags;
+      }
+    }
+  }
+  return [];
+}
+
+export async function generateAccountTagsFromMcp(
+  input: AccountTaggingInput,
+  config: CollectionMcpConfig,
+  taggingEnv: AccountTaggingEnv,
+  fetchImpl: typeof fetch = fetch,
+): Promise<string[]> {
+  const toolName = String(taggingEnv.AI_TAGGING_TOOL ?? "").trim();
+  if (!toolName || !config.endpoint.trim()) return [];
+  try {
+    const endpoint = buildMcpUrl(config);
+    const timeoutMs = Math.max(5_000, config.timeoutMs ?? 60_000);
+    const sessionId = await createMcpSession(fetchImpl, endpoint, timeoutMs);
+    const result = await invokeMcpTool(
+      fetchImpl,
+      endpoint,
+      sessionId,
+      timeoutMs,
+      toolName,
+      {
+        objective: "为KOC账号生成运营标签",
+        text: buildAccountTagText(input),
+        rules: [
+          "最多生成 5 个标签",
+          "每个标签不超过 8 个字",
+          "从内容领域、人设风格、受众画像等维度概括",
+          "只依据文本中给到的资料，不要编造",
+        ],
+        output_schema: {
+          type: "object",
+          properties: {
+            tags: { type: "array", items: { type: "string" } },
+          },
+          required: ["tags"],
+          additionalProperties: false,
+        },
+      },
+    );
+    if (result.isError) return [];
+    return parseTagsFromToolPayload(result.payload);
+  } catch {
+    return [];
+  }
+}
--- a/lib/resource-import.ts
+++ b/lib/resource-import.ts
@@ -279,6 +279,21 @@ export function resourceImportMissingFields(
  return missing;
 }

+export function needsResourceEnrichment(
+  row: Pick<
+    ResourceImportRow,
+    | "nickname"
+    | "publicAccountId"
+    | "ipLocation"
+    | "followersResolved"
+    | "gender"
+    | "bio"
+    | "tags"
+  >,
+) {
+  return resourceImportMissingFields(row).length > 0 || row.tags.length === 0;
+}
+
 function normalizeRows(rows: string[][]) {
  const header = findHeader(rows);
  if (!header) {
--- a/lib/resource-write.ts
+++ b/lib/resource-write.ts
@@ -1,8 +1,10 @@
 import { ensureSchema, getRawDb } from "./mvp-db";
 import {
+  generateAccountTagsFromMcp,
  resolveCollectionMcpConfig,
  resolveProfileDetailsFromMcp,
  resolveXhsPublicAccountDetails,
+  type AccountTaggingEnv,
  type CollectionMcpBindings,
 } from "./mcp-collection-client";
 import { getRuntimeEnv } from "./runtime-env";
@@ -135,9 +137,8 @@ export async function enrichRows(
  rows: ResourceImportRow[],
  accounts: AccountRow[],
 ) {
-  const mcpConfig = resolveCollectionMcpConfig(
-    getRuntimeEnv() as unknown as CollectionMcpBindings,
-  );
+  const env = getRuntimeEnv() as unknown as CollectionMcpBindings;
+  const mcpConfig = resolveCollectionMcpConfig(env);
  const baselineRows = mergeExistingFields(rows, accounts);
  return mapConcurrent(baselineRows, 4, async (baseline) => {
    const row = baseline;
@@ -216,6 +217,89 @@ export async function enrichRows(
  });
 }

+type UntaggedAccountRow = {
+  id: string;
+  platform: string;
+  platform_uid: string;
+  profile_url: string;
+  nickname: string;
+};
+
+export async function tagUntaggedAccounts(
+  rows: Array<Pick<AnalyzedRow, "platform" | "platformUid">>,
+) {
+  const env = getRuntimeEnv() as unknown as CollectionMcpBindings &
+    AccountTaggingEnv;
+  const toolName = String(env.AI_TAGGING_TOOL ?? "").trim();
+  const refs = new Set(
+    rows
+      .filter((row) => row.platformUid)
+      .map((row) => `${row.platform}:${row.platformUid}`),
+  );
+  if (!toolName || refs.size === 0) return;
+
+  const db = getRawDb();
+  const uids = [...new Set([...refs].map((key) => key.slice(key.indexOf(":") + 1)))];
+  const candidates: UntaggedAccountRow[] = [];
+  for (let index = 0; index < uids.length; index += 100) {
+    const chunk = uids.slice(index, index + 100);
+    const placeholders = chunk.map(() => "?").join(", ");
+    const result = await db
+      .prepare(
+        `SELECT id, platform, platform_uid, profile_url, nickname
+         FROM accounts
+         WHERE (tags = '' OR tags IS NULL)
+           AND profile_url IS NOT NULL AND profile_url != ''
+           AND platform_uid IN (${placeholders})`,
+      )
+      .bind(...chunk)
+      .all<UntaggedAccountRow>();
+    candidates.push(...result.results);
+  }
+  const targets = candidates.filter((account) =>
+    refs.has(`${account.platform}:${account.platform_uid}`),
+  );
+  if (targets.length === 0) return;
+
+  const mcpConfig = resolveCollectionMcpConfig(env);
+  await mapConcurrent(targets, 4, async (account) => {
+    const details = await resolveProfileDetailsFromMcp(
+      account.profile_url,
+      account.platform === "抖音" ? "抖音" : "小红书",
+      mcpConfig,
+    ).catch(() => null);
+    if (
+      !details ||
+      (!details.bio &&
+        details.providerTags.length === 0 &&
+        details.recentNoteTitles.length === 0)
+    ) {
+      return;
+    }
+    const tags = await generateAccountTagsFromMcp(
+      {
+        platform: account.platform,
+        nickname: details.nickname?.trim() || account.nickname || "",
+        bio: details.bio,
+        gender: details.gender,
+        ipLocation: details.ipLocation ?? "",
+        followers: details.followers,
+        providerTags: details.providerTags,
+        recentNoteTitles: details.recentNoteTitles,
+      },
+      mcpConfig,
+      { AI_TAGGING_TOOL: toolName },
+    ).catch(() => []);
+    if (tags.length === 0) return;
+    await db
+      .prepare(
+        `UPDATE accounts SET tags = ? WHERE id = ? AND (tags = '' OR tags IS NULL)`,
+      )
+      .bind(tags.join(","), account.id)
+      .run();
+  });
+}
+
 export function analyzeRows(
  rows: ResourceImportRow[],
  accountRows: AccountRow[],
--- a/lib/runtime-env.ts
+++ b/lib/runtime-env.ts
@@ -18,6 +18,7 @@ export type RuntimeEnv = {
  FEISHU_APP_SECRET?: string;
  AI_TOOL_CENTER_MCP_URL?: string;
  AI_TOOL_CENTER_MCP_KEY?: string;
+  AI_TAGGING_TOOL?: string;
  COLLECTION_MCP_URL?: string;
  COLLECTION_MCP_KEY?: string;
  WECOM_CORP_ID?: string;