feat: 完善视频任务与 KOC 资源库

This commit is contained in:
巫凤萍
2026-08-15 03:53:09 +08:00
parent ad3dbdcc86
commit f37d05dd88
66 changed files with 6633 additions and 558 deletions

View File

@@ -1,7 +1,7 @@
import {
resolveXhsPublicAccountDetails,
resolveXhsAccountProfileFromMcp,
resolveXhsProfileDetailsFromMcp,
resolveAccountProfileFromMcp,
resolveProfileDetailsFromMcp,
type CollectionMcpConfig,
} from "./mcp-collection-client";
import { hashText } from "./mvp-db";
@@ -11,15 +11,20 @@ type DistributionAccountRow = {
id: string;
account_id: string | null;
publish_url: string | null;
platform: string;
claimant_contact: string | null;
};
type BackfillRow = DistributionAccountRow & {
resolved_account_id: string | null;
nickname: string | null;
platform: string | null;
platform_uid: string | null;
public_account_id: string | null;
profile_url: string | null;
followers: number | null;
gender: string | null;
bio: string | null;
tags: string | null;
};
function isVerifiedXhsProfileUrl(value: string | null) {
@@ -37,6 +42,23 @@ function isVerifiedXhsProfileUrl(value: string | null) {
}
}
function isVerifiedDouyinProfileUrl(value: string | null) {
if (!value) return false;
try {
const url = new URL(value);
const secUid = decodeURIComponent(url.pathname.match(/^\/user\/([^/?#]+)/)?.[1] ?? "");
return (
url.protocol === "https:" &&
(url.hostname === "douyin.com" ||
url.hostname.endsWith(".douyin.com")) &&
/^[A-Za-z0-9_-]{20,220}$/.test(secUid) &&
!/^\d+$/.test(secUid)
);
} catch {
return false;
}
}
export async function enrichDistributionAccount(
db: DatabaseClient,
distributionId: string,
@@ -44,27 +66,42 @@ export async function enrichDistributionAccount(
fallbackNickname: string,
mcpConfig: CollectionMcpConfig,
) {
const profile = await resolveXhsAccountProfileFromMcp(
publishUrl,
fallbackNickname,
mcpConfig,
);
const current = await db
.prepare(
`SELECT id, account_id, publish_url
FROM distributions
WHERE id = ?`,
`SELECT d.id, d.account_id, d.publish_url, t.platform,
cl.claimant_name AS claimant_contact
FROM distributions d
JOIN tasks t ON t.id = d.task_id
LEFT JOIN claims cl ON cl.id = d.claim_id
WHERE d.id = ?`,
)
.bind(distributionId)
.first<DistributionAccountRow>();
if (!current || current.publish_url !== publishUrl) {
return { updated: false, reason: "stale" as const };
}
const platform = current.platform === "抖音" ? "抖音" : "小红书";
const profile = await resolveAccountProfileFromMcp(
publishUrl,
fallbackNickname,
platform,
mcpConfig,
);
const canonicalAccountId = `account-${hashText(
`小红书:${profile.platformUid}`,
`${platform}:${profile.platformUid}`,
)}`;
if (current.account_id === canonicalAccountId) {
const existingAccount = await db
.prepare(
`SELECT id FROM accounts
WHERE platform = ? AND platform_uid = ?
LIMIT 1`,
)
.bind(platform, profile.platformUid)
.first<{ id: string }>();
const targetAccountId = existingAccount?.id || canonicalAccountId;
const currentContact = (current.claimant_contact || "").trim();
if (current.account_id === targetAccountId) {
await db
.prepare(
`UPDATE accounts SET
@@ -82,6 +119,12 @@ export async function enrichDistributionAccount(
WHEN ? IS NOT NULL AND (? > 0 OR followers = 0) THEN ?
ELSE followers
END,
gender = CASE WHEN ? != '' THEN ? ELSE gender END,
bio = CASE WHEN ? != '' THEN ? ELSE bio END,
current_contact = CASE WHEN ? != '' THEN ? ELSE current_contact END,
post_count = (
SELECT COUNT(*) FROM distributions WHERE account_id = ?
),
last_seen_at = CURRENT_TIMESTAMP
WHERE id = ?`,
)
@@ -96,12 +139,19 @@ export async function enrichDistributionAccount(
profile.followers,
profile.followers,
profile.followers,
canonicalAccountId,
profile.gender,
profile.gender,
profile.bio,
profile.bio,
currentContact,
currentContact,
targetAccountId,
targetAccountId,
)
.run();
return {
updated: true,
accountId: canonicalAccountId,
accountId: targetAccountId,
profileUrl: profile.profileUrl,
};
}
@@ -111,8 +161,9 @@ export async function enrichDistributionAccount(
db
.prepare(
`INSERT INTO accounts
(id, platform, platform_uid, public_account_id, nickname, profile_url, ip_location, followers, post_count)
VALUES (?, '小红书', ?, ?, ?, ?, ?, ?, 1)
(id, platform, platform_uid, public_account_id, nickname, profile_url,
ip_location, followers, gender, bio, current_contact, post_count)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1)
ON CONFLICT(platform, platform_uid) DO UPDATE SET
public_account_id = CASE
WHEN excluded.public_account_id != ''
@@ -131,17 +182,29 @@ export async function enrichDistributionAccount(
THEN excluded.followers
ELSE accounts.followers
END,
post_count = accounts.post_count + 1,
gender = CASE
WHEN excluded.gender != '' THEN excluded.gender ELSE accounts.gender END,
bio = CASE
WHEN excluded.bio != '' THEN excluded.bio ELSE accounts.bio END,
current_contact = CASE
WHEN excluded.current_contact != ''
THEN excluded.current_contact
ELSE accounts.current_contact
END,
last_seen_at = CURRENT_TIMESTAMP`,
)
.bind(
canonicalAccountId,
targetAccountId,
platform,
profile.platformUid,
profile.redId,
profile.nickname || fallbackNickname,
profile.profileUrl,
profile.ipLocation,
profile.followers ?? 0,
profile.gender,
profile.bio,
currentContact,
),
db
.prepare(
@@ -150,10 +213,25 @@ export async function enrichDistributionAccount(
updated_at = CURRENT_TIMESTAMP
WHERE id = ? AND publish_url = ?`,
)
.bind(canonicalAccountId, distributionId, publishUrl),
.bind(targetAccountId, distributionId, publishUrl),
db
.prepare(
`UPDATE accounts SET post_count = (
SELECT COUNT(*) FROM distributions WHERE account_id = ?
) WHERE id = ?`,
)
.bind(targetAccountId, targetAccountId),
]);
if (provisionalAccountId) {
if (provisionalAccountId && provisionalAccountId !== targetAccountId) {
await db
.prepare(
`UPDATE accounts SET post_count = (
SELECT COUNT(*) FROM distributions WHERE account_id = ?
) WHERE id = ?`,
)
.bind(provisionalAccountId, provisionalAccountId)
.run();
await db
.prepare(
`DELETE FROM accounts
@@ -165,14 +243,14 @@ export async function enrichDistributionAccount(
)
.bind(
provisionalAccountId,
canonicalAccountId,
targetAccountId,
provisionalAccountId,
)
.run();
}
return {
updated: true,
accountId: canonicalAccountId,
accountId: targetAccountId,
profileUrl: profile.profileUrl,
};
}
@@ -188,15 +266,22 @@ export async function backfillAccountProfiles(
d.id,
d.account_id,
d.publish_url,
a.id AS resolved_account_id,
a.nickname,
a.platform,
COALESCE(a.platform, t.platform) AS platform,
a.platform_uid,
a.public_account_id,
a.profile_url,
a.followers
a.followers,
a.gender,
a.bio,
a.tags,
cl.claimant_name AS claimant_contact
FROM distributions d
JOIN tasks t ON t.id = d.task_id
LEFT JOIN accounts a ON a.id = d.account_id
WHERE d.publish_url IS NOT NULL
LEFT JOIN claims cl ON cl.id = d.claim_id
WHERE d.publish_url IS NOT NULL
AND d.publish_url != ''
ORDER BY d.updated_at DESC
LIMIT 100`,
@@ -205,8 +290,34 @@ export async function backfillAccountProfiles(
let attempted = 0;
let updated = 0;
let failed = 0;
const backfilledAccounts = new Set<string>();
for (const row of rows.results) {
if (
row.resolved_account_id &&
!backfilledAccounts.has(row.resolved_account_id)
) {
backfilledAccounts.add(row.resolved_account_id);
const claimantContact = row.claimant_contact?.trim() || "";
await db
.prepare(
`UPDATE accounts
SET current_contact = CASE
WHEN ? != '' THEN ? ELSE current_contact
END,
post_count = (
SELECT COUNT(*) FROM distributions WHERE account_id = ?
)
WHERE id = ?`,
)
.bind(
claimantContact,
claimantContact,
row.resolved_account_id,
row.resolved_account_id,
)
.run();
}
if (attempted >= Math.max(1, Math.min(25, limit))) break;
const noteId = (() => {
try {
@@ -227,19 +338,31 @@ export async function backfillAccountProfiles(
row.platform === "小红书" &&
!isDemoAccount &&
isVerifiedXhsProfileUrl(row.profile_url) &&
(!row.public_account_id || Number(row.followers ?? 0) === 0)
(!row.public_account_id ||
Number(row.followers ?? 0) === 0 ||
!row.gender ||
!row.bio)
) {
attempted += 1;
attemptedThisRow = true;
const details = await resolveXhsProfileDetailsFromMcp(
const details = await resolveProfileDetailsFromMcp(
row.profile_url ?? "",
"小红书",
mcpConfig,
).catch(() =>
resolveXhsPublicAccountDetails(row.profile_url ?? ""),
);
).catch(async () => ({
...(await resolveXhsPublicAccountDetails(row.profile_url ?? "")),
gender: "" as const,
bio: "",
recentNoteTitles: [] as string[],
providerTags: [] as string[],
}));
if (
row.account_id &&
(details.redId || details.followers !== null)
(details.redId ||
details.followers !== null ||
details.gender ||
details.bio ||
details.recentNoteTitles.length > 0)
) {
await db
.prepare(
@@ -256,6 +379,8 @@ export async function backfillAccountProfiles(
WHEN ? != '' AND ? != '待识别' THEN ?
ELSE ip_location
END,
gender = CASE WHEN ? != '' THEN ? ELSE gender END,
bio = CASE WHEN ? != '' THEN ? ELSE bio END,
last_seen_at = CURRENT_TIMESTAMP
WHERE id = ?`,
)
@@ -268,6 +393,10 @@ export async function backfillAccountProfiles(
details.ipLocation ?? "",
details.ipLocation ?? "",
details.ipLocation ?? "",
details.gender,
details.gender,
details.bio,
details.bio,
row.account_id,
)
.run();
@@ -287,12 +416,17 @@ export async function backfillAccountProfiles(
}
const needsEnrichment =
!isDemoAccount &&
(!row.account_id ||
(!row.resolved_account_id ||
(row.platform === "小红书" &&
!isVerifiedXhsProfileUrl(row.profile_url)) ||
(row.platform === "抖音" &&
!isVerifiedDouyinProfileUrl(row.profile_url)) ||
(row.platform === "小红书" && !row.public_account_id) ||
(row.platform === "抖音" && !row.public_account_id) ||
(row.platform === "小红书" &&
Number(row.followers ?? 0) === 0) ||
(row.platform === "抖音" &&
Number(row.followers ?? 0) === 0) ||
row.platform_uid?.startsWith("pending-") ||
Boolean(noteId && row.platform_uid === noteId));
if (!needsEnrichment || !row.publish_url) {

View File

@@ -1,5 +1,5 @@
import {
collectXhsMetricsFromMcp,
collectMetricsFromMcp,
type CollectionMcpConfig,
} from "./mcp-collection-client";
import { uid } from "./mvp-db";
@@ -10,6 +10,7 @@ type DistributionForCollection = {
task_id: string;
publish_url: string | null;
ocr_status: string;
platform: string;
};
type ScheduledTask = {
@@ -165,11 +166,16 @@ export async function collectDistributionMetrics(
mcpConfig: CollectionMcpConfig,
) {
const current = await db
.prepare("SELECT * FROM distributions WHERE id = ?")
.prepare(
`SELECT d.*, t.platform
FROM distributions d
JOIN tasks t ON t.id = d.task_id
WHERE d.id = ?`,
)
.bind(distributionId)
.first<DistributionForCollection>();
if (!current) throw new Error("分发记录不存在");
if (!current.publish_url) throw new Error("笔记尚未回填发布链接");
if (!current.publish_url) throw new Error("作品尚未回填发布链接");
const scheduledAt = `${scheduledDate}T09:00:00+08:00`;
const runId = uid("run");
@@ -203,7 +209,12 @@ export async function collectDistributionMetrics(
.bind(distributionId, scheduledDate)
.first<{ id: string; status: string }>();
if (!run) throw new Error("采集任务创建失败");
if (run.status === "success") return { skipped: true };
// Scheduled jobs should remain idempotent, but an operator clicking
// “立即采集” is explicitly asking for a fresh snapshot. Reusing the same
// daily run lets us correct stale or previously mis-mapped platform data.
if (run.status === "success" && source !== "manual") {
return { skipped: true };
}
const collectingDescription =
source === "automatic"
@@ -235,8 +246,12 @@ export async function collectDistributionMetrics(
]);
try {
const { likes, comments, collects } =
await collectXhsMetricsFromMcp(current.publish_url, mcpConfig);
const { likes, comments, collects, shares } =
await collectMetricsFromMcp(
current.publish_url,
current.platform === "抖音" ? "抖音" : "小红书",
mcpConfig,
);
const dayWeight = scheduleDay ?? 1;
const successDescription =
source === "automatic"
@@ -253,6 +268,7 @@ export async function collectDistributionMetrics(
likes = ?,
comments = ?,
collects = ?,
shares = ?,
status_description = ?,
completed_at = CURRENT_TIMESTAMP
WHERE id = ?`,
@@ -261,6 +277,7 @@ export async function collectDistributionMetrics(
likes,
comments,
collects,
shares,
successDescription,
run.id,
),
@@ -270,6 +287,7 @@ export async function collectDistributionMetrics(
SET latest_likes = ?,
latest_comments = ?,
latest_collects = ?,
latest_shares = ?,
collection_status = 'success',
collection_status_description = ?,
collection_updated_at = CURRENT_TIMESTAMP,
@@ -285,12 +303,13 @@ export async function collectDistributionMetrics(
likes,
comments,
collects,
shares,
successDescription,
scheduleDay,
distributionId,
),
]);
return { skipped: false, likes, comments, collects };
return { skipped: false, likes, comments, collects, shares };
} catch (error) {
const message =
error instanceof Error ? error.message : "公开数据采集失败";

View File

@@ -2,7 +2,7 @@ const FEISHU_API_ORIGIN = "https://open.feishu.cn";
const MAX_SHEET_ROWS = 5_000;
const MAX_SHEET_COLUMNS = 100;
const MAX_CONTENT_ROWS = 1_000;
const MAX_MEDIA_BYTES = 20_000_000;
const DEFAULT_MAX_MEDIA_BYTES = 200_000_000;
export type FeishuBindings = {
FEISHU_APP_ID?: string;
@@ -16,11 +16,20 @@ export type FeishuSourceImage = {
height: number | null;
};
export type FeishuSourceVideo = {
index: number;
fileToken: string;
name: string;
mimeType: string;
size: number | null;
};
export type FeishuSourceRow = {
sourceRow: number;
title: string;
body: string;
images: FeishuSourceImage[];
videos: FeishuSourceVideo[];
};
export type FeishuSource = {
@@ -106,12 +115,47 @@ function cellText(value: unknown): string {
.filter(Boolean)
.join("");
}
if (!isRecord(value) || value.type === "embed-image") return "";
if (
!isRecord(value) ||
value.type === "embed-image" ||
value.type === "attachment"
) return "";
if (typeof value.text === "string") return value.text.trim();
if (typeof value.value === "string") return value.value.trim();
return "";
}
function extractVideos(value: unknown, output: FeishuSourceVideo[]) {
if (Array.isArray(value)) {
for (const item of value) extractVideos(item, output);
return;
}
if (!isRecord(value)) return;
const fileToken = bindingValue(value.fileToken ?? value.file_token);
const mimeType = bindingValue(value.mimeType ?? value.mime_type);
const name = bindingValue(value.text ?? value.name ?? value.file_name);
const isVideo =
value.type === "attachment" &&
(mimeType.startsWith("video/") || /\.(?:mp4|mov|m4v|webm)$/i.test(name));
if (isVideo && fileToken) {
output.push({
index: 0,
fileToken,
name: name || "视频",
mimeType: mimeType || "video/mp4",
size:
typeof value.size === "number" && Number.isFinite(value.size)
? value.size
: null,
});
}
for (const child of Object.values(value)) {
if (child !== value.fileToken && child !== value.file_token) {
extractVideos(child, output);
}
}
}
function extractImages(value: unknown, output: FeishuSourceImage[]) {
if (Array.isArray(value)) {
for (const item of value) extractImages(item, output);
@@ -167,6 +211,7 @@ function findHeader(values: unknown[][]) {
titleIndex: number;
bodyIndex: number;
tagsIndex: number;
videoIndex: number;
score: number;
}
| undefined;
@@ -185,9 +230,13 @@ function findHeader(values: unknown[][]) {
const tagsIndex = headers.findIndex((header) =>
headerMatches(header, [/标签/, /话题/, /^tags?$/]),
);
const videoIndex = headers.findIndex((header) =>
headerMatches(header, [/^视频\d*$/, /视频文件/, /视频素材/]),
);
const score =
(titleIndex >= 0 ? 5 : 0) +
(bodyIndex >= 0 ? 5 : 0) +
(videoIndex >= 0 ? 2 : 0) +
(idIndex >= 0 ? 1 : 0) +
(tagsIndex >= 0 ? 1 : 0);
if (!best || score > best.score) {
@@ -197,6 +246,7 @@ function findHeader(values: unknown[][]) {
titleIndex,
bodyIndex,
tagsIndex,
videoIndex,
score,
};
}
@@ -217,6 +267,7 @@ function parseRows(values: unknown[][]) {
const usedSourceRows = new Set<number>();
const rows: FeishuSourceRow[] = [];
let maxImageCount = 0;
let maxVideoCount = 0;
for (
let rowIndex = header.rowIndex + 1;
@@ -253,7 +304,20 @@ function parseRows(values: unknown[][]) {
})
.map((image, imageIndex) => ({ ...image, index: imageIndex + 1 }));
maxImageCount = Math.max(maxImageCount, images.length);
rows.push({ sourceRow, title, body, images });
const collectedVideos: FeishuSourceVideo[] = [];
if (header.videoIndex >= 0) {
extractVideos(row[header.videoIndex], collectedVideos);
}
const seenVideoTokens = new Set<string>();
const videos = collectedVideos
.filter((video) => {
if (seenVideoTokens.has(video.fileToken)) return false;
seenVideoTokens.add(video.fileToken);
return true;
})
.map((video, videoIndex) => ({ ...video, index: videoIndex + 1 }));
maxVideoCount = Math.max(maxVideoCount, videos.length);
rows.push({ sourceRow, title, body, images, videos });
}
if (rows.length === 0) {
@@ -266,6 +330,7 @@ function parseRows(values: unknown[][]) {
header.tagsIndex >= 0 ? cellText(headerRow[header.tagsIndex]) : "",
cellText(headerRow[header.bodyIndex]) || "正文",
...Array.from({ length: maxImageCount }, (_, index) => `图片${index + 1}`),
...Array.from({ length: maxVideoCount }, (_, index) => `视频${index + 1}`),
].filter(Boolean);
return {
@@ -515,34 +580,37 @@ export async function downloadFeishuMedia(
fileToken: string,
bindings: FeishuBindings,
fetchImpl: FetchLike = fetch,
options: { maxBytes?: number; label?: string; timeoutMs?: number } = {},
) {
const maxBytes = Math.max(1, options.maxBytes ?? DEFAULT_MAX_MEDIA_BYTES);
const label = bindingValue(options.label) || "素材";
const normalizedToken = bindingValue(fileToken);
if (!/^[A-Za-z0-9_-]{8,240}$/.test(normalizedToken)) {
throw new FeishuSourceError("飞书图片标识无效", 400);
throw new FeishuSourceError(`飞书${label}标识无效`, 400);
}
const token = await accessToken(bindings, fetchImpl);
const response = await fetchImpl(
`${FEISHU_API_ORIGIN}/open-apis/drive/v1/medias/${encodeURIComponent(normalizedToken)}/download`,
{
headers: { Authorization: `Bearer ${token}` },
signal: AbortSignal.timeout(20_000),
signal: AbortSignal.timeout(options.timeoutMs ?? 120_000),
},
);
const declaredSize = Number(response.headers.get("content-length"));
if (Number.isFinite(declaredSize) && declaredSize > MAX_MEDIA_BYTES) {
throw new FeishuSourceError("飞书图片超过20MB,无法同步", 413);
if (Number.isFinite(declaredSize) && declaredSize > maxBytes) {
throw new FeishuSourceError(`飞书${label}文件过大,无法同步`, 413);
}
if (!response.ok) {
throw new FeishuSourceError(
response.status === 403
? "飞书应用没有这张图片的下载权限"
: `下载飞书图片失败HTTP ${response.status}`,
? `飞书应用没有这${label}的下载权限`
: `下载飞书${label}失败HTTP ${response.status}`,
response.status === 403 ? 403 : 502,
);
}
const bytes = await response.arrayBuffer();
if (bytes.byteLength > MAX_MEDIA_BYTES) {
throw new FeishuSourceError("飞书图片超过20MB,无法同步", 413);
if (bytes.byteLength > maxBytes) {
throw new FeishuSourceError(`飞书${label}文件过大,无法同步`, 413);
}
return {
bytes,

View File

@@ -17,6 +17,7 @@ export type XhsPublicMetrics = {
likes: number;
comments: number;
collects: number;
shares: number;
};
export type XhsAccountProfile = {
@@ -26,6 +27,10 @@ export type XhsAccountProfile = {
redId: string;
ipLocation: string;
followers: number | null;
gender: "" | "男" | "女";
bio: string;
recentNoteTitles: string[];
providerTags: string[];
};
type JsonRpcEnvelope = {
@@ -341,16 +346,118 @@ function metricsFromToolResult(result: ToolResult): XhsPublicMetrics {
}
if (!data) throw new Error("采集结果缺少互动数据");
const count = (value: unknown, label: string) =>
value === null || value === undefined || value === ""
? 0
: metricValue(value, label);
return {
likes: metricValue(data.likes, "点赞数"),
comments: metricValue(data.comments, "评论数"),
collects: metricValue(
data.collects ?? data.favorites ?? data.favourites,
likes: count(
data.likes ??
data.liked_count ??
data.likedCount ??
data.like_count ??
data.likeCount ??
data.digg_count ??
data.diggCount,
"点赞数",
),
comments: count(
data.comments ?? data.comment_count ?? data.commentCount,
"评论数",
),
collects: count(
data.collects ??
data.collected_count ??
data.collectedCount ??
data.favorites ??
data.favourites ??
data.collect_count ??
data.collectCount,
"收藏数",
),
shares: count(
data.shares ??
data.share_count ??
data.shareCount ??
data.forwards ??
data.forward_count ??
data.forwardCount,
"转发数",
),
};
}
function usableDouyinSecUid(value: unknown) {
const candidate = stringValue(value);
return candidate && !/^\d+$/.test(candidate) && /^[A-Za-z0-9_-]{20,220}$/.test(candidate)
? candidate
: "";
}
function verifiedDouyinProfileUrl(value: unknown) {
const candidate = stringValue(value);
if (!candidate) return "";
try {
const parsed = new URL(candidate);
const secUid = decodeURIComponent(parsed.pathname.match(/^\/user\/([^/?#]+)/)?.[1] ?? "");
if (
parsed.protocol === "https:" &&
(parsed.hostname === "douyin.com" || parsed.hostname.endsWith(".douyin.com")) &&
usableDouyinSecUid(secUid)
) {
return parsed.toString();
}
} catch {
// The public redirect fallback below can still recover the profile URL.
}
return "";
}
async function douyinProfileFromPublicRedirect(
publishUrl: string,
fetchImpl: typeof fetch,
timeoutMs: number,
) {
let parsed: URL;
try {
parsed = new URL(publishUrl);
} catch {
return null;
}
if (
parsed.protocol !== "https:" ||
!(parsed.hostname === "douyin.com" || parsed.hostname.endsWith(".douyin.com"))
) {
return null;
}
try {
const response = await fetchImpl(parsed.toString(), {
method: "GET",
redirect: "manual",
signal: AbortSignal.timeout(Math.min(timeoutMs, 15_000)),
headers: {
"user-agent":
"Mozilla/5.0 (iPhone; CPU iPhone OS 18_0 like Mac OS X) AppleWebKit/605.1.15 Mobile/15E148",
},
});
const location = response.headers.get("location");
if (!location) return null;
const redirectUrl = new URL(location, parsed);
const secUid = usableDouyinSecUid(
redirectUrl.searchParams.get("sec_uid") ??
redirectUrl.searchParams.get("sec_user_id"),
);
return secUid
? {
platformUid: secUid,
profileUrl: `https://www.douyin.com/user/${encodeURIComponent(secUid)}`,
}
: null;
} catch {
return null;
}
}
function successfulToolData(result: ToolResult, fallbackMessage: string) {
const root = asRecord(result.payload);
const response = asRecord(root?.response) ?? root;
@@ -446,6 +553,64 @@ function findValueByKeys(
return undefined;
}
function profileGender(value: unknown): "" | "男" | "女" {
if (value === 1) return "男";
if (value === 2) return "女";
const normalized = String(value ?? "").trim().toLocaleLowerCase("zh-CN");
if (["男", "男性", "male", "m", "1"].includes(normalized)) return "男";
if (["女", "女性", "female", "f", "2"].includes(normalized)) return "女";
return "";
}
function recentNoteTitlesFromPayload(value: unknown) {
const titles: string[] = [];
const visit = (current: unknown, depth = 0) => {
if (depth > 12 || titles.length >= 20) return;
if (Array.isArray(current)) {
current.forEach((item) => visit(item, depth + 1));
return;
}
const record = asRecord(current);
if (!record) return;
const title = stringValue(record.title ?? record.note_title ?? record.noteTitle);
if (
title &&
(record.note_id || record.noteId || record.url || record.cover) &&
!titles.includes(title)
) {
titles.push(title);
}
Object.values(record).forEach((child) => visit(child, depth + 1));
};
visit(value);
return titles;
}
function providerTagsFromUser(value: unknown) {
const user = findRecord(value, (record) =>
Boolean(
record.gender !== undefined ||
record.desc !== undefined ||
record.signature !== undefined ||
record.fansCount !== undefined ||
record.fans_count !== undefined,
),
);
const raw = user?.tags;
if (!Array.isArray(raw)) return [];
return [
...new Set(
raw
.map((item) =>
typeof item === "string"
? item.trim()
: stringValue(asRecord(item)?.name ?? asRecord(item)?.title),
)
.filter(Boolean),
),
].slice(0, 5);
}
const FOLLOWER_KEYS = new Set([
"fans",
"fans_count",
@@ -529,10 +694,13 @@ async function xhsNoteIdFromShortLink(
) {
return { noteId: "", profile: null };
}
if (
url.hostname !== "xhslink.cn" &&
!url.hostname.endsWith(".xhslink.cn")
) {
const isShortLink =
url.hostname === "xhslink.cn" ||
url.hostname.endsWith(".xhslink.cn");
const isXhsPage =
url.hostname === "xiaohongshu.com" ||
url.hostname.endsWith(".xiaohongshu.com");
if (!isShortLink && !isXhsPage) {
return { noteId: "", profile: null };
}
@@ -608,6 +776,10 @@ function accountProfileFromPublicPage(
redId,
ipLocation: "待识别",
followers: null,
gender: "",
bio: "",
recentNoteTitles: [],
providerTags: [],
};
}
@@ -698,15 +870,45 @@ function profileDetailsFromToolResult(result: ToolResult) {
const redId =
findStringByKey(payload, "red_id") ||
findStringByKey(payload, "redId") ||
findStringByKey(payload, "unique_id") ||
findStringByKey(payload, "uniqueId") ||
findStringByKey(payload, "short_id") ||
findStringByKey(payload, "shortId") ||
findStringByKey(payload, "douyin_id") ||
findStringByKey(payload, "userId") ||
findStringByKey(payload, "user_id");
const ipLocation =
findStringByKey(payload, "ip_location") ||
findStringByKey(payload, "ipLocation");
if (followers === null && !nickname && !redId && !ipLocation) {
const gender = profileGender(findValueByKeys(payload, new Set(["gender", "sex"])));
const bio =
findStringByKey(payload, "desc") ||
findStringByKey(payload, "description") ||
findStringByKey(payload, "signature") ||
findStringByKey(payload, "bio");
const recentNoteTitles = recentNoteTitlesFromPayload(payload);
const providerTags = providerTagsFromUser(payload);
if (
followers === null &&
!nickname &&
!redId &&
!ipLocation &&
!gender &&
!bio &&
recentNoteTitles.length === 0
) {
throw new Error("账号主页采集结果缺少可用字段");
}
return { nickname, followers, redId, ipLocation };
return {
nickname,
followers,
redId,
ipLocation,
gender,
bio,
recentNoteTitles,
providerTags,
};
}
function accountProfileFromToolResult(
@@ -718,14 +920,18 @@ function accountProfileFromToolResult(
data,
(record) =>
Boolean(
stringValue(record.user_id ?? record.userid) &&
stringValue(record.user_id ?? record.userid ?? record.userId) &&
(stringValue(record.profile_url) ||
stringValue(record.nickname ?? record.name)),
),
);
if (!user) throw new Error("账号主页识别结果缺少作者信息");
const platformUid = stringValue(user.user_id ?? user.userid);
const candidateProfileUrl = stringValue(user.profile_url);
const platformUid = stringValue(
user.user_id ?? user.userid ?? user.userId,
);
const candidateProfileUrl = stringValue(
user.profile_url ?? user.profileUrl,
);
let profileUrl = "";
if (candidateProfileUrl) {
try {
@@ -755,6 +961,10 @@ function accountProfileFromToolResult(
redId: stringValue(user.red_id),
ipLocation: findStringByKey(data, "ip_location") || "待识别",
followers: followerCountFromPayload(data),
gender: "",
bio: "",
recentNoteTitles: [],
providerTags: [],
};
}
@@ -771,18 +981,7 @@ async function completeAccountProfile(
"parse_xhs_user_summary",
{ url: profile.profileUrl, use_proxy: true },
],
[
"fetch_user_detail",
{ link: profile.profileUrl, plant: "xhs" },
],
] as const) {
if (
completed.followers !== null &&
completed.redId &&
completed.ipLocation !== "待识别"
) {
break;
}
try {
const result = await callMcpTool(
fetchImpl,
@@ -801,6 +1000,16 @@ async function completeAccountProfile(
details.ipLocation && details.ipLocation !== "待识别"
? details.ipLocation
: completed.ipLocation,
gender: details.gender || completed.gender,
bio: details.bio || completed.bio,
recentNoteTitles:
details.recentNoteTitles.length > 0
? details.recentNoteTitles
: completed.recentNoteTitles,
providerTags:
details.providerTags.length > 0
? details.providerTags
: completed.providerTags,
};
} catch (error) {
if (error instanceof McpSessionLostError) throw error;
@@ -852,17 +1061,14 @@ async function resolveAccountInSession(
const endpoint = buildMcpUrl(config);
const timeoutMs = Math.max(5_000, config.timeoutMs ?? 30_000);
let noteId = xhsNoteIdFromUrl(publishUrl);
let publicPageProfile: XhsAccountProfile | null = null;
if (!noteId) {
const shortLink = await xhsNoteIdFromShortLink(
publishUrl,
fallbackNickname,
fetchImpl,
timeoutMs,
);
noteId = shortLink.noteId;
publicPageProfile = shortLink.profile;
}
const linkPage = await xhsNoteIdFromShortLink(
publishUrl,
fallbackNickname,
fetchImpl,
timeoutMs,
);
noteId = noteId || linkPage.noteId;
const publicPageProfile = linkPage.profile;
try {
const sessionId = await createMcpSession(
@@ -870,47 +1076,39 @@ async function resolveAccountInSession(
endpoint,
timeoutMs,
);
if (!noteId) {
const noteResult = await callMcpTool(
fetchImpl,
endpoint,
sessionId,
timeoutMs,
"fetch_content_detail",
{
link: publishUrl,
plant: "xhs",
include_comments: false,
auto_cookie: true,
},
);
const noteData = successfulToolData(
noteResult,
"无法识别小红书笔记",
);
noteId =
stringValue(noteData.noteId ?? noteData.note_id) ||
findStringByKey(noteData, "noteId") ||
findStringByKey(noteData, "note_id");
}
if (!noteId) throw new Error("无法从发布链接识别小红书笔记ID");
const authorResult = await callMcpTool(
const noteResult = await callMcpTool(
fetchImpl,
endpoint,
sessionId,
timeoutMs,
"collect_xhs_wen_note_detail",
"fetch_content_detail",
{
note_id: noteId,
need_desc: false,
include_raw: false,
link: publishUrl,
plant: "xhs",
include_comments: false,
auto_cookie: true,
},
);
const profile = accountProfileFromToolResult(
authorResult,
fallbackNickname,
const noteData = successfulToolData(
noteResult,
"无法识别小红书笔记",
);
noteId =
noteId ||
stringValue(noteData.noteId ?? noteData.note_id) ||
findStringByKey(noteData, "noteId") ||
findStringByKey(noteData, "note_id");
if (!noteId) throw new Error("无法从发布链接识别小红书笔记ID");
let profile: XhsAccountProfile;
try {
profile = accountProfileFromToolResult(
noteResult,
fallbackNickname,
);
} catch {
if (!publicPageProfile) throw new Error("笔记数据缺少公开作者主页");
profile = publicPageProfile;
}
return completeAccountProfile(
profile,
fetchImpl,
@@ -937,6 +1135,7 @@ async function resolveAccountInSession(
async function collectInSession(
publishUrl: string,
platform: "小红书" | "抖音",
config: CollectionMcpConfig,
fetchImpl: typeof fetch,
) {
@@ -952,28 +1151,12 @@ async function collectInSession(
"fetch_content_detail",
{
link: publishUrl,
plant: "xhs",
plant: platform === "抖音" ? "dy" : "xhs",
include_comments: false,
auto_cookie: true,
},
);
try {
return metricsFromToolResult(primary);
} catch {
const fallback = await callMcpTool(
fetchImpl,
endpoint,
sessionId,
timeoutMs,
"parse_xhs_note",
{
url: publishUrl,
include_comments: false,
auto_cookie: true,
},
);
return metricsFromToolResult(fallback);
}
return metricsFromToolResult(primary);
}
export function resolveCollectionMcpConfig(
@@ -1005,7 +1188,7 @@ export async function collectXhsMetricsFromMcp(
let lastError: unknown;
for (let attempt = 0; attempt < MAX_MCP_ATTEMPTS; attempt += 1) {
try {
return await collectInSession(parsed.toString(), config, fetchImpl);
return await collectInSession(parsed.toString(), "小红书", config, fetchImpl);
} catch (error) {
lastError = error;
if (!isRetryableTransportError(error)) throw error;
@@ -1022,6 +1205,211 @@ export async function collectXhsMetricsFromMcp(
: new Error("MCP采集服务暂时不可用");
}
export async function collectMetricsFromMcp(
publishUrl: string,
platform: "小红书" | "抖音",
config: CollectionMcpConfig,
fetchImpl: typeof fetch = fetch,
) {
if (platform === "小红书") {
return collectXhsMetricsFromMcp(publishUrl, config, fetchImpl);
}
let parsed: URL;
try {
parsed = new URL(publishUrl);
} catch {
throw new Error("发布链接无效");
}
if (!["http:", "https:"].includes(parsed.protocol)) {
throw new Error("发布链接无效");
}
let lastError: unknown;
for (let attempt = 0; attempt < MAX_MCP_ATTEMPTS; attempt += 1) {
try {
return await collectInSession(parsed.toString(), platform, config, fetchImpl);
} catch (error) {
lastError = error;
if (!isRetryableTransportError(error)) throw error;
}
}
throw lastError instanceof Error
? lastError
: new Error("MCP采集服务暂时不可用");
}
function douyinProfileFromToolResult(
result: ToolResult,
fallbackNickname: string,
): XhsAccountProfile {
const data = successfulToolData(result, "无法识别抖音作品");
const author = findRecord(data, (record) =>
Boolean(
stringValue(
record.sec_uid ?? record.secUid ?? record.uid ?? record.user_id ?? record.userId,
) && stringValue(record.nickname ?? record.name ?? record.unique_id ?? record.uniqueId),
),
);
if (!author) throw new Error("抖音作品数据缺少作者信息");
const verifiedSecUid = usableDouyinSecUid(author.sec_uid ?? author.secUid);
const fallbackUid = stringValue(author.uid ?? author.user_id ?? author.userId);
const platformUid = verifiedSecUid || fallbackUid;
const publicId = stringValue(
author.unique_id ?? author.uniqueId ?? author.short_id ?? author.shortId ?? author.douyin_id,
);
const candidateProfileUrl = verifiedDouyinProfileUrl(
author.profile_url ?? author.profileUrl,
);
const profileUrl = candidateProfileUrl ||
(verifiedSecUid
? `https://www.douyin.com/user/${encodeURIComponent(verifiedSecUid)}`
: "");
return {
platformUid,
nickname: stringValue(author.nickname ?? author.name) || fallbackNickname.trim(),
profileUrl,
redId: publicId,
ipLocation:
stringValue(author.ip_location ?? author.ipLocation) ||
findStringByKey(data, "ip_location") ||
findStringByKey(data, "ipLocation") ||
"待识别",
followers: followerCountFromPayload(author) ?? followerCountFromPayload(data),
gender: profileGender(author.gender ?? author.sex),
bio: stringValue(author.desc ?? author.description ?? author.signature ?? author.bio),
recentNoteTitles: recentNoteTitlesFromPayload(data),
providerTags: providerTagsFromUser(data),
};
}
async function resolveDouyinAccountInSession(
publishUrl: string,
fallbackNickname: string,
config: CollectionMcpConfig,
fetchImpl: typeof fetch,
) {
const endpoint = buildMcpUrl(config);
const timeoutMs = Math.max(5_000, config.timeoutMs ?? 30_000);
const sessionId = await createMcpSession(fetchImpl, endpoint, timeoutMs);
const result = await callMcpTool(
fetchImpl,
endpoint,
sessionId,
timeoutMs,
"fetch_content_detail",
{
link: publishUrl,
plant: "dy",
include_comments: false,
auto_cookie: true,
},
);
let profile = douyinProfileFromToolResult(result, fallbackNickname);
if (!verifiedDouyinProfileUrl(profile.profileUrl)) {
const resolved = await douyinProfileFromPublicRedirect(
publishUrl,
fetchImpl,
timeoutMs,
);
if (resolved) {
profile = {
...profile,
platformUid: resolved.platformUid,
profileUrl: resolved.profileUrl,
};
}
}
if (!verifiedDouyinProfileUrl(profile.profileUrl)) return profile;
try {
const detailsResult = await callMcpTool(
fetchImpl,
endpoint,
sessionId,
timeoutMs,
"parse_dy_user_summary",
{ url: profile.profileUrl },
);
const details = profileDetailsFromToolResult(detailsResult);
profile = {
...profile,
nickname: details.nickname || profile.nickname,
redId: details.redId || profile.redId,
followers: details.followers ?? profile.followers,
ipLocation:
details.ipLocation && details.ipLocation !== "待识别"
? details.ipLocation
: profile.ipLocation,
gender: details.gender || profile.gender,
bio: details.bio || profile.bio,
recentNoteTitles:
details.recentNoteTitles.length > 0
? details.recentNoteTitles
: profile.recentNoteTitles,
providerTags:
details.providerTags.length > 0
? details.providerTags
: profile.providerTags,
};
} catch (error) {
if (error instanceof McpSessionLostError) throw error;
}
return profile;
}
export async function resolveAccountProfileFromMcp(
publishUrl: string,
fallbackNickname: string,
platform: "小红书" | "抖音",
config: CollectionMcpConfig,
fetchImpl: typeof fetch = fetch,
) {
if (platform === "小红书") {
return resolveXhsAccountProfileFromMcp(
publishUrl,
fallbackNickname,
config,
fetchImpl,
);
}
let lastError: unknown;
for (let attempt = 0; attempt < MAX_MCP_ATTEMPTS; attempt += 1) {
try {
return await resolveDouyinAccountInSession(
publishUrl,
fallbackNickname,
config,
fetchImpl,
);
} catch (error) {
lastError = error;
if (!isRetryableTransportError(error)) throw error;
}
}
throw lastError instanceof Error ? lastError : new Error("抖音账号识别失败");
}
export async function resolveProfileDetailsFromMcp(
profileUrl: string,
platform: "小红书" | "抖音",
config: CollectionMcpConfig,
fetchImpl: typeof fetch = fetch,
) {
if (platform === "小红书") {
return resolveXhsProfileDetailsFromMcp(profileUrl, config, fetchImpl);
}
const endpoint = buildMcpUrl(config);
const timeoutMs = Math.max(5_000, config.timeoutMs ?? 30_000);
const sessionId = await createMcpSession(fetchImpl, endpoint, timeoutMs);
const result = await callMcpTool(
fetchImpl,
endpoint,
sessionId,
timeoutMs,
"parse_dy_user_summary",
{ url: profileUrl },
);
return profileDetailsFromToolResult(result);
}
export async function resolveXhsAccountProfileFromMcp(
publishUrl: string,
fallbackNickname: string,

View File

@@ -12,7 +12,7 @@ import {
type CollectionMcpBindings,
} from "./mcp-collection-client";
import { ensureSchema, getRawDb } from "./mvp-db";
import { extractXhsPublishUrl } from "./publish-url";
import { extractAnyPublishUrl } from "./publish-url";
import { buildClaimUrl } from "./task-service";
export type McpOperationBindings = CollectionMcpBindings & {
@@ -123,9 +123,9 @@ export async function taskGet(taskId: string, portalUrl: string) {
const [notes, claims, runs] = await Promise.all([
db
.prepare(
`SELECT c.id AS content_id, c.source_row, c.title, c.body, c.image_assets, c.status AS content_status,
`SELECT c.id AS content_id, c.source_row, c.title, c.body, c.image_assets, c.video_assets, c.status AS content_status,
d.id AS distribution_id, d.status AS distribution_status, d.publish_url, d.publish_time,
d.exposure, d.views, d.latest_likes, d.latest_comments, d.latest_collects,
d.exposure, d.views, d.latest_likes, d.latest_comments, d.latest_collects, d.latest_shares,
d.collection_status, d.collection_status_description, d.collection_updated_at,
a.id AS account_id, a.nickname AS account_nickname, a.profile_url, a.followers,
p.name AS cooperation_source, cl.claimant_name
@@ -172,12 +172,14 @@ export async function taskGet(taskId: string, portalUrl: string) {
notes: notes.results.map((row) => ({
...row,
image_assets: parseJsonArray(row.image_assets),
video_assets: parseJsonArray(row.video_assets),
total_interactions:
row.latest_likes == null
? null
: Number(row.latest_likes) +
Number(row.latest_comments ?? 0) +
Number(row.latest_collects ?? 0),
Number(row.latest_collects ?? 0) +
Number(row.latest_shares ?? 0),
})),
claims: claims.results,
collection_runs: runs.results,
@@ -220,7 +222,8 @@ export async function recoveryList(
const [rows, count] = await Promise.all([
db
.prepare(
`SELECT d.*, t.name AS task_name, c.source_row, c.title,
`SELECT d.*, t.name AS task_name, t.platform AS task_platform,
t.content_format, c.source_row, c.title,
a.nickname AS account_nickname, a.profile_url, p.name AS cooperation_source,
cl.claimant_name ${base}
ORDER BY COALESCE(d.publish_time, d.claimed_at) DESC LIMIT ${limit} OFFSET ${offset}`,
@@ -241,7 +244,7 @@ export async function recoveryList(
total_interactions:
row.latest_likes == null
? null
: Number(row.latest_likes) + Number(row.latest_comments ?? 0) + Number(row.latest_collects ?? 0),
: Number(row.latest_likes) + Number(row.latest_comments ?? 0) + Number(row.latest_collects ?? 0) + Number(row.latest_shares ?? 0),
})),
};
}
@@ -358,9 +361,11 @@ function resourceWhere(input: ResourceFilters) {
const conditions: string[] = [];
const bindings: unknown[] = [];
if (input.query?.trim()) {
conditions.push("(a.nickname LIKE ? ESCAPE '\\' OR a.public_account_id LIKE ? ESCAPE '\\')");
conditions.push(
"(a.nickname LIKE ? ESCAPE '\\' OR a.public_account_id LIKE ? ESCAPE '\\' OR a.current_contact LIKE ? ESCAPE '\\' OR a.tags LIKE ? ESCAPE '\\' OR a.bio LIKE ? ESCAPE '\\')",
);
const pattern = like(input.query.trim());
bindings.push(pattern, pattern);
bindings.push(pattern, pattern, pattern, pattern, pattern);
}
if (input.ipLocation?.trim()) {
conditions.push("a.ip_location LIKE ? ESCAPE '\\'");
@@ -369,8 +374,12 @@ function resourceWhere(input: ResourceFilters) {
if (input.cooperationSource?.trim()) {
conditions.push(
`(a.cooperation_source LIKE ? ESCAPE '\\' OR
EXISTS (SELECT 1 FROM distributions dx JOIN partners px ON px.id = dx.partner_id
WHERE dx.account_id = a.id AND px.name LIKE ? ESCAPE '\\'))`,
EXISTS (SELECT 1 FROM distributions dx
JOIN partners px ON px.id = dx.partner_id
LEFT JOIN claims cx ON cx.id = dx.claim_id
WHERE dx.account_id = a.id
AND (cx.claimant_name IS NULL OR px.name != cx.claimant_name)
AND px.name LIKE ? ESCAPE '\\'))`,
);
const pattern = like(input.cooperationSource.trim());
bindings.push(pattern, pattern);
@@ -389,7 +398,12 @@ export async function resourceSearch(input: ResourceFilters) {
const db = getRawDb();
const select = `SELECT a.*,
(SELECT COUNT(*) FROM distributions d WHERE d.account_id = a.id) AS cooperation_count,
(SELECT GROUP_CONCAT(DISTINCT p.name) FROM distributions d JOIN partners p ON p.id = d.partner_id WHERE d.account_id = a.id) AS cooperation_sources`;
(SELECT GROUP_CONCAT(DISTINCT p.name)
FROM distributions d
JOIN partners p ON p.id = d.partner_id
LEFT JOIN claims c ON c.id = d.claim_id
WHERE d.account_id = a.id
AND (c.claimant_name IS NULL OR p.name != c.claimant_name)) AS cooperation_sources`;
const [rows, count] = await Promise.all([
db.prepare(`${select} FROM accounts a ${where} ORDER BY a.last_seen_at DESC LIMIT ${limit} OFFSET ${offset}`)
.bind(...bindings).all<Record<string, unknown>>(),
@@ -422,7 +436,7 @@ export async function resourceGet(accountId: string) {
if (!account) throw new Error("账号不存在");
const history = await db.prepare(
`SELECT d.id AS distribution_id, d.task_id, t.name AS task_name, c.title,
d.publish_url, d.publish_time, d.latest_likes, d.latest_comments, d.latest_collects,
d.publish_url, d.publish_time, d.latest_likes, d.latest_comments, d.latest_collects, d.latest_shares,
d.exposure, d.views, p.name AS cooperation_source, cl.claimant_name
FROM distributions d JOIN tasks t ON t.id = d.task_id
JOIN contents c ON c.id = d.content_id JOIN partners p ON p.id = d.partner_id
@@ -438,7 +452,7 @@ export async function backfillResourceProfile(
) {
await ensureSchema();
const db = getRawDb();
const publishUrl = input.publishUrl ? extractXhsPublishUrl(input.publishUrl) : "";
const publishUrl = input.publishUrl ? extractAnyPublishUrl(input.publishUrl) : "";
const row = input.distributionId
? await db.prepare(
`SELECT d.id, d.publish_url, COALESCE(a.nickname, '') AS nickname

View File

@@ -52,7 +52,7 @@ const pagination = {
};
const resourceFilters = {
query: z.string().max(100).optional().describe("账号名称或小红书号/抖音号,支持模糊搜索"),
query: z.string().max(100).optional().describe("账号名称、账号号、简介或标签,支持模糊搜索"),
ip_location: z.string().max(100).optional().describe("IP地区关键词支持模糊搜索"),
cooperation_source: z.string().max(100).optional().describe("历史合作来源关键词,支持模糊搜索"),
platform: z.string().max(30).optional().describe("平台,例如小红书;不传表示全部"),
@@ -159,7 +159,7 @@ export function registerMcpOperationTools(server: McpServer, options: Options) {
"collection_collect_now",
{
title: "立即采集指定笔记",
description: "对指定分发记录立即采集点赞收藏评论数据。",
description: "对指定分发记录立即采集互动数据;小红书为点赞/收藏/评论,抖音另含转发。",
inputSchema: z.object({
distribution_id: z.string().min(1).describe("分发记录ID可从 task_get 或 recovery_list 获取"),
schedule_day: z.number().int().min(1).max(7).optional().describe("标记为第几天采集,可不传"),
@@ -190,7 +190,7 @@ export function registerMcpOperationTools(server: McpServer, options: Options) {
"resource_search",
{
title: "搜索 KOC 账号资源",
description: "按账号名称/账号号、IP地区、合作来源或平台搜索 KOC 资源。",
description: "按账号名称/账号号/标签、IP地区、合作来源或平台搜索 KOC 资源。",
inputSchema: z.object({ ...resourceFilters, ...pagination }),
annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true },
},
@@ -204,7 +204,7 @@ export function registerMcpOperationTools(server: McpServer, options: Options) {
"resource_get",
{
title: "查看 KOC 账号详情",
description: "查看账号主页、账号号、粉丝数、IP地区以及全部合作记录。",
description: "查看账号主页、账号号、粉丝数、性别、简介、标签、IP地区以及全部合作记录。",
inputSchema: z.object({ account_id: z.string().min(1).describe("KOC LOOP 账号ID") }),
annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true },
},
@@ -218,10 +218,10 @@ export function registerMcpOperationTools(server: McpServer, options: Options) {
"resource_backfill_profile",
{
title: "补全公开账号信息",
description: "根据已回填的发布链接补全账号主页、昵称、小红书号、IP地区和粉丝数。",
description: "根据已回填的小红书或抖音作品链接补全账号主页、昵称、账号号、IP地区和粉丝数。",
inputSchema: z.object({
distribution_id: z.string().optional().describe("分发记录ID和发布链接二选一"),
publish_url: z.string().optional().describe("小红书发布链接或包含链接的分享文案"),
publish_url: z.string().optional().describe("小红书或抖音作品链接,也可传包含链接的分享文案"),
}).refine((value) => Boolean(value.distribution_id || value.publish_url), "请提供分发记录ID或发布链接"),
annotations: { readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: true },
},

View File

@@ -94,6 +94,8 @@ export async function ensureSchema(database?: DatabaseClient) {
due_at TEXT NOT NULL,
status TEXT NOT NULL DEFAULT 'active',
task_type TEXT NOT NULL DEFAULT 'content_publish',
platform TEXT NOT NULL DEFAULT '小红书',
content_format TEXT NOT NULL DEFAULT 'image_text',
source_url TEXT NOT NULL DEFAULT '',
source_sheet_id TEXT NOT NULL DEFAULT '',
source_sheet_name TEXT NOT NULL DEFAULT '',
@@ -110,6 +112,7 @@ export async function ensureSchema(database?: DatabaseClient) {
title TEXT NOT NULL,
body TEXT NOT NULL DEFAULT '',
image_assets TEXT NOT NULL DEFAULT '[]',
video_assets TEXT NOT NULL DEFAULT '[]',
status TEXT NOT NULL DEFAULT 'available',
source TEXT NOT NULL DEFAULT '飞书内容表',
source_row INTEGER,
@@ -124,9 +127,13 @@ export async function ensureSchema(database?: DatabaseClient) {
profile_url TEXT NOT NULL DEFAULT '',
ip_location TEXT NOT NULL DEFAULT '待识别',
followers INTEGER NOT NULL DEFAULT 0,
gender TEXT NOT NULL DEFAULT '',
bio TEXT NOT NULL DEFAULT '',
tags TEXT NOT NULL DEFAULT '',
post_count INTEGER NOT NULL DEFAULT 0,
avg_views INTEGER NOT NULL DEFAULT 0,
cooperation_source TEXT NOT NULL DEFAULT '',
current_contact TEXT NOT NULL DEFAULT '',
first_seen_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
last_seen_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
)`,
@@ -185,6 +192,7 @@ export async function ensureSchema(database?: DatabaseClient) {
latest_likes INTEGER,
latest_comments INTEGER,
latest_collects INTEGER,
latest_shares INTEGER,
collection_status TEXT NOT NULL DEFAULT 'pending',
collection_status_description TEXT,
collection_updated_at TEXT,
@@ -202,6 +210,7 @@ export async function ensureSchema(database?: DatabaseClient) {
likes INTEGER,
comments INTEGER,
collects INTEGER,
shares INTEGER,
status_description TEXT,
started_at TEXT,
completed_at TEXT,
@@ -293,6 +302,14 @@ export async function ensureSchema(database?: DatabaseClient) {
"public_account_id",
"public_account_id TEXT NOT NULL DEFAULT ''",
);
await ensureColumn("accounts", "gender", "gender TEXT NOT NULL DEFAULT ''");
await ensureColumn("accounts", "bio", "bio TEXT NOT NULL DEFAULT ''");
await ensureColumn("accounts", "tags", "tags TEXT NOT NULL DEFAULT ''");
await ensureColumn(
"accounts",
"current_contact",
"current_contact TEXT NOT NULL DEFAULT ''",
);
await ensureColumn("distributions", "claim_id", "claim_id TEXT");
await ensureColumn(
"distributions",
@@ -796,22 +813,44 @@ export async function getDashboardData() {
await Promise.all([
db.prepare("SELECT * FROM partners ORDER BY created_at DESC").all(),
db.prepare("SELECT * FROM tasks ORDER BY created_at DESC").all(),
db.prepare("SELECT * FROM accounts ORDER BY last_seen_at DESC").all(),
db
.prepare(
`SELECT
a.*,
COALESCE(
(
SELECT d.publish_url
FROM distributions d
WHERE d.account_id = a.id
AND TRIM(COALESCE(d.publish_url, '')) != ''
ORDER BY d.updated_at DESC, d.claimed_at DESC
LIMIT 1
),
''
) AS latest_publish_url
FROM accounts a
ORDER BY a.last_seen_at DESC`,
)
.all(),
db
.prepare(
`SELECT
d.*,
c.title AS content_title,
p.name AS partner_name,
cl.claimant_name AS claimant_name,
a.nickname AS account_nickname,
a.platform AS account_platform,
t.name AS task_name,
t.brand AS task_brand,
t.task_type AS task_type,
t.platform AS task_platform,
t.content_format AS content_format,
t.due_at AS due_at
FROM distributions d
JOIN contents c ON c.id = d.content_id
JOIN partners p ON p.id = d.partner_id
LEFT JOIN claims cl ON cl.id = d.claim_id
JOIN tasks t ON t.id = d.task_id
LEFT JOIN accounts a ON a.id = d.account_id
ORDER BY d.updated_at DESC, d.claimed_at DESC`,

View File

@@ -0,0 +1,547 @@
import path from "node:path";
import { strFromU8, unzipSync } from "fflate";
export const PARTNER_BATCH_HEADERS = [
"序号(不能改)",
"标题",
"笔记内容(正文+话题)",
"图片",
"发布链接",
"笔记截图",
"数据分析截图(单篇笔记数据分析截图)",
"_系统笔记ID",
"_原笔记截图",
"_原数据分析截图",
] as const;
export const PARTNER_BATCH_VISIBLE_COLUMN_COUNT = 7;
export const PARTNER_BATCH_MAX_BYTES = 80_000_000;
export type PartnerBatchWorkbookColumns = {
headers: string[];
columnWidths: number[];
sourceImageStartColumn: number;
sourceImageCount: number;
sourceVideoStartColumn: number;
sourceVideoCount: number;
publishUrlColumn: number;
publishScreenshotColumn: number;
creatorScreenshotColumn: number;
systemColumn: number;
};
function nonNegativeInteger(value: number) {
return Number.isFinite(value) ? Math.max(0, Math.floor(value)) : 0;
}
export function buildPartnerBatchWorkbookColumns(input: {
contentFormat: "image_text" | "video";
maxSourceImages: number;
maxSourceVideos: number;
}): PartnerBatchWorkbookColumns {
const sourceImageCount =
input.contentFormat === "video"
? 0
: Math.max(1, nonNegativeInteger(input.maxSourceImages));
const sourceVideoCount =
input.contentFormat === "video"
? Math.max(1, nonNegativeInteger(input.maxSourceVideos))
: 0;
const sourceImageStartColumn = 3;
const sourceVideoStartColumn = sourceImageStartColumn + sourceImageCount;
const publishUrlColumn = sourceVideoStartColumn + sourceVideoCount;
const publishScreenshotColumn = publishUrlColumn + 1;
const creatorScreenshotColumn = publishScreenshotColumn + 1;
const systemColumn = creatorScreenshotColumn + 1;
return {
headers: [
"序号(不能改)",
"标题",
"笔记内容(正文+话题)",
...Array.from(
{ length: sourceImageCount },
(_, index) => `图片${index + 1}`,
),
...Array.from(
{ length: sourceVideoCount },
(_, index) => `视频${index + 1}`,
),
"发布链接",
"笔记截图",
"数据分析截图(单篇笔记数据分析截图)",
"_系统笔记ID",
"_原笔记截图",
"_原数据分析截图",
],
columnWidths: [
14,
30,
62,
...Array.from({ length: sourceImageCount }, () => 24),
...Array.from({ length: sourceVideoCount }, () => 20),
45,
28,
32,
22,
22,
22,
],
sourceImageStartColumn,
sourceImageCount,
sourceVideoStartColumn,
sourceVideoCount,
publishUrlColumn,
publishScreenshotColumn,
creatorScreenshotColumn,
systemColumn,
};
}
function firstForwardedValue(value: string | null) {
return value?.split(",")[0]?.trim() ?? "";
}
function httpOrigin(value: string) {
try {
const url = new URL(value);
return url.protocol === "http:" || url.protocol === "https:"
? url.origin
: "";
} catch {
return "";
}
}
export function resolvePartnerWorkbookOrigin(
request: Request,
configuredOrigin = "",
) {
const requestUrl = new URL(request.url);
const host =
firstForwardedValue(request.headers.get("x-forwarded-host")) ||
firstForwardedValue(request.headers.get("host"));
const forwardedProtocol = firstForwardedValue(
request.headers.get("x-forwarded-proto"),
).toLowerCase();
const protocol = ["http", "https"].includes(forwardedProtocol)
? forwardedProtocol
: requestUrl.protocol.replace(":", "");
const proxyOrigin = host ? httpOrigin(`${protocol}://${host}`) : "";
return (
httpOrigin(configuredOrigin) ||
proxyOrigin ||
httpOrigin(requestUrl.origin) ||
requestUrl.origin
);
}
export type PartnerBatchImage = {
bytes: Uint8Array;
contentType: string;
fileName: string;
};
export type PartnerBatchImportRow = {
spreadsheetRow: number;
sequence: string;
title: string;
publishUrl: string;
distributionId: string;
originalPublishScreenshotKey: string;
originalCreatorScreenshotKey: string;
publishScreenshot: PartnerBatchImage | null;
creatorScreenshot: PartnerBatchImage | null;
};
function decodeXml(value: string) {
return value
.replace(/<[^>]+>/g, "")
.replace(/&lt;/g, "<")
.replace(/&gt;/g, ">")
.replace(/&quot;/g, '"')
.replace(/&apos;/g, "'")
.replace(/&amp;/g, "&")
.replace(/&#(\d+);/g, (_, code) => String.fromCodePoint(Number(code)))
.replace(/&#x([0-9a-f]+);/gi, (_, code) =>
String.fromCodePoint(Number.parseInt(code, 16)),
);
}
function xmlAttribute(value: string) {
return decodeXml(value);
}
function textNodes(xml: string) {
return [...xml.matchAll(/<t\b[^>]*>([\s\S]*?)<\/t>/g)]
.map((match) => decodeXml(match[1]))
.join("");
}
function columnIndex(reference: string) {
const letters = reference.match(/^[A-Z]+/i)?.[0]?.toUpperCase() ?? "";
let result = 0;
for (const letter of letters) result = result * 26 + letter.charCodeAt(0) - 64;
return Math.max(0, result - 1);
}
function parseWorksheet(xml: string, sharedStrings: string[]) {
const rows: string[][] = [];
for (const rowMatch of xml.matchAll(/<row\b([^>]*)>([\s\S]*?)<\/row>/g)) {
const rowNumber = Number(
rowMatch[1].match(/\br="(\d+)"/)?.[1] ?? rows.length + 1,
);
const values: string[] = [];
for (const cellMatch of rowMatch[2].matchAll(
/<c\b([^>]*)>([\s\S]*?)<\/c>/g,
)) {
const attributes = cellMatch[1];
const body = cellMatch[2];
const reference = attributes.match(/\br="([A-Z]+\d+)"/i)?.[1] ?? "A1";
const type = attributes.match(/\bt="([^"]+)"/)?.[1] ?? "";
const rawValue = body.match(/<v>([\s\S]*?)<\/v>/)?.[1] ?? "";
let value = "";
if (type === "s") value = sharedStrings[Number(rawValue)] ?? "";
else if (type === "inlineStr") value = textNodes(body);
else if (type === "b") value = rawValue === "1" ? "TRUE" : "FALSE";
else value = decodeXml(rawValue);
values[columnIndex(reference)] = value.trim();
}
while (rows.length < rowNumber - 1) rows.push([]);
rows[rowNumber - 1] = values;
}
return rows;
}
function normalizeHeader(value: string) {
return value.replace(/[\s_\-()]/g, "").toLocaleLowerCase("zh-CN");
}
function isSourceImageHeader(value: string) {
return /^(?:图片|发布配图)\d*$/.test(normalizeHeader(value));
}
function headerAliases(header: (typeof PARTNER_BATCH_HEADERS)[number]) {
const aliases: Record<string, string[]> = {
"序号(不能改)": ["序号(不能改)", "序号"],
: ["标题"],
"笔记内容(正文+话题)": ["笔记内容(正文+话题)", "笔记内容"],
: ["图片", "发布配图"],
: ["发布链接"],
: ["笔记截图", "发布截图"],
"数据分析截图(单篇笔记数据分析截图)": [
"数据分析截图(单篇笔记数据分析截图)",
"数据分析截图",
"创作者中心截图",
],
_系统笔记ID: ["_系统笔记ID", "系统笔记ID"],
_原笔记截图: ["_原笔记截图"],
_原数据分析截图: ["_原数据分析截图"],
};
return aliases[header] ?? [header];
}
function findHeader(rows: string[][]) {
for (let rowIndex = 0; rowIndex < Math.min(rows.length, 8); rowIndex += 1) {
const mapping = new Map<(typeof PARTNER_BATCH_HEADERS)[number], number>();
rows[rowIndex].forEach((value, column) => {
for (const header of PARTNER_BATCH_HEADERS) {
if (header === "图片" && isSourceImageHeader(value)) {
if (!mapping.has(header)) mapping.set(header, column);
break;
}
if (
headerAliases(header).some(
(alias) => normalizeHeader(alias) === normalizeHeader(value),
)
) {
mapping.set(header, column);
break;
}
}
});
if (
mapping.has("序号(不能改)") &&
mapping.has("标题") &&
mapping.has("发布链接") &&
mapping.has("笔记截图") &&
mapping.has("数据分析截图(单篇笔记数据分析截图)") &&
mapping.has("_系统笔记ID")
) {
return { rowIndex, mapping };
}
}
return null;
}
function relationshipMap(xml: string) {
const relationships = new Map<string, string>();
for (const match of xml.matchAll(/<Relationship\b([^>]*)\/?\s*>/g)) {
const attributes = match[1];
const id = attributes.match(/\bId="([^"]+)"/)?.[1] ?? "";
const target = attributes.match(/\bTarget="([^"]+)"/)?.[1] ?? "";
if (id && target) relationships.set(id, xmlAttribute(target));
}
return relationships;
}
function contentType(bytes: Uint8Array, fileName: string) {
if (bytes[0] === 0x89 && bytes[1] === 0x50) return "image/png";
if (bytes[0] === 0xff && bytes[1] === 0xd8) return "image/jpeg";
if (bytes[0] === 0x47 && bytes[1] === 0x49 && bytes[2] === 0x46) {
return "image/gif";
}
if (String.fromCharCode(...bytes.slice(8, 12)) === "WEBP") return "image/webp";
const extension = path.extname(fileName).toLowerCase();
return extension === ".png"
? "image/png"
: extension === ".gif"
? "image/gif"
: extension === ".webp"
? "image/webp"
: "image/jpeg";
}
function resolveZipPath(base: string, target: string) {
return path.posix.normalize(path.posix.join(path.posix.dirname(base), target));
}
function parseDrawingImages(entries: Record<string, Uint8Array>) {
const images = new Map<string, PartnerBatchImage>();
const sheetRelationshipsXml = entries["xl/worksheets/_rels/sheet1.xml.rels"]
? strFromU8(entries["xl/worksheets/_rels/sheet1.xml.rels"])
: "";
const sheetRelationships = relationshipMap(sheetRelationshipsXml);
const sheetXml = entries["xl/worksheets/sheet1.xml"]
? strFromU8(entries["xl/worksheets/sheet1.xml"])
: "";
const drawingId = sheetXml.match(/<drawing\b[^>]*r:id="([^"]+)"/)?.[1] ?? "";
const drawingTarget = sheetRelationships.get(drawingId);
if (!drawingTarget) return images;
const drawingPath = resolveZipPath("xl/worksheets/sheet1.xml", drawingTarget);
const drawingXml = entries[drawingPath] ? strFromU8(entries[drawingPath]) : "";
const drawingRelationshipsPath = path.posix.join(
path.posix.dirname(drawingPath),
"_rels",
`${path.posix.basename(drawingPath)}.rels`,
);
const drawingRelationships = relationshipMap(
entries[drawingRelationshipsPath]
? strFromU8(entries[drawingRelationshipsPath])
: "",
);
for (const anchor of drawingXml.matchAll(
/<xdr:(?:oneCellAnchor|twoCellAnchor)\b[^>]*>[\s\S]*?<xdr:from>([\s\S]*?)<\/xdr:from>[\s\S]*?<a:blip\b[^>]*r:embed="([^"]+)"[\s\S]*?<\/xdr:(?:oneCellAnchor|twoCellAnchor)>/g,
)) {
const column = Number(anchor[1].match(/<xdr:col>(\d+)<\/xdr:col>/)?.[1]);
const row = Number(anchor[1].match(/<xdr:row>(\d+)<\/xdr:row>/)?.[1]);
const mediaTarget = drawingRelationships.get(anchor[2]);
if (!Number.isInteger(column) || !Number.isInteger(row) || !mediaTarget) continue;
const mediaPath = resolveZipPath(drawingPath, mediaTarget);
const bytes = entries[mediaPath];
if (!bytes) continue;
images.set(`${row + 1}:${column}`, {
bytes,
contentType: contentType(bytes, mediaPath),
fileName: path.posix.basename(mediaPath),
});
}
return images;
}
function parseRichValueImages(entries: Record<string, Uint8Array>) {
const images = new Map<string, PartnerBatchImage>();
const worksheetXml = entries["xl/worksheets/sheet1.xml"]
? strFromU8(entries["xl/worksheets/sheet1.xml"])
: "";
const metadataXml = entries["xl/metadata.xml"]
? strFromU8(entries["xl/metadata.xml"])
: "";
const richValueXml = entries["xl/richData/rdrichvalue.xml"]
? strFromU8(entries["xl/richData/rdrichvalue.xml"])
: "";
const richValueRelXml = entries["xl/richData/richValueRel.xml"]
? strFromU8(entries["xl/richData/richValueRel.xml"])
: "";
const richValueRelRelationships = relationshipMap(
entries["xl/richData/_rels/richValueRel.xml.rels"]
? strFromU8(entries["xl/richData/_rels/richValueRel.xml.rels"])
: "",
);
if (
!worksheetXml ||
!metadataXml ||
!richValueXml ||
!richValueRelXml ||
!richValueRelRelationships.size
) {
return images;
}
const valueMetadataXml =
metadataXml.match(/<valueMetadata\b[^>]*>([\s\S]*?)<\/valueMetadata>/)?.[1] ??
"";
const metadataToRichValue = [
...valueMetadataXml.matchAll(/<bk\b[^>]*>[\s\S]*?<rc\b[^>]*\bv="(\d+)"[^>]*\/>[\s\S]*?<\/bk>/g),
].map((match) => Number(match[1]));
const richValueToRelationship = [
...richValueXml.matchAll(/<rv\b[^>]*>([\s\S]*?)<\/rv>/g),
].map((match) => Number(match[1].match(/<v>(\d+)<\/v>/)?.[1] ?? -1));
const relationshipIds = [
...richValueRelXml.matchAll(/<rel\b[^>]*\br:id="([^"]+)"[^>]*\/>/g),
].map((match) => match[1]);
for (const cell of worksheetXml.matchAll(
/<c\b([^>]*)>[\s\S]*?<\/c>/g,
)) {
const reference = cell[1].match(/\br="([A-Z]+\d+)"/i)?.[1] ?? "";
const metadataIndex = Number(cell[1].match(/\bvm="(\d+)"/)?.[1] ?? 0);
const row = Number(reference.match(/\d+$/)?.[0] ?? 0);
const column = columnIndex(reference);
if (!reference || !metadataIndex || !row) continue;
const richValueIndex = metadataToRichValue[metadataIndex - 1];
const relationshipIndex = richValueToRelationship[richValueIndex];
const relationshipId = relationshipIds[relationshipIndex];
const mediaTarget = richValueRelRelationships.get(relationshipId);
if (!mediaTarget) continue;
const mediaPath = resolveZipPath(
"xl/richData/richValueRel.xml",
mediaTarget,
);
const bytes = entries[mediaPath];
if (!bytes) continue;
images.set(`${row}:${column}`, {
bytes,
contentType: contentType(bytes, mediaPath),
fileName: path.posix.basename(mediaPath),
});
}
return images;
}
function parseWpsCellImages(entries: Record<string, Uint8Array>) {
const images = new Map<string, PartnerBatchImage>();
const worksheetXml = entries["xl/worksheets/sheet1.xml"]
? strFromU8(entries["xl/worksheets/sheet1.xml"])
: "";
const cellImagesXml = entries["xl/cellimages.xml"]
? strFromU8(entries["xl/cellimages.xml"])
: "";
const relationships = relationshipMap(
entries["xl/_rels/cellimages.xml.rels"]
? strFromU8(entries["xl/_rels/cellimages.xml.rels"])
: "",
);
if (!worksheetXml || !cellImagesXml || !relationships.size) return images;
const imageIdToRelationship = new Map<string, string>();
for (const match of cellImagesXml.matchAll(
/<(?:etc:)?cellImage\b[^>]*>([\s\S]*?)<\/(?:etc:)?cellImage>/g,
)) {
const imageId = match[1].match(
/<(?:xdr:)?cNvPr\b[^>]*\bname="([^"]+)"/,
)?.[1];
const relationshipId = match[1].match(
/<(?:a:)?blip\b[^>]*\br:embed="([^"]+)"/,
)?.[1];
if (imageId && relationshipId) {
imageIdToRelationship.set(imageId, relationshipId);
}
}
for (const cell of worksheetXml.matchAll(/<c\b([^>]*)>([\s\S]*?)<\/c>/g)) {
const reference = cell[1].match(/\br="([A-Z]+\d+)"/i)?.[1] ?? "";
const imageId = decodeXml(cell[2]).match(/DISPIMG\("([^"]+)"/i)?.[1];
if (!reference || !imageId) continue;
const relationshipId = imageIdToRelationship.get(imageId);
const mediaTarget = relationshipId
? relationships.get(relationshipId)
: undefined;
if (!mediaTarget) continue;
const mediaPath = resolveZipPath("xl/cellimages.xml", mediaTarget);
const bytes = entries[mediaPath];
if (!bytes) continue;
const row = Number(reference.match(/\d+$/)?.[0] ?? 0);
if (!row) continue;
images.set(`${row}:${columnIndex(reference)}`, {
bytes,
contentType: contentType(bytes, mediaPath),
fileName: path.posix.basename(mediaPath),
});
}
return images;
}
function parseImages(entries: Record<string, Uint8Array>) {
const images = parseDrawingImages(entries);
for (const [cell, image] of parseRichValueImages(entries)) {
images.set(cell, image);
}
for (const [cell, image] of parseWpsCellImages(entries)) {
images.set(cell, image);
}
return images;
}
function valueAt(
row: string[],
mapping: Map<(typeof PARTNER_BATCH_HEADERS)[number], number>,
header: (typeof PARTNER_BATCH_HEADERS)[number],
) {
const column = mapping.get(header);
return column === undefined ? "" : String(row[column] ?? "").trim();
}
export function parsePartnerBatchWorkbook(input: ArrayBuffer | Uint8Array) {
const bytes = input instanceof Uint8Array ? input : new Uint8Array(input);
if (bytes.byteLength > PARTNER_BATCH_MAX_BYTES) {
throw new Error("批量回填表不能超过80MB");
}
const entries = unzipSync(bytes);
const worksheetBytes = entries["xl/worksheets/sheet1.xml"];
if (!worksheetBytes) throw new Error("Excel 中没有可读取的批量回填工作表");
const sharedXml = entries["xl/sharedStrings.xml"]
? strFromU8(entries["xl/sharedStrings.xml"])
: "";
const sharedStrings = [
...sharedXml.matchAll(/<si\b[^>]*>([\s\S]*?)<\/si>/g),
].map((match) => textNodes(match[1]));
const rows = parseWorksheet(strFromU8(worksheetBytes), sharedStrings);
const header = findHeader(rows);
if (!header) {
throw new Error("表格结构不正确,请使用本领取页面导出的批量回填表");
}
const images = parseImages(entries);
const publishScreenshotColumn = header.mapping.get("笔记截图")!;
const creatorScreenshotColumn = header.mapping.get(
"数据分析截图(单篇笔记数据分析截图)",
)!;
const result: PartnerBatchImportRow[] = [];
for (let index = header.rowIndex + 1; index < rows.length; index += 1) {
const row = rows[index];
const distributionId = valueAt(row, header.mapping, "_系统笔记ID");
if (!distributionId && !row.some((value) => String(value ?? "").trim())) continue;
result.push({
spreadsheetRow: index + 1,
sequence: valueAt(row, header.mapping, "序号(不能改)"),
title: valueAt(row, header.mapping, "标题"),
publishUrl: valueAt(row, header.mapping, "发布链接"),
distributionId,
originalPublishScreenshotKey: valueAt(
row,
header.mapping,
"_原笔记截图",
),
originalCreatorScreenshotKey: valueAt(
row,
header.mapping,
"_原数据分析截图",
),
publishScreenshot:
images.get(`${index + 1}:${publishScreenshotColumn}`) ?? null,
creatorScreenshot:
images.get(`${index + 1}:${creatorScreenshotColumn}`) ?? null,
});
}
if (result.length === 0) throw new Error("表格中没有可回填的笔记");
return result;
}

View File

@@ -1,22 +1,31 @@
import { hashText } from "./mvp-db";
import {
extractPublishUrl,
extractXhsPublishUrl,
platformFromPublishUrl,
safeHttpUrl,
type SupportedPlatform,
} from "./publish-url";
export { extractXhsPublishUrl } from "./publish-url";
export function accountFromPublishLink(input: string) {
const url = safeHttpUrl(extractXhsPublishUrl(input));
export function accountFromPublishLink(
input: string,
expectedPlatform?: SupportedPlatform,
) {
const extracted = expectedPlatform
? extractPublishUrl(input, expectedPlatform)
: extractXhsPublishUrl(input) || extractPublishUrl(input, "抖音");
const url = safeHttpUrl(extracted);
if (!url) return null;
const platform =
url.hostname.includes("xiaohongshu") || url.hostname.includes("xhslink")
? "小红书"
: "其他平台";
const platform = platformFromPublishUrl(url.toString());
if (!platform || (expectedPlatform && platform !== expectedPlatform)) return null;
const noteId =
url.pathname.match(
/\/(?:discovery\/item|explore)\/([A-Za-z0-9_-]{8,80})/,
)?.[1] ?? "";
platform === "抖音"
? url.pathname.match(/\/(?:video|note)\/([A-Za-z0-9_-]{8,80})/)?.[1] ?? ""
: url.pathname.match(
/\/(?:discovery\/item|explore)\/([A-Za-z0-9_-]{8,80})/,
)?.[1] ?? "";
const platformUid = `pending-${hashText(
noteId || `${url.origin}${url.pathname}`,
)}`;

View File

@@ -25,3 +25,44 @@ export function extractXhsPublishUrl(input: string) {
}
return "";
}
export type SupportedPlatform = "小红书" | "抖音";
function platformMatches(url: URL, platform: SupportedPlatform) {
const hostname = url.hostname.toLowerCase();
if (platform === "抖音") {
return hostname === "douyin.com" || hostname.endsWith(".douyin.com");
}
return (
hostname === "xiaohongshu.com" ||
hostname.endsWith(".xiaohongshu.com") ||
hostname === "xhslink.cn" ||
hostname.endsWith(".xhslink.cn")
);
}
export function extractPublishUrl(
input: string,
platform: SupportedPlatform = "小红书",
) {
const candidates =
input.match(/https?:\/\/[^\s<>"',。!?;:、【】()]+/gi) ?? [];
for (const candidate of candidates) {
const cleaned = candidate.replace(/[.,!?;:~)\]}]+$/g, "");
const url = safeHttpUrl(cleaned);
if (url && platformMatches(url, platform)) return url.toString();
}
return "";
}
export function extractAnyPublishUrl(input: string) {
return extractPublishUrl(input, "小红书") || extractPublishUrl(input, "抖音");
}
export function platformFromPublishUrl(input: string): SupportedPlatform | "" {
const url = safeHttpUrl(input);
if (!url) return "";
if (platformMatches(url, "小红书")) return "小红书";
if (platformMatches(url, "抖音")) return "抖音";
return "";
}

View File

@@ -13,6 +13,9 @@ export type RecoveryWorkbookRow = {
images: Array<{
column: number;
image: RecoveryWorkbookImage;
offsetX?: number;
maxWidth?: number;
maxHeight?: number;
}>;
hyperlinks?: Array<{
column: number;
@@ -25,6 +28,7 @@ type WorkbookOptions = {
headers: string[];
columnWidths: number[];
rows: RecoveryWorkbookRow[];
hiddenColumns?: number[];
};
const XML_HEADER = '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>';
@@ -130,9 +134,17 @@ function imageDimensions(image: RecoveryWorkbookImage) {
return { width: 4, height: 3 };
}
function imageDisplaySize(image: RecoveryWorkbookImage) {
function imageDisplaySize(
image: RecoveryWorkbookImage,
maxWidth = 160,
maxHeight = 150,
) {
const dimensions = imageDimensions(image);
const scale = Math.min(160 / dimensions.width, 150 / dimensions.height, 1);
const scale = Math.min(
maxWidth / dimensions.width,
maxHeight / dimensions.height,
1,
);
return {
width: Math.max(28, Math.round(dimensions.width * scale)),
height: Math.max(28, Math.round(dimensions.height * scale)),
@@ -152,6 +164,14 @@ export function buildRecoveryWorkbook(options: WorkbookOptions) {
const imageEntries = options.rows.flatMap((row, rowIndex) =>
row.images.map((item) => ({ ...item, row: rowIndex + 1 })),
);
const imageCells = new Set<string>();
imageEntries.forEach((entry) => {
const key = `${entry.row}:${entry.column}`;
if (imageCells.has(key)) {
throw new Error("Excel 单元格内只能嵌入一张图片,请为每张图片分配独立列");
}
imageCells.add(key);
});
const hyperlinkEntries = options.rows.flatMap((row, rowIndex) =>
(row.hyperlinks ?? [])
.map((item) => ({
@@ -178,7 +198,7 @@ export function buildRecoveryWorkbook(options: WorkbookOptions) {
const reference = `${columnName(columnIndex)}${number}`;
const value = row.cells[columnIndex] ?? "";
if (imageColumns.has(columnIndex)) {
return inlineCell(reference, value || "见图", 4);
return inlineCell(reference, value, 4);
}
return typeof value === "number"
? numberCell(reference, value, 3)
@@ -196,17 +216,22 @@ export function buildRecoveryWorkbook(options: WorkbookOptions) {
const columns = options.headers
.map((_, index) => {
const width = Math.min(70, Math.max(8, options.columnWidths[index] ?? 14));
return `<col min="${index + 1}" max="${index + 1}" width="${width}" customWidth="1"/>`;
const hidden = options.hiddenColumns?.includes(index) ? ' hidden="1"' : "";
return `<col min="${index + 1}" max="${index + 1}" width="${width}" customWidth="1"${hidden}/>`;
})
.join("");
const drawingXml = imageEntries.length
? `${XML_HEADER}<xdr:wsDr xmlns:xdr="http://schemas.openxmlformats.org/drawingml/2006/spreadsheetDrawing" xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships">${imageEntries
.map((entry, index) => {
const size = imageDisplaySize(entry.image);
const width = size.width * 9525;
const height = size.height * 9525;
return `<xdr:oneCellAnchor><xdr:from><xdr:col>${entry.column}</xdr:col><xdr:colOff>57150</xdr:colOff><xdr:row>${entry.row}</xdr:row><xdr:rowOff>57150</xdr:rowOff></xdr:from><xdr:ext cx="${width}" cy="${height}"/><xdr:pic><xdr:nvPicPr><xdr:cNvPr id="${index + 1}" name="${cleanXmlText(entry.image.description || `图片${index + 1}`)}"/><xdr:cNvPicPr><a:picLocks noChangeAspect="1"/></xdr:cNvPicPr></xdr:nvPicPr><xdr:blipFill><a:blip r:embed="rId${index + 1}"/><a:stretch><a:fillRect/></a:stretch></xdr:blipFill><xdr:spPr><a:prstGeom prst="rect"><a:avLst/></a:prstGeom></xdr:spPr></xdr:pic><xdr:clientData/></xdr:oneCellAnchor>`;
const size = imageDisplaySize(
entry.image,
entry.maxWidth ?? 160,
entry.maxHeight ?? 150,
);
const offsetX = entry.offsetX ?? 6;
const offsetY = 6;
return `<xdr:twoCellAnchor editAs="twoCell"><xdr:from><xdr:col>${entry.column}</xdr:col><xdr:colOff>${offsetX * 9525}</xdr:colOff><xdr:row>${entry.row}</xdr:row><xdr:rowOff>${offsetY * 9525}</xdr:rowOff></xdr:from><xdr:to><xdr:col>${entry.column}</xdr:col><xdr:colOff>${(offsetX + size.width) * 9525}</xdr:colOff><xdr:row>${entry.row}</xdr:row><xdr:rowOff>${(offsetY + size.height) * 9525}</xdr:rowOff></xdr:to><xdr:pic><xdr:nvPicPr><xdr:cNvPr id="${index + 1}" name="${cleanXmlText(entry.image.description || `图片${index + 1}`)}"/><xdr:cNvPicPr><a:picLocks noChangeAspect="1"/></xdr:cNvPicPr></xdr:nvPicPr><xdr:blipFill><a:blip r:embed="rId${index + 1}"/><a:stretch><a:fillRect/></a:stretch></xdr:blipFill><xdr:spPr><a:prstGeom prst="rect"><a:avLst/></a:prstGeom></xdr:spPr></xdr:pic><xdr:clientData/></xdr:twoCellAnchor>`;
})
.join("")}</xdr:wsDr>`
: "";
@@ -227,7 +252,10 @@ export function buildRecoveryWorkbook(options: WorkbookOptions) {
const imageContentTypes = [...imageFormats.entries()]
.map(([extension, contentType]) => `<Default Extension="${extension}" ContentType="${contentType}"/>`)
.join("");
const contentTypes = `${XML_HEADER}<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types"><Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/><Default Extension="xml" ContentType="application/xml"/>${imageContentTypes}<Override PartName="/xl/workbook.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml"/><Override PartName="/xl/worksheets/sheet1.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.worksheet+xml"/><Override PartName="/xl/styles.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.styles+xml"/>${imageEntries.length ? '<Override PartName="/xl/drawings/drawing1.xml" ContentType="application/vnd.openxmlformats-officedocument.drawing+xml"/>' : ""}<Override PartName="/docProps/core.xml" ContentType="application/vnd.openxmlformats-package.core-properties+xml"/><Override PartName="/docProps/app.xml" ContentType="application/vnd.openxmlformats-officedocument.extended-properties+xml"/></Types>`;
const drawingContentType = imageEntries.length
? '<Override PartName="/xl/drawings/drawing1.xml" ContentType="application/vnd.openxmlformats-officedocument.drawing+xml"/>'
: "";
const contentTypes = `${XML_HEADER}<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types"><Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/><Default Extension="xml" ContentType="application/xml"/>${imageContentTypes}<Override PartName="/xl/workbook.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml"/><Override PartName="/xl/worksheets/sheet1.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.worksheet+xml"/><Override PartName="/xl/styles.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.styles+xml"/>${drawingContentType}<Override PartName="/docProps/core.xml" ContentType="application/vnd.openxmlformats-package.core-properties+xml"/><Override PartName="/docProps/app.xml" ContentType="application/vnd.openxmlformats-officedocument.extended-properties+xml"/></Types>`;
const hyperlinkRelationshipOffset = imageEntries.length ? 2 : 1;
const hyperlinksXml = hyperlinkEntries.length
? `<hyperlinks>${hyperlinkEntries
@@ -266,7 +294,9 @@ export function buildRecoveryWorkbook(options: WorkbookOptions) {
}
if (imageEntries.length) {
files["xl/drawings/drawing1.xml"] = strToU8(drawingXml);
files["xl/drawings/_rels/drawing1.xml.rels"] = strToU8(drawingRelationships);
files["xl/drawings/_rels/drawing1.xml.rels"] = strToU8(
drawingRelationships,
);
imageEntries.forEach((entry, index) => {
const format = imageFormat(entry.image.contentType, entry.image.bytes);
files[`xl/media/image${index + 1}.${format.extension}`] = entry.image.bytes;

View File

@@ -1,7 +1,7 @@
import { strFromU8, unzipSync } from "fflate";
export const RESOURCE_IMPORT_MAX_ROWS = 100;
export const RESOURCE_IMPORT_MAX_BYTES = 5 * 1024 * 1024;
export const RESOURCE_IMPORT_MAX_ROWS = 10_000;
export const RESOURCE_IMPORT_MAX_BYTES = 20 * 1024 * 1024;
export type ResourceImportRow = {
rowNumber: number;
@@ -12,6 +12,9 @@ export type ResourceImportRow = {
ipLocation: string;
followers: number;
followersResolved: boolean;
gender: "" | "男" | "女";
bio: string;
tags: string[];
cooperationSource: string;
errors: string[];
};
@@ -22,6 +25,9 @@ const HEADER_ALIASES = {
publicAccountId: ["账号ID", "账号号", "小红书号", "抖音号"],
ipLocation: ["IP属地", "IP地址", "IP地区", "IP所在地"],
followers: ["粉丝数", "粉丝", "粉丝量"],
gender: ["性别"],
bio: ["简介", "账号简介", "个人简介"],
tags: ["标签", "账号标签"],
cooperationSource: ["合作来源", "资源来源", "历史合作来源"],
} as const;
@@ -60,9 +66,11 @@ function parseWorksheet(xml: string, sharedStrings: string[]) {
const rowAttributes = rowMatch[1];
const rowNumber = Number(rowAttributes.match(/\br="(\d+)"/)?.[1] ?? rows.length + 1);
const values: string[] = [];
for (const cellMatch of rowMatch[2].matchAll(/<c\b([^>]*)>([\s\S]*?)<\/c>/g)) {
for (const cellMatch of rowMatch[2].matchAll(
/<c\b([^>]*?)(?:\/>|>([\s\S]*?)<\/c>)/g,
)) {
const attributes = cellMatch[1];
const body = cellMatch[2];
const body = cellMatch[2] ?? "";
const reference = attributes.match(/\br="([A-Z]+\d+)"/i)?.[1] ?? "A1";
const type = attributes.match(/\bt="([^"]+)"/)?.[1] ?? "";
const rawValue = body.match(/<v>([\s\S]*?)<\/v>/)?.[1] ?? "";
@@ -183,6 +191,15 @@ export function platformFromProfileUrl(profileUrl: string) {
) {
return "小红书";
}
if (
((url.hostname === "douyin.com" || url.hostname.endsWith(".douyin.com")) &&
/^\/user\/[^/]+/i.test(url.pathname)) ||
((url.hostname === "iesdouyin.com" ||
url.hostname.endsWith(".iesdouyin.com")) &&
/^\/share\/user\/[^/]+/i.test(url.pathname))
) {
return "抖音";
}
} catch {
// URL validation is reported by normalizeRows.
}
@@ -212,10 +229,42 @@ export function parseResourceFollowers(value: string) {
};
}
export function normalizeResourceGender(value: string) {
const normalized = value.trim().toLocaleLowerCase("zh-CN");
if (!normalized || ["未知", "未填写", "待识别", "unknown"].includes(normalized)) {
return { value: "" as const, valid: true };
}
if (["男", "男性", "male", "m"].includes(normalized)) {
return { value: "男" as const, valid: true };
}
if (["女", "女性", "female", "f"].includes(normalized)) {
return { value: "女" as const, valid: true };
}
return { value: "" as const, valid: false };
}
export function normalizeResourceTags(value: string | string[]) {
const source = Array.isArray(value) ? value.join(",") : value;
return [
...new Set(
source
.split(/[,,、;|]/)
.map((item) => item.trim().replace(/^#+/, ""))
.filter(Boolean),
),
];
}
export function resourceImportMissingFields(
row: Pick<
ResourceImportRow,
"nickname" | "publicAccountId" | "ipLocation" | "followersResolved"
| "nickname"
| "publicAccountId"
| "ipLocation"
| "followersResolved"
| "gender"
| "bio"
| "tags"
>,
) {
const missing: string[] = [];
@@ -225,6 +274,8 @@ export function resourceImportMissingFields(
missing.push("ipLocation");
}
if (!row.followersResolved) missing.push("followers");
if (!row.gender) missing.push("gender");
if (!row.bio.trim()) missing.push("bio");
return missing;
}
@@ -242,22 +293,39 @@ function normalizeRows(rows: string[][]) {
const platform = platformFromProfileUrl(profileUrl);
const rawFollowers = valueAt(source, header.mapping, "followers");
const parsedFollowers = parseResourceFollowers(rawFollowers);
const parsedGender = normalizeResourceGender(
valueAt(source, header.mapping, "gender"),
);
const tags = normalizeResourceTags(valueAt(source, header.mapping, "tags"));
const ipLocation = valueAt(source, header.mapping, "ipLocation");
const errors: string[] = [];
if (!rawProfileUrl) errors.push("账号主页不能为空");
else if (!profileUrl) errors.push("账号主页链接格式不正确");
else if (!platform) errors.push("当前自动解析仅支持小红书账号主页");
else if (!platform) errors.push("当前自动解析仅支持小红书或抖音账号主页");
if (!parsedFollowers.valid) {
errors.push("粉丝数格式不正确,请填写数字或如 1.3万、10+");
}
if (!parsedGender.valid) {
errors.push("性别格式不正确,请填写男、女或留空");
}
if (tags.length > 5) {
errors.push("单个账号最多填写 5 个标签,请用逗号分隔");
}
if (/^\d+$/.test(ipLocation)) {
errors.push("IP属地格式不正确请填写省份、地区或国家名称");
}
result.push({
rowNumber: index + 1,
platform,
nickname: valueAt(source, header.mapping, "nickname"),
publicAccountId: valueAt(source, header.mapping, "publicAccountId"),
profileUrl,
ipLocation: valueAt(source, header.mapping, "ipLocation"),
ipLocation,
followers: parsedFollowers.value,
followersResolved: parsedFollowers.resolved,
gender: parsedGender.value,
bio: valueAt(source, header.mapping, "bio"),
tags: tags.slice(0, 5),
cooperationSource: valueAt(source, header.mapping, "cooperationSource"),
errors,
});

View File

@@ -10,6 +10,8 @@ export type CreateDistributionTaskInput = {
name: string;
brand: string;
dueAt: string;
platform?: "小红书" | "抖音";
contentFormat?: "image_text" | "video";
};
export type CreateScreenshotTaskInput = {
@@ -33,6 +35,8 @@ export type DistributionTaskCreation = {
sheetId: string;
sheetName: string;
sourceUrl: string;
platform: "小红书" | "抖音";
contentFormat: "image_text" | "video";
};
export type ScreenshotTaskCreation = {
@@ -55,6 +59,8 @@ type TaskRow = {
source_url: string;
source_sheet_id: string;
source_sheet_name: string;
platform: "小红书" | "抖音";
content_format: "image_text" | "video";
};
function normalizedValue(value: string) {
@@ -99,17 +105,27 @@ async function findExistingTask(
return db
.prepare(
`SELECT id, share_token, name, brand, due_at, quantity,
source_url, source_sheet_id, source_sheet_name
source_url, source_sheet_id, source_sheet_name,
platform, content_format
FROM tasks
WHERE name = ?
AND brand = ?
AND due_at = ?
AND source_url = ?
AND platform = ?
AND content_format = ?
AND status IN ('active', 'importing')
ORDER BY created_at DESC
LIMIT 1`,
)
.bind(input.name, input.brand, input.dueAt, sourceUrl)
.bind(
input.name,
input.brand,
input.dueAt,
sourceUrl,
input.platform ?? "小红书",
input.contentFormat ?? "image_text",
)
.first<TaskRow>();
}
@@ -124,9 +140,10 @@ async function insertTaskFromSource(
.prepare(
`INSERT INTO tasks
(id, name, brand, quantity, claimed_quantity, due_at, status,
platform, content_format,
source_url, source_sheet_id, source_sheet_name, source_synced_at,
share_token)
VALUES (?, ?, ?, ?, 0, ?, 'importing', ?, ?, ?, ?, ?)`,
VALUES (?, ?, ?, ?, 0, ?, 'importing', ?, ?, ?, ?, ?, ?, ?)`,
)
.bind(
taskId,
@@ -134,6 +151,8 @@ async function insertTaskFromSource(
input.brand,
source.rows.length,
input.dueAt,
input.platform ?? "小红书",
input.contentFormat ?? "image_text",
source.url,
source.sheetId,
source.sheetName,
@@ -149,11 +168,15 @@ async function insertTaskFromSource(
...image,
key: `content-assets/${taskId}/${contentId}/${image.index}`,
}));
const videoAssets = row.videos.map((video) => ({
...video,
key: `content-videos/${taskId}/${contentId}/${video.index}`,
}));
return db
.prepare(
`INSERT INTO contents
(id, task_id, title, body, image_assets, status, source, source_row)
VALUES (?, ?, ?, ?, ?, 'available', ?, ?)`,
(id, task_id, title, body, image_assets, video_assets, status, source, source_row)
VALUES (?, ?, ?, ?, ?, ?, 'available', ?, ?)`,
)
.bind(
contentId,
@@ -161,6 +184,7 @@ async function insertTaskFromSource(
row.title,
row.body,
JSON.stringify(imageAssets),
JSON.stringify(videoAssets),
`飞书 · ${source.sheetName}`,
row.sourceRow,
);
@@ -189,11 +213,13 @@ export async function createDistributionTask(
options: { deduplicate?: boolean } = {},
): Promise<DistributionTaskCreation> {
await ensureSchema();
const input = {
const input: Required<CreateDistributionTaskInput> = {
feishuUrl: normalizedFeishuUrl(rawInput.feishuUrl),
name: normalizedValue(rawInput.name),
brand: normalizedValue(rawInput.brand),
dueAt: normalizedDueDate(rawInput.dueAt),
platform: rawInput.platform === "抖音" ? "抖音" : "小红书",
contentFormat: rawInput.contentFormat === "video" ? "video" : "image_text",
};
if (!input.feishuUrl || !input.name || !input.brand) {
throw new Error("请补全飞书链接、任务名称和品牌/项目");
@@ -213,11 +239,19 @@ export async function createDistributionTask(
sheetId: existing.source_sheet_id,
sheetName: existing.source_sheet_name,
sourceUrl: existing.source_url,
platform: existing.platform,
contentFormat: existing.content_format,
};
}
}
const source = await readFeishuSource(input.feishuUrl, bindings);
if (
input.contentFormat === "video" &&
source.rows.some((row) => row.videos.length === 0)
) {
throw new Error("视频任务中存在未识别到视频的内容行,请检查飞书“视频”列");
}
const inserted = await insertTaskFromSource(source, input);
return {
created: true,
@@ -230,6 +264,8 @@ export async function createDistributionTask(
sheetId: source.sheetId,
sheetName: source.sheetName,
sourceUrl: source.url,
platform: input.platform,
contentFormat: input.contentFormat,
};
}
@@ -287,8 +323,8 @@ export async function createScreenshotTask(
db
.prepare(
`INSERT INTO contents
(id, task_id, title, body, image_assets, status, source, source_row)
VALUES (?, ?, ?, ?, ?, 'available', '截图回收任务', ?)`,
(id, task_id, title, body, image_assets, video_assets, status, source, source_row)
VALUES (?, ?, ?, ?, ?, '[]', 'available', '截图回收任务', ?)`,
)
.bind(
uid("content"),

45
lib/video-file.ts Normal file
View File

@@ -0,0 +1,45 @@
const MP4_BRANDS = new Set([
"avc1",
"dash",
"isom",
"M4A ",
"M4B ",
"M4P ",
"M4V ",
"mp41",
"mp42",
"MSNV",
]);
function fourCharacters(bytes: Uint8Array, offset: number) {
return String.fromCharCode(...bytes.slice(offset, offset + 4));
}
function isMp4Brand(brand: string) {
return (
MP4_BRANDS.has(brand) ||
/^iso[2-9]$/.test(brand) ||
/^3g[2p]$/.test(brand.slice(0, 3))
);
}
/** Rejects HTML/JSON/error payloads and non-MP4 containers before download. */
export function hasMp4FileSignature(input: ArrayBuffer | Uint8Array) {
const bytes = input instanceof Uint8Array ? input : new Uint8Array(input);
if (bytes.byteLength < 12 || fourCharacters(bytes, 4) !== "ftyp") {
return false;
}
const declaredSize = new DataView(
bytes.buffer,
bytes.byteOffset,
bytes.byteLength,
).getUint32(0);
const boxEnd = Math.min(
bytes.byteLength,
declaredSize >= 12 ? declaredSize : bytes.byteLength,
);
for (let offset = 8; offset + 4 <= boxEnd; offset += 4) {
if (isMp4Brand(fourCharacters(bytes, offset))) return true;
}
return false;
}

83
lib/workbook-image.ts Normal file
View File

@@ -0,0 +1,83 @@
import sharp from "sharp";
export type WorkbookSourceImage = {
bytes: Uint8Array;
contentType: string;
width?: number | null;
height?: number | null;
description: string;
};
export type WorkbookImageNormalizationOptions = {
maxDimension?: number;
outputFormat?: "png" | "jpeg";
jpegQuality?: number;
};
const NORMALIZABLE_IMAGE = /^image\/(?:jpe?g|png|webp|gif|tiff?|avif|heic|heif)$/i;
function hasImageSignature(bytes: Uint8Array) {
if (bytes.length < 4) return false;
if (bytes[0] === 0xff && bytes[1] === 0xd8) return true;
if (bytes[0] === 0x89 && bytes[1] === 0x50 && bytes[2] === 0x4e && bytes[3] === 0x47) {
return true;
}
if (bytes[0] === 0x47 && bytes[1] === 0x49 && bytes[2] === 0x46) return true;
if (
bytes.length >= 12 &&
String.fromCharCode(...bytes.slice(0, 4)) === "RIFF" &&
String.fromCharCode(...bytes.slice(8, 12)) === "WEBP"
) {
return true;
}
return (
(bytes[0] === 0x49 && bytes[1] === 0x49 && bytes[2] === 0x2a && bytes[3] === 0x00) ||
(bytes[0] === 0x4d && bytes[1] === 0x4d && bytes[2] === 0x00 && bytes[3] === 0x2a)
);
}
/**
* Excel viewers disagree on whether JPEG EXIF orientation should be applied.
* Bake that orientation into the pixels before the image enters the workbook.
* Callers may also resize and encode large source images as JPEG to keep the
* generated workbook within the upload limit.
*/
export async function normalizeWorkbookImage(
image: WorkbookSourceImage,
options: WorkbookImageNormalizationOptions = {},
): Promise<WorkbookSourceImage> {
if (!NORMALIZABLE_IMAGE.test(image.contentType) && !hasImageSignature(image.bytes)) {
return image;
}
try {
let normalized = sharp(image.bytes, { animated: false }).rotate();
if (options.maxDimension && options.maxDimension > 0) {
normalized = normalized.resize({
width: Math.floor(options.maxDimension),
height: Math.floor(options.maxDimension),
fit: "inside",
withoutEnlargement: true,
});
}
normalized =
options.outputFormat === "jpeg"
? normalized
.flatten({ background: "#ffffff" })
.jpeg({
quality: Math.min(95, Math.max(50, options.jpegQuality ?? 82)),
mozjpeg: true,
})
: normalized.png({ compressionLevel: 6 });
const { data, info } = await normalized.toBuffer({ resolveWithObject: true });
return {
...image,
bytes: new Uint8Array(data),
contentType:
options.outputFormat === "jpeg" ? "image/jpeg" : "image/png",
width: info.width,
height: info.height,
};
} catch {
return image;
}
}