feat: 完善视频任务与 KOC 资源库
This commit is contained in:
@@ -1,7 +1,7 @@
|
||||
import {
|
||||
resolveXhsPublicAccountDetails,
|
||||
resolveXhsAccountProfileFromMcp,
|
||||
resolveXhsProfileDetailsFromMcp,
|
||||
resolveAccountProfileFromMcp,
|
||||
resolveProfileDetailsFromMcp,
|
||||
type CollectionMcpConfig,
|
||||
} from "./mcp-collection-client";
|
||||
import { hashText } from "./mvp-db";
|
||||
@@ -11,15 +11,20 @@ type DistributionAccountRow = {
|
||||
id: string;
|
||||
account_id: string | null;
|
||||
publish_url: string | null;
|
||||
platform: string;
|
||||
claimant_contact: string | null;
|
||||
};
|
||||
|
||||
type BackfillRow = DistributionAccountRow & {
|
||||
resolved_account_id: string | null;
|
||||
nickname: string | null;
|
||||
platform: string | null;
|
||||
platform_uid: string | null;
|
||||
public_account_id: string | null;
|
||||
profile_url: string | null;
|
||||
followers: number | null;
|
||||
gender: string | null;
|
||||
bio: string | null;
|
||||
tags: string | null;
|
||||
};
|
||||
|
||||
function isVerifiedXhsProfileUrl(value: string | null) {
|
||||
@@ -37,6 +42,23 @@ function isVerifiedXhsProfileUrl(value: string | null) {
|
||||
}
|
||||
}
|
||||
|
||||
function isVerifiedDouyinProfileUrl(value: string | null) {
|
||||
if (!value) return false;
|
||||
try {
|
||||
const url = new URL(value);
|
||||
const secUid = decodeURIComponent(url.pathname.match(/^\/user\/([^/?#]+)/)?.[1] ?? "");
|
||||
return (
|
||||
url.protocol === "https:" &&
|
||||
(url.hostname === "douyin.com" ||
|
||||
url.hostname.endsWith(".douyin.com")) &&
|
||||
/^[A-Za-z0-9_-]{20,220}$/.test(secUid) &&
|
||||
!/^\d+$/.test(secUid)
|
||||
);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export async function enrichDistributionAccount(
|
||||
db: DatabaseClient,
|
||||
distributionId: string,
|
||||
@@ -44,27 +66,42 @@ export async function enrichDistributionAccount(
|
||||
fallbackNickname: string,
|
||||
mcpConfig: CollectionMcpConfig,
|
||||
) {
|
||||
const profile = await resolveXhsAccountProfileFromMcp(
|
||||
publishUrl,
|
||||
fallbackNickname,
|
||||
mcpConfig,
|
||||
);
|
||||
const current = await db
|
||||
.prepare(
|
||||
`SELECT id, account_id, publish_url
|
||||
FROM distributions
|
||||
WHERE id = ?`,
|
||||
`SELECT d.id, d.account_id, d.publish_url, t.platform,
|
||||
cl.claimant_name AS claimant_contact
|
||||
FROM distributions d
|
||||
JOIN tasks t ON t.id = d.task_id
|
||||
LEFT JOIN claims cl ON cl.id = d.claim_id
|
||||
WHERE d.id = ?`,
|
||||
)
|
||||
.bind(distributionId)
|
||||
.first<DistributionAccountRow>();
|
||||
if (!current || current.publish_url !== publishUrl) {
|
||||
return { updated: false, reason: "stale" as const };
|
||||
}
|
||||
const platform = current.platform === "抖音" ? "抖音" : "小红书";
|
||||
const profile = await resolveAccountProfileFromMcp(
|
||||
publishUrl,
|
||||
fallbackNickname,
|
||||
platform,
|
||||
mcpConfig,
|
||||
);
|
||||
|
||||
const canonicalAccountId = `account-${hashText(
|
||||
`小红书:${profile.platformUid}`,
|
||||
`${platform}:${profile.platformUid}`,
|
||||
)}`;
|
||||
if (current.account_id === canonicalAccountId) {
|
||||
const existingAccount = await db
|
||||
.prepare(
|
||||
`SELECT id FROM accounts
|
||||
WHERE platform = ? AND platform_uid = ?
|
||||
LIMIT 1`,
|
||||
)
|
||||
.bind(platform, profile.platformUid)
|
||||
.first<{ id: string }>();
|
||||
const targetAccountId = existingAccount?.id || canonicalAccountId;
|
||||
const currentContact = (current.claimant_contact || "").trim();
|
||||
if (current.account_id === targetAccountId) {
|
||||
await db
|
||||
.prepare(
|
||||
`UPDATE accounts SET
|
||||
@@ -82,6 +119,12 @@ export async function enrichDistributionAccount(
|
||||
WHEN ? IS NOT NULL AND (? > 0 OR followers = 0) THEN ?
|
||||
ELSE followers
|
||||
END,
|
||||
gender = CASE WHEN ? != '' THEN ? ELSE gender END,
|
||||
bio = CASE WHEN ? != '' THEN ? ELSE bio END,
|
||||
current_contact = CASE WHEN ? != '' THEN ? ELSE current_contact END,
|
||||
post_count = (
|
||||
SELECT COUNT(*) FROM distributions WHERE account_id = ?
|
||||
),
|
||||
last_seen_at = CURRENT_TIMESTAMP
|
||||
WHERE id = ?`,
|
||||
)
|
||||
@@ -96,12 +139,19 @@ export async function enrichDistributionAccount(
|
||||
profile.followers,
|
||||
profile.followers,
|
||||
profile.followers,
|
||||
canonicalAccountId,
|
||||
profile.gender,
|
||||
profile.gender,
|
||||
profile.bio,
|
||||
profile.bio,
|
||||
currentContact,
|
||||
currentContact,
|
||||
targetAccountId,
|
||||
targetAccountId,
|
||||
)
|
||||
.run();
|
||||
return {
|
||||
updated: true,
|
||||
accountId: canonicalAccountId,
|
||||
accountId: targetAccountId,
|
||||
profileUrl: profile.profileUrl,
|
||||
};
|
||||
}
|
||||
@@ -111,8 +161,9 @@ export async function enrichDistributionAccount(
|
||||
db
|
||||
.prepare(
|
||||
`INSERT INTO accounts
|
||||
(id, platform, platform_uid, public_account_id, nickname, profile_url, ip_location, followers, post_count)
|
||||
VALUES (?, '小红书', ?, ?, ?, ?, ?, ?, 1)
|
||||
(id, platform, platform_uid, public_account_id, nickname, profile_url,
|
||||
ip_location, followers, gender, bio, current_contact, post_count)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1)
|
||||
ON CONFLICT(platform, platform_uid) DO UPDATE SET
|
||||
public_account_id = CASE
|
||||
WHEN excluded.public_account_id != ''
|
||||
@@ -131,17 +182,29 @@ export async function enrichDistributionAccount(
|
||||
THEN excluded.followers
|
||||
ELSE accounts.followers
|
||||
END,
|
||||
post_count = accounts.post_count + 1,
|
||||
gender = CASE
|
||||
WHEN excluded.gender != '' THEN excluded.gender ELSE accounts.gender END,
|
||||
bio = CASE
|
||||
WHEN excluded.bio != '' THEN excluded.bio ELSE accounts.bio END,
|
||||
current_contact = CASE
|
||||
WHEN excluded.current_contact != ''
|
||||
THEN excluded.current_contact
|
||||
ELSE accounts.current_contact
|
||||
END,
|
||||
last_seen_at = CURRENT_TIMESTAMP`,
|
||||
)
|
||||
.bind(
|
||||
canonicalAccountId,
|
||||
targetAccountId,
|
||||
platform,
|
||||
profile.platformUid,
|
||||
profile.redId,
|
||||
profile.nickname || fallbackNickname,
|
||||
profile.profileUrl,
|
||||
profile.ipLocation,
|
||||
profile.followers ?? 0,
|
||||
profile.gender,
|
||||
profile.bio,
|
||||
currentContact,
|
||||
),
|
||||
db
|
||||
.prepare(
|
||||
@@ -150,10 +213,25 @@ export async function enrichDistributionAccount(
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE id = ? AND publish_url = ?`,
|
||||
)
|
||||
.bind(canonicalAccountId, distributionId, publishUrl),
|
||||
.bind(targetAccountId, distributionId, publishUrl),
|
||||
db
|
||||
.prepare(
|
||||
`UPDATE accounts SET post_count = (
|
||||
SELECT COUNT(*) FROM distributions WHERE account_id = ?
|
||||
) WHERE id = ?`,
|
||||
)
|
||||
.bind(targetAccountId, targetAccountId),
|
||||
]);
|
||||
|
||||
if (provisionalAccountId) {
|
||||
if (provisionalAccountId && provisionalAccountId !== targetAccountId) {
|
||||
await db
|
||||
.prepare(
|
||||
`UPDATE accounts SET post_count = (
|
||||
SELECT COUNT(*) FROM distributions WHERE account_id = ?
|
||||
) WHERE id = ?`,
|
||||
)
|
||||
.bind(provisionalAccountId, provisionalAccountId)
|
||||
.run();
|
||||
await db
|
||||
.prepare(
|
||||
`DELETE FROM accounts
|
||||
@@ -165,14 +243,14 @@ export async function enrichDistributionAccount(
|
||||
)
|
||||
.bind(
|
||||
provisionalAccountId,
|
||||
canonicalAccountId,
|
||||
targetAccountId,
|
||||
provisionalAccountId,
|
||||
)
|
||||
.run();
|
||||
}
|
||||
return {
|
||||
updated: true,
|
||||
accountId: canonicalAccountId,
|
||||
accountId: targetAccountId,
|
||||
profileUrl: profile.profileUrl,
|
||||
};
|
||||
}
|
||||
@@ -188,15 +266,22 @@ export async function backfillAccountProfiles(
|
||||
d.id,
|
||||
d.account_id,
|
||||
d.publish_url,
|
||||
a.id AS resolved_account_id,
|
||||
a.nickname,
|
||||
a.platform,
|
||||
COALESCE(a.platform, t.platform) AS platform,
|
||||
a.platform_uid,
|
||||
a.public_account_id,
|
||||
a.profile_url,
|
||||
a.followers
|
||||
a.followers,
|
||||
a.gender,
|
||||
a.bio,
|
||||
a.tags,
|
||||
cl.claimant_name AS claimant_contact
|
||||
FROM distributions d
|
||||
JOIN tasks t ON t.id = d.task_id
|
||||
LEFT JOIN accounts a ON a.id = d.account_id
|
||||
WHERE d.publish_url IS NOT NULL
|
||||
LEFT JOIN claims cl ON cl.id = d.claim_id
|
||||
WHERE d.publish_url IS NOT NULL
|
||||
AND d.publish_url != ''
|
||||
ORDER BY d.updated_at DESC
|
||||
LIMIT 100`,
|
||||
@@ -205,8 +290,34 @@ export async function backfillAccountProfiles(
|
||||
let attempted = 0;
|
||||
let updated = 0;
|
||||
let failed = 0;
|
||||
const backfilledAccounts = new Set<string>();
|
||||
|
||||
for (const row of rows.results) {
|
||||
if (
|
||||
row.resolved_account_id &&
|
||||
!backfilledAccounts.has(row.resolved_account_id)
|
||||
) {
|
||||
backfilledAccounts.add(row.resolved_account_id);
|
||||
const claimantContact = row.claimant_contact?.trim() || "";
|
||||
await db
|
||||
.prepare(
|
||||
`UPDATE accounts
|
||||
SET current_contact = CASE
|
||||
WHEN ? != '' THEN ? ELSE current_contact
|
||||
END,
|
||||
post_count = (
|
||||
SELECT COUNT(*) FROM distributions WHERE account_id = ?
|
||||
)
|
||||
WHERE id = ?`,
|
||||
)
|
||||
.bind(
|
||||
claimantContact,
|
||||
claimantContact,
|
||||
row.resolved_account_id,
|
||||
row.resolved_account_id,
|
||||
)
|
||||
.run();
|
||||
}
|
||||
if (attempted >= Math.max(1, Math.min(25, limit))) break;
|
||||
const noteId = (() => {
|
||||
try {
|
||||
@@ -227,19 +338,31 @@ export async function backfillAccountProfiles(
|
||||
row.platform === "小红书" &&
|
||||
!isDemoAccount &&
|
||||
isVerifiedXhsProfileUrl(row.profile_url) &&
|
||||
(!row.public_account_id || Number(row.followers ?? 0) === 0)
|
||||
(!row.public_account_id ||
|
||||
Number(row.followers ?? 0) === 0 ||
|
||||
!row.gender ||
|
||||
!row.bio)
|
||||
) {
|
||||
attempted += 1;
|
||||
attemptedThisRow = true;
|
||||
const details = await resolveXhsProfileDetailsFromMcp(
|
||||
const details = await resolveProfileDetailsFromMcp(
|
||||
row.profile_url ?? "",
|
||||
"小红书",
|
||||
mcpConfig,
|
||||
).catch(() =>
|
||||
resolveXhsPublicAccountDetails(row.profile_url ?? ""),
|
||||
);
|
||||
).catch(async () => ({
|
||||
...(await resolveXhsPublicAccountDetails(row.profile_url ?? "")),
|
||||
gender: "" as const,
|
||||
bio: "",
|
||||
recentNoteTitles: [] as string[],
|
||||
providerTags: [] as string[],
|
||||
}));
|
||||
if (
|
||||
row.account_id &&
|
||||
(details.redId || details.followers !== null)
|
||||
(details.redId ||
|
||||
details.followers !== null ||
|
||||
details.gender ||
|
||||
details.bio ||
|
||||
details.recentNoteTitles.length > 0)
|
||||
) {
|
||||
await db
|
||||
.prepare(
|
||||
@@ -256,6 +379,8 @@ export async function backfillAccountProfiles(
|
||||
WHEN ? != '' AND ? != '待识别' THEN ?
|
||||
ELSE ip_location
|
||||
END,
|
||||
gender = CASE WHEN ? != '' THEN ? ELSE gender END,
|
||||
bio = CASE WHEN ? != '' THEN ? ELSE bio END,
|
||||
last_seen_at = CURRENT_TIMESTAMP
|
||||
WHERE id = ?`,
|
||||
)
|
||||
@@ -268,6 +393,10 @@ export async function backfillAccountProfiles(
|
||||
details.ipLocation ?? "",
|
||||
details.ipLocation ?? "",
|
||||
details.ipLocation ?? "",
|
||||
details.gender,
|
||||
details.gender,
|
||||
details.bio,
|
||||
details.bio,
|
||||
row.account_id,
|
||||
)
|
||||
.run();
|
||||
@@ -287,12 +416,17 @@ export async function backfillAccountProfiles(
|
||||
}
|
||||
const needsEnrichment =
|
||||
!isDemoAccount &&
|
||||
(!row.account_id ||
|
||||
(!row.resolved_account_id ||
|
||||
(row.platform === "小红书" &&
|
||||
!isVerifiedXhsProfileUrl(row.profile_url)) ||
|
||||
(row.platform === "抖音" &&
|
||||
!isVerifiedDouyinProfileUrl(row.profile_url)) ||
|
||||
(row.platform === "小红书" && !row.public_account_id) ||
|
||||
(row.platform === "抖音" && !row.public_account_id) ||
|
||||
(row.platform === "小红书" &&
|
||||
Number(row.followers ?? 0) === 0) ||
|
||||
(row.platform === "抖音" &&
|
||||
Number(row.followers ?? 0) === 0) ||
|
||||
row.platform_uid?.startsWith("pending-") ||
|
||||
Boolean(noteId && row.platform_uid === noteId));
|
||||
if (!needsEnrichment || !row.publish_url) {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import {
|
||||
collectXhsMetricsFromMcp,
|
||||
collectMetricsFromMcp,
|
||||
type CollectionMcpConfig,
|
||||
} from "./mcp-collection-client";
|
||||
import { uid } from "./mvp-db";
|
||||
@@ -10,6 +10,7 @@ type DistributionForCollection = {
|
||||
task_id: string;
|
||||
publish_url: string | null;
|
||||
ocr_status: string;
|
||||
platform: string;
|
||||
};
|
||||
|
||||
type ScheduledTask = {
|
||||
@@ -165,11 +166,16 @@ export async function collectDistributionMetrics(
|
||||
mcpConfig: CollectionMcpConfig,
|
||||
) {
|
||||
const current = await db
|
||||
.prepare("SELECT * FROM distributions WHERE id = ?")
|
||||
.prepare(
|
||||
`SELECT d.*, t.platform
|
||||
FROM distributions d
|
||||
JOIN tasks t ON t.id = d.task_id
|
||||
WHERE d.id = ?`,
|
||||
)
|
||||
.bind(distributionId)
|
||||
.first<DistributionForCollection>();
|
||||
if (!current) throw new Error("分发记录不存在");
|
||||
if (!current.publish_url) throw new Error("笔记尚未回填发布链接");
|
||||
if (!current.publish_url) throw new Error("作品尚未回填发布链接");
|
||||
|
||||
const scheduledAt = `${scheduledDate}T09:00:00+08:00`;
|
||||
const runId = uid("run");
|
||||
@@ -203,7 +209,12 @@ export async function collectDistributionMetrics(
|
||||
.bind(distributionId, scheduledDate)
|
||||
.first<{ id: string; status: string }>();
|
||||
if (!run) throw new Error("采集任务创建失败");
|
||||
if (run.status === "success") return { skipped: true };
|
||||
// Scheduled jobs should remain idempotent, but an operator clicking
|
||||
// “立即采集” is explicitly asking for a fresh snapshot. Reusing the same
|
||||
// daily run lets us correct stale or previously mis-mapped platform data.
|
||||
if (run.status === "success" && source !== "manual") {
|
||||
return { skipped: true };
|
||||
}
|
||||
|
||||
const collectingDescription =
|
||||
source === "automatic"
|
||||
@@ -235,8 +246,12 @@ export async function collectDistributionMetrics(
|
||||
]);
|
||||
|
||||
try {
|
||||
const { likes, comments, collects } =
|
||||
await collectXhsMetricsFromMcp(current.publish_url, mcpConfig);
|
||||
const { likes, comments, collects, shares } =
|
||||
await collectMetricsFromMcp(
|
||||
current.publish_url,
|
||||
current.platform === "抖音" ? "抖音" : "小红书",
|
||||
mcpConfig,
|
||||
);
|
||||
const dayWeight = scheduleDay ?? 1;
|
||||
const successDescription =
|
||||
source === "automatic"
|
||||
@@ -253,6 +268,7 @@ export async function collectDistributionMetrics(
|
||||
likes = ?,
|
||||
comments = ?,
|
||||
collects = ?,
|
||||
shares = ?,
|
||||
status_description = ?,
|
||||
completed_at = CURRENT_TIMESTAMP
|
||||
WHERE id = ?`,
|
||||
@@ -261,6 +277,7 @@ export async function collectDistributionMetrics(
|
||||
likes,
|
||||
comments,
|
||||
collects,
|
||||
shares,
|
||||
successDescription,
|
||||
run.id,
|
||||
),
|
||||
@@ -270,6 +287,7 @@ export async function collectDistributionMetrics(
|
||||
SET latest_likes = ?,
|
||||
latest_comments = ?,
|
||||
latest_collects = ?,
|
||||
latest_shares = ?,
|
||||
collection_status = 'success',
|
||||
collection_status_description = ?,
|
||||
collection_updated_at = CURRENT_TIMESTAMP,
|
||||
@@ -285,12 +303,13 @@ export async function collectDistributionMetrics(
|
||||
likes,
|
||||
comments,
|
||||
collects,
|
||||
shares,
|
||||
successDescription,
|
||||
scheduleDay,
|
||||
distributionId,
|
||||
),
|
||||
]);
|
||||
return { skipped: false, likes, comments, collects };
|
||||
return { skipped: false, likes, comments, collects, shares };
|
||||
} catch (error) {
|
||||
const message =
|
||||
error instanceof Error ? error.message : "公开数据采集失败";
|
||||
|
||||
@@ -2,7 +2,7 @@ const FEISHU_API_ORIGIN = "https://open.feishu.cn";
|
||||
const MAX_SHEET_ROWS = 5_000;
|
||||
const MAX_SHEET_COLUMNS = 100;
|
||||
const MAX_CONTENT_ROWS = 1_000;
|
||||
const MAX_MEDIA_BYTES = 20_000_000;
|
||||
const DEFAULT_MAX_MEDIA_BYTES = 200_000_000;
|
||||
|
||||
export type FeishuBindings = {
|
||||
FEISHU_APP_ID?: string;
|
||||
@@ -16,11 +16,20 @@ export type FeishuSourceImage = {
|
||||
height: number | null;
|
||||
};
|
||||
|
||||
export type FeishuSourceVideo = {
|
||||
index: number;
|
||||
fileToken: string;
|
||||
name: string;
|
||||
mimeType: string;
|
||||
size: number | null;
|
||||
};
|
||||
|
||||
export type FeishuSourceRow = {
|
||||
sourceRow: number;
|
||||
title: string;
|
||||
body: string;
|
||||
images: FeishuSourceImage[];
|
||||
videos: FeishuSourceVideo[];
|
||||
};
|
||||
|
||||
export type FeishuSource = {
|
||||
@@ -106,12 +115,47 @@ function cellText(value: unknown): string {
|
||||
.filter(Boolean)
|
||||
.join("");
|
||||
}
|
||||
if (!isRecord(value) || value.type === "embed-image") return "";
|
||||
if (
|
||||
!isRecord(value) ||
|
||||
value.type === "embed-image" ||
|
||||
value.type === "attachment"
|
||||
) return "";
|
||||
if (typeof value.text === "string") return value.text.trim();
|
||||
if (typeof value.value === "string") return value.value.trim();
|
||||
return "";
|
||||
}
|
||||
|
||||
function extractVideos(value: unknown, output: FeishuSourceVideo[]) {
|
||||
if (Array.isArray(value)) {
|
||||
for (const item of value) extractVideos(item, output);
|
||||
return;
|
||||
}
|
||||
if (!isRecord(value)) return;
|
||||
const fileToken = bindingValue(value.fileToken ?? value.file_token);
|
||||
const mimeType = bindingValue(value.mimeType ?? value.mime_type);
|
||||
const name = bindingValue(value.text ?? value.name ?? value.file_name);
|
||||
const isVideo =
|
||||
value.type === "attachment" &&
|
||||
(mimeType.startsWith("video/") || /\.(?:mp4|mov|m4v|webm)$/i.test(name));
|
||||
if (isVideo && fileToken) {
|
||||
output.push({
|
||||
index: 0,
|
||||
fileToken,
|
||||
name: name || "视频",
|
||||
mimeType: mimeType || "video/mp4",
|
||||
size:
|
||||
typeof value.size === "number" && Number.isFinite(value.size)
|
||||
? value.size
|
||||
: null,
|
||||
});
|
||||
}
|
||||
for (const child of Object.values(value)) {
|
||||
if (child !== value.fileToken && child !== value.file_token) {
|
||||
extractVideos(child, output);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function extractImages(value: unknown, output: FeishuSourceImage[]) {
|
||||
if (Array.isArray(value)) {
|
||||
for (const item of value) extractImages(item, output);
|
||||
@@ -167,6 +211,7 @@ function findHeader(values: unknown[][]) {
|
||||
titleIndex: number;
|
||||
bodyIndex: number;
|
||||
tagsIndex: number;
|
||||
videoIndex: number;
|
||||
score: number;
|
||||
}
|
||||
| undefined;
|
||||
@@ -185,9 +230,13 @@ function findHeader(values: unknown[][]) {
|
||||
const tagsIndex = headers.findIndex((header) =>
|
||||
headerMatches(header, [/标签/, /话题/, /^tags?$/]),
|
||||
);
|
||||
const videoIndex = headers.findIndex((header) =>
|
||||
headerMatches(header, [/^视频\d*$/, /视频文件/, /视频素材/]),
|
||||
);
|
||||
const score =
|
||||
(titleIndex >= 0 ? 5 : 0) +
|
||||
(bodyIndex >= 0 ? 5 : 0) +
|
||||
(videoIndex >= 0 ? 2 : 0) +
|
||||
(idIndex >= 0 ? 1 : 0) +
|
||||
(tagsIndex >= 0 ? 1 : 0);
|
||||
if (!best || score > best.score) {
|
||||
@@ -197,6 +246,7 @@ function findHeader(values: unknown[][]) {
|
||||
titleIndex,
|
||||
bodyIndex,
|
||||
tagsIndex,
|
||||
videoIndex,
|
||||
score,
|
||||
};
|
||||
}
|
||||
@@ -217,6 +267,7 @@ function parseRows(values: unknown[][]) {
|
||||
const usedSourceRows = new Set<number>();
|
||||
const rows: FeishuSourceRow[] = [];
|
||||
let maxImageCount = 0;
|
||||
let maxVideoCount = 0;
|
||||
|
||||
for (
|
||||
let rowIndex = header.rowIndex + 1;
|
||||
@@ -253,7 +304,20 @@ function parseRows(values: unknown[][]) {
|
||||
})
|
||||
.map((image, imageIndex) => ({ ...image, index: imageIndex + 1 }));
|
||||
maxImageCount = Math.max(maxImageCount, images.length);
|
||||
rows.push({ sourceRow, title, body, images });
|
||||
const collectedVideos: FeishuSourceVideo[] = [];
|
||||
if (header.videoIndex >= 0) {
|
||||
extractVideos(row[header.videoIndex], collectedVideos);
|
||||
}
|
||||
const seenVideoTokens = new Set<string>();
|
||||
const videos = collectedVideos
|
||||
.filter((video) => {
|
||||
if (seenVideoTokens.has(video.fileToken)) return false;
|
||||
seenVideoTokens.add(video.fileToken);
|
||||
return true;
|
||||
})
|
||||
.map((video, videoIndex) => ({ ...video, index: videoIndex + 1 }));
|
||||
maxVideoCount = Math.max(maxVideoCount, videos.length);
|
||||
rows.push({ sourceRow, title, body, images, videos });
|
||||
}
|
||||
|
||||
if (rows.length === 0) {
|
||||
@@ -266,6 +330,7 @@ function parseRows(values: unknown[][]) {
|
||||
header.tagsIndex >= 0 ? cellText(headerRow[header.tagsIndex]) : "",
|
||||
cellText(headerRow[header.bodyIndex]) || "正文",
|
||||
...Array.from({ length: maxImageCount }, (_, index) => `图片${index + 1}`),
|
||||
...Array.from({ length: maxVideoCount }, (_, index) => `视频${index + 1}`),
|
||||
].filter(Boolean);
|
||||
|
||||
return {
|
||||
@@ -515,34 +580,37 @@ export async function downloadFeishuMedia(
|
||||
fileToken: string,
|
||||
bindings: FeishuBindings,
|
||||
fetchImpl: FetchLike = fetch,
|
||||
options: { maxBytes?: number; label?: string; timeoutMs?: number } = {},
|
||||
) {
|
||||
const maxBytes = Math.max(1, options.maxBytes ?? DEFAULT_MAX_MEDIA_BYTES);
|
||||
const label = bindingValue(options.label) || "素材";
|
||||
const normalizedToken = bindingValue(fileToken);
|
||||
if (!/^[A-Za-z0-9_-]{8,240}$/.test(normalizedToken)) {
|
||||
throw new FeishuSourceError("飞书图片标识无效", 400);
|
||||
throw new FeishuSourceError(`飞书${label}标识无效`, 400);
|
||||
}
|
||||
const token = await accessToken(bindings, fetchImpl);
|
||||
const response = await fetchImpl(
|
||||
`${FEISHU_API_ORIGIN}/open-apis/drive/v1/medias/${encodeURIComponent(normalizedToken)}/download`,
|
||||
{
|
||||
headers: { Authorization: `Bearer ${token}` },
|
||||
signal: AbortSignal.timeout(20_000),
|
||||
signal: AbortSignal.timeout(options.timeoutMs ?? 120_000),
|
||||
},
|
||||
);
|
||||
const declaredSize = Number(response.headers.get("content-length"));
|
||||
if (Number.isFinite(declaredSize) && declaredSize > MAX_MEDIA_BYTES) {
|
||||
throw new FeishuSourceError("飞书图片超过20MB,无法同步", 413);
|
||||
if (Number.isFinite(declaredSize) && declaredSize > maxBytes) {
|
||||
throw new FeishuSourceError(`飞书${label}文件过大,无法同步`, 413);
|
||||
}
|
||||
if (!response.ok) {
|
||||
throw new FeishuSourceError(
|
||||
response.status === 403
|
||||
? "飞书应用没有这张图片的下载权限"
|
||||
: `下载飞书图片失败(HTTP ${response.status})`,
|
||||
? `飞书应用没有这个${label}的下载权限`
|
||||
: `下载飞书${label}失败(HTTP ${response.status})`,
|
||||
response.status === 403 ? 403 : 502,
|
||||
);
|
||||
}
|
||||
const bytes = await response.arrayBuffer();
|
||||
if (bytes.byteLength > MAX_MEDIA_BYTES) {
|
||||
throw new FeishuSourceError("飞书图片超过20MB,无法同步", 413);
|
||||
if (bytes.byteLength > maxBytes) {
|
||||
throw new FeishuSourceError(`飞书${label}文件过大,无法同步`, 413);
|
||||
}
|
||||
return {
|
||||
bytes,
|
||||
|
||||
@@ -17,6 +17,7 @@ export type XhsPublicMetrics = {
|
||||
likes: number;
|
||||
comments: number;
|
||||
collects: number;
|
||||
shares: number;
|
||||
};
|
||||
|
||||
export type XhsAccountProfile = {
|
||||
@@ -26,6 +27,10 @@ export type XhsAccountProfile = {
|
||||
redId: string;
|
||||
ipLocation: string;
|
||||
followers: number | null;
|
||||
gender: "" | "男" | "女";
|
||||
bio: string;
|
||||
recentNoteTitles: string[];
|
||||
providerTags: string[];
|
||||
};
|
||||
|
||||
type JsonRpcEnvelope = {
|
||||
@@ -341,16 +346,118 @@ function metricsFromToolResult(result: ToolResult): XhsPublicMetrics {
|
||||
}
|
||||
if (!data) throw new Error("采集结果缺少互动数据");
|
||||
|
||||
const count = (value: unknown, label: string) =>
|
||||
value === null || value === undefined || value === ""
|
||||
? 0
|
||||
: metricValue(value, label);
|
||||
return {
|
||||
likes: metricValue(data.likes, "点赞数"),
|
||||
comments: metricValue(data.comments, "评论数"),
|
||||
collects: metricValue(
|
||||
data.collects ?? data.favorites ?? data.favourites,
|
||||
likes: count(
|
||||
data.likes ??
|
||||
data.liked_count ??
|
||||
data.likedCount ??
|
||||
data.like_count ??
|
||||
data.likeCount ??
|
||||
data.digg_count ??
|
||||
data.diggCount,
|
||||
"点赞数",
|
||||
),
|
||||
comments: count(
|
||||
data.comments ?? data.comment_count ?? data.commentCount,
|
||||
"评论数",
|
||||
),
|
||||
collects: count(
|
||||
data.collects ??
|
||||
data.collected_count ??
|
||||
data.collectedCount ??
|
||||
data.favorites ??
|
||||
data.favourites ??
|
||||
data.collect_count ??
|
||||
data.collectCount,
|
||||
"收藏数",
|
||||
),
|
||||
shares: count(
|
||||
data.shares ??
|
||||
data.share_count ??
|
||||
data.shareCount ??
|
||||
data.forwards ??
|
||||
data.forward_count ??
|
||||
data.forwardCount,
|
||||
"转发数",
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
function usableDouyinSecUid(value: unknown) {
|
||||
const candidate = stringValue(value);
|
||||
return candidate && !/^\d+$/.test(candidate) && /^[A-Za-z0-9_-]{20,220}$/.test(candidate)
|
||||
? candidate
|
||||
: "";
|
||||
}
|
||||
|
||||
function verifiedDouyinProfileUrl(value: unknown) {
|
||||
const candidate = stringValue(value);
|
||||
if (!candidate) return "";
|
||||
try {
|
||||
const parsed = new URL(candidate);
|
||||
const secUid = decodeURIComponent(parsed.pathname.match(/^\/user\/([^/?#]+)/)?.[1] ?? "");
|
||||
if (
|
||||
parsed.protocol === "https:" &&
|
||||
(parsed.hostname === "douyin.com" || parsed.hostname.endsWith(".douyin.com")) &&
|
||||
usableDouyinSecUid(secUid)
|
||||
) {
|
||||
return parsed.toString();
|
||||
}
|
||||
} catch {
|
||||
// The public redirect fallback below can still recover the profile URL.
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
async function douyinProfileFromPublicRedirect(
|
||||
publishUrl: string,
|
||||
fetchImpl: typeof fetch,
|
||||
timeoutMs: number,
|
||||
) {
|
||||
let parsed: URL;
|
||||
try {
|
||||
parsed = new URL(publishUrl);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (
|
||||
parsed.protocol !== "https:" ||
|
||||
!(parsed.hostname === "douyin.com" || parsed.hostname.endsWith(".douyin.com"))
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
const response = await fetchImpl(parsed.toString(), {
|
||||
method: "GET",
|
||||
redirect: "manual",
|
||||
signal: AbortSignal.timeout(Math.min(timeoutMs, 15_000)),
|
||||
headers: {
|
||||
"user-agent":
|
||||
"Mozilla/5.0 (iPhone; CPU iPhone OS 18_0 like Mac OS X) AppleWebKit/605.1.15 Mobile/15E148",
|
||||
},
|
||||
});
|
||||
const location = response.headers.get("location");
|
||||
if (!location) return null;
|
||||
const redirectUrl = new URL(location, parsed);
|
||||
const secUid = usableDouyinSecUid(
|
||||
redirectUrl.searchParams.get("sec_uid") ??
|
||||
redirectUrl.searchParams.get("sec_user_id"),
|
||||
);
|
||||
return secUid
|
||||
? {
|
||||
platformUid: secUid,
|
||||
profileUrl: `https://www.douyin.com/user/${encodeURIComponent(secUid)}`,
|
||||
}
|
||||
: null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function successfulToolData(result: ToolResult, fallbackMessage: string) {
|
||||
const root = asRecord(result.payload);
|
||||
const response = asRecord(root?.response) ?? root;
|
||||
@@ -446,6 +553,64 @@ function findValueByKeys(
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function profileGender(value: unknown): "" | "男" | "女" {
|
||||
if (value === 1) return "男";
|
||||
if (value === 2) return "女";
|
||||
const normalized = String(value ?? "").trim().toLocaleLowerCase("zh-CN");
|
||||
if (["男", "男性", "male", "m", "1"].includes(normalized)) return "男";
|
||||
if (["女", "女性", "female", "f", "2"].includes(normalized)) return "女";
|
||||
return "";
|
||||
}
|
||||
|
||||
function recentNoteTitlesFromPayload(value: unknown) {
|
||||
const titles: string[] = [];
|
||||
const visit = (current: unknown, depth = 0) => {
|
||||
if (depth > 12 || titles.length >= 20) return;
|
||||
if (Array.isArray(current)) {
|
||||
current.forEach((item) => visit(item, depth + 1));
|
||||
return;
|
||||
}
|
||||
const record = asRecord(current);
|
||||
if (!record) return;
|
||||
const title = stringValue(record.title ?? record.note_title ?? record.noteTitle);
|
||||
if (
|
||||
title &&
|
||||
(record.note_id || record.noteId || record.url || record.cover) &&
|
||||
!titles.includes(title)
|
||||
) {
|
||||
titles.push(title);
|
||||
}
|
||||
Object.values(record).forEach((child) => visit(child, depth + 1));
|
||||
};
|
||||
visit(value);
|
||||
return titles;
|
||||
}
|
||||
|
||||
function providerTagsFromUser(value: unknown) {
|
||||
const user = findRecord(value, (record) =>
|
||||
Boolean(
|
||||
record.gender !== undefined ||
|
||||
record.desc !== undefined ||
|
||||
record.signature !== undefined ||
|
||||
record.fansCount !== undefined ||
|
||||
record.fans_count !== undefined,
|
||||
),
|
||||
);
|
||||
const raw = user?.tags;
|
||||
if (!Array.isArray(raw)) return [];
|
||||
return [
|
||||
...new Set(
|
||||
raw
|
||||
.map((item) =>
|
||||
typeof item === "string"
|
||||
? item.trim()
|
||||
: stringValue(asRecord(item)?.name ?? asRecord(item)?.title),
|
||||
)
|
||||
.filter(Boolean),
|
||||
),
|
||||
].slice(0, 5);
|
||||
}
|
||||
|
||||
const FOLLOWER_KEYS = new Set([
|
||||
"fans",
|
||||
"fans_count",
|
||||
@@ -529,10 +694,13 @@ async function xhsNoteIdFromShortLink(
|
||||
) {
|
||||
return { noteId: "", profile: null };
|
||||
}
|
||||
if (
|
||||
url.hostname !== "xhslink.cn" &&
|
||||
!url.hostname.endsWith(".xhslink.cn")
|
||||
) {
|
||||
const isShortLink =
|
||||
url.hostname === "xhslink.cn" ||
|
||||
url.hostname.endsWith(".xhslink.cn");
|
||||
const isXhsPage =
|
||||
url.hostname === "xiaohongshu.com" ||
|
||||
url.hostname.endsWith(".xiaohongshu.com");
|
||||
if (!isShortLink && !isXhsPage) {
|
||||
return { noteId: "", profile: null };
|
||||
}
|
||||
|
||||
@@ -608,6 +776,10 @@ function accountProfileFromPublicPage(
|
||||
redId,
|
||||
ipLocation: "待识别",
|
||||
followers: null,
|
||||
gender: "",
|
||||
bio: "",
|
||||
recentNoteTitles: [],
|
||||
providerTags: [],
|
||||
};
|
||||
}
|
||||
|
||||
@@ -698,15 +870,45 @@ function profileDetailsFromToolResult(result: ToolResult) {
|
||||
const redId =
|
||||
findStringByKey(payload, "red_id") ||
|
||||
findStringByKey(payload, "redId") ||
|
||||
findStringByKey(payload, "unique_id") ||
|
||||
findStringByKey(payload, "uniqueId") ||
|
||||
findStringByKey(payload, "short_id") ||
|
||||
findStringByKey(payload, "shortId") ||
|
||||
findStringByKey(payload, "douyin_id") ||
|
||||
findStringByKey(payload, "userId") ||
|
||||
findStringByKey(payload, "user_id");
|
||||
const ipLocation =
|
||||
findStringByKey(payload, "ip_location") ||
|
||||
findStringByKey(payload, "ipLocation");
|
||||
if (followers === null && !nickname && !redId && !ipLocation) {
|
||||
const gender = profileGender(findValueByKeys(payload, new Set(["gender", "sex"])));
|
||||
const bio =
|
||||
findStringByKey(payload, "desc") ||
|
||||
findStringByKey(payload, "description") ||
|
||||
findStringByKey(payload, "signature") ||
|
||||
findStringByKey(payload, "bio");
|
||||
const recentNoteTitles = recentNoteTitlesFromPayload(payload);
|
||||
const providerTags = providerTagsFromUser(payload);
|
||||
if (
|
||||
followers === null &&
|
||||
!nickname &&
|
||||
!redId &&
|
||||
!ipLocation &&
|
||||
!gender &&
|
||||
!bio &&
|
||||
recentNoteTitles.length === 0
|
||||
) {
|
||||
throw new Error("账号主页采集结果缺少可用字段");
|
||||
}
|
||||
return { nickname, followers, redId, ipLocation };
|
||||
return {
|
||||
nickname,
|
||||
followers,
|
||||
redId,
|
||||
ipLocation,
|
||||
gender,
|
||||
bio,
|
||||
recentNoteTitles,
|
||||
providerTags,
|
||||
};
|
||||
}
|
||||
|
||||
function accountProfileFromToolResult(
|
||||
@@ -718,14 +920,18 @@ function accountProfileFromToolResult(
|
||||
data,
|
||||
(record) =>
|
||||
Boolean(
|
||||
stringValue(record.user_id ?? record.userid) &&
|
||||
stringValue(record.user_id ?? record.userid ?? record.userId) &&
|
||||
(stringValue(record.profile_url) ||
|
||||
stringValue(record.nickname ?? record.name)),
|
||||
),
|
||||
);
|
||||
if (!user) throw new Error("账号主页识别结果缺少作者信息");
|
||||
const platformUid = stringValue(user.user_id ?? user.userid);
|
||||
const candidateProfileUrl = stringValue(user.profile_url);
|
||||
const platformUid = stringValue(
|
||||
user.user_id ?? user.userid ?? user.userId,
|
||||
);
|
||||
const candidateProfileUrl = stringValue(
|
||||
user.profile_url ?? user.profileUrl,
|
||||
);
|
||||
let profileUrl = "";
|
||||
if (candidateProfileUrl) {
|
||||
try {
|
||||
@@ -755,6 +961,10 @@ function accountProfileFromToolResult(
|
||||
redId: stringValue(user.red_id),
|
||||
ipLocation: findStringByKey(data, "ip_location") || "待识别",
|
||||
followers: followerCountFromPayload(data),
|
||||
gender: "",
|
||||
bio: "",
|
||||
recentNoteTitles: [],
|
||||
providerTags: [],
|
||||
};
|
||||
}
|
||||
|
||||
@@ -771,18 +981,7 @@ async function completeAccountProfile(
|
||||
"parse_xhs_user_summary",
|
||||
{ url: profile.profileUrl, use_proxy: true },
|
||||
],
|
||||
[
|
||||
"fetch_user_detail",
|
||||
{ link: profile.profileUrl, plant: "xhs" },
|
||||
],
|
||||
] as const) {
|
||||
if (
|
||||
completed.followers !== null &&
|
||||
completed.redId &&
|
||||
completed.ipLocation !== "待识别"
|
||||
) {
|
||||
break;
|
||||
}
|
||||
try {
|
||||
const result = await callMcpTool(
|
||||
fetchImpl,
|
||||
@@ -801,6 +1000,16 @@ async function completeAccountProfile(
|
||||
details.ipLocation && details.ipLocation !== "待识别"
|
||||
? details.ipLocation
|
||||
: completed.ipLocation,
|
||||
gender: details.gender || completed.gender,
|
||||
bio: details.bio || completed.bio,
|
||||
recentNoteTitles:
|
||||
details.recentNoteTitles.length > 0
|
||||
? details.recentNoteTitles
|
||||
: completed.recentNoteTitles,
|
||||
providerTags:
|
||||
details.providerTags.length > 0
|
||||
? details.providerTags
|
||||
: completed.providerTags,
|
||||
};
|
||||
} catch (error) {
|
||||
if (error instanceof McpSessionLostError) throw error;
|
||||
@@ -852,17 +1061,14 @@ async function resolveAccountInSession(
|
||||
const endpoint = buildMcpUrl(config);
|
||||
const timeoutMs = Math.max(5_000, config.timeoutMs ?? 30_000);
|
||||
let noteId = xhsNoteIdFromUrl(publishUrl);
|
||||
let publicPageProfile: XhsAccountProfile | null = null;
|
||||
if (!noteId) {
|
||||
const shortLink = await xhsNoteIdFromShortLink(
|
||||
publishUrl,
|
||||
fallbackNickname,
|
||||
fetchImpl,
|
||||
timeoutMs,
|
||||
);
|
||||
noteId = shortLink.noteId;
|
||||
publicPageProfile = shortLink.profile;
|
||||
}
|
||||
const linkPage = await xhsNoteIdFromShortLink(
|
||||
publishUrl,
|
||||
fallbackNickname,
|
||||
fetchImpl,
|
||||
timeoutMs,
|
||||
);
|
||||
noteId = noteId || linkPage.noteId;
|
||||
const publicPageProfile = linkPage.profile;
|
||||
|
||||
try {
|
||||
const sessionId = await createMcpSession(
|
||||
@@ -870,47 +1076,39 @@ async function resolveAccountInSession(
|
||||
endpoint,
|
||||
timeoutMs,
|
||||
);
|
||||
if (!noteId) {
|
||||
const noteResult = await callMcpTool(
|
||||
fetchImpl,
|
||||
endpoint,
|
||||
sessionId,
|
||||
timeoutMs,
|
||||
"fetch_content_detail",
|
||||
{
|
||||
link: publishUrl,
|
||||
plant: "xhs",
|
||||
include_comments: false,
|
||||
auto_cookie: true,
|
||||
},
|
||||
);
|
||||
const noteData = successfulToolData(
|
||||
noteResult,
|
||||
"无法识别小红书笔记",
|
||||
);
|
||||
noteId =
|
||||
stringValue(noteData.noteId ?? noteData.note_id) ||
|
||||
findStringByKey(noteData, "noteId") ||
|
||||
findStringByKey(noteData, "note_id");
|
||||
}
|
||||
if (!noteId) throw new Error("无法从发布链接识别小红书笔记ID");
|
||||
|
||||
const authorResult = await callMcpTool(
|
||||
const noteResult = await callMcpTool(
|
||||
fetchImpl,
|
||||
endpoint,
|
||||
sessionId,
|
||||
timeoutMs,
|
||||
"collect_xhs_wen_note_detail",
|
||||
"fetch_content_detail",
|
||||
{
|
||||
note_id: noteId,
|
||||
need_desc: false,
|
||||
include_raw: false,
|
||||
link: publishUrl,
|
||||
plant: "xhs",
|
||||
include_comments: false,
|
||||
auto_cookie: true,
|
||||
},
|
||||
);
|
||||
const profile = accountProfileFromToolResult(
|
||||
authorResult,
|
||||
fallbackNickname,
|
||||
const noteData = successfulToolData(
|
||||
noteResult,
|
||||
"无法识别小红书笔记",
|
||||
);
|
||||
noteId =
|
||||
noteId ||
|
||||
stringValue(noteData.noteId ?? noteData.note_id) ||
|
||||
findStringByKey(noteData, "noteId") ||
|
||||
findStringByKey(noteData, "note_id");
|
||||
if (!noteId) throw new Error("无法从发布链接识别小红书笔记ID");
|
||||
let profile: XhsAccountProfile;
|
||||
try {
|
||||
profile = accountProfileFromToolResult(
|
||||
noteResult,
|
||||
fallbackNickname,
|
||||
);
|
||||
} catch {
|
||||
if (!publicPageProfile) throw new Error("笔记数据缺少公开作者主页");
|
||||
profile = publicPageProfile;
|
||||
}
|
||||
return completeAccountProfile(
|
||||
profile,
|
||||
fetchImpl,
|
||||
@@ -937,6 +1135,7 @@ async function resolveAccountInSession(
|
||||
|
||||
async function collectInSession(
|
||||
publishUrl: string,
|
||||
platform: "小红书" | "抖音",
|
||||
config: CollectionMcpConfig,
|
||||
fetchImpl: typeof fetch,
|
||||
) {
|
||||
@@ -952,28 +1151,12 @@ async function collectInSession(
|
||||
"fetch_content_detail",
|
||||
{
|
||||
link: publishUrl,
|
||||
plant: "xhs",
|
||||
plant: platform === "抖音" ? "dy" : "xhs",
|
||||
include_comments: false,
|
||||
auto_cookie: true,
|
||||
},
|
||||
);
|
||||
try {
|
||||
return metricsFromToolResult(primary);
|
||||
} catch {
|
||||
const fallback = await callMcpTool(
|
||||
fetchImpl,
|
||||
endpoint,
|
||||
sessionId,
|
||||
timeoutMs,
|
||||
"parse_xhs_note",
|
||||
{
|
||||
url: publishUrl,
|
||||
include_comments: false,
|
||||
auto_cookie: true,
|
||||
},
|
||||
);
|
||||
return metricsFromToolResult(fallback);
|
||||
}
|
||||
return metricsFromToolResult(primary);
|
||||
}
|
||||
|
||||
export function resolveCollectionMcpConfig(
|
||||
@@ -1005,7 +1188,7 @@ export async function collectXhsMetricsFromMcp(
|
||||
let lastError: unknown;
|
||||
for (let attempt = 0; attempt < MAX_MCP_ATTEMPTS; attempt += 1) {
|
||||
try {
|
||||
return await collectInSession(parsed.toString(), config, fetchImpl);
|
||||
return await collectInSession(parsed.toString(), "小红书", config, fetchImpl);
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
if (!isRetryableTransportError(error)) throw error;
|
||||
@@ -1022,6 +1205,211 @@ export async function collectXhsMetricsFromMcp(
|
||||
: new Error("MCP采集服务暂时不可用");
|
||||
}
|
||||
|
||||
export async function collectMetricsFromMcp(
|
||||
publishUrl: string,
|
||||
platform: "小红书" | "抖音",
|
||||
config: CollectionMcpConfig,
|
||||
fetchImpl: typeof fetch = fetch,
|
||||
) {
|
||||
if (platform === "小红书") {
|
||||
return collectXhsMetricsFromMcp(publishUrl, config, fetchImpl);
|
||||
}
|
||||
let parsed: URL;
|
||||
try {
|
||||
parsed = new URL(publishUrl);
|
||||
} catch {
|
||||
throw new Error("发布链接无效");
|
||||
}
|
||||
if (!["http:", "https:"].includes(parsed.protocol)) {
|
||||
throw new Error("发布链接无效");
|
||||
}
|
||||
let lastError: unknown;
|
||||
for (let attempt = 0; attempt < MAX_MCP_ATTEMPTS; attempt += 1) {
|
||||
try {
|
||||
return await collectInSession(parsed.toString(), platform, config, fetchImpl);
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
if (!isRetryableTransportError(error)) throw error;
|
||||
}
|
||||
}
|
||||
throw lastError instanceof Error
|
||||
? lastError
|
||||
: new Error("MCP采集服务暂时不可用");
|
||||
}
|
||||
|
||||
function douyinProfileFromToolResult(
|
||||
result: ToolResult,
|
||||
fallbackNickname: string,
|
||||
): XhsAccountProfile {
|
||||
const data = successfulToolData(result, "无法识别抖音作品");
|
||||
const author = findRecord(data, (record) =>
|
||||
Boolean(
|
||||
stringValue(
|
||||
record.sec_uid ?? record.secUid ?? record.uid ?? record.user_id ?? record.userId,
|
||||
) && stringValue(record.nickname ?? record.name ?? record.unique_id ?? record.uniqueId),
|
||||
),
|
||||
);
|
||||
if (!author) throw new Error("抖音作品数据缺少作者信息");
|
||||
const verifiedSecUid = usableDouyinSecUid(author.sec_uid ?? author.secUid);
|
||||
const fallbackUid = stringValue(author.uid ?? author.user_id ?? author.userId);
|
||||
const platformUid = verifiedSecUid || fallbackUid;
|
||||
const publicId = stringValue(
|
||||
author.unique_id ?? author.uniqueId ?? author.short_id ?? author.shortId ?? author.douyin_id,
|
||||
);
|
||||
const candidateProfileUrl = verifiedDouyinProfileUrl(
|
||||
author.profile_url ?? author.profileUrl,
|
||||
);
|
||||
const profileUrl = candidateProfileUrl ||
|
||||
(verifiedSecUid
|
||||
? `https://www.douyin.com/user/${encodeURIComponent(verifiedSecUid)}`
|
||||
: "");
|
||||
return {
|
||||
platformUid,
|
||||
nickname: stringValue(author.nickname ?? author.name) || fallbackNickname.trim(),
|
||||
profileUrl,
|
||||
redId: publicId,
|
||||
ipLocation:
|
||||
stringValue(author.ip_location ?? author.ipLocation) ||
|
||||
findStringByKey(data, "ip_location") ||
|
||||
findStringByKey(data, "ipLocation") ||
|
||||
"待识别",
|
||||
followers: followerCountFromPayload(author) ?? followerCountFromPayload(data),
|
||||
gender: profileGender(author.gender ?? author.sex),
|
||||
bio: stringValue(author.desc ?? author.description ?? author.signature ?? author.bio),
|
||||
recentNoteTitles: recentNoteTitlesFromPayload(data),
|
||||
providerTags: providerTagsFromUser(data),
|
||||
};
|
||||
}
|
||||
|
||||
async function resolveDouyinAccountInSession(
|
||||
publishUrl: string,
|
||||
fallbackNickname: string,
|
||||
config: CollectionMcpConfig,
|
||||
fetchImpl: typeof fetch,
|
||||
) {
|
||||
const endpoint = buildMcpUrl(config);
|
||||
const timeoutMs = Math.max(5_000, config.timeoutMs ?? 30_000);
|
||||
const sessionId = await createMcpSession(fetchImpl, endpoint, timeoutMs);
|
||||
const result = await callMcpTool(
|
||||
fetchImpl,
|
||||
endpoint,
|
||||
sessionId,
|
||||
timeoutMs,
|
||||
"fetch_content_detail",
|
||||
{
|
||||
link: publishUrl,
|
||||
plant: "dy",
|
||||
include_comments: false,
|
||||
auto_cookie: true,
|
||||
},
|
||||
);
|
||||
let profile = douyinProfileFromToolResult(result, fallbackNickname);
|
||||
if (!verifiedDouyinProfileUrl(profile.profileUrl)) {
|
||||
const resolved = await douyinProfileFromPublicRedirect(
|
||||
publishUrl,
|
||||
fetchImpl,
|
||||
timeoutMs,
|
||||
);
|
||||
if (resolved) {
|
||||
profile = {
|
||||
...profile,
|
||||
platformUid: resolved.platformUid,
|
||||
profileUrl: resolved.profileUrl,
|
||||
};
|
||||
}
|
||||
}
|
||||
if (!verifiedDouyinProfileUrl(profile.profileUrl)) return profile;
|
||||
try {
|
||||
const detailsResult = await callMcpTool(
|
||||
fetchImpl,
|
||||
endpoint,
|
||||
sessionId,
|
||||
timeoutMs,
|
||||
"parse_dy_user_summary",
|
||||
{ url: profile.profileUrl },
|
||||
);
|
||||
const details = profileDetailsFromToolResult(detailsResult);
|
||||
profile = {
|
||||
...profile,
|
||||
nickname: details.nickname || profile.nickname,
|
||||
redId: details.redId || profile.redId,
|
||||
followers: details.followers ?? profile.followers,
|
||||
ipLocation:
|
||||
details.ipLocation && details.ipLocation !== "待识别"
|
||||
? details.ipLocation
|
||||
: profile.ipLocation,
|
||||
gender: details.gender || profile.gender,
|
||||
bio: details.bio || profile.bio,
|
||||
recentNoteTitles:
|
||||
details.recentNoteTitles.length > 0
|
||||
? details.recentNoteTitles
|
||||
: profile.recentNoteTitles,
|
||||
providerTags:
|
||||
details.providerTags.length > 0
|
||||
? details.providerTags
|
||||
: profile.providerTags,
|
||||
};
|
||||
} catch (error) {
|
||||
if (error instanceof McpSessionLostError) throw error;
|
||||
}
|
||||
return profile;
|
||||
}
|
||||
|
||||
export async function resolveAccountProfileFromMcp(
|
||||
publishUrl: string,
|
||||
fallbackNickname: string,
|
||||
platform: "小红书" | "抖音",
|
||||
config: CollectionMcpConfig,
|
||||
fetchImpl: typeof fetch = fetch,
|
||||
) {
|
||||
if (platform === "小红书") {
|
||||
return resolveXhsAccountProfileFromMcp(
|
||||
publishUrl,
|
||||
fallbackNickname,
|
||||
config,
|
||||
fetchImpl,
|
||||
);
|
||||
}
|
||||
let lastError: unknown;
|
||||
for (let attempt = 0; attempt < MAX_MCP_ATTEMPTS; attempt += 1) {
|
||||
try {
|
||||
return await resolveDouyinAccountInSession(
|
||||
publishUrl,
|
||||
fallbackNickname,
|
||||
config,
|
||||
fetchImpl,
|
||||
);
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
if (!isRetryableTransportError(error)) throw error;
|
||||
}
|
||||
}
|
||||
throw lastError instanceof Error ? lastError : new Error("抖音账号识别失败");
|
||||
}
|
||||
|
||||
export async function resolveProfileDetailsFromMcp(
|
||||
profileUrl: string,
|
||||
platform: "小红书" | "抖音",
|
||||
config: CollectionMcpConfig,
|
||||
fetchImpl: typeof fetch = fetch,
|
||||
) {
|
||||
if (platform === "小红书") {
|
||||
return resolveXhsProfileDetailsFromMcp(profileUrl, config, fetchImpl);
|
||||
}
|
||||
const endpoint = buildMcpUrl(config);
|
||||
const timeoutMs = Math.max(5_000, config.timeoutMs ?? 30_000);
|
||||
const sessionId = await createMcpSession(fetchImpl, endpoint, timeoutMs);
|
||||
const result = await callMcpTool(
|
||||
fetchImpl,
|
||||
endpoint,
|
||||
sessionId,
|
||||
timeoutMs,
|
||||
"parse_dy_user_summary",
|
||||
{ url: profileUrl },
|
||||
);
|
||||
return profileDetailsFromToolResult(result);
|
||||
}
|
||||
|
||||
export async function resolveXhsAccountProfileFromMcp(
|
||||
publishUrl: string,
|
||||
fallbackNickname: string,
|
||||
|
||||
@@ -12,7 +12,7 @@ import {
|
||||
type CollectionMcpBindings,
|
||||
} from "./mcp-collection-client";
|
||||
import { ensureSchema, getRawDb } from "./mvp-db";
|
||||
import { extractXhsPublishUrl } from "./publish-url";
|
||||
import { extractAnyPublishUrl } from "./publish-url";
|
||||
import { buildClaimUrl } from "./task-service";
|
||||
|
||||
export type McpOperationBindings = CollectionMcpBindings & {
|
||||
@@ -123,9 +123,9 @@ export async function taskGet(taskId: string, portalUrl: string) {
|
||||
const [notes, claims, runs] = await Promise.all([
|
||||
db
|
||||
.prepare(
|
||||
`SELECT c.id AS content_id, c.source_row, c.title, c.body, c.image_assets, c.status AS content_status,
|
||||
`SELECT c.id AS content_id, c.source_row, c.title, c.body, c.image_assets, c.video_assets, c.status AS content_status,
|
||||
d.id AS distribution_id, d.status AS distribution_status, d.publish_url, d.publish_time,
|
||||
d.exposure, d.views, d.latest_likes, d.latest_comments, d.latest_collects,
|
||||
d.exposure, d.views, d.latest_likes, d.latest_comments, d.latest_collects, d.latest_shares,
|
||||
d.collection_status, d.collection_status_description, d.collection_updated_at,
|
||||
a.id AS account_id, a.nickname AS account_nickname, a.profile_url, a.followers,
|
||||
p.name AS cooperation_source, cl.claimant_name
|
||||
@@ -172,12 +172,14 @@ export async function taskGet(taskId: string, portalUrl: string) {
|
||||
notes: notes.results.map((row) => ({
|
||||
...row,
|
||||
image_assets: parseJsonArray(row.image_assets),
|
||||
video_assets: parseJsonArray(row.video_assets),
|
||||
total_interactions:
|
||||
row.latest_likes == null
|
||||
? null
|
||||
: Number(row.latest_likes) +
|
||||
Number(row.latest_comments ?? 0) +
|
||||
Number(row.latest_collects ?? 0),
|
||||
Number(row.latest_collects ?? 0) +
|
||||
Number(row.latest_shares ?? 0),
|
||||
})),
|
||||
claims: claims.results,
|
||||
collection_runs: runs.results,
|
||||
@@ -220,7 +222,8 @@ export async function recoveryList(
|
||||
const [rows, count] = await Promise.all([
|
||||
db
|
||||
.prepare(
|
||||
`SELECT d.*, t.name AS task_name, c.source_row, c.title,
|
||||
`SELECT d.*, t.name AS task_name, t.platform AS task_platform,
|
||||
t.content_format, c.source_row, c.title,
|
||||
a.nickname AS account_nickname, a.profile_url, p.name AS cooperation_source,
|
||||
cl.claimant_name ${base}
|
||||
ORDER BY COALESCE(d.publish_time, d.claimed_at) DESC LIMIT ${limit} OFFSET ${offset}`,
|
||||
@@ -241,7 +244,7 @@ export async function recoveryList(
|
||||
total_interactions:
|
||||
row.latest_likes == null
|
||||
? null
|
||||
: Number(row.latest_likes) + Number(row.latest_comments ?? 0) + Number(row.latest_collects ?? 0),
|
||||
: Number(row.latest_likes) + Number(row.latest_comments ?? 0) + Number(row.latest_collects ?? 0) + Number(row.latest_shares ?? 0),
|
||||
})),
|
||||
};
|
||||
}
|
||||
@@ -358,9 +361,11 @@ function resourceWhere(input: ResourceFilters) {
|
||||
const conditions: string[] = [];
|
||||
const bindings: unknown[] = [];
|
||||
if (input.query?.trim()) {
|
||||
conditions.push("(a.nickname LIKE ? ESCAPE '\\' OR a.public_account_id LIKE ? ESCAPE '\\')");
|
||||
conditions.push(
|
||||
"(a.nickname LIKE ? ESCAPE '\\' OR a.public_account_id LIKE ? ESCAPE '\\' OR a.current_contact LIKE ? ESCAPE '\\' OR a.tags LIKE ? ESCAPE '\\' OR a.bio LIKE ? ESCAPE '\\')",
|
||||
);
|
||||
const pattern = like(input.query.trim());
|
||||
bindings.push(pattern, pattern);
|
||||
bindings.push(pattern, pattern, pattern, pattern, pattern);
|
||||
}
|
||||
if (input.ipLocation?.trim()) {
|
||||
conditions.push("a.ip_location LIKE ? ESCAPE '\\'");
|
||||
@@ -369,8 +374,12 @@ function resourceWhere(input: ResourceFilters) {
|
||||
if (input.cooperationSource?.trim()) {
|
||||
conditions.push(
|
||||
`(a.cooperation_source LIKE ? ESCAPE '\\' OR
|
||||
EXISTS (SELECT 1 FROM distributions dx JOIN partners px ON px.id = dx.partner_id
|
||||
WHERE dx.account_id = a.id AND px.name LIKE ? ESCAPE '\\'))`,
|
||||
EXISTS (SELECT 1 FROM distributions dx
|
||||
JOIN partners px ON px.id = dx.partner_id
|
||||
LEFT JOIN claims cx ON cx.id = dx.claim_id
|
||||
WHERE dx.account_id = a.id
|
||||
AND (cx.claimant_name IS NULL OR px.name != cx.claimant_name)
|
||||
AND px.name LIKE ? ESCAPE '\\'))`,
|
||||
);
|
||||
const pattern = like(input.cooperationSource.trim());
|
||||
bindings.push(pattern, pattern);
|
||||
@@ -389,7 +398,12 @@ export async function resourceSearch(input: ResourceFilters) {
|
||||
const db = getRawDb();
|
||||
const select = `SELECT a.*,
|
||||
(SELECT COUNT(*) FROM distributions d WHERE d.account_id = a.id) AS cooperation_count,
|
||||
(SELECT GROUP_CONCAT(DISTINCT p.name) FROM distributions d JOIN partners p ON p.id = d.partner_id WHERE d.account_id = a.id) AS cooperation_sources`;
|
||||
(SELECT GROUP_CONCAT(DISTINCT p.name)
|
||||
FROM distributions d
|
||||
JOIN partners p ON p.id = d.partner_id
|
||||
LEFT JOIN claims c ON c.id = d.claim_id
|
||||
WHERE d.account_id = a.id
|
||||
AND (c.claimant_name IS NULL OR p.name != c.claimant_name)) AS cooperation_sources`;
|
||||
const [rows, count] = await Promise.all([
|
||||
db.prepare(`${select} FROM accounts a ${where} ORDER BY a.last_seen_at DESC LIMIT ${limit} OFFSET ${offset}`)
|
||||
.bind(...bindings).all<Record<string, unknown>>(),
|
||||
@@ -422,7 +436,7 @@ export async function resourceGet(accountId: string) {
|
||||
if (!account) throw new Error("账号不存在");
|
||||
const history = await db.prepare(
|
||||
`SELECT d.id AS distribution_id, d.task_id, t.name AS task_name, c.title,
|
||||
d.publish_url, d.publish_time, d.latest_likes, d.latest_comments, d.latest_collects,
|
||||
d.publish_url, d.publish_time, d.latest_likes, d.latest_comments, d.latest_collects, d.latest_shares,
|
||||
d.exposure, d.views, p.name AS cooperation_source, cl.claimant_name
|
||||
FROM distributions d JOIN tasks t ON t.id = d.task_id
|
||||
JOIN contents c ON c.id = d.content_id JOIN partners p ON p.id = d.partner_id
|
||||
@@ -438,7 +452,7 @@ export async function backfillResourceProfile(
|
||||
) {
|
||||
await ensureSchema();
|
||||
const db = getRawDb();
|
||||
const publishUrl = input.publishUrl ? extractXhsPublishUrl(input.publishUrl) : "";
|
||||
const publishUrl = input.publishUrl ? extractAnyPublishUrl(input.publishUrl) : "";
|
||||
const row = input.distributionId
|
||||
? await db.prepare(
|
||||
`SELECT d.id, d.publish_url, COALESCE(a.nickname, '') AS nickname
|
||||
|
||||
@@ -52,7 +52,7 @@ const pagination = {
|
||||
};
|
||||
|
||||
const resourceFilters = {
|
||||
query: z.string().max(100).optional().describe("账号名称或小红书号/抖音号,支持模糊搜索"),
|
||||
query: z.string().max(100).optional().describe("账号名称、账号号、简介或标签,支持模糊搜索"),
|
||||
ip_location: z.string().max(100).optional().describe("IP地区关键词,支持模糊搜索"),
|
||||
cooperation_source: z.string().max(100).optional().describe("历史合作来源关键词,支持模糊搜索"),
|
||||
platform: z.string().max(30).optional().describe("平台,例如小红书;不传表示全部"),
|
||||
@@ -159,7 +159,7 @@ export function registerMcpOperationTools(server: McpServer, options: Options) {
|
||||
"collection_collect_now",
|
||||
{
|
||||
title: "立即采集指定笔记",
|
||||
description: "对指定分发记录立即采集点赞、收藏和评论数据。",
|
||||
description: "对指定分发记录立即采集互动数据;小红书为点赞/收藏/评论,抖音另含转发。",
|
||||
inputSchema: z.object({
|
||||
distribution_id: z.string().min(1).describe("分发记录ID,可从 task_get 或 recovery_list 获取"),
|
||||
schedule_day: z.number().int().min(1).max(7).optional().describe("标记为第几天采集,可不传"),
|
||||
@@ -190,7 +190,7 @@ export function registerMcpOperationTools(server: McpServer, options: Options) {
|
||||
"resource_search",
|
||||
{
|
||||
title: "搜索 KOC 账号资源",
|
||||
description: "按账号名称/账号号、IP地区、合作来源或平台搜索 KOC 资源。",
|
||||
description: "按账号名称/账号号/标签、IP地区、合作来源或平台搜索 KOC 资源。",
|
||||
inputSchema: z.object({ ...resourceFilters, ...pagination }),
|
||||
annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true },
|
||||
},
|
||||
@@ -204,7 +204,7 @@ export function registerMcpOperationTools(server: McpServer, options: Options) {
|
||||
"resource_get",
|
||||
{
|
||||
title: "查看 KOC 账号详情",
|
||||
description: "查看账号主页、账号号、粉丝数、IP地区以及全部合作记录。",
|
||||
description: "查看账号主页、账号号、粉丝数、性别、简介、标签、IP地区以及全部合作记录。",
|
||||
inputSchema: z.object({ account_id: z.string().min(1).describe("KOC LOOP 账号ID") }),
|
||||
annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true },
|
||||
},
|
||||
@@ -218,10 +218,10 @@ export function registerMcpOperationTools(server: McpServer, options: Options) {
|
||||
"resource_backfill_profile",
|
||||
{
|
||||
title: "补全公开账号信息",
|
||||
description: "根据已回填的发布链接补全账号主页、昵称、小红书号、IP地区和粉丝数。",
|
||||
description: "根据已回填的小红书或抖音作品链接补全账号主页、昵称、账号号、IP地区和粉丝数。",
|
||||
inputSchema: z.object({
|
||||
distribution_id: z.string().optional().describe("分发记录ID,和发布链接二选一"),
|
||||
publish_url: z.string().optional().describe("小红书发布链接或包含链接的分享文案"),
|
||||
publish_url: z.string().optional().describe("小红书或抖音作品链接,也可传包含链接的分享文案"),
|
||||
}).refine((value) => Boolean(value.distribution_id || value.publish_url), "请提供分发记录ID或发布链接"),
|
||||
annotations: { readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: true },
|
||||
},
|
||||
|
||||
@@ -94,6 +94,8 @@ export async function ensureSchema(database?: DatabaseClient) {
|
||||
due_at TEXT NOT NULL,
|
||||
status TEXT NOT NULL DEFAULT 'active',
|
||||
task_type TEXT NOT NULL DEFAULT 'content_publish',
|
||||
platform TEXT NOT NULL DEFAULT '小红书',
|
||||
content_format TEXT NOT NULL DEFAULT 'image_text',
|
||||
source_url TEXT NOT NULL DEFAULT '',
|
||||
source_sheet_id TEXT NOT NULL DEFAULT '',
|
||||
source_sheet_name TEXT NOT NULL DEFAULT '',
|
||||
@@ -110,6 +112,7 @@ export async function ensureSchema(database?: DatabaseClient) {
|
||||
title TEXT NOT NULL,
|
||||
body TEXT NOT NULL DEFAULT '',
|
||||
image_assets TEXT NOT NULL DEFAULT '[]',
|
||||
video_assets TEXT NOT NULL DEFAULT '[]',
|
||||
status TEXT NOT NULL DEFAULT 'available',
|
||||
source TEXT NOT NULL DEFAULT '飞书内容表',
|
||||
source_row INTEGER,
|
||||
@@ -124,9 +127,13 @@ export async function ensureSchema(database?: DatabaseClient) {
|
||||
profile_url TEXT NOT NULL DEFAULT '',
|
||||
ip_location TEXT NOT NULL DEFAULT '待识别',
|
||||
followers INTEGER NOT NULL DEFAULT 0,
|
||||
gender TEXT NOT NULL DEFAULT '',
|
||||
bio TEXT NOT NULL DEFAULT '',
|
||||
tags TEXT NOT NULL DEFAULT '',
|
||||
post_count INTEGER NOT NULL DEFAULT 0,
|
||||
avg_views INTEGER NOT NULL DEFAULT 0,
|
||||
cooperation_source TEXT NOT NULL DEFAULT '',
|
||||
current_contact TEXT NOT NULL DEFAULT '',
|
||||
first_seen_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
last_seen_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
)`,
|
||||
@@ -185,6 +192,7 @@ export async function ensureSchema(database?: DatabaseClient) {
|
||||
latest_likes INTEGER,
|
||||
latest_comments INTEGER,
|
||||
latest_collects INTEGER,
|
||||
latest_shares INTEGER,
|
||||
collection_status TEXT NOT NULL DEFAULT 'pending',
|
||||
collection_status_description TEXT,
|
||||
collection_updated_at TEXT,
|
||||
@@ -202,6 +210,7 @@ export async function ensureSchema(database?: DatabaseClient) {
|
||||
likes INTEGER,
|
||||
comments INTEGER,
|
||||
collects INTEGER,
|
||||
shares INTEGER,
|
||||
status_description TEXT,
|
||||
started_at TEXT,
|
||||
completed_at TEXT,
|
||||
@@ -293,6 +302,14 @@ export async function ensureSchema(database?: DatabaseClient) {
|
||||
"public_account_id",
|
||||
"public_account_id TEXT NOT NULL DEFAULT ''",
|
||||
);
|
||||
await ensureColumn("accounts", "gender", "gender TEXT NOT NULL DEFAULT ''");
|
||||
await ensureColumn("accounts", "bio", "bio TEXT NOT NULL DEFAULT ''");
|
||||
await ensureColumn("accounts", "tags", "tags TEXT NOT NULL DEFAULT ''");
|
||||
await ensureColumn(
|
||||
"accounts",
|
||||
"current_contact",
|
||||
"current_contact TEXT NOT NULL DEFAULT ''",
|
||||
);
|
||||
await ensureColumn("distributions", "claim_id", "claim_id TEXT");
|
||||
await ensureColumn(
|
||||
"distributions",
|
||||
@@ -796,22 +813,44 @@ export async function getDashboardData() {
|
||||
await Promise.all([
|
||||
db.prepare("SELECT * FROM partners ORDER BY created_at DESC").all(),
|
||||
db.prepare("SELECT * FROM tasks ORDER BY created_at DESC").all(),
|
||||
db.prepare("SELECT * FROM accounts ORDER BY last_seen_at DESC").all(),
|
||||
db
|
||||
.prepare(
|
||||
`SELECT
|
||||
a.*,
|
||||
COALESCE(
|
||||
(
|
||||
SELECT d.publish_url
|
||||
FROM distributions d
|
||||
WHERE d.account_id = a.id
|
||||
AND TRIM(COALESCE(d.publish_url, '')) != ''
|
||||
ORDER BY d.updated_at DESC, d.claimed_at DESC
|
||||
LIMIT 1
|
||||
),
|
||||
''
|
||||
) AS latest_publish_url
|
||||
FROM accounts a
|
||||
ORDER BY a.last_seen_at DESC`,
|
||||
)
|
||||
.all(),
|
||||
db
|
||||
.prepare(
|
||||
`SELECT
|
||||
d.*,
|
||||
c.title AS content_title,
|
||||
p.name AS partner_name,
|
||||
cl.claimant_name AS claimant_name,
|
||||
a.nickname AS account_nickname,
|
||||
a.platform AS account_platform,
|
||||
t.name AS task_name,
|
||||
t.brand AS task_brand,
|
||||
t.task_type AS task_type,
|
||||
t.platform AS task_platform,
|
||||
t.content_format AS content_format,
|
||||
t.due_at AS due_at
|
||||
FROM distributions d
|
||||
JOIN contents c ON c.id = d.content_id
|
||||
JOIN partners p ON p.id = d.partner_id
|
||||
LEFT JOIN claims cl ON cl.id = d.claim_id
|
||||
JOIN tasks t ON t.id = d.task_id
|
||||
LEFT JOIN accounts a ON a.id = d.account_id
|
||||
ORDER BY d.updated_at DESC, d.claimed_at DESC`,
|
||||
|
||||
547
lib/partner-batch-workbook.ts
Normal file
547
lib/partner-batch-workbook.ts
Normal file
@@ -0,0 +1,547 @@
|
||||
import path from "node:path";
|
||||
import { strFromU8, unzipSync } from "fflate";
|
||||
|
||||
export const PARTNER_BATCH_HEADERS = [
|
||||
"序号(不能改)",
|
||||
"标题",
|
||||
"笔记内容(正文+话题)",
|
||||
"图片",
|
||||
"发布链接",
|
||||
"笔记截图",
|
||||
"数据分析截图(单篇笔记数据分析截图)",
|
||||
"_系统笔记ID",
|
||||
"_原笔记截图",
|
||||
"_原数据分析截图",
|
||||
] as const;
|
||||
|
||||
export const PARTNER_BATCH_VISIBLE_COLUMN_COUNT = 7;
|
||||
export const PARTNER_BATCH_MAX_BYTES = 80_000_000;
|
||||
|
||||
export type PartnerBatchWorkbookColumns = {
|
||||
headers: string[];
|
||||
columnWidths: number[];
|
||||
sourceImageStartColumn: number;
|
||||
sourceImageCount: number;
|
||||
sourceVideoStartColumn: number;
|
||||
sourceVideoCount: number;
|
||||
publishUrlColumn: number;
|
||||
publishScreenshotColumn: number;
|
||||
creatorScreenshotColumn: number;
|
||||
systemColumn: number;
|
||||
};
|
||||
|
||||
function nonNegativeInteger(value: number) {
|
||||
return Number.isFinite(value) ? Math.max(0, Math.floor(value)) : 0;
|
||||
}
|
||||
|
||||
export function buildPartnerBatchWorkbookColumns(input: {
|
||||
contentFormat: "image_text" | "video";
|
||||
maxSourceImages: number;
|
||||
maxSourceVideos: number;
|
||||
}): PartnerBatchWorkbookColumns {
|
||||
const sourceImageCount =
|
||||
input.contentFormat === "video"
|
||||
? 0
|
||||
: Math.max(1, nonNegativeInteger(input.maxSourceImages));
|
||||
const sourceVideoCount =
|
||||
input.contentFormat === "video"
|
||||
? Math.max(1, nonNegativeInteger(input.maxSourceVideos))
|
||||
: 0;
|
||||
const sourceImageStartColumn = 3;
|
||||
const sourceVideoStartColumn = sourceImageStartColumn + sourceImageCount;
|
||||
const publishUrlColumn = sourceVideoStartColumn + sourceVideoCount;
|
||||
const publishScreenshotColumn = publishUrlColumn + 1;
|
||||
const creatorScreenshotColumn = publishScreenshotColumn + 1;
|
||||
const systemColumn = creatorScreenshotColumn + 1;
|
||||
return {
|
||||
headers: [
|
||||
"序号(不能改)",
|
||||
"标题",
|
||||
"笔记内容(正文+话题)",
|
||||
...Array.from(
|
||||
{ length: sourceImageCount },
|
||||
(_, index) => `图片${index + 1}`,
|
||||
),
|
||||
...Array.from(
|
||||
{ length: sourceVideoCount },
|
||||
(_, index) => `视频${index + 1}`,
|
||||
),
|
||||
"发布链接",
|
||||
"笔记截图",
|
||||
"数据分析截图(单篇笔记数据分析截图)",
|
||||
"_系统笔记ID",
|
||||
"_原笔记截图",
|
||||
"_原数据分析截图",
|
||||
],
|
||||
columnWidths: [
|
||||
14,
|
||||
30,
|
||||
62,
|
||||
...Array.from({ length: sourceImageCount }, () => 24),
|
||||
...Array.from({ length: sourceVideoCount }, () => 20),
|
||||
45,
|
||||
28,
|
||||
32,
|
||||
22,
|
||||
22,
|
||||
22,
|
||||
],
|
||||
sourceImageStartColumn,
|
||||
sourceImageCount,
|
||||
sourceVideoStartColumn,
|
||||
sourceVideoCount,
|
||||
publishUrlColumn,
|
||||
publishScreenshotColumn,
|
||||
creatorScreenshotColumn,
|
||||
systemColumn,
|
||||
};
|
||||
}
|
||||
|
||||
function firstForwardedValue(value: string | null) {
|
||||
return value?.split(",")[0]?.trim() ?? "";
|
||||
}
|
||||
|
||||
function httpOrigin(value: string) {
|
||||
try {
|
||||
const url = new URL(value);
|
||||
return url.protocol === "http:" || url.protocol === "https:"
|
||||
? url.origin
|
||||
: "";
|
||||
} catch {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
export function resolvePartnerWorkbookOrigin(
|
||||
request: Request,
|
||||
configuredOrigin = "",
|
||||
) {
|
||||
const requestUrl = new URL(request.url);
|
||||
const host =
|
||||
firstForwardedValue(request.headers.get("x-forwarded-host")) ||
|
||||
firstForwardedValue(request.headers.get("host"));
|
||||
const forwardedProtocol = firstForwardedValue(
|
||||
request.headers.get("x-forwarded-proto"),
|
||||
).toLowerCase();
|
||||
const protocol = ["http", "https"].includes(forwardedProtocol)
|
||||
? forwardedProtocol
|
||||
: requestUrl.protocol.replace(":", "");
|
||||
const proxyOrigin = host ? httpOrigin(`${protocol}://${host}`) : "";
|
||||
return (
|
||||
httpOrigin(configuredOrigin) ||
|
||||
proxyOrigin ||
|
||||
httpOrigin(requestUrl.origin) ||
|
||||
requestUrl.origin
|
||||
);
|
||||
}
|
||||
|
||||
export type PartnerBatchImage = {
|
||||
bytes: Uint8Array;
|
||||
contentType: string;
|
||||
fileName: string;
|
||||
};
|
||||
|
||||
export type PartnerBatchImportRow = {
|
||||
spreadsheetRow: number;
|
||||
sequence: string;
|
||||
title: string;
|
||||
publishUrl: string;
|
||||
distributionId: string;
|
||||
originalPublishScreenshotKey: string;
|
||||
originalCreatorScreenshotKey: string;
|
||||
publishScreenshot: PartnerBatchImage | null;
|
||||
creatorScreenshot: PartnerBatchImage | null;
|
||||
};
|
||||
|
||||
function decodeXml(value: string) {
|
||||
return value
|
||||
.replace(/<[^>]+>/g, "")
|
||||
.replace(/</g, "<")
|
||||
.replace(/>/g, ">")
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, "'")
|
||||
.replace(/&/g, "&")
|
||||
.replace(/&#(\d+);/g, (_, code) => String.fromCodePoint(Number(code)))
|
||||
.replace(/&#x([0-9a-f]+);/gi, (_, code) =>
|
||||
String.fromCodePoint(Number.parseInt(code, 16)),
|
||||
);
|
||||
}
|
||||
|
||||
function xmlAttribute(value: string) {
|
||||
return decodeXml(value);
|
||||
}
|
||||
|
||||
function textNodes(xml: string) {
|
||||
return [...xml.matchAll(/<t\b[^>]*>([\s\S]*?)<\/t>/g)]
|
||||
.map((match) => decodeXml(match[1]))
|
||||
.join("");
|
||||
}
|
||||
|
||||
function columnIndex(reference: string) {
|
||||
const letters = reference.match(/^[A-Z]+/i)?.[0]?.toUpperCase() ?? "";
|
||||
let result = 0;
|
||||
for (const letter of letters) result = result * 26 + letter.charCodeAt(0) - 64;
|
||||
return Math.max(0, result - 1);
|
||||
}
|
||||
|
||||
function parseWorksheet(xml: string, sharedStrings: string[]) {
|
||||
const rows: string[][] = [];
|
||||
for (const rowMatch of xml.matchAll(/<row\b([^>]*)>([\s\S]*?)<\/row>/g)) {
|
||||
const rowNumber = Number(
|
||||
rowMatch[1].match(/\br="(\d+)"/)?.[1] ?? rows.length + 1,
|
||||
);
|
||||
const values: string[] = [];
|
||||
for (const cellMatch of rowMatch[2].matchAll(
|
||||
/<c\b([^>]*)>([\s\S]*?)<\/c>/g,
|
||||
)) {
|
||||
const attributes = cellMatch[1];
|
||||
const body = cellMatch[2];
|
||||
const reference = attributes.match(/\br="([A-Z]+\d+)"/i)?.[1] ?? "A1";
|
||||
const type = attributes.match(/\bt="([^"]+)"/)?.[1] ?? "";
|
||||
const rawValue = body.match(/<v>([\s\S]*?)<\/v>/)?.[1] ?? "";
|
||||
let value = "";
|
||||
if (type === "s") value = sharedStrings[Number(rawValue)] ?? "";
|
||||
else if (type === "inlineStr") value = textNodes(body);
|
||||
else if (type === "b") value = rawValue === "1" ? "TRUE" : "FALSE";
|
||||
else value = decodeXml(rawValue);
|
||||
values[columnIndex(reference)] = value.trim();
|
||||
}
|
||||
while (rows.length < rowNumber - 1) rows.push([]);
|
||||
rows[rowNumber - 1] = values;
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
function normalizeHeader(value: string) {
|
||||
return value.replace(/[\s_\-()()]/g, "").toLocaleLowerCase("zh-CN");
|
||||
}
|
||||
|
||||
function isSourceImageHeader(value: string) {
|
||||
return /^(?:图片|发布配图)\d*$/.test(normalizeHeader(value));
|
||||
}
|
||||
|
||||
function headerAliases(header: (typeof PARTNER_BATCH_HEADERS)[number]) {
|
||||
const aliases: Record<string, string[]> = {
|
||||
"序号(不能改)": ["序号(不能改)", "序号"],
|
||||
标题: ["标题"],
|
||||
"笔记内容(正文+话题)": ["笔记内容(正文+话题)", "笔记内容"],
|
||||
图片: ["图片", "发布配图"],
|
||||
发布链接: ["发布链接"],
|
||||
笔记截图: ["笔记截图", "发布截图"],
|
||||
"数据分析截图(单篇笔记数据分析截图)": [
|
||||
"数据分析截图(单篇笔记数据分析截图)",
|
||||
"数据分析截图",
|
||||
"创作者中心截图",
|
||||
],
|
||||
_系统笔记ID: ["_系统笔记ID", "系统笔记ID"],
|
||||
_原笔记截图: ["_原笔记截图"],
|
||||
_原数据分析截图: ["_原数据分析截图"],
|
||||
};
|
||||
return aliases[header] ?? [header];
|
||||
}
|
||||
|
||||
function findHeader(rows: string[][]) {
|
||||
for (let rowIndex = 0; rowIndex < Math.min(rows.length, 8); rowIndex += 1) {
|
||||
const mapping = new Map<(typeof PARTNER_BATCH_HEADERS)[number], number>();
|
||||
rows[rowIndex].forEach((value, column) => {
|
||||
for (const header of PARTNER_BATCH_HEADERS) {
|
||||
if (header === "图片" && isSourceImageHeader(value)) {
|
||||
if (!mapping.has(header)) mapping.set(header, column);
|
||||
break;
|
||||
}
|
||||
if (
|
||||
headerAliases(header).some(
|
||||
(alias) => normalizeHeader(alias) === normalizeHeader(value),
|
||||
)
|
||||
) {
|
||||
mapping.set(header, column);
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
if (
|
||||
mapping.has("序号(不能改)") &&
|
||||
mapping.has("标题") &&
|
||||
mapping.has("发布链接") &&
|
||||
mapping.has("笔记截图") &&
|
||||
mapping.has("数据分析截图(单篇笔记数据分析截图)") &&
|
||||
mapping.has("_系统笔记ID")
|
||||
) {
|
||||
return { rowIndex, mapping };
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function relationshipMap(xml: string) {
|
||||
const relationships = new Map<string, string>();
|
||||
for (const match of xml.matchAll(/<Relationship\b([^>]*)\/?\s*>/g)) {
|
||||
const attributes = match[1];
|
||||
const id = attributes.match(/\bId="([^"]+)"/)?.[1] ?? "";
|
||||
const target = attributes.match(/\bTarget="([^"]+)"/)?.[1] ?? "";
|
||||
if (id && target) relationships.set(id, xmlAttribute(target));
|
||||
}
|
||||
return relationships;
|
||||
}
|
||||
|
||||
function contentType(bytes: Uint8Array, fileName: string) {
|
||||
if (bytes[0] === 0x89 && bytes[1] === 0x50) return "image/png";
|
||||
if (bytes[0] === 0xff && bytes[1] === 0xd8) return "image/jpeg";
|
||||
if (bytes[0] === 0x47 && bytes[1] === 0x49 && bytes[2] === 0x46) {
|
||||
return "image/gif";
|
||||
}
|
||||
if (String.fromCharCode(...bytes.slice(8, 12)) === "WEBP") return "image/webp";
|
||||
const extension = path.extname(fileName).toLowerCase();
|
||||
return extension === ".png"
|
||||
? "image/png"
|
||||
: extension === ".gif"
|
||||
? "image/gif"
|
||||
: extension === ".webp"
|
||||
? "image/webp"
|
||||
: "image/jpeg";
|
||||
}
|
||||
|
||||
function resolveZipPath(base: string, target: string) {
|
||||
return path.posix.normalize(path.posix.join(path.posix.dirname(base), target));
|
||||
}
|
||||
|
||||
function parseDrawingImages(entries: Record<string, Uint8Array>) {
|
||||
const images = new Map<string, PartnerBatchImage>();
|
||||
const sheetRelationshipsXml = entries["xl/worksheets/_rels/sheet1.xml.rels"]
|
||||
? strFromU8(entries["xl/worksheets/_rels/sheet1.xml.rels"])
|
||||
: "";
|
||||
const sheetRelationships = relationshipMap(sheetRelationshipsXml);
|
||||
const sheetXml = entries["xl/worksheets/sheet1.xml"]
|
||||
? strFromU8(entries["xl/worksheets/sheet1.xml"])
|
||||
: "";
|
||||
const drawingId = sheetXml.match(/<drawing\b[^>]*r:id="([^"]+)"/)?.[1] ?? "";
|
||||
const drawingTarget = sheetRelationships.get(drawingId);
|
||||
if (!drawingTarget) return images;
|
||||
const drawingPath = resolveZipPath("xl/worksheets/sheet1.xml", drawingTarget);
|
||||
const drawingXml = entries[drawingPath] ? strFromU8(entries[drawingPath]) : "";
|
||||
const drawingRelationshipsPath = path.posix.join(
|
||||
path.posix.dirname(drawingPath),
|
||||
"_rels",
|
||||
`${path.posix.basename(drawingPath)}.rels`,
|
||||
);
|
||||
const drawingRelationships = relationshipMap(
|
||||
entries[drawingRelationshipsPath]
|
||||
? strFromU8(entries[drawingRelationshipsPath])
|
||||
: "",
|
||||
);
|
||||
for (const anchor of drawingXml.matchAll(
|
||||
/<xdr:(?:oneCellAnchor|twoCellAnchor)\b[^>]*>[\s\S]*?<xdr:from>([\s\S]*?)<\/xdr:from>[\s\S]*?<a:blip\b[^>]*r:embed="([^"]+)"[\s\S]*?<\/xdr:(?:oneCellAnchor|twoCellAnchor)>/g,
|
||||
)) {
|
||||
const column = Number(anchor[1].match(/<xdr:col>(\d+)<\/xdr:col>/)?.[1]);
|
||||
const row = Number(anchor[1].match(/<xdr:row>(\d+)<\/xdr:row>/)?.[1]);
|
||||
const mediaTarget = drawingRelationships.get(anchor[2]);
|
||||
if (!Number.isInteger(column) || !Number.isInteger(row) || !mediaTarget) continue;
|
||||
const mediaPath = resolveZipPath(drawingPath, mediaTarget);
|
||||
const bytes = entries[mediaPath];
|
||||
if (!bytes) continue;
|
||||
images.set(`${row + 1}:${column}`, {
|
||||
bytes,
|
||||
contentType: contentType(bytes, mediaPath),
|
||||
fileName: path.posix.basename(mediaPath),
|
||||
});
|
||||
}
|
||||
return images;
|
||||
}
|
||||
|
||||
function parseRichValueImages(entries: Record<string, Uint8Array>) {
|
||||
const images = new Map<string, PartnerBatchImage>();
|
||||
const worksheetXml = entries["xl/worksheets/sheet1.xml"]
|
||||
? strFromU8(entries["xl/worksheets/sheet1.xml"])
|
||||
: "";
|
||||
const metadataXml = entries["xl/metadata.xml"]
|
||||
? strFromU8(entries["xl/metadata.xml"])
|
||||
: "";
|
||||
const richValueXml = entries["xl/richData/rdrichvalue.xml"]
|
||||
? strFromU8(entries["xl/richData/rdrichvalue.xml"])
|
||||
: "";
|
||||
const richValueRelXml = entries["xl/richData/richValueRel.xml"]
|
||||
? strFromU8(entries["xl/richData/richValueRel.xml"])
|
||||
: "";
|
||||
const richValueRelRelationships = relationshipMap(
|
||||
entries["xl/richData/_rels/richValueRel.xml.rels"]
|
||||
? strFromU8(entries["xl/richData/_rels/richValueRel.xml.rels"])
|
||||
: "",
|
||||
);
|
||||
if (
|
||||
!worksheetXml ||
|
||||
!metadataXml ||
|
||||
!richValueXml ||
|
||||
!richValueRelXml ||
|
||||
!richValueRelRelationships.size
|
||||
) {
|
||||
return images;
|
||||
}
|
||||
|
||||
const valueMetadataXml =
|
||||
metadataXml.match(/<valueMetadata\b[^>]*>([\s\S]*?)<\/valueMetadata>/)?.[1] ??
|
||||
"";
|
||||
const metadataToRichValue = [
|
||||
...valueMetadataXml.matchAll(/<bk\b[^>]*>[\s\S]*?<rc\b[^>]*\bv="(\d+)"[^>]*\/>[\s\S]*?<\/bk>/g),
|
||||
].map((match) => Number(match[1]));
|
||||
const richValueToRelationship = [
|
||||
...richValueXml.matchAll(/<rv\b[^>]*>([\s\S]*?)<\/rv>/g),
|
||||
].map((match) => Number(match[1].match(/<v>(\d+)<\/v>/)?.[1] ?? -1));
|
||||
const relationshipIds = [
|
||||
...richValueRelXml.matchAll(/<rel\b[^>]*\br:id="([^"]+)"[^>]*\/>/g),
|
||||
].map((match) => match[1]);
|
||||
|
||||
for (const cell of worksheetXml.matchAll(
|
||||
/<c\b([^>]*)>[\s\S]*?<\/c>/g,
|
||||
)) {
|
||||
const reference = cell[1].match(/\br="([A-Z]+\d+)"/i)?.[1] ?? "";
|
||||
const metadataIndex = Number(cell[1].match(/\bvm="(\d+)"/)?.[1] ?? 0);
|
||||
const row = Number(reference.match(/\d+$/)?.[0] ?? 0);
|
||||
const column = columnIndex(reference);
|
||||
if (!reference || !metadataIndex || !row) continue;
|
||||
const richValueIndex = metadataToRichValue[metadataIndex - 1];
|
||||
const relationshipIndex = richValueToRelationship[richValueIndex];
|
||||
const relationshipId = relationshipIds[relationshipIndex];
|
||||
const mediaTarget = richValueRelRelationships.get(relationshipId);
|
||||
if (!mediaTarget) continue;
|
||||
const mediaPath = resolveZipPath(
|
||||
"xl/richData/richValueRel.xml",
|
||||
mediaTarget,
|
||||
);
|
||||
const bytes = entries[mediaPath];
|
||||
if (!bytes) continue;
|
||||
images.set(`${row}:${column}`, {
|
||||
bytes,
|
||||
contentType: contentType(bytes, mediaPath),
|
||||
fileName: path.posix.basename(mediaPath),
|
||||
});
|
||||
}
|
||||
return images;
|
||||
}
|
||||
|
||||
function parseWpsCellImages(entries: Record<string, Uint8Array>) {
|
||||
const images = new Map<string, PartnerBatchImage>();
|
||||
const worksheetXml = entries["xl/worksheets/sheet1.xml"]
|
||||
? strFromU8(entries["xl/worksheets/sheet1.xml"])
|
||||
: "";
|
||||
const cellImagesXml = entries["xl/cellimages.xml"]
|
||||
? strFromU8(entries["xl/cellimages.xml"])
|
||||
: "";
|
||||
const relationships = relationshipMap(
|
||||
entries["xl/_rels/cellimages.xml.rels"]
|
||||
? strFromU8(entries["xl/_rels/cellimages.xml.rels"])
|
||||
: "",
|
||||
);
|
||||
if (!worksheetXml || !cellImagesXml || !relationships.size) return images;
|
||||
|
||||
const imageIdToRelationship = new Map<string, string>();
|
||||
for (const match of cellImagesXml.matchAll(
|
||||
/<(?:etc:)?cellImage\b[^>]*>([\s\S]*?)<\/(?:etc:)?cellImage>/g,
|
||||
)) {
|
||||
const imageId = match[1].match(
|
||||
/<(?:xdr:)?cNvPr\b[^>]*\bname="([^"]+)"/,
|
||||
)?.[1];
|
||||
const relationshipId = match[1].match(
|
||||
/<(?:a:)?blip\b[^>]*\br:embed="([^"]+)"/,
|
||||
)?.[1];
|
||||
if (imageId && relationshipId) {
|
||||
imageIdToRelationship.set(imageId, relationshipId);
|
||||
}
|
||||
}
|
||||
|
||||
for (const cell of worksheetXml.matchAll(/<c\b([^>]*)>([\s\S]*?)<\/c>/g)) {
|
||||
const reference = cell[1].match(/\br="([A-Z]+\d+)"/i)?.[1] ?? "";
|
||||
const imageId = decodeXml(cell[2]).match(/DISPIMG\("([^"]+)"/i)?.[1];
|
||||
if (!reference || !imageId) continue;
|
||||
const relationshipId = imageIdToRelationship.get(imageId);
|
||||
const mediaTarget = relationshipId
|
||||
? relationships.get(relationshipId)
|
||||
: undefined;
|
||||
if (!mediaTarget) continue;
|
||||
const mediaPath = resolveZipPath("xl/cellimages.xml", mediaTarget);
|
||||
const bytes = entries[mediaPath];
|
||||
if (!bytes) continue;
|
||||
const row = Number(reference.match(/\d+$/)?.[0] ?? 0);
|
||||
if (!row) continue;
|
||||
images.set(`${row}:${columnIndex(reference)}`, {
|
||||
bytes,
|
||||
contentType: contentType(bytes, mediaPath),
|
||||
fileName: path.posix.basename(mediaPath),
|
||||
});
|
||||
}
|
||||
return images;
|
||||
}
|
||||
|
||||
function parseImages(entries: Record<string, Uint8Array>) {
|
||||
const images = parseDrawingImages(entries);
|
||||
for (const [cell, image] of parseRichValueImages(entries)) {
|
||||
images.set(cell, image);
|
||||
}
|
||||
for (const [cell, image] of parseWpsCellImages(entries)) {
|
||||
images.set(cell, image);
|
||||
}
|
||||
return images;
|
||||
}
|
||||
|
||||
function valueAt(
|
||||
row: string[],
|
||||
mapping: Map<(typeof PARTNER_BATCH_HEADERS)[number], number>,
|
||||
header: (typeof PARTNER_BATCH_HEADERS)[number],
|
||||
) {
|
||||
const column = mapping.get(header);
|
||||
return column === undefined ? "" : String(row[column] ?? "").trim();
|
||||
}
|
||||
|
||||
export function parsePartnerBatchWorkbook(input: ArrayBuffer | Uint8Array) {
|
||||
const bytes = input instanceof Uint8Array ? input : new Uint8Array(input);
|
||||
if (bytes.byteLength > PARTNER_BATCH_MAX_BYTES) {
|
||||
throw new Error("批量回填表不能超过80MB");
|
||||
}
|
||||
const entries = unzipSync(bytes);
|
||||
const worksheetBytes = entries["xl/worksheets/sheet1.xml"];
|
||||
if (!worksheetBytes) throw new Error("Excel 中没有可读取的批量回填工作表");
|
||||
const sharedXml = entries["xl/sharedStrings.xml"]
|
||||
? strFromU8(entries["xl/sharedStrings.xml"])
|
||||
: "";
|
||||
const sharedStrings = [
|
||||
...sharedXml.matchAll(/<si\b[^>]*>([\s\S]*?)<\/si>/g),
|
||||
].map((match) => textNodes(match[1]));
|
||||
const rows = parseWorksheet(strFromU8(worksheetBytes), sharedStrings);
|
||||
const header = findHeader(rows);
|
||||
if (!header) {
|
||||
throw new Error("表格结构不正确,请使用本领取页面导出的批量回填表");
|
||||
}
|
||||
const images = parseImages(entries);
|
||||
const publishScreenshotColumn = header.mapping.get("笔记截图")!;
|
||||
const creatorScreenshotColumn = header.mapping.get(
|
||||
"数据分析截图(单篇笔记数据分析截图)",
|
||||
)!;
|
||||
const result: PartnerBatchImportRow[] = [];
|
||||
for (let index = header.rowIndex + 1; index < rows.length; index += 1) {
|
||||
const row = rows[index];
|
||||
const distributionId = valueAt(row, header.mapping, "_系统笔记ID");
|
||||
if (!distributionId && !row.some((value) => String(value ?? "").trim())) continue;
|
||||
result.push({
|
||||
spreadsheetRow: index + 1,
|
||||
sequence: valueAt(row, header.mapping, "序号(不能改)"),
|
||||
title: valueAt(row, header.mapping, "标题"),
|
||||
publishUrl: valueAt(row, header.mapping, "发布链接"),
|
||||
distributionId,
|
||||
originalPublishScreenshotKey: valueAt(
|
||||
row,
|
||||
header.mapping,
|
||||
"_原笔记截图",
|
||||
),
|
||||
originalCreatorScreenshotKey: valueAt(
|
||||
row,
|
||||
header.mapping,
|
||||
"_原数据分析截图",
|
||||
),
|
||||
publishScreenshot:
|
||||
images.get(`${index + 1}:${publishScreenshotColumn}`) ?? null,
|
||||
creatorScreenshot:
|
||||
images.get(`${index + 1}:${creatorScreenshotColumn}`) ?? null,
|
||||
});
|
||||
}
|
||||
if (result.length === 0) throw new Error("表格中没有可回填的笔记");
|
||||
return result;
|
||||
}
|
||||
@@ -1,22 +1,31 @@
|
||||
import { hashText } from "./mvp-db";
|
||||
import {
|
||||
extractPublishUrl,
|
||||
extractXhsPublishUrl,
|
||||
platformFromPublishUrl,
|
||||
safeHttpUrl,
|
||||
type SupportedPlatform,
|
||||
} from "./publish-url";
|
||||
|
||||
export { extractXhsPublishUrl } from "./publish-url";
|
||||
|
||||
export function accountFromPublishLink(input: string) {
|
||||
const url = safeHttpUrl(extractXhsPublishUrl(input));
|
||||
export function accountFromPublishLink(
|
||||
input: string,
|
||||
expectedPlatform?: SupportedPlatform,
|
||||
) {
|
||||
const extracted = expectedPlatform
|
||||
? extractPublishUrl(input, expectedPlatform)
|
||||
: extractXhsPublishUrl(input) || extractPublishUrl(input, "抖音");
|
||||
const url = safeHttpUrl(extracted);
|
||||
if (!url) return null;
|
||||
const platform =
|
||||
url.hostname.includes("xiaohongshu") || url.hostname.includes("xhslink")
|
||||
? "小红书"
|
||||
: "其他平台";
|
||||
const platform = platformFromPublishUrl(url.toString());
|
||||
if (!platform || (expectedPlatform && platform !== expectedPlatform)) return null;
|
||||
const noteId =
|
||||
url.pathname.match(
|
||||
/\/(?:discovery\/item|explore)\/([A-Za-z0-9_-]{8,80})/,
|
||||
)?.[1] ?? "";
|
||||
platform === "抖音"
|
||||
? url.pathname.match(/\/(?:video|note)\/([A-Za-z0-9_-]{8,80})/)?.[1] ?? ""
|
||||
: url.pathname.match(
|
||||
/\/(?:discovery\/item|explore)\/([A-Za-z0-9_-]{8,80})/,
|
||||
)?.[1] ?? "";
|
||||
const platformUid = `pending-${hashText(
|
||||
noteId || `${url.origin}${url.pathname}`,
|
||||
)}`;
|
||||
|
||||
@@ -25,3 +25,44 @@ export function extractXhsPublishUrl(input: string) {
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
export type SupportedPlatform = "小红书" | "抖音";
|
||||
|
||||
function platformMatches(url: URL, platform: SupportedPlatform) {
|
||||
const hostname = url.hostname.toLowerCase();
|
||||
if (platform === "抖音") {
|
||||
return hostname === "douyin.com" || hostname.endsWith(".douyin.com");
|
||||
}
|
||||
return (
|
||||
hostname === "xiaohongshu.com" ||
|
||||
hostname.endsWith(".xiaohongshu.com") ||
|
||||
hostname === "xhslink.cn" ||
|
||||
hostname.endsWith(".xhslink.cn")
|
||||
);
|
||||
}
|
||||
|
||||
export function extractPublishUrl(
|
||||
input: string,
|
||||
platform: SupportedPlatform = "小红书",
|
||||
) {
|
||||
const candidates =
|
||||
input.match(/https?:\/\/[^\s<>"',。!?;:、【】()]+/gi) ?? [];
|
||||
for (const candidate of candidates) {
|
||||
const cleaned = candidate.replace(/[.,!?;:~)\]}]+$/g, "");
|
||||
const url = safeHttpUrl(cleaned);
|
||||
if (url && platformMatches(url, platform)) return url.toString();
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
export function extractAnyPublishUrl(input: string) {
|
||||
return extractPublishUrl(input, "小红书") || extractPublishUrl(input, "抖音");
|
||||
}
|
||||
|
||||
export function platformFromPublishUrl(input: string): SupportedPlatform | "" {
|
||||
const url = safeHttpUrl(input);
|
||||
if (!url) return "";
|
||||
if (platformMatches(url, "小红书")) return "小红书";
|
||||
if (platformMatches(url, "抖音")) return "抖音";
|
||||
return "";
|
||||
}
|
||||
|
||||
@@ -13,6 +13,9 @@ export type RecoveryWorkbookRow = {
|
||||
images: Array<{
|
||||
column: number;
|
||||
image: RecoveryWorkbookImage;
|
||||
offsetX?: number;
|
||||
maxWidth?: number;
|
||||
maxHeight?: number;
|
||||
}>;
|
||||
hyperlinks?: Array<{
|
||||
column: number;
|
||||
@@ -25,6 +28,7 @@ type WorkbookOptions = {
|
||||
headers: string[];
|
||||
columnWidths: number[];
|
||||
rows: RecoveryWorkbookRow[];
|
||||
hiddenColumns?: number[];
|
||||
};
|
||||
|
||||
const XML_HEADER = '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>';
|
||||
@@ -130,9 +134,17 @@ function imageDimensions(image: RecoveryWorkbookImage) {
|
||||
return { width: 4, height: 3 };
|
||||
}
|
||||
|
||||
function imageDisplaySize(image: RecoveryWorkbookImage) {
|
||||
function imageDisplaySize(
|
||||
image: RecoveryWorkbookImage,
|
||||
maxWidth = 160,
|
||||
maxHeight = 150,
|
||||
) {
|
||||
const dimensions = imageDimensions(image);
|
||||
const scale = Math.min(160 / dimensions.width, 150 / dimensions.height, 1);
|
||||
const scale = Math.min(
|
||||
maxWidth / dimensions.width,
|
||||
maxHeight / dimensions.height,
|
||||
1,
|
||||
);
|
||||
return {
|
||||
width: Math.max(28, Math.round(dimensions.width * scale)),
|
||||
height: Math.max(28, Math.round(dimensions.height * scale)),
|
||||
@@ -152,6 +164,14 @@ export function buildRecoveryWorkbook(options: WorkbookOptions) {
|
||||
const imageEntries = options.rows.flatMap((row, rowIndex) =>
|
||||
row.images.map((item) => ({ ...item, row: rowIndex + 1 })),
|
||||
);
|
||||
const imageCells = new Set<string>();
|
||||
imageEntries.forEach((entry) => {
|
||||
const key = `${entry.row}:${entry.column}`;
|
||||
if (imageCells.has(key)) {
|
||||
throw new Error("Excel 单元格内只能嵌入一张图片,请为每张图片分配独立列");
|
||||
}
|
||||
imageCells.add(key);
|
||||
});
|
||||
const hyperlinkEntries = options.rows.flatMap((row, rowIndex) =>
|
||||
(row.hyperlinks ?? [])
|
||||
.map((item) => ({
|
||||
@@ -178,7 +198,7 @@ export function buildRecoveryWorkbook(options: WorkbookOptions) {
|
||||
const reference = `${columnName(columnIndex)}${number}`;
|
||||
const value = row.cells[columnIndex] ?? "";
|
||||
if (imageColumns.has(columnIndex)) {
|
||||
return inlineCell(reference, value || "见图", 4);
|
||||
return inlineCell(reference, value, 4);
|
||||
}
|
||||
return typeof value === "number"
|
||||
? numberCell(reference, value, 3)
|
||||
@@ -196,17 +216,22 @@ export function buildRecoveryWorkbook(options: WorkbookOptions) {
|
||||
const columns = options.headers
|
||||
.map((_, index) => {
|
||||
const width = Math.min(70, Math.max(8, options.columnWidths[index] ?? 14));
|
||||
return `<col min="${index + 1}" max="${index + 1}" width="${width}" customWidth="1"/>`;
|
||||
const hidden = options.hiddenColumns?.includes(index) ? ' hidden="1"' : "";
|
||||
return `<col min="${index + 1}" max="${index + 1}" width="${width}" customWidth="1"${hidden}/>`;
|
||||
})
|
||||
.join("");
|
||||
|
||||
const drawingXml = imageEntries.length
|
||||
? `${XML_HEADER}<xdr:wsDr xmlns:xdr="http://schemas.openxmlformats.org/drawingml/2006/spreadsheetDrawing" xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships">${imageEntries
|
||||
.map((entry, index) => {
|
||||
const size = imageDisplaySize(entry.image);
|
||||
const width = size.width * 9525;
|
||||
const height = size.height * 9525;
|
||||
return `<xdr:oneCellAnchor><xdr:from><xdr:col>${entry.column}</xdr:col><xdr:colOff>57150</xdr:colOff><xdr:row>${entry.row}</xdr:row><xdr:rowOff>57150</xdr:rowOff></xdr:from><xdr:ext cx="${width}" cy="${height}"/><xdr:pic><xdr:nvPicPr><xdr:cNvPr id="${index + 1}" name="${cleanXmlText(entry.image.description || `图片${index + 1}`)}"/><xdr:cNvPicPr><a:picLocks noChangeAspect="1"/></xdr:cNvPicPr></xdr:nvPicPr><xdr:blipFill><a:blip r:embed="rId${index + 1}"/><a:stretch><a:fillRect/></a:stretch></xdr:blipFill><xdr:spPr><a:prstGeom prst="rect"><a:avLst/></a:prstGeom></xdr:spPr></xdr:pic><xdr:clientData/></xdr:oneCellAnchor>`;
|
||||
const size = imageDisplaySize(
|
||||
entry.image,
|
||||
entry.maxWidth ?? 160,
|
||||
entry.maxHeight ?? 150,
|
||||
);
|
||||
const offsetX = entry.offsetX ?? 6;
|
||||
const offsetY = 6;
|
||||
return `<xdr:twoCellAnchor editAs="twoCell"><xdr:from><xdr:col>${entry.column}</xdr:col><xdr:colOff>${offsetX * 9525}</xdr:colOff><xdr:row>${entry.row}</xdr:row><xdr:rowOff>${offsetY * 9525}</xdr:rowOff></xdr:from><xdr:to><xdr:col>${entry.column}</xdr:col><xdr:colOff>${(offsetX + size.width) * 9525}</xdr:colOff><xdr:row>${entry.row}</xdr:row><xdr:rowOff>${(offsetY + size.height) * 9525}</xdr:rowOff></xdr:to><xdr:pic><xdr:nvPicPr><xdr:cNvPr id="${index + 1}" name="${cleanXmlText(entry.image.description || `图片${index + 1}`)}"/><xdr:cNvPicPr><a:picLocks noChangeAspect="1"/></xdr:cNvPicPr></xdr:nvPicPr><xdr:blipFill><a:blip r:embed="rId${index + 1}"/><a:stretch><a:fillRect/></a:stretch></xdr:blipFill><xdr:spPr><a:prstGeom prst="rect"><a:avLst/></a:prstGeom></xdr:spPr></xdr:pic><xdr:clientData/></xdr:twoCellAnchor>`;
|
||||
})
|
||||
.join("")}</xdr:wsDr>`
|
||||
: "";
|
||||
@@ -227,7 +252,10 @@ export function buildRecoveryWorkbook(options: WorkbookOptions) {
|
||||
const imageContentTypes = [...imageFormats.entries()]
|
||||
.map(([extension, contentType]) => `<Default Extension="${extension}" ContentType="${contentType}"/>`)
|
||||
.join("");
|
||||
const contentTypes = `${XML_HEADER}<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types"><Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/><Default Extension="xml" ContentType="application/xml"/>${imageContentTypes}<Override PartName="/xl/workbook.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml"/><Override PartName="/xl/worksheets/sheet1.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.worksheet+xml"/><Override PartName="/xl/styles.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.styles+xml"/>${imageEntries.length ? '<Override PartName="/xl/drawings/drawing1.xml" ContentType="application/vnd.openxmlformats-officedocument.drawing+xml"/>' : ""}<Override PartName="/docProps/core.xml" ContentType="application/vnd.openxmlformats-package.core-properties+xml"/><Override PartName="/docProps/app.xml" ContentType="application/vnd.openxmlformats-officedocument.extended-properties+xml"/></Types>`;
|
||||
const drawingContentType = imageEntries.length
|
||||
? '<Override PartName="/xl/drawings/drawing1.xml" ContentType="application/vnd.openxmlformats-officedocument.drawing+xml"/>'
|
||||
: "";
|
||||
const contentTypes = `${XML_HEADER}<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types"><Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/><Default Extension="xml" ContentType="application/xml"/>${imageContentTypes}<Override PartName="/xl/workbook.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml"/><Override PartName="/xl/worksheets/sheet1.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.worksheet+xml"/><Override PartName="/xl/styles.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.styles+xml"/>${drawingContentType}<Override PartName="/docProps/core.xml" ContentType="application/vnd.openxmlformats-package.core-properties+xml"/><Override PartName="/docProps/app.xml" ContentType="application/vnd.openxmlformats-officedocument.extended-properties+xml"/></Types>`;
|
||||
const hyperlinkRelationshipOffset = imageEntries.length ? 2 : 1;
|
||||
const hyperlinksXml = hyperlinkEntries.length
|
||||
? `<hyperlinks>${hyperlinkEntries
|
||||
@@ -266,7 +294,9 @@ export function buildRecoveryWorkbook(options: WorkbookOptions) {
|
||||
}
|
||||
if (imageEntries.length) {
|
||||
files["xl/drawings/drawing1.xml"] = strToU8(drawingXml);
|
||||
files["xl/drawings/_rels/drawing1.xml.rels"] = strToU8(drawingRelationships);
|
||||
files["xl/drawings/_rels/drawing1.xml.rels"] = strToU8(
|
||||
drawingRelationships,
|
||||
);
|
||||
imageEntries.forEach((entry, index) => {
|
||||
const format = imageFormat(entry.image.contentType, entry.image.bytes);
|
||||
files[`xl/media/image${index + 1}.${format.extension}`] = entry.image.bytes;
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { strFromU8, unzipSync } from "fflate";
|
||||
|
||||
export const RESOURCE_IMPORT_MAX_ROWS = 100;
|
||||
export const RESOURCE_IMPORT_MAX_BYTES = 5 * 1024 * 1024;
|
||||
export const RESOURCE_IMPORT_MAX_ROWS = 10_000;
|
||||
export const RESOURCE_IMPORT_MAX_BYTES = 20 * 1024 * 1024;
|
||||
|
||||
export type ResourceImportRow = {
|
||||
rowNumber: number;
|
||||
@@ -12,6 +12,9 @@ export type ResourceImportRow = {
|
||||
ipLocation: string;
|
||||
followers: number;
|
||||
followersResolved: boolean;
|
||||
gender: "" | "男" | "女";
|
||||
bio: string;
|
||||
tags: string[];
|
||||
cooperationSource: string;
|
||||
errors: string[];
|
||||
};
|
||||
@@ -22,6 +25,9 @@ const HEADER_ALIASES = {
|
||||
publicAccountId: ["账号ID", "账号号", "小红书号", "抖音号"],
|
||||
ipLocation: ["IP属地", "IP地址", "IP地区", "IP所在地"],
|
||||
followers: ["粉丝数", "粉丝", "粉丝量"],
|
||||
gender: ["性别"],
|
||||
bio: ["简介", "账号简介", "个人简介"],
|
||||
tags: ["标签", "账号标签"],
|
||||
cooperationSource: ["合作来源", "资源来源", "历史合作来源"],
|
||||
} as const;
|
||||
|
||||
@@ -60,9 +66,11 @@ function parseWorksheet(xml: string, sharedStrings: string[]) {
|
||||
const rowAttributes = rowMatch[1];
|
||||
const rowNumber = Number(rowAttributes.match(/\br="(\d+)"/)?.[1] ?? rows.length + 1);
|
||||
const values: string[] = [];
|
||||
for (const cellMatch of rowMatch[2].matchAll(/<c\b([^>]*)>([\s\S]*?)<\/c>/g)) {
|
||||
for (const cellMatch of rowMatch[2].matchAll(
|
||||
/<c\b([^>]*?)(?:\/>|>([\s\S]*?)<\/c>)/g,
|
||||
)) {
|
||||
const attributes = cellMatch[1];
|
||||
const body = cellMatch[2];
|
||||
const body = cellMatch[2] ?? "";
|
||||
const reference = attributes.match(/\br="([A-Z]+\d+)"/i)?.[1] ?? "A1";
|
||||
const type = attributes.match(/\bt="([^"]+)"/)?.[1] ?? "";
|
||||
const rawValue = body.match(/<v>([\s\S]*?)<\/v>/)?.[1] ?? "";
|
||||
@@ -183,6 +191,15 @@ export function platformFromProfileUrl(profileUrl: string) {
|
||||
) {
|
||||
return "小红书";
|
||||
}
|
||||
if (
|
||||
((url.hostname === "douyin.com" || url.hostname.endsWith(".douyin.com")) &&
|
||||
/^\/user\/[^/]+/i.test(url.pathname)) ||
|
||||
((url.hostname === "iesdouyin.com" ||
|
||||
url.hostname.endsWith(".iesdouyin.com")) &&
|
||||
/^\/share\/user\/[^/]+/i.test(url.pathname))
|
||||
) {
|
||||
return "抖音";
|
||||
}
|
||||
} catch {
|
||||
// URL validation is reported by normalizeRows.
|
||||
}
|
||||
@@ -212,10 +229,42 @@ export function parseResourceFollowers(value: string) {
|
||||
};
|
||||
}
|
||||
|
||||
export function normalizeResourceGender(value: string) {
|
||||
const normalized = value.trim().toLocaleLowerCase("zh-CN");
|
||||
if (!normalized || ["未知", "未填写", "待识别", "unknown"].includes(normalized)) {
|
||||
return { value: "" as const, valid: true };
|
||||
}
|
||||
if (["男", "男性", "male", "m"].includes(normalized)) {
|
||||
return { value: "男" as const, valid: true };
|
||||
}
|
||||
if (["女", "女性", "female", "f"].includes(normalized)) {
|
||||
return { value: "女" as const, valid: true };
|
||||
}
|
||||
return { value: "" as const, valid: false };
|
||||
}
|
||||
|
||||
export function normalizeResourceTags(value: string | string[]) {
|
||||
const source = Array.isArray(value) ? value.join(",") : value;
|
||||
return [
|
||||
...new Set(
|
||||
source
|
||||
.split(/[,,、;;|]/)
|
||||
.map((item) => item.trim().replace(/^#+/, ""))
|
||||
.filter(Boolean),
|
||||
),
|
||||
];
|
||||
}
|
||||
|
||||
export function resourceImportMissingFields(
|
||||
row: Pick<
|
||||
ResourceImportRow,
|
||||
"nickname" | "publicAccountId" | "ipLocation" | "followersResolved"
|
||||
| "nickname"
|
||||
| "publicAccountId"
|
||||
| "ipLocation"
|
||||
| "followersResolved"
|
||||
| "gender"
|
||||
| "bio"
|
||||
| "tags"
|
||||
>,
|
||||
) {
|
||||
const missing: string[] = [];
|
||||
@@ -225,6 +274,8 @@ export function resourceImportMissingFields(
|
||||
missing.push("ipLocation");
|
||||
}
|
||||
if (!row.followersResolved) missing.push("followers");
|
||||
if (!row.gender) missing.push("gender");
|
||||
if (!row.bio.trim()) missing.push("bio");
|
||||
return missing;
|
||||
}
|
||||
|
||||
@@ -242,22 +293,39 @@ function normalizeRows(rows: string[][]) {
|
||||
const platform = platformFromProfileUrl(profileUrl);
|
||||
const rawFollowers = valueAt(source, header.mapping, "followers");
|
||||
const parsedFollowers = parseResourceFollowers(rawFollowers);
|
||||
const parsedGender = normalizeResourceGender(
|
||||
valueAt(source, header.mapping, "gender"),
|
||||
);
|
||||
const tags = normalizeResourceTags(valueAt(source, header.mapping, "tags"));
|
||||
const ipLocation = valueAt(source, header.mapping, "ipLocation");
|
||||
const errors: string[] = [];
|
||||
if (!rawProfileUrl) errors.push("账号主页不能为空");
|
||||
else if (!profileUrl) errors.push("账号主页链接格式不正确");
|
||||
else if (!platform) errors.push("当前自动解析仅支持小红书账号主页");
|
||||
else if (!platform) errors.push("当前自动解析仅支持小红书或抖音账号主页");
|
||||
if (!parsedFollowers.valid) {
|
||||
errors.push("粉丝数格式不正确,请填写数字或如 1.3万、10+");
|
||||
}
|
||||
if (!parsedGender.valid) {
|
||||
errors.push("性别格式不正确,请填写男、女或留空");
|
||||
}
|
||||
if (tags.length > 5) {
|
||||
errors.push("单个账号最多填写 5 个标签,请用逗号分隔");
|
||||
}
|
||||
if (/^\d+$/.test(ipLocation)) {
|
||||
errors.push("IP属地格式不正确,请填写省份、地区或国家名称");
|
||||
}
|
||||
result.push({
|
||||
rowNumber: index + 1,
|
||||
platform,
|
||||
nickname: valueAt(source, header.mapping, "nickname"),
|
||||
publicAccountId: valueAt(source, header.mapping, "publicAccountId"),
|
||||
profileUrl,
|
||||
ipLocation: valueAt(source, header.mapping, "ipLocation"),
|
||||
ipLocation,
|
||||
followers: parsedFollowers.value,
|
||||
followersResolved: parsedFollowers.resolved,
|
||||
gender: parsedGender.value,
|
||||
bio: valueAt(source, header.mapping, "bio"),
|
||||
tags: tags.slice(0, 5),
|
||||
cooperationSource: valueAt(source, header.mapping, "cooperationSource"),
|
||||
errors,
|
||||
});
|
||||
|
||||
@@ -10,6 +10,8 @@ export type CreateDistributionTaskInput = {
|
||||
name: string;
|
||||
brand: string;
|
||||
dueAt: string;
|
||||
platform?: "小红书" | "抖音";
|
||||
contentFormat?: "image_text" | "video";
|
||||
};
|
||||
|
||||
export type CreateScreenshotTaskInput = {
|
||||
@@ -33,6 +35,8 @@ export type DistributionTaskCreation = {
|
||||
sheetId: string;
|
||||
sheetName: string;
|
||||
sourceUrl: string;
|
||||
platform: "小红书" | "抖音";
|
||||
contentFormat: "image_text" | "video";
|
||||
};
|
||||
|
||||
export type ScreenshotTaskCreation = {
|
||||
@@ -55,6 +59,8 @@ type TaskRow = {
|
||||
source_url: string;
|
||||
source_sheet_id: string;
|
||||
source_sheet_name: string;
|
||||
platform: "小红书" | "抖音";
|
||||
content_format: "image_text" | "video";
|
||||
};
|
||||
|
||||
function normalizedValue(value: string) {
|
||||
@@ -99,17 +105,27 @@ async function findExistingTask(
|
||||
return db
|
||||
.prepare(
|
||||
`SELECT id, share_token, name, brand, due_at, quantity,
|
||||
source_url, source_sheet_id, source_sheet_name
|
||||
source_url, source_sheet_id, source_sheet_name,
|
||||
platform, content_format
|
||||
FROM tasks
|
||||
WHERE name = ?
|
||||
AND brand = ?
|
||||
AND due_at = ?
|
||||
AND source_url = ?
|
||||
AND platform = ?
|
||||
AND content_format = ?
|
||||
AND status IN ('active', 'importing')
|
||||
ORDER BY created_at DESC
|
||||
LIMIT 1`,
|
||||
)
|
||||
.bind(input.name, input.brand, input.dueAt, sourceUrl)
|
||||
.bind(
|
||||
input.name,
|
||||
input.brand,
|
||||
input.dueAt,
|
||||
sourceUrl,
|
||||
input.platform ?? "小红书",
|
||||
input.contentFormat ?? "image_text",
|
||||
)
|
||||
.first<TaskRow>();
|
||||
}
|
||||
|
||||
@@ -124,9 +140,10 @@ async function insertTaskFromSource(
|
||||
.prepare(
|
||||
`INSERT INTO tasks
|
||||
(id, name, brand, quantity, claimed_quantity, due_at, status,
|
||||
platform, content_format,
|
||||
source_url, source_sheet_id, source_sheet_name, source_synced_at,
|
||||
share_token)
|
||||
VALUES (?, ?, ?, ?, 0, ?, 'importing', ?, ?, ?, ?, ?)`,
|
||||
VALUES (?, ?, ?, ?, 0, ?, 'importing', ?, ?, ?, ?, ?, ?, ?)`,
|
||||
)
|
||||
.bind(
|
||||
taskId,
|
||||
@@ -134,6 +151,8 @@ async function insertTaskFromSource(
|
||||
input.brand,
|
||||
source.rows.length,
|
||||
input.dueAt,
|
||||
input.platform ?? "小红书",
|
||||
input.contentFormat ?? "image_text",
|
||||
source.url,
|
||||
source.sheetId,
|
||||
source.sheetName,
|
||||
@@ -149,11 +168,15 @@ async function insertTaskFromSource(
|
||||
...image,
|
||||
key: `content-assets/${taskId}/${contentId}/${image.index}`,
|
||||
}));
|
||||
const videoAssets = row.videos.map((video) => ({
|
||||
...video,
|
||||
key: `content-videos/${taskId}/${contentId}/${video.index}`,
|
||||
}));
|
||||
return db
|
||||
.prepare(
|
||||
`INSERT INTO contents
|
||||
(id, task_id, title, body, image_assets, status, source, source_row)
|
||||
VALUES (?, ?, ?, ?, ?, 'available', ?, ?)`,
|
||||
(id, task_id, title, body, image_assets, video_assets, status, source, source_row)
|
||||
VALUES (?, ?, ?, ?, ?, ?, 'available', ?, ?)`,
|
||||
)
|
||||
.bind(
|
||||
contentId,
|
||||
@@ -161,6 +184,7 @@ async function insertTaskFromSource(
|
||||
row.title,
|
||||
row.body,
|
||||
JSON.stringify(imageAssets),
|
||||
JSON.stringify(videoAssets),
|
||||
`飞书 · ${source.sheetName}`,
|
||||
row.sourceRow,
|
||||
);
|
||||
@@ -189,11 +213,13 @@ export async function createDistributionTask(
|
||||
options: { deduplicate?: boolean } = {},
|
||||
): Promise<DistributionTaskCreation> {
|
||||
await ensureSchema();
|
||||
const input = {
|
||||
const input: Required<CreateDistributionTaskInput> = {
|
||||
feishuUrl: normalizedFeishuUrl(rawInput.feishuUrl),
|
||||
name: normalizedValue(rawInput.name),
|
||||
brand: normalizedValue(rawInput.brand),
|
||||
dueAt: normalizedDueDate(rawInput.dueAt),
|
||||
platform: rawInput.platform === "抖音" ? "抖音" : "小红书",
|
||||
contentFormat: rawInput.contentFormat === "video" ? "video" : "image_text",
|
||||
};
|
||||
if (!input.feishuUrl || !input.name || !input.brand) {
|
||||
throw new Error("请补全飞书链接、任务名称和品牌/项目");
|
||||
@@ -213,11 +239,19 @@ export async function createDistributionTask(
|
||||
sheetId: existing.source_sheet_id,
|
||||
sheetName: existing.source_sheet_name,
|
||||
sourceUrl: existing.source_url,
|
||||
platform: existing.platform,
|
||||
contentFormat: existing.content_format,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
const source = await readFeishuSource(input.feishuUrl, bindings);
|
||||
if (
|
||||
input.contentFormat === "video" &&
|
||||
source.rows.some((row) => row.videos.length === 0)
|
||||
) {
|
||||
throw new Error("视频任务中存在未识别到视频的内容行,请检查飞书“视频”列");
|
||||
}
|
||||
const inserted = await insertTaskFromSource(source, input);
|
||||
return {
|
||||
created: true,
|
||||
@@ -230,6 +264,8 @@ export async function createDistributionTask(
|
||||
sheetId: source.sheetId,
|
||||
sheetName: source.sheetName,
|
||||
sourceUrl: source.url,
|
||||
platform: input.platform,
|
||||
contentFormat: input.contentFormat,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -287,8 +323,8 @@ export async function createScreenshotTask(
|
||||
db
|
||||
.prepare(
|
||||
`INSERT INTO contents
|
||||
(id, task_id, title, body, image_assets, status, source, source_row)
|
||||
VALUES (?, ?, ?, ?, ?, 'available', '截图回收任务', ?)`,
|
||||
(id, task_id, title, body, image_assets, video_assets, status, source, source_row)
|
||||
VALUES (?, ?, ?, ?, ?, '[]', 'available', '截图回收任务', ?)`,
|
||||
)
|
||||
.bind(
|
||||
uid("content"),
|
||||
|
||||
45
lib/video-file.ts
Normal file
45
lib/video-file.ts
Normal file
@@ -0,0 +1,45 @@
|
||||
const MP4_BRANDS = new Set([
|
||||
"avc1",
|
||||
"dash",
|
||||
"isom",
|
||||
"M4A ",
|
||||
"M4B ",
|
||||
"M4P ",
|
||||
"M4V ",
|
||||
"mp41",
|
||||
"mp42",
|
||||
"MSNV",
|
||||
]);
|
||||
|
||||
function fourCharacters(bytes: Uint8Array, offset: number) {
|
||||
return String.fromCharCode(...bytes.slice(offset, offset + 4));
|
||||
}
|
||||
|
||||
function isMp4Brand(brand: string) {
|
||||
return (
|
||||
MP4_BRANDS.has(brand) ||
|
||||
/^iso[2-9]$/.test(brand) ||
|
||||
/^3g[2p]$/.test(brand.slice(0, 3))
|
||||
);
|
||||
}
|
||||
|
||||
/** Rejects HTML/JSON/error payloads and non-MP4 containers before download. */
|
||||
export function hasMp4FileSignature(input: ArrayBuffer | Uint8Array) {
|
||||
const bytes = input instanceof Uint8Array ? input : new Uint8Array(input);
|
||||
if (bytes.byteLength < 12 || fourCharacters(bytes, 4) !== "ftyp") {
|
||||
return false;
|
||||
}
|
||||
const declaredSize = new DataView(
|
||||
bytes.buffer,
|
||||
bytes.byteOffset,
|
||||
bytes.byteLength,
|
||||
).getUint32(0);
|
||||
const boxEnd = Math.min(
|
||||
bytes.byteLength,
|
||||
declaredSize >= 12 ? declaredSize : bytes.byteLength,
|
||||
);
|
||||
for (let offset = 8; offset + 4 <= boxEnd; offset += 4) {
|
||||
if (isMp4Brand(fourCharacters(bytes, offset))) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
83
lib/workbook-image.ts
Normal file
83
lib/workbook-image.ts
Normal file
@@ -0,0 +1,83 @@
|
||||
import sharp from "sharp";
|
||||
|
||||
export type WorkbookSourceImage = {
|
||||
bytes: Uint8Array;
|
||||
contentType: string;
|
||||
width?: number | null;
|
||||
height?: number | null;
|
||||
description: string;
|
||||
};
|
||||
|
||||
export type WorkbookImageNormalizationOptions = {
|
||||
maxDimension?: number;
|
||||
outputFormat?: "png" | "jpeg";
|
||||
jpegQuality?: number;
|
||||
};
|
||||
|
||||
const NORMALIZABLE_IMAGE = /^image\/(?:jpe?g|png|webp|gif|tiff?|avif|heic|heif)$/i;
|
||||
|
||||
function hasImageSignature(bytes: Uint8Array) {
|
||||
if (bytes.length < 4) return false;
|
||||
if (bytes[0] === 0xff && bytes[1] === 0xd8) return true;
|
||||
if (bytes[0] === 0x89 && bytes[1] === 0x50 && bytes[2] === 0x4e && bytes[3] === 0x47) {
|
||||
return true;
|
||||
}
|
||||
if (bytes[0] === 0x47 && bytes[1] === 0x49 && bytes[2] === 0x46) return true;
|
||||
if (
|
||||
bytes.length >= 12 &&
|
||||
String.fromCharCode(...bytes.slice(0, 4)) === "RIFF" &&
|
||||
String.fromCharCode(...bytes.slice(8, 12)) === "WEBP"
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
return (
|
||||
(bytes[0] === 0x49 && bytes[1] === 0x49 && bytes[2] === 0x2a && bytes[3] === 0x00) ||
|
||||
(bytes[0] === 0x4d && bytes[1] === 0x4d && bytes[2] === 0x00 && bytes[3] === 0x2a)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Excel viewers disagree on whether JPEG EXIF orientation should be applied.
|
||||
* Bake that orientation into the pixels before the image enters the workbook.
|
||||
* Callers may also resize and encode large source images as JPEG to keep the
|
||||
* generated workbook within the upload limit.
|
||||
*/
|
||||
export async function normalizeWorkbookImage(
|
||||
image: WorkbookSourceImage,
|
||||
options: WorkbookImageNormalizationOptions = {},
|
||||
): Promise<WorkbookSourceImage> {
|
||||
if (!NORMALIZABLE_IMAGE.test(image.contentType) && !hasImageSignature(image.bytes)) {
|
||||
return image;
|
||||
}
|
||||
try {
|
||||
let normalized = sharp(image.bytes, { animated: false }).rotate();
|
||||
if (options.maxDimension && options.maxDimension > 0) {
|
||||
normalized = normalized.resize({
|
||||
width: Math.floor(options.maxDimension),
|
||||
height: Math.floor(options.maxDimension),
|
||||
fit: "inside",
|
||||
withoutEnlargement: true,
|
||||
});
|
||||
}
|
||||
normalized =
|
||||
options.outputFormat === "jpeg"
|
||||
? normalized
|
||||
.flatten({ background: "#ffffff" })
|
||||
.jpeg({
|
||||
quality: Math.min(95, Math.max(50, options.jpegQuality ?? 82)),
|
||||
mozjpeg: true,
|
||||
})
|
||||
: normalized.png({ compressionLevel: 6 });
|
||||
const { data, info } = await normalized.toBuffer({ resolveWithObject: true });
|
||||
return {
|
||||
...image,
|
||||
bytes: new Uint8Array(data),
|
||||
contentType:
|
||||
options.outputFormat === "jpeg" ? "image/jpeg" : "image/png",
|
||||
width: info.width,
|
||||
height: info.height,
|
||||
};
|
||||
} catch {
|
||||
return image;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user