Compare commits
2 Commits
b882a5c7d7
...
bc81e0daf1
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
bc81e0daf1 | ||
|
|
ed13262166 |
@@ -54,5 +54,6 @@ WECOM_NOTIFY_ENABLED=true
|
|||||||
# ── 调度与入口 ────────────────────────────────────────────
|
# ── 调度与入口 ────────────────────────────────────────────
|
||||||
# 每天北京时间 09:00 自动执行采集计划,生产保持 true。
|
# 每天北京时间 09:00 自动执行采集计划,生产保持 true。
|
||||||
ENABLE_SCHEDULER=true
|
ENABLE_SCHEDULER=true
|
||||||
|
# 是否在空库时写入演示数据,生产保持 false。
|
||||||
SEED_DEMO_DATA=false
|
SEED_DEMO_DATA=false
|
||||||
HTTP_PORT=80
|
HTTP_PORT=80
|
||||||
|
|||||||
@@ -74,8 +74,9 @@ type Account = {
|
|||||||
last_seen_at: string;
|
last_seen_at: string;
|
||||||
};
|
};
|
||||||
|
|
||||||
type IdRefreshStatus = {
|
type FieldRefreshStatus = {
|
||||||
status: "running" | "done";
|
status: "running" | "done";
|
||||||
|
fields: string[];
|
||||||
total: number;
|
total: number;
|
||||||
processed: number;
|
processed: number;
|
||||||
updated: number;
|
updated: number;
|
||||||
@@ -85,6 +86,16 @@ type IdRefreshStatus = {
|
|||||||
finishedAt: string | null;
|
finishedAt: string | null;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const REFRESH_FIELD_OPTIONS = [
|
||||||
|
{ key: "publicAccountId", label: "账号ID(小红书号/抖音号)" },
|
||||||
|
{ key: "nickname", label: "昵称" },
|
||||||
|
{ key: "followers", label: "粉丝数" },
|
||||||
|
{ key: "ipLocation", label: "IP属地" },
|
||||||
|
{ key: "gender", label: "性别" },
|
||||||
|
{ key: "bio", label: "简介" },
|
||||||
|
{ key: "tags", label: "标签(AI 重新打标,覆盖已有标签)" },
|
||||||
|
] as const;
|
||||||
|
|
||||||
type ResourceImportPreview = {
|
type ResourceImportPreview = {
|
||||||
summary: { total: number; create: number; update: number; error: number };
|
summary: { total: number; create: number; update: number; error: number };
|
||||||
rows: Array<{
|
rows: Array<{
|
||||||
@@ -3228,10 +3239,20 @@ function ResourcesPage({
|
|||||||
const [editAccountId, setEditAccountId] = useState("");
|
const [editAccountId, setEditAccountId] = useState("");
|
||||||
const [editWorking, setEditWorking] = useState(false);
|
const [editWorking, setEditWorking] = useState(false);
|
||||||
const [editError, setEditError] = useState("");
|
const [editError, setEditError] = useState("");
|
||||||
const [idRefresh, setIdRefresh] = useState<IdRefreshStatus | null>(null);
|
const [idRefresh, setIdRefresh] = useState<FieldRefreshStatus | null>(null);
|
||||||
const [idRefreshStarting, setIdRefreshStarting] = useState(false);
|
const [idRefreshStarting, setIdRefreshStarting] = useState(false);
|
||||||
const idRefreshNotifiedRef = useRef(true);
|
const idRefreshNotifiedRef = useRef(true);
|
||||||
const [idRefreshError, setIdRefreshError] = useState("");
|
const [idRefreshError, setIdRefreshError] = useState("");
|
||||||
|
const [fieldPickOpen, setFieldPickOpen] = useState(false);
|
||||||
|
const [fieldChecks, setFieldChecks] = useState<Record<string, boolean>>(
|
||||||
|
() =>
|
||||||
|
Object.fromEntries(
|
||||||
|
REFRESH_FIELD_OPTIONS.map((option) => [
|
||||||
|
option.key,
|
||||||
|
option.key === "publicAccountId",
|
||||||
|
]),
|
||||||
|
),
|
||||||
|
);
|
||||||
const resourceAccounts = useMemo(() => {
|
const resourceAccounts = useMemo(() => {
|
||||||
const deliveriesByAccount = new Map<string, Distribution[]>();
|
const deliveriesByAccount = new Map<string, Distribution[]>();
|
||||||
distributions.forEach((distribution) => {
|
distributions.forEach((distribution) => {
|
||||||
@@ -3494,9 +3515,9 @@ function ResourcesPage({
|
|||||||
const response = await fetch("/api/action", {
|
const response = await fetch("/api/action", {
|
||||||
method: "POST",
|
method: "POST",
|
||||||
headers: { "Content-Type": "application/json" },
|
headers: { "Content-Type": "application/json" },
|
||||||
body: JSON.stringify({ action: "public_account_id_refresh_status" }),
|
body: JSON.stringify({ action: "account_field_refresh_status" }),
|
||||||
});
|
});
|
||||||
const result = await readApiResponse<{ refresh: IdRefreshStatus | null }>(
|
const result = await readApiResponse<{ refresh: FieldRefreshStatus | null }>(
|
||||||
response,
|
response,
|
||||||
"查询失败",
|
"查询失败",
|
||||||
);
|
);
|
||||||
@@ -3504,10 +3525,10 @@ function ResourcesPage({
|
|||||||
return result.refresh;
|
return result.refresh;
|
||||||
}, []);
|
}, []);
|
||||||
const applyIdRefreshStatus = useCallback(
|
const applyIdRefreshStatus = useCallback(
|
||||||
(status: IdRefreshStatus) => {
|
(status: FieldRefreshStatus) => {
|
||||||
if (status.status === "done" && !idRefreshNotifiedRef.current) {
|
if (status.status === "done" && !idRefreshNotifiedRef.current) {
|
||||||
idRefreshNotifiedRef.current = true;
|
idRefreshNotifiedRef.current = true;
|
||||||
const summary = `账号ID更新完成:成功 ${status.updated} 个${
|
const summary = `账号资料更新完成:成功 ${status.updated} 个${
|
||||||
status.skipped ? `,未识别 ${status.skipped} 个` : ""
|
status.skipped ? `,未识别 ${status.skipped} 个` : ""
|
||||||
}${status.failed ? `,失败 ${status.failed} 个` : ""}`;
|
}${status.failed ? `,失败 ${status.failed} 个` : ""}`;
|
||||||
void onImported(summary);
|
void onImported(summary);
|
||||||
@@ -3537,30 +3558,32 @@ function ResourcesPage({
|
|||||||
}, 3000);
|
}, 3000);
|
||||||
return () => window.clearInterval(timer);
|
return () => window.clearInterval(timer);
|
||||||
}, [applyIdRefreshStatus, fetchIdRefreshStatus, runningIdRefresh]);
|
}, [applyIdRefreshStatus, fetchIdRefreshStatus, runningIdRefresh]);
|
||||||
|
const closeFieldPick = () => {
|
||||||
|
if (idRefreshStarting) return;
|
||||||
|
setFieldPickOpen(false);
|
||||||
|
};
|
||||||
const startIdRefresh = async () => {
|
const startIdRefresh = async () => {
|
||||||
if (runningIdRefresh || idRefreshStarting) return;
|
if (runningIdRefresh || idRefreshStarting) return;
|
||||||
if (
|
const fields = REFRESH_FIELD_OPTIONS.filter(
|
||||||
!window.confirm(
|
(option) => fieldChecks[option.key],
|
||||||
"将逐个爬取全部账号主页以更新小红书号/抖音号(其他资料不变),账号较多时需要几分钟。确定开始?",
|
).map((option) => option.key);
|
||||||
)
|
if (fields.length === 0) return;
|
||||||
) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
try {
|
try {
|
||||||
setIdRefreshStarting(true);
|
setIdRefreshStarting(true);
|
||||||
setIdRefreshError("");
|
setIdRefreshError("");
|
||||||
const response = await fetch("/api/action", {
|
const response = await fetch("/api/action", {
|
||||||
method: "POST",
|
method: "POST",
|
||||||
headers: { "Content-Type": "application/json" },
|
headers: { "Content-Type": "application/json" },
|
||||||
body: JSON.stringify({ action: "refresh_public_account_ids" }),
|
body: JSON.stringify({ action: "refresh_account_fields", fields }),
|
||||||
});
|
});
|
||||||
const result = await readApiResponse<{ refresh: IdRefreshStatus }>(
|
const result = await readApiResponse<{ refresh: FieldRefreshStatus }>(
|
||||||
response,
|
response,
|
||||||
"启动失败",
|
"启动失败",
|
||||||
);
|
);
|
||||||
if (!response.ok) throw new Error(result.error || "启动失败");
|
if (!response.ok) throw new Error(result.error || "启动失败");
|
||||||
idRefreshNotifiedRef.current = false;
|
idRefreshNotifiedRef.current = false;
|
||||||
setIdRefresh(result.refresh);
|
setIdRefresh(result.refresh);
|
||||||
|
setFieldPickOpen(false);
|
||||||
} catch (reason) {
|
} catch (reason) {
|
||||||
setIdRefreshError(reason instanceof Error ? reason.message : "启动失败");
|
setIdRefreshError(reason instanceof Error ? reason.message : "启动失败");
|
||||||
} finally {
|
} finally {
|
||||||
@@ -3680,14 +3703,14 @@ function ResourcesPage({
|
|||||||
<button
|
<button
|
||||||
className="export-data-button"
|
className="export-data-button"
|
||||||
disabled={runningIdRefresh || idRefreshStarting}
|
disabled={runningIdRefresh || idRefreshStarting}
|
||||||
title="爬取全部账号主页,更新小红书号/抖音号,其他资料不变"
|
title="爬取全部账号主页,按选择的字段批量更新账号资料"
|
||||||
onClick={() => void startIdRefresh()}
|
onClick={() => setFieldPickOpen(true)}
|
||||||
>
|
>
|
||||||
{idRefreshStarting
|
{idRefreshStarting
|
||||||
? "正在启动…"
|
? "正在启动…"
|
||||||
: runningIdRefresh
|
: runningIdRefresh
|
||||||
? `更新中 ${idRefresh?.processed ?? 0}/${idRefresh?.total ?? 0}`
|
? `更新中 ${idRefresh?.processed ?? 0}/${idRefresh?.total ?? 0}`
|
||||||
: "更新全部账号ID"}
|
: "批量更新账号资料"}
|
||||||
</button>
|
</button>
|
||||||
<button
|
<button
|
||||||
className="export-data-button"
|
className="export-data-button"
|
||||||
@@ -3834,6 +3857,66 @@ function ResourcesPage({
|
|||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
</section>
|
</section>
|
||||||
|
{fieldPickOpen && (
|
||||||
|
<div className="modal-backdrop" role="presentation" onMouseDown={closeFieldPick}>
|
||||||
|
<div
|
||||||
|
className="modal-card compact"
|
||||||
|
role="dialog"
|
||||||
|
aria-modal="true"
|
||||||
|
aria-labelledby="field-pick-title"
|
||||||
|
onMouseDown={(event) => event.stopPropagation()}
|
||||||
|
>
|
||||||
|
<div className="modal-heading">
|
||||||
|
<div>
|
||||||
|
<p className="eyebrow">批量更新</p>
|
||||||
|
<h2 id="field-pick-title">选择要更新的字段</h2>
|
||||||
|
</div>
|
||||||
|
<button type="button" onClick={closeFieldPick} aria-label="关闭">×</button>
|
||||||
|
</div>
|
||||||
|
<p className="field-pick-note">
|
||||||
|
将逐个爬取全部账号主页(小红书 + 抖音),只写入选中的字段;
|
||||||
|
爬取为空的字段自动跳过,不会清空已有数据。账号较多时需要几分钟。
|
||||||
|
</p>
|
||||||
|
<div className="field-pick-list">
|
||||||
|
{REFRESH_FIELD_OPTIONS.map((option) => (
|
||||||
|
<label className="field-pick-item" key={option.key}>
|
||||||
|
<input
|
||||||
|
checked={Boolean(fieldChecks[option.key])}
|
||||||
|
onChange={(event) =>
|
||||||
|
setFieldChecks((checks) => ({
|
||||||
|
...checks,
|
||||||
|
[option.key]: event.target.checked,
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
type="checkbox"
|
||||||
|
/>
|
||||||
|
<span>{option.label}</span>
|
||||||
|
</label>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
{idRefreshError && <div className="form-error">{idRefreshError}</div>}
|
||||||
|
<div className="modal-actions">
|
||||||
|
<button type="button" className="ghost-button" onClick={closeFieldPick}>取消</button>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
className="primary-button"
|
||||||
|
disabled={
|
||||||
|
idRefreshStarting ||
|
||||||
|
runningIdRefresh ||
|
||||||
|
!REFRESH_FIELD_OPTIONS.some((option) => fieldChecks[option.key])
|
||||||
|
}
|
||||||
|
onClick={() => void startIdRefresh()}
|
||||||
|
>
|
||||||
|
{idRefreshStarting
|
||||||
|
? "正在启动…"
|
||||||
|
: runningIdRefresh
|
||||||
|
? `更新中 ${idRefresh?.processed ?? 0}/${idRefresh?.total ?? 0}`
|
||||||
|
: "开始更新"}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
{editAccount && (
|
{editAccount && (
|
||||||
<div className="modal-backdrop" role="presentation" onMouseDown={closeAccountEdit}>
|
<div className="modal-backdrop" role="presentation" onMouseDown={closeAccountEdit}>
|
||||||
<div
|
<div
|
||||||
|
|||||||
@@ -55,9 +55,9 @@ import {
|
|||||||
import { isManagerRequest } from "../../../lib/user-auth";
|
import { isManagerRequest } from "../../../lib/user-auth";
|
||||||
import { extractPublishUrl } from "../../../lib/publish-url";
|
import { extractPublishUrl } from "../../../lib/publish-url";
|
||||||
import {
|
import {
|
||||||
getPublicAccountIdRefreshStatus,
|
getAccountFieldRefreshStatus,
|
||||||
startPublicAccountIdRefresh,
|
startAccountFieldRefresh,
|
||||||
} from "../../../lib/account-id-refresh-service";
|
} from "../../../lib/account-field-refresh-service";
|
||||||
|
|
||||||
type ActionBody = {
|
type ActionBody = {
|
||||||
action?: string;
|
action?: string;
|
||||||
@@ -687,14 +687,15 @@ export async function POST(request: Request) {
|
|||||||
const taskId = String(body.taskId ?? "").trim();
|
const taskId = String(body.taskId ?? "").trim();
|
||||||
const pushes = await listGroupPushes(db, taskId || undefined);
|
const pushes = await listGroupPushes(db, taskId || undefined);
|
||||||
return Response.json({ pushes });
|
return Response.json({ pushes });
|
||||||
} else if (body.action === "refresh_public_account_ids") {
|
} else if (body.action === "refresh_account_fields") {
|
||||||
const status = startPublicAccountIdRefresh(
|
const status = startAccountFieldRefresh(
|
||||||
db,
|
db,
|
||||||
resolveCollectionMcpConfig(env as unknown as CollectionMcpBindings),
|
resolveCollectionMcpConfig(env as unknown as CollectionMcpBindings),
|
||||||
|
body.fields,
|
||||||
);
|
);
|
||||||
return Response.json({ refresh: status });
|
return Response.json({ refresh: status });
|
||||||
} else if (body.action === "public_account_id_refresh_status") {
|
} else if (body.action === "account_field_refresh_status") {
|
||||||
return Response.json({ refresh: getPublicAccountIdRefreshStatus() });
|
return Response.json({ refresh: getAccountFieldRefreshStatus() });
|
||||||
} else if (body.action === "wecom_group_chats") {
|
} else if (body.action === "wecom_group_chats") {
|
||||||
const groups = await listGroupChatRows(db);
|
const groups = await listGroupChatRows(db);
|
||||||
return Response.json({ groups });
|
return Response.json({ groups });
|
||||||
|
|||||||
@@ -3043,6 +3043,48 @@ a {
|
|||||||
font-size: 10px;
|
font-size: 10px;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.field-pick-note {
|
||||||
|
margin: 0 0 14px;
|
||||||
|
color: #8a978f;
|
||||||
|
font-size: 11px;
|
||||||
|
line-height: 1.7;
|
||||||
|
}
|
||||||
|
|
||||||
|
.field-pick-list {
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
gap: 6px;
|
||||||
|
margin-bottom: 12px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.field-pick-item {
|
||||||
|
flex-direction: row;
|
||||||
|
align-items: center;
|
||||||
|
gap: 10px;
|
||||||
|
margin-bottom: 0;
|
||||||
|
padding: 9px 11px;
|
||||||
|
border: 1px solid #eef1ef;
|
||||||
|
border-radius: 9px;
|
||||||
|
cursor: pointer;
|
||||||
|
}
|
||||||
|
|
||||||
|
.field-pick-item:hover {
|
||||||
|
border-color: #d8e2dd;
|
||||||
|
}
|
||||||
|
|
||||||
|
.field-pick-item input {
|
||||||
|
width: 15px;
|
||||||
|
height: 15px;
|
||||||
|
padding: 0;
|
||||||
|
accent-color: #70ad95;
|
||||||
|
cursor: pointer;
|
||||||
|
}
|
||||||
|
|
||||||
|
.field-pick-item span {
|
||||||
|
color: #4a5854;
|
||||||
|
font-size: 12px;
|
||||||
|
}
|
||||||
|
|
||||||
.resource-platform-line {
|
.resource-platform-line {
|
||||||
display: flex;
|
display: flex;
|
||||||
min-width: 0;
|
min-width: 0;
|
||||||
|
|||||||
253
lib/account-field-refresh-service.ts
Normal file
253
lib/account-field-refresh-service.ts
Normal file
@@ -0,0 +1,253 @@
|
|||||||
|
import { runInBackground } from "./background";
|
||||||
|
import {
|
||||||
|
generateAccountTagsFromMcp,
|
||||||
|
resolveProfileDetailsFromMcp,
|
||||||
|
type AccountTaggingEnv,
|
||||||
|
type CollectionMcpConfig,
|
||||||
|
} from "./mcp-collection-client";
|
||||||
|
import { getRuntimeEnv } from "./runtime-env";
|
||||||
|
import type { DatabaseClient } from "./database";
|
||||||
|
|
||||||
|
export const ACCOUNT_REFRESH_FIELDS = [
|
||||||
|
"publicAccountId",
|
||||||
|
"nickname",
|
||||||
|
"followers",
|
||||||
|
"ipLocation",
|
||||||
|
"gender",
|
||||||
|
"bio",
|
||||||
|
"tags",
|
||||||
|
] as const;
|
||||||
|
|
||||||
|
export type AccountRefreshField = (typeof ACCOUNT_REFRESH_FIELDS)[number];
|
||||||
|
|
||||||
|
export const ACCOUNT_REFRESH_FIELD_LABELS: Record<AccountRefreshField, string> = {
|
||||||
|
publicAccountId: "账号ID(小红书号/抖音号)",
|
||||||
|
nickname: "昵称",
|
||||||
|
followers: "粉丝数",
|
||||||
|
ipLocation: "IP属地",
|
||||||
|
gender: "性别",
|
||||||
|
bio: "简介",
|
||||||
|
tags: "标签(AI 重新打标,覆盖已有标签)",
|
||||||
|
};
|
||||||
|
|
||||||
|
const FIELD_COLUMNS: Record<AccountRefreshField, string> = {
|
||||||
|
publicAccountId: "public_account_id",
|
||||||
|
nickname: "nickname",
|
||||||
|
followers: "followers",
|
||||||
|
ipLocation: "ip_location",
|
||||||
|
gender: "gender",
|
||||||
|
bio: "bio",
|
||||||
|
tags: "tags",
|
||||||
|
};
|
||||||
|
|
||||||
|
export function sanitizeRefreshFields(value: unknown): AccountRefreshField[] {
|
||||||
|
const requested = Array.isArray(value) ? value : [];
|
||||||
|
const fields = ACCOUNT_REFRESH_FIELDS.filter((field) =>
|
||||||
|
requested.includes(field),
|
||||||
|
);
|
||||||
|
return fields.length > 0 ? fields : ["publicAccountId"];
|
||||||
|
}
|
||||||
|
|
||||||
|
export type AccountFieldRefreshStatus = {
|
||||||
|
status: "running" | "done";
|
||||||
|
fields: AccountRefreshField[];
|
||||||
|
total: number;
|
||||||
|
processed: number;
|
||||||
|
updated: number;
|
||||||
|
failed: number;
|
||||||
|
skipped: number;
|
||||||
|
startedAt: string;
|
||||||
|
finishedAt: string | null;
|
||||||
|
};
|
||||||
|
|
||||||
|
type RefreshTargetRow = {
|
||||||
|
id: string;
|
||||||
|
platform: string;
|
||||||
|
platform_uid: string;
|
||||||
|
profile_url: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type ScrapedProfile = {
|
||||||
|
nickname: string | null;
|
||||||
|
redId: string | null;
|
||||||
|
followers: number | null;
|
||||||
|
ipLocation: string | null;
|
||||||
|
gender: string;
|
||||||
|
bio: string;
|
||||||
|
recentNoteTitles: string[];
|
||||||
|
providerTags: string[];
|
||||||
|
};
|
||||||
|
|
||||||
|
export type ProfileScraper = (
|
||||||
|
profileUrl: string,
|
||||||
|
platform: "小红书" | "抖音",
|
||||||
|
) => Promise<ScrapedProfile>;
|
||||||
|
|
||||||
|
export type TagGenerator = (input: {
|
||||||
|
platform: string;
|
||||||
|
nickname: string;
|
||||||
|
bio: string;
|
||||||
|
gender: string;
|
||||||
|
ipLocation: string;
|
||||||
|
followers: number | null;
|
||||||
|
providerTags: string[];
|
||||||
|
recentNoteTitles: string[];
|
||||||
|
}) => Promise<string[]>;
|
||||||
|
|
||||||
|
function scrapedFieldValues(
|
||||||
|
details: ScrapedProfile,
|
||||||
|
): Partial<Record<AccountRefreshField, string | number>> {
|
||||||
|
const values: Partial<Record<AccountRefreshField, string | number>> = {};
|
||||||
|
const redId = details.redId?.trim();
|
||||||
|
if (redId) values.publicAccountId = redId;
|
||||||
|
const nickname = details.nickname?.trim();
|
||||||
|
if (nickname) values.nickname = nickname;
|
||||||
|
if (details.followers !== null && Number.isFinite(details.followers)) {
|
||||||
|
values.followers = details.followers;
|
||||||
|
}
|
||||||
|
const ipLocation = details.ipLocation?.trim();
|
||||||
|
if (ipLocation) values.ipLocation = ipLocation;
|
||||||
|
if (details.gender === "男" || details.gender === "女") {
|
||||||
|
values.gender = details.gender;
|
||||||
|
}
|
||||||
|
const bio = details.bio?.trim();
|
||||||
|
if (bio) values.bio = bio;
|
||||||
|
return values;
|
||||||
|
}
|
||||||
|
|
||||||
|
let currentJob: AccountFieldRefreshStatus | null = null;
|
||||||
|
|
||||||
|
export function getAccountFieldRefreshStatus(): AccountFieldRefreshStatus | null {
|
||||||
|
return currentJob ? { ...currentJob } : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function startAccountFieldRefresh(
|
||||||
|
db: DatabaseClient,
|
||||||
|
config: CollectionMcpConfig,
|
||||||
|
requestedFields: unknown,
|
||||||
|
): AccountFieldRefreshStatus {
|
||||||
|
if (currentJob?.status === "running") {
|
||||||
|
return { ...currentJob };
|
||||||
|
}
|
||||||
|
const fields = sanitizeRefreshFields(requestedFields);
|
||||||
|
const job: AccountFieldRefreshStatus = {
|
||||||
|
status: "running",
|
||||||
|
fields,
|
||||||
|
total: 0,
|
||||||
|
processed: 0,
|
||||||
|
updated: 0,
|
||||||
|
failed: 0,
|
||||||
|
skipped: 0,
|
||||||
|
startedAt: new Date().toISOString(),
|
||||||
|
finishedAt: null,
|
||||||
|
};
|
||||||
|
currentJob = job;
|
||||||
|
const scraper: ProfileScraper = (profileUrl, platform) =>
|
||||||
|
resolveProfileDetailsFromMcp(profileUrl, platform, config);
|
||||||
|
const tagger: TagGenerator = (input) =>
|
||||||
|
generateAccountTagsFromMcp(
|
||||||
|
input,
|
||||||
|
config,
|
||||||
|
getRuntimeEnv() as AccountTaggingEnv,
|
||||||
|
);
|
||||||
|
runInBackground(
|
||||||
|
executeAccountFieldRefresh(job, db, scraper, tagger),
|
||||||
|
"account field refresh",
|
||||||
|
);
|
||||||
|
return { ...job };
|
||||||
|
}
|
||||||
|
|
||||||
|
async function executeAccountFieldRefresh(
|
||||||
|
job: AccountFieldRefreshStatus,
|
||||||
|
db: DatabaseClient,
|
||||||
|
scraper: ProfileScraper,
|
||||||
|
tagger: TagGenerator,
|
||||||
|
) {
|
||||||
|
const rows = await db
|
||||||
|
.prepare(
|
||||||
|
`SELECT id, platform, platform_uid, profile_url
|
||||||
|
FROM accounts
|
||||||
|
WHERE profile_url IS NOT NULL AND profile_url != ''
|
||||||
|
AND platform IN ('小红书', '抖音')`,
|
||||||
|
)
|
||||||
|
.all<RefreshTargetRow>();
|
||||||
|
const targets = rows.results.filter(
|
||||||
|
(row) => !row.platform_uid.startsWith("xhs-"),
|
||||||
|
);
|
||||||
|
job.total = targets.length;
|
||||||
|
const fields = job.fields;
|
||||||
|
const wantTags = fields.includes("tags");
|
||||||
|
|
||||||
|
let cursor = 0;
|
||||||
|
const worker = async () => {
|
||||||
|
while (cursor < targets.length) {
|
||||||
|
const row = targets[cursor];
|
||||||
|
cursor += 1;
|
||||||
|
try {
|
||||||
|
const details = await scraper(
|
||||||
|
row.profile_url,
|
||||||
|
row.platform === "抖音" ? "抖音" : "小红书",
|
||||||
|
);
|
||||||
|
const values = scrapedFieldValues(details);
|
||||||
|
if (wantTags) {
|
||||||
|
const tags = await tagger({
|
||||||
|
platform: row.platform,
|
||||||
|
nickname: details.nickname?.trim() || "",
|
||||||
|
bio: details.bio,
|
||||||
|
gender: details.gender,
|
||||||
|
ipLocation: details.ipLocation ?? "",
|
||||||
|
followers: details.followers,
|
||||||
|
providerTags: details.providerTags,
|
||||||
|
recentNoteTitles: details.recentNoteTitles,
|
||||||
|
}).catch(() => []);
|
||||||
|
if (tags.length > 0) values.tags = tags.join(",");
|
||||||
|
}
|
||||||
|
const updates = fields.filter(
|
||||||
|
(field) => values[field] !== undefined,
|
||||||
|
);
|
||||||
|
if (updates.length === 0) {
|
||||||
|
job.skipped += 1;
|
||||||
|
} else {
|
||||||
|
const assignments = updates
|
||||||
|
.map((field) => `${FIELD_COLUMNS[field]} = ?`)
|
||||||
|
.join(", ");
|
||||||
|
await db
|
||||||
|
.prepare(`UPDATE accounts SET ${assignments} WHERE id = ?`)
|
||||||
|
.bind(
|
||||||
|
...updates.map((field) => values[field]),
|
||||||
|
row.id,
|
||||||
|
)
|
||||||
|
.run();
|
||||||
|
job.updated += 1;
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
job.failed += 1;
|
||||||
|
}
|
||||||
|
job.processed += 1;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
await Promise.all(Array.from({ length: Math.min(4, targets.length) }, worker));
|
||||||
|
job.status = "done";
|
||||||
|
job.finishedAt = new Date().toISOString();
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function runAccountFieldRefresh(
|
||||||
|
db: DatabaseClient,
|
||||||
|
scraper: ProfileScraper,
|
||||||
|
tagger: TagGenerator,
|
||||||
|
fields: AccountRefreshField[],
|
||||||
|
): Promise<AccountFieldRefreshStatus> {
|
||||||
|
const job: AccountFieldRefreshStatus = {
|
||||||
|
status: "running",
|
||||||
|
fields: sanitizeRefreshFields(fields),
|
||||||
|
total: 0,
|
||||||
|
processed: 0,
|
||||||
|
updated: 0,
|
||||||
|
failed: 0,
|
||||||
|
skipped: 0,
|
||||||
|
startedAt: new Date().toISOString(),
|
||||||
|
finishedAt: null,
|
||||||
|
};
|
||||||
|
await executeAccountFieldRefresh(job, db, scraper, tagger);
|
||||||
|
return job;
|
||||||
|
}
|
||||||
@@ -1,134 +0,0 @@
|
|||||||
import { runInBackground } from "./background";
|
|
||||||
import {
|
|
||||||
resolveProfileDetailsFromMcp,
|
|
||||||
type CollectionMcpConfig,
|
|
||||||
} from "./mcp-collection-client";
|
|
||||||
import type { DatabaseClient } from "./database";
|
|
||||||
|
|
||||||
export type PublicAccountIdRefreshStatus = {
|
|
||||||
status: "running" | "done";
|
|
||||||
total: number;
|
|
||||||
processed: number;
|
|
||||||
updated: number;
|
|
||||||
failed: number;
|
|
||||||
skipped: number;
|
|
||||||
startedAt: string;
|
|
||||||
finishedAt: string | null;
|
|
||||||
};
|
|
||||||
|
|
||||||
type RefreshTargetRow = {
|
|
||||||
id: string;
|
|
||||||
platform: string;
|
|
||||||
platform_uid: string;
|
|
||||||
profile_url: string;
|
|
||||||
};
|
|
||||||
|
|
||||||
export type ProfileScraper = (
|
|
||||||
profileUrl: string,
|
|
||||||
platform: "小红书" | "抖音",
|
|
||||||
) => Promise<{ redId: string | null }>;
|
|
||||||
|
|
||||||
const defaultScraper =
|
|
||||||
(config: CollectionMcpConfig): ProfileScraper =>
|
|
||||||
(profileUrl, platform) =>
|
|
||||||
resolveProfileDetailsFromMcp(profileUrl, platform, config);
|
|
||||||
|
|
||||||
let currentJob: PublicAccountIdRefreshStatus | null = null;
|
|
||||||
|
|
||||||
export function getPublicAccountIdRefreshStatus(): PublicAccountIdRefreshStatus | null {
|
|
||||||
return currentJob ? { ...currentJob } : null;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function startPublicAccountIdRefresh(
|
|
||||||
db: DatabaseClient,
|
|
||||||
config: CollectionMcpConfig,
|
|
||||||
): PublicAccountIdRefreshStatus {
|
|
||||||
if (currentJob?.status === "running") {
|
|
||||||
return { ...currentJob };
|
|
||||||
}
|
|
||||||
const job: PublicAccountIdRefreshStatus = {
|
|
||||||
status: "running",
|
|
||||||
total: 0,
|
|
||||||
processed: 0,
|
|
||||||
updated: 0,
|
|
||||||
failed: 0,
|
|
||||||
skipped: 0,
|
|
||||||
startedAt: new Date().toISOString(),
|
|
||||||
finishedAt: null,
|
|
||||||
};
|
|
||||||
currentJob = job;
|
|
||||||
runInBackground(
|
|
||||||
executePublicAccountIdRefresh(job, db, defaultScraper(config)),
|
|
||||||
"public account id refresh",
|
|
||||||
);
|
|
||||||
return { ...job };
|
|
||||||
}
|
|
||||||
|
|
||||||
async function executePublicAccountIdRefresh(
|
|
||||||
job: PublicAccountIdRefreshStatus,
|
|
||||||
db: DatabaseClient,
|
|
||||||
scraper: ProfileScraper,
|
|
||||||
) {
|
|
||||||
const rows = await db
|
|
||||||
.prepare(
|
|
||||||
`SELECT id, platform, platform_uid, profile_url
|
|
||||||
FROM accounts
|
|
||||||
WHERE profile_url IS NOT NULL AND profile_url != ''
|
|
||||||
AND platform IN ('小红书', '抖音')`,
|
|
||||||
)
|
|
||||||
.all<RefreshTargetRow>();
|
|
||||||
const targets = rows.results.filter(
|
|
||||||
(row) => !row.platform_uid.startsWith("xhs-"),
|
|
||||||
);
|
|
||||||
job.total = targets.length;
|
|
||||||
|
|
||||||
let cursor = 0;
|
|
||||||
const worker = async () => {
|
|
||||||
while (cursor < targets.length) {
|
|
||||||
const row = targets[cursor];
|
|
||||||
cursor += 1;
|
|
||||||
try {
|
|
||||||
const details = await scraper(
|
|
||||||
row.profile_url,
|
|
||||||
row.platform === "抖音" ? "抖音" : "小红书",
|
|
||||||
);
|
|
||||||
const redId = details.redId?.trim();
|
|
||||||
if (redId) {
|
|
||||||
await db
|
|
||||||
.prepare(`UPDATE accounts SET public_account_id = ? WHERE id = ?`)
|
|
||||||
.bind(redId, row.id)
|
|
||||||
.run();
|
|
||||||
job.updated += 1;
|
|
||||||
} else {
|
|
||||||
job.skipped += 1;
|
|
||||||
}
|
|
||||||
} catch {
|
|
||||||
job.failed += 1;
|
|
||||||
}
|
|
||||||
job.processed += 1;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
await Promise.all(
|
|
||||||
Array.from({ length: Math.min(4, targets.length) }, worker),
|
|
||||||
);
|
|
||||||
job.status = "done";
|
|
||||||
job.finishedAt = new Date().toISOString();
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function runPublicAccountIdRefresh(
|
|
||||||
db: DatabaseClient,
|
|
||||||
scraper: ProfileScraper,
|
|
||||||
): Promise<PublicAccountIdRefreshStatus> {
|
|
||||||
const job: PublicAccountIdRefreshStatus = {
|
|
||||||
status: "running",
|
|
||||||
total: 0,
|
|
||||||
processed: 0,
|
|
||||||
updated: 0,
|
|
||||||
failed: 0,
|
|
||||||
skipped: 0,
|
|
||||||
startedAt: new Date().toISOString(),
|
|
||||||
finishedAt: null,
|
|
||||||
};
|
|
||||||
await executePublicAccountIdRefresh(job, db, scraper);
|
|
||||||
return job;
|
|
||||||
}
|
|
||||||
141
tests/account-field-refresh.test.mjs
Normal file
141
tests/account-field-refresh.test.mjs
Normal file
@@ -0,0 +1,141 @@
|
|||||||
|
import assert from "node:assert/strict";
|
||||||
|
import test from "node:test";
|
||||||
|
import {
|
||||||
|
runAccountFieldRefresh,
|
||||||
|
sanitizeRefreshFields,
|
||||||
|
} from "../lib/account-field-refresh-service.ts";
|
||||||
|
|
||||||
|
function fakeDb(accounts) {
|
||||||
|
const updates = [];
|
||||||
|
return {
|
||||||
|
db: {
|
||||||
|
prepare: (sql) => {
|
||||||
|
if (sql.startsWith("SELECT")) {
|
||||||
|
return {
|
||||||
|
all: async () => ({
|
||||||
|
results: accounts.map((account) => ({
|
||||||
|
id: account.id,
|
||||||
|
platform: account.platform,
|
||||||
|
platform_uid: account.platform_uid,
|
||||||
|
profile_url: account.profile_url,
|
||||||
|
})),
|
||||||
|
success: true,
|
||||||
|
meta: { changes: 0, last_row_id: 0 },
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
assert.match(sql, /^UPDATE accounts SET [a-z_, ?= ]+ WHERE id = \?$/);
|
||||||
|
return {
|
||||||
|
bind: (...args) => ({
|
||||||
|
run: async () => {
|
||||||
|
updates.push({ sql, args });
|
||||||
|
return { success: true, meta: { changes: 1, last_row_id: 0 } };
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
},
|
||||||
|
},
|
||||||
|
updates,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const profile = (overrides = {}) => ({
|
||||||
|
nickname: "昵称A",
|
||||||
|
redId: "id-a",
|
||||||
|
followers: 123,
|
||||||
|
ipLocation: "上海",
|
||||||
|
gender: "女",
|
||||||
|
bio: "简介A",
|
||||||
|
recentNoteTitles: [],
|
||||||
|
providerTags: [],
|
||||||
|
...overrides,
|
||||||
|
});
|
||||||
|
|
||||||
|
const noTag = async () => [];
|
||||||
|
|
||||||
|
test("writes only the selected fields as dynamic columns", async () => {
|
||||||
|
const accounts = [
|
||||||
|
{ id: "a1", platform: "小红书", platform_uid: "uid-1", profile_url: "https://www.xiaohongshu.com/user/profile/a1" },
|
||||||
|
];
|
||||||
|
const { db, updates } = fakeDb(accounts);
|
||||||
|
const status = await runAccountFieldRefresh(
|
||||||
|
db,
|
||||||
|
async () => profile(),
|
||||||
|
noTag,
|
||||||
|
["nickname", "ipLocation"],
|
||||||
|
);
|
||||||
|
|
||||||
|
assert.equal(status.status, "done");
|
||||||
|
assert.equal(status.updated, 1);
|
||||||
|
assert.equal(status.fields.join(","), "nickname,ipLocation");
|
||||||
|
assert.equal(updates.length, 1);
|
||||||
|
assert.match(updates[0].sql, /SET nickname = \?, ip_location = \?/);
|
||||||
|
assert.doesNotMatch(updates[0].sql, /public_account_id|followers|gender|bio|tags/);
|
||||||
|
assert.deepEqual(updates[0].args, ["昵称A", "上海", "a1"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("tags field uses the tagger output and joins with commas", async () => {
|
||||||
|
const accounts = [
|
||||||
|
{ id: "a1", platform: "小红书", platform_uid: "uid-1", profile_url: "https://www.xiaohongshu.com/user/profile/a1" },
|
||||||
|
];
|
||||||
|
const { db, updates } = fakeDb(accounts);
|
||||||
|
const status = await runAccountFieldRefresh(
|
||||||
|
db,
|
||||||
|
async () => profile({ redId: "" }),
|
||||||
|
async () => ["美妆", "学生党"],
|
||||||
|
["tags"],
|
||||||
|
);
|
||||||
|
|
||||||
|
assert.equal(status.updated, 1);
|
||||||
|
assert.match(updates[0].sql, /SET tags = \?/);
|
||||||
|
assert.deepEqual(updates[0].args, ["美妆,学生党", "a1"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("empty scrape values and empty tagger result are skipped without writing", async () => {
|
||||||
|
const accounts = [
|
||||||
|
{ id: "a1", platform: "小红书", platform_uid: "uid-1", profile_url: "https://www.xiaohongshu.com/user/profile/a1" },
|
||||||
|
];
|
||||||
|
const { db, updates } = fakeDb(accounts);
|
||||||
|
const status = await runAccountFieldRefresh(
|
||||||
|
db,
|
||||||
|
async () => profile({ redId: " ", nickname: null, followers: null, ipLocation: "", gender: "", bio: "" }),
|
||||||
|
noTag,
|
||||||
|
["publicAccountId", "nickname", "followers", "ipLocation", "gender", "bio", "tags"],
|
||||||
|
);
|
||||||
|
|
||||||
|
assert.equal(status.updated, 0);
|
||||||
|
assert.equal(status.skipped, 1);
|
||||||
|
assert.equal(updates.length, 0);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("scrape failures count as failed and demo accounts are excluded", async () => {
|
||||||
|
const accounts = [
|
||||||
|
{ id: "boom", platform: "小红书", platform_uid: "uid-1", profile_url: "https://www.xiaohongshu.com/user/profile/boom" },
|
||||||
|
{ id: "demo", platform: "小红书", platform_uid: "xhs-8af3", profile_url: "https://www.xiaohongshu.com/user/profile/demo" },
|
||||||
|
{ id: "ok", platform: "抖音", platform_uid: "uid-2", profile_url: "https://www.douyin.com/user/ok" },
|
||||||
|
];
|
||||||
|
const { db, updates } = fakeDb(accounts);
|
||||||
|
const status = await runAccountFieldRefresh(
|
||||||
|
db,
|
||||||
|
async (url) => {
|
||||||
|
if (url.includes("boom")) throw new Error("mcp down");
|
||||||
|
return profile({ redId: "ok-2" });
|
||||||
|
},
|
||||||
|
noTag,
|
||||||
|
["publicAccountId"],
|
||||||
|
);
|
||||||
|
|
||||||
|
assert.equal(status.total, 2);
|
||||||
|
assert.equal(status.updated, 1);
|
||||||
|
assert.equal(status.failed, 1);
|
||||||
|
assert.deepEqual(updates[0].args.slice(-1), ["ok"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("sanitizeRefreshFields filters unknown values and falls back to account id", () => {
|
||||||
|
assert.deepEqual(sanitizeRefreshFields(["nickname", "hacked", "tags"]), [
|
||||||
|
"nickname",
|
||||||
|
"tags",
|
||||||
|
]);
|
||||||
|
assert.deepEqual(sanitizeRefreshFields([]), ["publicAccountId"]);
|
||||||
|
assert.deepEqual(sanitizeRefreshFields("bad"), ["publicAccountId"]);
|
||||||
|
});
|
||||||
@@ -1,98 +0,0 @@
|
|||||||
import assert from "node:assert/strict";
|
|
||||||
import test from "node:test";
|
|
||||||
import {
|
|
||||||
runPublicAccountIdRefresh,
|
|
||||||
} from "../lib/account-id-refresh-service.ts";
|
|
||||||
|
|
||||||
function fakeDb(accounts) {
|
|
||||||
const updates = [];
|
|
||||||
return {
|
|
||||||
db: {
|
|
||||||
prepare: (sql) => {
|
|
||||||
if (sql.startsWith("SELECT")) {
|
|
||||||
return {
|
|
||||||
all: async () => ({
|
|
||||||
results: accounts.map((account) => ({
|
|
||||||
id: account.id,
|
|
||||||
platform: account.platform,
|
|
||||||
platform_uid: account.platform_uid,
|
|
||||||
profile_url: account.profile_url,
|
|
||||||
})),
|
|
||||||
success: true,
|
|
||||||
meta: { changes: 0, last_row_id: 0 },
|
|
||||||
}),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
const match = sql.match(/WHERE id = \?$/);
|
|
||||||
assert.ok(match, `unexpected sql: ${sql}`);
|
|
||||||
assert.match(sql, /UPDATE accounts SET public_account_id = \?/);
|
|
||||||
assert.doesNotMatch(sql, /followers|tags|nickname|bio|gender|ip_location/);
|
|
||||||
return {
|
|
||||||
bind: (value, id) => ({
|
|
||||||
run: async () => {
|
|
||||||
updates.push({ value, id });
|
|
||||||
return { success: true, meta: { changes: 1, last_row_id: 0 } };
|
|
||||||
},
|
|
||||||
}),
|
|
||||||
};
|
|
||||||
},
|
|
||||||
},
|
|
||||||
updates,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
test("updates only public_account_id for scrapeable accounts", async () => {
|
|
||||||
const accounts = [
|
|
||||||
{ id: "a1", platform: "小红书", platform_uid: "uid-1", profile_url: "https://www.xiaohongshu.com/user/profile/a1" },
|
|
||||||
{ id: "a2", platform: "抖音", platform_uid: "uid-2", profile_url: "https://www.douyin.com/user/a2" },
|
|
||||||
];
|
|
||||||
const { db, updates } = fakeDb(accounts);
|
|
||||||
const status = await runPublicAccountIdRefresh(db, async (url) => {
|
|
||||||
if (url.endsWith("a1")) return { redId: "xhs-id-1" };
|
|
||||||
if (url.endsWith("a2")) return { redId: " dy-id-2 " };
|
|
||||||
return { redId: null };
|
|
||||||
});
|
|
||||||
|
|
||||||
assert.equal(status.status, "done");
|
|
||||||
assert.equal(status.total, 2);
|
|
||||||
assert.equal(status.processed, 2);
|
|
||||||
assert.equal(status.updated, 2);
|
|
||||||
assert.equal(status.failed, 0);
|
|
||||||
assert.deepEqual(updates, [
|
|
||||||
{ value: "xhs-id-1", id: "a1" },
|
|
||||||
{ value: "dy-id-2", id: "a2" },
|
|
||||||
]);
|
|
||||||
});
|
|
||||||
|
|
||||||
test("counts failures and skips empty redId without writing", async () => {
|
|
||||||
const accounts = [
|
|
||||||
{ id: "a1", platform: "小红书", platform_uid: "uid-1", profile_url: "https://www.xiaohongshu.com/user/profile/a1" },
|
|
||||||
{ id: "a2", platform: "小红书", platform_uid: "uid-2", profile_url: "https://www.xiaohongshu.com/user/profile/a2" },
|
|
||||||
{ id: "a3", platform: "小红书", platform_uid: "uid-3", profile_url: "https://www.xiaohongshu.com/user/profile/a3" },
|
|
||||||
];
|
|
||||||
const { db, updates } = fakeDb(accounts);
|
|
||||||
const status = await runPublicAccountIdRefresh(db, async (url) => {
|
|
||||||
if (url.endsWith("a1")) throw new Error("mcp down");
|
|
||||||
if (url.endsWith("a2")) return { redId: " " };
|
|
||||||
return { redId: "ok-3" };
|
|
||||||
});
|
|
||||||
|
|
||||||
assert.equal(status.total, 3);
|
|
||||||
assert.equal(status.updated, 1);
|
|
||||||
assert.equal(status.failed, 1);
|
|
||||||
assert.equal(status.skipped, 1);
|
|
||||||
assert.deepEqual(updates, [{ value: "ok-3", id: "a3" }]);
|
|
||||||
});
|
|
||||||
|
|
||||||
test("ignores demo accounts and accounts without profile_url at query level", async () => {
|
|
||||||
const accounts = [
|
|
||||||
{ id: "demo", platform: "小红书", platform_uid: "xhs-8af3", profile_url: "https://www.xiaohongshu.com/user/profile/demo" },
|
|
||||||
{ id: "real", platform: "小红书", platform_uid: "uid-9", profile_url: "https://www.xiaohongshu.com/user/profile/real" },
|
|
||||||
];
|
|
||||||
const { db, updates } = fakeDb(accounts);
|
|
||||||
const status = await runPublicAccountIdRefresh(db, async () => ({ redId: "id-9" }));
|
|
||||||
|
|
||||||
assert.equal(status.total, 1);
|
|
||||||
assert.equal(status.updated, 1);
|
|
||||||
assert.deepEqual(updates, [{ value: "id-9", id: "real" }]);
|
|
||||||
});
|
|
||||||
Reference in New Issue
Block a user