wufengping/koc-loop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: ship self-hosted KOC LOOP workflows

Browse Source
This commit is contained in:
巫凤萍
2026-08-11 23:07:26 +08:00
parent 1f1887c860
commit ddad4b7659
88 changed files with 6056 additions and 8938 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
lib/account-enrichment-service.ts
View File

@@ -5,6 +5,7 @@ import {
type CollectionMcpConfig,
} from "./mcp-collection-client";
import { hashText } from "./mvp-db";
import type { DatabaseClient } from "./database";
type DistributionAccountRow = {
id: string;
@@ -37,7 +38,7 @@ function isVerifiedXhsProfileUrl(value: string | null) {
}
export async function enrichDistributionAccount(
db: D1Database,
db: DatabaseClient,
distributionId: string,
publishUrl: string,
fallbackNickname: string,
@@ -177,7 +178,7 @@ export async function enrichDistributionAccount(
}
export async function backfillAccountProfiles(
db: D1Database,
db: DatabaseClient,
mcpConfig: CollectionMcpConfig,
limit = 10,
) {

11
lib/background.ts Normal file
View File

@@ -0,0 +1,11 @@
export function runInBackground(
operation: Promise<unknown>,
label: string,
) {
void operation.catch((error) => {
console.error(
`[KOC LOOP] ${label} failed`,
error instanceof Error ? error.message : error,
);
});
}

13
lib/collection-service.ts
View File

@@ -3,6 +3,7 @@ import {
type CollectionMcpConfig,
} from "./mcp-collection-client";
import { uid } from "./mvp-db";
import type { DatabaseClient, DatabaseStatement } from "./database";
type DistributionForCollection = {
id: string;
@@ -84,7 +85,7 @@ function dueSchedules(
}
export async function createCollectionRunTasks(
db: D1Database,
db: DatabaseClient,
taskId: string,
startDate: string,
days: number[],
@@ -111,7 +112,7 @@ export async function createCollectionRunTasks(
.bind(taskId)
.run();
const statements: D1PreparedStatement[] = [];
const statements: DatabaseStatement[] = [];
for (const distribution of distributions.results) {
for (const scheduleDay of normalizedDays) {
const scheduledDate = dateForScheduleDay(startDate, scheduleDay);
@@ -147,7 +148,7 @@ export async function createCollectionRunTasks(
}
export async function collectDistributionMetrics(
db: D1Database,
db: DatabaseClient,
distributionId: string,
scheduledDate: string,
scheduleDay: number | null,
@@ -310,7 +311,7 @@ export async function collectDistributionMetrics(
}
export async function runScheduledCollections(
db: D1Database,
db: DatabaseClient,
scheduledTimestamp: number,
mcpConfig: CollectionMcpConfig,
) {
@@ -323,7 +324,7 @@ export async function runScheduledCollections(
}
export async function runDueScheduledCollections(
db: D1Database,
db: DatabaseClient,
timestamp: number,
mcpConfig: CollectionMcpConfig,
source: Extract<CollectionSource, "automatic" | "catchup"> = "catchup",
@@ -404,7 +405,7 @@ export async function runDueScheduledCollections(
}
export async function retryFailedCollections(
db: D1Database,
db: DatabaseClient,
taskId: string,
mcpConfig: CollectionMcpConfig,
) {

210
lib/database.ts Normal file
View File

@@ -0,0 +1,210 @@
import mysql, {
type Pool,
type PoolConnection,
type ResultSetHeader,
type RowDataPacket,
} from "mysql2/promise";
import { getDatabaseUrl } from "./runtime-env";
export type DatabaseResult<T = Record<string, unknown>> = {
results: T[];
success: boolean;
meta: {
changes: number;
last_row_id: number;
};
};
type Executor = Pool | PoolConnection;
type MysqlBindValue =
| string
| number
| bigint
| boolean
| Date
| null
| Blob
| Buffer
| Uint8Array
| MysqlBindValue[]
| { [key: string]: MysqlBindValue };
export function normalizeSqlForMysql(input: string) {
let sql = input.trim();
sql = sql.replace(/^INSERT\s+OR\s+IGNORE\s+INTO\b/i, "INSERT IGNORE INTO");
sql = sql.replace(/\s+ESCAPE\s+'\\\\'/gi, "");
sql = sql.replace(
/datetime\(\s*'now'\s*,\s*'-([0-9]+)\s+minutes?'\s*\)/gi,
"DATE_SUB(UTC_TIMESTAMP(), INTERVAL $1 MINUTE)",
);
sql = sql.replace(
/datetime\(\s*([^,]+?)\s*,\s*'\+([0-9]+)\s+days?'\s*\)/gi,
"DATE_ADD($1, INTERVAL $2 DAY)",
);
const conflict = sql.match(
/\s+ON\s+CONFLICT\s*\(([^)]+)\)\s+DO\s+UPDATE\s+SET\s+([\s\S]+)$/i,
);
if (conflict) {
const assignments = conflict[2].replace(
/\bexcluded\.([a-zA-Z_][a-zA-Z0-9_]*)\b/g,
"VALUES($1)",
);
sql = `${sql.slice(0, conflict.index)} ON DUPLICATE KEY UPDATE ${assignments}`;
}
return sql;
}
function normalizeBindValue(value: unknown): MysqlBindValue {
if (value === undefined) return null;
if (
typeof value === "string" &&
/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,3})?Z$/.test(value)
) {
return value.replace("T", " ").replace(/Z$/, "");
}
return value as MysqlBindValue;
}
export class DatabaseStatement {
private params: unknown[] = [];
readonly database: DatabaseClient;
readonly sql: string;
constructor(database: DatabaseClient, sql: string) {
this.database = database;
this.sql = sql;
}
bind(...params: unknown[]) {
const statement = new DatabaseStatement(this.database, this.sql);
statement.params = params;
return statement;
}
async all<T = Record<string, unknown>>() {
return this.database.execute<T>(this.sql, this.params);
}
async first<T = Record<string, unknown>>(column?: string) {
const result = await this.all<T>();
const row = result.results[0];
if (!row) return null;
return column ? ((row as Record<string, unknown>)[column] as T) : row;
}
async run() {
return this.database.execute(this.sql, this.params);
}
values() {
return [...this.params];
}
}
export class DatabaseClient {
private readonly executor: Executor;
constructor(executor: Executor) {
this.executor = executor;
}
prepare(sql: string) {
return new DatabaseStatement(this, sql);
}
async execute<T = Record<string, unknown>>(sql: string, params: unknown[] = []) {
const [result] = await this.executor.execute(
normalizeSqlForMysql(sql),
params.map(normalizeBindValue),
);
if (Array.isArray(result)) {
return {
results: result as T[],
success: true,
meta: { changes: 0, last_row_id: 0 },
} satisfies DatabaseResult<T>;
}
const header = result as ResultSetHeader;
return {
results: [],
success: true,
meta: {
changes: header.affectedRows ?? 0,
last_row_id: header.insertId ?? 0,
},
} satisfies DatabaseResult<T>;
}
async batch(statements: DatabaseStatement[]) {
const pool = getPool();
const connection = await pool.getConnection();
try {
await connection.beginTransaction();
const tx = new DatabaseClient(connection);
const results = [];
for (const statement of statements) {
results.push(await tx.execute(statement.sql, statement.values()));
}
await connection.commit();
return results;
} catch (error) {
await connection.rollback();
throw error;
} finally {
connection.release();
}
}
}
declare global {
var __kocLoopMysqlPool: Pool | undefined;
}
export function getPool() {
if (!globalThis.__kocLoopMysqlPool) {
globalThis.__kocLoopMysqlPool = mysql.createPool({
uri: getDatabaseUrl(),
connectionLimit: 10,
waitForConnections: true,
queueLimit: 0,
charset: "utf8mb4",
timezone: "Z",
dateStrings: true,
decimalNumbers: true,
enableKeepAlive: true,
});
}
return globalThis.__kocLoopMysqlPool;
}
export function getDatabase() {
return new DatabaseClient(getPool());
}
export async function checkDatabaseConnection() {
const [rows] = await getPool().query<RowDataPacket[]>("SELECT 1 AS healthy");
return rows[0]?.healthy === 1;
}
export async function withDatabaseLock<T>(
name: string,
timeoutSeconds: number,
operation: () => Promise<T>,
) {
const connection = await getPool().getConnection();
try {
const [rows] = await connection.query<RowDataPacket[]>(
"SELECT GET_LOCK(?, ?) AS acquired",
[name, timeoutSeconds],
);
if (Number(rows[0]?.acquired ?? 0) !== 1) return null;
try {
return await operation();
} finally {
await connection.query("SELECT RELEASE_LOCK(?)", [name]);
}
} finally {
connection.release();
}
}

65
lib/mcp-collection-client.ts
View File

@@ -225,7 +225,7 @@ async function createMcpSession(
timeoutMs,
);
const sessionId = initialize.response.headers.get("mcp-session-id");
if (!sessionId) throw new Error("MCP采集服务未返回会话标识");
if (!sessionId) return undefined;
await postMcp(
fetchImpl,
@@ -242,10 +242,10 @@ async function createMcpSession(
return sessionId;
}
async function callMcpTool(
async function invokeMcpTool(
fetchImpl: typeof fetch,
endpoint: string,
sessionId: string,
sessionId: string | undefined,
timeoutMs: number,
name: string,
args: Record<string, unknown>,
@@ -272,6 +272,12 @@ async function callMcpTool(
try {
payload = JSON.parse(text);
} catch {
if (result.envelope?.result?.isError === true) {
return {
isError: true,
payload: { message: text },
};
}
throw new Error("MCP采集工具返回了无法解析的数据");
}
return {
@@ -280,6 +286,40 @@ async function callMcpTool(
};
}
function isToolArgumentShapeError(result: ToolResult) {
if (!result.isError) return false;
const root = asRecord(result.payload);
const message = String(root?.message ?? "");
return /input validation error/i.test(message);
}
async function callMcpTool(
fetchImpl: typeof fetch,
endpoint: string,
sessionId: string | undefined,
timeoutMs: number,
name: string,
args: Record<string, unknown>,
): Promise<ToolResult> {
const nested = await invokeMcpTool(
fetchImpl,
endpoint,
sessionId,
timeoutMs,
name,
{ request: args },
);
if (!isToolArgumentShapeError(nested)) return nested;
return invokeMcpTool(
fetchImpl,
endpoint,
sessionId,
timeoutMs,
name,
args,
);
}
function metricsFromToolResult(result: ToolResult): XhsPublicMetrics {
const root = asRecord(result.payload);
const response = asRecord(root?.response) ?? root;
@@ -579,7 +619,7 @@ export async function resolveXhsPublicAccountDetails(
try {
parsed = new URL(profileUrl);
} catch {
return { redId: "", followers: null, ipLocation: "" };
return { nickname: "", redId: "", followers: null, ipLocation: "" };
}
if (
parsed.protocol !== "https:" ||
@@ -589,7 +629,7 @@ export async function resolveXhsPublicAccountDetails(
) ||
!parsed.pathname.startsWith("/user/profile/")
) {
return { redId: "", followers: null, ipLocation: "" };
return { nickname: "", redId: "", followers: null, ipLocation: "" };
}
try {
const response = await fetchImpl(parsed.toString(), {
@@ -602,10 +642,12 @@ export async function resolveXhsPublicAccountDetails(
},
});
if (!response.ok) {
return { redId: "", followers: null, ipLocation: "" };
return { nickname: "", redId: "", followers: null, ipLocation: "" };
}
const html = await response.text();
return {
nickname:
html.match(/"(?:nickname|nickName)":"([^"]+)"/)?.[1] ?? "",
redId:
html.match(/"(?:redId|red_id)":"([^"]+)"/)?.[1] ??
html.match(/小红书号[：:]\s*([^<"\s]+)/)?.[1] ??
@@ -620,7 +662,7 @@ export async function resolveXhsPublicAccountDetails(
ipLocation: "",
};
} catch {
return { redId: "", followers: null, ipLocation: "" };
return { nickname: "", redId: "", followers: null, ipLocation: "" };
}
}
@@ -650,6 +692,9 @@ function profileDetailsFromToolResult(result: ToolResult) {
const payload =
asRecord(response?.data) ?? asRecord(root?.data) ?? result.payload;
const followers = followerCountFromPayload(payload);
const nickname =
findStringByKey(payload, "nickname") ||
findStringByKey(payload, "nickName");
const redId =
findStringByKey(payload, "red_id") ||
findStringByKey(payload, "redId") ||
@@ -658,10 +703,10 @@ function profileDetailsFromToolResult(result: ToolResult) {
const ipLocation =
findStringByKey(payload, "ip_location") ||
findStringByKey(payload, "ipLocation");
if (followers === null && !redId && !ipLocation) {
if (followers === null && !nickname && !redId && !ipLocation) {
throw new Error("账号主页采集结果缺少可用字段");
}
return { followers, redId, ipLocation };
return { nickname, followers, redId, ipLocation };
}
function accountProfileFromToolResult(
@@ -717,7 +762,7 @@ async function completeAccountProfile(
profile: XhsAccountProfile,
fetchImpl: typeof fetch,
endpoint: string,
sessionId: string,
sessionId: string | undefined,
timeoutMs: number,
) {
let completed = profile;

68
lib/mcp-operations.ts
View File

@@ -83,9 +83,9 @@ export async function taskList(
(SELECT COUNT(*) FROM distributions d WHERE d.task_id = t.id AND COALESCE(d.publish_url, '') != '') AS published_count,
(SELECT COUNT(*) FROM distributions d WHERE d.task_id = t.id AND d.exposure IS NOT NULL AND d.views IS NOT NULL) AS day7_count
FROM tasks t ${where}
ORDER BY t.created_at DESC LIMIT ? OFFSET ?`,
ORDER BY t.created_at DESC LIMIT ${limit} OFFSET ${offset}`,
)
.bind(...bindings, limit, offset)
.bind(...bindings)
.all<Record<string, unknown>>(),
db
.prepare(`SELECT COUNT(*) AS total FROM tasks t ${where}`)
@@ -96,14 +96,19 @@ export async function taskList(
total: count?.total ?? 0,
limit,
offset,
tasks: rows.results.map((row) => ({
...row,
collection_days: parseJsonArray(row.collection_days),
claim_url:
portalUrl && row.share_token
? buildClaimUrl(portalUrl, String(row.share_token))
: null,
})),
tasks: rows.results.map(
(row): Record<string, unknown> & {
collection_days: unknown[];
claim_url: string | null;
} => ({
...row,
collection_days: parseJsonArray(row.collection_days),
claim_url:
portalUrl && row.share_token
? buildClaimUrl(portalUrl, String(row.share_token))
: null,
}),
),
};
}
@@ -151,15 +156,19 @@ export async function taskGet(taskId: string, portalUrl: string) {
.bind(taskId)
.all<Record<string, unknown>>(),
]);
return {
task: {
const taskOutput: Record<string, unknown> & {
collection_days: unknown[];
claim_url: string | null;
} = {
...task,
collection_days: parseJsonArray(task.collection_days),
claim_url:
portalUrl && task.share_token
? buildClaimUrl(portalUrl, String(task.share_token))
: null,
},
};
return {
task: taskOutput,
notes: notes.results.map((row) => ({
...row,
image_assets: parseJsonArray(row.image_assets),
@@ -214,9 +223,9 @@ export async function recoveryList(
`SELECT d.*, t.name AS task_name, c.source_row, c.title,
a.nickname AS account_nickname, a.profile_url, p.name AS cooperation_source,
cl.claimant_name ${base}
ORDER BY COALESCE(d.publish_time, d.claimed_at) DESC LIMIT ? OFFSET ?`,
ORDER BY COALESCE(d.publish_time, d.claimed_at) DESC LIMIT ${limit} OFFSET ${offset}`,
)
.bind(...bindings, limit, offset)
.bind(...bindings)
.all<Record<string, unknown>>(),
db
.prepare(`SELECT COUNT(*) AS total ${base}`)
@@ -359,10 +368,12 @@ function resourceWhere(input: ResourceFilters) {
}
if (input.cooperationSource?.trim()) {
conditions.push(
`EXISTS (SELECT 1 FROM distributions dx JOIN partners px ON px.id = dx.partner_id
WHERE dx.account_id = a.id AND px.name LIKE ? ESCAPE '\\')`,
`(a.cooperation_source LIKE ? ESCAPE '\\' OR
EXISTS (SELECT 1 FROM distributions dx JOIN partners px ON px.id = dx.partner_id
WHERE dx.account_id = a.id AND px.name LIKE ? ESCAPE '\\'))`,
);
bindings.push(like(input.cooperationSource.trim()));
const pattern = like(input.cooperationSource.trim());
bindings.push(pattern, pattern);
}
if (input.platform?.trim() && input.platform !== "all") {
conditions.push("a.platform = ?");
@@ -380,18 +391,27 @@ export async function resourceSearch(input: ResourceFilters) {
(SELECT COUNT(*) FROM distributions d WHERE d.account_id = a.id) AS cooperation_count,
(SELECT GROUP_CONCAT(DISTINCT p.name) FROM distributions d JOIN partners p ON p.id = d.partner_id WHERE d.account_id = a.id) AS cooperation_sources`;
const [rows, count] = await Promise.all([
db.prepare(`${select} FROM accounts a ${where} ORDER BY a.last_seen_at DESC LIMIT ? OFFSET ?`)
.bind(...bindings, limit, offset).all<Record<string, unknown>>(),
db.prepare(`${select} FROM accounts a ${where} ORDER BY a.last_seen_at DESC LIMIT ${limit} OFFSET ${offset}`)
.bind(...bindings).all<Record<string, unknown>>(),
db.prepare(`SELECT COUNT(*) AS total FROM accounts a ${where}`).bind(...bindings).first<{ total: number }>(),
]);
return {
total: count?.total ?? 0,
limit,
offset,
accounts: rows.results.map((row) => ({
...row,
cooperation_sources: String(row.cooperation_sources ?? "").split(",").filter(Boolean),
})),
accounts: rows.results.map(
(row): Record<string, unknown> & { cooperation_sources: string[] } => ({
...row,
cooperation_sources: [
...new Set(
`${row.cooperation_sources ?? ""}、${row.cooperation_source ?? ""}`
.split(/[、,，;；|]/)
.map((item) => item.trim())
.filter(Boolean),
),
],
}),
),
};
}

106
lib/mvp-db.ts
View File

@@ -1,26 +1,80 @@
import { env } from "cloudflare:workers";
import { type DatabaseClient, getDatabase } from "./database";
import { getObjectStore } from "./object-store";
import { getRuntimeEnv, isEnabled } from "./runtime-env";
import feishuSnapshot from "./feishu-source-snapshot.json";
type D1ResultRow = Record<string, unknown>;
export function getRawDb(): D1Database {
const database = (env as unknown as { DB?: D1Database }).DB;
if (!database) {
throw new Error("数据库尚未连接");
}
return database;
export function getRawDb(): DatabaseClient {
return getDatabase();
}
export function getUploadBucket(): R2Bucket {
const bucket = (env as unknown as { UPLOADS?: R2Bucket }).UPLOADS;
if (!bucket) {
throw new Error("文件存储尚未连接");
}
return bucket;
export function getUploadBucket() {
return getObjectStore();
}
export async function ensureSchema(database?: D1Database) {
export async function ensureSchema(database?: DatabaseClient) {
const db = database ?? getRawDb();
{
await db.prepare("SELECT id FROM tasks LIMIT 1").all();
const tasksWithoutShare = await db
.prepare("SELECT id FROM tasks WHERE share_token IS NULL OR share_token = ''")
.all<{ id: string }>();
for (const task of tasksWithoutShare.results) {
await db
.prepare("UPDATE tasks SET share_token = ? WHERE id = ?")
.bind(crypto.randomUUID().replaceAll("-", ""), task.id)
.run();
}
await db
.prepare(
`UPDATE distributions
SET latest_likes = COALESCE(d7_likes, d5_likes, d2_likes),
latest_comments = COALESCE(d7_comments, d5_comments, d2_comments),
latest_collects = COALESCE(d7_collects, d5_collects, d2_collects),
collection_status = 'success',
collection_status_description = '历史采集数据已迁移',
collection_updated_at = updated_at,
last_collection_day = CASE
WHEN d7_likes IS NOT NULL THEN 7
WHEN d5_likes IS NOT NULL THEN 5
WHEN d2_likes IS NOT NULL THEN 2
ELSE NULL
END
WHERE latest_likes IS NULL
AND COALESCE(d7_likes, d5_likes, d2_likes) IS NOT NULL`,
)
.run();
const tasksNeedingImages = await db
.prepare(
`SELECT DISTINCT t.id
FROM tasks t
JOIN contents c ON c.task_id = t.id
WHERE t.source_sheet_id = ?
AND (c.image_assets IS NULL OR c.image_assets = '' OR c.image_assets = '[]')`,
)
.bind(feishuSnapshot.sheetId)
.all<{ id: string }>();
for (const task of tasksNeedingImages.results) {
const updates = feishuSnapshot.rows
.filter((row) => row.images.length > 0)
.map((row) =>
db
.prepare(
`UPDATE contents SET image_assets = ?
WHERE task_id = ? AND source_row = ?
AND (image_assets IS NULL OR image_assets = '' OR image_assets = '[]')`,
)
.bind(JSON.stringify(row.images), task.id, row.sourceRow),
);
if (updates.length > 0) await db.batch(updates);
}
}
return;
const statements = [
`CREATE TABLE IF NOT EXISTS partners (
id TEXT PRIMARY KEY,
@@ -39,6 +93,7 @@ export async function ensureSchema(database?: D1Database) {
claimed_quantity INTEGER NOT NULL DEFAULT 0,
due_at TEXT NOT NULL,
status TEXT NOT NULL DEFAULT 'active',
task_type TEXT NOT NULL DEFAULT 'content_publish',
source_url TEXT NOT NULL DEFAULT '',
source_sheet_id TEXT NOT NULL DEFAULT '',
source_sheet_name TEXT NOT NULL DEFAULT '',
@@ -71,6 +126,7 @@ export async function ensureSchema(database?: D1Database) {
followers INTEGER NOT NULL DEFAULT 0,
post_count INTEGER NOT NULL DEFAULT 0,
avg_views INTEGER NOT NULL DEFAULT 0,
cooperation_source TEXT NOT NULL DEFAULT '',
first_seen_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
last_seen_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
)`,
@@ -109,6 +165,8 @@ export async function ensureSchema(database?: D1Database) {
publish_url TEXT,
publish_time TEXT,
publish_screenshot_key TEXT,
result_screenshot_key TEXT,
result_submitted_at TEXT,
status TEXT NOT NULL DEFAULT 'claimed',
claimed_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
screenshot_key TEXT,
@@ -192,6 +250,11 @@ export async function ensureSchema(database?: D1Database) {
};
await ensureColumn("tasks", "source_url", "source_url TEXT NOT NULL DEFAULT ''");
await ensureColumn(
"tasks",
"task_type",
"task_type TEXT NOT NULL DEFAULT 'content_publish'",
);
await ensureColumn(
"tasks",
"source_sheet_id",
@@ -241,6 +304,16 @@ export async function ensureSchema(database?: D1Database) {
"publish_screenshot_key",
"publish_screenshot_key TEXT",
);
await ensureColumn(
"distributions",
"result_screenshot_key",
"result_screenshot_key TEXT",
);
await ensureColumn(
"distributions",
"result_submitted_at",
"result_submitted_at TEXT",
);
await ensureColumn(
"distributions",
"latest_likes",
@@ -399,6 +472,7 @@ export async function ensureSchema(database?: D1Database) {
}
export async function seedIfEmpty() {
if (!isEnabled(getRuntimeEnv().SEED_DEMO_DATA)) return;
const db = getRawDb();
const row = await db.prepare("SELECT COUNT(*) AS count FROM tasks").first<{
count: number;
@@ -733,6 +807,7 @@ export async function getDashboardData() {
a.platform AS account_platform,
t.name AS task_name,
t.brand AS task_brand,
t.task_type AS task_type,
t.due_at AS due_at
FROM distributions d
JOIN contents c ON c.id = d.content_id
@@ -749,8 +824,7 @@ export async function getDashboardData() {
tasks: tasksResult.results as D1ResultRow[],
accounts: accountsResult.results as D1ResultRow[],
distributions: distributionsResult.results as D1ResultRow[],
portal_url:
(env as unknown as { KOC_PORTAL_URL?: string }).KOC_PORTAL_URL ?? "",
portal_url: getRuntimeEnv().KOC_PORTAL_URL ?? "",
};
}

125
lib/object-store.ts Normal file
View File

@@ -0,0 +1,125 @@
import { mkdir, readFile, rename, stat, writeFile } from "node:fs/promises";
import path from "node:path";
import { getRuntimeEnv } from "./runtime-env";
type PutOptions = {
httpMetadata?: { contentType?: string };
customMetadata?: Record<string, string>;
};
type StoredMetadata = {
contentType: string;
customMetadata: Record<string, string>;
uploadedAt: string;
};
export class StoredObjectBody {
readonly body: Uint8Array;
private readonly bytes: Buffer;
private readonly metadata: StoredMetadata;
constructor(bytes: Buffer, metadata: StoredMetadata) {
this.bytes = bytes;
this.metadata = metadata;
this.body = new Uint8Array(bytes);
}
async arrayBuffer() {
return this.body.buffer.slice(
this.body.byteOffset,
this.body.byteOffset + this.body.byteLength,
) as ArrayBuffer;
}
writeHttpMetadata(headers: Headers) {
headers.set("Content-Type", this.metadata.contentType);
}
get customMetadata() {
return this.metadata.customMetadata;
}
}
export function normalizeObjectKey(key: string) {
const normalized = key.replaceAll("\\", "/").replace(/^\/+/, "");
if (!normalized || normalized.includes("\0")) {
throw new Error("文件存储键无效");
}
const parts = normalized.split("/");
if (parts.some((part) => !part || part === "." || part === "..")) {
throw new Error("文件存储键不安全");
}
return parts.join("/");
}
function defaultUploadDir() {
return path.join(process.cwd(), ".data", "uploads");
}
export class LocalObjectStore {
readonly root: string;
constructor(root = getRuntimeEnv().UPLOAD_DIR || defaultUploadDir()) {
this.root = root;
}
private resolve(key: string) {
const normalized = normalizeObjectKey(key);
const filePath = path.join(this.root, ...normalized.split("/"));
return { normalized, filePath, metadataPath: `${filePath}.metadata.json` };
}
async put(key: string, input: ArrayBuffer | Uint8Array, options: PutOptions = {}) {
const target = this.resolve(key);
await mkdir(path.dirname(target.filePath), { recursive: true });
const bytes = input instanceof Uint8Array ? input : new Uint8Array(input);
const metadata: StoredMetadata = {
contentType: options.httpMetadata?.contentType || "application/octet-stream",
customMetadata: options.customMetadata ?? {},
uploadedAt: new Date().toISOString(),
};
const suffix = `${process.pid}-${crypto.randomUUID()}`;
const temporaryFile = `${target.filePath}.${suffix}.tmp`;
const temporaryMetadata = `${target.metadataPath}.${suffix}.tmp`;
await Promise.all([
writeFile(temporaryFile, bytes),
writeFile(temporaryMetadata, JSON.stringify(metadata), "utf8"),
]);
await rename(temporaryFile, target.filePath);
await rename(temporaryMetadata, target.metadataPath);
return { key: target.normalized };
}
async get(key: string) {
const target = this.resolve(key);
try {
const [bytes, metadataText] = await Promise.all([
readFile(target.filePath),
readFile(target.metadataPath, "utf8").catch(() => ""),
]);
const fallback: StoredMetadata = {
contentType: "application/octet-stream",
customMetadata: {},
uploadedAt: (await stat(target.filePath)).mtime.toISOString(),
};
const metadata = metadataText
? ({ ...fallback, ...JSON.parse(metadataText) } as StoredMetadata)
: fallback;
return new StoredObjectBody(bytes, metadata);
} catch (error) {
if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
throw error;
}
}
}
declare global {
var __kocLoopObjectStore: LocalObjectStore | undefined;
}
export function getObjectStore() {
if (!globalThis.__kocLoopObjectStore) {
globalThis.__kocLoopObjectStore = new LocalObjectStore();
}
return globalThis.__kocLoopObjectStore;
}

18
lib/partner-cors.ts
View File

@@ -1,12 +1,22 @@
import { env } from "cloudflare:workers";
import { getRuntimeEnv } from "./runtime-env";
const env = getRuntimeEnv();
function allowedOrigin(request: Request) {
const origin = request.headers.get("origin");
if (!origin) return null;
const portalOrigin = String(
const portalUrl = String(
(env as unknown as { KOC_PORTAL_URL?: string }).KOC_PORTAL_URL ?? "",
).replace(/\/$/, "");
return origin === portalOrigin || origin === "http://localhost:3000"
).trim();
let portalOrigin = "";
try {
portalOrigin = portalUrl ? new URL(portalUrl).origin : "";
} catch {
portalOrigin = "";
}
return [portalOrigin, "http://localhost:3000", "http://localhost:3001"].includes(
origin,
)
? origin
: null;
}

274
lib/resource-import.ts Normal file
View File

@@ -0,0 +1,274 @@
import { strFromU8, unzipSync } from "fflate";
export const RESOURCE_IMPORT_MAX_ROWS = 100;
export const RESOURCE_IMPORT_MAX_BYTES = 5 * 1024 * 1024;
export type ResourceImportRow = {
rowNumber: number;
platform: string;
nickname: string;
publicAccountId: string;
profileUrl: string;
ipLocation: string;
followers: number;
cooperationSource: string;
errors: string[];
};
const HEADER_ALIASES = {
profileUrl: ["账号主页", "账号主页链接", "主页链接", "账号链接"],
cooperationSource: ["合作来源", "资源来源", "历史合作来源"],
} as const;
type CanonicalHeader = keyof typeof HEADER_ALIASES;
function decodeXml(value: string) {
return value
.replace(/<[^>]+>/g, "")
.replace(/&lt;/g, "<")
.replace(/&gt;/g, ">")
.replace(/&quot;/g, '"')
.replace(/&apos;/g, "'")
.replace(/&amp;/g, "&")
.replace(/&#(\d+);/g, (_, code) => String.fromCodePoint(Number(code)))
.replace(/&#x([0-9a-f]+);/gi, (_, code) =>
String.fromCodePoint(Number.parseInt(code, 16)),
);
}
function textNodes(xml: string) {
return [...xml.matchAll(/<t\b[^>]*>([\s\S]*?)<\/t>/g)]
.map((match) => decodeXml(match[1]))
.join("");
}
function columnIndex(reference: string) {
const letters = reference.match(/^[A-Z]+/i)?.[0]?.toUpperCase() ?? "";
let result = 0;
for (const letter of letters) result = result * 26 + letter.charCodeAt(0) - 64;
return Math.max(0, result - 1);
}
function parseWorksheet(xml: string, sharedStrings: string[]) {
const rows: string[][] = [];
for (const rowMatch of xml.matchAll(/<row\b([^>]*)>([\s\S]*?)<\/row>/g)) {
const rowAttributes = rowMatch[1];
const rowNumber = Number(rowAttributes.match(/\br="(\d+)"/)?.[1] ?? rows.length + 1);
const values: string[] = [];
for (const cellMatch of rowMatch[2].matchAll(/<c\b([^>]*)>([\s\S]*?)<\/c>/g)) {
const attributes = cellMatch[1];
const body = cellMatch[2];
const reference = attributes.match(/\br="([A-Z]+\d+)"/i)?.[1] ?? "A1";
const type = attributes.match(/\bt="([^"]+)"/)?.[1] ?? "";
const rawValue = body.match(/<v>([\s\S]*?)<\/v>/)?.[1] ?? "";
let value = "";
if (type === "s") value = sharedStrings[Number(rawValue)] ?? "";
else if (type === "inlineStr") value = textNodes(body);
else if (type === "b") value = rawValue === "1" ? "TRUE" : "FALSE";
else value = decodeXml(rawValue);
values[columnIndex(reference)] = value.trim();
}
while (rows.length < rowNumber - 1) rows.push([]);
rows[rowNumber - 1] = values;
}
return rows;
}
function parseXlsx(bytes: Uint8Array) {
const entries = unzipSync(bytes);
const sharedXml = entries["xl/sharedStrings.xml"]
? strFromU8(entries["xl/sharedStrings.xml"])
: "";
const sharedStrings = [...sharedXml.matchAll(/<si\b[^>]*>([\s\S]*?)<\/si>/g)].map(
(match) => textNodes(match[1]),
);
const sheets = Object.keys(entries)
.filter((name) => /^xl\/worksheets\/sheet\d+\.xml$/.test(name))
.sort((left, right) => left.localeCompare(right, undefined, { numeric: true }));
if (sheets.length === 0) throw new Error("Excel 中没有可读取的工作表");
return sheets.map((name) => parseWorksheet(strFromU8(entries[name]), sharedStrings));
}
function parseCsv(text: string) {
const rows: string[][] = [];
let row: string[] = [];
let cell = "";
let quoted = false;
for (let index = 0; index < text.length; index += 1) {
const char = text[index];
if (quoted) {
if (char === '"' && text[index + 1] === '"') {
cell += '"';
index += 1;
} else if (char === '"') quoted = false;
else cell += char;
} else if (char === '"') quoted = true;
else if (char === ",") {
row.push(cell.trim());
cell = "";
} else if (char === "\n" || char === "\r") {
if (char === "\r" && text[index + 1] === "\n") index += 1;
row.push(cell.trim());
if (row.some(Boolean)) rows.push(row);
row = [];
cell = "";
} else cell += char;
}
row.push(cell.trim());
if (row.some(Boolean)) rows.push(row);
return rows;
}
function normalizeHeader(value: string) {
return value.replace(/[\s_\-（）()]/g, "").toLocaleLowerCase("zh-CN");
}
function canonicalHeader(value: string): CanonicalHeader | null {
const normalized = normalizeHeader(value);
for (const [key, aliases] of Object.entries(HEADER_ALIASES)) {
if (aliases.some((alias) => normalizeHeader(alias) === normalized)) {
return key as CanonicalHeader;
}
}
return null;
}
function findHeader(rows: string[][]) {
for (let index = 0; index < Math.min(rows.length, 12); index += 1) {
const mapping = new Map<CanonicalHeader, number>();
rows[index].forEach((cell, column) => {
const header = canonicalHeader(cell);
if (header && !mapping.has(header)) mapping.set(header, column);
});
if (mapping.has("profileUrl")) {
return { rowIndex: index, mapping };
}
}
return null;
}
export function normalizeProfileUrl(value: string) {
const extracted = value.match(/https?:\/\/[^\s，,；;]+/i)?.[0] ?? value.trim();
if (!extracted) return "";
try {
const url = new URL(extracted);
if (!(["http:", "https:"].includes(url.protocol))) return "";
url.protocol = "https:";
url.hostname = url.hostname.toLowerCase();
url.search = "";
url.hash = "";
url.pathname = url.pathname.replace(/\/+$/, "") || "/";
return url.toString().replace(/\/$/, "");
} catch {
return "";
}
}
export function platformFromProfileUrl(profileUrl: string) {
if (!profileUrl) return "";
try {
const url = new URL(profileUrl);
if (
(url.hostname === "xiaohongshu.com" ||
url.hostname.endsWith(".xiaohongshu.com")) &&
/^\/user\/profile\/[^/]+/i.test(url.pathname)
) {
return "小红书";
}
} catch {
// URL validation is reported by normalizeRows.
}
return "";
}
function valueAt(row: string[], mapping: Map<CanonicalHeader, number>, key: CanonicalHeader) {
const index = mapping.get(key);
return index === undefined ? "" : String(row[index] ?? "").trim();
}
function normalizeRows(rows: string[][]) {
const header = findHeader(rows);
if (!header) {
throw new Error("没有找到模板表头，请使用系统提供的导入模板");
}
const result: ResourceImportRow[] = [];
for (let index = header.rowIndex + 1; index < rows.length; index += 1) {
const source = rows[index];
if (!source.some((cell) => String(cell ?? "").trim())) continue;
const rawProfileUrl = valueAt(source, header.mapping, "profileUrl");
const profileUrl = normalizeProfileUrl(rawProfileUrl);
const platform = platformFromProfileUrl(profileUrl);
const errors: string[] = [];
if (!rawProfileUrl) errors.push("账号主页不能为空");
else if (!profileUrl) errors.push("账号主页链接格式不正确");
else if (!platform) errors.push("当前自动解析仅支持小红书账号主页");
result.push({
rowNumber: index + 1,
platform,
nickname: "",
publicAccountId: "",
profileUrl,
ipLocation: "待识别",
followers: 0,
cooperationSource: valueAt(source, header.mapping, "cooperationSource"),
errors,
});
}
if (result.length === 0) throw new Error("表格中没有可导入的账号数据");
if (result.length > RESOURCE_IMPORT_MAX_ROWS) {
throw new Error(`单次最多导入 ${RESOURCE_IMPORT_MAX_ROWS} 个账号`);
}
return result;
}
export function parseResourceImportFile(fileName: string, bytes: Uint8Array) {
const extension = fileName.toLocaleLowerCase().split(".").pop();
const workbooks =
extension === "csv"
? [parseCsv(new TextDecoder("utf-8").decode(bytes).replace(/^\uFEFF/, ""))]
: extension === "xlsx"
? parseXlsx(bytes)
: null;
if (!workbooks) throw new Error("仅支持 .xlsx 或 .csv 文件");
for (const rows of workbooks) {
if (findHeader(rows)) return normalizeRows(rows);
}
throw new Error("没有找到模板表头，请使用系统提供的导入模板");
}
function shortHash(value: string) {
let hash = 2166136261;
for (const character of value) {
hash ^= character.charCodeAt(0);
hash = Math.imul(hash, 16777619);
}
return Math.abs(hash >>> 0).toString(36);
}
export function resourcePlatformUid(row: Pick<ResourceImportRow, "platform" | "profileUrl" | "publicAccountId">) {
if (row.profileUrl) {
try {
const url = new URL(row.profileUrl);
const candidate =
url.pathname.match(/\/user\/profile\/([^/]+)/i)?.[1] ??
url.pathname.match(/\/(?:user|profile)\/([^/]+)/i)?.[1] ??
url.pathname.split("/").filter(Boolean).at(-1);
if (candidate && candidate.length >= 3) return candidate;
} catch {
// Validation already reports malformed profile links.
}
}
const identity = row.publicAccountId || row.profileUrl;
return `manual-${shortHash(`${row.platform}:${identity}`)}`;
}
export function mergeCooperationSources(existing: string, incoming: string) {
return [
...new Set(
`${existing}、${incoming}`
.split(/[、,，;；|]/)
.map((item) => item.trim())
.filter(Boolean),
),
].join("、");
}

34
lib/result-screenshots.ts Normal file
View File

@@ -0,0 +1,34 @@
export const MAX_RESULT_SCREENSHOTS = 9;
function isResultScreenshotKey(value: unknown): value is string {
return (
typeof value === "string" &&
value.startsWith("task-results/") &&
value.length <= 512
);
}
export function parseResultScreenshotKeys(value: unknown): string[] {
const text = String(value ?? "").trim();
if (!text) return [];
if (isResultScreenshotKey(text)) return [text];
try {
const parsed = JSON.parse(text) as unknown;
if (!Array.isArray(parsed)) return [];
return [...new Set(parsed.filter(isResultScreenshotKey))].slice(
0,
MAX_RESULT_SCREENSHOTS,
);
} catch {
return [];
}
}
export function serializeResultScreenshotKeys(keys: string[]) {
return JSON.stringify(
[...new Set(keys.filter(isResultScreenshotKey))].slice(
0,
MAX_RESULT_SCREENSHOTS,
),
);
}

45
lib/runtime-env.ts Normal file
View File

@@ -0,0 +1,45 @@
export type RuntimeEnv = {
DATABASE_URL?: string;
MYSQL_HOST?: string;
MYSQL_PORT?: string;
MYSQL_USER?: string;
MYSQL_PASSWORD?: string;
MYSQL_DATABASE?: string;
UPLOAD_DIR?: string;
APP_ORIGIN?: string;
KOC_PORTAL_URL?: string;
SUPER_ADMIN_USERNAME?: string;
SUPER_ADMIN_PASSWORD?: string;
ADMIN_INTERNAL_TOKEN?: string;
KOC_MCP_API_KEY?: string;
KOC_LOOP_MCP_API_KEY?: string;
MCP_API_KEY?: string;
FEISHU_APP_ID?: string;
FEISHU_APP_SECRET?: string;
AI_TOOL_CENTER_MCP_URL?: string;
AI_TOOL_CENTER_MCP_KEY?: string;
COLLECTION_MCP_URL?: string;
COLLECTION_MCP_KEY?: string;
SEED_DEMO_DATA?: string;
ENABLE_SCHEDULER?: string;
};
export function getRuntimeEnv(): RuntimeEnv {
return process.env as RuntimeEnv;
}
export function getDatabaseUrl() {
const env = getRuntimeEnv();
if (env.DATABASE_URL) return env.DATABASE_URL;
const host = env.MYSQL_HOST ?? "127.0.0.1";
const port = env.MYSQL_PORT ?? "3306";
const user = encodeURIComponent(env.MYSQL_USER ?? "koc");
const password = encodeURIComponent(env.MYSQL_PASSWORD ?? "");
const database = env.MYSQL_DATABASE ?? "koc_loop";
return `mysql://${user}:${password}@${host}:${port}/${database}`;
}
export function isEnabled(value: string | undefined, defaultValue = false) {
if (value === undefined || value === "") return defaultValue;
return !["0", "false", "no", "off"].includes(value.toLowerCase());
}

43
lib/scheduler.ts Normal file
View File

@@ -0,0 +1,43 @@
import cron, { type ScheduledTask } from "node-cron";
import { backfillAccountProfiles } from "./account-enrichment-service";
import { runScheduledCollections } from "./collection-service";
import { withDatabaseLock } from "./database";
import {
resolveCollectionMcpConfig,
type CollectionMcpBindings,
} from "./mcp-collection-client";
import { ensureSchema, getRawDb } from "./mvp-db";
import { getRuntimeEnv, isEnabled } from "./runtime-env";
declare global {
var __kocLoopScheduler: ScheduledTask | undefined;
}
async function runDailyJob() {
await withDatabaseLock("koc-loop-daily-collection", 0, async () => {
await ensureSchema();
const db = getRawDb();
const config = resolveCollectionMcpConfig(
getRuntimeEnv() as unknown as CollectionMcpBindings,
);
const collections = await runScheduledCollections(db, Date.now(), config);
const accounts = await backfillAccountProfiles(db, config, 10);
console.info("[KOC LOOP] daily scheduler completed", {
collections,
accounts,
});
});
}
export function startScheduler() {
if (!isEnabled(getRuntimeEnv().ENABLE_SCHEDULER, true)) return;
if (globalThis.__kocLoopScheduler) return;
globalThis.__kocLoopScheduler = cron.schedule(
"0 10 * * *",
() => void runDailyJob().catch((error) => {
console.error("[KOC LOOP] daily scheduler failed", error);
}),
{ timezone: "Asia/Shanghai", noOverlap: true },
);
console.info("[KOC LOOP] scheduler enabled at 10:00 Asia/Shanghai");
}

108
lib/task-service.ts
View File

@@ -12,6 +12,16 @@ export type CreateDistributionTaskInput = {
dueAt: string;
};
export type CreateScreenshotTaskInput = {
name: string;
brand: string;
dueAt: string;
keyword: string;
instructions: string;
quantity: number;
exampleImageKey?: string;
};
export type DistributionTaskCreation = {
created: boolean;
taskId: string;
@@ -25,6 +35,16 @@ export type DistributionTaskCreation = {
sourceUrl: string;
};
export type ScreenshotTaskCreation = {
created: true;
taskId: string;
shareToken: string;
name: string;
brand: string;
dueAt: string;
quantity: number;
};
type TaskRow = {
id: string;
share_token: string | null;
@@ -213,6 +233,94 @@ export async function createDistributionTask(
};
}
export async function createScreenshotTask(
rawInput: CreateScreenshotTaskInput,
): Promise<ScreenshotTaskCreation> {
await ensureSchema();
const input = {
name: normalizedValue(rawInput.name),
brand: normalizedValue(rawInput.brand),
dueAt: normalizedDueDate(rawInput.dueAt),
keyword: normalizedValue(rawInput.keyword),
instructions: normalizedValue(rawInput.instructions),
quantity: Math.floor(Number(rawInput.quantity)),
exampleImageKey: normalizedValue(rawInput.exampleImageKey ?? ""),
};
if (!input.name || !input.brand || !input.keyword || !input.instructions) {
throw new Error("请补全任务名称、品牌/项目、搜索关键词和任务说明");
}
if (!Number.isInteger(input.quantity) || input.quantity < 1 || input.quantity > 500) {
throw new Error("任务数量需为 1—500 份");
}
if (input.exampleImageKey && !input.exampleImageKey.startsWith("task-assets/")) {
throw new Error("示例截图无效，请重新上传");
}
const db = getRawDb();
const taskId = uid("task");
const shareToken = crypto.randomUUID().replaceAll("-", "");
const imageAssets = input.exampleImageKey
? JSON.stringify([
{ index: 1, key: input.exampleImageKey, width: null, height: null },
])
: "[]";
await db
.prepare(
`INSERT INTO tasks
(id, name, brand, quantity, claimed_quantity, due_at, status,
task_type, source_url, source_sheet_id, source_sheet_name,
share_token, collection_days)
VALUES (?, ?, ?, ?, 0, ?, 'active', 'screenshot_collect', '', '', '', ?, '[]')`,
)
.bind(
taskId,
input.name,
input.brand,
input.quantity,
input.dueAt,
shareToken,
)
.run();
try {
const statements = Array.from({ length: input.quantity }, (_, index) =>
db
.prepare(
`INSERT INTO contents
(id, task_id, title, body, image_assets, status, source, source_row)
VALUES (?, ?, ?, ?, ?, 'available', '截图回收任务', ?)`,
)
.bind(
uid("content"),
taskId,
input.keyword,
input.instructions,
imageAssets,
index + 1,
),
);
for (let index = 0; index < statements.length; index += 100) {
await db.batch(statements.slice(index, index + 100));
}
} catch (error) {
await db.batch([
db.prepare("DELETE FROM contents WHERE task_id = ?").bind(taskId),
db.prepare("DELETE FROM tasks WHERE id = ?").bind(taskId),
]);
throw error;
}
return {
created: true,
taskId,
shareToken,
name: input.name,
brand: input.brand,
dueAt: input.dueAt,
quantity: input.quantity,
};
}
export function buildClaimUrl(portalOrigin: string, shareToken: string) {
const origin = normalizedValue(portalOrigin).replace(/\/$/, "");
if (!origin) throw new Error("KOC 领取站点地址尚未配置");

15
lib/user-auth.ts
View File

@@ -1,6 +1,8 @@
import { env } from "cloudflare:workers";
import { getRuntimeEnv } from "./runtime-env";
import { ensureSchema, getRawDb } from "./mvp-db";
const env = getRuntimeEnv();
export type UserRole = "super_admin" | "admin" | "user";
export type AuthUser = {
@@ -15,7 +17,7 @@ export type RequestPrincipal =
const SESSION_COOKIE = "koc_session";
const SESSION_MAX_AGE_SECONDS = 60 * 60 * 24 * 7;
// Cloudflare Workers caps PBKDF2 at 100,000 iterations.
// Keep existing password records compatible while using a production-safe cost.
const PASSWORD_ITERATIONS = 100_000;
function bytesToHex(bytes: Uint8Array) {
@@ -198,6 +200,15 @@ export function clearSessionCookie(secure = true) {
.join("; ");
}
export function requestUsesHttps(request: Request) {
const forwardedProtocol = request.headers
.get("x-forwarded-proto")
?.split(",")[0]
?.trim()
.toLowerCase();
return forwardedProtocol === "https" || new URL(request.url).protocol === "https:";
}
export async function createSession(userId: string) {
const tokenBytes = new Uint8Array(32);
crypto.getRandomValues(tokenBytes);