From ddad4b7659503356782886f626bbd0aba8aeb9b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B7=AB=E5=87=A4=E8=90=8D?= Date: Tue, 11 Aug 2026 23:07:26 +0800 Subject: [PATCH] feat: ship self-hosted KOC LOOP workflows --- .dockerignore | 12 + .env.self-hosted.example | 30 + .gitignore | 1 + Dockerfile | 31 + README.md | 39 +- app/admin-app.tsx | 802 +++- app/api/action/route.ts | 82 +- app/api/auth/login/route.ts | 5 +- app/api/auth/logout/route.ts | 5 +- app/api/bootstrap/route.ts | 15 +- app/api/content-image-upload/route.ts | 2 - app/api/creator-screenshot/route.ts | 4 +- app/api/health/route.ts | 24 + app/api/mcp/route.ts | 10 +- app/api/partner-image/route.ts | 61 +- app/api/partner-upload/route.ts | 63 +- app/api/partner/route.ts | 135 +- app/api/recovery-export/route.ts | 7 +- app/api/resources-export/route.ts | 13 +- app/api/resources-import/route.ts | 302 ++ app/api/screenshot-task-export/route.ts | 102 + app/api/task-example-upload/route.ts | 30 + app/api/task-result-image/route.ts | 49 + app/api/upload/route.ts | 2 - app/api/users/route.ts | 2 - app/globals.css | 588 ++- app/layout.tsx | 15 +- build/sites-vite-plugin.ts | 45 - db/index.ts | 12 +- db/schema.ts | 330 +- deploy/nginx/Dockerfile | 11 + deploy/nginx/koc-loop.conf | 39 + design-qa.md | 69 +- docker-compose.self-hosted.yml | 74 + docs/KOC LOOP 私有化部署指南.md | 196 + drizzle.config.ts | 5 +- instrumentation.ts | 5 + koc-portal/app/globals.css | 288 ++ koc-portal/app/layout.tsx | 75 +- koc-portal/app/page.tsx | 619 ++- koc-portal/build/sites-vite-plugin.ts | 45 - koc-portal/db/index.ts | 13 - koc-portal/db/schema.ts | 4 - koc-portal/drizzle.config.ts | 7 - koc-portal/drizzle/meta/_journal.json | 5 - koc-portal/next.config.ts | 8 +- koc-portal/package-lock.json | 4959 ++--------------------- koc-portal/package.json | 22 +- koc-portal/tests/rendered-html.test.mjs | 45 +- koc-portal/tsconfig.json | 2 +- koc-portal/vite.config.ts | 59 - koc-portal/worker/index.ts | 47 - lib/account-enrichment-service.ts | 5 +- lib/background.ts | 11 + lib/collection-service.ts | 13 +- lib/database.ts | 210 + lib/mcp-collection-client.ts | 65 +- lib/mcp-operations.ts | 68 +- lib/mvp-db.ts | 106 +- lib/object-store.ts | 125 + lib/partner-cors.ts | 18 +- lib/resource-import.ts | 274 ++ lib/result-screenshots.ts | 34 + lib/runtime-env.ts | 45 + lib/scheduler.ts | 43 + lib/task-service.ts | 108 + lib/user-auth.ts | 15 +- mysql/0001_init.sql | 195 + mysql/0002_resource_import.sql | 2 + mysql/0003_screenshot_tasks.sql | 8 + mysql/0004_multi_result_screenshots.sql | 2 + next.config.ts | 3 +- package-lock.json | 3448 ++-------------- package.json | 32 +- public/KOC资源导入模板.xlsx | Bin 0 -> 6396 bytes scripts/check-mysql.mjs | 24 + scripts/import-d1-json.mjs | 103 + scripts/import-object-directory.mjs | 59 + scripts/migrate-mysql.mjs | 63 + tests/mcp-collection-client.test.mjs | 54 +- tests/mcp-task-server.test.mjs | 6 +- tests/rendered-html.test.mjs | 120 +- tests/resource-import.test.mjs | 66 + tests/result-screenshots.test.mjs | 31 + tests/self-hosted-runtime.test.mjs | 75 + tsconfig.json | 2 +- vite.config.ts | 61 - worker/index.ts | 85 - 88 files changed, 6056 insertions(+), 8938 deletions(-) create mode 100644 .dockerignore create mode 100644 .env.self-hosted.example create mode 100644 Dockerfile create mode 100644 app/api/health/route.ts create mode 100644 app/api/resources-import/route.ts create mode 100644 app/api/screenshot-task-export/route.ts create mode 100644 app/api/task-example-upload/route.ts create mode 100644 app/api/task-result-image/route.ts delete mode 100644 build/sites-vite-plugin.ts create mode 100644 deploy/nginx/Dockerfile create mode 100644 deploy/nginx/koc-loop.conf create mode 100644 docker-compose.self-hosted.yml create mode 100644 docs/KOC LOOP 私有化部署指南.md create mode 100644 instrumentation.ts delete mode 100644 koc-portal/build/sites-vite-plugin.ts delete mode 100644 koc-portal/db/index.ts delete mode 100644 koc-portal/db/schema.ts delete mode 100644 koc-portal/drizzle.config.ts delete mode 100644 koc-portal/drizzle/meta/_journal.json delete mode 100644 koc-portal/vite.config.ts delete mode 100644 koc-portal/worker/index.ts create mode 100644 lib/background.ts create mode 100644 lib/database.ts create mode 100644 lib/object-store.ts create mode 100644 lib/resource-import.ts create mode 100644 lib/result-screenshots.ts create mode 100644 lib/runtime-env.ts create mode 100644 lib/scheduler.ts create mode 100644 mysql/0001_init.sql create mode 100644 mysql/0002_resource_import.sql create mode 100644 mysql/0003_screenshot_tasks.sql create mode 100644 mysql/0004_multi_result_screenshots.sql create mode 100644 public/KOC资源导入模板.xlsx create mode 100644 scripts/check-mysql.mjs create mode 100644 scripts/import-d1-json.mjs create mode 100644 scripts/import-object-directory.mjs create mode 100644 scripts/migrate-mysql.mjs create mode 100644 tests/resource-import.test.mjs create mode 100644 tests/result-screenshots.test.mjs create mode 100644 tests/self-hosted-runtime.test.mjs delete mode 100644 vite.config.ts delete mode 100644 worker/index.ts diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..003601a --- /dev/null +++ b/.dockerignore @@ -0,0 +1,12 @@ +.git +.next +dist +node_modules +koc-portal/.next +koc-portal/out +koc-portal/dist +koc-portal/node_modules +.data +.env* +!.env.self-hosted.example +npm-debug.log* diff --git a/.env.self-hosted.example b/.env.self-hosted.example new file mode 100644 index 0000000..a06b4ea --- /dev/null +++ b/.env.self-hosted.example @@ -0,0 +1,30 @@ +# 复制为 .env.self-hosted 后填写。不要把真实密钥提交到 Git。 +MYSQL_ROOT_PASSWORD=replace-with-strong-root-password +MYSQL_DATABASE=koc_loop +MYSQL_USER=koc +MYSQL_PASSWORD=replace-with-strong-app-password + +# 对外地址。部署域名确定后替换。 +APP_ORIGIN=https://koc.example.com +KOC_PORTAL_URL=https://koc.example.com/koc/ + +# 后台首次启动会创建唯一超级管理员；账号创建后建议移除明文密码并重启。 +SUPER_ADMIN_USERNAME=admin +SUPER_ADMIN_PASSWORD=replace-with-strong-admin-password +ADMIN_INTERNAL_TOKEN=replace-with-random-internal-token + +# KOC LOOP 自身 MCP 的访问密钥。 +KOC_MCP_API_KEY=replace-with-random-mcp-key + +# 飞书内容表读取。 +FEISHU_APP_ID= +FEISHU_APP_SECRET= + +# 小红书公开数据采集 MCP。 +AI_TOOL_CENTER_MCP_URL= +AI_TOOL_CENTER_MCP_KEY= + +# 每天北京时间 10:00 自动执行采集计划。 +ENABLE_SCHEDULER=true +SEED_DEMO_DATA=false +HTTP_PORT=80 diff --git a/.gitignore b/.gitignore index 6a66e65..ed0e05c 100644 --- a/.gitignore +++ b/.gitignore @@ -31,6 +31,7 @@ yarn-error.log* # env files (can opt-in for committing if needed) .env* +!.env.self-hosted.example .dev.vars # vercel diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..ded2622 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,31 @@ +FROM node:22.18.0-bookworm-slim AS dependencies +WORKDIR /app +COPY package.json package-lock.json ./ +RUN npm ci + +FROM node:22.18.0-bookworm-slim AS builder +WORKDIR /app +ENV NEXT_TELEMETRY_DISABLED=1 +COPY --from=dependencies /app/node_modules ./node_modules +COPY . . +RUN npm run build + +FROM node:22.18.0-bookworm-slim AS runner +WORKDIR /app +ENV NODE_ENV=production \ + NEXT_TELEMETRY_DISABLED=1 \ + PORT=3000 \ + HOSTNAME=0.0.0.0 \ + UPLOAD_DIR=/data/koc/uploads +RUN groupadd --system --gid 1001 nodejs \ + && useradd --system --uid 1001 --gid nodejs nextjs \ + && mkdir -p /data/koc/uploads \ + && chown -R nextjs:nodejs /data/koc +COPY --from=builder --chown=nextjs:nodejs /app/.next/standalone ./ +COPY --from=builder --chown=nextjs:nodejs /app/.next/static ./.next/static +COPY --from=builder --chown=nextjs:nodejs /app/public ./public +COPY --from=builder --chown=nextjs:nodejs /app/mysql ./mysql +COPY --from=builder --chown=nextjs:nodejs /app/scripts ./scripts +USER nextjs +EXPOSE 3000 +CMD ["sh", "-c", "node scripts/migrate-mysql.mjs && node server.js"] diff --git a/README.md b/README.md index 8750a78..0035a5f 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,22 @@ # KOC LOOP -KOC 内容分发与数据回收闭环，运行于 vinext、Cloudflare D1 和 R2。 +KOC 内容分发与数据回收闭环。当前私有化分支运行于标准 Next.js Node.js、MySQL 8、Nginx 和本地持久化文件存储。 ## Prerequisites - Node.js `>=22.13.0` +- MySQL `>=8.0`（本地完整运行） ## Quick Start ```bash npm install +npm run db:migrate npm run dev npm run build ``` -复制 `.dev.vars.example` 为 `.dev.vars` 并配置运行时变量。飞书动态导入需要： +复制 `.env.self-hosted.example` 为 `.env.self-hosted`，设置 `DATABASE_URL` 或 `MYSQL_*` 连接信息。飞书动态导入需要： - `FEISHU_APP_ID` - `FEISHU_APP_SECRET` @@ -31,16 +33,7 @@ npm run build 系统首次登录时创建唯一的超级管理员。后续管理员和普通用户均由“用户管理”页面创建，普通用户不能访问 KOC 资源库。 -This starter does not use `wrangler.jsonc`. - -## Included Shape - -- edit site code under `app/` -- `.openai/hosting.json` declares optional Sites D1 and R2 bindings -- `vite.config.ts` simulates declared bindings for local development -- `db/schema.ts` starts intentionally empty -- `examples/d1/` contains an optional D1 example surface -- `drizzle.config.ts` supports local migration generation when needed +完整私有化部署请看 [KOC LOOP 私有化部署指南](docs/KOC%20LOOP%20私有化部署指南.md)。Docker Compose 会启动 Nginx、KOC 服务和 MySQL；外部领取页与后台使用同一域名下的 `/koc/` 路径。 ## 后台账号与角色 @@ -50,6 +43,16 @@ This starter does not use `wrangler.jsonc`. 后台不提供注册、找回密码和普通用户个人改密。密码重置统一由管理员在后台完成。 +## KOC 资源导入 + +超级管理员和管理员可在“KOC资源”页面下载标准 Excel 模板，批量导入已有资源。模板只需填写小红书账号主页，合作来源可选填；上传后系统自动解析账号名称、小红书号、IP属地和粉丝数。 + +- 单次最多导入 100 个账号，支持 `.xlsx` 和 `.csv`，文件不超过 5MB。 +- 当前自动解析支持小红书账号主页；上传后先展示新增、更新和异常数据，存在异常时不会写入数据库。 +- 按“平台 + 账号主页”去重；解析出相同小红书号时也会更新已有账号。 +- 重复账号更新公开资料和合作来源，不产生两份资源。 +- 导入的合作来源会进入现有资源搜索、筛选和导出结果。 + ## Agent MCP 生产地址： @@ -110,11 +113,15 @@ Authorization: Bearer ## Useful Commands - `npm run dev`: start local development -- `npm run build`: verify the vinext build output -- `npm test`: build the starter and verify its rendered loading skeleton +- `npm run build`: 验证标准 Next.js Node.js 生产构建 +- `npm test`: 构建并执行业务与私有化架构测试 - `npm run db:generate`: generate Drizzle migrations after schema changes +- `npm run db:migrate`: 应用 MySQL 增量迁移 +- `npm run db:check`: 检查 MySQL 连接 +- `npm run db:import-json -- `: 导入 D1 JSON 数据 +- `npm run storage:import -- `: 导入 R2 对象目录 ## Learn More -- [vinext Documentation](https://github.com/cloudflare/vinext) -- [Drizzle D1 Guide](https://orm.drizzle.team/docs/get-started/d1-new) +- [Next.js Self-Hosting](https://nextjs.org/docs/app/guides/self-hosting) +- [Drizzle MySQL Guide](https://orm.drizzle.team/docs/get-started-mysql) diff --git a/app/admin-app.tsx b/app/admin-app.tsx index 0b1fb5a..fdacfe8 100644 --- a/app/admin-app.tsx +++ b/app/admin-app.tsx @@ -13,6 +13,7 @@ import { formatShanghaiToday, } from "../lib/date-utils"; import type { AuthUser } from "../lib/user-auth"; +import { parseResultScreenshotKeys } from "../lib/result-screenshots"; import UsersPage from "./users-page"; type Partner = { @@ -32,6 +33,7 @@ type Task = { claimed_quantity: number; due_at: string; status: string; + task_type: "content_publish" | "screenshot_collect"; source_url: string; source_sheet_id: string; source_sheet_name: string; @@ -53,10 +55,27 @@ type Account = { followers: number; post_count: number; avg_views: number; + cooperation_source: string; first_seen_at: string; last_seen_at: string; }; +type ResourceImportPreview = { + summary: { total: number; create: number; update: number; error: number }; + rows: Array<{ + rowNumber: number; + platform: string; + nickname: string; + publicAccountId: string; + ipLocation: string; + followers: number; + cooperationSource: string; + action: "create" | "update" | "error"; + errors: string[]; + }>; + truncated: boolean; +}; + type Distribution = { id: string; task_id: string; @@ -66,6 +85,8 @@ type Distribution = { account_id: string | null; publish_url: string | null; publish_time: string | null; + result_screenshot_key: string | null; + result_submitted_at: string | null; status: string; claimed_at: string; screenshot_key: string | null; @@ -95,6 +116,7 @@ type Distribution = { account_platform: string | null; task_name: string; task_brand: string; + task_type: "content_publish" | "screenshot_collect"; due_at: string; }; @@ -129,8 +151,8 @@ type NavKey = const NAV_ITEMS: Array<{ key: NavKey; label: string; mark: string }> = [ { key: "overview", label: "工作台", mark: "⌂" }, - { key: "tasks", label: "内容任务", mark: "▤" }, - { key: "distributions", label: "内容分发", mark: "↗" }, + { key: "tasks", label: "任务中心", mark: "▤" }, + { key: "distributions", label: "任务分发", mark: "↗" }, { key: "resources", label: "KOC资源", mark: "◎" }, { key: "recovery", label: "数据回收", mark: "◫" }, { key: "users", label: "用户管理", mark: "♙" }, @@ -253,6 +275,8 @@ function xhsPublishUrl(value: string | null) { const isXhsHost = hostname === "xiaohongshu.com" || hostname.endsWith(".xiaohongshu.com") || + hostname === "xhslink.cn" || + hostname.endsWith(".xhslink.cn") || hostname === "xhslink.com" || hostname.endsWith(".xhslink.com"); return ["http:", "https:"].includes(url.protocol) && isXhsHost @@ -316,11 +340,16 @@ export default function Home({ currentUser }: { currentUser: AuthUser }) { const [taskModalOpen, setTaskModalOpen] = useState(false); const [metricModal, setMetricModal] = useState(null); const [taskForm, setTaskForm] = useState({ + taskType: "content_publish" as "content_publish" | "screenshot_collect", name: "", brand: "", dueAt: "2026-08-12", feishuUrl: "", + keyword: "", + instructions: "在小红书搜索指定关键词，打开 AI 总结并上传完整截图。", + quantity: 1, }); + const [taskExampleImage, setTaskExampleImage] = useState(null); const [sourcePreview, setSourcePreview] = useState(null); const [sourceWorking, setSourceWorking] = useState(false); const [metricForm, setMetricForm] = useState({ exposure: "", views: "" }); @@ -386,6 +415,7 @@ export default function Home({ currentUser }: { currentUser: AuthUser }) { const stats = useMemo(() => { const published = data.distributions.filter((item) => + item.task_type !== "screenshot_collect" && ["published", "collecting", "complete"].includes(item.status), ).length; const completed = data.distributions.filter( @@ -406,7 +436,9 @@ export default function Home({ currentUser }: { currentUser: AuthUser }) { }; }, [data]); - const recentDistributions = data.distributions.slice(0, 6); + const recentDistributions = data.distributions + .filter((item) => item.task_type !== "screenshot_collect") + .slice(0, 6); const navigate = (key: NavKey) => { if (key === "resources" && currentUser.role === "user") return; @@ -417,22 +449,64 @@ export default function Home({ currentUser }: { currentUser: AuthUser }) { const submitTask = async (event: FormEvent) => { event.preventDefault(); - if (!sourcePreview) { + if (taskForm.taskType === "content_publish" && !sourcePreview) { setToast("请先读取飞书表格"); return; } + let exampleImageKey = ""; + if (taskForm.taskType === "screenshot_collect" && taskExampleImage) { + try { + setWorking(true); + const form = new FormData(); + form.set("file", await compressScreenshot(taskExampleImage)); + const uploadResponse = await fetch("/api/task-example-upload", { + method: "POST", + body: form, + }); + const uploadResult = await readApiResponse<{ key?: string }>( + uploadResponse, + "示例截图上传失败", + ); + if (!uploadResponse.ok || !uploadResult.key) { + throw new Error(uploadResult.error || "示例截图上传失败"); + } + exampleImageKey = uploadResult.key; + } catch (reason) { + setToast(reason instanceof Error ? reason.message : "示例截图上传失败"); + setWorking(false); + return; + } + } const success = await runAction( - { action: "create_task", ...taskForm }, - `任务已创建，${sourcePreview.rowCount} 篇内容进入可领取池`, + taskForm.taskType === "screenshot_collect" + ? { + action: "create_screenshot_task", + name: taskForm.name, + brand: taskForm.brand, + dueAt: taskForm.dueAt, + keyword: taskForm.keyword, + instructions: taskForm.instructions, + quantity: taskForm.quantity, + exampleImageKey, + } + : { action: "create_task", ...taskForm }, + taskForm.taskType === "screenshot_collect" + ? `截图回收任务已创建，${taskForm.quantity} 份任务进入可领取池` + : `任务已创建，${sourcePreview?.rowCount ?? 0} 篇内容进入可领取池`, ); if (success) { setTaskModalOpen(false); setSourcePreview(null); + setTaskExampleImage(null); setTaskForm({ + taskType: "content_publish", name: "", brand: "", dueAt: "2026-08-12", feishuUrl: "", + keyword: "", + instructions: "在小红书搜索指定关键词，打开 AI 总结并上传完整截图。", + quantity: 1, }); } }; @@ -485,6 +559,34 @@ export default function Home({ currentUser }: { currentUser: AuthUser }) { } }; + const exportScreenshotResults = async (task: Task) => { + try { + setExportingTaskId(task.id); + const response = await fetch( + `/api/screenshot-task-export?task=${encodeURIComponent(task.id)}`, + ); + if (!response.ok) { + const result = await readApiResponse>( + response, + "截图打包失败", + ); + throw new Error(result.error || "截图打包失败"); + } + const blob = await response.blob(); + const url = URL.createObjectURL(blob); + const anchor = document.createElement("a"); + anchor.href = url; + anchor.download = `${task.name}-截图回收.zip`; + anchor.click(); + URL.revokeObjectURL(url); + setToast("任务截图和回收清单已打包下载"); + } catch (reason) { + setToast(reason instanceof Error ? reason.message : "截图打包失败"); + } finally { + setExportingTaskId(null); + } + }; + const collectMetrics = async (distribution: Distribution) => { await runAction( { action: "collect_now", distributionId: distribution.id }, @@ -586,6 +688,11 @@ export default function Home({ currentUser }: { currentUser: AuthUser }) { } }; + const completeResourceImport = async (message: string) => { + await loadData(); + setToast(message); + }; + const uploadScreenshot = async ( distribution: Distribution, event: ChangeEvent, @@ -794,6 +901,7 @@ export default function Home({ currentUser }: { currentUser: AuthUser }) { distributions={data.distributions} exporting={exportingResources} onExport={exportResources} + onImported={completeResourceImport} /> )} {activeNav === "users" && isManager && ( @@ -802,7 +910,7 @@ export default function Home({ currentUser }: { currentUser: AuthUser }) { {activeNav === "recovery" && ( item.publish_url)} + distributions={data.distributions} working={working} onCollect={collectMetrics} onSaveSchedule={saveCollectionSchedule} @@ -810,6 +918,7 @@ export default function Home({ currentUser }: { currentUser: AuthUser }) { onFallback={openMetricFallback} onRetryFailed={retryFailedMetrics} onExport={exportRecoveryData} + onExportScreenshots={exportScreenshotResults} exportingTaskId={exportingTaskId} /> )} @@ -823,57 +932,129 @@ export default function Home({ currentUser }: { currentUser: AuthUser }) {
event.stopPropagation()}>
-

飞书链接导入

-

新建分发任务

+

任务类型明确，回收链路独立

+

新建任务

- - {sourcePreview && ( -
-
-
- ✓ -
- {sourcePreview.sheetName} - 工作表 {sourcePreview.sheetId} · 已匹配 {sourcePreview.columns.length} 个字段 +
+ + +
+ {taskForm.taskType === "content_publish" ? ( + <> + + {sourcePreview && ( +
+
+
+ ✓ +
+ {sourcePreview.sheetName} + 工作表 {sourcePreview.sheetId} · 已匹配 {sourcePreview.columns.length} 个字段 +
+
+ {sourcePreview.rowCount} 篇 +
+
+ {sourcePreview.columns.map((column) => {column})} +
+
+ {sourcePreview.preview.map((row) => ( +
+ {row.sourceRow} + {row.title} +
+ ))}
- {sourcePreview.rowCount} 篇 -
-
- {sourcePreview.columns.map((column) => {column})} -
-
- {sourcePreview.preview.map((row) => ( -
- {row.sourceRow} - {row.title} -
- ))} + )} + + ) : ( +
+
+ +
+