Initial commit: KOC LOOP platform
This commit is contained in:
巫凤萍
43
scripts/build-feishu-snapshot.mjs
Normal file
43
scripts/build-feishu-snapshot.mjs
Normal file
@@ -0,0 +1,43 @@
|
||||
import { readFile, writeFile } from "node:fs/promises";
|
||||
import { resolve } from "node:path";
|
||||
|
||||
const [, , inputPath, outputPath] = process.argv;
|
||||
|
||||
if (!inputPath || !outputPath) {
|
||||
throw new Error("Usage: node scripts/build-feishu-snapshot.mjs <input.json> <output.json>");
|
||||
}
|
||||
|
||||
const payload = JSON.parse(await readFile(resolve(inputPath), "utf8"));
|
||||
const sheet = payload?.data?.sheets?.[0];
|
||||
|
||||
if (!sheet || !Array.isArray(sheet.data)) {
|
||||
throw new Error("Invalid lark-cli +table-get payload");
|
||||
}
|
||||
|
||||
const rows = sheet.data
|
||||
.map((row, index) => ({
|
||||
sourceRow: Number(row[0]) || index + 1,
|
||||
title: String(row[2] ?? "").trim(),
|
||||
body: String(row[3] ?? "").trim(),
|
||||
}))
|
||||
.filter((row) => row.title);
|
||||
|
||||
const snapshot = {
|
||||
wikiToken: "BSzxwRbGJi5dWoksauicUjtonks",
|
||||
sheetId: "954953",
|
||||
sheetName: sheet.name,
|
||||
sourceRange: sheet.range,
|
||||
syncedAt: new Date().toISOString(),
|
||||
columns: [
|
||||
"序号（不能改）",
|
||||
"标题",
|
||||
"笔记内容（正文+话题）",
|
||||
"图片1",
|
||||
"图片2",
|
||||
"图片3",
|
||||
],
|
||||
rows,
|
||||
};
|
||||
|
||||
await writeFile(resolve(outputPath), `${JSON.stringify(snapshot, null, 2)}\n`, "utf8");
|
||||
console.log(`Wrote ${rows.length} rows to ${outputPath}`);
|
||||
315
scripts/sync-feishu-images.mjs
Normal file
315
scripts/sync-feishu-images.mjs
Normal file
@@ -0,0 +1,315 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import {
|
||||
mkdir,
|
||||
readFile,
|
||||
readdir,
|
||||
stat,
|
||||
writeFile,
|
||||
} from "node:fs/promises";
|
||||
import { extname, resolve } from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
const projectRoot = resolve(import.meta.dirname, "..");
|
||||
const snapshotPath = resolve(projectRoot, "lib/feishu-source-snapshot.json");
|
||||
const downloadDir = "/private/tmp/koc-feishu-images-954953";
|
||||
const uploadOriginArg = process.argv.find((item) =>
|
||||
item.startsWith("--upload-origin="),
|
||||
);
|
||||
const uploadOrigin = uploadOriginArg
|
||||
? uploadOriginArg.slice("--upload-origin=".length).replace(/\/$/, "")
|
||||
: "";
|
||||
const internalToken =
|
||||
process.env.KOC_ADMIN_INTERNAL_TOKEN ??
|
||||
process.env.ADMIN_INTERNAL_TOKEN ??
|
||||
"";
|
||||
const reuseDownloads = process.argv.includes("--reuse-downloads");
|
||||
const fromRowArg = process.argv.find((item) => item.startsWith("--from-row="));
|
||||
const fromRow = fromRowArg
|
||||
? Math.max(1, Number(fromRowArg.slice("--from-row=".length)) || 1)
|
||||
: 1;
|
||||
|
||||
if (uploadOrigin && !internalToken) {
|
||||
throw new Error(
|
||||
"KOC_ADMIN_INTERNAL_TOKEN is required when --upload-origin is provided",
|
||||
);
|
||||
}
|
||||
|
||||
const snapshot = JSON.parse(await readFile(snapshotPath, "utf8"));
|
||||
const sourceUrl =
|
||||
"https://eodzc79n5l.feishu.cn/wiki/BSzxwRbGJi5dWoksauicUjtonks?from=from_copylink&sheet=954953";
|
||||
const quietEnv = {
|
||||
...process.env,
|
||||
LARKSUITE_CLI_NO_UPDATE_NOTIFIER: "1",
|
||||
LARKSUITE_CLI_NO_SKILLS_NOTIFIER: "1",
|
||||
};
|
||||
|
||||
await mkdir(downloadDir, { recursive: true });
|
||||
|
||||
const { stdout: cellsStdout } = await execFileAsync(
|
||||
"lark-cli",
|
||||
[
|
||||
"sheets",
|
||||
"+cells-get",
|
||||
"--url",
|
||||
sourceUrl,
|
||||
"--sheet-id",
|
||||
snapshot.sheetId,
|
||||
"--range",
|
||||
"F2:H62",
|
||||
"--include",
|
||||
"value",
|
||||
"--max-chars",
|
||||
"500000",
|
||||
"--format",
|
||||
"json",
|
||||
],
|
||||
{
|
||||
cwd: projectRoot,
|
||||
env: quietEnv,
|
||||
maxBuffer: 2_000_000,
|
||||
},
|
||||
);
|
||||
|
||||
const cellsPayload = JSON.parse(cellsStdout);
|
||||
if (!cellsPayload.ok || cellsPayload.data?.has_more) {
|
||||
throw new Error("Feishu image cells were not returned completely");
|
||||
}
|
||||
const range = cellsPayload.data?.ranges?.[0];
|
||||
if (!range || range.truncated || range.actual_range !== "F2:H62") {
|
||||
throw new Error(
|
||||
`Unexpected Feishu image range: ${range?.actual_range ?? "missing"}`,
|
||||
);
|
||||
}
|
||||
|
||||
const sourceRows = new Map(
|
||||
snapshot.rows.map((row) => [Number(row.sourceRow), row]),
|
||||
);
|
||||
const assets = [];
|
||||
|
||||
for (let rowIndex = 0; rowIndex < range.cells.length; rowIndex += 1) {
|
||||
const sheetRow = Number(range.row_indices[rowIndex]);
|
||||
const sourceRow = sheetRow - 1;
|
||||
if (!sourceRows.has(sourceRow)) continue;
|
||||
const cells = range.cells[rowIndex] ?? [];
|
||||
for (let columnIndex = 0; columnIndex < cells.length; columnIndex += 1) {
|
||||
const column = range.col_indices[columnIndex];
|
||||
const imageIndex = ["F", "G", "H"].indexOf(column) + 1;
|
||||
if (imageIndex < 1) continue;
|
||||
const richText = cells[columnIndex]?.rich_text ?? [];
|
||||
const image = richText.find((item) => item.type === "embed-image");
|
||||
if (!image?.image_token) continue;
|
||||
assets.push({
|
||||
sourceRow,
|
||||
imageIndex,
|
||||
token: image.image_token,
|
||||
width: Number(image.image_width) || null,
|
||||
height: Number(image.image_height) || null,
|
||||
key: `content-assets/${snapshot.sheetId}/${sourceRow}/${imageIndex}`,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async function downloadAsset(asset) {
|
||||
const baseName = `row-${asset.sourceRow}-image-${asset.imageIndex}`;
|
||||
if (reuseDownloads) {
|
||||
const existingName = (await readdir(downloadDir)).find((name) =>
|
||||
name.startsWith(`${baseName}.`),
|
||||
);
|
||||
if (existingName) {
|
||||
const localPath = resolve(downloadDir, existingName);
|
||||
const extension = extname(existingName).toLowerCase();
|
||||
const fileInfo = await stat(localPath);
|
||||
return {
|
||||
...asset,
|
||||
localPath,
|
||||
contentType:
|
||||
extension === ".png"
|
||||
? "image/png"
|
||||
: extension === ".webp"
|
||||
? "image/webp"
|
||||
: "image/jpeg",
|
||||
sizeBytes: fileInfo.size,
|
||||
};
|
||||
}
|
||||
}
|
||||
const { stdout } = await execFileAsync(
|
||||
"lark-cli",
|
||||
[
|
||||
"docs",
|
||||
"+media-download",
|
||||
"--token",
|
||||
asset.token,
|
||||
"--output",
|
||||
`./${baseName}`,
|
||||
"--overwrite",
|
||||
],
|
||||
{
|
||||
cwd: downloadDir,
|
||||
env: quietEnv,
|
||||
maxBuffer: 200_000,
|
||||
},
|
||||
);
|
||||
const result = JSON.parse(stdout.slice(stdout.indexOf("{")));
|
||||
if (!result.ok || !result.data?.saved_path) {
|
||||
throw new Error(
|
||||
`Failed to download row ${asset.sourceRow} image ${asset.imageIndex}`,
|
||||
);
|
||||
}
|
||||
return {
|
||||
...asset,
|
||||
localPath: result.data.saved_path,
|
||||
contentType: result.data.content_type || "application/octet-stream",
|
||||
sizeBytes: Number(result.data.size_bytes) || 0,
|
||||
};
|
||||
}
|
||||
|
||||
const downloaded = [];
|
||||
const queue = [...assets];
|
||||
const workers = Array.from({ length: 4 }, async () => {
|
||||
while (queue.length > 0) {
|
||||
const asset = queue.shift();
|
||||
if (!asset) return;
|
||||
downloaded.push(await downloadAsset(asset));
|
||||
}
|
||||
});
|
||||
await Promise.all(workers);
|
||||
downloaded.sort(
|
||||
(left, right) =>
|
||||
left.sourceRow - right.sourceRow || left.imageIndex - right.imageIndex,
|
||||
);
|
||||
|
||||
for (const row of snapshot.rows) {
|
||||
row.images = downloaded
|
||||
.filter((asset) => asset.sourceRow === Number(row.sourceRow))
|
||||
.map((asset) => ({
|
||||
index: asset.imageIndex,
|
||||
key: asset.key,
|
||||
width: asset.width,
|
||||
height: asset.height,
|
||||
}));
|
||||
}
|
||||
await writeFile(snapshotPath, `${JSON.stringify(snapshot, null, 2)}\n`, "utf8");
|
||||
|
||||
if (uploadOrigin) {
|
||||
const sharp = (await import("sharp")).default;
|
||||
const uploadConcurrency = new URL(uploadOrigin).hostname === "localhost" ? 4 : 1;
|
||||
|
||||
async function prepareUpload(asset) {
|
||||
const original = await readFile(asset.localPath);
|
||||
if (original.byteLength < 800_000) {
|
||||
return {
|
||||
bytes: original,
|
||||
contentType: asset.contentType,
|
||||
extension: extname(asset.localPath) || ".bin",
|
||||
};
|
||||
}
|
||||
let quality = 86;
|
||||
let compressed = await sharp(original)
|
||||
.rotate()
|
||||
.flatten({ background: "#ffffff" })
|
||||
.resize({
|
||||
width: 1800,
|
||||
height: 2200,
|
||||
fit: "inside",
|
||||
withoutEnlargement: true,
|
||||
})
|
||||
.jpeg({ quality, mozjpeg: true })
|
||||
.toBuffer();
|
||||
while (compressed.byteLength > 800_000 && quality > 58) {
|
||||
quality -= 7;
|
||||
compressed = await sharp(original)
|
||||
.rotate()
|
||||
.flatten({ background: "#ffffff" })
|
||||
.resize({
|
||||
width: 1600,
|
||||
height: 2000,
|
||||
fit: "inside",
|
||||
withoutEnlargement: true,
|
||||
})
|
||||
.jpeg({ quality, mozjpeg: true })
|
||||
.toBuffer();
|
||||
}
|
||||
return {
|
||||
bytes: compressed,
|
||||
contentType: "image/jpeg",
|
||||
extension: ".jpg",
|
||||
};
|
||||
}
|
||||
|
||||
const uploadQueue = downloaded.filter((asset) => asset.sourceRow >= fromRow);
|
||||
const uploadWorkers = Array.from({ length: uploadConcurrency }, async () => {
|
||||
while (uploadQueue.length > 0) {
|
||||
const asset = uploadQueue.shift();
|
||||
if (!asset) return;
|
||||
const prepared = await prepareUpload(asset);
|
||||
let uploaded = false;
|
||||
let lastError = "unknown error";
|
||||
for (let attempt = 1; attempt <= 3; attempt += 1) {
|
||||
const form = new FormData();
|
||||
form.append("sheetId", snapshot.sheetId);
|
||||
form.append("sourceRow", String(asset.sourceRow));
|
||||
form.append("imageIndex", String(asset.imageIndex));
|
||||
form.append(
|
||||
"file",
|
||||
new File(
|
||||
[prepared.bytes],
|
||||
`image-${asset.imageIndex}${prepared.extension}`,
|
||||
{
|
||||
type: prepared.contentType,
|
||||
},
|
||||
),
|
||||
);
|
||||
const response = await fetch(
|
||||
`${uploadOrigin}/api/content-image-upload`,
|
||||
{
|
||||
method: "POST",
|
||||
headers: {
|
||||
"X-KOC-Admin-Token": internalToken,
|
||||
},
|
||||
body: form,
|
||||
},
|
||||
);
|
||||
const responseText = await response.text();
|
||||
let result = {};
|
||||
try {
|
||||
result = JSON.parse(responseText);
|
||||
} catch {
|
||||
result = { error: responseText || `HTTP ${response.status}` };
|
||||
}
|
||||
if (response.ok) {
|
||||
uploaded = true;
|
||||
break;
|
||||
}
|
||||
lastError = result.error ?? `HTTP ${response.status}`;
|
||||
if (response.status < 500 && response.status !== 404) break;
|
||||
await new Promise((resolveDelay) =>
|
||||
setTimeout(resolveDelay, attempt * 750),
|
||||
);
|
||||
}
|
||||
if (!uploaded) {
|
||||
throw new Error(
|
||||
`Upload failed for row ${asset.sourceRow} image ${asset.imageIndex}: ${lastError}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
await Promise.all(uploadWorkers);
|
||||
}
|
||||
|
||||
const totalBytes = downloaded.reduce(
|
||||
(sum, asset) => sum + asset.sizeBytes,
|
||||
0,
|
||||
);
|
||||
console.log(
|
||||
JSON.stringify({
|
||||
rows: snapshot.rows.length,
|
||||
images: downloaded.length,
|
||||
totalBytes,
|
||||
uploaded: Boolean(uploadOrigin),
|
||||
uploadedImages: uploadOrigin
|
||||
? downloaded.filter((asset) => asset.sourceRow >= fromRow).length
|
||||
: 0,
|
||||
downloadDir,
|
||||
}),
|
||||
);
|
||||
Reference in New Issue
Block a user