Files
koc-loop/lib/partner-batch-workbook.ts
2026-08-15 03:53:09 +08:00

548 lines
18 KiB
TypeScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import path from "node:path";
import { strFromU8, unzipSync } from "fflate";
export const PARTNER_BATCH_HEADERS = [
"序号(不能改)",
"标题",
"笔记内容(正文+话题)",
"图片",
"发布链接",
"笔记截图",
"数据分析截图(单篇笔记数据分析截图)",
"_系统笔记ID",
"_原笔记截图",
"_原数据分析截图",
] as const;
export const PARTNER_BATCH_VISIBLE_COLUMN_COUNT = 7;
export const PARTNER_BATCH_MAX_BYTES = 80_000_000;
export type PartnerBatchWorkbookColumns = {
headers: string[];
columnWidths: number[];
sourceImageStartColumn: number;
sourceImageCount: number;
sourceVideoStartColumn: number;
sourceVideoCount: number;
publishUrlColumn: number;
publishScreenshotColumn: number;
creatorScreenshotColumn: number;
systemColumn: number;
};
function nonNegativeInteger(value: number) {
return Number.isFinite(value) ? Math.max(0, Math.floor(value)) : 0;
}
export function buildPartnerBatchWorkbookColumns(input: {
contentFormat: "image_text" | "video";
maxSourceImages: number;
maxSourceVideos: number;
}): PartnerBatchWorkbookColumns {
const sourceImageCount =
input.contentFormat === "video"
? 0
: Math.max(1, nonNegativeInteger(input.maxSourceImages));
const sourceVideoCount =
input.contentFormat === "video"
? Math.max(1, nonNegativeInteger(input.maxSourceVideos))
: 0;
const sourceImageStartColumn = 3;
const sourceVideoStartColumn = sourceImageStartColumn + sourceImageCount;
const publishUrlColumn = sourceVideoStartColumn + sourceVideoCount;
const publishScreenshotColumn = publishUrlColumn + 1;
const creatorScreenshotColumn = publishScreenshotColumn + 1;
const systemColumn = creatorScreenshotColumn + 1;
return {
headers: [
"序号(不能改)",
"标题",
"笔记内容(正文+话题)",
...Array.from(
{ length: sourceImageCount },
(_, index) => `图片${index + 1}`,
),
...Array.from(
{ length: sourceVideoCount },
(_, index) => `视频${index + 1}`,
),
"发布链接",
"笔记截图",
"数据分析截图(单篇笔记数据分析截图)",
"_系统笔记ID",
"_原笔记截图",
"_原数据分析截图",
],
columnWidths: [
14,
30,
62,
...Array.from({ length: sourceImageCount }, () => 24),
...Array.from({ length: sourceVideoCount }, () => 20),
45,
28,
32,
22,
22,
22,
],
sourceImageStartColumn,
sourceImageCount,
sourceVideoStartColumn,
sourceVideoCount,
publishUrlColumn,
publishScreenshotColumn,
creatorScreenshotColumn,
systemColumn,
};
}
function firstForwardedValue(value: string | null) {
return value?.split(",")[0]?.trim() ?? "";
}
function httpOrigin(value: string) {
try {
const url = new URL(value);
return url.protocol === "http:" || url.protocol === "https:"
? url.origin
: "";
} catch {
return "";
}
}
export function resolvePartnerWorkbookOrigin(
request: Request,
configuredOrigin = "",
) {
const requestUrl = new URL(request.url);
const host =
firstForwardedValue(request.headers.get("x-forwarded-host")) ||
firstForwardedValue(request.headers.get("host"));
const forwardedProtocol = firstForwardedValue(
request.headers.get("x-forwarded-proto"),
).toLowerCase();
const protocol = ["http", "https"].includes(forwardedProtocol)
? forwardedProtocol
: requestUrl.protocol.replace(":", "");
const proxyOrigin = host ? httpOrigin(`${protocol}://${host}`) : "";
return (
httpOrigin(configuredOrigin) ||
proxyOrigin ||
httpOrigin(requestUrl.origin) ||
requestUrl.origin
);
}
export type PartnerBatchImage = {
bytes: Uint8Array;
contentType: string;
fileName: string;
};
export type PartnerBatchImportRow = {
spreadsheetRow: number;
sequence: string;
title: string;
publishUrl: string;
distributionId: string;
originalPublishScreenshotKey: string;
originalCreatorScreenshotKey: string;
publishScreenshot: PartnerBatchImage | null;
creatorScreenshot: PartnerBatchImage | null;
};
function decodeXml(value: string) {
return value
.replace(/<[^>]+>/g, "")
.replace(/&lt;/g, "<")
.replace(/&gt;/g, ">")
.replace(/&quot;/g, '"')
.replace(/&apos;/g, "'")
.replace(/&amp;/g, "&")
.replace(/&#(\d+);/g, (_, code) => String.fromCodePoint(Number(code)))
.replace(/&#x([0-9a-f]+);/gi, (_, code) =>
String.fromCodePoint(Number.parseInt(code, 16)),
);
}
function xmlAttribute(value: string) {
return decodeXml(value);
}
function textNodes(xml: string) {
return [...xml.matchAll(/<t\b[^>]*>([\s\S]*?)<\/t>/g)]
.map((match) => decodeXml(match[1]))
.join("");
}
function columnIndex(reference: string) {
const letters = reference.match(/^[A-Z]+/i)?.[0]?.toUpperCase() ?? "";
let result = 0;
for (const letter of letters) result = result * 26 + letter.charCodeAt(0) - 64;
return Math.max(0, result - 1);
}
function parseWorksheet(xml: string, sharedStrings: string[]) {
const rows: string[][] = [];
for (const rowMatch of xml.matchAll(/<row\b([^>]*)>([\s\S]*?)<\/row>/g)) {
const rowNumber = Number(
rowMatch[1].match(/\br="(\d+)"/)?.[1] ?? rows.length + 1,
);
const values: string[] = [];
for (const cellMatch of rowMatch[2].matchAll(
/<c\b([^>]*)>([\s\S]*?)<\/c>/g,
)) {
const attributes = cellMatch[1];
const body = cellMatch[2];
const reference = attributes.match(/\br="([A-Z]+\d+)"/i)?.[1] ?? "A1";
const type = attributes.match(/\bt="([^"]+)"/)?.[1] ?? "";
const rawValue = body.match(/<v>([\s\S]*?)<\/v>/)?.[1] ?? "";
let value = "";
if (type === "s") value = sharedStrings[Number(rawValue)] ?? "";
else if (type === "inlineStr") value = textNodes(body);
else if (type === "b") value = rawValue === "1" ? "TRUE" : "FALSE";
else value = decodeXml(rawValue);
values[columnIndex(reference)] = value.trim();
}
while (rows.length < rowNumber - 1) rows.push([]);
rows[rowNumber - 1] = values;
}
return rows;
}
function normalizeHeader(value: string) {
return value.replace(/[\s_\-()]/g, "").toLocaleLowerCase("zh-CN");
}
function isSourceImageHeader(value: string) {
return /^(?:图片|发布配图)\d*$/.test(normalizeHeader(value));
}
function headerAliases(header: (typeof PARTNER_BATCH_HEADERS)[number]) {
const aliases: Record<string, string[]> = {
"序号(不能改)": ["序号(不能改)", "序号"],
: ["标题"],
"笔记内容(正文+话题)": ["笔记内容(正文+话题)", "笔记内容"],
: ["图片", "发布配图"],
: ["发布链接"],
: ["笔记截图", "发布截图"],
"数据分析截图(单篇笔记数据分析截图)": [
"数据分析截图(单篇笔记数据分析截图)",
"数据分析截图",
"创作者中心截图",
],
_系统笔记ID: ["_系统笔记ID", "系统笔记ID"],
_原笔记截图: ["_原笔记截图"],
_原数据分析截图: ["_原数据分析截图"],
};
return aliases[header] ?? [header];
}
function findHeader(rows: string[][]) {
for (let rowIndex = 0; rowIndex < Math.min(rows.length, 8); rowIndex += 1) {
const mapping = new Map<(typeof PARTNER_BATCH_HEADERS)[number], number>();
rows[rowIndex].forEach((value, column) => {
for (const header of PARTNER_BATCH_HEADERS) {
if (header === "图片" && isSourceImageHeader(value)) {
if (!mapping.has(header)) mapping.set(header, column);
break;
}
if (
headerAliases(header).some(
(alias) => normalizeHeader(alias) === normalizeHeader(value),
)
) {
mapping.set(header, column);
break;
}
}
});
if (
mapping.has("序号(不能改)") &&
mapping.has("标题") &&
mapping.has("发布链接") &&
mapping.has("笔记截图") &&
mapping.has("数据分析截图(单篇笔记数据分析截图)") &&
mapping.has("_系统笔记ID")
) {
return { rowIndex, mapping };
}
}
return null;
}
function relationshipMap(xml: string) {
const relationships = new Map<string, string>();
for (const match of xml.matchAll(/<Relationship\b([^>]*)\/?\s*>/g)) {
const attributes = match[1];
const id = attributes.match(/\bId="([^"]+)"/)?.[1] ?? "";
const target = attributes.match(/\bTarget="([^"]+)"/)?.[1] ?? "";
if (id && target) relationships.set(id, xmlAttribute(target));
}
return relationships;
}
function contentType(bytes: Uint8Array, fileName: string) {
if (bytes[0] === 0x89 && bytes[1] === 0x50) return "image/png";
if (bytes[0] === 0xff && bytes[1] === 0xd8) return "image/jpeg";
if (bytes[0] === 0x47 && bytes[1] === 0x49 && bytes[2] === 0x46) {
return "image/gif";
}
if (String.fromCharCode(...bytes.slice(8, 12)) === "WEBP") return "image/webp";
const extension = path.extname(fileName).toLowerCase();
return extension === ".png"
? "image/png"
: extension === ".gif"
? "image/gif"
: extension === ".webp"
? "image/webp"
: "image/jpeg";
}
function resolveZipPath(base: string, target: string) {
return path.posix.normalize(path.posix.join(path.posix.dirname(base), target));
}
function parseDrawingImages(entries: Record<string, Uint8Array>) {
const images = new Map<string, PartnerBatchImage>();
const sheetRelationshipsXml = entries["xl/worksheets/_rels/sheet1.xml.rels"]
? strFromU8(entries["xl/worksheets/_rels/sheet1.xml.rels"])
: "";
const sheetRelationships = relationshipMap(sheetRelationshipsXml);
const sheetXml = entries["xl/worksheets/sheet1.xml"]
? strFromU8(entries["xl/worksheets/sheet1.xml"])
: "";
const drawingId = sheetXml.match(/<drawing\b[^>]*r:id="([^"]+)"/)?.[1] ?? "";
const drawingTarget = sheetRelationships.get(drawingId);
if (!drawingTarget) return images;
const drawingPath = resolveZipPath("xl/worksheets/sheet1.xml", drawingTarget);
const drawingXml = entries[drawingPath] ? strFromU8(entries[drawingPath]) : "";
const drawingRelationshipsPath = path.posix.join(
path.posix.dirname(drawingPath),
"_rels",
`${path.posix.basename(drawingPath)}.rels`,
);
const drawingRelationships = relationshipMap(
entries[drawingRelationshipsPath]
? strFromU8(entries[drawingRelationshipsPath])
: "",
);
for (const anchor of drawingXml.matchAll(
/<xdr:(?:oneCellAnchor|twoCellAnchor)\b[^>]*>[\s\S]*?<xdr:from>([\s\S]*?)<\/xdr:from>[\s\S]*?<a:blip\b[^>]*r:embed="([^"]+)"[\s\S]*?<\/xdr:(?:oneCellAnchor|twoCellAnchor)>/g,
)) {
const column = Number(anchor[1].match(/<xdr:col>(\d+)<\/xdr:col>/)?.[1]);
const row = Number(anchor[1].match(/<xdr:row>(\d+)<\/xdr:row>/)?.[1]);
const mediaTarget = drawingRelationships.get(anchor[2]);
if (!Number.isInteger(column) || !Number.isInteger(row) || !mediaTarget) continue;
const mediaPath = resolveZipPath(drawingPath, mediaTarget);
const bytes = entries[mediaPath];
if (!bytes) continue;
images.set(`${row + 1}:${column}`, {
bytes,
contentType: contentType(bytes, mediaPath),
fileName: path.posix.basename(mediaPath),
});
}
return images;
}
function parseRichValueImages(entries: Record<string, Uint8Array>) {
const images = new Map<string, PartnerBatchImage>();
const worksheetXml = entries["xl/worksheets/sheet1.xml"]
? strFromU8(entries["xl/worksheets/sheet1.xml"])
: "";
const metadataXml = entries["xl/metadata.xml"]
? strFromU8(entries["xl/metadata.xml"])
: "";
const richValueXml = entries["xl/richData/rdrichvalue.xml"]
? strFromU8(entries["xl/richData/rdrichvalue.xml"])
: "";
const richValueRelXml = entries["xl/richData/richValueRel.xml"]
? strFromU8(entries["xl/richData/richValueRel.xml"])
: "";
const richValueRelRelationships = relationshipMap(
entries["xl/richData/_rels/richValueRel.xml.rels"]
? strFromU8(entries["xl/richData/_rels/richValueRel.xml.rels"])
: "",
);
if (
!worksheetXml ||
!metadataXml ||
!richValueXml ||
!richValueRelXml ||
!richValueRelRelationships.size
) {
return images;
}
const valueMetadataXml =
metadataXml.match(/<valueMetadata\b[^>]*>([\s\S]*?)<\/valueMetadata>/)?.[1] ??
"";
const metadataToRichValue = [
...valueMetadataXml.matchAll(/<bk\b[^>]*>[\s\S]*?<rc\b[^>]*\bv="(\d+)"[^>]*\/>[\s\S]*?<\/bk>/g),
].map((match) => Number(match[1]));
const richValueToRelationship = [
...richValueXml.matchAll(/<rv\b[^>]*>([\s\S]*?)<\/rv>/g),
].map((match) => Number(match[1].match(/<v>(\d+)<\/v>/)?.[1] ?? -1));
const relationshipIds = [
...richValueRelXml.matchAll(/<rel\b[^>]*\br:id="([^"]+)"[^>]*\/>/g),
].map((match) => match[1]);
for (const cell of worksheetXml.matchAll(
/<c\b([^>]*)>[\s\S]*?<\/c>/g,
)) {
const reference = cell[1].match(/\br="([A-Z]+\d+)"/i)?.[1] ?? "";
const metadataIndex = Number(cell[1].match(/\bvm="(\d+)"/)?.[1] ?? 0);
const row = Number(reference.match(/\d+$/)?.[0] ?? 0);
const column = columnIndex(reference);
if (!reference || !metadataIndex || !row) continue;
const richValueIndex = metadataToRichValue[metadataIndex - 1];
const relationshipIndex = richValueToRelationship[richValueIndex];
const relationshipId = relationshipIds[relationshipIndex];
const mediaTarget = richValueRelRelationships.get(relationshipId);
if (!mediaTarget) continue;
const mediaPath = resolveZipPath(
"xl/richData/richValueRel.xml",
mediaTarget,
);
const bytes = entries[mediaPath];
if (!bytes) continue;
images.set(`${row}:${column}`, {
bytes,
contentType: contentType(bytes, mediaPath),
fileName: path.posix.basename(mediaPath),
});
}
return images;
}
function parseWpsCellImages(entries: Record<string, Uint8Array>) {
const images = new Map<string, PartnerBatchImage>();
const worksheetXml = entries["xl/worksheets/sheet1.xml"]
? strFromU8(entries["xl/worksheets/sheet1.xml"])
: "";
const cellImagesXml = entries["xl/cellimages.xml"]
? strFromU8(entries["xl/cellimages.xml"])
: "";
const relationships = relationshipMap(
entries["xl/_rels/cellimages.xml.rels"]
? strFromU8(entries["xl/_rels/cellimages.xml.rels"])
: "",
);
if (!worksheetXml || !cellImagesXml || !relationships.size) return images;
const imageIdToRelationship = new Map<string, string>();
for (const match of cellImagesXml.matchAll(
/<(?:etc:)?cellImage\b[^>]*>([\s\S]*?)<\/(?:etc:)?cellImage>/g,
)) {
const imageId = match[1].match(
/<(?:xdr:)?cNvPr\b[^>]*\bname="([^"]+)"/,
)?.[1];
const relationshipId = match[1].match(
/<(?:a:)?blip\b[^>]*\br:embed="([^"]+)"/,
)?.[1];
if (imageId && relationshipId) {
imageIdToRelationship.set(imageId, relationshipId);
}
}
for (const cell of worksheetXml.matchAll(/<c\b([^>]*)>([\s\S]*?)<\/c>/g)) {
const reference = cell[1].match(/\br="([A-Z]+\d+)"/i)?.[1] ?? "";
const imageId = decodeXml(cell[2]).match(/DISPIMG\("([^"]+)"/i)?.[1];
if (!reference || !imageId) continue;
const relationshipId = imageIdToRelationship.get(imageId);
const mediaTarget = relationshipId
? relationships.get(relationshipId)
: undefined;
if (!mediaTarget) continue;
const mediaPath = resolveZipPath("xl/cellimages.xml", mediaTarget);
const bytes = entries[mediaPath];
if (!bytes) continue;
const row = Number(reference.match(/\d+$/)?.[0] ?? 0);
if (!row) continue;
images.set(`${row}:${columnIndex(reference)}`, {
bytes,
contentType: contentType(bytes, mediaPath),
fileName: path.posix.basename(mediaPath),
});
}
return images;
}
function parseImages(entries: Record<string, Uint8Array>) {
const images = parseDrawingImages(entries);
for (const [cell, image] of parseRichValueImages(entries)) {
images.set(cell, image);
}
for (const [cell, image] of parseWpsCellImages(entries)) {
images.set(cell, image);
}
return images;
}
function valueAt(
row: string[],
mapping: Map<(typeof PARTNER_BATCH_HEADERS)[number], number>,
header: (typeof PARTNER_BATCH_HEADERS)[number],
) {
const column = mapping.get(header);
return column === undefined ? "" : String(row[column] ?? "").trim();
}
export function parsePartnerBatchWorkbook(input: ArrayBuffer | Uint8Array) {
const bytes = input instanceof Uint8Array ? input : new Uint8Array(input);
if (bytes.byteLength > PARTNER_BATCH_MAX_BYTES) {
throw new Error("批量回填表不能超过80MB");
}
const entries = unzipSync(bytes);
const worksheetBytes = entries["xl/worksheets/sheet1.xml"];
if (!worksheetBytes) throw new Error("Excel 中没有可读取的批量回填工作表");
const sharedXml = entries["xl/sharedStrings.xml"]
? strFromU8(entries["xl/sharedStrings.xml"])
: "";
const sharedStrings = [
...sharedXml.matchAll(/<si\b[^>]*>([\s\S]*?)<\/si>/g),
].map((match) => textNodes(match[1]));
const rows = parseWorksheet(strFromU8(worksheetBytes), sharedStrings);
const header = findHeader(rows);
if (!header) {
throw new Error("表格结构不正确,请使用本领取页面导出的批量回填表");
}
const images = parseImages(entries);
const publishScreenshotColumn = header.mapping.get("笔记截图")!;
const creatorScreenshotColumn = header.mapping.get(
"数据分析截图(单篇笔记数据分析截图)",
)!;
const result: PartnerBatchImportRow[] = [];
for (let index = header.rowIndex + 1; index < rows.length; index += 1) {
const row = rows[index];
const distributionId = valueAt(row, header.mapping, "_系统笔记ID");
if (!distributionId && !row.some((value) => String(value ?? "").trim())) continue;
result.push({
spreadsheetRow: index + 1,
sequence: valueAt(row, header.mapping, "序号(不能改)"),
title: valueAt(row, header.mapping, "标题"),
publishUrl: valueAt(row, header.mapping, "发布链接"),
distributionId,
originalPublishScreenshotKey: valueAt(
row,
header.mapping,
"_原笔记截图",
),
originalCreatorScreenshotKey: valueAt(
row,
header.mapping,
"_原数据分析截图",
),
publishScreenshot:
images.get(`${index + 1}:${publishScreenshotColumn}`) ?? null,
creatorScreenshot:
images.get(`${index + 1}:${creatorScreenshotColumn}`) ?? null,
});
}
if (result.length === 0) throw new Error("表格中没有可回填的笔记");
return result;
}