feat: ship self-hosted KOC LOOP workflows
This commit is contained in:
巫凤萍
24
scripts/check-mysql.mjs
Normal file
24
scripts/check-mysql.mjs
Normal file
@@ -0,0 +1,24 @@
|
||||
import mysql from "mysql2/promise";
|
||||
|
||||
function databaseUrl() {
|
||||
if (process.env.DATABASE_URL) return process.env.DATABASE_URL;
|
||||
const host = process.env.MYSQL_HOST ?? "127.0.0.1";
|
||||
const port = process.env.MYSQL_PORT ?? "3306";
|
||||
const user = encodeURIComponent(process.env.MYSQL_USER ?? "koc");
|
||||
const password = encodeURIComponent(process.env.MYSQL_PASSWORD ?? "");
|
||||
const database = process.env.MYSQL_DATABASE ?? "koc_loop";
|
||||
return `mysql://${user}:${password}@${host}:${port}/${database}`;
|
||||
}
|
||||
const connection = await mysql.createConnection({
|
||||
uri: databaseUrl(),
|
||||
charset: "utf8mb4",
|
||||
timezone: "Z",
|
||||
});
|
||||
try {
|
||||
const [rows] = await connection.query(
|
||||
"SELECT COUNT(*) AS table_count FROM information_schema.tables WHERE table_schema = DATABASE()",
|
||||
);
|
||||
console.info(`database ready, tables=${Number(rows[0]?.table_count ?? 0)}`);
|
||||
} finally {
|
||||
await connection.end();
|
||||
}
|
||||
103
scripts/import-d1-json.mjs
Normal file
103
scripts/import-d1-json.mjs
Normal file
@@ -0,0 +1,103 @@
|
||||
import { readFile } from "node:fs/promises";
|
||||
import mysql from "mysql2/promise";
|
||||
|
||||
const allowedTables = [
|
||||
"partners",
|
||||
"tasks",
|
||||
"contents",
|
||||
"accounts",
|
||||
"claims",
|
||||
"delegation_bundles",
|
||||
"distributions",
|
||||
"collection_runs",
|
||||
"users",
|
||||
"auth_sessions",
|
||||
"mcp_export_tokens",
|
||||
];
|
||||
|
||||
function databaseUrl() {
|
||||
if (process.env.DATABASE_URL) return process.env.DATABASE_URL;
|
||||
const host = process.env.MYSQL_HOST ?? "127.0.0.1";
|
||||
const port = process.env.MYSQL_PORT ?? "3306";
|
||||
const user = encodeURIComponent(process.env.MYSQL_USER ?? "koc");
|
||||
const password = encodeURIComponent(process.env.MYSQL_PASSWORD ?? "");
|
||||
const database = process.env.MYSQL_DATABASE ?? "koc_loop";
|
||||
return `mysql://${user}:${password}@${host}:${port}/${database}`;
|
||||
}
|
||||
|
||||
function normalizeImportedValue(value) {
|
||||
if (
|
||||
typeof value === "string" &&
|
||||
/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,3})?Z$/.test(value)
|
||||
) {
|
||||
return value.replace("T", " ").replace(/Z$/, "");
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
const inputPath = process.argv[2];
|
||||
if (!inputPath) {
|
||||
throw new Error("用法：npm run db:import-json -- /path/to/d1-export.json");
|
||||
}
|
||||
const parsed = JSON.parse(await readFile(inputPath, "utf8"));
|
||||
const tables = parsed.tables && typeof parsed.tables === "object"
|
||||
? parsed.tables
|
||||
: parsed;
|
||||
const pool = mysql.createPool({
|
||||
uri: databaseUrl(),
|
||||
connectionLimit: 2,
|
||||
charset: "utf8mb4",
|
||||
timezone: "Z",
|
||||
dateStrings: true,
|
||||
});
|
||||
|
||||
try {
|
||||
for (const table of allowedTables) {
|
||||
const rows = Array.isArray(tables[table]) ? tables[table] : [];
|
||||
if (rows.length === 0) continue;
|
||||
const [columnRows] = await pool.query(
|
||||
`SELECT column_name, extra FROM information_schema.columns
|
||||
WHERE table_schema = DATABASE() AND table_name = ? ORDER BY ordinal_position`,
|
||||
[table],
|
||||
);
|
||||
const allowedColumns = new Set(
|
||||
columnRows
|
||||
.filter((column) => !String(column.extra ?? "").includes("GENERATED"))
|
||||
.map((column) => String(column.column_name)),
|
||||
);
|
||||
const columns = [...new Set(rows.flatMap((row) => Object.keys(row)))]
|
||||
.filter((column) => allowedColumns.has(column));
|
||||
if (columns.length === 0) continue;
|
||||
const quotedColumns = columns.map((column) => `\`${column}\``).join(", ");
|
||||
const updates = columns
|
||||
.filter((column) => column !== "id" && column !== "token_hash")
|
||||
.map((column) => `\`${column}\` = VALUES(\`${column}\`)`)
|
||||
.join(", ");
|
||||
const connection = await pool.getConnection();
|
||||
try {
|
||||
await connection.beginTransaction();
|
||||
for (let offset = 0; offset < rows.length; offset += 200) {
|
||||
const chunk = rows.slice(offset, offset + 200);
|
||||
const placeholders = chunk
|
||||
.map(() => `(${columns.map(() => "?").join(", ")})`)
|
||||
.join(", ");
|
||||
const sql = `INSERT INTO \`${table}\` (${quotedColumns}) VALUES ${placeholders}${
|
||||
updates ? ` ON DUPLICATE KEY UPDATE ${updates}` : ""
|
||||
}`;
|
||||
const values = chunk.flatMap((row) =>
|
||||
columns.map((column) => normalizeImportedValue(row[column] ?? null)),
|
||||
);
|
||||
await connection.query(sql, values);
|
||||
}
|
||||
await connection.commit();
|
||||
console.info(`imported ${table}: ${rows.length}`);
|
||||
} catch (error) {
|
||||
await connection.rollback();
|
||||
throw error;
|
||||
} finally {
|
||||
connection.release();
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await pool.end();
|
||||
}
|
||||
59
scripts/import-object-directory.mjs
Normal file
59
scripts/import-object-directory.mjs
Normal file
@@ -0,0 +1,59 @@
|
||||
import { copyFile, mkdir, readdir, readFile, writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
|
||||
const sourceRoot = process.argv[2];
|
||||
const targetRoot = process.env.UPLOAD_DIR || process.argv[3];
|
||||
if (!sourceRoot || !targetRoot) {
|
||||
throw new Error(
|
||||
"用法：UPLOAD_DIR=/data/koc/uploads npm run storage:import -- /path/to/r2-export",
|
||||
);
|
||||
}
|
||||
|
||||
const contentTypes = {
|
||||
".avif": "image/avif",
|
||||
".gif": "image/gif",
|
||||
".jpeg": "image/jpeg",
|
||||
".jpg": "image/jpeg",
|
||||
".png": "image/png",
|
||||
".webp": "image/webp",
|
||||
};
|
||||
|
||||
let copied = 0;
|
||||
async function walk(directory) {
|
||||
const entries = await readdir(directory, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
const absolute = path.join(directory, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
await walk(absolute);
|
||||
continue;
|
||||
}
|
||||
if (entry.name.endsWith(".metadata.json")) continue;
|
||||
const relative = path.relative(sourceRoot, absolute);
|
||||
if (relative.startsWith("..") || path.isAbsolute(relative)) {
|
||||
throw new Error("导入目录越界");
|
||||
}
|
||||
const destination = path.join(targetRoot, relative);
|
||||
await mkdir(path.dirname(destination), { recursive: true });
|
||||
await copyFile(absolute, destination);
|
||||
const sourceMetadata = `${absolute}.metadata.json`;
|
||||
const metadataDestination = `${destination}.metadata.json`;
|
||||
let metadata;
|
||||
try {
|
||||
metadata = JSON.parse(await readFile(sourceMetadata, "utf8"));
|
||||
} catch {
|
||||
metadata = {
|
||||
contentType:
|
||||
contentTypes[path.extname(entry.name).toLowerCase()] ||
|
||||
"application/octet-stream",
|
||||
customMetadata: { source: "r2-export" },
|
||||
uploadedAt: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
await writeFile(metadataDestination, JSON.stringify(metadata), "utf8");
|
||||
copied += 1;
|
||||
}
|
||||
}
|
||||
|
||||
await mkdir(targetRoot, { recursive: true });
|
||||
await walk(sourceRoot);
|
||||
console.info(`imported objects: ${copied}`);
|
||||
63
scripts/migrate-mysql.mjs
Normal file
63
scripts/migrate-mysql.mjs
Normal file
@@ -0,0 +1,63 @@
|
||||
import { readFile, readdir } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import mysql from "mysql2/promise";
|
||||
|
||||
function databaseUrl() {
|
||||
if (process.env.DATABASE_URL) return process.env.DATABASE_URL;
|
||||
const host = process.env.MYSQL_HOST ?? "127.0.0.1";
|
||||
const port = process.env.MYSQL_PORT ?? "3306";
|
||||
const user = encodeURIComponent(process.env.MYSQL_USER ?? "koc");
|
||||
const password = encodeURIComponent(process.env.MYSQL_PASSWORD ?? "");
|
||||
const database = process.env.MYSQL_DATABASE ?? "koc_loop";
|
||||
return `mysql://${user}:${password}@${host}:${port}/${database}`;
|
||||
}
|
||||
|
||||
const pool = mysql.createPool({
|
||||
uri: databaseUrl(),
|
||||
connectionLimit: 2,
|
||||
charset: "utf8mb4",
|
||||
timezone: "Z",
|
||||
});
|
||||
|
||||
const connection = await pool.getConnection();
|
||||
try {
|
||||
const [lockRows] = await connection.query(
|
||||
"SELECT GET_LOCK('koc-loop-schema-migration', 60) AS acquired",
|
||||
);
|
||||
if (Number(lockRows[0]?.acquired ?? 0) !== 1) {
|
||||
throw new Error("无法获取数据库迁移锁");
|
||||
}
|
||||
await connection.query(`CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||
version VARCHAR(255) PRIMARY KEY,
|
||||
applied_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci`);
|
||||
|
||||
const migrationDir = path.join(process.cwd(), "mysql");
|
||||
const files = (await readdir(migrationDir))
|
||||
.filter((file) => file.endsWith(".sql"))
|
||||
.sort();
|
||||
for (const file of files) {
|
||||
const [rows] = await connection.query(
|
||||
"SELECT version FROM schema_migrations WHERE version = ?",
|
||||
[file],
|
||||
);
|
||||
if (rows.length > 0) continue;
|
||||
const source = await readFile(path.join(migrationDir, file), "utf8");
|
||||
const statements = source
|
||||
.split(/^-- statement-breakpoint\s*$/m)
|
||||
.map((statement) => statement.trim())
|
||||
.filter(Boolean);
|
||||
for (const statement of statements) {
|
||||
await connection.query(statement);
|
||||
}
|
||||
await connection.query(
|
||||
"INSERT INTO schema_migrations (version) VALUES (?)",
|
||||
[file],
|
||||
);
|
||||
console.info(`applied ${file}`);
|
||||
}
|
||||
await connection.query("SELECT RELEASE_LOCK('koc-loop-schema-migration')");
|
||||
} finally {
|
||||
connection.release();
|
||||
await pool.end();
|
||||
}
|
||||
Reference in New Issue
Block a user