feat: backup mysql and s3storage with script
This commit is contained in:
Methapon2001
parent
3066ec3996
commit
63846f57bd
2 changed files with 983 additions and 54 deletions
|
|
@ -1,36 +1,43 @@
|
|||
import { Body, Controller, Delete, Get, Post, Route, Security } from "tsoa";
|
||||
import { Client as MinioClient, BucketItem } from "minio";
|
||||
import HttpError from "../interfaces/http-error";
|
||||
import HttpStatus from "../interfaces/http-status";
|
||||
|
||||
function getEnvVar(name: string) {
|
||||
const value = process.env[name];
|
||||
if (!value) throw new Error(`${name} is required.`);
|
||||
return value;
|
||||
function getEnvVar(environmentName: string) {
|
||||
const environmentValue = process.env[environmentName];
|
||||
if (!environmentValue) throw new Error(`${environmentName} is required.`);
|
||||
return environmentValue;
|
||||
}
|
||||
|
||||
const WINDMILL_URL = getEnvVar("WINDMILL_URL");
|
||||
const WINDMILL_WORKSPACE = getEnvVar("WINDMILL_WORKSPACE");
|
||||
const WINDMILL_BACKUP_SCRIPT_PATH = getEnvVar("WINDMILL_BACKUP_SCRIPT_PATH");
|
||||
const WINDMILL_RESTORE_SCRIPT_PATH = getEnvVar("WINDMILL_RESTORE_SCRIPT_PATH");
|
||||
const WINDMILL_BACKUP_FLOW_PATH = getEnvVar("WINDMILL_BACKUP_FLOW_PATH");
|
||||
const WINDMILL_RESTORE_FLOW_PATH = getEnvVar("WINDMILL_RESTORE_FLOW_PATH");
|
||||
const WINDMILL_BACKUP_DELTE_SCRIPT_PATH = getEnvVar("WINDMILL_BACKUP_DELTE_SCRIPT_PATH");
|
||||
const WINDMILL_API_KEY = getEnvVar("WINDMILL_API_KEY");
|
||||
const DB_HOST = getEnvVar("DB_HOST");
|
||||
const DB_PORT = process.env.DB_PORT;
|
||||
const DB_USERNAME = getEnvVar("DB_USERNAME");
|
||||
const DB_PASSWORD = getEnvVar("DB_PASSWORD");
|
||||
const MINIO_USE_SSL = getEnvVar("MINIO_USE_SSL");
|
||||
const MINIO_HOST = getEnvVar("MINIO_HOST");
|
||||
const MINIO_PORT = process.env.MINIO_PORT;
|
||||
const MINIO_ACCESS_KEY = getEnvVar("MINIO_ACCESS_KEY");
|
||||
const MINIO_SECRET_KEY = getEnvVar("MINIO_SECRET_KEY");
|
||||
const MINIO_BUCKET = getEnvVar("MINIO_BUCKET");
|
||||
const MINIO_BACKUP_FILE_PREFIX =
|
||||
process.env.MINIO_BACKUP_FILE_PREFIX?.split("/").filter(Boolean).join("/").concat("/") || "";
|
||||
const MAIN_MINIO_USE_SSL = getEnvVar("MAIN_MINIO_USE_SSL");
|
||||
const MAIN_MINIO_HOST = getEnvVar("MAIN_MINIO_HOST");
|
||||
const MAIN_MINIO_PORT = process.env.MAIN_MINIO_PORT;
|
||||
const MAIN_MINIO_ACCESS_KEY = getEnvVar("MAIN_MINIO_ACCESS_KEY");
|
||||
const MAIN_MINIO_SECRET_KEY = getEnvVar("MAIN_MINIO_SECRET_KEY");
|
||||
const MAIN_MINIO_BUCKET = getEnvVar("MAIN_MINIO_BUCKET");
|
||||
const BACKUP_MINIO_USE_SSL = getEnvVar("BACKUP_MINIO_USE_SSL");
|
||||
const BACKUP_MINIO_HOST = getEnvVar("BACKUP_MINIO_HOST");
|
||||
const BACKUP_MINIO_PORT = process.env.BACKUP_MINIO_PORT;
|
||||
const BACKUP_MINIO_ACCESS_KEY = getEnvVar("BACKUP_MINIO_ACCESS_KEY");
|
||||
const BACKUP_MINIO_SECRET_KEY = getEnvVar("BACKUP_MINIO_SECRET_KEY");
|
||||
const BACKUP_MINIO_BUCKET = getEnvVar("BACKUP_MINIO_BUCKET");
|
||||
|
||||
const minio = new MinioClient({
|
||||
useSSL: MINIO_USE_SSL === "true",
|
||||
endPoint: MINIO_HOST,
|
||||
port: +(MINIO_PORT || "9000"),
|
||||
accessKey: MINIO_ACCESS_KEY,
|
||||
secretKey: MINIO_SECRET_KEY,
|
||||
useSSL: BACKUP_MINIO_USE_SSL === "true",
|
||||
endPoint: BACKUP_MINIO_HOST,
|
||||
port: +(BACKUP_MINIO_PORT || "9000"),
|
||||
accessKey: BACKUP_MINIO_ACCESS_KEY,
|
||||
secretKey: BACKUP_MINIO_SECRET_KEY,
|
||||
});
|
||||
|
||||
@Route("/api/v1/backup")
|
||||
|
|
@ -40,7 +47,7 @@ export class BackupController extends Controller {
|
|||
async listBackup() {
|
||||
return await new Promise((resolve, reject) => {
|
||||
const data: BucketItem[] = [];
|
||||
const stream = minio.listObjectsV2(MINIO_BUCKET, MINIO_BACKUP_FILE_PREFIX);
|
||||
const stream = minio.listObjectsV2(BACKUP_MINIO_BUCKET);
|
||||
stream.on("data", (obj) => data.unshift(obj));
|
||||
stream.on("end", () =>
|
||||
resolve(
|
||||
|
|
@ -48,7 +55,7 @@ export class BackupController extends Controller {
|
|||
"prefix" in v
|
||||
? []
|
||||
: {
|
||||
name: v.name.replace(MINIO_BACKUP_FILE_PREFIX || "", ""),
|
||||
name: v.name.replace(".sql.gz", ""),
|
||||
timestamp: v.lastModified,
|
||||
},
|
||||
),
|
||||
|
|
@ -61,7 +68,7 @@ export class BackupController extends Controller {
|
|||
@Get("backup-running-list")
|
||||
async runningBackupStatus() {
|
||||
return await fetch(
|
||||
`${WINDMILL_URL}/api/w/${WINDMILL_WORKSPACE}/jobs/list?running=true&script_path_exact=${WINDMILL_BACKUP_SCRIPT_PATH}`,
|
||||
`${WINDMILL_URL}/api/w/${WINDMILL_WORKSPACE}/jobs/list?running=true&script_path_exact=${WINDMILL_BACKUP_FLOW_PATH}`,
|
||||
{
|
||||
headers: { Authorization: `Bearer ${WINDMILL_API_KEY}` },
|
||||
},
|
||||
|
|
@ -69,16 +76,16 @@ export class BackupController extends Controller {
|
|||
const data = await r.json();
|
||||
if (typeof data === "object" && "error" in data) {
|
||||
console.error(data);
|
||||
throw new Error("Backup Error");
|
||||
throw new Error("Cannot get status.");
|
||||
}
|
||||
return data;
|
||||
return data as Record<string, any>[];
|
||||
});
|
||||
}
|
||||
|
||||
@Get("restore-running-list")
|
||||
async runningRestoreStatus() {
|
||||
return await fetch(
|
||||
`${WINDMILL_URL}/api/w/${WINDMILL_WORKSPACE}/jobs/list?running=true&script_path_exact=${WINDMILL_RESTORE_SCRIPT_PATH}`,
|
||||
`${WINDMILL_URL}/api/w/${WINDMILL_WORKSPACE}/jobs/list?running=true&script_path_exact=${WINDMILL_RESTORE_FLOW_PATH}`,
|
||||
{
|
||||
headers: { Authorization: `Bearer ${WINDMILL_API_KEY}` },
|
||||
},
|
||||
|
|
@ -86,16 +93,28 @@ export class BackupController extends Controller {
|
|||
const data = await r.json();
|
||||
if (typeof data === "object" && "error" in data) {
|
||||
console.error(data);
|
||||
throw new Error("Backup Error");
|
||||
throw new Error("Cannot get status.");
|
||||
}
|
||||
return data;
|
||||
});
|
||||
}
|
||||
|
||||
@Post("create")
|
||||
async runBackup() {
|
||||
async runBackup(@Body() body?: { name?: string }) {
|
||||
const timestamp = Math.round(Date.now() / 1000);
|
||||
const name =
|
||||
body?.name && body.name !== "auto-backup"
|
||||
? `${timestamp}-${body.name}`
|
||||
: `${timestamp}-manual`;
|
||||
|
||||
const listRunning = await this.runningBackupStatus();
|
||||
|
||||
if (!listRunning || listRunning.length > 0) {
|
||||
throw new HttpError(HttpStatus.NOT_ACCEPTABLE, "Cannot create two backup at the same time.");
|
||||
}
|
||||
|
||||
return await fetch(
|
||||
`${WINDMILL_URL}/api/w/${WINDMILL_WORKSPACE}/jobs/run/p/${WINDMILL_BACKUP_SCRIPT_PATH}`,
|
||||
`${WINDMILL_URL}/api/w/${WINDMILL_WORKSPACE}/jobs/run/f/${WINDMILL_BACKUP_FLOW_PATH}`,
|
||||
{
|
||||
method: "POST",
|
||||
headers: {
|
||||
|
|
@ -103,17 +122,26 @@ export class BackupController extends Controller {
|
|||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
s3_endpoint: `${MINIO_USE_SSL === "true" ? "https" : "http"}://${MINIO_HOST}${(MINIO_PORT && ":" + MINIO_PORT) || ""}`,
|
||||
s3_access: MINIO_ACCESS_KEY,
|
||||
s3_secret: MINIO_SECRET_KEY,
|
||||
s3_bucket: MINIO_BUCKET,
|
||||
s3_prefix: MINIO_BACKUP_FILE_PREFIX || "/",
|
||||
db_host: DB_HOST,
|
||||
db_port: DB_PORT,
|
||||
db_user: DB_USERNAME,
|
||||
db_password: DB_PASSWORD,
|
||||
backup_filename: "manual",
|
||||
num_versions_to_keep: 0,
|
||||
backup_name: name,
|
||||
storage: {
|
||||
s3_source_endpoint: `${MAIN_MINIO_USE_SSL === "true" ? "https://" : "http://"}${MAIN_MINIO_HOST}${(MAIN_MINIO_PORT && ":" + MAIN_MINIO_PORT) || ""}`,
|
||||
s3_source_access: MAIN_MINIO_ACCESS_KEY,
|
||||
s3_source_secret: MAIN_MINIO_SECRET_KEY,
|
||||
s3_source_bucket: MAIN_MINIO_BUCKET,
|
||||
s3_dest_endpoint: `${BACKUP_MINIO_USE_SSL === "true" ? "https" : "http://"}${BACKUP_MINIO_HOST}${(BACKUP_MINIO_PORT && ":" + BACKUP_MINIO_PORT) || ""}`,
|
||||
s3_dest_access: BACKUP_MINIO_ACCESS_KEY,
|
||||
s3_dest_secret: BACKUP_MINIO_SECRET_KEY,
|
||||
},
|
||||
database: {
|
||||
s3_endpoint: `${BACKUP_MINIO_USE_SSL === "true" ? "https" : "http://"}${BACKUP_MINIO_HOST}${(BACKUP_MINIO_PORT && ":" + BACKUP_MINIO_PORT) || ""}`,
|
||||
s3_access: BACKUP_MINIO_ACCESS_KEY,
|
||||
s3_secret: BACKUP_MINIO_SECRET_KEY,
|
||||
s3_bucket: BACKUP_MINIO_BUCKET,
|
||||
db_host: DB_HOST,
|
||||
db_port: DB_PORT,
|
||||
db_user: DB_USERNAME,
|
||||
db_password: DB_PASSWORD,
|
||||
},
|
||||
}),
|
||||
},
|
||||
).then(async (r) => {
|
||||
|
|
@ -127,9 +155,15 @@ export class BackupController extends Controller {
|
|||
}
|
||||
|
||||
@Post("restore")
|
||||
async restoreBackup(@Body() body: { filename: string }) {
|
||||
async restoreBackup(@Body() body: { name: string }) {
|
||||
const listRunning = await this.runningRestoreStatus();
|
||||
|
||||
if (!listRunning || listRunning.length > 0) {
|
||||
throw new HttpError(HttpStatus.NOT_ACCEPTABLE, "Cannot restore two backup at the same time.");
|
||||
}
|
||||
|
||||
return await fetch(
|
||||
`${WINDMILL_URL}/api/w/${WINDMILL_WORKSPACE}/jobs/run/p/${WINDMILL_RESTORE_SCRIPT_PATH}`,
|
||||
`${WINDMILL_URL}/api/w/${WINDMILL_WORKSPACE}/jobs/run/f/${WINDMILL_RESTORE_FLOW_PATH}`,
|
||||
{
|
||||
method: "POST",
|
||||
headers: {
|
||||
|
|
@ -137,16 +171,26 @@ export class BackupController extends Controller {
|
|||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
s3_endpoint: `${MINIO_USE_SSL === "true" ? "https" : "http"}://${MINIO_HOST}${(MINIO_PORT && ":" + MINIO_PORT) || ""}`,
|
||||
s3_access: MINIO_ACCESS_KEY,
|
||||
s3_secret: MINIO_SECRET_KEY,
|
||||
s3_bucket: MINIO_BUCKET,
|
||||
s3_prefix: MINIO_BACKUP_FILE_PREFIX || "/",
|
||||
db_host: DB_HOST,
|
||||
db_port: DB_PORT,
|
||||
db_user: DB_USERNAME,
|
||||
db_password: DB_PASSWORD,
|
||||
restore_filename: body.filename,
|
||||
backup_name: body.name,
|
||||
storage: {
|
||||
s3_restore_endpoint: `${MAIN_MINIO_USE_SSL === "true" ? "https://" : "http://"}${MAIN_MINIO_HOST}${(MAIN_MINIO_PORT && ":" + MAIN_MINIO_PORT) || ""}`,
|
||||
s3_restore_access: MAIN_MINIO_ACCESS_KEY,
|
||||
s3_restore_secret: MAIN_MINIO_SECRET_KEY,
|
||||
s3_restore_bucket: MAIN_MINIO_BUCKET,
|
||||
s3_backup_endpoint: `${BACKUP_MINIO_USE_SSL === "true" ? "https" : "http://"}${BACKUP_MINIO_HOST}${(BACKUP_MINIO_PORT && ":" + BACKUP_MINIO_PORT) || ""}`,
|
||||
s3_backup_access: BACKUP_MINIO_ACCESS_KEY,
|
||||
s3_backup_secret: BACKUP_MINIO_SECRET_KEY,
|
||||
},
|
||||
database: {
|
||||
s3_endpoint: `${BACKUP_MINIO_USE_SSL === "true" ? "https" : "http://"}${BACKUP_MINIO_HOST}${(BACKUP_MINIO_PORT && ":" + BACKUP_MINIO_PORT) || ""}`,
|
||||
s3_access: BACKUP_MINIO_ACCESS_KEY,
|
||||
s3_secret: BACKUP_MINIO_SECRET_KEY,
|
||||
s3_bucket: BACKUP_MINIO_BUCKET,
|
||||
db_host: DB_HOST,
|
||||
db_port: DB_PORT,
|
||||
db_user: DB_USERNAME,
|
||||
db_password: DB_PASSWORD,
|
||||
},
|
||||
}),
|
||||
},
|
||||
).then(async (r) => {
|
||||
|
|
@ -160,9 +204,30 @@ export class BackupController extends Controller {
|
|||
}
|
||||
|
||||
@Delete("delete")
|
||||
async deleteBackup(@Body() body: { filename: string }) {
|
||||
await minio.removeObject(MINIO_BUCKET, MINIO_BACKUP_FILE_PREFIX + body.filename, {
|
||||
forceDelete: true,
|
||||
async deleteBackup(@Body() body: { name: string }) {
|
||||
await fetch(
|
||||
`${WINDMILL_URL}/api/w/${WINDMILL_WORKSPACE}/jobs/run_wait_result/p/${WINDMILL_BACKUP_DELTE_SCRIPT_PATH}`,
|
||||
{
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${WINDMILL_API_KEY}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
backup_name: body.name,
|
||||
s3_backup_endpoint: `${BACKUP_MINIO_USE_SSL === "true" ? "https://" : "http://"}${BACKUP_MINIO_HOST}${(BACKUP_MINIO_PORT && ":" + BACKUP_MINIO_PORT) || ""}`,
|
||||
s3_backup_access: BACKUP_MINIO_ACCESS_KEY,
|
||||
s3_backup_secret: BACKUP_MINIO_SECRET_KEY,
|
||||
s3_backup_bucket: BACKUP_MINIO_BUCKET,
|
||||
}),
|
||||
},
|
||||
).then(async (r) => {
|
||||
const data = await r.text();
|
||||
if (data.includes("error")) {
|
||||
console.error(data);
|
||||
throw new Error("Error delete backup.");
|
||||
}
|
||||
return data;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue