From 8b47aac02ef49cb5a56a42f9edadecfca4adc69b Mon Sep 17 00:00:00 2001 From: Innei Date: Sun, 9 Jan 2022 15:20:41 +0800 Subject: [PATCH] fix: ignore analyzes collection in backup --- src/bootstrap.ts | 1 + src/modules/backup/backup.controller.ts | 6 +- src/modules/backup/backup.service.ts | 62 +++++++++++++++++++- src/processors/helper/helper.cron.service.ts | 47 +++------------ 4 files changed, 73 insertions(+), 43 deletions(-) diff --git a/src/bootstrap.ts b/src/bootstrap.ts index 0240f4d3..b03c225a 100644 --- a/src/bootstrap.ts +++ b/src/bootstrap.ts @@ -18,6 +18,7 @@ async function bootstrap() { const app = await NestFactory.create( AppModule, fastifyApp, + { logger: false }, ) const hosts = Origin.map((host) => new RegExp(host, 'i')) diff --git a/src/modules/backup/backup.controller.ts b/src/modules/backup/backup.controller.ts index 0a143c75..2f9965d5 100644 --- a/src/modules/backup/backup.controller.ts +++ b/src/modules/backup/backup.controller.ts @@ -42,13 +42,13 @@ export class BackupController { @Header('Content-Type', 'application/zip') @HTTPDecorators.Bypass async createNewBackup() { - const buffer = await this.cronService.backupDB({ uploadCOS: false }) - if (typeof buffer == 'undefined') { + const res = await this.backupService.backup() + if (typeof res == 'undefined' || typeof res.buffer === 'undefined') { throw new BadRequestException('请先开启在设置开启备份功能') } const stream = new Readable() - stream.push(buffer) + stream.push(res.buffer) stream.push(null) return stream } diff --git a/src/modules/backup/backup.service.ts b/src/modules/backup/backup.service.ts index 316783f3..ab733d46 100644 --- a/src/modules/backup/backup.service.ts +++ b/src/modules/backup/backup.service.ts @@ -5,22 +5,29 @@ import { Logger, Scope, } from '@nestjs/common' +import { exec } from 'child_process' +import dayjs from 'dayjs' import { existsSync, statSync } from 'fs' import { readdir, readFile, rm, writeFile } from 'fs/promises' import mkdirp from 'mkdirp' import { join, resolve } from 'path' import { Readable } from 'stream' +import { promisify } from 'util' import { MONGO_DB } from '~/app.config' -import { BACKUP_DIR } from '~/constants/path.constant' +import { BACKUP_DIR, DATA_DIR } from '~/constants/path.constant' import { AdminEventsGateway } from '~/processors/gateway/admin/events.gateway' import { EventTypes } from '~/processors/gateway/events.types' import { getFolderSize } from '~/utils/system.util' +import { ConfigsService } from '../configs/configs.service' @Injectable({ scope: Scope.REQUEST }) export class BackupService { private logger: Logger - constructor(private readonly adminGateway: AdminEventsGateway) { + constructor( + private readonly adminGateway: AdminEventsGateway, + private readonly configs: ConfigsService, + ) { this.logger = new Logger(BackupService.name) } @@ -52,6 +59,56 @@ export class BackupService { ) } + async backup() { + const nowStr = dayjs().format('YYYY-MM-DD-HH:mm:ss') + const { backupOptions: configs } = await this.configs.waitForConfigReady() + if (!configs.enable) { + return + } + this.logger.log('--> 备份数据库中') + // 用时间格式命名文件夹 + const dateDir = nowStr + + const backupDirPath = join(BACKUP_DIR, dateDir) + mkdirp.sync(backupDirPath) + try { + await $`mongodump -h ${MONGO_DB.host} --port ${MONGO_DB.port} -d ${MONGO_DB.dbName} --excludeCollection analyzes -o ${backupDirPath} >/dev/null 2>&1` + + // 打包 DB + await promisify(exec)( + `zip -r backup-${dateDir} mx-space/* && rm -rf mx-space`, + { + cwd: backupDirPath, + }, + ) + + // 打包数据目录 + await promisify(exec)( + `rsync -a . ./temp_copy_need --exclude temp_copy_need --exclude backup --exclude log && mv temp_copy_need backup_data && zip -r ${join( + backupDirPath, + `backup-${dateDir}`, + )} ./backup_data && rm -rf backup_data`, + { + cwd: DATA_DIR, + }, + ) + + this.logger.log('--> 备份成功') + } catch (e) { + this.logger.error( + '--> 备份失败, 请确保已安装 zip 或 mongo-tools, mongo-tools 的版本需要与 mongod 版本一致, ' + + e.message, + ) + throw e + } + const path = join(backupDirPath, 'backup-' + dateDir + '.zip') + + return { + buffer: await readFile(path), + path, + } + } + async getFileStream(dirname: string) { const path = this.checkBackupExist(dirname) const stream = new Readable() @@ -70,6 +127,7 @@ export class BackupService { return path } + // TODO 下面两个方法有重复代码 async saveTempBackupByUpload(buffer: Buffer) { const tempDirPath = '/tmp/mx-space/backup' const tempBackupPath = join(tempDirPath, 'backup.zip') diff --git a/src/processors/helper/helper.cron.service.ts b/src/processors/helper/helper.cron.service.ts index ad0c3879..deadfac4 100644 --- a/src/processors/helper/helper.cron.service.ts +++ b/src/processors/helper/helper.cron.service.ts @@ -1,25 +1,20 @@ import { forwardRef, Inject, Injectable, Logger } from '@nestjs/common' import { Cron, CronExpression } from '@nestjs/schedule' -import { exec } from 'child_process' import COS from 'cos-nodejs-sdk-v5' import dayjs from 'dayjs' import { existsSync } from 'fs' -import { readFile, rm, writeFile } from 'fs/promises' +import { rm, writeFile } from 'fs/promises' import mkdirp from 'mkdirp' import { InjectModel } from 'nestjs-typegoose' -import { join } from 'path' -import { promisify } from 'util' -import { $ } from 'zx' -import { MONGO_DB } from '~/app.config' import { CronDescription } from '~/common/decorator/cron-description.decorator' import { RedisItems, RedisKeys } from '~/constants/cache.constant' import { - BACKUP_DIR, LOCAL_BOT_LIST_DATA_FILE_PATH, TEMP_DIR, } from '~/constants/path.constant' import { AggregateService } from '~/modules/aggregate/aggregate.service' import { AnalyzeModel } from '~/modules/analyze/analyze.model' +import { BackupService } from '~/modules/backup/backup.service' import { ConfigsService } from '~/modules/configs/configs.service' import { NoteService } from '~/modules/note/note.service' import { PageService } from '~/modules/page/page.service' @@ -51,6 +46,8 @@ export class CronService { @Inject(forwardRef(() => PageService)) private readonly pageService: PageService, @Inject(forwardRef(() => SearchService)) + @Inject(forwardRef(() => BackupService)) + private readonly backupService: BackupService, private readonly searchService: SearchService, ) { this.logger = new Logger(CronService.name) @@ -82,35 +79,11 @@ export class CronService { @Cron(CronExpression.EVERY_DAY_AT_1AM, { name: 'backupDB' }) @CronDescription('备份 DB 并上传 COS') async backupDB({ uploadCOS = true }: { uploadCOS?: boolean } = {}) { - const { backupOptions: configs } = await this.configs.waitForConfigReady() - if (!configs.enable) { + const backup = await this.backupService.backup() + if (!backup) { + this.logger.log('没有开启备份') return } - this.logger.log('--> 备份数据库中') - - const dateDir = this.nowStr - - const backupDirPath = join(BACKUP_DIR, dateDir) - mkdirp.sync(backupDirPath) - try { - await $`mongodump -h ${MONGO_DB.host} --port ${MONGO_DB.port} -d ${MONGO_DB.dbName} -o ${backupDirPath} >/dev/null 2>&1` - - await promisify(exec)( - `zip -r backup-${dateDir} mx-space/* && rm -rf mx-space`, - { - cwd: backupDirPath, - }, - ) - - this.logger.log('--> 备份成功') - } catch (e) { - this.logger.error( - '--> 备份失败, 请确保已安装 zip 或 mongo-tools, mongo-tools 的版本需要与 mongod 版本一致, ' + - e.message, - ) - throw e - } - // 开始上传 COS process.nextTick(async () => { if (!uploadCOS) { @@ -126,7 +99,7 @@ export class CronService { ) { return } - const backupFilePath = join(backupDirPath, 'backup-' + dateDir + '.zip') + const backupFilePath = backup.path if (!existsSync(backupFilePath)) { this.logger.warn('文件不存在, 无法上传到 COS') @@ -142,7 +115,7 @@ export class CronService { { Bucket: backupOptions.bucket, Region: backupOptions.region, - Key: `backup-${dateDir}.zip`, + Key: backup.path.slice(backup.path.lastIndexOf('/') + 1), FilePath: backupFilePath, }, (err) => { @@ -155,8 +128,6 @@ export class CronService { }, ) }) - - return readFile(join(backupDirPath, 'backup-' + dateDir + '.zip')) } @Cron(CronExpression.EVERY_1ST_DAY_OF_MONTH_AT_MIDNIGHT, {