From fe5ec2bb1646feb7782a012caa189fd86def738e Mon Sep 17 00:00:00 2001 From: Innei Date: Sat, 27 Aug 2022 16:32:56 +0800 Subject: [PATCH] refactor: remove cos-sdk --- package.json | 2 +- pnpm-lock.yaml | 140 +------------------ src/processors/helper/helper.cron.service.ts | 64 ++++----- src/utils/cos.util.ts | 97 +++++++++++++ 4 files changed, 126 insertions(+), 177 deletions(-) create mode 100644 src/utils/cos.util.ts diff --git a/package.json b/package.json index 2243dfd1..28c4acc5 100644 --- a/package.json +++ b/package.json @@ -98,9 +98,9 @@ "class-validator": "0.13.2", "class-validator-jsonschema": "npm:@innei/class-validator-jsonschema@3.1.2", "consola": "*", - "cos-nodejs-sdk-v5": "2.11.12", "dayjs": "1.11.5", "ejs": "3.1.8", + "form-data": "4.0.0", "fs-extra": "*", "get-image-colors": "4.0.1", "image-size": "1.0.2", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7a5f9254..dd5aec5b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -66,13 +66,13 @@ specifiers: class-validator: 0.13.2 class-validator-jsonschema: npm:@innei/class-validator-jsonschema@3.1.2 consola: '*' - cos-nodejs-sdk-v5: 2.11.12 cron: '*' cross-env: 7.0.3 dayjs: 1.11.5 ejs: 3.1.8 eslint: '*' eslint-plugin-unused-imports: 2.0.0 + form-data: 4.0.0 fs-extra: '*' get-image-colors: 4.0.1 husky: 8.0.1 @@ -159,9 +159,9 @@ dependencies: class-validator: 0.13.2 class-validator-jsonschema: /@innei/class-validator-jsonschema/3.1.2_e6kgdsnyya5caxg3ysdyxrqm7a consola: 2.15.3 - cos-nodejs-sdk-v5: 2.11.12 dayjs: 1.11.5 ejs: 3.1.8 + form-data: 4.0.0 fs-extra: 10.1.0 get-image-colors: 4.0.1 image-size: 1.0.2 @@ -2414,15 +2414,6 @@ packages: indent-string: 4.0.0 dev: true - /ajv-formats/1.6.1: - resolution: {integrity: sha512-4CjkH20If1lhR5CGtqkrVg3bbOtFEG80X9v6jDOIUhbzzbB+UzPBGy8GQhUNVZ0yvMHdMpawCOcy5ydGMsagGQ==} - peerDependenciesMeta: - ajv: - optional: true - dependencies: - ajv: 7.2.4 - dev: false - /ajv-formats/2.1.1: resolution: {integrity: sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==} peerDependenciesMeta: @@ -2447,15 +2438,6 @@ packages: json-schema-traverse: 0.4.1 uri-js: 4.4.1 - /ajv/7.2.4: - resolution: {integrity: sha512-nBeQgg/ZZA3u3SYxyaDvpvDtgZ/EZPF547ARgZBrG9Bhu1vKDwAIjtIf+sDtJUKa2zOcEbmRLBRSyMraS/Oy1A==} - dependencies: - fast-deep-equal: 3.1.3 - json-schema-traverse: 1.0.0 - require-from-string: 2.0.2 - uri-js: 4.4.1 - dev: false - /ajv/8.11.0: resolution: {integrity: sha512-wGgprdCvMalC0BztXvitD2hC04YffAvtsUn93JbGXYLAtCUO4xd17mCCZQxUOItiBwZvJScWo8NIvQMQ71rdpg==} dependencies: @@ -2635,11 +2617,6 @@ packages: engines: {node: '>=8.0.0'} dev: false - /atomically/1.7.0: - resolution: {integrity: sha512-Xcz9l0z7y9yQ9rdDaxlmaI4uJHf/T8g9hOEzJcsEqX2SjCj4J20uK7+ldkDHMbpJDK76wF7xEIgxc/vSlsfw5w==} - engines: {node: '>=10.12.0'} - dev: false - /author-regex/1.0.0: resolution: {integrity: sha512-KbWgR8wOYRAPekEmMXrYYdc7BRyhn2Ftk7KWfMUnQ43hFdojWEFRxhhRUm3/OFEdPa1r0KAvTTg9YQK57xTe0g==} engines: {node: '>=0.8'} @@ -3154,23 +3131,6 @@ packages: /concat-map/0.0.1: resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==} - /conf/9.0.2: - resolution: {integrity: sha512-rLSiilO85qHgaTBIIHQpsv8z+NnVfZq3cKuYNCXN1AOqPzced0GWZEe/A517VldRLyQYXUMyV+vszavE2jSAqw==} - engines: {node: '>=10'} - dependencies: - ajv: 7.2.4 - ajv-formats: 1.6.1 - atomically: 1.7.0 - debounce-fn: 4.0.0 - dot-prop: 6.0.1 - env-paths: 2.2.1 - json-schema-typed: 7.0.3 - make-dir: 3.1.0 - onetime: 5.1.2 - pkg-up: 3.1.0 - semver: 7.3.7 - dev: false - /consola/2.15.3: resolution: {integrity: sha512-9vAdYbHj6x2fLKC4+oPH0kFzY/orMZyG2Aj+kNylHxKGJ/Ed4dpNyAQYwJOdqO4zdM7XpVHmyejQDcQHrnuXbw==} @@ -3214,16 +3174,6 @@ packages: object-assign: 4.1.1 vary: 1.1.2 - /cos-nodejs-sdk-v5/2.11.12: - resolution: {integrity: sha512-XtSlcrwgcyO8K0LCwNmimtkBErC1yJ55cvZ7nWFWsT0c2AWBw8F/ftGvUhZIZhh7B2SlPdXsFZg+QOU7cwI2GQ==} - engines: {node: '>= 6'} - dependencies: - conf: 9.0.2 - mime-types: 2.1.35 - request: 2.88.2 - xml2js: 0.4.23 - dev: false - /cosmiconfig/7.0.1: resolution: {integrity: sha512-a1YWNUV2HwGimB7dU2s1wUMurNKjpx60HxBB6xUM8Re+2s1g1IIfJvFR0/iCF+XHdE0GMTKTuLR32UQff4TEyQ==} engines: {node: '>=10'} @@ -3328,13 +3278,6 @@ packages: resolution: {integrity: sha512-CAdX5Q3YW3Gclyo5Vpqkgpj8fSdLQcRuzfX6mC6Phy0nfJ0eGYOeS7m4mt2plDWLAtA4TqTakvbboHvUxfe4iA==} dev: false - /debounce-fn/4.0.0: - resolution: {integrity: sha512-8pYCQiL9Xdcg0UPSD3d+0KMlOjp+KGU5EPwYddgzQ7DATsg4fuUDjQtsYLmWjnk2obnNHgV3vE2Y4jejSOJVBQ==} - engines: {node: '>=10'} - dependencies: - mimic-fn: 3.1.0 - dev: false - /debug/2.6.9: resolution: {integrity: sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==} peerDependencies: @@ -3405,7 +3348,7 @@ packages: object-keys: 1.1.1 /delayed-stream/1.0.0: - resolution: {integrity: sha1-3zrhmayt+31ECqrgsp4icrJOxhk=} + resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} engines: {node: '>=0.4.0'} dev: false @@ -3539,13 +3482,6 @@ packages: tslib: 2.4.0 dev: false - /dot-prop/6.0.1: - resolution: {integrity: sha512-tE7ztYzXHIeyvc7N+hR3oi7FIbf/NIjVP9hmAt3yMXzrQ072/fpjGLx2GxNxGxUl5V73MEqYzioOMoVhGMJ5cA==} - engines: {node: '>=10'} - dependencies: - is-obj: 2.0.0 - dev: false - /eastasianwidth/0.2.0: resolution: {integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==} dev: true @@ -3640,11 +3576,6 @@ packages: engines: {node: '>=0.12'} dev: false - /env-paths/2.2.1: - resolution: {integrity: sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==} - engines: {node: '>=6'} - dev: false - /error-ex/1.3.2: resolution: {integrity: sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==} dependencies: @@ -4232,13 +4163,6 @@ packages: dev: false optional: true - /find-up/3.0.0: - resolution: {integrity: sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==} - engines: {node: '>=6'} - dependencies: - locate-path: 3.0.0 - dev: false - /find-up/4.1.0: resolution: {integrity: sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==} engines: {node: '>=8'} @@ -4957,11 +4881,6 @@ packages: engines: {node: '>=0.12.0'} dev: true - /is-obj/2.0.0: - resolution: {integrity: sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==} - engines: {node: '>=8'} - dev: false - /is-regex/1.1.4: resolution: {integrity: sha512-kvRdxDsxZjhzUX07ZnLydzS1TU/TJlTUHHY4YLL87e37oUA49DfkLqgy+VjFocowy29cKvcSiu+kIv728jTTVg==} engines: {node: '>= 0.4'} @@ -5651,10 +5570,6 @@ packages: /json-schema-traverse/1.0.0: resolution: {integrity: sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==} - /json-schema-typed/7.0.3: - resolution: {integrity: sha512-7DE8mpG+/fVw+dTpjbxnx47TaMnDfOI1jwft9g1VybltZCduyRQPJPvc+zzKY9WPHxhPWczyFuYa6I8Mw4iU5A==} - dev: false - /json-schema/0.2.3: resolution: {integrity: sha512-a3xHnILGMtk+hDOqNwHzF6e2fNbiMrXZvxKQiEv2MlgQP+pjIOzqAmKYD2mDpXYE/44M7g+n9p2bKkYWDUcXCQ==} dev: false @@ -5848,14 +5763,6 @@ packages: engines: {node: '>=6.11.5'} dev: true - /locate-path/3.0.0: - resolution: {integrity: sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==} - engines: {node: '>=6'} - dependencies: - p-locate: 3.0.0 - path-exists: 3.0.0 - dev: false - /locate-path/5.0.0: resolution: {integrity: sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==} engines: {node: '>=8'} @@ -6130,11 +6037,6 @@ packages: resolution: {integrity: sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==} engines: {node: '>=6'} - /mimic-fn/3.1.0: - resolution: {integrity: sha512-Ysbi9uYW9hFyfrThdDEQuykN4Ey6BuwPD2kpI5ES/nFTDn/98yxYNLZJcgUAKPT/mcrLLKaGzJR9YVxJrIdASQ==} - engines: {node: '>=8'} - dev: false - /mimic-fn/4.0.0: resolution: {integrity: sha512-vqiC06CuhBTUdZH+RYl8sFrL096vA45Ok5ISO6sE/Mr1jRbGH4Csnhi8f3wKVl7x8mO4Au7Ir9D3Oyv1VYMFJw==} engines: {node: '>=12'} @@ -6628,13 +6530,6 @@ packages: dependencies: yocto-queue: 0.1.0 - /p-locate/3.0.0: - resolution: {integrity: sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==} - engines: {node: '>=6'} - dependencies: - p-limit: 2.3.0 - dev: false - /p-locate/4.1.0: resolution: {integrity: sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==} engines: {node: '>=8'} @@ -6693,11 +6588,6 @@ packages: lines-and-columns: 1.2.4 dev: true - /path-exists/3.0.0: - resolution: {integrity: sha512-bpC7GYwiDYQ4wYLe+FA8lhRjhQCMcQGuSgGGqDkg/QerRWw9CmGRT0iSOVRSZJ29NMLZgIzqaljJ63oaL4NIJQ==} - engines: {node: '>=4'} - dev: false - /path-exists/4.0.0: resolution: {integrity: sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==} engines: {node: '>=8'} @@ -6798,13 +6688,6 @@ packages: dependencies: find-up: 4.1.0 - /pkg-up/3.1.0: - resolution: {integrity: sha512-nDywThFk1i4BQK4twPQ6TA4RT8bDY96yeuCVBWL3ePARCiEKDRSrNGbFIgUJpLp+XeIR65v8ra7WuJOFUBtkMA==} - engines: {node: '>=8'} - dependencies: - find-up: 3.0.0 - dev: false - /pluralize/8.0.0: resolution: {integrity: sha512-Nc3IT5yHzflTfbjgqWcCPpo7DaKy4FnpB0l/zCAW0Tc7jxAiuqSxHasntB3D7887LSrA93kDJ9IXovxJYxyLCA==} engines: {node: '>=4'} @@ -7196,10 +7079,6 @@ packages: sparse-bitfield: 3.0.3 optional: true - /sax/1.2.4: - resolution: {integrity: sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==} - dev: false - /schema-utils/3.1.1: resolution: {integrity: sha512-Y5PQxS4ITlC+EahLuXaY86TXfR7Dc5lw294alXOq86JAHCihAIZfqv8nNCWvaEJvaC51uN9hbLGeV0cFBdH+Fw==} engines: {node: '>= 10.13.0'} @@ -8230,19 +8109,6 @@ packages: utf-8-validate: optional: true - /xml2js/0.4.23: - resolution: {integrity: sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==} - engines: {node: '>=4.0.0'} - dependencies: - sax: 1.2.4 - xmlbuilder: 11.0.1 - dev: false - - /xmlbuilder/11.0.1: - resolution: {integrity: sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==} - engines: {node: '>=4.0'} - dev: false - /xss/1.0.14: resolution: {integrity: sha512-og7TEJhXvn1a7kzZGQ7ETjdQVS2UfZyTlsEdDOqvQF7GoxNfY+0YLCzBy1kPdsDDx4QuNAonQPddpsn6Xl/7sw==} engines: {node: '>= 0.10.0'} diff --git a/src/processors/helper/helper.cron.service.ts b/src/processors/helper/helper.cron.service.ts index 27b12eb3..04212786 100644 --- a/src/processors/helper/helper.cron.service.ts +++ b/src/processors/helper/helper.cron.service.ts @@ -1,19 +1,12 @@ -import COS from 'cos-nodejs-sdk-v5' import dayjs from 'dayjs' -import { existsSync } from 'fs' import { readdir, rm } from 'fs/promises' import mkdirp from 'mkdirp' import { join } from 'path' -import { Inject, Injectable, Logger, forwardRef } from '@nestjs/common' -import { OnEvent } from '@nestjs/event-emitter' -import { CronExpression } from '@nestjs/schedule' +import { forwardRef, Inject, Injectable, Logger } from '@nestjs/common' import { DEMO_MODE } from '~/app.config' -import { CronDescription } from '~/common/decorator/cron-description.decorator' -import { CronOnce } from '~/common/decorator/cron-once.decorator' import { RedisKeys } from '~/constants/cache.constant' -import { EventBusEvents } from '~/constants/event-bus.constant' import { LOG_DIR, TEMP_DIR } from '~/constants/path.constant' import { AggregateService } from '~/modules/aggregate/aggregate.service' import { AnalyzeModel } from '~/modules/analyze/analyze.model' @@ -23,12 +16,18 @@ import { NoteService } from '~/modules/note/note.service' import { PageService } from '~/modules/page/page.service' import { PostService } from '~/modules/post/post.service' import { SearchService } from '~/modules/search/search.service' -import { InjectModel } from '~/transformers/model.transformer' +import { uploadFileToCOS } from '~/utils/cos.util' import { getRedisKey } from '~/utils/redis.util' import { CacheService } from '../redis/cache.service' import { HttpService } from './helper.http.service' import { JWTService, StoreJWTPayload } from './helper.jwt.service' +import { OnEvent } from '@nestjs/event-emitter' +import { CronExpression } from '@nestjs/schedule' +import { CronDescription } from '~/common/decorator/cron-description.decorator' +import { CronOnce } from '~/common/decorator/cron-once.decorator' +import { EventBusEvents } from '~/constants/event-bus.constant' +import { InjectModel } from '~/transformers/model.transformer' @Injectable() export class CronService { @@ -54,16 +53,14 @@ export class CronService { @Inject(forwardRef(() => BackupService)) private readonly backupService: BackupService, @Inject(forwardRef(() => SearchService)) - private readonly searchService: SearchService, - @Inject(forwardRef(() => JWTService)) - private readonly jwtService: JWTService, + private readonly searchService: SearchService, // @Inject(forwardRef(() => JWTService)) // private readonly jwtService: JWTService, ) { this.logger = new Logger(CronService.name) } @CronOnce(CronExpression.EVERY_DAY_AT_1AM, { name: 'backupDB' }) @CronDescription('备份 DB 并上传 COS') - async backupDB({ uploadCOS = true }: { uploadCOS?: boolean } = {}) { + async backupDB() { if (DEMO_MODE) { return } @@ -74,9 +71,6 @@ export class CronService { } // 开始上传 COS process.nextTick(async () => { - if (!uploadCOS) { - return - } const { backupOptions } = await this.configs.waitForConfigReady() if ( @@ -87,34 +81,26 @@ export class CronService { ) { return } - const backupFilePath = backup.path - if (!existsSync(backupFilePath)) { - this.logger.warn('文件不存在, 无法上传到 COS') - return - } this.logger.log('--> 开始上传到 COS') - const cos = new COS({ - SecretId: backupOptions.secretId, - SecretKey: backupOptions.secretKey, - }) - // 分片上传 - cos.sliceUploadFile( + + await uploadFileToCOS( + backup.buffer, + backup.path.slice(backup.path.lastIndexOf('/') + 1), { - Bucket: backupOptions.bucket, - Region: backupOptions.region, - Key: backup.path.slice(backup.path.lastIndexOf('/') + 1), - FilePath: backupFilePath, - }, - (err) => { - if (!err) { - this.logger.log('--> 上传成功') - } else { - this.logger.error('--> 上传失败了') - throw err - } + bucket: backupOptions.bucket, + region: backupOptions.region, + secretId: backupOptions.secretId, + secretKey: backupOptions.secretKey, }, ) + .then(() => { + this.logger.log('--> 上传成功') + }) + .catch((err) => { + this.logger.error('--> 上传失败了') + throw err + }) }) } diff --git a/src/utils/cos.util.ts b/src/utils/cos.util.ts new file mode 100644 index 00000000..4c400eab --- /dev/null +++ b/src/utils/cos.util.ts @@ -0,0 +1,97 @@ +import axios from 'axios' +import crypto from 'crypto' +import FormData from 'form-data' +import fs from 'fs-extra' + +const sha1 = (str: string) => + crypto.createHash('sha1').update(str).digest('hex').toLowerCase() +const hmac = (str: string, key: string) => + crypto.createHmac('sha1', key).update(str).digest('hex').toLowerCase() + +export const uploadFileToCOS = async ( + localFilePathOrBuffer: string | Buffer, + remoteFileKey: string, + options: { + bucket: string + region: string + secretId: string + secretKey: string + onProgress?: (progressFloat: number) => void + }, +) => { + const { + secretId, + secretKey, + bucket, + region, + onProgress = () => void 0, + } = options + const endpoint = `https://${bucket}.cos.${region}.myqcloud.com` + + const now = +new Date() + const startTime = now / 1000, + expireTime = now / 1000 + 900 + const keytime = `${startTime};${expireTime}` + const tickets = [ + { + 'q-ak': secretId, + }, + { + 'q-sign-algorithm': 'sha1', + }, + { + 'q-sign-time': keytime, + }, + ] + const policy = JSON.stringify({ + expiration: new Date(expireTime * 1000).toISOString(), + conditions: tickets, + }) + const signature = hmac(sha1(policy), hmac(keytime, secretKey)) + const formData = new FormData({ + maxDataSize: 10e10, + }) + Object.entries({ + key: remoteFileKey, + policy: Buffer.from(policy).toString('base64'), + 'q-key-time': keytime, + 'q-signature': signature, + + ...tickets.reduce((acc, cur) => ({ ...acc, ...cur }), {}), + }).forEach(([key, value]) => { + formData.append(key, value) + }) + + formData.append( + 'file', + typeof localFilePathOrBuffer == 'string' + ? await fs.readFile(localFilePathOrBuffer) + : Buffer.isBuffer(localFilePathOrBuffer) + ? localFilePathOrBuffer + : Buffer.from(localFilePathOrBuffer), + { + filename: remoteFileKey, + }, + ) + + await axios + .post(endpoint, formData, { + headers: { + ...formData.getHeaders(), + // NOTE: important do this, if post file is over than 300K + 'Content-Length': formData.getLengthSync(), + }, + onDownloadProgress: (progress) => { + if (onProgress) { + onProgress(progress.loaded / progress.total) + } + }, + }) + .catch((err) => { + if (isDev) { + console.dir(err) + } + console.log(err.response.data) + throw err + }) +}