refactor: remove cos-sdk

This commit is contained in:
Innei
2022-08-27 16:32:56 +08:00
parent 0c2249b7a5
commit fe5ec2bb16
4 changed files with 126 additions and 177 deletions

View File

@@ -98,9 +98,9 @@
"class-validator": "0.13.2",
"class-validator-jsonschema": "npm:@innei/class-validator-jsonschema@3.1.2",
"consola": "*",
"cos-nodejs-sdk-v5": "2.11.12",
"dayjs": "1.11.5",
"ejs": "3.1.8",
"form-data": "4.0.0",
"fs-extra": "*",
"get-image-colors": "4.0.1",
"image-size": "1.0.2",

140
pnpm-lock.yaml generated
View File

@@ -66,13 +66,13 @@ specifiers:
class-validator: 0.13.2
class-validator-jsonschema: npm:@innei/class-validator-jsonschema@3.1.2
consola: '*'
cos-nodejs-sdk-v5: 2.11.12
cron: '*'
cross-env: 7.0.3
dayjs: 1.11.5
ejs: 3.1.8
eslint: '*'
eslint-plugin-unused-imports: 2.0.0
form-data: 4.0.0
fs-extra: '*'
get-image-colors: 4.0.1
husky: 8.0.1
@@ -159,9 +159,9 @@ dependencies:
class-validator: 0.13.2
class-validator-jsonschema: /@innei/class-validator-jsonschema/3.1.2_e6kgdsnyya5caxg3ysdyxrqm7a
consola: 2.15.3
cos-nodejs-sdk-v5: 2.11.12
dayjs: 1.11.5
ejs: 3.1.8
form-data: 4.0.0
fs-extra: 10.1.0
get-image-colors: 4.0.1
image-size: 1.0.2
@@ -2414,15 +2414,6 @@ packages:
indent-string: 4.0.0
dev: true
/ajv-formats/1.6.1:
resolution: {integrity: sha512-4CjkH20If1lhR5CGtqkrVg3bbOtFEG80X9v6jDOIUhbzzbB+UzPBGy8GQhUNVZ0yvMHdMpawCOcy5ydGMsagGQ==}
peerDependenciesMeta:
ajv:
optional: true
dependencies:
ajv: 7.2.4
dev: false
/ajv-formats/2.1.1:
resolution: {integrity: sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==}
peerDependenciesMeta:
@@ -2447,15 +2438,6 @@ packages:
json-schema-traverse: 0.4.1
uri-js: 4.4.1
/ajv/7.2.4:
resolution: {integrity: sha512-nBeQgg/ZZA3u3SYxyaDvpvDtgZ/EZPF547ARgZBrG9Bhu1vKDwAIjtIf+sDtJUKa2zOcEbmRLBRSyMraS/Oy1A==}
dependencies:
fast-deep-equal: 3.1.3
json-schema-traverse: 1.0.0
require-from-string: 2.0.2
uri-js: 4.4.1
dev: false
/ajv/8.11.0:
resolution: {integrity: sha512-wGgprdCvMalC0BztXvitD2hC04YffAvtsUn93JbGXYLAtCUO4xd17mCCZQxUOItiBwZvJScWo8NIvQMQ71rdpg==}
dependencies:
@@ -2635,11 +2617,6 @@ packages:
engines: {node: '>=8.0.0'}
dev: false
/atomically/1.7.0:
resolution: {integrity: sha512-Xcz9l0z7y9yQ9rdDaxlmaI4uJHf/T8g9hOEzJcsEqX2SjCj4J20uK7+ldkDHMbpJDK76wF7xEIgxc/vSlsfw5w==}
engines: {node: '>=10.12.0'}
dev: false
/author-regex/1.0.0:
resolution: {integrity: sha512-KbWgR8wOYRAPekEmMXrYYdc7BRyhn2Ftk7KWfMUnQ43hFdojWEFRxhhRUm3/OFEdPa1r0KAvTTg9YQK57xTe0g==}
engines: {node: '>=0.8'}
@@ -3154,23 +3131,6 @@ packages:
/concat-map/0.0.1:
resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==}
/conf/9.0.2:
resolution: {integrity: sha512-rLSiilO85qHgaTBIIHQpsv8z+NnVfZq3cKuYNCXN1AOqPzced0GWZEe/A517VldRLyQYXUMyV+vszavE2jSAqw==}
engines: {node: '>=10'}
dependencies:
ajv: 7.2.4
ajv-formats: 1.6.1
atomically: 1.7.0
debounce-fn: 4.0.0
dot-prop: 6.0.1
env-paths: 2.2.1
json-schema-typed: 7.0.3
make-dir: 3.1.0
onetime: 5.1.2
pkg-up: 3.1.0
semver: 7.3.7
dev: false
/consola/2.15.3:
resolution: {integrity: sha512-9vAdYbHj6x2fLKC4+oPH0kFzY/orMZyG2Aj+kNylHxKGJ/Ed4dpNyAQYwJOdqO4zdM7XpVHmyejQDcQHrnuXbw==}
@@ -3214,16 +3174,6 @@ packages:
object-assign: 4.1.1
vary: 1.1.2
/cos-nodejs-sdk-v5/2.11.12:
resolution: {integrity: sha512-XtSlcrwgcyO8K0LCwNmimtkBErC1yJ55cvZ7nWFWsT0c2AWBw8F/ftGvUhZIZhh7B2SlPdXsFZg+QOU7cwI2GQ==}
engines: {node: '>= 6'}
dependencies:
conf: 9.0.2
mime-types: 2.1.35
request: 2.88.2
xml2js: 0.4.23
dev: false
/cosmiconfig/7.0.1:
resolution: {integrity: sha512-a1YWNUV2HwGimB7dU2s1wUMurNKjpx60HxBB6xUM8Re+2s1g1IIfJvFR0/iCF+XHdE0GMTKTuLR32UQff4TEyQ==}
engines: {node: '>=10'}
@@ -3328,13 +3278,6 @@ packages:
resolution: {integrity: sha512-CAdX5Q3YW3Gclyo5Vpqkgpj8fSdLQcRuzfX6mC6Phy0nfJ0eGYOeS7m4mt2plDWLAtA4TqTakvbboHvUxfe4iA==}
dev: false
/debounce-fn/4.0.0:
resolution: {integrity: sha512-8pYCQiL9Xdcg0UPSD3d+0KMlOjp+KGU5EPwYddgzQ7DATsg4fuUDjQtsYLmWjnk2obnNHgV3vE2Y4jejSOJVBQ==}
engines: {node: '>=10'}
dependencies:
mimic-fn: 3.1.0
dev: false
/debug/2.6.9:
resolution: {integrity: sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==}
peerDependencies:
@@ -3405,7 +3348,7 @@ packages:
object-keys: 1.1.1
/delayed-stream/1.0.0:
resolution: {integrity: sha1-3zrhmayt+31ECqrgsp4icrJOxhk=}
resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==}
engines: {node: '>=0.4.0'}
dev: false
@@ -3539,13 +3482,6 @@ packages:
tslib: 2.4.0
dev: false
/dot-prop/6.0.1:
resolution: {integrity: sha512-tE7ztYzXHIeyvc7N+hR3oi7FIbf/NIjVP9hmAt3yMXzrQ072/fpjGLx2GxNxGxUl5V73MEqYzioOMoVhGMJ5cA==}
engines: {node: '>=10'}
dependencies:
is-obj: 2.0.0
dev: false
/eastasianwidth/0.2.0:
resolution: {integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==}
dev: true
@@ -3640,11 +3576,6 @@ packages:
engines: {node: '>=0.12'}
dev: false
/env-paths/2.2.1:
resolution: {integrity: sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==}
engines: {node: '>=6'}
dev: false
/error-ex/1.3.2:
resolution: {integrity: sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==}
dependencies:
@@ -4232,13 +4163,6 @@ packages:
dev: false
optional: true
/find-up/3.0.0:
resolution: {integrity: sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==}
engines: {node: '>=6'}
dependencies:
locate-path: 3.0.0
dev: false
/find-up/4.1.0:
resolution: {integrity: sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==}
engines: {node: '>=8'}
@@ -4957,11 +4881,6 @@ packages:
engines: {node: '>=0.12.0'}
dev: true
/is-obj/2.0.0:
resolution: {integrity: sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==}
engines: {node: '>=8'}
dev: false
/is-regex/1.1.4:
resolution: {integrity: sha512-kvRdxDsxZjhzUX07ZnLydzS1TU/TJlTUHHY4YLL87e37oUA49DfkLqgy+VjFocowy29cKvcSiu+kIv728jTTVg==}
engines: {node: '>= 0.4'}
@@ -5651,10 +5570,6 @@ packages:
/json-schema-traverse/1.0.0:
resolution: {integrity: sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==}
/json-schema-typed/7.0.3:
resolution: {integrity: sha512-7DE8mpG+/fVw+dTpjbxnx47TaMnDfOI1jwft9g1VybltZCduyRQPJPvc+zzKY9WPHxhPWczyFuYa6I8Mw4iU5A==}
dev: false
/json-schema/0.2.3:
resolution: {integrity: sha512-a3xHnILGMtk+hDOqNwHzF6e2fNbiMrXZvxKQiEv2MlgQP+pjIOzqAmKYD2mDpXYE/44M7g+n9p2bKkYWDUcXCQ==}
dev: false
@@ -5848,14 +5763,6 @@ packages:
engines: {node: '>=6.11.5'}
dev: true
/locate-path/3.0.0:
resolution: {integrity: sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==}
engines: {node: '>=6'}
dependencies:
p-locate: 3.0.0
path-exists: 3.0.0
dev: false
/locate-path/5.0.0:
resolution: {integrity: sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==}
engines: {node: '>=8'}
@@ -6130,11 +6037,6 @@ packages:
resolution: {integrity: sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==}
engines: {node: '>=6'}
/mimic-fn/3.1.0:
resolution: {integrity: sha512-Ysbi9uYW9hFyfrThdDEQuykN4Ey6BuwPD2kpI5ES/nFTDn/98yxYNLZJcgUAKPT/mcrLLKaGzJR9YVxJrIdASQ==}
engines: {node: '>=8'}
dev: false
/mimic-fn/4.0.0:
resolution: {integrity: sha512-vqiC06CuhBTUdZH+RYl8sFrL096vA45Ok5ISO6sE/Mr1jRbGH4Csnhi8f3wKVl7x8mO4Au7Ir9D3Oyv1VYMFJw==}
engines: {node: '>=12'}
@@ -6628,13 +6530,6 @@ packages:
dependencies:
yocto-queue: 0.1.0
/p-locate/3.0.0:
resolution: {integrity: sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==}
engines: {node: '>=6'}
dependencies:
p-limit: 2.3.0
dev: false
/p-locate/4.1.0:
resolution: {integrity: sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==}
engines: {node: '>=8'}
@@ -6693,11 +6588,6 @@ packages:
lines-and-columns: 1.2.4
dev: true
/path-exists/3.0.0:
resolution: {integrity: sha512-bpC7GYwiDYQ4wYLe+FA8lhRjhQCMcQGuSgGGqDkg/QerRWw9CmGRT0iSOVRSZJ29NMLZgIzqaljJ63oaL4NIJQ==}
engines: {node: '>=4'}
dev: false
/path-exists/4.0.0:
resolution: {integrity: sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==}
engines: {node: '>=8'}
@@ -6798,13 +6688,6 @@ packages:
dependencies:
find-up: 4.1.0
/pkg-up/3.1.0:
resolution: {integrity: sha512-nDywThFk1i4BQK4twPQ6TA4RT8bDY96yeuCVBWL3ePARCiEKDRSrNGbFIgUJpLp+XeIR65v8ra7WuJOFUBtkMA==}
engines: {node: '>=8'}
dependencies:
find-up: 3.0.0
dev: false
/pluralize/8.0.0:
resolution: {integrity: sha512-Nc3IT5yHzflTfbjgqWcCPpo7DaKy4FnpB0l/zCAW0Tc7jxAiuqSxHasntB3D7887LSrA93kDJ9IXovxJYxyLCA==}
engines: {node: '>=4'}
@@ -7196,10 +7079,6 @@ packages:
sparse-bitfield: 3.0.3
optional: true
/sax/1.2.4:
resolution: {integrity: sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==}
dev: false
/schema-utils/3.1.1:
resolution: {integrity: sha512-Y5PQxS4ITlC+EahLuXaY86TXfR7Dc5lw294alXOq86JAHCihAIZfqv8nNCWvaEJvaC51uN9hbLGeV0cFBdH+Fw==}
engines: {node: '>= 10.13.0'}
@@ -8230,19 +8109,6 @@ packages:
utf-8-validate:
optional: true
/xml2js/0.4.23:
resolution: {integrity: sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==}
engines: {node: '>=4.0.0'}
dependencies:
sax: 1.2.4
xmlbuilder: 11.0.1
dev: false
/xmlbuilder/11.0.1:
resolution: {integrity: sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==}
engines: {node: '>=4.0'}
dev: false
/xss/1.0.14:
resolution: {integrity: sha512-og7TEJhXvn1a7kzZGQ7ETjdQVS2UfZyTlsEdDOqvQF7GoxNfY+0YLCzBy1kPdsDDx4QuNAonQPddpsn6Xl/7sw==}
engines: {node: '>= 0.10.0'}

View File

@@ -1,19 +1,12 @@
import COS from 'cos-nodejs-sdk-v5'
import dayjs from 'dayjs'
import { existsSync } from 'fs'
import { readdir, rm } from 'fs/promises'
import mkdirp from 'mkdirp'
import { join } from 'path'
import { Inject, Injectable, Logger, forwardRef } from '@nestjs/common'
import { OnEvent } from '@nestjs/event-emitter'
import { CronExpression } from '@nestjs/schedule'
import { forwardRef, Inject, Injectable, Logger } from '@nestjs/common'
import { DEMO_MODE } from '~/app.config'
import { CronDescription } from '~/common/decorator/cron-description.decorator'
import { CronOnce } from '~/common/decorator/cron-once.decorator'
import { RedisKeys } from '~/constants/cache.constant'
import { EventBusEvents } from '~/constants/event-bus.constant'
import { LOG_DIR, TEMP_DIR } from '~/constants/path.constant'
import { AggregateService } from '~/modules/aggregate/aggregate.service'
import { AnalyzeModel } from '~/modules/analyze/analyze.model'
@@ -23,12 +16,18 @@ import { NoteService } from '~/modules/note/note.service'
import { PageService } from '~/modules/page/page.service'
import { PostService } from '~/modules/post/post.service'
import { SearchService } from '~/modules/search/search.service'
import { InjectModel } from '~/transformers/model.transformer'
import { uploadFileToCOS } from '~/utils/cos.util'
import { getRedisKey } from '~/utils/redis.util'
import { CacheService } from '../redis/cache.service'
import { HttpService } from './helper.http.service'
import { JWTService, StoreJWTPayload } from './helper.jwt.service'
import { OnEvent } from '@nestjs/event-emitter'
import { CronExpression } from '@nestjs/schedule'
import { CronDescription } from '~/common/decorator/cron-description.decorator'
import { CronOnce } from '~/common/decorator/cron-once.decorator'
import { EventBusEvents } from '~/constants/event-bus.constant'
import { InjectModel } from '~/transformers/model.transformer'
@Injectable()
export class CronService {
@@ -54,16 +53,14 @@ export class CronService {
@Inject(forwardRef(() => BackupService))
private readonly backupService: BackupService,
@Inject(forwardRef(() => SearchService))
private readonly searchService: SearchService,
@Inject(forwardRef(() => JWTService))
private readonly jwtService: JWTService,
private readonly searchService: SearchService, // @Inject(forwardRef(() => JWTService)) // private readonly jwtService: JWTService,
) {
this.logger = new Logger(CronService.name)
}
@CronOnce(CronExpression.EVERY_DAY_AT_1AM, { name: 'backupDB' })
@CronDescription('备份 DB 并上传 COS')
async backupDB({ uploadCOS = true }: { uploadCOS?: boolean } = {}) {
async backupDB() {
if (DEMO_MODE) {
return
}
@@ -74,9 +71,6 @@ export class CronService {
}
// 开始上传 COS
process.nextTick(async () => {
if (!uploadCOS) {
return
}
const { backupOptions } = await this.configs.waitForConfigReady()
if (
@@ -87,34 +81,26 @@ export class CronService {
) {
return
}
const backupFilePath = backup.path
if (!existsSync(backupFilePath)) {
this.logger.warn('文件不存在, 无法上传到 COS')
return
}
this.logger.log('--> 开始上传到 COS')
const cos = new COS({
SecretId: backupOptions.secretId,
SecretKey: backupOptions.secretKey,
})
// 分片上传
cos.sliceUploadFile(
await uploadFileToCOS(
backup.buffer,
backup.path.slice(backup.path.lastIndexOf('/') + 1),
{
Bucket: backupOptions.bucket,
Region: backupOptions.region,
Key: backup.path.slice(backup.path.lastIndexOf('/') + 1),
FilePath: backupFilePath,
},
(err) => {
if (!err) {
this.logger.log('--> 上传成功')
} else {
this.logger.error('--> 上传失败了')
throw err
}
bucket: backupOptions.bucket,
region: backupOptions.region,
secretId: backupOptions.secretId,
secretKey: backupOptions.secretKey,
},
)
.then(() => {
this.logger.log('--> 上传成功')
})
.catch((err) => {
this.logger.error('--> 上传失败了')
throw err
})
})
}

97
src/utils/cos.util.ts Normal file
View File

@@ -0,0 +1,97 @@
import axios from 'axios'
import crypto from 'crypto'
import FormData from 'form-data'
import fs from 'fs-extra'
const sha1 = (str: string) =>
crypto.createHash('sha1').update(str).digest('hex').toLowerCase()
const hmac = (str: string, key: string) =>
crypto.createHmac('sha1', key).update(str).digest('hex').toLowerCase()
export const uploadFileToCOS = async (
localFilePathOrBuffer: string | Buffer,
remoteFileKey: string,
options: {
bucket: string
region: string
secretId: string
secretKey: string
onProgress?: (progressFloat: number) => void
},
) => {
const {
secretId,
secretKey,
bucket,
region,
onProgress = () => void 0,
} = options
const endpoint = `https://${bucket}.cos.${region}.myqcloud.com`
const now = +new Date()
const startTime = now / 1000,
expireTime = now / 1000 + 900
const keytime = `${startTime};${expireTime}`
const tickets = [
{
'q-ak': secretId,
},
{
'q-sign-algorithm': 'sha1',
},
{
'q-sign-time': keytime,
},
]
const policy = JSON.stringify({
expiration: new Date(expireTime * 1000).toISOString(),
conditions: tickets,
})
const signature = hmac(sha1(policy), hmac(keytime, secretKey))
const formData = new FormData({
maxDataSize: 10e10,
})
Object.entries({
key: remoteFileKey,
policy: Buffer.from(policy).toString('base64'),
'q-key-time': keytime,
'q-signature': signature,
...tickets.reduce((acc, cur) => ({ ...acc, ...cur }), {}),
}).forEach(([key, value]) => {
formData.append(key, value)
})
formData.append(
'file',
typeof localFilePathOrBuffer == 'string'
? await fs.readFile(localFilePathOrBuffer)
: Buffer.isBuffer(localFilePathOrBuffer)
? localFilePathOrBuffer
: Buffer.from(localFilePathOrBuffer),
{
filename: remoteFileKey,
},
)
await axios
.post(endpoint, formData, {
headers: {
...formData.getHeaders(),
// NOTE: important do this, if post file is over than 300K
'Content-Length': formData.getLengthSync(),
},
onDownloadProgress: (progress) => {
if (onProgress) {
onProgress(progress.loaded / progress.total)
}
},
})
.catch((err) => {
if (isDev) {
console.dir(err)
}
console.log(err.response.data)
throw err
})
}