diff --git a/.gitignore b/.gitignore index ec7a92b6..4d16205d 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,4 @@ lerna-debug.log* !.vscode/extensions.json patch/dist +tmp \ No newline at end of file diff --git a/package.json b/package.json index 711c4ab2..32d579a5 100644 --- a/package.json +++ b/package.json @@ -47,6 +47,7 @@ "@nestjs/passport": "^8.0.1", "@nestjs/platform-fastify": "^8.0.6", "@nestjs/platform-socket.io": "8.0.6", + "@nestjs/schedule": "^1.0.1", "@nestjs/swagger": "^5.0.9", "@nestjs/websockets": "^8.0.6", "@typegoose/auto-increment": "^0.9.0", diff --git a/paw.paw b/paw.paw index 1945d440..87dc68d6 100644 Binary files a/paw.paw and b/paw.paw differ diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b9662f5d..4a50011b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -12,6 +12,7 @@ specifiers: '@nestjs/passport': ^8.0.1 '@nestjs/platform-fastify': ^8.0.6 '@nestjs/platform-socket.io': 8.0.6 + '@nestjs/schedule': ^1.0.1 '@nestjs/schematics': ^8.0.2 '@nestjs/swagger': ^5.0.9 '@nestjs/testing': ^8.0.6 @@ -91,6 +92,7 @@ dependencies: '@nestjs/passport': 8.0.1_2c02db70fddcb59258fa0eed39c4b725 '@nestjs/platform-fastify': 8.0.6_67f7e5db8827badcb202b1d38f6b1aea '@nestjs/platform-socket.io': 8.0.6_875c1aa90becd3a53d7e39e33971fbfe + '@nestjs/schedule': 1.0.1_be74d10e7c1accb5ea6dd58471a1ec77 '@nestjs/swagger': 5.0.9_dc9defb2ccd6e5ace839769c2c65c2a3 '@nestjs/websockets': 8.0.6_d9cb7157596bf7c6176480174d173b36 '@typegoose/auto-increment': 0.9.0_mongoose@5.13.8 @@ -1165,6 +1167,20 @@ packages: - utf-8-validate dev: false + /@nestjs/schedule/1.0.1_be74d10e7c1accb5ea6dd58471a1ec77: + resolution: {integrity: sha512-EU2tB4rxuEgum8JlorAFvXkU982EYZm/IBa7n6kgkyps5BbxQSFf7iR1CLkP9zODO9ApZTWk5z3q9L3O7vrkoQ==} + peerDependencies: + '@nestjs/common': ^6.10.11 || ^7.0.0 || ^8.0.0 + '@nestjs/core': ^7.0.0 || ^8.0.0 + reflect-metadata: ^0.1.12 + dependencies: + '@nestjs/common': 8.0.6_4d0c20d2c2a765e9ff99ebac79ad2484 + '@nestjs/core': 8.0.6_214ebf00327c8ed1d6618d61764e6a91 + cron: 1.7.2 + reflect-metadata: 0.1.13 + uuid: 8.3.2 + dev: false + /@nestjs/schematics/8.0.3_typescript@4.3.5: resolution: {integrity: sha512-A5qyS9yv6v2RIBqbsyYG57NfYA8Jm/aypRV1nc7JXjhdfDHwWKqCsgQ/7/82vVjhlvVAfr5x/dpCWqcF3XYd7w==} peerDependencies: @@ -2721,6 +2737,12 @@ packages: resolution: {integrity: sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==} dev: true + /cron/1.7.2: + resolution: {integrity: sha512-+SaJ2OfeRvfQqwXQ2kgr0Y5pzBR/lijf5OpnnaruwWnmI799JfWr2jN2ItOV9s3A/+TFOt6mxvKzQq5F0Jp6VQ==} + dependencies: + moment-timezone: 0.5.33 + dev: false + /cross-env/7.0.3: resolution: {integrity: sha512-+/HKd6EgcQCJGh2PSjZuUitQBQynKor4wrFbRg4DtAgS1aWO+gU52xpH7M9ScGgXSYmAVS9bIJ8EzuaGw0oNAw==} engines: {node: '>=10.14', npm: '>=6', yarn: '>=1'} @@ -5111,6 +5133,16 @@ packages: hasBin: true dev: false + /moment-timezone/0.5.33: + resolution: {integrity: sha512-PTc2vcT8K9J5/9rDEPe5czSIKgLoGsH8UNpA4qZTVw0Vd/Uz19geE9abbIOQKaAQFcnQ3v5YEXrbSc5BpshH+w==} + dependencies: + moment: 2.29.1 + dev: false + + /moment/2.29.1: + resolution: {integrity: sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ==} + dev: false + /mongodb/3.6.11: resolution: {integrity: sha512-4Y4lTFHDHZZdgMaHmojtNAlqkvddX2QQBEN0K//GzxhGwlI9tZ9R0vhbjr1Decw+TF7qK0ZLjQT292XgHRRQgw==} engines: {node: '>=4'} diff --git a/src/common/middlewares/analyze.middleware.ts b/src/common/middlewares/analyze.middleware.ts index b2bc15b8..2dca143d 100644 --- a/src/common/middlewares/analyze.middleware.ts +++ b/src/common/middlewares/analyze.middleware.ts @@ -1,7 +1,140 @@ import { NestMiddleware } from '@nestjs/common' -// TODO: +import { ReturnModelType } from '@typegoose/typegoose' +import { readFileSync } from 'fs' +import { IncomingMessage, ServerResponse } from 'http' +import { InjectModel } from 'nestjs-typegoose' +import { UAParser } from 'ua-parser-js' +import { RedisKeys } from '~/constants/cache.constant' +import { localBotListDataFilePath } from '~/constants/path.constant' +import { AnalyzeModel } from '~/modules/analyze/analyze.model' +import { OptionModel } from '~/modules/configs/configs.model' +import { CacheService } from '~/processors/cache/cache.service' +import { CronService } from '~/processors/helper/helper.cron.service' +import { getIp } from '~/utils/ip.util' +import { getRedisKey } from '~/utils/redis.util' + export class AnalyzeMiddleware implements NestMiddleware { - use(req, res, next) { - next() + private parser: UAParser + private botListData: RegExp[] = [] + + constructor( + @InjectModel(AnalyzeModel) + private readonly model: ReturnModelType, + @InjectModel(OptionModel) + private readonly options: ReturnModelType, + private readonly cronService: CronService, + private readonly cacheService: CacheService, + ) { + this.init() + } + + init() { + this.parser = new UAParser() + this.botListData = this.getLocalBotList() + this.cronService.updateBotList().then((res) => { + this.botListData = this.pickPattern2Regexp(res) + }) + } + + getLocalBotList() { + try { + return this.pickPattern2Regexp( + JSON.parse( + readFileSync(localBotListDataFilePath, { + encoding: 'utf-8', + }), + ), + ) + } catch { + return [] + } + } + + private pickPattern2Regexp(data: any): RegExp[] { + return data.map((item) => new RegExp(item.pattern)) + } + + async use(req: IncomingMessage, res: ServerResponse, next: () => void) { + const ip = getIp(req) + // @ts-ignore + const url = req.originalUrl?.replace(/^\/api(\/v\d)?/, '') + + // if req from SSR server, like 127.0.0.1, skip + if (['127.0.0.1', 'localhost', '::-1'].includes(ip)) { + return next() + } + + // if is login and is master, skip + if (req.headers['Authorization'] || req.headers['authorization']) { + return next() + } + + // if user agent is in bot list, skip + if (this.botListData.some((rg) => rg.test(req.headers['user-agent']))) { + return next() + } + + try { + this.parser.setUA(req.headers['user-agent']) + + const ua = this.parser.getResult() + // @ts-ignore + await this.model.create({ + ip, + ua, + path: new URL('http://a.com' + url).pathname, + }) + const apiCallTimeRecord = await this.options.findOne({ + name: 'apiCallTime', + }) + if (!apiCallTimeRecord) { + await this.options.create({ + name: 'apiCallTime', + value: 1, + }) + } else { + await this.options.updateOne( + { name: 'apiCallTime' }, + { + $inc: { + // @ts-ignore + value: 1, + }, + }, + ) + } + // ip access in redis + const client = this.cacheService.getClient() + const fromRedisIps = await client.get( + getRedisKey(RedisKeys.Access, 'ips'), + ) + const ips = fromRedisIps ? JSON.parse(fromRedisIps) : [] + if (!ips.includes(ip)) { + await client.set( + getRedisKey(RedisKeys.Access, 'ips'), + JSON.stringify([...ips, ip]), + ) + // record uv to db + process.nextTick(async () => { + const uvRecord = await this.options.findOne({ name: 'uv' }) + if (uvRecord) { + await uvRecord.updateOne({ + $inc: { + value: 1, + }, + }) + } else { + await this.options.create({ + name: 'uv', + value: 1, + }) + } + }) + } + } catch (e) { + console.error(e) + } finally { + next() + } } } diff --git a/src/constants/path.constant.ts b/src/constants/path.constant.ts index b3744fe1..470935e6 100644 --- a/src/constants/path.constant.ts +++ b/src/constants/path.constant.ts @@ -1,21 +1,15 @@ -/* - * @Author: Innei - * @Date: 2020-08-01 19:49:31 - * @LastEditTime: 2021-03-21 19:36:20 - * @LastEditors: Innei - * @FilePath: /server/shared/constants/index.ts - * @Coding with Love - */ import { homedir } from 'os' import { join } from 'path' import { isDev } from '~/utils/index.util' export const HOME = homedir() -export const TEMP_DIR = isDev ? join(__dirname, '../tmp') : '/tmp/mx-space' +export const TEMP_DIR = isDev ? join(process.cwd(), './tmp') : '/tmp/mx-space' export const DATA_DIR = isDev - ? join(__dirname, '../tmp') + ? join(process.cwd(), './tmp') : join(HOME, '.mx-space') export const LOGGER_DIR = join(DATA_DIR, 'log') + +export const localBotListDataFilePath = join(DATA_DIR, 'bot_list.json') diff --git a/src/modules/link/link.controller.ts b/src/modules/link/link.controller.ts index 75452bd3..22b48d26 100644 --- a/src/modules/link/link.controller.ts +++ b/src/modules/link/link.controller.ts @@ -1,4 +1,13 @@ -import { Body, Get, HttpCode, Param, Patch, Post, Query } from '@nestjs/common' +import { + Body, + Controller, + Get, + HttpCode, + Param, + Patch, + Post, + Query, +} from '@nestjs/common' import { InjectModel } from 'nestjs-typegoose' import { Auth } from '~/common/decorator/auth.decorator' import { BaseCrudFactory } from '~/utils/crud.util' @@ -7,6 +16,7 @@ import { LinkQueryDto } from './link.dto' import { LinkModel } from './link.model' import { LinkService } from './link.service' +@Controller(['links', 'friends']) export class LinkController extends BaseCrudFactory({ model: LinkModel, }) { diff --git a/src/processors/helper/helper.cron.service.ts b/src/processors/helper/helper.cron.service.ts new file mode 100644 index 00000000..602f1315 --- /dev/null +++ b/src/processors/helper/helper.cron.service.ts @@ -0,0 +1,30 @@ +import { Injectable, Logger } from '@nestjs/common' +import { Cron, CronExpression } from '@nestjs/schedule' +import { writeFileSync } from 'fs' +import { localBotListDataFilePath } from '~/constants/path.constant' +import { HttpService } from './helper.http.service' +@Injectable() +export class CronService { + private logger: Logger + constructor(private readonly http: HttpService) { + this.logger = new Logger(CronService.name) + } + + @Cron(CronExpression.EVERY_WEEK) + async updateBotList() { + try { + const { data: json } = await this.http.axiosRef.get( + 'https://cdn.jsdelivr.net/gh/atmire/COUNTER-Robots@master/COUNTER_Robots_list.json', + ) + + writeFileSync(localBotListDataFilePath, JSON.stringify(json), { + encoding: 'utf-8', + flag: 'w+', + }) + + return json + } catch { + this.logger.warn('更新 Bot 列表错误') + } + } +} diff --git a/src/processors/helper/helper.module.ts b/src/processors/helper/helper.module.ts index 6049b2f9..09a8939c 100644 --- a/src/processors/helper/helper.module.ts +++ b/src/processors/helper/helper.module.ts @@ -1,16 +1,22 @@ import { Global, Module, Provider } from '@nestjs/common' +import { ScheduleModule } from '@nestjs/schedule' import { CountingService } from './helper.counting.service' +import { CronService } from './helper.cron.service' import { EmailService } from './helper.email.service' import { HttpService } from './helper.http.service' import { ImageService } from './helper.image.service' - const providers: Provider[] = [ EmailService, HttpService, ImageService, + CronService, CountingService, ] -@Module({ imports: [], providers: providers, exports: providers }) +@Module({ + imports: [ScheduleModule.forRoot()], + providers: providers, + exports: providers, +}) @Global() export class HelperModule {}