feat: anayzle module

package.json

@@ -76,11 +76,12 @@
     "nodemailer": "^6.6.3",
     "passport": "^0.4.1",
     "passport-jwt": "^4.0.0",
+    "pluralize": "*",
     "redis": "3.1.2",
     "reflect-metadata": "^0.1.13",
     "rxjs": "^7.3.0",
-    "pluralize": "*",
-    "snakecase-keys": "^4.0.2"
+    "snakecase-keys": "^4.0.2",
+    "ua-parser-js": "^0.7.28"
   },
   "devDependencies": {
     "@innei-util/eslint-config-ts": "^0.2.3",
@@ -100,6 +101,7 @@
     "@types/passport-local": "^1.0.34",
     "@types/redis": "2.8.31",
     "@types/supertest": "^2.0.11",
+    "@types/ua-parser-js": "^0.7.36",
     "cross-env": "^7.0.3",
     "eslint": "^7.32.0",
     "fastify": "*",

pnpm-lock.yaml

@@ -30,6 +30,7 @@ specifiers:
   '@types/passport-local': ^1.0.34
   '@types/redis': 2.8.31
   '@types/supertest': ^2.0.11
+  '@types/ua-parser-js': ^0.7.36
   argv: ^0.0.2
   axios: '*'
   bcrypt: ^5.0.1
@@ -77,6 +78,7 @@ specifiers:
   ts-node: ^10.2.1
   tsconfig-paths: ^3.10.1
   typescript: ^4.3.5
+  ua-parser-js: ^0.7.28
   webpack: ^5.51.1
   webpack-node-externals: ^3.0.0
 
@@ -123,6 +125,7 @@ dependencies:
   reflect-metadata: 0.1.13
   rxjs: 7.3.0
   snakecase-keys: 4.0.2
+  ua-parser-js: 0.7.28
 
 devDependencies:
   '@innei-util/eslint-config-ts': 0.2.3_typescript@4.4.2
@@ -142,6 +145,7 @@ devDependencies:
   '@types/passport-local': 1.0.34
   '@types/redis': 2.8.31
   '@types/supertest': 2.0.11
+  '@types/ua-parser-js': 0.7.36
   cross-env: 7.0.3
   eslint: 7.32.0
   fastify: 3.20.1
@@ -1618,6 +1622,10 @@ packages:
       '@types/superagent': 4.1.12
     dev: true
 
+  /@types/ua-parser-js/0.7.36:
+    resolution: {integrity: sha512-N1rW+njavs70y2cApeIw1vLMYXRwfBy+7trgavGuuTfOd7j1Yh7QTRc/yqsPl6ncokt72ZXuxEU0PiCp9bSwNQ==}
+    dev: true
+
   /@types/validator/13.6.3:
     resolution: {integrity: sha512-fWG42pMJOL4jKsDDZZREnXLjc3UE0R8LOJfARWYg6U966rxDT7TYejYzLnUF5cvSObGg34nd0+H2wHHU5Omdfw==}
     dev: false
@@ -6799,6 +6807,10 @@ packages:
     hasBin: true
     dev: true
 
+  /ua-parser-js/0.7.28:
+    resolution: {integrity: sha512-6Gurc1n//gjp9eQNXjD9O3M/sMwVtN5S8Lv9bvOYBfKfDNiIIhqiyi01vMBO45u4zkDE420w/e0se7Vs+sIg+g==}
+    dev: false
+
   /unbox-primitive/1.0.1:
     resolution: {integrity: sha512-tZU/3NqK3dA5gpE1KtyiJUrEB0lxnGkMFHptJ7q6ewdZ8s12QrODwNbhIJStmJkd1QDXa1NRA8aF2A1zk/Ypyw==}
     dependencies:

src/app.module.ts

@@ -17,6 +17,7 @@ import {
 import { AnalyzeMiddleware } from './common/middlewares/analyze.middleware'
 import { SkipBrowserDefaultRequestMiddleware } from './common/middlewares/favicon.middleware'
 import { SecurityMiddleware } from './common/middlewares/security.middleware'
+import { AnalyzeModule } from './modules/analyze/analyze.module'
 import { AuthModule } from './modules/auth/auth.module'
 import { RolesGuard } from './modules/auth/roles.guard'
 import { CategoryModule } from './modules/category/category.module'
@@ -39,6 +40,7 @@ import { HelperModule } from './processors/helper/helper.module'
 @Module({
   imports: [
     DbModule,
+    InitModule,
     CacheModule,
     ConfigModule.forRoot({
       envFilePath: [
@@ -51,20 +53,20 @@ import { HelperModule } from './processors/helper/helper.module'
       isGlobal: true,
     }),
 
-    InitModule,
-    UserModule,
-    PostModule,
-    NoteModule,
-    PageModule,
-    CategoryModule,
-    ProjectModule,
-    SayModule,
-    LinkModule,
+    AnalyzeModule,
     AuthModule,
-    UserModule,
+    CategoryModule,
     CommentModule,
     ConfigsModule,
+    LinkModule,
+    NoteModule,
     OptionModule,
+    PageModule,
+    PostModule,
+    ProjectModule,
+    SayModule,
+    UserModule,
+    UserModule,
 
     GatewayModule,
     HelperModule,

src/modules/analyze/analyze.controller.ts

@@ -0,0 +1,170 @@
+import { Controller, Delete, Get, HttpCode, Query } from '@nestjs/common'
+import { ApiTags } from '@nestjs/swagger'
+import dayjs from 'dayjs'
+import { Auth } from '~/common/decorator/auth.decorator'
+import { Paginator } from '~/common/decorator/http.decorator'
+import { RedisKeys } from '~/constants/cache.constant'
+import { CacheService } from '~/processors/cache/cache.service'
+import { PagerDto } from '~/shared/dto/pager.dto'
+import { getRedisKey } from '~/utils/redis.util'
+import { getTodayEarly, getWeekStart } from '~/utils/time.util'
+import { AnalyzeDto } from './analyze.dto'
+import { AnalyzeService } from './analyze.service'
+
+@Controller('analyze')
+@ApiTags('Analyze Routes')
+@Auth()
+export class AnalyzeController {
+  constructor(
+    private readonly service: AnalyzeService,
+    private readonly cacheService: CacheService,
+  ) {}
+
+  @Get('/')
+  @Paginator
+  async getAnalyze(@Query() query: AnalyzeDto & Partial<PagerDto>) {
+    const { from, to = new Date(), page = 1, size = 50 } = query
+
+    const data = await this.service.getRangeAnalyzeData(from, to, {
+      limit: size | 0,
+      page,
+    })
+
+    return data
+  }
+
+  @Get('/today')
+  @Paginator
+  async getAnalyzeToday(@Query() query: Partial<PagerDto>) {
+    const { page = 1, size = 50 } = query
+    const today = new Date()
+    const todayEarly = getTodayEarly(today)
+    return await this.service.getRangeAnalyzeData(todayEarly, today, {
+      limit: ~~size,
+      page,
+    })
+  }
+
+  @Get('/week')
+  @Paginator
+  async getAnalyzeWeek(@Query() query: Partial<PagerDto>) {
+    const { page = 1, size = 50 } = query
+    const today = new Date()
+    const weekStart = getWeekStart(today)
+    return await this.service.getRangeAnalyzeData(weekStart, today, {
+      limit: size,
+      page,
+    })
+  }
+
+  @Get('/aggregate')
+  async getFragment() {
+    const day = await this.service.getIpAndPvAggregate('day', true)
+
+    const now = new Date()
+    const nowHour = now.getHours()
+    const dayData = Array(24)
+      .fill(undefined)
+      .map((v, i) => {
+        return [
+          {
+            hour: i === nowHour ? '现在' : i + '时',
+            key: 'ip',
+            value: day[i.toString().padStart(2, '0')]?.ip || 0,
+          },
+          {
+            hour: i === nowHour ? '现在' : i + '时',
+            key: 'pv',
+            value: day[i.toString().padStart(2, '0')]?.pv || 0,
+          },
+        ]
+      })
+    const all = (await this.service.getIpAndPvAggregate('all')) as any[]
+
+    const weekData = all
+      .slice(0, 7)
+      .map((item) => {
+        const date =
+          '周' +
+          ['日', '一', '二', '三', '四', '五', '六'][
+            dayjs(item.date).get('day')
+          ]
+        return [
+          {
+            day: date,
+            key: 'ip',
+            value: item.ip,
+          },
+          {
+            day: date,
+            key: 'pv',
+            value: item.pv,
+          },
+        ]
+      })
+      .reverse()
+
+    const monthData = all
+      .slice(0, 30)
+      .map((item) => {
+        return [
+          {
+            date: item.date.split('-').slice(1, 3).join('-'),
+            key: 'ip',
+            value: item.ip,
+          },
+          {
+            date: item.date.split('-').slice(1, 3).join('-'),
+            key: 'pv',
+            value: item.pv,
+          },
+        ]
+      })
+      .reverse()
+
+    const paths = await this.service.getRangeOfTopPathVisitor()
+
+    const total = await this.service.getCallTime()
+
+    return {
+      today: dayData.flat(1),
+      weeks: weekData.flat(1),
+      months: monthData.flat(1),
+      paths: paths.slice(50),
+
+      total,
+      today_ips: await this.service.getTodayAccessIp(),
+    }
+  }
+
+  @Get('/like')
+  async getTodayLikedArticle() {
+    const client = this.cacheService.getClient()
+    const keys = await client.keys(getRedisKey(RedisKeys.Like, '*mx_like*'))
+    return await Promise.all(
+      keys.map(async (key) => {
+        const id = key.split('_').pop()
+        const json = await client.get(id)
+        return {
+          [id]: (
+            JSON.parse(json) as {
+              ip: string
+              created: string
+            }[]
+          ).sort(
+            (a, b) =>
+              new Date(a.created).getTime() - new Date(b.created).getTime(),
+          ),
+        }
+      }),
+    )
+  }
+
+  @Delete('/')
+  @HttpCode(204)
+  async clearAnalyze(@Query() query: AnalyzeDto) {
+    const { from = new Date('2020-01-01'), to = new Date() } = query
+    await this.service.cleanAnalyzeRange({ from, to })
+    return
+  }
+}

src/modules/analyze/analyze.dto.ts

@@ -0,0 +1,17 @@
+import { ApiProperty } from '@nestjs/swagger'
+import { Transform } from 'class-transformer'
+import { IsDate, IsOptional } from 'class-validator'
+
+export class AnalyzeDto {
+  @Transform(({ value: v }) => new Date(parseInt(v)))
+  @IsOptional()
+  @IsDate()
+  @ApiProperty({ type: 'string' })
+  from?: Date
+
+  @Transform(({ value: v }) => new Date(parseInt(v)))
+  @IsOptional()
+  @IsDate()
+  @ApiProperty({ type: 'string' })
+  to?: Date
+}

src/modules/analyze/analyze.model.ts

@@ -0,0 +1,30 @@
+import { ApiHideProperty } from '@nestjs/swagger'
+import { modelOptions, prop, Severity } from '@typegoose/typegoose'
+import { SchemaTypes } from 'mongoose'
+import { BaseModel } from '~/shared/model/base.model'
+import type { UAParser } from 'ua-parser-js'
+
+@modelOptions({
+  schemaOptions: {
+    timestamps: {
+      createdAt: 'timestamp',
+      updatedAt: false,
+    },
+  },
+  options: {
+    customName: 'Analyze',
+    allowMixed: Severity.ALLOW,
+  },
+})
+export class AnalyzeModel extends BaseModel {
+  @prop()
+  ip?: string
+
+  @prop({ type: SchemaTypes.Mixed })
+  ua: UAParser
+
+  @prop()
+  path?: string
+  @ApiHideProperty()
+  timestamp: Date
+}

src/modules/analyze/analyze.module.ts

@@ -0,0 +1,10 @@
+import { Module } from '@nestjs/common'
+import { AnalyzeController } from './analyze.controller'
+import { AnalyzeService } from './analyze.service'
+
+@Module({
+  controllers: [AnalyzeController],
+  exports: [AnalyzeService],
+  providers: [AnalyzeService],
+})
+export class AnalyzeModule {}

src/modules/analyze/analyze.service.ts

@@ -0,0 +1,292 @@
+import { Injectable } from '@nestjs/common'
+import { ReturnModelType } from '@typegoose/typegoose'
+import dayjs from 'dayjs'
+import { merge } from 'lodash'
+import { InjectModel } from 'nestjs-typegoose'
+import { RedisKeys } from '~/constants/cache.constant'
+import { CacheService } from '~/processors/cache/cache.service'
+import { getRedisKey } from '~/utils/redis.util'
+import { OptionModel } from '../configs/configs.model'
+import { AnalyzeModel } from './analyze.model'
+
+@Injectable()
+export class AnalyzeService {
+  constructor(
+    @InjectModel(OptionModel)
+    private readonly options: ReturnModelType<typeof OptionModel>,
+    @InjectModel(AnalyzeModel)
+    private readonly analyzeModel: MongooseModel<AnalyzeModel>,
+    private readonly cacheService: CacheService,
+  ) {}
+
+  public get model() {
+    return this.analyzeModel
+  }
+
+  async getRangeAnalyzeData(
+    from = new Date(new Date().getTime() - 1000 * 24 * 3600 * 3),
+    to = new Date(),
+    options?: {
+      limit?: number
+      page?: number
+    },
+  ) {
+    const { limit = 50, page = 1 } = options || {}
+    const condition = {
+      $and: [
+        {
+          timestamp: {
+            $gte: from,
+          },
+        },
+        {
+          timestamp: {
+            $lte: to,
+          },
+        },
+      ],
+    }
+
+    return await this.analyzeModel.paginate(condition, {
+      sort: { timestamp: -1 },
+      page,
+      limit,
+    })
+  }
+
+  async getCallTime() {
+    const callTime =
+      (
+        await this.options
+          .findOne({
+            name: 'apiCallTime',
+          })
+          .lean()
+      )?.value || 0
+
+    const uv =
+      (
+        await this.options
+          .findOne({
+            name: 'uv',
+          })
+          .lean()
+      )?.value || 0
+
+    return { callTime, uv }
+  }
+  async cleanAnalyzeRange(range: { from?: Date; to?: Date }) {
+    const { from, to } = range
+
+    await this.analyzeModel.deleteMany({
+      $and: [
+        {
+          timestamp: {
+            $gte: from,
+          },
+        },
+        {
+          timestamp: {
+            $lte: to,
+          },
+        },
+      ],
+    })
+  }
+
+  async getIpAndPvAggregate(
+    type: 'day' | 'week' | 'month' | 'all',
+    returnObj?: boolean,
+  ) {
+    let cond = {}
+    const now = dayjs()
+    const beforeDawn = now.set('minute', 0).set('second', 0).set('hour', 0)
+    switch (type) {
+      case 'day': {
+        cond = {
+          timestamp: {
+            $gte: beforeDawn.toDate(),
+          },
+        }
+        break
+      }
+      case 'month': {
+        cond = {
+          timestamp: {
+            $gte: beforeDawn.set('day', -30).toDate(),
+          },
+        }
+        break
+      }
+      case 'week': {
+        cond = {
+          timestamp: {
+            $gte: beforeDawn.set('day', -7).toDate(),
+          },
+        }
+        break
+      }
+      case 'all':
+      default: {
+        break
+      }
+    }
+
+    const res = await this.analyzeModel.aggregate([
+      { $match: cond },
+      {
+        $project: {
+          _id: 1,
+          timestamp: 1,
+          hour: {
+            $dateToString: {
+              format: '%H',
+              date: { $subtract: ['$timestamp', 0] },
+              timezone: '+08:00',
+            },
+          },
+          date: {
+            $dateToString: {
+              format: '%Y-%m-%d',
+              date: { $subtract: ['$timestamp', 0] },
+              timezone: '+08:00',
+            },
+          },
+        },
+      },
+      {
+        $group: {
+          _id: type === 'day' ? '$hour' : '$date',
+
+          pv: {
+            $sum: 1,
+          },
+        },
+      },
+      {
+        $project: {
+          _id: 0,
+          ...(type === 'day' ? { hour: '$_id' } : { date: '$_id' }),
+          pv: 1,
+        },
+      },
+      {
+        $sort: {
+          date: -1,
+        },
+      },
+    ])
+
+    const res2 = await this.analyzeModel.aggregate([
+      { $match: cond },
+      {
+        $project: {
+          _id: 1,
+          timestamp: 1,
+          ip: 1,
+          hour: {
+            $dateToString: {
+              format: '%H',
+              date: { $subtract: ['$timestamp', 0] },
+              timezone: '+08:00',
+            },
+          },
+          date: {
+            $dateToString: {
+              format: '%Y-%m-%d',
+              date: { $subtract: ['$timestamp', 0] },
+              timezone: '+08:00',
+            },
+          },
+        },
+      },
+      {
+        $group: {
+          _id:
+            type === 'day'
+              ? { ip: '$ip', hour: '$hour' }
+              : { ip: '$ip', date: '$date' },
+        },
+      },
+
+      {
+        $group: {
+          _id: type === 'day' ? '$_id.hour' : '$_id.date',
+          ip: {
+            $sum: 1,
+          },
+        },
+      },
+      {
+        $project: {
+          _id: 0,
+          ...(type === 'day' ? { hour: '$_id' } : { date: '$_id' }),
+          ip: 1,
+        },
+      },
+      {
+        $sort: {
+          date: -1,
+        },
+      },
+    ])
+    const arr = merge(res, res2)
+    if (returnObj) {
+      const obj = {}
+      for (const item of arr) {
+        obj[item.hour || item.date] = item
+      }
+
+      return obj
+    }
+    return arr
+  }
+
+  async getRangeOfTopPathVisitor(from?: Date, to?: Date): Promise<any[]> {
+    from = from ?? new Date(new Date().getTime() - 1000 * 24 * 3600 * 7)
+    to = to ?? new Date()
+
+    const pipeline = [
+      {
+        $match: {
+          timestamp: {
+            $gte: from,
+            $lte: to,
+          },
+        },
+      },
+      {
+        $group: {
+          _id: '$path',
+          count: {
+            $sum: 1,
+          },
+        },
+      },
+
+      {
+        $sort: {
+          count: -1,
+        },
+      },
+      {
+        $project: {
+          _id: 0,
+          path: '$_id',
+          count: 1,
+        },
+      },
+    ]
+
+    const res = await this.analyzeModel.aggregate(pipeline).exec()
+
+    return res
+  }
+
+  async getTodayAccessIp(): Promise<string[]> {
+    const redis = this.cacheService.getClient()
+    const fromRedisIps = await redis.get(getRedisKey(RedisKeys.Access, 'ips'))
+    const ips = fromRedisIps ? JSON.parse(fromRedisIps) : []
+    return ips
+  }
+}

src/processors/database/database.module.ts

@@ -1,6 +1,7 @@
 import { Global, Module } from '@nestjs/common'
 import { TypegooseModule } from 'nestjs-typegoose'
 import { MONGO_DB } from '~/app.config'
+import { AnalyzeModel } from '~/modules/analyze/analyze.model'
 import { CommentModel } from '~/modules/comment/comment.model'
 import { OptionModel } from '~/modules/configs/configs.model'
 import { LinkModel } from '~/modules/link/link.model'
@@ -23,6 +24,7 @@ const models = TypegooseModule.forFeature([
   LinkModel,
   ProjectModel,
   SayModel,
+  AnalyzeModel,
 ])
 @Module({
   imports: [

src/utils/time.util.ts

@@ -0,0 +1,24 @@
+import dayjs from 'dayjs'
+
+export const getTodayEarly = (today: Date) =>
+  dayjs(today).set('hour', 0).set('minute', 0).set('millisecond', 0).toDate()
+
+export const getWeekStart = (today: Date) =>
+  dayjs(today)
+    .set('day', 0)
+    .set('hour', 0)
+    .set('millisecond', 0)
+    .set('minute', 0)
+    .toDate()
+
+export const getMonthStart = (today: Date) =>
+  dayjs(today)
+    .set('date', 1)
+    .set('hour', 0)
+    .set('minute', 0)
+    .set('millisecond', 0)
+    .toDate()
+
+export function getMonthLength(month: number, year: number) {
+  return new Date(year, month, 0).getDate()
+}