feat: snippet and function refactor (#692)

.eslintrc.js

@@ -24,4 +24,12 @@ module.exports = {
       },
     ],
   },
+  overrides: [
+    {
+      files: ['src/migration/**/*.ts'],
+      rules: {
+        'import/no-default-export': 'off',
+      },
+    },
+  ],
 }

package.json

@@ -49,7 +49,6 @@
     "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand",
     "test:e2e": "jest --config ./test/jest-e2e.js",
     "test:all": "jest --config ./test/jest-e2e.js && jest",
-    "patch": "node bin/patch.js",
     "docs": "npx @compodoc/compodoc -p tsconfig.json -s -d docs",
     "changelog": "conventional-changelog -p angular -i CHANGELOG.md -s"
   },

patch/bootstrap.js

@@ -1,36 +0,0 @@
-const { MongoClient, Db } = require('mongodb')
-const path = require('path')
-const ts = require('typescript')
-const { readFileSync, writeFileSync } = require('fs')
-const appConfigFile = path.join(__dirname, '../src/app.config.ts')
-
-Object.assign(global, { isDev: false })
-
-const result = ts.transpileModule(
-  readFileSync(appConfigFile, { encoding: 'utf-8' }),
-  {
-    compilerOptions: { module: ts.ModuleKind.CommonJS, esModuleInterop: true },
-  },
-)
-const complied = result.outputText
-
-writeFileSync(appConfigFile.replace(/\.ts$/, '.js'), complied)
-
-const MONGO_DB = require('../src/app.config').MONGO_DB
-
-/**
- *
- * @param {(db: Db) => Promise<any>} cb
- */
-async function bootstrap(cb) {
-  const client = new MongoClient(`mongodb://${MONGO_DB.host}:${MONGO_DB.port}`)
-  await client.connect()
-  const db = client.db(MONGO_DB.dbName)
-
-  await cb(db)
-
-  await client.close()
-  process.exit(0)
-}
-
-module.exports = exports.bootstrap = bootstrap

patch/index.js

@@ -1 +0,0 @@
-require('./bootstrap')

patch/v3.30.0.js

@@ -1,7 +0,0 @@
-// patch for version lower than v2.0.0-alpha.1
-
-const bootstrap = require('./bootstrap')
-
-bootstrap(async (db) => {
-  await db.collection('users').updateMany({}, { $unset: { authCode: 1 } })
-})

paw.paw

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0ae55a0cf67aeb4f4a6b97931340ac1b11d609568ede07633b373463a0a20078
-size 94384
+oid sha256:50f786ff188b98559dd5f4c2cafe6b8e71cbbc03ce7743aefe5b297dd667dc16
+size 95577

src/bootstrap.ts

@@ -3,7 +3,7 @@ import { performance } from 'perf_hooks'
 import wcmatch from 'wildcard-match'
 
 import { LogLevel, Logger, ValidationPipe } from '@nestjs/common'
-import { NestFactory } from '@nestjs/core'
+import { ContextIdFactory, NestFactory } from '@nestjs/core'
 import { NestFastifyApplication } from '@nestjs/platform-fastify'
 
 import { API_VERSION, CROSS_DOMAIN, PORT, isMainProcess } from './app.config'
@@ -12,7 +12,9 @@ import { fastifyApp } from './common/adapters/fastify.adapter'
 import { RedisIoAdapter } from './common/adapters/socket.adapter'
 import { SpiderGuard } from './common/guard/spider.guard'
 import { LoggingInterceptor } from './common/interceptors/logging.interceptor'
+import { AggregateByTenantContextIdStrategy } from './common/strategies/context.strategy'
 import { isTest } from './global/env.global'
+import { migrateDatabase } from './migration/migrate'
 import { MyLogger } from './processors/logger/logger.service'
 
 const Origin: false | string[] = Array.isArray(CROSS_DOMAIN.allowedOrigins)
@@ -23,6 +25,7 @@ declare const module: any
 
 export async function bootstrap() {
   process.title = `Mix Space (${cluster.isPrimary ? 'master' : 'worker'})`
+  await migrateDatabase()
   const app = await NestFactory.create<NestFastifyApplication>(
     AppModule,
     fastifyApp,
@@ -72,6 +75,8 @@ export async function bootstrap() {
   app.useGlobalGuards(new SpiderGuard())
   !isTest && app.useWebSocketAdapter(new RedisIoAdapter(app))
 
+  ContextIdFactory.apply(new AggregateByTenantContextIdStrategy())
+
   if (isDev) {
     const { DocumentBuilder, SwaggerModule } = await import('@nestjs/swagger')
     const options = new DocumentBuilder()

src/common/interceptors/analyze.interceptor.ts

@@ -53,7 +53,8 @@ export class AnalyzeInterceptor implements NestInterceptor {
     if (!request) {
       return call$
     }
-    const method = request.routerMethod.toUpperCase()
+
+    const method = request.method.toUpperCase()
     if (method !== 'GET') {
       return call$
     }

src/common/strategies/context.strategy.ts

@@ -0,0 +1,29 @@
+import { FastifyRequest } from 'fastify'
+
+import {
+  ContextId,
+  ContextIdFactory,
+  ContextIdStrategy,
+  HostComponentInfo,
+} from '@nestjs/core'
+
+const tenants = new Map<string, ContextId>()
+
+export class AggregateByTenantContextIdStrategy implements ContextIdStrategy {
+  attach(contextId: ContextId, request: FastifyRequest) {
+    const tenantId = request.headers['x-tenant-id'] as string
+    let tenantSubTreeId: ContextId
+
+    if (tenants.has(tenantId)) {
+      // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+      tenantSubTreeId = tenants.get(tenantId)!
+    } else {
+      tenantSubTreeId = ContextIdFactory.create()
+      tenants.set(tenantId, tenantSubTreeId)
+    }
+
+    // If tree is not durable, return the original "contextId" object
+    return (info: HostComponentInfo) =>
+      info.isTreeDurable ? tenantSubTreeId : contextId
+  }
+}

src/migration/history.ts

@@ -0,0 +1,5 @@
+import v200Alpha1 from './version/v2.0.0-alpha.1'
+import v3330 from './version/v3.30.0'
+import v3360 from './version/v3.36.0'
+
+export default [v200Alpha1, v3330, v3360]

src/migration/migrate.ts

@@ -0,0 +1,44 @@
+import { existsSync } from 'fs-extra'
+import { MongoClient } from 'mongodb'
+
+import * as APP_CONFIG from '../app.config'
+import { DATA_DIR } from '../constants/path.constant'
+import VersionList from './history'
+
+const { MONGO_DB } = APP_CONFIG
+
+export async function migrateDatabase() {
+  if (!APP_CONFIG.isMainProcess) {
+    return
+  }
+
+  const migrateFilePath = path.join(DATA_DIR, 'migrate')
+  existsSync(migrateFilePath) || (await fs.writeFile(migrateFilePath, ''))
+
+  const migrateRecord = await fs.readFile(migrateFilePath, 'utf-8')
+
+  const migratedSet = new Set(migrateRecord.split('\n'))
+
+  const client = new MongoClient(`mongodb://${MONGO_DB.host}:${MONGO_DB.port}`)
+  await client.connect()
+  const db = client.db(MONGO_DB.dbName)
+
+  for (const migrate of VersionList) {
+    if (migratedSet.has(migrate.name)) {
+      continue
+    }
+
+    consola.log(`[Database] migrate ${migrate.name}`)
+
+    await migrate(db)
+    migratedSet.add(migrate.name)
+  }
+
+  await fs.unlink(migrateFilePath)
+
+  await fs.writeFile(migrateFilePath, [...migratedSet].join('\n'), {
+    flag: 'w+',
+  })
+
+  await client.close()
+}

src/migration/version/v2.0.0-alpha.1.ts

@@ -1,8 +1,7 @@
 // patch for version lower than v2.0.0-alpha.1
+import { Db } from 'mongodb'
 
-const bootstrap = require('./bootstrap')
-
-bootstrap(async (db) => {
+export default (async function v200Alpha1(db: Db) {
   return await Promise.all([
     ['notes', 'posts'].map(async (collectionName) => {
       return db

src/migration/version/v3.30.0.ts

@@ -0,0 +1,6 @@
+// patch for version lower than v3.30.0
+import { Db } from 'mongodb'
+
+export default (async function v3330(db: Db) {
+  await db.collection('users').updateMany({}, { $unset: { authCode: 1 } })
+})

src/migration/version/v3.36.0.ts

@@ -0,0 +1,18 @@
+// patch for version lower than v3.36.0
+import { Db } from 'mongodb'
+
+export default (async function v3360(db: Db) {
+  await db.collection('snippets').updateMany(
+    {
+      type: 'function',
+      method: undefined,
+      enable: undefined,
+    },
+    {
+      $set: {
+        method: 'GET',
+        enable: true,
+      },
+    },
+  )
+})

src/modules/debug/debug.controller.ts

@@ -1,4 +1,4 @@
-import { Body, Post, Query, Request, Response } from '@nestjs/common'
+import { Body, Get, Post, Query, Request, Response } from '@nestjs/common'
 
 import { ApiController } from '~/common/decorator/api-controller.decorator'
 import { HTTPDecorators } from '~/common/decorator/http.decorator'
@@ -9,6 +9,7 @@ import { EventManagerService } from '~/processors/helper/helper.event.service'
 import { createMockedContextResponse } from '../serverless/mock-response.util'
 import { ServerlessService } from '../serverless/serverless.service'
 import { SnippetModel, SnippetType } from '../snippet/snippet.model'
+import { DebugService } from './debug.service'
 
 @ApiName
 @ApiController('debug')
@@ -16,8 +17,16 @@ export class DebugController {
   constructor(
     private readonly serverlessService: ServerlessService,
     private readonly eventManager: EventManagerService,
+
+    private readonly debugService: DebugService,
   ) {}
 
+  @Get('/test')
+  test() {
+    this.debugService.test()
+    return ''
+  }
+
   @Post('/events')
   async sendEvent(
     @Query('type') type: 'web' | 'admin' | 'all',

src/modules/debug/debug.service.ts

@@ -1,6 +1,13 @@
-import { Injectable } from '@nestjs/common'
+import { Inject, Injectable, Scope } from '@nestjs/common'
+import { REQUEST } from '@nestjs/core'
 
-@Injectable()
+@Injectable({ scope: Scope.REQUEST })
 export class DebugService {
-  constructor() {}
+  constructor(@Inject(REQUEST) private req) {
+    console.log('DebugService created')
+  }
+
+  test() {
+    console.log('this.req', this.req.method)
+  }
 }

src/modules/serverless/serverless.controller.ts

@@ -1,6 +1,7 @@
 import { FastifyReply, FastifyRequest } from 'fastify'
 
 import {
+  All,
   CacheTTL,
   ForbiddenException,
   Get,
@@ -58,7 +59,7 @@ export class ServerlessController {
     return this.runServerlessFunction(param, isMaster, req, reply)
   }
 
-  @Get('/:reference/:name')
+  @All('/:reference/:name')
   @HTTPDecorators.Bypass
   async runServerlessFunction(
     @Param() param: ServerlessReferenceDto,
@@ -67,15 +68,25 @@ export class ServerlessController {
     @Request() req: FastifyRequest,
     @Response() reply: FastifyReply,
   ) {
+    const requestMethod = req.method.toUpperCase()
     const { name, reference } = param
-    const snippet = await this.serverlessService.model.findOne({
-      name,
-      reference,
-      type: SnippetType.Function,
-    })
+    const snippet = await this.serverlessService.model
+      .findOne({
+        name,
+        reference,
+        type: SnippetType.Function,
+        method: requestMethod,
+      })
+      .lean()
+
+    const notExistMessage = 'serverless function is not exist or not enabled'
 
     if (!snippet) {
-      throw new NotFoundException('serverless function is not exist')
+      throw new NotFoundException(notExistMessage)
+    }
+
+    if (snippet.method !== requestMethod || !snippet.enable) {
+      throw new NotFoundException(notExistMessage)
     }
 
     if (snippet.private && !isMaster) {

src/modules/serverless/serverless.service.ts

@@ -375,7 +375,7 @@ export class ServerlessService {
   private lruCache = new LRUCache({
     max: 100,
     ttl: 10 * 1000,
-    maxSize: 5000,
+    maxSize: 50000,
     sizeCalculation: (value: string, key: string) => {
       return value.length + key.length
     },
@@ -563,8 +563,12 @@ export class ServerlessService {
       const { body } = ast.program as t.Program
 
       const hasEntryFunction = body.some(
-        (node) => t.isFunction(node) && node.id && node.id.name === 'handler',
+        (node: t.Declaration) =>
+          (node.type == 'ExportDefaultDeclaration' &&
+            isHandlerFunction(node.declaration)) ||
+          isHandlerFunction(node),
       )
+
       return hasEntryFunction
     } catch (e) {
       if (isDev) {
@@ -572,5 +576,17 @@ export class ServerlessService {
       }
       return e.message?.split('\n').at(0)
     }
+
+    function isHandlerFunction(
+      node:
+        | t.Declaration
+        | t.FunctionDeclaration
+        | t.ClassDeclaration
+        | t.TSDeclareFunction
+        | t.Expression,
+    ): boolean {
+      // @ts-expect-error
+      return t.isFunction(node) && node?.id?.name === 'handler'
+    }
   }
 }

src/modules/snippet/snippet.controller.ts

@@ -3,10 +3,12 @@ import {
   Delete,
   ForbiddenException,
   Get,
+  NotFoundException,
   Param,
   Post,
   Put,
   Query,
+  UnprocessableEntityException,
 } from '@nestjs/common'
 
 import { ApiController } from '~/common/decorator/api-controller.decorator'
@@ -15,7 +17,6 @@ import { BanInDemo } from '~/common/decorator/demo.decorator'
 import { HTTPDecorators } from '~/common/decorator/http.decorator'
 import { ApiName } from '~/common/decorator/openapi.decorator'
 import { IsMaster } from '~/common/decorator/role.decorator'
-import { CacheService } from '~/processors/redis/cache.service'
 import { MongoIdDto } from '~/shared/dto/id.dto'
 import { PagerDto } from '~/shared/dto/pager.dto'
 import { transformDataToPaginate } from '~/transformers/paginate.transformer'
@@ -27,10 +28,7 @@ import { SnippetService } from './snippet.service'
 @ApiName
 @ApiController('snippets')
 export class SnippetController {
-  constructor(
-    private readonly snippetService: SnippetService,
-    private readonly redisService: CacheService,
-  ) {}
+  constructor(private readonly snippetService: SnippetService) {}
 
   @Get('/')
   @Auth()
@@ -50,9 +48,9 @@ export class SnippetController {
     )
   }
 
-  @Post('/more')
+  @Post('/import')
   @Auth()
-  async createMore(@Body() body: SnippetMoreDto) {
+  async importSnippets(@Body() body: SnippetMoreDto) {
     const { snippets } = body
     const tasks = snippets.map((snippet) => this.create(snippet))
 
@@ -78,6 +76,51 @@ export class SnippetController {
     return snippet
   }
 
+  @Get('/group')
+  @Auth()
+  @HTTPDecorators.Paginator
+  async getGroup(@Query() query: PagerDto) {
+    const { page, size = 30 } = query
+    return this.snippetService.model.aggregatePaginate(
+      this.snippetService.model.aggregate([
+        {
+          $group: {
+            _id: {
+              reference: '$reference',
+            },
+            count: { $sum: 1 },
+          },
+        },
+        {
+          $sort: {
+            '_id.reference': 1,
+          },
+        },
+        {
+          $project: {
+            _id: 0,
+            reference: '$_id.reference',
+            count: 1,
+          },
+        },
+      ]),
+      {
+        page,
+        limit: size,
+      },
+    )
+  }
+
+  @Get('/group/:reference')
+  @Auth()
+  async getGroupByReference(@Param('reference') reference: string) {
+    if (typeof reference !== 'string') {
+      throw new UnprocessableEntityException('reference should be string')
+    }
+
+    return this.snippetService.model.find({ reference }).lean()
+  }
+
   @Post('/aggregate')
   @Auth()
   async aggregate(@Body() body: any) {
@@ -123,16 +166,19 @@ export class SnippetController {
     }
 
     const snippet = await this.snippetService.getSnippetByName(name, reference)
+
+    if (snippet.type === SnippetType.Function) {
+      throw new NotFoundException()
+    }
+
     if (snippet.private && !isMaster) {
       throw new ForbiddenException('snippet is private')
     }
 
-    if (snippet.type !== SnippetType.Function) {
-      return this.snippetService.attachSnippet(snippet).then((res) => {
-        this.snippetService.cacheSnippet(res, res.data)
-        return res.data
-      })
-    }
+    return this.snippetService.attachSnippet(snippet).then((res) => {
+      this.snippetService.cacheSnippet(res, res.data)
+      return res.data
+    })
   }
 
   @Put('/:id')

src/modules/snippet/snippet.model.ts

@@ -8,8 +8,9 @@ import {
   Matches,
   MaxLength,
 } from 'class-validator'
+import aggregatePaginate from 'mongoose-aggregate-paginate-v2'
 
-import { index, modelOptions, prop } from '@typegoose/typegoose'
+import { index, modelOptions, plugin, prop } from '@typegoose/typegoose'
 
 import { BaseModel } from '~/shared/model/base.model'
 
@@ -32,6 +33,7 @@ export enum SnippetType {
     },
   },
 })
+@plugin(aggregatePaginate)
 @index({ name: 1, reference: 1 })
 @index({ type: 1 })
 export class SnippetModel extends BaseModel {
@@ -81,4 +83,15 @@ export class SnippetModel extends BaseModel {
   @IsString()
   @IsOptional()
   schema?: string
+
+  // for function
+  @prop()
+  @IsEnum(['GET', 'POST', 'PUT', 'DELETE', 'PATCH'])
+  @IsOptional()
+  method?: string
+
+  @prop()
+  @IsBoolean()
+  @IsOptional()
+  enable?: boolean
 }

src/modules/snippet/snippet.service.ts

@@ -1,5 +1,6 @@
 import { load } from 'js-yaml'
 import JSON5 from 'json5'
+import { AggregatePaginateModel, Document } from 'mongoose'
 
 import {
   BadRequestException,
@@ -21,7 +22,8 @@ import { SnippetModel, SnippetType } from './snippet.model'
 export class SnippetService {
   constructor(
     @InjectModel(SnippetModel)
-    private readonly snippetModel: MongooseModel<SnippetModel>,
+    private readonly snippetModel: MongooseModel<SnippetModel> &
+      AggregatePaginateModel<SnippetModel & Document>,
     @Inject(forwardRef(() => ServerlessService))
     private readonly serverlessService: ServerlessService,
     private readonly cacheService: CacheService,
@@ -32,26 +34,42 @@ export class SnippetService {
   }
 
   async create(model: SnippetModel) {
+    if (model.type === SnippetType.Function) {
+      model.method ??= 'GET'
+      model.enable ??= true
+    }
     const isExist = await this.model.countDocuments({
       name: model.name,
       reference: model.reference || 'root',
+      method: model.method,
     })
 
     if (isExist) {
       throw new BadRequestException('snippet is exist')
     }
     // 验证正确类型
-    await this.validateType(model)
+    await this.validateTypeAndCleanup(model)
     return await this.model.create({ ...model, created: new Date() })
   }
 
   async update(id: string, model: SnippetModel) {
-    await this.validateType(model)
+    await this.validateTypeAndCleanup(model)
     delete model.created
     const old = await this.model.findById(id).lean()
+
     if (!old) {
       throw new NotFoundException()
     }
+
+    if (
+      old.type === SnippetType.Function &&
+      model.type !== SnippetType.Function
+    ) {
+      throw new BadRequestException(
+        '`type` is not allowed to change if this snippet set to Function type.',
+      )
+    }
+
     await this.deleteCachedSnippet(old.reference, old.name)
     return await this.model.findByIdAndUpdate(
       id,
@@ -68,7 +86,7 @@ export class SnippetService {
     await this.deleteCachedSnippet(doc.reference, doc.name)
   }
 
-  private async validateType(model: SnippetModel) {
+  private async validateTypeAndCleanup(model: SnippetModel) {
     switch (model.type) {
       case SnippetType.JSON: {
         try {
@@ -113,6 +131,14 @@ export class SnippetService {
         break
       }
     }
+    // TODO refactor
+    // cleanup
+    if (model.type !== SnippetType.Function) {
+      const deleteKeys: (keyof SnippetModel)[] = ['enable', 'method']
+      deleteKeys.forEach((key) => {
+        Reflect.deleteProperty(model, key)
+      })
+    }
   }
 
   async getSnippetById(id: string) {