feat(ai): introduce deep reading functionality and refactor AI module

- Added `AiDeepReadingController`, `AiDeepReadingService`, and related DTOs and models to support deep reading capabilities using AI.
- Integrated new methods for generating and managing deep readings, including error handling and caching mechanisms.
- Updated `AiModule` to include dependencies for deep reading services and controllers.
- Removed the obsolete `AiAgentModule` and its associated test controller.
- Enhanced configuration options to enable or disable deep reading features.

Signed-off-by: Innei <tukon479@gmail.com>

apps/core/global.d.ts

@@ -1,8 +1,5 @@
-import type { Document, PaginateModel } from 'mongoose'
-
-import '@mx-space/compiled/zx-global'
-
 import type { ModelType } from '@typegoose/typegoose/lib/types'
+import type { Document, PaginateModel } from 'mongoose'
 
 declare global {
   export type KV<T = any> = Record<string, T>

apps/core/src/app.config.test.ts

@@ -1,5 +1,7 @@
 import type { AxiosRequestConfig } from 'axios'
 
+import { argv } from '@mx-space/compiled'
+
 export const PORT = process.env.PORT || 2333
 export const API_VERSION = 2
 

apps/core/src/bootstrap.ts

@@ -6,6 +6,7 @@ import type { LogLevel } from '@nestjs/common'
 import type { NestFastifyApplication } from '@nestjs/platform-fastify'
 
 import { Logger } from '@innei/pretty-logger-nestjs'
+import { chalk } from '@mx-space/compiled'
 import { NestFactory } from '@nestjs/core'
 
 import { CROSS_DOMAIN, DEBUG_MODE, PORT } from './app.config'

apps/core/src/common/exceptions/biz.exception.ts

@@ -4,11 +4,11 @@ import { ErrorCode, ErrorCodeEnum } from '~/constants/error-code.constant'
 
 export class BusinessException extends HttpException {
   public bizCode: ErrorCodeEnum
-  constructor(code: ErrorCodeEnum, extraMessage?: string)
+  constructor(code: ErrorCodeEnum, extraMessage?: string, trace?: string)
   constructor(message: string)
   constructor(...args: any[]) {
     let status = 500
-    const [bizCode, extraMessage] = args as any
+    const [bizCode, extraMessage, trace] = args as any
     const bizError = ErrorCode[bizCode] || []
     const [message] = bizError
     status = bizError[1] ?? status
@@ -27,6 +27,8 @@ export class BusinessException extends HttpException {
       status,
     )
 
+    this.stack = trace
+
     this.bizCode = typeof bizCode === 'number' ? bizCode : ErrorCodeEnum.Default
   }
 }

apps/core/src/common/filters/any-exception.filter.ts

@@ -1,8 +1,10 @@
+import { createWriteStream } from 'node:fs'
 import { resolve } from 'node:path'
 import type { ArgumentsHost, ExceptionFilter } from '@nestjs/common'
 import type { FastifyReply, FastifyRequest } from 'fastify'
 import type { WriteStream } from 'node:fs'
 
+import { chalk } from '@mx-space/compiled'
 import {
   Catch,
   HttpException,
@@ -122,7 +124,7 @@ export class AllExceptionsFilter implements ExceptionFilter {
       if (!isDev) {
         this.errorLogPipe =
           this.errorLogPipe ??
-          fs.createWriteStream(resolve(LOG_DIR, 'error.log'), {
+          createWriteStream(resolve(LOG_DIR, 'error.log'), {
             flags: 'a+',
             encoding: 'utf-8',
           })

apps/core/src/common/interceptors/logging.interceptor.ts

@@ -13,6 +13,7 @@ import type {
 } from '@nestjs/common'
 import type { Observable } from 'rxjs'
 
+import { chalk } from '@mx-space/compiled'
 import { Injectable, Logger, SetMetadata } from '@nestjs/common'
 
 import { HTTP_REQUEST_TIME } from '~/constants/meta.constant'

apps/core/src/constants/db.constant.ts

@@ -16,6 +16,7 @@ export const RECENTLY_COLLECTION_NAME = 'recentlies'
 export const ANALYZE_COLLECTION_NAME = 'analyzes'
 export const WEBHOOK_EVENT_COLLECTION_NAME = 'webhook_events'
 export const AI_SUMMARY_COLLECTION_NAME = 'ai_summaries'
+export const AI_DEEP_READING_COLLECTION_NAME = 'ai_deep_readings'
 
 export const USER_COLLECTION_NAME = 'users'
 export enum CollectionRefTypes {

apps/core/src/global/index.global.ts

@@ -1,5 +1,5 @@
 import cluster from 'node:cluster'
-import { mkdirSync, writeFileSync } from 'node:fs'
+import { existsSync, mkdirSync, writeFileSync } from 'node:fs'
 
 import { Logger } from '@nestjs/common'
 
@@ -18,7 +18,8 @@ import { cwd, isDev } from './env.global'
 import { registerJSONGlobal } from './json.global'
 
 import './dayjs.global'
-import '@mx-space/compiled/zx-global'
+
+import { $, chalk } from '@mx-space/compiled'
 
 // 建立目录
 function createAppFolders() {
@@ -37,7 +38,7 @@ function createAppFolders() {
     Logger.log(chalk.blue(`文件回收站目录已经建好：${STATIC_FILE_TRASH_DIR}`))
 
     const packageJSON = `${DATA_DIR}/package.json`
-    const hasPKG = fs.existsSync(packageJSON)
+    const hasPKG = existsSync(packageJSON)
     if (!hasPKG) {
       writeFileSync(packageJSON, '{"name":"modules"}', {
         flag: 'a',

apps/core/src/main.ts

@@ -1,6 +1,9 @@
 #!env node
 // register global
 import cluster from 'node:cluster'
+import { cpus } from 'node:os'
+
+import { argv } from '@mx-space/compiled'
 
 import { DEBUG_MODE } from './app.config'
 import { registerForMemoryDump } from './dump'
@@ -51,7 +54,7 @@ async function main() {
   DEBUG_MODE.memoryDump && registerForMemoryDump()
   if (CLUSTER.enable) {
     Cluster.register(
-      Number.parseInt(CLUSTER.workers) || os.cpus().length,
+      Number.parseInt(CLUSTER.workers) || cpus().length,
       bootstrap,
     )
   } else {

apps/core/src/modules/ai/ai-agent/ai-agent.module.ts

@@ -1,13 +0,0 @@
-import { forwardRef, Module } from '@nestjs/common'
-
-import { McpModule } from '../../mcp/mcp.module'
-import { AiModule } from '../ai.module'
-import { AIAgentService } from './ai-agent.service'
-import { TestController } from './test.controller'
-
-@Module({
-  imports: [McpModule, forwardRef(() => AiModule)],
-  providers: [AIAgentService],
-  controllers: isDev ? [TestController] : [],
-})
-export class AiAgentModule {}

apps/core/src/modules/ai/ai-agent/test.controller.ts

@@ -1,15 +0,0 @@
-import { Controller, Get } from '@nestjs/common'
-
-import { AIAgentService } from './ai-agent.service'
-
-@Controller('ai/test')
-export class TestController {
-  constructor(private readonly aiAgentService: AIAgentService) {}
-
-  @Get('/')
-  async test() {
-    return this.aiAgentService.runWithTools(
-      'Posts Id: 65e99e28eb677674816310c7 主要了写了什么内容',
-    )
-  }
-}

apps/core/src/modules/ai/ai-deep-reading/ai-deep-reading.controller.ts

@@ -0,0 +1,60 @@
+import { Delete, Get, Param, Post, Query } from '@nestjs/common'
+
+import { ApiController } from '~/common/decorators/api-controller.decorator'
+import { Auth } from '~/common/decorators/auth.decorator'
+import { BizException } from '~/common/exceptions/biz.exception'
+import { ErrorCodeEnum } from '~/constants/error-code.constant'
+import { MongoIdDto } from '~/shared/dto/id.dto'
+import { PagerDto } from '~/shared/dto/pager.dto'
+
+import { ConfigsService } from '../../configs/configs.service'
+import { GetDeepReadingQueryDto } from './ai-deep-reading.dto'
+import { AiDeepReadingService } from './ai-deep-reading.service'
+
+@ApiController('ai/deep-readings')
+export class AiDeepReadingController {
+  constructor(
+    private readonly service: AiDeepReadingService,
+    private readonly configService: ConfigsService,
+  ) {}
+
+  @Get('/')
+  @Auth()
+  async getDeepReadings(@Query() query: PagerDto) {
+    return this.service.getAllDeepReadings(query)
+  }
+
+  @Post('/generate/:id')
+  @Auth()
+  async generateDeepReading(@Param() params: MongoIdDto) {
+    return this.service.generateDeepReadingByOpenAI(params.id)
+  }
+
+  @Delete('/:id')
+  @Auth()
+  async deleteDeepReading(@Param() params: MongoIdDto) {
+    return this.service.deleteDeepReadingInDb(params.id)
+  }
+
+  @Get('/article/:id')
+  async getArticleDeepReading(
+    @Param() params: MongoIdDto,
+    @Query() query: GetDeepReadingQueryDto,
+  ) {
+    const dbStored = await this.service.getDeepReadingByArticleId(params.id)
+
+    const aiConfig = await this.configService.get('ai')
+    if (!dbStored && !query.onlyDb) {
+      const shouldGenerate = aiConfig?.enableDeepReading
+      if (shouldGenerate) {
+        return this.service.generateDeepReadingByOpenAI(params.id)
+      }
+    }
+
+    if (!dbStored && !aiConfig.enableDeepReading) {
+      throw new BizException(ErrorCodeEnum.AINotEnabled)
+    }
+
+    return dbStored
+  }
+}

apps/core/src/modules/ai/ai-deep-reading/ai-deep-reading.dto.ts

@@ -0,0 +1,38 @@
+import { IsBoolean, IsOptional, IsString } from 'class-validator'
+
+import { TransformBoolean } from '~/common/decorators/transform-boolean.decorator'
+
+class BaseLangQueryDto {
+  @IsString()
+  @IsOptional()
+  lang: string
+}
+
+export class GenerateAiDeepReadingDto extends BaseLangQueryDto {
+  @IsString()
+  refId: string
+}
+
+export class GetDeepReadingQueryDto extends BaseLangQueryDto {
+  @IsOptional()
+  @IsBoolean()
+  @TransformBoolean()
+  onlyDb?: boolean
+}
+
+export class UpdateDeepReadingDto {
+  @IsString()
+  deepReading: string
+
+  @IsString()
+  @IsOptional()
+  criticalAnalysis?: string
+
+  @IsString({ each: true })
+  @IsOptional()
+  keyPoints?: string[]
+
+  @IsString()
+  @IsOptional()
+  content?: string
+}

apps/core/src/modules/ai/ai-deep-reading/ai-deep-reading.model.ts

@@ -0,0 +1,32 @@
+import mongoose from 'mongoose'
+
+import { modelOptions, prop } from '@typegoose/typegoose'
+
+import { AI_DEEP_READING_COLLECTION_NAME } from '~/constants/db.constant'
+import { BaseModel } from '~/shared/model/base.model'
+
+@modelOptions({
+  options: {
+    customName: AI_DEEP_READING_COLLECTION_NAME,
+  },
+})
+export class AIDeepReadingModel extends BaseModel {
+  @prop({
+    required: true,
+  })
+  hash: string
+
+  @prop({
+    required: true,
+  })
+  refId: string
+
+  @prop({ type: [String] })
+  keyPoints?: mongoose.Types.Array<string>
+
+  @prop()
+  criticalAnalysis?: string
+
+  @prop()
+  content?: string
+}

apps/core/src/modules/ai/ai-deep-reading/ai-deep-reading.service.ts

@@ -0,0 +1,337 @@
+import { AgentExecutor, createOpenAIToolsAgent } from 'langchain/agents'
+import { StructuredOutputParser } from 'langchain/output_parsers'
+import type { PagerDto } from '~/shared/dto/pager.dto'
+
+import { ToolDefinition } from '@langchain/core/language_models/base'
+import { JsonOutputParser } from '@langchain/core/output_parsers'
+import { JsonOutputToolsParser } from '@langchain/core/output_parsers/openai_tools'
+import {
+  ChatPromptTemplate,
+  HumanMessagePromptTemplate,
+  MessagesPlaceholder,
+  SystemMessagePromptTemplate,
+} from '@langchain/core/prompts'
+import { DynamicStructuredTool } from '@langchain/core/tools'
+import { z } from '@mx-space/compiled/zod'
+import { Injectable, Logger } from '@nestjs/common'
+import { OnEvent } from '@nestjs/event-emitter'
+
+import { BizException } from '~/common/exceptions/biz.exception'
+import { CollectionRefTypes } from '~/constants/db.constant'
+import { ErrorCodeEnum } from '~/constants/error-code.constant'
+import { DatabaseService } from '~/processors/database/database.service'
+import { RedisService } from '~/processors/redis/redis.service'
+import { InjectModel } from '~/transformers/model.transformer'
+import { md5 } from '~/utils/tool.util'
+
+import { ConfigsService } from '../../configs/configs.service'
+import { AiService } from '../ai.service'
+import { AIDeepReadingModel } from './ai-deep-reading.model'
+
+@Injectable()
+export class AiDeepReadingService {
+  private readonly logger: Logger
+  constructor(
+    @InjectModel(AIDeepReadingModel)
+    private readonly aiDeepReadingModel: MongooseModel<AIDeepReadingModel>,
+    private readonly databaseService: DatabaseService,
+    private readonly configService: ConfigsService,
+
+    private readonly redisService: RedisService,
+    private readonly aiService: AiService,
+  ) {
+    this.logger = new Logger(AiDeepReadingService.name)
+  }
+
+  private cachedTaskId2AiPromise = new Map<string, Promise<any>>()
+
+  private async deepReadingAgentChain(articleId: string) {
+    const {
+      ai: { enableDeepReading },
+    } = await this.configService.waitForConfigReady()
+
+    if (!enableDeepReading) {
+      throw new BizException(ErrorCodeEnum.AINotEnabled)
+    }
+
+    const article = await this.databaseService.findGlobalById(articleId)
+    if (!article || article.type === CollectionRefTypes.Recently) {
+      throw new BizException(ErrorCodeEnum.ContentNotFoundCantProcess)
+    }
+
+    const llm = await this.aiService.getOpenAiChain({
+      maxTokens: 8192,
+    })
+
+    const dataModel = {
+      keyPoints: [] as string[],
+      criticalAnalysis: '',
+      content: '',
+    }
+
+    // 创建分析文章的工具
+    const tools = [
+      new DynamicStructuredTool({
+        name: 'deep_reading',
+        description: `获取深度阅读内容`,
+        schema: z.object({}),
+        func: async () => {
+          const llm = await this.aiService.getOpenAiChain({
+            maxTokens: 1024 * 10,
+          })
+
+          const result = await llm
+            .bind({
+              tool_choice: {
+                type: 'function',
+                function: { name: 'deep_reading' },
+              },
+              tools: [
+                {
+                  name: 'deep_reading',
+                  type: 'function',
+                  function: {
+                    name: 'deep_reading',
+                    parameters: {
+                      type: 'object',
+                      properties: {
+                        content: {
+                          type: 'string',
+                          description: '深度阅读内容',
+                        },
+                      },
+                      required: ['content'],
+                    },
+                    description: `创建一个全面的深度阅读Markdown文本，保持文章的原始结构但提供扩展的解释和见解。
+内容应该：
+1. 遵循原文的流程和主要论点
+2. 包含原文的所有关键技术细节
+3. 扩展未充分解释的复杂概念
+4. 在需要的地方提供额外背景和解释
+5. 保持文章的原始语调和语言风格
+6. 使用适当的Markdown格式，包括标题、代码块、列表等
+7. 输出的语言必须与原文的语言匹配`,
+                  },
+                },
+              ],
+            })
+
+            .pipe(new JsonOutputToolsParser())
+            .invoke([
+              {
+                content: `分析以下文章：${article.document.text}\n\n创建一个全面的深度阅读Markdown文本，保持文章的原始结构但提供扩展的解释和见解。`,
+                role: 'system',
+              },
+            ])
+            .then((result: any[]) => {
+              const content = result[0]?.args?.content
+              dataModel.content = content
+              return content
+            })
+          return result
+        },
+      }),
+      new DynamicStructuredTool({
+        name: 'save_key_points',
+        description: '保存关键点到数据库',
+        schema: z.object({
+          keyPoints: z.array(z.string()).describe('关键点数组'),
+        }),
+        func: async (data: { keyPoints: string[] }) => {
+          dataModel.keyPoints = data.keyPoints
+          return '关键点已保存'
+        },
+      }),
+
+      new DynamicStructuredTool({
+        name: 'save_critical_analysis',
+        description: '保存批判性分析到数据库',
+        schema: z.object({
+          criticalAnalysis: z.string().describe('批判性分析'),
+        }),
+        func: async (data: { criticalAnalysis: string }) => {
+          dataModel.criticalAnalysis = data.criticalAnalysis
+          return '批判性分析已保存'
+        },
+      }),
+    ]
+
+    // 创建Agent提示模板
+    const prompt = ChatPromptTemplate.fromMessages([
+      SystemMessagePromptTemplate.fromTemplate(
+        `你是一个专门进行文章深度阅读的AI助手，需要分析文章并提供详细的解读。
+分析过程：
+1. 首先提取文章关键点，然后使用 save_key_points 保存到数据库
+2. 然后进行批判性分析，包括文章的优点、缺点和改进建议，然后使用 save_critical_analysis 保存到数据库
+3. 最后使用 deep_reading 生成完整的深度阅读内容
+4. 返回完整结果，包括关键点、批判性分析和深度阅读内容
+`,
+      ),
+      new MessagesPlaceholder('chat_history'),
+      ['human', '文章标题: {article_title}\n文章内容: {article_content}'],
+      new MessagesPlaceholder('agent_scratchpad'),
+    ])
+
+    // 创建Agent
+    const agent = await createOpenAIToolsAgent({
+      llm,
+      tools,
+      prompt,
+    })
+
+    // 创建Agent执行器
+    const executor = new AgentExecutor({
+      agent,
+      tools,
+      verbose: false,
+    })
+
+    try {
+      // 执行Agent
+      await executor.invoke({
+        article_title: article.document.title,
+        article_content: article.document.text,
+        chat_history: [],
+      })
+
+      // 返回结果
+      return {
+        keyPoints: dataModel.keyPoints,
+        criticalAnalysis: dataModel.criticalAnalysis,
+        content: dataModel.content,
+      }
+    } catch (error) {
+      this.logger.error(`Agent execution error: ${error.message}`)
+      throw new BizException(
+        ErrorCodeEnum.AIException,
+        error.message,
+        error.stack,
+      )
+    }
+  }
+
+  async generateDeepReadingByOpenAI(articleId: string) {
+    const {
+      ai: { enableDeepReading },
+    } = await this.configService.waitForConfigReady()
+
+    if (!enableDeepReading) {
+      throw new BizException(ErrorCodeEnum.AINotEnabled)
+    }
+
+    const article = await this.databaseService.findGlobalById(articleId)
+    if (!article) {
+      throw new BizException(ErrorCodeEnum.ContentNotFoundCantProcess)
+    }
+
+    if (article.type === CollectionRefTypes.Recently) {
+      throw new BizException(ErrorCodeEnum.ContentNotFoundCantProcess)
+    }
+
+    const taskId = `ai:deepreading:${articleId}`
+    try {
+      if (this.cachedTaskId2AiPromise.has(taskId)) {
+        return this.cachedTaskId2AiPromise.get(taskId)
+      }
+      const redis = this.redisService.getClient()
+
+      const isProcessing = await redis.get(taskId)
+
+      if (isProcessing === 'processing' && !isDev) {
+        throw new BizException(ErrorCodeEnum.AIProcessing)
+      }
+
+      const taskPromise = handle.bind(this)(
+        articleId,
+        article.document.text,
+        article.document.title,
+      ) as Promise<any>
+
+      this.cachedTaskId2AiPromise.set(taskId, taskPromise)
+      return await taskPromise
+
+      async function handle(
+        this: AiDeepReadingService,
+        id: string,
+        text: string,
+      ) {
+        // 处理时间增加到5分钟
+        await redis.set(taskId, 'processing', 'EX', 300)
+
+        const result = await this.deepReadingAgentChain(id)
+
+        await redis.del(taskId)
+
+        const contentMd5 = md5(text)
+
+        const doc = await this.aiDeepReadingModel.create({
+          hash: contentMd5,
+          refId: id,
+          keyPoints: result.keyPoints,
+          criticalAnalysis: result.criticalAnalysis,
+          content: result.content,
+        })
+
+        return doc
+      }
+    } catch (error) {
+      console.error(error)
+      this.logger.error(
+        `OpenAI encountered an error processing article ${articleId}: ${error.message}`,
+      )
+
+      throw new BizException(
+        ErrorCodeEnum.AIException,
+        error.message,
+        error.stack,
+      )
+    } finally {
+      this.cachedTaskId2AiPromise.delete(taskId)
+    }
+  }
+
+  async getAllDeepReadings(pager: PagerDto) {
+    const { page, size } = pager
+    const deepReadings = await this.aiDeepReadingModel.paginate(
+      {},
+      {
+        page,
+        limit: size,
+        sort: {
+          created: -1,
+        },
+        lean: true,
+        leanWithId: true,
+      },
+    )
+    return deepReadings
+  }
+
+  async getDeepReadingByArticleId(articleId: string) {
+    const article = await this.databaseService.findGlobalById(articleId)
+    if (!article) {
+      throw new BizException(ErrorCodeEnum.ContentNotFound)
+    }
+
+    const docs = await this.aiDeepReadingModel.find({
+      refId: articleId,
+    })
+
+    return docs[0]
+  }
+
+  async deleteDeepReadingByArticleId(articleId: string) {
+    await this.aiDeepReadingModel.deleteMany({
+      refId: articleId,
+    })
+  }
+
+  async deleteDeepReadingInDb(id: string) {
+    return this.aiDeepReadingModel.findByIdAndDelete(id)
+  }
+
+  @OnEvent('article.delete')
+  async handleDeleteArticle(event: { id: string }) {
+    await this.deleteDeepReadingByArticleId(event.id)
+  }
+}

apps/core/src/modules/ai/ai.constants.ts

@@ -1,4 +1,5 @@
 export const DEFAULT_SUMMARY_LANG = 'zh'
+
 export const LANGUAGE_CODE_TO_NAME = {
   ar: 'Arabic',
   bg: 'Bulgarian',

apps/core/src/modules/ai/ai.module.ts

@@ -1,6 +1,9 @@
 import { forwardRef, Module } from '@nestjs/common'
 
-import { AiAgentModule } from './ai-agent/ai-agent.module'
+import { McpModule } from '../mcp/mcp.module'
+import { AIAgentService } from './ai-agent/ai-agent.service'
+import { AiDeepReadingController } from './ai-deep-reading/ai-deep-reading.controller'
+import { AiDeepReadingService } from './ai-deep-reading/ai-deep-reading.service'
 import { AiSummaryController } from './ai-summary/ai-summary.controller'
 import { AiSummaryService } from './ai-summary/ai-summary.service'
 import { AiWriterController } from './ai-writer/ai-writer.controller'
@@ -8,9 +11,19 @@ import { AiWriterService } from './ai-writer/ai-writer.service'
 import { AiService } from './ai.service'
 
 @Module({
-  imports: [forwardRef(() => AiAgentModule)],
-  providers: [AiSummaryService, AiService, AiWriterService],
-  controllers: [AiSummaryController, AiWriterController],
+  imports: [forwardRef(() => McpModule)],
+  providers: [
+    AiSummaryService,
+    AiService,
+    AiWriterService,
+    AiDeepReadingService,
+    AIAgentService,
+  ],
+  controllers: [
+    AiSummaryController,
+    AiWriterController,
+    AiDeepReadingController,
+  ],
   exports: [AiService],
 })
 export class AiModule {}

apps/core/src/modules/ai/ai.service.ts

@@ -10,7 +10,7 @@ import { ConfigsService } from '../configs/configs.service'
 export class AiService {
   constructor(private readonly configService: ConfigsService) {}
 
-  public async getOpenAiChain() {
+  public async getOpenAiChain(options?: { maxTokens?: number }) {
     const {
       ai: { openAiKey, openAiEndpoint, openAiPreferredModel },
     } = await this.configService.waitForConfigReady()
@@ -24,6 +24,7 @@ export class AiService {
       configuration: {
         baseURL: openAiEndpoint || void 0,
       },
+      maxTokens: options?.maxTokens,
     })
   }
 }

apps/core/src/modules/backup/backup.service.ts

@@ -1,10 +1,11 @@
-import { existsSync, statSync } from 'node:fs'
+import { createReadStream, existsSync, statSync } from 'node:fs'
 import { readdir, readFile, rm, writeFile } from 'node:fs/promises'
-import { join, resolve } from 'node:path'
+import path, { join, resolve } from 'node:path'
 import { flatten } from 'lodash'
 import { mkdirp } from 'mkdirp'
 
 import { PutObjectCommand, S3Client } from '@aws-sdk/client-s3'
+import { $, cd } from '@mx-space/compiled'
 import {
   BadRequestException,
   Injectable,
@@ -148,7 +149,7 @@ export class BackupService {
 
   async getFileStream(dirname: string) {
     const path = this.checkBackupExist(dirname)
-    const stream = fs.createReadStream(path)
+    const stream = createReadStream(path)
 
     return stream
   }
@@ -179,7 +180,7 @@ export class BackupService {
 
   async restore(restoreFilePath: string) {
     await this.backup()
-    const isExist = fs.existsSync(restoreFilePath)
+    const isExist = existsSync(restoreFilePath)
     if (!isExist) {
       throw new InternalServerErrorException('备份文件不存在')
     }

apps/core/src/modules/configs/configs.default.ts

@@ -90,6 +90,7 @@ export const generateDefaultConfig: () => IConfig = () => ({
     openAiPreferredModel: 'gpt-4o-mini',
     openAiKey: '',
     aiSummaryTargetLanguage: 'auto',
+    enableDeepReading: false,
   },
   oauth: {
     providers: [],

apps/core/src/modules/configs/configs.dto.ts

@@ -448,6 +448,13 @@ export class AIDto {
   })
   enableAutoGenerateSummary: boolean
 
+  @IsBoolean()
+  @IsOptional()
+  @JSONSchemaToggleField('开启 AI 深度阅读', {
+    description: '是否开启调用 AI 去生成深度阅读',
+  })
+  enableDeepReading: boolean
+
   @IsString()
   @IsOptional()
   @JSONSchemaPlainField('AI 摘要目标语言', {

apps/core/src/modules/configs/configs.service.ts

@@ -17,7 +17,7 @@ import { RedisService } from '~/processors/redis/redis.service'
 import { SubPubBridgeService } from '~/processors/redis/subpub.service'
 import { InjectModel } from '~/transformers/model.transformer'
 import { getRedisKey } from '~/utils/redis.util'
-import { camelcaseKeys } from '~/utils/tool.util'
+import { camelcaseKeys, sleep } from '~/utils/tool.util'
 
 import { generateDefaultConfig } from './configs.default'
 import { OAuthDto } from './configs.dto'

apps/core/src/modules/dependency/dependency.controller.ts

@@ -1,6 +1,8 @@
 import { readFile } from 'node:fs/promises'
+import path from 'node:path'
 import { Observable } from 'rxjs'
 
+import { chalk } from '@mx-space/compiled'
 import { BadRequestException, Get, Query, Sse } from '@nestjs/common'
 
 import { ApiController } from '~/common/decorators/api-controller.decorator'

apps/core/src/modules/file/file.controller.ts

@@ -1,4 +1,5 @@
 import fs from 'node:fs/promises'
+import path from 'node:path'
 import { FastifyReply, FastifyRequest } from 'fastify'
 import { lookup } from 'mime-types'
 

apps/core/src/modules/file/file.service.ts

@@ -1,8 +1,9 @@
 import { createWriteStream } from 'node:fs'
-import { resolve } from 'node:path'
+import path, { resolve } from 'node:path'
 import type { Readable } from 'node:stream'
 import type { FileType } from './file.type'
 
+import { fs } from '@mx-space/compiled'
 import {
   BadRequestException,
   Injectable,

apps/core/src/modules/health/sub-controller/log.controller.ts

@@ -1,5 +1,8 @@
+import os from 'node:os'
+import path from 'node:path'
 import type { Readable } from 'form-data'
 
+import { fs } from '@mx-space/compiled'
 import {
   BadRequestException,
   Delete,

apps/core/src/modules/pageproxy/pageproxy.controller.ts

@@ -1,6 +1,6 @@
 import { createReadStream, existsSync, statSync } from 'node:fs'
 import fs from 'node:fs/promises'
-import { extname, join } from 'node:path'
+import path, { extname, join } from 'node:path'
 import { render } from 'ejs'
 import { FastifyReply, FastifyRequest } from 'fastify'
 import { lookup } from 'mime-types'

apps/core/src/modules/pageproxy/pageproxy.service.ts

@@ -1,3 +1,4 @@
+import path from 'node:path'
 import { URL } from 'node:url'
 import { parseHTML } from 'linkedom'
 

apps/core/src/modules/update/update.controller.ts

@@ -1,7 +1,9 @@
+import path from 'node:path'
 import { isSemVer } from 'class-validator'
 import { catchError, lastValueFrom, Observable } from 'rxjs'
 import { lt, major, minor } from 'semver'
 
+import { chalk, fs } from '@mx-space/compiled'
 import { Query, Sse } from '@nestjs/common'
 
 import { dashboard } from '~/../package.json'

apps/core/src/modules/update/update.service.ts

@@ -1,9 +1,11 @@
 import { appendFile, rm, writeFile } from 'node:fs/promises'
+import path from 'node:path'
 import { inspect } from 'node:util'
 import axios from 'axios'
 import { catchError, Observable } from 'rxjs'
 import type { Subscriber } from 'rxjs'
 
+import { chalk } from '@mx-space/compiled'
 import { Injectable } from '@nestjs/common'
 
 import { dashboard } from '~/../package.json'

apps/core/src/modules/user/user.service.ts

@@ -16,7 +16,7 @@ import {
 } from '~/common/exceptions/biz.exception'
 import { ErrorCodeEnum } from '~/constants/error-code.constant'
 import { InjectModel } from '~/transformers/model.transformer'
-import { getAvatar } from '~/utils/tool.util'
+import { getAvatar, sleep } from '~/utils/tool.util'
 
 import { AuthService } from '../auth/auth.service'
 import { UserModel } from './user.model'

apps/core/src/processors/database/database.models.ts

@@ -1,4 +1,5 @@
 import { ActivityModel } from '~/modules/activity/activity.model'
+import { AIDeepReadingModel } from '~/modules/ai/ai-deep-reading/ai-deep-reading.model'
 import { AISummaryModel } from '~/modules/ai/ai-summary/ai-summary.model'
 import { AnalyzeModel } from '~/modules/analyze/analyze.model'
 import { AuthnModel } from '~/modules/authn/authn.model'
@@ -27,6 +28,7 @@ import { getProviderByTypegooseClass } from '~/transformers/model.transformer'
 export const databaseModels = [
   ActivityModel,
   AISummaryModel,
+  AIDeepReadingModel,
   AnalyzeModel,
   AuthnModel,
   CategoryModel,

apps/core/src/processors/gateway/admin/events.gateway.ts

@@ -1,3 +1,4 @@
+import { createReadStream } from 'node:fs'
 import { resolve } from 'node:path'
 import { Socket } from 'socket.io'
 import type {
@@ -51,8 +52,7 @@ export class AdminEventsGateway
 
     this.subscribeSocketToHandlerMap.set(client, handler)
     if (prevLog) {
-      const stream = fs
-        .createReadStream(resolve(LOG_DIR, getTodayLogFilePath()), {
+      const stream = createReadStream(resolve(LOG_DIR, getTodayLogFilePath()), {
         encoding: 'utf-8',
         highWaterMark: 32 * 1024,
       })

apps/core/src/processors/helper/helper.cron.service.ts

@@ -1,3 +1,4 @@
+import { statSync } from 'node:fs'
 import { readdir, rm } from 'node:fs/promises'
 import { join } from 'node:path'
 import dayjs from 'dayjs'
@@ -106,7 +107,7 @@ export class CronService {
     const rmTaskArr = [] as Promise<any>[]
     for (const file of files) {
       const filePath = join(LOG_DIR, file)
-      const state = fs.statSync(filePath)
+      const state = statSync(filePath)
       const oldThanWeek = dayjs().diff(state.mtime, 'day') > 7
       if (oldThanWeek) {
         rmTaskArr.push(rm(filePath))

apps/core/src/processors/helper/helper.http.service.ts

@@ -3,6 +3,7 @@ import axios from 'axios'
 import axiosRetry, { exponentialDelay } from 'axios-retry'
 import type { AxiosInstance, AxiosRequestConfig } from 'axios'
 
+import { chalk } from '@mx-space/compiled'
 import { Injectable, Logger } from '@nestjs/common'
 
 import { AXIOS_CONFIG, DEBUG_MODE } from '~/app.config'

apps/core/src/utils/database.util.ts

@@ -4,6 +4,8 @@
 import mongoose from 'mongoose'
 import type { CollectionRefTypes } from '~/constants/db.constant'
 
+import { chalk } from '@mx-space/compiled'
+
 import { MONGO_DB } from '~/app.config'
 import {
   NOTE_COLLECTION_NAME,

apps/core/src/utils/system.util.ts

@@ -1,7 +1,11 @@
 import cdp, { exec } from 'node:child_process'
 import { builtinModules } from 'node:module'
+import os from 'node:os'
+import path from 'node:path'
 import { promisify } from 'node:util'
 
+import { $, cd, fs } from '@mx-space/compiled'
+
 export async function getFolderSize(folderPath: string) {
   try {
     return (