From 57e4a8b932033764b954d9e679c315ae2b5c7527 Mon Sep 17 00:00:00 2001 From: lborv Date: Sat, 11 Oct 2025 16:21:03 +0300 Subject: [PATCH] feat: add logging functionality with LoggerService; implement log entity and controller; enhance query processing with logging support --- .../database/database.manager.controller.ts | 13 ++++ .../database/database.manager.service.ts | 23 ++++++- src/migrations/1760184857707-adminToken.ts | 17 +++++ src/migrations/1760188157352-logs.ts | 21 ++++++ src/project/project.controller.ts | 7 +- src/project/project.module.ts | 2 + src/project/project.service.ts | 17 +++-- src/query/base/base-query.controller.ts | 28 +++++++- src/query/command/command.controller.ts | 7 +- src/query/executer/query.executer.service.ts | 9 ++- src/query/handler/query.controller.ts | 7 +- src/query/logger/entities/log.entity.ts | 17 +++++ src/query/logger/logger.controller.ts | 32 +++++++++ src/query/logger/logger.service.ts | 66 +++++++++++++++++++ src/query/logger/logger.types.ts | 26 ++++++++ src/query/query.module.ts | 20 +++++- src/queue/processors/query.processor.ts | 7 +- src/vm/vm.class.ts | 31 +++++++-- src/vm/vm.constants.ts | 2 + 19 files changed, 328 insertions(+), 24 deletions(-) create mode 100644 src/migrations/1760184857707-adminToken.ts create mode 100644 src/migrations/1760188157352-logs.ts create mode 100644 src/query/logger/entities/log.entity.ts create mode 100644 src/query/logger/logger.controller.ts create mode 100644 src/query/logger/logger.service.ts create mode 100644 src/query/logger/logger.types.ts diff --git a/src/databaseManager/database/database.manager.controller.ts b/src/databaseManager/database/database.manager.controller.ts index 5df5e46..78e6752 100644 --- a/src/databaseManager/database/database.manager.controller.ts +++ b/src/databaseManager/database/database.manager.controller.ts @@ -39,6 +39,19 @@ export class DatabaseManagerController { ); } + @Get("tables/:databaseId") + getTables(@Param("databaseId") databaseId: string) { + return this.databaseManagerService.getTableList(databaseId); + } + + @Get("columns/:databaseId/:tableName") + getColumns( + @Param("databaseId") databaseId: string, + @Param("tableName") tableName: string + ) { + return this.databaseManagerService.getTableColumns(databaseId, tableName); + } + @Get("migration/up/:databaseId") migrateUp(@Param("databaseId") databaseId: string) { return this.migrationService.up(databaseId); diff --git a/src/databaseManager/database/database.manager.service.ts b/src/databaseManager/database/database.manager.service.ts index b5331aa..99903c4 100644 --- a/src/databaseManager/database/database.manager.service.ts +++ b/src/databaseManager/database/database.manager.service.ts @@ -1,4 +1,4 @@ -import { Inject, Injectable } from "@nestjs/common"; +import { forwardRef, Inject, Injectable } from "@nestjs/common"; import { InjectRepository } from "@nestjs/typeorm"; import { Database } from "../entities/database.entity"; import { Repository } from "typeorm"; @@ -13,6 +13,7 @@ export class DatabaseManagerService extends DatabaseEncryptionService { constructor( @InjectRepository(Database) private databaseRepository: Repository, + @Inject(forwardRef(() => ProjectService)) private readonly projectService: ProjectService, private readonly databaseNodeService: DatabaseNodeService, @Inject(RedisClient) @@ -146,4 +147,24 @@ export class DatabaseManagerService extends DatabaseEncryptionService { return await this.databaseRepository.save(database); } + + async getTableList(databaseId: string): Promise { + const results = await this.runQuery( + databaseId, + "SHOW TABLES;", + true /* use query user */ + ); + + return results.map((row) => Object.values(row)[0]); + } + + async getTableColumns(databaseId: string, tableName: string): Promise { + const results = await this.runQuery( + databaseId, + `SHOW COLUMNS FROM \`${tableName}\`;`, + true /* use query user */ + ); + + return results; + } } diff --git a/src/migrations/1760184857707-adminToken.ts b/src/migrations/1760184857707-adminToken.ts new file mode 100644 index 0000000..cdd6336 --- /dev/null +++ b/src/migrations/1760184857707-adminToken.ts @@ -0,0 +1,17 @@ +import { MigrationInterface, QueryRunner } from "typeorm"; + +export class AdminToken1760184857707 implements MigrationInterface { + name = "AdminToken1760184857707"; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `ALTER TABLE \`token\` CHANGE \`isActive\` \`isActive\` tinyint NOT NULL DEFAULT '1'` + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `ALTER TABLE \`token\` CHANGE \`isActive\` \`isActive\` tinyint NOT NULL DEFAULT 0` + ); + } +} diff --git a/src/migrations/1760188157352-logs.ts b/src/migrations/1760188157352-logs.ts new file mode 100644 index 0000000..770c93b --- /dev/null +++ b/src/migrations/1760188157352-logs.ts @@ -0,0 +1,21 @@ +import { MigrationInterface, QueryRunner } from "typeorm"; + +export class Logs1760188157352 implements MigrationInterface { + name = "Logs1760188157352"; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `CREATE TABLE \`log\` (\`id\` varchar(36) NOT NULL, \`traceId\` varchar(255) NOT NULL, \`content\` longtext NOT NULL, \`createdAt\` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP(), PRIMARY KEY (\`id\`)) ENGINE=InnoDB` + ); + await queryRunner.query( + `ALTER TABLE \`token\` CHANGE \`isActive\` \`isActive\` tinyint NOT NULL DEFAULT '1'` + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `ALTER TABLE \`token\` CHANGE \`isActive\` \`isActive\` tinyint NOT NULL DEFAULT 0` + ); + await queryRunner.query(`DROP TABLE \`log\``); + } +} diff --git a/src/project/project.controller.ts b/src/project/project.controller.ts index 5b6c51a..ebaaaef 100644 --- a/src/project/project.controller.ts +++ b/src/project/project.controller.ts @@ -5,7 +5,6 @@ import { AdminGuard } from "src/api/guards/admin.guard"; @Controller("project") @UseGuards(ApiTokenGuard) -@UseGuards(AdminGuard) export class ProjectController { constructor( @Inject(ProjectService) @@ -16,4 +15,10 @@ export class ProjectController { createProject(@Body() body: { name: string }) { return this.projectService.create(body.name); } + + @Put("create-without-db") + @UseGuards(AdminGuard) + createProjectWithoutDB(@Body() body: { name: string }) { + return this.projectService.create(body.name, false); + } } diff --git a/src/project/project.module.ts b/src/project/project.module.ts index 855d3b9..2b9e302 100644 --- a/src/project/project.module.ts +++ b/src/project/project.module.ts @@ -5,11 +5,13 @@ import { ProjectService } from "./project.service"; import { ProjectController } from "./project.controller"; import { ApiModule } from "src/api/api.module"; import { RedisModule } from "src/redis/redis.module"; +import { DatabaseManagerModule } from "src/databaseManager/database.manager.module"; @Module({ imports: [ forwardRef(() => ApiModule), forwardRef(() => RedisModule), + forwardRef(() => DatabaseManagerModule), TypeOrmModule.forFeature([Project]), ], controllers: [ProjectController], diff --git a/src/project/project.service.ts b/src/project/project.service.ts index a1069a4..7dd8249 100644 --- a/src/project/project.service.ts +++ b/src/project/project.service.ts @@ -1,8 +1,9 @@ -import { Inject, Injectable } from "@nestjs/common"; +import { forwardRef, Inject, Injectable } from "@nestjs/common"; import { InjectRepository } from "@nestjs/typeorm"; import { Repository } from "typeorm"; import { Project } from "./entities/project.entity"; import { RedisClient } from "src/redis/redis.service"; +import { DatabaseManagerService } from "src/databaseManager/database/database.manager.service"; @Injectable() export class ProjectService { @@ -10,12 +11,20 @@ export class ProjectService { @InjectRepository(Project) private readonly projectRepository: Repository, @Inject(RedisClient) - private readonly redisClient: RedisClient + private readonly redisClient: RedisClient, + @Inject(forwardRef(() => DatabaseManagerService)) + private readonly databaseManagerService: DatabaseManagerService ) {} - create(name: string) { + async create(name: string, createDatabase: boolean = true) { const project = this.projectRepository.create({ name }); - return this.projectRepository.save(project); + const projectSaved = await this.projectRepository.save(project); + + if (createDatabase) { + await this.databaseManagerService.createDatabase(projectSaved.id); + } + + return projectSaved; } async findById(id: string) { diff --git a/src/query/base/base-query.controller.ts b/src/query/base/base-query.controller.ts index ca5adf4..896523b 100644 --- a/src/query/base/base-query.controller.ts +++ b/src/query/base/base-query.controller.ts @@ -13,6 +13,8 @@ import { QueryHandlerService } from "../handler/query.handler.service"; import { ApiTokenGuard } from "src/api/guards/api-token.guard"; import { QueryExecuterService } from "../executer/query.executer.service"; import { QueryGuard } from "src/query/guards/query.guard"; +import { LoggerService } from "../logger/logger.service"; +import { TLogType } from "../logger/logger.types"; @UseGuards(ApiTokenGuard) export abstract class BaseQueryController { @@ -20,7 +22,9 @@ export abstract class BaseQueryController { @Inject(QueryHandlerService) protected readonly queryHandlerService: QueryHandlerService, @Inject(QueryExecuterService) - protected readonly queryExecuterService: QueryExecuterService + protected readonly queryExecuterService: QueryExecuterService, + @Inject(LoggerService) + protected readonly loggerService: LoggerService ) {} protected abstract getIsCommand(): boolean; @@ -49,9 +53,26 @@ export abstract class BaseQueryController { @Headers() headers: Record, @Res() res: Response ) { + const loggerTraceId = + headers["x-trace-id"] || LoggerService.generateTraceId(); + const log = LoggerService.log( + { + traceId: loggerTraceId, + startTime: new Date().getTime(), + payload: query, + headers: headers, + cookies: headers.cookie, + url: `/run/${id}`, + response: null, + content: [], + }, + { content: "", type: TLogType.info, timeStamp: new Date().getTime() } + ); + const queryResult = await this.queryExecuterService.runQueryQueued( id, query, + log, headers, headers.cookie.split("; ").reduce((acc, cookie) => { const [key, value] = cookie.split("="); @@ -60,6 +81,11 @@ export abstract class BaseQueryController { }, {}) ); + if (queryResult?.log) { + queryResult.log.endTime = new Date().getTime(); + await this.loggerService.create(queryResult.log.traceId, queryResult.log); + } + res.status(queryResult?.statusCode || 200); if (queryResult?.cookies) { diff --git a/src/query/command/command.controller.ts b/src/query/command/command.controller.ts index 35cf516..30e6900 100644 --- a/src/query/command/command.controller.ts +++ b/src/query/command/command.controller.ts @@ -2,6 +2,7 @@ import { Controller, Inject } from "@nestjs/common"; import { QueryHandlerService } from "../handler/query.handler.service"; import { QueryExecuterService } from "../executer/query.executer.service"; import { BaseQueryController } from "../base/base-query.controller"; +import { LoggerService } from "../logger/logger.service"; @Controller("command") export class CommandController extends BaseQueryController { @@ -9,9 +10,11 @@ export class CommandController extends BaseQueryController { @Inject(QueryHandlerService) queryHandlerService: QueryHandlerService, @Inject(QueryExecuterService) - queryExecuterService: QueryExecuterService + queryExecuterService: QueryExecuterService, + @Inject(LoggerService) + loggerService: LoggerService ) { - super(queryHandlerService, queryExecuterService); + super(queryHandlerService, queryExecuterService, loggerService); } protected getIsCommand(): boolean { diff --git a/src/query/executer/query.executer.service.ts b/src/query/executer/query.executer.service.ts index 60ba258..040516d 100644 --- a/src/query/executer/query.executer.service.ts +++ b/src/query/executer/query.executer.service.ts @@ -16,6 +16,7 @@ import { QUEUE_NAMES } from "src/queue/constants"; import { Queue, QueueEvents } from "bullmq"; import { FunctionService } from "src/query/function/function.service"; import { SessionService } from "../session/session.service"; +import { TLog } from "../logger/logger.types"; @Injectable() export class QueryExecuterService { @@ -56,6 +57,7 @@ export class QueryExecuterService { async runQueryQueued( token: string, queryData: any, + log: TLog, headers: Record = {}, cookies: Record = {} ): Promise { @@ -66,6 +68,7 @@ export class QueryExecuterService { queryData, headers, cookies, + log, }, { removeOnComplete: true, @@ -82,7 +85,8 @@ export class QueryExecuterService { token: string, queryData: any, headers: Record = {}, - cookies: Record = {} + cookies: Record = {}, + log: TLog = null ): Promise { const query = await this.queryRepository.findOne({ where: { id: token }, @@ -104,7 +108,8 @@ export class QueryExecuterService { const result = await vm.runScript( this.clearImports(query.source), queryData, - headers + headers, + log ); if (!this.checkResponse(result)) { diff --git a/src/query/handler/query.controller.ts b/src/query/handler/query.controller.ts index 752e3ce..0dcd4dc 100644 --- a/src/query/handler/query.controller.ts +++ b/src/query/handler/query.controller.ts @@ -2,6 +2,7 @@ import { Controller, Inject } from "@nestjs/common"; import { QueryHandlerService } from "./query.handler.service"; import { QueryExecuterService } from "../executer/query.executer.service"; import { BaseQueryController } from "../base/base-query.controller"; +import { LoggerService } from "../logger/logger.service"; @Controller("query") export class QueryController extends BaseQueryController { @@ -9,9 +10,11 @@ export class QueryController extends BaseQueryController { @Inject(QueryHandlerService) queryHandlerService: QueryHandlerService, @Inject(QueryExecuterService) - queryExecuterService: QueryExecuterService + queryExecuterService: QueryExecuterService, + @Inject(LoggerService) + loggerService: LoggerService ) { - super(queryHandlerService, queryExecuterService); + super(queryHandlerService, queryExecuterService, loggerService); } protected getIsCommand(): boolean { diff --git a/src/query/logger/entities/log.entity.ts b/src/query/logger/entities/log.entity.ts new file mode 100644 index 0000000..2607283 --- /dev/null +++ b/src/query/logger/entities/log.entity.ts @@ -0,0 +1,17 @@ +import { Column, Entity, PrimaryGeneratedColumn } from "typeorm"; +import { TLog } from "../logger.types"; + +@Entity() +export class Log { + @PrimaryGeneratedColumn("uuid") + id: string; + + @Column({ type: "varchar", length: 255 }) + traceId: string; + + @Column({ type: "longtext" }) + content: TLog; + + @Column({ type: "timestamp", default: () => "CURRENT_TIMESTAMP" }) + createdAt: Date; +} diff --git a/src/query/logger/logger.controller.ts b/src/query/logger/logger.controller.ts new file mode 100644 index 0000000..65b67a6 --- /dev/null +++ b/src/query/logger/logger.controller.ts @@ -0,0 +1,32 @@ +import { Body, Controller, Get, Inject, Post, UseGuards } from "@nestjs/common"; +import { LoggerService } from "./logger.service"; +import { ApiTokenGuard } from "src/api/guards/api-token.guard"; + +@Controller("logger") +@UseGuards(ApiTokenGuard) +export class LoggerController { + constructor( + @Inject(LoggerService) + private readonly loggerService: LoggerService + ) {} + + @Get("/:traceId") + getByTraceId(@Inject("traceId") traceId: string) { + return this.loggerService.findByTraceId(traceId); + } + + @Post("/find") + find( + @Body() + body: { + traceId?: string; + fromDate?: Date; + toDate?: Date; + url?: string; + limit: number; + offset: number; + } + ) { + return this.loggerService.find(body); + } +} diff --git a/src/query/logger/logger.service.ts b/src/query/logger/logger.service.ts new file mode 100644 index 0000000..c0f7470 --- /dev/null +++ b/src/query/logger/logger.service.ts @@ -0,0 +1,66 @@ +import { Injectable } from "@nestjs/common"; +import { Log } from "./entities/log.entity"; +import { InjectRepository } from "@nestjs/typeorm"; +import { Repository } from "typeorm"; +import { TLog, TLogLine } from "./logger.types"; + +@Injectable() +export class LoggerService { + constructor( + @InjectRepository(Log) + private readonly logRepository: Repository + ) {} + + async create(traceId: string, content: TLog): Promise { + const log = this.logRepository.create({ traceId, content }); + return await this.logRepository.save(log); + } + + async findByTraceId(traceId: string): Promise { + return await this.logRepository.find({ where: { traceId } }); + } + + async find(data: { + traceId?: string; + fromDate?: Date; + toDate?: Date; + url?: string; + limit: number; + offset: number; + }): Promise { + const query = this.logRepository.createQueryBuilder("log"); + + if (data.traceId) { + query.andWhere("log.traceId = :traceId", { traceId: data.traceId }); + } + + if (data.fromDate) { + query.andWhere("log.createdAt >= :fromDate", { fromDate: data.fromDate }); + } + + if (data.toDate) { + query.andWhere("log.createdAt <= :toDate", { toDate: data.toDate }); + } + + if (data.url) { + query.andWhere("log.content LIKE :url", { url: `%${data.url}%` }); + } + + query.skip(data.offset).take(data.limit).orderBy("log.createdAt", "DESC"); + + return await query.getMany(); + } + + static log(logStack: TLog, log: TLog | TLogLine): TLog { + logStack.content.push(log); + return logStack; + } + + static generateTraceId(): string { + return ( + Date.now().toString(36) + + Math.random().toString(36).substring(2, 15) + + Math.random().toString(36).substring(2, 15) + ); + } +} diff --git a/src/query/logger/logger.types.ts b/src/query/logger/logger.types.ts new file mode 100644 index 0000000..9c8bed9 --- /dev/null +++ b/src/query/logger/logger.types.ts @@ -0,0 +1,26 @@ +import { QueryResponse } from "src/vm/vm.constants"; + +export enum TLogType { + info = "info", + error = "error", + debug = "debug", + warn = "warn", +} + +export type TLogLine = { + content: string; + type: TLogType; + timeStamp: number; +}; + +export interface TLog { + traceId: string; + content: (TLogLine | TLog)[]; + payload?: any; + headers: Record; + cookies: Record; + url: string; + response: QueryResponse | null; + startTime: number; + endTime?: number; +} diff --git a/src/query/query.module.ts b/src/query/query.module.ts index b186533..53c19cd 100644 --- a/src/query/query.module.ts +++ b/src/query/query.module.ts @@ -15,6 +15,9 @@ import { FunctionController } from "src/query/function/function.controller"; import { RedisManagerModule } from "src/redisManager/redisManager.module"; import { RedisModule } from "src/redis/redis.module"; import { SessionService } from "./session/session.service"; +import { Log } from "./logger/entities/log.entity"; +import { LoggerService } from "./logger/logger.service"; +import { LoggerController } from "./logger/logger.controller"; @Module({ imports: [ @@ -24,15 +27,26 @@ import { SessionService } from "./session/session.service"; forwardRef(() => QueueModule), forwardRef(() => RedisModule), forwardRef(() => RedisManagerModule), - TypeOrmModule.forFeature([Query, FunctionEntity]), + TypeOrmModule.forFeature([Query, FunctionEntity, Log]), + ], + controllers: [ + QueryController, + CommandController, + FunctionController, + LoggerController, ], - controllers: [QueryController, CommandController, FunctionController], providers: [ QueryExecuterService, SessionService, + LoggerService, QueryHandlerService, FunctionService, ], - exports: [QueryExecuterService, TypeOrmModule, QueryHandlerService], + exports: [ + QueryExecuterService, + TypeOrmModule, + QueryHandlerService, + LoggerService, + ], }) export class QueryModule {} diff --git a/src/queue/processors/query.processor.ts b/src/queue/processors/query.processor.ts index 4e734fa..d5a2183 100644 --- a/src/queue/processors/query.processor.ts +++ b/src/queue/processors/query.processor.ts @@ -2,12 +2,14 @@ import { Processor, WorkerHost } from "@nestjs/bullmq"; import { Job } from "bullmq"; import { QueryExecuterService } from "src/query/executer/query.executer.service"; import { QUEUE_NAMES } from "../constants"; +import { TLog } from "src/query/logger/logger.types"; export interface QueryJob { token: string; queryData: any; headers: Record; cookies: Record; + log: TLog; } @Processor(QUEUE_NAMES.QUERY, { concurrency: 5 }) @@ -17,13 +19,14 @@ export class QueryProcessor extends WorkerHost { } async process(job: Job) { - const { token, queryData, headers, cookies } = job.data; + const { token, queryData, headers, cookies, log } = job.data; return await this.queryExecuterService.runQuery( token, queryData, headers, - cookies + cookies, + log ); } } diff --git a/src/vm/vm.class.ts b/src/vm/vm.class.ts index 11ace9c..d49b6b9 100644 --- a/src/vm/vm.class.ts +++ b/src/vm/vm.class.ts @@ -1,7 +1,10 @@ +import { TLogType } from "./../query/logger/logger.types"; import * as ivm from "isolated-vm"; import { VModule } from "./module.class"; import { Plugin } from "./plugin.class"; import { QueryResponse } from "./vm.constants"; +import { TLog } from "src/query/logger/logger.types"; +import { LoggerService } from "src/query/logger/logger.service"; export class Vm { private memoryLimit: number; @@ -83,7 +86,8 @@ export class Vm { async runScript( script: string, args: Record, - headers: Record + headers: Record, + log: TLog ): Promise { let resolvePromise: (value: any) => void; let rejectPromise: (reason?: any) => void; @@ -94,14 +98,29 @@ export class Vm { }); this.setFunction("returnResult", (res) => { - console.log("Returning result from VM:", res); - - resolvePromise(res); + resolvePromise({ ...res, log }); }); - // TODO: log this.setFunction("log", (...args) => { - console.log("vm log:", args); + if (!log) { + return; + } + + const logType = args.find( + (arg) => + arg && + typeof arg === "object" && + arg.type && + Object.values(TLogType).includes(arg.type) + ); + + log = LoggerService.log(log, { + content: args + .map((arg) => (typeof arg === "string" ? arg : JSON.stringify(arg))) + .join(" "), + type: logType?.type || TLogType.info, + timeStamp: new Date().getTime(), + }); }); this.setFunction("error", (error: any) => { diff --git a/src/vm/vm.constants.ts b/src/vm/vm.constants.ts index 86a5086..8d2c2c9 100644 --- a/src/vm/vm.constants.ts +++ b/src/vm/vm.constants.ts @@ -5,6 +5,7 @@ import { QueryPlugin } from "./plugins/query.plugin"; import { AxiosPlugin } from "./plugins/axios.plugin"; import { RedisPlugin } from "./plugins/redis.plugin"; import { SessionPlugin } from "./plugins/session.plugin"; +import { TLog } from "src/query/logger/logger.types"; export const registeredPlugins = { db: async (service: QueryExecuterService, query: Query) => { @@ -59,4 +60,5 @@ export type QueryResponse = { headers?: Record; redirect?: string; cookies?: Record; + log: TLog; };