feat: add logging functionality with LoggerService; implement log entity and controller; enhance query processing with logging support

src/databaseManager/database/database.manager.controller.ts

@@ -39,6 +39,19 @@ export class DatabaseManagerController {
     );
   }
 
+  @Get("tables/:databaseId")
+  getTables(@Param("databaseId") databaseId: string) {
+    return this.databaseManagerService.getTableList(databaseId);
+  }
+
+  @Get("columns/:databaseId/:tableName")
+  getColumns(
+    @Param("databaseId") databaseId: string,
+    @Param("tableName") tableName: string
+  ) {
+    return this.databaseManagerService.getTableColumns(databaseId, tableName);
+  }
+
   @Get("migration/up/:databaseId")
   migrateUp(@Param("databaseId") databaseId: string) {
     return this.migrationService.up(databaseId);

src/databaseManager/database/database.manager.service.ts

@@ -1,4 +1,4 @@
-import { Inject, Injectable } from "@nestjs/common";
+import { forwardRef, Inject, Injectable } from "@nestjs/common";
 import { InjectRepository } from "@nestjs/typeorm";
 import { Database } from "../entities/database.entity";
 import { Repository } from "typeorm";
@@ -13,6 +13,7 @@ export class DatabaseManagerService extends DatabaseEncryptionService {
   constructor(
     @InjectRepository(Database)
     private databaseRepository: Repository<Database>,
+    @Inject(forwardRef(() => ProjectService))
     private readonly projectService: ProjectService,
     private readonly databaseNodeService: DatabaseNodeService,
     @Inject(RedisClient)
@@ -146,4 +147,24 @@ export class DatabaseManagerService extends DatabaseEncryptionService {
 
     return await this.databaseRepository.save(database);
   }
+
+  async getTableList(databaseId: string): Promise<string[]> {
+    const results = await this.runQuery(
+      databaseId,
+      "SHOW TABLES;",
+      true /* use query user */
+    );
+
+    return results.map((row) => Object.values(row)[0]);
+  }
+
+  async getTableColumns(databaseId: string, tableName: string): Promise<any[]> {
+    const results = await this.runQuery(
+      databaseId,
+      `SHOW COLUMNS FROM \`${tableName}\`;`,
+      true /* use query user */
+    );
+
+    return results;
+  }
 }

src/migrations/1760184857707-adminToken.ts

@@ -0,0 +1,17 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class AdminToken1760184857707 implements MigrationInterface {
+  name = "AdminToken1760184857707";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE \`token\` CHANGE \`isActive\` \`isActive\` tinyint NOT NULL DEFAULT '1'`
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE \`token\` CHANGE \`isActive\` \`isActive\` tinyint NOT NULL DEFAULT 0`
+    );
+  }
+}

src/migrations/1760188157352-logs.ts

@@ -0,0 +1,21 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class Logs1760188157352 implements MigrationInterface {
+  name = "Logs1760188157352";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `CREATE TABLE \`log\` (\`id\` varchar(36) NOT NULL, \`traceId\` varchar(255) NOT NULL, \`content\` longtext NOT NULL, \`createdAt\` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP(), PRIMARY KEY (\`id\`)) ENGINE=InnoDB`
+    );
+    await queryRunner.query(
+      `ALTER TABLE \`token\` CHANGE \`isActive\` \`isActive\` tinyint NOT NULL DEFAULT '1'`
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE \`token\` CHANGE \`isActive\` \`isActive\` tinyint NOT NULL DEFAULT 0`
+    );
+    await queryRunner.query(`DROP TABLE \`log\``);
+  }
+}

src/project/project.controller.ts

@@ -5,7 +5,6 @@ import { AdminGuard } from "src/api/guards/admin.guard";
 
 @Controller("project")
 @UseGuards(ApiTokenGuard)
-@UseGuards(AdminGuard)
 export class ProjectController {
   constructor(
     @Inject(ProjectService)
@@ -16,4 +15,10 @@ export class ProjectController {
   createProject(@Body() body: { name: string }) {
     return this.projectService.create(body.name);
   }
+
+  @Put("create-without-db")
+  @UseGuards(AdminGuard)
+  createProjectWithoutDB(@Body() body: { name: string }) {
+    return this.projectService.create(body.name, false);
+  }
 }

src/project/project.module.ts

@@ -5,11 +5,13 @@ import { ProjectService } from "./project.service";
 import { ProjectController } from "./project.controller";
 import { ApiModule } from "src/api/api.module";
 import { RedisModule } from "src/redis/redis.module";
+import { DatabaseManagerModule } from "src/databaseManager/database.manager.module";
 
 @Module({
   imports: [
     forwardRef(() => ApiModule),
     forwardRef(() => RedisModule),
+    forwardRef(() => DatabaseManagerModule),
     TypeOrmModule.forFeature([Project]),
   ],
   controllers: [ProjectController],

src/project/project.service.ts

@@ -1,8 +1,9 @@
-import { Inject, Injectable } from "@nestjs/common";
+import { forwardRef, Inject, Injectable } from "@nestjs/common";
 import { InjectRepository } from "@nestjs/typeorm";
 import { Repository } from "typeorm";
 import { Project } from "./entities/project.entity";
 import { RedisClient } from "src/redis/redis.service";
+import { DatabaseManagerService } from "src/databaseManager/database/database.manager.service";
 
 @Injectable()
 export class ProjectService {
@@ -10,12 +11,20 @@ export class ProjectService {
     @InjectRepository(Project)
     private readonly projectRepository: Repository<Project>,
     @Inject(RedisClient)
-    private readonly redisClient: RedisClient
+    private readonly redisClient: RedisClient,
+    @Inject(forwardRef(() => DatabaseManagerService))
+    private readonly databaseManagerService: DatabaseManagerService
   ) {}
 
-  create(name: string) {
+  async create(name: string, createDatabase: boolean = true) {
     const project = this.projectRepository.create({ name });
-    return this.projectRepository.save(project);
+    const projectSaved = await this.projectRepository.save(project);
+
+    if (createDatabase) {
+      await this.databaseManagerService.createDatabase(projectSaved.id);
+    }
+
+    return projectSaved;
   }
 
   async findById(id: string) {

src/query/base/base-query.controller.ts

@@ -13,6 +13,8 @@ import { QueryHandlerService } from "../handler/query.handler.service";
 import { ApiTokenGuard } from "src/api/guards/api-token.guard";
 import { QueryExecuterService } from "../executer/query.executer.service";
 import { QueryGuard } from "src/query/guards/query.guard";
+import { LoggerService } from "../logger/logger.service";
+import { TLogType } from "../logger/logger.types";
 
 @UseGuards(ApiTokenGuard)
 export abstract class BaseQueryController {
@@ -20,7 +22,9 @@ export abstract class BaseQueryController {
     @Inject(QueryHandlerService)
     protected readonly queryHandlerService: QueryHandlerService,
     @Inject(QueryExecuterService)
-    protected readonly queryExecuterService: QueryExecuterService
+    protected readonly queryExecuterService: QueryExecuterService,
+    @Inject(LoggerService)
+    protected readonly loggerService: LoggerService
   ) {}
 
   protected abstract getIsCommand(): boolean;
@@ -49,9 +53,26 @@ export abstract class BaseQueryController {
     @Headers() headers: Record<string, any>,
     @Res() res: Response
   ) {
+    const loggerTraceId =
+      headers["x-trace-id"] || LoggerService.generateTraceId();
+    const log = LoggerService.log(
+      {
+        traceId: loggerTraceId,
+        startTime: new Date().getTime(),
+        payload: query,
+        headers: headers,
+        cookies: headers.cookie,
+        url: `/run/${id}`,
+        response: null,
+        content: [],
+      },
+      { content: "", type: TLogType.info, timeStamp: new Date().getTime() }
+    );
+
     const queryResult = await this.queryExecuterService.runQueryQueued(
       id,
       query,
+      log,
       headers,
       headers.cookie.split("; ").reduce((acc, cookie) => {
         const [key, value] = cookie.split("=");
@@ -60,6 +81,11 @@ export abstract class BaseQueryController {
       }, {})
     );
 
+    if (queryResult?.log) {
+      queryResult.log.endTime = new Date().getTime();
+      await this.loggerService.create(queryResult.log.traceId, queryResult.log);
+    }
+
     res.status(queryResult?.statusCode || 200);
 
     if (queryResult?.cookies) {

src/query/command/command.controller.ts

@@ -2,6 +2,7 @@ import { Controller, Inject } from "@nestjs/common";
 import { QueryHandlerService } from "../handler/query.handler.service";
 import { QueryExecuterService } from "../executer/query.executer.service";
 import { BaseQueryController } from "../base/base-query.controller";
+import { LoggerService } from "../logger/logger.service";
 
 @Controller("command")
 export class CommandController extends BaseQueryController {
@@ -9,9 +10,11 @@ export class CommandController extends BaseQueryController {
     @Inject(QueryHandlerService)
     queryHandlerService: QueryHandlerService,
     @Inject(QueryExecuterService)
-    queryExecuterService: QueryExecuterService
+    queryExecuterService: QueryExecuterService,
+    @Inject(LoggerService)
+    loggerService: LoggerService
   ) {
-    super(queryHandlerService, queryExecuterService);
+    super(queryHandlerService, queryExecuterService, loggerService);
   }
 
   protected getIsCommand(): boolean {

src/query/executer/query.executer.service.ts

@@ -16,6 +16,7 @@ import { QUEUE_NAMES } from "src/queue/constants";
 import { Queue, QueueEvents } from "bullmq";
 import { FunctionService } from "src/query/function/function.service";
 import { SessionService } from "../session/session.service";
+import { TLog } from "../logger/logger.types";
 
 @Injectable()
 export class QueryExecuterService {
@@ -56,6 +57,7 @@ export class QueryExecuterService {
   async runQueryQueued(
     token: string,
     queryData: any,
+    log: TLog,
     headers: Record<string, any> = {},
     cookies: Record<string, any> = {}
   ): Promise<QueryResponse> {
@@ -66,6 +68,7 @@ export class QueryExecuterService {
         queryData,
         headers,
         cookies,
+        log,
       },
       {
         removeOnComplete: true,
@@ -82,7 +85,8 @@ export class QueryExecuterService {
     token: string,
     queryData: any,
     headers: Record<string, any> = {},
-    cookies: Record<string, any> = {}
+    cookies: Record<string, any> = {},
+    log: TLog = null
   ): Promise<QueryResponse> {
     const query = await this.queryRepository.findOne({
       where: { id: token },
@@ -104,7 +108,8 @@ export class QueryExecuterService {
     const result = await vm.runScript(
       this.clearImports(query.source),
       queryData,
-      headers
+      headers,
+      log
     );
 
     if (!this.checkResponse(result)) {

src/query/handler/query.controller.ts

@@ -2,6 +2,7 @@ import { Controller, Inject } from "@nestjs/common";
 import { QueryHandlerService } from "./query.handler.service";
 import { QueryExecuterService } from "../executer/query.executer.service";
 import { BaseQueryController } from "../base/base-query.controller";
+import { LoggerService } from "../logger/logger.service";
 
 @Controller("query")
 export class QueryController extends BaseQueryController {
@@ -9,9 +10,11 @@ export class QueryController extends BaseQueryController {
     @Inject(QueryHandlerService)
     queryHandlerService: QueryHandlerService,
     @Inject(QueryExecuterService)
-    queryExecuterService: QueryExecuterService
+    queryExecuterService: QueryExecuterService,
+    @Inject(LoggerService)
+    loggerService: LoggerService
   ) {
-    super(queryHandlerService, queryExecuterService);
+    super(queryHandlerService, queryExecuterService, loggerService);
   }
 
   protected getIsCommand(): boolean {

src/query/logger/entities/log.entity.ts

@@ -0,0 +1,17 @@
+import { Column, Entity, PrimaryGeneratedColumn } from "typeorm";
+import { TLog } from "../logger.types";
+
+@Entity()
+export class Log {
+  @PrimaryGeneratedColumn("uuid")
+  id: string;
+
+  @Column({ type: "varchar", length: 255 })
+  traceId: string;
+
+  @Column({ type: "longtext" })
+  content: TLog;
+
+  @Column({ type: "timestamp", default: () => "CURRENT_TIMESTAMP" })
+  createdAt: Date;
+}

src/query/logger/logger.controller.ts

@@ -0,0 +1,32 @@
+import { Body, Controller, Get, Inject, Post, UseGuards } from "@nestjs/common";
+import { LoggerService } from "./logger.service";
+import { ApiTokenGuard } from "src/api/guards/api-token.guard";
+
+@Controller("logger")
+@UseGuards(ApiTokenGuard)
+export class LoggerController {
+  constructor(
+    @Inject(LoggerService)
+    private readonly loggerService: LoggerService
+  ) {}
+
+  @Get("/:traceId")
+  getByTraceId(@Inject("traceId") traceId: string) {
+    return this.loggerService.findByTraceId(traceId);
+  }
+
+  @Post("/find")
+  find(
+    @Body()
+    body: {
+      traceId?: string;
+      fromDate?: Date;
+      toDate?: Date;
+      url?: string;
+      limit: number;
+      offset: number;
+    }
+  ) {
+    return this.loggerService.find(body);
+  }
+}

src/query/logger/logger.service.ts

@@ -0,0 +1,66 @@
+import { Injectable } from "@nestjs/common";
+import { Log } from "./entities/log.entity";
+import { InjectRepository } from "@nestjs/typeorm";
+import { Repository } from "typeorm";
+import { TLog, TLogLine } from "./logger.types";
+
+@Injectable()
+export class LoggerService {
+  constructor(
+    @InjectRepository(Log)
+    private readonly logRepository: Repository<Log>
+  ) {}
+
+  async create(traceId: string, content: TLog): Promise<Log> {
+    const log = this.logRepository.create({ traceId, content });
+    return await this.logRepository.save(log);
+  }
+
+  async findByTraceId(traceId: string): Promise<Log[]> {
+    return await this.logRepository.find({ where: { traceId } });
+  }
+
+  async find(data: {
+    traceId?: string;
+    fromDate?: Date;
+    toDate?: Date;
+    url?: string;
+    limit: number;
+    offset: number;
+  }): Promise<Log[]> {
+    const query = this.logRepository.createQueryBuilder("log");
+
+    if (data.traceId) {
+      query.andWhere("log.traceId = :traceId", { traceId: data.traceId });
+    }
+
+    if (data.fromDate) {
+      query.andWhere("log.createdAt >= :fromDate", { fromDate: data.fromDate });
+    }
+
+    if (data.toDate) {
+      query.andWhere("log.createdAt <= :toDate", { toDate: data.toDate });
+    }
+
+    if (data.url) {
+      query.andWhere("log.content LIKE :url", { url: `%${data.url}%` });
+    }
+
+    query.skip(data.offset).take(data.limit).orderBy("log.createdAt", "DESC");
+
+    return await query.getMany();
+  }
+
+  static log(logStack: TLog, log: TLog | TLogLine): TLog {
+    logStack.content.push(log);
+    return logStack;
+  }
+
+  static generateTraceId(): string {
+    return (
+      Date.now().toString(36) +
+      Math.random().toString(36).substring(2, 15) +
+      Math.random().toString(36).substring(2, 15)
+    );
+  }
+}

src/query/logger/logger.types.ts

@@ -0,0 +1,26 @@
+import { QueryResponse } from "src/vm/vm.constants";
+
+export enum TLogType {
+  info = "info",
+  error = "error",
+  debug = "debug",
+  warn = "warn",
+}
+
+export type TLogLine = {
+  content: string;
+  type: TLogType;
+  timeStamp: number;
+};
+
+export interface TLog {
+  traceId: string;
+  content: (TLogLine | TLog)[];
+  payload?: any;
+  headers: Record<string, string | string[] | undefined>;
+  cookies: Record<string, string>;
+  url: string;
+  response: QueryResponse | null;
+  startTime: number;
+  endTime?: number;
+}

src/query/query.module.ts

@@ -15,6 +15,9 @@ import { FunctionController } from "src/query/function/function.controller";
 import { RedisManagerModule } from "src/redisManager/redisManager.module";
 import { RedisModule } from "src/redis/redis.module";
 import { SessionService } from "./session/session.service";
+import { Log } from "./logger/entities/log.entity";
+import { LoggerService } from "./logger/logger.service";
+import { LoggerController } from "./logger/logger.controller";
 
 @Module({
   imports: [
@@ -24,15 +27,26 @@ import { SessionService } from "./session/session.service";
     forwardRef(() => QueueModule),
     forwardRef(() => RedisModule),
     forwardRef(() => RedisManagerModule),
-    TypeOrmModule.forFeature([Query, FunctionEntity]),
+    TypeOrmModule.forFeature([Query, FunctionEntity, Log]),
+  ],
+  controllers: [
+    QueryController,
+    CommandController,
+    FunctionController,
+    LoggerController,
   ],
-  controllers: [QueryController, CommandController, FunctionController],
   providers: [
     QueryExecuterService,
     SessionService,
+    LoggerService,
     QueryHandlerService,
     FunctionService,
   ],
-  exports: [QueryExecuterService, TypeOrmModule, QueryHandlerService],
+  exports: [
+    QueryExecuterService,
+    TypeOrmModule,
+    QueryHandlerService,
+    LoggerService,
+  ],
 })
 export class QueryModule {}

src/queue/processors/query.processor.ts

@@ -2,12 +2,14 @@ import { Processor, WorkerHost } from "@nestjs/bullmq";
 import { Job } from "bullmq";
 import { QueryExecuterService } from "src/query/executer/query.executer.service";
 import { QUEUE_NAMES } from "../constants";
+import { TLog } from "src/query/logger/logger.types";
 
 export interface QueryJob {
   token: string;
   queryData: any;
   headers: Record<string, any>;
   cookies: Record<string, any>;
+  log: TLog;
 }
 
 @Processor(QUEUE_NAMES.QUERY, { concurrency: 5 })
@@ -17,13 +19,14 @@ export class QueryProcessor extends WorkerHost {
   }
 
   async process(job: Job<QueryJob>) {
-    const { token, queryData, headers, cookies } = job.data;
+    const { token, queryData, headers, cookies, log } = job.data;
 
     return await this.queryExecuterService.runQuery(
       token,
       queryData,
       headers,
-      cookies
+      cookies,
+      log
     );
   }
 }

src/vm/vm.class.ts

@@ -1,7 +1,10 @@
+import { TLogType } from "./../query/logger/logger.types";
 import * as ivm from "isolated-vm";
 import { VModule } from "./module.class";
 import { Plugin } from "./plugin.class";
 import { QueryResponse } from "./vm.constants";
+import { TLog } from "src/query/logger/logger.types";
+import { LoggerService } from "src/query/logger/logger.service";
 
 export class Vm {
   private memoryLimit: number;
@@ -83,7 +86,8 @@ export class Vm {
   async runScript(
     script: string,
     args: Record<string, any>,
-    headers: Record<string, any>
+    headers: Record<string, any>,
+    log: TLog
   ): Promise<QueryResponse> {
     let resolvePromise: (value: any) => void;
     let rejectPromise: (reason?: any) => void;
@@ -94,14 +98,29 @@ export class Vm {
     });
 
     this.setFunction("returnResult", (res) => {
-      console.log("Returning result from VM:", res);
-
-      resolvePromise(res);
+      resolvePromise({ ...res, log });
     });
 
-    // TODO: log
     this.setFunction("log", (...args) => {
-      console.log("vm log:", args);
+      if (!log) {
+        return;
+      }
+
+      const logType = args.find(
+        (arg) =>
+          arg &&
+          typeof arg === "object" &&
+          arg.type &&
+          Object.values(TLogType).includes(arg.type)
+      );
+
+      log = LoggerService.log(log, {
+        content: args
+          .map((arg) => (typeof arg === "string" ? arg : JSON.stringify(arg)))
+          .join(" "),
+        type: logType?.type || TLogType.info,
+        timeStamp: new Date().getTime(),
+      });
     });
 
     this.setFunction("error", (error: any) => {

src/vm/vm.constants.ts

@@ -5,6 +5,7 @@ import { QueryPlugin } from "./plugins/query.plugin";
 import { AxiosPlugin } from "./plugins/axios.plugin";
 import { RedisPlugin } from "./plugins/redis.plugin";
 import { SessionPlugin } from "./plugins/session.plugin";
+import { TLog } from "src/query/logger/logger.types";
 
 export const registeredPlugins = {
   db: async (service: QueryExecuterService, query: Query) => {
@@ -59,4 +60,5 @@ export type QueryResponse = {
   headers?: Record<string, string>;
   redirect?: string;
   cookies?: Record<string, string>;
+  log: TLog;
 };