feat: implement logging enhancements; add projectId and queryId to log entity; update query and logger services for improved logging; refactor query execution to support call stack tracking

src/migrations/1760199448968-logsPerQuery.ts

@@ -0,0 +1,31 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class LogsPerQuery1760199448968 implements MigrationInterface {
+  name = "LogsPerQuery1760199448968";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE \`log\` ADD \`projectId\` varchar(36) NULL`
+    );
+    await queryRunner.query(
+      `ALTER TABLE \`log\` ADD \`queryId\` varchar(36) NULL`
+    );
+    await queryRunner.query(
+      `ALTER TABLE \`log\` ADD CONSTRAINT \`FK_0c0ad31dd4033de83a2c47f2c82\` FOREIGN KEY (\`projectId\`) REFERENCES \`project\`(\`id\`) ON DELETE NO ACTION ON UPDATE NO ACTION`
+    );
+    await queryRunner.query(
+      `ALTER TABLE \`log\` ADD CONSTRAINT \`FK_7867d6fbda5d177a3727cedece3\` FOREIGN KEY (\`queryId\`) REFERENCES \`query\`(\`id\`) ON DELETE NO ACTION ON UPDATE NO ACTION`
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE \`log\` DROP FOREIGN KEY \`FK_7867d6fbda5d177a3727cedece3\``
+    );
+    await queryRunner.query(
+      `ALTER TABLE \`log\` DROP FOREIGN KEY \`FK_0c0ad31dd4033de83a2c47f2c82\``
+    );
+    await queryRunner.query(`ALTER TABLE \`log\` DROP COLUMN \`queryId\``);
+    await queryRunner.query(`ALTER TABLE \`log\` DROP COLUMN \`projectId\``);
+  }
+}

src/project/entities/project.entity.ts

@@ -13,6 +13,7 @@ import {
 import { Database } from "../../databaseManager/entities/database.entity";
 import { FunctionEntity } from "../../query/entities/function.entity";
 import { RedisNode } from "../../redisManager/entities/redis.node.entity";
+import { Log } from "../../query/logger/entities/log.entity";
 
 @Entity("project")
 export class Project {
@@ -29,6 +30,9 @@ export class Project {
   @JoinColumn()
   database: Database;
 
+  @OneToMany(() => Log, (log) => log.project)
+  logs: Log[];
+
   @OneToMany(() => Query, (query) => query.project)
   queries: Query[];
 

src/query/base/base-query.controller.ts

@@ -5,10 +5,11 @@ import {
   Inject,
   Param,
   Post,
+  Req,
   Res,
   UseGuards,
 } from "@nestjs/common";
-import { Response } from "express";
+import { Response, Request } from "express";
 import { QueryHandlerService } from "../handler/query.handler.service";
 import { ApiTokenGuard } from "src/api/guards/api-token.guard";
 import { QueryExecuterService } from "../executer/query.executer.service";
@@ -16,6 +17,7 @@ import { QueryGuard } from "src/query/guards/query.guard";
 import { LoggerService } from "../logger/logger.service";
 import { TLogType } from "../logger/logger.types";
 import { QueryResponse } from "src/vm/vm.constants";
+import { Query } from "../entities/query.entity";
 
 @UseGuards(ApiTokenGuard)
 export abstract class BaseQueryController {
@@ -52,7 +54,8 @@ export abstract class BaseQueryController {
     @Param("id") id: string,
     @Body() query: Record<string, any>,
     @Headers() headers: Record<string, any>,
-    @Res() res: Response
+    @Res() res: Response,
+    @Req() req: Request & { query: Query }
   ) {
     let queryResult: QueryResponse;
     const loggerTraceId =
@@ -68,7 +71,7 @@ export abstract class BaseQueryController {
         response: null,
         content: [],
       },
-      { content: "", type: TLogType.info, timeStamp: new Date().getTime() }
+      { content: `Query ${id} started`, type: TLogType.info }
     );
 
     try {
@@ -90,7 +93,7 @@ export abstract class BaseQueryController {
         timeStamp: new Date().getTime(),
       });
       log.endTime = new Date().getTime();
-      await this.loggerService.create(log.traceId, log);
+      await this.loggerService.create(log.traceId, log, req.query);
       res.status(500).send({ error: "Internal Server Error" });
       return;
     }
@@ -100,7 +103,11 @@ export abstract class BaseQueryController {
       const res = JSON.parse(JSON.stringify(queryResult));
       delete res.log;
       queryResult.log.response = res;
-      await this.loggerService.create(queryResult.log.traceId, queryResult.log);
+      await this.loggerService.create(
+        queryResult.log.traceId,
+        queryResult.log,
+        req.query
+      );
     }
 
     res.status(queryResult?.statusCode || 200);

src/query/entities/query.entity.ts

@@ -1,5 +1,12 @@
 import { Project } from "../../project/entities/project.entity";
-import { Column, Entity, ManyToOne, PrimaryGeneratedColumn } from "typeorm";
+import {
+  Column,
+  Entity,
+  ManyToOne,
+  OneToMany,
+  PrimaryGeneratedColumn,
+} from "typeorm";
+import { Log } from "../logger/entities/log.entity";
 
 @Entity("query")
 export class Query {
@@ -9,6 +16,9 @@ export class Query {
   @ManyToOne(() => Project, (project) => project.queries)
   project: Project;
 
+  @OneToMany(() => Log, (log) => log.query)
+  logs: Log[];
+
   @Column({ type: "longtext" })
   source: string;
 

src/query/executer/query.executer.service.ts

@@ -1,5 +1,5 @@
 import { RedisNodeService } from "./../../redisManager/redisNode/redis.node.service";
-import { Inject, Injectable, Logger } from "@nestjs/common";
+import { Inject, Injectable } from "@nestjs/common";
 import { InjectRepository } from "@nestjs/typeorm";
 import { Query } from "../entities/query.entity";
 import { Repository } from "typeorm";
@@ -62,6 +62,11 @@ export class QueryExecuterService {
     headers: Record<string, any> = {},
     cookies: Record<string, any> = {}
   ): Promise<QueryResponse> {
+    LoggerService.log(log, {
+      content: `Queueing query ${token}`,
+      type: TLogType.info,
+    });
+
     const job = await this.queryQueue.add(
       `${new Date().getTime()}_${token}`,
       {
@@ -85,6 +90,7 @@ export class QueryExecuterService {
   async runQuery(
     token: string,
     queryData: any,
+    callStack: number = 0,
     headers: Record<string, any> = {},
     cookies: Record<string, any> = {},
     log: TLog = null
@@ -95,6 +101,10 @@ export class QueryExecuterService {
     });
 
     if (!query) {
+      LoggerService.log(log, {
+        content: `Query with id ${token} not found`,
+        type: TLogType.error,
+      });
       throw new Error("Query not found");
     }
 
@@ -105,7 +115,17 @@ export class QueryExecuterService {
       cookies["x-session-id"] = session.sessionId;
     }
 
-    const vm = await this.createVm(query, log, cookies["x-session-id"]);
+    LoggerService.log(log, {
+      content: `Running query ${query.id}`,
+      type: TLogType.info,
+    });
+
+    const vm = await this.createVm(
+      query,
+      log,
+      callStack,
+      cookies["x-session-id"]
+    );
     const result = await vm.runScript(
       this.clearImports(query.source),
       queryData,
@@ -126,10 +146,20 @@ export class QueryExecuterService {
       ).sessionId;
     }
 
+    LoggerService.log(log, {
+      content: `Query ${query.id} executed successfully`,
+      type: TLogType.info,
+    });
+
     return result;
   }
 
-  private async createVm(query: Query, log: TLog, sessionId: string = null) {
+  private async createVm(
+    query: Query,
+    log: TLog,
+    callStack: number,
+    sessionId: string = null
+  ): Promise<Vm> {
     const imports = this.parseImports(query.source);
     const importsParsed = imports.map((imp) => {
       const item = imp.split("/");
@@ -172,7 +202,8 @@ export class QueryExecuterService {
       );
     }
 
-    const vm = new Vm({
+    const vm = new Vm(
+      {
         memoryLimit: 128,
         timeLimit: BigInt(100e9),
         cpuTimeLimit: BigInt(5e9),
@@ -180,7 +211,9 @@ export class QueryExecuterService {
         plugins: plugins,
         functions: functions,
         log,
-    });
+      },
+      callStack
+    );
 
     return await vm.init();
   }

src/query/logger/entities/log.entity.ts

@@ -1,5 +1,7 @@
-import { Column, Entity, PrimaryGeneratedColumn } from "typeorm";
+import { Column, Entity, ManyToOne, PrimaryGeneratedColumn } from "typeorm";
 import { TLog } from "../logger.types";
+import { Project } from "../../../project/entities/project.entity";
+import { Query } from "../../../query/entities/query.entity";
 
 @Entity()
 export class Log {
@@ -18,6 +20,12 @@ export class Log {
   })
   content: TLog;
 
+  @ManyToOne(() => Project, (project) => project.logs)
+  project: Project;
+
+  @ManyToOne(() => Query, (query) => query.logs)
+  query: Query;
+
   @Column({ type: "timestamp", default: () => "CURRENT_TIMESTAMP" })
   createdAt: Date;
 }

src/query/logger/logger.controller.ts

@@ -1,6 +1,15 @@
-import { Body, Controller, Get, Inject, Post, UseGuards } from "@nestjs/common";
+import {
+  Body,
+  Controller,
+  Get,
+  Inject,
+  Param,
+  Post,
+  UseGuards,
+} from "@nestjs/common";
 import { LoggerService } from "./logger.service";
 import { ApiTokenGuard } from "src/api/guards/api-token.guard";
+import { QueryGuard } from "../guards/query.guard";
 
 @Controller("logger")
 @UseGuards(ApiTokenGuard)
@@ -10,13 +19,14 @@ export class LoggerController {
     private readonly loggerService: LoggerService
   ) {}
 
-  @Get("/:traceId")
-  getByTraceId(@Inject("traceId") traceId: string) {
+  @Get("/:id/:traceId")
+  getByTraceId(@Param("traceId") traceId: string) {
     return this.loggerService.findByTraceId(traceId);
   }
 
-  @Post("/find")
-  find(
+  @Post("/:id/findAll")
+  findAll(
+    @Param("id") projectId: string,
     @Body()
     body: {
       traceId?: string;
@@ -27,6 +37,23 @@ export class LoggerController {
       offset: number;
     }
   ) {
-    return this.loggerService.find(body);
+    return this.loggerService.findByProjectId(projectId, body);
+  }
+
+  @Post("/:id/find")
+  @UseGuards(QueryGuard)
+  find(
+    @Param("id") _id: string,
+    @Body()
+    body: {
+      traceId?: string;
+      fromDate?: Date;
+      toDate?: Date;
+      url?: string;
+      limit: number;
+      offset: number;
+    }
+  ) {
+    return this.loggerService.find(_id, body);
   }
 }

src/query/logger/logger.service.ts

@@ -3,6 +3,7 @@ import { Log } from "./entities/log.entity";
 import { InjectRepository } from "@nestjs/typeorm";
 import { Repository } from "typeorm";
 import { TLog, TLogLine } from "./logger.types";
+import { Query } from "../entities/query.entity";
 
 @Injectable()
 export class LoggerService {
@@ -11,8 +12,13 @@ export class LoggerService {
     private readonly logRepository: Repository<Log>
   ) {}
 
-  async create(traceId: string, content: TLog): Promise<Log> {
-    const log = this.logRepository.create({ traceId, content });
+  async create(traceId: string, content: TLog, query: Query): Promise<Log> {
+    const log = this.logRepository.create({
+      traceId,
+      content,
+      query,
+      project: query.project,
+    });
     return await this.logRepository.save(log);
   }
 
@@ -20,14 +26,14 @@ export class LoggerService {
     return await this.logRepository.find({ where: { traceId } });
   }
 
-  async find(data: {
+  private prepareQuery(data: {
     traceId?: string;
     fromDate?: Date;
     toDate?: Date;
     url?: string;
     limit: number;
     offset: number;
-  }): Promise<Log[]> {
+  }) {
     const query = this.logRepository.createQueryBuilder("log");
 
     if (data.traceId) {
@@ -37,21 +43,57 @@ export class LoggerService {
     if (data.fromDate) {
       query.andWhere("log.createdAt >= :fromDate", { fromDate: data.fromDate });
     }
-
     if (data.toDate) {
       query.andWhere("log.createdAt <= :toDate", { toDate: data.toDate });
     }
-
     if (data.url) {
       query.andWhere("log.content LIKE :url", { url: `%${data.url}%` });
     }
 
     query.skip(data.offset).take(data.limit).orderBy("log.createdAt", "DESC");
 
+    return query;
+  }
+
+  async findByProjectId(
+    projectId: string,
+    data: {
+      traceId?: string;
+      fromDate?: Date;
+      toDate?: Date;
+      url?: string;
+      limit: number;
+      offset: number;
+    }
+  ): Promise<Log[]> {
+    const query = this.prepareQuery(data);
+    query.where("log.projectId = :projectId", { projectId });
+
     return await query.getMany();
   }
 
-  static log(logStack: TLog, log: TLog | TLogLine): TLog {
+  async find(
+    queryId: string,
+    data: {
+      traceId?: string;
+      fromDate?: Date;
+      toDate?: Date;
+      url?: string;
+      limit: number;
+      offset: number;
+    }
+  ): Promise<Log[]> {
+    const query = this.prepareQuery(data);
+    query.where("log.queryId = :queryId", { queryId });
+
+    return await query.getMany();
+  }
+
+  static log(logStack: TLog, log: TLogLine): TLog {
+    if (!log.timeStamp) {
+      log.timeStamp = Date.now();
+    }
+
     logStack.content.push(log);
     return logStack;
   }

src/query/logger/logger.types.ts

@@ -10,12 +10,12 @@ export enum TLogType {
 export type TLogLine = {
   content: string;
   type: TLogType;
-  timeStamp: number;
+  timeStamp?: number;
 };
 
 export interface TLog {
   traceId: string;
-  content: (TLogLine | TLog)[];
+  content: TLogLine[];
   payload?: any;
   headers: Record<string, string | string[] | undefined>;
   cookies: Record<string, string>;

src/queue/processors/query.processor.ts

@@ -24,6 +24,7 @@ export class QueryProcessor extends WorkerHost {
     return await this.queryExecuterService.runQuery(
       token,
       queryData,
+      0,
       headers,
       cookies,
       log

src/vm/plugin.class.ts

@@ -5,9 +5,11 @@ export abstract class Plugin {
   protected log: TLog;
   protected cookies: Record<string, string>;
   protected headers: Record<string, string>;
+  protected callStack: number;
 
   constructor(name: string, protected methods: string[] = []) {
     this.name = name;
+    this.callStack = 0;
   }
 
   getName(): string {
@@ -26,6 +28,10 @@ export abstract class Plugin {
     this.headers = headers;
   }
 
+  setCallStack(callStack: number) {
+    this.callStack = callStack;
+  }
+
   setCookies(cookies: Record<string, string>) {
     this.cookies = cookies;
   }

src/vm/plugins/query.plugin.ts

@@ -33,6 +33,7 @@ export class QueryPlugin extends Plugin {
     return await this.QueryExecuterService.runQuery(
       id,
       data,
+      this.callStack + 1,
       this.headers,
       this.cookies,
       this.log

src/vm/vm.class.ts

@@ -19,8 +19,10 @@ export class Vm {
   private log: TLog;
   private headers?: Record<string, string>;
   private cookies?: Record<string, string>;
+  private callStack = 0;
 
-  constructor(configs: {
+  constructor(
+    configs: {
       memoryLimit: number;
       timeLimit?: bigint;
       cpuTimeLimit?: bigint;
@@ -30,7 +32,15 @@ export class Vm {
       log: TLog;
       headers?: Record<string, string>;
       cookies?: Record<string, string>;
-  }) {
+    },
+    callStack = 0
+  ) {
+    this.callStack = callStack;
+
+    if (this.callStack > 5) {
+      throw new Error("Maximum call stack size exceeded");
+    }
+
     this.memoryLimit = configs.memoryLimit;
     this.modules = configs.modules;
     this.plugins = configs.plugins;
@@ -71,6 +81,7 @@ export class Vm {
           plugin.setLog(this.log);
           plugin.setHeaders(this.headers);
           plugin.setCookies(this.cookies);
+          plugin.setCallStack(this.callStack);
           const result = await plugin[method](...args);
 
           if (result && result.log) {

tests/base/case1-payload.js

@@ -6,7 +6,6 @@ import "plugin/db";
 import "plugin/axios";
 import "plugin/session";
 import "plugin/query";
-import { query } from "express";
 
 function createSQL(id) {
   return squel.select().from("test").where("id = ?", id).toString();