feat: implement logging enhancements; add projectId and queryId to log entity; update query and logger services for improved logging; refactor query execution to support call stack tracking

src/query/base/base-query.controller.ts

@@ -5,10 +5,11 @@ import {
   Inject,
   Param,
   Post,
+  Req,
   Res,
   UseGuards,
 } from "@nestjs/common";
-import { Response } from "express";
+import { Response, Request } from "express";
 import { QueryHandlerService } from "../handler/query.handler.service";
 import { ApiTokenGuard } from "src/api/guards/api-token.guard";
 import { QueryExecuterService } from "../executer/query.executer.service";
@@ -16,6 +17,7 @@ import { QueryGuard } from "src/query/guards/query.guard";
 import { LoggerService } from "../logger/logger.service";
 import { TLogType } from "../logger/logger.types";
 import { QueryResponse } from "src/vm/vm.constants";
+import { Query } from "../entities/query.entity";
 
 @UseGuards(ApiTokenGuard)
 export abstract class BaseQueryController {
@@ -52,7 +54,8 @@ export abstract class BaseQueryController {
     @Param("id") id: string,
     @Body() query: Record<string, any>,
     @Headers() headers: Record<string, any>,
-    @Res() res: Response
+    @Res() res: Response,
+    @Req() req: Request & { query: Query }
   ) {
     let queryResult: QueryResponse;
     const loggerTraceId =
@@ -68,7 +71,7 @@ export abstract class BaseQueryController {
         response: null,
         content: [],
       },
-      { content: "", type: TLogType.info, timeStamp: new Date().getTime() }
+      { content: `Query ${id} started`, type: TLogType.info }
     );
 
     try {
@@ -90,7 +93,7 @@ export abstract class BaseQueryController {
         timeStamp: new Date().getTime(),
       });
       log.endTime = new Date().getTime();
-      await this.loggerService.create(log.traceId, log);
+      await this.loggerService.create(log.traceId, log, req.query);
       res.status(500).send({ error: "Internal Server Error" });
       return;
     }
@@ -100,7 +103,11 @@ export abstract class BaseQueryController {
       const res = JSON.parse(JSON.stringify(queryResult));
       delete res.log;
       queryResult.log.response = res;
-      await this.loggerService.create(queryResult.log.traceId, queryResult.log);
+      await this.loggerService.create(
+        queryResult.log.traceId,
+        queryResult.log,
+        req.query
+      );
     }
 
     res.status(queryResult?.statusCode || 200);

src/query/entities/query.entity.ts

@@ -1,5 +1,12 @@
 import { Project } from "../../project/entities/project.entity";
-import { Column, Entity, ManyToOne, PrimaryGeneratedColumn } from "typeorm";
+import {
+  Column,
+  Entity,
+  ManyToOne,
+  OneToMany,
+  PrimaryGeneratedColumn,
+} from "typeorm";
+import { Log } from "../logger/entities/log.entity";
 
 @Entity("query")
 export class Query {
@@ -9,6 +16,9 @@ export class Query {
   @ManyToOne(() => Project, (project) => project.queries)
   project: Project;
 
+  @OneToMany(() => Log, (log) => log.query)
+  logs: Log[];
+
   @Column({ type: "longtext" })
   source: string;
 

src/query/executer/query.executer.service.ts

@@ -1,5 +1,5 @@
 import { RedisNodeService } from "./../../redisManager/redisNode/redis.node.service";
-import { Inject, Injectable, Logger } from "@nestjs/common";
+import { Inject, Injectable } from "@nestjs/common";
 import { InjectRepository } from "@nestjs/typeorm";
 import { Query } from "../entities/query.entity";
 import { Repository } from "typeorm";
@@ -62,6 +62,11 @@ export class QueryExecuterService {
     headers: Record<string, any> = {},
     cookies: Record<string, any> = {}
   ): Promise<QueryResponse> {
+    LoggerService.log(log, {
+      content: `Queueing query ${token}`,
+      type: TLogType.info,
+    });
+
     const job = await this.queryQueue.add(
       `${new Date().getTime()}_${token}`,
       {
@@ -85,6 +90,7 @@ export class QueryExecuterService {
   async runQuery(
     token: string,
     queryData: any,
+    callStack: number = 0,
     headers: Record<string, any> = {},
     cookies: Record<string, any> = {},
     log: TLog = null
@@ -95,6 +101,10 @@ export class QueryExecuterService {
     });
 
     if (!query) {
+      LoggerService.log(log, {
+        content: `Query with id ${token} not found`,
+        type: TLogType.error,
+      });
       throw new Error("Query not found");
     }
 
@@ -105,7 +115,17 @@ export class QueryExecuterService {
       cookies["x-session-id"] = session.sessionId;
     }
 
-    const vm = await this.createVm(query, log, cookies["x-session-id"]);
+    LoggerService.log(log, {
+      content: `Running query ${query.id}`,
+      type: TLogType.info,
+    });
+
+    const vm = await this.createVm(
+      query,
+      log,
+      callStack,
+      cookies["x-session-id"]
+    );
     const result = await vm.runScript(
       this.clearImports(query.source),
       queryData,
@@ -126,10 +146,20 @@ export class QueryExecuterService {
       ).sessionId;
     }
 
+    LoggerService.log(log, {
+      content: `Query ${query.id} executed successfully`,
+      type: TLogType.info,
+    });
+
     return result;
   }
 
-  private async createVm(query: Query, log: TLog, sessionId: string = null) {
+  private async createVm(
+    query: Query,
+    log: TLog,
+    callStack: number,
+    sessionId: string = null
+  ): Promise<Vm> {
     const imports = this.parseImports(query.source);
     const importsParsed = imports.map((imp) => {
       const item = imp.split("/");
@@ -172,15 +202,18 @@ export class QueryExecuterService {
       );
     }
 
-    const vm = new Vm({
-      memoryLimit: 128,
-      timeLimit: BigInt(100e9),
-      cpuTimeLimit: BigInt(5e9),
-      modules: modules,
-      plugins: plugins,
-      functions: functions,
-      log,
-    });
+    const vm = new Vm(
+      {
+        memoryLimit: 128,
+        timeLimit: BigInt(100e9),
+        cpuTimeLimit: BigInt(5e9),
+        modules: modules,
+        plugins: plugins,
+        functions: functions,
+        log,
+      },
+      callStack
+    );
 
     return await vm.init();
   }

src/query/logger/entities/log.entity.ts

@@ -1,5 +1,7 @@
-import { Column, Entity, PrimaryGeneratedColumn } from "typeorm";
+import { Column, Entity, ManyToOne, PrimaryGeneratedColumn } from "typeorm";
 import { TLog } from "../logger.types";
+import { Project } from "../../../project/entities/project.entity";
+import { Query } from "../../../query/entities/query.entity";
 
 @Entity()
 export class Log {
@@ -18,6 +20,12 @@ export class Log {
   })
   content: TLog;
 
+  @ManyToOne(() => Project, (project) => project.logs)
+  project: Project;
+
+  @ManyToOne(() => Query, (query) => query.logs)
+  query: Query;
+
   @Column({ type: "timestamp", default: () => "CURRENT_TIMESTAMP" })
   createdAt: Date;
 }

src/query/logger/logger.controller.ts

@@ -1,6 +1,15 @@
-import { Body, Controller, Get, Inject, Post, UseGuards } from "@nestjs/common";
+import {
+  Body,
+  Controller,
+  Get,
+  Inject,
+  Param,
+  Post,
+  UseGuards,
+} from "@nestjs/common";
 import { LoggerService } from "./logger.service";
 import { ApiTokenGuard } from "src/api/guards/api-token.guard";
+import { QueryGuard } from "../guards/query.guard";
 
 @Controller("logger")
 @UseGuards(ApiTokenGuard)
@@ -10,13 +19,14 @@ export class LoggerController {
     private readonly loggerService: LoggerService
   ) {}
 
-  @Get("/:traceId")
-  getByTraceId(@Inject("traceId") traceId: string) {
+  @Get("/:id/:traceId")
+  getByTraceId(@Param("traceId") traceId: string) {
     return this.loggerService.findByTraceId(traceId);
   }
 
-  @Post("/find")
-  find(
+  @Post("/:id/findAll")
+  findAll(
+    @Param("id") projectId: string,
     @Body()
     body: {
       traceId?: string;
@@ -27,6 +37,23 @@ export class LoggerController {
       offset: number;
     }
   ) {
-    return this.loggerService.find(body);
+    return this.loggerService.findByProjectId(projectId, body);
+  }
+
+  @Post("/:id/find")
+  @UseGuards(QueryGuard)
+  find(
+    @Param("id") _id: string,
+    @Body()
+    body: {
+      traceId?: string;
+      fromDate?: Date;
+      toDate?: Date;
+      url?: string;
+      limit: number;
+      offset: number;
+    }
+  ) {
+    return this.loggerService.find(_id, body);
   }
 }

src/query/logger/logger.service.ts

@@ -3,6 +3,7 @@ import { Log } from "./entities/log.entity";
 import { InjectRepository } from "@nestjs/typeorm";
 import { Repository } from "typeorm";
 import { TLog, TLogLine } from "./logger.types";
+import { Query } from "../entities/query.entity";
 
 @Injectable()
 export class LoggerService {
@@ -11,8 +12,13 @@ export class LoggerService {
     private readonly logRepository: Repository<Log>
   ) {}
 
-  async create(traceId: string, content: TLog): Promise<Log> {
-    const log = this.logRepository.create({ traceId, content });
+  async create(traceId: string, content: TLog, query: Query): Promise<Log> {
+    const log = this.logRepository.create({
+      traceId,
+      content,
+      query,
+      project: query.project,
+    });
     return await this.logRepository.save(log);
   }
 
@@ -20,14 +26,14 @@ export class LoggerService {
     return await this.logRepository.find({ where: { traceId } });
   }
 
-  async find(data: {
+  private prepareQuery(data: {
     traceId?: string;
     fromDate?: Date;
     toDate?: Date;
     url?: string;
     limit: number;
     offset: number;
-  }): Promise<Log[]> {
+  }) {
     const query = this.logRepository.createQueryBuilder("log");
 
     if (data.traceId) {
@@ -37,21 +43,57 @@ export class LoggerService {
     if (data.fromDate) {
       query.andWhere("log.createdAt >= :fromDate", { fromDate: data.fromDate });
     }
-
     if (data.toDate) {
       query.andWhere("log.createdAt <= :toDate", { toDate: data.toDate });
     }
-
     if (data.url) {
       query.andWhere("log.content LIKE :url", { url: `%${data.url}%` });
     }
 
     query.skip(data.offset).take(data.limit).orderBy("log.createdAt", "DESC");
 
+    return query;
+  }
+
+  async findByProjectId(
+    projectId: string,
+    data: {
+      traceId?: string;
+      fromDate?: Date;
+      toDate?: Date;
+      url?: string;
+      limit: number;
+      offset: number;
+    }
+  ): Promise<Log[]> {
+    const query = this.prepareQuery(data);
+    query.where("log.projectId = :projectId", { projectId });
+
     return await query.getMany();
   }
 
-  static log(logStack: TLog, log: TLog | TLogLine): TLog {
+  async find(
+    queryId: string,
+    data: {
+      traceId?: string;
+      fromDate?: Date;
+      toDate?: Date;
+      url?: string;
+      limit: number;
+      offset: number;
+    }
+  ): Promise<Log[]> {
+    const query = this.prepareQuery(data);
+    query.where("log.queryId = :queryId", { queryId });
+
+    return await query.getMany();
+  }
+
+  static log(logStack: TLog, log: TLogLine): TLog {
+    if (!log.timeStamp) {
+      log.timeStamp = Date.now();
+    }
+
     logStack.content.push(log);
     return logStack;
   }

src/query/logger/logger.types.ts

@@ -10,12 +10,12 @@ export enum TLogType {
 export type TLogLine = {
   content: string;
   type: TLogType;
-  timeStamp: number;
+  timeStamp?: number;
 };
 
 export interface TLog {
   traceId: string;
-  content: (TLogLine | TLog)[];
+  content: TLogLine[];
   payload?: any;
   headers: Record<string, string | string[] | undefined>;
   cookies: Record<string, string>;