From 967c89108a14ec3ddaeb5ccd5e15542dbc755de5 Mon Sep 17 00:00:00 2001 From: lborv Date: Sat, 11 Oct 2025 19:36:43 +0300 Subject: [PATCH] feat: implement logging enhancements; add projectId and queryId to log entity; update query and logger services for improved logging; refactor query execution to support call stack tracking --- src/migrations/1760199448968-logsPerQuery.ts | 31 +++++++++++ src/project/entities/project.entity.ts | 4 ++ src/query/base/base-query.controller.ts | 17 ++++-- src/query/entities/query.entity.ts | 12 ++++- src/query/executer/query.executer.service.ts | 57 +++++++++++++++----- src/query/logger/entities/log.entity.ts | 10 +++- src/query/logger/logger.controller.ts | 39 +++++++++++--- src/query/logger/logger.service.ts | 56 ++++++++++++++++--- src/query/logger/logger.types.ts | 4 +- src/queue/processors/query.processor.ts | 1 + src/vm/plugin.class.ts | 6 +++ src/vm/plugins/query.plugin.ts | 1 + src/vm/vm.class.ts | 33 ++++++++---- tests/base/case1-payload.js | 1 - 14 files changed, 226 insertions(+), 46 deletions(-) create mode 100644 src/migrations/1760199448968-logsPerQuery.ts diff --git a/src/migrations/1760199448968-logsPerQuery.ts b/src/migrations/1760199448968-logsPerQuery.ts new file mode 100644 index 0000000..3ae8ce5 --- /dev/null +++ b/src/migrations/1760199448968-logsPerQuery.ts @@ -0,0 +1,31 @@ +import { MigrationInterface, QueryRunner } from "typeorm"; + +export class LogsPerQuery1760199448968 implements MigrationInterface { + name = "LogsPerQuery1760199448968"; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `ALTER TABLE \`log\` ADD \`projectId\` varchar(36) NULL` + ); + await queryRunner.query( + `ALTER TABLE \`log\` ADD \`queryId\` varchar(36) NULL` + ); + await queryRunner.query( + `ALTER TABLE \`log\` ADD CONSTRAINT \`FK_0c0ad31dd4033de83a2c47f2c82\` FOREIGN KEY (\`projectId\`) REFERENCES \`project\`(\`id\`) ON DELETE NO ACTION ON UPDATE NO ACTION` + ); + await queryRunner.query( + `ALTER TABLE \`log\` ADD CONSTRAINT \`FK_7867d6fbda5d177a3727cedece3\` FOREIGN KEY (\`queryId\`) REFERENCES \`query\`(\`id\`) ON DELETE NO ACTION ON UPDATE NO ACTION` + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `ALTER TABLE \`log\` DROP FOREIGN KEY \`FK_7867d6fbda5d177a3727cedece3\`` + ); + await queryRunner.query( + `ALTER TABLE \`log\` DROP FOREIGN KEY \`FK_0c0ad31dd4033de83a2c47f2c82\`` + ); + await queryRunner.query(`ALTER TABLE \`log\` DROP COLUMN \`queryId\``); + await queryRunner.query(`ALTER TABLE \`log\` DROP COLUMN \`projectId\``); + } +} diff --git a/src/project/entities/project.entity.ts b/src/project/entities/project.entity.ts index 2d02b1c..0d094a5 100644 --- a/src/project/entities/project.entity.ts +++ b/src/project/entities/project.entity.ts @@ -13,6 +13,7 @@ import { import { Database } from "../../databaseManager/entities/database.entity"; import { FunctionEntity } from "../../query/entities/function.entity"; import { RedisNode } from "../../redisManager/entities/redis.node.entity"; +import { Log } from "../../query/logger/entities/log.entity"; @Entity("project") export class Project { @@ -29,6 +30,9 @@ export class Project { @JoinColumn() database: Database; + @OneToMany(() => Log, (log) => log.project) + logs: Log[]; + @OneToMany(() => Query, (query) => query.project) queries: Query[]; diff --git a/src/query/base/base-query.controller.ts b/src/query/base/base-query.controller.ts index 273d049..031da44 100644 --- a/src/query/base/base-query.controller.ts +++ b/src/query/base/base-query.controller.ts @@ -5,10 +5,11 @@ import { Inject, Param, Post, + Req, Res, UseGuards, } from "@nestjs/common"; -import { Response } from "express"; +import { Response, Request } from "express"; import { QueryHandlerService } from "../handler/query.handler.service"; import { ApiTokenGuard } from "src/api/guards/api-token.guard"; import { QueryExecuterService } from "../executer/query.executer.service"; @@ -16,6 +17,7 @@ import { QueryGuard } from "src/query/guards/query.guard"; import { LoggerService } from "../logger/logger.service"; import { TLogType } from "../logger/logger.types"; import { QueryResponse } from "src/vm/vm.constants"; +import { Query } from "../entities/query.entity"; @UseGuards(ApiTokenGuard) export abstract class BaseQueryController { @@ -52,7 +54,8 @@ export abstract class BaseQueryController { @Param("id") id: string, @Body() query: Record, @Headers() headers: Record, - @Res() res: Response + @Res() res: Response, + @Req() req: Request & { query: Query } ) { let queryResult: QueryResponse; const loggerTraceId = @@ -68,7 +71,7 @@ export abstract class BaseQueryController { response: null, content: [], }, - { content: "", type: TLogType.info, timeStamp: new Date().getTime() } + { content: `Query ${id} started`, type: TLogType.info } ); try { @@ -90,7 +93,7 @@ export abstract class BaseQueryController { timeStamp: new Date().getTime(), }); log.endTime = new Date().getTime(); - await this.loggerService.create(log.traceId, log); + await this.loggerService.create(log.traceId, log, req.query); res.status(500).send({ error: "Internal Server Error" }); return; } @@ -100,7 +103,11 @@ export abstract class BaseQueryController { const res = JSON.parse(JSON.stringify(queryResult)); delete res.log; queryResult.log.response = res; - await this.loggerService.create(queryResult.log.traceId, queryResult.log); + await this.loggerService.create( + queryResult.log.traceId, + queryResult.log, + req.query + ); } res.status(queryResult?.statusCode || 200); diff --git a/src/query/entities/query.entity.ts b/src/query/entities/query.entity.ts index e44fa0e..b1218da 100644 --- a/src/query/entities/query.entity.ts +++ b/src/query/entities/query.entity.ts @@ -1,5 +1,12 @@ import { Project } from "../../project/entities/project.entity"; -import { Column, Entity, ManyToOne, PrimaryGeneratedColumn } from "typeorm"; +import { + Column, + Entity, + ManyToOne, + OneToMany, + PrimaryGeneratedColumn, +} from "typeorm"; +import { Log } from "../logger/entities/log.entity"; @Entity("query") export class Query { @@ -9,6 +16,9 @@ export class Query { @ManyToOne(() => Project, (project) => project.queries) project: Project; + @OneToMany(() => Log, (log) => log.query) + logs: Log[]; + @Column({ type: "longtext" }) source: string; diff --git a/src/query/executer/query.executer.service.ts b/src/query/executer/query.executer.service.ts index eecf788..55cac36 100644 --- a/src/query/executer/query.executer.service.ts +++ b/src/query/executer/query.executer.service.ts @@ -1,5 +1,5 @@ import { RedisNodeService } from "./../../redisManager/redisNode/redis.node.service"; -import { Inject, Injectable, Logger } from "@nestjs/common"; +import { Inject, Injectable } from "@nestjs/common"; import { InjectRepository } from "@nestjs/typeorm"; import { Query } from "../entities/query.entity"; import { Repository } from "typeorm"; @@ -62,6 +62,11 @@ export class QueryExecuterService { headers: Record = {}, cookies: Record = {} ): Promise { + LoggerService.log(log, { + content: `Queueing query ${token}`, + type: TLogType.info, + }); + const job = await this.queryQueue.add( `${new Date().getTime()}_${token}`, { @@ -85,6 +90,7 @@ export class QueryExecuterService { async runQuery( token: string, queryData: any, + callStack: number = 0, headers: Record = {}, cookies: Record = {}, log: TLog = null @@ -95,6 +101,10 @@ export class QueryExecuterService { }); if (!query) { + LoggerService.log(log, { + content: `Query with id ${token} not found`, + type: TLogType.error, + }); throw new Error("Query not found"); } @@ -105,7 +115,17 @@ export class QueryExecuterService { cookies["x-session-id"] = session.sessionId; } - const vm = await this.createVm(query, log, cookies["x-session-id"]); + LoggerService.log(log, { + content: `Running query ${query.id}`, + type: TLogType.info, + }); + + const vm = await this.createVm( + query, + log, + callStack, + cookies["x-session-id"] + ); const result = await vm.runScript( this.clearImports(query.source), queryData, @@ -126,10 +146,20 @@ export class QueryExecuterService { ).sessionId; } + LoggerService.log(log, { + content: `Query ${query.id} executed successfully`, + type: TLogType.info, + }); + return result; } - private async createVm(query: Query, log: TLog, sessionId: string = null) { + private async createVm( + query: Query, + log: TLog, + callStack: number, + sessionId: string = null + ): Promise { const imports = this.parseImports(query.source); const importsParsed = imports.map((imp) => { const item = imp.split("/"); @@ -172,15 +202,18 @@ export class QueryExecuterService { ); } - const vm = new Vm({ - memoryLimit: 128, - timeLimit: BigInt(100e9), - cpuTimeLimit: BigInt(5e9), - modules: modules, - plugins: plugins, - functions: functions, - log, - }); + const vm = new Vm( + { + memoryLimit: 128, + timeLimit: BigInt(100e9), + cpuTimeLimit: BigInt(5e9), + modules: modules, + plugins: plugins, + functions: functions, + log, + }, + callStack + ); return await vm.init(); } diff --git a/src/query/logger/entities/log.entity.ts b/src/query/logger/entities/log.entity.ts index b031526..1fad412 100644 --- a/src/query/logger/entities/log.entity.ts +++ b/src/query/logger/entities/log.entity.ts @@ -1,5 +1,7 @@ -import { Column, Entity, PrimaryGeneratedColumn } from "typeorm"; +import { Column, Entity, ManyToOne, PrimaryGeneratedColumn } from "typeorm"; import { TLog } from "../logger.types"; +import { Project } from "../../../project/entities/project.entity"; +import { Query } from "../../../query/entities/query.entity"; @Entity() export class Log { @@ -18,6 +20,12 @@ export class Log { }) content: TLog; + @ManyToOne(() => Project, (project) => project.logs) + project: Project; + + @ManyToOne(() => Query, (query) => query.logs) + query: Query; + @Column({ type: "timestamp", default: () => "CURRENT_TIMESTAMP" }) createdAt: Date; } diff --git a/src/query/logger/logger.controller.ts b/src/query/logger/logger.controller.ts index 65b67a6..37885ea 100644 --- a/src/query/logger/logger.controller.ts +++ b/src/query/logger/logger.controller.ts @@ -1,6 +1,15 @@ -import { Body, Controller, Get, Inject, Post, UseGuards } from "@nestjs/common"; +import { + Body, + Controller, + Get, + Inject, + Param, + Post, + UseGuards, +} from "@nestjs/common"; import { LoggerService } from "./logger.service"; import { ApiTokenGuard } from "src/api/guards/api-token.guard"; +import { QueryGuard } from "../guards/query.guard"; @Controller("logger") @UseGuards(ApiTokenGuard) @@ -10,13 +19,14 @@ export class LoggerController { private readonly loggerService: LoggerService ) {} - @Get("/:traceId") - getByTraceId(@Inject("traceId") traceId: string) { + @Get("/:id/:traceId") + getByTraceId(@Param("traceId") traceId: string) { return this.loggerService.findByTraceId(traceId); } - @Post("/find") - find( + @Post("/:id/findAll") + findAll( + @Param("id") projectId: string, @Body() body: { traceId?: string; @@ -27,6 +37,23 @@ export class LoggerController { offset: number; } ) { - return this.loggerService.find(body); + return this.loggerService.findByProjectId(projectId, body); + } + + @Post("/:id/find") + @UseGuards(QueryGuard) + find( + @Param("id") _id: string, + @Body() + body: { + traceId?: string; + fromDate?: Date; + toDate?: Date; + url?: string; + limit: number; + offset: number; + } + ) { + return this.loggerService.find(_id, body); } } diff --git a/src/query/logger/logger.service.ts b/src/query/logger/logger.service.ts index c0f7470..32dc96c 100644 --- a/src/query/logger/logger.service.ts +++ b/src/query/logger/logger.service.ts @@ -3,6 +3,7 @@ import { Log } from "./entities/log.entity"; import { InjectRepository } from "@nestjs/typeorm"; import { Repository } from "typeorm"; import { TLog, TLogLine } from "./logger.types"; +import { Query } from "../entities/query.entity"; @Injectable() export class LoggerService { @@ -11,8 +12,13 @@ export class LoggerService { private readonly logRepository: Repository ) {} - async create(traceId: string, content: TLog): Promise { - const log = this.logRepository.create({ traceId, content }); + async create(traceId: string, content: TLog, query: Query): Promise { + const log = this.logRepository.create({ + traceId, + content, + query, + project: query.project, + }); return await this.logRepository.save(log); } @@ -20,14 +26,14 @@ export class LoggerService { return await this.logRepository.find({ where: { traceId } }); } - async find(data: { + private prepareQuery(data: { traceId?: string; fromDate?: Date; toDate?: Date; url?: string; limit: number; offset: number; - }): Promise { + }) { const query = this.logRepository.createQueryBuilder("log"); if (data.traceId) { @@ -37,21 +43,57 @@ export class LoggerService { if (data.fromDate) { query.andWhere("log.createdAt >= :fromDate", { fromDate: data.fromDate }); } - if (data.toDate) { query.andWhere("log.createdAt <= :toDate", { toDate: data.toDate }); } - if (data.url) { query.andWhere("log.content LIKE :url", { url: `%${data.url}%` }); } query.skip(data.offset).take(data.limit).orderBy("log.createdAt", "DESC"); + return query; + } + + async findByProjectId( + projectId: string, + data: { + traceId?: string; + fromDate?: Date; + toDate?: Date; + url?: string; + limit: number; + offset: number; + } + ): Promise { + const query = this.prepareQuery(data); + query.where("log.projectId = :projectId", { projectId }); + return await query.getMany(); } - static log(logStack: TLog, log: TLog | TLogLine): TLog { + async find( + queryId: string, + data: { + traceId?: string; + fromDate?: Date; + toDate?: Date; + url?: string; + limit: number; + offset: number; + } + ): Promise { + const query = this.prepareQuery(data); + query.where("log.queryId = :queryId", { queryId }); + + return await query.getMany(); + } + + static log(logStack: TLog, log: TLogLine): TLog { + if (!log.timeStamp) { + log.timeStamp = Date.now(); + } + logStack.content.push(log); return logStack; } diff --git a/src/query/logger/logger.types.ts b/src/query/logger/logger.types.ts index 9c8bed9..dee1f28 100644 --- a/src/query/logger/logger.types.ts +++ b/src/query/logger/logger.types.ts @@ -10,12 +10,12 @@ export enum TLogType { export type TLogLine = { content: string; type: TLogType; - timeStamp: number; + timeStamp?: number; }; export interface TLog { traceId: string; - content: (TLogLine | TLog)[]; + content: TLogLine[]; payload?: any; headers: Record; cookies: Record; diff --git a/src/queue/processors/query.processor.ts b/src/queue/processors/query.processor.ts index d5a2183..799e1ff 100644 --- a/src/queue/processors/query.processor.ts +++ b/src/queue/processors/query.processor.ts @@ -24,6 +24,7 @@ export class QueryProcessor extends WorkerHost { return await this.queryExecuterService.runQuery( token, queryData, + 0, headers, cookies, log diff --git a/src/vm/plugin.class.ts b/src/vm/plugin.class.ts index aaee4b3..2441df0 100644 --- a/src/vm/plugin.class.ts +++ b/src/vm/plugin.class.ts @@ -5,9 +5,11 @@ export abstract class Plugin { protected log: TLog; protected cookies: Record; protected headers: Record; + protected callStack: number; constructor(name: string, protected methods: string[] = []) { this.name = name; + this.callStack = 0; } getName(): string { @@ -26,6 +28,10 @@ export abstract class Plugin { this.headers = headers; } + setCallStack(callStack: number) { + this.callStack = callStack; + } + setCookies(cookies: Record) { this.cookies = cookies; } diff --git a/src/vm/plugins/query.plugin.ts b/src/vm/plugins/query.plugin.ts index 43547ad..f28eeed 100644 --- a/src/vm/plugins/query.plugin.ts +++ b/src/vm/plugins/query.plugin.ts @@ -33,6 +33,7 @@ export class QueryPlugin extends Plugin { return await this.QueryExecuterService.runQuery( id, data, + this.callStack + 1, this.headers, this.cookies, this.log diff --git a/src/vm/vm.class.ts b/src/vm/vm.class.ts index 0ae56c7..9db77bd 100644 --- a/src/vm/vm.class.ts +++ b/src/vm/vm.class.ts @@ -19,18 +19,28 @@ export class Vm { private log: TLog; private headers?: Record; private cookies?: Record; + private callStack = 0; + + constructor( + configs: { + memoryLimit: number; + timeLimit?: bigint; + cpuTimeLimit?: bigint; + modules: VModule[]; + plugins: Plugin[]; + functions: string[]; + log: TLog; + headers?: Record; + cookies?: Record; + }, + callStack = 0 + ) { + this.callStack = callStack; + + if (this.callStack > 5) { + throw new Error("Maximum call stack size exceeded"); + } - constructor(configs: { - memoryLimit: number; - timeLimit?: bigint; - cpuTimeLimit?: bigint; - modules: VModule[]; - plugins: Plugin[]; - functions: string[]; - log: TLog; - headers?: Record; - cookies?: Record; - }) { this.memoryLimit = configs.memoryLimit; this.modules = configs.modules; this.plugins = configs.plugins; @@ -71,6 +81,7 @@ export class Vm { plugin.setLog(this.log); plugin.setHeaders(this.headers); plugin.setCookies(this.cookies); + plugin.setCallStack(this.callStack); const result = await plugin[method](...args); if (result && result.log) { diff --git a/tests/base/case1-payload.js b/tests/base/case1-payload.js index 0289d47..e49ce78 100644 --- a/tests/base/case1-payload.js +++ b/tests/base/case1-payload.js @@ -6,7 +6,6 @@ import "plugin/db"; import "plugin/axios"; import "plugin/session"; import "plugin/query"; -import { query } from "express"; function createSQL(id) { return squel.select().from("test").where("id = ?", id).toString();