From fbbbd6183886f032adebba001e1d7cd0cf94b999 Mon Sep 17 00:00:00 2001 From: Boris D Date: Mon, 22 Sep 2025 18:35:02 +0300 Subject: [PATCH] DB --- .vscode/tasks.json | 32 +++++++ src/migrations/1758550333691-defaults.ts | 18 ++++ .../1758551707113-query-modules-join.ts | 20 +++++ src/project/project.controller.ts | 4 +- src/query/entities/module.entity.ts | 3 - src/query/entities/plugin.entity.ts | 6 +- src/query/entities/query.entity.ts | 9 +- src/query/executer/query.executer.service.ts | 21 +++-- src/query/handler/query.handler.service.ts | 84 ++++++++++++++++++- src/vm/modules/async.js | 10 ++- src/vm/plugins/database.plugin.ts | 26 ++++-- src/vm/vm.class.ts | 18 +++- tests/base/case1-payload.js | 28 +++++++ tests/base/case1.ts | 5 +- tests/config.ts | 3 + tests/functions/addModule.ts | 12 ++- tests/functions/addPlugin.ts | 12 ++- tests/functions/createModule.ts | 12 ++- tests/functions/createPluign.ts | 14 ++-- tests/functions/createProject.ts | 3 +- tests/functions/createQuery.ts | 3 +- tests/functions/runQuery.ts | 3 +- tests/functions/updateQuery.ts | 3 +- 23 files changed, 282 insertions(+), 67 deletions(-) create mode 100644 .vscode/tasks.json create mode 100644 src/migrations/1758550333691-defaults.ts create mode 100644 src/migrations/1758551707113-query-modules-join.ts create mode 100644 tests/base/case1-payload.js create mode 100644 tests/config.ts diff --git a/.vscode/tasks.json b/.vscode/tasks.json new file mode 100644 index 0000000..6f7635b --- /dev/null +++ b/.vscode/tasks.json @@ -0,0 +1,32 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "label": "Run case1 test", + "type": "shell", + "command": "npx ts-node tests/base/case1.ts", + "args": [], + "isBackground": false, + "problemMatcher": [], + "group": "build" + }, + { + "label": "Generate join table migration", + "type": "shell", + "command": "npm run migration:generate -- src/migrations/query-modules-join", + "args": [], + "isBackground": false, + "problemMatcher": [], + "group": "build" + }, + { + "label": "Run DB migrations", + "type": "shell", + "command": "npm run migration:run", + "args": [], + "isBackground": false, + "problemMatcher": [], + "group": "build" + } + ] +} \ No newline at end of file diff --git a/src/migrations/1758550333691-defaults.ts b/src/migrations/1758550333691-defaults.ts new file mode 100644 index 0000000..4df87b0 --- /dev/null +++ b/src/migrations/1758550333691-defaults.ts @@ -0,0 +1,18 @@ +import { MigrationInterface, QueryRunner } from "typeorm"; + +export class Defaults1758550333691 implements MigrationInterface { + name = 'Defaults1758550333691' + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE \`module\` DROP COLUMN \`isInjectable\``); + await queryRunner.query(`ALTER TABLE \`plugin\` ADD \`queryId\` uuid NULL`); + await queryRunner.query(`ALTER TABLE \`plugin\` ADD CONSTRAINT \`FK_5162d18c3653d35ff4d104dd940\` FOREIGN KEY (\`queryId\`) REFERENCES \`query\`(\`id\`) ON DELETE NO ACTION ON UPDATE NO ACTION`); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE \`plugin\` DROP FOREIGN KEY \`FK_5162d18c3653d35ff4d104dd940\``); + await queryRunner.query(`ALTER TABLE \`plugin\` DROP COLUMN \`queryId\``); + await queryRunner.query(`ALTER TABLE \`module\` ADD \`isInjectable\` tinyint NOT NULL DEFAULT 1`); + } + +} diff --git a/src/migrations/1758551707113-query-modules-join.ts b/src/migrations/1758551707113-query-modules-join.ts new file mode 100644 index 0000000..60ddd8c --- /dev/null +++ b/src/migrations/1758551707113-query-modules-join.ts @@ -0,0 +1,20 @@ +import { MigrationInterface, QueryRunner } from "typeorm"; + +export class QueryModulesJoin1758551707113 implements MigrationInterface { + name = 'QueryModulesJoin1758551707113' + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`CREATE TABLE \`query_modules_module\` (\`queryId\` uuid NOT NULL, \`moduleId\` uuid NOT NULL, INDEX \`IDX_12121324c524e12538de4948ce\` (\`queryId\`), INDEX \`IDX_7fb42dbc85874aafbd4bfb1c10\` (\`moduleId\`), PRIMARY KEY (\`queryId\`, \`moduleId\`)) ENGINE=InnoDB`); + await queryRunner.query(`ALTER TABLE \`query_modules_module\` ADD CONSTRAINT \`FK_12121324c524e12538de4948cee\` FOREIGN KEY (\`queryId\`) REFERENCES \`query\`(\`id\`) ON DELETE CASCADE ON UPDATE CASCADE`); + await queryRunner.query(`ALTER TABLE \`query_modules_module\` ADD CONSTRAINT \`FK_7fb42dbc85874aafbd4bfb1c101\` FOREIGN KEY (\`moduleId\`) REFERENCES \`module\`(\`id\`) ON DELETE NO ACTION ON UPDATE NO ACTION`); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE \`query_modules_module\` DROP FOREIGN KEY \`FK_7fb42dbc85874aafbd4bfb1c101\``); + await queryRunner.query(`ALTER TABLE \`query_modules_module\` DROP FOREIGN KEY \`FK_12121324c524e12538de4948cee\``); + await queryRunner.query(`DROP INDEX \`IDX_7fb42dbc85874aafbd4bfb1c10\` ON \`query_modules_module\``); + await queryRunner.query(`DROP INDEX \`IDX_12121324c524e12538de4948ce\` ON \`query_modules_module\``); + await queryRunner.query(`DROP TABLE \`query_modules_module\``); + } + +} diff --git a/src/project/project.controller.ts b/src/project/project.controller.ts index 6b06963..6956661 100644 --- a/src/project/project.controller.ts +++ b/src/project/project.controller.ts @@ -1,14 +1,14 @@ import { Body, Controller, Inject, Put } from "@nestjs/common"; import { ProjectService } from "./project.service"; -@Controller("") +@Controller("project") export class ProjectController { constructor( @Inject(ProjectService) private readonly projectService: ProjectService ) {} - @Put("project/create") + @Put("create") createProject(@Body() body: { name: string }) { return this.projectService.create(body.name); } diff --git a/src/query/entities/module.entity.ts b/src/query/entities/module.entity.ts index 08357e6..0e31c1b 100644 --- a/src/query/entities/module.entity.ts +++ b/src/query/entities/module.entity.ts @@ -12,9 +12,6 @@ export class VMModule { @Column({ type: "varchar", length: 255 }) name: string; - @Column({ type: "tinyint", default: 1 }) - isInjectable: number; - @ManyToMany(() => Query, (query) => query.modules) queries: Query[]; } diff --git a/src/query/entities/plugin.entity.ts b/src/query/entities/plugin.entity.ts index c5a9d8a..af8f102 100644 --- a/src/query/entities/plugin.entity.ts +++ b/src/query/entities/plugin.entity.ts @@ -1,4 +1,4 @@ -import { Column, Entity, ManyToMany, PrimaryGeneratedColumn } from "typeorm"; +import { Column, Entity, ManyToOne, PrimaryGeneratedColumn } from "typeorm"; import { Query } from "./query.entity"; @Entity("plugin") @@ -12,8 +12,8 @@ export class VmPlugin { @Column({ type: "varchar", length: 255, nullable: false }) name: string; - @ManyToMany(() => Query, (query) => query.plugins) - queries: Query[]; + @ManyToOne(() => Query, (query) => query.plugins) + query: Query; @Column({ type: "varchar", length: 255 }) config: string; diff --git a/src/query/entities/query.entity.ts b/src/query/entities/query.entity.ts index 906909f..a74c556 100644 --- a/src/query/entities/query.entity.ts +++ b/src/query/entities/query.entity.ts @@ -4,7 +4,9 @@ import { Entity, ManyToMany, ManyToOne, + OneToMany, PrimaryGeneratedColumn, + JoinTable, } from "typeorm"; import { VMModule } from "./module.entity"; import { VmPlugin } from "./plugin.entity"; @@ -24,8 +26,13 @@ export class Query { isActive: number; @ManyToMany(() => VMModule, (module) => module.queries) + @JoinTable({ + name: "query_modules_module", + joinColumn: { name: "queryId", referencedColumnName: "id" }, + inverseJoinColumn: { name: "moduleId", referencedColumnName: "id" }, + }) modules: VMModule[]; - @ManyToMany(() => VmPlugin, (plugin) => plugin.queries) + @OneToMany(() => VmPlugin, (plugin) => plugin.query) plugins: VmPlugin[]; } diff --git a/src/query/executer/query.executer.service.ts b/src/query/executer/query.executer.service.ts index 70ccdaa..4ea7ee8 100644 --- a/src/query/executer/query.executer.service.ts +++ b/src/query/executer/query.executer.service.ts @@ -16,6 +16,7 @@ export class QueryExecuterService { async runQuery(token: string, queryData: any) { const query = await this.queryRepository.findOne({ where: { id: token }, + relations: ["modules", "plugins"], }); if (!query) { @@ -42,16 +43,18 @@ export class QueryExecuterService { modules: query.modules.map((module) => { return new VModule(module.name, module.sourcePath); }), - plugins: query.plugins.map((plugin) => { - switch (plugin.class) { - case "DATABASE": { - const config = JSON.parse(plugin.config); - return DatabasePlugin.init(plugin.name, config); + plugins: await Promise.all( + query.plugins.map((plugin) => { + switch (plugin.class) { + case "DATABASE": { + const config = JSON.parse(plugin.config); + return DatabasePlugin.init(plugin.name, config); + } + default: + throw new Error(`Unknown plugin class: ${plugin.class}`); } - default: - throw new Error(`Unknown plugin class: ${plugin.class}`); - } - }), + }) + ), }); return await vm.init(); diff --git a/src/query/handler/query.handler.service.ts b/src/query/handler/query.handler.service.ts index b033076..caaf40c 100644 --- a/src/query/handler/query.handler.service.ts +++ b/src/query/handler/query.handler.service.ts @@ -19,6 +19,84 @@ export class QueryHandlerService { private readonly projectService: ProjectService ) {} + private async createOrGetVmModule( + name: string, + sourcePath: string + ): Promise { + const vmModule = await this.moduleRepository.findOne({ + where: { + name, + }, + }); + + if (vmModule) { + return vmModule; + } + + const newModule = this.moduleRepository.create({ + sourcePath, + name, + }); + + return this.moduleRepository.save(newModule); + } + + private async createOrGetPlugin( + name: string, + className: string, + query: Query, + config: Record + ): Promise { + const plugin = await this.pluginRepository.findOne({ + where: { + name, + query: { + id: query.id, + }, + }, + }); + + if (plugin) { + return plugin; + } + + const newPlugin = this.pluginRepository.create({ + name, + class: className, + config: JSON.stringify(config), + query, + }); + + return this.pluginRepository.save(newPlugin); + } + + // TODO: make it nice + async createDefaults(query: Query): Promise { + const asyncCallModule = await this.createOrGetVmModule( + "asyncCall", + "dist/vm/modules/async.js" + ); + const squelModule = await this.createOrGetVmModule( + "query", + "node_modules/squel/dist/squel.min.js" + ); + + query.modules = [asyncCallModule, squelModule]; + + const dbPlugin = await this.createOrGetPlugin("db", "DATABASE", query, { + // TODO: take it from database handler + host: process.env.DB_HOST, + user: process.env.DB_USERNAME, + port: process.env.DB_PORT, + password: process.env.DB_PASSWORD, + database: "test", + }); + + query.plugins = [dbPlugin]; + + await this.queryRepository.save(query); + } + async createQuery(queryData: { projectToken: string; source: string }) { const project = await this.projectService.findByToken( queryData.projectToken @@ -32,7 +110,11 @@ export class QueryHandlerService { delete queryData.projectToken; const query = this.queryRepository.create(queryData); - return this.queryRepository.save(query); + await this.queryRepository.save(query); + + await this.createDefaults(query); + + return query; } async updateQuery(id: string, updateData: Partial) { diff --git a/src/vm/modules/async.js b/src/vm/modules/async.js index 24b0133..3602c85 100644 --- a/src/vm/modules/async.js +++ b/src/vm/modules/async.js @@ -1,4 +1,12 @@ // eslint-disable-next-line @typescript-eslint/no-unused-vars async function asyncCall(reference, args) { - return await reference.apply(undefined, args, { result: { promise: true } }); + if (!Array.isArray(args)) { + args = [args]; + } + + const resJson = await reference.apply(undefined, args, { + result: { promise: true }, + }); + + return JSON.parse(resJson); } diff --git a/src/vm/plugins/database.plugin.ts b/src/vm/plugins/database.plugin.ts index f00f82a..92c42b7 100644 --- a/src/vm/plugins/database.plugin.ts +++ b/src/vm/plugins/database.plugin.ts @@ -12,31 +12,39 @@ export class DatabasePlugin extends Plugin { super(name); } - static init( + static async init( name: string, config: { host: string; + port: number; user: string; password: string; database: string; } - ): DatabasePlugin { - const dbConnection = mysql.createConnection({ + ): Promise { + const dbConnection = await mysql.createConnection({ host: config.host, user: config.user, + port: config.port, password: config.password, database: config.database, + enableKeepAlive: true, }); return new DatabasePlugin(name, dbConnection); } - async run(query): Promise<{ - rows: any[]; - fields: any[]; - }> { - const [rows, fields] = await this.dbConnection.execute(query); - return { rows, fields }; + async run(query): Promise { + try { + const [rows, fields] = await this.dbConnection.execute(query); + + return JSON.stringify({ + rows: rows, + fields: fields ?? [], + }); + } catch (error) { + console.log("error", error); + } } onFinish() { diff --git a/src/vm/vm.class.ts b/src/vm/vm.class.ts index c13892f..f756160 100644 --- a/src/vm/vm.class.ts +++ b/src/vm/vm.class.ts @@ -28,13 +28,17 @@ export class Vm { this.jail.set("global", this.jail.derefInto()); for (const mod of this.modules) { - this.jail.setSync(mod.getName(), mod.getSource()); + await this.context.eval(mod.getSource()); } for (const plugin of this.plugins) { this.jail.setSync( plugin.getName(), - new ivm.Reference(plugin.run.bind(plugin)) + new ivm.Reference(async (...args) => { + const res = await plugin.run(...args); + + return res; + }) ); } @@ -61,6 +65,9 @@ export class Vm { }); // TODO: log + this.setFunction("log", (...args) => { + console.log("vm log:", args); + }); this.setFunction("error", (error: any) => { console.error("Script error:", error); @@ -82,9 +89,12 @@ export class Vm { const compiledScript = await this.isolate.compileScript(scriptWithResult); compiledScript.run(this.context); - this.onFinish(); - return await resultPromise; + try { + return await resultPromise; + } finally { + this.onFinish(); + } } onFinish() { diff --git a/tests/base/case1-payload.js b/tests/base/case1-payload.js new file mode 100644 index 0000000..e09dd6d --- /dev/null +++ b/tests/base/case1-payload.js @@ -0,0 +1,28 @@ +/* eslint-disable @typescript-eslint/no-unused-vars */ +/* eslint-disable no-undef */ + +async function insert() { + const res = await asyncCall( + db, + squel.insert().into("testTable").set("col", "test me now").toString() + ); + + return res; +} + +function createSQL(id) { + return squel.select().from("testTable").where("id = ?", id).toString(); +} + +async function main(input) { + const inserted = await insert(); + + log(inserted); + + const sql = createSQL(inserted.rows.insertId); + const res = await asyncCall(db, sql); + + log(res.rows); + + return res; +} diff --git a/tests/base/case1.ts b/tests/base/case1.ts index 91290ca..9f805d8 100644 --- a/tests/base/case1.ts +++ b/tests/base/case1.ts @@ -1,14 +1,17 @@ import createProject from "../functions/createProject"; import createQuery from "../functions/createQuery"; import runQuery from "../functions/runQuery"; +import * as fs from "fs"; +import * as path from "path"; (async () => { try { const project = await createProject("Test Project"); + const payloadPath = path.join(__dirname, "case1-payload.js"); const query = await createQuery( project, - "async function main(input) { return `Hello, ${input.name}!`; }" + fs.readFileSync(payloadPath, { encoding: "utf-8" }) ); console.log(query); diff --git a/tests/config.ts b/tests/config.ts new file mode 100644 index 0000000..873878e --- /dev/null +++ b/tests/config.ts @@ -0,0 +1,3 @@ +export const config = { + url: "http://localhost:3054", +}; diff --git a/tests/functions/addModule.ts b/tests/functions/addModule.ts index 98670f8..e6256bb 100644 --- a/tests/functions/addModule.ts +++ b/tests/functions/addModule.ts @@ -1,14 +1,12 @@ import axios from "axios"; +import { config } from "tests/config"; export default async (query: { id: string }, module: { id: string }) => { try { - const response = await axios.post( - "http://localhost:3000/query/module/add", - { - queryId: query.id, - moduleId: module.id, - } - ); + const response = await axios.post(`${config.url}/query/module/add`, { + queryId: query.id, + moduleId: module.id, + }); return response; } catch (error) { diff --git a/tests/functions/addPlugin.ts b/tests/functions/addPlugin.ts index 8b2621e..3b08263 100644 --- a/tests/functions/addPlugin.ts +++ b/tests/functions/addPlugin.ts @@ -1,14 +1,12 @@ import axios from "axios"; +import { config } from "tests/config"; export default async (query: { id: string }, plugin: { id: string }) => { try { - const response = await axios.post( - "http://localhost:3000/query/plugin/add", - { - queryId: query.id, - pluginId: plugin.id, - } - ); + const response = await axios.post(`${config.url}/query/plugin/add`, { + queryId: query.id, + pluginId: plugin.id, + }); return response; } catch (error) { diff --git a/tests/functions/createModule.ts b/tests/functions/createModule.ts index a2b81e5..3115bea 100644 --- a/tests/functions/createModule.ts +++ b/tests/functions/createModule.ts @@ -1,14 +1,12 @@ import axios from "axios"; +import { config } from "tests/config"; export default async (name: string, sourcePath: string) => { try { - const response = await axios.post( - "http://localhost:3000/query/module/create", - { - name, - sourcePath, - } - ); + const response = await axios.post(`${config.url}/query/module/create`, { + name, + sourcePath, + }); return response; } catch (error) { diff --git a/tests/functions/createPluign.ts b/tests/functions/createPluign.ts index 6f13117..3037ae1 100644 --- a/tests/functions/createPluign.ts +++ b/tests/functions/createPluign.ts @@ -1,15 +1,13 @@ import axios from "axios"; +import { config as appConfig } from "../config"; export default async (name: string, className: string, config: string) => { try { - const response = await axios.post( - "http://localhost:3000/query/plugin/create", - { - name, - class: className, - config, - } - ); + const response = await axios.post(`${appConfig.url}/query/plugin/create`, { + name, + class: className, + config, + }); return response; } catch (error) { diff --git a/tests/functions/createProject.ts b/tests/functions/createProject.ts index cbdeda1..30547f1 100644 --- a/tests/functions/createProject.ts +++ b/tests/functions/createProject.ts @@ -1,8 +1,9 @@ import axios from "axios"; +import { config } from "../config"; const createProject = async (name: string) => { try { - const response = await axios.put("http://localhost:3000/project/create", { + const response = await axios.put(`${config.url}/project/create`, { name: name, }); diff --git a/tests/functions/createQuery.ts b/tests/functions/createQuery.ts index 0d02a12..5c87a2c 100644 --- a/tests/functions/createQuery.ts +++ b/tests/functions/createQuery.ts @@ -1,8 +1,9 @@ import axios from "axios"; +import { config } from "../config"; export default async (project: { token: string }, source: string) => { try { - const response = await axios.post("http://localhost:3000/query/create", { + const response = await axios.post(`${config.url}/query/create`, { source, projectToken: project.token, }); diff --git a/tests/functions/runQuery.ts b/tests/functions/runQuery.ts index ed62c77..4087a95 100644 --- a/tests/functions/runQuery.ts +++ b/tests/functions/runQuery.ts @@ -1,9 +1,10 @@ import axios from "axios"; +import { config } from "../config"; export default async (token: string, queryData: Record) => { try { const response = await axios.post( - `http://localhost:3000/query/run/${token}`, + `${config.url}/query/run/${token}`, queryData ); diff --git a/tests/functions/updateQuery.ts b/tests/functions/updateQuery.ts index d0c5d59..907e084 100644 --- a/tests/functions/updateQuery.ts +++ b/tests/functions/updateQuery.ts @@ -1,8 +1,9 @@ import axios from "axios"; +import { config } from "../config"; export default async (query: { id: string; source: string }) => { try { - const response = await axios.post("http://localhost:3000/query/update", { + const response = await axios.post(`${config.url}/query/update`, { queryId: query.id, source: query.source, });