Commit dfa55cc2 authored by jaden's avatar jaden 🏄

Initial commit

parents
File added
module.exports = {
parser: '@typescript-eslint/parser',
parserOptions: {
project: 'tsconfig.json',
tsconfigRootDir: __dirname,
sourceType: 'module',
},
plugins: ['@typescript-eslint/eslint-plugin'],
extends: [
'plugin:@typescript-eslint/recommended',
'plugin:prettier/recommended',
],
root: true,
env: {
node: true,
jest: true,
},
ignorePatterns: ['.eslintrc.js'],
rules: {
'@typescript-eslint/interface-name-prefix': 'off',
'@typescript-eslint/explicit-function-return-type': 'off',
'@typescript-eslint/explicit-module-boundary-types': 'off',
'@typescript-eslint/no-explicit-any': 'off',
},
};
name: Build and Push Docker Image
on:
push:
branches:
- fusionTeach
jobs:
build-and-push:
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v2
- name: Install dependencies & build Nest app
run: |
npm install -g pnpm
pnpm install
npm run build
- name: Login to Docker Hub
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Build and push Docker image
uses: docker/build-push-action@v2
with:
context: .
push: true
tags: jadenxiong/chat-query-backend:fusionTeach
build-args: |
OPEN_AI_API_KEY=${{ secrets.OPEN_AI_API_KEY }}
MODEL_NAME=${{ secrets.MODEL_NAME }}
BASE_URL=${{ secrets.BASE_URL }}
DB_HOST=${{ secrets.DB_HOST }}
DB_PORT=${{ secrets.DB_PORT }}
# compiled output
/dist
/node_modules
# Logs
logs
*.log
npm-debug.log*
pnpm-debug.log*
yarn-debug.log*
yarn-error.log*
lerna-debug.log*
# OS
.DS_Store
# Tests
/coverage
/.nyc_output
# IDEs and editors
/.idea
.project
.classpath
.c9/
*.launch
.settings/
*.sublime-workspace
# IDE - VSCode
.vscode/*
!.vscode/settings.json
!.vscode/tasks.json
!.vscode/launch.json
!.vscode/extensions.json
packages/*
.env
python
\ No newline at end of file
{
"singleQuote": true,
"trailingComma": "all"
}
\ No newline at end of file
[https://github.com/Mrxyy/chat-query](https://github.com/Mrxyy/chat-query)
\ No newline at end of file
FROM node:18
# 设置工作目录
WORKDIR /usr/src/app
# 复制 package.json 和 package-lock.json 文件到工作目录
COPY package*.json ./
COPY pnpm-lock.yaml ./
ARG OPEN_AI_API_KEY
ENV OPEN_AI_API_KEY $OPEN_AI_API_KEY
ARG MODEL_NAME
ENV MODEL_NAME $MODEL_NAME
ARG BASE_URL
ENV BASE_URL $BASE_URL
ARG DB_HOST
ENV DB_HOST $DB_HOST
ARG DB_PORT
ENV DB_PORT $DB_PORT
ARG DB_PASSWORD
ENV DB_PASSWORD $DB_PASSWORD
ARG DB_USER
ENV DB_USER $DB_USER
# 安装项目依赖
RUN npm i -g pnpm
RUN pnpm i
# 复制应用程序源代码到工作目录
COPY dist ./dist
# 暴露端口,确保与 Nest.js 应用程序中设置的端口一致
EXPOSE 3001
# 启动命令
CMD [ "npm", "run", "start:prod" ]
{
"$schema": "https://json.schemastore.org/nest-cli",
"collection": "@nestjs/schematics",
"sourceRoot": "src",
"compilerOptions": {
"deleteOutDir": true
}
}
This diff is collapsed.
{
"name": "chat-query",
"version": "0.0.1",
"description": "",
"author": "",
"private": true,
"license": "UNLICENSED",
"scripts": {
"build": "nest build",
"format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"",
"start": "nest start",
"start:dev": "nest start --watch",
"start:debug": "nest start --debug --watch",
"start:prod": "node dist/src/main",
"build-image": "npm run build && docker build -t jadenxiong/chat-query-backend:latest .",
"lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix",
"test": "jest",
"test:watch": "jest --watch",
"test:cov": "jest --coverage",
"test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand",
"test:e2e": "jest --config ./test/jest-e2e.json"
},
"dependencies": {
"@dbml/core": "^2.5.3",
"@nestjs/common": "^9.0.0",
"@nestjs/core": "^9.0.0",
"@nestjs/platform-express": "^9.0.0",
"@nestjs/sequelize": "^9.0.2",
"@types/sequelize": "^4.28.15",
"dotenv": "^16.1.4",
"knex": "^2.4.2",
"langchain": "^0.0.149",
"lodash": "^4.17.21",
"mysql2": "^3.3.3",
"reflect-metadata": "^0.1.13",
"rxjs": "^7.2.0",
"sequelize": "^6.32.0",
"sequelize-typescript": "^2.1.5",
"typeorm": "^0.3.16",
"zod": "^3.21.4"
},
"devDependencies": {
"@nestjs/cli": "^9.0.0",
"@nestjs/schematics": "^9.0.0",
"@nestjs/testing": "^9.0.0",
"@types/express": "^4.17.13",
"@types/jest": "29.5.1",
"@types/node": "18.16.12",
"@types/supertest": "^2.0.11",
"@typescript-eslint/eslint-plugin": "^5.0.0",
"@typescript-eslint/parser": "^5.0.0",
"eslint": "^8.0.1",
"eslint-config-prettier": "^8.3.0",
"eslint-plugin-prettier": "^4.0.0",
"jest": "29.5.0",
"prettier": "^2.3.2",
"source-map-support": "^0.5.20",
"supertest": "^6.1.3",
"ts-jest": "29.1.0",
"ts-loader": "^9.2.3",
"ts-node": "^10.0.0",
"tsconfig-paths": "4.2.0",
"typescript": "^5.0.0"
},
"jest": {
"moduleFileExtensions": [
"js",
"json",
"ts"
],
"rootDir": "src",
"testRegex": ".*\\.spec\\.ts$",
"transform": {
"^.+\\.(t|j)s$": "ts-jest"
},
"collectCoverageFrom": [
"**/*.(t|j)s"
],
"coverageDirectory": "../coverage",
"testEnvironment": "node"
},
"peerDependenciesMeta": {
"langchain": {
"injected": true
}
},
"dependenciesMeta": {}
}
This diff is collapsed.
import { Sequelize } from 'sequelize-typescript';
import { Module } from '@nestjs/common';
import { SChemaModel } from './models/Schema';
import { SequelizeModule } from '@nestjs/sequelize';
import { Kenx } from './utils/knex';
import { QueriesModel } from './models/Querys';
import { OpenAIModule } from './models/AI/openAi';
import { config } from 'dotenv';
config();
export const dbHost = process.env['DB_HOST'];
export const dbPort = Number(process.env['DB_PORT']);
export const dbUser = process.env['DB_USER'];
export const dbPassword = process.env['DB_PASSWORD'];
const env = [
SequelizeModule.forRoot({
dialect: 'mysql',
host: dbHost,
port: dbPort,
username: dbUser || 'root',
password: dbPassword || '123789',
database: 'chat_query_fusion',
autoLoadModels: true,
synchronize: true,
}),
];
@Module({
imports: [...env, SChemaModel, QueriesModel, Kenx, OpenAIModule],
})
export class AppModule {
constructor(sequelize: Sequelize) {
// console.log(sequelize, "sequelize实例")
}
}
import { NestFactory } from '@nestjs/core';
import { AppModule } from './app.module';
async function bootstrap() {
const app = await NestFactory.create(AppModule);
app.enableCors();
await app.listen(3001);
console.log('http://127.0.0.1:3001');
}
bootstrap();
import { Module } from '@nestjs/common';
import { OpenAIController } from './openAi.controller';
import { OpenAIService } from './openAi.service';
@Module({
imports: [],
providers: [OpenAIService],
controllers: [OpenAIController],
})
export class OpenAIModule {
constructor() {}
}
import { Body, Controller, Post } from '@nestjs/common';
import { OpenAIService } from './openAi.service';
@Controller('/openAi/api')
export class OpenAIController {
constructor(private server: OpenAIService) {
server.test();
}
@Post('reactLive')
ReactLive(@Body('props') props, @Body('need') need) {
return this.server.getReactLiveCode(props, need);
}
@Post('code')
Code(@Body('data') data, @Body('need') need) {
return this.server.getFunctionCode(data.slice(0, 2), need);
}
@Post('checkQuery')
checkQuery(@Body('messageList') messageList) {
return this.server.checkQuery(messageList);
}
}
import { async } from 'rxjs';
import { defaultScope, fxTepmlate } from './../../../utils/prompts/reactLive';
import { OpenAI } from 'langchain/llms/openai';
// import { PromptTemplate } from 'langchain/prompts';
// import { OpenAIEmbeddings } from 'langchain/embeddings/openai';
// import { APIChain } from 'langchain/chains';
import { DataSource } from 'typeorm';
import { SqlDatabase } from 'langchain/sql_db';
import { createSqlAgent, SqlToolkit } from 'langchain/agents/toolkits/sql';
// import { SqlDatabaseChain } from 'langchain/chains';
import { get, nth } from 'lodash';
import { Tool } from 'langchain/tools';
import {
disableConstraints,
enableConstraints,
} from 'src/utils/knex/executeSQLWithDisabledForeignKeys';
import { GET_COMPONENT_BY_DATA } from 'src/utils/prompts/reactLive';
import { extractCodeBlocks } from 'src/utils/parse/getCode';
import { GET_FUNCTION_CODE_CHAIN } from 'src/utils/prompts/getFunction';
import { GET_CHECK_RESULT } from 'src/utils/prompts/checkSql';
export const openAIApiKey = process.env['OPEN_AI_API_KEY'];
console.log(Tool);
export class TestSqlTool extends Tool {
name = 'execute-sql';
db: SqlDatabase;
constructor(db: SqlDatabase) {
super();
this.db = db;
}
/** @ignore */
async _call(input: string) {
try {
await this.db.appDataSource.query(disableConstraints['mysql']);
await this.db.appDataSource.query(input);
await this.db.appDataSource.query(enableConstraints['mysql']);
return input;
} catch (error) {
return `${error}`;
}
}
// 此工具的输入是一个逗号分隔的表列表,输出是这些表的模式和示例行。请务必先调用list-tables-sql来确保这些表实际存在!
description = `
Input to this tool is specifically for executing SQL DDL statements and insert, update, delete statements and only one SQL statement can be executed each time.
`;
}
export class OpenAIService {
async test() {
// this.run();
}
async run() {
const model = new OpenAI(
{
modelName: 'gpt-4-0613',
openAIApiKey: openAIApiKey,
temperature: 0,
},
{
basePath:
'https://chat-gpt-next-qwn676aj7-mrxyy.vercel.app/api/openai/v1/',
},
);
const datasource = new DataSource({
type: 'mysql',
host: '127.0.0.1',
port: 3306,
username: 'root',
password: '123789xyy',
database: 'test-10',
connectorPackage: 'mysql2',
});
const db = await SqlDatabase.fromDataSourceParams({
appDataSource: datasource,
});
console.log(db);
const toolkit = new SqlToolkit(db, model);
toolkit.tools.push(new TestSqlTool(db));
toolkit.dialect = 'mysql';
const executor = createSqlAgent(model, toolkit, {
prefix: `You are an agent designed to interact with a SQL database.
Given an input question, create a syntactically correct {dialect} query to run, then look at the results of the query and return the answer.
Unless the user specifies a specific number of examples they wish to obtain, always limit your query to at most {top_k} results using the LIMIT clause.
You can order the results by a relevant column to return the most interesting examples in the database.
Never query for all the columns from a specific table, only ask for a the few relevant columns given the question.
You have access to tools for interacting with the database.
Only use the below tools. Only use the information returned by the below tools to construct your final answer.
You MUST double check your query before executing it. If you get an error while executing a query, rewrite the query and try again.
If the question does not seem related to the database, just return "I don't know" as the answer.`,
});
// const chain = new SqlDatabaseChain({
// llm: model,
// database: db,
// sqlOutputKey: 'sql',
// });
const result = await executor.call({
input: '每个表中都插入一条假数据,id 为自增字段',
});
// const res = await chain.call({
// query: '查出所有订单对应的用户和对应地址?',
// });
console.log(get(nth(result.intermediateSteps, -1), 'observation'));
await datasource.destroy();
}
async getReactLiveCode(props: Record<string, any>, need: string) {
const result = await GET_COMPONENT_BY_DATA.call({
props: JSON.stringify(props),
need,
scope: defaultScope,
fxTepmlate: fxTepmlate,
});
return {
code: extractCodeBlocks(result.text)[0],
};
}
async getFunctionCode(data: Record<string, any>, need: string) {
const result = await GET_FUNCTION_CODE_CHAIN.call({
data: JSON.stringify(data),
need,
});
return {
code: result.text,
};
}
async checkQuery(messageList: any[]) {
const result = await GET_CHECK_RESULT.call({
messageList: JSON.stringify(
messageList.map((v) => ({
[v.role]: get(v, 'content') || get(v, 'function_call.arguments'),
})),
),
});
console.log(result, 'result');
return {
...result,
};
}
}
import { Schema } from '../Schema/schema.model';
import {
Table,
Column,
Model,
CreatedAt,
UpdatedAt,
DeletedAt,
PrimaryKey,
DataType,
Comment,
Default,
AllowNull,
BelongsTo,
ForeignKey,
HasMany,
} from 'sequelize-typescript';
import { Query } from './Query.model';
@Table
export class DB extends Model {
@PrimaryKey
@Comment('id')
@Default(DataType.UUIDV4)
@Column(DataType.UUID)
id: string;
@HasMany(() => Query, {
sourceKey: 'id',
foreignKey: 'DbID',
})
Queries: Query[];
@Comment('连接名称')
@AllowNull(false)
@Column
name: string;
@Comment('配置')
@AllowNull(false)
@Column({
type: DataType.JSON,
})
config: JSON;
@BelongsTo(() => Schema, {
foreignKey: 'schemaId',
})
Schema: string;
@ForeignKey(() => Schema)
@Column(DataType.UUID)
schemaId: string;
@CreatedAt
createdAt: Date;
@UpdatedAt
updatedAt: Date;
@DeletedAt
DeletedAt: Date;
}
import { Schema } from './../Schema/schema.model';
import {
Table,
Column,
Model,
CreatedAt,
UpdatedAt,
DeletedAt,
PrimaryKey,
DataType,
Comment,
Default,
AllowNull,
BelongsTo,
ForeignKey,
} from 'sequelize-typescript';
import { DB } from './DB.model';
@Table
export class Query extends Model {
@PrimaryKey
@Comment('id')
@Default(DataType.UUIDV4)
@Column(DataType.UUID)
id: string;
@Comment('Query名称')
@AllowNull(false)
@Column
name: string;
@Comment('内容')
@AllowNull(false)
@Column({
type: DataType.JSON,
})
content: string;
@BelongsTo(() => Schema, {
foreignKey: 'schemaId',
})
Schema: Schema;
@BelongsTo(() => DB, {
foreignKey: 'DbID',
})
DB: DB;
@ForeignKey(() => DB)
@Column(DataType.UUID)
DbID: string;
@ForeignKey(() => Schema)
@Column(DataType.UUID)
schemaId: string;
@CreatedAt
createdAt: Date;
@UpdatedAt
updatedAt: Date;
@DeletedAt
DeletedAt: Date;
}
import { CreatedAt } from 'sequelize-typescript';
import {
Body,
Controller,
Delete,
Get,
Param,
Post,
Put,
Query as QueryValue,
} from '@nestjs/common';
import { QueriesService } from './index.service';
import { DB } from './DB.model';
import { Query } from './Query.model';
import { Schema } from '../Schema/schema.model';
@Controller('query')
export class QueriesController {
constructor(private service: QueriesService) {}
@Post('/querySql')
executeQuery(@Body() pramas) {
return this.service.executeQuery(pramas);
}
@Post('/add')
addQuery(@Body() query: Query) {
return this.service.addQuery(query);
}
@Put('/:queryId')
updateQuery(@Param('queryId') queryId, @Body('functions') functions: string) {
return this.service.updateQuery(queryId, functions);
}
@Post('/getDbDBML')
getDbDBML(@Body() config) {
return this.service.getDbDBML(config);
}
@Post('/createDbConnect')
createDbConnect(@Body() dbConfig: Pick<DB, 'config' | 'schemaId' | 'name'>) {
return this.service.createDbConnectConfigWithSchema(dbConfig);
}
@Post('/testConnectDb')
testConnectDb(@Body() dbConfig: DB['config']) {
return this.service.testConnectDb(dbConfig);
}
@Delete('/DbConnect/:DbID')
deleteDbForSchema(@Param('DbID') DbID) {
return this.service.deleteDb(DbID);
}
@Post('/run/:queryId')
runQuery(
@Param('queryId') queryID,
@Body('params') params: Record<string, any>,
@QueryValue('type') type: 1 | undefined,
) {
return this.service.runQuery(queryID, params, type);
}
@Delete('/:queryId')
deleteQuery(@Param('queryId') queryId: Query['id']) {
return this.service.deteteQuery(queryId);
}
@Get('/:schemaId/queries')
getQueries(@Param('schemaId') schemaId: Schema['id']) {
return this.service.getQueries(schemaId);
}
@Get('/:schemaId/DbConnect')
getSchemaAllDb(@Param('schemaId') schemaId: Schema['id']) {
return this.service.getSchemaAllDb(schemaId);
}
@Post('/:schemaId')
create(@Param('schemaId') schemaId: Schema['id']) {
return this.service.createQuery(schemaId);
}
}
import { Schema } from './../Schema/schema.model';
import { Injectable } from '@nestjs/common';
import { KnexContainer } from 'src/utils/knex';
import { Query } from './Query.model';
import { InjectModel } from '@nestjs/sequelize';
import { DB } from './DB.model';
import knex from 'knex';
import { generateDbdiagramDsl } from 'src/utils/knex/DB2DBML';
import { get, pick } from 'lodash';
import { Knex } from 'knex';
import exportSQL from 'src/utils/knex/export-sql';
import { executeSQLWithDisabledForeignKeys } from 'src/utils/knex/executeSQLWithDisabledForeignKeys';
function pureCode(raw: string): string {
const codeRegex = /```.*\n([\s\S]*?)\n```/;
const match = raw.match(codeRegex);
if (match) {
const code = match[1];
return code;
} else {
return raw;
}
}
@Injectable()
export class QueriesService {
constructor(
private knex: KnexContainer,
@InjectModel(Query) private QueryModel: typeof Query,
@InjectModel(DB) private DbModel: typeof DB,
) {}
async executeQuery(pramas: {
config: Record<string, any>;
execution: {
content: string;
type: string;
}[];
dbID: string;
}) {
console.log(pramas);
let db = this.knex.get(pramas.dbID);
if (!db) {
const dbConfig = await this.DbModel.findByPk(pramas.dbID);
const { client, host, port, user, password, database }: any = get(
dbConfig,
'dataValues.config',
);
db = await this.knex.create({
client: client,
asyncStackTraces: true,
debug: true,
connection: {
host: host,
port: port,
user: user,
password: password,
database: database,
},
});
}
const { execution = [], config } = pramas;
if (db) {
const results: string[] = [];
const fx = async ({
content,
type,
}: {
content: string;
type: string;
}) => {
if (type === 'sql') {
const sqlArr = content.split(/;\n/);
const params = [];
const templateSqlArr = [];
for (let i = 0; i < sqlArr.length; i++) {
const paramsItem = [];
templateSqlArr.push(
pureCode(sqlArr[i]).replace(/\$.*?\$/g, (string) => {
paramsItem.push(config[string]);
return '?';
}),
);
params.push(paramsItem);
}
const data = await executeSQLWithDisabledForeignKeys(
db,
templateSqlArr,
params,
);
return data;
}
};
const data = [];
for (const item of execution) {
// 保证执行顺序
data.push(await fx(item));
}
return {
data,
};
}
}
async createQuery(schemaId: Schema['id']): Promise<Query> {
return this.QueryModel.create({
schemaId,
name: 'unknown',
content: '',
});
}
async createDatabaseAndExecuteDDL(dbConfig, ddl, dbName) {
// 创建连接到默认数据库的 knex 实例
const defaultKnex = knex(dbConfig);
// 根据不同数据库系统的语法,创建数据库
if (dbConfig.client === 'mysql' || dbConfig.client === 'mysql2') {
await defaultKnex.schema.raw(
`CREATE DATABASE IF NOT EXISTS \`${dbName}\` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;`,
);
} else if (dbConfig.client === 'mssql') {
await defaultKnex.schema.raw(
`CREATE DATABASE [${dbName}] COLLATE Latin1_General_CI_AS`,
);
} else if (dbConfig.client === 'pg') {
await defaultKnex.schema.raw(
`CREATE DATABASE "${dbName}" WITH ENCODING 'UTF8' LC_COLLATE = 'en_US.UTF-8' LC_CTYPE = 'en_US.UTF-8'`,
);
} else if (dbConfig.client === 'oracle') {
// Oracle 不支持在 CREATE DATABASE 语句中直接设置字符集,请参考官方文档进行配置。
await defaultKnex.schema.raw(`CREATE DATABASE "${dbName}"`);
} else {
throw new Error(`Unsupported database client: ${dbConfig.client}`);
}
// 断开连接到默认数据库的 knex 实例
await defaultKnex.destroy();
// 创建连接到新数据库的 knex 实例
const knexForNewDb = knex({
...dbConfig,
connection: {
...dbConfig.connection,
database: dbName,
},
});
// 执行原始的 DDL SQL 语句
// await knexForNewDb.schema.raw(a);
await executeSQLWithDisabledForeignKeys(knexForNewDb, ddl);
// 断开连接到新数据库的 knex 实例。
await knexForNewDb.destroy();
}
async createDbConnectConfigWithSchema(
dbConfig: Pick<DB, 'config' | 'schemaId' | 'name'>,
): Promise<DB | { err: string }> {
if (get(dbConfig, 'config.create')) {
if (!get(dbConfig, 'config.newDbName')) {
return { err: 'please input database info' };
}
const { Schema: schema } = await this.DbModel.findOne({
include: [
{
model: Schema,
required: false,
right: true,
},
],
where: {
'$Schema.id$': dbConfig.schemaId,
},
paranoid: false,
attributes: [],
});
const { tableDict, linkDict } = get(schema, 'dataValues.graph', {
tableDict: {},
linkDict: {},
});
const ddl = exportSQL(
tableDict,
linkDict,
// get(dbConfig, 'config.newDbType'),
);
const { client, host, port, user, password, database, newDbName }: any =
dbConfig.config;
if (ddl && get(dbConfig.config, 'newDbName')) {
await this.createDatabaseAndExecuteDDL(
{
client: client,
asyncStackTraces: true,
debug: true,
connection: {
host: host,
port: port,
user: user,
password: password,
database: database,
},
},
ddl,
newDbName,
);
}
}
const saveConfig = {
...dbConfig,
config: {
...dbConfig.config,
database: get(
dbConfig.config,
'newDbName',
get(dbConfig.config, 'newDbName', get(dbConfig.config, 'database')),
),
},
};
return this.DbModel.create(saveConfig);
}
async getSchemaAllDb(schemaId: Schema['id']): Promise<DB[]> {
return this.DbModel.findAll({
where: {
schemaId,
},
});
}
async deleteDb(id: DB['id']): Promise<{ id: DB['id'] }> {
await this.DbModel.destroy({
where: {
id,
},
});
return { id };
}
async testConnectDb(dbConfig: DB['config']) {
const { client, host, port, user, password, database } = dbConfig as any;
const db: Knex = this.knex.create({
client: client,
asyncStackTraces: true,
debug: true,
connection: {
host: host,
port: port,
user: user,
password: password,
database: database,
},
});
try {
await db.raw('SHOW TABLES;');
this.knex.destroy(db);
return {
status: 200,
};
} catch (err) {
console.log(err, 'err');
this.knex.destroy(db);
return {
err: '参数错误,连接失败',
status: 401,
};
}
}
async getDbDBML(config: Knex.Config) {
const { client, host, port, user, password, database }: any = config;
const db: Knex = this.knex.create({
client: client,
asyncStackTraces: true,
debug: true,
connection: {
host: host,
port: port,
user: user,
password: password,
database: database,
},
});
try {
const data = await generateDbdiagramDsl(db);
this.knex.destroy(db);
return {
data: data,
status: 200,
};
} catch (err) {
console.log(err, 'err');
this.knex.destroy(db);
return {
err: '参数错误,连接失败',
status: 401,
};
}
}
async addQuery(query: Query) {
return this.QueryModel.create(
pick(query, 'name', 'content', 'schemaId', 'DbID'),
);
}
async updateQuery(queryId: string, functions: string) {
const query = await this.QueryModel.findByPk(queryId);
// query.content
const content: any = query.content;
return await query.update({
content: {
...content,
functions,
},
});
}
async getQueries(schemaId: Schema['id']) {
return this.QueryModel.findAll({
where: {
schemaId,
},
});
}
async deteteQuery(queryId: Query['id']) {
return this.QueryModel.destroy({
where: {
id: queryId,
},
});
}
async runQuery(queryId: string, params: Record<string, any>, type?) {
const query = await this.QueryModel.findByPk(queryId);
console.log(query.content);
const data = await this.executeQuery({
config: params,
execution: get(query.content, 'executions'),
dbID: query.DbID,
});
if (type === '1') {
let result = [];
console.log(data, 'data');
get(data, 'data', []).map((item) => {
item.map((v) => {
result = result.concat(get(v, '0'));
});
});
console.log(result);
return result;
}
return data;
}
}
import { Module, Controller } from '@nestjs/common';
import { QueriesService } from './index.service';
import { QueriesController } from './index.controller';
import { SequelizeModule } from '@nestjs/sequelize';
import { Query } from './Query.model';
import { DB } from './DB.model';
@Module({
imports: [SequelizeModule.forFeature([Query, DB])],
providers: [QueriesService],
controllers: [QueriesController],
})
export class QueriesModel {}
import {
Table,
Column,
Model,
PrimaryKey,
DataType,
Comment,
Default,
AllowNull,
BelongsTo,
ForeignKey,
} from 'sequelize-typescript';
import { Schema } from './schema.model';
@Table
export class SchemaLog extends Model {
@PrimaryKey
@Comment('id')
@Default(DataType.UUIDV4)
@Column(DataType.UUID)
id: string;
@BelongsTo(() => Schema, {
foreignKey: 'schemaId',
})
Schema: string;
@ForeignKey(() => Schema)
@Column(DataType.UUID)
schemaId: string;
@Comment('模型名称')
@AllowNull(false)
@Column
name: string;
@Comment('模型结构')
@AllowNull(false)
@Column({
type: DataType.JSON,
})
graph: string;
createdAt: Date;
updatedAt: Date;
DeletedAt: Date;
}
import {
Body,
Controller,
Delete,
Get,
Next,
Param,
Post,
Put,
} from '@nestjs/common';
import { SchemaService } from './index.service';
import { Schema } from './schema.model';
import { get } from 'lodash';
import { SchemaLog } from './SchemaLog.model';
@Controller('schema')
export class SchemaController {
constructor(private service: SchemaService) {}
@Get('/all')
async findAll() {
return this.service.getAll();
}
@Get('/:id')
async getSchemaById(@Param('id') id) {
return this.service.findSchema(id);
}
@Post('/create')
async create(@Body() body) {
const { dataValues } = await this.service.addSchema(body);
return dataValues;
}
@Delete('/:id')
async delete(@Param('id') id) {
return await this.service.removeSchema(id);
}
@Put('/:id')
save(
@Param('id') id,
@Body('graph') body,
@Body('name') name,
): Promise<Schema> {
return this.service.updateSchema(id, body, name);
}
@Get('/getLogs/:id')
async getLogsbySchemaId(@Param('id') id): Promise<SchemaLog> {
const res = await this.service.getSchemaLogsById(id);
return get(res, 'dataValues.schemaLogs');
}
@Delete('/getLogs/:id')
async deleteLogsbySchemaId(@Param('id') id) {
return await this.service.deleteSchemaLogsById(id);
}
}
import { Injectable } from '@nestjs/common';
import { Schema } from './schema.model';
import { InjectModel } from '@nestjs/sequelize';
import { executeRes } from 'src/utils/response/sequeilze';
import { SchemaLog } from './SchemaLog.model';
import { get, omit } from 'lodash';
import { async } from 'rxjs';
import export_dbml from 'src/utils/knex/export-dbml';
import { GET_SCHEMA_INFO } from 'src/utils/prompts/schema';
@Injectable()
export class SchemaService {
constructor(
@InjectModel(Schema) private SchemaModel: typeof Schema,
@InjectModel(SchemaLog) private SchemaLogModel: typeof SchemaLog,
) {}
async addSchema(body: Pick<Schema, 'name' | 'graph'>): Promise<Schema> {
return await executeRes(() => this.SchemaModel.create(body));
}
async findSchema(id: string): Promise<Schema> {
const schema = await this.SchemaModel.findByPk(id);
if (!schema.description) {
this.updateSchema(schema.id, schema.graph, schema.name);
}
return await executeRes(() => this.SchemaModel.findByPk(id));
}
async getAll(): Promise<Schema[]> {
return await executeRes(() => this.SchemaModel.findAll());
}
async removeSchema(id: string): Promise<{
id: Schema['id'];
}> {
return executeRes(async () => {
const num = await this.SchemaModel.destroy({
where: {
id: id,
},
});
console.log(num);
return { id: id };
});
}
async updateSchema(
id: string,
graph: Schema['graph'],
name: Schema['name'],
): Promise<Schema> {
const schema = await this.SchemaModel.findByPk(id);
//添加日志、
await this.SchemaLogModel.create({
...omit(get(schema, 'dataValues'), 'id'),
schemaId: get(schema, 'dataValues.id'),
});
const { tableDict, linkDict } = (graph as any) || {
tableDict: {},
linkDict: {},
};
const dbml = export_dbml(tableDict, linkDict);
const description = await GET_SCHEMA_INFO.run(dbml);
return executeRes(() =>
schema.update({
name,
graph,
description,
}),
);
}
async getSchemaLogsById(id: string): Promise<Schema[]> {
const resuts = await executeRes(
async () =>
await executeRes(() =>
this.SchemaModel.findOne({
where: {
id,
},
attributes: [],
include: [SchemaLog],
}),
),
);
return resuts;
}
async deleteSchemaLogsById(id: string): Promise<{
id: string;
}> {
await this.SchemaLogModel.destroy({
where: {
id,
},
});
return { id: id };
}
}
import { Module } from '@nestjs/common';
import { SchemaController } from './index.controller';
import { SchemaService } from './index.service';
import { SequelizeModule } from '@nestjs/sequelize';
import { Schema } from './schema.model';
import { SchemaLog } from './SchemaLog.model';
@Module({
imports: [SequelizeModule.forFeature([Schema, SchemaLog])],
controllers: [SchemaController],
providers: [SchemaService],
})
export class SChemaModel {}
import {
Table,
Column,
Model,
CreatedAt,
UpdatedAt,
DeletedAt,
PrimaryKey,
DataType,
Comment,
Default,
AllowNull,
HasMany,
} from 'sequelize-typescript';
import { SchemaLog } from './SchemaLog.model';
@Table
export class Schema extends Model {
@PrimaryKey
@Comment('id')
@Default(DataType.UUIDV4)
@Column(DataType.UUID)
id: string;
@HasMany(() => SchemaLog, {
foreignKey: 'schemaId',
sourceKey: 'id',
})
schemaLogs: SchemaLog[];
@Comment('模型名称')
@AllowNull(false)
@Column
name: string;
@Comment('模型结构')
@AllowNull(false)
@Column({
type: DataType.JSON,
})
graph: typeof DataType.JSON;
@Comment('模型描述')
@AllowNull(true)
@Column({
type: DataType.STRING(10000),
})
description: string;
@CreatedAt
createdAt: Date;
@UpdatedAt
updatedAt: Date;
@DeletedAt
DeletedAt: Date;
}
import { config } from 'dotenv';
import { ChatOpenAI } from 'langchain/chat_models/openai';
import { OpenAI } from 'langchain/llms/openai';
config();
export const openAIApiKey = process.env['OPEN_AI_API_KEY'];
export const modelName = process.env['MODEL_NAME'];
export const basePath = process.env['BASE_URL'];
export function getOpenAi() {
return new OpenAI(
{
modelName: modelName,
openAIApiKey: openAIApiKey,
temperature: 0,
},
{
basePath: basePath,
},
);
}
export function getChatOpenAi() {
return new ChatOpenAI(
{
modelName: modelName,
openAIApiKey: openAIApiKey,
temperature: 0,
},
{
basePath: basePath,
},
);
}
import { Knex as KnexType } from 'knex';
export async function generateDbdiagramDsl(
knexInstance: KnexType,
): Promise<string> {
const dbType = knexInstance.client.config.client;
const isOracle = dbType === 'oracledb';
const databaseName = isOracle
? knexInstance.client.config.connection.user.toUpperCase()
: knexInstance.client.config.connection.database;
const tables = isOracle
? await knexInstance('all_tables')
.select('table_name')
.where('owner', '=', databaseName)
: await knexInstance('information_schema.tables')
.select('table_name')
.where('table_schema', '=', databaseName);
let dsl = '';
for (const table of tables) {
const table_name = table['TABLE_NAME'] || table['table_name'];
dsl += `Table "${table_name}" {\n`;
// console.log(databaseName,table_name)
const columns = isOracle
? await knexInstance('all_tab_columns')
.select('column_name', 'data_type')
.where('table_name', '=', table_name)
.andWhere('owner', '=', databaseName)
: await knexInstance('information_schema.columns')
.select('column_name', 'data_type', 'column_key')
.where('table_name', table_name)
.andWhere('table_schema', databaseName);
const foreignKeys = isOracle
? await knexInstance('all_cons_columns')
.select('column_name', 'r_constraint_name')
.where('table_name', '=', table_name)
.andWhere('owner', '=', databaseName)
.whereIn('constraint_name', function () {
this.select('constraint_name')
.from('all_constraints')
.where('constraint_type', '=', 'R')
.andWhere('table_name', '=', table_name)
.andWhere('owner', '=', databaseName);
})
: await knexInstance('information_schema.key_column_usage')
.select(
'column_name',
'referenced_table_name',
'referenced_column_name',
)
.where('table_name', '=', table_name)
.andWhere('table_schema', '=', databaseName)
.andWhereRaw('referenced_table_name IS NOT NULL');
const flag: Record<string, any> = {};
for (const item of foreignKeys) {
const column_name = item['COLUMN_NAME'] || item['column_name'];
const referenced_table_name =
item['REFERENCED_TABLE_NAME'] || item['referenced_table_name'];
const referenced_column_name =
item['REFERENCED_COLUMN_NAME'] || item['referenced_column_name'];
const data_type = item['DATA_TYPE'] || item['data_type'];
if (isOracle) {
const refInfo = await knexInstance('all_constraints')
.select('table_name', 'column_name')
.where('constraint_name', '=', referenced_table_name)
.andWhere('owner', '=', databaseName)
.first();
if (refInfo) {
dsl += ` "${column_name}" ${data_type.toUpperCase()} [ref: > ${referenced_table_name}.${referenced_column_name}]\n`;
}
} else {
const columnInfo = columns.find(
(column) =>
(column.COLUMN_NAME || column.column_name) === column_name,
);
const data_type = columnInfo
? columnInfo.DATA_TYPE || columnInfo.data_type
: '';
// 处理外键关系的代码
dsl += ` "${column_name}" ${data_type.toUpperCase()} [ref: > "${referenced_table_name}.${referenced_column_name}"]\n`;
}
flag[column_name] = true;
}
for (const col of columns) {
const column_name = col.COLUMN_NAME || col.column_name;
const data_type = col.DATA_TYPE || col.data_type;
const column_key = col.COLUMN_KEY || col.column_key;
if (!flag[column_name])
dsl += ` "${column_name}" ${data_type.toUpperCase()}${
column_key === 'PRI' ? ' [pk]' : ''
}\n`;
}
// console.log(await knexInstance('information_schema.key_column_usage'), 'foreignKeys');
dsl += '}\n\n';
}
// console.log('generateDbdiagramDsl', dsl);
return dsl;
}
import { Knex } from 'knex';
export const disableConstraints = {
mysql: 'SET FOREIGN_KEY_CHECKS = 0;',
mysql2: 'SET FOREIGN_KEY_CHECKS = 0;',
sqlServer: 'ALTER TABLE table_name NOCHECK CONSTRAINT ALL;',
postgresql: 'SET CONSTRAINTS ALL DEFERRED;',
oracle: 'ALTER TABLE table_name DISABLE ALL TRIGGERS;',
};
// Enable foreign key constraints for different databases
export const enableConstraints = {
mysql: 'SET FOREIGN_KEY_CHECKS = 1;',
mysql2: 'SET FOREIGN_KEY_CHECKS = 1;',
sqlServer: 'ALTER TABLE table_name CHECK CONSTRAINT ALL;',
postgresql: '', // Constraints will be enabled automatically at the end of the transaction
oracle: 'ALTER TABLE table_name ENABLE ALL TRIGGERS;',
};
const annotationMatching = /(--.*)|(((\/\*)+?[\w\W]+?(\*\/)+))/g;
export async function executeSQLWithDisabledForeignKeys(
kenx: Knex,
sqlArray: string[],
params: any[][] = [],
) {
try {
// Disable foreign key constraints for different databases
// Replace 'databaseType' with the actual type of your database (e.g., 'mysql', 'sqlServer', etc.)
const databaseType = kenx.client.config.client;
// Disable foreign key constraints
console.log(disableConstraints[databaseType]);
await kenx.schema.raw(disableConstraints[databaseType]);
// Execute the main SQL query
const result = [];
for (const index in sqlArray) {
result.push(
// ${sql}
await (kenx.schema.raw as any)(
sqlArray[index].replace(annotationMatching, ''),
params[index],
),
);
}
return result;
} catch (error) {
console.error('执行失败☹️:', error);
} finally {
const databaseType = kenx.client.config.client;
await kenx.schema.raw(enableConstraints[databaseType]);
}
}
import { ModelExporter, Parser, exporter } from '@dbml/core';
import { readFileSync, writeFileSync } from 'fs';
/**
* It takes a table dictionary and a link dictionary and returns a SQL string
* @param tableDict - a dictionary of tables, where the key is the table ID and the value is the tableobject
* @param linkDict - a dictionary of links, where the key is the link id and the value is the linkobject
* @param [databaseType=postgres] - The type of database you want to export to.
* @returns dbml string.
*/
const export_dbml = (
tableDict: any,
linkDict: any,
databaseType: 'postgres' | 'mysql' | 'dbml' | 'mssql' | 'json' = 'mysql',
) => {
const combined = {
name: 'public',
note: '',
tables: Object.values(tableDict || {}).map((table: any) => {
return {
name: table.name,
note: table.note,
fields: table.fields.map((field) => {
return {
...field,
type: {
// To lower case because of typing 'BIGINT' with upper case and increment get wrong pg sql type when export
type_name: field.type.toLowerCase(),
args: null,
},
};
}),
};
}),
enums: [],
tableGroups: [],
refs: Object.values(linkDict || {}).map((ref: any) => {
return {
...ref,
endpoints: ref.endpoints.map((endpoint) => {
return {
...endpoint,
tableName: tableDict[endpoint.id].name,
fieldNames: [
tableDict[endpoint.id].fields.find(
(field) => field.id == endpoint.fieldId,
).name,
],
};
}),
};
}),
};
const database = Parser.parse(combined as any, 'json');
const dbml = ModelExporter.export(database, 'dbml', false);
return dbml;
};
export default export_dbml;
import { ModelExporter, Parser, exporter } from '@dbml/core';
import { readFileSync, writeFileSync } from 'fs';
/**
* It takes a table dictionary and a link dictionary and returns a SQL string
* @param tableDict - a dictionary of tables, where the key is the table ID and the value is the tableobject
* @param linkDict - a dictionary of links, where the key is the link id and the value is the linkobject
* @param [databaseType=postgres] - The type of database you want to export to.
* @returns SQL string.
*/
const exportSQL = (
tableDict: any,
linkDict: any,
databaseType: 'postgres' | 'mysql' | 'dbml' | 'mssql' | 'json' = 'mysql',
) => {
const combined = {
name: 'public',
note: '',
tables: Object.values(tableDict || {}).map((table: any) => {
return {
name: table.name,
note: table.note,
fields: table.fields.map((field) => {
return {
...field,
type: {
// To lower case because of typing 'BIGINT' with upper case and increment get wrong pg sql type when export
type_name: field.type.toLowerCase(),
args: null,
},
};
}),
};
}),
enums: [],
tableGroups: [],
refs: Object.values(linkDict || {}).map((ref: any) => {
return {
...ref,
endpoints: ref.endpoints.map((endpoint) => {
return {
...endpoint,
tableName: tableDict[endpoint.id].name,
fieldNames: [
tableDict[endpoint.id].fields.find(
(field) => field.id == endpoint.fieldId,
).name,
],
};
}),
};
}),
};
const database = Parser.parse(combined as any, 'json');
const dbml = ModelExporter.export(database, 'dbml', false);
const sql = exporter.export(dbml, databaseType);
// console.log(sql);
// writeFileSync('./test.sql', sql);
return sql.split(/\n\s*\n/);
};
export default exportSQL;
import { Global, Injectable, Module } from '@nestjs/common';
import { randomUUID } from 'crypto';
import knex, { Knex } from 'knex';
import { get, set } from 'lodash';
@Injectable()
export class KnexContainer {
knexs = new Map();
create(config: Knex.Config, id?: string) {
const db = knex(config);
this.add(db, id);
return db;
}
add(kenx: Knex, id?: string) {
const key = id || randomUUID();
this.knexs.set(key, {
db: kenx,
});
set(kenx, '_key_', key);
}
get(id: string) {
let client;
if (!(client = this.knexs.get(id))) {
// client = create();
//Todo
}
return client?.db;
}
async destroy(key: string | Knex) {
const id = get(key || { _key_: key }, '_key_');
const client = this.get(id);
console.log(this.knexs.has(id));
if (client) {
await client.destroy();
this.knexs.delete(id);
}
console.log('db pools', id, this.knexs.size);
}
getAll() {
return this.knexs.values();
}
}
@Global()
@Module({
providers: [KnexContainer],
exports: [KnexContainer],
})
export class Kenx {}
import { Injectable } from '@nestjs/common';
import { OpenAI } from 'langchain/llms/openai';
class LangChain {
model = new OpenAI({ openAIApiKey: 'sk-...', temperature: 0.9 });
}
export function extractCodeBlocks(markdownText, lang = null) {
const codeBlockRegex = /```(\w+)?\n([\s\S]*?)```/g;
const codeBlocks = [];
let match;
while ((match = codeBlockRegex.exec(markdownText)) !== null) {
if (lang === null || match[1] === lang) {
codeBlocks.push(match[2]);
}
}
return codeBlocks;
}
import {
ChatPromptTemplate,
HumanMessagePromptTemplate,
SystemMessagePromptTemplate,
} from 'langchain/prompts';
import { z } from 'zod';
import { getChatOpenAi } from '../Ai';
import { createStructuredOutputChainFromZod } from 'langchain/chains/openai_functions';
const model = getChatOpenAi();
const zodSchema = z.object({
answerMeetsRequirements: z
.boolean()
.describe('判断沟通记录中给出的答案是否满足你最后一条记录中的需求'),
why: z
.string()
.describe(
'你现在是用户,下一次你将输入什么内容(中文)保证上面你最后一条记录中的需求能被正确的被理解。',
),
});
const prompt = ChatPromptTemplate.fromPromptMessages([
SystemMessagePromptTemplate.fromTemplate(`沟通记录:\n
{messageList}
`),
HumanMessagePromptTemplate.fromTemplate(
`你需要谨慎查看这个沟通记录,你扮演对话中的用户, 对比需求和回答,以判断你最后一条记录中的需求是否被正确理解且能被正确解决。
如果需求和答案没有对齐,你现在是用户,下一次你将输入什么内容(中文)保证上面你最后一条记录中的需求能被正确的被理解。`,
),
]);
export const GET_CHECK_RESULT = createStructuredOutputChainFromZod(zodSchema, {
llm: model,
prompt,
});
import { OpenAI } from 'langchain/llms/openai';
import { PromptTemplate } from 'langchain/prompts';
import { LLMChain } from 'langchain/chains';
import { getOpenAi } from '../Ai';
const model = getOpenAi();
const prompt = PromptTemplate.fromTemplate(
`作为js代码编程专家,你需要分析我的需求,请你编写一个函数处理我的需求并返回我的需要的结果,这个是固定第一个入参:
{data}。
需求:
{need}。
请按照需求生成代码之后,将结果按照以下模版中的标签中的内容进行分析并且替换模版中标签的内容(必须保留标签)作为最后结果输出:
<FunctionDescription>该函数描述说明</FunctionDescription>
<FunctionName>函数名称</FunctionName>
<FunctionCode>函数代码</FunctionCode>
<FunctionExample>函数实例</FunctionExample>
`,
);
export const GET_FUNCTION_CODE_CHAIN = new LLMChain({ llm: model, prompt });
import { OpenAI } from 'langchain/llms/openai';
import {
AIMessagePromptTemplate,
ChatPromptTemplate,
HumanMessagePromptTemplate,
} from 'langchain/prompts';
import { LLMChain } from 'langchain/chains';
import { getChatOpenAi } from '../Ai';
const model = getChatOpenAi();
export const defaultScope = `
{
data,
import: {
'styled-components':{
'描述':'A css in js Framework,Visual primitives for the component age. Use the best bits of ES6 and CSS to style your apps without stress'
},
'echarts':{
'描述':'Apache ECharts is a powerful, interactive charting and data visualization library for browser.',
'用法': "import echarts from 'echarts';"
},
'echarts-for-react':{
'描述':'Apache ECharts is a powerful, interactive charting and data visualization library for browser.',
'用法': "import React from 'react';
import ReactECharts from 'echarts-for-react'; // or var ReactECharts = require('echarts-for-react');
<ReactECharts
option={this.getOption()}
notMerge={true}
lazyUpdate={true}
theme={"theme_name"}
onChartReady={this.onChartReadyCallback}
onEvents={EventsDict}
opts={}
/>"
},
},
}
`;
export const fxTepmlate = `export default App() {
const props = data;
...
}`;
const Messages_1 =
HumanMessagePromptTemplate.fromTemplate(`我正在使用react-live作为一个实时编辑和编译React组件的工具。请根据我的需求,输出一段能在react-live运行的代码,只需要导出默认组件,不用挂载dom。可用的作用域如下:
{scope}
其中import可供导入的库,除此之外不能使用其它第三方库。 key为包名,value 是包的描述。其它每个属性将作为单独全局变量,变量名为属性名。请确保默认导出的组件接受null作为参数,并将传入props取自模块全局作用域变量data(const props = data),全局变量变量data会自动注入,请不要进行声明。例如:
{fxTepmlate}
`);
const Messages_2 =
AIMessagePromptTemplate.fromTemplate(`好的我明白啦,请问你的业务是什么?`);
const Messages_3 = HumanMessagePromptTemplate.fromTemplate(
`这是我传入的props:{props}。{need}`,
);
const prompt = ChatPromptTemplate.fromPromptMessages([
Messages_1,
Messages_2,
Messages_3,
]);
export const GET_COMPONENT_BY_DATA = new LLMChain({ llm: model, prompt });
import { OpenAI } from 'langchain/llms/openai';
import { PromptTemplate } from 'langchain/prompts';
import { LLMChain } from 'langchain/chains';
import { getOpenAi } from '../Ai';
const model = getOpenAi();
const prompt = PromptTemplate.fromTemplate(
`作为一个数据模型业务分析专家,您需要根据当前数据模型(DBML格式)进行精确的分析,根据模型的提供的信息,分析该模型并输出简要描述。当前数据库模型为:
<dbml>
{sql}
</dbml>`,
);
export const GET_SCHEMA_INFO = new LLMChain({ llm: model, prompt });
export async function executeRes(fx: () => any) {
let result;
try {
result = await fx();
} catch (err) {
result = { dataValues: err };
}
console.log('result', result, 'result');
return result;
}
import { Test, TestingModule } from '@nestjs/testing';
import { INestApplication } from '@nestjs/common';
import * as request from 'supertest';
import { AppModule } from './../src/app.module';
describe('AppController (e2e)', () => {
let app: INestApplication;
beforeEach(async () => {
const moduleFixture: TestingModule = await Test.createTestingModule({
imports: [AppModule],
}).compile();
app = moduleFixture.createNestApplication();
await app.init();
});
it('/ (GET)', () => {
return request(app.getHttpServer())
.get('/')
.expect(200)
.expect('Hello World!');
});
});
{
"moduleFileExtensions": ["js", "json", "ts"],
"rootDir": ".",
"testEnvironment": "node",
"testRegex": ".e2e-spec.ts$",
"transform": {
"^.+\\.(t|j)s$": "ts-jest"
}
}
import { GET_CHECK_RESULT } from 'src/utils/prompts/checkSql';
async function test() {
const result = await GET_CHECK_RESULT.call({
messageList: `[
{
"role": "system",
"content": "IMPRTANT: You are a virtual assistant powered by the gpt-3.5-turbo model, now time is 2023/5/30 16:57:14}"
},
{
"role": "user",
"content": "作为一个数据库模型业务分析专家,您需要根据当前数据库模型(DBML格式)生成对应的 MySQL 数据库可执行的 SQL 文本,并列出执行该 SQL 所需的变量及其解释,最后为此条查询命名。当前数据库模型为:\n\ndbml\nTable "orders" {\n "id" int [pk, increment]\n "patient_id" int\n "doctor_id" int\n "appointment_date" timestamp\n "status" varchar\n}\n\nTable "doctors" {\n "id" int [pk, increment]\n "department_id" int\n "name" varchar\n "specialization" varchar\n "phone_number" varchar\n "email" varchar\n}\n\nTable "patients" {\n "id" int [pk, increment]\n "name" varchar\n "gender" varchar\n "age" int\n "phone_number" varchar\n "email" varchar\n "address" varchar\n}\n\nTable "departments" {\n "id" int [pk, increment]\n "hospital_id" int\n "name" varchar\n "description" varchar\n}\n\nTable "hospitals" {\n "id" int [pk, increment]\n "name" varchar\n "address" varchar\n "phone_number" varchar\n "created_at" timestamp\n}\n\nRef:"doctors"."id" < "orders"."doctor_id"\n\nRef:"patients"."id" < "orders"."patient_id"\n\nRef:"hospitals"."id" < "departments"."hospital_id"\n\nRef:"departments"."id" < "doctors"."department_id"\n\n\n\n请按照需求生成结果之后,将结果按照xml语法,将模版中的标签中的内容进行分析(需要用户填入的信息作为变量,如果需要变量:变量的命名必须请放在
和之间,例如:
tab、
name...)。:\n\n根据您提供的数据库模型,已为您生成查询:\n<sql>按照需要生产的sql语句</sql>\n\n执行所需变量:\n\n判断解决当前问题的查询执行时是否需要变量。若不需要变量,则无需提供任何信息。 若条件成立,请使用:\n<var>
执行
所需变量名称
执行SQL所需变量名称</var>: <varDescription>变量解释</varDescription>\n\n查询命名和描述:\n<queryName>查询名称</queryName>\n<queryDescription>查询描述</queryDescription>\n"
},
{
"role": "system",
"content": "你第一步应该思考分析:用户是否指定生成模拟数据,如果有,你还需要生成模拟数据,你第二步应该思考分析:用户需求中是否需要变量(例如:生成模拟数据就不需要变量,因为数据由你生成),如果需要变量:变量的命名必须请放在
和之间,例如:"variablesArr": [\n {\n "variable": "
variable",\n "varDescription": "这个变量的描述"\n }):\n "
},
{
"role": "assistant",
"content": "好的,我明白了。请问你的业务是什么?"
},
{
"role": "user",
"content": "分析数据库,添加一批中文的模拟数据10-15条"
},
{
"role": "assistant",
"content": "",
"function_call": {
"name": "saveExecuteSqlInfo",
"arguments": "{\n "simulation": true,\n "sql": "INSERT INTO orders (patient_id, doctor_id, appointment_date, status) VALUES (1, 1, '2023-05-30 10:00:00', '待确认'), (2, 2, '2023-05-31 14:30:00', '已确认'), (3, 3, '2023-06-01 09:15:00', '已取消'), (4, 4, '2023-06-02 16:45:00', '已完成'), (5, 5, '2023-06-03 11:30:00', '待确认'), (6, 6, '2023-06-04 13:00:00', '已确认'), (7, 7, '2023-06-05 15:45:00', '已取消'), (8, 8, '2023-06-06 10:30:00', '已完成'), (9, 9, '2023-06-07 12:15:00', '待确认'), (10, 10, '2023-06-08 14:00:00', '已确认')",\n "queryName": "添加模拟数据",\n "queryDescription": "向订单表中添加10条模拟数据"\n}"
}
}
]`,
});
console.log(result, 'result');
}
describe('checkSql', () => {
test();
});
{
"extends": "./tsconfig.json",
"exclude": ["node_modules", "test", "dist", "**/*spec.ts", "packages/**"]
}
{
"extends": "./tsconfig.json",
"compilerOptions": {
"paths": {
"@dbml/core": ["../packages/dbml/packages/dbml-core"]
}
},
"exclude": ["node_modules", "test", "dist", "**/*spec.ts"]
}
{
"compilerOptions": {
"module": "commonjs",
"declaration": true,
"removeComments": true,
"emitDecoratorMetadata": true,
"experimentalDecorators": true,
"allowSyntheticDefaultImports": true,
"target": "es2017",
"sourceMap": true,
"outDir": "./dist",
"baseUrl": "./",
"incremental": true,
"skipLibCheck": true,
"strictNullChecks": false,
"noImplicitAny": false,
"strictBindCallApply": false,
"forceConsistentCasingInFileNames": false,
"noFallthroughCasesInSwitch": false
}
}
# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
# dependencies
node_modules
.pnp
.pnp.js
# testing
coverage
# next.js
out
# production
build
# misc
.DS_Store
*.pem
# debug
npm-debug.log*
yarn-debug.log*
yarn-error.log*
.pnpm-debug.log*
# local env files
.env*.local
# vercel
.vercel
# typescript
*.tsbuildinfo
next-env.d.ts
backend
TOKEN.ts
.env
.vscode
\ No newline at end of file
{
"extends": "next/core-web-vitals",
"rules": {
"react-hooks/rules-of-hooks": "off", // Checks rules of Hooks
"react-hooks/exhaustive-deps": "off",// Checks effect dependencies
}
}
name: Build and Push Docker Image
on:
push:
branches:
- fusionTeach
jobs:
build-and-push:
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v2
- name: Login to Docker Hub
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Build and push Docker image
uses: docker/build-push-action@v2
with:
context: .
push: true
tags: jadenxiong/chat-query:fusionTeach
build-args: |
NEXT_PUBLIC_BACKEND_URL=${{ secrets.NEXT_PUBLIC_BACKEND_URL }}
NEXT_PUBLIC_OPEN_AI_API_KEY=${{ secrets.NEXT_PUBLIC_OPEN_AI_API_KEY }}
OPENAI_PROXY_URL=${{ secrets.OPENAI_PROXY_URL }}
# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
# dependencies
/node_modules
/.pnp
.pnp.js
# testing
/coverage
# next.js
/.next/
/out/
# production
/build
# misc
.DS_Store
*.pem
# debug
npm-debug.log*
yarn-debug.log*
yarn-error.log*
.pnpm-debug.log*
# local env files
.env*.local
# vercel
.vercel
# typescript
*.tsbuildinfo
next-env.d.ts
backend
TOKEN.ts
.env
.vscode
public/service-worker.js
xflow-docs
draw-app
packages
react-live-runner-swc
**/*.svg
package.json
out
.DS_Store
*.png
.editorconfig
Dockerfile*
.gitignore
.prettierignore
LICENSE
.next
# .prettierrc
tabWidth: 4
semi: true
singleQuote: true
trailingComma: 'es5'
bracketSpacing: true
jsxBracketSameLine: false
arrowParens: 'avoid'
printWidth: 100
# CHAT-QUERY
[en](./README.md)
> Chat-Query 是一个基于元数据模型和 AI 技术,通过自然语言实现数据查询。
## 演示
> [在线预览](https://query.fusiontech.cn/)
> [demo gif](https://cdn.glitch.me/fd139a45-4a65-41b6-9634-41617ab20cdc/%E6%BC%94%E7%A4%BA.gif?v=1686907695067)
+ **功能特点🐂:**
- 支持导入 DDL、DBML 和数据库逆向解析,AI 自动生成业务模型。
- 提供业务模型的基本 CRUD 功能、AI 智能分析,支持模型导出为 DDL、DBML 以及与数据库同步。
- 结合模型和 AI 实现自然语言数据查询,可添加至查询列表并通过 API 调用。
- BI分析、Function 调用、react代码沙箱、支持导入npm包。
## 应用场景🎬
+ 从低代码到零代码开发。
+ 非业务人员快速进行数据分析。
+ 更多应用场景待探索...
## 部署
+ docker compose
```shell
NEXT_PUBLIC_OPEN_AI_API_KEY="sk-..." docker-compose up
```
+ chat-query和chat-query-backend都是支持docker构建和提供docker镜像.
## 开发环境设置
> 👏 欢迎参与 Chat-Query 的建设。
+ 后端:
```js
pnpm install
pnpm start:dev
```
- 在 .env 文件中添加
```shell
#openai
OPEN_AI_API_KEY='sk-...'
MODEL_NAME="gpt-3.5-turbo-16k-0613"
BASE_URL='https://open-ai-xyy.deno.dev/v1'
# database
DB_HOST="139.198.179.193"
DB_PORT=32094
```
+ 前端
```js
pnpm install
pnpm dev
```
- 在 .env 文件中添加
```shell
NEXT_PUBLIC_OPEN_AI_API_KEY='sk-...'
OPENAI_PROXY_URL="https://open-ai-xyy.deno.dev/"
NEXT_PUBLIC_BACKEND_URL="http://localhost:3001/"
```
## 系统架构
![架构](https://cdn.glitch.global/fd139a45-4a65-41b6-9634-41617ab20cdc/%E6%97%A0%E6%A0%87%E9%A2%98-2023-05-31-1202%20(1).png?v=1686908252244)
# CHAT-QUERY
[zh](./README-zh.md)
> Chat-Query is a data query implementation based on metadata models and AI technology through natural language.
## Demo
> [Online Preview](https://chat-query.vercel.app/)
> [demo gif](https://cdn.glitch.me/fd139a45-4a65-41b6-9634-41617ab20cdc/%E6%BC%94%E7%A4%BA.gif?v=1686907695067)
+ **Features🐂:**
- Supports DDL, DBML and database reverse parsing import, AI automatically generates business models.
- Provides basic CRUD functions of business models, AI intelligent analysis, supports model export to DDL, DBML and synchronization with the database.
- Combines models and AI to implement natural language data queries, can be added to the query list and called via API.
- BI analysis, Function call, react code sandbox, supports importing npm packages.
## Application Scenarios🎬
+ From low-code to no-code development.
+ Non-business personnel can quickly perform data analysis.
+ More application scenarios to be explored...
## Deployment
+ docker compose
```shell
NEXT_PUBLIC_OPEN_AI_API_KEY="sk-..." docker-compose up
```
+ Both chat-query and chat-query-backend support docker build and provide docker images.
## Development Environment Setup
> 👏 Welcome to participate in the construction of Chat-Query.
+ Backend:
```js
pnpm install
pnpm start:dev
```
- Add in .env file
```shell
#openai
OPEN_AI_API_KEY='sk-...'
MODEL_NAME="gpt-3.5-turbo-16k-0613"
BASE_URL='https://open-ai-xyy.deno.dev/v1'
# database
DB_HOST="139.198.179.193"
DB_PORT=32094
```
+ Frontend
```js
pnpm install
pnpm dev
```
- Add in .env file
```shell
NEXT_PUBLIC_OPEN_AI_API_KEY='sk-...'
OPENAI_PROXY_URL="https://open-ai-xyy.deno.dev/"
NEXT_PUBLIC_BACKEND_URL="http://localhost:3001/"
```
## System Architecture
![Architecture](https://cdn.glitch.global/fd139a45-4a65-41b6-9634-41617ab20cdc/%E6%97%A0%E6%A0%87%E9%A2%98-2023-05-31-1202%20(1).png?v=1686908252244)
\ No newline at end of file
import getConfig from 'next/config';
import APi, { backendApi } from '.';
export type DBInfo = {
host: string;
port: number;
user: string;
password: string;
database: string;
client: 'mysql2';
name?: string;
};
export interface Query {
schemaId: string;
DbID: string;
name: string;
content: {
executions: {
content: string;
type: string;
};
params: Record<string, any>;
info: { queryDescription: string; queryName: string };
functions?: string;
filed: string[];
};
}
export default class ConnectDb {
static create(config: DBInfo) {
return backendApi.post('query/testConnectDb', config);
}
static getDbDBML(config: DBInfo) {
return backendApi.post('query/getDbDBML', config);
}
static addDbForSchema(params: { config: DBInfo; schemaId: string; name: string }) {
return backendApi.post('query/createDbConnect', params);
}
static getAllForSchema(schemaId: string) {
return backendApi.get(`query/${schemaId}/DbConnect`);
}
static removeDbForSchema(DbID: string) {
return backendApi.delete(`query/DbConnect/${DbID}`);
}
static addQuery(query: Query) {
return backendApi.post('query/add', query);
}
static deleteQuery(queryId: string) {
return backendApi.delete(`/query/${queryId}`);
}
static getQueries(schemaId: string) {
return backendApi.get(`query/${schemaId}/queries`);
}
static runQuery(queryId: string, params: Record<string, any>) {
return backendApi.post(`query/run/${queryId}`, { params });
}
static updateQuery(queryId: string, functions: string) {
return backendApi.put(`/query/${queryId}`, {
functions,
});
}
}
import getConfig from 'next/config';
import APi, { backendApi } from '.';
export type ExecuteSqlPrams = Record<string, any>;
export default class ExecuteQuery {
static executeSql(
config: ExecuteSqlPrams,
execution: {
content: string;
type: string;
}[],
dbID: string
) {
return backendApi.post('/query/querySql', {
config,
execution,
dbID,
});
}
}
import getConfig from 'next/config';
import type { NextApiRequest, NextApiResponse } from 'next';
import { table } from 'console';
import APi from '.';
const {
publicRuntimeConfig: { apiPath },
} = getConfig();
export interface schemaParams {
type: 'schema' | 'table';
name?: string;
}
export default class getSchema {
static getTableList(config: schemaParams) {
return APi.post(apiPath + 'getSchema', config);
}
}
import { ChatMessage } from './../../components/AITool/index';
import APi, { backendApi } from '.';
import { get, isArray, max, min, pick } from 'lodash';
import { Options } from 'redaxios';
import OpenAI from './openAI';
export interface View {
type: 'schema' | 'table';
name?: string;
}
const privateConfig: Options & {
timeout: number;
} = {
timeout: 1000 * 60,
};
function extractCodeBlocks(markdownText: string, lang = null) {
const codeBlockRegex = /```(\w+)?\n([\s\S]*?)```/g;
const codeBlocks = [];
let match;
while ((match = codeBlockRegex.exec(markdownText)) !== null) {
if (lang === null || match[1] === lang) {
codeBlocks.push(match[2]);
}
}
return codeBlocks;
}
export default class getView {
static getViewComponent(params: { props: Record<string, any>; need: string }) {
return new Promise((res, rej) => {
OpenAI.request(
[
{
role: 'user',
content: `我正在使用react-live作为一个实时编辑和编译React组件的工具。请根据我的需求,输出一段能在react-live运行的代码,只需要导出默认组件,不用挂载dom。可用的作用域如下:
{
data,
import: {
'styled-components':{
'描述':'A css in js Framework,Visual primitives for the component age. Use the best bits of ES6 and CSS to style your apps without stress'
},
'echarts':{
'描述':'Apache ECharts is a powerful, interactive charting and data visualization library for browser.',
'用法': "import echarts from 'echarts';"
},
'echarts-for-react':{
'描述':'Apache ECharts is a powerful, interactive charting and data visualization library for browser.',
'用法': "import React from 'react';
import ReactECharts from 'echarts-for-react'; // or var ReactECharts = require('echarts-for-react');
<ReactECharts
option={this.getOption()}
notMerge={true}
lazyUpdate={true}
theme={"theme_name"}
onChartReady={this.onChartReadyCallback}
onEvents={EventsDict}
opts={}
/>"
},
},
}
其中import可供导入的库,除此之外不能使用其它第三方库。 key为包名,value 是包的描述。其它每个属性将作为单独全局变量,变量名为属性名。请确保默认导出的组件接受null作为参数,并将传入props取自模块全局作用域变量data(const props = data),全局变量变量data会自动注入,请不要进行声明。例如:
export default App() {
const props = data;
...
}
`,
},
{
role: 'assistant',
content: `好的我明白啦,请问你的业务是什么?`,
},
{
role: 'user',
content: `这是我传入的props:${JSON.stringify(
params.props.slice(0, min([params.props.length, 4]))
)}${params.need}`,
},
],
(responseText: string, cancel?: boolean) => {
let code;
try {
code = JSON.parse(get(JSON.parse(responseText), 'arguments')).reactCode;
} catch (e) {}
res({
data: {
code: code,
},
});
},
undefined,
undefined,
[
{
name: 'runReactCode',
description: '作为一个实时编译React组件代码的函数。',
parameters: {
type: 'object',
properties: {
reactCode: {
type: 'string',
description: '输出一段能在react-live运行的代码',
},
},
required: ['sql', 'queryName', 'queryDescription', 'simulation'],
},
},
],
undefined,
{ name: 'runReactCode' },
false
);
});
// return backendApi.post(
// '/openAi/api/reactLive',
// isArray(params.props)
// ? {
// ...params,
// props: params.props.slice(0, min([params.props.length, 4])),
// }
// : params,
// privateConfig
// );
}
static getViewFunction(params: { data: Record<string, any>; need: string }) {
return backendApi.post(
'/openAi/api/code',
isArray(params.data)
? {
...params,
data: params.data.slice(0, min([params.data.length, 4])),
}
: params,
privateConfig
);
}
static checkChatResult(messageList: ChatMessage[]) {
return backendApi.post('/openAi/api/checkQuery', { messageList }, privateConfig);
}
}
'use client';
import { set } from 'lodash';
import axios from 'redaxios';
const APi = axios.create({});
export const backendApi = axios.create({
baseURL:globalThis?.sessionStorage?.getItem("baseURL") || '/backend',
});
(() => {
if (globalThis.sessionStorage && !sessionStorage.getItem("baseURL")) {
const url = process.env.NEXT_PUBLIC_BACKEND_URL || 'http://localhost:3001'
axios.get((url) + "/schema/all").then(
() => {
set(backendApi.defaults, "baseURL", url)
sessionStorage.setItem("baseURL", url)
},
err => {
console.log(err);
sessionStorage.setItem("baseURL",'/backend')
}
);
}
})();
export default APi;
import getConfig from 'next/config';
import { backendApi } from '.';
import { getModel } from '@/utils/gpt';
import { fetchEventSource, EventStreamContentType } from '@microsoft/fetch-event-source';
import { MutableRefObject, SetStateAction } from 'react';
import { functions, get } from 'lodash';
export interface View {
type: 'schema' | 'table';
name?: string;
}
const model = getModel(process.env.NEXT_PUBLIC_MODEL_NAME || 'gpt-4o-mini');
export function prettyObject(msg: any) {
const obj = msg;
if (typeof msg !== 'string') {
msg = JSON.stringify(msg, null, ' ');
}
if (msg === '{}') {
return obj.toString();
}
if (msg.startsWith('```json')) {
return msg;
}
return ['```json', msg, '```'].join('\n');
}
type message = {
role: string;
content: string;
};
export default class OpenAI {
static async request(
messages: message[],
onFinish: (responseText: string, cancel?: boolean) => any,
onUpdate?: (responseText: string, delta: string) => any,
onError?: (e: Error) => any,
functions?: any[],
closeFn?: MutableRefObject<any>,
function_call?: any,
stream: boolean = true
) {
const requestPayload = {
messages: messages,
model: model.name,
temperature: model.temperature,
frequency_penalty: model.frequency_penalty,
presence_penalty: model.presence_penalty,
stream,
functions,
function_call,
};
const controller = new AbortController();
const chatPayload = {
method: 'POST',
body: JSON.stringify(requestPayload),
signal: controller.signal,
headers: {
'cache-control': 'no-cache',
'Content-Type': 'application/json',
'x-requested-with': 'XMLHttpRequest',
Accept: 'text/event-stream',
},
};
const requestTimeoutId = setTimeout(() => controller.abort(), 1000 * 120);
const chatPath = '/openai/v1/chat/completions';
if (closeFn) {
controller.signal.onabort = () => {
onFinish('', true);
};
closeFn.current = () => {
clearTimeout(requestTimeoutId);
controller.abort();
};
}
if (stream) {
let responseText = '';
let finished = false;
const finish = (cancel?: boolean) => {
if (!finished) {
onFinish(responseText, cancel);
finished = true;
}
};
controller.signal.onabort = () => {
finish(true);
};
fetchEventSource(chatPath, {
...chatPayload,
async onopen(res) {
clearTimeout(requestTimeoutId);
const contentType = res.headers.get('content-type');
// console.log('[OpenAI] request response content type: ', contentType);
if (contentType?.startsWith('text/plain')) {
responseText = await res.clone().text();
return finish();
}
if (
!res.ok ||
!res.headers.get('content-type')?.startsWith(EventStreamContentType) ||
res.status !== 200
) {
const responseTexts = [responseText];
let extraInfo = await res.clone().text();
try {
const resJson = await res.clone().json();
extraInfo = prettyObject(resJson);
} catch {}
if (extraInfo) {
responseTexts.push(extraInfo);
}
responseText = responseTexts.join('\n\n');
return finish();
}
},
onmessage(msg) {
if (msg.data === '[DONE]' || finished) {
return finish();
}
const text = msg.data;
try {
// console.log(text);
const json = JSON.parse(text);
const delta = json.choices[0].delta.content;
if (json.choices[0].finish_reason === 'function_call') {
// 🌟 output the Tree structure data
console.log(json);
}
if (delta) {
responseText += delta;
onUpdate?.(responseText, delta);
}
} catch (e) {
console.error('[Request] parse error', text, msg);
}
},
onclose() {
finish();
},
onerror(e) {
onError?.(e);
throw e;
},
openWhenHidden: true,
});
} else {
const res = await fetch(chatPath, chatPayload);
clearTimeout(requestTimeoutId);
const resJson = await res.json();
const message =
resJson.choices?.at(0)?.message?.content ??
(JSON.stringify(get(resJson, 'choices[0].message.function_call')) || '');
onFinish(message);
}
}
}
import React from 'react';
import MarkdownIt from 'markdown-it';
import mdHighlight from 'markdown-it-highlightjs';
// import 'katex/dist/katex.min.css';
import doMarkdownit from '@digitalocean/do-markdownit';
import Prism from '@digitalocean/do-markdownit/vendor/prismjs';
import prismTools from '@digitalocean/do-markdownit/vendor/prismjs/plugins/toolbar/prism-toolbar';
import prismCopyToClipboard from '@digitalocean/do-markdownit/vendor/prismjs/plugins/copy-to-clipboard/prism-copy-to-clipboard';
import { debounce } from 'lodash';
// import style manually
export interface ChatMessage {
role: 'system' | 'user' | 'assistant';
content: string;
}
interface Props {
role: ChatMessage['role'];
message: string;
}
// Finish!
const addPlugins = debounce(() => {
prismTools(Prism);
prismCopyToClipboard(Prism);
Prism.highlightAll();
}, 50);
export const htmlString = (message: string | (() => string)) => {
const md = MarkdownIt()
.use(mdHighlight)
.use(doMarkdownit, {
fence_environment: {
allowedEnvironments: '*',
},
fence_classes: {
allowedClasses: false,
},
callout: {
allowedClasses: ['note', 'warning', 'info', 'draft'],
},
});
addPlugins();
if (typeof message === 'function') {
return md.render(message());
} else if (typeof message === 'string') {
return md.render(message);
}
return '';
};
export function codeWrapper(type: string, code: string) {
return '``` ' + type + '\n' + code + '\n' + '```';
}
export default function MessageItem({ message, role }: Props) {
// role === 'user';
return (
<div
className={`flex gap-3 p-4 box-border mx-[5px] shadow rounded transition-colors mt-[20px] font-hm ${
role === 'user'
? 'bg-[rgb(var(--primary-6))] text-white shadow-[var(--pc)]'
: 'bg-[var(--white-bg)] text-[#333]'
}`}
>
<div
className="message prose text-slate break-words overflow-hidden"
dangerouslySetInnerHTML={{
__html: htmlString(message),
}}
/>
</div>
);
}
This diff is collapsed.
import { useCallback } from 'react';
import ReactFlow, {
MiniMap,
Controls,
Background,
useNodesState,
useEdgesState,
addEdge,
} from 'reactflow';
import 'reactflow/dist/style.css';
const initialNodes = [];
const initialEdges = [{ id: 'e1-2', source: '1', target: '2' }];
function Flow() {
const [nodes, setNodes, onNodesChange] = useNodesState(initialNodes);
const [edges, setEdges, onEdgesChange] = useEdgesState(initialEdges);
const onConnect = useCallback(params => setEdges(eds => addEdge(params, eds)), [setEdges]);
return (
<ReactFlow
className="canvas"
nodes={nodes}
edges={edges}
onNodesChange={onNodesChange}
onEdgesChange={onEdgesChange}
onConnect={onConnect}
>
<MiniMap />
<Controls />
<Background />
</ReactFlow>
);
}
export default Flow;
import { Dropdown, Menu, Space, Divider } from '@arco-design/web-react';
import graphState from '../hooks/use-graph-state';
import tableModel from '../hooks/table-model';
import { useTranslation } from 'react-i18next';
export default function ContextMenu({ setShowModal, children }) {
const { version } = graphState.useContainer();
const { updateGraph, addTable } = tableModel();
const { t } = useTranslation('ContextMenu');
const menus = [
{
key: 'N',
title: t('Add New Table'),
action: () => addTable(),
},
{
key: 'I',
title: t('Import Table'),
action: () => setShowModal('import'),
},
{
key: 'line',
},
{
key: 'S',
title: t('Save Change'),
action: () => updateGraph(),
},
{
key: 'E',
title: t('Export Database'),
action: () => setShowModal('export'),
},
];
return (
<Dropdown
trigger="contextMenu"
position="bl"
droplist={
version !== 'currentVersion' ? null : (
<Menu className="context-menu">
{menus.map(item =>
item.key === 'line' ? (
<Divider key={item.key} className="context-menu-line" />
) : (
<Menu.Item
key={item.key}
className="context-menu-item"
onClick={item.action}
>
{item.title}
<Space size={4}>
<div className="arco-home-key"></div>
<div className="arco-home-key">{item.key}</div>
</Space>
</Menu.Item>
)
)}
</Menu>
)
}
>
{children}
</Dropdown>
);
}
import { useState, useEffect } from 'react';
import { Modal, Notification, Tabs } from '@arco-design/web-react';
import Editor from '@monaco-editor/react';
import graphState from '../hooks/use-graph-state';
import exportSQL from '../utils/export-sql';
import { useTranslation } from 'react-i18next';
const TabPane = Tabs.TabPane;
/**
* It's a modal that displays the command to be exported
* @returns Modal component
*/
export default function ExportModal({ showModal, onCloseModal }) {
const [exportType, setExportType] = useState('dbml');
const [sqlValue, setSqlValue] = useState('');
const { t } = useTranslation('modal');
const { tableDict, linkDict, theme } = graphState.useContainer();
const copy = async () => {
try {
await window.navigator.clipboard.writeText(sqlValue);
Notification.success({
title: t('Copy Success'),
});
} catch (e) {
console.log(e);
Notification.error({
title: t('Copy Failed'),
});
}
};
useEffect(() => {
if (showModal === 'export') {
const sql = exportSQL(tableDict, linkDict, exportType);
setSqlValue(sql);
}
}, [showModal, exportType]);
const editor = (
<Editor
className={`!mt-0 ${theme === 'dark' ? 'bg-[#1e1e1e]' : ' bg-[#fff]'} mt-[10px]`}
language={exportType === 'dbml' ? 'apex' : 'sql'}
width="680px"
height="60vh"
theme={theme === 'dark' ? 'vs-dark' : 'light'}
value={sqlValue}
options={{
acceptSuggestionOnCommitCharacter: true,
acceptSuggestionOnEnter: 'on',
accessibilitySupport: 'auto',
autoIndent: false,
automaticLayout: true,
codeLens: true,
colorDecorators: true,
contextmenu: true,
cursorBlinking: 'blink',
cursorSmoothCaretAnimation: false,
cursorStyle: 'line',
disableLayerHinting: false,
disableMonospaceOptimizations: false,
dragAndDrop: false,
fixedOverflowWidgets: false,
folding: true,
foldingStrategy: 'auto',
fontLigatures: false,
formatOnPaste: false,
formatOnType: false,
hideCursorInOverviewRuler: false,
highlightActiveIndentGuide: true,
links: true,
mouseWheelZoom: false,
multiCursorMergeOverlapping: true,
multiCursorModifier: 'alt',
overviewRulerBorder: true,
overviewRulerLanes: 2,
quickSuggestions: true,
quickSuggestionsDelay: 100,
readOnly: false,
renderControlCharacters: false,
renderFinalNewline: true,
renderIndentGuides: true,
renderLineHighlight: 'line',
renderWhitespace: 'none',
revealHorizontalRightPadding: 300,
roundedSelection: true,
rulers: [],
scrollBeyondLastColumn: 5,
scrollBeyondLastLine: true,
selectOnLineNumbers: true,
selectionClipboard: true,
selectionHighlight: true,
showFoldingControls: 'mouseover',
smoothScrolling: false,
suggestOnTriggerCharacters: true,
wordBasedSuggestions: true,
wordSeparators: '~!@#$%^&*()-=+[{]}|;:\'",.<>/?',
wordWrap: 'wordWrapColumn',
wordWrapBreakAfterCharacters: '\t})]?|&,;',
wordWrapBreakBeforeCharacters: '{([+',
wordWrapBreakObtrusiveCharacters: '.',
wordWrapColumn: 80,
wordWrapMinified: true,
wrappingIndent: 'none',
// minimap: {
// autohide: true,
// },
}}
onChange={setSqlValue}
/>
);
return (
<Modal
title={null}
simple
visible={showModal === 'export'}
autoFocus={false}
onOk={() => copy()}
okText={t('Copy')}
cancelText={t('Close')}
onCancel={() => onCloseModal()}
style={{ width: 'auto' }}
>
<h5 className="text-[20px] py-[10px] font-bold">{t('Export ERD Data Model')}</h5>
<Tabs
activeTab={exportType}
onChange={val => setExportType(val)}
className="ring-2 ring-[#359c899a] p-0 w-[680px]"
>
<TabPane key="dbml" title="DBML">
{editor}
</TabPane>
<TabPane key="postgres" title="PostgreSQL">
{editor}
</TabPane>
<TabPane key="mysql" title="MySQL">
{editor}
</TabPane>
<TabPane key="mssql" title="MSSQL">
{editor}
</TabPane>
</Tabs>
</Modal>
);
}
import { Checkbox, Form, Input, Space, Tag, Modal, AutoComplete } from '@arco-design/web-react';
import fieldTypes from '../data/filed_typs';
import graphState from '../hooks/use-graph-state';
import tableModel from '../hooks/table-model';
import { useTranslation } from 'react-i18next';
/**
* It renders a form for editing a table
* @param props - The props passed to the component.
* @returns A TableForm component
*/
export default function FieldForm(props) {
const [form] = Form.useForm();
const { formChange, onFormChange } = props;
const { editingField, setEditingField, addingField, setAddingField } =
graphState.useContainer();
const { updateTable, removeField } = tableModel();
const { field, table } = editingField;
const save = values => {
const data = { ...field, ...values };
table.fields = table.fields.map(f => (f.id === data.id ? data : f));
updateTable(table);
};
const { t } = useTranslation('graph');
return table ? (
<Modal
title={
<div style={{ textAlign: 'left' }}>
{t('Edit')}
{table ? (
<Tag color="arcoblue" style={{ margin: '0 4px' }}>
{table.name}
</Tag>
) : (
''
)}
{t('Field')}
</div>
}
visible={!!table}
onCancel={() => {
if (addingField?.index) {
removeField(addingField.table, addingField.index);
}
setEditingField({});
}}
onOk={() => {
setAddingField(null);
form.submit();
}}
escToExit={!formChange}
maskClosable={!formChange}
afterClose={() => {
onFormChange(false);
}}
afterOpen={() => {
form.resetFields();
}}
style={{ width: 580 }}
okText="Commit"
cancelText={t('Cancel')}
>
{field && (
<Form
onSubmit={save}
form={form}
labelAlign="left"
requiredSymbol={false}
labelCol={{ span: 6 }}
wrapperCol={{ span: 18 }}
onValuesChange={(changedValues, allValues) => {
if (!formChange) onFormChange(true);
}}
>
<Space direction="vertical" style={{ width: '100%' }}>
<Space className="table-form-item">
<Form.Item
label={t('Name')}
field="name"
initialValue={field.name}
rules={[
{
required: true,
message: t('Please enter field name'),
},
{
validator: (value, cb) => {
return table.fields
.filter(item => item.id !== field.id)
.find(item => item.name === value)
? cb(t('have same name field'))
: cb();
},
},
]}
>
<Input allowClear placeholder={t('Name')} />
</Form.Item>
<Form.Item
label={t('Type')}
field="type"
initialValue={field.type}
rules={[
{
required: true,
message: t('Please choose field type'),
},
]}
>
<AutoComplete
data={fieldTypes}
placeholder={t('Type')}
></AutoComplete>
</Form.Item>
</Space>
<Space className="table-form-item">
<Form.Item
label={t('Comment')}
field="note"
initialValue={field.note || ''}
>
<Input allowClear placeholder={t('Comment')} />
</Form.Item>
<Form.Item
label={t('Default')}
field="dbdefault"
initialValue={field.dbdefault || ''}
>
<Input allowClear placeholder={t('Default')} />
</Form.Item>
</Space>
<Space className="table-form-item">
<Form.Item noStyle field="pk" initialValue={field.pk}>
<Checkbox defaultChecked={field.pk}>Primary</Checkbox>
</Form.Item>
<Form.Item noStyle field="unique" initialValue={field.unique}>
<Checkbox defaultChecked={field.unique}>Unique</Checkbox>
</Form.Item>
<Form.Item noStyle field="not_null" initialValue={field.not_null}>
<Checkbox defaultChecked={field.not_null}>Not Null</Checkbox>
</Form.Item>
<Form.Item noStyle field="increment" initialValue={field.increment}>
<Checkbox defaultChecked={field.increment}>Increment</Checkbox>
</Form.Item>
</Space>
</Space>
</Form>
)}
</Modal>
) : null;
}
import { Upload, Message, UploadProps } from '@arco-design/web-react';
import * as XLSX from 'xlsx';
import { useState } from 'react';
import pinyin from 'pinyin';
import { flatten, get, isDate, isString, map } from 'lodash';
const e = /^\s+$/g;
function getDataType(value: string) {
if (!value || (isString(value) && e.test(value))) {
console.log(value, 'value');
return 'VARCHAR(255)';
} else if (isDate(value) && !isNaN(Date.parse(value))) {
if (/^\d{4}-\d{2}-\d{2}$/.test(value)) {
return 'DATE';
} else {
if (/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{1,3})?$/.test(value)) {
return 'DATETIME';
}
return 'VARCHAR(255)';
}
} else if (isFinite(Number(value))) {
const number = Number(value);
if (Number.isInteger(number)) {
if (number >= -2147483648 && number <= 2147483647) {
return 'INT';
} else {
return 'BIGINT';
}
} else {
if (number >= -9999999 && number <= 9999999) {
return 'DECIMAL(10, 2)';
} else {
return 'DECIMAL(20, 2)';
}
}
} else if (value === 'true' || value === 'false') {
return 'BOOLEAN';
} else {
if (value.length <= 255) {
return 'VARCHAR(255)';
} else {
return 'TEXT';
}
}
}
function isChinese(text: string) {
var re = /[\u4E00-\u9FA5\uF900-\uFA2D]/;
return re.test(text);
}
export function handleUpload(files: File[]) {
return new Promise(res => {
const ddlArr: string[] = [];
for (let i = 0; i < files.length; i++) {
const file = files[i];
const reader = new FileReader();
reader.onload = (e: any) => {
const data = new Uint8Array(e.target.result);
const workbook = XLSX.read(data, {
type: 'array',
cellDates: true,
cellNF: true,
});
const sheetName = workbook.SheetNames[0];
const jsonData = XLSX.utils.sheet_to_json<any>(workbook.Sheets[sheetName], {
defval: '',
header: 1,
});
let tableName = file.name.split('.')[0];
if (isChinese(tableName)) {
tableName = flatten(
pinyin(tableName, {
style: 'normal',
})
).join('_');
}
const columns: any[] = jsonData[0];
const columnTypes = map(columns, (_, column) => {
const firstNonNullValue = get(
jsonData.slice(1).find(row => row[column] != null),
[column],
'VARCHAR(255)'
);
return getDataType(firstNonNullValue);
});
const ddl = `CREATE TABLE \`${tableName}\` (${map(
columns,
(column, i) => `\`${column}\` ${columnTypes[i]}`
).join(', ')});`;
ddlArr.push(ddl);
if (ddlArr.length === files.length) {
res(ddlArr);
}
};
reader.readAsArrayBuffer((file as File & { originFile: File })?.originFile);
}
});
}
const ImportExcel = ({ setList }: { setList: typeof useState<UploadProps['fileList']> }) => {
return (
<Upload
drag
multiple
accept=".xls,.xlsx,application/vnd.ms-excel,application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
onDrop={e => {
let uploadFile = e.dataTransfer.files;
}}
autoUpload={false}
onChange={lists => setList(lists)}
/>
);
};
export default ImportExcel;
This diff is collapsed.
import { Graph, Path } from '@antv/x6';
export default function ConnectorInit() {
Graph.registerConnector(
'algo-connector',
(s, e) => {
const offset = 4;
const deltaY = Math.abs(e.y - s.y);
const control = Math.floor((deltaY / 3) * 2);
const v1 = { x: s.x, y: s.y + offset + control };
const v2 = { x: e.x, y: e.y - offset - control };
return Path.normalize(
`M ${s.x} ${s.y}
L ${s.x} ${s.y + offset}
C ${v1.x} ${v1.y} ${v2.x} ${v2.y} ${e.x} ${e.y - offset}
L ${e.x} ${e.y}
`
);
},
true
);
}
import { Cell } from '@antv/x6';
import { ReactShapeConfig } from '@antv/x6-react-shape';
import { type } from 'os';
import React from 'react';
interface DagContext {
activeCell?: Cell;
}
export enum DagActions {
'CHANGE_ACTIVE_CELL',
}
let count = 1;
type dispatchType<T> = (type: T) => any;
type ExtractProps<T> = T extends React.ComponentType<infer P> ? P : never;
type NodeProps = ReactShapeConfig['component'];
export type CustomProps = ExtractProps<NodeProps>;
export type DagContextType = React.Context<
DagContext & {
dispatch: (actions: {
type: DagActions;
payload: (typeof actions)['type'] extends DagActions.CHANGE_ACTIVE_CELL ? Cell : never;
}) => any;
}
>;
export default function getDagContext() {
const DagContext = React.createContext<DagContext>({});
DagContext.displayName = 'DagContext' + count;
const { Provider, Consumer, displayName } = DagContext;
console.log(count, 'countcount');
count++;
return { Provider, Consumer, displayName, DagContext };
}
import { Graph } from '@antv/x6';
export default function EdgesInit(){
Graph.registerEdge(
'dag-edge',
{
inherit: 'edge',
attrs: {
line: {
stroke: '#C2C8D5',
strokeWidth: 1,
targetMarker: null,
},
},
},
true
);
}
\ No newline at end of file
'use client';
import React, {
useEffect,
useRef,
useCallback,
useReducer,
type Provider as ReactProvider,
useMemo,
useState,
} from 'react';
import { Graph, Path, Cell } from '@antv/x6';
import { Selection } from '@antv/x6-plugin-selection';
// import insertCss from 'insert-css';
import { History } from '@antv/x6-plugin-history';
import { DagreLayout } from '@antv/layout';
import styled from 'styled-components';
import { Button, Collapse, Layout, List } from '@arco-design/web-react';
import { IconLayout } from '@arco-design/web-react/icon';
import { Dnd } from '@antv/x6-plugin-dnd';
import { MiniMap } from '@antv/x6-plugin-minimap';
import nodeStatusList from './nodeStatus';
import { NodeStatus, NodeType } from './nodes/AlgoNode';
import ConnectorInit from './connectors';
import EdgesInit from './edges';
import nodesInit from './nodes';
import { Div } from './styleWidgets';
import { SlideForm } from './nodeConfigs';
import getDagContext, { DagActions } from './context';
import { produce } from 'immer';
import { set, uniqueId } from 'lodash';
import { Portal } from '@antv/x6-react-shape';
const CollapseItem = Collapse.Item;
//注册
ConnectorInit();
EdgesInit();
nodesInit();
const Sider = Layout.Sider;
const Header = Layout.Header;
const Content = Layout.Content;
const X6ReactPortalProvider = Portal.getProvider();
function A() {
console.log('A');
return <div>A</div>;
}
function useFirst(fn: typeof getDagContext) {
const flag = useRef<ReturnType<typeof fn>>();
if (!flag.current) {
flag.current = fn();
}
return flag.current;
}
export default function Demo() {
const el = useRef<HTMLDivElement>(null);
const graphEl = useRef<Graph | null>();
const dndEl = useRef<Dnd | null>();
const context = useFirst(getDagContext);
const graph = graphEl.current;
const dnd = dndEl.current;
const { Provider, DagContext } = context;
console.log(DagContext, 'DagContext');
const [state, dispatch] = useReducer(
produce(
(
state: typeof Provider extends ReactProvider<infer P> ? P : never,
action: {
type: DagActions;
payload: (typeof action)['type'] extends DagActions.CHANGE_ACTIVE_CELL ? Cell : never;
}
) => {
switch (action.type) {
case DagActions.CHANGE_ACTIVE_CELL: {
console.info(action);
state.activeCell = action.payload;
}
}
}
),
{}
);
const store = useMemo(() => {
return {
...state,
dispatch: dispatch,
};
}, [state]);
console.log(store);
useEffect(() => {
if (el) {
const w = (el?.current?.clientWidth || 400) - 400;
const h = el?.current?.clientHeight;
const graph: Graph & {
DagContext?: typeof DagContext;
} = (graphEl.current = new Graph({
container: el.current!,
width: w,
height: 418,
panning: {
enabled: true,
eventTypes: ['leftMouseDown', 'mouseWheel'],
},
mousewheel: {
enabled: true,
modifiers: 'ctrl',
factor: 1.1,
maxScale: 1.5,
minScale: 0.5,
},
highlighting: {
magnetAdsorbed: {
name: 'stroke',
args: {
attrs: {
fill: '#fff',
stroke: '#31d0c6',
strokeWidth: 4,
},
},
},
},
connecting: {
snap: true,
allowBlank: false,
allowLoop: false,
highlight: true,
connector: 'algo-connector',
connectionPoint: 'anchor',
anchor: 'center',
validateMagnet({ magnet }) {
return magnet.getAttribute('port-group') !== 'top';
},
createEdge() {
return graph.createEdge({
shape: 'dag-edge',
attrs: {
line: {
strokeDasharray: '5 5',
},
},
zIndex: -1,
});
},
},
} as any));
graph.DagContext = DagContext;
const dnd = (dndEl.current = new Dnd({
target: graph,
}));
// graph.use(
// new MiniMap({
// container: document.getElementById('minimap'),
// }),
// )
graph.use(
new Selection({
multiple: true,
rubberEdge: true,
rubberNode: true,
modifiers: 'shift',
rubberband: true,
})
);
graph.use(
new History({
enabled: true,
})
);
graph.on('edge:connected', ({ edge }) => {
edge.attr({
line: {
strokeDasharray: '',
},
});
});
graph.on('node:change:data', ({ node }) => {
const edges = graph.getIncomingEdges(node);
const { status } = node.getData() as NodeStatus;
edges?.forEach(edge => {
if (status === 'running') {
edge.attr('line/strokeDasharray', 5);
edge.attr('line/style/animation', 'running-line 30s infinite linear');
} else {
edge.attr('line/strokeDasharray', '');
edge.attr('line/style/animation', '');
}
});
});
graph.on('node:contextmenu', ({ node, view }) => {
dispatch({
type: DagActions.CHANGE_ACTIVE_CELL,
payload: node,
});
});
// 初始化节点/边
const init = (data: Cell.Metadata[]) => {
const cells: Cell[] = [];
data.forEach(item => {
cells.push(graph.createNode(item));
});
graph.resetCells(cells);
};
init([
{
id: 'flow' + '_' + Date.now(),
shape: 'dag-node',
x: 290,
y: 110,
data: {
label: '流程开始',
status: 'success',
nodeType: NodeType.FLOW_START,
},
ports: [],
},
]);
graph.centerContent();
return () => {
graph.dispose();
};
}
}, []);
const startDrag = (e: React.MouseEvent<HTMLDivElement, MouseEvent>) => {
// 该 node 为拖拽的节点,默认也是放置到画布上的节点,可以自定义任何属性
if (graph && dnd) {
const node = graph.createNode({
shape: 'rect',
width: 100,
height: 40,
});
dnd.start(node, e.nativeEvent);
}
};
return (
<Div className="react-portal-app">
<Layout className="h-[500px]">
<Header className="h-[40px] flex pl-[200px] bg-slate-50 border-b border-b-gray-200 relative z-[1] bg-clip-content">
<div className="px-[10px]">
<Button
iconOnly
icon={<IconLayout />}
onClick={() => {
// const dagreLayout = new DagreLayout({
// type: 'dagre',
// rankdir: 'LR',
// align: 'UR',
// ranksep: 35,
// nodesep: 15,
// });
// const model = dagreLayout.layout({
// nodes: graphEl.current?.getNodes().map((v,i) => {
// return {
// id: `${v.id}`,
// shape: 'circle',
// width: 32,
// height: 32,
// label: i,
// attrs: {
// body: {
// fill: '#5F95FF',
// stroke: 'transparent',
// },
// label: {
// fill: '#ffffff',
// },
// },
// };
// }),
// edges: graphEl.current?.getEdges(),
// });
// console.log(model);
// graphEl.current?.fromJSON(model);
}}
/>
</div>
</Header>
<Layout>
<Sider className="bg-slate-100 mt-[-40px] border-r border-r-gray-200 pt-[40px]">
<Collapse bordered={false} defaultActiveKey={['1']}>
<CollapseItem header="Query" name="1">
<List
style={{ width: '100%' }}
size="small"
header={null}
dataSource={['查询1', '查询2']}
render={(item, index) => (
<div className="m-[10px] text-center bg-slate-300 p-[5px] rounded cursor-pointer text-purple-400 font-bold shadow">
<span className="w-full" key={index}>
{item}
</span>
</div>
)}
/>
</CollapseItem>
</Collapse>
</Sider>
<Content>
<Provider value={store}>
<X6ReactPortalProvider />
</Provider>
<div ref={el}></div>
</Content>
<Sider
className="bg-slate-100 mt-[-40px] border-l border-l-gray-200 pt-[40px]"
collapsible
collapsedWidth={0}
>
<SlideForm />
</Sider>
</Layout>
</Layout>
</Div>
);
}
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
import { register } from '@antv/x6-react-shape';
import AlgoNode from './AlgoNode';
export default function nodesInit() {
register(AlgoNode);
}
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment