From 35bd4c70ceb9ef57da061cf9be2d5eda5ddbcdc5 Mon Sep 17 00:00:00 2001 From: inrgihc Date: Fri, 18 Oct 2024 22:49:50 +0800 Subject: [PATCH] =?UTF-8?q?=E6=94=AF=E6=8C=81=E9=85=8D=E7=BD=AE=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E5=88=87=E6=8D=A2=E7=8E=AF=E5=A2=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- LICENSE | 2 +- README.md | 78 ++++++++++++------- SUPPORTED_PRODUCTS.md | 2 +- .../conf/application-mysql.yml | 13 ++++ .../conf/application-postgres.yml | 13 ++++ .../dbswitch-release/conf/application.yml | 20 ++--- build-docker/install/docker-compose.yml | 1 + ...sample_mysql.yml => application-mysql.yml} | 22 ------ .../main/resources/application-postgres.yml | 13 ++++ .../src/main/resources/application.yml | 21 ++--- .../sample/application_sample_postgres.yml | 35 --------- .../data/service/MigrationService.java | 3 +- flyway-core/{REAME.md => README.md} | 0 13 files changed, 103 insertions(+), 120 deletions(-) create mode 100644 build-docker/dbswitch/dbswitch-release/conf/application-mysql.yml create mode 100644 build-docker/dbswitch/dbswitch-release/conf/application-postgres.yml rename dbswitch-admin/src/main/resources/{sample/application_sample_mysql.yml => application-mysql.yml} (51%) create mode 100644 dbswitch-admin/src/main/resources/application-postgres.yml delete mode 100644 dbswitch-admin/src/main/resources/sample/application_sample_postgres.yml rename flyway-core/{REAME.md => README.md} (100%) diff --git a/LICENSE b/LICENSE index b6265e98..56eae528 100644 --- a/LICENSE +++ b/LICENSE @@ -12,7 +12,7 @@ // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. -// * Neither the name of Shuo Chen nor the names of other contributors +// * Neither the name of tang(inrgihc) nor the names of other contributors // may be used to endorse or promote products derived from this software // without specific prior written permission. // diff --git a/README.md b/README.md index 45567839..7c038c47 100644 --- a/README.md +++ b/README.md @@ -23,11 +23,7 @@ 支持有主键表的 **增量变更同步** （变化数据计算Change Data Calculate）功能(千万级以上数据量的性能尚需在生产环境验证) -### 2、功能设计 - - ![function](images/function.PNG) - -### 3、结构设计 +### 2、结构设计 - 模块结构功能 @@ -53,11 +49,11 @@ ├── dbswitch-product-sqlite // -> sqlite方言实现类 ├── dbswitch-product-greenplum // -> greenplum方言实现类 ├── dbswitch-product-clickhouse // -> clickhouse方言实现类 - ├── dbswitch-product-mongodb // -> mongodb方言实现类 - ├── dbswitch-product-elasticsearch // -> elasticsearch方言实现类 ├── dbswitch-product-starrocks // -> starrocks方言实现类 ├── dbswitch-product-doris // -> doris方言实现类 ├── dbswitch-product-oceanbase // -> oceanbase方言实现类 + ├── dbswitch-product-mongodb // -> mongodb方言实现类 + ├── dbswitch-product-elasticsearch // -> elasticsearch方言实现类 ├── dbswitch-data // 工具入口模块，读取配置文件中的参数执行异构迁移同步 ├── dbswitch-admin // 在以上模块的基础上引入Quartz的调度服务与接口 ├── dbswitch-admin-ui // 基于Vue2的前段WEB交互页面 @@ -107,9 +103,12 @@ sh ./docker-maven-build.sh (2) docker容器方式部署： -假设已经部署好的MySQL数据库地址为192.168.31.57，端口为3306，账号为test，密码为123456 +- MYSQL做配置库部署 + +假设已经部署好的MySQL(5.7+)数据库地址为192.168.31.57，端口为3306，账号为test，密码为123456 ``` docker run -d --name dbswitch \ + -e DBTYPE=mysql \ -e MYSQLDB_HOST=192.168.31.57 \ -e MYSQLDB_PORT=3306 \ -e MYSQLDB_USERNAME=test \ @@ -120,6 +119,22 @@ docker run -d --name dbswitch \ registry.cn-hangzhou.aliyuncs.com/inrgihc/dbswitch:latest ``` +- PostgreSQL/OpenGauss做配置库部署 + +假设已经部署好的PostgreSQL/OpenGauss数据库地址为192.168.31.57，端口为5432，账号为test，数据库为dbswitch(需先建好), 密码为123456 +``` +docker run -d --name dbswitch \ + -e DBTYPE=postgres \ + -e PGDB_HOST=192.168.31.57 \ + -e PGDB_PORT=5432 \ + -e PGDB_USERNAME=test \ + -e PGDB_PASSWORD='123456' \ + -e PGDB_NAME='dbswitch' \ + -v /tmp:/tmp \ + -p 9088:9088 \ + registry.cn-hangzhou.aliyuncs.com/inrgihc/dbswitch:latest +``` + (3) 基于docker-compose提供linux联网环境下的**一键安装**，安装命令见 [发行版链接地址](https://gitee.com/inrgihc/dbswitch/releases) 文档详见: [build-docker/install/README.md](build-docker/install) @@ -278,15 +293,16 @@ dbswitch.target.writer-engine-insert=true > dbswitch-admin模块后端同时支持MySQL、PostgreSQL、OpenGauss作为配置数据库。 -#### (2)、配置conf/application.yml(MySQL可参考application_sample_mysql.yml配置，PostgreSQL/OpenGauss可参考application_sample_postgresql.yml配置) +#### (2)、配置conf/application.yml -MySQL的application.yml配置内容示例如下： +application.yml配置内容示例如下： ``` -server: - port: 9088 - spring: + profiles: + # 配置包含使用的配置库类型(可选值:mysql或postgres),对应在application-mysql.yml或application-postgres.yml中配置数据库信息 + # 如果使用OpenGauss作为配置数据库，请配置为postgres类型,dbswitch会使用postgres的jdbc驱动连接OpenGauss + include: mysql application: name: dbswitch-admin tomcat: @@ -295,25 +311,16 @@ spring: mvc: throw-exception-if-no-handler-found: false static-path-pattern: /statics/** - datasource: - driver-class-name: com.mysql.jdbc.Driver - url: jdbc:mysql://192.168.31.57:3306/dbswitch?createDatabaseIfNotExist=true&useUnicode=true&characterEncoding=UTF8&useSSL=false - username: test - password: 123456 - validation-query: SELECT 1 - test-on-borrow: true - flyway: - locations: classpath:db/migration - baseline-on-migrate: true - table: DBSWITCH_SCHEMA_HISTORY - enabled: true + +server: + port: 9088 mybatis: configuration: lazy-loading-enabled: true aggressive-lazy-loading: false map-underscore-to-camel-case: true - log-impl: org.apache.ibatis.logging.stdout.StdOutImpl + #log-impl: org.apache.ibatis.logging.stdout.StdOutImpl dbswitch: configuration: @@ -321,7 +328,18 @@ dbswitch: drivers-base-path: D:/Workspace/dbswitch/drivers ``` -按照上述配置，修改```conf/application.yml```配置文件中的```spring.datasource.url```和```spring.datasource.username```及```spring.datasource.password```及```dbswitch.configuration.drivers-base-path```四个字段值的配置。 +按照上述配置，只需修改```conf/application.yml```及```conf/application-???.yml```配置文件中的如下五个参数的配置: + +- ```spring.profiles.include``` +> 使用的数据库类型，可选值(单选): mysql,postgres +- ```dbswitch.configuration.drivers-base-path``` +> 驱动JAR文件所在的目录位置 +- ```spring.datasource.url``` +> 对应数据库类型的jdbc连接串(只需修改IP地址和端口号即可) +- ```spring.datasource.username``` +> 对应数据库认证的账号 +- ```spring.datasource.password``` +> 对应数据库认证的密码 #### (3)、启动dbswitch-admin系统 @@ -434,11 +452,11 @@ cd dbswitch && mvn clean install ``` // 构建并行读取任务执行的线程池 -AsyncTaskExecutor taskReadExecutor=new ThreadPoolTaskExecutor(); +AsyncTaskExecutor readExecutor = new ThreadPoolTaskExecutor(); taskReadExecutor.setXXXX(); // 构建并行写入任务执行的线程池 -AsyncTaskExecutor taskWriteExecutor=new ThreadPoolTaskExecutor(); +AsyncTaskExecutor writeExecutor = new ThreadPoolTaskExecutor(); taskWriteExecutor.setXXXX(); // 构造dbswitch所需的配置参数，参数说明请参考第三章第1小节 @@ -447,7 +465,7 @@ properties.setSource(XXX); properties.setTarget(YYY); // 将参数传递给dbswitch启动迁移同步方式执行 -MigrationService service = new MigrationService(properties, taskReadExecutor, taskWriteExecutor); +MigrationService service = new MigrationService(properties, readExecutor, writeExecutor); service.run(); ``` diff --git a/SUPPORTED_PRODUCTS.md b/SUPPORTED_PRODUCTS.md index cad30b25..60b0ba78 100644 --- a/SUPPORTED_PRODUCTS.md +++ b/SUPPORTED_PRODUCTS.md @@ -1,4 +1,4 @@ -## DBSWITCH对数据库产品的支持列表 +## DBSWITCH支持的数据库产品列表 支持的数据库产品及其JDBC驱动连接示例如下： diff --git a/build-docker/dbswitch/dbswitch-release/conf/application-mysql.yml b/build-docker/dbswitch/dbswitch-release/conf/application-mysql.yml new file mode 100644 index 00000000..64039af5 --- /dev/null +++ b/build-docker/dbswitch/dbswitch-release/conf/application-mysql.yml @@ -0,0 +1,13 @@ +spring: + datasource: + driver-class-name: com.mysql.jdbc.Driver + url: jdbc:mysql://${MYSQLDB_HOST}:${MYSQLDB_PORT:3306}/${MYSQLDB_NAME:dbswitch}?createDatabaseIfNotExist=true&useUnicode=true&characterEncoding=UTF8&autoReconnect=true&useSSL=false&allowMultiQueries=true&failOverReadOnly=false&connectTimeout=30000 + username: ${MYSQLDB_USERNAME} + password: ${MYSQLDB_PASSWORD} + validation-query: SELECT 1 + test-on-borrow: true + flyway: + locations: classpath:db/migration + baseline-on-migrate: true + table: DBSWITCH_SCHEMA_HISTORY + enabled: true diff --git a/build-docker/dbswitch/dbswitch-release/conf/application-postgres.yml b/build-docker/dbswitch/dbswitch-release/conf/application-postgres.yml new file mode 100644 index 00000000..2bd3cf47 --- /dev/null +++ b/build-docker/dbswitch/dbswitch-release/conf/application-postgres.yml @@ -0,0 +1,13 @@ +spring: + datasource: + driver-class-name: org.postgresql.Driver + url: jdbc:postgresql://${PGDB_HOST}:${PGDB_PORT:5432}/${PGDB_NAME:dbswitch}?currentSchema=public + username: ${PGDB_USERNAME} + password: ${PGDB_PASSWORD} + validation-query: SELECT 1 + test-on-borrow: true + flyway: + locations: classpath:db/postgres + baseline-on-migrate: true + table: dbswitch_schema_history + enabled: true diff --git a/build-docker/dbswitch/dbswitch-release/conf/application.yml b/build-docker/dbswitch/dbswitch-release/conf/application.yml index 62b3ee7a..a32c35ab 100644 --- a/build-docker/dbswitch/dbswitch-release/conf/application.yml +++ b/build-docker/dbswitch/dbswitch-release/conf/application.yml @@ -1,7 +1,6 @@ -server: - port: 9088 - spring: + profiles: + include: ${DBTYPE:mysql} application: name: dbswitch-admin tomcat: @@ -10,18 +9,9 @@ spring: mvc: throw-exception-if-no-handler-found: false static-path-pattern: /statics/** - datasource: - driver-class-name: com.mysql.jdbc.Driver - url: jdbc:mysql://${MYSQLDB_HOST:dbswitch_mysqldb}:${MYSQLDB_PORT:3306}/${MYSQLDB_NAME:dbswitch}?createDatabaseIfNotExist=true&useUnicode=true&characterEncoding=UTF8&autoReconnect=true&useSSL=false&allowMultiQueries=true&failOverReadOnly=false&connectTimeout=30000 - username: ${MYSQLDB_USERNAME:tangyibo} - password: ${MYSQLDB_PASSWORD:123456} - validation-query: SELECT 1 - test-on-borrow: true - flyway: - locations: classpath:db/migration - baseline-on-migrate: true - table: DBSWITCH_SCHEMA_HISTORY - enabled: true + +server: + port: 9088 mybatis: configuration: diff --git a/build-docker/install/docker-compose.yml b/build-docker/install/docker-compose.yml index 178c993b..af8a0ed7 100644 --- a/build-docker/install/docker-compose.yml +++ b/build-docker/install/docker-compose.yml @@ -5,6 +5,7 @@ services: volumes: - "/data/mysql:/var/lib/mysql" environment: + DBTYPE: mysql MYSQL_DATABASE: dbswitch MYSQL_USER: tangyibo MYSQL_PASSWORD: 123456 diff --git a/dbswitch-admin/src/main/resources/sample/application_sample_mysql.yml b/dbswitch-admin/src/main/resources/application-mysql.yml similarity index 51% rename from dbswitch-admin/src/main/resources/sample/application_sample_mysql.yml rename to dbswitch-admin/src/main/resources/application-mysql.yml index 906775bb..d59a8ee4 100644 --- a/dbswitch-admin/src/main/resources/sample/application_sample_mysql.yml +++ b/dbswitch-admin/src/main/resources/application-mysql.yml @@ -1,15 +1,4 @@ -server: - port: 9088 - spring: - application: - name: dbswitch-admin - tomcat: - uri-encoding: UTF-8 - max-http-header-size: 8096 - mvc: - throw-exception-if-no-handler-found: false - static-path-pattern: /statics/** datasource: driver-class-name: com.mysql.jdbc.Driver url: jdbc:mysql://192.168.31.57:3306/dbswitch?createDatabaseIfNotExist=true&useUnicode=true&characterEncoding=UTF8&autoReconnect=true&useSSL=false&allowMultiQueries=true&failOverReadOnly=false&connectTimeout=30000 @@ -22,14 +11,3 @@ spring: baseline-on-migrate: true table: DBSWITCH_SCHEMA_HISTORY enabled: true - -mybatis: - configuration: - lazy-loading-enabled: true - aggressive-lazy-loading: false - map-underscore-to-camel-case: true - #log-impl: org.apache.ibatis.logging.stdout.StdOutImpl - -dbswitch: - configuration: - drivers-base-path: ${APP_DRIVERS_PATH} diff --git a/dbswitch-admin/src/main/resources/application-postgres.yml b/dbswitch-admin/src/main/resources/application-postgres.yml new file mode 100644 index 00000000..282bc300 --- /dev/null +++ b/dbswitch-admin/src/main/resources/application-postgres.yml @@ -0,0 +1,13 @@ +spring: + datasource: + driver-class-name: org.postgresql.Driver + url: jdbc:postgresql://192.168.31.57:5432/dbswitch?currentSchema=public + username: tangyibo + password: 123456 + validation-query: SELECT 1 + test-on-borrow: true + flyway: + locations: classpath:db/postgres + baseline-on-migrate: true + table: dbswitch_schema_history + enabled: true diff --git a/dbswitch-admin/src/main/resources/application.yml b/dbswitch-admin/src/main/resources/application.yml index 906775bb..fc00cd17 100644 --- a/dbswitch-admin/src/main/resources/application.yml +++ b/dbswitch-admin/src/main/resources/application.yml @@ -1,7 +1,7 @@ -server: - port: 9088 - spring: + profiles: + include: mysql + #include: postgres application: name: dbswitch-admin tomcat: @@ -10,18 +10,9 @@ spring: mvc: throw-exception-if-no-handler-found: false static-path-pattern: /statics/** - datasource: - driver-class-name: com.mysql.jdbc.Driver - url: jdbc:mysql://192.168.31.57:3306/dbswitch?createDatabaseIfNotExist=true&useUnicode=true&characterEncoding=UTF8&autoReconnect=true&useSSL=false&allowMultiQueries=true&failOverReadOnly=false&connectTimeout=30000 - username: tangyibo - password: 123456 - validation-query: SELECT 1 - test-on-borrow: true - flyway: - locations: classpath:db/migration - baseline-on-migrate: true - table: DBSWITCH_SCHEMA_HISTORY - enabled: true + +server: + port: 9088 mybatis: configuration: diff --git a/dbswitch-admin/src/main/resources/sample/application_sample_postgres.yml b/dbswitch-admin/src/main/resources/sample/application_sample_postgres.yml deleted file mode 100644 index ee5c30f3..00000000 --- a/dbswitch-admin/src/main/resources/sample/application_sample_postgres.yml +++ /dev/null @@ -1,35 +0,0 @@ -server: - port: 9088 - -spring: - application: - name: dbswitch-admin - tomcat: - uri-encoding: UTF-8 - max-http-header-size: 8096 - mvc: - throw-exception-if-no-handler-found: false - static-path-pattern: /statics/** - datasource: - driver-class-name: org.postgresql.Driver - url: jdbc:postgresql://192.168.31.57:5432/dbswitch?currentSchema=public - username: tangyibo - password: 123456 - validation-query: SELECT 1 - test-on-borrow: true - flyway: - locations: classpath:db/postgres - baseline-on-migrate: true - table: dbswitch_schema_history - enabled: true - -mybatis: - configuration: - lazy-loading-enabled: true - aggressive-lazy-loading: false - map-underscore-to-camel-case: true - #log-impl: org.apache.ibatis.logging.stdout.StdOutImpl - -dbswitch: - configuration: - drivers-base-path: ${APP_DRIVERS_PATH} diff --git a/dbswitch-data/src/main/java/com/gitee/dbswitch/data/service/MigrationService.java b/dbswitch-data/src/main/java/com/gitee/dbswitch/data/service/MigrationService.java index 9e4dccd6..34cc30ea 100644 --- a/dbswitch-data/src/main/java/com/gitee/dbswitch/data/service/MigrationService.java +++ b/dbswitch-data/src/main/java/com/gitee/dbswitch/data/service/MigrationService.java @@ -39,7 +39,7 @@ import org.springframework.util.StopWatch; */ @Slf4j @Service -public class MigrationService { +public class MigrationService implements Runnable { /** * 性能统计记录表 @@ -93,6 +93,7 @@ public class MigrationService { /** * 执行入口 */ + @Override public void run() { if (Objects.nonNull(mdcKeyValue)) { Runnable runnable = new LoggingRunnable(this::doRun, this.mdcKeyValue); diff --git a/flyway-core/REAME.md b/flyway-core/README.md similarity index 100% rename from flyway-core/REAME.md rename to flyway-core/README.md