From d9737be7abef2cc33b8f79b00a59d25270f1bc50 Mon Sep 17 00:00:00 2001 From: AE86 <836391306@qq.com> Date: Tue, 14 Nov 2023 02:24:53 +0800 Subject: [PATCH 01/98] move module --- .../dbsyncer/biz/checker/AbstractChecker.java | 6 +- .../impl/connector/ConnectorChecker.java | 20 +- .../impl/connector/OracleConfigChecker.java | 14 +- .../impl/group/ProjectGroupChecker.java | 7 +- .../checker/impl/mapping/MappingChecker.java | 24 +- .../impl/system/SystemConfigChecker.java | 8 +- .../impl/tablegroup/TableGroupChecker.java | 26 +- .../dbsyncer/biz/impl/BaseServiceImpl.java | 14 +- .../biz/impl/ConditionServiceImpl.java | 14 +- .../biz/impl/ConnectorServiceImpl.java | 64 +++- .../dbsyncer/biz/impl/ConvertServiceImpl.java | 10 +- .../biz/impl/DataSyncServiceImpl.java | 51 ++- .../dbsyncer/biz/impl/MappingServiceImpl.java | 72 ++-- .../dbsyncer/biz/impl/MonitorServiceImpl.java | 61 +++- .../dbsyncer/biz/impl/PluginServiceImpl.java | 26 +- .../biz/impl/ProjectGroupServiceImpl.java | 32 +- .../biz/impl/SystemConfigServiceImpl.java | 22 +- .../biz/impl/TableGroupServiceImpl.java | 48 +-- .../biz/impl/UserConfigServiceImpl.java | 24 +- .../java/org/dbsyncer/manager/Command.java | 7 - .../org/dbsyncer/manager/CommandExecutor.java | 6 - .../java/org/dbsyncer/manager/Executor.java | 37 --- .../org/dbsyncer/manager/ManagerFactory.java | 310 +----------------- .../dbsyncer/manager/command/Persistence.java | 14 - .../org/dbsyncer/manager/command/Preload.java | 35 -- .../manager/listener/ClosedTaskListener.java | 29 ++ .../dbsyncer/manager/puller/FullPuller.java | 18 +- .../manager/puller/IncrementPuller.java | 160 +-------- .../manager/template/PreloadTemplate.java | 140 ++++---- .../java/org/dbsyncer/monitor/Monitor.java | 28 -- .../org/dbsyncer/monitor/MonitorFactory.java | 111 +------ .../{Parser.java => ParserComponent.java} | 97 +----- .../org/dbsyncer/parser/ProfileComponent.java | 164 +++++---- .../org/dbsyncer/parser/command/Command.java | 14 + .../parser/command/CommandExecutor.java | 14 + .../dbsyncer/parser/command/Persistence.java | 24 ++ .../parser}/command/PersistenceCommand.java | 13 +- .../org/dbsyncer/parser/command/Preload.java | 51 +++ .../parser}/command/PreloadCommand.java | 35 +- .../parser/consumer/AbstractConsumer.java | 84 +++++ .../parser/consumer/impl/LogConsumer.java | 87 +++++ .../parser/consumer/impl/QuartzConsumer.java | 42 +++ .../dbsyncer/parser}/enums/CommandEnum.java | 11 +- .../parser}/enums/GroupStrategyEnum.java | 13 +- .../flush/impl}/BufferActuatorRouter.java | 19 +- .../flush/impl/GeneralBufferActuator.java | 12 +- .../ParserComponentImpl.java} | 124 +------ .../parser/impl/ProfileComponentImpl.java | 208 ++++++++++++ .../parser/message/MessageServiceImpl.java | 59 +--- .../dbsyncer/parser}/model/FieldPicker.java | 13 +- .../parser}/model/OperationConfig.java | 10 +- .../dbsyncer/parser}/model/QueryConfig.java | 8 +- .../parser/strategy}/GroupStrategy.java | 5 +- .../strategy/impl}/DefaultGroupStrategy.java | 7 +- .../impl}/PreloadTableGroupStrategy.java | 7 +- .../strategy/impl}/TableGroupStrategy.java | 7 +- .../parser}/template/OperationTemplate.java | 39 ++- 57 files changed, 1255 insertions(+), 1350 deletions(-) delete mode 100644 dbsyncer-manager/src/main/java/org/dbsyncer/manager/Command.java delete mode 100644 dbsyncer-manager/src/main/java/org/dbsyncer/manager/CommandExecutor.java delete mode 100644 dbsyncer-manager/src/main/java/org/dbsyncer/manager/Executor.java delete mode 100644 dbsyncer-manager/src/main/java/org/dbsyncer/manager/command/Persistence.java delete mode 100644 dbsyncer-manager/src/main/java/org/dbsyncer/manager/command/Preload.java create mode 100644 dbsyncer-manager/src/main/java/org/dbsyncer/manager/listener/ClosedTaskListener.java rename dbsyncer-parser/src/main/java/org/dbsyncer/parser/{Parser.java => ParserComponent.java} (50%) rename dbsyncer-manager/src/main/java/org/dbsyncer/manager/Manager.java => dbsyncer-parser/src/main/java/org/dbsyncer/parser/ProfileComponent.java (48%) create mode 100644 dbsyncer-parser/src/main/java/org/dbsyncer/parser/command/Command.java create mode 100644 dbsyncer-parser/src/main/java/org/dbsyncer/parser/command/CommandExecutor.java create mode 100644 dbsyncer-parser/src/main/java/org/dbsyncer/parser/command/Persistence.java rename {dbsyncer-manager/src/main/java/org/dbsyncer/manager => dbsyncer-parser/src/main/java/org/dbsyncer/parser}/command/PersistenceCommand.java (77%) create mode 100644 dbsyncer-parser/src/main/java/org/dbsyncer/parser/command/Preload.java rename {dbsyncer-manager/src/main/java/org/dbsyncer/manager => dbsyncer-parser/src/main/java/org/dbsyncer/parser}/command/PreloadCommand.java (49%) create mode 100644 dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/AbstractConsumer.java create mode 100644 dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/impl/LogConsumer.java create mode 100644 dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/impl/QuartzConsumer.java rename {dbsyncer-manager/src/main/java/org/dbsyncer/manager => dbsyncer-parser/src/main/java/org/dbsyncer/parser}/enums/CommandEnum.java (91%) rename {dbsyncer-manager/src/main/java/org/dbsyncer/manager => dbsyncer-parser/src/main/java/org/dbsyncer/parser}/enums/GroupStrategyEnum.java (63%) rename {dbsyncer-manager/src/main/java/org/dbsyncer/manager/puller => dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl}/BufferActuatorRouter.java (89%) rename dbsyncer-parser/src/main/java/org/dbsyncer/parser/{ParserFactory.java => impl/ParserComponentImpl.java} (75%) create mode 100644 dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/ProfileComponentImpl.java rename {dbsyncer-manager/src/main/java/org/dbsyncer/manager => dbsyncer-parser/src/main/java/org/dbsyncer/parser}/model/FieldPicker.java (95%) rename {dbsyncer-manager/src/main/java/org/dbsyncer/manager => dbsyncer-parser/src/main/java/org/dbsyncer/parser}/model/OperationConfig.java (84%) rename {dbsyncer-manager/src/main/java/org/dbsyncer/manager => dbsyncer-parser/src/main/java/org/dbsyncer/parser}/model/QueryConfig.java (80%) rename {dbsyncer-manager/src/main/java/org/dbsyncer/manager => dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy}/GroupStrategy.java (67%) rename {dbsyncer-manager/src/main/java/org/dbsyncer/manager/strategy => dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/impl}/DefaultGroupStrategy.java (64%) rename {dbsyncer-manager/src/main/java/org/dbsyncer/manager/strategy => dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/impl}/PreloadTableGroupStrategy.java (72%) rename {dbsyncer-manager/src/main/java/org/dbsyncer/manager/strategy => dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/impl}/TableGroupStrategy.java (74%) rename {dbsyncer-manager/src/main/java/org/dbsyncer/manager => dbsyncer-parser/src/main/java/org/dbsyncer/parser}/template/OperationTemplate.java (87%) diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/AbstractChecker.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/AbstractChecker.java index 75962ab9..5d57bbdd 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/AbstractChecker.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/AbstractChecker.java @@ -6,7 +6,6 @@ import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.JsonUtil; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.connector.model.Filter; -import org.dbsyncer.manager.Manager; import org.dbsyncer.parser.model.AbstractConfigModel; import org.dbsyncer.parser.model.ConfigModel; import org.dbsyncer.parser.model.Convert; @@ -35,9 +34,6 @@ public abstract class AbstractChecker implements Checker { private static final String SYMBOL = "***"; - @Resource - private Manager manager; - @Resource private PluginFactory pluginFactory; @@ -95,7 +91,7 @@ public abstract class AbstractChecker implements Checker { String pluginId = params.get("pluginId"); Plugin plugin = null; if (StringUtil.isNotBlank(pluginId)) { - List plugins = manager.getPluginAll(); + List plugins = pluginFactory.getPluginAll(); if (!CollectionUtils.isEmpty(plugins)) { for (Plugin p : plugins) { if (StringUtil.equals(pluginFactory.createPluginId(p.getClassName(), p.getVersion()), pluginId)) { diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/ConnectorChecker.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/ConnectorChecker.java index 03bdcf34..28f4273b 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/ConnectorChecker.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/ConnectorChecker.java @@ -6,9 +6,11 @@ import org.dbsyncer.biz.checker.ConnectorConfigChecker; import org.dbsyncer.common.model.AbstractConnectorConfig; import org.dbsyncer.common.spi.ConnectorMapper; import org.dbsyncer.common.util.StringUtil; +import org.dbsyncer.connector.ConnectorFactory; import org.dbsyncer.connector.enums.ConnectorEnum; import org.dbsyncer.connector.model.Table; -import org.dbsyncer.manager.Manager; +import org.dbsyncer.parser.ParserComponent; +import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.parser.logger.LogService; import org.dbsyncer.parser.logger.LogType; import org.dbsyncer.parser.model.ConfigModel; @@ -34,11 +36,17 @@ public class ConnectorChecker extends AbstractChecker { private final Logger logger = LoggerFactory.getLogger(getClass()); @Resource - private Manager manager; + private ParserComponent parserComponent; + + @Resource + private ProfileComponent profileComponent; @Resource private LogService logService; + @Resource + private ConnectorFactory connectorFactory; + @Resource private Map map; @@ -75,7 +83,7 @@ public class ConnectorChecker extends AbstractChecker { printParams(params); Assert.notEmpty(params, "ConnectorChecker check params is null."); String id = params.get(ConfigConstant.CONFIG_MODEL_ID); - Connector connector = manager.getConnector(id); + Connector connector = profileComponent.getConnector(id); Assert.notNull(connector, "Can not find connector."); // 修改基本配置 @@ -106,14 +114,14 @@ public class ConnectorChecker extends AbstractChecker { } private void setTable(Connector connector) { - boolean isAlive = manager.refreshConnectorConfig(connector.getConfig()); + boolean isAlive = connectorFactory.isAlive(connector.getConfig()); if (!isAlive) { logService.log(LogType.ConnectorLog.FAILED); } Assert.isTrue(isAlive, "无法连接."); // 获取表信息 - ConnectorMapper connectorMapper = manager.connect(connector.getConfig()); - List table = manager.getTable(connectorMapper); + ConnectorMapper connectorMapper = connectorFactory.connect(connector.getConfig()); + List
table = parserComponent.getTable(connectorMapper); connector.setTable(table); } diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/OracleConfigChecker.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/OracleConfigChecker.java index 18b8c4b5..944fa695 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/OracleConfigChecker.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/OracleConfigChecker.java @@ -7,14 +7,15 @@ import org.dbsyncer.connector.config.DatabaseConfig; import org.dbsyncer.connector.model.Field; import org.dbsyncer.connector.model.MetaInfo; import org.dbsyncer.connector.model.Table; -import org.dbsyncer.manager.Manager; +import org.dbsyncer.parser.ParserComponent; +import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.parser.model.FieldMapping; import org.dbsyncer.parser.model.Mapping; import org.dbsyncer.parser.model.TableGroup; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.springframework.util.Assert; +import javax.annotation.Resource; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -32,8 +33,11 @@ import java.util.Map; @Component public class OracleConfigChecker extends AbstractDataBaseConfigChecker { - @Autowired - private Manager manager; + @Resource + private ParserComponent parserComponent; + + @Resource + private ProfileComponent profileComponent; @Override public void modify(DatabaseConfig connectorConfig, Map params) { @@ -139,7 +143,7 @@ public class OracleConfigChecker extends AbstractDataBaseConfigChecker { // 没有自定义主键，获取表元信息 if (null == pk) { Table targetTable = tableGroup.getTargetTable(); - MetaInfo metaInfo = manager.getMetaInfo(mapping.getTargetConnectorId(), targetTable.getName()); + MetaInfo metaInfo = parserComponent.getMetaInfo(mapping.getTargetConnectorId(), targetTable.getName()); List targetColumn = metaInfo.getColumn(); targetTable.setColumn(targetColumn); diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/group/ProjectGroupChecker.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/group/ProjectGroupChecker.java index b08c7c46..01bb68cf 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/group/ProjectGroupChecker.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/group/ProjectGroupChecker.java @@ -3,11 +3,10 @@ package org.dbsyncer.biz.checker.impl.group; import org.dbsyncer.biz.checker.AbstractChecker; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.manager.Manager; +import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.parser.model.ConfigModel; import org.dbsyncer.parser.model.ProjectGroup; import org.dbsyncer.storage.constant.ConfigConstant; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.springframework.util.Assert; @@ -25,7 +24,7 @@ import java.util.Map; public class ProjectGroupChecker extends AbstractChecker { @Resource - private Manager manager; + private ProfileComponent profileComponent; /** * 新增配置 @@ -56,7 +55,7 @@ public class ProjectGroupChecker extends AbstractChecker { @Override public ConfigModel checkEditConfigModel(Map params) { String id = params.get(ConfigConstant.CONFIG_MODEL_ID); - ProjectGroup projectGroup = manager.getProjectGroup(id); + ProjectGroup projectGroup = profileComponent.getProjectGroup(id); Assert.notNull(projectGroup, "Can not find project group."); modifyProjectGroup(projectGroup, params); diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/mapping/MappingChecker.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/mapping/MappingChecker.java index 5692b611..bbf6f2c5 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/mapping/MappingChecker.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/mapping/MappingChecker.java @@ -10,7 +10,7 @@ import org.dbsyncer.common.util.NumberUtil; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.listener.config.ListenerConfig; import org.dbsyncer.listener.enums.ListenerTypeEnum; -import org.dbsyncer.manager.Manager; +import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.parser.enums.ModelEnum; import org.dbsyncer.parser.model.ConfigModel; import org.dbsyncer.parser.model.Mapping; @@ -41,7 +41,7 @@ public class MappingChecker extends AbstractChecker { private final Logger logger = LoggerFactory.getLogger(getClass()); @Resource - private Manager manager; + private ProfileComponent profileComponent; @Resource private TableGroupChecker tableGroupChecker; @@ -81,7 +81,7 @@ public class MappingChecker extends AbstractChecker { logger.info("params:{}", params); Assert.notEmpty(params, "MappingChecker check params is null."); String id = params.get(ConfigConstant.CONFIG_MODEL_ID); - Mapping mapping = manager.getMapping(id); + Mapping mapping = profileComponent.getMapping(id); Assert.notNull(mapping, "Can not find mapping."); // 修改基本配置 @@ -124,7 +124,7 @@ public class MappingChecker extends AbstractChecker { // 修改基本配置 this.modifyConfigModel(meta, new HashMap<>()); - String id = manager.addConfigModel(meta); + String id = profileComponent.addConfigModel(meta); mapping.setMetaId(id); } @@ -144,7 +144,7 @@ public class MappingChecker extends AbstractChecker { * @param params */ public void batchMergeTableGroupConfig(Mapping mapping, Map params) { - List groupAll = manager.getTableGroupAll(mapping.getId()); + List groupAll = profileComponent.getTableGroupAll(mapping.getId()); if (!CollectionUtils.isEmpty(groupAll)) { // 手动排序 String[] sortedTableGroupIds = StringUtil.split(params.get("sortedTableGroupIds"), "|"); @@ -153,7 +153,7 @@ public class MappingChecker extends AbstractChecker { groupAll.clear(); int size = sortedTableGroupIds.length; int i = size; - while (i > 0){ + while (i > 0) { TableGroup g = tableGroupMap.get(sortedTableGroupIds[size - i]); Assert.notNull(g, "Invalid sorted tableGroup."); g.setIndex(i); @@ -165,7 +165,7 @@ public class MappingChecker extends AbstractChecker { // 合并配置 for (TableGroup g : groupAll) { tableGroupChecker.mergeConfig(mapping, g); - manager.editConfigModel(g); + profileComponent.editConfigModel(g); } } } @@ -194,16 +194,16 @@ public class MappingChecker extends AbstractChecker { * @param metaSnapshot */ private void updateMeta(Mapping mapping, String metaSnapshot) { - Meta meta = manager.getMeta(mapping.getMetaId()); + Meta meta = profileComponent.getMeta(mapping.getMetaId()); Assert.notNull(meta, "驱动meta不存在."); // 清空状态 meta.clear(); // 手动配置增量点 - if(StringUtil.isNotBlank(metaSnapshot)){ + if (StringUtil.isNotBlank(metaSnapshot)) { Map snapshot = JsonUtil.jsonToObj(metaSnapshot, HashMap.class); - if(!CollectionUtils.isEmpty(snapshot)){ + if (!CollectionUtils.isEmpty(snapshot)) { meta.setSnapshot(snapshot); } } @@ -211,7 +211,7 @@ public class MappingChecker extends AbstractChecker { getMetaTotal(meta, mapping.getModel()); meta.setUpdateTime(Instant.now().toEpochMilli()); - manager.editConfigModel(meta); + profileComponent.editConfigModel(meta); } private void getMetaTotal(Meta meta, String model) { @@ -219,7 +219,7 @@ public class MappingChecker extends AbstractChecker { if (ModelEnum.isFull(model)) { // 统计tableGroup总条数 AtomicLong count = new AtomicLong(0); - List groupAll = manager.getTableGroupAll(meta.getMappingId()); + List groupAll = profileComponent.getTableGroupAll(meta.getMappingId()); if (!CollectionUtils.isEmpty(groupAll)) { for (TableGroup g : groupAll) { count.getAndAdd(g.getSourceTable().getCount()); diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/system/SystemConfigChecker.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/system/SystemConfigChecker.java index f713b437..d318f50e 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/system/SystemConfigChecker.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/system/SystemConfigChecker.java @@ -3,7 +3,7 @@ package org.dbsyncer.biz.checker.impl.system; import org.dbsyncer.biz.checker.AbstractChecker; import org.dbsyncer.common.util.BeanUtil; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.manager.Manager; +import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.parser.logger.LogService; import org.dbsyncer.parser.logger.LogType; import org.dbsyncer.parser.model.ConfigModel; @@ -27,7 +27,7 @@ public class SystemConfigChecker extends AbstractChecker { private final Logger logger = LoggerFactory.getLogger(getClass()); @Resource - private Manager manager; + private ProfileComponent profileComponent; @Resource private LogService logService; @@ -40,7 +40,7 @@ public class SystemConfigChecker extends AbstractChecker { // 修改基本配置 this.modifyConfigModel(systemConfig, params); - manager.addConfigModel(systemConfig); + profileComponent.addConfigModel(systemConfig); return systemConfig; } @@ -50,7 +50,7 @@ public class SystemConfigChecker extends AbstractChecker { Assert.notEmpty(params, "Config check params is null."); params.put("enableCDN", StringUtil.isNotBlank(params.get("enableCDN")) ? "true" : "false"); - SystemConfig systemConfig = manager.getSystemConfig(); + SystemConfig systemConfig = profileComponent.getSystemConfig(); Assert.notNull(systemConfig, "配置文件为空."); BeanUtil.mapToBean(params, systemConfig); logService.log(LogType.SystemLog.INFO, "修改系统配置"); diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/tablegroup/TableGroupChecker.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/tablegroup/TableGroupChecker.java index 6d4db650..5b45072f 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/tablegroup/TableGroupChecker.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/tablegroup/TableGroupChecker.java @@ -10,7 +10,8 @@ import org.dbsyncer.connector.model.Field; import org.dbsyncer.connector.model.MetaInfo; import org.dbsyncer.connector.model.Table; import org.dbsyncer.connector.util.PrimaryKeyUtil; -import org.dbsyncer.manager.Manager; +import org.dbsyncer.parser.ParserComponent; +import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.parser.enums.ModelEnum; import org.dbsyncer.parser.model.ConfigModel; import org.dbsyncer.parser.model.FieldMapping; @@ -45,7 +46,10 @@ public class TableGroupChecker extends AbstractChecker { private final Logger logger = LoggerFactory.getLogger(getClass()); @Resource - private Manager manager; + private ParserComponent parserComponent; + + @Resource + private ProfileComponent profileComponent; @Resource private Map map; @@ -61,7 +65,7 @@ public class TableGroupChecker extends AbstractChecker { Assert.hasText(mappingId, "tableGroup mappingId is empty."); Assert.hasText(sourceTable, "tableGroup sourceTable is empty."); Assert.hasText(targetTable, "tableGroup targetTable is empty."); - Mapping mapping = manager.getMapping(mappingId); + Mapping mapping = profileComponent.getMapping(mappingId); Assert.notNull(mapping, "mapping can not be null."); // 检查是否存在重复映射关系 @@ -92,9 +96,9 @@ public class TableGroupChecker extends AbstractChecker { logger.info("params:{}", params); Assert.notEmpty(params, "TableGroupChecker check params is null."); String id = params.get(ConfigConstant.CONFIG_MODEL_ID); - TableGroup tableGroup = manager.getTableGroup(id); + TableGroup tableGroup = profileComponent.getTableGroup(id); Assert.notNull(tableGroup, "Can not find tableGroup."); - Mapping mapping = manager.getMapping(tableGroup.getMappingId()); + Mapping mapping = profileComponent.getMapping(tableGroup.getMappingId()); Assert.notNull(mapping, "mapping can not be null."); String fieldMappingJson = params.get("fieldMapping"); Assert.hasText(fieldMappingJson, "TableGroupChecker check params fieldMapping is empty"); @@ -120,7 +124,7 @@ public class TableGroupChecker extends AbstractChecker { * @param tableGroup */ public void refreshTableFields(TableGroup tableGroup) { - Mapping mapping = manager.getMapping(tableGroup.getMappingId()); + Mapping mapping = profileComponent.getMapping(tableGroup.getMappingId()); Assert.notNull(mapping, "mapping can not be null."); Table sourceTable = tableGroup.getSourceTable(); @@ -138,16 +142,16 @@ public class TableGroupChecker extends AbstractChecker { // 处理策略 dealIncrementStrategy(mapping, group); - Map command = manager.getCommand(mapping, group); + Map command = parserComponent.getCommand(mapping, group); tableGroup.setCommand(command); // 获取数据源总数 - long count = ModelEnum.isFull(mapping.getModel()) && !CollectionUtils.isEmpty(command) ? manager.getCount(mapping.getSourceConnectorId(), command) : 0; + long count = ModelEnum.isFull(mapping.getModel()) && !CollectionUtils.isEmpty(command) ? parserComponent.getCount(mapping.getSourceConnectorId(), command) : 0; tableGroup.getSourceTable().setCount(count); } public void dealIncrementStrategy(Mapping mapping, TableGroup tableGroup) { - String connectorType = manager.getConnector(mapping.getSourceConnectorId()).getConfig().getConnectorType(); + String connectorType = profileComponent.getConnector(mapping.getSourceConnectorId()).getConfig().getConnectorType(); String type = StringUtil.toLowerCaseFirstOne(connectorType).concat("ConfigChecker"); ConnectorConfigChecker checker = map.get(type); Assert.notNull(checker, "Checker can not be null."); @@ -155,7 +159,7 @@ public class TableGroupChecker extends AbstractChecker { } private Table getTable(String connectorId, String tableName, String primaryKeyStr) { - MetaInfo metaInfo = manager.getMetaInfo(connectorId, tableName); + MetaInfo metaInfo = parserComponent.getMetaInfo(connectorId, tableName); Assert.notNull(metaInfo, "无法获取连接器表信息:" + tableName); // 自定义主键 if (StringUtil.isNotBlank(primaryKeyStr) && !CollectionUtils.isEmpty(metaInfo.getColumn())) { @@ -173,7 +177,7 @@ public class TableGroupChecker extends AbstractChecker { } private void checkRepeatedTable(String mappingId, String sourceTable, String targetTable) { - List list = manager.getTableGroupAll(mappingId); + List list = profileComponent.getTableGroupAll(mappingId); if (!CollectionUtils.isEmpty(list)) { for (TableGroup g : list) { // 数据源表和目标表都存在 diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/BaseServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/BaseServiceImpl.java index 25e8fb75..fb6fca31 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/BaseServiceImpl.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/BaseServiceImpl.java @@ -1,6 +1,6 @@ package org.dbsyncer.biz.impl; -import org.dbsyncer.manager.Manager; +import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.parser.enums.MetaEnum; import org.dbsyncer.parser.enums.ModelEnum; import org.dbsyncer.parser.logger.LogService; @@ -13,12 +13,14 @@ import org.dbsyncer.parser.model.TableGroup; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.util.Assert; +import javax.annotation.Resource; + public class BaseServiceImpl { - @Autowired - protected Manager manager; + @Resource + private ProfileComponent profileComponent; - @Autowired + @Resource private LogService logService; @Autowired @@ -30,7 +32,7 @@ public class BaseServiceImpl { protected final static Object LOCK = new Object(); protected boolean isRunning(String metaId) { - Meta meta = manager.getMeta(metaId); + Meta meta = profileComponent.getMeta(metaId); if (null != meta) { int state = meta.getState(); return MetaEnum.isRunning(state); @@ -66,7 +68,7 @@ public class BaseServiceImpl { protected void log(LogType log, TableGroup tableGroup) { if (null != tableGroup) { - Mapping mapping = manager.getMapping(tableGroup.getMappingId()); + Mapping mapping = profileComponent.getMapping(tableGroup.getMappingId()); if (null != mapping) { // 新增驱动知识库(全量)映射关系:[My_User] >> [My_User_Target] String name = mapping.getName(); diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ConditionServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ConditionServiceImpl.java index 10e96ea1..cd94dd36 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ConditionServiceImpl.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ConditionServiceImpl.java @@ -5,10 +5,10 @@ import org.dbsyncer.biz.vo.ConditionVo; import org.dbsyncer.connector.enums.FilterEnum; import org.dbsyncer.connector.enums.OperationEnum; import org.dbsyncer.listener.enums.QuartzFilterEnum; -import org.dbsyncer.manager.Manager; -import org.springframework.beans.factory.annotation.Autowired; +import org.dbsyncer.parser.ProfileComponent; import org.springframework.stereotype.Component; +import javax.annotation.Resource; import java.util.List; /** @@ -19,14 +19,14 @@ import java.util.List; @Component public class ConditionServiceImpl implements ConditionService { - @Autowired - private Manager manager; + @Resource + private ProfileComponent profileComponent; @Override public ConditionVo getCondition() { - List operationEnumAll = manager.getOperationEnumAll(); - List quartzFilterEnumAll = manager.getQuartzFilterEnumAll(); - List filterEnumAll = manager.getFilterEnumAll(); + List operationEnumAll = profileComponent.getOperationEnumAll(); + List quartzFilterEnumAll = profileComponent.getQuartzFilterEnumAll(); + List filterEnumAll = profileComponent.getFilterEnumAll(); return new ConditionVo(operationEnumAll, quartzFilterEnumAll, filterEnumAll); } } \ No newline at end of file diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ConnectorServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ConnectorServiceImpl.java index 859fec7f..a9309558 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ConnectorServiceImpl.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ConnectorServiceImpl.java @@ -3,10 +3,13 @@ package org.dbsyncer.biz.impl; import org.dbsyncer.biz.BizException; import org.dbsyncer.biz.ConnectorService; import org.dbsyncer.biz.checker.Checker; +import org.dbsyncer.common.model.AbstractConnectorConfig; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.JsonUtil; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.manager.Manager; +import org.dbsyncer.connector.ConnectorFactory; +import org.dbsyncer.parser.ProfileComponent; +import org.dbsyncer.parser.logger.LogService; import org.dbsyncer.parser.logger.LogType; import org.dbsyncer.parser.model.ConfigModel; import org.dbsyncer.parser.model.Connector; @@ -14,10 +17,10 @@ import org.dbsyncer.parser.model.Mapping; import org.dbsyncer.storage.constant.ConfigConstant; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.util.Assert; +import javax.annotation.Resource; import java.util.ArrayList; import java.util.Comparator; import java.util.HashSet; @@ -39,10 +42,16 @@ public class ConnectorServiceImpl extends BaseServiceImpl implements ConnectorSe private Map health = new LinkedHashMap<>(); - @Autowired - private Manager manager; + @Resource + private ProfileComponent profileComponent; - @Autowired + @Resource + private ConnectorFactory connectorFactory; + + @Resource + private LogService logService; + + @Resource private Checker connectorChecker; @Override @@ -50,7 +59,7 @@ public class ConnectorServiceImpl extends BaseServiceImpl implements ConnectorSe ConfigModel model = connectorChecker.checkAddConfigModel(params); log(LogType.ConnectorLog.INSERT, model); - return manager.addConfigModel(model); + return profileComponent.addConfigModel(model); } @Override @@ -62,7 +71,7 @@ public class ConnectorServiceImpl extends BaseServiceImpl implements ConnectorSe params.put(ConfigConstant.CONFIG_MODEL_NAME, connector.getName() + "(复制)"); ConfigModel model = connectorChecker.checkAddConfigModel(params); log(LogType.ConnectorLog.COPY, model); - manager.addConfigModel(model); + profileComponent.addConfigModel(model); return String.format("复制成功[%s]", model.getName()); } @@ -72,12 +81,12 @@ public class ConnectorServiceImpl extends BaseServiceImpl implements ConnectorSe ConfigModel model = connectorChecker.checkEditConfigModel(params); log(LogType.ConnectorLog.UPDATE, model); - return manager.editConfigModel(model); + return profileComponent.editConfigModel(model); } @Override public String remove(String id) { - List mappingAll = manager.getMappingAll(); + List mappingAll = profileComponent.getMappingAll(); if (!CollectionUtils.isEmpty(mappingAll)) { mappingAll.forEach(mapping -> { if (StringUtil.equals(mapping.getSourceConnectorId(), id) || StringUtil.equals(mapping.getTargetConnectorId(), id)) { @@ -88,21 +97,21 @@ public class ConnectorServiceImpl extends BaseServiceImpl implements ConnectorSe }); } - Connector connector = manager.getConnector(id); + Connector connector = profileComponent.getConnector(id); log(LogType.ConnectorLog.DELETE, connector); - manager.removeConfigModel(id); + profileComponent.removeConfigModel(id); return "删除连接器成功!"; } @Override public Connector getConnector(String id) { - return StringUtil.isNotBlank(id) ? manager.getConnector(id) : null; + return StringUtil.isNotBlank(id) ? profileComponent.getConnector(id) : null; } @Override public List getConnectorAll() { - List list = manager.getConnectorAll() + List list = profileComponent.getConnectorAll() .stream() .sorted(Comparator.comparing(Connector::getUpdateTime).reversed()) .collect(Collectors.toList()); @@ -112,13 +121,13 @@ public class ConnectorServiceImpl extends BaseServiceImpl implements ConnectorSe @Override public List getConnectorTypeAll() { List list = new ArrayList<>(); - manager.getConnectorEnumAll().forEach(c -> list.add(c.getType())); + profileComponent.getConnectorEnumAll().forEach(c -> list.add(c.getType())); return list; } @Override public void refreshHealth() { - List list = manager.getConnectorAll(); + List list = profileComponent.getConnectorAll(); if (CollectionUtils.isEmpty(list)) { if (!CollectionUtils.isEmpty(health)) { health.clear(); @@ -129,7 +138,7 @@ public class ConnectorServiceImpl extends BaseServiceImpl implements ConnectorSe // 更新连接器状态 Set exist = new HashSet<>(); list.forEach(c -> { - health.put(c.getId(), manager.isAliveConnectorConfig(c.getConfig())); + health.put(c.getId(), isAliveConnectorConfig(c.getConfig())); exist.add(c.getId()); }); @@ -150,4 +159,27 @@ public class ConnectorServiceImpl extends BaseServiceImpl implements ConnectorSe public boolean isAlive(String id) { return health.containsKey(id) && health.get(id); } + + private boolean isAliveConnectorConfig(AbstractConnectorConfig config) { + boolean alive = false; + try { + alive = connectorFactory.isAlive(config); + } catch (Exception e) { + LogType.ConnectorLog logType = LogType.ConnectorLog.FAILED; + logService.log(logType, "%s%s", logType.getName(), e.getMessage()); + } + // 断线重连 + if (!alive) { + try { + alive = connectorFactory.refresh(config); + } catch (Exception e) { + logger.error(e.getMessage()); + } + if (alive) { + logger.info(LogType.ConnectorLog.RECONNECT_SUCCESS.getMessage()); + } + } + return alive; + } + } \ No newline at end of file diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ConvertServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ConvertServiceImpl.java index 64145306..11d5e4ec 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ConvertServiceImpl.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ConvertServiceImpl.java @@ -1,11 +1,11 @@ package org.dbsyncer.biz.impl; import org.dbsyncer.biz.ConvertService; -import org.dbsyncer.manager.Manager; +import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.parser.enums.ConvertEnum; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import javax.annotation.Resource; import java.util.List; /** @@ -16,12 +16,12 @@ import java.util.List; @Component public class ConvertServiceImpl implements ConvertService { - @Autowired - private Manager manager; + @Resource + private ProfileComponent profileComponent; @Override public List getConvertEnumAll() { - return manager.getConvertEnumAll(); + return profileComponent.getConvertEnumAll(); } } \ No newline at end of file diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/DataSyncServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/DataSyncServiceImpl.java index 2b166506..57af4fbc 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/DataSyncServiceImpl.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/DataSyncServiceImpl.java @@ -5,20 +5,24 @@ import org.dbsyncer.biz.DataSyncService; import org.dbsyncer.biz.vo.BinlogColumnVo; import org.dbsyncer.biz.vo.MessageVo; import org.dbsyncer.common.event.RowChangedEvent; +import org.dbsyncer.common.model.Paging; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.DateFormatUtil; import org.dbsyncer.common.util.JsonUtil; import org.dbsyncer.common.util.NumberUtil; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.connector.model.Field; -import org.dbsyncer.manager.Manager; -import org.dbsyncer.monitor.Monitor; -import org.dbsyncer.parser.Parser; +import org.dbsyncer.parser.ParserComponent; +import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.parser.model.Meta; import org.dbsyncer.parser.model.Picker; import org.dbsyncer.parser.model.TableGroup; +import org.dbsyncer.storage.StorageService; import org.dbsyncer.storage.binlog.proto.BinlogMap; import org.dbsyncer.storage.constant.ConfigConstant; +import org.dbsyncer.storage.enums.IndexFieldResolverEnum; +import org.dbsyncer.storage.enums.StorageEnum; +import org.dbsyncer.storage.query.Query; import org.dbsyncer.storage.util.BinlogMessageUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,6 +35,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -48,13 +53,13 @@ public class DataSyncServiceImpl implements DataSyncService { private final Logger logger = LoggerFactory.getLogger(getClass()); @Resource - private Monitor monitor; + private ParserComponent parserComponent; @Resource - private Manager manager; + private ProfileComponent profileComponent; @Resource - private Parser parser; + private StorageService storageService; @Override public MessageVo getMessageVo(String metaId, String messageId) { @@ -63,10 +68,10 @@ public class DataSyncServiceImpl implements DataSyncService { MessageVo messageVo = new MessageVo(); try { - Map row = monitor.getData(metaId, messageId); + Map row = getData(metaId, messageId); Map binlogData = getBinlogData(row, true); String tableGroupId = (String) row.get(ConfigConstant.DATA_TABLE_GROUP_ID); - TableGroup tableGroup = monitor.getTableGroup(tableGroupId); + TableGroup tableGroup = profileComponent.getTableGroup(tableGroupId); messageVo.setSourceTableName(tableGroup.getSourceTable().getName()); messageVo.setTargetTableName(tableGroup.getTargetTable().getName()); messageVo.setId(messageId); @@ -87,7 +92,7 @@ public class DataSyncServiceImpl implements DataSyncService { public Map getBinlogData(Map row, boolean prettyBytes) throws InvalidProtocolBufferException { String tableGroupId = (String) row.get(ConfigConstant.DATA_TABLE_GROUP_ID); // 1、获取配置信息 - final TableGroup tableGroup = manager.getTableGroup(tableGroupId); + final TableGroup tableGroup = profileComponent.getTableGroup(tableGroupId); if (tableGroup == null) { return Collections.EMPTY_MAP; } @@ -141,7 +146,7 @@ public class DataSyncServiceImpl implements DataSyncService { Assert.hasText(messageId, "The messageId is null."); try { - Map row = monitor.getData(metaId, messageId); + Map row = getData(metaId, messageId); Map binlogData = getBinlogData(row, false); if (CollectionUtils.isEmpty(binlogData)) { return messageId; @@ -153,26 +158,42 @@ public class DataSyncServiceImpl implements DataSyncService { if (StringUtil.isNotBlank(retryDataParams)) { JsonUtil.parseMap(retryDataParams).forEach((k, v) -> binlogData.put(k, convertValue(binlogData.get(k), (String) v))); } - TableGroup tableGroup = manager.getTableGroup(tableGroupId); + TableGroup tableGroup = profileComponent.getTableGroup(tableGroupId); String sourceTableName = tableGroup.getSourceTable().getName(); RowChangedEvent changedEvent = new RowChangedEvent(sourceTableName, event, Collections.EMPTY_LIST); // 转换为源字段 final Picker picker = new Picker(tableGroup.getFieldMapping()); changedEvent.setChangedRow(picker.pickSourceData(binlogData)); - parser.execute(tableGroupId, changedEvent); - monitor.removeData(metaId, messageId); + parserComponent.execute(tableGroupId, changedEvent); + storageService.remove(StorageEnum.DATA, metaId, messageId); // 更新失败数 - Meta meta = manager.getMeta(metaId); + Meta meta = profileComponent.getMeta(metaId); Assert.notNull(meta, "Meta can not be null."); meta.getFail().decrementAndGet(); meta.setUpdateTime(Instant.now().toEpochMilli()); - manager.editConfigModel(meta); + profileComponent.editConfigModel(meta); } catch (Exception e) { logger.error(e.getLocalizedMessage()); } return messageId; } + private Map getData(String metaId, String messageId) { + Query query = new Query(1, 1); + Map fieldResolvers = new LinkedHashMap<>(); + fieldResolvers.put(ConfigConstant.BINLOG_DATA, IndexFieldResolverEnum.BINARY); + query.setIndexFieldResolverMap(fieldResolvers); + query.addFilter(ConfigConstant.CONFIG_MODEL_ID, messageId); + query.setMetaId(metaId); + query.setType(StorageEnum.DATA); + Paging paging = storageService.query(query); + if (!CollectionUtils.isEmpty(paging.getData())) { + List data = (List) paging.getData(); + return data.get(0); + } + return Collections.EMPTY_MAP; + } + private Object convertValue(Object oldValue, String newValue) { if (oldValue == null) { return newValue; diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MappingServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MappingServiceImpl.java index b3be7f51..d448dac3 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MappingServiceImpl.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MappingServiceImpl.java @@ -3,6 +3,7 @@ package org.dbsyncer.biz.impl; import org.dbsyncer.biz.BizException; import org.dbsyncer.biz.ConnectorService; import org.dbsyncer.biz.MappingService; +import org.dbsyncer.biz.MonitorService; import org.dbsyncer.biz.TableGroupService; import org.dbsyncer.biz.checker.impl.mapping.MappingChecker; import org.dbsyncer.biz.vo.ConnectorVo; @@ -13,6 +14,8 @@ import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.JsonUtil; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.connector.model.Table; +import org.dbsyncer.manager.ManagerFactory; +import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.parser.enums.ModelEnum; import org.dbsyncer.parser.logger.LogType; import org.dbsyncer.parser.model.ConfigModel; @@ -22,10 +25,10 @@ import org.dbsyncer.parser.model.Meta; import org.dbsyncer.parser.model.TableGroup; import org.dbsyncer.storage.constant.ConfigConstant; import org.springframework.beans.BeanUtils; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.util.Assert; +import javax.annotation.Resource; import java.time.Instant; import java.util.Collections; import java.util.Comparator; @@ -42,24 +45,33 @@ import java.util.stream.Collectors; @Service public class MappingServiceImpl extends BaseServiceImpl implements MappingService { - @Autowired + @Resource private MappingChecker mappingChecker; - @Autowired + @Resource private TableGroupService tableGroupService; - @Autowired + @Resource private ConnectorService connectorService; - @Autowired + @Resource private SnowflakeIdWorker snowflakeIdWorker; + @Resource + private ProfileComponent profileComponent; + + @Resource + private MonitorService monitorService; + + @Resource + private ManagerFactory managerFactory; + @Override public String add(Map params) { ConfigModel model = mappingChecker.checkAddConfigModel(params); log(LogType.MappingLog.INSERT, (Mapping) model); - String id = manager.addConfigModel(model); + String id = profileComponent.addConfigModel(model); // 匹配相似表 on String autoMatchTable = params.get("autoMatchTable"); @@ -72,7 +84,7 @@ public class MappingServiceImpl extends BaseServiceImpl implements MappingServic @Override public String copy(String id) { - Mapping mapping = manager.getMapping(id); + Mapping mapping = profileComponent.getMapping(id); Assert.notNull(mapping, "The mapping id is invalid."); String json = JsonUtil.objToJson(mapping); @@ -82,18 +94,18 @@ public class MappingServiceImpl extends BaseServiceImpl implements MappingServic newMapping.setUpdateTime(Instant.now().toEpochMilli()); mappingChecker.addMeta(newMapping); - manager.addConfigModel(newMapping); + profileComponent.addConfigModel(newMapping); log(LogType.MappingLog.COPY, newMapping); // 复制映射表关系 - List groupList = manager.getTableGroupAll(mapping.getId()); - if(!CollectionUtils.isEmpty(groupList)){ + List groupList = profileComponent.getTableGroupAll(mapping.getId()); + if (!CollectionUtils.isEmpty(groupList)) { groupList.forEach(tableGroup -> { String tableGroupJson = JsonUtil.objToJson(tableGroup); TableGroup newTableGroup = JsonUtil.jsonToObj(tableGroupJson, TableGroup.class); newTableGroup.setId(String.valueOf(snowflakeIdWorker.nextId())); newTableGroup.setMappingId(newMapping.getId()); - manager.addTableGroup(newTableGroup); + profileComponent.addTableGroup(newTableGroup); log(LogType.TableGroupLog.COPY, newTableGroup); }); } @@ -110,7 +122,7 @@ public class MappingServiceImpl extends BaseServiceImpl implements MappingServic log(LogType.MappingLog.UPDATE, model); mappingChecker.batchMergeTableGroupConfig(model, params); - return manager.editConfigModel(model); + return profileComponent.editConfigModel(model); } } @@ -118,26 +130,26 @@ public class MappingServiceImpl extends BaseServiceImpl implements MappingServic public String remove(String id) { Mapping mapping = assertMappingExist(id); String metaId = mapping.getMetaId(); - Meta meta = manager.getMeta(metaId); + Meta meta = profileComponent.getMeta(metaId); synchronized (LOCK) { assertRunning(metaId); // 删除数据 - manager.clearData(metaId); + monitorService.clearData(metaId); log(LogType.MetaLog.CLEAR, meta); // 删除meta - manager.removeConfigModel(metaId); + profileComponent.removeConfigModel(metaId); log(LogType.MetaLog.DELETE, meta); // 删除tableGroup - List groupList = manager.getTableGroupAll(id); + List groupList = profileComponent.getTableGroupAll(id); if (!CollectionUtils.isEmpty(groupList)) { - groupList.forEach(t -> manager.removeTableGroup(t.getId())); + groupList.forEach(t -> profileComponent.removeTableGroup(t.getId())); } // 删除驱动 - manager.removeConfigModel(id); + profileComponent.removeConfigModel(id); log(LogType.MappingLog.DELETE, mapping); } return "驱动删除成功"; @@ -145,13 +157,13 @@ public class MappingServiceImpl extends BaseServiceImpl implements MappingServic @Override public MappingVo getMapping(String id) { - Mapping mapping = manager.getMapping(id); + Mapping mapping = profileComponent.getMapping(id); return convertMapping2Vo(mapping); } @Override public List getMappingAll() { - List list = manager.getMappingAll() + List list = profileComponent.getMappingAll() .stream() .map(m -> convertMapping2Vo(m)) .sorted(Comparator.comparing(MappingVo::getUpdateTime).reversed()) @@ -170,7 +182,7 @@ public class MappingServiceImpl extends BaseServiceImpl implements MappingServic assertRunning(metaId); // 启动 - manager.start(mapping); + managerFactory.start(mapping); log(LogType.MappingLog.RUNNING, mapping); } @@ -184,7 +196,7 @@ public class MappingServiceImpl extends BaseServiceImpl implements MappingServic if (!isRunning(mapping.getMetaId())) { throw new BizException("驱动已停止."); } - manager.close(mapping); + managerFactory.close(mapping); log(LogType.MappingLog.STOP, mapping); @@ -198,8 +210,8 @@ public class MappingServiceImpl extends BaseServiceImpl implements MappingServic private MappingVo convertMapping2Vo(Mapping mapping) { String model = mapping.getModel(); Assert.notNull(mapping, "Mapping can not be null."); - Connector s = manager.getConnector(mapping.getSourceConnectorId()); - Connector t = manager.getConnector(mapping.getTargetConnectorId()); + Connector s = profileComponent.getConnector(mapping.getSourceConnectorId()); + Connector t = profileComponent.getConnector(mapping.getTargetConnectorId()); ConnectorVo sConn = new ConnectorVo(connectorService.isAlive(s.getId())); BeanUtils.copyProperties(s, sConn); ConnectorVo tConn = new ConnectorVo(connectorService.isAlive(t.getId())); @@ -210,7 +222,7 @@ public class MappingServiceImpl extends BaseServiceImpl implements MappingServic Collections.sort(tConn.getTable(), Comparator.comparing(Table::getName)); // 元信息 - Meta meta = manager.getMeta(mapping.getMetaId()); + Meta meta = profileComponent.getMeta(mapping.getMetaId()); Assert.notNull(meta, "Meta can not be null."); MetaVo metaVo = new MetaVo(ModelEnum.getModelEnum(model).getName(), mapping.getName()); BeanUtils.copyProperties(meta, metaVo); @@ -227,7 +239,7 @@ public class MappingServiceImpl extends BaseServiceImpl implements MappingServic * @return */ private Mapping assertMappingExist(String mappingId) { - Mapping mapping = manager.getMapping(mappingId); + Mapping mapping = profileComponent.getMapping(mappingId); Assert.notNull(mapping, "驱动不存在."); return mapping; } @@ -239,8 +251,8 @@ public class MappingServiceImpl extends BaseServiceImpl implements MappingServic */ private void matchSimilarTable(ConfigModel model) { Mapping mapping = (Mapping) model; - Connector s = manager.getConnector(mapping.getSourceConnectorId()); - Connector t = manager.getConnector(mapping.getTargetConnectorId()); + Connector s = profileComponent.getConnector(mapping.getSourceConnectorId()); + Connector t = profileComponent.getConnector(mapping.getTargetConnectorId()); if (CollectionUtils.isEmpty(s.getTable()) || CollectionUtils.isEmpty(t.getTable())) { return; } @@ -262,13 +274,13 @@ public class MappingServiceImpl extends BaseServiceImpl implements MappingServic } private void clearMetaIfFinished(String metaId) { - Meta meta = manager.getMeta(metaId); + Meta meta = profileComponent.getMeta(metaId); Assert.notNull(meta, "Mapping meta can not be null."); // 完成任务则重置状态 if (meta.getTotal().get() <= (meta.getSuccess().get() + meta.getFail().get())) { meta.getFail().set(0); meta.getSuccess().set(0); - manager.editConfigModel(meta); + profileComponent.editConfigModel(meta); } } diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java index 3497762b..9d963c39 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java @@ -30,12 +30,16 @@ import org.dbsyncer.monitor.enums.DiskMetricEnum; import org.dbsyncer.monitor.enums.MetricEnum; import org.dbsyncer.monitor.model.AppReportMetric; import org.dbsyncer.monitor.model.MetricResponse; +import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.parser.enums.MetaEnum; import org.dbsyncer.parser.enums.ModelEnum; +import org.dbsyncer.parser.logger.LogService; +import org.dbsyncer.parser.logger.LogType; import org.dbsyncer.parser.model.Mapping; import org.dbsyncer.parser.model.Meta; import org.dbsyncer.storage.StorageService; import org.dbsyncer.storage.constant.ConfigConstant; +import org.dbsyncer.storage.enums.IndexFieldResolverEnum; import org.dbsyncer.storage.enums.StorageDataStatusEnum; import org.dbsyncer.storage.enums.StorageEnum; import org.dbsyncer.storage.query.BooleanFilter; @@ -72,6 +76,9 @@ public class MonitorServiceImpl extends BaseServiceImpl implements MonitorServic @Resource private Monitor monitor; + @Resource + private ProfileComponent profileComponent; + @Resource private DataSyncService dataSyncService; @@ -81,6 +88,9 @@ public class MonitorServiceImpl extends BaseServiceImpl implements MonitorServic @Resource private StorageService storageService; + @Resource + private LogService logService; + @Resource private SystemConfigService systemConfigService; @@ -110,7 +120,7 @@ public class MonitorServiceImpl extends BaseServiceImpl implements MonitorServic @Override public List getMetaAll() { - List list = monitor.getMetaAll() + List list = profileComponent.getMetaAll() .stream() .map(m -> convertMeta2Vo(m)) .sorted(Comparator.comparing(MetaVo::getUpdateTime).reversed()) @@ -120,7 +130,7 @@ public class MonitorServiceImpl extends BaseServiceImpl implements MonitorServic @Override public MetaVo getMetaVo(String metaId) { - Meta meta = monitor.getMeta(metaId); + Meta meta = profileComponent.getMeta(metaId); Assert.notNull(meta, "The meta is null."); return convertMeta2Vo(meta); @@ -140,7 +150,7 @@ public class MonitorServiceImpl extends BaseServiceImpl implements MonitorServic String error = params.get(ConfigConstant.DATA_ERROR); String success = params.get(ConfigConstant.DATA_SUCCESS); - Paging paging = monitor.queryData(getDefaultMetaId(id), pageNum, pageSize, error, success); + Paging paging = queryData(getDefaultMetaId(id), pageNum, pageSize, error, success); List data = (List) paging.getData(); List list = new ArrayList<>(); for (Map row : data) { @@ -160,7 +170,12 @@ public class MonitorServiceImpl extends BaseServiceImpl implements MonitorServic @Override public String clearData(String id) { Assert.hasText(id, "驱动不存在."); - monitor.clearData(id); + Meta meta = profileComponent.getMeta(id); + Mapping mapping = profileComponent.getMapping(meta.getMappingId()); + String model = ModelEnum.getModelEnum(mapping.getModel()).getName(); + LogType.MappingLog log = LogType.MappingLog.CLEAR_DATA; + logService.log(log, "%s:%s(%s)", log.getMessage(), mapping.getName(), model); + storageService.clear(StorageEnum.DATA, id); return "清空同步数据成功"; } @@ -169,7 +184,12 @@ public class MonitorServiceImpl extends BaseServiceImpl implements MonitorServic int pageNum = NumberUtil.toInt(params.get("pageNum"), 1); int pageSize = NumberUtil.toInt(params.get("pageSize"), 10); String json = params.get(ConfigConstant.CONFIG_MODEL_JSON); - Paging paging = monitor.queryLog(pageNum, pageSize, json); + Query query = new Query(pageNum, pageSize); + if (StringUtil.isNotBlank(json)) { + query.addFilter(ConfigConstant.CONFIG_MODEL_JSON, json, true); + } + query.setType(StorageEnum.LOG); + Paging paging = storageService.query(query); List data = (List) paging.getData(); paging.setData(data.stream() .map(m -> convert2Vo(m, LogVo.class)) @@ -179,7 +199,7 @@ public class MonitorServiceImpl extends BaseServiceImpl implements MonitorServic @Override public String clearLog() { - monitor.clearLog(); + storageService.clear(StorageEnum.LOG, null); return "清空日志成功"; } @@ -193,7 +213,7 @@ public class MonitorServiceImpl extends BaseServiceImpl implements MonitorServic @Override public List getStorageDataStatusEnumAll() { - return monitor.getStorageDataStatusEnumAll(); + return profileComponent.getStorageDataStatusEnumAll(); } @Override @@ -213,7 +233,7 @@ public class MonitorServiceImpl extends BaseServiceImpl implements MonitorServic @Override public void run() { // 预警：驱动出现失败记录，发送通知消息 - List metaAll = monitor.getMetaAll(); + List metaAll = profileComponent.getMetaAll(); if (CollectionUtils.isEmpty(metaAll)) { return; } @@ -222,7 +242,7 @@ public class MonitorServiceImpl extends BaseServiceImpl implements MonitorServic metaAll.forEach(meta -> { // 有失败记录 if (MetaEnum.isRunning(meta.getState()) && meta.getFail().get() > 0) { - Mapping mapping = monitor.getMapping(meta.getMappingId()); + Mapping mapping = profileComponent.getMapping(meta.getMappingId()); if (null != mapping) { ModelEnum modelEnum = ModelEnum.getModelEnum(mapping.getModel()); content.append("

"); @@ -241,6 +261,27 @@ public class MonitorServiceImpl extends BaseServiceImpl implements MonitorServic } } + private Paging queryData(String metaId, int pageNum, int pageSize, String error, String success) { + // 没有驱动 + if (StringUtil.isBlank(metaId)) { + return new Paging(pageNum, pageSize); + } + Query query = new Query(pageNum, pageSize); + Map fieldResolvers = new LinkedHashMap<>(); + fieldResolvers.put(ConfigConstant.BINLOG_DATA, IndexFieldResolverEnum.BINARY); + query.setIndexFieldResolverMap(fieldResolvers); + + // 查询异常信息 + if (StringUtil.isNotBlank(error)) { + query.addFilter(ConfigConstant.DATA_ERROR, error, true); + } + // 查询是否成功, 默认查询失败 + query.addFilter(ConfigConstant.DATA_SUCCESS, StringUtil.isNotBlank(success) ? NumberUtil.toInt(success) : StorageDataStatusEnum.FAIL.getValue()); + query.setMetaId(metaId); + query.setType(StorageEnum.DATA); + return storageService.query(query); + } + private void deleteExpiredData() { List metaAll = getMetaAll(); if (!CollectionUtils.isEmpty(metaAll)) { @@ -268,7 +309,7 @@ public class MonitorServiceImpl extends BaseServiceImpl implements MonitorServic } private MetaVo convertMeta2Vo(Meta meta) { - Mapping mapping = monitor.getMapping(meta.getMappingId()); + Mapping mapping = profileComponent.getMapping(meta.getMappingId()); Assert.notNull(mapping, "驱动不存在."); ModelEnum modelEnum = ModelEnum.getModelEnum(mapping.getModel()); MetaVo metaVo = new MetaVo(modelEnum.getName(), mapping.getName()); diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/PluginServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/PluginServiceImpl.java index 1a89e7df..b1beb0a6 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/PluginServiceImpl.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/PluginServiceImpl.java @@ -5,18 +5,19 @@ import org.dbsyncer.biz.PluginService; import org.dbsyncer.biz.vo.PluginVo; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.manager.Manager; +import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.parser.logger.LogService; import org.dbsyncer.parser.logger.LogType; import org.dbsyncer.parser.model.Mapping; import org.dbsyncer.parser.model.TableGroup; +import org.dbsyncer.plugin.PluginFactory; import org.dbsyncer.plugin.config.Plugin; import org.dbsyncer.plugin.enums.FileSuffixEnum; import org.springframework.beans.BeanUtils; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.springframework.util.Assert; +import javax.annotation.Resource; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -31,15 +32,18 @@ import java.util.stream.Collectors; @Component public class PluginServiceImpl implements PluginService { - @Autowired - private Manager manager; + @Resource + private PluginFactory pluginFactory; - @Autowired + @Resource + private ProfileComponent profileComponent; + + @Resource private LogService logService; @Override public List getPluginAll() { - List pluginAll = manager.getPluginAll(); + List pluginAll = pluginFactory.getPluginAll(); List vos = new ArrayList<>(); if (!CollectionUtils.isEmpty(pluginAll)) { Map> pluginClassNameMap = getPluginClassNameMap(); @@ -55,17 +59,17 @@ public class PluginServiceImpl implements PluginService { @Override public String getPluginPath() { - return manager.getPluginPath(); + return pluginFactory.getPluginPath(); } @Override public String getLibraryPath() { - return manager.getLibraryPath(); + return pluginFactory.getLibraryPath(); } @Override public void loadPlugins() { - manager.loadPlugins(); + pluginFactory.loadPlugins(); logService.log(LogType.PluginLog.UPDATE); } @@ -84,7 +88,7 @@ public class PluginServiceImpl implements PluginService { private Map> getPluginClassNameMap() { Map> map = new HashMap<>(); - List mappingAll = manager.getMappingAll(); + List mappingAll = profileComponent.getMappingAll(); if (CollectionUtils.isEmpty(mappingAll)) { return map; } @@ -97,7 +101,7 @@ public class PluginServiceImpl implements PluginService { continue; } - List tableGroupAll = manager.getTableGroupAll(m.getId()); + List tableGroupAll = profileComponent.getTableGroupAll(m.getId()); if (CollectionUtils.isEmpty(tableGroupAll)) { continue; } diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ProjectGroupServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ProjectGroupServiceImpl.java index f9444504..00c013b6 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ProjectGroupServiceImpl.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ProjectGroupServiceImpl.java @@ -8,17 +8,21 @@ import org.dbsyncer.biz.vo.MappingVo; import org.dbsyncer.biz.vo.ProjectGroupVo; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.manager.Manager; +import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.parser.logger.LogType; import org.dbsyncer.parser.model.ConfigModel; import org.dbsyncer.parser.model.Connector; import org.dbsyncer.parser.model.ProjectGroup; import org.springframework.beans.BeanUtils; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.util.Assert; -import java.util.*; +import javax.annotation.Resource; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; /** @@ -31,16 +35,16 @@ import java.util.stream.Collectors; @Service public class ProjectGroupServiceImpl extends BaseServiceImpl implements ProjectGroupService { - @Autowired + @Resource private ConnectorService connectorService; - @Autowired + @Resource private MappingService mappingService; - @Autowired - private Manager manager; + @Resource + private ProfileComponent profileComponent; - @Autowired + @Resource private Checker projectGroupChecker; @Override @@ -48,7 +52,7 @@ public class ProjectGroupServiceImpl extends BaseServiceImpl implements ProjectG ConfigModel model = projectGroupChecker.checkAddConfigModel(params); log(LogType.ConnectorLog.INSERT, model); - return manager.addConfigModel(model); + return profileComponent.addConfigModel(model); } @Override @@ -56,15 +60,15 @@ public class ProjectGroupServiceImpl extends BaseServiceImpl implements ProjectG ConfigModel model = projectGroupChecker.checkEditConfigModel(params); log(LogType.ConnectorLog.UPDATE, model); - return manager.editConfigModel(model); + return profileComponent.editConfigModel(model); } @Override public String remove(String id) { - ProjectGroup projectGroup = manager.getProjectGroup(id); + ProjectGroup projectGroup = profileComponent.getProjectGroup(id); log(LogType.ConnectorLog.DELETE, projectGroup); Assert.notNull(projectGroup, "该分组已被删除"); - manager.removeConfigModel(id); + profileComponent.removeConfigModel(id); return "删除分组成功!"; } @@ -80,7 +84,7 @@ public class ProjectGroupServiceImpl extends BaseServiceImpl implements ProjectG return vo; } - ProjectGroup projectGroup = manager.getProjectGroup(id); + ProjectGroup projectGroup = profileComponent.getProjectGroup(id); Assert.notNull(projectGroup, "该分组已被删除"); BeanUtils.copyProperties(projectGroup, vo); vo.setConnectors(Collections.EMPTY_LIST); @@ -115,7 +119,7 @@ public class ProjectGroupServiceImpl extends BaseServiceImpl implements ProjectG @Override public List getProjectGroupAll() { - return manager.getProjectGroupAll(); + return profileComponent.getProjectGroupAll(); } } \ No newline at end of file diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/SystemConfigServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/SystemConfigServiceImpl.java index 3f9479e7..bc14a13f 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/SystemConfigServiceImpl.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/SystemConfigServiceImpl.java @@ -6,12 +6,12 @@ import org.dbsyncer.biz.UserConfigService; import org.dbsyncer.biz.checker.Checker; import org.dbsyncer.biz.vo.SystemConfigVo; import org.dbsyncer.common.util.CollectionUtils; -import org.dbsyncer.manager.Manager; -import org.dbsyncer.manager.template.PreloadTemplate; +import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.parser.logger.LogService; import org.dbsyncer.parser.logger.LogType; import org.dbsyncer.parser.model.ConfigModel; import org.dbsyncer.parser.model.SystemConfig; +import org.dbsyncer.manager.template.PreloadTemplate; import org.dbsyncer.plugin.enums.FileSuffixEnum; import org.springframework.beans.BeanUtils; import org.springframework.stereotype.Service; @@ -35,13 +35,13 @@ import java.util.Map; public class SystemConfigServiceImpl implements SystemConfigService { @Resource - private Manager manager; + private ProfileComponent profileComponent; @Resource - private Checker systemConfigChecker; + private PreloadTemplate preloadTemplate; @Resource - private PreloadTemplate preloadTemplate; + private Checker systemConfigChecker; @Resource private LogService logService; @@ -52,7 +52,7 @@ public class SystemConfigServiceImpl implements SystemConfigService { @Override public String edit(Map params) { ConfigModel model = systemConfigChecker.checkEditConfigModel(params); - manager.editConfigModel(model); + profileComponent.editConfigModel(model); return "修改成功."; } @@ -66,9 +66,9 @@ public class SystemConfigServiceImpl implements SystemConfigService { List list = new ArrayList<>(); list.add(getSystemConfig()); list.add(userConfigService.getUserConfig()); - manager.getConnectorAll().forEach(config -> list.add(config)); - manager.getMappingAll().forEach(config -> list.add(config)); - manager.getMetaAll().forEach(config -> list.add(config)); + profileComponent.getConnectorAll().forEach(config -> list.add(config)); + profileComponent.getMappingAll().forEach(config -> list.add(config)); + profileComponent.getMetaAll().forEach(config -> list.add(config)); return list; } @@ -104,13 +104,13 @@ public class SystemConfigServiceImpl implements SystemConfigService { } private SystemConfig getSystemConfig() { - SystemConfig config = manager.getSystemConfig(); + SystemConfig config = profileComponent.getSystemConfig(); if (null != config) { return config; } synchronized (this) { - config = manager.getSystemConfig(); + config = profileComponent.getSystemConfig(); if (null == config) { config = (SystemConfig) systemConfigChecker.checkAddConfigModel(new HashMap<>()); } diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/TableGroupServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/TableGroupServiceImpl.java index 65c8175c..26585b3d 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/TableGroupServiceImpl.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/TableGroupServiceImpl.java @@ -6,14 +6,15 @@ import org.dbsyncer.biz.checker.impl.tablegroup.TableGroupChecker; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.connector.model.Field; +import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.parser.logger.LogType; import org.dbsyncer.parser.model.Mapping; import org.dbsyncer.parser.model.TableGroup; import org.dbsyncer.storage.constant.ConfigConstant; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.util.Assert; +import javax.annotation.Resource; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -29,15 +30,18 @@ import java.util.stream.Stream; @Service public class TableGroupServiceImpl extends BaseServiceImpl implements TableGroupService { - @Autowired + @Resource private Checker tableGroupChecker; + @Resource + private ProfileComponent profileComponent; + @Override public String add(Map params) { String mappingId = params.get("mappingId"); - assertRunning(manager.getMapping(mappingId)); + assertRunning(profileComponent.getMapping(mappingId)); - synchronized (LOCK){ + synchronized (LOCK) { // table1, table2 String[] sourceTableArray = StringUtil.split(params.get("sourceTable"), "|"); String[] targetTableArray = StringUtil.split(params.get("targetTable"), "|"); @@ -50,9 +54,9 @@ public class TableGroupServiceImpl extends BaseServiceImpl implements TableGroup params.put("targetTable", targetTableArray[i]); TableGroup model = (TableGroup) tableGroupChecker.checkAddConfigModel(params); log(LogType.TableGroupLog.INSERT, model); - int tableGroupCount = manager.getTableGroupCount(mappingId); + int tableGroupCount = profileComponent.getTableGroupCount(mappingId); model.setIndex(tableGroupCount + 1); - id = manager.addTableGroup(model); + id = profileComponent.addTableGroup(model); } // 合并驱动公共字段 @@ -64,39 +68,39 @@ public class TableGroupServiceImpl extends BaseServiceImpl implements TableGroup @Override public String edit(Map params) { String id = params.get(ConfigConstant.CONFIG_MODEL_ID); - TableGroup tableGroup = manager.getTableGroup(id); + TableGroup tableGroup = profileComponent.getTableGroup(id); Assert.notNull(tableGroup, "Can not find tableGroup."); - assertRunning(manager.getMapping(tableGroup.getMappingId())); + assertRunning(profileComponent.getMapping(tableGroup.getMappingId())); TableGroup model = (TableGroup) tableGroupChecker.checkEditConfigModel(params); log(LogType.TableGroupLog.UPDATE, model); - return manager.editTableGroup(model); + return profileComponent.editTableGroup(model); } @Override public String refreshFields(String id) { - TableGroup tableGroup = manager.getTableGroup(id); + TableGroup tableGroup = profileComponent.getTableGroup(id); Assert.notNull(tableGroup, "Can not find tableGroup."); - assertRunning(manager.getMapping(tableGroup.getMappingId())); + assertRunning(profileComponent.getMapping(tableGroup.getMappingId())); TableGroupChecker checker = (TableGroupChecker) tableGroupChecker; checker.refreshTableFields(tableGroup); - return manager.editTableGroup(tableGroup); + return profileComponent.editTableGroup(tableGroup); } @Override public boolean remove(String mappingId, String ids) { Assert.hasText(mappingId, "Mapping id can not be null"); Assert.hasText(ids, "TableGroup ids can not be null"); - assertRunning(manager.getMapping(mappingId)); + assertRunning(profileComponent.getMapping(mappingId)); // 批量删除表 Stream.of(StringUtil.split(ids, ",")).parallel().forEach(id -> { - TableGroup model = manager.getTableGroup(id); + TableGroup model = profileComponent.getTableGroup(id); log(LogType.TableGroupLog.DELETE, model); - manager.removeTableGroup(id); + profileComponent.removeTableGroup(id); }); // 合并驱动公共字段 @@ -109,34 +113,34 @@ public class TableGroupServiceImpl extends BaseServiceImpl implements TableGroup @Override public TableGroup getTableGroup(String id) { - TableGroup tableGroup = manager.getTableGroup(id); + TableGroup tableGroup = profileComponent.getTableGroup(id); Assert.notNull(tableGroup, "TableGroup can not be null"); return tableGroup; } @Override public List getTableGroupAll(String mappingId) { - return manager.getSortedTableGroupAll(mappingId); + return profileComponent.getSortedTableGroupAll(mappingId); } private void resetTableGroupAllIndex(String mappingId) { synchronized (LOCK) { - List list = manager.getSortedTableGroupAll(mappingId); + List list = profileComponent.getSortedTableGroupAll(mappingId); int size = list.size(); int i = size; while (i > 0) { TableGroup g = list.get(size - i); g.setIndex(i); - manager.editConfigModel(g); + profileComponent.editConfigModel(g); i--; } } } private void mergeMappingColumn(String mappingId) { - List groups = manager.getTableGroupAll(mappingId); + List groups = profileComponent.getTableGroupAll(mappingId); - Mapping mapping = manager.getMapping(mappingId); + Mapping mapping = profileComponent.getMapping(mappingId); Assert.notNull(mapping, "mapping not exist."); List sourceColumn = null; @@ -148,7 +152,7 @@ public class TableGroupServiceImpl extends BaseServiceImpl implements TableGroup mapping.setSourceColumn(sourceColumn); mapping.setTargetColumn(targetColumn); - manager.editConfigModel(mapping); + profileComponent.editConfigModel(mapping); } private List pickCommonFields(List column, List target) { diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/UserConfigServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/UserConfigServiceImpl.java index 58773e60..185903d2 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/UserConfigServiceImpl.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/UserConfigServiceImpl.java @@ -7,16 +7,16 @@ import org.dbsyncer.biz.enums.UserRoleEnum; import org.dbsyncer.biz.vo.UserInfoVo; import org.dbsyncer.common.util.SHA1Util; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.manager.Manager; +import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.parser.logger.LogService; import org.dbsyncer.parser.logger.LogType; import org.dbsyncer.parser.model.UserConfig; import org.dbsyncer.parser.model.UserInfo; import org.springframework.beans.BeanUtils; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.util.Assert; +import javax.annotation.Resource; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -35,13 +35,13 @@ public class UserConfigServiceImpl implements UserConfigService { private static final String DEFAULT_PASSWORD = "0DPiKuNIrrVmD8IUCuw1hQxNqZc="; - @Autowired - private Manager manager; + @Resource + private ProfileComponent profileComponent; - @Autowired + @Resource private UserConfigChecker userConfigChecker; - @Autowired + @Resource private LogService logService; @Override @@ -64,7 +64,7 @@ public class UserConfigServiceImpl implements UserConfigService { userConfig.getUserInfoList().add(new UserInfo(username, nickname, SHA1Util.b64_sha1(password), UserRoleEnum.USER.getCode(), mail)); logService.log(LogType.UserLog.INSERT, String.format("[%s]添加[%s]账号成功", currentUser.getUsername(), username)); - return manager.editConfigModel(userConfig); + return profileComponent.editConfigModel(userConfig); } @Override @@ -107,7 +107,7 @@ public class UserConfigServiceImpl implements UserConfigService { logService.log(LogType.UserLog.UPDATE, String.format("[%s]修改[%s]账号密码成功", currentUser.getUsername(), username)); } - return manager.editConfigModel(userConfig); + return profileComponent.editConfigModel(userConfig); } @Override @@ -127,7 +127,7 @@ public class UserConfigServiceImpl implements UserConfigService { UserInfo deleteUser = userConfig.getUserInfo(username); Assert.notNull(deleteUser, "用户已删除."); userConfig.removeUserInfo(username); - manager.editConfigModel(userConfig); + profileComponent.editConfigModel(userConfig); logService.log(LogType.UserLog.DELETE, String.format("[%s]删除[%s]账号成功", currentUser.getUsername(), username)); return "删除用户成功!"; } @@ -168,18 +168,18 @@ public class UserConfigServiceImpl implements UserConfigService { @Override public UserConfig getUserConfig() { - UserConfig config = manager.getUserConfig(); + UserConfig config = profileComponent.getUserConfig(); if (null != config) { return config; } synchronized (this) { - config = manager.getUserConfig(); + config = profileComponent.getUserConfig(); if (null == config) { config = (UserConfig) userConfigChecker.checkAddConfigModel(new HashMap<>()); UserRoleEnum admin = UserRoleEnum.ADMIN; config.getUserInfoList().add(new UserInfo(DEFAULT_USERNAME, DEFAULT_USERNAME, DEFAULT_PASSWORD, admin.getCode(), "")); - manager.addConfigModel(config); + profileComponent.addConfigModel(config); } return config; } diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/Command.java b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/Command.java deleted file mode 100644 index ffd6118a..00000000 --- a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/Command.java +++ /dev/null @@ -1,7 +0,0 @@ -package org.dbsyncer.manager; - -import org.dbsyncer.manager.command.Persistence; -import org.dbsyncer.manager.command.Preload; - -public interface Command extends Persistence, Preload { -} \ No newline at end of file diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/CommandExecutor.java b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/CommandExecutor.java deleted file mode 100644 index eb1d043a..00000000 --- a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/CommandExecutor.java +++ /dev/null @@ -1,6 +0,0 @@ -package org.dbsyncer.manager; - -public interface CommandExecutor { - - Object execute(Command cmd); -} \ No newline at end of file diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/Executor.java b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/Executor.java deleted file mode 100644 index 094e5d75..00000000 --- a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/Executor.java +++ /dev/null @@ -1,37 +0,0 @@ -package org.dbsyncer.manager; - -import org.dbsyncer.parser.enums.MetaEnum; -import org.dbsyncer.parser.model.Mapping; - -/** - * 同步任务执行器 - * - * @author AE86 - * @version 1.0.0 - * @date 2020/04/26 16:32 - */ -public interface Executor { - - /** - * 启动同步任务 - * - * @param mapping - */ - void start(Mapping mapping); - - /** - * 关闭同步任务 - * - * @param mapping - */ - void close(Mapping mapping); - - /** - * 切换meta状态 - * - * @param metaId - * @param metaEnum - */ - void changeMetaState(String metaId, MetaEnum metaEnum); - -} \ No newline at end of file diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/ManagerFactory.java b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/ManagerFactory.java index de5bb324..9605bc8f 100644 --- a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/ManagerFactory.java +++ b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/ManagerFactory.java @@ -1,44 +1,15 @@ package org.dbsyncer.manager; -import org.dbsyncer.common.event.ClosedEvent; -import org.dbsyncer.common.model.AbstractConnectorConfig; -import org.dbsyncer.common.model.Paging; -import org.dbsyncer.common.spi.ConnectorMapper; -import org.dbsyncer.common.util.CollectionUtils; -import org.dbsyncer.connector.enums.ConnectorEnum; -import org.dbsyncer.connector.enums.FilterEnum; -import org.dbsyncer.connector.enums.OperationEnum; -import org.dbsyncer.connector.model.MetaInfo; -import org.dbsyncer.connector.model.Table; -import org.dbsyncer.listener.enums.QuartzFilterEnum; -import org.dbsyncer.manager.enums.CommandEnum; -import org.dbsyncer.manager.enums.GroupStrategyEnum; -import org.dbsyncer.manager.model.OperationConfig; -import org.dbsyncer.manager.model.QueryConfig; -import org.dbsyncer.manager.template.OperationTemplate; -import org.dbsyncer.parser.Parser; -import org.dbsyncer.parser.enums.ConvertEnum; +import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.parser.enums.MetaEnum; -import org.dbsyncer.parser.enums.ModelEnum; -import org.dbsyncer.parser.logger.LogService; -import org.dbsyncer.parser.logger.LogType; -import org.dbsyncer.parser.model.*; -import org.dbsyncer.plugin.PluginFactory; -import org.dbsyncer.plugin.config.Plugin; -import org.dbsyncer.storage.StorageService; -import org.dbsyncer.storage.enums.StorageDataStatusEnum; -import org.dbsyncer.storage.enums.StorageEnum; -import org.dbsyncer.storage.query.Query; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationListener; +import org.dbsyncer.parser.model.Mapping; +import org.dbsyncer.parser.model.Meta; import org.springframework.stereotype.Component; import org.springframework.util.Assert; +import javax.annotation.Resource; import java.time.Instant; -import java.util.Comparator; -import java.util.List; import java.util.Map; -import java.util.stream.Collectors; /** * @author AE86 @@ -46,267 +17,14 @@ import java.util.stream.Collectors; * @date 2019/9/16 23:59 */ @Component -public class ManagerFactory implements Manager, ApplicationListener { +public class ManagerFactory { - @Autowired - private Parser parser; + @Resource + private ProfileComponent profileComponent; - @Autowired - private PluginFactory pluginFactory; - - @Autowired - private OperationTemplate operationTemplate; - - @Autowired - private StorageService storageService; - - @Autowired - private LogService logService; - - @Autowired + @Resource private Map map; - @Override - public String addConfigModel(ConfigModel model) { - return operationTemplate.execute(new OperationConfig(model, CommandEnum.OPR_ADD)); - } - - @Override - public String editConfigModel(ConfigModel model) { - return operationTemplate.execute(new OperationConfig(model, CommandEnum.OPR_EDIT)); - } - - @Override - public void removeConfigModel(String id) { - operationTemplate.remove(new OperationConfig(id)); - } - - @Override - public SystemConfig getSystemConfig() { - List list = operationTemplate.queryAll(SystemConfig.class); - return CollectionUtils.isEmpty(list) ? null : list.get(0); - } - - @Override - public UserConfig getUserConfig() { - List list = operationTemplate.queryAll(UserConfig.class); - return CollectionUtils.isEmpty(list) ? null : list.get(0); - } - - @Override - public ProjectGroup getProjectGroup(String id) { - return operationTemplate.queryObject(ProjectGroup.class, id); - } - - @Override - public List getProjectGroupAll() { - return operationTemplate.queryAll(ProjectGroup.class); - } - - @Override - public ConnectorMapper connect(AbstractConnectorConfig config) { - return parser.connect(config); - } - - @Override - public boolean refreshConnectorConfig(AbstractConnectorConfig config) { - return parser.refreshConnectorConfig(config); - } - - @Override - public boolean isAliveConnectorConfig(AbstractConnectorConfig config) { - return parser.isAliveConnectorConfig(config); - } - - @Override - public List

getTable(ConnectorMapper config) { - return parser.getTable(config); - } - - @Override - public MetaInfo getMetaInfo(String connectorId, String tableName) { - return parser.getMetaInfo(connectorId, tableName); - } - - @Override - public Connector getConnector(String connectorId) { - return operationTemplate.queryObject(Connector.class, connectorId); - } - - @Override - public List getConnectorAll() { - return operationTemplate.queryAll(Connector.class); - } - - @Override - public void checkAllConnectorStatus() { - List list = getConnectorAll(); - if (!CollectionUtils.isEmpty(list)) { - list.forEach(c -> { - try { - refreshConnectorConfig(c.getConfig()); - } catch (Exception e) { - LogType.ConnectorLog logType = LogType.ConnectorLog.FAILED; - logService.log(logType, "%s%s", logType.getName(), e.getMessage()); - } - }); - } - } - - @Override - public Mapping getMapping(String mappingId) { - return operationTemplate.queryObject(Mapping.class, mappingId); - } - - @Override - public List getMappingAll() { - return operationTemplate.queryAll(Mapping.class); - } - - @Override - public String addTableGroup(TableGroup model) { - return operationTemplate.execute(new OperationConfig(model, CommandEnum.OPR_ADD, GroupStrategyEnum.TABLE)); - } - - @Override - public String editTableGroup(TableGroup model) { - return operationTemplate.execute(new OperationConfig(model, CommandEnum.OPR_EDIT, GroupStrategyEnum.TABLE)); - } - - @Override - public void removeTableGroup(String id) { - operationTemplate.remove(new OperationConfig(id, GroupStrategyEnum.TABLE)); - } - - @Override - public TableGroup getTableGroup(String tableGroupId) { - return operationTemplate.queryObject(TableGroup.class, tableGroupId); - } - - @Override - public List getTableGroupAll(String mappingId) { - TableGroup tableGroup = new TableGroup().setMappingId(mappingId); - return operationTemplate.queryAll(new QueryConfig(tableGroup, GroupStrategyEnum.TABLE)); - } - - @Override - public List getSortedTableGroupAll(String mappingId) { - List list = getTableGroupAll(mappingId) - .stream() - .sorted(Comparator.comparing(TableGroup::getIndex).reversed()) - .collect(Collectors.toList()); - return list; - } - - @Override - public int getTableGroupCount(String mappingId) { - TableGroup tableGroup = new TableGroup().setMappingId(mappingId); - return operationTemplate.queryCount(new QueryConfig<>(tableGroup, GroupStrategyEnum.TABLE)); - } - - @Override - public Map getCommand(Mapping mapping, TableGroup tableGroup) { - return parser.getCommand(mapping, tableGroup); - } - - @Override - public long getCount(String connectorId, Map command) { - return parser.getCount(connectorId, command); - } - - @Override - public Meta getMeta(String metaId) { - return operationTemplate.queryObject(Meta.class, metaId); - } - - @Override - public List getMetaAll() { - return operationTemplate.queryAll(Meta.class); - } - - @Override - public Paging queryData(Query query) { - query.setType(StorageEnum.DATA); - return storageService.query(query); - } - - @Override - public void removeData(String metaId, String messageId) { - storageService.remove(StorageEnum.DATA, metaId, messageId); - } - - @Override - public void clearData(String metaId) { - Meta meta = getMeta(metaId); - Mapping mapping = getMapping(meta.getMappingId()); - String model = ModelEnum.getModelEnum(mapping.getModel()).getName(); - LogType.MappingLog log = LogType.MappingLog.CLEAR_DATA; - logService.log(log, "%s:%s(%s)", log.getMessage(), mapping.getName(), model); - storageService.clear(StorageEnum.DATA, metaId); - } - - @Override - public Paging queryLog(Query query) { - query.setType(StorageEnum.LOG); - return storageService.query(query); - } - - @Override - public void clearLog() { - storageService.clear(StorageEnum.LOG, null); - } - - @Override - public List getConnectorEnumAll() { - return parser.getConnectorEnumAll(); - } - - @Override - public List getOperationEnumAll() { - return parser.getOperationEnumAll(); - } - - @Override - public List getQuartzFilterEnumAll() { - return parser.getQuartzFilterEnumAll(); - } - - @Override - public List getFilterEnumAll() { - return parser.getFilterEnumAll(); - } - - @Override - public List getConvertEnumAll() { - return parser.getConvertEnumAll(); - } - - @Override - public List getStorageDataStatusEnumAll() { - return parser.getStorageDataStatusEnumAll(); - } - - @Override - public List getPluginAll() { - return pluginFactory.getPluginAll(); - } - - @Override - public String getPluginPath() { - return pluginFactory.getPluginPath(); - } - - @Override - public String getLibraryPath() { - return pluginFactory.getLibraryPath(); - } - - @Override - public void loadPlugins() { - pluginFactory.loadPlugins(); - } - - @Override public void start(Mapping mapping) { Puller puller = getPuller(mapping); @@ -316,7 +34,6 @@ public class ManagerFactory implements Manager, ApplicationListener puller.start(mapping); } - @Override public void close(Mapping mapping) { Puller puller = getPuller(mapping); @@ -327,23 +44,16 @@ public class ManagerFactory implements Manager, ApplicationListener puller.close(metaId); } - @Override public void changeMetaState(String metaId, MetaEnum metaEnum) { - Meta meta = getMeta(metaId); + Meta meta = profileComponent.getMeta(metaId); int code = metaEnum.getCode(); if (null != meta && meta.getState() != code) { meta.setState(code); meta.setUpdateTime(Instant.now().toEpochMilli()); - editConfigModel(meta); + profileComponent.editConfigModel(meta); } } - @Override - public void onApplicationEvent(ClosedEvent event) { - // 异步监听任务关闭事件 - changeMetaState(event.getMetaId(), MetaEnum.READY); - } - private Puller getPuller(Mapping mapping) { Assert.notNull(mapping, "驱动不能为空"); String model = mapping.getModel(); diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/command/Persistence.java b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/command/Persistence.java deleted file mode 100644 index e519c99d..00000000 --- a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/command/Persistence.java +++ /dev/null @@ -1,14 +0,0 @@ -package org.dbsyncer.manager.command; - -import org.dbsyncer.manager.ManagerException; - -public interface Persistence { - - default boolean addConfig() { - throw new ManagerException("Unsupported method addConfig"); - } - - default boolean editConfig() { - throw new ManagerException("Unsupported method editConfig"); - } -} \ No newline at end of file diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/command/Preload.java b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/command/Preload.java deleted file mode 100644 index 14a8ab7b..00000000 --- a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/command/Preload.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.dbsyncer.manager.command; - -import org.dbsyncer.manager.ManagerException; -import org.dbsyncer.parser.model.*; - -public interface Preload { - - default SystemConfig parseSystemConfig(){ - throw new ManagerException("Unsupported method parseSystemConfig"); - } - - default UserConfig parseUserConfig(){ - throw new ManagerException("Unsupported method parseUserConfig"); - } - - default Connector parseConnector(){ - throw new ManagerException("Unsupported method parseConnector"); - } - - default Mapping parseMapping(){ - throw new ManagerException("Unsupported method parseMapping"); - } - - default TableGroup parseTableGroup(){ - throw new ManagerException("Unsupported method parseTableGroup"); - } - - default Meta parseMeta(){ - throw new ManagerException("Unsupported method parseMeta"); - } - - default ProjectGroup parseProjectGroup(){ - throw new ManagerException("Unsupported method parseProjectGroup"); - } -} \ No newline at end of file diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/listener/ClosedTaskListener.java b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/listener/ClosedTaskListener.java new file mode 100644 index 00000000..031f4f14 --- /dev/null +++ b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/listener/ClosedTaskListener.java @@ -0,0 +1,29 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.manager.listener; + +import org.dbsyncer.common.event.ClosedEvent; +import org.dbsyncer.manager.ManagerFactory; +import org.dbsyncer.parser.enums.MetaEnum; +import org.springframework.context.ApplicationListener; +import org.springframework.stereotype.Component; + +import javax.annotation.Resource; + +/** + * @Version 1.0.0 + * @Author AE86 + * @Date 2023-11-14 01:48 + */ +@Component +public class ClosedTaskListener implements ApplicationListener { + + @Resource + private ManagerFactory managerFactory; + + @Override + public void onApplicationEvent(ClosedEvent event) { + managerFactory.changeMetaState(event.getMetaId(), MetaEnum.READY); + } +} \ No newline at end of file diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/puller/FullPuller.java b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/puller/FullPuller.java index d27dddbe..7a44a566 100644 --- a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/puller/FullPuller.java +++ b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/puller/FullPuller.java @@ -3,8 +3,8 @@ package org.dbsyncer.manager.puller; import org.dbsyncer.common.util.NumberUtil; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.connector.util.PrimaryKeyUtil; -import org.dbsyncer.manager.Manager; -import org.dbsyncer.parser.Parser; +import org.dbsyncer.parser.ParserComponent; +import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.parser.enums.ParserEnum; import org.dbsyncer.parser.event.FullRefreshEvent; import org.dbsyncer.parser.logger.LogService; @@ -41,10 +41,10 @@ public final class FullPuller extends AbstractPuller implements ApplicationListe private final Logger logger = LoggerFactory.getLogger(getClass()); @Resource - private Parser parser; + private ParserComponent parserComponent; @Resource - private Manager manager; + private ProfileComponent profileComponent; @Resource private LogService logService; @@ -57,7 +57,7 @@ public final class FullPuller extends AbstractPuller implements ApplicationListe final String metaId = mapping.getMetaId(); ExecutorService executor = null; try { - List list = manager.getSortedTableGroupAll(mapping.getId()); + List list = profileComponent.getSortedTableGroupAll(mapping.getId()); Assert.notEmpty(list, "映射关系不能为空"); logger.info("开始全量同步：{}, {}", metaId, mapping.getName()); map.putIfAbsent(metaId, new Task(metaId)); @@ -106,7 +106,7 @@ public final class FullPuller extends AbstractPuller implements ApplicationListe task.setEndTime(now); // 获取上次同步点 - Meta meta = manager.getMeta(task.getId()); + Meta meta = profileComponent.getMeta(task.getId()); Map snapshot = meta.getSnapshot(); task.setPageIndex(NumberUtil.toInt(snapshot.get(ParserEnum.PAGE_INDEX.getCode()), ParserEnum.PAGE_INDEX.getDefaultValue())); // 反序列化游标值类型(通常为数字或字符串类型) @@ -117,7 +117,7 @@ public final class FullPuller extends AbstractPuller implements ApplicationListe int i = task.getTableGroupIndex(); while (i < list.size()) { - parser.execute(task, mapping, list.get(i), executor); + parserComponent.execute(task, mapping, list.get(i), executor); if (!task.isRunning()) { break; } @@ -134,7 +134,7 @@ public final class FullPuller extends AbstractPuller implements ApplicationListe } private void flush(Task task) { - Meta meta = manager.getMeta(task.getId()); + Meta meta = profileComponent.getMeta(task.getId()); Assert.notNull(meta, "检查meta为空."); // 全量的过程中，有新数据则更新总数 @@ -150,7 +150,7 @@ public final class FullPuller extends AbstractPuller implements ApplicationListe snapshot.put(ParserEnum.PAGE_INDEX.getCode(), String.valueOf(task.getPageIndex())); snapshot.put(ParserEnum.CURSOR.getCode(), StringUtil.join(task.getCursors(), ",")); snapshot.put(ParserEnum.TABLE_GROUP_INDEX.getCode(), String.valueOf(task.getTableGroupIndex())); - manager.editConfigModel(meta); + profileComponent.editConfigModel(meta); } } \ No newline at end of file diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/puller/IncrementPuller.java b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/puller/IncrementPuller.java index 1d70134c..dd012e42 100644 --- a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/puller/IncrementPuller.java +++ b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/puller/IncrementPuller.java @@ -1,13 +1,7 @@ package org.dbsyncer.manager.puller; -import org.dbsyncer.common.event.ChangedEvent; import org.dbsyncer.common.event.ChangedOffset; -import org.dbsyncer.common.event.CommonChangedEvent; -import org.dbsyncer.common.event.DDLChangedEvent; import org.dbsyncer.common.event.RefreshOffsetEvent; -import org.dbsyncer.common.event.RowChangedEvent; -import org.dbsyncer.common.event.ScanChangedEvent; -import org.dbsyncer.common.event.Watcher; import org.dbsyncer.common.model.AbstractConnectorConfig; import org.dbsyncer.common.scheduled.ScheduledTaskJob; import org.dbsyncer.common.scheduled.ScheduledTaskService; @@ -21,16 +15,17 @@ import org.dbsyncer.listener.config.ListenerConfig; import org.dbsyncer.listener.enums.ListenerTypeEnum; import org.dbsyncer.listener.quartz.AbstractQuartzExtractor; import org.dbsyncer.listener.quartz.TableGroupQuartzCommand; -import org.dbsyncer.manager.Manager; import org.dbsyncer.manager.ManagerException; -import org.dbsyncer.manager.model.FieldPicker; +import org.dbsyncer.parser.ProfileComponent; +import org.dbsyncer.parser.consumer.impl.LogConsumer; +import org.dbsyncer.parser.consumer.impl.QuartzConsumer; +import org.dbsyncer.parser.flush.impl.BufferActuatorRouter; import org.dbsyncer.parser.logger.LogService; import org.dbsyncer.parser.logger.LogType; import org.dbsyncer.parser.model.Connector; import org.dbsyncer.parser.model.Mapping; import org.dbsyncer.parser.model.Meta; import org.dbsyncer.parser.model.TableGroup; -import org.dbsyncer.parser.util.PickerUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.ApplicationListener; @@ -42,13 +37,10 @@ import javax.annotation.Resource; import java.time.Instant; import java.util.ArrayList; import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import java.util.function.Consumer; import java.util.stream.Collectors; /** @@ -67,7 +59,7 @@ public final class IncrementPuller extends AbstractPuller implements Application private Listener listener; @Resource - private Manager manager; + private ProfileComponent profileComponent; @Resource private LogService logService; @@ -93,11 +85,11 @@ public final class IncrementPuller extends AbstractPuller implements Application final String mappingId = mapping.getId(); final String metaId = mapping.getMetaId(); logger.info("开始增量同步：{}, {}", metaId, mapping.getName()); - Connector connector = manager.getConnector(mapping.getSourceConnectorId()); + Connector connector = profileComponent.getConnector(mapping.getSourceConnectorId()); Assert.notNull(connector, "连接器不能为空."); - List list = manager.getSortedTableGroupAll(mappingId); + List list = profileComponent.getSortedTableGroupAll(mappingId); Assert.notEmpty(list, "映射关系不能为空."); - Meta meta = manager.getMeta(metaId); + Meta meta = profileComponent.getMeta(metaId); Assert.notNull(meta, "Meta不能为空."); Thread worker = new Thread(() -> { @@ -105,7 +97,7 @@ public final class IncrementPuller extends AbstractPuller implements Application long now = Instant.now().toEpochMilli(); meta.setBeginTime(now); meta.setEndTime(now); - manager.editConfigModel(meta); + profileComponent.editConfigModel(meta); map.putIfAbsent(metaId, getExtractor(mapping, connector, list, meta)); map.get(metaId).start(); } catch (Exception e) { @@ -160,14 +152,14 @@ public final class IncrementPuller extends AbstractPuller implements Application if (ListenerTypeEnum.isTiming(listenerType)) { AbstractQuartzExtractor quartzExtractor = listener.getExtractor(ListenerTypeEnum.TIMING, connectorConfig.getConnectorType(), AbstractQuartzExtractor.class); quartzExtractor.setCommands(list.stream().map(t -> new TableGroupQuartzCommand(t.getSourceTable(), t.getCommand())).collect(Collectors.toList())); - quartzExtractor.register(new QuartzConsumer(meta, mapping, list)); + quartzExtractor.register(new QuartzConsumer().init(bufferActuatorRouter, profileComponent, logService, meta, mapping, list)); extractor = quartzExtractor; } // 基于日志抽取 if (ListenerTypeEnum.isLog(listenerType)) { extractor = listener.getExtractor(ListenerTypeEnum.LOG, connectorConfig.getConnectorType(), AbstractExtractor.class); - extractor.register(new LogConsumer(meta, mapping, list)); + extractor.register(new LogConsumer().init(bufferActuatorRouter, profileComponent, logService, meta, mapping, list)); } if (null != extractor) { @@ -195,134 +187,4 @@ public final class IncrementPuller extends AbstractPuller implements Application throw new ManagerException("未知的监听配置."); } - abstract class AbstractConsumer implements Watcher { - protected Meta meta; - - public abstract void onChange(E e); - - public void onDDLChanged(DDLChangedEvent event) { - } - - @Override - public void changeEvent(ChangedEvent event) { - event.getChangedOffset().setMetaId(meta.getId()); - if (event instanceof DDLChangedEvent) { - onDDLChanged((DDLChangedEvent) event); - return; - } - onChange((E) event); - } - - @Override - public void flushEvent(Map snapshot) { - meta.setSnapshot(snapshot); - manager.editConfigModel(meta); - } - - @Override - public void errorEvent(Exception e) { - logService.log(LogType.TableGroupLog.INCREMENT_FAILED, e.getMessage()); - } - - @Override - public long getMetaUpdateTime() { - return meta.getUpdateTime(); - } - - protected void bind(String tableGroupId) { - bufferActuatorRouter.bind(meta.getId(), tableGroupId); - } - - protected void execute(String tableGroupId, ChangedEvent event) { - bufferActuatorRouter.execute(meta.getId(), tableGroupId, event); - } - - } - - final class QuartzConsumer extends AbstractConsumer { - private List tablePicker = new LinkedList<>(); - - public QuartzConsumer(Meta meta, Mapping mapping, List tableGroups) { - this.meta = meta; - tableGroups.forEach(t -> { - tablePicker.add(new FieldPicker(PickerUtil.mergeTableGroupConfig(mapping, t))); - bind(t.getId()); - }); - } - - @Override - public void onChange(ScanChangedEvent event) { - final FieldPicker picker = tablePicker.get(event.getTableGroupIndex()); - TableGroup tableGroup = picker.getTableGroup(); - event.setSourceTableName(tableGroup.getSourceTable().getName()); - - // 定时暂不支持触发刷新增量点事件 - execute(tableGroup.getId(), event); - } - } - - final class LogConsumer extends AbstractConsumer { - private Mapping mapping; - private List tableGroups; - private Map> tablePicker = new LinkedHashMap<>(); - - //判断上次是否为ddl，是ddl需要强制刷新下picker - private boolean ddlChanged; - - public LogConsumer(Meta meta, Mapping mapping, List tableGroups) { - this.tableGroups = tableGroups; - this.meta = meta; - this.mapping = mapping; - addTablePicker(true); - } - - @Override - public void onChange(RowChangedEvent event) { - // 需要强制刷新 fix https://gitee.com/ghi/dbsyncer/issues/I8DJUR - if (ddlChanged) { - addTablePicker(false); - ddlChanged = false; - } - process(event, picker -> { - final Map changedRow = picker.getColumns(event.getDataList()); - if (picker.filter(changedRow)) { - event.setChangedRow(changedRow); - execute(picker.getTableGroup().getId(), event); - } - }); - } - - @Override - public void onDDLChanged(DDLChangedEvent event) { - ddlChanged = true; - process(event, picker -> execute(picker.getTableGroup().getId(), event)); - } - - private void process(CommonChangedEvent event, Consumer consumer) { - // 处理过程有异常向上抛 - List pickers = tablePicker.get(event.getSourceTableName()); - if (!CollectionUtils.isEmpty(pickers)) { - // 触发刷新增量点事件 - event.getChangedOffset().setRefreshOffset(true); - pickers.forEach(picker -> consumer.accept(picker)); - } - } - - private void addTablePicker(boolean bindBufferActuatorRouter) { - this.tablePicker.clear(); - this.tableGroups.forEach(t -> { - final Table table = t.getSourceTable(); - final String tableName = table.getName(); - tablePicker.putIfAbsent(tableName, new ArrayList<>()); - TableGroup group = PickerUtil.mergeTableGroupConfig(mapping, t); - tablePicker.get(tableName).add(new FieldPicker(group, group.getFilter(), table.getColumn(), group.getFieldMapping())); - // 是否注册到路由服务中 - if (bindBufferActuatorRouter) { - bind(group.getId()); - } - }); - } - - } - } \ No newline at end of file diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/template/PreloadTemplate.java b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/template/PreloadTemplate.java index 6ce1e060..1f57b83b 100644 --- a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/template/PreloadTemplate.java +++ b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/template/PreloadTemplate.java @@ -4,17 +4,18 @@ import org.dbsyncer.common.event.PreloadCompletedEvent; import org.dbsyncer.common.model.Paging; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.JsonUtil; -import org.dbsyncer.manager.Manager; -import org.dbsyncer.manager.command.PreloadCommand; -import org.dbsyncer.manager.enums.CommandEnum; -import org.dbsyncer.manager.enums.GroupStrategyEnum; -import org.dbsyncer.manager.model.OperationConfig; -import org.dbsyncer.manager.template.OperationTemplate.Group; -import org.dbsyncer.parser.Parser; +import org.dbsyncer.manager.ManagerFactory; +import org.dbsyncer.parser.ProfileComponent; +import org.dbsyncer.parser.command.PreloadCommand; +import org.dbsyncer.parser.enums.CommandEnum; +import org.dbsyncer.parser.enums.GroupStrategyEnum; import org.dbsyncer.parser.enums.MetaEnum; import org.dbsyncer.parser.model.ConfigModel; import org.dbsyncer.parser.model.Mapping; import org.dbsyncer.parser.model.Meta; +import org.dbsyncer.parser.model.OperationConfig; +import org.dbsyncer.parser.template.OperationTemplate; +import org.dbsyncer.plugin.PluginFactory; import org.dbsyncer.storage.StorageService; import org.dbsyncer.storage.constant.ConfigConstant; import org.dbsyncer.storage.enums.StorageEnum; @@ -44,21 +45,75 @@ public final class PreloadTemplate implements ApplicationListener commandEnum.isPreload()).forEach(commandEnum -> execute(commandEnum)); + + // Load plugins + pluginFactory.loadPlugins(); + + // Launch drivers + launch(); + + // publish event + applicationContext.publishEvent(new PreloadCompletedEvent(applicationContext)); + } + + public void reload(String json) { + Map map = JsonUtil.jsonToObj(json, Map.class); + if (CollectionUtils.isEmpty(map)) { + return; + } + + // Load system + reload(map, CommandEnum.PRELOAD_SYSTEM); + // Load user + reload(map, CommandEnum.PRELOAD_USER); + // Load connectors + reload(map, CommandEnum.PRELOAD_CONNECTOR); + // Load mappings + reload(map, CommandEnum.PRELOAD_MAPPING); + // Load metas + reload(map, CommandEnum.PRELOAD_META); + // Load projectGroups + reload(map, CommandEnum.PRELOAD_PROJECT_GROUP); + launch(); + } + + private void launch() { + List metas = profileComponent.getMetaAll(); + if (!CollectionUtils.isEmpty(metas)) { + metas.forEach(m -> { + // 恢复驱动状态 + if (MetaEnum.RUNNING.getCode() == m.getState()) { + Mapping mapping = profileComponent.getMapping(m.getMappingId()); + managerFactory.start(mapping); + } else if (MetaEnum.STOPPING.getCode() == m.getState()) { + managerFactory.changeMetaState(m.getId(), MetaEnum.READY); + } + }); + } + } + + private void execute(CommandEnum commandEnum) { Query query = new Query(); query.setType(StorageEnum.CONFIG); String modelType = commandEnum.getModelType(); @@ -77,7 +132,7 @@ public final class PreloadTemplate implements ApplicationListener { String json = (String) map.get(ConfigConstant.CONFIG_MODEL_JSON); - ConfigModel model = (ConfigModel) commandEnum.getCommandExecutor().execute(new PreloadCommand(parser, json)); + ConfigModel model = (ConfigModel) commandEnum.getCommandExecutor().execute(new PreloadCommand(profileComponent, json)); if (null != model) { operationTemplate.cache(model, commandEnum.getGroupStrategyEnum()); } @@ -88,27 +143,6 @@ public final class PreloadTemplate implements ApplicationListener map = JsonUtil.jsonToObj(json, Map.class); - if (CollectionUtils.isEmpty(map)) { - return; - } - - // Load system - reload(map, CommandEnum.PRELOAD_SYSTEM); - // Load user - reload(map, CommandEnum.PRELOAD_USER); - // Load connectors - reload(map, CommandEnum.PRELOAD_CONNECTOR); - // Load mappings - reload(map, CommandEnum.PRELOAD_MAPPING); - // Load metas - reload(map, CommandEnum.PRELOAD_META); - // Load projectGroups - reload(map, CommandEnum.PRELOAD_PROJECT_GROUP); - launch(); - } - private void reload(Map map, CommandEnum commandEnum) { reload(map, commandEnum, commandEnum.getModelType()); } @@ -118,7 +152,7 @@ public final class PreloadTemplate implements ApplicationListener metas = manager.getMetaAll(); - if (!CollectionUtils.isEmpty(metas)) { - metas.forEach(m -> { - // 恢复驱动状态 - if (MetaEnum.RUNNING.getCode() == m.getState()) { - Mapping mapping = manager.getMapping(m.getMappingId()); - manager.start(mapping); - } else if (MetaEnum.STOPPING.getCode() == m.getState()) { - manager.changeMetaState(m.getId(), MetaEnum.READY); - } - }); - } - } - - @Override - public void onApplicationEvent(ContextRefreshedEvent contextRefreshedEvent) { - // Load configModels - Arrays.stream(CommandEnum.values()).filter(commandEnum -> commandEnum.isPreload()).forEach(commandEnum -> execute(commandEnum)); - - // Load plugins - manager.loadPlugins(); - - // Check connectors status - manager.checkAllConnectorStatus(); - - // Launch drivers - launch(); - - // publish event - applicationContext.publishEvent(new PreloadCompletedEvent(applicationContext)); - } - } \ No newline at end of file diff --git a/dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/Monitor.java b/dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/Monitor.java index 265e81e9..869cf1b1 100644 --- a/dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/Monitor.java +++ b/dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/Monitor.java @@ -1,16 +1,10 @@ package org.dbsyncer.monitor; -import org.dbsyncer.common.model.Paging; import org.dbsyncer.monitor.enums.MetricEnum; import org.dbsyncer.monitor.model.AppReportMetric; import org.dbsyncer.monitor.model.MetricResponse; -import org.dbsyncer.parser.model.Mapping; -import org.dbsyncer.parser.model.Meta; -import org.dbsyncer.parser.model.TableGroup; -import org.dbsyncer.storage.enums.StorageDataStatusEnum; import java.util.List; -import java.util.Map; /** * @author AE86 @@ -19,28 +13,6 @@ import java.util.Map; */ public interface Monitor { - Mapping getMapping(String mappingId); - - TableGroup getTableGroup(String tableGroupId); - - List getMetaAll(); - - Meta getMeta(String metaId); - - Paging queryData(String metaId, int pageNum, int pageSize, String error, String success); - - Map getData(String metaId, String messageId); - - void removeData(String metaId, String messageId); - - void clearData(String metaId); - - Paging queryLog(int pageNum, int pageSize, String json); - - void clearLog(); - - List getStorageDataStatusEnumAll(); - List getMetricEnumAll(); List getMetricInfo(); diff --git a/dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/MonitorFactory.java b/dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/MonitorFactory.java index fdb01584..142058aa 100644 --- a/dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/MonitorFactory.java +++ b/dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/MonitorFactory.java @@ -4,11 +4,8 @@ import org.dbsyncer.common.model.Paging; import org.dbsyncer.common.scheduled.ScheduledTaskJob; import org.dbsyncer.common.scheduled.ScheduledTaskService; import org.dbsyncer.common.util.CollectionUtils; -import org.dbsyncer.common.util.NumberUtil; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.connector.constant.ConnectorConstant; -import org.dbsyncer.manager.Manager; -import org.dbsyncer.manager.puller.BufferActuatorRouter; import org.dbsyncer.monitor.enums.BufferActuatorMetricEnum; import org.dbsyncer.monitor.enums.MetricEnum; import org.dbsyncer.monitor.enums.StatisticEnum; @@ -18,13 +15,15 @@ import org.dbsyncer.monitor.model.MappingReportMetric; import org.dbsyncer.monitor.model.MetricResponse; import org.dbsyncer.monitor.model.MetricResponseInfo; import org.dbsyncer.monitor.model.Sample; +import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.parser.flush.BufferActuator; +import org.dbsyncer.parser.flush.impl.BufferActuatorRouter; import org.dbsyncer.parser.flush.impl.TableGroupBufferActuator; import org.dbsyncer.parser.model.Mapping; import org.dbsyncer.parser.model.Meta; import org.dbsyncer.parser.model.TableGroup; +import org.dbsyncer.storage.StorageService; import org.dbsyncer.storage.constant.ConfigConstant; -import org.dbsyncer.storage.enums.IndexFieldResolverEnum; import org.dbsyncer.storage.enums.StorageDataStatusEnum; import org.dbsyncer.storage.enums.StorageEnum; import org.dbsyncer.storage.query.Query; @@ -38,11 +37,8 @@ import javax.annotation.Resource; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.Comparator; -import java.util.LinkedHashMap; import java.util.List; -import java.util.Map; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Consumer; @@ -59,7 +55,7 @@ public class MonitorFactory implements Monitor, ScheduledTaskJob { private final Logger logger = LoggerFactory.getLogger(getClass()); @Resource - private Manager manager; + private ProfileComponent profileComponent; @Resource private BufferActuator generalBufferActuator; @@ -73,6 +69,9 @@ public class MonitorFactory implements Monitor, ScheduledTaskJob { @Resource private ScheduledTaskService scheduledTaskService; + @Resource + private StorageService storageService; + private volatile boolean running; private LocalDateTime queryTime; @@ -86,92 +85,6 @@ public class MonitorFactory implements Monitor, ScheduledTaskJob { scheduledTaskService.start(5000, this); } - @Override - public Mapping getMapping(String mappingId) { - return manager.getMapping(mappingId); - } - - @Override - public TableGroup getTableGroup(String tableGroupId) { - return manager.getTableGroup(tableGroupId); - } - - @Override - public List getMetaAll() { - return manager.getMetaAll(); - } - - @Override - public Meta getMeta(String metaId) { - return manager.getMeta(metaId); - } - - @Override - public Paging queryData(String metaId, int pageNum, int pageSize, String error, String success) { - // 没有驱动 - if (StringUtil.isBlank(metaId)) { - return new Paging(pageNum, pageSize); - } - Query query = new Query(pageNum, pageSize); - Map fieldResolvers = new LinkedHashMap<>(); - fieldResolvers.put(ConfigConstant.BINLOG_DATA, IndexFieldResolverEnum.BINARY); - query.setIndexFieldResolverMap(fieldResolvers); - - // 查询异常信息 - if (StringUtil.isNotBlank(error)) { - query.addFilter(ConfigConstant.DATA_ERROR, error, true); - } - // 查询是否成功, 默认查询失败 - query.addFilter(ConfigConstant.DATA_SUCCESS, StringUtil.isNotBlank(success) ? NumberUtil.toInt(success) : StorageDataStatusEnum.FAIL.getValue()); - query.setMetaId(metaId); - return manager.queryData(query); - } - - @Override - public Map getData(String metaId, String messageId) { - Query query = new Query(1, 1); - Map fieldResolvers = new LinkedHashMap<>(); - fieldResolvers.put(ConfigConstant.BINLOG_DATA, IndexFieldResolverEnum.BINARY); - query.setIndexFieldResolverMap(fieldResolvers); - query.addFilter(ConfigConstant.CONFIG_MODEL_ID, messageId); - query.setMetaId(metaId); - Paging paging = manager.queryData(query); - if (!CollectionUtils.isEmpty(paging.getData())) { - List data = (List) paging.getData(); - return data.get(0); - } - return Collections.EMPTY_MAP; - } - - @Override - public void removeData(String metaId, String messageId) { - manager.removeData(metaId, messageId); - } - - @Override - public void clearData(String metaId) { - manager.clearData(metaId); - } - - @Override - public Paging queryLog(int pageNum, int pageSize, String json) { - Query query = new Query(pageNum, pageSize); - if (StringUtil.isNotBlank(json)) { - query.addFilter(ConfigConstant.CONFIG_MODEL_JSON, json, true); - } - return manager.queryLog(query); - } - - @Override - public void clearLog() { - manager.clearLog(); - } - - @Override - public List getStorageDataStatusEnumAll() { - return manager.getStorageDataStatusEnumAll(); - } - @Override public List getMetricEnumAll() { return Arrays.asList(MetricEnum.values()); @@ -188,12 +101,12 @@ public class MonitorFactory implements Monitor, ScheduledTaskJob { List tableList = new ArrayList<>(); String tableGroupCode = BufferActuatorMetricEnum.TABLE_GROUP.getCode(); bufferActuatorRouter.getRouter().forEach((metaId, group) -> { - Meta meta = manager.getMeta(metaId); - Mapping mapping = manager.getMapping(meta.getMappingId()); + Meta meta = profileComponent.getMeta(metaId); + Mapping mapping = profileComponent.getMapping(meta.getMappingId()); group.forEach((k, bufferActuator) -> { if (bufferActuator instanceof TableGroupBufferActuator) { TableGroupBufferActuator actuator = bufferActuator; - TableGroup tableGroup = manager.getTableGroup(actuator.getTableGroupId()); + TableGroup tableGroup = profileComponent.getTableGroup(actuator.getTableGroupId()); String metricName = new StringBuilder() .append(tableGroup.getSourceTable().getName()) .append(" > ") @@ -241,7 +154,7 @@ public class MonitorFactory implements Monitor, ScheduledTaskJob { // 刷新报表 try { running = true; - final List metaAll = manager.getMetaAll(); + final List metaAll = profileComponent.getMetaAll(); mappingReportMetric.setSuccess(getMappingSuccess(metaAll)); mappingReportMetric.setFail(getMappingFail(metaAll)); mappingReportMetric.setInsert(getMappingInsert(metaAll)); @@ -313,7 +226,7 @@ public class MonitorFactory implements Monitor, ScheduledTaskJob { operation.accept(query); metaAll.forEach(meta -> { query.setMetaId(meta.getId()); - Paging paging = manager.queryData(query); + Paging paging = storageService.query(query); total.getAndAdd(paging.getTotal()); }); } diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/Parser.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ParserComponent.java similarity index 50% rename from dbsyncer-parser/src/main/java/org/dbsyncer/parser/Parser.java rename to dbsyncer-parser/src/main/java/org/dbsyncer/parser/ParserComponent.java index 3e9026d1..bdebeaf7 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/Parser.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ParserComponent.java @@ -1,57 +1,30 @@ package org.dbsyncer.parser; import org.dbsyncer.common.event.ChangedEvent; -import org.dbsyncer.common.model.AbstractConnectorConfig; import org.dbsyncer.common.model.Result; import org.dbsyncer.common.spi.ConnectorMapper; import org.dbsyncer.common.spi.ConvertContext; -import org.dbsyncer.connector.enums.ConnectorEnum; -import org.dbsyncer.connector.enums.FilterEnum; -import org.dbsyncer.connector.enums.OperationEnum; import org.dbsyncer.connector.model.MetaInfo; import org.dbsyncer.connector.model.Table; -import org.dbsyncer.listener.enums.QuartzFilterEnum; -import org.dbsyncer.parser.enums.ConvertEnum; import org.dbsyncer.parser.model.BatchWriter; -import org.dbsyncer.parser.model.Connector; import org.dbsyncer.parser.model.Mapping; import org.dbsyncer.parser.model.TableGroup; import org.dbsyncer.parser.model.Task; -import org.dbsyncer.storage.enums.StorageDataStatusEnum; import java.util.List; import java.util.Map; import java.util.concurrent.Executor; /** + * 
+ *     实现全量和增量同步任务，以及连接器读写，插件转换，表元信息，心跳检测
+ *
+ * * @author AE86 * @version 1.0.0 * @date 2019/10/20 21:11 */ -public interface Parser { - - /** - * 获取连接配置 - * - * @param config - * @return - */ - ConnectorMapper connect(AbstractConnectorConfig config); - - /** - * 刷新连接配置 - * - * @param config - */ - boolean refreshConnectorConfig(AbstractConnectorConfig config); - - /** - * 连接配置是否可用 - * - * @param config - * @return - */ - boolean isAliveConnectorConfig(AbstractConnectorConfig config); +public interface ParserComponent { /** * 获取连接器表 @@ -88,66 +61,6 @@ public interface Parser { */ long getCount(String connectorId, Map command); - /** - * 解析连接器配置为Connector - * - * @param json - * @return - */ - Connector parseConnector(String json); - - /** - * 解析配置 - * - * @param json - * @param clazz - * @param - * @return - */ - T parseObject(String json, Class clazz); - - /** - * 获取所有连接器类型 - * - * @return - */ - List getConnectorEnumAll(); - - /** - * 获取所有条件类型 - * - * @return - */ - List getOperationEnumAll(); - - /** - * 获取过滤条件系统参数 - * - * @return - */ - List getQuartzFilterEnumAll(); - - /** - * 获取所有运算符类型 - * - * @return - */ - List getFilterEnumAll(); - - /** - * 获取所有转换类型 - * - * @return - */ - List getConvertEnumAll(); - - /** - * 获取所有同步数据状态类型 - * - * @return - */ - List getStorageDataStatusEnumAll(); - /** * 全量同步 * diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/Manager.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ProfileComponent.java similarity index 48% rename from dbsyncer-manager/src/main/java/org/dbsyncer/manager/Manager.java rename to dbsyncer-parser/src/main/java/org/dbsyncer/parser/ProfileComponent.java index d6466fa2..5842b5c3 100644 --- a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/Manager.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ProfileComponent.java @@ -1,31 +1,54 @@ -package org.dbsyncer.manager; +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.parser; -import org.dbsyncer.common.model.AbstractConnectorConfig; -import org.dbsyncer.common.model.Paging; -import org.dbsyncer.common.spi.ConnectorMapper; import org.dbsyncer.connector.enums.ConnectorEnum; import org.dbsyncer.connector.enums.FilterEnum; import org.dbsyncer.connector.enums.OperationEnum; -import org.dbsyncer.connector.model.MetaInfo; -import org.dbsyncer.connector.model.Table; import org.dbsyncer.listener.enums.QuartzFilterEnum; import org.dbsyncer.parser.enums.ConvertEnum; -import org.dbsyncer.parser.model.*; -import org.dbsyncer.plugin.config.Plugin; +import org.dbsyncer.parser.model.ConfigModel; +import org.dbsyncer.parser.model.Connector; +import org.dbsyncer.parser.model.Mapping; +import org.dbsyncer.parser.model.Meta; +import org.dbsyncer.parser.model.ProjectGroup; +import org.dbsyncer.parser.model.SystemConfig; +import org.dbsyncer.parser.model.TableGroup; +import org.dbsyncer.parser.model.UserConfig; +import org.dbsyncer.storage.constant.ConfigConstant; import org.dbsyncer.storage.enums.StorageDataStatusEnum; -import org.dbsyncer.storage.query.Query; import java.util.List; -import java.util.Map; /** - * 驱动配置 + * 配置文件组件（system/user/connector/mapping/tableGroup/meta/projectGroup） + *

+ * {@link ConfigConstant} * - * @author AE86 - * @version 1.0.0 - * @date 2019/9/30 20:31 + * @Version 1.0.0 + * @Author AE86 + * @Date 2023-11-13 20:48 */ -public interface Manager extends Executor { +public interface ProfileComponent { + + /** + * 解析连接器配置为Connector + * + * @param json + * @return + */ + Connector parseConnector(String json); + + /** + * 解析配置 + * + * @param json + * @param clazz + * @param + * @return + */ + T parseObject(String json, Class clazz); /** * 添加ConfigModel @@ -46,39 +69,55 @@ public interface Manager extends Executor { /** * 刪除ConfigModel * - * @param id + * @param id * @return */ void removeConfigModel(String id); - // system + /** + * 获取系统配置 + * + * @return + */ SystemConfig getSystemConfig(); - // user + /** + * 获取用户配置 + * + * @return + */ UserConfig getUserConfig(); - // project group + /** + * 根据ID获取分组配置 + * + * @param id + * @return + */ ProjectGroup getProjectGroup(String id); + /** + * 获取所有的分组配置 + * + * @return + */ List getProjectGroupAll(); - // Connector - ConnectorMapper connect(AbstractConnectorConfig config); - - boolean refreshConnectorConfig(AbstractConnectorConfig config); - - boolean isAliveConnectorConfig(AbstractConnectorConfig config); - - List

getTable(ConnectorMapper config); - - MetaInfo getMetaInfo(String connectorId, String tableName); - + /** + * 根据ID获取连接器 + * + * @param connectorId + * @return + */ Connector getConnector(String connectorId); + /** + * 获取所有的连接器 + * + * @return + */ List getConnectorAll(); - void checkAllConnectorStatus(); - // Mapping Mapping getMapping(String mappingId); @@ -99,51 +138,50 @@ public interface Manager extends Executor { int getTableGroupCount(String mappingId); - Map getCommand(Mapping mapping, TableGroup tableGroup); - - long getCount(String connectorId, Map command); - // Meta Meta getMeta(String metaId); List getMetaAll(); - // Data - Paging queryData(Query query); - - void removeData(String metaId, String messageId); - - void clearData(String metaId); - - // Log - Paging queryLog(Query query); - - void clearLog(); - - // ConnectorEnum + /** + * 获取所有连接器类型 + * + * @return + */ List getConnectorEnumAll(); - // OperationEnum + /** + * 获取所有条件类型 + * + * @return + */ List getOperationEnumAll(); - // QuartzFilterEnum + /** + * 获取过滤条件系统参数 + * + * @return + */ List getQuartzFilterEnumAll(); - // FilterEnum + /** + * 获取所有运算符类型 + * + * @return + */ List getFilterEnumAll(); - // ConvertEnum + /** + * 获取所有转换类型 + * + * @return + */ List getConvertEnumAll(); - // StorageDataStatusEnum + /** + * 获取所有同步数据状态类型 + * + * @return + */ List getStorageDataStatusEnumAll(); - - // Plugin - List getPluginAll(); - - String getPluginPath(); - - String getLibraryPath(); - - void loadPlugins(); } \ No newline at end of file diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/command/Command.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/command/Command.java new file mode 100644 index 00000000..3c33e05e --- /dev/null +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/command/Command.java @@ -0,0 +1,14 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.parser.command; + +/** + * 命令模式接口 + * + * @Version 1.0.0 + * @Author AE86 + * @Date 2023-11-12 01:32 + */ +public interface Command extends Persistence, Preload { +} \ No newline at end of file diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/command/CommandExecutor.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/command/CommandExecutor.java new file mode 100644 index 00000000..6ed55a9a --- /dev/null +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/command/CommandExecutor.java @@ -0,0 +1,14 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.parser.command; + +/** + * @Version 1.0.0 + * @Author AE86 + * @Date 2023-11-12 01:32 + */ +public interface CommandExecutor { + + Object execute(Command cmd); +} \ No newline at end of file diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/command/Persistence.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/command/Persistence.java new file mode 100644 index 00000000..cfc3b38d --- /dev/null +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/command/Persistence.java @@ -0,0 +1,24 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.parser.command; + +import org.dbsyncer.parser.ParserException; + +/** + * 序列化接口 + * + * @Version 1.0.0 + * @Author AE86 + * @Date 2023-11-12 01:32 + */ +public interface Persistence { + + default boolean addConfig() { + throw new ParserException("Unsupported method addConfig"); + } + + default boolean editConfig() { + throw new ParserException("Unsupported method editConfig"); + } +} \ No newline at end of file diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/command/PersistenceCommand.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/command/PersistenceCommand.java similarity index 77% rename from dbsyncer-manager/src/main/java/org/dbsyncer/manager/command/PersistenceCommand.java rename to dbsyncer-parser/src/main/java/org/dbsyncer/parser/command/PersistenceCommand.java index ff907b84..e207a1ef 100644 --- a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/command/PersistenceCommand.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/command/PersistenceCommand.java @@ -1,11 +1,20 @@ -package org.dbsyncer.manager.command; +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.parser.command; -import org.dbsyncer.manager.Command; import org.dbsyncer.storage.StorageService; import org.dbsyncer.storage.enums.StorageEnum; import java.util.Map; +/** + * 配置序列化 + * + * @Version 1.0.0 + * @Author AE86 + * @Date 2023-11-12 01:32 + */ public class PersistenceCommand implements Command { private StorageService storageService; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/command/Preload.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/command/Preload.java new file mode 100644 index 00000000..2c012ea4 --- /dev/null +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/command/Preload.java @@ -0,0 +1,51 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.parser.command; + +import org.dbsyncer.parser.ParserException; +import org.dbsyncer.parser.model.Connector; +import org.dbsyncer.parser.model.Mapping; +import org.dbsyncer.parser.model.Meta; +import org.dbsyncer.parser.model.ProjectGroup; +import org.dbsyncer.parser.model.SystemConfig; +import org.dbsyncer.parser.model.TableGroup; +import org.dbsyncer.parser.model.UserConfig; + +/** + * 预加载接口 + * + * @Version 1.0.0 + * @Author AE86 + * @Date 2023-11-12 01:32 + */ +public interface Preload { + + default SystemConfig parseSystemConfig(){ + throw new ParserException("Unsupported method parseSystemConfig"); + } + + default UserConfig parseUserConfig(){ + throw new ParserException("Unsupported method parseUserConfig"); + } + + default Connector parseConnector(){ + throw new ParserException("Unsupported method parseConnector"); + } + + default Mapping parseMapping(){ + throw new ParserException("Unsupported method parseMapping"); + } + + default TableGroup parseTableGroup(){ + throw new ParserException("Unsupported method parseTableGroup"); + } + + default Meta parseMeta(){ + throw new ParserException("Unsupported method parseMeta"); + } + + default ProjectGroup parseProjectGroup(){ + throw new ParserException("Unsupported method parseProjectGroup"); + } +} \ No newline at end of file diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/command/PreloadCommand.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/command/PreloadCommand.java similarity index 49% rename from dbsyncer-manager/src/main/java/org/dbsyncer/manager/command/PreloadCommand.java rename to dbsyncer-parser/src/main/java/org/dbsyncer/parser/command/PreloadCommand.java index 4c6af555..fba38307 100644 --- a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/command/PreloadCommand.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/command/PreloadCommand.java @@ -1,7 +1,9 @@ -package org.dbsyncer.manager.command; +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.parser.command; -import org.dbsyncer.manager.Command; -import org.dbsyncer.parser.Parser; +import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.parser.model.Connector; import org.dbsyncer.parser.model.Mapping; import org.dbsyncer.parser.model.Meta; @@ -10,50 +12,57 @@ import org.dbsyncer.parser.model.SystemConfig; import org.dbsyncer.parser.model.TableGroup; import org.dbsyncer.parser.model.UserConfig; +/** + * 预加载接口 + * + * @Version 1.0.0 + * @Author AE86 + * @Date 2023-11-12 01:32 + */ public class PreloadCommand implements Command { - private Parser parser; + private ProfileComponent profileComponent; private String json; - public PreloadCommand(Parser parser, String json) { - this.parser = parser; + public PreloadCommand(ProfileComponent profileComponent, String json) { + this.profileComponent = profileComponent; this.json = json; } @Override public SystemConfig parseSystemConfig() { - return parser.parseObject(json, SystemConfig.class); + return profileComponent.parseObject(json, SystemConfig.class); } @Override public UserConfig parseUserConfig() { - return parser.parseObject(json, UserConfig.class); + return profileComponent.parseObject(json, UserConfig.class); } @Override public Connector parseConnector() { - return parser.parseConnector(json); + return profileComponent.parseConnector(json); } @Override public Mapping parseMapping() { - return parser.parseObject(json, Mapping.class); + return profileComponent.parseObject(json, Mapping.class); } @Override public TableGroup parseTableGroup() { - return parser.parseObject(json, TableGroup.class); + return profileComponent.parseObject(json, TableGroup.class); } @Override public Meta parseMeta() { - return parser.parseObject(json, Meta.class); + return profileComponent.parseObject(json, Meta.class); } @Override public ProjectGroup parseProjectGroup() { - return parser.parseObject(json, ProjectGroup.class); + return profileComponent.parseObject(json, ProjectGroup.class); } } \ No newline at end of file diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/AbstractConsumer.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/AbstractConsumer.java new file mode 100644 index 00000000..7ec9e137 --- /dev/null +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/AbstractConsumer.java @@ -0,0 +1,84 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.parser.consumer; + +import org.dbsyncer.common.event.ChangedEvent; +import org.dbsyncer.common.event.DDLChangedEvent; +import org.dbsyncer.common.event.Watcher; +import org.dbsyncer.parser.ProfileComponent; +import org.dbsyncer.parser.flush.impl.BufferActuatorRouter; +import org.dbsyncer.parser.logger.LogService; +import org.dbsyncer.parser.logger.LogType; +import org.dbsyncer.parser.model.Mapping; +import org.dbsyncer.parser.model.Meta; +import org.dbsyncer.parser.model.TableGroup; + +import java.util.List; +import java.util.Map; + +/** + * @Version 1.0.0 + * @Author AE86 + * @Date 2023-11-12 01:32 + */ +public abstract class AbstractConsumer implements Watcher { + private BufferActuatorRouter bufferActuatorRouter; + private ProfileComponent profileComponent; + private LogService logService; + private Meta meta; + protected Mapping mapping; + protected List tableGroups; + + public AbstractConsumer init(BufferActuatorRouter bufferActuatorRouter, ProfileComponent profileComponent, LogService logService, Meta meta, Mapping mapping, List tableGroups) { + this.bufferActuatorRouter = bufferActuatorRouter; + this.profileComponent = profileComponent; + this.logService = logService; + this.meta = meta; + this.mapping = mapping; + this.tableGroups = tableGroups; + postProcessBeforeInitialization(); + return this; + } + + public abstract void postProcessBeforeInitialization(); + + public abstract void onChange(E e); + + public void onDDLChanged(DDLChangedEvent event) { + } + + @Override + public void changeEvent(ChangedEvent event) { + event.getChangedOffset().setMetaId(meta.getId()); + if (event instanceof DDLChangedEvent) { + onDDLChanged((DDLChangedEvent) event); + return; + } + onChange((E) event); + } + + @Override + public void flushEvent(Map snapshot) { + meta.setSnapshot(snapshot); + profileComponent.editConfigModel(meta); + } + + @Override + public void errorEvent(Exception e) { + logService.log(LogType.TableGroupLog.INCREMENT_FAILED, e.getMessage()); + } + + @Override + public long getMetaUpdateTime() { + return meta.getUpdateTime(); + } + + protected void bind(String tableGroupId) { + bufferActuatorRouter.bind(meta.getId(), tableGroupId); + } + + protected void execute(String tableGroupId, ChangedEvent event) { + bufferActuatorRouter.execute(meta.getId(), tableGroupId, event); + } +} \ No newline at end of file diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/impl/LogConsumer.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/impl/LogConsumer.java new file mode 100644 index 00000000..eb664a44 --- /dev/null +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/impl/LogConsumer.java @@ -0,0 +1,87 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.parser.consumer.impl; + +import org.dbsyncer.common.event.CommonChangedEvent; +import org.dbsyncer.common.event.DDLChangedEvent; +import org.dbsyncer.common.event.RowChangedEvent; +import org.dbsyncer.common.util.CollectionUtils; +import org.dbsyncer.connector.model.Table; +import org.dbsyncer.parser.consumer.AbstractConsumer; +import org.dbsyncer.parser.model.FieldPicker; +import org.dbsyncer.parser.model.TableGroup; +import org.dbsyncer.parser.util.PickerUtil; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; + +/** + * 日志消费 + * + * @Version 1.0.0 + * @Author AE86 + * @Date 2023-11-12 02:25 + */ +public final class LogConsumer extends AbstractConsumer { + private Map> tablePicker = new LinkedHashMap<>(); + + //判断上次是否为ddl，是ddl需要强制刷新下picker + private boolean ddlChanged; + + @Override + public void postProcessBeforeInitialization() { + addTablePicker(true); + } + + @Override + public void onChange(RowChangedEvent event) { + // 需要强制刷新 fix https://gitee.com/ghi/dbsyncer/issues/I8DJUR + if (ddlChanged) { + addTablePicker(false); + ddlChanged = false; + } + process(event, picker -> { + final Map changedRow = picker.getColumns(event.getDataList()); + if (picker.filter(changedRow)) { + event.setChangedRow(changedRow); + execute(picker.getTableGroup().getId(), event); + } + }); + } + + @Override + public void onDDLChanged(DDLChangedEvent event) { + ddlChanged = true; + process(event, picker -> execute(picker.getTableGroup().getId(), event)); + } + + private void process(CommonChangedEvent event, Consumer consumer) { + // 处理过程有异常向上抛 + List pickers = tablePicker.get(event.getSourceTableName()); + if (!CollectionUtils.isEmpty(pickers)) { + // 触发刷新增量点事件 + event.getChangedOffset().setRefreshOffset(true); + pickers.forEach(picker -> consumer.accept(picker)); + } + } + + private void addTablePicker(boolean bindBufferActuatorRouter) { + this.tablePicker.clear(); + this.tableGroups.forEach(t -> { + final Table table = t.getSourceTable(); + final String tableName = table.getName(); + tablePicker.putIfAbsent(tableName, new ArrayList<>()); + TableGroup group = PickerUtil.mergeTableGroupConfig(mapping, t); + tablePicker.get(tableName).add(new FieldPicker(group, group.getFilter(), table.getColumn(), group.getFieldMapping())); + // 是否注册到路由服务中 + if (bindBufferActuatorRouter) { + bind(group.getId()); + } + }); + } + +} \ No newline at end of file diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/impl/QuartzConsumer.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/impl/QuartzConsumer.java new file mode 100644 index 00000000..5bb77a6c --- /dev/null +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/impl/QuartzConsumer.java @@ -0,0 +1,42 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.parser.consumer.impl; + +import org.dbsyncer.common.event.ScanChangedEvent; +import org.dbsyncer.parser.consumer.AbstractConsumer; +import org.dbsyncer.parser.model.FieldPicker; +import org.dbsyncer.parser.model.TableGroup; +import org.dbsyncer.parser.util.PickerUtil; + +import java.util.LinkedList; +import java.util.List; + +/** + * 定时消费 + * + * @Version 1.0.0 + * @Author AE86 + * @Date 2023-11-12 02:18 + */ +public final class QuartzConsumer extends AbstractConsumer { + private List tablePicker = new LinkedList<>(); + + @Override + public void postProcessBeforeInitialization() { + tableGroups.forEach(t -> { + tablePicker.add(new FieldPicker(PickerUtil.mergeTableGroupConfig(mapping, t))); + bind(t.getId()); + }); + } + + @Override + public void onChange(ScanChangedEvent event) { + final FieldPicker picker = tablePicker.get(event.getTableGroupIndex()); + TableGroup tableGroup = picker.getTableGroup(); + event.setSourceTableName(tableGroup.getSourceTable().getName()); + + // 定时暂不支持触发刷新增量点事件 + execute(tableGroup.getId(), event); + } +} \ No newline at end of file diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/enums/CommandEnum.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/enums/CommandEnum.java similarity index 91% rename from dbsyncer-manager/src/main/java/org/dbsyncer/manager/enums/CommandEnum.java rename to dbsyncer-parser/src/main/java/org/dbsyncer/parser/enums/CommandEnum.java index 37585935..afa27bd4 100644 --- a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/enums/CommandEnum.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/enums/CommandEnum.java @@ -1,8 +1,11 @@ -package org.dbsyncer.manager.enums; +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.parser.enums; -import org.dbsyncer.manager.CommandExecutor; -import org.dbsyncer.manager.command.Persistence; -import org.dbsyncer.manager.command.Preload; +import org.dbsyncer.parser.command.CommandExecutor; +import org.dbsyncer.parser.command.Persistence; +import org.dbsyncer.parser.command.Preload; import org.dbsyncer.storage.constant.ConfigConstant; /** diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/enums/GroupStrategyEnum.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/enums/GroupStrategyEnum.java similarity index 63% rename from dbsyncer-manager/src/main/java/org/dbsyncer/manager/enums/GroupStrategyEnum.java rename to dbsyncer-parser/src/main/java/org/dbsyncer/parser/enums/GroupStrategyEnum.java index 97cc5bdb..6fdfacdf 100644 --- a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/enums/GroupStrategyEnum.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/enums/GroupStrategyEnum.java @@ -1,9 +1,12 @@ -package org.dbsyncer.manager.enums; +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.parser.enums; -import org.dbsyncer.manager.GroupStrategy; -import org.dbsyncer.manager.strategy.DefaultGroupStrategy; -import org.dbsyncer.manager.strategy.PreloadTableGroupStrategy; -import org.dbsyncer.manager.strategy.TableGroupStrategy; +import org.dbsyncer.parser.strategy.GroupStrategy; +import org.dbsyncer.parser.strategy.impl.DefaultGroupStrategy; +import org.dbsyncer.parser.strategy.impl.PreloadTableGroupStrategy; +import org.dbsyncer.parser.strategy.impl.TableGroupStrategy; /** * @author AE86 diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/puller/BufferActuatorRouter.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/BufferActuatorRouter.java similarity index 89% rename from dbsyncer-manager/src/main/java/org/dbsyncer/manager/puller/BufferActuatorRouter.java rename to dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/BufferActuatorRouter.java index 4b853ec6..04d73611 100644 --- a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/puller/BufferActuatorRouter.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/BufferActuatorRouter.java @@ -1,9 +1,11 @@ -package org.dbsyncer.manager.puller; +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.parser.flush.impl; import org.dbsyncer.common.config.TableGroupBufferConfig; import org.dbsyncer.common.event.ChangedEvent; -import org.dbsyncer.parser.Parser; -import org.dbsyncer.parser.flush.impl.TableGroupBufferActuator; +import org.dbsyncer.parser.flush.BufferActuator; import org.dbsyncer.parser.model.WriterRequest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -16,6 +18,13 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicLong; +/** + * 缓存执行器路由 + * + * @Version 1.0.0 + * @Author AE86 + * @Date 2023-11-12 01:32 + */ @Component public final class BufferActuatorRouter implements DisposableBean { @@ -28,7 +37,7 @@ public final class BufferActuatorRouter implements DisposableBean { private TableGroupBufferActuator tableGroupBufferActuator; @Resource - private Parser parser; + private BufferActuator generalBufferActuator; /** * 驱动缓存执行路由列表 @@ -40,7 +49,7 @@ public final class BufferActuatorRouter implements DisposableBean { router.get(metaId).get(tableGroupId).offer(new WriterRequest(tableGroupId, event)); return; } - parser.execute(tableGroupId, event); + generalBufferActuator.offer(new WriterRequest(tableGroupId, event)); } public void bind(String metaId, String tableGroupId) { diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/GeneralBufferActuator.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/GeneralBufferActuator.java index 56d0bf3a..b03f3771 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/GeneralBufferActuator.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/GeneralBufferActuator.java @@ -14,7 +14,7 @@ import org.dbsyncer.connector.config.DDLConfig; import org.dbsyncer.connector.constant.ConnectorConstant; import org.dbsyncer.connector.enums.ConnectorEnum; import org.dbsyncer.connector.model.MetaInfo; -import org.dbsyncer.parser.Parser; +import org.dbsyncer.parser.ParserComponent; import org.dbsyncer.parser.ddl.DDLParser; import org.dbsyncer.parser.flush.AbstractBufferActuator; import org.dbsyncer.parser.model.BatchWriter; @@ -68,7 +68,7 @@ public class GeneralBufferActuator extends AbstractBufferActuator commands = parser.getCommand(mapping, tableGroup); + Map commands = parserComponent.getCommand(mapping, tableGroup); tableGroup.setCommand(commands); // 5.持久化存储 & 更新缓存配置 diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ParserFactory.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/ParserComponentImpl.java similarity index 75% rename from dbsyncer-parser/src/main/java/org/dbsyncer/parser/ParserFactory.java rename to dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/ParserComponentImpl.java index 86c11523..47d2d989 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ParserFactory.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/ParserComponentImpl.java @@ -1,6 +1,5 @@ -package org.dbsyncer.parser; +package org.dbsyncer.parser.impl; -import org.dbsyncer.cache.CacheService; import org.dbsyncer.common.event.ChangedEvent; import org.dbsyncer.common.model.AbstractConnectorConfig; import org.dbsyncer.common.model.FullConvertContext; @@ -8,26 +7,20 @@ import org.dbsyncer.common.model.Result; import org.dbsyncer.common.spi.ConnectorMapper; import org.dbsyncer.common.spi.ConvertContext; import org.dbsyncer.common.util.CollectionUtils; -import org.dbsyncer.common.util.JsonUtil; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.connector.ConnectorFactory; import org.dbsyncer.connector.config.CommandConfig; import org.dbsyncer.connector.config.ReaderConfig; import org.dbsyncer.connector.config.WriterBatchConfig; import org.dbsyncer.connector.constant.ConnectorConstant; -import org.dbsyncer.connector.enums.ConnectorEnum; -import org.dbsyncer.connector.enums.FilterEnum; -import org.dbsyncer.connector.enums.OperationEnum; import org.dbsyncer.connector.model.Field; import org.dbsyncer.connector.model.MetaInfo; import org.dbsyncer.connector.model.Table; import org.dbsyncer.connector.util.PrimaryKeyUtil; -import org.dbsyncer.listener.enums.QuartzFilterEnum; -import org.dbsyncer.parser.enums.ConvertEnum; +import org.dbsyncer.parser.ParserComponent; +import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.parser.event.FullRefreshEvent; import org.dbsyncer.parser.flush.BufferActuator; -import org.dbsyncer.parser.logger.LogService; -import org.dbsyncer.parser.logger.LogType; import org.dbsyncer.parser.model.BatchWriter; import org.dbsyncer.parser.model.Connector; import org.dbsyncer.parser.model.FieldMapping; @@ -40,7 +33,6 @@ import org.dbsyncer.parser.strategy.FlushStrategy; import org.dbsyncer.parser.util.ConvertUtil; import org.dbsyncer.parser.util.PickerUtil; import org.dbsyncer.plugin.PluginFactory; -import org.dbsyncer.storage.enums.StorageDataStatusEnum; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.ApplicationContext; @@ -50,7 +42,6 @@ import org.springframework.util.Assert; import javax.annotation.Resource; import java.time.Instant; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.concurrent.CountDownLatch; @@ -62,7 +53,7 @@ import java.util.concurrent.Executor; * @date 2019/9/29 22:38 */ @Component -public class ParserFactory implements Parser { +public class ParserComponentImpl implements ParserComponent { private final Logger logger = LoggerFactory.getLogger(getClass()); @@ -73,13 +64,10 @@ public class ParserFactory implements Parser { private PluginFactory pluginFactory; @Resource - private CacheService cacheService; - - @Resource - private LogService logService; + private FlushStrategy flushStrategy; @Resource - private FlushStrategy flushStrategy; + private ProfileComponent profileComponent; @Resource private ApplicationContext applicationContext; @@ -87,39 +75,6 @@ public class ParserFactory implements Parser { @Resource private BufferActuator generalBufferActuator; - @Override - public ConnectorMapper connect(AbstractConnectorConfig config) { - return connectorFactory.connect(config); - } - - @Override - public boolean refreshConnectorConfig(AbstractConnectorConfig config) { - return connectorFactory.refresh(config); - } - - @Override - public boolean isAliveConnectorConfig(AbstractConnectorConfig config) { - boolean alive = false; - try { - alive = connectorFactory.isAlive(config); - } catch (Exception e) { - LogType.ConnectorLog logType = LogType.ConnectorLog.FAILED; - logService.log(logType, "%s%s", logType.getName(), e.getMessage()); - } - // 断线重连 - if (!alive) { - try { - alive = connectorFactory.refresh(config); - } catch (Exception e) { - logger.error(e.getMessage()); - } - if (alive) { - logger.info(LogType.ConnectorLog.RECONNECT_SUCCESS.getMessage()); - } - } - return alive; - } - @Override public List
getTable(ConnectorMapper config) { return connectorFactory.getTable(config); @@ -127,7 +82,7 @@ public class ParserFactory implements Parser { @Override public MetaInfo getMetaInfo(String connectorId, String tableName) { - Connector connector = getConnector(connectorId); + Connector connector = profileComponent.getConnector(connectorId); ConnectorMapper connectorMapper = connectorFactory.connect(connector.getConfig()); MetaInfo metaInfo = connectorFactory.getMetaInfo(connectorMapper, tableName); if (!CollectionUtils.isEmpty(connector.getTable())) { @@ -174,56 +129,6 @@ public class ParserFactory implements Parser { return connectorFactory.getCount(connectorMapper, command); } - @Override - public Connector parseConnector(String json) { - Map conn = JsonUtil.parseMap(json); - Map config = (Map) conn.remove("config"); - Connector connector = JsonUtil.jsonToObj(conn.toString(), Connector.class); - Assert.notNull(connector, "Connector can not be null."); - String connectorType = (String) config.get("connectorType"); - ConnectorEnum connectorEnum = ConnectorEnum.getConnectorEnum(connectorType); - AbstractConnectorConfig obj = JsonUtil.jsonToObj(config.toString(), connectorEnum.getConfigClass()); - obj.setConnectorType(connectorEnum.getType()); - connector.setConfig(obj); - - return connector; - } - - @Override - public T parseObject(String json, Class clazz) { - return JsonUtil.jsonToObj(json, clazz); - } - - @Override - public List getConnectorEnumAll() { - return Arrays.asList(ConnectorEnum.values()); - } - - @Override - public List getOperationEnumAll() { - return Arrays.asList(OperationEnum.values()); - } - - @Override - public List getQuartzFilterEnumAll() { - return Arrays.asList(QuartzFilterEnum.values()); - } - - @Override - public List getFilterEnumAll() { - return Arrays.asList(FilterEnum.values()); - } - - @Override - public List getConvertEnumAll() { - return Arrays.asList(ConvertEnum.values()); - } - - @Override - public List getStorageDataStatusEnumAll() { - return Arrays.asList(StorageDataStatusEnum.values()); - } - @Override public void execute(Task task, Mapping mapping, TableGroup tableGroup, Executor executor) { final String metaId = task.getId(); @@ -380,19 +285,6 @@ public class ParserFactory implements Parser { applicationContext.publishEvent(new FullRefreshEvent(applicationContext, task)); } - /** - * 获取连接器 - * - * @param connectorId - * @return - */ - private Connector getConnector(String connectorId) { - Assert.hasText(connectorId, "Connector id can not be empty."); - Connector conn = cacheService.get(connectorId, Connector.class); - Assert.notNull(conn, "Connector can not be null."); - return conn; - } - /** * 获取连接配置 * @@ -400,7 +292,7 @@ public class ParserFactory implements Parser { * @return */ private AbstractConnectorConfig getConnectorConfig(String connectorId) { - return getConnector(connectorId).getConfig(); + return profileComponent.getConnector(connectorId).getConfig(); } } \ No newline at end of file diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/ProfileComponentImpl.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/ProfileComponentImpl.java new file mode 100644 index 00000000..383fb5e4 --- /dev/null +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/ProfileComponentImpl.java @@ -0,0 +1,208 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.parser.impl; + +import org.dbsyncer.common.model.AbstractConnectorConfig; +import org.dbsyncer.common.util.CollectionUtils; +import org.dbsyncer.common.util.JsonUtil; +import org.dbsyncer.connector.enums.ConnectorEnum; +import org.dbsyncer.connector.enums.FilterEnum; +import org.dbsyncer.connector.enums.OperationEnum; +import org.dbsyncer.listener.enums.QuartzFilterEnum; +import org.dbsyncer.parser.ProfileComponent; +import org.dbsyncer.parser.enums.CommandEnum; +import org.dbsyncer.parser.enums.ConvertEnum; +import org.dbsyncer.parser.enums.GroupStrategyEnum; +import org.dbsyncer.parser.model.ConfigModel; +import org.dbsyncer.parser.model.Connector; +import org.dbsyncer.parser.model.Mapping; +import org.dbsyncer.parser.model.Meta; +import org.dbsyncer.parser.model.OperationConfig; +import org.dbsyncer.parser.model.ProjectGroup; +import org.dbsyncer.parser.model.QueryConfig; +import org.dbsyncer.parser.model.SystemConfig; +import org.dbsyncer.parser.model.TableGroup; +import org.dbsyncer.parser.model.UserConfig; +import org.dbsyncer.parser.template.OperationTemplate; +import org.dbsyncer.storage.enums.StorageDataStatusEnum; +import org.springframework.stereotype.Component; +import org.springframework.util.Assert; + +import javax.annotation.Resource; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * @Version 1.0.0 + * @Author AE86 + * @Date 2023-11-13 21:16 + */ +@Component +public class ProfileComponentImpl implements ProfileComponent { + + @Resource + private OperationTemplate operationTemplate; + + @Override + public Connector parseConnector(String json) { + Map conn = JsonUtil.parseMap(json); + Map config = (Map) conn.remove("config"); + Connector connector = JsonUtil.jsonToObj(conn.toString(), Connector.class); + Assert.notNull(connector, "Connector can not be null."); + String connectorType = (String) config.get("connectorType"); + ConnectorEnum connectorEnum = ConnectorEnum.getConnectorEnum(connectorType); + AbstractConnectorConfig obj = JsonUtil.jsonToObj(config.toString(), connectorEnum.getConfigClass()); + obj.setConnectorType(connectorEnum.getType()); + connector.setConfig(obj); + + return connector; + } + + @Override + public T parseObject(String json, Class clazz) { + return JsonUtil.jsonToObj(json, clazz); + } + + @Override + public String addConfigModel(ConfigModel model) { + return operationTemplate.execute(new OperationConfig(model, CommandEnum.OPR_ADD)); + } + + @Override + public String editConfigModel(ConfigModel model) { + return operationTemplate.execute(new OperationConfig(model, CommandEnum.OPR_EDIT)); + } + + @Override + public void removeConfigModel(String id) { + operationTemplate.remove(new OperationConfig(id)); + } + + @Override + public SystemConfig getSystemConfig() { + List list = operationTemplate.queryAll(SystemConfig.class); + return CollectionUtils.isEmpty(list) ? null : list.get(0); + } + + @Override + public UserConfig getUserConfig() { + List list = operationTemplate.queryAll(UserConfig.class); + return CollectionUtils.isEmpty(list) ? null : list.get(0); + } + + @Override + public ProjectGroup getProjectGroup(String id) { + return operationTemplate.queryObject(ProjectGroup.class, id); + } + + @Override + public List getProjectGroupAll() { + return operationTemplate.queryAll(ProjectGroup.class); + } + + @Override + public Connector getConnector(String connectorId) { + return operationTemplate.queryObject(Connector.class, connectorId); + } + + @Override + public List getConnectorAll() { + return operationTemplate.queryAll(Connector.class); + } + + @Override + public Mapping getMapping(String mappingId) { + return operationTemplate.queryObject(Mapping.class, mappingId); + } + + @Override + public List getMappingAll() { + return operationTemplate.queryAll(Mapping.class); + } + + @Override + public String addTableGroup(TableGroup model) { + return operationTemplate.execute(new OperationConfig(model, CommandEnum.OPR_ADD, GroupStrategyEnum.TABLE)); + } + + @Override + public String editTableGroup(TableGroup model) { + return operationTemplate.execute(new OperationConfig(model, CommandEnum.OPR_EDIT, GroupStrategyEnum.TABLE)); + } + + @Override + public void removeTableGroup(String id) { + operationTemplate.remove(new OperationConfig(id, GroupStrategyEnum.TABLE)); + } + + @Override + public TableGroup getTableGroup(String tableGroupId) { + return operationTemplate.queryObject(TableGroup.class, tableGroupId); + } + + @Override + public List getTableGroupAll(String mappingId) { + TableGroup tableGroup = new TableGroup().setMappingId(mappingId); + return operationTemplate.queryAll(new QueryConfig(tableGroup, GroupStrategyEnum.TABLE)); + } + + @Override + public List getSortedTableGroupAll(String mappingId) { + List list = getTableGroupAll(mappingId) + .stream() + .sorted(Comparator.comparing(TableGroup::getIndex).reversed()) + .collect(Collectors.toList()); + return list; + } + + @Override + public int getTableGroupCount(String mappingId) { + TableGroup tableGroup = new TableGroup().setMappingId(mappingId); + return operationTemplate.queryCount(new QueryConfig<>(tableGroup, GroupStrategyEnum.TABLE)); + } + + @Override + public Meta getMeta(String metaId) { + return operationTemplate.queryObject(Meta.class, metaId); + } + + @Override + public List getMetaAll() { + return operationTemplate.queryAll(Meta.class); + } + + @Override + public List getConnectorEnumAll() { + return Arrays.asList(ConnectorEnum.values()); + } + + @Override + public List getOperationEnumAll() { + return Arrays.asList(OperationEnum.values()); + } + + @Override + public List getQuartzFilterEnumAll() { + return Arrays.asList(QuartzFilterEnum.values()); + } + + @Override + public List getFilterEnumAll() { + return Arrays.asList(FilterEnum.values()); + } + + @Override + public List getConvertEnumAll() { + return Arrays.asList(ConvertEnum.values()); + } + + @Override + public List getStorageDataStatusEnumAll() { + return Arrays.asList(StorageDataStatusEnum.values()); + } + +} \ No newline at end of file diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/message/MessageServiceImpl.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/message/MessageServiceImpl.java index e5afc81e..de21ac29 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/message/MessageServiceImpl.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/message/MessageServiceImpl.java @@ -1,42 +1,30 @@ package org.dbsyncer.parser.message; -import org.dbsyncer.cache.CacheService; import org.dbsyncer.common.model.NotifyMessage; -import org.dbsyncer.common.model.Paging; import org.dbsyncer.common.spi.NotifyService; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.parser.Parser; +import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.parser.model.UserConfig; -import org.dbsyncer.storage.StorageService; -import org.dbsyncer.storage.constant.ConfigConstant; -import org.dbsyncer.storage.enums.StorageEnum; -import org.dbsyncer.storage.query.Query; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import java.util.*; +import javax.annotation.Resource; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; @Component public class MessageServiceImpl implements MessageService { - private String userConfigId; - - @Autowired - private CacheService cacheService; - - @Autowired + @Resource private NotifyService notifyService; - @Autowired - private StorageService storageService; - - @Autowired - private Parser parser; + @Resource + private ProfileComponent profileComponent; @Override public void sendMessage(String title, String content) { - UserConfig userConfig = getUserConfig(); + UserConfig userConfig = profileComponent.getUserConfig(); if (null == userConfig) { return; } @@ -53,33 +41,4 @@ public class MessageServiceImpl implements MessageService { notifyService.sendMessage(NotifyMessage.newBuilder().setTitle(title).setContent(content).setReceivers(mails)); } - private UserConfig getUserConfig() { - if (StringUtil.isNotBlank(userConfigId)) { - return cacheService.get(userConfigId, UserConfig.class); - } - - UserConfig userConfig = null; - synchronized (this) { - Query query = new Query(); - query.setType(StorageEnum.CONFIG); - query.addFilter(ConfigConstant.CONFIG_MODEL_TYPE, ConfigConstant.USER); - query.setPageNum(1); - query.setPageSize(1); - Paging paging = storageService.query(query); - List data = (List) paging.getData(); - if (!CollectionUtils.isEmpty(data)) { - Iterator iterator = data.iterator(); - if (iterator.hasNext()) { - Map map = (Map) iterator.next(); - String json = (String) map.get(ConfigConstant.CONFIG_MODEL_JSON); - userConfig = parser.parseObject(json, UserConfig.class); - userConfigId = userConfig.getId(); - } - - } - - } - return userConfig; - } - } \ No newline at end of file diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/model/FieldPicker.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/FieldPicker.java similarity index 95% rename from dbsyncer-manager/src/main/java/org/dbsyncer/manager/model/FieldPicker.java rename to dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/FieldPicker.java index f80fb344..cae543de 100644 --- a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/model/FieldPicker.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/FieldPicker.java @@ -1,4 +1,7 @@ -package org.dbsyncer.manager.model; +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.parser.model; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.DateFormatUtil; @@ -8,13 +11,15 @@ import org.dbsyncer.connector.enums.FilterEnum; import org.dbsyncer.connector.enums.OperationEnum; import org.dbsyncer.connector.model.Field; import org.dbsyncer.connector.model.Filter; -import org.dbsyncer.parser.model.FieldMapping; -import org.dbsyncer.parser.model.TableGroup; import org.springframework.util.Assert; import java.sql.Date; import java.sql.Timestamp; -import java.util.*; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; import java.util.stream.Collectors; public class FieldPicker { diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/model/OperationConfig.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/OperationConfig.java similarity index 84% rename from dbsyncer-manager/src/main/java/org/dbsyncer/manager/model/OperationConfig.java rename to dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/OperationConfig.java index ed6b5799..75955cce 100644 --- a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/model/OperationConfig.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/OperationConfig.java @@ -1,8 +1,10 @@ -package org.dbsyncer.manager.model; +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.parser.model; -import org.dbsyncer.manager.enums.CommandEnum; -import org.dbsyncer.manager.enums.GroupStrategyEnum; -import org.dbsyncer.parser.model.ConfigModel; +import org.dbsyncer.parser.enums.CommandEnum; +import org.dbsyncer.parser.enums.GroupStrategyEnum; public class OperationConfig { diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/model/QueryConfig.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/QueryConfig.java similarity index 80% rename from dbsyncer-manager/src/main/java/org/dbsyncer/manager/model/QueryConfig.java rename to dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/QueryConfig.java index 6b9e7b9f..e2316c32 100644 --- a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/model/QueryConfig.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/QueryConfig.java @@ -1,7 +1,9 @@ -package org.dbsyncer.manager.model; +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.parser.model; -import org.dbsyncer.manager.enums.GroupStrategyEnum; -import org.dbsyncer.parser.model.ConfigModel; +import org.dbsyncer.parser.enums.GroupStrategyEnum; public class QueryConfig { diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/GroupStrategy.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/GroupStrategy.java similarity index 67% rename from dbsyncer-manager/src/main/java/org/dbsyncer/manager/GroupStrategy.java rename to dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/GroupStrategy.java index 10a0b68d..a3702ddd 100644 --- a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/GroupStrategy.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/GroupStrategy.java @@ -1,4 +1,7 @@ -package org.dbsyncer.manager; +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.parser.strategy; import org.dbsyncer.parser.model.ConfigModel; diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/strategy/DefaultGroupStrategy.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/impl/DefaultGroupStrategy.java similarity index 64% rename from dbsyncer-manager/src/main/java/org/dbsyncer/manager/strategy/DefaultGroupStrategy.java rename to dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/impl/DefaultGroupStrategy.java index 80a8677c..a6aea0d2 100644 --- a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/strategy/DefaultGroupStrategy.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/impl/DefaultGroupStrategy.java @@ -1,7 +1,10 @@ -package org.dbsyncer.manager.strategy; +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.parser.strategy.impl; -import org.dbsyncer.manager.GroupStrategy; import org.dbsyncer.parser.model.ConfigModel; +import org.dbsyncer.parser.strategy.GroupStrategy; /** * @author AE86 diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/strategy/PreloadTableGroupStrategy.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/impl/PreloadTableGroupStrategy.java similarity index 72% rename from dbsyncer-manager/src/main/java/org/dbsyncer/manager/strategy/PreloadTableGroupStrategy.java rename to dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/impl/PreloadTableGroupStrategy.java index 35eca005..446d958c 100644 --- a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/strategy/PreloadTableGroupStrategy.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/impl/PreloadTableGroupStrategy.java @@ -1,7 +1,10 @@ -package org.dbsyncer.manager.strategy; +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.parser.strategy.impl; -import org.dbsyncer.manager.GroupStrategy; import org.dbsyncer.parser.model.Mapping; +import org.dbsyncer.parser.strategy.GroupStrategy; import org.dbsyncer.storage.constant.ConfigConstant; /** diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/strategy/TableGroupStrategy.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/impl/TableGroupStrategy.java similarity index 74% rename from dbsyncer-manager/src/main/java/org/dbsyncer/manager/strategy/TableGroupStrategy.java rename to dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/impl/TableGroupStrategy.java index c769676e..72acaa1b 100644 --- a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/strategy/TableGroupStrategy.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/impl/TableGroupStrategy.java @@ -1,7 +1,10 @@ -package org.dbsyncer.manager.strategy; +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.parser.strategy.impl; -import org.dbsyncer.manager.GroupStrategy; import org.dbsyncer.parser.model.TableGroup; +import org.dbsyncer.parser.strategy.GroupStrategy; import org.dbsyncer.storage.constant.ConfigConstant; /** diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/template/OperationTemplate.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/template/OperationTemplate.java similarity index 87% rename from dbsyncer-manager/src/main/java/org/dbsyncer/manager/template/OperationTemplate.java rename to dbsyncer-parser/src/main/java/org/dbsyncer/parser/template/OperationTemplate.java index 2b1d819b..f4ee07d1 100644 --- a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/template/OperationTemplate.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/template/OperationTemplate.java @@ -1,27 +1,34 @@ -package org.dbsyncer.manager.template; +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.parser.template; import org.dbsyncer.cache.CacheService; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.manager.GroupStrategy; -import org.dbsyncer.manager.ManagerException; -import org.dbsyncer.manager.command.PersistenceCommand; -import org.dbsyncer.manager.enums.CommandEnum; -import org.dbsyncer.manager.enums.GroupStrategyEnum; -import org.dbsyncer.manager.model.OperationConfig; -import org.dbsyncer.manager.model.QueryConfig; +import org.dbsyncer.parser.ParserException; +import org.dbsyncer.parser.command.PersistenceCommand; +import org.dbsyncer.parser.enums.CommandEnum; +import org.dbsyncer.parser.enums.GroupStrategyEnum; import org.dbsyncer.parser.model.ConfigModel; +import org.dbsyncer.parser.model.OperationConfig; +import org.dbsyncer.parser.model.QueryConfig; +import org.dbsyncer.parser.strategy.GroupStrategy; import org.dbsyncer.parser.util.ConfigModelUtil; import org.dbsyncer.storage.StorageService; import org.dbsyncer.storage.enums.StorageEnum; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.BeanUtils; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.springframework.util.Assert; -import java.util.*; +import javax.annotation.Resource; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; /** * 操作配置模板 @@ -35,10 +42,10 @@ public final class OperationTemplate { private final Logger logger = LoggerFactory.getLogger(getClass()); - @Autowired + @Resource private StorageService storageService; - @Autowired + @Resource private CacheService cacheService; public List queryAll(Class valueType) { @@ -46,7 +53,7 @@ public final class OperationTemplate { ConfigModel configModel = (ConfigModel) valueType.newInstance(); return queryAll(new QueryConfig(configModel)); } catch (Exception e) { - throw new ManagerException(e); + throw new ParserException(e); } } @@ -154,13 +161,13 @@ public final class OperationTemplate { BeanUtils.copyProperties(o, t); return t; } catch (InstantiationException e) { - throw new ManagerException(e.getMessage()); + throw new ParserException(e); } catch (IllegalAccessException e) { - throw new ManagerException(e.getMessage()); + throw new ParserException(e); } } - class Group { + public class Group { private List index; -- Gitee From 8bac39fc4c71e0b9b0edc79b6c7e221749e8a84f Mon Sep 17 00:00:00 2001 From: AE86 <836391306@qq.com> Date: Tue, 14 Nov 2023 22:03:46 +0800 Subject: [PATCH 02/98] move cache --- .../impl/connector/ConnectorChecker.java | 4 +- .../impl/system/SystemConfigChecker.java | 4 +- .../dbsyncer/biz/impl/BaseServiceImpl.java | 6 +- .../biz/impl/ConnectorServiceImpl.java | 4 +- .../dbsyncer/biz/impl/MappingServiceImpl.java | 2 +- .../dbsyncer/biz/impl/MonitorServiceImpl.java | 4 +- .../dbsyncer/biz/impl/PluginServiceImpl.java | 4 +- .../biz/impl/ProjectGroupServiceImpl.java | 2 +- .../biz/impl/SystemConfigServiceImpl.java | 4 +- .../biz/impl/TableGroupServiceImpl.java | 2 +- .../biz/impl/UserConfigServiceImpl.java | 4 +- dbsyncer-cache/pom.xml | 21 - .../org/dbsyncer/cache/CacheException.java | 27 - .../dbsyncer/manager/puller/FullPuller.java | 4 +- .../manager/puller/IncrementPuller.java | 4 +- dbsyncer-parser/pom.xml | 7 - .../org/dbsyncer/parser}/CacheService.java | 2 +- .../parser/{logger => }/LogService.java | 28 +- .../dbsyncer/parser/{logger => }/LogType.java | 610 +++++++++--------- .../parser/{message => }/MessageService.java | 2 +- .../parser/consumer/AbstractConsumer.java | 4 +- .../parser/flush/AbstractBufferActuator.java | 2 +- .../parser/flush/AbstractFlushStrategy.java | 2 +- .../flush/impl/GeneralBufferActuator.java | 2 +- .../parser/impl}/CacheServiceImpl.java | 3 +- .../{logger => impl}/LogServiceImpl.java | 64 +- .../{message => impl}/MessageServiceImpl.java | 3 +- .../impl/DisableFullFlushStrategy.java | 4 +- .../parser/template/OperationTemplate.java | 2 +- .../controller/config/ConfigController.java | 18 +- pom.xml | 1 - 31 files changed, 399 insertions(+), 451 deletions(-) delete mode 100644 dbsyncer-cache/pom.xml delete mode 100644 dbsyncer-cache/src/main/java/org/dbsyncer/cache/CacheException.java rename {dbsyncer-cache/src/main/java/org/dbsyncer/cache => dbsyncer-parser/src/main/java/org/dbsyncer/parser}/CacheService.java (96%) rename dbsyncer-parser/src/main/java/org/dbsyncer/parser/{logger => }/LogService.java (82%) rename dbsyncer-parser/src/main/java/org/dbsyncer/parser/{logger => }/LogType.java (94%) rename dbsyncer-parser/src/main/java/org/dbsyncer/parser/{message => }/MessageService.java (81%) rename {dbsyncer-cache/src/main/java/org/dbsyncer/cache => dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl}/CacheServiceImpl.java (92%) rename dbsyncer-parser/src/main/java/org/dbsyncer/parser/{logger => impl}/LogServiceImpl.java (88%) rename dbsyncer-parser/src/main/java/org/dbsyncer/parser/{message => impl}/MessageServiceImpl.java (94%) diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/ConnectorChecker.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/ConnectorChecker.java index 28f4273b..239608d7 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/ConnectorChecker.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/ConnectorChecker.java @@ -11,8 +11,8 @@ import org.dbsyncer.connector.enums.ConnectorEnum; import org.dbsyncer.connector.model.Table; import org.dbsyncer.parser.ParserComponent; import org.dbsyncer.parser.ProfileComponent; -import org.dbsyncer.parser.logger.LogService; -import org.dbsyncer.parser.logger.LogType; +import org.dbsyncer.parser.LogService; +import org.dbsyncer.parser.LogType; import org.dbsyncer.parser.model.ConfigModel; import org.dbsyncer.parser.model.Connector; import org.dbsyncer.storage.constant.ConfigConstant; diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/system/SystemConfigChecker.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/system/SystemConfigChecker.java index d318f50e..527cd478 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/system/SystemConfigChecker.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/system/SystemConfigChecker.java @@ -4,8 +4,8 @@ import org.dbsyncer.biz.checker.AbstractChecker; import org.dbsyncer.common.util.BeanUtil; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.parser.ProfileComponent; -import org.dbsyncer.parser.logger.LogService; -import org.dbsyncer.parser.logger.LogType; +import org.dbsyncer.parser.LogService; +import org.dbsyncer.parser.LogType; import org.dbsyncer.parser.model.ConfigModel; import org.dbsyncer.parser.model.SystemConfig; import org.slf4j.Logger; diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/BaseServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/BaseServiceImpl.java index fb6fca31..5dc8244b 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/BaseServiceImpl.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/BaseServiceImpl.java @@ -3,9 +3,9 @@ package org.dbsyncer.biz.impl; import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.parser.enums.MetaEnum; import org.dbsyncer.parser.enums.ModelEnum; -import org.dbsyncer.parser.logger.LogService; -import org.dbsyncer.parser.logger.LogType; -import org.dbsyncer.parser.message.MessageService; +import org.dbsyncer.parser.LogService; +import org.dbsyncer.parser.LogType; +import org.dbsyncer.parser.MessageService; import org.dbsyncer.parser.model.ConfigModel; import org.dbsyncer.parser.model.Mapping; import org.dbsyncer.parser.model.Meta; diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ConnectorServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ConnectorServiceImpl.java index a9309558..f96fcb8d 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ConnectorServiceImpl.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ConnectorServiceImpl.java @@ -9,8 +9,8 @@ import org.dbsyncer.common.util.JsonUtil; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.connector.ConnectorFactory; import org.dbsyncer.parser.ProfileComponent; -import org.dbsyncer.parser.logger.LogService; -import org.dbsyncer.parser.logger.LogType; +import org.dbsyncer.parser.LogService; +import org.dbsyncer.parser.LogType; import org.dbsyncer.parser.model.ConfigModel; import org.dbsyncer.parser.model.Connector; import org.dbsyncer.parser.model.Mapping; diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MappingServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MappingServiceImpl.java index d448dac3..efaaa4de 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MappingServiceImpl.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MappingServiceImpl.java @@ -17,7 +17,7 @@ import org.dbsyncer.connector.model.Table; import org.dbsyncer.manager.ManagerFactory; import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.parser.enums.ModelEnum; -import org.dbsyncer.parser.logger.LogType; +import org.dbsyncer.parser.LogType; import org.dbsyncer.parser.model.ConfigModel; import org.dbsyncer.parser.model.Connector; import org.dbsyncer.parser.model.Mapping; diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java index 9d963c39..5317a696 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java @@ -33,8 +33,8 @@ import org.dbsyncer.monitor.model.MetricResponse; import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.parser.enums.MetaEnum; import org.dbsyncer.parser.enums.ModelEnum; -import org.dbsyncer.parser.logger.LogService; -import org.dbsyncer.parser.logger.LogType; +import org.dbsyncer.parser.LogService; +import org.dbsyncer.parser.LogType; import org.dbsyncer.parser.model.Mapping; import org.dbsyncer.parser.model.Meta; import org.dbsyncer.storage.StorageService; diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/PluginServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/PluginServiceImpl.java index b1beb0a6..e23b1586 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/PluginServiceImpl.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/PluginServiceImpl.java @@ -6,8 +6,8 @@ import org.dbsyncer.biz.vo.PluginVo; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.parser.ProfileComponent; -import org.dbsyncer.parser.logger.LogService; -import org.dbsyncer.parser.logger.LogType; +import org.dbsyncer.parser.LogService; +import org.dbsyncer.parser.LogType; import org.dbsyncer.parser.model.Mapping; import org.dbsyncer.parser.model.TableGroup; import org.dbsyncer.plugin.PluginFactory; diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ProjectGroupServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ProjectGroupServiceImpl.java index 00c013b6..31c5b8e2 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ProjectGroupServiceImpl.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ProjectGroupServiceImpl.java @@ -9,7 +9,7 @@ import org.dbsyncer.biz.vo.ProjectGroupVo; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.parser.ProfileComponent; -import org.dbsyncer.parser.logger.LogType; +import org.dbsyncer.parser.LogType; import org.dbsyncer.parser.model.ConfigModel; import org.dbsyncer.parser.model.Connector; import org.dbsyncer.parser.model.ProjectGroup; diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/SystemConfigServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/SystemConfigServiceImpl.java index bc14a13f..2e246773 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/SystemConfigServiceImpl.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/SystemConfigServiceImpl.java @@ -7,8 +7,8 @@ import org.dbsyncer.biz.checker.Checker; import org.dbsyncer.biz.vo.SystemConfigVo; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.parser.ProfileComponent; -import org.dbsyncer.parser.logger.LogService; -import org.dbsyncer.parser.logger.LogType; +import org.dbsyncer.parser.LogService; +import org.dbsyncer.parser.LogType; import org.dbsyncer.parser.model.ConfigModel; import org.dbsyncer.parser.model.SystemConfig; import org.dbsyncer.manager.template.PreloadTemplate; diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/TableGroupServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/TableGroupServiceImpl.java index 26585b3d..0d809fe4 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/TableGroupServiceImpl.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/TableGroupServiceImpl.java @@ -7,7 +7,7 @@ import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.connector.model.Field; import org.dbsyncer.parser.ProfileComponent; -import org.dbsyncer.parser.logger.LogType; +import org.dbsyncer.parser.LogType; import org.dbsyncer.parser.model.Mapping; import org.dbsyncer.parser.model.TableGroup; import org.dbsyncer.storage.constant.ConfigConstant; diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/UserConfigServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/UserConfigServiceImpl.java index 185903d2..179da1a7 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/UserConfigServiceImpl.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/UserConfigServiceImpl.java @@ -8,8 +8,8 @@ import org.dbsyncer.biz.vo.UserInfoVo; import org.dbsyncer.common.util.SHA1Util; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.parser.ProfileComponent; -import org.dbsyncer.parser.logger.LogService; -import org.dbsyncer.parser.logger.LogType; +import org.dbsyncer.parser.LogService; +import org.dbsyncer.parser.LogType; import org.dbsyncer.parser.model.UserConfig; import org.dbsyncer.parser.model.UserInfo; import org.springframework.beans.BeanUtils; diff --git a/dbsyncer-cache/pom.xml b/dbsyncer-cache/pom.xml deleted file mode 100644 index 201c5803..00000000 --- a/dbsyncer-cache/pom.xml +++ /dev/null @@ -1,21 +0,0 @@ - - - - dbsyncer - org.ghi - 1.2.7_1103 - - 4.0.0 - dbsyncer-cache - - - - - org.ghi - dbsyncer-common - ${project.parent.version} - - - - \ No newline at end of file diff --git a/dbsyncer-cache/src/main/java/org/dbsyncer/cache/CacheException.java b/dbsyncer-cache/src/main/java/org/dbsyncer/cache/CacheException.java deleted file mode 100644 index 1fa3bd19..00000000 --- a/dbsyncer-cache/src/main/java/org/dbsyncer/cache/CacheException.java +++ /dev/null @@ -1,27 +0,0 @@ -package org.dbsyncer.cache; - -/** - * @author AE86 - * @version 1.0.0 - * @date 2020/1/8 15:17 - */ -public class CacheException extends RuntimeException { - - private static final long serialVersionUID = 1L; - - public CacheException(String message) { - super(message); - } - - public CacheException(String message, Throwable cause) { - super(message, cause); - } - - public CacheException(Throwable cause) { - super(cause); - } - - protected CacheException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { - super(message, cause, enableSuppression, writableStackTrace); - } -} \ No newline at end of file diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/puller/FullPuller.java b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/puller/FullPuller.java index 7a44a566..86e47e9c 100644 --- a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/puller/FullPuller.java +++ b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/puller/FullPuller.java @@ -7,8 +7,8 @@ import org.dbsyncer.parser.ParserComponent; import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.parser.enums.ParserEnum; import org.dbsyncer.parser.event.FullRefreshEvent; -import org.dbsyncer.parser.logger.LogService; -import org.dbsyncer.parser.logger.LogType; +import org.dbsyncer.parser.LogService; +import org.dbsyncer.parser.LogType; import org.dbsyncer.parser.model.Mapping; import org.dbsyncer.parser.model.Meta; import org.dbsyncer.parser.model.TableGroup; diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/puller/IncrementPuller.java b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/puller/IncrementPuller.java index dd012e42..03dd70d7 100644 --- a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/puller/IncrementPuller.java +++ b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/puller/IncrementPuller.java @@ -20,8 +20,8 @@ import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.parser.consumer.impl.LogConsumer; import org.dbsyncer.parser.consumer.impl.QuartzConsumer; import org.dbsyncer.parser.flush.impl.BufferActuatorRouter; -import org.dbsyncer.parser.logger.LogService; -import org.dbsyncer.parser.logger.LogType; +import org.dbsyncer.parser.LogService; +import org.dbsyncer.parser.LogType; import org.dbsyncer.parser.model.Connector; import org.dbsyncer.parser.model.Mapping; import org.dbsyncer.parser.model.Meta; diff --git a/dbsyncer-parser/pom.xml b/dbsyncer-parser/pom.xml index 00f69b6e..f262f71c 100644 --- a/dbsyncer-parser/pom.xml +++ b/dbsyncer-parser/pom.xml @@ -18,13 +18,6 @@ ${project.parent.version} - - - org.ghi - dbsyncer-cache - ${project.parent.version} - - org.ghi diff --git a/dbsyncer-cache/src/main/java/org/dbsyncer/cache/CacheService.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/CacheService.java similarity index 96% rename from dbsyncer-cache/src/main/java/org/dbsyncer/cache/CacheService.java rename to dbsyncer-parser/src/main/java/org/dbsyncer/parser/CacheService.java index 7ae7c07b..34b6eee8 100644 --- a/dbsyncer-cache/src/main/java/org/dbsyncer/cache/CacheService.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/CacheService.java @@ -1,4 +1,4 @@ -package org.dbsyncer.cache; +package org.dbsyncer.parser; import java.util.Map; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/logger/LogService.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/LogService.java similarity index 82% rename from dbsyncer-parser/src/main/java/org/dbsyncer/parser/logger/LogService.java rename to dbsyncer-parser/src/main/java/org/dbsyncer/parser/LogService.java index d0256381..f9efe9c7 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/logger/LogService.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/LogService.java @@ -1,15 +1,15 @@ -package org.dbsyncer.parser.logger; - -/** - * @version 1.0.0 - * @Author AE86 - * @Date 2020-05-21 23:18 - */ -public interface LogService { - - void log(LogType logType); - - void log(LogType logType, String msg); - - void log(LogType logType, String format, Object... args); +package org.dbsyncer.parser; + +/** + * @version 1.0.0 + * @Author AE86 + * @Date 2020-05-21 23:18 + */ +public interface LogService { + + void log(LogType logType); + + void log(LogType logType, String msg); + + void log(LogType logType, String format, Object... args); } \ No newline at end of file diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/logger/LogType.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/LogType.java similarity index 94% rename from dbsyncer-parser/src/main/java/org/dbsyncer/parser/logger/LogType.java rename to dbsyncer-parser/src/main/java/org/dbsyncer/parser/LogType.java index 5889caa6..df4bef06 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/logger/LogType.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/LogType.java @@ -1,306 +1,306 @@ -package org.dbsyncer.parser.logger; - -/** - * 日志类型枚举 - * - * @author AE86 - * @version 1.0.0 - * @date 2020/04/21 16:19 - */ -public interface LogType { - - /** - * 分类名称 - * - * @return - */ - String getName(); - - /** - * 类型 - * - * @return - */ - String getType(); - - /** - * 内容 - * - * @return - */ - String getMessage(); - - /** - * 系统日志1 - */ - enum SystemLog implements LogType { - /** - * 正常 - */ - INFO("10", "正常"), - /** - * 警告 - */ - WARN("11", "警告"), - /** - * 错误 - */ - ERROR("12", "错误"); - - private String type; - private String message; - - SystemLog(String type, String message) { - this.type = type; - this.message = message; - } - - @Override - public String getName() { - return "系统日志"; - } - - @Override - public String getType() { - return type; - } - - @Override - public String getMessage() { - return message; - } - } - - /** - * 连接器2 - */ - enum ConnectorLog implements LogType { - INSERT("20", "新增"), - UPDATE("21", "修改"), - DELETE("22", "删除"), - FAILED("23", "连接失败"), - RECONNECT_SUCCESS("24", "重连成功"), - COPY("25", "复制"); - - private String type; - private String message; - - ConnectorLog(String type, String message) { - this.type = type; - this.message = message; - } - - @Override - public String getName() { - return "连接器"; - } - - @Override - public String getType() { - return type; - } - - @Override - public String getMessage() { - return message; - } - } - - /** - * 驱动3 - */ - enum MappingLog implements LogType { - INSERT("30", "新增"), - UPDATE("31", "修改"), - DELETE("32", "删除"), - RUNNING("33", "启动"), - STOP("34", "停止"), - CLEAR_DATA("35", "清空同步数据"), - COPY("36", "复制"); - - private String type; - private String message; - - MappingLog(String type, String message) { - this.type = type; - this.message = message; - } - - @Override - public String getName() { - return "驱动"; - } - - @Override - public String getType() { - return type; - } - - @Override - public String getMessage() { - return message; - } - } - - /** - * 映射关系4 - */ - enum TableGroupLog implements LogType { - INSERT("40", "新增"), - UPDATE("41", "修改"), - DELETE("42", "删除"), - INCREMENT_FAILED("43", "增量同步异常"), - FULL_FAILED("44", "全量同步异常"), - COPY("45", "复制"); - - private String type; - private String message; - - TableGroupLog(String type, String message) { - this.type = type; - this.message = message; - } - - @Override - public String getName() { - return "映射关系"; - } - - @Override - public String getType() { - return type; - } - - @Override - public String getMessage() { - return message; - } - } - - /** - * 元信息5 - */ - enum MetaLog implements LogType { - DELETE("50", "删除"), - CLEAR("51", "删除数据"); - - private String type; - private String message; - - MetaLog(String type, String message) { - this.type = type; - this.message = message; - } - - @Override - public String getName() { - return "元信息"; - } - - @Override - public String getType() { - return type; - } - - @Override - public String getMessage() { - return message; - } - } - - /** - * 插件信息6 - */ - enum PluginLog implements LogType { - UPDATE("60", "上传成功"), - CHECK_ERROR("61", "格式不正确"); - - private String type; - private String message; - - PluginLog(String type, String message) { - this.type = type; - this.message = message; - } - - @Override - public String getName() { - return "插件"; - } - - @Override - public String getType() { - return type; - } - - @Override - public String getMessage() { - return message; - } - } - - /** - * 缓存配置信息7 - */ - enum CacheLog implements LogType { - IMPORT("70", "导入配置"), - IMPORT_ERROR("71", "导入配置异常"), - EXPORT("72", "导出配置"); - - private String type; - private String message; - - CacheLog(String type, String message) { - this.type = type; - this.message = message; - } - - @Override - public String getName() { - return "插件"; - } - - @Override - public String getType() { - return type; - } - - @Override - public String getMessage() { - return message; - } - } - - /** - * 用户日志8 - */ - enum UserLog implements LogType { - INSERT("80", "新增"), - UPDATE("81", "修改"), - DELETE("82", "删除"); - - private String type; - private String message; - - UserLog(String type, String message) { - this.type = type; - this.message = message; - } - - @Override - public String getName() { - return "用户日志"; - } - - @Override - public String getType() { - return type; - } - - @Override - public String getMessage() { - return message; - } - } - +package org.dbsyncer.parser; + +/** + * 日志类型枚举 + * + * @author AE86 + * @version 1.0.0 + * @date 2020/04/21 16:19 + */ +public interface LogType { + + /** + * 分类名称 + * + * @return + */ + String getName(); + + /** + * 类型 + * + * @return + */ + String getType(); + + /** + * 内容 + * + * @return + */ + String getMessage(); + + /** + * 系统日志1 + */ + enum SystemLog implements LogType { + /** + * 正常 + */ + INFO("10", "正常"), + /** + * 警告 + */ + WARN("11", "警告"), + /** + * 错误 + */ + ERROR("12", "错误"); + + private String type; + private String message; + + SystemLog(String type, String message) { + this.type = type; + this.message = message; + } + + @Override + public String getName() { + return "系统日志"; + } + + @Override + public String getType() { + return type; + } + + @Override + public String getMessage() { + return message; + } + } + + /** + * 连接器2 + */ + enum ConnectorLog implements LogType { + INSERT("20", "新增"), + UPDATE("21", "修改"), + DELETE("22", "删除"), + FAILED("23", "连接失败"), + RECONNECT_SUCCESS("24", "重连成功"), + COPY("25", "复制"); + + private String type; + private String message; + + ConnectorLog(String type, String message) { + this.type = type; + this.message = message; + } + + @Override + public String getName() { + return "连接器"; + } + + @Override + public String getType() { + return type; + } + + @Override + public String getMessage() { + return message; + } + } + + /** + * 驱动3 + */ + enum MappingLog implements LogType { + INSERT("30", "新增"), + UPDATE("31", "修改"), + DELETE("32", "删除"), + RUNNING("33", "启动"), + STOP("34", "停止"), + CLEAR_DATA("35", "清空同步数据"), + COPY("36", "复制"); + + private String type; + private String message; + + MappingLog(String type, String message) { + this.type = type; + this.message = message; + } + + @Override + public String getName() { + return "驱动"; + } + + @Override + public String getType() { + return type; + } + + @Override + public String getMessage() { + return message; + } + } + + /** + * 映射关系4 + */ + enum TableGroupLog implements LogType { + INSERT("40", "新增"), + UPDATE("41", "修改"), + DELETE("42", "删除"), + INCREMENT_FAILED("43", "增量同步异常"), + FULL_FAILED("44", "全量同步异常"), + COPY("45", "复制"); + + private String type; + private String message; + + TableGroupLog(String type, String message) { + this.type = type; + this.message = message; + } + + @Override + public String getName() { + return "映射关系"; + } + + @Override + public String getType() { + return type; + } + + @Override + public String getMessage() { + return message; + } + } + + /** + * 元信息5 + */ + enum MetaLog implements LogType { + DELETE("50", "删除"), + CLEAR("51", "删除数据"); + + private String type; + private String message; + + MetaLog(String type, String message) { + this.type = type; + this.message = message; + } + + @Override + public String getName() { + return "元信息"; + } + + @Override + public String getType() { + return type; + } + + @Override + public String getMessage() { + return message; + } + } + + /** + * 插件信息6 + */ + enum PluginLog implements LogType { + UPDATE("60", "上传成功"), + CHECK_ERROR("61", "格式不正确"); + + private String type; + private String message; + + PluginLog(String type, String message) { + this.type = type; + this.message = message; + } + + @Override + public String getName() { + return "插件"; + } + + @Override + public String getType() { + return type; + } + + @Override + public String getMessage() { + return message; + } + } + + /** + * 缓存配置信息7 + */ + enum CacheLog implements LogType { + IMPORT("70", "导入配置"), + IMPORT_ERROR("71", "导入配置异常"), + EXPORT("72", "导出配置"); + + private String type; + private String message; + + CacheLog(String type, String message) { + this.type = type; + this.message = message; + } + + @Override + public String getName() { + return "插件"; + } + + @Override + public String getType() { + return type; + } + + @Override + public String getMessage() { + return message; + } + } + + /** + * 用户日志8 + */ + enum UserLog implements LogType { + INSERT("80", "新增"), + UPDATE("81", "修改"), + DELETE("82", "删除"); + + private String type; + private String message; + + UserLog(String type, String message) { + this.type = type; + this.message = message; + } + + @Override + public String getName() { + return "用户日志"; + } + + @Override + public String getType() { + return type; + } + + @Override + public String getMessage() { + return message; + } + } + } \ No newline at end of file diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/message/MessageService.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/MessageService.java similarity index 81% rename from dbsyncer-parser/src/main/java/org/dbsyncer/parser/message/MessageService.java rename to dbsyncer-parser/src/main/java/org/dbsyncer/parser/MessageService.java index 35da969f..61d7bd6f 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/message/MessageService.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/MessageService.java @@ -1,4 +1,4 @@ -package org.dbsyncer.parser.message; +package org.dbsyncer.parser; public interface MessageService { diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/AbstractConsumer.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/AbstractConsumer.java index 7ec9e137..627a1d83 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/AbstractConsumer.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/AbstractConsumer.java @@ -8,8 +8,8 @@ import org.dbsyncer.common.event.DDLChangedEvent; import org.dbsyncer.common.event.Watcher; import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.parser.flush.impl.BufferActuatorRouter; -import org.dbsyncer.parser.logger.LogService; -import org.dbsyncer.parser.logger.LogType; +import org.dbsyncer.parser.LogService; +import org.dbsyncer.parser.LogType; import org.dbsyncer.parser.model.Mapping; import org.dbsyncer.parser.model.Meta; import org.dbsyncer.parser.model.TableGroup; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/AbstractBufferActuator.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/AbstractBufferActuator.java index 4746ba3f..fe8e3b2c 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/AbstractBufferActuator.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/AbstractBufferActuator.java @@ -1,9 +1,9 @@ package org.dbsyncer.parser.flush; -import org.dbsyncer.cache.CacheService; import org.dbsyncer.common.config.BufferActuatorConfig; import org.dbsyncer.common.scheduled.ScheduledTaskJob; import org.dbsyncer.common.scheduled.ScheduledTaskService; +import org.dbsyncer.parser.CacheService; import org.dbsyncer.parser.enums.MetaEnum; import org.dbsyncer.parser.model.Meta; import org.slf4j.Logger; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/AbstractFlushStrategy.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/AbstractFlushStrategy.java index 408aa1e2..42e5cb5d 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/AbstractFlushStrategy.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/AbstractFlushStrategy.java @@ -1,6 +1,6 @@ package org.dbsyncer.parser.flush; -import org.dbsyncer.cache.CacheService; +import org.dbsyncer.parser.CacheService; import org.dbsyncer.common.config.StorageConfig; import org.dbsyncer.common.model.Result; import org.dbsyncer.common.util.CollectionUtils; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/GeneralBufferActuator.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/GeneralBufferActuator.java index b03f3771..aed90b4c 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/GeneralBufferActuator.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/GeneralBufferActuator.java @@ -1,6 +1,6 @@ package org.dbsyncer.parser.flush.impl; -import org.dbsyncer.cache.CacheService; +import org.dbsyncer.parser.CacheService; import org.dbsyncer.common.config.GeneralBufferConfig; import org.dbsyncer.common.event.RefreshOffsetEvent; import org.dbsyncer.common.model.AbstractConnectorConfig; diff --git a/dbsyncer-cache/src/main/java/org/dbsyncer/cache/CacheServiceImpl.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/CacheServiceImpl.java similarity index 92% rename from dbsyncer-cache/src/main/java/org/dbsyncer/cache/CacheServiceImpl.java rename to dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/CacheServiceImpl.java index cb02328f..56c9e2cd 100644 --- a/dbsyncer-cache/src/main/java/org/dbsyncer/cache/CacheServiceImpl.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/CacheServiceImpl.java @@ -1,5 +1,6 @@ -package org.dbsyncer.cache; +package org.dbsyncer.parser.impl; +import org.dbsyncer.parser.CacheService; import org.springframework.stereotype.Component; import java.util.Collections; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/logger/LogServiceImpl.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/LogServiceImpl.java similarity index 88% rename from dbsyncer-parser/src/main/java/org/dbsyncer/parser/logger/LogServiceImpl.java rename to dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/LogServiceImpl.java index c7fa3b97..f658d34c 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/logger/LogServiceImpl.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/LogServiceImpl.java @@ -1,32 +1,34 @@ -package org.dbsyncer.parser.logger; - -import org.dbsyncer.parser.flush.FlushService; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -/** - * @version 1.0.0 - * @Author AE86 - * @Date 2020-05-21 23:18 - */ -@Component -public class LogServiceImpl implements LogService { - - @Autowired - private FlushService flushService; - - @Override - public void log(LogType logType) { - flushService.asyncWrite(logType.getType(), String.format("%s%s", logType.getName(), logType.getMessage())); - } - - @Override - public void log(LogType logType, String msg) { - flushService.asyncWrite(logType.getType(), null == msg ? logType.getMessage() : msg); - } - - @Override - public void log(LogType logType, String format, Object... args) { - flushService.asyncWrite(logType.getType(), String.format(format, args)); - } +package org.dbsyncer.parser.impl; + +import org.dbsyncer.parser.LogService; +import org.dbsyncer.parser.LogType; +import org.dbsyncer.parser.flush.FlushService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +/** + * @version 1.0.0 + * @Author AE86 + * @Date 2020-05-21 23:18 + */ +@Component +public class LogServiceImpl implements LogService { + + @Autowired + private FlushService flushService; + + @Override + public void log(LogType logType) { + flushService.asyncWrite(logType.getType(), String.format("%s%s", logType.getName(), logType.getMessage())); + } + + @Override + public void log(LogType logType, String msg) { + flushService.asyncWrite(logType.getType(), null == msg ? logType.getMessage() : msg); + } + + @Override + public void log(LogType logType, String format, Object... args) { + flushService.asyncWrite(logType.getType(), String.format(format, args)); + } } \ No newline at end of file diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/message/MessageServiceImpl.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/MessageServiceImpl.java similarity index 94% rename from dbsyncer-parser/src/main/java/org/dbsyncer/parser/message/MessageServiceImpl.java rename to dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/MessageServiceImpl.java index de21ac29..03e1e319 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/message/MessageServiceImpl.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/MessageServiceImpl.java @@ -1,9 +1,10 @@ -package org.dbsyncer.parser.message; +package org.dbsyncer.parser.impl; import org.dbsyncer.common.model.NotifyMessage; import org.dbsyncer.common.spi.NotifyService; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.StringUtil; +import org.dbsyncer.parser.MessageService; import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.parser.model.UserConfig; import org.springframework.stereotype.Component; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/impl/DisableFullFlushStrategy.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/impl/DisableFullFlushStrategy.java index be0ed506..f55078b0 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/impl/DisableFullFlushStrategy.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/impl/DisableFullFlushStrategy.java @@ -3,8 +3,8 @@ package org.dbsyncer.parser.strategy.impl; import org.dbsyncer.common.model.Result; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.parser.flush.AbstractFlushStrategy; -import org.dbsyncer.parser.logger.LogService; -import org.dbsyncer.parser.logger.LogType; +import org.dbsyncer.parser.LogService; +import org.dbsyncer.parser.LogType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/template/OperationTemplate.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/template/OperationTemplate.java index f4ee07d1..c09bdca7 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/template/OperationTemplate.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/template/OperationTemplate.java @@ -3,7 +3,7 @@ */ package org.dbsyncer.parser.template; -import org.dbsyncer.cache.CacheService; +import org.dbsyncer.parser.CacheService; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.parser.ParserException; diff --git a/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/config/ConfigController.java b/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/config/ConfigController.java index 8d3598da..4e0fabf9 100644 --- a/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/config/ConfigController.java +++ b/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/config/ConfigController.java @@ -4,15 +4,14 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.dbsyncer.biz.SystemConfigService; import org.dbsyncer.biz.vo.RestResult; -import org.dbsyncer.cache.CacheService; +import org.dbsyncer.parser.CacheService; import org.dbsyncer.common.config.AppConfig; import org.dbsyncer.common.snowflake.SnowflakeIdWorker; import org.dbsyncer.common.util.JsonUtil; -import org.dbsyncer.parser.logger.LogService; -import org.dbsyncer.parser.logger.LogType; +import org.dbsyncer.parser.LogService; +import org.dbsyncer.parser.LogType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Controller; import org.springframework.ui.ModelMap; import org.springframework.web.bind.annotation.GetMapping; @@ -21,6 +20,7 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.multipart.MultipartFile; +import javax.annotation.Resource; import javax.servlet.http.HttpServletResponse; import java.io.File; import java.io.IOException; @@ -33,19 +33,19 @@ public class ConfigController { private final Logger logger = LoggerFactory.getLogger(getClass()); - @Autowired + @Resource private SystemConfigService systemConfigService; - @Autowired + @Resource private CacheService cacheService; - @Autowired + @Resource private LogService logService; - @Autowired + @Resource private AppConfig appConfig; - @Autowired + @Resource private SnowflakeIdWorker snowflakeIdWorker; @RequestMapping("") diff --git a/pom.xml b/pom.xml index 3733d8ef..69b35ada 100644 --- a/pom.xml +++ b/pom.xml @@ -14,7 +14,6 @@ dbsyncer-biz - dbsyncer-cache dbsyncer-cluster dbsyncer-common dbsyncer-connector -- Gitee From ed1b8c641cb5d69f43a18e6c3b5d0a6f5538935b Mon Sep 17 00:00:00 2001 From: AE86 <836391306@qq.com> Date: Tue, 14 Nov 2023 22:20:10 +0800 Subject: [PATCH 03/98] move monitor --- dbsyncer-biz/pom.xml | 7 - .../java/org/dbsyncer/biz/MonitorService.java | 4 +- .../biz}/enums/BufferActuatorMetricEnum.java | 2 +- .../dbsyncer/biz}/enums/DiskMetricEnum.java | 96 ++-- .../org/dbsyncer/biz}/enums/MetricEnum.java | 162 +++--- .../dbsyncer/biz}/enums/StatisticEnum.java | 108 ++-- .../biz}/enums/ThreadPoolMetricEnum.java | 2 +- .../org/dbsyncer/biz/impl/MetricReporter.java | 493 +++++++++--------- .../dbsyncer/biz/impl/MonitorServiceImpl.java | 20 +- .../metric/impl/CpuMetricDetailFormatter.java | 2 +- .../impl/DiskMetricDetailFormatter.java | 2 +- .../DoubleRoundMetricDetailFormatter.java | 2 +- .../metric/impl/GCMetricDetailFormatter.java | 2 +- .../impl/MemoryMetricDetailFormatter.java | 2 +- .../impl/ValueMetricDetailFormatter.java | 2 +- .../dbsyncer/biz}/model/AppReportMetric.java | 2 +- .../biz}/model/MappingReportMetric.java | 138 ++--- .../dbsyncer/biz}/model/MetricResponse.java | 110 ++-- .../biz}/model/MetricResponseInfo.java | 46 +- .../java/org/dbsyncer/biz}/model/Sample.java | 56 +- .../dbsyncer/biz/vo/AppReportMetricVo.java | 2 +- .../org/dbsyncer/biz/vo/MetricResponseVo.java | 2 +- dbsyncer-monitor/pom.xml | 22 - .../java/org/dbsyncer/monitor/Monitor.java | 22 - .../dbsyncer/monitor/MonitorException.java | 27 - .../controller/monitor/MonitorController.java | 10 +- pom.xml | 3 +- 27 files changed, 630 insertions(+), 716 deletions(-) rename {dbsyncer-monitor/src/main/java/org/dbsyncer/monitor => dbsyncer-biz/src/main/java/org/dbsyncer/biz}/enums/BufferActuatorMetricEnum.java (96%) rename {dbsyncer-monitor/src/main/java/org/dbsyncer/monitor => dbsyncer-biz/src/main/java/org/dbsyncer/biz}/enums/DiskMetricEnum.java (90%) rename {dbsyncer-monitor/src/main/java/org/dbsyncer/monitor => dbsyncer-biz/src/main/java/org/dbsyncer/biz}/enums/MetricEnum.java (85%) rename {dbsyncer-monitor/src/main/java/org/dbsyncer/monitor => dbsyncer-biz/src/main/java/org/dbsyncer/biz}/enums/StatisticEnum.java (92%) rename {dbsyncer-monitor/src/main/java/org/dbsyncer/monitor => dbsyncer-biz/src/main/java/org/dbsyncer/biz}/enums/ThreadPoolMetricEnum.java (94%) rename dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/MonitorFactory.java => dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MetricReporter.java (90%) rename {dbsyncer-monitor/src/main/java/org/dbsyncer/monitor => dbsyncer-biz/src/main/java/org/dbsyncer/biz}/model/AppReportMetric.java (96%) rename {dbsyncer-monitor/src/main/java/org/dbsyncer/monitor => dbsyncer-biz/src/main/java/org/dbsyncer/biz}/model/MappingReportMetric.java (90%) rename {dbsyncer-monitor/src/main/java/org/dbsyncer/monitor => dbsyncer-biz/src/main/java/org/dbsyncer/biz}/model/MetricResponse.java (92%) rename {dbsyncer-monitor/src/main/java/org/dbsyncer/monitor => dbsyncer-biz/src/main/java/org/dbsyncer/biz}/model/MetricResponseInfo.java (87%) rename {dbsyncer-monitor/src/main/java/org/dbsyncer/monitor => dbsyncer-biz/src/main/java/org/dbsyncer/biz}/model/Sample.java (88%) delete mode 100644 dbsyncer-monitor/pom.xml delete mode 100644 dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/Monitor.java delete mode 100644 dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/MonitorException.java diff --git a/dbsyncer-biz/pom.xml b/dbsyncer-biz/pom.xml index 9f9902f6..f30e14aa 100644 --- a/dbsyncer-biz/pom.xml +++ b/dbsyncer-biz/pom.xml @@ -18,12 +18,5 @@ ${project.parent.version} - - - org.ghi - dbsyncer-monitor - ${project.parent.version} - - diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/MonitorService.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/MonitorService.java index c103d3fc..d2d5c07c 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/MonitorService.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/MonitorService.java @@ -3,8 +3,8 @@ package org.dbsyncer.biz; import org.dbsyncer.biz.vo.AppReportMetricVo; import org.dbsyncer.biz.vo.MetaVo; import org.dbsyncer.common.model.Paging; -import org.dbsyncer.monitor.enums.MetricEnum; -import org.dbsyncer.monitor.model.MetricResponse; +import org.dbsyncer.biz.enums.MetricEnum; +import org.dbsyncer.biz.model.MetricResponse; import org.dbsyncer.storage.enums.StorageDataStatusEnum; import java.util.List; diff --git a/dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/enums/BufferActuatorMetricEnum.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/enums/BufferActuatorMetricEnum.java similarity index 96% rename from dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/enums/BufferActuatorMetricEnum.java rename to dbsyncer-biz/src/main/java/org/dbsyncer/biz/enums/BufferActuatorMetricEnum.java index d7939c87..e2325fb3 100644 --- a/dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/enums/BufferActuatorMetricEnum.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/enums/BufferActuatorMetricEnum.java @@ -1,4 +1,4 @@ -package org.dbsyncer.monitor.enums; +package org.dbsyncer.biz.enums; /** * 缓存执行器指标 diff --git a/dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/enums/DiskMetricEnum.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/enums/DiskMetricEnum.java similarity index 90% rename from dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/enums/DiskMetricEnum.java rename to dbsyncer-biz/src/main/java/org/dbsyncer/biz/enums/DiskMetricEnum.java index 6975879a..110e7425 100644 --- a/dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/enums/DiskMetricEnum.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/enums/DiskMetricEnum.java @@ -1,48 +1,48 @@ -package org.dbsyncer.monitor.enums; - -/** - * 硬盘指标 - * - * @author AE86 - * @version 1.0.0 - * @date 2021/07/23 0:19 - */ -public enum DiskMetricEnum { - - /** - * 已用 - */ - THRESHOLD("disk.space.threshold", "硬盘", "已用"), - - /** - * 空闲 - */ - FREE("disk.space.free", "硬盘", "空闲"), - - /** - * 总共 - */ - TOTAL("disk.space.total", "硬盘", "总共"),; - - private String code; - private String group; - private String metricName; - - DiskMetricEnum(String code, String group, String metricName) { - this.code = code; - this.group = group; - this.metricName = metricName; - } - - public String getCode() { - return code; - } - - public String getGroup() { - return group; - } - - public String getMetricName() { - return metricName; - } -} +package org.dbsyncer.biz.enums; + +/** + * 硬盘指标 + * + * @author AE86 + * @version 1.0.0 + * @date 2021/07/23 0:19 + */ +public enum DiskMetricEnum { + + /** + * 已用 + */ + THRESHOLD("disk.space.threshold", "硬盘", "已用"), + + /** + * 空闲 + */ + FREE("disk.space.free", "硬盘", "空闲"), + + /** + * 总共 + */ + TOTAL("disk.space.total", "硬盘", "总共"),; + + private String code; + private String group; + private String metricName; + + DiskMetricEnum(String code, String group, String metricName) { + this.code = code; + this.group = group; + this.metricName = metricName; + } + + public String getCode() { + return code; + } + + public String getGroup() { + return group; + } + + public String getMetricName() { + return metricName; + } +} diff --git a/dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/enums/MetricEnum.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/enums/MetricEnum.java similarity index 85% rename from dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/enums/MetricEnum.java rename to dbsyncer-biz/src/main/java/org/dbsyncer/biz/enums/MetricEnum.java index 476ba7a4..22042d55 100644 --- a/dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/enums/MetricEnum.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/enums/MetricEnum.java @@ -1,82 +1,82 @@ -package org.dbsyncer.monitor.enums; - -import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.ConnectorException; -import org.dbsyncer.monitor.MonitorException; - -/** - * 系统指标 - * - * @author AE86 - * @version 1.0.0 - * @date 2021/07/22 19:19 - */ -public enum MetricEnum { - - /** - * 线程活跃数 - */ - THREADS_LIVE("jvm.threads.live", "应用线程", "活跃数"), - - /** - * 线程峰值 - */ - THREADS_PEAK("jvm.threads.peak", "应用线程", "峰值数"), - - /** - * 内存已用 - */ - MEMORY_USED("jvm.memory.used", "内存", "已用"), - - /** - * 内存空闲 - */ - MEMORY_COMMITTED("jvm.memory.committed", "内存", "空闲"), - - /** - * 内存总共 - */ - MEMORY_MAX("jvm.memory.max", "内存", "总共"), - - /** - * GC - */ - GC_PAUSE("jvm.gc.pause", "GC", "已用"), - - /** - * CPU已用 - */ - CPU_USAGE("system.cpu.usage", "CPU", "已用"); - - private String code; - private String group; - private String metricName; - - MetricEnum(String code, String group, String metricName) { - this.code = code; - this.group = group; - this.metricName = metricName; - } - - public static MetricEnum getMetric(String code) throws ConnectorException { - for (MetricEnum e : MetricEnum.values()) { - if (StringUtil.equals(code, e.getCode())) { - return e; - } - } - throw new MonitorException(String.format("Metric code \"%s\" does not exist.", code)); - } - - public String getCode() { - return code; - } - - public String getGroup() { - return group; - } - - public String getMetricName() { - return metricName; - } - +package org.dbsyncer.biz.enums; + +import org.dbsyncer.biz.BizException; +import org.dbsyncer.common.util.StringUtil; +import org.dbsyncer.connector.ConnectorException; + +/** + * 系统指标 + * + * @author AE86 + * @version 1.0.0 + * @date 2021/07/22 19:19 + */ +public enum MetricEnum { + + /** + * 线程活跃数 + */ + THREADS_LIVE("jvm.threads.live", "应用线程", "活跃数"), + + /** + * 线程峰值 + */ + THREADS_PEAK("jvm.threads.peak", "应用线程", "峰值数"), + + /** + * 内存已用 + */ + MEMORY_USED("jvm.memory.used", "内存", "已用"), + + /** + * 内存空闲 + */ + MEMORY_COMMITTED("jvm.memory.committed", "内存", "空闲"), + + /** + * 内存总共 + */ + MEMORY_MAX("jvm.memory.max", "内存", "总共"), + + /** + * GC + */ + GC_PAUSE("jvm.gc.pause", "GC", "已用"), + + /** + * CPU已用 + */ + CPU_USAGE("system.cpu.usage", "CPU", "已用"); + + private String code; + private String group; + private String metricName; + + MetricEnum(String code, String group, String metricName) { + this.code = code; + this.group = group; + this.metricName = metricName; + } + + public static MetricEnum getMetric(String code) throws ConnectorException { + for (MetricEnum e : MetricEnum.values()) { + if (StringUtil.equals(code, e.getCode())) { + return e; + } + } + throw new BizException(String.format("Metric code \"%s\" does not exist.", code)); + } + + public String getCode() { + return code; + } + + public String getGroup() { + return group; + } + + public String getMetricName() { + return metricName; + } + } \ No newline at end of file diff --git a/dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/enums/StatisticEnum.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/enums/StatisticEnum.java similarity index 92% rename from dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/enums/StatisticEnum.java rename to dbsyncer-biz/src/main/java/org/dbsyncer/biz/enums/StatisticEnum.java index cddcbb47..5b417159 100644 --- a/dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/enums/StatisticEnum.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/enums/StatisticEnum.java @@ -1,55 +1,55 @@ -package org.dbsyncer.monitor.enums; - -public enum StatisticEnum { - - /** - * The sum of the amounts recorded. - */ - TOTAL("total"), - - /** - * The sum of the times recorded. Reported in the monitoring system's base unit of time - */ - TOTAL_TIME("total"), - - /** - * Rate per second for calls. - */ - COUNT("count"), - - /** - * The maximum amount recorded. When this represents a time, it is reported in the monitoring system's base unit of time. - */ - MAX("max"), - - /** - * Instantaneous value, such as those reported by gauges. - */ - VALUE("value"), - - /** - * Undetermined. - */ - UNKNOWN("unknown"), - - /** - * Number of currently active tasks for a long task timer. - */ - ACTIVE_TASKS("active"), - - /** - * Duration of a running task in a long task timer. Always reported in the monitoring system's base unit of time. - */ - DURATION("duration"); - - private String tagValueRepresentation; - - StatisticEnum(String tagValueRepresentation) { - this.tagValueRepresentation = tagValueRepresentation; - } - - public String getTagValueRepresentation() { - return tagValueRepresentation; - } - +package org.dbsyncer.biz.enums; + +public enum StatisticEnum { + + /** + * The sum of the amounts recorded. + */ + TOTAL("total"), + + /** + * The sum of the times recorded. Reported in the monitoring system's base unit of time + */ + TOTAL_TIME("total"), + + /** + * Rate per second for calls. + */ + COUNT("count"), + + /** + * The maximum amount recorded. When this represents a time, it is reported in the monitoring system's base unit of time. + */ + MAX("max"), + + /** + * Instantaneous value, such as those reported by gauges. + */ + VALUE("value"), + + /** + * Undetermined. + */ + UNKNOWN("unknown"), + + /** + * Number of currently active tasks for a long task timer. + */ + ACTIVE_TASKS("active"), + + /** + * Duration of a running task in a long task timer. Always reported in the monitoring system's base unit of time. + */ + DURATION("duration"); + + private String tagValueRepresentation; + + StatisticEnum(String tagValueRepresentation) { + this.tagValueRepresentation = tagValueRepresentation; + } + + public String getTagValueRepresentation() { + return tagValueRepresentation; + } + } \ No newline at end of file diff --git a/dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/enums/ThreadPoolMetricEnum.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/enums/ThreadPoolMetricEnum.java similarity index 94% rename from dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/enums/ThreadPoolMetricEnum.java rename to dbsyncer-biz/src/main/java/org/dbsyncer/biz/enums/ThreadPoolMetricEnum.java index 8c8bbff0..8b3439cd 100644 --- a/dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/enums/ThreadPoolMetricEnum.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/enums/ThreadPoolMetricEnum.java @@ -1,4 +1,4 @@ -package org.dbsyncer.monitor.enums; +package org.dbsyncer.biz.enums; /** * 线程池指标 diff --git a/dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/MonitorFactory.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MetricReporter.java similarity index 90% rename from dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/MonitorFactory.java rename to dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MetricReporter.java index 142058aa..cfff49d0 100644 --- a/dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/MonitorFactory.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MetricReporter.java @@ -1,251 +1,244 @@ -package org.dbsyncer.monitor; - -import org.dbsyncer.common.model.Paging; -import org.dbsyncer.common.scheduled.ScheduledTaskJob; -import org.dbsyncer.common.scheduled.ScheduledTaskService; -import org.dbsyncer.common.util.CollectionUtils; -import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.constant.ConnectorConstant; -import org.dbsyncer.monitor.enums.BufferActuatorMetricEnum; -import org.dbsyncer.monitor.enums.MetricEnum; -import org.dbsyncer.monitor.enums.StatisticEnum; -import org.dbsyncer.monitor.enums.ThreadPoolMetricEnum; -import org.dbsyncer.monitor.model.AppReportMetric; -import org.dbsyncer.monitor.model.MappingReportMetric; -import org.dbsyncer.monitor.model.MetricResponse; -import org.dbsyncer.monitor.model.MetricResponseInfo; -import org.dbsyncer.monitor.model.Sample; -import org.dbsyncer.parser.ProfileComponent; -import org.dbsyncer.parser.flush.BufferActuator; -import org.dbsyncer.parser.flush.impl.BufferActuatorRouter; -import org.dbsyncer.parser.flush.impl.TableGroupBufferActuator; -import org.dbsyncer.parser.model.Mapping; -import org.dbsyncer.parser.model.Meta; -import org.dbsyncer.parser.model.TableGroup; -import org.dbsyncer.storage.StorageService; -import org.dbsyncer.storage.constant.ConfigConstant; -import org.dbsyncer.storage.enums.StorageDataStatusEnum; -import org.dbsyncer.storage.enums.StorageEnum; -import org.dbsyncer.storage.query.Query; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; -import org.springframework.stereotype.Component; - -import javax.annotation.PostConstruct; -import javax.annotation.Resource; -import java.time.LocalDateTime; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Comparator; -import java.util.List; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.atomic.AtomicLong; -import java.util.function.Consumer; -import java.util.stream.Collectors; - -/** - * @author AE86 - * @version 1.0.0 - * @date 2020/04/23 11:30 - */ -@Component -public class MonitorFactory implements Monitor, ScheduledTaskJob { - - private final Logger logger = LoggerFactory.getLogger(getClass()); - - @Resource - private ProfileComponent profileComponent; - - @Resource - private BufferActuator generalBufferActuator; - - @Resource - private BufferActuator storageBufferActuator; - - @Resource - private BufferActuatorRouter bufferActuatorRouter; - - @Resource - private ScheduledTaskService scheduledTaskService; - - @Resource - private StorageService storageService; - - private volatile boolean running; - - private LocalDateTime queryTime; - - private final MappingReportMetric mappingReportMetric = new MappingReportMetric(); - - private final int SHOW_BUFFER_ACTUATOR_SIZE = 6; - - @PostConstruct - private void init() { - scheduledTaskService.start(5000, this); - } - - @Override - public List getMetricEnumAll() { - return Arrays.asList(MetricEnum.values()); - } - - @Override - public List getMetricInfo() { - List list = new ArrayList<>(); - BufferActuatorMetricEnum general = BufferActuatorMetricEnum.GENERAL; - BufferActuatorMetricEnum storage = BufferActuatorMetricEnum.STORAGE; - list.add(collect(generalBufferActuator, general.getCode(), general.getGroup(), general.getMetricName())); - list.add(collect(storageBufferActuator, storage.getCode(), storage.getGroup(), storage.getMetricName())); - if (!CollectionUtils.isEmpty(bufferActuatorRouter.getRouter())) { - List tableList = new ArrayList<>(); - String tableGroupCode = BufferActuatorMetricEnum.TABLE_GROUP.getCode(); - bufferActuatorRouter.getRouter().forEach((metaId, group) -> { - Meta meta = profileComponent.getMeta(metaId); - Mapping mapping = profileComponent.getMapping(meta.getMappingId()); - group.forEach((k, bufferActuator) -> { - if (bufferActuator instanceof TableGroupBufferActuator) { - TableGroupBufferActuator actuator = bufferActuator; - TableGroup tableGroup = profileComponent.getTableGroup(actuator.getTableGroupId()); - String metricName = new StringBuilder() - .append(tableGroup.getSourceTable().getName()) - .append(" > ") - .append(tableGroup.getTargetTable().getName()).toString(); - tableList.add(collect(bufferActuator, tableGroupCode, mapping.getName(), metricName)); - } - }); - }); - List sortList = tableList.stream() - .sorted(Comparator.comparing(MetricResponseInfo::getQueueUp).reversed()) - .collect(Collectors.toList()); - list.addAll(sortList.size() <= SHOW_BUFFER_ACTUATOR_SIZE ? sortList : sortList.subList(0, SHOW_BUFFER_ACTUATOR_SIZE)); - } - return list.stream().map(info -> info.getResponse()).collect(Collectors.toList()); - } - - @Override - public AppReportMetric getAppReportMetric() { - queryTime = LocalDateTime.now(); - AppReportMetric report = new AppReportMetric(); - report.setSuccess(mappingReportMetric.getSuccess()); - report.setFail(mappingReportMetric.getFail()); - report.setInsert(mappingReportMetric.getInsert()); - report.setUpdate(mappingReportMetric.getUpdate()); - report.setDelete(mappingReportMetric.getDelete()); - // 堆积任务(通用执行器 + 表执行器) - report.setQueueUp(bufferActuatorRouter.getQueueSize().addAndGet(generalBufferActuator.getQueue().size())); - report.setQueueCapacity(bufferActuatorRouter.getQueueCapacity().addAndGet(generalBufferActuator.getQueueCapacity())); - // 持久化任务 - report.setStorageQueueUp(storageBufferActuator.getQueue().size()); - report.setStorageQueueCapacity(storageBufferActuator.getQueueCapacity()); - return report; - } - - @Override - public void run() { - if (running || null == queryTime) { - return; - } - // 非活动时间范围(30s内) - if (LocalDateTime.now().minusSeconds(30).isAfter(queryTime)) { - return; - } - - // 刷新报表 - try { - running = true; - final List metaAll = profileComponent.getMetaAll(); - mappingReportMetric.setSuccess(getMappingSuccess(metaAll)); - mappingReportMetric.setFail(getMappingFail(metaAll)); - mappingReportMetric.setInsert(getMappingInsert(metaAll)); - mappingReportMetric.setUpdate(getMappingUpdate(metaAll)); - mappingReportMetric.setDelete(getMappingDelete(metaAll)); - } catch (Exception e) { - logger.error(e.getMessage()); - } finally { - running = false; - } - } - - /** - * 获取所有驱动成功数 - * - * @param metaAll - * @return - */ - private long getMappingSuccess(List metaAll) { - return queryMappingMetricCount(metaAll, (query) -> query.addFilter(ConfigConstant.DATA_SUCCESS, StorageDataStatusEnum.SUCCESS.getValue())); - } - - /** - * 获取所有驱动失败数 - * - * @param metaAll - * @return - */ - private long getMappingFail(List metaAll) { - return queryMappingMetricCount(metaAll, (query) -> query.addFilter(ConfigConstant.DATA_SUCCESS, StorageDataStatusEnum.FAIL.getValue())); - } - - /** - * 获取所有驱动事件插入数 - * - * @param metaAll - * @return - */ - private long getMappingInsert(List metaAll) { - return queryMappingMetricCount(metaAll, (query) -> query.addFilter(ConfigConstant.DATA_EVENT, ConnectorConstant.OPERTION_INSERT)); - } - - /** - * 获取所有驱动事件更新数 - * - * @param metaAll - * @return - */ - private long getMappingUpdate(List metaAll) { - return queryMappingMetricCount(metaAll, (query) -> query.addFilter(ConfigConstant.DATA_EVENT, ConnectorConstant.OPERTION_UPDATE)); - } - - /** - * 获取所有驱动事件删除数 - * - * @param metaAll - * @return - */ - private long getMappingDelete(List metaAll) { - return queryMappingMetricCount(metaAll, (query) -> query.addFilter(ConfigConstant.DATA_EVENT, ConnectorConstant.OPERTION_DELETE)); - } - - private long queryMappingMetricCount(List metaAll, Consumer operation) { - AtomicLong total = new AtomicLong(0); - if (!CollectionUtils.isEmpty(metaAll)) { - Query query = new Query(1, 1); - query.setQueryTotal(true); - query.setType(StorageEnum.DATA); - operation.accept(query); - metaAll.forEach(meta -> { - query.setMetaId(meta.getId()); - Paging paging = storageService.query(query); - total.getAndAdd(paging.getTotal()); - }); - } - return total.get(); - } - - private MetricResponseInfo collect(BufferActuator bufferActuator, String code, String group, String metricName) { - MetricResponseInfo info = new MetricResponseInfo(); - ThreadPoolTaskExecutor threadTask = (ThreadPoolTaskExecutor) bufferActuator.getExecutor(); - ThreadPoolExecutor pool = threadTask.getThreadPoolExecutor(); - info.setQueueUp(bufferActuator.getQueue().size()); - StringBuilder msg = new StringBuilder(); - msg.append("堆积").append(StringUtil.COLON).append(info.getQueueUp()); - msg.append(StringUtil.FORWARD_SLASH).append(bufferActuator.getQueueCapacity()).append(StringUtil.SPACE); - msg.append(ThreadPoolMetricEnum.CORE_SIZE.getMetricName()).append(StringUtil.COLON).append(pool.getActiveCount()); - msg.append(StringUtil.FORWARD_SLASH).append(pool.getCorePoolSize()).append(StringUtil.SPACE); - msg.append(ThreadPoolMetricEnum.COMPLETED.getMetricName()).append(StringUtil.COLON).append(pool.getCompletedTaskCount()); - info.setResponse(new MetricResponse(code, group, metricName, Arrays.asList(new Sample(StatisticEnum.COUNT.getTagValueRepresentation(), msg.toString())))); - return info; - } - +package org.dbsyncer.biz.impl; + +import org.dbsyncer.biz.enums.BufferActuatorMetricEnum; +import org.dbsyncer.biz.enums.MetricEnum; +import org.dbsyncer.biz.enums.StatisticEnum; +import org.dbsyncer.biz.enums.ThreadPoolMetricEnum; +import org.dbsyncer.biz.model.AppReportMetric; +import org.dbsyncer.biz.model.MappingReportMetric; +import org.dbsyncer.biz.model.MetricResponse; +import org.dbsyncer.biz.model.MetricResponseInfo; +import org.dbsyncer.biz.model.Sample; +import org.dbsyncer.common.model.Paging; +import org.dbsyncer.common.scheduled.ScheduledTaskJob; +import org.dbsyncer.common.scheduled.ScheduledTaskService; +import org.dbsyncer.common.util.CollectionUtils; +import org.dbsyncer.common.util.StringUtil; +import org.dbsyncer.connector.constant.ConnectorConstant; +import org.dbsyncer.parser.ProfileComponent; +import org.dbsyncer.parser.flush.BufferActuator; +import org.dbsyncer.parser.flush.impl.BufferActuatorRouter; +import org.dbsyncer.parser.flush.impl.TableGroupBufferActuator; +import org.dbsyncer.parser.model.Mapping; +import org.dbsyncer.parser.model.Meta; +import org.dbsyncer.parser.model.TableGroup; +import org.dbsyncer.storage.StorageService; +import org.dbsyncer.storage.constant.ConfigConstant; +import org.dbsyncer.storage.enums.StorageDataStatusEnum; +import org.dbsyncer.storage.enums.StorageEnum; +import org.dbsyncer.storage.query.Query; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import javax.annotation.Resource; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +/** + * @author AE86 + * @version 1.0.0 + * @date 2020/04/23 11:30 + */ +@Component +public class MetricReporter implements ScheduledTaskJob { + + private final Logger logger = LoggerFactory.getLogger(getClass()); + + @Resource + private ProfileComponent profileComponent; + + @Resource + private BufferActuator generalBufferActuator; + + @Resource + private BufferActuator storageBufferActuator; + + @Resource + private BufferActuatorRouter bufferActuatorRouter; + + @Resource + private ScheduledTaskService scheduledTaskService; + + @Resource + private StorageService storageService; + + private volatile boolean running; + + private LocalDateTime queryTime; + + private final MappingReportMetric mappingReportMetric = new MappingReportMetric(); + + private final int SHOW_BUFFER_ACTUATOR_SIZE = 6; + + @PostConstruct + private void init() { + scheduledTaskService.start(5000, this); + } + + public List getMetricInfo() { + List list = new ArrayList<>(); + BufferActuatorMetricEnum general = BufferActuatorMetricEnum.GENERAL; + BufferActuatorMetricEnum storage = BufferActuatorMetricEnum.STORAGE; + list.add(collect(generalBufferActuator, general.getCode(), general.getGroup(), general.getMetricName())); + list.add(collect(storageBufferActuator, storage.getCode(), storage.getGroup(), storage.getMetricName())); + if (!CollectionUtils.isEmpty(bufferActuatorRouter.getRouter())) { + List tableList = new ArrayList<>(); + String tableGroupCode = BufferActuatorMetricEnum.TABLE_GROUP.getCode(); + bufferActuatorRouter.getRouter().forEach((metaId, group) -> { + Meta meta = profileComponent.getMeta(metaId); + Mapping mapping = profileComponent.getMapping(meta.getMappingId()); + group.forEach((k, bufferActuator) -> { + if (bufferActuator instanceof TableGroupBufferActuator) { + TableGroupBufferActuator actuator = bufferActuator; + TableGroup tableGroup = profileComponent.getTableGroup(actuator.getTableGroupId()); + String metricName = new StringBuilder() + .append(tableGroup.getSourceTable().getName()) + .append(" > ") + .append(tableGroup.getTargetTable().getName()).toString(); + tableList.add(collect(bufferActuator, tableGroupCode, mapping.getName(), metricName)); + } + }); + }); + List sortList = tableList.stream() + .sorted(Comparator.comparing(MetricResponseInfo::getQueueUp).reversed()) + .collect(Collectors.toList()); + list.addAll(sortList.size() <= SHOW_BUFFER_ACTUATOR_SIZE ? sortList : sortList.subList(0, SHOW_BUFFER_ACTUATOR_SIZE)); + } + return list.stream().map(info -> info.getResponse()).collect(Collectors.toList()); + } + + public AppReportMetric getAppReportMetric() { + queryTime = LocalDateTime.now(); + AppReportMetric report = new AppReportMetric(); + report.setSuccess(mappingReportMetric.getSuccess()); + report.setFail(mappingReportMetric.getFail()); + report.setInsert(mappingReportMetric.getInsert()); + report.setUpdate(mappingReportMetric.getUpdate()); + report.setDelete(mappingReportMetric.getDelete()); + // 堆积任务(通用执行器 + 表执行器) + report.setQueueUp(bufferActuatorRouter.getQueueSize().addAndGet(generalBufferActuator.getQueue().size())); + report.setQueueCapacity(bufferActuatorRouter.getQueueCapacity().addAndGet(generalBufferActuator.getQueueCapacity())); + // 持久化任务 + report.setStorageQueueUp(storageBufferActuator.getQueue().size()); + report.setStorageQueueCapacity(storageBufferActuator.getQueueCapacity()); + return report; + } + + @Override + public void run() { + if (running || null == queryTime) { + return; + } + // 非活动时间范围(30s内) + if (LocalDateTime.now().minusSeconds(30).isAfter(queryTime)) { + return; + } + + // 刷新报表 + try { + running = true; + final List metaAll = profileComponent.getMetaAll(); + mappingReportMetric.setSuccess(getMappingSuccess(metaAll)); + mappingReportMetric.setFail(getMappingFail(metaAll)); + mappingReportMetric.setInsert(getMappingInsert(metaAll)); + mappingReportMetric.setUpdate(getMappingUpdate(metaAll)); + mappingReportMetric.setDelete(getMappingDelete(metaAll)); + } catch (Exception e) { + logger.error(e.getMessage()); + } finally { + running = false; + } + } + + /** + * 获取所有驱动成功数 + * + * @param metaAll + * @return + */ + private long getMappingSuccess(List metaAll) { + return queryMappingMetricCount(metaAll, (query) -> query.addFilter(ConfigConstant.DATA_SUCCESS, StorageDataStatusEnum.SUCCESS.getValue())); + } + + /** + * 获取所有驱动失败数 + * + * @param metaAll + * @return + */ + private long getMappingFail(List metaAll) { + return queryMappingMetricCount(metaAll, (query) -> query.addFilter(ConfigConstant.DATA_SUCCESS, StorageDataStatusEnum.FAIL.getValue())); + } + + /** + * 获取所有驱动事件插入数 + * + * @param metaAll + * @return + */ + private long getMappingInsert(List metaAll) { + return queryMappingMetricCount(metaAll, (query) -> query.addFilter(ConfigConstant.DATA_EVENT, ConnectorConstant.OPERTION_INSERT)); + } + + /** + * 获取所有驱动事件更新数 + * + * @param metaAll + * @return + */ + private long getMappingUpdate(List metaAll) { + return queryMappingMetricCount(metaAll, (query) -> query.addFilter(ConfigConstant.DATA_EVENT, ConnectorConstant.OPERTION_UPDATE)); + } + + /** + * 获取所有驱动事件删除数 + * + * @param metaAll + * @return + */ + private long getMappingDelete(List metaAll) { + return queryMappingMetricCount(metaAll, (query) -> query.addFilter(ConfigConstant.DATA_EVENT, ConnectorConstant.OPERTION_DELETE)); + } + + private long queryMappingMetricCount(List metaAll, Consumer operation) { + AtomicLong total = new AtomicLong(0); + if (!CollectionUtils.isEmpty(metaAll)) { + Query query = new Query(1, 1); + query.setQueryTotal(true); + query.setType(StorageEnum.DATA); + operation.accept(query); + metaAll.forEach(meta -> { + query.setMetaId(meta.getId()); + Paging paging = storageService.query(query); + total.getAndAdd(paging.getTotal()); + }); + } + return total.get(); + } + + private MetricResponseInfo collect(BufferActuator bufferActuator, String code, String group, String metricName) { + MetricResponseInfo info = new MetricResponseInfo(); + ThreadPoolTaskExecutor threadTask = (ThreadPoolTaskExecutor) bufferActuator.getExecutor(); + ThreadPoolExecutor pool = threadTask.getThreadPoolExecutor(); + info.setQueueUp(bufferActuator.getQueue().size()); + StringBuilder msg = new StringBuilder(); + msg.append("堆积").append(StringUtil.COLON).append(info.getQueueUp()); + msg.append(StringUtil.FORWARD_SLASH).append(bufferActuator.getQueueCapacity()).append(StringUtil.SPACE); + msg.append(ThreadPoolMetricEnum.CORE_SIZE.getMetricName()).append(StringUtil.COLON).append(pool.getActiveCount()); + msg.append(StringUtil.FORWARD_SLASH).append(pool.getCorePoolSize()).append(StringUtil.SPACE); + msg.append(ThreadPoolMetricEnum.COMPLETED.getMetricName()).append(StringUtil.COLON).append(pool.getCompletedTaskCount()); + info.setResponse(new MetricResponse(code, group, metricName, Arrays.asList(new Sample(StatisticEnum.COUNT.getTagValueRepresentation(), msg.toString())))); + return info; + } + } \ No newline at end of file diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java index 5317a696..bd79fffb 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java @@ -24,12 +24,11 @@ import org.dbsyncer.common.util.JsonUtil; import org.dbsyncer.common.util.NumberUtil; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.connector.enums.FilterEnum; -import org.dbsyncer.monitor.Monitor; -import org.dbsyncer.monitor.enums.BufferActuatorMetricEnum; -import org.dbsyncer.monitor.enums.DiskMetricEnum; -import org.dbsyncer.monitor.enums.MetricEnum; -import org.dbsyncer.monitor.model.AppReportMetric; -import org.dbsyncer.monitor.model.MetricResponse; +import org.dbsyncer.biz.enums.BufferActuatorMetricEnum; +import org.dbsyncer.biz.enums.DiskMetricEnum; +import org.dbsyncer.biz.enums.MetricEnum; +import org.dbsyncer.biz.model.AppReportMetric; +import org.dbsyncer.biz.model.MetricResponse; import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.parser.enums.MetaEnum; import org.dbsyncer.parser.enums.ModelEnum; @@ -57,6 +56,7 @@ import javax.annotation.Resource; import java.sql.Timestamp; import java.time.LocalDateTime; import java.util.ArrayList; +import java.util.Arrays; import java.util.Comparator; import java.util.LinkedHashMap; import java.util.List; @@ -74,7 +74,7 @@ public class MonitorServiceImpl extends BaseServiceImpl implements MonitorServic private final Logger logger = LoggerFactory.getLogger(getClass()); @Resource - private Monitor monitor; + private MetricReporter metricReporter; @Resource private ProfileComponent profileComponent; @@ -218,12 +218,12 @@ public class MonitorServiceImpl extends BaseServiceImpl implements MonitorServic @Override public List getMetricEnumAll() { - return monitor.getMetricEnumAll(); + return Arrays.asList(MetricEnum.values()); } @Override public AppReportMetricVo queryAppReportMetric(List metrics) { - AppReportMetric appReportMetric = monitor.getAppReportMetric(); + AppReportMetric appReportMetric = metricReporter.getAppReportMetric(); AppReportMetricVo vo = new AppReportMetricVo(); BeanUtils.copyProperties(appReportMetric, vo); vo.setMetrics(getMetrics(metrics)); @@ -334,7 +334,7 @@ public class MonitorServiceImpl extends BaseServiceImpl implements MonitorServic private List getMetrics(List metrics) { // 系统指标 - List metricList = monitor.getMetricInfo(); + List metricList = metricReporter.getMetricInfo(); // 线程池状态 metrics.addAll(metricList); diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/metric/impl/CpuMetricDetailFormatter.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/metric/impl/CpuMetricDetailFormatter.java index dc465898..cda5fa7e 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/metric/impl/CpuMetricDetailFormatter.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/metric/impl/CpuMetricDetailFormatter.java @@ -2,7 +2,7 @@ package org.dbsyncer.biz.metric.impl; import org.dbsyncer.biz.metric.AbstractMetricDetailFormatter; import org.dbsyncer.biz.vo.MetricResponseVo; -import org.dbsyncer.monitor.model.Sample; +import org.dbsyncer.biz.model.Sample; public final class CpuMetricDetailFormatter extends AbstractMetricDetailFormatter { diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/metric/impl/DiskMetricDetailFormatter.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/metric/impl/DiskMetricDetailFormatter.java index 90181256..2d78fab3 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/metric/impl/DiskMetricDetailFormatter.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/metric/impl/DiskMetricDetailFormatter.java @@ -2,7 +2,7 @@ package org.dbsyncer.biz.metric.impl; import org.dbsyncer.biz.metric.AbstractMetricDetailFormatter; import org.dbsyncer.biz.vo.MetricResponseVo; -import org.dbsyncer.monitor.model.Sample; +import org.dbsyncer.biz.model.Sample; import java.math.BigDecimal; diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/metric/impl/DoubleRoundMetricDetailFormatter.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/metric/impl/DoubleRoundMetricDetailFormatter.java index d9f19730..8f02e4ee 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/metric/impl/DoubleRoundMetricDetailFormatter.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/metric/impl/DoubleRoundMetricDetailFormatter.java @@ -2,7 +2,7 @@ package org.dbsyncer.biz.metric.impl; import org.dbsyncer.biz.metric.AbstractMetricDetailFormatter; import org.dbsyncer.biz.vo.MetricResponseVo; -import org.dbsyncer.monitor.model.Sample; +import org.dbsyncer.biz.model.Sample; public final class DoubleRoundMetricDetailFormatter extends AbstractMetricDetailFormatter { diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/metric/impl/GCMetricDetailFormatter.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/metric/impl/GCMetricDetailFormatter.java index 6cdb596b..0604985d 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/metric/impl/GCMetricDetailFormatter.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/metric/impl/GCMetricDetailFormatter.java @@ -2,7 +2,7 @@ package org.dbsyncer.biz.metric.impl; import org.dbsyncer.biz.metric.AbstractMetricDetailFormatter; import org.dbsyncer.biz.vo.MetricResponseVo; -import org.dbsyncer.monitor.model.Sample; +import org.dbsyncer.biz.model.Sample; import java.util.List; diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/metric/impl/MemoryMetricDetailFormatter.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/metric/impl/MemoryMetricDetailFormatter.java index 331b83d9..9b447a5c 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/metric/impl/MemoryMetricDetailFormatter.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/metric/impl/MemoryMetricDetailFormatter.java @@ -2,7 +2,7 @@ package org.dbsyncer.biz.metric.impl; import org.dbsyncer.biz.metric.AbstractMetricDetailFormatter; import org.dbsyncer.biz.vo.MetricResponseVo; -import org.dbsyncer.monitor.model.Sample; +import org.dbsyncer.biz.model.Sample; public final class MemoryMetricDetailFormatter extends AbstractMetricDetailFormatter { diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/metric/impl/ValueMetricDetailFormatter.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/metric/impl/ValueMetricDetailFormatter.java index 02c4c427..51ac221a 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/metric/impl/ValueMetricDetailFormatter.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/metric/impl/ValueMetricDetailFormatter.java @@ -2,7 +2,7 @@ package org.dbsyncer.biz.metric.impl; import org.dbsyncer.biz.metric.AbstractMetricDetailFormatter; import org.dbsyncer.biz.vo.MetricResponseVo; -import org.dbsyncer.monitor.model.Sample; +import org.dbsyncer.biz.model.Sample; public final class ValueMetricDetailFormatter extends AbstractMetricDetailFormatter { diff --git a/dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/model/AppReportMetric.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/model/AppReportMetric.java similarity index 96% rename from dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/model/AppReportMetric.java rename to dbsyncer-biz/src/main/java/org/dbsyncer/biz/model/AppReportMetric.java index 7c75ea9c..3f62040c 100644 --- a/dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/model/AppReportMetric.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/model/AppReportMetric.java @@ -1,4 +1,4 @@ -package org.dbsyncer.monitor.model; +package org.dbsyncer.biz.model; public class AppReportMetric extends MappingReportMetric{ diff --git a/dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/model/MappingReportMetric.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/model/MappingReportMetric.java similarity index 90% rename from dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/model/MappingReportMetric.java rename to dbsyncer-biz/src/main/java/org/dbsyncer/biz/model/MappingReportMetric.java index 5f4c546c..2f9fe938 100644 --- a/dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/model/MappingReportMetric.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/model/MappingReportMetric.java @@ -1,70 +1,70 @@ -package org.dbsyncer.monitor.model; - -public class MappingReportMetric { - - /** - * 已处理成功数 - */ - private long success; - - /** - * 已处理失败数 - */ - private long fail; - - /** - * 插入事件 - */ - private long insert; - - /** - * 更新事件 - */ - private long update; - - /** - * 删除事件 - */ - private long delete; - - public long getSuccess() { - return success; - } - - public void setSuccess(long success) { - this.success = success; - } - - public long getFail() { - return fail; - } - - public void setFail(long fail) { - this.fail = fail; - } - - public long getInsert() { - return insert; - } - - public void setInsert(long insert) { - this.insert = insert; - } - - public long getUpdate() { - return update; - } - - public void setUpdate(long update) { - this.update = update; - } - - public long getDelete() { - return delete; - } - - public void setDelete(long delete) { - this.delete = delete; - } - +package org.dbsyncer.biz.model; + +public class MappingReportMetric { + + /** + * 已处理成功数 + */ + private long success; + + /** + * 已处理失败数 + */ + private long fail; + + /** + * 插入事件 + */ + private long insert; + + /** + * 更新事件 + */ + private long update; + + /** + * 删除事件 + */ + private long delete; + + public long getSuccess() { + return success; + } + + public void setSuccess(long success) { + this.success = success; + } + + public long getFail() { + return fail; + } + + public void setFail(long fail) { + this.fail = fail; + } + + public long getInsert() { + return insert; + } + + public void setInsert(long insert) { + this.insert = insert; + } + + public long getUpdate() { + return update; + } + + public void setUpdate(long update) { + this.update = update; + } + + public long getDelete() { + return delete; + } + + public void setDelete(long delete) { + this.delete = delete; + } + } \ No newline at end of file diff --git a/dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/model/MetricResponse.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/model/MetricResponse.java similarity index 92% rename from dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/model/MetricResponse.java rename to dbsyncer-biz/src/main/java/org/dbsyncer/biz/model/MetricResponse.java index 1461774c..fb217787 100644 --- a/dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/model/MetricResponse.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/model/MetricResponse.java @@ -1,56 +1,56 @@ -package org.dbsyncer.monitor.model; - -import java.util.List; - -public class MetricResponse { - - private String code; - - private String group; - - private String metricName; - - private List measurements; - - public MetricResponse() { - } - - public MetricResponse(String code, String group, String metricName, List measurements) { - this.code = code; - this.group = group; - this.metricName = metricName; - this.measurements = measurements; - } - - public String getCode() { - return code; - } - - public void setCode(String code) { - this.code = code; - } - - public String getGroup() { - return group; - } - - public void setGroup(String group) { - this.group = group; - } - - public String getMetricName() { - return metricName; - } - - public void setMetricName(String metricName) { - this.metricName = metricName; - } - - public List getMeasurements() { - return measurements; - } - - public void setMeasurements(List measurements) { - this.measurements = measurements; - } +package org.dbsyncer.biz.model; + +import java.util.List; + +public class MetricResponse { + + private String code; + + private String group; + + private String metricName; + + private List measurements; + + public MetricResponse() { + } + + public MetricResponse(String code, String group, String metricName, List measurements) { + this.code = code; + this.group = group; + this.metricName = metricName; + this.measurements = measurements; + } + + public String getCode() { + return code; + } + + public void setCode(String code) { + this.code = code; + } + + public String getGroup() { + return group; + } + + public void setGroup(String group) { + this.group = group; + } + + public String getMetricName() { + return metricName; + } + + public void setMetricName(String metricName) { + this.metricName = metricName; + } + + public List getMeasurements() { + return measurements; + } + + public void setMeasurements(List measurements) { + this.measurements = measurements; + } } \ No newline at end of file diff --git a/dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/model/MetricResponseInfo.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/model/MetricResponseInfo.java similarity index 87% rename from dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/model/MetricResponseInfo.java rename to dbsyncer-biz/src/main/java/org/dbsyncer/biz/model/MetricResponseInfo.java index b815879a..b1b90cee 100644 --- a/dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/model/MetricResponseInfo.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/model/MetricResponseInfo.java @@ -1,24 +1,24 @@ -package org.dbsyncer.monitor.model; - -public class MetricResponseInfo { - - private MetricResponse response; - - private long queueUp; - - public MetricResponse getResponse() { - return response; - } - - public void setResponse(MetricResponse response) { - this.response = response; - } - - public long getQueueUp() { - return queueUp; - } - - public void setQueueUp(long queueUp) { - this.queueUp = queueUp; - } +package org.dbsyncer.biz.model; + +public class MetricResponseInfo { + + private MetricResponse response; + + private long queueUp; + + public MetricResponse getResponse() { + return response; + } + + public void setResponse(MetricResponse response) { + this.response = response; + } + + public long getQueueUp() { + return queueUp; + } + + public void setQueueUp(long queueUp) { + this.queueUp = queueUp; + } } \ No newline at end of file diff --git a/dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/model/Sample.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/model/Sample.java similarity index 88% rename from dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/model/Sample.java rename to dbsyncer-biz/src/main/java/org/dbsyncer/biz/model/Sample.java index 9c68aa24..f23c7cc3 100644 --- a/dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/model/Sample.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/model/Sample.java @@ -1,29 +1,29 @@ -package org.dbsyncer.monitor.model; - -public final class Sample { - - private String statistic; - - private Object value; - - public Sample(String statistic, Object value) { - this.statistic = statistic; - this.value = value; - } - - public String getStatistic() { - return statistic; - } - - public void setStatistic(String statistic) { - this.statistic = statistic; - } - - public Object getValue() { - return value; - } - - public void setValue(Object value) { - this.value = value; - } +package org.dbsyncer.biz.model; + +public final class Sample { + + private String statistic; + + private Object value; + + public Sample(String statistic, Object value) { + this.statistic = statistic; + this.value = value; + } + + public String getStatistic() { + return statistic; + } + + public void setStatistic(String statistic) { + this.statistic = statistic; + } + + public Object getValue() { + return value; + } + + public void setValue(Object value) { + this.value = value; + } } \ No newline at end of file diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/vo/AppReportMetricVo.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/vo/AppReportMetricVo.java index 058d612a..ac816bc1 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/vo/AppReportMetricVo.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/vo/AppReportMetricVo.java @@ -1,6 +1,6 @@ package org.dbsyncer.biz.vo; -import org.dbsyncer.monitor.model.AppReportMetric; +import org.dbsyncer.biz.model.AppReportMetric; import java.util.List; diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/vo/MetricResponseVo.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/vo/MetricResponseVo.java index 6c4f25b6..71863ee8 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/vo/MetricResponseVo.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/vo/MetricResponseVo.java @@ -1,6 +1,6 @@ package org.dbsyncer.biz.vo; -import org.dbsyncer.monitor.model.MetricResponse; +import org.dbsyncer.biz.model.MetricResponse; public class MetricResponseVo extends MetricResponse { diff --git a/dbsyncer-monitor/pom.xml b/dbsyncer-monitor/pom.xml deleted file mode 100644 index aff70d1c..00000000 --- a/dbsyncer-monitor/pom.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - dbsyncer - org.ghi - 1.2.7_1103 - - 4.0.0 - dbsyncer-monitor - - - - - org.ghi - dbsyncer-manager - ${project.parent.version} - - - - \ No newline at end of file diff --git a/dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/Monitor.java b/dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/Monitor.java deleted file mode 100644 index 869cf1b1..00000000 --- a/dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/Monitor.java +++ /dev/null @@ -1,22 +0,0 @@ -package org.dbsyncer.monitor; - -import org.dbsyncer.monitor.enums.MetricEnum; -import org.dbsyncer.monitor.model.AppReportMetric; -import org.dbsyncer.monitor.model.MetricResponse; - -import java.util.List; - -/** - * @author AE86 - * @version 1.0.0 - * @date 2020/04/23 11:30 - */ -public interface Monitor { - - List getMetricEnumAll(); - - List getMetricInfo(); - - AppReportMetric getAppReportMetric(); - -} \ No newline at end of file diff --git a/dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/MonitorException.java b/dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/MonitorException.java deleted file mode 100644 index 5d92a6f1..00000000 --- a/dbsyncer-monitor/src/main/java/org/dbsyncer/monitor/MonitorException.java +++ /dev/null @@ -1,27 +0,0 @@ -package org.dbsyncer.monitor; - -/** - * @author AE86 - * @version 1.0.0 - * @date 2021/7/23 22:39 - */ -public class MonitorException extends RuntimeException { - - private static final long serialVersionUID = 1L; - - public MonitorException(String message) { - super(message); - } - - public MonitorException(String message, Throwable cause) { - super(message, cause); - } - - public MonitorException(Throwable cause) { - super(cause); - } - - protected MonitorException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { - super(message, cause, enableSuppression, writableStackTrace); - } -} diff --git a/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/monitor/MonitorController.java b/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/monitor/MonitorController.java index 00d2ba22..fab2c423 100644 --- a/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/monitor/MonitorController.java +++ b/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/monitor/MonitorController.java @@ -10,11 +10,11 @@ import org.dbsyncer.biz.vo.RestResult; import org.dbsyncer.biz.vo.SystemConfigVo; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.DateFormatUtil; -import org.dbsyncer.monitor.enums.DiskMetricEnum; -import org.dbsyncer.monitor.enums.MetricEnum; -import org.dbsyncer.monitor.enums.StatisticEnum; -import org.dbsyncer.monitor.model.MetricResponse; -import org.dbsyncer.monitor.model.Sample; +import org.dbsyncer.biz.enums.DiskMetricEnum; +import org.dbsyncer.biz.enums.MetricEnum; +import org.dbsyncer.biz.enums.StatisticEnum; +import org.dbsyncer.biz.model.MetricResponse; +import org.dbsyncer.biz.model.Sample; import org.dbsyncer.web.controller.BaseController; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/pom.xml b/pom.xml index 69b35ada..949fa21b 100644 --- a/pom.xml +++ b/pom.xml @@ -11,7 +11,7 @@ dbsyncer https://gitee.com/ghi/dbsyncer - + dbsyncer-biz dbsyncer-cluster @@ -19,7 +19,6 @@ dbsyncer-connector dbsyncer-listener dbsyncer-manager - dbsyncer-monitor dbsyncer-parser dbsyncer-plugin dbsyncer-storage -- Gitee From f3df2e433c94b7d3f320098f3231da8802ef2817 Mon Sep 17 00:00:00 2001 From: AE86 <836391306@qq.com> Date: Tue, 14 Nov 2023 23:15:22 +0800 Subject: [PATCH 04/98] move event --- .../biz/impl/DataSyncServiceImpl.java | 2 +- .../dbsyncer/biz/impl/MonitorServiceImpl.java | 2 +- .../listener/AbstractDatabaseExtractor.java | 3 +- .../dbsyncer/listener/AbstractExtractor.java | 4 +- .../org/dbsyncer/listener}/ChangedEvent.java | 90 +++++------ .../java/org/dbsyncer/listener/Extractor.java | 4 +- .../java/org/dbsyncer/listener}/Watcher.java | 2 +- .../listener}/event/CommonChangedEvent.java | 147 +++++++++--------- .../listener}/event/DDLChangedEvent.java | 2 +- .../listener}/event/RowChangedEvent.java | 66 ++++---- .../listener}/event/ScanChangedEvent.java | 2 +- .../dbsyncer/listener/file/FileExtractor.java | 4 +- .../listener/model}/ChangedOffset.java | 116 +++++++------- .../listener/mysql/DqlMySQLExtractor.java | 2 +- .../listener/mysql/MySQLExtractor.java | 6 +- .../listener/oracle/DqlOracleExtractor.java | 2 +- .../oracle/dcn/DBChangeNotification.java | 2 +- .../listener/oracle/dcn/RowEventListener.java | 2 +- .../postgresql/DqlPostgreSQLExtractor.java | 2 +- .../listener/postgresql/MessageDecoder.java | 2 +- .../postgresql/PostgreSQLExtractor.java | 4 +- .../decoder/PgOutputMessageDecoder.java | 2 +- .../decoder/TestDecodingMessageDecoder.java | 2 +- .../quartz/AbstractQuartzExtractor.java | 2 +- .../sqlserver/DqlSqlServerExtractor.java | 2 +- .../sqlserver/SqlServerExtractor.java | 4 +- .../dbsyncer/manager}/event/ClosedEvent.java | 2 +- .../manager}/event/PreloadCompletedEvent.java | 46 +++--- .../manager/listener/ClosedTaskListener.java | 2 +- .../manager/puller/AbstractPuller.java | 2 +- .../manager/puller/IncrementPuller.java | 4 +- .../manager/template/PreloadTemplate.java | 2 +- .../org/dbsyncer/parser/ParserComponent.java | 2 +- .../parser/consumer/AbstractConsumer.java | 6 +- .../parser/consumer/impl/LogConsumer.java | 6 +- .../parser/consumer/impl/QuartzConsumer.java | 2 +- .../parser}/event/RefreshOffsetEvent.java | 65 ++++---- .../flush/impl/BufferActuatorRouter.java | 2 +- .../flush/impl/GeneralBufferActuator.java | 2 +- .../parser/impl/ParserComponentImpl.java | 2 +- .../dbsyncer/parser/model/WriterRequest.java | 6 +- .../dbsyncer/parser/model/WriterResponse.java | 2 +- 42 files changed, 316 insertions(+), 315 deletions(-) rename {dbsyncer-common/src/main/java/org/dbsyncer/common/event => dbsyncer-listener/src/main/java/org/dbsyncer/listener}/ChangedEvent.java (88%) rename {dbsyncer-common/src/main/java/org/dbsyncer/common/event => dbsyncer-listener/src/main/java/org/dbsyncer/listener}/Watcher.java (94%) rename {dbsyncer-common/src/main/java/org/dbsyncer/common => dbsyncer-listener/src/main/java/org/dbsyncer/listener}/event/CommonChangedEvent.java (91%) rename {dbsyncer-common/src/main/java/org/dbsyncer/common => dbsyncer-listener/src/main/java/org/dbsyncer/listener}/event/DDLChangedEvent.java (95%) rename {dbsyncer-common/src/main/java/org/dbsyncer/common => dbsyncer-listener/src/main/java/org/dbsyncer/listener}/event/RowChangedEvent.java (92%) rename {dbsyncer-common/src/main/java/org/dbsyncer/common => dbsyncer-listener/src/main/java/org/dbsyncer/listener}/event/ScanChangedEvent.java (93%) rename {dbsyncer-common/src/main/java/org/dbsyncer/common/event => dbsyncer-listener/src/main/java/org/dbsyncer/listener/model}/ChangedOffset.java (91%) rename {dbsyncer-common/src/main/java/org/dbsyncer/common => dbsyncer-manager/src/main/java/org/dbsyncer/manager}/event/ClosedEvent.java (94%) rename {dbsyncer-common/src/main/java/org/dbsyncer/common => dbsyncer-manager/src/main/java/org/dbsyncer/manager}/event/PreloadCompletedEvent.java (90%) rename {dbsyncer-common/src/main/java/org/dbsyncer/common => dbsyncer-parser/src/main/java/org/dbsyncer/parser}/event/RefreshOffsetEvent.java (90%) diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/DataSyncServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/DataSyncServiceImpl.java index 57af4fbc..99cc6039 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/DataSyncServiceImpl.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/DataSyncServiceImpl.java @@ -4,7 +4,7 @@ import com.google.protobuf.InvalidProtocolBufferException; import org.dbsyncer.biz.DataSyncService; import org.dbsyncer.biz.vo.BinlogColumnVo; import org.dbsyncer.biz.vo.MessageVo; -import org.dbsyncer.common.event.RowChangedEvent; +import org.dbsyncer.listener.event.RowChangedEvent; import org.dbsyncer.common.model.Paging; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.DateFormatUtil; diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java index bd79fffb..9881005b 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java @@ -15,7 +15,7 @@ import org.dbsyncer.biz.vo.DataVo; import org.dbsyncer.biz.vo.LogVo; import org.dbsyncer.biz.vo.MetaVo; import org.dbsyncer.biz.vo.MetricResponseVo; -import org.dbsyncer.common.event.PreloadCompletedEvent; +import org.dbsyncer.manager.event.PreloadCompletedEvent; import org.dbsyncer.common.model.Paging; import org.dbsyncer.common.scheduled.ScheduledTaskJob; import org.dbsyncer.common.scheduled.ScheduledTaskService; diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/AbstractDatabaseExtractor.java b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/AbstractDatabaseExtractor.java index 9f212005..b6d1f28d 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/AbstractDatabaseExtractor.java +++ b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/AbstractDatabaseExtractor.java @@ -1,7 +1,6 @@ package org.dbsyncer.listener; -import org.dbsyncer.common.event.ChangedEvent; -import org.dbsyncer.common.event.RowChangedEvent; +import org.dbsyncer.listener.event.RowChangedEvent; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.connector.constant.ConnectorConstant; diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/AbstractExtractor.java b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/AbstractExtractor.java index 9069aabd..43084663 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/AbstractExtractor.java +++ b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/AbstractExtractor.java @@ -1,8 +1,6 @@ package org.dbsyncer.listener; -import org.dbsyncer.common.event.ChangedEvent; -import org.dbsyncer.common.event.ChangedOffset; -import org.dbsyncer.common.event.Watcher; +import org.dbsyncer.listener.model.ChangedOffset; import org.dbsyncer.common.model.AbstractConnectorConfig; import org.dbsyncer.common.scheduled.ScheduledTaskService; import org.dbsyncer.common.util.CollectionUtils; diff --git a/dbsyncer-common/src/main/java/org/dbsyncer/common/event/ChangedEvent.java b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/ChangedEvent.java similarity index 88% rename from dbsyncer-common/src/main/java/org/dbsyncer/common/event/ChangedEvent.java rename to dbsyncer-listener/src/main/java/org/dbsyncer/listener/ChangedEvent.java index 2b516981..cb9a60cb 100644 --- a/dbsyncer-common/src/main/java/org/dbsyncer/common/event/ChangedEvent.java +++ b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/ChangedEvent.java @@ -1,45 +1,47 @@ -/** - * DBSyncer Copyright 2019-2024 All Rights Reserved. - */ -package org.dbsyncer.common.event; - -import java.util.Map; - -/** - * 变更事件 - * - * @version 1.0.0 - * @Author AE86 - * @Date 2023-08-20 20:00 - */ -public interface ChangedEvent { - - /** - * 获取变更表名称 - * - * @return - */ - String getSourceTableName(); - - /** - * 获取变更事件 - * - * @return - */ - String getEvent(); - - /** - * 获取变更行数据 - * - * @return - */ - Map getChangedRow(); - - /** - * 获取增量偏移量 - * - * @return - */ - ChangedOffset getChangedOffset(); - +/** + * DBSyncer Copyright 2019-2024 All Rights Reserved. + */ +package org.dbsyncer.listener; + +import org.dbsyncer.listener.model.ChangedOffset; + +import java.util.Map; + +/** + * 变更事件 + * + * @version 1.0.0 + * @Author AE86 + * @Date 2023-08-20 20:00 + */ +public interface ChangedEvent { + + /** + * 获取变更表名称 + * + * @return + */ + String getSourceTableName(); + + /** + * 获取变更事件 + * + * @return + */ + String getEvent(); + + /** + * 获取变更行数据 + * + * @return + */ + Map getChangedRow(); + + /** + * 获取增量偏移量 + * + * @return + */ + ChangedOffset getChangedOffset(); + } \ No newline at end of file diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/Extractor.java b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/Extractor.java index ca810114..f1e9b8dc 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/Extractor.java +++ b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/Extractor.java @@ -1,8 +1,6 @@ package org.dbsyncer.listener; -import org.dbsyncer.common.event.ChangedEvent; -import org.dbsyncer.common.event.ChangedOffset; -import org.dbsyncer.common.event.Watcher; +import org.dbsyncer.listener.model.ChangedOffset; public interface Extractor { diff --git a/dbsyncer-common/src/main/java/org/dbsyncer/common/event/Watcher.java b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/Watcher.java similarity index 94% rename from dbsyncer-common/src/main/java/org/dbsyncer/common/event/Watcher.java rename to dbsyncer-listener/src/main/java/org/dbsyncer/listener/Watcher.java index 43ecd48e..9d41c055 100644 --- a/dbsyncer-common/src/main/java/org/dbsyncer/common/event/Watcher.java +++ b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/Watcher.java @@ -1,4 +1,4 @@ -package org.dbsyncer.common.event; +package org.dbsyncer.listener; import java.util.Map; diff --git a/dbsyncer-common/src/main/java/org/dbsyncer/common/event/CommonChangedEvent.java b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/event/CommonChangedEvent.java similarity index 91% rename from dbsyncer-common/src/main/java/org/dbsyncer/common/event/CommonChangedEvent.java rename to dbsyncer-listener/src/main/java/org/dbsyncer/listener/event/CommonChangedEvent.java index 782682a8..570e175e 100644 --- a/dbsyncer-common/src/main/java/org/dbsyncer/common/event/CommonChangedEvent.java +++ b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/event/CommonChangedEvent.java @@ -1,73 +1,76 @@ -/** - * DBSyncer Copyright 2019-2024 All Rights Reserved. - */ -package org.dbsyncer.common.event; - -import java.util.Map; - -/** - * 通用变更事件 - * - * @version 1.0.0 - * @Author AE86 - * @Date 2023-08-20 20:00 - */ -public class CommonChangedEvent implements ChangedEvent { - - /** - * 变更表名称 - */ - private String sourceTableName; - /** - * 变更事件 - */ - private String event; - /** - * 变更行数据 - */ - private Map changedRow; - /** - * 增量偏移量 - */ - private ChangedOffset changedOffset = new ChangedOffset(); - - @Override - public String getSourceTableName() { - return sourceTableName; - } - - public void setSourceTableName(String sourceTableName) { - this.sourceTableName = sourceTableName; - } - - @Override - public String getEvent() { - return event; - } - - public void setEvent(String event) { - this.event = event; - } - - @Override - public Map getChangedRow() { - return changedRow; - } - - public void setChangedRow(Map changedRow) { - this.changedRow = changedRow; - } - - @Override - public ChangedOffset getChangedOffset() { - return changedOffset; - } - - public void setNextFileName(String nextFileName) { - changedOffset.setNextFileName(nextFileName); - } - - public void setPosition(Object position) { - changedOffset.setPosition(position); - } +/** + * DBSyncer Copyright 2019-2024 All Rights Reserved. + */ +package org.dbsyncer.listener.event; + +import org.dbsyncer.listener.ChangedEvent; +import org.dbsyncer.listener.model.ChangedOffset; + +import java.util.Map; + +/** + * 通用变更事件 + * + * @version 1.0.0 + * @Author AE86 + * @Date 2023-08-20 20:00 + */ +public class CommonChangedEvent implements ChangedEvent { + + /** + * 变更表名称 + */ + private String sourceTableName; + /** + * 变更事件 + */ + private String event; + /** + * 变更行数据 + */ + private Map changedRow; + /** + * 增量偏移量 + */ + private ChangedOffset changedOffset = new ChangedOffset(); + + @Override + public String getSourceTableName() { + return sourceTableName; + } + + public void setSourceTableName(String sourceTableName) { + this.sourceTableName = sourceTableName; + } + + @Override + public String getEvent() { + return event; + } + + public void setEvent(String event) { + this.event = event; + } + + @Override + public Map getChangedRow() { + return changedRow; + } + + public void setChangedRow(Map changedRow) { + this.changedRow = changedRow; + } + + @Override + public ChangedOffset getChangedOffset() { + return changedOffset; + } + + public void setNextFileName(String nextFileName) { + changedOffset.setNextFileName(nextFileName); + } + + public void setPosition(Object position) { + changedOffset.setPosition(position); + } } \ No newline at end of file diff --git a/dbsyncer-common/src/main/java/org/dbsyncer/common/event/DDLChangedEvent.java b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/event/DDLChangedEvent.java similarity index 95% rename from dbsyncer-common/src/main/java/org/dbsyncer/common/event/DDLChangedEvent.java rename to dbsyncer-listener/src/main/java/org/dbsyncer/listener/event/DDLChangedEvent.java index ae8a75d9..577bbecb 100644 --- a/dbsyncer-common/src/main/java/org/dbsyncer/common/event/DDLChangedEvent.java +++ b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/event/DDLChangedEvent.java @@ -1,4 +1,4 @@ -package org.dbsyncer.common.event; +package org.dbsyncer.listener.event; /** * DDL变更事件 diff --git a/dbsyncer-common/src/main/java/org/dbsyncer/common/event/RowChangedEvent.java b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/event/RowChangedEvent.java similarity index 92% rename from dbsyncer-common/src/main/java/org/dbsyncer/common/event/RowChangedEvent.java rename to dbsyncer-listener/src/main/java/org/dbsyncer/listener/event/RowChangedEvent.java index 83a10b10..3f2d922a 100644 --- a/dbsyncer-common/src/main/java/org/dbsyncer/common/event/RowChangedEvent.java +++ b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/event/RowChangedEvent.java @@ -1,34 +1,34 @@ -/** - * DBSyncer Copyright 2019-2024 All Rights Reserved. - */ -package org.dbsyncer.common.event; - -import java.util.List; - -/** - * 监听行变更事件 - * - * @version 1.0.0 - * @Author AE86 - * @Date 2020-06-15 20:00 - */ -public final class RowChangedEvent extends CommonChangedEvent { - private List dataList; - - public RowChangedEvent(String sourceTableName, String event, List data) { - this(sourceTableName, event, data, null, null); - } - - public RowChangedEvent(String sourceTableName, String event, List data, String nextFileName, Object position) { - setSourceTableName(sourceTableName); - setEvent(event); - setNextFileName(nextFileName); - setPosition(position); - this.dataList = data; - } - - public List getDataList() { - return dataList; - } - +/** + * DBSyncer Copyright 2019-2024 All Rights Reserved. + */ +package org.dbsyncer.listener.event; + +import java.util.List; + +/** + * 监听行变更事件 + * + * @version 1.0.0 + * @Author AE86 + * @Date 2020-06-15 20:00 + */ +public final class RowChangedEvent extends CommonChangedEvent { + private List dataList; + + public RowChangedEvent(String sourceTableName, String event, List data) { + this(sourceTableName, event, data, null, null); + } + + public RowChangedEvent(String sourceTableName, String event, List data, String nextFileName, Object position) { + setSourceTableName(sourceTableName); + setEvent(event); + setNextFileName(nextFileName); + setPosition(position); + this.dataList = data; + } + + public List getDataList() { + return dataList; + } + } \ No newline at end of file diff --git a/dbsyncer-common/src/main/java/org/dbsyncer/common/event/ScanChangedEvent.java b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/event/ScanChangedEvent.java similarity index 93% rename from dbsyncer-common/src/main/java/org/dbsyncer/common/event/ScanChangedEvent.java rename to dbsyncer-listener/src/main/java/org/dbsyncer/listener/event/ScanChangedEvent.java index 1f45e2ae..3fc58218 100644 --- a/dbsyncer-common/src/main/java/org/dbsyncer/common/event/ScanChangedEvent.java +++ b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/event/ScanChangedEvent.java @@ -1,7 +1,7 @@ /** * DBSyncer Copyright 2019-2024 All Rights Reserved. */ -package org.dbsyncer.common.event; +package org.dbsyncer.listener.event; import java.util.Map; diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/file/FileExtractor.java b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/file/FileExtractor.java index d40c66d0..6e013b12 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/file/FileExtractor.java +++ b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/file/FileExtractor.java @@ -1,8 +1,8 @@ package org.dbsyncer.listener.file; import org.apache.commons.io.IOUtils; -import org.dbsyncer.common.event.ChangedOffset; -import org.dbsyncer.common.event.RowChangedEvent; +import org.dbsyncer.listener.model.ChangedOffset; +import org.dbsyncer.listener.event.RowChangedEvent; import org.dbsyncer.common.file.BufferedRandomAccessFile; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.NumberUtil; diff --git a/dbsyncer-common/src/main/java/org/dbsyncer/common/event/ChangedOffset.java b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/model/ChangedOffset.java similarity index 91% rename from dbsyncer-common/src/main/java/org/dbsyncer/common/event/ChangedOffset.java rename to dbsyncer-listener/src/main/java/org/dbsyncer/listener/model/ChangedOffset.java index 8e92f5cf..d0050946 100644 --- a/dbsyncer-common/src/main/java/org/dbsyncer/common/event/ChangedOffset.java +++ b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/model/ChangedOffset.java @@ -1,59 +1,59 @@ -package org.dbsyncer.common.event; - -/** - * 增量偏移量 - * - * @version 1.0.0 - * @Author AE86 - * @Date 2023-08-23 20:00 - */ -public final class ChangedOffset { - /** - * 驱动ID - */ - private String metaId; - /** - * 增量文件名称 - */ - private String nextFileName; - /** - * 增量偏移量 - */ - private Object position; - /** - * 是否触发刷新增量点事件 - */ - private boolean refreshOffset; - - public String getMetaId() { - return metaId; - } - - public void setMetaId(String metaId) { - this.metaId = metaId; - } - - public String getNextFileName() { - return nextFileName; - } - - public void setNextFileName(String nextFileName) { - this.nextFileName = nextFileName; - } - - public Object getPosition() { - return position; - } - - public void setPosition(Object position) { - this.position = position; - } - - public boolean isRefreshOffset() { - return refreshOffset; - } - - public void setRefreshOffset(boolean refreshOffset) { - this.refreshOffset = refreshOffset; - } +package org.dbsyncer.listener.model; + +/** + * 增量偏移量 + * + * @version 1.0.0 + * @Author AE86 + * @Date 2023-08-23 20:00 + */ +public final class ChangedOffset { + /** + * 驱动ID + */ + private String metaId; + /** + * 增量文件名称 + */ + private String nextFileName; + /** + * 增量偏移量 + */ + private Object position; + /** + * 是否触发刷新增量点事件 + */ + private boolean refreshOffset; + + public String getMetaId() { + return metaId; + } + + public void setMetaId(String metaId) { + this.metaId = metaId; + } + + public String getNextFileName() { + return nextFileName; + } + + public void setNextFileName(String nextFileName) { + this.nextFileName = nextFileName; + } + + public Object getPosition() { + return position; + } + + public void setPosition(Object position) { + this.position = position; + } + + public boolean isRefreshOffset() { + return refreshOffset; + } + + public void setRefreshOffset(boolean refreshOffset) { + this.refreshOffset = refreshOffset; + } } \ No newline at end of file diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/mysql/DqlMySQLExtractor.java b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/mysql/DqlMySQLExtractor.java index 7e85ea14..ee73eb87 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/mysql/DqlMySQLExtractor.java +++ b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/mysql/DqlMySQLExtractor.java @@ -1,6 +1,6 @@ package org.dbsyncer.listener.mysql; -import org.dbsyncer.common.event.ChangedEvent; +import org.dbsyncer.listener.ChangedEvent; /** * @author AE86 diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/mysql/MySQLExtractor.java b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/mysql/MySQLExtractor.java index d5b9d8a6..44a6d15e 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/mysql/MySQLExtractor.java +++ b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/mysql/MySQLExtractor.java @@ -14,9 +14,9 @@ import com.github.shyiko.mysql.binlog.network.ServerException; import net.sf.jsqlparser.JSQLParserException; import net.sf.jsqlparser.parser.CCJSqlParserUtil; import net.sf.jsqlparser.statement.alter.Alter; -import org.dbsyncer.common.event.ChangedOffset; -import org.dbsyncer.common.event.DDLChangedEvent; -import org.dbsyncer.common.event.RowChangedEvent; +import org.dbsyncer.listener.model.ChangedOffset; +import org.dbsyncer.listener.event.DDLChangedEvent; +import org.dbsyncer.listener.event.RowChangedEvent; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.connector.config.DatabaseConfig; import org.dbsyncer.connector.constant.ConnectorConstant; diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/oracle/DqlOracleExtractor.java b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/oracle/DqlOracleExtractor.java index 5b28bb1b..95e6f27f 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/oracle/DqlOracleExtractor.java +++ b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/oracle/DqlOracleExtractor.java @@ -1,6 +1,6 @@ package org.dbsyncer.listener.oracle; -import org.dbsyncer.common.event.ChangedEvent; +import org.dbsyncer.listener.ChangedEvent; import org.dbsyncer.connector.model.Field; import java.util.List; diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/oracle/dcn/DBChangeNotification.java b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/oracle/dcn/DBChangeNotification.java index 12ede471..8e920870 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/oracle/dcn/DBChangeNotification.java +++ b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/oracle/dcn/DBChangeNotification.java @@ -11,7 +11,7 @@ import oracle.jdbc.dcn.DatabaseChangeRegistration; import oracle.jdbc.dcn.RowChangeDescription; import oracle.jdbc.dcn.TableChangeDescription; import oracle.jdbc.driver.OracleConnection; -import org.dbsyncer.common.event.RowChangedEvent; +import org.dbsyncer.listener.event.RowChangedEvent; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.connector.constant.ConnectorConstant; import org.dbsyncer.listener.ListenerException; diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/oracle/dcn/RowEventListener.java b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/oracle/dcn/RowEventListener.java index 69886190..a5e9e358 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/oracle/dcn/RowEventListener.java +++ b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/oracle/dcn/RowEventListener.java @@ -3,7 +3,7 @@ */ package org.dbsyncer.listener.oracle.dcn; -import org.dbsyncer.common.event.RowChangedEvent; +import org.dbsyncer.listener.event.RowChangedEvent; /** * 行变更监听器 diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/DqlPostgreSQLExtractor.java b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/DqlPostgreSQLExtractor.java index 4503864d..6db7f7f4 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/DqlPostgreSQLExtractor.java +++ b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/DqlPostgreSQLExtractor.java @@ -1,6 +1,6 @@ package org.dbsyncer.listener.postgresql; -import org.dbsyncer.common.event.ChangedEvent; +import org.dbsyncer.listener.ChangedEvent; /** * @author AE86 diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/MessageDecoder.java b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/MessageDecoder.java index f093726f..ebb711fd 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/MessageDecoder.java +++ b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/MessageDecoder.java @@ -1,6 +1,6 @@ package org.dbsyncer.listener.postgresql; -import org.dbsyncer.common.event.RowChangedEvent; +import org.dbsyncer.listener.event.RowChangedEvent; import org.dbsyncer.connector.ConnectorFactory; import org.dbsyncer.connector.config.DatabaseConfig; import org.dbsyncer.connector.database.DatabaseConnectorMapper; diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/PostgreSQLExtractor.java b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/PostgreSQLExtractor.java index d5a787bb..68466eed 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/PostgreSQLExtractor.java +++ b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/PostgreSQLExtractor.java @@ -1,7 +1,7 @@ package org.dbsyncer.listener.postgresql; -import org.dbsyncer.common.event.ChangedOffset; -import org.dbsyncer.common.event.RowChangedEvent; +import org.dbsyncer.listener.model.ChangedOffset; +import org.dbsyncer.listener.event.RowChangedEvent; import org.dbsyncer.common.util.BooleanUtil; import org.dbsyncer.connector.config.DatabaseConfig; import org.dbsyncer.connector.database.DatabaseConnectorMapper; diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/decoder/PgOutputMessageDecoder.java b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/decoder/PgOutputMessageDecoder.java index eb8c4c94..cc9857dd 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/decoder/PgOutputMessageDecoder.java +++ b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/decoder/PgOutputMessageDecoder.java @@ -1,6 +1,6 @@ package org.dbsyncer.listener.postgresql.decoder; -import org.dbsyncer.common.event.RowChangedEvent; +import org.dbsyncer.listener.event.RowChangedEvent; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.connector.ConnectorFactory; import org.dbsyncer.connector.database.DatabaseConnectorMapper; diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/decoder/TestDecodingMessageDecoder.java b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/decoder/TestDecodingMessageDecoder.java index cddb1e9d..9e46deae 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/decoder/TestDecodingMessageDecoder.java +++ b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/decoder/TestDecodingMessageDecoder.java @@ -1,7 +1,7 @@ package org.dbsyncer.listener.postgresql.decoder; import org.dbsyncer.common.column.Lexer; -import org.dbsyncer.common.event.RowChangedEvent; +import org.dbsyncer.listener.event.RowChangedEvent; import org.dbsyncer.connector.constant.ConnectorConstant; import org.dbsyncer.listener.postgresql.AbstractMessageDecoder; import org.dbsyncer.listener.postgresql.enums.MessageDecoderEnum; diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/AbstractQuartzExtractor.java b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/AbstractQuartzExtractor.java index e1e25571..b52f5242 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/AbstractQuartzExtractor.java +++ b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/AbstractQuartzExtractor.java @@ -1,6 +1,6 @@ package org.dbsyncer.listener.quartz; -import org.dbsyncer.common.event.ScanChangedEvent; +import org.dbsyncer.listener.event.ScanChangedEvent; import org.dbsyncer.common.model.Result; import org.dbsyncer.common.scheduled.ScheduledTaskJob; import org.dbsyncer.common.spi.ConnectorMapper; diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/sqlserver/DqlSqlServerExtractor.java b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/sqlserver/DqlSqlServerExtractor.java index 7aff02bf..ba8f306a 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/sqlserver/DqlSqlServerExtractor.java +++ b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/sqlserver/DqlSqlServerExtractor.java @@ -1,6 +1,6 @@ package org.dbsyncer.listener.sqlserver; -import org.dbsyncer.common.event.ChangedEvent; +import org.dbsyncer.listener.ChangedEvent; /** * @author AE86 diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/sqlserver/SqlServerExtractor.java b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/sqlserver/SqlServerExtractor.java index 07c51272..e029ce0a 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/sqlserver/SqlServerExtractor.java +++ b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/sqlserver/SqlServerExtractor.java @@ -1,8 +1,8 @@ package org.dbsyncer.listener.sqlserver; import com.microsoft.sqlserver.jdbc.SQLServerException; -import org.dbsyncer.common.event.ChangedOffset; -import org.dbsyncer.common.event.RowChangedEvent; +import org.dbsyncer.listener.model.ChangedOffset; +import org.dbsyncer.listener.event.RowChangedEvent; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.connector.config.DatabaseConfig; import org.dbsyncer.connector.constant.ConnectorConstant; diff --git a/dbsyncer-common/src/main/java/org/dbsyncer/common/event/ClosedEvent.java b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/event/ClosedEvent.java similarity index 94% rename from dbsyncer-common/src/main/java/org/dbsyncer/common/event/ClosedEvent.java rename to dbsyncer-manager/src/main/java/org/dbsyncer/manager/event/ClosedEvent.java index 53a3deed..c598d240 100644 --- a/dbsyncer-common/src/main/java/org/dbsyncer/common/event/ClosedEvent.java +++ b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/event/ClosedEvent.java @@ -1,4 +1,4 @@ -package org.dbsyncer.common.event; +package org.dbsyncer.manager.event; import org.springframework.context.ApplicationContext; import org.springframework.context.event.ApplicationContextEvent; diff --git a/dbsyncer-common/src/main/java/org/dbsyncer/common/event/PreloadCompletedEvent.java b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/event/PreloadCompletedEvent.java similarity index 90% rename from dbsyncer-common/src/main/java/org/dbsyncer/common/event/PreloadCompletedEvent.java rename to dbsyncer-manager/src/main/java/org/dbsyncer/manager/event/PreloadCompletedEvent.java index d977b872..15efd27b 100644 --- a/dbsyncer-common/src/main/java/org/dbsyncer/common/event/PreloadCompletedEvent.java +++ b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/event/PreloadCompletedEvent.java @@ -1,24 +1,24 @@ -package org.dbsyncer.common.event; - -import org.springframework.context.ApplicationContext; -import org.springframework.context.event.ApplicationContextEvent; - -/** - * 预加载完成事件 - * - * @version 1.0.0 - * @Author AE86 - * @Date 2020-08-26 22:45 - */ -public final class PreloadCompletedEvent extends ApplicationContextEvent { - - /** - * Create a new ContextStartedEvent. - * - * @param source the {@code ApplicationContext} that the event is raised for (must not be {@code null}) - */ - public PreloadCompletedEvent(ApplicationContext source) { - super(source); - } - +package org.dbsyncer.manager.event; + +import org.springframework.context.ApplicationContext; +import org.springframework.context.event.ApplicationContextEvent; + +/** + * 预加载完成事件 + * + * @version 1.0.0 + * @Author AE86 + * @Date 2020-08-26 22:45 + */ +public final class PreloadCompletedEvent extends ApplicationContextEvent { + + /** + * Create a new ContextStartedEvent. + * + * @param source the {@code ApplicationContext} that the event is raised for (must not be {@code null}) + */ + public PreloadCompletedEvent(ApplicationContext source) { + super(source); + } + } \ No newline at end of file diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/listener/ClosedTaskListener.java b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/listener/ClosedTaskListener.java index 031f4f14..4f8404be 100644 --- a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/listener/ClosedTaskListener.java +++ b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/listener/ClosedTaskListener.java @@ -3,7 +3,7 @@ */ package org.dbsyncer.manager.listener; -import org.dbsyncer.common.event.ClosedEvent; +import org.dbsyncer.manager.event.ClosedEvent; import org.dbsyncer.manager.ManagerFactory; import org.dbsyncer.parser.enums.MetaEnum; import org.springframework.context.ApplicationListener; diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/puller/AbstractPuller.java b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/puller/AbstractPuller.java index 94ac2fab..c5315a9d 100644 --- a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/puller/AbstractPuller.java +++ b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/puller/AbstractPuller.java @@ -1,6 +1,6 @@ package org.dbsyncer.manager.puller; -import org.dbsyncer.common.event.ClosedEvent; +import org.dbsyncer.manager.event.ClosedEvent; import org.dbsyncer.manager.Puller; import org.springframework.context.ApplicationContext; diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/puller/IncrementPuller.java b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/puller/IncrementPuller.java index 03dd70d7..6cb7e826 100644 --- a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/puller/IncrementPuller.java +++ b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/puller/IncrementPuller.java @@ -1,7 +1,7 @@ package org.dbsyncer.manager.puller; -import org.dbsyncer.common.event.ChangedOffset; -import org.dbsyncer.common.event.RefreshOffsetEvent; +import org.dbsyncer.listener.model.ChangedOffset; +import org.dbsyncer.parser.event.RefreshOffsetEvent; import org.dbsyncer.common.model.AbstractConnectorConfig; import org.dbsyncer.common.scheduled.ScheduledTaskJob; import org.dbsyncer.common.scheduled.ScheduledTaskService; diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/template/PreloadTemplate.java b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/template/PreloadTemplate.java index 1f57b83b..de7025a0 100644 --- a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/template/PreloadTemplate.java +++ b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/template/PreloadTemplate.java @@ -1,6 +1,6 @@ package org.dbsyncer.manager.template; -import org.dbsyncer.common.event.PreloadCompletedEvent; +import org.dbsyncer.manager.event.PreloadCompletedEvent; import org.dbsyncer.common.model.Paging; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.JsonUtil; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ParserComponent.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ParserComponent.java index bdebeaf7..62361540 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ParserComponent.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ParserComponent.java @@ -1,6 +1,6 @@ package org.dbsyncer.parser; -import org.dbsyncer.common.event.ChangedEvent; +import org.dbsyncer.listener.ChangedEvent; import org.dbsyncer.common.model.Result; import org.dbsyncer.common.spi.ConnectorMapper; import org.dbsyncer.common.spi.ConvertContext; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/AbstractConsumer.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/AbstractConsumer.java index 627a1d83..680d498a 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/AbstractConsumer.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/AbstractConsumer.java @@ -3,9 +3,9 @@ */ package org.dbsyncer.parser.consumer; -import org.dbsyncer.common.event.ChangedEvent; -import org.dbsyncer.common.event.DDLChangedEvent; -import org.dbsyncer.common.event.Watcher; +import org.dbsyncer.listener.ChangedEvent; +import org.dbsyncer.listener.event.DDLChangedEvent; +import org.dbsyncer.listener.Watcher; import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.parser.flush.impl.BufferActuatorRouter; import org.dbsyncer.parser.LogService; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/impl/LogConsumer.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/impl/LogConsumer.java index eb664a44..c35ee769 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/impl/LogConsumer.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/impl/LogConsumer.java @@ -3,9 +3,9 @@ */ package org.dbsyncer.parser.consumer.impl; -import org.dbsyncer.common.event.CommonChangedEvent; -import org.dbsyncer.common.event.DDLChangedEvent; -import org.dbsyncer.common.event.RowChangedEvent; +import org.dbsyncer.listener.event.CommonChangedEvent; +import org.dbsyncer.listener.event.DDLChangedEvent; +import org.dbsyncer.listener.event.RowChangedEvent; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.connector.model.Table; import org.dbsyncer.parser.consumer.AbstractConsumer; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/impl/QuartzConsumer.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/impl/QuartzConsumer.java index 5bb77a6c..839c6c6e 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/impl/QuartzConsumer.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/impl/QuartzConsumer.java @@ -3,7 +3,7 @@ */ package org.dbsyncer.parser.consumer.impl; -import org.dbsyncer.common.event.ScanChangedEvent; +import org.dbsyncer.listener.event.ScanChangedEvent; import org.dbsyncer.parser.consumer.AbstractConsumer; import org.dbsyncer.parser.model.FieldPicker; import org.dbsyncer.parser.model.TableGroup; diff --git a/dbsyncer-common/src/main/java/org/dbsyncer/common/event/RefreshOffsetEvent.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/event/RefreshOffsetEvent.java similarity index 90% rename from dbsyncer-common/src/main/java/org/dbsyncer/common/event/RefreshOffsetEvent.java rename to dbsyncer-parser/src/main/java/org/dbsyncer/parser/event/RefreshOffsetEvent.java index a752c9e9..87809d4f 100644 --- a/dbsyncer-common/src/main/java/org/dbsyncer/common/event/RefreshOffsetEvent.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/event/RefreshOffsetEvent.java @@ -1,33 +1,34 @@ -package org.dbsyncer.common.event; - -import org.springframework.context.ApplicationContext; -import org.springframework.context.event.ApplicationContextEvent; - -import java.util.List; - -/** - * 刷新偏移量事件 - * - * @version 1.0.0 - * @Author AE86 - * @Date 2023-08-23 22:45 - */ -public final class RefreshOffsetEvent extends ApplicationContextEvent { - - private List offsetList; - - /** - * Create a new ContextStartedEvent. - * - * @param source the {@code ApplicationContext} that the event is raised for - * (must not be {@code null}) - */ - public RefreshOffsetEvent(ApplicationContext source, List offsetList) { - super(source); - this.offsetList = offsetList; - } - - public List getOffsetList() { - return offsetList; - } +package org.dbsyncer.parser.event; + +import org.dbsyncer.listener.model.ChangedOffset; +import org.springframework.context.ApplicationContext; +import org.springframework.context.event.ApplicationContextEvent; + +import java.util.List; + +/** + * 刷新偏移量事件 + * + * @version 1.0.0 + * @Author AE86 + * @Date 2023-08-23 22:45 + */ +public final class RefreshOffsetEvent extends ApplicationContextEvent { + + private List offsetList; + + /** + * Create a new ContextStartedEvent. + * + * @param source the {@code ApplicationContext} that the event is raised for + * (must not be {@code null}) + */ + public RefreshOffsetEvent(ApplicationContext source, List offsetList) { + super(source); + this.offsetList = offsetList; + } + + public List getOffsetList() { + return offsetList; + } } \ No newline at end of file diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/BufferActuatorRouter.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/BufferActuatorRouter.java index 04d73611..e9828654 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/BufferActuatorRouter.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/BufferActuatorRouter.java @@ -4,7 +4,7 @@ package org.dbsyncer.parser.flush.impl; import org.dbsyncer.common.config.TableGroupBufferConfig; -import org.dbsyncer.common.event.ChangedEvent; +import org.dbsyncer.listener.ChangedEvent; import org.dbsyncer.parser.flush.BufferActuator; import org.dbsyncer.parser.model.WriterRequest; import org.slf4j.Logger; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/GeneralBufferActuator.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/GeneralBufferActuator.java index aed90b4c..8e44fcb1 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/GeneralBufferActuator.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/GeneralBufferActuator.java @@ -2,7 +2,7 @@ package org.dbsyncer.parser.flush.impl; import org.dbsyncer.parser.CacheService; import org.dbsyncer.common.config.GeneralBufferConfig; -import org.dbsyncer.common.event.RefreshOffsetEvent; +import org.dbsyncer.parser.event.RefreshOffsetEvent; import org.dbsyncer.common.model.AbstractConnectorConfig; import org.dbsyncer.common.model.IncrementConvertContext; import org.dbsyncer.common.model.Result; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/ParserComponentImpl.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/ParserComponentImpl.java index 47d2d989..7621c461 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/ParserComponentImpl.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/ParserComponentImpl.java @@ -1,6 +1,6 @@ package org.dbsyncer.parser.impl; -import org.dbsyncer.common.event.ChangedEvent; +import org.dbsyncer.listener.ChangedEvent; import org.dbsyncer.common.model.AbstractConnectorConfig; import org.dbsyncer.common.model.FullConvertContext; import org.dbsyncer.common.model.Result; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/WriterRequest.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/WriterRequest.java index 8a661c8c..81b1d4a7 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/WriterRequest.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/WriterRequest.java @@ -1,8 +1,8 @@ package org.dbsyncer.parser.model; -import org.dbsyncer.common.event.ChangedEvent; -import org.dbsyncer.common.event.ChangedOffset; -import org.dbsyncer.common.event.DDLChangedEvent; +import org.dbsyncer.listener.ChangedEvent; +import org.dbsyncer.listener.model.ChangedOffset; +import org.dbsyncer.listener.event.DDLChangedEvent; import org.dbsyncer.parser.flush.BufferRequest; import java.util.Map; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/WriterResponse.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/WriterResponse.java index cebb603e..526a89a0 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/WriterResponse.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/WriterResponse.java @@ -1,6 +1,6 @@ package org.dbsyncer.parser.model; -import org.dbsyncer.common.event.ChangedOffset; +import org.dbsyncer.listener.model.ChangedOffset; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.parser.flush.BufferResponse; -- Gitee From 1817e62d79ffa0850d90874423f76b4414150b95 Mon Sep 17 00:00:00 2001 From: AE86 <836391306@qq.com> Date: Wed, 15 Nov 2023 00:33:00 +0800 Subject: [PATCH 05/98] add sdk --- dbsyncer-biz/pom.xml | 2 +- .../impl/connector/ConnectorChecker.java | 12 +- .../biz/impl/ConnectorServiceImpl.java | 4 +- dbsyncer-cluster/pom.xml | 2 +- dbsyncer-common/pom.xml | 3 +- dbsyncer-connector/pom.xml | 9 +- .../dbsyncer/connector/AbstractConnector.java | 2 +- .../connector/AbstractValueMapper.java | 10 +- .../org/dbsyncer/connector/Connector.java | 4 +- .../dbsyncer/connector/ConnectorFactory.java | 13 +- .../org/dbsyncer/connector/ValueMapper.java | 2 +- .../connector/config/CommandConfig.java | 8 +- .../connector/config/DatabaseConfig.java | 4 +- .../dbsyncer/connector/config/ESConfig.java | 4 +- .../dbsyncer/connector/config/FileConfig.java | 4 +- .../connector/config/KafkaConfig.java | 4 +- .../database/AbstractDatabaseConnector.java | 2 +- .../database/DatabaseConnectorMapper.java | 2 +- .../connector/enums/ConnectorEnum.java | 8 +- .../dbsyncer/connector/es/ESConnector.java | 2 +- .../connector/es/ESConnectorMapper.java | 2 +- .../connector/es/ESDateValueMapper.java | 2 +- .../connector/es/ESOtherValueMapper.java | 2 +- .../connector/file/FileConnector.java | 2 +- .../connector/file/FileConnectorMapper.java | 2 +- .../connector/kafka/KafkaConnector.java | 2 +- .../connector/kafka/KafkaConnectorMapper.java | 2 +- .../oracle/OracleOtherValueMapper.java | 2 +- .../postgresql/PostgreSQLBitValueMapper.java | 2 +- .../PostgreSQLOtherValueMapper.java | 2 +- .../connector/schema/BigintValueMapper.java | 2 +- .../connector/schema/BinaryValueMapper.java | 2 +- .../connector/schema/BitValueMapper.java | 2 +- .../connector/schema/BlobValueMapper.java | 2 +- .../connector/schema/CharValueMapper.java | 2 +- .../connector/schema/ClobValueMapper.java | 2 +- .../connector/schema/DateValueMapper.java | 2 +- .../connector/schema/DecimalValueMapper.java | 2 +- .../connector/schema/DoubleValueMapper.java | 2 +- .../connector/schema/FloatValueMapper.java | 2 +- .../connector/schema/IntegerValueMapper.java | 2 +- .../schema/LongVarBinaryValueMapper.java | 2 +- .../schema/LongVarcharValueMapper.java | 2 +- .../connector/schema/NCharValueMapper.java | 2 +- .../connector/schema/NClobValueMapper.java | 2 +- .../connector/schema/NVarcharValueMapper.java | 2 +- .../connector/schema/NumberValueMapper.java | 2 +- .../connector/schema/OtherValueMapper.java | 2 +- .../connector/schema/RealValueMapper.java | 2 +- .../connector/schema/RowIdValueMapper.java | 2 +- .../connector/schema/SmallintValueMapper.java | 2 +- .../connector/schema/TimeValueMapper.java | 2 +- .../schema/TimestampValueMapper.java | 2 +- .../connector/schema/TinyintValueMapper.java | 2 +- .../schema/VarBinaryValueMapper.java | 2 +- .../connector/schema/VarcharValueMapper.java | 2 +- dbsyncer-listener/pom.xml | 2 +- .../dbsyncer/listener/AbstractExtractor.java | 6 +- .../quartz/AbstractQuartzExtractor.java | 2 +- dbsyncer-manager/pom.xml | 9 +- .../manager/puller/IncrementPuller.java | 4 +- dbsyncer-parser/pom.xml | 2 +- .../org/dbsyncer/parser/ParserComponent.java | 4 +- .../parser/event/FullRefreshEvent.java | 2 +- .../flush/impl/GeneralBufferActuator.java | 16 +- .../parser/impl/MessageServiceImpl.java | 4 +- .../parser/impl/ParserComponentImpl.java | 18 +-- .../parser/impl/ProfileComponentImpl.java | 4 +- .../dbsyncer/parser/model/BatchWriter.java | 2 +- .../org/dbsyncer/parser/model/Connector.java | 8 +- dbsyncer-plugin/pom.xml | 9 +- .../plugin/NotifySupportConfiguration.java | 2 +- .../org/dbsyncer/plugin/PluginFactory.java | 8 +- .../plugin/notify/DefaultNotifyService.java | 4 +- .../plugin/notify/MailNotifyService.java | 14 +- .../proxy/ProxyApplicationContextImpl.java | 13 +- .../service/DemoConvertServiceImpl.java | 9 +- dbsyncer-sdk/pom.xml | 21 +++ .../java/org/dbsyncer/sdk/RemoteClient.java | 12 ++ .../sdk}/model/AbstractConvertContext.java | 8 +- .../dbsyncer/sdk/model/ConnectorConfig.java | 6 +- .../sdk}/model/FullConvertContext.java | 4 +- .../sdk}/model/IncrementConvertContext.java | 4 +- .../dbsyncer/sdk}/model/NotifyMessage.java | 2 +- .../dbsyncer/sdk}/spi/ConnectorMapper.java | 24 +-- .../org/dbsyncer/sdk}/spi/ConvertContext.java | 138 +++++++++--------- .../org/dbsyncer/sdk}/spi/ConvertService.java | 114 +++++++-------- .../org/dbsyncer/sdk}/spi/NotifyService.java | 4 +- .../sdk}/spi/ProxyApplicationContext.java | 2 +- dbsyncer-storage/pom.xml | 9 +- .../storage/binlog/AbstractBinlogService.java | 10 +- dbsyncer-web/pom.xml | 2 +- .../main/resources/public/plugin/plugin.html | 4 +- pom.xml | 5 +- version.cmd | 2 +- 95 files changed, 358 insertions(+), 328 deletions(-) create mode 100644 dbsyncer-sdk/pom.xml create mode 100644 dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/RemoteClient.java rename {dbsyncer-common/src/main/java/org/dbsyncer/common => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/model/AbstractConvertContext.java (93%) rename dbsyncer-common/src/main/java/org/dbsyncer/common/model/AbstractConnectorConfig.java => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/ConnectorConfig.java (66%) rename {dbsyncer-common/src/main/java/org/dbsyncer/common => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/model/FullConvertContext.java (83%) rename {dbsyncer-common/src/main/java/org/dbsyncer/common => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/model/IncrementConvertContext.java (86%) rename {dbsyncer-common/src/main/java/org/dbsyncer/common => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/model/NotifyMessage.java (96%) rename {dbsyncer-common/src/main/java/org/dbsyncer/common => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/spi/ConnectorMapper.java (62%) rename {dbsyncer-common/src/main/java/org/dbsyncer/common => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/spi/ConvertContext.java (91%) rename {dbsyncer-common/src/main/java/org/dbsyncer/common => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/spi/ConvertService.java (91%) rename {dbsyncer-common/src/main/java/org/dbsyncer/common => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/spi/NotifyService.java (79%) rename {dbsyncer-common/src/main/java/org/dbsyncer/common => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/spi/ProxyApplicationContext.java (87%) diff --git a/dbsyncer-biz/pom.xml b/dbsyncer-biz/pom.xml index f30e14aa..dace0f78 100644 --- a/dbsyncer-biz/pom.xml +++ b/dbsyncer-biz/pom.xml @@ -5,7 +5,7 @@ dbsyncer org.ghi - 1.2.7_1103 + 2.0.0 4.0.0 dbsyncer-biz diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/ConnectorChecker.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/ConnectorChecker.java index 239608d7..0373902f 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/ConnectorChecker.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/ConnectorChecker.java @@ -3,8 +3,6 @@ package org.dbsyncer.biz.checker.impl.connector; import org.dbsyncer.biz.BizException; import org.dbsyncer.biz.checker.AbstractChecker; import org.dbsyncer.biz.checker.ConnectorConfigChecker; -import org.dbsyncer.common.model.AbstractConnectorConfig; -import org.dbsyncer.common.spi.ConnectorMapper; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.connector.ConnectorFactory; import org.dbsyncer.connector.enums.ConnectorEnum; @@ -15,6 +13,8 @@ import org.dbsyncer.parser.LogService; import org.dbsyncer.parser.LogType; import org.dbsyncer.parser.model.ConfigModel; import org.dbsyncer.parser.model.Connector; +import org.dbsyncer.sdk.model.ConnectorConfig; +import org.dbsyncer.sdk.spi.ConnectorMapper; import org.dbsyncer.storage.constant.ConfigConstant; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -60,7 +60,7 @@ public class ConnectorChecker extends AbstractChecker { Connector connector = new Connector(); connector.setName(name); - AbstractConnectorConfig config = getConfig(connectorType); + ConnectorConfig config = getConfig(connectorType); connector.setConfig(config); // 配置连接器配置 @@ -90,7 +90,7 @@ public class ConnectorChecker extends AbstractChecker { this.modifyConfigModel(connector, params); // 配置连接器配置 - AbstractConnectorConfig config = connector.getConfig(); + ConnectorConfig config = connector.getConfig(); String type = StringUtil.toLowerCaseFirstOne(config.getConnectorType()).concat("ConfigChecker"); ConnectorConfigChecker checker = map.get(type); Assert.notNull(checker, "Checker can not be null."); @@ -102,9 +102,9 @@ public class ConnectorChecker extends AbstractChecker { return connector; } - private AbstractConnectorConfig getConfig(String connectorType) { + private ConnectorConfig getConfig(String connectorType) { try { - AbstractConnectorConfig config = ConnectorEnum.getConnectorEnum(connectorType).getConfigClass().newInstance(); + ConnectorConfig config = ConnectorEnum.getConnectorEnum(connectorType).getConfigClass().newInstance(); config.setConnectorType(connectorType); return config; } catch (Exception e) { diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ConnectorServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ConnectorServiceImpl.java index f96fcb8d..b2a43182 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ConnectorServiceImpl.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ConnectorServiceImpl.java @@ -3,7 +3,6 @@ package org.dbsyncer.biz.impl; import org.dbsyncer.biz.BizException; import org.dbsyncer.biz.ConnectorService; import org.dbsyncer.biz.checker.Checker; -import org.dbsyncer.common.model.AbstractConnectorConfig; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.JsonUtil; import org.dbsyncer.common.util.StringUtil; @@ -14,6 +13,7 @@ import org.dbsyncer.parser.LogType; import org.dbsyncer.parser.model.ConfigModel; import org.dbsyncer.parser.model.Connector; import org.dbsyncer.parser.model.Mapping; +import org.dbsyncer.sdk.model.ConnectorConfig; import org.dbsyncer.storage.constant.ConfigConstant; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -160,7 +160,7 @@ public class ConnectorServiceImpl extends BaseServiceImpl implements ConnectorSe return health.containsKey(id) && health.get(id); } - private boolean isAliveConnectorConfig(AbstractConnectorConfig config) { + private boolean isAliveConnectorConfig(ConnectorConfig config) { boolean alive = false; try { alive = connectorFactory.isAlive(config); diff --git a/dbsyncer-cluster/pom.xml b/dbsyncer-cluster/pom.xml index 0787ec7e..e08b4203 100644 --- a/dbsyncer-cluster/pom.xml +++ b/dbsyncer-cluster/pom.xml @@ -5,7 +5,7 @@ dbsyncer org.ghi - 1.2.7_1103 + 2.0.0 4.0.0 dbsyncer-cluster diff --git a/dbsyncer-common/pom.xml b/dbsyncer-common/pom.xml index b68760b7..4f143607 100644 --- a/dbsyncer-common/pom.xml +++ b/dbsyncer-common/pom.xml @@ -5,13 +5,12 @@ dbsyncer org.ghi - 1.2.7_1103 + 2.0.0 4.0.0 dbsyncer-common - org.springframework.boot diff --git a/dbsyncer-connector/pom.xml b/dbsyncer-connector/pom.xml index 255c27b8..02043833 100644 --- a/dbsyncer-connector/pom.xml +++ b/dbsyncer-connector/pom.xml @@ -5,12 +5,19 @@ dbsyncer org.ghi - 1.2.7_1103 + 2.0.0 4.0.0 dbsyncer-connector + + + org.ghi + dbsyncer-sdk + ${project.parent.version} + + org.ghi diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/AbstractConnector.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/AbstractConnector.java index 0b43981c..7780da38 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/AbstractConnector.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/AbstractConnector.java @@ -1,6 +1,5 @@ package org.dbsyncer.connector; -import org.dbsyncer.common.spi.ConnectorMapper; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.connector.config.WriterBatchConfig; @@ -32,6 +31,7 @@ import org.dbsyncer.connector.schema.TimestampValueMapper; import org.dbsyncer.connector.schema.TinyintValueMapper; import org.dbsyncer.connector.schema.VarBinaryValueMapper; import org.dbsyncer.connector.schema.VarcharValueMapper; +import org.dbsyncer.sdk.spi.ConnectorMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/AbstractValueMapper.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/AbstractValueMapper.java index 3c2d6801..5789fc0d 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/AbstractValueMapper.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/AbstractValueMapper.java @@ -1,6 +1,6 @@ package org.dbsyncer.connector; -import org.dbsyncer.common.spi.ConnectorMapper; +import org.dbsyncer.sdk.spi.ConnectorMapper; import java.lang.reflect.ParameterizedType; @@ -30,7 +30,7 @@ public abstract class AbstractValueMapper implements ValueMapper { * @param val * @return */ - protected boolean skipConvert(Object val){ + protected boolean skipConvert(Object val) { return false; } @@ -46,14 +46,14 @@ public abstract class AbstractValueMapper implements ValueMapper { @Override public Object convertValue(ConnectorMapper connectorMapper, Object val) throws Exception { - if(null != val){ + if (null != val) { // 是否需要跳过转换 - if(skipConvert(val)){ + if (skipConvert(val)) { return val; } // 当数据类型不同时，返回转换值 - if(!val.getClass().equals(parameterClazz)){ + if (!val.getClass().equals(parameterClazz)) { return convert(connectorMapper, val); } } diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/Connector.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/Connector.java index 012effea..f200f37b 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/Connector.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/Connector.java @@ -1,13 +1,13 @@ package org.dbsyncer.connector; import org.dbsyncer.common.model.Result; -import org.dbsyncer.common.spi.ConnectorMapper; import org.dbsyncer.connector.config.CommandConfig; import org.dbsyncer.connector.config.DDLConfig; import org.dbsyncer.connector.config.ReaderConfig; import org.dbsyncer.connector.config.WriterBatchConfig; import org.dbsyncer.connector.model.MetaInfo; import org.dbsyncer.connector.model.Table; +import org.dbsyncer.sdk.spi.ConnectorMapper; import java.util.List; import java.util.Map; @@ -103,7 +103,7 @@ public interface Connector { * @param ddlConfig * @return */ - default Result writerDDL(M connectorMapper, DDLConfig ddlConfig){ + default Result writerDDL(M connectorMapper, DDLConfig ddlConfig) { throw new ConnectorException("Unsupported method."); } diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/ConnectorFactory.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/ConnectorFactory.java index bac47c61..bcafc6df 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/ConnectorFactory.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/ConnectorFactory.java @@ -1,8 +1,6 @@ package org.dbsyncer.connector; -import org.dbsyncer.common.model.AbstractConnectorConfig; import org.dbsyncer.common.model.Result; -import org.dbsyncer.common.spi.ConnectorMapper; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.connector.config.CommandConfig; import org.dbsyncer.connector.config.DDLConfig; @@ -11,6 +9,8 @@ import org.dbsyncer.connector.config.WriterBatchConfig; import org.dbsyncer.connector.enums.ConnectorEnum; import org.dbsyncer.connector.model.MetaInfo; import org.dbsyncer.connector.model.Table; +import org.dbsyncer.sdk.model.ConnectorConfig; +import org.dbsyncer.sdk.spi.ConnectorMapper; import org.springframework.beans.factory.DisposableBean; import org.springframework.util.Assert; @@ -41,7 +41,7 @@ public class ConnectorFactory implements DisposableBean { * * @param config */ - public ConnectorMapper connect(AbstractConnectorConfig config) { + public ConnectorMapper connect(ConnectorConfig config) { Assert.notNull(config, "ConnectorConfig can not be null."); Connector connector = getConnector(config.getConnectorType()); String cacheKey = connector.getConnectorMapperCacheKey(config); @@ -68,7 +68,7 @@ public class ConnectorFactory implements DisposableBean { * @param config * @return */ - public boolean refresh(AbstractConnectorConfig config) { + public boolean refresh(ConnectorConfig config) { Assert.notNull(config, "ConnectorConfig can not be null."); Connector connector = getConnector(config.getConnectorType()); String cacheKey = connector.getConnectorMapperCacheKey(config); @@ -86,7 +86,7 @@ public class ConnectorFactory implements DisposableBean { * @param config * @return */ - public boolean isAlive(AbstractConnectorConfig config) { + public boolean isAlive(ConnectorConfig config) { Assert.notNull(config, "ConnectorConfig can not be null."); Connector connector = getConnector(config.getConnectorType()); String cacheKey = connector.getConnectorMapperCacheKey(config); @@ -180,7 +180,8 @@ public class ConnectorFactory implements DisposableBean { } public Connector getConnector(ConnectorMapper connectorMapper) { - return getConnector(connectorMapper.getConnectorType()); + ConnectorConfig connectorConfig = (ConnectorConfig) connectorMapper.getConfig(); + return getConnector(connectorConfig.getConnectorType()); } /** diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/ValueMapper.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/ValueMapper.java index bd7bf5d3..adfdf0d3 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/ValueMapper.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/ValueMapper.java @@ -1,6 +1,6 @@ package org.dbsyncer.connector; -import org.dbsyncer.common.spi.ConnectorMapper; +import org.dbsyncer.sdk.spi.ConnectorMapper; /** * 值转换器 diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/CommandConfig.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/CommandConfig.java index c484ab66..8a65eab2 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/CommandConfig.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/CommandConfig.java @@ -1,8 +1,8 @@ package org.dbsyncer.connector.config; -import org.dbsyncer.common.model.AbstractConnectorConfig; import org.dbsyncer.connector.model.Filter; import org.dbsyncer.connector.model.Table; +import org.dbsyncer.sdk.model.ConnectorConfig; import java.util.List; @@ -21,9 +21,9 @@ public class CommandConfig { private List filter; - private AbstractConnectorConfig connectorConfig; + private ConnectorConfig connectorConfig; - public CommandConfig(String type, Table table, AbstractConnectorConfig connectorConfig, List filter) { + public CommandConfig(String type, Table table, ConnectorConfig connectorConfig, List filter) { this.type = type; this.table = table; this.filter = filter; @@ -42,7 +42,7 @@ public class CommandConfig { return filter; } - public AbstractConnectorConfig getConnectorConfig() { + public ConnectorConfig getConnectorConfig() { return connectorConfig; } } \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/DatabaseConfig.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/DatabaseConfig.java index d89356bc..18e7e9f4 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/DatabaseConfig.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/DatabaseConfig.java @@ -1,7 +1,7 @@ package org.dbsyncer.connector.config; -import org.dbsyncer.common.model.AbstractConnectorConfig; import org.dbsyncer.connector.model.SqlTable; +import org.dbsyncer.sdk.model.ConnectorConfig; import java.util.LinkedHashMap; import java.util.List; @@ -13,7 +13,7 @@ import java.util.Map; * @Description: 数据库连接配置 * @date: 2017年7月20日 下午3:40:59 */ -public class DatabaseConfig extends AbstractConnectorConfig { +public class DatabaseConfig extends ConnectorConfig { /** * 驱动 diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/ESConfig.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/ESConfig.java index 3c3bb074..c592812e 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/ESConfig.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/ESConfig.java @@ -1,6 +1,6 @@ package org.dbsyncer.connector.config; -import org.dbsyncer.common.model.AbstractConnectorConfig; +import org.dbsyncer.sdk.model.ConnectorConfig; /** * @author AE86 @@ -8,7 +8,7 @@ import org.dbsyncer.common.model.AbstractConnectorConfig; * @Description: ES连接配置 * @date: 2021年8月23日 下午8:00:00 */ -public class ESConfig extends AbstractConnectorConfig { +public class ESConfig extends ConnectorConfig { /** * 集群地址, http(s)-9200, tcp-9300 http://192.168.1.100:9200,http://192.168.1.200:9200 diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/FileConfig.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/FileConfig.java index b7520e1e..ac4b2c7a 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/FileConfig.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/FileConfig.java @@ -1,13 +1,13 @@ package org.dbsyncer.connector.config; -import org.dbsyncer.common.model.AbstractConnectorConfig; +import org.dbsyncer.sdk.model.ConnectorConfig; /** * @author AE86 * @version 1.0.0 * @date 2022/5/5 23:19 */ -public class FileConfig extends AbstractConnectorConfig { +public class FileConfig extends ConnectorConfig { /** * 文件目录 diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/KafkaConfig.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/KafkaConfig.java index 8b2897ab..53e1d63a 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/KafkaConfig.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/KafkaConfig.java @@ -1,6 +1,6 @@ package org.dbsyncer.connector.config; -import org.dbsyncer.common.model.AbstractConnectorConfig; +import org.dbsyncer.sdk.model.ConnectorConfig; /** * @author AE86 @@ -8,7 +8,7 @@ import org.dbsyncer.common.model.AbstractConnectorConfig; * @Description: Kafka连接配置 * @date: 2021年11月4日 下午8:00:00 */ -public class KafkaConfig extends AbstractConnectorConfig { +public class KafkaConfig extends ConnectorConfig { private String bootstrapServers; private String topic; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/AbstractDatabaseConnector.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/AbstractDatabaseConnector.java index e9bb7905..fd24e8ca 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/AbstractDatabaseConnector.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/AbstractDatabaseConnector.java @@ -1,7 +1,6 @@ package org.dbsyncer.connector.database; import org.dbsyncer.common.model.Result; -import org.dbsyncer.common.spi.ConnectorMapper; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.connector.AbstractConnector; @@ -25,6 +24,7 @@ import org.dbsyncer.connector.model.PageSql; import org.dbsyncer.connector.model.Table; import org.dbsyncer.connector.util.DatabaseUtil; import org.dbsyncer.connector.util.PrimaryKeyUtil; +import org.dbsyncer.sdk.spi.ConnectorMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.jdbc.support.rowset.ResultSetWrappingSqlRowSet; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/DatabaseConnectorMapper.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/DatabaseConnectorMapper.java index a15c1a45..7a63d70a 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/DatabaseConnectorMapper.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/DatabaseConnectorMapper.java @@ -1,10 +1,10 @@ package org.dbsyncer.connector.database; -import org.dbsyncer.common.spi.ConnectorMapper; import org.dbsyncer.connector.ConnectorException; import org.dbsyncer.connector.config.DatabaseConfig; import org.dbsyncer.connector.database.ds.SimpleConnection; import org.dbsyncer.connector.database.ds.SimpleDataSource; +import org.dbsyncer.sdk.spi.ConnectorMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.dao.EmptyResultDataAccessException; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/enums/ConnectorEnum.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/enums/ConnectorEnum.java index e4b12f68..b095b871 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/enums/ConnectorEnum.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/enums/ConnectorEnum.java @@ -1,6 +1,5 @@ package org.dbsyncer.connector.enums; -import org.dbsyncer.common.model.AbstractConnectorConfig; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.connector.Connector; import org.dbsyncer.connector.ConnectorException; @@ -19,6 +18,7 @@ import org.dbsyncer.connector.sql.DQLOracleConnector; import org.dbsyncer.connector.sql.DQLPostgreSQLConnector; import org.dbsyncer.connector.sql.DQLSqlServerConnector; import org.dbsyncer.connector.sqlserver.SqlServerConnector; +import org.dbsyncer.sdk.model.ConnectorConfig; /** * 支持的连接器类型 @@ -81,9 +81,9 @@ public enum ConnectorEnum { private Connector connector; // 配置 - private Class configClass; + private Class configClass; - ConnectorEnum(String type, Connector connector, Class configClass) { + ConnectorEnum(String type, Connector connector, Class configClass) { this.type = type; this.connector = connector; this.configClass = configClass; @@ -113,7 +113,7 @@ public enum ConnectorEnum { return connector; } - public Class getConfigClass() { + public Class getConfigClass() { return configClass; } } \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESConnector.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESConnector.java index 75715387..3543fd77 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESConnector.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESConnector.java @@ -1,7 +1,6 @@ package org.dbsyncer.connector.es; import org.dbsyncer.common.model.Result; -import org.dbsyncer.common.spi.ConnectorMapper; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.JsonUtil; import org.dbsyncer.common.util.StringUtil; @@ -22,6 +21,7 @@ import org.dbsyncer.connector.model.MetaInfo; import org.dbsyncer.connector.model.Table; import org.dbsyncer.connector.util.ESUtil; import org.dbsyncer.connector.util.PrimaryKeyUtil; +import org.dbsyncer.sdk.spi.ConnectorMapper; import org.elasticsearch.Version; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESConnectorMapper.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESConnectorMapper.java index 3e7d9ab6..484cc360 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESConnectorMapper.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESConnectorMapper.java @@ -1,9 +1,9 @@ package org.dbsyncer.connector.es; -import org.dbsyncer.common.spi.ConnectorMapper; import org.dbsyncer.connector.ConnectorException; import org.dbsyncer.connector.config.ESConfig; import org.dbsyncer.connector.util.ESUtil; +import org.dbsyncer.sdk.spi.ConnectorMapper; import org.elasticsearch.Version; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.core.MainResponse; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESDateValueMapper.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESDateValueMapper.java index 49f2031d..04010769 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESDateValueMapper.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESDateValueMapper.java @@ -1,8 +1,8 @@ package org.dbsyncer.connector.es; -import org.dbsyncer.common.spi.ConnectorMapper; import org.dbsyncer.connector.AbstractValueMapper; import org.dbsyncer.connector.ConnectorException; +import org.dbsyncer.sdk.spi.ConnectorMapper; import java.sql.Date; import java.sql.Timestamp; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESOtherValueMapper.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESOtherValueMapper.java index f6094e66..96d50e45 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESOtherValueMapper.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESOtherValueMapper.java @@ -1,9 +1,9 @@ package org.dbsyncer.connector.es; -import org.dbsyncer.common.spi.ConnectorMapper; import org.dbsyncer.common.util.JsonUtil; import org.dbsyncer.connector.AbstractValueMapper; import org.dbsyncer.connector.ConnectorException; +import org.dbsyncer.sdk.spi.ConnectorMapper; import org.postgresql.util.PGobject; import java.util.Map; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileConnector.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileConnector.java index 82c6c148..a5d178ab 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileConnector.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileConnector.java @@ -3,7 +3,6 @@ package org.dbsyncer.connector.file; import org.apache.commons.io.IOUtils; import org.apache.commons.io.LineIterator; import org.dbsyncer.common.model.Result; -import org.dbsyncer.common.spi.ConnectorMapper; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.connector.AbstractConnector; @@ -17,6 +16,7 @@ import org.dbsyncer.connector.model.Field; import org.dbsyncer.connector.model.FileSchema; import org.dbsyncer.connector.model.MetaInfo; import org.dbsyncer.connector.model.Table; +import org.dbsyncer.sdk.spi.ConnectorMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.util.Assert; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileConnectorMapper.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileConnectorMapper.java index c08ebfe1..aac59a57 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileConnectorMapper.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileConnectorMapper.java @@ -1,10 +1,10 @@ package org.dbsyncer.connector.file; -import org.dbsyncer.common.spi.ConnectorMapper; import org.dbsyncer.common.util.JsonUtil; import org.dbsyncer.connector.config.FileConfig; import org.dbsyncer.connector.model.Field; import org.dbsyncer.connector.model.FileSchema; +import org.dbsyncer.sdk.spi.ConnectorMapper; import org.springframework.util.Assert; import java.io.File; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/kafka/KafkaConnector.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/kafka/KafkaConnector.java index 11b4c49e..049aa33d 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/kafka/KafkaConnector.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/kafka/KafkaConnector.java @@ -1,7 +1,6 @@ package org.dbsyncer.connector.kafka; import org.dbsyncer.common.model.Result; -import org.dbsyncer.common.spi.ConnectorMapper; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.JsonUtil; import org.dbsyncer.connector.AbstractConnector; @@ -15,6 +14,7 @@ import org.dbsyncer.connector.model.Field; import org.dbsyncer.connector.model.MetaInfo; import org.dbsyncer.connector.model.Table; import org.dbsyncer.connector.util.PrimaryKeyUtil; +import org.dbsyncer.sdk.spi.ConnectorMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/kafka/KafkaConnectorMapper.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/kafka/KafkaConnectorMapper.java index 3b7703ea..fcfb2bfc 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/kafka/KafkaConnectorMapper.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/kafka/KafkaConnectorMapper.java @@ -1,8 +1,8 @@ package org.dbsyncer.connector.kafka; -import org.dbsyncer.common.spi.ConnectorMapper; import org.dbsyncer.connector.config.KafkaConfig; import org.dbsyncer.connector.util.KafkaUtil; +import org.dbsyncer.sdk.spi.ConnectorMapper; public final class KafkaConnectorMapper implements ConnectorMapper { private KafkaConfig config; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/OracleOtherValueMapper.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/OracleOtherValueMapper.java index 1a75ef9e..4b956c15 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/OracleOtherValueMapper.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/OracleOtherValueMapper.java @@ -3,11 +3,11 @@ package org.dbsyncer.connector.oracle; import com.microsoft.sqlserver.jdbc.Geometry; import oracle.jdbc.OracleConnection; import org.dbsyncer.connector.oracle.geometry.JGeometry; -import org.dbsyncer.common.spi.ConnectorMapper; import org.dbsyncer.connector.AbstractValueMapper; import org.dbsyncer.connector.ConnectorException; import org.dbsyncer.connector.database.ds.SimpleConnection; import org.dbsyncer.connector.util.DatabaseUtil; +import org.dbsyncer.sdk.spi.ConnectorMapper; import java.sql.Struct; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLBitValueMapper.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLBitValueMapper.java index 6d96b08e..e278403d 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLBitValueMapper.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLBitValueMapper.java @@ -1,8 +1,8 @@ package org.dbsyncer.connector.postgresql; -import org.dbsyncer.common.spi.ConnectorMapper; import org.dbsyncer.connector.AbstractValueMapper; import org.dbsyncer.connector.ConnectorException; +import org.dbsyncer.sdk.spi.ConnectorMapper; /** * @author AE86 diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLOtherValueMapper.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLOtherValueMapper.java index 7bafc361..6b3ac2d7 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLOtherValueMapper.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLOtherValueMapper.java @@ -1,8 +1,8 @@ package org.dbsyncer.connector.postgresql; -import org.dbsyncer.common.spi.ConnectorMapper; import org.dbsyncer.connector.AbstractValueMapper; import org.dbsyncer.connector.ConnectorException; +import org.dbsyncer.sdk.spi.ConnectorMapper; import org.postgis.Geometry; import org.postgis.PGgeometry; import org.postgis.binary.BinaryParser; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/BigintValueMapper.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/BigintValueMapper.java index c7e328a3..a9027d1b 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/BigintValueMapper.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/BigintValueMapper.java @@ -1,8 +1,8 @@ package org.dbsyncer.connector.schema; -import org.dbsyncer.common.spi.ConnectorMapper; import org.dbsyncer.connector.AbstractValueMapper; import org.dbsyncer.connector.ConnectorException; +import org.dbsyncer.sdk.spi.ConnectorMapper; import java.math.BigDecimal; import java.math.BigInteger; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/BinaryValueMapper.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/BinaryValueMapper.java index 4f2e6f2d..414f1407 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/BinaryValueMapper.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/BinaryValueMapper.java @@ -1,8 +1,8 @@ package org.dbsyncer.connector.schema; -import org.dbsyncer.common.spi.ConnectorMapper; import org.dbsyncer.connector.AbstractValueMapper; import org.dbsyncer.connector.ConnectorException; +import org.dbsyncer.sdk.spi.ConnectorMapper; /** * @author AE86 diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/BitValueMapper.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/BitValueMapper.java index 29ebe4be..bd2e40af 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/BitValueMapper.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/BitValueMapper.java @@ -1,8 +1,8 @@ package org.dbsyncer.connector.schema; -import org.dbsyncer.common.spi.ConnectorMapper; import org.dbsyncer.connector.AbstractValueMapper; import org.dbsyncer.connector.ConnectorException; +import org.dbsyncer.sdk.spi.ConnectorMapper; import java.nio.ByteBuffer; import java.util.BitSet; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/BlobValueMapper.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/BlobValueMapper.java index 6001bb85..6eec09e0 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/BlobValueMapper.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/BlobValueMapper.java @@ -1,8 +1,8 @@ package org.dbsyncer.connector.schema; -import org.dbsyncer.common.spi.ConnectorMapper; import org.dbsyncer.connector.AbstractValueMapper; import org.dbsyncer.connector.ConnectorException; +import org.dbsyncer.sdk.spi.ConnectorMapper; import java.sql.Blob; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/CharValueMapper.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/CharValueMapper.java index f356b34b..52c039f5 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/CharValueMapper.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/CharValueMapper.java @@ -1,8 +1,8 @@ package org.dbsyncer.connector.schema; -import org.dbsyncer.common.spi.ConnectorMapper; import org.dbsyncer.connector.AbstractValueMapper; import org.dbsyncer.connector.ConnectorException; +import org.dbsyncer.sdk.spi.ConnectorMapper; /** * @author AE86 diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/ClobValueMapper.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/ClobValueMapper.java index 51ac1121..b4da6e22 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/ClobValueMapper.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/ClobValueMapper.java @@ -1,8 +1,8 @@ package org.dbsyncer.connector.schema; -import org.dbsyncer.common.spi.ConnectorMapper; import org.dbsyncer.connector.AbstractValueMapper; import org.dbsyncer.connector.ConnectorException; +import org.dbsyncer.sdk.spi.ConnectorMapper; import java.sql.Clob; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/DateValueMapper.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/DateValueMapper.java index 5b6d2a7c..5d6a5ac0 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/DateValueMapper.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/DateValueMapper.java @@ -1,9 +1,9 @@ package org.dbsyncer.connector.schema; -import org.dbsyncer.common.spi.ConnectorMapper; import org.dbsyncer.common.util.DateFormatUtil; import org.dbsyncer.connector.AbstractValueMapper; import org.dbsyncer.connector.ConnectorException; +import org.dbsyncer.sdk.spi.ConnectorMapper; import java.sql.Date; import java.sql.Timestamp; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/DecimalValueMapper.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/DecimalValueMapper.java index a400fd13..6212d55c 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/DecimalValueMapper.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/DecimalValueMapper.java @@ -1,8 +1,8 @@ package org.dbsyncer.connector.schema; -import org.dbsyncer.common.spi.ConnectorMapper; import org.dbsyncer.connector.AbstractValueMapper; import org.dbsyncer.connector.ConnectorException; +import org.dbsyncer.sdk.spi.ConnectorMapper; import java.math.BigDecimal; import java.math.BigInteger; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/DoubleValueMapper.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/DoubleValueMapper.java index c1a4fdbd..632ce6f1 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/DoubleValueMapper.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/DoubleValueMapper.java @@ -1,8 +1,8 @@ package org.dbsyncer.connector.schema; -import org.dbsyncer.common.spi.ConnectorMapper; import org.dbsyncer.connector.AbstractValueMapper; import org.dbsyncer.connector.ConnectorException; +import org.dbsyncer.sdk.spi.ConnectorMapper; import java.math.BigDecimal; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/FloatValueMapper.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/FloatValueMapper.java index 80d949e7..75845e0d 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/FloatValueMapper.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/FloatValueMapper.java @@ -1,8 +1,8 @@ package org.dbsyncer.connector.schema; -import org.dbsyncer.common.spi.ConnectorMapper; import org.dbsyncer.connector.AbstractValueMapper; import org.dbsyncer.connector.ConnectorException; +import org.dbsyncer.sdk.spi.ConnectorMapper; import java.math.BigDecimal; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/IntegerValueMapper.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/IntegerValueMapper.java index 1cc9267b..00cb0fa8 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/IntegerValueMapper.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/IntegerValueMapper.java @@ -1,8 +1,8 @@ package org.dbsyncer.connector.schema; -import org.dbsyncer.common.spi.ConnectorMapper; import org.dbsyncer.connector.AbstractValueMapper; import org.dbsyncer.connector.ConnectorException; +import org.dbsyncer.sdk.spi.ConnectorMapper; import java.math.BigDecimal; import java.math.BigInteger; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/LongVarBinaryValueMapper.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/LongVarBinaryValueMapper.java index 76267fb6..21176ff3 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/LongVarBinaryValueMapper.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/LongVarBinaryValueMapper.java @@ -1,8 +1,8 @@ package org.dbsyncer.connector.schema; -import org.dbsyncer.common.spi.ConnectorMapper; import org.dbsyncer.connector.AbstractValueMapper; import org.dbsyncer.connector.ConnectorException; +import org.dbsyncer.sdk.spi.ConnectorMapper; /** * @author AE86 diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/LongVarcharValueMapper.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/LongVarcharValueMapper.java index 1405ad18..3ff9dad7 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/LongVarcharValueMapper.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/LongVarcharValueMapper.java @@ -1,8 +1,8 @@ package org.dbsyncer.connector.schema; -import org.dbsyncer.common.spi.ConnectorMapper; import org.dbsyncer.connector.AbstractValueMapper; import org.dbsyncer.connector.ConnectorException; +import org.dbsyncer.sdk.spi.ConnectorMapper; import java.sql.Date; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/NCharValueMapper.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/NCharValueMapper.java index cecbf0c2..4c99700a 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/NCharValueMapper.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/NCharValueMapper.java @@ -1,8 +1,8 @@ package org.dbsyncer.connector.schema; -import org.dbsyncer.common.spi.ConnectorMapper; import org.dbsyncer.connector.AbstractValueMapper; import org.dbsyncer.connector.ConnectorException; +import org.dbsyncer.sdk.spi.ConnectorMapper; /** * @author AE86 diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/NClobValueMapper.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/NClobValueMapper.java index 2da1957a..84cb844d 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/NClobValueMapper.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/NClobValueMapper.java @@ -1,8 +1,8 @@ package org.dbsyncer.connector.schema; -import org.dbsyncer.common.spi.ConnectorMapper; import org.dbsyncer.connector.AbstractValueMapper; import org.dbsyncer.connector.ConnectorException; +import org.dbsyncer.sdk.spi.ConnectorMapper; import java.sql.NClob; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/NVarcharValueMapper.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/NVarcharValueMapper.java index ba412073..cdd638d8 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/NVarcharValueMapper.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/NVarcharValueMapper.java @@ -1,8 +1,8 @@ package org.dbsyncer.connector.schema; -import org.dbsyncer.common.spi.ConnectorMapper; import org.dbsyncer.connector.AbstractValueMapper; import org.dbsyncer.connector.ConnectorException; +import org.dbsyncer.sdk.spi.ConnectorMapper; /** * @author AE86 diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/NumberValueMapper.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/NumberValueMapper.java index ed917a96..449f44b6 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/NumberValueMapper.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/NumberValueMapper.java @@ -1,8 +1,8 @@ package org.dbsyncer.connector.schema; -import org.dbsyncer.common.spi.ConnectorMapper; import org.dbsyncer.connector.AbstractValueMapper; import org.dbsyncer.connector.ConnectorException; +import org.dbsyncer.sdk.spi.ConnectorMapper; import java.math.BigDecimal; import java.math.BigInteger; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/OtherValueMapper.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/OtherValueMapper.java index fb2184bc..5b6a9f28 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/OtherValueMapper.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/OtherValueMapper.java @@ -1,8 +1,8 @@ package org.dbsyncer.connector.schema; -import org.dbsyncer.common.spi.ConnectorMapper; import org.dbsyncer.connector.AbstractValueMapper; import org.dbsyncer.connector.ConnectorException; +import org.dbsyncer.sdk.spi.ConnectorMapper; /** * @author AE86 diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/RealValueMapper.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/RealValueMapper.java index f872598d..8e6da43f 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/RealValueMapper.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/RealValueMapper.java @@ -1,8 +1,8 @@ package org.dbsyncer.connector.schema; -import org.dbsyncer.common.spi.ConnectorMapper; import org.dbsyncer.connector.AbstractValueMapper; import org.dbsyncer.connector.ConnectorException; +import org.dbsyncer.sdk.spi.ConnectorMapper; /** * @author AE86 diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/RowIdValueMapper.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/RowIdValueMapper.java index 9ae8266b..7c091b20 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/RowIdValueMapper.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/RowIdValueMapper.java @@ -1,8 +1,8 @@ package org.dbsyncer.connector.schema; -import org.dbsyncer.common.spi.ConnectorMapper; import org.dbsyncer.connector.AbstractValueMapper; import org.dbsyncer.connector.ConnectorException; +import org.dbsyncer.sdk.spi.ConnectorMapper; import java.sql.RowId; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/SmallintValueMapper.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/SmallintValueMapper.java index e7a373db..fdc38c3f 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/SmallintValueMapper.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/SmallintValueMapper.java @@ -1,8 +1,8 @@ package org.dbsyncer.connector.schema; -import org.dbsyncer.common.spi.ConnectorMapper; import org.dbsyncer.connector.AbstractValueMapper; import org.dbsyncer.connector.ConnectorException; +import org.dbsyncer.sdk.spi.ConnectorMapper; /** * @author AE86 diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/TimeValueMapper.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/TimeValueMapper.java index ad008b59..f2bf2a51 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/TimeValueMapper.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/TimeValueMapper.java @@ -1,8 +1,8 @@ package org.dbsyncer.connector.schema; -import org.dbsyncer.common.spi.ConnectorMapper; import org.dbsyncer.connector.AbstractValueMapper; import org.dbsyncer.connector.ConnectorException; +import org.dbsyncer.sdk.spi.ConnectorMapper; import java.sql.Time; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/TimestampValueMapper.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/TimestampValueMapper.java index 930e9d3e..ed2271ed 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/TimestampValueMapper.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/TimestampValueMapper.java @@ -1,9 +1,9 @@ package org.dbsyncer.connector.schema; -import org.dbsyncer.common.spi.ConnectorMapper; import org.dbsyncer.common.util.DateFormatUtil; import org.dbsyncer.connector.AbstractValueMapper; import org.dbsyncer.connector.ConnectorException; +import org.dbsyncer.sdk.spi.ConnectorMapper; import java.sql.Date; import java.sql.SQLException; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/TinyintValueMapper.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/TinyintValueMapper.java index cdad9836..b2ee953f 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/TinyintValueMapper.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/TinyintValueMapper.java @@ -1,8 +1,8 @@ package org.dbsyncer.connector.schema; -import org.dbsyncer.common.spi.ConnectorMapper; import org.dbsyncer.connector.AbstractValueMapper; import org.dbsyncer.connector.ConnectorException; +import org.dbsyncer.sdk.spi.ConnectorMapper; import java.math.BigDecimal; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/VarBinaryValueMapper.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/VarBinaryValueMapper.java index c62b9f42..a5fc15e2 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/VarBinaryValueMapper.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/VarBinaryValueMapper.java @@ -1,8 +1,8 @@ package org.dbsyncer.connector.schema; -import org.dbsyncer.common.spi.ConnectorMapper; import org.dbsyncer.connector.AbstractValueMapper; import org.dbsyncer.connector.ConnectorException; +import org.dbsyncer.sdk.spi.ConnectorMapper; import java.nio.ByteBuffer; import java.util.BitSet; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/VarcharValueMapper.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/VarcharValueMapper.java index 36688411..f4452c81 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/VarcharValueMapper.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/VarcharValueMapper.java @@ -1,9 +1,9 @@ package org.dbsyncer.connector.schema; -import org.dbsyncer.common.spi.ConnectorMapper; import org.dbsyncer.common.util.DateFormatUtil; import org.dbsyncer.connector.AbstractValueMapper; import org.dbsyncer.connector.ConnectorException; +import org.dbsyncer.sdk.spi.ConnectorMapper; import org.postgresql.util.PGobject; import java.sql.Date; diff --git a/dbsyncer-listener/pom.xml b/dbsyncer-listener/pom.xml index 5a11faf3..0405d8b7 100644 --- a/dbsyncer-listener/pom.xml +++ b/dbsyncer-listener/pom.xml @@ -5,7 +5,7 @@ dbsyncer org.ghi - 1.2.7_1103 + 2.0.0 4.0.0 dbsyncer-listener diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/AbstractExtractor.java b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/AbstractExtractor.java index 43084663..53af585d 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/AbstractExtractor.java +++ b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/AbstractExtractor.java @@ -1,13 +1,13 @@ package org.dbsyncer.listener; import org.dbsyncer.listener.model.ChangedOffset; -import org.dbsyncer.common.model.AbstractConnectorConfig; import org.dbsyncer.common.scheduled.ScheduledTaskService; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.connector.ConnectorFactory; import org.dbsyncer.connector.constant.ConnectorConstant; import org.dbsyncer.connector.model.Table; import org.dbsyncer.listener.config.ListenerConfig; +import org.dbsyncer.sdk.model.ConnectorConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -28,7 +28,7 @@ public abstract class AbstractExtractor implements Extractor { private final Logger logger = LoggerFactory.getLogger(getClass()); protected ConnectorFactory connectorFactory; protected ScheduledTaskService scheduledTaskService; - protected AbstractConnectorConfig connectorConfig; + protected ConnectorConfig connectorConfig; protected ListenerConfig listenerConfig; protected Set filterTable; protected List
sourceTable; @@ -125,7 +125,7 @@ public abstract class AbstractExtractor implements Extractor { this.scheduledTaskService = scheduledTaskService; } - public void setConnectorConfig(AbstractConnectorConfig connectorConfig) { + public void setConnectorConfig(ConnectorConfig connectorConfig) { this.connectorConfig = connectorConfig; } diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/AbstractQuartzExtractor.java b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/AbstractQuartzExtractor.java index b52f5242..b137a1a9 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/AbstractQuartzExtractor.java +++ b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/AbstractQuartzExtractor.java @@ -3,7 +3,6 @@ package org.dbsyncer.listener.quartz; import org.dbsyncer.listener.event.ScanChangedEvent; import org.dbsyncer.common.model.Result; import org.dbsyncer.common.scheduled.ScheduledTaskJob; -import org.dbsyncer.common.spi.ConnectorMapper; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.common.util.UUIDUtil; @@ -12,6 +11,7 @@ import org.dbsyncer.connector.constant.ConnectorConstant; import org.dbsyncer.connector.model.Table; import org.dbsyncer.connector.util.PrimaryKeyUtil; import org.dbsyncer.listener.AbstractExtractor; +import org.dbsyncer.sdk.spi.ConnectorMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dbsyncer-manager/pom.xml b/dbsyncer-manager/pom.xml index 213adb3c..108d2a65 100644 --- a/dbsyncer-manager/pom.xml +++ b/dbsyncer-manager/pom.xml @@ -5,12 +5,19 @@ dbsyncer org.ghi - 1.2.7_1103 + 2.0.0 4.0.0dbsyncer-manager + + + org.ghi + dbsyncer-cluster + ${project.parent.version} + + org.ghi diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/puller/IncrementPuller.java b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/puller/IncrementPuller.java index 6cb7e826..b3500a19 100644 --- a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/puller/IncrementPuller.java +++ b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/puller/IncrementPuller.java @@ -2,7 +2,6 @@ package org.dbsyncer.manager.puller; import org.dbsyncer.listener.model.ChangedOffset; import org.dbsyncer.parser.event.RefreshOffsetEvent; -import org.dbsyncer.common.model.AbstractConnectorConfig; import org.dbsyncer.common.scheduled.ScheduledTaskJob; import org.dbsyncer.common.scheduled.ScheduledTaskService; import org.dbsyncer.common.util.CollectionUtils; @@ -26,6 +25,7 @@ import org.dbsyncer.parser.model.Connector; import org.dbsyncer.parser.model.Mapping; import org.dbsyncer.parser.model.Meta; import org.dbsyncer.parser.model.TableGroup; +import org.dbsyncer.sdk.model.ConnectorConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.ApplicationListener; @@ -142,7 +142,7 @@ public final class IncrementPuller extends AbstractPuller implements Application } private AbstractExtractor getExtractor(Mapping mapping, Connector connector, List list, Meta meta) throws InstantiationException, IllegalAccessException { - AbstractConnectorConfig connectorConfig = connector.getConfig(); + ConnectorConfig connectorConfig = connector.getConfig(); ListenerConfig listenerConfig = mapping.getListener(); // timing/log final String listenerType = listenerConfig.getListenerType(); diff --git a/dbsyncer-parser/pom.xml b/dbsyncer-parser/pom.xml index f262f71c..b578b146 100644 --- a/dbsyncer-parser/pom.xml +++ b/dbsyncer-parser/pom.xml @@ -5,7 +5,7 @@ dbsyncer org.ghi - 1.2.7_1103 + 2.0.0 4.0.0 dbsyncer-parser diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ParserComponent.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ParserComponent.java index 62361540..b955da97 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ParserComponent.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ParserComponent.java @@ -2,14 +2,14 @@ package org.dbsyncer.parser; import org.dbsyncer.listener.ChangedEvent; import org.dbsyncer.common.model.Result; -import org.dbsyncer.common.spi.ConnectorMapper; -import org.dbsyncer.common.spi.ConvertContext; import org.dbsyncer.connector.model.MetaInfo; import org.dbsyncer.connector.model.Table; import org.dbsyncer.parser.model.BatchWriter; import org.dbsyncer.parser.model.Mapping; import org.dbsyncer.parser.model.TableGroup; import org.dbsyncer.parser.model.Task; +import org.dbsyncer.sdk.spi.ConnectorMapper; +import org.dbsyncer.sdk.spi.ConvertContext; import java.util.List; import java.util.Map; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/event/FullRefreshEvent.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/event/FullRefreshEvent.java index baec0dbd..fe3c99d0 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/event/FullRefreshEvent.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/event/FullRefreshEvent.java @@ -4,7 +4,7 @@ import org.dbsyncer.parser.model.Task; import org.springframework.context.ApplicationContext; import org.springframework.context.event.ApplicationContextEvent; -public class FullRefreshEvent extends ApplicationContextEvent { +public final class FullRefreshEvent extends ApplicationContextEvent { private Task task; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/GeneralBufferActuator.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/GeneralBufferActuator.java index 8e44fcb1..78382ccc 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/GeneralBufferActuator.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/GeneralBufferActuator.java @@ -1,12 +1,7 @@ package org.dbsyncer.parser.flush.impl; -import org.dbsyncer.parser.CacheService; import org.dbsyncer.common.config.GeneralBufferConfig; -import org.dbsyncer.parser.event.RefreshOffsetEvent; -import org.dbsyncer.common.model.AbstractConnectorConfig; -import org.dbsyncer.common.model.IncrementConvertContext; import org.dbsyncer.common.model.Result; -import org.dbsyncer.common.spi.ConnectorMapper; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.connector.ConnectorFactory; @@ -14,8 +9,10 @@ import org.dbsyncer.connector.config.DDLConfig; import org.dbsyncer.connector.constant.ConnectorConstant; import org.dbsyncer.connector.enums.ConnectorEnum; import org.dbsyncer.connector.model.MetaInfo; +import org.dbsyncer.parser.CacheService; import org.dbsyncer.parser.ParserComponent; import org.dbsyncer.parser.ddl.DDLParser; +import org.dbsyncer.parser.event.RefreshOffsetEvent; import org.dbsyncer.parser.flush.AbstractBufferActuator; import org.dbsyncer.parser.model.BatchWriter; import org.dbsyncer.parser.model.ConfigModel; @@ -31,6 +28,9 @@ import org.dbsyncer.parser.util.ConfigModelUtil; import org.dbsyncer.parser.util.ConvertUtil; import org.dbsyncer.parser.util.PickerUtil; import org.dbsyncer.plugin.PluginFactory; +import org.dbsyncer.sdk.model.ConnectorConfig; +import org.dbsyncer.sdk.model.IncrementConvertContext; +import org.dbsyncer.sdk.spi.ConnectorMapper; import org.dbsyncer.storage.StorageService; import org.dbsyncer.storage.enums.StorageEnum; import org.slf4j.Logger; @@ -186,8 +186,8 @@ public class GeneralBufferActuator extends AbstractBufferActuator getCommand(Mapping mapping, TableGroup tableGroup) { - AbstractConnectorConfig sConnConfig = getConnectorConfig(mapping.getSourceConnectorId()); - AbstractConnectorConfig tConnConfig = getConnectorConfig(mapping.getTargetConnectorId()); + ConnectorConfig sConnConfig = getConnectorConfig(mapping.getSourceConnectorId()); + ConnectorConfig tConnConfig = getConnectorConfig(mapping.getTargetConnectorId()); Table sourceTable = tableGroup.getSourceTable(); Table targetTable = tableGroup.getTargetTable(); Table sTable = new Table(sourceTable.getName(), sourceTable.getType(), new ArrayList<>(), sourceTable.getSql()); @@ -135,9 +135,9 @@ public class ParserComponentImpl implements ParserComponent { final String sourceConnectorId = mapping.getSourceConnectorId(); final String targetConnectorId = mapping.getTargetConnectorId(); - AbstractConnectorConfig sConfig = getConnectorConfig(sourceConnectorId); + ConnectorConfig sConfig = getConnectorConfig(sourceConnectorId); Assert.notNull(sConfig, "数据源配置不能为空."); - AbstractConnectorConfig tConfig = getConnectorConfig(targetConnectorId); + ConnectorConfig tConfig = getConnectorConfig(targetConnectorId); Assert.notNull(tConfig, "目标源配置不能为空."); TableGroup group = PickerUtil.mergeTableGroupConfig(mapping, tableGroup); Map command = group.getCommand(); @@ -291,7 +291,7 @@ public class ParserComponentImpl implements ParserComponent { * @param connectorId * @return */ - private AbstractConnectorConfig getConnectorConfig(String connectorId) { + private ConnectorConfig getConnectorConfig(String connectorId) { return profileComponent.getConnector(connectorId).getConfig(); } diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/ProfileComponentImpl.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/ProfileComponentImpl.java index 383fb5e4..13fb2a92 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/ProfileComponentImpl.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/ProfileComponentImpl.java @@ -3,7 +3,6 @@ */ package org.dbsyncer.parser.impl; -import org.dbsyncer.common.model.AbstractConnectorConfig; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.JsonUtil; import org.dbsyncer.connector.enums.ConnectorEnum; @@ -25,6 +24,7 @@ import org.dbsyncer.parser.model.SystemConfig; import org.dbsyncer.parser.model.TableGroup; import org.dbsyncer.parser.model.UserConfig; import org.dbsyncer.parser.template.OperationTemplate; +import org.dbsyncer.sdk.model.ConnectorConfig; import org.dbsyncer.storage.enums.StorageDataStatusEnum; import org.springframework.stereotype.Component; import org.springframework.util.Assert; @@ -55,7 +55,7 @@ public class ProfileComponentImpl implements ProfileComponent { Assert.notNull(connector, "Connector can not be null."); String connectorType = (String) config.get("connectorType"); ConnectorEnum connectorEnum = ConnectorEnum.getConnectorEnum(connectorType); - AbstractConnectorConfig obj = JsonUtil.jsonToObj(config.toString(), connectorEnum.getConfigClass()); + ConnectorConfig obj = JsonUtil.jsonToObj(config.toString(), connectorEnum.getConfigClass()); obj.setConnectorType(connectorEnum.getType()); connector.setConfig(obj); diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/BatchWriter.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/BatchWriter.java index 694b354f..6ea8bb0e 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/BatchWriter.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/BatchWriter.java @@ -1,7 +1,7 @@ package org.dbsyncer.parser.model; -import org.dbsyncer.common.spi.ConnectorMapper; import org.dbsyncer.connector.model.Field; +import org.dbsyncer.sdk.spi.ConnectorMapper; import java.util.List; import java.util.Map; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/Connector.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/Connector.java index b83a95ce..26d92fab 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/Connector.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/Connector.java @@ -1,7 +1,7 @@ package org.dbsyncer.parser.model; -import org.dbsyncer.common.model.AbstractConnectorConfig; import org.dbsyncer.connector.model.Table; +import org.dbsyncer.sdk.model.ConnectorConfig; import org.dbsyncer.storage.constant.ConfigConstant; import java.util.List; @@ -25,7 +25,7 @@ public class Connector extends ConfigModel{ /** * 连接器配置 */ - private AbstractConnectorConfig config; + private ConnectorConfig config; public List
getTable() { return table; @@ -36,11 +36,11 @@ public class Connector extends ConfigModel{ return this; } - public AbstractConnectorConfig getConfig() { + public ConnectorConfig getConfig() { return config; } - public Connector setConfig(AbstractConnectorConfig config) { + public Connector setConfig(ConnectorConfig config) { this.config = config; return this; } diff --git a/dbsyncer-plugin/pom.xml b/dbsyncer-plugin/pom.xml index 53f71ad8..2a73c372 100644 --- a/dbsyncer-plugin/pom.xml +++ b/dbsyncer-plugin/pom.xml @@ -5,12 +5,19 @@ dbsyncer org.ghi - 1.2.7_1103 + 2.0.0 4.0.0dbsyncer-plugin + + + org.ghi + dbsyncer-sdk + ${project.parent.version} + + org.ghi diff --git a/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/NotifySupportConfiguration.java b/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/NotifySupportConfiguration.java index e6d66c12..cfbc4f12 100644 --- a/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/NotifySupportConfiguration.java +++ b/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/NotifySupportConfiguration.java @@ -1,7 +1,7 @@ package org.dbsyncer.plugin; -import org.dbsyncer.common.spi.NotifyService; import org.dbsyncer.plugin.notify.DefaultNotifyService; +import org.dbsyncer.sdk.spi.NotifyService; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; diff --git a/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/PluginFactory.java b/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/PluginFactory.java index 51b6f0c4..3ffe84dc 100644 --- a/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/PluginFactory.java +++ b/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/PluginFactory.java @@ -1,12 +1,12 @@ package org.dbsyncer.plugin; import org.apache.commons.io.FileUtils; -import org.dbsyncer.common.model.AbstractConvertContext; -import org.dbsyncer.common.spi.ConvertContext; -import org.dbsyncer.common.spi.ConvertService; -import org.dbsyncer.common.spi.ProxyApplicationContext; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.plugin.config.Plugin; +import org.dbsyncer.sdk.model.AbstractConvertContext; +import org.dbsyncer.sdk.spi.ConvertContext; +import org.dbsyncer.sdk.spi.ConvertService; +import org.dbsyncer.sdk.spi.ProxyApplicationContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.DisposableBean; diff --git a/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/notify/DefaultNotifyService.java b/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/notify/DefaultNotifyService.java index 9be51ec5..47b5cbe1 100644 --- a/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/notify/DefaultNotifyService.java +++ b/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/notify/DefaultNotifyService.java @@ -1,7 +1,7 @@ package org.dbsyncer.plugin.notify; -import org.dbsyncer.common.model.NotifyMessage; -import org.dbsyncer.common.spi.NotifyService; +import org.dbsyncer.sdk.model.NotifyMessage; +import org.dbsyncer.sdk.spi.NotifyService; /** * 默认通知服务 diff --git a/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/notify/MailNotifyService.java b/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/notify/MailNotifyService.java index b36e77f5..d78c4c84 100644 --- a/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/notify/MailNotifyService.java +++ b/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/notify/MailNotifyService.java @@ -1,19 +1,23 @@ package org.dbsyncer.plugin.notify; import org.dbsyncer.common.config.AppConfig; -import org.dbsyncer.common.model.NotifyMessage; -import org.dbsyncer.common.spi.NotifyService; import org.dbsyncer.common.util.StringUtil; +import org.dbsyncer.sdk.model.NotifyMessage; +import org.dbsyncer.sdk.spi.NotifyService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.stereotype.Component; import org.springframework.util.Assert; import javax.annotation.PostConstruct; -import javax.mail.*; +import javax.annotation.Resource; +import javax.mail.Authenticator; +import javax.mail.Message; +import javax.mail.PasswordAuthentication; +import javax.mail.Session; +import javax.mail.Transport; import javax.mail.internet.InternetAddress; import javax.mail.internet.MimeMessage; import java.util.List; @@ -33,7 +37,7 @@ public class MailNotifyService implements NotifyService { private final Logger logger = LoggerFactory.getLogger(getClass()); - @Autowired + @Resource private AppConfig appConfig; /** diff --git a/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/proxy/ProxyApplicationContextImpl.java b/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/proxy/ProxyApplicationContextImpl.java index bcde9aa7..4de16e86 100644 --- a/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/proxy/ProxyApplicationContextImpl.java +++ b/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/proxy/ProxyApplicationContextImpl.java @@ -1,12 +1,10 @@ package org.dbsyncer.plugin.proxy; -import org.dbsyncer.common.spi.ProxyApplicationContext; +import org.dbsyncer.sdk.spi.ProxyApplicationContext; import org.springframework.beans.BeansException; import org.springframework.beans.factory.BeanFactory; import org.springframework.beans.factory.NoSuchBeanDefinitionException; import org.springframework.beans.factory.ObjectProvider; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.config.AutowireCapableBeanFactory; import org.springframework.context.ApplicationContext; import org.springframework.context.ApplicationEvent; @@ -14,9 +12,9 @@ import org.springframework.context.MessageSourceResolvable; import org.springframework.context.NoSuchMessageException; import org.springframework.core.ResolvableType; import org.springframework.core.env.Environment; -import org.springframework.core.io.Resource; import org.springframework.stereotype.Component; +import javax.annotation.Resource; import java.io.IOException; import java.lang.annotation.Annotation; import java.util.Locale; @@ -30,8 +28,7 @@ import java.util.Map; @Component("proxyApplicationContext") public class ProxyApplicationContextImpl implements ProxyApplicationContext { - @Qualifier("webApplicationContext") - @Autowired + @Resource private ApplicationContext applicationContext; @Override @@ -250,12 +247,12 @@ public class ProxyApplicationContextImpl implements ProxyApplicationContext { } @Override - public Resource[] getResources(String locationPattern) throws IOException { + public org.springframework.core.io.Resource[] getResources(String locationPattern) throws IOException { return applicationContext.getResources(locationPattern); } @Override - public Resource getResource(String location) { + public org.springframework.core.io.Resource getResource(String location) { return applicationContext.getResource(location); } diff --git a/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/service/DemoConvertServiceImpl.java b/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/service/DemoConvertServiceImpl.java index d0781106..fcce323a 100644 --- a/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/service/DemoConvertServiceImpl.java +++ b/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/service/DemoConvertServiceImpl.java @@ -1,19 +1,20 @@ package org.dbsyncer.plugin.service; import org.dbsyncer.common.config.AppConfig; -import org.dbsyncer.common.spi.ConvertContext; -import org.dbsyncer.common.spi.ConvertService; +import org.dbsyncer.sdk.spi.ConvertContext; +import org.dbsyncer.sdk.spi.ConvertService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import javax.annotation.Resource; + @Component public class DemoConvertServiceImpl implements ConvertService { private final Logger logger = LoggerFactory.getLogger(getClass()); - @Autowired + @Resource private AppConfig appConfig; @Override diff --git a/dbsyncer-sdk/pom.xml b/dbsyncer-sdk/pom.xml new file mode 100644 index 00000000..6b6a34b7 --- /dev/null +++ b/dbsyncer-sdk/pom.xml @@ -0,0 +1,21 @@ + + + + dbsyncer + org.ghi + 2.0.0 + + 4.0.0 + dbsyncer-sdk + + + + + org.ghi + dbsyncer-common + ${project.parent.version} + + + \ No newline at end of file diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/RemoteClient.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/RemoteClient.java new file mode 100644 index 00000000..dabc20c9 --- /dev/null +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/RemoteClient.java @@ -0,0 +1,12 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.sdk; + +/** + * @Version 1.0.0 + * @Author AE86 + * @Date 2023-11-14 23:34 + */ +public class RemoteClient { +} \ No newline at end of file diff --git a/dbsyncer-common/src/main/java/org/dbsyncer/common/model/AbstractConvertContext.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/AbstractConvertContext.java similarity index 93% rename from dbsyncer-common/src/main/java/org/dbsyncer/common/model/AbstractConvertContext.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/AbstractConvertContext.java index 0dc924c5..a61a3ec0 100644 --- a/dbsyncer-common/src/main/java/org/dbsyncer/common/model/AbstractConvertContext.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/AbstractConvertContext.java @@ -1,8 +1,8 @@ -package org.dbsyncer.common.model; +package org.dbsyncer.sdk.model; -import org.dbsyncer.common.spi.ConnectorMapper; -import org.dbsyncer.common.spi.ConvertContext; -import org.dbsyncer.common.spi.ProxyApplicationContext; +import org.dbsyncer.sdk.spi.ConnectorMapper; +import org.dbsyncer.sdk.spi.ConvertContext; +import org.dbsyncer.sdk.spi.ProxyApplicationContext; import java.util.List; import java.util.Map; diff --git a/dbsyncer-common/src/main/java/org/dbsyncer/common/model/AbstractConnectorConfig.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/ConnectorConfig.java similarity index 66% rename from dbsyncer-common/src/main/java/org/dbsyncer/common/model/AbstractConnectorConfig.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/ConnectorConfig.java index 7d692106..227d7043 100644 --- a/dbsyncer-common/src/main/java/org/dbsyncer/common/model/AbstractConnectorConfig.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/ConnectorConfig.java @@ -1,4 +1,4 @@ -package org.dbsyncer.common.model; +package org.dbsyncer.sdk.model; /** * 连接器配置 @@ -6,7 +6,7 @@ package org.dbsyncer.common.model; * @version 1.0.0 * @date 2019/9/21 0:01 */ -public abstract class AbstractConnectorConfig { +public abstract class ConnectorConfig { /** * 连接器类型 @@ -17,7 +17,7 @@ public abstract class AbstractConnectorConfig { return connectorType; } - public AbstractConnectorConfig setConnectorType(String connectorType) { + public ConnectorConfig setConnectorType(String connectorType) { this.connectorType = connectorType; return this; } diff --git a/dbsyncer-common/src/main/java/org/dbsyncer/common/model/FullConvertContext.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/FullConvertContext.java similarity index 83% rename from dbsyncer-common/src/main/java/org/dbsyncer/common/model/FullConvertContext.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/FullConvertContext.java index ff767445..ea7f2370 100644 --- a/dbsyncer-common/src/main/java/org/dbsyncer/common/model/FullConvertContext.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/FullConvertContext.java @@ -1,6 +1,6 @@ -package org.dbsyncer.common.model; +package org.dbsyncer.sdk.model; -import org.dbsyncer.common.spi.ConnectorMapper; +import org.dbsyncer.sdk.spi.ConnectorMapper; /** * @author AE86 diff --git a/dbsyncer-common/src/main/java/org/dbsyncer/common/model/IncrementConvertContext.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/IncrementConvertContext.java similarity index 86% rename from dbsyncer-common/src/main/java/org/dbsyncer/common/model/IncrementConvertContext.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/IncrementConvertContext.java index 969398e2..702af7ee 100644 --- a/dbsyncer-common/src/main/java/org/dbsyncer/common/model/IncrementConvertContext.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/IncrementConvertContext.java @@ -1,6 +1,6 @@ -package org.dbsyncer.common.model; +package org.dbsyncer.sdk.model; -import org.dbsyncer.common.spi.ConnectorMapper; +import org.dbsyncer.sdk.spi.ConnectorMapper; import java.util.List; import java.util.Map; diff --git a/dbsyncer-common/src/main/java/org/dbsyncer/common/model/NotifyMessage.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/NotifyMessage.java similarity index 96% rename from dbsyncer-common/src/main/java/org/dbsyncer/common/model/NotifyMessage.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/NotifyMessage.java index 2df750d4..3db4281b 100644 --- a/dbsyncer-common/src/main/java/org/dbsyncer/common/model/NotifyMessage.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/NotifyMessage.java @@ -1,4 +1,4 @@ -package org.dbsyncer.common.model; +package org.dbsyncer.sdk.model; import java.util.List; diff --git a/dbsyncer-common/src/main/java/org/dbsyncer/common/spi/ConnectorMapper.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/ConnectorMapper.java similarity index 62% rename from dbsyncer-common/src/main/java/org/dbsyncer/common/spi/ConnectorMapper.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/ConnectorMapper.java index d72d303b..f40a093b 100644 --- a/dbsyncer-common/src/main/java/org/dbsyncer/common/spi/ConnectorMapper.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/ConnectorMapper.java @@ -1,7 +1,4 @@ -package org.dbsyncer.common.spi; - - -import org.dbsyncer.common.model.AbstractConnectorConfig; +package org.dbsyncer.sdk.spi; /** * 连接器实例，管理连接生命周期 @@ -14,24 +11,6 @@ import org.dbsyncer.common.model.AbstractConnectorConfig; */ public interface ConnectorMapper extends Cloneable { - /** - * 获取连接配置 - * - * @return - */ - default AbstractConnectorConfig getOriginalConfig() { - return (AbstractConnectorConfig) getConfig(); - } - - /** - * 获取连接器类型 - * - * @return - */ - default String getConnectorType() { - return getOriginalConfig().getConnectorType(); - } - /** * 获取连接配置 * @@ -41,6 +20,7 @@ public interface ConnectorMapper extends Cloneable { /** * 设置 + * * @param k */ void setConfig(K k); diff --git a/dbsyncer-common/src/main/java/org/dbsyncer/common/spi/ConvertContext.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/ConvertContext.java similarity index 91% rename from dbsyncer-common/src/main/java/org/dbsyncer/common/spi/ConvertContext.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/ConvertContext.java index 7b04ef7e..b07a4a77 100644 --- a/dbsyncer-common/src/main/java/org/dbsyncer/common/spi/ConvertContext.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/ConvertContext.java @@ -1,70 +1,70 @@ -package org.dbsyncer.common.spi; - -import java.util.List; -import java.util.Map; - -/** - * 插件转换上下文 - * - * @author AE86 - * @version 1.0.0 - * @date 2022/10/28 20:26 - */ -public interface ConvertContext { - - /** - * 是否终止同步数据到目标源库 - * - * @return - */ - boolean isTerminated(); - - /** - * 是否终止同步数据到目标源库 - *

true: 终止，默认值false - * - * @param terminated - */ - void setTerminated(boolean terminated); - - /** - * Spring上下文 - */ - ProxyApplicationContext getContext(); - - /** - * 数据源连接实例 - */ - ConnectorMapper getSourceConnectorMapper(); - - /** - * 目标源连接实例 - */ - ConnectorMapper getTargetConnectorMapper(); - - /** - * 数据源表 - */ - String getSourceTableName(); - - /** - * 目标源表 - */ - String getTargetTableName(); - - /** - * 增量同步，事件（INSERT/UPDATE/DELETE） - */ - String getEvent(); - - /** - * 数据源数据集合 - */ - List getSourceList(); - - /** - * 目标源源数据集合 - */ - List getTargetList(); - +package org.dbsyncer.sdk.spi; + +import java.util.List; +import java.util.Map; + +/** + * 插件转换上下文 + * + * @author AE86 + * @version 1.0.0 + * @date 2022/10/28 20:26 + */ +public interface ConvertContext { + + /** + * 是否终止同步数据到目标源库 + * + * @return + */ + boolean isTerminated(); + + /** + * 是否终止同步数据到目标源库 + *

true: 终止，默认值false + * + * @param terminated + */ + void setTerminated(boolean terminated); + + /** + * Spring上下文 + */ + ProxyApplicationContext getContext(); + + /** + * 数据源连接实例 + */ + ConnectorMapper getSourceConnectorMapper(); + + /** + * 目标源连接实例 + */ + ConnectorMapper getTargetConnectorMapper(); + + /** + * 数据源表 + */ + String getSourceTableName(); + + /** + * 目标源表 + */ + String getTargetTableName(); + + /** + * 增量同步，事件（INSERT/UPDATE/DELETE） + */ + String getEvent(); + + /** + * 数据源数据集合 + */ + List getSourceList(); + + /** + * 目标源源数据集合 + */ + List getTargetList(); + } \ No newline at end of file diff --git a/dbsyncer-common/src/main/java/org/dbsyncer/common/spi/ConvertService.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/ConvertService.java similarity index 91% rename from dbsyncer-common/src/main/java/org/dbsyncer/common/spi/ConvertService.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/ConvertService.java index d4c2d809..22564a3a 100644 --- a/dbsyncer-common/src/main/java/org/dbsyncer/common/spi/ConvertService.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/ConvertService.java @@ -1,58 +1,58 @@ -package org.dbsyncer.common.spi; - -/** - * 插件扩展服务接口 - *

全量同步/增量同步,扩展转换

- * - * @author AE86 - * @version 1.0.0 - * @date 2021/02/22 20:26 - */ -public interface ConvertService { - - /** - * 初始化加载 - */ - default void init(){ - - } - - /** - * 全量同步/增量同步 - * - * @param context 上下文 - */ - void convert(ConvertContext context); - - /** - * 全量同步/增量同步完成后执行处理 - * - * @param context 上下文 - */ - default void postProcessAfter(ConvertContext context) { - } - - /** - * 版本号 - * - * @return - */ - default String getVersion() { - return "1.0.0"; - } - - /** - * 插件名称 - * - * @return - */ - default String getName() { - return getClass().getSimpleName(); - } - - /** - * 关闭服务(上传相同插件或服务关闭时触发) - */ - default void close() { - } +package org.dbsyncer.sdk.spi; + +/** + * 插件扩展服务接口 + *

全量同步/增量同步,扩展转换

+ * + * @author AE86 + * @version 1.0.0 + * @date 2021/02/22 20:26 + */ +public interface ConvertService { + + /** + * 初始化加载 + */ + default void init(){ + + } + + /** + * 全量同步/增量同步 + * + * @param context 上下文 + */ + void convert(ConvertContext context); + + /** + * 全量同步/增量同步完成后执行处理 + * + * @param context 上下文 + */ + default void postProcessAfter(ConvertContext context) { + } + + /** + * 版本号 + * + * @return + */ + default String getVersion() { + return "1.0.0"; + } + + /** + * 插件名称 + * + * @return + */ + default String getName() { + return getClass().getSimpleName(); + } + + /** + * 关闭服务(上传相同插件或服务关闭时触发) + */ + default void close() { + } } \ No newline at end of file diff --git a/dbsyncer-common/src/main/java/org/dbsyncer/common/spi/NotifyService.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/NotifyService.java similarity index 79% rename from dbsyncer-common/src/main/java/org/dbsyncer/common/spi/NotifyService.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/NotifyService.java index 77f36980..cdbd1c2a 100644 --- a/dbsyncer-common/src/main/java/org/dbsyncer/common/spi/NotifyService.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/NotifyService.java @@ -1,6 +1,6 @@ -package org.dbsyncer.common.spi; +package org.dbsyncer.sdk.spi; -import org.dbsyncer.common.model.NotifyMessage; +import org.dbsyncer.sdk.model.NotifyMessage; /** * 通知服务（同步异常、连接器异常、应用异常等消息） diff --git a/dbsyncer-common/src/main/java/org/dbsyncer/common/spi/ProxyApplicationContext.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/ProxyApplicationContext.java similarity index 87% rename from dbsyncer-common/src/main/java/org/dbsyncer/common/spi/ProxyApplicationContext.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/ProxyApplicationContext.java index 84ea059a..239879af 100644 --- a/dbsyncer-common/src/main/java/org/dbsyncer/common/spi/ProxyApplicationContext.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/ProxyApplicationContext.java @@ -1,4 +1,4 @@ -package org.dbsyncer.common.spi; +package org.dbsyncer.sdk.spi; import org.springframework.context.ApplicationContext; diff --git a/dbsyncer-storage/pom.xml b/dbsyncer-storage/pom.xml index f7546d6b..a8a21263 100644 --- a/dbsyncer-storage/pom.xml +++ b/dbsyncer-storage/pom.xml @@ -5,19 +5,12 @@ dbsyncer org.ghi - 1.2.7_1103 + 2.0.0 4.0.0 dbsyncer-storage - - - org.ghi - dbsyncer-cluster - ${project.parent.version} - - org.ghi diff --git a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/binlog/AbstractBinlogService.java b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/binlog/AbstractBinlogService.java index 6df98511..e1489cbe 100644 --- a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/binlog/AbstractBinlogService.java +++ b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/binlog/AbstractBinlogService.java @@ -22,9 +22,9 @@ import org.dbsyncer.storage.query.filter.IntFilter; import org.dbsyncer.storage.query.filter.LongFilter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; import javax.annotation.PostConstruct; +import javax.annotation.Resource; import java.sql.Timestamp; import java.time.Instant; import java.time.LocalDateTime; @@ -48,13 +48,13 @@ public abstract class AbstractBinlogService implements BinlogRecorder { private final Logger logger = LoggerFactory.getLogger(getClass()); - @Autowired + @Resource private StorageService storageService; - @Autowired + @Resource private ScheduledTaskService scheduledTaskService; - @Autowired + @Resource private SnowflakeIdWorker snowflakeIdWorker; private BinlogRecorderConfig binlogRecorderConfig = new BinlogRecorderConfig(); @@ -161,7 +161,7 @@ public abstract class AbstractBinlogService implements BinlogRecorder { } catch (Exception e) { logger.error(e.getMessage(), e); } finally { - if (locked) { + if (locked) { binlogLock.unlock(); } } diff --git a/dbsyncer-web/pom.xml b/dbsyncer-web/pom.xml index 6d07df1f..d09a42ea 100644 --- a/dbsyncer-web/pom.xml +++ b/dbsyncer-web/pom.xml @@ -5,7 +5,7 @@ dbsyncer org.ghi - 1.2.7_1103 + 2.0.0 4.0.0 dbsyncer-web diff --git a/dbsyncer-web/src/main/resources/public/plugin/plugin.html b/dbsyncer-web/src/main/resources/public/plugin/plugin.html index 8022d20d..5df029fe 100644 --- a/dbsyncer-web/src/main/resources/public/plugin/plugin.html +++ b/dbsyncer-web/src/main/resources/public/plugin/plugin.html @@ -19,11 +19,11 @@
  • 导入开发包：
    • 方式1：导入jar -

      dbsyncer-common-[[${version}]].jar

      +

      dbsyncer-sdk-[[${version}]].jar

      dbsyncer-connector-[[${version}]].jar (非必须)

    • 方式2：引入pom（需要安装到本地） - 
      <dependency>
        <groupId>org.ghi</groupId>
        <artifactId>dbsyncer-common</artifactId>
        <version>[[${version}]]</version>
      </dependency>
      <dependency>
        <groupId>org.ghi</groupId>
        <artifactId>dbsyncer-connector</artifactId>
        <version>[[${version}]]</version>
      </dependency>
      + 
      <dependency>
        <groupId>org.ghi</groupId>
        <artifactId>dbsyncer-sdk</artifactId>
        <version>[[${version}]]</version>
      </dependency>
      <dependency>
        <groupId>org.ghi</groupId>
        <artifactId>dbsyncer-connector</artifactId>
        <version>[[${version}]]</version>
      </dependency>
    • diff --git a/pom.xml b/pom.xml index 949fa21b..ab9f1691 100644 --- a/pom.xml +++ b/pom.xml @@ -6,12 +6,12 @@ org.ghi dbsyncer - 1.2.7_1103 + 2.0.0 pom dbsyncer https://gitee.com/ghi/dbsyncer - + dbsyncer-biz dbsyncer-cluster @@ -21,6 +21,7 @@ dbsyncer-manager dbsyncer-parser dbsyncer-plugin + dbsyncer-sdk dbsyncer-storage dbsyncer-web diff --git a/version.cmd b/version.cmd index 7abd903f..790d3ab5 100644 --- a/version.cmd +++ b/version.cmd @@ -1,7 +1,7 @@ @echo off set CURRENT_DATE=%date:~5,2%%date:~8,2% -set VERSION=1.2.7_%CURRENT_DATE% +set VERSION=2.0.0_%CURRENT_DATE% set /p APP_VERSION=Please enter a new version number(%VERSION%): || set APP_VERSION=%VERSION% echo %APP_VERSION% -- Gitee From c9ae8380bc9b3a4d936eff29dfc346bf13af5c3a Mon Sep 17 00:00:00 2001 From: AE86 <836391306@qq.com> Date: Wed, 15 Nov 2023 00:57:43 +0800 Subject: [PATCH 06/98] fix pom --- dbsyncer-common/pom.xml | 5 ----- dbsyncer-connector/pom.xml | 7 ------- dbsyncer-listener/pom.xml | 5 +++++ dbsyncer-plugin/pom.xml | 7 ------- 4 files changed, 5 insertions(+), 19 deletions(-) diff --git a/dbsyncer-common/pom.xml b/dbsyncer-common/pom.xml index 4f143607..448942be 100644 --- a/dbsyncer-common/pom.xml +++ b/dbsyncer-common/pom.xml @@ -56,11 +56,6 @@ fastjson2     - - com.github.jsqlparser - jsqlparser - -     \ No newline at end of file diff --git a/dbsyncer-connector/pom.xml b/dbsyncer-connector/pom.xml index 02043833..e8cb7978 100644 --- a/dbsyncer-connector/pom.xml +++ b/dbsyncer-connector/pom.xml @@ -18,13 +18,6 @@ ${project.parent.version}     - - - org.ghi - dbsyncer-common - ${project.parent.version} - - org.springframework diff --git a/dbsyncer-listener/pom.xml b/dbsyncer-listener/pom.xml index 0405d8b7..b76ac45f 100644 --- a/dbsyncer-listener/pom.xml +++ b/dbsyncer-listener/pom.xml @@ -18,6 +18,11 @@ ${project.parent.version} + + com.github.jsqlparser + jsqlparser + + com.zendesk mysql-binlog-connector-java diff --git a/dbsyncer-plugin/pom.xml b/dbsyncer-plugin/pom.xml index 2a73c372..547d80d8 100644 --- a/dbsyncer-plugin/pom.xml +++ b/dbsyncer-plugin/pom.xml @@ -18,13 +18,6 @@ ${project.parent.version} - - - org.ghi - dbsyncer-connector - ${project.parent.version} - - org.springframework.boot -- Gitee From 6e96cd023fc853e2f4ff4f54430cecf8c6934092 Mon Sep 17 00:00:00 2001 From: AE86 <836391306@qq.com> Date: Thu, 16 Nov 2023 00:32:26 +0800 Subject: [PATCH 07/98] fix code --- .../org/dbsyncer/manager/ManagerFactory.java | 9 +++++- .../manager/listener/ClosedTaskListener.java | 29 ------------------- 2 files changed, 8 insertions(+), 30 deletions(-) delete mode 100644 dbsyncer-manager/src/main/java/org/dbsyncer/manager/listener/ClosedTaskListener.java diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/ManagerFactory.java b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/ManagerFactory.java index 9605bc8f..2370d2f4 100644 --- a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/ManagerFactory.java +++ b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/ManagerFactory.java @@ -1,9 +1,11 @@ package org.dbsyncer.manager; +import org.dbsyncer.manager.event.ClosedEvent; import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.parser.enums.MetaEnum; import org.dbsyncer.parser.model.Mapping; import org.dbsyncer.parser.model.Meta; +import org.springframework.context.ApplicationListener; import org.springframework.stereotype.Component; import org.springframework.util.Assert; @@ -17,7 +19,7 @@ import java.util.Map; * @date 2019/9/16 23:59 */ @Component -public class ManagerFactory { +public class ManagerFactory implements ApplicationListener { @Resource private ProfileComponent profileComponent; @@ -25,6 +27,11 @@ public class ManagerFactory { @Resource private Map map; + @Override + public void onApplicationEvent(ClosedEvent event) { + changeMetaState(event.getMetaId(), MetaEnum.READY); + } + public void start(Mapping mapping) { Puller puller = getPuller(mapping); diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/listener/ClosedTaskListener.java b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/listener/ClosedTaskListener.java deleted file mode 100644 index 4f8404be..00000000 --- a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/listener/ClosedTaskListener.java +++ /dev/null @@ -1,29 +0,0 @@ -/** - * DBSyncer Copyright 2020-2023 All Rights Reserved. - */ -package org.dbsyncer.manager.listener; - -import org.dbsyncer.manager.event.ClosedEvent; -import org.dbsyncer.manager.ManagerFactory; -import org.dbsyncer.parser.enums.MetaEnum; -import org.springframework.context.ApplicationListener; -import org.springframework.stereotype.Component; - -import javax.annotation.Resource; - -/** - * @Version 1.0.0 - * @Author AE86 - * @Date 2023-11-14 01:48 - */ -@Component -public class ClosedTaskListener implements ApplicationListener { - - @Resource - private ManagerFactory managerFactory; - - @Override - public void onApplicationEvent(ClosedEvent event) { - managerFactory.changeMetaState(event.getMetaId(), MetaEnum.READY); - } -} \ No newline at end of file -- Gitee From c8a4d8b71a15373b2e346c26890c6dee3741fd50 Mon Sep 17 00:00:00 2001 From: AE86 <836391306@qq.com> Date: Thu, 16 Nov 2023 00:55:36 +0800 Subject: [PATCH 08/98] fix code --- .../checker/impl/mapping/MappingChecker.java | 2 +- .../impl/tablegroup/TableGroupChecker.java | 2 +- .../dbsyncer/biz/impl/BaseServiceImpl.java | 2 +- .../dbsyncer/biz/impl/MappingServiceImpl.java | 2 +- .../dbsyncer/biz/impl/MonitorServiceImpl.java | 2 +- .../biz/impl/SystemConfigServiceImpl.java | 2 +- .../manager/{puller => }/AbstractPuller.java | 3 +- .../manager/{puller => impl}/FullPuller.java | 3 +- .../{puller => impl}/IncrementPuller.java | 3 +- .../{template => impl}/PreloadTemplate.java | 4 +- .../{template => impl}/OperationTemplate.java | 2 +- .../parser/impl/ProfileComponentImpl.java | 1 - .../org/dbsyncer/parser/model/Mapping.java | 2 +- .../org/dbsyncer/plugin/PluginFactory.java | 25 +- .../proxy/ProxyApplicationContextImpl.java | 268 ------------------ .../java/org/dbsyncer/sdk/SdkException.java | 30 ++ .../org/dbsyncer/sdk}/enums/ModelEnum.java | 14 +- .../sdk/model/AbstractConvertContext.java | 15 - .../org/dbsyncer/sdk/spi/ConvertContext.java | 5 - .../sdk/spi/ProxyApplicationContext.java | 13 - 20 files changed, 63 insertions(+), 337 deletions(-) rename dbsyncer-manager/src/main/java/org/dbsyncer/manager/{puller => }/AbstractPuller.java (84%) rename dbsyncer-manager/src/main/java/org/dbsyncer/manager/{puller => impl}/FullPuller.java (98%) rename dbsyncer-manager/src/main/java/org/dbsyncer/manager/{puller => impl}/IncrementPuller.java (98%) rename dbsyncer-manager/src/main/java/org/dbsyncer/manager/{template => impl}/PreloadTemplate.java (98%) rename dbsyncer-parser/src/main/java/org/dbsyncer/parser/{template => impl}/OperationTemplate.java (99%) delete mode 100644 dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/proxy/ProxyApplicationContextImpl.java create mode 100644 dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/SdkException.java rename {dbsyncer-parser/src/main/java/org/dbsyncer/parser => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/enums/ModelEnum.java (74%) delete mode 100644 dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/ProxyApplicationContext.java diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/mapping/MappingChecker.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/mapping/MappingChecker.java index bbf6f2c5..a352ee62 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/mapping/MappingChecker.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/mapping/MappingChecker.java @@ -11,7 +11,7 @@ import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.listener.config.ListenerConfig; import org.dbsyncer.listener.enums.ListenerTypeEnum; import org.dbsyncer.parser.ProfileComponent; -import org.dbsyncer.parser.enums.ModelEnum; +import org.dbsyncer.sdk.enums.ModelEnum; import org.dbsyncer.parser.model.ConfigModel; import org.dbsyncer.parser.model.Mapping; import org.dbsyncer.parser.model.Meta; diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/tablegroup/TableGroupChecker.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/tablegroup/TableGroupChecker.java index 5b45072f..79d07579 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/tablegroup/TableGroupChecker.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/tablegroup/TableGroupChecker.java @@ -12,7 +12,7 @@ import org.dbsyncer.connector.model.Table; import org.dbsyncer.connector.util.PrimaryKeyUtil; import org.dbsyncer.parser.ParserComponent; import org.dbsyncer.parser.ProfileComponent; -import org.dbsyncer.parser.enums.ModelEnum; +import org.dbsyncer.sdk.enums.ModelEnum; import org.dbsyncer.parser.model.ConfigModel; import org.dbsyncer.parser.model.FieldMapping; import org.dbsyncer.parser.model.Mapping; diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/BaseServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/BaseServiceImpl.java index 5dc8244b..60f54387 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/BaseServiceImpl.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/BaseServiceImpl.java @@ -2,7 +2,7 @@ package org.dbsyncer.biz.impl; import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.parser.enums.MetaEnum; -import org.dbsyncer.parser.enums.ModelEnum; +import org.dbsyncer.sdk.enums.ModelEnum; import org.dbsyncer.parser.LogService; import org.dbsyncer.parser.LogType; import org.dbsyncer.parser.MessageService; diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MappingServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MappingServiceImpl.java index efaaa4de..1232a9eb 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MappingServiceImpl.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MappingServiceImpl.java @@ -16,7 +16,7 @@ import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.connector.model.Table; import org.dbsyncer.manager.ManagerFactory; import org.dbsyncer.parser.ProfileComponent; -import org.dbsyncer.parser.enums.ModelEnum; +import org.dbsyncer.sdk.enums.ModelEnum; import org.dbsyncer.parser.LogType; import org.dbsyncer.parser.model.ConfigModel; import org.dbsyncer.parser.model.Connector; diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java index 9881005b..1aa2cc9c 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java @@ -31,7 +31,7 @@ import org.dbsyncer.biz.model.AppReportMetric; import org.dbsyncer.biz.model.MetricResponse; import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.parser.enums.MetaEnum; -import org.dbsyncer.parser.enums.ModelEnum; +import org.dbsyncer.sdk.enums.ModelEnum; import org.dbsyncer.parser.LogService; import org.dbsyncer.parser.LogType; import org.dbsyncer.parser.model.Mapping; diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/SystemConfigServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/SystemConfigServiceImpl.java index 2e246773..453e17c7 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/SystemConfigServiceImpl.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/SystemConfigServiceImpl.java @@ -11,7 +11,7 @@ import org.dbsyncer.parser.LogService; import org.dbsyncer.parser.LogType; import org.dbsyncer.parser.model.ConfigModel; import org.dbsyncer.parser.model.SystemConfig; -import org.dbsyncer.manager.template.PreloadTemplate; +import org.dbsyncer.manager.impl.PreloadTemplate; import org.dbsyncer.plugin.enums.FileSuffixEnum; import org.springframework.beans.BeanUtils; import org.springframework.stereotype.Service; diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/puller/AbstractPuller.java b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/AbstractPuller.java similarity index 84% rename from dbsyncer-manager/src/main/java/org/dbsyncer/manager/puller/AbstractPuller.java rename to dbsyncer-manager/src/main/java/org/dbsyncer/manager/AbstractPuller.java index c5315a9d..d68fe5f8 100644 --- a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/puller/AbstractPuller.java +++ b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/AbstractPuller.java @@ -1,7 +1,6 @@ -package org.dbsyncer.manager.puller; +package org.dbsyncer.manager; import org.dbsyncer.manager.event.ClosedEvent; -import org.dbsyncer.manager.Puller; import org.springframework.context.ApplicationContext; import javax.annotation.Resource; diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/puller/FullPuller.java b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/FullPuller.java similarity index 98% rename from dbsyncer-manager/src/main/java/org/dbsyncer/manager/puller/FullPuller.java rename to dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/FullPuller.java index 86e47e9c..25772868 100644 --- a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/puller/FullPuller.java +++ b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/FullPuller.java @@ -1,8 +1,9 @@ -package org.dbsyncer.manager.puller; +package org.dbsyncer.manager.impl; import org.dbsyncer.common.util.NumberUtil; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.connector.util.PrimaryKeyUtil; +import org.dbsyncer.manager.AbstractPuller; import org.dbsyncer.parser.ParserComponent; import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.parser.enums.ParserEnum; diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/puller/IncrementPuller.java b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/IncrementPuller.java similarity index 98% rename from dbsyncer-manager/src/main/java/org/dbsyncer/manager/puller/IncrementPuller.java rename to dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/IncrementPuller.java index b3500a19..e383c146 100644 --- a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/puller/IncrementPuller.java +++ b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/IncrementPuller.java @@ -1,6 +1,7 @@ -package org.dbsyncer.manager.puller; +package org.dbsyncer.manager.impl; import org.dbsyncer.listener.model.ChangedOffset; +import org.dbsyncer.manager.AbstractPuller; import org.dbsyncer.parser.event.RefreshOffsetEvent; import org.dbsyncer.common.scheduled.ScheduledTaskJob; import org.dbsyncer.common.scheduled.ScheduledTaskService; diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/template/PreloadTemplate.java b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/PreloadTemplate.java similarity index 98% rename from dbsyncer-manager/src/main/java/org/dbsyncer/manager/template/PreloadTemplate.java rename to dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/PreloadTemplate.java index de7025a0..9f29d9f9 100644 --- a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/template/PreloadTemplate.java +++ b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/PreloadTemplate.java @@ -1,4 +1,4 @@ -package org.dbsyncer.manager.template; +package org.dbsyncer.manager.impl; import org.dbsyncer.manager.event.PreloadCompletedEvent; import org.dbsyncer.common.model.Paging; @@ -14,7 +14,7 @@ import org.dbsyncer.parser.model.ConfigModel; import org.dbsyncer.parser.model.Mapping; import org.dbsyncer.parser.model.Meta; import org.dbsyncer.parser.model.OperationConfig; -import org.dbsyncer.parser.template.OperationTemplate; +import org.dbsyncer.parser.impl.OperationTemplate; import org.dbsyncer.plugin.PluginFactory; import org.dbsyncer.storage.StorageService; import org.dbsyncer.storage.constant.ConfigConstant; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/template/OperationTemplate.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/OperationTemplate.java similarity index 99% rename from dbsyncer-parser/src/main/java/org/dbsyncer/parser/template/OperationTemplate.java rename to dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/OperationTemplate.java index c09bdca7..e84af993 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/template/OperationTemplate.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/OperationTemplate.java @@ -1,7 +1,7 @@ /** * DBSyncer Copyright 2020-2023 All Rights Reserved. */ -package org.dbsyncer.parser.template; +package org.dbsyncer.parser.impl; import org.dbsyncer.parser.CacheService; import org.dbsyncer.common.util.CollectionUtils; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/ProfileComponentImpl.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/ProfileComponentImpl.java index 13fb2a92..e03f3371 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/ProfileComponentImpl.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/ProfileComponentImpl.java @@ -23,7 +23,6 @@ import org.dbsyncer.parser.model.QueryConfig; import org.dbsyncer.parser.model.SystemConfig; import org.dbsyncer.parser.model.TableGroup; import org.dbsyncer.parser.model.UserConfig; -import org.dbsyncer.parser.template.OperationTemplate; import org.dbsyncer.sdk.model.ConnectorConfig; import org.dbsyncer.storage.enums.StorageDataStatusEnum; import org.springframework.stereotype.Component; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/Mapping.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/Mapping.java index e1e9840e..2f2c4249 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/Mapping.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/Mapping.java @@ -2,7 +2,7 @@ package org.dbsyncer.parser.model; import org.dbsyncer.connector.model.Field; import org.dbsyncer.listener.config.ListenerConfig; -import org.dbsyncer.parser.enums.ModelEnum; +import org.dbsyncer.sdk.enums.ModelEnum; import org.dbsyncer.storage.constant.ConfigConstant; import java.util.List; diff --git a/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/PluginFactory.java b/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/PluginFactory.java index 3ffe84dc..be0b00f0 100644 --- a/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/PluginFactory.java +++ b/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/PluginFactory.java @@ -3,13 +3,12 @@ package org.dbsyncer.plugin; import org.apache.commons.io.FileUtils; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.plugin.config.Plugin; -import org.dbsyncer.sdk.model.AbstractConvertContext; import org.dbsyncer.sdk.spi.ConvertContext; import org.dbsyncer.sdk.spi.ConvertService; -import org.dbsyncer.sdk.spi.ProxyApplicationContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.DisposableBean; +import org.springframework.context.ApplicationContext; import org.springframework.stereotype.Component; import javax.annotation.PostConstruct; @@ -55,11 +54,11 @@ public class PluginFactory implements DisposableBean { private final Map service = new LinkedHashMap<>(); @Resource - private ProxyApplicationContext proxyApplicationContext; + private ApplicationContext applicationContext; @PostConstruct private void init() { - Map services = proxyApplicationContext.getBeansOfType(ConvertService.class); + Map services = applicationContext.getBeansOfType(ConvertService.class); if (!CollectionUtils.isEmpty(services)) { services.forEach((k, s) -> { String pluginId = createPluginId(s.getClass().getName(), s.getVersion()); @@ -114,13 +113,10 @@ public class PluginFactory implements DisposableBean { public void convert(Plugin plugin, ConvertContext context) { if (null != plugin) { String pluginId = createPluginId(plugin.getClassName(), plugin.getVersion()); - if (service.containsKey(pluginId)) { - if (context instanceof AbstractConvertContext) { - AbstractConvertContext ctx = (AbstractConvertContext) context; - ctx.setContext(proxyApplicationContext); - } - service.get(pluginId).convert(context); - } + service.computeIfPresent(pluginId, (k, c) -> { + c.convert(context); + return c; + }); } } @@ -133,9 +129,10 @@ public class PluginFactory implements DisposableBean { public void postProcessAfter(Plugin plugin, ConvertContext context) { if (null != plugin) { String pluginId = createPluginId(plugin.getClassName(), plugin.getVersion()); - if (service.containsKey(pluginId)) { - service.get(pluginId).postProcessAfter(context); - } + service.computeIfPresent(pluginId, (k, c) -> { + c.postProcessAfter(context); + return c; + }); } } diff --git a/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/proxy/ProxyApplicationContextImpl.java b/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/proxy/ProxyApplicationContextImpl.java deleted file mode 100644 index 4de16e86..00000000 --- a/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/proxy/ProxyApplicationContextImpl.java +++ /dev/null @@ -1,268 +0,0 @@ -package org.dbsyncer.plugin.proxy; - -import org.dbsyncer.sdk.spi.ProxyApplicationContext; -import org.springframework.beans.BeansException; -import org.springframework.beans.factory.BeanFactory; -import org.springframework.beans.factory.NoSuchBeanDefinitionException; -import org.springframework.beans.factory.ObjectProvider; -import org.springframework.beans.factory.config.AutowireCapableBeanFactory; -import org.springframework.context.ApplicationContext; -import org.springframework.context.ApplicationEvent; -import org.springframework.context.MessageSourceResolvable; -import org.springframework.context.NoSuchMessageException; -import org.springframework.core.ResolvableType; -import org.springframework.core.env.Environment; -import org.springframework.stereotype.Component; - -import javax.annotation.Resource; -import java.io.IOException; -import java.lang.annotation.Annotation; -import java.util.Locale; -import java.util.Map; - -/** - * @author AE86 - * @version 1.0.0 - * @date 2022/6/30 15:14 - */ -@Component("proxyApplicationContext") -public class ProxyApplicationContextImpl implements ProxyApplicationContext { - - @Resource - private ApplicationContext applicationContext; - - @Override - public String getId() { - return applicationContext.getId(); - } - - @Override - public String getApplicationName() { - return applicationContext.getApplicationName(); - } - - @Override - public String getDisplayName() { - return applicationContext.getDisplayName(); - } - - @Override - public long getStartupDate() { - return applicationContext.getStartupDate(); - } - - @Override - public ApplicationContext getParent() { - return applicationContext.getParent(); - } - - @Override - public AutowireCapableBeanFactory getAutowireCapableBeanFactory() throws IllegalStateException { - return applicationContext.getAutowireCapableBeanFactory(); - } - - @Override - public BeanFactory getParentBeanFactory() { - return applicationContext.getParentBeanFactory(); - } - - @Override - public boolean containsLocalBean(String name) { - return applicationContext.containsLocalBean(name); - } - - @Override - public boolean containsBeanDefinition(String beanName) { - return applicationContext.containsBeanDefinition(beanName); - } - - @Override - public int getBeanDefinitionCount() { - return applicationContext.getBeanDefinitionCount(); - } - - @Override - public String[] getBeanDefinitionNames() { - return applicationContext.getBeanDefinitionNames(); - } - - @Override - public ObjectProvider getBeanProvider(Class aClass, boolean b) { - return applicationContext.getBeanProvider(aClass, b); - } - - @Override - public ObjectProvider getBeanProvider(ResolvableType resolvableType, boolean b) { - return applicationContext.getBeanProvider(resolvableType, b); - } - - @Override - public String[] getBeanNamesForType(ResolvableType type) { - return applicationContext.getBeanNamesForType(type); - } - - @Override - public String[] getBeanNamesForType(ResolvableType type, boolean includeNonSingletons, boolean allowEagerInit) { - return applicationContext.getBeanNamesForType(type, includeNonSingletons, allowEagerInit); - } - - @Override - public String[] getBeanNamesForType(Class type) { - return applicationContext.getBeanNamesForType(type); - } - - @Override - public String[] getBeanNamesForType(Class type, boolean includeNonSingletons, boolean allowEagerInit) { - return applicationContext.getBeanNamesForType(type, includeNonSingletons, allowEagerInit); - } - - @Override - public Map getBeansOfType(Class type) throws BeansException { - return applicationContext.getBeansOfType(type); - } - - @Override - public Map getBeansOfType(Class type, boolean includeNonSingletons, boolean allowEagerInit) throws BeansException { - return applicationContext.getBeansOfType(type, includeNonSingletons, allowEagerInit); - } - - @Override - public String[] getBeanNamesForAnnotation(Class annotationType) { - return applicationContext.getBeanNamesForAnnotation(annotationType); - } - - @Override - public Map getBeansWithAnnotation(Class annotationType) throws BeansException { - return applicationContext.getBeansWithAnnotation(annotationType); - } - - @Override - public A findAnnotationOnBean(String beanName, Class annotationType) throws NoSuchBeanDefinitionException { - return applicationContext.findAnnotationOnBean(beanName, annotationType); - } - - @Override - public A findAnnotationOnBean(String s, Class aClass, boolean b) throws NoSuchBeanDefinitionException { - return applicationContext.findAnnotationOnBean(s, aClass, b); - } - - @Override - public Object getBean(String name) throws BeansException { - return applicationContext.getBean(name); - } - - @Override - public T getBean(String name, Class requiredType) throws BeansException { - return applicationContext.getBean(name, requiredType); - } - - @Override - public Object getBean(String name, Object... args) throws BeansException { - return applicationContext.getBean(name, args); - } - - @Override - public T getBean(Class requiredType) throws BeansException { - return applicationContext.getBean(requiredType); - } - - @Override - public T getBean(Class requiredType, Object... args) throws BeansException { - return applicationContext.getBean(requiredType, args); - } - - @Override - public ObjectProvider getBeanProvider(Class requiredType) { - return applicationContext.getBeanProvider(requiredType); - } - - @Override - public ObjectProvider getBeanProvider(ResolvableType requiredType) { - return applicationContext.getBeanProvider(requiredType); - } - - @Override - public boolean containsBean(String name) { - return applicationContext.containsBean(name); - } - - @Override - public boolean isSingleton(String name) throws NoSuchBeanDefinitionException { - return applicationContext.isSingleton(name); - } - - @Override - public boolean isPrototype(String name) throws NoSuchBeanDefinitionException { - return applicationContext.isPrototype(name); - } - - @Override - public boolean isTypeMatch(String name, ResolvableType typeToMatch) throws NoSuchBeanDefinitionException { - return applicationContext.isTypeMatch(name, typeToMatch); - } - - @Override - public boolean isTypeMatch(String name, Class typeToMatch) throws NoSuchBeanDefinitionException { - return applicationContext.isTypeMatch(name, typeToMatch); - } - - @Override - public Class getType(String name) throws NoSuchBeanDefinitionException { - return applicationContext.getType(name); - } - - @Override - public Class getType(String name, boolean allowFactoryBeanInit) throws NoSuchBeanDefinitionException { - return applicationContext.getType(name, allowFactoryBeanInit); - } - - @Override - public String[] getAliases(String name) { - return applicationContext.getAliases(name); - } - - @Override - public void publishEvent(Object event) { - applicationContext.publishEvent(event); - } - - @Override - public String getMessage(String code, Object[] args, String defaultMessage, Locale locale) { - return applicationContext.getMessage(code, args, defaultMessage, locale); - } - - @Override - public String getMessage(String code, Object[] args, Locale locale) throws NoSuchMessageException { - return applicationContext.getMessage(code, args, locale); - } - - @Override - public String getMessage(MessageSourceResolvable resolvable, Locale locale) throws NoSuchMessageException { - return applicationContext.getMessage(resolvable, locale); - } - - @Override - public Environment getEnvironment() { - return applicationContext.getEnvironment(); - } - - @Override - public org.springframework.core.io.Resource[] getResources(String locationPattern) throws IOException { - return applicationContext.getResources(locationPattern); - } - - @Override - public org.springframework.core.io.Resource getResource(String location) { - return applicationContext.getResource(location); - } - - @Override - public ClassLoader getClassLoader() { - return applicationContext.getClassLoader(); - } - - @Override - public void publishEvent(ApplicationEvent event) { - applicationContext.publishEvent(event); - } -} \ No newline at end of file diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/SdkException.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/SdkException.java new file mode 100644 index 00000000..3e03e174 --- /dev/null +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/SdkException.java @@ -0,0 +1,30 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.sdk; + +/** + * @Version 1.0.0 + * @Author AE86 + * @Date 2023-11-16 00:51 + */ +public class SdkException extends RuntimeException { + + private static final long serialVersionUID = 1L; + + public SdkException(String message) { + super(message); + } + + public SdkException(String message, Throwable cause) { + super(message, cause); + } + + public SdkException(Throwable cause) { + super(cause); + } + + protected SdkException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } +} \ No newline at end of file diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/enums/ModelEnum.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/ModelEnum.java similarity index 74% rename from dbsyncer-parser/src/main/java/org/dbsyncer/parser/enums/ModelEnum.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/ModelEnum.java index a196ab12..6b768a0a 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/enums/ModelEnum.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/ModelEnum.java @@ -1,14 +1,14 @@ -package org.dbsyncer.parser.enums; +package org.dbsyncer.sdk.enums; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.parser.ParserException; +import org.dbsyncer.sdk.SdkException; /** * 驱动同步方式枚举 * - * @author AE86 - * @version 1.0.0 - * @date 2020/04/21 16:19 + * @Version 1.0.0 + * @Author AE86 + * @Date 2023-11-16 00:51 */ public enum ModelEnum { @@ -29,13 +29,13 @@ public enum ModelEnum { this.name = name; } - public static ModelEnum getModelEnum(String code) throws ParserException { + public static ModelEnum getModelEnum(String code) throws SdkException { for (ModelEnum e : ModelEnum.values()) { if (StringUtil.equals(code, e.getCode())) { return e; } } - throw new ParserException(String.format("Model code \"%s\" does not exist.", code)); + throw new SdkException(String.format("Model code \"%s\" does not exist.", code)); } public static boolean isFull(String model) { diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/AbstractConvertContext.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/AbstractConvertContext.java index a61a3ec0..8cf761d6 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/AbstractConvertContext.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/AbstractConvertContext.java @@ -2,7 +2,6 @@ package org.dbsyncer.sdk.model; import org.dbsyncer.sdk.spi.ConnectorMapper; import org.dbsyncer.sdk.spi.ConvertContext; -import org.dbsyncer.sdk.spi.ProxyApplicationContext; import java.util.List; import java.util.Map; @@ -20,11 +19,6 @@ public abstract class AbstractConvertContext implements ConvertContext { */ private boolean terminated; - /** - * Spring上下文 - */ - protected ProxyApplicationContext context; - /** * 数据源连接实例 */ @@ -71,10 +65,6 @@ public abstract class AbstractConvertContext implements ConvertContext { this.targetList = targetList; } - public void setContext(ProxyApplicationContext context) { - this.context = context; - } - @Override public boolean isTerminated() { return terminated; @@ -85,11 +75,6 @@ public abstract class AbstractConvertContext implements ConvertContext { this.terminated = terminated; } - @Override - public ProxyApplicationContext getContext() { - return context; - } - @Override public ConnectorMapper getSourceConnectorMapper() { return sourceConnectorMapper; diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/ConvertContext.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/ConvertContext.java index b07a4a77..0bd9b54b 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/ConvertContext.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/ConvertContext.java @@ -27,11 +27,6 @@ public interface ConvertContext { */ void setTerminated(boolean terminated); - /** - * Spring上下文 - */ - ProxyApplicationContext getContext(); - /** * 数据源连接实例 */ diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/ProxyApplicationContext.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/ProxyApplicationContext.java deleted file mode 100644 index 239879af..00000000 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/ProxyApplicationContext.java +++ /dev/null @@ -1,13 +0,0 @@ -package org.dbsyncer.sdk.spi; - -import org.springframework.context.ApplicationContext; - -/** - * Spring上下文代理对象 - * - * @author AE86 - * @version 1.0.0 - * @date 2022/6/30 15:13 - */ -public interface ProxyApplicationContext extends ApplicationContext { -} \ No newline at end of file -- Gitee From bcc63f4328db619748baafc37ddf986e99aa50bb Mon Sep 17 00:00:00 2001 From: AE86 <836391306@qq.com> Date: Thu, 16 Nov 2023 01:08:43 +0800 Subject: [PATCH 09/98] fix package --- .../dbsyncer/biz/checker/AbstractChecker.java | 2 +- .../dbsyncer/biz/impl/PluginServiceImpl.java | 4 +- .../biz/impl/SystemConfigServiceImpl.java | 2 +- .../java/org/dbsyncer/biz/vo/PluginVo.java | 2 +- .../common}/enums/FileSuffixEnum.java | 94 +++++++++---------- .../{ => impl}/ScheduledTaskServiceImpl.java | 4 +- .../file/BufferedRandomAccessFile.java | 2 +- .../dbsyncer/listener/file/FileExtractor.java | 1 - .../parser/model/AbstractConfigModel.java | 2 +- .../plugin/NotifySupportConfiguration.java | 2 +- .../org/dbsyncer/plugin/PluginFactory.java | 2 +- .../DefaultNotifyService.java | 2 +- .../DemoConvertServiceImpl.java | 2 +- .../{notify => impl}/MailNotifyService.java | 2 +- .../plugin/{config => model}/Plugin.java | 2 +- 15 files changed, 63 insertions(+), 62 deletions(-) rename {dbsyncer-plugin/src/main/java/org/dbsyncer/plugin => dbsyncer-common/src/main/java/org/dbsyncer/common}/enums/FileSuffixEnum.java (90%) rename dbsyncer-common/src/main/java/org/dbsyncer/common/scheduled/{ => impl}/ScheduledTaskServiceImpl.java (94%) rename {dbsyncer-common/src/main/java/org/dbsyncer/common => dbsyncer-listener/src/main/java/org/dbsyncer/listener}/file/BufferedRandomAccessFile.java (99%) rename dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/{notify => impl}/DefaultNotifyService.java (89%) rename dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/{service => impl}/DemoConvertServiceImpl.java (97%) rename dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/{notify => impl}/MailNotifyService.java (99%) rename dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/{config => model}/Plugin.java (97%) diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/AbstractChecker.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/AbstractChecker.java index 5d57bbdd..e9d4c1cb 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/AbstractChecker.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/AbstractChecker.java @@ -10,7 +10,7 @@ import org.dbsyncer.parser.model.AbstractConfigModel; import org.dbsyncer.parser.model.ConfigModel; import org.dbsyncer.parser.model.Convert; import org.dbsyncer.plugin.PluginFactory; -import org.dbsyncer.plugin.config.Plugin; +import org.dbsyncer.plugin.model.Plugin; import org.dbsyncer.storage.constant.ConfigConstant; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/PluginServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/PluginServiceImpl.java index e23b1586..0f9efa02 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/PluginServiceImpl.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/PluginServiceImpl.java @@ -11,8 +11,8 @@ import org.dbsyncer.parser.LogType; import org.dbsyncer.parser.model.Mapping; import org.dbsyncer.parser.model.TableGroup; import org.dbsyncer.plugin.PluginFactory; -import org.dbsyncer.plugin.config.Plugin; -import org.dbsyncer.plugin.enums.FileSuffixEnum; +import org.dbsyncer.plugin.model.Plugin; +import org.dbsyncer.common.enums.FileSuffixEnum; import org.springframework.beans.BeanUtils; import org.springframework.stereotype.Component; import org.springframework.util.Assert; diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/SystemConfigServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/SystemConfigServiceImpl.java index 453e17c7..8ee8970b 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/SystemConfigServiceImpl.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/SystemConfigServiceImpl.java @@ -12,7 +12,7 @@ import org.dbsyncer.parser.LogType; import org.dbsyncer.parser.model.ConfigModel; import org.dbsyncer.parser.model.SystemConfig; import org.dbsyncer.manager.impl.PreloadTemplate; -import org.dbsyncer.plugin.enums.FileSuffixEnum; +import org.dbsyncer.common.enums.FileSuffixEnum; import org.springframework.beans.BeanUtils; import org.springframework.stereotype.Service; import org.springframework.util.Assert; diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/vo/PluginVo.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/vo/PluginVo.java index 6fa51a11..af37077e 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/vo/PluginVo.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/vo/PluginVo.java @@ -1,6 +1,6 @@ package org.dbsyncer.biz.vo; -import org.dbsyncer.plugin.config.Plugin; +import org.dbsyncer.plugin.model.Plugin; public class PluginVo extends Plugin { diff --git a/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/enums/FileSuffixEnum.java b/dbsyncer-common/src/main/java/org/dbsyncer/common/enums/FileSuffixEnum.java similarity index 90% rename from dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/enums/FileSuffixEnum.java rename to dbsyncer-common/src/main/java/org/dbsyncer/common/enums/FileSuffixEnum.java index c5bfc3a6..c197999b 100644 --- a/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/enums/FileSuffixEnum.java +++ b/dbsyncer-common/src/main/java/org/dbsyncer/common/enums/FileSuffixEnum.java @@ -1,48 +1,48 @@ -package org.dbsyncer.plugin.enums; - -import org.dbsyncer.common.util.StringUtil; - -/** - * 文件格式 - * - * @author AE86 - * @version 1.0.0 - * @date 2021/2/23 23:21 - */ -public enum FileSuffixEnum { - - /** - * jar - */ - JAR("jar"), - - /** - * JSON - */ - JSON("json"); - - private String name; - - FileSuffixEnum(String name) { - this.name = name; - } - - /** - * 获取文件类型 - * - * @param suffix - * @return - */ - public static FileSuffixEnum getFileSuffix(String suffix) { - for (FileSuffixEnum e : FileSuffixEnum.values()) { - if (StringUtil.equals(suffix, e.getName())) { - return e; - } - } - return null; - } - - public String getName() { - return name; - } +package org.dbsyncer.common.enums; + +import org.dbsyncer.common.util.StringUtil; + +/** + * 文件格式 + * + * @author AE86 + * @version 1.0.0 + * @date 2021/2/23 23:21 + */ +public enum FileSuffixEnum { + + /** + * jar + */ + JAR("jar"), + + /** + * JSON + */ + JSON("json"); + + private String name; + + FileSuffixEnum(String name) { + this.name = name; + } + + /** + * 获取文件类型 + * + * @param suffix + * @return + */ + public static FileSuffixEnum getFileSuffix(String suffix) { + for (FileSuffixEnum e : FileSuffixEnum.values()) { + if (StringUtil.equals(suffix, e.getName())) { + return e; + } + } + return null; + } + + public String getName() { + return name; + } } \ No newline at end of file diff --git a/dbsyncer-common/src/main/java/org/dbsyncer/common/scheduled/ScheduledTaskServiceImpl.java b/dbsyncer-common/src/main/java/org/dbsyncer/common/scheduled/impl/ScheduledTaskServiceImpl.java similarity index 94% rename from dbsyncer-common/src/main/java/org/dbsyncer/common/scheduled/ScheduledTaskServiceImpl.java rename to dbsyncer-common/src/main/java/org/dbsyncer/common/scheduled/impl/ScheduledTaskServiceImpl.java index d9ac61b1..96125f54 100644 --- a/dbsyncer-common/src/main/java/org/dbsyncer/common/scheduled/ScheduledTaskServiceImpl.java +++ b/dbsyncer-common/src/main/java/org/dbsyncer/common/scheduled/impl/ScheduledTaskServiceImpl.java @@ -1,6 +1,8 @@ -package org.dbsyncer.common.scheduled; +package org.dbsyncer.common.scheduled.impl; import org.dbsyncer.common.CommonException; +import org.dbsyncer.common.scheduled.ScheduledTaskJob; +import org.dbsyncer.common.scheduled.ScheduledTaskService; import org.dbsyncer.common.util.UUIDUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dbsyncer-common/src/main/java/org/dbsyncer/common/file/BufferedRandomAccessFile.java b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/file/BufferedRandomAccessFile.java similarity index 99% rename from dbsyncer-common/src/main/java/org/dbsyncer/common/file/BufferedRandomAccessFile.java rename to dbsyncer-listener/src/main/java/org/dbsyncer/listener/file/BufferedRandomAccessFile.java index 03270f86..6b560f28 100644 --- a/dbsyncer-common/src/main/java/org/dbsyncer/common/file/BufferedRandomAccessFile.java +++ b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/file/BufferedRandomAccessFile.java @@ -1,4 +1,4 @@ -package org.dbsyncer.common.file; +package org.dbsyncer.listener.file; import java.io.File; import java.io.FileNotFoundException; diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/file/FileExtractor.java b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/file/FileExtractor.java index 6e013b12..171d376f 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/file/FileExtractor.java +++ b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/file/FileExtractor.java @@ -3,7 +3,6 @@ package org.dbsyncer.listener.file; import org.apache.commons.io.IOUtils; import org.dbsyncer.listener.model.ChangedOffset; import org.dbsyncer.listener.event.RowChangedEvent; -import org.dbsyncer.common.file.BufferedRandomAccessFile; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.NumberUtil; import org.dbsyncer.common.util.StringUtil; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/AbstractConfigModel.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/AbstractConfigModel.java index cc38294d..9bfb4e2e 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/AbstractConfigModel.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/AbstractConfigModel.java @@ -1,7 +1,7 @@ package org.dbsyncer.parser.model; import org.dbsyncer.connector.model.Filter; -import org.dbsyncer.plugin.config.Plugin; +import org.dbsyncer.plugin.model.Plugin; import java.util.List; import java.util.Map; diff --git a/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/NotifySupportConfiguration.java b/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/NotifySupportConfiguration.java index cfbc4f12..fd80e923 100644 --- a/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/NotifySupportConfiguration.java +++ b/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/NotifySupportConfiguration.java @@ -1,6 +1,6 @@ package org.dbsyncer.plugin; -import org.dbsyncer.plugin.notify.DefaultNotifyService; +import org.dbsyncer.plugin.impl.DefaultNotifyService; import org.dbsyncer.sdk.spi.NotifyService; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.context.annotation.Bean; diff --git a/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/PluginFactory.java b/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/PluginFactory.java index be0b00f0..104bb661 100644 --- a/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/PluginFactory.java +++ b/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/PluginFactory.java @@ -2,7 +2,7 @@ package org.dbsyncer.plugin; import org.apache.commons.io.FileUtils; import org.dbsyncer.common.util.CollectionUtils; -import org.dbsyncer.plugin.config.Plugin; +import org.dbsyncer.plugin.model.Plugin; import org.dbsyncer.sdk.spi.ConvertContext; import org.dbsyncer.sdk.spi.ConvertService; import org.slf4j.Logger; diff --git a/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/notify/DefaultNotifyService.java b/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/impl/DefaultNotifyService.java similarity index 89% rename from dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/notify/DefaultNotifyService.java rename to dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/impl/DefaultNotifyService.java index 47b5cbe1..6dd8903e 100644 --- a/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/notify/DefaultNotifyService.java +++ b/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/impl/DefaultNotifyService.java @@ -1,4 +1,4 @@ -package org.dbsyncer.plugin.notify; +package org.dbsyncer.plugin.impl; import org.dbsyncer.sdk.model.NotifyMessage; import org.dbsyncer.sdk.spi.NotifyService; diff --git a/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/service/DemoConvertServiceImpl.java b/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/impl/DemoConvertServiceImpl.java similarity index 97% rename from dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/service/DemoConvertServiceImpl.java rename to dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/impl/DemoConvertServiceImpl.java index fcce323a..7af8df85 100644 --- a/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/service/DemoConvertServiceImpl.java +++ b/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/impl/DemoConvertServiceImpl.java @@ -1,4 +1,4 @@ -package org.dbsyncer.plugin.service; +package org.dbsyncer.plugin.impl; import org.dbsyncer.common.config.AppConfig; import org.dbsyncer.sdk.spi.ConvertContext; diff --git a/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/notify/MailNotifyService.java b/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/impl/MailNotifyService.java similarity index 99% rename from dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/notify/MailNotifyService.java rename to dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/impl/MailNotifyService.java index d78c4c84..a0a1a5b2 100644 --- a/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/notify/MailNotifyService.java +++ b/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/impl/MailNotifyService.java @@ -1,4 +1,4 @@ -package org.dbsyncer.plugin.notify; +package org.dbsyncer.plugin.impl; import org.dbsyncer.common.config.AppConfig; import org.dbsyncer.common.util.StringUtil; diff --git a/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/config/Plugin.java b/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/model/Plugin.java similarity index 97% rename from dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/config/Plugin.java rename to dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/model/Plugin.java index b49722c3..fa003be1 100644 --- a/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/config/Plugin.java +++ b/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/model/Plugin.java @@ -1,4 +1,4 @@ -package org.dbsyncer.plugin.config; +package org.dbsyncer.plugin.model; /** * @author AE86 -- Gitee From 3918facce6a67e8ef156884b9776ed7c874a1721 Mon Sep 17 00:00:00 2001 From: AE86 <836391306@qq.com> Date: Thu, 16 Nov 2023 01:31:52 +0800 Subject: [PATCH 10/98] fix package --- .../main/java/org/dbsyncer/biz/checker/AbstractChecker.java | 2 +- .../main/java/org/dbsyncer/biz/impl/MappingServiceImpl.java | 2 +- .../src/main/java/org/dbsyncer/biz/impl/MetricReporter.java | 5 ++--- .../main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java | 4 ++-- .../org/dbsyncer/connector}/scheduled/ScheduledTaskJob.java | 2 +- .../dbsyncer/connector}/scheduled/ScheduledTaskService.java | 2 +- .../connector}/scheduled/impl/ScheduledTaskServiceImpl.java | 6 +++--- .../main/java/org/dbsyncer/listener/AbstractExtractor.java | 2 +- .../dbsyncer/listener/quartz/AbstractQuartzExtractor.java | 2 +- .../java/org/dbsyncer/manager/impl/IncrementPuller.java | 4 ++-- .../org/dbsyncer/parser/flush/AbstractBufferActuator.java | 4 ++-- .../org/dbsyncer/parser/flush/impl/FlushServiceImpl.java | 2 +- .../parser/flush/impl/TableGroupBufferActuator.java | 2 +- .../main/java/org/dbsyncer/parser/impl/LogServiceImpl.java | 5 +++-- .../dbsyncer/parser/strategy/impl/DefaultGroupStrategy.java | 2 +- .../parser/strategy/impl/PreloadTableGroupStrategy.java | 2 +- .../dbsyncer/parser/strategy/impl/TableGroupStrategy.java | 2 +- .../java/org/dbsyncer/sdk/{ => client}/RemoteClient.java | 2 +- .../org/dbsyncer/storage/StorageSupportConfiguration.java | 2 +- .../org/dbsyncer/storage/binlog/AbstractBinlogService.java | 6 +++--- .../storage/{support => impl}/DiskStorageServiceImpl.java | 2 +- .../MySQLStorageServiceImpl.java} | 4 ++-- .../java/org/dbsyncer/storage/impl}/SnowflakeIdWorker.java | 2 +- .../dbsyncer/web/controller/config/ConfigController.java | 2 +- 24 files changed, 35 insertions(+), 35 deletions(-) rename {dbsyncer-common/src/main/java/org/dbsyncer/common => dbsyncer-connector/src/main/java/org/dbsyncer/connector}/scheduled/ScheduledTaskJob.java (74%) rename {dbsyncer-common/src/main/java/org/dbsyncer/common => dbsyncer-connector/src/main/java/org/dbsyncer/connector}/scheduled/ScheduledTaskService.java (94%) rename {dbsyncer-common/src/main/java/org/dbsyncer/common => dbsyncer-connector/src/main/java/org/dbsyncer/connector}/scheduled/impl/ScheduledTaskServiceImpl.java (94%) rename dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/{ => client}/RemoteClient.java (82%) rename dbsyncer-storage/src/main/java/org/dbsyncer/storage/{support => impl}/DiskStorageServiceImpl.java (99%) rename dbsyncer-storage/src/main/java/org/dbsyncer/storage/{support/MysqlStorageServiceImpl.java => impl/MySQLStorageServiceImpl.java} (99%) rename {dbsyncer-common/src/main/java/org/dbsyncer/common/snowflake => dbsyncer-storage/src/main/java/org/dbsyncer/storage/impl}/SnowflakeIdWorker.java (99%) diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/AbstractChecker.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/AbstractChecker.java index e9d4c1cb..556e5a6e 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/AbstractChecker.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/AbstractChecker.java @@ -1,7 +1,7 @@ package org.dbsyncer.biz.checker; import org.dbsyncer.biz.enums.SafeInfoEnum; -import org.dbsyncer.common.snowflake.SnowflakeIdWorker; +import org.dbsyncer.storage.impl.SnowflakeIdWorker; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.JsonUtil; import org.dbsyncer.common.util.StringUtil; diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MappingServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MappingServiceImpl.java index 1232a9eb..0ba9fa98 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MappingServiceImpl.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MappingServiceImpl.java @@ -9,7 +9,7 @@ import org.dbsyncer.biz.checker.impl.mapping.MappingChecker; import org.dbsyncer.biz.vo.ConnectorVo; import org.dbsyncer.biz.vo.MappingVo; import org.dbsyncer.biz.vo.MetaVo; -import org.dbsyncer.common.snowflake.SnowflakeIdWorker; +import org.dbsyncer.storage.impl.SnowflakeIdWorker; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.JsonUtil; import org.dbsyncer.common.util.StringUtil; diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MetricReporter.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MetricReporter.java index cfff49d0..b968916c 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MetricReporter.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MetricReporter.java @@ -1,7 +1,6 @@ package org.dbsyncer.biz.impl; import org.dbsyncer.biz.enums.BufferActuatorMetricEnum; -import org.dbsyncer.biz.enums.MetricEnum; import org.dbsyncer.biz.enums.StatisticEnum; import org.dbsyncer.biz.enums.ThreadPoolMetricEnum; import org.dbsyncer.biz.model.AppReportMetric; @@ -10,8 +9,8 @@ import org.dbsyncer.biz.model.MetricResponse; import org.dbsyncer.biz.model.MetricResponseInfo; import org.dbsyncer.biz.model.Sample; import org.dbsyncer.common.model.Paging; -import org.dbsyncer.common.scheduled.ScheduledTaskJob; -import org.dbsyncer.common.scheduled.ScheduledTaskService; +import org.dbsyncer.connector.scheduled.ScheduledTaskJob; +import org.dbsyncer.connector.scheduled.ScheduledTaskService; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.connector.constant.ConnectorConstant; diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java index 1aa2cc9c..0051b9b1 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java @@ -17,8 +17,8 @@ import org.dbsyncer.biz.vo.MetaVo; import org.dbsyncer.biz.vo.MetricResponseVo; import org.dbsyncer.manager.event.PreloadCompletedEvent; import org.dbsyncer.common.model.Paging; -import org.dbsyncer.common.scheduled.ScheduledTaskJob; -import org.dbsyncer.common.scheduled.ScheduledTaskService; +import org.dbsyncer.connector.scheduled.ScheduledTaskJob; +import org.dbsyncer.connector.scheduled.ScheduledTaskService; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.JsonUtil; import org.dbsyncer.common.util.NumberUtil; diff --git a/dbsyncer-common/src/main/java/org/dbsyncer/common/scheduled/ScheduledTaskJob.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/scheduled/ScheduledTaskJob.java similarity index 74% rename from dbsyncer-common/src/main/java/org/dbsyncer/common/scheduled/ScheduledTaskJob.java rename to dbsyncer-connector/src/main/java/org/dbsyncer/connector/scheduled/ScheduledTaskJob.java index 586c46c7..d00f8b98 100644 --- a/dbsyncer-common/src/main/java/org/dbsyncer/common/scheduled/ScheduledTaskJob.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/scheduled/ScheduledTaskJob.java @@ -1,4 +1,4 @@ -package org.dbsyncer.common.scheduled; +package org.dbsyncer.connector.scheduled; /** * @version 1.0.0 diff --git a/dbsyncer-common/src/main/java/org/dbsyncer/common/scheduled/ScheduledTaskService.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/scheduled/ScheduledTaskService.java similarity index 94% rename from dbsyncer-common/src/main/java/org/dbsyncer/common/scheduled/ScheduledTaskService.java rename to dbsyncer-connector/src/main/java/org/dbsyncer/connector/scheduled/ScheduledTaskService.java index 34caa6c0..35b6159a 100644 --- a/dbsyncer-common/src/main/java/org/dbsyncer/common/scheduled/ScheduledTaskService.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/scheduled/ScheduledTaskService.java @@ -1,4 +1,4 @@ -package org.dbsyncer.common.scheduled; +package org.dbsyncer.connector.scheduled; public interface ScheduledTaskService { diff --git a/dbsyncer-common/src/main/java/org/dbsyncer/common/scheduled/impl/ScheduledTaskServiceImpl.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/scheduled/impl/ScheduledTaskServiceImpl.java similarity index 94% rename from dbsyncer-common/src/main/java/org/dbsyncer/common/scheduled/impl/ScheduledTaskServiceImpl.java rename to dbsyncer-connector/src/main/java/org/dbsyncer/connector/scheduled/impl/ScheduledTaskServiceImpl.java index 96125f54..5184a95b 100644 --- a/dbsyncer-common/src/main/java/org/dbsyncer/common/scheduled/impl/ScheduledTaskServiceImpl.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/scheduled/impl/ScheduledTaskServiceImpl.java @@ -1,8 +1,8 @@ -package org.dbsyncer.common.scheduled.impl; +package org.dbsyncer.connector.scheduled.impl; import org.dbsyncer.common.CommonException; -import org.dbsyncer.common.scheduled.ScheduledTaskJob; -import org.dbsyncer.common.scheduled.ScheduledTaskService; +import org.dbsyncer.connector.scheduled.ScheduledTaskJob; +import org.dbsyncer.connector.scheduled.ScheduledTaskService; import org.dbsyncer.common.util.UUIDUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/AbstractExtractor.java b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/AbstractExtractor.java index 53af585d..61218f6c 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/AbstractExtractor.java +++ b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/AbstractExtractor.java @@ -1,7 +1,7 @@ package org.dbsyncer.listener; +import org.dbsyncer.connector.scheduled.ScheduledTaskService; import org.dbsyncer.listener.model.ChangedOffset; -import org.dbsyncer.common.scheduled.ScheduledTaskService; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.connector.ConnectorFactory; import org.dbsyncer.connector.constant.ConnectorConstant; diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/AbstractQuartzExtractor.java b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/AbstractQuartzExtractor.java index b137a1a9..1f559759 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/AbstractQuartzExtractor.java +++ b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/AbstractQuartzExtractor.java @@ -2,7 +2,7 @@ package org.dbsyncer.listener.quartz; import org.dbsyncer.listener.event.ScanChangedEvent; import org.dbsyncer.common.model.Result; -import org.dbsyncer.common.scheduled.ScheduledTaskJob; +import org.dbsyncer.connector.scheduled.ScheduledTaskJob; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.common.util.UUIDUtil; diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/IncrementPuller.java b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/IncrementPuller.java index e383c146..dcbff6d9 100644 --- a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/IncrementPuller.java +++ b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/IncrementPuller.java @@ -3,8 +3,8 @@ package org.dbsyncer.manager.impl; import org.dbsyncer.listener.model.ChangedOffset; import org.dbsyncer.manager.AbstractPuller; import org.dbsyncer.parser.event.RefreshOffsetEvent; -import org.dbsyncer.common.scheduled.ScheduledTaskJob; -import org.dbsyncer.common.scheduled.ScheduledTaskService; +import org.dbsyncer.connector.scheduled.ScheduledTaskJob; +import org.dbsyncer.connector.scheduled.ScheduledTaskService; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.connector.ConnectorFactory; import org.dbsyncer.connector.model.Table; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/AbstractBufferActuator.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/AbstractBufferActuator.java index fe8e3b2c..81b64334 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/AbstractBufferActuator.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/AbstractBufferActuator.java @@ -1,8 +1,8 @@ package org.dbsyncer.parser.flush; import org.dbsyncer.common.config.BufferActuatorConfig; -import org.dbsyncer.common.scheduled.ScheduledTaskJob; -import org.dbsyncer.common.scheduled.ScheduledTaskService; +import org.dbsyncer.connector.scheduled.ScheduledTaskJob; +import org.dbsyncer.connector.scheduled.ScheduledTaskService; import org.dbsyncer.parser.CacheService; import org.dbsyncer.parser.enums.MetaEnum; import org.dbsyncer.parser.model.Meta; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/FlushServiceImpl.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/FlushServiceImpl.java index 36d4b6ad..bf379b53 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/FlushServiceImpl.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/FlushServiceImpl.java @@ -1,7 +1,7 @@ package org.dbsyncer.parser.flush.impl; import org.dbsyncer.common.config.StorageConfig; -import org.dbsyncer.common.snowflake.SnowflakeIdWorker; +import org.dbsyncer.storage.impl.SnowflakeIdWorker; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.parser.flush.BufferActuator; import org.dbsyncer.parser.flush.FlushService; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/TableGroupBufferActuator.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/TableGroupBufferActuator.java index cef57d43..c2627c47 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/TableGroupBufferActuator.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/TableGroupBufferActuator.java @@ -1,7 +1,7 @@ package org.dbsyncer.parser.flush.impl; import org.dbsyncer.common.config.TableGroupBufferConfig; -import org.dbsyncer.common.scheduled.ScheduledTaskService; +import org.dbsyncer.connector.scheduled.ScheduledTaskService; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.common.util.ThreadPoolUtil; import org.dbsyncer.common.util.UUIDUtil; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/LogServiceImpl.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/LogServiceImpl.java index f658d34c..5e2d027a 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/LogServiceImpl.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/LogServiceImpl.java @@ -3,9 +3,10 @@ package org.dbsyncer.parser.impl; import org.dbsyncer.parser.LogService; import org.dbsyncer.parser.LogType; import org.dbsyncer.parser.flush.FlushService; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import javax.annotation.Resource; + /** * @version 1.0.0 * @Author AE86 @@ -14,7 +15,7 @@ import org.springframework.stereotype.Component; @Component public class LogServiceImpl implements LogService { - @Autowired + @Resource private FlushService flushService; @Override diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/impl/DefaultGroupStrategy.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/impl/DefaultGroupStrategy.java index a6aea0d2..25b2876b 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/impl/DefaultGroupStrategy.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/impl/DefaultGroupStrategy.java @@ -11,7 +11,7 @@ import org.dbsyncer.parser.strategy.GroupStrategy; * @version 1.0.0 * @date 2019/12/2 22:53 */ -public class DefaultGroupStrategy implements GroupStrategy { +public final class DefaultGroupStrategy implements GroupStrategy { @Override public String getGroupId(ConfigModel model) { diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/impl/PreloadTableGroupStrategy.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/impl/PreloadTableGroupStrategy.java index 446d958c..3dfa2086 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/impl/PreloadTableGroupStrategy.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/impl/PreloadTableGroupStrategy.java @@ -12,7 +12,7 @@ import org.dbsyncer.storage.constant.ConfigConstant; * @version 1.0.0 * @date 2019/12/2 22:53 */ -public class PreloadTableGroupStrategy implements GroupStrategy { +public final class PreloadTableGroupStrategy implements GroupStrategy { @Override public String getGroupId(Mapping model) { diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/impl/TableGroupStrategy.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/impl/TableGroupStrategy.java index 72acaa1b..c8f2ad2d 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/impl/TableGroupStrategy.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/impl/TableGroupStrategy.java @@ -12,7 +12,7 @@ import org.dbsyncer.storage.constant.ConfigConstant; * @version 1.0.0 * @date 2019/12/2 22:53 */ -public class TableGroupStrategy implements GroupStrategy { +public final class TableGroupStrategy implements GroupStrategy { @Override public String getGroupId(TableGroup model) { diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/RemoteClient.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/client/RemoteClient.java similarity index 82% rename from dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/RemoteClient.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/client/RemoteClient.java index dabc20c9..98e2b005 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/RemoteClient.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/client/RemoteClient.java @@ -1,7 +1,7 @@ /** * DBSyncer Copyright 2020-2023 All Rights Reserved. */ -package org.dbsyncer.sdk; +package org.dbsyncer.sdk.client; /** * @Version 1.0.0 diff --git a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/StorageSupportConfiguration.java b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/StorageSupportConfiguration.java index c1953832..7a64b66f 100644 --- a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/StorageSupportConfiguration.java +++ b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/StorageSupportConfiguration.java @@ -1,6 +1,6 @@ package org.dbsyncer.storage; -import org.dbsyncer.storage.support.DiskStorageServiceImpl; +import org.dbsyncer.storage.impl.DiskStorageServiceImpl; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; diff --git a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/binlog/AbstractBinlogService.java b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/binlog/AbstractBinlogService.java index e1489cbe..bd91617f 100644 --- a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/binlog/AbstractBinlogService.java +++ b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/binlog/AbstractBinlogService.java @@ -3,9 +3,9 @@ package org.dbsyncer.storage.binlog; import com.google.protobuf.InvalidProtocolBufferException; import org.dbsyncer.common.config.BinlogRecorderConfig; import org.dbsyncer.common.model.Paging; -import org.dbsyncer.common.scheduled.ScheduledTaskJob; -import org.dbsyncer.common.scheduled.ScheduledTaskService; -import org.dbsyncer.common.snowflake.SnowflakeIdWorker; +import org.dbsyncer.storage.impl.SnowflakeIdWorker; +import org.dbsyncer.connector.scheduled.ScheduledTaskJob; +import org.dbsyncer.connector.scheduled.ScheduledTaskService; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.connector.enums.FilterEnum; import org.dbsyncer.connector.enums.OperationEnum; diff --git a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/support/DiskStorageServiceImpl.java b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/impl/DiskStorageServiceImpl.java similarity index 99% rename from dbsyncer-storage/src/main/java/org/dbsyncer/storage/support/DiskStorageServiceImpl.java rename to dbsyncer-storage/src/main/java/org/dbsyncer/storage/impl/DiskStorageServiceImpl.java index 3232ce5e..bb3c0fac 100644 --- a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/support/DiskStorageServiceImpl.java +++ b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/impl/DiskStorageServiceImpl.java @@ -1,4 +1,4 @@ -package org.dbsyncer.storage.support; +package org.dbsyncer.storage.impl; import org.apache.commons.io.FileUtils; import org.apache.lucene.document.Document; diff --git a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/support/MysqlStorageServiceImpl.java b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/impl/MySQLStorageServiceImpl.java similarity index 99% rename from dbsyncer-storage/src/main/java/org/dbsyncer/storage/support/MysqlStorageServiceImpl.java rename to dbsyncer-storage/src/main/java/org/dbsyncer/storage/impl/MySQLStorageServiceImpl.java index db6d3069..264c1d25 100644 --- a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/support/MysqlStorageServiceImpl.java +++ b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/impl/MySQLStorageServiceImpl.java @@ -1,4 +1,4 @@ -package org.dbsyncer.storage.support; +package org.dbsyncer.storage.impl; import org.apache.commons.io.IOUtils; import org.dbsyncer.common.model.Paging; @@ -58,7 +58,7 @@ import java.util.stream.Stream; @Component @ConditionalOnProperty(value = "dbsyncer.storage.support.mysql.enabled", havingValue = "true") @ConfigurationProperties(prefix = "dbsyncer.storage.support.mysql") -public class MysqlStorageServiceImpl extends AbstractStorageService { +public class MySQLStorageServiceImpl extends AbstractStorageService { private final Logger logger = LoggerFactory.getLogger(getClass()); diff --git a/dbsyncer-common/src/main/java/org/dbsyncer/common/snowflake/SnowflakeIdWorker.java b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/impl/SnowflakeIdWorker.java similarity index 99% rename from dbsyncer-common/src/main/java/org/dbsyncer/common/snowflake/SnowflakeIdWorker.java rename to dbsyncer-storage/src/main/java/org/dbsyncer/storage/impl/SnowflakeIdWorker.java index 83d9e232..c84e05c6 100644 --- a/dbsyncer-common/src/main/java/org/dbsyncer/common/snowflake/SnowflakeIdWorker.java +++ b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/impl/SnowflakeIdWorker.java @@ -1,4 +1,4 @@ -package org.dbsyncer.common.snowflake; +package org.dbsyncer.storage.impl; import org.dbsyncer.common.CommonException; import org.springframework.boot.context.properties.ConfigurationProperties; diff --git a/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/config/ConfigController.java b/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/config/ConfigController.java index 4e0fabf9..339ce7a8 100644 --- a/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/config/ConfigController.java +++ b/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/config/ConfigController.java @@ -6,7 +6,7 @@ import org.dbsyncer.biz.SystemConfigService; import org.dbsyncer.biz.vo.RestResult; import org.dbsyncer.parser.CacheService; import org.dbsyncer.common.config.AppConfig; -import org.dbsyncer.common.snowflake.SnowflakeIdWorker; +import org.dbsyncer.storage.impl.SnowflakeIdWorker; import org.dbsyncer.common.util.JsonUtil; import org.dbsyncer.parser.LogService; import org.dbsyncer.parser.LogType; -- Gitee From 7d64ba976d41e365f92d2a56a6de411a8dbfcc28 Mon Sep 17 00:00:00 2001 From: AE86 <836391306@qq.com> Date: Thu, 16 Nov 2023 01:49:45 +0800 Subject: [PATCH 11/98] add open api --- .../dbsyncer/biz/impl/BaseServiceImpl.java | 9 +++--- .../storage/AbstractStorageService.java | 4 +-- .../org/dbsyncer/web/config/WebAppConfig.java | 4 +-- .../web/controller/BaseController.java | 8 ++--- .../controller/index/ConnectorController.java | 11 +++++-- .../controller/index/MappingController.java | 17 +++++++---- .../web/controller/index/MetaController.java | 5 ++-- .../index/ProjectGroupController.java | 14 +++++---- .../index/TableGroupController.java | 13 +++++--- .../OpenApiController.java} | 30 ++++++++++++------- .../controller/plugin/PluginController.java | 13 +++++--- .../controller/system/SystemController.java | 4 +-- .../web/controller/user/UserController.java | 4 +-- .../org/dbsyncer/web/remote/UserService.java | 12 -------- .../dbsyncer/web/remote/UserServiceImpl.java | 14 --------- 15 files changed, 85 insertions(+), 77 deletions(-) rename dbsyncer-web/src/main/java/org/dbsyncer/web/controller/{TestController.java => openapi/OpenApiController.java} (90%) delete mode 100644 dbsyncer-web/src/main/java/org/dbsyncer/web/remote/UserService.java delete mode 100644 dbsyncer-web/src/main/java/org/dbsyncer/web/remote/UserServiceImpl.java diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/BaseServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/BaseServiceImpl.java index 60f54387..0527076f 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/BaseServiceImpl.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/BaseServiceImpl.java @@ -1,16 +1,15 @@ package org.dbsyncer.biz.impl; -import org.dbsyncer.parser.ProfileComponent; -import org.dbsyncer.parser.enums.MetaEnum; -import org.dbsyncer.sdk.enums.ModelEnum; import org.dbsyncer.parser.LogService; import org.dbsyncer.parser.LogType; import org.dbsyncer.parser.MessageService; +import org.dbsyncer.parser.ProfileComponent; +import org.dbsyncer.parser.enums.MetaEnum; import org.dbsyncer.parser.model.ConfigModel; import org.dbsyncer.parser.model.Mapping; import org.dbsyncer.parser.model.Meta; import org.dbsyncer.parser.model.TableGroup; -import org.springframework.beans.factory.annotation.Autowired; +import org.dbsyncer.sdk.enums.ModelEnum; import org.springframework.util.Assert; import javax.annotation.Resource; @@ -23,7 +22,7 @@ public class BaseServiceImpl { @Resource private LogService logService; - @Autowired + @Resource private MessageService messageService; /** diff --git a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/AbstractStorageService.java b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/AbstractStorageService.java index f6634748..54ce5959 100644 --- a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/AbstractStorageService.java +++ b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/AbstractStorageService.java @@ -9,9 +9,9 @@ import org.dbsyncer.storage.strategy.Strategy; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.DisposableBean; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.util.Assert; +import javax.annotation.Resource; import java.io.File; import java.util.ArrayList; import java.util.List; @@ -29,7 +29,7 @@ public abstract class AbstractStorageService implements StorageService, Disposab private final Logger logger = LoggerFactory.getLogger(getClass()); - @Autowired + @Resource private Map map; private final Lock lock = new ReentrantLock(); diff --git a/dbsyncer-web/src/main/java/org/dbsyncer/web/config/WebAppConfig.java b/dbsyncer-web/src/main/java/org/dbsyncer/web/config/WebAppConfig.java index e16ce11c..ff9d5aff 100644 --- a/dbsyncer-web/src/main/java/org/dbsyncer/web/config/WebAppConfig.java +++ b/dbsyncer-web/src/main/java/org/dbsyncer/web/config/WebAppConfig.java @@ -8,7 +8,6 @@ import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.parser.model.UserInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.security.authentication.AuthenticationProvider; @@ -25,6 +24,7 @@ import org.springframework.security.web.authentication.AuthenticationFailureHand import org.springframework.security.web.authentication.SavedRequestAwareAuthenticationSuccessHandler; import org.springframework.security.web.authentication.logout.LogoutSuccessHandler; +import javax.annotation.Resource; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import javax.servlet.http.HttpSessionEvent; @@ -59,7 +59,7 @@ public class WebAppConfig extends WebSecurityConfigurerAdapter implements Authen */ private static final int MAXIMUM_SESSIONS = 1; - @Autowired + @Resource private UserConfigService userConfigService; diff --git a/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/BaseController.java b/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/BaseController.java index 96b39ade..3e1ac7f1 100644 --- a/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/BaseController.java +++ b/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/BaseController.java @@ -3,9 +3,9 @@ package org.dbsyncer.web.controller; import org.dbsyncer.biz.ConditionService; import org.dbsyncer.biz.ConvertService; import org.dbsyncer.biz.PluginService; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.ui.ModelMap; +import javax.annotation.Resource; import javax.servlet.http.HttpServletRequest; import java.util.HashMap; import java.util.Map; @@ -17,13 +17,13 @@ import java.util.Map; */ public abstract class BaseController { - @Autowired + @Resource private ConditionService filterService; - @Autowired + @Resource private ConvertService convertService; - @Autowired + @Resource private PluginService pluginService; /** diff --git a/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/index/ConnectorController.java b/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/index/ConnectorController.java index ec83b64a..a85a9c56 100644 --- a/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/index/ConnectorController.java +++ b/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/index/ConnectorController.java @@ -5,11 +5,16 @@ import org.dbsyncer.biz.vo.RestResult; import org.dbsyncer.web.controller.BaseController; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Controller; import org.springframework.ui.ModelMap; -import org.springframework.web.bind.annotation.*; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.ResponseBody; +import javax.annotation.Resource; import javax.servlet.http.HttpServletRequest; import java.util.Map; @@ -19,7 +24,7 @@ public class ConnectorController extends BaseController { private final Logger logger = LoggerFactory.getLogger(getClass()); - @Autowired + @Resource private ConnectorService connectorService; @GetMapping("/page/add") diff --git a/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/index/MappingController.java b/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/index/MappingController.java index 19192c57..10f2b979 100644 --- a/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/index/MappingController.java +++ b/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/index/MappingController.java @@ -7,11 +7,16 @@ import org.dbsyncer.biz.vo.RestResult; import org.dbsyncer.web.controller.BaseController; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Controller; import org.springframework.ui.ModelMap; -import org.springframework.web.bind.annotation.*; - +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.ResponseBody; + +import javax.annotation.Resource; import javax.servlet.http.HttpServletRequest; import java.util.Map; @@ -21,13 +26,13 @@ public class MappingController extends BaseController { private final Logger logger = LoggerFactory.getLogger(getClass()); - @Autowired + @Resource private ConnectorService connectorService; - @Autowired + @Resource private MappingService mappingService; - @Autowired + @Resource private TableGroupService tableGroupService; @GetMapping("/pageAdd") diff --git a/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/index/MetaController.java b/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/index/MetaController.java index 48b92c2c..5c1973ae 100644 --- a/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/index/MetaController.java +++ b/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/index/MetaController.java @@ -4,19 +4,20 @@ import org.dbsyncer.biz.MonitorService; import org.dbsyncer.biz.vo.RestResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.ResponseBody; +import javax.annotation.Resource; + @Controller @RequestMapping("/meta") public class MetaController { private final Logger logger = LoggerFactory.getLogger(getClass()); - @Autowired + @Resource private MonitorService monitorService; @GetMapping("/getAll") diff --git a/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/index/ProjectGroupController.java b/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/index/ProjectGroupController.java index e62ccd8a..58932897 100644 --- a/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/index/ProjectGroupController.java +++ b/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/index/ProjectGroupController.java @@ -7,11 +7,15 @@ import org.dbsyncer.biz.vo.RestResult; import org.dbsyncer.web.controller.BaseController; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Controller; import org.springframework.ui.ModelMap; -import org.springframework.web.bind.annotation.*; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.ResponseBody; +import javax.annotation.Resource; import javax.servlet.http.HttpServletRequest; import java.util.Map; @@ -27,13 +31,13 @@ public class ProjectGroupController extends BaseController { private final Logger logger = LoggerFactory.getLogger(getClass()); - @Autowired + @Resource private ProjectGroupService projectGroupService; - @Autowired + @Resource private ConnectorService connectorService; - @Autowired + @Resource private MappingService mappingService; @GetMapping("/page/add") diff --git a/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/index/TableGroupController.java b/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/index/TableGroupController.java index f10e369b..4acbf374 100644 --- a/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/index/TableGroupController.java +++ b/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/index/TableGroupController.java @@ -7,11 +7,16 @@ import org.dbsyncer.parser.model.TableGroup; import org.dbsyncer.web.controller.BaseController; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Controller; import org.springframework.ui.ModelMap; -import org.springframework.web.bind.annotation.*; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.ResponseBody; +import javax.annotation.Resource; import javax.servlet.http.HttpServletRequest; import java.util.Map; @@ -21,10 +26,10 @@ public class TableGroupController extends BaseController { private final Logger logger = LoggerFactory.getLogger(getClass()); - @Autowired + @Resource private TableGroupService tableGroupService; - @Autowired + @Resource private MappingService mappingService; @GetMapping("/page/{page}") diff --git a/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/TestController.java b/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/openapi/OpenApiController.java similarity index 90% rename from dbsyncer-web/src/main/java/org/dbsyncer/web/controller/TestController.java rename to dbsyncer-web/src/main/java/org/dbsyncer/web/controller/openapi/OpenApiController.java index deb8f9e7..cb341182 100644 --- a/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/TestController.java +++ b/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/openapi/OpenApiController.java @@ -1,5 +1,9 @@ -package org.dbsyncer.web.controller; +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.web.controller.openapi; +import org.dbsyncer.biz.vo.RestResult; import org.dbsyncer.common.util.RandomUtil; import org.dbsyncer.common.util.StringUtil; import org.slf4j.Logger; @@ -34,9 +38,16 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +/** + * OpenAPI + * + * @Version 1.0.0 + * @Author AE86 + * @Date 2023-11-16 01:38 + */ @Controller -@RequestMapping("/test") -public class TestController implements InitializingBean { +@RequestMapping("/gateway") +public class OpenApiController implements InitializingBean { private final Logger logger = LoggerFactory.getLogger(getClass()); @@ -52,18 +63,17 @@ public class TestController implements InitializingBean { private ParameterNameDiscoverer parameterNameDiscoverer = new DefaultParameterNameDiscoverer(); @ResponseBody - @RequestMapping("/demo") - public String demo(ModelMap modelMap, Long id, String version) { + @RequestMapping("/demo.json") + public RestResult demo(Long id, String version) { logger.info("id:{},version:{}", id, version); - modelMap.put("data", RandomUtil.nextInt(1, 100)); - return id + version; + return RestResult.restSuccess(RandomUtil.nextInt(1, 100)); } @ResponseBody - @RequestMapping("/adapter.json") + @RequestMapping("/api.json") public Object adapter(HttpServletRequest request, HttpServletResponse response) { try { - InvocableHandlerMethod invocableMethod = handlers.get("/test/demo"); + InvocableHandlerMethod invocableMethod = handlers.get("/gateway/demo.json"); // 模拟参数 Map params = new HashMap<>(); params.put("id", 1000L); @@ -100,7 +110,7 @@ public class TestController implements InitializingBean { } private void initHandlerMapping() { - parsePackage.put("/test/", ""); + parsePackage.put("/gateway/", ""); RequestMappingHandlerMapping mapping = applicationContext.getBean(RequestMappingHandlerMapping.class); // 获取url与类和方法的对应信息 Map map = mapping.getHandlerMethods(); diff --git a/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/plugin/PluginController.java b/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/plugin/PluginController.java index 0aa7ee4e..cc1eb201 100644 --- a/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/plugin/PluginController.java +++ b/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/plugin/PluginController.java @@ -8,7 +8,6 @@ import org.dbsyncer.common.config.AppConfig; import org.dbsyncer.common.util.JsonUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Controller; import org.springframework.ui.ModelMap; import org.springframework.web.bind.annotation.GetMapping; @@ -17,8 +16,14 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.multipart.MultipartFile; +import javax.annotation.Resource; import javax.servlet.http.HttpServletResponse; -import java.io.*; +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintWriter; @Controller @RequestMapping("/plugin") @@ -26,10 +31,10 @@ public class PluginController { private final Logger logger = LoggerFactory.getLogger(getClass()); - @Autowired + @Resource private PluginService pluginService; - @Autowired + @Resource private AppConfig appConfig; @RequestMapping("") diff --git a/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/system/SystemController.java b/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/system/SystemController.java index 742c92ce..49b8af37 100644 --- a/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/system/SystemController.java +++ b/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/system/SystemController.java @@ -5,7 +5,6 @@ import org.dbsyncer.biz.vo.RestResult; import org.dbsyncer.web.controller.BaseController; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Controller; import org.springframework.ui.ModelMap; import org.springframework.web.bind.annotation.GetMapping; @@ -13,6 +12,7 @@ import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.ResponseBody; +import javax.annotation.Resource; import javax.servlet.http.HttpServletRequest; import java.util.Map; @@ -22,7 +22,7 @@ public class SystemController extends BaseController { private final Logger logger = LoggerFactory.getLogger(getClass()); - @Autowired + @Resource private SystemConfigService systemConfigService; @RequestMapping("") diff --git a/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/user/UserController.java b/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/user/UserController.java index 291ac7cd..c8a995ba 100644 --- a/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/user/UserController.java +++ b/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/user/UserController.java @@ -6,7 +6,6 @@ import org.dbsyncer.biz.vo.UserInfoVo; import org.dbsyncer.web.controller.BaseController; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.security.core.Authentication; import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.stereotype.Controller; @@ -17,6 +16,7 @@ import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.ResponseBody; +import javax.annotation.Resource; import javax.servlet.http.HttpServletRequest; import java.util.Map; @@ -33,7 +33,7 @@ public class UserController extends BaseController { private final Logger logger = LoggerFactory.getLogger(getClass()); - @Autowired + @Resource private UserConfigService userConfigService; @RequestMapping("") diff --git a/dbsyncer-web/src/main/java/org/dbsyncer/web/remote/UserService.java b/dbsyncer-web/src/main/java/org/dbsyncer/web/remote/UserService.java deleted file mode 100644 index 25fe3629..00000000 --- a/dbsyncer-web/src/main/java/org/dbsyncer/web/remote/UserService.java +++ /dev/null @@ -1,12 +0,0 @@ -package org.dbsyncer.web.remote; - -public interface UserService { - - /** - * 远程调用接口 - * @param params - * @return - */ - String hello(String params); - -} \ No newline at end of file diff --git a/dbsyncer-web/src/main/java/org/dbsyncer/web/remote/UserServiceImpl.java b/dbsyncer-web/src/main/java/org/dbsyncer/web/remote/UserServiceImpl.java deleted file mode 100644 index 7654d85b..00000000 --- a/dbsyncer-web/src/main/java/org/dbsyncer/web/remote/UserServiceImpl.java +++ /dev/null @@ -1,14 +0,0 @@ -package org.dbsyncer.web.remote; - -import java.time.LocalDateTime; - -public class UserServiceImpl implements UserService { - - private String name = "def"; - - @Override - public String hello(String params) { - return name + LocalDateTime.now().toString(); - } - -} \ No newline at end of file -- Gitee From be11697cbc453adce1b783878d31c8f6296418ad Mon Sep 17 00:00:00 2001 From: AE86 <836391306@qq.com> Date: Fri, 17 Nov 2023 01:49:08 +0800 Subject: [PATCH 12/98] fix bug --- .../impl/connector/ConnectorChecker.java | 32 ++++--------------- .../biz/impl/ConnectorServiceImpl.java | 30 ++++++----------- .../biz/impl/DataSyncServiceImpl.java | 8 ++--- .../dbsyncer/connector/ConnectorFactory.java | 23 ++++++------- .../database/AbstractDatabaseConnector.java | 7 +--- .../connector/kafka/KafkaConnector.java | 6 +--- .../java/org/dbsyncer/parser/LogType.java | 1 - .../org/dbsyncer/parser/ParserComponent.java | 13 +------- .../parser/impl/ParserComponentImpl.java | 5 --- .../controller/openapi/OpenApiController.java | 1 - 10 files changed, 32 insertions(+), 94 deletions(-) diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/ConnectorChecker.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/ConnectorChecker.java index 0373902f..577f73ed 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/ConnectorChecker.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/ConnectorChecker.java @@ -6,11 +6,7 @@ import org.dbsyncer.biz.checker.ConnectorConfigChecker; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.connector.ConnectorFactory; import org.dbsyncer.connector.enums.ConnectorEnum; -import org.dbsyncer.connector.model.Table; -import org.dbsyncer.parser.ParserComponent; import org.dbsyncer.parser.ProfileComponent; -import org.dbsyncer.parser.LogService; -import org.dbsyncer.parser.LogType; import org.dbsyncer.parser.model.ConfigModel; import org.dbsyncer.parser.model.Connector; import org.dbsyncer.sdk.model.ConnectorConfig; @@ -22,7 +18,6 @@ import org.springframework.stereotype.Component; import org.springframework.util.Assert; import javax.annotation.Resource; -import java.util.List; import java.util.Map; /** @@ -35,15 +30,9 @@ public class ConnectorChecker extends AbstractChecker { private final Logger logger = LoggerFactory.getLogger(getClass()); - @Resource - private ParserComponent parserComponent; - @Resource private ProfileComponent profileComponent; - @Resource - private LogService logService; - @Resource private ConnectorFactory connectorFactory; @@ -70,7 +59,8 @@ public class ConnectorChecker extends AbstractChecker { checker.modify(config, params); // 获取表 - setTable(connector); + ConnectorMapper mapper = connectorFactory.connect(connector.getConfig()); + connector.setTable(connectorFactory.getTable(mapper)); // 修改基本配置 this.modifyConfigModel(connector, params); @@ -85,19 +75,21 @@ public class ConnectorChecker extends AbstractChecker { String id = params.get(ConfigConstant.CONFIG_MODEL_ID); Connector connector = profileComponent.getConnector(id); Assert.notNull(connector, "Can not find connector."); + ConnectorConfig config = connector.getConfig(); + connectorFactory.disconnect(config); // 修改基本配置 this.modifyConfigModel(connector, params); // 配置连接器配置 - ConnectorConfig config = connector.getConfig(); String type = StringUtil.toLowerCaseFirstOne(config.getConnectorType()).concat("ConfigChecker"); ConnectorConfigChecker checker = map.get(type); Assert.notNull(checker, "Checker can not be null."); checker.modify(config, params); // 获取表 - setTable(connector); + ConnectorMapper mapper = connectorFactory.connect(config); + connector.setTable(connectorFactory.getTable(mapper)); return connector; } @@ -113,16 +105,4 @@ public class ConnectorChecker extends AbstractChecker { } } - private void setTable(Connector connector) { - boolean isAlive = connectorFactory.isAlive(connector.getConfig()); - if (!isAlive) { - logService.log(LogType.ConnectorLog.FAILED); - } - Assert.isTrue(isAlive, "无法连接."); - // 获取表信息 - ConnectorMapper connectorMapper = connectorFactory.connect(connector.getConfig()); - List
    table = parserComponent.getTable(connectorMapper); - connector.setTable(table); - } - } \ No newline at end of file diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ConnectorServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ConnectorServiceImpl.java index b2a43182..2c7d331c 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ConnectorServiceImpl.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ConnectorServiceImpl.java @@ -7,9 +7,9 @@ import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.JsonUtil; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.connector.ConnectorFactory; -import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.parser.LogService; import org.dbsyncer.parser.LogType; +import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.parser.model.ConfigModel; import org.dbsyncer.parser.model.Connector; import org.dbsyncer.parser.model.Mapping; @@ -98,9 +98,11 @@ public class ConnectorServiceImpl extends BaseServiceImpl implements ConnectorSe } Connector connector = profileComponent.getConnector(id); - log(LogType.ConnectorLog.DELETE, connector); - - profileComponent.removeConfigModel(id); + if (connector != null) { + connectorFactory.disconnect(connector.getConfig()); + log(LogType.ConnectorLog.DELETE, connector); + profileComponent.removeConfigModel(id); + } return "删除连接器成功!"; } @@ -138,7 +140,7 @@ public class ConnectorServiceImpl extends BaseServiceImpl implements ConnectorSe // 更新连接器状态 Set exist = new HashSet<>(); list.forEach(c -> { - health.put(c.getId(), isAliveConnectorConfig(c.getConfig())); + health.put(c.getId(), isAlive(c.getConfig())); exist.add(c.getId()); }); @@ -160,26 +162,14 @@ public class ConnectorServiceImpl extends BaseServiceImpl implements ConnectorSe return health.containsKey(id) && health.get(id); } - private boolean isAliveConnectorConfig(ConnectorConfig config) { - boolean alive = false; + private boolean isAlive(ConnectorConfig config) { try { - alive = connectorFactory.isAlive(config); + return connectorFactory.isAlive(config); } catch (Exception e) { LogType.ConnectorLog logType = LogType.ConnectorLog.FAILED; logService.log(logType, "%s%s", logType.getName(), e.getMessage()); + return false; } - // 断线重连 - if (!alive) { - try { - alive = connectorFactory.refresh(config); - } catch (Exception e) { - logger.error(e.getMessage()); - } - if (alive) { - logger.info(LogType.ConnectorLog.RECONNECT_SUCCESS.getMessage()); - } - } - return alive; } } \ No newline at end of file diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/DataSyncServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/DataSyncServiceImpl.java index 99cc6039..d8c507aa 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/DataSyncServiceImpl.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/DataSyncServiceImpl.java @@ -4,7 +4,6 @@ import com.google.protobuf.InvalidProtocolBufferException; import org.dbsyncer.biz.DataSyncService; import org.dbsyncer.biz.vo.BinlogColumnVo; import org.dbsyncer.biz.vo.MessageVo; -import org.dbsyncer.listener.event.RowChangedEvent; import org.dbsyncer.common.model.Paging; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.DateFormatUtil; @@ -12,8 +11,9 @@ import org.dbsyncer.common.util.JsonUtil; import org.dbsyncer.common.util.NumberUtil; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.connector.model.Field; -import org.dbsyncer.parser.ParserComponent; +import org.dbsyncer.listener.event.RowChangedEvent; import org.dbsyncer.parser.ProfileComponent; +import org.dbsyncer.parser.flush.impl.BufferActuatorRouter; import org.dbsyncer.parser.model.Meta; import org.dbsyncer.parser.model.Picker; import org.dbsyncer.parser.model.TableGroup; @@ -53,7 +53,7 @@ public class DataSyncServiceImpl implements DataSyncService { private final Logger logger = LoggerFactory.getLogger(getClass()); @Resource - private ParserComponent parserComponent; + private BufferActuatorRouter bufferActuatorRouter; @Resource private ProfileComponent profileComponent; @@ -164,7 +164,7 @@ public class DataSyncServiceImpl implements DataSyncService { // 转换为源字段 final Picker picker = new Picker(tableGroup.getFieldMapping()); changedEvent.setChangedRow(picker.pickSourceData(binlogData)); - parserComponent.execute(tableGroupId, changedEvent); + bufferActuatorRouter.execute(metaId, tableGroupId, changedEvent); storageService.remove(StorageEnum.DATA, metaId, messageId); // 更新失败数 Meta meta = profileComponent.getMeta(metaId); diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/ConnectorFactory.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/ConnectorFactory.java index bcafc6df..9d44b7f4 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/ConnectorFactory.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/ConnectorFactory.java @@ -48,12 +48,14 @@ public class ConnectorFactory implements DisposableBean { if (!connectorCache.containsKey(cacheKey)) { synchronized (connectorCache) { if (!connectorCache.containsKey(cacheKey)) { - connectorCache.putIfAbsent(cacheKey, connector.connect(config)); + ConnectorMapper mapper = connector.connect(config); + Assert.isTrue(connector.isAlive(mapper), "连接配置异常"); + connectorCache.putIfAbsent(cacheKey, mapper); } } } - ConnectorMapper connectorMapper = connectorCache.get(cacheKey); try { + ConnectorMapper connectorMapper = connectorCache.get(cacheKey); ConnectorMapper clone = (ConnectorMapper) connectorMapper.clone(); clone.setConfig(config); return clone; @@ -63,21 +65,20 @@ public class ConnectorFactory implements DisposableBean { } /** - * 刷新连接配置 + * 断开连接 * * @param config * @return */ - public boolean refresh(ConnectorConfig config) { + public void disconnect(ConnectorConfig config) { Assert.notNull(config, "ConnectorConfig can not be null."); Connector connector = getConnector(config.getConnectorType()); String cacheKey = connector.getConnectorMapperCacheKey(config); - if (connectorCache.containsKey(cacheKey)) { - disconnect(connectorCache.get(cacheKey)); + ConnectorMapper connectorMapper = connectorCache.get(cacheKey); + if (connectorMapper != null) { + disconnect(connectorMapper); connectorCache.remove(cacheKey); } - connect(config); - return connector.isAlive(connectorCache.get(cacheKey)); } /** @@ -184,12 +185,6 @@ public class ConnectorFactory implements DisposableBean { return getConnector(connectorConfig.getConnectorType()); } - /** - * 获取连接器 - * - * @param connectorType - * @return - */ public Connector getConnector(String connectorType) { return ConnectorEnum.getConnectorEnum(connectorType).getConnector(); } diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/AbstractDatabaseConnector.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/AbstractDatabaseConnector.java index fd24e8ca..957960a1 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/AbstractDatabaseConnector.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/AbstractDatabaseConnector.java @@ -59,12 +59,7 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector implem @Override public ConnectorMapper connect(DatabaseConfig config) { - try { - return new DatabaseConnectorMapper(config); - } catch (Exception e) { - logger.error("Failed to connect:{}, message:{}", config.getUrl(), e.getMessage()); - } - throw new ConnectorException(String.format("Failed to connect:%s", config.getUrl())); + return new DatabaseConnectorMapper(config); } @Override diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/kafka/KafkaConnector.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/kafka/KafkaConnector.java index 049aa33d..2510da32 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/kafka/KafkaConnector.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/kafka/KafkaConnector.java @@ -29,11 +29,7 @@ public class KafkaConnector extends AbstractConnector implements Connector getTable(ConnectorMapper config); - /** * 获取表元信息 * diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/ParserComponentImpl.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/ParserComponentImpl.java index 70831463..2e47aebd 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/ParserComponentImpl.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/ParserComponentImpl.java @@ -75,11 +75,6 @@ public class ParserComponentImpl implements ParserComponent { @Resource private BufferActuator generalBufferActuator; - @Override - public List
    getTable(ConnectorMapper config) { - return connectorFactory.getTable(config); - } - @Override public MetaInfo getMetaInfo(String connectorId, String tableName) { Connector connector = profileComponent.getConnector(connectorId); diff --git a/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/openapi/OpenApiController.java b/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/openapi/OpenApiController.java index cb341182..f847dd71 100644 --- a/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/openapi/OpenApiController.java +++ b/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/openapi/OpenApiController.java @@ -13,7 +13,6 @@ import org.springframework.context.ApplicationContext; import org.springframework.core.DefaultParameterNameDiscoverer; import org.springframework.core.ParameterNameDiscoverer; import org.springframework.stereotype.Controller; -import org.springframework.ui.ModelMap; import org.springframework.util.ObjectUtils; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.ResponseBody; -- Gitee From da6cb12ac694fd90afe83bc6abeee134c0d49461 Mon Sep 17 00:00:00 2001 From: AE86 <836391306@qq.com> Date: Fri, 17 Nov 2023 02:11:26 +0800 Subject: [PATCH 13/98] fix package --- .../org/dbsyncer/parser/ParserComponent.java | 9 ---- .../parser/flush/AbstractBufferActuator.java | 8 ++-- .../parser/flush/AbstractFlushStrategy.java | 6 +-- .../flush/impl/GeneralBufferActuator.java | 46 ++++--------------- .../parser/impl/ParserComponentImpl.java | 11 ----- 5 files changed, 15 insertions(+), 65 deletions(-) diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ParserComponent.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ParserComponent.java index dc0ce2bf..4c10d740 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ParserComponent.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ParserComponent.java @@ -2,7 +2,6 @@ package org.dbsyncer.parser; import org.dbsyncer.common.model.Result; import org.dbsyncer.connector.model.MetaInfo; -import org.dbsyncer.listener.ChangedEvent; import org.dbsyncer.parser.model.BatchWriter; import org.dbsyncer.parser.model.Mapping; import org.dbsyncer.parser.model.TableGroup; @@ -60,14 +59,6 @@ public interface ParserComponent { */ void execute(Task task, Mapping mapping, TableGroup tableGroup, Executor executor); - /** - * 增量同步 - * - * @param tableGroupId 表关系ID - * @param changedEvent 增量事件 - */ - void execute(String tableGroupId, ChangedEvent changedEvent); - /** * 批执行 * diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/AbstractBufferActuator.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/AbstractBufferActuator.java index 81b64334..9c35277d 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/AbstractBufferActuator.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/AbstractBufferActuator.java @@ -3,7 +3,7 @@ package org.dbsyncer.parser.flush; import org.dbsyncer.common.config.BufferActuatorConfig; import org.dbsyncer.connector.scheduled.ScheduledTaskJob; import org.dbsyncer.connector.scheduled.ScheduledTaskService; -import org.dbsyncer.parser.CacheService; +import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.parser.enums.MetaEnum; import org.dbsyncer.parser.model.Meta; import org.slf4j.Logger; @@ -50,7 +50,7 @@ public abstract class AbstractBufferActuator params = ConfigModelUtil.convertModelToMap(model); - logger.debug("params:{}", params); - storageService.edit(StorageEnum.CONFIG, params); - - // 3、缓存 - Assert.notNull(model, "ConfigModel can not be null."); - cacheService.put(model.getId(), model); - } - public void setGeneralExecutor(Executor generalExecutor) { this.generalExecutor = generalExecutor; } diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/ParserComponentImpl.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/ParserComponentImpl.java index 2e47aebd..8a2c0586 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/ParserComponentImpl.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/ParserComponentImpl.java @@ -1,6 +1,5 @@ package org.dbsyncer.parser.impl; -import org.dbsyncer.listener.ChangedEvent; import org.dbsyncer.common.model.Result; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.StringUtil; @@ -16,7 +15,6 @@ import org.dbsyncer.connector.util.PrimaryKeyUtil; import org.dbsyncer.parser.ParserComponent; import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.parser.event.FullRefreshEvent; -import org.dbsyncer.parser.flush.BufferActuator; import org.dbsyncer.parser.model.BatchWriter; import org.dbsyncer.parser.model.Connector; import org.dbsyncer.parser.model.FieldMapping; @@ -24,7 +22,6 @@ import org.dbsyncer.parser.model.Mapping; import org.dbsyncer.parser.model.Picker; import org.dbsyncer.parser.model.TableGroup; import org.dbsyncer.parser.model.Task; -import org.dbsyncer.parser.model.WriterRequest; import org.dbsyncer.parser.strategy.FlushStrategy; import org.dbsyncer.parser.util.ConvertUtil; import org.dbsyncer.parser.util.PickerUtil; @@ -72,9 +69,6 @@ public class ParserComponentImpl implements ParserComponent { @Resource private ApplicationContext applicationContext; - @Resource - private BufferActuator generalBufferActuator; - @Override public MetaInfo getMetaInfo(String connectorId, String tableName) { Connector connector = profileComponent.getConnector(connectorId); @@ -201,11 +195,6 @@ public class ParserComponentImpl implements ParserComponent { } } - @Override - public void execute(String tableGroupId, ChangedEvent event) { - generalBufferActuator.offer(new WriterRequest(tableGroupId, event)); - } - @Override public Result writeBatch(ConvertContext context, BatchWriter batchWriter, Executor executor) { final Result result = new Result(); -- Gitee From cb48612fe3b44d5b298b9422f03c757acadb9344 Mon Sep 17 00:00:00 2001 From: AE86 <836391306@qq.com> Date: Fri, 17 Nov 2023 10:37:01 +0000 Subject: [PATCH 14/98] simple code --- .../dbsyncer/biz/impl/MonitorServiceImpl.java | 36 ++++----- .../database/DatabaseConnectorMapper.java | 7 +- .../connector/es/ESConnectorMapper.java | 5 ++ .../connector/file/FileConnectorMapper.java | 7 +- .../connector/kafka/KafkaConnectorMapper.java | 7 +- .../manager/event/PreloadCompletedEvent.java | 1 + .../manager/impl/PreloadTemplate.java | 76 ++++++++++++++----- .../org/dbsyncer/sdk/spi/ConnectorMapper.java | 7 ++ .../controller/monitor/MonitorController.java | 12 ++- 9 files changed, 111 insertions(+), 47 deletions(-) diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java index 0051b9b1..c59a3f8e 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java @@ -3,6 +3,9 @@ package org.dbsyncer.biz.impl; import org.dbsyncer.biz.DataSyncService; import org.dbsyncer.biz.MonitorService; import org.dbsyncer.biz.SystemConfigService; +import org.dbsyncer.biz.enums.BufferActuatorMetricEnum; +import org.dbsyncer.biz.enums.DiskMetricEnum; +import org.dbsyncer.biz.enums.MetricEnum; import org.dbsyncer.biz.metric.MetricDetailFormatter; import org.dbsyncer.biz.metric.impl.CpuMetricDetailFormatter; import org.dbsyncer.biz.metric.impl.DiskMetricDetailFormatter; @@ -10,32 +13,28 @@ import org.dbsyncer.biz.metric.impl.DoubleRoundMetricDetailFormatter; import org.dbsyncer.biz.metric.impl.GCMetricDetailFormatter; import org.dbsyncer.biz.metric.impl.MemoryMetricDetailFormatter; import org.dbsyncer.biz.metric.impl.ValueMetricDetailFormatter; +import org.dbsyncer.biz.model.AppReportMetric; +import org.dbsyncer.biz.model.MetricResponse; import org.dbsyncer.biz.vo.AppReportMetricVo; import org.dbsyncer.biz.vo.DataVo; import org.dbsyncer.biz.vo.LogVo; import org.dbsyncer.biz.vo.MetaVo; import org.dbsyncer.biz.vo.MetricResponseVo; -import org.dbsyncer.manager.event.PreloadCompletedEvent; import org.dbsyncer.common.model.Paging; -import org.dbsyncer.connector.scheduled.ScheduledTaskJob; -import org.dbsyncer.connector.scheduled.ScheduledTaskService; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.JsonUtil; import org.dbsyncer.common.util.NumberUtil; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.connector.enums.FilterEnum; -import org.dbsyncer.biz.enums.BufferActuatorMetricEnum; -import org.dbsyncer.biz.enums.DiskMetricEnum; -import org.dbsyncer.biz.enums.MetricEnum; -import org.dbsyncer.biz.model.AppReportMetric; -import org.dbsyncer.biz.model.MetricResponse; -import org.dbsyncer.parser.ProfileComponent; -import org.dbsyncer.parser.enums.MetaEnum; -import org.dbsyncer.sdk.enums.ModelEnum; +import org.dbsyncer.connector.scheduled.ScheduledTaskJob; +import org.dbsyncer.connector.scheduled.ScheduledTaskService; import org.dbsyncer.parser.LogService; import org.dbsyncer.parser.LogType; +import org.dbsyncer.parser.ProfileComponent; +import org.dbsyncer.parser.enums.MetaEnum; import org.dbsyncer.parser.model.Mapping; import org.dbsyncer.parser.model.Meta; +import org.dbsyncer.sdk.enums.ModelEnum; import org.dbsyncer.storage.StorageService; import org.dbsyncer.storage.constant.ConfigConstant; import org.dbsyncer.storage.enums.IndexFieldResolverEnum; @@ -47,7 +46,6 @@ import org.dbsyncer.storage.query.filter.LongFilter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.BeanUtils; -import org.springframework.context.ApplicationListener; import org.springframework.stereotype.Service; import org.springframework.util.Assert; @@ -69,7 +67,7 @@ import java.util.stream.Collectors; * @date 2020/04/27 10:20 */ @Service -public class MonitorServiceImpl extends BaseServiceImpl implements MonitorService, ScheduledTaskJob, ApplicationListener { +public class MonitorServiceImpl extends BaseServiceImpl implements MonitorService, ScheduledTaskJob { private final Logger logger = LoggerFactory.getLogger(getClass()); @@ -94,8 +92,6 @@ public class MonitorServiceImpl extends BaseServiceImpl implements MonitorServic @Resource private SystemConfigService systemConfigService; - private boolean preloadCompleted; - private Map metricDetailFormatterMap = new LinkedHashMap<>(); @PostConstruct @@ -205,10 +201,8 @@ public class MonitorServiceImpl extends BaseServiceImpl implements MonitorServic @Override public void deleteExpiredDataAndLog() { - if (preloadCompleted) { - deleteExpiredData(); - deleteExpiredLog(); - } + deleteExpiredData(); + deleteExpiredLog(); } @Override @@ -350,8 +344,4 @@ public class MonitorServiceImpl extends BaseServiceImpl implements MonitorServic }).collect(Collectors.toList()); } - @Override - public void onApplicationEvent(PreloadCompletedEvent event) { - preloadCompleted = true; - } } \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/DatabaseConnectorMapper.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/DatabaseConnectorMapper.java index 7a63d70a..7e03bebf 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/DatabaseConnectorMapper.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/DatabaseConnectorMapper.java @@ -36,6 +36,11 @@ public class DatabaseConnectorMapper implements ConnectorMapper reload(map, commandEnum)); + + // Load connectorMappers + loadConnectorMapper(); + + // Launch drivers launch(); } @@ -140,7 +160,7 @@ public final class PreloadTemplate implements ApplicationListener map, CommandEnum commandEnum) { @@ -172,4 +192,22 @@ public final class PreloadTemplate implements ApplicationListener list = profileComponent.getConnectorAll(); + if (!CollectionUtils.isEmpty(list)) { + list.forEach(connector -> { + generalExecutor.execute(() -> { + try { + connectorFactory.disconnect(connector.getConfig()); + ConnectorMapper mapper = connectorFactory.connect(connector.getConfig()); + logger.info("Completed connection {} {}", connector.getConfig().getConnectorType(), mapper.getServiceUrl()); + } catch (Exception e) { + logger.error("连接配置异常", e); + logService.log(LogType.ConnectorLog.FAILED, e.getMessage()); + } + }); + }); + } + } } \ No newline at end of file diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/ConnectorMapper.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/ConnectorMapper.java index f40a093b..bb7e7e66 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/ConnectorMapper.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/ConnectorMapper.java @@ -11,6 +11,13 @@ package org.dbsyncer.sdk.spi; */ public interface ConnectorMapper extends Cloneable { + /** + * 获取服务地址 + * + * @return + */ + String getServiceUrl(); + /** * 获取连接配置 * diff --git a/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/monitor/MonitorController.java b/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/monitor/MonitorController.java index fab2c423..5211b256 100644 --- a/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/monitor/MonitorController.java +++ b/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/monitor/MonitorController.java @@ -15,6 +15,7 @@ import org.dbsyncer.biz.enums.MetricEnum; import org.dbsyncer.biz.enums.StatisticEnum; import org.dbsyncer.biz.model.MetricResponse; import org.dbsyncer.biz.model.Sample; +import org.dbsyncer.manager.impl.PreloadTemplate; import org.dbsyncer.web.controller.BaseController; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -60,6 +61,9 @@ public class MonitorController extends BaseController { @Resource private SystemConfigService systemConfigService; + @Resource + private PreloadTemplate preloadTemplate; + @Resource private MetricsEndpoint metricsEndpoint; @@ -98,12 +102,16 @@ public class MonitorController extends BaseController { @Scheduled(fixedRate = 10000) public void refreshConnectorHealth() { - connectorService.refreshHealth(); + if (preloadTemplate.isPreloadCompleted()) { + connectorService.refreshHealth(); + } } @Scheduled(fixedRate = 30000) public void deleteExpiredDataAndLog() { - monitorService.deleteExpiredDataAndLog(); + if (preloadTemplate.isPreloadCompleted()) { + monitorService.deleteExpiredDataAndLog(); + } } @GetMapping("/queryData") -- Gitee From 8ce766d088c3b7ef3152c48861db4da5db20f9a8 Mon Sep 17 00:00:00 2001 From: AE86 <836391306@qq.com> Date: Sat, 18 Nov 2023 01:01:11 +0800 Subject: [PATCH 15/98] rm binlog --- .../sqlserver/SqlServerConnector.java | 21 -- .../manager/event/PreloadCompletedEvent.java | 25 -- .../storage/binlog/AbstractBinlogService.java | 233 ------------------ .../storage/binlog/BinlogRecorder.java | 37 --- .../dbsyncer/storage/enums/StorageEnum.java | 7 +- .../src/main/resources/dbsyncer_binlog.sql | 8 - .../src/main/resources/dbsyncer_upgrade.sql | 2 - 7 files changed, 1 insertion(+), 332 deletions(-) delete mode 100644 dbsyncer-manager/src/main/java/org/dbsyncer/manager/event/PreloadCompletedEvent.java delete mode 100644 dbsyncer-storage/src/main/java/org/dbsyncer/storage/binlog/AbstractBinlogService.java delete mode 100644 dbsyncer-storage/src/main/java/org/dbsyncer/storage/binlog/BinlogRecorder.java delete mode 100644 dbsyncer-storage/src/main/resources/dbsyncer_binlog.sql delete mode 100644 dbsyncer-storage/src/main/resources/dbsyncer_upgrade.sql diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/SqlServerConnector.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/SqlServerConnector.java index 445d4752..a00c6545 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/SqlServerConnector.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/SqlServerConnector.java @@ -25,11 +25,6 @@ public final class SqlServerConnector extends AbstractDatabaseConnector { private static final String QUERY_TABLE = "select name from sys.tables where schema_id = schema_id('%s') and is_ms_shipped = 0"; - /** - * 系统函数表达式convert/varchar/getdate - */ - private final String SYS_EXPRESSION = "(convert\\().+?(\\))|(varchar\\().+?(\\))|(getdate\\(\\))"; - @Override public List
    getTable(DatabaseConnectorMapper connectorMapper) { DatabaseConfig config = connectorMapper.getConfig(); @@ -52,22 +47,6 @@ public final class SqlServerConnector extends AbstractDatabaseConnector { return new Object[] {(pageIndex - 1) * pageSize + 1, pageIndex * pageSize}; } - /** - * TODO 待废弃 推荐使用系统参数表达式$xxx$ - * - * @param value - * @return - */ - @Deprecated - @Override - public String buildFilterValue(String value) { - // 支持SqlServer系统函数, Example: (select CONVERT(varchar(10),GETDATE(),120)) - if (containsKeyword(SYS_EXPRESSION, value)) { - return value; - } - return super.buildFilterValue(value); - } - @Override public String buildTableName(String tableName) { return convertKey(tableName); diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/event/PreloadCompletedEvent.java b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/event/PreloadCompletedEvent.java deleted file mode 100644 index 762d4df7..00000000 --- a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/event/PreloadCompletedEvent.java +++ /dev/null @@ -1,25 +0,0 @@ -package org.dbsyncer.manager.event; - -import org.springframework.context.ApplicationContext; -import org.springframework.context.event.ApplicationContextEvent; - -/** - * 预加载完成事件 - * - * @version 1.0.0 - * @Author AE86 - * @Date 2020-08-26 22:45 - */ -@Deprecated -public final class PreloadCompletedEvent extends ApplicationContextEvent { - - /** - * Create a new ContextStartedEvent. - * - * @param source the {@code ApplicationContext} that the event is raised for (must not be {@code null}) - */ - public PreloadCompletedEvent(ApplicationContext source) { - super(source); - } - -} \ No newline at end of file diff --git a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/binlog/AbstractBinlogService.java b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/binlog/AbstractBinlogService.java deleted file mode 100644 index bd91617f..00000000 --- a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/binlog/AbstractBinlogService.java +++ /dev/null @@ -1,233 +0,0 @@ -package org.dbsyncer.storage.binlog; - -import com.google.protobuf.InvalidProtocolBufferException; -import org.dbsyncer.common.config.BinlogRecorderConfig; -import org.dbsyncer.common.model.Paging; -import org.dbsyncer.storage.impl.SnowflakeIdWorker; -import org.dbsyncer.connector.scheduled.ScheduledTaskJob; -import org.dbsyncer.connector.scheduled.ScheduledTaskService; -import org.dbsyncer.common.util.CollectionUtils; -import org.dbsyncer.connector.enums.FilterEnum; -import org.dbsyncer.connector.enums.OperationEnum; -import org.dbsyncer.storage.StorageService; -import org.dbsyncer.storage.binlog.proto.BinlogMessage; -import org.dbsyncer.storage.constant.BinlogConstant; -import org.dbsyncer.storage.constant.ConfigConstant; -import org.dbsyncer.storage.enums.BinlogSortEnum; -import org.dbsyncer.storage.enums.IndexFieldResolverEnum; -import org.dbsyncer.storage.enums.StorageEnum; -import org.dbsyncer.storage.query.BooleanFilter; -import org.dbsyncer.storage.query.Query; -import org.dbsyncer.storage.query.filter.IntFilter; -import org.dbsyncer.storage.query.filter.LongFilter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.annotation.PostConstruct; -import javax.annotation.Resource; -import java.sql.Timestamp; -import java.time.Instant; -import java.time.LocalDateTime; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Queue; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; - -/** - * @author AE86 - * @version 1.0.0 - * @date 2022/11/25 0:53 - */ -public abstract class AbstractBinlogService implements BinlogRecorder { - - private final Logger logger = LoggerFactory.getLogger(getClass()); - - @Resource - private StorageService storageService; - - @Resource - private ScheduledTaskService scheduledTaskService; - - @Resource - private SnowflakeIdWorker snowflakeIdWorker; - - private BinlogRecorderConfig binlogRecorderConfig = new BinlogRecorderConfig(); - - private Queue queue; - - private WriterTask writerTask = new WriterTask(); - - private ReaderTask readerTask = new ReaderTask(); - - private AtomicLong activeTasks = new AtomicLong(); - - @PostConstruct - private void init() { - queue = new LinkedBlockingQueue(binlogRecorderConfig.getQueueCapacity()); - scheduledTaskService.start(binlogRecorderConfig.getWriterPeriodMillisecond(), writerTask); - scheduledTaskService.start(binlogRecorderConfig.getReaderPeriodMillisecond(), readerTask); - } - - /** - * 反序列化消息 - * - * @param message - * @return - */ - protected abstract Message deserialize(String messageId, BinlogMessage message); - - @Override - public void flush(BinlogMessage message) { - queue.offer(message); - } - - @Override - public void complete(List messageIds) { - if (!CollectionUtils.isEmpty(messageIds)) { - try { - storageService.removeBatch(StorageEnum.BINLOG, messageIds); - } catch (Exception e) { - logger.error(e.getLocalizedMessage(), e); - } - messageIds.forEach((id) -> activeTasks.decrementAndGet()); - } - } - - /** - * 合并缓存队列任务到磁盘 - */ - final class WriterTask implements ScheduledTaskJob { - - @Override - public void run() { - if (queue.isEmpty()) { - return; - } - - try { - List tasks = new ArrayList<>(); - int count = 0; - long now = Instant.now().toEpochMilli(); - Map task = null; - while (!queue.isEmpty() && count < binlogRecorderConfig.getBatchCount()) { - BinlogMessage message = queue.poll(); - if (null != message) { - task = new HashMap(); - task.put(ConfigConstant.CONFIG_MODEL_ID, String.valueOf(snowflakeIdWorker.nextId())); - task.put(ConfigConstant.BINLOG_STATUS, BinlogConstant.READY); - task.put(ConfigConstant.BINLOG_DATA, message.toByteArray()); - task.put(ConfigConstant.CONFIG_MODEL_CREATE_TIME, now); - tasks.add(task); - } - count++; - } - - if (!CollectionUtils.isEmpty(tasks)) { - storageService.addBatch(StorageEnum.BINLOG, tasks); - } - } catch (Exception e) { - logger.error(e.getMessage()); - } - } - } - - /** - * 从磁盘读取日志到任务队列 - */ - final class ReaderTask implements ScheduledTaskJob { - - private final Lock lock = new ReentrantLock(); - - @Override - public void run() { - // 是否运行中 或 还有提交的任务未处理完成 - if (activeTasks.get() > 0) { - return; - } - - final Lock binlogLock = lock; - boolean locked = false; - try { - locked = binlogLock.tryLock(); - if (locked) { - doParse(); - } - } catch (Exception e) { - logger.error(e.getMessage(), e); - } finally { - if (locked) { - binlogLock.unlock(); - } - } - } - - private void doParse() { - // 查询[待处理] 或 [处理中 & 处理超时] - Query query = new Query(); - query.setType(StorageEnum.BINLOG); - - IntFilter ready = new IntFilter(ConfigConstant.BINLOG_STATUS, FilterEnum.EQUAL, BinlogConstant.READY); - IntFilter processing = new IntFilter(ConfigConstant.BINLOG_STATUS, FilterEnum.EQUAL, BinlogConstant.PROCESSING); - long maxProcessingSeconds = Timestamp.valueOf(LocalDateTime.now().minusSeconds(binlogRecorderConfig.getMaxProcessingSeconds())).getTime(); - LongFilter processingTimeout = new LongFilter(ConfigConstant.CONFIG_MODEL_CREATE_TIME, FilterEnum.LT, maxProcessingSeconds); - BooleanFilter booleanFilter = new BooleanFilter() - .add(new BooleanFilter().add(ready), OperationEnum.OR) - .add(new BooleanFilter().add(processing).add(processingTimeout), OperationEnum.OR); - query.setBooleanFilter(booleanFilter); - - // 指定返回值类型 - Map fieldResolvers = new LinkedHashMap<>(); - fieldResolvers.put(ConfigConstant.BINLOG_STATUS, IndexFieldResolverEnum.INT); - fieldResolvers.put(ConfigConstant.BINLOG_DATA, IndexFieldResolverEnum.BINARY); - fieldResolvers.put(ConfigConstant.CONFIG_MODEL_CREATE_TIME, IndexFieldResolverEnum.LONG); - query.setIndexFieldResolverMap(fieldResolvers); - query.setPageNum(1); - query.setPageSize(binlogRecorderConfig.getBatchCount()); - query.setSort(BinlogSortEnum.ASC); - Paging paging = storageService.query(query); - if (CollectionUtils.isEmpty(paging.getData())) { - return; - } - - List list = (List) paging.getData(); - final int size = list.size(); - final List messages = new ArrayList<>(size); - final List updateTasks = new ArrayList<>(size); - boolean existProcessing = false; - for (int i = 0; i < size; i++) { - Map row = list.get(i); - String id = (String) row.get(ConfigConstant.CONFIG_MODEL_ID); - Integer status = (Integer) row.get(ConfigConstant.BINLOG_STATUS); - byte[] bytes = (byte[]) row.get(ConfigConstant.BINLOG_DATA); - if (BinlogConstant.PROCESSING == status) { - existProcessing = true; - } - try { - Message message = deserialize(id, BinlogMessage.parseFrom(bytes)); - if (null != message) { - messages.add(message); - row.put(ConfigConstant.CONFIG_MODEL_CREATE_TIME, Instant.now().toEpochMilli()); - updateTasks.add(row); - } - } catch (InvalidProtocolBufferException e) { - logger.error(e.getMessage()); - } - } - if (existProcessing) { - logger.warn("存在超时未处理数据，正在重试，建议优化配置参数[max-processing-seconds={}].", binlogRecorderConfig.getMaxProcessingSeconds()); - } - - // 如果在更新消息状态的过程中服务被中止，为保证数据的安全性，重启后消息可能会同步2次） - storageService.editBatch(StorageEnum.BINLOG, updateTasks); - activeTasks.set(messages.size()); - getQueue().addAll(messages); - } - } - -} \ No newline at end of file diff --git a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/binlog/BinlogRecorder.java b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/binlog/BinlogRecorder.java deleted file mode 100644 index 6036e2ef..00000000 --- a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/binlog/BinlogRecorder.java +++ /dev/null @@ -1,37 +0,0 @@ -package org.dbsyncer.storage.binlog; - -import org.dbsyncer.storage.binlog.proto.BinlogMessage; - -import java.io.IOException; -import java.util.List; -import java.util.Queue; - -/** - * @author AE86 - * @version 1.0.0 - * @date 2022/6/8 23:34 - */ -public interface BinlogRecorder { - - /** - * 将任务序列化刷入磁盘 - * - * @param message - */ - void flush(BinlogMessage message) throws IOException; - - /** - * 消息同步完成后，删除消息记录 - * - * @param messageIds - */ - void complete(List messageIds); - - /** - * 获取缓存队列 - * - * @return - */ - Queue getQueue(); - -} \ No newline at end of file diff --git a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/enums/StorageEnum.java b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/enums/StorageEnum.java index 2877e367..b5a48082 100644 --- a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/enums/StorageEnum.java +++ b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/enums/StorageEnum.java @@ -18,12 +18,7 @@ public enum StorageEnum { /** * 数据：全量或增量数据 */ - DATA("data"), - /** - * Binlog：缓存队列数据 - */ - @Deprecated - BINLOG("binlog"); + DATA("data"); private String type; diff --git a/dbsyncer-storage/src/main/resources/dbsyncer_binlog.sql b/dbsyncer-storage/src/main/resources/dbsyncer_binlog.sql deleted file mode 100644 index 25570e2a..00000000 --- a/dbsyncer-storage/src/main/resources/dbsyncer_binlog.sql +++ /dev/null @@ -1,8 +0,0 @@ -CREATE TABLE `dbsyncer_binlog` ( - `ID` varchar(64) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT '唯一ID', - `STATUS` int(1) NOT NULL COMMENT '状态，0-待同步；1-同步中', - `CREATE_TIME` bigint(0) NOT NULL COMMENT '创建时间', - `DATA` blob NOT NULL COMMENT '同步数据', - PRIMARY KEY (`ID`) USING BTREE, - INDEX `IDX_TYPE_CREATE_TIME`(`STATUS`, `CREATE_TIME`) USING BTREE -) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_bin COMMENT = '缓存队列任务表' ROW_FORMAT = Dynamic; \ No newline at end of file diff --git a/dbsyncer-storage/src/main/resources/dbsyncer_upgrade.sql b/dbsyncer-storage/src/main/resources/dbsyncer_upgrade.sql deleted file mode 100644 index d1b58ce5..00000000 --- a/dbsyncer-storage/src/main/resources/dbsyncer_upgrade.sql +++ /dev/null @@ -1,2 +0,0 @@ -ALTER TABLE `dbsyncer_upgrade` DROP COLUMN `JSON`; -ALTER TABLE `dbsyncer_upgrade` ADD COLUMN `DATA` blob NOT NULL COMMENT '同步数据'; \ No newline at end of file -- Gitee From 2be4851d2fbbb84f32db09e3223085beeec2b9dd Mon Sep 17 00:00:00 2001 From: AE86 <836391306@qq.com> Date: Sun, 19 Nov 2023 23:18:28 +0800 Subject: [PATCH 16/98] rename plugin --- .../org/dbsyncer/parser/ParserComponent.java | 6 +++--- .../parser/flush/impl/GeneralBufferActuator.java | 4 ++-- .../parser/impl/ParserComponentImpl.java | 10 +++++----- .../java/org/dbsyncer/plugin/PluginFactory.java | 16 ++++++++-------- ...eImpl.java => DemoPluginServiceProvider.java} | 12 ++++++------ .../AbstractPluginContext.java} | 5 ++--- .../PluginContext.java} | 14 ++++++++++++-- .../impl/FullPluginContext.java} | 13 ++++++++++--- .../impl/IncrementPluginContext.java} | 13 ++++++++++--- .../{ConvertService.java => PluginService.java} | 14 ++++++++------ 10 files changed, 66 insertions(+), 41 deletions(-) rename dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/impl/{DemoConvertServiceImpl.java => DemoPluginServiceProvider.java} (64%) rename dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/{model/AbstractConvertContext.java => plugin/AbstractPluginContext.java} (94%) rename dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/{spi/ConvertContext.java => plugin/PluginContext.java} (80%) rename dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/{model/FullConvertContext.java => plugin/impl/FullPluginContext.java} (34%) rename dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/{model/IncrementConvertContext.java => plugin/impl/IncrementPluginContext.java} (36%) rename dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/{ConvertService.java => PluginService.java} (71%) diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ParserComponent.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ParserComponent.java index 4c10d740..62163576 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ParserComponent.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ParserComponent.java @@ -6,7 +6,7 @@ import org.dbsyncer.parser.model.BatchWriter; import org.dbsyncer.parser.model.Mapping; import org.dbsyncer.parser.model.TableGroup; import org.dbsyncer.parser.model.Task; -import org.dbsyncer.sdk.spi.ConvertContext; +import org.dbsyncer.sdk.plugin.PluginContext; import java.util.Map; import java.util.concurrent.Executor; @@ -62,11 +62,11 @@ public interface ParserComponent { /** * 批执行 * - * @param context + * @param pluginContext * @param batchWriter * @param executor * @return */ - Result writeBatch(ConvertContext context, BatchWriter batchWriter, Executor executor); + Result writeBatch(PluginContext pluginContext, BatchWriter batchWriter, Executor executor); } \ No newline at end of file diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/GeneralBufferActuator.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/GeneralBufferActuator.java index 41799b45..d7bde8c9 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/GeneralBufferActuator.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/GeneralBufferActuator.java @@ -27,7 +27,7 @@ import org.dbsyncer.parser.util.ConvertUtil; import org.dbsyncer.parser.util.PickerUtil; import org.dbsyncer.plugin.PluginFactory; import org.dbsyncer.sdk.model.ConnectorConfig; -import org.dbsyncer.sdk.model.IncrementConvertContext; +import org.dbsyncer.sdk.plugin.impl.IncrementPluginContext; import org.dbsyncer.sdk.spi.ConnectorMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -141,7 +141,7 @@ public class GeneralBufferActuator extends AbstractBufferActuator plugins = new LinkedList<>(); - private final Map service = new LinkedHashMap<>(); + private final Map service = new LinkedHashMap<>(); @Resource private ApplicationContext applicationContext; @PostConstruct private void init() { - Map services = applicationContext.getBeansOfType(ConvertService.class); + Map services = applicationContext.getBeansOfType(PluginService.class); if (!CollectionUtils.isEmpty(services)) { services.forEach((k, s) -> { String pluginId = createPluginId(s.getClass().getName(), s.getVersion()); @@ -110,7 +110,7 @@ public class PluginFactory implements DisposableBean { * @param plugin * @param context */ - public void convert(Plugin plugin, ConvertContext context) { + public void convert(Plugin plugin, PluginContext context) { if (null != plugin) { String pluginId = createPluginId(plugin.getClassName(), plugin.getVersion()); service.computeIfPresent(pluginId, (k, c) -> { @@ -126,7 +126,7 @@ public class PluginFactory implements DisposableBean { * @param plugin * @param context */ - public void postProcessAfter(Plugin plugin, ConvertContext context) { + public void postProcessAfter(Plugin plugin, PluginContext context) { if (null != plugin) { String pluginId = createPluginId(plugin.getClassName(), plugin.getVersion()); service.computeIfPresent(pluginId, (k, c) -> { @@ -150,8 +150,8 @@ public class PluginFactory implements DisposableBean { String fileName = jar.getName(); URL url = jar.toURI().toURL(); URLClassLoader loader = new URLClassLoader(new URL[]{url}, Thread.currentThread().getContextClassLoader()); - ServiceLoader services = ServiceLoader.load(ConvertService.class, loader); - for (ConvertService s : services) { + ServiceLoader services = ServiceLoader.load(PluginService.class, loader); + for (PluginService s : services) { String pluginId = createPluginId(s.getClass().getName(), s.getVersion()); // 先释放历史版本 if (service.containsKey(pluginId)) { diff --git a/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/impl/DemoConvertServiceImpl.java b/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/impl/DemoPluginServiceProvider.java similarity index 64% rename from dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/impl/DemoConvertServiceImpl.java rename to dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/impl/DemoPluginServiceProvider.java index 7af8df85..d597fa79 100644 --- a/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/impl/DemoConvertServiceImpl.java +++ b/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/impl/DemoPluginServiceProvider.java @@ -1,8 +1,8 @@ package org.dbsyncer.plugin.impl; import org.dbsyncer.common.config.AppConfig; -import org.dbsyncer.sdk.spi.ConvertContext; -import org.dbsyncer.sdk.spi.ConvertService; +import org.dbsyncer.sdk.plugin.PluginContext; +import org.dbsyncer.sdk.spi.PluginService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; @@ -10,7 +10,7 @@ import org.springframework.stereotype.Component; import javax.annotation.Resource; @Component -public class DemoConvertServiceImpl implements ConvertService { +public class DemoPluginServiceProvider implements PluginService { private final Logger logger = LoggerFactory.getLogger(getClass()); @@ -18,14 +18,14 @@ public class DemoConvertServiceImpl implements ConvertService { private AppConfig appConfig; @Override - public void convert(ConvertContext context) { + public void convert(PluginContext context) { context.setTerminated(true); - logger.info("插件正在处理同步数据，数据源表:{}，目标源表:{}，事件:{}，条数:{}", context.getSourceTableName(), context.getTargetTableName(), + logger.info("插件正在处理{}，数据源表:{}，目标源表:{}，事件:{}，条数:{}", context.getModelEnum().getName(), context.getSourceTableName(), context.getTargetTableName(), context.getEvent(), context.getTargetList().size()); } @Override - public void postProcessAfter(ConvertContext context) { + public void postProcessAfter(PluginContext context) { logger.info("插件正在处理同步成功的数据，目标源表:{}，事件:{}，条数:{}", context.getTargetTableName(), context.getEvent(), context.getTargetList().size()); } diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/AbstractConvertContext.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/plugin/AbstractPluginContext.java similarity index 94% rename from dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/AbstractConvertContext.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/plugin/AbstractPluginContext.java index 8cf761d6..a702c7df 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/AbstractConvertContext.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/plugin/AbstractPluginContext.java @@ -1,7 +1,6 @@ -package org.dbsyncer.sdk.model; +package org.dbsyncer.sdk.plugin; import org.dbsyncer.sdk.spi.ConnectorMapper; -import org.dbsyncer.sdk.spi.ConvertContext; import java.util.List; import java.util.Map; @@ -11,7 +10,7 @@ import java.util.Map; * @version 1.0.0 * @date 2022/6/30 16:00 */ -public abstract class AbstractConvertContext implements ConvertContext { +public abstract class AbstractPluginContext implements PluginContext { /** * 是否终止任务 diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/ConvertContext.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/plugin/PluginContext.java similarity index 80% rename from dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/ConvertContext.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/plugin/PluginContext.java index 0bd9b54b..d05b94ff 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/ConvertContext.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/plugin/PluginContext.java @@ -1,4 +1,7 @@ -package org.dbsyncer.sdk.spi; +package org.dbsyncer.sdk.plugin; + +import org.dbsyncer.sdk.enums.ModelEnum; +import org.dbsyncer.sdk.spi.ConnectorMapper; import java.util.List; import java.util.Map; @@ -10,7 +13,14 @@ import java.util.Map; * @version 1.0.0 * @date 2022/10/28 20:26 */ -public interface ConvertContext { +public interface PluginContext { + + /** + * 获取同步方式 + * + * @return + */ + ModelEnum getModelEnum(); /** * 是否终止同步数据到目标源库 diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/FullConvertContext.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/plugin/impl/FullPluginContext.java similarity index 34% rename from dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/FullConvertContext.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/plugin/impl/FullPluginContext.java index ea7f2370..7082d07f 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/FullConvertContext.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/plugin/impl/FullPluginContext.java @@ -1,5 +1,7 @@ -package org.dbsyncer.sdk.model; +package org.dbsyncer.sdk.plugin.impl; +import org.dbsyncer.sdk.enums.ModelEnum; +import org.dbsyncer.sdk.plugin.AbstractPluginContext; import org.dbsyncer.sdk.spi.ConnectorMapper; /** @@ -7,9 +9,14 @@ import org.dbsyncer.sdk.spi.ConnectorMapper; * @version 1.0.0 * @date 2022/6/30 16:04 */ -public final class FullConvertContext extends AbstractConvertContext { +public final class FullPluginContext extends AbstractPluginContext { - public FullConvertContext(ConnectorMapper sourceConnectorMapper, ConnectorMapper targetConnectorMapper, String sourceTableName, String targetTableName, String event) { + public FullPluginContext(ConnectorMapper sourceConnectorMapper, ConnectorMapper targetConnectorMapper, String sourceTableName, String targetTableName, String event) { super.init(sourceConnectorMapper, targetConnectorMapper, sourceTableName, targetTableName, event, null, null); } + + @Override + public ModelEnum getModelEnum() { + return ModelEnum.FULL; + } } \ No newline at end of file diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/IncrementConvertContext.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/plugin/impl/IncrementPluginContext.java similarity index 36% rename from dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/IncrementConvertContext.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/plugin/impl/IncrementPluginContext.java index 702af7ee..0757d0df 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/IncrementConvertContext.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/plugin/impl/IncrementPluginContext.java @@ -1,5 +1,7 @@ -package org.dbsyncer.sdk.model; +package org.dbsyncer.sdk.plugin.impl; +import org.dbsyncer.sdk.enums.ModelEnum; +import org.dbsyncer.sdk.plugin.AbstractPluginContext; import org.dbsyncer.sdk.spi.ConnectorMapper; import java.util.List; @@ -10,9 +12,14 @@ import java.util.Map; * @version 1.0.0 * @date 2022/6/30 16:06 */ -public final class IncrementConvertContext extends AbstractConvertContext { +public final class IncrementPluginContext extends AbstractPluginContext { - public IncrementConvertContext(ConnectorMapper sourceConnectorMapper, ConnectorMapper targetConnectorMapper, String sourceTableName, String targetTableName, String event, List sourceList, List targetList) { + public IncrementPluginContext(ConnectorMapper sourceConnectorMapper, ConnectorMapper targetConnectorMapper, String sourceTableName, String targetTableName, String event, List sourceList, List targetList) { super.init(sourceConnectorMapper, targetConnectorMapper, sourceTableName, targetTableName, event, sourceList, targetList); } + + @Override + public ModelEnum getModelEnum() { + return ModelEnum.INCREMENT; + } } \ No newline at end of file diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/ConvertService.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/PluginService.java similarity index 71% rename from dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/ConvertService.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/PluginService.java index 22564a3a..ee08b1c5 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/ConvertService.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/PluginService.java @@ -1,5 +1,7 @@ package org.dbsyncer.sdk.spi; +import org.dbsyncer.sdk.plugin.PluginContext; + /** * 插件扩展服务接口 *

    全量同步/增量同步,扩展转换

    @@ -8,28 +10,28 @@ package org.dbsyncer.sdk.spi; * @version 1.0.0 * @date 2021/02/22 20:26 */ -public interface ConvertService { +public interface PluginService { /** * 初始化加载 */ - default void init(){ + default void init() { } /** * 全量同步/增量同步 * - * @param context 上下文 + * @param pluginContext 上下文 */ - void convert(ConvertContext context); + void convert(PluginContext pluginContext); /** * 全量同步/增量同步完成后执行处理 * - * @param context 上下文 + * @param pluginContext 上下文 */ - default void postProcessAfter(ConvertContext context) { + default void postProcessAfter(PluginContext pluginContext) { } /** -- Gitee From 25154f8bb8cf8074f4c798372aef58f1aecc6004 Mon Sep 17 00:00:00 2001 From: AE86 <836391306@qq.com> Date: Sun, 19 Nov 2023 23:23:31 +0800 Subject: [PATCH 17/98] rename plugin --- .../org/dbsyncer/parser/flush/impl/GeneralBufferActuator.java | 2 +- .../java/org/dbsyncer/parser/impl/ParserComponentImpl.java | 2 +- .../main/java/org/dbsyncer}/plugin/AbstractPluginContext.java | 3 ++- .../java/org/dbsyncer}/plugin/impl/FullPluginContext.java | 4 ++-- .../org/dbsyncer}/plugin/impl/IncrementPluginContext.java | 4 ++-- 5 files changed, 8 insertions(+), 7 deletions(-) rename {dbsyncer-sdk/src/main/java/org/dbsyncer/sdk => dbsyncer-plugin/src/main/java/org/dbsyncer}/plugin/AbstractPluginContext.java (97%) rename {dbsyncer-sdk/src/main/java/org/dbsyncer/sdk => dbsyncer-plugin/src/main/java/org/dbsyncer}/plugin/impl/FullPluginContext.java (86%) rename {dbsyncer-sdk/src/main/java/org/dbsyncer/sdk => dbsyncer-plugin/src/main/java/org/dbsyncer}/plugin/impl/IncrementPluginContext.java (88%) diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/GeneralBufferActuator.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/GeneralBufferActuator.java index d7bde8c9..eeec03d2 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/GeneralBufferActuator.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/GeneralBufferActuator.java @@ -27,7 +27,7 @@ import org.dbsyncer.parser.util.ConvertUtil; import org.dbsyncer.parser.util.PickerUtil; import org.dbsyncer.plugin.PluginFactory; import org.dbsyncer.sdk.model.ConnectorConfig; -import org.dbsyncer.sdk.plugin.impl.IncrementPluginContext; +import org.dbsyncer.plugin.impl.IncrementPluginContext; import org.dbsyncer.sdk.spi.ConnectorMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/ParserComponentImpl.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/ParserComponentImpl.java index 0f0cc996..2966e7d8 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/ParserComponentImpl.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/ParserComponentImpl.java @@ -27,7 +27,7 @@ import org.dbsyncer.parser.util.ConvertUtil; import org.dbsyncer.parser.util.PickerUtil; import org.dbsyncer.plugin.PluginFactory; import org.dbsyncer.sdk.model.ConnectorConfig; -import org.dbsyncer.sdk.plugin.impl.FullPluginContext; +import org.dbsyncer.plugin.impl.FullPluginContext; import org.dbsyncer.sdk.spi.ConnectorMapper; import org.dbsyncer.sdk.plugin.PluginContext; import org.slf4j.Logger; diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/plugin/AbstractPluginContext.java b/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/AbstractPluginContext.java similarity index 97% rename from dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/plugin/AbstractPluginContext.java rename to dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/AbstractPluginContext.java index a702c7df..471adacd 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/plugin/AbstractPluginContext.java +++ b/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/AbstractPluginContext.java @@ -1,5 +1,6 @@ -package org.dbsyncer.sdk.plugin; +package org.dbsyncer.plugin; +import org.dbsyncer.sdk.plugin.PluginContext; import org.dbsyncer.sdk.spi.ConnectorMapper; import java.util.List; diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/plugin/impl/FullPluginContext.java b/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/impl/FullPluginContext.java similarity index 86% rename from dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/plugin/impl/FullPluginContext.java rename to dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/impl/FullPluginContext.java index 7082d07f..54054bce 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/plugin/impl/FullPluginContext.java +++ b/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/impl/FullPluginContext.java @@ -1,7 +1,7 @@ -package org.dbsyncer.sdk.plugin.impl; +package org.dbsyncer.plugin.impl; import org.dbsyncer.sdk.enums.ModelEnum; -import org.dbsyncer.sdk.plugin.AbstractPluginContext; +import org.dbsyncer.plugin.AbstractPluginContext; import org.dbsyncer.sdk.spi.ConnectorMapper; /** diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/plugin/impl/IncrementPluginContext.java b/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/impl/IncrementPluginContext.java similarity index 88% rename from dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/plugin/impl/IncrementPluginContext.java rename to dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/impl/IncrementPluginContext.java index 0757d0df..6a0d7216 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/plugin/impl/IncrementPluginContext.java +++ b/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/impl/IncrementPluginContext.java @@ -1,7 +1,7 @@ -package org.dbsyncer.sdk.plugin.impl; +package org.dbsyncer.plugin.impl; import org.dbsyncer.sdk.enums.ModelEnum; -import org.dbsyncer.sdk.plugin.AbstractPluginContext; +import org.dbsyncer.plugin.AbstractPluginContext; import org.dbsyncer.sdk.spi.ConnectorMapper; import java.util.List; -- Gitee From 361c6b62d158ab34577ddf27c1903026c044458c Mon Sep 17 00:00:00 2001 From: AE86 <836391306@qq.com> Date: Sun, 19 Nov 2023 23:35:18 +0800 Subject: [PATCH 18/98] add deployment --- .../manager/ManagerSupportConfiguration.java | 23 +++++++++++++++++++ .../deployment/StandaloneProvider.java | 16 +++++++++++++ .../dbsyncer/sdk/spi/DeploymentService.java | 14 +++++++++++ 3 files changed, 53 insertions(+) create mode 100644 dbsyncer-manager/src/main/java/org/dbsyncer/manager/ManagerSupportConfiguration.java create mode 100644 dbsyncer-manager/src/main/java/org/dbsyncer/manager/deployment/StandaloneProvider.java create mode 100644 dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/DeploymentService.java diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/ManagerSupportConfiguration.java b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/ManagerSupportConfiguration.java new file mode 100644 index 00000000..1eac8c9b --- /dev/null +++ b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/ManagerSupportConfiguration.java @@ -0,0 +1,23 @@ +package org.dbsyncer.manager; + +import org.dbsyncer.manager.deployment.StandaloneProvider; +import org.dbsyncer.sdk.spi.DeploymentService; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * @author AE86 + * @version 1.0.0 + * @Date 2023-11-19 23:29 + */ +@Configuration +public class ManagerSupportConfiguration { + + @Bean + @ConditionalOnMissingBean + public DeploymentService deploymentService() { + return new StandaloneProvider(); + } + +} \ No newline at end of file diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/deployment/StandaloneProvider.java b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/deployment/StandaloneProvider.java new file mode 100644 index 00000000..0e11c722 --- /dev/null +++ b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/deployment/StandaloneProvider.java @@ -0,0 +1,16 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.manager.deployment; + +import org.dbsyncer.sdk.spi.DeploymentService; + +/** + * 单机部署 + * + * @Version 1.0.0 + * @Author AE86 + * @Date 2023-11-19 23:29 + */ +public class StandaloneProvider implements DeploymentService { +} \ No newline at end of file diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/DeploymentService.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/DeploymentService.java new file mode 100644 index 00000000..3d3417c7 --- /dev/null +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/DeploymentService.java @@ -0,0 +1,14 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.sdk.spi; + +/** + * 部署服务接口 + * + * @Version 1.0.0 + * @Author AE86 + * @Date 2023-11-19 23:24 + */ +public interface DeploymentService { +} \ No newline at end of file -- Gitee From 918dd0782209089a18103726ff4e478160e7b8e5 Mon Sep 17 00:00:00 2001 From: AE86 <836391306@qq.com> Date: Sun, 19 Nov 2023 23:43:23 +0800 Subject: [PATCH 19/98] move module --- .../java/org/dbsyncer/parser/impl/MessageServiceImpl.java | 4 ++-- .../src/main/java/org/dbsyncer/plugin}/NotifyService.java | 4 ++-- .../org/dbsyncer/plugin/NotifySupportConfiguration.java | 5 ++--- ...NotifyService.java => DefaultNotifyServiceProvider.java} | 6 +++--- .../org/dbsyncer/plugin/impl/DemoPluginServiceProvider.java | 2 +- ...ailNotifyService.java => MailNotifyServiceProvider.java} | 6 +++--- .../main/java/org/dbsyncer/plugin}/model/NotifyMessage.java | 2 +- 7 files changed, 14 insertions(+), 15 deletions(-) rename {dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi => dbsyncer-plugin/src/main/java/org/dbsyncer/plugin}/NotifyService.java (79%) rename dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/impl/{DefaultNotifyService.java => DefaultNotifyServiceProvider.java} (56%) rename dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/impl/{MailNotifyService.java => MailNotifyServiceProvider.java} (96%) rename {dbsyncer-sdk/src/main/java/org/dbsyncer/sdk => dbsyncer-plugin/src/main/java/org/dbsyncer/plugin}/model/NotifyMessage.java (96%) diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/MessageServiceImpl.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/MessageServiceImpl.java index 6185b937..7e72b622 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/MessageServiceImpl.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/MessageServiceImpl.java @@ -5,8 +5,8 @@ import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.parser.MessageService; import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.parser.model.UserConfig; -import org.dbsyncer.sdk.model.NotifyMessage; -import org.dbsyncer.sdk.spi.NotifyService; +import org.dbsyncer.plugin.model.NotifyMessage; +import org.dbsyncer.plugin.NotifyService; import org.springframework.stereotype.Component; import javax.annotation.Resource; diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/NotifyService.java b/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/NotifyService.java similarity index 79% rename from dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/NotifyService.java rename to dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/NotifyService.java index cdbd1c2a..ea8f4951 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/NotifyService.java +++ b/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/NotifyService.java @@ -1,6 +1,6 @@ -package org.dbsyncer.sdk.spi; +package org.dbsyncer.plugin; -import org.dbsyncer.sdk.model.NotifyMessage; +import org.dbsyncer.plugin.model.NotifyMessage; /** * 通知服务（同步异常、连接器异常、应用异常等消息） diff --git a/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/NotifySupportConfiguration.java b/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/NotifySupportConfiguration.java index fd80e923..fa66216f 100644 --- a/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/NotifySupportConfiguration.java +++ b/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/NotifySupportConfiguration.java @@ -1,7 +1,6 @@ package org.dbsyncer.plugin; -import org.dbsyncer.plugin.impl.DefaultNotifyService; -import org.dbsyncer.sdk.spi.NotifyService; +import org.dbsyncer.plugin.impl.DefaultNotifyServiceProvider; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -17,7 +16,7 @@ public class NotifySupportConfiguration { @Bean @ConditionalOnMissingBean public NotifyService notifyService() { - return new DefaultNotifyService(); + return new DefaultNotifyServiceProvider(); } } \ No newline at end of file diff --git a/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/impl/DefaultNotifyService.java b/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/impl/DefaultNotifyServiceProvider.java similarity index 56% rename from dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/impl/DefaultNotifyService.java rename to dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/impl/DefaultNotifyServiceProvider.java index 6dd8903e..4ec3cbfa 100644 --- a/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/impl/DefaultNotifyService.java +++ b/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/impl/DefaultNotifyServiceProvider.java @@ -1,7 +1,7 @@ package org.dbsyncer.plugin.impl; -import org.dbsyncer.sdk.model.NotifyMessage; -import org.dbsyncer.sdk.spi.NotifyService; +import org.dbsyncer.plugin.model.NotifyMessage; +import org.dbsyncer.plugin.NotifyService; /** * 默认通知服务 @@ -10,7 +10,7 @@ import org.dbsyncer.sdk.spi.NotifyService; * @version 1.0.0 * @date 2022/11/13 22:57 */ -public class DefaultNotifyService implements NotifyService { +public class DefaultNotifyServiceProvider implements NotifyService { @Override public void sendMessage(NotifyMessage notifyMessage) { diff --git a/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/impl/DemoPluginServiceProvider.java b/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/impl/DemoPluginServiceProvider.java index d597fa79..2d0f3e79 100644 --- a/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/impl/DemoPluginServiceProvider.java +++ b/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/impl/DemoPluginServiceProvider.java @@ -10,7 +10,7 @@ import org.springframework.stereotype.Component; import javax.annotation.Resource; @Component -public class DemoPluginServiceProvider implements PluginService { +public final class DemoPluginServiceProvider implements PluginService { private final Logger logger = LoggerFactory.getLogger(getClass()); diff --git a/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/impl/MailNotifyService.java b/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/impl/MailNotifyServiceProvider.java similarity index 96% rename from dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/impl/MailNotifyService.java rename to dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/impl/MailNotifyServiceProvider.java index a0a1a5b2..8627e868 100644 --- a/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/impl/MailNotifyService.java +++ b/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/impl/MailNotifyServiceProvider.java @@ -2,8 +2,8 @@ package org.dbsyncer.plugin.impl; import org.dbsyncer.common.config.AppConfig; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.sdk.model.NotifyMessage; -import org.dbsyncer.sdk.spi.NotifyService; +import org.dbsyncer.plugin.model.NotifyMessage; +import org.dbsyncer.plugin.NotifyService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; @@ -33,7 +33,7 @@ import java.util.Properties; @Component @ConditionalOnProperty(value = "dbsyncer.plugin.notify.mail.enabled", havingValue = "true") @ConfigurationProperties(prefix = "dbsyncer.plugin.notify.mail") -public class MailNotifyService implements NotifyService { +public final class MailNotifyServiceProvider implements NotifyService { private final Logger logger = LoggerFactory.getLogger(getClass()); diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/NotifyMessage.java b/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/model/NotifyMessage.java similarity index 96% rename from dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/NotifyMessage.java rename to dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/model/NotifyMessage.java index 3db4281b..d3cf56f0 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/NotifyMessage.java +++ b/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/model/NotifyMessage.java @@ -1,4 +1,4 @@ -package org.dbsyncer.sdk.model; +package org.dbsyncer.plugin.model; import java.util.List; -- Gitee From 0af3cd083001b39a50c852ce97eda769eaec0f78 Mon Sep 17 00:00:00 2001 From: AE86 <836391306@qq.com> Date: Mon, 20 Nov 2023 00:00:26 +0800 Subject: [PATCH 20/98] add conn --- .../impl/DefaultNotifyServiceProvider.java | 2 +- .../dbsyncer/sdk/config/CommandConfig.java | 48 +++++++ .../org/dbsyncer/sdk/config/DDLConfig.java | 74 ++++++++++ .../org/dbsyncer/sdk/config/ReaderConfig.java | 55 ++++++++ .../sdk/config/WriterBatchConfig.java | 59 ++++++++ .../dbsyncer/sdk/connector/CompareFilter.java | 7 + .../dbsyncer/sdk/enums/DDLOperationEnum.java | 15 +++ .../org/dbsyncer/sdk/enums/FilterEnum.java | 114 ++++++++++++++++ .../org/dbsyncer/sdk/enums/OperationEnum.java | 40 ++++++ .../org/dbsyncer/sdk/enums/TableTypeEnum.java | 47 +++++++ .../java/org/dbsyncer/sdk/model/Field.java | 123 +++++++++++++++++ .../java/org/dbsyncer/sdk/model/Filter.java | 65 +++++++++ .../java/org/dbsyncer/sdk/model/MetaInfo.java | 62 +++++++++ .../java/org/dbsyncer/sdk/model/Table.java | 95 +++++++++++++ .../dbsyncer/sdk/spi/ConnectorInstance.java | 55 ++++++++ .../dbsyncer/sdk/spi/ConnectorService.java | 127 ++++++++++++++++++ 16 files changed, 987 insertions(+), 1 deletion(-) create mode 100644 dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/config/CommandConfig.java create mode 100644 dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/config/DDLConfig.java create mode 100644 dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/config/ReaderConfig.java create mode 100644 dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/config/WriterBatchConfig.java create mode 100644 dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/CompareFilter.java create mode 100644 dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/DDLOperationEnum.java create mode 100644 dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/FilterEnum.java create mode 100644 dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/OperationEnum.java create mode 100644 dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/TableTypeEnum.java create mode 100644 dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/Field.java create mode 100644 dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/Filter.java create mode 100644 dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/MetaInfo.java create mode 100644 dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/Table.java create mode 100644 dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/ConnectorInstance.java create mode 100644 dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/ConnectorService.java diff --git a/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/impl/DefaultNotifyServiceProvider.java b/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/impl/DefaultNotifyServiceProvider.java index 4ec3cbfa..17218023 100644 --- a/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/impl/DefaultNotifyServiceProvider.java +++ b/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/impl/DefaultNotifyServiceProvider.java @@ -10,7 +10,7 @@ import org.dbsyncer.plugin.NotifyService; * @version 1.0.0 * @date 2022/11/13 22:57 */ -public class DefaultNotifyServiceProvider implements NotifyService { +public final class DefaultNotifyServiceProvider implements NotifyService { @Override public void sendMessage(NotifyMessage notifyMessage) { diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/config/CommandConfig.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/config/CommandConfig.java new file mode 100644 index 00000000..2eecead1 --- /dev/null +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/config/CommandConfig.java @@ -0,0 +1,48 @@ +package org.dbsyncer.sdk.config; + +import org.dbsyncer.sdk.model.ConnectorConfig; +import org.dbsyncer.sdk.model.Filter; +import org.dbsyncer.sdk.model.Table; + +import java.util.List; + +/** + * 查询同步参数模板 + * + * @author AE86 + * @version 1.0.0 + * @date 2019/9/16 23:59 + */ +public class CommandConfig { + + private String type; + + private Table table; + + private List filter; + + private ConnectorConfig connectorConfig; + + public CommandConfig(String type, Table table, ConnectorConfig connectorConfig, List filter) { + this.type = type; + this.table = table; + this.filter = filter; + this.connectorConfig = connectorConfig; + } + + public String getType() { + return type; + } + + public Table getTable() { + return table; + } + + public List getFilter() { + return filter; + } + + public ConnectorConfig getConnectorConfig() { + return connectorConfig; + } +} \ No newline at end of file diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/config/DDLConfig.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/config/DDLConfig.java new file mode 100644 index 00000000..6c172059 --- /dev/null +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/config/DDLConfig.java @@ -0,0 +1,74 @@ +package org.dbsyncer.sdk.config; + +import org.dbsyncer.sdk.enums.DDLOperationEnum; +import org.dbsyncer.sdk.model.Field; + +import java.util.LinkedList; +import java.util.List; + +public class DDLConfig { + /** + * 执行命令 + */ + private String sql; + + private DDLOperationEnum ddlOperationEnum; + + private List addFields = new LinkedList<>(); + + private List removeFields = new LinkedList<>(); + + //记录源表的源字段名称 + private String sourceColumnName; + + //记录改变后的字段名称 + private String changedColumnName; + + public String getSql() { + return sql; + } + + public void setSql(String sql) { + this.sql = sql; + } + + public List getAddFields() { + return addFields; + } + + public void setAddFields(List addFields) { + this.addFields = addFields; + } + + public List getRemoveFields() { + return removeFields; + } + + public void setRemoveFields(List removeFields) { + this.removeFields = removeFields; + } + + public String getSourceColumnName() { + return sourceColumnName; + } + + public void setSourceColumnName(String sourceColumnName) { + this.sourceColumnName = sourceColumnName; + } + + public String getChangedColumnName() { + return changedColumnName; + } + + public void setChangedColumnName(String changedColumnName) { + this.changedColumnName = changedColumnName; + } + + public DDLOperationEnum getDdlOperationEnum() { + return ddlOperationEnum; + } + + public void setDdlOperationEnum(DDLOperationEnum ddlOperationEnum) { + this.ddlOperationEnum = ddlOperationEnum; + } +} \ No newline at end of file diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/config/ReaderConfig.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/config/ReaderConfig.java new file mode 100644 index 00000000..99923947 --- /dev/null +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/config/ReaderConfig.java @@ -0,0 +1,55 @@ +package org.dbsyncer.sdk.config; + +import org.dbsyncer.sdk.model.Table; + +import java.util.List; +import java.util.Map; + +public class ReaderConfig { + + private Table table; + private Map command; + private List args; + private boolean supportedCursor; + private Object[] cursors; + private int pageIndex; + private int pageSize; + + public ReaderConfig(Table table, Map command, List args, boolean supportedCursor, Object[] cursors, int pageIndex, int pageSize) { + this.table = table; + this.command = command; + this.args = args; + this.supportedCursor = supportedCursor; + this.cursors = cursors; + this.pageIndex = pageIndex; + this.pageSize = pageSize; + } + + public Table getTable() { + return table; + } + + public Map getCommand() { + return command; + } + + public List getArgs() { + return args; + } + + public Object[] getCursors() { + return cursors; + } + + public int getPageIndex() { + return pageIndex; + } + + public int getPageSize() { + return pageSize; + } + + public boolean isSupportedCursor() { + return supportedCursor; + } +} \ No newline at end of file diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/config/WriterBatchConfig.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/config/WriterBatchConfig.java new file mode 100644 index 00000000..6a5f33ee --- /dev/null +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/config/WriterBatchConfig.java @@ -0,0 +1,59 @@ +package org.dbsyncer.sdk.config; + +import org.dbsyncer.sdk.model.Field; + +import java.util.List; +import java.util.Map; + +public class WriterBatchConfig { + + /** + * 表名 + */ + private String tableName; + /** + * 事件 + */ + private String event; + /** + * 执行命令 + */ + private Map command; + /** + * 字段信息 + */ + private List fields; + /** + * 集合数据 + */ + private List data; + + public WriterBatchConfig(String tableName, String event, Map command, List fields, List data) { + this.tableName = tableName; + this.event = event; + this.command = command; + this.fields = fields; + this.data = data; + } + + public String getTableName() { + return tableName; + } + + public String getEvent() { + return event; + } + + public Map getCommand() { + return command; + } + + public List getFields() { + return fields; + } + + public List getData() { + return data; + } + +} \ No newline at end of file diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/CompareFilter.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/CompareFilter.java new file mode 100644 index 00000000..219c6d13 --- /dev/null +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/CompareFilter.java @@ -0,0 +1,7 @@ +package org.dbsyncer.sdk.connector; + +public interface CompareFilter { + + boolean compare(String value, String filterValue); + +} \ No newline at end of file diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/DDLOperationEnum.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/DDLOperationEnum.java new file mode 100644 index 00000000..807df0be --- /dev/null +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/DDLOperationEnum.java @@ -0,0 +1,15 @@ +package org.dbsyncer.sdk.enums; + +/** + * 支持同步的DDL命令 + * + * @version 1.0.0 + * @Author life + * @Date 2023-09-24 14:24 + */ +public enum DDLOperationEnum { + ALTER_MODIFY, + ALTER_ADD, + ALTER_DROP, + ALTER_CHANGE; +} \ No newline at end of file diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/FilterEnum.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/FilterEnum.java new file mode 100644 index 00000000..9cb59482 --- /dev/null +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/FilterEnum.java @@ -0,0 +1,114 @@ +package org.dbsyncer.sdk.enums; + +import org.dbsyncer.common.util.NumberUtil; +import org.dbsyncer.common.util.StringUtil; +import org.dbsyncer.sdk.SdkException; +import org.dbsyncer.sdk.connector.CompareFilter; + +/** + * 运算符表达式类型 + * + * @author AE86 + * @version 1.0.0 + * @date 2019/9/26 23:21 + */ +public enum FilterEnum { + + /** + * 等于 + */ + EQUAL("=", (value, filterValue) -> StringUtil.equals(value, filterValue)), + /** + * 不等于 + */ + NOT_EQUAL("!=", (value, filterValue) -> !StringUtil.equals(value, filterValue)), + /** + * 大于 + */ + GT(">", (value, filterValue) -> NumberUtil.toLong(value) > NumberUtil.toLong(filterValue)), + /** + * 小于 + */ + LT("<", (value, filterValue) -> NumberUtil.toLong(value) < NumberUtil.toLong(filterValue)), + /** + * 大于等于 + */ + GT_AND_EQUAL(">=", (value, filterValue) -> NumberUtil.toLong(value) >= NumberUtil.toLong(filterValue)), + /** + * 小于等于 + */ + LT_AND_EQUAL("<=", (value, filterValue) -> NumberUtil.toLong(value) <= NumberUtil.toLong(filterValue)), + /** + * 模糊匹配 + */ + LIKE("like", (value, filterValue) -> { + boolean startsWith = StringUtil.startsWith(filterValue, "%") || StringUtil.startsWith(filterValue, "*"); + boolean endsWith = StringUtil.endsWith(filterValue, "%") || StringUtil.endsWith(filterValue, "*"); + String compareValue = StringUtil.replace(filterValue, "%", ""); + compareValue = StringUtil.replace(compareValue, "*", ""); + // 模糊匹配 + if(startsWith && endsWith){ + return StringUtil.contains(value, compareValue); + } + // 前缀匹配 + if(endsWith){ + return StringUtil.startsWith(value, compareValue); + } + // 后缀匹配 + if(startsWith){ + return StringUtil.endsWith(value, compareValue); + } + return false; + }); + + // 运算符名称 + private String name; + // 比较器 + private CompareFilter compareFilter; + + FilterEnum(String name, CompareFilter compareFilter) { + this.name = name; + this.compareFilter = compareFilter; + } + + /** + * 获取表达式 + * + * @param name + * @return + * @throws SdkException + */ + public static FilterEnum getFilterEnum(String name) throws SdkException { + for (FilterEnum e : FilterEnum.values()) { + if (StringUtil.equals(name, e.getName())) { + return e; + } + } + throw new SdkException(String.format("FilterEnum name \"%s\" does not exist.", name)); + } + + /** + * 获取比较器 + * + * @param filterName + * @return + * @throws SdkException + */ + public static CompareFilter getCompareFilter(String filterName) throws SdkException { + for (FilterEnum e : FilterEnum.values()) { + if (StringUtil.equals(filterName, e.getName())) { + return e.getCompareFilter(); + } + } + throw new SdkException(String.format("FilterEnum name \"%s\" does not exist.", filterName)); + } + + public String getName() { + return name; + } + + public CompareFilter getCompareFilter() { + return compareFilter; + } + +} \ No newline at end of file diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/OperationEnum.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/OperationEnum.java new file mode 100644 index 00000000..8a969109 --- /dev/null +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/OperationEnum.java @@ -0,0 +1,40 @@ +package org.dbsyncer.sdk.enums; + +/** + * 条件表达式类型 + * + * @author AE86 + * @version 1.0.0 + * @date 2019/9/26 23:21 + */ +public enum OperationEnum { + + /** + * 并且 + */ + AND("and"), + /** + * 或者 + */ + OR("or"); + + // 描述 + private String name; + + OperationEnum(String name) { + this.name = name; + } + + public static boolean isAnd(String name) { + return AND.getName().equals(name); + } + + public static boolean isOr(String name) { + return OR.getName().equals(name); + } + + public String getName() { + return name; + } + +} \ No newline at end of file diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/TableTypeEnum.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/TableTypeEnum.java new file mode 100644 index 00000000..03580c27 --- /dev/null +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/TableTypeEnum.java @@ -0,0 +1,47 @@ +package org.dbsyncer.sdk.enums; + +/** + * 表类型 + * + * @author AE86 + * @version 1.0.0 + * @date 2021/08/26 21:13 + */ +public enum TableTypeEnum { + + /** + * 表 + */ + TABLE("TABLE"), + + /** + * 视图 + */ + VIEW("VIEW"), + + /** + * 物化视图 + */ + MATERIALIZED_VIEW("MATERIALIZED VIEW"); + + private String code; + + TableTypeEnum(String code) { + this.code = code; + } + + /** + * 是否视图类型 + * + * @param type + * @return + */ + public static boolean isView(String type) { + return VIEW.getCode().equals(type); + } + + public String getCode() { + return code; + } + +} \ No newline at end of file diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/Field.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/Field.java new file mode 100644 index 00000000..fcbd4521 --- /dev/null +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/Field.java @@ -0,0 +1,123 @@ +package org.dbsyncer.sdk.model; + +import org.dbsyncer.common.util.JsonUtil; + +/** + * 字段属性 + * + * @author AE86 + * @version 1.0.0 + * @date 2019/9/30 15:10 + */ +public class Field { + + /** + * 字段名，ID + */ + private String name; + + /** + * 类型名，INT + */ + private String typeName; + + /** + * 类型编码，4 + */ + private int type; + + /** + * 主键 + */ + private boolean pk; + + /** + * 字段别名 + */ + private String labelName; + + /** + * 是否系统字段 + */ + private boolean unmodifiabled; + + public Field() { + } + + public Field(String name, String typeName, int type) { + this.name = name; + this.typeName = typeName; + this.type = type; + } + + public Field(String name, String typeName, int type, boolean pk) { + this.name = name; + this.typeName = typeName; + this.type = type; + this.pk = pk; + } + + public Field(String name, String typeName, int type, boolean pk, String labelName, boolean unmodifiabled) { + this.name = name; + this.typeName = typeName; + this.type = type; + this.pk = pk; + this.labelName = labelName; + this.unmodifiabled = unmodifiabled; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getTypeName() { + return typeName; + } + + public void setTypeName(String typeName) { + this.typeName = typeName; + } + + public int getType() { + return type; + } + + public void setType(int type) { + this.type = type; + } + + public boolean isPk() { + return pk; + } + + public void setPk(boolean pk) { + this.pk = pk; + } + + public String getLabelName() { + return labelName; + } + + public Field setLabelName(String labelName) { + this.labelName = labelName; + return this; + } + + public boolean isUnmodifiabled() { + return unmodifiabled; + } + + public Field setUnmodifiabled(boolean unmodifiabled) { + this.unmodifiabled = unmodifiabled; + return this; + } + + @Override + public String toString() { + return JsonUtil.objToJson(this); + } +} \ No newline at end of file diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/Filter.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/Filter.java new file mode 100644 index 00000000..7e169d4a --- /dev/null +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/Filter.java @@ -0,0 +1,65 @@ +package org.dbsyncer.sdk.model; + +import org.dbsyncer.sdk.enums.FilterEnum; +import org.dbsyncer.sdk.enums.OperationEnum; + +/** + * 字段属性条件 + * @author AE86 + * @version 1.0.0 + * @date 2019/9/30 15:10 + */ +public class Filter { + + /** + * 字段名，ID + */ + private String name; + + /** + * @see OperationEnum + */ + private String operation; + + /** + * @see FilterEnum + */ + private String filter; + + /** + * 值 + */ + private String value; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getOperation() { + return operation; + } + + public void setOperation(String operation) { + this.operation = operation; + } + + public String getFilter() { + return filter; + } + + public void setFilter(String filter) { + this.filter = filter; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } +} \ No newline at end of file diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/MetaInfo.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/MetaInfo.java new file mode 100644 index 00000000..b80ae0e8 --- /dev/null +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/MetaInfo.java @@ -0,0 +1,62 @@ +package org.dbsyncer.sdk.model; + +import java.util.List; + +/** + * 连接器基本信息 + * + * @author AE86 + * @ClassName: MetaInfo + * @Description: 包括字段信息、总条数 + * @date: 2017年7月20日 下午3:37:59 + */ +public class MetaInfo { + + /** + * 表类型 + */ + private String tableType; + + /** + * 属性字段 + * 格式：[{"name":"ID","typeName":"INT","type":"4"},{"name":"NAME","typeName":"VARCHAR","type":"12"}] + */ + private List column; + + /** + * sql + */ + private String sql; + + public String getTableType() { + return tableType; + } + + public MetaInfo setTableType(String tableType) { + this.tableType = tableType; + return this; + } + + public List getColumn() { + return column; + } + + public MetaInfo setColumn(List column) { + this.column = column; + return this; + } + + public String getSql() { + return sql; + } + + public MetaInfo setSql(String sql) { + this.sql = sql; + return this; + } + + @Override + public String toString() { + return new StringBuilder().append("MetaInfo{").append("tableType=").append(tableType).append(", ").append("column=").append(column).append('}').toString(); + } +} \ No newline at end of file diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/Table.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/Table.java new file mode 100644 index 00000000..98c14b63 --- /dev/null +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/Table.java @@ -0,0 +1,95 @@ +package org.dbsyncer.sdk.model; + +import org.dbsyncer.sdk.enums.TableTypeEnum; + +import java.util.List; + +/** + * @author AE86 + * @version 1.0.0 + * @date 2019/10/15 23:58 + */ +public class Table { + + /** + * 表名 + */ + private String name; + + /** + * 表类型[TABLE、VIEW、MATERIALIZED VIEW] + */ + private String type; + + /** + * 属性字段 + * 格式：[{"name":"ID","typeName":"INT","type":"4"},{"name":"NAME","typeName":"VARCHAR","type":"12"}] + */ + private List column; + + /** + * sql + */ + private String sql; + + // 总数 + private long count; + + public Table() { + } + + public Table(String name) { + this(name, TableTypeEnum.TABLE.getCode()); + } + + public Table(String name, String type) { + this(name, type, null, null); + } + + public Table(String name, String type, List column, String sql) { + this.name = name; + this.type = type; + this.column = column; + this.sql = sql; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public List getColumn() { + return column; + } + + public void setColumn(List column) { + this.column = column; + } + + public String getSql() { + return sql; + } + + public void setSql(String sql) { + this.sql = sql; + } + + public long getCount() { + return count; + } + + public void setCount(long count) { + this.count = count; + } +} \ No newline at end of file diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/ConnectorInstance.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/ConnectorInstance.java new file mode 100644 index 00000000..10f5287a --- /dev/null +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/ConnectorInstance.java @@ -0,0 +1,55 @@ +package org.dbsyncer.sdk.spi; + +/** + * 连接器实例，管理连接生命周期 + * + * @param 配置 + * @param 实例 + * @author AE86 + * @version 1.0.0 + * @date 2022/3/20 23:00 + */ +public interface ConnectorInstance extends Cloneable { + + /** + * 获取服务地址 + * + * @return + */ + String getServiceUrl(); + + /** + * 获取连接配置 + * + * @return + */ + K getConfig(); + + /** + * 设置 + * + * @param k + */ + void setConfig(K k); + + /** + * 获取连接通道实例 + * + * @return + * @throws Exception + */ + V getConnection() throws Exception; + + /** + * 关闭连接器 + */ + void close(); + + /** + * 浅拷贝连接器 + * + * @return + * @throws CloneNotSupportedException + */ + Object clone() throws CloneNotSupportedException; +} \ No newline at end of file diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/ConnectorService.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/ConnectorService.java new file mode 100644 index 00000000..7f759f23 --- /dev/null +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/ConnectorService.java @@ -0,0 +1,127 @@ +package org.dbsyncer.sdk.spi; + +import org.dbsyncer.common.model.Result; +import org.dbsyncer.sdk.SdkException; +import org.dbsyncer.sdk.config.CommandConfig; +import org.dbsyncer.sdk.config.DDLConfig; +import org.dbsyncer.sdk.config.ReaderConfig; +import org.dbsyncer.sdk.config.WriterBatchConfig; +import org.dbsyncer.sdk.model.MetaInfo; +import org.dbsyncer.sdk.model.Table; + +import java.util.List; +import java.util.Map; + +/** + * 连接器基础功能 + * + * @param ConnectorInstance + * @param ConnectorConfig + * @Version 1.0.0 + * @Author AE86 + * @Date 2023-11-19 23:24 + */ +public interface ConnectorService { + + /** + * 建立连接 + * + * @param config + * @return + */ + ConnectorInstance connect(C config); + + /** + * 断开连接 + * + * @param connectorInstance + */ + void disconnect(I connectorInstance); + + /** + * 检查连接器是否连接正常 + * + * @param connectorInstance + * @return + */ + boolean isAlive(I connectorInstance); + + /** + * 获取连接缓存key + * + * @param config + * @return + */ + String getConnectorInstanceCacheKey(C config); + + /** + * 获取所有表名 + * + * @param connectorInstance + * @return + */ + List
    getTable(I connectorInstance); + + /** + * 获取表元信息 + * + * @param connectorInstance + * @param tableNamePattern + * @return + */ + MetaInfo getMetaInfo(I connectorInstance, String tableNamePattern); + + /** + * 获取总数 + * + * @param connectorInstance + * @param command + * @return + */ + long getCount(I connectorInstance, Map command); + + /** + * 分页获取数据源数据 + * + * @param connectorInstance + * @param config + * @return + */ + Result reader(I connectorInstance, ReaderConfig config); + + /** + * 批量写入目标源数据 + * + * @param connectorInstance + * @param config + * @return + */ + Result writer(I connectorInstance, WriterBatchConfig config); + + /** + * 执行DDL命令 + * + * @param connectorInstance + * @param ddlConfig + * @return + */ + default Result writerDDL(I connectorInstance, DDLConfig ddlConfig) { + throw new SdkException("Unsupported method."); + } + + /** + * 获取数据源同步参数 + * + * @param commandConfig + * @return + */ + Map getSourceCommand(CommandConfig commandConfig); + + /** + * 获取目标源同步参数 + * + * @param commandConfig + * @return + */ + Map getTargetCommand(CommandConfig commandConfig); +} \ No newline at end of file -- Gitee From 8277b768773a844f7243c1146282f683cad137f2 Mon Sep 17 00:00:00 2001 From: AE86 <836391306@qq.com> Date: Tue, 21 Nov 2023 01:29:48 +0800 Subject: [PATCH 21/98] move conn --- .../dbsyncer/biz/checker/AbstractChecker.java | 2 +- .../AbstractDataBaseConfigChecker.java | 4 +- .../impl/connector/ConnectorChecker.java | 18 +- .../impl/connector/DqlMySQLConfigChecker.java | 2 +- .../connector/DqlOracleConfigChecker.java | 2 +- .../connector/DqlPostgreSQLConfigChecker.java | 2 +- .../connector/DqlSqlServerConfigChecker.java | 2 +- .../impl/connector/OracleConfigChecker.java | 8 +- .../connector/PostgreSQLConfigChecker.java | 2 +- .../connector/SqlServerConfigChecker.java | 2 +- .../impl/tablegroup/TableGroupChecker.java | 8 +- .../biz/impl/ConditionServiceImpl.java | 2 +- .../biz/impl/ConnectorServiceImpl.java | 7 +- .../biz/impl/DataSyncServiceImpl.java | 2 +- .../dbsyncer/biz/impl/MappingServiceImpl.java | 8 +- .../org/dbsyncer/biz/impl/MetricReporter.java | 2 +- .../biz/impl/TableGroupServiceImpl.java | 4 +- .../java/org/dbsyncer/biz/vo/ConditionVo.java | 2 +- .../org/dbsyncer/connector/Connector.java | 125 ---- .../dbsyncer/connector/ConnectorFactory.java | 164 +++-- .../connector/config/CommandConfig.java | 48 -- .../dbsyncer/connector/config/DDLConfig.java | 74 -- .../connector/config/ReaderConfig.java | 55 -- .../connector/config/WriterBatchConfig.java | 59 -- .../connector/enums/ConnectorEnum.java | 119 ---- .../connector/enums/DDLOperationEnum.java | 16 - .../connector/enums/OperationEnum.java | 40 -- .../connector/enums/TableTypeEnum.java | 47 -- .../dbsyncer/connector/es/ESConnector.java | 86 ++- ...orMapper.java => ESConnectorInstance.java} | 116 ++-- .../connector/es/ESDateValueMapper.java | 6 +- .../connector/es/ESOtherValueMapper.java | 6 +- .../connector/file/FileConnector.java | 69 +- ...Mapper.java => FileConnectorInstance.java} | 8 +- .../dbsyncer/connector/file/FileResolver.java | 2 +- .../file/column/FileColumnValue.java | 1 - .../connector/kafka/KafkaConnector.java | 66 +- ...apper.java => KafkaConnectorInstance.java} | 88 +-- .../org/dbsyncer/connector/model/Field.java | 123 ---- .../dbsyncer/connector/model/FileSchema.java | 2 + .../org/dbsyncer/connector/model/Filter.java | 65 -- .../dbsyncer/connector/model/MetaInfo.java | 62 -- .../org/dbsyncer/connector/model/Table.java | 95 --- .../connector/mysql/MySQLConnector.java | 19 +- .../connector/oracle/OracleConnector.java | 55 +- .../oracle/OracleOtherValueMapper.java | 14 +- .../postgresql/PostgreSQLBitValueMapper.java | 6 +- .../postgresql/PostgreSQLConnector.java | 31 +- .../PostgreSQLOtherValueMapper.java | 6 +- .../connector/schema/CharValueMapper.java | 18 - .../connector/schema/NCharValueMapper.java | 18 - .../connector/schema/NVarcharValueMapper.java | 18 - .../connector/schema/OtherValueMapper.java | 18 - .../connector/sql/DQLMySQLConnector.java | 18 +- .../connector/sql/DQLOracleConnector.java | 16 +- .../connector/sql/DQLPostgreSQLConnector.java | 16 +- .../connector/sql/DQLSqlServerConnector.java | 18 +- .../sqlserver/SqlServerConnector.java | 37 +- .../src/main/test/ConnectionTest.java | 28 +- .../listener/AbstractDatabaseExtractor.java | 34 +- .../dbsyncer/listener/AbstractExtractor.java | 4 +- .../dbsyncer/listener/ListenerFactory.java | 9 - .../listener/enums/LogExtractorEnum.java | 98 --- .../listener/enums/TimingExtractorEnum.java | 86 --- .../dbsyncer/listener/file/FileExtractor.java | 18 +- .../listener/mysql/MySQLExtractor.java | 6 +- .../listener/oracle/DqlOracleExtractor.java | 2 +- .../listener/oracle/OracleExtractor.java | 2 +- .../oracle/dcn/DBChangeNotification.java | 2 +- .../postgresql/AbstractMessageDecoder.java | 2 +- .../listener/postgresql/MessageDecoder.java | 6 +- .../postgresql/PostgreSQLExtractor.java | 26 +- .../decoder/PgOutputMessageDecoder.java | 24 +- .../decoder/TestDecodingMessageDecoder.java | 2 +- .../quartz/AbstractQuartzExtractor.java | 18 +- .../quartz/DatabaseQuartzExtractor.java | 2 +- .../listener/quartz/ESQuartzExtractor.java | 10 +- .../quartz/TableGroupQuartzCommand.java | 4 +- .../sqlserver/SqlServerExtractor.java | 16 +- .../main/test/BinaryLogRemoteClientTest.java | 2 +- .../src/main/test/KafkaClientTest.java | 7 +- .../org/dbsyncer/manager/impl/FullPuller.java | 2 +- .../manager/impl/IncrementPuller.java | 16 +- .../manager/impl/PreloadTemplate.java | 6 +- .../org/dbsyncer/parser/ParserComponent.java | 2 +- .../org/dbsyncer/parser/ProfileComponent.java | 10 +- .../parser/consumer/impl/LogConsumer.java | 2 +- .../dbsyncer/parser/ddl/AlterStrategy.java | 2 +- .../org/dbsyncer/parser/ddl/DDLParser.java | 4 +- .../parser/ddl/alter/AddStrategy.java | 6 +- .../parser/ddl/alter/ChangeStrategy.java | 4 +- .../parser/ddl/alter/DropStrategy.java | 28 +- .../parser/ddl/alter/ModifyStrategy.java | 4 +- .../parser/ddl/impl/DDLParserImpl.java | 16 +- .../flush/impl/GeneralBufferActuator.java | 23 +- .../flush/impl/StorageBufferActuator.java | 2 +- .../parser/impl/ParserComponentImpl.java | 40 +- .../parser/impl/ProfileComponentImpl.java | 20 +- .../parser/model/AbstractConfigModel.java | 2 +- .../dbsyncer/parser/model/BatchWriter.java | 14 +- .../org/dbsyncer/parser/model/Connector.java | 2 +- .../dbsyncer/parser/model/FieldMapping.java | 2 +- .../dbsyncer/parser/model/FieldPicker.java | 6 +- .../org/dbsyncer/parser/model/Mapping.java | 2 +- .../org/dbsyncer/parser/model/Picker.java | 2 +- .../org/dbsyncer/parser/model/TableGroup.java | 2 +- .../org/dbsyncer/parser/util/PickerUtil.java | 7 +- .../plugin/AbstractPluginContext.java | 21 +- .../plugin/impl/FullPluginContext.java | 6 +- .../plugin/impl/IncrementPluginContext.java | 6 +- dbsyncer-sdk/pom.xml | 19 + .../dbsyncer/sdk/config/CommandConfig.java | 10 +- .../dbsyncer/sdk}/config/DatabaseConfig.java | 4 +- .../sdk}/config/SqlBuilderConfig.java | 118 ++-- .../sdk}/connector/AbstractConnector.java | 70 +- .../sdk}/connector/AbstractValueMapper.java | 10 +- .../{spi => connector}/ConnectorInstance.java | 6 +- .../dbsyncer/sdk}/connector/ValueMapper.java | 8 +- .../database}/AbstractDQLConnector.java | 27 +- .../database/AbstractDatabaseConnector.java | 107 +-- .../database/AbstractSqlBuilder.java | 24 +- .../sdk}/connector/database/Database.java | 12 +- .../database/DatabaseConnectorInstance.java | 136 ++-- .../connector/database/DatabaseTemplate.java | 8 +- .../connector/database/HandleCallback.java | 2 +- .../database/ds/SimpleConnection.java | 640 +++++++++--------- .../database/ds/SimpleDataSource.java | 274 ++++---- .../database/sqlbuilder/SqlBuilder.java | 4 +- .../database/sqlbuilder/SqlBuilderDelete.java | 10 +- .../database/sqlbuilder/SqlBuilderInsert.java | 10 +- .../database/sqlbuilder/SqlBuilderQuery.java | 12 +- .../sqlbuilder/SqlBuilderQueryCount.java | 82 +-- .../sqlbuilder/SqlBuilderQueryCursor.java | 10 +- .../sqlbuilder/SqlBuilderQueryExist.java | 64 +- .../database/sqlbuilder/SqlBuilderUpdate.java | 12 +- .../connector/schema/BigintValueMapper.java | 12 +- .../connector/schema/BinaryValueMapper.java | 14 +- .../sdk}/connector/schema/BitValueMapper.java | 12 +- .../connector/schema/BlobValueMapper.java | 12 +- .../sdk/connector/schema/CharValueMapper.java | 18 + .../connector/schema/ClobValueMapper.java | 12 +- .../connector/schema/DateValueMapper.java | 12 +- .../connector/schema/DecimalValueMapper.java | 12 +- .../connector/schema/DoubleValueMapper.java | 12 +- .../connector/schema/FloatValueMapper.java | 12 +- .../connector/schema/IntegerValueMapper.java | 12 +- .../schema/LongVarBinaryValueMapper.java | 12 +- .../schema/LongVarcharValueMapper.java | 12 +- .../connector/schema/NCharValueMapper.java | 18 + .../connector/schema/NClobValueMapper.java | 12 +- .../connector/schema/NVarcharValueMapper.java | 18 + .../connector/schema/NumberValueMapper.java | 12 +- .../connector/schema/OtherValueMapper.java | 18 + .../connector/schema/RealValueMapper.java | 12 +- .../connector/schema/RowIdValueMapper.java | 12 +- .../connector/schema/SmallintValueMapper.java | 14 +- .../connector/schema/TimeValueMapper.java | 12 +- .../schema/TimestampValueMapper.java | 12 +- .../connector/schema/TinyintValueMapper.java | 14 +- .../schema/VarBinaryValueMapper.java | 12 +- .../connector/schema/VarcharValueMapper.java | 12 +- .../sdk}/constant/ConnectorConstant.java | 2 +- .../sdk}/constant/DatabaseConstant.java | 2 +- .../dbsyncer/sdk}/enums/SqlBuilderEnum.java | 20 +- .../java/org/dbsyncer/sdk}/model/PageSql.java | 2 +- .../org/dbsyncer/sdk}/model/SqlTable.java | 2 +- .../java/org/dbsyncer/sdk/model/Table.java | 1 + .../dbsyncer/sdk/plugin/PluginContext.java | 6 +- .../org/dbsyncer/sdk/spi/ConnectorMapper.java | 55 -- .../dbsyncer/sdk/spi/ConnectorService.java | 47 +- .../org/dbsyncer/sdk}/util/DatabaseUtil.java | 11 +- .../dbsyncer/sdk}/util/PrimaryKeyUtil.java | 366 +++++----- .../storage/impl/DiskStorageServiceImpl.java | 2 +- .../storage/impl/MySQLStorageServiceImpl.java | 41 +- .../storage/query/AbstractFilter.java | 2 +- .../dbsyncer/storage/query/BooleanFilter.java | 3 +- .../src/main/test/LuceneFactoryTest.java | 2 +- .../src/main/resources/application.properties | 4 +- 178 files changed, 2107 insertions(+), 3078 deletions(-) delete mode 100644 dbsyncer-connector/src/main/java/org/dbsyncer/connector/Connector.java delete mode 100644 dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/CommandConfig.java delete mode 100644 dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/DDLConfig.java delete mode 100644 dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/ReaderConfig.java delete mode 100644 dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/WriterBatchConfig.java delete mode 100644 dbsyncer-connector/src/main/java/org/dbsyncer/connector/enums/ConnectorEnum.java delete mode 100644 dbsyncer-connector/src/main/java/org/dbsyncer/connector/enums/DDLOperationEnum.java delete mode 100644 dbsyncer-connector/src/main/java/org/dbsyncer/connector/enums/OperationEnum.java delete mode 100644 dbsyncer-connector/src/main/java/org/dbsyncer/connector/enums/TableTypeEnum.java rename dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/{ESConnectorMapper.java => ESConnectorInstance.java} (84%) rename dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/{FileConnectorMapper.java => FileConnectorInstance.java} (91%) rename dbsyncer-connector/src/main/java/org/dbsyncer/connector/kafka/{KafkaConnectorMapper.java => KafkaConnectorInstance.java} (78%) delete mode 100644 dbsyncer-connector/src/main/java/org/dbsyncer/connector/model/Field.java delete mode 100644 dbsyncer-connector/src/main/java/org/dbsyncer/connector/model/Filter.java delete mode 100644 dbsyncer-connector/src/main/java/org/dbsyncer/connector/model/MetaInfo.java delete mode 100644 dbsyncer-connector/src/main/java/org/dbsyncer/connector/model/Table.java delete mode 100644 dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/CharValueMapper.java delete mode 100644 dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/NCharValueMapper.java delete mode 100644 dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/NVarcharValueMapper.java delete mode 100644 dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/OtherValueMapper.java delete mode 100644 dbsyncer-listener/src/main/java/org/dbsyncer/listener/enums/LogExtractorEnum.java delete mode 100644 dbsyncer-listener/src/main/java/org/dbsyncer/listener/enums/TimingExtractorEnum.java rename {dbsyncer-connector/src/main/java/org/dbsyncer/connector => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/config/DatabaseConfig.java (96%) rename {dbsyncer-connector/src/main/java/org/dbsyncer/connector => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/config/SqlBuilderConfig.java (85%) rename {dbsyncer-connector/src/main/java/org/dbsyncer => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/connector/AbstractConnector.java (65%) rename {dbsyncer-connector/src/main/java/org/dbsyncer => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/connector/AbstractValueMapper.java (77%) rename dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/{spi => connector}/ConnectorInstance.java (81%) rename {dbsyncer-connector/src/main/java/org/dbsyncer => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/connector/ValueMapper.java (49%) rename {dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database}/AbstractDQLConnector.java (79%) rename {dbsyncer-connector/src/main/java/org/dbsyncer => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/connector/database/AbstractDatabaseConnector.java (87%) rename {dbsyncer-connector/src/main/java/org/dbsyncer => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/connector/database/AbstractSqlBuilder.java (34%) rename {dbsyncer-connector/src/main/java/org/dbsyncer => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/connector/database/Database.java (84%) rename dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/DatabaseConnectorMapper.java => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/DatabaseConnectorInstance.java (71%) rename {dbsyncer-connector/src/main/java/org/dbsyncer => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/connector/database/DatabaseTemplate.java (99%) rename {dbsyncer-connector/src/main/java/org/dbsyncer => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/connector/database/HandleCallback.java (69%) rename {dbsyncer-connector/src/main/java/org/dbsyncer => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/connector/database/ds/SimpleConnection.java (95%) rename {dbsyncer-connector/src/main/java/org/dbsyncer => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/connector/database/ds/SimpleDataSource.java (91%) rename {dbsyncer-connector/src/main/java/org/dbsyncer => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/connector/database/sqlbuilder/SqlBuilder.java (76%) rename {dbsyncer-connector/src/main/java/org/dbsyncer => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/connector/database/sqlbuilder/SqlBuilderDelete.java (77%) rename {dbsyncer-connector/src/main/java/org/dbsyncer => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/connector/database/sqlbuilder/SqlBuilderInsert.java (83%) rename {dbsyncer-connector/src/main/java/org/dbsyncer => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/connector/database/sqlbuilder/SqlBuilderQuery.java (87%) rename {dbsyncer-connector/src/main/java/org/dbsyncer => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/connector/database/sqlbuilder/SqlBuilderQueryCount.java (83%) rename {dbsyncer-connector/src/main/java/org/dbsyncer => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/connector/database/sqlbuilder/SqlBuilderQueryCursor.java (72%) rename {dbsyncer-connector/src/main/java/org/dbsyncer => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/connector/database/sqlbuilder/SqlBuilderQueryExist.java (78%) rename {dbsyncer-connector/src/main/java/org/dbsyncer => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/connector/database/sqlbuilder/SqlBuilderUpdate.java (83%) rename {dbsyncer-connector/src/main/java/org/dbsyncer => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/connector/schema/BigintValueMapper.java (62%) rename {dbsyncer-connector/src/main/java/org/dbsyncer => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/connector/schema/BinaryValueMapper.java (44%) rename {dbsyncer-connector/src/main/java/org/dbsyncer => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/connector/schema/BitValueMapper.java (64%) rename {dbsyncer-connector/src/main/java/org/dbsyncer => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/connector/schema/BlobValueMapper.java (44%) create mode 100644 dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/CharValueMapper.java rename {dbsyncer-connector/src/main/java/org/dbsyncer => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/connector/schema/ClobValueMapper.java (45%) rename {dbsyncer-connector/src/main/java/org/dbsyncer => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/connector/schema/DateValueMapper.java (64%) rename {dbsyncer-connector/src/main/java/org/dbsyncer => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/connector/schema/DecimalValueMapper.java (73%) rename {dbsyncer-connector/src/main/java/org/dbsyncer => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/connector/schema/DoubleValueMapper.java (45%) rename {dbsyncer-connector/src/main/java/org/dbsyncer => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/connector/schema/FloatValueMapper.java (53%) rename {dbsyncer-connector/src/main/java/org/dbsyncer => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/connector/schema/IntegerValueMapper.java (65%) rename {dbsyncer-connector/src/main/java/org/dbsyncer => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/connector/schema/LongVarBinaryValueMapper.java (41%) rename {dbsyncer-connector/src/main/java/org/dbsyncer => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/connector/schema/LongVarcharValueMapper.java (52%) create mode 100644 dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/NCharValueMapper.java rename {dbsyncer-connector/src/main/java/org/dbsyncer => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/connector/schema/NClobValueMapper.java (44%) create mode 100644 dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/NVarcharValueMapper.java rename {dbsyncer-connector/src/main/java/org/dbsyncer => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/connector/schema/NumberValueMapper.java (59%) create mode 100644 dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/OtherValueMapper.java rename {dbsyncer-connector/src/main/java/org/dbsyncer => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/connector/schema/RealValueMapper.java (42%) rename {dbsyncer-connector/src/main/java/org/dbsyncer => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/connector/schema/RowIdValueMapper.java (30%) rename {dbsyncer-connector/src/main/java/org/dbsyncer => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/connector/schema/SmallintValueMapper.java (35%) rename {dbsyncer-connector/src/main/java/org/dbsyncer => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/connector/schema/TimeValueMapper.java (30%) rename {dbsyncer-connector/src/main/java/org/dbsyncer => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/connector/schema/TimestampValueMapper.java (79%) rename {dbsyncer-connector/src/main/java/org/dbsyncer => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/connector/schema/TinyintValueMapper.java (60%) rename {dbsyncer-connector/src/main/java/org/dbsyncer => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/connector/schema/VarBinaryValueMapper.java (70%) rename {dbsyncer-connector/src/main/java/org/dbsyncer => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/connector/schema/VarcharValueMapper.java (75%) rename {dbsyncer-connector/src/main/java/org/dbsyncer/connector => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/constant/ConnectorConstant.java (96%) rename {dbsyncer-connector/src/main/java/org/dbsyncer/connector => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/constant/DatabaseConstant.java (97%) rename {dbsyncer-connector/src/main/java/org/dbsyncer/connector => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/enums/SqlBuilderEnum.java (66%) rename {dbsyncer-connector/src/main/java/org/dbsyncer/connector => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/model/PageSql.java (95%) rename {dbsyncer-connector/src/main/java/org/dbsyncer/connector => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/model/SqlTable.java (94%) delete mode 100644 dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/ConnectorMapper.java rename {dbsyncer-connector/src/main/java/org/dbsyncer/connector => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/util/DatabaseUtil.java (84%) rename {dbsyncer-connector/src/main/java/org/dbsyncer/connector => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/util/PrimaryKeyUtil.java (87%) diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/AbstractChecker.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/AbstractChecker.java index 556e5a6e..0ee63f9e 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/AbstractChecker.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/AbstractChecker.java @@ -1,11 +1,11 @@ package org.dbsyncer.biz.checker; import org.dbsyncer.biz.enums.SafeInfoEnum; +import org.dbsyncer.sdk.model.Filter; import org.dbsyncer.storage.impl.SnowflakeIdWorker; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.JsonUtil; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.model.Filter; import org.dbsyncer.parser.model.AbstractConfigModel; import org.dbsyncer.parser.model.ConfigModel; import org.dbsyncer.parser.model.Convert; diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/AbstractDataBaseConfigChecker.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/AbstractDataBaseConfigChecker.java index 58396ca7..156b000f 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/AbstractDataBaseConfigChecker.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/AbstractDataBaseConfigChecker.java @@ -3,8 +3,8 @@ package org.dbsyncer.biz.checker.impl.connector; import org.dbsyncer.biz.checker.ConnectorConfigChecker; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.JsonUtil; -import org.dbsyncer.connector.config.DatabaseConfig; -import org.dbsyncer.connector.model.SqlTable; +import org.dbsyncer.sdk.config.DatabaseConfig; +import org.dbsyncer.sdk.model.SqlTable; import org.springframework.util.Assert; import java.util.List; diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/ConnectorChecker.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/ConnectorChecker.java index 577f73ed..4aaddd5f 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/ConnectorChecker.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/ConnectorChecker.java @@ -5,12 +5,12 @@ import org.dbsyncer.biz.checker.AbstractChecker; import org.dbsyncer.biz.checker.ConnectorConfigChecker; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.connector.ConnectorFactory; -import org.dbsyncer.connector.enums.ConnectorEnum; import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.parser.model.ConfigModel; import org.dbsyncer.parser.model.Connector; +import org.dbsyncer.sdk.connector.ConnectorInstance; import org.dbsyncer.sdk.model.ConnectorConfig; -import org.dbsyncer.sdk.spi.ConnectorMapper; +import org.dbsyncer.sdk.spi.ConnectorService; import org.dbsyncer.storage.constant.ConfigConstant; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -59,8 +59,8 @@ public class ConnectorChecker extends AbstractChecker { checker.modify(config, params); // 获取表 - ConnectorMapper mapper = connectorFactory.connect(connector.getConfig()); - connector.setTable(connectorFactory.getTable(mapper)); + ConnectorInstance connectorInstance = connectorFactory.connect(connector.getConfig()); + connector.setTable(connectorFactory.getTable(connectorInstance)); // 修改基本配置 this.modifyConfigModel(connector, params); @@ -88,16 +88,18 @@ public class ConnectorChecker extends AbstractChecker { checker.modify(config, params); // 获取表 - ConnectorMapper mapper = connectorFactory.connect(config); - connector.setTable(connectorFactory.getTable(mapper)); + ConnectorInstance connectorInstance = connectorFactory.connect(config); + connector.setTable(connectorFactory.getTable(connectorInstance)); return connector; } private ConnectorConfig getConfig(String connectorType) { try { - ConnectorConfig config = ConnectorEnum.getConnectorEnum(connectorType).getConfigClass().newInstance(); - config.setConnectorType(connectorType); + ConnectorService connectorService = connectorFactory.getConnectorService(connectorType); + Class configClass = connectorService.getConfigClass(); + ConnectorConfig config = configClass.newInstance(); + config.setConnectorType(connectorService.getConnectorType()); return config; } catch (Exception e) { logger.error(e.getMessage()); diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlMySQLConfigChecker.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlMySQLConfigChecker.java index b4fb0bc1..64ce4ada 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlMySQLConfigChecker.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlMySQLConfigChecker.java @@ -1,6 +1,6 @@ package org.dbsyncer.biz.checker.impl.connector; -import org.dbsyncer.connector.config.DatabaseConfig; +import org.dbsyncer.sdk.config.DatabaseConfig; import org.springframework.stereotype.Component; import java.util.Map; diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlOracleConfigChecker.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlOracleConfigChecker.java index fa3c5368..14e6d45f 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlOracleConfigChecker.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlOracleConfigChecker.java @@ -1,7 +1,7 @@ package org.dbsyncer.biz.checker.impl.connector; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.config.DatabaseConfig; +import org.dbsyncer.sdk.config.DatabaseConfig; import org.springframework.stereotype.Component; import java.util.Map; diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlPostgreSQLConfigChecker.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlPostgreSQLConfigChecker.java index 74b8e497..1e6a00b6 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlPostgreSQLConfigChecker.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlPostgreSQLConfigChecker.java @@ -1,7 +1,7 @@ package org.dbsyncer.biz.checker.impl.connector; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.config.DatabaseConfig; +import org.dbsyncer.sdk.config.DatabaseConfig; import org.springframework.stereotype.Component; import java.util.Map; diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlSqlServerConfigChecker.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlSqlServerConfigChecker.java index 1a1dbdde..3b3f8de0 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlSqlServerConfigChecker.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlSqlServerConfigChecker.java @@ -1,6 +1,6 @@ package org.dbsyncer.biz.checker.impl.connector; -import org.dbsyncer.connector.config.DatabaseConfig; +import org.dbsyncer.sdk.config.DatabaseConfig; import org.springframework.stereotype.Component; import java.util.Map; diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/OracleConfigChecker.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/OracleConfigChecker.java index 944fa695..5f9c4081 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/OracleConfigChecker.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/OracleConfigChecker.java @@ -3,15 +3,15 @@ package org.dbsyncer.biz.checker.impl.connector; import org.dbsyncer.biz.enums.OracleIncrementEnum; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.config.DatabaseConfig; -import org.dbsyncer.connector.model.Field; -import org.dbsyncer.connector.model.MetaInfo; -import org.dbsyncer.connector.model.Table; +import org.dbsyncer.sdk.config.DatabaseConfig; import org.dbsyncer.parser.ParserComponent; import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.parser.model.FieldMapping; import org.dbsyncer.parser.model.Mapping; import org.dbsyncer.parser.model.TableGroup; +import org.dbsyncer.sdk.model.Field; +import org.dbsyncer.sdk.model.MetaInfo; +import org.dbsyncer.sdk.model.Table; import org.springframework.stereotype.Component; import org.springframework.util.Assert; diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/PostgreSQLConfigChecker.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/PostgreSQLConfigChecker.java index 382e6ed5..6f049bc5 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/PostgreSQLConfigChecker.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/PostgreSQLConfigChecker.java @@ -1,7 +1,7 @@ package org.dbsyncer.biz.checker.impl.connector; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.config.DatabaseConfig; +import org.dbsyncer.sdk.config.DatabaseConfig; import org.springframework.stereotype.Component; import java.util.Map; diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/SqlServerConfigChecker.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/SqlServerConfigChecker.java index c9e4cc2e..b6e7db1a 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/SqlServerConfigChecker.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/SqlServerConfigChecker.java @@ -1,6 +1,6 @@ package org.dbsyncer.biz.checker.impl.connector; -import org.dbsyncer.connector.config.DatabaseConfig; +import org.dbsyncer.sdk.config.DatabaseConfig; import org.springframework.stereotype.Component; import java.util.Map; diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/tablegroup/TableGroupChecker.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/tablegroup/TableGroupChecker.java index 79d07579..d1c62e97 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/tablegroup/TableGroupChecker.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/tablegroup/TableGroupChecker.java @@ -6,10 +6,10 @@ import org.dbsyncer.biz.checker.ConnectorConfigChecker; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.JsonUtil; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.model.Field; -import org.dbsyncer.connector.model.MetaInfo; -import org.dbsyncer.connector.model.Table; -import org.dbsyncer.connector.util.PrimaryKeyUtil; +import org.dbsyncer.sdk.model.Field; +import org.dbsyncer.sdk.model.MetaInfo; +import org.dbsyncer.sdk.model.Table; +import org.dbsyncer.sdk.util.PrimaryKeyUtil; import org.dbsyncer.parser.ParserComponent; import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.sdk.enums.ModelEnum; diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ConditionServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ConditionServiceImpl.java index cd94dd36..2f61d900 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ConditionServiceImpl.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ConditionServiceImpl.java @@ -3,9 +3,9 @@ package org.dbsyncer.biz.impl; import org.dbsyncer.biz.ConditionService; import org.dbsyncer.biz.vo.ConditionVo; import org.dbsyncer.connector.enums.FilterEnum; -import org.dbsyncer.connector.enums.OperationEnum; import org.dbsyncer.listener.enums.QuartzFilterEnum; import org.dbsyncer.parser.ProfileComponent; +import org.dbsyncer.sdk.enums.OperationEnum; import org.springframework.stereotype.Component; import javax.annotation.Resource; diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ConnectorServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ConnectorServiceImpl.java index 2c7d331c..13b10c37 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ConnectorServiceImpl.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ConnectorServiceImpl.java @@ -22,6 +22,7 @@ import org.springframework.util.Assert; import javax.annotation.Resource; import java.util.ArrayList; +import java.util.Collections; import java.util.Comparator; import java.util.HashSet; import java.util.LinkedHashMap; @@ -122,9 +123,9 @@ public class ConnectorServiceImpl extends BaseServiceImpl implements ConnectorSe @Override public List getConnectorTypeAll() { - List list = new ArrayList<>(); - profileComponent.getConnectorEnumAll().forEach(c -> list.add(c.getType())); - return list; + ArrayList connectorTypes = new ArrayList<>(connectorFactory.getConnectorTypeAll()); + Collections.sort(connectorTypes, Comparator.comparing(String::toString)); + return connectorTypes; } @Override diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/DataSyncServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/DataSyncServiceImpl.java index d8c507aa..23fe23d1 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/DataSyncServiceImpl.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/DataSyncServiceImpl.java @@ -10,13 +10,13 @@ import org.dbsyncer.common.util.DateFormatUtil; import org.dbsyncer.common.util.JsonUtil; import org.dbsyncer.common.util.NumberUtil; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.model.Field; import org.dbsyncer.listener.event.RowChangedEvent; import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.parser.flush.impl.BufferActuatorRouter; import org.dbsyncer.parser.model.Meta; import org.dbsyncer.parser.model.Picker; import org.dbsyncer.parser.model.TableGroup; +import org.dbsyncer.sdk.model.Field; import org.dbsyncer.storage.StorageService; import org.dbsyncer.storage.binlog.proto.BinlogMap; import org.dbsyncer.storage.constant.ConfigConstant; diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MappingServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MappingServiceImpl.java index 0ba9fa98..265a9e1e 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MappingServiceImpl.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MappingServiceImpl.java @@ -9,21 +9,21 @@ import org.dbsyncer.biz.checker.impl.mapping.MappingChecker; import org.dbsyncer.biz.vo.ConnectorVo; import org.dbsyncer.biz.vo.MappingVo; import org.dbsyncer.biz.vo.MetaVo; -import org.dbsyncer.storage.impl.SnowflakeIdWorker; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.JsonUtil; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.model.Table; import org.dbsyncer.manager.ManagerFactory; -import org.dbsyncer.parser.ProfileComponent; -import org.dbsyncer.sdk.enums.ModelEnum; import org.dbsyncer.parser.LogType; +import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.parser.model.ConfigModel; import org.dbsyncer.parser.model.Connector; import org.dbsyncer.parser.model.Mapping; import org.dbsyncer.parser.model.Meta; import org.dbsyncer.parser.model.TableGroup; +import org.dbsyncer.sdk.enums.ModelEnum; +import org.dbsyncer.sdk.model.Table; import org.dbsyncer.storage.constant.ConfigConstant; +import org.dbsyncer.storage.impl.SnowflakeIdWorker; import org.springframework.beans.BeanUtils; import org.springframework.stereotype.Service; import org.springframework.util.Assert; diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MetricReporter.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MetricReporter.java index b968916c..c598a895 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MetricReporter.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MetricReporter.java @@ -13,7 +13,7 @@ import org.dbsyncer.connector.scheduled.ScheduledTaskJob; import org.dbsyncer.connector.scheduled.ScheduledTaskService; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.constant.ConnectorConstant; +import org.dbsyncer.sdk.constant.ConnectorConstant; import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.parser.flush.BufferActuator; import org.dbsyncer.parser.flush.impl.BufferActuatorRouter; diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/TableGroupServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/TableGroupServiceImpl.java index 0d809fe4..a7b7971e 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/TableGroupServiceImpl.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/TableGroupServiceImpl.java @@ -5,11 +5,11 @@ import org.dbsyncer.biz.checker.Checker; import org.dbsyncer.biz.checker.impl.tablegroup.TableGroupChecker; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.model.Field; -import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.parser.LogType; +import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.parser.model.Mapping; import org.dbsyncer.parser.model.TableGroup; +import org.dbsyncer.sdk.model.Field; import org.dbsyncer.storage.constant.ConfigConstant; import org.springframework.stereotype.Service; import org.springframework.util.Assert; diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/vo/ConditionVo.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/vo/ConditionVo.java index a403a947..60233d4a 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/vo/ConditionVo.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/vo/ConditionVo.java @@ -1,8 +1,8 @@ package org.dbsyncer.biz.vo; import org.dbsyncer.connector.enums.FilterEnum; -import org.dbsyncer.connector.enums.OperationEnum; import org.dbsyncer.listener.enums.QuartzFilterEnum; +import org.dbsyncer.sdk.enums.OperationEnum; import java.util.List; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/Connector.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/Connector.java deleted file mode 100644 index f200f37b..00000000 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/Connector.java +++ /dev/null @@ -1,125 +0,0 @@ -package org.dbsyncer.connector; - -import org.dbsyncer.common.model.Result; -import org.dbsyncer.connector.config.CommandConfig; -import org.dbsyncer.connector.config.DDLConfig; -import org.dbsyncer.connector.config.ReaderConfig; -import org.dbsyncer.connector.config.WriterBatchConfig; -import org.dbsyncer.connector.model.MetaInfo; -import org.dbsyncer.connector.model.Table; -import org.dbsyncer.sdk.spi.ConnectorMapper; - -import java.util.List; -import java.util.Map; - -/** - * 连接器基础功能 - * - * @param ConnectorMapper - * @param ConnectorConfig - * @author AE86 - * @version 1.0.0 - * @date 2019/9/18 23:30 - */ -public interface Connector { - - /** - * 建立连接 - * - * @param config - * @return - */ - ConnectorMapper connect(C config); - - /** - * 断开连接 - * - * @param connectorMapper - */ - void disconnect(M connectorMapper); - - /** - * 检查连接器是否连接正常 - * - * @param connectorMapper - * @return - */ - boolean isAlive(M connectorMapper); - - /** - * 获取连接缓存key - * - * @param config - * @return - */ - String getConnectorMapperCacheKey(C config); - - /** - * 获取所有表名 - * - * @param connectorMapper - * @return - */ - List
    getTable(M connectorMapper); - - /** - * 获取表元信息 - * - * @param connectorMapper - * @param tableNamePattern - * @return - */ - MetaInfo getMetaInfo(M connectorMapper, String tableNamePattern); - - /** - * 获取总数 - * - * @param connectorMapper - * @param command - * @return - */ - long getCount(M connectorMapper, Map command); - - /** - * 分页获取数据源数据 - * - * @param config - * @return - */ - Result reader(M connectorMapper, ReaderConfig config); - - /** - * 批量写入目标源数据 - * - * @param config - * @return - */ - Result writer(M connectorMapper, WriterBatchConfig config); - - /** - * 执行DDL命令 - * - * @param connectorMapper - * @param ddlConfig - * @return - */ - default Result writerDDL(M connectorMapper, DDLConfig ddlConfig) { - throw new ConnectorException("Unsupported method."); - } - - /** - * 获取数据源同步参数 - * - * @param commandConfig - * @return - */ - Map getSourceCommand(CommandConfig commandConfig); - - /** - * 获取目标源同步参数 - * - * @param commandConfig - * @return - */ - Map getTargetCommand(CommandConfig commandConfig); -} \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/ConnectorFactory.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/ConnectorFactory.java index 9d44b7f4..df200505 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/ConnectorFactory.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/ConnectorFactory.java @@ -2,21 +2,27 @@ package org.dbsyncer.connector; import org.dbsyncer.common.model.Result; import org.dbsyncer.common.util.CollectionUtils; -import org.dbsyncer.connector.config.CommandConfig; -import org.dbsyncer.connector.config.DDLConfig; -import org.dbsyncer.connector.config.ReaderConfig; -import org.dbsyncer.connector.config.WriterBatchConfig; -import org.dbsyncer.connector.enums.ConnectorEnum; -import org.dbsyncer.connector.model.MetaInfo; -import org.dbsyncer.connector.model.Table; +import org.dbsyncer.sdk.config.CommandConfig; +import org.dbsyncer.sdk.config.DDLConfig; +import org.dbsyncer.sdk.config.ReaderConfig; +import org.dbsyncer.sdk.config.WriterBatchConfig; +import org.dbsyncer.sdk.connector.AbstractConnector; +import org.dbsyncer.sdk.connector.ConnectorInstance; import org.dbsyncer.sdk.model.ConnectorConfig; -import org.dbsyncer.sdk.spi.ConnectorMapper; +import org.dbsyncer.sdk.model.MetaInfo; +import org.dbsyncer.sdk.model.Table; +import org.dbsyncer.sdk.spi.ConnectorService; import org.springframework.beans.factory.DisposableBean; +import org.springframework.context.ApplicationContext; import org.springframework.util.Assert; +import javax.annotation.PostConstruct; +import javax.annotation.Resource; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; /** @@ -28,12 +34,28 @@ import java.util.concurrent.ConcurrentHashMap; */ public class ConnectorFactory implements DisposableBean { - private final Map connectorCache = new ConcurrentHashMap<>(); + private final Map pool = new ConcurrentHashMap<>(); + + private final Map service = new ConcurrentHashMap<>(); + + private final Set connectorTypes = new HashSet<>(); + + @Resource + private ApplicationContext applicationContext; + + @PostConstruct + private void init() { + Map beans = applicationContext.getBeansOfType(ConnectorService.class); + if (!CollectionUtils.isEmpty(beans)) { + beans.values().forEach(s -> service.putIfAbsent(s.getConnectorType(), s)); + } + service.values().forEach(s -> connectorTypes.add(s.getConnectorType())); + } @Override public void destroy() { - connectorCache.values().forEach(this::disconnect); - connectorCache.clear(); + pool.values().forEach(this::disconnect); + pool.clear(); } /** @@ -41,22 +63,22 @@ public class ConnectorFactory implements DisposableBean { * * @param config */ - public ConnectorMapper connect(ConnectorConfig config) { + public ConnectorInstance connect(ConnectorConfig config) { Assert.notNull(config, "ConnectorConfig can not be null."); - Connector connector = getConnector(config.getConnectorType()); - String cacheKey = connector.getConnectorMapperCacheKey(config); - if (!connectorCache.containsKey(cacheKey)) { - synchronized (connectorCache) { - if (!connectorCache.containsKey(cacheKey)) { - ConnectorMapper mapper = connector.connect(config); - Assert.isTrue(connector.isAlive(mapper), "连接配置异常"); - connectorCache.putIfAbsent(cacheKey, mapper); + ConnectorService connectorService = getConnectorService(config); + String cacheKey = connectorService.getConnectorInstanceCacheKey(config); + if (!pool.containsKey(cacheKey)) { + synchronized (pool) { + if (!pool.containsKey(cacheKey)) { + ConnectorInstance instance = connectorService.connect(config); + Assert.isTrue(connectorService.isAlive(instance), "连接配置异常"); + pool.putIfAbsent(cacheKey, instance); } } } try { - ConnectorMapper connectorMapper = connectorCache.get(cacheKey); - ConnectorMapper clone = (ConnectorMapper) connectorMapper.clone(); + ConnectorInstance connectorInstance = pool.get(cacheKey); + ConnectorInstance clone = (ConnectorInstance) connectorInstance.clone(); clone.setConfig(config); return clone; } catch (CloneNotSupportedException e) { @@ -72,12 +94,11 @@ public class ConnectorFactory implements DisposableBean { */ public void disconnect(ConnectorConfig config) { Assert.notNull(config, "ConnectorConfig can not be null."); - Connector connector = getConnector(config.getConnectorType()); - String cacheKey = connector.getConnectorMapperCacheKey(config); - ConnectorMapper connectorMapper = connectorCache.get(cacheKey); - if (connectorMapper != null) { - disconnect(connectorMapper); - connectorCache.remove(cacheKey); + String cacheKey = getConnectorService(config).getConnectorInstanceCacheKey(config); + ConnectorInstance connectorInstance = pool.get(cacheKey); + if (connectorInstance != null) { + disconnect(connectorInstance); + pool.remove(cacheKey); } } @@ -89,10 +110,10 @@ public class ConnectorFactory implements DisposableBean { */ public boolean isAlive(ConnectorConfig config) { Assert.notNull(config, "ConnectorConfig can not be null."); - Connector connector = getConnector(config.getConnectorType()); - String cacheKey = connector.getConnectorMapperCacheKey(config); - if (connectorCache.containsKey(cacheKey)) { - return connector.isAlive(connectorCache.get(cacheKey)); + ConnectorService connectorService = getConnectorService(config); + String cacheKey = connectorService.getConnectorInstanceCacheKey(config); + if (pool.containsKey(cacheKey)) { + return connectorService.isAlive(pool.get(cacheKey)); } return false; } @@ -100,25 +121,25 @@ public class ConnectorFactory implements DisposableBean { /** * 获取配置表 * - * @param connectionMapper + * @param connectorInstance * @return */ - public List
    getTable(ConnectorMapper connectionMapper) { - Assert.notNull(connectionMapper, "ConnectorMapper can not be null."); - return getConnector(connectionMapper).getTable(connectionMapper); + public List
    getTable(ConnectorInstance connectorInstance) { + Assert.notNull(connectorInstance, "ConnectorInstance can not be null."); + return getConnectorService(connectorInstance.getConfig()).getTable(connectorInstance); } /** * 获取配置表元信息 * - * @param connectionMapper + * @param connectorInstance * @param tableName * @return */ - public MetaInfo getMetaInfo(ConnectorMapper connectionMapper, String tableName) { - Assert.notNull(connectionMapper, "ConnectorMapper can not be null."); + public MetaInfo getMetaInfo(ConnectorInstance connectorInstance, String tableName) { + Assert.notNull(connectorInstance, "ConnectorInstance can not be null."); Assert.hasText(tableName, "tableName can not be empty."); - return getConnector(connectionMapper).getMetaInfo(connectionMapper, tableName); + return getConnectorService(connectorInstance.getConfig()).getMetaInfo(connectorInstance, tableName); } /** @@ -129,37 +150,43 @@ public class ConnectorFactory implements DisposableBean { * @return */ public Map getCommand(CommandConfig sourceCommandConfig, CommandConfig targetCommandConfig) { - String sType = sourceCommandConfig.getType(); - String tType = targetCommandConfig.getType(); + Assert.notNull(sourceCommandConfig, "SourceCommandConfig can not be null."); + Assert.notNull(targetCommandConfig, "TargetCommandConfig can not be null."); Map map = new HashMap<>(); - Map sCmd = getConnector(sType).getSourceCommand(sourceCommandConfig); + Map sCmd = getConnectorService(sourceCommandConfig.getConnectorType()).getSourceCommand(sourceCommandConfig); if (!CollectionUtils.isEmpty(sCmd)) { map.putAll(sCmd); } - Map tCmd = getConnector(tType).getTargetCommand(targetCommandConfig); + Map tCmd = getConnectorService(targetCommandConfig.getConnectorType()).getTargetCommand(targetCommandConfig); if (!CollectionUtils.isEmpty(tCmd)) { map.putAll(tCmd); } return map; } - public long getCount(ConnectorMapper connectorMapper, Map command) { - return getConnector(connectorMapper).getCount(connectorMapper, command); + public long getCount(ConnectorInstance connectorInstance, Map command) { + Assert.notNull(connectorInstance, "ConnectorInstance can not null"); + Assert.notNull(command, "command can not null"); + return getConnectorService(connectorInstance.getConfig()).getCount(connectorInstance, command); } - public Result reader(ConnectorMapper connectorMapper, ReaderConfig config) { - Result result = getConnector(connectorMapper).reader(connectorMapper, config); + public Result reader(ConnectorInstance connectorInstance, ReaderConfig config) { + Assert.notNull(connectorInstance, "ConnectorInstance can not null"); + Assert.notNull(config, "ReaderConfig can not null"); + Result result = getConnectorService(connectorInstance.getConfig()).reader(connectorInstance, config); Assert.notNull(result, "Connector reader result can not null"); return result; } - public Result writer(ConnectorMapper connectorMapper, WriterBatchConfig config) { - Connector connector = getConnector(connectorMapper); + public Result writer(ConnectorInstance connectorInstance, WriterBatchConfig config) { + Assert.notNull(connectorInstance, "ConnectorInstance can not null"); + Assert.notNull(config, "WriterBatchConfig can not null"); + ConnectorService connector = getConnectorService(connectorInstance.getConfig()); if (connector instanceof AbstractConnector) { AbstractConnector conn = (AbstractConnector) connector; try { - conn.convertProcessBeforeWriter(connectorMapper, config); + conn.convertProcessBeforeWriter(connectorInstance, config); } catch (Exception e) { Result result = new Result(); result.getError().append(e.getMessage()); @@ -168,34 +195,43 @@ public class ConnectorFactory implements DisposableBean { } } - Result result = connector.writer(connectorMapper, config); + Result result = connector.writer(connectorInstance, config); Assert.notNull(result, "Connector writer batch result can not null"); return result; } - public Result writerDDL(ConnectorMapper connectorMapper, DDLConfig ddlConfig) { - Connector connector = getConnector(connectorMapper); - Result result = connector.writerDDL(connectorMapper, ddlConfig); + public Result writerDDL(ConnectorInstance connectorInstance, DDLConfig ddlConfig) { + Assert.notNull(connectorInstance, "ConnectorInstance can not null"); + Result result = getConnectorService(connectorInstance.getConfig()).writerDDL(connectorInstance, ddlConfig); Assert.notNull(result, "Connector writer batch result can not null"); return result; } - public Connector getConnector(ConnectorMapper connectorMapper) { - ConnectorConfig connectorConfig = (ConnectorConfig) connectorMapper.getConfig(); - return getConnector(connectorConfig.getConnectorType()); + private ConnectorService getConnectorService(ConnectorConfig connectorConfig) { + Assert.notNull(connectorConfig, "ConnectorConfig can not null"); + return getConnectorService(connectorConfig.getConnectorType()); } - public Connector getConnector(String connectorType) { - return ConnectorEnum.getConnectorEnum(connectorType).getConnector(); + public ConnectorService getConnectorService(String connectorType) { + ConnectorService connectorService = service.get(connectorType); + if (connectorService == null) { + Assert.isTrue(false, "Unsupported connector type:" + connectorType); + } + return connectorService; + } + + public Set getConnectorTypeAll() { + return connectorTypes; } /** * 断开连接 * - * @param connectorMapper + * @param connectorInstance */ - private void disconnect(ConnectorMapper connectorMapper) { - Assert.notNull(connectorMapper, "ConnectorMapper can not be null."); - getConnector(connectorMapper).disconnect(connectorMapper); + private void disconnect(ConnectorInstance connectorInstance) { + Assert.notNull(connectorInstance, "ConnectorInstance can not be null."); + getConnectorService(connectorInstance.getConfig()).disconnect(connectorInstance); } + } \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/CommandConfig.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/CommandConfig.java deleted file mode 100644 index 8a65eab2..00000000 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/CommandConfig.java +++ /dev/null @@ -1,48 +0,0 @@ -package org.dbsyncer.connector.config; - -import org.dbsyncer.connector.model.Filter; -import org.dbsyncer.connector.model.Table; -import org.dbsyncer.sdk.model.ConnectorConfig; - -import java.util.List; - -/** - * 查询同步参数模板 - * - * @author AE86 - * @version 1.0.0 - * @date 2019/9/16 23:59 - */ -public class CommandConfig { - - private String type; - - private Table table; - - private List filter; - - private ConnectorConfig connectorConfig; - - public CommandConfig(String type, Table table, ConnectorConfig connectorConfig, List filter) { - this.type = type; - this.table = table; - this.filter = filter; - this.connectorConfig = connectorConfig; - } - - public String getType() { - return type; - } - - public Table getTable() { - return table; - } - - public List getFilter() { - return filter; - } - - public ConnectorConfig getConnectorConfig() { - return connectorConfig; - } -} \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/DDLConfig.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/DDLConfig.java deleted file mode 100644 index 41e80156..00000000 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/DDLConfig.java +++ /dev/null @@ -1,74 +0,0 @@ -package org.dbsyncer.connector.config; - -import org.dbsyncer.connector.enums.DDLOperationEnum; -import org.dbsyncer.connector.model.Field; - -import java.util.LinkedList; -import java.util.List; - -public class DDLConfig { - /** - * 执行命令 - */ - private String sql; - - private DDLOperationEnum ddlOperationEnum; - - private List addFields = new LinkedList<>(); - - private List removeFields = new LinkedList<>(); - - //记录源表的源字段名称 - private String sourceColumnName; - - //记录改变后的字段名称 - private String changedColumnName; - - public String getSql() { - return sql; - } - - public void setSql(String sql) { - this.sql = sql; - } - - public List getAddFields() { - return addFields; - } - - public void setAddFields(List addFields) { - this.addFields = addFields; - } - - public List getRemoveFields() { - return removeFields; - } - - public void setRemoveFields(List removeFields) { - this.removeFields = removeFields; - } - - public String getSourceColumnName() { - return sourceColumnName; - } - - public void setSourceColumnName(String sourceColumnName) { - this.sourceColumnName = sourceColumnName; - } - - public String getChangedColumnName() { - return changedColumnName; - } - - public void setChangedColumnName(String changedColumnName) { - this.changedColumnName = changedColumnName; - } - - public DDLOperationEnum getDdlOperationEnum() { - return ddlOperationEnum; - } - - public void setDdlOperationEnum(DDLOperationEnum ddlOperationEnum) { - this.ddlOperationEnum = ddlOperationEnum; - } -} \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/ReaderConfig.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/ReaderConfig.java deleted file mode 100644 index a2142cb3..00000000 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/ReaderConfig.java +++ /dev/null @@ -1,55 +0,0 @@ -package org.dbsyncer.connector.config; - -import org.dbsyncer.connector.model.Table; - -import java.util.List; -import java.util.Map; - -public class ReaderConfig { - - private Table table; - private Map command; - private List args; - private boolean supportedCursor; - private Object[] cursors; - private int pageIndex; - private int pageSize; - - public ReaderConfig(Table table, Map command, List args, boolean supportedCursor, Object[] cursors, int pageIndex, int pageSize) { - this.table = table; - this.command = command; - this.args = args; - this.supportedCursor = supportedCursor; - this.cursors = cursors; - this.pageIndex = pageIndex; - this.pageSize = pageSize; - } - - public Table getTable() { - return table; - } - - public Map getCommand() { - return command; - } - - public List getArgs() { - return args; - } - - public Object[] getCursors() { - return cursors; - } - - public int getPageIndex() { - return pageIndex; - } - - public int getPageSize() { - return pageSize; - } - - public boolean isSupportedCursor() { - return supportedCursor; - } -} \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/WriterBatchConfig.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/WriterBatchConfig.java deleted file mode 100644 index ac0af83d..00000000 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/WriterBatchConfig.java +++ /dev/null @@ -1,59 +0,0 @@ -package org.dbsyncer.connector.config; - -import org.dbsyncer.connector.model.Field; - -import java.util.List; -import java.util.Map; - -public class WriterBatchConfig { - - /** - * 表名 - */ - private String tableName; - /** - * 事件 - */ - private String event; - /** - * 执行命令 - */ - private Map command; - /** - * 字段信息 - */ - private List fields; - /** - * 集合数据 - */ - private List data; - - public WriterBatchConfig(String tableName, String event, Map command, List fields, List data) { - this.tableName = tableName; - this.event = event; - this.command = command; - this.fields = fields; - this.data = data; - } - - public String getTableName() { - return tableName; - } - - public String getEvent() { - return event; - } - - public Map getCommand() { - return command; - } - - public List getFields() { - return fields; - } - - public List getData() { - return data; - } - -} \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/enums/ConnectorEnum.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/enums/ConnectorEnum.java deleted file mode 100644 index b095b871..00000000 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/enums/ConnectorEnum.java +++ /dev/null @@ -1,119 +0,0 @@ -package org.dbsyncer.connector.enums; - -import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.Connector; -import org.dbsyncer.connector.ConnectorException; -import org.dbsyncer.connector.config.DatabaseConfig; -import org.dbsyncer.connector.config.ESConfig; -import org.dbsyncer.connector.config.FileConfig; -import org.dbsyncer.connector.config.KafkaConfig; -import org.dbsyncer.connector.es.ESConnector; -import org.dbsyncer.connector.file.FileConnector; -import org.dbsyncer.connector.kafka.KafkaConnector; -import org.dbsyncer.connector.mysql.MySQLConnector; -import org.dbsyncer.connector.oracle.OracleConnector; -import org.dbsyncer.connector.postgresql.PostgreSQLConnector; -import org.dbsyncer.connector.sql.DQLMySQLConnector; -import org.dbsyncer.connector.sql.DQLOracleConnector; -import org.dbsyncer.connector.sql.DQLPostgreSQLConnector; -import org.dbsyncer.connector.sql.DQLSqlServerConnector; -import org.dbsyncer.connector.sqlserver.SqlServerConnector; -import org.dbsyncer.sdk.model.ConnectorConfig; - -/** - * 支持的连接器类型 - * - * @author AE86 - * @version 1.0.0 - * @date 2019/9/19 23:56 - */ -public enum ConnectorEnum { - - /** - * MySQL 连接器 - */ - MYSQL("MySQL", new MySQLConnector(), DatabaseConfig.class), - /** - * Oracle 连接器 - */ - ORACLE("Oracle", new OracleConnector(), DatabaseConfig.class), - /** - * SqlServer 连接器 - */ - SQL_SERVER("SqlServer", new SqlServerConnector(), DatabaseConfig.class), - /** - * PostgreSQL 连接器 - */ - POSTGRE_SQL("PostgreSQL", new PostgreSQLConnector(), DatabaseConfig.class), - /** - * Elasticsearch 连接器 - */ - ELASTIC_SEARCH("Elasticsearch", new ESConnector(), ESConfig.class), - /** - * Kafka 连接器 - */ - KAFKA("Kafka", new KafkaConnector(), KafkaConfig.class), - /** - * File 连接器 - */ - FILE("File", new FileConnector(), FileConfig.class), - /** - * DqlMySQL 连接器 - */ - DQL_MYSQL("DqlMySQL", new DQLMySQLConnector(), DatabaseConfig.class), - /** - * DqlOracle 连接器 - */ - DQL_ORACLE("DqlOracle", new DQLOracleConnector(), DatabaseConfig.class), - /** - * DqlSqlServer 连接器 - */ - DQL_SQL_SERVER("DqlSqlServer", new DQLSqlServerConnector(), DatabaseConfig.class), - /** - * DqlPostgreSQL 连接器 - */ - DQL_POSTGRE_SQL("DqlPostgreSQL", new DQLPostgreSQLConnector(), DatabaseConfig.class); - - // 连接器名称 - private String type; - - // 连接器 - private Connector connector; - - // 配置 - private Class configClass; - - ConnectorEnum(String type, Connector connector, Class configClass) { - this.type = type; - this.connector = connector; - this.configClass = configClass; - } - - /** - * 获取连接枚举 - * - * @param type - * @return - * @throws ConnectorException - */ - public static ConnectorEnum getConnectorEnum(String type) throws ConnectorException { - for (ConnectorEnum e : ConnectorEnum.values()) { - if (StringUtil.equalsIgnoreCase(type, e.getType())) { - return e; - } - } - throw new ConnectorException(String.format("Connector type \"%s\" does not exist.", type)); - } - - public String getType() { - return type; - } - - public Connector getConnector() { - return connector; - } - - public Class getConfigClass() { - return configClass; - } -} \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/enums/DDLOperationEnum.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/enums/DDLOperationEnum.java deleted file mode 100644 index 826f49af..00000000 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/enums/DDLOperationEnum.java +++ /dev/null @@ -1,16 +0,0 @@ -package org.dbsyncer.connector.enums; - - -/** - * 支持同步的DDL命令 - * - * @version 1.0.0 - * @Author life - * @Date 2023-09-24 14:24 - */ -public enum DDLOperationEnum { - ALTER_MODIFY, - ALTER_ADD, - ALTER_DROP, - ALTER_CHANGE; -} \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/enums/OperationEnum.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/enums/OperationEnum.java deleted file mode 100644 index 0d9e8a61..00000000 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/enums/OperationEnum.java +++ /dev/null @@ -1,40 +0,0 @@ -package org.dbsyncer.connector.enums; - -/** - * 条件表达式类型 - * - * @author AE86 - * @version 1.0.0 - * @date 2019/9/26 23:21 - */ -public enum OperationEnum { - - /** - * 并且 - */ - AND("and"), - /** - * 或者 - */ - OR("or"); - - // 描述 - private String name; - - OperationEnum(String name) { - this.name = name; - } - - public static boolean isAnd(String name) { - return AND.getName().equals(name); - } - - public static boolean isOr(String name) { - return OR.getName().equals(name); - } - - public String getName() { - return name; - } - -} \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/enums/TableTypeEnum.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/enums/TableTypeEnum.java deleted file mode 100644 index 4e93dc21..00000000 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/enums/TableTypeEnum.java +++ /dev/null @@ -1,47 +0,0 @@ -package org.dbsyncer.connector.enums; - -/** - * 表类型 - * - * @author AE86 - * @version 1.0.0 - * @date 2021/08/26 21:13 - */ -public enum TableTypeEnum { - - /** - * 表 - */ - TABLE("TABLE"), - - /** - * 视图 - */ - VIEW("VIEW"), - - /** - * 物化视图 - */ - MATERIALIZED_VIEW("MATERIALIZED VIEW"); - - private String code; - - TableTypeEnum(String code) { - this.code = code; - } - - /** - * 是否视图类型 - * - * @param type - * @return - */ - public static boolean isView(String type) { - return VIEW.getCode().equals(type); - } - - public String getCode() { - return code; - } - -} \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESConnector.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESConnector.java index 3543fd77..01fe6ed5 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESConnector.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESConnector.java @@ -4,24 +4,24 @@ import org.dbsyncer.common.model.Result; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.JsonUtil; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.AbstractConnector; -import org.dbsyncer.connector.Connector; import org.dbsyncer.connector.ConnectorException; -import org.dbsyncer.connector.config.CommandConfig; import org.dbsyncer.connector.config.ESConfig; -import org.dbsyncer.connector.config.ReaderConfig; -import org.dbsyncer.connector.config.WriterBatchConfig; -import org.dbsyncer.connector.constant.ConnectorConstant; import org.dbsyncer.connector.enums.ESFieldTypeEnum; import org.dbsyncer.connector.enums.FilterEnum; -import org.dbsyncer.connector.enums.OperationEnum; -import org.dbsyncer.connector.model.Field; -import org.dbsyncer.connector.model.Filter; -import org.dbsyncer.connector.model.MetaInfo; -import org.dbsyncer.connector.model.Table; import org.dbsyncer.connector.util.ESUtil; -import org.dbsyncer.connector.util.PrimaryKeyUtil; -import org.dbsyncer.sdk.spi.ConnectorMapper; +import org.dbsyncer.sdk.config.CommandConfig; +import org.dbsyncer.sdk.config.ReaderConfig; +import org.dbsyncer.sdk.config.WriterBatchConfig; +import org.dbsyncer.sdk.connector.AbstractConnector; +import org.dbsyncer.sdk.connector.ConnectorInstance; +import org.dbsyncer.sdk.constant.ConnectorConstant; +import org.dbsyncer.sdk.enums.OperationEnum; +import org.dbsyncer.sdk.model.Field; +import org.dbsyncer.sdk.model.Filter; +import org.dbsyncer.sdk.model.MetaInfo; +import org.dbsyncer.sdk.model.Table; +import org.dbsyncer.sdk.spi.ConnectorService; +import org.dbsyncer.sdk.util.PrimaryKeyUtil; import org.elasticsearch.Version; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; @@ -47,7 +47,9 @@ import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.sort.SortOrder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; +import javax.annotation.PostConstruct; import java.io.IOException; import java.sql.Types; import java.util.ArrayList; @@ -58,13 +60,20 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; -public final class ESConnector extends AbstractConnector implements Connector { +@Component +public final class ESConnector extends AbstractConnector implements ConnectorService { private final Logger logger = LoggerFactory.getLogger(getClass()); - private static Map filters = new LinkedHashMap<>(); + private final String TYPE = "Elasticsearch"; + + private final Map filters = new LinkedHashMap<>(); + + @PostConstruct + private void init() { + VALUE_MAPPERS.put(Types.DATE, new ESDateValueMapper()); + VALUE_MAPPERS.put(Types.OTHER, new ESOtherValueMapper()); - static { filters.putIfAbsent(FilterEnum.EQUAL.getName(), (builder, k, v) -> builder.must(QueryBuilders.matchQuery(k, v))); filters.putIfAbsent(FilterEnum.NOT_EQUAL.getName(), (builder, k, v) -> builder.mustNot(QueryBuilders.matchQuery(k, v))); filters.putIfAbsent(FilterEnum.GT.getName(), (builder, k, v) -> builder.filter(QueryBuilders.rangeQuery(k).gt(v))); @@ -74,23 +83,38 @@ public final class ESConnector extends AbstractConnector implements Connector builder.filter(QueryBuilders.wildcardQuery(k, v))); } - public ESConnector() { - VALUE_MAPPERS.put(Types.DATE, new ESDateValueMapper()); - VALUE_MAPPERS.put(Types.OTHER, new ESOtherValueMapper()); + @Override + public String getConnectorType() { + return TYPE; + } + + @Override + public boolean isSupportedTiming() { + return true; + } + + @Override + public boolean isSupportedLog() { + return false; + } + + @Override + public Class getConfigClass() { + return ESConfig.class; } @Override - public ConnectorMapper connect(ESConfig config) { - return new ESConnectorMapper(config); + public ConnectorInstance connect(ESConfig config) { + return new ESConnectorInstance(config); } @Override - public void disconnect(ESConnectorMapper connectorMapper) { + public void disconnect(ESConnectorInstance connectorMapper) { connectorMapper.close(); } @Override - public boolean isAlive(ESConnectorMapper connectorMapper) { + public boolean isAlive(ESConnectorInstance connectorMapper) { try { RestHighLevelClient client = connectorMapper.getConnection(); return client.ping(RequestOptions.DEFAULT); @@ -101,12 +125,12 @@ public final class ESConnector extends AbstractConnector implements Connector getTable(ESConnectorMapper connectorMapper) { + public List
    getTable(ESConnectorInstance connectorMapper) { try { ESConfig config = connectorMapper.getConfig(); GetIndexRequest request = new GetIndexRequest(config.getIndex()); @@ -130,7 +154,7 @@ public final class ESConnector extends AbstractConnector implements Connector fields = new ArrayList<>(); try { ESConfig config = connectorMapper.getConfig(); @@ -154,7 +178,7 @@ public final class ESConnector extends AbstractConnector implements Connector command) { + public long getCount(ESConnectorInstance connectorMapper, Map command) { try { ESConfig config = connectorMapper.getConfig(); SearchSourceBuilder builder = new SearchSourceBuilder(); @@ -165,7 +189,7 @@ public final class ESConnector extends AbstractConnector implements Connector data = config.getData(); if (CollectionUtils.isEmpty(data) || CollectionUtils.isEmpty(config.getFields())) { logger.error("writer data can not be empty."); diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESConnectorMapper.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESConnectorInstance.java similarity index 84% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESConnectorMapper.java rename to dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESConnectorInstance.java index 6ac7bbef..2412ceab 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESConnectorMapper.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESConnectorInstance.java @@ -1,59 +1,59 @@ -package org.dbsyncer.connector.es; - -import org.dbsyncer.connector.ConnectorException; -import org.dbsyncer.connector.config.ESConfig; -import org.dbsyncer.connector.util.ESUtil; -import org.dbsyncer.sdk.spi.ConnectorMapper; -import org.elasticsearch.Version; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.core.MainResponse; - -public final class ESConnectorMapper implements ConnectorMapper { - private ESConfig config; - private EasyRestHighLevelClient client; - - public ESConnectorMapper(ESConfig config) { - this.config = config; - this.client = ESUtil.getConnection(config); - try { - MainResponse info = client.info(RequestOptions.DEFAULT); - client.setVersion(Version.fromString(info.getVersion().getNumber())); - } catch (Exception e) { - throw new ConnectorException(String.format("获取ES版本信息异常 %s, %s", config.getUrl(), e.getMessage())); - } - } - - @Override - public String getServiceUrl() { - return config.getUrl(); - } - - @Override - public ESConfig getConfig() { - return config; - } - - @Override - public void setConfig(ESConfig config) { - this.config = config; - } - - @Override - public EasyRestHighLevelClient getConnection() { - return client; - } - - public Version getVersion() { - return client.getVersion(); - } - - @Override - public void close() { - ESUtil.close(client); - } - - @Override - public Object clone() throws CloneNotSupportedException { - return super.clone(); - } +package org.dbsyncer.connector.es; + +import org.dbsyncer.connector.ConnectorException; +import org.dbsyncer.connector.config.ESConfig; +import org.dbsyncer.connector.util.ESUtil; +import org.dbsyncer.sdk.connector.ConnectorInstance; +import org.elasticsearch.Version; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.core.MainResponse; + +public final class ESConnectorInstance implements ConnectorInstance { + private ESConfig config; + private EasyRestHighLevelClient client; + + public ESConnectorInstance(ESConfig config) { + this.config = config; + this.client = ESUtil.getConnection(config); + try { + MainResponse info = client.info(RequestOptions.DEFAULT); + client.setVersion(Version.fromString(info.getVersion().getNumber())); + } catch (Exception e) { + throw new ConnectorException(String.format("获取ES版本信息异常 %s, %s", config.getUrl(), e.getMessage())); + } + } + + @Override + public String getServiceUrl() { + return config.getUrl(); + } + + @Override + public ESConfig getConfig() { + return config; + } + + @Override + public void setConfig(ESConfig config) { + this.config = config; + } + + @Override + public EasyRestHighLevelClient getConnection() { + return client; + } + + public Version getVersion() { + return client.getVersion(); + } + + @Override + public void close() { + ESUtil.close(client); + } + + @Override + public Object clone() throws CloneNotSupportedException { + return super.clone(); + } } \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESDateValueMapper.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESDateValueMapper.java index 04010769..99e963f6 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESDateValueMapper.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESDateValueMapper.java @@ -1,8 +1,8 @@ package org.dbsyncer.connector.es; -import org.dbsyncer.connector.AbstractValueMapper; import org.dbsyncer.connector.ConnectorException; -import org.dbsyncer.sdk.spi.ConnectorMapper; +import org.dbsyncer.sdk.connector.AbstractValueMapper; +import org.dbsyncer.sdk.connector.ConnectorInstance; import java.sql.Date; import java.sql.Timestamp; @@ -15,7 +15,7 @@ import java.sql.Timestamp; public class ESDateValueMapper extends AbstractValueMapper { @Override - protected java.util.Date convert(ConnectorMapper connectorMapper, Object val) { + protected java.util.Date convert(ConnectorInstance connectorInstance, Object val) { if (val instanceof Timestamp) { Timestamp timestamp = (Timestamp) val; return new java.util.Date(timestamp.getTime()); diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESOtherValueMapper.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESOtherValueMapper.java index 96d50e45..7b2ab9cc 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESOtherValueMapper.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESOtherValueMapper.java @@ -1,9 +1,9 @@ package org.dbsyncer.connector.es; import org.dbsyncer.common.util.JsonUtil; -import org.dbsyncer.connector.AbstractValueMapper; import org.dbsyncer.connector.ConnectorException; -import org.dbsyncer.sdk.spi.ConnectorMapper; +import org.dbsyncer.sdk.connector.AbstractValueMapper; +import org.dbsyncer.sdk.connector.ConnectorInstance; import org.postgresql.util.PGobject; import java.util.Map; @@ -16,7 +16,7 @@ import java.util.Map; public class ESOtherValueMapper extends AbstractValueMapper { @Override - protected Map convert(ConnectorMapper connectorMapper, Object val) { + protected Map convert(ConnectorInstance connectorInstance, Object val) { if (val instanceof String) { return JsonUtil.jsonToObj((String) val, Map.class); } diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileConnector.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileConnector.java index a5d178ab..78c9031d 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileConnector.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileConnector.java @@ -5,20 +5,21 @@ import org.apache.commons.io.LineIterator; import org.dbsyncer.common.model.Result; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.AbstractConnector; -import org.dbsyncer.connector.Connector; import org.dbsyncer.connector.ConnectorException; -import org.dbsyncer.connector.config.CommandConfig; import org.dbsyncer.connector.config.FileConfig; -import org.dbsyncer.connector.config.ReaderConfig; -import org.dbsyncer.connector.config.WriterBatchConfig; -import org.dbsyncer.connector.model.Field; import org.dbsyncer.connector.model.FileSchema; -import org.dbsyncer.connector.model.MetaInfo; -import org.dbsyncer.connector.model.Table; -import org.dbsyncer.sdk.spi.ConnectorMapper; +import org.dbsyncer.sdk.config.CommandConfig; +import org.dbsyncer.sdk.config.ReaderConfig; +import org.dbsyncer.sdk.config.WriterBatchConfig; +import org.dbsyncer.sdk.connector.AbstractConnector; +import org.dbsyncer.sdk.connector.ConnectorInstance; +import org.dbsyncer.sdk.model.Field; +import org.dbsyncer.sdk.model.MetaInfo; +import org.dbsyncer.sdk.model.Table; +import org.dbsyncer.sdk.spi.ConnectorService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; import org.springframework.util.Assert; import java.io.File; @@ -40,26 +41,48 @@ import java.util.stream.Collectors; * @version 1.0.0 * @date 2022/5/5 23:19 */ -public final class FileConnector extends AbstractConnector implements Connector { +@Component +public final class FileConnector extends AbstractConnector implements ConnectorService { private final Logger logger = LoggerFactory.getLogger(getClass()); - private static final String FILE_NAME = "fileName"; - private static final String FILE_PATH = "filePath"; + private final String TYPE = "File"; + private final String FILE_NAME = "fileName"; + private final String FILE_PATH = "filePath"; private final FileResolver fileResolver = new FileResolver(); @Override - public ConnectorMapper connect(FileConfig config) { - return new FileConnectorMapper(config); + public String getConnectorType() { + return TYPE; } @Override - public void disconnect(FileConnectorMapper connectorMapper) { + public boolean isSupportedTiming() { + return false; + } + + @Override + public boolean isSupportedLog() { + return true; + } + + @Override + public Class getConfigClass() { + return FileConfig.class; + } + + @Override + public ConnectorInstance connect(FileConfig config) { + return new FileConnectorInstance(config); + } + + @Override + public void disconnect(FileConnectorInstance connectorMapper) { } @Override - public boolean isAlive(FileConnectorMapper connectorMapper) { + public boolean isAlive(FileConnectorInstance connectorMapper) { String fileDir = connectorMapper.getConnection(); boolean alive = new File(fileDir).exists(); if (!alive) { @@ -77,7 +100,7 @@ public final class FileConnector extends AbstractConnector implements Connector< } @Override - public String getConnectorMapperCacheKey(FileConfig config) { + public String getConnectorInstanceCacheKey(FileConfig config) { String localIP; try { localIP = InetAddress.getLocalHost().getHostAddress(); @@ -89,18 +112,18 @@ public final class FileConnector extends AbstractConnector implements Connector< } @Override - public List
    getTable(FileConnectorMapper connectorMapper) { + public List
    getTable(FileConnectorInstance connectorMapper) { return connectorMapper.getFileSchemaList().stream().map(fileSchema -> new Table(fileSchema.getFileName())).collect(Collectors.toList()); } @Override - public MetaInfo getMetaInfo(FileConnectorMapper connectorMapper, String tableName) { + public MetaInfo getMetaInfo(FileConnectorInstance connectorMapper, String tableName) { FileSchema fileSchema = connectorMapper.getFileSchema(tableName); return new MetaInfo().setColumn(fileSchema.getFields()); } @Override - public long getCount(FileConnectorMapper connectorMapper, Map command) { + public long getCount(FileConnectorInstance connectorMapper, Map command) { AtomicLong count = new AtomicLong(); FileReader reader = null; try { @@ -119,7 +142,7 @@ public final class FileConnector extends AbstractConnector implements Connector< } @Override - public Result reader(FileConnectorMapper connectorMapper, ReaderConfig config) { + public Result reader(FileConnectorInstance connectorMapper, ReaderConfig config) { List> list = new ArrayList<>(); FileReader reader = null; try { @@ -154,7 +177,7 @@ public final class FileConnector extends AbstractConnector implements Connector< } @Override - public Result writer(FileConnectorMapper connectorMapper, WriterBatchConfig config) { + public Result writer(FileConnectorInstance connectorMapper, WriterBatchConfig config) { List data = config.getData(); if (CollectionUtils.isEmpty(data)) { logger.error("writer data can not be empty."); @@ -162,7 +185,7 @@ public final class FileConnector extends AbstractConnector implements Connector< } final List fields = config.getFields(); - final String separator = new String(new char[] {connectorMapper.getConfig().getSeparator()}); + final String separator = new String(new char[]{connectorMapper.getConfig().getSeparator()}); Result result = new Result(); OutputStream output = null; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileConnectorMapper.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileConnectorInstance.java similarity index 91% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileConnectorMapper.java rename to dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileConnectorInstance.java index fde0a082..01d4c5de 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileConnectorMapper.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileConnectorInstance.java @@ -2,9 +2,9 @@ package org.dbsyncer.connector.file; import org.dbsyncer.common.util.JsonUtil; import org.dbsyncer.connector.config.FileConfig; -import org.dbsyncer.connector.model.Field; import org.dbsyncer.connector.model.FileSchema; -import org.dbsyncer.sdk.spi.ConnectorMapper; +import org.dbsyncer.sdk.connector.ConnectorInstance; +import org.dbsyncer.sdk.model.Field; import org.springframework.util.Assert; import java.io.File; @@ -17,13 +17,13 @@ import java.util.concurrent.ConcurrentHashMap; * @version 1.0.0 * @date 2022/5/5 23:19 */ -public final class FileConnectorMapper implements ConnectorMapper { +public final class FileConnectorInstance implements ConnectorInstance { private FileConfig config; private List fileSchemaList; private Map fileSchemaMap = new ConcurrentHashMap<>(); - public FileConnectorMapper(FileConfig config) { + public FileConnectorInstance(FileConfig config) { this.config = config; fileSchemaList = JsonUtil.jsonToArray(config.getSchema(), FileSchema.class); Assert.notEmpty(fileSchemaList, "The schema is empty."); diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileResolver.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileResolver.java index a743fc6c..36172438 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileResolver.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileResolver.java @@ -3,7 +3,7 @@ package org.dbsyncer.connector.file; import org.dbsyncer.common.column.Lexer; import org.dbsyncer.connector.file.column.ColumnValue; import org.dbsyncer.connector.file.column.FileColumnValue; -import org.dbsyncer.connector.model.Field; +import org.dbsyncer.sdk.model.Field; import java.util.ArrayList; import java.util.LinkedHashMap; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/column/FileColumnValue.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/column/FileColumnValue.java index 081c3cdb..a87a8537 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/column/FileColumnValue.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/column/FileColumnValue.java @@ -6,7 +6,6 @@ import org.dbsyncer.common.util.StringUtil; import java.sql.Date; import java.sql.Timestamp; -import java.time.LocalDateTime; /** * @author AE86 diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/kafka/KafkaConnector.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/kafka/KafkaConnector.java index 2510da32..a730b35c 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/kafka/KafkaConnector.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/kafka/KafkaConnector.java @@ -3,76 +3,100 @@ package org.dbsyncer.connector.kafka; import org.dbsyncer.common.model.Result; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.JsonUtil; -import org.dbsyncer.connector.AbstractConnector; -import org.dbsyncer.connector.Connector; +import org.dbsyncer.sdk.config.CommandConfig; +import org.dbsyncer.sdk.config.ReaderConfig; +import org.dbsyncer.sdk.config.WriterBatchConfig; +import org.dbsyncer.sdk.connector.AbstractConnector; import org.dbsyncer.connector.ConnectorException; -import org.dbsyncer.connector.config.CommandConfig; import org.dbsyncer.connector.config.KafkaConfig; -import org.dbsyncer.connector.config.ReaderConfig; -import org.dbsyncer.connector.config.WriterBatchConfig; -import org.dbsyncer.connector.model.Field; -import org.dbsyncer.connector.model.MetaInfo; -import org.dbsyncer.connector.model.Table; -import org.dbsyncer.connector.util.PrimaryKeyUtil; -import org.dbsyncer.sdk.spi.ConnectorMapper; +import org.dbsyncer.sdk.connector.ConnectorInstance; +import org.dbsyncer.sdk.model.Field; +import org.dbsyncer.sdk.model.MetaInfo; +import org.dbsyncer.sdk.model.Table; +import org.dbsyncer.sdk.spi.ConnectorService; +import org.dbsyncer.sdk.util.PrimaryKeyUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; -public class KafkaConnector extends AbstractConnector implements Connector { +@Component +public class KafkaConnector extends AbstractConnector implements ConnectorService { private final Logger logger = LoggerFactory.getLogger(getClass()); + private final String TYPE = "Kafka"; + + @Override + public String getConnectorType() { + return TYPE; + } + + @Override + public boolean isSupportedTiming() { + return false; + } + + @Override + public boolean isSupportedLog() { + return false; + } + + @Override + public Class getConfigClass() { + return KafkaConfig.class; + } + @Override - public ConnectorMapper connect(KafkaConfig config) { - return new KafkaConnectorMapper(config); + public ConnectorInstance connect(KafkaConfig config) { + return new KafkaConnectorInstance(config); } @Override - public void disconnect(KafkaConnectorMapper connectorMapper) { + public void disconnect(KafkaConnectorInstance connectorMapper) { connectorMapper.close(); } @Override - public boolean isAlive(KafkaConnectorMapper connectorMapper) { + public boolean isAlive(KafkaConnectorInstance connectorMapper) { return connectorMapper.getConnection().ping(); } @Override - public String getConnectorMapperCacheKey(KafkaConfig config) { + public String getConnectorInstanceCacheKey(KafkaConfig config) { return String.format("%s-%s-%s-%s", config.getConnectorType(), config.getBootstrapServers(), config.getTopic(), config.getGroupId()); } @Override - public List
    getTable(KafkaConnectorMapper connectorMapper) { + public List
    getTable(KafkaConnectorInstance connectorMapper) { List
    topics = new ArrayList<>(); topics.add(new Table(connectorMapper.getConfig().getTopic())); return topics; } @Override - public MetaInfo getMetaInfo(KafkaConnectorMapper connectorMapper, String tableName) { + public MetaInfo getMetaInfo(KafkaConnectorInstance connectorMapper, String tableName) { KafkaConfig config = connectorMapper.getConfig(); List fields = JsonUtil.jsonToArray(config.getFields(), Field.class); return new MetaInfo().setColumn(fields); } @Override - public long getCount(KafkaConnectorMapper connectorMapper, Map command) { + public long getCount(KafkaConnectorInstance connectorMapper, Map command) { return 0; } @Override - public Result reader(KafkaConnectorMapper connectorMapper, ReaderConfig config) { + public Result reader(KafkaConnectorInstance connectorMapper, ReaderConfig config) { throw new ConnectorException("Full synchronization is not supported"); } @Override - public Result writer(KafkaConnectorMapper connectorMapper, WriterBatchConfig config) { + public Result writer(KafkaConnectorInstance connectorMapper, WriterBatchConfig config) { List data = config.getData(); if (CollectionUtils.isEmpty(data) || CollectionUtils.isEmpty(config.getFields())) { logger.error("writer data can not be empty."); diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/kafka/KafkaConnectorMapper.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/kafka/KafkaConnectorInstance.java similarity index 78% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/kafka/KafkaConnectorMapper.java rename to dbsyncer-connector/src/main/java/org/dbsyncer/connector/kafka/KafkaConnectorInstance.java index 5d28de99..d32ba3f4 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/kafka/KafkaConnectorMapper.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/kafka/KafkaConnectorInstance.java @@ -1,45 +1,45 @@ -package org.dbsyncer.connector.kafka; - -import org.dbsyncer.connector.config.KafkaConfig; -import org.dbsyncer.connector.util.KafkaUtil; -import org.dbsyncer.sdk.spi.ConnectorMapper; - -public final class KafkaConnectorMapper implements ConnectorMapper { - private KafkaConfig config; - private KafkaClient client; - - public KafkaConnectorMapper(KafkaConfig config) { - this.config = config; - this.client = KafkaUtil.getConnection(config); - } - - @Override - public String getServiceUrl() { - return config.getBootstrapServers(); - } - - @Override - public KafkaConfig getConfig() { - return config; - } - - @Override - public void setConfig(KafkaConfig config) { - this.config = config; - } - - @Override - public KafkaClient getConnection() { - return client; - } - - @Override - public void close() { - KafkaUtil.close(client); - } - - @Override - public Object clone() throws CloneNotSupportedException { - return super.clone(); - } +package org.dbsyncer.connector.kafka; + +import org.dbsyncer.connector.config.KafkaConfig; +import org.dbsyncer.connector.util.KafkaUtil; +import org.dbsyncer.sdk.connector.ConnectorInstance; + +public final class KafkaConnectorInstance implements ConnectorInstance { + private KafkaConfig config; + private KafkaClient client; + + public KafkaConnectorInstance(KafkaConfig config) { + this.config = config; + this.client = KafkaUtil.getConnection(config); + } + + @Override + public String getServiceUrl() { + return config.getBootstrapServers(); + } + + @Override + public KafkaConfig getConfig() { + return config; + } + + @Override + public void setConfig(KafkaConfig config) { + this.config = config; + } + + @Override + public KafkaClient getConnection() { + return client; + } + + @Override + public void close() { + KafkaUtil.close(client); + } + + @Override + public Object clone() throws CloneNotSupportedException { + return super.clone(); + } } \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/model/Field.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/model/Field.java deleted file mode 100644 index c5391145..00000000 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/model/Field.java +++ /dev/null @@ -1,123 +0,0 @@ -package org.dbsyncer.connector.model; - -import org.dbsyncer.common.util.JsonUtil; - -/** - * 字段属性 - * - * @author AE86 - * @version 1.0.0 - * @date 2019/9/30 15:10 - */ -public class Field { - - /** - * 字段名，ID - */ - private String name; - - /** - * 类型名，INT - */ - private String typeName; - - /** - * 类型编码，4 - */ - private int type; - - /** - * 主键 - */ - private boolean pk; - - /** - * 字段别名 - */ - private String labelName; - - /** - * 是否系统字段 - */ - private boolean unmodifiabled; - - public Field() { - } - - public Field(String name, String typeName, int type) { - this.name = name; - this.typeName = typeName; - this.type = type; - } - - public Field(String name, String typeName, int type, boolean pk) { - this.name = name; - this.typeName = typeName; - this.type = type; - this.pk = pk; - } - - public Field(String name, String typeName, int type, boolean pk, String labelName, boolean unmodifiabled) { - this.name = name; - this.typeName = typeName; - this.type = type; - this.pk = pk; - this.labelName = labelName; - this.unmodifiabled = unmodifiabled; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getTypeName() { - return typeName; - } - - public void setTypeName(String typeName) { - this.typeName = typeName; - } - - public int getType() { - return type; - } - - public void setType(int type) { - this.type = type; - } - - public boolean isPk() { - return pk; - } - - public void setPk(boolean pk) { - this.pk = pk; - } - - public String getLabelName() { - return labelName; - } - - public Field setLabelName(String labelName) { - this.labelName = labelName; - return this; - } - - public boolean isUnmodifiabled() { - return unmodifiabled; - } - - public Field setUnmodifiabled(boolean unmodifiabled) { - this.unmodifiabled = unmodifiabled; - return this; - } - - @Override - public String toString() { - return JsonUtil.objToJson(this); - } -} \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/model/FileSchema.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/model/FileSchema.java index eefcdc4d..28d4cf5d 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/model/FileSchema.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/model/FileSchema.java @@ -1,5 +1,7 @@ package org.dbsyncer.connector.model; +import org.dbsyncer.sdk.model.Field; + import java.util.List; /** diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/model/Filter.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/model/Filter.java deleted file mode 100644 index fbf40d1c..00000000 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/model/Filter.java +++ /dev/null @@ -1,65 +0,0 @@ -package org.dbsyncer.connector.model; - -import org.dbsyncer.connector.enums.FilterEnum; -import org.dbsyncer.connector.enums.OperationEnum; - -/** - * 字段属性条件 - * @author AE86 - * @version 1.0.0 - * @date 2019/9/30 15:10 - */ -public class Filter { - - /** - * 字段名，ID - */ - private String name; - - /** - * @see OperationEnum - */ - private String operation; - - /** - * @see FilterEnum - */ - private String filter; - - /** - * 值 - */ - private String value; - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getOperation() { - return operation; - } - - public void setOperation(String operation) { - this.operation = operation; - } - - public String getFilter() { - return filter; - } - - public void setFilter(String filter) { - this.filter = filter; - } - - public String getValue() { - return value; - } - - public void setValue(String value) { - this.value = value; - } -} \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/model/MetaInfo.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/model/MetaInfo.java deleted file mode 100644 index b4730ac3..00000000 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/model/MetaInfo.java +++ /dev/null @@ -1,62 +0,0 @@ -package org.dbsyncer.connector.model; - -import java.util.List; - -/** - * 连接器基本信息 - * - * @author AE86 - * @ClassName: MetaInfo - * @Description: 包括字段信息、总条数 - * @date: 2017年7月20日 下午3:37:59 - */ -public class MetaInfo { - - /** - * 表类型 - */ - private String tableType; - - /** - * 属性字段 - * 格式：[{"name":"ID","typeName":"INT","type":"4"},{"name":"NAME","typeName":"VARCHAR","type":"12"}] - */ - private List column; - - /** - * sql - */ - private String sql; - - public String getTableType() { - return tableType; - } - - public MetaInfo setTableType(String tableType) { - this.tableType = tableType; - return this; - } - - public List getColumn() { - return column; - } - - public MetaInfo setColumn(List column) { - this.column = column; - return this; - } - - public String getSql() { - return sql; - } - - public MetaInfo setSql(String sql) { - this.sql = sql; - return this; - } - - @Override - public String toString() { - return new StringBuilder().append("MetaInfo{").append("tableType=").append(tableType).append(", ").append("column=").append(column).append('}').toString(); - } -} \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/model/Table.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/model/Table.java deleted file mode 100644 index 6d887537..00000000 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/model/Table.java +++ /dev/null @@ -1,95 +0,0 @@ -package org.dbsyncer.connector.model; - -import org.dbsyncer.connector.enums.TableTypeEnum; - -import java.util.List; - -/** - * @author AE86 - * @version 1.0.0 - * @date 2019/10/15 23:58 - */ -public class Table { - - /** - * 表名 - */ - private String name; - - /** - * 表类型[TABLE、VIEW、MATERIALIZED VIEW] - */ - private String type; - - /** - * 属性字段 - * 格式：[{"name":"ID","typeName":"INT","type":"4"},{"name":"NAME","typeName":"VARCHAR","type":"12"}] - */ - private List column; - - /** - * sql - */ - private String sql; - - // 总数 - private long count; - - public Table() { - } - - public Table(String name) { - this(name, TableTypeEnum.TABLE.getCode()); - } - - public Table(String name, String type) { - this(name, type, null, null); - } - - public Table(String name, String type, List column, String sql) { - this.name = name; - this.type = type; - this.column = column; - this.sql = sql; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type; - } - - public List getColumn() { - return column; - } - - public void setColumn(List column) { - this.column = column; - } - - public String getSql() { - return sql; - } - - public void setSql(String sql) { - this.sql = sql; - } - - public long getCount() { - return count; - } - - public void setCount(long count) { - this.count = count; - } -} \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/MySQLConnector.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/MySQLConnector.java index be2e8b58..4382eea1 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/MySQLConnector.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/MySQLConnector.java @@ -1,20 +1,29 @@ package org.dbsyncer.connector.mysql; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.config.ReaderConfig; -import org.dbsyncer.connector.constant.DatabaseConstant; -import org.dbsyncer.connector.database.AbstractDatabaseConnector; -import org.dbsyncer.connector.model.PageSql; -import org.dbsyncer.connector.util.PrimaryKeyUtil; +import org.dbsyncer.sdk.config.ReaderConfig; +import org.dbsyncer.sdk.constant.DatabaseConstant; +import org.dbsyncer.sdk.connector.database.AbstractDatabaseConnector; +import org.dbsyncer.sdk.model.PageSql; +import org.dbsyncer.sdk.util.PrimaryKeyUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; import java.util.List; +@Component public final class MySQLConnector extends AbstractDatabaseConnector { private final Logger logger = LoggerFactory.getLogger(getClass()); + private final String TYPE = "MySQL"; + + @Override + public String getConnectorType() { + return TYPE; + } + @Override public String buildSqlWithQuotation() { return "`"; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/OracleConnector.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/OracleConnector.java index bfa92292..02198409 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/OracleConnector.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/OracleConnector.java @@ -1,36 +1,40 @@ package org.dbsyncer.connector.oracle; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.config.CommandConfig; -import org.dbsyncer.connector.config.DatabaseConfig; -import org.dbsyncer.connector.config.ReaderConfig; -import org.dbsyncer.connector.constant.DatabaseConstant; -import org.dbsyncer.connector.database.AbstractDatabaseConnector; -import org.dbsyncer.connector.enums.TableTypeEnum; -import org.dbsyncer.connector.model.PageSql; -import org.dbsyncer.connector.model.Table; -import org.dbsyncer.connector.util.PrimaryKeyUtil; +import org.dbsyncer.sdk.config.CommandConfig; +import org.dbsyncer.sdk.config.DatabaseConfig; +import org.dbsyncer.sdk.config.ReaderConfig; +import org.dbsyncer.sdk.connector.database.AbstractDatabaseConnector; +import org.dbsyncer.sdk.constant.DatabaseConstant; +import org.dbsyncer.sdk.enums.TableTypeEnum; +import org.dbsyncer.sdk.model.PageSql; +import org.dbsyncer.sdk.model.Table; +import org.dbsyncer.sdk.util.PrimaryKeyUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; +import javax.annotation.PostConstruct; import java.sql.Types; import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; +@Component public final class OracleConnector extends AbstractDatabaseConnector { private final Logger logger = LoggerFactory.getLogger(getClass()); - /** - * 系统函数表达式to_char/to_date/to_timestamp/to_number - */ - private final String SYS_EXPRESSION = "(to_char\\().+?(\\))|(to_date\\().+?(\\))|(to_timestamp\\().+?(\\))|(to_number\\().+?(\\))"; + private final String TYPE = "Oracle"; - public OracleConnector() { + @PostConstruct + private void init() { VALUE_MAPPERS.put(Types.OTHER, new OracleOtherValueMapper()); } + @Override + public String getConnectorType() { + return TYPE; + } + @Override public String buildSqlWithQuotation() { return "\""; @@ -102,25 +106,6 @@ public final class OracleConnector extends AbstractDatabaseConnector { return true; } - /** - * TODO 待废弃 推荐使用系统参数表达式$xxx$ - * - * @param value - * @return - */ - @Override - public String buildFilterValue(String value) { - if (StringUtil.isNotBlank(value)) { - String val = value.toLowerCase(); - // 支持Oracle系统函数, Example: to_char(sysdate, 'YYYY-MM-DD HH24:MI:SS') - Matcher matcher = Pattern.compile(SYS_EXPRESSION).matcher(val); - if (matcher.find()) { - return value; - } - } - return super.buildFilterValue(value); - } - @Override public String getValidationQuery() { return "select 1 from dual"; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/OracleOtherValueMapper.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/OracleOtherValueMapper.java index 4b956c15..8450be7e 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/OracleOtherValueMapper.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/OracleOtherValueMapper.java @@ -2,12 +2,12 @@ package org.dbsyncer.connector.oracle; import com.microsoft.sqlserver.jdbc.Geometry; import oracle.jdbc.OracleConnection; -import org.dbsyncer.connector.oracle.geometry.JGeometry; -import org.dbsyncer.connector.AbstractValueMapper; import org.dbsyncer.connector.ConnectorException; -import org.dbsyncer.connector.database.ds.SimpleConnection; -import org.dbsyncer.connector.util.DatabaseUtil; -import org.dbsyncer.sdk.spi.ConnectorMapper; +import org.dbsyncer.connector.oracle.geometry.JGeometry; +import org.dbsyncer.sdk.connector.AbstractValueMapper; +import org.dbsyncer.sdk.connector.ConnectorInstance; +import org.dbsyncer.sdk.connector.database.ds.SimpleConnection; +import org.dbsyncer.sdk.util.DatabaseUtil; import java.sql.Struct; @@ -30,10 +30,10 @@ public final class OracleOtherValueMapper extends AbstractValueMapper { } @Override - protected Struct convert(ConnectorMapper connectorMapper, Object val) throws Exception { + protected Struct convert(ConnectorInstance connectorInstance, Object val) throws Exception { // SqlServer Geometry if (val instanceof byte[]) { - Object conn = connectorMapper.getConnection(); + Object conn = connectorInstance.getConnection(); if (conn instanceof SimpleConnection) { SimpleConnection connection = null; try { diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLBitValueMapper.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLBitValueMapper.java index e278403d..e5e17632 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLBitValueMapper.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLBitValueMapper.java @@ -1,8 +1,8 @@ package org.dbsyncer.connector.postgresql; -import org.dbsyncer.connector.AbstractValueMapper; import org.dbsyncer.connector.ConnectorException; -import org.dbsyncer.sdk.spi.ConnectorMapper; +import org.dbsyncer.sdk.connector.AbstractValueMapper; +import org.dbsyncer.sdk.connector.ConnectorInstance; /** * @author AE86 @@ -12,7 +12,7 @@ import org.dbsyncer.sdk.spi.ConnectorMapper; public class PostgreSQLBitValueMapper extends AbstractValueMapper { @Override - protected Boolean convert(ConnectorMapper connectorMapper, Object val) { + protected Boolean convert(ConnectorInstance connectorInstance, Object val) { if (val instanceof Integer) { Integer i = (Integer) val; return i == 1; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLConnector.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLConnector.java index b3d70a96..b8a4df7e 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLConnector.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLConnector.java @@ -1,26 +1,37 @@ package org.dbsyncer.connector.postgresql; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.config.CommandConfig; -import org.dbsyncer.connector.config.DatabaseConfig; -import org.dbsyncer.connector.config.ReaderConfig; -import org.dbsyncer.connector.constant.DatabaseConstant; -import org.dbsyncer.connector.database.AbstractDatabaseConnector; -import org.dbsyncer.connector.enums.TableTypeEnum; -import org.dbsyncer.connector.model.PageSql; -import org.dbsyncer.connector.model.Table; -import org.dbsyncer.connector.util.PrimaryKeyUtil; +import org.dbsyncer.sdk.config.CommandConfig; +import org.dbsyncer.sdk.config.DatabaseConfig; +import org.dbsyncer.sdk.config.ReaderConfig; +import org.dbsyncer.sdk.connector.database.AbstractDatabaseConnector; +import org.dbsyncer.sdk.constant.DatabaseConstant; +import org.dbsyncer.sdk.enums.TableTypeEnum; +import org.dbsyncer.sdk.model.PageSql; +import org.dbsyncer.sdk.model.Table; +import org.dbsyncer.sdk.util.PrimaryKeyUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; +import javax.annotation.PostConstruct; import java.sql.Types; import java.util.List; +@Component public final class PostgreSQLConnector extends AbstractDatabaseConnector { private final Logger logger = LoggerFactory.getLogger(getClass()); - public PostgreSQLConnector() { + private final String TYPE = "PostgreSQL"; + + @Override + public String getConnectorType() { + return TYPE; + } + + @PostConstruct + private void init() { VALUE_MAPPERS.put(Types.BIT, new PostgreSQLBitValueMapper()); VALUE_MAPPERS.put(Types.OTHER, new PostgreSQLOtherValueMapper()); } diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLOtherValueMapper.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLOtherValueMapper.java index 6b3ac2d7..a808e37e 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLOtherValueMapper.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLOtherValueMapper.java @@ -1,8 +1,8 @@ package org.dbsyncer.connector.postgresql; -import org.dbsyncer.connector.AbstractValueMapper; import org.dbsyncer.connector.ConnectorException; -import org.dbsyncer.sdk.spi.ConnectorMapper; +import org.dbsyncer.sdk.connector.AbstractValueMapper; +import org.dbsyncer.sdk.connector.ConnectorInstance; import org.postgis.Geometry; import org.postgis.PGgeometry; import org.postgis.binary.BinaryParser; @@ -38,7 +38,7 @@ public final class PostgreSQLOtherValueMapper extends AbstractValueMapper { - - @Override - protected String convert(ConnectorMapper connectorMapper, Object val) { - throw new ConnectorException(String.format("%s can not find type [%s], val [%s]", getClass().getSimpleName(), val.getClass(), val)); - } -} \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/NCharValueMapper.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/NCharValueMapper.java deleted file mode 100644 index 4c99700a..00000000 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/NCharValueMapper.java +++ /dev/null @@ -1,18 +0,0 @@ -package org.dbsyncer.connector.schema; - -import org.dbsyncer.connector.AbstractValueMapper; -import org.dbsyncer.connector.ConnectorException; -import org.dbsyncer.sdk.spi.ConnectorMapper; - -/** - * @author AE86 - * @version 1.0.0 - * @date 2022/8/25 0:07 - */ -public class NCharValueMapper extends AbstractValueMapper { - - @Override - protected String convert(ConnectorMapper connectorMapper, Object val) { - throw new ConnectorException(String.format("%s can not find type [%s], val [%s]", getClass().getSimpleName(), val.getClass(), val)); - } -} \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/NVarcharValueMapper.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/NVarcharValueMapper.java deleted file mode 100644 index cdd638d8..00000000 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/NVarcharValueMapper.java +++ /dev/null @@ -1,18 +0,0 @@ -package org.dbsyncer.connector.schema; - -import org.dbsyncer.connector.AbstractValueMapper; -import org.dbsyncer.connector.ConnectorException; -import org.dbsyncer.sdk.spi.ConnectorMapper; - -/** - * @author AE86 - * @version 1.0.0 - * @date 2022/8/24 23:43 - */ -public class NVarcharValueMapper extends AbstractValueMapper { - - @Override - protected String convert(ConnectorMapper connectorMapper, Object val) { - throw new ConnectorException(String.format("%s can not find type [%s], val [%s]", getClass().getSimpleName(), val.getClass(), val)); - } -} \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/OtherValueMapper.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/OtherValueMapper.java deleted file mode 100644 index 5b6a9f28..00000000 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/OtherValueMapper.java +++ /dev/null @@ -1,18 +0,0 @@ -package org.dbsyncer.connector.schema; - -import org.dbsyncer.connector.AbstractValueMapper; -import org.dbsyncer.connector.ConnectorException; -import org.dbsyncer.sdk.spi.ConnectorMapper; - -/** - * @author AE86 - * @version 1.0.0 - * @date 2022/9/16 16:54 - */ -public class OtherValueMapper extends AbstractValueMapper { - - @Override - protected Object convert(ConnectorMapper connectorMapper, Object val) { - throw new ConnectorException(String.format("%s can not find type [%s], val [%s]", getClass().getSimpleName(), val.getClass(), val)); - } -} \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLMySQLConnector.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLMySQLConnector.java index bd7b00cb..74745416 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLMySQLConnector.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLMySQLConnector.java @@ -1,14 +1,24 @@ package org.dbsyncer.connector.sql; -import org.dbsyncer.connector.config.CommandConfig; -import org.dbsyncer.connector.config.ReaderConfig; -import org.dbsyncer.connector.constant.DatabaseConstant; -import org.dbsyncer.connector.model.PageSql; +import org.dbsyncer.sdk.config.CommandConfig; +import org.dbsyncer.sdk.config.ReaderConfig; +import org.dbsyncer.sdk.connector.database.AbstractDQLConnector; +import org.dbsyncer.sdk.constant.DatabaseConstant; +import org.dbsyncer.sdk.model.PageSql; +import org.springframework.stereotype.Component; import java.util.Map; +@Component public final class DQLMySQLConnector extends AbstractDQLConnector { + private final String TYPE = "DqlMySQL"; + + @Override + public String getConnectorType() { + return TYPE; + } + @Override public String getPageSql(PageSql config) { return config.getQuerySql() + DatabaseConstant.MYSQL_PAGE_SQL; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLOracleConnector.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLOracleConnector.java index bc6f554e..f7cca7fb 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLOracleConnector.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLOracleConnector.java @@ -1,11 +1,21 @@ package org.dbsyncer.connector.sql; -import org.dbsyncer.connector.config.ReaderConfig; -import org.dbsyncer.connector.constant.DatabaseConstant; -import org.dbsyncer.connector.model.PageSql; +import org.dbsyncer.sdk.config.ReaderConfig; +import org.dbsyncer.sdk.connector.database.AbstractDQLConnector; +import org.dbsyncer.sdk.constant.DatabaseConstant; +import org.dbsyncer.sdk.model.PageSql; +import org.springframework.stereotype.Component; +@Component public final class DQLOracleConnector extends AbstractDQLConnector { + private final String TYPE = "DqlOracle"; + + @Override + public String getConnectorType() { + return TYPE; + } + @Override public String getPageSql(PageSql config) { return DatabaseConstant.ORACLE_PAGE_SQL_START + config.getQuerySql() + DatabaseConstant.ORACLE_PAGE_SQL_END; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLPostgreSQLConnector.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLPostgreSQLConnector.java index 09a95226..37f087cc 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLPostgreSQLConnector.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLPostgreSQLConnector.java @@ -1,11 +1,21 @@ package org.dbsyncer.connector.sql; -import org.dbsyncer.connector.config.ReaderConfig; -import org.dbsyncer.connector.constant.DatabaseConstant; -import org.dbsyncer.connector.model.PageSql; +import org.dbsyncer.sdk.config.ReaderConfig; +import org.dbsyncer.sdk.connector.database.AbstractDQLConnector; +import org.dbsyncer.sdk.constant.DatabaseConstant; +import org.dbsyncer.sdk.model.PageSql; +import org.springframework.stereotype.Component; +@Component public final class DQLPostgreSQLConnector extends AbstractDQLConnector { + private final String TYPE = "DqlPostgreSQL"; + + @Override + public String getConnectorType() { + return TYPE; + } + @Override public String getPageSql(PageSql config) { return config.getQuerySql() + DatabaseConstant.POSTGRESQL_PAGE_SQL; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLSqlServerConnector.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLSqlServerConnector.java index c2294e83..e7ad4199 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLSqlServerConnector.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLSqlServerConnector.java @@ -1,15 +1,24 @@ package org.dbsyncer.connector.sql; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.config.ReaderConfig; -import org.dbsyncer.connector.constant.DatabaseConstant; -import org.dbsyncer.connector.model.PageSql; -import org.dbsyncer.connector.util.PrimaryKeyUtil; +import org.dbsyncer.sdk.config.ReaderConfig; +import org.dbsyncer.sdk.connector.database.AbstractDQLConnector; +import org.dbsyncer.sdk.constant.DatabaseConstant; +import org.dbsyncer.sdk.model.PageSql; +import org.springframework.stereotype.Component; import java.util.List; +@Component public final class DQLSqlServerConnector extends AbstractDQLConnector { + private final String TYPE = "DqlSqlServer"; + + @Override + public String getConnectorType() { + return TYPE; + } + @Override public String getPageSql(PageSql config) { List primaryKeys = config.getPrimaryKeys(); @@ -23,5 +32,4 @@ public final class DQLSqlServerConnector extends AbstractDQLConnector { int pageIndex = config.getPageIndex(); return new Object[]{(pageIndex - 1) * pageSize + 1, pageIndex * pageSize}; } - } \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/SqlServerConnector.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/SqlServerConnector.java index a00c6545..19bb5e8a 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/SqlServerConnector.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/SqlServerConnector.java @@ -2,16 +2,17 @@ package org.dbsyncer.connector.sqlserver; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.config.CommandConfig; -import org.dbsyncer.connector.config.DatabaseConfig; -import org.dbsyncer.connector.config.ReaderConfig; -import org.dbsyncer.connector.constant.DatabaseConstant; -import org.dbsyncer.connector.database.AbstractDatabaseConnector; -import org.dbsyncer.connector.database.DatabaseConnectorMapper; -import org.dbsyncer.connector.enums.TableTypeEnum; -import org.dbsyncer.connector.model.Field; -import org.dbsyncer.connector.model.PageSql; -import org.dbsyncer.connector.model.Table; +import org.dbsyncer.sdk.config.DatabaseConfig; +import org.dbsyncer.sdk.config.CommandConfig; +import org.dbsyncer.sdk.config.ReaderConfig; +import org.dbsyncer.sdk.constant.DatabaseConstant; +import org.dbsyncer.sdk.connector.database.AbstractDatabaseConnector; +import org.dbsyncer.sdk.connector.database.DatabaseConnectorInstance; +import org.dbsyncer.sdk.enums.TableTypeEnum; +import org.dbsyncer.sdk.model.PageSql; +import org.dbsyncer.sdk.model.Field; +import org.dbsyncer.sdk.model.Table; +import org.springframework.stereotype.Component; import java.util.ArrayList; import java.util.List; @@ -19,14 +20,22 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; +@Component public final class SqlServerConnector extends AbstractDatabaseConnector { - private static final String QUERY_VIEW = "select name from sysobjects where xtype in('v')"; + private final String QUERY_VIEW = "select name from sysobjects where xtype in('v')"; - private static final String QUERY_TABLE = "select name from sys.tables where schema_id = schema_id('%s') and is_ms_shipped = 0"; + private final String QUERY_TABLE = "select name from sys.tables where schema_id = schema_id('%s') and is_ms_shipped = 0"; + + private final String TYPE = "SqlServer"; + + @Override + public String getConnectorType() { + return TYPE; + } @Override - public List
    getTable(DatabaseConnectorMapper connectorMapper) { + public List
    getTable(DatabaseConnectorInstance connectorMapper) { DatabaseConfig config = connectorMapper.getConfig(); List
    tables = getTables(connectorMapper, String.format(QUERY_TABLE, config.getSchema()), TableTypeEnum.TABLE); tables.addAll(getTables(connectorMapper, QUERY_VIEW, TableTypeEnum.VIEW)); @@ -79,7 +88,7 @@ public final class SqlServerConnector extends AbstractDatabaseConnector { buildTableName(table.getName())); } - private List
    getTables(DatabaseConnectorMapper connectorMapper, String sql, TableTypeEnum type) { + private List
    getTables(DatabaseConnectorInstance connectorMapper, String sql, TableTypeEnum type) { List tableNames = connectorMapper.execute(databaseTemplate -> databaseTemplate.queryForList(sql, String.class)); if (!CollectionUtils.isEmpty(tableNames)) { return tableNames.stream().map(name -> new Table(name, type.getCode())).collect(Collectors.toList()); diff --git a/dbsyncer-connector/src/main/test/ConnectionTest.java b/dbsyncer-connector/src/main/test/ConnectionTest.java index 81c85d13..9e85c0f7 100644 --- a/dbsyncer-connector/src/main/test/ConnectionTest.java +++ b/dbsyncer-connector/src/main/test/ConnectionTest.java @@ -2,12 +2,12 @@ import oracle.jdbc.OracleConnection; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.RandomUtil; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.config.DatabaseConfig; -import org.dbsyncer.connector.database.DatabaseConnectorMapper; -import org.dbsyncer.connector.database.ds.SimpleConnection; -import org.dbsyncer.connector.enums.TableTypeEnum; -import org.dbsyncer.connector.model.Field; -import org.dbsyncer.connector.model.Table; +import org.dbsyncer.sdk.config.DatabaseConfig; +import org.dbsyncer.sdk.connector.database.DatabaseConnectorInstance; +import org.dbsyncer.sdk.connector.database.ds.SimpleConnection; +import org.dbsyncer.sdk.enums.TableTypeEnum; +import org.dbsyncer.sdk.model.Field; +import org.dbsyncer.sdk.model.Table; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -32,7 +32,7 @@ public class ConnectionTest { @Test public void testByte() { - final DatabaseConnectorMapper connectorMapper = new DatabaseConnectorMapper(createOracleConfig()); + final DatabaseConnectorInstance connectorMapper = new DatabaseConnectorInstance(createOracleConfig()); String executeSql = "UPDATE \"my_user\" SET \"name\"=?,\"clo\"=? WHERE \"id\"=?"; int[] execute = connectorMapper.execute(databaseTemplate -> @@ -64,7 +64,7 @@ public class ConnectionTest { @Test public void testConnection() throws InterruptedException { - final DatabaseConnectorMapper connectorMapper = new DatabaseConnectorMapper(createSqlServerConfig()); + final DatabaseConnectorInstance connectorMapper = new DatabaseConnectorInstance(createSqlServerConfig()); // 模拟并发 final int threadSize = 100; @@ -109,7 +109,7 @@ public class ConnectionTest { @Test public void testBatchInsert() { - final DatabaseConnectorMapper connectorMapper = new DatabaseConnectorMapper(createMysqlConfig()); + final DatabaseConnectorInstance connectorMapper = new DatabaseConnectorInstance(createMysqlConfig()); long begin = Instant.now().toEpochMilli(); final int threadSize = 10; @@ -147,7 +147,7 @@ public class ConnectionTest { @Test public void testBatchUpdate() { - final DatabaseConnectorMapper connectorMapper = new DatabaseConnectorMapper(createMysqlConfig()); + final DatabaseConnectorInstance connectorMapper = new DatabaseConnectorInstance(createMysqlConfig()); long begin = Instant.now().toEpochMilli(); final int threadSize = 10; @@ -185,7 +185,7 @@ public class ConnectionTest { @Test public void testBatchDelete() { - final DatabaseConnectorMapper connectorMapper = new DatabaseConnectorMapper(createMysqlConfig()); + final DatabaseConnectorInstance connectorMapper = new DatabaseConnectorInstance(createMysqlConfig()); long begin = Instant.now().toEpochMilli(); final int threadSize = 10; @@ -226,7 +226,7 @@ public class ConnectionTest { return s.toString(); } - private void batchUpdate(DatabaseConnectorMapper connectorMapper, ExecutorService pool, String sql, List dataList, int batchSize) { + private void batchUpdate(DatabaseConnectorInstance connectorMapper, ExecutorService pool, String sql, List dataList, int batchSize) { int total = dataList.size(); int taskSize = total % batchSize == 0 ? total / batchSize : total / batchSize + 1; final CountDownLatch latch = new CountDownLatch(taskSize); @@ -279,7 +279,7 @@ public class ConnectionTest { public void testGetColumnsDetails() { final String schema = "root"; final String tableNamePattern = "sw_test"; - final DatabaseConnectorMapper connectorMapper = new DatabaseConnectorMapper(createMysqlConfig()); + final DatabaseConnectorInstance connectorMapper = new DatabaseConnectorInstance(createMysqlConfig()); connectorMapper.execute(databaseTemplate -> { SimpleConnection connection = databaseTemplate.getSimpleConnection(); Connection conn = connection.getConnection(); @@ -299,7 +299,7 @@ public class ConnectionTest { } private List
    getTables(DatabaseConfig config, final String catalog, final String schema, final String tableNamePattern) { - final DatabaseConnectorMapper connectorMapper = new DatabaseConnectorMapper(config); + final DatabaseConnectorInstance connectorMapper = new DatabaseConnectorInstance(config); List
    tables = new ArrayList<>(); connectorMapper.execute(databaseTemplate -> { SimpleConnection connection = databaseTemplate.getSimpleConnection(); diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/AbstractDatabaseExtractor.java b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/AbstractDatabaseExtractor.java index b6d1f28d..6eea6cd8 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/AbstractDatabaseExtractor.java +++ b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/AbstractDatabaseExtractor.java @@ -3,13 +3,13 @@ package org.dbsyncer.listener; import org.dbsyncer.listener.event.RowChangedEvent; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.constant.ConnectorConstant; -import org.dbsyncer.connector.database.AbstractDatabaseConnector; -import org.dbsyncer.connector.database.DatabaseConnectorMapper; -import org.dbsyncer.connector.model.Field; -import org.dbsyncer.connector.model.MetaInfo; -import org.dbsyncer.connector.model.Table; -import org.dbsyncer.connector.util.PrimaryKeyUtil; +import org.dbsyncer.sdk.constant.ConnectorConstant; +import org.dbsyncer.sdk.connector.database.AbstractDatabaseConnector; +import org.dbsyncer.sdk.connector.database.DatabaseConnectorInstance; +import org.dbsyncer.sdk.model.Field; +import org.dbsyncer.sdk.model.MetaInfo; +import org.dbsyncer.sdk.model.Table; +import org.dbsyncer.sdk.util.PrimaryKeyUtil; import org.springframework.util.Assert; import java.util.ArrayList; @@ -77,13 +77,13 @@ public abstract class AbstractDatabaseExtractor extends AbstractExtractor { * 初始化Dql连接配置 */ protected void postProcessDqlBeforeInitialization() { - DatabaseConnectorMapper mapper = (DatabaseConnectorMapper) connectorFactory.connect(connectorConfig); - AbstractDatabaseConnector connector = (AbstractDatabaseConnector) connectorFactory.getConnector(mapper); - String quotation = connector.buildSqlWithQuotation(); + DatabaseConnectorInstance connectorInstance = (DatabaseConnectorInstance) connectorFactory.connect(connectorConfig); + AbstractDatabaseConnector connectorService = (AbstractDatabaseConnector) connectorFactory.getConnectorService(connectorConfig.getConnectorType()); + String quotation = connectorService.buildSqlWithQuotation(); // <用户表, MY_USER> Map tableMap = new HashMap<>(); - mapper.getConfig().getSqlTables().forEach(s -> tableMap.put(s.getSqlName(), s.getTable())); + connectorInstance.getConfig().getSqlTables().forEach(s -> tableMap.put(s.getSqlName(), s.getTable())); // 清空默认表名 filterTable.clear(); for (Table t : sourceTable) { @@ -94,7 +94,7 @@ public abstract class AbstractDatabaseExtractor extends AbstractExtractor { Assert.hasText(sql, "The sql is null."); Assert.hasText(tableName, "The tableName is null."); - MetaInfo metaInfo = connectorFactory.getMetaInfo(mapper, sqlName); + MetaInfo metaInfo = connectorFactory.getMetaInfo(connectorInstance, sqlName); final List column = metaInfo.getColumn(); Assert.notEmpty(column, String.format("The column of table name '%s' is empty.", sqlName)); @@ -106,7 +106,7 @@ public abstract class AbstractDatabaseExtractor extends AbstractExtractor { boolean notContainsWhere = !StringUtil.contains(sql, " WHERE "); querySql.append(notContainsWhere ? " WHERE " : " AND "); PrimaryKeyUtil.buildSql(querySql, primaryKeys, quotation, " AND ", " = ? ", notContainsWhere); - DqlMapper dqlMapper = new DqlMapper(mapper, sqlName, querySql.toString(), column, getPrimaryKeyIndexArray(column, primaryKeys)); + DqlMapper dqlMapper = new DqlMapper(connectorInstance, sqlName, querySql.toString(), column, getPrimaryKeyIndexArray(column, primaryKeys)); if (!dqlMap.containsKey(tableName)) { dqlMap.putIfAbsent(tableName, new ArrayList<>()); } @@ -139,7 +139,7 @@ public abstract class AbstractDatabaseExtractor extends AbstractExtractor { private void queryDqlData(DqlMapper dqlMapper, List data) { if (!CollectionUtils.isEmpty(data)) { - Map row = dqlMapper.mapper.execute(databaseTemplate -> { + Map row = dqlMapper.connectorInstance.execute(databaseTemplate -> { int size = dqlMapper.primaryKeyIndexArray.length; Object[] args = new Object[size]; for (int i = 0; i < size; i++) { @@ -155,14 +155,14 @@ public abstract class AbstractDatabaseExtractor extends AbstractExtractor { } final class DqlMapper { - DatabaseConnectorMapper mapper; + DatabaseConnectorInstance connectorInstance; String sqlName; String sql; List column; Integer[] primaryKeyIndexArray; - public DqlMapper(DatabaseConnectorMapper mapper, String sqlName, String sql, List column, Integer[] primaryKeyIndexArray) { - this.mapper = mapper; + public DqlMapper(DatabaseConnectorInstance connectorInstance, String sqlName, String sql, List column, Integer[] primaryKeyIndexArray) { + this.connectorInstance = connectorInstance; this.sqlName = sqlName; this.sql = sql; this.column = column; diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/AbstractExtractor.java b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/AbstractExtractor.java index 61218f6c..30e3dc5f 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/AbstractExtractor.java +++ b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/AbstractExtractor.java @@ -4,10 +4,10 @@ import org.dbsyncer.connector.scheduled.ScheduledTaskService; import org.dbsyncer.listener.model.ChangedOffset; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.connector.ConnectorFactory; -import org.dbsyncer.connector.constant.ConnectorConstant; -import org.dbsyncer.connector.model.Table; +import org.dbsyncer.sdk.constant.ConnectorConstant; import org.dbsyncer.listener.config.ListenerConfig; import org.dbsyncer.sdk.model.ConnectorConfig; +import org.dbsyncer.sdk.model.Table; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/ListenerFactory.java b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/ListenerFactory.java index 5cff71c9..55efab3c 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/ListenerFactory.java +++ b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/ListenerFactory.java @@ -1,11 +1,8 @@ package org.dbsyncer.listener; import org.dbsyncer.listener.enums.ListenerTypeEnum; -import org.dbsyncer.listener.enums.LogExtractorEnum; -import org.dbsyncer.listener.enums.TimingExtractorEnum; import org.springframework.stereotype.Component; -import javax.annotation.PostConstruct; import java.util.LinkedHashMap; import java.util.Map; import java.util.function.Function; @@ -15,12 +12,6 @@ public class ListenerFactory implements Listener { private Map> map = new LinkedHashMap<>(); - @PostConstruct - private void init() { - map.putIfAbsent(ListenerTypeEnum.LOG, (connectorType) -> LogExtractorEnum.getExtractor(connectorType)); - map.putIfAbsent(ListenerTypeEnum.TIMING, (connectorType) -> TimingExtractorEnum.getExtractor(connectorType)); - } - @Override public T getExtractor(ListenerTypeEnum listenerTypeEnum, String connectorType, Class valueType) throws IllegalAccessException, InstantiationException { Function function = map.get(listenerTypeEnum); diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/enums/LogExtractorEnum.java b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/enums/LogExtractorEnum.java deleted file mode 100644 index 7003734c..00000000 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/enums/LogExtractorEnum.java +++ /dev/null @@ -1,98 +0,0 @@ -package org.dbsyncer.listener.enums; - -import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.enums.ConnectorEnum; -import org.dbsyncer.listener.ListenerException; -import org.dbsyncer.listener.file.FileExtractor; -import org.dbsyncer.listener.kafka.KafkaExtractor; -import org.dbsyncer.listener.mysql.DqlMySQLExtractor; -import org.dbsyncer.listener.mysql.MySQLExtractor; -import org.dbsyncer.listener.oracle.DqlOracleExtractor; -import org.dbsyncer.listener.oracle.OracleExtractor; -import org.dbsyncer.listener.postgresql.DqlPostgreSQLExtractor; -import org.dbsyncer.listener.postgresql.PostgreSQLExtractor; -import org.dbsyncer.listener.sqlserver.DqlSqlServerExtractor; -import org.dbsyncer.listener.sqlserver.SqlServerExtractor; - -/** - * 日志模式支持类型 - * - * @author AE86 - * @version 1.0.0 - * @date 2020/04/24 14:19 - */ -public enum LogExtractorEnum { - - /** - * Mysql - */ - MYSQL(ConnectorEnum.MYSQL.getType(), MySQLExtractor.class), - /** - * Oracle - */ - ORACLE(ConnectorEnum.ORACLE.getType(), OracleExtractor.class), - /** - * SqlServer - */ - SQL_SERVER(ConnectorEnum.SQL_SERVER.getType(), SqlServerExtractor.class), - /** - * PostgreSQL - */ - POSTGRE_SQL(ConnectorEnum.POSTGRE_SQL.getType(), PostgreSQLExtractor.class), - /** - * Kafka - */ - KAFKA(ConnectorEnum.KAFKA.getType(), KafkaExtractor.class), - /** - * File - */ - FILE(ConnectorEnum.FILE.getType(), FileExtractor.class), - /** - * DqlMysql - */ - DQL_MYSQL(ConnectorEnum.DQL_MYSQL.getType(), DqlMySQLExtractor.class), - /** - * DqlOracle - */ - DQL_ORACLE(ConnectorEnum.DQL_ORACLE.getType(), DqlOracleExtractor.class), - /** - * DqlSqlServer - */ - DQL_SQL_SERVER(ConnectorEnum.DQL_SQL_SERVER.getType(), DqlSqlServerExtractor.class), - /** - * DqlPostgreSQL - */ - DQL_POSTGRE_SQL(ConnectorEnum.DQL_POSTGRE_SQL.getType(), DqlPostgreSQLExtractor.class); - - private String type; - private Class clazz; - - LogExtractorEnum(String type, Class clazz) { - this.type = type; - this.clazz = clazz; - } - - /** - * 获取抽取器 - * - * @param type - * @return - * @throws ListenerException - */ - public static Class getExtractor(String type) throws ListenerException { - for (LogExtractorEnum e : LogExtractorEnum.values()) { - if (StringUtil.equals(type, e.getType())) { - return e.getClazz(); - } - } - throw new ListenerException(String.format("LogExtractorEnum type \"%s\" does not exist.", type)); - } - - public String getType() { - return type; - } - - public Class getClazz() { - return clazz; - } -} \ No newline at end of file diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/enums/TimingExtractorEnum.java b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/enums/TimingExtractorEnum.java deleted file mode 100644 index 7172b83f..00000000 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/enums/TimingExtractorEnum.java +++ /dev/null @@ -1,86 +0,0 @@ -package org.dbsyncer.listener.enums; - -import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.enums.ConnectorEnum; -import org.dbsyncer.listener.ListenerException; -import org.dbsyncer.listener.quartz.DatabaseQuartzExtractor; -import org.dbsyncer.listener.quartz.ESQuartzExtractor; - -/** - * 定时模式支持类型 - * - * @author AE86 - * @version 1.0.0 - * @date 2020/04/24 14:19 - */ -public enum TimingExtractorEnum { - - /** - * Mysql - */ - MYSQL(ConnectorEnum.MYSQL.getType(), DatabaseQuartzExtractor.class), - /** - * Mysql - */ - ORACLE(ConnectorEnum.ORACLE.getType(), DatabaseQuartzExtractor.class), - /** - * SqlServer - */ - SQL_SERVER(ConnectorEnum.SQL_SERVER.getType(), DatabaseQuartzExtractor.class), - /** - * PostgreSQL - */ - POSTGRE_SQL(ConnectorEnum.POSTGRE_SQL.getType(), DatabaseQuartzExtractor.class), - /** - * Elasticsearch - */ - ELASTIC_SEARCH(ConnectorEnum.ELASTIC_SEARCH.getType(), ESQuartzExtractor.class), - /** - * DqlMysql - */ - DQL_MYSQL(ConnectorEnum.DQL_MYSQL.getType(), DatabaseQuartzExtractor.class), - /** - * DqlOracle - */ - DQL_ORACLE(ConnectorEnum.DQL_ORACLE.getType(), DatabaseQuartzExtractor.class), - /** - * DqlSqlServer - */ - DQL_SQL_SERVER(ConnectorEnum.DQL_SQL_SERVER.getType(), DatabaseQuartzExtractor.class), - /** - * DqlPostgreSQL - */ - DQL_POSTGRE_SQL(ConnectorEnum.DQL_POSTGRE_SQL.getType(), DatabaseQuartzExtractor.class); - - private String type; - private Class clazz; - - TimingExtractorEnum(String type, Class clazz) { - this.type = type; - this.clazz = clazz; - } - - /** - * 获取抽取器 - * - * @param type - * @return - * @throws ListenerException - */ - public static Class getExtractor(String type) throws ListenerException { - for (TimingExtractorEnum e : TimingExtractorEnum.values()) { - if (StringUtil.equals(type, e.getType())) { - return e.getClazz(); - } - } - throw new ListenerException(String.format("TimingListenerEnum type \"%s\" does not exist.", type)); - } - - public String getType() { - return type; - } - - public Class getClazz() { - return clazz; - } -} \ No newline at end of file diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/file/FileExtractor.java b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/file/FileExtractor.java index 171d376f..b39931f2 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/file/FileExtractor.java +++ b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/file/FileExtractor.java @@ -7,13 +7,13 @@ import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.NumberUtil; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.connector.config.FileConfig; -import org.dbsyncer.connector.constant.ConnectorConstant; -import org.dbsyncer.connector.file.FileConnectorMapper; +import org.dbsyncer.sdk.constant.ConnectorConstant; +import org.dbsyncer.connector.file.FileConnectorInstance; import org.dbsyncer.connector.file.FileResolver; -import org.dbsyncer.connector.model.Field; import org.dbsyncer.connector.model.FileSchema; import org.dbsyncer.listener.AbstractExtractor; import org.dbsyncer.listener.ListenerException; +import org.dbsyncer.sdk.model.Field; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.util.Assert; @@ -51,7 +51,7 @@ public class FileExtractor extends AbstractExtractor { private static final String CHARSET_NAME = "UTF-8"; private final Lock connectLock = new ReentrantLock(); private volatile boolean connected; - private FileConnectorMapper connectorMapper; + private FileConnectorInstance connectorInstance; private WatchService watchService; private Worker worker; private Map pipeline = new ConcurrentHashMap<>(); @@ -67,9 +67,9 @@ public class FileExtractor extends AbstractExtractor { return; } - connectorMapper = (FileConnectorMapper) connectorFactory.connect(connectorConfig); - final FileConfig config = connectorMapper.getConfig(); - final String mapperCacheKey = connectorFactory.getConnector(connectorMapper).getConnectorMapperCacheKey(connectorConfig); + connectorInstance = (FileConnectorInstance) connectorFactory.connect(connectorConfig); + final FileConfig config = connectorInstance.getConfig(); + final String cacheKey = connectorFactory.getConnectorService(connectorConfig.getConnectorType()).getConnectorInstanceCacheKey(connectorConfig); connected = true; separator = config.getSeparator(); @@ -83,7 +83,7 @@ public class FileExtractor extends AbstractExtractor { } worker = new Worker(); - worker.setName(new StringBuilder("file-parser-").append(mapperCacheKey).append("_").append(worker.hashCode()).toString()); + worker.setName(new StringBuilder("file-parser-").append(cacheKey).append("_").append(worker.hashCode()).toString()); worker.setDaemon(false); worker.start(); } catch (Exception e) { @@ -96,7 +96,7 @@ public class FileExtractor extends AbstractExtractor { } private void initPipeline(String fileDir) throws IOException { - for (FileSchema fileSchema : connectorMapper.getFileSchemaList()) { + for (FileSchema fileSchema : connectorInstance.getFileSchemaList()) { String fileName = fileSchema.getFileName(); String file = fileDir.concat(fileName); Assert.isTrue(new File(file).exists(), String.format("found not file '%s'", file)); diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/mysql/MySQLExtractor.java b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/mysql/MySQLExtractor.java index 44a6d15e..3c21070e 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/mysql/MySQLExtractor.java +++ b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/mysql/MySQLExtractor.java @@ -18,9 +18,9 @@ import org.dbsyncer.listener.model.ChangedOffset; import org.dbsyncer.listener.event.DDLChangedEvent; import org.dbsyncer.listener.event.RowChangedEvent; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.config.DatabaseConfig; -import org.dbsyncer.connector.constant.ConnectorConstant; -import org.dbsyncer.connector.util.DatabaseUtil; +import org.dbsyncer.sdk.config.DatabaseConfig; +import org.dbsyncer.sdk.constant.ConnectorConstant; +import org.dbsyncer.sdk.util.DatabaseUtil; import org.dbsyncer.listener.AbstractDatabaseExtractor; import org.dbsyncer.listener.ListenerException; import org.dbsyncer.listener.config.Host; diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/oracle/DqlOracleExtractor.java b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/oracle/DqlOracleExtractor.java index 95e6f27f..f76e369d 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/oracle/DqlOracleExtractor.java +++ b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/oracle/DqlOracleExtractor.java @@ -1,7 +1,7 @@ package org.dbsyncer.listener.oracle; import org.dbsyncer.listener.ChangedEvent; -import org.dbsyncer.connector.model.Field; +import org.dbsyncer.sdk.model.Field; import java.util.List; diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/oracle/OracleExtractor.java b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/oracle/OracleExtractor.java index 5a75ff8c..24e72c77 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/oracle/OracleExtractor.java +++ b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/oracle/OracleExtractor.java @@ -1,6 +1,6 @@ package org.dbsyncer.listener.oracle; -import org.dbsyncer.connector.config.DatabaseConfig; +import org.dbsyncer.sdk.config.DatabaseConfig; import org.dbsyncer.listener.AbstractDatabaseExtractor; import org.dbsyncer.listener.ListenerException; import org.dbsyncer.listener.oracle.dcn.DBChangeNotification; diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/oracle/dcn/DBChangeNotification.java b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/oracle/dcn/DBChangeNotification.java index 8e920870..516a46a2 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/oracle/dcn/DBChangeNotification.java +++ b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/oracle/dcn/DBChangeNotification.java @@ -13,7 +13,7 @@ import oracle.jdbc.dcn.TableChangeDescription; import oracle.jdbc.driver.OracleConnection; import org.dbsyncer.listener.event.RowChangedEvent; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.constant.ConnectorConstant; +import org.dbsyncer.sdk.constant.ConnectorConstant; import org.dbsyncer.listener.ListenerException; import org.dbsyncer.listener.oracle.event.DCNEvent; import org.slf4j.Logger; diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/AbstractMessageDecoder.java b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/AbstractMessageDecoder.java index ad4b7098..7c16fa7a 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/AbstractMessageDecoder.java +++ b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/AbstractMessageDecoder.java @@ -1,6 +1,6 @@ package org.dbsyncer.listener.postgresql; -import org.dbsyncer.connector.config.DatabaseConfig; +import org.dbsyncer.sdk.config.DatabaseConfig; import org.dbsyncer.listener.postgresql.column.PgColumnValue; import org.dbsyncer.listener.postgresql.enums.MessageTypeEnum; import org.postgresql.replication.LogSequenceNumber; diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/MessageDecoder.java b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/MessageDecoder.java index ebb711fd..86021363 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/MessageDecoder.java +++ b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/MessageDecoder.java @@ -2,8 +2,8 @@ package org.dbsyncer.listener.postgresql; import org.dbsyncer.listener.event.RowChangedEvent; import org.dbsyncer.connector.ConnectorFactory; -import org.dbsyncer.connector.config.DatabaseConfig; -import org.dbsyncer.connector.database.DatabaseConnectorMapper; +import org.dbsyncer.sdk.config.DatabaseConfig; +import org.dbsyncer.sdk.connector.database.DatabaseConnectorInstance; import org.postgresql.replication.LogSequenceNumber; import org.postgresql.replication.fluent.logical.ChainedLogicalStreamBuilder; @@ -16,7 +16,7 @@ import java.nio.ByteBuffer; */ public interface MessageDecoder { - default void postProcessBeforeInitialization(ConnectorFactory connectorFactory, DatabaseConnectorMapper connectorMapper) { + default void postProcessBeforeInitialization(ConnectorFactory connectorFactory, DatabaseConnectorInstance connectorInstance) { } boolean skipMessage(ByteBuffer buffer, LogSequenceNumber startLsn, LogSequenceNumber lastReceiveLsn); diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/PostgreSQLExtractor.java b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/PostgreSQLExtractor.java index 68466eed..236750b2 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/PostgreSQLExtractor.java +++ b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/PostgreSQLExtractor.java @@ -3,9 +3,9 @@ package org.dbsyncer.listener.postgresql; import org.dbsyncer.listener.model.ChangedOffset; import org.dbsyncer.listener.event.RowChangedEvent; import org.dbsyncer.common.util.BooleanUtil; -import org.dbsyncer.connector.config.DatabaseConfig; -import org.dbsyncer.connector.database.DatabaseConnectorMapper; -import org.dbsyncer.connector.util.DatabaseUtil; +import org.dbsyncer.sdk.config.DatabaseConfig; +import org.dbsyncer.sdk.connector.database.DatabaseConnectorInstance; +import org.dbsyncer.sdk.util.DatabaseUtil; import org.dbsyncer.listener.AbstractDatabaseExtractor; import org.dbsyncer.listener.ListenerException; import org.dbsyncer.listener.postgresql.enums.MessageDecoderEnum; @@ -52,7 +52,7 @@ public class PostgreSQLExtractor extends AbstractDatabaseExtractor { private final Lock connectLock = new ReentrantLock(); private volatile boolean connected; private DatabaseConfig config; - private DatabaseConnectorMapper connectorMapper; + private DatabaseConnectorInstance connectorInstance; private Connection connection; private PGReplicationStream stream; private boolean dropSlotOnClose; @@ -70,15 +70,15 @@ public class PostgreSQLExtractor extends AbstractDatabaseExtractor { return; } - connectorMapper = (DatabaseConnectorMapper) connectorFactory.connect(connectorConfig); - config = connectorMapper.getConfig(); + connectorInstance = (DatabaseConnectorInstance) connectorFactory.connect(connectorConfig); + config = connectorInstance.getConfig(); - final String walLevel = connectorMapper.execute(databaseTemplate -> databaseTemplate.queryForObject(GET_WAL_LEVEL, String.class)); + final String walLevel = connectorInstance.execute(databaseTemplate -> databaseTemplate.queryForObject(GET_WAL_LEVEL, String.class)); if (!DEFAULT_WAL_LEVEL.equals(walLevel)) { throw new ListenerException(String.format("Postgres server wal_level property must be \"%s\" but is: %s", DEFAULT_WAL_LEVEL, walLevel)); } - final boolean hasAuth = connectorMapper.execute(databaseTemplate -> { + final boolean hasAuth = connectorInstance.execute(databaseTemplate -> { Map rs = databaseTemplate.queryForMap(GET_ROLE); Boolean login = (Boolean) rs.getOrDefault("login", false); Boolean replication = (Boolean) rs.getOrDefault("replication", false); @@ -91,11 +91,11 @@ public class PostgreSQLExtractor extends AbstractDatabaseExtractor { throw new ListenerException(String.format("Postgres roles LOGIN and REPLICATION are not assigned to user: %s", config.getUsername())); } - database = connectorMapper.execute(databaseTemplate -> databaseTemplate.queryForObject(GET_DATABASE, String.class)); + database = connectorInstance.execute(databaseTemplate -> databaseTemplate.queryForObject(GET_DATABASE, String.class)); messageDecoder = MessageDecoderEnum.getMessageDecoder(config.getProperty(PLUGIN_NAME)); messageDecoder.setMetaId(metaId); messageDecoder.setConfig(config); - messageDecoder.postProcessBeforeInitialization(connectorFactory, connectorMapper); + messageDecoder.postProcessBeforeInitialization(connectorFactory, connectorInstance); dropSlotOnClose = BooleanUtil.toBoolean(config.getProperty(DROP_SLOT_ON_CLOSE, "true")); connect(); @@ -170,7 +170,7 @@ public class PostgreSQLExtractor extends AbstractDatabaseExtractor { private void createReplicationSlot(PGConnection pgConnection) throws SQLException { String slotName = messageDecoder.getSlotName(); String plugin = messageDecoder.getOutputPlugin(); - boolean existSlot = connectorMapper.execute(databaseTemplate -> databaseTemplate.queryForObject(GET_SLOT, new Object[]{database, slotName, plugin}, Integer.class) > 0); + boolean existSlot = connectorInstance.execute(databaseTemplate -> databaseTemplate.queryForObject(GET_SLOT, new Object[]{database, slotName, plugin}, Integer.class) > 0); if (!existSlot) { pgConnection.getReplicationAPI() .createReplicationSlot() @@ -184,7 +184,7 @@ public class PostgreSQLExtractor extends AbstractDatabaseExtractor { } if (!snapshot.containsKey(LSN_POSITION)) { - LogSequenceNumber lsn = connectorMapper.execute(databaseTemplate -> LogSequenceNumber.valueOf(databaseTemplate.queryForObject(GET_RESTART_LSN, new Object[] {database, slotName, plugin}, String.class))); + LogSequenceNumber lsn = connectorInstance.execute(databaseTemplate -> LogSequenceNumber.valueOf(databaseTemplate.queryForObject(GET_RESTART_LSN, new Object[] {database, slotName, plugin}, String.class))); if (null == lsn || lsn.asLong() == 0) { throw new ListenerException("No maximum LSN recorded in the database"); } @@ -204,7 +204,7 @@ public class PostgreSQLExtractor extends AbstractDatabaseExtractor { final int ATTEMPTS = 3; for (int i = 0; i < ATTEMPTS; i++) { try { - connectorMapper.execute(databaseTemplate -> { + connectorInstance.execute(databaseTemplate -> { databaseTemplate.execute(String.format("select pg_drop_replication_slot('%s')", slotName)); return true; }); diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/decoder/PgOutputMessageDecoder.java b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/decoder/PgOutputMessageDecoder.java index cc9857dd..fb981cd4 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/decoder/PgOutputMessageDecoder.java +++ b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/decoder/PgOutputMessageDecoder.java @@ -1,15 +1,15 @@ package org.dbsyncer.listener.postgresql.decoder; -import org.dbsyncer.listener.event.RowChangedEvent; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.connector.ConnectorFactory; -import org.dbsyncer.connector.database.DatabaseConnectorMapper; -import org.dbsyncer.connector.model.Field; -import org.dbsyncer.connector.model.MetaInfo; import org.dbsyncer.listener.ListenerException; +import org.dbsyncer.listener.event.RowChangedEvent; import org.dbsyncer.listener.postgresql.AbstractMessageDecoder; import org.dbsyncer.listener.postgresql.enums.MessageDecoderEnum; import org.dbsyncer.listener.postgresql.enums.MessageTypeEnum; +import org.dbsyncer.sdk.connector.database.DatabaseConnectorInstance; +import org.dbsyncer.sdk.model.Field; +import org.dbsyncer.sdk.model.MetaInfo; import org.postgresql.replication.fluent.logical.ChainedLogicalStreamBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,12 +35,12 @@ public class PgOutputMessageDecoder extends AbstractMessageDecoder { private static final String GET_TABLE_SCHEMA = "select t.oid,t.relname as tableName from pg_class t inner join (select ns.oid as nspoid, ns.nspname from pg_namespace ns where ns.nspname = '%s') as n on n.nspoid = t.relnamespace where relkind = 'r'"; private static final Map tables = new LinkedHashMap<>(); private ConnectorFactory connectorFactory; - private DatabaseConnectorMapper connectorMapper; + private DatabaseConnectorInstance connectorInstance; @Override - public void postProcessBeforeInitialization(ConnectorFactory connectorFactory, DatabaseConnectorMapper connectorMapper) { + public void postProcessBeforeInitialization(ConnectorFactory connectorFactory, DatabaseConnectorInstance connectorInstance) { this.connectorFactory = connectorFactory; - this.connectorMapper = connectorMapper; + this.connectorInstance = connectorInstance; initPublication(); readSchema(); } @@ -98,7 +98,7 @@ public class PgOutputMessageDecoder extends AbstractMessageDecoder { private void initPublication() { String pubName = getPubName(); String selectPublication = String.format("SELECT COUNT(1) FROM pg_publication WHERE pubname = '%s'", pubName); - Integer count = connectorMapper.execute(databaseTemplate -> databaseTemplate.queryForObject(selectPublication, Integer.class)); + Integer count = connectorInstance.execute(databaseTemplate -> databaseTemplate.queryForObject(selectPublication, Integer.class)); if (0 < count) { return; } @@ -107,7 +107,7 @@ public class PgOutputMessageDecoder extends AbstractMessageDecoder { try { String createPublication = String.format("CREATE PUBLICATION %s FOR ALL TABLES", pubName); logger.info("Creating Publication with statement '{}'", createPublication); - connectorMapper.execute(databaseTemplate -> { + connectorInstance.execute(databaseTemplate -> { databaseTemplate.execute(createPublication); return true; }); @@ -118,12 +118,12 @@ public class PgOutputMessageDecoder extends AbstractMessageDecoder { private void readSchema() { final String querySchema = String.format(GET_TABLE_SCHEMA, config.getSchema()); - List schemas = connectorMapper.execute(databaseTemplate -> databaseTemplate.queryForList(querySchema)); + List schemas = connectorInstance.execute(databaseTemplate -> databaseTemplate.queryForList(querySchema)); if (!CollectionUtils.isEmpty(schemas)) { schemas.forEach(map -> { Long oid = (Long) map.get("oid"); String tableName = (String) map.get("tableName"); - MetaInfo metaInfo = connectorFactory.getMetaInfo(connectorMapper, tableName); + MetaInfo metaInfo = connectorFactory.getMetaInfo(connectorInstance, tableName); Assert.notEmpty(metaInfo.getColumn(), String.format("The table column for '%s' must not be empty.", tableName)); tables.put(oid.intValue(), new TableId(oid.intValue(), tableName, metaInfo.getColumn())); }); @@ -156,7 +156,7 @@ public class PgOutputMessageDecoder extends AbstractMessageDecoder { logger.warn("The column size of table '{}' is {}, but we has been received column size is {}.", tableId.tableName, tableId.fields.size(), nColumn); // The table schema has been changed, we should be get a new table schema from db. - MetaInfo metaInfo = connectorFactory.getMetaInfo(connectorMapper, tableId.tableName); + MetaInfo metaInfo = connectorFactory.getMetaInfo(connectorInstance, tableId.tableName); if (CollectionUtils.isEmpty(metaInfo.getColumn())) { throw new ListenerException(String.format("The table column for '%s' is empty.", tableId.tableName)); } diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/decoder/TestDecodingMessageDecoder.java b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/decoder/TestDecodingMessageDecoder.java index 9e46deae..02392d45 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/decoder/TestDecodingMessageDecoder.java +++ b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/decoder/TestDecodingMessageDecoder.java @@ -2,7 +2,7 @@ package org.dbsyncer.listener.postgresql.decoder; import org.dbsyncer.common.column.Lexer; import org.dbsyncer.listener.event.RowChangedEvent; -import org.dbsyncer.connector.constant.ConnectorConstant; +import org.dbsyncer.sdk.constant.ConnectorConstant; import org.dbsyncer.listener.postgresql.AbstractMessageDecoder; import org.dbsyncer.listener.postgresql.enums.MessageDecoderEnum; import org.dbsyncer.listener.postgresql.enums.MessageTypeEnum; diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/AbstractQuartzExtractor.java b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/AbstractQuartzExtractor.java index 1f559759..4341d85c 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/AbstractQuartzExtractor.java +++ b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/AbstractQuartzExtractor.java @@ -1,17 +1,17 @@ package org.dbsyncer.listener.quartz; -import org.dbsyncer.listener.event.ScanChangedEvent; import org.dbsyncer.common.model.Result; -import org.dbsyncer.connector.scheduled.ScheduledTaskJob; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.common.util.UUIDUtil; -import org.dbsyncer.connector.config.ReaderConfig; -import org.dbsyncer.connector.constant.ConnectorConstant; -import org.dbsyncer.connector.model.Table; -import org.dbsyncer.connector.util.PrimaryKeyUtil; +import org.dbsyncer.connector.scheduled.ScheduledTaskJob; import org.dbsyncer.listener.AbstractExtractor; -import org.dbsyncer.sdk.spi.ConnectorMapper; +import org.dbsyncer.listener.event.ScanChangedEvent; +import org.dbsyncer.sdk.config.ReaderConfig; +import org.dbsyncer.sdk.connector.ConnectorInstance; +import org.dbsyncer.sdk.constant.ConnectorConstant; +import org.dbsyncer.sdk.model.Table; +import org.dbsyncer.sdk.util.PrimaryKeyUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -103,14 +103,14 @@ public abstract class AbstractQuartzExtractor extends AbstractExtractor implemen boolean supportedCursor = StringUtil.isNotBlank(command.get(ConnectorConstant.OPERTION_QUERY_CURSOR)); // 检查增量点 - ConnectorMapper connectionMapper = connectorFactory.connect(connectorConfig); + ConnectorInstance connectionInstance = connectorFactory.connect(connectorConfig); Point point = checkLastPoint(command, index); int pageIndex = 1; Object[] cursors = PrimaryKeyUtil.getLastCursors(snapshot.get(index + CURSOR)); while (running) { ReaderConfig readerConfig = new ReaderConfig(table, point.getCommand(), point.getArgs(), supportedCursor, cursors, pageIndex++, READ_NUM); - Result reader = connectorFactory.reader(connectionMapper, readerConfig); + Result reader = connectorFactory.reader(connectionInstance, readerConfig); List data = reader.getSuccessData(); if (CollectionUtils.isEmpty(data)) { cursors = new Object[0]; diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/DatabaseQuartzExtractor.java b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/DatabaseQuartzExtractor.java index ab53fdbb..ebf84282 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/DatabaseQuartzExtractor.java +++ b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/DatabaseQuartzExtractor.java @@ -2,7 +2,7 @@ package org.dbsyncer.listener.quartz; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.constant.ConnectorConstant; +import org.dbsyncer.sdk.constant.ConnectorConstant; import org.dbsyncer.listener.enums.QuartzFilterEnum; import org.springframework.util.Assert; diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/ESQuartzExtractor.java b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/ESQuartzExtractor.java index 3b4adb9c..fd60924f 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/ESQuartzExtractor.java +++ b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/ESQuartzExtractor.java @@ -3,12 +3,16 @@ package org.dbsyncer.listener.quartz; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.JsonUtil; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.model.Filter; -import org.dbsyncer.connector.constant.ConnectorConstant; import org.dbsyncer.listener.ListenerException; import org.dbsyncer.listener.enums.QuartzFilterEnum; +import org.dbsyncer.sdk.constant.ConnectorConstant; +import org.dbsyncer.sdk.model.Filter; -import java.util.*; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; /** * ES定时抽取 diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/TableGroupQuartzCommand.java b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/TableGroupQuartzCommand.java index 72990e73..a2a734a1 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/TableGroupQuartzCommand.java +++ b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/TableGroupQuartzCommand.java @@ -1,7 +1,7 @@ package org.dbsyncer.listener.quartz; -import org.dbsyncer.connector.model.Table; -import org.dbsyncer.connector.util.PrimaryKeyUtil; +import org.dbsyncer.sdk.model.Table; +import org.dbsyncer.sdk.util.PrimaryKeyUtil; import java.util.List; import java.util.Map; diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/sqlserver/SqlServerExtractor.java b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/sqlserver/SqlServerExtractor.java index e029ce0a..37cbb6dd 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/sqlserver/SqlServerExtractor.java +++ b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/sqlserver/SqlServerExtractor.java @@ -4,9 +4,9 @@ import com.microsoft.sqlserver.jdbc.SQLServerException; import org.dbsyncer.listener.model.ChangedOffset; import org.dbsyncer.listener.event.RowChangedEvent; import org.dbsyncer.common.util.CollectionUtils; -import org.dbsyncer.connector.config.DatabaseConfig; -import org.dbsyncer.connector.constant.ConnectorConstant; -import org.dbsyncer.connector.database.DatabaseConnectorMapper; +import org.dbsyncer.sdk.config.DatabaseConfig; +import org.dbsyncer.sdk.constant.ConnectorConstant; +import org.dbsyncer.sdk.connector.database.DatabaseConnectorInstance; import org.dbsyncer.listener.AbstractDatabaseExtractor; import org.dbsyncer.listener.ListenerException; import org.dbsyncer.listener.enums.TableOperationEnum; @@ -62,7 +62,7 @@ public class SqlServerExtractor extends AbstractDatabaseExtractor { private volatile boolean connected; private Set tables; private Set changeTables; - private DatabaseConnectorMapper connectorMapper; + private DatabaseConnectorInstance connectorInstance; private Worker worker; private Lsn lastLsn; private String serverName; @@ -136,8 +136,8 @@ public class SqlServerExtractor extends AbstractDatabaseExtractor { private void connect() { if (connectorFactory.isAlive(connectorConfig)) { - connectorMapper = (DatabaseConnectorMapper) connectorFactory.connect(connectorConfig); - DatabaseConfig cfg = connectorMapper.getConfig(); + connectorInstance = (DatabaseConnectorInstance) connectorFactory.connect(connectorConfig); + DatabaseConfig cfg = connectorInstance.getConfig(); serverName = cfg.getUrl(); schema = cfg.getSchema(); } @@ -221,7 +221,7 @@ public class SqlServerExtractor extends AbstractDatabaseExtractor { } private void execute(String... sqlStatements) { - connectorMapper.execute(databaseTemplate -> { + connectorInstance.execute(databaseTemplate -> { for (String sqlStatement : sqlStatements) { if (sqlStatement != null) { logger.info("executing '{}'", sqlStatement); @@ -313,7 +313,7 @@ public class SqlServerExtractor extends AbstractDatabaseExtractor { } private T query(String preparedQuerySql, StatementPreparer statementPreparer, ResultSetMapper mapper) { - Object execute = connectorMapper.execute(databaseTemplate -> { + Object execute = connectorInstance.execute(databaseTemplate -> { PreparedStatement ps = null; ResultSet rs = null; T apply = null; diff --git a/dbsyncer-listener/src/main/test/BinaryLogRemoteClientTest.java b/dbsyncer-listener/src/main/test/BinaryLogRemoteClientTest.java index 67ccaea9..b6083e11 100644 --- a/dbsyncer-listener/src/main/test/BinaryLogRemoteClientTest.java +++ b/dbsyncer-listener/src/main/test/BinaryLogRemoteClientTest.java @@ -1,5 +1,5 @@ import com.github.shyiko.mysql.binlog.event.*; -import org.dbsyncer.connector.constant.ConnectorConstant; +import org.dbsyncer.sdk.constant.ConnectorConstant; import org.dbsyncer.listener.mysql.BinaryLogClient; import org.dbsyncer.listener.mysql.BinaryLogRemoteClient; import org.junit.Test; diff --git a/dbsyncer-listener/src/main/test/KafkaClientTest.java b/dbsyncer-listener/src/main/test/KafkaClientTest.java index bf10561f..0f33214f 100644 --- a/dbsyncer-listener/src/main/test/KafkaClientTest.java +++ b/dbsyncer-listener/src/main/test/KafkaClientTest.java @@ -6,16 +6,17 @@ import org.dbsyncer.connector.enums.KafkaFieldTypeEnum; import org.dbsyncer.connector.kafka.KafkaClient; import org.dbsyncer.connector.kafka.serialization.JsonToMapDeserializer; import org.dbsyncer.connector.kafka.serialization.MapToJsonSerializer; -import org.dbsyncer.connector.model.Field; import org.dbsyncer.connector.util.KafkaUtil; +import org.dbsyncer.sdk.model.Field; import org.junit.After; import org.junit.Before; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.sql.Timestamp; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; import java.util.concurrent.TimeUnit; /** diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/FullPuller.java b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/FullPuller.java index 25772868..44ffa5ae 100644 --- a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/FullPuller.java +++ b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/FullPuller.java @@ -2,7 +2,7 @@ package org.dbsyncer.manager.impl; import org.dbsyncer.common.util.NumberUtil; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.util.PrimaryKeyUtil; +import org.dbsyncer.sdk.util.PrimaryKeyUtil; import org.dbsyncer.manager.AbstractPuller; import org.dbsyncer.parser.ParserComponent; import org.dbsyncer.parser.ProfileComponent; diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/IncrementPuller.java b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/IncrementPuller.java index dcbff6d9..df4d2430 100644 --- a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/IncrementPuller.java +++ b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/IncrementPuller.java @@ -1,32 +1,32 @@ package org.dbsyncer.manager.impl; -import org.dbsyncer.listener.model.ChangedOffset; -import org.dbsyncer.manager.AbstractPuller; -import org.dbsyncer.parser.event.RefreshOffsetEvent; -import org.dbsyncer.connector.scheduled.ScheduledTaskJob; -import org.dbsyncer.connector.scheduled.ScheduledTaskService; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.connector.ConnectorFactory; -import org.dbsyncer.connector.model.Table; +import org.dbsyncer.connector.scheduled.ScheduledTaskJob; +import org.dbsyncer.connector.scheduled.ScheduledTaskService; import org.dbsyncer.listener.AbstractExtractor; import org.dbsyncer.listener.Extractor; import org.dbsyncer.listener.Listener; import org.dbsyncer.listener.config.ListenerConfig; import org.dbsyncer.listener.enums.ListenerTypeEnum; +import org.dbsyncer.listener.model.ChangedOffset; import org.dbsyncer.listener.quartz.AbstractQuartzExtractor; import org.dbsyncer.listener.quartz.TableGroupQuartzCommand; +import org.dbsyncer.manager.AbstractPuller; import org.dbsyncer.manager.ManagerException; +import org.dbsyncer.parser.LogService; +import org.dbsyncer.parser.LogType; import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.parser.consumer.impl.LogConsumer; import org.dbsyncer.parser.consumer.impl.QuartzConsumer; +import org.dbsyncer.parser.event.RefreshOffsetEvent; import org.dbsyncer.parser.flush.impl.BufferActuatorRouter; -import org.dbsyncer.parser.LogService; -import org.dbsyncer.parser.LogType; import org.dbsyncer.parser.model.Connector; import org.dbsyncer.parser.model.Mapping; import org.dbsyncer.parser.model.Meta; import org.dbsyncer.parser.model.TableGroup; import org.dbsyncer.sdk.model.ConnectorConfig; +import org.dbsyncer.sdk.model.Table; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.ApplicationListener; diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/PreloadTemplate.java b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/PreloadTemplate.java index 67e82630..bc3f0b7d 100644 --- a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/PreloadTemplate.java +++ b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/PreloadTemplate.java @@ -19,7 +19,7 @@ import org.dbsyncer.parser.model.Mapping; import org.dbsyncer.parser.model.Meta; import org.dbsyncer.parser.model.OperationConfig; import org.dbsyncer.plugin.PluginFactory; -import org.dbsyncer.sdk.spi.ConnectorMapper; +import org.dbsyncer.sdk.connector.ConnectorInstance; import org.dbsyncer.storage.StorageService; import org.dbsyncer.storage.constant.ConfigConstant; import org.dbsyncer.storage.enums.StorageEnum; @@ -200,8 +200,8 @@ public final class PreloadTemplate implements ApplicationListener { try { connectorFactory.disconnect(connector.getConfig()); - ConnectorMapper mapper = connectorFactory.connect(connector.getConfig()); - logger.info("Completed connection {} {}", connector.getConfig().getConnectorType(), mapper.getServiceUrl()); + ConnectorInstance connectorInstance = connectorFactory.connect(connector.getConfig()); + logger.info("Completed connection {} {}", connector.getConfig().getConnectorType(), connectorInstance.getServiceUrl()); } catch (Exception e) { logger.error("连接配置异常", e); logService.log(LogType.ConnectorLog.FAILED, e.getMessage()); diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ParserComponent.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ParserComponent.java index 62163576..67494d57 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ParserComponent.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ParserComponent.java @@ -1,11 +1,11 @@ package org.dbsyncer.parser; import org.dbsyncer.common.model.Result; -import org.dbsyncer.connector.model.MetaInfo; import org.dbsyncer.parser.model.BatchWriter; import org.dbsyncer.parser.model.Mapping; import org.dbsyncer.parser.model.TableGroup; import org.dbsyncer.parser.model.Task; +import org.dbsyncer.sdk.model.MetaInfo; import org.dbsyncer.sdk.plugin.PluginContext; import java.util.Map; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ProfileComponent.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ProfileComponent.java index 5842b5c3..4358e4c4 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ProfileComponent.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ProfileComponent.java @@ -3,9 +3,7 @@ */ package org.dbsyncer.parser; -import org.dbsyncer.connector.enums.ConnectorEnum; import org.dbsyncer.connector.enums.FilterEnum; -import org.dbsyncer.connector.enums.OperationEnum; import org.dbsyncer.listener.enums.QuartzFilterEnum; import org.dbsyncer.parser.enums.ConvertEnum; import org.dbsyncer.parser.model.ConfigModel; @@ -16,6 +14,7 @@ import org.dbsyncer.parser.model.ProjectGroup; import org.dbsyncer.parser.model.SystemConfig; import org.dbsyncer.parser.model.TableGroup; import org.dbsyncer.parser.model.UserConfig; +import org.dbsyncer.sdk.enums.OperationEnum; import org.dbsyncer.storage.constant.ConfigConstant; import org.dbsyncer.storage.enums.StorageDataStatusEnum; @@ -143,13 +142,6 @@ public interface ProfileComponent { List getMetaAll(); - /** - * 获取所有连接器类型 - * - * @return - */ - List getConnectorEnumAll(); - /** * 获取所有条件类型 * diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/impl/LogConsumer.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/impl/LogConsumer.java index c35ee769..2fdc61db 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/impl/LogConsumer.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/impl/LogConsumer.java @@ -7,11 +7,11 @@ import org.dbsyncer.listener.event.CommonChangedEvent; import org.dbsyncer.listener.event.DDLChangedEvent; import org.dbsyncer.listener.event.RowChangedEvent; import org.dbsyncer.common.util.CollectionUtils; -import org.dbsyncer.connector.model.Table; import org.dbsyncer.parser.consumer.AbstractConsumer; import org.dbsyncer.parser.model.FieldPicker; import org.dbsyncer.parser.model.TableGroup; import org.dbsyncer.parser.util.PickerUtil; +import org.dbsyncer.sdk.model.Table; import java.util.ArrayList; import java.util.LinkedHashMap; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ddl/AlterStrategy.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ddl/AlterStrategy.java index c317ee78..3d463ad8 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ddl/AlterStrategy.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ddl/AlterStrategy.java @@ -1,8 +1,8 @@ package org.dbsyncer.parser.ddl; import net.sf.jsqlparser.statement.alter.AlterExpression; -import org.dbsyncer.connector.config.DDLConfig; import org.dbsyncer.parser.model.FieldMapping; +import org.dbsyncer.sdk.config.DDLConfig; import java.util.List; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ddl/DDLParser.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ddl/DDLParser.java index a7bb91ee..8517291a 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ddl/DDLParser.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ddl/DDLParser.java @@ -1,8 +1,8 @@ package org.dbsyncer.parser.ddl; -import org.dbsyncer.connector.config.DDLConfig; -import org.dbsyncer.connector.model.MetaInfo; import org.dbsyncer.parser.model.FieldMapping; +import org.dbsyncer.sdk.config.DDLConfig; +import org.dbsyncer.sdk.model.MetaInfo; import java.util.List; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ddl/alter/AddStrategy.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ddl/alter/AddStrategy.java index 5139c925..85342452 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ddl/alter/AddStrategy.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ddl/alter/AddStrategy.java @@ -6,11 +6,11 @@ import net.sf.jsqlparser.statement.alter.AlterExpression; import net.sf.jsqlparser.statement.create.table.Index; import net.sf.jsqlparser.statement.create.table.Index.ColumnParams; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.config.DDLConfig; -import org.dbsyncer.connector.enums.DDLOperationEnum; -import org.dbsyncer.connector.model.Field; import org.dbsyncer.parser.ddl.AlterStrategy; import org.dbsyncer.parser.model.FieldMapping; +import org.dbsyncer.sdk.config.DDLConfig; +import org.dbsyncer.sdk.enums.DDLOperationEnum; +import org.dbsyncer.sdk.model.Field; /** * 解析add的属性 exampleSql: ALTER TABLE cost ADD duan INT after(before) `tmp`; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ddl/alter/ChangeStrategy.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ddl/alter/ChangeStrategy.java index 3144af57..2fa1c50a 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ddl/alter/ChangeStrategy.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ddl/alter/ChangeStrategy.java @@ -2,10 +2,10 @@ package org.dbsyncer.parser.ddl.alter; import net.sf.jsqlparser.statement.alter.AlterExpression; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.config.DDLConfig; -import org.dbsyncer.connector.enums.DDLOperationEnum; import org.dbsyncer.parser.ddl.AlterStrategy; import org.dbsyncer.parser.model.FieldMapping; +import org.dbsyncer.sdk.config.DDLConfig; +import org.dbsyncer.sdk.enums.DDLOperationEnum; import java.util.List; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ddl/alter/DropStrategy.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ddl/alter/DropStrategy.java index 58d13e2e..f1a47003 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ddl/alter/DropStrategy.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ddl/alter/DropStrategy.java @@ -1,21 +1,17 @@ package org.dbsyncer.parser.ddl.alter; -import java.util.LinkedList; import net.sf.jsqlparser.statement.alter.AlterExpression; -import net.sf.jsqlparser.statement.create.table.Index; -import net.sf.jsqlparser.statement.create.table.Index.ColumnParams; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.config.DDLConfig; -import org.dbsyncer.connector.enums.DDLOperationEnum; -import org.dbsyncer.connector.model.Field; import org.dbsyncer.parser.ddl.AlterStrategy; import org.dbsyncer.parser.model.FieldMapping; +import org.dbsyncer.sdk.config.DDLConfig; +import org.dbsyncer.sdk.enums.DDLOperationEnum; +import org.dbsyncer.sdk.model.Field; import java.util.List; /** * 解析drop - * * @author life */ @@ -23,22 +19,23 @@ public class DropStrategy implements AlterStrategy { @Override public void parse(AlterExpression expression, DDLConfig ddlConfig, List originalFieldMappings) { - if (expression.getColumnName() !=null){ - dropColumn(expression,ddlConfig,originalFieldMappings); + if (expression.getColumnName() != null) { + dropColumn(expression, ddlConfig, originalFieldMappings); } - if (expression.getIndex() !=null){ - dropIndex(expression,originalFieldMappings); + if (expression.getIndex() != null) { + dropIndex(expression, originalFieldMappings); } ddlConfig.setDdlOperationEnum(DDLOperationEnum.ALTER_DROP); } /** - * example: ALTER TABLE test_table DROP dis; + * example: ALTER TABLE test_table DROP dis; + * * @param expression * @param ddlConfig * @param originalFieldMappings */ - private void dropColumn(AlterExpression expression, DDLConfig ddlConfig, List originalFieldMappings){ + private void dropColumn(AlterExpression expression, DDLConfig ddlConfig, List originalFieldMappings) { String columnName = StringUtil.replace(expression.getColumnName(), "`", ""); Field field = new Field(columnName, null, 0); //需要把列替换成目标的列名 @@ -51,11 +48,12 @@ public class DropStrategy implements AlterStrategy { /** * 貌似不需要做什么，我们目前没有字段分索引，再考虑 - * * example: ALTER TABLE test_table drop index name; + * example: ALTER TABLE test_table drop index name; + * * @param expression * @param originalFieldMappings */ - private void dropIndex(AlterExpression expression, List originalFieldMappings){ + private void dropIndex(AlterExpression expression, List originalFieldMappings) { // Index index = expression.getIndex(); // String names= index.getName(); // String[] nameList = StringUtil.split(names,"."); diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ddl/alter/ModifyStrategy.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ddl/alter/ModifyStrategy.java index 212d1ccd..fd960fdb 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ddl/alter/ModifyStrategy.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ddl/alter/ModifyStrategy.java @@ -2,10 +2,10 @@ package org.dbsyncer.parser.ddl.alter; import net.sf.jsqlparser.statement.alter.AlterExpression; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.config.DDLConfig; -import org.dbsyncer.connector.enums.DDLOperationEnum; import org.dbsyncer.parser.ddl.AlterStrategy; import org.dbsyncer.parser.model.FieldMapping; +import org.dbsyncer.sdk.config.DDLConfig; +import org.dbsyncer.sdk.enums.DDLOperationEnum; import java.util.List; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ddl/impl/DDLParserImpl.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ddl/impl/DDLParserImpl.java index 90bba6b2..d2c6ac35 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ddl/impl/DDLParserImpl.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ddl/impl/DDLParserImpl.java @@ -7,13 +7,9 @@ import net.sf.jsqlparser.statement.alter.Alter; import net.sf.jsqlparser.statement.alter.AlterExpression; import net.sf.jsqlparser.statement.alter.AlterOperation; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.Connector; import org.dbsyncer.connector.ConnectorFactory; -import org.dbsyncer.connector.config.DDLConfig; -import org.dbsyncer.connector.database.Database; -import org.dbsyncer.connector.enums.DDLOperationEnum; -import org.dbsyncer.connector.model.Field; -import org.dbsyncer.connector.model.MetaInfo; +import org.dbsyncer.sdk.config.DDLConfig; +import org.dbsyncer.sdk.connector.database.Database; import org.dbsyncer.parser.ddl.AlterStrategy; import org.dbsyncer.parser.ddl.DDLParser; import org.dbsyncer.parser.ddl.alter.AddStrategy; @@ -21,6 +17,10 @@ import org.dbsyncer.parser.ddl.alter.ChangeStrategy; import org.dbsyncer.parser.ddl.alter.DropStrategy; import org.dbsyncer.parser.ddl.alter.ModifyStrategy; import org.dbsyncer.parser.model.FieldMapping; +import org.dbsyncer.sdk.enums.DDLOperationEnum; +import org.dbsyncer.sdk.model.Field; +import org.dbsyncer.sdk.model.MetaInfo; +import org.dbsyncer.sdk.spi.ConnectorService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; @@ -63,14 +63,14 @@ public class DDLParserImpl implements DDLParser { @Override public DDLConfig parseDDlConfig(String sql, String targetConnectorType, String targetTableName, List originalFieldMappings) { - Connector connector = connectorFactory.getConnector(targetConnectorType); + ConnectorService connectorService = connectorFactory.getConnectorService(targetConnectorType); // 替换为目标库执行SQL DDLConfig ddlConfig = new DDLConfig(); try { Statement statement = CCJSqlParserUtil.parse(sql); if (statement instanceof Alter) { Alter alter = (Alter) statement; - Database database = (Database) connector; + Database database = (Database) connectorService; String quotation = database.buildSqlWithQuotation(); // 替换成目标表名 alter.getTable().setName(new StringBuilder(quotation).append(targetTableName).append(quotation).toString()); diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/GeneralBufferActuator.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/GeneralBufferActuator.java index eeec03d2..396de955 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/GeneralBufferActuator.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/GeneralBufferActuator.java @@ -5,10 +5,8 @@ import org.dbsyncer.common.model.Result; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.connector.ConnectorFactory; -import org.dbsyncer.connector.config.DDLConfig; -import org.dbsyncer.connector.constant.ConnectorConstant; -import org.dbsyncer.connector.enums.ConnectorEnum; -import org.dbsyncer.connector.model.MetaInfo; +import org.dbsyncer.sdk.config.DDLConfig; +import org.dbsyncer.sdk.constant.ConnectorConstant; import org.dbsyncer.parser.ParserComponent; import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.parser.ddl.DDLParser; @@ -26,9 +24,10 @@ import org.dbsyncer.parser.strategy.FlushStrategy; import org.dbsyncer.parser.util.ConvertUtil; import org.dbsyncer.parser.util.PickerUtil; import org.dbsyncer.plugin.PluginFactory; +import org.dbsyncer.sdk.connector.ConnectorInstance; import org.dbsyncer.sdk.model.ConnectorConfig; import org.dbsyncer.plugin.impl.IncrementPluginContext; -import org.dbsyncer.sdk.spi.ConnectorMapper; +import org.dbsyncer.sdk.model.MetaInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.ApplicationContext; @@ -139,13 +138,13 @@ public class GeneralBufferActuator extends AbstractBufferActuator originalFieldMappings = tableGroup.getFieldMapping(); DDLConfig targetDDLConfig = ddlParser.parseDDlConfig(response.getSql(), tConnType, targetTableName, originalFieldMappings); - final ConnectorMapper tConnectorMapper = connectorFactory.connect(tConnConfig); - Result result = connectorFactory.writerDDL(tConnectorMapper, targetDDLConfig); + final ConnectorInstance tConnectorInstance = connectorFactory.connect(tConnConfig); + Result result = connectorFactory.writerDDL(tConnectorInstance, targetDDLConfig); result.setTableGroupId(tableGroup.getId()); result.setTargetTableGroupName(targetTableName); diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/StorageBufferActuator.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/StorageBufferActuator.java index f5d25408..75057b12 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/StorageBufferActuator.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/StorageBufferActuator.java @@ -32,7 +32,7 @@ public final class StorageBufferActuator extends AbstractBufferActuator command) { - ConnectorMapper connectorMapper = connectorFactory.connect(getConnectorConfig(connectorId)); - return connectorFactory.getCount(connectorMapper, command); + ConnectorInstance connectorInstance = connectorFactory.connect(getConnectorConfig(connectorId)); + return connectorFactory.getCount(connectorInstance, command); } @Override @@ -142,10 +142,10 @@ public class ParserComponentImpl implements ParserComponent { boolean supportedCursor = StringUtil.isNotBlank(command.get(ConnectorConstant.OPERTION_QUERY_CURSOR)); int pageSize = mapping.getReadNum(); int batchSize = mapping.getBatchNum(); - final ConnectorMapper sConnectorMapper = connectorFactory.connect(sConfig); - final ConnectorMapper tConnectorMapper = connectorFactory.connect(tConfig); + final ConnectorInstance sConnectorInstance = connectorFactory.connect(sConfig); + final ConnectorInstance tConnectorInstance = connectorFactory.connect(tConfig); final String event = ConnectorConstant.OPERTION_INSERT; - final FullPluginContext context = new FullPluginContext(sConnectorMapper, tConnectorMapper, sTableName, tTableName, event); + final FullPluginContext context = new FullPluginContext(sConnectorInstance, tConnectorInstance, sTableName, tTableName, event); for (; ; ) { if (!task.isRunning()) { @@ -155,7 +155,7 @@ public class ParserComponentImpl implements ParserComponent { // 1、获取数据源数据 ReaderConfig readerConfig = new ReaderConfig(sourceTable, command, new ArrayList<>(), supportedCursor, task.getCursors(), task.getPageIndex(), pageSize); - Result reader = connectorFactory.reader(sConnectorMapper, readerConfig); + Result reader = connectorFactory.reader(sConnectorInstance, readerConfig); List source = reader.getSuccessData(); if (CollectionUtils.isEmpty(source)) { logger.info("完成全量同步任务:{}, [{}] >> [{}]", metaId, sTableName, tTableName); @@ -174,7 +174,7 @@ public class ParserComponentImpl implements ParserComponent { pluginFactory.convert(group.getPlugin(), context); // 5、写入目标源 - BatchWriter batchWriter = new BatchWriter(tConnectorMapper, command, tTableName, event, picker.getTargetFields(), target, batchSize); + BatchWriter batchWriter = new BatchWriter(tConnectorInstance, command, tTableName, event, picker.getTargetFields(), target, batchSize); Result result = writeBatch(context, batchWriter, executor); // 6、更新结果 @@ -214,7 +214,7 @@ public class ParserComponentImpl implements ParserComponent { int total = dataList.size(); // 单次任务 if (total <= batchSize) { - return connectorFactory.writer(batchWriter.getConnectorMapper(), new WriterBatchConfig(tableName, event, command, fields, dataList)); + return connectorFactory.writer(batchWriter.getConnectorInstance(), new WriterBatchConfig(tableName, event, command, fields, dataList)); } // 批量任务, 拆分 @@ -236,7 +236,7 @@ public class ParserComponentImpl implements ParserComponent { executor.execute(() -> { try { - Result w = connectorFactory.writer(batchWriter.getConnectorMapper(), new WriterBatchConfig(tableName, event, command, fields, data)); + Result w = connectorFactory.writer(batchWriter.getConnectorInstance(), new WriterBatchConfig(tableName, event, command, fields, data)); result.addSuccessData(w.getSuccessData()); result.addFailData(w.getFailData()); result.getError().append(w.getError()); diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/ProfileComponentImpl.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/ProfileComponentImpl.java index e03f3371..3f617efd 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/ProfileComponentImpl.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/ProfileComponentImpl.java @@ -5,9 +5,8 @@ package org.dbsyncer.parser.impl; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.JsonUtil; -import org.dbsyncer.connector.enums.ConnectorEnum; +import org.dbsyncer.connector.ConnectorFactory; import org.dbsyncer.connector.enums.FilterEnum; -import org.dbsyncer.connector.enums.OperationEnum; import org.dbsyncer.listener.enums.QuartzFilterEnum; import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.parser.enums.CommandEnum; @@ -23,7 +22,9 @@ import org.dbsyncer.parser.model.QueryConfig; import org.dbsyncer.parser.model.SystemConfig; import org.dbsyncer.parser.model.TableGroup; import org.dbsyncer.parser.model.UserConfig; +import org.dbsyncer.sdk.enums.OperationEnum; import org.dbsyncer.sdk.model.ConnectorConfig; +import org.dbsyncer.sdk.spi.ConnectorService; import org.dbsyncer.storage.enums.StorageDataStatusEnum; import org.springframework.stereotype.Component; import org.springframework.util.Assert; @@ -46,6 +47,9 @@ public class ProfileComponentImpl implements ProfileComponent { @Resource private OperationTemplate operationTemplate; + @Resource + private ConnectorFactory connectorFactory; + @Override public Connector parseConnector(String json) { Map conn = JsonUtil.parseMap(json); @@ -53,10 +57,9 @@ public class ProfileComponentImpl implements ProfileComponent { Connector connector = JsonUtil.jsonToObj(conn.toString(), Connector.class); Assert.notNull(connector, "Connector can not be null."); String connectorType = (String) config.get("connectorType"); - ConnectorEnum connectorEnum = ConnectorEnum.getConnectorEnum(connectorType); - ConnectorConfig obj = JsonUtil.jsonToObj(config.toString(), connectorEnum.getConfigClass()); - obj.setConnectorType(connectorEnum.getType()); - connector.setConfig(obj); + ConnectorService connectorService = connectorFactory.getConnectorService(connectorType); + Class configClass = connectorService.getConfigClass(); + connector.setConfig(JsonUtil.jsonToObj(config.toString(), configClass)); return connector; } @@ -174,11 +177,6 @@ public class ProfileComponentImpl implements ProfileComponent { return operationTemplate.queryAll(Meta.class); } - @Override - public List getConnectorEnumAll() { - return Arrays.asList(ConnectorEnum.values()); - } - @Override public List getOperationEnumAll() { return Arrays.asList(OperationEnum.values()); diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/AbstractConfigModel.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/AbstractConfigModel.java index 9bfb4e2e..98ffa14b 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/AbstractConfigModel.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/AbstractConfigModel.java @@ -1,7 +1,7 @@ package org.dbsyncer.parser.model; -import org.dbsyncer.connector.model.Filter; import org.dbsyncer.plugin.model.Plugin; +import org.dbsyncer.sdk.model.Filter; import java.util.List; import java.util.Map; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/BatchWriter.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/BatchWriter.java index 6ea8bb0e..5a1055c2 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/BatchWriter.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/BatchWriter.java @@ -1,14 +1,14 @@ package org.dbsyncer.parser.model; -import org.dbsyncer.connector.model.Field; -import org.dbsyncer.sdk.spi.ConnectorMapper; +import org.dbsyncer.sdk.connector.ConnectorInstance; +import org.dbsyncer.sdk.model.Field; import java.util.List; import java.util.Map; public final class BatchWriter { - private ConnectorMapper connectorMapper; + private ConnectorInstance connectorInstance; private Map command; private String tableName; private String event; @@ -16,9 +16,9 @@ public final class BatchWriter { private List dataList; private int batchSize; - public BatchWriter(ConnectorMapper connectorMapper, Map command, String tableName, String event, + public BatchWriter(ConnectorInstance connectorInstance, Map command, String tableName, String event, List fields, List dataList, int batchSize) { - this.connectorMapper = connectorMapper; + this.connectorInstance = connectorInstance; this.command = command; this.tableName = tableName; this.event = event; @@ -27,8 +27,8 @@ public final class BatchWriter { this.batchSize = batchSize; } - public ConnectorMapper getConnectorMapper() { - return connectorMapper; + public ConnectorInstance getConnectorInstance() { + return connectorInstance; } public Map getCommand() { diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/Connector.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/Connector.java index 26d92fab..2e7c5004 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/Connector.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/Connector.java @@ -1,7 +1,7 @@ package org.dbsyncer.parser.model; -import org.dbsyncer.connector.model.Table; import org.dbsyncer.sdk.model.ConnectorConfig; +import org.dbsyncer.sdk.model.Table; import org.dbsyncer.storage.constant.ConfigConstant; import java.util.List; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/FieldMapping.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/FieldMapping.java index 4bc40d8c..f5d05a37 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/FieldMapping.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/FieldMapping.java @@ -1,6 +1,6 @@ package org.dbsyncer.parser.model; -import org.dbsyncer.connector.model.Field; +import org.dbsyncer.sdk.model.Field; /** * 字段映射关系 diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/FieldPicker.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/FieldPicker.java index cae543de..ce02b15c 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/FieldPicker.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/FieldPicker.java @@ -8,9 +8,9 @@ import org.dbsyncer.common.util.DateFormatUtil; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.connector.CompareFilter; import org.dbsyncer.connector.enums.FilterEnum; -import org.dbsyncer.connector.enums.OperationEnum; -import org.dbsyncer.connector.model.Field; -import org.dbsyncer.connector.model.Filter; +import org.dbsyncer.sdk.enums.OperationEnum; +import org.dbsyncer.sdk.model.Field; +import org.dbsyncer.sdk.model.Filter; import org.springframework.util.Assert; import java.sql.Date; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/Mapping.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/Mapping.java index 2f2c4249..06c01af2 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/Mapping.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/Mapping.java @@ -1,8 +1,8 @@ package org.dbsyncer.parser.model; -import org.dbsyncer.connector.model.Field; import org.dbsyncer.listener.config.ListenerConfig; import org.dbsyncer.sdk.enums.ModelEnum; +import org.dbsyncer.sdk.model.Field; import org.dbsyncer.storage.constant.ConfigConstant; import java.util.List; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/Picker.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/Picker.java index 08b52717..9e555501 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/Picker.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/Picker.java @@ -2,7 +2,7 @@ package org.dbsyncer.parser.model; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.model.Field; +import org.dbsyncer.sdk.model.Field; import java.util.ArrayList; import java.util.Collections; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/TableGroup.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/TableGroup.java index cdd2353b..53184f3d 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/TableGroup.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/TableGroup.java @@ -1,6 +1,6 @@ package org.dbsyncer.parser.model; -import org.dbsyncer.connector.model.Table; +import org.dbsyncer.sdk.model.Table; import org.dbsyncer.storage.constant.ConfigConstant; import java.util.List; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/util/PickerUtil.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/util/PickerUtil.java index bfacd225..a0d64406 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/util/PickerUtil.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/util/PickerUtil.java @@ -2,12 +2,12 @@ package org.dbsyncer.parser.util; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.model.Field; -import org.dbsyncer.connector.model.Filter; import org.dbsyncer.parser.model.Convert; import org.dbsyncer.parser.model.FieldMapping; import org.dbsyncer.parser.model.Mapping; import org.dbsyncer.parser.model.TableGroup; +import org.dbsyncer.sdk.model.Field; +import org.dbsyncer.sdk.model.Filter; import java.util.List; import java.util.Map; @@ -15,9 +15,6 @@ import java.util.stream.Collectors; public abstract class PickerUtil { - private PickerUtil() { - } - /** * 合并参数配置、过滤条件、转换配置、插件配置、目标源字段、数据源字段 * diff --git a/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/AbstractPluginContext.java b/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/AbstractPluginContext.java index 471adacd..1c966ae7 100644 --- a/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/AbstractPluginContext.java +++ b/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/AbstractPluginContext.java @@ -1,7 +1,7 @@ package org.dbsyncer.plugin; +import org.dbsyncer.sdk.connector.ConnectorInstance; import org.dbsyncer.sdk.plugin.PluginContext; -import org.dbsyncer.sdk.spi.ConnectorMapper; import java.util.List; import java.util.Map; @@ -22,12 +22,12 @@ public abstract class AbstractPluginContext implements PluginContext { /** * 数据源连接实例 */ - protected ConnectorMapper sourceConnectorMapper; + protected ConnectorInstance sourceConnectorInstance; /** * 目标源连接实例 */ - protected ConnectorMapper targetConnectorMapper; + protected ConnectorInstance targetConnectorInstance; /** * 数据源表 @@ -54,10 +54,9 @@ public abstract class AbstractPluginContext implements PluginContext { */ protected List targetList; - public void init(ConnectorMapper sourceConnectorMapper, ConnectorMapper targetConnectorMapper, String sourceTableName, String targetTableName, String event, - List sourceList, List targetList) { - this.sourceConnectorMapper = sourceConnectorMapper; - this.targetConnectorMapper = targetConnectorMapper; + public void init(ConnectorInstance sourceConnectorInstance, ConnectorInstance targetConnectorInstance, String sourceTableName, String targetTableName, String event, List sourceList, List targetList) { + this.sourceConnectorInstance = sourceConnectorInstance; + this.targetConnectorInstance = targetConnectorInstance; this.sourceTableName = sourceTableName; this.targetTableName = targetTableName; this.event = event; @@ -76,13 +75,13 @@ public abstract class AbstractPluginContext implements PluginContext { } @Override - public ConnectorMapper getSourceConnectorMapper() { - return sourceConnectorMapper; + public ConnectorInstance getSourceConnectorInstance() { + return sourceConnectorInstance; } @Override - public ConnectorMapper getTargetConnectorMapper() { - return targetConnectorMapper; + public ConnectorInstance getTargetConnectorInstance() { + return targetConnectorInstance; } @Override diff --git a/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/impl/FullPluginContext.java b/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/impl/FullPluginContext.java index 54054bce..d022df0e 100644 --- a/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/impl/FullPluginContext.java +++ b/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/impl/FullPluginContext.java @@ -1,8 +1,8 @@ package org.dbsyncer.plugin.impl; +import org.dbsyncer.sdk.connector.ConnectorInstance; import org.dbsyncer.sdk.enums.ModelEnum; import org.dbsyncer.plugin.AbstractPluginContext; -import org.dbsyncer.sdk.spi.ConnectorMapper; /** * @author AE86 @@ -11,8 +11,8 @@ import org.dbsyncer.sdk.spi.ConnectorMapper; */ public final class FullPluginContext extends AbstractPluginContext { - public FullPluginContext(ConnectorMapper sourceConnectorMapper, ConnectorMapper targetConnectorMapper, String sourceTableName, String targetTableName, String event) { - super.init(sourceConnectorMapper, targetConnectorMapper, sourceTableName, targetTableName, event, null, null); + public FullPluginContext(ConnectorInstance sourceConnectorInstance, ConnectorInstance targetConnectorInstance, String sourceTableName, String targetTableName, String event) { + super.init(sourceConnectorInstance, targetConnectorInstance, sourceTableName, targetTableName, event, null, null); } @Override diff --git a/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/impl/IncrementPluginContext.java b/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/impl/IncrementPluginContext.java index 6a0d7216..5de217dd 100644 --- a/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/impl/IncrementPluginContext.java +++ b/dbsyncer-plugin/src/main/java/org/dbsyncer/plugin/impl/IncrementPluginContext.java @@ -1,8 +1,8 @@ package org.dbsyncer.plugin.impl; +import org.dbsyncer.sdk.connector.ConnectorInstance; import org.dbsyncer.sdk.enums.ModelEnum; import org.dbsyncer.plugin.AbstractPluginContext; -import org.dbsyncer.sdk.spi.ConnectorMapper; import java.util.List; import java.util.Map; @@ -14,8 +14,8 @@ import java.util.Map; */ public final class IncrementPluginContext extends AbstractPluginContext { - public IncrementPluginContext(ConnectorMapper sourceConnectorMapper, ConnectorMapper targetConnectorMapper, String sourceTableName, String targetTableName, String event, List sourceList, List targetList) { - super.init(sourceConnectorMapper, targetConnectorMapper, sourceTableName, targetTableName, event, sourceList, targetList); + public IncrementPluginContext(ConnectorInstance sourceConnectorInstance, ConnectorInstance targetConnectorInstance, String sourceTableName, String targetTableName, String event, List sourceList, List targetList) { + super.init(sourceConnectorInstance, targetConnectorInstance, sourceTableName, targetTableName, event, sourceList, targetList); } @Override diff --git a/dbsyncer-sdk/pom.xml b/dbsyncer-sdk/pom.xml index 6b6a34b7..8e8e3711 100644 --- a/dbsyncer-sdk/pom.xml +++ b/dbsyncer-sdk/pom.xml @@ -17,5 +17,24 @@ dbsyncer-common ${project.parent.version} + + + + org.springframework + spring-jdbc + + + + + com.oracle + ojdbc6 + + + + + org.postgresql + postgresql + + \ No newline at end of file diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/config/CommandConfig.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/config/CommandConfig.java index 2eecead1..9eb16c7e 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/config/CommandConfig.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/config/CommandConfig.java @@ -15,7 +15,7 @@ import java.util.List; */ public class CommandConfig { - private String type; + private String connectorType; private Table table; @@ -23,15 +23,15 @@ public class CommandConfig { private ConnectorConfig connectorConfig; - public CommandConfig(String type, Table table, ConnectorConfig connectorConfig, List filter) { - this.type = type; + public CommandConfig(String connectorType, Table table, ConnectorConfig connectorConfig, List filter) { + this.connectorType = connectorType; this.table = table; this.filter = filter; this.connectorConfig = connectorConfig; } - public String getType() { - return type; + public String getConnectorType() { + return connectorType; } public Table getTable() { diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/DatabaseConfig.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/config/DatabaseConfig.java similarity index 96% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/DatabaseConfig.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/config/DatabaseConfig.java index 18e7e9f4..34ebb1d7 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/DatabaseConfig.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/config/DatabaseConfig.java @@ -1,7 +1,7 @@ -package org.dbsyncer.connector.config; +package org.dbsyncer.sdk.config; -import org.dbsyncer.connector.model.SqlTable; import org.dbsyncer.sdk.model.ConnectorConfig; +import org.dbsyncer.sdk.model.SqlTable; import java.util.LinkedHashMap; import java.util.List; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/SqlBuilderConfig.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/config/SqlBuilderConfig.java similarity index 85% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/SqlBuilderConfig.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/config/SqlBuilderConfig.java index 641aeb30..a0bc9f22 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/SqlBuilderConfig.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/config/SqlBuilderConfig.java @@ -1,60 +1,60 @@ -package org.dbsyncer.connector.config; - -import org.dbsyncer.connector.database.Database; -import org.dbsyncer.connector.model.Field; - -import java.util.List; - -public class SqlBuilderConfig { - - private Database database; - - // 架构名 - private String schema; - - // 表名 - private String tableName; - - // 主键列表 - private List primaryKeys; - - // 字段 - private List fields; - - // 过滤条件 - private String queryFilter; - - public SqlBuilderConfig(Database database, String schema, String tableName, List primaryKeys, - List fields, String queryFilter) { - this.database = database; - this.schema = schema; - this.tableName = tableName; - this.primaryKeys = primaryKeys; - this.fields = fields; - this.queryFilter = queryFilter; - } - - public Database getDatabase() { - return database; - } - - public String getSchema() { - return schema; - } - - public String getTableName() { - return tableName; - } - - public List getPrimaryKeys() { - return primaryKeys; - } - - public List getFields() { - return fields; - } - - public String getQueryFilter() { - return queryFilter; - } +package org.dbsyncer.sdk.config; + +import org.dbsyncer.sdk.connector.database.Database; +import org.dbsyncer.sdk.model.Field; + +import java.util.List; + +public class SqlBuilderConfig { + + private Database database; + + // 架构名 + private String schema; + + // 表名 + private String tableName; + + // 主键列表 + private List primaryKeys; + + // 字段 + private List fields; + + // 过滤条件 + private String queryFilter; + + public SqlBuilderConfig(Database database, String schema, String tableName, List primaryKeys, + List fields, String queryFilter) { + this.database = database; + this.schema = schema; + this.tableName = tableName; + this.primaryKeys = primaryKeys; + this.fields = fields; + this.queryFilter = queryFilter; + } + + public Database getDatabase() { + return database; + } + + public String getSchema() { + return schema; + } + + public String getTableName() { + return tableName; + } + + public List getPrimaryKeys() { + return primaryKeys; + } + + public List getFields() { + return fields; + } + + public String getQueryFilter() { + return queryFilter; + } } \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/AbstractConnector.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/AbstractConnector.java similarity index 65% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/AbstractConnector.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/AbstractConnector.java index 7780da38..b330cd08 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/AbstractConnector.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/AbstractConnector.java @@ -1,37 +1,37 @@ -package org.dbsyncer.connector; +package org.dbsyncer.sdk.connector; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.config.WriterBatchConfig; -import org.dbsyncer.connector.constant.ConnectorConstant; -import org.dbsyncer.connector.model.Field; -import org.dbsyncer.connector.schema.BigintValueMapper; -import org.dbsyncer.connector.schema.BinaryValueMapper; -import org.dbsyncer.connector.schema.BitValueMapper; -import org.dbsyncer.connector.schema.BlobValueMapper; -import org.dbsyncer.connector.schema.CharValueMapper; -import org.dbsyncer.connector.schema.ClobValueMapper; -import org.dbsyncer.connector.schema.DateValueMapper; -import org.dbsyncer.connector.schema.DecimalValueMapper; -import org.dbsyncer.connector.schema.DoubleValueMapper; -import org.dbsyncer.connector.schema.FloatValueMapper; -import org.dbsyncer.connector.schema.IntegerValueMapper; -import org.dbsyncer.connector.schema.LongVarBinaryValueMapper; -import org.dbsyncer.connector.schema.LongVarcharValueMapper; -import org.dbsyncer.connector.schema.NCharValueMapper; -import org.dbsyncer.connector.schema.NClobValueMapper; -import org.dbsyncer.connector.schema.NVarcharValueMapper; -import org.dbsyncer.connector.schema.NumberValueMapper; -import org.dbsyncer.connector.schema.OtherValueMapper; -import org.dbsyncer.connector.schema.RealValueMapper; -import org.dbsyncer.connector.schema.RowIdValueMapper; -import org.dbsyncer.connector.schema.SmallintValueMapper; -import org.dbsyncer.connector.schema.TimeValueMapper; -import org.dbsyncer.connector.schema.TimestampValueMapper; -import org.dbsyncer.connector.schema.TinyintValueMapper; -import org.dbsyncer.connector.schema.VarBinaryValueMapper; -import org.dbsyncer.connector.schema.VarcharValueMapper; -import org.dbsyncer.sdk.spi.ConnectorMapper; +import org.dbsyncer.sdk.SdkException; +import org.dbsyncer.sdk.config.WriterBatchConfig; +import org.dbsyncer.sdk.connector.schema.BigintValueMapper; +import org.dbsyncer.sdk.connector.schema.BinaryValueMapper; +import org.dbsyncer.sdk.connector.schema.BitValueMapper; +import org.dbsyncer.sdk.connector.schema.BlobValueMapper; +import org.dbsyncer.sdk.connector.schema.CharValueMapper; +import org.dbsyncer.sdk.connector.schema.ClobValueMapper; +import org.dbsyncer.sdk.connector.schema.DateValueMapper; +import org.dbsyncer.sdk.connector.schema.DecimalValueMapper; +import org.dbsyncer.sdk.connector.schema.DoubleValueMapper; +import org.dbsyncer.sdk.connector.schema.FloatValueMapper; +import org.dbsyncer.sdk.connector.schema.IntegerValueMapper; +import org.dbsyncer.sdk.connector.schema.LongVarBinaryValueMapper; +import org.dbsyncer.sdk.connector.schema.LongVarcharValueMapper; +import org.dbsyncer.sdk.connector.schema.NCharValueMapper; +import org.dbsyncer.sdk.connector.schema.NClobValueMapper; +import org.dbsyncer.sdk.connector.schema.NVarcharValueMapper; +import org.dbsyncer.sdk.connector.schema.NumberValueMapper; +import org.dbsyncer.sdk.connector.schema.OtherValueMapper; +import org.dbsyncer.sdk.connector.schema.RealValueMapper; +import org.dbsyncer.sdk.connector.schema.RowIdValueMapper; +import org.dbsyncer.sdk.connector.schema.SmallintValueMapper; +import org.dbsyncer.sdk.connector.schema.TimeValueMapper; +import org.dbsyncer.sdk.connector.schema.TimestampValueMapper; +import org.dbsyncer.sdk.connector.schema.TinyintValueMapper; +import org.dbsyncer.sdk.connector.schema.VarBinaryValueMapper; +import org.dbsyncer.sdk.connector.schema.VarcharValueMapper; +import org.dbsyncer.sdk.constant.ConnectorConstant; +import org.dbsyncer.sdk.model.Field; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -82,10 +82,10 @@ public abstract class AbstractConnector { /** * 转换字段值 * - * @param connectorMapper + * @param connectorInstance * @param config */ - protected void convertProcessBeforeWriter(ConnectorMapper connectorMapper, WriterBatchConfig config) { + public void convertProcessBeforeWriter(ConnectorInstance connectorInstance, WriterBatchConfig config) { if (CollectionUtils.isEmpty(config.getFields()) || CollectionUtils.isEmpty(config.getData())) { return; } @@ -102,10 +102,10 @@ public abstract class AbstractConnector { if (null != valueMapper) { // 当数据类型不同时，转换值类型 try { - row.put(f.getName(), valueMapper.convertValue(connectorMapper, row.get(f.getName()))); + row.put(f.getName(), valueMapper.convertValue(connectorInstance, row.get(f.getName()))); } catch (Exception e) { logger.error("convert value error: ({}, {})", f.getName(), row.get(f.getName())); - throw new ConnectorException(e); + throw new SdkException(e); } } } diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/AbstractValueMapper.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/AbstractValueMapper.java similarity index 77% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/AbstractValueMapper.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/AbstractValueMapper.java index 5789fc0d..f7de26ea 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/AbstractValueMapper.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/AbstractValueMapper.java @@ -1,6 +1,4 @@ -package org.dbsyncer.connector; - -import org.dbsyncer.sdk.spi.ConnectorMapper; +package org.dbsyncer.sdk.connector; import java.lang.reflect.ParameterizedType; @@ -22,7 +20,7 @@ public abstract class AbstractValueMapper implements ValueMapper { * * @param val */ - protected abstract T convert(ConnectorMapper connectorMapper, Object val) throws Exception; + protected abstract T convert(ConnectorInstance connectorInstance, Object val) throws Exception; /** * 是否跳过类型转换 @@ -45,7 +43,7 @@ public abstract class AbstractValueMapper implements ValueMapper { } @Override - public Object convertValue(ConnectorMapper connectorMapper, Object val) throws Exception { + public Object convertValue(ConnectorInstance connectorInstance, Object val) throws Exception { if (null != val) { // 是否需要跳过转换 if (skipConvert(val)) { @@ -54,7 +52,7 @@ public abstract class AbstractValueMapper implements ValueMapper { // 当数据类型不同时，返回转换值 if (!val.getClass().equals(parameterClazz)) { - return convert(connectorMapper, val); + return convert(connectorInstance, val); } } return getDefaultVal(val); diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/ConnectorInstance.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/ConnectorInstance.java similarity index 81% rename from dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/ConnectorInstance.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/ConnectorInstance.java index 10f5287a..ae48f12a 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/ConnectorInstance.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/ConnectorInstance.java @@ -1,4 +1,6 @@ -package org.dbsyncer.sdk.spi; +package org.dbsyncer.sdk.connector; + +import org.dbsyncer.sdk.model.ConnectorConfig; /** * 连接器实例，管理连接生命周期 @@ -9,7 +11,7 @@ package org.dbsyncer.sdk.spi; * @version 1.0.0 * @date 2022/3/20 23:00 */ -public interface ConnectorInstance extends Cloneable { +public interface ConnectorInstance extends Cloneable { /** * 获取服务地址 diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/ValueMapper.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/ValueMapper.java similarity index 49% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/ValueMapper.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/ValueMapper.java index adfdf0d3..176494a2 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/ValueMapper.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/ValueMapper.java @@ -1,6 +1,4 @@ -package org.dbsyncer.connector; - -import org.dbsyncer.sdk.spi.ConnectorMapper; +package org.dbsyncer.sdk.connector; /** * 值转换器 @@ -14,9 +12,9 @@ public interface ValueMapper { /** * 转换值 * - * @param connectorMapper + * @param connectorInstance * @param val * @return */ - Object convertValue(ConnectorMapper connectorMapper, Object val) throws Exception; + Object convertValue(ConnectorInstance connectorInstance, Object val) throws Exception; } \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/AbstractDQLConnector.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/AbstractDQLConnector.java similarity index 79% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/AbstractDQLConnector.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/AbstractDQLConnector.java index e339a24e..08e9e168 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/AbstractDQLConnector.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/AbstractDQLConnector.java @@ -1,19 +1,16 @@ -package org.dbsyncer.connector.sql; +package org.dbsyncer.sdk.connector.database; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.config.CommandConfig; -import org.dbsyncer.connector.config.DatabaseConfig; -import org.dbsyncer.connector.constant.ConnectorConstant; -import org.dbsyncer.connector.database.AbstractDatabaseConnector; -import org.dbsyncer.connector.database.DatabaseConnectorMapper; -import org.dbsyncer.connector.enums.SqlBuilderEnum; -import org.dbsyncer.connector.enums.TableTypeEnum; -import org.dbsyncer.connector.model.MetaInfo; -import org.dbsyncer.connector.model.PageSql; -import org.dbsyncer.connector.model.SqlTable; -import org.dbsyncer.connector.model.Table; -import org.dbsyncer.connector.util.PrimaryKeyUtil; +import org.dbsyncer.sdk.config.CommandConfig; +import org.dbsyncer.sdk.config.DatabaseConfig; +import org.dbsyncer.sdk.enums.SqlBuilderEnum; +import org.dbsyncer.sdk.enums.TableTypeEnum; +import org.dbsyncer.sdk.model.MetaInfo; +import org.dbsyncer.sdk.model.PageSql; +import org.dbsyncer.sdk.model.SqlTable; +import org.dbsyncer.sdk.model.Table; +import org.dbsyncer.sdk.util.PrimaryKeyUtil; import java.util.ArrayList; import java.util.Collections; @@ -29,7 +26,7 @@ import java.util.Map; public abstract class AbstractDQLConnector extends AbstractDatabaseConnector { @Override - public List
    getTable(DatabaseConnectorMapper connectorMapper) { + public List
    getTable(DatabaseConnectorInstance connectorMapper) { DatabaseConfig cfg = connectorMapper.getConfig(); List sqlTables = cfg.getSqlTables(); List
    tables = new ArrayList<>(); @@ -47,7 +44,7 @@ public abstract class AbstractDQLConnector extends AbstractDatabaseConnector { } @Override - public MetaInfo getMetaInfo(DatabaseConnectorMapper connectorMapper, String sqlName) { + public MetaInfo getMetaInfo(DatabaseConnectorInstance connectorMapper, String sqlName) { DatabaseConfig cfg = connectorMapper.getConfig(); List sqlTables = cfg.getSqlTables(); for (SqlTable s : sqlTables) { diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/AbstractDatabaseConnector.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/AbstractDatabaseConnector.java similarity index 87% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/AbstractDatabaseConnector.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/AbstractDatabaseConnector.java index 957960a1..464b36f1 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/AbstractDatabaseConnector.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/AbstractDatabaseConnector.java @@ -1,30 +1,30 @@ -package org.dbsyncer.connector.database; +package org.dbsyncer.sdk.connector.database; import org.dbsyncer.common.model.Result; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.AbstractConnector; -import org.dbsyncer.connector.Connector; -import org.dbsyncer.connector.ConnectorException; -import org.dbsyncer.connector.config.CommandConfig; -import org.dbsyncer.connector.config.DDLConfig; -import org.dbsyncer.connector.config.DatabaseConfig; -import org.dbsyncer.connector.config.ReaderConfig; -import org.dbsyncer.connector.config.SqlBuilderConfig; -import org.dbsyncer.connector.config.WriterBatchConfig; -import org.dbsyncer.connector.constant.ConnectorConstant; -import org.dbsyncer.connector.database.ds.SimpleConnection; -import org.dbsyncer.connector.enums.OperationEnum; -import org.dbsyncer.connector.enums.SqlBuilderEnum; -import org.dbsyncer.connector.enums.TableTypeEnum; -import org.dbsyncer.connector.model.Field; -import org.dbsyncer.connector.model.Filter; -import org.dbsyncer.connector.model.MetaInfo; -import org.dbsyncer.connector.model.PageSql; -import org.dbsyncer.connector.model.Table; -import org.dbsyncer.connector.util.DatabaseUtil; -import org.dbsyncer.connector.util.PrimaryKeyUtil; -import org.dbsyncer.sdk.spi.ConnectorMapper; +import org.dbsyncer.sdk.SdkException; +import org.dbsyncer.sdk.config.CommandConfig; +import org.dbsyncer.sdk.config.DDLConfig; +import org.dbsyncer.sdk.config.DatabaseConfig; +import org.dbsyncer.sdk.config.ReaderConfig; +import org.dbsyncer.sdk.config.SqlBuilderConfig; +import org.dbsyncer.sdk.config.WriterBatchConfig; +import org.dbsyncer.sdk.connector.AbstractConnector; +import org.dbsyncer.sdk.connector.ConnectorInstance; +import org.dbsyncer.sdk.connector.database.ds.SimpleConnection; +import org.dbsyncer.sdk.constant.ConnectorConstant; +import org.dbsyncer.sdk.enums.OperationEnum; +import org.dbsyncer.sdk.enums.SqlBuilderEnum; +import org.dbsyncer.sdk.enums.TableTypeEnum; +import org.dbsyncer.sdk.model.Field; +import org.dbsyncer.sdk.model.Filter; +import org.dbsyncer.sdk.model.MetaInfo; +import org.dbsyncer.sdk.model.PageSql; +import org.dbsyncer.sdk.model.Table; +import org.dbsyncer.sdk.spi.ConnectorService; +import org.dbsyncer.sdk.util.DatabaseUtil; +import org.dbsyncer.sdk.util.PrimaryKeyUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.jdbc.support.rowset.ResultSetWrappingSqlRowSet; @@ -48,7 +48,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; -public abstract class AbstractDatabaseConnector extends AbstractConnector implements Connector, Database { +public abstract class AbstractDatabaseConnector extends AbstractConnector implements ConnectorService, Database { private final Logger logger = LoggerFactory.getLogger(getClass()); @@ -58,33 +58,48 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector implem private final String SYS_EXPRESSION = "^[$].*[$]$"; @Override - public ConnectorMapper connect(DatabaseConfig config) { - return new DatabaseConnectorMapper(config); + public boolean isSupportedTiming() { + return true; } @Override - public void disconnect(DatabaseConnectorMapper connectorMapper) { + public boolean isSupportedLog() { + return true; + } + + @Override + public Class getConfigClass() { + return DatabaseConfig.class; + } + + @Override + public ConnectorInstance connect(DatabaseConfig config) { + return new DatabaseConnectorInstance(config); + } + + @Override + public void disconnect(DatabaseConnectorInstance connectorMapper) { connectorMapper.close(); } @Override - public boolean isAlive(DatabaseConnectorMapper connectorMapper) { + public boolean isAlive(DatabaseConnectorInstance connectorMapper) { Integer count = connectorMapper.execute(databaseTemplate -> databaseTemplate.queryForObject(getValidationQuery(), Integer.class)); return null != count && count > 0; } @Override - public String getConnectorMapperCacheKey(DatabaseConfig config) { + public String getConnectorInstanceCacheKey(DatabaseConfig config) { return String.format("%s-%s-%s", config.getConnectorType(), config.getUrl(), config.getUsername()); } @Override - public List
    getTable(DatabaseConnectorMapper connectorMapper) { + public List
    getTable(DatabaseConnectorInstance connectorMapper) { return getTable(connectorMapper, null, getSchema(connectorMapper.getConfig()), null); } @Override - public MetaInfo getMetaInfo(DatabaseConnectorMapper connectorMapper, String tableNamePattern) { + public MetaInfo getMetaInfo(DatabaseConnectorInstance connectorMapper, String tableNamePattern) { List fields = new ArrayList<>(); final String schema = getSchema(connectorMapper.getConfig()); connectorMapper.execute(databaseTemplate -> { @@ -107,7 +122,7 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector implem } @Override - public long getCount(DatabaseConnectorMapper connectorMapper, Map command) { + public long getCount(DatabaseConnectorInstance connectorMapper, Map command) { if (CollectionUtils.isEmpty(command)) { return 0L; } @@ -126,7 +141,7 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector implem } @Override - public Result reader(DatabaseConnectorMapper connectorMapper, ReaderConfig config) { + public Result reader(DatabaseConnectorInstance connectorMapper, ReaderConfig config) { // 1、获取select SQL boolean supportedCursor = enableCursor() && config.isSupportedCursor() && null != config.getCursors(); String queryKey = supportedCursor ? ConnectorConstant.OPERTION_QUERY_CURSOR : ConnectorConstant.OPERTION_QUERY; @@ -144,7 +159,7 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector implem } @Override - public Result writer(DatabaseConnectorMapper connectorMapper, WriterBatchConfig config) { + public Result writer(DatabaseConnectorInstance connectorMapper, WriterBatchConfig config) { String event = config.getEvent(); List data = config.getData(); @@ -153,11 +168,11 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector implem Assert.hasText(executeSql, "执行SQL语句不能为空."); if (CollectionUtils.isEmpty(config.getFields())) { logger.error("writer fields can not be empty."); - throw new ConnectorException("writer fields can not be empty."); + throw new SdkException("writer fields can not be empty."); } if (CollectionUtils.isEmpty(data)) { logger.error("writer data can not be empty."); - throw new ConnectorException("writer data can not be empty."); + throw new SdkException("writer data can not be empty."); } List fields = new ArrayList<>(config.getFields()); List pkFields = PrimaryKeyUtil.findConfigPrimaryKeyFields(config); @@ -204,7 +219,7 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector implem String tableName = table.getName(); if (StringUtil.isBlank(tableName)) { logger.error("数据源表不能为空."); - throw new ConnectorException("数据源表不能为空."); + throw new SdkException("数据源表不能为空."); } // 架构名 @@ -238,7 +253,7 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector implem String tableName = table.getName(); if (StringUtil.isBlank(tableName)) { logger.error("目标源表不能为空."); - throw new ConnectorException("目标源表不能为空."); + throw new SdkException("目标源表不能为空."); } // 架构名 @@ -310,7 +325,7 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector implem // 查询表字段信息 int columnCount = metaData.getColumnCount(); if (1 > columnCount) { - throw new ConnectorException("查询表字段不能为空."); + throw new SdkException("查询表字段不能为空."); } List fields = new ArrayList<>(columnCount); Map> tables = new HashMap<>(); @@ -412,7 +427,7 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector implem for (Field c : column) { String name = c.getName(); if (StringUtil.isBlank(name)) { - throw new ConnectorException("The field name can not be empty."); + throw new SdkException("The field name can not be empty."); } if (!mark.contains(name)) { fields.add(c); @@ -431,7 +446,7 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector implem * @param tableNamePattern * @return */ - private List
    getTable(DatabaseConnectorMapper connectorMapper, String catalog, String schema, String tableNamePattern) { + private List
    getTable(DatabaseConnectorInstance connectorMapper, String catalog, String schema, String tableNamePattern) { return connectorMapper.execute(databaseTemplate -> { List
    tables = new ArrayList<>(); SimpleConnection connection = databaseTemplate.getSimpleConnection(); @@ -554,7 +569,7 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector implem return args; } - private void forceUpdate(Result result, DatabaseConnectorMapper connectorMapper, WriterBatchConfig config, List pkFields, + private void forceUpdate(Result result, DatabaseConnectorInstance connectorMapper, WriterBatchConfig config, List pkFields, Map row) { if (isUpdate(config.getEvent()) || isInsert(config.getEvent())) { // 存在执行覆盖更新，否则写入 @@ -571,7 +586,7 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector implem } } - private void writer(Result result, DatabaseConnectorMapper connectorMapper, WriterBatchConfig config, List pkFields, Map row, + private void writer(Result result, DatabaseConnectorInstance connectorMapper, WriterBatchConfig config, List pkFields, Map row, String event) { // 1、获取 SQL String sql = config.getCommand().get(event); @@ -591,7 +606,7 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector implem // 2、设置参数 int execute = connectorMapper.execute(databaseTemplate -> databaseTemplate.update(sql, batchRow(fields, row))); if (execute == 0) { - throw new ConnectorException(String.format("尝试执行[%s]失败", event)); + throw new SdkException(String.format("尝试执行[%s]失败", event)); } result.getSuccessData().add(row); } catch (Exception e) { @@ -603,7 +618,7 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector implem } } - private boolean existRow(DatabaseConnectorMapper connectorMapper, String sql, Object[] args) { + private boolean existRow(DatabaseConnectorInstance connectorMapper, String sql, Object[] args) { int rowNum = 0; try { rowNum = connectorMapper.execute(databaseTemplate -> databaseTemplate.queryForObject(sql, Integer.class, args)); @@ -636,7 +651,7 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector implem } @Override - public Result writerDDL(DatabaseConnectorMapper connectorMapper, DDLConfig config) { + public Result writerDDL(DatabaseConnectorInstance connectorMapper, DDLConfig config) { Result result = new Result(); try { Assert.hasText(config.getSql(), "执行SQL语句不能为空."); diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/AbstractSqlBuilder.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/AbstractSqlBuilder.java similarity index 34% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/AbstractSqlBuilder.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/AbstractSqlBuilder.java index 5a48ac90..e15ec10e 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/AbstractSqlBuilder.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/AbstractSqlBuilder.java @@ -1,13 +1,13 @@ -package org.dbsyncer.connector.database; - -import org.dbsyncer.connector.ConnectorException; -import org.dbsyncer.connector.config.SqlBuilderConfig; -import org.dbsyncer.connector.database.sqlbuilder.SqlBuilder; - -public abstract class AbstractSqlBuilder implements SqlBuilder { - - @Override - public String buildQuerySql(SqlBuilderConfig config) { - throw new ConnectorException("Not implemented"); - } +package org.dbsyncer.sdk.connector.database; + +import org.dbsyncer.sdk.SdkException; +import org.dbsyncer.sdk.config.SqlBuilderConfig; +import org.dbsyncer.sdk.connector.database.sqlbuilder.SqlBuilder; + +public abstract class AbstractSqlBuilder implements SqlBuilder { + + @Override + public String buildQuerySql(SqlBuilderConfig config) { + throw new SdkException("Not implemented"); + } } \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/Database.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/Database.java similarity index 84% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/Database.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/Database.java index 5814b71d..ccd89e18 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/Database.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/Database.java @@ -1,10 +1,10 @@ -package org.dbsyncer.connector.database; +package org.dbsyncer.sdk.connector.database; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.ConnectorException; -import org.dbsyncer.connector.config.ReaderConfig; -import org.dbsyncer.connector.model.Field; -import org.dbsyncer.connector.model.PageSql; +import org.dbsyncer.sdk.SdkException; +import org.dbsyncer.sdk.config.ReaderConfig; +import org.dbsyncer.sdk.model.Field; +import org.dbsyncer.sdk.model.PageSql; import java.util.List; @@ -82,7 +82,7 @@ public interface Database { * @return */ default Object[] getPageCursorArgs(ReaderConfig config) { - throw new ConnectorException("Unsupported override method getPageCursorArgs:" + getClass().getName()); + throw new SdkException("Unsupported override method getPageCursorArgs:" + getClass().getName()); } /** diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/DatabaseConnectorMapper.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/DatabaseConnectorInstance.java similarity index 71% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/DatabaseConnectorMapper.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/DatabaseConnectorInstance.java index 7e03bebf..f161cfd7 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/DatabaseConnectorMapper.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/DatabaseConnectorInstance.java @@ -1,69 +1,69 @@ -package org.dbsyncer.connector.database; - -import org.dbsyncer.connector.ConnectorException; -import org.dbsyncer.connector.config.DatabaseConfig; -import org.dbsyncer.connector.database.ds.SimpleConnection; -import org.dbsyncer.connector.database.ds.SimpleDataSource; -import org.dbsyncer.sdk.spi.ConnectorMapper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.dao.EmptyResultDataAccessException; - -import java.sql.Connection; - -public class DatabaseConnectorMapper implements ConnectorMapper { - private final Logger logger = LoggerFactory.getLogger(getClass()); - private DatabaseConfig config; - private SimpleDataSource dataSource; - - public DatabaseConnectorMapper(DatabaseConfig config) { - this.config = config; - this.dataSource = new SimpleDataSource(config.getDriverClassName(), config.getUrl(), config.getUsername(), config.getPassword()); - } - - public T execute(HandleCallback callback) { - Connection connection = null; - try { - connection = getConnection(); - return (T) callback.apply(new DatabaseTemplate((SimpleConnection) connection)); - } catch (EmptyResultDataAccessException e) { - throw e; - } catch (Exception e) { - logger.error(e.getMessage()); - throw new ConnectorException(e.getMessage(), e.getCause()); - } finally { - dataSource.close(connection); - } - } - - @Override - public String getServiceUrl() { - return config.getUrl(); - } - - @Override - public DatabaseConfig getConfig() { - return config; - } - - @Override - public void setConfig(DatabaseConfig config) { - this.config = config; - } - - @Override - public Connection getConnection() throws Exception { - return dataSource.getConnection(); - } - - @Override - public void close() { - dataSource.close(); - } - - @Override - public Object clone() throws CloneNotSupportedException { - return super.clone(); - } - +package org.dbsyncer.sdk.connector.database; + +import org.dbsyncer.sdk.SdkException; +import org.dbsyncer.sdk.config.DatabaseConfig; +import org.dbsyncer.sdk.connector.ConnectorInstance; +import org.dbsyncer.sdk.connector.database.ds.SimpleConnection; +import org.dbsyncer.sdk.connector.database.ds.SimpleDataSource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.dao.EmptyResultDataAccessException; + +import java.sql.Connection; + +public class DatabaseConnectorInstance implements ConnectorInstance { + private final Logger logger = LoggerFactory.getLogger(getClass()); + private DatabaseConfig config; + private SimpleDataSource dataSource; + + public DatabaseConnectorInstance(DatabaseConfig config) { + this.config = config; + this.dataSource = new SimpleDataSource(config.getDriverClassName(), config.getUrl(), config.getUsername(), config.getPassword()); + } + + public T execute(HandleCallback callback) { + Connection connection = null; + try { + connection = getConnection(); + return (T) callback.apply(new DatabaseTemplate((SimpleConnection) connection)); + } catch (EmptyResultDataAccessException e) { + throw e; + } catch (Exception e) { + logger.error(e.getMessage()); + throw new SdkException(e.getMessage(), e.getCause()); + } finally { + dataSource.close(connection); + } + } + + @Override + public String getServiceUrl() { + return config.getUrl(); + } + + @Override + public DatabaseConfig getConfig() { + return config; + } + + @Override + public void setConfig(DatabaseConfig config) { + this.config = config; + } + + @Override + public Connection getConnection() throws Exception { + return dataSource.getConnection(); + } + + @Override + public void close() { + dataSource.close(); + } + + @Override + public Object clone() throws CloneNotSupportedException { + return super.clone(); + } + } \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/DatabaseTemplate.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/DatabaseTemplate.java similarity index 99% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/DatabaseTemplate.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/DatabaseTemplate.java index 9bc5d344..f169ff78 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/DatabaseTemplate.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/DatabaseTemplate.java @@ -1,7 +1,7 @@ -package org.dbsyncer.connector.database; +package org.dbsyncer.sdk.connector.database; -import org.dbsyncer.connector.ConnectorException; -import org.dbsyncer.connector.database.ds.SimpleConnection; +import org.dbsyncer.sdk.SdkException; +import org.dbsyncer.sdk.connector.database.ds.SimpleConnection; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.dao.DataAccessException; @@ -1515,7 +1515,7 @@ public class DatabaseTemplate implements JdbcOperations { } catch (IllegalAccessException e) { logger.error(e.getMessage()); } - throw new ConnectorException(String.format("Can't invoke '%s'.", fieldName)); + throw new SdkException(String.format("Can't invoke '%s'.", fieldName)); } /** diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/HandleCallback.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/HandleCallback.java similarity index 69% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/HandleCallback.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/HandleCallback.java index 30fc64bb..970c879e 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/HandleCallback.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/HandleCallback.java @@ -1,4 +1,4 @@ -package org.dbsyncer.connector.database; +package org.dbsyncer.sdk.connector.database; public interface HandleCallback { diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/ds/SimpleConnection.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/ds/SimpleConnection.java similarity index 95% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/ds/SimpleConnection.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/ds/SimpleConnection.java index bc0a56bc..fa8326bb 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/ds/SimpleConnection.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/ds/SimpleConnection.java @@ -1,321 +1,321 @@ -package org.dbsyncer.connector.database.ds; - -import org.dbsyncer.connector.util.DatabaseUtil; - -import java.sql.Array; -import java.sql.Blob; -import java.sql.CallableStatement; -import java.sql.Clob; -import java.sql.Connection; -import java.sql.DatabaseMetaData; -import java.sql.NClob; -import java.sql.PreparedStatement; -import java.sql.SQLClientInfoException; -import java.sql.SQLException; -import java.sql.SQLWarning; -import java.sql.SQLXML; -import java.sql.Savepoint; -import java.sql.Statement; -import java.sql.Struct; -import java.time.Instant; -import java.util.Map; -import java.util.Properties; -import java.util.concurrent.Executor; - -public class SimpleConnection implements Connection { - private final Connection connection; - private final long ACTIVE_TIME = Instant.now().toEpochMilli(); - private boolean oracleDriver; - - public SimpleConnection(Connection connection, boolean oracleDriver) { - this.connection = connection; - this.oracleDriver = oracleDriver; - } - - @Override - public Statement createStatement() throws SQLException { - return connection.createStatement(); - } - - @Override - public PreparedStatement prepareStatement(String sql) throws SQLException { - return connection.prepareStatement(sql); - } - - @Override - public CallableStatement prepareCall(String sql) throws SQLException { - return connection.prepareCall(sql); - } - - @Override - public String nativeSQL(String sql) throws SQLException { - return connection.nativeSQL(sql); - } - - @Override - public void setAutoCommit(boolean autoCommit) throws SQLException { - connection.setAutoCommit(autoCommit); - } - - @Override - public boolean getAutoCommit() throws SQLException { - return connection.getAutoCommit(); - } - - @Override - public void commit() throws SQLException { - connection.commit(); - } - - @Override - public void rollback() throws SQLException { - connection.rollback(); - } - - @Override - public void close() { - DatabaseUtil.close(connection); - } - - @Override - public boolean isClosed() throws SQLException { - return connection.isClosed(); - } - - @Override - public DatabaseMetaData getMetaData() throws SQLException { - return connection.getMetaData(); - } - - @Override - public void setReadOnly(boolean readOnly) throws SQLException { - connection.setReadOnly(readOnly); - } - - @Override - public boolean isReadOnly() throws SQLException { - return connection.isReadOnly(); - } - - @Override - public void setCatalog(String catalog) throws SQLException { - connection.setCatalog(catalog); - } - - @Override - public String getCatalog() throws SQLException { - return connection.getCatalog(); - } - - @Override - public void setTransactionIsolation(int level) throws SQLException { - connection.setTransactionIsolation(level); - } - - @Override - public int getTransactionIsolation() throws SQLException { - return connection.getTransactionIsolation(); - } - - @Override - public SQLWarning getWarnings() throws SQLException { - return connection.getWarnings(); - } - - @Override - public void clearWarnings() throws SQLException { - connection.clearWarnings(); - } - - @Override - public Statement createStatement(int resultSetType, int resultSetConcurrency) throws SQLException { - return connection.createStatement(resultSetType, resultSetConcurrency); - } - - @Override - public PreparedStatement prepareStatement(String sql, int resultSetType, int resultSetConcurrency) throws SQLException { - return connection.prepareStatement(sql, resultSetType, resultSetConcurrency); - } - - @Override - public CallableStatement prepareCall(String sql, int resultSetType, int resultSetConcurrency) throws SQLException { - return connection.prepareCall(sql, resultSetType, resultSetConcurrency); - } - - @Override - public Map> getTypeMap() throws SQLException { - return connection.getTypeMap(); - } - - @Override - public void setTypeMap(Map> map) throws SQLException { - connection.setTypeMap(map); - } - - @Override - public void setHoldability(int holdability) throws SQLException { - connection.setHoldability(holdability); - } - - @Override - public int getHoldability() throws SQLException { - return connection.getHoldability(); - } - - @Override - public Savepoint setSavepoint() throws SQLException { - return connection.setSavepoint(); - } - - @Override - public Savepoint setSavepoint(String name) throws SQLException { - return connection.setSavepoint(name); - } - - @Override - public void rollback(Savepoint savepoint) throws SQLException { - connection.rollback(savepoint); - } - - @Override - public void releaseSavepoint(Savepoint savepoint) throws SQLException { - connection.releaseSavepoint(savepoint); - } - - @Override - public Statement createStatement(int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException { - return connection.createStatement(resultSetType, resultSetConcurrency, resultSetHoldability); - } - - @Override - public PreparedStatement prepareStatement(String sql, int resultSetType, int resultSetConcurrency, int resultSetHoldability) - throws SQLException { - return connection.prepareStatement(sql, resultSetType, resultSetConcurrency, resultSetHoldability); - } - - @Override - public CallableStatement prepareCall(String sql, int resultSetType, int resultSetConcurrency, int resultSetHoldability) - throws SQLException { - return connection.prepareCall(sql, resultSetType, resultSetConcurrency, resultSetHoldability); - } - - @Override - public PreparedStatement prepareStatement(String sql, int autoGeneratedKeys) throws SQLException { - return connection.prepareStatement(sql, autoGeneratedKeys); - } - - @Override - public PreparedStatement prepareStatement(String sql, int[] columnIndexes) throws SQLException { - return connection.prepareStatement(sql, columnIndexes); - } - - @Override - public PreparedStatement prepareStatement(String sql, String[] columnNames) throws SQLException { - return connection.prepareStatement(sql, columnNames); - } - - @Override - public Clob createClob() throws SQLException { - return connection.createClob(); - } - - @Override - public Blob createBlob() throws SQLException { - return connection.createBlob(); - } - - @Override - public NClob createNClob() throws SQLException { - return connection.createNClob(); - } - - @Override - public SQLXML createSQLXML() throws SQLException { - return connection.createSQLXML(); - } - - @Override - public boolean isValid(int timeout) throws SQLException { - return connection.isValid(timeout); - } - - @Override - public void setClientInfo(String name, String value) throws SQLClientInfoException { - connection.setClientInfo(name, value); - } - - @Override - public void setClientInfo(Properties properties) throws SQLClientInfoException { - connection.setClientInfo(properties); - } - - @Override - public String getClientInfo(String name) throws SQLException { - return connection.getClientInfo(name); - } - - @Override - public Properties getClientInfo() throws SQLException { - return connection.getClientInfo(); - } - - @Override - public Array createArrayOf(String typeName, Object[] elements) throws SQLException { - return connection.createArrayOf(typeName, elements); - } - - @Override - public Struct createStruct(String typeName, Object[] attributes) throws SQLException { - return connection.createStruct(typeName, attributes); - } - - @Override - public void setSchema(String schema) throws SQLException { - connection.setSchema(schema); - } - - @Override - public String getSchema() throws SQLException { - return connection.getSchema(); - } - - @Override - public void abort(Executor executor) throws SQLException { - connection.abort(executor); - } - - @Override - public void setNetworkTimeout(Executor executor, int milliseconds) throws SQLException { - connection.setNetworkTimeout(executor, milliseconds); - } - - @Override - public int getNetworkTimeout() throws SQLException { - return connection.getNetworkTimeout(); - } - - @Override - public T unwrap(Class iface) throws SQLException { - if (iface.isAssignableFrom(connection.getClass())) { - return iface.cast(connection); - } - throw new SQLException("Cannot unwrap to " + iface.getName()); - } - - @Override - public boolean isWrapperFor(Class iface) throws SQLException { - return iface.isAssignableFrom(connection.getClass()); - } - - public Connection getConnection() { - return connection; - } - - public long getActiveTime() { - return ACTIVE_TIME; - } - - public boolean isOracleDriver() { - return oracleDriver; - } +package org.dbsyncer.sdk.connector.database.ds; + +import org.dbsyncer.sdk.util.DatabaseUtil; + +import java.sql.Array; +import java.sql.Blob; +import java.sql.CallableStatement; +import java.sql.Clob; +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.NClob; +import java.sql.PreparedStatement; +import java.sql.SQLClientInfoException; +import java.sql.SQLException; +import java.sql.SQLWarning; +import java.sql.SQLXML; +import java.sql.Savepoint; +import java.sql.Statement; +import java.sql.Struct; +import java.time.Instant; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.Executor; + +public class SimpleConnection implements Connection { + private final Connection connection; + private final long ACTIVE_TIME = Instant.now().toEpochMilli(); + private boolean oracleDriver; + + public SimpleConnection(Connection connection, boolean oracleDriver) { + this.connection = connection; + this.oracleDriver = oracleDriver; + } + + @Override + public Statement createStatement() throws SQLException { + return connection.createStatement(); + } + + @Override + public PreparedStatement prepareStatement(String sql) throws SQLException { + return connection.prepareStatement(sql); + } + + @Override + public CallableStatement prepareCall(String sql) throws SQLException { + return connection.prepareCall(sql); + } + + @Override + public String nativeSQL(String sql) throws SQLException { + return connection.nativeSQL(sql); + } + + @Override + public void setAutoCommit(boolean autoCommit) throws SQLException { + connection.setAutoCommit(autoCommit); + } + + @Override + public boolean getAutoCommit() throws SQLException { + return connection.getAutoCommit(); + } + + @Override + public void commit() throws SQLException { + connection.commit(); + } + + @Override + public void rollback() throws SQLException { + connection.rollback(); + } + + @Override + public void close() { + DatabaseUtil.close(connection); + } + + @Override + public boolean isClosed() throws SQLException { + return connection.isClosed(); + } + + @Override + public DatabaseMetaData getMetaData() throws SQLException { + return connection.getMetaData(); + } + + @Override + public void setReadOnly(boolean readOnly) throws SQLException { + connection.setReadOnly(readOnly); + } + + @Override + public boolean isReadOnly() throws SQLException { + return connection.isReadOnly(); + } + + @Override + public void setCatalog(String catalog) throws SQLException { + connection.setCatalog(catalog); + } + + @Override + public String getCatalog() throws SQLException { + return connection.getCatalog(); + } + + @Override + public void setTransactionIsolation(int level) throws SQLException { + connection.setTransactionIsolation(level); + } + + @Override + public int getTransactionIsolation() throws SQLException { + return connection.getTransactionIsolation(); + } + + @Override + public SQLWarning getWarnings() throws SQLException { + return connection.getWarnings(); + } + + @Override + public void clearWarnings() throws SQLException { + connection.clearWarnings(); + } + + @Override + public Statement createStatement(int resultSetType, int resultSetConcurrency) throws SQLException { + return connection.createStatement(resultSetType, resultSetConcurrency); + } + + @Override + public PreparedStatement prepareStatement(String sql, int resultSetType, int resultSetConcurrency) throws SQLException { + return connection.prepareStatement(sql, resultSetType, resultSetConcurrency); + } + + @Override + public CallableStatement prepareCall(String sql, int resultSetType, int resultSetConcurrency) throws SQLException { + return connection.prepareCall(sql, resultSetType, resultSetConcurrency); + } + + @Override + public Map> getTypeMap() throws SQLException { + return connection.getTypeMap(); + } + + @Override + public void setTypeMap(Map> map) throws SQLException { + connection.setTypeMap(map); + } + + @Override + public void setHoldability(int holdability) throws SQLException { + connection.setHoldability(holdability); + } + + @Override + public int getHoldability() throws SQLException { + return connection.getHoldability(); + } + + @Override + public Savepoint setSavepoint() throws SQLException { + return connection.setSavepoint(); + } + + @Override + public Savepoint setSavepoint(String name) throws SQLException { + return connection.setSavepoint(name); + } + + @Override + public void rollback(Savepoint savepoint) throws SQLException { + connection.rollback(savepoint); + } + + @Override + public void releaseSavepoint(Savepoint savepoint) throws SQLException { + connection.releaseSavepoint(savepoint); + } + + @Override + public Statement createStatement(int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException { + return connection.createStatement(resultSetType, resultSetConcurrency, resultSetHoldability); + } + + @Override + public PreparedStatement prepareStatement(String sql, int resultSetType, int resultSetConcurrency, int resultSetHoldability) + throws SQLException { + return connection.prepareStatement(sql, resultSetType, resultSetConcurrency, resultSetHoldability); + } + + @Override + public CallableStatement prepareCall(String sql, int resultSetType, int resultSetConcurrency, int resultSetHoldability) + throws SQLException { + return connection.prepareCall(sql, resultSetType, resultSetConcurrency, resultSetHoldability); + } + + @Override + public PreparedStatement prepareStatement(String sql, int autoGeneratedKeys) throws SQLException { + return connection.prepareStatement(sql, autoGeneratedKeys); + } + + @Override + public PreparedStatement prepareStatement(String sql, int[] columnIndexes) throws SQLException { + return connection.prepareStatement(sql, columnIndexes); + } + + @Override + public PreparedStatement prepareStatement(String sql, String[] columnNames) throws SQLException { + return connection.prepareStatement(sql, columnNames); + } + + @Override + public Clob createClob() throws SQLException { + return connection.createClob(); + } + + @Override + public Blob createBlob() throws SQLException { + return connection.createBlob(); + } + + @Override + public NClob createNClob() throws SQLException { + return connection.createNClob(); + } + + @Override + public SQLXML createSQLXML() throws SQLException { + return connection.createSQLXML(); + } + + @Override + public boolean isValid(int timeout) throws SQLException { + return connection.isValid(timeout); + } + + @Override + public void setClientInfo(String name, String value) throws SQLClientInfoException { + connection.setClientInfo(name, value); + } + + @Override + public void setClientInfo(Properties properties) throws SQLClientInfoException { + connection.setClientInfo(properties); + } + + @Override + public String getClientInfo(String name) throws SQLException { + return connection.getClientInfo(name); + } + + @Override + public Properties getClientInfo() throws SQLException { + return connection.getClientInfo(); + } + + @Override + public Array createArrayOf(String typeName, Object[] elements) throws SQLException { + return connection.createArrayOf(typeName, elements); + } + + @Override + public Struct createStruct(String typeName, Object[] attributes) throws SQLException { + return connection.createStruct(typeName, attributes); + } + + @Override + public void setSchema(String schema) throws SQLException { + connection.setSchema(schema); + } + + @Override + public String getSchema() throws SQLException { + return connection.getSchema(); + } + + @Override + public void abort(Executor executor) throws SQLException { + connection.abort(executor); + } + + @Override + public void setNetworkTimeout(Executor executor, int milliseconds) throws SQLException { + connection.setNetworkTimeout(executor, milliseconds); + } + + @Override + public int getNetworkTimeout() throws SQLException { + return connection.getNetworkTimeout(); + } + + @Override + public T unwrap(Class iface) throws SQLException { + if (iface.isAssignableFrom(connection.getClass())) { + return iface.cast(connection); + } + throw new SQLException("Cannot unwrap to " + iface.getName()); + } + + @Override + public boolean isWrapperFor(Class iface) throws SQLException { + return iface.isAssignableFrom(connection.getClass()); + } + + public Connection getConnection() { + return connection; + } + + public long getActiveTime() { + return ACTIVE_TIME; + } + + public boolean isOracleDriver() { + return oracleDriver; + } } \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/ds/SimpleDataSource.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/ds/SimpleDataSource.java similarity index 91% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/ds/SimpleDataSource.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/ds/SimpleDataSource.java index 2239ff14..b7d204d4 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/ds/SimpleDataSource.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/ds/SimpleDataSource.java @@ -1,138 +1,138 @@ -package org.dbsyncer.connector.database.ds; - -import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.ConnectorException; -import org.dbsyncer.connector.util.DatabaseUtil; - -import javax.sql.DataSource; -import java.io.PrintWriter; -import java.sql.Connection; -import java.sql.SQLException; -import java.sql.SQLFeatureNotSupportedException; -import java.time.Instant; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.logging.Logger; - -public class SimpleDataSource implements DataSource, AutoCloseable { - - private final BlockingQueue pool = new LinkedBlockingQueue<>(300); - /** - * 有效期（毫秒），默认60s - */ - private final long KEEP_ALIVE = 60000; - /** - * 有效检测时间（秒），默认10s - */ - private final int VALID_TIMEOUT_SECONDS = 10; - private String driverClassName; - private String url; - private String username; - private String password; - - public SimpleDataSource(String driverClassName, String url, String username, String password) { - this.driverClassName = driverClassName; - this.url = url; - this.username = username; - this.password = password; - } - - @Override - public Connection getConnection() throws SQLException { - SimpleConnection poll = pool.poll(); - if (null == poll) { - return createConnection(); - } - - // 连接无效 - if (!poll.isValid(VALID_TIMEOUT_SECONDS)) { - return createConnection(); - } - - // 连接过期 - if (isExpired(poll)) { - return createConnection(); - } - return poll; - } - - @Override - public Connection getConnection(String username, String password) throws SQLException { - throw new ConnectorException("Unsupported method."); - } - - @Override - public T unwrap(Class iface) throws SQLException { - return null; - } - - @Override - public boolean isWrapperFor(Class iface) throws SQLException { - return false; - } - - @Override - public PrintWriter getLogWriter() throws SQLException { - return null; - } - - @Override - public void setLogWriter(PrintWriter out) throws SQLException { - - } - - @Override - public void setLoginTimeout(int seconds) throws SQLException { - - } - - @Override - public int getLoginTimeout() throws SQLException { - return 0; - } - - @Override - public Logger getParentLogger() throws SQLFeatureNotSupportedException { - return null; - } - - @Override - public void close() { - pool.forEach(c -> c.close()); - } - - public void close(Connection connection) { - if (connection != null && connection instanceof SimpleConnection) { - SimpleConnection simpleConnection = (SimpleConnection) connection; - // 连接过期 - if (isExpired(simpleConnection)) { - simpleConnection.close(); - return; - } - - // 回收连接 - pool.offer(simpleConnection); - } - } - - /** - * 连接是否过期 - * - * @param connection - * @return - */ - private boolean isExpired(SimpleConnection connection) { - return connection.getActiveTime() + KEEP_ALIVE < Instant.now().toEpochMilli(); - } - - /** - * 创建新连接 - * - * @return - * @throws SQLException - */ - private SimpleConnection createConnection() throws SQLException { - return new SimpleConnection(DatabaseUtil.getConnection(driverClassName, url, username, password), StringUtil.equals(driverClassName, "oracle.jdbc.OracleDriver")); - } - +package org.dbsyncer.sdk.connector.database.ds; + +import org.dbsyncer.common.util.StringUtil; +import org.dbsyncer.sdk.SdkException; +import org.dbsyncer.sdk.util.DatabaseUtil; + +import javax.sql.DataSource; +import java.io.PrintWriter; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.SQLFeatureNotSupportedException; +import java.time.Instant; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.logging.Logger; + +public class SimpleDataSource implements DataSource, AutoCloseable { + + private final BlockingQueue pool = new LinkedBlockingQueue<>(300); + /** + * 有效期（毫秒），默认60s + */ + private final long KEEP_ALIVE = 60000; + /** + * 有效检测时间（秒），默认10s + */ + private final int VALID_TIMEOUT_SECONDS = 10; + private String driverClassName; + private String url; + private String username; + private String password; + + public SimpleDataSource(String driverClassName, String url, String username, String password) { + this.driverClassName = driverClassName; + this.url = url; + this.username = username; + this.password = password; + } + + @Override + public Connection getConnection() throws SQLException { + SimpleConnection poll = pool.poll(); + if (null == poll) { + return createConnection(); + } + + // 连接无效 + if (!poll.isValid(VALID_TIMEOUT_SECONDS)) { + return createConnection(); + } + + // 连接过期 + if (isExpired(poll)) { + return createConnection(); + } + return poll; + } + + @Override + public Connection getConnection(String username, String password) throws SQLException { + throw new SdkException("Unsupported method."); + } + + @Override + public T unwrap(Class iface) throws SQLException { + return null; + } + + @Override + public boolean isWrapperFor(Class iface) throws SQLException { + return false; + } + + @Override + public PrintWriter getLogWriter() throws SQLException { + return null; + } + + @Override + public void setLogWriter(PrintWriter out) throws SQLException { + + } + + @Override + public void setLoginTimeout(int seconds) throws SQLException { + + } + + @Override + public int getLoginTimeout() throws SQLException { + return 0; + } + + @Override + public Logger getParentLogger() throws SQLFeatureNotSupportedException { + return null; + } + + @Override + public void close() { + pool.forEach(c -> c.close()); + } + + public void close(Connection connection) { + if (connection != null && connection instanceof SimpleConnection) { + SimpleConnection simpleConnection = (SimpleConnection) connection; + // 连接过期 + if (isExpired(simpleConnection)) { + simpleConnection.close(); + return; + } + + // 回收连接 + pool.offer(simpleConnection); + } + } + + /** + * 连接是否过期 + * + * @param connection + * @return + */ + private boolean isExpired(SimpleConnection connection) { + return connection.getActiveTime() + KEEP_ALIVE < Instant.now().toEpochMilli(); + } + + /** + * 创建新连接 + * + * @return + * @throws SQLException + */ + private SimpleConnection createConnection() throws SQLException { + return new SimpleConnection(DatabaseUtil.getConnection(driverClassName, url, username, password), StringUtil.equals(driverClassName, "oracle.jdbc.OracleDriver")); + } + } \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/sqlbuilder/SqlBuilder.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilder.java similarity index 76% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/sqlbuilder/SqlBuilder.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilder.java index 0e7e2a6f..209405f1 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/sqlbuilder/SqlBuilder.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilder.java @@ -1,6 +1,6 @@ -package org.dbsyncer.connector.database.sqlbuilder; +package org.dbsyncer.sdk.connector.database.sqlbuilder; -import org.dbsyncer.connector.config.SqlBuilderConfig; +import org.dbsyncer.sdk.config.SqlBuilderConfig; /** * SQL生成器 diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/sqlbuilder/SqlBuilderDelete.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderDelete.java similarity index 77% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/sqlbuilder/SqlBuilderDelete.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderDelete.java index fbf9d506..60e20a5f 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/sqlbuilder/SqlBuilderDelete.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderDelete.java @@ -1,9 +1,9 @@ -package org.dbsyncer.connector.database.sqlbuilder; +package org.dbsyncer.sdk.connector.database.sqlbuilder; -import org.dbsyncer.connector.config.SqlBuilderConfig; -import org.dbsyncer.connector.database.AbstractSqlBuilder; -import org.dbsyncer.connector.database.Database; -import org.dbsyncer.connector.util.PrimaryKeyUtil; +import org.dbsyncer.sdk.config.SqlBuilderConfig; +import org.dbsyncer.sdk.connector.database.AbstractSqlBuilder; +import org.dbsyncer.sdk.connector.database.Database; +import org.dbsyncer.sdk.util.PrimaryKeyUtil; import java.util.List; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/sqlbuilder/SqlBuilderInsert.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderInsert.java similarity index 83% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/sqlbuilder/SqlBuilderInsert.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderInsert.java index 7fdb7c99..67635148 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/sqlbuilder/SqlBuilderInsert.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderInsert.java @@ -1,9 +1,9 @@ -package org.dbsyncer.connector.database.sqlbuilder; +package org.dbsyncer.sdk.connector.database.sqlbuilder; -import org.dbsyncer.connector.config.SqlBuilderConfig; -import org.dbsyncer.connector.database.AbstractSqlBuilder; -import org.dbsyncer.connector.database.Database; -import org.dbsyncer.connector.model.Field; +import org.dbsyncer.sdk.config.SqlBuilderConfig; +import org.dbsyncer.sdk.connector.database.AbstractSqlBuilder; +import org.dbsyncer.sdk.connector.database.Database; +import org.dbsyncer.sdk.model.Field; import java.util.List; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/sqlbuilder/SqlBuilderQuery.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderQuery.java similarity index 87% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/sqlbuilder/SqlBuilderQuery.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderQuery.java index 351b094c..41116bc7 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/sqlbuilder/SqlBuilderQuery.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderQuery.java @@ -1,11 +1,11 @@ -package org.dbsyncer.connector.database.sqlbuilder; +package org.dbsyncer.sdk.connector.database.sqlbuilder; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.config.SqlBuilderConfig; -import org.dbsyncer.connector.database.AbstractSqlBuilder; -import org.dbsyncer.connector.database.Database; -import org.dbsyncer.connector.model.Field; -import org.dbsyncer.connector.model.PageSql; +import org.dbsyncer.sdk.config.SqlBuilderConfig; +import org.dbsyncer.sdk.connector.database.AbstractSqlBuilder; +import org.dbsyncer.sdk.connector.database.Database; +import org.dbsyncer.sdk.model.Field; +import org.dbsyncer.sdk.model.PageSql; import java.util.List; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/sqlbuilder/SqlBuilderQueryCount.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderQueryCount.java similarity index 83% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/sqlbuilder/SqlBuilderQueryCount.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderQueryCount.java index b1c3d3b6..dc01f1d2 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/sqlbuilder/SqlBuilderQueryCount.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderQueryCount.java @@ -1,42 +1,42 @@ -package org.dbsyncer.connector.database.sqlbuilder; - -import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.config.SqlBuilderConfig; -import org.dbsyncer.connector.database.Database; -import org.dbsyncer.connector.util.PrimaryKeyUtil; - -import java.util.List; - -/** - * @author AE86 - * @version 1.0.0 - * @date 2023/8/8 0:03 - */ -public class SqlBuilderQueryCount extends SqlBuilderQuery { - - @Override - public String buildSql(SqlBuilderConfig config) { - Database database = config.getDatabase(); - String quotation = database.buildSqlWithQuotation(); - String tableName = config.getTableName(); - List primaryKeys = database.buildPrimaryKeys(config.getPrimaryKeys()); - String schema = config.getSchema(); - String queryFilter = config.getQueryFilter(); - - StringBuilder sql = new StringBuilder(); - sql.append("SELECT COUNT(1) FROM (SELECT 1 AS ").append(quotation).append("_ROW").append(quotation).append(" FROM "); - sql.append(schema); - sql.append(quotation); - sql.append(database.buildTableName(tableName)); - sql.append(quotation); - if (StringUtil.isNotBlank(queryFilter)) { - sql.append(queryFilter); - } - sql.append(" GROUP BY "); - // id,uid - PrimaryKeyUtil.buildSql(sql, primaryKeys, quotation, ",", "", true); - sql.append(") DBSYNCER_T"); - return sql.toString(); - } - +package org.dbsyncer.sdk.connector.database.sqlbuilder; + +import org.dbsyncer.common.util.StringUtil; +import org.dbsyncer.sdk.config.SqlBuilderConfig; +import org.dbsyncer.sdk.connector.database.Database; +import org.dbsyncer.sdk.util.PrimaryKeyUtil; + +import java.util.List; + +/** + * @author AE86 + * @version 1.0.0 + * @date 2023/8/8 0:03 + */ +public class SqlBuilderQueryCount extends SqlBuilderQuery { + + @Override + public String buildSql(SqlBuilderConfig config) { + Database database = config.getDatabase(); + String quotation = database.buildSqlWithQuotation(); + String tableName = config.getTableName(); + List primaryKeys = database.buildPrimaryKeys(config.getPrimaryKeys()); + String schema = config.getSchema(); + String queryFilter = config.getQueryFilter(); + + StringBuilder sql = new StringBuilder(); + sql.append("SELECT COUNT(1) FROM (SELECT 1 AS ").append(quotation).append("_ROW").append(quotation).append(" FROM "); + sql.append(schema); + sql.append(quotation); + sql.append(database.buildTableName(tableName)); + sql.append(quotation); + if (StringUtil.isNotBlank(queryFilter)) { + sql.append(queryFilter); + } + sql.append(" GROUP BY "); + // id,uid + PrimaryKeyUtil.buildSql(sql, primaryKeys, quotation, ",", "", true); + sql.append(") DBSYNCER_T"); + return sql.toString(); + } + } \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/sqlbuilder/SqlBuilderQueryCursor.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderQueryCursor.java similarity index 72% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/sqlbuilder/SqlBuilderQueryCursor.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderQueryCursor.java index fa135ab3..e33aa073 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/sqlbuilder/SqlBuilderQueryCursor.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderQueryCursor.java @@ -1,9 +1,9 @@ -package org.dbsyncer.connector.database.sqlbuilder; +package org.dbsyncer.sdk.connector.database.sqlbuilder; -import org.dbsyncer.connector.config.SqlBuilderConfig; -import org.dbsyncer.connector.database.Database; -import org.dbsyncer.connector.model.Field; -import org.dbsyncer.connector.model.PageSql; +import org.dbsyncer.sdk.config.SqlBuilderConfig; +import org.dbsyncer.sdk.connector.database.Database; +import org.dbsyncer.sdk.model.Field; +import org.dbsyncer.sdk.model.PageSql; import java.util.List; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/sqlbuilder/SqlBuilderQueryExist.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderQueryExist.java similarity index 78% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/sqlbuilder/SqlBuilderQueryExist.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderQueryExist.java index 6708b690..d72f33dd 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/sqlbuilder/SqlBuilderQueryExist.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderQueryExist.java @@ -1,33 +1,33 @@ -package org.dbsyncer.connector.database.sqlbuilder; - -import org.dbsyncer.connector.config.SqlBuilderConfig; -import org.dbsyncer.connector.database.Database; -import org.dbsyncer.connector.util.PrimaryKeyUtil; - -import java.util.List; - -/** - * @author AE86 - * @version 1.0.0 - * @date 2023/8/8 0:03 - */ -public class SqlBuilderQueryExist extends SqlBuilderQuery { - - @Override - public String buildSql(SqlBuilderConfig config) { - Database database = config.getDatabase(); - String quotation = database.buildSqlWithQuotation(); - String tableName = config.getTableName(); - String schema = config.getSchema(); - List primaryKeys = database.buildPrimaryKeys(config.getPrimaryKeys()); - StringBuilder sql = new StringBuilder("SELECT COUNT(1) FROM "); - sql.append(schema).append(quotation); - sql.append(database.buildTableName(tableName)); - sql.append(quotation); - sql.append(" WHERE "); - // id = ? AND uid = ? - PrimaryKeyUtil.buildSql(sql, primaryKeys, quotation, " AND ", " = ? ", true); - return sql.toString(); - } - +package org.dbsyncer.sdk.connector.database.sqlbuilder; + +import org.dbsyncer.sdk.config.SqlBuilderConfig; +import org.dbsyncer.sdk.connector.database.Database; +import org.dbsyncer.sdk.util.PrimaryKeyUtil; + +import java.util.List; + +/** + * @author AE86 + * @version 1.0.0 + * @date 2023/8/8 0:03 + */ +public class SqlBuilderQueryExist extends SqlBuilderQuery { + + @Override + public String buildSql(SqlBuilderConfig config) { + Database database = config.getDatabase(); + String quotation = database.buildSqlWithQuotation(); + String tableName = config.getTableName(); + String schema = config.getSchema(); + List primaryKeys = database.buildPrimaryKeys(config.getPrimaryKeys()); + StringBuilder sql = new StringBuilder("SELECT COUNT(1) FROM "); + sql.append(schema).append(quotation); + sql.append(database.buildTableName(tableName)); + sql.append(quotation); + sql.append(" WHERE "); + // id = ? AND uid = ? + PrimaryKeyUtil.buildSql(sql, primaryKeys, quotation, " AND ", " = ? ", true); + return sql.toString(); + } + } \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/sqlbuilder/SqlBuilderUpdate.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderUpdate.java similarity index 83% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/sqlbuilder/SqlBuilderUpdate.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderUpdate.java index 606ab458..934affd1 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/database/sqlbuilder/SqlBuilderUpdate.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderUpdate.java @@ -1,11 +1,11 @@ -package org.dbsyncer.connector.database.sqlbuilder; +package org.dbsyncer.sdk.connector.database.sqlbuilder; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.config.SqlBuilderConfig; -import org.dbsyncer.connector.database.AbstractSqlBuilder; -import org.dbsyncer.connector.database.Database; -import org.dbsyncer.connector.model.Field; -import org.dbsyncer.connector.util.PrimaryKeyUtil; +import org.dbsyncer.sdk.config.SqlBuilderConfig; +import org.dbsyncer.sdk.connector.database.AbstractSqlBuilder; +import org.dbsyncer.sdk.connector.database.Database; +import org.dbsyncer.sdk.model.Field; +import org.dbsyncer.sdk.util.PrimaryKeyUtil; import java.util.List; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/BigintValueMapper.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/BigintValueMapper.java similarity index 62% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/BigintValueMapper.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/BigintValueMapper.java index a9027d1b..b70d933b 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/BigintValueMapper.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/BigintValueMapper.java @@ -1,8 +1,8 @@ -package org.dbsyncer.connector.schema; +package org.dbsyncer.sdk.connector.schema; -import org.dbsyncer.connector.AbstractValueMapper; -import org.dbsyncer.connector.ConnectorException; -import org.dbsyncer.sdk.spi.ConnectorMapper; +import org.dbsyncer.sdk.SdkException; +import org.dbsyncer.sdk.connector.AbstractValueMapper; +import org.dbsyncer.sdk.connector.ConnectorInstance; import java.math.BigDecimal; import java.math.BigInteger; @@ -15,7 +15,7 @@ import java.math.BigInteger; public class BigintValueMapper extends AbstractValueMapper { @Override - protected Long convert(ConnectorMapper connectorMapper, Object val) { + protected Long convert(ConnectorInstance connectorInstance, Object val) { if (val instanceof BigDecimal) { BigDecimal bitDec = (BigDecimal) val; return bitDec.longValue(); @@ -31,6 +31,6 @@ public class BigintValueMapper extends AbstractValueMapper { return new Long((String) val); } - throw new ConnectorException(String.format("%s can not find type [%s], val [%s]", getClass().getSimpleName(), val.getClass(), val)); + throw new SdkException(String.format("%s can not find type [%s], val [%s]", getClass().getSimpleName(), val.getClass(), val)); } } \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/BinaryValueMapper.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/BinaryValueMapper.java similarity index 44% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/BinaryValueMapper.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/BinaryValueMapper.java index 414f1407..b82a7d9b 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/BinaryValueMapper.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/BinaryValueMapper.java @@ -1,8 +1,8 @@ -package org.dbsyncer.connector.schema; +package org.dbsyncer.sdk.connector.schema; -import org.dbsyncer.connector.AbstractValueMapper; -import org.dbsyncer.connector.ConnectorException; -import org.dbsyncer.sdk.spi.ConnectorMapper; +import org.dbsyncer.sdk.SdkException; +import org.dbsyncer.sdk.connector.AbstractValueMapper; +import org.dbsyncer.sdk.connector.ConnectorInstance; /** * @author AE86 @@ -17,11 +17,11 @@ public class BinaryValueMapper extends AbstractValueMapper { } @Override - protected byte[] convert(ConnectorMapper connectorMapper, Object val) { - if(val instanceof String){ + protected byte[] convert(ConnectorInstance connectorInstance, Object val) { + if (val instanceof String) { String s = (String) val; return s.getBytes(); } - throw new ConnectorException(String.format("%s can not find type [%s], val [%s]", getClass().getSimpleName(), val.getClass(), val)); + throw new SdkException(String.format("%s can not find type [%s], val [%s]", getClass().getSimpleName(), val.getClass(), val)); } } \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/BitValueMapper.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/BitValueMapper.java similarity index 64% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/BitValueMapper.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/BitValueMapper.java index bd2e40af..0e877eae 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/BitValueMapper.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/BitValueMapper.java @@ -1,8 +1,8 @@ -package org.dbsyncer.connector.schema; +package org.dbsyncer.sdk.connector.schema; -import org.dbsyncer.connector.AbstractValueMapper; -import org.dbsyncer.connector.ConnectorException; -import org.dbsyncer.sdk.spi.ConnectorMapper; +import org.dbsyncer.sdk.SdkException; +import org.dbsyncer.sdk.connector.AbstractValueMapper; +import org.dbsyncer.sdk.connector.ConnectorInstance; import java.nio.ByteBuffer; import java.util.BitSet; @@ -15,7 +15,7 @@ import java.util.BitSet; public class BitValueMapper extends AbstractValueMapper { @Override - protected byte[] convert(ConnectorMapper connectorMapper, Object val) { + protected byte[] convert(ConnectorInstance connectorInstance, Object val) { if (val instanceof BitSet) { BitSet bitSet = (BitSet) val; return bitSet.toByteArray(); @@ -32,7 +32,7 @@ public class BitValueMapper extends AbstractValueMapper { return buffer.array(); } - throw new ConnectorException(String.format("%s can not find type [%s], val [%s]", getClass().getSimpleName(), val.getClass(), val)); + throw new SdkException(String.format("%s can not find type [%s], val [%s]", getClass().getSimpleName(), val.getClass(), val)); } } \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/BlobValueMapper.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/BlobValueMapper.java similarity index 44% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/BlobValueMapper.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/BlobValueMapper.java index 6eec09e0..a316a470 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/BlobValueMapper.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/BlobValueMapper.java @@ -1,8 +1,8 @@ -package org.dbsyncer.connector.schema; +package org.dbsyncer.sdk.connector.schema; -import org.dbsyncer.connector.AbstractValueMapper; -import org.dbsyncer.connector.ConnectorException; -import org.dbsyncer.sdk.spi.ConnectorMapper; +import org.dbsyncer.sdk.SdkException; +import org.dbsyncer.sdk.connector.AbstractValueMapper; +import org.dbsyncer.sdk.connector.ConnectorInstance; import java.sql.Blob; @@ -19,7 +19,7 @@ public class BlobValueMapper extends AbstractValueMapper { } @Override - protected Blob convert(ConnectorMapper connectorMapper, Object val) { - throw new ConnectorException(String.format("%s can not find type [%s], val [%s]", getClass().getSimpleName(), val.getClass(), val)); + protected Blob convert(ConnectorInstance connectorInstance, Object val) { + throw new SdkException(String.format("%s can not find type [%s], val [%s]", getClass().getSimpleName(), val.getClass(), val)); } } \ No newline at end of file diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/CharValueMapper.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/CharValueMapper.java new file mode 100644 index 00000000..499a9b2f --- /dev/null +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/CharValueMapper.java @@ -0,0 +1,18 @@ +package org.dbsyncer.sdk.connector.schema; + +import org.dbsyncer.sdk.SdkException; +import org.dbsyncer.sdk.connector.AbstractValueMapper; +import org.dbsyncer.sdk.connector.ConnectorInstance; + +/** + * @author AE86 + * @version 1.0.0 + * @date 2022/8/25 0:07 + */ +public class CharValueMapper extends AbstractValueMapper { + + @Override + protected String convert(ConnectorInstance connectorInstance, Object val) { + throw new SdkException(String.format("%s can not find type [%s], val [%s]", getClass().getSimpleName(), val.getClass(), val)); + } +} \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/ClobValueMapper.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/ClobValueMapper.java similarity index 45% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/ClobValueMapper.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/ClobValueMapper.java index b4da6e22..c5013e00 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/ClobValueMapper.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/ClobValueMapper.java @@ -1,8 +1,8 @@ -package org.dbsyncer.connector.schema; +package org.dbsyncer.sdk.connector.schema; -import org.dbsyncer.connector.AbstractValueMapper; -import org.dbsyncer.connector.ConnectorException; -import org.dbsyncer.sdk.spi.ConnectorMapper; +import org.dbsyncer.sdk.SdkException; +import org.dbsyncer.sdk.connector.AbstractValueMapper; +import org.dbsyncer.sdk.connector.ConnectorInstance; import java.sql.Clob; @@ -19,7 +19,7 @@ public class ClobValueMapper extends AbstractValueMapper { } @Override - protected Clob convert(ConnectorMapper connectorMapper, Object val) { - throw new ConnectorException(String.format("%s can not find type [%s], val [%s]", getClass().getSimpleName(), val.getClass(), val)); + protected Clob convert(ConnectorInstance connectorInstance, Object val) { + throw new SdkException(String.format("%s can not find type [%s], val [%s]", getClass().getSimpleName(), val.getClass(), val)); } } \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/DateValueMapper.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/DateValueMapper.java similarity index 64% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/DateValueMapper.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/DateValueMapper.java index 5d6a5ac0..757fc814 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/DateValueMapper.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/DateValueMapper.java @@ -1,9 +1,9 @@ -package org.dbsyncer.connector.schema; +package org.dbsyncer.sdk.connector.schema; import org.dbsyncer.common.util.DateFormatUtil; -import org.dbsyncer.connector.AbstractValueMapper; -import org.dbsyncer.connector.ConnectorException; -import org.dbsyncer.sdk.spi.ConnectorMapper; +import org.dbsyncer.sdk.SdkException; +import org.dbsyncer.sdk.connector.AbstractValueMapper; +import org.dbsyncer.sdk.connector.ConnectorInstance; import java.sql.Date; import java.sql.Timestamp; @@ -16,7 +16,7 @@ import java.sql.Timestamp; public class DateValueMapper extends AbstractValueMapper { @Override - protected Date convert(ConnectorMapper connectorMapper, Object val) { + protected Date convert(ConnectorInstance connectorInstance, Object val) { if (val instanceof Timestamp) { Timestamp timestamp = (Timestamp) val; return Date.valueOf(timestamp.toLocalDateTime().toLocalDate()); @@ -30,6 +30,6 @@ public class DateValueMapper extends AbstractValueMapper { } } - throw new ConnectorException(String.format("%s can not find type [%s], val [%s]", getClass().getSimpleName(), val.getClass(), val)); + throw new SdkException(String.format("%s can not find type [%s], val [%s]", getClass().getSimpleName(), val.getClass(), val)); } } \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/DecimalValueMapper.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/DecimalValueMapper.java similarity index 73% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/DecimalValueMapper.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/DecimalValueMapper.java index 6212d55c..bc54b311 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/DecimalValueMapper.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/DecimalValueMapper.java @@ -1,8 +1,8 @@ -package org.dbsyncer.connector.schema; +package org.dbsyncer.sdk.connector.schema; -import org.dbsyncer.connector.AbstractValueMapper; -import org.dbsyncer.connector.ConnectorException; -import org.dbsyncer.sdk.spi.ConnectorMapper; +import org.dbsyncer.sdk.SdkException; +import org.dbsyncer.sdk.connector.AbstractValueMapper; +import org.dbsyncer.sdk.connector.ConnectorInstance; import java.math.BigDecimal; import java.math.BigInteger; @@ -15,7 +15,7 @@ import java.math.BigInteger; public class DecimalValueMapper extends AbstractValueMapper { @Override - protected BigDecimal convert(ConnectorMapper connectorMapper, Object val) { + protected BigDecimal convert(ConnectorInstance connectorInstance, Object val) { if (val instanceof Integer) { Integer integer = (Integer) val; return new BigDecimal(integer); @@ -44,6 +44,6 @@ public class DecimalValueMapper extends AbstractValueMapper { String s = (String) val; return new BigDecimal(s); } - throw new ConnectorException(String.format("%s can not find type [%s], val [%s]", getClass().getSimpleName(), val.getClass(), val)); + throw new SdkException(String.format("%s can not find type [%s], val [%s]", getClass().getSimpleName(), val.getClass(), val)); } } \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/DoubleValueMapper.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/DoubleValueMapper.java similarity index 45% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/DoubleValueMapper.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/DoubleValueMapper.java index 632ce6f1..b820aa6e 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/DoubleValueMapper.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/DoubleValueMapper.java @@ -1,8 +1,8 @@ -package org.dbsyncer.connector.schema; +package org.dbsyncer.sdk.connector.schema; -import org.dbsyncer.connector.AbstractValueMapper; -import org.dbsyncer.connector.ConnectorException; -import org.dbsyncer.sdk.spi.ConnectorMapper; +import org.dbsyncer.sdk.SdkException; +import org.dbsyncer.sdk.connector.AbstractValueMapper; +import org.dbsyncer.sdk.connector.ConnectorInstance; import java.math.BigDecimal; @@ -14,11 +14,11 @@ import java.math.BigDecimal; public class DoubleValueMapper extends AbstractValueMapper { @Override - protected Double convert(ConnectorMapper connectorMapper, Object val) { + protected Double convert(ConnectorInstance connectorInstance, Object val) { if (val instanceof BigDecimal) { BigDecimal bigDecimal = (BigDecimal) val; return bigDecimal.doubleValue(); } - throw new ConnectorException(String.format("%s can not find type [%s], val [%s]", getClass().getSimpleName(), val.getClass(), val)); + throw new SdkException(String.format("%s can not find type [%s], val [%s]", getClass().getSimpleName(), val.getClass(), val)); } } \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/FloatValueMapper.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/FloatValueMapper.java similarity index 53% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/FloatValueMapper.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/FloatValueMapper.java index 75845e0d..ec9b0160 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/FloatValueMapper.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/FloatValueMapper.java @@ -1,8 +1,8 @@ -package org.dbsyncer.connector.schema; +package org.dbsyncer.sdk.connector.schema; -import org.dbsyncer.connector.AbstractValueMapper; -import org.dbsyncer.connector.ConnectorException; -import org.dbsyncer.sdk.spi.ConnectorMapper; +import org.dbsyncer.sdk.SdkException; +import org.dbsyncer.sdk.connector.AbstractValueMapper; +import org.dbsyncer.sdk.connector.ConnectorInstance; import java.math.BigDecimal; @@ -14,7 +14,7 @@ import java.math.BigDecimal; public class FloatValueMapper extends AbstractValueMapper { @Override - protected Float convert(ConnectorMapper connectorMapper, Object val) { + protected Float convert(ConnectorInstance connectorInstance, Object val) { if (val instanceof BigDecimal) { BigDecimal bigDecimal = (BigDecimal) val; return bigDecimal.floatValue(); @@ -23,6 +23,6 @@ public class FloatValueMapper extends AbstractValueMapper { Double dbl = (Double) val; return dbl.floatValue(); } - throw new ConnectorException(String.format("%s can not find type [%s], val [%s]", getClass().getSimpleName(), val.getClass(), val)); + throw new SdkException(String.format("%s can not find type [%s], val [%s]", getClass().getSimpleName(), val.getClass(), val)); } } \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/IntegerValueMapper.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/IntegerValueMapper.java similarity index 65% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/IntegerValueMapper.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/IntegerValueMapper.java index 00cb0fa8..2c2f7fe6 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/IntegerValueMapper.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/IntegerValueMapper.java @@ -1,8 +1,8 @@ -package org.dbsyncer.connector.schema; +package org.dbsyncer.sdk.connector.schema; -import org.dbsyncer.connector.AbstractValueMapper; -import org.dbsyncer.connector.ConnectorException; -import org.dbsyncer.sdk.spi.ConnectorMapper; +import org.dbsyncer.sdk.SdkException; +import org.dbsyncer.sdk.connector.AbstractValueMapper; +import org.dbsyncer.sdk.connector.ConnectorInstance; import java.math.BigDecimal; import java.math.BigInteger; @@ -14,7 +14,7 @@ import java.math.BigInteger; */ public class IntegerValueMapper extends AbstractValueMapper { @Override - protected Integer convert(ConnectorMapper connectorMapper, Object val) { + protected Integer convert(ConnectorInstance connectorInstance, Object val) { if (val instanceof BigInteger) { BigInteger bigInteger = (BigInteger) val; return bigInteger.intValue(); @@ -35,6 +35,6 @@ public class IntegerValueMapper extends AbstractValueMapper { return new Integer(b ? 1 : 0); } - throw new ConnectorException(String.format("%s can not find type [%s], val [%s]", getClass().getSimpleName(), val.getClass(), val)); + throw new SdkException(String.format("%s can not find type [%s], val [%s]", getClass().getSimpleName(), val.getClass(), val)); } } \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/LongVarBinaryValueMapper.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/LongVarBinaryValueMapper.java similarity index 41% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/LongVarBinaryValueMapper.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/LongVarBinaryValueMapper.java index 21176ff3..0acac03c 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/LongVarBinaryValueMapper.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/LongVarBinaryValueMapper.java @@ -1,8 +1,8 @@ -package org.dbsyncer.connector.schema; +package org.dbsyncer.sdk.connector.schema; -import org.dbsyncer.connector.AbstractValueMapper; -import org.dbsyncer.connector.ConnectorException; -import org.dbsyncer.sdk.spi.ConnectorMapper; +import org.dbsyncer.sdk.SdkException; +import org.dbsyncer.sdk.connector.AbstractValueMapper; +import org.dbsyncer.sdk.connector.ConnectorInstance; /** * @author AE86 @@ -17,7 +17,7 @@ public class LongVarBinaryValueMapper extends AbstractValueMapper { } @Override - protected byte[] convert(ConnectorMapper connectorMapper, Object val) { - throw new ConnectorException(String.format("%s can not find type [%s], val [%s]", getClass().getSimpleName(), val.getClass(), val)); + protected byte[] convert(ConnectorInstance connectorInstance, Object val) { + throw new SdkException(String.format("%s can not find type [%s], val [%s]", getClass().getSimpleName(), val.getClass(), val)); } } \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/LongVarcharValueMapper.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/LongVarcharValueMapper.java similarity index 52% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/LongVarcharValueMapper.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/LongVarcharValueMapper.java index 3ff9dad7..95c42ad0 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/LongVarcharValueMapper.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/LongVarcharValueMapper.java @@ -1,8 +1,8 @@ -package org.dbsyncer.connector.schema; +package org.dbsyncer.sdk.connector.schema; -import org.dbsyncer.connector.AbstractValueMapper; -import org.dbsyncer.connector.ConnectorException; -import org.dbsyncer.sdk.spi.ConnectorMapper; +import org.dbsyncer.sdk.SdkException; +import org.dbsyncer.sdk.connector.AbstractValueMapper; +import org.dbsyncer.sdk.connector.ConnectorInstance; import java.sql.Date; @@ -14,7 +14,7 @@ import java.sql.Date; public class LongVarcharValueMapper extends AbstractValueMapper { @Override - protected String convert(ConnectorMapper connectorMapper, Object val) { + protected String convert(ConnectorInstance connectorInstance, Object val) { if (val instanceof byte[]) { return new String((byte[]) val); } @@ -24,6 +24,6 @@ public class LongVarcharValueMapper extends AbstractValueMapper { if (val instanceof Integer) { return String.valueOf(val); } - throw new ConnectorException(String.format("%s can not find type [%s], val [%s]", getClass().getSimpleName(), val.getClass(), val)); + throw new SdkException(String.format("%s can not find type [%s], val [%s]", getClass().getSimpleName(), val.getClass(), val)); } } \ No newline at end of file diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/NCharValueMapper.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/NCharValueMapper.java new file mode 100644 index 00000000..e3e8c401 --- /dev/null +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/NCharValueMapper.java @@ -0,0 +1,18 @@ +package org.dbsyncer.sdk.connector.schema; + +import org.dbsyncer.sdk.SdkException; +import org.dbsyncer.sdk.connector.AbstractValueMapper; +import org.dbsyncer.sdk.connector.ConnectorInstance; + +/** + * @author AE86 + * @version 1.0.0 + * @date 2022/8/25 0:07 + */ +public class NCharValueMapper extends AbstractValueMapper { + + @Override + protected String convert(ConnectorInstance connectorInstance, Object val) { + throw new SdkException(String.format("%s can not find type [%s], val [%s]", getClass().getSimpleName(), val.getClass(), val)); + } +} \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/NClobValueMapper.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/NClobValueMapper.java similarity index 44% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/NClobValueMapper.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/NClobValueMapper.java index 84cb844d..aa72f6e1 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/NClobValueMapper.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/NClobValueMapper.java @@ -1,8 +1,8 @@ -package org.dbsyncer.connector.schema; +package org.dbsyncer.sdk.connector.schema; -import org.dbsyncer.connector.AbstractValueMapper; -import org.dbsyncer.connector.ConnectorException; -import org.dbsyncer.sdk.spi.ConnectorMapper; +import org.dbsyncer.sdk.SdkException; +import org.dbsyncer.sdk.connector.AbstractValueMapper; +import org.dbsyncer.sdk.connector.ConnectorInstance; import java.sql.NClob; @@ -19,7 +19,7 @@ public class NClobValueMapper extends AbstractValueMapper { } @Override - protected NClob convert(ConnectorMapper connectorMapper, Object val) { - throw new ConnectorException(String.format("%s can not find type [%s], val [%s]", getClass().getSimpleName(), val.getClass(), val)); + protected NClob convert(ConnectorInstance connectorInstance, Object val) { + throw new SdkException(String.format("%s can not find type [%s], val [%s]", getClass().getSimpleName(), val.getClass(), val)); } } \ No newline at end of file diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/NVarcharValueMapper.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/NVarcharValueMapper.java new file mode 100644 index 00000000..4bb1fc2f --- /dev/null +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/NVarcharValueMapper.java @@ -0,0 +1,18 @@ +package org.dbsyncer.sdk.connector.schema; + +import org.dbsyncer.sdk.SdkException; +import org.dbsyncer.sdk.connector.AbstractValueMapper; +import org.dbsyncer.sdk.connector.ConnectorInstance; + +/** + * @author AE86 + * @version 1.0.0 + * @date 2022/8/24 23:43 + */ +public class NVarcharValueMapper extends AbstractValueMapper { + + @Override + protected String convert(ConnectorInstance connectorInstance, Object val) { + throw new SdkException(String.format("%s can not find type [%s], val [%s]", getClass().getSimpleName(), val.getClass(), val)); + } +} \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/NumberValueMapper.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/NumberValueMapper.java similarity index 59% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/NumberValueMapper.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/NumberValueMapper.java index 449f44b6..ae22bc22 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/NumberValueMapper.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/NumberValueMapper.java @@ -1,8 +1,8 @@ -package org.dbsyncer.connector.schema; +package org.dbsyncer.sdk.connector.schema; -import org.dbsyncer.connector.AbstractValueMapper; -import org.dbsyncer.connector.ConnectorException; -import org.dbsyncer.sdk.spi.ConnectorMapper; +import org.dbsyncer.sdk.SdkException; +import org.dbsyncer.sdk.connector.AbstractValueMapper; +import org.dbsyncer.sdk.connector.ConnectorInstance; import java.math.BigDecimal; import java.math.BigInteger; @@ -15,7 +15,7 @@ import java.math.BigInteger; public class NumberValueMapper extends AbstractValueMapper { @Override - protected BigDecimal convert(ConnectorMapper connectorMapper, Object val) { + protected BigDecimal convert(ConnectorInstance connectorInstance, Object val) { if (val instanceof Integer) { Integer integer = (Integer) val; return new BigDecimal(integer); @@ -29,6 +29,6 @@ public class NumberValueMapper extends AbstractValueMapper { return new BigDecimal(b); } - throw new ConnectorException(String.format("%s can not find type [%s], val [%s]", getClass().getSimpleName(), val.getClass(), val)); + throw new SdkException(String.format("%s can not find type [%s], val [%s]", getClass().getSimpleName(), val.getClass(), val)); } } \ No newline at end of file diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/OtherValueMapper.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/OtherValueMapper.java new file mode 100644 index 00000000..a33637c3 --- /dev/null +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/OtherValueMapper.java @@ -0,0 +1,18 @@ +package org.dbsyncer.sdk.connector.schema; + +import org.dbsyncer.sdk.SdkException; +import org.dbsyncer.sdk.connector.AbstractValueMapper; +import org.dbsyncer.sdk.connector.ConnectorInstance; + +/** + * @author AE86 + * @version 1.0.0 + * @date 2022/9/16 16:54 + */ +public class OtherValueMapper extends AbstractValueMapper { + + @Override + protected Object convert(ConnectorInstance connectorInstance, Object val) { + throw new SdkException(String.format("%s can not find type [%s], val [%s]", getClass().getSimpleName(), val.getClass(), val)); + } +} \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/RealValueMapper.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/RealValueMapper.java similarity index 42% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/RealValueMapper.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/RealValueMapper.java index 8e6da43f..ab9ce942 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/RealValueMapper.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/RealValueMapper.java @@ -1,8 +1,8 @@ -package org.dbsyncer.connector.schema; +package org.dbsyncer.sdk.connector.schema; -import org.dbsyncer.connector.AbstractValueMapper; -import org.dbsyncer.connector.ConnectorException; -import org.dbsyncer.sdk.spi.ConnectorMapper; +import org.dbsyncer.sdk.SdkException; +import org.dbsyncer.sdk.connector.AbstractValueMapper; +import org.dbsyncer.sdk.connector.ConnectorInstance; /** * @author AE86 @@ -12,12 +12,12 @@ import org.dbsyncer.sdk.spi.ConnectorMapper; public class RealValueMapper extends AbstractValueMapper { @Override - protected Float convert(ConnectorMapper connectorMapper, Object val) { + protected Float convert(ConnectorInstance connectorInstance, Object val) { if (val instanceof Double) { Double dob = (Double) val; return Float.valueOf(dob.floatValue()); } - throw new ConnectorException(String.format("%s can not find type [%s], val [%s]", getClass().getSimpleName(), val.getClass(), val)); + throw new SdkException(String.format("%s can not find type [%s], val [%s]", getClass().getSimpleName(), val.getClass(), val)); } } \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/RowIdValueMapper.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/RowIdValueMapper.java similarity index 30% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/RowIdValueMapper.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/RowIdValueMapper.java index 7c091b20..52e1f22f 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/RowIdValueMapper.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/RowIdValueMapper.java @@ -1,8 +1,8 @@ -package org.dbsyncer.connector.schema; +package org.dbsyncer.sdk.connector.schema; -import org.dbsyncer.connector.AbstractValueMapper; -import org.dbsyncer.connector.ConnectorException; -import org.dbsyncer.sdk.spi.ConnectorMapper; +import org.dbsyncer.sdk.SdkException; +import org.dbsyncer.sdk.connector.AbstractValueMapper; +import org.dbsyncer.sdk.connector.ConnectorInstance; import java.sql.RowId; @@ -14,7 +14,7 @@ import java.sql.RowId; public class RowIdValueMapper extends AbstractValueMapper { @Override - protected RowId convert(ConnectorMapper connectorMapper, Object val) { - throw new ConnectorException(String.format("%s can not find type [%s], val [%s]", getClass().getSimpleName(), val.getClass(), val)); + protected RowId convert(ConnectorInstance connectorInstance, Object val) { + throw new SdkException(String.format("%s can not find type [%s], val [%s]", getClass().getSimpleName(), val.getClass(), val)); } } \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/SmallintValueMapper.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/SmallintValueMapper.java similarity index 35% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/SmallintValueMapper.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/SmallintValueMapper.java index fdc38c3f..85bdfbd3 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/SmallintValueMapper.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/SmallintValueMapper.java @@ -1,8 +1,8 @@ -package org.dbsyncer.connector.schema; +package org.dbsyncer.sdk.connector.schema; -import org.dbsyncer.connector.AbstractValueMapper; -import org.dbsyncer.connector.ConnectorException; -import org.dbsyncer.sdk.spi.ConnectorMapper; +import org.dbsyncer.sdk.SdkException; +import org.dbsyncer.sdk.connector.AbstractValueMapper; +import org.dbsyncer.sdk.connector.ConnectorInstance; /** * @author AE86 @@ -12,12 +12,12 @@ import org.dbsyncer.sdk.spi.ConnectorMapper; public class SmallintValueMapper extends AbstractValueMapper { @Override - protected Integer convert(ConnectorMapper connectorMapper, Object val) { - if(val instanceof Short){ + protected Integer convert(ConnectorInstance connectorInstance, Object val) { + if (val instanceof Short) { Short s = (Short) val; return new Integer(s); } - throw new ConnectorException(String.format("%s can not find type [%s], val [%s]", getClass().getSimpleName(), val.getClass(), val)); + throw new SdkException(String.format("%s can not find type [%s], val [%s]", getClass().getSimpleName(), val.getClass(), val)); } } \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/TimeValueMapper.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/TimeValueMapper.java similarity index 30% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/TimeValueMapper.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/TimeValueMapper.java index f2bf2a51..0c0c7c1a 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/schema/TimeValueMapper.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/TimeValueMapper.java @@ -1,8 +1,8 @@ -package org.dbsyncer.connector.schema; +package org.dbsyncer.sdk.connector.schema; -import org.dbsyncer.connector.AbstractValueMapper; -import org.dbsyncer.connector.ConnectorException; -import org.dbsyncer.sdk.spi.ConnectorMapper; +import org.dbsyncer.sdk.SdkException; +import org.dbsyncer.sdk.connector.AbstractValueMapper; +import org.dbsyncer.sdk.connector.ConnectorInstance; import java.sql.Time; @@ -14,7 +14,7 @@ import java.sql.Time; public class TimeValueMapper extends AbstractValueMapper
    sourceTable) { + public AbstractListener setSourceTable(List
    sourceTable) { this.sourceTable = sourceTable; return this; } diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/ConnectorFactory.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/ConnectorFactory.java index 551ba837..9ea164e9 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/ConnectorFactory.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/ConnectorFactory.java @@ -8,6 +8,7 @@ import org.dbsyncer.sdk.config.ReaderConfig; import org.dbsyncer.sdk.config.WriterBatchConfig; import org.dbsyncer.sdk.connector.AbstractConnector; import org.dbsyncer.sdk.connector.ConnectorInstance; +import org.dbsyncer.sdk.listener.Listener; import org.dbsyncer.sdk.model.ConnectorConfig; import org.dbsyncer.sdk.model.MetaInfo; import org.dbsyncer.sdk.model.Table; @@ -89,19 +90,14 @@ public class ConnectorFactory implements DisposableBean { } /** - * 断开连接 + * 获取监听器 * - * @param config + * @param connectorType + * @param listenerType * @return */ - public void disconnect(ConnectorConfig config) { - Assert.notNull(config, "ConnectorConfig can not be null."); - String cacheKey = getConnectorService(config).getConnectorInstanceCacheKey(config); - ConnectorInstance connectorInstance = pool.get(cacheKey); - if (connectorInstance != null) { - disconnect(connectorInstance); - pool.remove(cacheKey); - } + public Listener getListener(String connectorType, String listenerType) { + return getConnectorService(connectorType).getListener(listenerType); } /** @@ -209,7 +205,7 @@ public class ConnectorFactory implements DisposableBean { return result; } - private ConnectorService getConnectorService(ConnectorConfig connectorConfig) { + public ConnectorService getConnectorService(ConnectorConfig connectorConfig) { Assert.notNull(connectorConfig, "ConnectorConfig can not null"); return getConnectorService(connectorConfig.getConnectorType()); } @@ -229,8 +225,19 @@ public class ConnectorFactory implements DisposableBean { /** * 断开连接 * - * @param connectorInstance + * @param config + * @return */ + public void disconnect(ConnectorConfig config) { + Assert.notNull(config, "ConnectorConfig can not be null."); + String cacheKey = getConnectorService(config).getConnectorInstanceCacheKey(config); + ConnectorInstance connectorInstance = pool.get(cacheKey); + if (connectorInstance != null) { + disconnect(connectorInstance); + pool.remove(cacheKey); + } + } + private void disconnect(ConnectorInstance connectorInstance) { Assert.notNull(connectorInstance, "ConnectorInstance can not be null."); getConnectorService(connectorInstance.getConfig()).disconnect(connectorInstance); diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/config/ListenerConfig.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/ListenerConfig.java similarity index 96% rename from dbsyncer-listener/src/main/java/org/dbsyncer/listener/config/ListenerConfig.java rename to dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/ListenerConfig.java index 2a23cedd..75cbf8d6 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/config/ListenerConfig.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/ListenerConfig.java @@ -1,6 +1,6 @@ -package org.dbsyncer.listener.config; +package org.dbsyncer.connector.config; -import org.dbsyncer.listener.enums.ListenerTypeEnum; +import org.dbsyncer.sdk.enums.ListenerTypeEnum; /** * @author AE86 diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/enums/QuartzFilterEnum.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/enums/QuartzFilterEnum.java similarity index 87% rename from dbsyncer-listener/src/main/java/org/dbsyncer/listener/enums/QuartzFilterEnum.java rename to dbsyncer-connector/src/main/java/org/dbsyncer/connector/enums/QuartzFilterEnum.java index d9f34c6e..e769410e 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/enums/QuartzFilterEnum.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/enums/QuartzFilterEnum.java @@ -1,11 +1,11 @@ -package org.dbsyncer.listener.enums; +package org.dbsyncer.connector.enums; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.listener.quartz.QuartzFilter; -import org.dbsyncer.listener.quartz.filter.DateFilter; -import org.dbsyncer.listener.quartz.filter.TimestampFilter; -import org.dbsyncer.listener.quartz.filter.YesDateFilter; -import org.dbsyncer.listener.quartz.filter.YesTimestampFilter; +import org.dbsyncer.connector.quartz.QuartzFilter; +import org.dbsyncer.connector.quartz.filter.DateFilter; +import org.dbsyncer.connector.quartz.filter.TimestampFilter; +import org.dbsyncer.connector.quartz.filter.YesDateFilter; +import org.dbsyncer.connector.quartz.filter.YesTimestampFilter; /** * @author AE86 diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/enums/TableOperationEnum.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/enums/TableOperationEnum.java similarity index 95% rename from dbsyncer-listener/src/main/java/org/dbsyncer/listener/enums/TableOperationEnum.java rename to dbsyncer-connector/src/main/java/org/dbsyncer/connector/enums/TableOperationEnum.java index ec117f9e..af235645 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/enums/TableOperationEnum.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/enums/TableOperationEnum.java @@ -1,4 +1,4 @@ -package org.dbsyncer.listener.enums; +package org.dbsyncer.connector.enums; public enum TableOperationEnum { diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESConnector.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESConnector.java index 01fe6ed5..d0b9a6f0 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESConnector.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESConnector.java @@ -15,7 +15,9 @@ import org.dbsyncer.sdk.config.WriterBatchConfig; import org.dbsyncer.sdk.connector.AbstractConnector; import org.dbsyncer.sdk.connector.ConnectorInstance; import org.dbsyncer.sdk.constant.ConnectorConstant; +import org.dbsyncer.sdk.enums.ListenerTypeEnum; import org.dbsyncer.sdk.enums.OperationEnum; +import org.dbsyncer.sdk.listener.Listener; import org.dbsyncer.sdk.model.Field; import org.dbsyncer.sdk.model.Filter; import org.dbsyncer.sdk.model.MetaInfo; @@ -289,6 +291,14 @@ public final class ESConnector extends AbstractConnector implements ConnectorSer return Collections.EMPTY_MAP; } + @Override + public Listener getListener(String listenerType) { + if (ListenerTypeEnum.isTiming(listenerType)) { + return new ESQuartzListener(); + } + return null; + } + private void parseProperties(List fields, Map sourceMap) { Map properties = (Map) sourceMap.get(ESUtil.PROPERTIES); if (CollectionUtils.isEmpty(properties)) { diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/ESQuartzExtractor.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESQuartzListener.java similarity index 84% rename from dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/ESQuartzExtractor.java rename to dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESQuartzListener.java index fd60924f..7eebed34 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/ESQuartzExtractor.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESQuartzListener.java @@ -1,10 +1,13 @@ -package org.dbsyncer.listener.quartz; +package org.dbsyncer.connector.es; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.JsonUtil; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.listener.ListenerException; -import org.dbsyncer.listener.enums.QuartzFilterEnum; +import org.dbsyncer.connector.ConnectorException; +import org.dbsyncer.connector.enums.QuartzFilterEnum; +import org.dbsyncer.connector.quartz.AbstractQuartzListener; +import org.dbsyncer.connector.quartz.Point; +import org.dbsyncer.connector.quartz.QuartzFilter; import org.dbsyncer.sdk.constant.ConnectorConstant; import org.dbsyncer.sdk.model.Filter; @@ -21,7 +24,7 @@ import java.util.Set; * @Author AE86 * @Date 2021-09-01 20:35 */ -public final class ESQuartzExtractor extends AbstractQuartzExtractor { +public final class ESQuartzListener extends AbstractQuartzListener { @Override protected Point checkLastPoint(Map command, int index) { @@ -40,7 +43,7 @@ public final class ESQuartzExtractor extends AbstractQuartzExtractor { Set set = new HashSet<>(); for (Filter f : filters) { if (set.contains(f.getValue())) { - throw new ListenerException(String.format("系统参数%s存在多个.", f.getValue())); + throw new ConnectorException(String.format("系统参数%s存在多个.", f.getValue())); } QuartzFilterEnum filterEnum = QuartzFilterEnum.getQuartzFilterEnum(f.getValue()); if (null != filterEnum) { diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/file/BufferedRandomAccessFile.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/BufferedRandomAccessFile.java similarity index 99% rename from dbsyncer-listener/src/main/java/org/dbsyncer/listener/file/BufferedRandomAccessFile.java rename to dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/BufferedRandomAccessFile.java index 6b560f28..44f6be4d 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/file/BufferedRandomAccessFile.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/BufferedRandomAccessFile.java @@ -1,4 +1,4 @@ -package org.dbsyncer.listener.file; +package org.dbsyncer.connector.file; import java.io.File; import java.io.FileNotFoundException; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileConnector.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileConnector.java index 78c9031d..fed6638f 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileConnector.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileConnector.java @@ -7,12 +7,13 @@ import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.connector.ConnectorException; import org.dbsyncer.connector.config.FileConfig; -import org.dbsyncer.connector.model.FileSchema; import org.dbsyncer.sdk.config.CommandConfig; import org.dbsyncer.sdk.config.ReaderConfig; import org.dbsyncer.sdk.config.WriterBatchConfig; import org.dbsyncer.sdk.connector.AbstractConnector; import org.dbsyncer.sdk.connector.ConnectorInstance; +import org.dbsyncer.sdk.enums.ListenerTypeEnum; +import org.dbsyncer.sdk.listener.Listener; import org.dbsyncer.sdk.model.Field; import org.dbsyncer.sdk.model.MetaInfo; import org.dbsyncer.sdk.model.Table; @@ -233,4 +234,12 @@ public final class FileConnector extends AbstractConnector implements ConnectorS return command; } + @Override + public Listener getListener(String listenerType) { + if (ListenerTypeEnum.isLog(listenerType)) { + return new FileListener(); + } + return null; + } + } \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileConnectorInstance.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileConnectorInstance.java index 01d4c5de..9c402011 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileConnectorInstance.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileConnectorInstance.java @@ -2,7 +2,6 @@ package org.dbsyncer.connector.file; import org.dbsyncer.common.util.JsonUtil; import org.dbsyncer.connector.config.FileConfig; -import org.dbsyncer.connector.model.FileSchema; import org.dbsyncer.sdk.connector.ConnectorInstance; import org.dbsyncer.sdk.model.Field; import org.springframework.util.Assert; diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/file/FileExtractor.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileListener.java similarity index 94% rename from dbsyncer-listener/src/main/java/org/dbsyncer/listener/file/FileExtractor.java rename to dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileListener.java index b39931f2..3057d2b8 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/file/FileExtractor.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileListener.java @@ -1,18 +1,15 @@ -package org.dbsyncer.listener.file; +package org.dbsyncer.connector.file; import org.apache.commons.io.IOUtils; -import org.dbsyncer.listener.model.ChangedOffset; -import org.dbsyncer.listener.event.RowChangedEvent; +import org.dbsyncer.connector.ConnectorException; +import org.dbsyncer.sdk.model.ChangedOffset; +import org.dbsyncer.sdk.listener.event.RowChangedEvent; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.NumberUtil; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.connector.config.FileConfig; import org.dbsyncer.sdk.constant.ConnectorConstant; -import org.dbsyncer.connector.file.FileConnectorInstance; -import org.dbsyncer.connector.file.FileResolver; -import org.dbsyncer.connector.model.FileSchema; -import org.dbsyncer.listener.AbstractExtractor; -import org.dbsyncer.listener.ListenerException; +import org.dbsyncer.connector.AbstractListener; import org.dbsyncer.sdk.model.Field; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -43,7 +40,7 @@ import java.util.concurrent.locks.ReentrantLock; * @version 1.0.0 * @date 2022/5/6 21:42 */ -public class FileExtractor extends AbstractExtractor { +public class FileListener extends AbstractListener { private final Logger logger = LoggerFactory.getLogger(getClass()); @@ -89,7 +86,7 @@ public class FileExtractor extends AbstractExtractor { } catch (Exception e) { logger.error("启动失败:{}", e.getMessage()); closePipelineAndWatch(); - throw new ListenerException(e); + throw new ConnectorException(e); } finally { connectLock.unlock(); } diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/model/FileSchema.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileSchema.java similarity index 93% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/model/FileSchema.java rename to dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileSchema.java index 28d4cf5d..0ce273c8 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/model/FileSchema.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileSchema.java @@ -1,4 +1,4 @@ -package org.dbsyncer.connector.model; +package org.dbsyncer.connector.file; import org.dbsyncer.sdk.model.Field; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/kafka/KafkaConnector.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/kafka/KafkaConnector.java index a730b35c..76158000 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/kafka/KafkaConnector.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/kafka/KafkaConnector.java @@ -10,6 +10,7 @@ import org.dbsyncer.sdk.connector.AbstractConnector; import org.dbsyncer.connector.ConnectorException; import org.dbsyncer.connector.config.KafkaConfig; import org.dbsyncer.sdk.connector.ConnectorInstance; +import org.dbsyncer.sdk.listener.Listener; import org.dbsyncer.sdk.model.Field; import org.dbsyncer.sdk.model.MetaInfo; import org.dbsyncer.sdk.model.Table; @@ -131,4 +132,9 @@ public class KafkaConnector extends AbstractConnector implements ConnectorServic return Collections.EMPTY_MAP; } + @Override + public Listener getListener(String listenerType) { + return null; + } + } \ No newline at end of file diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/mysql/BinaryLogClient.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/BinaryLogClient.java similarity index 96% rename from dbsyncer-listener/src/main/java/org/dbsyncer/listener/mysql/BinaryLogClient.java rename to dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/BinaryLogClient.java index 91bc508c..e9cb6ecb 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/mysql/BinaryLogClient.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/BinaryLogClient.java @@ -1,117 +1,117 @@ -package org.dbsyncer.listener.mysql; - -import com.github.shyiko.mysql.binlog.event.EventType; -import com.github.shyiko.mysql.binlog.event.TableMapEventData; -import com.github.shyiko.mysql.binlog.event.deserialization.EventDeserializer; -import com.github.shyiko.mysql.binlog.network.AuthenticationException; -import com.github.shyiko.mysql.binlog.network.ServerException; - -import java.util.Map; - -public interface BinaryLogClient { - - /** - * Connect to the replication stream in a separate thread. - * - * @throws AuthenticationException if authentication fails - * @throws ServerException if MySQL server responds with an error - * @throws Exception if anything goes wrong while trying to connect - */ - void connect() throws Exception; - - /** - * Disconnect from the replication stream. Note that this does not reset binlogFilename/binlogPosition. Calling {@link #connect(int)}} - * again resumes client from where it left off. - */ - void disconnect() throws Exception; - - /** - * @return true if client is connected, false otherwise - */ - boolean isConnected(); - - /** - * Register event listener. Note that multiple event listeners will be called in order they where registered. - */ - void registerEventListener(BinaryLogRemoteClient.EventListener eventListener); - - /** - * Register lifecycle listener. Note that multiple lifecycle listeners will be called in order they where registered. - */ - void registerLifecycleListener(BinaryLogRemoteClient.LifecycleListener lifecycleListener); - - /** - * @return binary log filename, nullable (and null be default). Note that this value is automatically tracked by the client and thus is - * subject to change (in response to {@link EventType#ROTATE}, for example). - * @see #setBinlogFilename(String) - */ - String getBinlogFilename(); - - /** - * @param binlogFilename binary log filename. Special values are: - *
      - *
    • null, which turns on automatic resolution (resulting in the last known binlog and position). This is what - * happens by default when you don't specify binary log filename explicitly.
      • - *
    • "" (empty string), which instructs server to stream events starting from the oldest known binlog.
      • - *
    - * @see #getBinlogFilename() - */ - void setBinlogFilename(String binlogFilename); - - /** - * @return binary log position of the next event, 4 by default (which is a position of first event). Note that this value changes with - * each incoming event. - * @see #setBinlogPosition(long) - */ - long getBinlogPosition(); - - /** - * @param binlogPosition binary log position. Any value less than 4 gets automatically adjusted to 4 on connect. - * @see #getBinlogPosition() - */ - void setBinlogPosition(long binlogPosition); - - /** - * @return event deserializer - * @see #setEventDeserializer(EventDeserializer) - */ - EventDeserializer getEventDeserializer(); - - /** - * @param eventDeserializer custom event deserializer - */ - void setEventDeserializer(EventDeserializer eventDeserializer); - - /** - * @return tableMapEventByTableId - */ - Map getTableMapEventByTableId(); - - /** - * @param tableMapEventByTableId tableMapEventMetadata - */ - void setTableMapEventByTableId(Map tableMapEventByTableId); - - /** - * 是否支持ddl - * - * @return - */ - boolean isEnableDDL(); - - /** - *

    true: ROTATE > FORMAT_DESCRIPTION > TABLE_MAP > WRITE_ROWS > UPDATE_ROWS > DELETE_ROWS > XID - *

    false: Support all events - * - * @param enableDDL - */ - void setEnableDDL(boolean enableDDL); - - /** - * binlog-parser-127.0.0.1_3306_1 - * - * @return workerThreadName - */ - String getWorkerThreadName(); - +package org.dbsyncer.connector.mysql; + +import com.github.shyiko.mysql.binlog.event.EventType; +import com.github.shyiko.mysql.binlog.event.TableMapEventData; +import com.github.shyiko.mysql.binlog.event.deserialization.EventDeserializer; +import com.github.shyiko.mysql.binlog.network.AuthenticationException; +import com.github.shyiko.mysql.binlog.network.ServerException; + +import java.util.Map; + +public interface BinaryLogClient { + + /** + * Connect to the replication stream in a separate thread. + * + * @throws AuthenticationException if authentication fails + * @throws ServerException if MySQL server responds with an error + * @throws Exception if anything goes wrong while trying to connect + */ + void connect() throws Exception; + + /** + * Disconnect from the replication stream. Note that this does not reset binlogFilename/binlogPosition. Calling {@link #connect(int)}} + * again resumes client from where it left off. + */ + void disconnect() throws Exception; + + /** + * @return true if client is connected, false otherwise + */ + boolean isConnected(); + + /** + * Register event listener. Note that multiple event listeners will be called in order they where registered. + */ + void registerEventListener(BinaryLogRemoteClient.EventListener eventListener); + + /** + * Register lifecycle listener. Note that multiple lifecycle listeners will be called in order they where registered. + */ + void registerLifecycleListener(BinaryLogRemoteClient.LifecycleListener lifecycleListener); + + /** + * @return binary log filename, nullable (and null be default). Note that this value is automatically tracked by the client and thus is + * subject to change (in response to {@link EventType#ROTATE}, for example). + * @see #setBinlogFilename(String) + */ + String getBinlogFilename(); + + /** + * @param binlogFilename binary log filename. Special values are: + *

      + *
    • null, which turns on automatic resolution (resulting in the last known binlog and position). This is what + * happens by default when you don't specify binary log filename explicitly.
      • + *
    • "" (empty string), which instructs server to stream events starting from the oldest known binlog.
      • + *
    + * @see #getBinlogFilename() + */ + void setBinlogFilename(String binlogFilename); + + /** + * @return binary log position of the next event, 4 by default (which is a position of first event). Note that this value changes with + * each incoming event. + * @see #setBinlogPosition(long) + */ + long getBinlogPosition(); + + /** + * @param binlogPosition binary log position. Any value less than 4 gets automatically adjusted to 4 on connect. + * @see #getBinlogPosition() + */ + void setBinlogPosition(long binlogPosition); + + /** + * @return event deserializer + * @see #setEventDeserializer(EventDeserializer) + */ + EventDeserializer getEventDeserializer(); + + /** + * @param eventDeserializer custom event deserializer + */ + void setEventDeserializer(EventDeserializer eventDeserializer); + + /** + * @return tableMapEventByTableId + */ + Map getTableMapEventByTableId(); + + /** + * @param tableMapEventByTableId tableMapEventMetadata + */ + void setTableMapEventByTableId(Map tableMapEventByTableId); + + /** + * 是否支持ddl + * + * @return + */ + boolean isEnableDDL(); + + /** + *

    true: ROTATE > FORMAT_DESCRIPTION > TABLE_MAP > WRITE_ROWS > UPDATE_ROWS > DELETE_ROWS > XID + *

    false: Support all events + * + * @param enableDDL + */ + void setEnableDDL(boolean enableDDL); + + /** + * binlog-parser-127.0.0.1_3306_1 + * + * @return workerThreadName + */ + String getWorkerThreadName(); + } \ No newline at end of file diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/mysql/BinaryLogRemoteClient.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/BinaryLogRemoteClient.java similarity index 96% rename from dbsyncer-listener/src/main/java/org/dbsyncer/listener/mysql/BinaryLogRemoteClient.java rename to dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/BinaryLogRemoteClient.java index 637c7122..03565dc3 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/mysql/BinaryLogRemoteClient.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/BinaryLogRemoteClient.java @@ -1,837 +1,837 @@ -package org.dbsyncer.listener.mysql; - -import com.github.shyiko.mysql.binlog.GtidSet; -import com.github.shyiko.mysql.binlog.MariadbGtidSet; -import com.github.shyiko.mysql.binlog.event.*; -import com.github.shyiko.mysql.binlog.event.deserialization.*; -import com.github.shyiko.mysql.binlog.io.ByteArrayInputStream; -import com.github.shyiko.mysql.binlog.network.*; -import com.github.shyiko.mysql.binlog.network.protocol.*; -import com.github.shyiko.mysql.binlog.network.protocol.command.*; -import org.dbsyncer.listener.ListenerException; -import org.dbsyncer.listener.mysql.deserializer.DeleteDeserializer; -import org.dbsyncer.listener.mysql.deserializer.UpdateDeserializer; -import org.dbsyncer.listener.mysql.deserializer.WriteDeserializer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.net.ssl.SSLContext; -import javax.net.ssl.TrustManager; -import javax.net.ssl.X509TrustManager; -import java.io.EOFException; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.net.ServerSocket; -import java.net.Socket; -import java.net.SocketException; -import java.security.GeneralSecurityException; -import java.security.cert.X509Certificate; -import java.util.*; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; - -public class BinaryLogRemoteClient implements BinaryLogClient { - - private final Logger logger = LoggerFactory.getLogger(getClass()); - - private static final SSLSocketFactory DEFAULT_REQUIRED_SSL_MODE_SOCKET_FACTORY = new DefaultSSLSocketFactory() { - - @Override - protected void initSSLContext(SSLContext sc) throws GeneralSecurityException { - sc.init(null, new TrustManager[]{ - new X509TrustManager() { - - @Override - public void checkClientTrusted(X509Certificate[] x509Certificates, String s) { - } - - @Override - public void checkServerTrusted(X509Certificate[] x509Certificates, String s) { - } - - @Override - public X509Certificate[] getAcceptedIssuers() { - return new X509Certificate[0]; - } - } - }, null); - } - }; - private static final SSLSocketFactory DEFAULT_VERIFY_CA_SSL_MODE_SOCKET_FACTORY = new DefaultSSLSocketFactory(); - - // https://dev.mysql.com/doc/internals/en/sending-more-than-16mbyte.html - private static final int MAX_PACKET_LENGTH = 16777215; - - private final String hostname; - private final int port; - private final String schema; - private final String username; - private final String password; - private SSLMode sslMode = SSLMode.DISABLED; - - private EventDeserializer eventDeserializer; - private Map tableMapEventByTableId; - private boolean blocking = true; - private boolean enableDDL = false; - private long serverId = 65535; - private volatile String binlogFilename; - private volatile long binlogPosition = 4; - private volatile long connectionId; - private volatile PacketChannel channel; - private volatile boolean connected; - private Thread worker; - private Thread keepAlive; - private String workerThreadName; - - private final Lock connectLock = new ReentrantLock(); - private boolean gtidEnabled = false; - private final Object gtidSetAccessLock = new Object(); - private GtidSet gtidSet; - private String gtid; - private boolean tx; - private boolean gtidSetFallbackToPurged; - private boolean useBinlogFilenamePositionInGtidMode; - private Boolean isMariaDB; - - private final List eventListeners = new CopyOnWriteArrayList<>(); - private final List lifecycleListeners = new CopyOnWriteArrayList<>(); - - /** - * Alias for BinaryLogRemoteClient(hostname, port, <no schema> = null, username, password). - * - * @see BinaryLogRemoteClient#BinaryLogRemoteClient(String, int, String, String, String, long) - */ - public BinaryLogRemoteClient(String hostname, int port, String username, String password) throws IOException { - this(hostname, port, null, username, password, 0L); - } - - /** - * @param hostname mysql server hostname - * @param port mysql server port - * @param schema database name, nullable. Note that this parameter has nothing to do with event filtering. It's used only during the - * authentication. - * @param username login name - * @param password password - * @param serverId serverId - */ - public BinaryLogRemoteClient(String hostname, int port, String schema, String username, String password, long serverId) throws IOException { - this.hostname = hostname; - this.port = port; - this.schema = schema; - this.username = username; - this.password = password; - this.serverId = randomPort(serverId); - } - - @Override - public void connect() throws Exception { - try { - connectLock.lock(); - if (connected) { - throw new IllegalStateException("BinaryLogRemoteClient is already connected"); - } - setConfig(); - openChannel(); - connected = true; - // new keepalive thread - spawnKeepAliveThread(); - - // dump binary log - requestBinaryLogStream(); - ensureEventDeserializerHasRequiredEDDs(); - - // new listen thread - spawnWorkerThread(); - lifecycleListeners.forEach(listener -> listener.onConnect(this)); - } finally { - connectLock.unlock(); - } - } - - @Override - public void disconnect() throws Exception { - if (connected) { - try { - connectLock.lock(); - closeChannel(channel); - connected = false; - if (null != this.worker && !worker.isInterrupted()) { - this.worker.interrupt(); - this.worker = null; - } - if (null != this.keepAlive && !keepAlive.isInterrupted()) { - this.keepAlive.interrupt(); - this.keepAlive = null; - } - lifecycleListeners.forEach(listener -> listener.onDisconnect(this)); - } finally { - connectLock.unlock(); - } - } - } - - @Override - public boolean isConnected() { - return this.connected; - } - - @Override - public void registerEventListener(BinaryLogRemoteClient.EventListener eventListener) { - eventListeners.add(eventListener); - } - - @Override - public void registerLifecycleListener(BinaryLogRemoteClient.LifecycleListener lifecycleListener) { - lifecycleListeners.add(lifecycleListener); - } - - private String createClientId() { - return new StringBuilder(hostname).append(":").append(port).append("_").append(connectionId).toString(); - } - - private void openChannel() throws IOException { - try { - Socket socket = new Socket(); - socket.connect(new InetSocketAddress(hostname, port), 3000); - channel = new PacketChannel(socket); - if (channel.getInputStream().peek() == -1) { - throw new EOFException(); - } - } catch (IOException e) { - closeChannel(channel); - throw new IOException("Failed to connect to MySQL on " + hostname + ":" + port + ". Please make sure it's running.", e); - } - GreetingPacket greetingPacket = receiveGreeting(channel); - detectMariaDB(greetingPacket); - tryUpgradeToSSL(greetingPacket); - new Authenticator(greetingPacket, channel, schema, username, password).authenticate(); - channel.authenticationComplete(); - - connectionId = greetingPacket.getThreadId(); - if ("".equals(binlogFilename)) { - setupGtidSet(); - } - if (binlogFilename == null) { - fetchBinlogFilenameAndPosition(); - } - if (binlogPosition < 4) { - logger.warn("Binary log position adjusted from {} to {}", binlogPosition, 4); - binlogPosition = 4; - } - ChecksumType checksumType = fetchBinlogChecksum(); - if (checksumType != ChecksumType.NONE) { - confirmSupportOfChecksum(channel, checksumType); - } - synchronized (gtidSetAccessLock) { - String position = gtidSet != null ? gtidSet.toString() : binlogFilename + "/" + binlogPosition; - logger.info("Connected to {}:{} at {} (sid:{}, cid:{})", hostname, port, position, serverId, connectionId); - } - } - - private void detectMariaDB(GreetingPacket packet) { - String serverVersion = packet.getServerVersion(); - if (serverVersion == null) { - return; - } - this.isMariaDB = serverVersion.toLowerCase().contains("mariadb"); - } - - private boolean tryUpgradeToSSL(GreetingPacket greetingPacket) throws IOException { - if (sslMode != SSLMode.DISABLED) { - boolean serverSupportsSSL = (greetingPacket.getServerCapabilities() & ClientCapabilities.SSL) != 0; - if (!serverSupportsSSL && (sslMode == SSLMode.REQUIRED || sslMode == SSLMode.VERIFY_CA || - sslMode == SSLMode.VERIFY_IDENTITY)) { - throw new IOException("MySQL server does not support SSL"); - } - if (serverSupportsSSL) { - SSLRequestCommand sslRequestCommand = new SSLRequestCommand(); - int collation = greetingPacket.getServerCollation(); - sslRequestCommand.setCollation(collation); - channel.write(sslRequestCommand); - SSLSocketFactory sslSocketFactory = sslMode == SSLMode.REQUIRED || sslMode == SSLMode.PREFERRED ? - DEFAULT_REQUIRED_SSL_MODE_SOCKET_FACTORY : - DEFAULT_VERIFY_CA_SSL_MODE_SOCKET_FACTORY; - channel.upgradeToSSL(sslSocketFactory, sslMode == SSLMode.VERIFY_IDENTITY ? new TLSHostnameVerifier() : null); - logger.info("SSL enabled"); - return true; - } - } - return false; - } - - private void listenForEventPackets(final PacketChannel channel) { - ByteArrayInputStream inputStream = channel.getInputStream(); - try { - while (inputStream.peek() != -1) { - int packetLength = inputStream.readInteger(3); - //noinspection ResultOfMethodCallIgnored - // 1 byte for sequence - inputStream.skip(1); - int marker = inputStream.read(); - // 255 error - if (marker == 0xFF) { - ErrorPacket errorPacket = new ErrorPacket(inputStream.read(packetLength - 1)); - throw new ServerException(errorPacket.getErrorMessage(), errorPacket.getErrorCode(), - errorPacket.getSqlState()); - } - // 254 empty - if (marker == 0xFE && !blocking) { - break; - } - Event event; - try { - event = eventDeserializer.nextEvent(packetLength == MAX_PACKET_LENGTH ? - new ByteArrayInputStream(readPacketSplitInChunks(inputStream, packetLength - 1)) : - inputStream); - if (event == null) { - throw new EOFException(); - } - } catch (Exception e) { - Throwable cause = e instanceof EventDataDeserializationException ? e.getCause() : e; - if (cause instanceof EOFException || cause instanceof SocketException) { - throw e; - } - lifecycleListeners.forEach(listener -> listener.onEventDeserializationFailure(this, e)); - continue; - } - if (connected) { - updateGtidSet(event); - notifyEventListeners(event); - updateClientBinlogFilenameAndPosition(event); - } - } - } catch (Exception e) { - lifecycleListeners.forEach(listener -> listener.onCommunicationFailure(this, e)); - } - } - - private void ensureEventDeserializerHasRequiredEDDs() { - ensureEventDataDeserializerIfPresent(EventType.ROTATE, RotateEventDataDeserializer.class); - synchronized (gtidSetAccessLock) { - if (this.gtidEnabled) { - ensureEventDataDeserializerIfPresent(EventType.GTID, GtidEventDataDeserializer.class); - ensureEventDataDeserializerIfPresent(EventType.QUERY, QueryEventDataDeserializer.class); - ensureEventDataDeserializerIfPresent(EventType.ANNOTATE_ROWS, AnnotateRowsEventDataDeserializer.class); - ensureEventDataDeserializerIfPresent(EventType.MARIADB_GTID, MariadbGtidEventDataDeserializer.class); - ensureEventDataDeserializerIfPresent(EventType.MARIADB_GTID_LIST, MariadbGtidListEventDataDeserializer.class); - } - } - } - - protected void checkError(byte[] packet) throws IOException { - if (packet[0] == (byte) 0xFF /* error */) { - byte[] bytes = Arrays.copyOfRange(packet, 1, packet.length); - ErrorPacket errorPacket = new ErrorPacket(bytes); - throw new ServerException(errorPacket.getErrorMessage(), errorPacket.getErrorCode(), - errorPacket.getSqlState()); - } - } - - private GreetingPacket receiveGreeting(final PacketChannel channel) throws IOException { - byte[] initialHandshakePacket = channel.read(); - /* error */ - if (initialHandshakePacket[0] == (byte) 0xFF) { - byte[] bytes = Arrays.copyOfRange(initialHandshakePacket, 1, initialHandshakePacket.length); - ErrorPacket errorPacket = new ErrorPacket(bytes); - throw new ServerException(errorPacket.getErrorMessage(), errorPacket.getErrorCode(), - errorPacket.getSqlState()); - } - return new GreetingPacket(initialHandshakePacket); - } - - private void requestBinaryLogStream() throws IOException { - long serverId = blocking ? this.serverId : 0; // http://bugs.mysql.com/bug.php?id=71178 - if (this.isMariaDB) { - requestBinaryLogStreamMaria(serverId); - return; - } - - requestBinaryLogStreamMysql(serverId); - } - - private void requestBinaryLogStreamMysql(long serverId) throws IOException { - Command dumpBinaryLogCommand; - synchronized (gtidSetAccessLock) { - if (this.gtidEnabled) { - dumpBinaryLogCommand = new DumpBinaryLogGtidCommand(serverId, - useBinlogFilenamePositionInGtidMode ? binlogFilename : "", - useBinlogFilenamePositionInGtidMode ? binlogPosition : 4, - gtidSet); - } else { - dumpBinaryLogCommand = new DumpBinaryLogCommand(serverId, binlogFilename, binlogPosition); - } - } - channel.write(dumpBinaryLogCommand); - } - - private void requestBinaryLogStreamMaria(long serverId) throws IOException { - Command dumpBinaryLogCommand; - synchronized (gtidSetAccessLock) { - if (this.gtidEnabled) { - channel.write(new QueryCommand("SET @mariadb_slave_capability=4")); - checkError(channel.read()); - logger.info(gtidSet.toString()); - channel.write(new QueryCommand("SET @slave_connect_state = '" + gtidSet.toString() + "'")); - checkError(channel.read()); - channel.write(new QueryCommand("SET @slave_gtid_strict_mode = 0")); - checkError(channel.read()); - channel.write(new QueryCommand("SET @slave_gtid_ignore_duplicates = 0")); - checkError(channel.read()); - dumpBinaryLogCommand = new DumpBinaryLogCommand(serverId, "", 0L, false); - } else { - dumpBinaryLogCommand = new DumpBinaryLogCommand(serverId, binlogFilename, binlogPosition); - } - } - channel.write(dumpBinaryLogCommand); - } - - private void ensureEventDataDeserializerIfPresent(EventType eventType, - Class> eventDataDeserializerClass) { - EventDataDeserializer eventDataDeserializer = eventDeserializer.getEventDataDeserializer(eventType); - if (eventDataDeserializer.getClass() != eventDataDeserializerClass && - eventDataDeserializer.getClass() != EventDeserializer.EventDataWrapper.Deserializer.class) { - EventDataDeserializer internalEventDataDeserializer; - try { - internalEventDataDeserializer = eventDataDeserializerClass.newInstance(); - } catch (Exception e) { - throw new RuntimeException(e); - } - eventDeserializer.setEventDataDeserializer(eventType, - new EventDeserializer.EventDataWrapper.Deserializer(internalEventDataDeserializer, - eventDataDeserializer)); - } - } - - private String fetchGtidPurged() throws IOException { - channel.write(new QueryCommand("show global variables like 'gtid_purged'")); - ResultSetRowPacket[] resultSet = readResultSet(); - if (resultSet.length != 0) { - return resultSet[0].getValue(1).toUpperCase(); - } - return ""; - } - - private void setupGtidSet() throws IOException { - if (!this.gtidEnabled) - return; - - synchronized (gtidSetAccessLock) { - if (this.isMariaDB) { - if (gtidSet == null) { - gtidSet = new MariadbGtidSet(""); - } else if (!(gtidSet instanceof MariadbGtidSet)) { - throw new RuntimeException("Connected to MariaDB but given a mysql GTID set!"); - } - } else { - if (gtidSet == null && gtidSetFallbackToPurged) { - gtidSet = new GtidSet(fetchGtidPurged()); - } else if (gtidSet == null) { - gtidSet = new GtidSet(""); - } else if (gtidSet instanceof MariadbGtidSet) { - throw new RuntimeException("Connected to Mysql but given a MariaDB GTID set!"); - } - } - } - - } - - private void fetchBinlogFilenameAndPosition() throws IOException { - channel.write(new QueryCommand("show master status")); - ResultSetRowPacket[] resultSet = readResultSet(); - if (resultSet.length == 0) { - throw new IOException("Failed to determine binlog filename/position"); - } - ResultSetRowPacket resultSetRow = resultSet[0]; - binlogFilename = resultSetRow.getValue(0); - binlogPosition = Long.parseLong(resultSetRow.getValue(1)); - } - - private ChecksumType fetchBinlogChecksum() throws IOException { - channel.write(new QueryCommand("show global variables like 'binlog_checksum'")); - ResultSetRowPacket[] resultSet = readResultSet(); - if (resultSet.length == 0) { - return ChecksumType.NONE; - } - return ChecksumType.valueOf(resultSet[0].getValue(1).toUpperCase()); - } - - private void confirmSupportOfChecksum(final PacketChannel channel, ChecksumType checksumType) throws IOException { - channel.write(new QueryCommand("set @master_binlog_checksum= @@global.binlog_checksum")); - byte[] statementResult = channel.read(); - checkError(statementResult); - eventDeserializer.setChecksumType(checksumType); - } - - private byte[] readPacketSplitInChunks(ByteArrayInputStream inputStream, int packetLength) throws IOException { - byte[] result = inputStream.read(packetLength); - int chunkLength; - do { - chunkLength = inputStream.readInteger(3); - //noinspection ResultOfMethodCallIgnored - inputStream.skip(1); // 1 byte for sequence - result = Arrays.copyOf(result, result.length + chunkLength); - inputStream.fill(result, result.length - chunkLength, chunkLength); - } while (chunkLength == Packet.MAX_LENGTH); - return result; - } - - private void updateClientBinlogFilenameAndPosition(Event event) { - EventHeader eventHeader = event.getHeader(); - EventType eventType = eventHeader.getEventType(); - if (eventType == EventType.ROTATE) { - RotateEventData rotateEventData = (RotateEventData) EventDeserializer.EventDataWrapper.internal(event.getData()); - binlogFilename = rotateEventData.getBinlogFilename(); - binlogPosition = rotateEventData.getBinlogPosition(); - } else - // do not update binlogPosition on TABLE_MAP so that in case of reconnect (using a different instance of - // client) table mapping cache could be reconstructed before hitting row mutation event - if (eventType != EventType.TABLE_MAP && eventHeader instanceof EventHeaderV4) { - EventHeaderV4 trackableEventHeader = (EventHeaderV4) eventHeader; - long nextBinlogPosition = trackableEventHeader.getNextPosition(); - if (nextBinlogPosition > 0) { - binlogPosition = nextBinlogPosition; - } - } - } - - private void updateGtidSet(Event event) { - synchronized (gtidSetAccessLock) { - if (gtidSet == null) { - return; - } - } - EventHeader eventHeader = event.getHeader(); - switch (eventHeader.getEventType()) { - case GTID: - GtidEventData gtidEventData = (GtidEventData) EventDeserializer.EventDataWrapper.internal(event.getData()); - gtid = gtidEventData.getGtid(); - break; - case XID: - commitGtid(); - tx = false; - break; - case QUERY: - QueryEventData queryEventData = (QueryEventData) EventDeserializer.EventDataWrapper.internal(event.getData()); - String sql = queryEventData.getSql(); - if (sql == null) { - break; - } - if ("BEGIN".equals(sql)) { - tx = true; - } else if ("COMMIT".equals(sql) || "ROLLBACK".equals(sql)) { - commitGtid(); - tx = false; - } else if (!tx) { - // auto-commit query, likely DDL - commitGtid(); - } - default: - } - } - - private void commitGtid() { - if (gtid != null) { - synchronized (gtidSetAccessLock) { - gtidSet.add(gtid); - } - } - } - - private ResultSetRowPacket[] readResultSet() throws IOException { - List resultSet = new LinkedList(); - byte[] statementResult = channel.read(); - if (statementResult[0] == (byte) 0xFF /* error */) { - byte[] bytes = Arrays.copyOfRange(statementResult, 1, statementResult.length); - ErrorPacket errorPacket = new ErrorPacket(bytes); - throw new ServerException(errorPacket.getErrorMessage(), errorPacket.getErrorCode(), - errorPacket.getSqlState()); - } - while ((channel.read())[0] != (byte) 0xFE /* eof */) { /* skip */ } - for (byte[] bytes; (bytes = channel.read())[0] != (byte) 0xFE /* eof */; ) { - resultSet.add(new ResultSetRowPacket(bytes)); - } - return resultSet.toArray(new ResultSetRowPacket[0]); - } - - private void closeChannel(final PacketChannel channel) throws IOException { - if (channel != null && channel.isOpen()) { - channel.close(); - } - } - - private void setConfig() { - if (null == tableMapEventByTableId) { - tableMapEventByTableId = new HashMap<>(); - } - - IdentityHashMap eventDataDeserializers = new IdentityHashMap(); - if (null == eventDeserializer) { - this.eventDeserializer = new EventDeserializer(new EventHeaderV4Deserializer(), new NullEventDataDeserializer(), eventDataDeserializers, tableMapEventByTableId); - } - - // Process event priority: RotateEvent > FormatDescriptionEvent > TableMapEvent > RowsEvent > XidEvent - eventDataDeserializers.put(EventType.ROTATE, new RotateEventDataDeserializer()); - eventDataDeserializers.put(EventType.FORMAT_DESCRIPTION, new FormatDescriptionEventDataDeserializer()); - eventDataDeserializers.put(EventType.TABLE_MAP, new TableMapEventDataDeserializer()); - eventDataDeserializers.put(EventType.UPDATE_ROWS, new UpdateRowsEventDataDeserializer(tableMapEventByTableId)); - eventDataDeserializers.put(EventType.WRITE_ROWS, new WriteRowsEventDataDeserializer(tableMapEventByTableId)); - eventDataDeserializers.put(EventType.DELETE_ROWS, new DeleteRowsEventDataDeserializer(tableMapEventByTableId)); - eventDataDeserializers.put(EventType.EXT_WRITE_ROWS, (new WriteDeserializer(tableMapEventByTableId)).setMayContainExtraInformation(true)); - eventDataDeserializers.put(EventType.EXT_UPDATE_ROWS, (new UpdateDeserializer(tableMapEventByTableId)).setMayContainExtraInformation(true)); - eventDataDeserializers.put(EventType.EXT_DELETE_ROWS, (new DeleteDeserializer(tableMapEventByTableId)).setMayContainExtraInformation(true)); - eventDataDeserializers.put(EventType.XID, new XidEventDataDeserializer()); - - if (enableDDL) { - eventDataDeserializers.put(EventType.INTVAR, new IntVarEventDataDeserializer()); - eventDataDeserializers.put(EventType.QUERY, new QueryEventDataDeserializer()); - eventDataDeserializers.put(EventType.ROWS_QUERY, new RowsQueryEventDataDeserializer()); - eventDataDeserializers.put(EventType.GTID, new GtidEventDataDeserializer()); - eventDataDeserializers.put(EventType.PREVIOUS_GTIDS, new PreviousGtidSetDeserializer()); - eventDataDeserializers.put(EventType.XA_PREPARE, new XAPrepareEventDataDeserializer()); - } - - } - - private void notifyEventListeners(Event event) { - if (event.getData() instanceof EventDeserializer.EventDataWrapper) { - event = new Event(event.getHeader(), ((EventDeserializer.EventDataWrapper) event.getData()).getExternal()); - } - for (BinaryLogRemoteClient.EventListener eventListener : eventListeners) { - try { - eventListener.onEvent(event); - } catch (Exception e) { - if (logger.isWarnEnabled()) { - logger.warn(eventListener + " choked on " + event, e); - } - } - } - } - - private long randomPort(long serverId) throws IOException { - if (0 == serverId) { - ServerSocket serverSocket = null; - try { - serverSocket = new ServerSocket(0); - return serverSocket.getLocalPort(); - } finally { - if (null != serverSocket) { - serverSocket.close(); - } - } - } - return serverId; - } - - private void spawnWorkerThread() { - this.worker = new Thread(() -> listenForEventPackets(channel)); - this.worker.setDaemon(false); - this.workerThreadName = new StringBuilder("binlog-parser-").append(createClientId()).toString(); - this.worker.setName(workerThreadName); - this.worker.start(); - } - - private void spawnKeepAliveThread() { - long keepAliveInterval = TimeUnit.SECONDS.toMillis(30); - String clientId = createClientId(); - this.keepAlive = new Thread(() -> { - while (connected) { - try { - Thread.sleep(keepAliveInterval); - } catch (InterruptedException e) { - // expected in case of disconnect - } - boolean connectionLost = false; - try { - channel.write(new PingCommand()); - } catch (IOException e) { - connectionLost = true; - } - if (connectionLost) { - if (connected) { - String error = String.format("keepalive: Trying to restore lost connection to %s", clientId); - logger.info(error); - try { - lifecycleListeners.forEach(listener -> listener.onCommunicationFailure(this, new ListenerException(error))); - } catch (Exception e) { - logger.warn("keepalive error", e); - } - } - break; - } - } - }); - this.keepAlive.setDaemon(false); - this.keepAlive.setName(new StringBuilder("binlog-keepalive-").append(clientId).toString()); - this.keepAlive.start(); - } - - @Override - public String getBinlogFilename() { - return binlogFilename; - } - - @Override - public void setBinlogFilename(String binlogFilename) { - this.binlogFilename = binlogFilename; - } - - @Override - public long getBinlogPosition() { - return binlogPosition; - } - - @Override - public void setBinlogPosition(long binlogPosition) { - this.binlogPosition = binlogPosition; - } - - @Override - public EventDeserializer getEventDeserializer() { - return eventDeserializer; - } - - @Override - public void setEventDeserializer(EventDeserializer eventDeserializer) { - if (eventDeserializer == null) { - throw new IllegalArgumentException("Event deserializer cannot be NULL"); - } - this.eventDeserializer = eventDeserializer; - } - - @Override - public Map getTableMapEventByTableId() { - return tableMapEventByTableId; - } - - @Override - public void setTableMapEventByTableId(Map tableMapEventByTableId) { - this.tableMapEventByTableId = tableMapEventByTableId; - } - - public boolean isEnableDDL() { - return enableDDL; - } - - public void setEnableDDL(boolean enableDDL) { - this.enableDDL = enableDDL; - } - - @Override - public String getWorkerThreadName() { - return workerThreadName; - } - - public SSLMode getSSLMode() { - return sslMode; - } - - public void setSSLMode(SSLMode sslMode) { - if (sslMode == null) { - throw new IllegalArgumentException("SSL mode cannot be NULL"); - } - this.sslMode = sslMode; - } - - /** - * @return GTID set. Note that this value changes with each received GTID event (provided client is in GTID mode). - * @see #setGtidSet(String) - */ - public String getGtidSet() { - synchronized (gtidSetAccessLock) { - return gtidSet != null ? gtidSet.toString() : null; - } - } - - /** - * @param gtidStr GTID set string (can be an empty string). - *

    NOTE #1: Any value but null will switch BinaryLogRemoteClient into a GTID mode (this will also set binlogFilename - * to "" (provided it's null) forcing MySQL to send events starting from the oldest known binlog (keep in mind - * that connection will fail if gtid_purged is anything but empty (unless - * {@link #setGtidSetFallbackToPurged(boolean)} is set to true))). - *

    NOTE #2: GTID set is automatically updated with each incoming GTID event (provided GTID mode is on). - * @see #getGtidSet() - * @see #setGtidSetFallbackToPurged(boolean) - */ - public void setGtidSet(String gtidStr) { - if (gtidStr == null) - return; - - this.gtidEnabled = true; - - if (this.binlogFilename == null) { - this.binlogFilename = ""; - } - synchronized (gtidSetAccessLock) { - if (!gtidStr.equals("")) { - if (MariadbGtidSet.isMariaGtidSet(gtidStr)) { - this.gtidSet = new MariadbGtidSet(gtidStr); - } else { - this.gtidSet = new GtidSet(gtidStr); - } - } - } - } - - /** - * @see #setGtidSetFallbackToPurged(boolean) - */ - public boolean isGtidSetFallbackToPurged() { - return gtidSetFallbackToPurged; - } - - /** - * @param gtidSetFallbackToPurged true if gtid_purged should be used as a fallback when gtidSet is set to "" and MySQL server has purged - * some of the binary logs, false otherwise (default). - */ - public void setGtidSetFallbackToPurged(boolean gtidSetFallbackToPurged) { - this.gtidSetFallbackToPurged = gtidSetFallbackToPurged; - } - - /** - * @see #setUseBinlogFilenamePositionInGtidMode(boolean) - */ - public boolean isUseBinlogFilenamePositionInGtidMode() { - return useBinlogFilenamePositionInGtidMode; - } - - /** - * @param useBinlogFilenamePositionInGtidMode true if MySQL server should start streaming events from a given {@link - * #getBinlogFilename()} and {@link #getBinlogPosition()} instead of "the oldest known - * binlog" when {@link #getGtidSet()} is set, false otherwise (default). - */ - public void setUseBinlogFilenamePositionInGtidMode(boolean useBinlogFilenamePositionInGtidMode) { - this.useBinlogFilenamePositionInGtidMode = useBinlogFilenamePositionInGtidMode; - } - - public interface EventListener { - - void onEvent(Event event); - } - - public interface LifecycleListener { - - /** - * Called once client has successfully logged in but before started to receive binlog events. - */ - void onConnect(BinaryLogRemoteClient client); - - /** - * It's guarantied to be called before {@link #onDisconnect(BinaryLogRemoteClient)}) in case of communication failure. - */ - void onCommunicationFailure(BinaryLogRemoteClient client, Exception ex); - - /** - * Called in case of failed event deserialization. Note this type of error does NOT cause client to disconnect. If you wish to stop - * receiving events you'll need to fire client.disconnect() manually. - */ - void onEventDeserializationFailure(BinaryLogRemoteClient client, Exception ex); - - /** - * Called upon disconnect (regardless of the reason). - */ - void onDisconnect(BinaryLogRemoteClient client); - } - +package org.dbsyncer.connector.mysql; + +import com.github.shyiko.mysql.binlog.GtidSet; +import com.github.shyiko.mysql.binlog.MariadbGtidSet; +import com.github.shyiko.mysql.binlog.event.*; +import com.github.shyiko.mysql.binlog.event.deserialization.*; +import com.github.shyiko.mysql.binlog.io.ByteArrayInputStream; +import com.github.shyiko.mysql.binlog.network.*; +import com.github.shyiko.mysql.binlog.network.protocol.*; +import com.github.shyiko.mysql.binlog.network.protocol.command.*; +import org.dbsyncer.connector.ConnectorException; +import org.dbsyncer.connector.mysql.deserializer.DeleteDeserializer; +import org.dbsyncer.connector.mysql.deserializer.UpdateDeserializer; +import org.dbsyncer.connector.mysql.deserializer.WriteDeserializer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; +import java.io.EOFException; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.net.Socket; +import java.net.SocketException; +import java.security.GeneralSecurityException; +import java.security.cert.X509Certificate; +import java.util.*; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +public class BinaryLogRemoteClient implements BinaryLogClient { + + private final Logger logger = LoggerFactory.getLogger(getClass()); + + private static final SSLSocketFactory DEFAULT_REQUIRED_SSL_MODE_SOCKET_FACTORY = new DefaultSSLSocketFactory() { + + @Override + protected void initSSLContext(SSLContext sc) throws GeneralSecurityException { + sc.init(null, new TrustManager[]{ + new X509TrustManager() { + + @Override + public void checkClientTrusted(X509Certificate[] x509Certificates, String s) { + } + + @Override + public void checkServerTrusted(X509Certificate[] x509Certificates, String s) { + } + + @Override + public X509Certificate[] getAcceptedIssuers() { + return new X509Certificate[0]; + } + } + }, null); + } + }; + private static final SSLSocketFactory DEFAULT_VERIFY_CA_SSL_MODE_SOCKET_FACTORY = new DefaultSSLSocketFactory(); + + // https://dev.mysql.com/doc/internals/en/sending-more-than-16mbyte.html + private static final int MAX_PACKET_LENGTH = 16777215; + + private final String hostname; + private final int port; + private final String schema; + private final String username; + private final String password; + private SSLMode sslMode = SSLMode.DISABLED; + + private EventDeserializer eventDeserializer; + private Map tableMapEventByTableId; + private boolean blocking = true; + private boolean enableDDL = false; + private long serverId = 65535; + private volatile String binlogFilename; + private volatile long binlogPosition = 4; + private volatile long connectionId; + private volatile PacketChannel channel; + private volatile boolean connected; + private Thread worker; + private Thread keepAlive; + private String workerThreadName; + + private final Lock connectLock = new ReentrantLock(); + private boolean gtidEnabled = false; + private final Object gtidSetAccessLock = new Object(); + private GtidSet gtidSet; + private String gtid; + private boolean tx; + private boolean gtidSetFallbackToPurged; + private boolean useBinlogFilenamePositionInGtidMode; + private Boolean isMariaDB; + + private final List eventListeners = new CopyOnWriteArrayList<>(); + private final List lifecycleListeners = new CopyOnWriteArrayList<>(); + + /** + * Alias for BinaryLogRemoteClient(hostname, port, <no schema> = null, username, password). + * + * @see BinaryLogRemoteClient#BinaryLogRemoteClient(String, int, String, String, String, long) + */ + public BinaryLogRemoteClient(String hostname, int port, String username, String password) throws IOException { + this(hostname, port, null, username, password, 0L); + } + + /** + * @param hostname mysql server hostname + * @param port mysql server port + * @param schema database name, nullable. Note that this parameter has nothing to do with event filtering. It's used only during the + * authentication. + * @param username login name + * @param password password + * @param serverId serverId + */ + public BinaryLogRemoteClient(String hostname, int port, String schema, String username, String password, long serverId) throws IOException { + this.hostname = hostname; + this.port = port; + this.schema = schema; + this.username = username; + this.password = password; + this.serverId = randomPort(serverId); + } + + @Override + public void connect() throws Exception { + try { + connectLock.lock(); + if (connected) { + throw new IllegalStateException("BinaryLogRemoteClient is already connected"); + } + setConfig(); + openChannel(); + connected = true; + // new keepalive thread + spawnKeepAliveThread(); + + // dump binary log + requestBinaryLogStream(); + ensureEventDeserializerHasRequiredEDDs(); + + // new listen thread + spawnWorkerThread(); + lifecycleListeners.forEach(listener -> listener.onConnect(this)); + } finally { + connectLock.unlock(); + } + } + + @Override + public void disconnect() throws Exception { + if (connected) { + try { + connectLock.lock(); + closeChannel(channel); + connected = false; + if (null != this.worker && !worker.isInterrupted()) { + this.worker.interrupt(); + this.worker = null; + } + if (null != this.keepAlive && !keepAlive.isInterrupted()) { + this.keepAlive.interrupt(); + this.keepAlive = null; + } + lifecycleListeners.forEach(listener -> listener.onDisconnect(this)); + } finally { + connectLock.unlock(); + } + } + } + + @Override + public boolean isConnected() { + return this.connected; + } + + @Override + public void registerEventListener(BinaryLogRemoteClient.EventListener eventListener) { + eventListeners.add(eventListener); + } + + @Override + public void registerLifecycleListener(BinaryLogRemoteClient.LifecycleListener lifecycleListener) { + lifecycleListeners.add(lifecycleListener); + } + + private String createClientId() { + return new StringBuilder(hostname).append(":").append(port).append("_").append(connectionId).toString(); + } + + private void openChannel() throws IOException { + try { + Socket socket = new Socket(); + socket.connect(new InetSocketAddress(hostname, port), 3000); + channel = new PacketChannel(socket); + if (channel.getInputStream().peek() == -1) { + throw new EOFException(); + } + } catch (IOException e) { + closeChannel(channel); + throw new IOException("Failed to connect to MySQL on " + hostname + ":" + port + ". Please make sure it's running.", e); + } + GreetingPacket greetingPacket = receiveGreeting(channel); + detectMariaDB(greetingPacket); + tryUpgradeToSSL(greetingPacket); + new Authenticator(greetingPacket, channel, schema, username, password).authenticate(); + channel.authenticationComplete(); + + connectionId = greetingPacket.getThreadId(); + if ("".equals(binlogFilename)) { + setupGtidSet(); + } + if (binlogFilename == null) { + fetchBinlogFilenameAndPosition(); + } + if (binlogPosition < 4) { + logger.warn("Binary log position adjusted from {} to {}", binlogPosition, 4); + binlogPosition = 4; + } + ChecksumType checksumType = fetchBinlogChecksum(); + if (checksumType != ChecksumType.NONE) { + confirmSupportOfChecksum(channel, checksumType); + } + synchronized (gtidSetAccessLock) { + String position = gtidSet != null ? gtidSet.toString() : binlogFilename + "/" + binlogPosition; + logger.info("Connected to {}:{} at {} (sid:{}, cid:{})", hostname, port, position, serverId, connectionId); + } + } + + private void detectMariaDB(GreetingPacket packet) { + String serverVersion = packet.getServerVersion(); + if (serverVersion == null) { + return; + } + this.isMariaDB = serverVersion.toLowerCase().contains("mariadb"); + } + + private boolean tryUpgradeToSSL(GreetingPacket greetingPacket) throws IOException { + if (sslMode != SSLMode.DISABLED) { + boolean serverSupportsSSL = (greetingPacket.getServerCapabilities() & ClientCapabilities.SSL) != 0; + if (!serverSupportsSSL && (sslMode == SSLMode.REQUIRED || sslMode == SSLMode.VERIFY_CA || + sslMode == SSLMode.VERIFY_IDENTITY)) { + throw new IOException("MySQL server does not support SSL"); + } + if (serverSupportsSSL) { + SSLRequestCommand sslRequestCommand = new SSLRequestCommand(); + int collation = greetingPacket.getServerCollation(); + sslRequestCommand.setCollation(collation); + channel.write(sslRequestCommand); + SSLSocketFactory sslSocketFactory = sslMode == SSLMode.REQUIRED || sslMode == SSLMode.PREFERRED ? + DEFAULT_REQUIRED_SSL_MODE_SOCKET_FACTORY : + DEFAULT_VERIFY_CA_SSL_MODE_SOCKET_FACTORY; + channel.upgradeToSSL(sslSocketFactory, sslMode == SSLMode.VERIFY_IDENTITY ? new TLSHostnameVerifier() : null); + logger.info("SSL enabled"); + return true; + } + } + return false; + } + + private void listenForEventPackets(final PacketChannel channel) { + ByteArrayInputStream inputStream = channel.getInputStream(); + try { + while (inputStream.peek() != -1) { + int packetLength = inputStream.readInteger(3); + //noinspection ResultOfMethodCallIgnored + // 1 byte for sequence + inputStream.skip(1); + int marker = inputStream.read(); + // 255 error + if (marker == 0xFF) { + ErrorPacket errorPacket = new ErrorPacket(inputStream.read(packetLength - 1)); + throw new ServerException(errorPacket.getErrorMessage(), errorPacket.getErrorCode(), + errorPacket.getSqlState()); + } + // 254 empty + if (marker == 0xFE && !blocking) { + break; + } + Event event; + try { + event = eventDeserializer.nextEvent(packetLength == MAX_PACKET_LENGTH ? + new ByteArrayInputStream(readPacketSplitInChunks(inputStream, packetLength - 1)) : + inputStream); + if (event == null) { + throw new EOFException(); + } + } catch (Exception e) { + Throwable cause = e instanceof EventDataDeserializationException ? e.getCause() : e; + if (cause instanceof EOFException || cause instanceof SocketException) { + throw e; + } + lifecycleListeners.forEach(listener -> listener.onEventDeserializationFailure(this, e)); + continue; + } + if (connected) { + updateGtidSet(event); + notifyEventListeners(event); + updateClientBinlogFilenameAndPosition(event); + } + } + } catch (Exception e) { + lifecycleListeners.forEach(listener -> listener.onCommunicationFailure(this, e)); + } + } + + private void ensureEventDeserializerHasRequiredEDDs() { + ensureEventDataDeserializerIfPresent(EventType.ROTATE, RotateEventDataDeserializer.class); + synchronized (gtidSetAccessLock) { + if (this.gtidEnabled) { + ensureEventDataDeserializerIfPresent(EventType.GTID, GtidEventDataDeserializer.class); + ensureEventDataDeserializerIfPresent(EventType.QUERY, QueryEventDataDeserializer.class); + ensureEventDataDeserializerIfPresent(EventType.ANNOTATE_ROWS, AnnotateRowsEventDataDeserializer.class); + ensureEventDataDeserializerIfPresent(EventType.MARIADB_GTID, MariadbGtidEventDataDeserializer.class); + ensureEventDataDeserializerIfPresent(EventType.MARIADB_GTID_LIST, MariadbGtidListEventDataDeserializer.class); + } + } + } + + protected void checkError(byte[] packet) throws IOException { + if (packet[0] == (byte) 0xFF /* error */) { + byte[] bytes = Arrays.copyOfRange(packet, 1, packet.length); + ErrorPacket errorPacket = new ErrorPacket(bytes); + throw new ServerException(errorPacket.getErrorMessage(), errorPacket.getErrorCode(), + errorPacket.getSqlState()); + } + } + + private GreetingPacket receiveGreeting(final PacketChannel channel) throws IOException { + byte[] initialHandshakePacket = channel.read(); + /* error */ + if (initialHandshakePacket[0] == (byte) 0xFF) { + byte[] bytes = Arrays.copyOfRange(initialHandshakePacket, 1, initialHandshakePacket.length); + ErrorPacket errorPacket = new ErrorPacket(bytes); + throw new ServerException(errorPacket.getErrorMessage(), errorPacket.getErrorCode(), + errorPacket.getSqlState()); + } + return new GreetingPacket(initialHandshakePacket); + } + + private void requestBinaryLogStream() throws IOException { + long serverId = blocking ? this.serverId : 0; // http://bugs.mysql.com/bug.php?id=71178 + if (this.isMariaDB) { + requestBinaryLogStreamMaria(serverId); + return; + } + + requestBinaryLogStreamMysql(serverId); + } + + private void requestBinaryLogStreamMysql(long serverId) throws IOException { + Command dumpBinaryLogCommand; + synchronized (gtidSetAccessLock) { + if (this.gtidEnabled) { + dumpBinaryLogCommand = new DumpBinaryLogGtidCommand(serverId, + useBinlogFilenamePositionInGtidMode ? binlogFilename : "", + useBinlogFilenamePositionInGtidMode ? binlogPosition : 4, + gtidSet); + } else { + dumpBinaryLogCommand = new DumpBinaryLogCommand(serverId, binlogFilename, binlogPosition); + } + } + channel.write(dumpBinaryLogCommand); + } + + private void requestBinaryLogStreamMaria(long serverId) throws IOException { + Command dumpBinaryLogCommand; + synchronized (gtidSetAccessLock) { + if (this.gtidEnabled) { + channel.write(new QueryCommand("SET @mariadb_slave_capability=4")); + checkError(channel.read()); + logger.info(gtidSet.toString()); + channel.write(new QueryCommand("SET @slave_connect_state = '" + gtidSet.toString() + "'")); + checkError(channel.read()); + channel.write(new QueryCommand("SET @slave_gtid_strict_mode = 0")); + checkError(channel.read()); + channel.write(new QueryCommand("SET @slave_gtid_ignore_duplicates = 0")); + checkError(channel.read()); + dumpBinaryLogCommand = new DumpBinaryLogCommand(serverId, "", 0L, false); + } else { + dumpBinaryLogCommand = new DumpBinaryLogCommand(serverId, binlogFilename, binlogPosition); + } + } + channel.write(dumpBinaryLogCommand); + } + + private void ensureEventDataDeserializerIfPresent(EventType eventType, + Class> eventDataDeserializerClass) { + EventDataDeserializer eventDataDeserializer = eventDeserializer.getEventDataDeserializer(eventType); + if (eventDataDeserializer.getClass() != eventDataDeserializerClass && + eventDataDeserializer.getClass() != EventDeserializer.EventDataWrapper.Deserializer.class) { + EventDataDeserializer internalEventDataDeserializer; + try { + internalEventDataDeserializer = eventDataDeserializerClass.newInstance(); + } catch (Exception e) { + throw new RuntimeException(e); + } + eventDeserializer.setEventDataDeserializer(eventType, + new EventDeserializer.EventDataWrapper.Deserializer(internalEventDataDeserializer, + eventDataDeserializer)); + } + } + + private String fetchGtidPurged() throws IOException { + channel.write(new QueryCommand("show global variables like 'gtid_purged'")); + ResultSetRowPacket[] resultSet = readResultSet(); + if (resultSet.length != 0) { + return resultSet[0].getValue(1).toUpperCase(); + } + return ""; + } + + private void setupGtidSet() throws IOException { + if (!this.gtidEnabled) + return; + + synchronized (gtidSetAccessLock) { + if (this.isMariaDB) { + if (gtidSet == null) { + gtidSet = new MariadbGtidSet(""); + } else if (!(gtidSet instanceof MariadbGtidSet)) { + throw new RuntimeException("Connected to MariaDB but given a mysql GTID set!"); + } + } else { + if (gtidSet == null && gtidSetFallbackToPurged) { + gtidSet = new GtidSet(fetchGtidPurged()); + } else if (gtidSet == null) { + gtidSet = new GtidSet(""); + } else if (gtidSet instanceof MariadbGtidSet) { + throw new RuntimeException("Connected to Mysql but given a MariaDB GTID set!"); + } + } + } + + } + + private void fetchBinlogFilenameAndPosition() throws IOException { + channel.write(new QueryCommand("show master status")); + ResultSetRowPacket[] resultSet = readResultSet(); + if (resultSet.length == 0) { + throw new IOException("Failed to determine binlog filename/position"); + } + ResultSetRowPacket resultSetRow = resultSet[0]; + binlogFilename = resultSetRow.getValue(0); + binlogPosition = Long.parseLong(resultSetRow.getValue(1)); + } + + private ChecksumType fetchBinlogChecksum() throws IOException { + channel.write(new QueryCommand("show global variables like 'binlog_checksum'")); + ResultSetRowPacket[] resultSet = readResultSet(); + if (resultSet.length == 0) { + return ChecksumType.NONE; + } + return ChecksumType.valueOf(resultSet[0].getValue(1).toUpperCase()); + } + + private void confirmSupportOfChecksum(final PacketChannel channel, ChecksumType checksumType) throws IOException { + channel.write(new QueryCommand("set @master_binlog_checksum= @@global.binlog_checksum")); + byte[] statementResult = channel.read(); + checkError(statementResult); + eventDeserializer.setChecksumType(checksumType); + } + + private byte[] readPacketSplitInChunks(ByteArrayInputStream inputStream, int packetLength) throws IOException { + byte[] result = inputStream.read(packetLength); + int chunkLength; + do { + chunkLength = inputStream.readInteger(3); + //noinspection ResultOfMethodCallIgnored + inputStream.skip(1); // 1 byte for sequence + result = Arrays.copyOf(result, result.length + chunkLength); + inputStream.fill(result, result.length - chunkLength, chunkLength); + } while (chunkLength == Packet.MAX_LENGTH); + return result; + } + + private void updateClientBinlogFilenameAndPosition(Event event) { + EventHeader eventHeader = event.getHeader(); + EventType eventType = eventHeader.getEventType(); + if (eventType == EventType.ROTATE) { + RotateEventData rotateEventData = (RotateEventData) EventDeserializer.EventDataWrapper.internal(event.getData()); + binlogFilename = rotateEventData.getBinlogFilename(); + binlogPosition = rotateEventData.getBinlogPosition(); + } else + // do not update binlogPosition on TABLE_MAP so that in case of reconnect (using a different instance of + // client) table mapping cache could be reconstructed before hitting row mutation event + if (eventType != EventType.TABLE_MAP && eventHeader instanceof EventHeaderV4) { + EventHeaderV4 trackableEventHeader = (EventHeaderV4) eventHeader; + long nextBinlogPosition = trackableEventHeader.getNextPosition(); + if (nextBinlogPosition > 0) { + binlogPosition = nextBinlogPosition; + } + } + } + + private void updateGtidSet(Event event) { + synchronized (gtidSetAccessLock) { + if (gtidSet == null) { + return; + } + } + EventHeader eventHeader = event.getHeader(); + switch (eventHeader.getEventType()) { + case GTID: + GtidEventData gtidEventData = (GtidEventData) EventDeserializer.EventDataWrapper.internal(event.getData()); + gtid = gtidEventData.getGtid(); + break; + case XID: + commitGtid(); + tx = false; + break; + case QUERY: + QueryEventData queryEventData = (QueryEventData) EventDeserializer.EventDataWrapper.internal(event.getData()); + String sql = queryEventData.getSql(); + if (sql == null) { + break; + } + if ("BEGIN".equals(sql)) { + tx = true; + } else if ("COMMIT".equals(sql) || "ROLLBACK".equals(sql)) { + commitGtid(); + tx = false; + } else if (!tx) { + // auto-commit query, likely DDL + commitGtid(); + } + default: + } + } + + private void commitGtid() { + if (gtid != null) { + synchronized (gtidSetAccessLock) { + gtidSet.add(gtid); + } + } + } + + private ResultSetRowPacket[] readResultSet() throws IOException { + List resultSet = new LinkedList(); + byte[] statementResult = channel.read(); + if (statementResult[0] == (byte) 0xFF /* error */) { + byte[] bytes = Arrays.copyOfRange(statementResult, 1, statementResult.length); + ErrorPacket errorPacket = new ErrorPacket(bytes); + throw new ServerException(errorPacket.getErrorMessage(), errorPacket.getErrorCode(), + errorPacket.getSqlState()); + } + while ((channel.read())[0] != (byte) 0xFE /* eof */) { /* skip */ } + for (byte[] bytes; (bytes = channel.read())[0] != (byte) 0xFE /* eof */; ) { + resultSet.add(new ResultSetRowPacket(bytes)); + } + return resultSet.toArray(new ResultSetRowPacket[0]); + } + + private void closeChannel(final PacketChannel channel) throws IOException { + if (channel != null && channel.isOpen()) { + channel.close(); + } + } + + private void setConfig() { + if (null == tableMapEventByTableId) { + tableMapEventByTableId = new HashMap<>(); + } + + IdentityHashMap eventDataDeserializers = new IdentityHashMap(); + if (null == eventDeserializer) { + this.eventDeserializer = new EventDeserializer(new EventHeaderV4Deserializer(), new NullEventDataDeserializer(), eventDataDeserializers, tableMapEventByTableId); + } + + // Process event priority: RotateEvent > FormatDescriptionEvent > TableMapEvent > RowsEvent > XidEvent + eventDataDeserializers.put(EventType.ROTATE, new RotateEventDataDeserializer()); + eventDataDeserializers.put(EventType.FORMAT_DESCRIPTION, new FormatDescriptionEventDataDeserializer()); + eventDataDeserializers.put(EventType.TABLE_MAP, new TableMapEventDataDeserializer()); + eventDataDeserializers.put(EventType.UPDATE_ROWS, new UpdateRowsEventDataDeserializer(tableMapEventByTableId)); + eventDataDeserializers.put(EventType.WRITE_ROWS, new WriteRowsEventDataDeserializer(tableMapEventByTableId)); + eventDataDeserializers.put(EventType.DELETE_ROWS, new DeleteRowsEventDataDeserializer(tableMapEventByTableId)); + eventDataDeserializers.put(EventType.EXT_WRITE_ROWS, (new WriteDeserializer(tableMapEventByTableId)).setMayContainExtraInformation(true)); + eventDataDeserializers.put(EventType.EXT_UPDATE_ROWS, (new UpdateDeserializer(tableMapEventByTableId)).setMayContainExtraInformation(true)); + eventDataDeserializers.put(EventType.EXT_DELETE_ROWS, (new DeleteDeserializer(tableMapEventByTableId)).setMayContainExtraInformation(true)); + eventDataDeserializers.put(EventType.XID, new XidEventDataDeserializer()); + + if (enableDDL) { + eventDataDeserializers.put(EventType.INTVAR, new IntVarEventDataDeserializer()); + eventDataDeserializers.put(EventType.QUERY, new QueryEventDataDeserializer()); + eventDataDeserializers.put(EventType.ROWS_QUERY, new RowsQueryEventDataDeserializer()); + eventDataDeserializers.put(EventType.GTID, new GtidEventDataDeserializer()); + eventDataDeserializers.put(EventType.PREVIOUS_GTIDS, new PreviousGtidSetDeserializer()); + eventDataDeserializers.put(EventType.XA_PREPARE, new XAPrepareEventDataDeserializer()); + } + + } + + private void notifyEventListeners(Event event) { + if (event.getData() instanceof EventDeserializer.EventDataWrapper) { + event = new Event(event.getHeader(), ((EventDeserializer.EventDataWrapper) event.getData()).getExternal()); + } + for (BinaryLogRemoteClient.EventListener eventListener : eventListeners) { + try { + eventListener.onEvent(event); + } catch (Exception e) { + if (logger.isWarnEnabled()) { + logger.warn(eventListener + " choked on " + event, e); + } + } + } + } + + private long randomPort(long serverId) throws IOException { + if (0 == serverId) { + ServerSocket serverSocket = null; + try { + serverSocket = new ServerSocket(0); + return serverSocket.getLocalPort(); + } finally { + if (null != serverSocket) { + serverSocket.close(); + } + } + } + return serverId; + } + + private void spawnWorkerThread() { + this.worker = new Thread(() -> listenForEventPackets(channel)); + this.worker.setDaemon(false); + this.workerThreadName = new StringBuilder("binlog-parser-").append(createClientId()).toString(); + this.worker.setName(workerThreadName); + this.worker.start(); + } + + private void spawnKeepAliveThread() { + long keepAliveInterval = TimeUnit.SECONDS.toMillis(30); + String clientId = createClientId(); + this.keepAlive = new Thread(() -> { + while (connected) { + try { + Thread.sleep(keepAliveInterval); + } catch (InterruptedException e) { + // expected in case of disconnect + } + boolean connectionLost = false; + try { + channel.write(new PingCommand()); + } catch (IOException e) { + connectionLost = true; + } + if (connectionLost) { + if (connected) { + String error = String.format("keepalive: Trying to restore lost connection to %s", clientId); + logger.info(error); + try { + lifecycleListeners.forEach(listener -> listener.onCommunicationFailure(this, new ConnectorException(error))); + } catch (Exception e) { + logger.warn("keepalive error", e); + } + } + break; + } + } + }); + this.keepAlive.setDaemon(false); + this.keepAlive.setName(new StringBuilder("binlog-keepalive-").append(clientId).toString()); + this.keepAlive.start(); + } + + @Override + public String getBinlogFilename() { + return binlogFilename; + } + + @Override + public void setBinlogFilename(String binlogFilename) { + this.binlogFilename = binlogFilename; + } + + @Override + public long getBinlogPosition() { + return binlogPosition; + } + + @Override + public void setBinlogPosition(long binlogPosition) { + this.binlogPosition = binlogPosition; + } + + @Override + public EventDeserializer getEventDeserializer() { + return eventDeserializer; + } + + @Override + public void setEventDeserializer(EventDeserializer eventDeserializer) { + if (eventDeserializer == null) { + throw new IllegalArgumentException("Event deserializer cannot be NULL"); + } + this.eventDeserializer = eventDeserializer; + } + + @Override + public Map getTableMapEventByTableId() { + return tableMapEventByTableId; + } + + @Override + public void setTableMapEventByTableId(Map tableMapEventByTableId) { + this.tableMapEventByTableId = tableMapEventByTableId; + } + + public boolean isEnableDDL() { + return enableDDL; + } + + public void setEnableDDL(boolean enableDDL) { + this.enableDDL = enableDDL; + } + + @Override + public String getWorkerThreadName() { + return workerThreadName; + } + + public SSLMode getSSLMode() { + return sslMode; + } + + public void setSSLMode(SSLMode sslMode) { + if (sslMode == null) { + throw new IllegalArgumentException("SSL mode cannot be NULL"); + } + this.sslMode = sslMode; + } + + /** + * @return GTID set. Note that this value changes with each received GTID event (provided client is in GTID mode). + * @see #setGtidSet(String) + */ + public String getGtidSet() { + synchronized (gtidSetAccessLock) { + return gtidSet != null ? gtidSet.toString() : null; + } + } + + /** + * @param gtidStr GTID set string (can be an empty string). + *

    NOTE #1: Any value but null will switch BinaryLogRemoteClient into a GTID mode (this will also set binlogFilename + * to "" (provided it's null) forcing MySQL to send events starting from the oldest known binlog (keep in mind + * that connection will fail if gtid_purged is anything but empty (unless + * {@link #setGtidSetFallbackToPurged(boolean)} is set to true))). + *

    NOTE #2: GTID set is automatically updated with each incoming GTID event (provided GTID mode is on). + * @see #getGtidSet() + * @see #setGtidSetFallbackToPurged(boolean) + */ + public void setGtidSet(String gtidStr) { + if (gtidStr == null) + return; + + this.gtidEnabled = true; + + if (this.binlogFilename == null) { + this.binlogFilename = ""; + } + synchronized (gtidSetAccessLock) { + if (!gtidStr.equals("")) { + if (MariadbGtidSet.isMariaGtidSet(gtidStr)) { + this.gtidSet = new MariadbGtidSet(gtidStr); + } else { + this.gtidSet = new GtidSet(gtidStr); + } + } + } + } + + /** + * @see #setGtidSetFallbackToPurged(boolean) + */ + public boolean isGtidSetFallbackToPurged() { + return gtidSetFallbackToPurged; + } + + /** + * @param gtidSetFallbackToPurged true if gtid_purged should be used as a fallback when gtidSet is set to "" and MySQL server has purged + * some of the binary logs, false otherwise (default). + */ + public void setGtidSetFallbackToPurged(boolean gtidSetFallbackToPurged) { + this.gtidSetFallbackToPurged = gtidSetFallbackToPurged; + } + + /** + * @see #setUseBinlogFilenamePositionInGtidMode(boolean) + */ + public boolean isUseBinlogFilenamePositionInGtidMode() { + return useBinlogFilenamePositionInGtidMode; + } + + /** + * @param useBinlogFilenamePositionInGtidMode true if MySQL server should start streaming events from a given {@link + * #getBinlogFilename()} and {@link #getBinlogPosition()} instead of "the oldest known + * binlog" when {@link #getGtidSet()} is set, false otherwise (default). + */ + public void setUseBinlogFilenamePositionInGtidMode(boolean useBinlogFilenamePositionInGtidMode) { + this.useBinlogFilenamePositionInGtidMode = useBinlogFilenamePositionInGtidMode; + } + + public interface EventListener { + + void onEvent(Event event); + } + + public interface LifecycleListener { + + /** + * Called once client has successfully logged in but before started to receive binlog events. + */ + void onConnect(BinaryLogRemoteClient client); + + /** + * It's guarantied to be called before {@link #onDisconnect(BinaryLogRemoteClient)}) in case of communication failure. + */ + void onCommunicationFailure(BinaryLogRemoteClient client, Exception ex); + + /** + * Called in case of failed event deserialization. Note this type of error does NOT cause client to disconnect. If you wish to stop + * receiving events you'll need to fire client.disconnect() manually. + */ + void onEventDeserializationFailure(BinaryLogRemoteClient client, Exception ex); + + /** + * Called upon disconnect (regardless of the reason). + */ + void onDisconnect(BinaryLogRemoteClient client); + } + } \ No newline at end of file diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/mysql/DqlMySQLExtractor.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/DqlMySQLListener.java similarity index 68% rename from dbsyncer-listener/src/main/java/org/dbsyncer/listener/mysql/DqlMySQLExtractor.java rename to dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/DqlMySQLListener.java index ee73eb87..6566010e 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/mysql/DqlMySQLExtractor.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/DqlMySQLListener.java @@ -1,13 +1,13 @@ -package org.dbsyncer.listener.mysql; +package org.dbsyncer.connector.mysql; -import org.dbsyncer.listener.ChangedEvent; +import org.dbsyncer.sdk.listener.ChangedEvent; /** * @author AE86 * @version 1.0.0 * @date 2022/5/28 22:02 */ -public class DqlMySQLExtractor extends MySQLExtractor { +public class DqlMySQLListener extends MySQLListener { @Override public void start() { diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/MySQLConnector.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/MySQLConnector.java index 4382eea1..81dd3b82 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/MySQLConnector.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/MySQLConnector.java @@ -1,9 +1,12 @@ package org.dbsyncer.connector.mysql; import org.dbsyncer.common.util.StringUtil; +import org.dbsyncer.connector.quartz.DatabaseQuartzListener; import org.dbsyncer.sdk.config.ReaderConfig; -import org.dbsyncer.sdk.constant.DatabaseConstant; import org.dbsyncer.sdk.connector.database.AbstractDatabaseConnector; +import org.dbsyncer.sdk.constant.DatabaseConstant; +import org.dbsyncer.sdk.enums.ListenerTypeEnum; +import org.dbsyncer.sdk.listener.Listener; import org.dbsyncer.sdk.model.PageSql; import org.dbsyncer.sdk.util.PrimaryKeyUtil; import org.slf4j.Logger; @@ -24,6 +27,18 @@ public final class MySQLConnector extends AbstractDatabaseConnector { return TYPE; } + @Override + public Listener getListener(String listenerType) { + if (ListenerTypeEnum.isTiming(listenerType)) { + return new DatabaseQuartzListener(); + } + + if (ListenerTypeEnum.isLog(listenerType)) { + return new MySQLListener(); + } + return null; + } + @Override public String buildSqlWithQuotation() { return "`"; diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/mysql/MySQLExtractor.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/MySQLListener.java similarity index 91% rename from dbsyncer-listener/src/main/java/org/dbsyncer/listener/mysql/MySQLExtractor.java rename to dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/MySQLListener.java index 3c21070e..dd172696 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/mysql/MySQLExtractor.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/MySQLListener.java @@ -1,342 +1,359 @@ -package org.dbsyncer.listener.mysql; - -import com.github.shyiko.mysql.binlog.event.DeleteRowsEventData; -import com.github.shyiko.mysql.binlog.event.Event; -import com.github.shyiko.mysql.binlog.event.EventHeader; -import com.github.shyiko.mysql.binlog.event.EventHeaderV4; -import com.github.shyiko.mysql.binlog.event.EventType; -import com.github.shyiko.mysql.binlog.event.QueryEventData; -import com.github.shyiko.mysql.binlog.event.RotateEventData; -import com.github.shyiko.mysql.binlog.event.TableMapEventData; -import com.github.shyiko.mysql.binlog.event.UpdateRowsEventData; -import com.github.shyiko.mysql.binlog.event.WriteRowsEventData; -import com.github.shyiko.mysql.binlog.network.ServerException; -import net.sf.jsqlparser.JSQLParserException; -import net.sf.jsqlparser.parser.CCJSqlParserUtil; -import net.sf.jsqlparser.statement.alter.Alter; -import org.dbsyncer.listener.model.ChangedOffset; -import org.dbsyncer.listener.event.DDLChangedEvent; -import org.dbsyncer.listener.event.RowChangedEvent; -import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.sdk.config.DatabaseConfig; -import org.dbsyncer.sdk.constant.ConnectorConstant; -import org.dbsyncer.sdk.util.DatabaseUtil; -import org.dbsyncer.listener.AbstractDatabaseExtractor; -import org.dbsyncer.listener.ListenerException; -import org.dbsyncer.listener.config.Host; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.util.Assert; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; -import java.util.regex.Matcher; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static java.util.regex.Pattern.compile; - -/** - * @version 1.0.0 - * @Author AE86 - * @Date 2020-05-12 21:14 - */ -public class MySQLExtractor extends AbstractDatabaseExtractor { - - private final Logger logger = LoggerFactory.getLogger(getClass()); - - private final String BINLOG_FILENAME = "fileName"; - private final String BINLOG_POSITION = "position"; - private final int RETRY_TIMES = 10; - private final int MASTER = 0; - private Map tables = new HashMap<>(); - private BinaryLogClient client; - private List cluster; - private String database; - private final Lock connectLock = new ReentrantLock(); - private volatile boolean connected; - private volatile boolean recovery; - - @Override - public void start() { - try { - connectLock.lock(); - if (connected) { - logger.error("MySQLExtractor is already started"); - return; - } - run(); - connected = true; - } catch (Exception e) { - logger.error("启动失败:{}", e.getMessage()); - throw new ListenerException(e); - } finally { - connectLock.unlock(); - } - } - - @Override - public void close() { - try { - connectLock.lock(); - connected = false; - if (null != client) { - client.disconnect(); - } - } catch (Exception e) { - logger.error("关闭失败:{}", e.getMessage()); - } finally { - connectLock.unlock(); - } - } - - @Override - public void refreshEvent(ChangedOffset offset) { - refreshSnapshot(offset.getNextFileName(), (Long) offset.getPosition()); - } - - private void run() throws Exception { - final DatabaseConfig config = (DatabaseConfig) connectorConfig; - if (StringUtil.isBlank(config.getUrl())) { - throw new ListenerException("url is invalid"); - } - database = DatabaseUtil.getDatabaseName(config.getUrl()); - cluster = readNodes(config.getUrl()); - Assert.notEmpty(cluster, "MySQL连接地址有误."); - - final Host host = cluster.get(MASTER); - final String username = config.getUsername(); - final String password = config.getPassword(); - boolean containsPos = snapshot.containsKey(BINLOG_POSITION); - client = new BinaryLogRemoteClient(host.getIp(), host.getPort(), username, password); - client.setEnableDDL(true); - client.setBinlogFilename(snapshot.get(BINLOG_FILENAME)); - client.setBinlogPosition(containsPos ? Long.parseLong(snapshot.get(BINLOG_POSITION)) : 0); - client.setTableMapEventByTableId(tables); - client.registerEventListener(new MysqlEventListener()); - client.registerLifecycleListener(new MysqlLifecycleListener()); - - client.connect(); - - if (!containsPos) { - refreshSnapshot(client.getBinlogFilename(), client.getBinlogPosition()); - super.forceFlushEvent(); - } - } - - private List readNodes(String url) { - Matcher matcher = compile("(//)(?!(/)).+?(/)").matcher(url); - while (matcher.find()) { - url = matcher.group(0); - break; - } - url = StringUtil.replace(url, "/", ""); - - List cluster = new ArrayList<>(); - String[] arr = StringUtil.split(url, ","); - int size = arr.length; - for (int i = 0; i < size; i++) { - String[] host = StringUtil.split(arr[i], ":"); - if (2 == host.length) { - cluster.add(new Host(host[0], Integer.parseInt(host[1]))); - } - } - return cluster; - } - - private void reStart() { - try { - connectLock.lock(); - if (recovery) { - return; - } - recovery = true; - } finally { - connectLock.unlock(); - } - - for (int i = 1; i <= RETRY_TIMES; i++) { - try { - if (null != client) { - client.disconnect(); - } - run(); - - errorEvent(new ListenerException(String.format("重启成功, %s", client.getWorkerThreadName()))); - logger.error("第{}次重启成功, ThreadName:{} ", i, client.getWorkerThreadName()); - recovery = false; - break; - } catch (Exception e) { - logger.error("第{}次重启异常, ThreadName:{}, {}", i, client.getWorkerThreadName(), e.getMessage()); - // 无法连接，关闭任务 - if (i == RETRY_TIMES) { - errorEvent(new ListenerException(String.format("重启异常, %s, %s", client.getWorkerThreadName(), e.getMessage()))); - } - } - try { - TimeUnit.SECONDS.sleep(i * 2); - } catch (InterruptedException e) { - logger.error(e.getMessage()); - } - } - } - - private void refresh(EventHeader header) { - EventHeaderV4 eventHeaderV4 = (EventHeaderV4) header; - refresh(null, eventHeaderV4.getNextPosition()); - } - - private void refresh(String binlogFilename, long nextPosition) { - if (StringUtil.isNotBlank(binlogFilename)) { - client.setBinlogFilename(binlogFilename); - } - if (0 < nextPosition) { - client.setBinlogPosition(nextPosition); - } - } - - private void refreshSnapshot(String binlogFilename, long nextPosition) { - snapshot.put(BINLOG_FILENAME, binlogFilename); - snapshot.put(BINLOG_POSITION, String.valueOf(nextPosition)); - } - - final class MysqlLifecycleListener implements BinaryLogRemoteClient.LifecycleListener { - - @Override - public void onConnect(BinaryLogRemoteClient client) { - // 记录binlog增量点 - refresh(client.getBinlogFilename(), client.getBinlogPosition()); - } - - @Override - public void onCommunicationFailure(BinaryLogRemoteClient client, Exception e) { - if (!connected) { - return; - } - logger.error(e.getMessage()); - /** - * e: - * case1> Due to the automatic expiration and deletion mechanism of MySQL binlog files, the binlog file cannot be found. - * case2> Got fatal error 1236 from master when reading data from binary log. - * case3> Log event entry exceeded max_allowed_packet; Increase max_allowed_packet on master. - */ - if (e instanceof ServerException) { - ServerException serverException = (ServerException) e; - if (serverException.getErrorCode() == 1236) { - close(); - String log = String.format("线程[%s]执行异常。由于MySQL配置了过期binlog文件自动删除机制，已无法找到原binlog文件%s。建议先保存驱动（加载最新的binlog文件），再启动驱动。", - client.getWorkerThreadName(), - client.getBinlogFilename()); - errorEvent(new ListenerException(log)); - return; - } - } - - reStart(); - } - - @Override - public void onEventDeserializationFailure(BinaryLogRemoteClient client, Exception ex) { - } - - @Override - public void onDisconnect(BinaryLogRemoteClient client) { - } - - } - - final class MysqlEventListener implements BinaryLogRemoteClient.EventListener { - - @Override - public void onEvent(Event event) { - // ROTATE > FORMAT_DESCRIPTION > TABLE_MAP > WRITE_ROWS > UPDATE_ROWS > DELETE_ROWS > XID - EventHeader header = event.getHeader(); - if (header.getEventType() == EventType.XID) { - refresh(header); - return; - } - - if (EventType.isUpdate(header.getEventType())) { - refresh(header); - UpdateRowsEventData data = event.getData(); - if (isFilterTable(data.getTableId())) { - data.getRows().forEach(m -> { - List after = Stream.of(m.getValue()).collect(Collectors.toList()); - sendChangedEvent(new RowChangedEvent(getTableName(data.getTableId()), ConnectorConstant.OPERTION_UPDATE, after, client.getBinlogFilename(), client.getBinlogPosition())); - }); - } - return; - } - if (EventType.isWrite(header.getEventType())) { - refresh(header); - WriteRowsEventData data = event.getData(); - if (isFilterTable(data.getTableId())) { - data.getRows().forEach(m -> { - List after = Stream.of(m).collect(Collectors.toList()); - sendChangedEvent(new RowChangedEvent(getTableName(data.getTableId()), ConnectorConstant.OPERTION_INSERT, after, client.getBinlogFilename(), client.getBinlogPosition())); - }); - } - return; - } - if (EventType.isDelete(header.getEventType())) { - refresh(header); - DeleteRowsEventData data = event.getData(); - if (isFilterTable(data.getTableId())) { - data.getRows().forEach(m -> { - List before = Stream.of(m).collect(Collectors.toList()); - sendChangedEvent(new RowChangedEvent(getTableName(data.getTableId()), ConnectorConstant.OPERTION_DELETE, before, client.getBinlogFilename(), client.getBinlogPosition())); - }); - } - return; - } - - if (client.isEnableDDL() && EventType.QUERY == header.getEventType()) { - refresh(header); - parseDDL(event.getData()); - return; - } - - // 切换binlog - if (header.getEventType() == EventType.ROTATE) { - RotateEventData data = event.getData(); - refresh(data.getBinlogFilename(), data.getBinlogPosition()); - } - } - - private void parseDDL(QueryEventData data) { - if (StringUtil.startsWith(data.getSql(), ConnectorConstant.OPERTION_ALTER)) { - try { - // ALTER TABLE `test`.`my_user` MODIFY COLUMN `name` varchar(128) CHARACTER SET utf8 COLLATE utf8_bin NULL DEFAULT NULL AFTER `id` - Alter alter = (Alter) CCJSqlParserUtil.parse(data.getSql()); - String tableName = StringUtil.replace(alter.getTable().getName(),"`",""); - if (isFilterTable(data.getDatabase(), tableName)) { - logger.info("sql:{}", data.getSql()); - changeEvent(new DDLChangedEvent(data.getDatabase(), tableName, ConnectorConstant.OPERTION_ALTER, data.getSql(), client.getBinlogFilename(), client.getBinlogPosition())); - } - } catch (JSQLParserException e) { - logger.error("不支持ddl sql，支持标准的sql格式，请查看文档https://gitee.com/ghi/dbsyncer/wikis/%E5%BF%AB%E9%80%9F%E4%BA%86%E8%A7%A3/%E8%A1%A8%E7%BB%93%E6%9E%84%E5%90%8C%E6%AD%A5"); - } - } - } - - private String getTableName(long tableId) { - return tables.get(tableId).getTable(); - } - - private boolean isFilterTable(long tableId) { - final TableMapEventData tableMap = tables.get(tableId); - return isFilterTable(tableMap.getDatabase(), tableMap.getTable()); - } - - private boolean isFilterTable(String dbName, String tableName) { - return StringUtil.equalsIgnoreCase(database, dbName) && filterTable.contains(tableName); - } - - } - +package org.dbsyncer.connector.mysql; + +import com.github.shyiko.mysql.binlog.event.DeleteRowsEventData; +import com.github.shyiko.mysql.binlog.event.Event; +import com.github.shyiko.mysql.binlog.event.EventHeader; +import com.github.shyiko.mysql.binlog.event.EventHeaderV4; +import com.github.shyiko.mysql.binlog.event.EventType; +import com.github.shyiko.mysql.binlog.event.QueryEventData; +import com.github.shyiko.mysql.binlog.event.RotateEventData; +import com.github.shyiko.mysql.binlog.event.TableMapEventData; +import com.github.shyiko.mysql.binlog.event.UpdateRowsEventData; +import com.github.shyiko.mysql.binlog.event.WriteRowsEventData; +import com.github.shyiko.mysql.binlog.network.ServerException; +import net.sf.jsqlparser.JSQLParserException; +import net.sf.jsqlparser.parser.CCJSqlParserUtil; +import net.sf.jsqlparser.statement.alter.Alter; +import org.dbsyncer.common.util.StringUtil; +import org.dbsyncer.connector.AbstractDatabaseListener; +import org.dbsyncer.connector.ConnectorException; +import org.dbsyncer.sdk.config.DatabaseConfig; +import org.dbsyncer.sdk.constant.ConnectorConstant; +import org.dbsyncer.sdk.listener.event.DDLChangedEvent; +import org.dbsyncer.sdk.listener.event.RowChangedEvent; +import org.dbsyncer.sdk.model.ChangedOffset; +import org.dbsyncer.sdk.util.DatabaseUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.util.Assert; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import java.util.regex.Matcher; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static java.util.regex.Pattern.compile; + +/** + * @version 1.0.0 + * @Author AE86 + * @Date 2020-05-12 21:14 + */ +public class MySQLListener extends AbstractDatabaseListener { + + private final Logger logger = LoggerFactory.getLogger(getClass()); + + private final String BINLOG_FILENAME = "fileName"; + private final String BINLOG_POSITION = "position"; + private final int RETRY_TIMES = 10; + private final int MASTER = 0; + private Map tables = new HashMap<>(); + private BinaryLogClient client; + private List cluster; + private String database; + private final Lock connectLock = new ReentrantLock(); + private volatile boolean connected; + private volatile boolean recovery; + + @Override + public void start() { + try { + connectLock.lock(); + if (connected) { + logger.error("MySQLExtractor is already started"); + return; + } + run(); + connected = true; + } catch (Exception e) { + logger.error("启动失败:{}", e.getMessage()); + throw new ConnectorException(e); + } finally { + connectLock.unlock(); + } + } + + @Override + public void close() { + try { + connectLock.lock(); + connected = false; + if (null != client) { + client.disconnect(); + } + } catch (Exception e) { + logger.error("关闭失败:{}", e.getMessage()); + } finally { + connectLock.unlock(); + } + } + + @Override + public void refreshEvent(ChangedOffset offset) { + refreshSnapshot(offset.getNextFileName(), (Long) offset.getPosition()); + } + + private void run() throws Exception { + final DatabaseConfig config = (DatabaseConfig) connectorConfig; + if (StringUtil.isBlank(config.getUrl())) { + throw new ConnectorException("url is invalid"); + } + database = DatabaseUtil.getDatabaseName(config.getUrl()); + cluster = readNodes(config.getUrl()); + Assert.notEmpty(cluster, "MySQL连接地址有误."); + + final Host host = cluster.get(MASTER); + final String username = config.getUsername(); + final String password = config.getPassword(); + boolean containsPos = snapshot.containsKey(BINLOG_POSITION); + client = new BinaryLogRemoteClient(host.getIp(), host.getPort(), username, password); + client.setEnableDDL(true); + client.setBinlogFilename(snapshot.get(BINLOG_FILENAME)); + client.setBinlogPosition(containsPos ? Long.parseLong(snapshot.get(BINLOG_POSITION)) : 0); + client.setTableMapEventByTableId(tables); + client.registerEventListener(new MysqlEventListener()); + client.registerLifecycleListener(new MysqlLifecycleListener()); + + client.connect(); + + if (!containsPos) { + refreshSnapshot(client.getBinlogFilename(), client.getBinlogPosition()); + super.forceFlushEvent(); + } + } + + private List readNodes(String url) { + Matcher matcher = compile("(//)(?!(/)).+?(/)").matcher(url); + while (matcher.find()) { + url = matcher.group(0); + break; + } + url = StringUtil.replace(url, "/", ""); + + List cluster = new ArrayList<>(); + String[] arr = StringUtil.split(url, ","); + int size = arr.length; + for (int i = 0; i < size; i++) { + String[] host = StringUtil.split(arr[i], ":"); + if (2 == host.length) { + cluster.add(new Host(host[0], Integer.parseInt(host[1]))); + } + } + return cluster; + } + + private void reStart() { + try { + connectLock.lock(); + if (recovery) { + return; + } + recovery = true; + } finally { + connectLock.unlock(); + } + + for (int i = 1; i <= RETRY_TIMES; i++) { + try { + if (null != client) { + client.disconnect(); + } + run(); + + errorEvent(new ConnectorException(String.format("重启成功, %s", client.getWorkerThreadName()))); + logger.error("第{}次重启成功, ThreadName:{} ", i, client.getWorkerThreadName()); + recovery = false; + break; + } catch (Exception e) { + logger.error("第{}次重启异常, ThreadName:{}, {}", i, client.getWorkerThreadName(), e.getMessage()); + // 无法连接，关闭任务 + if (i == RETRY_TIMES) { + errorEvent(new ConnectorException(String.format("重启异常, %s, %s", client.getWorkerThreadName(), e.getMessage()))); + } + } + try { + TimeUnit.SECONDS.sleep(i * 2); + } catch (InterruptedException e) { + logger.error(e.getMessage()); + } + } + } + + private void refresh(EventHeader header) { + EventHeaderV4 eventHeaderV4 = (EventHeaderV4) header; + refresh(null, eventHeaderV4.getNextPosition()); + } + + private void refresh(String binlogFilename, long nextPosition) { + if (StringUtil.isNotBlank(binlogFilename)) { + client.setBinlogFilename(binlogFilename); + } + if (0 < nextPosition) { + client.setBinlogPosition(nextPosition); + } + } + + private void refreshSnapshot(String binlogFilename, long nextPosition) { + snapshot.put(BINLOG_FILENAME, binlogFilename); + snapshot.put(BINLOG_POSITION, String.valueOf(nextPosition)); + } + + final class Host { + private String ip; + private int port; + + public Host(String ip, int port) { + this.ip = ip; + this.port = port; + } + + public String getIp() { + return ip; + } + + public int getPort() { + return port; + } + } + + final class MysqlLifecycleListener implements BinaryLogRemoteClient.LifecycleListener { + + @Override + public void onConnect(BinaryLogRemoteClient client) { + // 记录binlog增量点 + refresh(client.getBinlogFilename(), client.getBinlogPosition()); + } + + @Override + public void onCommunicationFailure(BinaryLogRemoteClient client, Exception e) { + if (!connected) { + return; + } + logger.error(e.getMessage()); + /** + * e: + * case1> Due to the automatic expiration and deletion mechanism of MySQL binlog files, the binlog file cannot be found. + * case2> Got fatal error 1236 from master when reading data from binary log. + * case3> Log event entry exceeded max_allowed_packet; Increase max_allowed_packet on master. + */ + if (e instanceof ServerException) { + ServerException serverException = (ServerException) e; + if (serverException.getErrorCode() == 1236) { + close(); + String log = String.format("线程[%s]执行异常。由于MySQL配置了过期binlog文件自动删除机制，已无法找到原binlog文件%s。建议先保存驱动（加载最新的binlog文件），再启动驱动。", + client.getWorkerThreadName(), + client.getBinlogFilename()); + errorEvent(new ConnectorException(log)); + return; + } + } + + reStart(); + } + + @Override + public void onEventDeserializationFailure(BinaryLogRemoteClient client, Exception ex) { + } + + @Override + public void onDisconnect(BinaryLogRemoteClient client) { + } + + } + + final class MysqlEventListener implements BinaryLogRemoteClient.EventListener { + + @Override + public void onEvent(Event event) { + // ROTATE > FORMAT_DESCRIPTION > TABLE_MAP > WRITE_ROWS > UPDATE_ROWS > DELETE_ROWS > XID + EventHeader header = event.getHeader(); + if (header.getEventType() == EventType.XID) { + refresh(header); + return; + } + + if (EventType.isUpdate(header.getEventType())) { + refresh(header); + UpdateRowsEventData data = event.getData(); + if (isFilterTable(data.getTableId())) { + data.getRows().forEach(m -> { + List after = Stream.of(m.getValue()).collect(Collectors.toList()); + sendChangedEvent(new RowChangedEvent(getTableName(data.getTableId()), ConnectorConstant.OPERTION_UPDATE, after, client.getBinlogFilename(), client.getBinlogPosition())); + }); + } + return; + } + if (EventType.isWrite(header.getEventType())) { + refresh(header); + WriteRowsEventData data = event.getData(); + if (isFilterTable(data.getTableId())) { + data.getRows().forEach(m -> { + List after = Stream.of(m).collect(Collectors.toList()); + sendChangedEvent(new RowChangedEvent(getTableName(data.getTableId()), ConnectorConstant.OPERTION_INSERT, after, client.getBinlogFilename(), client.getBinlogPosition())); + }); + } + return; + } + if (EventType.isDelete(header.getEventType())) { + refresh(header); + DeleteRowsEventData data = event.getData(); + if (isFilterTable(data.getTableId())) { + data.getRows().forEach(m -> { + List before = Stream.of(m).collect(Collectors.toList()); + sendChangedEvent(new RowChangedEvent(getTableName(data.getTableId()), ConnectorConstant.OPERTION_DELETE, before, client.getBinlogFilename(), client.getBinlogPosition())); + }); + } + return; + } + + if (client.isEnableDDL() && EventType.QUERY == header.getEventType()) { + refresh(header); + parseDDL(event.getData()); + return; + } + + // 切换binlog + if (header.getEventType() == EventType.ROTATE) { + RotateEventData data = event.getData(); + refresh(data.getBinlogFilename(), data.getBinlogPosition()); + } + } + + private void parseDDL(QueryEventData data) { + if (StringUtil.startsWith(data.getSql(), ConnectorConstant.OPERTION_ALTER)) { + try { + // ALTER TABLE `test`.`my_user` MODIFY COLUMN `name` varchar(128) CHARACTER SET utf8 COLLATE utf8_bin NULL DEFAULT NULL AFTER `id` + Alter alter = (Alter) CCJSqlParserUtil.parse(data.getSql()); + String tableName = StringUtil.replace(alter.getTable().getName(), "`", ""); + if (isFilterTable(data.getDatabase(), tableName)) { + logger.info("sql:{}", data.getSql()); + changeEvent(new DDLChangedEvent(data.getDatabase(), tableName, ConnectorConstant.OPERTION_ALTER, data.getSql(), client.getBinlogFilename(), client.getBinlogPosition())); + } + } catch (JSQLParserException e) { + logger.error("不支持ddl sql，支持标准的sql格式，请查看文档https://gitee.com/ghi/dbsyncer/wikis/%E5%BF%AB%E9%80%9F%E4%BA%86%E8%A7%A3/%E8%A1%A8%E7%BB%93%E6%9E%84%E5%90%8C%E6%AD%A5"); + } + } + } + + private String getTableName(long tableId) { + return tables.get(tableId).getTable(); + } + + private boolean isFilterTable(long tableId) { + final TableMapEventData tableMap = tables.get(tableId); + return isFilterTable(tableMap.getDatabase(), tableMap.getTable()); + } + + private boolean isFilterTable(String dbName, String tableName) { + return StringUtil.equalsIgnoreCase(database, dbName) && filterTable.contains(tableName); + } + + } + } \ No newline at end of file diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/mysql/deserializer/DatetimeV2Deserialize.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/deserializer/DatetimeV2Deserialize.java similarity index 97% rename from dbsyncer-listener/src/main/java/org/dbsyncer/listener/mysql/deserializer/DatetimeV2Deserialize.java rename to dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/deserializer/DatetimeV2Deserialize.java index b9b09dd8..93e0ab60 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/mysql/deserializer/DatetimeV2Deserialize.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/deserializer/DatetimeV2Deserialize.java @@ -1,4 +1,4 @@ -package org.dbsyncer.listener.mysql.deserializer; +package org.dbsyncer.connector.mysql.deserializer; import com.github.shyiko.mysql.binlog.io.ByteArrayInputStream; diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/mysql/deserializer/DeleteDeserializer.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/deserializer/DeleteDeserializer.java similarity index 95% rename from dbsyncer-listener/src/main/java/org/dbsyncer/listener/mysql/deserializer/DeleteDeserializer.java rename to dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/deserializer/DeleteDeserializer.java index 1542c141..54d52a64 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/mysql/deserializer/DeleteDeserializer.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/deserializer/DeleteDeserializer.java @@ -1,4 +1,4 @@ -package org.dbsyncer.listener.mysql.deserializer; +package org.dbsyncer.connector.mysql.deserializer; import com.github.shyiko.mysql.binlog.event.TableMapEventData; import com.github.shyiko.mysql.binlog.event.deserialization.DeleteRowsEventDataDeserializer; diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/mysql/deserializer/JsonBinaryDeserialize.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/deserializer/JsonBinaryDeserialize.java similarity index 92% rename from dbsyncer-listener/src/main/java/org/dbsyncer/listener/mysql/deserializer/JsonBinaryDeserialize.java rename to dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/deserializer/JsonBinaryDeserialize.java index 5ffa7dfc..72458216 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/mysql/deserializer/JsonBinaryDeserialize.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/deserializer/JsonBinaryDeserialize.java @@ -1,4 +1,4 @@ -package org.dbsyncer.listener.mysql.deserializer; +package org.dbsyncer.connector.mysql.deserializer; import com.github.shyiko.mysql.binlog.event.deserialization.json.JsonBinary; import com.github.shyiko.mysql.binlog.io.ByteArrayInputStream; diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/mysql/deserializer/UpdateDeserializer.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/deserializer/UpdateDeserializer.java similarity index 95% rename from dbsyncer-listener/src/main/java/org/dbsyncer/listener/mysql/deserializer/UpdateDeserializer.java rename to dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/deserializer/UpdateDeserializer.java index f105fba0..04f68e6d 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/mysql/deserializer/UpdateDeserializer.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/deserializer/UpdateDeserializer.java @@ -1,4 +1,4 @@ -package org.dbsyncer.listener.mysql.deserializer; +package org.dbsyncer.connector.mysql.deserializer; import com.github.shyiko.mysql.binlog.event.TableMapEventData; import com.github.shyiko.mysql.binlog.event.deserialization.UpdateRowsEventDataDeserializer; diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/mysql/deserializer/WriteDeserializer.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/deserializer/WriteDeserializer.java similarity index 95% rename from dbsyncer-listener/src/main/java/org/dbsyncer/listener/mysql/deserializer/WriteDeserializer.java rename to dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/deserializer/WriteDeserializer.java index c46e65dd..e1c5b277 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/mysql/deserializer/WriteDeserializer.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/deserializer/WriteDeserializer.java @@ -1,4 +1,4 @@ -package org.dbsyncer.listener.mysql.deserializer; +package org.dbsyncer.connector.mysql.deserializer; import com.github.shyiko.mysql.binlog.event.TableMapEventData; import com.github.shyiko.mysql.binlog.event.deserialization.WriteRowsEventDataDeserializer; diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/oracle/event/DCNEvent.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/DCNEvent.java similarity index 87% rename from dbsyncer-listener/src/main/java/org/dbsyncer/listener/oracle/event/DCNEvent.java rename to dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/DCNEvent.java index 96c65e54..3438696f 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/oracle/event/DCNEvent.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/DCNEvent.java @@ -1,29 +1,29 @@ -/** - * DBSyncer Copyright 2019-2024 All Rights Reserved. - */ -package org.dbsyncer.listener.oracle.event; - -public final class DCNEvent { - - private String tableName; - private String rowId; - private int code; - - public DCNEvent(String tableName, String rowId, int code) { - this.tableName = tableName; - this.rowId = rowId; - this.code = code; - } - - public String getTableName() { - return tableName; - } - - public String getRowId() { - return rowId; - } - - public int getCode() { - return code; - } +/** + * DBSyncer Copyright 2019-2024 All Rights Reserved. + */ +package org.dbsyncer.connector.oracle; + +public final class DCNEvent { + + private String tableName; + private String rowId; + private int code; + + public DCNEvent(String tableName, String rowId, int code) { + this.tableName = tableName; + this.rowId = rowId; + this.code = code; + } + + public String getTableName() { + return tableName; + } + + public String getRowId() { + return rowId; + } + + public int getCode() { + return code; + } } \ No newline at end of file diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/oracle/DqlOracleExtractor.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/DqlOracleListener.java similarity index 79% rename from dbsyncer-listener/src/main/java/org/dbsyncer/listener/oracle/DqlOracleExtractor.java rename to dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/DqlOracleListener.java index f76e369d..017732ac 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/oracle/DqlOracleExtractor.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/DqlOracleListener.java @@ -1,6 +1,6 @@ -package org.dbsyncer.listener.oracle; +package org.dbsyncer.connector.oracle; -import org.dbsyncer.listener.ChangedEvent; +import org.dbsyncer.sdk.listener.ChangedEvent; import org.dbsyncer.sdk.model.Field; import java.util.List; @@ -10,7 +10,7 @@ import java.util.List; * @version 1.0.0 * @date 2022/5/29 22:44 */ -public class DqlOracleExtractor extends OracleExtractor { +public class DqlOracleListener extends OracleListener { @Override public void start() { diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/OracleConnector.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/OracleConnector.java index 02198409..5c5faf20 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/OracleConnector.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/OracleConnector.java @@ -1,12 +1,15 @@ package org.dbsyncer.connector.oracle; import org.dbsyncer.common.util.StringUtil; +import org.dbsyncer.connector.quartz.DatabaseQuartzListener; import org.dbsyncer.sdk.config.CommandConfig; import org.dbsyncer.sdk.config.DatabaseConfig; import org.dbsyncer.sdk.config.ReaderConfig; import org.dbsyncer.sdk.connector.database.AbstractDatabaseConnector; import org.dbsyncer.sdk.constant.DatabaseConstant; +import org.dbsyncer.sdk.enums.ListenerTypeEnum; import org.dbsyncer.sdk.enums.TableTypeEnum; +import org.dbsyncer.sdk.listener.Listener; import org.dbsyncer.sdk.model.PageSql; import org.dbsyncer.sdk.model.Table; import org.dbsyncer.sdk.util.PrimaryKeyUtil; @@ -35,6 +38,18 @@ public final class OracleConnector extends AbstractDatabaseConnector { return TYPE; } + @Override + public Listener getListener(String listenerType) { + if (ListenerTypeEnum.isTiming(listenerType)) { + return new DatabaseQuartzListener(); + } + + if (ListenerTypeEnum.isLog(listenerType)) { + return new OracleListener(); + } + return null; + } + @Override public String buildSqlWithQuotation() { return "\""; diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/oracle/OracleExtractor.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/OracleListener.java similarity index 76% rename from dbsyncer-listener/src/main/java/org/dbsyncer/listener/oracle/OracleExtractor.java rename to dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/OracleListener.java index 24e72c77..2f933309 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/oracle/OracleExtractor.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/OracleListener.java @@ -1,9 +1,9 @@ -package org.dbsyncer.listener.oracle; +package org.dbsyncer.connector.oracle; +import org.dbsyncer.connector.AbstractDatabaseListener; +import org.dbsyncer.connector.ConnectorException; +import org.dbsyncer.connector.oracle.dcn.DBChangeNotification; import org.dbsyncer.sdk.config.DatabaseConfig; -import org.dbsyncer.listener.AbstractDatabaseExtractor; -import org.dbsyncer.listener.ListenerException; -import org.dbsyncer.listener.oracle.dcn.DBChangeNotification; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -12,7 +12,7 @@ import org.slf4j.LoggerFactory; * @Author AE86 * @Date 2020-05-12 21:14 */ -public class OracleExtractor extends AbstractDatabaseExtractor { +public class OracleListener extends AbstractDatabaseListener { private final Logger logger = LoggerFactory.getLogger(getClass()); @@ -31,7 +31,7 @@ public class OracleExtractor extends AbstractDatabaseExtractor { client.start(); } catch (Exception e) { logger.error("启动失败:{}", e.getMessage()); - throw new ListenerException(e); + throw new ConnectorException(e); } } diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/oracle/dcn/DBChangeNotification.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/dcn/DBChangeNotification.java similarity index 96% rename from dbsyncer-listener/src/main/java/org/dbsyncer/listener/oracle/dcn/DBChangeNotification.java rename to dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/dcn/DBChangeNotification.java index 516a46a2..6cf4a7d9 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/oracle/dcn/DBChangeNotification.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/dcn/DBChangeNotification.java @@ -1,7 +1,7 @@ /** * DBSyncer Copyright 2019-2024 All Rights Reserved. */ -package org.dbsyncer.listener.oracle.dcn; +package org.dbsyncer.connector.oracle.dcn; import oracle.jdbc.OracleDriver; import oracle.jdbc.OracleStatement; @@ -11,11 +11,11 @@ import oracle.jdbc.dcn.DatabaseChangeRegistration; import oracle.jdbc.dcn.RowChangeDescription; import oracle.jdbc.dcn.TableChangeDescription; import oracle.jdbc.driver.OracleConnection; -import org.dbsyncer.listener.event.RowChangedEvent; import org.dbsyncer.common.util.StringUtil; +import org.dbsyncer.connector.ConnectorException; +import org.dbsyncer.connector.oracle.DCNEvent; import org.dbsyncer.sdk.constant.ConnectorConstant; -import org.dbsyncer.listener.ListenerException; -import org.dbsyncer.listener.oracle.event.DCNEvent; +import org.dbsyncer.sdk.listener.event.RowChangedEvent; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.util.Assert; @@ -248,7 +248,7 @@ public class DBChangeNotification { } catch (InvocationTargetException e) { logger.error(e.getMessage()); } - throw new ListenerException(String.format("Can't invoke '%s'.", declaredMethod)); + throw new ConnectorException(String.format("Can't invoke '%s'.", declaredMethod)); } /** @@ -265,7 +265,7 @@ public class DBChangeNotification { } // TNS - if(StringUtil.startsWith(url, "jdbc:oracle:thin:@(")){ + if (StringUtil.startsWith(url, "jdbc:oracle:thin:@(")) { Object obj = invokeDCR(dcr, "getClientHost"); return String.valueOf(obj); } @@ -356,14 +356,14 @@ public class DBChangeNotification { } // 断线 - if(eventType == DatabaseChangeEvent.EventType.SHUTDOWN){ + if (eventType == DatabaseChangeEvent.EventType.SHUTDOWN) { connected = false; logger.error("连接中断，等待Oracle数据库重启中..."); return; } // 重启 - if(eventType == DatabaseChangeEvent.EventType.STARTUP){ + if (eventType == DatabaseChangeEvent.EventType.STARTUP) { try { conn = connect(); connected = true; diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/oracle/dcn/RowEventListener.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/dcn/RowEventListener.java similarity index 68% rename from dbsyncer-listener/src/main/java/org/dbsyncer/listener/oracle/dcn/RowEventListener.java rename to dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/dcn/RowEventListener.java index a5e9e358..34e3eae0 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/oracle/dcn/RowEventListener.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/dcn/RowEventListener.java @@ -1,19 +1,19 @@ -/** - * DBSyncer Copyright 2019-2024 All Rights Reserved. - */ -package org.dbsyncer.listener.oracle.dcn; - -import org.dbsyncer.listener.event.RowChangedEvent; - -/** - * 行变更监听器 - * - * @version 1.0.0 - * @Author AE86 - * @Date 2020-06-15 20:00 - */ -public interface RowEventListener { - - void onEvents(RowChangedEvent rowChangedEvent); - +/** + * DBSyncer Copyright 2019-2024 All Rights Reserved. + */ +package org.dbsyncer.connector.oracle.dcn; + +import org.dbsyncer.sdk.listener.event.RowChangedEvent; + +/** + * 行变更监听器 + * + * @version 1.0.0 + * @Author AE86 + * @Date 2020-06-15 20:00 + */ +public interface RowEventListener { + + void onEvents(RowChangedEvent rowChangedEvent); + } \ No newline at end of file diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/AbstractMessageDecoder.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/AbstractMessageDecoder.java similarity index 97% rename from dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/AbstractMessageDecoder.java rename to dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/AbstractMessageDecoder.java index 7c16fa7a..a6549d66 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/AbstractMessageDecoder.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/AbstractMessageDecoder.java @@ -1,8 +1,8 @@ -package org.dbsyncer.listener.postgresql; +package org.dbsyncer.connector.postgresql; import org.dbsyncer.sdk.config.DatabaseConfig; -import org.dbsyncer.listener.postgresql.column.PgColumnValue; -import org.dbsyncer.listener.postgresql.enums.MessageTypeEnum; +import org.dbsyncer.connector.postgresql.column.PgColumnValue; +import org.dbsyncer.connector.postgresql.enums.MessageTypeEnum; import org.postgresql.replication.LogSequenceNumber; import org.postgresql.util.PGmoney; diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/DqlPostgreSQLExtractor.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/DqlPostgreSQLListener.java similarity index 66% rename from dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/DqlPostgreSQLExtractor.java rename to dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/DqlPostgreSQLListener.java index 6db7f7f4..c95d208f 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/DqlPostgreSQLExtractor.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/DqlPostgreSQLListener.java @@ -1,13 +1,13 @@ -package org.dbsyncer.listener.postgresql; +package org.dbsyncer.connector.postgresql; -import org.dbsyncer.listener.ChangedEvent; +import org.dbsyncer.sdk.listener.ChangedEvent; /** * @author AE86 * @version 1.0.0 * @date 2022/5/29 22:44 */ -public class DqlPostgreSQLExtractor extends PostgreSQLExtractor { +public class DqlPostgreSQLListener extends PostgreSQLListener { @Override public void start() { diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/MessageDecoder.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/MessageDecoder.java similarity index 90% rename from dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/MessageDecoder.java rename to dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/MessageDecoder.java index 86021363..2c3ed00d 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/MessageDecoder.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/MessageDecoder.java @@ -1,6 +1,6 @@ -package org.dbsyncer.listener.postgresql; +package org.dbsyncer.connector.postgresql; -import org.dbsyncer.listener.event.RowChangedEvent; +import org.dbsyncer.sdk.listener.event.RowChangedEvent; import org.dbsyncer.connector.ConnectorFactory; import org.dbsyncer.sdk.config.DatabaseConfig; import org.dbsyncer.sdk.connector.database.DatabaseConnectorInstance; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLConnector.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLConnector.java index b8a4df7e..b4efc875 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLConnector.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLConnector.java @@ -1,12 +1,15 @@ package org.dbsyncer.connector.postgresql; import org.dbsyncer.common.util.StringUtil; +import org.dbsyncer.connector.quartz.DatabaseQuartzListener; import org.dbsyncer.sdk.config.CommandConfig; import org.dbsyncer.sdk.config.DatabaseConfig; import org.dbsyncer.sdk.config.ReaderConfig; import org.dbsyncer.sdk.connector.database.AbstractDatabaseConnector; import org.dbsyncer.sdk.constant.DatabaseConstant; +import org.dbsyncer.sdk.enums.ListenerTypeEnum; import org.dbsyncer.sdk.enums.TableTypeEnum; +import org.dbsyncer.sdk.listener.Listener; import org.dbsyncer.sdk.model.PageSql; import org.dbsyncer.sdk.model.Table; import org.dbsyncer.sdk.util.PrimaryKeyUtil; @@ -30,6 +33,18 @@ public final class PostgreSQLConnector extends AbstractDatabaseConnector { return TYPE; } + @Override + public Listener getListener(String listenerType) { + if (ListenerTypeEnum.isTiming(listenerType)) { + return new DatabaseQuartzListener(); + } + + if (ListenerTypeEnum.isLog(listenerType)) { + return new PostgreSQLListener(); + } + return null; + } + @PostConstruct private void init() { VALUE_MAPPERS.put(Types.BIT, new PostgreSQLBitValueMapper()); diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/PostgreSQLExtractor.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLListener.java similarity index 92% rename from dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/PostgreSQLExtractor.java rename to dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLListener.java index 236750b2..2218eb09 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/PostgreSQLExtractor.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLListener.java @@ -1,14 +1,14 @@ -package org.dbsyncer.listener.postgresql; +package org.dbsyncer.connector.postgresql; -import org.dbsyncer.listener.model.ChangedOffset; -import org.dbsyncer.listener.event.RowChangedEvent; import org.dbsyncer.common.util.BooleanUtil; +import org.dbsyncer.connector.AbstractDatabaseListener; +import org.dbsyncer.connector.ConnectorException; +import org.dbsyncer.connector.postgresql.enums.MessageDecoderEnum; import org.dbsyncer.sdk.config.DatabaseConfig; import org.dbsyncer.sdk.connector.database.DatabaseConnectorInstance; +import org.dbsyncer.sdk.listener.event.RowChangedEvent; +import org.dbsyncer.sdk.model.ChangedOffset; import org.dbsyncer.sdk.util.DatabaseUtil; -import org.dbsyncer.listener.AbstractDatabaseExtractor; -import org.dbsyncer.listener.ListenerException; -import org.dbsyncer.listener.postgresql.enums.MessageDecoderEnum; import org.postgresql.PGConnection; import org.postgresql.PGProperty; import org.postgresql.replication.LogSequenceNumber; @@ -36,7 +36,7 @@ import java.util.concurrent.locks.ReentrantLock; * @version 1.0.0 * @date 2022/4/10 22:36 */ -public class PostgreSQLExtractor extends AbstractDatabaseExtractor { +public class PostgreSQLListener extends AbstractDatabaseListener { private final Logger logger = LoggerFactory.getLogger(getClass()); @@ -75,7 +75,7 @@ public class PostgreSQLExtractor extends AbstractDatabaseExtractor { final String walLevel = connectorInstance.execute(databaseTemplate -> databaseTemplate.queryForObject(GET_WAL_LEVEL, String.class)); if (!DEFAULT_WAL_LEVEL.equals(walLevel)) { - throw new ListenerException(String.format("Postgres server wal_level property must be \"%s\" but is: %s", DEFAULT_WAL_LEVEL, walLevel)); + throw new ConnectorException(String.format("Postgres server wal_level property must be \"%s\" but is: %s", DEFAULT_WAL_LEVEL, walLevel)); } final boolean hasAuth = connectorInstance.execute(databaseTemplate -> { @@ -88,7 +88,7 @@ public class PostgreSQLExtractor extends AbstractDatabaseExtractor { return login && (replication || superuser || admin || repAdmin); }); if (!hasAuth) { - throw new ListenerException(String.format("Postgres roles LOGIN and REPLICATION are not assigned to user: %s", config.getUsername())); + throw new ConnectorException(String.format("Postgres roles LOGIN and REPLICATION are not assigned to user: %s", config.getUsername())); } database = connectorInstance.execute(databaseTemplate -> databaseTemplate.queryForObject(GET_DATABASE, String.class)); @@ -109,7 +109,7 @@ public class PostgreSQLExtractor extends AbstractDatabaseExtractor { logger.error("启动失败:{}", e.getMessage()); DatabaseUtil.close(stream); DatabaseUtil.close(connection); - throw new ListenerException(e); + throw new ConnectorException(e); } finally { connectLock.unlock(); } @@ -184,9 +184,9 @@ public class PostgreSQLExtractor extends AbstractDatabaseExtractor { } if (!snapshot.containsKey(LSN_POSITION)) { - LogSequenceNumber lsn = connectorInstance.execute(databaseTemplate -> LogSequenceNumber.valueOf(databaseTemplate.queryForObject(GET_RESTART_LSN, new Object[] {database, slotName, plugin}, String.class))); + LogSequenceNumber lsn = connectorInstance.execute(databaseTemplate -> LogSequenceNumber.valueOf(databaseTemplate.queryForObject(GET_RESTART_LSN, new Object[]{database, slotName, plugin}, String.class))); if (null == lsn || lsn.asLong() == 0) { - throw new ListenerException("No maximum LSN recorded in the database"); + throw new ConnectorException("No maximum LSN recorded in the database"); } snapshot.put(LSN_POSITION, lsn.asString()); super.forceFlushEvent(); @@ -280,7 +280,7 @@ public class PostgreSQLExtractor extends AbstractDatabaseExtractor { // process decoder RowChangedEvent event = messageDecoder.processMessage(msg); - if(event != null){ + if (event != null) { event.setPosition(lsn.asString()); sendChangedEvent(event); } @@ -289,7 +289,7 @@ public class PostgreSQLExtractor extends AbstractDatabaseExtractor { stream.setAppliedLSN(lsn); stream.setFlushedLSN(lsn); stream.forceUpdateStatus(); - } catch (IllegalStateException | ListenerException e) { + } catch (IllegalStateException | ConnectorException e) { logger.error(e.getMessage()); } catch (Exception e) { logger.error(e.getMessage(), e); diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/column/PgColumnValue.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/column/PgColumnValue.java similarity index 86% rename from dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/column/PgColumnValue.java rename to dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/column/PgColumnValue.java index 0eb68011..e32522a6 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/column/PgColumnValue.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/column/PgColumnValue.java @@ -1,11 +1,17 @@ -package org.dbsyncer.listener.postgresql.column; +package org.dbsyncer.connector.postgresql.column; import org.dbsyncer.common.column.AbstractColumnValue; import org.dbsyncer.common.util.DateFormatUtil; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.listener.ListenerException; +import org.dbsyncer.connector.ConnectorException; import org.postgresql.PGStatement; -import org.postgresql.geometric.*; +import org.postgresql.geometric.PGbox; +import org.postgresql.geometric.PGcircle; +import org.postgresql.geometric.PGline; +import org.postgresql.geometric.PGlseg; +import org.postgresql.geometric.PGpath; +import org.postgresql.geometric.PGpoint; +import org.postgresql.geometric.PGpolygon; import org.postgresql.util.PGInterval; import org.postgresql.util.PGmoney; import org.slf4j.Logger; @@ -16,7 +22,11 @@ import java.sql.Date; import java.sql.SQLException; import java.sql.Time; import java.sql.Timestamp; -import java.time.*; +import java.time.Instant; +import java.time.LocalTime; +import java.time.OffsetDateTime; +import java.time.OffsetTime; +import java.time.ZoneOffset; import java.util.concurrent.TimeUnit; public final class PgColumnValue extends AbstractColumnValue { @@ -116,7 +126,7 @@ public final class PgColumnValue extends AbstractColumnValue { return new PGbox(asString()); } catch (final SQLException e) { logger.error("Failed to parse point {}, {}", asString(), e); - throw new ListenerException(e); + throw new ConnectorException(e); } } @@ -125,7 +135,7 @@ public final class PgColumnValue extends AbstractColumnValue { return new PGcircle(asString()); } catch (final SQLException e) { logger.error("Failed to parse circle {}, {}", asString(), e); - throw new ListenerException(e); + throw new ConnectorException(e); } } @@ -134,7 +144,7 @@ public final class PgColumnValue extends AbstractColumnValue { return new PGInterval(asString()); } catch (final SQLException e) { logger.error("Failed to parse point {}, {}", asString(), e); - throw new ListenerException(e); + throw new ConnectorException(e); } } @@ -143,7 +153,7 @@ public final class PgColumnValue extends AbstractColumnValue { return new PGline(asString()); } catch (final SQLException e) { logger.error("Failed to parse point {}, {}", asString(), e); - throw new ListenerException(e); + throw new ConnectorException(e); } } @@ -152,7 +162,7 @@ public final class PgColumnValue extends AbstractColumnValue { return new PGlseg(asString()); } catch (final SQLException e) { logger.error("Failed to parse point {}, {}", asString(), e); - throw new ListenerException(e); + throw new ConnectorException(e); } } @@ -166,7 +176,7 @@ public final class PgColumnValue extends AbstractColumnValue { return new PGmoney(asString()); } catch (final SQLException e) { logger.error("Failed to parse money {}, {}", asString(), e); - throw new ListenerException(e); + throw new ConnectorException(e); } } @@ -175,7 +185,7 @@ public final class PgColumnValue extends AbstractColumnValue { return new PGpath(asString()); } catch (final SQLException e) { logger.error("Failed to parse point {}, {}", asString(), e); - throw new ListenerException(e); + throw new ConnectorException(e); } } @@ -184,7 +194,7 @@ public final class PgColumnValue extends AbstractColumnValue { return new PGpoint(asString()); } catch (final SQLException e) { logger.error("Failed to parse point {}, {}", asString(), e); - throw new ListenerException(e); + throw new ConnectorException(e); } } @@ -193,7 +203,7 @@ public final class PgColumnValue extends AbstractColumnValue { return new PGpolygon(asString()); } catch (final SQLException e) { logger.error("Failed to parse point {}, {}", asString(), e); - throw new ListenerException(e); + throw new ConnectorException(e); } } diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/decoder/PgOutputMessageDecoder.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/decoder/PgOutputMessageDecoder.java similarity index 93% rename from dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/decoder/PgOutputMessageDecoder.java rename to dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/decoder/PgOutputMessageDecoder.java index fb981cd4..11c9118f 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/decoder/PgOutputMessageDecoder.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/decoder/PgOutputMessageDecoder.java @@ -1,13 +1,13 @@ -package org.dbsyncer.listener.postgresql.decoder; +package org.dbsyncer.connector.postgresql.decoder; import org.dbsyncer.common.util.CollectionUtils; +import org.dbsyncer.connector.ConnectorException; import org.dbsyncer.connector.ConnectorFactory; -import org.dbsyncer.listener.ListenerException; -import org.dbsyncer.listener.event.RowChangedEvent; -import org.dbsyncer.listener.postgresql.AbstractMessageDecoder; -import org.dbsyncer.listener.postgresql.enums.MessageDecoderEnum; -import org.dbsyncer.listener.postgresql.enums.MessageTypeEnum; +import org.dbsyncer.connector.postgresql.AbstractMessageDecoder; +import org.dbsyncer.connector.postgresql.enums.MessageDecoderEnum; +import org.dbsyncer.connector.postgresql.enums.MessageTypeEnum; import org.dbsyncer.sdk.connector.database.DatabaseConnectorInstance; +import org.dbsyncer.sdk.listener.event.RowChangedEvent; import org.dbsyncer.sdk.model.Field; import org.dbsyncer.sdk.model.MetaInfo; import org.postgresql.replication.fluent.logical.ChainedLogicalStreamBuilder; @@ -112,7 +112,7 @@ public class PgOutputMessageDecoder extends AbstractMessageDecoder { return true; }); } catch (Exception e) { - throw new ListenerException(e.getCause()); + throw new ConnectorException(e.getCause()); } } @@ -158,7 +158,7 @@ public class PgOutputMessageDecoder extends AbstractMessageDecoder { // The table schema has been changed, we should be get a new table schema from db. MetaInfo metaInfo = connectorFactory.getMetaInfo(connectorInstance, tableId.tableName); if (CollectionUtils.isEmpty(metaInfo.getColumn())) { - throw new ListenerException(String.format("The table column for '%s' is empty.", tableId.tableName)); + throw new ConnectorException(String.format("The table column for '%s' is empty.", tableId.tableName)); } tableId.fields = metaInfo.getColumn(); return; diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/decoder/TestDecodingMessageDecoder.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/decoder/TestDecodingMessageDecoder.java similarity index 91% rename from dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/decoder/TestDecodingMessageDecoder.java rename to dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/decoder/TestDecodingMessageDecoder.java index 02392d45..97d704a0 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/decoder/TestDecodingMessageDecoder.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/decoder/TestDecodingMessageDecoder.java @@ -1,11 +1,11 @@ -package org.dbsyncer.listener.postgresql.decoder; +package org.dbsyncer.connector.postgresql.decoder; import org.dbsyncer.common.column.Lexer; -import org.dbsyncer.listener.event.RowChangedEvent; +import org.dbsyncer.connector.postgresql.AbstractMessageDecoder; +import org.dbsyncer.connector.postgresql.enums.MessageDecoderEnum; +import org.dbsyncer.connector.postgresql.enums.MessageTypeEnum; import org.dbsyncer.sdk.constant.ConnectorConstant; -import org.dbsyncer.listener.postgresql.AbstractMessageDecoder; -import org.dbsyncer.listener.postgresql.enums.MessageDecoderEnum; -import org.dbsyncer.listener.postgresql.enums.MessageTypeEnum; +import org.dbsyncer.sdk.listener.event.RowChangedEvent; import org.postgresql.replication.fluent.logical.ChainedLogicalStreamBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/enums/MessageDecoderEnum.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/enums/MessageDecoderEnum.java similarity index 72% rename from dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/enums/MessageDecoderEnum.java rename to dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/enums/MessageDecoderEnum.java index c3a8d8e6..288c0bb5 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/enums/MessageDecoderEnum.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/enums/MessageDecoderEnum.java @@ -1,10 +1,10 @@ -package org.dbsyncer.listener.postgresql.enums; +package org.dbsyncer.connector.postgresql.enums; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.listener.ListenerException; -import org.dbsyncer.listener.postgresql.MessageDecoder; -import org.dbsyncer.listener.postgresql.decoder.PgOutputMessageDecoder; -import org.dbsyncer.listener.postgresql.decoder.TestDecodingMessageDecoder; +import org.dbsyncer.connector.ConnectorException; +import org.dbsyncer.connector.postgresql.MessageDecoder; +import org.dbsyncer.connector.postgresql.decoder.PgOutputMessageDecoder; +import org.dbsyncer.connector.postgresql.decoder.TestDecodingMessageDecoder; /** * @author AE86 @@ -31,7 +31,7 @@ public enum MessageDecoderEnum { this.clazz = clazz; } - public static MessageDecoder getMessageDecoder(String type) throws ListenerException, IllegalAccessException, InstantiationException { + public static MessageDecoder getMessageDecoder(String type) throws ConnectorException, IllegalAccessException, InstantiationException { for (MessageDecoderEnum e : MessageDecoderEnum.values()) { if (StringUtil.equals(type, e.getType())) { return (MessageDecoder) e.getClazz().newInstance(); diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/enums/MessageTypeEnum.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/enums/MessageTypeEnum.java similarity index 90% rename from dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/enums/MessageTypeEnum.java rename to dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/enums/MessageTypeEnum.java index d444bf8b..b8a660a4 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/postgresql/enums/MessageTypeEnum.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/enums/MessageTypeEnum.java @@ -1,42 +1,42 @@ -package org.dbsyncer.listener.postgresql.enums; - -public enum MessageTypeEnum { - BEGIN, - COMMIT, - TABLE, - INSERT, - UPDATE, - DELETE, - RELATION, - TRUNCATE, - TYPE, - ORIGIN, - NONE; - - public static MessageTypeEnum getType(char type) { - switch (type) { - case 'B': - return BEGIN; - case 'C': - return COMMIT; - case 't': - return TABLE; - case 'I': - return INSERT; - case 'U': - return UPDATE; - case 'D': - return DELETE; - case 'R': - return RELATION; - case 'Y': - return TYPE; - case 'O': - return ORIGIN; - case 'T': - return TRUNCATE; - default: - return NONE; - } - } +package org.dbsyncer.connector.postgresql.enums; + +public enum MessageTypeEnum { + BEGIN, + COMMIT, + TABLE, + INSERT, + UPDATE, + DELETE, + RELATION, + TRUNCATE, + TYPE, + ORIGIN, + NONE; + + public static MessageTypeEnum getType(char type) { + switch (type) { + case 'B': + return BEGIN; + case 'C': + return COMMIT; + case 't': + return TABLE; + case 'I': + return INSERT; + case 'U': + return UPDATE; + case 'D': + return DELETE; + case 'R': + return RELATION; + case 'Y': + return TYPE; + case 'O': + return ORIGIN; + case 'T': + return TRUNCATE; + default: + return NONE; + } + } } \ No newline at end of file diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/AbstractQuartzExtractor.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/quartz/AbstractQuartzListener.java similarity index 95% rename from dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/AbstractQuartzExtractor.java rename to dbsyncer-connector/src/main/java/org/dbsyncer/connector/quartz/AbstractQuartzListener.java index 4341d85c..3a1ea6bd 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/AbstractQuartzExtractor.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/quartz/AbstractQuartzListener.java @@ -1,15 +1,15 @@ -package org.dbsyncer.listener.quartz; +package org.dbsyncer.connector.quartz; import org.dbsyncer.common.model.Result; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.common.util.UUIDUtil; +import org.dbsyncer.connector.AbstractListener; import org.dbsyncer.connector.scheduled.ScheduledTaskJob; -import org.dbsyncer.listener.AbstractExtractor; -import org.dbsyncer.listener.event.ScanChangedEvent; import org.dbsyncer.sdk.config.ReaderConfig; import org.dbsyncer.sdk.connector.ConnectorInstance; import org.dbsyncer.sdk.constant.ConnectorConstant; +import org.dbsyncer.sdk.listener.event.ScanChangedEvent; import org.dbsyncer.sdk.model.Table; import org.dbsyncer.sdk.util.PrimaryKeyUtil; import org.slf4j.Logger; @@ -30,7 +30,7 @@ import java.util.stream.Stream; * @Author AE86 * @Date 2020-05-12 20:35 */ -public abstract class AbstractQuartzExtractor extends AbstractExtractor implements ScheduledTaskJob { +public abstract class AbstractQuartzListener extends AbstractListener implements ScheduledTaskJob { private final Logger logger = LoggerFactory.getLogger(getClass()); private final String CURSOR = "cursor"; diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/DatabaseQuartzExtractor.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/quartz/DatabaseQuartzListener.java similarity index 96% rename from dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/DatabaseQuartzExtractor.java rename to dbsyncer-connector/src/main/java/org/dbsyncer/connector/quartz/DatabaseQuartzListener.java index ebf84282..7d973ca6 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/DatabaseQuartzExtractor.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/quartz/DatabaseQuartzListener.java @@ -1,9 +1,9 @@ -package org.dbsyncer.listener.quartz; +package org.dbsyncer.connector.quartz; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.StringUtil; +import org.dbsyncer.connector.enums.QuartzFilterEnum; import org.dbsyncer.sdk.constant.ConnectorConstant; -import org.dbsyncer.listener.enums.QuartzFilterEnum; import org.springframework.util.Assert; import java.util.ArrayList; @@ -22,7 +22,7 @@ import java.util.stream.Stream; * @Author AE86 * @Date 2021-09-01 20:35 */ -public final class DatabaseQuartzExtractor extends AbstractQuartzExtractor { +public final class DatabaseQuartzListener extends AbstractQuartzListener { @Override protected Point checkLastPoint(Map command, int index) { diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/Point.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/quartz/Point.java similarity index 97% rename from dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/Point.java rename to dbsyncer-connector/src/main/java/org/dbsyncer/connector/quartz/Point.java index d939e96c..e790c8ad 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/Point.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/quartz/Point.java @@ -1,4 +1,4 @@ -package org.dbsyncer.listener.quartz; +package org.dbsyncer.connector.quartz; import org.dbsyncer.common.util.StringUtil; diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/QuartzFilter.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/quartz/QuartzFilter.java similarity index 91% rename from dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/QuartzFilter.java rename to dbsyncer-connector/src/main/java/org/dbsyncer/connector/quartz/QuartzFilter.java index 7c7202fc..62d171fa 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/QuartzFilter.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/quartz/QuartzFilter.java @@ -1,4 +1,4 @@ -package org.dbsyncer.listener.quartz; +package org.dbsyncer.connector.quartz; public interface QuartzFilter { diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/TableGroupQuartzCommand.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/quartz/TableGroupQuartzCommand.java similarity index 94% rename from dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/TableGroupQuartzCommand.java rename to dbsyncer-connector/src/main/java/org/dbsyncer/connector/quartz/TableGroupQuartzCommand.java index a2a734a1..13dcf479 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/TableGroupQuartzCommand.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/quartz/TableGroupQuartzCommand.java @@ -1,4 +1,4 @@ -package org.dbsyncer.listener.quartz; +package org.dbsyncer.connector.quartz; import org.dbsyncer.sdk.model.Table; import org.dbsyncer.sdk.util.PrimaryKeyUtil; diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/filter/DateFilter.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/quartz/filter/DateFilter.java similarity index 86% rename from dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/filter/DateFilter.java rename to dbsyncer-connector/src/main/java/org/dbsyncer/connector/quartz/filter/DateFilter.java index 54ba664f..0ade9c69 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/filter/DateFilter.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/quartz/filter/DateFilter.java @@ -1,7 +1,7 @@ -package org.dbsyncer.listener.quartz.filter; +package org.dbsyncer.connector.quartz.filter; import org.dbsyncer.common.util.DateFormatUtil; -import org.dbsyncer.listener.quartz.QuartzFilter; +import org.dbsyncer.connector.quartz.QuartzFilter; import java.sql.Date; import java.time.Instant; diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/filter/TimestampFilter.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/quartz/filter/TimestampFilter.java similarity index 86% rename from dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/filter/TimestampFilter.java rename to dbsyncer-connector/src/main/java/org/dbsyncer/connector/quartz/filter/TimestampFilter.java index a42d4e7e..c04447d9 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/filter/TimestampFilter.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/quartz/filter/TimestampFilter.java @@ -1,6 +1,6 @@ -package org.dbsyncer.listener.quartz.filter; +package org.dbsyncer.connector.quartz.filter; -import org.dbsyncer.listener.quartz.QuartzFilter; +import org.dbsyncer.connector.quartz.QuartzFilter; import java.sql.Timestamp; import java.time.Instant; diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/filter/YesDateFilter.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/quartz/filter/YesDateFilter.java similarity index 92% rename from dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/filter/YesDateFilter.java rename to dbsyncer-connector/src/main/java/org/dbsyncer/connector/quartz/filter/YesDateFilter.java index f72ae498..0601f378 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/filter/YesDateFilter.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/quartz/filter/YesDateFilter.java @@ -1,7 +1,7 @@ -package org.dbsyncer.listener.quartz.filter; +package org.dbsyncer.connector.quartz.filter; import org.dbsyncer.common.util.DateFormatUtil; -import org.dbsyncer.listener.quartz.QuartzFilter; +import org.dbsyncer.connector.quartz.QuartzFilter; import java.sql.Date; import java.time.LocalDateTime; diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/filter/YesTimestampFilter.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/quartz/filter/YesTimestampFilter.java similarity index 90% rename from dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/filter/YesTimestampFilter.java rename to dbsyncer-connector/src/main/java/org/dbsyncer/connector/quartz/filter/YesTimestampFilter.java index d265b6f7..67e3f2ea 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/quartz/filter/YesTimestampFilter.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/quartz/filter/YesTimestampFilter.java @@ -1,6 +1,6 @@ -package org.dbsyncer.listener.quartz.filter; +package org.dbsyncer.connector.quartz.filter; -import org.dbsyncer.listener.quartz.QuartzFilter; +import org.dbsyncer.connector.quartz.QuartzFilter; import java.sql.Timestamp; import java.time.LocalDateTime; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLMySQLConnector.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLMySQLConnector.java index 74745416..259157d3 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLMySQLConnector.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLMySQLConnector.java @@ -1,9 +1,13 @@ package org.dbsyncer.connector.sql; +import org.dbsyncer.connector.mysql.DqlMySQLListener; +import org.dbsyncer.connector.quartz.DatabaseQuartzListener; import org.dbsyncer.sdk.config.CommandConfig; import org.dbsyncer.sdk.config.ReaderConfig; import org.dbsyncer.sdk.connector.database.AbstractDQLConnector; import org.dbsyncer.sdk.constant.DatabaseConstant; +import org.dbsyncer.sdk.enums.ListenerTypeEnum; +import org.dbsyncer.sdk.listener.Listener; import org.dbsyncer.sdk.model.PageSql; import org.springframework.stereotype.Component; @@ -35,4 +39,16 @@ public final class DQLMySQLConnector extends AbstractDQLConnector { public Map getSourceCommand(CommandConfig commandConfig) { return super.getDqlSourceCommand(commandConfig, true); } + + @Override + public Listener getListener(String listenerType) { + if (ListenerTypeEnum.isTiming(listenerType)) { + return new DatabaseQuartzListener(); + } + + if (ListenerTypeEnum.isLog(listenerType)) { + return new DqlMySQLListener(); + } + return null; + } } \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLOracleConnector.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLOracleConnector.java index f7cca7fb..48b9aac2 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLOracleConnector.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLOracleConnector.java @@ -1,8 +1,12 @@ package org.dbsyncer.connector.sql; +import org.dbsyncer.connector.oracle.DqlOracleListener; +import org.dbsyncer.connector.quartz.DatabaseQuartzListener; import org.dbsyncer.sdk.config.ReaderConfig; import org.dbsyncer.sdk.connector.database.AbstractDQLConnector; import org.dbsyncer.sdk.constant.DatabaseConstant; +import org.dbsyncer.sdk.enums.ListenerTypeEnum; +import org.dbsyncer.sdk.listener.Listener; import org.dbsyncer.sdk.model.PageSql; import org.springframework.stereotype.Component; @@ -16,6 +20,18 @@ public final class DQLOracleConnector extends AbstractDQLConnector { return TYPE; } + @Override + public Listener getListener(String listenerType) { + if (ListenerTypeEnum.isTiming(listenerType)) { + return new DatabaseQuartzListener(); + } + + if (ListenerTypeEnum.isLog(listenerType)) { + return new DqlOracleListener(); + } + return null; + } + @Override public String getPageSql(PageSql config) { return DatabaseConstant.ORACLE_PAGE_SQL_START + config.getQuerySql() + DatabaseConstant.ORACLE_PAGE_SQL_END; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLPostgreSQLConnector.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLPostgreSQLConnector.java index 37f087cc..8720faa9 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLPostgreSQLConnector.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLPostgreSQLConnector.java @@ -1,8 +1,12 @@ package org.dbsyncer.connector.sql; +import org.dbsyncer.connector.postgresql.DqlPostgreSQLListener; +import org.dbsyncer.connector.quartz.DatabaseQuartzListener; import org.dbsyncer.sdk.config.ReaderConfig; import org.dbsyncer.sdk.connector.database.AbstractDQLConnector; import org.dbsyncer.sdk.constant.DatabaseConstant; +import org.dbsyncer.sdk.enums.ListenerTypeEnum; +import org.dbsyncer.sdk.listener.Listener; import org.dbsyncer.sdk.model.PageSql; import org.springframework.stereotype.Component; @@ -16,6 +20,18 @@ public final class DQLPostgreSQLConnector extends AbstractDQLConnector { return TYPE; } + @Override + public Listener getListener(String listenerType) { + if (ListenerTypeEnum.isTiming(listenerType)) { + return new DatabaseQuartzListener(); + } + + if (ListenerTypeEnum.isLog(listenerType)) { + return new DqlPostgreSQLListener(); + } + return null; + } + @Override public String getPageSql(PageSql config) { return config.getQuerySql() + DatabaseConstant.POSTGRESQL_PAGE_SQL; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLSqlServerConnector.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLSqlServerConnector.java index e7ad4199..691893c3 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLSqlServerConnector.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLSqlServerConnector.java @@ -1,9 +1,13 @@ package org.dbsyncer.connector.sql; import org.dbsyncer.common.util.StringUtil; +import org.dbsyncer.connector.quartz.DatabaseQuartzListener; +import org.dbsyncer.connector.sqlserver.DqlSqlServerListener; import org.dbsyncer.sdk.config.ReaderConfig; import org.dbsyncer.sdk.connector.database.AbstractDQLConnector; import org.dbsyncer.sdk.constant.DatabaseConstant; +import org.dbsyncer.sdk.enums.ListenerTypeEnum; +import org.dbsyncer.sdk.listener.Listener; import org.dbsyncer.sdk.model.PageSql; import org.springframework.stereotype.Component; @@ -19,6 +23,18 @@ public final class DQLSqlServerConnector extends AbstractDQLConnector { return TYPE; } + @Override + public Listener getListener(String listenerType) { + if (ListenerTypeEnum.isTiming(listenerType)) { + return new DatabaseQuartzListener(); + } + + if (ListenerTypeEnum.isLog(listenerType)) { + return new DqlSqlServerListener(); + } + return null; + } + @Override public String getPageSql(PageSql config) { List primaryKeys = config.getPrimaryKeys(); diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/sqlserver/CDCEvent.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/CDCEvent.java similarity index 92% rename from dbsyncer-listener/src/main/java/org/dbsyncer/listener/sqlserver/CDCEvent.java rename to dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/CDCEvent.java index 7ebe1da9..6491f4e0 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/sqlserver/CDCEvent.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/CDCEvent.java @@ -1,7 +1,7 @@ /** * DBSyncer Copyright 2019-2024 All Rights Reserved. */ -package org.dbsyncer.listener.sqlserver; +package org.dbsyncer.connector.sqlserver; import java.util.List; diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/sqlserver/DqlSqlServerExtractor.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/DqlSqlServerListener.java similarity index 67% rename from dbsyncer-listener/src/main/java/org/dbsyncer/listener/sqlserver/DqlSqlServerExtractor.java rename to dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/DqlSqlServerListener.java index ba8f306a..db33209b 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/sqlserver/DqlSqlServerExtractor.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/DqlSqlServerListener.java @@ -1,13 +1,13 @@ -package org.dbsyncer.listener.sqlserver; +package org.dbsyncer.connector.sqlserver; -import org.dbsyncer.listener.ChangedEvent; +import org.dbsyncer.sdk.listener.ChangedEvent; /** * @author AE86 * @version 1.0.0 * @date 2022/5/22 22:56 */ -public class DqlSqlServerExtractor extends SqlServerExtractor { +public class DqlSqlServerListener extends SqlServerListener { @Override public void start() { diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/sqlserver/Lsn.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/Lsn.java similarity index 98% rename from dbsyncer-listener/src/main/java/org/dbsyncer/listener/sqlserver/Lsn.java rename to dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/Lsn.java index ade94565..31fcb499 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/sqlserver/Lsn.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/Lsn.java @@ -1,4 +1,4 @@ -package org.dbsyncer.listener.sqlserver; +package org.dbsyncer.connector.sqlserver; import java.util.Arrays; diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/sqlserver/LsnPuller.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/LsnPuller.java similarity index 80% rename from dbsyncer-listener/src/main/java/org/dbsyncer/listener/sqlserver/LsnPuller.java rename to dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/LsnPuller.java index f17a8d9d..a807fc35 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/sqlserver/LsnPuller.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/LsnPuller.java @@ -1,4 +1,4 @@ -package org.dbsyncer.listener.sqlserver; +package org.dbsyncer.connector.sqlserver; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -21,7 +21,7 @@ public class LsnPuller { */ private static final long DEFAULT_POLL_INTERVAL_MILLIS = 100; private static volatile LsnPuller instance = null; - private final Map map = new ConcurrentHashMap<>(); + private final Map map = new ConcurrentHashMap<>(); private Worker worker; private LsnPuller() { @@ -46,8 +46,8 @@ public class LsnPuller { worker.start(); } - public static void addExtractor(String metaId, SqlServerExtractor extractor) { - getInstance().map.put(metaId, extractor); + public static void addExtractor(String metaId, SqlServerListener listener) { + getInstance().map.put(metaId, listener); } public static void removeExtractor(String metaId) { @@ -65,10 +65,10 @@ public class LsnPuller { continue; } Lsn maxLsn = null; - for (SqlServerExtractor extractor : map.values()) { - maxLsn = extractor.getMaxLsn(); - if (null != maxLsn && maxLsn.isAvailable() && maxLsn.compareTo(extractor.getLastLsn()) > 0) { - extractor.pushStopLsn(maxLsn); + for (SqlServerListener listener : map.values()) { + maxLsn = listener.getMaxLsn(); + if (null != maxLsn && maxLsn.isAvailable() && maxLsn.compareTo(listener.getLastLsn()) > 0) { + listener.pushStopLsn(maxLsn); } } TimeUnit.MILLISECONDS.sleep(DEFAULT_POLL_INTERVAL_MILLIS); diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/sqlserver/SqlServerChangeTable.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/SqlServerChangeTable.java similarity index 85% rename from dbsyncer-listener/src/main/java/org/dbsyncer/listener/sqlserver/SqlServerChangeTable.java rename to dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/SqlServerChangeTable.java index 3511b6d5..fd29591e 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/sqlserver/SqlServerChangeTable.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/SqlServerChangeTable.java @@ -1,4 +1,4 @@ -package org.dbsyncer.listener.sqlserver; +package org.dbsyncer.connector.sqlserver; import org.dbsyncer.common.util.JsonUtil; @@ -12,9 +12,7 @@ public class SqlServerChangeTable { private byte[] stopLsn; private String capturedColumns; - public SqlServerChangeTable(String schemaName, String tableName, String captureInstance, - int changeTableObjectId, - byte[] startLsn, byte[] stopLsn, String capturedColumns) { + public SqlServerChangeTable(String schemaName, String tableName, String captureInstance, int changeTableObjectId, byte[] startLsn, byte[] stopLsn, String capturedColumns) { this.schemaName = schemaName; this.tableName = tableName; this.captureInstance = captureInstance; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/SqlServerConnector.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/SqlServerConnector.java index 19bb5e8a..ca194281 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/SqlServerConnector.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/SqlServerConnector.java @@ -2,22 +2,23 @@ package org.dbsyncer.connector.sqlserver; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.sdk.config.DatabaseConfig; +import org.dbsyncer.connector.quartz.DatabaseQuartzListener; import org.dbsyncer.sdk.config.CommandConfig; +import org.dbsyncer.sdk.config.DatabaseConfig; import org.dbsyncer.sdk.config.ReaderConfig; -import org.dbsyncer.sdk.constant.DatabaseConstant; import org.dbsyncer.sdk.connector.database.AbstractDatabaseConnector; import org.dbsyncer.sdk.connector.database.DatabaseConnectorInstance; +import org.dbsyncer.sdk.constant.DatabaseConstant; +import org.dbsyncer.sdk.enums.ListenerTypeEnum; import org.dbsyncer.sdk.enums.TableTypeEnum; -import org.dbsyncer.sdk.model.PageSql; +import org.dbsyncer.sdk.listener.Listener; import org.dbsyncer.sdk.model.Field; +import org.dbsyncer.sdk.model.PageSql; import org.dbsyncer.sdk.model.Table; import org.springframework.stereotype.Component; import java.util.ArrayList; import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import java.util.stream.Collectors; @Component @@ -42,6 +43,18 @@ public final class SqlServerConnector extends AbstractDatabaseConnector { return tables; } + @Override + public Listener getListener(String listenerType) { + if (ListenerTypeEnum.isTiming(listenerType)) { + return new DatabaseQuartzListener(); + } + + if (ListenerTypeEnum.isLog(listenerType)) { + return new SqlServerListener(); + } + return null; + } + @Override public String getPageSql(PageSql config) { List primaryKeys = buildPrimaryKeys(config.getPrimaryKeys()); @@ -53,7 +66,7 @@ public final class SqlServerConnector extends AbstractDatabaseConnector { public Object[] getPageArgs(ReaderConfig config) { int pageSize = config.getPageSize(); int pageIndex = config.getPageIndex(); - return new Object[] {(pageIndex - 1) * pageSize + 1, pageIndex * pageSize}; + return new Object[]{(pageIndex - 1) * pageSize + 1, pageIndex * pageSize}; } @Override @@ -100,18 +113,4 @@ public final class SqlServerConnector extends AbstractDatabaseConnector { return new StringBuilder("[").append(key).append("]").toString(); } - /** - * 是否包含系统关键字 - * - * @param regex - * @param val - * @return - */ - private boolean containsKeyword(String regex, String val) { - if (StringUtil.isNotBlank(val)) { - Matcher matcher = Pattern.compile(regex).matcher(val.toLowerCase()); - return matcher.find(); - } - return false; - } } \ No newline at end of file diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/sqlserver/SqlServerExtractor.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/SqlServerListener.java similarity index 96% rename from dbsyncer-listener/src/main/java/org/dbsyncer/listener/sqlserver/SqlServerExtractor.java rename to dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/SqlServerListener.java index 37cbb6dd..da7cb5ab 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/sqlserver/SqlServerExtractor.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/SqlServerListener.java @@ -1,15 +1,15 @@ -package org.dbsyncer.listener.sqlserver; +package org.dbsyncer.connector.sqlserver; import com.microsoft.sqlserver.jdbc.SQLServerException; -import org.dbsyncer.listener.model.ChangedOffset; -import org.dbsyncer.listener.event.RowChangedEvent; +import org.dbsyncer.connector.ConnectorException; +import org.dbsyncer.connector.enums.TableOperationEnum; +import org.dbsyncer.sdk.model.ChangedOffset; +import org.dbsyncer.sdk.listener.event.RowChangedEvent; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.sdk.config.DatabaseConfig; import org.dbsyncer.sdk.constant.ConnectorConstant; import org.dbsyncer.sdk.connector.database.DatabaseConnectorInstance; -import org.dbsyncer.listener.AbstractDatabaseExtractor; -import org.dbsyncer.listener.ListenerException; -import org.dbsyncer.listener.enums.TableOperationEnum; +import org.dbsyncer.connector.AbstractDatabaseListener; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.util.Assert; @@ -36,7 +36,7 @@ import java.util.concurrent.locks.ReentrantLock; * @Author AE86 * @Date 2021-06-18 01:20 */ -public class SqlServerExtractor extends AbstractDatabaseExtractor { +public class SqlServerListener extends AbstractDatabaseListener { private final Logger logger = LoggerFactory.getLogger(getClass()); @@ -99,7 +99,7 @@ public class SqlServerExtractor extends AbstractDatabaseExtractor { } catch (Exception e) { close(); logger.error("启动失败:{}", e.getMessage()); - throw new ListenerException(e); + throw new ConnectorException(e); } finally { connectLock.unlock(); } @@ -152,7 +152,7 @@ public class SqlServerExtractor extends AbstractDatabaseExtractor { return; } // Shouldn't happen if the agent is running, but it is better to guard against such situation - throw new ListenerException("No maximum LSN recorded in the database"); + throw new ConnectorException("No maximum LSN recorded in the database"); } lastLsn = Lsn.valueOf(snapshot.get(LSN_POSITION)); } diff --git a/dbsyncer-listener/src/main/test/BinaryLogRemoteClientTest.java b/dbsyncer-connector/src/main/test/BinaryLogRemoteClientTest.java similarity index 97% rename from dbsyncer-listener/src/main/test/BinaryLogRemoteClientTest.java rename to dbsyncer-connector/src/main/test/BinaryLogRemoteClientTest.java index b6083e11..c008f8e4 100644 --- a/dbsyncer-listener/src/main/test/BinaryLogRemoteClientTest.java +++ b/dbsyncer-connector/src/main/test/BinaryLogRemoteClientTest.java @@ -1,7 +1,7 @@ import com.github.shyiko.mysql.binlog.event.*; import org.dbsyncer.sdk.constant.ConnectorConstant; -import org.dbsyncer.listener.mysql.BinaryLogClient; -import org.dbsyncer.listener.mysql.BinaryLogRemoteClient; +import org.dbsyncer.connector.mysql.BinaryLogClient; +import org.dbsyncer.connector.mysql.BinaryLogRemoteClient; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dbsyncer-listener/src/main/test/ChangeDataCaptureTest.java b/dbsyncer-connector/src/main/test/ChangeDataCaptureTest.java similarity index 97% rename from dbsyncer-listener/src/main/test/ChangeDataCaptureTest.java rename to dbsyncer-connector/src/main/test/ChangeDataCaptureTest.java index d1094cb0..c9853785 100644 --- a/dbsyncer-listener/src/main/test/ChangeDataCaptureTest.java +++ b/dbsyncer-connector/src/main/test/ChangeDataCaptureTest.java @@ -1,315 +1,315 @@ -import com.microsoft.sqlserver.jdbc.SQLServerException; -import org.dbsyncer.common.util.CollectionUtils; -import org.dbsyncer.listener.sqlserver.Lsn; -import org.dbsyncer.listener.sqlserver.SqlServerChangeTable; -import org.junit.Assert; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.sql.*; -import java.util.*; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.regex.Matcher; - -/** - * @version 1.0.0 - * @Author AE86 - * @Date 2021-06-14 01:55 - * @Link https://www.red-gate.com/simple-talk/sql/learn-sql-server/introduction-to-change-data-capture-cdc-in-sql-server-2008/ - */ -public class ChangeDataCaptureTest { - - private final Logger logger = LoggerFactory.getLogger(getClass()); - - private static final String STATEMENTS_PLACEHOLDER = "#"; - private static final String GET_DATABASE_NAME = "SELECT db_name()"; - private static final String GET_DATABASE_VERSION = "SELECT @@VERSION AS 'SQL Server Version'"; - private static final String GET_TABLE_LIST = "SELECT NAME FROM SYS.TABLES WHERE SCHEMA_ID = SCHEMA_ID('#') AND IS_MS_SHIPPED = 0"; - private static final String IS_SERVER_AGENT_RUNNING = "EXEC master.#.xp_servicecontrol N'QUERYSTATE', N'SQLSERVERAGENT'"; - private static final String IS_DB_CDC_ENABLED = "SELECT is_cdc_enabled FROM sys.databases WHERE name = '#'"; - private static final String IS_TABLE_CDC_ENABLED = "SELECT COUNT(*) FROM sys.tables tb WHERE tb.is_tracked_by_cdc = 1 AND tb.name='#'"; - private static final String ENABLE_DB_CDC = "IF EXISTS(select 1 from sys.databases where name = '#' AND is_cdc_enabled=0) EXEC sys.sp_cdc_enable_db"; - private static final String ENABLE_TABLE_CDC = "IF EXISTS(select 1 from sys.tables where name = '#' AND is_tracked_by_cdc=0) EXEC sys.sp_cdc_enable_table @source_schema = N'%s', @source_name = N'#', @role_name = NULL, @supports_net_changes = 0"; - private static final String DISABLE_TABLE_CDC = "EXEC sys.sp_cdc_disable_table @source_schema = N'%s', @source_name = N'#', @capture_instance = 'all'"; - - private static final String AT_TIME_ZONE_UTC = " AT TIME ZONE 'UTC'"; - private static final String GET_ALL_CHANGES_FOR_TABLE = "SELECT sys.fn_cdc_map_lsn_to_time([__$start_lsn])#, * FROM cdc.[fn_cdc_get_all_changes_#](?, ?, N'all update old') order by [__$start_lsn] ASC, [__$seqval] ASC, [__$operation] ASC"; - private static final String GET_TABLES_CDC_ENABLED = "EXEC sys.sp_cdc_help_change_data_capture"; - private static final String GET_MAX_LSN = "SELECT sys.fn_cdc_get_max_lsn()"; - private static final String GET_MIN_LSN = "SELECT sys.fn_cdc_get_min_lsn('#')"; - private static final String GET_INCREMENT_LSN = "SELECT sys.fn_cdc_increment_lsn(?)"; - - private String realDatabaseName; - private String getAllChangesForTable; - private Connection connection = null; - private Set tables; - private String schema; - - /** - *

    cdc.captured_columns – 此表返回捕获列列表的结果。

    - *

    cdc.change_tables – 此表返回所有启用捕获的表的列表。

    - *

    cdc.ddl_history – 此表包含自启用捕获数据以来所有 DDL 更改的历史记录。

    - *

    cdc.index_columns – 该表包含与变更表相关的索引。

    - *

    cdc.lsn_time_mapping – 此表映射 LSN编号(唯一序列号标识, 增加数字) 和时间。

    - *

    cdc.fn_cdc_get_all_changes_MY_USER - 可用于获取在特定时间段内发生的事件

    - *

    sys.fn_cdc_map_time_to_lsn - 表中是否有 tran_end_time值大于或等于指定时间的行。例如，可以用此查询来确定捕获进程是否已处理完截至前指定时间提交的更改

    - *

    sys.fn_cdc_get_max_lsn

    - *

    sys.sp_cdc_cleanup_change_table 默认情况下间隔为3天清理日志数据

    - * - * @throws SQLException - */ - @Test - public void testConnect() throws SQLException, InterruptedException { - ChangeDataCaptureTest cdc = new ChangeDataCaptureTest(); - cdc.start(); - - // 获取数据库名 test - realDatabaseName = cdc.queryAndMap(GET_DATABASE_NAME, rs -> rs.getString(1)); - logger.info("数据库名:{}", realDatabaseName); - // As per https://www.mssqltips.com/sqlservertip/1140/how-to-tell-what-sql-server-version-you-are-running/ - // Always beginning with 'Microsoft SQL Server NNNN' but only in case SQL Server is standalone - String version = cdc.queryAndMap(GET_DATABASE_VERSION, rs -> rs.getString(1)); - boolean supportsAtTimeZone = false; - if (version.startsWith("Microsoft SQL Server ")) { - supportsAtTimeZone = 2016 < Integer.valueOf(version.substring(21, 25)); - } - logger.info("数据库版本:{}", version); - tables = cdc.queryAndMapList(GET_TABLE_LIST.replace(STATEMENTS_PLACEHOLDER, realDatabaseName), rs -> { - Set table = new LinkedHashSet<>(); - while (rs.next()) { - table.add(rs.getString(1)); - } - return table; - }); - logger.info("所有表:{}", tables); - // 获取Agent服务状态 Stopped. Running. - boolean enabledServerAgent = cdc.queryAndMap(IS_SERVER_AGENT_RUNNING.replace(STATEMENTS_PLACEHOLDER, realDatabaseName), rs -> "Running.".equals(rs.getString(1))); - logger.info("是否启动Agent服务:{}", enabledServerAgent); - Assert.assertTrue("The agent server is not running", enabledServerAgent); - boolean enabledCDC = cdc.queryAndMap(IS_DB_CDC_ENABLED.replace(STATEMENTS_PLACEHOLDER, realDatabaseName), rs -> rs.getBoolean(1)); - logger.info("是否启用CDC库[{}]:{}", realDatabaseName, enabledCDC); - if (!enabledCDC) { - cdc.execute(ENABLE_DB_CDC.replace(STATEMENTS_PLACEHOLDER, realDatabaseName)); - // make sure DB has cdc-enabled before proceeding - TimeUnit.SECONDS.sleep(3); - } - - // 注册CDC表 - tables.forEach(table -> { - try { - boolean enabledTableCDC = cdc.queryAndMap(IS_TABLE_CDC_ENABLED.replace(STATEMENTS_PLACEHOLDER, table), rs -> rs.getInt(1) > 0); - logger.info("是否启用CDC表[{}]:{}", table, enabledTableCDC); - if (!enabledTableCDC) { - cdc.execute(String.format(ENABLE_TABLE_CDC.replace(STATEMENTS_PLACEHOLDER, table), schema)); - Lsn minLsn = cdc.queryAndMap(GET_MIN_LSN.replace(STATEMENTS_PLACEHOLDER, table), rs -> new Lsn(rs.getBytes(1))); - logger.info("启用CDC表[{}]:{}", table, minLsn.isAvailable()); - } - } catch (SQLException e) { - logger.error(e.getMessage()); - } - }); - - // 支持UTC - getAllChangesForTable = GET_ALL_CHANGES_FOR_TABLE.replaceFirst(STATEMENTS_PLACEHOLDER, Matcher.quoteReplacement(supportsAtTimeZone ? AT_TIME_ZONE_UTC : "")); - - // 读取增量 - Set changeTables = cdc.queryAndMapList(GET_TABLES_CDC_ENABLED, rs -> { - final Set tables = new HashSet<>(); - while (rs.next()) { - SqlServerChangeTable changeTable = new SqlServerChangeTable( - // schemaName - rs.getString(1), - // tableName - rs.getString(2), - // captureInstance - rs.getString(3), - // changeTableObjectId - rs.getInt(4), - // startLsn - rs.getBytes(6), - // stopLsn - rs.getBytes(7), - // capturedColumns - rs.getString(15)); - logger.info(changeTable.toString()); - tables.add(changeTable); - } - return tables; - }); - logger.info("监听表数:{} ", changeTables.size()); - - if (!CollectionUtils.isEmpty(changeTables)) { - AtomicInteger count = new AtomicInteger(0); - Lsn lastLsn = cdc.queryAndMap(GET_MAX_LSN, rs -> new Lsn(rs.getBytes(1))); - - while (true && count.getAndAdd(1) < 30) { - Lsn stopLsn = cdc.queryAndMap(GET_MAX_LSN, rs -> new Lsn(rs.getBytes(1))); - if (!stopLsn.isAvailable()) { - logger.warn("No maximum LSN recorded in the database; please ensure that the SQL Server Agent is running"); - cdc.pause(); - continue; - } - - if (stopLsn.compareTo(lastLsn) <= 0) { - cdc.pause(); - continue; - } - - Lsn startLsn = getIncrementLsn(cdc, lastLsn); - changeTables.forEach(changeTable -> { - try { - final String query = getAllChangesForTable.replace(STATEMENTS_PLACEHOLDER, changeTable.getCaptureInstance()); - logger.info("Getting changes for table {} in range[{}, {}]", changeTable.getTableName(), startLsn, stopLsn); - - cdc.queryAndMapList(query, statement -> { - statement.setBytes(1, startLsn.getBinary()); - statement.setBytes(2, stopLsn.getBinary()); - }, rs -> { - int columnCount = rs.getMetaData().getColumnCount(); - List> data = new ArrayList<>(columnCount); - List row = null; - while (rs.next()) { - row = new ArrayList<>(columnCount); - for (int i = 1; i <= columnCount; i++) { - row.add(rs.getObject(i)); - } - logger.info("rows:{}", row); - data.add(row); - } - return data; - }); - } catch (Exception e) { - e.printStackTrace(); - } - }); - - lastLsn = stopLsn; - } - } - - // 注销CDC表 - for (String table : tables) { - cdc.execute(String.format(DISABLE_TABLE_CDC.replace(STATEMENTS_PLACEHOLDER, table), schema)); - } - cdc.close(); - } - - public void start() throws SQLException { - String username = "sa"; - String password = "123"; - String url = "jdbc:sqlserver://127.0.0.1:1433;DatabaseName=test"; - schema = "dbo"; - connection = DriverManager.getConnection(url, username, password); - if (connection != null) { - DatabaseMetaData dm = (DatabaseMetaData) connection.getMetaData(); - logger.info("Driver name: " + dm.getDriverName()); - logger.info("Driver version: " + dm.getDriverVersion()); - logger.info("Product name: " + dm.getDatabaseProductName()); - logger.info("Product version: " + dm.getDatabaseProductVersion()); - } - } - - public void close() { - if (null != connection) { - close(connection); - } - } - - private void close(AutoCloseable closeable) { - if (null != closeable) { - try { - closeable.close(); - } catch (Exception e) { - logger.error(e.getMessage()); - } - } - } - - private Lsn getIncrementLsn(ChangeDataCaptureTest cdc, Lsn lastLsn) { - return cdc.queryAndMap(GET_INCREMENT_LSN, statement -> statement.setBytes(1, lastLsn.getBinary()), rs -> Lsn.valueOf(rs.getBytes(1))); - } - - private void pause() throws InterruptedException { - TimeUnit.SECONDS.sleep(2); - } - - private void execute(String... sqlStatements) throws SQLException { - Statement statement = connection.createStatement(); - try { - for (String sqlStatement : sqlStatements) { - if (sqlStatement != null) { - logger.info("executing '{}'", sqlStatement); - statement.execute(sqlStatement); - } - } - } catch (Exception e) { - logger.error(e.getMessage()); - } finally { - close(statement); - } - } - - public interface ResultSetMapper { - T apply(ResultSet rs) throws SQLException; - } - - public interface StatementPreparer { - void accept(PreparedStatement statement) throws SQLException; - } - - public T queryAndMap(String sql, ResultSetMapper mapper) { - return queryAndMap(sql, null, mapper); - } - - public T queryAndMap(String sql, StatementPreparer statementPreparer, ResultSetMapper mapper) { - PreparedStatement ps = null; - ResultSet rs = null; - T apply = null; - try { - ps = connection.prepareStatement(sql); - if (null != statementPreparer) { - statementPreparer.accept(ps); - } - rs = ps.executeQuery(); - if (rs.next()) { - apply = mapper.apply(rs); - } - } catch (Exception e) { - logger.error(e.getMessage()); - } finally { - close(rs); - close(ps); - } - return apply; - } - - public T queryAndMapList(String sql, ResultSetMapper mapper) { - return queryAndMapList(sql, null, mapper); - } - - public T queryAndMapList(String sql, StatementPreparer statementPreparer, ResultSetMapper mapper) { - PreparedStatement ps = null; - ResultSet rs = null; - T apply = null; - try { - ps = connection.prepareStatement(sql); - if (null != statementPreparer) { - statementPreparer.accept(ps); - } - rs = ps.executeQuery(); - apply = mapper.apply(rs); - } catch (SQLServerException e) { - // 为过程或函数 cdc.fn_cdc_get_all_changes_ ... 提供的参数数目不足。 - logger.warn(e.getMessage()); - } catch (Exception e) { - logger.error(e.getMessage()); - } finally { - close(rs); - close(ps); - } - return apply; - } - +import com.microsoft.sqlserver.jdbc.SQLServerException; +import org.dbsyncer.common.util.CollectionUtils; +import org.dbsyncer.connector.sqlserver.Lsn; +import org.dbsyncer.connector.sqlserver.SqlServerChangeTable; +import org.junit.Assert; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.*; +import java.util.*; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.regex.Matcher; + +/** + * @version 1.0.0 + * @Author AE86 + * @Date 2021-06-14 01:55 + * @Link https://www.red-gate.com/simple-talk/sql/learn-sql-server/introduction-to-change-data-capture-cdc-in-sql-server-2008/ + */ +public class ChangeDataCaptureTest { + + private final Logger logger = LoggerFactory.getLogger(getClass()); + + private static final String STATEMENTS_PLACEHOLDER = "#"; + private static final String GET_DATABASE_NAME = "SELECT db_name()"; + private static final String GET_DATABASE_VERSION = "SELECT @@VERSION AS 'SQL Server Version'"; + private static final String GET_TABLE_LIST = "SELECT NAME FROM SYS.TABLES WHERE SCHEMA_ID = SCHEMA_ID('#') AND IS_MS_SHIPPED = 0"; + private static final String IS_SERVER_AGENT_RUNNING = "EXEC master.#.xp_servicecontrol N'QUERYSTATE', N'SQLSERVERAGENT'"; + private static final String IS_DB_CDC_ENABLED = "SELECT is_cdc_enabled FROM sys.databases WHERE name = '#'"; + private static final String IS_TABLE_CDC_ENABLED = "SELECT COUNT(*) FROM sys.tables tb WHERE tb.is_tracked_by_cdc = 1 AND tb.name='#'"; + private static final String ENABLE_DB_CDC = "IF EXISTS(select 1 from sys.databases where name = '#' AND is_cdc_enabled=0) EXEC sys.sp_cdc_enable_db"; + private static final String ENABLE_TABLE_CDC = "IF EXISTS(select 1 from sys.tables where name = '#' AND is_tracked_by_cdc=0) EXEC sys.sp_cdc_enable_table @source_schema = N'%s', @source_name = N'#', @role_name = NULL, @supports_net_changes = 0"; + private static final String DISABLE_TABLE_CDC = "EXEC sys.sp_cdc_disable_table @source_schema = N'%s', @source_name = N'#', @capture_instance = 'all'"; + + private static final String AT_TIME_ZONE_UTC = " AT TIME ZONE 'UTC'"; + private static final String GET_ALL_CHANGES_FOR_TABLE = "SELECT sys.fn_cdc_map_lsn_to_time([__$start_lsn])#, * FROM cdc.[fn_cdc_get_all_changes_#](?, ?, N'all update old') order by [__$start_lsn] ASC, [__$seqval] ASC, [__$operation] ASC"; + private static final String GET_TABLES_CDC_ENABLED = "EXEC sys.sp_cdc_help_change_data_capture"; + private static final String GET_MAX_LSN = "SELECT sys.fn_cdc_get_max_lsn()"; + private static final String GET_MIN_LSN = "SELECT sys.fn_cdc_get_min_lsn('#')"; + private static final String GET_INCREMENT_LSN = "SELECT sys.fn_cdc_increment_lsn(?)"; + + private String realDatabaseName; + private String getAllChangesForTable; + private Connection connection = null; + private Set tables; + private String schema; + + /** + *

    cdc.captured_columns – 此表返回捕获列列表的结果。

    + *

    cdc.change_tables – 此表返回所有启用捕获的表的列表。

    + *

    cdc.ddl_history – 此表包含自启用捕获数据以来所有 DDL 更改的历史记录。

    + *

    cdc.index_columns – 该表包含与变更表相关的索引。

    + *

    cdc.lsn_time_mapping – 此表映射 LSN编号(唯一序列号标识, 增加数字) 和时间。

    + *

    cdc.fn_cdc_get_all_changes_MY_USER - 可用于获取在特定时间段内发生的事件

    + *

    sys.fn_cdc_map_time_to_lsn - 表中是否有 tran_end_time值大于或等于指定时间的行。例如，可以用此查询来确定捕获进程是否已处理完截至前指定时间提交的更改

    + *

    sys.fn_cdc_get_max_lsn

    + *

    sys.sp_cdc_cleanup_change_table 默认情况下间隔为3天清理日志数据

    + * + * @throws SQLException + */ + @Test + public void testConnect() throws SQLException, InterruptedException { + ChangeDataCaptureTest cdc = new ChangeDataCaptureTest(); + cdc.start(); + + // 获取数据库名 test + realDatabaseName = cdc.queryAndMap(GET_DATABASE_NAME, rs -> rs.getString(1)); + logger.info("数据库名:{}", realDatabaseName); + // As per https://www.mssqltips.com/sqlservertip/1140/how-to-tell-what-sql-server-version-you-are-running/ + // Always beginning with 'Microsoft SQL Server NNNN' but only in case SQL Server is standalone + String version = cdc.queryAndMap(GET_DATABASE_VERSION, rs -> rs.getString(1)); + boolean supportsAtTimeZone = false; + if (version.startsWith("Microsoft SQL Server ")) { + supportsAtTimeZone = 2016 < Integer.valueOf(version.substring(21, 25)); + } + logger.info("数据库版本:{}", version); + tables = cdc.queryAndMapList(GET_TABLE_LIST.replace(STATEMENTS_PLACEHOLDER, realDatabaseName), rs -> { + Set table = new LinkedHashSet<>(); + while (rs.next()) { + table.add(rs.getString(1)); + } + return table; + }); + logger.info("所有表:{}", tables); + // 获取Agent服务状态 Stopped. Running. + boolean enabledServerAgent = cdc.queryAndMap(IS_SERVER_AGENT_RUNNING.replace(STATEMENTS_PLACEHOLDER, realDatabaseName), rs -> "Running.".equals(rs.getString(1))); + logger.info("是否启动Agent服务:{}", enabledServerAgent); + Assert.assertTrue("The agent server is not running", enabledServerAgent); + boolean enabledCDC = cdc.queryAndMap(IS_DB_CDC_ENABLED.replace(STATEMENTS_PLACEHOLDER, realDatabaseName), rs -> rs.getBoolean(1)); + logger.info("是否启用CDC库[{}]:{}", realDatabaseName, enabledCDC); + if (!enabledCDC) { + cdc.execute(ENABLE_DB_CDC.replace(STATEMENTS_PLACEHOLDER, realDatabaseName)); + // make sure DB has cdc-enabled before proceeding + TimeUnit.SECONDS.sleep(3); + } + + // 注册CDC表 + tables.forEach(table -> { + try { + boolean enabledTableCDC = cdc.queryAndMap(IS_TABLE_CDC_ENABLED.replace(STATEMENTS_PLACEHOLDER, table), rs -> rs.getInt(1) > 0); + logger.info("是否启用CDC表[{}]:{}", table, enabledTableCDC); + if (!enabledTableCDC) { + cdc.execute(String.format(ENABLE_TABLE_CDC.replace(STATEMENTS_PLACEHOLDER, table), schema)); + Lsn minLsn = cdc.queryAndMap(GET_MIN_LSN.replace(STATEMENTS_PLACEHOLDER, table), rs -> new Lsn(rs.getBytes(1))); + logger.info("启用CDC表[{}]:{}", table, minLsn.isAvailable()); + } + } catch (SQLException e) { + logger.error(e.getMessage()); + } + }); + + // 支持UTC + getAllChangesForTable = GET_ALL_CHANGES_FOR_TABLE.replaceFirst(STATEMENTS_PLACEHOLDER, Matcher.quoteReplacement(supportsAtTimeZone ? AT_TIME_ZONE_UTC : "")); + + // 读取增量 + Set changeTables = cdc.queryAndMapList(GET_TABLES_CDC_ENABLED, rs -> { + final Set tables = new HashSet<>(); + while (rs.next()) { + SqlServerChangeTable changeTable = new SqlServerChangeTable( + // schemaName + rs.getString(1), + // tableName + rs.getString(2), + // captureInstance + rs.getString(3), + // changeTableObjectId + rs.getInt(4), + // startLsn + rs.getBytes(6), + // stopLsn + rs.getBytes(7), + // capturedColumns + rs.getString(15)); + logger.info(changeTable.toString()); + tables.add(changeTable); + } + return tables; + }); + logger.info("监听表数:{} ", changeTables.size()); + + if (!CollectionUtils.isEmpty(changeTables)) { + AtomicInteger count = new AtomicInteger(0); + Lsn lastLsn = cdc.queryAndMap(GET_MAX_LSN, rs -> new Lsn(rs.getBytes(1))); + + while (true && count.getAndAdd(1) < 30) { + Lsn stopLsn = cdc.queryAndMap(GET_MAX_LSN, rs -> new Lsn(rs.getBytes(1))); + if (!stopLsn.isAvailable()) { + logger.warn("No maximum LSN recorded in the database; please ensure that the SQL Server Agent is running"); + cdc.pause(); + continue; + } + + if (stopLsn.compareTo(lastLsn) <= 0) { + cdc.pause(); + continue; + } + + Lsn startLsn = getIncrementLsn(cdc, lastLsn); + changeTables.forEach(changeTable -> { + try { + final String query = getAllChangesForTable.replace(STATEMENTS_PLACEHOLDER, changeTable.getCaptureInstance()); + logger.info("Getting changes for table {} in range[{}, {}]", changeTable.getTableName(), startLsn, stopLsn); + + cdc.queryAndMapList(query, statement -> { + statement.setBytes(1, startLsn.getBinary()); + statement.setBytes(2, stopLsn.getBinary()); + }, rs -> { + int columnCount = rs.getMetaData().getColumnCount(); + List> data = new ArrayList<>(columnCount); + List row = null; + while (rs.next()) { + row = new ArrayList<>(columnCount); + for (int i = 1; i <= columnCount; i++) { + row.add(rs.getObject(i)); + } + logger.info("rows:{}", row); + data.add(row); + } + return data; + }); + } catch (Exception e) { + e.printStackTrace(); + } + }); + + lastLsn = stopLsn; + } + } + + // 注销CDC表 + for (String table : tables) { + cdc.execute(String.format(DISABLE_TABLE_CDC.replace(STATEMENTS_PLACEHOLDER, table), schema)); + } + cdc.close(); + } + + public void start() throws SQLException { + String username = "sa"; + String password = "123"; + String url = "jdbc:sqlserver://127.0.0.1:1433;DatabaseName=test"; + schema = "dbo"; + connection = DriverManager.getConnection(url, username, password); + if (connection != null) { + DatabaseMetaData dm = (DatabaseMetaData) connection.getMetaData(); + logger.info("Driver name: " + dm.getDriverName()); + logger.info("Driver version: " + dm.getDriverVersion()); + logger.info("Product name: " + dm.getDatabaseProductName()); + logger.info("Product version: " + dm.getDatabaseProductVersion()); + } + } + + public void close() { + if (null != connection) { + close(connection); + } + } + + private void close(AutoCloseable closeable) { + if (null != closeable) { + try { + closeable.close(); + } catch (Exception e) { + logger.error(e.getMessage()); + } + } + } + + private Lsn getIncrementLsn(ChangeDataCaptureTest cdc, Lsn lastLsn) { + return cdc.queryAndMap(GET_INCREMENT_LSN, statement -> statement.setBytes(1, lastLsn.getBinary()), rs -> Lsn.valueOf(rs.getBytes(1))); + } + + private void pause() throws InterruptedException { + TimeUnit.SECONDS.sleep(2); + } + + private void execute(String... sqlStatements) throws SQLException { + Statement statement = connection.createStatement(); + try { + for (String sqlStatement : sqlStatements) { + if (sqlStatement != null) { + logger.info("executing '{}'", sqlStatement); + statement.execute(sqlStatement); + } + } + } catch (Exception e) { + logger.error(e.getMessage()); + } finally { + close(statement); + } + } + + public interface ResultSetMapper { + T apply(ResultSet rs) throws SQLException; + } + + public interface StatementPreparer { + void accept(PreparedStatement statement) throws SQLException; + } + + public T queryAndMap(String sql, ResultSetMapper mapper) { + return queryAndMap(sql, null, mapper); + } + + public T queryAndMap(String sql, StatementPreparer statementPreparer, ResultSetMapper mapper) { + PreparedStatement ps = null; + ResultSet rs = null; + T apply = null; + try { + ps = connection.prepareStatement(sql); + if (null != statementPreparer) { + statementPreparer.accept(ps); + } + rs = ps.executeQuery(); + if (rs.next()) { + apply = mapper.apply(rs); + } + } catch (Exception e) { + logger.error(e.getMessage()); + } finally { + close(rs); + close(ps); + } + return apply; + } + + public T queryAndMapList(String sql, ResultSetMapper mapper) { + return queryAndMapList(sql, null, mapper); + } + + public T queryAndMapList(String sql, StatementPreparer statementPreparer, ResultSetMapper mapper) { + PreparedStatement ps = null; + ResultSet rs = null; + T apply = null; + try { + ps = connection.prepareStatement(sql); + if (null != statementPreparer) { + statementPreparer.accept(ps); + } + rs = ps.executeQuery(); + apply = mapper.apply(rs); + } catch (SQLServerException e) { + // 为过程或函数 cdc.fn_cdc_get_all_changes_ ... 提供的参数数目不足。 + logger.warn(e.getMessage()); + } catch (Exception e) { + logger.error(e.getMessage()); + } finally { + close(rs); + close(ps); + } + return apply; + } + } \ No newline at end of file diff --git a/dbsyncer-listener/src/main/test/DBChangeNotificationTest.java b/dbsyncer-connector/src/main/test/DBChangeNotificationTest.java similarity index 95% rename from dbsyncer-listener/src/main/test/DBChangeNotificationTest.java rename to dbsyncer-connector/src/main/test/DBChangeNotificationTest.java index dad62485..b6595348 100644 --- a/dbsyncer-listener/src/main/test/DBChangeNotificationTest.java +++ b/dbsyncer-connector/src/main/test/DBChangeNotificationTest.java @@ -1,101 +1,101 @@ -import oracle.jdbc.OracleStatement; -import oracle.jdbc.driver.OracleConnection; -import org.dbsyncer.listener.oracle.dcn.DBChangeNotification; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.*; - -/** - * @version 1.0.0 - * @Author AE86 - * @Date 2021-05-10 22:25 - */ -public class DBChangeNotificationTest { - - private final Logger logger = LoggerFactory.getLogger(getClass()); - - @Test - public void testConnect() throws Exception { - String username = "ae86"; - String password = "123"; - String url = "jdbc:oracle:thin:@127.0.0.1:1521:XE"; - - final DBChangeNotification dcn = new DBChangeNotification(username, password, url); - dcn.addRowEventListener((e) -> - logger.info("{}触发{}, data:{}", e.getSourceTableName(), e.getEvent(), e.getDataList()) - ); - dcn.start(); - - // 模拟并发 - final int threadSize = 301; - final ExecutorService pool = Executors.newFixedThreadPool(threadSize); - final CyclicBarrier barrier = new CyclicBarrier(threadSize); - final CountDownLatch latch = new CountDownLatch(threadSize); - - for (int i = 0; i < threadSize; i++) { - final int k = i + 3; - pool.submit(() -> { - try { - barrier.await(); - //read(k, dcn); - - // 模拟写入操作 - insert(k, dcn); - - } catch (InterruptedException e) { - logger.error(e.getMessage()); - } catch (BrokenBarrierException e) { - logger.error(e.getMessage()); - } finally { - latch.countDown(); - } - }); - } - - try { - latch.await(); - logger.info("try to close"); - } catch (InterruptedException e) { - logger.error(e.getMessage()); - } - pool.shutdown(); - - TimeUnit.SECONDS.sleep(20); - dcn.close(); - logger.info("test end"); - - } - - private void insert(int k, DBChangeNotification dcn) { - OracleConnection conn = dcn.getOracleConnection(); - OracleStatement os = null; - ResultSet rs = null; - try { - os = (OracleStatement) conn.createStatement(); - String sql = "INSERT INTO \"AE86\".\"my_user\"(\"id\", \"name\", \"age\", \"phone\", \"create_date\", \"last_time\", \"money\", \"car\", \"big\", \"clo\", \"rel\") VALUES (" + k + ", '红包', '2', '18200001111', TO_DATE('2015-10-23 00:00:00', 'SYYYY-MM-DD HH24:MI:SS'), TO_TIMESTAMP('2021-01-23 00:00:00.000000', 'SYYYY-MM-DD HH24:MI:SS:FF6'), '200.00000000000000', '4', null, '888', '3.0000000000000000')"; - - int i = os.executeUpdate(sql); - logger.info("insert:{}, {}", k, i); - } catch (SQLException e) { - logger.error(e.getMessage()); - } finally { - dcn.close(rs); - dcn.close(os); - } - } - - private void read(final int k, DBChangeNotification dcn) { - final String tableName = "my_user"; - final String rowId = "AAAE5fAABAAALCJAAx"; - List data = new ArrayList<>(); - dcn.read(tableName, rowId, data); - logger.info("{}, 【{}】, data:{}", k, data.size(), data); - } - +import oracle.jdbc.OracleStatement; +import oracle.jdbc.driver.OracleConnection; +import org.dbsyncer.connector.oracle.dcn.DBChangeNotification; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.*; + +/** + * @version 1.0.0 + * @Author AE86 + * @Date 2021-05-10 22:25 + */ +public class DBChangeNotificationTest { + + private final Logger logger = LoggerFactory.getLogger(getClass()); + + @Test + public void testConnect() throws Exception { + String username = "ae86"; + String password = "123"; + String url = "jdbc:oracle:thin:@127.0.0.1:1521:XE"; + + final DBChangeNotification dcn = new DBChangeNotification(username, password, url); + dcn.addRowEventListener((e) -> + logger.info("{}触发{}, data:{}", e.getSourceTableName(), e.getEvent(), e.getDataList()) + ); + dcn.start(); + + // 模拟并发 + final int threadSize = 301; + final ExecutorService pool = Executors.newFixedThreadPool(threadSize); + final CyclicBarrier barrier = new CyclicBarrier(threadSize); + final CountDownLatch latch = new CountDownLatch(threadSize); + + for (int i = 0; i < threadSize; i++) { + final int k = i + 3; + pool.submit(() -> { + try { + barrier.await(); + //read(k, dcn); + + // 模拟写入操作 + insert(k, dcn); + + } catch (InterruptedException e) { + logger.error(e.getMessage()); + } catch (BrokenBarrierException e) { + logger.error(e.getMessage()); + } finally { + latch.countDown(); + } + }); + } + + try { + latch.await(); + logger.info("try to close"); + } catch (InterruptedException e) { + logger.error(e.getMessage()); + } + pool.shutdown(); + + TimeUnit.SECONDS.sleep(20); + dcn.close(); + logger.info("test end"); + + } + + private void insert(int k, DBChangeNotification dcn) { + OracleConnection conn = dcn.getOracleConnection(); + OracleStatement os = null; + ResultSet rs = null; + try { + os = (OracleStatement) conn.createStatement(); + String sql = "INSERT INTO \"AE86\".\"my_user\"(\"id\", \"name\", \"age\", \"phone\", \"create_date\", \"last_time\", \"money\", \"car\", \"big\", \"clo\", \"rel\") VALUES (" + k + ", '红包', '2', '18200001111', TO_DATE('2015-10-23 00:00:00', 'SYYYY-MM-DD HH24:MI:SS'), TO_TIMESTAMP('2021-01-23 00:00:00.000000', 'SYYYY-MM-DD HH24:MI:SS:FF6'), '200.00000000000000', '4', null, '888', '3.0000000000000000')"; + + int i = os.executeUpdate(sql); + logger.info("insert:{}, {}", k, i); + } catch (SQLException e) { + logger.error(e.getMessage()); + } finally { + dcn.close(rs); + dcn.close(os); + } + } + + private void read(final int k, DBChangeNotification dcn) { + final String tableName = "my_user"; + final String rowId = "AAAE5fAABAAALCJAAx"; + List data = new ArrayList<>(); + dcn.read(tableName, rowId, data); + logger.info("{}, 【{}】, data:{}", k, data.size(), data); + } + } \ No newline at end of file diff --git a/dbsyncer-listener/src/main/test/ESClientTest.java b/dbsyncer-connector/src/main/test/ESClientTest.java similarity index 100% rename from dbsyncer-listener/src/main/test/ESClientTest.java rename to dbsyncer-connector/src/main/test/ESClientTest.java diff --git a/dbsyncer-listener/src/main/test/FileWatchTest.java b/dbsyncer-connector/src/main/test/FileWatchTest.java similarity index 100% rename from dbsyncer-listener/src/main/test/FileWatchTest.java rename to dbsyncer-connector/src/main/test/FileWatchTest.java diff --git a/dbsyncer-listener/src/main/test/KafkaClientTest.java b/dbsyncer-connector/src/main/test/KafkaClientTest.java similarity index 100% rename from dbsyncer-listener/src/main/test/KafkaClientTest.java rename to dbsyncer-connector/src/main/test/KafkaClientTest.java diff --git a/dbsyncer-listener/src/main/test/LinkedBlockingQueueTest.java b/dbsyncer-connector/src/main/test/LinkedBlockingQueueTest.java similarity index 90% rename from dbsyncer-listener/src/main/test/LinkedBlockingQueueTest.java rename to dbsyncer-connector/src/main/test/LinkedBlockingQueueTest.java index 9874752c..c60d4e26 100644 --- a/dbsyncer-listener/src/main/test/LinkedBlockingQueueTest.java +++ b/dbsyncer-connector/src/main/test/LinkedBlockingQueueTest.java @@ -1,86 +1,86 @@ -import oracle.jdbc.dcn.TableChangeDescription; -import org.dbsyncer.common.util.RandomUtil; -import org.dbsyncer.listener.oracle.event.DCNEvent; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; - -public class LinkedBlockingQueueTest { - - private final Logger logger = LoggerFactory.getLogger(getClass()); - private BlockingQueue queue = new LinkedBlockingQueue<>(10); - - @Test - public void testProducerAndConsumer() throws InterruptedException { - logger.info("test begin"); - new Producer(queue).start(); - new Consumer(queue).start(); - new Consumer(queue).start(); - TimeUnit.SECONDS.sleep(60); - logger.info("test end"); - } - - /** - * 生产 - */ - class Producer extends Thread { - - BlockingQueue queue; - int taskNumber = 50; - - public Producer(BlockingQueue queue) { - setName("Producer-thread"); - this.queue = queue; - } - - @Override - public void run() { - logger.info("生产线程{}开始工作", Thread.currentThread().getName()); - for (int i = 0; i < taskNumber; i++) { - DCNEvent event = new DCNEvent("my_user" + i, "AAAF8BAABAAALJBAAA", TableChangeDescription.TableOperation.INSERT.getCode()); - try { - // 如果BlockQueue没有空间,则调用此方法的线程被阻断直到BlockingQueue里面有空间再继续 - queue.put(event); - } catch (InterruptedException e) { - logger.error("添加消息：{}, 失败", event, e.getMessage()); - } - } - logger.info("生产线程{}结束工作", Thread.currentThread().getName()); - } - } - - /** - * 消费 - */ - class Consumer extends Thread { - - BlockingQueue queue; - - public Consumer(BlockingQueue queue) { - setName("Consumer-thread-" + RandomUtil.nextInt(1, 100)); - this.queue = queue; - } - - @Override - public void run() { - String threadName = Thread.currentThread().getName(); - logger.info("消费线程{}开始工作", threadName); - while (true) { - try { - // 模拟耗时 - TimeUnit.SECONDS.sleep(RandomUtil.nextInt(0, 3)); - // 取走BlockingQueue里排在首位的对象,若BlockingQueue为空,阻断进入等待状态直到Blocking有新的对象被加入为止 - DCNEvent event = queue.take(); - logger.error("消费线程{}接受消息：{}", threadName, event.getTableName()); - } catch (InterruptedException e) { - logger.error(e.getMessage()); - } - } - } - } - +import oracle.jdbc.dcn.TableChangeDescription; +import org.dbsyncer.common.util.RandomUtil; +import org.dbsyncer.connector.oracle.DCNEvent; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; + +public class LinkedBlockingQueueTest { + + private final Logger logger = LoggerFactory.getLogger(getClass()); + private BlockingQueue queue = new LinkedBlockingQueue<>(10); + + @Test + public void testProducerAndConsumer() throws InterruptedException { + logger.info("test begin"); + new Producer(queue).start(); + new Consumer(queue).start(); + new Consumer(queue).start(); + TimeUnit.SECONDS.sleep(60); + logger.info("test end"); + } + + /** + * 生产 + */ + class Producer extends Thread { + + BlockingQueue queue; + int taskNumber = 50; + + public Producer(BlockingQueue queue) { + setName("Producer-thread"); + this.queue = queue; + } + + @Override + public void run() { + logger.info("生产线程{}开始工作", Thread.currentThread().getName()); + for (int i = 0; i < taskNumber; i++) { + DCNEvent event = new DCNEvent("my_user" + i, "AAAF8BAABAAALJBAAA", TableChangeDescription.TableOperation.INSERT.getCode()); + try { + // 如果BlockQueue没有空间,则调用此方法的线程被阻断直到BlockingQueue里面有空间再继续 + queue.put(event); + } catch (InterruptedException e) { + logger.error("添加消息：{}, 失败", event, e.getMessage()); + } + } + logger.info("生产线程{}结束工作", Thread.currentThread().getName()); + } + } + + /** + * 消费 + */ + class Consumer extends Thread { + + BlockingQueue queue; + + public Consumer(BlockingQueue queue) { + setName("Consumer-thread-" + RandomUtil.nextInt(1, 100)); + this.queue = queue; + } + + @Override + public void run() { + String threadName = Thread.currentThread().getName(); + logger.info("消费线程{}开始工作", threadName); + while (true) { + try { + // 模拟耗时 + TimeUnit.SECONDS.sleep(RandomUtil.nextInt(0, 3)); + // 取走BlockingQueue里排在首位的对象,若BlockingQueue为空,阻断进入等待状态直到Blocking有新的对象被加入为止 + DCNEvent event = queue.take(); + logger.error("消费线程{}接受消息：{}", threadName, event.getTableName()); + } catch (InterruptedException e) { + logger.error(e.getMessage()); + } + } + } + } + } \ No newline at end of file diff --git a/dbsyncer-listener/src/main/test/PGReplicationTest.java b/dbsyncer-connector/src/main/test/PGReplicationTest.java similarity index 100% rename from dbsyncer-listener/src/main/test/PGReplicationTest.java rename to dbsyncer-connector/src/main/test/PGReplicationTest.java diff --git a/dbsyncer-listener/pom.xml b/dbsyncer-listener/pom.xml deleted file mode 100644 index b76ac45f..00000000 --- a/dbsyncer-listener/pom.xml +++ /dev/null @@ -1,43 +0,0 @@ - - - - dbsyncer - org.ghi - 2.0.0 - - 4.0.0 - dbsyncer-listener - - - - - org.ghi - dbsyncer-connector - ${project.parent.version} - - - - com.github.jsqlparser - jsqlparser - - - - com.zendesk - mysql-binlog-connector-java - - - - org.springframework.boot - spring-boot-starter-log4j2 - - - - junit - junit - provided - - - - \ No newline at end of file diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/Listener.java b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/Listener.java deleted file mode 100644 index 5704391c..00000000 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/Listener.java +++ /dev/null @@ -1,9 +0,0 @@ -package org.dbsyncer.listener; - -import org.dbsyncer.listener.enums.ListenerTypeEnum; - -public interface Listener { - - T getExtractor(ListenerTypeEnum listenerTypeEnum, String connectorType, Class valueType) throws IllegalAccessException, InstantiationException; - -} \ No newline at end of file diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/ListenerException.java b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/ListenerException.java deleted file mode 100644 index e8e054e4..00000000 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/ListenerException.java +++ /dev/null @@ -1,27 +0,0 @@ -package org.dbsyncer.listener; - -/** - * @author AE86 - * @version 1.0.0 - * @date 2019/9/28 22:39 - */ -public class ListenerException extends RuntimeException { - - private static final long serialVersionUID = 1L; - - public ListenerException(String message) { - super(message); - } - - public ListenerException(String message, Throwable cause) { - super(message, cause); - } - - public ListenerException(Throwable cause) { - super(cause); - } - - protected ListenerException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { - super(message, cause, enableSuppression, writableStackTrace); - } -} diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/ListenerFactory.java b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/ListenerFactory.java deleted file mode 100644 index 55efab3c..00000000 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/ListenerFactory.java +++ /dev/null @@ -1,26 +0,0 @@ -package org.dbsyncer.listener; - -import org.dbsyncer.listener.enums.ListenerTypeEnum; -import org.springframework.stereotype.Component; - -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.function.Function; - -@Component -public class ListenerFactory implements Listener { - - private Map> map = new LinkedHashMap<>(); - - @Override - public T getExtractor(ListenerTypeEnum listenerTypeEnum, String connectorType, Class valueType) throws IllegalAccessException, InstantiationException { - Function function = map.get(listenerTypeEnum); - if (null == function) { - throw new ListenerException(String.format("Unsupported type \"%s\" for extractor \"%s\".", listenerTypeEnum, connectorType)); - } - - Class clazz = (Class) function.apply(connectorType); - return clazz.newInstance(); - } - -} \ No newline at end of file diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/config/Host.java b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/config/Host.java deleted file mode 100644 index 5f308f52..00000000 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/config/Host.java +++ /dev/null @@ -1,19 +0,0 @@ -package org.dbsyncer.listener.config; - -public class Host { - private String ip; - private int port; - - public Host(String ip, int port) { - this.ip = ip; - this.port = port; - } - - public String getIp() { - return ip; - } - - public int getPort() { - return port; - } -} \ No newline at end of file diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/kafka/KafkaExtractor.java b/dbsyncer-listener/src/main/java/org/dbsyncer/listener/kafka/KafkaExtractor.java deleted file mode 100644 index a9301d90..00000000 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/kafka/KafkaExtractor.java +++ /dev/null @@ -1,22 +0,0 @@ -package org.dbsyncer.listener.kafka; - -import org.dbsyncer.listener.AbstractExtractor; -import org.dbsyncer.listener.ListenerException; - -/** - * @author AE86 - * @version 1.0.0 - * @date 2021/12/18 22:19 - */ -public class KafkaExtractor extends AbstractExtractor { - - @Override - public void start() { - throw new ListenerException("抱歉，kafka监听功能正在开发中..."); - } - - @Override - public void close() { - - } -} diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/IncrementPuller.java b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/IncrementPuller.java index df4d2430..f0073726 100644 --- a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/IncrementPuller.java +++ b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/IncrementPuller.java @@ -1,17 +1,13 @@ package org.dbsyncer.manager.impl; import org.dbsyncer.common.util.CollectionUtils; +import org.dbsyncer.connector.AbstractListener; import org.dbsyncer.connector.ConnectorFactory; +import org.dbsyncer.connector.config.ListenerConfig; +import org.dbsyncer.connector.quartz.AbstractQuartzListener; +import org.dbsyncer.connector.quartz.TableGroupQuartzCommand; import org.dbsyncer.connector.scheduled.ScheduledTaskJob; import org.dbsyncer.connector.scheduled.ScheduledTaskService; -import org.dbsyncer.listener.AbstractExtractor; -import org.dbsyncer.listener.Extractor; -import org.dbsyncer.listener.Listener; -import org.dbsyncer.listener.config.ListenerConfig; -import org.dbsyncer.listener.enums.ListenerTypeEnum; -import org.dbsyncer.listener.model.ChangedOffset; -import org.dbsyncer.listener.quartz.AbstractQuartzExtractor; -import org.dbsyncer.listener.quartz.TableGroupQuartzCommand; import org.dbsyncer.manager.AbstractPuller; import org.dbsyncer.manager.ManagerException; import org.dbsyncer.parser.LogService; @@ -25,6 +21,9 @@ import org.dbsyncer.parser.model.Connector; import org.dbsyncer.parser.model.Mapping; import org.dbsyncer.parser.model.Meta; import org.dbsyncer.parser.model.TableGroup; +import org.dbsyncer.sdk.enums.ListenerTypeEnum; +import org.dbsyncer.sdk.listener.Listener; +import org.dbsyncer.sdk.model.ChangedOffset; import org.dbsyncer.sdk.model.ConnectorConfig; import org.dbsyncer.sdk.model.Table; import org.slf4j.Logger; @@ -57,13 +56,7 @@ public final class IncrementPuller extends AbstractPuller implements Application private final Logger logger = LoggerFactory.getLogger(getClass()); @Resource - private Listener listener; - - @Resource - private ProfileComponent profileComponent; - - @Resource - private LogService logService; + private BufferActuatorRouter bufferActuatorRouter; @Resource private ScheduledTaskService scheduledTaskService; @@ -72,9 +65,12 @@ public final class IncrementPuller extends AbstractPuller implements Application private ConnectorFactory connectorFactory; @Resource - private BufferActuatorRouter bufferActuatorRouter; + private ProfileComponent profileComponent; - private Map map = new ConcurrentHashMap<>(); + @Resource + private LogService logService; + + private Map map = new ConcurrentHashMap<>(); @PostConstruct private void init() { @@ -99,7 +95,7 @@ public final class IncrementPuller extends AbstractPuller implements Application meta.setBeginTime(now); meta.setEndTime(now); profileComponent.editConfigModel(meta); - map.putIfAbsent(metaId, getExtractor(mapping, connector, list, meta)); + map.putIfAbsent(metaId, getListener(mapping, connector, list, meta)); map.get(metaId).start(); } catch (Exception e) { close(metaId); @@ -114,10 +110,10 @@ public final class IncrementPuller extends AbstractPuller implements Application @Override public void close(String metaId) { - Extractor extractor = map.get(metaId); - if (null != extractor) { + Listener listener = map.get(metaId); + if (null != listener) { bufferActuatorRouter.unbind(metaId); - extractor.close(); + listener.close(); } map.remove(metaId); publishClosedEvent(metaId); @@ -139,31 +135,34 @@ public final class IncrementPuller extends AbstractPuller implements Application @Override public void run() { // 定时同步增量信息 - map.values().forEach(extractor -> extractor.flushEvent()); + map.values().forEach(listener -> listener.flushEvent()); } - private AbstractExtractor getExtractor(Mapping mapping, Connector connector, List list, Meta meta) throws InstantiationException, IllegalAccessException { + private Listener getListener(Mapping mapping, Connector connector, List list, Meta meta) { ConnectorConfig connectorConfig = connector.getConfig(); ListenerConfig listenerConfig = mapping.getListener(); - // timing/log - final String listenerType = listenerConfig.getListenerType(); + String listenerType = listenerConfig.getListenerType(); + + Listener listener = connectorFactory.getListener(connectorConfig.getConnectorType(), listenerType); + if (null == listener) { + throw new ManagerException(String.format("Unsupported listener type \"%s\".", connectorConfig.getConnectorType())); + } - AbstractExtractor extractor = null; // 默认定时抽取 - if (ListenerTypeEnum.isTiming(listenerType)) { - AbstractQuartzExtractor quartzExtractor = listener.getExtractor(ListenerTypeEnum.TIMING, connectorConfig.getConnectorType(), AbstractQuartzExtractor.class); - quartzExtractor.setCommands(list.stream().map(t -> new TableGroupQuartzCommand(t.getSourceTable(), t.getCommand())).collect(Collectors.toList())); - quartzExtractor.register(new QuartzConsumer().init(bufferActuatorRouter, profileComponent, logService, meta, mapping, list)); - extractor = quartzExtractor; + if (ListenerTypeEnum.isTiming(listenerType) && listener instanceof AbstractQuartzListener) { + AbstractQuartzListener quartzListener = (AbstractQuartzListener) listener; + quartzListener.setCommands(list.stream().map(t -> new TableGroupQuartzCommand(t.getSourceTable(), t.getCommand())).collect(Collectors.toList())); + quartzListener.register(new QuartzConsumer().init(bufferActuatorRouter, profileComponent, logService, meta, mapping, list)); } // 基于日志抽取 - if (ListenerTypeEnum.isLog(listenerType)) { - extractor = listener.getExtractor(ListenerTypeEnum.LOG, connectorConfig.getConnectorType(), AbstractExtractor.class); - extractor.register(new LogConsumer().init(bufferActuatorRouter, profileComponent, logService, meta, mapping, list)); + if (ListenerTypeEnum.isLog(listenerType) && listener instanceof AbstractListener) { + AbstractListener abstractListener = (AbstractListener) listener; + abstractListener.register(new LogConsumer().init(bufferActuatorRouter, profileComponent, logService, meta, mapping, list)); } - if (null != extractor) { + if (listener instanceof AbstractListener) { + AbstractListener abstractListener = (AbstractListener) listener; Set filterTable = new HashSet<>(); List

    sourceTable = new ArrayList<>(); list.forEach(t -> { @@ -174,18 +173,17 @@ public final class IncrementPuller extends AbstractPuller implements Application filterTable.add(table.getName()); }); - extractor.setConnectorFactory(connectorFactory); - extractor.setScheduledTaskService(scheduledTaskService); - extractor.setConnectorConfig(connectorConfig); - extractor.setListenerConfig(listenerConfig); - extractor.setFilterTable(filterTable); - extractor.setSourceTable(sourceTable); - extractor.setSnapshot(meta.getSnapshot()); - extractor.setMetaId(meta.getId()); - return extractor; + abstractListener.setConnectorFactory(connectorFactory); + abstractListener.setScheduledTaskService(scheduledTaskService); + abstractListener.setConnectorConfig(connectorConfig); + abstractListener.setListenerConfig(listenerConfig); + abstractListener.setFilterTable(filterTable); + abstractListener.setSourceTable(sourceTable); + abstractListener.setSnapshot(meta.getSnapshot()); + abstractListener.setMetaId(meta.getId()); } - throw new ManagerException("未知的监听配置."); + return listener; } } \ No newline at end of file diff --git a/dbsyncer-parser/pom.xml b/dbsyncer-parser/pom.xml index b578b146..8acf90b8 100644 --- a/dbsyncer-parser/pom.xml +++ b/dbsyncer-parser/pom.xml @@ -11,13 +11,6 @@ dbsyncer-parser - - - org.ghi - dbsyncer-listener - ${project.parent.version} - - org.ghi diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ProfileComponent.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ProfileComponent.java index 4358e4c4..ca60997c 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ProfileComponent.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ProfileComponent.java @@ -4,7 +4,7 @@ package org.dbsyncer.parser; import org.dbsyncer.connector.enums.FilterEnum; -import org.dbsyncer.listener.enums.QuartzFilterEnum; +import org.dbsyncer.connector.enums.QuartzFilterEnum; import org.dbsyncer.parser.enums.ConvertEnum; import org.dbsyncer.parser.model.ConfigModel; import org.dbsyncer.parser.model.Connector; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/AbstractConsumer.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/AbstractConsumer.java index 680d498a..87027b77 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/AbstractConsumer.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/AbstractConsumer.java @@ -3,9 +3,9 @@ */ package org.dbsyncer.parser.consumer; -import org.dbsyncer.listener.ChangedEvent; -import org.dbsyncer.listener.event.DDLChangedEvent; -import org.dbsyncer.listener.Watcher; +import org.dbsyncer.sdk.listener.ChangedEvent; +import org.dbsyncer.sdk.listener.event.DDLChangedEvent; +import org.dbsyncer.sdk.listener.Watcher; import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.parser.flush.impl.BufferActuatorRouter; import org.dbsyncer.parser.LogService; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/impl/LogConsumer.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/impl/LogConsumer.java index 2fdc61db..026012cd 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/impl/LogConsumer.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/impl/LogConsumer.java @@ -3,9 +3,9 @@ */ package org.dbsyncer.parser.consumer.impl; -import org.dbsyncer.listener.event.CommonChangedEvent; -import org.dbsyncer.listener.event.DDLChangedEvent; -import org.dbsyncer.listener.event.RowChangedEvent; +import org.dbsyncer.sdk.listener.event.CommonChangedEvent; +import org.dbsyncer.sdk.listener.event.DDLChangedEvent; +import org.dbsyncer.sdk.listener.event.RowChangedEvent; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.parser.consumer.AbstractConsumer; import org.dbsyncer.parser.model.FieldPicker; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/impl/QuartzConsumer.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/impl/QuartzConsumer.java index 839c6c6e..f1cac10a 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/impl/QuartzConsumer.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/impl/QuartzConsumer.java @@ -3,7 +3,7 @@ */ package org.dbsyncer.parser.consumer.impl; -import org.dbsyncer.listener.event.ScanChangedEvent; +import org.dbsyncer.sdk.listener.event.ScanChangedEvent; import org.dbsyncer.parser.consumer.AbstractConsumer; import org.dbsyncer.parser.model.FieldPicker; import org.dbsyncer.parser.model.TableGroup; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/event/RefreshOffsetEvent.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/event/RefreshOffsetEvent.java index 87809d4f..af8e76b9 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/event/RefreshOffsetEvent.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/event/RefreshOffsetEvent.java @@ -1,6 +1,6 @@ package org.dbsyncer.parser.event; -import org.dbsyncer.listener.model.ChangedOffset; +import org.dbsyncer.sdk.model.ChangedOffset; import org.springframework.context.ApplicationContext; import org.springframework.context.event.ApplicationContextEvent; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/BufferActuatorRouter.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/BufferActuatorRouter.java index e9828654..9b1b221f 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/BufferActuatorRouter.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/BufferActuatorRouter.java @@ -4,7 +4,7 @@ package org.dbsyncer.parser.flush.impl; import org.dbsyncer.common.config.TableGroupBufferConfig; -import org.dbsyncer.listener.ChangedEvent; +import org.dbsyncer.sdk.listener.ChangedEvent; import org.dbsyncer.parser.flush.BufferActuator; import org.dbsyncer.parser.model.WriterRequest; import org.slf4j.Logger; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/ProfileComponentImpl.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/ProfileComponentImpl.java index 3f617efd..3dc02c89 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/ProfileComponentImpl.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/ProfileComponentImpl.java @@ -7,7 +7,7 @@ import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.JsonUtil; import org.dbsyncer.connector.ConnectorFactory; import org.dbsyncer.connector.enums.FilterEnum; -import org.dbsyncer.listener.enums.QuartzFilterEnum; +import org.dbsyncer.connector.enums.QuartzFilterEnum; import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.parser.enums.CommandEnum; import org.dbsyncer.parser.enums.ConvertEnum; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/Mapping.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/Mapping.java index 06c01af2..d72d68c9 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/Mapping.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/Mapping.java @@ -1,6 +1,6 @@ package org.dbsyncer.parser.model; -import org.dbsyncer.listener.config.ListenerConfig; +import org.dbsyncer.connector.config.ListenerConfig; import org.dbsyncer.sdk.enums.ModelEnum; import org.dbsyncer.sdk.model.Field; import org.dbsyncer.storage.constant.ConfigConstant; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/WriterRequest.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/WriterRequest.java index 81b1d4a7..dcb4543c 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/WriterRequest.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/WriterRequest.java @@ -1,8 +1,8 @@ package org.dbsyncer.parser.model; -import org.dbsyncer.listener.ChangedEvent; -import org.dbsyncer.listener.model.ChangedOffset; -import org.dbsyncer.listener.event.DDLChangedEvent; +import org.dbsyncer.sdk.listener.ChangedEvent; +import org.dbsyncer.sdk.model.ChangedOffset; +import org.dbsyncer.sdk.listener.event.DDLChangedEvent; import org.dbsyncer.parser.flush.BufferRequest; import java.util.Map; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/WriterResponse.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/WriterResponse.java index 526a89a0..099ac868 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/WriterResponse.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/WriterResponse.java @@ -1,6 +1,6 @@ package org.dbsyncer.parser.model; -import org.dbsyncer.listener.model.ChangedOffset; +import org.dbsyncer.sdk.model.ChangedOffset; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.parser.flush.BufferResponse; diff --git a/dbsyncer-sdk/pom.xml b/dbsyncer-sdk/pom.xml index 8e8e3711..d6fc5968 100644 --- a/dbsyncer-sdk/pom.xml +++ b/dbsyncer-sdk/pom.xml @@ -22,18 +22,21 @@ org.springframework spring-jdbc + provided com.oracle ojdbc6 + provided org.postgresql postgresql + provided diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/enums/ListenerTypeEnum.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/ListenerTypeEnum.java similarity index 94% rename from dbsyncer-listener/src/main/java/org/dbsyncer/listener/enums/ListenerTypeEnum.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/ListenerTypeEnum.java index 1a1ec17e..ab7a1262 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/enums/ListenerTypeEnum.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/ListenerTypeEnum.java @@ -1,4 +1,4 @@ -package org.dbsyncer.listener.enums; +package org.dbsyncer.sdk.enums; import org.dbsyncer.common.util.StringUtil; diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/ChangedEvent.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/ChangedEvent.java similarity index 80% rename from dbsyncer-listener/src/main/java/org/dbsyncer/listener/ChangedEvent.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/ChangedEvent.java index cb9a60cb..5a2b62d8 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/ChangedEvent.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/ChangedEvent.java @@ -1,9 +1,9 @@ /** - * DBSyncer Copyright 2019-2024 All Rights Reserved. + * DBSyncer Copyright 2019-2023 All Rights Reserved. */ -package org.dbsyncer.listener; +package org.dbsyncer.sdk.listener; -import org.dbsyncer.listener.model.ChangedOffset; +import org.dbsyncer.sdk.model.ChangedOffset; import java.util.Map; diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/Extractor.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/Listener.java similarity index 69% rename from dbsyncer-listener/src/main/java/org/dbsyncer/listener/Extractor.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/Listener.java index f1e9b8dc..abe1681f 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/Extractor.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/Listener.java @@ -1,9 +1,18 @@ -package org.dbsyncer.listener; - -import org.dbsyncer.listener.model.ChangedOffset; - -public interface Extractor { - +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.sdk.listener; + +import org.dbsyncer.sdk.model.ChangedOffset; + +/** + * 监听器接口，提供实现增量同步功能，支持定时和日志解析 + * + * @Author AE86 + * @Version 1.0.0 + * @Date 2023-11-21 22:48 + */ +public interface Listener { /** * 启动定时/日志抽取任务 */ @@ -51,5 +60,4 @@ public interface Extractor { * @param e */ void errorEvent(Exception e); - } \ No newline at end of file diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/Watcher.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/Watcher.java similarity index 94% rename from dbsyncer-listener/src/main/java/org/dbsyncer/listener/Watcher.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/Watcher.java index 9d41c055..1328a54a 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/Watcher.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/Watcher.java @@ -1,4 +1,4 @@ -package org.dbsyncer.listener; +package org.dbsyncer.sdk.listener; import java.util.Map; diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/event/CommonChangedEvent.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/event/CommonChangedEvent.java similarity index 91% rename from dbsyncer-listener/src/main/java/org/dbsyncer/listener/event/CommonChangedEvent.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/event/CommonChangedEvent.java index 570e175e..a8f95df7 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/event/CommonChangedEvent.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/event/CommonChangedEvent.java @@ -1,10 +1,10 @@ /** * DBSyncer Copyright 2019-2024 All Rights Reserved. */ -package org.dbsyncer.listener.event; +package org.dbsyncer.sdk.listener.event; -import org.dbsyncer.listener.ChangedEvent; -import org.dbsyncer.listener.model.ChangedOffset; +import org.dbsyncer.sdk.listener.ChangedEvent; +import org.dbsyncer.sdk.model.ChangedOffset; import java.util.Map; diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/event/DDLChangedEvent.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/event/DDLChangedEvent.java similarity index 86% rename from dbsyncer-listener/src/main/java/org/dbsyncer/listener/event/DDLChangedEvent.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/event/DDLChangedEvent.java index 577bbecb..5cf81f36 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/event/DDLChangedEvent.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/event/DDLChangedEvent.java @@ -1,4 +1,4 @@ -package org.dbsyncer.listener.event; +package org.dbsyncer.sdk.listener.event; /** * DDL变更事件 @@ -7,7 +7,7 @@ package org.dbsyncer.listener.event; * @Author AE86 * @Date 2023-09-18 23:00 */ -public class DDLChangedEvent extends CommonChangedEvent { +public final class DDLChangedEvent extends CommonChangedEvent { /** * 变更数据库 diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/event/RowChangedEvent.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/event/RowChangedEvent.java similarity index 95% rename from dbsyncer-listener/src/main/java/org/dbsyncer/listener/event/RowChangedEvent.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/event/RowChangedEvent.java index 3f2d922a..307b710d 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/event/RowChangedEvent.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/event/RowChangedEvent.java @@ -1,7 +1,7 @@ /** * DBSyncer Copyright 2019-2024 All Rights Reserved. */ -package org.dbsyncer.listener.event; +package org.dbsyncer.sdk.listener.event; import java.util.List; diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/event/ScanChangedEvent.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/event/ScanChangedEvent.java similarity index 92% rename from dbsyncer-listener/src/main/java/org/dbsyncer/listener/event/ScanChangedEvent.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/event/ScanChangedEvent.java index 3fc58218..07abad00 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/event/ScanChangedEvent.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/event/ScanChangedEvent.java @@ -1,7 +1,7 @@ /** * DBSyncer Copyright 2019-2024 All Rights Reserved. */ -package org.dbsyncer.listener.event; +package org.dbsyncer.sdk.listener.event; import java.util.Map; diff --git a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/model/ChangedOffset.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/ChangedOffset.java similarity index 96% rename from dbsyncer-listener/src/main/java/org/dbsyncer/listener/model/ChangedOffset.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/ChangedOffset.java index d0050946..a7276a88 100644 --- a/dbsyncer-listener/src/main/java/org/dbsyncer/listener/model/ChangedOffset.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/ChangedOffset.java @@ -1,4 +1,4 @@ -package org.dbsyncer.listener.model; +package org.dbsyncer.sdk.model; /** * 增量偏移量 diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/ConnectorService.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/ConnectorService.java index 3d8d710a..93eeb172 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/ConnectorService.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/ConnectorService.java @@ -7,6 +7,8 @@ import org.dbsyncer.sdk.config.DDLConfig; import org.dbsyncer.sdk.config.ReaderConfig; import org.dbsyncer.sdk.config.WriterBatchConfig; import org.dbsyncer.sdk.connector.ConnectorInstance; +import org.dbsyncer.sdk.enums.ListenerTypeEnum; +import org.dbsyncer.sdk.listener.Listener; import org.dbsyncer.sdk.model.ConnectorConfig; import org.dbsyncer.sdk.model.MetaInfo; import org.dbsyncer.sdk.model.Table; @@ -153,4 +155,11 @@ public interface ConnectorService getTargetCommand(CommandConfig commandConfig); + /** + * 获取监听器 + * + * @param listenerType {@link ListenerTypeEnum} + * @return + */ + Listener getListener(String listenerType); } \ No newline at end of file diff --git a/dbsyncer-web/src/main/resources/application.properties b/dbsyncer-web/src/main/resources/application.properties index 525d92ad..c33a57f1 100644 --- a/dbsyncer-web/src/main/resources/application.properties +++ b/dbsyncer-web/src/main/resources/application.properties @@ -46,8 +46,8 @@ dbsyncer.parser.table.group.buffer-period-millisecond=300 #storage # 是否使用MySQL存储配置（false-关闭; true-开启） # false: 保存磁盘/data/config(驱动配置)|data(按驱动分别存储增量数据)|log(系统日志)} -dbsyncer.storage.support.mysql.enabled=true -dbsyncer.storage.support.mysql.config.url=jdbc:mysql://127.0.0.1:3305/dbsyncer?rewriteBatchedStatements=true&seUnicode=true&characterEncoding=UTF8&serverTimezone=Asia/Shanghai&useSSL=false&verifyServerCertificate=false&autoReconnect=true +dbsyncer.storage.support.mysql.enabled=false +dbsyncer.storage.support.mysql.config.url=jdbc:mysql://127.0.0.1:3306/dbsyncer?rewriteBatchedStatements=true&seUnicode=true&characterEncoding=UTF8&serverTimezone=Asia/Shanghai&useSSL=false&verifyServerCertificate=false&autoReconnect=true dbsyncer.storage.support.mysql.config.username=root dbsyncer.storage.support.mysql.config.password=123 # [StorageBufferActuator]线程数 diff --git a/pom.xml b/pom.xml index ab9f1691..0cf612bb 100644 --- a/pom.xml +++ b/pom.xml @@ -17,7 +17,6 @@ dbsyncer-cluster dbsyncer-common dbsyncer-connector - dbsyncer-listener dbsyncer-manager dbsyncer-parser dbsyncer-plugin -- Gitee From d15a1020cf59c6a32160909a7d4cdcea8bde8af9 Mon Sep 17 00:00:00 2001 From: AE86 <836391306@qq.com> Date: Wed, 22 Nov 2023 01:29:35 +0800 Subject: [PATCH 24/98] fix bug --- .../impl/mapping/LogConfigChecker.java | 2 +- .../checker/impl/mapping/MappingChecker.java | 2 +- .../impl/mapping/TimingConfigChecker.java | 2 +- .../dbsyncer/connector/AbstractListener.java | 4 +-- .../manager/impl/IncrementPuller.java | 6 ++-- .../parser/consumer/AbstractConsumer.java | 22 +++++++++------ .../parser/flush/AbstractFlushStrategy.java | 28 ++++++++++--------- .../org/dbsyncer/parser/model/Mapping.java | 2 +- .../dbsyncer/sdk}/config/ListenerConfig.java | 2 +- 9 files changed, 38 insertions(+), 32 deletions(-) rename {dbsyncer-connector/src/main/java/org/dbsyncer/connector => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/config/ListenerConfig.java (98%) diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/mapping/LogConfigChecker.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/mapping/LogConfigChecker.java index 38a664c4..418ef9b4 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/mapping/LogConfigChecker.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/mapping/LogConfigChecker.java @@ -1,7 +1,7 @@ package org.dbsyncer.biz.checker.impl.mapping; import org.dbsyncer.biz.checker.MappingConfigChecker; -import org.dbsyncer.connector.config.ListenerConfig; +import org.dbsyncer.sdk.config.ListenerConfig; import org.dbsyncer.sdk.enums.ListenerTypeEnum; import org.dbsyncer.parser.model.Mapping; import org.springframework.stereotype.Component; diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/mapping/MappingChecker.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/mapping/MappingChecker.java index 6d558ddc..dec9cd1b 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/mapping/MappingChecker.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/mapping/MappingChecker.java @@ -8,7 +8,7 @@ import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.JsonUtil; import org.dbsyncer.common.util.NumberUtil; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.config.ListenerConfig; +import org.dbsyncer.sdk.config.ListenerConfig; import org.dbsyncer.sdk.enums.ListenerTypeEnum; import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.sdk.enums.ModelEnum; diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/mapping/TimingConfigChecker.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/mapping/TimingConfigChecker.java index 5f714b31..24397168 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/mapping/TimingConfigChecker.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/mapping/TimingConfigChecker.java @@ -2,7 +2,7 @@ package org.dbsyncer.biz.checker.impl.mapping; import org.dbsyncer.biz.checker.MappingConfigChecker; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.config.ListenerConfig; +import org.dbsyncer.sdk.config.ListenerConfig; import org.dbsyncer.sdk.enums.ListenerTypeEnum; import org.dbsyncer.parser.model.Mapping; import org.springframework.stereotype.Component; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/AbstractListener.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/AbstractListener.java index 3f18fce4..a541b48b 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/AbstractListener.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/AbstractListener.java @@ -1,8 +1,8 @@ package org.dbsyncer.connector; import org.dbsyncer.common.util.CollectionUtils; -import org.dbsyncer.connector.config.ListenerConfig; import org.dbsyncer.connector.scheduled.ScheduledTaskService; +import org.dbsyncer.sdk.config.ListenerConfig; import org.dbsyncer.sdk.constant.ConnectorConstant; import org.dbsyncer.sdk.listener.ChangedEvent; import org.dbsyncer.sdk.listener.Listener; @@ -28,6 +28,7 @@ import java.util.concurrent.TimeUnit; public abstract class AbstractListener implements Listener { private final Logger logger = LoggerFactory.getLogger(getClass()); + private final int FLUSH_DELAYED_SECONDS = 20; protected ConnectorFactory connectorFactory; protected ScheduledTaskService scheduledTaskService; protected ConnectorConfig connectorConfig; @@ -37,7 +38,6 @@ public abstract class AbstractListener implements Listener { protected Map snapshot; protected String metaId; private Watcher watcher; - private final int FLUSH_DELAYED_SECONDS = 20; @Override public void register(Watcher watcher) { diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/IncrementPuller.java b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/IncrementPuller.java index f0073726..cb2a848e 100644 --- a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/IncrementPuller.java +++ b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/IncrementPuller.java @@ -3,7 +3,6 @@ package org.dbsyncer.manager.impl; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.connector.AbstractListener; import org.dbsyncer.connector.ConnectorFactory; -import org.dbsyncer.connector.config.ListenerConfig; import org.dbsyncer.connector.quartz.AbstractQuartzListener; import org.dbsyncer.connector.quartz.TableGroupQuartzCommand; import org.dbsyncer.connector.scheduled.ScheduledTaskJob; @@ -21,6 +20,7 @@ import org.dbsyncer.parser.model.Connector; import org.dbsyncer.parser.model.Mapping; import org.dbsyncer.parser.model.Meta; import org.dbsyncer.parser.model.TableGroup; +import org.dbsyncer.sdk.config.ListenerConfig; import org.dbsyncer.sdk.enums.ListenerTypeEnum; import org.dbsyncer.sdk.listener.Listener; import org.dbsyncer.sdk.model.ChangedOffset; @@ -152,13 +152,13 @@ public final class IncrementPuller extends AbstractPuller implements Application if (ListenerTypeEnum.isTiming(listenerType) && listener instanceof AbstractQuartzListener) { AbstractQuartzListener quartzListener = (AbstractQuartzListener) listener; quartzListener.setCommands(list.stream().map(t -> new TableGroupQuartzCommand(t.getSourceTable(), t.getCommand())).collect(Collectors.toList())); - quartzListener.register(new QuartzConsumer().init(bufferActuatorRouter, profileComponent, logService, meta, mapping, list)); + quartzListener.register(new QuartzConsumer().init(bufferActuatorRouter, profileComponent, logService, meta.getId(), mapping, list)); } // 基于日志抽取 if (ListenerTypeEnum.isLog(listenerType) && listener instanceof AbstractListener) { AbstractListener abstractListener = (AbstractListener) listener; - abstractListener.register(new LogConsumer().init(bufferActuatorRouter, profileComponent, logService, meta, mapping, list)); + abstractListener.register(new LogConsumer().init(bufferActuatorRouter, profileComponent, logService, meta.getId(), mapping, list)); } if (listener instanceof AbstractListener) { diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/AbstractConsumer.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/AbstractConsumer.java index 87027b77..eab740a1 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/AbstractConsumer.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/AbstractConsumer.java @@ -26,15 +26,15 @@ public abstract class AbstractConsumer implements Watche private BufferActuatorRouter bufferActuatorRouter; private ProfileComponent profileComponent; private LogService logService; - private Meta meta; + private String metaId; protected Mapping mapping; protected List tableGroups; - public AbstractConsumer init(BufferActuatorRouter bufferActuatorRouter, ProfileComponent profileComponent, LogService logService, Meta meta, Mapping mapping, List tableGroups) { + public AbstractConsumer init(BufferActuatorRouter bufferActuatorRouter, ProfileComponent profileComponent, LogService logService, String metaId, Mapping mapping, List tableGroups) { this.bufferActuatorRouter = bufferActuatorRouter; this.profileComponent = profileComponent; this.logService = logService; - this.meta = meta; + this.metaId = metaId; this.mapping = mapping; this.tableGroups = tableGroups; postProcessBeforeInitialization(); @@ -50,7 +50,7 @@ public abstract class AbstractConsumer implements Watche @Override public void changeEvent(ChangedEvent event) { - event.getChangedOffset().setMetaId(meta.getId()); + event.getChangedOffset().setMetaId(metaId); if (event instanceof DDLChangedEvent) { onDDLChanged((DDLChangedEvent) event); return; @@ -60,8 +60,11 @@ public abstract class AbstractConsumer implements Watche @Override public void flushEvent(Map snapshot) { - meta.setSnapshot(snapshot); - profileComponent.editConfigModel(meta); + Meta meta = profileComponent.getMeta(metaId); + if (meta != null) { + meta.setSnapshot(snapshot); + profileComponent.editConfigModel(meta); + } } @Override @@ -71,14 +74,15 @@ public abstract class AbstractConsumer implements Watche @Override public long getMetaUpdateTime() { - return meta.getUpdateTime(); + Meta meta = profileComponent.getMeta(metaId); + return meta != null ? meta.getUpdateTime() : 0L; } protected void bind(String tableGroupId) { - bufferActuatorRouter.bind(meta.getId(), tableGroupId); + bufferActuatorRouter.bind(metaId, tableGroupId); } protected void execute(String tableGroupId, ChangedEvent event) { - bufferActuatorRouter.execute(meta.getId(), tableGroupId, event); + bufferActuatorRouter.execute(metaId, tableGroupId, event); } } \ No newline at end of file diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/AbstractFlushStrategy.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/AbstractFlushStrategy.java index 6bc1e822..3759a962 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/AbstractFlushStrategy.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/AbstractFlushStrategy.java @@ -4,7 +4,7 @@ import org.dbsyncer.common.config.StorageConfig; import org.dbsyncer.common.model.Result; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.parser.ProfileComponent; +import org.dbsyncer.parser.CacheService; import org.dbsyncer.parser.model.Meta; import org.dbsyncer.parser.strategy.FlushStrategy; import org.springframework.util.Assert; @@ -23,7 +23,7 @@ public abstract class AbstractFlushStrategy implements FlushStrategy { private FlushService flushService; @Resource - private ProfileComponent profileComponent; + private CacheService cacheService; @Resource private StorageConfig storageConfig; @@ -38,27 +38,29 @@ public abstract class AbstractFlushStrategy implements FlushStrategy { flush(metaId, result, event); } - protected void flush(String metaId, Result result, String event) { + protected void refreshTotal(String metaId, Result writer) { + Assert.hasText(metaId, "Meta id can not be empty."); + Meta meta = cacheService.get(metaId, Meta.class); + if (meta != null) { + meta.getFail().getAndAdd(writer.getFailData().size()); + meta.getSuccess().getAndAdd(writer.getSuccessData().size()); + meta.setUpdateTime(Instant.now().toEpochMilli()); + } + } + + private void flush(String metaId, Result result, String event) { refreshTotal(metaId, result); + // 是否写失败数据 if (storageConfig.isWriteFail() && !CollectionUtils.isEmpty(result.getFailData())) { final String error = StringUtil.substring(result.getError().toString(), 0, storageConfig.getMaxErrorLength()); flushService.asyncWrite(metaId, result.getTableGroupId(), result.getTargetTableGroupName(), event, false, result.getFailData(), error); } - // 是否写增量数据 + // 是否写成功数据 if (storageConfig.isWriteSuccess() && !CollectionUtils.isEmpty(result.getSuccessData())) { flushService.asyncWrite(metaId, result.getTableGroupId(), result.getTargetTableGroupName(), event, true, result.getSuccessData(), ""); } } - protected void refreshTotal(String metaId, Result writer) { - Assert.hasText(metaId, "Meta id can not be empty."); - Meta meta = profileComponent.getMeta(metaId); - Assert.notNull(meta, "Meta can not be null."); - meta.getFail().getAndAdd(writer.getFailData().size()); - meta.getSuccess().getAndAdd(writer.getSuccessData().size()); - meta.setUpdateTime(Instant.now().toEpochMilli()); - } - } \ No newline at end of file diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/Mapping.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/Mapping.java index d72d68c9..a52ae871 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/Mapping.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/Mapping.java @@ -1,6 +1,6 @@ package org.dbsyncer.parser.model; -import org.dbsyncer.connector.config.ListenerConfig; +import org.dbsyncer.sdk.config.ListenerConfig; import org.dbsyncer.sdk.enums.ModelEnum; import org.dbsyncer.sdk.model.Field; import org.dbsyncer.storage.constant.ConfigConstant; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/ListenerConfig.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/config/ListenerConfig.java similarity index 98% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/ListenerConfig.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/config/ListenerConfig.java index 75cbf8d6..41a66836 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/ListenerConfig.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/config/ListenerConfig.java @@ -1,4 +1,4 @@ -package org.dbsyncer.connector.config; +package org.dbsyncer.sdk.config; import org.dbsyncer.sdk.enums.ListenerTypeEnum; -- Gitee From 9d0ba06a125257a5808927182092ff2ed1ca58a2 Mon Sep 17 00:00:00 2001 From: AE86 <836391306@qq.com> Date: Wed, 22 Nov 2023 02:26:03 +0800 Subject: [PATCH 25/98] fix bug --- .../biz/impl/ConditionServiceImpl.java | 2 +- .../org/dbsyncer/biz/impl/MetricReporter.java | 4 ++-- .../dbsyncer/biz/impl/MonitorServiceImpl.java | 4 ++-- .../java/org/dbsyncer/biz/vo/ConditionVo.java | 2 +- .../connector/es/ESQuartzListener.java | 8 +++---- .../dbsyncer/connector/file/FileListener.java | 18 +++++++-------- .../connector/mysql/MySQLConnector.java | 2 +- .../connector/mysql/MySQLListener.java | 2 +- .../connector/oracle/OracleConnector.java | 2 +- .../connector/oracle/OracleListener.java | 2 +- .../connector/postgresql/MessageDecoder.java | 6 ++--- .../postgresql/PostgreSQLConnector.java | 2 +- .../postgresql/PostgreSQLListener.java | 22 +++++++++--------- .../decoder/PgOutputMessageDecoder.java | 12 +++++----- .../connector/sql/DQLMySQLConnector.java | 2 +- .../connector/sql/DQLOracleConnector.java | 2 +- .../connector/sql/DQLPostgreSQLConnector.java | 2 +- .../connector/sql/DQLSqlServerConnector.java | 2 +- .../sqlserver/SqlServerConnector.java | 2 +- .../sqlserver/SqlServerListener.java | 16 +++++++------ .../manager/impl/IncrementPuller.java | 13 ++++++----- .../org/dbsyncer/parser/ProfileComponent.java | 2 +- .../parser/flush/AbstractBufferActuator.java | 4 ++-- .../flush/impl/TableGroupBufferActuator.java | 2 +- .../parser/impl/ProfileComponentImpl.java | 2 +- .../dbsyncer/sdk}/enums/QuartzFilterEnum.java | 12 +++++----- .../listener}/AbstractDatabaseListener.java | 23 +++++++++---------- .../sdk/listener}/AbstractListener.java | 20 +++++++++------- .../sdk/listener}/AbstractQuartzListener.java | 11 ++++----- .../sdk}/quartz/DatabaseQuartzListener.java | 5 ++-- .../java/org/dbsyncer/sdk}/quartz/Point.java | 2 +- .../dbsyncer/sdk}/quartz/QuartzFilter.java | 2 +- .../sdk}/quartz/TableGroupQuartzCommand.java | 2 +- .../sdk}/quartz/filter/DateFilter.java | 4 ++-- .../sdk}/quartz/filter/TimestampFilter.java | 4 ++-- .../sdk}/quartz/filter/YesDateFilter.java | 4 ++-- .../quartz/filter/YesTimestampFilter.java | 4 ++-- .../sdk}/scheduled/ScheduledTaskJob.java | 2 +- .../sdk}/scheduled/ScheduledTaskService.java | 2 +- .../impl/ScheduledTaskServiceImpl.java | 6 ++--- .../src/main/resources/application.properties | 4 ++-- 41 files changed, 125 insertions(+), 119 deletions(-) rename {dbsyncer-connector/src/main/java/org/dbsyncer/connector => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/enums/QuartzFilterEnum.java (87%) rename {dbsyncer-connector/src/main/java/org/dbsyncer/connector => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener}/AbstractDatabaseListener.java (82%) rename {dbsyncer-connector/src/main/java/org/dbsyncer/connector => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener}/AbstractListener.java (88%) rename {dbsyncer-connector/src/main/java/org/dbsyncer/connector/quartz => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener}/AbstractQuartzListener.java (93%) rename {dbsyncer-connector/src/main/java/org/dbsyncer/connector => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/quartz/DatabaseQuartzListener.java (96%) rename {dbsyncer-connector/src/main/java/org/dbsyncer/connector => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/quartz/Point.java (97%) rename {dbsyncer-connector/src/main/java/org/dbsyncer/connector => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/quartz/QuartzFilter.java (91%) rename {dbsyncer-connector/src/main/java/org/dbsyncer/connector => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/quartz/TableGroupQuartzCommand.java (94%) rename {dbsyncer-connector/src/main/java/org/dbsyncer/connector => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/quartz/filter/DateFilter.java (86%) rename {dbsyncer-connector/src/main/java/org/dbsyncer/connector => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/quartz/filter/TimestampFilter.java (86%) rename {dbsyncer-connector/src/main/java/org/dbsyncer/connector => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/quartz/filter/YesDateFilter.java (92%) rename {dbsyncer-connector/src/main/java/org/dbsyncer/connector => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/quartz/filter/YesTimestampFilter.java (90%) rename {dbsyncer-connector/src/main/java/org/dbsyncer/connector => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/scheduled/ScheduledTaskJob.java (74%) rename {dbsyncer-connector/src/main/java/org/dbsyncer/connector => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/scheduled/ScheduledTaskService.java (94%) rename {dbsyncer-connector/src/main/java/org/dbsyncer/connector => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/scheduled/impl/ScheduledTaskServiceImpl.java (94%) diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ConditionServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ConditionServiceImpl.java index 8e02de1b..e06b17f4 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ConditionServiceImpl.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ConditionServiceImpl.java @@ -3,7 +3,7 @@ package org.dbsyncer.biz.impl; import org.dbsyncer.biz.ConditionService; import org.dbsyncer.biz.vo.ConditionVo; import org.dbsyncer.connector.enums.FilterEnum; -import org.dbsyncer.connector.enums.QuartzFilterEnum; +import org.dbsyncer.sdk.enums.QuartzFilterEnum; import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.sdk.enums.OperationEnum; import org.springframework.stereotype.Component; diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MetricReporter.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MetricReporter.java index c598a895..d62e488d 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MetricReporter.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MetricReporter.java @@ -9,8 +9,8 @@ import org.dbsyncer.biz.model.MetricResponse; import org.dbsyncer.biz.model.MetricResponseInfo; import org.dbsyncer.biz.model.Sample; import org.dbsyncer.common.model.Paging; -import org.dbsyncer.connector.scheduled.ScheduledTaskJob; -import org.dbsyncer.connector.scheduled.ScheduledTaskService; +import org.dbsyncer.sdk.scheduled.ScheduledTaskJob; +import org.dbsyncer.sdk.scheduled.ScheduledTaskService; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.sdk.constant.ConnectorConstant; diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java index c59a3f8e..8ce1909c 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java @@ -26,8 +26,8 @@ import org.dbsyncer.common.util.JsonUtil; import org.dbsyncer.common.util.NumberUtil; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.connector.enums.FilterEnum; -import org.dbsyncer.connector.scheduled.ScheduledTaskJob; -import org.dbsyncer.connector.scheduled.ScheduledTaskService; +import org.dbsyncer.sdk.scheduled.ScheduledTaskJob; +import org.dbsyncer.sdk.scheduled.ScheduledTaskService; import org.dbsyncer.parser.LogService; import org.dbsyncer.parser.LogType; import org.dbsyncer.parser.ProfileComponent; diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/vo/ConditionVo.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/vo/ConditionVo.java index 02139eca..c2837983 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/vo/ConditionVo.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/vo/ConditionVo.java @@ -1,7 +1,7 @@ package org.dbsyncer.biz.vo; import org.dbsyncer.connector.enums.FilterEnum; -import org.dbsyncer.connector.enums.QuartzFilterEnum; +import org.dbsyncer.sdk.enums.QuartzFilterEnum; import org.dbsyncer.sdk.enums.OperationEnum; import java.util.List; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESQuartzListener.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESQuartzListener.java index 7eebed34..81b5e89d 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESQuartzListener.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESQuartzListener.java @@ -4,10 +4,10 @@ import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.JsonUtil; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.connector.ConnectorException; -import org.dbsyncer.connector.enums.QuartzFilterEnum; -import org.dbsyncer.connector.quartz.AbstractQuartzListener; -import org.dbsyncer.connector.quartz.Point; -import org.dbsyncer.connector.quartz.QuartzFilter; +import org.dbsyncer.sdk.enums.QuartzFilterEnum; +import org.dbsyncer.sdk.listener.AbstractQuartzListener; +import org.dbsyncer.sdk.quartz.Point; +import org.dbsyncer.sdk.quartz.QuartzFilter; import org.dbsyncer.sdk.constant.ConnectorConstant; import org.dbsyncer.sdk.model.Filter; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileListener.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileListener.java index 3057d2b8..5415b6b8 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileListener.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileListener.java @@ -1,15 +1,15 @@ package org.dbsyncer.connector.file; import org.apache.commons.io.IOUtils; -import org.dbsyncer.connector.ConnectorException; -import org.dbsyncer.sdk.model.ChangedOffset; -import org.dbsyncer.sdk.listener.event.RowChangedEvent; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.NumberUtil; import org.dbsyncer.common.util.StringUtil; +import org.dbsyncer.connector.ConnectorException; import org.dbsyncer.connector.config.FileConfig; import org.dbsyncer.sdk.constant.ConnectorConstant; -import org.dbsyncer.connector.AbstractListener; +import org.dbsyncer.sdk.listener.AbstractListener; +import org.dbsyncer.sdk.listener.event.RowChangedEvent; +import org.dbsyncer.sdk.model.ChangedOffset; import org.dbsyncer.sdk.model.Field; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -48,7 +48,7 @@ public class FileListener extends AbstractListener { private static final String CHARSET_NAME = "UTF-8"; private final Lock connectLock = new ReentrantLock(); private volatile boolean connected; - private FileConnectorInstance connectorInstance; + private FileConnectorInstance instance; private WatchService watchService; private Worker worker; private Map pipeline = new ConcurrentHashMap<>(); @@ -64,9 +64,9 @@ public class FileListener extends AbstractListener { return; } - connectorInstance = (FileConnectorInstance) connectorFactory.connect(connectorConfig); - final FileConfig config = connectorInstance.getConfig(); - final String cacheKey = connectorFactory.getConnectorService(connectorConfig.getConnectorType()).getConnectorInstanceCacheKey(connectorConfig); + instance = (FileConnectorInstance) connectorInstance; + final FileConfig config = instance.getConfig(); + final String cacheKey = connectorService.getConnectorInstanceCacheKey(config); connected = true; separator = config.getSeparator(); @@ -93,7 +93,7 @@ public class FileListener extends AbstractListener { } private void initPipeline(String fileDir) throws IOException { - for (FileSchema fileSchema : connectorInstance.getFileSchemaList()) { + for (FileSchema fileSchema : instance.getFileSchemaList()) { String fileName = fileSchema.getFileName(); String file = fileDir.concat(fileName); Assert.isTrue(new File(file).exists(), String.format("found not file '%s'", file)); diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/MySQLConnector.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/MySQLConnector.java index 81dd3b82..ee0d1ee0 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/MySQLConnector.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/MySQLConnector.java @@ -1,7 +1,7 @@ package org.dbsyncer.connector.mysql; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.quartz.DatabaseQuartzListener; +import org.dbsyncer.sdk.quartz.DatabaseQuartzListener; import org.dbsyncer.sdk.config.ReaderConfig; import org.dbsyncer.sdk.connector.database.AbstractDatabaseConnector; import org.dbsyncer.sdk.constant.DatabaseConstant; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/MySQLListener.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/MySQLListener.java index dd172696..b5426e2f 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/MySQLListener.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/MySQLListener.java @@ -15,7 +15,7 @@ import net.sf.jsqlparser.JSQLParserException; import net.sf.jsqlparser.parser.CCJSqlParserUtil; import net.sf.jsqlparser.statement.alter.Alter; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.AbstractDatabaseListener; +import org.dbsyncer.sdk.listener.AbstractDatabaseListener; import org.dbsyncer.connector.ConnectorException; import org.dbsyncer.sdk.config.DatabaseConfig; import org.dbsyncer.sdk.constant.ConnectorConstant; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/OracleConnector.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/OracleConnector.java index 5c5faf20..3db09341 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/OracleConnector.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/OracleConnector.java @@ -1,7 +1,7 @@ package org.dbsyncer.connector.oracle; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.quartz.DatabaseQuartzListener; +import org.dbsyncer.sdk.quartz.DatabaseQuartzListener; import org.dbsyncer.sdk.config.CommandConfig; import org.dbsyncer.sdk.config.DatabaseConfig; import org.dbsyncer.sdk.config.ReaderConfig; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/OracleListener.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/OracleListener.java index 2f933309..89d363fd 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/OracleListener.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/OracleListener.java @@ -1,6 +1,6 @@ package org.dbsyncer.connector.oracle; -import org.dbsyncer.connector.AbstractDatabaseListener; +import org.dbsyncer.sdk.listener.AbstractDatabaseListener; import org.dbsyncer.connector.ConnectorException; import org.dbsyncer.connector.oracle.dcn.DBChangeNotification; import org.dbsyncer.sdk.config.DatabaseConfig; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/MessageDecoder.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/MessageDecoder.java index 2c3ed00d..4ceddaf5 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/MessageDecoder.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/MessageDecoder.java @@ -1,9 +1,9 @@ package org.dbsyncer.connector.postgresql; -import org.dbsyncer.sdk.listener.event.RowChangedEvent; -import org.dbsyncer.connector.ConnectorFactory; import org.dbsyncer.sdk.config.DatabaseConfig; import org.dbsyncer.sdk.connector.database.DatabaseConnectorInstance; +import org.dbsyncer.sdk.listener.event.RowChangedEvent; +import org.dbsyncer.sdk.spi.ConnectorService; import org.postgresql.replication.LogSequenceNumber; import org.postgresql.replication.fluent.logical.ChainedLogicalStreamBuilder; @@ -16,7 +16,7 @@ import java.nio.ByteBuffer; */ public interface MessageDecoder { - default void postProcessBeforeInitialization(ConnectorFactory connectorFactory, DatabaseConnectorInstance connectorInstance) { + default void postProcessBeforeInitialization(ConnectorService connectorService, DatabaseConnectorInstance connectorInstance) { } boolean skipMessage(ByteBuffer buffer, LogSequenceNumber startLsn, LogSequenceNumber lastReceiveLsn); diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLConnector.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLConnector.java index b4efc875..c48bea8e 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLConnector.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLConnector.java @@ -1,7 +1,7 @@ package org.dbsyncer.connector.postgresql; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.quartz.DatabaseQuartzListener; +import org.dbsyncer.sdk.quartz.DatabaseQuartzListener; import org.dbsyncer.sdk.config.CommandConfig; import org.dbsyncer.sdk.config.DatabaseConfig; import org.dbsyncer.sdk.config.ReaderConfig; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLListener.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLListener.java index 2218eb09..c07dff95 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLListener.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLListener.java @@ -1,7 +1,7 @@ package org.dbsyncer.connector.postgresql; import org.dbsyncer.common.util.BooleanUtil; -import org.dbsyncer.connector.AbstractDatabaseListener; +import org.dbsyncer.sdk.listener.AbstractDatabaseListener; import org.dbsyncer.connector.ConnectorException; import org.dbsyncer.connector.postgresql.enums.MessageDecoderEnum; import org.dbsyncer.sdk.config.DatabaseConfig; @@ -52,7 +52,7 @@ public class PostgreSQLListener extends AbstractDatabaseListener { private final Lock connectLock = new ReentrantLock(); private volatile boolean connected; private DatabaseConfig config; - private DatabaseConnectorInstance connectorInstance; + private DatabaseConnectorInstance instance; private Connection connection; private PGReplicationStream stream; private boolean dropSlotOnClose; @@ -70,15 +70,15 @@ public class PostgreSQLListener extends AbstractDatabaseListener { return; } - connectorInstance = (DatabaseConnectorInstance) connectorFactory.connect(connectorConfig); - config = connectorInstance.getConfig(); + instance = (DatabaseConnectorInstance) connectorInstance; + config = instance.getConfig(); - final String walLevel = connectorInstance.execute(databaseTemplate -> databaseTemplate.queryForObject(GET_WAL_LEVEL, String.class)); + final String walLevel = instance.execute(databaseTemplate -> databaseTemplate.queryForObject(GET_WAL_LEVEL, String.class)); if (!DEFAULT_WAL_LEVEL.equals(walLevel)) { throw new ConnectorException(String.format("Postgres server wal_level property must be \"%s\" but is: %s", DEFAULT_WAL_LEVEL, walLevel)); } - final boolean hasAuth = connectorInstance.execute(databaseTemplate -> { + final boolean hasAuth = instance.execute(databaseTemplate -> { Map rs = databaseTemplate.queryForMap(GET_ROLE); Boolean login = (Boolean) rs.getOrDefault("login", false); Boolean replication = (Boolean) rs.getOrDefault("replication", false); @@ -91,11 +91,11 @@ public class PostgreSQLListener extends AbstractDatabaseListener { throw new ConnectorException(String.format("Postgres roles LOGIN and REPLICATION are not assigned to user: %s", config.getUsername())); } - database = connectorInstance.execute(databaseTemplate -> databaseTemplate.queryForObject(GET_DATABASE, String.class)); + database = instance.execute(databaseTemplate -> databaseTemplate.queryForObject(GET_DATABASE, String.class)); messageDecoder = MessageDecoderEnum.getMessageDecoder(config.getProperty(PLUGIN_NAME)); messageDecoder.setMetaId(metaId); messageDecoder.setConfig(config); - messageDecoder.postProcessBeforeInitialization(connectorFactory, connectorInstance); + messageDecoder.postProcessBeforeInitialization(connectorService, instance); dropSlotOnClose = BooleanUtil.toBoolean(config.getProperty(DROP_SLOT_ON_CLOSE, "true")); connect(); @@ -170,7 +170,7 @@ public class PostgreSQLListener extends AbstractDatabaseListener { private void createReplicationSlot(PGConnection pgConnection) throws SQLException { String slotName = messageDecoder.getSlotName(); String plugin = messageDecoder.getOutputPlugin(); - boolean existSlot = connectorInstance.execute(databaseTemplate -> databaseTemplate.queryForObject(GET_SLOT, new Object[]{database, slotName, plugin}, Integer.class) > 0); + boolean existSlot = instance.execute(databaseTemplate -> databaseTemplate.queryForObject(GET_SLOT, new Object[]{database, slotName, plugin}, Integer.class) > 0); if (!existSlot) { pgConnection.getReplicationAPI() .createReplicationSlot() @@ -184,7 +184,7 @@ public class PostgreSQLListener extends AbstractDatabaseListener { } if (!snapshot.containsKey(LSN_POSITION)) { - LogSequenceNumber lsn = connectorInstance.execute(databaseTemplate -> LogSequenceNumber.valueOf(databaseTemplate.queryForObject(GET_RESTART_LSN, new Object[]{database, slotName, plugin}, String.class))); + LogSequenceNumber lsn = instance.execute(databaseTemplate -> LogSequenceNumber.valueOf(databaseTemplate.queryForObject(GET_RESTART_LSN, new Object[]{database, slotName, plugin}, String.class))); if (null == lsn || lsn.asLong() == 0) { throw new ConnectorException("No maximum LSN recorded in the database"); } @@ -204,7 +204,7 @@ public class PostgreSQLListener extends AbstractDatabaseListener { final int ATTEMPTS = 3; for (int i = 0; i < ATTEMPTS; i++) { try { - connectorInstance.execute(databaseTemplate -> { + instance.execute(databaseTemplate -> { databaseTemplate.execute(String.format("select pg_drop_replication_slot('%s')", slotName)); return true; }); diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/decoder/PgOutputMessageDecoder.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/decoder/PgOutputMessageDecoder.java index 11c9118f..1cd19eae 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/decoder/PgOutputMessageDecoder.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/decoder/PgOutputMessageDecoder.java @@ -2,7 +2,6 @@ package org.dbsyncer.connector.postgresql.decoder; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.connector.ConnectorException; -import org.dbsyncer.connector.ConnectorFactory; import org.dbsyncer.connector.postgresql.AbstractMessageDecoder; import org.dbsyncer.connector.postgresql.enums.MessageDecoderEnum; import org.dbsyncer.connector.postgresql.enums.MessageTypeEnum; @@ -10,6 +9,7 @@ import org.dbsyncer.sdk.connector.database.DatabaseConnectorInstance; import org.dbsyncer.sdk.listener.event.RowChangedEvent; import org.dbsyncer.sdk.model.Field; import org.dbsyncer.sdk.model.MetaInfo; +import org.dbsyncer.sdk.spi.ConnectorService; import org.postgresql.replication.fluent.logical.ChainedLogicalStreamBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,12 +34,12 @@ public class PgOutputMessageDecoder extends AbstractMessageDecoder { private static final LocalDateTime PG_EPOCH = LocalDateTime.of(2000, 1, 1, 0, 0, 0); private static final String GET_TABLE_SCHEMA = "select t.oid,t.relname as tableName from pg_class t inner join (select ns.oid as nspoid, ns.nspname from pg_namespace ns where ns.nspname = '%s') as n on n.nspoid = t.relnamespace where relkind = 'r'"; private static final Map tables = new LinkedHashMap<>(); - private ConnectorFactory connectorFactory; + private ConnectorService connectorService; private DatabaseConnectorInstance connectorInstance; @Override - public void postProcessBeforeInitialization(ConnectorFactory connectorFactory, DatabaseConnectorInstance connectorInstance) { - this.connectorFactory = connectorFactory; + public void postProcessBeforeInitialization(ConnectorService connectorService, DatabaseConnectorInstance connectorInstance) { + this.connectorService = connectorService; this.connectorInstance = connectorInstance; initPublication(); readSchema(); @@ -123,7 +123,7 @@ public class PgOutputMessageDecoder extends AbstractMessageDecoder { schemas.forEach(map -> { Long oid = (Long) map.get("oid"); String tableName = (String) map.get("tableName"); - MetaInfo metaInfo = connectorFactory.getMetaInfo(connectorInstance, tableName); + MetaInfo metaInfo = connectorService.getMetaInfo(connectorInstance, tableName); Assert.notEmpty(metaInfo.getColumn(), String.format("The table column for '%s' must not be empty.", tableName)); tables.put(oid.intValue(), new TableId(oid.intValue(), tableName, metaInfo.getColumn())); }); @@ -156,7 +156,7 @@ public class PgOutputMessageDecoder extends AbstractMessageDecoder { logger.warn("The column size of table '{}' is {}, but we has been received column size is {}.", tableId.tableName, tableId.fields.size(), nColumn); // The table schema has been changed, we should be get a new table schema from db. - MetaInfo metaInfo = connectorFactory.getMetaInfo(connectorInstance, tableId.tableName); + MetaInfo metaInfo = connectorService.getMetaInfo(connectorInstance, tableId.tableName); if (CollectionUtils.isEmpty(metaInfo.getColumn())) { throw new ConnectorException(String.format("The table column for '%s' is empty.", tableId.tableName)); } diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLMySQLConnector.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLMySQLConnector.java index 259157d3..22b55456 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLMySQLConnector.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLMySQLConnector.java @@ -1,7 +1,7 @@ package org.dbsyncer.connector.sql; import org.dbsyncer.connector.mysql.DqlMySQLListener; -import org.dbsyncer.connector.quartz.DatabaseQuartzListener; +import org.dbsyncer.sdk.quartz.DatabaseQuartzListener; import org.dbsyncer.sdk.config.CommandConfig; import org.dbsyncer.sdk.config.ReaderConfig; import org.dbsyncer.sdk.connector.database.AbstractDQLConnector; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLOracleConnector.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLOracleConnector.java index 48b9aac2..e37cbd65 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLOracleConnector.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLOracleConnector.java @@ -1,7 +1,7 @@ package org.dbsyncer.connector.sql; import org.dbsyncer.connector.oracle.DqlOracleListener; -import org.dbsyncer.connector.quartz.DatabaseQuartzListener; +import org.dbsyncer.sdk.quartz.DatabaseQuartzListener; import org.dbsyncer.sdk.config.ReaderConfig; import org.dbsyncer.sdk.connector.database.AbstractDQLConnector; import org.dbsyncer.sdk.constant.DatabaseConstant; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLPostgreSQLConnector.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLPostgreSQLConnector.java index 8720faa9..a8ff60da 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLPostgreSQLConnector.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLPostgreSQLConnector.java @@ -1,7 +1,7 @@ package org.dbsyncer.connector.sql; import org.dbsyncer.connector.postgresql.DqlPostgreSQLListener; -import org.dbsyncer.connector.quartz.DatabaseQuartzListener; +import org.dbsyncer.sdk.quartz.DatabaseQuartzListener; import org.dbsyncer.sdk.config.ReaderConfig; import org.dbsyncer.sdk.connector.database.AbstractDQLConnector; import org.dbsyncer.sdk.constant.DatabaseConstant; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLSqlServerConnector.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLSqlServerConnector.java index 691893c3..9538d614 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLSqlServerConnector.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLSqlServerConnector.java @@ -1,7 +1,7 @@ package org.dbsyncer.connector.sql; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.quartz.DatabaseQuartzListener; +import org.dbsyncer.sdk.quartz.DatabaseQuartzListener; import org.dbsyncer.connector.sqlserver.DqlSqlServerListener; import org.dbsyncer.sdk.config.ReaderConfig; import org.dbsyncer.sdk.connector.database.AbstractDQLConnector; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/SqlServerConnector.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/SqlServerConnector.java index ca194281..878541c6 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/SqlServerConnector.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/SqlServerConnector.java @@ -2,7 +2,7 @@ package org.dbsyncer.connector.sqlserver; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.quartz.DatabaseQuartzListener; +import org.dbsyncer.sdk.quartz.DatabaseQuartzListener; import org.dbsyncer.sdk.config.CommandConfig; import org.dbsyncer.sdk.config.DatabaseConfig; import org.dbsyncer.sdk.config.ReaderConfig; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/SqlServerListener.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/SqlServerListener.java index da7cb5ab..59b4ee36 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/SqlServerListener.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/SqlServerListener.java @@ -3,13 +3,14 @@ package org.dbsyncer.connector.sqlserver; import com.microsoft.sqlserver.jdbc.SQLServerException; import org.dbsyncer.connector.ConnectorException; import org.dbsyncer.connector.enums.TableOperationEnum; +import org.dbsyncer.sdk.connector.database.AbstractDatabaseConnector; import org.dbsyncer.sdk.model.ChangedOffset; import org.dbsyncer.sdk.listener.event.RowChangedEvent; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.sdk.config.DatabaseConfig; import org.dbsyncer.sdk.constant.ConnectorConstant; import org.dbsyncer.sdk.connector.database.DatabaseConnectorInstance; -import org.dbsyncer.connector.AbstractDatabaseListener; +import org.dbsyncer.sdk.listener.AbstractDatabaseListener; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.util.Assert; @@ -62,7 +63,7 @@ public class SqlServerListener extends AbstractDatabaseListener { private volatile boolean connected; private Set tables; private Set changeTables; - private DatabaseConnectorInstance connectorInstance; + private DatabaseConnectorInstance instance; private Worker worker; private Lsn lastLsn; private String serverName; @@ -135,9 +136,10 @@ public class SqlServerListener extends AbstractDatabaseListener { } private void connect() { - if (connectorFactory.isAlive(connectorConfig)) { - connectorInstance = (DatabaseConnectorInstance) connectorFactory.connect(connectorConfig); - DatabaseConfig cfg = connectorInstance.getConfig(); + instance = (DatabaseConnectorInstance) connectorInstance; + AbstractDatabaseConnector service = (AbstractDatabaseConnector) connectorService; + if (service.isAlive(instance)) { + DatabaseConfig cfg = instance.getConfig(); serverName = cfg.getUrl(); schema = cfg.getSchema(); } @@ -221,7 +223,7 @@ public class SqlServerListener extends AbstractDatabaseListener { } private void execute(String... sqlStatements) { - connectorInstance.execute(databaseTemplate -> { + instance.execute(databaseTemplate -> { for (String sqlStatement : sqlStatements) { if (sqlStatement != null) { logger.info("executing '{}'", sqlStatement); @@ -313,7 +315,7 @@ public class SqlServerListener extends AbstractDatabaseListener { } private T query(String preparedQuerySql, StatementPreparer statementPreparer, ResultSetMapper mapper) { - Object execute = connectorInstance.execute(databaseTemplate -> { + Object execute = instance.execute(databaseTemplate -> { PreparedStatement ps = null; ResultSet rs = null; T apply = null; diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/IncrementPuller.java b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/IncrementPuller.java index cb2a848e..baa5ff5c 100644 --- a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/IncrementPuller.java +++ b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/IncrementPuller.java @@ -1,12 +1,7 @@ package org.dbsyncer.manager.impl; import org.dbsyncer.common.util.CollectionUtils; -import org.dbsyncer.connector.AbstractListener; import org.dbsyncer.connector.ConnectorFactory; -import org.dbsyncer.connector.quartz.AbstractQuartzListener; -import org.dbsyncer.connector.quartz.TableGroupQuartzCommand; -import org.dbsyncer.connector.scheduled.ScheduledTaskJob; -import org.dbsyncer.connector.scheduled.ScheduledTaskService; import org.dbsyncer.manager.AbstractPuller; import org.dbsyncer.manager.ManagerException; import org.dbsyncer.parser.LogService; @@ -22,10 +17,15 @@ import org.dbsyncer.parser.model.Meta; import org.dbsyncer.parser.model.TableGroup; import org.dbsyncer.sdk.config.ListenerConfig; import org.dbsyncer.sdk.enums.ListenerTypeEnum; +import org.dbsyncer.sdk.listener.AbstractListener; +import org.dbsyncer.sdk.listener.AbstractQuartzListener; import org.dbsyncer.sdk.listener.Listener; import org.dbsyncer.sdk.model.ChangedOffset; import org.dbsyncer.sdk.model.ConnectorConfig; import org.dbsyncer.sdk.model.Table; +import org.dbsyncer.sdk.quartz.TableGroupQuartzCommand; +import org.dbsyncer.sdk.scheduled.ScheduledTaskJob; +import org.dbsyncer.sdk.scheduled.ScheduledTaskService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.ApplicationListener; @@ -173,7 +173,8 @@ public final class IncrementPuller extends AbstractPuller implements Application filterTable.add(table.getName()); }); - abstractListener.setConnectorFactory(connectorFactory); + abstractListener.setConnectorService(connectorFactory.getConnectorService(connectorConfig.getConnectorType())); + abstractListener.setConnectorInstance(connectorFactory.connect(connectorConfig)); abstractListener.setScheduledTaskService(scheduledTaskService); abstractListener.setConnectorConfig(connectorConfig); abstractListener.setListenerConfig(listenerConfig); diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ProfileComponent.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ProfileComponent.java index ca60997c..8d1ffb6e 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ProfileComponent.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ProfileComponent.java @@ -4,7 +4,7 @@ package org.dbsyncer.parser; import org.dbsyncer.connector.enums.FilterEnum; -import org.dbsyncer.connector.enums.QuartzFilterEnum; +import org.dbsyncer.sdk.enums.QuartzFilterEnum; import org.dbsyncer.parser.enums.ConvertEnum; import org.dbsyncer.parser.model.ConfigModel; import org.dbsyncer.parser.model.Connector; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/AbstractBufferActuator.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/AbstractBufferActuator.java index 9c35277d..15febcf9 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/AbstractBufferActuator.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/AbstractBufferActuator.java @@ -1,8 +1,8 @@ package org.dbsyncer.parser.flush; import org.dbsyncer.common.config.BufferActuatorConfig; -import org.dbsyncer.connector.scheduled.ScheduledTaskJob; -import org.dbsyncer.connector.scheduled.ScheduledTaskService; +import org.dbsyncer.sdk.scheduled.ScheduledTaskJob; +import org.dbsyncer.sdk.scheduled.ScheduledTaskService; import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.parser.enums.MetaEnum; import org.dbsyncer.parser.model.Meta; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/TableGroupBufferActuator.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/TableGroupBufferActuator.java index f71e1559..285c2176 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/TableGroupBufferActuator.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/TableGroupBufferActuator.java @@ -1,7 +1,7 @@ package org.dbsyncer.parser.flush.impl; import org.dbsyncer.common.config.TableGroupBufferConfig; -import org.dbsyncer.connector.scheduled.ScheduledTaskService; +import org.dbsyncer.sdk.scheduled.ScheduledTaskService; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.common.util.ThreadPoolUtil; import org.dbsyncer.common.util.UUIDUtil; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/ProfileComponentImpl.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/ProfileComponentImpl.java index 3dc02c89..52a5b278 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/ProfileComponentImpl.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/ProfileComponentImpl.java @@ -7,7 +7,7 @@ import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.JsonUtil; import org.dbsyncer.connector.ConnectorFactory; import org.dbsyncer.connector.enums.FilterEnum; -import org.dbsyncer.connector.enums.QuartzFilterEnum; +import org.dbsyncer.sdk.enums.QuartzFilterEnum; import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.parser.enums.CommandEnum; import org.dbsyncer.parser.enums.ConvertEnum; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/enums/QuartzFilterEnum.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/QuartzFilterEnum.java similarity index 87% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/enums/QuartzFilterEnum.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/QuartzFilterEnum.java index e769410e..f54771f2 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/enums/QuartzFilterEnum.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/QuartzFilterEnum.java @@ -1,11 +1,11 @@ -package org.dbsyncer.connector.enums; +package org.dbsyncer.sdk.enums; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.quartz.QuartzFilter; -import org.dbsyncer.connector.quartz.filter.DateFilter; -import org.dbsyncer.connector.quartz.filter.TimestampFilter; -import org.dbsyncer.connector.quartz.filter.YesDateFilter; -import org.dbsyncer.connector.quartz.filter.YesTimestampFilter; +import org.dbsyncer.sdk.quartz.QuartzFilter; +import org.dbsyncer.sdk.quartz.filter.DateFilter; +import org.dbsyncer.sdk.quartz.filter.TimestampFilter; +import org.dbsyncer.sdk.quartz.filter.YesDateFilter; +import org.dbsyncer.sdk.quartz.filter.YesTimestampFilter; /** * @author AE86 diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/AbstractDatabaseListener.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/AbstractDatabaseListener.java similarity index 82% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/AbstractDatabaseListener.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/AbstractDatabaseListener.java index fc609a1d..4e93cef6 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/AbstractDatabaseListener.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/AbstractDatabaseListener.java @@ -1,11 +1,10 @@ -package org.dbsyncer.connector; +package org.dbsyncer.sdk.listener; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.sdk.connector.database.AbstractDatabaseConnector; import org.dbsyncer.sdk.connector.database.DatabaseConnectorInstance; import org.dbsyncer.sdk.constant.ConnectorConstant; -import org.dbsyncer.sdk.listener.ChangedEvent; import org.dbsyncer.sdk.listener.event.RowChangedEvent; import org.dbsyncer.sdk.model.Field; import org.dbsyncer.sdk.model.MetaInfo; @@ -78,13 +77,13 @@ public abstract class AbstractDatabaseListener extends AbstractListener { * 初始化Dql连接配置 */ protected void postProcessDqlBeforeInitialization() { - DatabaseConnectorInstance connectorInstance = (DatabaseConnectorInstance) connectorFactory.connect(connectorConfig); - AbstractDatabaseConnector connectorService = (AbstractDatabaseConnector) connectorFactory.getConnectorService(connectorConfig.getConnectorType()); - String quotation = connectorService.buildSqlWithQuotation(); + DatabaseConnectorInstance instance = (DatabaseConnectorInstance) connectorInstance; + AbstractDatabaseConnector service = (AbstractDatabaseConnector) connectorService; + String quotation = service.buildSqlWithQuotation(); // <用户表, MY_USER> Map tableMap = new HashMap<>(); - connectorInstance.getConfig().getSqlTables().forEach(s -> tableMap.put(s.getSqlName(), s.getTable())); + instance.getConfig().getSqlTables().forEach(s -> tableMap.put(s.getSqlName(), s.getTable())); // 清空默认表名 filterTable.clear(); for (Table t : sourceTable) { @@ -95,7 +94,7 @@ public abstract class AbstractDatabaseListener extends AbstractListener { Assert.hasText(sql, "The sql is null."); Assert.hasText(tableName, "The tableName is null."); - MetaInfo metaInfo = connectorFactory.getMetaInfo(connectorInstance, sqlName); + MetaInfo metaInfo = service.getMetaInfo(instance, sqlName); final List column = metaInfo.getColumn(); Assert.notEmpty(column, String.format("The column of table name '%s' is empty.", sqlName)); @@ -107,7 +106,7 @@ public abstract class AbstractDatabaseListener extends AbstractListener { boolean notContainsWhere = !StringUtil.contains(sql, " WHERE "); querySql.append(notContainsWhere ? " WHERE " : " AND "); PrimaryKeyUtil.buildSql(querySql, primaryKeys, quotation, " AND ", " = ? ", notContainsWhere); - DqlMapper dqlMapper = new DqlMapper(connectorInstance, sqlName, querySql.toString(), column, getPrimaryKeyIndexArray(column, primaryKeys)); + DqlMapper dqlMapper = new DqlMapper(instance, sqlName, querySql.toString(), column, getPrimaryKeyIndexArray(column, primaryKeys)); if (!dqlMap.containsKey(tableName)) { dqlMap.putIfAbsent(tableName, new ArrayList<>()); } @@ -140,7 +139,7 @@ public abstract class AbstractDatabaseListener extends AbstractListener { private void queryDqlData(DqlMapper dqlMapper, List data) { if (!CollectionUtils.isEmpty(data)) { - Map row = dqlMapper.connectorInstance.execute(databaseTemplate -> { + Map row = dqlMapper.instance.execute(databaseTemplate -> { int size = dqlMapper.primaryKeyIndexArray.length; Object[] args = new Object[size]; for (int i = 0; i < size; i++) { @@ -156,14 +155,14 @@ public abstract class AbstractDatabaseListener extends AbstractListener { } final class DqlMapper { - DatabaseConnectorInstance connectorInstance; + DatabaseConnectorInstance instance; String sqlName; String sql; List column; Integer[] primaryKeyIndexArray; - public DqlMapper(DatabaseConnectorInstance connectorInstance, String sqlName, String sql, List column, Integer[] primaryKeyIndexArray) { - this.connectorInstance = connectorInstance; + public DqlMapper(DatabaseConnectorInstance instance, String sqlName, String sql, List column, Integer[] primaryKeyIndexArray) { + this.instance = instance; this.sqlName = sqlName; this.sql = sql; this.column = column; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/AbstractListener.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/AbstractListener.java similarity index 88% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/AbstractListener.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/AbstractListener.java index a541b48b..483d15f3 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/AbstractListener.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/AbstractListener.java @@ -1,15 +1,14 @@ -package org.dbsyncer.connector; +package org.dbsyncer.sdk.listener; import org.dbsyncer.common.util.CollectionUtils; -import org.dbsyncer.connector.scheduled.ScheduledTaskService; +import org.dbsyncer.sdk.scheduled.ScheduledTaskService; import org.dbsyncer.sdk.config.ListenerConfig; +import org.dbsyncer.sdk.connector.ConnectorInstance; import org.dbsyncer.sdk.constant.ConnectorConstant; -import org.dbsyncer.sdk.listener.ChangedEvent; -import org.dbsyncer.sdk.listener.Listener; -import org.dbsyncer.sdk.listener.Watcher; import org.dbsyncer.sdk.model.ChangedOffset; import org.dbsyncer.sdk.model.ConnectorConfig; import org.dbsyncer.sdk.model.Table; +import org.dbsyncer.sdk.spi.ConnectorService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -29,7 +28,8 @@ public abstract class AbstractListener implements Listener { private final Logger logger = LoggerFactory.getLogger(getClass()); private final int FLUSH_DELAYED_SECONDS = 20; - protected ConnectorFactory connectorFactory; + protected ConnectorInstance connectorInstance; + protected ConnectorService connectorService; protected ScheduledTaskService scheduledTaskService; protected ConnectorConfig connectorConfig; protected ListenerConfig listenerConfig; @@ -119,8 +119,12 @@ public abstract class AbstractListener implements Listener { } } - public void setConnectorFactory(ConnectorFactory connectorFactory) { - this.connectorFactory = connectorFactory; + public void setConnectorInstance(ConnectorInstance connectorInstance) { + this.connectorInstance = connectorInstance; + } + + public void setConnectorService(ConnectorService connectorService) { + this.connectorService = connectorService; } public void setScheduledTaskService(ScheduledTaskService scheduledTaskService) { diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/quartz/AbstractQuartzListener.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/AbstractQuartzListener.java similarity index 93% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/quartz/AbstractQuartzListener.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/AbstractQuartzListener.java index 3a1ea6bd..0f541027 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/quartz/AbstractQuartzListener.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/AbstractQuartzListener.java @@ -1,16 +1,16 @@ -package org.dbsyncer.connector.quartz; +package org.dbsyncer.sdk.listener; import org.dbsyncer.common.model.Result; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.common.util.UUIDUtil; -import org.dbsyncer.connector.AbstractListener; -import org.dbsyncer.connector.scheduled.ScheduledTaskJob; import org.dbsyncer.sdk.config.ReaderConfig; -import org.dbsyncer.sdk.connector.ConnectorInstance; import org.dbsyncer.sdk.constant.ConnectorConstant; import org.dbsyncer.sdk.listener.event.ScanChangedEvent; import org.dbsyncer.sdk.model.Table; +import org.dbsyncer.sdk.quartz.Point; +import org.dbsyncer.sdk.quartz.TableGroupQuartzCommand; +import org.dbsyncer.sdk.scheduled.ScheduledTaskJob; import org.dbsyncer.sdk.util.PrimaryKeyUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -103,14 +103,13 @@ public abstract class AbstractQuartzListener extends AbstractListener implements boolean supportedCursor = StringUtil.isNotBlank(command.get(ConnectorConstant.OPERTION_QUERY_CURSOR)); // 检查增量点 - ConnectorInstance connectionInstance = connectorFactory.connect(connectorConfig); Point point = checkLastPoint(command, index); int pageIndex = 1; Object[] cursors = PrimaryKeyUtil.getLastCursors(snapshot.get(index + CURSOR)); while (running) { ReaderConfig readerConfig = new ReaderConfig(table, point.getCommand(), point.getArgs(), supportedCursor, cursors, pageIndex++, READ_NUM); - Result reader = connectorFactory.reader(connectionInstance, readerConfig); + Result reader = connectorService.reader(connectorInstance, readerConfig); List data = reader.getSuccessData(); if (CollectionUtils.isEmpty(data)) { cursors = new Object[0]; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/quartz/DatabaseQuartzListener.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/quartz/DatabaseQuartzListener.java similarity index 96% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/quartz/DatabaseQuartzListener.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/quartz/DatabaseQuartzListener.java index 7d973ca6..8a7ae7c7 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/quartz/DatabaseQuartzListener.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/quartz/DatabaseQuartzListener.java @@ -1,9 +1,10 @@ -package org.dbsyncer.connector.quartz; +package org.dbsyncer.sdk.quartz; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.enums.QuartzFilterEnum; +import org.dbsyncer.sdk.enums.QuartzFilterEnum; import org.dbsyncer.sdk.constant.ConnectorConstant; +import org.dbsyncer.sdk.listener.AbstractQuartzListener; import org.springframework.util.Assert; import java.util.ArrayList; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/quartz/Point.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/quartz/Point.java similarity index 97% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/quartz/Point.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/quartz/Point.java index e790c8ad..8c47d390 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/quartz/Point.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/quartz/Point.java @@ -1,4 +1,4 @@ -package org.dbsyncer.connector.quartz; +package org.dbsyncer.sdk.quartz; import org.dbsyncer.common.util.StringUtil; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/quartz/QuartzFilter.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/quartz/QuartzFilter.java similarity index 91% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/quartz/QuartzFilter.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/quartz/QuartzFilter.java index 62d171fa..ec64c6c7 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/quartz/QuartzFilter.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/quartz/QuartzFilter.java @@ -1,4 +1,4 @@ -package org.dbsyncer.connector.quartz; +package org.dbsyncer.sdk.quartz; public interface QuartzFilter { diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/quartz/TableGroupQuartzCommand.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/quartz/TableGroupQuartzCommand.java similarity index 94% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/quartz/TableGroupQuartzCommand.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/quartz/TableGroupQuartzCommand.java index 13dcf479..3cb740ac 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/quartz/TableGroupQuartzCommand.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/quartz/TableGroupQuartzCommand.java @@ -1,4 +1,4 @@ -package org.dbsyncer.connector.quartz; +package org.dbsyncer.sdk.quartz; import org.dbsyncer.sdk.model.Table; import org.dbsyncer.sdk.util.PrimaryKeyUtil; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/quartz/filter/DateFilter.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/quartz/filter/DateFilter.java similarity index 86% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/quartz/filter/DateFilter.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/quartz/filter/DateFilter.java index 0ade9c69..3d56586c 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/quartz/filter/DateFilter.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/quartz/filter/DateFilter.java @@ -1,7 +1,7 @@ -package org.dbsyncer.connector.quartz.filter; +package org.dbsyncer.sdk.quartz.filter; import org.dbsyncer.common.util.DateFormatUtil; -import org.dbsyncer.connector.quartz.QuartzFilter; +import org.dbsyncer.sdk.quartz.QuartzFilter; import java.sql.Date; import java.time.Instant; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/quartz/filter/TimestampFilter.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/quartz/filter/TimestampFilter.java similarity index 86% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/quartz/filter/TimestampFilter.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/quartz/filter/TimestampFilter.java index c04447d9..0f3d85bc 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/quartz/filter/TimestampFilter.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/quartz/filter/TimestampFilter.java @@ -1,6 +1,6 @@ -package org.dbsyncer.connector.quartz.filter; +package org.dbsyncer.sdk.quartz.filter; -import org.dbsyncer.connector.quartz.QuartzFilter; +import org.dbsyncer.sdk.quartz.QuartzFilter; import java.sql.Timestamp; import java.time.Instant; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/quartz/filter/YesDateFilter.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/quartz/filter/YesDateFilter.java similarity index 92% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/quartz/filter/YesDateFilter.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/quartz/filter/YesDateFilter.java index 0601f378..c3c41622 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/quartz/filter/YesDateFilter.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/quartz/filter/YesDateFilter.java @@ -1,7 +1,7 @@ -package org.dbsyncer.connector.quartz.filter; +package org.dbsyncer.sdk.quartz.filter; import org.dbsyncer.common.util.DateFormatUtil; -import org.dbsyncer.connector.quartz.QuartzFilter; +import org.dbsyncer.sdk.quartz.QuartzFilter; import java.sql.Date; import java.time.LocalDateTime; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/quartz/filter/YesTimestampFilter.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/quartz/filter/YesTimestampFilter.java similarity index 90% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/quartz/filter/YesTimestampFilter.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/quartz/filter/YesTimestampFilter.java index 67e3f2ea..67634e87 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/quartz/filter/YesTimestampFilter.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/quartz/filter/YesTimestampFilter.java @@ -1,6 +1,6 @@ -package org.dbsyncer.connector.quartz.filter; +package org.dbsyncer.sdk.quartz.filter; -import org.dbsyncer.connector.quartz.QuartzFilter; +import org.dbsyncer.sdk.quartz.QuartzFilter; import java.sql.Timestamp; import java.time.LocalDateTime; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/scheduled/ScheduledTaskJob.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/scheduled/ScheduledTaskJob.java similarity index 74% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/scheduled/ScheduledTaskJob.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/scheduled/ScheduledTaskJob.java index d00f8b98..b40e5f2a 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/scheduled/ScheduledTaskJob.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/scheduled/ScheduledTaskJob.java @@ -1,4 +1,4 @@ -package org.dbsyncer.connector.scheduled; +package org.dbsyncer.sdk.scheduled; /** * @version 1.0.0 diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/scheduled/ScheduledTaskService.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/scheduled/ScheduledTaskService.java similarity index 94% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/scheduled/ScheduledTaskService.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/scheduled/ScheduledTaskService.java index 35b6159a..f73367d3 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/scheduled/ScheduledTaskService.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/scheduled/ScheduledTaskService.java @@ -1,4 +1,4 @@ -package org.dbsyncer.connector.scheduled; +package org.dbsyncer.sdk.scheduled; public interface ScheduledTaskService { diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/scheduled/impl/ScheduledTaskServiceImpl.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/scheduled/impl/ScheduledTaskServiceImpl.java similarity index 94% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/scheduled/impl/ScheduledTaskServiceImpl.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/scheduled/impl/ScheduledTaskServiceImpl.java index 5184a95b..3f636360 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/scheduled/impl/ScheduledTaskServiceImpl.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/scheduled/impl/ScheduledTaskServiceImpl.java @@ -1,8 +1,8 @@ -package org.dbsyncer.connector.scheduled.impl; +package org.dbsyncer.sdk.scheduled.impl; import org.dbsyncer.common.CommonException; -import org.dbsyncer.connector.scheduled.ScheduledTaskJob; -import org.dbsyncer.connector.scheduled.ScheduledTaskService; +import org.dbsyncer.sdk.scheduled.ScheduledTaskJob; +import org.dbsyncer.sdk.scheduled.ScheduledTaskService; import org.dbsyncer.common.util.UUIDUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dbsyncer-web/src/main/resources/application.properties b/dbsyncer-web/src/main/resources/application.properties index c33a57f1..525d92ad 100644 --- a/dbsyncer-web/src/main/resources/application.properties +++ b/dbsyncer-web/src/main/resources/application.properties @@ -46,8 +46,8 @@ dbsyncer.parser.table.group.buffer-period-millisecond=300 #storage # 是否使用MySQL存储配置（false-关闭; true-开启） # false: 保存磁盘/data/config(驱动配置)|data(按驱动分别存储增量数据)|log(系统日志)} -dbsyncer.storage.support.mysql.enabled=false -dbsyncer.storage.support.mysql.config.url=jdbc:mysql://127.0.0.1:3306/dbsyncer?rewriteBatchedStatements=true&seUnicode=true&characterEncoding=UTF8&serverTimezone=Asia/Shanghai&useSSL=false&verifyServerCertificate=false&autoReconnect=true +dbsyncer.storage.support.mysql.enabled=true +dbsyncer.storage.support.mysql.config.url=jdbc:mysql://127.0.0.1:3305/dbsyncer?rewriteBatchedStatements=true&seUnicode=true&characterEncoding=UTF8&serverTimezone=Asia/Shanghai&useSSL=false&verifyServerCertificate=false&autoReconnect=true dbsyncer.storage.support.mysql.config.username=root dbsyncer.storage.support.mysql.config.password=123 # [StorageBufferActuator]线程数 -- Gitee From 0a76d71cd8a54add9ff5f5c0d327c9da63f7c773 Mon Sep 17 00:00:00 2001 From: AE86 <836391306@qq.com> Date: Wed, 22 Nov 2023 02:34:42 +0800 Subject: [PATCH 26/98] fix bug --- .../org/dbsyncer/connector/es/ESQuartzListener.java | 4 ++-- .../org/dbsyncer/connector/mysql/MySQLConnector.java | 2 +- .../org/dbsyncer/connector/oracle/OracleConnector.java | 2 +- .../connector/postgresql/PostgreSQLConnector.java | 2 +- .../org/dbsyncer/connector/sql/DQLMySQLConnector.java | 2 +- .../org/dbsyncer/connector/sql/DQLOracleConnector.java | 2 +- .../dbsyncer/connector/sql/DQLPostgreSQLConnector.java | 2 +- .../dbsyncer/connector/sql/DQLSqlServerConnector.java | 2 +- .../connector/sqlserver/SqlServerConnector.java | 2 +- .../org/dbsyncer/manager/impl/IncrementPuller.java | 2 +- .../java/org/dbsyncer/sdk/enums/QuartzFilterEnum.java | 10 +++++----- .../dbsyncer/sdk/listener/AbstractQuartzListener.java | 4 ++-- .../{quartz => listener}/DatabaseQuartzListener.java | 6 +++--- .../sdk/{quartz => listener}/QuartzFilter.java | 2 +- .../sdk/{quartz => listener}/filter/DateFilter.java | 4 ++-- .../{quartz => listener}/filter/TimestampFilter.java | 4 ++-- .../sdk/{quartz => listener}/filter/YesDateFilter.java | 4 ++-- .../filter/YesTimestampFilter.java | 4 ++-- .../java/org/dbsyncer/sdk/{quartz => model}/Point.java | 10 +++++++--- .../sdk/{quartz => model}/TableGroupQuartzCommand.java | 3 +-- dbsyncer-web/src/main/resources/application.properties | 4 ++-- 21 files changed, 40 insertions(+), 37 deletions(-) rename dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/{quartz => listener}/DatabaseQuartzListener.java (97%) rename dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/{quartz => listener}/QuartzFilter.java (92%) rename dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/{quartz => listener}/filter/DateFilter.java (87%) rename dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/{quartz => listener}/filter/TimestampFilter.java (87%) rename dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/{quartz => listener}/filter/YesDateFilter.java (92%) rename dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/{quartz => listener}/filter/YesTimestampFilter.java (91%) rename dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/{quartz => model}/Point.java (85%) rename dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/{quartz => model}/TableGroupQuartzCommand.java (90%) diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESQuartzListener.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESQuartzListener.java index 81b5e89d..7f7de604 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESQuartzListener.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESQuartzListener.java @@ -6,8 +6,8 @@ import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.connector.ConnectorException; import org.dbsyncer.sdk.enums.QuartzFilterEnum; import org.dbsyncer.sdk.listener.AbstractQuartzListener; -import org.dbsyncer.sdk.quartz.Point; -import org.dbsyncer.sdk.quartz.QuartzFilter; +import org.dbsyncer.sdk.model.Point; +import org.dbsyncer.sdk.listener.QuartzFilter; import org.dbsyncer.sdk.constant.ConnectorConstant; import org.dbsyncer.sdk.model.Filter; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/MySQLConnector.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/MySQLConnector.java index ee0d1ee0..12abccbe 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/MySQLConnector.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/MySQLConnector.java @@ -1,7 +1,7 @@ package org.dbsyncer.connector.mysql; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.sdk.quartz.DatabaseQuartzListener; +import org.dbsyncer.sdk.listener.DatabaseQuartzListener; import org.dbsyncer.sdk.config.ReaderConfig; import org.dbsyncer.sdk.connector.database.AbstractDatabaseConnector; import org.dbsyncer.sdk.constant.DatabaseConstant; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/OracleConnector.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/OracleConnector.java index 3db09341..8a8ee131 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/OracleConnector.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/OracleConnector.java @@ -1,7 +1,7 @@ package org.dbsyncer.connector.oracle; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.sdk.quartz.DatabaseQuartzListener; +import org.dbsyncer.sdk.listener.DatabaseQuartzListener; import org.dbsyncer.sdk.config.CommandConfig; import org.dbsyncer.sdk.config.DatabaseConfig; import org.dbsyncer.sdk.config.ReaderConfig; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLConnector.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLConnector.java index c48bea8e..fed43089 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLConnector.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLConnector.java @@ -1,7 +1,7 @@ package org.dbsyncer.connector.postgresql; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.sdk.quartz.DatabaseQuartzListener; +import org.dbsyncer.sdk.listener.DatabaseQuartzListener; import org.dbsyncer.sdk.config.CommandConfig; import org.dbsyncer.sdk.config.DatabaseConfig; import org.dbsyncer.sdk.config.ReaderConfig; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLMySQLConnector.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLMySQLConnector.java index 22b55456..0468042d 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLMySQLConnector.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLMySQLConnector.java @@ -1,7 +1,7 @@ package org.dbsyncer.connector.sql; import org.dbsyncer.connector.mysql.DqlMySQLListener; -import org.dbsyncer.sdk.quartz.DatabaseQuartzListener; +import org.dbsyncer.sdk.listener.DatabaseQuartzListener; import org.dbsyncer.sdk.config.CommandConfig; import org.dbsyncer.sdk.config.ReaderConfig; import org.dbsyncer.sdk.connector.database.AbstractDQLConnector; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLOracleConnector.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLOracleConnector.java index e37cbd65..9387496b 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLOracleConnector.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLOracleConnector.java @@ -1,7 +1,7 @@ package org.dbsyncer.connector.sql; import org.dbsyncer.connector.oracle.DqlOracleListener; -import org.dbsyncer.sdk.quartz.DatabaseQuartzListener; +import org.dbsyncer.sdk.listener.DatabaseQuartzListener; import org.dbsyncer.sdk.config.ReaderConfig; import org.dbsyncer.sdk.connector.database.AbstractDQLConnector; import org.dbsyncer.sdk.constant.DatabaseConstant; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLPostgreSQLConnector.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLPostgreSQLConnector.java index a8ff60da..8e6547c0 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLPostgreSQLConnector.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLPostgreSQLConnector.java @@ -1,7 +1,7 @@ package org.dbsyncer.connector.sql; import org.dbsyncer.connector.postgresql.DqlPostgreSQLListener; -import org.dbsyncer.sdk.quartz.DatabaseQuartzListener; +import org.dbsyncer.sdk.listener.DatabaseQuartzListener; import org.dbsyncer.sdk.config.ReaderConfig; import org.dbsyncer.sdk.connector.database.AbstractDQLConnector; import org.dbsyncer.sdk.constant.DatabaseConstant; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLSqlServerConnector.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLSqlServerConnector.java index 9538d614..7949e12b 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLSqlServerConnector.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLSqlServerConnector.java @@ -1,7 +1,7 @@ package org.dbsyncer.connector.sql; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.sdk.quartz.DatabaseQuartzListener; +import org.dbsyncer.sdk.listener.DatabaseQuartzListener; import org.dbsyncer.connector.sqlserver.DqlSqlServerListener; import org.dbsyncer.sdk.config.ReaderConfig; import org.dbsyncer.sdk.connector.database.AbstractDQLConnector; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/SqlServerConnector.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/SqlServerConnector.java index 878541c6..12eeaada 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/SqlServerConnector.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/SqlServerConnector.java @@ -2,7 +2,7 @@ package org.dbsyncer.connector.sqlserver; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.sdk.quartz.DatabaseQuartzListener; +import org.dbsyncer.sdk.listener.DatabaseQuartzListener; import org.dbsyncer.sdk.config.CommandConfig; import org.dbsyncer.sdk.config.DatabaseConfig; import org.dbsyncer.sdk.config.ReaderConfig; diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/IncrementPuller.java b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/IncrementPuller.java index baa5ff5c..71b9a9f6 100644 --- a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/IncrementPuller.java +++ b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/IncrementPuller.java @@ -23,7 +23,7 @@ import org.dbsyncer.sdk.listener.Listener; import org.dbsyncer.sdk.model.ChangedOffset; import org.dbsyncer.sdk.model.ConnectorConfig; import org.dbsyncer.sdk.model.Table; -import org.dbsyncer.sdk.quartz.TableGroupQuartzCommand; +import org.dbsyncer.sdk.model.TableGroupQuartzCommand; import org.dbsyncer.sdk.scheduled.ScheduledTaskJob; import org.dbsyncer.sdk.scheduled.ScheduledTaskService; import org.slf4j.Logger; diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/QuartzFilterEnum.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/QuartzFilterEnum.java index f54771f2..7da35266 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/QuartzFilterEnum.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/QuartzFilterEnum.java @@ -1,11 +1,11 @@ package org.dbsyncer.sdk.enums; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.sdk.quartz.QuartzFilter; -import org.dbsyncer.sdk.quartz.filter.DateFilter; -import org.dbsyncer.sdk.quartz.filter.TimestampFilter; -import org.dbsyncer.sdk.quartz.filter.YesDateFilter; -import org.dbsyncer.sdk.quartz.filter.YesTimestampFilter; +import org.dbsyncer.sdk.listener.QuartzFilter; +import org.dbsyncer.sdk.listener.filter.DateFilter; +import org.dbsyncer.sdk.listener.filter.TimestampFilter; +import org.dbsyncer.sdk.listener.filter.YesDateFilter; +import org.dbsyncer.sdk.listener.filter.YesTimestampFilter; /** * @author AE86 diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/AbstractQuartzListener.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/AbstractQuartzListener.java index 0f541027..b1d05743 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/AbstractQuartzListener.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/AbstractQuartzListener.java @@ -7,9 +7,9 @@ import org.dbsyncer.common.util.UUIDUtil; import org.dbsyncer.sdk.config.ReaderConfig; import org.dbsyncer.sdk.constant.ConnectorConstant; import org.dbsyncer.sdk.listener.event.ScanChangedEvent; +import org.dbsyncer.sdk.model.Point; import org.dbsyncer.sdk.model.Table; -import org.dbsyncer.sdk.quartz.Point; -import org.dbsyncer.sdk.quartz.TableGroupQuartzCommand; +import org.dbsyncer.sdk.model.TableGroupQuartzCommand; import org.dbsyncer.sdk.scheduled.ScheduledTaskJob; import org.dbsyncer.sdk.util.PrimaryKeyUtil; import org.slf4j.Logger; diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/quartz/DatabaseQuartzListener.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/DatabaseQuartzListener.java similarity index 97% rename from dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/quartz/DatabaseQuartzListener.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/DatabaseQuartzListener.java index 8a7ae7c7..ceaf66a0 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/quartz/DatabaseQuartzListener.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/DatabaseQuartzListener.java @@ -1,10 +1,10 @@ -package org.dbsyncer.sdk.quartz; +package org.dbsyncer.sdk.listener; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.sdk.enums.QuartzFilterEnum; import org.dbsyncer.sdk.constant.ConnectorConstant; -import org.dbsyncer.sdk.listener.AbstractQuartzListener; +import org.dbsyncer.sdk.enums.QuartzFilterEnum; +import org.dbsyncer.sdk.model.Point; import org.springframework.util.Assert; import java.util.ArrayList; diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/quartz/QuartzFilter.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/QuartzFilter.java similarity index 92% rename from dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/quartz/QuartzFilter.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/QuartzFilter.java index ec64c6c7..36e24910 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/quartz/QuartzFilter.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/QuartzFilter.java @@ -1,4 +1,4 @@ -package org.dbsyncer.sdk.quartz; +package org.dbsyncer.sdk.listener; public interface QuartzFilter { diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/quartz/filter/DateFilter.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/filter/DateFilter.java similarity index 87% rename from dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/quartz/filter/DateFilter.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/filter/DateFilter.java index 3d56586c..05865815 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/quartz/filter/DateFilter.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/filter/DateFilter.java @@ -1,7 +1,7 @@ -package org.dbsyncer.sdk.quartz.filter; +package org.dbsyncer.sdk.listener.filter; import org.dbsyncer.common.util.DateFormatUtil; -import org.dbsyncer.sdk.quartz.QuartzFilter; +import org.dbsyncer.sdk.listener.QuartzFilter; import java.sql.Date; import java.time.Instant; diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/quartz/filter/TimestampFilter.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/filter/TimestampFilter.java similarity index 87% rename from dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/quartz/filter/TimestampFilter.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/filter/TimestampFilter.java index 0f3d85bc..a36a34c8 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/quartz/filter/TimestampFilter.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/filter/TimestampFilter.java @@ -1,6 +1,6 @@ -package org.dbsyncer.sdk.quartz.filter; +package org.dbsyncer.sdk.listener.filter; -import org.dbsyncer.sdk.quartz.QuartzFilter; +import org.dbsyncer.sdk.listener.QuartzFilter; import java.sql.Timestamp; import java.time.Instant; diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/quartz/filter/YesDateFilter.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/filter/YesDateFilter.java similarity index 92% rename from dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/quartz/filter/YesDateFilter.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/filter/YesDateFilter.java index c3c41622..f18f95cd 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/quartz/filter/YesDateFilter.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/filter/YesDateFilter.java @@ -1,7 +1,7 @@ -package org.dbsyncer.sdk.quartz.filter; +package org.dbsyncer.sdk.listener.filter; import org.dbsyncer.common.util.DateFormatUtil; -import org.dbsyncer.sdk.quartz.QuartzFilter; +import org.dbsyncer.sdk.listener.QuartzFilter; import java.sql.Date; import java.time.LocalDateTime; diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/quartz/filter/YesTimestampFilter.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/filter/YesTimestampFilter.java similarity index 91% rename from dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/quartz/filter/YesTimestampFilter.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/filter/YesTimestampFilter.java index 67634e87..d9c83fd5 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/quartz/filter/YesTimestampFilter.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/filter/YesTimestampFilter.java @@ -1,6 +1,6 @@ -package org.dbsyncer.sdk.quartz.filter; +package org.dbsyncer.sdk.listener.filter; -import org.dbsyncer.sdk.quartz.QuartzFilter; +import org.dbsyncer.sdk.listener.QuartzFilter; import java.sql.Timestamp; import java.time.LocalDateTime; diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/quartz/Point.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/Point.java similarity index 85% rename from dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/quartz/Point.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/Point.java index 8c47d390..1e02ca0f 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/quartz/Point.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/Point.java @@ -1,8 +1,12 @@ -package org.dbsyncer.sdk.quartz; +package org.dbsyncer.sdk.model; import org.dbsyncer.common.util.StringUtil; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; public class Point { @@ -33,7 +37,7 @@ public class Point { } public void refresh() { - if(StringUtil.isNotBlank(beginKey) && StringUtil.isNotBlank(beginValue)){ + if (StringUtil.isNotBlank(beginKey) && StringUtil.isNotBlank(beginValue)) { position.put(beginKey, beginValue); refreshed = true; } diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/quartz/TableGroupQuartzCommand.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/TableGroupQuartzCommand.java similarity index 90% rename from dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/quartz/TableGroupQuartzCommand.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/TableGroupQuartzCommand.java index 3cb740ac..90da4f87 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/quartz/TableGroupQuartzCommand.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/TableGroupQuartzCommand.java @@ -1,6 +1,5 @@ -package org.dbsyncer.sdk.quartz; +package org.dbsyncer.sdk.model; -import org.dbsyncer.sdk.model.Table; import org.dbsyncer.sdk.util.PrimaryKeyUtil; import java.util.List; diff --git a/dbsyncer-web/src/main/resources/application.properties b/dbsyncer-web/src/main/resources/application.properties index 525d92ad..c33a57f1 100644 --- a/dbsyncer-web/src/main/resources/application.properties +++ b/dbsyncer-web/src/main/resources/application.properties @@ -46,8 +46,8 @@ dbsyncer.parser.table.group.buffer-period-millisecond=300 #storage # 是否使用MySQL存储配置（false-关闭; true-开启） # false: 保存磁盘/data/config(驱动配置)|data(按驱动分别存储增量数据)|log(系统日志)} -dbsyncer.storage.support.mysql.enabled=true -dbsyncer.storage.support.mysql.config.url=jdbc:mysql://127.0.0.1:3305/dbsyncer?rewriteBatchedStatements=true&seUnicode=true&characterEncoding=UTF8&serverTimezone=Asia/Shanghai&useSSL=false&verifyServerCertificate=false&autoReconnect=true +dbsyncer.storage.support.mysql.enabled=false +dbsyncer.storage.support.mysql.config.url=jdbc:mysql://127.0.0.1:3306/dbsyncer?rewriteBatchedStatements=true&seUnicode=true&characterEncoding=UTF8&serverTimezone=Asia/Shanghai&useSSL=false&verifyServerCertificate=false&autoReconnect=true dbsyncer.storage.support.mysql.config.username=root dbsyncer.storage.support.mysql.config.password=123 # [StorageBufferActuator]线程数 -- Gitee From 7fd023d952d7310a49191e017740fe45c7152a06 Mon Sep 17 00:00:00 2001 From: AE86 <836391306@qq.com> Date: Wed, 22 Nov 2023 02:52:17 +0800 Subject: [PATCH 27/98] rename --- .../dbsyncer/connector/es/ESConnector.java | 44 +++++++------- .../connector/file/FileConnector.java | 32 +++++----- .../connector/kafka/KafkaConnector.java | 26 ++++----- .../sqlserver/SqlServerConnector.java | 12 ++-- .../src/main/test/ConnectionTest.java | 38 ++++++------ .../manager/impl/PreloadTemplate.java | 10 ++-- .../database/AbstractDQLConnector.java | 10 ++-- .../database/AbstractDatabaseConnector.java | 58 +++++++++---------- .../main/resources/public/plugin/plugin.html | 6 +- 9 files changed, 118 insertions(+), 118 deletions(-) diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESConnector.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESConnector.java index d0b9a6f0..2f28378b 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESConnector.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESConnector.java @@ -111,14 +111,14 @@ public final class ESConnector extends AbstractConnector implements ConnectorSer } @Override - public void disconnect(ESConnectorInstance connectorMapper) { - connectorMapper.close(); + public void disconnect(ESConnectorInstance connectorInstance) { + connectorInstance.close(); } @Override - public boolean isAlive(ESConnectorInstance connectorMapper) { + public boolean isAlive(ESConnectorInstance connectorInstance) { try { - RestHighLevelClient client = connectorMapper.getConnection(); + RestHighLevelClient client = connectorInstance.getConnection(); return client.ping(RequestOptions.DEFAULT); } catch (IOException e) { logger.error(e.getMessage()); @@ -132,15 +132,15 @@ public final class ESConnector extends AbstractConnector implements ConnectorSer } @Override - public List
    getTable(ESConnectorInstance connectorMapper) { + public List
    getTable(ESConnectorInstance connectorInstance) { try { - ESConfig config = connectorMapper.getConfig(); + ESConfig config = connectorInstance.getConfig(); GetIndexRequest request = new GetIndexRequest(config.getIndex()); - GetIndexResponse indexResponse = connectorMapper.getConnection().indices().get(request, RequestOptions.DEFAULT); + GetIndexResponse indexResponse = connectorInstance.getConnection().indices().get(request, RequestOptions.DEFAULT); MappingMetadata mappingMetaData = indexResponse.getMappings().get(config.getIndex()); List
    tables = new ArrayList<>(); // 6.x 版本 - if (Version.V_7_0_0.after(connectorMapper.getVersion())) { + if (Version.V_7_0_0.after(connectorInstance.getVersion())) { Map sourceMap = XContentHelper.convertToMap(mappingMetaData.source().compressedReference(), true, null).v2(); sourceMap.keySet().forEach(tableName -> tables.add(new Table(tableName))); return tables; @@ -156,15 +156,15 @@ public final class ESConnector extends AbstractConnector implements ConnectorSer } @Override - public MetaInfo getMetaInfo(ESConnectorInstance connectorMapper, String tableName) { + public MetaInfo getMetaInfo(ESConnectorInstance connectorInstance, String tableName) { List fields = new ArrayList<>(); try { - ESConfig config = connectorMapper.getConfig(); + ESConfig config = connectorInstance.getConfig(); GetIndexRequest request = new GetIndexRequest(config.getIndex()); - GetIndexResponse indexResponse = connectorMapper.getConnection().indices().get(request, RequestOptions.DEFAULT); + GetIndexResponse indexResponse = connectorInstance.getConnection().indices().get(request, RequestOptions.DEFAULT); MappingMetadata mappingMetaData = indexResponse.getMappings().get(config.getIndex()); // 6.x 版本 - if (Version.V_7_0_0.after(connectorMapper.getVersion())) { + if (Version.V_7_0_0.after(connectorInstance.getVersion())) { Map sourceMap = XContentHelper.convertToMap(mappingMetaData.source().compressedReference(), true, null).v2(); parseProperties(fields, (Map) sourceMap.get(tableName)); return new MetaInfo().setColumn(fields); @@ -180,19 +180,19 @@ public final class ESConnector extends AbstractConnector implements ConnectorSer } @Override - public long getCount(ESConnectorInstance connectorMapper, Map command) { + public long getCount(ESConnectorInstance connectorInstance, Map command) { try { - ESConfig config = connectorMapper.getConfig(); + ESConfig config = connectorInstance.getConfig(); SearchSourceBuilder builder = new SearchSourceBuilder(); genSearchSourceBuilder(builder, command); // 7.x 版本以上 - if (Version.V_7_0_0.onOrBefore(connectorMapper.getVersion())) { + if (Version.V_7_0_0.onOrBefore(connectorInstance.getVersion())) { builder.trackTotalHits(true); } builder.from(0); builder.size(0); SearchRequest request = new SearchRequest(new String[]{config.getIndex()}, builder); - SearchResponse response = connectorMapper.getConnection().searchWithVersion(request, RequestOptions.DEFAULT); + SearchResponse response = connectorInstance.getConnection().searchWithVersion(request, RequestOptions.DEFAULT); return response.getHits().getTotalHits().value; } catch (IOException e) { logger.error(e.getMessage()); @@ -201,8 +201,8 @@ public final class ESConnector extends AbstractConnector implements ConnectorSer } @Override - public Result reader(ESConnectorInstance connectorMapper, ReaderConfig config) { - ESConfig cfg = connectorMapper.getConfig(); + public Result reader(ESConnectorInstance connectorInstance, ReaderConfig config) { + ESConfig cfg = connectorInstance.getConfig(); SearchSourceBuilder builder = new SearchSourceBuilder(); genSearchSourceBuilder(builder, config.getCommand()); builder.from((config.getPageIndex() - 1) * config.getPageSize()); @@ -220,7 +220,7 @@ public final class ESConnector extends AbstractConnector implements ConnectorSer try { SearchRequest rq = new SearchRequest(new String[]{cfg.getIndex()}, builder); - SearchResponse searchResponse = connectorMapper.getConnection().searchWithVersion(rq, RequestOptions.DEFAULT); + SearchResponse searchResponse = connectorInstance.getConnection().searchWithVersion(rq, RequestOptions.DEFAULT); SearchHits hits = searchResponse.getHits(); SearchHit[] searchHits = hits.getHits(); List> list = new ArrayList<>(); @@ -235,7 +235,7 @@ public final class ESConnector extends AbstractConnector implements ConnectorSer } @Override - public Result writer(ESConnectorInstance connectorMapper, WriterBatchConfig config) { + public Result writer(ESConnectorInstance connectorInstance, WriterBatchConfig config) { List data = config.getData(); if (CollectionUtils.isEmpty(data) || CollectionUtils.isEmpty(config.getFields())) { logger.error("writer data can not be empty."); @@ -243,7 +243,7 @@ public final class ESConnector extends AbstractConnector implements ConnectorSer } final Result result = new Result(); - final ESConfig cfg = connectorMapper.getConfig(); + final ESConfig cfg = connectorInstance.getConfig(); final List pkFields = PrimaryKeyUtil.findConfigPrimaryKeyFields(config); try { BulkRequest request = new BulkRequest(); @@ -251,7 +251,7 @@ public final class ESConnector extends AbstractConnector implements ConnectorSer final String pk = pkFields.get(0).getName(); data.forEach(row -> addRequest(request, cfg.getIndex(), config.getTableName(), config.getEvent(), String.valueOf(row.get(pk)), row)); - BulkResponse response = connectorMapper.getConnection().bulkWithVersion(request, RequestOptions.DEFAULT); + BulkResponse response = connectorInstance.getConnection().bulkWithVersion(request, RequestOptions.DEFAULT); RestStatus restStatus = response.status(); if (restStatus.getStatus() != RestStatus.OK.getStatus()) { throw new ConnectorException(String.format("error code:%s", restStatus.getStatus())); diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileConnector.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileConnector.java index fed6638f..5232e4b6 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileConnector.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileConnector.java @@ -78,20 +78,20 @@ public final class FileConnector extends AbstractConnector implements ConnectorS } @Override - public void disconnect(FileConnectorInstance connectorMapper) { + public void disconnect(FileConnectorInstance connectorInstance) { } @Override - public boolean isAlive(FileConnectorInstance connectorMapper) { - String fileDir = connectorMapper.getConnection(); + public boolean isAlive(FileConnectorInstance connectorInstance) { + String fileDir = connectorInstance.getConnection(); boolean alive = new File(fileDir).exists(); if (!alive) { logger.warn("can not find fileDir:{}", fileDir); return false; } - for (FileSchema fileSchema : connectorMapper.getFileSchemaList()) { - String filePath = connectorMapper.getFilePath(fileSchema.getFileName()); + for (FileSchema fileSchema : connectorInstance.getFileSchemaList()) { + String filePath = connectorInstance.getFilePath(fileSchema.getFileName()); if (!new File(filePath).exists()) { logger.warn("can not find file:{}", filePath); alive = false; @@ -113,18 +113,18 @@ public final class FileConnector extends AbstractConnector implements ConnectorS } @Override - public List
    getTable(FileConnectorInstance connectorMapper) { - return connectorMapper.getFileSchemaList().stream().map(fileSchema -> new Table(fileSchema.getFileName())).collect(Collectors.toList()); + public List
    getTable(FileConnectorInstance connectorInstance) { + return connectorInstance.getFileSchemaList().stream().map(fileSchema -> new Table(fileSchema.getFileName())).collect(Collectors.toList()); } @Override - public MetaInfo getMetaInfo(FileConnectorInstance connectorMapper, String tableName) { - FileSchema fileSchema = connectorMapper.getFileSchema(tableName); + public MetaInfo getMetaInfo(FileConnectorInstance connectorInstance, String tableName) { + FileSchema fileSchema = connectorInstance.getFileSchema(tableName); return new MetaInfo().setColumn(fileSchema.getFields()); } @Override - public long getCount(FileConnectorInstance connectorMapper, Map command) { + public long getCount(FileConnectorInstance connectorInstance, Map command) { AtomicLong count = new AtomicLong(); FileReader reader = null; try { @@ -143,12 +143,12 @@ public final class FileConnector extends AbstractConnector implements ConnectorS } @Override - public Result reader(FileConnectorInstance connectorMapper, ReaderConfig config) { + public Result reader(FileConnectorInstance connectorInstance, ReaderConfig config) { List> list = new ArrayList<>(); FileReader reader = null; try { - FileConfig fileConfig = connectorMapper.getConfig(); - FileSchema fileSchema = connectorMapper.getFileSchema(config.getCommand().get(FILE_NAME)); + FileConfig fileConfig = connectorInstance.getConfig(); + FileSchema fileSchema = connectorInstance.getFileSchema(config.getCommand().get(FILE_NAME)); final List fields = fileSchema.getFields(); Assert.notEmpty(fields, "The fields of file schema is empty."); final char separator = fileConfig.getSeparator(); @@ -178,7 +178,7 @@ public final class FileConnector extends AbstractConnector implements ConnectorS } @Override - public Result writer(FileConnectorInstance connectorMapper, WriterBatchConfig config) { + public Result writer(FileConnectorInstance connectorInstance, WriterBatchConfig config) { List data = config.getData(); if (CollectionUtils.isEmpty(data)) { logger.error("writer data can not be empty."); @@ -186,12 +186,12 @@ public final class FileConnector extends AbstractConnector implements ConnectorS } final List fields = config.getFields(); - final String separator = new String(new char[]{connectorMapper.getConfig().getSeparator()}); + final String separator = new String(new char[]{connectorInstance.getConfig().getSeparator()}); Result result = new Result(); OutputStream output = null; try { - final String filePath = connectorMapper.getFilePath(config.getCommand().get(FILE_NAME)); + final String filePath = connectorInstance.getFilePath(config.getCommand().get(FILE_NAME)); output = new FileOutputStream(filePath, true); List lines = data.stream().map(row -> { List array = new ArrayList<>(); diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/kafka/KafkaConnector.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/kafka/KafkaConnector.java index 76158000..5aaaafb4 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/kafka/KafkaConnector.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/kafka/KafkaConnector.java @@ -58,13 +58,13 @@ public class KafkaConnector extends AbstractConnector implements ConnectorServic } @Override - public void disconnect(KafkaConnectorInstance connectorMapper) { - connectorMapper.close(); + public void disconnect(KafkaConnectorInstance connectorInstance) { + connectorInstance.close(); } @Override - public boolean isAlive(KafkaConnectorInstance connectorMapper) { - return connectorMapper.getConnection().ping(); + public boolean isAlive(KafkaConnectorInstance connectorInstance) { + return connectorInstance.getConnection().ping(); } @Override @@ -73,31 +73,31 @@ public class KafkaConnector extends AbstractConnector implements ConnectorServic } @Override - public List
    getTable(KafkaConnectorInstance connectorMapper) { + public List
    getTable(KafkaConnectorInstance connectorInstance) { List
    topics = new ArrayList<>(); - topics.add(new Table(connectorMapper.getConfig().getTopic())); + topics.add(new Table(connectorInstance.getConfig().getTopic())); return topics; } @Override - public MetaInfo getMetaInfo(KafkaConnectorInstance connectorMapper, String tableName) { - KafkaConfig config = connectorMapper.getConfig(); + public MetaInfo getMetaInfo(KafkaConnectorInstance connectorInstance, String tableName) { + KafkaConfig config = connectorInstance.getConfig(); List fields = JsonUtil.jsonToArray(config.getFields(), Field.class); return new MetaInfo().setColumn(fields); } @Override - public long getCount(KafkaConnectorInstance connectorMapper, Map command) { + public long getCount(KafkaConnectorInstance connectorInstance, Map command) { return 0; } @Override - public Result reader(KafkaConnectorInstance connectorMapper, ReaderConfig config) { + public Result reader(KafkaConnectorInstance connectorInstance, ReaderConfig config) { throw new ConnectorException("Full synchronization is not supported"); } @Override - public Result writer(KafkaConnectorInstance connectorMapper, WriterBatchConfig config) { + public Result writer(KafkaConnectorInstance connectorInstance, WriterBatchConfig config) { List data = config.getData(); if (CollectionUtils.isEmpty(data) || CollectionUtils.isEmpty(config.getFields())) { logger.error("writer data can not be empty."); @@ -105,13 +105,13 @@ public class KafkaConnector extends AbstractConnector implements ConnectorServic } Result result = new Result(); - final KafkaConfig cfg = connectorMapper.getConfig(); + final KafkaConfig cfg = connectorInstance.getConfig(); final List pkFields = PrimaryKeyUtil.findConfigPrimaryKeyFields(config); try { String topic = cfg.getTopic(); // 默认取第一个主键 final String pk = pkFields.get(0).getName(); - data.forEach(row -> connectorMapper.getConnection().send(topic, String.valueOf(row.get(pk)), row)); + data.forEach(row -> connectorInstance.getConnection().send(topic, String.valueOf(row.get(pk)), row)); result.addSuccessData(data); } catch (Exception e) { // 记录错误数据 diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/SqlServerConnector.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/SqlServerConnector.java index 12eeaada..56d335fe 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/SqlServerConnector.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/SqlServerConnector.java @@ -36,10 +36,10 @@ public final class SqlServerConnector extends AbstractDatabaseConnector { } @Override - public List
    getTable(DatabaseConnectorInstance connectorMapper) { - DatabaseConfig config = connectorMapper.getConfig(); - List
    tables = getTables(connectorMapper, String.format(QUERY_TABLE, config.getSchema()), TableTypeEnum.TABLE); - tables.addAll(getTables(connectorMapper, QUERY_VIEW, TableTypeEnum.VIEW)); + public List
    getTable(DatabaseConnectorInstance connectorInstance) { + DatabaseConfig config = connectorInstance.getConfig(); + List
    tables = getTables(connectorInstance, String.format(QUERY_TABLE, config.getSchema()), TableTypeEnum.TABLE); + tables.addAll(getTables(connectorInstance, QUERY_VIEW, TableTypeEnum.VIEW)); return tables; } @@ -101,8 +101,8 @@ public final class SqlServerConnector extends AbstractDatabaseConnector { buildTableName(table.getName())); } - private List
    getTables(DatabaseConnectorInstance connectorMapper, String sql, TableTypeEnum type) { - List tableNames = connectorMapper.execute(databaseTemplate -> databaseTemplate.queryForList(sql, String.class)); + private List
    getTables(DatabaseConnectorInstance connectorInstance, String sql, TableTypeEnum type) { + List tableNames = connectorInstance.execute(databaseTemplate -> databaseTemplate.queryForList(sql, String.class)); if (!CollectionUtils.isEmpty(tableNames)) { return tableNames.stream().map(name -> new Table(name, type.getCode())).collect(Collectors.toList()); } diff --git a/dbsyncer-connector/src/main/test/ConnectionTest.java b/dbsyncer-connector/src/main/test/ConnectionTest.java index 9e85c0f7..99ab8f9c 100644 --- a/dbsyncer-connector/src/main/test/ConnectionTest.java +++ b/dbsyncer-connector/src/main/test/ConnectionTest.java @@ -32,10 +32,10 @@ public class ConnectionTest { @Test public void testByte() { - final DatabaseConnectorInstance connectorMapper = new DatabaseConnectorInstance(createOracleConfig()); + final DatabaseConnectorInstance connectorInstance = new DatabaseConnectorInstance(createOracleConfig()); String executeSql = "UPDATE \"my_user\" SET \"name\"=?,\"clo\"=? WHERE \"id\"=?"; - int[] execute = connectorMapper.execute(databaseTemplate -> + int[] execute = connectorInstance.execute(databaseTemplate -> databaseTemplate.batchUpdate(executeSql, new BatchPreparedStatementSetter() { @Override public void setValues(PreparedStatement ps, int i) { @@ -64,7 +64,7 @@ public class ConnectionTest { @Test public void testConnection() throws InterruptedException { - final DatabaseConnectorInstance connectorMapper = new DatabaseConnectorInstance(createSqlServerConfig()); + final DatabaseConnectorInstance connectorInstance = new DatabaseConnectorInstance(createSqlServerConfig()); // 模拟并发 final int threadSize = 100; @@ -80,7 +80,7 @@ public class ConnectionTest { // 模拟操作 System.out.println(String.format("%s %s:%s", LocalDateTime.now(), Thread.currentThread().getName(), k)); - Object execute = connectorMapper.execute(tem -> tem.queryForObject("select 1", Integer.class)); + Object execute = connectorInstance.execute(tem -> tem.queryForObject("select 1", Integer.class)); System.out.println(String.format("%s %s:%s execute=>%s", LocalDateTime.now(), Thread.currentThread().getName(), k, execute)); } catch (InterruptedException e) { @@ -109,7 +109,7 @@ public class ConnectionTest { @Test public void testBatchInsert() { - final DatabaseConnectorInstance connectorMapper = new DatabaseConnectorInstance(createMysqlConfig()); + final DatabaseConnectorInstance connectorInstance = new DatabaseConnectorInstance(createMysqlConfig()); long begin = Instant.now().toEpochMilli(); final int threadSize = 10; @@ -131,14 +131,14 @@ public class ConnectionTest { if (i % 10000 == 0) { System.out.println(i + "-----------------正在处理"); - batchUpdate(connectorMapper, pool, sql, dataList, 1000); + batchUpdate(connectorInstance, pool, sql, dataList, 1000); dataList.clear(); } } if(!CollectionUtils.isEmpty(dataList)){ System.out.println("-----------------正在处理剩余数据"); - batchUpdate(connectorMapper, pool, sql, dataList, 1000); + batchUpdate(connectorInstance, pool, sql, dataList, 1000); } pool.shutdown(); @@ -147,7 +147,7 @@ public class ConnectionTest { @Test public void testBatchUpdate() { - final DatabaseConnectorInstance connectorMapper = new DatabaseConnectorInstance(createMysqlConfig()); + final DatabaseConnectorInstance connectorInstance = new DatabaseConnectorInstance(createMysqlConfig()); long begin = Instant.now().toEpochMilli(); final int threadSize = 10; @@ -167,14 +167,14 @@ public class ConnectionTest { if (i % 10000 == 0) { System.out.println(i + "-----------------正在处理"); - batchUpdate(connectorMapper, pool, sql, dataList, 1000); + batchUpdate(connectorInstance, pool, sql, dataList, 1000); dataList.clear(); } } if (!CollectionUtils.isEmpty(dataList)) { System.out.println("-----------------正在处理剩余数据"); - batchUpdate(connectorMapper, pool, sql, dataList, 1000); + batchUpdate(connectorInstance, pool, sql, dataList, 1000); } k--; } @@ -185,7 +185,7 @@ public class ConnectionTest { @Test public void testBatchDelete() { - final DatabaseConnectorInstance connectorMapper = new DatabaseConnectorInstance(createMysqlConfig()); + final DatabaseConnectorInstance connectorInstance = new DatabaseConnectorInstance(createMysqlConfig()); long begin = Instant.now().toEpochMilli(); final int threadSize = 10; @@ -201,14 +201,14 @@ public class ConnectionTest { if (i % 10000 == 0) { System.out.println(i + "-----------------正在处理"); - batchUpdate(connectorMapper, pool, sql, dataList, 1000); + batchUpdate(connectorInstance, pool, sql, dataList, 1000); dataList.clear(); } } if (!CollectionUtils.isEmpty(dataList)) { System.out.println("-----------------正在处理剩余数据"); - batchUpdate(connectorMapper, pool, sql, dataList, 1000); + batchUpdate(connectorInstance, pool, sql, dataList, 1000); } pool.shutdown(); @@ -226,7 +226,7 @@ public class ConnectionTest { return s.toString(); } - private void batchUpdate(DatabaseConnectorInstance connectorMapper, ExecutorService pool, String sql, List dataList, int batchSize) { + private void batchUpdate(DatabaseConnectorInstance connectorInstance, ExecutorService pool, String sql, List dataList, int batchSize) { int total = dataList.size(); int taskSize = total % batchSize == 0 ? total / batchSize : total / batchSize + 1; final CountDownLatch latch = new CountDownLatch(taskSize); @@ -245,7 +245,7 @@ public class ConnectionTest { pool.submit(() -> { try { - connectorMapper.execute(databaseTemplate -> databaseTemplate.batchUpdate(sql, data)); + connectorInstance.execute(databaseTemplate -> databaseTemplate.batchUpdate(sql, data)); } catch (Exception e) { logger.error(e.getMessage()); } finally { @@ -279,8 +279,8 @@ public class ConnectionTest { public void testGetColumnsDetails() { final String schema = "root"; final String tableNamePattern = "sw_test"; - final DatabaseConnectorInstance connectorMapper = new DatabaseConnectorInstance(createMysqlConfig()); - connectorMapper.execute(databaseTemplate -> { + final DatabaseConnectorInstance connectorInstance = new DatabaseConnectorInstance(createMysqlConfig()); + connectorInstance.execute(databaseTemplate -> { SimpleConnection connection = databaseTemplate.getSimpleConnection(); Connection conn = connection.getConnection(); String databaseCatalog = conn.getCatalog(); @@ -299,9 +299,9 @@ public class ConnectionTest { } private List
    getTables(DatabaseConfig config, final String catalog, final String schema, final String tableNamePattern) { - final DatabaseConnectorInstance connectorMapper = new DatabaseConnectorInstance(config); + final DatabaseConnectorInstance connectorInstance = new DatabaseConnectorInstance(config); List
    tables = new ArrayList<>(); - connectorMapper.execute(databaseTemplate -> { + connectorInstance.execute(databaseTemplate -> { SimpleConnection connection = databaseTemplate.getSimpleConnection(); Connection conn = connection.getConnection(); String databaseCatalog = null == catalog ? conn.getCatalog() : catalog; diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/PreloadTemplate.java b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/PreloadTemplate.java index bc3f0b7d..0fd208b5 100644 --- a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/PreloadTemplate.java +++ b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/PreloadTemplate.java @@ -83,8 +83,8 @@ public final class PreloadTemplate implements ApplicationListener reload(map, commandEnum)); - // Load connectorMappers - loadConnectorMapper(); + // Load connectorInstances + loadConnectorInstance(); // Launch drivers launch(); @@ -193,7 +193,7 @@ public final class PreloadTemplate implements ApplicationListener list = profileComponent.getConnectorAll(); if (!CollectionUtils.isEmpty(list)) { list.forEach(connector -> { diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/AbstractDQLConnector.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/AbstractDQLConnector.java index 08e9e168..cf76a765 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/AbstractDQLConnector.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/AbstractDQLConnector.java @@ -26,8 +26,8 @@ import java.util.Map; public abstract class AbstractDQLConnector extends AbstractDatabaseConnector { @Override - public List
    getTable(DatabaseConnectorInstance connectorMapper) { - DatabaseConfig cfg = connectorMapper.getConfig(); + public List
    getTable(DatabaseConnectorInstance connectorInstance) { + DatabaseConfig cfg = connectorInstance.getConfig(); List sqlTables = cfg.getSqlTables(); List
    tables = new ArrayList<>(); if (!CollectionUtils.isEmpty(sqlTables)) { @@ -44,8 +44,8 @@ public abstract class AbstractDQLConnector extends AbstractDatabaseConnector { } @Override - public MetaInfo getMetaInfo(DatabaseConnectorInstance connectorMapper, String sqlName) { - DatabaseConfig cfg = connectorMapper.getConfig(); + public MetaInfo getMetaInfo(DatabaseConnectorInstance connectorInstance, String sqlName) { + DatabaseConfig cfg = connectorInstance.getConfig(); List sqlTables = cfg.getSqlTables(); for (SqlTable s : sqlTables) { if (StringUtil.equals(s.getSqlName(), sqlName)) { @@ -54,7 +54,7 @@ public abstract class AbstractDQLConnector extends AbstractDatabaseConnector { sql = sql.replace("\r", " "); sql = sql.replace("\n", " "); String queryMetaSql = StringUtil.contains(sql, " WHERE ") ? s.getSql() + " AND 1!=1 " : s.getSql() + " WHERE 1!=1 "; - return connectorMapper.execute(databaseTemplate -> super.getMetaInfo(databaseTemplate, queryMetaSql, getSchema(cfg), s.getTable())); + return connectorInstance.execute(databaseTemplate -> super.getMetaInfo(databaseTemplate, queryMetaSql, getSchema(cfg), s.getTable())); } } return null; diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/AbstractDatabaseConnector.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/AbstractDatabaseConnector.java index 464b36f1..d556eba2 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/AbstractDatabaseConnector.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/AbstractDatabaseConnector.java @@ -78,13 +78,13 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector implem } @Override - public void disconnect(DatabaseConnectorInstance connectorMapper) { - connectorMapper.close(); + public void disconnect(DatabaseConnectorInstance connectorInstance) { + connectorInstance.close(); } @Override - public boolean isAlive(DatabaseConnectorInstance connectorMapper) { - Integer count = connectorMapper.execute(databaseTemplate -> databaseTemplate.queryForObject(getValidationQuery(), Integer.class)); + public boolean isAlive(DatabaseConnectorInstance connectorInstance) { + Integer count = connectorInstance.execute(databaseTemplate -> databaseTemplate.queryForObject(getValidationQuery(), Integer.class)); return null != count && count > 0; } @@ -94,15 +94,15 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector implem } @Override - public List
    getTable(DatabaseConnectorInstance connectorMapper) { - return getTable(connectorMapper, null, getSchema(connectorMapper.getConfig()), null); + public List
    getTable(DatabaseConnectorInstance connectorInstance) { + return getTable(connectorInstance, null, getSchema(connectorInstance.getConfig()), null); } @Override - public MetaInfo getMetaInfo(DatabaseConnectorInstance connectorMapper, String tableNamePattern) { + public MetaInfo getMetaInfo(DatabaseConnectorInstance connectorInstance, String tableNamePattern) { List fields = new ArrayList<>(); - final String schema = getSchema(connectorMapper.getConfig()); - connectorMapper.execute(databaseTemplate -> { + final String schema = getSchema(connectorInstance.getConfig()); + connectorInstance.execute(databaseTemplate -> { SimpleConnection connection = databaseTemplate.getSimpleConnection(); Connection conn = connection.getConnection(); String catalog = conn.getCatalog(); @@ -122,7 +122,7 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector implem } @Override - public long getCount(DatabaseConnectorInstance connectorMapper, Map command) { + public long getCount(DatabaseConnectorInstance connectorInstance, Map command) { if (CollectionUtils.isEmpty(command)) { return 0L; } @@ -134,14 +134,14 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector implem } // 2、返回结果集 - return connectorMapper.execute(databaseTemplate -> { + return connectorInstance.execute(databaseTemplate -> { Long count = databaseTemplate.queryForObject(queryCountSql, Long.class); return count == null ? 0 : count; }); } @Override - public Result reader(DatabaseConnectorInstance connectorMapper, ReaderConfig config) { + public Result reader(DatabaseConnectorInstance connectorInstance, ReaderConfig config) { // 1、获取select SQL boolean supportedCursor = enableCursor() && config.isSupportedCursor() && null != config.getCursors(); String queryKey = supportedCursor ? ConnectorConstant.OPERTION_QUERY_CURSOR : ConnectorConstant.OPERTION_QUERY; @@ -152,14 +152,14 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector implem Collections.addAll(config.getArgs(), supportedCursor ? getPageCursorArgs(config) : getPageArgs(config)); // 3、执行SQL - List> list = connectorMapper.execute(databaseTemplate -> databaseTemplate.queryForList(querySql, config.getArgs().toArray())); + List> list = connectorInstance.execute(databaseTemplate -> databaseTemplate.queryForList(querySql, config.getArgs().toArray())); // 4、返回结果集 return new Result(list); } @Override - public Result writer(DatabaseConnectorInstance connectorMapper, WriterBatchConfig config) { + public Result writer(DatabaseConnectorInstance connectorInstance, WriterBatchConfig config) { String event = config.getEvent(); List data = config.getData(); @@ -190,9 +190,9 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector implem int[] execute = null; try { // 2、设置参数 - execute = connectorMapper.execute(databaseTemplate -> databaseTemplate.batchUpdate(executeSql, batchRows(fields, data))); + execute = connectorInstance.execute(databaseTemplate -> databaseTemplate.batchUpdate(executeSql, batchRows(fields, data))); } catch (Exception e) { - data.forEach(row -> forceUpdate(result, connectorMapper, config, pkFields, row)); + data.forEach(row -> forceUpdate(result, connectorInstance, config, pkFields, row)); } if (null != execute) { @@ -202,7 +202,7 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector implem result.getSuccessData().add(data.get(i)); continue; } - forceUpdate(result, connectorMapper, config, pkFields, data.get(i)); + forceUpdate(result, connectorInstance, config, pkFields, data.get(i)); } } return result; @@ -440,14 +440,14 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector implem /** * 获取表列表 * - * @param connectorMapper + * @param connectorInstance * @param catalog * @param schema * @param tableNamePattern * @return */ - private List
    getTable(DatabaseConnectorInstance connectorMapper, String catalog, String schema, String tableNamePattern) { - return connectorMapper.execute(databaseTemplate -> { + private List
    getTable(DatabaseConnectorInstance connectorInstance, String catalog, String schema, String tableNamePattern) { + return connectorInstance.execute(databaseTemplate -> { List
    tables = new ArrayList<>(); SimpleConnection connection = databaseTemplate.getSimpleConnection(); Connection conn = connection.getConnection(); @@ -569,7 +569,7 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector implem return args; } - private void forceUpdate(Result result, DatabaseConnectorInstance connectorMapper, WriterBatchConfig config, List pkFields, + private void forceUpdate(Result result, DatabaseConnectorInstance connectorInstance, WriterBatchConfig config, List pkFields, Map row) { if (isUpdate(config.getEvent()) || isInsert(config.getEvent())) { // 存在执行覆盖更新，否则写入 @@ -579,14 +579,14 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector implem for (int i = 0; i < size; i++) { args[i] = row.get(pkFields.get(i).getName()); } - final String event = existRow(connectorMapper, queryCount, args) ? ConnectorConstant.OPERTION_UPDATE + final String event = existRow(connectorInstance, queryCount, args) ? ConnectorConstant.OPERTION_UPDATE : ConnectorConstant.OPERTION_INSERT; logger.warn("{}表执行{}失败, 重新执行{}, {}", config.getTableName(), config.getEvent(), event, row); - writer(result, connectorMapper, config, pkFields, row, event); + writer(result, connectorInstance, config, pkFields, row, event); } } - private void writer(Result result, DatabaseConnectorInstance connectorMapper, WriterBatchConfig config, List pkFields, Map row, + private void writer(Result result, DatabaseConnectorInstance connectorInstance, WriterBatchConfig config, List pkFields, Map row, String event) { // 1、获取 SQL String sql = config.getCommand().get(event); @@ -604,7 +604,7 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector implem try { // 2、设置参数 - int execute = connectorMapper.execute(databaseTemplate -> databaseTemplate.update(sql, batchRow(fields, row))); + int execute = connectorInstance.execute(databaseTemplate -> databaseTemplate.update(sql, batchRow(fields, row))); if (execute == 0) { throw new SdkException(String.format("尝试执行[%s]失败", event)); } @@ -618,10 +618,10 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector implem } } - private boolean existRow(DatabaseConnectorInstance connectorMapper, String sql, Object[] args) { + private boolean existRow(DatabaseConnectorInstance connectorInstance, String sql, Object[] args) { int rowNum = 0; try { - rowNum = connectorMapper.execute(databaseTemplate -> databaseTemplate.queryForObject(sql, Integer.class, args)); + rowNum = connectorInstance.execute(databaseTemplate -> databaseTemplate.queryForObject(sql, Integer.class, args)); } catch (Exception e) { logger.error("检查数据行存在异常:{}，SQL:{},参数:{}", e.getMessage(), sql, args); } @@ -651,11 +651,11 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector implem } @Override - public Result writerDDL(DatabaseConnectorInstance connectorMapper, DDLConfig config) { + public Result writerDDL(DatabaseConnectorInstance connectorInstance, DDLConfig config) { Result result = new Result(); try { Assert.hasText(config.getSql(), "执行SQL语句不能为空."); - connectorMapper.execute(databaseTemplate -> { + connectorInstance.execute(databaseTemplate -> { databaseTemplate.execute(config.getSql()); return true; }); diff --git a/dbsyncer-web/src/main/resources/public/plugin/plugin.html b/dbsyncer-web/src/main/resources/public/plugin/plugin.html index 5df029fe..d9ed91fd 100644 --- a/dbsyncer-web/src/main/resources/public/plugin/plugin.html +++ b/dbsyncer-web/src/main/resources/public/plugin/plugin.html @@ -77,11 +77,11 @@ public class MyPlugin implements ConvertService { // 完成同步后调用该方法 logger.info("插件正在处理同步成功的数据，目标源表:{}，事件:{}，条数:{}", context.getTargetTableName(), context.getEvent(), context.getTargetList().size()); - ConnectorMapper connectorMapper = context.getSourceConnectorMapper(); + ConnectorInstance connectorInstance = context.getSourceConnectorInstance(); // 获取关系型数据库连接，实现自己的业务逻辑... - if (connectorMapper instanceof DatabaseConnectorMapper) { - DatabaseConnectorMapper db = (DatabaseConnectorMapper) connectorMapper; + if (connectorInstance instanceof DatabaseConnectorInstance) { + DatabaseConnectorInstance db = (DatabaseConnectorInstance) connectorInstance; // 方式一（推荐）： String query = "select * from my_user"; db.execute(databaseTemplate -> databaseTemplate.queryForList(query)); -- Gitee From 5bc3b53cc852b4f11cf0bd383f44022e4f65ca4b Mon Sep 17 00:00:00 2001 From: AE86 <836391306@qq.com> Date: Sun, 26 Nov 2023 02:11:59 +0800 Subject: [PATCH 28/98] add module dbsyncer-connector-elasticsearch --- ...a => AbstractDataBaseConfigValidator.java} | 15 +- .../impl/connector/ConnectorChecker.java | 17 +- .../impl/connector/DqlMySQLConfigChecker.java | 11 +- .../connector/DqlOracleConfigChecker.java | 11 +- .../connector/DqlPostgreSQLConfigChecker.java | 11 +- ....java => DqlSqlServerConfigValidator.java} | 47 ++--- ...gChecker.java => FileConfigValidator.java} | 15 +- ...Checker.java => KafkaConfigValidator.java} | 15 +- .../impl/connector/MySQLConfigChecker.java | 11 +- .../impl/connector/OracleConfigChecker.java | 176 ------------------ .../impl/connector/OracleConfigValidator.java | 28 +++ .../connector/PostgreSQLConfigChecker.java | 11 +- .../connector/SqlServerConfigChecker.java | 11 +- .../impl/tablegroup/TableGroupChecker.java | 34 ++-- .../biz/impl/ConditionServiceImpl.java | 13 +- .../dbsyncer/biz/impl/MonitorServiceImpl.java | 11 +- .../java/org/dbsyncer/biz/vo/ConditionVo.java | 11 +- .../dbsyncer-connector-elasticsearch/pom.xml | 42 +++++ .../elasticsearch}/ESConnectorInstance.java | 21 ++- .../ElasticsearchConnector.java} | 54 ++++-- .../api}/EasyRestHighLevelClient.java | 14 +- .../elasticsearch/api}/RequestConverters.java | 12 +- .../elasticsearch/api}/RethrottleRequest.java | 9 +- .../elasticsearch/cdc}/ESQuartzListener.java | 17 +- .../elasticsearch}/config/ESConfig.java | 132 ++++++------- .../elasticsearch}/enums/ESFieldTypeEnum.java | 11 +- .../schema}/ESDateValueMapper.java | 65 ++++--- .../schema}/ESOtherValueMapper.java | 73 ++++---- .../connector/elasticsearch}/util/ESUtil.java | 132 +++++++------ .../validator/ESConfigValidator.java | 21 ++- .../org.dbsyncer.sdk.spi.ConnectorService | 1 + .../src/main/test/ESClientTest.java | 15 +- dbsyncer-connector/pom.xml | 11 ++ .../connector/ConnectorConfiguration.java | 48 ++++- .../dbsyncer/connector/ConnectorFactory.java | 11 +- .../dbsyncer/connector/enums/FilterEnum.java | 114 ------------ .../connector/file/FileConnector.java | 17 +- .../connector/kafka/KafkaConnector.java | 20 +- .../main/resources/META-INF/spring.factories | 3 - .../org/dbsyncer/parser/ProfileComponent.java | 2 +- .../parser/impl/ProfileComponentImpl.java | 2 +- .../dbsyncer/parser/model/FieldPicker.java | 4 +- .../sdk/connector/ConfigValidator.java | 27 +-- .../database/AbstractDatabaseConnector.java | 16 ++ .../org/dbsyncer/sdk/enums/FilterEnum.java | 17 +- .../dbsyncer/sdk/filter}/CompareFilter.java | 12 +- .../dbsyncer/sdk/spi/ConnectorService.java | 13 +- .../storage/impl/DiskStorageServiceImpl.java | 11 +- .../storage/impl/MySQLStorageServiceImpl.java | 11 +- .../storage/query/AbstractFilter.java | 13 +- .../org/dbsyncer/storage/query/Query.java | 11 +- .../storage/query/filter/IntFilter.java | 5 +- .../storage/query/filter/LongFilter.java | 5 +- .../storage/query/filter/StringFilter.java | 5 +- pom.xml | 1 - 55 files changed, 748 insertions(+), 698 deletions(-) rename dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/{AbstractDataBaseConfigChecker.java => AbstractDataBaseConfigValidator.java} (85%) rename dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/{DqlSqlServerConfigChecker.java => DqlSqlServerConfigValidator.java} (63%) rename dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/{FileConfigChecker.java => FileConfigValidator.java} (83%) rename dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/{KafkaConfigChecker.java => KafkaConfigValidator.java} (89%) delete mode 100644 dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/OracleConfigChecker.java create mode 100644 dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/OracleConfigValidator.java create mode 100644 dbsyncer-connector/dbsyncer-connector-elasticsearch/pom.xml rename dbsyncer-connector/{src/main/java/org/dbsyncer/connector/es => dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch}/ESConnectorInstance.java (70%) rename dbsyncer-connector/{src/main/java/org/dbsyncer/connector/es/ESConnector.java => dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/ElasticsearchConnector.java} (90%) rename dbsyncer-connector/{src/main/java/org/dbsyncer/connector/es => dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/api}/EasyRestHighLevelClient.java (93%) rename dbsyncer-connector/{src/main/java/org/dbsyncer/connector/es => dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/api}/RequestConverters.java (99%) rename dbsyncer-connector/{src/main/java/org/dbsyncer/connector/es => dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/api}/RethrottleRequest.java (91%) rename dbsyncer-connector/{src/main/java/org/dbsyncer/connector/es => dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/cdc}/ESQuartzListener.java (90%) rename dbsyncer-connector/{src/main/java/org/dbsyncer/connector => dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch}/config/ESConfig.java (82%) rename dbsyncer-connector/{src/main/java/org/dbsyncer/connector => dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch}/enums/ESFieldTypeEnum.java (90%) rename dbsyncer-connector/{src/main/java/org/dbsyncer/connector/es => dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/schema}/ESDateValueMapper.java (62%) rename dbsyncer-connector/{src/main/java/org/dbsyncer/connector/es => dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/schema}/ESOtherValueMapper.java (66%) rename dbsyncer-connector/{src/main/java/org/dbsyncer/connector => dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch}/util/ESUtil.java (79%) rename dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/ElasticsearchConfigChecker.java => dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/validator/ESConfigValidator.java (64%) create mode 100644 dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/resources/META-INF/services/org.dbsyncer.sdk.spi.ConnectorService rename dbsyncer-connector/{ => dbsyncer-connector-elasticsearch}/src/main/test/ESClientTest.java (97%) delete mode 100644 dbsyncer-connector/src/main/java/org/dbsyncer/connector/enums/FilterEnum.java delete mode 100644 dbsyncer-connector/src/main/resources/META-INF/spring.factories rename dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/ConnectorConfigChecker.java => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/ConfigValidator.java (31%) rename {dbsyncer-connector/src/main/java/org/dbsyncer/connector => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/filter}/CompareFilter.java (32%) diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/AbstractDataBaseConfigChecker.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/AbstractDataBaseConfigValidator.java similarity index 85% rename from dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/AbstractDataBaseConfigChecker.java rename to dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/AbstractDataBaseConfigValidator.java index 156b000f..fc913071 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/AbstractDataBaseConfigChecker.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/AbstractDataBaseConfigValidator.java @@ -1,9 +1,12 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.biz.checker.impl.connector; -import org.dbsyncer.biz.checker.ConnectorConfigChecker; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.JsonUtil; import org.dbsyncer.sdk.config.DatabaseConfig; +import org.dbsyncer.sdk.connector.ConfigValidator; import org.dbsyncer.sdk.model.SqlTable; import org.springframework.util.Assert; @@ -11,11 +14,13 @@ import java.util.List; import java.util.Map; /** - * @author AE86 - * @version 1.0.0 - * @date 2020/1/8 15:17 + * 关系型数据库连接配置校验器 + * + * @Author AE86 + * @Version 1.0.0 + * @Date 2020-01-08 15:17 */ -public abstract class AbstractDataBaseConfigChecker implements ConnectorConfigChecker { +public abstract class AbstractDataBaseConfigValidator implements ConfigValidator { @Override public void modify(DatabaseConfig connectorConfig, Map params) { diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/ConnectorChecker.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/ConnectorChecker.java index 4aaddd5f..6ce1a827 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/ConnectorChecker.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/ConnectorChecker.java @@ -1,8 +1,11 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.biz.checker.impl.connector; import org.dbsyncer.biz.BizException; import org.dbsyncer.biz.checker.AbstractChecker; -import org.dbsyncer.biz.checker.ConnectorConfigChecker; +import org.dbsyncer.sdk.connector.ConfigValidator; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.connector.ConnectorFactory; import org.dbsyncer.parser.ProfileComponent; @@ -21,9 +24,9 @@ import javax.annotation.Resource; import java.util.Map; /** - * @author AE86 - * @version 1.0.0 - * @date 2020/1/8 15:17 + * @Author AE86 + * @Version 1.0.0 + * @Date 2020-01-08 15:17 */ @Component public class ConnectorChecker extends AbstractChecker { @@ -37,7 +40,7 @@ public class ConnectorChecker extends AbstractChecker { private ConnectorFactory connectorFactory; @Resource - private Map map; + private Map map; @Override public ConfigModel checkAddConfigModel(Map params) { @@ -54,7 +57,7 @@ public class ConnectorChecker extends AbstractChecker { // 配置连接器配置 String type = StringUtil.toLowerCaseFirstOne(connectorType).concat("ConfigChecker"); - ConnectorConfigChecker checker = map.get(type); + ConfigValidator checker = map.get(type); Assert.notNull(checker, "Checker can not be null."); checker.modify(config, params); @@ -83,7 +86,7 @@ public class ConnectorChecker extends AbstractChecker { // 配置连接器配置 String type = StringUtil.toLowerCaseFirstOne(config.getConnectorType()).concat("ConfigChecker"); - ConnectorConfigChecker checker = map.get(type); + ConfigValidator checker = map.get(type); Assert.notNull(checker, "Checker can not be null."); checker.modify(config, params); diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlMySQLConfigChecker.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlMySQLConfigChecker.java index 64ce4ada..255f1b51 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlMySQLConfigChecker.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlMySQLConfigChecker.java @@ -1,3 +1,6 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.biz.checker.impl.connector; import org.dbsyncer.sdk.config.DatabaseConfig; @@ -6,12 +9,12 @@ import org.springframework.stereotype.Component; import java.util.Map; /** - * @author AE86 - * @version 1.0.0 - * @date 2020/1/8 15:17 + * @Author AE86 + * @Version 1.0.0 + * @Date 2020-01-07 15:17 */ @Component -public class DqlMySQLConfigChecker extends AbstractDataBaseConfigChecker { +public class DqlMySQLConfigChecker extends AbstractDataBaseConfigValidator { @Override public void modify(DatabaseConfig connectorConfig, Map params) { diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlOracleConfigChecker.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlOracleConfigChecker.java index 14e6d45f..38c2756d 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlOracleConfigChecker.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlOracleConfigChecker.java @@ -1,3 +1,6 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.biz.checker.impl.connector; import org.dbsyncer.common.util.StringUtil; @@ -7,12 +10,12 @@ import org.springframework.stereotype.Component; import java.util.Map; /** - * @author AE86 - * @version 1.0.0 - * @date 2020/1/8 15:17 + * @Author AE86 + * @Version 1.0.0 + * @Date 2020-01-08 15:17 */ @Component -public class DqlOracleConfigChecker extends AbstractDataBaseConfigChecker { +public class DqlOracleConfigChecker extends AbstractDataBaseConfigValidator { @Override public void modify(DatabaseConfig connectorConfig, Map params) { diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlPostgreSQLConfigChecker.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlPostgreSQLConfigChecker.java index 1e6a00b6..cb6defbe 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlPostgreSQLConfigChecker.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlPostgreSQLConfigChecker.java @@ -1,3 +1,6 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.biz.checker.impl.connector; import org.dbsyncer.common.util.StringUtil; @@ -7,12 +10,12 @@ import org.springframework.stereotype.Component; import java.util.Map; /** - * @author AE86 - * @version 1.0.0 - * @date 2022/4/5 22:14 + * @Author AE86 + * @Version 1.0.0 + * @Date 2022-04-05 22:14 */ @Component -public class DqlPostgreSQLConfigChecker extends AbstractDataBaseConfigChecker { +public class DqlPostgreSQLConfigChecker extends AbstractDataBaseConfigValidator { @Override public void modify(DatabaseConfig connectorConfig, Map params) { diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlSqlServerConfigChecker.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlSqlServerConfigValidator.java similarity index 63% rename from dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlSqlServerConfigChecker.java rename to dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlSqlServerConfigValidator.java index 3b3f8de0..76523ff8 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlSqlServerConfigChecker.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlSqlServerConfigValidator.java @@ -1,22 +1,27 @@ -package org.dbsyncer.biz.checker.impl.connector; - -import org.dbsyncer.sdk.config.DatabaseConfig; -import org.springframework.stereotype.Component; - -import java.util.Map; - -/** - * @author AE86 - * @version 1.0.0 - * @date 2021/5/18 20:17 - */ -@Component -public class DqlSqlServerConfigChecker extends AbstractDataBaseConfigChecker { - - @Override - public void modify(DatabaseConfig connectorConfig, Map params) { - super.modify(connectorConfig, params); - super.modifyDql(connectorConfig, params); - super.modifySchema(connectorConfig, params); - } +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.biz.checker.impl.connector; + +import org.dbsyncer.sdk.config.DatabaseConfig; +import org.springframework.stereotype.Component; + +import java.util.Map; + +/** + * DqlSqlServer连接配置校验器实现 + * + * @Author AE86 + * @Version 1.0.0 + * @Date 2022-05-18 20:17 + */ +@Component +public class DqlSqlServerConfigValidator extends AbstractDataBaseConfigValidator { + + @Override + public void modify(DatabaseConfig connectorConfig, Map params) { + super.modify(connectorConfig, params); + super.modifyDql(connectorConfig, params); + super.modifySchema(connectorConfig, params); + } } \ No newline at end of file diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/FileConfigChecker.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/FileConfigValidator.java similarity index 83% rename from dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/FileConfigChecker.java rename to dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/FileConfigValidator.java index e0949d41..9b9f5fea 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/FileConfigChecker.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/FileConfigValidator.java @@ -1,6 +1,9 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.biz.checker.impl.connector; -import org.dbsyncer.biz.checker.ConnectorConfigChecker; +import org.dbsyncer.sdk.connector.ConfigValidator; import org.dbsyncer.common.util.JsonUtil; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.connector.config.FileConfig; @@ -13,12 +16,14 @@ import java.util.List; import java.util.Map; /** - * @author AE86 - * @version 1.0.0 - * @date 2022/5/6 0:04 + * 文件连接配置校验器实现 + * + * @Author AE86 + * @Version 1.0.0 + * @Date 2022-05-06 00:04 */ @Component -public class FileConfigChecker implements ConnectorConfigChecker { +public class FileConfigValidator implements ConfigValidator { @Override public void modify(FileConfig fileConfig, Map params) { diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/KafkaConfigChecker.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/KafkaConfigValidator.java similarity index 89% rename from dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/KafkaConfigChecker.java rename to dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/KafkaConfigValidator.java index 3b574921..6496c9a1 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/KafkaConfigChecker.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/KafkaConfigValidator.java @@ -1,6 +1,9 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.biz.checker.impl.connector; -import org.dbsyncer.biz.checker.ConnectorConfigChecker; +import org.dbsyncer.sdk.connector.ConfigValidator; import org.dbsyncer.common.util.NumberUtil; import org.dbsyncer.connector.config.KafkaConfig; import org.springframework.stereotype.Component; @@ -9,12 +12,14 @@ import org.springframework.util.Assert; import java.util.Map; /** - * @author AE86 - * @version 1.0.0 - * @date 2021/11/22 23:55 + * Kafka连接配置校验器实现 + * + * @Author AE86 + * @Version 1.0.0 + * @Date 2021-11-22 23:55 */ @Component -public class KafkaConfigChecker implements ConnectorConfigChecker { +public class KafkaConfigValidator implements ConfigValidator { @Override public void modify(KafkaConfig connectorConfig, Map params) { diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/MySQLConfigChecker.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/MySQLConfigChecker.java index 6036ad57..fd283ebc 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/MySQLConfigChecker.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/MySQLConfigChecker.java @@ -1,13 +1,16 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.biz.checker.impl.connector; import org.springframework.stereotype.Component; /** - * @author AE86 - * @version 1.0.0 - * @date 2020/1/8 15:17 + * @Author AE86 + * @Version 1.0.0 + * @Date 2020-01-08 15:17 */ @Component -public class MySQLConfigChecker extends AbstractDataBaseConfigChecker { +public class MySQLConfigChecker extends AbstractDataBaseConfigValidator { } \ No newline at end of file diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/OracleConfigChecker.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/OracleConfigChecker.java deleted file mode 100644 index 5f9c4081..00000000 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/OracleConfigChecker.java +++ /dev/null @@ -1,176 +0,0 @@ -package org.dbsyncer.biz.checker.impl.connector; - -import org.dbsyncer.biz.enums.OracleIncrementEnum; -import org.dbsyncer.common.util.CollectionUtils; -import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.sdk.config.DatabaseConfig; -import org.dbsyncer.parser.ParserComponent; -import org.dbsyncer.parser.ProfileComponent; -import org.dbsyncer.parser.model.FieldMapping; -import org.dbsyncer.parser.model.Mapping; -import org.dbsyncer.parser.model.TableGroup; -import org.dbsyncer.sdk.model.Field; -import org.dbsyncer.sdk.model.MetaInfo; -import org.dbsyncer.sdk.model.Table; -import org.springframework.stereotype.Component; -import org.springframework.util.Assert; - -import javax.annotation.Resource; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -/** - *

    1、增量同步时，目标源必须有一个主键字段用于接收ROW_ID值。

    - *

    2、全局可配置目标源ROW_ID字段名称，默认为ROW_ID_LABEL_NAME。

    - *

    3、如果配置了接收字段，添加字段映射关系[ROW_ID_LABEL_NAME] > [ROW_ID_LABEL_NAME]，并将ROW_ID_LABEL_NAME字段设置为目标源的唯一主键。

    - *

    4、全量同步时，ROW_ID_LABEL_NAME参数非必须。

    - * - * @author AE86 - * @version 1.0.0 - * @date 2020/1/8 15:17 - */ -@Component -public class OracleConfigChecker extends AbstractDataBaseConfigChecker { - - @Resource - private ParserComponent parserComponent; - - @Resource - private ProfileComponent profileComponent; - - @Override - public void modify(DatabaseConfig connectorConfig, Map params) { - super.modify(connectorConfig, params); - - String schema = params.get("schema"); - connectorConfig.setSchema(StringUtil.isBlank(schema) ? connectorConfig.getUsername().toUpperCase() : schema.toUpperCase()); - } - - @Override - public void dealIncrementStrategy(Mapping mapping, TableGroup tableGroup) { - String rowIdLabelName = OracleIncrementEnum.ROW_ID_LABEL_NAME.getName(); - Map params = tableGroup.getParams(); - if (CollectionUtils.isEmpty(params) || !params.containsKey(rowIdLabelName)) { - revert(mapping, tableGroup); - return; - } - String targetRowIdName = params.get(rowIdLabelName); - - // 检查目标源是否支持该自定义字段 - List targetColumn = tableGroup.getTargetTable().getColumn(); - Field targetField = null; - for (Field f : targetColumn) { - if (StringUtil.equals(f.getName(), targetRowIdName)) { - targetField = f; - targetField.setPk(true); - continue; - } - f.setPk(false); - } - Assert.isTrue(null != targetField, - String.format("[%s 同步到 %s]，目标源表不存在字段%s", tableGroup.getSourceTable().getName(), tableGroup.getTargetTable().getName(), - targetRowIdName)); - - // 检查是否更新 - List fieldMapping = tableGroup.getFieldMapping(); - List list = new ArrayList<>(); - for (FieldMapping m : fieldMapping) { - if (null != m.getSource() && OracleIncrementEnum.isRowId(m.getSource().getName())) { - m.getTarget().setName(targetRowIdName); - return; - } - - // 排除主键关系 - if (null != m.getTarget() && StringUtil.equals(m.getTarget().getName(), targetRowIdName)) { - continue; - } - - list.add(m); - } - - Field sourceField = new Field(OracleIncrementEnum.ROW_ID.getName(), "VARCHAR2", 12, false, - OracleIncrementEnum.ROW_ID_LABEL_NAME.getName(), true); - tableGroup.getSourceTable().getColumn().add(0, sourceField); - - list.add(0, new FieldMapping(sourceField, targetField)); - fieldMapping.clear(); - fieldMapping.addAll(list); - } - - /** - * 还原字段和映射关系 - * - * @param mapping - * @param tableGroup - */ - private void revert(Mapping mapping, TableGroup tableGroup) { - List fieldMapping = tableGroup.getFieldMapping(); - if (CollectionUtils.isEmpty(fieldMapping)) { - return; - } - - // 还原字段 - Table sourceTable = tableGroup.getSourceTable(); - List sourceColumn = sourceTable.getColumn(); - List sourceFields = new ArrayList<>(); - boolean existRowId = false; - for (Field f : sourceColumn) { - if (OracleIncrementEnum.isRowId(f.getName())) { - existRowId = true; - continue; - } - sourceFields.add(f); - } - sourceTable.setColumn(sourceFields); - - if (!existRowId) { - return; - } - - // 存在自定义主键 - String pk = null; - for (FieldMapping m : tableGroup.getFieldMapping()) { - if (null != m.getSource() && OracleIncrementEnum.isRowId(m.getSource().getName())) { - continue; - } - if (null != m.getTarget() && m.getTarget().isPk()) { - pk = m.getTarget().getName(); - break; - } - } - - // 没有自定义主键，获取表元信息 - if (null == pk) { - Table targetTable = tableGroup.getTargetTable(); - MetaInfo metaInfo = parserComponent.getMetaInfo(mapping.getTargetConnectorId(), targetTable.getName()); - List targetColumn = metaInfo.getColumn(); - targetTable.setColumn(targetColumn); - - if (!CollectionUtils.isEmpty(targetColumn)) { - for (Field f : targetColumn) { - if (f.isPk()) { - pk = f.getName(); - break; - } - } - } - } - - // 剔除映射关系 - List list = new ArrayList<>(); - for (FieldMapping m : tableGroup.getFieldMapping()) { - if (null != m.getSource() && OracleIncrementEnum.isRowId(m.getSource().getName())) { - continue; - } - - if (null != m.getTarget() && StringUtil.equals(m.getTarget().getName(), pk)) { - m.getTarget().setPk(true); - } - list.add(m); - } - fieldMapping.clear(); - fieldMapping.addAll(list); - } - -} \ No newline at end of file diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/OracleConfigValidator.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/OracleConfigValidator.java new file mode 100644 index 00000000..af71854a --- /dev/null +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/OracleConfigValidator.java @@ -0,0 +1,28 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.biz.checker.impl.connector; + +import org.dbsyncer.common.util.StringUtil; +import org.dbsyncer.sdk.config.DatabaseConfig; + +import java.util.Map; + +/** + * Oracle连接配置校验器实现 + * + * @Author AE86 + * @Version 1.0.0 + * @Date 2020-01-08 15:17 + */ +public class OracleConfigValidator extends AbstractDataBaseConfigValidator { + + @Override + public void modify(DatabaseConfig connectorConfig, Map params) { + super.modify(connectorConfig, params); + + String schema = params.get("schema"); + connectorConfig.setSchema(StringUtil.isBlank(schema) ? connectorConfig.getUsername().toUpperCase() : schema.toUpperCase()); + } + +} \ No newline at end of file diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/PostgreSQLConfigChecker.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/PostgreSQLConfigChecker.java index 6f049bc5..2acba096 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/PostgreSQLConfigChecker.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/PostgreSQLConfigChecker.java @@ -1,3 +1,6 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.biz.checker.impl.connector; import org.dbsyncer.common.util.StringUtil; @@ -7,12 +10,12 @@ import org.springframework.stereotype.Component; import java.util.Map; /** - * @author AE86 - * @version 1.0.0 - * @date 2022/4/5 22:14 + * @Author AE86 + * @Version 1.0.0 + * @Date 2022-04-05 22:14 */ @Component -public class PostgreSQLConfigChecker extends AbstractDataBaseConfigChecker { +public class PostgreSQLConfigChecker extends AbstractDataBaseConfigValidator { @Override public void modify(DatabaseConfig connectorConfig, Map params) { super.modify(connectorConfig, params); diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/SqlServerConfigChecker.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/SqlServerConfigChecker.java index b6e7db1a..a73fff89 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/SqlServerConfigChecker.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/SqlServerConfigChecker.java @@ -1,3 +1,6 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.biz.checker.impl.connector; import org.dbsyncer.sdk.config.DatabaseConfig; @@ -6,12 +9,12 @@ import org.springframework.stereotype.Component; import java.util.Map; /** - * @author AE86 - * @version 1.0.0 - * @date 2021/5/18 20:17 + * @Author AE86 + * @Version 1.0.0 + * @Date 2023-11-25 23:10 */ @Component -public class SqlServerConfigChecker extends AbstractDataBaseConfigChecker { +public class SqlServerConfigChecker extends AbstractDataBaseConfigValidator { @Override public void modify(DatabaseConfig connectorConfig, Map params) { super.modify(connectorConfig, params); diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/tablegroup/TableGroupChecker.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/tablegroup/TableGroupChecker.java index d1c62e97..ba921085 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/tablegroup/TableGroupChecker.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/tablegroup/TableGroupChecker.java @@ -1,23 +1,26 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.biz.checker.impl.tablegroup; import org.dbsyncer.biz.BizException; import org.dbsyncer.biz.checker.AbstractChecker; -import org.dbsyncer.biz.checker.ConnectorConfigChecker; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.JsonUtil; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.sdk.model.Field; -import org.dbsyncer.sdk.model.MetaInfo; -import org.dbsyncer.sdk.model.Table; -import org.dbsyncer.sdk.util.PrimaryKeyUtil; import org.dbsyncer.parser.ParserComponent; import org.dbsyncer.parser.ProfileComponent; -import org.dbsyncer.sdk.enums.ModelEnum; import org.dbsyncer.parser.model.ConfigModel; import org.dbsyncer.parser.model.FieldMapping; import org.dbsyncer.parser.model.Mapping; import org.dbsyncer.parser.model.TableGroup; import org.dbsyncer.parser.util.PickerUtil; +import org.dbsyncer.sdk.connector.ConfigValidator; +import org.dbsyncer.sdk.enums.ModelEnum; +import org.dbsyncer.sdk.model.Field; +import org.dbsyncer.sdk.model.MetaInfo; +import org.dbsyncer.sdk.model.Table; +import org.dbsyncer.sdk.util.PrimaryKeyUtil; import org.dbsyncer.storage.constant.ConfigConstant; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,9 +39,9 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; /** - * @author AE86 - * @version 1.0.0 - * @date 2020/1/8 15:17 + * @Author AE86 + * @Version 1.0.0 + * @Date 2020-01-08 15:17 */ @Component public class TableGroupChecker extends AbstractChecker { @@ -52,7 +55,7 @@ public class TableGroupChecker extends AbstractChecker { private ProfileComponent profileComponent; @Resource - private Map map; + private Map map; @Override public ConfigModel checkAddConfigModel(Map params) { @@ -139,9 +142,6 @@ public class TableGroupChecker extends AbstractChecker { // 合并高级配置 TableGroup group = PickerUtil.mergeTableGroupConfig(mapping, tableGroup); - // 处理策略 - dealIncrementStrategy(mapping, group); - Map command = parserComponent.getCommand(mapping, group); tableGroup.setCommand(command); @@ -150,14 +150,6 @@ public class TableGroupChecker extends AbstractChecker { tableGroup.getSourceTable().setCount(count); } - public void dealIncrementStrategy(Mapping mapping, TableGroup tableGroup) { - String connectorType = profileComponent.getConnector(mapping.getSourceConnectorId()).getConfig().getConnectorType(); - String type = StringUtil.toLowerCaseFirstOne(connectorType).concat("ConfigChecker"); - ConnectorConfigChecker checker = map.get(type); - Assert.notNull(checker, "Checker can not be null."); - checker.dealIncrementStrategy(mapping, tableGroup); - } - private Table getTable(String connectorId, String tableName, String primaryKeyStr) { MetaInfo metaInfo = parserComponent.getMetaInfo(connectorId, tableName); Assert.notNull(metaInfo, "无法获取连接器表信息:" + tableName); diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ConditionServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ConditionServiceImpl.java index e06b17f4..235792c1 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ConditionServiceImpl.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ConditionServiceImpl.java @@ -1,8 +1,11 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.biz.impl; import org.dbsyncer.biz.ConditionService; import org.dbsyncer.biz.vo.ConditionVo; -import org.dbsyncer.connector.enums.FilterEnum; +import org.dbsyncer.sdk.enums.FilterEnum; import org.dbsyncer.sdk.enums.QuartzFilterEnum; import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.sdk.enums.OperationEnum; @@ -12,9 +15,11 @@ import javax.annotation.Resource; import java.util.List; /** - * @author AE86 - * @version 1.0.0 - * @date 2020/1/19 16:02 + * 支持的条件和运算符类型 + * + * @Author AE86 + * @Version 1.0.0 + * @Date 2020-01-19 16:02 */ @Component public class ConditionServiceImpl implements ConditionService { diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java index 8ce1909c..3bdaa364 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java @@ -1,3 +1,6 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.biz.impl; import org.dbsyncer.biz.DataSyncService; @@ -25,7 +28,7 @@ import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.JsonUtil; import org.dbsyncer.common.util.NumberUtil; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.enums.FilterEnum; +import org.dbsyncer.sdk.enums.FilterEnum; import org.dbsyncer.sdk.scheduled.ScheduledTaskJob; import org.dbsyncer.sdk.scheduled.ScheduledTaskService; import org.dbsyncer.parser.LogService; @@ -62,9 +65,9 @@ import java.util.Map; import java.util.stream.Collectors; /** - * @author AE86 - * @version 1.0.0 - * @date 2020/04/27 10:20 + * @Author AE86 + * @Version 1.0.0 + * @Date 2020-04-27 10:20 */ @Service public class MonitorServiceImpl extends BaseServiceImpl implements MonitorService, ScheduledTaskJob { diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/vo/ConditionVo.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/vo/ConditionVo.java index c2837983..c1ee949e 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/vo/ConditionVo.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/vo/ConditionVo.java @@ -1,15 +1,18 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.biz.vo; -import org.dbsyncer.connector.enums.FilterEnum; +import org.dbsyncer.sdk.enums.FilterEnum; import org.dbsyncer.sdk.enums.QuartzFilterEnum; import org.dbsyncer.sdk.enums.OperationEnum; import java.util.List; /** - * @author AE86 - * @version 1.0.0 - * @date 2020/01/19 17:20 + * @Author AE86 + * @Version 1.0.0 + * @Date 2020-01-19 17:20 */ public class ConditionVo { diff --git a/dbsyncer-connector/dbsyncer-connector-elasticsearch/pom.xml b/dbsyncer-connector/dbsyncer-connector-elasticsearch/pom.xml new file mode 100644 index 00000000..90a0e958 --- /dev/null +++ b/dbsyncer-connector/dbsyncer-connector-elasticsearch/pom.xml @@ -0,0 +1,42 @@ + + + + dbsyncer-connector + org.ghi + 2.0.0 + + 4.0.0 + + dbsyncer-connector-elasticsearch + + + + + org.ghi + dbsyncer-sdk + ${project.parent.version} + provided + + + + + org.elasticsearch.client + elasticsearch-rest-high-level-client + + + + org.springframework.boot + spring-boot-starter-log4j2 + provided + + + + junit + junit + provided + + + + \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESConnectorInstance.java b/dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/ESConnectorInstance.java similarity index 70% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESConnectorInstance.java rename to dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/ESConnectorInstance.java index 2412ceab..a2dbdf05 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESConnectorInstance.java +++ b/dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/ESConnectorInstance.java @@ -1,13 +1,24 @@ -package org.dbsyncer.connector.es; +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.connector.elasticsearch; -import org.dbsyncer.connector.ConnectorException; -import org.dbsyncer.connector.config.ESConfig; -import org.dbsyncer.connector.util.ESUtil; +import org.dbsyncer.connector.elasticsearch.api.EasyRestHighLevelClient; +import org.dbsyncer.connector.elasticsearch.config.ESConfig; +import org.dbsyncer.connector.elasticsearch.util.ESUtil; import org.dbsyncer.sdk.connector.ConnectorInstance; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.core.MainResponse; +/** + * ES连接器实例 + * + * @Author AE86 + * @Version 1.0.0 + * @Date 2023-11-25 23:10 + */ public final class ESConnectorInstance implements ConnectorInstance { private ESConfig config; private EasyRestHighLevelClient client; @@ -19,7 +30,7 @@ public final class ESConnectorInstance implements ConnectorInstance { +/** + * ES连接器实现 + * + * @Author AE86 + * @Version 1.0.0 + * @Date 2023-11-25 23:10 + */ +public final class ElasticsearchConnector extends AbstractConnector implements ConnectorService { private final Logger logger = LoggerFactory.getLogger(getClass()); @@ -71,6 +84,8 @@ public final class ESConnector extends AbstractConnector implements ConnectorSer private final Map filters = new LinkedHashMap<>(); + private final ESConfigValidator configValidator = new ESConfigValidator(); + @PostConstruct private void init() { VALUE_MAPPERS.put(Types.DATE, new ESDateValueMapper()); @@ -110,6 +125,11 @@ public final class ESConnector extends AbstractConnector implements ConnectorSer return new ESConnectorInstance(config); } + @Override + public ConfigValidator getConfigValidator() { + return configValidator; + } + @Override public void disconnect(ESConnectorInstance connectorInstance) { connectorInstance.close(); @@ -122,7 +142,7 @@ public final class ESConnector extends AbstractConnector implements ConnectorSer return client.ping(RequestOptions.DEFAULT); } catch (IOException e) { logger.error(e.getMessage()); - throw new ConnectorException(e.getMessage()); + throw new ElasticsearchException(e.getMessage()); } } @@ -151,7 +171,7 @@ public final class ESConnector extends AbstractConnector implements ConnectorSer return tables; } catch (IOException e) { logger.error(e.getMessage()); - throw new ConnectorException(e); + throw new ElasticsearchException(e); } } @@ -175,7 +195,7 @@ public final class ESConnector extends AbstractConnector implements ConnectorSer return new MetaInfo().setColumn(fields); } catch (IOException e) { logger.error(e.getMessage()); - throw new ConnectorException(e); + throw new ElasticsearchException(e); } } @@ -196,7 +216,7 @@ public final class ESConnector extends AbstractConnector implements ConnectorSer return response.getHits().getTotalHits().value; } catch (IOException e) { logger.error(e.getMessage()); - throw new ConnectorException(e); + throw new ElasticsearchException(e); } } @@ -230,7 +250,7 @@ public final class ESConnector extends AbstractConnector implements ConnectorSer return new Result(list); } catch (IOException e) { logger.error(e.getMessage()); - throw new ConnectorException(e.getMessage()); + throw new ElasticsearchException(e.getMessage()); } } @@ -239,7 +259,7 @@ public final class ESConnector extends AbstractConnector implements ConnectorSer List data = config.getData(); if (CollectionUtils.isEmpty(data) || CollectionUtils.isEmpty(config.getFields())) { logger.error("writer data can not be empty."); - throw new ConnectorException("writer data can not be empty."); + throw new ElasticsearchException("writer data can not be empty."); } final Result result = new Result(); @@ -254,7 +274,7 @@ public final class ESConnector extends AbstractConnector implements ConnectorSer BulkResponse response = connectorInstance.getConnection().bulkWithVersion(request, RequestOptions.DEFAULT); RestStatus restStatus = response.status(); if (restStatus.getStatus() != RestStatus.OK.getStatus()) { - throw new ConnectorException(String.format("error code:%s", restStatus.getStatus())); + throw new ElasticsearchException(String.format("error code:%s", restStatus.getStatus())); } result.addSuccessData(data); } catch (Exception e) { @@ -302,7 +322,7 @@ public final class ESConnector extends AbstractConnector implements ConnectorSer private void parseProperties(List fields, Map sourceMap) { Map properties = (Map) sourceMap.get(ESUtil.PROPERTIES); if (CollectionUtils.isEmpty(properties)) { - throw new ConnectorException("查询字段不能为空."); + throw new ElasticsearchException("查询字段不能为空."); } properties.forEach((fieldName, c) -> { Map fieldDesc = (Map) c; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/EasyRestHighLevelClient.java b/dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/api/EasyRestHighLevelClient.java similarity index 93% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/EasyRestHighLevelClient.java rename to dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/api/EasyRestHighLevelClient.java index 12cc2849..de7f867d 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/EasyRestHighLevelClient.java +++ b/dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/api/EasyRestHighLevelClient.java @@ -1,4 +1,7 @@ -package org.dbsyncer.connector.es; +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.connector.elasticsearch.api; import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; @@ -22,7 +25,14 @@ import java.util.List; import static java.util.Collections.emptySet; -public class EasyRestHighLevelClient extends RestHighLevelClient { +/** + * ES客户端（同时兼容6-7版本API） + * + * @Author AE86 + * @Version 1.0.0 + * @Date 2023-11-25 23:10 + */ +public final class EasyRestHighLevelClient extends RestHighLevelClient { private Version version; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/RequestConverters.java b/dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/api/RequestConverters.java similarity index 99% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/RequestConverters.java rename to dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/api/RequestConverters.java index 5e8e799d..73f89267 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/RequestConverters.java +++ b/dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/api/RequestConverters.java @@ -1,4 +1,7 @@ -package org.dbsyncer.connector.es; +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.connector.elasticsearch.api; import org.apache.http.HttpEntity; import org.apache.http.client.methods.HttpDelete; @@ -81,6 +84,13 @@ import java.util.Locale; import java.util.Map; import java.util.StringJoiner; +/** + * ES请求（同时兼容6-7版本API） + * + * @Author AE86 + * @Version 1.0.0 + * @Date 2023-11-25 23:10 + */ final class RequestConverters { static final XContentType REQUEST_BODY_CONTENT_TYPE = XContentType.JSON; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/RethrottleRequest.java b/dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/api/RethrottleRequest.java similarity index 91% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/RethrottleRequest.java rename to dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/api/RethrottleRequest.java index 0ef8e65e..512e0ed9 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/RethrottleRequest.java +++ b/dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/api/RethrottleRequest.java @@ -1,4 +1,7 @@ -package org.dbsyncer.connector.es; +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.connector.elasticsearch.api; import org.elasticsearch.client.Validatable; import org.elasticsearch.tasks.TaskId; @@ -7,6 +10,10 @@ import java.util.Objects; /** * A request changing throttling of a task. + * + * @Author AE86 + * @Version 1.0.0 + * @Date 2023-11-25 23:10 */ public class RethrottleRequest implements Validatable { diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESQuartzListener.java b/dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/cdc/ESQuartzListener.java similarity index 90% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESQuartzListener.java rename to dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/cdc/ESQuartzListener.java index 7f7de604..eb9b867a 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESQuartzListener.java +++ b/dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/cdc/ESQuartzListener.java @@ -1,15 +1,18 @@ -package org.dbsyncer.connector.es; +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.connector.elasticsearch.cdc; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.JsonUtil; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.ConnectorException; +import org.dbsyncer.sdk.constant.ConnectorConstant; import org.dbsyncer.sdk.enums.QuartzFilterEnum; import org.dbsyncer.sdk.listener.AbstractQuartzListener; -import org.dbsyncer.sdk.model.Point; import org.dbsyncer.sdk.listener.QuartzFilter; -import org.dbsyncer.sdk.constant.ConnectorConstant; import org.dbsyncer.sdk.model.Filter; +import org.dbsyncer.sdk.model.Point; +import org.elasticsearch.ElasticsearchException; import java.util.ArrayList; import java.util.HashSet; @@ -18,10 +21,10 @@ import java.util.Map; import java.util.Set; /** - * ES定时抽取 + * CDC-ES定时监听器 * - * @version 1.0.0 * @Author AE86 + * @Version 1.0.0 * @Date 2021-09-01 20:35 */ public final class ESQuartzListener extends AbstractQuartzListener { @@ -43,7 +46,7 @@ public final class ESQuartzListener extends AbstractQuartzListener { Set set = new HashSet<>(); for (Filter f : filters) { if (set.contains(f.getValue())) { - throw new ConnectorException(String.format("系统参数%s存在多个.", f.getValue())); + throw new ElasticsearchException(String.format("系统参数%s存在多个.", f.getValue())); } QuartzFilterEnum filterEnum = QuartzFilterEnum.getQuartzFilterEnum(f.getValue()); if (null != filterEnum) { diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/ESConfig.java b/dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/config/ESConfig.java similarity index 82% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/ESConfig.java rename to dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/config/ESConfig.java index c592812e..4e3558ad 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/ESConfig.java +++ b/dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/config/ESConfig.java @@ -1,65 +1,69 @@ -package org.dbsyncer.connector.config; - -import org.dbsyncer.sdk.model.ConnectorConfig; - -/** - * @author AE86 - * @ClassName: ESConfig - * @Description: ES连接配置 - * @date: 2021年8月23日 下午8:00:00 - */ -public class ESConfig extends ConnectorConfig { - - /** - * 集群地址, http(s)-9200, tcp-9300 http://192.168.1.100:9200,http://192.168.1.200:9200 - */ - private String url; - - /** - * 帐号 - */ - private String username; - - /** - * 密码 - */ - private String password; - - /** - * 索引(相当于数据库) - */ - private String index; - - public String getUrl() { - return url; - } - - public void setUrl(String url) { - this.url = url; - } - - public String getUsername() { - return username; - } - - public void setUsername(String username) { - this.username = username; - } - - public String getPassword() { - return password; - } - - public void setPassword(String password) { - this.password = password; - } - - public String getIndex() { - return index; - } - - public void setIndex(String index) { - this.index = index; - } - +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.connector.elasticsearch.config; + +import org.dbsyncer.sdk.model.ConnectorConfig; + +/** + * ES连接配置 + * + * @Author AE86 + * @Version 1.0.0 + * @Date 2021-08-23 20:00 + */ +public class ESConfig extends ConnectorConfig { + + /** + * 集群地址, http(s)-9200, tcp-9300 http://192.168.1.100:9200,http://192.168.1.200:9200 + */ + private String url; + + /** + * 帐号 + */ + private String username; + + /** + * 密码 + */ + private String password; + + /** + * 索引(相当于数据库) + */ + private String index; + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + public String getPassword() { + return password; + } + + public void setPassword(String password) { + this.password = password; + } + + public String getIndex() { + return index; + } + + public void setIndex(String index) { + this.index = index; + } + } \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/enums/ESFieldTypeEnum.java b/dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/enums/ESFieldTypeEnum.java similarity index 90% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/enums/ESFieldTypeEnum.java rename to dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/enums/ESFieldTypeEnum.java index a8054d27..fe72c83e 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/enums/ESFieldTypeEnum.java +++ b/dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/enums/ESFieldTypeEnum.java @@ -1,7 +1,10 @@ -package org.dbsyncer.connector.enums; +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.connector.elasticsearch.enums; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.ConnectorException; +import org.elasticsearch.ElasticsearchException; import java.sql.Types; @@ -10,7 +13,7 @@ import java.sql.Types; * * @author AE86 * @version 1.0.0 - * @date 2021/08/26 21:13 + * @Date 2023-08-26 21:13 */ public enum ESFieldTypeEnum { @@ -80,7 +83,7 @@ public enum ESFieldTypeEnum { this.type = type; } - public static int getType(String code) throws ConnectorException { + public static int getType(String code) throws ElasticsearchException { for (ESFieldTypeEnum e : ESFieldTypeEnum.values()) { if (StringUtil.equals(e.getCode(), code)) { return e.getType(); diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESDateValueMapper.java b/dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/schema/ESDateValueMapper.java similarity index 62% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESDateValueMapper.java rename to dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/schema/ESDateValueMapper.java index 99e963f6..5eec3f9e 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESDateValueMapper.java +++ b/dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/schema/ESDateValueMapper.java @@ -1,31 +1,36 @@ -package org.dbsyncer.connector.es; - -import org.dbsyncer.connector.ConnectorException; -import org.dbsyncer.sdk.connector.AbstractValueMapper; -import org.dbsyncer.sdk.connector.ConnectorInstance; - -import java.sql.Date; -import java.sql.Timestamp; - -/** - * @author moyu - * @version 1.0.0 - * @date 2023/10/12 0:07 - */ -public class ESDateValueMapper extends AbstractValueMapper { - - @Override - protected java.util.Date convert(ConnectorInstance connectorInstance, Object val) { - if (val instanceof Timestamp) { - Timestamp timestamp = (Timestamp) val; - return new java.util.Date(timestamp.getTime()); - } - - if (val instanceof Date) { - Date date = (Date) val; - return new java.util.Date(date.getTime()); - } - - throw new ConnectorException(String.format("%s can not find type [%s], val [%s]", getClass().getSimpleName(), val.getClass(), val)); - } +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.connector.elasticsearch.schema; + +import org.dbsyncer.sdk.connector.AbstractValueMapper; +import org.dbsyncer.sdk.connector.ConnectorInstance; +import org.elasticsearch.ElasticsearchException; + +import java.sql.Date; +import java.sql.Timestamp; + +/** + * ES日期字段值转换 + * + * @Author AE86 + * @Version 1.0.0 + * @Date 2023-11-25 23:10 + */ +public class ESDateValueMapper extends AbstractValueMapper { + + @Override + protected java.util.Date convert(ConnectorInstance connectorInstance, Object val) { + if (val instanceof Timestamp) { + Timestamp timestamp = (Timestamp) val; + return new java.util.Date(timestamp.getTime()); + } + + if (val instanceof Date) { + Date date = (Date) val; + return new java.util.Date(date.getTime()); + } + + throw new ElasticsearchException(String.format("%s can not find type [%s], val [%s]", getClass().getSimpleName(), val.getClass(), val)); + } } \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESOtherValueMapper.java b/dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/schema/ESOtherValueMapper.java similarity index 66% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESOtherValueMapper.java rename to dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/schema/ESOtherValueMapper.java index 7b2ab9cc..9cf07f76 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/es/ESOtherValueMapper.java +++ b/dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/schema/ESOtherValueMapper.java @@ -1,35 +1,40 @@ -package org.dbsyncer.connector.es; - -import org.dbsyncer.common.util.JsonUtil; -import org.dbsyncer.connector.ConnectorException; -import org.dbsyncer.sdk.connector.AbstractValueMapper; -import org.dbsyncer.sdk.connector.ConnectorInstance; -import org.postgresql.util.PGobject; - -import java.util.Map; - -/** - * @author moyu - * @version 1.0.0 - * @date 2023/10/12 0:07 - */ -public class ESOtherValueMapper extends AbstractValueMapper { - - @Override - protected Map convert(ConnectorInstance connectorInstance, Object val) { - if (val instanceof String) { - return JsonUtil.jsonToObj((String) val, Map.class); - } - - if (val instanceof byte[]) { - return JsonUtil.jsonToObj(new String((byte[]) val), Map.class); - } - - if (val instanceof PGobject) { - PGobject pgObject = (PGobject) val; - return JsonUtil.jsonToObj(pgObject.getValue(), Map.class); - } - - throw new ConnectorException(String.format("%s can not find type [%s], val [%s]", getClass().getSimpleName(), val.getClass(), val)); - } +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.connector.elasticsearch.schema; + +import org.dbsyncer.common.util.JsonUtil; +import org.dbsyncer.sdk.connector.AbstractValueMapper; +import org.dbsyncer.sdk.connector.ConnectorInstance; +import org.elasticsearch.ElasticsearchException; +import org.postgresql.util.PGobject; + +import java.util.Map; + +/** + * ES异构字段值转换 + * + * @Author AE86 + * @Version 1.0.0 + * @Date 2023-11-25 23:10 + */ +public class ESOtherValueMapper extends AbstractValueMapper { + + @Override + protected Map convert(ConnectorInstance connectorInstance, Object val) { + if (val instanceof String) { + return JsonUtil.jsonToObj((String) val, Map.class); + } + + if (val instanceof byte[]) { + return JsonUtil.jsonToObj(new String((byte[]) val), Map.class); + } + + if (val instanceof PGobject) { + PGobject pgObject = (PGobject) val; + return JsonUtil.jsonToObj(pgObject.getValue(), Map.class); + } + + throw new ElasticsearchException(String.format("%s can not find type [%s], val [%s]", getClass().getSimpleName(), val.getClass(), val)); + } } \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/util/ESUtil.java b/dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/util/ESUtil.java similarity index 79% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/util/ESUtil.java rename to dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/util/ESUtil.java index 037037f2..e0c400b6 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/util/ESUtil.java +++ b/dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/util/ESUtil.java @@ -1,62 +1,72 @@ -package org.dbsyncer.connector.util; - -import org.apache.http.HttpHost; -import org.apache.http.auth.AuthScope; -import org.apache.http.auth.UsernamePasswordCredentials; -import org.apache.http.client.CredentialsProvider; -import org.apache.http.conn.ssl.NoopHostnameVerifier; -import org.apache.http.conn.ssl.TrustAllStrategy; -import org.apache.http.impl.client.BasicCredentialsProvider; -import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy; -import org.apache.http.ssl.SSLContextBuilder; -import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.ConnectorException; -import org.dbsyncer.connector.config.ESConfig; -import org.dbsyncer.connector.es.EasyRestHighLevelClient; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.RestClient; -import org.elasticsearch.client.RestClientBuilder; -import org.elasticsearch.client.RestHighLevelClient; - -import javax.net.ssl.SSLContext; -import java.io.IOException; -import java.util.Arrays; -import java.util.Objects; - -public abstract class ESUtil { - - public static final String PROPERTIES = "properties"; - - private ESUtil() { - } - - public static EasyRestHighLevelClient getConnection(ESConfig config) { - String[] ipAddress = StringUtil.split(config.getUrl(), ","); - HttpHost[] hosts = Arrays.stream(ipAddress).map(node -> HttpHost.create(node)).filter(Objects::nonNull).toArray( - HttpHost[]::new); - RestClientBuilder builder = RestClient.builder(hosts); - CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); - credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(config.getUsername(), config.getPassword())); - try { - SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(new TrustAllStrategy()).build(); - SSLIOSessionStrategy sessionStrategy = new SSLIOSessionStrategy(sslContext, NoopHostnameVerifier.INSTANCE); - builder.setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider).setSSLStrategy(sessionStrategy)); - final EasyRestHighLevelClient client = new EasyRestHighLevelClient(builder); - client.ping(RequestOptions.DEFAULT); - return client; - } catch (Exception e) { - throw new ConnectorException(String.format("Failed to connect to ElasticSearch on %s, %s", config.getUrl(), e.getMessage())); - } - } - - public static void close(RestHighLevelClient client) { - if (null != client) { - try { - client.close(); - } catch (IOException e) { - throw new ConnectorException(e.getMessage()); - } - } - } - +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.connector.elasticsearch.util; + +import org.apache.http.HttpHost; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.conn.ssl.NoopHostnameVerifier; +import org.apache.http.conn.ssl.TrustAllStrategy; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy; +import org.apache.http.ssl.SSLContextBuilder; +import org.dbsyncer.common.util.StringUtil; +import org.dbsyncer.connector.elasticsearch.api.EasyRestHighLevelClient; +import org.dbsyncer.connector.elasticsearch.config.ESConfig; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestClientBuilder; +import org.elasticsearch.client.RestHighLevelClient; + +import javax.net.ssl.SSLContext; +import java.io.IOException; +import java.util.Arrays; +import java.util.Objects; + +/** + * ES连接器工具类 + * + * @Author AE86 + * @Version 1.0.0 + * @Date 2023-11-25 23:10 + */ +public abstract class ESUtil { + + public static final String PROPERTIES = "properties"; + + private ESUtil() { + } + + public static EasyRestHighLevelClient getConnection(ESConfig config) { + String[] ipAddress = StringUtil.split(config.getUrl(), ","); + HttpHost[] hosts = Arrays.stream(ipAddress).map(node -> HttpHost.create(node)).filter(Objects::nonNull).toArray( + HttpHost[]::new); + RestClientBuilder builder = RestClient.builder(hosts); + CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(config.getUsername(), config.getPassword())); + try { + SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(new TrustAllStrategy()).build(); + SSLIOSessionStrategy sessionStrategy = new SSLIOSessionStrategy(sslContext, NoopHostnameVerifier.INSTANCE); + builder.setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider).setSSLStrategy(sessionStrategy)); + final EasyRestHighLevelClient client = new EasyRestHighLevelClient(builder); + client.ping(RequestOptions.DEFAULT); + return client; + } catch (Exception e) { + throw new ElasticsearchException(String.format("Failed to connect to ElasticSearch on %s, %s", config.getUrl(), e.getMessage())); + } + } + + public static void close(RestHighLevelClient client) { + if (null != client) { + try { + client.close(); + } catch (IOException e) { + throw new ElasticsearchException(e.getMessage()); + } + } + } + } \ No newline at end of file diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/ElasticsearchConfigChecker.java b/dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/validator/ESConfigValidator.java similarity index 64% rename from dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/ElasticsearchConfigChecker.java rename to dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/validator/ESConfigValidator.java index f350b45e..f1a1881b 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/ElasticsearchConfigChecker.java +++ b/dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/validator/ESConfigValidator.java @@ -1,19 +1,22 @@ -package org.dbsyncer.biz.checker.impl.connector; +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.connector.elasticsearch.validator; -import org.dbsyncer.biz.checker.ConnectorConfigChecker; -import org.dbsyncer.connector.config.ESConfig; -import org.springframework.stereotype.Component; +import org.dbsyncer.connector.elasticsearch.config.ESConfig; +import org.dbsyncer.sdk.connector.ConfigValidator; import org.springframework.util.Assert; import java.util.Map; /** - * @author AE86 - * @version 1.0.0 - * @date 2021/8/25 23:30 + * ES连接配置校验器实现 + * + * @Author AE86 + * @Version 1.0.0 + * @Date 2021-08-25 23:30 */ -@Component -public class ElasticsearchConfigChecker implements ConnectorConfigChecker { +public final class ESConfigValidator implements ConfigValidator { @Override public void modify(ESConfig connectorConfig, Map params) { diff --git a/dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/resources/META-INF/services/org.dbsyncer.sdk.spi.ConnectorService b/dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/resources/META-INF/services/org.dbsyncer.sdk.spi.ConnectorService new file mode 100644 index 00000000..5e693dc3 --- /dev/null +++ b/dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/resources/META-INF/services/org.dbsyncer.sdk.spi.ConnectorService @@ -0,0 +1 @@ +org.dbsyncer.connector.elasticsearch.ElasticsearchConnector \ No newline at end of file diff --git a/dbsyncer-connector/src/main/test/ESClientTest.java b/dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/test/ESClientTest.java similarity index 97% rename from dbsyncer-connector/src/main/test/ESClientTest.java rename to dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/test/ESClientTest.java index 24572776..0e4ae7f3 100644 --- a/dbsyncer-connector/src/main/test/ESClientTest.java +++ b/dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/test/ESClientTest.java @@ -1,5 +1,9 @@ -import org.dbsyncer.connector.config.ESConfig; -import org.dbsyncer.connector.util.ESUtil; +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ + +import org.dbsyncer.connector.elasticsearch.config.ESConfig; +import org.dbsyncer.connector.elasticsearch.util.ESUtil; import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkRequest; @@ -39,6 +43,13 @@ import java.util.Arrays; import java.util.HashMap; import java.util.Map; +/** + * ES测试用例 + * + * @Author AE86 + * @Version 1.0.0 + * @Date 2023-11-25 23:10 + */ public class ESClientTest { private final Logger logger = LoggerFactory.getLogger(getClass()); private RestHighLevelClient client; diff --git a/dbsyncer-connector/pom.xml b/dbsyncer-connector/pom.xml index 07cddc8b..977a2131 100644 --- a/dbsyncer-connector/pom.xml +++ b/dbsyncer-connector/pom.xml @@ -9,6 +9,10 @@ 4.0.0 dbsyncer-connector + pom + + dbsyncer-connector-elasticsearch + @@ -18,6 +22,13 @@ ${project.parent.version} + + + org.ghi + dbsyncer-connector-elasticsearch + ${project.parent.version} + + org.springframework diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/ConnectorConfiguration.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/ConnectorConfiguration.java index 4912eb33..5f4326b1 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/ConnectorConfiguration.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/ConnectorConfiguration.java @@ -1,19 +1,47 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.connector; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; +import org.dbsyncer.sdk.spi.ConnectorService; +import org.springframework.beans.BeansException; +import org.springframework.beans.factory.config.BeanDefinition; +import org.springframework.beans.factory.config.ConfigurableListableBeanFactory; +import org.springframework.beans.factory.support.BeanDefinitionBuilder; +import org.springframework.beans.factory.support.BeanDefinitionRegistry; +import org.springframework.beans.factory.support.BeanDefinitionRegistryPostProcessor; +import org.springframework.stereotype.Service; +import org.springframework.util.Assert; +import org.springframework.util.ClassUtils; + +import java.beans.Introspector; +import java.util.ServiceLoader; /** - * @author AE86 - * @version 1.0.0 - * @date 2019/9/19 23:17 + * 连接器配置 + * + * @Author AE86 + * @Version 1.0.0 + * @Date 2019-09-19 23:17 */ -@Configuration -public class ConnectorConfiguration { +@Service +public class ConnectorConfiguration implements BeanDefinitionRegistryPostProcessor { - @Bean - public ConnectorFactory connectorFactory() { - return new ConnectorFactory(); + @Override + public void postProcessBeanDefinitionRegistry(BeanDefinitionRegistry beanDefinitionRegistry) throws BeansException { + ServiceLoader services = ServiceLoader.load(ConnectorService.class, Thread.currentThread().getContextClassLoader()); + for (ConnectorService s : services) { + BeanDefinition beanDefinition = BeanDefinitionBuilder.genericBeanDefinition(s.getClass()).getBeanDefinition(); + String beanClassName = beanDefinition.getBeanClassName(); + Assert.state(beanClassName != null, "No bean class name set"); + String shortClassName = ClassUtils.getShortName(beanClassName); + String decapitalize = Introspector.decapitalize(shortClassName); + beanDefinitionRegistry.registerBeanDefinition(decapitalize, beanDefinition); + } } + @Override + public void postProcessBeanFactory(ConfigurableListableBeanFactory configurableListableBeanFactory) throws BeansException { + + } } diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/ConnectorFactory.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/ConnectorFactory.java index 9ea164e9..f24dc935 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/ConnectorFactory.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/ConnectorFactory.java @@ -1,3 +1,6 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.connector; import org.dbsyncer.common.model.Result; @@ -15,6 +18,7 @@ import org.dbsyncer.sdk.model.Table; import org.dbsyncer.sdk.spi.ConnectorService; import org.springframework.beans.factory.DisposableBean; import org.springframework.context.ApplicationContext; +import org.springframework.stereotype.Component; import org.springframework.util.Assert; import javax.annotation.PostConstruct; @@ -29,10 +33,11 @@ import java.util.concurrent.ConcurrentHashMap; /** * 连接器工厂 * - * @author AE86 - * @version 1.0.0 - * @date 2019/9/18 23:30 + * @Author AE86 + * @Version 1.0.0 + * @Date 2019-09-18 23:30 */ +@Component public class ConnectorFactory implements DisposableBean { private final Map pool = new ConcurrentHashMap<>(); diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/enums/FilterEnum.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/enums/FilterEnum.java deleted file mode 100644 index 7f4d6966..00000000 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/enums/FilterEnum.java +++ /dev/null @@ -1,114 +0,0 @@ -package org.dbsyncer.connector.enums; - -import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.common.util.NumberUtil; -import org.dbsyncer.connector.CompareFilter; -import org.dbsyncer.connector.ConnectorException; - -/** - * 运算符表达式类型 - * - * @author AE86 - * @version 1.0.0 - * @date 2019/9/26 23:21 - */ -public enum FilterEnum { - - /** - * 等于 - */ - EQUAL("=", (value, filterValue) -> StringUtil.equals(value, filterValue)), - /** - * 不等于 - */ - NOT_EQUAL("!=", (value, filterValue) -> !StringUtil.equals(value, filterValue)), - /** - * 大于 - */ - GT(">", (value, filterValue) -> NumberUtil.toLong(value) > NumberUtil.toLong(filterValue)), - /** - * 小于 - */ - LT("<", (value, filterValue) -> NumberUtil.toLong(value) < NumberUtil.toLong(filterValue)), - /** - * 大于等于 - */ - GT_AND_EQUAL(">=", (value, filterValue) -> NumberUtil.toLong(value) >= NumberUtil.toLong(filterValue)), - /** - * 小于等于 - */ - LT_AND_EQUAL("<=", (value, filterValue) -> NumberUtil.toLong(value) <= NumberUtil.toLong(filterValue)), - /** - * 模糊匹配 - */ - LIKE("like", (value, filterValue) -> { - boolean startsWith = StringUtil.startsWith(filterValue, "%") || StringUtil.startsWith(filterValue, "*"); - boolean endsWith = StringUtil.endsWith(filterValue, "%") || StringUtil.endsWith(filterValue, "*"); - String compareValue = StringUtil.replace(filterValue, "%", ""); - compareValue = StringUtil.replace(compareValue, "*", ""); - // 模糊匹配 - if(startsWith && endsWith){ - return StringUtil.contains(value, compareValue); - } - // 前缀匹配 - if(endsWith){ - return StringUtil.startsWith(value, compareValue); - } - // 后缀匹配 - if(startsWith){ - return StringUtil.endsWith(value, compareValue); - } - return false; - }); - - // 运算符名称 - private String name; - // 比较器 - private CompareFilter compareFilter; - - FilterEnum(String name, CompareFilter compareFilter) { - this.name = name; - this.compareFilter = compareFilter; - } - - /** - * 获取表达式 - * - * @param name - * @return - * @throws ConnectorException - */ - public static FilterEnum getFilterEnum(String name) throws ConnectorException { - for (FilterEnum e : FilterEnum.values()) { - if (StringUtil.equals(name, e.getName())) { - return e; - } - } - throw new ConnectorException(String.format("FilterEnum name \"%s\" does not exist.", name)); - } - - /** - * 获取比较器 - * - * @param filterName - * @return - * @throws ConnectorException - */ - public static CompareFilter getCompareFilter(String filterName) throws ConnectorException { - for (FilterEnum e : FilterEnum.values()) { - if (StringUtil.equals(filterName, e.getName())) { - return e.getCompareFilter(); - } - } - throw new ConnectorException(String.format("FilterEnum name \"%s\" does not exist.", filterName)); - } - - public String getName() { - return name; - } - - public CompareFilter getCompareFilter() { - return compareFilter; - } - -} \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileConnector.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileConnector.java index 5232e4b6..0f2cd11d 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileConnector.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileConnector.java @@ -1,3 +1,6 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.connector.file; import org.apache.commons.io.IOUtils; @@ -11,6 +14,7 @@ import org.dbsyncer.sdk.config.CommandConfig; import org.dbsyncer.sdk.config.ReaderConfig; import org.dbsyncer.sdk.config.WriterBatchConfig; import org.dbsyncer.sdk.connector.AbstractConnector; +import org.dbsyncer.sdk.connector.ConfigValidator; import org.dbsyncer.sdk.connector.ConnectorInstance; import org.dbsyncer.sdk.enums.ListenerTypeEnum; import org.dbsyncer.sdk.listener.Listener; @@ -38,9 +42,11 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; /** - * @author AE86 - * @version 1.0.0 - * @date 2022/5/5 23:19 + * 文件连接器实现 + * + * @Author AE86 + * @Version 1.0.0 + * @Date 2022-05-05 23:19 */ @Component public final class FileConnector extends AbstractConnector implements ConnectorService { @@ -77,6 +83,11 @@ public final class FileConnector extends AbstractConnector implements ConnectorS return new FileConnectorInstance(config); } + @Override + public ConfigValidator getConfigValidator() { + return null; + } + @Override public void disconnect(FileConnectorInstance connectorInstance) { diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/kafka/KafkaConnector.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/kafka/KafkaConnector.java index 5aaaafb4..4eb3abb9 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/kafka/KafkaConnector.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/kafka/KafkaConnector.java @@ -1,14 +1,18 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.connector.kafka; import org.dbsyncer.common.model.Result; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.JsonUtil; +import org.dbsyncer.connector.ConnectorException; +import org.dbsyncer.connector.config.KafkaConfig; import org.dbsyncer.sdk.config.CommandConfig; import org.dbsyncer.sdk.config.ReaderConfig; import org.dbsyncer.sdk.config.WriterBatchConfig; import org.dbsyncer.sdk.connector.AbstractConnector; -import org.dbsyncer.connector.ConnectorException; -import org.dbsyncer.connector.config.KafkaConfig; +import org.dbsyncer.sdk.connector.ConfigValidator; import org.dbsyncer.sdk.connector.ConnectorInstance; import org.dbsyncer.sdk.listener.Listener; import org.dbsyncer.sdk.model.Field; @@ -25,6 +29,13 @@ import java.util.Collections; import java.util.List; import java.util.Map; +/** + * Kafka连接器实现 + * + * @Author AE86 + * @Version 1.0.0 + * @Date 2021-11-22 23:55 + */ @Component public class KafkaConnector extends AbstractConnector implements ConnectorService { @@ -57,6 +68,11 @@ public class KafkaConnector extends AbstractConnector implements ConnectorServic return new KafkaConnectorInstance(config); } + @Override + public ConfigValidator getConfigValidator() { + return null; + } + @Override public void disconnect(KafkaConnectorInstance connectorInstance) { connectorInstance.close(); diff --git a/dbsyncer-connector/src/main/resources/META-INF/spring.factories b/dbsyncer-connector/src/main/resources/META-INF/spring.factories deleted file mode 100644 index 364ac5a7..00000000 --- a/dbsyncer-connector/src/main/resources/META-INF/spring.factories +++ /dev/null @@ -1,3 +0,0 @@ -# 在这里开启自动状态，遵循spring 扫描规则 -org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ -org.dbsyncer.connector.ConnectorConfiguration \ No newline at end of file diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ProfileComponent.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ProfileComponent.java index 8d1ffb6e..297da059 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ProfileComponent.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ProfileComponent.java @@ -3,7 +3,7 @@ */ package org.dbsyncer.parser; -import org.dbsyncer.connector.enums.FilterEnum; +import org.dbsyncer.sdk.enums.FilterEnum; import org.dbsyncer.sdk.enums.QuartzFilterEnum; import org.dbsyncer.parser.enums.ConvertEnum; import org.dbsyncer.parser.model.ConfigModel; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/ProfileComponentImpl.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/ProfileComponentImpl.java index 52a5b278..0c49978c 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/ProfileComponentImpl.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/ProfileComponentImpl.java @@ -6,7 +6,7 @@ package org.dbsyncer.parser.impl; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.JsonUtil; import org.dbsyncer.connector.ConnectorFactory; -import org.dbsyncer.connector.enums.FilterEnum; +import org.dbsyncer.sdk.enums.FilterEnum; import org.dbsyncer.sdk.enums.QuartzFilterEnum; import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.parser.enums.CommandEnum; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/FieldPicker.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/FieldPicker.java index ce02b15c..f43bb3ad 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/FieldPicker.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/FieldPicker.java @@ -6,8 +6,8 @@ package org.dbsyncer.parser.model; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.DateFormatUtil; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.CompareFilter; -import org.dbsyncer.connector.enums.FilterEnum; +import org.dbsyncer.sdk.filter.CompareFilter; +import org.dbsyncer.sdk.enums.FilterEnum; import org.dbsyncer.sdk.enums.OperationEnum; import org.dbsyncer.sdk.model.Field; import org.dbsyncer.sdk.model.Filter; diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/ConnectorConfigChecker.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/ConfigValidator.java similarity index 31% rename from dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/ConnectorConfigChecker.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/ConfigValidator.java index f53b43b2..2c02a902 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/ConnectorConfigChecker.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/ConfigValidator.java @@ -1,25 +1,18 @@ -package org.dbsyncer.biz.checker; - -import org.dbsyncer.parser.model.Mapping; -import org.dbsyncer.parser.model.TableGroup; +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.sdk.connector; import java.util.Map; /** - * @param ConnectorConfig - * @author AE86 - * @version 1.0.0 - * @date 2020/1/7 23:17 + * 连接器配置校验器 + * + * @Author AE86 + * @Version 1.0.0 + * @Date 2020-01-07 23:17 */ -public interface ConnectorConfigChecker { - - /** - * 处理增量同步策略 - * - * @param mapping - * @param tableGroup - */ - default void dealIncrementStrategy(Mapping mapping, TableGroup tableGroup) {} +public interface ConfigValidator { /** * 修改配置 diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/AbstractDatabaseConnector.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/AbstractDatabaseConnector.java index d556eba2..2d5940a9 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/AbstractDatabaseConnector.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/AbstractDatabaseConnector.java @@ -1,3 +1,6 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.sdk.connector.database; import org.dbsyncer.common.model.Result; @@ -11,6 +14,7 @@ import org.dbsyncer.sdk.config.ReaderConfig; import org.dbsyncer.sdk.config.SqlBuilderConfig; import org.dbsyncer.sdk.config.WriterBatchConfig; import org.dbsyncer.sdk.connector.AbstractConnector; +import org.dbsyncer.sdk.connector.ConfigValidator; import org.dbsyncer.sdk.connector.ConnectorInstance; import org.dbsyncer.sdk.connector.database.ds.SimpleConnection; import org.dbsyncer.sdk.constant.ConnectorConstant; @@ -48,6 +52,13 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; +/** + * 关系型数据库连接器实现 + * + * @Author AE86 + * @Version 1.0.0 + * @Date 2020-01-08 15:17 + */ public abstract class AbstractDatabaseConnector extends AbstractConnector implements ConnectorService, Database { private final Logger logger = LoggerFactory.getLogger(getClass()); @@ -77,6 +88,11 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector implem return new DatabaseConnectorInstance(config); } + @Override + public ConfigValidator getConfigValidator() { + return null; + } + @Override public void disconnect(DatabaseConnectorInstance connectorInstance) { connectorInstance.close(); diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/FilterEnum.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/FilterEnum.java index 9cb59482..b4deaf32 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/FilterEnum.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/FilterEnum.java @@ -1,16 +1,19 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.sdk.enums; import org.dbsyncer.common.util.NumberUtil; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.sdk.SdkException; -import org.dbsyncer.sdk.connector.CompareFilter; +import org.dbsyncer.sdk.filter.CompareFilter; /** * 运算符表达式类型 * - * @author AE86 - * @version 1.0.0 - * @date 2019/9/26 23:21 + * @Author AE86 + * @Version 1.0.0 + * @Date 2019-09-26 23:21 */ public enum FilterEnum { @@ -47,15 +50,15 @@ public enum FilterEnum { String compareValue = StringUtil.replace(filterValue, "%", ""); compareValue = StringUtil.replace(compareValue, "*", ""); // 模糊匹配 - if(startsWith && endsWith){ + if (startsWith && endsWith) { return StringUtil.contains(value, compareValue); } // 前缀匹配 - if(endsWith){ + if (endsWith) { return StringUtil.startsWith(value, compareValue); } // 后缀匹配 - if(startsWith){ + if (startsWith) { return StringUtil.endsWith(value, compareValue); } return false; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/CompareFilter.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/filter/CompareFilter.java similarity index 32% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/CompareFilter.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/filter/CompareFilter.java index c1a34378..3f0da1dc 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/CompareFilter.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/filter/CompareFilter.java @@ -1,5 +1,15 @@ -package org.dbsyncer.connector; +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.sdk.filter; +/** + * 值比较器实现 + * + * @Author AE86 + * @Version 1.0.0 + * @Date 2019-11-17 23:56 + */ public interface CompareFilter { boolean compare(String value, String filterValue); diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/ConnectorService.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/ConnectorService.java index 93eeb172..ed301f31 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/ConnectorService.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/ConnectorService.java @@ -1,3 +1,6 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.sdk.spi; import org.dbsyncer.common.model.Result; @@ -6,6 +9,7 @@ import org.dbsyncer.sdk.config.CommandConfig; import org.dbsyncer.sdk.config.DDLConfig; import org.dbsyncer.sdk.config.ReaderConfig; import org.dbsyncer.sdk.config.WriterBatchConfig; +import org.dbsyncer.sdk.connector.ConfigValidator; import org.dbsyncer.sdk.connector.ConnectorInstance; import org.dbsyncer.sdk.enums.ListenerTypeEnum; import org.dbsyncer.sdk.listener.Listener; @@ -21,8 +25,8 @@ import java.util.Map; * * @param ConnectorInstance * @param ConnectorConfig - * @Version 1.0.0 * @Author AE86 + * @Version 1.0.0 * @Date 2023-11-19 23:24 */ public interface ConnectorService { @@ -61,6 +65,13 @@ public interface ConnectorServicedbsyncer https://gitee.com/ghi/dbsyncer - dbsyncer-biz dbsyncer-cluster -- Gitee From 6a9d4f5ebb1b7fdc3019cb0bc5d65916d857a2fe Mon Sep 17 00:00:00 2001 From: AE86 <836391306@qq.com> Date: Sun, 26 Nov 2023 21:17:55 +0800 Subject: [PATCH 29/98] add module dbsyncer-connector-file --- .../src/{main => }/test/ESClientTest.java | 0 .../dbsyncer-connector-file/pom.xml | 36 +++++++++++++ .../connector/file/FileConnector.java | 18 ++++--- .../connector/file/FileConnectorInstance.java | 6 ++- .../connector/file/FileException.java | 30 +++++++++++ .../file/cdc}/BufferedRandomAccessFile.java | 11 ++-- .../connector/file/cdc}/FileListener.java | 20 ++++--- .../connector/file/column/ColumnValue.java | 9 ++-- .../file/column/impl}/FileColumnValue.java | 12 +++-- .../connector/file/config/FileConfig.java | 53 +++++++++++++++++++ .../connector/file/model}/FileResolver.java | 13 +++-- .../connector/file/model}/FileSchema.java | 11 ++-- .../file/validator}/FileConfigValidator.java | 10 ++-- .../org.dbsyncer.sdk.spi.ConnectorService | 1 + dbsyncer-connector/pom.xml | 10 +++- 15 files changed, 197 insertions(+), 43 deletions(-) rename dbsyncer-connector/dbsyncer-connector-elasticsearch/src/{main => }/test/ESClientTest.java (100%) create mode 100644 dbsyncer-connector/dbsyncer-connector-file/pom.xml rename dbsyncer-connector/{ => dbsyncer-connector-file}/src/main/java/org/dbsyncer/connector/file/FileConnector.java (93%) rename dbsyncer-connector/{ => dbsyncer-connector-file}/src/main/java/org/dbsyncer/connector/file/FileConnectorInstance.java (94%) create mode 100644 dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/FileException.java rename dbsyncer-connector/{src/main/java/org/dbsyncer/connector/file => dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/cdc}/BufferedRandomAccessFile.java (98%) rename dbsyncer-connector/{src/main/java/org/dbsyncer/connector/file => dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/cdc}/FileListener.java (94%) rename dbsyncer-connector/{ => dbsyncer-connector-file}/src/main/java/org/dbsyncer/connector/file/column/ColumnValue.java (78%) rename dbsyncer-connector/{src/main/java/org/dbsyncer/connector/file/column => dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/column/impl}/FileColumnValue.java (87%) create mode 100644 dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/config/FileConfig.java rename dbsyncer-connector/{src/main/java/org/dbsyncer/connector/file => dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/model}/FileResolver.java (91%) rename dbsyncer-connector/{src/main/java/org/dbsyncer/connector/file => dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/model}/FileSchema.java (76%) rename {dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector => dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/validator}/FileConfigValidator.java (88%) create mode 100644 dbsyncer-connector/dbsyncer-connector-file/src/main/resources/META-INF/services/org.dbsyncer.sdk.spi.ConnectorService diff --git a/dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/test/ESClientTest.java b/dbsyncer-connector/dbsyncer-connector-elasticsearch/src/test/ESClientTest.java similarity index 100% rename from dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/test/ESClientTest.java rename to dbsyncer-connector/dbsyncer-connector-elasticsearch/src/test/ESClientTest.java diff --git a/dbsyncer-connector/dbsyncer-connector-file/pom.xml b/dbsyncer-connector/dbsyncer-connector-file/pom.xml new file mode 100644 index 00000000..9c71355e --- /dev/null +++ b/dbsyncer-connector/dbsyncer-connector-file/pom.xml @@ -0,0 +1,36 @@ + + + + dbsyncer-connector + org.ghi + 2.0.0 + + 4.0.0 + + dbsyncer-connector-file + + + + + org.ghi + dbsyncer-sdk + ${project.parent.version} + provided + + + + org.springframework.boot + spring-boot-starter-log4j2 + provided + + + + junit + junit + provided + + + + \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileConnector.java b/dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/FileConnector.java similarity index 93% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileConnector.java rename to dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/FileConnector.java index 0f2cd11d..a05b8425 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileConnector.java +++ b/dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/FileConnector.java @@ -8,8 +8,11 @@ import org.apache.commons.io.LineIterator; import org.dbsyncer.common.model.Result; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.ConnectorException; -import org.dbsyncer.connector.config.FileConfig; +import org.dbsyncer.connector.file.cdc.FileListener; +import org.dbsyncer.connector.file.model.FileResolver; +import org.dbsyncer.connector.file.config.FileConfig; +import org.dbsyncer.connector.file.model.FileSchema; +import org.dbsyncer.connector.file.validator.FileConfigValidator; import org.dbsyncer.sdk.config.CommandConfig; import org.dbsyncer.sdk.config.ReaderConfig; import org.dbsyncer.sdk.config.WriterBatchConfig; @@ -24,7 +27,6 @@ import org.dbsyncer.sdk.model.Table; import org.dbsyncer.sdk.spi.ConnectorService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Component; import org.springframework.util.Assert; import java.io.File; @@ -48,7 +50,6 @@ import java.util.stream.Collectors; * @Version 1.0.0 * @Date 2022-05-05 23:19 */ -@Component public final class FileConnector extends AbstractConnector implements ConnectorService { private final Logger logger = LoggerFactory.getLogger(getClass()); @@ -57,6 +58,7 @@ public final class FileConnector extends AbstractConnector implements ConnectorS private final String FILE_NAME = "fileName"; private final String FILE_PATH = "filePath"; private final FileResolver fileResolver = new FileResolver(); + private final FileConfigValidator configValidator = new FileConfigValidator(); @Override public String getConnectorType() { @@ -85,7 +87,7 @@ public final class FileConnector extends AbstractConnector implements ConnectorS @Override public ConfigValidator getConfigValidator() { - return null; + return configValidator; } @Override @@ -146,7 +148,7 @@ public final class FileConnector extends AbstractConnector implements ConnectorS count.addAndGet(1); } } catch (IOException e) { - throw new ConnectorException(e.getCause()); + throw new FileException(e.getCause()); } finally { IOUtils.closeQuietly(reader); } @@ -181,7 +183,7 @@ public final class FileConnector extends AbstractConnector implements ConnectorS } } } catch (IOException e) { - throw new ConnectorException(e.getCause()); + throw new FileException(e.getCause()); } finally { IOUtils.closeQuietly(reader); } @@ -193,7 +195,7 @@ public final class FileConnector extends AbstractConnector implements ConnectorS List data = config.getData(); if (CollectionUtils.isEmpty(data)) { logger.error("writer data can not be empty."); - throw new ConnectorException("writer data can not be empty."); + throw new FileException("writer data can not be empty."); } final List fields = config.getFields(); diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileConnectorInstance.java b/dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/FileConnectorInstance.java similarity index 94% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileConnectorInstance.java rename to dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/FileConnectorInstance.java index 9c402011..3c068a1e 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileConnectorInstance.java +++ b/dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/FileConnectorInstance.java @@ -1,7 +1,11 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.connector.file; import org.dbsyncer.common.util.JsonUtil; -import org.dbsyncer.connector.config.FileConfig; +import org.dbsyncer.connector.file.config.FileConfig; +import org.dbsyncer.connector.file.model.FileSchema; import org.dbsyncer.sdk.connector.ConnectorInstance; import org.dbsyncer.sdk.model.Field; import org.springframework.util.Assert; diff --git a/dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/FileException.java b/dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/FileException.java new file mode 100644 index 00000000..4a971993 --- /dev/null +++ b/dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/FileException.java @@ -0,0 +1,30 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.connector.file; + +/** + * @Author AE86 + * @Version 1.0.0 + * @Date 2022-05-05 23:19 + */ +public class FileException extends RuntimeException { + + private static final long serialVersionUID = 1L; + + public FileException(String message) { + super(message); + } + + public FileException(String message, Throwable cause) { + super(message, cause); + } + + public FileException(Throwable cause) { + super(cause); + } + + protected FileException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } +} diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/BufferedRandomAccessFile.java b/dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/cdc/BufferedRandomAccessFile.java similarity index 98% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/BufferedRandomAccessFile.java rename to dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/cdc/BufferedRandomAccessFile.java index 44f6be4d..c4495468 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/BufferedRandomAccessFile.java +++ b/dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/cdc/BufferedRandomAccessFile.java @@ -1,4 +1,7 @@ -package org.dbsyncer.connector.file; +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.connector.file.cdc; import java.io.File; import java.io.FileNotFoundException; @@ -7,9 +10,9 @@ import java.io.RandomAccessFile; import java.util.Arrays; /** - * @author AE86 - * @version 1.0.0 - * @date 2022/5/7 22:27 + * @Author AE86 + * @Version 1.0.0 + * @Date 2022-05-07 22:27 */ public class BufferedRandomAccessFile extends RandomAccessFile { static final int LogBuffSz_ = 16; // 64K buffer diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileListener.java b/dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/cdc/FileListener.java similarity index 94% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileListener.java rename to dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/cdc/FileListener.java index 5415b6b8..c0449e1a 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileListener.java +++ b/dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/cdc/FileListener.java @@ -1,11 +1,17 @@ -package org.dbsyncer.connector.file; +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.connector.file.cdc; import org.apache.commons.io.IOUtils; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.NumberUtil; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.ConnectorException; -import org.dbsyncer.connector.config.FileConfig; +import org.dbsyncer.connector.file.FileConnectorInstance; +import org.dbsyncer.connector.file.FileException; +import org.dbsyncer.connector.file.model.FileSchema; +import org.dbsyncer.connector.file.config.FileConfig; +import org.dbsyncer.connector.file.model.FileResolver; import org.dbsyncer.sdk.constant.ConnectorConstant; import org.dbsyncer.sdk.listener.AbstractListener; import org.dbsyncer.sdk.listener.event.RowChangedEvent; @@ -36,9 +42,9 @@ import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; /** - * @author AE86 - * @version 1.0.0 - * @date 2022/5/6 21:42 + * @Author AE86 + * @Version 1.0.0 + * @Date 2022-05-05 23:19 */ public class FileListener extends AbstractListener { @@ -86,7 +92,7 @@ public class FileListener extends AbstractListener { } catch (Exception e) { logger.error("启动失败:{}", e.getMessage()); closePipelineAndWatch(); - throw new ConnectorException(e); + throw new FileException(e); } finally { connectLock.unlock(); } diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/column/ColumnValue.java b/dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/column/ColumnValue.java similarity index 78% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/column/ColumnValue.java rename to dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/column/ColumnValue.java index 3d76e21a..a45960e0 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/column/ColumnValue.java +++ b/dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/column/ColumnValue.java @@ -1,12 +1,15 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.connector.file.column; import java.sql.Date; import java.sql.Timestamp; /** - * @author AE86 - * @version 1.0.0 - * @date 2022/5/5 22:39 + * @Author AE86 + * @Version 1.0.0 + * @Date 2022-05-05 23:19 */ public interface ColumnValue { diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/column/FileColumnValue.java b/dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/column/impl/FileColumnValue.java similarity index 87% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/column/FileColumnValue.java rename to dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/column/impl/FileColumnValue.java index a87a8537..a6e3d5c3 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/column/FileColumnValue.java +++ b/dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/column/impl/FileColumnValue.java @@ -1,16 +1,20 @@ -package org.dbsyncer.connector.file.column; +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.connector.file.column.impl; import org.dbsyncer.common.util.DateFormatUtil; import org.dbsyncer.common.util.NumberUtil; import org.dbsyncer.common.util.StringUtil; +import org.dbsyncer.connector.file.column.ColumnValue; import java.sql.Date; import java.sql.Timestamp; /** - * @author AE86 - * @version 1.0.0 - * @date 2022/5/6 15:48 + * @Author AE86 + * @Version 1.0.0 + * @Date 2022-05-05 23:19 */ public class FileColumnValue implements ColumnValue { diff --git a/dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/config/FileConfig.java b/dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/config/FileConfig.java new file mode 100644 index 00000000..f89908aa --- /dev/null +++ b/dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/config/FileConfig.java @@ -0,0 +1,53 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.connector.file.config; + +import org.dbsyncer.sdk.model.ConnectorConfig; + +/** + * @Author AE86 + * @Version 1.0.0 + * @Date 2022-05-05 23:19 + */ +public class FileConfig extends ConnectorConfig { + + /** + * 文件目录 + */ + private String fileDir; + + /** + * 分隔符 + */ + private char separator; + + /** + * 文件描述信息 + */ + private String schema; + + public String getFileDir() { + return fileDir; + } + + public void setFileDir(String fileDir) { + this.fileDir = fileDir; + } + + public char getSeparator() { + return separator; + } + + public void setSeparator(char separator) { + this.separator = separator; + } + + public String getSchema() { + return schema; + } + + public void setSchema(String schema) { + this.schema = schema; + } +} \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileResolver.java b/dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/model/FileResolver.java similarity index 91% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileResolver.java rename to dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/model/FileResolver.java index 36172438..d272e8ae 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileResolver.java +++ b/dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/model/FileResolver.java @@ -1,8 +1,11 @@ -package org.dbsyncer.connector.file; +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.connector.file.model; import org.dbsyncer.common.column.Lexer; import org.dbsyncer.connector.file.column.ColumnValue; -import org.dbsyncer.connector.file.column.FileColumnValue; +import org.dbsyncer.connector.file.column.impl.FileColumnValue; import org.dbsyncer.sdk.model.Field; import java.util.ArrayList; @@ -11,9 +14,9 @@ import java.util.List; import java.util.Map; /** - * @author AE86 - * @version 1.0.0 - * @date 2022/5/6 15:46 + * @Author AE86 + * @Version 1.0.0 + * @Date 2022-05-06 00:04 */ public class FileResolver { diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileSchema.java b/dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/model/FileSchema.java similarity index 76% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileSchema.java rename to dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/model/FileSchema.java index 0ce273c8..ae0e80ea 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/file/FileSchema.java +++ b/dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/model/FileSchema.java @@ -1,13 +1,16 @@ -package org.dbsyncer.connector.file; +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.connector.file.model; import org.dbsyncer.sdk.model.Field; import java.util.List; /** - * @author AE86 - * @version 1.0.0 - * @date 2022/5/6 0:04 + * @Author AE86 + * @Version 1.0.0 + * @Date 2022-05-06 00:04 */ public class FileSchema { diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/FileConfigValidator.java b/dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/validator/FileConfigValidator.java similarity index 88% rename from dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/FileConfigValidator.java rename to dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/validator/FileConfigValidator.java index 9b9f5fea..62c54490 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/FileConfigValidator.java +++ b/dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/validator/FileConfigValidator.java @@ -1,14 +1,13 @@ /** * DBSyncer Copyright 2020-2023 All Rights Reserved. */ -package org.dbsyncer.biz.checker.impl.connector; +package org.dbsyncer.connector.file.validator; -import org.dbsyncer.sdk.connector.ConfigValidator; import org.dbsyncer.common.util.JsonUtil; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.config.FileConfig; -import org.dbsyncer.connector.file.FileSchema; -import org.springframework.stereotype.Component; +import org.dbsyncer.connector.file.model.FileSchema; +import org.dbsyncer.connector.file.config.FileConfig; +import org.dbsyncer.sdk.connector.ConfigValidator; import org.springframework.util.Assert; import java.io.File; @@ -22,7 +21,6 @@ import java.util.Map; * @Version 1.0.0 * @Date 2022-05-06 00:04 */ -@Component public class FileConfigValidator implements ConfigValidator { @Override diff --git a/dbsyncer-connector/dbsyncer-connector-file/src/main/resources/META-INF/services/org.dbsyncer.sdk.spi.ConnectorService b/dbsyncer-connector/dbsyncer-connector-file/src/main/resources/META-INF/services/org.dbsyncer.sdk.spi.ConnectorService new file mode 100644 index 00000000..62da8519 --- /dev/null +++ b/dbsyncer-connector/dbsyncer-connector-file/src/main/resources/META-INF/services/org.dbsyncer.sdk.spi.ConnectorService @@ -0,0 +1 @@ +org.dbsyncer.connector.file.FileConnector \ No newline at end of file diff --git a/dbsyncer-connector/pom.xml b/dbsyncer-connector/pom.xml index 977a2131..2ebc53af 100644 --- a/dbsyncer-connector/pom.xml +++ b/dbsyncer-connector/pom.xml @@ -12,6 +12,7 @@ pom dbsyncer-connector-elasticsearch + dbsyncer-connector-file @@ -22,11 +23,18 @@ ${project.parent.version} - + + + + + org.ghi + dbsyncer-connector-file + ${project.parent.version} -- Gitee From e489185c90bf79d84848e5dfb48958d5a64b0a20 Mon Sep 17 00:00:00 2001 From: AE86 <836391306@qq.com> Date: Sun, 26 Nov 2023 21:38:23 +0800 Subject: [PATCH 30/98] add module dbsyncer-connector-kafka --- .../src/test/{ => java}/ESClientTest.java | 0 .../dbsyncer-connector-kafka/pom.xml | 42 +++ .../dbsyncer/connector/kafka/KafkaClient.java | 203 ++++++------ .../connector/kafka/KafkaConnector.java | 310 +++++++++--------- .../kafka/KafkaConnectorInstance.java | 14 +- .../connector/kafka}/config/KafkaConfig.java | 288 ++++++++-------- .../kafka}/enums/KafkaFieldTypeEnum.java | 14 +- .../serialization/JsonToMapDeserializer.java | 9 +- .../serialization/MapToJsonSerializer.java | 9 +- .../connector/kafka}/util/KafkaUtil.java | 14 +- .../validator}/KafkaConfigValidator.java | 6 +- .../org.dbsyncer.sdk.spi.ConnectorService | 1 + .../src/test/java}/KafkaClientTest.java | 10 +- dbsyncer-connector/pom.xml | 16 +- .../dbsyncer/connector/config/FileConfig.java | 50 --- 15 files changed, 512 insertions(+), 474 deletions(-) rename dbsyncer-connector/dbsyncer-connector-elasticsearch/src/test/{ => java}/ESClientTest.java (100%) create mode 100644 dbsyncer-connector/dbsyncer-connector-kafka/pom.xml rename dbsyncer-connector/{ => dbsyncer-connector-kafka}/src/main/java/org/dbsyncer/connector/kafka/KafkaClient.java (90%) rename dbsyncer-connector/{ => dbsyncer-connector-kafka}/src/main/java/org/dbsyncer/connector/kafka/KafkaConnector.java (90%) rename dbsyncer-connector/{ => dbsyncer-connector-kafka}/src/main/java/org/dbsyncer/connector/kafka/KafkaConnectorInstance.java (78%) rename dbsyncer-connector/{src/main/java/org/dbsyncer/connector => dbsyncer-connector-kafka/src/main/java/org/dbsyncer/connector/kafka}/config/KafkaConfig.java (90%) rename dbsyncer-connector/{src/main/java/org/dbsyncer/connector => dbsyncer-connector-kafka/src/main/java/org/dbsyncer/connector/kafka}/enums/KafkaFieldTypeEnum.java (81%) rename dbsyncer-connector/{ => dbsyncer-connector-kafka}/src/main/java/org/dbsyncer/connector/kafka/serialization/JsonToMapDeserializer.java (92%) rename dbsyncer-connector/{ => dbsyncer-connector-kafka}/src/main/java/org/dbsyncer/connector/kafka/serialization/MapToJsonSerializer.java (91%) rename dbsyncer-connector/{src/main/java/org/dbsyncer/connector => dbsyncer-connector-kafka/src/main/java/org/dbsyncer/connector/kafka}/util/KafkaUtil.java (88%) rename {dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector => dbsyncer-connector/dbsyncer-connector-kafka/src/main/java/org/dbsyncer/connector/kafka/validator}/KafkaConfigValidator.java (96%) create mode 100644 dbsyncer-connector/dbsyncer-connector-kafka/src/main/resources/META-INF/services/org.dbsyncer.sdk.spi.ConnectorService rename dbsyncer-connector/{src/main/test => dbsyncer-connector-kafka/src/test/java}/KafkaClientTest.java (95%) delete mode 100644 dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/FileConfig.java diff --git a/dbsyncer-connector/dbsyncer-connector-elasticsearch/src/test/ESClientTest.java b/dbsyncer-connector/dbsyncer-connector-elasticsearch/src/test/java/ESClientTest.java similarity index 100% rename from dbsyncer-connector/dbsyncer-connector-elasticsearch/src/test/ESClientTest.java rename to dbsyncer-connector/dbsyncer-connector-elasticsearch/src/test/java/ESClientTest.java diff --git a/dbsyncer-connector/dbsyncer-connector-kafka/pom.xml b/dbsyncer-connector/dbsyncer-connector-kafka/pom.xml new file mode 100644 index 00000000..ffbdb6bf --- /dev/null +++ b/dbsyncer-connector/dbsyncer-connector-kafka/pom.xml @@ -0,0 +1,42 @@ + + + + dbsyncer-connector + org.ghi + 2.0.0 + + 4.0.0 + + dbsyncer-connector-kafka + + + + + org.ghi + dbsyncer-sdk + ${project.parent.version} + provided + + + + + org.apache.kafka + kafka-clients + + + + org.springframework.boot + spring-boot-starter-log4j2 + provided + + + + junit + junit + provided + + + + \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/kafka/KafkaClient.java b/dbsyncer-connector/dbsyncer-connector-kafka/src/main/java/org/dbsyncer/connector/kafka/KafkaClient.java similarity index 90% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/kafka/KafkaClient.java rename to dbsyncer-connector/dbsyncer-connector-kafka/src/main/java/org/dbsyncer/connector/kafka/KafkaClient.java index e4dd8412..68b28336 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/kafka/KafkaClient.java +++ b/dbsyncer-connector/dbsyncer-connector-kafka/src/main/java/org/dbsyncer/connector/kafka/KafkaClient.java @@ -1,99 +1,106 @@ -package org.dbsyncer.connector.kafka; - -import org.apache.kafka.clients.CommonClientConfigs; -import org.apache.kafka.clients.NetworkClient; -import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.apache.kafka.clients.consumer.KafkaConsumer; -import org.apache.kafka.clients.producer.KafkaProducer; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.common.Node; -import org.dbsyncer.connector.ConnectorException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.lang.reflect.Field; -import java.net.InetSocketAddress; -import java.net.Socket; -import java.util.List; -import java.util.Map; - -/** - * Kafka客户端，集成消费者、生产者API - */ -public class KafkaClient { - - private final Logger logger = LoggerFactory.getLogger(getClass()); - - private KafkaConsumer consumer; - private KafkaProducer producer; - private NetworkClient networkClient; - - public KafkaClient(KafkaConsumer consumer, KafkaProducer producer) { - this.consumer = consumer; - this.producer = producer; - } - - public boolean ping() { - if (null == networkClient) { - synchronized (this) { - if (null == networkClient) { - try { - networkClient = (NetworkClient) invoke(invoke(consumer, "client"), "client"); - } catch (NoSuchFieldException e) { - logger.error(e.getMessage()); - } catch (IllegalAccessException e) { - logger.error(e.getMessage()); - } - } - } - } - final Node node = networkClient.leastLoadedNode(0); - telnet(node.host(), node.port(), 5000); - return true; - } - - private boolean telnet(String host, int port, int timeout) { - Socket socket = new Socket(); - try { - socket.connect(new InetSocketAddress(host, port), timeout); - return socket.isConnected(); - } catch (IOException e) { - throw new ConnectorException(String.format("DNS resolution failed for url in %s %s:%s", CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, host, port)); - } finally { - try { - socket.close(); - } catch (IOException e) { - // nothing to do - } - } - } - - private Object invoke(Object obj, String declaredFieldName) throws NoSuchFieldException, IllegalAccessException { - final Field field = obj.getClass().getDeclaredField(declaredFieldName); - field.setAccessible(true); - return field.get(obj); - } - - public void close() { - if (null != producer) { - producer.close(); - } - if (null != consumer) { - consumer.close(); - } - } - - public void subscribe(List topics) { - consumer.subscribe(topics); - } - - public ConsumerRecords poll(long timeout) { - return consumer.poll(timeout); - } - - public void send(String topic, String key, Map map) { - producer.send(new ProducerRecord<>(topic, key, map)); - } - +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.connector.kafka; + +import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.clients.NetworkClient; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.KafkaException; +import org.apache.kafka.common.Node; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.lang.reflect.Field; +import java.net.InetSocketAddress; +import java.net.Socket; +import java.util.List; +import java.util.Map; + +/** + * Kafka客户端，集成消费者、生产者API + * + * @Author AE86 + * @Version 1.0.0 + * @Date 2021-12-16 23:09 + */ +public class KafkaClient { + + private final Logger logger = LoggerFactory.getLogger(getClass()); + + private KafkaConsumer consumer; + private KafkaProducer producer; + private NetworkClient networkClient; + + public KafkaClient(KafkaConsumer consumer, KafkaProducer producer) { + this.consumer = consumer; + this.producer = producer; + } + + public boolean ping() { + if (null == networkClient) { + synchronized (this) { + if (null == networkClient) { + try { + networkClient = (NetworkClient) invoke(invoke(consumer, "client"), "client"); + } catch (NoSuchFieldException e) { + logger.error(e.getMessage()); + } catch (IllegalAccessException e) { + logger.error(e.getMessage()); + } + } + } + } + final Node node = networkClient.leastLoadedNode(0); + telnet(node.host(), node.port(), 5000); + return true; + } + + private boolean telnet(String host, int port, int timeout) { + Socket socket = new Socket(); + try { + socket.connect(new InetSocketAddress(host, port), timeout); + return socket.isConnected(); + } catch (IOException e) { + throw new KafkaException(String.format("DNS resolution failed for url in %s %s:%s", CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, host, port)); + } finally { + try { + socket.close(); + } catch (IOException e) { + // nothing to do + } + } + } + + private Object invoke(Object obj, String declaredFieldName) throws NoSuchFieldException, IllegalAccessException { + final Field field = obj.getClass().getDeclaredField(declaredFieldName); + field.setAccessible(true); + return field.get(obj); + } + + public void close() { + if (null != producer) { + producer.close(); + } + if (null != consumer) { + consumer.close(); + } + } + + public void subscribe(List topics) { + consumer.subscribe(topics); + } + + public ConsumerRecords poll(long timeout) { + return consumer.poll(timeout); + } + + public void send(String topic, String key, Map map) { + producer.send(new ProducerRecord<>(topic, key, map)); + } + } \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/kafka/KafkaConnector.java b/dbsyncer-connector/dbsyncer-connector-kafka/src/main/java/org/dbsyncer/connector/kafka/KafkaConnector.java similarity index 90% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/kafka/KafkaConnector.java rename to dbsyncer-connector/dbsyncer-connector-kafka/src/main/java/org/dbsyncer/connector/kafka/KafkaConnector.java index 4eb3abb9..832f2d3d 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/kafka/KafkaConnector.java +++ b/dbsyncer-connector/dbsyncer-connector-kafka/src/main/java/org/dbsyncer/connector/kafka/KafkaConnector.java @@ -1,156 +1,156 @@ -/** - * DBSyncer Copyright 2020-2023 All Rights Reserved. - */ -package org.dbsyncer.connector.kafka; - -import org.dbsyncer.common.model.Result; -import org.dbsyncer.common.util.CollectionUtils; -import org.dbsyncer.common.util.JsonUtil; -import org.dbsyncer.connector.ConnectorException; -import org.dbsyncer.connector.config.KafkaConfig; -import org.dbsyncer.sdk.config.CommandConfig; -import org.dbsyncer.sdk.config.ReaderConfig; -import org.dbsyncer.sdk.config.WriterBatchConfig; -import org.dbsyncer.sdk.connector.AbstractConnector; -import org.dbsyncer.sdk.connector.ConfigValidator; -import org.dbsyncer.sdk.connector.ConnectorInstance; -import org.dbsyncer.sdk.listener.Listener; -import org.dbsyncer.sdk.model.Field; -import org.dbsyncer.sdk.model.MetaInfo; -import org.dbsyncer.sdk.model.Table; -import org.dbsyncer.sdk.spi.ConnectorService; -import org.dbsyncer.sdk.util.PrimaryKeyUtil; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Component; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; - -/** - * Kafka连接器实现 - * - * @Author AE86 - * @Version 1.0.0 - * @Date 2021-11-22 23:55 - */ -@Component -public class KafkaConnector extends AbstractConnector implements ConnectorService { - - private final Logger logger = LoggerFactory.getLogger(getClass()); - - private final String TYPE = "Kafka"; - - @Override - public String getConnectorType() { - return TYPE; - } - - @Override - public boolean isSupportedTiming() { - return false; - } - - @Override - public boolean isSupportedLog() { - return false; - } - - @Override - public Class getConfigClass() { - return KafkaConfig.class; - } - - @Override - public ConnectorInstance connect(KafkaConfig config) { - return new KafkaConnectorInstance(config); - } - - @Override - public ConfigValidator getConfigValidator() { - return null; - } - - @Override - public void disconnect(KafkaConnectorInstance connectorInstance) { - connectorInstance.close(); - } - - @Override - public boolean isAlive(KafkaConnectorInstance connectorInstance) { - return connectorInstance.getConnection().ping(); - } - - @Override - public String getConnectorInstanceCacheKey(KafkaConfig config) { - return String.format("%s-%s-%s-%s", config.getConnectorType(), config.getBootstrapServers(), config.getTopic(), config.getGroupId()); - } - - @Override - public List
    getTable(KafkaConnectorInstance connectorInstance) { - List
    topics = new ArrayList<>(); - topics.add(new Table(connectorInstance.getConfig().getTopic())); - return topics; - } - - @Override - public MetaInfo getMetaInfo(KafkaConnectorInstance connectorInstance, String tableName) { - KafkaConfig config = connectorInstance.getConfig(); - List fields = JsonUtil.jsonToArray(config.getFields(), Field.class); - return new MetaInfo().setColumn(fields); - } - - @Override - public long getCount(KafkaConnectorInstance connectorInstance, Map command) { - return 0; - } - - @Override - public Result reader(KafkaConnectorInstance connectorInstance, ReaderConfig config) { - throw new ConnectorException("Full synchronization is not supported"); - } - - @Override - public Result writer(KafkaConnectorInstance connectorInstance, WriterBatchConfig config) { - List data = config.getData(); - if (CollectionUtils.isEmpty(data) || CollectionUtils.isEmpty(config.getFields())) { - logger.error("writer data can not be empty."); - throw new ConnectorException("writer data can not be empty."); - } - - Result result = new Result(); - final KafkaConfig cfg = connectorInstance.getConfig(); - final List pkFields = PrimaryKeyUtil.findConfigPrimaryKeyFields(config); - try { - String topic = cfg.getTopic(); - // 默认取第一个主键 - final String pk = pkFields.get(0).getName(); - data.forEach(row -> connectorInstance.getConnection().send(topic, String.valueOf(row.get(pk)), row)); - result.addSuccessData(data); - } catch (Exception e) { - // 记录错误数据 - result.addFailData(data); - result.getError().append(e.getMessage()).append(System.lineSeparator()); - logger.error(e.getMessage()); - } - return result; - } - - @Override - public Map getSourceCommand(CommandConfig commandConfig) { - return Collections.EMPTY_MAP; - } - - @Override - public Map getTargetCommand(CommandConfig commandConfig) { - return Collections.EMPTY_MAP; - } - - @Override - public Listener getListener(String listenerType) { - return null; - } - +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.connector.kafka; + +import org.apache.kafka.common.KafkaException; +import org.dbsyncer.common.model.Result; +import org.dbsyncer.common.util.CollectionUtils; +import org.dbsyncer.common.util.JsonUtil; +import org.dbsyncer.connector.kafka.config.KafkaConfig; +import org.dbsyncer.connector.kafka.validator.KafkaConfigValidator; +import org.dbsyncer.sdk.config.CommandConfig; +import org.dbsyncer.sdk.config.ReaderConfig; +import org.dbsyncer.sdk.config.WriterBatchConfig; +import org.dbsyncer.sdk.connector.AbstractConnector; +import org.dbsyncer.sdk.connector.ConfigValidator; +import org.dbsyncer.sdk.connector.ConnectorInstance; +import org.dbsyncer.sdk.listener.Listener; +import org.dbsyncer.sdk.model.Field; +import org.dbsyncer.sdk.model.MetaInfo; +import org.dbsyncer.sdk.model.Table; +import org.dbsyncer.sdk.spi.ConnectorService; +import org.dbsyncer.sdk.util.PrimaryKeyUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * Kafka连接器实现 + * + * @Author AE86 + * @Version 1.0.0 + * @Date 2021-11-22 23:55 + */ +public class KafkaConnector extends AbstractConnector implements ConnectorService { + + private final Logger logger = LoggerFactory.getLogger(getClass()); + + private final String TYPE = "Kafka"; + private final KafkaConfigValidator configValidator = new KafkaConfigValidator(); + + @Override + public String getConnectorType() { + return TYPE; + } + + @Override + public boolean isSupportedTiming() { + return false; + } + + @Override + public boolean isSupportedLog() { + return false; + } + + @Override + public Class getConfigClass() { + return KafkaConfig.class; + } + + @Override + public ConnectorInstance connect(KafkaConfig config) { + return new KafkaConnectorInstance(config); + } + + @Override + public ConfigValidator getConfigValidator() { + return configValidator; + } + + @Override + public void disconnect(KafkaConnectorInstance connectorInstance) { + connectorInstance.close(); + } + + @Override + public boolean isAlive(KafkaConnectorInstance connectorInstance) { + return connectorInstance.getConnection().ping(); + } + + @Override + public String getConnectorInstanceCacheKey(KafkaConfig config) { + return String.format("%s-%s-%s-%s", config.getConnectorType(), config.getBootstrapServers(), config.getTopic(), config.getGroupId()); + } + + @Override + public List
    getTable(KafkaConnectorInstance connectorInstance) { + List
    topics = new ArrayList<>(); + topics.add(new Table(connectorInstance.getConfig().getTopic())); + return topics; + } + + @Override + public MetaInfo getMetaInfo(KafkaConnectorInstance connectorInstance, String tableName) { + KafkaConfig config = connectorInstance.getConfig(); + List fields = JsonUtil.jsonToArray(config.getFields(), Field.class); + return new MetaInfo().setColumn(fields); + } + + @Override + public long getCount(KafkaConnectorInstance connectorInstance, Map command) { + return 0; + } + + @Override + public Result reader(KafkaConnectorInstance connectorInstance, ReaderConfig config) { + throw new KafkaException("Full synchronization is not supported"); + } + + @Override + public Result writer(KafkaConnectorInstance connectorInstance, WriterBatchConfig config) { + List data = config.getData(); + if (CollectionUtils.isEmpty(data) || CollectionUtils.isEmpty(config.getFields())) { + logger.error("writer data can not be empty."); + throw new KafkaException("writer data can not be empty."); + } + + Result result = new Result(); + final KafkaConfig cfg = connectorInstance.getConfig(); + final List pkFields = PrimaryKeyUtil.findConfigPrimaryKeyFields(config); + try { + String topic = cfg.getTopic(); + // 默认取第一个主键 + final String pk = pkFields.get(0).getName(); + data.forEach(row -> connectorInstance.getConnection().send(topic, String.valueOf(row.get(pk)), row)); + result.addSuccessData(data); + } catch (Exception e) { + // 记录错误数据 + result.addFailData(data); + result.getError().append(e.getMessage()).append(System.lineSeparator()); + logger.error(e.getMessage()); + } + return result; + } + + @Override + public Map getSourceCommand(CommandConfig commandConfig) { + return Collections.EMPTY_MAP; + } + + @Override + public Map getTargetCommand(CommandConfig commandConfig) { + return Collections.EMPTY_MAP; + } + + @Override + public Listener getListener(String listenerType) { + return null; + } + } \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/kafka/KafkaConnectorInstance.java b/dbsyncer-connector/dbsyncer-connector-kafka/src/main/java/org/dbsyncer/connector/kafka/KafkaConnectorInstance.java similarity index 78% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/kafka/KafkaConnectorInstance.java rename to dbsyncer-connector/dbsyncer-connector-kafka/src/main/java/org/dbsyncer/connector/kafka/KafkaConnectorInstance.java index d32ba3f4..fbd7465b 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/kafka/KafkaConnectorInstance.java +++ b/dbsyncer-connector/dbsyncer-connector-kafka/src/main/java/org/dbsyncer/connector/kafka/KafkaConnectorInstance.java @@ -1,9 +1,19 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.connector.kafka; -import org.dbsyncer.connector.config.KafkaConfig; -import org.dbsyncer.connector.util.KafkaUtil; +import org.dbsyncer.connector.kafka.config.KafkaConfig; +import org.dbsyncer.connector.kafka.util.KafkaUtil; import org.dbsyncer.sdk.connector.ConnectorInstance; +/** + * Kafka连接器实例 + * + * @Author AE86 + * @Version 1.0.0 + * @Date 2021-11-22 23:55 + */ public final class KafkaConnectorInstance implements ConnectorInstance { private KafkaConfig config; private KafkaClient client; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/KafkaConfig.java b/dbsyncer-connector/dbsyncer-connector-kafka/src/main/java/org/dbsyncer/connector/kafka/config/KafkaConfig.java similarity index 90% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/KafkaConfig.java rename to dbsyncer-connector/dbsyncer-connector-kafka/src/main/java/org/dbsyncer/connector/kafka/config/KafkaConfig.java index 53e1d63a..8907c28c 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/KafkaConfig.java +++ b/dbsyncer-connector/dbsyncer-connector-kafka/src/main/java/org/dbsyncer/connector/kafka/config/KafkaConfig.java @@ -1,143 +1,147 @@ -package org.dbsyncer.connector.config; - -import org.dbsyncer.sdk.model.ConnectorConfig; - -/** - * @author AE86 - * @ClassName: KafkaConfig - * @Description: Kafka连接配置 - * @date: 2021年11月4日 下午8:00:00 - */ -public class KafkaConfig extends ConnectorConfig { - - private String bootstrapServers; - private String topic; - private String fields; - - // 消费者 - private String deserializer; - private String groupId; - private int sessionTimeoutMs; - private int maxPartitionFetchBytes; - - // 生产者 - private String serializer; - private int bufferMemory; - private int batchSize; - private int lingerMs; - private String acks; - private int retries; - private int maxRequestSize; - - public String getBootstrapServers() { - return bootstrapServers; - } - - public void setBootstrapServers(String bootstrapServers) { - this.bootstrapServers = bootstrapServers; - } - - public String getTopic() { - return topic; - } - - public void setTopic(String topic) { - this.topic = topic; - } - - public String getFields() { - return fields; - } - - public void setFields(String fields) { - this.fields = fields; - } - - public String getDeserializer() { - return deserializer; - } - - public void setDeserializer(String deserializer) { - this.deserializer = deserializer; - } - - public String getGroupId() { - return groupId; - } - - public void setGroupId(String groupId) { - this.groupId = groupId; - } - - public int getSessionTimeoutMs() { - return sessionTimeoutMs; - } - - public void setSessionTimeoutMs(int sessionTimeoutMs) { - this.sessionTimeoutMs = sessionTimeoutMs; - } - - public int getMaxPartitionFetchBytes() { - return maxPartitionFetchBytes; - } - - public void setMaxPartitionFetchBytes(int maxPartitionFetchBytes) { - this.maxPartitionFetchBytes = maxPartitionFetchBytes; - } - - public String getSerializer() { - return serializer; - } - - public void setSerializer(String serializer) { - this.serializer = serializer; - } - - public int getBufferMemory() { - return bufferMemory; - } - - public void setBufferMemory(int bufferMemory) { - this.bufferMemory = bufferMemory; - } - - public int getBatchSize() { - return batchSize; - } - - public void setBatchSize(int batchSize) { - this.batchSize = batchSize; - } - - public int getLingerMs() { - return lingerMs; - } - - public void setLingerMs(int lingerMs) { - this.lingerMs = lingerMs; - } - - public String getAcks() { - return acks; - } - - public void setAcks(String acks) { - this.acks = acks; - } - - public int getRetries() { - return retries; - } - - public void setRetries(int retries) { - this.retries = retries; - } - - public int getMaxRequestSize() { - return maxRequestSize; - } - - public void setMaxRequestSize(int maxRequestSize) { - this.maxRequestSize = maxRequestSize; - } +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.connector.kafka.config; + +import org.dbsyncer.sdk.model.ConnectorConfig; + +/** + * Kafka连接配置 + * + * @Author AE86 + * @Version 1.0.0 + * @Date 2021-11-04 20:10 + */ +public class KafkaConfig extends ConnectorConfig { + + private String bootstrapServers; + private String topic; + private String fields; + + // 消费者 + private String deserializer; + private String groupId; + private int sessionTimeoutMs; + private int maxPartitionFetchBytes; + + // 生产者 + private String serializer; + private int bufferMemory; + private int batchSize; + private int lingerMs; + private String acks; + private int retries; + private int maxRequestSize; + + public String getBootstrapServers() { + return bootstrapServers; + } + + public void setBootstrapServers(String bootstrapServers) { + this.bootstrapServers = bootstrapServers; + } + + public String getTopic() { + return topic; + } + + public void setTopic(String topic) { + this.topic = topic; + } + + public String getFields() { + return fields; + } + + public void setFields(String fields) { + this.fields = fields; + } + + public String getDeserializer() { + return deserializer; + } + + public void setDeserializer(String deserializer) { + this.deserializer = deserializer; + } + + public String getGroupId() { + return groupId; + } + + public void setGroupId(String groupId) { + this.groupId = groupId; + } + + public int getSessionTimeoutMs() { + return sessionTimeoutMs; + } + + public void setSessionTimeoutMs(int sessionTimeoutMs) { + this.sessionTimeoutMs = sessionTimeoutMs; + } + + public int getMaxPartitionFetchBytes() { + return maxPartitionFetchBytes; + } + + public void setMaxPartitionFetchBytes(int maxPartitionFetchBytes) { + this.maxPartitionFetchBytes = maxPartitionFetchBytes; + } + + public String getSerializer() { + return serializer; + } + + public void setSerializer(String serializer) { + this.serializer = serializer; + } + + public int getBufferMemory() { + return bufferMemory; + } + + public void setBufferMemory(int bufferMemory) { + this.bufferMemory = bufferMemory; + } + + public int getBatchSize() { + return batchSize; + } + + public void setBatchSize(int batchSize) { + this.batchSize = batchSize; + } + + public int getLingerMs() { + return lingerMs; + } + + public void setLingerMs(int lingerMs) { + this.lingerMs = lingerMs; + } + + public String getAcks() { + return acks; + } + + public void setAcks(String acks) { + this.acks = acks; + } + + public int getRetries() { + return retries; + } + + public void setRetries(int retries) { + this.retries = retries; + } + + public int getMaxRequestSize() { + return maxRequestSize; + } + + public void setMaxRequestSize(int maxRequestSize) { + this.maxRequestSize = maxRequestSize; + } } \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/enums/KafkaFieldTypeEnum.java b/dbsyncer-connector/dbsyncer-connector-kafka/src/main/java/org/dbsyncer/connector/kafka/enums/KafkaFieldTypeEnum.java similarity index 81% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/enums/KafkaFieldTypeEnum.java rename to dbsyncer-connector/dbsyncer-connector-kafka/src/main/java/org/dbsyncer/connector/kafka/enums/KafkaFieldTypeEnum.java index 0fbb46ae..d0f7e857 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/enums/KafkaFieldTypeEnum.java +++ b/dbsyncer-connector/dbsyncer-connector-kafka/src/main/java/org/dbsyncer/connector/kafka/enums/KafkaFieldTypeEnum.java @@ -1,7 +1,10 @@ -package org.dbsyncer.connector.enums; +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.connector.kafka.enums; +import org.apache.kafka.common.KafkaException; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.ConnectorException; import java.sql.Date; import java.sql.Time; @@ -13,8 +16,7 @@ import java.sql.Types; * * @author AE86 * @version 1.0.0 - * @date 2021/12/08 21:13 - * @date 2021/12/17 0:02 + * @date 2021/11/23 23:13 */ public enum KafkaFieldTypeEnum { @@ -47,13 +49,13 @@ public enum KafkaFieldTypeEnum { this.type = type; } - public static Class getType(String code) throws ConnectorException { + public static Class getType(String code) throws KafkaException { for (KafkaFieldTypeEnum e : KafkaFieldTypeEnum.values()) { if (StringUtil.equals(e.getCode(), code)) { return e.getClazz(); } } - throw new ConnectorException(String.format("Unsupported code: %s", code)); + throw new KafkaException(String.format("Unsupported code: %s", code)); } public String getCode() { diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/kafka/serialization/JsonToMapDeserializer.java b/dbsyncer-connector/dbsyncer-connector-kafka/src/main/java/org/dbsyncer/connector/kafka/serialization/JsonToMapDeserializer.java similarity index 92% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/kafka/serialization/JsonToMapDeserializer.java rename to dbsyncer-connector/dbsyncer-connector-kafka/src/main/java/org/dbsyncer/connector/kafka/serialization/JsonToMapDeserializer.java index 11e89796..67ee02ba 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/kafka/serialization/JsonToMapDeserializer.java +++ b/dbsyncer-connector/dbsyncer-connector-kafka/src/main/java/org/dbsyncer/connector/kafka/serialization/JsonToMapDeserializer.java @@ -1,3 +1,6 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.connector.kafka.serialization; import org.apache.kafka.common.errors.SerializationException; @@ -9,9 +12,9 @@ import java.util.LinkedHashMap; import java.util.Map; /** - * @author AE86 - * @version 1.0.0 - * @date 2021/12/16 23:09 + * @Author AE86 + * @Version 1.0.0 + * @Date 2021-12-16 23:09 */ public class JsonToMapDeserializer implements Deserializer { private String encoding = "UTF8"; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/kafka/serialization/MapToJsonSerializer.java b/dbsyncer-connector/dbsyncer-connector-kafka/src/main/java/org/dbsyncer/connector/kafka/serialization/MapToJsonSerializer.java similarity index 91% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/kafka/serialization/MapToJsonSerializer.java rename to dbsyncer-connector/dbsyncer-connector-kafka/src/main/java/org/dbsyncer/connector/kafka/serialization/MapToJsonSerializer.java index a91d94f0..c8353981 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/kafka/serialization/MapToJsonSerializer.java +++ b/dbsyncer-connector/dbsyncer-connector-kafka/src/main/java/org/dbsyncer/connector/kafka/serialization/MapToJsonSerializer.java @@ -1,3 +1,6 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.connector.kafka.serialization; import org.apache.kafka.common.errors.SerializationException; @@ -8,9 +11,9 @@ import java.io.UnsupportedEncodingException; import java.util.Map; /** - * @author AE86 - * @version 1.0.0 - * @date 2021/12/16 23:09 + * @Author AE86 + * @Version 1.0.0 + * @Date 2021-12-16 23:09 */ public class MapToJsonSerializer implements Serializer { private String encoding = "UTF8"; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/util/KafkaUtil.java b/dbsyncer-connector/dbsyncer-connector-kafka/src/main/java/org/dbsyncer/connector/kafka/util/KafkaUtil.java similarity index 88% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/util/KafkaUtil.java rename to dbsyncer-connector/dbsyncer-connector-kafka/src/main/java/org/dbsyncer/connector/kafka/util/KafkaUtil.java index bba9ff8b..7f77f091 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/util/KafkaUtil.java +++ b/dbsyncer-connector/dbsyncer-connector-kafka/src/main/java/org/dbsyncer/connector/kafka/util/KafkaUtil.java @@ -1,12 +1,22 @@ -package org.dbsyncer.connector.util; +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.connector.kafka.util; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.clients.producer.KafkaProducer; -import org.dbsyncer.connector.config.KafkaConfig; import org.dbsyncer.connector.kafka.KafkaClient; +import org.dbsyncer.connector.kafka.config.KafkaConfig; import java.util.Properties; +/** + * Kafka连接器工具类 + * + * @Author AE86 + * @Version 1.0.0 + * @Date 2021-12-16 23:09 + */ public abstract class KafkaUtil { public static KafkaClient getConnection(KafkaConfig config) { diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/KafkaConfigValidator.java b/dbsyncer-connector/dbsyncer-connector-kafka/src/main/java/org/dbsyncer/connector/kafka/validator/KafkaConfigValidator.java similarity index 96% rename from dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/KafkaConfigValidator.java rename to dbsyncer-connector/dbsyncer-connector-kafka/src/main/java/org/dbsyncer/connector/kafka/validator/KafkaConfigValidator.java index 6496c9a1..50815092 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/KafkaConfigValidator.java +++ b/dbsyncer-connector/dbsyncer-connector-kafka/src/main/java/org/dbsyncer/connector/kafka/validator/KafkaConfigValidator.java @@ -1,11 +1,11 @@ /** * DBSyncer Copyright 2020-2023 All Rights Reserved. */ -package org.dbsyncer.biz.checker.impl.connector; +package org.dbsyncer.connector.kafka.validator; -import org.dbsyncer.sdk.connector.ConfigValidator; import org.dbsyncer.common.util.NumberUtil; -import org.dbsyncer.connector.config.KafkaConfig; +import org.dbsyncer.connector.kafka.config.KafkaConfig; +import org.dbsyncer.sdk.connector.ConfigValidator; import org.springframework.stereotype.Component; import org.springframework.util.Assert; diff --git a/dbsyncer-connector/dbsyncer-connector-kafka/src/main/resources/META-INF/services/org.dbsyncer.sdk.spi.ConnectorService b/dbsyncer-connector/dbsyncer-connector-kafka/src/main/resources/META-INF/services/org.dbsyncer.sdk.spi.ConnectorService new file mode 100644 index 00000000..f0bdacb7 --- /dev/null +++ b/dbsyncer-connector/dbsyncer-connector-kafka/src/main/resources/META-INF/services/org.dbsyncer.sdk.spi.ConnectorService @@ -0,0 +1 @@ +org.dbsyncer.connector.kafka.KafkaConnector \ No newline at end of file diff --git a/dbsyncer-connector/src/main/test/KafkaClientTest.java b/dbsyncer-connector/dbsyncer-connector-kafka/src/test/java/KafkaClientTest.java similarity index 95% rename from dbsyncer-connector/src/main/test/KafkaClientTest.java rename to dbsyncer-connector/dbsyncer-connector-kafka/src/test/java/KafkaClientTest.java index 0f33214f..e6cb4c02 100644 --- a/dbsyncer-connector/src/main/test/KafkaClientTest.java +++ b/dbsyncer-connector/dbsyncer-connector-kafka/src/test/java/KafkaClientTest.java @@ -1,12 +1,16 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ + import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.ConsumerRecords; import org.dbsyncer.common.util.JsonUtil; -import org.dbsyncer.connector.config.KafkaConfig; -import org.dbsyncer.connector.enums.KafkaFieldTypeEnum; import org.dbsyncer.connector.kafka.KafkaClient; +import org.dbsyncer.connector.kafka.config.KafkaConfig; +import org.dbsyncer.connector.kafka.enums.KafkaFieldTypeEnum; import org.dbsyncer.connector.kafka.serialization.JsonToMapDeserializer; import org.dbsyncer.connector.kafka.serialization.MapToJsonSerializer; -import org.dbsyncer.connector.util.KafkaUtil; +import org.dbsyncer.connector.kafka.util.KafkaUtil; import org.dbsyncer.sdk.model.Field; import org.junit.After; import org.junit.Before; diff --git a/dbsyncer-connector/pom.xml b/dbsyncer-connector/pom.xml index 2ebc53af..ae12e26f 100644 --- a/dbsyncer-connector/pom.xml +++ b/dbsyncer-connector/pom.xml @@ -13,6 +13,7 @@ dbsyncer-connector-elasticsearch dbsyncer-connector-file + dbsyncer-connector-kafka @@ -30,11 +31,18 @@ ${project.parent.version} --> - + + + + + org.ghi + dbsyncer-connector-kafka + ${project.parent.version} @@ -88,12 +96,6 @@ elasticsearch-rest-high-level-client - - - org.apache.kafka - kafka-clients - - com.github.jsqlparser jsqlparser diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/FileConfig.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/FileConfig.java deleted file mode 100644 index ac4b2c7a..00000000 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/config/FileConfig.java +++ /dev/null @@ -1,50 +0,0 @@ -package org.dbsyncer.connector.config; - -import org.dbsyncer.sdk.model.ConnectorConfig; - -/** - * @author AE86 - * @version 1.0.0 - * @date 2022/5/5 23:19 - */ -public class FileConfig extends ConnectorConfig { - - /** - * 文件目录 - */ - private String fileDir; - - /** - * 分隔符 - */ - private char separator; - - /** - * 文件描述信息 - */ - private String schema; - - public String getFileDir() { - return fileDir; - } - - public void setFileDir(String fileDir) { - this.fileDir = fileDir; - } - - public char getSeparator() { - return separator; - } - - public void setSeparator(char separator) { - this.separator = separator; - } - - public String getSchema() { - return schema; - } - - public void setSchema(String schema) { - this.schema = schema; - } -} \ No newline at end of file -- Gitee From 5f8f3fe4a5f3ba2e45b1067193a4b49a91a5264f Mon Sep 17 00:00:00 2001 From: AE86 <836391306@qq.com> Date: Sun, 26 Nov 2023 22:20:06 +0800 Subject: [PATCH 31/98] add module dbsyncer-connector-mysql --- ...ker.java => DqlOracleConfigValidator.java} | 55 ++++++++++--------- ...java => DqlPostgreSQLConfigValidator.java} | 3 +- .../DqlSqlServerConfigValidator.java | 1 + .../impl/connector/OracleConfigValidator.java | 1 + ...er.java => PostgreSQLConfigValidator.java} | 3 +- ...ker.java => SqlServerConfigValidator.java} | 45 +++++++-------- .../dbsyncer-connector-mysql/pom.xml | 48 ++++++++++++++++ .../connector/mysql}/DQLMySQLConnector.java | 26 +++++++-- .../connector/mysql/MySQLConnector.java | 21 ++++++- .../connector/mysql/MySQLException.java | 30 ++++++++++ .../mysql/binlog}/BinaryLogClient.java | 5 +- .../mysql/binlog}/BinaryLogRemoteClient.java | 9 ++- .../mysql/cdc}/DqlMySQLListener.java | 11 ++-- .../connector/mysql/cdc}/MySQLListener.java | 25 +++++---- .../deserializer/DatetimeV2Deserialize.java | 3 + .../deserializer/DeleteDeserializer.java | 3 + .../deserializer/JsonBinaryDeserialize.java | 9 ++- .../deserializer/UpdateDeserializer.java | 3 + .../mysql/deserializer/WriteDeserializer.java | 3 + .../validator/DqlMySQLConfigValidator.java | 7 +-- .../mysql/validator/MySQLConfigValidator.java | 7 +-- .../org.dbsyncer.sdk.spi.ConnectorService | 2 + .../test/java}/BinaryLogRemoteClientTest.java | 11 ++-- dbsyncer-connector/pom.xml | 21 +++---- .../AbstractDataBaseConfigValidator.java | 3 +- 25 files changed, 250 insertions(+), 105 deletions(-) rename dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/{DqlOracleConfigChecker.java => DqlOracleConfigValidator.java} (83%) rename dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/{DqlPostgreSQLConfigChecker.java => DqlPostgreSQLConfigValidator.java} (84%) rename dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/{PostgreSQLConfigChecker.java => PostgreSQLConfigValidator.java} (84%) rename dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/{SqlServerConfigChecker.java => SqlServerConfigValidator.java} (77%) create mode 100644 dbsyncer-connector/dbsyncer-connector-mysql/pom.xml rename dbsyncer-connector/{src/main/java/org/dbsyncer/connector/sql => dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql}/DQLMySQLConnector.java (71%) rename dbsyncer-connector/{ => dbsyncer-connector-mysql}/src/main/java/org/dbsyncer/connector/mysql/MySQLConnector.java (87%) create mode 100644 dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/MySQLException.java rename dbsyncer-connector/{src/main/java/org/dbsyncer/connector/mysql => dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/binlog}/BinaryLogClient.java (97%) rename dbsyncer-connector/{src/main/java/org/dbsyncer/connector/mysql => dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/binlog}/BinaryLogRemoteClient.java (99%) rename dbsyncer-connector/{src/main/java/org/dbsyncer/connector/mysql => dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/cdc}/DqlMySQLListener.java (68%) rename dbsyncer-connector/{src/main/java/org/dbsyncer/connector/mysql => dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/cdc}/MySQLListener.java (94%) rename dbsyncer-connector/{ => dbsyncer-connector-mysql}/src/main/java/org/dbsyncer/connector/mysql/deserializer/DatetimeV2Deserialize.java (97%) rename dbsyncer-connector/{ => dbsyncer-connector-mysql}/src/main/java/org/dbsyncer/connector/mysql/deserializer/DeleteDeserializer.java (94%) rename dbsyncer-connector/{ => dbsyncer-connector-mysql}/src/main/java/org/dbsyncer/connector/mysql/deserializer/JsonBinaryDeserialize.java (84%) rename dbsyncer-connector/{ => dbsyncer-connector-mysql}/src/main/java/org/dbsyncer/connector/mysql/deserializer/UpdateDeserializer.java (94%) rename dbsyncer-connector/{ => dbsyncer-connector-mysql}/src/main/java/org/dbsyncer/connector/mysql/deserializer/WriteDeserializer.java (94%) rename dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlMySQLConfigChecker.java => dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/validator/DqlMySQLConfigValidator.java (67%) rename dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/MySQLConfigChecker.java => dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/validator/MySQLConfigValidator.java (40%) create mode 100644 dbsyncer-connector/dbsyncer-connector-mysql/src/main/resources/META-INF/services/org.dbsyncer.sdk.spi.ConnectorService rename dbsyncer-connector/{src/main/test => dbsyncer-connector-mysql/src/test/java}/BinaryLogRemoteClientTest.java (94%) rename {dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/connector/AbstractDataBaseConfigValidator.java (95%) diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlOracleConfigChecker.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlOracleConfigValidator.java similarity index 83% rename from dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlOracleConfigChecker.java rename to dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlOracleConfigValidator.java index 38c2756d..72ceaf94 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlOracleConfigChecker.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlOracleConfigValidator.java @@ -1,28 +1,29 @@ -/** - * DBSyncer Copyright 2020-2023 All Rights Reserved. - */ -package org.dbsyncer.biz.checker.impl.connector; - -import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.sdk.config.DatabaseConfig; -import org.springframework.stereotype.Component; - -import java.util.Map; - -/** - * @Author AE86 - * @Version 1.0.0 - * @Date 2020-01-08 15:17 - */ -@Component -public class DqlOracleConfigChecker extends AbstractDataBaseConfigValidator { - - @Override - public void modify(DatabaseConfig connectorConfig, Map params) { - super.modify(connectorConfig, params); - super.modifyDql(connectorConfig, params); - - String schema = params.get("schema"); - connectorConfig.setSchema(StringUtil.isBlank(schema) ? connectorConfig.getUsername().toUpperCase() : schema.toUpperCase()); - } +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.biz.checker.impl.connector; + +import org.dbsyncer.common.util.StringUtil; +import org.dbsyncer.sdk.config.DatabaseConfig; +import org.dbsyncer.sdk.connector.AbstractDataBaseConfigValidator; +import org.springframework.stereotype.Component; + +import java.util.Map; + +/** + * @Author AE86 + * @Version 1.0.0 + * @Date 2020-01-08 15:17 + */ +@Component +public class DqlOracleConfigValidator extends AbstractDataBaseConfigValidator { + + @Override + public void modify(DatabaseConfig connectorConfig, Map params) { + super.modify(connectorConfig, params); + super.modifyDql(connectorConfig, params); + + String schema = params.get("schema"); + connectorConfig.setSchema(StringUtil.isBlank(schema) ? connectorConfig.getUsername().toUpperCase() : schema.toUpperCase()); + } } \ No newline at end of file diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlPostgreSQLConfigChecker.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlPostgreSQLConfigValidator.java similarity index 84% rename from dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlPostgreSQLConfigChecker.java rename to dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlPostgreSQLConfigValidator.java index cb6defbe..92ddbe9b 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlPostgreSQLConfigChecker.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlPostgreSQLConfigValidator.java @@ -5,6 +5,7 @@ package org.dbsyncer.biz.checker.impl.connector; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.sdk.config.DatabaseConfig; +import org.dbsyncer.sdk.connector.AbstractDataBaseConfigValidator; import org.springframework.stereotype.Component; import java.util.Map; @@ -15,7 +16,7 @@ import java.util.Map; * @Date 2022-04-05 22:14 */ @Component -public class DqlPostgreSQLConfigChecker extends AbstractDataBaseConfigValidator { +public class DqlPostgreSQLConfigValidator extends AbstractDataBaseConfigValidator { @Override public void modify(DatabaseConfig connectorConfig, Map params) { diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlSqlServerConfigValidator.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlSqlServerConfigValidator.java index 76523ff8..c2ef80bf 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlSqlServerConfigValidator.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlSqlServerConfigValidator.java @@ -4,6 +4,7 @@ package org.dbsyncer.biz.checker.impl.connector; import org.dbsyncer.sdk.config.DatabaseConfig; +import org.dbsyncer.sdk.connector.AbstractDataBaseConfigValidator; import org.springframework.stereotype.Component; import java.util.Map; diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/OracleConfigValidator.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/OracleConfigValidator.java index af71854a..b8d823b3 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/OracleConfigValidator.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/OracleConfigValidator.java @@ -5,6 +5,7 @@ package org.dbsyncer.biz.checker.impl.connector; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.sdk.config.DatabaseConfig; +import org.dbsyncer.sdk.connector.AbstractDataBaseConfigValidator; import java.util.Map; diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/PostgreSQLConfigChecker.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/PostgreSQLConfigValidator.java similarity index 84% rename from dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/PostgreSQLConfigChecker.java rename to dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/PostgreSQLConfigValidator.java index 2acba096..4a87ee01 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/PostgreSQLConfigChecker.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/PostgreSQLConfigValidator.java @@ -5,6 +5,7 @@ package org.dbsyncer.biz.checker.impl.connector; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.sdk.config.DatabaseConfig; +import org.dbsyncer.sdk.connector.AbstractDataBaseConfigValidator; import org.springframework.stereotype.Component; import java.util.Map; @@ -15,7 +16,7 @@ import java.util.Map; * @Date 2022-04-05 22:14 */ @Component -public class PostgreSQLConfigChecker extends AbstractDataBaseConfigValidator { +public class PostgreSQLConfigValidator extends AbstractDataBaseConfigValidator { @Override public void modify(DatabaseConfig connectorConfig, Map params) { super.modify(connectorConfig, params); diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/SqlServerConfigChecker.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/SqlServerConfigValidator.java similarity index 77% rename from dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/SqlServerConfigChecker.java rename to dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/SqlServerConfigValidator.java index a73fff89..f460c21d 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/SqlServerConfigChecker.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/SqlServerConfigValidator.java @@ -1,23 +1,24 @@ -/** - * DBSyncer Copyright 2020-2023 All Rights Reserved. - */ -package org.dbsyncer.biz.checker.impl.connector; - -import org.dbsyncer.sdk.config.DatabaseConfig; -import org.springframework.stereotype.Component; - -import java.util.Map; - -/** - * @Author AE86 - * @Version 1.0.0 - * @Date 2023-11-25 23:10 - */ -@Component -public class SqlServerConfigChecker extends AbstractDataBaseConfigValidator { - @Override - public void modify(DatabaseConfig connectorConfig, Map params) { - super.modify(connectorConfig, params); - super.modifySchema(connectorConfig, params); - } +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.biz.checker.impl.connector; + +import org.dbsyncer.sdk.config.DatabaseConfig; +import org.dbsyncer.sdk.connector.AbstractDataBaseConfigValidator; +import org.springframework.stereotype.Component; + +import java.util.Map; + +/** + * @Author AE86 + * @Version 1.0.0 + * @Date 2023-11-25 23:10 + */ +@Component +public class SqlServerConfigValidator extends AbstractDataBaseConfigValidator { + @Override + public void modify(DatabaseConfig connectorConfig, Map params) { + super.modify(connectorConfig, params); + super.modifySchema(connectorConfig, params); + } } \ No newline at end of file diff --git a/dbsyncer-connector/dbsyncer-connector-mysql/pom.xml b/dbsyncer-connector/dbsyncer-connector-mysql/pom.xml new file mode 100644 index 00000000..d7deffbb --- /dev/null +++ b/dbsyncer-connector/dbsyncer-connector-mysql/pom.xml @@ -0,0 +1,48 @@ + + + + dbsyncer-connector + org.ghi + 2.0.0 + + 4.0.0 + + dbsyncer-connector-mysql + + + + + org.ghi + dbsyncer-sdk + ${project.parent.version} + provided + + + + + mysql + mysql-connector-java + + + + + com.zendesk + mysql-binlog-connector-java + + + + org.springframework.boot + spring-boot-starter-log4j2 + provided + + + + junit + junit + provided + + + + \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLMySQLConnector.java b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/DQLMySQLConnector.java similarity index 71% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLMySQLConnector.java rename to dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/DQLMySQLConnector.java index 0468042d..bb877528 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLMySQLConnector.java +++ b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/DQLMySQLConnector.java @@ -1,28 +1,44 @@ -package org.dbsyncer.connector.sql; +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.connector.mysql; -import org.dbsyncer.connector.mysql.DqlMySQLListener; -import org.dbsyncer.sdk.listener.DatabaseQuartzListener; +import org.dbsyncer.connector.mysql.cdc.DqlMySQLListener; +import org.dbsyncer.connector.mysql.validator.DqlMySQLConfigValidator; import org.dbsyncer.sdk.config.CommandConfig; import org.dbsyncer.sdk.config.ReaderConfig; +import org.dbsyncer.sdk.connector.ConfigValidator; import org.dbsyncer.sdk.connector.database.AbstractDQLConnector; import org.dbsyncer.sdk.constant.DatabaseConstant; import org.dbsyncer.sdk.enums.ListenerTypeEnum; +import org.dbsyncer.sdk.listener.DatabaseQuartzListener; import org.dbsyncer.sdk.listener.Listener; import org.dbsyncer.sdk.model.PageSql; -import org.springframework.stereotype.Component; import java.util.Map; -@Component +/** + * DQLMySQL连接器实现 + * + * @Author AE86 + * @Version 1.0.0 + * @Date 2021-11-22 23:55 + */ public final class DQLMySQLConnector extends AbstractDQLConnector { private final String TYPE = "DqlMySQL"; + private final DqlMySQLConfigValidator configValidator = new DqlMySQLConfigValidator(); @Override public String getConnectorType() { return TYPE; } + @Override + public ConfigValidator getConfigValidator() { + return configValidator; + } + @Override public String getPageSql(PageSql config) { return config.getQuerySql() + DatabaseConstant.MYSQL_PAGE_SQL; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/MySQLConnector.java b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/MySQLConnector.java similarity index 87% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/MySQLConnector.java rename to dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/MySQLConnector.java index 12abccbe..8d22b57f 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/MySQLConnector.java +++ b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/MySQLConnector.java @@ -1,6 +1,12 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.connector.mysql; import org.dbsyncer.common.util.StringUtil; +import org.dbsyncer.connector.mysql.cdc.MySQLListener; +import org.dbsyncer.connector.mysql.validator.MySQLConfigValidator; +import org.dbsyncer.sdk.connector.ConfigValidator; import org.dbsyncer.sdk.listener.DatabaseQuartzListener; import org.dbsyncer.sdk.config.ReaderConfig; import org.dbsyncer.sdk.connector.database.AbstractDatabaseConnector; @@ -11,22 +17,33 @@ import org.dbsyncer.sdk.model.PageSql; import org.dbsyncer.sdk.util.PrimaryKeyUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Component; import java.util.List; -@Component +/** + * MySQL连接器实现 + * + * @Author AE86 + * @Version 1.0.0 + * @Date 2021-11-22 23:55 + */ public final class MySQLConnector extends AbstractDatabaseConnector { private final Logger logger = LoggerFactory.getLogger(getClass()); private final String TYPE = "MySQL"; + private final MySQLConfigValidator configValidator = new MySQLConfigValidator(); @Override public String getConnectorType() { return TYPE; } + @Override + public ConfigValidator getConfigValidator() { + return configValidator; + } + @Override public Listener getListener(String listenerType) { if (ListenerTypeEnum.isTiming(listenerType)) { diff --git a/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/MySQLException.java b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/MySQLException.java new file mode 100644 index 00000000..1eadda56 --- /dev/null +++ b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/MySQLException.java @@ -0,0 +1,30 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.connector.mysql; + +/** + * @author AE86 + * @version 1.0.0 + * @date 2019/9/28 22:39 + */ +public class MySQLException extends RuntimeException { + + private static final long serialVersionUID = 1L; + + public MySQLException(String message) { + super(message); + } + + public MySQLException(String message, Throwable cause) { + super(message, cause); + } + + public MySQLException(Throwable cause) { + super(cause); + } + + protected MySQLException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } +} diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/BinaryLogClient.java b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/binlog/BinaryLogClient.java similarity index 97% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/BinaryLogClient.java rename to dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/binlog/BinaryLogClient.java index e9cb6ecb..147adb10 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/BinaryLogClient.java +++ b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/binlog/BinaryLogClient.java @@ -1,4 +1,7 @@ -package org.dbsyncer.connector.mysql; +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.connector.mysql.binlog; import com.github.shyiko.mysql.binlog.event.EventType; import com.github.shyiko.mysql.binlog.event.TableMapEventData; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/BinaryLogRemoteClient.java b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/binlog/BinaryLogRemoteClient.java similarity index 99% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/BinaryLogRemoteClient.java rename to dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/binlog/BinaryLogRemoteClient.java index 03565dc3..daeefe02 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/BinaryLogRemoteClient.java +++ b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/binlog/BinaryLogRemoteClient.java @@ -1,4 +1,7 @@ -package org.dbsyncer.connector.mysql; +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.connector.mysql.binlog; import com.github.shyiko.mysql.binlog.GtidSet; import com.github.shyiko.mysql.binlog.MariadbGtidSet; @@ -8,7 +11,7 @@ import com.github.shyiko.mysql.binlog.io.ByteArrayInputStream; import com.github.shyiko.mysql.binlog.network.*; import com.github.shyiko.mysql.binlog.network.protocol.*; import com.github.shyiko.mysql.binlog.network.protocol.command.*; -import org.dbsyncer.connector.ConnectorException; +import org.dbsyncer.connector.mysql.MySQLException; import org.dbsyncer.connector.mysql.deserializer.DeleteDeserializer; import org.dbsyncer.connector.mysql.deserializer.UpdateDeserializer; import org.dbsyncer.connector.mysql.deserializer.WriteDeserializer; @@ -653,7 +656,7 @@ public class BinaryLogRemoteClient implements BinaryLogClient { String error = String.format("keepalive: Trying to restore lost connection to %s", clientId); logger.info(error); try { - lifecycleListeners.forEach(listener -> listener.onCommunicationFailure(this, new ConnectorException(error))); + lifecycleListeners.forEach(listener -> listener.onCommunicationFailure(this, new MySQLException(error))); } catch (Exception e) { logger.warn("keepalive error", e); } diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/DqlMySQLListener.java b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/cdc/DqlMySQLListener.java similarity index 68% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/DqlMySQLListener.java rename to dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/cdc/DqlMySQLListener.java index 6566010e..6a73d098 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/DqlMySQLListener.java +++ b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/cdc/DqlMySQLListener.java @@ -1,11 +1,14 @@ -package org.dbsyncer.connector.mysql; +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.connector.mysql.cdc; import org.dbsyncer.sdk.listener.ChangedEvent; /** - * @author AE86 - * @version 1.0.0 - * @date 2022/5/28 22:02 + * @Author AE86 + * @Version 1.0.0 + * @Date 2022-05-28 22:02 */ public class DqlMySQLListener extends MySQLListener { diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/MySQLListener.java b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/cdc/MySQLListener.java similarity index 94% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/MySQLListener.java rename to dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/cdc/MySQLListener.java index b5426e2f..f72ec2b1 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/MySQLListener.java +++ b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/cdc/MySQLListener.java @@ -1,4 +1,7 @@ -package org.dbsyncer.connector.mysql; +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.connector.mysql.cdc; import com.github.shyiko.mysql.binlog.event.DeleteRowsEventData; import com.github.shyiko.mysql.binlog.event.Event; @@ -15,10 +18,12 @@ import net.sf.jsqlparser.JSQLParserException; import net.sf.jsqlparser.parser.CCJSqlParserUtil; import net.sf.jsqlparser.statement.alter.Alter; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.sdk.listener.AbstractDatabaseListener; -import org.dbsyncer.connector.ConnectorException; +import org.dbsyncer.connector.mysql.binlog.BinaryLogClient; +import org.dbsyncer.connector.mysql.binlog.BinaryLogRemoteClient; +import org.dbsyncer.connector.mysql.MySQLException; import org.dbsyncer.sdk.config.DatabaseConfig; import org.dbsyncer.sdk.constant.ConnectorConstant; +import org.dbsyncer.sdk.listener.AbstractDatabaseListener; import org.dbsyncer.sdk.listener.event.DDLChangedEvent; import org.dbsyncer.sdk.listener.event.RowChangedEvent; import org.dbsyncer.sdk.model.ChangedOffset; @@ -41,9 +46,9 @@ import java.util.stream.Stream; import static java.util.regex.Pattern.compile; /** - * @version 1.0.0 * @Author AE86 - * @Date 2020-05-12 21:14 + * @Version 1.0.0 + * @Date 2022-05-28 22:02 */ public class MySQLListener extends AbstractDatabaseListener { @@ -73,7 +78,7 @@ public class MySQLListener extends AbstractDatabaseListener { connected = true; } catch (Exception e) { logger.error("启动失败:{}", e.getMessage()); - throw new ConnectorException(e); + throw new MySQLException(e); } finally { connectLock.unlock(); } @@ -102,7 +107,7 @@ public class MySQLListener extends AbstractDatabaseListener { private void run() throws Exception { final DatabaseConfig config = (DatabaseConfig) connectorConfig; if (StringUtil.isBlank(config.getUrl())) { - throw new ConnectorException("url is invalid"); + throw new MySQLException("url is invalid"); } database = DatabaseUtil.getDatabaseName(config.getUrl()); cluster = readNodes(config.getUrl()); @@ -166,7 +171,7 @@ public class MySQLListener extends AbstractDatabaseListener { } run(); - errorEvent(new ConnectorException(String.format("重启成功, %s", client.getWorkerThreadName()))); + errorEvent(new MySQLException(String.format("重启成功, %s", client.getWorkerThreadName()))); logger.error("第{}次重启成功, ThreadName:{} ", i, client.getWorkerThreadName()); recovery = false; break; @@ -174,7 +179,7 @@ public class MySQLListener extends AbstractDatabaseListener { logger.error("第{}次重启异常, ThreadName:{}, {}", i, client.getWorkerThreadName(), e.getMessage()); // 无法连接，关闭任务 if (i == RETRY_TIMES) { - errorEvent(new ConnectorException(String.format("重启异常, %s, %s", client.getWorkerThreadName(), e.getMessage()))); + errorEvent(new MySQLException(String.format("重启异常, %s, %s", client.getWorkerThreadName(), e.getMessage()))); } } try { @@ -249,7 +254,7 @@ public class MySQLListener extends AbstractDatabaseListener { String log = String.format("线程[%s]执行异常。由于MySQL配置了过期binlog文件自动删除机制，已无法找到原binlog文件%s。建议先保存驱动（加载最新的binlog文件），再启动驱动。", client.getWorkerThreadName(), client.getBinlogFilename()); - errorEvent(new ConnectorException(log)); + errorEvent(new MySQLException(log)); return; } } diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/deserializer/DatetimeV2Deserialize.java b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/deserializer/DatetimeV2Deserialize.java similarity index 97% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/deserializer/DatetimeV2Deserialize.java rename to dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/deserializer/DatetimeV2Deserialize.java index 93e0ab60..43db7e5f 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/deserializer/DatetimeV2Deserialize.java +++ b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/deserializer/DatetimeV2Deserialize.java @@ -1,3 +1,6 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.connector.mysql.deserializer; import com.github.shyiko.mysql.binlog.io.ByteArrayInputStream; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/deserializer/DeleteDeserializer.java b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/deserializer/DeleteDeserializer.java similarity index 94% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/deserializer/DeleteDeserializer.java rename to dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/deserializer/DeleteDeserializer.java index 54d52a64..67b34ef0 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/deserializer/DeleteDeserializer.java +++ b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/deserializer/DeleteDeserializer.java @@ -1,3 +1,6 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.connector.mysql.deserializer; import com.github.shyiko.mysql.binlog.event.TableMapEventData; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/deserializer/JsonBinaryDeserialize.java b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/deserializer/JsonBinaryDeserialize.java similarity index 84% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/deserializer/JsonBinaryDeserialize.java rename to dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/deserializer/JsonBinaryDeserialize.java index 72458216..d14fa2de 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/deserializer/JsonBinaryDeserialize.java +++ b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/deserializer/JsonBinaryDeserialize.java @@ -1,3 +1,6 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.connector.mysql.deserializer; import com.github.shyiko.mysql.binlog.event.deserialization.json.JsonBinary; @@ -6,9 +9,9 @@ import com.github.shyiko.mysql.binlog.io.ByteArrayInputStream; import java.io.IOException; /** - * @author AE86 - * @version 1.0.0 - * @date 2022/8/30 0:22 + * @Author AE86 + * @Version 1.0.0 + * @Date 2022-08-30 00:22 */ public final class JsonBinaryDeserialize { diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/deserializer/UpdateDeserializer.java b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/deserializer/UpdateDeserializer.java similarity index 94% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/deserializer/UpdateDeserializer.java rename to dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/deserializer/UpdateDeserializer.java index 04f68e6d..f6af7244 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/deserializer/UpdateDeserializer.java +++ b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/deserializer/UpdateDeserializer.java @@ -1,3 +1,6 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.connector.mysql.deserializer; import com.github.shyiko.mysql.binlog.event.TableMapEventData; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/deserializer/WriteDeserializer.java b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/deserializer/WriteDeserializer.java similarity index 94% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/deserializer/WriteDeserializer.java rename to dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/deserializer/WriteDeserializer.java index e1c5b277..0dd84194 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/mysql/deserializer/WriteDeserializer.java +++ b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/deserializer/WriteDeserializer.java @@ -1,3 +1,6 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.connector.mysql.deserializer; import com.github.shyiko.mysql.binlog.event.TableMapEventData; diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlMySQLConfigChecker.java b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/validator/DqlMySQLConfigValidator.java similarity index 67% rename from dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlMySQLConfigChecker.java rename to dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/validator/DqlMySQLConfigValidator.java index 255f1b51..56a621fd 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlMySQLConfigChecker.java +++ b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/validator/DqlMySQLConfigValidator.java @@ -1,10 +1,10 @@ /** * DBSyncer Copyright 2020-2023 All Rights Reserved. */ -package org.dbsyncer.biz.checker.impl.connector; +package org.dbsyncer.connector.mysql.validator; import org.dbsyncer.sdk.config.DatabaseConfig; -import org.springframework.stereotype.Component; +import org.dbsyncer.sdk.connector.AbstractDataBaseConfigValidator; import java.util.Map; @@ -13,8 +13,7 @@ import java.util.Map; * @Version 1.0.0 * @Date 2020-01-07 15:17 */ -@Component -public class DqlMySQLConfigChecker extends AbstractDataBaseConfigValidator { +public class DqlMySQLConfigValidator extends AbstractDataBaseConfigValidator { @Override public void modify(DatabaseConfig connectorConfig, Map params) { diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/MySQLConfigChecker.java b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/validator/MySQLConfigValidator.java similarity index 40% rename from dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/MySQLConfigChecker.java rename to dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/validator/MySQLConfigValidator.java index fd283ebc..2b2cc90d 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/MySQLConfigChecker.java +++ b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/validator/MySQLConfigValidator.java @@ -1,16 +1,15 @@ /** * DBSyncer Copyright 2020-2023 All Rights Reserved. */ -package org.dbsyncer.biz.checker.impl.connector; +package org.dbsyncer.connector.mysql.validator; -import org.springframework.stereotype.Component; +import org.dbsyncer.sdk.connector.AbstractDataBaseConfigValidator; /** * @Author AE86 * @Version 1.0.0 * @Date 2020-01-08 15:17 */ -@Component -public class MySQLConfigChecker extends AbstractDataBaseConfigValidator { +public class MySQLConfigValidator extends AbstractDataBaseConfigValidator { } \ No newline at end of file diff --git a/dbsyncer-connector/dbsyncer-connector-mysql/src/main/resources/META-INF/services/org.dbsyncer.sdk.spi.ConnectorService b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/resources/META-INF/services/org.dbsyncer.sdk.spi.ConnectorService new file mode 100644 index 00000000..acec8d6b --- /dev/null +++ b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/resources/META-INF/services/org.dbsyncer.sdk.spi.ConnectorService @@ -0,0 +1,2 @@ +org.dbsyncer.connector.mysql.MySQLConnector +org.dbsyncer.connector.mysql.DQLMySQLConnector \ No newline at end of file diff --git a/dbsyncer-connector/src/main/test/BinaryLogRemoteClientTest.java b/dbsyncer-connector/dbsyncer-connector-mysql/src/test/java/BinaryLogRemoteClientTest.java similarity index 94% rename from dbsyncer-connector/src/main/test/BinaryLogRemoteClientTest.java rename to dbsyncer-connector/dbsyncer-connector-mysql/src/test/java/BinaryLogRemoteClientTest.java index c008f8e4..de5667fd 100644 --- a/dbsyncer-connector/src/main/test/BinaryLogRemoteClientTest.java +++ b/dbsyncer-connector/dbsyncer-connector-mysql/src/test/java/BinaryLogRemoteClientTest.java @@ -1,7 +1,10 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ import com.github.shyiko.mysql.binlog.event.*; import org.dbsyncer.sdk.constant.ConnectorConstant; -import org.dbsyncer.connector.mysql.BinaryLogClient; -import org.dbsyncer.connector.mysql.BinaryLogRemoteClient; +import org.dbsyncer.connector.mysql.binlog.BinaryLogClient; +import org.dbsyncer.connector.mysql.binlog.BinaryLogRemoteClient; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -12,9 +15,9 @@ import java.util.stream.Collectors; import java.util.stream.Stream; /** - * @version 1.0.0 * @Author AE86 - * @Date 2020-11-13 22:25 + * @Version 1.0.0 + * @Date 2021-11-22 23:55 */ public class BinaryLogRemoteClientTest { diff --git a/dbsyncer-connector/pom.xml b/dbsyncer-connector/pom.xml index ae12e26f..c6de6c71 100644 --- a/dbsyncer-connector/pom.xml +++ b/dbsyncer-connector/pom.xml @@ -14,6 +14,7 @@ dbsyncer-connector-elasticsearch dbsyncer-connector-file dbsyncer-connector-kafka + dbsyncer-connector-mysql @@ -38,11 +39,18 @@ ${project.parent.version} --> - + + + + + org.ghi + dbsyncer-connector-mysql + ${project.parent.version} @@ -51,12 +59,6 @@ spring-jdbc - - - mysql - mysql-connector-java - - com.oracle @@ -101,11 +103,6 @@ jsqlparser - - com.zendesk - mysql-binlog-connector-java - - org.springframework.boot spring-boot-starter-log4j2 diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/AbstractDataBaseConfigValidator.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/AbstractDataBaseConfigValidator.java similarity index 95% rename from dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/AbstractDataBaseConfigValidator.java rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/AbstractDataBaseConfigValidator.java index fc913071..48de6b1a 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/AbstractDataBaseConfigValidator.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/AbstractDataBaseConfigValidator.java @@ -1,12 +1,11 @@ /** * DBSyncer Copyright 2020-2023 All Rights Reserved. */ -package org.dbsyncer.biz.checker.impl.connector; +package org.dbsyncer.sdk.connector; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.JsonUtil; import org.dbsyncer.sdk.config.DatabaseConfig; -import org.dbsyncer.sdk.connector.ConfigValidator; import org.dbsyncer.sdk.model.SqlTable; import org.springframework.util.Assert; -- Gitee From 5a00b95e47853eaa619f04be5619380904db362a Mon Sep 17 00:00:00 2001 From: AE86 <836391306@qq.com> Date: Sun, 26 Nov 2023 22:45:59 +0800 Subject: [PATCH 32/98] add module dbsyncer-connector-oracle --- .../src/test/java}/FileConnectionTest.java | 136 +++++++++--------- .../src/test/java}/FileWatchTest.java | 0 .../connector/mysql/MySQLException.java | 6 +- .../dbsyncer-connector-oracle/pom.xml | 47 ++++++ .../connector/oracle}/DQLOracleConnector.java | 26 +++- .../connector/oracle/OracleConnector.java | 24 +++- .../connector/oracle/OracleException.java | 30 ++++ .../oracle/cdc}/DqlOracleListener.java | 11 +- .../connector/oracle/cdc}/OracleListener.java | 15 +- .../oracle/dcn/DBChangeNotification.java | 12 +- .../oracle/dcn/RowEventListener.java | 6 +- .../oracle/geometry/DataException.java | 3 + .../oracle/geometry/J3D_Geometry.java | 3 + .../connector/oracle/geometry/JGeometry.java | 3 + .../connector/oracle/geometry/JPoint2DD.java | 3 + .../connector/oracle/geometry/SdoPickler.java | 3 + .../connector/oracle/model}/DCNEvent.java | 4 +- .../schema}/OracleOtherValueMapper.java | 15 +- .../validator}/DqlOracleConfigValidator.java | 4 +- .../validator}/OracleConfigValidator.java | 2 +- .../org.dbsyncer.sdk.spi.ConnectorService | 2 + .../test/java}/DBChangeNotificationTest.java | 7 +- .../test/java}/LinkedBlockingQueueTest.java | 10 +- dbsyncer-connector/pom.xml | 21 ++- .../src/main/test/CgLibTest.java | 31 ---- 25 files changed, 268 insertions(+), 156 deletions(-) rename dbsyncer-connector/{src/main/test => dbsyncer-connector-file/src/test/java}/FileConnectionTest.java (96%) rename dbsyncer-connector/{src/main/test => dbsyncer-connector-file/src/test/java}/FileWatchTest.java (100%) create mode 100644 dbsyncer-connector/dbsyncer-connector-oracle/pom.xml rename dbsyncer-connector/{src/main/java/org/dbsyncer/connector/sql => dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle}/DQLOracleConnector.java (71%) rename dbsyncer-connector/{ => dbsyncer-connector-oracle}/src/main/java/org/dbsyncer/connector/oracle/OracleConnector.java (89%) create mode 100644 dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/OracleException.java rename dbsyncer-connector/{src/main/java/org/dbsyncer/connector/oracle => dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc}/DqlOracleListener.java (77%) rename dbsyncer-connector/{src/main/java/org/dbsyncer/connector/oracle => dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc}/OracleListener.java (82%) rename dbsyncer-connector/{ => dbsyncer-connector-oracle}/src/main/java/org/dbsyncer/connector/oracle/dcn/DBChangeNotification.java (97%) rename dbsyncer-connector/{ => dbsyncer-connector-oracle}/src/main/java/org/dbsyncer/connector/oracle/dcn/RowEventListener.java (71%) rename dbsyncer-connector/{ => dbsyncer-connector-oracle}/src/main/java/org/dbsyncer/connector/oracle/geometry/DataException.java (95%) rename dbsyncer-connector/{ => dbsyncer-connector-oracle}/src/main/java/org/dbsyncer/connector/oracle/geometry/J3D_Geometry.java (96%) rename dbsyncer-connector/{ => dbsyncer-connector-oracle}/src/main/java/org/dbsyncer/connector/oracle/geometry/JGeometry.java (99%) rename dbsyncer-connector/{ => dbsyncer-connector-oracle}/src/main/java/org/dbsyncer/connector/oracle/geometry/JPoint2DD.java (94%) rename dbsyncer-connector/{ => dbsyncer-connector-oracle}/src/main/java/org/dbsyncer/connector/oracle/geometry/SdoPickler.java (99%) rename dbsyncer-connector/{src/main/java/org/dbsyncer/connector/oracle => dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/model}/DCNEvent.java (82%) rename dbsyncer-connector/{src/main/java/org/dbsyncer/connector/oracle => dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/schema}/OracleOtherValueMapper.java (83%) rename {dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector => dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/validator}/DqlOracleConfigValidator.java (87%) rename {dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector => dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/validator}/OracleConfigValidator.java (93%) create mode 100644 dbsyncer-connector/dbsyncer-connector-oracle/src/main/resources/META-INF/services/org.dbsyncer.sdk.spi.ConnectorService rename dbsyncer-connector/{src/main/test => dbsyncer-connector-oracle/src/test/java}/DBChangeNotificationTest.java (96%) rename dbsyncer-connector/{src/main/test => dbsyncer-connector-oracle/src/test/java}/LinkedBlockingQueueTest.java (94%) delete mode 100644 dbsyncer-connector/src/main/test/CgLibTest.java diff --git a/dbsyncer-connector/src/main/test/FileConnectionTest.java b/dbsyncer-connector/dbsyncer-connector-file/src/test/java/FileConnectionTest.java similarity index 96% rename from dbsyncer-connector/src/main/test/FileConnectionTest.java rename to dbsyncer-connector/dbsyncer-connector-file/src/test/java/FileConnectionTest.java index 5718eb11..19d88533 100644 --- a/dbsyncer-connector/src/main/test/FileConnectionTest.java +++ b/dbsyncer-connector/dbsyncer-connector-file/src/test/java/FileConnectionTest.java @@ -1,69 +1,69 @@ -import org.apache.commons.io.IOUtils; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.*; -import java.nio.ByteBuffer; -import java.nio.channels.FileChannel; -import java.nio.charset.Charset; -import java.util.List; -import java.util.concurrent.CountDownLatch; - -/** - * @author AE86 - * @version 1.0.0 - * @date 2022/5/11 20:19 - */ -public class FileConnectionTest { - - private final Logger logger = LoggerFactory.getLogger(getClass()); - - private CountDownLatch latch; - - @Test - public void testConnection() throws InterruptedException, IOException { - File f = new File("D:\\test\\abc.txt"); - RandomAccessFile file = new RandomAccessFile(f, "rw"); - - int count = 10; - latch = new CountDownLatch(count); - for (int i = 0; i < count; i++) { - Thread t = new Thread(new WritingThread(i, file.getChannel())); - t.start(); - } - latch.await(); - file.close(); - InputStream fileR = new FileInputStream(f); - List strings = IOUtils.readLines(fileR, Charset.defaultCharset()); - strings.forEach(line -> logger.info("{}", line)); - fileR.close(); - } - - class WritingThread implements Runnable { - - private FileChannel channel; - - private int id; - - public WritingThread(int id, FileChannel channel) { - this.channel = channel; - this.id = id; - } - - @Override - public void run() { - logger.info("Thread {} is Writing", id); - try { - for (int i = 1; i <= 5; i++) { - String msg = String.format("%s, %s, %s\n", Thread.currentThread().getName(), id, i); - this.channel.write(ByteBuffer.wrap(msg.getBytes())); - } - } catch (IOException e) { - logger.error(e.getMessage()); - } - latch.countDown(); - } - } - +import org.apache.commons.io.IOUtils; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.*; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.charset.Charset; +import java.util.List; +import java.util.concurrent.CountDownLatch; + +/** + * @author AE86 + * @version 1.0.0 + * @date 2022/5/11 20:19 + */ +public class FileConnectionTest { + + private final Logger logger = LoggerFactory.getLogger(getClass()); + + private CountDownLatch latch; + + @Test + public void testConnection() throws InterruptedException, IOException { + File f = new File("D:\\test\\abc.txt"); + RandomAccessFile file = new RandomAccessFile(f, "rw"); + + int count = 10; + latch = new CountDownLatch(count); + for (int i = 0; i < count; i++) { + Thread t = new Thread(new WritingThread(i, file.getChannel())); + t.start(); + } + latch.await(); + file.close(); + InputStream fileR = new FileInputStream(f); + List strings = IOUtils.readLines(fileR, Charset.defaultCharset()); + strings.forEach(line -> logger.info("{}", line)); + fileR.close(); + } + + class WritingThread implements Runnable { + + private FileChannel channel; + + private int id; + + public WritingThread(int id, FileChannel channel) { + this.channel = channel; + this.id = id; + } + + @Override + public void run() { + logger.info("Thread {} is Writing", id); + try { + for (int i = 1; i <= 5; i++) { + String msg = String.format("%s, %s, %s\n", Thread.currentThread().getName(), id, i); + this.channel.write(ByteBuffer.wrap(msg.getBytes())); + } + } catch (IOException e) { + logger.error(e.getMessage()); + } + latch.countDown(); + } + } + } \ No newline at end of file diff --git a/dbsyncer-connector/src/main/test/FileWatchTest.java b/dbsyncer-connector/dbsyncer-connector-file/src/test/java/FileWatchTest.java similarity index 100% rename from dbsyncer-connector/src/main/test/FileWatchTest.java rename to dbsyncer-connector/dbsyncer-connector-file/src/test/java/FileWatchTest.java diff --git a/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/MySQLException.java b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/MySQLException.java index 1eadda56..453399f4 100644 --- a/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/MySQLException.java +++ b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/MySQLException.java @@ -4,9 +4,9 @@ package org.dbsyncer.connector.mysql; /** - * @author AE86 - * @version 1.0.0 - * @date 2019/9/28 22:39 + * @Author AE86 + * @Version 1.0.0 + * @Date 2021-11-22 23:55 */ public class MySQLException extends RuntimeException { diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/pom.xml b/dbsyncer-connector/dbsyncer-connector-oracle/pom.xml new file mode 100644 index 00000000..ab0327ad --- /dev/null +++ b/dbsyncer-connector/dbsyncer-connector-oracle/pom.xml @@ -0,0 +1,47 @@ + + + + dbsyncer-connector + org.ghi + 2.0.0 + + 4.0.0 + + dbsyncer-connector-oracle + + + + + org.ghi + dbsyncer-sdk + ${project.parent.version} + provided + + + + + com.oracle + ojdbc6 + + + + com.github.jsqlparser + jsqlparser + + + + org.springframework.boot + spring-boot-starter-log4j2 + provided + + + + junit + junit + provided + + + + \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLOracleConnector.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/DQLOracleConnector.java similarity index 71% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLOracleConnector.java rename to dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/DQLOracleConnector.java index 9387496b..8722af45 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLOracleConnector.java +++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/DQLOracleConnector.java @@ -1,25 +1,41 @@ -package org.dbsyncer.connector.sql; +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.connector.oracle; -import org.dbsyncer.connector.oracle.DqlOracleListener; -import org.dbsyncer.sdk.listener.DatabaseQuartzListener; +import org.dbsyncer.connector.oracle.cdc.DqlOracleListener; +import org.dbsyncer.connector.oracle.validator.DqlOracleConfigValidator; import org.dbsyncer.sdk.config.ReaderConfig; +import org.dbsyncer.sdk.connector.ConfigValidator; import org.dbsyncer.sdk.connector.database.AbstractDQLConnector; import org.dbsyncer.sdk.constant.DatabaseConstant; import org.dbsyncer.sdk.enums.ListenerTypeEnum; +import org.dbsyncer.sdk.listener.DatabaseQuartzListener; import org.dbsyncer.sdk.listener.Listener; import org.dbsyncer.sdk.model.PageSql; -import org.springframework.stereotype.Component; -@Component +/** + * DQLOracle连接器实现 + * + * @Author AE86 + * @Version 1.0.0 + * @Date 2022-05-12 21:14 + */ public final class DQLOracleConnector extends AbstractDQLConnector { private final String TYPE = "DqlOracle"; + private final DqlOracleConfigValidator configValidator = new DqlOracleConfigValidator(); @Override public String getConnectorType() { return TYPE; } + @Override + public ConfigValidator getConfigValidator() { + return configValidator; + } + @Override public Listener getListener(String listenerType) { if (ListenerTypeEnum.isTiming(listenerType)) { diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/OracleConnector.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/OracleConnector.java similarity index 89% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/OracleConnector.java rename to dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/OracleConnector.java index 8a8ee131..92a414d5 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/OracleConnector.java +++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/OracleConnector.java @@ -1,32 +1,45 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.connector.oracle; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.sdk.listener.DatabaseQuartzListener; +import org.dbsyncer.connector.oracle.cdc.OracleListener; +import org.dbsyncer.connector.oracle.schema.OracleOtherValueMapper; +import org.dbsyncer.connector.oracle.validator.OracleConfigValidator; import org.dbsyncer.sdk.config.CommandConfig; import org.dbsyncer.sdk.config.DatabaseConfig; import org.dbsyncer.sdk.config.ReaderConfig; +import org.dbsyncer.sdk.connector.ConfigValidator; import org.dbsyncer.sdk.connector.database.AbstractDatabaseConnector; import org.dbsyncer.sdk.constant.DatabaseConstant; import org.dbsyncer.sdk.enums.ListenerTypeEnum; import org.dbsyncer.sdk.enums.TableTypeEnum; +import org.dbsyncer.sdk.listener.DatabaseQuartzListener; import org.dbsyncer.sdk.listener.Listener; import org.dbsyncer.sdk.model.PageSql; import org.dbsyncer.sdk.model.Table; import org.dbsyncer.sdk.util.PrimaryKeyUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Component; import javax.annotation.PostConstruct; import java.sql.Types; import java.util.List; -@Component +/** + * Oracle连接器实现 + * + * @Author AE86 + * @Version 1.0.0 + * @Date 2022-05-12 21:14 + */ public final class OracleConnector extends AbstractDatabaseConnector { private final Logger logger = LoggerFactory.getLogger(getClass()); private final String TYPE = "Oracle"; + private final OracleConfigValidator configValidator = new OracleConfigValidator(); @PostConstruct private void init() { @@ -38,6 +51,11 @@ public final class OracleConnector extends AbstractDatabaseConnector { return TYPE; } + @Override + public ConfigValidator getConfigValidator() { + return configValidator; + } + @Override public Listener getListener(String listenerType) { if (ListenerTypeEnum.isTiming(listenerType)) { diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/OracleException.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/OracleException.java new file mode 100644 index 00000000..f5cb492d --- /dev/null +++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/OracleException.java @@ -0,0 +1,30 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.connector.oracle; + +/** + * @Author AE86 + * @Version 1.0.0 + * @Date 2020-01-08 15:17 + */ +public class OracleException extends RuntimeException { + + private static final long serialVersionUID = 1L; + + public OracleException(String message) { + super(message); + } + + public OracleException(String message, Throwable cause) { + super(message, cause); + } + + public OracleException(Throwable cause) { + super(cause); + } + + protected OracleException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } +} diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/DqlOracleListener.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/DqlOracleListener.java similarity index 77% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/DqlOracleListener.java rename to dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/DqlOracleListener.java index 017732ac..df1ebdcf 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/DqlOracleListener.java +++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/DqlOracleListener.java @@ -1,4 +1,7 @@ -package org.dbsyncer.connector.oracle; +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.connector.oracle.cdc; import org.dbsyncer.sdk.listener.ChangedEvent; import org.dbsyncer.sdk.model.Field; @@ -6,9 +9,9 @@ import org.dbsyncer.sdk.model.Field; import java.util.List; /** - * @author AE86 - * @version 1.0.0 - * @date 2022/5/29 22:44 + * @Author AE86 + * @Version 1.0.0 + * @Date 2022-05-29 22:44 */ public class DqlOracleListener extends OracleListener { diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/OracleListener.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java similarity index 82% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/OracleListener.java rename to dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java index 89d363fd..7a4fbd92 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/OracleListener.java +++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java @@ -1,16 +1,19 @@ -package org.dbsyncer.connector.oracle; +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.connector.oracle.cdc; -import org.dbsyncer.sdk.listener.AbstractDatabaseListener; -import org.dbsyncer.connector.ConnectorException; +import org.dbsyncer.connector.oracle.OracleException; import org.dbsyncer.connector.oracle.dcn.DBChangeNotification; import org.dbsyncer.sdk.config.DatabaseConfig; +import org.dbsyncer.sdk.listener.AbstractDatabaseListener; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * @version 1.0.0 * @Author AE86 - * @Date 2020-05-12 21:14 + * @Version 1.0.0 + * @Date 2022-05-12 21:14 */ public class OracleListener extends AbstractDatabaseListener { @@ -31,7 +34,7 @@ public class OracleListener extends AbstractDatabaseListener { client.start(); } catch (Exception e) { logger.error("启动失败:{}", e.getMessage()); - throw new ConnectorException(e); + throw new OracleException(e); } } diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/dcn/DBChangeNotification.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/dcn/DBChangeNotification.java similarity index 97% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/dcn/DBChangeNotification.java rename to dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/dcn/DBChangeNotification.java index 6cf4a7d9..93661494 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/dcn/DBChangeNotification.java +++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/dcn/DBChangeNotification.java @@ -1,5 +1,5 @@ /** - * DBSyncer Copyright 2019-2024 All Rights Reserved. + * DBSyncer Copyright 2020-2023 All Rights Reserved. */ package org.dbsyncer.connector.oracle.dcn; @@ -12,8 +12,8 @@ import oracle.jdbc.dcn.RowChangeDescription; import oracle.jdbc.dcn.TableChangeDescription; import oracle.jdbc.driver.OracleConnection; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.ConnectorException; -import org.dbsyncer.connector.oracle.DCNEvent; +import org.dbsyncer.connector.oracle.OracleException; +import org.dbsyncer.connector.oracle.model.DCNEvent; import org.dbsyncer.sdk.constant.ConnectorConstant; import org.dbsyncer.sdk.listener.event.RowChangedEvent; import org.slf4j.Logger; @@ -39,9 +39,9 @@ import java.util.concurrent.locks.ReentrantLock; *

    *

    grant change notification to AE86 * - * @version 1.0.0 * @Author AE86 - * @Date 2020-06-08 21:53 + * @Version 1.0.0 + * @Date 2022-06-08 21:53 */ public class DBChangeNotification { @@ -248,7 +248,7 @@ public class DBChangeNotification { } catch (InvocationTargetException e) { logger.error(e.getMessage()); } - throw new ConnectorException(String.format("Can't invoke '%s'.", declaredMethod)); + throw new OracleException(String.format("Can't invoke '%s'.", declaredMethod)); } /** diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/dcn/RowEventListener.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/dcn/RowEventListener.java similarity index 71% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/dcn/RowEventListener.java rename to dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/dcn/RowEventListener.java index 34e3eae0..fbf816d5 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/dcn/RowEventListener.java +++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/dcn/RowEventListener.java @@ -1,5 +1,5 @@ /** - * DBSyncer Copyright 2019-2024 All Rights Reserved. + * DBSyncer Copyright 2020-2023 All Rights Reserved. */ package org.dbsyncer.connector.oracle.dcn; @@ -8,9 +8,9 @@ import org.dbsyncer.sdk.listener.event.RowChangedEvent; /** * 行变更监听器 * - * @version 1.0.0 * @Author AE86 - * @Date 2020-06-15 20:00 + * @Version 1.0.0 + * @Date 2022-06-08 21:53 */ public interface RowEventListener { diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/geometry/DataException.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/geometry/DataException.java similarity index 95% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/geometry/DataException.java rename to dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/geometry/DataException.java index ef6cbccf..678d14e0 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/geometry/DataException.java +++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/geometry/DataException.java @@ -1,3 +1,6 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.connector.oracle.geometry; import java.io.PrintStream; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/geometry/J3D_Geometry.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/geometry/J3D_Geometry.java similarity index 96% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/geometry/J3D_Geometry.java rename to dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/geometry/J3D_Geometry.java index e100d0db..0b4c61a1 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/geometry/J3D_Geometry.java +++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/geometry/J3D_Geometry.java @@ -1,3 +1,6 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.connector.oracle.geometry; public class J3D_Geometry extends JGeometry { diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/geometry/JGeometry.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/geometry/JGeometry.java similarity index 99% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/geometry/JGeometry.java rename to dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/geometry/JGeometry.java index 3a5d0f7b..d3cf0b91 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/geometry/JGeometry.java +++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/geometry/JGeometry.java @@ -1,3 +1,6 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.connector.oracle.geometry; import java.awt.Shape; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/geometry/JPoint2DD.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/geometry/JPoint2DD.java similarity index 94% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/geometry/JPoint2DD.java rename to dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/geometry/JPoint2DD.java index b4d2ff9a..da6e2a83 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/geometry/JPoint2DD.java +++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/geometry/JPoint2DD.java @@ -1,3 +1,6 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.connector.oracle.geometry; import java.io.Serializable; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/geometry/SdoPickler.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/geometry/SdoPickler.java similarity index 99% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/geometry/SdoPickler.java rename to dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/geometry/SdoPickler.java index 74163ad2..5f446d03 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/geometry/SdoPickler.java +++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/geometry/SdoPickler.java @@ -1,3 +1,6 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.connector.oracle.geometry; import java.sql.SQLException; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/DCNEvent.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/model/DCNEvent.java similarity index 82% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/DCNEvent.java rename to dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/model/DCNEvent.java index 3438696f..2e89a4a9 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/DCNEvent.java +++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/model/DCNEvent.java @@ -1,7 +1,7 @@ /** - * DBSyncer Copyright 2019-2024 All Rights Reserved. + * DBSyncer Copyright 2020-2023 All Rights Reserved. */ -package org.dbsyncer.connector.oracle; +package org.dbsyncer.connector.oracle.model; public final class DCNEvent { diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/OracleOtherValueMapper.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/schema/OracleOtherValueMapper.java similarity index 83% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/OracleOtherValueMapper.java rename to dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/schema/OracleOtherValueMapper.java index 8450be7e..231a4275 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/oracle/OracleOtherValueMapper.java +++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/schema/OracleOtherValueMapper.java @@ -1,8 +1,11 @@ -package org.dbsyncer.connector.oracle; +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.connector.oracle.schema; import com.microsoft.sqlserver.jdbc.Geometry; import oracle.jdbc.OracleConnection; -import org.dbsyncer.connector.ConnectorException; +import org.dbsyncer.connector.oracle.OracleException; import org.dbsyncer.connector.oracle.geometry.JGeometry; import org.dbsyncer.sdk.connector.AbstractValueMapper; import org.dbsyncer.sdk.connector.ConnectorInstance; @@ -18,9 +21,9 @@ import java.sql.Struct; *

  • VARCHAR2
    • * * - * @author AE86 - * @version 1.0.0 - * @date 2022/12/22 22:59 + * @Author AE86 + * @Version 1.0.0 + * @Date 2022-12-22 22:59 */ public final class OracleOtherValueMapper extends AbstractValueMapper { @@ -50,6 +53,6 @@ public final class OracleOtherValueMapper extends AbstractValueMapper { } } } - throw new ConnectorException(String.format("%s can not find type [%s], val [%s]", getClass().getSimpleName(), val.getClass(), val)); + throw new OracleException(String.format("%s can not find type [%s], val [%s]", getClass().getSimpleName(), val.getClass(), val)); } } \ No newline at end of file diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlOracleConfigValidator.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/validator/DqlOracleConfigValidator.java similarity index 87% rename from dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlOracleConfigValidator.java rename to dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/validator/DqlOracleConfigValidator.java index 72ceaf94..616c9471 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlOracleConfigValidator.java +++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/validator/DqlOracleConfigValidator.java @@ -1,12 +1,11 @@ /** * DBSyncer Copyright 2020-2023 All Rights Reserved. */ -package org.dbsyncer.biz.checker.impl.connector; +package org.dbsyncer.connector.oracle.validator; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.sdk.config.DatabaseConfig; import org.dbsyncer.sdk.connector.AbstractDataBaseConfigValidator; -import org.springframework.stereotype.Component; import java.util.Map; @@ -15,7 +14,6 @@ import java.util.Map; * @Version 1.0.0 * @Date 2020-01-08 15:17 */ -@Component public class DqlOracleConfigValidator extends AbstractDataBaseConfigValidator { @Override diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/OracleConfigValidator.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/validator/OracleConfigValidator.java similarity index 93% rename from dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/OracleConfigValidator.java rename to dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/validator/OracleConfigValidator.java index b8d823b3..aec0121c 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/OracleConfigValidator.java +++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/validator/OracleConfigValidator.java @@ -1,7 +1,7 @@ /** * DBSyncer Copyright 2020-2023 All Rights Reserved. */ -package org.dbsyncer.biz.checker.impl.connector; +package org.dbsyncer.connector.oracle.validator; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.sdk.config.DatabaseConfig; diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/resources/META-INF/services/org.dbsyncer.sdk.spi.ConnectorService b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/resources/META-INF/services/org.dbsyncer.sdk.spi.ConnectorService new file mode 100644 index 00000000..7693b7a2 --- /dev/null +++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/resources/META-INF/services/org.dbsyncer.sdk.spi.ConnectorService @@ -0,0 +1,2 @@ +org.dbsyncer.connector.oracle.OracleConnector +org.dbsyncer.connector.oracle.DQLOracleConnector \ No newline at end of file diff --git a/dbsyncer-connector/src/main/test/DBChangeNotificationTest.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/test/java/DBChangeNotificationTest.java similarity index 96% rename from dbsyncer-connector/src/main/test/DBChangeNotificationTest.java rename to dbsyncer-connector/dbsyncer-connector-oracle/src/test/java/DBChangeNotificationTest.java index b6595348..58dab872 100644 --- a/dbsyncer-connector/src/main/test/DBChangeNotificationTest.java +++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/test/java/DBChangeNotificationTest.java @@ -1,3 +1,6 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ import oracle.jdbc.OracleStatement; import oracle.jdbc.driver.OracleConnection; import org.dbsyncer.connector.oracle.dcn.DBChangeNotification; @@ -12,9 +15,9 @@ import java.util.List; import java.util.concurrent.*; /** - * @version 1.0.0 * @Author AE86 - * @Date 2021-05-10 22:25 + * @Version 1.0.0 + * @Date 2022-06-08 21:53 */ public class DBChangeNotificationTest { diff --git a/dbsyncer-connector/src/main/test/LinkedBlockingQueueTest.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/test/java/LinkedBlockingQueueTest.java similarity index 94% rename from dbsyncer-connector/src/main/test/LinkedBlockingQueueTest.java rename to dbsyncer-connector/dbsyncer-connector-oracle/src/test/java/LinkedBlockingQueueTest.java index c60d4e26..58d4ba9f 100644 --- a/dbsyncer-connector/src/main/test/LinkedBlockingQueueTest.java +++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/test/java/LinkedBlockingQueueTest.java @@ -1,6 +1,9 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ import oracle.jdbc.dcn.TableChangeDescription; import org.dbsyncer.common.util.RandomUtil; -import org.dbsyncer.connector.oracle.DCNEvent; +import org.dbsyncer.connector.oracle.model.DCNEvent; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -9,6 +12,11 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; +/** + * @Author AE86 + * @Version 1.0.0 + * @Date 2022-06-08 21:53 + */ public class LinkedBlockingQueueTest { private final Logger logger = LoggerFactory.getLogger(getClass()); diff --git a/dbsyncer-connector/pom.xml b/dbsyncer-connector/pom.xml index c6de6c71..c5d79727 100644 --- a/dbsyncer-connector/pom.xml +++ b/dbsyncer-connector/pom.xml @@ -15,6 +15,7 @@ dbsyncer-connector-file dbsyncer-connector-kafka dbsyncer-connector-mysql + dbsyncer-connector-oracle @@ -46,11 +47,18 @@ ${project.parent.version}     --> - + + + + + org.ghi + dbsyncer-connector-oracle + ${project.parent.version} @@ -59,12 +67,6 @@ spring-jdbc - - - com.oracle - ojdbc6 - - com.microsoft.sqlserver @@ -98,11 +100,6 @@ elasticsearch-rest-high-level-client - - com.github.jsqlparser - jsqlparser - - org.springframework.boot spring-boot-starter-log4j2 diff --git a/dbsyncer-connector/src/main/test/CgLibTest.java b/dbsyncer-connector/src/main/test/CgLibTest.java deleted file mode 100644 index 9ab0aa8a..00000000 --- a/dbsyncer-connector/src/main/test/CgLibTest.java +++ /dev/null @@ -1,31 +0,0 @@ -import org.dbsyncer.common.util.JsonUtil; -import org.junit.Assert; -import org.junit.Test; -import org.springframework.cglib.beans.BeanGenerator; - -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; - -/** - * @author AE86 - * @version 1.0.0 - * @date 2021/12/12 15:45 - */ -public class CgLibTest { - - @Test - public void testCgLib() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { - BeanGenerator beanGenerator = new BeanGenerator(); - beanGenerator.addProperty("value", String.class); - - Object myBean = beanGenerator.create(); - Method setter = myBean.getClass().getMethod("setValue", String.class); - setter.invoke(myBean, "Hello cglib"); - - Method getter = myBean.getClass().getMethod("getValue"); - Assert.assertEquals("Hello cglib", getter.invoke(myBean)); - - String json = JsonUtil.objToJson(myBean); - System.out.println(json); - } -} -- Gitee From 8121fc4fbbc56abd553989231c5576ce0d29f3e5 Mon Sep 17 00:00:00 2001 From: AE86 <836391306@qq.com> Date: Sun, 26 Nov 2023 23:09:34 +0800 Subject: [PATCH 33/98] add module dbsyncer-connector-sqlserver --- .../dbsyncer-connector-oracle/pom.xml | 5 - .../dbsyncer-connector-sqlserver/pom.xml | 42 ++ .../sqlserver}/DQLSqlServerConnector.java | 116 +++--- .../sqlserver/SqlServerConnector.java | 24 +- .../sqlserver/SqlServerException.java | 30 ++ .../sqlserver/cdc}/DqlSqlServerListener.java | 11 +- .../connector/sqlserver/cdc}/Lsn.java | 8 +- .../connector/sqlserver/cdc}/LsnPuller.java | 12 +- .../sqlserver/cdc}/SqlServerListener.java | 27 +- .../sqlserver}/enums/TableOperationEnum.java | 5 +- .../connector/sqlserver/model}/CDCEvent.java | 4 +- .../model}/SqlServerChangeTable.java | 5 +- .../DqlSqlServerConfigValidator.java | 4 +- .../validator}/SqlServerConfigValidator.java | 6 +- .../org.dbsyncer.sdk.spi.ConnectorService | 2 + .../src/test/java}/ChangeDataCaptureTest.java | 11 +- dbsyncer-connector/pom.xml | 16 +- .../postgresql/PostgreSQLConnector.java | 6 + .../connector/sql/DQLPostgreSQLConnector.java | 6 + .../src/main/test/ConnectionTest.java | 360 ------------------ dbsyncer-parser/pom.xml | 5 + .../database/AbstractDatabaseConnector.java | 6 - dbsyncer-storage/pom.xml | 6 + 23 files changed, 249 insertions(+), 468 deletions(-) create mode 100644 dbsyncer-connector/dbsyncer-connector-sqlserver/pom.xml rename dbsyncer-connector/{src/main/java/org/dbsyncer/connector/sql => dbsyncer-connector-sqlserver/src/main/java/org/dbsyncer/connector/sqlserver}/DQLSqlServerConnector.java (71%) rename dbsyncer-connector/{ => dbsyncer-connector-sqlserver}/src/main/java/org/dbsyncer/connector/sqlserver/SqlServerConnector.java (89%) create mode 100644 dbsyncer-connector/dbsyncer-connector-sqlserver/src/main/java/org/dbsyncer/connector/sqlserver/SqlServerException.java rename dbsyncer-connector/{src/main/java/org/dbsyncer/connector/sqlserver => dbsyncer-connector-sqlserver/src/main/java/org/dbsyncer/connector/sqlserver/cdc}/DqlSqlServerListener.java (68%) rename dbsyncer-connector/{src/main/java/org/dbsyncer/connector/sqlserver => dbsyncer-connector-sqlserver/src/main/java/org/dbsyncer/connector/sqlserver/cdc}/Lsn.java (96%) rename dbsyncer-connector/{src/main/java/org/dbsyncer/connector/sqlserver => dbsyncer-connector-sqlserver/src/main/java/org/dbsyncer/connector/sqlserver/cdc}/LsnPuller.java (93%) rename dbsyncer-connector/{src/main/java/org/dbsyncer/connector/sqlserver => dbsyncer-connector-sqlserver/src/main/java/org/dbsyncer/connector/sqlserver/cdc}/SqlServerListener.java (96%) rename dbsyncer-connector/{src/main/java/org/dbsyncer/connector => dbsyncer-connector-sqlserver/src/main/java/org/dbsyncer/connector/sqlserver}/enums/TableOperationEnum.java (88%) rename dbsyncer-connector/{src/main/java/org/dbsyncer/connector/sqlserver => dbsyncer-connector-sqlserver/src/main/java/org/dbsyncer/connector/sqlserver/model}/CDCEvent.java (82%) rename dbsyncer-connector/{src/main/java/org/dbsyncer/connector/sqlserver => dbsyncer-connector-sqlserver/src/main/java/org/dbsyncer/connector/sqlserver/model}/SqlServerChangeTable.java (92%) rename {dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector => dbsyncer-connector/dbsyncer-connector-sqlserver/src/main/java/org/dbsyncer/connector/sqlserver/validator}/DqlSqlServerConfigValidator.java (85%) rename {dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector => dbsyncer-connector/dbsyncer-connector-sqlserver/src/main/java/org/dbsyncer/connector/sqlserver/validator}/SqlServerConfigValidator.java (83%) create mode 100644 dbsyncer-connector/dbsyncer-connector-sqlserver/src/main/resources/META-INF/services/org.dbsyncer.sdk.spi.ConnectorService rename dbsyncer-connector/{src/main/test => dbsyncer-connector-sqlserver/src/test/java}/ChangeDataCaptureTest.java (98%) delete mode 100644 dbsyncer-connector/src/main/test/ConnectionTest.java diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/pom.xml b/dbsyncer-connector/dbsyncer-connector-oracle/pom.xml index ab0327ad..c8e10695 100644 --- a/dbsyncer-connector/dbsyncer-connector-oracle/pom.xml +++ b/dbsyncer-connector/dbsyncer-connector-oracle/pom.xml @@ -26,11 +26,6 @@ ojdbc6 - - com.github.jsqlparser - jsqlparser - - org.springframework.boot spring-boot-starter-log4j2 diff --git a/dbsyncer-connector/dbsyncer-connector-sqlserver/pom.xml b/dbsyncer-connector/dbsyncer-connector-sqlserver/pom.xml new file mode 100644 index 00000000..b3af0431 --- /dev/null +++ b/dbsyncer-connector/dbsyncer-connector-sqlserver/pom.xml @@ -0,0 +1,42 @@ + + + + dbsyncer-connector + org.ghi + 2.0.0 + + 4.0.0 + + dbsyncer-connector-sqlserver + + + + + org.ghi + dbsyncer-sdk + ${project.parent.version} + provided + + + + + com.microsoft.sqlserver + mssql-jdbc + + + + org.springframework.boot + spring-boot-starter-log4j2 + provided + + + + junit + junit + provided + + + + \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLSqlServerConnector.java b/dbsyncer-connector/dbsyncer-connector-sqlserver/src/main/java/org/dbsyncer/connector/sqlserver/DQLSqlServerConnector.java similarity index 71% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLSqlServerConnector.java rename to dbsyncer-connector/dbsyncer-connector-sqlserver/src/main/java/org/dbsyncer/connector/sqlserver/DQLSqlServerConnector.java index 7949e12b..dcbd2370 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLSqlServerConnector.java +++ b/dbsyncer-connector/dbsyncer-connector-sqlserver/src/main/java/org/dbsyncer/connector/sqlserver/DQLSqlServerConnector.java @@ -1,51 +1,67 @@ -package org.dbsyncer.connector.sql; - -import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.sdk.listener.DatabaseQuartzListener; -import org.dbsyncer.connector.sqlserver.DqlSqlServerListener; -import org.dbsyncer.sdk.config.ReaderConfig; -import org.dbsyncer.sdk.connector.database.AbstractDQLConnector; -import org.dbsyncer.sdk.constant.DatabaseConstant; -import org.dbsyncer.sdk.enums.ListenerTypeEnum; -import org.dbsyncer.sdk.listener.Listener; -import org.dbsyncer.sdk.model.PageSql; -import org.springframework.stereotype.Component; - -import java.util.List; - -@Component -public final class DQLSqlServerConnector extends AbstractDQLConnector { - - private final String TYPE = "DqlSqlServer"; - - @Override - public String getConnectorType() { - return TYPE; - } - - @Override - public Listener getListener(String listenerType) { - if (ListenerTypeEnum.isTiming(listenerType)) { - return new DatabaseQuartzListener(); - } - - if (ListenerTypeEnum.isLog(listenerType)) { - return new DqlSqlServerListener(); - } - return null; - } - - @Override - public String getPageSql(PageSql config) { - List primaryKeys = config.getPrimaryKeys(); - String orderBy = StringUtil.join(primaryKeys, ","); - return String.format(DatabaseConstant.SQLSERVER_PAGE_SQL, orderBy, config.getQuerySql()); - } - - @Override - public Object[] getPageArgs(ReaderConfig config) { - int pageSize = config.getPageSize(); - int pageIndex = config.getPageIndex(); - return new Object[]{(pageIndex - 1) * pageSize + 1, pageIndex * pageSize}; - } +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.connector.sqlserver; + +import org.dbsyncer.common.util.StringUtil; +import org.dbsyncer.connector.sqlserver.cdc.DqlSqlServerListener; +import org.dbsyncer.connector.sqlserver.validator.DqlSqlServerConfigValidator; +import org.dbsyncer.sdk.config.ReaderConfig; +import org.dbsyncer.sdk.connector.ConfigValidator; +import org.dbsyncer.sdk.connector.database.AbstractDQLConnector; +import org.dbsyncer.sdk.constant.DatabaseConstant; +import org.dbsyncer.sdk.enums.ListenerTypeEnum; +import org.dbsyncer.sdk.listener.DatabaseQuartzListener; +import org.dbsyncer.sdk.listener.Listener; +import org.dbsyncer.sdk.model.PageSql; + +import java.util.List; + +/** + * DQLSqlServer连接器实现 + * + * @Author AE86 + * @Version 1.0.0 + * @Date 2022-05-22 22:56 + */ +public final class DQLSqlServerConnector extends AbstractDQLConnector { + + private final String TYPE = "DqlSqlServer"; + private final DqlSqlServerConfigValidator configValidator = new DqlSqlServerConfigValidator(); + + @Override + public String getConnectorType() { + return TYPE; + } + + @Override + public ConfigValidator getConfigValidator() { + return null; + } + + @Override + public Listener getListener(String listenerType) { + if (ListenerTypeEnum.isTiming(listenerType)) { + return new DatabaseQuartzListener(); + } + + if (ListenerTypeEnum.isLog(listenerType)) { + return new DqlSqlServerListener(); + } + return null; + } + + @Override + public String getPageSql(PageSql config) { + List primaryKeys = config.getPrimaryKeys(); + String orderBy = StringUtil.join(primaryKeys, ","); + return String.format(DatabaseConstant.SQLSERVER_PAGE_SQL, orderBy, config.getQuerySql()); + } + + @Override + public Object[] getPageArgs(ReaderConfig config) { + int pageSize = config.getPageSize(); + int pageIndex = config.getPageIndex(); + return new Object[]{(pageIndex - 1) * pageSize + 1, pageIndex * pageSize}; + } } \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/SqlServerConnector.java b/dbsyncer-connector/dbsyncer-connector-sqlserver/src/main/java/org/dbsyncer/connector/sqlserver/SqlServerConnector.java similarity index 89% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/SqlServerConnector.java rename to dbsyncer-connector/dbsyncer-connector-sqlserver/src/main/java/org/dbsyncer/connector/sqlserver/SqlServerConnector.java index 56d335fe..1cb4c251 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/SqlServerConnector.java +++ b/dbsyncer-connector/dbsyncer-connector-sqlserver/src/main/java/org/dbsyncer/connector/sqlserver/SqlServerConnector.java @@ -1,40 +1,56 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.connector.sqlserver; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.sdk.listener.DatabaseQuartzListener; +import org.dbsyncer.connector.sqlserver.cdc.SqlServerListener; +import org.dbsyncer.connector.sqlserver.validator.SqlServerConfigValidator; import org.dbsyncer.sdk.config.CommandConfig; import org.dbsyncer.sdk.config.DatabaseConfig; import org.dbsyncer.sdk.config.ReaderConfig; +import org.dbsyncer.sdk.connector.ConfigValidator; import org.dbsyncer.sdk.connector.database.AbstractDatabaseConnector; import org.dbsyncer.sdk.connector.database.DatabaseConnectorInstance; import org.dbsyncer.sdk.constant.DatabaseConstant; import org.dbsyncer.sdk.enums.ListenerTypeEnum; import org.dbsyncer.sdk.enums.TableTypeEnum; +import org.dbsyncer.sdk.listener.DatabaseQuartzListener; import org.dbsyncer.sdk.listener.Listener; import org.dbsyncer.sdk.model.Field; import org.dbsyncer.sdk.model.PageSql; import org.dbsyncer.sdk.model.Table; -import org.springframework.stereotype.Component; import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; -@Component +/** + * SqlServer连接器实现 + * + * @Author AE86 + * @Version 1.0.0 + * @Date 2022-05-22 22:56 + */ public final class SqlServerConnector extends AbstractDatabaseConnector { private final String QUERY_VIEW = "select name from sysobjects where xtype in('v')"; - private final String QUERY_TABLE = "select name from sys.tables where schema_id = schema_id('%s') and is_ms_shipped = 0"; private final String TYPE = "SqlServer"; + private final SqlServerConfigValidator configValidator = new SqlServerConfigValidator(); @Override public String getConnectorType() { return TYPE; } + @Override + public ConfigValidator getConfigValidator() { + return configValidator; + } + @Override public List
    getTable(DatabaseConnectorInstance connectorInstance) { DatabaseConfig config = connectorInstance.getConfig(); diff --git a/dbsyncer-connector/dbsyncer-connector-sqlserver/src/main/java/org/dbsyncer/connector/sqlserver/SqlServerException.java b/dbsyncer-connector/dbsyncer-connector-sqlserver/src/main/java/org/dbsyncer/connector/sqlserver/SqlServerException.java new file mode 100644 index 00000000..426bf5d7 --- /dev/null +++ b/dbsyncer-connector/dbsyncer-connector-sqlserver/src/main/java/org/dbsyncer/connector/sqlserver/SqlServerException.java @@ -0,0 +1,30 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.connector.sqlserver; + +/** + * @Author AE86 + * @Version 1.0.0 + * @Date 2020-01-08 15:17 + */ +public class SqlServerException extends RuntimeException { + + private static final long serialVersionUID = 1L; + + public SqlServerException(String message) { + super(message); + } + + public SqlServerException(String message, Throwable cause) { + super(message, cause); + } + + public SqlServerException(Throwable cause) { + super(cause); + } + + protected SqlServerException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } +} diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/DqlSqlServerListener.java b/dbsyncer-connector/dbsyncer-connector-sqlserver/src/main/java/org/dbsyncer/connector/sqlserver/cdc/DqlSqlServerListener.java similarity index 68% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/DqlSqlServerListener.java rename to dbsyncer-connector/dbsyncer-connector-sqlserver/src/main/java/org/dbsyncer/connector/sqlserver/cdc/DqlSqlServerListener.java index db33209b..057be279 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/DqlSqlServerListener.java +++ b/dbsyncer-connector/dbsyncer-connector-sqlserver/src/main/java/org/dbsyncer/connector/sqlserver/cdc/DqlSqlServerListener.java @@ -1,11 +1,14 @@ -package org.dbsyncer.connector.sqlserver; +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.connector.sqlserver.cdc; import org.dbsyncer.sdk.listener.ChangedEvent; /** - * @author AE86 - * @version 1.0.0 - * @date 2022/5/22 22:56 + * @Author AE86 + * @Version 1.0.0 + * @Date 2022-05-22 22:56 */ public class DqlSqlServerListener extends SqlServerListener { diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/Lsn.java b/dbsyncer-connector/dbsyncer-connector-sqlserver/src/main/java/org/dbsyncer/connector/sqlserver/cdc/Lsn.java similarity index 96% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/Lsn.java rename to dbsyncer-connector/dbsyncer-connector-sqlserver/src/main/java/org/dbsyncer/connector/sqlserver/cdc/Lsn.java index 31fcb499..0080695b 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/Lsn.java +++ b/dbsyncer-connector/dbsyncer-connector-sqlserver/src/main/java/org/dbsyncer/connector/sqlserver/cdc/Lsn.java @@ -1,4 +1,7 @@ -package org.dbsyncer.connector.sqlserver; +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.connector.sqlserver.cdc; import java.util.Arrays; @@ -8,7 +11,8 @@ import org.dbsyncer.common.util.StringUtil; * SQL Server LSN（日志序列号）位置的逻辑表示, LSN不可用时为NULL。 * * @Author AE86 - * @Date 2021-06-04 22:25 + * @Version 1.0.0 + * @Date 2022-05-22 22:56 */ public class Lsn implements Comparable { private static final String NULL_STRING = "NULL"; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/LsnPuller.java b/dbsyncer-connector/dbsyncer-connector-sqlserver/src/main/java/org/dbsyncer/connector/sqlserver/cdc/LsnPuller.java similarity index 93% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/LsnPuller.java rename to dbsyncer-connector/dbsyncer-connector-sqlserver/src/main/java/org/dbsyncer/connector/sqlserver/cdc/LsnPuller.java index a807fc35..7b981713 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/LsnPuller.java +++ b/dbsyncer-connector/dbsyncer-connector-sqlserver/src/main/java/org/dbsyncer/connector/sqlserver/cdc/LsnPuller.java @@ -1,4 +1,7 @@ -package org.dbsyncer.connector.sqlserver; +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.connector.sqlserver.cdc; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -8,10 +11,9 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; /** - * @author Xinpeng.Fu - * @version V1.0 - * @description - * @date 2022/8/30 10:04 + * @Author Xinpeng.Fu + * @Version 1.0.0 + * @Date 2022-05-22 22:56 */ public class LsnPuller { private final Logger logger = LoggerFactory.getLogger(getClass()); diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/SqlServerListener.java b/dbsyncer-connector/dbsyncer-connector-sqlserver/src/main/java/org/dbsyncer/connector/sqlserver/cdc/SqlServerListener.java similarity index 96% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/SqlServerListener.java rename to dbsyncer-connector/dbsyncer-connector-sqlserver/src/main/java/org/dbsyncer/connector/sqlserver/cdc/SqlServerListener.java index 59b4ee36..198bcbee 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/SqlServerListener.java +++ b/dbsyncer-connector/dbsyncer-connector-sqlserver/src/main/java/org/dbsyncer/connector/sqlserver/cdc/SqlServerListener.java @@ -1,16 +1,21 @@ -package org.dbsyncer.connector.sqlserver; +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.connector.sqlserver.cdc; import com.microsoft.sqlserver.jdbc.SQLServerException; -import org.dbsyncer.connector.ConnectorException; -import org.dbsyncer.connector.enums.TableOperationEnum; -import org.dbsyncer.sdk.connector.database.AbstractDatabaseConnector; -import org.dbsyncer.sdk.model.ChangedOffset; -import org.dbsyncer.sdk.listener.event.RowChangedEvent; import org.dbsyncer.common.util.CollectionUtils; +import org.dbsyncer.connector.sqlserver.model.SqlServerChangeTable; +import org.dbsyncer.connector.sqlserver.SqlServerException; +import org.dbsyncer.connector.sqlserver.enums.TableOperationEnum; +import org.dbsyncer.connector.sqlserver.model.CDCEvent; import org.dbsyncer.sdk.config.DatabaseConfig; -import org.dbsyncer.sdk.constant.ConnectorConstant; +import org.dbsyncer.sdk.connector.database.AbstractDatabaseConnector; import org.dbsyncer.sdk.connector.database.DatabaseConnectorInstance; +import org.dbsyncer.sdk.constant.ConnectorConstant; import org.dbsyncer.sdk.listener.AbstractDatabaseListener; +import org.dbsyncer.sdk.listener.event.RowChangedEvent; +import org.dbsyncer.sdk.model.ChangedOffset; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.util.Assert; @@ -33,9 +38,9 @@ import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; /** - * @version 1.0.0 * @Author AE86 - * @Date 2021-06-18 01:20 + * @Version 1.0.0 + * @Date 2022-05-22 22:56 */ public class SqlServerListener extends AbstractDatabaseListener { @@ -100,7 +105,7 @@ public class SqlServerListener extends AbstractDatabaseListener { } catch (Exception e) { close(); logger.error("启动失败:{}", e.getMessage()); - throw new ConnectorException(e); + throw new SqlServerException(e); } finally { connectLock.unlock(); } @@ -154,7 +159,7 @@ public class SqlServerListener extends AbstractDatabaseListener { return; } // Shouldn't happen if the agent is running, but it is better to guard against such situation - throw new ConnectorException("No maximum LSN recorded in the database"); + throw new SqlServerException("No maximum LSN recorded in the database"); } lastLsn = Lsn.valueOf(snapshot.get(LSN_POSITION)); } diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/enums/TableOperationEnum.java b/dbsyncer-connector/dbsyncer-connector-sqlserver/src/main/java/org/dbsyncer/connector/sqlserver/enums/TableOperationEnum.java similarity index 88% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/enums/TableOperationEnum.java rename to dbsyncer-connector/dbsyncer-connector-sqlserver/src/main/java/org/dbsyncer/connector/sqlserver/enums/TableOperationEnum.java index af235645..929e3979 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/enums/TableOperationEnum.java +++ b/dbsyncer-connector/dbsyncer-connector-sqlserver/src/main/java/org/dbsyncer/connector/sqlserver/enums/TableOperationEnum.java @@ -1,4 +1,7 @@ -package org.dbsyncer.connector.enums; +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.connector.sqlserver.enums; public enum TableOperationEnum { diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/CDCEvent.java b/dbsyncer-connector/dbsyncer-connector-sqlserver/src/main/java/org/dbsyncer/connector/sqlserver/model/CDCEvent.java similarity index 82% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/CDCEvent.java rename to dbsyncer-connector/dbsyncer-connector-sqlserver/src/main/java/org/dbsyncer/connector/sqlserver/model/CDCEvent.java index 6491f4e0..509d4481 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/CDCEvent.java +++ b/dbsyncer-connector/dbsyncer-connector-sqlserver/src/main/java/org/dbsyncer/connector/sqlserver/model/CDCEvent.java @@ -1,7 +1,7 @@ /** - * DBSyncer Copyright 2019-2024 All Rights Reserved. + * DBSyncer Copyright 2020-2023 All Rights Reserved. */ -package org.dbsyncer.connector.sqlserver; +package org.dbsyncer.connector.sqlserver.model; import java.util.List; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/SqlServerChangeTable.java b/dbsyncer-connector/dbsyncer-connector-sqlserver/src/main/java/org/dbsyncer/connector/sqlserver/model/SqlServerChangeTable.java similarity index 92% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/SqlServerChangeTable.java rename to dbsyncer-connector/dbsyncer-connector-sqlserver/src/main/java/org/dbsyncer/connector/sqlserver/model/SqlServerChangeTable.java index fd29591e..a21b6c85 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sqlserver/SqlServerChangeTable.java +++ b/dbsyncer-connector/dbsyncer-connector-sqlserver/src/main/java/org/dbsyncer/connector/sqlserver/model/SqlServerChangeTable.java @@ -1,4 +1,7 @@ -package org.dbsyncer.connector.sqlserver; +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.connector.sqlserver.model; import org.dbsyncer.common.util.JsonUtil; diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlSqlServerConfigValidator.java b/dbsyncer-connector/dbsyncer-connector-sqlserver/src/main/java/org/dbsyncer/connector/sqlserver/validator/DqlSqlServerConfigValidator.java similarity index 85% rename from dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlSqlServerConfigValidator.java rename to dbsyncer-connector/dbsyncer-connector-sqlserver/src/main/java/org/dbsyncer/connector/sqlserver/validator/DqlSqlServerConfigValidator.java index c2ef80bf..9c492e0f 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/DqlSqlServerConfigValidator.java +++ b/dbsyncer-connector/dbsyncer-connector-sqlserver/src/main/java/org/dbsyncer/connector/sqlserver/validator/DqlSqlServerConfigValidator.java @@ -1,11 +1,10 @@ /** * DBSyncer Copyright 2020-2023 All Rights Reserved. */ -package org.dbsyncer.biz.checker.impl.connector; +package org.dbsyncer.connector.sqlserver.validator; import org.dbsyncer.sdk.config.DatabaseConfig; import org.dbsyncer.sdk.connector.AbstractDataBaseConfigValidator; -import org.springframework.stereotype.Component; import java.util.Map; @@ -16,7 +15,6 @@ import java.util.Map; * @Version 1.0.0 * @Date 2022-05-18 20:17 */ -@Component public class DqlSqlServerConfigValidator extends AbstractDataBaseConfigValidator { @Override diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/SqlServerConfigValidator.java b/dbsyncer-connector/dbsyncer-connector-sqlserver/src/main/java/org/dbsyncer/connector/sqlserver/validator/SqlServerConfigValidator.java similarity index 83% rename from dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/SqlServerConfigValidator.java rename to dbsyncer-connector/dbsyncer-connector-sqlserver/src/main/java/org/dbsyncer/connector/sqlserver/validator/SqlServerConfigValidator.java index f460c21d..4095228f 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/SqlServerConfigValidator.java +++ b/dbsyncer-connector/dbsyncer-connector-sqlserver/src/main/java/org/dbsyncer/connector/sqlserver/validator/SqlServerConfigValidator.java @@ -1,20 +1,20 @@ /** * DBSyncer Copyright 2020-2023 All Rights Reserved. */ -package org.dbsyncer.biz.checker.impl.connector; +package org.dbsyncer.connector.sqlserver.validator; import org.dbsyncer.sdk.config.DatabaseConfig; import org.dbsyncer.sdk.connector.AbstractDataBaseConfigValidator; -import org.springframework.stereotype.Component; import java.util.Map; /** + * SqlServer连接配置校验器实现 + * * @Author AE86 * @Version 1.0.0 * @Date 2023-11-25 23:10 */ -@Component public class SqlServerConfigValidator extends AbstractDataBaseConfigValidator { @Override public void modify(DatabaseConfig connectorConfig, Map params) { diff --git a/dbsyncer-connector/dbsyncer-connector-sqlserver/src/main/resources/META-INF/services/org.dbsyncer.sdk.spi.ConnectorService b/dbsyncer-connector/dbsyncer-connector-sqlserver/src/main/resources/META-INF/services/org.dbsyncer.sdk.spi.ConnectorService new file mode 100644 index 00000000..d24d17f0 --- /dev/null +++ b/dbsyncer-connector/dbsyncer-connector-sqlserver/src/main/resources/META-INF/services/org.dbsyncer.sdk.spi.ConnectorService @@ -0,0 +1,2 @@ +org.dbsyncer.connector.sqlserver.SqlServerConnector +org.dbsyncer.connector.sqlserver.DQLSqlServerConnector \ No newline at end of file diff --git a/dbsyncer-connector/src/main/test/ChangeDataCaptureTest.java b/dbsyncer-connector/dbsyncer-connector-sqlserver/src/test/java/ChangeDataCaptureTest.java similarity index 98% rename from dbsyncer-connector/src/main/test/ChangeDataCaptureTest.java rename to dbsyncer-connector/dbsyncer-connector-sqlserver/src/test/java/ChangeDataCaptureTest.java index c9853785..2198dedb 100644 --- a/dbsyncer-connector/src/main/test/ChangeDataCaptureTest.java +++ b/dbsyncer-connector/dbsyncer-connector-sqlserver/src/test/java/ChangeDataCaptureTest.java @@ -1,7 +1,10 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ import com.microsoft.sqlserver.jdbc.SQLServerException; import org.dbsyncer.common.util.CollectionUtils; -import org.dbsyncer.connector.sqlserver.Lsn; -import org.dbsyncer.connector.sqlserver.SqlServerChangeTable; +import org.dbsyncer.connector.sqlserver.model.SqlServerChangeTable; +import org.dbsyncer.connector.sqlserver.cdc.Lsn; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; @@ -14,9 +17,9 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.regex.Matcher; /** - * @version 1.0.0 * @Author AE86 - * @Date 2021-06-14 01:55 + * @Version 1.0.0 + * @Date 2022-05-22 22:56 * @Link https://www.red-gate.com/simple-talk/sql/learn-sql-server/introduction-to-change-data-capture-cdc-in-sql-server-2008/ */ public class ChangeDataCaptureTest { diff --git a/dbsyncer-connector/pom.xml b/dbsyncer-connector/pom.xml index c5d79727..a5ddbc65 100644 --- a/dbsyncer-connector/pom.xml +++ b/dbsyncer-connector/pom.xml @@ -16,6 +16,7 @@ dbsyncer-connector-kafka dbsyncer-connector-mysql dbsyncer-connector-oracle + dbsyncer-connector-sqlserver @@ -54,11 +55,18 @@ ${project.parent.version} --> - + + + + + org.ghi + dbsyncer-connector-sqlserver + ${project.parent.version} @@ -67,12 +75,6 @@ spring-jdbc - - - com.microsoft.sqlserver - mssql-jdbc - - org.antlr antlr4-runtime diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLConnector.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLConnector.java index fed43089..ebf954e8 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLConnector.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLConnector.java @@ -1,6 +1,7 @@ package org.dbsyncer.connector.postgresql; import org.dbsyncer.common.util.StringUtil; +import org.dbsyncer.sdk.connector.ConfigValidator; import org.dbsyncer.sdk.listener.DatabaseQuartzListener; import org.dbsyncer.sdk.config.CommandConfig; import org.dbsyncer.sdk.config.DatabaseConfig; @@ -33,6 +34,11 @@ public final class PostgreSQLConnector extends AbstractDatabaseConnector { return TYPE; } + @Override + public ConfigValidator getConfigValidator() { + return null; + } + @Override public Listener getListener(String listenerType) { if (ListenerTypeEnum.isTiming(listenerType)) { diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLPostgreSQLConnector.java b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLPostgreSQLConnector.java index 8e6547c0..c02727dd 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLPostgreSQLConnector.java +++ b/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLPostgreSQLConnector.java @@ -1,6 +1,7 @@ package org.dbsyncer.connector.sql; import org.dbsyncer.connector.postgresql.DqlPostgreSQLListener; +import org.dbsyncer.sdk.connector.ConfigValidator; import org.dbsyncer.sdk.listener.DatabaseQuartzListener; import org.dbsyncer.sdk.config.ReaderConfig; import org.dbsyncer.sdk.connector.database.AbstractDQLConnector; @@ -20,6 +21,11 @@ public final class DQLPostgreSQLConnector extends AbstractDQLConnector { return TYPE; } + @Override + public ConfigValidator getConfigValidator() { + return null; + } + @Override public Listener getListener(String listenerType) { if (ListenerTypeEnum.isTiming(listenerType)) { diff --git a/dbsyncer-connector/src/main/test/ConnectionTest.java b/dbsyncer-connector/src/main/test/ConnectionTest.java deleted file mode 100644 index 99ab8f9c..00000000 --- a/dbsyncer-connector/src/main/test/ConnectionTest.java +++ /dev/null @@ -1,360 +0,0 @@ -import oracle.jdbc.OracleConnection; -import org.dbsyncer.common.util.CollectionUtils; -import org.dbsyncer.common.util.RandomUtil; -import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.sdk.config.DatabaseConfig; -import org.dbsyncer.sdk.connector.database.DatabaseConnectorInstance; -import org.dbsyncer.sdk.connector.database.ds.SimpleConnection; -import org.dbsyncer.sdk.enums.TableTypeEnum; -import org.dbsyncer.sdk.model.Field; -import org.dbsyncer.sdk.model.Table; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.jdbc.core.BatchPreparedStatementSetter; - -import java.nio.charset.Charset; -import java.sql.*; -import java.time.Instant; -import java.time.LocalDateTime; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.*; - -/** - * @author AE86 - * @version 1.0.0 - * @date 2022/4/11 20:19 - */ -public class ConnectionTest { - - private final Logger logger = LoggerFactory.getLogger(getClass()); - - @Test - public void testByte() { - final DatabaseConnectorInstance connectorInstance = new DatabaseConnectorInstance(createOracleConfig()); - - String executeSql = "UPDATE \"my_user\" SET \"name\"=?,\"clo\"=? WHERE \"id\"=?"; - int[] execute = connectorInstance.execute(databaseTemplate -> - databaseTemplate.batchUpdate(executeSql, new BatchPreparedStatementSetter() { - @Override - public void setValues(PreparedStatement ps, int i) { - try { - SimpleConnection connection = databaseTemplate.getSimpleConnection(); - OracleConnection conn = (OracleConnection) connection.getConnection(); - Clob clob = conn.createClob(); - clob.setString(1, new String("中文888".getBytes(Charset.defaultCharset()))); - - ps.setString(1, "hello888"); - ps.setClob(2, clob); - ps.setInt(3, 2); - } catch (SQLException e) { - e.printStackTrace(); - } - } - - @Override - public int getBatchSize() { - return 1; - } - }) - ); - logger.info("execute:{}", execute); - } - - @Test - public void testConnection() throws InterruptedException { - final DatabaseConnectorInstance connectorInstance = new DatabaseConnectorInstance(createSqlServerConfig()); - - // 模拟并发 - final int threadSize = 100; - final ExecutorService pool = Executors.newFixedThreadPool(threadSize); - final CyclicBarrier barrier = new CyclicBarrier(threadSize); - final CountDownLatch latch = new CountDownLatch(threadSize); - for (int i = 0; i < threadSize; i++) { - final int k = i + 3; - pool.submit(() -> { - try { - barrier.await(); - - // 模拟操作 - System.out.println(String.format("%s %s:%s", LocalDateTime.now(), Thread.currentThread().getName(), k)); - - Object execute = connectorInstance.execute(tem -> tem.queryForObject("select 1", Integer.class)); - System.out.println(String.format("%s %s:%s execute=>%s", LocalDateTime.now(), Thread.currentThread().getName(), k, execute)); - - } catch (InterruptedException e) { - logger.error(e.getMessage()); - } catch (BrokenBarrierException e) { - logger.error(e.getMessage()); - } catch (Exception e) { - logger.error(e.getMessage()); - } finally { - latch.countDown(); - } - }); - } - - try { - latch.await(); - logger.info("try to shutdown"); - pool.shutdown(); - } catch (InterruptedException e) { - logger.error(e.getMessage()); - } - - TimeUnit.SECONDS.sleep(3); - logger.info("test end"); - } - - @Test - public void testBatchInsert() { - final DatabaseConnectorInstance connectorInstance = new DatabaseConnectorInstance(createMysqlConfig()); - - long begin = Instant.now().toEpochMilli(); - final int threadSize = 10; - final ExecutorService pool = Executors.newFixedThreadPool(threadSize); - final String sql = "INSERT INTO `vote_records_copy` (`id`, `user_id`, `vote_num`, `group_id`, `status`, `create_time`) VALUES (?, ?, ?, ?, ?, ?)"; - - // 模拟1000w条数据 - List dataList = new ArrayList<>(); - for (int i = 1; i <= 200001; i++) { - // 442001, 'dA8LeJLtX9MgQgDe7H1O', 9620, 1, 2, '2022-11-17 16:35:21' - Object[] args = new Object[6]; - args[0] = i; - args[1] = randomUserId(20); - args[2] = RandomUtil.nextInt(1, 9999); - args[3] = RandomUtil.nextInt(0, 3); - args[4] = RandomUtil.nextInt(1, 3); - args[5] = Timestamp.valueOf(LocalDateTime.now()); - dataList.add(args); - - if (i % 10000 == 0) { - System.out.println(i + "-----------------正在处理"); - batchUpdate(connectorInstance, pool, sql, dataList, 1000); - dataList.clear(); - } - } - - if(!CollectionUtils.isEmpty(dataList)){ - System.out.println("-----------------正在处理剩余数据"); - batchUpdate(connectorInstance, pool, sql, dataList, 1000); - } - - pool.shutdown(); - logger.info("总共耗时：{}秒", (Instant.now().toEpochMilli() - begin) / 1000); - } - - @Test - public void testBatchUpdate() { - final DatabaseConnectorInstance connectorInstance = new DatabaseConnectorInstance(createMysqlConfig()); - - long begin = Instant.now().toEpochMilli(); - final int threadSize = 10; - final ExecutorService pool = Executors.newFixedThreadPool(threadSize); - final String sql = "UPDATE `test`.`vote_records` SET `user_id` = ?, `create_time` = now() WHERE `id` = ?"; - - // 模拟100w条数据 - int k = 10; - while (k > 0) { - List dataList = new ArrayList<>(); - for (int i = 1; i <= 100000; i++) { - // 'dA8LeJLtX9MgQgDe7H1O', '2022-11-17 16:35:21', 1 - Object[] args = new Object[2]; - args[0] = randomUserId(20); - args[1] = i; - dataList.add(args); - - if (i % 10000 == 0) { - System.out.println(i + "-----------------正在处理"); - batchUpdate(connectorInstance, pool, sql, dataList, 1000); - dataList.clear(); - } - } - - if (!CollectionUtils.isEmpty(dataList)) { - System.out.println("-----------------正在处理剩余数据"); - batchUpdate(connectorInstance, pool, sql, dataList, 1000); - } - k--; - } - - pool.shutdown(); - logger.info("总共耗时：{}秒", (Instant.now().toEpochMilli() - begin) / 1000); - } - - @Test - public void testBatchDelete() { - final DatabaseConnectorInstance connectorInstance = new DatabaseConnectorInstance(createMysqlConfig()); - - long begin = Instant.now().toEpochMilli(); - final int threadSize = 10; - final ExecutorService pool = Executors.newFixedThreadPool(threadSize); - final String sql = "delete from `test`.`vote_records` WHERE `id` = ?"; - - List dataList = new ArrayList<>(); - for (int i = 1; i <= 3259000; i++) { - // 'dA8LeJLtX9MgQgDe7H1O', '2022-11-17 16:35:21', 1 - Object[] args = new Object[1]; - args[0] = i; - dataList.add(args); - - if (i % 10000 == 0) { - System.out.println(i + "-----------------正在处理"); - batchUpdate(connectorInstance, pool, sql, dataList, 1000); - dataList.clear(); - } - } - - if (!CollectionUtils.isEmpty(dataList)) { - System.out.println("-----------------正在处理剩余数据"); - batchUpdate(connectorInstance, pool, sql, dataList, 1000); - } - - pool.shutdown(); - logger.info("总共耗时：{}秒", (Instant.now().toEpochMilli() - begin) / 1000); - } - - private final static String STR = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; - - private String randomUserId(int i) { - StringBuilder s = new StringBuilder(); - for (int j = 0; j < i; j++) { - int r = RandomUtil.nextInt(0, 62); - s.append(StringUtil.substring(STR, r, r + 1)); - } - return s.toString(); - } - - private void batchUpdate(DatabaseConnectorInstance connectorInstance, ExecutorService pool, String sql, List dataList, int batchSize) { - int total = dataList.size(); - int taskSize = total % batchSize == 0 ? total / batchSize : total / batchSize + 1; - final CountDownLatch latch = new CountDownLatch(taskSize); - int fromIndex = 0; - int toIndex = batchSize; - for (int i = 0; i < taskSize; i++) { - final List data; - if (toIndex > total) { - toIndex = fromIndex + (total % batchSize); - data = dataList.subList(fromIndex, toIndex); - } else { - data = dataList.subList(fromIndex, toIndex); - fromIndex += batchSize; - toIndex += batchSize; - } - - pool.submit(() -> { - try { - connectorInstance.execute(databaseTemplate -> databaseTemplate.batchUpdate(sql, data)); - } catch (Exception e) { - logger.error(e.getMessage()); - } finally { - latch.countDown(); - } - }); - } - try { - latch.await(); - } catch (InterruptedException e) { - logger.error(e.getMessage()); - } - } - - @Test - public void testReadSchema() { - getTables(createOracleConfig(), "test", "AE86", "MY_ORG"); - getTables(createOracleConfig(), "test", "AE86", null); - - getTables(createMysqlConfig(), "test", "root", "MY_ORG"); - getTables(createMysqlConfig(), "test", "root", null); - - getTables(createSqlServerConfig(), "test", "dbo", "MY_ORG"); - getTables(createSqlServerConfig(), "test", "dbo", null); - - getTables(createPostgresConfig(), "postgres", "public", "MY_ORG"); - getTables(createPostgresConfig(), "postgres", "public", null); - } - - @Test - public void testGetColumnsDetails() { - final String schema = "root"; - final String tableNamePattern = "sw_test"; - final DatabaseConnectorInstance connectorInstance = new DatabaseConnectorInstance(createMysqlConfig()); - connectorInstance.execute(databaseTemplate -> { - SimpleConnection connection = databaseTemplate.getSimpleConnection(); - Connection conn = connection.getConnection(); - String databaseCatalog = conn.getCatalog(); - String schemaNamePattern = null == schema ? conn.getSchema() : schema; - List fields = new ArrayList<>(); - DatabaseMetaData metaData = conn.getMetaData(); - ResultSet columnMetadata = metaData.getColumns(databaseCatalog, schemaNamePattern, tableNamePattern, null); - while (columnMetadata.next()) { - String columnName = columnMetadata.getString(4); - int columnType = columnMetadata.getInt(5); - String typeName = columnMetadata.getString(6); - fields.add(new Field(columnName, typeName, columnType)); - } - return fields; - }); - } - - private List
    getTables(DatabaseConfig config, final String catalog, final String schema, final String tableNamePattern) { - final DatabaseConnectorInstance connectorInstance = new DatabaseConnectorInstance(config); - List
    tables = new ArrayList<>(); - connectorInstance.execute(databaseTemplate -> { - SimpleConnection connection = databaseTemplate.getSimpleConnection(); - Connection conn = connection.getConnection(); - String databaseCatalog = null == catalog ? conn.getCatalog() : catalog; - String schemaNamePattern = null == schema ? conn.getSchema() : schema; - String[] types = {TableTypeEnum.TABLE.getCode(), TableTypeEnum.VIEW.getCode(), TableTypeEnum.MATERIALIZED_VIEW.getCode()}; - final ResultSet rs = conn.getMetaData().getTables(databaseCatalog, schemaNamePattern, tableNamePattern, types); - while (rs.next()) { - final String tableName = rs.getString("TABLE_NAME"); - final String tableType = rs.getString("TABLE_TYPE"); - tables.add(new Table(tableName, tableType)); - } - return tables; - }); - - logger.info("\r 表总数{}", tables.size()); - tables.forEach(t -> logger.info("{} {}", t.getName(), t.getType())); - - return tables; - } - - private DatabaseConfig createSqlServerConfig() { - DatabaseConfig config = new DatabaseConfig(); - config.setUrl("jdbc:sqlserver://127.0.0.1:1433;DatabaseName=test"); - config.setUsername("sa"); - config.setPassword("123"); - config.setDriverClassName("com.microsoft.sqlserver.jdbc.SQLServerDriver"); - return config; - } - - private DatabaseConfig createOracleConfig() { - DatabaseConfig config = new DatabaseConfig(); - config.setUrl("jdbc:oracle:thin:@127.0.0.1:1521:ORCL"); - config.setUsername("ae86"); - config.setPassword("123"); - config.setDriverClassName("oracle.jdbc.OracleDriver"); - return config; - } - - private DatabaseConfig createMysqlConfig() { - DatabaseConfig config = new DatabaseConfig(); - config.setUrl("jdbc:mysql://127.0.0.1:3305/test?rewriteBatchedStatements=true&useUnicode=true&characterEncoding=UTF8&serverTimezone=Asia/Shanghai&useSSL=false&verifyServerCertificate=false&autoReconnect=true&failOverReadOnly=false"); - config.setUsername("root"); - config.setPassword("123"); - config.setDriverClassName("com.mysql.cj.jdbc.Driver"); - return config; - } - - private DatabaseConfig createPostgresConfig() { - DatabaseConfig config = new DatabaseConfig(); - config.setUrl("jdbc:postgresql://127.0.0.1:5432/postgres"); - config.setUsername("postgres"); - config.setPassword("123456"); - config.setDriverClassName("org.postgresql.Driver"); - return config; - } -} \ No newline at end of file diff --git a/dbsyncer-parser/pom.xml b/dbsyncer-parser/pom.xml index 8acf90b8..20dc5ece 100644 --- a/dbsyncer-parser/pom.xml +++ b/dbsyncer-parser/pom.xml @@ -25,6 +25,11 @@ ${project.parent.version} + + com.github.jsqlparser + jsqlparser + + junit junit diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/AbstractDatabaseConnector.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/AbstractDatabaseConnector.java index 2d5940a9..e0c086d7 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/AbstractDatabaseConnector.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/AbstractDatabaseConnector.java @@ -14,7 +14,6 @@ import org.dbsyncer.sdk.config.ReaderConfig; import org.dbsyncer.sdk.config.SqlBuilderConfig; import org.dbsyncer.sdk.config.WriterBatchConfig; import org.dbsyncer.sdk.connector.AbstractConnector; -import org.dbsyncer.sdk.connector.ConfigValidator; import org.dbsyncer.sdk.connector.ConnectorInstance; import org.dbsyncer.sdk.connector.database.ds.SimpleConnection; import org.dbsyncer.sdk.constant.ConnectorConstant; @@ -88,11 +87,6 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector implem return new DatabaseConnectorInstance(config); } - @Override - public ConfigValidator getConfigValidator() { - return null; - } - @Override public void disconnect(DatabaseConnectorInstance connectorInstance) { connectorInstance.close(); diff --git a/dbsyncer-storage/pom.xml b/dbsyncer-storage/pom.xml index a8a21263..ee45bb16 100644 --- a/dbsyncer-storage/pom.xml +++ b/dbsyncer-storage/pom.xml @@ -24,6 +24,12 @@ protobuf-java + + + com.oracle + ojdbc6 + + org.springframework.boot spring-boot-starter-log4j2 -- Gitee From 996a31788bab26df07d036fa3a087c3ecc981763 Mon Sep 17 00:00:00 2001 From: AE86 <836391306@qq.com> Date: Sun, 26 Nov 2023 23:38:58 +0800 Subject: [PATCH 34/98] add module dbsyncer-connector-postgresql --- .../impl/connector/ConnectorChecker.java | 24 ++++----- .../dbsyncer-connector-postgresql/pom.xml | 46 +++++++++++++++++ .../postgresql}/DQLPostgreSQLConnector.java | 24 ++++++--- .../postgresql/PostgreSQLConnector.java | 23 +++++++-- .../postgresql/PostgreSQLException.java | 30 +++++++++++ .../cdc}/DqlPostgreSQLListener.java | 11 ++-- .../postgresql/cdc}/PostgreSQLListener.java | 24 +++++---- .../postgresql/column/PgColumnValue.java | 23 +++++---- .../decoder}/AbstractMessageDecoder.java | 11 ++-- .../postgresql/decoder}/MessageDecoder.java | 11 ++-- .../decoder/impl}/PgOutputMessageDecoder.java | 19 ++++--- .../impl}/TestDecodingMessageDecoder.java | 13 +++-- .../postgresql/enums/MessageDecoderEnum.java | 19 ++++--- .../postgresql/enums/MessageTypeEnum.java | 3 ++ .../schema}/PostgreSQLBitValueMapper.java | 15 +++--- .../schema}/PostgreSQLOtherValueMapper.java | 15 +++--- .../DqlPostgreSQLConfigValidator.java | 4 +- .../validator}/PostgreSQLConfigValidator.java | 4 +- .../org.dbsyncer.sdk.spi.ConnectorService | 2 + .../src/test/java}/PGReplicationTest.java | 0 dbsyncer-connector/pom.xml | 50 +++---------------- dbsyncer-parser/pom.xml | 6 +++ dbsyncer-sdk/pom.xml | 1 - dbsyncer-storage/pom.xml | 12 +++++ 24 files changed, 248 insertions(+), 142 deletions(-) create mode 100644 dbsyncer-connector/dbsyncer-connector-postgresql/pom.xml rename dbsyncer-connector/{src/main/java/org/dbsyncer/connector/sql => dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql}/DQLPostgreSQLConnector.java (74%) rename dbsyncer-connector/{ => dbsyncer-connector-postgresql}/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLConnector.java (89%) create mode 100644 dbsyncer-connector/dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLException.java rename dbsyncer-connector/{src/main/java/org/dbsyncer/connector/postgresql => dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/cdc}/DqlPostgreSQLListener.java (68%) rename dbsyncer-connector/{src/main/java/org/dbsyncer/connector/postgresql => dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/cdc}/PostgreSQLListener.java (93%) rename dbsyncer-connector/{ => dbsyncer-connector-postgresql}/src/main/java/org/dbsyncer/connector/postgresql/column/PgColumnValue.java (92%) rename dbsyncer-connector/{src/main/java/org/dbsyncer/connector/postgresql => dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/decoder}/AbstractMessageDecoder.java (97%) rename dbsyncer-connector/{src/main/java/org/dbsyncer/connector/postgresql => dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/decoder}/MessageDecoder.java (84%) rename dbsyncer-connector/{src/main/java/org/dbsyncer/connector/postgresql/decoder => dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/decoder/impl}/PgOutputMessageDecoder.java (94%) rename dbsyncer-connector/{src/main/java/org/dbsyncer/connector/postgresql/decoder => dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/decoder/impl}/TestDecodingMessageDecoder.java (93%) rename dbsyncer-connector/{ => dbsyncer-connector-postgresql}/src/main/java/org/dbsyncer/connector/postgresql/enums/MessageDecoderEnum.java (67%) rename dbsyncer-connector/{ => dbsyncer-connector-postgresql}/src/main/java/org/dbsyncer/connector/postgresql/enums/MessageTypeEnum.java (93%) rename dbsyncer-connector/{src/main/java/org/dbsyncer/connector/postgresql => dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/schema}/PostgreSQLBitValueMapper.java (58%) rename dbsyncer-connector/{src/main/java/org/dbsyncer/connector/postgresql => dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/schema}/PostgreSQLOtherValueMapper.java (77%) rename {dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector => dbsyncer-connector/dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/validator}/DqlPostgreSQLConfigValidator.java (88%) rename {dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector => dbsyncer-connector/dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/validator}/PostgreSQLConfigValidator.java (88%) create mode 100644 dbsyncer-connector/dbsyncer-connector-postgresql/src/main/resources/META-INF/services/org.dbsyncer.sdk.spi.ConnectorService rename dbsyncer-connector/{src/main/test => dbsyncer-connector-postgresql/src/test/java}/PGReplicationTest.java (100%) diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/ConnectorChecker.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/ConnectorChecker.java index 6ce1a827..6790720f 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/ConnectorChecker.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/ConnectorChecker.java @@ -5,12 +5,11 @@ package org.dbsyncer.biz.checker.impl.connector; import org.dbsyncer.biz.BizException; import org.dbsyncer.biz.checker.AbstractChecker; -import org.dbsyncer.sdk.connector.ConfigValidator; -import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.connector.ConnectorFactory; import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.parser.model.ConfigModel; import org.dbsyncer.parser.model.Connector; +import org.dbsyncer.sdk.connector.ConfigValidator; import org.dbsyncer.sdk.connector.ConnectorInstance; import org.dbsyncer.sdk.model.ConnectorConfig; import org.dbsyncer.sdk.spi.ConnectorService; @@ -39,9 +38,6 @@ public class ConnectorChecker extends AbstractChecker { @Resource private ConnectorFactory connectorFactory; - @Resource - private Map map; - @Override public ConfigModel checkAddConfigModel(Map params) { printParams(params); @@ -55,11 +51,10 @@ public class ConnectorChecker extends AbstractChecker { ConnectorConfig config = getConfig(connectorType); connector.setConfig(config); - // 配置连接器配置 - String type = StringUtil.toLowerCaseFirstOne(connectorType).concat("ConfigChecker"); - ConfigValidator checker = map.get(type); - Assert.notNull(checker, "Checker can not be null."); - checker.modify(config, params); + // 连接器配置校验 + ConfigValidator configValidator = connectorFactory.getConnectorService(connectorType).getConfigValidator(); + Assert.notNull(configValidator, "ConfigValidator can not be null."); + configValidator.modify(config, params); // 获取表 ConnectorInstance connectorInstance = connectorFactory.connect(connector.getConfig()); @@ -84,11 +79,10 @@ public class ConnectorChecker extends AbstractChecker { // 修改基本配置 this.modifyConfigModel(connector, params); - // 配置连接器配置 - String type = StringUtil.toLowerCaseFirstOne(config.getConnectorType()).concat("ConfigChecker"); - ConfigValidator checker = map.get(type); - Assert.notNull(checker, "Checker can not be null."); - checker.modify(config, params); + // 连接器配置校验 + ConfigValidator configValidator = connectorFactory.getConnectorService(config.getConnectorType()).getConfigValidator(); + Assert.notNull(configValidator, "ConfigValidator can not be null."); + configValidator.modify(config, params); // 获取表 ConnectorInstance connectorInstance = connectorFactory.connect(config); diff --git a/dbsyncer-connector/dbsyncer-connector-postgresql/pom.xml b/dbsyncer-connector/dbsyncer-connector-postgresql/pom.xml new file mode 100644 index 00000000..deeba571 --- /dev/null +++ b/dbsyncer-connector/dbsyncer-connector-postgresql/pom.xml @@ -0,0 +1,46 @@ + + + + dbsyncer-connector + org.ghi + 2.0.0 + + 4.0.0 + + dbsyncer-connector-postgresql + + + + + org.ghi + dbsyncer-sdk + ${project.parent.version} + provided + + + + + org.postgresql + postgresql + + + net.postgis + postgis-jdbc + + + + org.springframework.boot + spring-boot-starter-log4j2 + provided + + + + junit + junit + provided + + + + \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLPostgreSQLConnector.java b/dbsyncer-connector/dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/DQLPostgreSQLConnector.java similarity index 74% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLPostgreSQLConnector.java rename to dbsyncer-connector/dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/DQLPostgreSQLConnector.java index c02727dd..bd180c3b 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/sql/DQLPostgreSQLConnector.java +++ b/dbsyncer-connector/dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/DQLPostgreSQLConnector.java @@ -1,20 +1,30 @@ -package org.dbsyncer.connector.sql; +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.connector.postgresql; -import org.dbsyncer.connector.postgresql.DqlPostgreSQLListener; -import org.dbsyncer.sdk.connector.ConfigValidator; -import org.dbsyncer.sdk.listener.DatabaseQuartzListener; +import org.dbsyncer.connector.postgresql.cdc.DqlPostgreSQLListener; +import org.dbsyncer.connector.postgresql.validator.DqlPostgreSQLConfigValidator; import org.dbsyncer.sdk.config.ReaderConfig; +import org.dbsyncer.sdk.connector.ConfigValidator; import org.dbsyncer.sdk.connector.database.AbstractDQLConnector; import org.dbsyncer.sdk.constant.DatabaseConstant; import org.dbsyncer.sdk.enums.ListenerTypeEnum; +import org.dbsyncer.sdk.listener.DatabaseQuartzListener; import org.dbsyncer.sdk.listener.Listener; import org.dbsyncer.sdk.model.PageSql; -import org.springframework.stereotype.Component; -@Component +/** + * DQLSqlServer连接器实现 + * + * @Author AE86 + * @Version 1.0.0 + * @Date 2022-05-22 22:56 + */ public final class DQLPostgreSQLConnector extends AbstractDQLConnector { private final String TYPE = "DqlPostgreSQL"; + private final DqlPostgreSQLConfigValidator configValidator = new DqlPostgreSQLConfigValidator(); @Override public String getConnectorType() { @@ -23,7 +33,7 @@ public final class DQLPostgreSQLConnector extends AbstractDQLConnector { @Override public ConfigValidator getConfigValidator() { - return null; + return configValidator; } @Override diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLConnector.java b/dbsyncer-connector/dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLConnector.java similarity index 89% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLConnector.java rename to dbsyncer-connector/dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLConnector.java index ebf954e8..de222a49 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLConnector.java +++ b/dbsyncer-connector/dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLConnector.java @@ -1,33 +1,46 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.connector.postgresql; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.sdk.connector.ConfigValidator; -import org.dbsyncer.sdk.listener.DatabaseQuartzListener; +import org.dbsyncer.connector.postgresql.cdc.PostgreSQLListener; +import org.dbsyncer.connector.postgresql.schema.PostgreSQLBitValueMapper; +import org.dbsyncer.connector.postgresql.schema.PostgreSQLOtherValueMapper; +import org.dbsyncer.connector.postgresql.validator.PostgreSQLConfigValidator; import org.dbsyncer.sdk.config.CommandConfig; import org.dbsyncer.sdk.config.DatabaseConfig; import org.dbsyncer.sdk.config.ReaderConfig; +import org.dbsyncer.sdk.connector.ConfigValidator; import org.dbsyncer.sdk.connector.database.AbstractDatabaseConnector; import org.dbsyncer.sdk.constant.DatabaseConstant; import org.dbsyncer.sdk.enums.ListenerTypeEnum; import org.dbsyncer.sdk.enums.TableTypeEnum; +import org.dbsyncer.sdk.listener.DatabaseQuartzListener; import org.dbsyncer.sdk.listener.Listener; import org.dbsyncer.sdk.model.PageSql; import org.dbsyncer.sdk.model.Table; import org.dbsyncer.sdk.util.PrimaryKeyUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Component; import javax.annotation.PostConstruct; import java.sql.Types; import java.util.List; -@Component +/** + * SqlServer连接器实现 + * + * @Author AE86 + * @Version 1.0.0 + * @Date 2022-05-22 22:56 + */ public final class PostgreSQLConnector extends AbstractDatabaseConnector { private final Logger logger = LoggerFactory.getLogger(getClass()); private final String TYPE = "PostgreSQL"; + private final PostgreSQLConfigValidator configValidator = new PostgreSQLConfigValidator(); @Override public String getConnectorType() { @@ -36,7 +49,7 @@ public final class PostgreSQLConnector extends AbstractDatabaseConnector { @Override public ConfigValidator getConfigValidator() { - return null; + return configValidator; } @Override diff --git a/dbsyncer-connector/dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLException.java b/dbsyncer-connector/dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLException.java new file mode 100644 index 00000000..213893e6 --- /dev/null +++ b/dbsyncer-connector/dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLException.java @@ -0,0 +1,30 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.connector.postgresql; + +/** + * @Author AE86 + * @Version 1.0.0 + * @Date 2022-05-18 20:17 + */ +public class PostgreSQLException extends RuntimeException { + + private static final long serialVersionUID = 1L; + + public PostgreSQLException(String message) { + super(message); + } + + public PostgreSQLException(String message, Throwable cause) { + super(message, cause); + } + + public PostgreSQLException(Throwable cause) { + super(cause); + } + + protected PostgreSQLException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } +} diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/DqlPostgreSQLListener.java b/dbsyncer-connector/dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/cdc/DqlPostgreSQLListener.java similarity index 68% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/DqlPostgreSQLListener.java rename to dbsyncer-connector/dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/cdc/DqlPostgreSQLListener.java index c95d208f..48fde42e 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/DqlPostgreSQLListener.java +++ b/dbsyncer-connector/dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/cdc/DqlPostgreSQLListener.java @@ -1,11 +1,14 @@ -package org.dbsyncer.connector.postgresql; +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.connector.postgresql.cdc; import org.dbsyncer.sdk.listener.ChangedEvent; /** - * @author AE86 - * @version 1.0.0 - * @date 2022/5/29 22:44 + * @Author AE86 + * @Version 1.0.0 + * @Date 2022-05-18 20:17 */ public class DqlPostgreSQLListener extends PostgreSQLListener { diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLListener.java b/dbsyncer-connector/dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/cdc/PostgreSQLListener.java similarity index 93% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLListener.java rename to dbsyncer-connector/dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/cdc/PostgreSQLListener.java index c07dff95..471d1ff7 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLListener.java +++ b/dbsyncer-connector/dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/cdc/PostgreSQLListener.java @@ -1,8 +1,12 @@ -package org.dbsyncer.connector.postgresql; +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.connector.postgresql.cdc; import org.dbsyncer.common.util.BooleanUtil; +import org.dbsyncer.connector.postgresql.decoder.MessageDecoder; +import org.dbsyncer.connector.postgresql.PostgreSQLException; import org.dbsyncer.sdk.listener.AbstractDatabaseListener; -import org.dbsyncer.connector.ConnectorException; import org.dbsyncer.connector.postgresql.enums.MessageDecoderEnum; import org.dbsyncer.sdk.config.DatabaseConfig; import org.dbsyncer.sdk.connector.database.DatabaseConnectorInstance; @@ -32,9 +36,9 @@ import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; /** - * @author AE86 - * @version 1.0.0 - * @date 2022/4/10 22:36 + * @Author AE86 + * @Version 1.0.0 + * @Date 2022-04-10 22:36 */ public class PostgreSQLListener extends AbstractDatabaseListener { @@ -75,7 +79,7 @@ public class PostgreSQLListener extends AbstractDatabaseListener { final String walLevel = instance.execute(databaseTemplate -> databaseTemplate.queryForObject(GET_WAL_LEVEL, String.class)); if (!DEFAULT_WAL_LEVEL.equals(walLevel)) { - throw new ConnectorException(String.format("Postgres server wal_level property must be \"%s\" but is: %s", DEFAULT_WAL_LEVEL, walLevel)); + throw new PostgreSQLException(String.format("Postgres server wal_level property must be \"%s\" but is: %s", DEFAULT_WAL_LEVEL, walLevel)); } final boolean hasAuth = instance.execute(databaseTemplate -> { @@ -88,7 +92,7 @@ public class PostgreSQLListener extends AbstractDatabaseListener { return login && (replication || superuser || admin || repAdmin); }); if (!hasAuth) { - throw new ConnectorException(String.format("Postgres roles LOGIN and REPLICATION are not assigned to user: %s", config.getUsername())); + throw new PostgreSQLException(String.format("Postgres roles LOGIN and REPLICATION are not assigned to user: %s", config.getUsername())); } database = instance.execute(databaseTemplate -> databaseTemplate.queryForObject(GET_DATABASE, String.class)); @@ -109,7 +113,7 @@ public class PostgreSQLListener extends AbstractDatabaseListener { logger.error("启动失败:{}", e.getMessage()); DatabaseUtil.close(stream); DatabaseUtil.close(connection); - throw new ConnectorException(e); + throw new PostgreSQLException(e); } finally { connectLock.unlock(); } @@ -186,7 +190,7 @@ public class PostgreSQLListener extends AbstractDatabaseListener { if (!snapshot.containsKey(LSN_POSITION)) { LogSequenceNumber lsn = instance.execute(databaseTemplate -> LogSequenceNumber.valueOf(databaseTemplate.queryForObject(GET_RESTART_LSN, new Object[]{database, slotName, plugin}, String.class))); if (null == lsn || lsn.asLong() == 0) { - throw new ConnectorException("No maximum LSN recorded in the database"); + throw new PostgreSQLException("No maximum LSN recorded in the database"); } snapshot.put(LSN_POSITION, lsn.asString()); super.forceFlushEvent(); @@ -289,7 +293,7 @@ public class PostgreSQLListener extends AbstractDatabaseListener { stream.setAppliedLSN(lsn); stream.setFlushedLSN(lsn); stream.forceUpdateStatus(); - } catch (IllegalStateException | ConnectorException e) { + } catch (IllegalStateException | PostgreSQLException e) { logger.error(e.getMessage()); } catch (Exception e) { logger.error(e.getMessage(), e); diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/column/PgColumnValue.java b/dbsyncer-connector/dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/column/PgColumnValue.java similarity index 92% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/column/PgColumnValue.java rename to dbsyncer-connector/dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/column/PgColumnValue.java index e32522a6..cec2273e 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/column/PgColumnValue.java +++ b/dbsyncer-connector/dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/column/PgColumnValue.java @@ -1,9 +1,12 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.connector.postgresql.column; import org.dbsyncer.common.column.AbstractColumnValue; import org.dbsyncer.common.util.DateFormatUtil; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.ConnectorException; +import org.dbsyncer.connector.postgresql.PostgreSQLException; import org.postgresql.PGStatement; import org.postgresql.geometric.PGbox; import org.postgresql.geometric.PGcircle; @@ -126,7 +129,7 @@ public final class PgColumnValue extends AbstractColumnValue { return new PGbox(asString()); } catch (final SQLException e) { logger.error("Failed to parse point {}, {}", asString(), e); - throw new ConnectorException(e); + throw new PostgreSQLException(e); } } @@ -135,7 +138,7 @@ public final class PgColumnValue extends AbstractColumnValue { return new PGcircle(asString()); } catch (final SQLException e) { logger.error("Failed to parse circle {}, {}", asString(), e); - throw new ConnectorException(e); + throw new PostgreSQLException(e); } } @@ -144,7 +147,7 @@ public final class PgColumnValue extends AbstractColumnValue { return new PGInterval(asString()); } catch (final SQLException e) { logger.error("Failed to parse point {}, {}", asString(), e); - throw new ConnectorException(e); + throw new PostgreSQLException(e); } } @@ -153,7 +156,7 @@ public final class PgColumnValue extends AbstractColumnValue { return new PGline(asString()); } catch (final SQLException e) { logger.error("Failed to parse point {}, {}", asString(), e); - throw new ConnectorException(e); + throw new PostgreSQLException(e); } } @@ -162,7 +165,7 @@ public final class PgColumnValue extends AbstractColumnValue { return new PGlseg(asString()); } catch (final SQLException e) { logger.error("Failed to parse point {}, {}", asString(), e); - throw new ConnectorException(e); + throw new PostgreSQLException(e); } } @@ -176,7 +179,7 @@ public final class PgColumnValue extends AbstractColumnValue { return new PGmoney(asString()); } catch (final SQLException e) { logger.error("Failed to parse money {}, {}", asString(), e); - throw new ConnectorException(e); + throw new PostgreSQLException(e); } } @@ -185,7 +188,7 @@ public final class PgColumnValue extends AbstractColumnValue { return new PGpath(asString()); } catch (final SQLException e) { logger.error("Failed to parse point {}, {}", asString(), e); - throw new ConnectorException(e); + throw new PostgreSQLException(e); } } @@ -194,7 +197,7 @@ public final class PgColumnValue extends AbstractColumnValue { return new PGpoint(asString()); } catch (final SQLException e) { logger.error("Failed to parse point {}, {}", asString(), e); - throw new ConnectorException(e); + throw new PostgreSQLException(e); } } @@ -203,7 +206,7 @@ public final class PgColumnValue extends AbstractColumnValue { return new PGpolygon(asString()); } catch (final SQLException e) { logger.error("Failed to parse point {}, {}", asString(), e); - throw new ConnectorException(e); + throw new PostgreSQLException(e); } } diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/AbstractMessageDecoder.java b/dbsyncer-connector/dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/decoder/AbstractMessageDecoder.java similarity index 97% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/AbstractMessageDecoder.java rename to dbsyncer-connector/dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/decoder/AbstractMessageDecoder.java index a6549d66..0737427f 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/AbstractMessageDecoder.java +++ b/dbsyncer-connector/dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/decoder/AbstractMessageDecoder.java @@ -1,4 +1,7 @@ -package org.dbsyncer.connector.postgresql; +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.connector.postgresql.decoder; import org.dbsyncer.sdk.config.DatabaseConfig; import org.dbsyncer.connector.postgresql.column.PgColumnValue; @@ -9,9 +12,9 @@ import org.postgresql.util.PGmoney; import java.nio.ByteBuffer; /** - * @author AE86 - * @version 1.0.0 - * @date 2022/4/17 23:04 + * @Author AE86 + * @Version 1.0.0 + * @Date 2022-04-10 22:36 */ public abstract class AbstractMessageDecoder implements MessageDecoder { diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/MessageDecoder.java b/dbsyncer-connector/dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/decoder/MessageDecoder.java similarity index 84% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/MessageDecoder.java rename to dbsyncer-connector/dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/decoder/MessageDecoder.java index 4ceddaf5..ebab469c 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/MessageDecoder.java +++ b/dbsyncer-connector/dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/decoder/MessageDecoder.java @@ -1,4 +1,7 @@ -package org.dbsyncer.connector.postgresql; +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.connector.postgresql.decoder; import org.dbsyncer.sdk.config.DatabaseConfig; import org.dbsyncer.sdk.connector.database.DatabaseConnectorInstance; @@ -10,9 +13,9 @@ import org.postgresql.replication.fluent.logical.ChainedLogicalStreamBuilder; import java.nio.ByteBuffer; /** - * @author AE86 - * @version 1.0.0 - * @date 2022/4/17 22:59 + * @Author AE86 + * @Version 1.0.0 + * @Date 2022-04-10 22:36 */ public interface MessageDecoder { diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/decoder/PgOutputMessageDecoder.java b/dbsyncer-connector/dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/decoder/impl/PgOutputMessageDecoder.java similarity index 94% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/decoder/PgOutputMessageDecoder.java rename to dbsyncer-connector/dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/decoder/impl/PgOutputMessageDecoder.java index 1cd19eae..fa90a205 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/decoder/PgOutputMessageDecoder.java +++ b/dbsyncer-connector/dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/decoder/impl/PgOutputMessageDecoder.java @@ -1,8 +1,11 @@ -package org.dbsyncer.connector.postgresql.decoder; +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.connector.postgresql.decoder.impl; import org.dbsyncer.common.util.CollectionUtils; -import org.dbsyncer.connector.ConnectorException; -import org.dbsyncer.connector.postgresql.AbstractMessageDecoder; +import org.dbsyncer.connector.postgresql.PostgreSQLException; +import org.dbsyncer.connector.postgresql.decoder.AbstractMessageDecoder; import org.dbsyncer.connector.postgresql.enums.MessageDecoderEnum; import org.dbsyncer.connector.postgresql.enums.MessageTypeEnum; import org.dbsyncer.sdk.connector.database.DatabaseConnectorInstance; @@ -23,9 +26,9 @@ import java.util.List; import java.util.Map; /** - * @author AE86 - * @version 1.0.0 - * @date 2022/4/17 23:00 + * @Author AE86 + * @Version 1.0.0 + * @Date 2022-04-10 22:36 */ public class PgOutputMessageDecoder extends AbstractMessageDecoder { @@ -112,7 +115,7 @@ public class PgOutputMessageDecoder extends AbstractMessageDecoder { return true; }); } catch (Exception e) { - throw new ConnectorException(e.getCause()); + throw new PostgreSQLException(e.getCause()); } } @@ -158,7 +161,7 @@ public class PgOutputMessageDecoder extends AbstractMessageDecoder { // The table schema has been changed, we should be get a new table schema from db. MetaInfo metaInfo = connectorService.getMetaInfo(connectorInstance, tableId.tableName); if (CollectionUtils.isEmpty(metaInfo.getColumn())) { - throw new ConnectorException(String.format("The table column for '%s' is empty.", tableId.tableName)); + throw new PostgreSQLException(String.format("The table column for '%s' is empty.", tableId.tableName)); } tableId.fields = metaInfo.getColumn(); return; diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/decoder/TestDecodingMessageDecoder.java b/dbsyncer-connector/dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/decoder/impl/TestDecodingMessageDecoder.java similarity index 93% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/decoder/TestDecodingMessageDecoder.java rename to dbsyncer-connector/dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/decoder/impl/TestDecodingMessageDecoder.java index 97d704a0..3c35b43e 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/decoder/TestDecodingMessageDecoder.java +++ b/dbsyncer-connector/dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/decoder/impl/TestDecodingMessageDecoder.java @@ -1,7 +1,10 @@ -package org.dbsyncer.connector.postgresql.decoder; +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.connector.postgresql.decoder.impl; import org.dbsyncer.common.column.Lexer; -import org.dbsyncer.connector.postgresql.AbstractMessageDecoder; +import org.dbsyncer.connector.postgresql.decoder.AbstractMessageDecoder; import org.dbsyncer.connector.postgresql.enums.MessageDecoderEnum; import org.dbsyncer.connector.postgresql.enums.MessageTypeEnum; import org.dbsyncer.sdk.constant.ConnectorConstant; @@ -15,9 +18,9 @@ import java.util.ArrayList; import java.util.List; /** - * @author AE86 - * @version 1.0.0 - * @date 2022/4/17 23:00 + * @Author AE86 + * @Version 1.0.0 + * @Date 2022-04-10 22:36 */ public class TestDecodingMessageDecoder extends AbstractMessageDecoder { diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/enums/MessageDecoderEnum.java b/dbsyncer-connector/dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/enums/MessageDecoderEnum.java similarity index 67% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/enums/MessageDecoderEnum.java rename to dbsyncer-connector/dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/enums/MessageDecoderEnum.java index 288c0bb5..84ad9bea 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/enums/MessageDecoderEnum.java +++ b/dbsyncer-connector/dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/enums/MessageDecoderEnum.java @@ -1,15 +1,18 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.connector.postgresql.enums; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.ConnectorException; -import org.dbsyncer.connector.postgresql.MessageDecoder; -import org.dbsyncer.connector.postgresql.decoder.PgOutputMessageDecoder; -import org.dbsyncer.connector.postgresql.decoder.TestDecodingMessageDecoder; +import org.dbsyncer.connector.postgresql.decoder.MessageDecoder; +import org.dbsyncer.connector.postgresql.PostgreSQLException; +import org.dbsyncer.connector.postgresql.decoder.impl.PgOutputMessageDecoder; +import org.dbsyncer.connector.postgresql.decoder.impl.TestDecodingMessageDecoder; /** - * @author AE86 - * @version 1.0.0 - * @date 2022/4/17 23:05 + * @Author AE86 + * @Version 1.0.0 + * @Date 2022-04-10 22:36 */ public enum MessageDecoderEnum { @@ -31,7 +34,7 @@ public enum MessageDecoderEnum { this.clazz = clazz; } - public static MessageDecoder getMessageDecoder(String type) throws ConnectorException, IllegalAccessException, InstantiationException { + public static MessageDecoder getMessageDecoder(String type) throws PostgreSQLException, IllegalAccessException, InstantiationException { for (MessageDecoderEnum e : MessageDecoderEnum.values()) { if (StringUtil.equals(type, e.getType())) { return (MessageDecoder) e.getClazz().newInstance(); diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/enums/MessageTypeEnum.java b/dbsyncer-connector/dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/enums/MessageTypeEnum.java similarity index 93% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/enums/MessageTypeEnum.java rename to dbsyncer-connector/dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/enums/MessageTypeEnum.java index b8a660a4..705be0f2 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/enums/MessageTypeEnum.java +++ b/dbsyncer-connector/dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/enums/MessageTypeEnum.java @@ -1,3 +1,6 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.connector.postgresql.enums; public enum MessageTypeEnum { diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLBitValueMapper.java b/dbsyncer-connector/dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/schema/PostgreSQLBitValueMapper.java similarity index 58% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLBitValueMapper.java rename to dbsyncer-connector/dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/schema/PostgreSQLBitValueMapper.java index e5e17632..a65467b9 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLBitValueMapper.java +++ b/dbsyncer-connector/dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/schema/PostgreSQLBitValueMapper.java @@ -1,13 +1,16 @@ -package org.dbsyncer.connector.postgresql; +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.connector.postgresql.schema; -import org.dbsyncer.connector.ConnectorException; +import org.dbsyncer.connector.postgresql.PostgreSQLException; import org.dbsyncer.sdk.connector.AbstractValueMapper; import org.dbsyncer.sdk.connector.ConnectorInstance; /** - * @author AE86 - * @version 1.0.0 - * @date 2022/8/25 0:07 + * @Author AE86 + * @Version 1.0.0 + * @Date 2022-04-10 22:36 */ public class PostgreSQLBitValueMapper extends AbstractValueMapper { @@ -22,7 +25,7 @@ public class PostgreSQLBitValueMapper extends AbstractValueMapper { return s == 1; } - throw new ConnectorException(String.format("%s can not find type [%s], val [%s]", getClass().getSimpleName(), val.getClass(), val)); + throw new PostgreSQLException(String.format("%s can not find type [%s], val [%s]", getClass().getSimpleName(), val.getClass(), val)); } } \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLOtherValueMapper.java b/dbsyncer-connector/dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/schema/PostgreSQLOtherValueMapper.java similarity index 77% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLOtherValueMapper.java rename to dbsyncer-connector/dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/schema/PostgreSQLOtherValueMapper.java index a808e37e..57694dce 100644 --- a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLOtherValueMapper.java +++ b/dbsyncer-connector/dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/schema/PostgreSQLOtherValueMapper.java @@ -1,6 +1,9 @@ -package org.dbsyncer.connector.postgresql; +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.connector.postgresql.schema; -import org.dbsyncer.connector.ConnectorException; +import org.dbsyncer.connector.postgresql.PostgreSQLException; import org.dbsyncer.sdk.connector.AbstractValueMapper; import org.dbsyncer.sdk.connector.ConnectorInstance; import org.postgis.Geometry; @@ -26,9 +29,9 @@ import org.postgresql.geometric.PGpoint; *
  • varbit
    • * * - * @author AE86 - * @version 1.0.0 - * @date 2022/12/22 22:59 + * @Author AE86 + * @Version 1.0.0 + * @Date 2022-12-22 22:59 */ public final class PostgreSQLOtherValueMapper extends AbstractValueMapper { @@ -45,6 +48,6 @@ public final class PostgreSQLOtherValueMapper extends AbstractValueMapper params) { diff --git a/dbsyncer-connector/dbsyncer-connector-postgresql/src/main/resources/META-INF/services/org.dbsyncer.sdk.spi.ConnectorService b/dbsyncer-connector/dbsyncer-connector-postgresql/src/main/resources/META-INF/services/org.dbsyncer.sdk.spi.ConnectorService new file mode 100644 index 00000000..09b208d3 --- /dev/null +++ b/dbsyncer-connector/dbsyncer-connector-postgresql/src/main/resources/META-INF/services/org.dbsyncer.sdk.spi.ConnectorService @@ -0,0 +1,2 @@ +org.dbsyncer.connector.postgresql.PostgreSQLConnector +org.dbsyncer.connector.postgresql.DQLPostgreSQLConnector \ No newline at end of file diff --git a/dbsyncer-connector/src/main/test/PGReplicationTest.java b/dbsyncer-connector/dbsyncer-connector-postgresql/src/test/java/PGReplicationTest.java similarity index 100% rename from dbsyncer-connector/src/main/test/PGReplicationTest.java rename to dbsyncer-connector/dbsyncer-connector-postgresql/src/test/java/PGReplicationTest.java diff --git a/dbsyncer-connector/pom.xml b/dbsyncer-connector/pom.xml index a5ddbc65..544aa766 100644 --- a/dbsyncer-connector/pom.xml +++ b/dbsyncer-connector/pom.xml @@ -17,6 +17,7 @@ dbsyncer-connector-mysql dbsyncer-connector-oracle dbsyncer-connector-sqlserver + dbsyncer-connector-postgresql @@ -62,55 +63,18 @@ ${project.parent.version}     --> - + - - org.springframework - spring-jdbc - - - - org.antlr - antlr4-runtime - - - - - org.postgresql - postgresql - - - net.postgis - postgis-jdbc - - - - - org.apache.lucene - lucene-analyzers-smartcn - - - - - org.elasticsearch.client - elasticsearch-rest-high-level-client - - - - org.springframework.boot - spring-boot-starter-log4j2 - + --> + - junit - junit - provided + org.ghi + dbsyncer-connector-postgresql + ${project.parent.version} diff --git a/dbsyncer-parser/pom.xml b/dbsyncer-parser/pom.xml index 20dc5ece..96118c03 100644 --- a/dbsyncer-parser/pom.xml +++ b/dbsyncer-parser/pom.xml @@ -30,6 +30,12 @@ jsqlparser + + + org.antlr + antlr4-runtime + + junit junit diff --git a/dbsyncer-sdk/pom.xml b/dbsyncer-sdk/pom.xml index d6fc5968..78020b93 100644 --- a/dbsyncer-sdk/pom.xml +++ b/dbsyncer-sdk/pom.xml @@ -22,7 +22,6 @@ org.springframework spring-jdbc - provided diff --git a/dbsyncer-storage/pom.xml b/dbsyncer-storage/pom.xml index ee45bb16..c2372900 100644 --- a/dbsyncer-storage/pom.xml +++ b/dbsyncer-storage/pom.xml @@ -30,6 +30,18 @@ ojdbc6 + + + org.apache.lucene + lucene-analyzers-smartcn + + + + + org.elasticsearch.client + elasticsearch-rest-high-level-client + + org.springframework.boot spring-boot-starter-log4j2 -- Gitee From 3de5e00b41e59aa616a177de0699c91c4d5159d6 Mon Sep 17 00:00:00 2001 From: AE86 <836391306@qq.com> Date: Sun, 26 Nov 2023 23:56:10 +0800 Subject: [PATCH 35/98] add module dbsyncer-connector-base --- .../org/dbsyncer/biz/enums/MetricEnum.java | 3 +- .../dbsyncer-connector-base/pom.xml | 71 +++++++++++++++++++ .../connector/ConnectorConfiguration.java | 0 .../connector/ConnectorException.java | 0 .../dbsyncer/connector/ConnectorFactory.java | 0 .../dbsyncer-connector-elasticsearch/pom.xml | 7 ++ .../dbsyncer-connector-mysql/pom.xml | 6 ++ .../dbsyncer-connector-oracle/pom.xml | 7 ++ dbsyncer-connector/pom.xml | 61 +--------------- dbsyncer-storage/pom.xml | 2 +- 10 files changed, 94 insertions(+), 63 deletions(-) create mode 100644 dbsyncer-connector/dbsyncer-connector-base/pom.xml rename dbsyncer-connector/{ => dbsyncer-connector-base}/src/main/java/org/dbsyncer/connector/ConnectorConfiguration.java (100%) rename dbsyncer-connector/{ => dbsyncer-connector-base}/src/main/java/org/dbsyncer/connector/ConnectorException.java (100%) rename dbsyncer-connector/{ => dbsyncer-connector-base}/src/main/java/org/dbsyncer/connector/ConnectorFactory.java (100%) diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/enums/MetricEnum.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/enums/MetricEnum.java index 22042d55..58fa8071 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/enums/MetricEnum.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/enums/MetricEnum.java @@ -2,7 +2,6 @@ package org.dbsyncer.biz.enums; import org.dbsyncer.biz.BizException; import org.dbsyncer.common.util.StringUtil; -import org.dbsyncer.connector.ConnectorException; /** * 系统指标 @@ -58,7 +57,7 @@ public enum MetricEnum { this.metricName = metricName; } - public static MetricEnum getMetric(String code) throws ConnectorException { + public static MetricEnum getMetric(String code) throws BizException { for (MetricEnum e : MetricEnum.values()) { if (StringUtil.equals(code, e.getCode())) { return e; diff --git a/dbsyncer-connector/dbsyncer-connector-base/pom.xml b/dbsyncer-connector/dbsyncer-connector-base/pom.xml new file mode 100644 index 00000000..1e64c761 --- /dev/null +++ b/dbsyncer-connector/dbsyncer-connector-base/pom.xml @@ -0,0 +1,71 @@ + + + + dbsyncer-connector + org.ghi + 2.0.0 + + 4.0.0 + + dbsyncer-connector-base + + + + + org.ghi + dbsyncer-sdk + ${project.parent.version} + + + + + org.ghi + dbsyncer-connector-elasticsearch + ${project.parent.version} + + + + + org.ghi + dbsyncer-connector-file + ${project.parent.version} + + + + + org.ghi + dbsyncer-connector-kafka + ${project.parent.version} + + + + + org.ghi + dbsyncer-connector-mysql + ${project.parent.version} + + + + + org.ghi + dbsyncer-connector-oracle + ${project.parent.version} + + + + + org.ghi + dbsyncer-connector-sqlserver + ${project.parent.version} + + + + + org.ghi + dbsyncer-connector-postgresql + ${project.parent.version} + + + \ No newline at end of file diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/ConnectorConfiguration.java b/dbsyncer-connector/dbsyncer-connector-base/src/main/java/org/dbsyncer/connector/ConnectorConfiguration.java similarity index 100% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/ConnectorConfiguration.java rename to dbsyncer-connector/dbsyncer-connector-base/src/main/java/org/dbsyncer/connector/ConnectorConfiguration.java diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/ConnectorException.java b/dbsyncer-connector/dbsyncer-connector-base/src/main/java/org/dbsyncer/connector/ConnectorException.java similarity index 100% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/ConnectorException.java rename to dbsyncer-connector/dbsyncer-connector-base/src/main/java/org/dbsyncer/connector/ConnectorException.java diff --git a/dbsyncer-connector/src/main/java/org/dbsyncer/connector/ConnectorFactory.java b/dbsyncer-connector/dbsyncer-connector-base/src/main/java/org/dbsyncer/connector/ConnectorFactory.java similarity index 100% rename from dbsyncer-connector/src/main/java/org/dbsyncer/connector/ConnectorFactory.java rename to dbsyncer-connector/dbsyncer-connector-base/src/main/java/org/dbsyncer/connector/ConnectorFactory.java diff --git a/dbsyncer-connector/dbsyncer-connector-elasticsearch/pom.xml b/dbsyncer-connector/dbsyncer-connector-elasticsearch/pom.xml index 90a0e958..40574412 100644 --- a/dbsyncer-connector/dbsyncer-connector-elasticsearch/pom.xml +++ b/dbsyncer-connector/dbsyncer-connector-elasticsearch/pom.xml @@ -26,6 +26,13 @@ elasticsearch-rest-high-level-client + + + org.postgresql + postgresql + provided + + org.springframework.boot spring-boot-starter-log4j2 diff --git a/dbsyncer-connector/dbsyncer-connector-mysql/pom.xml b/dbsyncer-connector/dbsyncer-connector-mysql/pom.xml index d7deffbb..db89afce 100644 --- a/dbsyncer-connector/dbsyncer-connector-mysql/pom.xml +++ b/dbsyncer-connector/dbsyncer-connector-mysql/pom.xml @@ -32,6 +32,12 @@ mysql-binlog-connector-java + + com.github.jsqlparser + jsqlparser + provided + + org.springframework.boot spring-boot-starter-log4j2 diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/pom.xml b/dbsyncer-connector/dbsyncer-connector-oracle/pom.xml index c8e10695..c0bcafe8 100644 --- a/dbsyncer-connector/dbsyncer-connector-oracle/pom.xml +++ b/dbsyncer-connector/dbsyncer-connector-oracle/pom.xml @@ -26,6 +26,13 @@ ojdbc6 + + + com.microsoft.sqlserver + mssql-jdbc + provided + + org.springframework.boot spring-boot-starter-log4j2 diff --git a/dbsyncer-connector/pom.xml b/dbsyncer-connector/pom.xml index 544aa766..acb9e4ee 100644 --- a/dbsyncer-connector/pom.xml +++ b/dbsyncer-connector/pom.xml @@ -11,6 +11,7 @@ dbsyncer-connector pom + dbsyncer-connector-base dbsyncer-connector-elasticsearch dbsyncer-connector-file dbsyncer-connector-kafka @@ -19,64 +20,4 @@ dbsyncer-connector-sqlserver dbsyncer-connector-postgresql - - - - - org.ghi - dbsyncer-sdk - ${project.parent.version} - - - - - - - - - - - - - - - - - org.ghi - dbsyncer-connector-postgresql - ${project.parent.version} - - - - \ No newline at end of file diff --git a/dbsyncer-storage/pom.xml b/dbsyncer-storage/pom.xml index c2372900..1b713703 100644 --- a/dbsyncer-storage/pom.xml +++ b/dbsyncer-storage/pom.xml @@ -14,7 +14,7 @@ org.ghi - dbsyncer-connector + dbsyncer-connector-base ${project.parent.version} -- Gitee From 894533cd84f18e6b1017f7a61a3820dccc0645fe Mon Sep 17 00:00:00 2001 From: AE86 <836391306@qq.com> Date: Sun, 26 Nov 2023 23:58:38 +0800 Subject: [PATCH 36/98] add test --- .../dbsyncer-connector-base/pom.xml | 6 + .../src/test/java/ConnectionTest.java | 363 ++++++++++++++++++ 2 files changed, 369 insertions(+) create mode 100644 dbsyncer-connector/dbsyncer-connector-base/src/test/java/ConnectionTest.java diff --git a/dbsyncer-connector/dbsyncer-connector-base/pom.xml b/dbsyncer-connector/dbsyncer-connector-base/pom.xml index 1e64c761..43df45ec 100644 --- a/dbsyncer-connector/dbsyncer-connector-base/pom.xml +++ b/dbsyncer-connector/dbsyncer-connector-base/pom.xml @@ -67,5 +67,11 @@ dbsyncer-connector-postgresql ${project.parent.version} + + + junit + junit + provided + \ No newline at end of file diff --git a/dbsyncer-connector/dbsyncer-connector-base/src/test/java/ConnectionTest.java b/dbsyncer-connector/dbsyncer-connector-base/src/test/java/ConnectionTest.java new file mode 100644 index 00000000..8f614e17 --- /dev/null +++ b/dbsyncer-connector/dbsyncer-connector-base/src/test/java/ConnectionTest.java @@ -0,0 +1,363 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +import oracle.jdbc.OracleConnection; +import org.dbsyncer.common.util.CollectionUtils; +import org.dbsyncer.common.util.RandomUtil; +import org.dbsyncer.common.util.StringUtil; +import org.dbsyncer.sdk.config.DatabaseConfig; +import org.dbsyncer.sdk.connector.database.DatabaseConnectorInstance; +import org.dbsyncer.sdk.connector.database.ds.SimpleConnection; +import org.dbsyncer.sdk.enums.TableTypeEnum; +import org.dbsyncer.sdk.model.Field; +import org.dbsyncer.sdk.model.Table; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.jdbc.core.BatchPreparedStatementSetter; + +import java.nio.charset.Charset; +import java.sql.*; +import java.time.Instant; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.*; + +/** + * @Author AE86 + * @Version 1.0.0 + * @Date 2022-04-11 20:19 + */ +public class ConnectionTest { + + private final Logger logger = LoggerFactory.getLogger(getClass()); + + @Test + public void testByte() { + final DatabaseConnectorInstance connectorInstance = new DatabaseConnectorInstance(createOracleConfig()); + + String executeSql = "UPDATE \"my_user\" SET \"name\"=?,\"clo\"=? WHERE \"id\"=?"; + int[] execute = connectorInstance.execute(databaseTemplate -> + databaseTemplate.batchUpdate(executeSql, new BatchPreparedStatementSetter() { + @Override + public void setValues(PreparedStatement ps, int i) { + try { + SimpleConnection connection = databaseTemplate.getSimpleConnection(); + OracleConnection conn = (OracleConnection) connection.getConnection(); + Clob clob = conn.createClob(); + clob.setString(1, new String("中文888".getBytes(Charset.defaultCharset()))); + + ps.setString(1, "hello888"); + ps.setClob(2, clob); + ps.setInt(3, 2); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Override + public int getBatchSize() { + return 1; + } + }) + ); + logger.info("execute:{}", execute); + } + + @Test + public void testConnection() throws InterruptedException { + final DatabaseConnectorInstance connectorInstance = new DatabaseConnectorInstance(createSqlServerConfig()); + + // 模拟并发 + final int threadSize = 100; + final ExecutorService pool = Executors.newFixedThreadPool(threadSize); + final CyclicBarrier barrier = new CyclicBarrier(threadSize); + final CountDownLatch latch = new CountDownLatch(threadSize); + for (int i = 0; i < threadSize; i++) { + final int k = i + 3; + pool.submit(() -> { + try { + barrier.await(); + + // 模拟操作 + System.out.println(String.format("%s %s:%s", LocalDateTime.now(), Thread.currentThread().getName(), k)); + + Object execute = connectorInstance.execute(tem -> tem.queryForObject("select 1", Integer.class)); + System.out.println(String.format("%s %s:%s execute=>%s", LocalDateTime.now(), Thread.currentThread().getName(), k, execute)); + + } catch (InterruptedException e) { + logger.error(e.getMessage()); + } catch (BrokenBarrierException e) { + logger.error(e.getMessage()); + } catch (Exception e) { + logger.error(e.getMessage()); + } finally { + latch.countDown(); + } + }); + } + + try { + latch.await(); + logger.info("try to shutdown"); + pool.shutdown(); + } catch (InterruptedException e) { + logger.error(e.getMessage()); + } + + TimeUnit.SECONDS.sleep(3); + logger.info("test end"); + } + + @Test + public void testBatchInsert() { + final DatabaseConnectorInstance connectorInstance = new DatabaseConnectorInstance(createMysqlConfig()); + + long begin = Instant.now().toEpochMilli(); + final int threadSize = 10; + final ExecutorService pool = Executors.newFixedThreadPool(threadSize); + final String sql = "INSERT INTO `vote_records_copy` (`id`, `user_id`, `vote_num`, `group_id`, `status`, `create_time`) VALUES (?, ?, ?, ?, ?, ?)"; + + // 模拟1000w条数据 + List dataList = new ArrayList<>(); + for (int i = 1; i <= 200001; i++) { + // 442001, 'dA8LeJLtX9MgQgDe7H1O', 9620, 1, 2, '2022-11-17 16:35:21' + Object[] args = new Object[6]; + args[0] = i; + args[1] = randomUserId(20); + args[2] = RandomUtil.nextInt(1, 9999); + args[3] = RandomUtil.nextInt(0, 3); + args[4] = RandomUtil.nextInt(1, 3); + args[5] = Timestamp.valueOf(LocalDateTime.now()); + dataList.add(args); + + if (i % 10000 == 0) { + System.out.println(i + "-----------------正在处理"); + batchUpdate(connectorInstance, pool, sql, dataList, 1000); + dataList.clear(); + } + } + + if(!CollectionUtils.isEmpty(dataList)){ + System.out.println("-----------------正在处理剩余数据"); + batchUpdate(connectorInstance, pool, sql, dataList, 1000); + } + + pool.shutdown(); + logger.info("总共耗时：{}秒", (Instant.now().toEpochMilli() - begin) / 1000); + } + + @Test + public void testBatchUpdate() { + final DatabaseConnectorInstance connectorInstance = new DatabaseConnectorInstance(createMysqlConfig()); + + long begin = Instant.now().toEpochMilli(); + final int threadSize = 10; + final ExecutorService pool = Executors.newFixedThreadPool(threadSize); + final String sql = "UPDATE `test`.`vote_records` SET `user_id` = ?, `create_time` = now() WHERE `id` = ?"; + + // 模拟100w条数据 + int k = 10; + while (k > 0) { + List dataList = new ArrayList<>(); + for (int i = 1; i <= 100000; i++) { + // 'dA8LeJLtX9MgQgDe7H1O', '2022-11-17 16:35:21', 1 + Object[] args = new Object[2]; + args[0] = randomUserId(20); + args[1] = i; + dataList.add(args); + + if (i % 10000 == 0) { + System.out.println(i + "-----------------正在处理"); + batchUpdate(connectorInstance, pool, sql, dataList, 1000); + dataList.clear(); + } + } + + if (!CollectionUtils.isEmpty(dataList)) { + System.out.println("-----------------正在处理剩余数据"); + batchUpdate(connectorInstance, pool, sql, dataList, 1000); + } + k--; + } + + pool.shutdown(); + logger.info("总共耗时：{}秒", (Instant.now().toEpochMilli() - begin) / 1000); + } + + @Test + public void testBatchDelete() { + final DatabaseConnectorInstance connectorInstance = new DatabaseConnectorInstance(createMysqlConfig()); + + long begin = Instant.now().toEpochMilli(); + final int threadSize = 10; + final ExecutorService pool = Executors.newFixedThreadPool(threadSize); + final String sql = "delete from `test`.`vote_records` WHERE `id` = ?"; + + List dataList = new ArrayList<>(); + for (int i = 1; i <= 3259000; i++) { + // 'dA8LeJLtX9MgQgDe7H1O', '2022-11-17 16:35:21', 1 + Object[] args = new Object[1]; + args[0] = i; + dataList.add(args); + + if (i % 10000 == 0) { + System.out.println(i + "-----------------正在处理"); + batchUpdate(connectorInstance, pool, sql, dataList, 1000); + dataList.clear(); + } + } + + if (!CollectionUtils.isEmpty(dataList)) { + System.out.println("-----------------正在处理剩余数据"); + batchUpdate(connectorInstance, pool, sql, dataList, 1000); + } + + pool.shutdown(); + logger.info("总共耗时：{}秒", (Instant.now().toEpochMilli() - begin) / 1000); + } + + private final static String STR = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; + + private String randomUserId(int i) { + StringBuilder s = new StringBuilder(); + for (int j = 0; j < i; j++) { + int r = RandomUtil.nextInt(0, 62); + s.append(StringUtil.substring(STR, r, r + 1)); + } + return s.toString(); + } + + private void batchUpdate(DatabaseConnectorInstance connectorInstance, ExecutorService pool, String sql, List dataList, int batchSize) { + int total = dataList.size(); + int taskSize = total % batchSize == 0 ? total / batchSize : total / batchSize + 1; + final CountDownLatch latch = new CountDownLatch(taskSize); + int fromIndex = 0; + int toIndex = batchSize; + for (int i = 0; i < taskSize; i++) { + final List data; + if (toIndex > total) { + toIndex = fromIndex + (total % batchSize); + data = dataList.subList(fromIndex, toIndex); + } else { + data = dataList.subList(fromIndex, toIndex); + fromIndex += batchSize; + toIndex += batchSize; + } + + pool.submit(() -> { + try { + connectorInstance.execute(databaseTemplate -> databaseTemplate.batchUpdate(sql, data)); + } catch (Exception e) { + logger.error(e.getMessage()); + } finally { + latch.countDown(); + } + }); + } + try { + latch.await(); + } catch (InterruptedException e) { + logger.error(e.getMessage()); + } + } + + @Test + public void testReadSchema() { + getTables(createOracleConfig(), "test", "AE86", "MY_ORG"); + getTables(createOracleConfig(), "test", "AE86", null); + + getTables(createMysqlConfig(), "test", "root", "MY_ORG"); + getTables(createMysqlConfig(), "test", "root", null); + + getTables(createSqlServerConfig(), "test", "dbo", "MY_ORG"); + getTables(createSqlServerConfig(), "test", "dbo", null); + + getTables(createPostgresConfig(), "postgres", "public", "MY_ORG"); + getTables(createPostgresConfig(), "postgres", "public", null); + } + + @Test + public void testGetColumnsDetails() { + final String schema = "root"; + final String tableNamePattern = "sw_test"; + final DatabaseConnectorInstance connectorInstance = new DatabaseConnectorInstance(createMysqlConfig()); + connectorInstance.execute(databaseTemplate -> { + SimpleConnection connection = databaseTemplate.getSimpleConnection(); + Connection conn = connection.getConnection(); + String databaseCatalog = conn.getCatalog(); + String schemaNamePattern = null == schema ? conn.getSchema() : schema; + List fields = new ArrayList<>(); + DatabaseMetaData metaData = conn.getMetaData(); + ResultSet columnMetadata = metaData.getColumns(databaseCatalog, schemaNamePattern, tableNamePattern, null); + while (columnMetadata.next()) { + String columnName = columnMetadata.getString(4); + int columnType = columnMetadata.getInt(5); + String typeName = columnMetadata.getString(6); + fields.add(new Field(columnName, typeName, columnType)); + } + return fields; + }); + } + + private List
    getTables(DatabaseConfig config, final String catalog, final String schema, final String tableNamePattern) { + final DatabaseConnectorInstance connectorInstance = new DatabaseConnectorInstance(config); + List
    tables = new ArrayList<>(); + connectorInstance.execute(databaseTemplate -> { + SimpleConnection connection = databaseTemplate.getSimpleConnection(); + Connection conn = connection.getConnection(); + String databaseCatalog = null == catalog ? conn.getCatalog() : catalog; + String schemaNamePattern = null == schema ? conn.getSchema() : schema; + String[] types = {TableTypeEnum.TABLE.getCode(), TableTypeEnum.VIEW.getCode(), TableTypeEnum.MATERIALIZED_VIEW.getCode()}; + final ResultSet rs = conn.getMetaData().getTables(databaseCatalog, schemaNamePattern, tableNamePattern, types); + while (rs.next()) { + final String tableName = rs.getString("TABLE_NAME"); + final String tableType = rs.getString("TABLE_TYPE"); + tables.add(new Table(tableName, tableType)); + } + return tables; + }); + + logger.info("\r 表总数{}", tables.size()); + tables.forEach(t -> logger.info("{} {}", t.getName(), t.getType())); + + return tables; + } + + private DatabaseConfig createSqlServerConfig() { + DatabaseConfig config = new DatabaseConfig(); + config.setUrl("jdbc:sqlserver://127.0.0.1:1433;DatabaseName=test"); + config.setUsername("sa"); + config.setPassword("123"); + config.setDriverClassName("com.microsoft.sqlserver.jdbc.SQLServerDriver"); + return config; + } + + private DatabaseConfig createOracleConfig() { + DatabaseConfig config = new DatabaseConfig(); + config.setUrl("jdbc:oracle:thin:@127.0.0.1:1521:ORCL"); + config.setUsername("ae86"); + config.setPassword("123"); + config.setDriverClassName("oracle.jdbc.OracleDriver"); + return config; + } + + private DatabaseConfig createMysqlConfig() { + DatabaseConfig config = new DatabaseConfig(); + config.setUrl("jdbc:mysql://127.0.0.1:3305/test?rewriteBatchedStatements=true&useUnicode=true&characterEncoding=UTF8&serverTimezone=Asia/Shanghai&useSSL=false&verifyServerCertificate=false&autoReconnect=true&failOverReadOnly=false"); + config.setUsername("root"); + config.setPassword("123"); + config.setDriverClassName("com.mysql.cj.jdbc.Driver"); + return config; + } + + private DatabaseConfig createPostgresConfig() { + DatabaseConfig config = new DatabaseConfig(); + config.setUrl("jdbc:postgresql://127.0.0.1:5432/postgres"); + config.setUsername("postgres"); + config.setPassword("123456"); + config.setDriverClassName("org.postgresql.Driver"); + return config; + } +} \ No newline at end of file -- Gitee From f8592030637232ce60403c17387a5fcdfb633f6f Mon Sep 17 00:00:00 2001 From: AE86 <836391306@qq.com> Date: Mon, 27 Nov 2023 00:38:10 +0800 Subject: [PATCH 37/98] fix bug --- .../impl/tablegroup/TableGroupChecker.java | 4 --- .../dbsyncer-connector-base/pom.xml | 7 +++++ .../convert/handler/BlobToStringHandler.java | 30 ------------------- .../convert/handler/ClobToStringHandler.java | 30 ------------------- .../dbsyncer/parser/enums/ConvertEnum.java | 8 ----- dbsyncer-storage/pom.xml | 20 +++++++++---- 6 files changed, 21 insertions(+), 78 deletions(-) delete mode 100644 dbsyncer-parser/src/main/java/org/dbsyncer/parser/convert/handler/BlobToStringHandler.java delete mode 100644 dbsyncer-parser/src/main/java/org/dbsyncer/parser/convert/handler/ClobToStringHandler.java diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/tablegroup/TableGroupChecker.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/tablegroup/TableGroupChecker.java index ba921085..b7c4949f 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/tablegroup/TableGroupChecker.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/tablegroup/TableGroupChecker.java @@ -15,7 +15,6 @@ import org.dbsyncer.parser.model.FieldMapping; import org.dbsyncer.parser.model.Mapping; import org.dbsyncer.parser.model.TableGroup; import org.dbsyncer.parser.util.PickerUtil; -import org.dbsyncer.sdk.connector.ConfigValidator; import org.dbsyncer.sdk.enums.ModelEnum; import org.dbsyncer.sdk.model.Field; import org.dbsyncer.sdk.model.MetaInfo; @@ -54,9 +53,6 @@ public class TableGroupChecker extends AbstractChecker { @Resource private ProfileComponent profileComponent; - @Resource - private Map map; - @Override public ConfigModel checkAddConfigModel(Map params) { logger.info("params:{}", params); diff --git a/dbsyncer-connector/dbsyncer-connector-base/pom.xml b/dbsyncer-connector/dbsyncer-connector-base/pom.xml index 43df45ec..b436a54c 100644 --- a/dbsyncer-connector/dbsyncer-connector-base/pom.xml +++ b/dbsyncer-connector/dbsyncer-connector-base/pom.xml @@ -24,6 +24,7 @@ org.ghi dbsyncer-connector-elasticsearch ${project.parent.version} + provided @@ -31,6 +32,7 @@ org.ghi dbsyncer-connector-file ${project.parent.version} + provided @@ -38,6 +40,7 @@ org.ghi dbsyncer-connector-kafka ${project.parent.version} + provided @@ -45,6 +48,7 @@ org.ghi dbsyncer-connector-mysql ${project.parent.version} + provided @@ -52,6 +56,7 @@ org.ghi dbsyncer-connector-oracle ${project.parent.version} + provided @@ -59,6 +64,7 @@ org.ghi dbsyncer-connector-sqlserver ${project.parent.version} + provided @@ -66,6 +72,7 @@ org.ghi dbsyncer-connector-postgresql ${project.parent.version} + provided diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/convert/handler/BlobToStringHandler.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/convert/handler/BlobToStringHandler.java deleted file mode 100644 index fb092810..00000000 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/convert/handler/BlobToStringHandler.java +++ /dev/null @@ -1,30 +0,0 @@ -package org.dbsyncer.parser.convert.handler; - -import org.dbsyncer.parser.ParserException; -import org.dbsyncer.parser.convert.AbstractHandler; - -import java.sql.SQLException; - -/** - * Blob转String - * - * @author AE86 - * @version 1.0.0 - * @date 2022/1/20 23:04 - */ -public class BlobToStringHandler extends AbstractHandler { - - @Override - public Object convert(String args, Object value) { - if (value instanceof oracle.sql.BLOB) { - oracle.sql.BLOB blob = (oracle.sql.BLOB) value; - try { - value = getString(blob.getBinaryStream(), (int) blob.length()); - } catch (SQLException e) { - throw new ParserException(e.getMessage()); - } - } - return value; - } - -} \ No newline at end of file diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/convert/handler/ClobToStringHandler.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/convert/handler/ClobToStringHandler.java deleted file mode 100644 index 21e17ff3..00000000 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/convert/handler/ClobToStringHandler.java +++ /dev/null @@ -1,30 +0,0 @@ -package org.dbsyncer.parser.convert.handler; - -import org.dbsyncer.parser.ParserException; -import org.dbsyncer.parser.convert.AbstractHandler; - -import java.sql.SQLException; - -/** - * Clob转String - * - * @author AE86 - * @version 1.0.0 - * @date 2022/1/20 23:04 - */ -public class ClobToStringHandler extends AbstractHandler { - - @Override - public Object convert(String args, Object value) { - if (value instanceof oracle.sql.CLOB) { - oracle.sql.CLOB clob = (oracle.sql.CLOB) value; - try { - value = getString(clob.getAsciiStream(), (int) clob.length()); - } catch (SQLException e) { - throw new ParserException(e.getMessage()); - } - } - return value; - } - -} \ No newline at end of file diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/enums/ConvertEnum.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/enums/ConvertEnum.java index d5ef0aed..03ce76ee 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/enums/ConvertEnum.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/enums/ConvertEnum.java @@ -54,14 +54,6 @@ public enum ConvertEnum { * Byte[]转String */ BYTES_TO_STRING("BYTES_TO_STRING", "Byte[]转String", 0, new BytesToStringHandler()), - /** - * Clob转String - */ - CLOB_TO_STRING("CLOB_TO_STRING", "Clob转String", 0, new ClobToStringHandler()), - /** - * Blob转String - */ - BLOB_TO_STRING("BLOB_TO_STRING", "Blob转String", 0, new BlobToStringHandler()), /** * 替换 */ diff --git a/dbsyncer-storage/pom.xml b/dbsyncer-storage/pom.xml index 1b713703..124091e6 100644 --- a/dbsyncer-storage/pom.xml +++ b/dbsyncer-storage/pom.xml @@ -24,12 +24,6 @@ protobuf-java - - - com.oracle - ojdbc6 - - org.apache.lucene @@ -47,6 +41,20 @@ spring-boot-starter-log4j2 + + + com.oracle + ojdbc6 + provided + + + + + org.postgresql + postgresql + provided + + junit junit -- Gitee From 8ca9647c1798066646bc9b9129f57630d8096587 Mon Sep 17 00:00:00 2001 From: AE86 <836391306@qq.com> Date: Mon, 27 Nov 2023 00:58:18 +0800 Subject: [PATCH 38/98] mv scheduled --- .../src/main/java/org/dbsyncer/biz/impl/MetricReporter.java | 4 ++-- .../main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java | 4 ++-- .../org/dbsyncer/common}/scheduled/ScheduledTaskJob.java | 2 +- .../dbsyncer/common}/scheduled/ScheduledTaskService.java | 2 +- .../common}/scheduled/impl/ScheduledTaskServiceImpl.java | 6 +++--- .../java/org/dbsyncer/manager/impl/IncrementPuller.java | 4 ++-- .../org/dbsyncer/parser/flush/AbstractBufferActuator.java | 4 ++-- .../parser/flush/impl/TableGroupBufferActuator.java | 2 +- .../java/org/dbsyncer/sdk/listener/AbstractListener.java | 2 +- .../org/dbsyncer/sdk/listener/AbstractQuartzListener.java | 2 +- 10 files changed, 16 insertions(+), 16 deletions(-) rename {dbsyncer-sdk/src/main/java/org/dbsyncer/sdk => dbsyncer-common/src/main/java/org/dbsyncer/common}/scheduled/ScheduledTaskJob.java (75%) rename {dbsyncer-sdk/src/main/java/org/dbsyncer/sdk => dbsyncer-common/src/main/java/org/dbsyncer/common}/scheduled/ScheduledTaskService.java (94%) rename {dbsyncer-sdk/src/main/java/org/dbsyncer/sdk => dbsyncer-common/src/main/java/org/dbsyncer/common}/scheduled/impl/ScheduledTaskServiceImpl.java (94%) diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MetricReporter.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MetricReporter.java index d62e488d..55f382d6 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MetricReporter.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MetricReporter.java @@ -9,8 +9,8 @@ import org.dbsyncer.biz.model.MetricResponse; import org.dbsyncer.biz.model.MetricResponseInfo; import org.dbsyncer.biz.model.Sample; import org.dbsyncer.common.model.Paging; -import org.dbsyncer.sdk.scheduled.ScheduledTaskJob; -import org.dbsyncer.sdk.scheduled.ScheduledTaskService; +import org.dbsyncer.common.scheduled.ScheduledTaskJob; +import org.dbsyncer.common.scheduled.ScheduledTaskService; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.sdk.constant.ConnectorConstant; diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java index 3bdaa364..0b7fd342 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java @@ -29,8 +29,8 @@ import org.dbsyncer.common.util.JsonUtil; import org.dbsyncer.common.util.NumberUtil; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.sdk.enums.FilterEnum; -import org.dbsyncer.sdk.scheduled.ScheduledTaskJob; -import org.dbsyncer.sdk.scheduled.ScheduledTaskService; +import org.dbsyncer.common.scheduled.ScheduledTaskJob; +import org.dbsyncer.common.scheduled.ScheduledTaskService; import org.dbsyncer.parser.LogService; import org.dbsyncer.parser.LogType; import org.dbsyncer.parser.ProfileComponent; diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/scheduled/ScheduledTaskJob.java b/dbsyncer-common/src/main/java/org/dbsyncer/common/scheduled/ScheduledTaskJob.java similarity index 75% rename from dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/scheduled/ScheduledTaskJob.java rename to dbsyncer-common/src/main/java/org/dbsyncer/common/scheduled/ScheduledTaskJob.java index b40e5f2a..586c46c7 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/scheduled/ScheduledTaskJob.java +++ b/dbsyncer-common/src/main/java/org/dbsyncer/common/scheduled/ScheduledTaskJob.java @@ -1,4 +1,4 @@ -package org.dbsyncer.sdk.scheduled; +package org.dbsyncer.common.scheduled; /** * @version 1.0.0 diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/scheduled/ScheduledTaskService.java b/dbsyncer-common/src/main/java/org/dbsyncer/common/scheduled/ScheduledTaskService.java similarity index 94% rename from dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/scheduled/ScheduledTaskService.java rename to dbsyncer-common/src/main/java/org/dbsyncer/common/scheduled/ScheduledTaskService.java index f73367d3..34caa6c0 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/scheduled/ScheduledTaskService.java +++ b/dbsyncer-common/src/main/java/org/dbsyncer/common/scheduled/ScheduledTaskService.java @@ -1,4 +1,4 @@ -package org.dbsyncer.sdk.scheduled; +package org.dbsyncer.common.scheduled; public interface ScheduledTaskService { diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/scheduled/impl/ScheduledTaskServiceImpl.java b/dbsyncer-common/src/main/java/org/dbsyncer/common/scheduled/impl/ScheduledTaskServiceImpl.java similarity index 94% rename from dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/scheduled/impl/ScheduledTaskServiceImpl.java rename to dbsyncer-common/src/main/java/org/dbsyncer/common/scheduled/impl/ScheduledTaskServiceImpl.java index 3f636360..96125f54 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/scheduled/impl/ScheduledTaskServiceImpl.java +++ b/dbsyncer-common/src/main/java/org/dbsyncer/common/scheduled/impl/ScheduledTaskServiceImpl.java @@ -1,8 +1,8 @@ -package org.dbsyncer.sdk.scheduled.impl; +package org.dbsyncer.common.scheduled.impl; import org.dbsyncer.common.CommonException; -import org.dbsyncer.sdk.scheduled.ScheduledTaskJob; -import org.dbsyncer.sdk.scheduled.ScheduledTaskService; +import org.dbsyncer.common.scheduled.ScheduledTaskJob; +import org.dbsyncer.common.scheduled.ScheduledTaskService; import org.dbsyncer.common.util.UUIDUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/IncrementPuller.java b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/IncrementPuller.java index 71b9a9f6..8386ab8d 100644 --- a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/IncrementPuller.java +++ b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/IncrementPuller.java @@ -24,8 +24,8 @@ import org.dbsyncer.sdk.model.ChangedOffset; import org.dbsyncer.sdk.model.ConnectorConfig; import org.dbsyncer.sdk.model.Table; import org.dbsyncer.sdk.model.TableGroupQuartzCommand; -import org.dbsyncer.sdk.scheduled.ScheduledTaskJob; -import org.dbsyncer.sdk.scheduled.ScheduledTaskService; +import org.dbsyncer.common.scheduled.ScheduledTaskJob; +import org.dbsyncer.common.scheduled.ScheduledTaskService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.ApplicationListener; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/AbstractBufferActuator.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/AbstractBufferActuator.java index 15febcf9..eec99734 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/AbstractBufferActuator.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/AbstractBufferActuator.java @@ -1,8 +1,8 @@ package org.dbsyncer.parser.flush; import org.dbsyncer.common.config.BufferActuatorConfig; -import org.dbsyncer.sdk.scheduled.ScheduledTaskJob; -import org.dbsyncer.sdk.scheduled.ScheduledTaskService; +import org.dbsyncer.common.scheduled.ScheduledTaskJob; +import org.dbsyncer.common.scheduled.ScheduledTaskService; import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.parser.enums.MetaEnum; import org.dbsyncer.parser.model.Meta; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/TableGroupBufferActuator.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/TableGroupBufferActuator.java index 285c2176..bd372f86 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/TableGroupBufferActuator.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/TableGroupBufferActuator.java @@ -1,7 +1,7 @@ package org.dbsyncer.parser.flush.impl; import org.dbsyncer.common.config.TableGroupBufferConfig; -import org.dbsyncer.sdk.scheduled.ScheduledTaskService; +import org.dbsyncer.common.scheduled.ScheduledTaskService; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.common.util.ThreadPoolUtil; import org.dbsyncer.common.util.UUIDUtil; diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/AbstractListener.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/AbstractListener.java index 483d15f3..f0a06462 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/AbstractListener.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/AbstractListener.java @@ -1,7 +1,7 @@ package org.dbsyncer.sdk.listener; import org.dbsyncer.common.util.CollectionUtils; -import org.dbsyncer.sdk.scheduled.ScheduledTaskService; +import org.dbsyncer.common.scheduled.ScheduledTaskService; import org.dbsyncer.sdk.config.ListenerConfig; import org.dbsyncer.sdk.connector.ConnectorInstance; import org.dbsyncer.sdk.constant.ConnectorConstant; diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/AbstractQuartzListener.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/AbstractQuartzListener.java index b1d05743..ee386cea 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/AbstractQuartzListener.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/AbstractQuartzListener.java @@ -10,7 +10,7 @@ import org.dbsyncer.sdk.listener.event.ScanChangedEvent; import org.dbsyncer.sdk.model.Point; import org.dbsyncer.sdk.model.Table; import org.dbsyncer.sdk.model.TableGroupQuartzCommand; -import org.dbsyncer.sdk.scheduled.ScheduledTaskJob; +import org.dbsyncer.common.scheduled.ScheduledTaskJob; import org.dbsyncer.sdk.util.PrimaryKeyUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -- Gitee From 2b3bf1aff8be490f35b5c3084ac5b19849097271 Mon Sep 17 00:00:00 2001 From: AE86 <836391306@qq.com> Date: Tue, 28 Nov 2023 01:16:20 +0800 Subject: [PATCH 39/98] fix --- .../dbsyncer/sdk/connector/CompareFilter.java | 7 ------- .../java/org/dbsyncer/sdk/enums/FilterEnum.java | 7 ++++++- .../org/dbsyncer/sdk/filter/CompareFilter.java | 17 ----------------- 3 files changed, 6 insertions(+), 25 deletions(-) delete mode 100644 dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/CompareFilter.java delete mode 100644 dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/filter/CompareFilter.java diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/CompareFilter.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/CompareFilter.java deleted file mode 100644 index 219c6d13..00000000 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/CompareFilter.java +++ /dev/null @@ -1,7 +0,0 @@ -package org.dbsyncer.sdk.connector; - -public interface CompareFilter { - - boolean compare(String value, String filterValue); - -} \ No newline at end of file diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/FilterEnum.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/FilterEnum.java index b4deaf32..2de0b0f0 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/FilterEnum.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/FilterEnum.java @@ -6,7 +6,6 @@ package org.dbsyncer.sdk.enums; import org.dbsyncer.common.util.NumberUtil; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.sdk.SdkException; -import org.dbsyncer.sdk.filter.CompareFilter; /** * 运算符表达式类型 @@ -114,4 +113,10 @@ public enum FilterEnum { return compareFilter; } + private interface CompareFilter { + + boolean compare(String value, String filterValue); + + } + } \ No newline at end of file diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/filter/CompareFilter.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/filter/CompareFilter.java deleted file mode 100644 index 3f0da1dc..00000000 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/filter/CompareFilter.java +++ /dev/null @@ -1,17 +0,0 @@ -/** - * DBSyncer Copyright 2020-2023 All Rights Reserved. - */ -package org.dbsyncer.sdk.filter; - -/** - * 值比较器实现 - * - * @Author AE86 - * @Version 1.0.0 - * @Date 2019-11-17 23:56 - */ -public interface CompareFilter { - - boolean compare(String value, String filterValue); - -} \ No newline at end of file -- Gitee From 7f09f0902c84fab410d511af08babc0a618645cf Mon Sep 17 00:00:00 2001 From: bble Date: Tue, 28 Nov 2023 14:26:42 +0800 Subject: [PATCH 40/98] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E8=BF=9E=E6=8E=A5?= =?UTF-8?q?=E5=99=A8=E5=AE=9E=E7=8E=B0=E5=89=8D=E7=AB=AF=E9=9B=B6=E6=94=B9?= =?UTF-8?q?=E9=80=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../public/connector/addElasticsearch.html | 0 .../resources/public/connector/addFile.html | 0 .../resources/public/connector/addKafka.html | 0 .../public/connector/addDqlMySQL.html | 0 .../resources/public/connector/addMySQL.html | 0 .../public/connector/addDqlOracle.html | 0 .../resources/public/connector/addOracle.html | 0 .../public/connector/addDqlPostgreSQL.html | 0 .../public/connector/addPostgreSQL.html | 0 .../public/connector/addDqlSqlServer.html | 108 +++++++++--------- .../public/connector/addSqlServer.html | 0 11 files changed, 54 insertions(+), 54 deletions(-) rename {dbsyncer-web => dbsyncer-connector/dbsyncer-connector-elasticsearch}/src/main/resources/public/connector/addElasticsearch.html (100%) rename {dbsyncer-web => dbsyncer-connector/dbsyncer-connector-file}/src/main/resources/public/connector/addFile.html (100%) rename {dbsyncer-web => dbsyncer-connector/dbsyncer-connector-kafka}/src/main/resources/public/connector/addKafka.html (100%) rename {dbsyncer-web => dbsyncer-connector/dbsyncer-connector-mysql}/src/main/resources/public/connector/addDqlMySQL.html (100%) rename {dbsyncer-web => dbsyncer-connector/dbsyncer-connector-mysql}/src/main/resources/public/connector/addMySQL.html (100%) rename {dbsyncer-web => dbsyncer-connector/dbsyncer-connector-oracle}/src/main/resources/public/connector/addDqlOracle.html (100%) rename {dbsyncer-web => dbsyncer-connector/dbsyncer-connector-oracle}/src/main/resources/public/connector/addOracle.html (100%) rename {dbsyncer-web => dbsyncer-connector/dbsyncer-connector-postgresql}/src/main/resources/public/connector/addDqlPostgreSQL.html (100%) rename {dbsyncer-web => dbsyncer-connector/dbsyncer-connector-postgresql}/src/main/resources/public/connector/addPostgreSQL.html (100%) rename {dbsyncer-web => dbsyncer-connector/dbsyncer-connector-sqlserver}/src/main/resources/public/connector/addDqlSqlServer.html (97%) rename {dbsyncer-web => dbsyncer-connector/dbsyncer-connector-sqlserver}/src/main/resources/public/connector/addSqlServer.html (100%) diff --git a/dbsyncer-web/src/main/resources/public/connector/addElasticsearch.html b/dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/resources/public/connector/addElasticsearch.html similarity index 100% rename from dbsyncer-web/src/main/resources/public/connector/addElasticsearch.html rename to dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/resources/public/connector/addElasticsearch.html diff --git a/dbsyncer-web/src/main/resources/public/connector/addFile.html b/dbsyncer-connector/dbsyncer-connector-file/src/main/resources/public/connector/addFile.html similarity index 100% rename from dbsyncer-web/src/main/resources/public/connector/addFile.html rename to dbsyncer-connector/dbsyncer-connector-file/src/main/resources/public/connector/addFile.html diff --git a/dbsyncer-web/src/main/resources/public/connector/addKafka.html b/dbsyncer-connector/dbsyncer-connector-kafka/src/main/resources/public/connector/addKafka.html similarity index 100% rename from dbsyncer-web/src/main/resources/public/connector/addKafka.html rename to dbsyncer-connector/dbsyncer-connector-kafka/src/main/resources/public/connector/addKafka.html diff --git a/dbsyncer-web/src/main/resources/public/connector/addDqlMySQL.html b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/resources/public/connector/addDqlMySQL.html similarity index 100% rename from dbsyncer-web/src/main/resources/public/connector/addDqlMySQL.html rename to dbsyncer-connector/dbsyncer-connector-mysql/src/main/resources/public/connector/addDqlMySQL.html diff --git a/dbsyncer-web/src/main/resources/public/connector/addMySQL.html b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/resources/public/connector/addMySQL.html similarity index 100% rename from dbsyncer-web/src/main/resources/public/connector/addMySQL.html rename to dbsyncer-connector/dbsyncer-connector-mysql/src/main/resources/public/connector/addMySQL.html diff --git a/dbsyncer-web/src/main/resources/public/connector/addDqlOracle.html b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/resources/public/connector/addDqlOracle.html similarity index 100% rename from dbsyncer-web/src/main/resources/public/connector/addDqlOracle.html rename to dbsyncer-connector/dbsyncer-connector-oracle/src/main/resources/public/connector/addDqlOracle.html diff --git a/dbsyncer-web/src/main/resources/public/connector/addOracle.html b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/resources/public/connector/addOracle.html similarity index 100% rename from dbsyncer-web/src/main/resources/public/connector/addOracle.html rename to dbsyncer-connector/dbsyncer-connector-oracle/src/main/resources/public/connector/addOracle.html diff --git a/dbsyncer-web/src/main/resources/public/connector/addDqlPostgreSQL.html b/dbsyncer-connector/dbsyncer-connector-postgresql/src/main/resources/public/connector/addDqlPostgreSQL.html similarity index 100% rename from dbsyncer-web/src/main/resources/public/connector/addDqlPostgreSQL.html rename to dbsyncer-connector/dbsyncer-connector-postgresql/src/main/resources/public/connector/addDqlPostgreSQL.html diff --git a/dbsyncer-web/src/main/resources/public/connector/addPostgreSQL.html b/dbsyncer-connector/dbsyncer-connector-postgresql/src/main/resources/public/connector/addPostgreSQL.html similarity index 100% rename from dbsyncer-web/src/main/resources/public/connector/addPostgreSQL.html rename to dbsyncer-connector/dbsyncer-connector-postgresql/src/main/resources/public/connector/addPostgreSQL.html diff --git a/dbsyncer-web/src/main/resources/public/connector/addDqlSqlServer.html b/dbsyncer-connector/dbsyncer-connector-sqlserver/src/main/resources/public/connector/addDqlSqlServer.html similarity index 97% rename from dbsyncer-web/src/main/resources/public/connector/addDqlSqlServer.html rename to dbsyncer-connector/dbsyncer-connector-sqlserver/src/main/resources/public/connector/addDqlSqlServer.html index ba54b23d..2e298a62 100644 --- a/dbsyncer-web/src/main/resources/public/connector/addDqlSqlServer.html +++ b/dbsyncer-connector/dbsyncer-connector-sqlserver/src/main/resources/public/connector/addDqlSqlServer.html @@ -1,55 +1,55 @@ - - - -
    -
    - -
    - -
    - -
    - -
    -
    -
    - -
    - -
    -
    -
    - -
    - -
    -
    -
    -
    - -
    - -
    -
    - - -
    - -
    - -
    - -
    -
    - - -
    - + + + +
    +
    + +
    + +
    + +
    + +
    +
    +
    + +
    + +
    +
    +
    + +
    + +
    +
    +
    +
    + +
    + +
    +
    + + +
    + +
    + +
    + +
    +
    + + +
    + \ No newline at end of file diff --git a/dbsyncer-web/src/main/resources/public/connector/addSqlServer.html b/dbsyncer-connector/dbsyncer-connector-sqlserver/src/main/resources/public/connector/addSqlServer.html similarity index 100% rename from dbsyncer-web/src/main/resources/public/connector/addSqlServer.html rename to dbsyncer-connector/dbsyncer-connector-sqlserver/src/main/resources/public/connector/addSqlServer.html -- Gitee From 17a61c642c3881fc0ebcac184f0744c39a269de1 Mon Sep 17 00:00:00 2001 From: AE86 <836391306@qq.com> Date: Tue, 28 Nov 2023 07:41:49 +0000 Subject: [PATCH 41/98] revert Signed-off-by: AE86 <836391306@qq.com> --- .../java/org/dbsyncer/sdk/enums/FilterEnum.java | 7 +------ .../org/dbsyncer/sdk/filter/CompareFilter.java | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 6 deletions(-) create mode 100644 dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/filter/CompareFilter.java diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/FilterEnum.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/FilterEnum.java index 2de0b0f0..b4deaf32 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/FilterEnum.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/FilterEnum.java @@ -6,6 +6,7 @@ package org.dbsyncer.sdk.enums; import org.dbsyncer.common.util.NumberUtil; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.sdk.SdkException; +import org.dbsyncer.sdk.filter.CompareFilter; /** * 运算符表达式类型 @@ -113,10 +114,4 @@ public enum FilterEnum { return compareFilter; } - private interface CompareFilter { - - boolean compare(String value, String filterValue); - - } - } \ No newline at end of file diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/filter/CompareFilter.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/filter/CompareFilter.java new file mode 100644 index 00000000..49183a2e --- /dev/null +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/filter/CompareFilter.java @@ -0,0 +1,17 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.sdk.filter; + +/** + * 值比较器实现 + * + * @Author AE86 + * @Version 1.0.0 + * @Date 2019-11-17 23:56 + */ +public interface CompareFilter { + + boolean compare(String value, String filterValue); + +} \ No newline at end of file -- Gitee From 58205ef704882025dafc2e2b6627cb5bc21a4104 Mon Sep 17 00:00:00 2001 From: bble <1058519553@qq.com> Date: Wed, 29 Nov 2023 10:06:08 +0800 Subject: [PATCH 42/98] =?UTF-8?q?=E8=B0=83=E6=95=B4=E8=BF=9E=E6=8E=A5?= =?UTF-8?q?=E5=99=A8=E5=89=8D=E7=AB=AF=E8=B5=84=E6=BA=90=E7=9B=AE=E5=BD=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/resources/static/img/Elasticsearch.png | Bin .../src/main/resources/static/img/File.png | Bin .../src/main/resources/static/img/Kafka.png | Bin .../src/main/resources/static/img/DqlMySQL.png | Bin .../src/main/resources/static/img/MySQL.png | Bin .../src/main/resources/static/img/DqlOracle.png | Bin .../src/main/resources/static/img/Oracle.png | Bin .../src/main/resources/static/img/DqlPostgreSQL.png | Bin .../src/main/resources/static/img/PostgreSQL.png | Bin .../src/main/resources/static/img/DqlSqlServer.png | Bin .../src/main/resources/static/img/SqlServer.png | Bin 11 files changed, 0 insertions(+), 0 deletions(-) rename {dbsyncer-web => dbsyncer-connector/dbsyncer-connector-elasticsearch}/src/main/resources/static/img/Elasticsearch.png (100%) rename {dbsyncer-web => dbsyncer-connector/dbsyncer-connector-file}/src/main/resources/static/img/File.png (100%) rename {dbsyncer-web => dbsyncer-connector/dbsyncer-connector-kafka}/src/main/resources/static/img/Kafka.png (100%) rename {dbsyncer-web => dbsyncer-connector/dbsyncer-connector-mysql}/src/main/resources/static/img/DqlMySQL.png (100%) rename {dbsyncer-web => dbsyncer-connector/dbsyncer-connector-mysql}/src/main/resources/static/img/MySQL.png (100%) rename {dbsyncer-web => dbsyncer-connector/dbsyncer-connector-oracle}/src/main/resources/static/img/DqlOracle.png (100%) rename {dbsyncer-web => dbsyncer-connector/dbsyncer-connector-oracle}/src/main/resources/static/img/Oracle.png (100%) rename {dbsyncer-web => dbsyncer-connector/dbsyncer-connector-postgresql}/src/main/resources/static/img/DqlPostgreSQL.png (100%) rename {dbsyncer-web => dbsyncer-connector/dbsyncer-connector-postgresql}/src/main/resources/static/img/PostgreSQL.png (100%) rename {dbsyncer-web => dbsyncer-connector/dbsyncer-connector-sqlserver}/src/main/resources/static/img/DqlSqlServer.png (100%) rename {dbsyncer-web => dbsyncer-connector/dbsyncer-connector-sqlserver}/src/main/resources/static/img/SqlServer.png (100%) diff --git a/dbsyncer-web/src/main/resources/static/img/Elasticsearch.png b/dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/resources/static/img/Elasticsearch.png similarity index 100% rename from dbsyncer-web/src/main/resources/static/img/Elasticsearch.png rename to dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/resources/static/img/Elasticsearch.png diff --git a/dbsyncer-web/src/main/resources/static/img/File.png b/dbsyncer-connector/dbsyncer-connector-file/src/main/resources/static/img/File.png similarity index 100% rename from dbsyncer-web/src/main/resources/static/img/File.png rename to dbsyncer-connector/dbsyncer-connector-file/src/main/resources/static/img/File.png diff --git a/dbsyncer-web/src/main/resources/static/img/Kafka.png b/dbsyncer-connector/dbsyncer-connector-kafka/src/main/resources/static/img/Kafka.png similarity index 100% rename from dbsyncer-web/src/main/resources/static/img/Kafka.png rename to dbsyncer-connector/dbsyncer-connector-kafka/src/main/resources/static/img/Kafka.png diff --git a/dbsyncer-web/src/main/resources/static/img/DqlMySQL.png b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/resources/static/img/DqlMySQL.png similarity index 100% rename from dbsyncer-web/src/main/resources/static/img/DqlMySQL.png rename to dbsyncer-connector/dbsyncer-connector-mysql/src/main/resources/static/img/DqlMySQL.png diff --git a/dbsyncer-web/src/main/resources/static/img/MySQL.png b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/resources/static/img/MySQL.png similarity index 100% rename from dbsyncer-web/src/main/resources/static/img/MySQL.png rename to dbsyncer-connector/dbsyncer-connector-mysql/src/main/resources/static/img/MySQL.png diff --git a/dbsyncer-web/src/main/resources/static/img/DqlOracle.png b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/resources/static/img/DqlOracle.png similarity index 100% rename from dbsyncer-web/src/main/resources/static/img/DqlOracle.png rename to dbsyncer-connector/dbsyncer-connector-oracle/src/main/resources/static/img/DqlOracle.png diff --git a/dbsyncer-web/src/main/resources/static/img/Oracle.png b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/resources/static/img/Oracle.png similarity index 100% rename from dbsyncer-web/src/main/resources/static/img/Oracle.png rename to dbsyncer-connector/dbsyncer-connector-oracle/src/main/resources/static/img/Oracle.png diff --git a/dbsyncer-web/src/main/resources/static/img/DqlPostgreSQL.png b/dbsyncer-connector/dbsyncer-connector-postgresql/src/main/resources/static/img/DqlPostgreSQL.png similarity index 100% rename from dbsyncer-web/src/main/resources/static/img/DqlPostgreSQL.png rename to dbsyncer-connector/dbsyncer-connector-postgresql/src/main/resources/static/img/DqlPostgreSQL.png diff --git a/dbsyncer-web/src/main/resources/static/img/PostgreSQL.png b/dbsyncer-connector/dbsyncer-connector-postgresql/src/main/resources/static/img/PostgreSQL.png similarity index 100% rename from dbsyncer-web/src/main/resources/static/img/PostgreSQL.png rename to dbsyncer-connector/dbsyncer-connector-postgresql/src/main/resources/static/img/PostgreSQL.png diff --git a/dbsyncer-web/src/main/resources/static/img/DqlSqlServer.png b/dbsyncer-connector/dbsyncer-connector-sqlserver/src/main/resources/static/img/DqlSqlServer.png similarity index 100% rename from dbsyncer-web/src/main/resources/static/img/DqlSqlServer.png rename to dbsyncer-connector/dbsyncer-connector-sqlserver/src/main/resources/static/img/DqlSqlServer.png diff --git a/dbsyncer-web/src/main/resources/static/img/SqlServer.png b/dbsyncer-connector/dbsyncer-connector-sqlserver/src/main/resources/static/img/SqlServer.png similarity index 100% rename from dbsyncer-web/src/main/resources/static/img/SqlServer.png rename to dbsyncer-connector/dbsyncer-connector-sqlserver/src/main/resources/static/img/SqlServer.png -- Gitee From fa84e34638c360e746c2d5faffceb176b1c7a51e Mon Sep 17 00:00:00 2001 From: bble <1058519553@qq.com> Date: Wed, 29 Nov 2023 14:17:39 +0800 Subject: [PATCH 43/98] add sqlite --- .../dbsyncer-connector-sqlite/pom.xml | 42 +++++++ .../connector/sqlite/DqlSQLiteConnector.java | 64 ++++++++++ .../connector/sqlite/SQLiteConnector.java | 118 ++++++++++++++++++ .../connector/sqlite/SQLiteException.java | 30 +++++ .../validator/DqlSQLiteConfigValidator.java | 27 ++++ .../validator/SQLiteConfigValidator.java | 24 ++++ .../org.dbsyncer.sdk.spi.ConnectorService | 2 + .../public/connector/addDqlSQLite.html | 46 +++++++ .../resources/public/connector/addSQLite.html | 29 +++++ .../src/main/resources/static/img/sqlite.png | Bin 0 -> 2449 bytes .../sdk/constant/DatabaseConstant.java | 6 + 11 files changed, 388 insertions(+) create mode 100644 dbsyncer-connector/dbsyncer-connector-sqlite/pom.xml create mode 100644 dbsyncer-connector/dbsyncer-connector-sqlite/src/main/java/org/dbsyncer/connector/sqlite/DqlSQLiteConnector.java create mode 100644 dbsyncer-connector/dbsyncer-connector-sqlite/src/main/java/org/dbsyncer/connector/sqlite/SQLiteConnector.java create mode 100644 dbsyncer-connector/dbsyncer-connector-sqlite/src/main/java/org/dbsyncer/connector/sqlite/SQLiteException.java create mode 100644 dbsyncer-connector/dbsyncer-connector-sqlite/src/main/java/org/dbsyncer/connector/sqlite/validator/DqlSQLiteConfigValidator.java create mode 100644 dbsyncer-connector/dbsyncer-connector-sqlite/src/main/java/org/dbsyncer/connector/sqlite/validator/SQLiteConfigValidator.java create mode 100644 dbsyncer-connector/dbsyncer-connector-sqlite/src/main/resources/META-INF/services/org.dbsyncer.sdk.spi.ConnectorService create mode 100644 dbsyncer-connector/dbsyncer-connector-sqlite/src/main/resources/public/connector/addDqlSQLite.html create mode 100644 dbsyncer-connector/dbsyncer-connector-sqlite/src/main/resources/public/connector/addSQLite.html create mode 100644 dbsyncer-connector/dbsyncer-connector-sqlite/src/main/resources/static/img/sqlite.png diff --git a/dbsyncer-connector/dbsyncer-connector-sqlite/pom.xml b/dbsyncer-connector/dbsyncer-connector-sqlite/pom.xml new file mode 100644 index 00000000..a40ced1c --- /dev/null +++ b/dbsyncer-connector/dbsyncer-connector-sqlite/pom.xml @@ -0,0 +1,42 @@ + + + + dbsyncer-connector + org.ghi + 2.0.0 + + 4.0.0 + + dbsyncer-connector-sqlite + + + + + org.ghi + dbsyncer-sdk + ${project.parent.version} + provided + + + + + org.xerial + sqlite-jdbc + + + + org.springframework.boot + spring-boot-starter-log4j2 + provided + + + + junit + junit + provided + + + + \ No newline at end of file diff --git a/dbsyncer-connector/dbsyncer-connector-sqlite/src/main/java/org/dbsyncer/connector/sqlite/DqlSQLiteConnector.java b/dbsyncer-connector/dbsyncer-connector-sqlite/src/main/java/org/dbsyncer/connector/sqlite/DqlSQLiteConnector.java new file mode 100644 index 00000000..21f9e595 --- /dev/null +++ b/dbsyncer-connector/dbsyncer-connector-sqlite/src/main/java/org/dbsyncer/connector/sqlite/DqlSQLiteConnector.java @@ -0,0 +1,64 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ + +package org.dbsyncer.connector.sqlite; + +import org.dbsyncer.common.util.StringUtil; +import org.dbsyncer.connector.sqlite.validator.DqlSQLiteConfigValidator; +import org.dbsyncer.sdk.config.ReaderConfig; +import org.dbsyncer.sdk.connector.ConfigValidator; +import org.dbsyncer.sdk.connector.database.AbstractDQLConnector; +import org.dbsyncer.sdk.constant.DatabaseConstant; +import org.dbsyncer.sdk.enums.ListenerTypeEnum; +import org.dbsyncer.sdk.listener.DatabaseQuartzListener; +import org.dbsyncer.sdk.listener.Listener; +import org.dbsyncer.sdk.model.PageSql; + +import java.util.List; + +/** + * DQLSQLite连接器实现 + * + * @Author bble + * @Version 1.0.0 + * @Date 2023-11-28 16:22 + */ +public final class DqlSQLiteConnector extends AbstractDQLConnector { + + private final String TYPE = "DqlSQLite"; + + private final DqlSQLiteConfigValidator configValidator = new DqlSQLiteConfigValidator(); + + @Override + public String getConnectorType() { + return TYPE; + } + + @Override + public ConfigValidator getConfigValidator() { + return configValidator; + } + + @Override + public Listener getListener(String listenerType) { + if (ListenerTypeEnum.isTiming(listenerType)) { + return new DatabaseQuartzListener(); + } + return null; + } + + @Override + public String getPageSql(PageSql config) { + List primaryKeys = config.getPrimaryKeys(); + String orderBy = StringUtil.join(primaryKeys, ","); + return String.format(DatabaseConstant.SQLITE_PAGE_SQL, orderBy, config.getQuerySql()); + } + + @Override + public Object[] getPageArgs(ReaderConfig config) { + int pageSize = config.getPageSize(); + int pageIndex = config.getPageIndex(); + return new Object[] {(pageIndex - 1) * pageSize + 1, pageIndex * pageSize}; + } +} \ No newline at end of file diff --git a/dbsyncer-connector/dbsyncer-connector-sqlite/src/main/java/org/dbsyncer/connector/sqlite/SQLiteConnector.java b/dbsyncer-connector/dbsyncer-connector-sqlite/src/main/java/org/dbsyncer/connector/sqlite/SQLiteConnector.java new file mode 100644 index 00000000..6622c000 --- /dev/null +++ b/dbsyncer-connector/dbsyncer-connector-sqlite/src/main/java/org/dbsyncer/connector/sqlite/SQLiteConnector.java @@ -0,0 +1,118 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.connector.sqlite; + +import org.dbsyncer.common.util.CollectionUtils; +import org.dbsyncer.common.util.StringUtil; +import org.dbsyncer.connector.sqlite.validator.SQLiteConfigValidator; +import org.dbsyncer.sdk.config.CommandConfig; +import org.dbsyncer.sdk.config.DatabaseConfig; +import org.dbsyncer.sdk.config.ReaderConfig; +import org.dbsyncer.sdk.connector.ConfigValidator; +import org.dbsyncer.sdk.connector.database.AbstractDatabaseConnector; +import org.dbsyncer.sdk.connector.database.DatabaseConnectorInstance; +import org.dbsyncer.sdk.constant.DatabaseConstant; +import org.dbsyncer.sdk.enums.ListenerTypeEnum; +import org.dbsyncer.sdk.enums.TableTypeEnum; +import org.dbsyncer.sdk.listener.DatabaseQuartzListener; +import org.dbsyncer.sdk.listener.Listener; +import org.dbsyncer.sdk.model.Field; +import org.dbsyncer.sdk.model.PageSql; +import org.dbsyncer.sdk.model.Table; +import org.dbsyncer.sdk.util.PrimaryKeyUtil; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + + +/** + * SQLite连接器实现 + * @Author bble + * @Version 1.0.0 + * @Date 2023-11-28 16:22 + */ +public final class SQLiteConnector extends AbstractDatabaseConnector { + + private final String QUERY_VIEW = "SELECT name FROM sqlite_master WHERE type = 'view'"; + private final String QUERY_TABLE = "SELECT name FROM sqlite_master WHERE type='table'"; + + private final String TYPE = "SQLite"; + private final SQLiteConfigValidator configValidator = new SQLiteConfigValidator(); + + @Override + public String getConnectorType() { + return TYPE; + } + + @Override + public ConfigValidator getConfigValidator() { + return configValidator; + } + + @Override + public List
    getTable(DatabaseConnectorInstance connectorInstance) { + DatabaseConfig config = connectorInstance.getConfig(); + List
    tables = getTables(connectorInstance, String.format(QUERY_TABLE, config.getSchema()), TableTypeEnum.TABLE); + tables.addAll(getTables(connectorInstance, QUERY_VIEW, TableTypeEnum.VIEW)); + return tables; + } + + @Override + public Listener getListener(String listenerType) { + if (ListenerTypeEnum.isTiming(listenerType)) { + return new DatabaseQuartzListener(); + } + return null; + } + + @Override + public String getPageSql(PageSql config) { + // select * from "my_user" where "id" > ? and "uid" > ? order by "id","uid" limit ? OFFSET ? + StringBuilder sql = new StringBuilder(config.getQuerySql()); + if (PrimaryKeyUtil.isSupportedCursor(config.getFields())) { + appendOrderByPk(config, sql); + } + sql.append(DatabaseConstant.SQLITE_PAGE_SQL); + return sql.toString(); + } + + @Override + public Object[] getPageArgs(ReaderConfig config) { + int pageIndex = config.getPageIndex(); + int pageSize = config.getPageSize(); + return new Object[]{pageSize, (pageIndex - 1) * pageSize}; + } + + @Override + public String buildTableName(String tableName) { + return convertKey(tableName); + } + + @Override + public String buildFieldName(Field field) { + return convertKey(field.getName()); + } + + @Override + public List buildPrimaryKeys(List primaryKeys) { + if (CollectionUtils.isEmpty(primaryKeys)) { + return primaryKeys; + } + return primaryKeys.stream().map(pk -> convertKey(pk)).collect(Collectors.toList()); + } + + private List
    getTables(DatabaseConnectorInstance connectorInstance, String sql, TableTypeEnum type) { + List tableNames = connectorInstance.execute(databaseTemplate -> databaseTemplate.queryForList(sql, String.class)); + if (!CollectionUtils.isEmpty(tableNames)) { + return tableNames.stream().map(name -> new Table(name, type.getCode())).collect(Collectors.toList()); + } + return new ArrayList<>(); + } + + private String convertKey(String key) { + return new StringBuilder("\"").append(key).append("\"").toString(); + } + +} \ No newline at end of file diff --git a/dbsyncer-connector/dbsyncer-connector-sqlite/src/main/java/org/dbsyncer/connector/sqlite/SQLiteException.java b/dbsyncer-connector/dbsyncer-connector-sqlite/src/main/java/org/dbsyncer/connector/sqlite/SQLiteException.java new file mode 100644 index 00000000..30568beb --- /dev/null +++ b/dbsyncer-connector/dbsyncer-connector-sqlite/src/main/java/org/dbsyncer/connector/sqlite/SQLiteException.java @@ -0,0 +1,30 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.connector.sqlite; + +/** + * @Author bble + * @Version 1.0.0 + * @Date 2023-11-28 16:22 + */ +public class SQLiteException extends RuntimeException { + + private static final long serialVersionUID = 1L; + + public SQLiteException(String message) { + super(message); + } + + public SQLiteException(String message, Throwable cause) { + super(message, cause); + } + + public SQLiteException(Throwable cause) { + super(cause); + } + + protected SQLiteException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } +} diff --git a/dbsyncer-connector/dbsyncer-connector-sqlite/src/main/java/org/dbsyncer/connector/sqlite/validator/DqlSQLiteConfigValidator.java b/dbsyncer-connector/dbsyncer-connector-sqlite/src/main/java/org/dbsyncer/connector/sqlite/validator/DqlSQLiteConfigValidator.java new file mode 100644 index 00000000..f74b80f6 --- /dev/null +++ b/dbsyncer-connector/dbsyncer-connector-sqlite/src/main/java/org/dbsyncer/connector/sqlite/validator/DqlSQLiteConfigValidator.java @@ -0,0 +1,27 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ + +package org.dbsyncer.connector.sqlite.validator; + +import org.dbsyncer.sdk.config.DatabaseConfig; +import org.dbsyncer.sdk.connector.AbstractDataBaseConfigValidator; + +import java.util.Map; + +/** + * SQLite连接配置校验器实现 + * + * @Author bble + * @Version 1.0.0 + * @Date 2023-11-28 16:22 + */ +public class DqlSQLiteConfigValidator extends AbstractDataBaseConfigValidator { + + @Override + public void modify(DatabaseConfig connectorConfig, Map params) { + super.modify(connectorConfig, params); + super.modifyDql(connectorConfig, params); + super.modifySchema(connectorConfig, params); + } +} \ No newline at end of file diff --git a/dbsyncer-connector/dbsyncer-connector-sqlite/src/main/java/org/dbsyncer/connector/sqlite/validator/SQLiteConfigValidator.java b/dbsyncer-connector/dbsyncer-connector-sqlite/src/main/java/org/dbsyncer/connector/sqlite/validator/SQLiteConfigValidator.java new file mode 100644 index 00000000..d8cec034 --- /dev/null +++ b/dbsyncer-connector/dbsyncer-connector-sqlite/src/main/java/org/dbsyncer/connector/sqlite/validator/SQLiteConfigValidator.java @@ -0,0 +1,24 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.connector.sqlite.validator; + +import org.dbsyncer.sdk.config.DatabaseConfig; +import org.dbsyncer.sdk.connector.AbstractDataBaseConfigValidator; + +import java.util.Map; + +/** + * SQLite连接配置校验器实现 + * + * @Author bble + * @Version 1.0.0 + * @Date 2023-11-28 16:22 + */ +public class SQLiteConfigValidator extends AbstractDataBaseConfigValidator { + @Override + public void modify(DatabaseConfig connectorConfig, Map params) { + super.modify(connectorConfig, params); + super.modifySchema(connectorConfig, params); + } +} \ No newline at end of file diff --git a/dbsyncer-connector/dbsyncer-connector-sqlite/src/main/resources/META-INF/services/org.dbsyncer.sdk.spi.ConnectorService b/dbsyncer-connector/dbsyncer-connector-sqlite/src/main/resources/META-INF/services/org.dbsyncer.sdk.spi.ConnectorService new file mode 100644 index 00000000..b44ab43b --- /dev/null +++ b/dbsyncer-connector/dbsyncer-connector-sqlite/src/main/resources/META-INF/services/org.dbsyncer.sdk.spi.ConnectorService @@ -0,0 +1,2 @@ +org.dbsyncer.connector.sqlite.SQLiteConnector +org.dbsyncer.connector.sqlite.DqlSQLiteConnector \ No newline at end of file diff --git a/dbsyncer-connector/dbsyncer-connector-sqlite/src/main/resources/public/connector/addDqlSQLite.html b/dbsyncer-connector/dbsyncer-connector-sqlite/src/main/resources/public/connector/addDqlSQLite.html new file mode 100644 index 00000000..cab4001a --- /dev/null +++ b/dbsyncer-connector/dbsyncer-connector-sqlite/src/main/resources/public/connector/addDqlSQLite.html @@ -0,0 +1,46 @@ + + + +
    +
    + + + +
    + +
    +
    + +
    + +
    + +
    +
    +
    + +
    + +
    +
    + + +
    + +
    + +
    + +
    +
    + + +
    + + \ No newline at end of file diff --git a/dbsyncer-connector/dbsyncer-connector-sqlite/src/main/resources/public/connector/addSQLite.html b/dbsyncer-connector/dbsyncer-connector-sqlite/src/main/resources/public/connector/addSQLite.html new file mode 100644 index 00000000..d7d5614e --- /dev/null +++ b/dbsyncer-connector/dbsyncer-connector-sqlite/src/main/resources/public/connector/addSQLite.html @@ -0,0 +1,29 @@ + + + +
    +
    + + + +
    + +
    +
    + +
    + +
    + +
    +
    +
    + +
    + +
    +
    +
    + + \ No newline at end of file diff --git a/dbsyncer-connector/dbsyncer-connector-sqlite/src/main/resources/static/img/sqlite.png b/dbsyncer-connector/dbsyncer-connector-sqlite/src/main/resources/static/img/sqlite.png new file mode 100644 index 0000000000000000000000000000000000000000..6fcf7196037d145557774df339fa7cb08816a730 GIT binary patch literal 2449 zcmV;C32yd@P)@@Qk{5BI*mv$#|eYqp?)VX{>{o-`cSY1jcOQobuo{K6#cAimy@`z_JAm81o=i zSExDQaDs=jC!G znb^hZXed}y6ij%B!yYsSrRj#E>UG_EUEa*grD`HUh|<>`Fwq#C(;hU`Yuo5YNO~_> ztJPebGuY*kWMZ2hpVuxlng*4c(pi$JT}6>S90$@!rl%qSzinvXDNXMrHCZ`0*fR`0 zHrSUD3OZa%QA3d)yU?hFHyuTk{%-)@vhCJJHCR-bo5YK@_WDzt7D% z^W`em``oVKUOUl9n$}6Oxs0Z%Bhj!>t)ipAdLSF>4cUmMrs^F;&E>KPX&57Um`C&> z0iGY~4|I9#LL=)Pdm2&E3PP4)=u_d)dSZ{fK-1T!#_fRe8jS+2n%-6_l2k6`!I&-= zpUjai!sl_%jg8odM$+}N)?S2iQL0rVe*a!x&~+V)V*}B%{k?XgF^me#&`t`KvMh_e zL7*v+zLjCl4-K3OciV^tys2vKMU-JU9qRYcw#-YCXI(OvZtB5Pk=_z5y@Ijdq-4Ft`y)Mn50^hqTPWX zU%rUS@FKDIDxLXeWgW`M>#Fv2CkffyJHStvNapjG$^GAbeed8TozM~4PsgWHEzmUM zb@QkX(O}@S%RkTz{Y7FQCF1Vtx~!ULH0{|=>gi4jHyaa8%0h{|&hP(mFIz0142^CO zK$d9r37MnEQ6w+VWlLpM*I|SHp1`x6qIf@M!O zNo&^a>k?}cmNk?Wt727>k40mu*IGnZ6-8HNpNr23B@K>Kl-{6kc{?#SNP;}t?YAz7 z=-Bh+^2G4KNMB_9R3GT_@Z~nP#bTTx?0?e<6)Hd@9fO84$aS-gz!3`Q;)@>~@|^RB z?~S=QZZ9X`62m(kqtS?iWra!=uDLKg=;1j$L`QZe8eWU1@}-KCWwE!z{ImvcsFJLj z6SeLA(7gG=Rfq#qjQWJk5jBd`9|~?~vL2rQ>HC+^fF2ghR*8UXtgp{^Q&%U>G4-%V z6?>W5hdF7pj5I5Q*(8%{z=UITv<=}-i)e=RD*_5s_+WDE!tj8DVM0Fdzn0h5lY5{x zOVjCG0SARs8asA-J&1LbSj?L<6QH1RtJHu*B~h8mSRBGb1mnJyT}?D^d#C}3;V72l z91geB`NfUP$jCTrY@qLt-#+^G`34q3Su&gRd0Z^RG_-OK7HMfWlWReP`~=bztF@}6 z++W{Z-bx5&w^SCThc9$cIUw{W}`isn52F841# zx`GbN+H`(=^o#2YFH@PX?k>W}$OtUcS}0hSY0lJjlr+n-pcI&nZXcG$CvdE8Ck8@6 z+^LZ>>%8e8PQEJ|^k18dfhE?ar=#JUGZToQ`!6<8ExBz8gAeeO$ec;cp%_5h;S&@LRRIN3? zj0Gi8#67ST4Nbdw4xbqK1l8~faEJ_~j2|t`po?OtBZu45H5BRQ>o-eif_ISjU>-n3 zEWqG6mQ|RnfwF)bAo4=XRy0Lpp9UIW0m(rx{0H`2yAXqaNk%xLLBMGMx$zYxDP?wm zVD#kaL4bqH(IY1ga*Z`OE?7Ji2^mcWN6f>PmTAqe1{CXN*%xJbdSnPpKyVR=&Ua|o z7(Bc4f9OuJsrh|PP4%qgMH?hy7jU5zccv>inm(HDbu*|2kQ+Tqp6-WL+ zUWVMBo5t>YxdQ6L(q?=uk^cMRr|1GQg;KUyDv1*8y|SHHi>G@7e&or&JzV+A_bbGM zKAs&9yE{1xRi+sz&IB}QZZi2ITn;qQ`sv&ho-A+0^8`)$c|3hjR*n7?!3!84dh;F8bwc8~N}|uo);^b9-(I z3JKbvwt!g{Ox@7H|^x+XrMXIB2|30Ff-5-M0j4G z8nfy%vdD)s=TAp^?%ZF3U#?D^wTKVhU*5iXZZJ9&=|TMmPtpDSynr_CPm50ws|Xz^L(T|W=rxUj*5dQ47g0Az3i!YNXnG79*r3HDM=oZ;Q?LZVFCN*AcN+f% Date: Thu, 30 Nov 2023 09:23:23 +0000 Subject: [PATCH 44/98] =?UTF-8?q?=E6=94=AF=E6=8C=81MySQL=E5=8F=8C=E5=90=91?= =?UTF-8?q?=E5=90=8C=E6=AD=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../connector/mysql/DQLMySQLConnector.java | 5 +++ .../connector/mysql/MySQLConnector.java | 5 +++ .../mysql/binlog/BinaryLogClient.java | 15 -------- .../mysql/binlog/BinaryLogRemoteClient.java | 27 ++++--------- .../connector/mysql/cdc/MySQLListener.java | 38 +++++++++++++------ .../database/AbstractDatabaseConnector.java | 4 +- .../sdk/connector/database/Database.java | 9 +++++ .../database/sqlbuilder/SqlBuilderDelete.java | 3 +- .../database/sqlbuilder/SqlBuilderInsert.java | 3 +- .../database/sqlbuilder/SqlBuilderUpdate.java | 2 +- .../sdk/constant/DatabaseConstant.java | 5 +++ .../src/main/test/LuceneFactoryTest.java | 2 +- 12 files changed, 67 insertions(+), 51 deletions(-) diff --git a/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/DQLMySQLConnector.java b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/DQLMySQLConnector.java index bb877528..d5c16dab 100644 --- a/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/DQLMySQLConnector.java +++ b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/DQLMySQLConnector.java @@ -67,4 +67,9 @@ public final class DQLMySQLConnector extends AbstractDQLConnector { } return null; } + + @Override + public String generateUniqueCode() { + return DatabaseConstant.DBS_UNIQUE_CODE; + } } \ No newline at end of file diff --git a/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/MySQLConnector.java b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/MySQLConnector.java index 8d22b57f..3e6471d9 100644 --- a/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/MySQLConnector.java +++ b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/MySQLConnector.java @@ -56,6 +56,11 @@ public final class MySQLConnector extends AbstractDatabaseConnector { return null; } + @Override + public String generateUniqueCode() { + return DatabaseConstant.DBS_UNIQUE_CODE; + } + @Override public String buildSqlWithQuotation() { return "`"; diff --git a/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/binlog/BinaryLogClient.java b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/binlog/BinaryLogClient.java index 147adb10..cc8f8554 100644 --- a/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/binlog/BinaryLogClient.java +++ b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/binlog/BinaryLogClient.java @@ -95,21 +95,6 @@ public interface BinaryLogClient { */ void setTableMapEventByTableId(Map tableMapEventByTableId); - /** - * 是否支持ddl - * - * @return - */ - boolean isEnableDDL(); - - /** - *

    true: ROTATE > FORMAT_DESCRIPTION > TABLE_MAP > WRITE_ROWS > UPDATE_ROWS > DELETE_ROWS > XID - *

    false: Support all events - * - * @param enableDDL - */ - void setEnableDDL(boolean enableDDL); - /** * binlog-parser-127.0.0.1_3306_1 * diff --git a/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/binlog/BinaryLogRemoteClient.java b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/binlog/BinaryLogRemoteClient.java index daeefe02..90adbb1c 100644 --- a/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/binlog/BinaryLogRemoteClient.java +++ b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/binlog/BinaryLogRemoteClient.java @@ -77,8 +77,7 @@ public class BinaryLogRemoteClient implements BinaryLogClient { private EventDeserializer eventDeserializer; private Map tableMapEventByTableId; private boolean blocking = true; - private boolean enableDDL = false; - private long serverId = 65535; + private long serverId = 65535L; private volatile String binlogFilename; private volatile long binlogPosition = 4; private volatile long connectionId; @@ -585,16 +584,12 @@ public class BinaryLogRemoteClient implements BinaryLogClient { eventDataDeserializers.put(EventType.EXT_UPDATE_ROWS, (new UpdateDeserializer(tableMapEventByTableId)).setMayContainExtraInformation(true)); eventDataDeserializers.put(EventType.EXT_DELETE_ROWS, (new DeleteDeserializer(tableMapEventByTableId)).setMayContainExtraInformation(true)); eventDataDeserializers.put(EventType.XID, new XidEventDataDeserializer()); - - if (enableDDL) { - eventDataDeserializers.put(EventType.INTVAR, new IntVarEventDataDeserializer()); - eventDataDeserializers.put(EventType.QUERY, new QueryEventDataDeserializer()); - eventDataDeserializers.put(EventType.ROWS_QUERY, new RowsQueryEventDataDeserializer()); - eventDataDeserializers.put(EventType.GTID, new GtidEventDataDeserializer()); - eventDataDeserializers.put(EventType.PREVIOUS_GTIDS, new PreviousGtidSetDeserializer()); - eventDataDeserializers.put(EventType.XA_PREPARE, new XAPrepareEventDataDeserializer()); - } - + eventDataDeserializers.put(EventType.INTVAR, new IntVarEventDataDeserializer()); + eventDataDeserializers.put(EventType.QUERY, new QueryEventDataDeserializer()); + eventDataDeserializers.put(EventType.ROWS_QUERY, new RowsQueryEventDataDeserializer()); + eventDataDeserializers.put(EventType.GTID, new GtidEventDataDeserializer()); + eventDataDeserializers.put(EventType.PREVIOUS_GTIDS, new PreviousGtidSetDeserializer()); + eventDataDeserializers.put(EventType.XA_PREPARE, new XAPrepareEventDataDeserializer()); } private void notifyEventListeners(Event event) { @@ -713,14 +708,6 @@ public class BinaryLogRemoteClient implements BinaryLogClient { this.tableMapEventByTableId = tableMapEventByTableId; } - public boolean isEnableDDL() { - return enableDDL; - } - - public void setEnableDDL(boolean enableDDL) { - this.enableDDL = enableDDL; - } - @Override public String getWorkerThreadName() { return workerThreadName; diff --git a/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/cdc/MySQLListener.java b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/cdc/MySQLListener.java index f72ec2b1..8d18a2f1 100644 --- a/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/cdc/MySQLListener.java +++ b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/cdc/MySQLListener.java @@ -10,6 +10,7 @@ import com.github.shyiko.mysql.binlog.event.EventHeaderV4; import com.github.shyiko.mysql.binlog.event.EventType; import com.github.shyiko.mysql.binlog.event.QueryEventData; import com.github.shyiko.mysql.binlog.event.RotateEventData; +import com.github.shyiko.mysql.binlog.event.RowsQueryEventData; import com.github.shyiko.mysql.binlog.event.TableMapEventData; import com.github.shyiko.mysql.binlog.event.UpdateRowsEventData; import com.github.shyiko.mysql.binlog.event.WriteRowsEventData; @@ -18,11 +19,12 @@ import net.sf.jsqlparser.JSQLParserException; import net.sf.jsqlparser.parser.CCJSqlParserUtil; import net.sf.jsqlparser.statement.alter.Alter; import org.dbsyncer.common.util.StringUtil; +import org.dbsyncer.connector.mysql.MySQLException; import org.dbsyncer.connector.mysql.binlog.BinaryLogClient; import org.dbsyncer.connector.mysql.binlog.BinaryLogRemoteClient; -import org.dbsyncer.connector.mysql.MySQLException; import org.dbsyncer.sdk.config.DatabaseConfig; import org.dbsyncer.sdk.constant.ConnectorConstant; +import org.dbsyncer.sdk.constant.DatabaseConstant; import org.dbsyncer.sdk.listener.AbstractDatabaseListener; import org.dbsyncer.sdk.listener.event.DDLChangedEvent; import org.dbsyncer.sdk.listener.event.RowChangedEvent; @@ -118,12 +120,11 @@ public class MySQLListener extends AbstractDatabaseListener { final String password = config.getPassword(); boolean containsPos = snapshot.containsKey(BINLOG_POSITION); client = new BinaryLogRemoteClient(host.getIp(), host.getPort(), username, password); - client.setEnableDDL(true); client.setBinlogFilename(snapshot.get(BINLOG_FILENAME)); client.setBinlogPosition(containsPos ? Long.parseLong(snapshot.get(BINLOG_POSITION)) : 0); client.setTableMapEventByTableId(tables); - client.registerEventListener(new MysqlEventListener()); - client.registerLifecycleListener(new MysqlLifecycleListener()); + client.registerEventListener(new InnerEventListener()); + client.registerLifecycleListener(new InnerLifecycleListener()); client.connect(); @@ -227,7 +228,7 @@ public class MySQLListener extends AbstractDatabaseListener { } } - final class MysqlLifecycleListener implements BinaryLogRemoteClient.LifecycleListener { + final class InnerLifecycleListener implements BinaryLogRemoteClient.LifecycleListener { @Override public void onConnect(BinaryLogRemoteClient client) { @@ -272,7 +273,12 @@ public class MySQLListener extends AbstractDatabaseListener { } - final class MysqlEventListener implements BinaryLogRemoteClient.EventListener { + final class InnerEventListener implements BinaryLogRemoteClient.EventListener { + + /** + * 只处理非dbs写入事件（单线程消费，不存在并发竞争） + */ + private boolean notUniqueCodeEvent = true; @Override public void onEvent(Event event) { @@ -283,7 +289,13 @@ public class MySQLListener extends AbstractDatabaseListener { return; } - if (EventType.isUpdate(header.getEventType())) { + if (header.getEventType() == EventType.ROWS_QUERY) { + RowsQueryEventData data = event.getData(); + notUniqueCodeEvent = isNotUniqueCodeEvent(data.getQuery()); + return; + } + + if (notUniqueCodeEvent && EventType.isUpdate(header.getEventType())) { refresh(header); UpdateRowsEventData data = event.getData(); if (isFilterTable(data.getTableId())) { @@ -294,7 +306,7 @@ public class MySQLListener extends AbstractDatabaseListener { } return; } - if (EventType.isWrite(header.getEventType())) { + if (notUniqueCodeEvent && EventType.isWrite(header.getEventType())) { refresh(header); WriteRowsEventData data = event.getData(); if (isFilterTable(data.getTableId())) { @@ -305,7 +317,7 @@ public class MySQLListener extends AbstractDatabaseListener { } return; } - if (EventType.isDelete(header.getEventType())) { + if (notUniqueCodeEvent && EventType.isDelete(header.getEventType())) { refresh(header); DeleteRowsEventData data = event.getData(); if (isFilterTable(data.getTableId())) { @@ -317,7 +329,7 @@ public class MySQLListener extends AbstractDatabaseListener { return; } - if (client.isEnableDDL() && EventType.QUERY == header.getEventType()) { + if (EventType.QUERY == header.getEventType()) { refresh(header); parseDDL(event.getData()); return; @@ -331,7 +343,7 @@ public class MySQLListener extends AbstractDatabaseListener { } private void parseDDL(QueryEventData data) { - if (StringUtil.startsWith(data.getSql(), ConnectorConstant.OPERTION_ALTER)) { + if (isNotUniqueCodeEvent(data.getSql()) && StringUtil.startsWith(data.getSql(), ConnectorConstant.OPERTION_ALTER)) { try { // ALTER TABLE `test`.`my_user` MODIFY COLUMN `name` varchar(128) CHARACTER SET utf8 COLLATE utf8_bin NULL DEFAULT NULL AFTER `id` Alter alter = (Alter) CCJSqlParserUtil.parse(data.getSql()); @@ -359,6 +371,10 @@ public class MySQLListener extends AbstractDatabaseListener { return StringUtil.equalsIgnoreCase(database, dbName) && filterTable.contains(tableName); } + private boolean isNotUniqueCodeEvent(String sql){ + return !StringUtil.startsWith(sql, DatabaseConstant.DBS_UNIQUE_CODE); + } + } } \ No newline at end of file diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/AbstractDatabaseConnector.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/AbstractDatabaseConnector.java index e0c086d7..411e22fe 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/AbstractDatabaseConnector.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/AbstractDatabaseConnector.java @@ -17,6 +17,7 @@ import org.dbsyncer.sdk.connector.AbstractConnector; import org.dbsyncer.sdk.connector.ConnectorInstance; import org.dbsyncer.sdk.connector.database.ds.SimpleConnection; import org.dbsyncer.sdk.constant.ConnectorConstant; +import org.dbsyncer.sdk.constant.DatabaseConstant; import org.dbsyncer.sdk.enums.OperationEnum; import org.dbsyncer.sdk.enums.SqlBuilderEnum; import org.dbsyncer.sdk.enums.TableTypeEnum; @@ -666,7 +667,8 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector implem try { Assert.hasText(config.getSql(), "执行SQL语句不能为空."); connectorInstance.execute(databaseTemplate -> { - databaseTemplate.execute(config.getSql()); + // 执行ddl时, 带上dbs唯一标识码，防止双向同步导致死循环 + databaseTemplate.execute(DatabaseConstant.DBS_UNIQUE_CODE.concat(config.getSql())); return true; }); } catch (Exception e) { diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/Database.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/Database.java index ccd89e18..102adcb3 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/Database.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/Database.java @@ -10,6 +10,15 @@ import java.util.List; public interface Database { + /** + * 获取dbs唯一标识码 + * + * @return + */ + default String generateUniqueCode() { + return StringUtil.EMPTY; + } + /** * 查询语句表名和字段带上引号（默认不加） * diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderDelete.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderDelete.java index 60e20a5f..3bf201f5 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderDelete.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderDelete.java @@ -21,7 +21,8 @@ public class SqlBuilderDelete extends AbstractSqlBuilder { String tableName = config.getTableName(); List primaryKeys = database.buildPrimaryKeys(config.getPrimaryKeys()); // DELETE FROM "USER" WHERE "ID"=? AND "UID" = ? - StringBuilder sql = new StringBuilder().append("DELETE FROM ").append(config.getSchema()); + StringBuilder sql = new StringBuilder(database.generateUniqueCode()); + sql.append("DELETE FROM ").append(config.getSchema()); sql.append(quotation); sql.append(database.buildTableName(tableName)); sql.append(quotation); diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderInsert.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderInsert.java index 67635148..4b69000f 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderInsert.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderInsert.java @@ -37,7 +37,8 @@ public class SqlBuilderInsert extends AbstractSqlBuilder { } } // INSERT INTO "USER"("USERNAME","AGE") VALUES (?,?) - StringBuilder sql = new StringBuilder("INSERT INTO "); + StringBuilder sql = new StringBuilder(database.generateUniqueCode()); + sql.append("INSERT INTO "); sql.append(config.getSchema()); sql.append(quotation); sql.append(database.buildTableName(config.getTableName())); diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderUpdate.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderUpdate.java index 934affd1..bdd98acf 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderUpdate.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderUpdate.java @@ -22,7 +22,7 @@ public class SqlBuilderUpdate extends AbstractSqlBuilder { String quotation = database.buildSqlWithQuotation(); List fields = config.getFields(); - StringBuilder sql = new StringBuilder(); + StringBuilder sql = new StringBuilder(database.generateUniqueCode()); sql.append("UPDATE ").append(config.getSchema()); sql.append(quotation); sql.append(database.buildTableName(config.getTableName())); diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/constant/DatabaseConstant.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/constant/DatabaseConstant.java index a49db55d..4742fa67 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/constant/DatabaseConstant.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/constant/DatabaseConstant.java @@ -2,6 +2,11 @@ package org.dbsyncer.sdk.constant; public class DatabaseConstant { + /** + * dbs唯一标识码 + */ + public static final String DBS_UNIQUE_CODE = "/*dbs*/"; + //*********************************** Mysql **************************************// /** * Mysql分页语句 diff --git a/dbsyncer-storage/src/main/test/LuceneFactoryTest.java b/dbsyncer-storage/src/main/test/LuceneFactoryTest.java index 0da49fc1..e218935a 100644 --- a/dbsyncer-storage/src/main/test/LuceneFactoryTest.java +++ b/dbsyncer-storage/src/main/test/LuceneFactoryTest.java @@ -34,12 +34,12 @@ import org.apache.lucene.search.highlight.QueryScorer; import org.apache.lucene.search.highlight.SimpleHTMLFormatter; import org.apache.lucene.util.BytesRef; import org.dbsyncer.common.model.Paging; -import org.dbsyncer.common.snowflake.SnowflakeIdWorker; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.RandomUtil; import org.dbsyncer.sdk.constant.ConnectorConstant; import org.dbsyncer.storage.constant.ConfigConstant; import org.dbsyncer.storage.enums.StorageDataStatusEnum; +import org.dbsyncer.storage.impl.SnowflakeIdWorker; import org.dbsyncer.storage.lucene.Option; import org.dbsyncer.storage.lucene.Shard; import org.dbsyncer.storage.util.BinlogMessageUtil; -- Gitee From a46fb3fb00f5fd8e444f2fd1e6d189d18837fe46 Mon Sep 17 00:00:00 2001 From: bble <1058519553@qq.com> Date: Thu, 30 Nov 2023 17:47:57 +0800 Subject: [PATCH 45/98] add sqlite pom --- dbsyncer-connector/pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/dbsyncer-connector/pom.xml b/dbsyncer-connector/pom.xml index acb9e4ee..b6d3379f 100644 --- a/dbsyncer-connector/pom.xml +++ b/dbsyncer-connector/pom.xml @@ -19,5 +19,6 @@ dbsyncer-connector-oracle dbsyncer-connector-sqlserver dbsyncer-connector-postgresql + dbsyncer-connector-sqlite \ No newline at end of file -- Gitee From c4f2b5c5ae94af256dfaec8a385607d23bc6f55a Mon Sep 17 00:00:00 2001 From: AE86 <836391306@qq.com> Date: Fri, 1 Dec 2023 00:18:15 +0800 Subject: [PATCH 46/98] revert --- .../connector/file/FileBitValueMapper.java | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/FileBitValueMapper.java diff --git a/dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/FileBitValueMapper.java b/dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/FileBitValueMapper.java new file mode 100644 index 00000000..34a8410b --- /dev/null +++ b/dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/FileBitValueMapper.java @@ -0,0 +1,26 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ +package org.dbsyncer.connector.file; + +import org.dbsyncer.sdk.connector.AbstractValueMapper; +import org.dbsyncer.sdk.connector.ConnectorInstance; + +/** + * @author AE86 + * @version 1.0.0 + * @date 2023/11/15 0:07 + */ +public class FileBitValueMapper extends AbstractValueMapper { + + @Override + protected Integer convert(ConnectorInstance connectorInstance, Object val) { + if (val instanceof Boolean) { + Boolean b = (Boolean) val; + return b ? 1 : 0; + } + + throw new FileException(String.format("%s can not find type [%s], val [%s]", getClass().getSimpleName(), val.getClass(), val)); + } + +} \ No newline at end of file -- Gitee From ec8e681ef04c740482b0515155a0795333bf6d8b Mon Sep 17 00:00:00 2001 From: AE86 <836391306@qq.com> Date: Fri, 1 Dec 2023 00:19:11 +0800 Subject: [PATCH 47/98] rm pom --- dbsyncer-cache/pom.xml | 0 dbsyncer-monitor/pom.xml | 0 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 dbsyncer-cache/pom.xml delete mode 100644 dbsyncer-monitor/pom.xml diff --git a/dbsyncer-cache/pom.xml b/dbsyncer-cache/pom.xml deleted file mode 100644 index e69de29b..00000000 diff --git a/dbsyncer-monitor/pom.xml b/dbsyncer-monitor/pom.xml deleted file mode 100644 index e69de29b..00000000 -- Gitee From 6c3a2e4e1d5a0c9494e5dc60a5c766c295441819 Mon Sep 17 00:00:00 2001 From: AE86 <836391306@qq.com> Date: Fri, 1 Dec 2023 00:23:08 +0800 Subject: [PATCH 48/98] add license --- .../org/dbsyncer/sdk/connector/AbstractConnector.java | 3 +++ .../org/dbsyncer/sdk/connector/AbstractValueMapper.java | 3 +++ .../org/dbsyncer/sdk/connector/ConnectorInstance.java | 3 +++ .../java/org/dbsyncer/sdk/connector/ValueMapper.java | 3 +++ .../sdk/connector/database/AbstractDQLConnector.java | 3 +++ .../sdk/connector/database/AbstractSqlBuilder.java | 3 +++ .../org/dbsyncer/sdk/connector/database/Database.java | 3 +++ .../connector/database/DatabaseConnectorInstance.java | 3 +++ .../sdk/connector/database/DatabaseTemplate.java | 3 +++ .../dbsyncer/sdk/connector/database/HandleCallback.java | 3 +++ .../sdk/connector/database/ds/SimpleConnection.java | 3 +++ .../sdk/connector/database/ds/SimpleDataSource.java | 9 ++++++--- .../sdk/connector/database/sqlbuilder/SqlBuilder.java | 3 +++ .../connector/database/sqlbuilder/SqlBuilderDelete.java | 3 +++ .../connector/database/sqlbuilder/SqlBuilderInsert.java | 3 +++ .../connector/database/sqlbuilder/SqlBuilderQuery.java | 3 +++ .../database/sqlbuilder/SqlBuilderQueryCount.java | 3 +++ .../database/sqlbuilder/SqlBuilderQueryCursor.java | 3 +++ .../database/sqlbuilder/SqlBuilderQueryExist.java | 3 +++ .../connector/database/sqlbuilder/SqlBuilderUpdate.java | 3 +++ 20 files changed, 63 insertions(+), 3 deletions(-) diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/AbstractConnector.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/AbstractConnector.java index b330cd08..0ab1a5b6 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/AbstractConnector.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/AbstractConnector.java @@ -1,3 +1,6 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.sdk.connector; import org.dbsyncer.common.util.CollectionUtils; diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/AbstractValueMapper.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/AbstractValueMapper.java index f7de26ea..f98d6531 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/AbstractValueMapper.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/AbstractValueMapper.java @@ -1,3 +1,6 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.sdk.connector; import java.lang.reflect.ParameterizedType; diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/ConnectorInstance.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/ConnectorInstance.java index ae48f12a..f1c4f4f4 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/ConnectorInstance.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/ConnectorInstance.java @@ -1,3 +1,6 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.sdk.connector; import org.dbsyncer.sdk.model.ConnectorConfig; diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/ValueMapper.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/ValueMapper.java index 176494a2..f89dad6d 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/ValueMapper.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/ValueMapper.java @@ -1,3 +1,6 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.sdk.connector; /** diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/AbstractDQLConnector.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/AbstractDQLConnector.java index cf76a765..7f5f0daa 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/AbstractDQLConnector.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/AbstractDQLConnector.java @@ -1,3 +1,6 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.sdk.connector.database; import org.dbsyncer.common.util.CollectionUtils; diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/AbstractSqlBuilder.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/AbstractSqlBuilder.java index e15ec10e..a96b7c16 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/AbstractSqlBuilder.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/AbstractSqlBuilder.java @@ -1,3 +1,6 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.sdk.connector.database; import org.dbsyncer.sdk.SdkException; diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/Database.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/Database.java index 102adcb3..0c653fa3 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/Database.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/Database.java @@ -1,3 +1,6 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.sdk.connector.database; import org.dbsyncer.common.util.StringUtil; diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/DatabaseConnectorInstance.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/DatabaseConnectorInstance.java index f161cfd7..573a0fd2 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/DatabaseConnectorInstance.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/DatabaseConnectorInstance.java @@ -1,3 +1,6 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.sdk.connector.database; import org.dbsyncer.sdk.SdkException; diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/DatabaseTemplate.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/DatabaseTemplate.java index f169ff78..0e053b1f 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/DatabaseTemplate.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/DatabaseTemplate.java @@ -1,3 +1,6 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.sdk.connector.database; import org.dbsyncer.sdk.SdkException; diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/HandleCallback.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/HandleCallback.java index 970c879e..57e00f8e 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/HandleCallback.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/HandleCallback.java @@ -1,3 +1,6 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.sdk.connector.database; public interface HandleCallback { diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/ds/SimpleConnection.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/ds/SimpleConnection.java index fa8326bb..f9f58b43 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/ds/SimpleConnection.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/ds/SimpleConnection.java @@ -1,3 +1,6 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.sdk.connector.database.ds; import org.dbsyncer.sdk.util.DatabaseUtil; diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/ds/SimpleDataSource.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/ds/SimpleDataSource.java index f4fc33f4..12596cbd 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/ds/SimpleDataSource.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/ds/SimpleDataSource.java @@ -1,3 +1,6 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.sdk.connector.database.ds; import org.dbsyncer.common.util.StringUtil; @@ -66,7 +69,7 @@ public class SimpleDataSource implements DataSource, AutoCloseable { //等待3秒 TimeUnit.SECONDS.sleep(MAX_WAIT_SECONDS); if (activeNum.get() >= MAX_IDLE) { - throw new ConnectorException(String.format("数据库连接数超过上限%d，url=%s", MAX_IDLE, url)); + throw new SdkException(String.format("数据库连接数超过上限%d，url=%s", MAX_IDLE, url)); } } SimpleConnection poll = pool.poll(); @@ -84,7 +87,7 @@ public class SimpleDataSource implements DataSource, AutoCloseable { } return poll; } catch (InterruptedException e) { - throw new ConnectorException(e); + throw new SdkException(e); } finally { lock.unlock(); } @@ -174,7 +177,7 @@ public class SimpleDataSource implements DataSource, AutoCloseable { simpleConnection = new SimpleConnection(DatabaseUtil.getConnection(driverClassName, url, username, password), StringUtil.equals(driverClassName, "oracle.jdbc.OracleDriver")); activeNum.incrementAndGet(); } catch (SQLException e) { - throw new ConnectorException(e); + throw new SdkException(e); } return simpleConnection; } diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilder.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilder.java index 209405f1..210c351c 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilder.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilder.java @@ -1,3 +1,6 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.sdk.connector.database.sqlbuilder; import org.dbsyncer.sdk.config.SqlBuilderConfig; diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderDelete.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderDelete.java index 3bf201f5..97697852 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderDelete.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderDelete.java @@ -1,3 +1,6 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.sdk.connector.database.sqlbuilder; import org.dbsyncer.sdk.config.SqlBuilderConfig; diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderInsert.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderInsert.java index 4b69000f..4cf47c40 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderInsert.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderInsert.java @@ -1,3 +1,6 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.sdk.connector.database.sqlbuilder; import org.dbsyncer.sdk.config.SqlBuilderConfig; diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderQuery.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderQuery.java index 41116bc7..ec8ca18e 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderQuery.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderQuery.java @@ -1,3 +1,6 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.sdk.connector.database.sqlbuilder; import org.dbsyncer.common.util.StringUtil; diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderQueryCount.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderQueryCount.java index dc01f1d2..d41d86a3 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderQueryCount.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderQueryCount.java @@ -1,3 +1,6 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.sdk.connector.database.sqlbuilder; import org.dbsyncer.common.util.StringUtil; diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderQueryCursor.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderQueryCursor.java index e33aa073..fc0be1c7 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderQueryCursor.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderQueryCursor.java @@ -1,3 +1,6 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.sdk.connector.database.sqlbuilder; import org.dbsyncer.sdk.config.SqlBuilderConfig; diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderQueryExist.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderQueryExist.java index d72f33dd..ab3dfbc1 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderQueryExist.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderQueryExist.java @@ -1,3 +1,6 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.sdk.connector.database.sqlbuilder; import org.dbsyncer.sdk.config.SqlBuilderConfig; diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderUpdate.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderUpdate.java index bdd98acf..3268297c 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderUpdate.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/sqlbuilder/SqlBuilderUpdate.java @@ -1,3 +1,6 @@ +/** + * DBSyncer Copyright 2020-2023 All Rights Reserved. + */ package org.dbsyncer.sdk.connector.database.sqlbuilder; import org.dbsyncer.common.util.StringUtil; -- Gitee From ef487a4fbadbbc7069f878b94a72cbb176c7de9f Mon Sep 17 00:00:00 2001 From: AE86 <836391306@qq.com> Date: Fri, 1 Dec 2023 00:28:07 +0800 Subject: [PATCH 49/98] fix code --- .../org/dbsyncer/biz/impl/MappingServiceImpl.java | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MappingServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MappingServiceImpl.java index 691ff1c8..904d7044 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MappingServiceImpl.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MappingServiceImpl.java @@ -9,11 +9,10 @@ import org.dbsyncer.biz.checker.impl.mapping.MappingChecker; import org.dbsyncer.biz.vo.ConnectorVo; import org.dbsyncer.biz.vo.MappingVo; import org.dbsyncer.biz.vo.MetaVo; -import org.dbsyncer.common.snowflake.SnowflakeIdWorker; -import org.dbsyncer.common.spi.ConnectorMapper; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.common.util.JsonUtil; import org.dbsyncer.common.util.StringUtil; +import org.dbsyncer.connector.ConnectorFactory; import org.dbsyncer.manager.ManagerFactory; import org.dbsyncer.parser.LogType; import org.dbsyncer.parser.ProfileComponent; @@ -22,6 +21,7 @@ import org.dbsyncer.parser.model.Connector; import org.dbsyncer.parser.model.Mapping; import org.dbsyncer.parser.model.Meta; import org.dbsyncer.parser.model.TableGroup; +import org.dbsyncer.sdk.connector.ConnectorInstance; import org.dbsyncer.sdk.enums.ModelEnum; import org.dbsyncer.sdk.model.Table; import org.dbsyncer.storage.constant.ConfigConstant; @@ -68,6 +68,9 @@ public class MappingServiceImpl extends BaseServiceImpl implements MappingServic @Resource private ManagerFactory managerFactory; + @Resource + private ConnectorFactory connectorFactory; + @Override public String add(Map params) { ConfigModel model = mappingChecker.checkAddConfigModel(params); @@ -289,10 +292,10 @@ public class MappingServiceImpl extends BaseServiceImpl implements MappingServic @Override public void refreshTables(Connector connector) { // 刷新数据表 - ConnectorMapper connectorMapper = manager.connect(connector.getConfig()); - List

    table = manager.getTable(connectorMapper); + ConnectorInstance connectorInstance = connectorFactory.connect(connector.getConfig()); + List
    table = connectorFactory.getTable(connectorInstance); connector.setTable(table); - manager.editConfigModel(connector); + profileComponent.editConfigModel(connector); } } \ No newline at end of file -- Gitee From c9088eaf0c9c99772535dd966c3457f1a5567b6e Mon Sep 17 00:00:00 2001 From: AE86 <836391306@qq.com> Date: Fri, 1 Dec 2023 00:33:51 +0800 Subject: [PATCH 50/98] fix pom --- .../dbsyncer-connector-base/pom.xml | 23 ++++++++++--------- dbsyncer-connector/pom.xml | 2 +- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/dbsyncer-connector/dbsyncer-connector-base/pom.xml b/dbsyncer-connector/dbsyncer-connector-base/pom.xml index b436a54c..37f8cfdf 100644 --- a/dbsyncer-connector/dbsyncer-connector-base/pom.xml +++ b/dbsyncer-connector/dbsyncer-connector-base/pom.xml @@ -24,7 +24,6 @@ org.ghi dbsyncer-connector-elasticsearch ${project.parent.version} - provided @@ -32,7 +31,6 @@ org.ghi dbsyncer-connector-file ${project.parent.version} - provided @@ -40,7 +38,6 @@ org.ghi dbsyncer-connector-kafka ${project.parent.version} - provided @@ -48,7 +45,6 @@ org.ghi dbsyncer-connector-mysql ${project.parent.version} - provided @@ -56,23 +52,28 @@ org.ghi dbsyncer-connector-oracle ${project.parent.version} - provided - + org.ghi - dbsyncer-connector-sqlserver + dbsyncer-connector-postgresql ${project.parent.version} - provided - + org.ghi - dbsyncer-connector-postgresql + dbsyncer-connector-sqlite + ${project.parent.version} + + + + + + org.ghi + dbsyncer-connector-sqlserver ${project.parent.version} - provided diff --git a/dbsyncer-connector/pom.xml b/dbsyncer-connector/pom.xml index b6d3379f..075571f0 100644 --- a/dbsyncer-connector/pom.xml +++ b/dbsyncer-connector/pom.xml @@ -17,8 +17,8 @@ dbsyncer-connector-kafka dbsyncer-connector-mysql dbsyncer-connector-oracle - dbsyncer-connector-sqlserver dbsyncer-connector-postgresql dbsyncer-connector-sqlite + dbsyncer-connector-sqlserver \ No newline at end of file -- Gitee From d3d6848e49be7bbca4bc4f5513c16d0c33838a35 Mon Sep 17 00:00:00 2001 From: AE86 <836391306@qq.com> Date: Fri, 1 Dec 2023 00:50:14 +0800 Subject: [PATCH 51/98] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=88=B7=E6=96=B0?= =?UTF-8?q?=E8=A1=A8=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/org/dbsyncer/biz/MappingService.java | 6 ++--- .../dbsyncer/biz/impl/MappingServiceImpl.java | 27 ++++++++++++------- .../controller/index/MappingController.java | 6 +---- .../resources/public/mapping/editTable.html | 4 +-- .../main/resources/static/js/mapping/edit.js | 4 +-- 5 files changed, 26 insertions(+), 21 deletions(-) diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/MappingService.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/MappingService.java index 204d3b9e..307a3e98 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/MappingService.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/MappingService.java @@ -71,10 +71,10 @@ public interface MappingService { String stop(String id); /** - * 刷新表 + * 刷新驱动数据源和目标源表 * - * @param connector + * @param id */ - void refreshTables(Connector connector); + String refreshMappingTables(String id); } \ No newline at end of file diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MappingServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MappingServiceImpl.java index 904d7044..80207014 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MappingServiceImpl.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MappingServiceImpl.java @@ -212,6 +212,24 @@ public class MappingServiceImpl extends BaseServiceImpl implements MappingServic return "驱动停止成功"; } + @Override + public String refreshMappingTables(String id) { + Mapping mapping = profileComponent.getMapping(id); + Assert.notNull(mapping, "The mapping id is invalid."); + updateConnectorTables(mapping.getSourceConnectorId()); + updateConnectorTables(mapping.getTargetConnectorId()); + return "刷新驱动表成功"; + } + + private void updateConnectorTables(String connectorId) { + Connector connector = profileComponent.getConnector(connectorId); + Assert.notNull(connector, "The connector id is invalid."); + // 刷新数据表 + ConnectorInstance connectorInstance = connectorFactory.connect(connector.getConfig()); + connector.setTable(connectorFactory.getTable(connectorInstance)); + profileComponent.editConfigModel(connector); + } + private MappingVo convertMapping2Vo(Mapping mapping) { String model = mapping.getModel(); Assert.notNull(mapping, "Mapping can not be null."); @@ -289,13 +307,4 @@ public class MappingServiceImpl extends BaseServiceImpl implements MappingServic } } - @Override - public void refreshTables(Connector connector) { - // 刷新数据表 - ConnectorInstance connectorInstance = connectorFactory.connect(connector.getConfig()); - List
    table = connectorFactory.getTable(connectorInstance); - connector.setTable(table); - profileComponent.editConfigModel(connector); - } - } \ No newline at end of file diff --git a/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/index/MappingController.java b/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/index/MappingController.java index 7ea955d7..6020feeb 100644 --- a/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/index/MappingController.java +++ b/dbsyncer-web/src/main/java/org/dbsyncer/web/controller/index/MappingController.java @@ -3,7 +3,6 @@ package org.dbsyncer.web.controller.index; import org.dbsyncer.biz.ConnectorService; import org.dbsyncer.biz.MappingService; import org.dbsyncer.biz.TableGroupService; -import org.dbsyncer.biz.vo.MappingVo; import org.dbsyncer.biz.vo.RestResult; import org.dbsyncer.web.controller.BaseController; import org.slf4j.Logger; @@ -144,10 +143,7 @@ public class MappingController extends BaseController { @ResponseBody public RestResult refreshTables(@RequestParam(value = "id") String id) { try { - MappingVo mapping = mappingService.getMapping(id); - mappingService.refreshTables(mapping.getSourceConnector()); - mappingService.refreshTables(mapping.getTargetConnector()); - return RestResult.restSuccess(id); + return RestResult.restSuccess(mappingService.refreshMappingTables(id)); } catch (Exception e) { logger.error(e.getLocalizedMessage(), e); return RestResult.restFail(e.getMessage()); diff --git a/dbsyncer-web/src/main/resources/public/mapping/editTable.html b/dbsyncer-web/src/main/resources/public/mapping/editTable.html index 90deae2e..cacf53c6 100644 --- a/dbsyncer-web/src/main/resources/public/mapping/editTable.html +++ b/dbsyncer-web/src/main/resources/public/mapping/editTable.html @@ -57,8 +57,8 @@
      -
    diff --git a/dbsyncer-web/src/main/resources/static/js/mapping/edit.js b/dbsyncer-web/src/main/resources/static/js/mapping/edit.js index 017b3898..efd006a6 100644 --- a/dbsyncer-web/src/main/resources/static/js/mapping/edit.js +++ b/dbsyncer-web/src/main/resources/static/js/mapping/edit.js @@ -201,14 +201,14 @@ function mappingModifyName(){ }); } -// 绑定刷新表字段事件 +// 绑定刷新表事件 function bindRefreshTablesClick() { let $refreshBtn = $("#refreshTableBtn"); $refreshBtn.bind('click', function(){ let id = $(this).attr("tableGroupId"); doPoster("/mapping/refreshTables", {'id': id}, function (data) { if (data.success == true) { - bootGrowl("刷新数据表成功!", "success"); + bootGrowl("刷新表成功!", "success"); doLoader('/mapping/page/edit?id=' + id); } else { bootGrowl(data.resultValue, "danger"); -- Gitee From 63ebb61c7923a33896799c14d99f3ddb1a17b5e1 Mon Sep 17 00:00:00 2001 From: AE86 <836391306@qq.com> Date: Tue, 5 Dec 2023 02:01:31 +0800 Subject: [PATCH 52/98] =?UTF-8?q?=E6=80=A7=E8=83=BD=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../org/dbsyncer/biz/DataSyncService.java | 2 +- .../biz/impl/DataSyncServiceImpl.java | 57 ++++++------ .../common/QueueOverflowException.java | 27 ++++++ .../mysql/binlog/BinaryLogRemoteClient.java | 75 ++++++--------- .../connector/mysql/cdc/MySQLListener.java | 92 ++++++++----------- .../test/java/BinaryLogRemoteClientTest.java | 7 +- .../connector/oracle/cdc/OracleListener.java | 18 +++- .../oracle/dcn/DBChangeNotification.java | 4 + .../postgresql/cdc/PostgreSQLListener.java | 14 ++- .../sqlserver/cdc/SqlServerListener.java | 22 ++++- .../parser/flush/AbstractBufferActuator.java | 43 +++------ .../flush/impl/GeneralBufferActuator.java | 7 ++ .../flush/impl/StorageBufferActuator.java | 29 ++++++ .../flush/impl/TableGroupBufferActuator.java | 1 - .../src/main/resources/application.properties | 12 +-- 15 files changed, 228 insertions(+), 182 deletions(-) create mode 100644 dbsyncer-common/src/main/java/org/dbsyncer/common/QueueOverflowException.java diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/DataSyncService.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/DataSyncService.java index c51f796c..2d6f9d4f 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/DataSyncService.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/DataSyncService.java @@ -32,6 +32,6 @@ public interface DataSyncService { * @param params * @return */ - String sync(Map params); + String sync(Map params) throws InvalidProtocolBufferException; } \ No newline at end of file diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/DataSyncServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/DataSyncServiceImpl.java index 7ea55694..45ed37e1 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/DataSyncServiceImpl.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/DataSyncServiceImpl.java @@ -139,42 +139,39 @@ public class DataSyncServiceImpl implements DataSyncService { } @Override - public String sync(Map params) { + public String sync(Map params) throws InvalidProtocolBufferException { String metaId = params.get("metaId"); String messageId = params.get("messageId"); Assert.hasText(metaId, "The metaId is null."); Assert.hasText(messageId, "The messageId is null."); - try { - Map row = getData(metaId, messageId); - Map binlogData = getBinlogData(row, false); - if (CollectionUtils.isEmpty(binlogData)) { - return messageId; - } - String tableGroupId = (String) row.get(ConfigConstant.DATA_TABLE_GROUP_ID); - String event = (String) row.get(ConfigConstant.DATA_EVENT); - // 有修改同步值 - String retryDataParams = params.get("retryDataParams"); - if (StringUtil.isNotBlank(retryDataParams)) { - JsonUtil.parseMap(retryDataParams).forEach((k, v) -> binlogData.put(k, convertValue(binlogData.get(k), (String) v))); - } - TableGroup tableGroup = profileComponent.getTableGroup(tableGroupId); - String sourceTableName = tableGroup.getSourceTable().getName(); - RowChangedEvent changedEvent = new RowChangedEvent(sourceTableName, event, Collections.EMPTY_LIST); - // 转换为源字段 - final Picker picker = new Picker(tableGroup.getFieldMapping()); - changedEvent.setChangedRow(picker.pickSourceData(binlogData)); - bufferActuatorRouter.execute(metaId, tableGroupId, changedEvent); - storageService.remove(StorageEnum.DATA, metaId, messageId); - // 更新失败数 - Meta meta = profileComponent.getMeta(metaId); - Assert.notNull(meta, "Meta can not be null."); - meta.getFail().decrementAndGet(); - meta.setUpdateTime(Instant.now().toEpochMilli()); - profileComponent.editConfigModel(meta); - } catch (Exception e) { - logger.error(e.getLocalizedMessage()); + Map row = getData(metaId, messageId); + Map binlogData = getBinlogData(row, false); + if (CollectionUtils.isEmpty(binlogData)) { + return messageId; } + String tableGroupId = (String) row.get(ConfigConstant.DATA_TABLE_GROUP_ID); + String event = (String) row.get(ConfigConstant.DATA_EVENT); + // 有修改同步值 + String retryDataParams = params.get("retryDataParams"); + if (StringUtil.isNotBlank(retryDataParams)) { + JsonUtil.parseMap(retryDataParams).forEach((k, v) -> binlogData.put(k, convertValue(binlogData.get(k), (String) v))); + } + TableGroup tableGroup = profileComponent.getTableGroup(tableGroupId); + String sourceTableName = tableGroup.getSourceTable().getName(); + RowChangedEvent changedEvent = new RowChangedEvent(sourceTableName, event, Collections.EMPTY_LIST); + // 转换为源字段 + final Picker picker = new Picker(tableGroup.getFieldMapping()); + changedEvent.setChangedRow(picker.pickSourceData(binlogData)); + // 执行同步是否成功 + bufferActuatorRouter.execute(metaId, tableGroupId, changedEvent); + storageService.remove(StorageEnum.DATA, metaId, messageId); + // 更新失败数 + Meta meta = profileComponent.getMeta(metaId); + Assert.notNull(meta, "Meta can not be null."); + meta.getFail().decrementAndGet(); + meta.setUpdateTime(Instant.now().toEpochMilli()); + profileComponent.editConfigModel(meta); return messageId; } diff --git a/dbsyncer-common/src/main/java/org/dbsyncer/common/QueueOverflowException.java b/dbsyncer-common/src/main/java/org/dbsyncer/common/QueueOverflowException.java new file mode 100644 index 00000000..91cc9806 --- /dev/null +++ b/dbsyncer-common/src/main/java/org/dbsyncer/common/QueueOverflowException.java @@ -0,0 +1,27 @@ +package org.dbsyncer.common; + +/** + * @author AE86 + * @version 1.0.0 + * @date 2023/12/04 22:39 + */ +public class QueueOverflowException extends RuntimeException { + + private static final long serialVersionUID = 1L; + + public QueueOverflowException(String message) { + super(message); + } + + public QueueOverflowException(String message, Throwable cause) { + super(message, cause); + } + + public QueueOverflowException(Throwable cause) { + super(cause); + } + + protected QueueOverflowException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } +} diff --git a/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/binlog/BinaryLogRemoteClient.java b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/binlog/BinaryLogRemoteClient.java index 90adbb1c..9c247e24 100644 --- a/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/binlog/BinaryLogRemoteClient.java +++ b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/binlog/BinaryLogRemoteClient.java @@ -11,7 +11,6 @@ import com.github.shyiko.mysql.binlog.io.ByteArrayInputStream; import com.github.shyiko.mysql.binlog.network.*; import com.github.shyiko.mysql.binlog.network.protocol.*; import com.github.shyiko.mysql.binlog.network.protocol.command.*; -import org.dbsyncer.connector.mysql.MySQLException; import org.dbsyncer.connector.mysql.deserializer.DeleteDeserializer; import org.dbsyncer.connector.mysql.deserializer.UpdateDeserializer; import org.dbsyncer.connector.mysql.deserializer.WriteDeserializer; @@ -26,7 +25,6 @@ import java.io.IOException; import java.net.InetSocketAddress; import java.net.ServerSocket; import java.net.Socket; -import java.net.SocketException; import java.security.GeneralSecurityException; import java.security.cert.X509Certificate; import java.util.*; @@ -83,6 +81,7 @@ public class BinaryLogRemoteClient implements BinaryLogClient { private volatile long connectionId; private volatile PacketChannel channel; private volatile boolean connected; + private volatile boolean connectedError; private Thread worker; private Thread keepAlive; private String workerThreadName; @@ -282,30 +281,17 @@ public class BinaryLogRemoteClient implements BinaryLogClient { if (marker == 0xFE && !blocking) { break; } - Event event; - try { - event = eventDeserializer.nextEvent(packetLength == MAX_PACKET_LENGTH ? - new ByteArrayInputStream(readPacketSplitInChunks(inputStream, packetLength - 1)) : - inputStream); - if (event == null) { - throw new EOFException(); - } - } catch (Exception e) { - Throwable cause = e instanceof EventDataDeserializationException ? e.getCause() : e; - if (cause instanceof EOFException || cause instanceof SocketException) { - throw e; - } - lifecycleListeners.forEach(listener -> listener.onEventDeserializationFailure(this, e)); - continue; - } - if (connected) { + Event event = eventDeserializer.nextEvent(packetLength == MAX_PACKET_LENGTH ? new ByteArrayInputStream(readPacketSplitInChunks(inputStream, packetLength - 1)) : inputStream); + if (event != null) { updateGtidSet(event); notifyEventListeners(event); updateClientBinlogFilenameAndPosition(event); + continue; } + throw new EOFException("event data deserialization exception"); } } catch (Exception e) { - lifecycleListeners.forEach(listener -> listener.onCommunicationFailure(this, e)); + notifyException(e); } } @@ -631,32 +617,27 @@ public class BinaryLogRemoteClient implements BinaryLogClient { } private void spawnKeepAliveThread() { - long keepAliveInterval = TimeUnit.SECONDS.toMillis(30); String clientId = createClientId(); this.keepAlive = new Thread(() -> { while (connected) { try { - Thread.sleep(keepAliveInterval); - } catch (InterruptedException e) { - // expected in case of disconnect - } - boolean connectionLost = false; - try { + TimeUnit.SECONDS.sleep(30); channel.write(new PingCommand()); - } catch (IOException e) { - connectionLost = true; + } catch (Exception e) { + notifyException(e); + break; } - if (connectionLost) { - if (connected) { - String error = String.format("keepalive: Trying to restore lost connection to %s", clientId); - logger.info(error); - try { - lifecycleListeners.forEach(listener -> listener.onCommunicationFailure(this, new MySQLException(error))); - } catch (Exception e) { - logger.warn("keepalive error", e); - } - } + } + while (connectedError) { + try { + logger.info("Trying to restore lost connection to {}}", createClientId()); + TimeUnit.MILLISECONDS.sleep(500); + disconnect(); + connect(); + connectedError = false; break; + } catch (Exception e) { + logger.error(e.getMessage(), e); } } }); @@ -665,6 +646,14 @@ public class BinaryLogRemoteClient implements BinaryLogClient { this.keepAlive.start(); } + private void notifyException(Exception e) { + if (connected) { + logger.error(e.getMessage(), e); + lifecycleListeners.forEach(listener -> listener.onException(this, e)); + connectedError = true; + } + } + @Override public String getBinlogFilename() { return binlogFilename; @@ -810,13 +799,7 @@ public class BinaryLogRemoteClient implements BinaryLogClient { /** * It's guarantied to be called before {@link #onDisconnect(BinaryLogRemoteClient)}) in case of communication failure. */ - void onCommunicationFailure(BinaryLogRemoteClient client, Exception ex); - - /** - * Called in case of failed event deserialization. Note this type of error does NOT cause client to disconnect. If you wish to stop - * receiving events you'll need to fire client.disconnect() manually. - */ - void onEventDeserializationFailure(BinaryLogRemoteClient client, Exception ex); + void onException(BinaryLogRemoteClient client, Exception ex); /** * Called upon disconnect (regardless of the reason). diff --git a/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/cdc/MySQLListener.java b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/cdc/MySQLListener.java index 8d18a2f1..134106fb 100644 --- a/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/cdc/MySQLListener.java +++ b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/cdc/MySQLListener.java @@ -18,6 +18,7 @@ import com.github.shyiko.mysql.binlog.network.ServerException; import net.sf.jsqlparser.JSQLParserException; import net.sf.jsqlparser.parser.CCJSqlParserUtil; import net.sf.jsqlparser.statement.alter.Alter; +import org.dbsyncer.common.QueueOverflowException; import org.dbsyncer.common.util.StringUtil; import org.dbsyncer.connector.mysql.MySQLException; import org.dbsyncer.connector.mysql.binlog.BinaryLogClient; @@ -34,6 +35,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.util.Assert; +import java.time.Instant; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -58,26 +60,22 @@ public class MySQLListener extends AbstractDatabaseListener { private final String BINLOG_FILENAME = "fileName"; private final String BINLOG_POSITION = "position"; - private final int RETRY_TIMES = 10; private final int MASTER = 0; private Map tables = new HashMap<>(); private BinaryLogClient client; private List cluster; private String database; private final Lock connectLock = new ReentrantLock(); - private volatile boolean connected; - private volatile boolean recovery; @Override public void start() { try { connectLock.lock(); - if (connected) { + if (client != null && client.isConnected()) { logger.error("MySQLExtractor is already started"); return; } run(); - connected = true; } catch (Exception e) { logger.error("启动失败:{}", e.getMessage()); throw new MySQLException(e); @@ -90,8 +88,7 @@ public class MySQLListener extends AbstractDatabaseListener { public void close() { try { connectLock.lock(); - connected = false; - if (null != client) { + if (client != null && client.isConnected()) { client.disconnect(); } } catch (Exception e) { @@ -154,43 +151,6 @@ public class MySQLListener extends AbstractDatabaseListener { return cluster; } - private void reStart() { - try { - connectLock.lock(); - if (recovery) { - return; - } - recovery = true; - } finally { - connectLock.unlock(); - } - - for (int i = 1; i <= RETRY_TIMES; i++) { - try { - if (null != client) { - client.disconnect(); - } - run(); - - errorEvent(new MySQLException(String.format("重启成功, %s", client.getWorkerThreadName()))); - logger.error("第{}次重启成功, ThreadName:{} ", i, client.getWorkerThreadName()); - recovery = false; - break; - } catch (Exception e) { - logger.error("第{}次重启异常, ThreadName:{}, {}", i, client.getWorkerThreadName(), e.getMessage()); - // 无法连接，关闭任务 - if (i == RETRY_TIMES) { - errorEvent(new MySQLException(String.format("重启异常, %s, %s", client.getWorkerThreadName(), e.getMessage()))); - } - } - try { - TimeUnit.SECONDS.sleep(i * 2); - } catch (InterruptedException e) { - logger.error(e.getMessage()); - } - } - } - private void refresh(EventHeader header) { EventHeaderV4 eventHeaderV4 = (EventHeaderV4) header; refresh(null, eventHeaderV4.getNextPosition()); @@ -210,6 +170,32 @@ public class MySQLListener extends AbstractDatabaseListener { snapshot.put(BINLOG_POSITION, String.valueOf(nextPosition)); } + private void trySendEvent(RowChangedEvent event){ + try { + // 如果消费事件失败，重试 + long now = Instant.now().toEpochMilli(); + boolean isReTry = false; + while (client.isConnected()){ + try { + sendChangedEvent(event); + break; + } catch (QueueOverflowException e) { + isReTry = true; + try { + TimeUnit.MILLISECONDS.sleep(1); + } catch (InterruptedException ex) { + logger.error(ex.getMessage(), ex); + } + } + } + if (isReTry) { + logger.info("重试耗时：{}ms", Instant.now().toEpochMilli() - now); + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + final class Host { private String ip; private int port; @@ -237,11 +223,11 @@ public class MySQLListener extends AbstractDatabaseListener { } @Override - public void onCommunicationFailure(BinaryLogRemoteClient client, Exception e) { - if (!connected) { + public void onException(BinaryLogRemoteClient client, Exception e) { + if (!client.isConnected()) { return; } - logger.error(e.getMessage()); + errorEvent(new MySQLException(e.getMessage())); /** * e: * case1> Due to the automatic expiration and deletion mechanism of MySQL binlog files, the binlog file cannot be found. @@ -259,12 +245,6 @@ public class MySQLListener extends AbstractDatabaseListener { return; } } - - reStart(); - } - - @Override - public void onEventDeserializationFailure(BinaryLogRemoteClient client, Exception ex) { } @Override @@ -301,7 +281,7 @@ public class MySQLListener extends AbstractDatabaseListener { if (isFilterTable(data.getTableId())) { data.getRows().forEach(m -> { List after = Stream.of(m.getValue()).collect(Collectors.toList()); - sendChangedEvent(new RowChangedEvent(getTableName(data.getTableId()), ConnectorConstant.OPERTION_UPDATE, after, client.getBinlogFilename(), client.getBinlogPosition())); + trySendEvent(new RowChangedEvent(getTableName(data.getTableId()), ConnectorConstant.OPERTION_UPDATE, after, client.getBinlogFilename(), client.getBinlogPosition())); }); } return; @@ -312,7 +292,7 @@ public class MySQLListener extends AbstractDatabaseListener { if (isFilterTable(data.getTableId())) { data.getRows().forEach(m -> { List after = Stream.of(m).collect(Collectors.toList()); - sendChangedEvent(new RowChangedEvent(getTableName(data.getTableId()), ConnectorConstant.OPERTION_INSERT, after, client.getBinlogFilename(), client.getBinlogPosition())); + trySendEvent(new RowChangedEvent(getTableName(data.getTableId()), ConnectorConstant.OPERTION_INSERT, after, client.getBinlogFilename(), client.getBinlogPosition())); }); } return; @@ -323,7 +303,7 @@ public class MySQLListener extends AbstractDatabaseListener { if (isFilterTable(data.getTableId())) { data.getRows().forEach(m -> { List before = Stream.of(m).collect(Collectors.toList()); - sendChangedEvent(new RowChangedEvent(getTableName(data.getTableId()), ConnectorConstant.OPERTION_DELETE, before, client.getBinlogFilename(), client.getBinlogPosition())); + trySendEvent(new RowChangedEvent(getTableName(data.getTableId()), ConnectorConstant.OPERTION_DELETE, before, client.getBinlogFilename(), client.getBinlogPosition())); }); } return; diff --git a/dbsyncer-connector/dbsyncer-connector-mysql/src/test/java/BinaryLogRemoteClientTest.java b/dbsyncer-connector/dbsyncer-connector-mysql/src/test/java/BinaryLogRemoteClientTest.java index de5667fd..cafc1d5a 100644 --- a/dbsyncer-connector/dbsyncer-connector-mysql/src/test/java/BinaryLogRemoteClientTest.java +++ b/dbsyncer-connector/dbsyncer-connector-mysql/src/test/java/BinaryLogRemoteClientTest.java @@ -81,15 +81,10 @@ public class BinaryLogRemoteClientTest { } @Override - public void onCommunicationFailure(BinaryLogRemoteClient client, Exception ex) { + public void onException(BinaryLogRemoteClient client, Exception ex) { logger.error("连接异常", ex); } - @Override - public void onEventDeserializationFailure(BinaryLogRemoteClient client, Exception ex) { - logger.error("解析异常", ex); - } - @Override public void onDisconnect(BinaryLogRemoteClient client) { logger.error("断开连接"); diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java index 7a4fbd92..f828750f 100644 --- a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java +++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java @@ -3,6 +3,7 @@ */ package org.dbsyncer.connector.oracle.cdc; +import org.dbsyncer.common.QueueOverflowException; import org.dbsyncer.connector.oracle.OracleException; import org.dbsyncer.connector.oracle.dcn.DBChangeNotification; import org.dbsyncer.sdk.config.DatabaseConfig; @@ -10,6 +11,8 @@ import org.dbsyncer.sdk.listener.AbstractDatabaseListener; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.concurrent.TimeUnit; + /** * @Author AE86 * @Version 1.0.0 @@ -30,7 +33,20 @@ public class OracleListener extends AbstractDatabaseListener { String url = config.getUrl(); client = new DBChangeNotification(username, password, url); client.setFilterTable(filterTable); - client.addRowEventListener((e) -> sendChangedEvent(e)); + client.addRowEventListener((event) -> { + while (client.isConnected()){ + try { + sendChangedEvent(event); + break; + } catch (QueueOverflowException ex) { + try { + TimeUnit.MILLISECONDS.sleep(1); + } catch (InterruptedException exe) { + logger.error(exe.getMessage(), exe); + } + } + } + }); client.start(); } catch (Exception e) { logger.error("启动失败:{}", e.getMessage()); diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/dcn/DBChangeNotification.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/dcn/DBChangeNotification.java index 93661494..c4f3287e 100644 --- a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/dcn/DBChangeNotification.java +++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/dcn/DBChangeNotification.java @@ -130,6 +130,10 @@ public class DBChangeNotification { return conn; } + public boolean isConnected() { + return connected; + } + public void setFilterTable(Set filterTable) { this.filterTable = filterTable; } diff --git a/dbsyncer-connector/dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/cdc/PostgreSQLListener.java b/dbsyncer-connector/dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/cdc/PostgreSQLListener.java index 471d1ff7..0e8938a3 100644 --- a/dbsyncer-connector/dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/cdc/PostgreSQLListener.java +++ b/dbsyncer-connector/dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/cdc/PostgreSQLListener.java @@ -3,6 +3,7 @@ */ package org.dbsyncer.connector.postgresql.cdc; +import org.dbsyncer.common.QueueOverflowException; import org.dbsyncer.common.util.BooleanUtil; import org.dbsyncer.connector.postgresql.decoder.MessageDecoder; import org.dbsyncer.connector.postgresql.PostgreSQLException; @@ -286,7 +287,18 @@ public class PostgreSQLListener extends AbstractDatabaseListener { RowChangedEvent event = messageDecoder.processMessage(msg); if (event != null) { event.setPosition(lsn.asString()); - sendChangedEvent(event); + while (connected){ + try { + sendChangedEvent(event); + break; + } catch (QueueOverflowException ex) { + try { + TimeUnit.MILLISECONDS.sleep(1); + } catch (InterruptedException exe) { + logger.error(exe.getMessage(), exe); + } + } + } } // feedback diff --git a/dbsyncer-connector/dbsyncer-connector-sqlserver/src/main/java/org/dbsyncer/connector/sqlserver/cdc/SqlServerListener.java b/dbsyncer-connector/dbsyncer-connector-sqlserver/src/main/java/org/dbsyncer/connector/sqlserver/cdc/SqlServerListener.java index 198bcbee..1f231fb7 100644 --- a/dbsyncer-connector/dbsyncer-connector-sqlserver/src/main/java/org/dbsyncer/connector/sqlserver/cdc/SqlServerListener.java +++ b/dbsyncer-connector/dbsyncer-connector-sqlserver/src/main/java/org/dbsyncer/connector/sqlserver/cdc/SqlServerListener.java @@ -4,6 +4,7 @@ package org.dbsyncer.connector.sqlserver.cdc; import com.microsoft.sqlserver.jdbc.SQLServerException; +import org.dbsyncer.common.QueueOverflowException; import org.dbsyncer.common.util.CollectionUtils; import org.dbsyncer.connector.sqlserver.model.SqlServerChangeTable; import org.dbsyncer.connector.sqlserver.SqlServerException; @@ -271,23 +272,38 @@ public class SqlServerListener extends AbstractDatabaseListener { }); } + private void trySendEvent(RowChangedEvent event){ + while (connected){ + try { + sendChangedEvent(event); + break; + } catch (QueueOverflowException ex) { + try { + TimeUnit.MILLISECONDS.sleep(1); + } catch (InterruptedException exe) { + logger.error(exe.getMessage(), exe); + } + } + } + } + private void parseEvent(List list, Lsn stopLsn) { int size = list.size(); for (int i = 0; i < size; i++) { boolean isEnd = i == size - 1; CDCEvent event = list.get(i); if (TableOperationEnum.isUpdateAfter(event.getCode())) { - sendChangedEvent(new RowChangedEvent(event.getTableName(), ConnectorConstant.OPERTION_UPDATE, event.getRow(), null, (isEnd ? stopLsn : null))); + trySendEvent(new RowChangedEvent(event.getTableName(), ConnectorConstant.OPERTION_UPDATE, event.getRow(), null, (isEnd ? stopLsn : null))); continue; } if (TableOperationEnum.isInsert(event.getCode())) { - sendChangedEvent(new RowChangedEvent(event.getTableName(), ConnectorConstant.OPERTION_INSERT, event.getRow(), null, (isEnd ? stopLsn : null))); + trySendEvent(new RowChangedEvent(event.getTableName(), ConnectorConstant.OPERTION_INSERT, event.getRow(), null, (isEnd ? stopLsn : null))); continue; } if (TableOperationEnum.isDelete(event.getCode())) { - sendChangedEvent(new RowChangedEvent(event.getTableName(), ConnectorConstant.OPERTION_DELETE, event.getRow(), null, (isEnd ? stopLsn : null))); + trySendEvent(new RowChangedEvent(event.getTableName(), ConnectorConstant.OPERTION_DELETE, event.getRow(), null, (isEnd ? stopLsn : null))); } } } diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/AbstractBufferActuator.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/AbstractBufferActuator.java index eec99734..5d7676df 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/AbstractBufferActuator.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/AbstractBufferActuator.java @@ -12,17 +12,13 @@ import org.springframework.util.Assert; import javax.annotation.Resource; import java.lang.reflect.ParameterizedType; -import java.time.Duration; import java.time.Instant; -import java.time.temporal.ChronoUnit; import java.util.LinkedHashMap; import java.util.Map; import java.util.Queue; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; @@ -40,9 +36,6 @@ public abstract class AbstractBufferActuator responseClazz; private Lock taskLock; - private Lock queueLock; - private Condition isFull; - private final Duration offerInterval = Duration.of(500, ChronoUnit.MILLIS); private BufferActuatorConfig config; private BlockingQueue queue; @@ -71,7 +64,6 @@ public abstract class AbstractBufferActuator queue, Request request); + @Override public void offer(BufferRequest request) { - if (!queue.offer((Request) request)) { - try { - // 公平锁，有序执行，容量上限，阻塞重试 - queueLock.lock(); - while (isRunning(request) && !queue.offer((Request) request)) { - try { - isFull.await(offerInterval.toMillis(), TimeUnit.MILLISECONDS); - } catch (InterruptedException e) { - break; - } - } - } finally { - queueLock.unlock(); - } + if (queue.offer((Request) request)) { + return; } + offerFailed(queue, (Request) request); } @Override diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/GeneralBufferActuator.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/GeneralBufferActuator.java index ca613d31..0519e335 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/GeneralBufferActuator.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/GeneralBufferActuator.java @@ -1,5 +1,6 @@ package org.dbsyncer.parser.flush.impl; +import org.dbsyncer.common.QueueOverflowException; import org.dbsyncer.common.config.GeneralBufferConfig; import org.dbsyncer.common.model.Result; import org.dbsyncer.common.util.CollectionUtils; @@ -38,6 +39,7 @@ import javax.annotation.PostConstruct; import javax.annotation.Resource; import java.util.List; import java.util.Map; +import java.util.concurrent.BlockingQueue; import java.util.concurrent.Executor; /** @@ -159,6 +161,11 @@ public class GeneralBufferActuator extends AbstractBufferActuator queue, WriterRequest request) { + throw new QueueOverflowException("缓存队列已满"); + } + @Override public Executor getExecutor() { return generalExecutor; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/StorageBufferActuator.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/StorageBufferActuator.java index 75057b12..283c8337 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/StorageBufferActuator.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/StorageBufferActuator.java @@ -10,7 +10,14 @@ import org.springframework.stereotype.Component; import javax.annotation.PostConstruct; import javax.annotation.Resource; +import java.time.Duration; +import java.time.temporal.ChronoUnit; +import java.util.concurrent.BlockingQueue; import java.util.concurrent.Executor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Condition; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; /** * 持久化执行器 @@ -22,6 +29,10 @@ import java.util.concurrent.Executor; @Component public final class StorageBufferActuator extends AbstractBufferActuator { + private final Duration offerInterval = Duration.of(500, ChronoUnit.MILLIS); + private final Lock queueLock = new ReentrantLock(true); + private final Condition queueCondition = queueLock.newCondition(); + @Resource private StorageConfig storageConfig; @@ -53,6 +64,24 @@ public final class StorageBufferActuator extends AbstractBufferActuator queue, StorageRequest request) { + final Lock lock = queueLock; + try { + // 公平锁，有序执行，容量上限，阻塞重试 + lock.lock(); + while (isRunning(request) && !queue.offer(request)) { + try { + queueCondition.await(offerInterval.toMillis(), TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + break; + } + } + } finally { + lock.unlock(); + } + } + @Override public Executor getExecutor() { return storageExecutor; diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/TableGroupBufferActuator.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/TableGroupBufferActuator.java index bd372f86..efe407e7 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/TableGroupBufferActuator.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/TableGroupBufferActuator.java @@ -53,7 +53,6 @@ public final class TableGroupBufferActuator extends GeneralBufferActuator implem public void buildConfig() { super.setConfig(tableGroupBufferConfig); - super.buildLock(); super.buildQueueConfig(); taskKey = UUIDUtil.getUUID(); int coreSize = tableGroupBufferConfig.getThreadCoreSize(); diff --git a/dbsyncer-web/src/main/resources/application.properties b/dbsyncer-web/src/main/resources/application.properties index c33a57f1..675047f3 100644 --- a/dbsyncer-web/src/main/resources/application.properties +++ b/dbsyncer-web/src/main/resources/application.properties @@ -29,17 +29,17 @@ dbsyncer.parser.general.buffer-queue-capacity=60000 dbsyncer.parser.general.buffer-period-millisecond=300 # *********************** 表执行器配置 *********************** # 每个驱动最多可分配的表执行器个数 -dbsyncer.parser.table.group.max-buffer-actuator-size=10 +dbsyncer.parser.table.group.max-buffer-actuator-size=20 # [TableGroupBufferActuator]线程数 -dbsyncer.parser.table.group.thread-core-size=2 +dbsyncer.parser.table.group.thread-core-size=10 # [TableGroupBufferActuator]线程池队列 dbsyncer.parser.table.group.thread-queue-capacity=16 # [TableGroupBufferActuator]单次执行任务数 -dbsyncer.parser.table.group.buffer-writer-count=100 +dbsyncer.parser.table.group.buffer-writer-count=1000 # [TableGroupBufferActuator]每次消费缓存队列的任务数 -dbsyncer.parser.table.group.buffer-pull-count=1000 +dbsyncer.parser.table.group.buffer-pull-count=20000 # [TableGroupBufferActuator]缓存队列容量 -dbsyncer.parser.table.group.buffer-queue-capacity=10000 +dbsyncer.parser.table.group.buffer-queue-capacity=40000 # [TableGroupBufferActuator]定时消费缓存队列间隔(毫秒) dbsyncer.parser.table.group.buffer-period-millisecond=300 @@ -65,7 +65,7 @@ dbsyncer.storage.buffer-period-millisecond=300 # 是否记录全量数据（false-关闭; true-开启） dbsyncer.storage.write.full.enabled=false # 是否记录同步成功数据（false-关闭; true-开启） -dbsyncer.storage.write-success=true +dbsyncer.storage.write-success=false # 是否记录同步失败数据（false-关闭; true-开启） dbsyncer.storage.write-fail=true # 记录同步失败日志最大长度 -- Gitee From 61c710ea1543651ee868c0cdb171da2fcd64dd18 Mon Sep 17 00:00:00 2001 From: AE86 <836391306@qq.com> Date: Wed, 6 Dec 2023 02:03:11 +0800 Subject: [PATCH 53/98] =?UTF-8?q?=E6=80=A7=E8=83=BD=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../org/dbsyncer/biz/impl/MetricReporter.java | 2 +- .../common/config/GeneralBufferConfig.java | 15 +++++++- .../dbsyncer/common/config/StorageConfig.java | 15 +++++++- .../common/config/TableGroupBufferConfig.java | 13 +++++++ .../flush/impl/StorageBufferActuator.java | 37 ++++++++----------- .../flush/impl/TableGroupBufferActuator.java | 3 +- .../database/ds/SimpleDataSource.java | 14 +------ .../src/main/resources/application.properties | 24 +++++++----- 8 files changed, 75 insertions(+), 48 deletions(-) diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MetricReporter.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MetricReporter.java index 55f382d6..0b638f69 100644 --- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MetricReporter.java +++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MetricReporter.java @@ -234,7 +234,7 @@ public class MetricReporter implements ScheduledTaskJob { msg.append("堆积").append(StringUtil.COLON).append(info.getQueueUp()); msg.append(StringUtil.FORWARD_SLASH).append(bufferActuator.getQueueCapacity()).append(StringUtil.SPACE); msg.append(ThreadPoolMetricEnum.CORE_SIZE.getMetricName()).append(StringUtil.COLON).append(pool.getActiveCount()); - msg.append(StringUtil.FORWARD_SLASH).append(pool.getCorePoolSize()).append(StringUtil.SPACE); + msg.append(StringUtil.FORWARD_SLASH).append(pool.getMaximumPoolSize()).append(StringUtil.SPACE); msg.append(ThreadPoolMetricEnum.COMPLETED.getMetricName()).append(StringUtil.COLON).append(pool.getCompletedTaskCount()); info.setResponse(new MetricResponse(code, group, metricName, Arrays.asList(new Sample(StatisticEnum.COUNT.getTagValueRepresentation(), msg.toString())))); return info; diff --git a/dbsyncer-common/src/main/java/org/dbsyncer/common/config/GeneralBufferConfig.java b/dbsyncer-common/src/main/java/org/dbsyncer/common/config/GeneralBufferConfig.java index 60415921..4c6c875b 100644 --- a/dbsyncer-common/src/main/java/org/dbsyncer/common/config/GeneralBufferConfig.java +++ b/dbsyncer-common/src/main/java/org/dbsyncer/common/config/GeneralBufferConfig.java @@ -21,6 +21,11 @@ public class GeneralBufferConfig extends BufferActuatorConfig { */ private int threadCoreSize = Runtime.getRuntime().availableProcessors() * 2; + /** + * 最大工作线程数 + */ + private int maxThreadSize = 16; + /** * 工作线任务队列 */ @@ -28,7 +33,7 @@ public class GeneralBufferConfig extends BufferActuatorConfig { @Bean(name = "generalExecutor", destroyMethod = "shutdown") public ThreadPoolTaskExecutor generalExecutor() { - return ThreadPoolUtil.newThreadPoolTaskExecutor(threadCoreSize, threadCoreSize, threadQueueCapacity, 30, "GeneralExecutor-"); + return ThreadPoolUtil.newThreadPoolTaskExecutor(threadCoreSize, maxThreadSize, threadQueueCapacity, 30, "GeneralExecutor-"); } public int getThreadCoreSize() { @@ -39,6 +44,14 @@ public class GeneralBufferConfig extends BufferActuatorConfig { this.threadCoreSize = threadCoreSize; } + public int getMaxThreadSize() { + return maxThreadSize; + } + + public void setMaxThreadSize(int maxThreadSize) { + this.maxThreadSize = maxThreadSize; + } + public int getThreadQueueCapacity() { return threadQueueCapacity; } diff --git a/dbsyncer-common/src/main/java/org/dbsyncer/common/config/StorageConfig.java b/dbsyncer-common/src/main/java/org/dbsyncer/common/config/StorageConfig.java index 04bb0d79..bc41f668 100644 --- a/dbsyncer-common/src/main/java/org/dbsyncer/common/config/StorageConfig.java +++ b/dbsyncer-common/src/main/java/org/dbsyncer/common/config/StorageConfig.java @@ -22,6 +22,11 @@ public class StorageConfig extends BufferActuatorConfig { */ private int threadCoreSize = Runtime.getRuntime().availableProcessors(); + /** + * 最大工作线程数 + */ + private int maxThreadSize = 16; + /** * 工作线任务队列 */ @@ -44,7 +49,7 @@ public class StorageConfig extends BufferActuatorConfig { @Bean(name = "storageExecutor", destroyMethod = "shutdown") public ThreadPoolTaskExecutor storageExecutor() { - return ThreadPoolUtil.newThreadPoolTaskExecutor(threadCoreSize, threadCoreSize, threadQueueCapacity, 30, "StorageExecutor-"); + return ThreadPoolUtil.newThreadPoolTaskExecutor(threadCoreSize, maxThreadSize, threadQueueCapacity, 30, "StorageExecutor-"); } public int getThreadCoreSize() { @@ -55,6 +60,14 @@ public class StorageConfig extends BufferActuatorConfig { this.threadCoreSize = threadCoreSize; } + public int getMaxThreadSize() { + return maxThreadSize; + } + + public void setMaxThreadSize(int maxThreadSize) { + this.maxThreadSize = maxThreadSize; + } + public int getThreadQueueCapacity() { return threadQueueCapacity; } diff --git a/dbsyncer-common/src/main/java/org/dbsyncer/common/config/TableGroupBufferConfig.java b/dbsyncer-common/src/main/java/org/dbsyncer/common/config/TableGroupBufferConfig.java index fca9fcd8..560cad61 100644 --- a/dbsyncer-common/src/main/java/org/dbsyncer/common/config/TableGroupBufferConfig.java +++ b/dbsyncer-common/src/main/java/org/dbsyncer/common/config/TableGroupBufferConfig.java @@ -23,6 +23,11 @@ public class TableGroupBufferConfig extends BufferActuatorConfig { */ private int threadCoreSize = Runtime.getRuntime().availableProcessors() * 2; + /** + * 最大工作线程数 + */ + private int maxThreadSize = 10; + /** * 工作线任务队列 */ @@ -44,6 +49,14 @@ public class TableGroupBufferConfig extends BufferActuatorConfig { this.threadCoreSize = threadCoreSize; } + public int getMaxThreadSize() { + return maxThreadSize; + } + + public void setMaxThreadSize(int maxThreadSize) { + this.maxThreadSize = maxThreadSize; + } + public int getThreadQueueCapacity() { return threadQueueCapacity; } diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/StorageBufferActuator.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/StorageBufferActuator.java index 283c8337..d36d3545 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/StorageBufferActuator.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/StorageBufferActuator.java @@ -1,23 +1,22 @@ package org.dbsyncer.parser.flush.impl; import org.dbsyncer.common.config.StorageConfig; +import org.dbsyncer.parser.ProfileComponent; import org.dbsyncer.parser.flush.AbstractBufferActuator; +import org.dbsyncer.parser.model.Mapping; +import org.dbsyncer.parser.model.Meta; import org.dbsyncer.parser.model.StorageRequest; import org.dbsyncer.parser.model.StorageResponse; import org.dbsyncer.storage.StorageService; import org.dbsyncer.storage.enums.StorageEnum; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; import javax.annotation.PostConstruct; import javax.annotation.Resource; -import java.time.Duration; -import java.time.temporal.ChronoUnit; import java.util.concurrent.BlockingQueue; import java.util.concurrent.Executor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.Condition; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; /** * 持久化执行器 @@ -29,9 +28,7 @@ import java.util.concurrent.locks.ReentrantLock; @Component public final class StorageBufferActuator extends AbstractBufferActuator { - private final Duration offerInterval = Duration.of(500, ChronoUnit.MILLIS); - private final Lock queueLock = new ReentrantLock(true); - private final Condition queueCondition = queueLock.newCondition(); + private final Logger logger = LoggerFactory.getLogger(getClass()); @Resource private StorageConfig storageConfig; @@ -42,6 +39,9 @@ public final class StorageBufferActuator extends AbstractBufferActuator storageService.addBatch(StorageEnum.DATA, response.getMetaId(), response.getDataList())); } @Override protected void offerFailed(BlockingQueue queue, StorageRequest request) { - final Lock lock = queueLock; - try { - // 公平锁，有序执行，容量上限，阻塞重试 - lock.lock(); - while (isRunning(request) && !queue.offer(request)) { - try { - queueCondition.await(offerInterval.toMillis(), TimeUnit.MILLISECONDS); - } catch (InterruptedException e) { - break; - } + Meta meta = profileComponent.getMeta(request.getMetaId()); + if (meta != null) { + Mapping mapping = profileComponent.getMapping(meta.getMappingId()); + if (mapping != null) { + logger.info("{}, data={}", mapping.getName(), request.getRow()); } - } finally { - lock.unlock(); } } diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/TableGroupBufferActuator.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/TableGroupBufferActuator.java index efe407e7..9613dc7a 100644 --- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/TableGroupBufferActuator.java +++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/TableGroupBufferActuator.java @@ -56,9 +56,10 @@ public final class TableGroupBufferActuator extends GeneralBufferActuator implem super.buildQueueConfig(); taskKey = UUIDUtil.getUUID(); int coreSize = tableGroupBufferConfig.getThreadCoreSize(); + int maxSize = tableGroupBufferConfig.getMaxThreadSize(); int queueCapacity = tableGroupBufferConfig.getThreadQueueCapacity(); String threadNamePrefix = new StringBuilder("TableGroupExecutor-").append(tableGroupId).append(StringUtil.SYMBOL).toString(); - threadPoolTaskExecutor = ThreadPoolUtil.newThreadPoolTaskExecutor(coreSize, coreSize, queueCapacity, 30, threadNamePrefix); + threadPoolTaskExecutor = ThreadPoolUtil.newThreadPoolTaskExecutor(coreSize, maxSize, queueCapacity, 30, threadNamePrefix); running = true; scheduledTaskService.start(taskKey, tableGroupBufferConfig.getBufferPeriodMillisecond(), this); } diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/ds/SimpleDataSource.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/ds/SimpleDataSource.java index cdcaf67d..a8628f9c 100644 --- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/ds/SimpleDataSource.java +++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/ds/SimpleDataSource.java @@ -15,7 +15,6 @@ import java.sql.SQLFeatureNotSupportedException; import java.time.Instant; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.ReentrantLock; import java.util.logging.Logger; @@ -27,11 +26,6 @@ public class SimpleDataSource implements DataSource, AutoCloseable { */ private final int MAX_IDLE = 300; - /** - * 连接上限后最大等待时间（秒） - */ - private final int MAX_WAIT_SECONDS = 3; - /** * 从缓存队列获取连接次数 */ @@ -71,11 +65,7 @@ public class SimpleDataSource implements DataSource, AutoCloseable { lock.lock(); //如果当前连接数大于或等于最大连接数 if (activeNum.get() >= MAX_IDLE) { - //等待3秒 - TimeUnit.SECONDS.sleep(MAX_WAIT_SECONDS); - if (activeNum.get() >= MAX_IDLE) { - throw new SdkException(String.format("数据库连接数超过上限%d，url=%s", MAX_IDLE, url)); - } + throw new SdkException(String.format("数据库连接数超过上限%d，url=%s", MAX_IDLE, url)); } int time = MAX_PULL_TIME; while (time-- > 0){ @@ -94,8 +84,6 @@ public class SimpleDataSource implements DataSource, AutoCloseable { // 兜底方案，保证一定能获取连接 return createConnection(); - } catch (InterruptedException e) { - throw new SdkException(e); } finally { lock.unlock(); } diff --git a/dbsyncer-web/src/main/resources/application.properties b/dbsyncer-web/src/main/resources/application.properties index 675047f3..edf20cca 100644 --- a/dbsyncer-web/src/main/resources/application.properties +++ b/dbsyncer-web/src/main/resources/application.properties @@ -17,21 +17,25 @@ dbsyncer.web.scheduler.pool-size=8 # *********************** 通用执行器配置 *********************** # [GeneralBufferActuator]线程数 dbsyncer.parser.general.thread-core-size=8 +# [GeneralBufferActuator]最大线程数 +dbsyncer.parser.general.max-thread-size=16 # [GeneralBufferActuator]线程池队列 dbsyncer.parser.general.thread-queue-capacity=64 # [GeneralBufferActuator]单次执行任务数 -dbsyncer.parser.general.buffer-writer-count=100 +dbsyncer.parser.general.buffer-writer-count=1000 # [GeneralBufferActuator]每次消费缓存队列的任务数 -dbsyncer.parser.general.buffer-pull-count=1000 +dbsyncer.parser.general.buffer-pull-count=20000 # [GeneralBufferActuator]缓存队列容量 -dbsyncer.parser.general.buffer-queue-capacity=60000 +dbsyncer.parser.general.buffer-queue-capacity=100000 # [GeneralBufferActuator]定时消费缓存队列间隔(毫秒) dbsyncer.parser.general.buffer-period-millisecond=300 # *********************** 表执行器配置 *********************** # 每个驱动最多可分配的表执行器个数 dbsyncer.parser.table.group.max-buffer-actuator-size=20 # [TableGroupBufferActuator]线程数 -dbsyncer.parser.table.group.thread-core-size=10 +dbsyncer.parser.table.group.thread-core-size=2 +# [TableGroupBufferActuator]最大线程数 +dbsyncer.parser.table.group.max-thread-size=10 # [TableGroupBufferActuator]线程池队列 dbsyncer.parser.table.group.thread-queue-capacity=16 # [TableGroupBufferActuator]单次执行任务数 @@ -52,20 +56,22 @@ dbsyncer.storage.support.mysql.config.username=root dbsyncer.storage.support.mysql.config.password=123 # [StorageBufferActuator]线程数 dbsyncer.storage.thread-core-size=4 +# [StorageBufferActuator]最大线程数 +dbsyncer.storage.max-thread-size=8 # [StorageBufferActuator]线程池队列 -dbsyncer.storage.thread-queue-capacity=32 +dbsyncer.storage.thread-queue-capacity=64 # [StorageBufferActuator]单次执行任务数 -dbsyncer.storage.buffer-writer-count=100 +dbsyncer.storage.buffer-writer-count=1000 # [StorageBufferActuator]每次消费缓存队列的任务数 -dbsyncer.storage.buffer-pull-count=1000 +dbsyncer.storage.buffer-pull-count=20000 # [StorageBufferActuator]缓存队列容量 -dbsyncer.storage.buffer-queue-capacity=50000 +dbsyncer.storage.buffer-queue-capacity=100000 # [StorageBufferActuator]定时消费缓存队列间隔(毫秒) dbsyncer.storage.buffer-period-millisecond=300 # 是否记录全量数据（false-关闭; true-开启） dbsyncer.storage.write.full.enabled=false # 是否记录同步成功数据（false-关闭; true-开启） -dbsyncer.storage.write-success=false +dbsyncer.storage.write-success=true # 是否记录同步失败数据（false-关闭; true-开启） dbsyncer.storage.write-fail=true # 记录同步失败日志最大长度 -- Gitee From b7300d2cfac862d99b8d457517b3436e92a19e2c Mon Sep 17 00:00:00 2001 From: AE86 <836391306@qq.com> Date: Wed, 6 Dec 2023 02:07:57 +0800 Subject: [PATCH 54/98] =?UTF-8?q?=E6=80=A7=E8=83=BD=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dbsyncer-biz/pom.xml | 2 +- dbsyncer-cluster/pom.xml | 2 +- dbsyncer-common/pom.xml | 2 +- dbsyncer-connector/dbsyncer-connector-base/pom.xml | 2 +- dbsyncer-connector/dbsyncer-connector-elasticsearch/pom.xml | 2 +- dbsyncer-connector/dbsyncer-connector-file/pom.xml | 2 +- dbsyncer-connector/dbsyncer-connector-kafka/pom.xml | 2 +- dbsyncer-connector/dbsyncer-connector-mysql/pom.xml | 2 +- dbsyncer-connector/dbsyncer-connector-oracle/pom.xml | 2 +- dbsyncer-connector/dbsyncer-connector-postgresql/pom.xml | 2 +- dbsyncer-connector/dbsyncer-connector-sqlite/pom.xml | 2 +- dbsyncer-connector/dbsyncer-connector-sqlserver/pom.xml | 2 +- dbsyncer-connector/pom.xml | 2 +- dbsyncer-manager/pom.xml | 2 +- dbsyncer-parser/pom.xml | 2 +- dbsyncer-plugin/pom.xml | 2 +- dbsyncer-sdk/pom.xml | 2 +- dbsyncer-storage/pom.xml | 2 +- dbsyncer-web/pom.xml | 2 +- pom.xml | 2 +- 20 files changed, 20 insertions(+), 20 deletions(-) diff --git a/dbsyncer-biz/pom.xml b/dbsyncer-biz/pom.xml index dace0f78..1e7940ed 100644 --- a/dbsyncer-biz/pom.xml +++ b/dbsyncer-biz/pom.xml @@ -5,7 +5,7 @@ dbsyncer org.ghi - 2.0.0 + 2.0.0_1206 4.0.0 dbsyncer-biz diff --git a/dbsyncer-cluster/pom.xml b/dbsyncer-cluster/pom.xml index e08b4203..3249784d 100644 --- a/dbsyncer-cluster/pom.xml +++ b/dbsyncer-cluster/pom.xml @@ -5,7 +5,7 @@ dbsyncer org.ghi - 2.0.0 + 2.0.0_1206 4.0.0 dbsyncer-cluster diff --git a/dbsyncer-common/pom.xml b/dbsyncer-common/pom.xml index 448942be..ae80e561 100644 --- a/dbsyncer-common/pom.xml +++ b/dbsyncer-common/pom.xml @@ -5,7 +5,7 @@ dbsyncer org.ghi - 2.0.0 + 2.0.0_1206 4.0.0 dbsyncer-common diff --git a/dbsyncer-connector/dbsyncer-connector-base/pom.xml b/dbsyncer-connector/dbsyncer-connector-base/pom.xml index 37f8cfdf..edf1fcc7 100644 --- a/dbsyncer-connector/dbsyncer-connector-base/pom.xml +++ b/dbsyncer-connector/dbsyncer-connector-base/pom.xml @@ -5,7 +5,7 @@ dbsyncer-connector org.ghi - 2.0.0 + 2.0.0_1206 4.0.0 diff --git a/dbsyncer-connector/dbsyncer-connector-elasticsearch/pom.xml b/dbsyncer-connector/dbsyncer-connector-elasticsearch/pom.xml index 40574412..771df73b 100644 --- a/dbsyncer-connector/dbsyncer-connector-elasticsearch/pom.xml +++ b/dbsyncer-connector/dbsyncer-connector-elasticsearch/pom.xml @@ -5,7 +5,7 @@ dbsyncer-connector org.ghi - 2.0.0 + 2.0.0_1206 4.0.0 diff --git a/dbsyncer-connector/dbsyncer-connector-file/pom.xml b/dbsyncer-connector/dbsyncer-connector-file/pom.xml index 9c71355e..3b6b388b 100644 --- a/dbsyncer-connector/dbsyncer-connector-file/pom.xml +++ b/dbsyncer-connector/dbsyncer-connector-file/pom.xml @@ -5,7 +5,7 @@ dbsyncer-connector org.ghi - 2.0.0 + 2.0.0_1206 4.0.0 diff --git a/dbsyncer-connector/dbsyncer-connector-kafka/pom.xml b/dbsyncer-connector/dbsyncer-connector-kafka/pom.xml index ffbdb6bf..4e870aff 100644 --- a/dbsyncer-connector/dbsyncer-connector-kafka/pom.xml +++ b/dbsyncer-connector/dbsyncer-connector-kafka/pom.xml @@ -5,7 +5,7 @@ dbsyncer-connector org.ghi - 2.0.0 + 2.0.0_1206 4.0.0 diff --git a/dbsyncer-connector/dbsyncer-connector-mysql/pom.xml b/dbsyncer-connector/dbsyncer-connector-mysql/pom.xml index db89afce..ad441514 100644 --- a/dbsyncer-connector/dbsyncer-connector-mysql/pom.xml +++ b/dbsyncer-connector/dbsyncer-connector-mysql/pom.xml @@ -5,7 +5,7 @@ dbsyncer-connector org.ghi - 2.0.0 + 2.0.0_1206 4.0.0 diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/pom.xml b/dbsyncer-connector/dbsyncer-connector-oracle/pom.xml index c0bcafe8..3b027210 100644 --- a/dbsyncer-connector/dbsyncer-connector-oracle/pom.xml +++ b/dbsyncer-connector/dbsyncer-connector-oracle/pom.xml @@ -5,7 +5,7 @@ dbsyncer-connector org.ghi - 2.0.0 + 2.0.0_1206 4.0.0 diff --git a/dbsyncer-connector/dbsyncer-connector-postgresql/pom.xml b/dbsyncer-connector/dbsyncer-connector-postgresql/pom.xml index deeba571..fb1ae866 100644 --- a/dbsyncer-connector/dbsyncer-connector-postgresql/pom.xml +++ b/dbsyncer-connector/dbsyncer-connector-postgresql/pom.xml @@ -5,7 +5,7 @@ dbsyncer-connector org.ghi - 2.0.0 + 2.0.0_1206 4.0.0 diff --git a/dbsyncer-connector/dbsyncer-connector-sqlite/pom.xml b/dbsyncer-connector/dbsyncer-connector-sqlite/pom.xml index a40ced1c..c4248dc2 100644 --- a/dbsyncer-connector/dbsyncer-connector-sqlite/pom.xml +++ b/dbsyncer-connector/dbsyncer-connector-sqlite/pom.xml @@ -5,7 +5,7 @@ dbsyncer-connector org.ghi - 2.0.0 + 2.0.0_1206 4.0.0 diff --git a/dbsyncer-connector/dbsyncer-connector-sqlserver/pom.xml b/dbsyncer-connector/dbsyncer-connector-sqlserver/pom.xml index b3af0431..250217da 100644 --- a/dbsyncer-connector/dbsyncer-connector-sqlserver/pom.xml +++ b/dbsyncer-connector/dbsyncer-connector-sqlserver/pom.xml @@ -5,7 +5,7 @@ dbsyncer-connector org.ghi - 2.0.0 + 2.0.0_1206 4.0.0 diff --git a/dbsyncer-connector/pom.xml b/dbsyncer-connector/pom.xml index 075571f0..2a83ade4 100644 --- a/dbsyncer-connector/pom.xml +++ b/dbsyncer-connector/pom.xml @@ -5,7 +5,7 @@ dbsyncer org.ghi - 2.0.0 + 2.0.0_1206 4.0.0 dbsyncer-connector diff --git a/dbsyncer-manager/pom.xml b/dbsyncer-manager/pom.xml index 108d2a65..4bbe4ad7 100644 --- a/dbsyncer-manager/pom.xml +++ b/dbsyncer-manager/pom.xml @@ -5,7 +5,7 @@ dbsyncer org.ghi - 2.0.0 + 2.0.0_1206 4.0.0 dbsyncer-manager diff --git a/dbsyncer-parser/pom.xml b/dbsyncer-parser/pom.xml index 96118c03..b7652383 100644 --- a/dbsyncer-parser/pom.xml +++ b/dbsyncer-parser/pom.xml @@ -5,7 +5,7 @@ dbsyncer org.ghi - 2.0.0 + 2.0.0_1206 4.0.0 dbsyncer-parser diff --git a/dbsyncer-plugin/pom.xml b/dbsyncer-plugin/pom.xml index 547d80d8..0851b160 100644 --- a/dbsyncer-plugin/pom.xml +++ b/dbsyncer-plugin/pom.xml @@ -5,7 +5,7 @@ dbsyncer org.ghi - 2.0.0 + 2.0.0_1206 4.0.0 dbsyncer-plugin diff --git a/dbsyncer-sdk/pom.xml b/dbsyncer-sdk/pom.xml index 78020b93..7a7aca99 100644 --- a/dbsyncer-sdk/pom.xml +++ b/dbsyncer-sdk/pom.xml @@ -5,7 +5,7 @@ dbsyncer org.ghi - 2.0.0 + 2.0.0_1206 4.0.0 dbsyncer-sdk diff --git a/dbsyncer-storage/pom.xml b/dbsyncer-storage/pom.xml index 124091e6..3614b147 100644 --- a/dbsyncer-storage/pom.xml +++ b/dbsyncer-storage/pom.xml @@ -5,7 +5,7 @@ dbsyncer org.ghi - 2.0.0 + 2.0.0_1206 4.0.0 dbsyncer-storage diff --git a/dbsyncer-web/pom.xml b/dbsyncer-web/pom.xml index d09a42ea..83eb3d92 100644 --- a/dbsyncer-web/pom.xml +++ b/dbsyncer-web/pom.xml @@ -5,7 +5,7 @@ dbsyncer org.ghi - 2.0.0 + 2.0.0_1206 4.0.0 dbsyncer-web diff --git a/pom.xml b/pom.xml index d4c640e5..acb2a2bb 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ org.ghi dbsyncer - 2.0.0 + 2.0.0_1206 pom dbsyncer https://gitee.com/ghi/dbsyncer -- Gitee From 9b335e43bbd5dbe6414d1af93cf031d36a3353cd Mon Sep 17 00:00:00 2001 From: AE86 <836391306@qq.com> Date: Wed, 6 Dec 2023 02:10:52 +0800 Subject: [PATCH 55/98] upgrade version --- dbsyncer-biz/pom.xml | 2 +- dbsyncer-cluster/pom.xml | 2 +- dbsyncer-common/pom.xml | 2 +- dbsyncer-connector/dbsyncer-connector-base/pom.xml | 2 +- dbsyncer-connector/dbsyncer-connector-elasticsearch/pom.xml | 2 +- dbsyncer-connector/dbsyncer-connector-file/pom.xml | 2 +- dbsyncer-connector/dbsyncer-connector-kafka/pom.xml | 2 +- dbsyncer-connector/dbsyncer-connector-mysql/pom.xml | 2 +- dbsyncer-connector/dbsyncer-connector-oracle/pom.xml | 2 +- dbsyncer-connector/dbsyncer-connector-postgresql/pom.xml | 2 +- dbsyncer-connector/dbsyncer-connector-sqlite/pom.xml | 2 +- dbsyncer-connector/dbsyncer-connector-sqlserver/pom.xml | 2 +- dbsyncer-connector/pom.xml | 2 +- dbsyncer-manager/pom.xml | 2 +- dbsyncer-parser/pom.xml | 2 +- dbsyncer-plugin/pom.xml | 2 +- dbsyncer-sdk/pom.xml | 2 +- dbsyncer-storage/pom.xml | 2 +- dbsyncer-web/pom.xml | 2 +- pom.xml | 2 +- 20 files changed, 20 insertions(+), 20 deletions(-) diff --git a/dbsyncer-biz/pom.xml b/dbsyncer-biz/pom.xml index 1e7940ed..b6d8a905 100644 --- a/dbsyncer-biz/pom.xml +++ b/dbsyncer-biz/pom.xml @@ -5,7 +5,7 @@ dbsyncer org.ghi - 2.0.0_1206 + 1.2.8_1206_pre 4.0.0 dbsyncer-biz diff --git a/dbsyncer-cluster/pom.xml b/dbsyncer-cluster/pom.xml index 3249784d..cb4f01cc 100644 --- a/dbsyncer-cluster/pom.xml +++ b/dbsyncer-cluster/pom.xml @@ -5,7 +5,7 @@ dbsyncer org.ghi - 2.0.0_1206 + 1.2.8_1206_pre 4.0.0 dbsyncer-cluster diff --git a/dbsyncer-common/pom.xml b/dbsyncer-common/pom.xml index ae80e561..9b403bee 100644 --- a/dbsyncer-common/pom.xml +++ b/dbsyncer-common/pom.xml @@ -5,7 +5,7 @@ dbsyncer org.ghi - 2.0.0_1206 + 1.2.8_1206_pre 4.0.0 dbsyncer-common diff --git a/dbsyncer-connector/dbsyncer-connector-base/pom.xml b/dbsyncer-connector/dbsyncer-connector-base/pom.xml index edf1fcc7..dfb0ef81 100644 --- a/dbsyncer-connector/dbsyncer-connector-base/pom.xml +++ b/dbsyncer-connector/dbsyncer-connector-base/pom.xml @@ -5,7 +5,7 @@ dbsyncer-connector org.ghi - 2.0.0_1206 + 1.2.8_1206_pre 4.0.0 diff --git a/dbsyncer-connector/dbsyncer-connector-elasticsearch/pom.xml b/dbsyncer-connector/dbsyncer-connector-elasticsearch/pom.xml index 771df73b..529ef199 100644 --- a/dbsyncer-connector/dbsyncer-connector-elasticsearch/pom.xml +++ b/dbsyncer-connector/dbsyncer-connector-elasticsearch/pom.xml @@ -5,7 +5,7 @@ dbsyncer-connector org.ghi - 2.0.0_1206 + 1.2.8_1206_pre 4.0.0 diff --git a/dbsyncer-connector/dbsyncer-connector-file/pom.xml b/dbsyncer-connector/dbsyncer-connector-file/pom.xml index 3b6b388b..98ab006c 100644 --- a/dbsyncer-connector/dbsyncer-connector-file/pom.xml +++ b/dbsyncer-connector/dbsyncer-connector-file/pom.xml @@ -5,7 +5,7 @@ dbsyncer-connector org.ghi - 2.0.0_1206 + 1.2.8_1206_pre 4.0.0 diff --git a/dbsyncer-connector/dbsyncer-connector-kafka/pom.xml b/dbsyncer-connector/dbsyncer-connector-kafka/pom.xml index 4e870aff..30fb7606 100644 --- a/dbsyncer-connector/dbsyncer-connector-kafka/pom.xml +++ b/dbsyncer-connector/dbsyncer-connector-kafka/pom.xml @@ -5,7 +5,7 @@ dbsyncer-connector org.ghi - 2.0.0_1206 + 1.2.8_1206_pre 4.0.0 diff --git a/dbsyncer-connector/dbsyncer-connector-mysql/pom.xml b/dbsyncer-connector/dbsyncer-connector-mysql/pom.xml index ad441514..8fe1a0ca 100644 --- a/dbsyncer-connector/dbsyncer-connector-mysql/pom.xml +++ b/dbsyncer-connector/dbsyncer-connector-mysql/pom.xml @@ -5,7 +5,7 @@ dbsyncer-connector org.ghi - 2.0.0_1206 + 1.2.8_1206_pre 4.0.0 diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/pom.xml b/dbsyncer-connector/dbsyncer-connector-oracle/pom.xml index 3b027210..b1ce2f4b 100644 --- a/dbsyncer-connector/dbsyncer-connector-oracle/pom.xml +++ b/dbsyncer-connector/dbsyncer-connector-oracle/pom.xml @@ -5,7 +5,7 @@ dbsyncer-connector org.ghi - 2.0.0_1206 + 1.2.8_1206_pre 4.0.0 diff --git a/dbsyncer-connector/dbsyncer-connector-postgresql/pom.xml b/dbsyncer-connector/dbsyncer-connector-postgresql/pom.xml index fb1ae866..b43f7d6f 100644 --- a/dbsyncer-connector/dbsyncer-connector-postgresql/pom.xml +++ b/dbsyncer-connector/dbsyncer-connector-postgresql/pom.xml @@ -5,7 +5,7 @@ dbsyncer-connector org.ghi - 2.0.0_1206 + 1.2.8_1206_pre 4.0.0 diff --git a/dbsyncer-connector/dbsyncer-connector-sqlite/pom.xml b/dbsyncer-connector/dbsyncer-connector-sqlite/pom.xml index c4248dc2..8d623b3d 100644 --- a/dbsyncer-connector/dbsyncer-connector-sqlite/pom.xml +++ b/dbsyncer-connector/dbsyncer-connector-sqlite/pom.xml @@ -5,7 +5,7 @@ dbsyncer-connector org.ghi - 2.0.0_1206 + 1.2.8_1206_pre 4.0.0 diff --git a/dbsyncer-connector/dbsyncer-connector-sqlserver/pom.xml b/dbsyncer-connector/dbsyncer-connector-sqlserver/pom.xml index 250217da..d55aad45 100644 --- a/dbsyncer-connector/dbsyncer-connector-sqlserver/pom.xml +++ b/dbsyncer-connector/dbsyncer-connector-sqlserver/pom.xml @@ -5,7 +5,7 @@ dbsyncer-connector org.ghi - 2.0.0_1206 + 1.2.8_1206_pre 4.0.0 diff --git a/dbsyncer-connector/pom.xml b/dbsyncer-connector/pom.xml index 2a83ade4..94afb27d 100644 --- a/dbsyncer-connector/pom.xml +++ b/dbsyncer-connector/pom.xml @@ -5,7 +5,7 @@ dbsyncer org.ghi - 2.0.0_1206 + 1.2.8_1206_pre 4.0.0 dbsyncer-connector diff --git a/dbsyncer-manager/pom.xml b/dbsyncer-manager/pom.xml index 4bbe4ad7..8cac578d 100644 --- a/dbsyncer-manager/pom.xml +++ b/dbsyncer-manager/pom.xml @@ -5,7 +5,7 @@ dbsyncer org.ghi - 2.0.0_1206 + 1.2.8_1206_pre 4.0.0 dbsyncer-manager diff --git a/dbsyncer-parser/pom.xml b/dbsyncer-parser/pom.xml index b7652383..db153972 100644 --- a/dbsyncer-parser/pom.xml +++ b/dbsyncer-parser/pom.xml @@ -5,7 +5,7 @@ dbsyncer org.ghi - 2.0.0_1206 + 1.2.8_1206_pre 4.0.0 dbsyncer-parser diff --git a/dbsyncer-plugin/pom.xml b/dbsyncer-plugin/pom.xml index 0851b160..620e617c 100644 --- a/dbsyncer-plugin/pom.xml +++ b/dbsyncer-plugin/pom.xml @@ -5,7 +5,7 @@ dbsyncer org.ghi - 2.0.0_1206 + 1.2.8_1206_pre 4.0.0 dbsyncer-plugin diff --git a/dbsyncer-sdk/pom.xml b/dbsyncer-sdk/pom.xml index 7a7aca99..a932fbab 100644 --- a/dbsyncer-sdk/pom.xml +++ b/dbsyncer-sdk/pom.xml @@ -5,7 +5,7 @@ dbsyncer org.ghi - 2.0.0_1206 + 1.2.8_1206_pre 4.0.0 dbsyncer-sdk diff --git a/dbsyncer-storage/pom.xml b/dbsyncer-storage/pom.xml index 3614b147..078e4fb5 100644 --- a/dbsyncer-storage/pom.xml +++ b/dbsyncer-storage/pom.xml @@ -5,7 +5,7 @@ dbsyncer org.ghi - 2.0.0_1206 + 1.2.8_1206_pre 4.0.0 dbsyncer-storage diff --git a/dbsyncer-web/pom.xml b/dbsyncer-web/pom.xml index 83eb3d92..0d3a3461 100644 --- a/dbsyncer-web/pom.xml +++ b/dbsyncer-web/pom.xml @@ -5,7 +5,7 @@ dbsyncer org.ghi - 2.0.0_1206 + 1.2.8_1206_pre 4.0.0 dbsyncer-web diff --git a/pom.xml b/pom.xml index acb2a2bb..383afc90 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ org.ghi dbsyncer - 2.0.0_1206 + 1.2.8_1206_pre pom dbsyncer https://gitee.com/ghi/dbsyncer -- Gitee From 6d8d2d290a338465b1f73730480e1190c1b8c33e Mon Sep 17 00:00:00 2001 From: AE86 <836391306@qq.com> Date: Wed, 6 Dec 2023 02:52:37 +0800 Subject: [PATCH 56/98] upgrade version --- .../main/resources/public/plugin/plugin.html | 36 ++++++++++--------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/dbsyncer-web/src/main/resources/public/plugin/plugin.html b/dbsyncer-web/src/main/resources/public/plugin/plugin.html index d9ed91fd..b1585c26 100644 --- a/dbsyncer-web/src/main/resources/public/plugin/plugin.html +++ b/dbsyncer-web/src/main/resources/public/plugin/plugin.html @@ -28,13 +28,20 @@
  • - 新建一个类，比如MyPlugin，实现接口ConvertService方法 + 新建一个类，比如MyPlugin，实现接口PluginService方法 
    package org.test;
 
-import org.dbsyncer.common.spi.ConvertContext;
-import org.dbsyncer.common.spi.ConvertService;
+import org.dbsyncer.sdk.connector.ConnectorInstance;
+import org.dbsyncer.sdk.connector.database.DatabaseConnectorInstance;
+import org.dbsyncer.sdk.connector.database.ds.SimpleConnection;
+import org.dbsyncer.sdk.plugin.PluginContext;
+import org.dbsyncer.sdk.spi.PluginService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
-public class MyPlugin implements ConvertService {
+public class MyPlugin implements PluginService {
+
+    private final Logger logger = LoggerFactory.getLogger(getClass());
 
     /**
      * 全量同步/增量同步
@@ -42,29 +49,26 @@ public class MyPlugin implements ConvertService {
      * @param convertContext
      */
     @Override
-    public void convert(ConvertContext convertContext) {
+    public void convert(PluginContext pluginContext) {
         // TODO 消费或处理数据
         System.out.println("插件消费数据中...");
 
         // 是否终止同步到目标库开关，默认false
-        convertContext.setTerminated(false);
-
-        // 获取Spring上下文，当然也可获取dbs注册的bean对象
-        convertContext.getContext();
+        pluginContext.setTerminated(false);
 
         // 数据源表和目标源表
-        convertContext.getSourceTableName();
-        convertContext.getTargetTableName();
+        pluginContext.getSourceTableName();
+        pluginContext.getTargetTableName();
 
         // 捕获的事件（INSERT/UPDATE/DELETE）
-        convertContext.getEvent();
+        pluginContext.getEvent();
 
         // 数据源和目标源表全量或增量数据
-        convertContext.getSourceList();
-        convertContext.getTargetList();
+        pluginContext.getSourceList();
+        pluginContext.getTargetList();
 
         // 获取目标库连接器实例（如果需要用到连接器，必须引入dbsyncer-connector-[[${version}]].jar）
-        convertContext.getTargetConnectorMapper();
+        pluginContext.getTargetConnectorInstance();
     }
 
     /**
@@ -73,7 +77,7 @@ public class MyPlugin implements ConvertService {
      * @param context
      */
     @Override
-    public void postProcessAfter(ConvertContext context) {
+    public void postProcessAfter(PluginContext context) {
         // 完成同步后调用该方法
        logger.info("插件正在处理同步成功的数据，目标源表:{}，事件:{}，条数:{}", context.getTargetTableName(), context.getEvent(), context.getTargetList().size());
 
-- 
Gitee


From 0eec418b8dca2268d607e96d865a91957dfcf537 Mon Sep 17 00:00:00 2001
From: AE86 <836391306@qq.com>
Date: Thu, 7 Dec 2023 23:55:27 +0800
Subject: [PATCH 57/98] =?UTF-8?q?=E6=94=AF=E6=8C=81mysql5.7=20tinyint?=
 =?UTF-8?q?=E7=B1=BB=E5=9E=8B=E5=90=8C=E6=AD=A5=E5=88=B0postgresql11=20int?=
 =?UTF-8?q?2=E7=B1=BB=E5=9E=8B?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../parser/convert/handler/NumberToStringHandler.java         | 2 --
 .../dbsyncer/sdk/connector/schema/SmallintValueMapper.java    | 4 ++++
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/convert/handler/NumberToStringHandler.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/convert/handler/NumberToStringHandler.java
index 489beb1e..21428314 100644
--- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/convert/handler/NumberToStringHandler.java
+++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/convert/handler/NumberToStringHandler.java
@@ -2,8 +2,6 @@ package org.dbsyncer.parser.convert.handler;
 
 import org.dbsyncer.parser.convert.AbstractHandler;
 
-import java.sql.Timestamp;
-
 /**
  * Number转String
  *
diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/SmallintValueMapper.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/SmallintValueMapper.java
index 85bdfbd3..21f108e6 100644
--- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/SmallintValueMapper.java
+++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/SmallintValueMapper.java
@@ -17,6 +17,10 @@ public class SmallintValueMapper extends AbstractValueMapper {
             Short s = (Short) val;
             return new Integer(s);
         }
+        if (val instanceof Boolean) {
+            Boolean b = (Boolean) val;
+            return new Integer(b ? 1 : 0);
+        }
 
         throw new SdkException(String.format("%s can not find type [%s], val [%s]", getClass().getSimpleName(), val.getClass(), val));
     }
-- 
Gitee


From ee094f3e8ff349c6eb862603b9ebf65a5ba037c5 Mon Sep 17 00:00:00 2001
From: AE86 <836391306@qq.com>
Date: Fri, 8 Dec 2023 10:40:23 +0000
Subject: [PATCH 58/98] fix bug

---
 .../main/java/org/dbsyncer/parser/util/PickerUtil.java    | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/util/PickerUtil.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/util/PickerUtil.java
index a0d64406..742ec854 100644
--- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/util/PickerUtil.java
+++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/util/PickerUtil.java
@@ -8,6 +8,7 @@ import org.dbsyncer.parser.model.Mapping;
 import org.dbsyncer.parser.model.TableGroup;
 import org.dbsyncer.sdk.model.Field;
 import org.dbsyncer.sdk.model.Filter;
+import org.springframework.beans.BeanUtils;
 
 import java.util.List;
 import java.util.Map;
@@ -23,12 +24,7 @@ public abstract class PickerUtil {
      */
     public static TableGroup mergeTableGroupConfig(Mapping mapping, TableGroup tableGroup) {
         TableGroup group = new TableGroup();
-        group.setId(tableGroup.getId());
-        group.setMappingId(mapping.getId());
-        group.setFieldMapping(tableGroup.getFieldMapping());
-        group.setSourceTable(tableGroup.getSourceTable());
-        group.setTargetTable(tableGroup.getTargetTable());
-        group.setCommand(tableGroup.getCommand());
+        BeanUtils.copyProperties(tableGroup, group);
 
         // 参数配置(默认使用全局)
         group.setParams(CollectionUtils.isEmpty(tableGroup.getParams()) ? mapping.getParams() : tableGroup.getParams());
-- 
Gitee


From ee08b57acac6afc09b9e3b0e908e4811420c4e05 Mon Sep 17 00:00:00 2001
From: AE86 <836391306@qq.com>
Date: Sun, 10 Dec 2023 01:19:27 +0800
Subject: [PATCH 59/98] merge into v2

---
 .../org/dbsyncer/common/util/StringUtil.java  |   2 +
 .../dbsyncer-connector-oracle/pom.xml         |   6 +
 .../connector/oracle/cdc/OracleListener.java  |  83 +++--
 .../connector/oracle/logminer/LogFile.java    |  46 +++
 .../connector/oracle/logminer/LogMiner.java   | 321 ++++++++++++++++++
 .../oracle/logminer/LogMinerHelper.java       | 240 +++++++++++++
 .../connector/oracle/logminer/RedoEvent.java  |  99 ++++++
 .../oracle/logminer/TransactionalBuffer.java  | 107 ++++++
 .../parser/consumer/AbstractConsumer.java     |  20 +-
 .../parser/consumer/impl/LogConsumer.java     |   6 +
 .../flush/impl/GeneralBufferActuator.java     |  71 +++-
 .../dbsyncer/parser/model/WriterRequest.java  |  17 +-
 .../dbsyncer/parser/model/WriterResponse.java |  11 +
 .../org/dbsyncer/parser/sql/SqlParser.java    |  14 +
 .../dbsyncer/parser/sql/impl/DeleteSql.java   |  76 +++++
 .../dbsyncer/parser/sql/impl/InsertSql.java   |  59 ++++
 .../dbsyncer/parser/sql/impl/UpdateSql.java   |  87 +++++
 .../sdk/enums/ChangedEventTypeEnum.java       |  35 ++
 .../dbsyncer/sdk/listener/ChangedEvent.java   |  15 +
 .../listener/event/CommonChangedEvent.java    |  15 +-
 .../sdk/listener/event/DDLChangedEvent.java   |  14 +-
 .../sdk/listener/event/RowChangedEvent.java   |   8 +-
 .../sdk/listener/event/ScanChangedEvent.java  |   7 +
 .../sdk/listener/event/SqlChangedEvent.java   |  27 ++
 24 files changed, 1337 insertions(+), 49 deletions(-)
 create mode 100644 dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/LogFile.java
 create mode 100644 dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/LogMiner.java
 create mode 100644 dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/LogMinerHelper.java
 create mode 100644 dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/RedoEvent.java
 create mode 100644 dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/TransactionalBuffer.java
 create mode 100644 dbsyncer-parser/src/main/java/org/dbsyncer/parser/sql/SqlParser.java
 create mode 100644 dbsyncer-parser/src/main/java/org/dbsyncer/parser/sql/impl/DeleteSql.java
 create mode 100644 dbsyncer-parser/src/main/java/org/dbsyncer/parser/sql/impl/InsertSql.java
 create mode 100644 dbsyncer-parser/src/main/java/org/dbsyncer/parser/sql/impl/UpdateSql.java
 create mode 100644 dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/ChangedEventTypeEnum.java
 create mode 100644 dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/event/SqlChangedEvent.java

diff --git a/dbsyncer-common/src/main/java/org/dbsyncer/common/util/StringUtil.java b/dbsyncer-common/src/main/java/org/dbsyncer/common/util/StringUtil.java
index fd452a87..30b5f6f2 100644
--- a/dbsyncer-common/src/main/java/org/dbsyncer/common/util/StringUtil.java
+++ b/dbsyncer-common/src/main/java/org/dbsyncer/common/util/StringUtil.java
@@ -14,6 +14,8 @@ public abstract class StringUtil {
 
     public static final String SINGLE_QUOTATION = "'";
 
+    public static final String DOUBLE_QUOTATION = "\"";
+
     public static final String FORWARD_SLASH = "/";
 
     public static boolean equals(CharSequence cs1, CharSequence cs2) {
diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/pom.xml b/dbsyncer-connector/dbsyncer-connector-oracle/pom.xml
index b1ce2f4b..c39b7517 100644
--- a/dbsyncer-connector/dbsyncer-connector-oracle/pom.xml
+++ b/dbsyncer-connector/dbsyncer-connector-oracle/pom.xml
@@ -26,6 +26,12 @@
             ojdbc6
         
 
+        
+            com.github.jsqlparser
+            jsqlparser
+            provided
+        
+
         
         
             com.microsoft.sqlserver
diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java
index f828750f..e9b33c82 100644
--- a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java
+++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java
@@ -3,15 +3,26 @@
  */
 package org.dbsyncer.connector.oracle.cdc;
 
-import org.dbsyncer.common.QueueOverflowException;
+import net.sf.jsqlparser.JSQLParserException;
+import net.sf.jsqlparser.parser.CCJSqlParserUtil;
+import net.sf.jsqlparser.schema.Table;
+import net.sf.jsqlparser.statement.Statement;
+import net.sf.jsqlparser.statement.alter.Alter;
+import net.sf.jsqlparser.statement.delete.Delete;
+import net.sf.jsqlparser.statement.insert.Insert;
+import net.sf.jsqlparser.statement.update.Update;
+import org.dbsyncer.common.util.StringUtil;
 import org.dbsyncer.connector.oracle.OracleException;
-import org.dbsyncer.connector.oracle.dcn.DBChangeNotification;
+import org.dbsyncer.connector.oracle.logminer.LogMiner;
 import org.dbsyncer.sdk.config.DatabaseConfig;
+import org.dbsyncer.sdk.constant.ConnectorConstant;
 import org.dbsyncer.sdk.listener.AbstractDatabaseListener;
+import org.dbsyncer.sdk.listener.event.DDLChangedEvent;
+import org.dbsyncer.sdk.listener.event.SqlChangedEvent;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.concurrent.TimeUnit;
+import java.sql.SQLException;
 
 /**
  * @Author AE86
@@ -21,33 +32,54 @@ import java.util.concurrent.TimeUnit;
 public class OracleListener extends AbstractDatabaseListener {
 
     private final Logger logger = LoggerFactory.getLogger(getClass());
+    private final String REDO_POSITION = "position";
 
-    private DBChangeNotification client;
+    private LogMiner logMiner;
 
     @Override
     public void start() {
         try {
             final DatabaseConfig config = (DatabaseConfig) connectorConfig;
+            String driverClassName = config.getDriverClassName();
             String username = config.getUsername();
             String password = config.getPassword();
             String url = config.getUrl();
-            client = new DBChangeNotification(username, password, url);
-            client.setFilterTable(filterTable);
-            client.addRowEventListener((event) -> {
-                while (client.isConnected()){
-                    try {
-                        sendChangedEvent(event);
-                        break;
-                    } catch (QueueOverflowException ex) {
-                        try {
-                            TimeUnit.MILLISECONDS.sleep(1);
-                        } catch (InterruptedException exe) {
-                            logger.error(exe.getMessage(), exe);
-                        }
+            String schema = config.getSchema();
+            boolean containsPos = snapshot.containsKey(REDO_POSITION);
+            logMiner = new LogMiner(username, password, url, schema, driverClassName);
+            logMiner.setStartScn(containsPos ? Long.parseLong(snapshot.get(REDO_POSITION)) : 0);
+            logMiner.registerEventListener((event) -> {
+                try {
+                    Statement statement = CCJSqlParserUtil.parse(event.getRedoSql());
+                    if (statement instanceof Update) {
+                        Update update = (Update) statement;
+                        sendChangedEvent(new SqlChangedEvent(replaceTableName(update.getTable()), ConnectorConstant.OPERTION_UPDATE, event.getRedoSql(), null, event.getScn()));
+                        return;
                     }
+
+                    if (statement instanceof Insert) {
+                        Insert insert = (Insert) statement;
+                        sendChangedEvent(new SqlChangedEvent(replaceTableName(insert.getTable()), ConnectorConstant.OPERTION_INSERT, event.getRedoSql(), null, event.getScn()));
+                        return;
+                    }
+
+                    if (statement instanceof Delete) {
+                        Delete delete = (Delete) statement;
+                        sendChangedEvent(new SqlChangedEvent(replaceTableName(delete.getTable()), ConnectorConstant.OPERTION_DELETE, event.getRedoSql(), null, event.getScn()));
+                        return;
+                    }
+
+                    if (statement instanceof Alter) {
+                        Alter alter = (Alter) statement;
+                        sendChangedEvent(new DDLChangedEvent("", replaceTableName(alter.getTable()), ConnectorConstant.OPERTION_ALTER, event.getRedoSql(), null, event.getScn()));
+                        return;
+                    }
+                } catch (JSQLParserException e) {
+                    logger.error("parse redoSql=" + event.getRedoSql(), e);
                 }
             });
-            client.start();
+            logMiner.start();
+
         } catch (Exception e) {
             logger.error("启动失败:{}", e.getMessage());
             throw new OracleException(e);
@@ -56,9 +88,20 @@ public class OracleListener extends AbstractDatabaseListener {
 
     @Override
     public void close() {
-        if (null != client) {
-            client.close();
+        try {
+            if (logMiner != null) {
+                logMiner.close();
+            }
+        } catch (SQLException e) {
+            logger.error(e.getMessage(), e);
+        }
+    }
+
+    private String replaceTableName(Table table) {
+        if (table == null) {
+            return StringUtil.EMPTY;
         }
+        return StringUtil.replace(table.getName(), StringUtil.DOUBLE_QUOTATION, StringUtil.EMPTY);
     }
 
 }
\ No newline at end of file
diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/LogFile.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/LogFile.java
new file mode 100644
index 00000000..7f2cbe6c
--- /dev/null
+++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/LogFile.java
@@ -0,0 +1,46 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
+package org.dbsyncer.connector.oracle.logminer;
+
+import java.math.BigInteger;
+import java.util.Objects;
+
+/**
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2023-12-09 20:20
+ */
+public class LogFile {
+    private final String fileName;
+    private final BigInteger firstScn;
+    private final BigInteger nextScn;
+    private final boolean current;
+
+    public LogFile(String fileName, BigInteger firstScn, BigInteger nextScn, boolean current) {
+        this.fileName = fileName;
+        this.firstScn = firstScn;
+        this.nextScn = nextScn;
+        this.current = current;
+    }
+
+    public String getFileName() {
+        return fileName;
+    }
+
+    public BigInteger getFirstScn() {
+        return firstScn;
+    }
+
+    public BigInteger getNextScn() {
+        return nextScn;
+    }
+
+    public boolean isCurrent() {
+        return current;
+    }
+
+    public boolean isSameRange(LogFile other) {
+        return Objects.equals(firstScn, other.getFirstScn()) && Objects.equals(nextScn, other.getNextScn());
+    }
+}
\ No newline at end of file
diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/LogMiner.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/LogMiner.java
new file mode 100644
index 00000000..be46509d
--- /dev/null
+++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/LogMiner.java
@@ -0,0 +1,321 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
+package org.dbsyncer.connector.oracle.logminer;
+
+import org.apache.commons.lang3.time.StopWatch;
+import org.dbsyncer.sdk.util.DatabaseUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.math.BigInteger;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.Timestamp;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.stream.Collectors;
+
+/**
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2023-12-09 20:21
+ */
+public class LogMiner {
+
+    private final Logger logger = LoggerFactory.getLogger(getClass());
+    private Lock lock = new ReentrantLock();
+    private String username;
+    private String password;
+    private String url;
+    private String schema;
+    private String driverClassName;
+    private String miningStrategy = "DBMS_LOGMNR.DICT_FROM_ONLINE_CATALOG";
+    private volatile boolean connected = false;
+    private final StopWatch stopWatch = StopWatch.create();
+    private Connection connection;
+    private List currentRedoLogSequences;
+    private TransactionalBuffer transactionalBuffer = new TransactionalBuffer();
+    // 已提交的位点
+    private Long committedScn = 0L;
+    // 初始位点
+    private long startScn = 0;
+    private EventListener listener;
+
+    public LogMiner(String username, String password, String url, String schema, String driverClassName) {
+        this.username = username;
+        this.password = password;
+        this.url = url;
+        this.schema = schema;
+        this.driverClassName = driverClassName;
+    }
+
+    public void close() throws SQLException {
+        lock.lock();
+        if (!connected) {
+            logger.error("LogMiner is already stop");
+            lock.unlock();
+            return;
+        }
+        this.connection.close();
+        connected = false;
+        lock.unlock();
+    }
+
+    public void start() throws SQLException {
+        lock.lock();
+        if (connected) {
+            logger.error("LogMiner is already started");
+            lock.unlock();
+            return;
+        }
+        this.connection = DatabaseUtil.getConnection(driverClassName, url, username, password);
+        connected = true;
+        lock.unlock();
+        //get current scn 判断是否第一次没有存储
+        if (startScn == 0) {
+            startScn = getCurrentScn(connection);
+        }
+
+        logger.info("scn start '{}'", startScn);
+        logger.info("start LogMiner...");
+//        LogMinerHelper.resetSessionToCdbIfNecessary(connection);
+        LogMinerHelper.setSessionParameter(connection);
+
+        // 1.记录当前redoLog，用于下文判断redoLog 是否切换
+        currentRedoLogSequences = LogMinerHelper.getCurrentRedoLogSequences(connection);
+
+        // 2.构建数据字典 && add redo / archived log
+        initializeLogMiner();
+
+        String minerViewQuery = LogMinerHelper.logMinerViewQuery(schema, username);
+        logger.debug(minerViewQuery);
+
+        try (PreparedStatement minerViewStatement = connection.prepareStatement(minerViewQuery, ResultSet.TYPE_FORWARD_ONLY,
+                ResultSet.CONCUR_READ_ONLY, ResultSet.HOLD_CURSORS_OVER_COMMIT)) {
+            // while
+            while (connected) {
+                // 3.确定 endScn
+                BigInteger endScn = determineEndScn();
+
+                // 4.是否发生redoLog切换
+                if (redoLogSwitchOccurred()) {
+                    // 如果切换则重启logMiner会话
+                    logger.debug("restart LogMiner Session");
+                    restartLogMiner();
+                    currentRedoLogSequences = LogMinerHelper.getCurrentRedoLogSequences(connection);
+                }
+
+                // 5.start logMiner
+                LogMinerHelper.startLogMiner(connection, BigInteger.valueOf(startScn), endScn, miningStrategy);
+
+                // 6.查询 logMiner view, 处理结果集
+                minerViewStatement.setFetchSize(2000);
+                minerViewStatement.setFetchDirection(ResultSet.FETCH_FORWARD);
+                minerViewStatement.setString(1, String.valueOf(startScn));
+                minerViewStatement.setString(2, endScn.toString());
+
+                stopWatch.start();
+
+                try (ResultSet rs = minerViewStatement.executeQuery()) {
+                    logger.trace("Query V$LOGMNR_CONTENTS spend time {} ms", stopWatch.getTime(TimeUnit.MILLISECONDS));
+                    stopWatch.reset();
+                    logMinerViewProcessor(rs);
+                }
+
+                // 7.确定新的SCN
+                startScn = Long.parseLong(endScn.toString());
+
+                try {
+                    // 避免频繁的执行导致 PGA 内存超出 PGA_AGGREGATE_LIMIT
+                    TimeUnit.SECONDS.sleep(2);
+                } catch (InterruptedException e) {
+                    logger.error(e.getMessage(), e);
+                }
+            }
+
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+        }
+    }
+
+    public long getCurrentScn(Connection connection) throws SQLException {
+        try (Statement statement = connection.createStatement()) {
+            ResultSet rs = statement.executeQuery("select CURRENT_SCN from V$DATABASE");
+
+            if (!rs.next()) {
+                throw new IllegalStateException("Couldn't get SCN");
+            }
+
+            return rs.getLong(1);
+        }
+    }
+
+    private void restartLogMiner() throws SQLException {
+        LogMinerHelper.endLogMiner(connection);
+        initializeLogMiner();
+    }
+
+    private boolean redoLogSwitchOccurred() throws SQLException {
+        final List newSequences = LogMinerHelper.getCurrentRedoLogSequences(connection);
+        if (!newSequences.equals(currentRedoLogSequences)) {
+            currentRedoLogSequences = newSequences;
+            return true;
+        }
+        return false;
+    }
+
+    private BigInteger determineEndScn() throws SQLException {
+        return BigInteger.valueOf(getCurrentScn(connection));
+    }
+
+    private void initializeLogMiner() throws SQLException {
+        // 默认使用在线数据字典，所以此处不做数据字典相关操作
+        LogMinerHelper.buildDataDictionary(connection, miningStrategy);
+
+        setRedoLog();
+    }
+
+    private void setRedoLog() throws SQLException {
+        LogMinerHelper.removeLogFilesFromMining(connection);
+        List onlineLogFiles = LogMinerHelper.getOnlineLogFilesForOffsetScn(connection, BigInteger.valueOf(startScn));
+        List archivedLogFiles = LogMinerHelper.getArchivedLogFilesForOffsetScn(connection, BigInteger.valueOf(startScn));
+        List logFilesNames = archivedLogFiles.stream().map(LogFile::getFileName).collect(Collectors.toList());
+        for (LogFile onlineLogFile : onlineLogFiles) {
+            boolean found = false;
+            for (LogFile archivedLogFile : archivedLogFiles) {
+                if (onlineLogFile.isSameRange(archivedLogFile)) {
+                    // 如果redo 已经被归档，那么就不需要加载这个redo了
+                    found = true;
+                    break;
+                }
+            }
+            if (!found)
+                logFilesNames.add(onlineLogFile.getFileName());
+        }
+
+        // 加载所需要的redo / archived
+        for (String fileName : logFilesNames) {
+            LogMinerHelper.addLogFile(connection, fileName);
+        }
+    }
+
+    private void logMinerViewProcessor(ResultSet rs) throws SQLException {
+        while (rs.next()) {
+            BigInteger scn = rs.getBigDecimal("SCN").toBigInteger();
+            String tableName = rs.getString("TABLE_NAME");
+            String segOwner = rs.getString("SEG_OWNER");
+            int operationCode = rs.getInt("OPERATION_CODE");
+            Timestamp changeTime = rs.getTimestamp("TIMESTAMP");
+            String txId = rs.getString("XID");
+            String operation = rs.getString("OPERATION");
+            String username = rs.getString("USERNAME");
+
+            logger.trace("Capture record, SCN:{}, TABLE_NAME:{}, SEG_OWNER:{}, OPERATION_CODE:{}, TIMESTAMP:{}, XID:{}, OPERATION:{}, USERNAME:{}",
+                    scn, tableName, segOwner, operationCode, changeTime, txId, operation, username);
+
+            // Commit
+            if (operationCode == LogMinerHelper.LOG_MINER_OC_COMMIT) {
+                // 将TransactionalBuffer中当前事务的DML 转移到消费者处理
+                if (transactionalBuffer.commit(txId, scn, committedScn)) {
+                    logger.debug("txId: {} commit", txId);
+                }
+                continue;
+            }
+
+            // Rollback
+            if (operationCode == LogMinerHelper.LOG_MINER_OC_ROLLBACK) {
+                // 清空TransactionalBuffer中当前事务
+                if (transactionalBuffer.rollback(txId)) {
+                    logger.debug("txId: {} rollback", txId);
+                }
+                continue;
+            }
+
+            // MISSING_SCN
+            if (operationCode == LogMinerHelper.LOG_MINER_OC_MISSING_SCN) {
+                logger.warn("Found MISSING_SCN");
+                continue;
+            }
+
+            // DDL
+            String redoSql = getRedoSQL(rs);
+            if (operationCode == LogMinerHelper.LOG_MINER_OC_DDL) {
+                updateCommittedScn(scn.longValue());
+                listener.onEvent(new RedoEvent(scn.longValue(), operationCode, redoSql, segOwner, tableName, changeTime, txId));
+                continue;
+            }
+
+            // DML
+            if (operationCode == LogMinerHelper.LOG_MINER_OC_INSERT
+                    || operationCode == LogMinerHelper.LOG_MINER_OC_DELETE
+                    || operationCode == LogMinerHelper.LOG_MINER_OC_UPDATE) {
+                // 内部维护 TransactionalBuffer，将每条DML注册到Buffer中
+                // 根据事务提交或者回滚情况决定如何处理
+                if (redoSql != null) {
+                    final RedoEvent event = new RedoEvent(scn.longValue(), operationCode, redoSql, segOwner, tableName, changeTime, txId);
+                    // Transactional Commit Callback
+                    TransactionalBuffer.CommitCallback commitCallback = (smallestScn, commitScn, counter) -> {
+                        if (smallestScn == null || scn.compareTo(smallestScn) < 0) {
+                            // 当前SCN 事务已经提交 并且 小于事务缓冲区中所有的开始SCN，所以可以更新offsetScn
+                            startScn = scn.longValue();
+                        }
+
+                        if (counter == 0) {
+                            updateCommittedScn(commitScn.longValue());
+                        }
+
+                        event.setScn(committedScn);
+                        listener.onEvent(event);
+                    };
+                    transactionalBuffer.registerCommitCallback(txId, scn, commitCallback);
+                }
+            }
+        }
+    }
+
+    private void updateCommittedScn(long newScn) {
+        committedScn = newScn > committedScn ? newScn : committedScn;
+    }
+
+    private String getRedoSQL(ResultSet rs) throws SQLException {
+        String redoSql = rs.getString("SQL_REDO");
+        if (redoSql == null) {
+            return null;
+        }
+        StringBuilder redoBuilder = new StringBuilder(redoSql);
+
+        // https://docs.oracle.com/cd/B19306_01/server.102/b14237/dynviews_1154.htm#REFRN30132
+        // Continuation SQL flag. Possible values are:
+        // 0 = indicates SQL_REDO and SQL_UNDO is contained within the same row
+        // 1 = indicates that either SQL_REDO or SQL_UNDO is greater than 4000 bytes in size and is continued in the next row returned by the view
+        int csf = rs.getInt("CSF");
+
+        while (csf == 1) {
+            rs.next();
+            redoBuilder.append(rs.getString("SQL_REDO"));
+            csf = rs.getInt("CSF");
+        }
+
+        return redoBuilder.toString();
+    }
+
+    public void registerEventListener(EventListener listener){
+        this.listener = listener;
+    }
+
+    public void setStartScn(long startScn) {
+        this.startScn = startScn;
+    }
+
+    public interface EventListener {
+
+        void onEvent(RedoEvent redoEvent);
+    }
+
+}
\ No newline at end of file
diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/LogMinerHelper.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/LogMinerHelper.java
new file mode 100644
index 00000000..e3530d7b
--- /dev/null
+++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/LogMinerHelper.java
@@ -0,0 +1,240 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
+package org.dbsyncer.connector.oracle.logminer;
+
+import org.apache.commons.lang3.StringUtils;
+import org.dbsyncer.common.util.StringUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.math.BigInteger;
+import java.sql.CallableStatement;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+
+/**
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2023-12-09 20:23
+ */
+public class LogMinerHelper {
+    private static final Logger LOGGER = LoggerFactory.getLogger(LogMinerHelper.class);
+    public static final int LOG_MINER_OC_INSERT = 1;
+    public static final int LOG_MINER_OC_DELETE = 2;
+    public static final int LOG_MINER_OC_UPDATE = 3;
+    public static final int LOG_MINER_OC_DDL = 5;
+    public static final int LOG_MINER_OC_COMMIT = 7;
+    public static final int LOG_MINER_OC_MISSING_SCN = 34;
+    public static final int LOG_MINER_OC_ROLLBACK = 36;
+
+    public static void removeLogFilesFromMining(Connection conn) throws SQLException {
+        try (PreparedStatement ps = conn.prepareStatement("SELECT FILENAME AS NAME FROM V$LOGMNR_LOGS");
+             ResultSet result = ps.executeQuery()) {
+            Set files = new LinkedHashSet<>();
+            while (result.next()) {
+                files.add(result.getString(1));
+            }
+            for (String fileName : files) {
+                String sql = String.format("BEGIN SYS.DBMS_LOGMNR.REMOVE_LOGFILE(LOGFILENAME => '%s');END;", fileName);
+                executeCallableStatement(conn, sql);
+                LOGGER.debug("File {} was removed from mining", fileName);
+            }
+        }
+    }
+
+    public static void resetSessionToCdbIfNecessary(Connection connection) {
+        Statement statement = null;
+        try {
+            statement = connection.createStatement();
+            statement.execute("alter session set container=cdb$root");
+        } catch (SQLException e) {
+            throw new RuntimeException(e);
+        } finally {
+            if (statement != null) {
+                try {
+                    statement.close();
+                } catch (SQLException e) {
+                    LOGGER.error("Couldn't close statement", e);
+                }
+            }
+        }
+    }
+
+    public static void executeCallableStatement(Connection connection, String statement) throws SQLException {
+        Objects.requireNonNull(statement);
+        try (CallableStatement s = connection.prepareCall(statement)) {
+            s.execute();
+        }
+    }
+
+    public static List getOnlineLogFilesForOffsetScn(Connection connection, BigInteger offsetScn) throws SQLException {
+        List redoLogFiles = new ArrayList<>();
+
+        String onlineLogQuery = "SELECT MIN(F.MEMBER) AS FILE_NAME, L.NEXT_CHANGE# AS NEXT_CHANGE, F.GROUP#, L.FIRST_CHANGE# AS FIRST_CHANGE, L.STATUS " +
+                " FROM V$LOG L, V$LOGFILE F " +
+                " WHERE F.GROUP# = L.GROUP# AND L.NEXT_CHANGE# > 0 " +
+                " GROUP BY F.GROUP#, L.NEXT_CHANGE#, L.FIRST_CHANGE#, L.STATUS ORDER BY 3";
+
+        try (PreparedStatement s = connection.prepareStatement(onlineLogQuery)) {
+            try (ResultSet rs = s.executeQuery()) {
+                while (rs.next()) {
+                    String fileName = rs.getString(1);
+                    BigInteger nextChangeNumber = new BigInteger(rs.getString(2));
+                    BigInteger firstChangeNumber = new BigInteger(rs.getString(4));
+                    String status = rs.getString(5);
+                    LogFile logFile = new LogFile(fileName, firstChangeNumber, nextChangeNumber, "CURRENT".equalsIgnoreCase(status));
+                    // 添加Current Redo || scn 范围符合的
+                    if (logFile.isCurrent() || logFile.getNextScn().compareTo(offsetScn) >= 0) {
+                        redoLogFiles.add(logFile);
+                    }
+                }
+            }
+        }
+        return redoLogFiles;
+    }
+
+    public static List getArchivedLogFilesForOffsetScn(Connection connection, BigInteger offsetScn) throws SQLException {
+        String archiveLogsQuery = String.format("SELECT NAME AS FILE_NAME, NEXT_CHANGE# AS NEXT_CHANGE, FIRST_CHANGE# AS FIRST_CHANGE FROM V$ARCHIVED_LOG " +
+                "WHERE NAME IS NOT NULL AND ARCHIVED = 'YES' " +
+                "AND STATUS = 'A' AND NEXT_CHANGE# > %s ORDER BY 2", offsetScn);
+
+        final List archiveLogFiles = new ArrayList<>();
+        try (PreparedStatement s = connection.prepareStatement(archiveLogsQuery)) {
+            try (ResultSet rs = s.executeQuery()) {
+                while (rs.next()) {
+                    String fileName = rs.getString(1);
+                    BigInteger firstChangeNumber = new BigInteger(rs.getString(3));
+                    BigInteger nextChangeNumber = new BigInteger(rs.getString(2));
+                    archiveLogFiles.add(new LogFile(fileName, firstChangeNumber, nextChangeNumber, false));
+                }
+            }
+        }
+        return archiveLogFiles;
+    }
+
+    public static void addLogFile(Connection connection, String fileName) throws SQLException {
+        String addLogFile = "BEGIN sys.dbms_logmnr.add_logfile(LOGFILENAME => '%s', OPTIONS => %s);END;";
+        String options = "DBMS_LOGMNR.ADDFILE";
+//        String options = "DBMS_LOGMNR.NEW";
+        executeCallableStatement(connection, String.format(addLogFile, fileName, options));
+    }
+
+    public static List getCurrentRedoLogSequences(Connection connection) throws SQLException {
+        String currentRedoSequence = "SELECT SEQUENCE# FROM V$LOG WHERE STATUS = 'CURRENT'";
+        try (Statement statement = connection.createStatement();
+             ResultSet rs = statement.executeQuery(currentRedoSequence)) {
+            List sequences = new ArrayList<>();
+            if (rs.next()) {
+                sequences.add(new BigInteger(rs.getString(1)));
+            }
+            // 如果是RAC则会返回多个SEQUENCE
+            return sequences;
+        }
+    }
+
+    public static void buildDataDictionary(Connection connection, String miningStrategy) throws SQLException {
+        if (StringUtils.isBlank(miningStrategy)) {
+            // default
+            String sql = "BEGIN DBMS_LOGMNR_D.BUILD (options => DBMS_LOGMNR_D.STORE_IN_REDO_LOGS); END;";
+            executeCallableStatement(connection, sql);
+        }
+    }
+
+    public static void startLogMiner(Connection connection, BigInteger startScn, BigInteger endScn, String miningStrategy) throws SQLException {
+        LOGGER.debug("startLogMiner... startScn {}, endScn {}", startScn, endScn);
+        // default
+        if (StringUtils.isBlank(miningStrategy)) {
+            miningStrategy = "DBMS_LOGMNR.DICT_FROM_REDO_LOGS + DBMS_LOGMNR.DDL_DICT_TRACKING ";
+        }
+
+        String startLogMiner = "BEGIN sys.dbms_logmnr.start_logmnr(" +
+                "startScn => '" + startScn + "', " +
+                "endScn => '" + endScn + "', " +
+                "OPTIONS => " + miningStrategy +
+                " + DBMS_LOGMNR.NO_ROWID_IN_STMT);" +
+                "END;";
+
+        executeCallableStatement(connection, startLogMiner);
+    }
+
+    public static void endLogMiner(Connection connection) {
+        try {
+            executeCallableStatement(connection, "BEGIN SYS.DBMS_LOGMNR.END_LOGMNR(); END;");
+        } catch (SQLException e) {
+            if (e.getMessage().toUpperCase().contains("ORA-01307")) {
+                LOGGER.info("LogMiner session was already closed");
+            } else {
+                LOGGER.error("Cannot close LogMiner session gracefully: {}", e);
+            }
+        }
+    }
+
+    public static String logMinerViewQuery(String schema, String logMinerUser) {
+        StringBuilder query = new StringBuilder();
+//        query.append("SELECT SCN, SQL_REDO, OPERATION_CODE, TIMESTAMP, XID, CSF, TABLE_NAME, SEG_OWNER, OPERATION, USERNAME ");
+        query.append("SELECT * ");
+        query.append("FROM V$LOGMNR_CONTENTS ");
+        query.append("WHERE ");
+        // 这里由原来的 SCN > ? AND SCN <= ? 改为如下
+        // 原因：
+        // 在测试的时候发现一个情况会丢失部分数据
+        // 结论：
+        // START_SCN = X , END_SCN = Y, 此时查询条件 SCN >= X AND SCN <= Y
+        // 查询 V$LOGMNR_CONTENTS, 此时如果SQL的SCN恰好等于Y, 那么这次可能不会查出SCN=Y 的SQL(并不是百分之百)
+        // 但是当指定 SCN >= Y 时, 貌似一定能查到
+        // 这个问题很奇怪，有待研究
+        query.append("SCN >= ? AND SCN < ? ");
+        query.append("AND (");
+        // MISSING_SCN/DDL only when not performed by excluded users
+        query.append("(OPERATION_CODE IN (5,34) AND USERNAME NOT IN (").append(getExcludedUsers(logMinerUser)).append(")) ");
+        // COMMIT/ROLLBACK
+        query.append("OR (OPERATION_CODE IN (7,36)) ");
+        // INSERT/UPDATE/DELETE
+        query.append("OR ");
+        query.append("(OPERATION_CODE IN (1,2,3) ");
+        query.append(" AND SEG_OWNER NOT IN ('APPQOSSYS','AUDSYS','CTXSYS','DVSYS','DBSFWUSER','DBSNMP','GSMADMIN_INTERNAL','LBACSYS','MDSYS','OJVMSYS','OLAPSYS','ORDDATA','ORDSYS','OUTLN','SYS','SYSTEM','WMSYS','XDB') ");
+
+        if (StringUtils.isNotBlank(schema)) {
+            query.append(String.format(" AND (REGEXP_LIKE(SEG_OWNER,'^%s$','i')) ", schema));
+//            query.append(" AND ");
+//            query.append("USERNAME = '");
+//            query.append(schema);
+//            query.append("' ");
+        }
+
+        query.append(" ))");
+
+        return query.toString();
+    }
+
+    private static String getExcludedUsers(String logMinerUser) {
+        return "'SYS','SYSTEM','" + logMinerUser.toUpperCase() + "'";
+    }
+
+    public static void setSessionParameter(Connection connection) throws SQLException {
+        String sql = "ALTER SESSION SET "
+                + "  NLS_DATE_FORMAT = 'YYYY-MM-DD HH24:MI:SS'"
+                + "  NLS_TIMESTAMP_FORMAT = 'YYYY-MM-DD HH24:MI:SS.FF'"
+                + "  NLS_TIMESTAMP_TZ_FORMAT = 'YYYY-MM-DD HH24:MI:SS.FF TZH:TZM'"
+                + "  NLS_NUMERIC_CHARACTERS = '.,'";
+
+        executeCallableStatement(connection, sql);
+        executeCallableStatement(connection, "ALTER SESSION SET TIME_ZONE = '00:00'");
+    }
+
+    public void executeWithoutCommitting(Connection connection, String sql) throws SQLException {
+        try (Statement statement = connection.createStatement()) {
+            statement.execute(sql);
+        }
+    }
+
+}
\ No newline at end of file
diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/RedoEvent.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/RedoEvent.java
new file mode 100644
index 00000000..f338de8a
--- /dev/null
+++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/RedoEvent.java
@@ -0,0 +1,99 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
+package org.dbsyncer.connector.oracle.logminer;
+
+import java.sql.Timestamp;
+
+/**
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2023-12-09 20:22
+ */
+public final class RedoEvent {
+    private long scn;
+    private int operationCode;
+    private String redoSql;
+    private String objectOwner;
+    private String objectName;
+    private Timestamp sourceTime;
+    private String transactionId;
+
+    public RedoEvent(long scn, int operationCode, String redoSql, String objectOwner, String objectName, Timestamp sourceTime, String transactionId) {
+        this.scn = scn;
+        this.operationCode = operationCode;
+        this.redoSql = redoSql;
+        this.objectOwner = objectOwner;
+        this.objectName = objectName;
+        this.sourceTime = sourceTime;
+        this.transactionId = transactionId;
+    }
+
+    public long getScn() {
+        return scn;
+    }
+
+    public void setScn(long scn) {
+        this.scn = scn;
+    }
+
+    public int getOperationCode() {
+        return operationCode;
+    }
+
+    public void setOperationCode(int operationCode) {
+        this.operationCode = operationCode;
+    }
+
+    public String getRedoSql() {
+        return redoSql;
+    }
+
+    public void setRedoSql(String redoSql) {
+        this.redoSql = redoSql;
+    }
+
+    public String getObjectOwner() {
+        return objectOwner;
+    }
+
+    public void setObjectOwner(String objectOwner) {
+        this.objectOwner = objectOwner;
+    }
+
+    public String getObjectName() {
+        return objectName;
+    }
+
+    public void setObjectName(String objectName) {
+        this.objectName = objectName;
+    }
+
+    public Timestamp getSourceTime() {
+        return sourceTime;
+    }
+
+    public void setSourceTime(Timestamp sourceTime) {
+        this.sourceTime = sourceTime;
+    }
+
+    public String getTransactionId() {
+        return transactionId;
+    }
+
+    public void setTransactionId(String transactionId) {
+        this.transactionId = transactionId;
+    }
+
+    @Override
+    public String toString() {
+        return "LogMinerDmlObject{" +
+                "redoSql='" + redoSql + '\'' +
+                ", objectOwner='" + objectOwner + '\'' +
+                ", objectName='" + objectName + '\'' +
+                ", sourceTime=" + sourceTime +
+                ", transactionId='" + transactionId + '\'' +
+                ", scn=" + scn +
+                '}';
+    }
+}
\ No newline at end of file
diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/TransactionalBuffer.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/TransactionalBuffer.java
new file mode 100644
index 00000000..77a326ec
--- /dev/null
+++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/TransactionalBuffer.java
@@ -0,0 +1,107 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
+package org.dbsyncer.connector.oracle.logminer;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.math.BigInteger;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2023-12-09 20:26
+ */
+public class TransactionalBuffer {
+    private final Logger LOGGER = LoggerFactory.getLogger(TransactionalBuffer.class);
+    private final Map transactions;
+    private BigInteger lastCommittedScn;
+
+    public TransactionalBuffer() {
+        this.transactions = new HashMap<>();
+        this.lastCommittedScn = BigInteger.ZERO;
+    }
+
+    public interface CommitCallback {
+        void execute(BigInteger smallestScn, BigInteger commitScn, int callbackNumber) throws InterruptedException;
+    }
+
+    private final class Transaction {
+        private final BigInteger firstScn;
+        private BigInteger lastScn;
+        private final List commitCallbacks;
+
+        private Transaction(BigInteger firstScn) {
+            this.firstScn = firstScn;
+            this.commitCallbacks = new ArrayList<>();
+            this.lastScn = firstScn;
+        }
+
+        @Override
+        public String toString() {
+            return "Transaction{" +
+                    "firstScn=" + firstScn +
+                    ", lastScn=" + lastScn +
+                    '}';
+        }
+    }
+
+    public boolean isEmpty() {
+        return this.transactions.isEmpty();
+    }
+
+    private BigInteger calculateSmallestScn() {
+        return transactions.isEmpty() ? null
+                : transactions.values()
+                .stream()
+                .map(transaction -> transaction.firstScn)
+                .min(BigInteger::compareTo)
+                .orElseThrow(() -> new RuntimeException("Cannot calculate smallest SCN"));
+    }
+
+    public void registerCommitCallback(String transactionId, BigInteger scn, CommitCallback callback) {
+        transactions.computeIfAbsent(transactionId, s -> new Transaction(scn));
+
+        Transaction transaction = transactions.get(transactionId);
+        if (transaction != null) {
+            transaction.commitCallbacks.add(callback);
+        }
+    }
+
+    public boolean commit(String txId, BigInteger commitScn, long committedScn) {
+        Transaction transaction = transactions.remove(txId);
+        if (transaction == null) {
+            return false;
+        }
+
+        BigInteger smallestScn = calculateSmallestScn();
+
+        if (committedScn > commitScn.longValue() || lastCommittedScn.longValue() > commitScn.longValue()) {
+            LOGGER.warn("txId {} already commit, ignore.", txId);
+            return false;
+        }
+
+        int counter = transaction.commitCallbacks.size();
+        for (CommitCallback callback : transaction.commitCallbacks) {
+            try {
+                callback.execute(smallestScn, commitScn, --counter);
+            } catch (InterruptedException e) {
+                LOGGER.error(e.getMessage(), e);
+            }
+        }
+
+        lastCommittedScn = commitScn;
+        return true;
+    }
+
+    public boolean rollback(String txId) {
+        return transactions.remove(txId) != null;
+    }
+
+}
\ No newline at end of file
diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/AbstractConsumer.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/AbstractConsumer.java
index eab740a1..4fd461ed 100644
--- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/AbstractConsumer.java
+++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/AbstractConsumer.java
@@ -3,6 +3,7 @@
  */
 package org.dbsyncer.parser.consumer;
 
+import org.dbsyncer.sdk.enums.ChangedEventTypeEnum;
 import org.dbsyncer.sdk.listener.ChangedEvent;
 import org.dbsyncer.sdk.listener.event.DDLChangedEvent;
 import org.dbsyncer.sdk.listener.Watcher;
@@ -13,6 +14,7 @@ import org.dbsyncer.parser.LogType;
 import org.dbsyncer.parser.model.Mapping;
 import org.dbsyncer.parser.model.Meta;
 import org.dbsyncer.parser.model.TableGroup;
+import org.dbsyncer.sdk.listener.event.SqlChangedEvent;
 
 import java.util.List;
 import java.util.Map;
@@ -48,14 +50,24 @@ public abstract class AbstractConsumer implements Watche
     public void onDDLChanged(DDLChangedEvent event) {
     }
 
+    public void onSqlChanged(SqlChangedEvent event) {
+    }
+
     @Override
     public void changeEvent(ChangedEvent event) {
         event.getChangedOffset().setMetaId(metaId);
-        if (event instanceof DDLChangedEvent) {
-            onDDLChanged((DDLChangedEvent) event);
-            return;
+        switch (event.getType()){
+            case ROW:
+            case SCAN:
+                onChange((E) event);
+                break;
+            case SQL:
+                onSqlChanged((SqlChangedEvent) event);
+                break;
+            case DDL:
+                onDDLChanged((DDLChangedEvent) event);
+                break;
         }
-        onChange((E) event);
     }
 
     @Override
diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/impl/LogConsumer.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/impl/LogConsumer.java
index 026012cd..78310710 100644
--- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/impl/LogConsumer.java
+++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/impl/LogConsumer.java
@@ -11,6 +11,7 @@ import org.dbsyncer.parser.consumer.AbstractConsumer;
 import org.dbsyncer.parser.model.FieldPicker;
 import org.dbsyncer.parser.model.TableGroup;
 import org.dbsyncer.parser.util.PickerUtil;
+import org.dbsyncer.sdk.listener.event.SqlChangedEvent;
 import org.dbsyncer.sdk.model.Table;
 
 import java.util.ArrayList;
@@ -59,6 +60,11 @@ public final class LogConsumer extends AbstractConsumer {
         process(event, picker -> execute(picker.getTableGroup().getId(), event));
     }
 
+    @Override
+    public void onSqlChanged(SqlChangedEvent event) {
+        process(event, picker -> execute(picker.getTableGroup().getId(), event));
+    }
+
     private void process(CommonChangedEvent event, Consumer consumer) {
         // 处理过程有异常向上抛
         List pickers = tablePicker.get(event.getSourceTableName());
diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/GeneralBufferActuator.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/GeneralBufferActuator.java
index 0519e335..3a02bde1 100644
--- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/GeneralBufferActuator.java
+++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/GeneralBufferActuator.java
@@ -6,8 +6,6 @@ import org.dbsyncer.common.model.Result;
 import org.dbsyncer.common.util.CollectionUtils;
 import org.dbsyncer.common.util.StringUtil;
 import org.dbsyncer.connector.ConnectorFactory;
-import org.dbsyncer.sdk.config.DDLConfig;
-import org.dbsyncer.sdk.constant.ConnectorConstant;
 import org.dbsyncer.parser.ParserComponent;
 import org.dbsyncer.parser.ProfileComponent;
 import org.dbsyncer.parser.ddl.DDLParser;
@@ -21,13 +19,20 @@ import org.dbsyncer.parser.model.Picker;
 import org.dbsyncer.parser.model.TableGroup;
 import org.dbsyncer.parser.model.WriterRequest;
 import org.dbsyncer.parser.model.WriterResponse;
+import org.dbsyncer.parser.sql.SqlParser;
+import org.dbsyncer.parser.sql.impl.DeleteSql;
+import org.dbsyncer.parser.sql.impl.InsertSql;
+import org.dbsyncer.parser.sql.impl.UpdateSql;
 import org.dbsyncer.parser.strategy.FlushStrategy;
 import org.dbsyncer.parser.util.ConvertUtil;
 import org.dbsyncer.parser.util.PickerUtil;
 import org.dbsyncer.plugin.PluginFactory;
+import org.dbsyncer.plugin.impl.IncrementPluginContext;
+import org.dbsyncer.sdk.config.DDLConfig;
 import org.dbsyncer.sdk.connector.ConnectorInstance;
+import org.dbsyncer.sdk.constant.ConnectorConstant;
+import org.dbsyncer.sdk.enums.ChangedEventTypeEnum;
 import org.dbsyncer.sdk.model.ConnectorConfig;
-import org.dbsyncer.plugin.impl.IncrementPluginContext;
 import org.dbsyncer.sdk.model.MetaInfo;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -103,6 +108,7 @@ public class GeneralBufferActuator extends AbstractBufferActuator Delete > Insert，批处理任务中出现不同事件时，跳过分区处理
         // 跳过表结构修改事件（保证表结构修改原子性）
-        return !StringUtil.equals(nextRequest.getEvent(), response.getEvent()) || isDDLEvent(response.getEvent());
+        return !StringUtil.equals(nextRequest.getEvent(), response.getEvent()) || ChangedEventTypeEnum.isDDL(response.getTypeEnum());
     }
 
     @Override
@@ -123,10 +129,14 @@ public class GeneralBufferActuator extends AbstractBufferActuator dataList = new LinkedList<>();
 
     private List offsetList = new LinkedList<>();
@@ -33,6 +36,14 @@ public class WriterResponse extends AbstractWriter implements BufferResponse {
         return StringUtil.SYMBOL.concat(getEvent());
     }
 
+    public ChangedEventTypeEnum getTypeEnum() {
+        return typeEnum;
+    }
+
+    public void setTypeEnum(ChangedEventTypeEnum typeEnum) {
+        this.typeEnum = typeEnum;
+    }
+
     public List getDataList() {
         return dataList;
     }
diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/sql/SqlParser.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/sql/SqlParser.java
new file mode 100644
index 00000000..30651e58
--- /dev/null
+++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/sql/SqlParser.java
@@ -0,0 +1,14 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
+package org.dbsyncer.parser.sql;
+
+/**
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2023-12-10 00:43
+ */
+public interface SqlParser {
+
+    public String parse();
+}
\ No newline at end of file
diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/sql/impl/DeleteSql.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/sql/impl/DeleteSql.java
new file mode 100644
index 00000000..e6d6f933
--- /dev/null
+++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/sql/impl/DeleteSql.java
@@ -0,0 +1,76 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
+package org.dbsyncer.parser.sql.impl;
+
+import net.sf.jsqlparser.JSQLParserException;
+import net.sf.jsqlparser.expression.BinaryExpression;
+import net.sf.jsqlparser.expression.Expression;
+import net.sf.jsqlparser.parser.CCJSqlParserUtil;
+import net.sf.jsqlparser.schema.Column;
+import net.sf.jsqlparser.schema.Table;
+import net.sf.jsqlparser.statement.delete.Delete;
+import org.dbsyncer.parser.model.FieldMapping;
+import org.dbsyncer.parser.sql.SqlParser;
+
+import java.util.List;
+
+/**
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2023-12-10 00:43
+ */
+public final class DeleteSql implements SqlParser {
+
+    private String sql;
+
+    private String sourceTableName;
+
+    private String targetTableName;
+
+    private List fieldMappingList;
+
+    public DeleteSql(String sql, String sourceTableName, String targetTableName, List fieldMappingList) {
+        this.sql = sql;
+        this.sourceTableName = sourceTableName;
+        this.targetTableName = targetTableName;
+        this.fieldMappingList = fieldMappingList;
+    }
+
+    @Override
+    public String parse() {
+        try {
+            Delete delete = (Delete) CCJSqlParserUtil.parse(sql);
+            //替换表名
+            Table table = new Table();
+            table.setName(targetTableName);
+            delete.setTable(table);
+            whereParse(delete.getWhere());
+            return delete.toString();
+        } catch (JSQLParserException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private void whereParse(Expression expression){
+        BinaryExpression binaryExpression = (BinaryExpression) expression;
+        Expression left = binaryExpression.getLeftExpression();
+        Expression right = binaryExpression.getRightExpression();
+        findColumn((BinaryExpression) left);
+        findColumn((BinaryExpression) right);
+    }
+
+    private void findColumn(BinaryExpression binaryExpression){
+        if (binaryExpression.getLeftExpression() instanceof Column){
+            Column column = (Column) binaryExpression.getLeftExpression();
+            fieldMappingList.stream()
+                    .filter(x -> x.getSource().getName()
+                            .equals(column.getColumnName().replaceAll("\"", "")))
+                    .findFirst().ifPresent(
+                    fieldMapping -> column.setColumnName(fieldMapping.getTarget().getName()));
+            return;
+        }
+        findColumn((BinaryExpression) binaryExpression.getLeftExpression());
+        findColumn((BinaryExpression) binaryExpression.getRightExpression());
+    }
+}
\ No newline at end of file
diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/sql/impl/InsertSql.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/sql/impl/InsertSql.java
new file mode 100644
index 00000000..f1478c1b
--- /dev/null
+++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/sql/impl/InsertSql.java
@@ -0,0 +1,59 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
+package org.dbsyncer.parser.sql.impl;
+
+import net.sf.jsqlparser.JSQLParserException;
+import net.sf.jsqlparser.parser.CCJSqlParserUtil;
+import net.sf.jsqlparser.schema.Column;
+import net.sf.jsqlparser.schema.Table;
+import net.sf.jsqlparser.statement.insert.Insert;
+import org.dbsyncer.parser.model.FieldMapping;
+import org.dbsyncer.parser.sql.SqlParser;
+
+import java.util.List;
+
+/**
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2023-12-10 00:44
+ */
+public final class InsertSql implements SqlParser {
+
+    private String sql;
+
+    private String sourceTableName;
+
+    private String targetTableName;
+
+    private List fieldMappingList;
+
+    public InsertSql(String sql, String sourceTableName, String targetTableName, List fieldMappingList) {
+        this.sql = sql;
+        this.sourceTableName = sourceTableName;
+        this.targetTableName = targetTableName;
+        this.fieldMappingList = fieldMappingList;
+    }
+
+    public String parse() {
+        try {
+            Insert insert = (Insert) CCJSqlParserUtil.parse(this.sql);
+            //替换表名
+            Table table = new Table();
+            table.setName(targetTableName);
+            insert.setTable(table);
+            //替换字段
+            List columns = insert.getColumns();
+            for (Column column : columns) {
+                fieldMappingList.stream()
+                        .filter(x -> x.getSource().getName()
+                                .equals(column.getColumnName().replaceAll("\"", "")))
+                        .findFirst().ifPresent(
+                        fieldMapping -> column.setColumnName(fieldMapping.getTarget().getName()));
+            }
+            return insert.toString();
+        } catch (JSQLParserException e) {
+            throw new RuntimeException(e);
+        }
+    }
+}
\ No newline at end of file
diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/sql/impl/UpdateSql.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/sql/impl/UpdateSql.java
new file mode 100644
index 00000000..c0ec61a6
--- /dev/null
+++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/sql/impl/UpdateSql.java
@@ -0,0 +1,87 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
+package org.dbsyncer.parser.sql.impl;
+
+import net.sf.jsqlparser.JSQLParserException;
+import net.sf.jsqlparser.expression.BinaryExpression;
+import net.sf.jsqlparser.expression.Expression;
+import net.sf.jsqlparser.parser.CCJSqlParserUtil;
+import net.sf.jsqlparser.schema.Column;
+import net.sf.jsqlparser.schema.Table;
+import net.sf.jsqlparser.statement.update.Update;
+import net.sf.jsqlparser.statement.update.UpdateSet;
+import org.dbsyncer.parser.model.FieldMapping;
+import org.dbsyncer.parser.sql.SqlParser;
+
+import java.util.List;
+
+/**
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2023-12-10 00:45
+ */
+public final class UpdateSql implements SqlParser {
+
+    private String sql;
+
+    private String sourceTableName;
+
+    private String targetTableName;
+
+    private List fieldMappingList;
+
+    public UpdateSql(String sql, String sourceTableName, String targetTableName, List fieldMappingList) {
+        this.sql = sql;
+        this.sourceTableName = sourceTableName;
+        this.targetTableName = targetTableName;
+        this.fieldMappingList = fieldMappingList;
+    }
+
+    @Override
+    public String parse() {
+        try {
+            Update update = (Update) CCJSqlParserUtil.parse(sql);
+            //替换表名
+            Table table = new Table();
+            table.setName(targetTableName);
+            update.setTable(table);
+            for (UpdateSet updateSet : update.getUpdateSets()) {
+                List columns = updateSet.getColumns();
+                for (Column column : columns) {
+                    fieldMappingList.stream()
+                            .filter(x -> x.getSource().getName()
+                                    .equals(column.getColumnName().replaceAll("\"", "")))
+                            .findFirst().ifPresent(
+                            fieldMapping -> column.setColumnName(fieldMapping.getTarget().getName()));
+                }
+            }
+            whereParse(update.getWhere());
+            return update.toString();
+        } catch (JSQLParserException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private void whereParse(Expression expression) {
+        BinaryExpression binaryExpression = (BinaryExpression) expression;
+        Expression left = binaryExpression.getLeftExpression();
+        Expression right = binaryExpression.getRightExpression();
+        findColumn((BinaryExpression) left);
+        findColumn((BinaryExpression) right);
+    }
+
+    private void findColumn(BinaryExpression binaryExpression) {
+        if (binaryExpression.getLeftExpression() instanceof Column) {
+            Column column = (Column) binaryExpression.getLeftExpression();
+            fieldMappingList.stream()
+                    .filter(x -> x.getSource().getName()
+                            .equals(column.getColumnName().replaceAll("\"", "")))
+                    .findFirst().ifPresent(
+                    fieldMapping -> column.setColumnName(fieldMapping.getTarget().getName()));
+            return;
+        }
+        findColumn((BinaryExpression) binaryExpression.getLeftExpression());
+        findColumn((BinaryExpression) binaryExpression.getRightExpression());
+    }
+}
\ No newline at end of file
diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/ChangedEventTypeEnum.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/ChangedEventTypeEnum.java
new file mode 100644
index 00000000..a4b97bb4
--- /dev/null
+++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/ChangedEventTypeEnum.java
@@ -0,0 +1,35 @@
+package org.dbsyncer.sdk.enums;
+
+/**
+ * 变更事件类型枚举
+ *
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2023-12-09 20:34
+ */
+public enum ChangedEventTypeEnum {
+    /**
+     * ddl变更
+     */
+    DDL,
+    /**
+     * 表列变更，比如oracle, 会获取变动列的sql
+     */
+    SQL,
+    /**
+     * 定时变更
+     */
+    SCAN,
+    /**
+     * 表行变更，比如mysql, 会获取变动行所有列的值
+     */
+    ROW;
+
+    public static boolean isDDL(ChangedEventTypeEnum event) {
+        return event != null && DDL == event;
+    }
+
+    public static boolean isSQL(ChangedEventTypeEnum event) {
+        return event != null && SQL == event;
+    }
+}
\ No newline at end of file
diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/ChangedEvent.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/ChangedEvent.java
index 5a2b62d8..ec9669ba 100644
--- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/ChangedEvent.java
+++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/ChangedEvent.java
@@ -3,6 +3,7 @@
  */
 package org.dbsyncer.sdk.listener;
 
+import org.dbsyncer.sdk.enums.ChangedEventTypeEnum;
 import org.dbsyncer.sdk.model.ChangedOffset;
 
 import java.util.Map;
@@ -16,6 +17,13 @@ import java.util.Map;
  */
 public interface ChangedEvent {
 
+    /**
+     * 获取变更事件类型
+     *
+     * @return
+     */
+    ChangedEventTypeEnum getType();
+
     /**
      * 获取变更表名称
      *
@@ -30,6 +38,13 @@ public interface ChangedEvent {
      */
     String getEvent();
 
+    /**
+     * 获取变更SQL
+     *
+     * @return
+     */
+    String getSql();
+
     /**
      * 获取变更行数据
      *
diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/event/CommonChangedEvent.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/event/CommonChangedEvent.java
index a8f95df7..6b7e4fbf 100644
--- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/event/CommonChangedEvent.java
+++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/event/CommonChangedEvent.java
@@ -15,7 +15,7 @@ import java.util.Map;
  * @Author AE86
  * @Date 2023-08-20 20:00
  */
-public class CommonChangedEvent implements ChangedEvent {
+public abstract class CommonChangedEvent implements ChangedEvent {
 
     /**
      * 变更表名称
@@ -25,6 +25,10 @@ public class CommonChangedEvent implements ChangedEvent {
      * 变更事件
      */
     private String event;
+    /**
+     * 变更sql
+     */
+    private String sql;
     /**
      * 变更行数据
      */
@@ -52,6 +56,15 @@ public class CommonChangedEvent implements ChangedEvent {
         this.event = event;
     }
 
+    @Override
+    public String getSql() {
+        return sql;
+    }
+
+    public void setSql(String sql) {
+        this.sql = sql;
+    }
+
     @Override
     public Map getChangedRow() {
         return changedRow;
diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/event/DDLChangedEvent.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/event/DDLChangedEvent.java
index 5cf81f36..c6574858 100644
--- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/event/DDLChangedEvent.java
+++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/event/DDLChangedEvent.java
@@ -1,5 +1,7 @@
 package org.dbsyncer.sdk.listener.event;
 
+import org.dbsyncer.sdk.enums.ChangedEventTypeEnum;
+
 /**
  * DDL变更事件
  *
@@ -14,25 +16,21 @@ public final class DDLChangedEvent extends CommonChangedEvent {
       */
     private String database;
 
-    /**
-     * 变更SQL
-     */
-    private String sql;
-
     public DDLChangedEvent(String database, String sourceTableName, String event, String sql, String nextFileName, Object position) {
         setSourceTableName(sourceTableName);
         setEvent(event);
         setNextFileName(nextFileName);
         setPosition(position);
+        setSql(sql);
         this.database = database;
-        this.sql = sql;
     }
 
     public String getDatabase() {
         return database;
     }
 
-    public String getSql() {
-        return sql;
+    @Override
+    public ChangedEventTypeEnum getType() {
+        return ChangedEventTypeEnum.DDL;
     }
 }
diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/event/RowChangedEvent.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/event/RowChangedEvent.java
index 307b710d..72af8f2e 100644
--- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/event/RowChangedEvent.java
+++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/event/RowChangedEvent.java
@@ -3,6 +3,8 @@
  */
 package org.dbsyncer.sdk.listener.event;
 
+import org.dbsyncer.sdk.enums.ChangedEventTypeEnum;
+
 import java.util.List;
 
 /**
@@ -12,7 +14,7 @@ import java.util.List;
  * @Author AE86
  * @Date 2020-06-15 20:00
  */
-public final class RowChangedEvent extends CommonChangedEvent {
+public class RowChangedEvent extends CommonChangedEvent {
     private List dataList;
 
     public RowChangedEvent(String sourceTableName, String event, List data) {
@@ -31,4 +33,8 @@ public final class RowChangedEvent extends CommonChangedEvent {
         return dataList;
     }
 
+    @Override
+    public ChangedEventTypeEnum getType() {
+        return ChangedEventTypeEnum.ROW;
+    }
 }
\ No newline at end of file
diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/event/ScanChangedEvent.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/event/ScanChangedEvent.java
index 07abad00..35c14b7f 100644
--- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/event/ScanChangedEvent.java
+++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/event/ScanChangedEvent.java
@@ -3,6 +3,8 @@
  */
 package org.dbsyncer.sdk.listener.event;
 
+import org.dbsyncer.sdk.enums.ChangedEventTypeEnum;
+
 import java.util.Map;
 
 /**
@@ -25,4 +27,9 @@ public final class ScanChangedEvent extends CommonChangedEvent {
     public int getTableGroupIndex() {
         return tableGroupIndex;
     }
+
+    @Override
+    public ChangedEventTypeEnum getType() {
+        return ChangedEventTypeEnum.SCAN;
+    }
 }
\ No newline at end of file
diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/event/SqlChangedEvent.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/event/SqlChangedEvent.java
new file mode 100644
index 00000000..824d7878
--- /dev/null
+++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/event/SqlChangedEvent.java
@@ -0,0 +1,27 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
+package org.dbsyncer.sdk.listener.event;
+
+import org.dbsyncer.sdk.enums.ChangedEventTypeEnum;
+
+/**
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2023-12-09 20:34
+ */
+public final class SqlChangedEvent extends CommonChangedEvent {
+
+    public SqlChangedEvent(String sourceTableName, String event, String sql, String nextFileName, Object position) {
+        setSourceTableName(sourceTableName);
+        setEvent(event);
+        setNextFileName(nextFileName);
+        setPosition(position);
+        setSql(sql);
+    }
+
+    @Override
+    public ChangedEventTypeEnum getType() {
+        return ChangedEventTypeEnum.SQL;
+    }
+}
\ No newline at end of file
-- 
Gitee


From ee1533dfd237ea89b8241100567fa0860b675f79 Mon Sep 17 00:00:00 2001
From: life <13122192336@163.com>
Date: Sun, 10 Dec 2023 10:37:08 +0800
Subject: [PATCH 60/98] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=88=B7=E6=96=B0?=
 =?UTF-8?q?=E7=82=B9=E4=BD=8D=E5=88=B0=E5=AD=98=E5=82=A8=E5=92=8C=E6=95=B0?=
 =?UTF-8?q?=E9=87=8F=E6=98=BE=E7=A4=BA?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../org/dbsyncer/connector/oracle/cdc/OracleListener.java   | 6 ++++++
 .../sdk/connector/database/AbstractDatabaseConnector.java   | 3 +++
 2 files changed, 9 insertions(+)

diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java
index e9b33c82..1b2f3b12 100644
--- a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java
+++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java
@@ -49,6 +49,12 @@ public class OracleListener extends AbstractDatabaseListener {
             logMiner = new LogMiner(username, password, url, schema, driverClassName);
             logMiner.setStartScn(containsPos ? Long.parseLong(snapshot.get(REDO_POSITION)) : 0);
             logMiner.registerEventListener((event) -> {
+                if (snapshot.containsKey(REDO_POSITION)){
+                    snapshot.replace(REDO_POSITION,String.valueOf(event.getScn()));
+                }else{
+                    snapshot.putIfAbsent(REDO_POSITION,String.valueOf(event.getScn()));
+                }
+
                 try {
                     Statement statement = CCJSqlParserUtil.parse(event.getRedoSql());
                     if (statement instanceof Update) {
diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/AbstractDatabaseConnector.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/AbstractDatabaseConnector.java
index 411e22fe..fb6cc455 100644
--- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/AbstractDatabaseConnector.java
+++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/AbstractDatabaseConnector.java
@@ -671,6 +671,9 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector implem
                 databaseTemplate.execute(DatabaseConstant.DBS_UNIQUE_CODE.concat(config.getSql()));
                 return true;
             });
+            Map successMap = new HashMap<>();
+            successMap.put("sql",config.getSql());
+            result.addSuccessData(Collections.singletonList(successMap));
         } catch (Exception e) {
             result.getError().append(String.format("执行ddl: %s, 异常：%s", config.getSql(), e.getMessage()));
         }
-- 
Gitee


From 2adacf6ffaa9e1430685977d8eb7aa3b6b5d24fe Mon Sep 17 00:00:00 2001
From: AE86 <836391306@qq.com>
Date: Mon, 11 Dec 2023 00:29:34 +0800
Subject: [PATCH 61/98] fix bug

---
 .../connector/oracle/cdc/OracleListener.java  | 16 ++++++----
 .../connector/oracle/logminer/LogMiner.java   |  5 ++-
 .../oracle/logminer/LogMinerHelper.java       | 31 ++-----------------
 .../database/AbstractDatabaseConnector.java   |  4 +--
 4 files changed, 19 insertions(+), 37 deletions(-)

diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java
index 1b2f3b12..f26f7954 100644
--- a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java
+++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java
@@ -19,6 +19,7 @@ import org.dbsyncer.sdk.constant.ConnectorConstant;
 import org.dbsyncer.sdk.listener.AbstractDatabaseListener;
 import org.dbsyncer.sdk.listener.event.DDLChangedEvent;
 import org.dbsyncer.sdk.listener.event.SqlChangedEvent;
+import org.dbsyncer.sdk.model.ChangedOffset;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -49,12 +50,6 @@ public class OracleListener extends AbstractDatabaseListener {
             logMiner = new LogMiner(username, password, url, schema, driverClassName);
             logMiner.setStartScn(containsPos ? Long.parseLong(snapshot.get(REDO_POSITION)) : 0);
             logMiner.registerEventListener((event) -> {
-                if (snapshot.containsKey(REDO_POSITION)){
-                    snapshot.replace(REDO_POSITION,String.valueOf(event.getScn()));
-                }else{
-                    snapshot.putIfAbsent(REDO_POSITION,String.valueOf(event.getScn()));
-                }
-
                 try {
                     Statement statement = CCJSqlParserUtil.parse(event.getRedoSql());
                     if (statement instanceof Update) {
@@ -86,6 +81,10 @@ public class OracleListener extends AbstractDatabaseListener {
             });
             logMiner.start();
 
+            if (!containsPos) {
+                snapshot.put(REDO_POSITION, String.valueOf(logMiner.getStartScn()));
+                super.forceFlushEvent();
+            }
         } catch (Exception e) {
             logger.error("启动失败:{}", e.getMessage());
             throw new OracleException(e);
@@ -103,6 +102,11 @@ public class OracleListener extends AbstractDatabaseListener {
         }
     }
 
+    @Override
+    public void refreshEvent(ChangedOffset offset) {
+        snapshot.put(REDO_POSITION, String.valueOf(offset.getPosition()));
+    }
+
     private String replaceTableName(Table table) {
         if (table == null) {
             return StringUtil.EMPTY;
diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/LogMiner.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/LogMiner.java
index be46509d..dd3580b3 100644
--- a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/LogMiner.java
+++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/LogMiner.java
@@ -84,7 +84,6 @@ public class LogMiner {
 
         logger.info("scn start '{}'", startScn);
         logger.info("start LogMiner...");
-//        LogMinerHelper.resetSessionToCdbIfNecessary(connection);
         LogMinerHelper.setSessionParameter(connection);
 
         // 1.记录当前redoLog，用于下文判断redoLog 是否切换
@@ -309,6 +308,10 @@ public class LogMiner {
         this.listener = listener;
     }
 
+    public long getStartScn() {
+        return startScn;
+    }
+
     public void setStartScn(long startScn) {
         this.startScn = startScn;
     }
diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/LogMinerHelper.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/LogMinerHelper.java
index e3530d7b..2b70ddef 100644
--- a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/LogMinerHelper.java
+++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/LogMinerHelper.java
@@ -3,7 +3,6 @@
  */
 package org.dbsyncer.connector.oracle.logminer;
 
-import org.apache.commons.lang3.StringUtils;
 import org.dbsyncer.common.util.StringUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -51,24 +50,6 @@ public class LogMinerHelper {
         }
     }
 
-    public static void resetSessionToCdbIfNecessary(Connection connection) {
-        Statement statement = null;
-        try {
-            statement = connection.createStatement();
-            statement.execute("alter session set container=cdb$root");
-        } catch (SQLException e) {
-            throw new RuntimeException(e);
-        } finally {
-            if (statement != null) {
-                try {
-                    statement.close();
-                } catch (SQLException e) {
-                    LOGGER.error("Couldn't close statement", e);
-                }
-            }
-        }
-    }
-
     public static void executeCallableStatement(Connection connection, String statement) throws SQLException {
         Objects.requireNonNull(statement);
         try (CallableStatement s = connection.prepareCall(statement)) {
@@ -142,7 +123,7 @@ public class LogMinerHelper {
     }
 
     public static void buildDataDictionary(Connection connection, String miningStrategy) throws SQLException {
-        if (StringUtils.isBlank(miningStrategy)) {
+        if (StringUtil.isBlank(miningStrategy)) {
             // default
             String sql = "BEGIN DBMS_LOGMNR_D.BUILD (options => DBMS_LOGMNR_D.STORE_IN_REDO_LOGS); END;";
             executeCallableStatement(connection, sql);
@@ -152,7 +133,7 @@ public class LogMinerHelper {
     public static void startLogMiner(Connection connection, BigInteger startScn, BigInteger endScn, String miningStrategy) throws SQLException {
         LOGGER.debug("startLogMiner... startScn {}, endScn {}", startScn, endScn);
         // default
-        if (StringUtils.isBlank(miningStrategy)) {
+        if (StringUtil.isBlank(miningStrategy)) {
             miningStrategy = "DBMS_LOGMNR.DICT_FROM_REDO_LOGS + DBMS_LOGMNR.DDL_DICT_TRACKING ";
         }
 
@@ -203,7 +184,7 @@ public class LogMinerHelper {
         query.append("(OPERATION_CODE IN (1,2,3) ");
         query.append(" AND SEG_OWNER NOT IN ('APPQOSSYS','AUDSYS','CTXSYS','DVSYS','DBSFWUSER','DBSNMP','GSMADMIN_INTERNAL','LBACSYS','MDSYS','OJVMSYS','OLAPSYS','ORDDATA','ORDSYS','OUTLN','SYS','SYSTEM','WMSYS','XDB') ");
 
-        if (StringUtils.isNotBlank(schema)) {
+        if (StringUtil.isNotBlank(schema)) {
             query.append(String.format(" AND (REGEXP_LIKE(SEG_OWNER,'^%s$','i')) ", schema));
 //            query.append(" AND ");
 //            query.append("USERNAME = '");
@@ -231,10 +212,4 @@ public class LogMinerHelper {
         executeCallableStatement(connection, "ALTER SESSION SET TIME_ZONE = '00:00'");
     }
 
-    public void executeWithoutCommitting(Connection connection, String sql) throws SQLException {
-        try (Statement statement = connection.createStatement()) {
-            statement.execute(sql);
-        }
-    }
-
 }
\ No newline at end of file
diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/AbstractDatabaseConnector.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/AbstractDatabaseConnector.java
index fb6cc455..a6ec57d7 100644
--- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/AbstractDatabaseConnector.java
+++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/AbstractDatabaseConnector.java
@@ -671,8 +671,8 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector implem
                 databaseTemplate.execute(DatabaseConstant.DBS_UNIQUE_CODE.concat(config.getSql()));
                 return true;
             });
-            Map successMap = new HashMap<>();
-            successMap.put("sql",config.getSql());
+            Map successMap = new HashMap<>();
+            successMap.put("sql", config.getSql());
             result.addSuccessData(Collections.singletonList(successMap));
         } catch (Exception e) {
             result.getError().append(String.format("执行ddl: %s, 异常：%s", config.getSql(), e.getMessage()));
-- 
Gitee


From 41e1eac507da985a7bcbcbc81761d0e6f1410e56 Mon Sep 17 00:00:00 2001
From: life <13122192336@163.com>
Date: Mon, 11 Dec 2023 10:18:36 +0800
Subject: [PATCH 62/98] =?UTF-8?q?=E5=8E=BB=E6=8E=89=E4=B8=8D=E5=8F=AF?=
 =?UTF-8?q?=E8=BE=BE=E8=AF=AD=E5=8F=A5=EF=BC=8C=E4=BF=AE=E6=94=B9=E4=B8=8D?=
 =?UTF-8?q?=E6=AD=A2sql=E6=B3=A8=E9=87=8A?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../org/dbsyncer/connector/oracle/cdc/OracleListener.java  | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java
index f26f7954..05cdb9c9 100644
--- a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java
+++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java
@@ -76,15 +76,10 @@ public class OracleListener extends AbstractDatabaseListener {
                         return;
                     }
                 } catch (JSQLParserException e) {
-                    logger.error("parse redoSql=" + event.getRedoSql(), e);
+                    logger.error("不支持sql:" + event.getRedoSql());
                 }
             });
             logMiner.start();
-
-            if (!containsPos) {
-                snapshot.put(REDO_POSITION, String.valueOf(logMiner.getStartScn()));
-                super.forceFlushEvent();
-            }
         } catch (Exception e) {
             logger.error("启动失败:{}", e.getMessage());
             throw new OracleException(e);
-- 
Gitee


From 92fdbd2371244c96aa0a419e9bb79c944bab9d72 Mon Sep 17 00:00:00 2001
From: AE86 <836391306@qq.com>
Date: Mon, 11 Dec 2023 23:00:53 +0800
Subject: [PATCH 63/98] move storage

---
 .../biz/impl/DataSyncServiceImpl.java         | 15 ++--
 .../org/dbsyncer/biz/impl/MetricReporter.java |  6 +-
 .../dbsyncer/biz/impl/MonitorServiceImpl.java | 23 +++---
 .../manager/impl/PreloadTemplate.java         |  4 +-
 .../parser/command/PersistenceCommand.java    |  2 +-
 .../parser/flush/impl/FlushServiceImpl.java   |  4 +-
 .../flush/impl/StorageBufferActuator.java     |  2 +-
 .../parser/impl/OperationTemplate.java        |  4 +-
 .../dbsyncer/sdk/enums/FilterTypeEnum.java    | 34 +++++++++
 .../java/org/dbsyncer/sdk/enums/SortEnum.java | 75 +++++++++----------
 .../org/dbsyncer/sdk}/enums/StorageEnum.java  | 65 ++++++++--------
 .../java/org/dbsyncer/sdk/model/Filter.java   |  3 +
 .../dbsyncer/sdk/storage}/AbstractFilter.java | 33 ++++----
 .../dbsyncer/sdk/storage}/BooleanFilter.java  |  2 +-
 .../dbsyncer/sdk/storage/FieldResolver.java   | 16 ++++
 .../java/org/dbsyncer/sdk/storage}/Query.java | 27 ++++---
 .../sdk/storage/filter/IntFilter.java         | 24 ++++++
 .../sdk/storage}/filter/LongFilter.java       | 23 +++---
 .../sdk/storage}/filter/StringFilter.java     | 23 +++---
 .../storage/AbstractStorageService.java       | 30 +++-----
 .../org/dbsyncer/storage/StorageService.java  | 30 +-------
 .../storage/constant/BinlogConstant.java      | 19 -----
 .../storage/enums/BinlogStatusEnum.java       | 18 -----
 .../storage/enums/IndexFieldResolverEnum.java | 24 ------
 .../storage/impl/DiskQueryHelper.java         | 57 ++++++++++++++
 .../storage/impl/DiskStorageServiceImpl.java  | 16 ++--
 .../storage/impl/MySQLStorageServiceImpl.java | 12 +--
 .../storage/lucene/IndexFieldResolver.java    |  9 ---
 .../org/dbsyncer/storage/lucene/Option.java   | 28 +++----
 .../org/dbsyncer/storage/lucene/Shard.java    | 11 ++-
 .../storage/query/filter/IntFilter.java       | 31 --------
 .../storage/strategy/impl/ConfigStrategy.java |  2 +-
 .../storage/strategy/impl/DataStrategy.java   |  2 +-
 .../storage/strategy/impl/LogStrategy.java    |  2 +-
 34 files changed, 336 insertions(+), 340 deletions(-)
 create mode 100644 dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/FilterTypeEnum.java
 rename dbsyncer-storage/src/main/java/org/dbsyncer/storage/enums/BinlogSortEnum.java => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/SortEnum.java (72%)
 rename {dbsyncer-storage/src/main/java/org/dbsyncer/storage => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/enums/StorageEnum.java (77%)
 rename {dbsyncer-storage/src/main/java/org/dbsyncer/storage/query => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage}/AbstractFilter.java (45%)
 rename {dbsyncer-storage/src/main/java/org/dbsyncer/storage/query => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage}/BooleanFilter.java (97%)
 create mode 100644 dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/FieldResolver.java
 rename {dbsyncer-storage/src/main/java/org/dbsyncer/storage/query => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage}/Query.java (74%)
 create mode 100644 dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/filter/IntFilter.java
 rename {dbsyncer-storage/src/main/java/org/dbsyncer/storage/query => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage}/filter/LongFilter.java (32%)
 rename {dbsyncer-storage/src/main/java/org/dbsyncer/storage/query => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage}/filter/StringFilter.java (35%)
 delete mode 100644 dbsyncer-storage/src/main/java/org/dbsyncer/storage/constant/BinlogConstant.java
 delete mode 100644 dbsyncer-storage/src/main/java/org/dbsyncer/storage/enums/BinlogStatusEnum.java
 delete mode 100644 dbsyncer-storage/src/main/java/org/dbsyncer/storage/enums/IndexFieldResolverEnum.java
 create mode 100644 dbsyncer-storage/src/main/java/org/dbsyncer/storage/impl/DiskQueryHelper.java
 delete mode 100644 dbsyncer-storage/src/main/java/org/dbsyncer/storage/lucene/IndexFieldResolver.java
 delete mode 100644 dbsyncer-storage/src/main/java/org/dbsyncer/storage/query/filter/IntFilter.java

diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/DataSyncServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/DataSyncServiceImpl.java
index 45ed37e1..a5a96921 100644
--- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/DataSyncServiceImpl.java
+++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/DataSyncServiceImpl.java
@@ -1,6 +1,7 @@
 package org.dbsyncer.biz.impl;
 
 import com.google.protobuf.InvalidProtocolBufferException;
+import org.apache.lucene.index.IndexableField;
 import org.dbsyncer.biz.DataSyncService;
 import org.dbsyncer.biz.vo.BinlogColumnVo;
 import org.dbsyncer.biz.vo.MessageVo;
@@ -10,19 +11,19 @@ import org.dbsyncer.common.util.DateFormatUtil;
 import org.dbsyncer.common.util.JsonUtil;
 import org.dbsyncer.common.util.NumberUtil;
 import org.dbsyncer.common.util.StringUtil;
-import org.dbsyncer.sdk.listener.event.RowChangedEvent;
 import org.dbsyncer.parser.ProfileComponent;
 import org.dbsyncer.parser.flush.impl.BufferActuatorRouter;
 import org.dbsyncer.parser.model.Meta;
 import org.dbsyncer.parser.model.Picker;
 import org.dbsyncer.parser.model.TableGroup;
+import org.dbsyncer.sdk.enums.StorageEnum;
+import org.dbsyncer.sdk.listener.event.RowChangedEvent;
 import org.dbsyncer.sdk.model.Field;
+import org.dbsyncer.sdk.storage.FieldResolver;
+import org.dbsyncer.sdk.storage.Query;
 import org.dbsyncer.storage.StorageService;
 import org.dbsyncer.storage.binlog.proto.BinlogMap;
 import org.dbsyncer.storage.constant.ConfigConstant;
-import org.dbsyncer.storage.enums.IndexFieldResolverEnum;
-import org.dbsyncer.storage.enums.StorageEnum;
-import org.dbsyncer.storage.query.Query;
 import org.dbsyncer.storage.util.BinlogMessageUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -177,9 +178,9 @@ public class DataSyncServiceImpl implements DataSyncService {
 
     private Map getData(String metaId, String messageId) {
         Query query = new Query(1, 1);
-        Map fieldResolvers = new LinkedHashMap<>();
-        fieldResolvers.put(ConfigConstant.BINLOG_DATA, IndexFieldResolverEnum.BINARY);
-        query.setIndexFieldResolverMap(fieldResolvers);
+        Map fieldResolvers = new LinkedHashMap<>();
+        fieldResolvers.put(ConfigConstant.BINLOG_DATA, (FieldResolver) field -> field.binaryValue().bytes);
+        query.setFieldResolverMap(fieldResolvers);
         query.addFilter(ConfigConstant.CONFIG_MODEL_ID, messageId);
         query.setMetaId(metaId);
         query.setType(StorageEnum.DATA);
diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MetricReporter.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MetricReporter.java
index 0b638f69..c7b835f2 100644
--- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MetricReporter.java
+++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MetricReporter.java
@@ -13,7 +13,6 @@ import org.dbsyncer.common.scheduled.ScheduledTaskJob;
 import org.dbsyncer.common.scheduled.ScheduledTaskService;
 import org.dbsyncer.common.util.CollectionUtils;
 import org.dbsyncer.common.util.StringUtil;
-import org.dbsyncer.sdk.constant.ConnectorConstant;
 import org.dbsyncer.parser.ProfileComponent;
 import org.dbsyncer.parser.flush.BufferActuator;
 import org.dbsyncer.parser.flush.impl.BufferActuatorRouter;
@@ -21,11 +20,12 @@ import org.dbsyncer.parser.flush.impl.TableGroupBufferActuator;
 import org.dbsyncer.parser.model.Mapping;
 import org.dbsyncer.parser.model.Meta;
 import org.dbsyncer.parser.model.TableGroup;
+import org.dbsyncer.sdk.constant.ConnectorConstant;
+import org.dbsyncer.sdk.enums.StorageEnum;
+import org.dbsyncer.sdk.storage.Query;
 import org.dbsyncer.storage.StorageService;
 import org.dbsyncer.storage.constant.ConfigConstant;
 import org.dbsyncer.storage.enums.StorageDataStatusEnum;
-import org.dbsyncer.storage.enums.StorageEnum;
-import org.dbsyncer.storage.query.Query;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java
index 0b7fd342..02f47a68 100644
--- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java
+++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java
@@ -3,6 +3,7 @@
  */
 package org.dbsyncer.biz.impl;
 
+import org.apache.lucene.index.IndexableField;
 import org.dbsyncer.biz.DataSyncService;
 import org.dbsyncer.biz.MonitorService;
 import org.dbsyncer.biz.SystemConfigService;
@@ -24,28 +25,28 @@ import org.dbsyncer.biz.vo.LogVo;
 import org.dbsyncer.biz.vo.MetaVo;
 import org.dbsyncer.biz.vo.MetricResponseVo;
 import org.dbsyncer.common.model.Paging;
+import org.dbsyncer.common.scheduled.ScheduledTaskJob;
+import org.dbsyncer.common.scheduled.ScheduledTaskService;
 import org.dbsyncer.common.util.CollectionUtils;
 import org.dbsyncer.common.util.JsonUtil;
 import org.dbsyncer.common.util.NumberUtil;
 import org.dbsyncer.common.util.StringUtil;
-import org.dbsyncer.sdk.enums.FilterEnum;
-import org.dbsyncer.common.scheduled.ScheduledTaskJob;
-import org.dbsyncer.common.scheduled.ScheduledTaskService;
 import org.dbsyncer.parser.LogService;
 import org.dbsyncer.parser.LogType;
 import org.dbsyncer.parser.ProfileComponent;
 import org.dbsyncer.parser.enums.MetaEnum;
 import org.dbsyncer.parser.model.Mapping;
 import org.dbsyncer.parser.model.Meta;
+import org.dbsyncer.sdk.enums.FilterEnum;
 import org.dbsyncer.sdk.enums.ModelEnum;
+import org.dbsyncer.sdk.enums.StorageEnum;
+import org.dbsyncer.sdk.storage.BooleanFilter;
+import org.dbsyncer.sdk.storage.FieldResolver;
+import org.dbsyncer.sdk.storage.Query;
+import org.dbsyncer.sdk.storage.filter.LongFilter;
 import org.dbsyncer.storage.StorageService;
 import org.dbsyncer.storage.constant.ConfigConstant;
-import org.dbsyncer.storage.enums.IndexFieldResolverEnum;
 import org.dbsyncer.storage.enums.StorageDataStatusEnum;
-import org.dbsyncer.storage.enums.StorageEnum;
-import org.dbsyncer.storage.query.BooleanFilter;
-import org.dbsyncer.storage.query.Query;
-import org.dbsyncer.storage.query.filter.LongFilter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.BeanUtils;
@@ -264,9 +265,9 @@ public class MonitorServiceImpl extends BaseServiceImpl implements MonitorServic
             return new Paging(pageNum, pageSize);
         }
         Query query = new Query(pageNum, pageSize);
-        Map fieldResolvers = new LinkedHashMap<>();
-        fieldResolvers.put(ConfigConstant.BINLOG_DATA, IndexFieldResolverEnum.BINARY);
-        query.setIndexFieldResolverMap(fieldResolvers);
+        Map fieldResolvers = new LinkedHashMap<>();
+        fieldResolvers.put(ConfigConstant.BINLOG_DATA, (FieldResolver) field -> field.binaryValue().bytes);
+        query.setFieldResolverMap(fieldResolvers);
 
         // 查询异常信息
         if (StringUtil.isNotBlank(error)) {
diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/PreloadTemplate.java b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/PreloadTemplate.java
index 0fd208b5..8f623cf5 100644
--- a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/PreloadTemplate.java
+++ b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/PreloadTemplate.java
@@ -20,10 +20,10 @@ import org.dbsyncer.parser.model.Meta;
 import org.dbsyncer.parser.model.OperationConfig;
 import org.dbsyncer.plugin.PluginFactory;
 import org.dbsyncer.sdk.connector.ConnectorInstance;
+import org.dbsyncer.sdk.enums.StorageEnum;
+import org.dbsyncer.sdk.storage.Query;
 import org.dbsyncer.storage.StorageService;
 import org.dbsyncer.storage.constant.ConfigConstant;
-import org.dbsyncer.storage.enums.StorageEnum;
-import org.dbsyncer.storage.query.Query;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.context.ApplicationListener;
diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/command/PersistenceCommand.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/command/PersistenceCommand.java
index e207a1ef..0d586642 100644
--- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/command/PersistenceCommand.java
+++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/command/PersistenceCommand.java
@@ -3,8 +3,8 @@
  */
 package org.dbsyncer.parser.command;
 
+import org.dbsyncer.sdk.enums.StorageEnum;
 import org.dbsyncer.storage.StorageService;
-import org.dbsyncer.storage.enums.StorageEnum;
 
 import java.util.Map;
 
diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/FlushServiceImpl.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/FlushServiceImpl.java
index bf379b53..450b4756 100644
--- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/FlushServiceImpl.java
+++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/FlushServiceImpl.java
@@ -1,15 +1,15 @@
 package org.dbsyncer.parser.flush.impl;
 
 import org.dbsyncer.common.config.StorageConfig;
-import org.dbsyncer.storage.impl.SnowflakeIdWorker;
 import org.dbsyncer.common.util.StringUtil;
 import org.dbsyncer.parser.flush.BufferActuator;
 import org.dbsyncer.parser.flush.FlushService;
 import org.dbsyncer.parser.model.StorageRequest;
+import org.dbsyncer.sdk.enums.StorageEnum;
 import org.dbsyncer.storage.StorageService;
 import org.dbsyncer.storage.constant.ConfigConstant;
 import org.dbsyncer.storage.enums.StorageDataStatusEnum;
-import org.dbsyncer.storage.enums.StorageEnum;
+import org.dbsyncer.storage.impl.SnowflakeIdWorker;
 import org.dbsyncer.storage.util.BinlogMessageUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/StorageBufferActuator.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/StorageBufferActuator.java
index d36d3545..175aefff 100644
--- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/StorageBufferActuator.java
+++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/StorageBufferActuator.java
@@ -7,8 +7,8 @@ import org.dbsyncer.parser.model.Mapping;
 import org.dbsyncer.parser.model.Meta;
 import org.dbsyncer.parser.model.StorageRequest;
 import org.dbsyncer.parser.model.StorageResponse;
+import org.dbsyncer.sdk.enums.StorageEnum;
 import org.dbsyncer.storage.StorageService;
-import org.dbsyncer.storage.enums.StorageEnum;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.stereotype.Component;
diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/OperationTemplate.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/OperationTemplate.java
index e84af993..c898f9b9 100644
--- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/OperationTemplate.java
+++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/OperationTemplate.java
@@ -3,9 +3,9 @@
  */
 package org.dbsyncer.parser.impl;
 
-import org.dbsyncer.parser.CacheService;
 import org.dbsyncer.common.util.CollectionUtils;
 import org.dbsyncer.common.util.StringUtil;
+import org.dbsyncer.parser.CacheService;
 import org.dbsyncer.parser.ParserException;
 import org.dbsyncer.parser.command.PersistenceCommand;
 import org.dbsyncer.parser.enums.CommandEnum;
@@ -15,8 +15,8 @@ import org.dbsyncer.parser.model.OperationConfig;
 import org.dbsyncer.parser.model.QueryConfig;
 import org.dbsyncer.parser.strategy.GroupStrategy;
 import org.dbsyncer.parser.util.ConfigModelUtil;
+import org.dbsyncer.sdk.enums.StorageEnum;
 import org.dbsyncer.storage.StorageService;
-import org.dbsyncer.storage.enums.StorageEnum;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.BeanUtils;
diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/FilterTypeEnum.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/FilterTypeEnum.java
new file mode 100644
index 00000000..ce1f9448
--- /dev/null
+++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/FilterTypeEnum.java
@@ -0,0 +1,34 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
+package org.dbsyncer.sdk.enums;
+
+/**
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2023-12-11 20:29
+ */
+public enum FilterTypeEnum {
+    /**
+     * string
+     */
+    STRING("string"),
+    /**
+     * int
+     */
+    INT("int"),
+    /**
+     * long
+     */
+    LONG("long");
+
+    private String type;
+
+    FilterTypeEnum(String type) {
+        this.type = type;
+    }
+
+    public String getType() {
+        return type;
+    }
+}
\ No newline at end of file
diff --git a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/enums/BinlogSortEnum.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/SortEnum.java
similarity index 72%
rename from dbsyncer-storage/src/main/java/org/dbsyncer/storage/enums/BinlogSortEnum.java
rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/SortEnum.java
index 668f4546..823704ec 100644
--- a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/enums/BinlogSortEnum.java
+++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/SortEnum.java
@@ -1,39 +1,38 @@
-package org.dbsyncer.storage.enums;
-
-/**
- * 支持的排序方式
- *
- * @author AE86
- * @version 1.0.0
- * @date 2023/6/1 00:07
- */
-public enum BinlogSortEnum {
-    /**
-     * 升序
-     */
-    ASC("asc"),
-
-    /**
-     * 降序
-     */
-    DESC("desc");
-
-    BinlogSortEnum(String code) {
-        this.code = code;
-    }
-
-    String code;
-
-    public String getCode() {
-        return code;
-    }
-
-    /**
-     * 是否降序
-     *
-     * @return
-     */
-    public boolean isDesc() {
-        return this == DESC;
-    }
+package org.dbsyncer.sdk.enums;
+
+/**
+ * 支持的排序方式
+ *
+ * @author AE86
+ * @version 1.0.0
+ * @date 2023/6/1 00:07
+ */
+public enum SortEnum {
+    /**
+     * 升序
+     */
+    ASC("asc"),
+    /**
+     * 降序
+     */
+    DESC("desc");
+
+    SortEnum(String code) {
+        this.code = code;
+    }
+
+    private String code;
+
+    public String getCode() {
+        return code;
+    }
+
+    /**
+     * 是否降序
+     *
+     * @return
+     */
+    public boolean isDesc() {
+        return this == DESC;
+    }
 }
\ No newline at end of file
diff --git a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/enums/StorageEnum.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/StorageEnum.java
similarity index 77%
rename from dbsyncer-storage/src/main/java/org/dbsyncer/storage/enums/StorageEnum.java
rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/StorageEnum.java
index b5a48082..12920b5d 100644
--- a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/enums/StorageEnum.java
+++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/StorageEnum.java
@@ -1,32 +1,35 @@
-package org.dbsyncer.storage.enums;
-
-/**
- * @author AE86
- * @version 1.0.0
- * @date 2019/11/16 20:31
- */
-public enum StorageEnum {
-
-    /**
-     * 配置：连接器、驱动、映射关系、同步信息、分組配置、系统配置、用戶配置
-     */
-    CONFIG("config"),
-    /**
-     * 日志：连接器、驱动、映射关系、同步信息、系统日志
-     */
-    LOG("log"),
-    /**
-     * 数据：全量或增量数据
-     */
-    DATA("data");
-
-    private String type;
-
-    StorageEnum(String type) {
-        this.type = type;
-    }
-
-    public String getType() {
-        return type;
-    }
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
+package org.dbsyncer.sdk.enums;
+
+/**
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2019-11-12 20:29
+ */
+public enum StorageEnum {
+
+    /**
+     * 配置：连接器、驱动、映射关系、同步信息、分組配置、系统配置、用戶配置
+     */
+    CONFIG("config"),
+    /**
+     * 日志：连接器、驱动、映射关系、同步信息、系统日志
+     */
+    LOG("log"),
+    /**
+     * 数据：全量或增量数据
+     */
+    DATA("data");
+
+    private String type;
+
+    StorageEnum(String type) {
+        this.type = type;
+    }
+
+    public String getType() {
+        return type;
+    }
 }
\ No newline at end of file
diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/Filter.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/Filter.java
index 7e169d4a..01e70869 100644
--- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/Filter.java
+++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/model/Filter.java
@@ -1,3 +1,6 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
 package org.dbsyncer.sdk.model;
 
 import org.dbsyncer.sdk.enums.FilterEnum;
diff --git a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/query/AbstractFilter.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/AbstractFilter.java
similarity index 45%
rename from dbsyncer-storage/src/main/java/org/dbsyncer/storage/query/AbstractFilter.java
rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/AbstractFilter.java
index c7e22770..3d0c02ca 100644
--- a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/query/AbstractFilter.java
+++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/AbstractFilter.java
@@ -1,10 +1,9 @@
 /**
  * DBSyncer Copyright 2020-2023 All Rights Reserved.
  */
-package org.dbsyncer.storage.query;
+package org.dbsyncer.sdk.storage;
 
-import org.apache.lucene.search.Query;
-import org.dbsyncer.sdk.enums.FilterEnum;
+import org.dbsyncer.sdk.enums.FilterTypeEnum;
 import org.dbsyncer.sdk.model.Filter;
 
 /**
@@ -15,24 +14,24 @@ import org.dbsyncer.sdk.model.Filter;
  * @Date 2019-11-17 23:56
  */
 public abstract class AbstractFilter extends Filter {
-    private boolean enableHighLightSearch;
-
-    public AbstractFilter(String name, FilterEnum filterEnum, Object value) {
-        this(name, filterEnum, value, false);
-    }
-
-    public AbstractFilter(String name, FilterEnum filterEnum, Object value, boolean enableHighLightSearch) {
-        setName(name);
-        setFilter(filterEnum.getName());
-        setValue(String.valueOf(value));
-        this.enableHighLightSearch = enableHighLightSearch;
-    }
 
-    public abstract Query newEqual();
+    /**
+     * 返回是否显示高亮
+     */
+    private boolean enableHighLightSearch;
 
-    public abstract Query newLessThan();
+    /**
+     * 参数类型
+     *
+     * @return
+     */
+    public abstract FilterTypeEnum getFilterTypeEnum();
 
     public boolean isEnableHighLightSearch() {
         return enableHighLightSearch;
     }
+
+    public void setEnableHighLightSearch(boolean enableHighLightSearch) {
+        this.enableHighLightSearch = enableHighLightSearch;
+    }
 }
\ No newline at end of file
diff --git a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/query/BooleanFilter.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/BooleanFilter.java
similarity index 97%
rename from dbsyncer-storage/src/main/java/org/dbsyncer/storage/query/BooleanFilter.java
rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/BooleanFilter.java
index dfdf2412..47f683b7 100644
--- a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/query/BooleanFilter.java
+++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/BooleanFilter.java
@@ -1,4 +1,4 @@
-package org.dbsyncer.storage.query;
+package org.dbsyncer.sdk.storage;
 
 import org.dbsyncer.sdk.enums.OperationEnum;
 
diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/FieldResolver.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/FieldResolver.java
new file mode 100644
index 00000000..a6e0a05d
--- /dev/null
+++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/FieldResolver.java
@@ -0,0 +1,16 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
+package org.dbsyncer.sdk.storage;
+
+/**
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2023-12-11 17:36
+ */
+@FunctionalInterface
+public interface FieldResolver {
+
+    Object getValue(F field);
+
+}
\ No newline at end of file
diff --git a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/query/Query.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/Query.java
similarity index 74%
rename from dbsyncer-storage/src/main/java/org/dbsyncer/storage/query/Query.java
rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/Query.java
index 235e3078..5694473b 100644
--- a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/query/Query.java
+++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/Query.java
@@ -1,14 +1,13 @@
 /**
  * DBSyncer Copyright 2020-2023 All Rights Reserved.
  */
-package org.dbsyncer.storage.query;
+package org.dbsyncer.sdk.storage;
 
+import org.dbsyncer.sdk.enums.SortEnum;
 import org.dbsyncer.sdk.enums.FilterEnum;
-import org.dbsyncer.storage.enums.BinlogSortEnum;
-import org.dbsyncer.storage.enums.IndexFieldResolverEnum;
-import org.dbsyncer.storage.enums.StorageEnum;
-import org.dbsyncer.storage.query.filter.IntFilter;
-import org.dbsyncer.storage.query.filter.StringFilter;
+import org.dbsyncer.sdk.enums.StorageEnum;
+import org.dbsyncer.sdk.storage.filter.IntFilter;
+import org.dbsyncer.sdk.storage.filter.StringFilter;
 
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
@@ -41,12 +40,12 @@ public class Query {
     /**
      * 修改时间和创建默认降序返回
      */
-    private BinlogSortEnum sort = BinlogSortEnum.DESC;
+    private SortEnum sort = SortEnum.DESC;
 
     /**
      * 返回值转换器，限Disk使用
      */
-    private Map indexFieldResolverMap = new ConcurrentHashMap<>();
+    private Map fieldResolverMap = new ConcurrentHashMap<>();
 
     public Query() {
     }
@@ -116,20 +115,20 @@ public class Query {
         this.pageSize = pageSize;
     }
 
-    public BinlogSortEnum getSort() {
+    public SortEnum getSort() {
         return sort;
     }
 
-    public void setSort(BinlogSortEnum sort) {
+    public void setSort(SortEnum sort) {
         this.sort = sort;
     }
 
-    public Map getIndexFieldResolverMap() {
-        return indexFieldResolverMap;
+    public Map getFieldResolverMap() {
+        return fieldResolverMap;
     }
 
-    public void setIndexFieldResolverMap(Map indexFieldResolverMap) {
-        this.indexFieldResolverMap = indexFieldResolverMap;
+    public void setFieldResolverMap(Map fieldResolverMap) {
+        this.fieldResolverMap = fieldResolverMap;
     }
 
 }
\ No newline at end of file
diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/filter/IntFilter.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/filter/IntFilter.java
new file mode 100644
index 00000000..17f25730
--- /dev/null
+++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/filter/IntFilter.java
@@ -0,0 +1,24 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
+package org.dbsyncer.sdk.storage.filter;
+
+import org.dbsyncer.sdk.enums.FilterEnum;
+import org.dbsyncer.sdk.enums.FilterTypeEnum;
+import org.dbsyncer.sdk.storage.AbstractFilter;
+
+import java.util.Objects;
+
+public class IntFilter extends AbstractFilter {
+
+    public IntFilter(String name, int value) {
+        setName(name);
+        setFilter(FilterEnum.EQUAL.getName());
+        setValue(Objects.toString(value));
+    }
+
+    @Override
+    public FilterTypeEnum getFilterTypeEnum() {
+        return FilterTypeEnum.INT;
+    }
+}
\ No newline at end of file
diff --git a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/query/filter/LongFilter.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/filter/LongFilter.java
similarity index 32%
rename from dbsyncer-storage/src/main/java/org/dbsyncer/storage/query/filter/LongFilter.java
rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/filter/LongFilter.java
index bb3c33ec..53ded3fb 100644
--- a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/query/filter/LongFilter.java
+++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/filter/LongFilter.java
@@ -1,27 +1,24 @@
 /**
  * DBSyncer Copyright 2020-2023 All Rights Reserved.
  */
-package org.dbsyncer.storage.query.filter;
+package org.dbsyncer.sdk.storage.filter;
 
-import org.apache.lucene.document.LongPoint;
-import org.apache.lucene.search.Query;
-import org.dbsyncer.common.util.NumberUtil;
 import org.dbsyncer.sdk.enums.FilterEnum;
-import org.dbsyncer.storage.query.AbstractFilter;
+import org.dbsyncer.sdk.enums.FilterTypeEnum;
+import org.dbsyncer.sdk.storage.AbstractFilter;
+
+import java.util.Objects;
 
 public class LongFilter extends AbstractFilter {
 
     public LongFilter(String name, FilterEnum filterEnum, long value) {
-        super(name, filterEnum, value);
-    }
-
-    @Override
-    public Query newEqual() {
-        return LongPoint.newSetQuery(getName(), NumberUtil.toLong(getValue()));
+        setName(name);
+        setFilter(filterEnum.getName());
+        setValue(Objects.toString(value));
     }
 
     @Override
-    public Query newLessThan() {
-        return LongPoint.newRangeQuery(getName(), Long.MIN_VALUE, NumberUtil.toLong(getValue()));
+    public FilterTypeEnum getFilterTypeEnum() {
+        return FilterTypeEnum.LONG;
     }
 }
\ No newline at end of file
diff --git a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/query/filter/StringFilter.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/filter/StringFilter.java
similarity index 35%
rename from dbsyncer-storage/src/main/java/org/dbsyncer/storage/query/filter/StringFilter.java
rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/filter/StringFilter.java
index 686b9642..ed80e250 100644
--- a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/query/filter/StringFilter.java
+++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/filter/StringFilter.java
@@ -1,28 +1,23 @@
 /**
  * DBSyncer Copyright 2020-2023 All Rights Reserved.
  */
-package org.dbsyncer.storage.query.filter;
+package org.dbsyncer.sdk.storage.filter;
 
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TermQuery;
 import org.dbsyncer.sdk.enums.FilterEnum;
-import org.dbsyncer.storage.StorageException;
-import org.dbsyncer.storage.query.AbstractFilter;
+import org.dbsyncer.sdk.enums.FilterTypeEnum;
+import org.dbsyncer.sdk.storage.AbstractFilter;
 
 public class StringFilter extends AbstractFilter {
 
     public StringFilter(String name, FilterEnum filterEnum, String value, boolean enableHighLightSearch) {
-        super(name, filterEnum, value, enableHighLightSearch);
+        setName(name);
+        setFilter(filterEnum.getName());
+        setValue(value);
+        setEnableHighLightSearch(enableHighLightSearch);
     }
 
     @Override
-    public Query newEqual() {
-        return new TermQuery(new Term(getName(), getValue()));
-    }
-
-    @Override
-    public Query newLessThan() {
-        throw new StorageException("Unsupported method newLessThan.");
+    public FilterTypeEnum getFilterTypeEnum() {
+        return FilterTypeEnum.STRING;
     }
 }
\ No newline at end of file
diff --git a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/AbstractStorageService.java b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/AbstractStorageService.java
index 54ce5959..84872892 100644
--- a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/AbstractStorageService.java
+++ b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/AbstractStorageService.java
@@ -1,10 +1,13 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
 package org.dbsyncer.storage;
 
 import org.dbsyncer.common.model.Paging;
 import org.dbsyncer.common.util.CollectionUtils;
-import org.dbsyncer.storage.enums.StorageEnum;
-import org.dbsyncer.storage.query.BooleanFilter;
-import org.dbsyncer.storage.query.Query;
+import org.dbsyncer.sdk.enums.StorageEnum;
+import org.dbsyncer.sdk.storage.BooleanFilter;
+import org.dbsyncer.sdk.storage.Query;
 import org.dbsyncer.storage.strategy.Strategy;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -21,9 +24,9 @@ import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
 
 /**
- * @author AE86
- * @version 1.0.0
- * @date 2019/11/16 1:28
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2019-11-16 23:22
  */
 public abstract class AbstractStorageService implements StorageService, DisposableBean {
 
@@ -124,11 +127,6 @@ public abstract class AbstractStorageService implements StorageService, Disposab
         addBatch(type, metaId, newArrayList(params));
     }
 
-    @Override
-    public void addBatch(StorageEnum type, List list) {
-        addBatch(type, null, list);
-    }
-
     @Override
     public void addBatch(StorageEnum type, String metaId, List list) {
         if (!CollectionUtils.isEmpty(list)) {
@@ -146,11 +144,6 @@ public abstract class AbstractStorageService implements StorageService, Disposab
         editBatch(type, metaId, newArrayList(params));
     }
 
-    @Override
-    public void editBatch(StorageEnum type, List list) {
-        editBatch(type, null, list);
-    }
-
     @Override
     public void editBatch(StorageEnum type, String metaId, List list) {
         if (!CollectionUtils.isEmpty(list)) {
@@ -168,11 +161,6 @@ public abstract class AbstractStorageService implements StorageService, Disposab
         removeBatch(type, metaId, newArrayList(id));
     }
 
-    @Override
-    public void removeBatch(StorageEnum type, List ids) {
-        removeBatch(type, null, ids);
-    }
-
     @Override
     public void removeBatch(StorageEnum type, String metaId, List ids) {
         if (!CollectionUtils.isEmpty(ids)) {
diff --git a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/StorageService.java b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/StorageService.java
index adb22837..232e6a6f 100644
--- a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/StorageService.java
+++ b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/StorageService.java
@@ -1,8 +1,8 @@
 package org.dbsyncer.storage;
 
 import org.dbsyncer.common.model.Paging;
-import org.dbsyncer.storage.enums.StorageEnum;
-import org.dbsyncer.storage.query.Query;
+import org.dbsyncer.sdk.enums.StorageEnum;
+import org.dbsyncer.sdk.storage.Query;
 
 import java.util.List;
 import java.util.Map;
@@ -54,14 +54,6 @@ public interface StorageService {
      */
     void add(StorageEnum type, String metaId, Map params);
 
-    /**
-     * 批量添加
-     *
-     * @param type
-     * @param list
-     */
-    void addBatch(StorageEnum type, List list);
-
     /**
      * 批量添加
      *
@@ -88,14 +80,6 @@ public interface StorageService {
      */
     void edit(StorageEnum type, String metaId, Map params);
 
-    /**
-     * 批量修改
-     *
-     * @param type
-     * @param list
-     */
-    void editBatch(StorageEnum type, List list);
-
     /**
      * 批量修改
      *
@@ -122,20 +106,12 @@ public interface StorageService {
      */
     void remove(StorageEnum type, String metaId, String id);
 
-    /**
-     * 批量删除
-     *
-     * @param type
-     * @param ids
-     */
-    void removeBatch(StorageEnum type, List ids);
-
     /**
      * 批量删除
      *
      * @param type
      * @param metaId
-     * @param list
+     * @param ids
      */
     void removeBatch(StorageEnum type, String metaId, List ids);
 }
\ No newline at end of file
diff --git a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/constant/BinlogConstant.java b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/constant/BinlogConstant.java
deleted file mode 100644
index 7c00a1d3..00000000
--- a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/constant/BinlogConstant.java
+++ /dev/null
@@ -1,19 +0,0 @@
-package org.dbsyncer.storage.constant;
-
-/**
- * @author AE86
- * @version 1.0.0
- * @date 2022/7/13 22:14
- */
-public class BinlogConstant {
-
-    /**
-     * 待同步
-     */
-    public static final int READY = 0;
-    /**
-     * 同步中
-     */
-    public static final int PROCESSING = 1;
-
-}
\ No newline at end of file
diff --git a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/enums/BinlogStatusEnum.java b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/enums/BinlogStatusEnum.java
deleted file mode 100644
index 4d6e7820..00000000
--- a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/enums/BinlogStatusEnum.java
+++ /dev/null
@@ -1,18 +0,0 @@
-package org.dbsyncer.storage.enums;
-
-/**
- * @author AE86
- * @version 1.0.0
- * @date 2022/6/29 20:31
- */
-public enum BinlogStatusEnum {
-
-    /**
-     * 运行中
-     */
-    RUNNING,
-    /**
-     * 停止
-     */
-    STOP
-}
\ No newline at end of file
diff --git a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/enums/IndexFieldResolverEnum.java b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/enums/IndexFieldResolverEnum.java
deleted file mode 100644
index b426ba43..00000000
--- a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/enums/IndexFieldResolverEnum.java
+++ /dev/null
@@ -1,24 +0,0 @@
-package org.dbsyncer.storage.enums;
-
-import org.dbsyncer.storage.lucene.IndexFieldResolver;
-
-public enum IndexFieldResolverEnum {
-
-    LONG((f) -> f.numericValue().longValue()),
-
-    INT((f) -> f.numericValue().intValue()),
-
-    STRING((f) -> f.stringValue()),
-
-    BINARY((f) -> f.binaryValue().bytes);
-
-    private IndexFieldResolver indexFieldResolver;
-
-    IndexFieldResolverEnum(IndexFieldResolver indexFieldResolver) {
-        this.indexFieldResolver = indexFieldResolver;
-    }
-
-    public IndexFieldResolver getIndexFieldResolver() {
-        return indexFieldResolver;
-    }
-}
\ No newline at end of file
diff --git a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/impl/DiskQueryHelper.java b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/impl/DiskQueryHelper.java
new file mode 100644
index 00000000..eeecac83
--- /dev/null
+++ b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/impl/DiskQueryHelper.java
@@ -0,0 +1,57 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
+package org.dbsyncer.storage.impl;
+
+import org.apache.lucene.document.IntPoint;
+import org.apache.lucene.document.LongPoint;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
+import org.dbsyncer.common.util.NumberUtil;
+import org.dbsyncer.sdk.storage.AbstractFilter;
+import org.dbsyncer.storage.StorageException;
+
+/**
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2023-12-11 22:27
+ */
+public class DiskQueryHelper {
+
+    public static Query newEqual(AbstractFilter filter) {
+        Query query = null;
+         switch (filter.getFilterTypeEnum()) {
+            case STRING:
+                query = new TermQuery(new Term(filter.getName(), filter.getValue()));
+                break;
+            case LONG:
+                query = LongPoint.newSetQuery(filter.getName(), NumberUtil.toLong(filter.getValue()));
+                break;
+            case INT:
+                query = IntPoint.newSetQuery(filter.getName(), NumberUtil.toInt(filter.getValue()));
+                break;
+        }
+        if (query == null) {
+            throw new StorageException("Unsupported method newEqual.");
+        }
+        return query;
+    }
+
+    public static Query newLessThan(AbstractFilter filter) {
+        Query query = null;
+        switch (filter.getFilterTypeEnum()) {
+            case LONG:
+                query = LongPoint.newRangeQuery(filter.getName(), Long.MIN_VALUE, NumberUtil.toLong(filter.getValue()));
+                break;
+            case INT:
+                query = IntPoint.newRangeQuery(filter.getName(), Integer.MIN_VALUE, NumberUtil.toInt(filter.getValue()));
+                break;
+        }
+        if (query == null) {
+            throw new StorageException("Unsupported method newLessThan.");
+        }
+        return query;
+    }
+
+}
\ No newline at end of file
diff --git a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/impl/DiskStorageServiceImpl.java b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/impl/DiskStorageServiceImpl.java
index 8826c77a..0bb904cc 100644
--- a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/impl/DiskStorageServiceImpl.java
+++ b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/impl/DiskStorageServiceImpl.java
@@ -21,12 +21,12 @@ import org.dbsyncer.sdk.enums.OperationEnum;
 import org.dbsyncer.storage.AbstractStorageService;
 import org.dbsyncer.storage.StorageException;
 import org.dbsyncer.storage.constant.ConfigConstant;
-import org.dbsyncer.storage.enums.StorageEnum;
+import org.dbsyncer.sdk.enums.StorageEnum;
 import org.dbsyncer.storage.lucene.Option;
 import org.dbsyncer.storage.lucene.Shard;
-import org.dbsyncer.storage.query.AbstractFilter;
-import org.dbsyncer.storage.query.BooleanFilter;
-import org.dbsyncer.storage.query.Query;
+import org.dbsyncer.sdk.storage.AbstractFilter;
+import org.dbsyncer.sdk.storage.BooleanFilter;
+import org.dbsyncer.sdk.storage.Query;
 import org.dbsyncer.storage.util.DocumentUtil;
 
 import javax.annotation.PostConstruct;
@@ -77,7 +77,7 @@ public class DiskStorageServiceImpl extends AbstractStorageService {
                     new SortField(ConfigConstant.CONFIG_MODEL_CREATE_TIME, SortField.Type.LONG, desc));
             Option option = new Option();
             option.setQueryTotal(query.isQueryTotal());
-            option.setIndexFieldResolverMap(query.getIndexFieldResolverMap());
+            option.setFieldResolverMap(query.getFieldResolverMap());
             // 设置参数
             BooleanFilter baseQuery = query.getBooleanFilter();
             List filters = baseQuery.getFilters();
@@ -166,13 +166,13 @@ public class DiskStorageServiceImpl extends AbstractStorageService {
         filters.forEach(p -> {
             FilterEnum filterEnum = FilterEnum.getFilterEnum(p.getFilter());
             BooleanClause.Occur occur = getOccur(p.getOperation());
-            switch (filterEnum) {
+             switch (filterEnum) {
                 case EQUAL:
                 case LIKE:
-                    builder.add(p.newEqual(), occur);
+                    builder.add(DiskQueryHelper.newEqual(p), occur);
                     break;
                 case LT:
-                    builder.add(p.newLessThan(), occur);
+                    builder.add(DiskQueryHelper.newLessThan(p), occur);
                     break;
             }
 
diff --git a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/impl/MySQLStorageServiceImpl.java b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/impl/MySQLStorageServiceImpl.java
index 1922e271..62e86380 100644
--- a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/impl/MySQLStorageServiceImpl.java
+++ b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/impl/MySQLStorageServiceImpl.java
@@ -10,20 +10,20 @@ import org.dbsyncer.common.util.StringUtil;
 import org.dbsyncer.connector.ConnectorFactory;
 import org.dbsyncer.sdk.config.DatabaseConfig;
 import org.dbsyncer.sdk.config.SqlBuilderConfig;
-import org.dbsyncer.sdk.constant.DatabaseConstant;
 import org.dbsyncer.sdk.connector.database.Database;
 import org.dbsyncer.sdk.connector.database.DatabaseConnectorInstance;
+import org.dbsyncer.sdk.constant.DatabaseConstant;
 import org.dbsyncer.sdk.enums.FilterEnum;
 import org.dbsyncer.sdk.enums.SqlBuilderEnum;
+import org.dbsyncer.sdk.enums.StorageEnum;
 import org.dbsyncer.sdk.model.Field;
+import org.dbsyncer.sdk.storage.AbstractFilter;
+import org.dbsyncer.sdk.storage.BooleanFilter;
+import org.dbsyncer.sdk.storage.Query;
 import org.dbsyncer.sdk.util.DatabaseUtil;
 import org.dbsyncer.storage.AbstractStorageService;
 import org.dbsyncer.storage.NullExecutorException;
 import org.dbsyncer.storage.constant.ConfigConstant;
-import org.dbsyncer.storage.enums.StorageEnum;
-import org.dbsyncer.storage.query.AbstractFilter;
-import org.dbsyncer.storage.query.BooleanFilter;
-import org.dbsyncer.storage.query.Query;
 import org.dbsyncer.storage.util.UnderlineToCamelUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -149,7 +149,7 @@ public class MySQLStorageServiceImpl extends AbstractStorageService {
             return executor;
         });
         // 非系统表
-        if(!systemTable.get()){
+        if (!systemTable.get()) {
             tables.remove(sharding);
         }
     }
diff --git a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/lucene/IndexFieldResolver.java b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/lucene/IndexFieldResolver.java
deleted file mode 100644
index eb504ed2..00000000
--- a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/lucene/IndexFieldResolver.java
+++ /dev/null
@@ -1,9 +0,0 @@
-package org.dbsyncer.storage.lucene;
-
-import org.apache.lucene.index.IndexableField;
-
-public interface IndexFieldResolver {
-
-    Object getValue(IndexableField field);
-
-}
\ No newline at end of file
diff --git a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/lucene/Option.java b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/lucene/Option.java
index 6d42cecf..1667ef61 100644
--- a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/lucene/Option.java
+++ b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/lucene/Option.java
@@ -1,17 +1,20 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
 package org.dbsyncer.storage.lucene;
 
+import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.highlight.Highlighter;
-import org.dbsyncer.storage.enums.IndexFieldResolverEnum;
-import org.dbsyncer.storage.lucene.IndexFieldResolver;
+import org.dbsyncer.sdk.storage.FieldResolver;
 
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 
 /**
- * @version 1.0.0
  * @Author AE86
+ * @Version 1.0.0
  * @Date 2020-06-01 22:57
  */
 public class Option {
@@ -32,7 +35,7 @@ public class Option {
     /**
      * 返回值转换器
      */
-    private Map indexFieldResolverMap = new ConcurrentHashMap<>();
+    private Map fieldResolverMap = new ConcurrentHashMap<>();
 
     /**
      * 指定返回的值类型
@@ -40,12 +43,11 @@ public class Option {
      * @param name
      * @return
      */
-    public IndexFieldResolver getIndexFieldResolver(String name) {
-        IndexFieldResolverEnum indexFieldResolverEnum = indexFieldResolverMap.get(name);
-        if (null != indexFieldResolverEnum) {
-            return indexFieldResolverEnum.getIndexFieldResolver();
+    public FieldResolver getFieldResolver(String name) {
+        if (fieldResolverMap.containsKey(name)) {
+            return fieldResolverMap.get(name);
         }
-        return IndexFieldResolverEnum.STRING.getIndexFieldResolver();
+        return (f) -> f.stringValue();
     }
 
     public Query getQuery() {
@@ -88,11 +90,11 @@ public class Option {
         this.queryTotal = queryTotal;
     }
 
-    public Map getIndexFieldResolverMap() {
-        return indexFieldResolverMap;
+    public Map getFieldResolverMap() {
+        return fieldResolverMap;
     }
 
-    public void setIndexFieldResolverMap(Map indexFieldResolverMap) {
-        this.indexFieldResolverMap = indexFieldResolverMap;
+    public void setFieldResolverMap(Map fieldResolverMap) {
+        this.fieldResolverMap = fieldResolverMap;
     }
 }
\ No newline at end of file
diff --git a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/lucene/Shard.java b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/lucene/Shard.java
index 07182ebf..29f89f8c 100644
--- a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/lucene/Shard.java
+++ b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/lucene/Shard.java
@@ -1,3 +1,6 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
 package org.dbsyncer.storage.lucene;
 
 import org.apache.commons.io.FileUtils;
@@ -37,9 +40,9 @@ import java.util.List;
 import java.util.Map;
 
 /**
- * @author AE86
- * @version 1.0.0
- * @date 2019/11/12 20:29
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2019-11-12 20:29
  */
 public class Shard {
 
@@ -206,7 +209,7 @@ public class Shard {
                 }
 
                 // 解析value类型
-                r.put(f.name(), option.getIndexFieldResolver(f.name()).getValue(f));
+                r.put(f.name(), option.getFieldResolver(f.name()).getValue(f));
             }
             list.add(r);
         }
diff --git a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/query/filter/IntFilter.java b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/query/filter/IntFilter.java
deleted file mode 100644
index af45b52b..00000000
--- a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/query/filter/IntFilter.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/**
- * DBSyncer Copyright 2020-2023 All Rights Reserved.
- */
-package org.dbsyncer.storage.query.filter;
-
-import org.apache.lucene.document.IntPoint;
-import org.apache.lucene.search.Query;
-import org.dbsyncer.common.util.NumberUtil;
-import org.dbsyncer.sdk.enums.FilterEnum;
-import org.dbsyncer.storage.query.AbstractFilter;
-
-public class IntFilter extends AbstractFilter {
-
-    public IntFilter(String name, int value) {
-        super(name, FilterEnum.EQUAL, value);
-    }
-
-    public IntFilter(String name, FilterEnum filterEnum, int value) {
-        super(name, filterEnum, value);
-    }
-
-    @Override
-    public Query newEqual() {
-        return IntPoint.newSetQuery(getName(), NumberUtil.toInt(getValue()));
-    }
-
-    @Override
-    public Query newLessThan() {
-        return IntPoint.newRangeQuery(getName(), Integer.MIN_VALUE, NumberUtil.toInt(getValue()));
-    }
-}
\ No newline at end of file
diff --git a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/strategy/impl/ConfigStrategy.java b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/strategy/impl/ConfigStrategy.java
index 023e957c..28622b29 100644
--- a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/strategy/impl/ConfigStrategy.java
+++ b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/strategy/impl/ConfigStrategy.java
@@ -1,6 +1,6 @@
 package org.dbsyncer.storage.strategy.impl;
 
-import org.dbsyncer.storage.enums.StorageEnum;
+import org.dbsyncer.sdk.enums.StorageEnum;
 import org.dbsyncer.storage.strategy.Strategy;
 import org.springframework.stereotype.Component;
 
diff --git a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/strategy/impl/DataStrategy.java b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/strategy/impl/DataStrategy.java
index b96fdd6d..0120617a 100644
--- a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/strategy/impl/DataStrategy.java
+++ b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/strategy/impl/DataStrategy.java
@@ -1,6 +1,6 @@
 package org.dbsyncer.storage.strategy.impl;
 
-import org.dbsyncer.storage.enums.StorageEnum;
+import org.dbsyncer.sdk.enums.StorageEnum;
 import org.dbsyncer.storage.strategy.Strategy;
 import org.springframework.stereotype.Component;
 import org.springframework.util.Assert;
diff --git a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/strategy/impl/LogStrategy.java b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/strategy/impl/LogStrategy.java
index 821dd089..90ddc2b8 100644
--- a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/strategy/impl/LogStrategy.java
+++ b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/strategy/impl/LogStrategy.java
@@ -1,6 +1,6 @@
 package org.dbsyncer.storage.strategy.impl;
 
-import org.dbsyncer.storage.enums.StorageEnum;
+import org.dbsyncer.sdk.enums.StorageEnum;
 import org.dbsyncer.storage.strategy.Strategy;
 import org.springframework.stereotype.Component;
 
-- 
Gitee


From 4f4deae5b72bd70a55f6e7cabfe0c433ecc2ba8b Mon Sep 17 00:00:00 2001
From: AE86 <836391306@qq.com>
Date: Mon, 11 Dec 2023 23:16:26 +0800
Subject: [PATCH 64/98] move storage

---
 .../biz/impl/DataSyncServiceImpl.java         |  4 +--
 .../org/dbsyncer/biz/impl/MetricReporter.java | 11 ++++---
 .../dbsyncer/biz/impl/MonitorServiceImpl.java |  8 ++---
 .../common/column/AbstractColumnValue.java    |  3 ++
 .../dbsyncer/common/column/ColumnValue.java   |  9 ++++--
 .../manager/impl/PreloadTemplate.java         |  2 +-
 .../{storage => filter}/AbstractFilter.java   |  2 +-
 .../{storage => filter}/BooleanFilter.java    |  2 +-
 .../{storage => filter}/FieldResolver.java    |  2 +-
 .../sdk/{storage => filter}/Query.java        |  6 ++--
 .../filter => filter/impl}/IntFilter.java     |  4 +--
 .../filter => filter/impl}/LongFilter.java    |  4 +--
 .../filter => filter/impl}/StringFilter.java  |  4 +--
 .../storage/AbstractStorageService.java       |  4 +--
 .../org/dbsyncer/storage/StorageService.java  |  2 +-
 .../storage/binlog/BinlogColumnValue.java     |  9 ++++--
 .../storage/constant/ConfigConstant.java      |  9 ++++--
 .../storage/enums/StorageDataStatusEnum.java  | 30 +++++++------------
 .../storage/impl/DiskQueryHelper.java         |  2 +-
 .../storage/impl/DiskStorageServiceImpl.java  |  6 ++--
 .../storage/impl/MySQLStorageServiceImpl.java |  6 ++--
 .../org/dbsyncer/storage/lucene/Option.java   |  2 +-
 22 files changed, 69 insertions(+), 62 deletions(-)
 rename dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/{storage => filter}/AbstractFilter.java (95%)
 rename dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/{storage => filter}/BooleanFilter.java (97%)
 rename dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/{storage => filter}/FieldResolver.java (86%)
 rename dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/{storage => filter}/Query.java (95%)
 rename dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/{storage/filter => filter/impl}/IntFilter.java (84%)
 rename dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/{storage/filter => filter/impl}/LongFilter.java (85%)
 rename dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/{storage/filter => filter/impl}/StringFilter.java (86%)

diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/DataSyncServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/DataSyncServiceImpl.java
index a5a96921..d1c47746 100644
--- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/DataSyncServiceImpl.java
+++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/DataSyncServiceImpl.java
@@ -19,8 +19,8 @@ import org.dbsyncer.parser.model.TableGroup;
 import org.dbsyncer.sdk.enums.StorageEnum;
 import org.dbsyncer.sdk.listener.event.RowChangedEvent;
 import org.dbsyncer.sdk.model.Field;
-import org.dbsyncer.sdk.storage.FieldResolver;
-import org.dbsyncer.sdk.storage.Query;
+import org.dbsyncer.sdk.filter.FieldResolver;
+import org.dbsyncer.sdk.filter.Query;
 import org.dbsyncer.storage.StorageService;
 import org.dbsyncer.storage.binlog.proto.BinlogMap;
 import org.dbsyncer.storage.constant.ConfigConstant;
diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MetricReporter.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MetricReporter.java
index c7b835f2..c42d773f 100644
--- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MetricReporter.java
+++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MetricReporter.java
@@ -1,3 +1,6 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
 package org.dbsyncer.biz.impl;
 
 import org.dbsyncer.biz.enums.BufferActuatorMetricEnum;
@@ -22,7 +25,7 @@ import org.dbsyncer.parser.model.Meta;
 import org.dbsyncer.parser.model.TableGroup;
 import org.dbsyncer.sdk.constant.ConnectorConstant;
 import org.dbsyncer.sdk.enums.StorageEnum;
-import org.dbsyncer.sdk.storage.Query;
+import org.dbsyncer.sdk.filter.Query;
 import org.dbsyncer.storage.StorageService;
 import org.dbsyncer.storage.constant.ConfigConstant;
 import org.dbsyncer.storage.enums.StorageDataStatusEnum;
@@ -44,9 +47,9 @@ import java.util.function.Consumer;
 import java.util.stream.Collectors;
 
 /**
- * @author AE86
- * @version 1.0.0
- * @date 2020/04/23 11:30
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2020-04-23 11:30
  */
 @Component
 public class MetricReporter implements ScheduledTaskJob {
diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java
index 02f47a68..cdbd9a71 100644
--- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java
+++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java
@@ -40,10 +40,10 @@ import org.dbsyncer.parser.model.Meta;
 import org.dbsyncer.sdk.enums.FilterEnum;
 import org.dbsyncer.sdk.enums.ModelEnum;
 import org.dbsyncer.sdk.enums.StorageEnum;
-import org.dbsyncer.sdk.storage.BooleanFilter;
-import org.dbsyncer.sdk.storage.FieldResolver;
-import org.dbsyncer.sdk.storage.Query;
-import org.dbsyncer.sdk.storage.filter.LongFilter;
+import org.dbsyncer.sdk.filter.BooleanFilter;
+import org.dbsyncer.sdk.filter.FieldResolver;
+import org.dbsyncer.sdk.filter.Query;
+import org.dbsyncer.sdk.filter.impl.LongFilter;
 import org.dbsyncer.storage.StorageService;
 import org.dbsyncer.storage.constant.ConfigConstant;
 import org.dbsyncer.storage.enums.StorageDataStatusEnum;
diff --git a/dbsyncer-common/src/main/java/org/dbsyncer/common/column/AbstractColumnValue.java b/dbsyncer-common/src/main/java/org/dbsyncer/common/column/AbstractColumnValue.java
index 91f3eea4..be7ecbbb 100644
--- a/dbsyncer-common/src/main/java/org/dbsyncer/common/column/AbstractColumnValue.java
+++ b/dbsyncer-common/src/main/java/org/dbsyncer/common/column/AbstractColumnValue.java
@@ -1,3 +1,6 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
 package org.dbsyncer.common.column;
 
 public abstract class AbstractColumnValue implements ColumnValue {
diff --git a/dbsyncer-common/src/main/java/org/dbsyncer/common/column/ColumnValue.java b/dbsyncer-common/src/main/java/org/dbsyncer/common/column/ColumnValue.java
index 4872d2e6..f94b2b13 100644
--- a/dbsyncer-common/src/main/java/org/dbsyncer/common/column/ColumnValue.java
+++ b/dbsyncer-common/src/main/java/org/dbsyncer/common/column/ColumnValue.java
@@ -1,3 +1,6 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
 package org.dbsyncer.common.column;
 
 import java.math.BigDecimal;
@@ -6,9 +9,9 @@ import java.sql.Time;
 import java.sql.Timestamp;
 
 /**
- * @author AE86
- * @version 1.0.0
- * @date 2022/4/22 22:39
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2022-04-22 22:39
  */
 public interface ColumnValue {
 
diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/PreloadTemplate.java b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/PreloadTemplate.java
index 8f623cf5..bca19d15 100644
--- a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/PreloadTemplate.java
+++ b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/PreloadTemplate.java
@@ -21,7 +21,7 @@ import org.dbsyncer.parser.model.OperationConfig;
 import org.dbsyncer.plugin.PluginFactory;
 import org.dbsyncer.sdk.connector.ConnectorInstance;
 import org.dbsyncer.sdk.enums.StorageEnum;
-import org.dbsyncer.sdk.storage.Query;
+import org.dbsyncer.sdk.filter.Query;
 import org.dbsyncer.storage.StorageService;
 import org.dbsyncer.storage.constant.ConfigConstant;
 import org.slf4j.Logger;
diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/AbstractFilter.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/filter/AbstractFilter.java
similarity index 95%
rename from dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/AbstractFilter.java
rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/filter/AbstractFilter.java
index 3d0c02ca..7e1ac91c 100644
--- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/AbstractFilter.java
+++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/filter/AbstractFilter.java
@@ -1,7 +1,7 @@
 /**
  * DBSyncer Copyright 2020-2023 All Rights Reserved.
  */
-package org.dbsyncer.sdk.storage;
+package org.dbsyncer.sdk.filter;
 
 import org.dbsyncer.sdk.enums.FilterTypeEnum;
 import org.dbsyncer.sdk.model.Filter;
diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/BooleanFilter.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/filter/BooleanFilter.java
similarity index 97%
rename from dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/BooleanFilter.java
rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/filter/BooleanFilter.java
index 47f683b7..cae94c8e 100644
--- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/BooleanFilter.java
+++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/filter/BooleanFilter.java
@@ -1,4 +1,4 @@
-package org.dbsyncer.sdk.storage;
+package org.dbsyncer.sdk.filter;
 
 import org.dbsyncer.sdk.enums.OperationEnum;
 
diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/FieldResolver.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/filter/FieldResolver.java
similarity index 86%
rename from dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/FieldResolver.java
rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/filter/FieldResolver.java
index a6e0a05d..ff401b4f 100644
--- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/FieldResolver.java
+++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/filter/FieldResolver.java
@@ -1,7 +1,7 @@
 /**
  * DBSyncer Copyright 2020-2023 All Rights Reserved.
  */
-package org.dbsyncer.sdk.storage;
+package org.dbsyncer.sdk.filter;
 
 /**
  * @Author AE86
diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/Query.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/filter/Query.java
similarity index 95%
rename from dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/Query.java
rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/filter/Query.java
index 5694473b..5285ddb9 100644
--- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/Query.java
+++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/filter/Query.java
@@ -1,13 +1,13 @@
 /**
  * DBSyncer Copyright 2020-2023 All Rights Reserved.
  */
-package org.dbsyncer.sdk.storage;
+package org.dbsyncer.sdk.filter;
 
 import org.dbsyncer.sdk.enums.SortEnum;
 import org.dbsyncer.sdk.enums.FilterEnum;
 import org.dbsyncer.sdk.enums.StorageEnum;
-import org.dbsyncer.sdk.storage.filter.IntFilter;
-import org.dbsyncer.sdk.storage.filter.StringFilter;
+import org.dbsyncer.sdk.filter.impl.IntFilter;
+import org.dbsyncer.sdk.filter.impl.StringFilter;
 
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/filter/IntFilter.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/filter/impl/IntFilter.java
similarity index 84%
rename from dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/filter/IntFilter.java
rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/filter/impl/IntFilter.java
index 17f25730..10e47bf7 100644
--- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/filter/IntFilter.java
+++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/filter/impl/IntFilter.java
@@ -1,11 +1,11 @@
 /**
  * DBSyncer Copyright 2020-2023 All Rights Reserved.
  */
-package org.dbsyncer.sdk.storage.filter;
+package org.dbsyncer.sdk.filter.impl;
 
 import org.dbsyncer.sdk.enums.FilterEnum;
 import org.dbsyncer.sdk.enums.FilterTypeEnum;
-import org.dbsyncer.sdk.storage.AbstractFilter;
+import org.dbsyncer.sdk.filter.AbstractFilter;
 
 import java.util.Objects;
 
diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/filter/LongFilter.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/filter/impl/LongFilter.java
similarity index 85%
rename from dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/filter/LongFilter.java
rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/filter/impl/LongFilter.java
index 53ded3fb..d99f641f 100644
--- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/filter/LongFilter.java
+++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/filter/impl/LongFilter.java
@@ -1,11 +1,11 @@
 /**
  * DBSyncer Copyright 2020-2023 All Rights Reserved.
  */
-package org.dbsyncer.sdk.storage.filter;
+package org.dbsyncer.sdk.filter.impl;
 
 import org.dbsyncer.sdk.enums.FilterEnum;
 import org.dbsyncer.sdk.enums.FilterTypeEnum;
-import org.dbsyncer.sdk.storage.AbstractFilter;
+import org.dbsyncer.sdk.filter.AbstractFilter;
 
 import java.util.Objects;
 
diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/filter/StringFilter.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/filter/impl/StringFilter.java
similarity index 86%
rename from dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/filter/StringFilter.java
rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/filter/impl/StringFilter.java
index ed80e250..505825df 100644
--- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/filter/StringFilter.java
+++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/filter/impl/StringFilter.java
@@ -1,11 +1,11 @@
 /**
  * DBSyncer Copyright 2020-2023 All Rights Reserved.
  */
-package org.dbsyncer.sdk.storage.filter;
+package org.dbsyncer.sdk.filter.impl;
 
 import org.dbsyncer.sdk.enums.FilterEnum;
 import org.dbsyncer.sdk.enums.FilterTypeEnum;
-import org.dbsyncer.sdk.storage.AbstractFilter;
+import org.dbsyncer.sdk.filter.AbstractFilter;
 
 public class StringFilter extends AbstractFilter {
 
diff --git a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/AbstractStorageService.java b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/AbstractStorageService.java
index 84872892..b500911f 100644
--- a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/AbstractStorageService.java
+++ b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/AbstractStorageService.java
@@ -6,8 +6,8 @@ package org.dbsyncer.storage;
 import org.dbsyncer.common.model.Paging;
 import org.dbsyncer.common.util.CollectionUtils;
 import org.dbsyncer.sdk.enums.StorageEnum;
-import org.dbsyncer.sdk.storage.BooleanFilter;
-import org.dbsyncer.sdk.storage.Query;
+import org.dbsyncer.sdk.filter.BooleanFilter;
+import org.dbsyncer.sdk.filter.Query;
 import org.dbsyncer.storage.strategy.Strategy;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
diff --git a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/StorageService.java b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/StorageService.java
index 232e6a6f..77127d28 100644
--- a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/StorageService.java
+++ b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/StorageService.java
@@ -2,7 +2,7 @@ package org.dbsyncer.storage;
 
 import org.dbsyncer.common.model.Paging;
 import org.dbsyncer.sdk.enums.StorageEnum;
-import org.dbsyncer.sdk.storage.Query;
+import org.dbsyncer.sdk.filter.Query;
 
 import java.util.List;
 import java.util.Map;
diff --git a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/binlog/BinlogColumnValue.java b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/binlog/BinlogColumnValue.java
index 2ec38b30..a0fcb3e6 100644
--- a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/binlog/BinlogColumnValue.java
+++ b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/binlog/BinlogColumnValue.java
@@ -1,3 +1,6 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
 package org.dbsyncer.storage.binlog;
 
 import com.google.protobuf.ByteString;
@@ -11,9 +14,9 @@ import java.sql.Time;
 import java.sql.Timestamp;
 
 /**
- * @author AE86
- * @version 1.0.0
- * @date 2022/6/30 22:39
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2022-06-30 22:39
  */
 public class BinlogColumnValue extends AbstractColumnValue {
 
diff --git a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/constant/ConfigConstant.java b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/constant/ConfigConstant.java
index 1f2edc1f..f9a4ca35 100644
--- a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/constant/ConfigConstant.java
+++ b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/constant/ConfigConstant.java
@@ -1,9 +1,12 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
 package org.dbsyncer.storage.constant;
 
 /**
- * @author AE86
- * @version 1.0.0
- * @date 2019/11/16 22:14
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2019-11-16 22:14
  */
 public class ConfigConstant {
 
diff --git a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/enums/StorageDataStatusEnum.java b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/enums/StorageDataStatusEnum.java
index 2c940892..353d3d7e 100644
--- a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/enums/StorageDataStatusEnum.java
+++ b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/enums/StorageDataStatusEnum.java
@@ -1,42 +1,34 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
 package org.dbsyncer.storage.enums;
 
 /**
- * @author AE86
- * @version 1.0.0
- * @date 2021/3/10 20:31
+ * 同步数据状态枚举
+ *
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2021-03-10 20:31
  */
 public enum StorageDataStatusEnum {
 
     /**
      * 失败
      */
-    FAIL(0, "0", "失败"),
+    FAIL(0),
     /**
      * 成功
      */
-    SUCCESS(1, "1", "成功");
+    SUCCESS(1);
 
     private Integer value;
 
-    private String code;
-
-    private String message;
-
-    StorageDataStatusEnum(Integer value, String code, String message) {
+    StorageDataStatusEnum(Integer value) {
         this.value = value;
-        this.code = code;
-        this.message = message;
     }
 
     public Integer getValue() {
         return value;
     }
 
-    public String getCode() {
-        return code;
-    }
-
-    public String getMessage() {
-        return message;
-    }
 }
\ No newline at end of file
diff --git a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/impl/DiskQueryHelper.java b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/impl/DiskQueryHelper.java
index eeecac83..447fe792 100644
--- a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/impl/DiskQueryHelper.java
+++ b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/impl/DiskQueryHelper.java
@@ -9,7 +9,7 @@ import org.apache.lucene.index.Term;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
 import org.dbsyncer.common.util.NumberUtil;
-import org.dbsyncer.sdk.storage.AbstractFilter;
+import org.dbsyncer.sdk.filter.AbstractFilter;
 import org.dbsyncer.storage.StorageException;
 
 /**
diff --git a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/impl/DiskStorageServiceImpl.java b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/impl/DiskStorageServiceImpl.java
index 0bb904cc..b5c85f88 100644
--- a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/impl/DiskStorageServiceImpl.java
+++ b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/impl/DiskStorageServiceImpl.java
@@ -24,9 +24,9 @@ import org.dbsyncer.storage.constant.ConfigConstant;
 import org.dbsyncer.sdk.enums.StorageEnum;
 import org.dbsyncer.storage.lucene.Option;
 import org.dbsyncer.storage.lucene.Shard;
-import org.dbsyncer.sdk.storage.AbstractFilter;
-import org.dbsyncer.sdk.storage.BooleanFilter;
-import org.dbsyncer.sdk.storage.Query;
+import org.dbsyncer.sdk.filter.AbstractFilter;
+import org.dbsyncer.sdk.filter.BooleanFilter;
+import org.dbsyncer.sdk.filter.Query;
 import org.dbsyncer.storage.util.DocumentUtil;
 
 import javax.annotation.PostConstruct;
diff --git a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/impl/MySQLStorageServiceImpl.java b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/impl/MySQLStorageServiceImpl.java
index 62e86380..505ea593 100644
--- a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/impl/MySQLStorageServiceImpl.java
+++ b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/impl/MySQLStorageServiceImpl.java
@@ -17,9 +17,9 @@ import org.dbsyncer.sdk.enums.FilterEnum;
 import org.dbsyncer.sdk.enums.SqlBuilderEnum;
 import org.dbsyncer.sdk.enums.StorageEnum;
 import org.dbsyncer.sdk.model.Field;
-import org.dbsyncer.sdk.storage.AbstractFilter;
-import org.dbsyncer.sdk.storage.BooleanFilter;
-import org.dbsyncer.sdk.storage.Query;
+import org.dbsyncer.sdk.filter.AbstractFilter;
+import org.dbsyncer.sdk.filter.BooleanFilter;
+import org.dbsyncer.sdk.filter.Query;
 import org.dbsyncer.sdk.util.DatabaseUtil;
 import org.dbsyncer.storage.AbstractStorageService;
 import org.dbsyncer.storage.NullExecutorException;
diff --git a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/lucene/Option.java b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/lucene/Option.java
index 1667ef61..146327d6 100644
--- a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/lucene/Option.java
+++ b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/lucene/Option.java
@@ -6,7 +6,7 @@ package org.dbsyncer.storage.lucene;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.highlight.Highlighter;
-import org.dbsyncer.sdk.storage.FieldResolver;
+import org.dbsyncer.sdk.filter.FieldResolver;
 
 import java.util.Map;
 import java.util.Set;
-- 
Gitee


From cc0b5978488a1b51d6aa75903d2683d6ad0bcd55 Mon Sep 17 00:00:00 2001
From: AE86 <836391306@qq.com>
Date: Mon, 11 Dec 2023 23:25:34 +0800
Subject: [PATCH 65/98] move storage

---
 .../storage/enums/StorageDataStatusEnum.java       | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/enums/StorageDataStatusEnum.java b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/enums/StorageDataStatusEnum.java
index 353d3d7e..daad7eaf 100644
--- a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/enums/StorageDataStatusEnum.java
+++ b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/enums/StorageDataStatusEnum.java
@@ -11,24 +11,30 @@ package org.dbsyncer.storage.enums;
  * @Date 2021-03-10 20:31
  */
 public enum StorageDataStatusEnum {
-
     /**
      * 失败
      */
-    FAIL(0),
+    FAIL(0, "失败"),
     /**
      * 成功
      */
-    SUCCESS(1);
+    SUCCESS(1, "成功");
 
     private Integer value;
 
-    StorageDataStatusEnum(Integer value) {
+    private String message;
+
+    StorageDataStatusEnum(Integer value, String message) {
         this.value = value;
+        this.message = message;
     }
 
     public Integer getValue() {
         return value;
     }
 
+    public String getMessage() {
+        return message;
+    }
+
 }
\ No newline at end of file
-- 
Gitee


From 7b7fe25b5e715fe393c9ecadc18d667a88e6133e Mon Sep 17 00:00:00 2001
From: AE86 <836391306@qq.com>
Date: Tue, 12 Dec 2023 01:53:04 +0800
Subject: [PATCH 66/98] move storage

---
 .../dbsyncer/biz/checker/AbstractChecker.java |   2 +-
 .../impl/connector/ConnectorChecker.java      |   2 +-
 .../impl/group/ProjectGroupChecker.java       |   2 +-
 .../checker/impl/mapping/MappingChecker.java  |   2 +-
 .../impl/tablegroup/TableGroupChecker.java    |   2 +-
 .../biz/impl/ConnectorServiceImpl.java        |   2 +-
 .../biz/impl/DataSyncServiceImpl.java         |   4 +-
 .../dbsyncer/biz/impl/MappingServiceImpl.java |   2 +-
 .../org/dbsyncer/biz/impl/MetricReporter.java |   4 +-
 .../dbsyncer/biz/impl/MonitorServiceImpl.java |   4 +-
 .../biz/impl/TableGroupServiceImpl.java       |   2 +-
 .../common}/util/UnderlineToCamelUtils.java   | 110 +++++++++---------
 .../connector/ConnectorConfiguration.java     |  18 +++
 .../connector/mysql/cdc/MySQLListener.java    |   4 +-
 .../storage/MySQLStorageServiceProvider.java  |  58 +++------
 .../org.dbsyncer.sdk.spi.StorageService       |   1 +
 .../src/main/resources/dbsyncer_config.sql    |  18 +--
 .../src/main/resources/dbsyncer_data.sql      |  28 ++---
 .../src/main/resources/dbsyncer_log.sql       |  14 +--
 .../manager/impl/PreloadTemplate.java         |   4 +-
 .../org/dbsyncer/parser/ProfileComponent.java |   2 +-
 .../parser/command/PersistenceCommand.java    |   2 +-
 .../dbsyncer/parser/enums/CommandEnum.java    |   2 +-
 .../parser/flush/impl/FlushServiceImpl.java   |   4 +-
 .../flush/impl/StorageBufferActuator.java     |   2 +-
 .../parser/impl/OperationTemplate.java        |   2 +-
 .../dbsyncer/parser/model/ConfigModel.java    |   2 +-
 .../org/dbsyncer/parser/model/Connector.java  |   2 +-
 .../org/dbsyncer/parser/model/Mapping.java    |   2 +-
 .../java/org/dbsyncer/parser/model/Meta.java  |   2 +-
 .../dbsyncer/parser/model/ProjectGroup.java   |   2 +-
 .../dbsyncer/parser/model/SystemConfig.java   |   2 +-
 .../org/dbsyncer/parser/model/TableGroup.java |   2 +-
 .../org/dbsyncer/parser/model/UserConfig.java |   2 +-
 .../impl/PreloadTableGroupStrategy.java       |   2 +-
 .../strategy/impl/TableGroupStrategy.java     |   2 +-
 .../dbsyncer/parser/util/ConfigModelUtil.java |   2 +-
 .../dbsyncer/sdk}/NullExecutorException.java  |  31 ++---
 .../org/dbsyncer/sdk/client/RemoteClient.java |   4 +-
 .../sdk}/constant/ConfigConstant.java         |   2 +-
 .../org/dbsyncer/sdk/enums/ModelEnum.java     |   5 +-
 .../sdk/enums/StorageStrategyEnum.java        |  60 ++++++++++
 .../org/dbsyncer/sdk/spi}/StorageService.java |  13 ++-
 .../sdk}/storage/AbstractStorageService.java  |  17 ++-
 .../org/dbsyncer/sdk/storage}/Strategy.java   |  11 +-
 .../sdk/storage/strategy}/ConfigStrategy.java |  17 +--
 .../sdk/storage/strategy}/DataStrategy.java   |  17 +--
 .../sdk/storage/strategy}/LogStrategy.java    |  17 +--
 .../org/dbsyncer/sdk/util/DatabaseUtil.java   |   3 +
 .../org/dbsyncer/sdk/util/PrimaryKeyUtil.java |   3 +
 .../dbsyncer/storage/StorageException.java    |  11 +-
 .../storage/StorageSupportConfiguration.java  |   1 +
 .../storage/impl/DiskStorageServiceImpl.java  |   4 +-
 .../dbsyncer/storage/util/DocumentUtil.java   |   2 +-
 .../src/main/resources/application.properties |  10 +-
 55 files changed, 310 insertions(+), 237 deletions(-)
 rename {dbsyncer-storage/src/main/java/org/dbsyncer/storage => dbsyncer-common/src/main/java/org/dbsyncer/common}/util/UnderlineToCamelUtils.java (95%)
 rename dbsyncer-storage/src/main/java/org/dbsyncer/storage/impl/MySQLStorageServiceImpl.java => dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/storage/MySQLStorageServiceProvider.java (93%)
 create mode 100644 dbsyncer-connector/dbsyncer-connector-mysql/src/main/resources/META-INF/services/org.dbsyncer.sdk.spi.StorageService
 rename {dbsyncer-storage => dbsyncer-connector/dbsyncer-connector-mysql}/src/main/resources/dbsyncer_config.sql (93%)
 rename {dbsyncer-storage => dbsyncer-connector/dbsyncer-connector-mysql}/src/main/resources/dbsyncer_data.sql (94%)
 rename {dbsyncer-storage => dbsyncer-connector/dbsyncer-connector-mysql}/src/main/resources/dbsyncer_log.sql (91%)
 rename {dbsyncer-storage/src/main/java/org/dbsyncer/storage => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/NullExecutorException.java (57%)
 rename {dbsyncer-storage/src/main/java/org/dbsyncer/storage => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/constant/ConfigConstant.java (97%)
 create mode 100644 dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/StorageStrategyEnum.java
 rename {dbsyncer-storage/src/main/java/org/dbsyncer/storage => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi}/StorageService.java (89%)
 rename {dbsyncer-storage/src/main/java/org/dbsyncer => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk}/storage/AbstractStorageService.java (92%)
 rename {dbsyncer-storage/src/main/java/org/dbsyncer/storage/strategy => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage}/Strategy.java (74%)
 rename {dbsyncer-storage/src/main/java/org/dbsyncer/storage/strategy/impl => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/strategy}/ConfigStrategy.java (52%)
 rename {dbsyncer-storage/src/main/java/org/dbsyncer/storage/strategy/impl => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/strategy}/DataStrategy.java (64%)
 rename {dbsyncer-storage/src/main/java/org/dbsyncer/storage/strategy/impl => dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/strategy}/LogStrategy.java (50%)

diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/AbstractChecker.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/AbstractChecker.java
index 0ee63f9e..a5f99654 100644
--- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/AbstractChecker.java
+++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/AbstractChecker.java
@@ -11,7 +11,7 @@ import org.dbsyncer.parser.model.ConfigModel;
 import org.dbsyncer.parser.model.Convert;
 import org.dbsyncer.plugin.PluginFactory;
 import org.dbsyncer.plugin.model.Plugin;
-import org.dbsyncer.storage.constant.ConfigConstant;
+import org.dbsyncer.sdk.constant.ConfigConstant;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.util.Assert;
diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/ConnectorChecker.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/ConnectorChecker.java
index 6790720f..b60cbb0b 100644
--- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/ConnectorChecker.java
+++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/ConnectorChecker.java
@@ -13,7 +13,7 @@ import org.dbsyncer.sdk.connector.ConfigValidator;
 import org.dbsyncer.sdk.connector.ConnectorInstance;
 import org.dbsyncer.sdk.model.ConnectorConfig;
 import org.dbsyncer.sdk.spi.ConnectorService;
-import org.dbsyncer.storage.constant.ConfigConstant;
+import org.dbsyncer.sdk.constant.ConfigConstant;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.stereotype.Component;
diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/group/ProjectGroupChecker.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/group/ProjectGroupChecker.java
index 01bb68cf..97295e83 100644
--- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/group/ProjectGroupChecker.java
+++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/group/ProjectGroupChecker.java
@@ -6,7 +6,7 @@ import org.dbsyncer.common.util.StringUtil;
 import org.dbsyncer.parser.ProfileComponent;
 import org.dbsyncer.parser.model.ConfigModel;
 import org.dbsyncer.parser.model.ProjectGroup;
-import org.dbsyncer.storage.constant.ConfigConstant;
+import org.dbsyncer.sdk.constant.ConfigConstant;
 import org.springframework.stereotype.Component;
 import org.springframework.util.Assert;
 
diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/mapping/MappingChecker.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/mapping/MappingChecker.java
index dec9cd1b..5b2a67a3 100644
--- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/mapping/MappingChecker.java
+++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/mapping/MappingChecker.java
@@ -16,7 +16,7 @@ import org.dbsyncer.parser.model.ConfigModel;
 import org.dbsyncer.parser.model.Mapping;
 import org.dbsyncer.parser.model.Meta;
 import org.dbsyncer.parser.model.TableGroup;
-import org.dbsyncer.storage.constant.ConfigConstant;
+import org.dbsyncer.sdk.constant.ConfigConstant;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.stereotype.Component;
diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/tablegroup/TableGroupChecker.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/tablegroup/TableGroupChecker.java
index b7c4949f..91a2eb99 100644
--- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/tablegroup/TableGroupChecker.java
+++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/tablegroup/TableGroupChecker.java
@@ -20,7 +20,7 @@ import org.dbsyncer.sdk.model.Field;
 import org.dbsyncer.sdk.model.MetaInfo;
 import org.dbsyncer.sdk.model.Table;
 import org.dbsyncer.sdk.util.PrimaryKeyUtil;
-import org.dbsyncer.storage.constant.ConfigConstant;
+import org.dbsyncer.sdk.constant.ConfigConstant;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.stereotype.Component;
diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ConnectorServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ConnectorServiceImpl.java
index 13b10c37..fb70cc2f 100644
--- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ConnectorServiceImpl.java
+++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ConnectorServiceImpl.java
@@ -14,7 +14,7 @@ import org.dbsyncer.parser.model.ConfigModel;
 import org.dbsyncer.parser.model.Connector;
 import org.dbsyncer.parser.model.Mapping;
 import org.dbsyncer.sdk.model.ConnectorConfig;
-import org.dbsyncer.storage.constant.ConfigConstant;
+import org.dbsyncer.sdk.constant.ConfigConstant;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.stereotype.Service;
diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/DataSyncServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/DataSyncServiceImpl.java
index d1c47746..60904792 100644
--- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/DataSyncServiceImpl.java
+++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/DataSyncServiceImpl.java
@@ -21,9 +21,9 @@ import org.dbsyncer.sdk.listener.event.RowChangedEvent;
 import org.dbsyncer.sdk.model.Field;
 import org.dbsyncer.sdk.filter.FieldResolver;
 import org.dbsyncer.sdk.filter.Query;
-import org.dbsyncer.storage.StorageService;
+import org.dbsyncer.sdk.spi.StorageService;
 import org.dbsyncer.storage.binlog.proto.BinlogMap;
-import org.dbsyncer.storage.constant.ConfigConstant;
+import org.dbsyncer.sdk.constant.ConfigConstant;
 import org.dbsyncer.storage.util.BinlogMessageUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MappingServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MappingServiceImpl.java
index 80207014..06fb0fc0 100644
--- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MappingServiceImpl.java
+++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MappingServiceImpl.java
@@ -24,7 +24,7 @@ import org.dbsyncer.parser.model.TableGroup;
 import org.dbsyncer.sdk.connector.ConnectorInstance;
 import org.dbsyncer.sdk.enums.ModelEnum;
 import org.dbsyncer.sdk.model.Table;
-import org.dbsyncer.storage.constant.ConfigConstant;
+import org.dbsyncer.sdk.constant.ConfigConstant;
 import org.dbsyncer.storage.impl.SnowflakeIdWorker;
 import org.springframework.beans.BeanUtils;
 import org.springframework.stereotype.Service;
diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MetricReporter.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MetricReporter.java
index c42d773f..ec33bf4a 100644
--- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MetricReporter.java
+++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MetricReporter.java
@@ -26,8 +26,8 @@ import org.dbsyncer.parser.model.TableGroup;
 import org.dbsyncer.sdk.constant.ConnectorConstant;
 import org.dbsyncer.sdk.enums.StorageEnum;
 import org.dbsyncer.sdk.filter.Query;
-import org.dbsyncer.storage.StorageService;
-import org.dbsyncer.storage.constant.ConfigConstant;
+import org.dbsyncer.sdk.spi.StorageService;
+import org.dbsyncer.sdk.constant.ConfigConstant;
 import org.dbsyncer.storage.enums.StorageDataStatusEnum;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java
index cdbd9a71..3b0f051f 100644
--- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java
+++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java
@@ -44,8 +44,8 @@ import org.dbsyncer.sdk.filter.BooleanFilter;
 import org.dbsyncer.sdk.filter.FieldResolver;
 import org.dbsyncer.sdk.filter.Query;
 import org.dbsyncer.sdk.filter.impl.LongFilter;
-import org.dbsyncer.storage.StorageService;
-import org.dbsyncer.storage.constant.ConfigConstant;
+import org.dbsyncer.sdk.spi.StorageService;
+import org.dbsyncer.sdk.constant.ConfigConstant;
 import org.dbsyncer.storage.enums.StorageDataStatusEnum;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/TableGroupServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/TableGroupServiceImpl.java
index a7b7971e..d3e7f68a 100644
--- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/TableGroupServiceImpl.java
+++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/TableGroupServiceImpl.java
@@ -10,7 +10,7 @@ import org.dbsyncer.parser.ProfileComponent;
 import org.dbsyncer.parser.model.Mapping;
 import org.dbsyncer.parser.model.TableGroup;
 import org.dbsyncer.sdk.model.Field;
-import org.dbsyncer.storage.constant.ConfigConstant;
+import org.dbsyncer.sdk.constant.ConfigConstant;
 import org.springframework.stereotype.Service;
 import org.springframework.util.Assert;
 
diff --git a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/util/UnderlineToCamelUtils.java b/dbsyncer-common/src/main/java/org/dbsyncer/common/util/UnderlineToCamelUtils.java
similarity index 95%
rename from dbsyncer-storage/src/main/java/org/dbsyncer/storage/util/UnderlineToCamelUtils.java
rename to dbsyncer-common/src/main/java/org/dbsyncer/common/util/UnderlineToCamelUtils.java
index f130cfdf..8856110a 100644
--- a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/util/UnderlineToCamelUtils.java
+++ b/dbsyncer-common/src/main/java/org/dbsyncer/common/util/UnderlineToCamelUtils.java
@@ -1,56 +1,56 @@
-package org.dbsyncer.storage.util;
-
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-public abstract class UnderlineToCamelUtils {
-
-    /**
-     * 下划线转驼峰法
-     *
-     * @param line       源字符串
-     * @param smallCamel 大小驼峰,是否为小驼峰
-     * @return 转换后的字符串
-     */
-    public static String underlineToCamel(String line, boolean smallCamel) {
-        if (line == null || "".equals(line)) {
-            return "";
-        }
-        StringBuffer sb = new StringBuffer();
-        Pattern pattern = Pattern.compile("([A-Za-z\\d]+)(_)?");
-        Matcher matcher = pattern.matcher(line);
-        while (matcher.find()) {
-            String word = matcher.group();
-            sb.append(smallCamel && matcher.start() == 0 ? Character.toLowerCase(word.charAt(0)) : Character.toUpperCase(word.charAt(0)));
-            int index = word.lastIndexOf('_');
-            if (index > 0) {
-                sb.append(word.substring(1, index).toLowerCase());
-            } else {
-                sb.append(word.substring(1).toLowerCase());
-            }
-        }
-        return sb.toString();
-    }
-
-    /**
-     * 驼峰法转下划线
-     *
-     * @param line 源字符串
-     * @return 转换后的字符串
-     */
-    public static String camelToUnderline(String line) {
-        if (line == null || "".equals(line)) {
-            return "";
-        }
-        line = String.valueOf(line.charAt(0)).toUpperCase().concat(line.substring(1));
-        StringBuffer sb = new StringBuffer();
-        Pattern pattern = Pattern.compile("[A-Z]([a-z\\d]+)?");
-        Matcher matcher = pattern.matcher(line);
-        while (matcher.find()) {
-            String word = matcher.group();
-            sb.append(word.toUpperCase());
-            sb.append(matcher.end() == line.length() ? "" : "_");
-        }
-        return sb.toString();
-    }
+package org.dbsyncer.common.util;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public abstract class UnderlineToCamelUtils {
+
+    /**
+     * 下划线转驼峰法
+     *
+     * @param line       源字符串
+     * @param smallCamel 大小驼峰,是否为小驼峰
+     * @return 转换后的字符串
+     */
+    public static String underlineToCamel(String line, boolean smallCamel) {
+        if (line == null || "".equals(line)) {
+            return "";
+        }
+        StringBuffer sb = new StringBuffer();
+        Pattern pattern = Pattern.compile("([A-Za-z\\d]+)(_)?");
+        Matcher matcher = pattern.matcher(line);
+        while (matcher.find()) {
+            String word = matcher.group();
+            sb.append(smallCamel && matcher.start() == 0 ? Character.toLowerCase(word.charAt(0)) : Character.toUpperCase(word.charAt(0)));
+            int index = word.lastIndexOf('_');
+            if (index > 0) {
+                sb.append(word.substring(1, index).toLowerCase());
+            } else {
+                sb.append(word.substring(1).toLowerCase());
+            }
+        }
+        return sb.toString();
+    }
+
+    /**
+     * 驼峰法转下划线
+     *
+     * @param line 源字符串
+     * @return 转换后的字符串
+     */
+    public static String camelToUnderline(String line) {
+        if (line == null || "".equals(line)) {
+            return "";
+        }
+        line = String.valueOf(line.charAt(0)).toUpperCase().concat(line.substring(1));
+        StringBuffer sb = new StringBuffer();
+        Pattern pattern = Pattern.compile("[A-Z]([a-z\\d]+)?");
+        Matcher matcher = pattern.matcher(line);
+        while (matcher.find()) {
+            String word = matcher.group();
+            sb.append(word.toUpperCase());
+            sb.append(matcher.end() == line.length() ? "" : "_");
+        }
+        return sb.toString();
+    }
 }
\ No newline at end of file
diff --git a/dbsyncer-connector/dbsyncer-connector-base/src/main/java/org/dbsyncer/connector/ConnectorConfiguration.java b/dbsyncer-connector/dbsyncer-connector-base/src/main/java/org/dbsyncer/connector/ConnectorConfiguration.java
index 5f4326b1..7f8ddb8d 100644
--- a/dbsyncer-connector/dbsyncer-connector-base/src/main/java/org/dbsyncer/connector/ConnectorConfiguration.java
+++ b/dbsyncer-connector/dbsyncer-connector-base/src/main/java/org/dbsyncer/connector/ConnectorConfiguration.java
@@ -4,6 +4,7 @@
 package org.dbsyncer.connector;
 
 import org.dbsyncer.sdk.spi.ConnectorService;
+import org.dbsyncer.sdk.spi.StorageService;
 import org.springframework.beans.BeansException;
 import org.springframework.beans.factory.config.BeanDefinition;
 import org.springframework.beans.factory.config.ConfigurableListableBeanFactory;
@@ -29,6 +30,23 @@ public class ConnectorConfiguration implements BeanDefinitionRegistryPostProcess
 
     @Override
     public void postProcessBeanDefinitionRegistry(BeanDefinitionRegistry beanDefinitionRegistry) throws BeansException {
+        loadConnectorServices(beanDefinitionRegistry);
+//        loadStorageServices(beanDefinitionRegistry);
+    }
+
+    private void loadStorageServices(BeanDefinitionRegistry beanDefinitionRegistry) {
+        ServiceLoader services = ServiceLoader.load(StorageService.class, Thread.currentThread().getContextClassLoader());
+        for (StorageService s : services) {
+            BeanDefinition beanDefinition = BeanDefinitionBuilder.genericBeanDefinition(s.getClass()).getBeanDefinition();
+            String beanClassName = beanDefinition.getBeanClassName();
+            Assert.state(beanClassName != null, "No bean class name set");
+            String shortClassName = ClassUtils.getShortName(beanClassName);
+            String decapitalize = Introspector.decapitalize(shortClassName);
+            beanDefinitionRegistry.registerBeanDefinition(decapitalize, beanDefinition);
+        }
+    }
+
+    private void loadConnectorServices(BeanDefinitionRegistry beanDefinitionRegistry) {
         ServiceLoader services = ServiceLoader.load(ConnectorService.class, Thread.currentThread().getContextClassLoader());
         for (ConnectorService s : services) {
             BeanDefinition beanDefinition = BeanDefinitionBuilder.genericBeanDefinition(s.getClass()).getBeanDefinition();
diff --git a/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/cdc/MySQLListener.java b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/cdc/MySQLListener.java
index 134106fb..cfcf75ec 100644
--- a/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/cdc/MySQLListener.java
+++ b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/cdc/MySQLListener.java
@@ -227,7 +227,6 @@ public class MySQLListener extends AbstractDatabaseListener {
             if (!client.isConnected()) {
                 return;
             }
-            errorEvent(new MySQLException(e.getMessage()));
             /**
              * e:
              * case1> Due to the automatic expiration and deletion mechanism of MySQL binlog files, the binlog file cannot be found.
@@ -237,14 +236,15 @@ public class MySQLListener extends AbstractDatabaseListener {
             if (e instanceof ServerException) {
                 ServerException serverException = (ServerException) e;
                 if (serverException.getErrorCode() == 1236) {
-                    close();
                     String log = String.format("线程[%s]执行异常。由于MySQL配置了过期binlog文件自动删除机制，已无法找到原binlog文件%s。建议先保存驱动（加载最新的binlog文件），再启动驱动。",
                             client.getWorkerThreadName(),
                             client.getBinlogFilename());
                     errorEvent(new MySQLException(log));
+                    close();
                     return;
                 }
             }
+            errorEvent(new MySQLException(e.getMessage()));
         }
 
         @Override
diff --git a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/impl/MySQLStorageServiceImpl.java b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/storage/MySQLStorageServiceProvider.java
similarity index 93%
rename from dbsyncer-storage/src/main/java/org/dbsyncer/storage/impl/MySQLStorageServiceImpl.java
rename to dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/storage/MySQLStorageServiceProvider.java
index 505ea593..e74f0986 100644
--- a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/impl/MySQLStorageServiceImpl.java
+++ b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/storage/MySQLStorageServiceProvider.java
@@ -1,30 +1,29 @@
 /**
  * DBSyncer Copyright 2020-2023 All Rights Reserved.
  */
-package org.dbsyncer.storage.impl;
+package org.dbsyncer.connector.mysql.storage;
 
 import org.apache.commons.io.IOUtils;
 import org.dbsyncer.common.model.Paging;
 import org.dbsyncer.common.util.CollectionUtils;
 import org.dbsyncer.common.util.StringUtil;
-import org.dbsyncer.connector.ConnectorFactory;
+import org.dbsyncer.common.util.UnderlineToCamelUtils;
+import org.dbsyncer.connector.mysql.MySQLConnector;
+import org.dbsyncer.sdk.NullExecutorException;
 import org.dbsyncer.sdk.config.DatabaseConfig;
 import org.dbsyncer.sdk.config.SqlBuilderConfig;
-import org.dbsyncer.sdk.connector.database.Database;
 import org.dbsyncer.sdk.connector.database.DatabaseConnectorInstance;
+import org.dbsyncer.sdk.constant.ConfigConstant;
 import org.dbsyncer.sdk.constant.DatabaseConstant;
 import org.dbsyncer.sdk.enums.FilterEnum;
 import org.dbsyncer.sdk.enums.SqlBuilderEnum;
 import org.dbsyncer.sdk.enums.StorageEnum;
-import org.dbsyncer.sdk.model.Field;
 import org.dbsyncer.sdk.filter.AbstractFilter;
 import org.dbsyncer.sdk.filter.BooleanFilter;
 import org.dbsyncer.sdk.filter.Query;
+import org.dbsyncer.sdk.model.Field;
+import org.dbsyncer.sdk.storage.AbstractStorageService;
 import org.dbsyncer.sdk.util.DatabaseUtil;
-import org.dbsyncer.storage.AbstractStorageService;
-import org.dbsyncer.storage.NullExecutorException;
-import org.dbsyncer.storage.constant.ConfigConstant;
-import org.dbsyncer.storage.util.UnderlineToCamelUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
@@ -34,12 +33,10 @@ import org.springframework.stereotype.Component;
 import org.springframework.util.Assert;
 
 import javax.annotation.PostConstruct;
-import javax.annotation.Resource;
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
-import java.sql.SQLSyntaxErrorException;
 import java.sql.Types;
 import java.util.ArrayList;
 import java.util.List;
@@ -58,9 +55,9 @@ import java.util.stream.Stream;
  * @Date 2020-01-08 15:17
  */
 @Component
-@ConditionalOnProperty(value = "dbsyncer.storage.support.mysql.enabled", havingValue = "true")
-@ConfigurationProperties(prefix = "dbsyncer.storage.support.mysql")
-public class MySQLStorageServiceImpl extends AbstractStorageService {
+@ConditionalOnProperty(value = "dbsyncer.storage.type", havingValue = "MySQL")
+@ConfigurationProperties(prefix = "dbsyncer.storage.mysql")
+public class MySQLStorageServiceProvider extends AbstractStorageService {
 
     private final Logger logger = LoggerFactory.getLogger(getClass());
 
@@ -68,31 +65,18 @@ public class MySQLStorageServiceImpl extends AbstractStorageService {
     private final String SHOW_TABLE = "show tables where Tables_in_%s = '%s'";
     private final String DROP_TABLE = "DROP TABLE %s";
     private final String TRUNCATE_TABLE = "TRUNCATE TABLE %s";
-
-    @Resource
-    private ConnectorFactory connectorFactory;
-
     private Map tables = new ConcurrentHashMap<>();
-
-    private Database connector;
-
+    private MySQLConnector connector = new MySQLConnector();
     private DatabaseConnectorInstance connectorInstance;
-
     private DatabaseConfig config;
-
     private String database;
 
     @PostConstruct
     private void init() throws InterruptedException {
         logger.info("url:{}", config.getUrl());
-        config.setConnectorType("MySQL");
-        connectorInstance = (DatabaseConnectorInstance) connectorFactory.connect(config);
-        connector = (Database) connectorFactory.getConnectorService(config.getConnectorType());
+        config.setConnectorType(connector.getConnectorType());
         database = DatabaseUtil.getDatabaseName(config.getUrl());
-
-        // 升级脚本
-        initUpgradeSql();
-
+        connectorInstance = (DatabaseConnectorInstance) connector.connect(config);
         // 初始化表
         initTable();
     }
@@ -296,6 +280,7 @@ public class MySQLStorageServiceImpl extends AbstractStorageService {
     private void buildQuerySqlWithFilters(List filters, List args, StringBuilder sql, List highLightKeys) {
         // 过滤值
         int size = filters.size();
+
         String quotation = connector.buildSqlWithQuotation();
         for (int i = 0; i < size; i++) {
             AbstractFilter p = filters.get(i);
@@ -353,21 +338,6 @@ public class MySQLStorageServiceImpl extends AbstractStorageService {
         }
     }
 
-    private void initUpgradeSql() {
-        try {
-            executeSql("drop table if exists `dbsyncer_binlog`;");
-        } catch (Exception e) {
-            if (e.getCause() instanceof SQLSyntaxErrorException) {
-                SQLSyntaxErrorException ex = (SQLSyntaxErrorException) e.getCause();
-                if (ex.getSQLState().equals("42S21")) {
-                    // ignore
-                    return;
-                }
-            }
-            logger.error(e.getMessage());
-        }
-    }
-
     private void initTable() throws InterruptedException {
         // 配置
         FieldBuilder builder = new FieldBuilder();
diff --git a/dbsyncer-connector/dbsyncer-connector-mysql/src/main/resources/META-INF/services/org.dbsyncer.sdk.spi.StorageService b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/resources/META-INF/services/org.dbsyncer.sdk.spi.StorageService
new file mode 100644
index 00000000..049e45c2
--- /dev/null
+++ b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/resources/META-INF/services/org.dbsyncer.sdk.spi.StorageService
@@ -0,0 +1 @@
+org.dbsyncer.connector.mysql.storage.MySQLStorageServiceProvider
\ No newline at end of file
diff --git a/dbsyncer-storage/src/main/resources/dbsyncer_config.sql b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/resources/dbsyncer_config.sql
similarity index 93%
rename from dbsyncer-storage/src/main/resources/dbsyncer_config.sql
rename to dbsyncer-connector/dbsyncer-connector-mysql/src/main/resources/dbsyncer_config.sql
index 888dcd21..fc1f46ec 100644
--- a/dbsyncer-storage/src/main/resources/dbsyncer_config.sql
+++ b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/resources/dbsyncer_config.sql
@@ -1,10 +1,10 @@
-CREATE TABLE `dbsyncer_config`  (
-  `ID` varchar(64) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT '唯一ID',
-  `NAME` varchar(50) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT '名称',
-  `TYPE` varchar(24) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'system、user、connector、mapping、tableGroup、meta、projectGroup',
-  `CREATE_TIME` bigint(0) NOT NULL COMMENT '创建时间',
-  `UPDATE_TIME` bigint(0) NOT NULL COMMENT '修改时间',
-  `JSON` mediumtext CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT '配置信息',
-  PRIMARY KEY (`ID`) USING BTREE,
-  INDEX `IDX_TYPE_UPDATE_CREATE_TIME`(`TYPE`, `UPDATE_TIME`, `CREATE_TIME`) USING BTREE
+CREATE TABLE `dbsyncer_config`  (
+  `ID` varchar(64) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT '唯一ID',
+  `NAME` varchar(50) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT '名称',
+  `TYPE` varchar(24) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'system、user、connector、mapping、tableGroup、meta、projectGroup',
+  `CREATE_TIME` bigint(0) NOT NULL COMMENT '创建时间',
+  `UPDATE_TIME` bigint(0) NOT NULL COMMENT '修改时间',
+  `JSON` mediumtext CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT '配置信息',
+  PRIMARY KEY (`ID`) USING BTREE,
+  INDEX `IDX_TYPE_UPDATE_CREATE_TIME`(`TYPE`, `UPDATE_TIME`, `CREATE_TIME`) USING BTREE
 ) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_bin COMMENT = '配置信息表' ROW_FORMAT = Dynamic;
\ No newline at end of file
diff --git a/dbsyncer-storage/src/main/resources/dbsyncer_data.sql b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/resources/dbsyncer_data.sql
similarity index 94%
rename from dbsyncer-storage/src/main/resources/dbsyncer_data.sql
rename to dbsyncer-connector/dbsyncer-connector-mysql/src/main/resources/dbsyncer_data.sql
index 7b5eea65..1610ed7f 100644
--- a/dbsyncer-storage/src/main/resources/dbsyncer_data.sql
+++ b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/resources/dbsyncer_data.sql
@@ -1,15 +1,15 @@
-CREATE TABLE `dbsyncer_data` (
-  `ID` varchar(64) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT '唯一ID',
-  `SUCCESS` int(1) NOT NULL COMMENT '成功1/失败0',
-  `TABLE_GROUP_ID` varchar(64) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT '' COMMENT '驱动表映射关系ID',
-  `TARGET_TABLE_NAME` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT '' COMMENT '目标表名称',
-  `EVENT` varchar(10) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT '事件',
-  `ERROR` mediumtext CHARACTER SET utf8 COLLATE utf8_bin NULL COMMENT '异常信息',
-  `CREATE_TIME` bigint(0) NOT NULL COMMENT '创建时间',
-  `DATA` blob NOT NULL COMMENT '同步数据',
-  PRIMARY KEY (`ID`) USING BTREE,
-  INDEX `IDX_SUCCESS`(`SUCCESS`) USING BTREE,
-  INDEX `IDX_EVENT`(`EVENT`) USING BTREE,
-  INDEX `IDX_SUCCESS_CREATE_TIME`(`SUCCESS`, `CREATE_TIME`) USING BTREE,
-  FULLTEXT INDEX `FULL_TEXT_ERROR`(`ERROR`)
+CREATE TABLE `dbsyncer_data` (
+  `ID` varchar(64) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT '唯一ID',
+  `SUCCESS` int(1) NOT NULL COMMENT '成功1/失败0',
+  `TABLE_GROUP_ID` varchar(64) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT '' COMMENT '驱动表映射关系ID',
+  `TARGET_TABLE_NAME` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT '' COMMENT '目标表名称',
+  `EVENT` varchar(10) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT '事件',
+  `ERROR` mediumtext CHARACTER SET utf8 COLLATE utf8_bin NULL COMMENT '异常信息',
+  `CREATE_TIME` bigint(0) NOT NULL COMMENT '创建时间',
+  `DATA` blob NOT NULL COMMENT '同步数据',
+  PRIMARY KEY (`ID`) USING BTREE,
+  INDEX `IDX_SUCCESS`(`SUCCESS`) USING BTREE,
+  INDEX `IDX_EVENT`(`EVENT`) USING BTREE,
+  INDEX `IDX_SUCCESS_CREATE_TIME`(`SUCCESS`, `CREATE_TIME`) USING BTREE,
+  FULLTEXT INDEX `FULL_TEXT_ERROR`(`ERROR`)
 ) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_bin COMMENT = '同步数据表' ROW_FORMAT = Dynamic;
\ No newline at end of file
diff --git a/dbsyncer-storage/src/main/resources/dbsyncer_log.sql b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/resources/dbsyncer_log.sql
similarity index 91%
rename from dbsyncer-storage/src/main/resources/dbsyncer_log.sql
rename to dbsyncer-connector/dbsyncer-connector-mysql/src/main/resources/dbsyncer_log.sql
index e7626be4..4440e779 100644
--- a/dbsyncer-storage/src/main/resources/dbsyncer_log.sql
+++ b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/resources/dbsyncer_log.sql
@@ -1,8 +1,8 @@
-CREATE TABLE `dbsyncer_log`  (
-  `ID` varchar(64) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT '唯一ID',
-  `TYPE` varchar(24) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT '连接器、映射配置、表映射、元信息、系统日志',
-  `CREATE_TIME` bigint(0) NOT NULL COMMENT '创建时间',
-  `JSON` mediumtext CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT '日志信息',
-  PRIMARY KEY (`ID`) USING BTREE,
-  FULLTEXT INDEX `FULL_TEXT_JSON`(`JSON`)
+CREATE TABLE `dbsyncer_log`  (
+  `ID` varchar(64) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT '唯一ID',
+  `TYPE` varchar(24) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT '连接器、映射配置、表映射、元信息、系统日志',
+  `CREATE_TIME` bigint(0) NOT NULL COMMENT '创建时间',
+  `JSON` mediumtext CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT '日志信息',
+  PRIMARY KEY (`ID`) USING BTREE,
+  FULLTEXT INDEX `FULL_TEXT_JSON`(`JSON`)
 ) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_bin COMMENT = '操作日志表' ROW_FORMAT = Dynamic;
\ No newline at end of file
diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/PreloadTemplate.java b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/PreloadTemplate.java
index bca19d15..73ccc9c1 100644
--- a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/PreloadTemplate.java
+++ b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/PreloadTemplate.java
@@ -22,8 +22,8 @@ import org.dbsyncer.plugin.PluginFactory;
 import org.dbsyncer.sdk.connector.ConnectorInstance;
 import org.dbsyncer.sdk.enums.StorageEnum;
 import org.dbsyncer.sdk.filter.Query;
-import org.dbsyncer.storage.StorageService;
-import org.dbsyncer.storage.constant.ConfigConstant;
+import org.dbsyncer.sdk.spi.StorageService;
+import org.dbsyncer.sdk.constant.ConfigConstant;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.context.ApplicationListener;
diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ProfileComponent.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ProfileComponent.java
index 297da059..61dd07f7 100644
--- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ProfileComponent.java
+++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ProfileComponent.java
@@ -15,7 +15,7 @@ import org.dbsyncer.parser.model.SystemConfig;
 import org.dbsyncer.parser.model.TableGroup;
 import org.dbsyncer.parser.model.UserConfig;
 import org.dbsyncer.sdk.enums.OperationEnum;
-import org.dbsyncer.storage.constant.ConfigConstant;
+import org.dbsyncer.sdk.constant.ConfigConstant;
 import org.dbsyncer.storage.enums.StorageDataStatusEnum;
 
 import java.util.List;
diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/command/PersistenceCommand.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/command/PersistenceCommand.java
index 0d586642..74ef06c7 100644
--- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/command/PersistenceCommand.java
+++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/command/PersistenceCommand.java
@@ -4,7 +4,7 @@
 package org.dbsyncer.parser.command;
 
 import org.dbsyncer.sdk.enums.StorageEnum;
-import org.dbsyncer.storage.StorageService;
+import org.dbsyncer.sdk.spi.StorageService;
 
 import java.util.Map;
 
diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/enums/CommandEnum.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/enums/CommandEnum.java
index afa27bd4..bca8b483 100644
--- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/enums/CommandEnum.java
+++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/enums/CommandEnum.java
@@ -6,7 +6,7 @@ package org.dbsyncer.parser.enums;
 import org.dbsyncer.parser.command.CommandExecutor;
 import org.dbsyncer.parser.command.Persistence;
 import org.dbsyncer.parser.command.Preload;
-import org.dbsyncer.storage.constant.ConfigConstant;
+import org.dbsyncer.sdk.constant.ConfigConstant;
 
 /**
  * 枚举命令模式: 持久化和预加载
diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/FlushServiceImpl.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/FlushServiceImpl.java
index 450b4756..c49520ea 100644
--- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/FlushServiceImpl.java
+++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/FlushServiceImpl.java
@@ -6,8 +6,8 @@ import org.dbsyncer.parser.flush.BufferActuator;
 import org.dbsyncer.parser.flush.FlushService;
 import org.dbsyncer.parser.model.StorageRequest;
 import org.dbsyncer.sdk.enums.StorageEnum;
-import org.dbsyncer.storage.StorageService;
-import org.dbsyncer.storage.constant.ConfigConstant;
+import org.dbsyncer.sdk.spi.StorageService;
+import org.dbsyncer.sdk.constant.ConfigConstant;
 import org.dbsyncer.storage.enums.StorageDataStatusEnum;
 import org.dbsyncer.storage.impl.SnowflakeIdWorker;
 import org.dbsyncer.storage.util.BinlogMessageUtil;
diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/StorageBufferActuator.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/StorageBufferActuator.java
index 175aefff..c2c163bf 100644
--- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/StorageBufferActuator.java
+++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/StorageBufferActuator.java
@@ -8,7 +8,7 @@ import org.dbsyncer.parser.model.Meta;
 import org.dbsyncer.parser.model.StorageRequest;
 import org.dbsyncer.parser.model.StorageResponse;
 import org.dbsyncer.sdk.enums.StorageEnum;
-import org.dbsyncer.storage.StorageService;
+import org.dbsyncer.sdk.spi.StorageService;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.stereotype.Component;
diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/OperationTemplate.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/OperationTemplate.java
index c898f9b9..b90ab21f 100644
--- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/OperationTemplate.java
+++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/OperationTemplate.java
@@ -16,7 +16,7 @@ import org.dbsyncer.parser.model.QueryConfig;
 import org.dbsyncer.parser.strategy.GroupStrategy;
 import org.dbsyncer.parser.util.ConfigModelUtil;
 import org.dbsyncer.sdk.enums.StorageEnum;
-import org.dbsyncer.storage.StorageService;
+import org.dbsyncer.sdk.spi.StorageService;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.BeanUtils;
diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/ConfigModel.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/ConfigModel.java
index 5daf8803..d8a7e3bc 100644
--- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/ConfigModel.java
+++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/ConfigModel.java
@@ -1,6 +1,6 @@
 package org.dbsyncer.parser.model;
 
-import org.dbsyncer.storage.constant.ConfigConstant;
+import org.dbsyncer.sdk.constant.ConfigConstant;
 
 /**
  * @author AE86
diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/Connector.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/Connector.java
index 2e7c5004..ad508486 100644
--- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/Connector.java
+++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/Connector.java
@@ -2,7 +2,7 @@ package org.dbsyncer.parser.model;
 
 import org.dbsyncer.sdk.model.ConnectorConfig;
 import org.dbsyncer.sdk.model.Table;
-import org.dbsyncer.storage.constant.ConfigConstant;
+import org.dbsyncer.sdk.constant.ConfigConstant;
 
 import java.util.List;
 
diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/Mapping.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/Mapping.java
index a52ae871..08bbfe08 100644
--- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/Mapping.java
+++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/Mapping.java
@@ -3,7 +3,7 @@ package org.dbsyncer.parser.model;
 import org.dbsyncer.sdk.config.ListenerConfig;
 import org.dbsyncer.sdk.enums.ModelEnum;
 import org.dbsyncer.sdk.model.Field;
-import org.dbsyncer.storage.constant.ConfigConstant;
+import org.dbsyncer.sdk.constant.ConfigConstant;
 
 import java.util.List;
 
diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/Meta.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/Meta.java
index 437fb02c..94625238 100644
--- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/Meta.java
+++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/Meta.java
@@ -1,7 +1,7 @@
 package org.dbsyncer.parser.model;
 
 import org.dbsyncer.parser.enums.MetaEnum;
-import org.dbsyncer.storage.constant.ConfigConstant;
+import org.dbsyncer.sdk.constant.ConfigConstant;
 
 import java.util.LinkedHashMap;
 import java.util.Map;
diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/ProjectGroup.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/ProjectGroup.java
index 31be4f00..b04ab5b1 100644
--- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/ProjectGroup.java
+++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/ProjectGroup.java
@@ -1,6 +1,6 @@
 package org.dbsyncer.parser.model;
 
-import org.dbsyncer.storage.constant.ConfigConstant;
+import org.dbsyncer.sdk.constant.ConfigConstant;
 
 import java.util.List;
 
diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/SystemConfig.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/SystemConfig.java
index 9f28fb17..f79bb321 100644
--- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/SystemConfig.java
+++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/SystemConfig.java
@@ -1,6 +1,6 @@
 package org.dbsyncer.parser.model;
 
-import org.dbsyncer.storage.constant.ConfigConstant;
+import org.dbsyncer.sdk.constant.ConfigConstant;
 
 /**
  * 系统配置
diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/TableGroup.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/TableGroup.java
index 53184f3d..8fb558e6 100644
--- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/TableGroup.java
+++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/TableGroup.java
@@ -1,7 +1,7 @@
 package org.dbsyncer.parser.model;
 
 import org.dbsyncer.sdk.model.Table;
-import org.dbsyncer.storage.constant.ConfigConstant;
+import org.dbsyncer.sdk.constant.ConfigConstant;
 
 import java.util.List;
 import java.util.Map;
diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/UserConfig.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/UserConfig.java
index 1227e85f..b54049bc 100644
--- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/UserConfig.java
+++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/UserConfig.java
@@ -1,6 +1,6 @@
 package org.dbsyncer.parser.model;
 
-import org.dbsyncer.storage.constant.ConfigConstant;
+import org.dbsyncer.sdk.constant.ConfigConstant;
 
 import java.util.ArrayList;
 import java.util.Iterator;
diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/impl/PreloadTableGroupStrategy.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/impl/PreloadTableGroupStrategy.java
index 3dfa2086..0d41e756 100644
--- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/impl/PreloadTableGroupStrategy.java
+++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/impl/PreloadTableGroupStrategy.java
@@ -5,7 +5,7 @@ package org.dbsyncer.parser.strategy.impl;
 
 import org.dbsyncer.parser.model.Mapping;
 import org.dbsyncer.parser.strategy.GroupStrategy;
-import org.dbsyncer.storage.constant.ConfigConstant;
+import org.dbsyncer.sdk.constant.ConfigConstant;
 
 /**
  * @author AE86
diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/impl/TableGroupStrategy.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/impl/TableGroupStrategy.java
index c8f2ad2d..a08541c5 100644
--- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/impl/TableGroupStrategy.java
+++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/impl/TableGroupStrategy.java
@@ -5,7 +5,7 @@ package org.dbsyncer.parser.strategy.impl;
 
 import org.dbsyncer.parser.model.TableGroup;
 import org.dbsyncer.parser.strategy.GroupStrategy;
-import org.dbsyncer.storage.constant.ConfigConstant;
+import org.dbsyncer.sdk.constant.ConfigConstant;
 
 /**
  * @author AE86
diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/util/ConfigModelUtil.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/util/ConfigModelUtil.java
index 43d09f62..b6128032 100644
--- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/util/ConfigModelUtil.java
+++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/util/ConfigModelUtil.java
@@ -2,7 +2,7 @@ package org.dbsyncer.parser.util;
 
 import org.dbsyncer.common.util.JsonUtil;
 import org.dbsyncer.parser.model.ConfigModel;
-import org.dbsyncer.storage.constant.ConfigConstant;
+import org.dbsyncer.sdk.constant.ConfigConstant;
 
 import java.util.HashMap;
 import java.util.Map;
diff --git a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/NullExecutorException.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/NullExecutorException.java
similarity index 57%
rename from dbsyncer-storage/src/main/java/org/dbsyncer/storage/NullExecutorException.java
rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/NullExecutorException.java
index 316fb970..fd284396 100644
--- a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/NullExecutorException.java
+++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/NullExecutorException.java
@@ -1,15 +1,18 @@
-package org.dbsyncer.storage;
-
-/**
- * @author AE86
- * @version 1.0.0
- * @date 2023/10/13 21:38
- */
-public class NullExecutorException extends RuntimeException {
-
-	private static final long serialVersionUID = 1L;
-
-	public NullExecutorException(String message) {
-        super(message);
-    }
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
+package org.dbsyncer.sdk;
+
+/**
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2023-10-13 21:38
+ */
+public class NullExecutorException extends RuntimeException {
+
+	private static final long serialVersionUID = 1L;
+
+	public NullExecutorException(String message) {
+        super(message);
+    }
 }
\ No newline at end of file
diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/client/RemoteClient.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/client/RemoteClient.java
index 98e2b005..b442290e 100644
--- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/client/RemoteClient.java
+++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/client/RemoteClient.java
@@ -4,9 +4,9 @@
 package org.dbsyncer.sdk.client;
 
 /**
- * @Version 1.0.0
  * @Author AE86
+ * @Version 1.0.0
  * @Date 2023-11-14 23:34
  */
-public class RemoteClient {
+public interface RemoteClient {
 }
\ No newline at end of file
diff --git a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/constant/ConfigConstant.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/constant/ConfigConstant.java
similarity index 97%
rename from dbsyncer-storage/src/main/java/org/dbsyncer/storage/constant/ConfigConstant.java
rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/constant/ConfigConstant.java
index f9a4ca35..619afb7e 100644
--- a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/constant/ConfigConstant.java
+++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/constant/ConfigConstant.java
@@ -1,7 +1,7 @@
 /**
  * DBSyncer Copyright 2020-2023 All Rights Reserved.
  */
-package org.dbsyncer.storage.constant;
+package org.dbsyncer.sdk.constant;
 
 /**
  * @Author AE86
diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/ModelEnum.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/ModelEnum.java
index 6b768a0a..b980429e 100644
--- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/ModelEnum.java
+++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/ModelEnum.java
@@ -1,3 +1,6 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
 package org.dbsyncer.sdk.enums;
 
 import org.dbsyncer.common.util.StringUtil;
@@ -6,8 +9,8 @@ import org.dbsyncer.sdk.SdkException;
 /**
  * 驱动同步方式枚举
  *
- * @Version 1.0.0
  * @Author AE86
+ * @Version 1.0.0
  * @Date 2023-11-16 00:51
  */
 public enum ModelEnum {
diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/StorageStrategyEnum.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/StorageStrategyEnum.java
new file mode 100644
index 00000000..32422320
--- /dev/null
+++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/enums/StorageStrategyEnum.java
@@ -0,0 +1,60 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
+package org.dbsyncer.sdk.enums;
+
+import org.dbsyncer.sdk.SdkException;
+import org.dbsyncer.sdk.storage.Strategy;
+import org.dbsyncer.sdk.storage.strategy.ConfigStrategy;
+import org.dbsyncer.sdk.storage.strategy.DataStrategy;
+import org.dbsyncer.sdk.storage.strategy.LogStrategy;
+
+/**
+ * 存储策略枚举
+ *
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2023-12-12 00:51
+ */
+public enum StorageStrategyEnum {
+
+    /**
+     * 配置策略
+     */
+    CONFIG(StorageEnum.CONFIG, new ConfigStrategy()),
+
+    /**
+     * 数据策略
+     */
+    DATA(StorageEnum.DATA, new DataStrategy()),
+
+    /**
+     * 日志策略
+     */
+    LOG(StorageEnum.LOG, new LogStrategy());
+
+    private StorageEnum type;
+    private Strategy strategy;
+
+    StorageStrategyEnum(StorageEnum type, Strategy strategy) {
+        this.type = type;
+        this.strategy = strategy;
+    }
+
+    public static Strategy getStrategy(StorageEnum type) throws SdkException {
+        for (StorageStrategyEnum e : StorageStrategyEnum.values()) {
+            if (type == e.getType()) {
+                return e.getStrategy();
+            }
+        }
+        throw new SdkException(String.format("StorageStrategy type \"%s\" does not exist.", type));
+    }
+
+    public StorageEnum getType() {
+        return type;
+    }
+
+    public Strategy getStrategy() {
+        return strategy;
+    }
+}
\ No newline at end of file
diff --git a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/StorageService.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/StorageService.java
similarity index 89%
rename from dbsyncer-storage/src/main/java/org/dbsyncer/storage/StorageService.java
rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/StorageService.java
index 77127d28..fd9bf7ff 100644
--- a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/StorageService.java
+++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/StorageService.java
@@ -1,4 +1,7 @@
-package org.dbsyncer.storage;
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
+package org.dbsyncer.sdk.spi;
 
 import org.dbsyncer.common.model.Paging;
 import org.dbsyncer.sdk.enums.StorageEnum;
@@ -8,9 +11,11 @@ import java.util.List;
 import java.util.Map;
 
 /**
- * @author AE86
- * @version 1.0.0
- * @date 2019/10/1 15:18
+ * 存储服务（支持记录配置/日志/同步数据）
+ *
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2019-11-16 23:22
  */
 public interface StorageService {
 
diff --git a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/AbstractStorageService.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/AbstractStorageService.java
similarity index 92%
rename from dbsyncer-storage/src/main/java/org/dbsyncer/storage/AbstractStorageService.java
rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/AbstractStorageService.java
index b500911f..414bebaa 100644
--- a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/AbstractStorageService.java
+++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/AbstractStorageService.java
@@ -1,20 +1,22 @@
 /**
  * DBSyncer Copyright 2020-2023 All Rights Reserved.
  */
-package org.dbsyncer.storage;
+package org.dbsyncer.sdk.storage;
 
 import org.dbsyncer.common.model.Paging;
 import org.dbsyncer.common.util.CollectionUtils;
+import org.dbsyncer.sdk.NullExecutorException;
+import org.dbsyncer.sdk.SdkException;
 import org.dbsyncer.sdk.enums.StorageEnum;
+import org.dbsyncer.sdk.enums.StorageStrategyEnum;
 import org.dbsyncer.sdk.filter.BooleanFilter;
 import org.dbsyncer.sdk.filter.Query;
-import org.dbsyncer.storage.strategy.Strategy;
+import org.dbsyncer.sdk.spi.StorageService;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.DisposableBean;
 import org.springframework.util.Assert;
 
-import javax.annotation.Resource;
 import java.io.File;
 import java.util.ArrayList;
 import java.util.List;
@@ -32,9 +34,6 @@ public abstract class AbstractStorageService implements StorageService, Disposab
 
     private final Logger logger = LoggerFactory.getLogger(getClass());
 
-    @Resource
-    private Map map;
-
     private final Lock lock = new ReentrantLock();
 
     protected abstract Paging select(String sharding, Query query);
@@ -51,9 +50,7 @@ public abstract class AbstractStorageService implements StorageService, Disposab
 
     protected String getSharding(StorageEnum type, String collectionId) {
         Assert.notNull(type, "StorageEnum type can not be null.");
-        Strategy strategy = map.get(type.getType().concat("Strategy"));
-        Assert.notNull(strategy, "Strategy does not exist.");
-        return strategy.createSharding(getSeparator(), collectionId);
+        return StorageStrategyEnum.getStrategy(type).createSharding(getSeparator(), collectionId);
     }
 
     protected String getSeparator() {
@@ -85,7 +82,7 @@ public abstract class AbstractStorageService implements StorageService, Disposab
     public void delete(Query query) {
         BooleanFilter q = query.getBooleanFilter();
         if (CollectionUtils.isEmpty(q.getClauses()) && CollectionUtils.isEmpty(q.getFilters())) {
-            throw new StorageException("必须包含删除条件");
+            throw new SdkException("必须包含删除条件");
         }
 
         boolean locked = false;
diff --git a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/strategy/Strategy.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/Strategy.java
similarity index 74%
rename from dbsyncer-storage/src/main/java/org/dbsyncer/storage/strategy/Strategy.java
rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/Strategy.java
index fc7aed52..7f26685a 100644
--- a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/strategy/Strategy.java
+++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/Strategy.java
@@ -1,4 +1,7 @@
-package org.dbsyncer.storage.strategy;
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
+package org.dbsyncer.sdk.storage;
 
 /**
  * 创建数据存放的集合ID
@@ -12,9 +15,9 @@ package org.dbsyncer.storage.strategy;
  * --------/driver...
  * 
  *
- * @author AE86
- * @version 1.0.0
- * @date 2019/11/15 22:31
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2019-11-15 22:31
  */
 public interface Strategy {
 
diff --git a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/strategy/impl/ConfigStrategy.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/strategy/ConfigStrategy.java
similarity index 52%
rename from dbsyncer-storage/src/main/java/org/dbsyncer/storage/strategy/impl/ConfigStrategy.java
rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/strategy/ConfigStrategy.java
index 28622b29..a70d21ab 100644
--- a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/strategy/impl/ConfigStrategy.java
+++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/strategy/ConfigStrategy.java
@@ -1,18 +1,19 @@
-package org.dbsyncer.storage.strategy.impl;
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
+package org.dbsyncer.sdk.storage.strategy;
 
 import org.dbsyncer.sdk.enums.StorageEnum;
-import org.dbsyncer.storage.strategy.Strategy;
-import org.springframework.stereotype.Component;
+import org.dbsyncer.sdk.storage.Strategy;
 
 /**
  * 配置：Connector、Mapping、TableGroup、Meta、ProjectGroup、SystemConfig、UserConfig
  *
- * @author AE86
- * @version 1.0.0
- * @date 2019/11/15 22:39
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2019-11-16 23:22
  */
-@Component
-public class ConfigStrategy implements Strategy {
+public final class ConfigStrategy implements Strategy {
 
     @Override
     public String createSharding(String separator, String collectionId) {
diff --git a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/strategy/impl/DataStrategy.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/strategy/DataStrategy.java
similarity index 64%
rename from dbsyncer-storage/src/main/java/org/dbsyncer/storage/strategy/impl/DataStrategy.java
rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/strategy/DataStrategy.java
index 0120617a..431371a5 100644
--- a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/strategy/impl/DataStrategy.java
+++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/strategy/DataStrategy.java
@@ -1,19 +1,20 @@
-package org.dbsyncer.storage.strategy.impl;
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
+package org.dbsyncer.sdk.storage.strategy;
 
 import org.dbsyncer.sdk.enums.StorageEnum;
-import org.dbsyncer.storage.strategy.Strategy;
-import org.springframework.stereotype.Component;
+import org.dbsyncer.sdk.storage.Strategy;
 import org.springframework.util.Assert;
 
 /**
  * 数据：全量或增量数据
  *
- * @author AE86
- * @version 1.0.0
- * @date 2019/11/15 22:40
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2019-11-16 23:22
  */
-@Component
-public class DataStrategy implements Strategy {
+public final class DataStrategy implements Strategy {
 
     @Override
     public String createSharding(String separator, String collectionId) {
diff --git a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/strategy/impl/LogStrategy.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/strategy/LogStrategy.java
similarity index 50%
rename from dbsyncer-storage/src/main/java/org/dbsyncer/storage/strategy/impl/LogStrategy.java
rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/strategy/LogStrategy.java
index 90ddc2b8..7e265067 100644
--- a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/strategy/impl/LogStrategy.java
+++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/strategy/LogStrategy.java
@@ -1,18 +1,19 @@
-package org.dbsyncer.storage.strategy.impl;
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
+package org.dbsyncer.sdk.storage.strategy;
 
 import org.dbsyncer.sdk.enums.StorageEnum;
-import org.dbsyncer.storage.strategy.Strategy;
-import org.springframework.stereotype.Component;
+import org.dbsyncer.sdk.storage.Strategy;
 
 /**
  * 日志：Connector、Mapping、TableGroup、Meta、系统日志
  *
- * @author AE86
- * @version 1.0.0
- * @date 2019/11/15 22:39
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2019-11-16 23:22
  */
-@Component
-public class LogStrategy implements Strategy {
+public final class LogStrategy implements Strategy {
 
     @Override
     public String createSharding(String separator, String collectionId) {
diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/util/DatabaseUtil.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/util/DatabaseUtil.java
index 984dd07a..3a23679a 100644
--- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/util/DatabaseUtil.java
+++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/util/DatabaseUtil.java
@@ -1,3 +1,6 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
 package org.dbsyncer.sdk.util;
 
 import org.dbsyncer.common.util.StringUtil;
diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/util/PrimaryKeyUtil.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/util/PrimaryKeyUtil.java
index c1c9e1a4..8f7a64ee 100644
--- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/util/PrimaryKeyUtil.java
+++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/util/PrimaryKeyUtil.java
@@ -1,3 +1,6 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
 package org.dbsyncer.sdk.util;
 
 import org.dbsyncer.common.util.CollectionUtils;
diff --git a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/StorageException.java b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/StorageException.java
index 5eb25d61..f44ebf4d 100644
--- a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/StorageException.java
+++ b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/StorageException.java
@@ -1,9 +1,12 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
 package org.dbsyncer.storage;
 
 /**
- * @author AE86
- * @version 1.0.0
- * @date 2019/11/10 21:38
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2019-11-10 21:38
  */
 public class StorageException extends RuntimeException {
 
@@ -24,4 +27,4 @@ public class StorageException extends RuntimeException {
     protected StorageException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
         super(message, cause, enableSuppression, writableStackTrace);
     }
-}
+}
\ No newline at end of file
diff --git a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/StorageSupportConfiguration.java b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/StorageSupportConfiguration.java
index 7a64b66f..4fd89af6 100644
--- a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/StorageSupportConfiguration.java
+++ b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/StorageSupportConfiguration.java
@@ -1,5 +1,6 @@
 package org.dbsyncer.storage;
 
+import org.dbsyncer.sdk.spi.StorageService;
 import org.dbsyncer.storage.impl.DiskStorageServiceImpl;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
 import org.springframework.context.annotation.Bean;
diff --git a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/impl/DiskStorageServiceImpl.java b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/impl/DiskStorageServiceImpl.java
index b5c85f88..9ceaddca 100644
--- a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/impl/DiskStorageServiceImpl.java
+++ b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/impl/DiskStorageServiceImpl.java
@@ -18,9 +18,9 @@ import org.dbsyncer.common.model.Paging;
 import org.dbsyncer.common.util.CollectionUtils;
 import org.dbsyncer.sdk.enums.FilterEnum;
 import org.dbsyncer.sdk.enums.OperationEnum;
-import org.dbsyncer.storage.AbstractStorageService;
+import org.dbsyncer.sdk.storage.AbstractStorageService;
 import org.dbsyncer.storage.StorageException;
-import org.dbsyncer.storage.constant.ConfigConstant;
+import org.dbsyncer.sdk.constant.ConfigConstant;
 import org.dbsyncer.sdk.enums.StorageEnum;
 import org.dbsyncer.storage.lucene.Option;
 import org.dbsyncer.storage.lucene.Shard;
diff --git a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/util/DocumentUtil.java b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/util/DocumentUtil.java
index 808533b1..bc80e3cd 100644
--- a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/util/DocumentUtil.java
+++ b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/util/DocumentUtil.java
@@ -2,7 +2,7 @@ package org.dbsyncer.storage.util;
 
 import org.apache.lucene.document.*;
 import org.apache.lucene.util.BytesRef;
-import org.dbsyncer.storage.constant.ConfigConstant;
+import org.dbsyncer.sdk.constant.ConfigConstant;
 import org.springframework.util.Assert;
 
 import java.util.Map;
diff --git a/dbsyncer-web/src/main/resources/application.properties b/dbsyncer-web/src/main/resources/application.properties
index edf20cca..d8640266 100644
--- a/dbsyncer-web/src/main/resources/application.properties
+++ b/dbsyncer-web/src/main/resources/application.properties
@@ -48,12 +48,12 @@ dbsyncer.parser.table.group.buffer-queue-capacity=40000
 dbsyncer.parser.table.group.buffer-period-millisecond=300
 
 #storage
-# 是否使用MySQL存储配置（false-关闭; true-开启）
+# 是否使用MySQL存储配置类型（MySQL）
 # false: 保存磁盘/data/config(驱动配置)|data(按驱动分别存储增量数据)|log(系统日志)}
-dbsyncer.storage.support.mysql.enabled=false
-dbsyncer.storage.support.mysql.config.url=jdbc:mysql://127.0.0.1:3306/dbsyncer?rewriteBatchedStatements=true&seUnicode=true&characterEncoding=UTF8&serverTimezone=Asia/Shanghai&useSSL=false&verifyServerCertificate=false&autoReconnect=true
-dbsyncer.storage.support.mysql.config.username=root
-dbsyncer.storage.support.mysql.config.password=123
+dbsyncer.storage.type=MySQL
+dbsyncer.storage.mysql.config.url=jdbc:mysql://127.0.0.1:3306/dbsyncer?rewriteBatchedStatements=true&seUnicode=true&characterEncoding=UTF8&serverTimezone=Asia/Shanghai&useSSL=false&verifyServerCertificate=false&autoReconnect=true
+dbsyncer.storage.mysql.config.username=root
+dbsyncer.storage.mysql.config.password=123
 # [StorageBufferActuator]线程数
 dbsyncer.storage.thread-core-size=4
 # [StorageBufferActuator]最大线程数
-- 
Gitee


From d2cbb15c47cb68efe9bebf0a377a815185adf323 Mon Sep 17 00:00:00 2001
From: AE86 <836391306@qq.com>
Date: Tue, 12 Dec 2023 01:58:33 +0800
Subject: [PATCH 67/98] move storage

---
 dbsyncer-web/src/main/resources/application.properties | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/dbsyncer-web/src/main/resources/application.properties b/dbsyncer-web/src/main/resources/application.properties
index d8640266..7f4b534d 100644
--- a/dbsyncer-web/src/main/resources/application.properties
+++ b/dbsyncer-web/src/main/resources/application.properties
@@ -48,9 +48,9 @@ dbsyncer.parser.table.group.buffer-queue-capacity=40000
 dbsyncer.parser.table.group.buffer-period-millisecond=300
 
 #storage
-# 是否使用MySQL存储配置类型（MySQL）
+# 存储配置类型（MySQL）
 # false: 保存磁盘/data/config(驱动配置)|data(按驱动分别存储增量数据)|log(系统日志)}
-dbsyncer.storage.type=MySQL
+dbsyncer.storage.type=none
 dbsyncer.storage.mysql.config.url=jdbc:mysql://127.0.0.1:3306/dbsyncer?rewriteBatchedStatements=true&seUnicode=true&characterEncoding=UTF8&serverTimezone=Asia/Shanghai&useSSL=false&verifyServerCertificate=false&autoReconnect=true
 dbsyncer.storage.mysql.config.username=root
 dbsyncer.storage.mysql.config.password=123
-- 
Gitee


From 33f4d27606952280f636ef675706b11299ce72d0 Mon Sep 17 00:00:00 2001
From: AE86 <836391306@qq.com>
Date: Wed, 13 Dec 2023 01:04:38 +0800
Subject: [PATCH 68/98] move storage

---
 .../org/dbsyncer/common/util/StringUtil.java  |  5 ++
 .../connector/ConnectorConfiguration.java     | 65 ----------------
 .../dbsyncer/connector/ConnectorFactory.java  | 32 ++++----
 .../connector/mysql/MySQLConnector.java       |  7 ++
 ...Provider.java => MySQLStorageService.java} | 55 ++++++++------
 .../org.dbsyncer.sdk.spi.StorageService       |  1 -
 ...r_config.sql => dbsyncer_mysql_config.sql} |  0
 ...yncer_data.sql => dbsyncer_mysql_data.sql} |  0
 ...bsyncer_log.sql => dbsyncer_mysql_log.sql} |  0
 .../manager/impl/PreloadTemplate.java         |  5 +-
 .../dbsyncer/sdk/spi/ConnectorService.java    | 10 +++
 .../org/dbsyncer/sdk/spi/StorageService.java  |  6 ++
 .../storage/StorageSupportConfiguration.java  | 74 ++++++++++++++++++-
 ...rviceImpl.java => DiskStorageService.java} | 23 +++---
 .../src/main/resources/application.properties | 10 +--
 15 files changed, 167 insertions(+), 126 deletions(-)
 delete mode 100644 dbsyncer-connector/dbsyncer-connector-base/src/main/java/org/dbsyncer/connector/ConnectorConfiguration.java
 rename dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/storage/{MySQLStorageServiceProvider.java => MySQLStorageService.java} (92%)
 delete mode 100644 dbsyncer-connector/dbsyncer-connector-mysql/src/main/resources/META-INF/services/org.dbsyncer.sdk.spi.StorageService
 rename dbsyncer-connector/dbsyncer-connector-mysql/src/main/resources/{dbsyncer_config.sql => dbsyncer_mysql_config.sql} (100%)
 rename dbsyncer-connector/dbsyncer-connector-mysql/src/main/resources/{dbsyncer_data.sql => dbsyncer_mysql_data.sql} (100%)
 rename dbsyncer-connector/dbsyncer-connector-mysql/src/main/resources/{dbsyncer_log.sql => dbsyncer_mysql_log.sql} (100%)
 rename dbsyncer-storage/src/main/java/org/dbsyncer/storage/impl/{DiskStorageServiceImpl.java => DiskStorageService.java} (96%)

diff --git a/dbsyncer-common/src/main/java/org/dbsyncer/common/util/StringUtil.java b/dbsyncer-common/src/main/java/org/dbsyncer/common/util/StringUtil.java
index 30b5f6f2..0ad555d3 100644
--- a/dbsyncer-common/src/main/java/org/dbsyncer/common/util/StringUtil.java
+++ b/dbsyncer-common/src/main/java/org/dbsyncer/common/util/StringUtil.java
@@ -1,3 +1,6 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
 package org.dbsyncer.common.util;
 
 import org.apache.commons.lang3.StringUtils;
@@ -8,6 +11,8 @@ public abstract class StringUtil {
 
     public static final String SYMBOL = "-";
 
+    public static final String UNDERLINE = "_";
+
     public static final String COLON = ":";
 
     public static final String SPACE = " ";
diff --git a/dbsyncer-connector/dbsyncer-connector-base/src/main/java/org/dbsyncer/connector/ConnectorConfiguration.java b/dbsyncer-connector/dbsyncer-connector-base/src/main/java/org/dbsyncer/connector/ConnectorConfiguration.java
deleted file mode 100644
index 7f8ddb8d..00000000
--- a/dbsyncer-connector/dbsyncer-connector-base/src/main/java/org/dbsyncer/connector/ConnectorConfiguration.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/**
- * DBSyncer Copyright 2020-2023 All Rights Reserved.
- */
-package org.dbsyncer.connector;
-
-import org.dbsyncer.sdk.spi.ConnectorService;
-import org.dbsyncer.sdk.spi.StorageService;
-import org.springframework.beans.BeansException;
-import org.springframework.beans.factory.config.BeanDefinition;
-import org.springframework.beans.factory.config.ConfigurableListableBeanFactory;
-import org.springframework.beans.factory.support.BeanDefinitionBuilder;
-import org.springframework.beans.factory.support.BeanDefinitionRegistry;
-import org.springframework.beans.factory.support.BeanDefinitionRegistryPostProcessor;
-import org.springframework.stereotype.Service;
-import org.springframework.util.Assert;
-import org.springframework.util.ClassUtils;
-
-import java.beans.Introspector;
-import java.util.ServiceLoader;
-
-/**
- * 连接器配置
- *
- * @Author AE86
- * @Version 1.0.0
- * @Date 2019-09-19 23:17
- */
-@Service
-public class ConnectorConfiguration implements BeanDefinitionRegistryPostProcessor {
-
-    @Override
-    public void postProcessBeanDefinitionRegistry(BeanDefinitionRegistry beanDefinitionRegistry) throws BeansException {
-        loadConnectorServices(beanDefinitionRegistry);
-//        loadStorageServices(beanDefinitionRegistry);
-    }
-
-    private void loadStorageServices(BeanDefinitionRegistry beanDefinitionRegistry) {
-        ServiceLoader services = ServiceLoader.load(StorageService.class, Thread.currentThread().getContextClassLoader());
-        for (StorageService s : services) {
-            BeanDefinition beanDefinition = BeanDefinitionBuilder.genericBeanDefinition(s.getClass()).getBeanDefinition();
-            String beanClassName = beanDefinition.getBeanClassName();
-            Assert.state(beanClassName != null, "No bean class name set");
-            String shortClassName = ClassUtils.getShortName(beanClassName);
-            String decapitalize = Introspector.decapitalize(shortClassName);
-            beanDefinitionRegistry.registerBeanDefinition(decapitalize, beanDefinition);
-        }
-    }
-
-    private void loadConnectorServices(BeanDefinitionRegistry beanDefinitionRegistry) {
-        ServiceLoader services = ServiceLoader.load(ConnectorService.class, Thread.currentThread().getContextClassLoader());
-        for (ConnectorService s : services) {
-            BeanDefinition beanDefinition = BeanDefinitionBuilder.genericBeanDefinition(s.getClass()).getBeanDefinition();
-            String beanClassName = beanDefinition.getBeanClassName();
-            Assert.state(beanClassName != null, "No bean class name set");
-            String shortClassName = ClassUtils.getShortName(beanClassName);
-            String decapitalize = Introspector.decapitalize(shortClassName);
-            beanDefinitionRegistry.registerBeanDefinition(decapitalize, beanDefinition);
-        }
-    }
-
-    @Override
-    public void postProcessBeanFactory(ConfigurableListableBeanFactory configurableListableBeanFactory) throws BeansException {
-
-    }
-}
diff --git a/dbsyncer-connector/dbsyncer-connector-base/src/main/java/org/dbsyncer/connector/ConnectorFactory.java b/dbsyncer-connector/dbsyncer-connector-base/src/main/java/org/dbsyncer/connector/ConnectorFactory.java
index f24dc935..d861aa81 100644
--- a/dbsyncer-connector/dbsyncer-connector-base/src/main/java/org/dbsyncer/connector/ConnectorFactory.java
+++ b/dbsyncer-connector/dbsyncer-connector-base/src/main/java/org/dbsyncer/connector/ConnectorFactory.java
@@ -16,17 +16,19 @@ import org.dbsyncer.sdk.model.ConnectorConfig;
 import org.dbsyncer.sdk.model.MetaInfo;
 import org.dbsyncer.sdk.model.Table;
 import org.dbsyncer.sdk.spi.ConnectorService;
+import org.springframework.beans.BeansException;
 import org.springframework.beans.factory.DisposableBean;
-import org.springframework.context.ApplicationContext;
+import org.springframework.beans.factory.config.ConfigurableListableBeanFactory;
+import org.springframework.beans.factory.support.BeanDefinitionRegistry;
+import org.springframework.beans.factory.support.BeanDefinitionRegistryPostProcessor;
 import org.springframework.stereotype.Component;
 import org.springframework.util.Assert;
 
-import javax.annotation.PostConstruct;
-import javax.annotation.Resource;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.ServiceLoader;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 
@@ -38,7 +40,7 @@ import java.util.concurrent.ConcurrentHashMap;
  * @Date 2019-09-18 23:30
  */
 @Component
-public class ConnectorFactory implements DisposableBean {
+public class ConnectorFactory implements BeanDefinitionRegistryPostProcessor, DisposableBean {
 
     private final Map pool = new ConcurrentHashMap<>();
 
@@ -46,20 +48,20 @@ public class ConnectorFactory implements DisposableBean {
 
     private final Set connectorTypes = new HashSet<>();
 
-    @Resource
-    private ApplicationContext applicationContext;
-
-    @PostConstruct
-    private void init() {
-        Map beans = applicationContext.getBeansOfType(ConnectorService.class);
-        if (!CollectionUtils.isEmpty(beans)) {
-            beans.values().forEach(s -> {
-                service.putIfAbsent(s.getConnectorType(), s);
-                connectorTypes.add(s.getConnectorType());
-            });
+    @Override
+    public void postProcessBeanDefinitionRegistry(BeanDefinitionRegistry beanDefinitionRegistry) throws BeansException {
+        ServiceLoader services = ServiceLoader.load(ConnectorService.class, Thread.currentThread().getContextClassLoader());
+        for (ConnectorService s : services) {
+            service.putIfAbsent(s.getConnectorType(), s);
+            connectorTypes.add(s.getConnectorType());
         }
     }
 
+    @Override
+    public void postProcessBeanFactory(ConfigurableListableBeanFactory configurableListableBeanFactory) throws BeansException {
+
+    }
+
     @Override
     public void destroy() {
         pool.values().forEach(this::disconnect);
diff --git a/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/MySQLConnector.java b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/MySQLConnector.java
index 3e6471d9..30283c48 100644
--- a/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/MySQLConnector.java
+++ b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/MySQLConnector.java
@@ -5,6 +5,7 @@ package org.dbsyncer.connector.mysql;
 
 import org.dbsyncer.common.util.StringUtil;
 import org.dbsyncer.connector.mysql.cdc.MySQLListener;
+import org.dbsyncer.connector.mysql.storage.MySQLStorageService;
 import org.dbsyncer.connector.mysql.validator.MySQLConfigValidator;
 import org.dbsyncer.sdk.connector.ConfigValidator;
 import org.dbsyncer.sdk.listener.DatabaseQuartzListener;
@@ -14,6 +15,7 @@ import org.dbsyncer.sdk.constant.DatabaseConstant;
 import org.dbsyncer.sdk.enums.ListenerTypeEnum;
 import org.dbsyncer.sdk.listener.Listener;
 import org.dbsyncer.sdk.model.PageSql;
+import org.dbsyncer.sdk.spi.StorageService;
 import org.dbsyncer.sdk.util.PrimaryKeyUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -56,6 +58,11 @@ public final class MySQLConnector extends AbstractDatabaseConnector {
         return null;
     }
 
+    @Override
+    public StorageService getStorageService() {
+        return new MySQLStorageService();
+    }
+
     @Override
     public String generateUniqueCode() {
         return DatabaseConstant.DBS_UNIQUE_CODE;
diff --git a/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/storage/MySQLStorageServiceProvider.java b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/storage/MySQLStorageService.java
similarity index 92%
rename from dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/storage/MySQLStorageServiceProvider.java
rename to dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/storage/MySQLStorageService.java
index e74f0986..3632325d 100644
--- a/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/storage/MySQLStorageServiceProvider.java
+++ b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/storage/MySQLStorageService.java
@@ -26,13 +26,9 @@ import org.dbsyncer.sdk.storage.AbstractStorageService;
 import org.dbsyncer.sdk.util.DatabaseUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
-import org.springframework.boot.context.properties.ConfigurationProperties;
 import org.springframework.dao.EmptyResultDataAccessException;
-import org.springframework.stereotype.Component;
 import org.springframework.util.Assert;
 
-import javax.annotation.PostConstruct;
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
@@ -41,6 +37,7 @@ import java.sql.Types;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import java.util.Properties;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -54,10 +51,7 @@ import java.util.stream.Stream;
  * @Version 1.0.0
  * @Date 2020-01-08 15:17
  */
-@Component
-@ConditionalOnProperty(value = "dbsyncer.storage.type", havingValue = "MySQL")
-@ConfigurationProperties(prefix = "dbsyncer.storage.mysql")
-public class MySQLStorageServiceProvider extends AbstractStorageService {
+public class MySQLStorageService extends AbstractStorageService {
 
     private final Logger logger = LoggerFactory.getLogger(getClass());
 
@@ -65,25 +59,29 @@ public class MySQLStorageServiceProvider extends AbstractStorageService {
     private final String SHOW_TABLE = "show tables where Tables_in_%s = '%s'";
     private final String DROP_TABLE = "DROP TABLE %s";
     private final String TRUNCATE_TABLE = "TRUNCATE TABLE %s";
+    private final MySQLConnector connector = new MySQLConnector();
     private Map tables = new ConcurrentHashMap<>();
-    private MySQLConnector connector = new MySQLConnector();
     private DatabaseConnectorInstance connectorInstance;
-    private DatabaseConfig config;
     private String database;
 
-    @PostConstruct
-    private void init() throws InterruptedException {
+    @Override
+    public void init(Properties properties) {
+        DatabaseConfig config = new DatabaseConfig();
+        config.setConnectorType(properties.getProperty("dbsyncer.storage.type"));
+        config.setUrl(properties.getProperty("dbsyncer.storage.mysql.url", "jdbc:mysql://127.0.0.1:3306/dbsyncer?rewriteBatchedStatements=true&seUnicode=true&characterEncoding=UTF8&serverTimezone=Asia/Shanghai&useSSL=false&verifyServerCertificate=false&autoReconnect=true"));
+        config.setUsername(properties.getProperty("dbsyncer.storage.mysql.username", "admin"));
+        config.setPassword(properties.getProperty("dbsyncer.storage.mysql.password", "admin"));
+        config.setDriverClassName(properties.getProperty("dbsyncer.storage.mysql.driver-class-name"));
         logger.info("url:{}", config.getUrl());
-        config.setConnectorType(connector.getConnectorType());
         database = DatabaseUtil.getDatabaseName(config.getUrl());
-        connectorInstance = (DatabaseConnectorInstance) connector.connect(config);
+        connectorInstance = new DatabaseConnectorInstance(config);
         // 初始化表
         initTable();
     }
 
     @Override
     protected String getSeparator() {
-        return "_";
+        return StringUtil.UNDERLINE;
     }
 
     @Override
@@ -338,7 +336,7 @@ public class MySQLStorageServiceProvider extends AbstractStorageService {
         }
     }
 
-    private void initTable() throws InterruptedException {
+    private void initTable() {
         // 配置
         FieldBuilder builder = new FieldBuilder();
         builder.build(ConfigConstant.CONFIG_MODEL_ID, ConfigConstant.CONFIG_MODEL_NAME, ConfigConstant.CONFIG_MODEL_TYPE, ConfigConstant.CONFIG_MODEL_CREATE_TIME, ConfigConstant.CONFIG_MODEL_UPDATE_TIME, ConfigConstant.CONFIG_MODEL_JSON);
@@ -363,7 +361,11 @@ public class MySQLStorageServiceProvider extends AbstractStorageService {
         });
 
         // wait few seconds for execute sql
-        TimeUnit.SECONDS.sleep(1);
+        try {
+            TimeUnit.SECONDS.sleep(1);
+        } catch (InterruptedException e) {
+            logger.error(e.getMessage(), e);
+        }
     }
 
     private Executor createTableIfNotExist(String table, Executor executor) {
@@ -392,9 +394,7 @@ public class MySQLStorageServiceProvider extends AbstractStorageService {
     }
 
     private String readSql(String type, boolean systemTable, String table) {
-        String template = PREFIX_TABLE.concat(type);
-        String filePath = "/".concat(template).concat(".sql");
-
+        String filePath = getSqlFilePath(type);
         StringBuilder res = new StringBuilder();
         InputStream in = null;
         InputStreamReader isr = null;
@@ -417,11 +417,22 @@ public class MySQLStorageServiceProvider extends AbstractStorageService {
 
         // 动态替换表名
         if (!systemTable) {
+            String template = PREFIX_TABLE.concat(type);
             return StringUtil.replace(res.toString(), template, table);
         }
         return res.toString();
     }
 
+    /**
+     * 获取sql脚本路径
+     *
+     * @param type
+     * @return /dbsyncer_mysql_config.sql
+     */
+    private String getSqlFilePath(String type) {
+        return new StringBuilder(StringUtil.FORWARD_SLASH).append(PREFIX_TABLE).append(connector.getConnectorType().toLowerCase()).append(StringUtil.UNDERLINE).append(type).append(".sql").toString();
+    }
+
     private void executeSql(String ddl) {
         connectorInstance.execute(databaseTemplate -> {
             databaseTemplate.execute(ddl);
@@ -443,10 +454,6 @@ public class MySQLStorageServiceProvider extends AbstractStorageService {
         }
     }
 
-    public void setConfig(DatabaseConfig config) {
-        this.config = config;
-    }
-
     final class FieldBuilder {
         Map fieldMap;
         List fields;
diff --git a/dbsyncer-connector/dbsyncer-connector-mysql/src/main/resources/META-INF/services/org.dbsyncer.sdk.spi.StorageService b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/resources/META-INF/services/org.dbsyncer.sdk.spi.StorageService
deleted file mode 100644
index 049e45c2..00000000
--- a/dbsyncer-connector/dbsyncer-connector-mysql/src/main/resources/META-INF/services/org.dbsyncer.sdk.spi.StorageService
+++ /dev/null
@@ -1 +0,0 @@
-org.dbsyncer.connector.mysql.storage.MySQLStorageServiceProvider
\ No newline at end of file
diff --git a/dbsyncer-connector/dbsyncer-connector-mysql/src/main/resources/dbsyncer_config.sql b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/resources/dbsyncer_mysql_config.sql
similarity index 100%
rename from dbsyncer-connector/dbsyncer-connector-mysql/src/main/resources/dbsyncer_config.sql
rename to dbsyncer-connector/dbsyncer-connector-mysql/src/main/resources/dbsyncer_mysql_config.sql
diff --git a/dbsyncer-connector/dbsyncer-connector-mysql/src/main/resources/dbsyncer_data.sql b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/resources/dbsyncer_mysql_data.sql
similarity index 100%
rename from dbsyncer-connector/dbsyncer-connector-mysql/src/main/resources/dbsyncer_data.sql
rename to dbsyncer-connector/dbsyncer-connector-mysql/src/main/resources/dbsyncer_mysql_data.sql
diff --git a/dbsyncer-connector/dbsyncer-connector-mysql/src/main/resources/dbsyncer_log.sql b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/resources/dbsyncer_mysql_log.sql
similarity index 100%
rename from dbsyncer-connector/dbsyncer-connector-mysql/src/main/resources/dbsyncer_log.sql
rename to dbsyncer-connector/dbsyncer-connector-mysql/src/main/resources/dbsyncer_mysql_log.sql
diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/PreloadTemplate.java b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/PreloadTemplate.java
index 73ccc9c1..dca14652 100644
--- a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/PreloadTemplate.java
+++ b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/PreloadTemplate.java
@@ -1,3 +1,6 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
 package org.dbsyncer.manager.impl;
 
 import org.dbsyncer.common.model.Paging;
@@ -20,10 +23,10 @@ import org.dbsyncer.parser.model.Meta;
 import org.dbsyncer.parser.model.OperationConfig;
 import org.dbsyncer.plugin.PluginFactory;
 import org.dbsyncer.sdk.connector.ConnectorInstance;
+import org.dbsyncer.sdk.constant.ConfigConstant;
 import org.dbsyncer.sdk.enums.StorageEnum;
 import org.dbsyncer.sdk.filter.Query;
 import org.dbsyncer.sdk.spi.StorageService;
-import org.dbsyncer.sdk.constant.ConfigConstant;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.context.ApplicationListener;
diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/ConnectorService.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/ConnectorService.java
index ed301f31..96caa617 100644
--- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/ConnectorService.java
+++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/ConnectorService.java
@@ -173,4 +173,14 @@ public interface ConnectorService propertySource : propertySources) {
+                boolean applicationConfig = propertySource.getName().contains("application");
+                if (!applicationConfig) {
+                    continue;
+                }
+                Map props = (Map) propertySource.getSource();
+                props.forEach((k, v) -> {
+                    if (StringUtil.startsWith(k, PREFIX_STORAGE)) {
+                        properties.put(k, v.getValue());
+                    }
+                });
+            }
+        }
+
+        // 指定存储类型
+        String storageType = properties.getProperty(STORAGE_TYPE);
+        if (StringUtil.isNotBlank(storageType)) {
+            String connectorType = getConnectorType(storageType);
+            if (StringUtil.isNotBlank(connectorType)) {
+                StorageService storageService = connectorFactory.getConnectorService(connectorType).getStorageService();
+                if (storageService != null) {
+                    storageService.init(properties);
+                    return storageService;
+                }
+            }
+        }
+
+        // 默认磁盘存储
+        DiskStorageService storageService = new DiskStorageService();
+        storageService.init(properties);
+        return storageService;
+    }
+
+    private String getConnectorType(String storageType) {
+        Iterator iterator = connectorFactory.getConnectorTypeAll().iterator();
+        while (iterator.hasNext()) {
+            String connectorType = iterator.next();
+            if (StringUtil.equalsIgnoreCase(storageType, connectorType)) {
+                return connectorType;
+            }
+        }
+        return null;
     }
 
 }
\ No newline at end of file
diff --git a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/impl/DiskStorageServiceImpl.java b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/impl/DiskStorageService.java
similarity index 96%
rename from dbsyncer-storage/src/main/java/org/dbsyncer/storage/impl/DiskStorageServiceImpl.java
rename to dbsyncer-storage/src/main/java/org/dbsyncer/storage/impl/DiskStorageService.java
index 9ceaddca..fc7f16d8 100644
--- a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/impl/DiskStorageServiceImpl.java
+++ b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/impl/DiskStorageService.java
@@ -3,7 +3,6 @@
  */
 package org.dbsyncer.storage.impl;
 
-import org.apache.commons.io.FileUtils;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.BooleanClause;
@@ -16,26 +15,26 @@ import org.apache.lucene.search.highlight.QueryScorer;
 import org.apache.lucene.search.highlight.SimpleHTMLFormatter;
 import org.dbsyncer.common.model.Paging;
 import org.dbsyncer.common.util.CollectionUtils;
+import org.dbsyncer.sdk.constant.ConfigConstant;
 import org.dbsyncer.sdk.enums.FilterEnum;
 import org.dbsyncer.sdk.enums.OperationEnum;
-import org.dbsyncer.sdk.storage.AbstractStorageService;
-import org.dbsyncer.storage.StorageException;
-import org.dbsyncer.sdk.constant.ConfigConstant;
 import org.dbsyncer.sdk.enums.StorageEnum;
-import org.dbsyncer.storage.lucene.Option;
-import org.dbsyncer.storage.lucene.Shard;
 import org.dbsyncer.sdk.filter.AbstractFilter;
 import org.dbsyncer.sdk.filter.BooleanFilter;
 import org.dbsyncer.sdk.filter.Query;
+import org.dbsyncer.sdk.storage.AbstractStorageService;
+import org.dbsyncer.storage.StorageException;
+import org.dbsyncer.storage.lucene.Option;
+import org.dbsyncer.storage.lucene.Shard;
 import org.dbsyncer.storage.util.DocumentUtil;
 
-import javax.annotation.PostConstruct;
 import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Properties;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 
@@ -46,7 +45,7 @@ import java.util.concurrent.ConcurrentHashMap;
  * @Version 1.0.0
  * @Date 2023-09-10 23:22
  */
-public class DiskStorageServiceImpl extends AbstractStorageService {
+public class DiskStorageService extends AbstractStorageService {
 
     private Map shards = new ConcurrentHashMap();
 
@@ -56,10 +55,8 @@ public class DiskStorageServiceImpl extends AbstractStorageService {
     private static final String PATH = new StringBuilder(System.getProperty("user.dir")).append(File.separatorChar).append("data")
             .append(File.separatorChar).toString();
 
-    @PostConstruct
-    private void init() {
-        // 废弃binlog
-        FileUtils.deleteQuietly(new File(PATH + "binlog"));
+    @Override
+    public void init(Properties properties) {
         // 创建配置和日志索引shard
         getShard(getSharding(StorageEnum.CONFIG, null));
         getShard(getSharding(StorageEnum.LOG, null));
@@ -166,7 +163,7 @@ public class DiskStorageServiceImpl extends AbstractStorageService {
         filters.forEach(p -> {
             FilterEnum filterEnum = FilterEnum.getFilterEnum(p.getFilter());
             BooleanClause.Occur occur = getOccur(p.getOperation());
-             switch (filterEnum) {
+            switch (filterEnum) {
                 case EQUAL:
                 case LIKE:
                     builder.add(DiskQueryHelper.newEqual(p), occur);
diff --git a/dbsyncer-web/src/main/resources/application.properties b/dbsyncer-web/src/main/resources/application.properties
index 7f4b534d..eb4b2040 100644
--- a/dbsyncer-web/src/main/resources/application.properties
+++ b/dbsyncer-web/src/main/resources/application.properties
@@ -48,12 +48,12 @@ dbsyncer.parser.table.group.buffer-queue-capacity=40000
 dbsyncer.parser.table.group.buffer-period-millisecond=300
 
 #storage
-# 存储配置类型（MySQL）
+# 数据存储类型:disk(默认)/mysql(推荐生产环境使用)
 # false: 保存磁盘/data/config(驱动配置)|data(按驱动分别存储增量数据)|log(系统日志)}
-dbsyncer.storage.type=none
-dbsyncer.storage.mysql.config.url=jdbc:mysql://127.0.0.1:3306/dbsyncer?rewriteBatchedStatements=true&seUnicode=true&characterEncoding=UTF8&serverTimezone=Asia/Shanghai&useSSL=false&verifyServerCertificate=false&autoReconnect=true
-dbsyncer.storage.mysql.config.username=root
-dbsyncer.storage.mysql.config.password=123
+dbsyncer.storage.type=disk
+dbsyncer.storage.mysql.url=jdbc:mysql://127.0.0.1:3306/dbsyncer?rewriteBatchedStatements=true&seUnicode=true&characterEncoding=UTF8&serverTimezone=Asia/Shanghai&useSSL=false&verifyServerCertificate=false&autoReconnect=true
+dbsyncer.storage.mysql.username=root
+dbsyncer.storage.mysql.password=123
 # [StorageBufferActuator]线程数
 dbsyncer.storage.thread-core-size=4
 # [StorageBufferActuator]最大线程数
-- 
Gitee


From 1b3c7334a4d7ed27dc60ab4b658ec5a974192db4 Mon Sep 17 00:00:00 2001
From: AE86 <836391306@qq.com>
Date: Wed, 13 Dec 2023 01:41:58 +0800
Subject: [PATCH 69/98] fixbug

---
 .../database/AbstractDatabaseConnector.java          | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/AbstractDatabaseConnector.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/AbstractDatabaseConnector.java
index a6ec57d7..80350955 100644
--- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/AbstractDatabaseConnector.java
+++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/database/AbstractDatabaseConnector.java
@@ -19,6 +19,7 @@ import org.dbsyncer.sdk.connector.database.ds.SimpleConnection;
 import org.dbsyncer.sdk.constant.ConnectorConstant;
 import org.dbsyncer.sdk.constant.DatabaseConstant;
 import org.dbsyncer.sdk.enums.OperationEnum;
+import org.dbsyncer.sdk.enums.QuartzFilterEnum;
 import org.dbsyncer.sdk.enums.SqlBuilderEnum;
 import org.dbsyncer.sdk.enums.TableTypeEnum;
 import org.dbsyncer.sdk.model.Field;
@@ -522,10 +523,13 @@ public abstract class AbstractDatabaseConnector extends AbstractConnector implem
      */
     protected String buildFilterValue(String value) {
         if (StringUtil.isNotBlank(value)) {
-            // 系统函数表达式 $select max(update_time)$
-            Matcher matcher = Pattern.compile(SYS_EXPRESSION).matcher(value);
-            if (matcher.find()) {
-                return StringUtil.substring(value, 1, value.length() - 1);
+            // 排除定时表达式
+            if (QuartzFilterEnum.getQuartzFilterEnum(value) == null) {
+                // 系统函数表达式 $select max(update_time)$
+                Matcher matcher = Pattern.compile(SYS_EXPRESSION).matcher(value);
+                if (matcher.find()) {
+                    return StringUtil.substring(value, 1, value.length() - 1);
+                }
             }
         }
         return new StringBuilder(StringUtil.SINGLE_QUOTATION).append(value).append(StringUtil.SINGLE_QUOTATION).toString();
-- 
Gitee


From 44143847d39f683e7e370ea3d7c51b8b6d164474 Mon Sep 17 00:00:00 2001
From: AE86 <836391306@qq.com>
Date: Wed, 13 Dec 2023 02:31:20 +0800
Subject: [PATCH 70/98] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=90=8C=E6=AD=A5?=
 =?UTF-8?q?=E6=95=B0=E6=8D=AE=E5=BC=80=E5=85=B3=E5=88=B0=E7=B3=BB=E7=BB=9F?=
 =?UTF-8?q?=E9=85=8D=E7=BD=AE=E9=A1=B5=E9=9D=A2?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../impl/system/SystemConfigChecker.java      |  3 +
 .../dbsyncer/common/config/StorageConfig.java | 41 +------------
 .../parser/ParserStrategyConfiguration.java   | 23 --------
 .../parser/flush/impl/FlushServiceImpl.java   | 25 ++++----
 .../FlushStrategyImpl.java}                   | 52 ++++++++++++++---
 .../flush/impl/GeneralBufferActuator.java     |  3 +
 .../dbsyncer/parser/model/SystemConfig.java   | 57 ++++++++++++++++++-
 .../impl/DisableFullFlushStrategy.java        | 39 -------------
 .../strategy/impl/EnableFlushStrategy.java    | 18 ------
 .../src/main/resources/application.properties |  8 ---
 .../main/resources/public/system/system.html  | 26 ++++++++-
 .../main/resources/static/js/system/index.js  |  2 +-
 12 files changed, 149 insertions(+), 148 deletions(-)
 delete mode 100644 dbsyncer-parser/src/main/java/org/dbsyncer/parser/ParserStrategyConfiguration.java
 rename dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/{AbstractFlushStrategy.java => impl/FlushStrategyImpl.java} (50%)
 delete mode 100644 dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/impl/DisableFullFlushStrategy.java
 delete mode 100644 dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/impl/EnableFlushStrategy.java

diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/system/SystemConfigChecker.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/system/SystemConfigChecker.java
index 527cd478..4bdfd072 100644
--- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/system/SystemConfigChecker.java
+++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/system/SystemConfigChecker.java
@@ -49,6 +49,9 @@ public class SystemConfigChecker extends AbstractChecker {
         logger.info("params:{}", params);
         Assert.notEmpty(params, "Config check params is null.");
         params.put("enableCDN", StringUtil.isNotBlank(params.get("enableCDN")) ? "true" : "false");
+        params.put("enableStorageWriteFull", StringUtil.isNotBlank(params.get("enableStorageWriteFull")) ? "true" : "false");
+        params.put("enableStorageWriteSuccess", StringUtil.isNotBlank(params.get("enableStorageWriteSuccess")) ? "true" : "false");
+        params.put("enableStorageWriteFail", StringUtil.isNotBlank(params.get("enableStorageWriteFail")) ? "true" : "false");
 
         SystemConfig systemConfig = profileComponent.getSystemConfig();
         Assert.notNull(systemConfig, "配置文件为空.");
diff --git a/dbsyncer-common/src/main/java/org/dbsyncer/common/config/StorageConfig.java b/dbsyncer-common/src/main/java/org/dbsyncer/common/config/StorageConfig.java
index bc41f668..1a815c61 100644
--- a/dbsyncer-common/src/main/java/org/dbsyncer/common/config/StorageConfig.java
+++ b/dbsyncer-common/src/main/java/org/dbsyncer/common/config/StorageConfig.java
@@ -1,3 +1,6 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
 package org.dbsyncer.common.config;
 
 import org.dbsyncer.common.util.ThreadPoolUtil;
@@ -32,21 +35,6 @@ public class StorageConfig extends BufferActuatorConfig {
      */
     private int threadQueueCapacity = 500;
 
-    /**
-     * 是否记录同步成功数据
-     */
-    private boolean writeSuccess;
-
-    /**
-     * 是否记录同步失败数据
-     */
-    private boolean writeFail;
-
-    /**
-     * 最大记录异常信息长度
-     */
-    private int maxErrorLength;
-
     @Bean(name = "storageExecutor", destroyMethod = "shutdown")
     public ThreadPoolTaskExecutor storageExecutor() {
         return ThreadPoolUtil.newThreadPoolTaskExecutor(threadCoreSize, maxThreadSize, threadQueueCapacity, 30, "StorageExecutor-");
@@ -76,27 +64,4 @@ public class StorageConfig extends BufferActuatorConfig {
         this.threadQueueCapacity = threadQueueCapacity;
     }
 
-    public boolean isWriteSuccess() {
-        return writeSuccess;
-    }
-
-    public void setWriteSuccess(boolean writeSuccess) {
-        this.writeSuccess = writeSuccess;
-    }
-
-    public boolean isWriteFail() {
-        return writeFail;
-    }
-
-    public void setWriteFail(boolean writeFail) {
-        this.writeFail = writeFail;
-    }
-
-    public int getMaxErrorLength() {
-        return maxErrorLength;
-    }
-
-    public void setMaxErrorLength(int maxErrorLength) {
-        this.maxErrorLength = maxErrorLength;
-    }
 }
\ No newline at end of file
diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ParserStrategyConfiguration.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ParserStrategyConfiguration.java
deleted file mode 100644
index fa06133f..00000000
--- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ParserStrategyConfiguration.java
+++ /dev/null
@@ -1,23 +0,0 @@
-package org.dbsyncer.parser;
-
-import org.dbsyncer.parser.strategy.FlushStrategy;
-import org.dbsyncer.parser.strategy.impl.DisableFullFlushStrategy;
-import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Configuration;
-
-/**
- * @author AE86
- * @version 1.0.0
- * @date 2021/11/18 21:36
- */
-@Configuration
-public class ParserStrategyConfiguration {
-
-    @Bean
-    @ConditionalOnMissingBean
-    public FlushStrategy flushStrategy() {
-        return new DisableFullFlushStrategy();
-    }
-
-}
\ No newline at end of file
diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/FlushServiceImpl.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/FlushServiceImpl.java
index c49520ea..9be498b5 100644
--- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/FlushServiceImpl.java
+++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/FlushServiceImpl.java
@@ -1,13 +1,17 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
 package org.dbsyncer.parser.flush.impl;
 
-import org.dbsyncer.common.config.StorageConfig;
 import org.dbsyncer.common.util.StringUtil;
+import org.dbsyncer.parser.ProfileComponent;
 import org.dbsyncer.parser.flush.BufferActuator;
 import org.dbsyncer.parser.flush.FlushService;
 import org.dbsyncer.parser.model.StorageRequest;
+import org.dbsyncer.parser.model.SystemConfig;
+import org.dbsyncer.sdk.constant.ConfigConstant;
 import org.dbsyncer.sdk.enums.StorageEnum;
 import org.dbsyncer.sdk.spi.StorageService;
-import org.dbsyncer.sdk.constant.ConfigConstant;
 import org.dbsyncer.storage.enums.StorageDataStatusEnum;
 import org.dbsyncer.storage.impl.SnowflakeIdWorker;
 import org.dbsyncer.storage.util.BinlogMessageUtil;
@@ -46,10 +50,10 @@ public class FlushServiceImpl implements FlushService {
     private BufferActuator storageBufferActuator;
 
     @Resource
-    private StorageConfig storageConfig;
+    private Executor storageExecutor;
 
     @Resource
-    private Executor storageExecutor;
+    private ProfileComponent profileComponent;
 
     @Override
     public void asyncWrite(String type, String error) {
@@ -57,7 +61,7 @@ public class FlushServiceImpl implements FlushService {
             Map params = new HashMap();
             params.put(ConfigConstant.CONFIG_MODEL_ID, String.valueOf(snowflakeIdWorker.nextId()));
             params.put(ConfigConstant.CONFIG_MODEL_TYPE, type);
-            params.put(ConfigConstant.CONFIG_MODEL_JSON, substring(error));
+            params.put(ConfigConstant.CONFIG_MODEL_JSON, StringUtil.substring(error, 0, getSystemConfig().getMaxStorageErrorLength()));
             params.put(ConfigConstant.CONFIG_MODEL_CREATE_TIME, Instant.now().toEpochMilli());
             storageService.add(StorageEnum.LOG, params);
         });
@@ -73,7 +77,7 @@ public class FlushServiceImpl implements FlushService {
             row.put(ConfigConstant.DATA_TABLE_GROUP_ID, tableGroupId);
             row.put(ConfigConstant.DATA_TARGET_TABLE_NAME, targetTableGroupName);
             row.put(ConfigConstant.DATA_EVENT, event);
-            row.put(ConfigConstant.DATA_ERROR, substring(error));
+            row.put(ConfigConstant.DATA_ERROR, error);
             row.put(ConfigConstant.CONFIG_MODEL_CREATE_TIME, now);
             try {
                 byte[] bytes = BinlogMessageUtil.toBinlogMap(r).toByteArray();
@@ -81,19 +85,16 @@ public class FlushServiceImpl implements FlushService {
             } catch (Exception e) {
                 logger.warn("可能存在Blob或inputStream大文件类型, 无法序列化:{}", r);
             }
-
             storageBufferActuator.offer(new StorageRequest(metaId, row));
         });
     }
 
     /**
-     * 限制记录异常信息长度
+     * TODO 加缓存过期
      *
-     * @param error
      * @return
      */
-    private String substring(String error) {
-        return StringUtil.substring(error, 0, storageConfig.getMaxErrorLength());
+    private SystemConfig getSystemConfig() {
+        return profileComponent.getSystemConfig();
     }
-
 }
\ No newline at end of file
diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/AbstractFlushStrategy.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/FlushStrategyImpl.java
similarity index 50%
rename from dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/AbstractFlushStrategy.java
rename to dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/FlushStrategyImpl.java
index 3759a962..b870ee29 100644
--- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/AbstractFlushStrategy.java
+++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/FlushStrategyImpl.java
@@ -1,12 +1,22 @@
-package org.dbsyncer.parser.flush;
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
+package org.dbsyncer.parser.flush.impl;
 
-import org.dbsyncer.common.config.StorageConfig;
 import org.dbsyncer.common.model.Result;
 import org.dbsyncer.common.util.CollectionUtils;
 import org.dbsyncer.common.util.StringUtil;
 import org.dbsyncer.parser.CacheService;
+import org.dbsyncer.parser.LogService;
+import org.dbsyncer.parser.LogType;
+import org.dbsyncer.parser.ProfileComponent;
+import org.dbsyncer.parser.flush.FlushService;
 import org.dbsyncer.parser.model.Meta;
+import org.dbsyncer.parser.model.SystemConfig;
 import org.dbsyncer.parser.strategy.FlushStrategy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Component;
 import org.springframework.util.Assert;
 
 import javax.annotation.Resource;
@@ -17,7 +27,10 @@ import java.time.Instant;
  * @version 1.0.0
  * @date 2021/11/18 22:22
  */
-public abstract class AbstractFlushStrategy implements FlushStrategy {
+@Component
+public final class FlushStrategyImpl implements FlushStrategy {
+
+    private final Logger logger = LoggerFactory.getLogger(getClass());
 
     @Resource
     private FlushService flushService;
@@ -26,10 +39,26 @@ public abstract class AbstractFlushStrategy implements FlushStrategy {
     private CacheService cacheService;
 
     @Resource
-    private StorageConfig storageConfig;
+    private ProfileComponent profileComponent;
+
+    @Resource
+    private LogService logService;
 
     @Override
     public void flushFullData(String metaId, Result result, String event) {
+        // 不记录全量数据, 只记录增量同步数据, 将异常记录到系统日志中
+        if (!getSystemConfig().isEnableStorageWriteFull()) {
+            // 不记录全量数据，只统计成功失败总数
+            refreshTotal(metaId, result);
+
+            if (!CollectionUtils.isEmpty(result.getFailData())) {
+                logger.error(result.getError().toString());
+                LogType logType = LogType.TableGroupLog.FULL_FAILED;
+                logService.log(logType, "%s:%s:%s", result.getTargetTableGroupName(), logType.getMessage(), result.getError().toString());
+            }
+            return;
+        }
+
         flush(metaId, result, event);
     }
 
@@ -52,15 +81,24 @@ public abstract class AbstractFlushStrategy implements FlushStrategy {
         refreshTotal(metaId, result);
 
         // 是否写失败数据
-        if (storageConfig.isWriteFail() && !CollectionUtils.isEmpty(result.getFailData())) {
-            final String error = StringUtil.substring(result.getError().toString(), 0, storageConfig.getMaxErrorLength());
+        if (getSystemConfig().isEnableStorageWriteFail() && !CollectionUtils.isEmpty(result.getFailData())) {
+            final String error = StringUtil.substring(result.getError().toString(), 0, getSystemConfig().getMaxStorageErrorLength());
             flushService.asyncWrite(metaId, result.getTableGroupId(), result.getTargetTableGroupName(), event, false, result.getFailData(), error);
         }
 
         // 是否写成功数据
-        if (storageConfig.isWriteSuccess() && !CollectionUtils.isEmpty(result.getSuccessData())) {
+        if (getSystemConfig().isEnableStorageWriteSuccess() && !CollectionUtils.isEmpty(result.getSuccessData())) {
             flushService.asyncWrite(metaId, result.getTableGroupId(), result.getTargetTableGroupName(), event, true, result.getSuccessData(), "");
         }
     }
 
+    /**
+     * TODO 加缓存过期
+     *
+     * @return
+     */
+    private SystemConfig getSystemConfig() {
+        return profileComponent.getSystemConfig();
+    }
+
 }
\ No newline at end of file
diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/GeneralBufferActuator.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/GeneralBufferActuator.java
index 3a02bde1..9ee1e570 100644
--- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/GeneralBufferActuator.java
+++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/GeneralBufferActuator.java
@@ -1,3 +1,6 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
 package org.dbsyncer.parser.flush.impl;
 
 import org.dbsyncer.common.QueueOverflowException;
diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/SystemConfig.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/SystemConfig.java
index f79bb321..d71cce24 100644
--- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/SystemConfig.java
+++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/model/SystemConfig.java
@@ -1,3 +1,6 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
 package org.dbsyncer.parser.model;
 
 import org.dbsyncer.sdk.constant.ConfigConstant;
@@ -31,10 +34,30 @@ public class SystemConfig extends ConfigModel {
     private int refreshIntervalSeconds = 5;
 
     /**
-     * 是否启用CDN加速访问静态资源(false-禁用；true-启动)
+     * 是否启用CDN加速访问静态资源(false-关闭; true-开启）
      */
     private boolean enableCDN;
 
+    /**
+     * 是否记录全量数据（false-关闭; true-开启）
+     */
+    private boolean enableStorageWriteFull;
+
+    /**
+     * 是否记录同步成功数据（false-关闭; true-开启）
+     */
+    private boolean enableStorageWriteSuccess;
+
+    /**
+     * 是否记录同步失败数据（false-关闭; true-开启）
+     */
+    private boolean enableStorageWriteFail = true;
+
+    /**
+     * 记录同步失败日志最大长度
+     */
+    private int maxStorageErrorLength = 2048;
+
     public int getExpireDataDays() {
         return expireDataDays;
     }
@@ -66,4 +89,36 @@ public class SystemConfig extends ConfigModel {
     public void setEnableCDN(boolean enableCDN) {
         this.enableCDN = enableCDN;
     }
+
+    public boolean isEnableStorageWriteFull() {
+        return enableStorageWriteFull;
+    }
+
+    public void setEnableStorageWriteFull(boolean enableStorageWriteFull) {
+        this.enableStorageWriteFull = enableStorageWriteFull;
+    }
+
+    public boolean isEnableStorageWriteSuccess() {
+        return enableStorageWriteSuccess;
+    }
+
+    public void setEnableStorageWriteSuccess(boolean enableStorageWriteSuccess) {
+        this.enableStorageWriteSuccess = enableStorageWriteSuccess;
+    }
+
+    public boolean isEnableStorageWriteFail() {
+        return enableStorageWriteFail;
+    }
+
+    public void setEnableStorageWriteFail(boolean enableStorageWriteFail) {
+        this.enableStorageWriteFail = enableStorageWriteFail;
+    }
+
+    public int getMaxStorageErrorLength() {
+        return maxStorageErrorLength;
+    }
+
+    public void setMaxStorageErrorLength(int maxStorageErrorLength) {
+        this.maxStorageErrorLength = maxStorageErrorLength;
+    }
 }
\ No newline at end of file
diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/impl/DisableFullFlushStrategy.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/impl/DisableFullFlushStrategy.java
deleted file mode 100644
index f55078b0..00000000
--- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/impl/DisableFullFlushStrategy.java
+++ /dev/null
@@ -1,39 +0,0 @@
-package org.dbsyncer.parser.strategy.impl;
-
-import org.dbsyncer.common.model.Result;
-import org.dbsyncer.common.util.CollectionUtils;
-import org.dbsyncer.parser.flush.AbstractFlushStrategy;
-import org.dbsyncer.parser.LogService;
-import org.dbsyncer.parser.LogType;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.annotation.Resource;
-
-/**
- * 不记录全量数据, 只记录增量同步数据, 将异常记录到系统日志中
- *
- * @author AE86
- * @version 1.0.0
- * @date 2021/11/18 21:49
- */
-public final class DisableFullFlushStrategy extends AbstractFlushStrategy {
-
-    private final Logger logger = LoggerFactory.getLogger(getClass());
-
-    @Resource
-    private LogService logService;
-
-    @Override
-    public void flushFullData(String metaId, Result result, String event) {
-        // 不记录全量数据，只统计成功失败总数
-        refreshTotal(metaId, result);
-
-        if (!CollectionUtils.isEmpty(result.getFailData())) {
-            logger.error(result.getError().toString());
-            LogType logType = LogType.TableGroupLog.FULL_FAILED;
-            logService.log(logType, "%s:%s:%s", result.getTargetTableGroupName(), logType.getMessage(), result.getError().toString());
-        }
-    }
-
-}
\ No newline at end of file
diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/impl/EnableFlushStrategy.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/impl/EnableFlushStrategy.java
deleted file mode 100644
index a8d59618..00000000
--- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/impl/EnableFlushStrategy.java
+++ /dev/null
@@ -1,18 +0,0 @@
-package org.dbsyncer.parser.strategy.impl;
-
-import org.dbsyncer.parser.flush.AbstractFlushStrategy;
-import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
-import org.springframework.stereotype.Component;
-
-/**
- * 记录全量和增量同步数据
- *
- * @author AE86
- * @version 1.0.0
- * @date 2021/11/18 22:21
- */
-@Component
-@ConditionalOnProperty(value = "dbsyncer.storage.write.full.enabled", havingValue = "true")
-public final class EnableFlushStrategy extends AbstractFlushStrategy {
-
-}
\ No newline at end of file
diff --git a/dbsyncer-web/src/main/resources/application.properties b/dbsyncer-web/src/main/resources/application.properties
index eb4b2040..9a483523 100644
--- a/dbsyncer-web/src/main/resources/application.properties
+++ b/dbsyncer-web/src/main/resources/application.properties
@@ -68,14 +68,6 @@ dbsyncer.storage.buffer-pull-count=20000
 dbsyncer.storage.buffer-queue-capacity=100000
 # [StorageBufferActuator]定时消费缓存队列间隔(毫秒)
 dbsyncer.storage.buffer-period-millisecond=300
-# 是否记录全量数据（false-关闭; true-开启）
-dbsyncer.storage.write.full.enabled=false
-# 是否记录同步成功数据（false-关闭; true-开启）
-dbsyncer.storage.write-success=true
-# 是否记录同步失败数据（false-关闭; true-开启）
-dbsyncer.storage.write-fail=true
-# 记录同步失败日志最大长度
-dbsyncer.storage.max-error-length=2048
 
 #plugin
 # 是否开启邮箱通知功能（false-关闭; true-开启）
diff --git a/dbsyncer-web/src/main/resources/public/system/system.html b/dbsyncer-web/src/main/resources/public/system/system.html
index 07711757..112ec446 100644
--- a/dbsyncer-web/src/main/resources/public/system/system.html
+++ b/dbsyncer-web/src/main/resources/public/system/system.html
@@ -33,10 +33,34 @@
                         
                     
                 
+                
    
+                    
+                    
    
+                        
+                    
    
+                
    
+                
    
+                    
+                    
    
+                        
+                    
    
+                
    
+                
    
+                    
+                    
    
+                        
+                    
    
+                
    
+                
    
+                    
+                    
    
+                        
+                    
    
+                
    
                 
    
                     
                     
    
-                        
+                        
                     
    
                 
    
                 
    
diff --git a/dbsyncer-web/src/main/resources/static/js/system/index.js b/dbsyncer-web/src/main/resources/static/js/system/index.js
index 8eb6d9f3..951dc665 100644
--- a/dbsyncer-web/src/main/resources/static/js/system/index.js
+++ b/dbsyncer-web/src/main/resources/static/js/system/index.js
@@ -10,7 +10,7 @@ function submit(data) {
 }
 
 $(function () {
-    $('#enableCDNSwitch').bootstrapSwitch({
+    $('.systemConfigSwitch').bootstrapSwitch({
         onText: "Yes",
         offText: "No",
         onColor: "success",
-- 
Gitee


From 96db141201de8ce698a1be52578c0abdf6431d57 Mon Sep 17 00:00:00 2001
From: AE86 <836391306@qq.com>
Date: Wed, 13 Dec 2023 06:19:30 +0000
Subject: [PATCH 71/98] =?UTF-8?q?=E6=94=AF=E6=8C=81BigDecimal=E8=BD=ACvarc?=
 =?UTF-8?q?har?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../sdk/connector/schema/LongVarcharValueMapper.java      | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/LongVarcharValueMapper.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/LongVarcharValueMapper.java
index 95c42ad0..c3498d42 100644
--- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/LongVarcharValueMapper.java
+++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/LongVarcharValueMapper.java
@@ -1,9 +1,13 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
 package org.dbsyncer.sdk.connector.schema;
 
 import org.dbsyncer.sdk.SdkException;
 import org.dbsyncer.sdk.connector.AbstractValueMapper;
 import org.dbsyncer.sdk.connector.ConnectorInstance;
 
+import java.math.BigDecimal;
 import java.sql.Date;
 
 /**
@@ -24,6 +28,10 @@ public class LongVarcharValueMapper extends AbstractValueMapper {
         if (val instanceof Integer) {
             return String.valueOf(val);
         }
+        if (val instanceof BigDecimal) {
+            BigDecimal bigDecimal = (BigDecimal) val;
+            return bigDecimal.toString();
+        }
         throw new SdkException(String.format("%s can not find type [%s], val [%s]", getClass().getSimpleName(), val.getClass(), val));
     }
 }
\ No newline at end of file
-- 
Gitee


From 0ba325157d3b252dc768dd931889d787fc8f6c72 Mon Sep 17 00:00:00 2001
From: AE86 <836391306@qq.com>
Date: Wed, 13 Dec 2023 07:26:03 +0000
Subject: [PATCH 72/98] fix pck

Signed-off-by: AE86 <836391306@qq.com>
---
 .../dbsyncer/parser/consumer/AbstractConsumer.java    | 11 +++++------
 dbsyncer-storage/src/main/test/LuceneFactoryTest.java |  8 ++++++--
 .../src/main/resources/application.properties         |  2 +-
 3 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/AbstractConsumer.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/AbstractConsumer.java
index 4fd461ed..69f47dc8 100644
--- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/AbstractConsumer.java
+++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/consumer/AbstractConsumer.java
@@ -3,17 +3,16 @@
  */
 package org.dbsyncer.parser.consumer;
 
-import org.dbsyncer.sdk.enums.ChangedEventTypeEnum;
-import org.dbsyncer.sdk.listener.ChangedEvent;
-import org.dbsyncer.sdk.listener.event.DDLChangedEvent;
-import org.dbsyncer.sdk.listener.Watcher;
-import org.dbsyncer.parser.ProfileComponent;
-import org.dbsyncer.parser.flush.impl.BufferActuatorRouter;
 import org.dbsyncer.parser.LogService;
 import org.dbsyncer.parser.LogType;
+import org.dbsyncer.parser.ProfileComponent;
+import org.dbsyncer.parser.flush.impl.BufferActuatorRouter;
 import org.dbsyncer.parser.model.Mapping;
 import org.dbsyncer.parser.model.Meta;
 import org.dbsyncer.parser.model.TableGroup;
+import org.dbsyncer.sdk.listener.ChangedEvent;
+import org.dbsyncer.sdk.listener.Watcher;
+import org.dbsyncer.sdk.listener.event.DDLChangedEvent;
 import org.dbsyncer.sdk.listener.event.SqlChangedEvent;
 
 import java.util.List;
diff --git a/dbsyncer-storage/src/main/test/LuceneFactoryTest.java b/dbsyncer-storage/src/main/test/LuceneFactoryTest.java
index e218935a..a1332d4d 100644
--- a/dbsyncer-storage/src/main/test/LuceneFactoryTest.java
+++ b/dbsyncer-storage/src/main/test/LuceneFactoryTest.java
@@ -1,3 +1,7 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
+
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.cn.smart.SmartChineseAnalyzer;
@@ -36,8 +40,8 @@ import org.apache.lucene.util.BytesRef;
 import org.dbsyncer.common.model.Paging;
 import org.dbsyncer.common.util.CollectionUtils;
 import org.dbsyncer.common.util.RandomUtil;
+import org.dbsyncer.sdk.constant.ConfigConstant;
 import org.dbsyncer.sdk.constant.ConnectorConstant;
-import org.dbsyncer.storage.constant.ConfigConstant;
 import org.dbsyncer.storage.enums.StorageDataStatusEnum;
 import org.dbsyncer.storage.impl.SnowflakeIdWorker;
 import org.dbsyncer.storage.lucene.Option;
@@ -131,7 +135,7 @@ public class LuceneFactoryTest {
         } catch (InterruptedException e) {
             logger.error(e.getMessage());
         } catch (IOException e) {
-            e.printStackTrace();
+            logger.error(e.getMessage());
         }
 
         TimeUnit.SECONDS.sleep(3);
diff --git a/dbsyncer-web/src/main/resources/application.properties b/dbsyncer-web/src/main/resources/application.properties
index 9a483523..b059a833 100644
--- a/dbsyncer-web/src/main/resources/application.properties
+++ b/dbsyncer-web/src/main/resources/application.properties
@@ -49,7 +49,7 @@ dbsyncer.parser.table.group.buffer-period-millisecond=300
 
 #storage
 # 数据存储类型:disk(默认)/mysql(推荐生产环境使用)
-# false: 保存磁盘/data/config(驱动配置)|data(按驱动分别存储增量数据)|log(系统日志)}
+# disk-磁盘:/data/config(驱动配置)|data(按驱动分别存储增量数据)|log(系统日志)
 dbsyncer.storage.type=disk
 dbsyncer.storage.mysql.url=jdbc:mysql://127.0.0.1:3306/dbsyncer?rewriteBatchedStatements=true&seUnicode=true&characterEncoding=UTF8&serverTimezone=Asia/Shanghai&useSSL=false&verifyServerCertificate=false&autoReconnect=true
 dbsyncer.storage.mysql.username=root
-- 
Gitee


From a3ccb9613effe5bc2fc4db92d799681216c19b56 Mon Sep 17 00:00:00 2001
From: life <13122192336@163.com>
Date: Thu, 14 Dec 2023 15:36:34 +0800
Subject: [PATCH 73/98] =?UTF-8?q?logminer=20cdc=E4=BD=BF=E7=94=A8=E5=9B=9E?=
 =?UTF-8?q?=E6=9F=A5?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../connector/oracle/cdc/OracleListener.java  | 112 +++++++++++++-----
 .../oracle/logminer/parser/DeleteSql.java     |  75 ++++++++++++
 .../oracle/logminer/parser/InsertSql.java     |  79 ++++++++++++
 .../oracle/logminer/parser/Parser.java        |   8 ++
 .../oracle/logminer/parser/UpdateSql.java     |  93 +++++++++++++++
 5 files changed, 340 insertions(+), 27 deletions(-)
 create mode 100644 dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/DeleteSql.java
 create mode 100644 dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/InsertSql.java
 create mode 100644 dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/Parser.java
 create mode 100644 dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/UpdateSql.java

diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java
index 05cdb9c9..c69597af 100644
--- a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java
+++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java
@@ -3,6 +3,7 @@
  */
 package org.dbsyncer.connector.oracle.cdc;
 
+import java.sql.SQLException;
 import net.sf.jsqlparser.JSQLParserException;
 import net.sf.jsqlparser.parser.CCJSqlParserUtil;
 import net.sf.jsqlparser.schema.Table;
@@ -14,17 +15,22 @@ import net.sf.jsqlparser.statement.update.Update;
 import org.dbsyncer.common.util.StringUtil;
 import org.dbsyncer.connector.oracle.OracleException;
 import org.dbsyncer.connector.oracle.logminer.LogMiner;
+import org.dbsyncer.connector.oracle.logminer.RedoEvent;
+import org.dbsyncer.connector.oracle.logminer.parser.DeleteSql;
+import org.dbsyncer.connector.oracle.logminer.parser.InsertSql;
+import org.dbsyncer.connector.oracle.logminer.parser.Parser;
+import org.dbsyncer.connector.oracle.logminer.parser.UpdateSql;
 import org.dbsyncer.sdk.config.DatabaseConfig;
+import org.dbsyncer.sdk.connector.database.DatabaseConnectorInstance;
 import org.dbsyncer.sdk.constant.ConnectorConstant;
 import org.dbsyncer.sdk.listener.AbstractDatabaseListener;
 import org.dbsyncer.sdk.listener.event.DDLChangedEvent;
+import org.dbsyncer.sdk.listener.event.RowChangedEvent;
 import org.dbsyncer.sdk.listener.event.SqlChangedEvent;
 import org.dbsyncer.sdk.model.ChangedOffset;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.sql.SQLException;
-
 /**
  * @Author AE86
  * @Version 1.0.0
@@ -48,35 +54,17 @@ public class OracleListener extends AbstractDatabaseListener {
             String schema = config.getSchema();
             boolean containsPos = snapshot.containsKey(REDO_POSITION);
             logMiner = new LogMiner(username, password, url, schema, driverClassName);
-            logMiner.setStartScn(containsPos ? Long.parseLong(snapshot.get(REDO_POSITION)) : 0);
+            if (!snapshot.get(REDO_POSITION).equals("null")){
+                logMiner.setStartScn(containsPos ? Long.parseLong(snapshot.get(REDO_POSITION)) : 0);
+            }
             logMiner.registerEventListener((event) -> {
+//                sendSql(event);
                 try {
-                    Statement statement = CCJSqlParserUtil.parse(event.getRedoSql());
-                    if (statement instanceof Update) {
-                        Update update = (Update) statement;
-                        sendChangedEvent(new SqlChangedEvent(replaceTableName(update.getTable()), ConnectorConstant.OPERTION_UPDATE, event.getRedoSql(), null, event.getScn()));
-                        return;
-                    }
-
-                    if (statement instanceof Insert) {
-                        Insert insert = (Insert) statement;
-                        sendChangedEvent(new SqlChangedEvent(replaceTableName(insert.getTable()), ConnectorConstant.OPERTION_INSERT, event.getRedoSql(), null, event.getScn()));
-                        return;
-                    }
-
-                    if (statement instanceof Delete) {
-                        Delete delete = (Delete) statement;
-                        sendChangedEvent(new SqlChangedEvent(replaceTableName(delete.getTable()), ConnectorConstant.OPERTION_DELETE, event.getRedoSql(), null, event.getScn()));
-                        return;
-                    }
-
-                    if (statement instanceof Alter) {
-                        Alter alter = (Alter) statement;
-                        sendChangedEvent(new DDLChangedEvent("", replaceTableName(alter.getTable()), ConnectorConstant.OPERTION_ALTER, event.getRedoSql(), null, event.getScn()));
-                        return;
-                    }
+                    parseSqlToPk(event);
                 } catch (JSQLParserException e) {
                     logger.error("不支持sql:" + event.getRedoSql());
+                }catch (Exception e){
+                    logger.error(e.getMessage());
                 }
             });
             logMiner.start();
@@ -86,6 +74,75 @@ public class OracleListener extends AbstractDatabaseListener {
         }
     }
 
+    //发送sql解析时间
+    private void sendSql(RedoEvent event){
+        try {
+            Statement statement = CCJSqlParserUtil.parse(event.getRedoSql());
+            if (statement instanceof Update) {
+                Update update = (Update) statement;
+                sendChangedEvent(new SqlChangedEvent(replaceTableName(update.getTable()), ConnectorConstant.OPERTION_UPDATE, event.getRedoSql(), null, event.getScn()));
+                return;
+            }
+
+            if (statement instanceof Insert) {
+                Insert insert = (Insert) statement;
+                sendChangedEvent(new SqlChangedEvent(replaceTableName(insert.getTable()), ConnectorConstant.OPERTION_INSERT, event.getRedoSql(), null, event.getScn()));
+                return;
+            }
+
+            if (statement instanceof Delete) {
+                Delete delete = (Delete) statement;
+                sendChangedEvent(new SqlChangedEvent(replaceTableName(delete.getTable()), ConnectorConstant.OPERTION_DELETE, event.getRedoSql(), null, event.getScn()));
+                return;
+            }
+
+            if (statement instanceof Alter) {
+                Alter alter = (Alter) statement;
+                sendChangedEvent(new DDLChangedEvent("", replaceTableName(alter.getTable()), ConnectorConstant.OPERTION_ALTER, event.getRedoSql(), null, event.getScn()));
+                return;
+            }
+        } catch (JSQLParserException e) {
+            logger.error("不支持sql:" + event.getRedoSql());
+        }catch (Exception e){
+            logger.error(e.getMessage());
+        }
+    }
+
+    //解析sql出来主键数据
+    private void parseSqlToPk(RedoEvent event) throws Exception {
+        Statement statement = CCJSqlParserUtil.parse(event.getRedoSql());
+        if (statement instanceof Insert){
+            Insert insert = (Insert) statement;
+            org.dbsyncer.sdk.model.Table table1 =sourceTable.stream()
+                    .filter(x->x.getName().equals(replaceTableName(insert.getTable())))
+                    .findFirst().orElse(null);
+            if (table1 != null){
+                Parser parser = new InsertSql(insert,table1.getColumn(),(DatabaseConnectorInstance) connectorInstance);
+                sendChangedEvent(new RowChangedEvent(replaceTableName(insert.getTable()), ConnectorConstant.OPERTION_INSERT,parser.parseSql() ,null,event.getScn()));
+            }
+
+        }else if (statement instanceof Update){
+            Update update = (Update) statement;
+            org.dbsyncer.sdk.model.Table table1 =sourceTable.stream()
+                    .filter(x->x.getName().equals(replaceTableName(update.getTable())))
+                    .findFirst().orElse(null);
+            if (table1 != null){
+                Parser parser = new UpdateSql(update,table1.getColumn(),(DatabaseConnectorInstance) connectorInstance);
+                sendChangedEvent(new RowChangedEvent(replaceTableName(update.getTable()), ConnectorConstant.OPERTION_UPDATE, parser.parseSql(),null,event.getScn()));
+            }
+        }else if (statement instanceof Delete){
+            Delete delete = (Delete) statement;
+            org.dbsyncer.sdk.model.Table table1 =sourceTable.stream()
+                    .filter(x->x.getName().equals(replaceTableName(delete.getTable())))
+                    .findFirst().orElse(null);
+            if (table1 !=null){
+                Parser parser = new DeleteSql(delete,table1.getColumn());
+                sendChangedEvent(new RowChangedEvent(replaceTableName(delete.getTable()), ConnectorConstant.OPERTION_DELETE, parser.parseSql(),null,event.getScn()));
+            }
+        }
+
+    }
+
     @Override
     public void close() {
         try {
@@ -109,4 +166,5 @@ public class OracleListener extends AbstractDatabaseListener {
         return StringUtil.replace(table.getName(), StringUtil.DOUBLE_QUOTATION, StringUtil.EMPTY);
     }
 
+
 }
\ No newline at end of file
diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/DeleteSql.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/DeleteSql.java
new file mode 100644
index 00000000..f67df1a2
--- /dev/null
+++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/DeleteSql.java
@@ -0,0 +1,75 @@
+package org.dbsyncer.connector.oracle.logminer.parser;
+
+import java.math.BigInteger;
+import java.sql.Types;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import net.sf.jsqlparser.expression.BinaryExpression;
+import net.sf.jsqlparser.expression.Expression;
+import net.sf.jsqlparser.schema.Column;
+import net.sf.jsqlparser.statement.delete.Delete;
+import org.dbsyncer.common.util.StringUtil;
+import org.dbsyncer.sdk.model.Field;
+
+/**
+ * @author : life
+ * @Description :
+ * @Param :
+ * @return :
+ * @date : 2023/12/14  14:58
+ */
+public class DeleteSql implements Parser {
+
+    Delete delete;
+
+    Map cloumnMap;
+
+    List columns;
+
+    public DeleteSql(Delete delete,  List columns) {
+        this.delete = delete;
+        this.columns = columns;
+        cloumnMap = new HashMap<>();
+    }
+
+    @Override
+    public List parseSql() {
+        findColumn(delete.getWhere());
+        return columnMapToData();
+    }
+
+    private void findColumn(Expression expression) {
+        BinaryExpression binaryExpression = (BinaryExpression) expression;
+        if (binaryExpression.getLeftExpression() instanceof Column) {
+            Column column = (Column) binaryExpression.getLeftExpression();
+            String value = binaryExpression.getRightExpression().toString();
+            cloumnMap.put(StringUtil.replace(column.getColumnName(), StringUtil.DOUBLE_QUOTATION,
+                    StringUtil.EMPTY), value);
+            return;
+        }
+        findColumn(binaryExpression.getLeftExpression());
+        findColumn(binaryExpression.getRightExpression());
+    }
+
+    private List columnMapToData(){
+        List data = new LinkedList<>();
+        for (Field field : columns) {
+            if (field.isPk()) {
+                Object value = cloumnMap.get(field.getName());
+                switch (field.getType()) {
+                    case Types.DECIMAL:
+                        value = new BigInteger(
+                                StringUtil.replace(value.toString(), StringUtil.SINGLE_QUOTATION,
+                                        StringUtil.EMPTY));
+                }
+                data.add(value);
+            } else {
+                data.add(null);
+            }
+        }
+        return data;
+    }
+
+}
diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/InsertSql.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/InsertSql.java
new file mode 100644
index 00000000..f9753d28
--- /dev/null
+++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/InsertSql.java
@@ -0,0 +1,79 @@
+package org.dbsyncer.connector.oracle.logminer.parser;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import net.sf.jsqlparser.expression.operators.relational.ExpressionList;
+import net.sf.jsqlparser.schema.Column;
+import net.sf.jsqlparser.statement.insert.Insert;
+import org.dbsyncer.common.util.CollectionUtils;
+import org.dbsyncer.common.util.StringUtil;
+import org.dbsyncer.sdk.connector.database.DatabaseConnectorInstance;
+import org.dbsyncer.sdk.model.Field;
+
+public class InsertSql implements Parser{
+
+    Insert insert;
+
+    Map cloumnMap;
+
+    List columns;
+
+    DatabaseConnectorInstance instance;
+
+    public InsertSql(Insert insert, List columns,
+            DatabaseConnectorInstance instance) {
+        this.insert = insert;
+        this.columns = columns;
+        this.instance = instance;
+        cloumnMap = new HashMap<>();
+    }
+
+    @Override
+    public List parseSql() {
+        List columns= insert.getColumns();
+        ExpressionList values = insert.getSelect().getValues().getExpressions();
+        for (int i = 0; i < columns.size(); i++) {
+            cloumnMap.put(StringUtil.replace(columns.get(i).getColumnName(), StringUtil.DOUBLE_QUOTATION, StringUtil.EMPTY),values.get(i).toString());
+        }
+        String toSqlString = toSql(StringUtil.replace(insert.getTable().getName(), StringUtil.DOUBLE_QUOTATION, StringUtil.EMPTY));
+        List data= ToResult(toSqlString);
+        return data;
+    }
+
+    private String toSql(String tableName){
+        StringBuilder sql = new StringBuilder("SELECT *");
+        sql.append(" FROM ").append(tableName).append(" WHERE ");
+        int pkCount = 0;
+        for (Field field: columns) {
+            if (field.isPk()){
+                String value = cloumnMap.get(field.getName());
+                if (StringUtil.isNotBlank(value)){
+                    if (pkCount > 0){
+                        sql.append(" AND ");
+                    }
+                    sql.append(field.getName()).append("=").append(value);
+                    pkCount ++;
+                }
+            }
+        }
+        return sql.toString();
+    }
+
+    //从主键解析出来的map装载成sql并运行sql找出对应的数据
+    private List ToResult(String sql){
+        List> results = instance.execute(databaseTemplate -> databaseTemplate.queryForList(sql));
+        List list = new LinkedList<>();
+        if (!CollectionUtils.isEmpty(results)){
+            results.forEach(map->{
+                for (String key:map.keySet()) {
+                    list.add(map.get(key));
+                }
+            });
+            return list;
+        }
+        return Collections.EMPTY_LIST;
+    }
+}
diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/Parser.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/Parser.java
new file mode 100644
index 00000000..0585b404
--- /dev/null
+++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/Parser.java
@@ -0,0 +1,8 @@
+package org.dbsyncer.connector.oracle.logminer.parser;
+
+import java.util.List;
+
+public interface Parser {
+
+    List parseSql();
+}
diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/UpdateSql.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/UpdateSql.java
new file mode 100644
index 00000000..d96eabd5
--- /dev/null
+++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/UpdateSql.java
@@ -0,0 +1,93 @@
+package org.dbsyncer.connector.oracle.logminer.parser;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import net.sf.jsqlparser.expression.BinaryExpression;
+import net.sf.jsqlparser.expression.Expression;
+import net.sf.jsqlparser.schema.Column;
+import net.sf.jsqlparser.statement.update.Update;
+import org.dbsyncer.common.util.CollectionUtils;
+import org.dbsyncer.common.util.StringUtil;
+import org.dbsyncer.sdk.connector.database.DatabaseConnectorInstance;
+import org.dbsyncer.sdk.model.Field;
+
+public class UpdateSql implements Parser {
+
+    Update update;
+
+    Map cloumnMap;
+
+    List columns;
+
+    DatabaseConnectorInstance instance;
+
+    public UpdateSql(Update update, List columns,
+            DatabaseConnectorInstance instance) {
+        this.update = update;
+        this.columns = columns;
+        this.instance = instance;
+        cloumnMap = new HashMap<>();
+    }
+
+    @Override
+    public List parseSql() {
+        findColumn(update.getWhere());
+        String toSqlString = toSql(
+                StringUtil.replace(update.getTable().getName(), StringUtil.DOUBLE_QUOTATION,
+                        StringUtil.EMPTY));
+        List data = ToResult(toSqlString);
+        return data;
+    }
+
+
+    private void findColumn(Expression expression) {
+        BinaryExpression binaryExpression = (BinaryExpression) expression;
+        if (binaryExpression.getLeftExpression() instanceof Column) {
+            Column column = (Column) binaryExpression.getLeftExpression();
+            String value = binaryExpression.getRightExpression().toString();
+            cloumnMap.put(StringUtil.replace(column.getColumnName(), StringUtil.DOUBLE_QUOTATION,
+                    StringUtil.EMPTY), value);
+            return;
+        }
+        findColumn(binaryExpression.getLeftExpression());
+        findColumn(binaryExpression.getRightExpression());
+    }
+
+    private String toSql(String tableName) {
+        StringBuilder sql = new StringBuilder("SELECT *");
+        sql.append(" FROM ").append(tableName).append(" WHERE ");
+        int pkCount = 0;
+        for (Field field : columns) {
+            if (field.isPk()) {
+                String value = cloumnMap.get(field.getName());
+                if (StringUtil.isNotBlank(value)) {
+                    if (pkCount > 0) {
+                        sql.append(" AND ");
+                    }
+                    sql.append(field.getName()).append("=").append(value);
+                    pkCount++;
+                }
+            }
+        }
+        return sql.toString();
+    }
+
+    //从主键解析出来的map装载成sql并运行sql找出对应的数据
+    private List ToResult(String sql) {
+        List> results = instance.execute(
+                databaseTemplate -> databaseTemplate.queryForList(sql));
+        List list = new LinkedList<>();
+        if (!CollectionUtils.isEmpty(results)) {
+            results.forEach(map -> {
+                for (String key : map.keySet()) {
+                    list.add(map.get(key));
+                }
+            });
+            return list;
+        }
+        return Collections.EMPTY_LIST;
+    }
+}
-- 
Gitee


From 4b909acd6e47579ad16dd5078d96ede60f94c749 Mon Sep 17 00:00:00 2001
From: life <13122192336@163.com>
Date: Thu, 14 Dec 2023 21:50:19 +0800
Subject: [PATCH 74/98] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=97=B6=E9=97=B4?=
 =?UTF-8?q?=E6=94=BE=E7=BD=AEscn=E8=BF=87=E5=A4=9A=E9=94=99=E8=AF=AF?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../connector/oracle/cdc/OracleListener.java  | 70 +++++++++++--------
 .../connector/oracle/logminer/LogMiner.java   |  2 +-
 2 files changed, 43 insertions(+), 29 deletions(-)

diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java
index c69597af..32dcb468 100644
--- a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java
+++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java
@@ -54,16 +54,14 @@ public class OracleListener extends AbstractDatabaseListener {
             String schema = config.getSchema();
             boolean containsPos = snapshot.containsKey(REDO_POSITION);
             logMiner = new LogMiner(username, password, url, schema, driverClassName);
-            if (!snapshot.get(REDO_POSITION).equals("null")){
-                logMiner.setStartScn(containsPos ? Long.parseLong(snapshot.get(REDO_POSITION)) : 0);
-            }
+            logMiner.setStartScn(containsPos ? Long.parseLong(snapshot.get(REDO_POSITION)) : 0);
             logMiner.registerEventListener((event) -> {
 //                sendSql(event);
                 try {
                     parseSqlToPk(event);
                 } catch (JSQLParserException e) {
                     logger.error("不支持sql:" + event.getRedoSql());
-                }catch (Exception e){
+                } catch (Exception e) {
                     logger.error(e.getMessage());
                 }
             });
@@ -75,35 +73,43 @@ public class OracleListener extends AbstractDatabaseListener {
     }
 
     //发送sql解析时间
-    private void sendSql(RedoEvent event){
+    private void sendSql(RedoEvent event) {
         try {
             Statement statement = CCJSqlParserUtil.parse(event.getRedoSql());
             if (statement instanceof Update) {
                 Update update = (Update) statement;
-                sendChangedEvent(new SqlChangedEvent(replaceTableName(update.getTable()), ConnectorConstant.OPERTION_UPDATE, event.getRedoSql(), null, event.getScn()));
+                sendChangedEvent(new SqlChangedEvent(replaceTableName(update.getTable()),
+                        ConnectorConstant.OPERTION_UPDATE, event.getRedoSql(), null,
+                        event.getScn()));
                 return;
             }
 
             if (statement instanceof Insert) {
                 Insert insert = (Insert) statement;
-                sendChangedEvent(new SqlChangedEvent(replaceTableName(insert.getTable()), ConnectorConstant.OPERTION_INSERT, event.getRedoSql(), null, event.getScn()));
+                sendChangedEvent(new SqlChangedEvent(replaceTableName(insert.getTable()),
+                        ConnectorConstant.OPERTION_INSERT, event.getRedoSql(), null,
+                        event.getScn()));
                 return;
             }
 
             if (statement instanceof Delete) {
                 Delete delete = (Delete) statement;
-                sendChangedEvent(new SqlChangedEvent(replaceTableName(delete.getTable()), ConnectorConstant.OPERTION_DELETE, event.getRedoSql(), null, event.getScn()));
+                sendChangedEvent(new SqlChangedEvent(replaceTableName(delete.getTable()),
+                        ConnectorConstant.OPERTION_DELETE, event.getRedoSql(), null,
+                        event.getScn()));
                 return;
             }
 
             if (statement instanceof Alter) {
                 Alter alter = (Alter) statement;
-                sendChangedEvent(new DDLChangedEvent("", replaceTableName(alter.getTable()), ConnectorConstant.OPERTION_ALTER, event.getRedoSql(), null, event.getScn()));
+                sendChangedEvent(new DDLChangedEvent("", replaceTableName(alter.getTable()),
+                        ConnectorConstant.OPERTION_ALTER, event.getRedoSql(), null,
+                        event.getScn()));
                 return;
             }
         } catch (JSQLParserException e) {
             logger.error("不支持sql:" + event.getRedoSql());
-        }catch (Exception e){
+        } catch (Exception e) {
             logger.error(e.getMessage());
         }
     }
@@ -111,33 +117,41 @@ public class OracleListener extends AbstractDatabaseListener {
     //解析sql出来主键数据
     private void parseSqlToPk(RedoEvent event) throws Exception {
         Statement statement = CCJSqlParserUtil.parse(event.getRedoSql());
-        if (statement instanceof Insert){
+        if (statement instanceof Insert) {
             Insert insert = (Insert) statement;
-            org.dbsyncer.sdk.model.Table table1 =sourceTable.stream()
-                    .filter(x->x.getName().equals(replaceTableName(insert.getTable())))
+            org.dbsyncer.sdk.model.Table table1 = sourceTable.stream()
+                    .filter(x -> x.getName().equals(replaceTableName(insert.getTable())))
                     .findFirst().orElse(null);
-            if (table1 != null){
-                Parser parser = new InsertSql(insert,table1.getColumn(),(DatabaseConnectorInstance) connectorInstance);
-                sendChangedEvent(new RowChangedEvent(replaceTableName(insert.getTable()), ConnectorConstant.OPERTION_INSERT,parser.parseSql() ,null,event.getScn()));
+            if (table1 != null) {
+                Parser parser = new InsertSql(insert, table1.getColumn(),
+                        (DatabaseConnectorInstance) connectorInstance);
+                sendChangedEvent(new RowChangedEvent(replaceTableName(insert.getTable()),
+                        ConnectorConstant.OPERTION_INSERT, parser.parseSql(), null,
+                        event.getScn()));
             }
 
-        }else if (statement instanceof Update){
+        } else if (statement instanceof Update) {
             Update update = (Update) statement;
-            org.dbsyncer.sdk.model.Table table1 =sourceTable.stream()
-                    .filter(x->x.getName().equals(replaceTableName(update.getTable())))
+            org.dbsyncer.sdk.model.Table table1 = sourceTable.stream()
+                    .filter(x -> x.getName().equals(replaceTableName(update.getTable())))
                     .findFirst().orElse(null);
-            if (table1 != null){
-                Parser parser = new UpdateSql(update,table1.getColumn(),(DatabaseConnectorInstance) connectorInstance);
-                sendChangedEvent(new RowChangedEvent(replaceTableName(update.getTable()), ConnectorConstant.OPERTION_UPDATE, parser.parseSql(),null,event.getScn()));
+            if (table1 != null) {
+                Parser parser = new UpdateSql(update, table1.getColumn(),
+                        (DatabaseConnectorInstance) connectorInstance);
+                sendChangedEvent(new RowChangedEvent(replaceTableName(update.getTable()),
+                        ConnectorConstant.OPERTION_UPDATE, parser.parseSql(), null,
+                        event.getScn()));
             }
-        }else if (statement instanceof Delete){
+        } else if (statement instanceof Delete) {
             Delete delete = (Delete) statement;
-            org.dbsyncer.sdk.model.Table table1 =sourceTable.stream()
-                    .filter(x->x.getName().equals(replaceTableName(delete.getTable())))
+            org.dbsyncer.sdk.model.Table table1 = sourceTable.stream()
+                    .filter(x -> x.getName().equals(replaceTableName(delete.getTable())))
                     .findFirst().orElse(null);
-            if (table1 !=null){
-                Parser parser = new DeleteSql(delete,table1.getColumn());
-                sendChangedEvent(new RowChangedEvent(replaceTableName(delete.getTable()), ConnectorConstant.OPERTION_DELETE, parser.parseSql(),null,event.getScn()));
+            if (table1 != null) {
+                Parser parser = new DeleteSql(delete, table1.getColumn());
+                sendChangedEvent(new RowChangedEvent(replaceTableName(delete.getTable()),
+                        ConnectorConstant.OPERTION_DELETE, parser.parseSql(), null,
+                        event.getScn()));
             }
         }
 
diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/LogMiner.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/LogMiner.java
index dd3580b3..810b7ea2 100644
--- a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/LogMiner.java
+++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/LogMiner.java
@@ -132,7 +132,7 @@ public class LogMiner {
 
                 try {
                     // 避免频繁的执行导致 PGA 内存超出 PGA_AGGREGATE_LIMIT
-                    TimeUnit.SECONDS.sleep(2);
+                    TimeUnit.SECONDS.sleep(5);
                 } catch (InterruptedException e) {
                     logger.error(e.getMessage(), e);
                 }
-- 
Gitee


From ef19df5ab78c010dcaca829123931baa959d78fa Mon Sep 17 00:00:00 2001
From: AE86 <836391306@qq.com>
Date: Thu, 14 Dec 2023 21:55:43 +0800
Subject: [PATCH 75/98] fix

---
 .../org/dbsyncer/parser/flush/impl/FlushServiceImpl.java   | 7 +------
 .../org/dbsyncer/parser/flush/impl/FlushStrategyImpl.java  | 5 -----
 .../main/java/org/dbsyncer/web/config/WebAppConfig.java    | 1 -
 3 files changed, 1 insertion(+), 12 deletions(-)

diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/FlushServiceImpl.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/FlushServiceImpl.java
index 9be498b5..1d3a09f3 100644
--- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/FlushServiceImpl.java
+++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/FlushServiceImpl.java
@@ -36,7 +36,7 @@ import java.util.concurrent.Executor;
  * @date 2020/05/19 18:38
  */
 @Component
-public class FlushServiceImpl implements FlushService {
+public final class FlushServiceImpl implements FlushService {
 
     private final Logger logger = LoggerFactory.getLogger(getClass());
 
@@ -89,11 +89,6 @@ public class FlushServiceImpl implements FlushService {
         });
     }
 
-    /**
-     * TODO 加缓存过期
-     *
-     * @return
-     */
     private SystemConfig getSystemConfig() {
         return profileComponent.getSystemConfig();
     }
diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/FlushStrategyImpl.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/FlushStrategyImpl.java
index b870ee29..a96127b6 100644
--- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/FlushStrategyImpl.java
+++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/FlushStrategyImpl.java
@@ -92,11 +92,6 @@ public final class FlushStrategyImpl implements FlushStrategy {
         }
     }
 
-    /**
-     * TODO 加缓存过期
-     *
-     * @return
-     */
     private SystemConfig getSystemConfig() {
         return profileComponent.getSystemConfig();
     }
diff --git a/dbsyncer-web/src/main/java/org/dbsyncer/web/config/WebAppConfig.java b/dbsyncer-web/src/main/java/org/dbsyncer/web/config/WebAppConfig.java
index ff9d5aff..75ddcaf2 100644
--- a/dbsyncer-web/src/main/java/org/dbsyncer/web/config/WebAppConfig.java
+++ b/dbsyncer-web/src/main/java/org/dbsyncer/web/config/WebAppConfig.java
@@ -62,7 +62,6 @@ public class WebAppConfig extends WebSecurityConfigurerAdapter implements Authen
     @Resource
     private UserConfigService userConfigService;
 
-
     /**
      * 登录失败
      *
-- 
Gitee


From 1b8ee933c2ffd568889fc916bb5a2966b932b148 Mon Sep 17 00:00:00 2001
From: AE86 <836391306@qq.com>
Date: Thu, 14 Dec 2023 23:33:53 +0800
Subject: [PATCH 76/98] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=BB=A3=E7=A0=81?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../elasticsearch/cdc/ESQuartzListener.java   |   2 +-
 .../connector/file/cdc/FileListener.java      |   2 +-
 .../connector/oracle/cdc/OracleListener.java  | 144 ++++++------------
 .../{UpdateSql.java => AbstractParser.java}   | 110 ++++++-------
 .../oracle/logminer/parser/DeleteSql.java     |  75 ---------
 .../oracle/logminer/parser/InsertSql.java     |  79 ----------
 .../oracle/logminer/parser/Parser.java        |   7 +-
 .../logminer/parser/impl/DeleteSql.java       |  49 ++++++
 .../logminer/parser/impl/InsertSql.java       |  37 +++++
 .../logminer/parser/impl/UpdateSql.java       |  30 ++++
 .../manager/impl/IncrementPuller.java         |   1 +
 .../listener/AbstractDatabaseListener.java    |   5 +-
 .../sdk/listener/AbstractListener.java        |  11 +-
 .../sdk/listener/AbstractQuartzListener.java  |   5 +-
 .../sdk/listener/DatabaseQuartzListener.java  |   5 +-
 .../org/dbsyncer/sdk/listener/Listener.java   |   6 +
 16 files changed, 256 insertions(+), 312 deletions(-)
 rename dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/{UpdateSql.java => AbstractParser.java} (52%)
 delete mode 100644 dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/DeleteSql.java
 delete mode 100644 dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/InsertSql.java
 create mode 100644 dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/impl/DeleteSql.java
 create mode 100644 dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/impl/InsertSql.java
 create mode 100644 dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/impl/UpdateSql.java

diff --git a/dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/cdc/ESQuartzListener.java b/dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/cdc/ESQuartzListener.java
index eb9b867a..a19f2912 100644
--- a/dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/cdc/ESQuartzListener.java
+++ b/dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/cdc/ESQuartzListener.java
@@ -67,7 +67,7 @@ public final class ESQuartzListener extends AbstractQuartzListener {
                     }
 
                     // 读取历史增量点
-                    f.setValue(snapshot.get(key));
+                    f.setValue((String) snapshot.get(key));
                     point.setBeginKey(key);
                     point.setBeginValue(quartzFilter.toString(quartzFilter.getObject()));
                     continue;
diff --git a/dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/cdc/FileListener.java b/dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/cdc/FileListener.java
index c0449e1a..7f9327c8 100644
--- a/dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/cdc/FileListener.java
+++ b/dbsyncer-connector/dbsyncer-connector-file/src/main/java/org/dbsyncer/connector/file/cdc/FileListener.java
@@ -107,7 +107,7 @@ public class FileListener extends AbstractListener {
             final RandomAccessFile raf = new BufferedRandomAccessFile(file, "r");
             final String filePosKey = getFilePosKey(fileName);
             if (snapshot.containsKey(filePosKey)) {
-                raf.seek(NumberUtil.toLong(snapshot.get(filePosKey), 0L));
+                raf.seek(NumberUtil.toLong((String) snapshot.get(filePosKey), 0L));
             } else {
                 raf.seek(raf.length());
                 snapshot.put(filePosKey, String.valueOf(raf.getFilePointer()));
diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java
index 32dcb468..ff01ec36 100644
--- a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java
+++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java
@@ -3,34 +3,35 @@
  */
 package org.dbsyncer.connector.oracle.cdc;
 
-import java.sql.SQLException;
 import net.sf.jsqlparser.JSQLParserException;
 import net.sf.jsqlparser.parser.CCJSqlParserUtil;
 import net.sf.jsqlparser.schema.Table;
 import net.sf.jsqlparser.statement.Statement;
-import net.sf.jsqlparser.statement.alter.Alter;
 import net.sf.jsqlparser.statement.delete.Delete;
 import net.sf.jsqlparser.statement.insert.Insert;
 import net.sf.jsqlparser.statement.update.Update;
 import org.dbsyncer.common.util.StringUtil;
 import org.dbsyncer.connector.oracle.OracleException;
+import org.dbsyncer.connector.oracle.logminer.parser.AbstractParser;
 import org.dbsyncer.connector.oracle.logminer.LogMiner;
 import org.dbsyncer.connector.oracle.logminer.RedoEvent;
-import org.dbsyncer.connector.oracle.logminer.parser.DeleteSql;
-import org.dbsyncer.connector.oracle.logminer.parser.InsertSql;
-import org.dbsyncer.connector.oracle.logminer.parser.Parser;
-import org.dbsyncer.connector.oracle.logminer.parser.UpdateSql;
+import org.dbsyncer.connector.oracle.logminer.parser.impl.DeleteSql;
+import org.dbsyncer.connector.oracle.logminer.parser.impl.InsertSql;
+import org.dbsyncer.connector.oracle.logminer.parser.impl.UpdateSql;
 import org.dbsyncer.sdk.config.DatabaseConfig;
-import org.dbsyncer.sdk.connector.database.DatabaseConnectorInstance;
 import org.dbsyncer.sdk.constant.ConnectorConstant;
 import org.dbsyncer.sdk.listener.AbstractDatabaseListener;
-import org.dbsyncer.sdk.listener.event.DDLChangedEvent;
 import org.dbsyncer.sdk.listener.event.RowChangedEvent;
-import org.dbsyncer.sdk.listener.event.SqlChangedEvent;
 import org.dbsyncer.sdk.model.ChangedOffset;
+import org.dbsyncer.sdk.model.Field;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.sql.SQLException;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
 /**
  * @Author AE86
  * @Version 1.0.0
@@ -40,9 +41,14 @@ public class OracleListener extends AbstractDatabaseListener {
 
     private final Logger logger = LoggerFactory.getLogger(getClass());
     private final String REDO_POSITION = "position";
-
+    private final Map> tableFiledMap = new ConcurrentHashMap<>();
     private LogMiner logMiner;
 
+    @Override
+    public void init() {
+        sourceTable.forEach(table -> tableFiledMap.put(table.getName(), table.getColumn()));
+    }
+
     @Override
     public void start() {
         try {
@@ -56,13 +62,12 @@ public class OracleListener extends AbstractDatabaseListener {
             logMiner = new LogMiner(username, password, url, schema, driverClassName);
             logMiner.setStartScn(containsPos ? Long.parseLong(snapshot.get(REDO_POSITION)) : 0);
             logMiner.registerEventListener((event) -> {
-//                sendSql(event);
                 try {
-                    parseSqlToPk(event);
+                    parseEvent(event);
                 } catch (JSQLParserException e) {
-                    logger.error("不支持sql:" + event.getRedoSql());
+                    logger.error("不支持sql:{}", event.getRedoSql());
                 } catch (Exception e) {
-                    logger.error(e.getMessage());
+                    logger.error(e.getMessage(), e);
                 }
             });
             logMiner.start();
@@ -72,89 +77,39 @@ public class OracleListener extends AbstractDatabaseListener {
         }
     }
 
-    //发送sql解析时间
-    private void sendSql(RedoEvent event) {
-        try {
-            Statement statement = CCJSqlParserUtil.parse(event.getRedoSql());
-            if (statement instanceof Update) {
-                Update update = (Update) statement;
-                sendChangedEvent(new SqlChangedEvent(replaceTableName(update.getTable()),
-                        ConnectorConstant.OPERTION_UPDATE, event.getRedoSql(), null,
-                        event.getScn()));
-                return;
-            }
-
-            if (statement instanceof Insert) {
-                Insert insert = (Insert) statement;
-                sendChangedEvent(new SqlChangedEvent(replaceTableName(insert.getTable()),
-                        ConnectorConstant.OPERTION_INSERT, event.getRedoSql(), null,
-                        event.getScn()));
-                return;
-            }
-
-            if (statement instanceof Delete) {
-                Delete delete = (Delete) statement;
-                sendChangedEvent(new SqlChangedEvent(replaceTableName(delete.getTable()),
-                        ConnectorConstant.OPERTION_DELETE, event.getRedoSql(), null,
-                        event.getScn()));
-                return;
-            }
-
-            if (statement instanceof Alter) {
-                Alter alter = (Alter) statement;
-                sendChangedEvent(new DDLChangedEvent("", replaceTableName(alter.getTable()),
-                        ConnectorConstant.OPERTION_ALTER, event.getRedoSql(), null,
-                        event.getScn()));
-                return;
-            }
-        } catch (JSQLParserException e) {
-            logger.error("不支持sql:" + event.getRedoSql());
-        } catch (Exception e) {
-            logger.error(e.getMessage());
+    /**
+     * 解析时间
+     *
+     * @param event
+     * @throws Exception
+     */
+    private void parseEvent(RedoEvent event) throws Exception {
+        // TODO life 注意拦截子查询, 或修改主键值情况
+        Statement statement = CCJSqlParserUtil.parse(event.getRedoSql());
+        if (statement instanceof Update) {
+            Update update = (Update) statement;
+            UpdateSql parser = new UpdateSql(update);
+            setTable(parser, update.getTable());
+            sendChangedEvent(new RowChangedEvent(parser.getTableName(), ConnectorConstant.OPERTION_UPDATE, parser.parseColumns(), null, event.getScn()));
+            return;
         }
-    }
 
-    //解析sql出来主键数据
-    private void parseSqlToPk(RedoEvent event) throws Exception {
-        Statement statement = CCJSqlParserUtil.parse(event.getRedoSql());
         if (statement instanceof Insert) {
             Insert insert = (Insert) statement;
-            org.dbsyncer.sdk.model.Table table1 = sourceTable.stream()
-                    .filter(x -> x.getName().equals(replaceTableName(insert.getTable())))
-                    .findFirst().orElse(null);
-            if (table1 != null) {
-                Parser parser = new InsertSql(insert, table1.getColumn(),
-                        (DatabaseConnectorInstance) connectorInstance);
-                sendChangedEvent(new RowChangedEvent(replaceTableName(insert.getTable()),
-                        ConnectorConstant.OPERTION_INSERT, parser.parseSql(), null,
-                        event.getScn()));
-            }
+            InsertSql parser = new InsertSql(insert);
+            setTable(parser, insert.getTable());
+            sendChangedEvent(new RowChangedEvent(parser.getTableName(), ConnectorConstant.OPERTION_INSERT, parser.parseColumns(), null, event.getScn()));
+            return;
+        }
 
-        } else if (statement instanceof Update) {
-            Update update = (Update) statement;
-            org.dbsyncer.sdk.model.Table table1 = sourceTable.stream()
-                    .filter(x -> x.getName().equals(replaceTableName(update.getTable())))
-                    .findFirst().orElse(null);
-            if (table1 != null) {
-                Parser parser = new UpdateSql(update, table1.getColumn(),
-                        (DatabaseConnectorInstance) connectorInstance);
-                sendChangedEvent(new RowChangedEvent(replaceTableName(update.getTable()),
-                        ConnectorConstant.OPERTION_UPDATE, parser.parseSql(), null,
-                        event.getScn()));
-            }
-        } else if (statement instanceof Delete) {
+        if (statement instanceof Delete) {
             Delete delete = (Delete) statement;
-            org.dbsyncer.sdk.model.Table table1 = sourceTable.stream()
-                    .filter(x -> x.getName().equals(replaceTableName(delete.getTable())))
-                    .findFirst().orElse(null);
-            if (table1 != null) {
-                Parser parser = new DeleteSql(delete, table1.getColumn());
-                sendChangedEvent(new RowChangedEvent(replaceTableName(delete.getTable()),
-                        ConnectorConstant.OPERTION_DELETE, parser.parseSql(), null,
-                        event.getScn()));
-            }
+            DeleteSql parser = new DeleteSql(delete);
+            setTable(parser, delete.getTable());
+            sendChangedEvent(new RowChangedEvent(parser.getTableName(), ConnectorConstant.OPERTION_DELETE, parser.parseColumns(), null, event.getScn()));
         }
 
+        // TODO ddl
     }
 
     @Override
@@ -173,12 +128,11 @@ public class OracleListener extends AbstractDatabaseListener {
         snapshot.put(REDO_POSITION, String.valueOf(offset.getPosition()));
     }
 
-    private String replaceTableName(Table table) {
-        if (table == null) {
-            return StringUtil.EMPTY;
-        }
-        return StringUtil.replace(table.getName(), StringUtil.DOUBLE_QUOTATION, StringUtil.EMPTY);
+    private AbstractParser setTable(AbstractParser parser, Table table) {
+        parser.setTableName(table == null ? StringUtil.EMPTY : StringUtil.replace(table.getName(), StringUtil.DOUBLE_QUOTATION, StringUtil.EMPTY));
+        parser.setFields(tableFiledMap.get(parser.getTableName()));
+        parser.setInstance(getConnectorInstance());
+        return parser;
     }
 
-
 }
\ No newline at end of file
diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/UpdateSql.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/AbstractParser.java
similarity index 52%
rename from dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/UpdateSql.java
rename to dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/AbstractParser.java
index d96eabd5..3f0aa8a1 100644
--- a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/UpdateSql.java
+++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/AbstractParser.java
@@ -1,68 +1,67 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
 package org.dbsyncer.connector.oracle.logminer.parser;
 
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
 import net.sf.jsqlparser.expression.BinaryExpression;
 import net.sf.jsqlparser.expression.Expression;
 import net.sf.jsqlparser.schema.Column;
-import net.sf.jsqlparser.statement.update.Update;
 import org.dbsyncer.common.util.CollectionUtils;
 import org.dbsyncer.common.util.StringUtil;
 import org.dbsyncer.sdk.connector.database.DatabaseConnectorInstance;
 import org.dbsyncer.sdk.model.Field;
 
-public class UpdateSql implements Parser {
-
-    Update update;
-
-    Map cloumnMap;
-
-    List columns;
-
-    DatabaseConnectorInstance instance;
-
-    public UpdateSql(Update update, List columns,
-            DatabaseConnectorInstance instance) {
-        this.update = update;
-        this.columns = columns;
-        this.instance = instance;
-        cloumnMap = new HashMap<>();
-    }
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
 
-    @Override
-    public List parseSql() {
-        findColumn(update.getWhere());
-        String toSqlString = toSql(
-                StringUtil.replace(update.getTable().getName(), StringUtil.DOUBLE_QUOTATION,
-                        StringUtil.EMPTY));
-        List data = ToResult(toSqlString);
-        return data;
-    }
+/**
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2023-12-14 22:39
+ */
+public abstract class AbstractParser implements Parser {
 
+    protected Map columnMap = new HashMap<>();
+    protected String tableName;
+    protected List fields;
+    protected DatabaseConnectorInstance instance;
 
-    private void findColumn(Expression expression) {
+    public void findColumn(Expression expression) {
         BinaryExpression binaryExpression = (BinaryExpression) expression;
         if (binaryExpression.getLeftExpression() instanceof Column) {
             Column column = (Column) binaryExpression.getLeftExpression();
             String value = binaryExpression.getRightExpression().toString();
-            cloumnMap.put(StringUtil.replace(column.getColumnName(), StringUtil.DOUBLE_QUOTATION,
-                    StringUtil.EMPTY), value);
+            columnMap.put(StringUtil.replace(column.getColumnName(), StringUtil.DOUBLE_QUOTATION, StringUtil.EMPTY), value);
             return;
         }
         findColumn(binaryExpression.getLeftExpression());
         findColumn(binaryExpression.getRightExpression());
     }
 
-    private String toSql(String tableName) {
-        StringBuilder sql = new StringBuilder("SELECT *");
-        sql.append(" FROM ").append(tableName).append(" WHERE ");
+    // 从主键解析出来的map装载成sql并运行sql找出对应的数据
+    public List getColumnsFromDB() {
+        List> rows = instance.execute(databaseTemplate -> databaseTemplate.queryForList(getRowByPk()));
+        List list = new LinkedList<>();
+        if (!CollectionUtils.isEmpty(rows)) {
+            rows.forEach(map -> {
+                for (String key : map.keySet()) {
+                    list.add(map.get(key));
+                }
+            });
+            return list;
+        }
+        return Collections.EMPTY_LIST;
+    }
+
+    private String getRowByPk() {
+        StringBuilder sql = new StringBuilder("SELECT * FROM ").append(getTableName()).append(" WHERE ");
         int pkCount = 0;
-        for (Field field : columns) {
+        for (Field field : fields) {
             if (field.isPk()) {
-                String value = cloumnMap.get(field.getName());
+                String value = columnMap.get(field.getName());
                 if (StringUtil.isNotBlank(value)) {
                     if (pkCount > 0) {
                         sql.append(" AND ");
@@ -75,19 +74,20 @@ public class UpdateSql implements Parser {
         return sql.toString();
     }
 
-    //从主键解析出来的map装载成sql并运行sql找出对应的数据
-    private List ToResult(String sql) {
-        List> results = instance.execute(
-                databaseTemplate -> databaseTemplate.queryForList(sql));
-        List list = new LinkedList<>();
-        if (!CollectionUtils.isEmpty(results)) {
-            results.forEach(map -> {
-                for (String key : map.keySet()) {
-                    list.add(map.get(key));
-                }
-            });
-            return list;
-        }
-        return Collections.EMPTY_LIST;
+    @Override
+    public String getTableName() {
+        return tableName;
+    }
+
+    public void setTableName(String tableName) {
+        this.tableName = tableName;
+    }
+
+    public void setFields(List fields) {
+        this.fields = fields;
+    }
+
+    public void setInstance(DatabaseConnectorInstance instance) {
+        this.instance = instance;
     }
-}
+}
\ No newline at end of file
diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/DeleteSql.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/DeleteSql.java
deleted file mode 100644
index f67df1a2..00000000
--- a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/DeleteSql.java
+++ /dev/null
@@ -1,75 +0,0 @@
-package org.dbsyncer.connector.oracle.logminer.parser;
-
-import java.math.BigInteger;
-import java.sql.Types;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import net.sf.jsqlparser.expression.BinaryExpression;
-import net.sf.jsqlparser.expression.Expression;
-import net.sf.jsqlparser.schema.Column;
-import net.sf.jsqlparser.statement.delete.Delete;
-import org.dbsyncer.common.util.StringUtil;
-import org.dbsyncer.sdk.model.Field;
-
-/**
- * @author : life
- * @Description :
- * @Param :
- * @return :
- * @date : 2023/12/14  14:58
- */
-public class DeleteSql implements Parser {
-
-    Delete delete;
-
-    Map cloumnMap;
-
-    List columns;
-
-    public DeleteSql(Delete delete,  List columns) {
-        this.delete = delete;
-        this.columns = columns;
-        cloumnMap = new HashMap<>();
-    }
-
-    @Override
-    public List parseSql() {
-        findColumn(delete.getWhere());
-        return columnMapToData();
-    }
-
-    private void findColumn(Expression expression) {
-        BinaryExpression binaryExpression = (BinaryExpression) expression;
-        if (binaryExpression.getLeftExpression() instanceof Column) {
-            Column column = (Column) binaryExpression.getLeftExpression();
-            String value = binaryExpression.getRightExpression().toString();
-            cloumnMap.put(StringUtil.replace(column.getColumnName(), StringUtil.DOUBLE_QUOTATION,
-                    StringUtil.EMPTY), value);
-            return;
-        }
-        findColumn(binaryExpression.getLeftExpression());
-        findColumn(binaryExpression.getRightExpression());
-    }
-
-    private List columnMapToData(){
-        List data = new LinkedList<>();
-        for (Field field : columns) {
-            if (field.isPk()) {
-                Object value = cloumnMap.get(field.getName());
-                switch (field.getType()) {
-                    case Types.DECIMAL:
-                        value = new BigInteger(
-                                StringUtil.replace(value.toString(), StringUtil.SINGLE_QUOTATION,
-                                        StringUtil.EMPTY));
-                }
-                data.add(value);
-            } else {
-                data.add(null);
-            }
-        }
-        return data;
-    }
-
-}
diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/InsertSql.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/InsertSql.java
deleted file mode 100644
index f9753d28..00000000
--- a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/InsertSql.java
+++ /dev/null
@@ -1,79 +0,0 @@
-package org.dbsyncer.connector.oracle.logminer.parser;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import net.sf.jsqlparser.expression.operators.relational.ExpressionList;
-import net.sf.jsqlparser.schema.Column;
-import net.sf.jsqlparser.statement.insert.Insert;
-import org.dbsyncer.common.util.CollectionUtils;
-import org.dbsyncer.common.util.StringUtil;
-import org.dbsyncer.sdk.connector.database.DatabaseConnectorInstance;
-import org.dbsyncer.sdk.model.Field;
-
-public class InsertSql implements Parser{
-
-    Insert insert;
-
-    Map cloumnMap;
-
-    List columns;
-
-    DatabaseConnectorInstance instance;
-
-    public InsertSql(Insert insert, List columns,
-            DatabaseConnectorInstance instance) {
-        this.insert = insert;
-        this.columns = columns;
-        this.instance = instance;
-        cloumnMap = new HashMap<>();
-    }
-
-    @Override
-    public List parseSql() {
-        List columns= insert.getColumns();
-        ExpressionList values = insert.getSelect().getValues().getExpressions();
-        for (int i = 0; i < columns.size(); i++) {
-            cloumnMap.put(StringUtil.replace(columns.get(i).getColumnName(), StringUtil.DOUBLE_QUOTATION, StringUtil.EMPTY),values.get(i).toString());
-        }
-        String toSqlString = toSql(StringUtil.replace(insert.getTable().getName(), StringUtil.DOUBLE_QUOTATION, StringUtil.EMPTY));
-        List data= ToResult(toSqlString);
-        return data;
-    }
-
-    private String toSql(String tableName){
-        StringBuilder sql = new StringBuilder("SELECT *");
-        sql.append(" FROM ").append(tableName).append(" WHERE ");
-        int pkCount = 0;
-        for (Field field: columns) {
-            if (field.isPk()){
-                String value = cloumnMap.get(field.getName());
-                if (StringUtil.isNotBlank(value)){
-                    if (pkCount > 0){
-                        sql.append(" AND ");
-                    }
-                    sql.append(field.getName()).append("=").append(value);
-                    pkCount ++;
-                }
-            }
-        }
-        return sql.toString();
-    }
-
-    //从主键解析出来的map装载成sql并运行sql找出对应的数据
-    private List ToResult(String sql){
-        List> results = instance.execute(databaseTemplate -> databaseTemplate.queryForList(sql));
-        List list = new LinkedList<>();
-        if (!CollectionUtils.isEmpty(results)){
-            results.forEach(map->{
-                for (String key:map.keySet()) {
-                    list.add(map.get(key));
-                }
-            });
-            return list;
-        }
-        return Collections.EMPTY_LIST;
-    }
-}
diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/Parser.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/Parser.java
index 0585b404..670f21fb 100644
--- a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/Parser.java
+++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/Parser.java
@@ -1,8 +1,13 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
 package org.dbsyncer.connector.oracle.logminer.parser;
 
 import java.util.List;
 
 public interface Parser {
 
-    List parseSql();
+    String getTableName();
+
+    List parseColumns();
 }
diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/impl/DeleteSql.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/impl/DeleteSql.java
new file mode 100644
index 00000000..ad34384a
--- /dev/null
+++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/impl/DeleteSql.java
@@ -0,0 +1,49 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
+package org.dbsyncer.connector.oracle.logminer.parser.impl;
+
+import net.sf.jsqlparser.statement.delete.Delete;
+import org.dbsyncer.common.util.StringUtil;
+import org.dbsyncer.connector.oracle.logminer.parser.AbstractParser;
+import org.dbsyncer.sdk.model.Field;
+
+import java.math.BigInteger;
+import java.sql.Types;
+import java.util.LinkedList;
+import java.util.List;
+
+/**
+ * @Author life
+ * @Version 1.0.0
+ * @Date 2023-12-14 14:58
+ */
+public class DeleteSql extends AbstractParser {
+
+    private Delete delete;
+
+    public DeleteSql(Delete delete) {
+        this.delete = delete;
+    }
+
+    @Override
+    public List parseColumns() {
+        findColumn(delete.getWhere());
+        List data = new LinkedList<>();
+        for (Field field : fields) {
+            if (field.isPk()) {
+                Object value = columnMap.get(field.getName());
+                switch (field.getType()) {
+                    case Types.DECIMAL:
+                        value = new BigInteger(StringUtil.replace(value.toString(), StringUtil.SINGLE_QUOTATION, StringUtil.EMPTY));
+                        break;
+                }
+                data.add(value);
+            } else {
+                data.add(null);
+            }
+        }
+        return data;
+    }
+
+}
diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/impl/InsertSql.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/impl/InsertSql.java
new file mode 100644
index 00000000..01c325c9
--- /dev/null
+++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/impl/InsertSql.java
@@ -0,0 +1,37 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
+package org.dbsyncer.connector.oracle.logminer.parser.impl;
+
+import net.sf.jsqlparser.expression.operators.relational.ExpressionList;
+import net.sf.jsqlparser.schema.Column;
+import net.sf.jsqlparser.statement.insert.Insert;
+import org.dbsyncer.common.util.StringUtil;
+import org.dbsyncer.connector.oracle.logminer.parser.AbstractParser;
+
+import java.util.List;
+
+/**
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2023-12-14 14:58
+ */
+public class InsertSql extends AbstractParser {
+
+    private Insert insert;
+
+    public InsertSql(Insert insert) {
+        this.insert = insert;
+    }
+
+    @Override
+    public List parseColumns() {
+        List columns = insert.getColumns();
+        ExpressionList values = insert.getSelect().getValues().getExpressions();
+        for (int i = 0; i < columns.size(); i++) {
+            columnMap.put(StringUtil.replace(columns.get(i).getColumnName(), StringUtil.DOUBLE_QUOTATION, StringUtil.EMPTY), values.get(i).toString());
+        }
+        return getColumnsFromDB();
+    }
+
+}
diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/impl/UpdateSql.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/impl/UpdateSql.java
new file mode 100644
index 00000000..73e10bff
--- /dev/null
+++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/impl/UpdateSql.java
@@ -0,0 +1,30 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
+package org.dbsyncer.connector.oracle.logminer.parser.impl;
+
+import net.sf.jsqlparser.statement.update.Update;
+import org.dbsyncer.connector.oracle.logminer.parser.AbstractParser;
+
+import java.util.List;
+
+/**
+ * @Author AE86
+ * @Version 1.0.0
+ * @Date 2023-12-14 14:58
+ */
+public class UpdateSql extends AbstractParser {
+
+    private Update update;
+
+    public UpdateSql(Update update) {
+        this.update = update;
+    }
+
+    @Override
+    public List parseColumns() {
+        findColumn(update.getWhere());
+        return getColumnsFromDB();
+    }
+
+}
diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/IncrementPuller.java b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/IncrementPuller.java
index 8386ab8d..5b399027 100644
--- a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/IncrementPuller.java
+++ b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/IncrementPuller.java
@@ -184,6 +184,7 @@ public final class IncrementPuller extends AbstractPuller implements Application
             abstractListener.setMetaId(meta.getId());
         }
 
+        listener.init();
         return listener;
     }
 
diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/AbstractDatabaseListener.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/AbstractDatabaseListener.java
index 4e93cef6..6b8fe224 100644
--- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/AbstractDatabaseListener.java
+++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/AbstractDatabaseListener.java
@@ -1,3 +1,6 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
 package org.dbsyncer.sdk.listener;
 
 import org.dbsyncer.common.util.CollectionUtils;
@@ -23,7 +26,7 @@ import java.util.concurrent.ConcurrentHashMap;
  * @version 1.0.0
  * @date 2022/5/29 21:46
  */
-public abstract class AbstractDatabaseListener extends AbstractListener {
+public abstract class AbstractDatabaseListener extends AbstractListener {
 
     /**
      * 自定义SQL，支持1对多
diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/AbstractListener.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/AbstractListener.java
index f0a06462..24b0d2be 100644
--- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/AbstractListener.java
+++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/AbstractListener.java
@@ -1,3 +1,6 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
 package org.dbsyncer.sdk.listener;
 
 import org.dbsyncer.common.util.CollectionUtils;
@@ -24,7 +27,7 @@ import java.util.concurrent.TimeUnit;
  * @Author AE86
  * @Date 2020-05-25 22:35
  */
-public abstract class AbstractListener implements Listener {
+public abstract class AbstractListener implements Listener {
 
     private final Logger logger = LoggerFactory.getLogger(getClass());
     private final int FLUSH_DELAYED_SECONDS = 20;
@@ -54,7 +57,7 @@ public abstract class AbstractListener implements Listener {
                     break;
                 case ConnectorConstant.OPERTION_INSERT:
                     // 是否支持监听新增事件
-                    processEvent(!listenerConfig.isBanInsert(), event);
+                     processEvent(!listenerConfig.isBanInsert(), event);
                     break;
                 case ConnectorConstant.OPERTION_DELETE:
                     // 是否支持监听删除事件
@@ -127,6 +130,10 @@ public abstract class AbstractListener implements Listener {
         this.connectorService = connectorService;
     }
 
+    public C getConnectorInstance() {
+        return (C) connectorInstance;
+    }
+
     public void setScheduledTaskService(ScheduledTaskService scheduledTaskService) {
         this.scheduledTaskService = scheduledTaskService;
     }
diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/AbstractQuartzListener.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/AbstractQuartzListener.java
index ee386cea..c9f63fdf 100644
--- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/AbstractQuartzListener.java
+++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/AbstractQuartzListener.java
@@ -1,3 +1,6 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
 package org.dbsyncer.sdk.listener;
 
 import org.dbsyncer.common.model.Result;
@@ -105,7 +108,7 @@ public abstract class AbstractQuartzListener extends AbstractListener implements
         // 检查增量点
         Point point = checkLastPoint(command, index);
         int pageIndex = 1;
-        Object[] cursors = PrimaryKeyUtil.getLastCursors(snapshot.get(index + CURSOR));
+        Object[] cursors = PrimaryKeyUtil.getLastCursors((String) snapshot.get(index + CURSOR));
 
         while (running) {
             ReaderConfig readerConfig = new ReaderConfig(table, point.getCommand(), point.getArgs(), supportedCursor, cursors, pageIndex++, READ_NUM);
diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/DatabaseQuartzListener.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/DatabaseQuartzListener.java
index ceaf66a0..b2d29178 100644
--- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/DatabaseQuartzListener.java
+++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/DatabaseQuartzListener.java
@@ -1,3 +1,6 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
 package org.dbsyncer.sdk.listener;
 
 import org.dbsyncer.common.util.CollectionUtils;
@@ -79,7 +82,7 @@ public final class DatabaseQuartzListener extends AbstractQuartzListener {
                 }
 
                 // 读取历史增量点
-                Object val = f.getObject(snapshot.get(key));
+                Object val = f.getObject((String) snapshot.get(key));
                 point.addArg(val);
                 point.setBeginKey(key);
                 point.setBeginValue(f.toString(f.getObject()));
diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/Listener.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/Listener.java
index abe1681f..8d08418e 100644
--- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/Listener.java
+++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/listener/Listener.java
@@ -13,6 +13,12 @@ import org.dbsyncer.sdk.model.ChangedOffset;
  * @Date 2023-11-21 22:48
  */
 public interface Listener {
+
+    /**
+     * 初始化
+     */
+    default void init(){};
+
     /**
      * 启动定时/日志抽取任务
      */
-- 
Gitee


From a05f5c217bfc0d92ee08ae6478297e8fbfbe6d1c Mon Sep 17 00:00:00 2001
From: AE86 <836391306@qq.com>
Date: Thu, 14 Dec 2023 23:42:23 +0800
Subject: [PATCH 77/98] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=BB=A3=E7=A0=81?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../java/org/dbsyncer/connector/oracle/cdc/OracleListener.java  | 2 +-
 .../connector/oracle/logminer/parser/impl/DeleteSql.java        | 2 +-
 .../connector/oracle/logminer/parser/impl/InsertSql.java        | 2 +-
 .../connector/oracle/logminer/parser/impl/UpdateSql.java        | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java
index ff01ec36..554e29ac 100644
--- a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java
+++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java
@@ -78,7 +78,7 @@ public class OracleListener extends AbstractDatabaseListener {
     }
 
     /**
-     * 解析时间
+     * 解析事件
      *
      * @param event
      * @throws Exception
diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/impl/DeleteSql.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/impl/DeleteSql.java
index ad34384a..3059442a 100644
--- a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/impl/DeleteSql.java
+++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/impl/DeleteSql.java
@@ -46,4 +46,4 @@ public class DeleteSql extends AbstractParser {
         return data;
     }
 
-}
+}
\ No newline at end of file
diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/impl/InsertSql.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/impl/InsertSql.java
index 01c325c9..3218bcc1 100644
--- a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/impl/InsertSql.java
+++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/impl/InsertSql.java
@@ -34,4 +34,4 @@ public class InsertSql extends AbstractParser {
         return getColumnsFromDB();
     }
 
-}
+}
\ No newline at end of file
diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/impl/UpdateSql.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/impl/UpdateSql.java
index 73e10bff..e019a7e8 100644
--- a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/impl/UpdateSql.java
+++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/impl/UpdateSql.java
@@ -27,4 +27,4 @@ public class UpdateSql extends AbstractParser {
         return getColumnsFromDB();
     }
 
-}
+}
\ No newline at end of file
-- 
Gitee


From b3aeb34767df05d7407b0766353a9be5abb55f02 Mon Sep 17 00:00:00 2001
From: AE86 <836391306@qq.com>
Date: Fri, 15 Dec 2023 00:24:28 +0800
Subject: [PATCH 78/98] add shell

---
 version.sh | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 version.sh

diff --git a/version.sh b/version.sh
new file mode 100644
index 00000000..cbc4353e
--- /dev/null
+++ b/version.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+VERSION=1.2.7_$(date +"%m%d")
+read -p "Please enter a new version number($VERSION)：" APP_VERSION
+if [ -z "$APP_VERSION" ]; then
+  APP_VERSION=$VERSION
+fi
+
+echo "Clean Project ..."
+mvn clean -f pom.xml
+
+echo "Update version ..."
+mvn versions:set -DnewVersion=$APP_VERSION -DprocessAllModules=true -DallowSnapshots=true -DgenerateBackupPoms=false
+mvn -N versions:update-child-modules
+mvn versions:commit
\ No newline at end of file
-- 
Gitee


From 0653d763d8390fc104f2be6ba226be2e53adb086 Mon Sep 17 00:00:00 2001
From: AE86 <836391306@qq.com>
Date: Fri, 15 Dec 2023 01:37:06 +0800
Subject: [PATCH 79/98] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=BB=A3=E7=A0=81?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../biz/impl/DataSyncServiceImpl.java         |  2 +-
 .../org/dbsyncer/biz/impl/MetricReporter.java |  2 +-
 .../dbsyncer/biz/impl/MonitorServiceImpl.java |  2 +-
 .../connector/mysql/MySQLConnector.java       |  2 +-
 .../connector/mysql/cdc/MySQLListener.java    |  2 +-
 .../connector/oracle/cdc/OracleListener.java  |  3 +-
 .../org/dbsyncer/manager/impl/FullPuller.java |  9 +-
 .../manager/impl/IncrementPuller.java         |  9 +-
 .../manager/impl/PreloadTemplate.java         |  4 +-
 .../{ => impl}/PersistenceCommand.java        |  7 +-
 .../command/{ => impl}/PreloadCommand.java    |  5 +-
 .../dbsyncer/parser/flush/FlushService.java   | 27 ------
 .../parser/flush/impl/FlushServiceImpl.java   | 95 -------------------
 .../flush/impl/GeneralBufferActuator.java     |  6 +-
 .../flush/impl/StorageBufferActuator.java     | 11 ++-
 .../flush/impl/TableGroupBufferActuator.java  |  9 +-
 .../dbsyncer/parser/impl/LogServiceImpl.java  | 47 ++++++++-
 .../parser/impl/OperationTemplate.java        |  4 +-
 .../impl/FlushStrategyImpl.java               | 50 ++++++++--
 .../dbsyncer/sdk/spi/ConnectorService.java    |  1 +
 .../sdk/storage/AbstractStorageService.java   |  1 -
 .../sdk/{spi => storage}/StorageService.java  |  2 +-
 .../storage/StorageSupportConfiguration.java  |  2 +-
 .../src/main/resources/application.properties |  4 +-
 24 files changed, 134 insertions(+), 172 deletions(-)
 rename dbsyncer-parser/src/main/java/org/dbsyncer/parser/command/{ => impl}/PersistenceCommand.java (78%)
 rename dbsyncer-parser/src/main/java/org/dbsyncer/parser/command/{ => impl}/PreloadCommand.java (92%)
 delete mode 100644 dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/FlushService.java
 delete mode 100644 dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/FlushServiceImpl.java
 rename dbsyncer-parser/src/main/java/org/dbsyncer/parser/{flush => strategy}/impl/FlushStrategyImpl.java (57%)
 rename dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/{spi => storage}/StorageService.java (98%)

diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/DataSyncServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/DataSyncServiceImpl.java
index 60904792..3a711037 100644
--- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/DataSyncServiceImpl.java
+++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/DataSyncServiceImpl.java
@@ -21,7 +21,7 @@ import org.dbsyncer.sdk.listener.event.RowChangedEvent;
 import org.dbsyncer.sdk.model.Field;
 import org.dbsyncer.sdk.filter.FieldResolver;
 import org.dbsyncer.sdk.filter.Query;
-import org.dbsyncer.sdk.spi.StorageService;
+import org.dbsyncer.sdk.storage.StorageService;
 import org.dbsyncer.storage.binlog.proto.BinlogMap;
 import org.dbsyncer.sdk.constant.ConfigConstant;
 import org.dbsyncer.storage.util.BinlogMessageUtil;
diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MetricReporter.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MetricReporter.java
index ec33bf4a..a25ccf0a 100644
--- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MetricReporter.java
+++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MetricReporter.java
@@ -26,7 +26,7 @@ import org.dbsyncer.parser.model.TableGroup;
 import org.dbsyncer.sdk.constant.ConnectorConstant;
 import org.dbsyncer.sdk.enums.StorageEnum;
 import org.dbsyncer.sdk.filter.Query;
-import org.dbsyncer.sdk.spi.StorageService;
+import org.dbsyncer.sdk.storage.StorageService;
 import org.dbsyncer.sdk.constant.ConfigConstant;
 import org.dbsyncer.storage.enums.StorageDataStatusEnum;
 import org.slf4j.Logger;
diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java
index 3b0f051f..6dadb82f 100644
--- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java
+++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MonitorServiceImpl.java
@@ -44,7 +44,7 @@ import org.dbsyncer.sdk.filter.BooleanFilter;
 import org.dbsyncer.sdk.filter.FieldResolver;
 import org.dbsyncer.sdk.filter.Query;
 import org.dbsyncer.sdk.filter.impl.LongFilter;
-import org.dbsyncer.sdk.spi.StorageService;
+import org.dbsyncer.sdk.storage.StorageService;
 import org.dbsyncer.sdk.constant.ConfigConstant;
 import org.dbsyncer.storage.enums.StorageDataStatusEnum;
 import org.slf4j.Logger;
diff --git a/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/MySQLConnector.java b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/MySQLConnector.java
index 30283c48..365acc66 100644
--- a/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/MySQLConnector.java
+++ b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/MySQLConnector.java
@@ -15,7 +15,7 @@ import org.dbsyncer.sdk.constant.DatabaseConstant;
 import org.dbsyncer.sdk.enums.ListenerTypeEnum;
 import org.dbsyncer.sdk.listener.Listener;
 import org.dbsyncer.sdk.model.PageSql;
-import org.dbsyncer.sdk.spi.StorageService;
+import org.dbsyncer.sdk.storage.StorageService;
 import org.dbsyncer.sdk.util.PrimaryKeyUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
diff --git a/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/cdc/MySQLListener.java b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/cdc/MySQLListener.java
index cfcf75ec..b4497e5a 100644
--- a/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/cdc/MySQLListener.java
+++ b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/cdc/MySQLListener.java
@@ -104,7 +104,7 @@ public class MySQLListener extends AbstractDatabaseListener {
     }
 
     private void run() throws Exception {
-        final DatabaseConfig config = (DatabaseConfig) connectorConfig;
+        final DatabaseConfig config = getConnectorInstance().getConfig();
         if (StringUtil.isBlank(config.getUrl())) {
             throw new MySQLException("url is invalid");
         }
diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java
index 554e29ac..4bb58c43 100644
--- a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java
+++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java
@@ -52,7 +52,8 @@ public class OracleListener extends AbstractDatabaseListener {
     @Override
     public void start() {
         try {
-            final DatabaseConfig config = (DatabaseConfig) connectorConfig;
+            // TODO [increment-worker-1184659161326161921] 这里应该单独启动一个线程，线程名要有一定意义，如:binlog-parser-127.0.0.1:3306_123，便于监控排查问题
+            final DatabaseConfig config = getConnectorInstance().getConfig();
             String driverClassName = config.getDriverClassName();
             String username = config.getUsername();
             String password = config.getPassword();
diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/FullPuller.java b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/FullPuller.java
index 44ffa5ae..ec19d138 100644
--- a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/FullPuller.java
+++ b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/FullPuller.java
@@ -1,3 +1,6 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
 package org.dbsyncer.manager.impl;
 
 import org.dbsyncer.common.util.NumberUtil;
@@ -32,9 +35,9 @@ import java.util.concurrent.Executors;
 /**
  * 全量同步
  *
- * @author AE86
- * @version 1.0.0
- * @date 2020/04/26 15:28
+ * @Version 1.0.0
+ * @Author AE86
+ * @Date 2020-04-26 15:28
  */
 @Component
 public final class FullPuller extends AbstractPuller implements ApplicationListener {
diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/IncrementPuller.java b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/IncrementPuller.java
index 5b399027..3e95beb1 100644
--- a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/IncrementPuller.java
+++ b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/IncrementPuller.java
@@ -1,3 +1,6 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
 package org.dbsyncer.manager.impl;
 
 import org.dbsyncer.common.util.CollectionUtils;
@@ -46,9 +49,9 @@ import java.util.stream.Collectors;
 /**
  * 增量同步
  *
- * @author AE86
- * @version 1.0.0
- * @date 2020/04/26 15:28
+ * @Version 1.0.0
+ * @Author AE86
+ * @Date 2020-04-26 15:28
  */
 @Component
 public final class IncrementPuller extends AbstractPuller implements ApplicationListener, ScheduledTaskJob {
diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/PreloadTemplate.java b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/PreloadTemplate.java
index dca14652..145b5c19 100644
--- a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/PreloadTemplate.java
+++ b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/PreloadTemplate.java
@@ -11,7 +11,7 @@ import org.dbsyncer.manager.ManagerFactory;
 import org.dbsyncer.parser.LogService;
 import org.dbsyncer.parser.LogType;
 import org.dbsyncer.parser.ProfileComponent;
-import org.dbsyncer.parser.command.PreloadCommand;
+import org.dbsyncer.parser.command.impl.PreloadCommand;
 import org.dbsyncer.parser.enums.CommandEnum;
 import org.dbsyncer.parser.enums.GroupStrategyEnum;
 import org.dbsyncer.parser.enums.MetaEnum;
@@ -26,7 +26,7 @@ import org.dbsyncer.sdk.connector.ConnectorInstance;
 import org.dbsyncer.sdk.constant.ConfigConstant;
 import org.dbsyncer.sdk.enums.StorageEnum;
 import org.dbsyncer.sdk.filter.Query;
-import org.dbsyncer.sdk.spi.StorageService;
+import org.dbsyncer.sdk.storage.StorageService;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.context.ApplicationListener;
diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/command/PersistenceCommand.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/command/impl/PersistenceCommand.java
similarity index 78%
rename from dbsyncer-parser/src/main/java/org/dbsyncer/parser/command/PersistenceCommand.java
rename to dbsyncer-parser/src/main/java/org/dbsyncer/parser/command/impl/PersistenceCommand.java
index 74ef06c7..f26ba476 100644
--- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/command/PersistenceCommand.java
+++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/command/impl/PersistenceCommand.java
@@ -1,10 +1,11 @@
 /**
  * DBSyncer Copyright 2020-2023 All Rights Reserved.
  */
-package org.dbsyncer.parser.command;
+package org.dbsyncer.parser.command.impl;
 
+import org.dbsyncer.parser.command.Command;
 import org.dbsyncer.sdk.enums.StorageEnum;
-import org.dbsyncer.sdk.spi.StorageService;
+import org.dbsyncer.sdk.storage.StorageService;
 
 import java.util.Map;
 
@@ -15,7 +16,7 @@ import java.util.Map;
  * @Author AE86
  * @Date 2023-11-12 01:32
  */
-public class PersistenceCommand implements Command {
+public final class PersistenceCommand implements Command {
 
     private StorageService storageService;
 
diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/command/PreloadCommand.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/command/impl/PreloadCommand.java
similarity index 92%
rename from dbsyncer-parser/src/main/java/org/dbsyncer/parser/command/PreloadCommand.java
rename to dbsyncer-parser/src/main/java/org/dbsyncer/parser/command/impl/PreloadCommand.java
index fba38307..9b69e08b 100644
--- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/command/PreloadCommand.java
+++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/command/impl/PreloadCommand.java
@@ -1,9 +1,10 @@
 /**
  * DBSyncer Copyright 2020-2023 All Rights Reserved.
  */
-package org.dbsyncer.parser.command;
+package org.dbsyncer.parser.command.impl;
 
 import org.dbsyncer.parser.ProfileComponent;
+import org.dbsyncer.parser.command.Command;
 import org.dbsyncer.parser.model.Connector;
 import org.dbsyncer.parser.model.Mapping;
 import org.dbsyncer.parser.model.Meta;
@@ -19,7 +20,7 @@ import org.dbsyncer.parser.model.UserConfig;
  * @Author AE86
  * @Date 2023-11-12 01:32
  */
-public class PreloadCommand implements Command {
+public final class PreloadCommand implements Command {
 
     private ProfileComponent profileComponent;
 
diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/FlushService.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/FlushService.java
deleted file mode 100644
index 6a0a998f..00000000
--- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/FlushService.java
+++ /dev/null
@@ -1,27 +0,0 @@
-package org.dbsyncer.parser.flush;
-
-import java.util.List;
-import java.util.Map;
-
-public interface FlushService {
-
-    /**
-     * 记录错误日志
-     *
-     * @param type
-     * @param error
-     */
-    void asyncWrite(String type, String error);
-
-    /**
-     * 记录数据
-     *
-     * @param metaId
-     * @param tableGroupId
-     * @param targetTableGroupName
-     * @param event
-     * @param success
-     * @param data
-     */
-    void asyncWrite(String metaId, String tableGroupId, String targetTableGroupName, String event, boolean success, List data, String error);
-}
\ No newline at end of file
diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/FlushServiceImpl.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/FlushServiceImpl.java
deleted file mode 100644
index 1d3a09f3..00000000
--- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/FlushServiceImpl.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/**
- * DBSyncer Copyright 2020-2023 All Rights Reserved.
- */
-package org.dbsyncer.parser.flush.impl;
-
-import org.dbsyncer.common.util.StringUtil;
-import org.dbsyncer.parser.ProfileComponent;
-import org.dbsyncer.parser.flush.BufferActuator;
-import org.dbsyncer.parser.flush.FlushService;
-import org.dbsyncer.parser.model.StorageRequest;
-import org.dbsyncer.parser.model.SystemConfig;
-import org.dbsyncer.sdk.constant.ConfigConstant;
-import org.dbsyncer.sdk.enums.StorageEnum;
-import org.dbsyncer.sdk.spi.StorageService;
-import org.dbsyncer.storage.enums.StorageDataStatusEnum;
-import org.dbsyncer.storage.impl.SnowflakeIdWorker;
-import org.dbsyncer.storage.util.BinlogMessageUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.stereotype.Component;
-
-import javax.annotation.Resource;
-import java.time.Instant;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.Executor;
-
-/**
- * 持久化
- * 
    全量或增量数据
    
- * 
    系统日志
    
- *
- * @author AE86
- * @version 1.0.0
- * @date 2020/05/19 18:38
- */
-@Component
-public final class FlushServiceImpl implements FlushService {
-
-    private final Logger logger = LoggerFactory.getLogger(getClass());
-
-    @Resource
-    private StorageService storageService;
-
-    @Resource
-    private SnowflakeIdWorker snowflakeIdWorker;
-
-    @Resource
-    private BufferActuator storageBufferActuator;
-
-    @Resource
-    private Executor storageExecutor;
-
-    @Resource
-    private ProfileComponent profileComponent;
-
-    @Override
-    public void asyncWrite(String type, String error) {
-        storageExecutor.execute(() -> {
-            Map params = new HashMap();
-            params.put(ConfigConstant.CONFIG_MODEL_ID, String.valueOf(snowflakeIdWorker.nextId()));
-            params.put(ConfigConstant.CONFIG_MODEL_TYPE, type);
-            params.put(ConfigConstant.CONFIG_MODEL_JSON, StringUtil.substring(error, 0, getSystemConfig().getMaxStorageErrorLength()));
-            params.put(ConfigConstant.CONFIG_MODEL_CREATE_TIME, Instant.now().toEpochMilli());
-            storageService.add(StorageEnum.LOG, params);
-        });
-    }
-
-    @Override
-    public void asyncWrite(String metaId, String tableGroupId, String targetTableGroupName, String event, boolean success, List data, String error) {
-        long now = Instant.now().toEpochMilli();
-        data.forEach(r -> {
-            Map row = new HashMap();
-            row.put(ConfigConstant.CONFIG_MODEL_ID, String.valueOf(snowflakeIdWorker.nextId()));
-            row.put(ConfigConstant.DATA_SUCCESS, success ? StorageDataStatusEnum.SUCCESS.getValue() : StorageDataStatusEnum.FAIL.getValue());
-            row.put(ConfigConstant.DATA_TABLE_GROUP_ID, tableGroupId);
-            row.put(ConfigConstant.DATA_TARGET_TABLE_NAME, targetTableGroupName);
-            row.put(ConfigConstant.DATA_EVENT, event);
-            row.put(ConfigConstant.DATA_ERROR, error);
-            row.put(ConfigConstant.CONFIG_MODEL_CREATE_TIME, now);
-            try {
-                byte[] bytes = BinlogMessageUtil.toBinlogMap(r).toByteArray();
-                row.put(ConfigConstant.BINLOG_DATA, bytes);
-            } catch (Exception e) {
-                logger.warn("可能存在Blob或inputStream大文件类型, 无法序列化:{}", r);
-            }
-            storageBufferActuator.offer(new StorageRequest(metaId, row));
-        });
-    }
-
-    private SystemConfig getSystemConfig() {
-        return profileComponent.getSystemConfig();
-    }
-}
\ No newline at end of file
diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/GeneralBufferActuator.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/GeneralBufferActuator.java
index 9ee1e570..ee70d2fa 100644
--- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/GeneralBufferActuator.java
+++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/GeneralBufferActuator.java
@@ -53,9 +53,9 @@ import java.util.concurrent.Executor;
 /**
  * 通用执行器（单线程消费，多线程批量写，按序执行）
  *
- * @author AE86
- * @version 1.0.0
- * @date 2022/3/27 16:50
+ * @Version 1.0.0
+ * @Author AE86
+ * @Date 2022-03-27 16:50
  */
 @Component
 public class GeneralBufferActuator extends AbstractBufferActuator {
diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/StorageBufferActuator.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/StorageBufferActuator.java
index c2c163bf..211ffc2f 100644
--- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/StorageBufferActuator.java
+++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/StorageBufferActuator.java
@@ -1,3 +1,6 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
 package org.dbsyncer.parser.flush.impl;
 
 import org.dbsyncer.common.config.StorageConfig;
@@ -8,7 +11,7 @@ import org.dbsyncer.parser.model.Meta;
 import org.dbsyncer.parser.model.StorageRequest;
 import org.dbsyncer.parser.model.StorageResponse;
 import org.dbsyncer.sdk.enums.StorageEnum;
-import org.dbsyncer.sdk.spi.StorageService;
+import org.dbsyncer.sdk.storage.StorageService;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.stereotype.Component;
@@ -21,9 +24,9 @@ import java.util.concurrent.Executor;
 /**
  * 持久化执行器
  *
- * @author AE86
- * @version 1.0.0
- * @date 2022/3/27 16:50
+ * @Version 1.0.0
+ * @Author AE86
+ * @Date 2023-03-27 16:50
  */
 @Component
 public final class StorageBufferActuator extends AbstractBufferActuator {
diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/TableGroupBufferActuator.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/TableGroupBufferActuator.java
index 9613dc7a..fe73f9aa 100644
--- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/TableGroupBufferActuator.java
+++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/TableGroupBufferActuator.java
@@ -1,3 +1,6 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
 package org.dbsyncer.parser.flush.impl;
 
 import org.dbsyncer.common.config.TableGroupBufferConfig;
@@ -15,9 +18,9 @@ import java.util.concurrent.Executor;
 /**
  * 表执行器（根据表消费数据，多线程批量写，按序执行）
  *
- * @author AE86
- * @version 1.0.0
- * @date 2022/3/27 16:50
+ * @Version 1.0.0
+ * @Author AE86
+ * @Date 2023-03-27 16:50
  */
 @Component
 public final class TableGroupBufferActuator extends GeneralBufferActuator implements Cloneable {
diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/LogServiceImpl.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/LogServiceImpl.java
index 5e2d027a..d9a01aa7 100644
--- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/LogServiceImpl.java
+++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/LogServiceImpl.java
@@ -1,11 +1,24 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
 package org.dbsyncer.parser.impl;
 
+import org.dbsyncer.common.util.StringUtil;
 import org.dbsyncer.parser.LogService;
 import org.dbsyncer.parser.LogType;
-import org.dbsyncer.parser.flush.FlushService;
+import org.dbsyncer.parser.ProfileComponent;
+import org.dbsyncer.parser.model.SystemConfig;
+import org.dbsyncer.sdk.constant.ConfigConstant;
+import org.dbsyncer.sdk.enums.StorageEnum;
+import org.dbsyncer.sdk.storage.StorageService;
+import org.dbsyncer.storage.impl.SnowflakeIdWorker;
 import org.springframework.stereotype.Component;
 
 import javax.annotation.Resource;
+import java.time.Instant;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.Executor;
 
 /**
  * @version 1.0.0
@@ -16,20 +29,44 @@ import javax.annotation.Resource;
 public class LogServiceImpl implements LogService {
 
     @Resource
-    private FlushService flushService;
+    private StorageService storageService;
+
+    @Resource
+    private SnowflakeIdWorker snowflakeIdWorker;
+
+    @Resource
+    private Executor storageExecutor;
+
+    @Resource
+    private ProfileComponent profileComponent;
 
     @Override
     public void log(LogType logType) {
-        flushService.asyncWrite(logType.getType(), String.format("%s%s", logType.getName(), logType.getMessage()));
+        asyncWrite(logType.getType(), String.format("%s%s", logType.getName(), logType.getMessage()));
     }
 
     @Override
     public void log(LogType logType, String msg) {
-        flushService.asyncWrite(logType.getType(), null == msg ? logType.getMessage() : msg);
+        asyncWrite(logType.getType(), null == msg ? logType.getMessage() : msg);
     }
 
     @Override
     public void log(LogType logType, String format, Object... args) {
-        flushService.asyncWrite(logType.getType(), String.format(format, args));
+        asyncWrite(logType.getType(), String.format(format, args));
+    }
+
+    private void asyncWrite(String type, String error) {
+        storageExecutor.execute(() -> {
+            Map params = new HashMap();
+            params.put(ConfigConstant.CONFIG_MODEL_ID, String.valueOf(snowflakeIdWorker.nextId()));
+            params.put(ConfigConstant.CONFIG_MODEL_TYPE, type);
+            params.put(ConfigConstant.CONFIG_MODEL_JSON, StringUtil.substring(error, 0, getSystemConfig().getMaxStorageErrorLength()));
+            params.put(ConfigConstant.CONFIG_MODEL_CREATE_TIME, Instant.now().toEpochMilli());
+            storageService.add(StorageEnum.LOG, params);
+        });
+    }
+
+    private SystemConfig getSystemConfig() {
+        return profileComponent.getSystemConfig();
     }
 }
\ No newline at end of file
diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/OperationTemplate.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/OperationTemplate.java
index b90ab21f..26f22da3 100644
--- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/OperationTemplate.java
+++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/OperationTemplate.java
@@ -7,7 +7,7 @@ import org.dbsyncer.common.util.CollectionUtils;
 import org.dbsyncer.common.util.StringUtil;
 import org.dbsyncer.parser.CacheService;
 import org.dbsyncer.parser.ParserException;
-import org.dbsyncer.parser.command.PersistenceCommand;
+import org.dbsyncer.parser.command.impl.PersistenceCommand;
 import org.dbsyncer.parser.enums.CommandEnum;
 import org.dbsyncer.parser.enums.GroupStrategyEnum;
 import org.dbsyncer.parser.model.ConfigModel;
@@ -16,7 +16,7 @@ import org.dbsyncer.parser.model.QueryConfig;
 import org.dbsyncer.parser.strategy.GroupStrategy;
 import org.dbsyncer.parser.util.ConfigModelUtil;
 import org.dbsyncer.sdk.enums.StorageEnum;
-import org.dbsyncer.sdk.spi.StorageService;
+import org.dbsyncer.sdk.storage.StorageService;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.BeanUtils;
diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/FlushStrategyImpl.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/impl/FlushStrategyImpl.java
similarity index 57%
rename from dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/FlushStrategyImpl.java
rename to dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/impl/FlushStrategyImpl.java
index a96127b6..a5a4c222 100644
--- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/FlushStrategyImpl.java
+++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/strategy/impl/FlushStrategyImpl.java
@@ -1,7 +1,7 @@
 /**
  * DBSyncer Copyright 2020-2023 All Rights Reserved.
  */
-package org.dbsyncer.parser.flush.impl;
+package org.dbsyncer.parser.strategy.impl;
 
 import org.dbsyncer.common.model.Result;
 import org.dbsyncer.common.util.CollectionUtils;
@@ -10,10 +10,15 @@ import org.dbsyncer.parser.CacheService;
 import org.dbsyncer.parser.LogService;
 import org.dbsyncer.parser.LogType;
 import org.dbsyncer.parser.ProfileComponent;
-import org.dbsyncer.parser.flush.FlushService;
+import org.dbsyncer.parser.flush.BufferActuator;
 import org.dbsyncer.parser.model.Meta;
+import org.dbsyncer.parser.model.StorageRequest;
 import org.dbsyncer.parser.model.SystemConfig;
 import org.dbsyncer.parser.strategy.FlushStrategy;
+import org.dbsyncer.sdk.constant.ConfigConstant;
+import org.dbsyncer.storage.enums.StorageDataStatusEnum;
+import org.dbsyncer.storage.impl.SnowflakeIdWorker;
+import org.dbsyncer.storage.util.BinlogMessageUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.stereotype.Component;
@@ -21,11 +26,14 @@ import org.springframework.util.Assert;
 
 import javax.annotation.Resource;
 import java.time.Instant;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 
 /**
- * @author AE86
- * @version 1.0.0
- * @date 2021/11/18 22:22
+ * @Version 1.0.0
+ * @Author AE86
+ * @Date 2021-11-18 22:22
  */
 @Component
 public final class FlushStrategyImpl implements FlushStrategy {
@@ -33,7 +41,7 @@ public final class FlushStrategyImpl implements FlushStrategy {
     private final Logger logger = LoggerFactory.getLogger(getClass());
 
     @Resource
-    private FlushService flushService;
+    private SnowflakeIdWorker snowflakeIdWorker;
 
     @Resource
     private CacheService cacheService;
@@ -44,6 +52,9 @@ public final class FlushStrategyImpl implements FlushStrategy {
     @Resource
     private LogService logService;
 
+    @Resource
+    private BufferActuator storageBufferActuator;
+
     @Override
     public void flushFullData(String metaId, Result result, String event) {
         // 不记录全量数据, 只记录增量同步数据, 将异常记录到系统日志中
@@ -67,7 +78,28 @@ public final class FlushStrategyImpl implements FlushStrategy {
         flush(metaId, result, event);
     }
 
-    protected void refreshTotal(String metaId, Result writer) {
+    private void asyncWrite(String metaId, String tableGroupId, String targetTableGroupName, String event, boolean success, List data, String error) {
+        long now = Instant.now().toEpochMilli();
+        data.forEach(r -> {
+            Map row = new HashMap();
+            row.put(ConfigConstant.CONFIG_MODEL_ID, String.valueOf(snowflakeIdWorker.nextId()));
+            row.put(ConfigConstant.DATA_SUCCESS, success ? StorageDataStatusEnum.SUCCESS.getValue() : StorageDataStatusEnum.FAIL.getValue());
+            row.put(ConfigConstant.DATA_TABLE_GROUP_ID, tableGroupId);
+            row.put(ConfigConstant.DATA_TARGET_TABLE_NAME, targetTableGroupName);
+            row.put(ConfigConstant.DATA_EVENT, event);
+            row.put(ConfigConstant.DATA_ERROR, error);
+            row.put(ConfigConstant.CONFIG_MODEL_CREATE_TIME, now);
+            try {
+                byte[] bytes = BinlogMessageUtil.toBinlogMap(r).toByteArray();
+                row.put(ConfigConstant.BINLOG_DATA, bytes);
+            } catch (Exception e) {
+                logger.warn("可能存在Blob或inputStream大文件类型, 无法序列化:{}", r);
+            }
+            storageBufferActuator.offer(new StorageRequest(metaId, row));
+        });
+    }
+
+    private void refreshTotal(String metaId, Result writer) {
         Assert.hasText(metaId, "Meta id can not be empty.");
         Meta meta = cacheService.get(metaId, Meta.class);
         if (meta != null) {
@@ -83,12 +115,12 @@ public final class FlushStrategyImpl implements FlushStrategy {
         // 是否写失败数据
         if (getSystemConfig().isEnableStorageWriteFail() && !CollectionUtils.isEmpty(result.getFailData())) {
             final String error = StringUtil.substring(result.getError().toString(), 0, getSystemConfig().getMaxStorageErrorLength());
-            flushService.asyncWrite(metaId, result.getTableGroupId(), result.getTargetTableGroupName(), event, false, result.getFailData(), error);
+            asyncWrite(metaId, result.getTableGroupId(), result.getTargetTableGroupName(), event, false, result.getFailData(), error);
         }
 
         // 是否写成功数据
         if (getSystemConfig().isEnableStorageWriteSuccess() && !CollectionUtils.isEmpty(result.getSuccessData())) {
-            flushService.asyncWrite(metaId, result.getTableGroupId(), result.getTargetTableGroupName(), event, true, result.getSuccessData(), "");
+            asyncWrite(metaId, result.getTableGroupId(), result.getTargetTableGroupName(), event, true, result.getSuccessData(), "");
         }
     }
 
diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/ConnectorService.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/ConnectorService.java
index 96caa617..a9236962 100644
--- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/ConnectorService.java
+++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/ConnectorService.java
@@ -16,6 +16,7 @@ import org.dbsyncer.sdk.listener.Listener;
 import org.dbsyncer.sdk.model.ConnectorConfig;
 import org.dbsyncer.sdk.model.MetaInfo;
 import org.dbsyncer.sdk.model.Table;
+import org.dbsyncer.sdk.storage.StorageService;
 
 import java.util.List;
 import java.util.Map;
diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/AbstractStorageService.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/AbstractStorageService.java
index 414bebaa..1f32c388 100644
--- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/AbstractStorageService.java
+++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/AbstractStorageService.java
@@ -11,7 +11,6 @@ import org.dbsyncer.sdk.enums.StorageEnum;
 import org.dbsyncer.sdk.enums.StorageStrategyEnum;
 import org.dbsyncer.sdk.filter.BooleanFilter;
 import org.dbsyncer.sdk.filter.Query;
-import org.dbsyncer.sdk.spi.StorageService;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.DisposableBean;
diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/StorageService.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/StorageService.java
similarity index 98%
rename from dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/StorageService.java
rename to dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/StorageService.java
index 39e2a698..241a1f57 100644
--- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/spi/StorageService.java
+++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/storage/StorageService.java
@@ -1,7 +1,7 @@
 /**
  * DBSyncer Copyright 2020-2023 All Rights Reserved.
  */
-package org.dbsyncer.sdk.spi;
+package org.dbsyncer.sdk.storage;
 
 import org.dbsyncer.common.model.Paging;
 import org.dbsyncer.sdk.enums.StorageEnum;
diff --git a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/StorageSupportConfiguration.java b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/StorageSupportConfiguration.java
index a22856ee..890f391e 100644
--- a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/StorageSupportConfiguration.java
+++ b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/StorageSupportConfiguration.java
@@ -5,7 +5,7 @@ package org.dbsyncer.storage;
 
 import org.dbsyncer.common.util.StringUtil;
 import org.dbsyncer.connector.ConnectorFactory;
-import org.dbsyncer.sdk.spi.StorageService;
+import org.dbsyncer.sdk.storage.StorageService;
 import org.dbsyncer.storage.impl.DiskStorageService;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
 import org.springframework.boot.origin.OriginTrackedValue;
diff --git a/dbsyncer-web/src/main/resources/application.properties b/dbsyncer-web/src/main/resources/application.properties
index b059a833..6d230996 100644
--- a/dbsyncer-web/src/main/resources/application.properties
+++ b/dbsyncer-web/src/main/resources/application.properties
@@ -50,8 +50,8 @@ dbsyncer.parser.table.group.buffer-period-millisecond=300
 #storage
 # 数据存储类型:disk(默认)/mysql(推荐生产环境使用)
 # disk-磁盘:/data/config(驱动配置)|data(按驱动分别存储增量数据)|log(系统日志)
-dbsyncer.storage.type=disk
-dbsyncer.storage.mysql.url=jdbc:mysql://127.0.0.1:3306/dbsyncer?rewriteBatchedStatements=true&seUnicode=true&characterEncoding=UTF8&serverTimezone=Asia/Shanghai&useSSL=false&verifyServerCertificate=false&autoReconnect=true
+dbsyncer.storage.type=mysql
+dbsyncer.storage.mysql.url=jdbc:mysql://127.0.0.1:3305/dbsyncer?rewriteBatchedStatements=true&seUnicode=true&characterEncoding=UTF8&serverTimezone=Asia/Shanghai&useSSL=false&verifyServerCertificate=false&autoReconnect=true
 dbsyncer.storage.mysql.username=root
 dbsyncer.storage.mysql.password=123
 # [StorageBufferActuator]线程数
-- 
Gitee


From 7b73eee7baf70ef682ef5c0a5238b052e83db393 Mon Sep 17 00:00:00 2001
From: life <13122192336@163.com>
Date: Fri, 15 Dec 2023 13:08:38 +0800
Subject: [PATCH 80/98] =?UTF-8?q?=E4=BF=AE=E6=94=B9logminer=E8=A7=A3?=
 =?UTF-8?q?=E6=9E=90=E4=B8=BAsql=E8=A7=A3=E6=9E=90?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../logminer/parser/AbstractParser.java       | 74 ++++++++++---------
 .../logminer/parser/OracleTypeParser.java     | 34 +++++++++
 .../logminer/parser/impl/InsertSql.java       | 11 +--
 .../logminer/parser/impl/UpdateSql.java       | 15 +++-
 4 files changed, 92 insertions(+), 42 deletions(-)
 create mode 100644 dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/OracleTypeParser.java

diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/AbstractParser.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/AbstractParser.java
index 3f0aa8a1..1bfab0cf 100644
--- a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/AbstractParser.java
+++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/AbstractParser.java
@@ -3,20 +3,19 @@
  */
 package org.dbsyncer.connector.oracle.logminer.parser;
 
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
 import net.sf.jsqlparser.expression.BinaryExpression;
 import net.sf.jsqlparser.expression.Expression;
+import net.sf.jsqlparser.expression.Function;
+import net.sf.jsqlparser.expression.operators.relational.IsNullExpression;
 import net.sf.jsqlparser.schema.Column;
-import org.dbsyncer.common.util.CollectionUtils;
 import org.dbsyncer.common.util.StringUtil;
 import org.dbsyncer.sdk.connector.database.DatabaseConnectorInstance;
 import org.dbsyncer.sdk.model.Field;
 
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-
 /**
  * @Author AE86
  * @Version 1.0.0
@@ -30,48 +29,51 @@ public abstract class AbstractParser implements Parser {
     protected DatabaseConnectorInstance instance;
 
     public void findColumn(Expression expression) {
+        if (expression instanceof IsNullExpression){
+            IsNullExpression isNullExpression = (IsNullExpression) expression;
+            Column column = (Column) isNullExpression.getLeftExpression();
+            columnMap.put(StringUtil.replace(column.getColumnName(), StringUtil.DOUBLE_QUOTATION, StringUtil.EMPTY),
+                    null);
+            return;
+        }
+
         BinaryExpression binaryExpression = (BinaryExpression) expression;
         if (binaryExpression.getLeftExpression() instanceof Column) {
             Column column = (Column) binaryExpression.getLeftExpression();
-            String value = binaryExpression.getRightExpression().toString();
-            columnMap.put(StringUtil.replace(column.getColumnName(), StringUtil.DOUBLE_QUOTATION, StringUtil.EMPTY), value);
+            columnMap.put(StringUtil.replace(column.getColumnName(), StringUtil.DOUBLE_QUOTATION, StringUtil.EMPTY),
+                    parserValue(binaryExpression.getRightExpression()));
             return;
         }
         findColumn(binaryExpression.getLeftExpression());
         findColumn(binaryExpression.getRightExpression());
     }
 
-    // 从主键解析出来的map装载成sql并运行sql找出对应的数据
-    public List getColumnsFromDB() {
-        List> rows = instance.execute(databaseTemplate -> databaseTemplate.queryForList(getRowByPk()));
-        List list = new LinkedList<>();
-        if (!CollectionUtils.isEmpty(rows)) {
-            rows.forEach(map -> {
-                for (String key : map.keySet()) {
-                    list.add(map.get(key));
-                }
-            });
-            return list;
+    public String parserValue(Expression expression){
+        String value = "";
+        if (expression instanceof Function){
+            value = parserFunction((Function) expression);
+        }else{
+            value = expression.toString();
         }
-        return Collections.EMPTY_LIST;
+        return value;
     }
 
-    private String getRowByPk() {
-        StringBuilder sql = new StringBuilder("SELECT * FROM ").append(getTableName()).append(" WHERE ");
-        int pkCount = 0;
-        for (Field field : fields) {
-            if (field.isPk()) {
-                String value = columnMap.get(field.getName());
-                if (StringUtil.isNotBlank(value)) {
-                    if (pkCount > 0) {
-                        sql.append(" AND ");
-                    }
-                    sql.append(field.getName()).append("=").append(value);
-                    pkCount++;
-                }
-            }
+    //解析sql的function，只取到关键的字符串
+    public String parserFunction(Function function){
+        if (function.getMultipartName().get(0).equals("TO_DATE")){
+            return StringUtil.replace(function.getParameters().get(0).toString(),StringUtil.SINGLE_QUOTATION,StringUtil.EMPTY);
+        }
+        return "";
+    }
+
+
+    public List columnMapToData(){
+        List data = new LinkedList<>();
+        for (Field field: fields) {
+            Object value = OracleTypeParser.convertToJavaType(field,columnMap.get(field.getName()));
+            data.add(value);
         }
-        return sql.toString();
+        return data;
     }
 
     @Override
diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/OracleTypeParser.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/OracleTypeParser.java
new file mode 100644
index 00000000..3db8d737
--- /dev/null
+++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/OracleTypeParser.java
@@ -0,0 +1,34 @@
+package org.dbsyncer.connector.oracle.logminer.parser;
+
+import java.math.BigInteger;
+import java.sql.Types;
+import org.dbsyncer.common.util.DateFormatUtil;
+import org.dbsyncer.common.util.StringUtil;
+import org.dbsyncer.sdk.model.Field;
+
+/**
+ * @Description :oracle解析到java类
+ * @Param :
+ * @return :
+ * @author : life
+ * @date : 2023/12/15  11:25
+ */
+public class OracleTypeParser {
+    public static Object convertToJavaType(Field field,String value){
+        if (value == null){
+            return null;
+        }
+        switch (field.getType()) {
+            case Types.DECIMAL:
+                return new BigInteger(
+                        StringUtil.replace(value, StringUtil.SINGLE_QUOTATION, StringUtil.EMPTY));
+            case Types.TIMESTAMP:
+                return DateFormatUtil.stringToTimestamp(StringUtil.replace(value,StringUtil.SINGLE_QUOTATION,StringUtil.EMPTY));
+            default:
+                return value;
+        }
+    }
+
+
+
+}
diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/impl/InsertSql.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/impl/InsertSql.java
index 3218bcc1..00b58dab 100644
--- a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/impl/InsertSql.java
+++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/impl/InsertSql.java
@@ -3,14 +3,14 @@
  */
 package org.dbsyncer.connector.oracle.logminer.parser.impl;
 
+import java.util.List;
+import net.sf.jsqlparser.expression.Expression;
 import net.sf.jsqlparser.expression.operators.relational.ExpressionList;
 import net.sf.jsqlparser.schema.Column;
 import net.sf.jsqlparser.statement.insert.Insert;
 import org.dbsyncer.common.util.StringUtil;
 import org.dbsyncer.connector.oracle.logminer.parser.AbstractParser;
 
-import java.util.List;
-
 /**
  * @Author AE86
  * @Version 1.0.0
@@ -27,11 +27,12 @@ public class InsertSql extends AbstractParser {
     @Override
     public List parseColumns() {
         List columns = insert.getColumns();
-        ExpressionList values = insert.getSelect().getValues().getExpressions();
+        ExpressionList values = (ExpressionList) insert.getSelect().getValues().getExpressions();
         for (int i = 0; i < columns.size(); i++) {
-            columnMap.put(StringUtil.replace(columns.get(i).getColumnName(), StringUtil.DOUBLE_QUOTATION, StringUtil.EMPTY), values.get(i).toString());
+            columnMap.put(StringUtil.replace(columns.get(i).getColumnName(), StringUtil.DOUBLE_QUOTATION, StringUtil.EMPTY),
+                    parserValue(values.get(i)));
         }
-        return getColumnsFromDB();
+        return columnMapToData();
     }
 
 }
\ No newline at end of file
diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/impl/UpdateSql.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/impl/UpdateSql.java
index e019a7e8..f97c8460 100644
--- a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/impl/UpdateSql.java
+++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/impl/UpdateSql.java
@@ -4,6 +4,8 @@
 package org.dbsyncer.connector.oracle.logminer.parser.impl;
 
 import net.sf.jsqlparser.statement.update.Update;
+import net.sf.jsqlparser.statement.update.UpdateSet;
+import org.dbsyncer.common.util.StringUtil;
 import org.dbsyncer.connector.oracle.logminer.parser.AbstractParser;
 
 import java.util.List;
@@ -24,7 +26,18 @@ public class UpdateSql extends AbstractParser {
     @Override
     public List parseColumns() {
         findColumn(update.getWhere());
-        return getColumnsFromDB();
+        passerSet(update.getUpdateSets());
+        return columnMapToData();
+    }
+
+    private void passerSet(List updateSets){
+        //解析替换
+        for (UpdateSet updateSet:updateSets) {
+            String columnName = StringUtil.replace(updateSet.getColumn(0).getColumnName(),
+                    StringUtil.DOUBLE_QUOTATION,StringUtil.EMPTY);
+            String value = parserValue(updateSet.getValue(0));
+            columnMap.put(columnName,value);
+        }
     }
 
 }
\ No newline at end of file
-- 
Gitee


From fce4b991b07fc00ccbed892950962799cdf3e980 Mon Sep 17 00:00:00 2001
From: life <13122192336@163.com>
Date: Fri, 15 Dec 2023 13:12:46 +0800
Subject: [PATCH 81/98] =?UTF-8?q?=E5=88=A0=E9=99=A4=E4=B8=8D=E5=BF=85?=
 =?UTF-8?q?=E8=A6=81=E7=9A=84=EF=BC=8C=E4=BB=A5=E5=8F=8Astring=E5=81=9A?=
 =?UTF-8?q?=E6=A0=BC=E5=BC=8F=E5=8C=96?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../org/dbsyncer/connector/oracle/cdc/OracleListener.java     | 1 -
 .../connector/oracle/logminer/parser/AbstractParser.java      | 4 ----
 .../connector/oracle/logminer/parser/OracleTypeParser.java    | 2 +-
 3 files changed, 1 insertion(+), 6 deletions(-)

diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java
index 4bb58c43..182a9a9f 100644
--- a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java
+++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java
@@ -132,7 +132,6 @@ public class OracleListener extends AbstractDatabaseListener {
     private AbstractParser setTable(AbstractParser parser, Table table) {
         parser.setTableName(table == null ? StringUtil.EMPTY : StringUtil.replace(table.getName(), StringUtil.DOUBLE_QUOTATION, StringUtil.EMPTY));
         parser.setFields(tableFiledMap.get(parser.getTableName()));
-        parser.setInstance(getConnectorInstance());
         return parser;
     }
 
diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/AbstractParser.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/AbstractParser.java
index 1bfab0cf..27f1296c 100644
--- a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/AbstractParser.java
+++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/AbstractParser.java
@@ -26,7 +26,6 @@ public abstract class AbstractParser implements Parser {
     protected Map columnMap = new HashMap<>();
     protected String tableName;
     protected List fields;
-    protected DatabaseConnectorInstance instance;
 
     public void findColumn(Expression expression) {
         if (expression instanceof IsNullExpression){
@@ -89,7 +88,4 @@ public abstract class AbstractParser implements Parser {
         this.fields = fields;
     }
 
-    public void setInstance(DatabaseConnectorInstance instance) {
-        this.instance = instance;
-    }
 }
\ No newline at end of file
diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/OracleTypeParser.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/OracleTypeParser.java
index 3db8d737..528919a5 100644
--- a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/OracleTypeParser.java
+++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/OracleTypeParser.java
@@ -25,7 +25,7 @@ public class OracleTypeParser {
             case Types.TIMESTAMP:
                 return DateFormatUtil.stringToTimestamp(StringUtil.replace(value,StringUtil.SINGLE_QUOTATION,StringUtil.EMPTY));
             default:
-                return value;
+                return StringUtil.replace(value,StringUtil.SINGLE_QUOTATION,StringUtil.EMPTY);
         }
     }
 
-- 
Gitee


From 8f05f755eb07dbda2170644731b725f8699d4feb Mon Sep 17 00:00:00 2001
From: life <13122192336@163.com>
Date: Fri, 15 Dec 2023 14:50:28 +0800
Subject: [PATCH 82/98] =?UTF-8?q?=E5=A2=9E=E5=8A=A0trysend?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../connector/oracle/cdc/OracleListener.java  | 35 +++++++++++++++++--
 .../connector/oracle/logminer/LogMiner.java   |  4 +++
 .../logminer/parser/AbstractParser.java       | 12 +++----
 3 files changed, 42 insertions(+), 9 deletions(-)

diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java
index 182a9a9f..1c5d9f53 100644
--- a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java
+++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java
@@ -3,6 +3,8 @@
  */
 package org.dbsyncer.connector.oracle.cdc;
 
+import java.time.Instant;
+import java.util.concurrent.TimeUnit;
 import net.sf.jsqlparser.JSQLParserException;
 import net.sf.jsqlparser.parser.CCJSqlParserUtil;
 import net.sf.jsqlparser.schema.Table;
@@ -10,6 +12,7 @@ import net.sf.jsqlparser.statement.Statement;
 import net.sf.jsqlparser.statement.delete.Delete;
 import net.sf.jsqlparser.statement.insert.Insert;
 import net.sf.jsqlparser.statement.update.Update;
+import org.dbsyncer.common.QueueOverflowException;
 import org.dbsyncer.common.util.StringUtil;
 import org.dbsyncer.connector.oracle.OracleException;
 import org.dbsyncer.connector.oracle.logminer.parser.AbstractParser;
@@ -78,6 +81,32 @@ public class OracleListener extends AbstractDatabaseListener {
         }
     }
 
+    private void trySendEvent(RowChangedEvent event){
+        try {
+            // 如果消费事件失败，重试
+            long now = Instant.now().toEpochMilli();
+            boolean isReTry = false;
+            while (logMiner.isConnected()){
+                try {
+                    sendChangedEvent(event);
+                    break;
+                } catch (QueueOverflowException e) {
+                    isReTry = true;
+                    try {
+                        TimeUnit.MILLISECONDS.sleep(1);
+                    } catch (InterruptedException ex) {
+                        logger.error(ex.getMessage(), ex);
+                    }
+                }
+            }
+            if (isReTry) {
+                logger.info("重试耗时：{}ms", Instant.now().toEpochMilli() - now);
+            }
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+        }
+    }
+
     /**
      * 解析事件
      *
@@ -91,7 +120,7 @@ public class OracleListener extends AbstractDatabaseListener {
             Update update = (Update) statement;
             UpdateSql parser = new UpdateSql(update);
             setTable(parser, update.getTable());
-            sendChangedEvent(new RowChangedEvent(parser.getTableName(), ConnectorConstant.OPERTION_UPDATE, parser.parseColumns(), null, event.getScn()));
+            trySendEvent(new RowChangedEvent(parser.getTableName(), ConnectorConstant.OPERTION_UPDATE, parser.parseColumns(), null, event.getScn()));
             return;
         }
 
@@ -99,7 +128,7 @@ public class OracleListener extends AbstractDatabaseListener {
             Insert insert = (Insert) statement;
             InsertSql parser = new InsertSql(insert);
             setTable(parser, insert.getTable());
-            sendChangedEvent(new RowChangedEvent(parser.getTableName(), ConnectorConstant.OPERTION_INSERT, parser.parseColumns(), null, event.getScn()));
+            trySendEvent(new RowChangedEvent(parser.getTableName(), ConnectorConstant.OPERTION_INSERT, parser.parseColumns(), null, event.getScn()));
             return;
         }
 
@@ -107,7 +136,7 @@ public class OracleListener extends AbstractDatabaseListener {
             Delete delete = (Delete) statement;
             DeleteSql parser = new DeleteSql(delete);
             setTable(parser, delete.getTable());
-            sendChangedEvent(new RowChangedEvent(parser.getTableName(), ConnectorConstant.OPERTION_DELETE, parser.parseColumns(), null, event.getScn()));
+            trySendEvent(new RowChangedEvent(parser.getTableName(), ConnectorConstant.OPERTION_DELETE, parser.parseColumns(), null, event.getScn()));
         }
 
         // TODO ddl
diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/LogMiner.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/LogMiner.java
index 810b7ea2..697bced7 100644
--- a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/LogMiner.java
+++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/LogMiner.java
@@ -321,4 +321,8 @@ public class LogMiner {
         void onEvent(RedoEvent redoEvent);
     }
 
+    public boolean isConnected(){
+        return connected;
+    }
+
 }
\ No newline at end of file
diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/AbstractParser.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/AbstractParser.java
index 27f1296c..068b4e38 100644
--- a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/AbstractParser.java
+++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/AbstractParser.java
@@ -10,10 +10,10 @@ import java.util.Map;
 import net.sf.jsqlparser.expression.BinaryExpression;
 import net.sf.jsqlparser.expression.Expression;
 import net.sf.jsqlparser.expression.Function;
+import net.sf.jsqlparser.expression.NullValue;
 import net.sf.jsqlparser.expression.operators.relational.IsNullExpression;
 import net.sf.jsqlparser.schema.Column;
 import org.dbsyncer.common.util.StringUtil;
-import org.dbsyncer.sdk.connector.database.DatabaseConnectorInstance;
 import org.dbsyncer.sdk.model.Field;
 
 /**
@@ -48,13 +48,13 @@ public abstract class AbstractParser implements Parser {
     }
 
     public String parserValue(Expression expression){
-        String value = "";
         if (expression instanceof Function){
-            value = parserFunction((Function) expression);
-        }else{
-            value = expression.toString();
+            return parserFunction((Function) expression);
         }
-        return value;
+        if (expression instanceof NullValue){
+            return null;
+        }
+        return expression.toString();
     }
 
     //解析sql的function，只取到关键的字符串
-- 
Gitee


From 02fb4aed9c53e8fa23aa19ce8a157c05384dde31 Mon Sep 17 00:00:00 2001
From: life <13122192336@163.com>
Date: Sun, 17 Dec 2023 13:23:23 +0800
Subject: [PATCH 83/98] =?UTF-8?q?=E5=B0=9D=E8=AF=95=E5=B1=8F=E8=94=BD?=
 =?UTF-8?q?=E6=8E=89ORA-00310?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../connector/oracle/logminer/LogMiner.java         | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/LogMiner.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/LogMiner.java
index 697bced7..0c1e8d16 100644
--- a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/LogMiner.java
+++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/LogMiner.java
@@ -124,7 +124,14 @@ public class LogMiner {
                 try (ResultSet rs = minerViewStatement.executeQuery()) {
                     logger.trace("Query V$LOGMNR_CONTENTS spend time {} ms", stopWatch.getTime(TimeUnit.MILLISECONDS));
                     stopWatch.reset();
-                    logMinerViewProcessor(rs);
+                    try{
+                        logMinerViewProcessor(rs);
+                    }catch (SQLException e){
+                        if (e.getMessage().contains("ORA-00310")){
+                            continue;
+                        }
+                        throw e;
+                    }
                 }
 
                 // 7.确定新的SCN
@@ -264,12 +271,10 @@ public class LogMiner {
                             // 当前SCN 事务已经提交 并且 小于事务缓冲区中所有的开始SCN，所以可以更新offsetScn
                             startScn = scn.longValue();
                         }
-
                         if (counter == 0) {
                             updateCommittedScn(commitScn.longValue());
                         }
-
-                        event.setScn(committedScn);
+                        event.setScn(startScn < committedScn ? committedScn:startScn);
                         listener.onEvent(event);
                     };
                     transactionalBuffer.registerCommitCallback(txId, scn, commitCallback);
-- 
Gitee


From ee120aa250bb542363955147c48439c2037ea5b1 Mon Sep 17 00:00:00 2001
From: life <13122192336@163.com>
Date: Sun, 17 Dec 2023 13:24:57 +0800
Subject: [PATCH 84/98] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=97=A5=E5=BF=97?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../java/org/dbsyncer/connector/oracle/logminer/LogMiner.java    | 1 +
 1 file changed, 1 insertion(+)

diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/LogMiner.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/LogMiner.java
index 0c1e8d16..791f85cc 100644
--- a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/LogMiner.java
+++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/LogMiner.java
@@ -128,6 +128,7 @@ public class LogMiner {
                         logMinerViewProcessor(rs);
                     }catch (SQLException e){
                         if (e.getMessage().contains("ORA-00310")){
+                            logger.error("ORA-00310 try continue");
                             continue;
                         }
                         throw e;
-- 
Gitee


From 40ca74e3f07f0103a189d596bbe71494e55703b9 Mon Sep 17 00:00:00 2001
From: life <13122192336@163.com>
Date: Sun, 17 Dec 2023 13:32:40 +0800
Subject: [PATCH 85/98] =?UTF-8?q?=E5=A4=84=E7=90=86=E8=BF=99=E7=A7=8D?=
 =?UTF-8?q?=E9=80=9A=E8=BF=87=E9=87=8D=E5=90=AFlogminer?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../java/org/dbsyncer/connector/oracle/logminer/LogMiner.java   | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/LogMiner.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/LogMiner.java
index 791f85cc..99db94d3 100644
--- a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/LogMiner.java
+++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/LogMiner.java
@@ -129,6 +129,8 @@ public class LogMiner {
                     }catch (SQLException e){
                         if (e.getMessage().contains("ORA-00310")){
                             logger.error("ORA-00310 try continue");
+                            restartLogMiner();
+                            currentRedoLogSequences = LogMinerHelper.getCurrentRedoLogSequences(connection);
                             continue;
                         }
                         throw e;
-- 
Gitee


From 43c9a9e686d6ef3251166890dd66f44ebaf99442 Mon Sep 17 00:00:00 2001
From: AE86 <836391306@qq.com>
Date: Sun, 17 Dec 2023 17:11:48 +0800
Subject: [PATCH 86/98] mv pck

---
 .../dbsyncer/biz/checker/impl/connector/ConnectorChecker.java   | 2 +-
 .../main/java/org/dbsyncer/biz/impl/ConnectorServiceImpl.java   | 2 +-
 .../src/main/java/org/dbsyncer/biz/impl/MappingServiceImpl.java | 2 +-
 .../org/dbsyncer/connector/{ => base}/ConnectorException.java   | 2 +-
 .../org/dbsyncer/connector/{ => base}/ConnectorFactory.java     | 2 +-
 .../main/java/org/dbsyncer/manager/impl/IncrementPuller.java    | 2 +-
 .../main/java/org/dbsyncer/manager/impl/PreloadTemplate.java    | 2 +-
 .../main/java/org/dbsyncer/parser/ddl/impl/DDLParserImpl.java   | 2 +-
 .../org/dbsyncer/parser/flush/impl/GeneralBufferActuator.java   | 2 +-
 .../main/java/org/dbsyncer/parser/impl/ParserComponentImpl.java | 2 +-
 .../java/org/dbsyncer/parser/impl/ProfileComponentImpl.java     | 2 +-
 .../java/org/dbsyncer/storage/StorageSupportConfiguration.java  | 2 +-
 12 files changed, 12 insertions(+), 12 deletions(-)
 rename dbsyncer-connector/dbsyncer-connector-base/src/main/java/org/dbsyncer/connector/{ => base}/ConnectorException.java (94%)
 rename dbsyncer-connector/dbsyncer-connector-base/src/main/java/org/dbsyncer/connector/{ => base}/ConnectorFactory.java (99%)

diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/ConnectorChecker.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/ConnectorChecker.java
index b60cbb0b..6c6f7246 100644
--- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/ConnectorChecker.java
+++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/checker/impl/connector/ConnectorChecker.java
@@ -5,7 +5,7 @@ package org.dbsyncer.biz.checker.impl.connector;
 
 import org.dbsyncer.biz.BizException;
 import org.dbsyncer.biz.checker.AbstractChecker;
-import org.dbsyncer.connector.ConnectorFactory;
+import org.dbsyncer.connector.base.ConnectorFactory;
 import org.dbsyncer.parser.ProfileComponent;
 import org.dbsyncer.parser.model.ConfigModel;
 import org.dbsyncer.parser.model.Connector;
diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ConnectorServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ConnectorServiceImpl.java
index fb70cc2f..4888516b 100644
--- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ConnectorServiceImpl.java
+++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/ConnectorServiceImpl.java
@@ -6,7 +6,7 @@ import org.dbsyncer.biz.checker.Checker;
 import org.dbsyncer.common.util.CollectionUtils;
 import org.dbsyncer.common.util.JsonUtil;
 import org.dbsyncer.common.util.StringUtil;
-import org.dbsyncer.connector.ConnectorFactory;
+import org.dbsyncer.connector.base.ConnectorFactory;
 import org.dbsyncer.parser.LogService;
 import org.dbsyncer.parser.LogType;
 import org.dbsyncer.parser.ProfileComponent;
diff --git a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MappingServiceImpl.java b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MappingServiceImpl.java
index 06fb0fc0..0d1a8a81 100644
--- a/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MappingServiceImpl.java
+++ b/dbsyncer-biz/src/main/java/org/dbsyncer/biz/impl/MappingServiceImpl.java
@@ -12,7 +12,7 @@ import org.dbsyncer.biz.vo.MetaVo;
 import org.dbsyncer.common.util.CollectionUtils;
 import org.dbsyncer.common.util.JsonUtil;
 import org.dbsyncer.common.util.StringUtil;
-import org.dbsyncer.connector.ConnectorFactory;
+import org.dbsyncer.connector.base.ConnectorFactory;
 import org.dbsyncer.manager.ManagerFactory;
 import org.dbsyncer.parser.LogType;
 import org.dbsyncer.parser.ProfileComponent;
diff --git a/dbsyncer-connector/dbsyncer-connector-base/src/main/java/org/dbsyncer/connector/ConnectorException.java b/dbsyncer-connector/dbsyncer-connector-base/src/main/java/org/dbsyncer/connector/base/ConnectorException.java
similarity index 94%
rename from dbsyncer-connector/dbsyncer-connector-base/src/main/java/org/dbsyncer/connector/ConnectorException.java
rename to dbsyncer-connector/dbsyncer-connector-base/src/main/java/org/dbsyncer/connector/base/ConnectorException.java
index 21126f21..5ca9a386 100644
--- a/dbsyncer-connector/dbsyncer-connector-base/src/main/java/org/dbsyncer/connector/ConnectorException.java
+++ b/dbsyncer-connector/dbsyncer-connector-base/src/main/java/org/dbsyncer/connector/base/ConnectorException.java
@@ -1,4 +1,4 @@
-package org.dbsyncer.connector;
+package org.dbsyncer.connector.base;
 
 /**
  * @author AE86
diff --git a/dbsyncer-connector/dbsyncer-connector-base/src/main/java/org/dbsyncer/connector/ConnectorFactory.java b/dbsyncer-connector/dbsyncer-connector-base/src/main/java/org/dbsyncer/connector/base/ConnectorFactory.java
similarity index 99%
rename from dbsyncer-connector/dbsyncer-connector-base/src/main/java/org/dbsyncer/connector/ConnectorFactory.java
rename to dbsyncer-connector/dbsyncer-connector-base/src/main/java/org/dbsyncer/connector/base/ConnectorFactory.java
index d861aa81..53faf8ef 100644
--- a/dbsyncer-connector/dbsyncer-connector-base/src/main/java/org/dbsyncer/connector/ConnectorFactory.java
+++ b/dbsyncer-connector/dbsyncer-connector-base/src/main/java/org/dbsyncer/connector/base/ConnectorFactory.java
@@ -1,7 +1,7 @@
 /**
  * DBSyncer Copyright 2020-2023 All Rights Reserved.
  */
-package org.dbsyncer.connector;
+package org.dbsyncer.connector.base;
 
 import org.dbsyncer.common.model.Result;
 import org.dbsyncer.common.util.CollectionUtils;
diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/IncrementPuller.java b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/IncrementPuller.java
index 3e95beb1..3da8bffc 100644
--- a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/IncrementPuller.java
+++ b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/IncrementPuller.java
@@ -4,7 +4,7 @@
 package org.dbsyncer.manager.impl;
 
 import org.dbsyncer.common.util.CollectionUtils;
-import org.dbsyncer.connector.ConnectorFactory;
+import org.dbsyncer.connector.base.ConnectorFactory;
 import org.dbsyncer.manager.AbstractPuller;
 import org.dbsyncer.manager.ManagerException;
 import org.dbsyncer.parser.LogService;
diff --git a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/PreloadTemplate.java b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/PreloadTemplate.java
index 145b5c19..34831927 100644
--- a/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/PreloadTemplate.java
+++ b/dbsyncer-manager/src/main/java/org/dbsyncer/manager/impl/PreloadTemplate.java
@@ -6,7 +6,7 @@ package org.dbsyncer.manager.impl;
 import org.dbsyncer.common.model.Paging;
 import org.dbsyncer.common.util.CollectionUtils;
 import org.dbsyncer.common.util.JsonUtil;
-import org.dbsyncer.connector.ConnectorFactory;
+import org.dbsyncer.connector.base.ConnectorFactory;
 import org.dbsyncer.manager.ManagerFactory;
 import org.dbsyncer.parser.LogService;
 import org.dbsyncer.parser.LogType;
diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ddl/impl/DDLParserImpl.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ddl/impl/DDLParserImpl.java
index d2c6ac35..1a9fc645 100644
--- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ddl/impl/DDLParserImpl.java
+++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ddl/impl/DDLParserImpl.java
@@ -7,7 +7,7 @@ import net.sf.jsqlparser.statement.alter.Alter;
 import net.sf.jsqlparser.statement.alter.AlterExpression;
 import net.sf.jsqlparser.statement.alter.AlterOperation;
 import org.dbsyncer.common.util.StringUtil;
-import org.dbsyncer.connector.ConnectorFactory;
+import org.dbsyncer.connector.base.ConnectorFactory;
 import org.dbsyncer.sdk.config.DDLConfig;
 import org.dbsyncer.sdk.connector.database.Database;
 import org.dbsyncer.parser.ddl.AlterStrategy;
diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/GeneralBufferActuator.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/GeneralBufferActuator.java
index ee70d2fa..dbee58e8 100644
--- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/GeneralBufferActuator.java
+++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/GeneralBufferActuator.java
@@ -8,7 +8,7 @@ import org.dbsyncer.common.config.GeneralBufferConfig;
 import org.dbsyncer.common.model.Result;
 import org.dbsyncer.common.util.CollectionUtils;
 import org.dbsyncer.common.util.StringUtil;
-import org.dbsyncer.connector.ConnectorFactory;
+import org.dbsyncer.connector.base.ConnectorFactory;
 import org.dbsyncer.parser.ParserComponent;
 import org.dbsyncer.parser.ProfileComponent;
 import org.dbsyncer.parser.ddl.DDLParser;
diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/ParserComponentImpl.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/ParserComponentImpl.java
index 4600a152..520ce09c 100644
--- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/ParserComponentImpl.java
+++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/ParserComponentImpl.java
@@ -3,7 +3,7 @@ package org.dbsyncer.parser.impl;
 import org.dbsyncer.common.model.Result;
 import org.dbsyncer.common.util.CollectionUtils;
 import org.dbsyncer.common.util.StringUtil;
-import org.dbsyncer.connector.ConnectorFactory;
+import org.dbsyncer.connector.base.ConnectorFactory;
 import org.dbsyncer.sdk.config.CommandConfig;
 import org.dbsyncer.sdk.config.ReaderConfig;
 import org.dbsyncer.sdk.config.WriterBatchConfig;
diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/ProfileComponentImpl.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/ProfileComponentImpl.java
index 0c49978c..2264f5f1 100644
--- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/ProfileComponentImpl.java
+++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/impl/ProfileComponentImpl.java
@@ -5,7 +5,7 @@ package org.dbsyncer.parser.impl;
 
 import org.dbsyncer.common.util.CollectionUtils;
 import org.dbsyncer.common.util.JsonUtil;
-import org.dbsyncer.connector.ConnectorFactory;
+import org.dbsyncer.connector.base.ConnectorFactory;
 import org.dbsyncer.sdk.enums.FilterEnum;
 import org.dbsyncer.sdk.enums.QuartzFilterEnum;
 import org.dbsyncer.parser.ProfileComponent;
diff --git a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/StorageSupportConfiguration.java b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/StorageSupportConfiguration.java
index 890f391e..ad5f7056 100644
--- a/dbsyncer-storage/src/main/java/org/dbsyncer/storage/StorageSupportConfiguration.java
+++ b/dbsyncer-storage/src/main/java/org/dbsyncer/storage/StorageSupportConfiguration.java
@@ -4,7 +4,7 @@
 package org.dbsyncer.storage;
 
 import org.dbsyncer.common.util.StringUtil;
-import org.dbsyncer.connector.ConnectorFactory;
+import org.dbsyncer.connector.base.ConnectorFactory;
 import org.dbsyncer.sdk.storage.StorageService;
 import org.dbsyncer.storage.impl.DiskStorageService;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
-- 
Gitee


From 22569e869b5ea9635caffd5c55211e0eca428b4c Mon Sep 17 00:00:00 2001
From: AE86 <836391306@qq.com>
Date: Mon, 18 Dec 2023 00:37:24 +0800
Subject: [PATCH 87/98] fix bat

---
 dbsyncer-web/src/main/assembly/bin/startup.bat | 18 ++++++++++++------
 .../java/org/dbsyncer/web/Application.java     | 17 +++++++++++++++--
 .../src/main/resources/application.properties  |  4 ++--
 3 files changed, 29 insertions(+), 10 deletions(-)

diff --git a/dbsyncer-web/src/main/assembly/bin/startup.bat b/dbsyncer-web/src/main/assembly/bin/startup.bat
index d8b63eb0..0f669a66 100644
--- a/dbsyncer-web/src/main/assembly/bin/startup.bat
+++ b/dbsyncer-web/src/main/assembly/bin/startup.bat
@@ -5,10 +5,10 @@ rem set JAVA_HOME=D:\java\jdk1.8.0_40
 rem set PATH=%JAVA_HOME%\bin;%JAVA_HOME%\jre\bin;
 rem set CLASSPATH=.;%JAVA_HOME%\lib;%JAVA_HOME%\lib\dt.jar;%JAVA_HOME%\lib\tools.jar
 
+for %%F in ("%~dp0\..\") do set "DBS_HOME=%%~dpF"
+echo DBS_HOME=%DBS_HOME%
 cd ../
 echo starting up ...
-set CURRENT_DIR=%cd%
-echo %CURRENT_DIR%
 
 set SERVER_OPTS=-Xms1024m -Xmx1024m -Xss1m -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=256m
 rem debug model
@@ -18,15 +18,21 @@ rem set SERVER_OPTS=%SERVER_OPTS% -Djava.net.preferIPv4Stack=true -Djava.net.pre
 
 rem set jmxremote args
 set HOST=127.0.0.1
-set JMXREMOTE_CONFIG_PATH=%CURRENT_DIR%\config
 set JMXREMOTE_HOSTNAME=-Djava.rmi.server.hostname=%HOST%
 set JMXREMOTE_PORT=-Dcom.sun.management.jmxremote.port=15099
 set JMXREMOTE_SSL=-Dcom.sun.management.jmxremote.ssl=false
 set JMXREMOTE_AUTH=-Dcom.sun.management.jmxremote.authenticate=true
-set JMXREMOTE_ACCESS=-Dcom.sun.management.jmxremote.access.file=%JMXREMOTE_CONFIG_PATH%\jmxremote.access
-set JMXREMOTE_PASSWORD=-Dcom.sun.management.jmxremote.password.file=%JMXREMOTE_CONFIG_PATH%\jmxremote.password
+set JMXREMOTE_ACCESS=-Dcom.sun.management.jmxremote.access.file=%DBS_HOME%\config\jmxremote.access
+set JMXREMOTE_PASSWORD=-Dcom.sun.management.jmxremote.password.file=%DBS_HOME%\config\jmxremote.password
 rem jmxremote model
 rem SERVER_OPTS=%SERVER_OPTS% %JMXREMOTE_HOSTNAME% %JMXREMOTE_PORT% %JMXREMOTE_SSL% %JMXREMOTE_AUTH% %JMXREMOTE_ACCESS% %JMXREMOTE_PASSWORD%
 
+set ENCRYPT_FILE=%DBS_HOME%bin\dbsyncer_decrypt.dll
+if exist %ENCRYPT_FILE% (
+set SERVER_OPTS=%SERVER_OPTS% -agentpath:%ENCRYPT_FILE%
+)
+
+set SERVER_OPTS=%SERVER_OPTS% -Djava.ext.dirs=%JAVA_HOME%\jre\lib\ext;%DBS_HOME%lib
+set SERVER_OPTS=%SERVER_OPTS% -Dspring.config.location=%DBS_HOME%conf\application.properties
 echo %SERVER_OPTS%
-java %SERVER_OPTS% -Dfile.encoding=GBK -Djava.ext.dirs="%JAVA_HOME%\jre\lib\ext;./lib" -Dspring.config.location=%cd%\conf\application.properties org.dbsyncer.web.Application
\ No newline at end of file
+java %SERVER_OPTS% -Dfile.encoding=GBK org.dbsyncer.web.Application
\ No newline at end of file
diff --git a/dbsyncer-web/src/main/java/org/dbsyncer/web/Application.java b/dbsyncer-web/src/main/java/org/dbsyncer/web/Application.java
index a9a184ef..435f2331 100644
--- a/dbsyncer-web/src/main/java/org/dbsyncer/web/Application.java
+++ b/dbsyncer-web/src/main/java/org/dbsyncer/web/Application.java
@@ -1,9 +1,11 @@
+/**
+ * DBSyncer Copyright 2020-2023 All Rights Reserved.
+ */
 package org.dbsyncer.web;
 
 import org.dbsyncer.common.util.DateFormatUtil;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
-import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;
 import org.springframework.boot.info.BuildProperties;
 import org.springframework.cache.annotation.EnableCaching;
 import org.springframework.core.io.ClassPathResource;
@@ -19,7 +21,18 @@ import java.util.Properties;
 @EnableAsync
 @EnableScheduling
 @EnableCaching
-@SpringBootApplication(scanBasePackages = "org.dbsyncer", exclude = DataSourceAutoConfiguration.class)
+@SpringBootApplication(scanBasePackages = {
+        "org.dbsyncer.biz",
+        "org.dbsyncer.cluster",
+        "org.dbsyncer.common",
+        "org.dbsyncer.manager",
+        "org.dbsyncer.connector.base",
+        "org.dbsyncer.parser",
+        "org.dbsyncer.plugin",
+        "org.dbsyncer.sdk",
+        "org.dbsyncer.storage",
+        "org.dbsyncer.web"
+})
 public class Application {
 
     public static void main(String[] args) throws IOException {
diff --git a/dbsyncer-web/src/main/resources/application.properties b/dbsyncer-web/src/main/resources/application.properties
index 6d230996..b059a833 100644
--- a/dbsyncer-web/src/main/resources/application.properties
+++ b/dbsyncer-web/src/main/resources/application.properties
@@ -50,8 +50,8 @@ dbsyncer.parser.table.group.buffer-period-millisecond=300
 #storage
 # 数据存储类型:disk(默认)/mysql(推荐生产环境使用)
 # disk-磁盘:/data/config(驱动配置)|data(按驱动分别存储增量数据)|log(系统日志)
-dbsyncer.storage.type=mysql
-dbsyncer.storage.mysql.url=jdbc:mysql://127.0.0.1:3305/dbsyncer?rewriteBatchedStatements=true&seUnicode=true&characterEncoding=UTF8&serverTimezone=Asia/Shanghai&useSSL=false&verifyServerCertificate=false&autoReconnect=true
+dbsyncer.storage.type=disk
+dbsyncer.storage.mysql.url=jdbc:mysql://127.0.0.1:3306/dbsyncer?rewriteBatchedStatements=true&seUnicode=true&characterEncoding=UTF8&serverTimezone=Asia/Shanghai&useSSL=false&verifyServerCertificate=false&autoReconnect=true
 dbsyncer.storage.mysql.username=root
 dbsyncer.storage.mysql.password=123
 # [StorageBufferActuator]线程数
-- 
Gitee


From c1592c62124804c278dea7b76ebbabaa9c86353d Mon Sep 17 00:00:00 2001
From: life <13122192336@163.com>
Date: Mon, 18 Dec 2023 14:10:43 +0800
Subject: [PATCH 88/98] =?UTF-8?q?=E5=85=BC=E5=AE=B9oracle=20ddl=E5=90=8C?=
 =?UTF-8?q?=E6=AD=A5?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../org/dbsyncer/connector/oracle/cdc/OracleListener.java | 8 ++++++++
 .../java/org/dbsyncer/parser/ddl/alter/AddStrategy.java   | 1 +
 .../dbsyncer/parser/flush/impl/GeneralBufferActuator.java | 4 ++--
 3 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java
index 1c5d9f53..6c487c5b 100644
--- a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java
+++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java
@@ -9,6 +9,7 @@ import net.sf.jsqlparser.JSQLParserException;
 import net.sf.jsqlparser.parser.CCJSqlParserUtil;
 import net.sf.jsqlparser.schema.Table;
 import net.sf.jsqlparser.statement.Statement;
+import net.sf.jsqlparser.statement.alter.Alter;
 import net.sf.jsqlparser.statement.delete.Delete;
 import net.sf.jsqlparser.statement.insert.Insert;
 import net.sf.jsqlparser.statement.update.Update;
@@ -24,6 +25,7 @@ import org.dbsyncer.connector.oracle.logminer.parser.impl.UpdateSql;
 import org.dbsyncer.sdk.config.DatabaseConfig;
 import org.dbsyncer.sdk.constant.ConnectorConstant;
 import org.dbsyncer.sdk.listener.AbstractDatabaseListener;
+import org.dbsyncer.sdk.listener.event.DDLChangedEvent;
 import org.dbsyncer.sdk.listener.event.RowChangedEvent;
 import org.dbsyncer.sdk.model.ChangedOffset;
 import org.dbsyncer.sdk.model.Field;
@@ -140,6 +142,12 @@ public class OracleListener extends AbstractDatabaseListener {
         }
 
         // TODO ddl
+        if (statement instanceof Alter){
+            Alter alter = (Alter) statement;
+            String tableName = StringUtil.replace(alter.getTable().getName(), StringUtil.DOUBLE_QUOTATION, "");
+            logger.info("sql:{}", event.getRedoSql());
+            changeEvent(new DDLChangedEvent(null, tableName, ConnectorConstant.OPERTION_ALTER, event.getRedoSql(), null, event.getScn()));
+        }
     }
 
     @Override
diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ddl/alter/AddStrategy.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ddl/alter/AddStrategy.java
index 85342452..9c31faf8 100644
--- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ddl/alter/AddStrategy.java
+++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ddl/alter/AddStrategy.java
@@ -61,6 +61,7 @@ public class AddStrategy implements AlterStrategy {
             columnDataType.setColumnSpecs(columnSpecs);
             String columName = columnDataType.getColumnName();
             columName = StringUtil.replace(columName, "`", "");
+            columName = StringUtil.replace(columName, StringUtil.DOUBLE_QUOTATION, "");
             Field field = new Field(columName, columnDataType.getColDataType().getDataType(),
                     0);//感觉不需要都行，只需要名称，后续可以自己刷新
             ddlConfig.getAddFields().add(field);
diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/GeneralBufferActuator.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/GeneralBufferActuator.java
index dbee58e8..30301714 100644
--- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/GeneralBufferActuator.java
+++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/GeneralBufferActuator.java
@@ -198,7 +198,7 @@ public class GeneralBufferActuator extends AbstractBufferActuator originalFieldMappings = tableGroup.getFieldMapping();
@@ -233,7 +233,7 @@ public class GeneralBufferActuator extends AbstractBufferActuator
Date: Mon, 18 Dec 2023 18:22:44 +0800
Subject: [PATCH 89/98] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=85=B6=E4=BB=96?=
 =?UTF-8?q?=E7=9A=84=E5=85=BC=E5=AE=B9oracle=E5=B9=B6=E4=B8=94=E8=A7=84?=
 =?UTF-8?q?=E8=8C=83=E5=BC=95=E7=94=A8?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../src/main/java/org/dbsyncer/common/util/StringUtil.java | 2 ++
 .../java/org/dbsyncer/parser/ddl/alter/AddStrategy.java    | 7 ++++---
 .../java/org/dbsyncer/parser/ddl/alter/ChangeStrategy.java | 7 +++++--
 .../java/org/dbsyncer/parser/ddl/alter/DropStrategy.java   | 6 ++++--
 .../java/org/dbsyncer/parser/ddl/alter/ModifyStrategy.java | 3 ++-
 .../dbsyncer/parser/flush/impl/GeneralBufferActuator.java  | 2 +-
 6 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/dbsyncer-common/src/main/java/org/dbsyncer/common/util/StringUtil.java b/dbsyncer-common/src/main/java/org/dbsyncer/common/util/StringUtil.java
index 0ad555d3..60241d26 100644
--- a/dbsyncer-common/src/main/java/org/dbsyncer/common/util/StringUtil.java
+++ b/dbsyncer-common/src/main/java/org/dbsyncer/common/util/StringUtil.java
@@ -23,6 +23,8 @@ public abstract class StringUtil {
 
     public static final String FORWARD_SLASH = "/";
 
+    public static final String BACK_QUOTE = "`";
+
     public static boolean equals(CharSequence cs1, CharSequence cs2) {
         return StringUtils.equals(cs1, cs2);
     }
diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ddl/alter/AddStrategy.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ddl/alter/AddStrategy.java
index 9c31faf8..98f9a1c1 100644
--- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ddl/alter/AddStrategy.java
+++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ddl/alter/AddStrategy.java
@@ -40,7 +40,8 @@ public class AddStrategy implements AlterStrategy {
             boolean findColumn = false;
             List columnSpecs = new LinkedList<>();
             for (String spe : columnDataType.getColumnSpecs()) {//对一before，after进行处理
-                spe = StringUtil.replace(spe, "`", "");
+                spe = StringUtil.replace(spe, StringUtil.BACK_QUOTE, StringUtil.EMPTY);
+                spe = StringUtil.replace(spe, StringUtil.DOUBLE_QUOTATION, StringUtil.EMPTY);
                 if (findColumn) {
                     //对before（after）字段进行映射
                     String finalSpe = spe;
@@ -60,8 +61,8 @@ public class AddStrategy implements AlterStrategy {
             }
             columnDataType.setColumnSpecs(columnSpecs);
             String columName = columnDataType.getColumnName();
-            columName = StringUtil.replace(columName, "`", "");
-            columName = StringUtil.replace(columName, StringUtil.DOUBLE_QUOTATION, "");
+            columName = StringUtil.replace(columName, StringUtil.BACK_QUOTE, StringUtil.EMPTY);
+            columName = StringUtil.replace(columName, StringUtil.DOUBLE_QUOTATION, StringUtil.EMPTY);
             Field field = new Field(columName, columnDataType.getColDataType().getDataType(),
                     0);//感觉不需要都行，只需要名称，后续可以自己刷新
             ddlConfig.getAddFields().add(field);
diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ddl/alter/ChangeStrategy.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ddl/alter/ChangeStrategy.java
index 2fa1c50a..0a7b777d 100644
--- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ddl/alter/ChangeStrategy.java
+++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ddl/alter/ChangeStrategy.java
@@ -19,9 +19,12 @@ public class ChangeStrategy implements AlterStrategy {
 
     @Override
     public void parse(AlterExpression expression, DDLConfig ddlConfig, List originalFieldMappings) {
-        String oldColumnName = StringUtil.replace(expression.getColumnOldName(), "`", "");
+        String oldColumnName = StringUtil.replace(expression.getColumnOldName(), StringUtil.BACK_QUOTE, StringUtil.EMPTY);
+        oldColumnName = StringUtil.replace(oldColumnName, StringUtil.DOUBLE_QUOTATION, StringUtil.EMPTY);
         ddlConfig.setSourceColumnName(oldColumnName);
-        FieldMapping fieldMapping = originalFieldMappings.stream().filter(x -> StringUtil.equals(x.getSource().getName(), oldColumnName)).findFirst().orElse(null);
+        String finalOldColumnName = oldColumnName;
+        FieldMapping fieldMapping = originalFieldMappings.stream().filter(x -> StringUtil.equals(x.getSource().getName(),
+                finalOldColumnName)).findFirst().orElse(null);
         if (fieldMapping != null) {
             expression.setColumnOldName(fieldMapping.getTarget().getName());
             for (AlterExpression.ColumnDataType columnDataType : expression.getColDataTypeList()) {
diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ddl/alter/DropStrategy.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ddl/alter/DropStrategy.java
index f1a47003..abd22557 100644
--- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ddl/alter/DropStrategy.java
+++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ddl/alter/DropStrategy.java
@@ -36,11 +36,13 @@ public class DropStrategy implements AlterStrategy {
      * @param originalFieldMappings
      */
     private void dropColumn(AlterExpression expression, DDLConfig ddlConfig, List originalFieldMappings) {
-        String columnName = StringUtil.replace(expression.getColumnName(), "`", "");
+        String columnName = StringUtil.replace(expression.getColumnName(), StringUtil.BACK_QUOTE, StringUtil.EMPTY);
+        columnName = StringUtil.replace(columnName,StringUtil.DOUBLE_QUOTATION,StringUtil.EMPTY);
         Field field = new Field(columnName, null, 0);
         //需要把列替换成目标的列名
+        String finalColumnName = columnName;
         originalFieldMappings.stream()
-                .filter(x -> StringUtil.equals(x.getSource().getName(), columnName)).findFirst()
+                .filter(x -> StringUtil.equals(x.getSource().getName(), finalColumnName)).findFirst()
                 .ifPresent(fieldMapping -> expression.setColumnName(fieldMapping.getTarget().getName()));
         //加入还是原名
         ddlConfig.getRemoveFields().add(field);
diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ddl/alter/ModifyStrategy.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ddl/alter/ModifyStrategy.java
index fd960fdb..6f1b4cfd 100644
--- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ddl/alter/ModifyStrategy.java
+++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/ddl/alter/ModifyStrategy.java
@@ -22,7 +22,8 @@ public class ModifyStrategy implements AlterStrategy {
     public void parse(AlterExpression expression, DDLConfig ddlConfig, List originalFieldMappings) {
         //先查找到当前的表和目标的表对应的字段
         for (AlterExpression.ColumnDataType columnDataType : expression.getColDataTypeList()) {
-            String columnName = StringUtil.replace(columnDataType.getColumnName(), "`", "");
+            String columnName = StringUtil.replace(columnDataType.getColumnName(), StringUtil.BACK_QUOTE, StringUtil.EMPTY);
+            columnName =StringUtil.replace(columnName,StringUtil.DOUBLE_QUOTATION,StringUtil.EMPTY);
             for (FieldMapping fieldMapping : originalFieldMappings) {
                 if (StringUtil.equals(fieldMapping.getSource().getName(), columnName)) {
                     //TODO life 找到目标的表名，先是alter进行属性替换，然后config记录新的
diff --git a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/GeneralBufferActuator.java b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/GeneralBufferActuator.java
index 30301714..62670209 100644
--- a/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/GeneralBufferActuator.java
+++ b/dbsyncer-parser/src/main/java/org/dbsyncer/parser/flush/impl/GeneralBufferActuator.java
@@ -197,7 +197,7 @@ public class GeneralBufferActuator extends AbstractBufferActuator
Date: Tue, 19 Dec 2023 00:33:17 +0800
Subject: [PATCH 90/98] =?UTF-8?q?=E4=BC=98=E5=8C=96redo=E8=A7=A3=E6=9E=90?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../org/dbsyncer/common/util/StringUtil.java  |  2 +
 .../connector/oracle/cdc/OracleListener.java  | 51 +++++-----
 .../logminer/parser/AbstractParser.java       | 96 ++++++++++++-------
 .../logminer/parser/OracleTypeParser.java     | 34 -------
 .../oracle/logminer/parser/Parser.java        |  3 -
 .../logminer/parser/impl/DeleteSql.java       | 14 +--
 .../logminer/parser/impl/InsertSql.java       |  4 +-
 .../logminer/parser/impl/UpdateSql.java       | 15 ++-
 .../connector/schema/FloatValueMapper.java    |  4 +
 9 files changed, 109 insertions(+), 114 deletions(-)
 delete mode 100644 dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/OracleTypeParser.java

diff --git a/dbsyncer-common/src/main/java/org/dbsyncer/common/util/StringUtil.java b/dbsyncer-common/src/main/java/org/dbsyncer/common/util/StringUtil.java
index 60241d26..57e4142e 100644
--- a/dbsyncer-common/src/main/java/org/dbsyncer/common/util/StringUtil.java
+++ b/dbsyncer-common/src/main/java/org/dbsyncer/common/util/StringUtil.java
@@ -25,6 +25,8 @@ public abstract class StringUtil {
 
     public static final String BACK_QUOTE = "`";
 
+    public static final String POINT = ".";
+
     public static boolean equals(CharSequence cs1, CharSequence cs2) {
         return StringUtils.equals(cs1, cs2);
     }
diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java
index 6c487c5b..109010cb 100644
--- a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java
+++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java
@@ -3,8 +3,6 @@
  */
 package org.dbsyncer.connector.oracle.cdc;
 
-import java.time.Instant;
-import java.util.concurrent.TimeUnit;
 import net.sf.jsqlparser.JSQLParserException;
 import net.sf.jsqlparser.parser.CCJSqlParserUtil;
 import net.sf.jsqlparser.schema.Table;
@@ -16,7 +14,6 @@ import net.sf.jsqlparser.statement.update.Update;
 import org.dbsyncer.common.QueueOverflowException;
 import org.dbsyncer.common.util.StringUtil;
 import org.dbsyncer.connector.oracle.OracleException;
-import org.dbsyncer.connector.oracle.logminer.parser.AbstractParser;
 import org.dbsyncer.connector.oracle.logminer.LogMiner;
 import org.dbsyncer.connector.oracle.logminer.RedoEvent;
 import org.dbsyncer.connector.oracle.logminer.parser.impl.DeleteSql;
@@ -33,9 +30,11 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.sql.SQLException;
+import java.time.Instant;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
 
 /**
  * @Author AE86
@@ -83,12 +82,12 @@ public class OracleListener extends AbstractDatabaseListener {
         }
     }
 
-    private void trySendEvent(RowChangedEvent event){
+    private void trySendEvent(RowChangedEvent event) {
         try {
             // 如果消费事件失败，重试
             long now = Instant.now().toEpochMilli();
             boolean isReTry = false;
-            while (logMiner.isConnected()){
+            while (logMiner.isConnected()) {
                 try {
                     sendChangedEvent(event);
                     break;
@@ -116,37 +115,43 @@ public class OracleListener extends AbstractDatabaseListener {
      * @throws Exception
      */
     private void parseEvent(RedoEvent event) throws Exception {
-        // TODO life 注意拦截子查询, 或修改主键值情况
         Statement statement = CCJSqlParserUtil.parse(event.getRedoSql());
         if (statement instanceof Update) {
             Update update = (Update) statement;
-            UpdateSql parser = new UpdateSql(update);
-            setTable(parser, update.getTable());
-            trySendEvent(new RowChangedEvent(parser.getTableName(), ConnectorConstant.OPERTION_UPDATE, parser.parseColumns(), null, event.getScn()));
+            String tableName = getTableName(update.getTable());
+            if (tableFiledMap.containsKey(tableName)) {
+                UpdateSql parser = new UpdateSql(update, tableFiledMap.get(tableName));
+                trySendEvent(new RowChangedEvent(tableName, ConnectorConstant.OPERTION_UPDATE, parser.parseColumns(), null, event.getScn()));
+            }
             return;
         }
 
         if (statement instanceof Insert) {
             Insert insert = (Insert) statement;
-            InsertSql parser = new InsertSql(insert);
-            setTable(parser, insert.getTable());
-            trySendEvent(new RowChangedEvent(parser.getTableName(), ConnectorConstant.OPERTION_INSERT, parser.parseColumns(), null, event.getScn()));
+            String tableName = getTableName(insert.getTable());
+            if (tableFiledMap.containsKey(tableName)) {
+                InsertSql parser = new InsertSql(insert, tableFiledMap.get(tableName));
+                trySendEvent(new RowChangedEvent(tableName, ConnectorConstant.OPERTION_INSERT, parser.parseColumns(), null, event.getScn()));
+            }
             return;
         }
 
         if (statement instanceof Delete) {
             Delete delete = (Delete) statement;
-            DeleteSql parser = new DeleteSql(delete);
-            setTable(parser, delete.getTable());
-            trySendEvent(new RowChangedEvent(parser.getTableName(), ConnectorConstant.OPERTION_DELETE, parser.parseColumns(), null, event.getScn()));
+            String tableName = getTableName(delete.getTable());
+            if (tableFiledMap.containsKey(tableName)) {
+                DeleteSql parser = new DeleteSql(delete, tableFiledMap.get(tableName));
+                trySendEvent(new RowChangedEvent(tableName, ConnectorConstant.OPERTION_DELETE, parser.parseColumns(), null, event.getScn()));
+            }
         }
 
-        // TODO ddl
-        if (statement instanceof Alter){
+        if (statement instanceof Alter) {
             Alter alter = (Alter) statement;
-            String tableName = StringUtil.replace(alter.getTable().getName(), StringUtil.DOUBLE_QUOTATION, "");
-            logger.info("sql:{}", event.getRedoSql());
-            changeEvent(new DDLChangedEvent(null, tableName, ConnectorConstant.OPERTION_ALTER, event.getRedoSql(), null, event.getScn()));
+            String tableName = getTableName(alter.getTable());
+            if (tableFiledMap.containsKey(tableName)) {
+                logger.info("sql:{}", event.getRedoSql());
+                changeEvent(new DDLChangedEvent(null, tableName, ConnectorConstant.OPERTION_ALTER, event.getRedoSql(), null, event.getScn()));
+            }
         }
     }
 
@@ -166,10 +171,8 @@ public class OracleListener extends AbstractDatabaseListener {
         snapshot.put(REDO_POSITION, String.valueOf(offset.getPosition()));
     }
 
-    private AbstractParser setTable(AbstractParser parser, Table table) {
-        parser.setTableName(table == null ? StringUtil.EMPTY : StringUtil.replace(table.getName(), StringUtil.DOUBLE_QUOTATION, StringUtil.EMPTY));
-        parser.setFields(tableFiledMap.get(parser.getTableName()));
-        return parser;
+    private String getTableName(Table table) {
+        return table == null ? StringUtil.EMPTY : StringUtil.replace(table.getName(), StringUtil.DOUBLE_QUOTATION, StringUtil.EMPTY);
     }
 
 }
\ No newline at end of file
diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/AbstractParser.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/AbstractParser.java
index 068b4e38..3ccb027b 100644
--- a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/AbstractParser.java
+++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/AbstractParser.java
@@ -3,18 +3,26 @@
  */
 package org.dbsyncer.connector.oracle.logminer.parser;
 
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
 import net.sf.jsqlparser.expression.BinaryExpression;
 import net.sf.jsqlparser.expression.Expression;
 import net.sf.jsqlparser.expression.Function;
 import net.sf.jsqlparser.expression.NullValue;
+import net.sf.jsqlparser.expression.StringValue;
+import net.sf.jsqlparser.expression.operators.relational.ExpressionList;
 import net.sf.jsqlparser.expression.operators.relational.IsNullExpression;
 import net.sf.jsqlparser.schema.Column;
+import org.dbsyncer.common.util.CollectionUtils;
+import org.dbsyncer.common.util.DateFormatUtil;
 import org.dbsyncer.common.util.StringUtil;
 import org.dbsyncer.sdk.model.Field;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
 
 /**
  * @Author AE86
@@ -23,65 +31,87 @@ import org.dbsyncer.sdk.model.Field;
  */
 public abstract class AbstractParser implements Parser {
 
-    protected Map columnMap = new HashMap<>();
-    protected String tableName;
+    private final Logger logger = LoggerFactory.getLogger(getClass());
+    protected Map columnMap = new HashMap<>();
     protected List fields;
 
     public void findColumn(Expression expression) {
-        if (expression instanceof IsNullExpression){
+        if (expression instanceof IsNullExpression) {
             IsNullExpression isNullExpression = (IsNullExpression) expression;
             Column column = (Column) isNullExpression.getLeftExpression();
-            columnMap.put(StringUtil.replace(column.getColumnName(), StringUtil.DOUBLE_QUOTATION, StringUtil.EMPTY),
-                    null);
+            columnMap.put(StringUtil.replace(column.getColumnName(), StringUtil.DOUBLE_QUOTATION, StringUtil.EMPTY), null);
             return;
         }
 
         BinaryExpression binaryExpression = (BinaryExpression) expression;
         if (binaryExpression.getLeftExpression() instanceof Column) {
             Column column = (Column) binaryExpression.getLeftExpression();
-            columnMap.put(StringUtil.replace(column.getColumnName(), StringUtil.DOUBLE_QUOTATION, StringUtil.EMPTY),
-                    parserValue(binaryExpression.getRightExpression()));
+            columnMap.put(StringUtil.replace(column.getColumnName(), StringUtil.DOUBLE_QUOTATION, StringUtil.EMPTY), parserValue(binaryExpression.getRightExpression()));
             return;
         }
         findColumn(binaryExpression.getLeftExpression());
         findColumn(binaryExpression.getRightExpression());
     }
 
-    public String parserValue(Expression expression){
-        if (expression instanceof Function){
-            return parserFunction((Function) expression);
-        }
-        if (expression instanceof NullValue){
+    public Object parserValue(Expression expression) {
+        if (expression instanceof NullValue) {
             return null;
         }
-        return expression.toString();
+        // 解析sql的function，只取到关键的字符串
+        if (expression instanceof Function) {
+            return parseFunction((Function) expression);
+        }
+        if (expression instanceof StringValue) {
+            StringValue val = (StringValue) expression;
+            return val.getValue();
+        }
+        return null;
     }
 
-    //解析sql的function，只取到关键的字符串
-    public String parserFunction(Function function){
-        if (function.getMultipartName().get(0).equals("TO_DATE")){
-            return StringUtil.replace(function.getParameters().get(0).toString(),StringUtil.SINGLE_QUOTATION,StringUtil.EMPTY);
+    public List columnMapToData() {
+        List data = new LinkedList<>();
+        for (Field field : fields) {
+            data.add(columnMap.get(field.getName()));
         }
-        return "";
+        return data;
     }
 
+    private Object parseFunction(Function function) {
+        List multipartName = function.getMultipartName();
+        ExpressionList parameters = function.getParameters();
+        if (CollectionUtils.isEmpty(multipartName) || CollectionUtils.isEmpty(parameters)) {
+            return null;
+        }
 
-    public List columnMapToData(){
-        List data = new LinkedList<>();
-        for (Field field: fields) {
-            Object value = OracleTypeParser.convertToJavaType(field,columnMap.get(field.getName()));
-            data.add(value);
+        String nameType = Objects.toString(multipartName.get(0));
+        Object value = parameters.get(0);
+        if (nameType == null || value == null) {
+            return null;
         }
-        return data;
+
+        if (value instanceof StringValue) {
+            StringValue val = (StringValue) value;
+            value = val.getValue();
+        }
+        try {
+            switch (nameType) {
+                case "TO_DATE":
+                    return toDate(value);
+                case "TO_TIMESTAMP":
+                    return toTimestamp(value);
+            }
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+        }
+        return value;
     }
 
-    @Override
-    public String getTableName() {
-        return tableName;
+    private Object toDate(Object value) {
+        return DateFormatUtil.stringToTimestamp(Objects.toString(value));
     }
 
-    public void setTableName(String tableName) {
-        this.tableName = tableName;
+    private Object toTimestamp(Object value) {
+        return DateFormatUtil.stringToTimestamp(StringUtil.replace(Objects.toString(value), StringUtil.POINT, StringUtil.EMPTY));
     }
 
     public void setFields(List fields) {
diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/OracleTypeParser.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/OracleTypeParser.java
deleted file mode 100644
index 528919a5..00000000
--- a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/OracleTypeParser.java
+++ /dev/null
@@ -1,34 +0,0 @@
-package org.dbsyncer.connector.oracle.logminer.parser;
-
-import java.math.BigInteger;
-import java.sql.Types;
-import org.dbsyncer.common.util.DateFormatUtil;
-import org.dbsyncer.common.util.StringUtil;
-import org.dbsyncer.sdk.model.Field;
-
-/**
- * @Description :oracle解析到java类
- * @Param :
- * @return :
- * @author : life
- * @date : 2023/12/15  11:25
- */
-public class OracleTypeParser {
-    public static Object convertToJavaType(Field field,String value){
-        if (value == null){
-            return null;
-        }
-        switch (field.getType()) {
-            case Types.DECIMAL:
-                return new BigInteger(
-                        StringUtil.replace(value, StringUtil.SINGLE_QUOTATION, StringUtil.EMPTY));
-            case Types.TIMESTAMP:
-                return DateFormatUtil.stringToTimestamp(StringUtil.replace(value,StringUtil.SINGLE_QUOTATION,StringUtil.EMPTY));
-            default:
-                return StringUtil.replace(value,StringUtil.SINGLE_QUOTATION,StringUtil.EMPTY);
-        }
-    }
-
-
-
-}
diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/Parser.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/Parser.java
index 670f21fb..0042728f 100644
--- a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/Parser.java
+++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/Parser.java
@@ -6,8 +6,5 @@ package org.dbsyncer.connector.oracle.logminer.parser;
 import java.util.List;
 
 public interface Parser {
-
-    String getTableName();
-
     List parseColumns();
 }
diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/impl/DeleteSql.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/impl/DeleteSql.java
index 3059442a..0c4d1cad 100644
--- a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/impl/DeleteSql.java
+++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/impl/DeleteSql.java
@@ -4,12 +4,9 @@
 package org.dbsyncer.connector.oracle.logminer.parser.impl;
 
 import net.sf.jsqlparser.statement.delete.Delete;
-import org.dbsyncer.common.util.StringUtil;
 import org.dbsyncer.connector.oracle.logminer.parser.AbstractParser;
 import org.dbsyncer.sdk.model.Field;
 
-import java.math.BigInteger;
-import java.sql.Types;
 import java.util.LinkedList;
 import java.util.List;
 
@@ -22,8 +19,9 @@ public class DeleteSql extends AbstractParser {
 
     private Delete delete;
 
-    public DeleteSql(Delete delete) {
+    public DeleteSql(Delete delete, List fields) {
         this.delete = delete;
+        setFields(fields);
     }
 
     @Override
@@ -32,13 +30,7 @@ public class DeleteSql extends AbstractParser {
         List data = new LinkedList<>();
         for (Field field : fields) {
             if (field.isPk()) {
-                Object value = columnMap.get(field.getName());
-                switch (field.getType()) {
-                    case Types.DECIMAL:
-                        value = new BigInteger(StringUtil.replace(value.toString(), StringUtil.SINGLE_QUOTATION, StringUtil.EMPTY));
-                        break;
-                }
-                data.add(value);
+                data.add(columnMap.get(field.getName()));
             } else {
                 data.add(null);
             }
diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/impl/InsertSql.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/impl/InsertSql.java
index 00b58dab..bd536c66 100644
--- a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/impl/InsertSql.java
+++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/impl/InsertSql.java
@@ -10,6 +10,7 @@ import net.sf.jsqlparser.schema.Column;
 import net.sf.jsqlparser.statement.insert.Insert;
 import org.dbsyncer.common.util.StringUtil;
 import org.dbsyncer.connector.oracle.logminer.parser.AbstractParser;
+import org.dbsyncer.sdk.model.Field;
 
 /**
  * @Author AE86
@@ -20,8 +21,9 @@ public class InsertSql extends AbstractParser {
 
     private Insert insert;
 
-    public InsertSql(Insert insert) {
+    public InsertSql(Insert insert, List fields) {
         this.insert = insert;
+        setFields(fields);
     }
 
     @Override
diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/impl/UpdateSql.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/impl/UpdateSql.java
index f97c8460..79b6ba08 100644
--- a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/impl/UpdateSql.java
+++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/impl/UpdateSql.java
@@ -7,6 +7,7 @@ import net.sf.jsqlparser.statement.update.Update;
 import net.sf.jsqlparser.statement.update.UpdateSet;
 import org.dbsyncer.common.util.StringUtil;
 import org.dbsyncer.connector.oracle.logminer.parser.AbstractParser;
+import org.dbsyncer.sdk.model.Field;
 
 import java.util.List;
 
@@ -19,8 +20,9 @@ public class UpdateSql extends AbstractParser {
 
     private Update update;
 
-    public UpdateSql(Update update) {
+    public UpdateSql(Update update, List fields) {
         this.update = update;
+        setFields(fields);
     }
 
     @Override
@@ -30,13 +32,10 @@ public class UpdateSql extends AbstractParser {
         return columnMapToData();
     }
 
-    private void passerSet(List updateSets){
-        //解析替换
-        for (UpdateSet updateSet:updateSets) {
-            String columnName = StringUtil.replace(updateSet.getColumn(0).getColumnName(),
-                    StringUtil.DOUBLE_QUOTATION,StringUtil.EMPTY);
-            String value = parserValue(updateSet.getValue(0));
-            columnMap.put(columnName,value);
+    private void passerSet(List updateSets) {
+        for (UpdateSet updateSet : updateSets) {
+            String columnName = StringUtil.replace(updateSet.getColumn(0).getColumnName(), StringUtil.DOUBLE_QUOTATION, StringUtil.EMPTY);
+            columnMap.put(columnName, parserValue(updateSet.getValue(0)));
         }
     }
 
diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/FloatValueMapper.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/FloatValueMapper.java
index ec9b0160..3cbb2204 100644
--- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/FloatValueMapper.java
+++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/connector/schema/FloatValueMapper.java
@@ -23,6 +23,10 @@ public class FloatValueMapper extends AbstractValueMapper {
             Double dbl = (Double) val;
             return dbl.floatValue();
         }
+        if (val instanceof String) {
+            String strVal = (String) val;
+            return Float.parseFloat(strVal);
+        }
         throw new SdkException(String.format("%s can not find type [%s], val [%s]", getClass().getSimpleName(), val.getClass(), val));
     }
 }
\ No newline at end of file
-- 
Gitee


From 7007001437863f65a609eb44bedae44dca6338e7 Mon Sep 17 00:00:00 2001
From: AE86 <836391306@qq.com>
Date: Tue, 19 Dec 2023 00:53:00 +0800
Subject: [PATCH 91/98] upgrade version

---
 dbsyncer-biz/pom.xml                                        | 2 +-
 dbsyncer-cluster/pom.xml                                    | 2 +-
 dbsyncer-common/pom.xml                                     | 2 +-
 dbsyncer-connector/dbsyncer-connector-base/pom.xml          | 2 +-
 dbsyncer-connector/dbsyncer-connector-elasticsearch/pom.xml | 2 +-
 dbsyncer-connector/dbsyncer-connector-file/pom.xml          | 2 +-
 dbsyncer-connector/dbsyncer-connector-kafka/pom.xml         | 2 +-
 dbsyncer-connector/dbsyncer-connector-mysql/pom.xml         | 2 +-
 dbsyncer-connector/dbsyncer-connector-oracle/pom.xml        | 2 +-
 dbsyncer-connector/dbsyncer-connector-postgresql/pom.xml    | 2 +-
 dbsyncer-connector/dbsyncer-connector-sqlite/pom.xml        | 2 +-
 dbsyncer-connector/dbsyncer-connector-sqlserver/pom.xml     | 2 +-
 dbsyncer-connector/pom.xml                                  | 2 +-
 dbsyncer-manager/pom.xml                                    | 2 +-
 dbsyncer-parser/pom.xml                                     | 2 +-
 dbsyncer-plugin/pom.xml                                     | 2 +-
 dbsyncer-sdk/pom.xml                                        | 2 +-
 dbsyncer-storage/pom.xml                                    | 2 +-
 dbsyncer-web/pom.xml                                        | 2 +-
 pom.xml                                                     | 2 +-
 version.sh                                                  | 2 +-
 21 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/dbsyncer-biz/pom.xml b/dbsyncer-biz/pom.xml
index b6d8a905..dace0f78 100644
--- a/dbsyncer-biz/pom.xml
+++ b/dbsyncer-biz/pom.xml
@@ -5,7 +5,7 @@
 	
         dbsyncer
         org.ghi
-        1.2.8_1206_pre
+        2.0.0
     
 	4.0.0
 	dbsyncer-biz
diff --git a/dbsyncer-cluster/pom.xml b/dbsyncer-cluster/pom.xml
index cb4f01cc..e08b4203 100644
--- a/dbsyncer-cluster/pom.xml
+++ b/dbsyncer-cluster/pom.xml
@@ -5,7 +5,7 @@
     
         dbsyncer
         org.ghi
-        1.2.8_1206_pre
+        2.0.0
     
     4.0.0
     dbsyncer-cluster
diff --git a/dbsyncer-common/pom.xml b/dbsyncer-common/pom.xml
index 9b403bee..448942be 100644
--- a/dbsyncer-common/pom.xml
+++ b/dbsyncer-common/pom.xml
@@ -5,7 +5,7 @@
     
         dbsyncer
         org.ghi
-        1.2.8_1206_pre
+        2.0.0
     
     4.0.0
     dbsyncer-common
diff --git a/dbsyncer-connector/dbsyncer-connector-base/pom.xml b/dbsyncer-connector/dbsyncer-connector-base/pom.xml
index dfb0ef81..37f8cfdf 100644
--- a/dbsyncer-connector/dbsyncer-connector-base/pom.xml
+++ b/dbsyncer-connector/dbsyncer-connector-base/pom.xml
@@ -5,7 +5,7 @@
     
         dbsyncer-connector
         org.ghi
-        1.2.8_1206_pre
+        2.0.0
     
     4.0.0
 
diff --git a/dbsyncer-connector/dbsyncer-connector-elasticsearch/pom.xml b/dbsyncer-connector/dbsyncer-connector-elasticsearch/pom.xml
index 529ef199..40574412 100644
--- a/dbsyncer-connector/dbsyncer-connector-elasticsearch/pom.xml
+++ b/dbsyncer-connector/dbsyncer-connector-elasticsearch/pom.xml
@@ -5,7 +5,7 @@
     
         dbsyncer-connector
         org.ghi
-        1.2.8_1206_pre
+        2.0.0
     
     4.0.0
 
diff --git a/dbsyncer-connector/dbsyncer-connector-file/pom.xml b/dbsyncer-connector/dbsyncer-connector-file/pom.xml
index 98ab006c..9c71355e 100644
--- a/dbsyncer-connector/dbsyncer-connector-file/pom.xml
+++ b/dbsyncer-connector/dbsyncer-connector-file/pom.xml
@@ -5,7 +5,7 @@
     
         dbsyncer-connector
         org.ghi
-        1.2.8_1206_pre
+        2.0.0
     
     4.0.0
 
diff --git a/dbsyncer-connector/dbsyncer-connector-kafka/pom.xml b/dbsyncer-connector/dbsyncer-connector-kafka/pom.xml
index 30fb7606..ffbdb6bf 100644
--- a/dbsyncer-connector/dbsyncer-connector-kafka/pom.xml
+++ b/dbsyncer-connector/dbsyncer-connector-kafka/pom.xml
@@ -5,7 +5,7 @@
     
         dbsyncer-connector
         org.ghi
-        1.2.8_1206_pre
+        2.0.0
     
     4.0.0
 
diff --git a/dbsyncer-connector/dbsyncer-connector-mysql/pom.xml b/dbsyncer-connector/dbsyncer-connector-mysql/pom.xml
index 8fe1a0ca..db89afce 100644
--- a/dbsyncer-connector/dbsyncer-connector-mysql/pom.xml
+++ b/dbsyncer-connector/dbsyncer-connector-mysql/pom.xml
@@ -5,7 +5,7 @@
     
         dbsyncer-connector
         org.ghi
-        1.2.8_1206_pre
+        2.0.0
     
     4.0.0
 
diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/pom.xml b/dbsyncer-connector/dbsyncer-connector-oracle/pom.xml
index c39b7517..ec1e6f83 100644
--- a/dbsyncer-connector/dbsyncer-connector-oracle/pom.xml
+++ b/dbsyncer-connector/dbsyncer-connector-oracle/pom.xml
@@ -5,7 +5,7 @@
     
         dbsyncer-connector
         org.ghi
-        1.2.8_1206_pre
+        2.0.0
     
     4.0.0
 
diff --git a/dbsyncer-connector/dbsyncer-connector-postgresql/pom.xml b/dbsyncer-connector/dbsyncer-connector-postgresql/pom.xml
index b43f7d6f..deeba571 100644
--- a/dbsyncer-connector/dbsyncer-connector-postgresql/pom.xml
+++ b/dbsyncer-connector/dbsyncer-connector-postgresql/pom.xml
@@ -5,7 +5,7 @@
     
         dbsyncer-connector
         org.ghi
-        1.2.8_1206_pre
+        2.0.0
     
     4.0.0
 
diff --git a/dbsyncer-connector/dbsyncer-connector-sqlite/pom.xml b/dbsyncer-connector/dbsyncer-connector-sqlite/pom.xml
index 8d623b3d..a40ced1c 100644
--- a/dbsyncer-connector/dbsyncer-connector-sqlite/pom.xml
+++ b/dbsyncer-connector/dbsyncer-connector-sqlite/pom.xml
@@ -5,7 +5,7 @@
     
         dbsyncer-connector
         org.ghi
-        1.2.8_1206_pre
+        2.0.0
     
     4.0.0
 
diff --git a/dbsyncer-connector/dbsyncer-connector-sqlserver/pom.xml b/dbsyncer-connector/dbsyncer-connector-sqlserver/pom.xml
index d55aad45..b3af0431 100644
--- a/dbsyncer-connector/dbsyncer-connector-sqlserver/pom.xml
+++ b/dbsyncer-connector/dbsyncer-connector-sqlserver/pom.xml
@@ -5,7 +5,7 @@
     
         dbsyncer-connector
         org.ghi
-        1.2.8_1206_pre
+        2.0.0
     
     4.0.0
 
diff --git a/dbsyncer-connector/pom.xml b/dbsyncer-connector/pom.xml
index 94afb27d..075571f0 100644
--- a/dbsyncer-connector/pom.xml
+++ b/dbsyncer-connector/pom.xml
@@ -5,7 +5,7 @@
     
         dbsyncer
         org.ghi
-        1.2.8_1206_pre
+        2.0.0
     
     4.0.0
     dbsyncer-connector
diff --git a/dbsyncer-manager/pom.xml b/dbsyncer-manager/pom.xml
index 8cac578d..108d2a65 100644
--- a/dbsyncer-manager/pom.xml
+++ b/dbsyncer-manager/pom.xml
@@ -5,7 +5,7 @@
     
         dbsyncer
         org.ghi
-        1.2.8_1206_pre
+        2.0.0
     
     4.0.0
     dbsyncer-manager
diff --git a/dbsyncer-parser/pom.xml b/dbsyncer-parser/pom.xml
index db153972..96118c03 100644
--- a/dbsyncer-parser/pom.xml
+++ b/dbsyncer-parser/pom.xml
@@ -5,7 +5,7 @@
     
         dbsyncer
         org.ghi
-        1.2.8_1206_pre
+        2.0.0
     
     4.0.0
     dbsyncer-parser
diff --git a/dbsyncer-plugin/pom.xml b/dbsyncer-plugin/pom.xml
index 620e617c..547d80d8 100644
--- a/dbsyncer-plugin/pom.xml
+++ b/dbsyncer-plugin/pom.xml
@@ -5,7 +5,7 @@
     
         dbsyncer
         org.ghi
-        1.2.8_1206_pre
+        2.0.0
     
     4.0.0
     dbsyncer-plugin
diff --git a/dbsyncer-sdk/pom.xml b/dbsyncer-sdk/pom.xml
index a932fbab..78020b93 100644
--- a/dbsyncer-sdk/pom.xml
+++ b/dbsyncer-sdk/pom.xml
@@ -5,7 +5,7 @@
     
         dbsyncer
         org.ghi
-        1.2.8_1206_pre
+        2.0.0
     
     4.0.0
     dbsyncer-sdk
diff --git a/dbsyncer-storage/pom.xml b/dbsyncer-storage/pom.xml
index 078e4fb5..124091e6 100644
--- a/dbsyncer-storage/pom.xml
+++ b/dbsyncer-storage/pom.xml
@@ -5,7 +5,7 @@
     
         dbsyncer
         org.ghi
-        1.2.8_1206_pre
+        2.0.0
     
     4.0.0
     dbsyncer-storage
diff --git a/dbsyncer-web/pom.xml b/dbsyncer-web/pom.xml
index 0d3a3461..d09a42ea 100644
--- a/dbsyncer-web/pom.xml
+++ b/dbsyncer-web/pom.xml
@@ -5,7 +5,7 @@
     
         dbsyncer
         org.ghi
-        1.2.8_1206_pre
+        2.0.0
     
     4.0.0
     dbsyncer-web
diff --git a/pom.xml b/pom.xml
index 383afc90..d4c640e5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -6,7 +6,7 @@
 
     org.ghi
     dbsyncer
-    1.2.8_1206_pre
+    2.0.0
     pom
     dbsyncer
     https://gitee.com/ghi/dbsyncer
diff --git a/version.sh b/version.sh
index cbc4353e..4415f0e7 100644
--- a/version.sh
+++ b/version.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-VERSION=1.2.7_$(date +"%m%d")
+VERSION=2.0.0_$(date +"%m%d")
 read -p "Please enter a new version number($VERSION)：" APP_VERSION
 if [ -z "$APP_VERSION" ]; then
   APP_VERSION=$VERSION
-- 
Gitee


From c06fb4851e19377da5973d3015f6ca3ec44cfeba Mon Sep 17 00:00:00 2001
From: AE86 <836391306@qq.com>
Date: Tue, 19 Dec 2023 23:43:51 +0800
Subject: [PATCH 92/98] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E6=8F=92=E4=BB=B6?=
 =?UTF-8?q?=E6=89=8B=E5=86=8C?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 dbsyncer-web/src/main/resources/public/plugin/plugin.html | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/dbsyncer-web/src/main/resources/public/plugin/plugin.html b/dbsyncer-web/src/main/resources/public/plugin/plugin.html
index b1585c26..4ad7a968 100644
--- a/dbsyncer-web/src/main/resources/public/plugin/plugin.html
+++ b/dbsyncer-web/src/main/resources/public/plugin/plugin.html
@@ -20,10 +20,9 @@
                         
      
                             
    • 方式1：导入jar
                                 
      dbsyncer-sdk-[[${version}]].jar
      
-                                
      dbsyncer-connector-[[${version}]].jar (非必须)
      
                             
      • 
                             
    • 方式2：引入pom（需要安装到本地）
-                                
      <dependency>
        <groupId>org.ghi</groupId>
        <artifactId>dbsyncer-sdk</artifactId>
        <version>[[${version}]]</version>
      </dependency>
      <dependency>
        <groupId>org.ghi</groupId>
        <artifactId>dbsyncer-connector</artifactId>
        <version>[[${version}]]</version>
      </dependency>
      
+                                
      <dependency>
        <groupId>org.ghi</groupId>
        <artifactId>dbsyncer-sdk</artifactId>
        <version>[[${version}]]</version>
      </dependency>
      
                             
      • 
                         
    
                     
-- 
Gitee


From 927c50af306856a88c6ce1079a78b8824dcf1280 Mon Sep 17 00:00:00 2001
From: life <13122192336@163.com>
Date: Wed, 20 Dec 2023 16:05:21 +0800
Subject: [PATCH 93/98] =?UTF-8?q?=E7=A1=AE=E4=BF=9D=E4=BA=8B=E4=BB=B6?=
 =?UTF-8?q?=E5=8F=91=E9=80=81=E9=A1=BA=E5=BA=8F?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../java/org/dbsyncer/connector/mysql/cdc/MySQLListener.java | 5 +++--
 .../org/dbsyncer/connector/oracle/cdc/OracleListener.java    | 5 +++--
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/cdc/MySQLListener.java b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/cdc/MySQLListener.java
index b4497e5a..deb90a58 100644
--- a/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/cdc/MySQLListener.java
+++ b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/cdc/MySQLListener.java
@@ -27,6 +27,7 @@ import org.dbsyncer.sdk.config.DatabaseConfig;
 import org.dbsyncer.sdk.constant.ConnectorConstant;
 import org.dbsyncer.sdk.constant.DatabaseConstant;
 import org.dbsyncer.sdk.listener.AbstractDatabaseListener;
+import org.dbsyncer.sdk.listener.ChangedEvent;
 import org.dbsyncer.sdk.listener.event.DDLChangedEvent;
 import org.dbsyncer.sdk.listener.event.RowChangedEvent;
 import org.dbsyncer.sdk.model.ChangedOffset;
@@ -170,7 +171,7 @@ public class MySQLListener extends AbstractDatabaseListener {
         snapshot.put(BINLOG_POSITION, String.valueOf(nextPosition));
     }
 
-    private void trySendEvent(RowChangedEvent event){
+    private void trySendEvent(ChangedEvent event){
         try {
             // 如果消费事件失败，重试
             long now = Instant.now().toEpochMilli();
@@ -330,7 +331,7 @@ public class MySQLListener extends AbstractDatabaseListener {
                     String tableName = StringUtil.replace(alter.getTable().getName(), "`", "");
                     if (isFilterTable(data.getDatabase(), tableName)) {
                         logger.info("sql:{}", data.getSql());
-                        changeEvent(new DDLChangedEvent(data.getDatabase(), tableName, ConnectorConstant.OPERTION_ALTER, data.getSql(), client.getBinlogFilename(), client.getBinlogPosition()));
+                        trySendEvent(new DDLChangedEvent(data.getDatabase(), tableName, ConnectorConstant.OPERTION_ALTER, data.getSql(), client.getBinlogFilename(), client.getBinlogPosition()));
                     }
                 } catch (JSQLParserException e) {
                     logger.error("不支持ddl sql，支持标准的sql格式，请查看文档https://gitee.com/ghi/dbsyncer/wikis/%E5%BF%AB%E9%80%9F%E4%BA%86%E8%A7%A3/%E8%A1%A8%E7%BB%93%E6%9E%84%E5%90%8C%E6%AD%A5");
diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java
index 109010cb..dbc1f2c7 100644
--- a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java
+++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/cdc/OracleListener.java
@@ -22,6 +22,7 @@ import org.dbsyncer.connector.oracle.logminer.parser.impl.UpdateSql;
 import org.dbsyncer.sdk.config.DatabaseConfig;
 import org.dbsyncer.sdk.constant.ConnectorConstant;
 import org.dbsyncer.sdk.listener.AbstractDatabaseListener;
+import org.dbsyncer.sdk.listener.ChangedEvent;
 import org.dbsyncer.sdk.listener.event.DDLChangedEvent;
 import org.dbsyncer.sdk.listener.event.RowChangedEvent;
 import org.dbsyncer.sdk.model.ChangedOffset;
@@ -82,7 +83,7 @@ public class OracleListener extends AbstractDatabaseListener {
         }
     }
 
-    private void trySendEvent(RowChangedEvent event) {
+    private void trySendEvent(ChangedEvent event) {
         try {
             // 如果消费事件失败，重试
             long now = Instant.now().toEpochMilli();
@@ -150,7 +151,7 @@ public class OracleListener extends AbstractDatabaseListener {
             String tableName = getTableName(alter.getTable());
             if (tableFiledMap.containsKey(tableName)) {
                 logger.info("sql:{}", event.getRedoSql());
-                changeEvent(new DDLChangedEvent(null, tableName, ConnectorConstant.OPERTION_ALTER, event.getRedoSql(), null, event.getScn()));
+                trySendEvent(new DDLChangedEvent(null, tableName, ConnectorConstant.OPERTION_ALTER, event.getRedoSql(), null, event.getScn()));
             }
         }
     }
-- 
Gitee


From 48413e1fba3c5767380e399e8ad46838336d705b Mon Sep 17 00:00:00 2001
From: AE86 <836391306@qq.com>
Date: Thu, 21 Dec 2023 01:44:24 +0800
Subject: [PATCH 94/98] fix bat

---
 dbsyncer-web/src/main/assembly/bin/startup.bat | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/dbsyncer-web/src/main/assembly/bin/startup.bat b/dbsyncer-web/src/main/assembly/bin/startup.bat
index 0f669a66..4deeb1b2 100644
--- a/dbsyncer-web/src/main/assembly/bin/startup.bat
+++ b/dbsyncer-web/src/main/assembly/bin/startup.bat
@@ -8,7 +8,6 @@ rem set CLASSPATH=.;%JAVA_HOME%\lib;%JAVA_HOME%\lib\dt.jar;%JAVA_HOME%\lib\tools
 for %%F in ("%~dp0\..\") do set "DBS_HOME=%%~dpF"
 echo DBS_HOME=%DBS_HOME%
 cd ../
-echo starting up ...
 
 set SERVER_OPTS=-Xms1024m -Xmx1024m -Xss1m -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=256m
 rem debug model
@@ -17,15 +16,14 @@ rem set IPv4
 rem set SERVER_OPTS=%SERVER_OPTS% -Djava.net.preferIPv4Stack=true -Djava.net.preferIPv4Addresses
 
 rem set jmxremote args
-set HOST=127.0.0.1
-set JMXREMOTE_HOSTNAME=-Djava.rmi.server.hostname=%HOST%
+set JMXREMOTE_HOSTNAME=-Djava.rmi.server.hostname=127.0.0.1
 set JMXREMOTE_PORT=-Dcom.sun.management.jmxremote.port=15099
 set JMXREMOTE_SSL=-Dcom.sun.management.jmxremote.ssl=false
 set JMXREMOTE_AUTH=-Dcom.sun.management.jmxremote.authenticate=true
-set JMXREMOTE_ACCESS=-Dcom.sun.management.jmxremote.access.file=%DBS_HOME%\config\jmxremote.access
-set JMXREMOTE_PASSWORD=-Dcom.sun.management.jmxremote.password.file=%DBS_HOME%\config\jmxremote.password
+set JMXREMOTE_ACCESS=-Dcom.sun.management.jmxremote.access.file=%DBS_HOME%conf\jmxremote.access
+set JMXREMOTE_PASSWORD=-Dcom.sun.management.jmxremote.password.file=%DBS_HOME%conf\jmxremote.password
 rem jmxremote model
-rem SERVER_OPTS=%SERVER_OPTS% %JMXREMOTE_HOSTNAME% %JMXREMOTE_PORT% %JMXREMOTE_SSL% %JMXREMOTE_AUTH% %JMXREMOTE_ACCESS% %JMXREMOTE_PASSWORD%
+rem set SERVER_OPTS=%SERVER_OPTS% %JMXREMOTE_HOSTNAME% %JMXREMOTE_PORT% %JMXREMOTE_SSL% %JMXREMOTE_AUTH% %JMXREMOTE_ACCESS% %JMXREMOTE_PASSWORD%
 
 set ENCRYPT_FILE=%DBS_HOME%bin\dbsyncer_decrypt.dll
 if exist %ENCRYPT_FILE% (
-- 
Gitee


From b2846ffb55e69bbbde7714e89bd3a23180d0c328 Mon Sep 17 00:00:00 2001
From: life <13122192336@163.com>
Date: Sat, 23 Dec 2023 17:37:55 +0800
Subject: [PATCH 95/98] =?UTF-8?q?=E4=BF=AE=E6=AD=A3=E8=A7=A3=E6=9E=90?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../dbsyncer/common/column/ColumnValue.java   |   5 +
 .../logminer/parser/AbstractParser.java       |  90 +++---------
 .../logminer/parser/OracleColumnValue.java    | 138 ++++++++++++++++++
 .../logminer/parser/impl/InsertSql.java       |   2 +-
 .../logminer/parser/impl/UpdateSql.java       |   2 +-
 5 files changed, 168 insertions(+), 69 deletions(-)
 create mode 100644 dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/OracleColumnValue.java

diff --git a/dbsyncer-common/src/main/java/org/dbsyncer/common/column/ColumnValue.java b/dbsyncer-common/src/main/java/org/dbsyncer/common/column/ColumnValue.java
index f94b2b13..efafe7ac 100644
--- a/dbsyncer-common/src/main/java/org/dbsyncer/common/column/ColumnValue.java
+++ b/dbsyncer-common/src/main/java/org/dbsyncer/common/column/ColumnValue.java
@@ -4,6 +4,7 @@
 package org.dbsyncer.common.column;
 
 import java.math.BigDecimal;
+import java.math.BigInteger;
 import java.sql.Date;
 import java.sql.Time;
 import java.sql.Timestamp;
@@ -40,4 +41,8 @@ public interface ColumnValue {
     Timestamp asTimestamp();
 
     Time asTime();
+
+    default BigInteger asBigInteger(){
+        return null;
+    }
 }
\ No newline at end of file
diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/AbstractParser.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/AbstractParser.java
index 3ccb027b..efecff9a 100644
--- a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/AbstractParser.java
+++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/AbstractParser.java
@@ -3,27 +3,20 @@
  */
 package org.dbsyncer.connector.oracle.logminer.parser;
 
+import java.sql.Types;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
 import net.sf.jsqlparser.expression.BinaryExpression;
 import net.sf.jsqlparser.expression.Expression;
-import net.sf.jsqlparser.expression.Function;
-import net.sf.jsqlparser.expression.NullValue;
-import net.sf.jsqlparser.expression.StringValue;
-import net.sf.jsqlparser.expression.operators.relational.ExpressionList;
 import net.sf.jsqlparser.expression.operators.relational.IsNullExpression;
 import net.sf.jsqlparser.schema.Column;
-import org.dbsyncer.common.util.CollectionUtils;
-import org.dbsyncer.common.util.DateFormatUtil;
 import org.dbsyncer.common.util.StringUtil;
 import org.dbsyncer.sdk.model.Field;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-
 /**
  * @Author AE86
  * @Version 1.0.0
@@ -32,7 +25,7 @@ import java.util.Objects;
 public abstract class AbstractParser implements Parser {
 
     private final Logger logger = LoggerFactory.getLogger(getClass());
-    protected Map columnMap = new HashMap<>();
+    protected Map columnMap = new HashMap<>();
     protected List fields;
 
     public void findColumn(Expression expression) {
@@ -46,73 +39,36 @@ public abstract class AbstractParser implements Parser {
         BinaryExpression binaryExpression = (BinaryExpression) expression;
         if (binaryExpression.getLeftExpression() instanceof Column) {
             Column column = (Column) binaryExpression.getLeftExpression();
-            columnMap.put(StringUtil.replace(column.getColumnName(), StringUtil.DOUBLE_QUOTATION, StringUtil.EMPTY), parserValue(binaryExpression.getRightExpression()));
+            columnMap.put(StringUtil.replace(column.getColumnName(), StringUtil.DOUBLE_QUOTATION, StringUtil.EMPTY), binaryExpression.getRightExpression());
             return;
         }
         findColumn(binaryExpression.getLeftExpression());
         findColumn(binaryExpression.getRightExpression());
     }
 
-    public Object parserValue(Expression expression) {
-        if (expression instanceof NullValue) {
-            return null;
-        }
-        // 解析sql的function，只取到关键的字符串
-        if (expression instanceof Function) {
-            return parseFunction((Function) expression);
-        }
-        if (expression instanceof StringValue) {
-            StringValue val = (StringValue) expression;
-            return val.getValue();
-        }
-        return null;
-    }
-
     public List columnMapToData() {
         List data = new LinkedList<>();
+        //需要进行数据库类型
         for (Field field : fields) {
-            data.add(columnMap.get(field.getName()));
-        }
-        return data;
-    }
-
-    private Object parseFunction(Function function) {
-        List multipartName = function.getMultipartName();
-        ExpressionList parameters = function.getParameters();
-        if (CollectionUtils.isEmpty(multipartName) || CollectionUtils.isEmpty(parameters)) {
-            return null;
-        }
-
-        String nameType = Objects.toString(multipartName.get(0));
-        Object value = parameters.get(0);
-        if (nameType == null || value == null) {
-            return null;
-        }
-
-        if (value instanceof StringValue) {
-            StringValue val = (StringValue) value;
-            value = val.getValue();
-        }
-        try {
-            switch (nameType) {
-                case "TO_DATE":
-                    return toDate(value);
-                case "TO_TIMESTAMP":
-                    return toTimestamp(value);
+           OracleColumnValue oracleColumnValue = new OracleColumnValue(columnMap.get(field.getName()));
+            switch (field.getType()){
+                case Types.BIGINT:
+                    data.add(oracleColumnValue.asBigInteger());
+                    break;
+                case Types.DECIMAL:
+                    data.add(oracleColumnValue.asBigDecimal());
+                    break;
+                case Types.TIME:
+                case Types.TIMESTAMP:
+                    data.add(oracleColumnValue.asTime());
+                    break;
+                default:
+                    data.add(oracleColumnValue.asString());
             }
-        } catch (Exception e) {
-            logger.error(e.getMessage(), e);
         }
-        return value;
-    }
-
-    private Object toDate(Object value) {
-        return DateFormatUtil.stringToTimestamp(Objects.toString(value));
+        return data;
     }
 
-    private Object toTimestamp(Object value) {
-        return DateFormatUtil.stringToTimestamp(StringUtil.replace(Objects.toString(value), StringUtil.POINT, StringUtil.EMPTY));
-    }
 
     public void setFields(List fields) {
         this.fields = fields;
diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/OracleColumnValue.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/OracleColumnValue.java
new file mode 100644
index 00000000..0b4b065c
--- /dev/null
+++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/OracleColumnValue.java
@@ -0,0 +1,138 @@
+package org.dbsyncer.connector.oracle.logminer.parser;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.sql.Date;
+import java.sql.Time;
+import java.sql.Timestamp;
+import java.util.List;
+import java.util.Objects;
+import net.sf.jsqlparser.expression.Expression;
+import net.sf.jsqlparser.expression.Function;
+import net.sf.jsqlparser.expression.NullValue;
+import net.sf.jsqlparser.expression.StringValue;
+import net.sf.jsqlparser.expression.operators.relational.ExpressionList;
+import org.dbsyncer.common.column.AbstractColumnValue;
+import org.dbsyncer.common.util.CollectionUtils;
+import org.dbsyncer.common.util.DateFormatUtil;
+import org.dbsyncer.common.util.StringUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class OracleColumnValue extends AbstractColumnValue {
+
+    private final Logger logger = LoggerFactory.getLogger(getClass());
+
+
+    public OracleColumnValue(Expression value) {
+        setValue(value);
+    }
+
+
+    @Override
+    public String asString() {
+        if (getValue() instanceof NullValue){
+            return null;
+        }
+        if (getValue() instanceof StringValue){
+            return StringUtil.replace(((StringValue) getValue()).getValue(),StringUtil.DOUBLE_QUOTATION,StringUtil.EMPTY);
+        }
+        return getValue().toString();
+    }
+
+    @Override
+    public byte[] asByteArray() {
+        return new byte[0];
+    }
+
+    @Override
+    public Short asShort() {
+        return Short.valueOf(asString());
+    }
+
+    @Override
+    public Integer asInteger() {
+       return Integer.valueOf(asString());
+    }
+
+    @Override
+    public Long asLong() {
+        return Long.valueOf(asString());
+    }
+
+    @Override
+    public Float asFloat() {
+        return Float.valueOf(asString());
+    }
+
+    @Override
+    public Double asDouble() {
+        return Double.valueOf(asString());
+    }
+
+    @Override
+    public Boolean asBoolean() {
+        return null;
+    }
+
+    @Override
+    public BigDecimal asBigDecimal() {
+        return new BigDecimal(asString());
+    }
+
+    @Override
+    public Date asDate() {
+        return null;
+    }
+
+    @Override
+    public Timestamp asTimestamp() {
+        Function function = (Function) getValue();
+        List multipartName = function.getMultipartName();
+        ExpressionList parameters = function.getParameters();
+        if (CollectionUtils.isEmpty(multipartName) || CollectionUtils.isEmpty(parameters)) {
+            return null;
+        }
+
+        String nameType = Objects.toString(multipartName.get(0));
+        Object value = parameters.get(0);
+        if (nameType == null || value == null) {
+            return null;
+        }
+
+        if (value instanceof StringValue) {
+            StringValue val = (StringValue) value;
+            value = val.getValue();
+        }
+
+        try {
+            switch (nameType) {
+                case "TO_DATE":
+                    return toDate(value);
+                case "TO_TIMESTAMP":
+                    return toTimestamp(value);
+            }
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+        }
+        return null;
+    }
+
+    @Override
+    public Time asTime() {
+        return null;
+    }
+
+    @Override
+    public BigInteger asBigInteger() {
+        return new BigInteger(asString());
+    }
+
+    private Timestamp toDate(Object value) {
+        return DateFormatUtil.stringToTimestamp(Objects.toString(value));
+    }
+
+    private Timestamp toTimestamp(Object value) {
+        return DateFormatUtil.stringToTimestamp(StringUtil.replace(Objects.toString(value), StringUtil.POINT, StringUtil.EMPTY));
+    }
+}
diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/impl/InsertSql.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/impl/InsertSql.java
index bd536c66..b248e6c0 100644
--- a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/impl/InsertSql.java
+++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/impl/InsertSql.java
@@ -32,7 +32,7 @@ public class InsertSql extends AbstractParser {
         ExpressionList values = (ExpressionList) insert.getSelect().getValues().getExpressions();
         for (int i = 0; i < columns.size(); i++) {
             columnMap.put(StringUtil.replace(columns.get(i).getColumnName(), StringUtil.DOUBLE_QUOTATION, StringUtil.EMPTY),
-                    parserValue(values.get(i)));
+                    values.get(i));
         }
         return columnMapToData();
     }
diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/impl/UpdateSql.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/impl/UpdateSql.java
index 79b6ba08..25271ed6 100644
--- a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/impl/UpdateSql.java
+++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/logminer/parser/impl/UpdateSql.java
@@ -35,7 +35,7 @@ public class UpdateSql extends AbstractParser {
     private void passerSet(List updateSets) {
         for (UpdateSet updateSet : updateSets) {
             String columnName = StringUtil.replace(updateSet.getColumn(0).getColumnName(), StringUtil.DOUBLE_QUOTATION, StringUtil.EMPTY);
-            columnMap.put(columnName, parserValue(updateSet.getValue(0)));
+            columnMap.put(columnName, updateSet.getValue(0));
         }
     }
 
-- 
Gitee


From 2057c754027f45ba624e622d4331345e8bc109d8 Mon Sep 17 00:00:00 2001
From: AE86 <836391306@qq.com>
Date: Sun, 24 Dec 2023 11:52:50 +0800
Subject: [PATCH 96/98] fix schema

---
 .../elasticsearch/ElasticsearchConnector.java        |  4 +---
 .../dbsyncer/connector/oracle/OracleConnector.java   |  4 +---
 .../connector/postgresql/PostgreSQLConnector.java    | 12 +++++-------
 3 files changed, 7 insertions(+), 13 deletions(-)

diff --git a/dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/ElasticsearchConnector.java b/dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/ElasticsearchConnector.java
index 26f2d20e..1979beca 100644
--- a/dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/ElasticsearchConnector.java
+++ b/dbsyncer-connector/dbsyncer-connector-elasticsearch/src/main/java/org/dbsyncer/connector/elasticsearch/ElasticsearchConnector.java
@@ -58,7 +58,6 @@ import org.elasticsearch.search.sort.SortOrder;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import javax.annotation.PostConstruct;
 import java.io.IOException;
 import java.sql.Types;
 import java.util.ArrayList;
@@ -86,8 +85,7 @@ public final class ElasticsearchConnector extends AbstractConnector implements C
 
     private final ESConfigValidator configValidator = new ESConfigValidator();
 
-    @PostConstruct
-    private void init() {
+    public ElasticsearchConnector() {
         VALUE_MAPPERS.put(Types.DATE, new ESDateValueMapper());
         VALUE_MAPPERS.put(Types.OTHER, new ESOtherValueMapper());
 
diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/OracleConnector.java b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/OracleConnector.java
index 92a414d5..ca0db035 100644
--- a/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/OracleConnector.java
+++ b/dbsyncer-connector/dbsyncer-connector-oracle/src/main/java/org/dbsyncer/connector/oracle/OracleConnector.java
@@ -23,7 +23,6 @@ import org.dbsyncer.sdk.util.PrimaryKeyUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import javax.annotation.PostConstruct;
 import java.sql.Types;
 import java.util.List;
 
@@ -41,8 +40,7 @@ public final class OracleConnector extends AbstractDatabaseConnector {
     private final String TYPE = "Oracle";
     private final OracleConfigValidator configValidator = new OracleConfigValidator();
 
-    @PostConstruct
-    private void init() {
+    public OracleConnector() {
         VALUE_MAPPERS.put(Types.OTHER, new OracleOtherValueMapper());
     }
 
diff --git a/dbsyncer-connector/dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLConnector.java b/dbsyncer-connector/dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLConnector.java
index de222a49..eeae7c5a 100644
--- a/dbsyncer-connector/dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLConnector.java
+++ b/dbsyncer-connector/dbsyncer-connector-postgresql/src/main/java/org/dbsyncer/connector/postgresql/PostgreSQLConnector.java
@@ -24,7 +24,6 @@ import org.dbsyncer.sdk.util.PrimaryKeyUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import javax.annotation.PostConstruct;
 import java.sql.Types;
 import java.util.List;
 
@@ -42,6 +41,11 @@ public final class PostgreSQLConnector extends AbstractDatabaseConnector {
     private final String TYPE = "PostgreSQL";
     private final PostgreSQLConfigValidator configValidator = new PostgreSQLConfigValidator();
 
+    public PostgreSQLConnector() {
+        VALUE_MAPPERS.put(Types.BIT, new PostgreSQLBitValueMapper());
+        VALUE_MAPPERS.put(Types.OTHER, new PostgreSQLOtherValueMapper());
+    }
+
     @Override
     public String getConnectorType() {
         return TYPE;
@@ -64,12 +68,6 @@ public final class PostgreSQLConnector extends AbstractDatabaseConnector {
         return null;
     }
 
-    @PostConstruct
-    private void init() {
-        VALUE_MAPPERS.put(Types.BIT, new PostgreSQLBitValueMapper());
-        VALUE_MAPPERS.put(Types.OTHER, new PostgreSQLOtherValueMapper());
-    }
-
     @Override
     public String buildSqlWithQuotation() {
         return "\"";
-- 
Gitee


From 236096b32974e4e3fa1e1b7ce955cfe835795f95 Mon Sep 17 00:00:00 2001
From: AE86 <836391306@qq.com>
Date: Tue, 26 Dec 2023 00:59:14 +0800
Subject: [PATCH 97/98] pub 2.0 beta

---
 dbsyncer-biz/pom.xml                                  |  2 +-
 dbsyncer-cluster/pom.xml                              |  2 +-
 dbsyncer-common/pom.xml                               |  2 +-
 dbsyncer-connector/dbsyncer-connector-base/pom.xml    |  2 +-
 .../dbsyncer-connector-elasticsearch/pom.xml          |  2 +-
 dbsyncer-connector/dbsyncer-connector-file/pom.xml    |  2 +-
 dbsyncer-connector/dbsyncer-connector-kafka/pom.xml   |  2 +-
 dbsyncer-connector/dbsyncer-connector-mysql/pom.xml   |  2 +-
 dbsyncer-connector/dbsyncer-connector-oracle/pom.xml  |  2 +-
 .../dbsyncer-connector-postgresql/pom.xml             |  2 +-
 dbsyncer-connector/dbsyncer-connector-sqlite/pom.xml  |  2 +-
 .../dbsyncer-connector-sqlserver/pom.xml              |  2 +-
 dbsyncer-connector/pom.xml                            |  2 +-
 dbsyncer-manager/pom.xml                              |  2 +-
 dbsyncer-monitor/pom.xml                              | 11 -----------
 dbsyncer-parser/pom.xml                               |  2 +-
 dbsyncer-plugin/pom.xml                               |  2 +-
 dbsyncer-sdk/pom.xml                                  |  2 +-
 dbsyncer-storage/pom.xml                              |  2 +-
 dbsyncer-web/pom.xml                                  |  2 +-
 pom.xml                                               |  2 +-
 version.cmd                                           |  2 +-
 version.sh                                            |  2 +-
 23 files changed, 22 insertions(+), 33 deletions(-)
 delete mode 100644 dbsyncer-monitor/pom.xml

diff --git a/dbsyncer-biz/pom.xml b/dbsyncer-biz/pom.xml
index dace0f78..94f5feb1 100644
--- a/dbsyncer-biz/pom.xml
+++ b/dbsyncer-biz/pom.xml
@@ -5,7 +5,7 @@
 	
         dbsyncer
         org.ghi
-        2.0.0
+        2.0.0_1226-Beta
     
 	4.0.0
 	dbsyncer-biz
diff --git a/dbsyncer-cluster/pom.xml b/dbsyncer-cluster/pom.xml
index e08b4203..7a6427d4 100644
--- a/dbsyncer-cluster/pom.xml
+++ b/dbsyncer-cluster/pom.xml
@@ -5,7 +5,7 @@
     
         dbsyncer
         org.ghi
-        2.0.0
+        2.0.0_1226-Beta
     
     4.0.0
     dbsyncer-cluster
diff --git a/dbsyncer-common/pom.xml b/dbsyncer-common/pom.xml
index 448942be..c60addee 100644
--- a/dbsyncer-common/pom.xml
+++ b/dbsyncer-common/pom.xml
@@ -5,7 +5,7 @@
     
         dbsyncer
         org.ghi
-        2.0.0
+        2.0.0_1226-Beta
     
     4.0.0
     dbsyncer-common
diff --git a/dbsyncer-connector/dbsyncer-connector-base/pom.xml b/dbsyncer-connector/dbsyncer-connector-base/pom.xml
index 37f8cfdf..d00dcda2 100644
--- a/dbsyncer-connector/dbsyncer-connector-base/pom.xml
+++ b/dbsyncer-connector/dbsyncer-connector-base/pom.xml
@@ -5,7 +5,7 @@
     
         dbsyncer-connector
         org.ghi
-        2.0.0
+        2.0.0_1226-Beta
     
     4.0.0
 
diff --git a/dbsyncer-connector/dbsyncer-connector-elasticsearch/pom.xml b/dbsyncer-connector/dbsyncer-connector-elasticsearch/pom.xml
index 40574412..de93989a 100644
--- a/dbsyncer-connector/dbsyncer-connector-elasticsearch/pom.xml
+++ b/dbsyncer-connector/dbsyncer-connector-elasticsearch/pom.xml
@@ -5,7 +5,7 @@
     
         dbsyncer-connector
         org.ghi
-        2.0.0
+        2.0.0_1226-Beta
     
     4.0.0
 
diff --git a/dbsyncer-connector/dbsyncer-connector-file/pom.xml b/dbsyncer-connector/dbsyncer-connector-file/pom.xml
index 9c71355e..49f71422 100644
--- a/dbsyncer-connector/dbsyncer-connector-file/pom.xml
+++ b/dbsyncer-connector/dbsyncer-connector-file/pom.xml
@@ -5,7 +5,7 @@
     
         dbsyncer-connector
         org.ghi
-        2.0.0
+        2.0.0_1226-Beta
     
     4.0.0
 
diff --git a/dbsyncer-connector/dbsyncer-connector-kafka/pom.xml b/dbsyncer-connector/dbsyncer-connector-kafka/pom.xml
index ffbdb6bf..49c035a4 100644
--- a/dbsyncer-connector/dbsyncer-connector-kafka/pom.xml
+++ b/dbsyncer-connector/dbsyncer-connector-kafka/pom.xml
@@ -5,7 +5,7 @@
     
         dbsyncer-connector
         org.ghi
-        2.0.0
+        2.0.0_1226-Beta
     
     4.0.0
 
diff --git a/dbsyncer-connector/dbsyncer-connector-mysql/pom.xml b/dbsyncer-connector/dbsyncer-connector-mysql/pom.xml
index db89afce..d02e2d4d 100644
--- a/dbsyncer-connector/dbsyncer-connector-mysql/pom.xml
+++ b/dbsyncer-connector/dbsyncer-connector-mysql/pom.xml
@@ -5,7 +5,7 @@
     
         dbsyncer-connector
         org.ghi
-        2.0.0
+        2.0.0_1226-Beta
     
     4.0.0
 
diff --git a/dbsyncer-connector/dbsyncer-connector-oracle/pom.xml b/dbsyncer-connector/dbsyncer-connector-oracle/pom.xml
index ec1e6f83..fa081db4 100644
--- a/dbsyncer-connector/dbsyncer-connector-oracle/pom.xml
+++ b/dbsyncer-connector/dbsyncer-connector-oracle/pom.xml
@@ -5,7 +5,7 @@
     
         dbsyncer-connector
         org.ghi
-        2.0.0
+        2.0.0_1226-Beta
     
     4.0.0
 
diff --git a/dbsyncer-connector/dbsyncer-connector-postgresql/pom.xml b/dbsyncer-connector/dbsyncer-connector-postgresql/pom.xml
index deeba571..6883aea1 100644
--- a/dbsyncer-connector/dbsyncer-connector-postgresql/pom.xml
+++ b/dbsyncer-connector/dbsyncer-connector-postgresql/pom.xml
@@ -5,7 +5,7 @@
     
         dbsyncer-connector
         org.ghi
-        2.0.0
+        2.0.0_1226-Beta
     
     4.0.0
 
diff --git a/dbsyncer-connector/dbsyncer-connector-sqlite/pom.xml b/dbsyncer-connector/dbsyncer-connector-sqlite/pom.xml
index a40ced1c..5274faff 100644
--- a/dbsyncer-connector/dbsyncer-connector-sqlite/pom.xml
+++ b/dbsyncer-connector/dbsyncer-connector-sqlite/pom.xml
@@ -5,7 +5,7 @@
     
         dbsyncer-connector
         org.ghi
-        2.0.0
+        2.0.0_1226-Beta
     
     4.0.0
 
diff --git a/dbsyncer-connector/dbsyncer-connector-sqlserver/pom.xml b/dbsyncer-connector/dbsyncer-connector-sqlserver/pom.xml
index b3af0431..45b0e959 100644
--- a/dbsyncer-connector/dbsyncer-connector-sqlserver/pom.xml
+++ b/dbsyncer-connector/dbsyncer-connector-sqlserver/pom.xml
@@ -5,7 +5,7 @@
     
         dbsyncer-connector
         org.ghi
-        2.0.0
+        2.0.0_1226-Beta
     
     4.0.0
 
diff --git a/dbsyncer-connector/pom.xml b/dbsyncer-connector/pom.xml
index 075571f0..32be40a8 100644
--- a/dbsyncer-connector/pom.xml
+++ b/dbsyncer-connector/pom.xml
@@ -5,7 +5,7 @@
     
         dbsyncer
         org.ghi
-        2.0.0
+        2.0.0_1226-Beta
     
     4.0.0
     dbsyncer-connector
diff --git a/dbsyncer-manager/pom.xml b/dbsyncer-manager/pom.xml
index 108d2a65..b8a8383a 100644
--- a/dbsyncer-manager/pom.xml
+++ b/dbsyncer-manager/pom.xml
@@ -5,7 +5,7 @@
     
         dbsyncer
         org.ghi
-        2.0.0
+        2.0.0_1226-Beta
     
     4.0.0
     dbsyncer-manager
diff --git a/dbsyncer-monitor/pom.xml b/dbsyncer-monitor/pom.xml
deleted file mode 100644
index 1fee95e7..00000000
--- a/dbsyncer-monitor/pom.xml
+++ /dev/null
@@ -1,11 +0,0 @@
-
-
-    
-        dbsyncer
-        org.ghi
-        1.2.7_1201
-    
-    4.0.0
-    dbsyncer-monitor
diff --git a/dbsyncer-parser/pom.xml b/dbsyncer-parser/pom.xml
index 96118c03..5ea051f7 100644
--- a/dbsyncer-parser/pom.xml
+++ b/dbsyncer-parser/pom.xml
@@ -5,7 +5,7 @@
     
         dbsyncer
         org.ghi
-        2.0.0
+        2.0.0_1226-Beta
     
     4.0.0
     dbsyncer-parser
diff --git a/dbsyncer-plugin/pom.xml b/dbsyncer-plugin/pom.xml
index 547d80d8..2e0873a6 100644
--- a/dbsyncer-plugin/pom.xml
+++ b/dbsyncer-plugin/pom.xml
@@ -5,7 +5,7 @@
     
         dbsyncer
         org.ghi
-        2.0.0
+        2.0.0_1226-Beta
     
     4.0.0
     dbsyncer-plugin
diff --git a/dbsyncer-sdk/pom.xml b/dbsyncer-sdk/pom.xml
index 78020b93..c345dbfc 100644
--- a/dbsyncer-sdk/pom.xml
+++ b/dbsyncer-sdk/pom.xml
@@ -5,7 +5,7 @@
     
         dbsyncer
         org.ghi
-        2.0.0
+        2.0.0_1226-Beta
     
     4.0.0
     dbsyncer-sdk
diff --git a/dbsyncer-storage/pom.xml b/dbsyncer-storage/pom.xml
index 124091e6..0ef9fdb0 100644
--- a/dbsyncer-storage/pom.xml
+++ b/dbsyncer-storage/pom.xml
@@ -5,7 +5,7 @@
     
         dbsyncer
         org.ghi
-        2.0.0
+        2.0.0_1226-Beta
     
     4.0.0
     dbsyncer-storage
diff --git a/dbsyncer-web/pom.xml b/dbsyncer-web/pom.xml
index d09a42ea..a432bd27 100644
--- a/dbsyncer-web/pom.xml
+++ b/dbsyncer-web/pom.xml
@@ -5,7 +5,7 @@
     
         dbsyncer
         org.ghi
-        2.0.0
+        2.0.0_1226-Beta
     
     4.0.0
     dbsyncer-web
diff --git a/pom.xml b/pom.xml
index d4c640e5..a1c622c8 100644
--- a/pom.xml
+++ b/pom.xml
@@ -6,7 +6,7 @@
 
     org.ghi
     dbsyncer
-    2.0.0
+    2.0.0_1226-Beta
     pom
     dbsyncer
     https://gitee.com/ghi/dbsyncer
diff --git a/version.cmd b/version.cmd
index 790d3ab5..8ea1ae8d 100644
--- a/version.cmd
+++ b/version.cmd
@@ -1,7 +1,7 @@
 @echo off
 
 set CURRENT_DATE=%date:~5,2%%date:~8,2%
-set VERSION=2.0.0_%CURRENT_DATE%
+set VERSION=2.0.0_%CURRENT_DATE%-Beta
 set /p APP_VERSION=Please enter a new version number(%VERSION%): || set APP_VERSION=%VERSION%
 echo %APP_VERSION%
 
diff --git a/version.sh b/version.sh
index 4415f0e7..f0154534 100644
--- a/version.sh
+++ b/version.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-VERSION=2.0.0_$(date +"%m%d")
+VERSION=2.0.0_$(date +"%m%d")-Beta
 read -p "Please enter a new version number($VERSION)：" APP_VERSION
 if [ -z "$APP_VERSION" ]; then
   APP_VERSION=$VERSION
-- 
Gitee


From aac5c7a67d91d495c7b749030f9d5e10e61e2e3e Mon Sep 17 00:00:00 2001
From: AE86 <836391306@qq.com>
Date: Tue, 26 Dec 2023 01:05:54 +0800
Subject: [PATCH 98/98] fix pmd

---
 .../java/org/dbsyncer/connector/mysql/cdc/MySQLListener.java   | 3 +--
 .../src/main/java/org/dbsyncer/sdk/util/DatabaseUtil.java      | 3 +--
 dbsyncer-storage/src/{main => }/test/LuceneFactoryTest.java    | 3 +--
 3 files changed, 3 insertions(+), 6 deletions(-)
 rename dbsyncer-storage/src/{main => }/test/LuceneFactoryTest.java (99%)

diff --git a/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/cdc/MySQLListener.java b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/cdc/MySQLListener.java
index deb90a58..0289444d 100644
--- a/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/cdc/MySQLListener.java
+++ b/dbsyncer-connector/dbsyncer-connector-mysql/src/main/java/org/dbsyncer/connector/mysql/cdc/MySQLListener.java
@@ -134,9 +134,8 @@ public class MySQLListener extends AbstractDatabaseListener {
 
     private List readNodes(String url) {
         Matcher matcher = compile("(//)(?!(/)).+?(/)").matcher(url);
-        while (matcher.find()) {
+        if (matcher.find()) {
             url = matcher.group(0);
-            break;
         }
         url = StringUtil.replace(url, "/", "");
 
diff --git a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/util/DatabaseUtil.java b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/util/DatabaseUtil.java
index 3a23679a..33f2607e 100644
--- a/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/util/DatabaseUtil.java
+++ b/dbsyncer-sdk/src/main/java/org/dbsyncer/sdk/util/DatabaseUtil.java
@@ -38,9 +38,8 @@ public abstract class DatabaseUtil {
 
     public static String getDatabaseName(String url) {
         Matcher matcher = compile("(//)(?!(\\?)).+?(\\?)").matcher(url);
-        while (matcher.find()) {
+        if (matcher.find()) {
             url = matcher.group(0);
-            break;
         }
         int s = url.lastIndexOf("/");
         int e = url.lastIndexOf("?");
diff --git a/dbsyncer-storage/src/main/test/LuceneFactoryTest.java b/dbsyncer-storage/src/test/LuceneFactoryTest.java
similarity index 99%
rename from dbsyncer-storage/src/main/test/LuceneFactoryTest.java
rename to dbsyncer-storage/src/test/LuceneFactoryTest.java
index a1332d4d..af9e7814 100644
--- a/dbsyncer-storage/src/main/test/LuceneFactoryTest.java
+++ b/dbsyncer-storage/src/test/LuceneFactoryTest.java
@@ -261,8 +261,7 @@ public class LuceneFactoryTest {
         System.out.println("新增后：");
         maps = query(new MatchAllDocsQuery());
         maps.forEach(m -> {
-            m.get("content");
-            System.out.println(m);
+            logger.info(String.valueOf(m.get("content")));
         });
         check();
 
-- 
Gitee