From 92afe23a4209de3b1185a87c7e29e5a208ebb2d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=9E=BF=E4=BD=90=E9=B9=8F?= Date: Wed, 28 Aug 2024 11:33:37 +0800 Subject: [PATCH 1/3] =?UTF-8?q?=E6=8F=90=E4=BA=A4sharding=E9=9B=86?= =?UTF-8?q?=E6=88=90=E4=BD=BF=E7=94=A8=E5=AE=9E=E4=BE=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pom.xml | 4 +- .../yudao-module-system-biz/pom.xml | 7 ++ .../dal/mysql/logger/OperateLogMapper.java | 2 + .../DataPreciseShardingAlgorithm.java | 90 +++++++++++++++++++ .../sharding/MyDataSourceConfiguration.java | 73 +++++++++++++++ ...rdingsphere.sharding.spi.ShardingAlgorithm | 1 + .../src/main/resources/application-local.yaml | 38 +++++++- 7 files changed, 212 insertions(+), 3 deletions(-) create mode 100644 yudao-module-system/yudao-module-system-biz/src/main/java/cn/iocoder/yudao/module/system/framework/sharding/DataPreciseShardingAlgorithm.java create mode 100644 yudao-module-system/yudao-module-system-biz/src/main/java/cn/iocoder/yudao/module/system/framework/sharding/MyDataSourceConfiguration.java create mode 100644 yudao-module-system/yudao-module-system-biz/src/main/resources/META-INF/services/org.apache.shardingsphere.sharding.spi.ShardingAlgorithm diff --git a/pom.xml b/pom.xml index 2b8e647d0..b668423e6 100644 --- a/pom.xml +++ b/pom.xml @@ -17,12 +17,12 @@ yudao-module-member yudao-module-bpm yudao-module-pay - yudao-module-report + yudao-module-mp yudao-module-mall yudao-module-crm yudao-module-erp - yudao-module-ai + ${project.artifactId} diff --git a/yudao-module-system/yudao-module-system-biz/pom.xml b/yudao-module-system/yudao-module-system-biz/pom.xml index 5ca619768..600c140f1 100644 --- a/yudao-module-system/yudao-module-system-biz/pom.xml +++ b/yudao-module-system/yudao-module-system-biz/pom.xml @@ -152,6 +152,13 @@ hutool-extra + + + org.apache.shardingsphere + shardingsphere-jdbc-core-spring-boot-starter + 5.2.1 + + diff --git a/yudao-module-system/yudao-module-system-biz/src/main/java/cn/iocoder/yudao/module/system/dal/mysql/logger/OperateLogMapper.java b/yudao-module-system/yudao-module-system-biz/src/main/java/cn/iocoder/yudao/module/system/dal/mysql/logger/OperateLogMapper.java index 14e308b56..56d149bc4 100644 --- a/yudao-module-system/yudao-module-system-biz/src/main/java/cn/iocoder/yudao/module/system/dal/mysql/logger/OperateLogMapper.java +++ b/yudao-module-system/yudao-module-system-biz/src/main/java/cn/iocoder/yudao/module/system/dal/mysql/logger/OperateLogMapper.java @@ -6,9 +6,11 @@ import cn.iocoder.yudao.framework.mybatis.core.query.LambdaQueryWrapperX; import cn.iocoder.yudao.module.system.api.logger.dto.OperateLogPageReqDTO; import cn.iocoder.yudao.module.system.controller.admin.logger.vo.operatelog.OperateLogPageReqVO; import cn.iocoder.yudao.module.system.dal.dataobject.logger.OperateLogDO; +import com.baomidou.dynamic.datasource.annotation.DS; import org.apache.ibatis.annotations.Mapper; @Mapper +@DS("sharding") public interface OperateLogMapper extends BaseMapperX { default PageResult selectPage(OperateLogPageReqVO pageReqDTO) { diff --git a/yudao-module-system/yudao-module-system-biz/src/main/java/cn/iocoder/yudao/module/system/framework/sharding/DataPreciseShardingAlgorithm.java b/yudao-module-system/yudao-module-system-biz/src/main/java/cn/iocoder/yudao/module/system/framework/sharding/DataPreciseShardingAlgorithm.java new file mode 100644 index 000000000..339523701 --- /dev/null +++ b/yudao-module-system/yudao-module-system-biz/src/main/java/cn/iocoder/yudao/module/system/framework/sharding/DataPreciseShardingAlgorithm.java @@ -0,0 +1,90 @@ +package cn.iocoder.yudao.module.system.framework.sharding; + +import cn.hutool.core.date.DateUtil; +import cn.iocoder.yudao.framework.common.exception.ServiceException; +import com.google.common.collect.Range; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.apache.shardingsphere.sharding.api.sharding.standard.PreciseShardingValue; +import org.apache.shardingsphere.sharding.api.sharding.standard.RangeShardingValue; +import org.apache.shardingsphere.sharding.api.sharding.standard.StandardShardingAlgorithm; + +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Properties; + +/** + * @Description + * 目前分页查询是有点问题,不过我项目里没有用到没有研究那块,貌似是selectPage会执行mp的查询count,会报错 + * + * @Date 2024/6/13 10:07 + * @Author tom + **/ +@Getter +@Slf4j +public class DataPreciseShardingAlgorithm implements StandardShardingAlgorithm { + + @Override + public Collection doSharding(Collection collection, RangeShardingValue rangeShardingValue) { + Range valueRange = rangeShardingValue.getValueRange(); + LocalDateTime lowerEndpoint = valueRange.lowerEndpoint(); + LocalDateTime upperEndpoint = valueRange.upperEndpoint(); + + // 逻辑表名 + String logicTableName = rangeShardingValue.getLogicTableName(); + List tables = new ArrayList<>(); + + LocalDateTime now = LocalDateTime.now(); + if (upperEndpoint.getYear() > now.getYear() || upperEndpoint.getMonthValue() > now.getMonthValue()) { + log.warn("时间区间异常 不能查询未来数据 {} {}",lowerEndpoint,upperEndpoint); + throw new ServiceException(40001,"不能查询未来数据"); + } + while (lowerEndpoint.compareTo(upperEndpoint) <= 0) { + // 添加到集合 + tables.add(buildTable(logicTableName, lowerEndpoint)); + // 往后加一个月 + lowerEndpoint = lowerEndpoint.plusMonths(1).withDayOfMonth(1); + } + return tables; + } + + private String buildTable(String logicTableName, LocalDateTime startTime) { + String timeStr = DateUtil.formatLocalDateTime(startTime); + String year = timeStr.substring(2, 4); + String month = timeStr.substring(5, 7); + + return logicTableName + "_" + year + month; + } + + @Override + public String doSharding(Collection collection, PreciseShardingValue preciseShardingValue) { + String timeStr = DateUtil.formatLocalDateTime(preciseShardingValue.getValue()); + String year = timeStr.substring(2, 4); + String month = timeStr.substring(5, 7); + String tableSuffix = year + month; + for (String s : collection) { + if (s.contains(tableSuffix)){ + return s; + } + } + return null; + } + + + @Override + public Properties getProps() { + return null; + } + + @Override + public void init(Properties properties) { + + } + + @Override + public String getType() { + return "HIS_DATA_SPI_BASED"; + } +} diff --git a/yudao-module-system/yudao-module-system-biz/src/main/java/cn/iocoder/yudao/module/system/framework/sharding/MyDataSourceConfiguration.java b/yudao-module-system/yudao-module-system-biz/src/main/java/cn/iocoder/yudao/module/system/framework/sharding/MyDataSourceConfiguration.java new file mode 100644 index 000000000..b7b29f2d3 --- /dev/null +++ b/yudao-module-system/yudao-module-system-biz/src/main/java/cn/iocoder/yudao/module/system/framework/sharding/MyDataSourceConfiguration.java @@ -0,0 +1,73 @@ +package cn.iocoder.yudao.module.system.framework.sharding; + +import com.baomidou.dynamic.datasource.DynamicRoutingDataSource; +import com.baomidou.dynamic.datasource.creator.DefaultDataSourceCreator; +import com.baomidou.dynamic.datasource.provider.AbstractDataSourceProvider; +import com.baomidou.dynamic.datasource.provider.DynamicDataSourceProvider; +import com.baomidou.dynamic.datasource.spring.boot.autoconfigure.DynamicDataSourceProperties; +import jakarta.annotation.Resource; +import org.apache.shardingsphere.driver.jdbc.core.datasource.ShardingSphereDataSource; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; + +import javax.sql.DataSource; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Configuration +public class MyDataSourceConfiguration { + + @Resource + private DynamicDataSourceProperties properties; + + /** + * 分片: shardingDataSource + * 主从: masterSlaveDataSource + * 根据自己场景修改注入 + */ +// @Lazy +// @Resource +// private MasterSlaveDataSource masterSlaveDataSource; + +// @Lazy + @Resource + private ShardingSphereDataSource shardingSphereDataSource; + + @Bean + public DynamicDataSourceProvider dynamicDataSourceProvider(DefaultDataSourceCreator dataSourceCreator) { + return new AbstractDataSourceProvider(dataSourceCreator) { + + @Override + public Map loadDataSources() { + Map dataSourceMap = new HashMap<>(); + dataSourceMap.put("sharding", shardingSphereDataSource); +//下面的代码可以把 shardingJdbc 内部管理的子数据源也同时添加到动态数据源里 (根据自己需要选择开启+注解了@Lazy被代理了不可以) +// Map shardingInnerDataSources = shardingSphereDataSource(); +// dataSourceMap.putAll(shardingInnerDataSources); + return dataSourceMap; + } + }; + } + + /** + * 将动态数据源设置为首选的 + * 当spring存在多个数据源时, 自动注入的是首选的对象 + * 设置为主要的数据源之后，就可以支持shardingjdbc原生的配置方式了 + * 3.4.0版本及以上使用以下方式注入,老版本请阅读文档 进阶-手动注入多数据源 + */ + @Primary + @Bean + public DataSource dataSource(List providers) { + DynamicRoutingDataSource dataSource = new DynamicRoutingDataSource(providers); + dataSource.setPrimary(properties.getPrimary()); + dataSource.setStrict(properties.getStrict()); + dataSource.setStrategy(properties.getStrategy()); + dataSource.setSeata(properties.getSeata()); + dataSource.setP6spy(properties.getP6spy()); + dataSource.setSeata(properties.getSeata()); + return dataSource; + } + +} \ No newline at end of file diff --git a/yudao-module-system/yudao-module-system-biz/src/main/resources/META-INF/services/org.apache.shardingsphere.sharding.spi.ShardingAlgorithm b/yudao-module-system/yudao-module-system-biz/src/main/resources/META-INF/services/org.apache.shardingsphere.sharding.spi.ShardingAlgorithm new file mode 100644 index 000000000..5230bf72f --- /dev/null +++ b/yudao-module-system/yudao-module-system-biz/src/main/resources/META-INF/services/org.apache.shardingsphere.sharding.spi.ShardingAlgorithm @@ -0,0 +1 @@ +cn.iocoder.yudao.module.system.framework.sharding.DataPreciseShardingAlgorithm \ No newline at end of file diff --git a/yudao-module-system/yudao-module-system-biz/src/main/resources/application-local.yaml b/yudao-module-system/yudao-module-system-biz/src/main/resources/application-local.yaml index ca8b88d7a..09de691b4 100644 --- a/yudao-module-system/yudao-module-system-biz/src/main/resources/application-local.yaml +++ b/yudao-module-system/yudao-module-system-biz/src/main/resources/application-local.yaml @@ -73,7 +73,43 @@ spring: url: jdbc:mysql://127.0.0.1:3306/ruoyi-vue-pro?useSSL=false&serverTimezone=Asia/Shanghai&allowPublicKeyRetrieval=true&nullCatalogMeansCurrent=true username: root password: 123456 - + ## sharding 配置 + shardingsphere: + props: + sql: + show: true + sql-show: true + datasource: + names: db-0 + db-0: + type: com.alibaba.druid.pool.DruidDataSource + driver-class-name: com.mysql.cj.jdbc.Driver + username: root + password: 123456 + url: jdbc:mysql://127.0.0.1:3306/test?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&useSSL=false&zeroDateTimeBehavior=convertToNull&serverTimezone=GMT%2b8&rewriteBatchedStatements=true + rules: + sharding: + tables: + system_operate_log: + key-generator: + column: id + type: SNOWFLAKE + #分片策略 + actual-data-nodes: db-0.system_operate_log_${24..50}0${1..9},db-0.system_operate_log_${24..50}1${0..2} + table-strategy: #表分片策略 + standard: + sharding-column: create_time + sharding-algorithm-name: his_month_sharding + # 配置分片算法 + sharding-algorithms: + # person_sync_record_inline: + # type: INLINE + # props: + # algorithm-expression: person_sync_record_$->{id % 16} + # 分片算法名称 + his_month_sharding: + # 分片算法类型，这个type是我们的分片算法实现类中 getType()的返回值，SPI适用于这种方式 + type: HIS_DATA_SPI_BASED # Redis 配置。Redisson 默认的配置足够使用，一般不需要进行调优 data: redis: -- Gitee From c5e2713020b10fd20228963fe984c0740b677dd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=9E=BF=E4=BD=90=E9=B9=8F?= Date: Wed, 28 Aug 2024 11:37:44 +0800 Subject: [PATCH 2/3] =?UTF-8?q?=E6=8F=90=E4=BA=A4snakeyaml=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0init=E6=96=B9=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../snakeyaml/representer/Representer.java | 206 ++++++++++++++++++ 1 file changed, 206 insertions(+) create mode 100644 yudao-module-system/yudao-module-system-biz/src/main/java/org/yaml/snakeyaml/representer/Representer.java diff --git a/yudao-module-system/yudao-module-system-biz/src/main/java/org/yaml/snakeyaml/representer/Representer.java b/yudao-module-system/yudao-module-system-biz/src/main/java/org/yaml/snakeyaml/representer/Representer.java new file mode 100644 index 000000000..104c6096d --- /dev/null +++ b/yudao-module-system/yudao-module-system-biz/src/main/java/org/yaml/snakeyaml/representer/Representer.java @@ -0,0 +1,206 @@ +// +// Source code recreated from a .class file by IntelliJ IDEA +// (powered by FernFlower decompiler) +// + +package org.yaml.snakeyaml.representer; + +import org.yaml.snakeyaml.DumperOptions; +import org.yaml.snakeyaml.DumperOptions.FlowStyle; +import org.yaml.snakeyaml.TypeDescription; +import org.yaml.snakeyaml.introspector.Property; +import org.yaml.snakeyaml.introspector.PropertyUtils; +import org.yaml.snakeyaml.nodes.*; + +import java.util.*; + +public class Representer extends SafeRepresenter { + protected Map, TypeDescription> typeDefinitions = Collections.emptyMap(); + + + public Representer() { + super(new DumperOptions()); + this.representers.put(null, new RepresentJavaBean()); + } + + public Representer(DumperOptions options) { + super(options); + this.representers.put(null, new RepresentJavaBean()); + } + + public TypeDescription addTypeDescription(TypeDescription td) { + if (Collections.EMPTY_MAP == this.typeDefinitions) { + this.typeDefinitions = new HashMap(); + } + + if (td.getTag() != null) { + this.addClassTag(td.getType(), td.getTag()); + } + + td.setPropertyUtils(this.getPropertyUtils()); + return (TypeDescription)this.typeDefinitions.put(td.getType(), td); + } + + public void setPropertyUtils(PropertyUtils propertyUtils) { + super.setPropertyUtils(propertyUtils); + Collection tds = this.typeDefinitions.values(); + Iterator var3 = tds.iterator(); + + while(var3.hasNext()) { + TypeDescription typeDescription = (TypeDescription)var3.next(); + typeDescription.setPropertyUtils(propertyUtils); + } + + } + + protected MappingNode representJavaBean(Set properties, Object javaBean) { + List value = new ArrayList(properties.size()); + Tag customTag = (Tag)this.classTags.get(javaBean.getClass()); + Tag tag = customTag != null ? customTag : new Tag(javaBean.getClass()); + MappingNode node = new MappingNode(tag, value, FlowStyle.AUTO); + this.representedObjects.put(javaBean, node); + FlowStyle bestStyle = FlowStyle.FLOW; + Iterator var8 = properties.iterator(); + + while(true) { + NodeTuple tuple; + do { + if (!var8.hasNext()) { + if (this.defaultFlowStyle != FlowStyle.AUTO) { + node.setFlowStyle(this.defaultFlowStyle); + } else { + node.setFlowStyle(bestStyle); + } + + return node; + } + + Property property = (Property)var8.next(); + Object memberValue = property.get(javaBean); + Tag customPropertyTag = memberValue == null ? null : (Tag)this.classTags.get(memberValue.getClass()); + tuple = this.representJavaBeanProperty(javaBean, property, memberValue, customPropertyTag); + } while(tuple == null); + + if (!((ScalarNode)tuple.getKeyNode()).isPlain()) { + bestStyle = FlowStyle.BLOCK; + } + + Node nodeValue = tuple.getValueNode(); + if (!(nodeValue instanceof ScalarNode) || !((ScalarNode)nodeValue).isPlain()) { + bestStyle = FlowStyle.BLOCK; + } + + value.add(tuple); + } + } + + protected NodeTuple representJavaBeanProperty(Object javaBean, Property property, Object propertyValue, Tag customTag) { + ScalarNode nodeKey = (ScalarNode)this.representData(property.getName()); + boolean hasAlias = this.representedObjects.containsKey(propertyValue); + Node nodeValue = this.representData(propertyValue); + if (propertyValue != null && !hasAlias) { + NodeId nodeId = nodeValue.getNodeId(); + if (customTag == null) { + if (nodeId == NodeId.scalar) { + if (property.getType() != Enum.class && propertyValue instanceof Enum) { + nodeValue.setTag(Tag.STR); + } + } else { + if (nodeId == NodeId.mapping && property.getType() == propertyValue.getClass() && !(propertyValue instanceof Map) && !nodeValue.getTag().equals(Tag.SET)) { + nodeValue.setTag(Tag.MAP); + } + + this.checkGlobalTag(property, nodeValue, propertyValue); + } + } + } + + return new NodeTuple(nodeKey, nodeValue); + } + + protected void checkGlobalTag(Property property, Node node, Object object) { + if (!object.getClass().isArray() || !object.getClass().getComponentType().isPrimitive()) { + Class[] arguments = property.getActualTypeArguments(); + if (arguments != null) { + Class t; + Iterator iter; + Iterator var9; + if (node.getNodeId() == NodeId.sequence) { + t = arguments[0]; + SequenceNode snode = (SequenceNode)node; + Iterable memberList = Collections.emptyList(); + if (object.getClass().isArray()) { + memberList = Arrays.asList((Object[])object); + } else if (object instanceof Iterable) { + memberList = (Iterable)object; + } + + iter = ((Iterable)memberList).iterator(); + if (iter.hasNext()) { + var9 = snode.getValue().iterator(); + + while(var9.hasNext()) { + Node childNode = (Node)var9.next(); + Object member = iter.next(); + if (member != null && t.equals(member.getClass()) && childNode.getNodeId() == NodeId.mapping) { + childNode.setTag(Tag.MAP); + } + } + } + } else if (object instanceof Set) { + t = arguments[0]; + MappingNode mnode = (MappingNode)node; + iter = mnode.getValue().iterator(); + Set set = (Set)object; + var9 = set.iterator(); + + while(var9.hasNext()) { + Object member = var9.next(); + NodeTuple tuple = (NodeTuple)iter.next(); + Node keyNode = tuple.getKeyNode(); + if (t.equals(member.getClass()) && keyNode.getNodeId() == NodeId.mapping) { + keyNode.setTag(Tag.MAP); + } + } + } else if (object instanceof Map) { + t = arguments[0]; + Class valueType = arguments[1]; + MappingNode mnode = (MappingNode)node; + iter = mnode.getValue().iterator(); + + while(iter.hasNext()) { + NodeTuple tuple = (NodeTuple)iter.next(); + this.resetTag(t, tuple.getKeyNode()); + this.resetTag(valueType, tuple.getValueNode()); + } + } + } + + } + } + + private void resetTag(Class type, Node node) { + Tag tag = node.getTag(); + if (tag.matches(type)) { + if (Enum.class.isAssignableFrom(type)) { + node.setTag(Tag.STR); + } else { + node.setTag(Tag.MAP); + } + } + + } + + protected Set getProperties(Class type) { + return this.typeDefinitions.containsKey(type) ? ((TypeDescription)this.typeDefinitions.get(type)).getProperties() : this.getPropertyUtils().getProperties(type); + } + + protected class RepresentJavaBean implements Represent { + protected RepresentJavaBean() { + } + + public Node representData(Object data) { + return Representer.this.representJavaBean(Representer.this.getProperties(data.getClass()), data); + } + } +} -- Gitee From e44a738bc8e7ceed2541d01583b5af7ce798ee85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=9E=BF=E4=BD=90=E9=B9=8F?= Date: Wed, 28 Aug 2024 09:46:16 +0000 Subject: [PATCH 3/3] =?UTF-8?q?=E5=88=A0=E9=99=A4=E6=96=87=E4=BB=B6=20pom.?= =?UTF-8?q?xml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pom.xml | 165 -------------------------------------------------------- 1 file changed, 165 deletions(-) delete mode 100644 pom.xml diff --git a/pom.xml b/pom.xml deleted file mode 100644 index b668423e6..000000000 --- a/pom.xml +++ /dev/null @@ -1,165 +0,0 @@ - - - 4.0.0 - cn.iocoder.cloud - yudao - ${revision} - pom - - yudao-dependencies - yudao-gateway - yudao-framework - - yudao-module-system - yudao-module-infra - yudao-module-member - yudao-module-bpm - yudao-module-pay - - yudao-module-mp - yudao-module-mall - yudao-module-crm - yudao-module-erp - - - - ${project.artifactId} - 芋道项目基础脚手架 - https://github.com/YunaiV/ruoyi-vue-pro - - - 2.2.0-snapshot - - 17 - ${java.version} - ${java.version} - 3.2.2 - 3.13.0 - 1.6.0 - - 1.18.34 - 3.3.1 - 1.5.5.Final - UTF-8 - - - - - - cn.iocoder.cloud - yudao-dependencies - ${revision} - pom - import - - - - - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - ${maven-surefire-plugin.version} - - - - - org.apache.maven.plugins - maven-compiler-plugin - ${maven-compiler-plugin.version} - - - - org.springframework.boot - spring-boot-configuration-processor - ${spring.boot.version} - - - org.projectlombok - lombok - ${lombok.version} - - - org.mapstruct - mapstruct-processor - ${mapstruct.version} - - - - false - - -parameters - - - - - - - - - - org.codehaus.mojo - flatten-maven-plugin - ${flatten-maven-plugin.version} - - resolveCiFriendliesOnly - true - - - - - flatten - - flatten - process-resources - - - - clean - - flatten.clean - clean - - - - - - - - - - huaweicloud - huawei - https://mirrors.huaweicloud.com/repository/maven/ - - - aliyunmaven - aliyun - https://maven.aliyun.com/repository/public - - - - spring-milestones - Spring Milestones - https://repo.spring.io/milestone - - false - - - - spring-snapshots - Spring Snapshots - https://repo.spring.io/snapshot - - false - - - - - -- Gitee