From e1f1229916d2fd21243c7b35b6e2e951cfbf49fa Mon Sep 17 00:00:00 2001 From: CaiHQ Date: Tue, 20 Jun 2023 11:24:42 +0800 Subject: [PATCH 01/14] update doip-sdk version --- build.gradle | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build.gradle b/build.gradle index 8c8bf5d..d1f9791 100644 --- a/build.gradle +++ b/build.gradle @@ -49,15 +49,15 @@ dependencies { implementation 'org.apache.httpcomponents:httpclient:4.5.13' implementation 'io.netty:netty-codec:4.1.77.Final' implementation 'io.netty:netty-codec-http:4.1.77.Final' - implementation 'com.google.code.gson:gson:2.8.6' + implementation 'com.google.code.gson:gson:2.10.1' implementation 'org.bouncycastle:bcpkix-jdk15on:1.69' implementation 'org.bouncycastle:bcprov-jdk15on:1.69' implementation 'com.nimbusds:nimbus-jose-jwt:9.10' //---- repo example - implementation 'mysql:mysql-connector-java:8.0.27' + implementation 'mysql:mysql-connector-java:8.0.33' implementation 'org.rocksdb:rocksdbjni:6.22.1' - implementation 'org.postgresql:postgresql:42.3.4' - implementation 'org.bdware.sc:common:1.6.3' + implementation 'org.postgresql:postgresql:42.6.0' + implementation 'org.bdware.sc:common:1.7.5' implementation fileTree(dir: 'lib', include: '*.jar') //---- testImplementation 'io.netty:netty-all:4.1.77.Final' -- Gitee From 48c18d18e7a0ee313d3ae13086050886b3b4fdac Mon Sep 17 00:00:00 2001 From: "haoeliu@foxmail.com" Date: Sat, 24 Jun 2023 17:16:57 +0800 Subject: [PATCH 02/14] add brief SmartClusterClient and relevant tools --- .../doip/cluster/smart/ClusterTools.java | 74 +++++++++++++++++++ .../cluster/smart/SmartClusterClient.java | 40 ++++++++++ 2 files changed, 114 insertions(+) create mode 100644 src/main/java/org/bdware/doip/cluster/smart/ClusterTools.java create mode 100644 src/main/java/org/bdware/doip/cluster/smart/SmartClusterClient.java diff --git a/src/main/java/org/bdware/doip/cluster/smart/ClusterTools.java b/src/main/java/org/bdware/doip/cluster/smart/ClusterTools.java new file mode 100644 index 0000000..e92209d --- /dev/null +++ b/src/main/java/org/bdware/doip/cluster/smart/ClusterTools.java @@ -0,0 +1,74 @@ +package org.bdware.doip.cluster.smart; + +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.bdware.sc.bean.JoinInfo; +import org.bdware.sc.bean.RouteInfo; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class ClusterTools { + private static final Logger Logger = LogManager.getLogger(ClusterTools.class); + private static final Gson gson = new Gson(); + + public static String[] getDOIDClusterInfo(JsonObject BDOClusterInfo) { + JsonArray bdoList = BDOClusterInfo.get("bdoList").getAsJsonArray(); + JsonObject appendixes = BDOClusterInfo.get("appendixes").getAsJsonObject(); + List subDOIDsList = new ArrayList<>(); + for (JsonElement bdoId : bdoList) { + String bdo = bdoId.getAsString(); + if (appendixes.get(bdo) != null || !appendixes.get(bdo).isJsonNull()) { + subDOIDsList.add(bdo); + } else { + Logger.error(bdo + " cannot be parsed by Router"); + } + } + + String[] subDOIDs = new String[subDOIDsList.size()]; + subDOIDsList.toArray(subDOIDs); + return subDOIDs; + } + + public static Map getDOIDRouteInfo(JsonObject accessRulesJO) { + // 解析RouteInfo + JsonElement routeInfos = accessRulesJO.get("routeInfo"); + Map res = new HashMap<>(); + + assert routeInfos != null && !routeInfos.isJsonNull() && routeInfos.isJsonObject(); + + JsonObject routeInfosMap = routeInfos.getAsJsonObject(); + for (String doipOperationName : routeInfosMap.keySet()) { + JsonObject doipFunctionRouteInfoJson = routeInfosMap.get(doipOperationName).getAsJsonObject(); + RouteInfo doipFunctionRouteInfo = gson.fromJson(doipFunctionRouteInfoJson, RouteInfo.class); + res.put(doipOperationName, doipFunctionRouteInfo); + } + + + return res; + } + + public static Map getDOIDJoinInfo(JsonObject accessRulesJO) { + // 解析JoinInfo + JsonElement joinInfos = accessRulesJO.get("joinInfo"); + Map res = new HashMap<>(); + + assert joinInfos != null && !joinInfos.isJsonNull() && joinInfos.isJsonObject(); + + JsonObject joinInfosMap = joinInfos.getAsJsonObject(); + for (String doipOperationName : joinInfosMap.keySet()) { + JsonObject doipFunctionJoinInfoJson = joinInfosMap.get(doipOperationName).getAsJsonObject(); + JoinInfo doipFunctionJoinInfo = gson.fromJson(doipFunctionJoinInfoJson, JoinInfo.class); + res.put(doipOperationName, doipFunctionJoinInfo); + } + + return res; + } + +} diff --git a/src/main/java/org/bdware/doip/cluster/smart/SmartClusterClient.java b/src/main/java/org/bdware/doip/cluster/smart/SmartClusterClient.java new file mode 100644 index 0000000..bb83ab0 --- /dev/null +++ b/src/main/java/org/bdware/doip/cluster/smart/SmartClusterClient.java @@ -0,0 +1,40 @@ +package org.bdware.doip.cluster.smart; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.bdware.doip.audit.EndpointConfig; +import org.bdware.doip.audit.client.AuditIrpClient; +import org.bdware.doip.cluster.util.RouterTool; +import org.bdware.doip.endpoint.client.DoipClient; +import org.bdware.doip.endpoint.client.DoipClientImpl; +import wrp.jdk.nashorn.api.scripting.NashornScriptEngineUtil; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +public class SmartClusterClient extends DoipClientImpl { + private final String clientDoid; + // router config -> provide to DoipClusterClient + private final EndpointConfig routerConfig; + // irsClient -> connect to router to resolve doid + private final AuditIrpClient irsClient; + + private static final Map doid2CachedClusterInfo= new ConcurrentHashMap(); + + private static final Logger Logger = LogManager.getLogger(SmartClusterClient.class); + + public SmartClusterClient(String clientDoid, EndpointConfig config) { + this.clientDoid = clientDoid; + this.routerConfig = config; + this.irsClient = new AuditIrpClient(config); + + Logger.info("The DoipClusterClient has been initialized"); + } + + public DoipClient smartSelectDoipClient(String doid) { + + return null; + } + + +} -- Gitee From 443b3df5f001aeaf0e94202b130bbb9582c2bef7 Mon Sep 17 00:00:00 2001 From: "haoeliu@foxmail.com" Date: Mon, 3 Jul 2023 14:22:36 +0800 Subject: [PATCH 03/14] fulfill some functions in SmartContractClient --- .../bdware/doip/cluster/entity/DDOEntity.java | 46 +++++++ .../bdware/doip/cluster/smart/BDWType.java | 8 ++ .../cluster/smart/SmartClusterClient.java | 117 ++++++++++++++++-- .../{ClusterTools.java => SmartTools.java} | 73 ++++++++++- .../bdware/doip/cluster/util/RouterTool.java | 20 +-- 5 files changed, 239 insertions(+), 25 deletions(-) create mode 100644 src/main/java/org/bdware/doip/cluster/entity/DDOEntity.java create mode 100644 src/main/java/org/bdware/doip/cluster/smart/BDWType.java rename src/main/java/org/bdware/doip/cluster/smart/{ClusterTools.java => SmartTools.java} (43%) diff --git a/src/main/java/org/bdware/doip/cluster/entity/DDOEntity.java b/src/main/java/org/bdware/doip/cluster/entity/DDOEntity.java new file mode 100644 index 0000000..6806519 --- /dev/null +++ b/src/main/java/org/bdware/doip/cluster/entity/DDOEntity.java @@ -0,0 +1,46 @@ +package org.bdware.doip.cluster.entity; + +import org.bdware.sc.bean.JoinInfo; +import org.bdware.sc.bean.RouteInfo; +import wrp.jdk.nashorn.api.scripting.NashornScriptEngineUtil; + +import java.util.Map; + +public class DDOEntity { + // hello -> routeInfo + private Map ddoRouteInfo; + // hello -> joinInfo + private Map ddoJoinInfo; + // script engine + private NashornScriptEngineUtil engineUtil; + + public DDOEntity(Map ddoRouteInfo, Map ddoJoinInfo, NashornScriptEngineUtil engineUtil) { + this.ddoRouteInfo = ddoRouteInfo; + this.ddoJoinInfo = ddoJoinInfo; + this.engineUtil = engineUtil; + } + + public Map getDDORouteInfo() { + return ddoRouteInfo; + } + + public void setDDORouteInfo(Map ddoRouteInfo) { + this.ddoRouteInfo = ddoRouteInfo; + } + + public Map getDDOJoinInfo() { + return ddoJoinInfo; + } + + public void setDDOJoinInfo(Map ddoJoinInfo) { + this.ddoJoinInfo = ddoJoinInfo; + } + + public NashornScriptEngineUtil getEngineUtil() { + return engineUtil; + } + + public void setEngineUtil(NashornScriptEngineUtil engineUtil) { + this.engineUtil = engineUtil; + } +} diff --git a/src/main/java/org/bdware/doip/cluster/smart/BDWType.java b/src/main/java/org/bdware/doip/cluster/smart/BDWType.java new file mode 100644 index 0000000..ca375a1 --- /dev/null +++ b/src/main/java/org/bdware/doip/cluster/smart/BDWType.java @@ -0,0 +1,8 @@ +package org.bdware.doip.cluster.smart; + +public enum BDWType { + DDO, + BDO, + BDRepo, + BCRepo +} \ No newline at end of file diff --git a/src/main/java/org/bdware/doip/cluster/smart/SmartClusterClient.java b/src/main/java/org/bdware/doip/cluster/smart/SmartClusterClient.java index bb83ab0..3bef520 100644 --- a/src/main/java/org/bdware/doip/cluster/smart/SmartClusterClient.java +++ b/src/main/java/org/bdware/doip/cluster/smart/SmartClusterClient.java @@ -1,40 +1,139 @@ package org.bdware.doip.cluster.smart; +import com.google.gson.Gson; +import com.google.gson.JsonObject; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.bdware.doip.audit.EndpointConfig; +import org.bdware.doip.audit.client.AuditDoipClient; import org.bdware.doip.audit.client.AuditIrpClient; -import org.bdware.doip.cluster.util.RouterTool; +import org.bdware.doip.audit.writer.AuditConfig; +import org.bdware.doip.audit.writer.AuditType; +import org.bdware.doip.cluster.entity.DDOEntity; +import org.bdware.doip.codec.doipMessage.DoipMessageSigner; +import org.bdware.doip.encrypt.SM2Signer; import org.bdware.doip.endpoint.client.DoipClient; import org.bdware.doip.endpoint.client.DoipClientImpl; -import wrp.jdk.nashorn.api.scripting.NashornScriptEngineUtil; +import org.bdware.irp.exception.IrpClientException; +import org.zz.gmhelper.SM2KeyPair; +import org.zz.gmhelper.SM2Util; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.*; public class SmartClusterClient extends DoipClientImpl { + private final DoipMessageSigner doipMessageSigner; private final String clientDoid; // router config -> provide to DoipClusterClient private final EndpointConfig routerConfig; // irsClient -> connect to router to resolve doid private final AuditIrpClient irsClient; + // DDOID -> Contract Method -> RouteInfo + private Map ddoCache = new ConcurrentHashMap<>(); + // DOID -> Connection + private final Map addressToClient = new ConcurrentHashMap<>(); - private static final Map doid2CachedClusterInfo= new ConcurrentHashMap(); - - private static final Logger Logger = LogManager.getLogger(SmartClusterClient.class); + private static final Logger LOGGER = LogManager.getLogger(SmartClusterClient.class); public SmartClusterClient(String clientDoid, EndpointConfig config) { this.clientDoid = clientDoid; this.routerConfig = config; this.irsClient = new AuditIrpClient(config); + if (routerConfig.publicKey != null && routerConfig.privateKey != null) { + doipMessageSigner = new SM2Signer(SM2KeyPair.fromJson(new Gson().toJson(routerConfig))); + } else + doipMessageSigner = new SM2Signer(SM2Util.generateSM2KeyPair()); - Logger.info("The DoipClusterClient has been initialized"); + LOGGER.info("The DoipClusterClient has been initialized"); } - public DoipClient smartSelectDoipClient(String doid) { - + public DoipClient smartSelectDoipClient(String doid) throws IrpClientException { + JsonObject doInfo = SmartTools.getDOInfo(irsClient, doid); + BDWType doType = SmartTools.getDOType(doInfo); + switch (doType) { + case DDO: + SmartTools.verifyDDOInfo(doInfo); + System.out.println("Hello world"); + break; + case BDO: + case BDRepo: + String address = doInfo.get("address").getAsString(); + String version = doInfo.get("version").getAsString(); + return getClientByUrl(address, version); + case BCRepo: + System.out.println("Hello world3"); + break; + default: + LOGGER.error("Wrong doType: " + doType); + throw new IrpClientException("doType cannot be parsed"); + } + return null; } + public AuditDoipClient getClientByUrl(String address, String version) { + if (address == null && version == null) return null; + AuditDoipClient doipClientImpl = addressToClient.get(address); + if (doipClientImpl == null) { + // Create the client only if it doesn't exist in the map + doipClientImpl = createWithoutConnect(address, version); + // Use putIfAbsent() to ensure thread-safety + AuditDoipClient existingClient = addressToClient.putIfAbsent(address, doipClientImpl); + if (existingClient != null) { + // If another thread added a client while we were creating ours, use that instead + doipClientImpl = existingClient; + } + } + try { + // Always try to reconnect, regardless of whether the client was just created or already existed + tryReconnectSync(doipClientImpl, 2000); + } catch (Exception e) { + e.printStackTrace(); + } + return doipClientImpl; + } + private AuditDoipClient createWithoutConnect(String address, String version) { + AuditDoipClient doipClientImpl = new AuditDoipClient(AuditConfig.newInstance(null, AuditType.None, null), null); + LOGGER.info("connect to:" + address); + doipClientImpl.setRepoUrl(address); + return doipClientImpl; + } + + ExecutorService connectionTaskPool = Executors.newFixedThreadPool(8, new ThreadFactory() { + @Override + public Thread newThread(Runnable r) { + Thread t = new Thread(r); + t.setDaemon(true); + return t; + } + }); + private AuditDoipClient tryReconnectSync(AuditDoipClient doipClientImpl, long timeToWait) { + CompletableFuture future = CompletableFuture.supplyAsync(() -> { + try { + synchronized (doipClientImpl) { + if (!doipClientImpl.isConnected()) { + // LOGGER.info("=========ACTUALLY try reconnect to:" + doipClientImpl.getRepoUrl()); + doipClientImpl.reconnect(); + } + } + return doipClientImpl; + } catch (Exception e) { + ByteArrayOutputStream bo = new ByteArrayOutputStream(); + e.printStackTrace(new PrintStream(bo)); + LOGGER.info("failed to connect to:" + doipClientImpl.getRepoUrl() + " " + bo); + return null; + } + }, connectionTaskPool); + + try { + return future.get(timeToWait, TimeUnit.MILLISECONDS); + } catch (InterruptedException | ExecutionException | TimeoutException e) { + LOGGER.error("============Reconnect Exception " + doipClientImpl.getRepoUrl() + "============", e); + future.cancel(true); + return null; + } + } } diff --git a/src/main/java/org/bdware/doip/cluster/smart/ClusterTools.java b/src/main/java/org/bdware/doip/cluster/smart/SmartTools.java similarity index 43% rename from src/main/java/org/bdware/doip/cluster/smart/ClusterTools.java rename to src/main/java/org/bdware/doip/cluster/smart/SmartTools.java index e92209d..e8cadd9 100644 --- a/src/main/java/org/bdware/doip/cluster/smart/ClusterTools.java +++ b/src/main/java/org/bdware/doip/cluster/smart/SmartTools.java @@ -1,11 +1,11 @@ package org.bdware.doip.cluster.smart; -import com.google.gson.Gson; -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; +import com.google.gson.*; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.bdware.doip.audit.client.AuditIrpClient; +import org.bdware.irp.exception.IrpClientException; +import org.bdware.irp.stateinfo.StateInfoBase; import org.bdware.sc.bean.JoinInfo; import org.bdware.sc.bean.RouteInfo; @@ -14,8 +14,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -public class ClusterTools { - private static final Logger Logger = LogManager.getLogger(ClusterTools.class); +public class SmartTools { + private static final Logger Logger = LogManager.getLogger(SmartTools.class); private static final Gson gson = new Gson(); public static String[] getDOIDClusterInfo(JsonObject BDOClusterInfo) { @@ -71,4 +71,65 @@ public class ClusterTools { return res; } + public static JsonObject getAndVerifyDDOInfo(AuditIrpClient irsClient, String doid) throws IrpClientException { + StateInfoBase content = irsClient.resolve(doid); + if (content == null || content.handleValues == null || content.handleValues.isJsonNull()) { + throw new IrpClientException("DDO's info isn't in the router, id cannot be resolved"); + } + + return verifyDDOInfo(content.handleValues); + } + + public static JsonObject getDOInfo(AuditIrpClient irsClient, String doid) throws IrpClientException { + StateInfoBase content = irsClient.resolve(doid); + if (content == null || content.handleValues == null || content.handleValues.isJsonNull()) { + throw new IrpClientException("DO's info isn't in the router, id cannot be resolved"); + } + + return content.handleValues; + } + + public static JsonObject verifyDDOInfo(JsonObject doInfo) throws IrpClientException { + // 如果不含有 ClusterInfo 或者 BCOID(ClusterInfo & JoinInfo) + if (doInfo.get("bdoList") == null || doInfo.get("bdoList").isJsonNull() + || doInfo.get("bcoId") == null || doInfo.get("bcoId").isJsonNull() + || doInfo.get("appendixes") == null || doInfo.get("appendixes").isJsonNull()) { + throw new IrpClientException("DDO doesn't have enough info, bdoList or bcoId or appendixes cannot be verified"); + } + + // 如果不含有 ClusterInfo & JoinInfo + String bcoId = doInfo.get("bcoId").getAsString(); + JsonObject appendixes = doInfo.get("appendixes").getAsJsonObject(); + if (appendixes.get(bcoId) == null || appendixes.get(bcoId).isJsonNull() + || appendixes.get(bcoId).getAsJsonObject().get("accessRules") == null + || appendixes.get(bcoId).getAsJsonObject().get("accessRules").isJsonNull()) { + throw new IrpClientException("DDO doesn't have enough info , accessRules cannot be verified"); + } + Logger.info(new GsonBuilder().setPrettyPrinting().create().toJson(doInfo)); + return doInfo; + } + + public static BDWType getDOType(AuditIrpClient irsClient, String doid) throws IrpClientException { + StateInfoBase content = irsClient.resolve(doid); + if (content == null || content.handleValues == null || content.handleValues.isJsonNull()) { + throw new IrpClientException("DO's info isn't in the router, id cannot be resolved"); + } + + JsonObject doInfo = content.handleValues; + if(doInfo.get("bdwType") != null && !doInfo.get("bdwType").isJsonNull()) { + String bdwType = doInfo.get("bdwType").getAsString(); + return BDWType.valueOf(bdwType); + } + + return null; + } + + public static BDWType getDOType(JsonObject doInfo) throws IrpClientException { + if(doInfo.get("bdwType") == null || doInfo.get("bdwType").isJsonNull()) { + throw new IrpClientException("DO's info has not bdwType, please check the DO in the router"); + } + + String bdwType = doInfo.get("bdwType").getAsString(); + return BDWType.valueOf(bdwType); + } } diff --git a/src/main/java/org/bdware/doip/cluster/util/RouterTool.java b/src/main/java/org/bdware/doip/cluster/util/RouterTool.java index 4023ba9..4fc81ae 100644 --- a/src/main/java/org/bdware/doip/cluster/util/RouterTool.java +++ b/src/main/java/org/bdware/doip/cluster/util/RouterTool.java @@ -27,30 +27,30 @@ public class RouterTool { this.engineUtil = engineUtil; } - public JsonObject verifyCluster(String id) throws IrpClientException { - StateInfoBase content = irsClient.resolve(id); + public JsonObject verifyCluster(String doid) throws IrpClientException { + StateInfoBase content = irsClient.resolve(doid); if (content == null || content.handleValues == null || content.handleValues.isJsonNull()) { throw new IrpClientException("BDO's info isn't in the router, id cannot be resolved"); } // 如果不含有clusterInfo或者BCOID(ClusterInfo & JoinInfo) - JsonObject BDOInfo = content.handleValues; - if (BDOInfo.get("bdoList") == null || BDOInfo.get("bdoList").isJsonNull() - || BDOInfo.get("bcoId") == null || BDOInfo.get("bcoId").isJsonNull() - || BDOInfo.get("appendixes") == null || BDOInfo.get("appendixes").isJsonNull()) { + JsonObject DOInfo = content.handleValues; + if (DOInfo.get("bdoList") == null || DOInfo.get("bdoList").isJsonNull() + || DOInfo.get("bcoId") == null || DOInfo.get("bcoId").isJsonNull() + || DOInfo.get("appendixes") == null || DOInfo.get("appendixes").isJsonNull()) { throw new IrpClientException("BDO doesn't have enough cluster info"); } // 如果不有ClusterInfo & JoinInfo - String bcoId = BDOInfo.get("bcoId").getAsString(); - JsonObject appendixes = BDOInfo.get("appendixes").getAsJsonObject(); + String bcoId = DOInfo.get("bcoId").getAsString(); + JsonObject appendixes = DOInfo.get("appendixes").getAsJsonObject(); if (appendixes.get(bcoId) == null || appendixes.get(bcoId).isJsonNull() || appendixes.get(bcoId).getAsJsonObject().get("accessRules") == null || appendixes.get(bcoId).getAsJsonObject().get("accessRules").isJsonNull()) { throw new IrpClientException("BDO doesn't have enough cluster info"); } - Logger.info(new GsonBuilder().setPrettyPrinting().create().toJson(BDOInfo)); - return BDOInfo; + Logger.info(new GsonBuilder().setPrettyPrinting().create().toJson(DOInfo)); + return DOInfo; } public void parseClusterInfo(JsonObject BDOInfo, NashornScriptEngineUtil engineUtil) { -- Gitee From d91bb185a041570558858634e1c9483d3cfcadab Mon Sep 17 00:00:00 2001 From: "haoeliu@foxmail.com" Date: Sun, 9 Jul 2023 11:09:19 +0800 Subject: [PATCH 04/14] complete draft TreeClusterClient --- .../BDODelegateDoipMessageCallback.java | 53 +++++ .../DDODelegateDoipMessageCallback.java | 205 ++++++++++++++++++ .../DelegateDoipMessageCallback.java | 4 +- .../cluster/client/TreeClusterClient.java | 171 +++++++++++++++ .../bdware/doip/cluster/entity/BDOEntity.java | 27 +++ .../cluster/{smart => entity}/BDWType.java | 2 +- .../bdware/doip/cluster/entity/DDOEntity.java | 13 +- .../ClusterClientUtil.java} | 43 +++- .../ConnectionUtil.java} | 111 ++-------- .../doip/cluster/util/ReconnectionUtil.java | 104 +++++++++ .../doip/cluster/util/ResultCollector.java | 5 +- .../doip/cluster/util/RouteJoinUtil.java | 133 ++++++++++++ 12 files changed, 760 insertions(+), 111 deletions(-) create mode 100644 src/main/java/org/bdware/doip/cluster/callback/BDODelegateDoipMessageCallback.java create mode 100644 src/main/java/org/bdware/doip/cluster/callback/DDODelegateDoipMessageCallback.java rename src/main/java/org/bdware/doip/cluster/{client => callback}/DelegateDoipMessageCallback.java (96%) create mode 100644 src/main/java/org/bdware/doip/cluster/client/TreeClusterClient.java create mode 100644 src/main/java/org/bdware/doip/cluster/entity/BDOEntity.java rename src/main/java/org/bdware/doip/cluster/{smart => entity}/BDWType.java (60%) rename src/main/java/org/bdware/doip/cluster/{smart/SmartTools.java => util/ClusterClientUtil.java} (74%) rename src/main/java/org/bdware/doip/cluster/{smart/SmartClusterClient.java => util/ConnectionUtil.java} (30%) create mode 100644 src/main/java/org/bdware/doip/cluster/util/ReconnectionUtil.java create mode 100644 src/main/java/org/bdware/doip/cluster/util/RouteJoinUtil.java diff --git a/src/main/java/org/bdware/doip/cluster/callback/BDODelegateDoipMessageCallback.java b/src/main/java/org/bdware/doip/cluster/callback/BDODelegateDoipMessageCallback.java new file mode 100644 index 0000000..a23093a --- /dev/null +++ b/src/main/java/org/bdware/doip/cluster/callback/BDODelegateDoipMessageCallback.java @@ -0,0 +1,53 @@ +package org.bdware.doip.cluster.callback; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.bdware.doip.audit.client.AuditDoipClient; +import org.bdware.doip.cluster.client.TreeClusterClient; +import org.bdware.doip.codec.doipMessage.DoipMessage; +import org.bdware.doip.codec.doipMessage.DoipResponseCode; +import org.bdware.doip.endpoint.client.DoipMessageCallback; + +import java.net.URI; + +public class BDODelegateDoipMessageCallback implements DoipMessageCallback { + static Logger LOGGER = LogManager.getLogger(DelegateDoipMessageCallback.class); + private final DoipMessageCallback originalCallback; + DoipMessage originalMessage; + AuditDoipClient previousClient; + + public BDODelegateDoipMessageCallback(AuditDoipClient previousClient, DoipMessageCallback doipMessageCallback, DoipMessage originalMessage) { + this.previousClient = previousClient; + this.originalCallback = doipMessageCallback; + this.originalMessage = originalMessage; + } + + @Override + public void onResult(DoipMessage msg) { + if (msg.header.parameters.response == DoipResponseCode.DelegateRequired) { + + try { + if (msg.header.parameters.attributes.has("port")) { + int port = msg.header.parameters.attributes.get("port").getAsInt(); + URI uri = new URI(previousClient.getRepoUrl()); + String host = uri.getHost(); + if (msg.header.parameters.attributes.has("host")) + host = msg.header.parameters.attributes.get("host").getAsString(); + String newAddress = String.format("%s://%s:%d", uri.getScheme(), host, port); + String version = "2.1"; + if (msg.header.parameters.attributes.has("version")) + version = msg.header.parameters.attributes.get("version").getAsString(); + LOGGER.info("recv delegate:"+newAddress +" previousRepo:"+previousClient.getRepoUrl()); + // BDRepo doipMsgCallback -> BDOClient，维护repoToBDO的映射关系 + AuditDoipClient client = TreeClusterClient.getClientByUrl(newAddress, version); + previousClient = client; + client.sendRawMessage(originalMessage, this); + } + } catch (Exception e) { + e.printStackTrace(); + originalCallback.onResult(msg); + } + } else originalCallback.onResult(msg); + } +} + diff --git a/src/main/java/org/bdware/doip/cluster/callback/DDODelegateDoipMessageCallback.java b/src/main/java/org/bdware/doip/cluster/callback/DDODelegateDoipMessageCallback.java new file mode 100644 index 0000000..b3a1c59 --- /dev/null +++ b/src/main/java/org/bdware/doip/cluster/callback/DDODelegateDoipMessageCallback.java @@ -0,0 +1,205 @@ +package org.bdware.doip.cluster.callback; + +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.bdware.doip.cluster.util.ReconnectionUtil; +import org.bdware.doip.cluster.client.TreeClusterClient; +import org.bdware.doip.cluster.util.ResultCollector; +import org.bdware.doip.codec.JsonDoipMessage; +import org.bdware.doip.codec.doipMessage.DoipMessage; +import org.bdware.doip.codec.doipMessage.DoipMessageFactory; +import org.bdware.doip.codec.doipMessage.DoipResponseCode; +import org.bdware.doip.endpoint.client.DoipMessageCallback; +import org.bdware.sc.bean.JoinInfo; +import org.bdware.sc.util.JsonUtil; +import wrp.jdk.nashorn.api.scripting.NashornScriptEngineUtil; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; + +public class DDODelegateDoipMessageCallback implements DoipMessageCallback { + static org.apache.logging.log4j.Logger LOGGER = LogManager.getLogger(ResultCollector.class); + private final DoipMessage request; + // joinCount记录开始Merge的上限 + int joinCount; + AtomicInteger count = new AtomicInteger(0); + DoipMessageCallback originalCallback; + Logger Logger = LogManager.getLogger(ResultCollector.class); + Map successResMap = new HashMap<>(); + Set problemResult = new HashSet<>(); + // 记录所有收到结果列表 + Set nodeIDs = new HashSet<>(); + NashornScriptEngineUtil engineUtil; + JoinInfo joinInfo; + String[] targetServers; + TreeClusterClient treeClusterClient; + ReconnectionUtil reconnectionUtil; + + + public DDODelegateDoipMessageCallback(TreeClusterClient treeClusterClient, String[] targetServers, DoipMessage msg, DoipMessageCallback cb, int count, NashornScriptEngineUtil engineUtil, JoinInfo joinInfo, JsonObject appendixes) { + this.request = msg; + this.originalCallback = cb; + this.joinCount = count; + this.engineUtil = engineUtil; + this.joinInfo = joinInfo; + this.targetServers = targetServers; + this.treeClusterClient = treeClusterClient; + reconnectionUtil = new ReconnectionUtil(appendixes); + } + + @Override + public void onResult(DoipMessage msg) { + logDoipMsgInfo(msg); + // 如果nodeID不存在，或者该节点的返回值已经被处理过了，不需要再次调用onResult了 + String nodeID = getNodeID(msg); + if (nodeID == null) { + // 如果当前nodeID == null, 记录到problemResult中 + problemResult.add(msg); + } else if (nodeIDs.contains(nodeID)) { + LOGGER.info("Receive Duplicated Msg"); + } else { + nodeIDs.add(nodeID); + addDoipResult(nodeID, msg); + } + + int curCount = count.incrementAndGet(); + // 小于joinCount，意味着还没达到聚合的标准，需要继续处理 + if (curCount <= joinCount) { + // 达到聚合标准 + if (curCount == joinCount) { + // 完成聚合 & 结果统计 + DoipMessage finalResult = executeJoinFunc(msg); + originalCallback.onResult(finalResult); + } + } + } + + public DoipMessageCallback getOriginalCallback() { + return originalCallback; + } + + public void logDoipMsgInfo(DoipMessage msg) { + try { + String nodeID = getNodeID(msg); + if (nodeID != null) { + popReconnectRequest(); + } else { + Logger.error("收到错误的节点回复 : " + "Something wrong happens in" + new Gson().toJson(targetServers) + " msg:" + new Gson().toJson(msg.header.parameters)); + pushReconnectRequest(); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + private void popReconnectRequest() { + reconnectionUtil.popReconnectRequest(targetServers); + } + + private void pushReconnectRequest() { + reconnectionUtil.pushReconnectRequest(targetServers); + } + + public synchronized void addDoipResult(String nodeID, DoipMessage msg) { + successResMap.put(nodeID, msg); + } + + + public String getNodeID(DoipMessage msg) { + String nodeID = null; + if (msg != null && msg.header != null && msg.header.parameters != null && msg.header.parameters.attributes != null) { + if (msg.header.parameters.attributes.get("nodeID") != null) { + nodeID = msg.header.parameters.attributes.get("nodeID").getAsString(); + } + } + + return nodeID; + } + + public DoipMessage executeJoinFunc(DoipMessage request) { + DoipMessage resultMessage; + if (joinInfo != null && problemResult.size() == 0) { + resultMessage = handleJoinInfo(joinInfo); + return resultMessage; + } + if (successResMap.size() + problemResult.size() == 1) { + problemResult.addAll(successResMap.values()); + for (DoipMessage message : problemResult) + return message; + } + + DoipMessageFactory.DoipMessageBuilder builder = new DoipMessageFactory.DoipMessageBuilder(); + if (problemResult.size() > 0) + builder.createResponse(DoipResponseCode.UnKnownError, request); + else + builder.createResponse(DoipResponseCode.Success, request); + JsonObject successResponses = new JsonObject(); + for (String key : successResMap.keySet()) { + successResponses.add(key, JsonUtil.parseObject(JsonDoipMessage.fromDoipMessage(successResMap.get(key)))); + } + JsonArray problemResponses = new JsonArray(); + for (DoipMessage msg : problemResult) { + problemResponses.add(JsonUtil.parseObject(JsonDoipMessage.fromDoipMessage(msg))); + } + builder.addAttributes("successResponses", successResponses); + builder.addAttributes("problemResponses", problemResponses); + return builder.create(); + } + + private DoipMessage handleJoinInfo(JoinInfo joinInfo) { + try { + if (joinInfo.useDefault == null) { + if (joinInfo.joinFuncName != null) { + JsonObject jo = new JsonObject(); + for (String key : successResMap.keySet()) { + jo.add(key, JsonUtil.parseObject(JsonDoipMessage.fromDoipMessage(successResMap.get(key)))); + } + JsonArray array = new JsonArray(); + for (DoipMessage resp : problemResult) { + array.add(JsonUtil.parseObject(JsonDoipMessage.fromDoipMessage(resp))); + } + JsonElement returnVal = engineUtil.invokeFunction(joinInfo.joinFuncName, JsonElement.class, jo, array); + JsonDoipMessage result = JsonUtil.fromJson(returnVal, JsonDoipMessage.class); + return result.toResponseDoipMessage(request); + } else throw new IllegalArgumentException("missing joinFuncName"); + } + DoipMessage returnedMsg = null; + switch (joinInfo.useDefault) { + case add: + double val = 0; + for (DoipMessage msg : successResMap.values()) { + val += Double.valueOf(msg.body.getDataAsJsonString()); + returnedMsg = msg; + } + returnedMsg.body.encodedData = (val + "").getBytes(); + return returnedMsg; + case multiply: + val = 1; + for (DoipMessage msg : successResMap.values()) { + val *= Double.valueOf(msg.body.getDataAsJsonString()); + returnedMsg = msg; + } + return returnedMsg; + default: + throw new IllegalArgumentException("missing default merge rule"); + } + } catch (Exception e) { + DoipMessageFactory.DoipMessageBuilder builder = new DoipMessageFactory.DoipMessageBuilder(); + builder.createResponse(DoipResponseCode.Declined, request); + ByteArrayOutputStream bo = new ByteArrayOutputStream(); + e.printStackTrace(new PrintStream(bo)); + builder.setBody(bo.toByteArray()); + e.printStackTrace(); + return builder.create(); + } + } +} diff --git a/src/main/java/org/bdware/doip/cluster/client/DelegateDoipMessageCallback.java b/src/main/java/org/bdware/doip/cluster/callback/DelegateDoipMessageCallback.java similarity index 96% rename from src/main/java/org/bdware/doip/cluster/client/DelegateDoipMessageCallback.java rename to src/main/java/org/bdware/doip/cluster/callback/DelegateDoipMessageCallback.java index 7393cc7..a68328b 100644 --- a/src/main/java/org/bdware/doip/cluster/client/DelegateDoipMessageCallback.java +++ b/src/main/java/org/bdware/doip/cluster/callback/DelegateDoipMessageCallback.java @@ -1,11 +1,11 @@ -package org.bdware.doip.cluster.client; +package org.bdware.doip.cluster.callback; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.bdware.doip.audit.client.AuditDoipClient; +import org.bdware.doip.cluster.client.DoipClusterClient; import org.bdware.doip.codec.doipMessage.DoipMessage; import org.bdware.doip.codec.doipMessage.DoipResponseCode; -import org.bdware.doip.endpoint.client.DoipClientImpl; import org.bdware.doip.endpoint.client.DoipMessageCallback; import java.net.URI; diff --git a/src/main/java/org/bdware/doip/cluster/client/TreeClusterClient.java b/src/main/java/org/bdware/doip/cluster/client/TreeClusterClient.java new file mode 100644 index 0000000..c66b506 --- /dev/null +++ b/src/main/java/org/bdware/doip/cluster/client/TreeClusterClient.java @@ -0,0 +1,171 @@ +package org.bdware.doip.cluster.client; + +import com.google.gson.Gson; +import com.google.gson.JsonObject; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.bdware.doip.audit.EndpointConfig; +import org.bdware.doip.audit.client.AuditDoipClient; +import org.bdware.doip.audit.client.AuditIrpClient; +import org.bdware.doip.cluster.callback.BDODelegateDoipMessageCallback; +import org.bdware.doip.cluster.entity.BDOEntity; +import org.bdware.doip.cluster.entity.BDWType; +import org.bdware.doip.cluster.entity.DDOEntity; +import org.bdware.doip.cluster.util.ClusterClientUtil; +import org.bdware.doip.cluster.util.ConnectionUtil; +import org.bdware.doip.cluster.util.RouteJoinUtil; +import org.bdware.doip.codec.JsonDoipMessage; +import org.bdware.doip.codec.doipMessage.DoipMessage; +import org.bdware.doip.codec.doipMessage.DoipMessageFactory; +import org.bdware.doip.codec.doipMessage.DoipMessageSigner; +import org.bdware.doip.codec.doipMessage.DoipResponseCode; +import org.bdware.doip.codec.operations.BasicOperations; +import org.bdware.doip.encrypt.SM2Signer; +import org.bdware.doip.endpoint.client.DoipClientImpl; +import org.bdware.doip.endpoint.client.DoipMessageCallback; +import org.bdware.irp.exception.IrpClientException; +import org.bdware.sc.bean.JoinInfo; +import org.bdware.sc.bean.RouteInfo; +import org.zz.gmhelper.SM2KeyPair; +import org.zz.gmhelper.SM2Util; +import wrp.jdk.nashorn.api.scripting.NashornScriptEngineUtil; + +import javax.script.ScriptException; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +public class TreeClusterClient extends DoipClientImpl { + public final DoipMessageSigner doipMessageSigner; + // router config -> provide to DoipClusterClient + public final EndpointConfig routerConfig; + // irsClient -> connect to router to resolve doid + private final AuditIrpClient irsClient; + // DDOID -> Contract Method -> RouteInfo + private static final Map ddoCache = new ConcurrentHashMap<>(); + private static final Map bdoCache = new ConcurrentHashMap<>(); + // DOID -> Connection + private static final Map addressToClient = new ConcurrentHashMap<>(); + + private static final Logger LOGGER = LogManager.getLogger(TreeClusterClient.class); + + public TreeClusterClient(EndpointConfig config) { + this.routerConfig = config; + this.irsClient = new AuditIrpClient(config); + if (routerConfig.publicKey != null && routerConfig.privateKey != null) { + doipMessageSigner = new SM2Signer(SM2KeyPair.fromJson(new Gson().toJson(routerConfig))); + } else + doipMessageSigner = new SM2Signer(SM2Util.generateSM2KeyPair()); + + LOGGER.info("The SmartClusterClient has been initialized"); + } + + @Override + public void sendMessage(DoipMessage doipMessage, DoipMessageCallback cb) { + try { + parseDORecursive(doipMessage.header.parameters.id, doipMessage, cb); + } catch (Exception e) { + e.printStackTrace(); + DoipMessageFactory.DoipMessageBuilder builder = new DoipMessageFactory.DoipMessageBuilder(); + builder.createResponse(DoipResponseCode.UnKnownError, doipMessage); + cb.onResult(builder.create()); + } + } + + public void sendMessage(DoipMessage doipMessage, String targetDOID, DoipMessageCallback cb) { + try { + parseDORecursive(targetDOID, doipMessage, cb); + } catch (Exception e) { + e.printStackTrace(); + DoipMessageFactory.DoipMessageBuilder builder = new DoipMessageFactory.DoipMessageBuilder(); + builder.createResponse(DoipResponseCode.UnKnownError, doipMessage); + cb.onResult(builder.create()); + } + } + + // 返回当前doid对应的所有的BDRepo + public void parseDORecursive(String doid, DoipMessage doipMessage, DoipMessageCallback cb) throws IrpClientException, ScriptException { + JsonObject doInfo = ClusterClientUtil.getDOInfo(irsClient, doid); + BDWType doType = ClusterClientUtil.getDOType(doInfo); + switch (doType) { + case DDO: + ClusterClientUtil.verifyDDOInfo(doInfo); + DDOEntity ddoEntity = ddoInfo2DDOEntity(doid, doInfo); + ddoCall(doid, ddoEntity, doipMessage, cb); + break; + case BDO: + case BDRepo: + BDOEntity bdoEntity = bdoInfo2BDOEntity(doid, doInfo); + bdoCall(bdoEntity, doipMessage, cb); + case BCRepo: + throw new IrpClientException("BCRepo should not appear in SmartClusterClient verification"); + default: + LOGGER.error("Wrong doType: " + doType); + throw new IrpClientException("doType cannot be parsed"); + } + } + + public static AuditDoipClient getClientByUrl(String address, String version) { + if (address == null && version == null) return null; + AuditDoipClient doipClientImpl = addressToClient.get(address); + if (doipClientImpl == null) { + // Create the client only if it doesn't exist in the map + doipClientImpl = ConnectionUtil.createWithoutConnect(address, version); + // Use putIfAbsent() to ensure thread-safety + AuditDoipClient existingClient = addressToClient.putIfAbsent(address, doipClientImpl); + if (existingClient != null) { + // If another thread added a client while we were creating ours, use that instead + doipClientImpl = existingClient; + } + } + try { + // Always try to reconnect, regardless of whether the client was just created or already existed + ConnectionUtil.tryReconnectSync(doipClientImpl, 2000); + } catch (Exception e) { + e.printStackTrace(); + } + return doipClientImpl; + } + + public DDOEntity ddoInfo2DDOEntity(String ddoID, JsonObject ddoInfo) throws ScriptException { + DDOEntity ddoEntity = ddoCache.get(ddoID); + if(ddoEntity != null) return ddoEntity; + + String bcoId = ddoInfo.get("bcoId").getAsString(); + JsonObject appendixes = ddoInfo.get("appendixes").getAsJsonObject(); + // 将所有的functions通过engine注册到functionBindings中 + JsonObject bcoAccessRules = appendixes.get(bcoId).getAsJsonObject().get("accessRules").getAsJsonObject(); + Map ddoRouteInfo = ClusterClientUtil.getDDORouteInfo(bcoAccessRules); + Map ddoJoinInfo = ClusterClientUtil.getDDOJoinInfo(bcoAccessRules); + NashornScriptEngineUtil engineUtil = ClusterClientUtil.instantiateNashornScriptEngineUtil(ddoInfo); + ddoCache.putIfAbsent(ddoID, new DDOEntity(ddoRouteInfo, ddoJoinInfo, engineUtil, appendixes)); + return ddoCache.get(ddoID); + } + + public BDOEntity bdoInfo2BDOEntity(String bdoID, JsonObject bdoInfo) throws ScriptException { + String address = bdoInfo.get("address").getAsString(); + String version = bdoInfo.get("version").getAsString(); + BDOEntity bdoEntity = bdoCache.get(bdoID); + if(bdoEntity != null) return bdoEntity; + + bdoCache.putIfAbsent(bdoID, new BDOEntity(address, version)); + return bdoCache.get(bdoID); + } + + public void bdoCall(BDOEntity bdoEntity, DoipMessage doipMessage, DoipMessageCallback cb) { + String address = bdoEntity.getAddress(); + String version = bdoEntity.getVersion(); + AuditDoipClient doipClient = getClientByUrl(address, version); + + assert doipClient != null; + doipMessageSigner.signMessage(doipMessage); + BDODelegateDoipMessageCallback BDODelegateDoipMessageCallback = new BDODelegateDoipMessageCallback(doipClient, cb, doipMessage); + doipClient.sendMessage(doipMessage, BDODelegateDoipMessageCallback); + } + + public void ddoCall(String ddoID, DDOEntity ddoEntity, DoipMessage doipMessage, DoipMessageCallback cb) { + JsonDoipMessage doipParam = JsonDoipMessage.fromDoipMessage(doipMessage); + BasicOperations operation = BasicOperations.getDoOp(doipParam.header.operation); + String[] toCalls = RouteJoinUtil.route(doipParam, routerConfig, ddoEntity.getDDORouteInfo().get(operation.getName()), ddoEntity.getEngineUtil()); + RouteJoinUtil.sendMsgAndJoin(this, ddoEntity.getEngineUtil(), doipMessage, ddoEntity.getDDOJoinInfo().get(operation.getName()), toCalls, cb, ddoEntity.getAppendixes()); + } +} diff --git a/src/main/java/org/bdware/doip/cluster/entity/BDOEntity.java b/src/main/java/org/bdware/doip/cluster/entity/BDOEntity.java new file mode 100644 index 0000000..227371a --- /dev/null +++ b/src/main/java/org/bdware/doip/cluster/entity/BDOEntity.java @@ -0,0 +1,27 @@ +package org.bdware.doip.cluster.entity; + +public class BDOEntity { + private String address; + private String version; + + public String getAddress() { + return address; + } + + public void setAddress(String address) { + this.address = address; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public BDOEntity(String address, String version) { + this.address = address; + this.version = version; + } +} diff --git a/src/main/java/org/bdware/doip/cluster/smart/BDWType.java b/src/main/java/org/bdware/doip/cluster/entity/BDWType.java similarity index 60% rename from src/main/java/org/bdware/doip/cluster/smart/BDWType.java rename to src/main/java/org/bdware/doip/cluster/entity/BDWType.java index ca375a1..309a541 100644 --- a/src/main/java/org/bdware/doip/cluster/smart/BDWType.java +++ b/src/main/java/org/bdware/doip/cluster/entity/BDWType.java @@ -1,4 +1,4 @@ -package org.bdware.doip.cluster.smart; +package org.bdware.doip.cluster.entity; public enum BDWType { DDO, diff --git a/src/main/java/org/bdware/doip/cluster/entity/DDOEntity.java b/src/main/java/org/bdware/doip/cluster/entity/DDOEntity.java index 6806519..547d6fc 100644 --- a/src/main/java/org/bdware/doip/cluster/entity/DDOEntity.java +++ b/src/main/java/org/bdware/doip/cluster/entity/DDOEntity.java @@ -1,5 +1,6 @@ package org.bdware.doip.cluster.entity; +import com.google.gson.JsonObject; import org.bdware.sc.bean.JoinInfo; import org.bdware.sc.bean.RouteInfo; import wrp.jdk.nashorn.api.scripting.NashornScriptEngineUtil; @@ -13,11 +14,21 @@ public class DDOEntity { private Map ddoJoinInfo; // script engine private NashornScriptEngineUtil engineUtil; + private JsonObject appendixes; - public DDOEntity(Map ddoRouteInfo, Map ddoJoinInfo, NashornScriptEngineUtil engineUtil) { + public DDOEntity(Map ddoRouteInfo, Map ddoJoinInfo, NashornScriptEngineUtil engineUtil, JsonObject appendixes) { this.ddoRouteInfo = ddoRouteInfo; this.ddoJoinInfo = ddoJoinInfo; this.engineUtil = engineUtil; + this.appendixes = appendixes; + } + + public JsonObject getAppendixes() { + return appendixes; + } + + public void setAppendixes(JsonObject appendixes) { + this.appendixes = appendixes; } public Map getDDORouteInfo() { diff --git a/src/main/java/org/bdware/doip/cluster/smart/SmartTools.java b/src/main/java/org/bdware/doip/cluster/util/ClusterClientUtil.java similarity index 74% rename from src/main/java/org/bdware/doip/cluster/smart/SmartTools.java rename to src/main/java/org/bdware/doip/cluster/util/ClusterClientUtil.java index e8cadd9..de2c7cc 100644 --- a/src/main/java/org/bdware/doip/cluster/smart/SmartTools.java +++ b/src/main/java/org/bdware/doip/cluster/util/ClusterClientUtil.java @@ -1,26 +1,29 @@ -package org.bdware.doip.cluster.smart; +package org.bdware.doip.cluster.util; import com.google.gson.*; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.bdware.doip.audit.client.AuditIrpClient; +import org.bdware.doip.cluster.entity.BDWType; import org.bdware.irp.exception.IrpClientException; import org.bdware.irp.stateinfo.StateInfoBase; import org.bdware.sc.bean.JoinInfo; import org.bdware.sc.bean.RouteInfo; +import wrp.jdk.nashorn.api.scripting.NashornScriptEngineUtil; +import javax.script.ScriptException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -public class SmartTools { - private static final Logger Logger = LogManager.getLogger(SmartTools.class); +public class ClusterClientUtil { + private static final Logger Logger = LogManager.getLogger(ClusterClientUtil.class); private static final Gson gson = new Gson(); - public static String[] getDOIDClusterInfo(JsonObject BDOClusterInfo) { - JsonArray bdoList = BDOClusterInfo.get("bdoList").getAsJsonArray(); - JsonObject appendixes = BDOClusterInfo.get("appendixes").getAsJsonObject(); + public static String[] getDOIDClusterInfo(JsonObject ddoInfo) { + JsonArray bdoList = ddoInfo.get("bdoList").getAsJsonArray(); + JsonObject appendixes = ddoInfo.get("appendixes").getAsJsonObject(); List subDOIDsList = new ArrayList<>(); for (JsonElement bdoId : bdoList) { String bdo = bdoId.getAsString(); @@ -36,9 +39,29 @@ public class SmartTools { return subDOIDs; } - public static Map getDOIDRouteInfo(JsonObject accessRulesJO) { + + public static NashornScriptEngineUtil instantiateNashornScriptEngineUtil(JsonObject ddoInfo) throws ScriptException { + NashornScriptEngineUtil engineUtil = new NashornScriptEngineUtil(); + String bcoId = ddoInfo.get("bcoId").getAsString(); + JsonObject appendixes = ddoInfo.get("appendixes").getAsJsonObject(); + + // 将所有的functions通过engine注册到functionBindings中 + JsonObject bcoAccessRules = appendixes.get(bcoId).getAsJsonObject().get("accessRules").getAsJsonObject(); + JsonElement functions = bcoAccessRules.get("functions"); + if (functions != null && !functions.isJsonNull() && functions.isJsonObject()) { + JsonObject functionsObj = functions.getAsJsonObject(); + for (String funcName : functionsObj.keySet()) { + String funcScript = functionsObj.get(funcName).getAsString(); + engineUtil.evalFunction(funcScript); + } + } + + return engineUtil; + } + + public static Map getDDORouteInfo(JsonObject bcoAccessRules) { // 解析RouteInfo - JsonElement routeInfos = accessRulesJO.get("routeInfo"); + JsonElement routeInfos = bcoAccessRules.get("routeInfo"); Map res = new HashMap<>(); assert routeInfos != null && !routeInfos.isJsonNull() && routeInfos.isJsonObject(); @@ -54,9 +77,9 @@ public class SmartTools { return res; } - public static Map getDOIDJoinInfo(JsonObject accessRulesJO) { + public static Map getDDOJoinInfo(JsonObject bcoAccessRules) { // 解析JoinInfo - JsonElement joinInfos = accessRulesJO.get("joinInfo"); + JsonElement joinInfos = bcoAccessRules.get("joinInfo"); Map res = new HashMap<>(); assert joinInfos != null && !joinInfos.isJsonNull() && joinInfos.isJsonObject(); diff --git a/src/main/java/org/bdware/doip/cluster/smart/SmartClusterClient.java b/src/main/java/org/bdware/doip/cluster/util/ConnectionUtil.java similarity index 30% rename from src/main/java/org/bdware/doip/cluster/smart/SmartClusterClient.java rename to src/main/java/org/bdware/doip/cluster/util/ConnectionUtil.java index 3bef520..32f1755 100644 --- a/src/main/java/org/bdware/doip/cluster/smart/SmartClusterClient.java +++ b/src/main/java/org/bdware/doip/cluster/util/ConnectionUtil.java @@ -1,116 +1,38 @@ -package org.bdware.doip.cluster.smart; +package org.bdware.doip.cluster.util; -import com.google.gson.Gson; -import com.google.gson.JsonObject; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.bdware.doip.audit.EndpointConfig; import org.bdware.doip.audit.client.AuditDoipClient; -import org.bdware.doip.audit.client.AuditIrpClient; import org.bdware.doip.audit.writer.AuditConfig; import org.bdware.doip.audit.writer.AuditType; -import org.bdware.doip.cluster.entity.DDOEntity; -import org.bdware.doip.codec.doipMessage.DoipMessageSigner; -import org.bdware.doip.encrypt.SM2Signer; -import org.bdware.doip.endpoint.client.DoipClient; -import org.bdware.doip.endpoint.client.DoipClientImpl; -import org.bdware.irp.exception.IrpClientException; -import org.zz.gmhelper.SM2KeyPair; -import org.zz.gmhelper.SM2Util; import java.io.ByteArrayOutputStream; import java.io.PrintStream; -import java.util.Map; import java.util.concurrent.*; -public class SmartClusterClient extends DoipClientImpl { - private final DoipMessageSigner doipMessageSigner; - private final String clientDoid; - // router config -> provide to DoipClusterClient - private final EndpointConfig routerConfig; - // irsClient -> connect to router to resolve doid - private final AuditIrpClient irsClient; - // DDOID -> Contract Method -> RouteInfo - private Map ddoCache = new ConcurrentHashMap<>(); - // DOID -> Connection - private final Map addressToClient = new ConcurrentHashMap<>(); +/** + * @author liuyihao + */ +public class ConnectionUtil { + private static final Logger LOGGER = LogManager.getLogger(ConnectionUtil.class); - private static final Logger LOGGER = LogManager.getLogger(SmartClusterClient.class); - - public SmartClusterClient(String clientDoid, EndpointConfig config) { - this.clientDoid = clientDoid; - this.routerConfig = config; - this.irsClient = new AuditIrpClient(config); - if (routerConfig.publicKey != null && routerConfig.privateKey != null) { - doipMessageSigner = new SM2Signer(SM2KeyPair.fromJson(new Gson().toJson(routerConfig))); - } else - doipMessageSigner = new SM2Signer(SM2Util.generateSM2KeyPair()); - - LOGGER.info("The DoipClusterClient has been initialized"); - } - - public DoipClient smartSelectDoipClient(String doid) throws IrpClientException { - JsonObject doInfo = SmartTools.getDOInfo(irsClient, doid); - BDWType doType = SmartTools.getDOType(doInfo); - switch (doType) { - case DDO: - SmartTools.verifyDDOInfo(doInfo); - System.out.println("Hello world"); - break; - case BDO: - case BDRepo: - String address = doInfo.get("address").getAsString(); - String version = doInfo.get("version").getAsString(); - return getClientByUrl(address, version); - case BCRepo: - System.out.println("Hello world3"); - break; - default: - LOGGER.error("Wrong doType: " + doType); - throw new IrpClientException("doType cannot be parsed"); - } - - return null; - } - - public AuditDoipClient getClientByUrl(String address, String version) { - if (address == null && version == null) return null; - AuditDoipClient doipClientImpl = addressToClient.get(address); - if (doipClientImpl == null) { - // Create the client only if it doesn't exist in the map - doipClientImpl = createWithoutConnect(address, version); - // Use putIfAbsent() to ensure thread-safety - AuditDoipClient existingClient = addressToClient.putIfAbsent(address, doipClientImpl); - if (existingClient != null) { - // If another thread added a client while we were creating ours, use that instead - doipClientImpl = existingClient; - } - } - try { - // Always try to reconnect, regardless of whether the client was just created or already existed - tryReconnectSync(doipClientImpl, 2000); - } catch (Exception e) { - e.printStackTrace(); + private static final ExecutorService connectionTaskPool = Executors.newFixedThreadPool(8, new ThreadFactory() { + @Override + public Thread newThread(Runnable r) { + Thread t = new Thread(r); + t.setDaemon(true); + return t; } - return doipClientImpl; - } + }); - private AuditDoipClient createWithoutConnect(String address, String version) { + public static AuditDoipClient createWithoutConnect(String address, String version) { AuditDoipClient doipClientImpl = new AuditDoipClient(AuditConfig.newInstance(null, AuditType.None, null), null); LOGGER.info("connect to:" + address); doipClientImpl.setRepoUrl(address); return doipClientImpl; } - ExecutorService connectionTaskPool = Executors.newFixedThreadPool(8, new ThreadFactory() { - @Override - public Thread newThread(Runnable r) { - Thread t = new Thread(r); - t.setDaemon(true); - return t; - } - }); - private AuditDoipClient tryReconnectSync(AuditDoipClient doipClientImpl, long timeToWait) { + public static void tryReconnectSync(AuditDoipClient doipClientImpl, long timeToWait) { CompletableFuture future = CompletableFuture.supplyAsync(() -> { try { synchronized (doipClientImpl) { @@ -129,11 +51,10 @@ public class SmartClusterClient extends DoipClientImpl { }, connectionTaskPool); try { - return future.get(timeToWait, TimeUnit.MILLISECONDS); + future.get(timeToWait, TimeUnit.MILLISECONDS); } catch (InterruptedException | ExecutionException | TimeoutException e) { LOGGER.error("============Reconnect Exception " + doipClientImpl.getRepoUrl() + "============", e); future.cancel(true); - return null; } } } diff --git a/src/main/java/org/bdware/doip/cluster/util/ReconnectionUtil.java b/src/main/java/org/bdware/doip/cluster/util/ReconnectionUtil.java new file mode 100644 index 0000000..7bb23f6 --- /dev/null +++ b/src/main/java/org/bdware/doip/cluster/util/ReconnectionUtil.java @@ -0,0 +1,104 @@ +package org.bdware.doip.cluster.util; + +import com.google.gson.JsonObject; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.bdware.doip.audit.client.AuditDoipClient; +import org.bdware.doip.cluster.client.TreeClusterClient; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.atomic.AtomicInteger; + +public class ReconnectionUtil { + public static final Logger LOGGER = LogManager.getLogger(ReconnectionUtil.class); + private final Map reconnectRequests; + private final Map lastReconnectDate; + private final JsonObject appendixes; + private static final ExecutorService connectionTaskPool = Executors.newFixedThreadPool(8, new ThreadFactory() { + @Override + public Thread newThread(Runnable r) { + Thread t = new Thread(r); + t.setDaemon(true); + return t; + } + }); + + public ReconnectionUtil(JsonObject appendixes) { + reconnectRequests = new ConcurrentHashMap<>(); + lastReconnectDate = new ConcurrentHashMap<>(); + this.appendixes = appendixes; + } + + public void popReconnectRequest(String[] targetServers) { + if (targetServers != null) { + for (String str : targetServers) { + if (str == null) continue; + synchronized (reconnectRequests) { + if (!reconnectRequests.containsKey(str)) { + reconnectRequests.put(str, new AtomicInteger(0)); + } + } + AtomicInteger i = reconnectRequests.get(str); + if (i.decrementAndGet() < 0) { + i.set(0); + } + } + } + } + + public void pushReconnectRequest(String[] targetServers) { + if (targetServers != null) { + for (String str : targetServers) { + if (str == null) continue; + synchronized (reconnectRequests) { + if (!reconnectRequests.containsKey(str)) { + reconnectRequests.put(str, new AtomicInteger(0)); + } + } + AtomicInteger i = reconnectRequests.get(str); + LOGGER.info("===== pushReconnectRequest add:" + str + " --> " + i.get()); + if (i.incrementAndGet() > 10) { + connectionTaskPool.submit(new Runnable() { + @Override + public void run() { + triggerReconnect(str); + } + }); + } + } + } + } + + public void triggerReconnect(String doid) { + try { + String address = appendixes.get(doid).getAsJsonObject().get("address").getAsString(); + String version = appendixes.get(doid).getAsJsonObject().get("version").getAsString(); + AuditDoipClient doipClient = TreeClusterClient.getClientByUrl(address, null); + LOGGER.info("==========TRIGGER Reconnect:" + doid + " address:" + address); + Long date = lastReconnectDate.get(doid); + if (date == null || System.currentTimeMillis() - date > 10000) { + lastReconnectDate.put(doid, System.currentTimeMillis()); + reconnectRequests.get(doid).set(0); + } else { + LOGGER.info("==========Reject Reconnect:" + doid + " " + address + ", try too fast!"); + return; + } + if (doipClient != null) { + String oldAddress = doipClient.getRepoUrl(); + doipClient.disconnectAndSet(oldAddress); + ConnectionUtil.tryReconnectSync(doipClient, 10000L); + } else { + doipClient = TreeClusterClient.getClientByUrl(address, version); + String oldAddress = doipClient.getRepoUrl(); + doipClient.disconnectAndSet(oldAddress); + ConnectionUtil.tryReconnectSync(doipClient, 10000L); + } + } catch (Exception e) { + e.printStackTrace(); + } + } +} diff --git a/src/main/java/org/bdware/doip/cluster/util/ResultCollector.java b/src/main/java/org/bdware/doip/cluster/util/ResultCollector.java index ffa5bc3..48a6381 100644 --- a/src/main/java/org/bdware/doip/cluster/util/ResultCollector.java +++ b/src/main/java/org/bdware/doip/cluster/util/ResultCollector.java @@ -173,15 +173,16 @@ public class ResultCollector implements DoipMessageCallback { case add: double val = 0; for (DoipMessage msg : successResMap.values()) { - val += Double.valueOf(msg.body.getDataAsJsonString()); + val += Double.parseDouble(msg.body.getDataAsJsonString()); returnedMsg = msg; } + assert returnedMsg != null; returnedMsg.body.encodedData = (val + "").getBytes(); return returnedMsg; case multiply: val = 1; for (DoipMessage msg : successResMap.values()) { - val *= Double.valueOf(msg.body.getDataAsJsonString()); + val *= Double.parseDouble(msg.body.getDataAsJsonString()); returnedMsg = msg; } return returnedMsg; diff --git a/src/main/java/org/bdware/doip/cluster/util/RouteJoinUtil.java b/src/main/java/org/bdware/doip/cluster/util/RouteJoinUtil.java new file mode 100644 index 0000000..22e8a79 --- /dev/null +++ b/src/main/java/org/bdware/doip/cluster/util/RouteJoinUtil.java @@ -0,0 +1,133 @@ +package org.bdware.doip.cluster.util; + +import com.google.gson.JsonElement; +import com.google.gson.JsonNull; +import com.google.gson.JsonObject; +import com.google.gson.JsonPrimitive; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.bdware.doip.audit.EndpointConfig; +import org.bdware.doip.cluster.callback.DDODelegateDoipMessageCallback; +import org.bdware.doip.cluster.client.DoipClusterClient; +import org.bdware.doip.cluster.client.TreeClusterClient; +import org.bdware.doip.codec.JsonDoipMessage; +import org.bdware.doip.codec.doipMessage.DoipMessage; +import org.bdware.doip.endpoint.client.DoipMessageCallback; +import org.bdware.sc.bean.JoinInfo; +import org.bdware.sc.bean.RouteInfo; +import org.bdware.sc.util.JsonUtil; +import wrp.jdk.nashorn.api.scripting.NashornScriptEngineUtil; + +import java.math.BigInteger; + +public class RouteJoinUtil { + public static final Logger LOGGER = LogManager.getLogger(DoipClusterClient.class); + + public static String[] route(JsonDoipMessage doipParam, EndpointConfig routerConfig, RouteInfo routeInfo, NashornScriptEngineUtil engineUtil) { + try { + // If the operation truly has routeInfo + String[] BDRepoIDs = engineUtil.doipClusterUtil.doipServers; + if (routeInfo != null) { + if (routeInfo.useDefault == null) { + JsonElement requester = routerConfig.publicKey == null ? JsonNull.INSTANCE : new JsonPrimitive(routerConfig.publicKey); + return engineUtil.invokeFunction(routeInfo.funcName, String[].class, JsonUtil.parseObject(doipParam), requester); + } + + switch (routeInfo.useDefault) { + case byRequester: + int val = new BigInteger(routerConfig.publicKey, 16).mod(new BigInteger("" + BDRepoIDs.length)).intValue(); + while (val < 0 && BDRepoIDs.length > 0) { + val = val + BDRepoIDs.length; + } + return new String[]{BDRepoIDs[val]}; + case byArgHash: + val = JsonUtil.toJson(doipParam).hashCode(); + val = val % BDRepoIDs.length; + while (val < 0 && BDRepoIDs.length > 0) { + val += BDRepoIDs.length; + } + return new String[]{BDRepoIDs[val]}; + case byJsonPropHash: + JsonElement jo = tryLoadJsonProp(JsonUtil.parseObject(doipParam).getAsJsonObject(), routeInfo.param); + val = jo.toString().hashCode() % BDRepoIDs.length; + while (val < 0 && BDRepoIDs.length > 0) { + val += BDRepoIDs.length; + } + return new String[]{BDRepoIDs[val]}; + default: + return BDRepoIDs; + } + + // 如果没有RouteInfo，默认随机选择一台机器发送 + } else { + LOGGER.warn("Not RouteInfo in the function"); + return BDRepoIDs.length != 0 ? new String[]{BDRepoIDs[(int) (Math.random() * BDRepoIDs.length)]} : BDRepoIDs; + } + } catch (Exception e) { + e.printStackTrace(); + } + + return null; + } + + private static JsonElement tryLoadJsonProp(JsonObject routeInfoArg, String param) { + try { + if (routeInfoArg == null) + return JsonNull.INSTANCE; + JsonObject arg; + if (routeInfoArg.isJsonPrimitive()) { + arg = JsonUtil.parseString(routeInfoArg.getAsString()).getAsJsonObject(); + } else arg = routeInfoArg.getAsJsonObject(); + if (!param.contains(".")) { + return arg.get(param); + } else { + String[] props = param.split("\\."); + JsonElement result = arg; + for (String str : props) + result = result.getAsJsonObject().get(str); + return result; + } + } catch (Exception e) { + e.printStackTrace(); + return JsonNull.INSTANCE; + } + } + + public static void sendMsgAndJoin(TreeClusterClient treeClusterClient, NashornScriptEngineUtil engineUtil, DoipMessage msg, JoinInfo joinInfo, String[] toCalls, DoipMessageCallback cb, JsonObject appendixes) { + int serversNum = toCalls.length; + int count = getJoinCount(treeClusterClient.routerConfig, engineUtil, joinInfo, serversNum, new JsonObject()); + // 如果joinCountFunc执行算出的joinCount，比所有的servers还要多，就更新为servers的数量 + if (serversNum < count) { + count = serversNum; + } + + if (count > 0) { + DDODelegateDoipMessageCallback resultCollector = new DDODelegateDoipMessageCallback(treeClusterClient, toCalls, msg, cb, count, engineUtil, joinInfo, appendixes); + for (String toCall : toCalls) { + // todo 对于DDO嵌套 -> appendix可以只放bdwtype + // todo 消息嵌套？同一个消息发给多个Server？每个消息改一个requestId = 0 & DOID -> DDOID/BDOID，并且发给多个Server？ + // TODO TODO TODO + // DoipMessage toSend = msg.deepCopy(); + // 要么response wait中的waitObj改为非static，并加入一点流控制机制。 + // 这里的doipClient为BDRepoClient或者BDOClient，无所谓，如果是BDRepoClient，DelegateDoipMessageCallback会出手转为BDOClient + treeClusterClient.sendMessage(msg, toCall, resultCollector); + } + } else { + LOGGER.error("joinCount is not a positive num"); + } + } + + protected static int getJoinCount(EndpointConfig routerConfig, NashornScriptEngineUtil engineUtil, JoinInfo joinInfo, int serversNum, JsonObject joinParams) { + try { + if (joinInfo == null) return serversNum; + if (joinInfo.joinCountFuncName != null) { + JsonElement requester = routerConfig.publicKey == null ? JsonNull.INSTANCE : new JsonPrimitive(routerConfig.publicKey); + return engineUtil.invokeFunction(joinInfo.joinCountFuncName, Integer.class, joinParams, requester); + } + if (joinInfo.joinCount != 0) return joinInfo.joinCount; + } catch (Exception e) { + e.printStackTrace(); + } + return serversNum; + } +} -- Gitee From 3b9c121ddebd6d7a0a119c7946d5eb4aee41d8a2 Mon Sep 17 00:00:00 2001 From: "haoeliu@foxmail.com" Date: Mon, 10 Jul 2023 23:37:42 +0800 Subject: [PATCH 05/14] update ddoCall --- .../bdware/doip/cluster/client/TreeClusterClient.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/bdware/doip/cluster/client/TreeClusterClient.java b/src/main/java/org/bdware/doip/cluster/client/TreeClusterClient.java index c66b506..3a6bfaa 100644 --- a/src/main/java/org/bdware/doip/cluster/client/TreeClusterClient.java +++ b/src/main/java/org/bdware/doip/cluster/client/TreeClusterClient.java @@ -34,6 +34,7 @@ import javax.script.ScriptException; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +// todo add flow control public class TreeClusterClient extends DoipClientImpl { public final DoipMessageSigner doipMessageSigner; // router config -> provide to DoipClusterClient @@ -90,7 +91,7 @@ public class TreeClusterClient extends DoipClientImpl { case DDO: ClusterClientUtil.verifyDDOInfo(doInfo); DDOEntity ddoEntity = ddoInfo2DDOEntity(doid, doInfo); - ddoCall(doid, ddoEntity, doipMessage, cb); + ddoCall(ddoEntity, doipMessage, cb); break; case BDO: case BDRepo: @@ -142,11 +143,11 @@ public class TreeClusterClient extends DoipClientImpl { } public BDOEntity bdoInfo2BDOEntity(String bdoID, JsonObject bdoInfo) throws ScriptException { - String address = bdoInfo.get("address").getAsString(); - String version = bdoInfo.get("version").getAsString(); BDOEntity bdoEntity = bdoCache.get(bdoID); if(bdoEntity != null) return bdoEntity; + String address = bdoInfo.get("address").getAsString(); + String version = bdoInfo.get("version").getAsString(); bdoCache.putIfAbsent(bdoID, new BDOEntity(address, version)); return bdoCache.get(bdoID); } @@ -162,7 +163,7 @@ public class TreeClusterClient extends DoipClientImpl { doipClient.sendMessage(doipMessage, BDODelegateDoipMessageCallback); } - public void ddoCall(String ddoID, DDOEntity ddoEntity, DoipMessage doipMessage, DoipMessageCallback cb) { + public void ddoCall(DDOEntity ddoEntity, DoipMessage doipMessage, DoipMessageCallback cb) { JsonDoipMessage doipParam = JsonDoipMessage.fromDoipMessage(doipMessage); BasicOperations operation = BasicOperations.getDoOp(doipParam.header.operation); String[] toCalls = RouteJoinUtil.route(doipParam, routerConfig, ddoEntity.getDDORouteInfo().get(operation.getName()), ddoEntity.getEngineUtil()); -- Gitee From bd71b97756f7a9da9d182596a703f8d77b8bbac6 Mon Sep 17 00:00:00 2001 From: "haoeliu@foxmail.com" Date: Tue, 11 Jul 2023 23:04:44 +0800 Subject: [PATCH 06/14] refactor doipclusterclient and treeclusterclient --- .../bdware/doip/cluster/ClusterDoaClient.java | 5 +- .../BDODelegateDoipMessageCallback.java | 5 +- ... => ClusterClientDoipMessageCallback.java} | 6 +- .../DDODelegateDoipMessageCallback.java | 27 +- .../bdware/doip/cluster/client/DOAClient.java | 2 +- .../cluster/client/DoipClusterClient.java | 239 ++---------------- .../cluster/client/TreeClusterClient.java | 41 +-- .../doip/cluster/flowcontrol/RequestPack.java | 19 ++ .../{ => core}/RateThresholdFlowControl.java | 4 +- .../{ => core}/SlidingWindowFlowControl.java | 4 +- .../{ => core}/TokenBucketFlowControl.java | 4 +- ...rClientUtil.java => ClientCommonUtil.java} | 41 ++- .../doip/cluster/util/ConnectionUtil.java | 83 +++++- .../doip/cluster/util/ReconnectionUtil.java | 104 -------- .../doip/cluster/util/ResultCollector.java | 9 +- .../doip/cluster/util/RouteJoinUtil.java | 18 +- 16 files changed, 210 insertions(+), 401 deletions(-) rename src/main/java/org/bdware/doip/cluster/callback/{DelegateDoipMessageCallback.java => ClusterClientDoipMessageCallback.java} (87%) create mode 100644 src/main/java/org/bdware/doip/cluster/flowcontrol/RequestPack.java rename src/main/java/org/bdware/doip/cluster/flowcontrol/{ => core}/RateThresholdFlowControl.java (91%) rename src/main/java/org/bdware/doip/cluster/flowcontrol/{ => core}/SlidingWindowFlowControl.java (93%) rename src/main/java/org/bdware/doip/cluster/flowcontrol/{ => core}/TokenBucketFlowControl.java (93%) rename src/main/java/org/bdware/doip/cluster/util/{ClusterClientUtil.java => ClientCommonUtil.java} (81%) delete mode 100644 src/main/java/org/bdware/doip/cluster/util/ReconnectionUtil.java diff --git a/src/main/java/org/bdware/doip/cluster/ClusterDoaClient.java b/src/main/java/org/bdware/doip/cluster/ClusterDoaClient.java index 9bb5f42..08638d1 100644 --- a/src/main/java/org/bdware/doip/cluster/ClusterDoaClient.java +++ b/src/main/java/org/bdware/doip/cluster/ClusterDoaClient.java @@ -4,6 +4,7 @@ import com.google.gson.JsonObject; import org.bdware.doip.audit.EndpointConfig; import org.bdware.doip.audit.client.AuditIrpClient; import org.bdware.doip.cluster.client.DoipClusterClient; +import org.bdware.doip.cluster.util.RouteJoinUtil; import org.bdware.doip.codec.JsonDoipMessage; import org.bdware.doip.codec.digitalObject.DigitalObject; import org.bdware.doip.codec.doipMessage.DoipMessage; @@ -104,6 +105,7 @@ public class ClusterDoaClient extends DoipClientImpl { JsonDoipMessage message = JsonDoipMessage.fromDoipMessage(doipMessage); String[] BDRepoIDs = route(message); // joinParams都来自于callback，joinParams就是所有节点callback到达之后，merge之后的结果 + // todo optimize getJoinInfo JoinInfo joinInfo = getJoinInfo(op); sendMsgAndJoin(doipMessage, joinInfo, BDRepoIDs, cb); @@ -112,7 +114,8 @@ public class ClusterDoaClient extends DoipClientImpl { @Override public void sendMsgAndJoin(DoipMessage msg, JoinInfo joinInfo, String[] BDRepoIDs, DoipMessageCallback cb) { int serversNum = BDRepoIDs.length; - int count = this.getJoinCount(joinInfo, serversNum, new JsonObject()); + // todo optimize getJoinInfo + int count = RouteJoinUtil.getJoinCount(null, joinInfo, serversNum, new JsonObject()); // 如果joinCountFunc执行算出的joinCount，比所有的servers还要多，就更新为servers的数量 if (serversNum < count) { count = serversNum; diff --git a/src/main/java/org/bdware/doip/cluster/callback/BDODelegateDoipMessageCallback.java b/src/main/java/org/bdware/doip/cluster/callback/BDODelegateDoipMessageCallback.java index a23093a..6ff44bc 100644 --- a/src/main/java/org/bdware/doip/cluster/callback/BDODelegateDoipMessageCallback.java +++ b/src/main/java/org/bdware/doip/cluster/callback/BDODelegateDoipMessageCallback.java @@ -4,6 +4,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.bdware.doip.audit.client.AuditDoipClient; import org.bdware.doip.cluster.client.TreeClusterClient; +import org.bdware.doip.cluster.util.ClientCommonUtil; import org.bdware.doip.codec.doipMessage.DoipMessage; import org.bdware.doip.codec.doipMessage.DoipResponseCode; import org.bdware.doip.endpoint.client.DoipMessageCallback; @@ -11,7 +12,7 @@ import org.bdware.doip.endpoint.client.DoipMessageCallback; import java.net.URI; public class BDODelegateDoipMessageCallback implements DoipMessageCallback { - static Logger LOGGER = LogManager.getLogger(DelegateDoipMessageCallback.class); + static Logger LOGGER = LogManager.getLogger(ClusterClientDoipMessageCallback.class); private final DoipMessageCallback originalCallback; DoipMessage originalMessage; AuditDoipClient previousClient; @@ -39,7 +40,7 @@ public class BDODelegateDoipMessageCallback implements DoipMessageCallback { version = msg.header.parameters.attributes.get("version").getAsString(); LOGGER.info("recv delegate:"+newAddress +" previousRepo:"+previousClient.getRepoUrl()); // BDRepo doipMsgCallback -> BDOClient，维护repoToBDO的映射关系 - AuditDoipClient client = TreeClusterClient.getClientByUrl(newAddress, version); + AuditDoipClient client = ClientCommonUtil.getClientByUrl(newAddress, version); previousClient = client; client.sendRawMessage(originalMessage, this); } diff --git a/src/main/java/org/bdware/doip/cluster/callback/DelegateDoipMessageCallback.java b/src/main/java/org/bdware/doip/cluster/callback/ClusterClientDoipMessageCallback.java similarity index 87% rename from src/main/java/org/bdware/doip/cluster/callback/DelegateDoipMessageCallback.java rename to src/main/java/org/bdware/doip/cluster/callback/ClusterClientDoipMessageCallback.java index a68328b..656972f 100644 --- a/src/main/java/org/bdware/doip/cluster/callback/DelegateDoipMessageCallback.java +++ b/src/main/java/org/bdware/doip/cluster/callback/ClusterClientDoipMessageCallback.java @@ -10,14 +10,14 @@ import org.bdware.doip.endpoint.client.DoipMessageCallback; import java.net.URI; -public class DelegateDoipMessageCallback implements DoipMessageCallback { - static Logger LOGGER = LogManager.getLogger(DelegateDoipMessageCallback.class); +public class ClusterClientDoipMessageCallback implements DoipMessageCallback { + static Logger LOGGER = LogManager.getLogger(ClusterClientDoipMessageCallback.class); private final DoipMessageCallback originalCallback; DoipMessage originalMessage; AuditDoipClient previousClient; DoipClusterClient doipClusterClient; - public DelegateDoipMessageCallback(AuditDoipClient previousClient, DoipClusterClient doipClusterClient, DoipMessageCallback doipMessageCallback, DoipMessage originalMessage) { + public ClusterClientDoipMessageCallback(AuditDoipClient previousClient, DoipClusterClient doipClusterClient, DoipMessageCallback doipMessageCallback, DoipMessage originalMessage) { this.previousClient = previousClient; this.originalCallback = doipMessageCallback; this.originalMessage = originalMessage; diff --git a/src/main/java/org/bdware/doip/cluster/callback/DDODelegateDoipMessageCallback.java b/src/main/java/org/bdware/doip/cluster/callback/DDODelegateDoipMessageCallback.java index b3a1c59..9aa4b68 100644 --- a/src/main/java/org/bdware/doip/cluster/callback/DDODelegateDoipMessageCallback.java +++ b/src/main/java/org/bdware/doip/cluster/callback/DDODelegateDoipMessageCallback.java @@ -6,7 +6,7 @@ import com.google.gson.JsonElement; import com.google.gson.JsonObject; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.bdware.doip.cluster.util.ReconnectionUtil; +import org.bdware.doip.cluster.util.ConnectionUtil; import org.bdware.doip.cluster.client.TreeClusterClient; import org.bdware.doip.cluster.util.ResultCollector; import org.bdware.doip.codec.JsonDoipMessage; @@ -42,7 +42,7 @@ public class DDODelegateDoipMessageCallback implements DoipMessageCallback { JoinInfo joinInfo; String[] targetServers; TreeClusterClient treeClusterClient; - ReconnectionUtil reconnectionUtil; + JsonObject appendixes; public DDODelegateDoipMessageCallback(TreeClusterClient treeClusterClient, String[] targetServers, DoipMessage msg, DoipMessageCallback cb, int count, NashornScriptEngineUtil engineUtil, JoinInfo joinInfo, JsonObject appendixes) { @@ -53,7 +53,7 @@ public class DDODelegateDoipMessageCallback implements DoipMessageCallback { this.joinInfo = joinInfo; this.targetServers = targetServers; this.treeClusterClient = treeClusterClient; - reconnectionUtil = new ReconnectionUtil(appendixes); + this.appendixes = appendixes; } @Override @@ -83,32 +83,20 @@ public class DDODelegateDoipMessageCallback implements DoipMessageCallback { } } - public DoipMessageCallback getOriginalCallback() { - return originalCallback; - } - public void logDoipMsgInfo(DoipMessage msg) { try { String nodeID = getNodeID(msg); if (nodeID != null) { - popReconnectRequest(); + ConnectionUtil.popReconnectRequest(targetServers); } else { Logger.error("收到错误的节点回复 : " + "Something wrong happens in" + new Gson().toJson(targetServers) + " msg:" + new Gson().toJson(msg.header.parameters)); - pushReconnectRequest(); + ConnectionUtil.pushReconnectRequest(targetServers, appendixes); } } catch (Exception e) { e.printStackTrace(); } } - private void popReconnectRequest() { - reconnectionUtil.popReconnectRequest(targetServers); - } - - private void pushReconnectRequest() { - reconnectionUtil.pushReconnectRequest(targetServers); - } - public synchronized void addDoipResult(String nodeID, DoipMessage msg) { successResMap.put(nodeID, msg); } @@ -177,15 +165,16 @@ public class DDODelegateDoipMessageCallback implements DoipMessageCallback { case add: double val = 0; for (DoipMessage msg : successResMap.values()) { - val += Double.valueOf(msg.body.getDataAsJsonString()); + val += Double.parseDouble(msg.body.getDataAsJsonString()); returnedMsg = msg; } + assert returnedMsg != null; returnedMsg.body.encodedData = (val + "").getBytes(); return returnedMsg; case multiply: val = 1; for (DoipMessage msg : successResMap.values()) { - val *= Double.valueOf(msg.body.getDataAsJsonString()); + val *= Double.parseDouble(msg.body.getDataAsJsonString()); returnedMsg = msg; } return returnedMsg; diff --git a/src/main/java/org/bdware/doip/cluster/client/DOAClient.java b/src/main/java/org/bdware/doip/cluster/client/DOAClient.java index 32016e1..c53b4b5 100644 --- a/src/main/java/org/bdware/doip/cluster/client/DOAClient.java +++ b/src/main/java/org/bdware/doip/cluster/client/DOAClient.java @@ -369,7 +369,7 @@ public class DOAClient extends DoipClientImpl { if (count > 0) { //TODO//TODO//TODO//TODO//TODO//TODO//TODO//TODO//TODO - ResultCollector resultCollector = new ResultCollector(null, null, null, cb, count, engineUtil, joinInfo); + ResultCollector resultCollector = new ResultCollector(null, null, null, cb, count, engineUtil, joinInfo, null); for (String targetDoipServer : targetDoipServers) { DoipClientImpl doipClientImpl = getDoipClientForServer(targetDoipServer); switch (doipParam.operation) { diff --git a/src/main/java/org/bdware/doip/cluster/client/DoipClusterClient.java b/src/main/java/org/bdware/doip/cluster/client/DoipClusterClient.java index d7e43fc..0cb15cf 100644 --- a/src/main/java/org/bdware/doip/cluster/client/DoipClusterClient.java +++ b/src/main/java/org/bdware/doip/cluster/client/DoipClusterClient.java @@ -6,11 +6,13 @@ import org.apache.logging.log4j.Logger; import org.bdware.doip.audit.EndpointConfig; import org.bdware.doip.audit.client.AuditDoipClient; import org.bdware.doip.audit.client.AuditIrpClient; -import org.bdware.doip.audit.writer.AuditConfig; -import org.bdware.doip.audit.writer.AuditType; +import org.bdware.doip.cluster.callback.ClusterClientDoipMessageCallback; import org.bdware.doip.cluster.flowcontrol.FlowControl; -import org.bdware.doip.cluster.flowcontrol.RateThresholdFlowControl; +import org.bdware.doip.cluster.flowcontrol.RequestPack; +import org.bdware.doip.cluster.flowcontrol.core.RateThresholdFlowControl; +import org.bdware.doip.cluster.util.ClientCommonUtil; import org.bdware.doip.cluster.util.ResultCollector; +import org.bdware.doip.cluster.util.RouteJoinUtil; import org.bdware.doip.cluster.util.RouterTool; import org.bdware.doip.codec.JsonDoipMessage; import org.bdware.doip.codec.doipMessage.DoipMessage; @@ -31,14 +33,11 @@ import org.zz.gmhelper.SM2Util; import wrp.jdk.nashorn.api.scripting.NashornScriptEngineUtil; import javax.script.ScriptException; -import java.io.ByteArrayOutputStream; -import java.io.PrintStream; import java.math.BigInteger; import java.util.HashMap; import java.util.Map; import java.util.Queue; import java.util.concurrent.*; -import java.util.concurrent.atomic.AtomicInteger; //Client for single bdo running in cluster model. @@ -53,33 +52,13 @@ public class DoipClusterClient extends DoipClientImpl { // all the doipOperation and joinInfo binding relationships private final Map doipOperationToJoinInfo; // 维护了地址 与 doipPort之间的映射关系 - private static Map addressToClient; // BDO和DoipClusterClient是一一对应的关系，一个BDO对应一组BDRepo，一个BDRepo对应了一个DoipServer端口，因此这里维护Repo -> 端口的映射 private final Map bdRepoToBDOClient; - private final Map lastReconnectDate; - private final Map reconnectRequests; //Queue length based control. private final static Queue sendMessageTaskQueue = new ConcurrentLinkedDeque<>(); - private FlowControl flowControl; + private final FlowControl flowControl; // todo optimize thread pool -// int corePoolSize = 8; // 核心线程数 -// int maxPoolSize = 16; // 最大线程数 -// int queueCapacity = 100; // 等待队列长度 -// ExecutorService sendMessagePool = new ThreadPoolExecutor(corePoolSize, -// maxPoolSize, -// 0L, -// TimeUnit.MILLISECONDS, -// new ArrayBlockingQueue<>(queueCapacity) -// ); -// -// ExecutorService connectionTaskPool = new ThreadPoolExecutor(corePoolSize, -// maxPoolSize, -// 0L, -// TimeUnit.MILLISECONDS, -// new ArrayBlockingQueue<>(queueCapacity) -// ); - ExecutorService sendMessagePool = Executors.newFixedThreadPool(8, new ThreadFactory() { @Override public Thread newThread(Runnable r) { @@ -89,28 +68,6 @@ public class DoipClusterClient extends DoipClientImpl { } }); - ExecutorService connectionTaskPool = Executors.newFixedThreadPool(8, new ThreadFactory() { - @Override - public Thread newThread(Runnable r) { - Thread t = new Thread(r); - t.setDaemon(true); - return t; - } - }); - - static class RequestPack { - private final DoipMessage message; - private final JoinInfo joinInfo; - private final String[] bdRepoIds; - private final DoipMessageCallback cb; - - public RequestPack(DoipMessage message, JoinInfo joinInfo, String[] bdRepoIDs, DoipMessageCallback cb) { - this.message = message; - this.joinInfo = joinInfo; - this.bdRepoIds = bdRepoIDs; - this.cb = cb; - } - } Logger Logger = LogManager.getLogger(DoipClusterClient.class); private final RouterTool routerTool; @@ -142,10 +99,7 @@ public class DoipClusterClient extends DoipClientImpl { engineUtil = new NashornScriptEngineUtil(); doipOperationToRouteInfo = new HashMap<>(); doipOperationToJoinInfo = new HashMap<>(); - addressToClient = new ConcurrentHashMap<>(); bdRepoToBDOClient = new ConcurrentHashMap<>(); - lastReconnectDate = new ConcurrentHashMap<>(); - reconnectRequests = new ConcurrentHashMap<>(); routerTool = new RouterTool(irsClient, engineUtil); // todo optimize flowControl // flowControl = new TokenBucket(10240, 2048); @@ -158,7 +112,7 @@ public class DoipClusterClient extends DoipClientImpl { public void parseDOID(String doid) { - for (int retry = 0; retry < 5; retry++) + for (int retry = 0; retry < 5; retry++) { try { // 如果已经解析过了，就不需要再次进行解析了 synchronized (this) { @@ -186,9 +140,9 @@ public class DoipClusterClient extends DoipClientImpl { throw new RuntimeException(ex); } } + } } - @Override public void sendMessage(DoipMessage doipMessage, DoipMessageCallback cb) { try { @@ -196,7 +150,7 @@ public class DoipClusterClient extends DoipClientImpl { JsonDoipMessage message = JsonDoipMessage.fromDoipMessage(doipMessage); String[] BDRepoIDs = route(message); // joinParams都来自于callback，joinParams就是所有节点callback到达之后，merge之后的结果 - JoinInfo joinInfo = getJoinInfo(op); + JoinInfo joinInfo = doipOperationToJoinInfo.get(op.getName()); RequestPack pack = new RequestPack(doipMessage, joinInfo, BDRepoIDs, new DoipMessageCallback() { @Override public void onResult(DoipMessage msg) { @@ -273,7 +227,7 @@ public class DoipClusterClient extends DoipClientImpl { if (existingConn != null) { return existingConn; } - AuditDoipClient newGetClient = getClientByUrl(address, version); + AuditDoipClient newGetClient = ClientCommonUtil.getClientByUrl(address, version); if (newGetClient == null) { return null; } @@ -281,112 +235,6 @@ public class DoipClusterClient extends DoipClientImpl { return existingConn == null ? newGetClient : existingConn; } - public AuditDoipClient getClientByUrl(String address, String version) { - if (address == null && version == null) return null; - AuditDoipClient doipClientImpl = addressToClient.get(address); - if (doipClientImpl == null) { - // Create the client only if it doesn't exist in the map - doipClientImpl = createWithoutConnect(address, version); - // Use putIfAbsent() to ensure thread-safety - AuditDoipClient existingClient = addressToClient.putIfAbsent(address, doipClientImpl); - if (existingClient != null) { - // If another thread added a client while we were creating ours, use that instead - doipClientImpl = existingClient; - } - } - try { - // Always try to reconnect, regardless of whether the client was just created or already existed - tryReconnectSync(doipClientImpl, 2000); - } catch (Exception e) { - e.printStackTrace(); - } - return doipClientImpl; - } - - public void pushReconnectRequest(String[] targetServers) { - if (targetServers != null) { - for (String str : targetServers) { - if (str == null) continue; - AtomicInteger i = reconnectRequests.computeIfAbsent(str, k -> new AtomicInteger(0)); - LOGGER.info("===== pushReconnectRequest add:" + str + " --> " + i.get()); - if (i.incrementAndGet() > 10) { - connectionTaskPool.submit(() -> triggerReconnect(str)); - } - } - } - } - - public void popReconnectRequest(String[] targetServers) { - if (targetServers != null) { - for (String str : targetServers) { - if (str == null) continue; - AtomicInteger i = reconnectRequests.computeIfAbsent(str, k -> new AtomicInteger(0)); - if (i.decrementAndGet() < 0) { - i.set(0); - } - } - } - } - - public void triggerReconnect(String doid) { - try { - String address = appendixes.get(doid).getAsJsonObject().get("address").getAsString(); - String version = appendixes.get(doid).getAsJsonObject().get("version").getAsString(); - AuditDoipClient doipClient = getFastClientByUrl(address, null, null); - LOGGER.info("==========TRIGGER Reconnect:" + doid + " address:" + address); - Long date = lastReconnectDate.get(doid); - if (date != null && System.currentTimeMillis() - date <= 10000) { - LOGGER.info("==========Reject Reconnect:" + doid + " " + address + ", try too fast!"); - return; - } - lastReconnectDate.put(doid, System.currentTimeMillis()); - reconnectRequests.get(doid).set(0); - - if (doipClient == null) { - doipClient = getClientByUrl(address, version); - } - String oldAddress = doipClient.getRepoUrl(); - doipClient.disconnectAndSet(oldAddress); - tryReconnectSync(doipClient, 10000L); - } catch (Exception e) { - e.printStackTrace(); - } - } - - private AuditDoipClient tryReconnectSync(AuditDoipClient doipClientImpl, long timeToWait) { - CompletableFuture future = CompletableFuture.supplyAsync(() -> { - try { - synchronized (doipClientImpl) { - if (!doipClientImpl.isConnected()) { - // LOGGER.info("=========ACTUALLY try reconnect to:" + doipClientImpl.getRepoUrl()); - doipClientImpl.reconnect(); - } - } - return doipClientImpl; - } catch (Exception e) { - ByteArrayOutputStream bo = new ByteArrayOutputStream(); - e.printStackTrace(new PrintStream(bo)); - LOGGER.info("failed to connect to:" + doipClientImpl.getRepoUrl() + " " + bo); - return null; - } - }, connectionTaskPool); - - try { - return future.get(timeToWait, TimeUnit.MILLISECONDS); - } catch (InterruptedException | ExecutionException | TimeoutException e) { - LOGGER.error("============Reconnect Exception " + doipClientImpl.getRepoUrl() + "============", e); - future.cancel(true); - return null; - } - } - - private AuditDoipClient createWithoutConnect(String address, String version) { - AuditDoipClient doipClientImpl = new AuditDoipClient(AuditConfig.newInstance(null, AuditType.None, null), null); - // LOGGER.info("connect to:" + address); - doipClientImpl.setRepoUrl(address); - return doipClientImpl; - } - static Logger LOGGER = LogManager.getLogger(DoipClusterClient.class); // 根据请求参数，执行RouteInfo，寻找DoipServers @@ -414,14 +262,14 @@ public class DoipClusterClient extends DoipClientImpl { case byArgHash: val = JsonUtil.toJson(doipParam).hashCode(); val = val % BDRepoIDs.length; - while (val < 0 && BDRepoIDs.length > 0) { + while (val < 0) { val += BDRepoIDs.length; } return new String[]{BDRepoIDs[val]}; case byJsonPropHash: - JsonElement jo = tryLoadJsonProp(JsonUtil.parseObject(doipParam).getAsJsonObject(), routeInfo.param); + JsonElement jo = RouteJoinUtil.tryLoadJsonProp(JsonUtil.parseObject(doipParam).getAsJsonObject(), routeInfo.param); val = jo.toString().hashCode() % BDRepoIDs.length; - while (val < 0 && BDRepoIDs.length > 0) { + while (val < 0) { val += BDRepoIDs.length; } return new String[]{BDRepoIDs[val]}; @@ -441,64 +289,24 @@ public class DoipClusterClient extends DoipClientImpl { return null; } - private JsonElement tryLoadJsonProp(JsonObject routeInfoArg, String param) { - try { - if (routeInfoArg == null) - return JsonNull.INSTANCE; - JsonObject arg; - if (routeInfoArg.isJsonPrimitive()) { - arg = JsonUtil.parseString(routeInfoArg.getAsString()).getAsJsonObject(); - } else arg = routeInfoArg.getAsJsonObject(); - if (!param.contains(".")) { - return arg.get(param); - } else { - String[] props = param.split("\\."); - JsonElement result = arg; - for (String str : props) - result = result.getAsJsonObject().get(str); - return result; - } - } catch (Exception e) { - e.printStackTrace(); - return JsonNull.INSTANCE; - } - } - - protected int getJoinCount(JoinInfo joinInfo, int serversNum, JsonObject joinParams) { - try { - if (joinInfo == null) return serversNum; - if (joinInfo.joinCountFuncName != null) { - JsonElement requester = routerConfig.publicKey == null ? JsonNull.INSTANCE : new JsonPrimitive(routerConfig.publicKey); - return engineUtil.invokeFunction(joinInfo.joinCountFuncName, Integer.class, joinParams, requester); - } - if (joinInfo.joinCount != 0) return joinInfo.joinCount; - } catch (Exception e) { - e.printStackTrace(); - } - return serversNum; - } - - public JoinInfo getJoinInfo(BasicOperations operation) { - return doipOperationToJoinInfo.getOrDefault(operation.getName(), null); - } - public void sendMsgAndJoin(DoipMessage msg, JoinInfo joinInfo, String[] BDRepoIDs, DoipMessageCallback cb) { int serversNum = BDRepoIDs.length; - int count = getJoinCount(joinInfo, serversNum, new JsonObject()); + // todo optimize joinParams + int count = RouteJoinUtil.getJoinCount(routerConfig.publicKey, engineUtil, joinInfo, serversNum, new JsonObject()); // 如果joinCountFunc执行算出的joinCount，比所有的servers还要多，就更新为servers的数量 if (serversNum < count) { count = serversNum; } if (count > 0) { - ResultCollector resultCollector = new ResultCollector(this, BDRepoIDs, msg, cb, count, engineUtil, joinInfo); + ResultCollector resultCollector = new ResultCollector(this, BDRepoIDs, msg, cb, count, engineUtil, joinInfo, appendixes); for (String BDRepoId : BDRepoIDs) { // 如果已经建立过BDRepo -> BDO的映射，可以通过BDRepo的地址，直接找到BDO的Client，对其进行操作（本质BDRepo和BDO是一一对应的关系） String address = appendixes.get(BDRepoId).getAsJsonObject().get("address").getAsString(); String version = appendixes.get(BDRepoId).getAsJsonObject().get("version").getAsString(); AuditDoipClient doipClient = getFastClientByUrl(address, null, null); if (doipClient == null) - doipClient = getClientByUrl(address, version); + doipClient = ClientCommonUtil.getClientByUrl(address, version); if (doipMessageSigner != null) { doipMessageSigner.signMessage(msg); } @@ -506,8 +314,8 @@ public class DoipClusterClient extends DoipClientImpl { // DoipMessage toSend = msg.deepCopy(); // 要么response wait中的waitObj改为非static，并加入一点流控制机制。 // 这里的doipClient为BDRepoClient或者BDOClient，无所谓，如果是BDRepoClient，DelegateDoipMessageCallback会出手转为BDOClient - DelegateDoipMessageCallback delegateDoipMessageCallback = new DelegateDoipMessageCallback(doipClient, this, resultCollector, msg); - doipClient.sendMessage(msg, delegateDoipMessageCallback); + ClusterClientDoipMessageCallback clusterClientDoipMessageCallback = new ClusterClientDoipMessageCallback(doipClient, this, resultCollector, msg); + doipClient.sendMessage(msg, clusterClientDoipMessageCallback); } } else { Logger.error("joinCount is not a positive num"); @@ -529,14 +337,7 @@ public class DoipClusterClient extends DoipClientImpl { public void closeAll() { close(); - for (AuditDoipClient client : addressToClient.values()) { - try { - if (client.isConnected()) client.close(); - } catch (Exception e) { - e.printStackTrace(); - } - } - addressToClient.clear(); + ClientCommonUtil.closeAll(); } } diff --git a/src/main/java/org/bdware/doip/cluster/client/TreeClusterClient.java b/src/main/java/org/bdware/doip/cluster/client/TreeClusterClient.java index 3a6bfaa..00f85ee 100644 --- a/src/main/java/org/bdware/doip/cluster/client/TreeClusterClient.java +++ b/src/main/java/org/bdware/doip/cluster/client/TreeClusterClient.java @@ -11,8 +11,7 @@ import org.bdware.doip.cluster.callback.BDODelegateDoipMessageCallback; import org.bdware.doip.cluster.entity.BDOEntity; import org.bdware.doip.cluster.entity.BDWType; import org.bdware.doip.cluster.entity.DDOEntity; -import org.bdware.doip.cluster.util.ClusterClientUtil; -import org.bdware.doip.cluster.util.ConnectionUtil; +import org.bdware.doip.cluster.util.ClientCommonUtil; import org.bdware.doip.cluster.util.RouteJoinUtil; import org.bdware.doip.codec.JsonDoipMessage; import org.bdware.doip.codec.doipMessage.DoipMessage; @@ -44,8 +43,6 @@ public class TreeClusterClient extends DoipClientImpl { // DDOID -> Contract Method -> RouteInfo private static final Map ddoCache = new ConcurrentHashMap<>(); private static final Map bdoCache = new ConcurrentHashMap<>(); - // DOID -> Connection - private static final Map addressToClient = new ConcurrentHashMap<>(); private static final Logger LOGGER = LogManager.getLogger(TreeClusterClient.class); @@ -85,11 +82,11 @@ public class TreeClusterClient extends DoipClientImpl { // 返回当前doid对应的所有的BDRepo public void parseDORecursive(String doid, DoipMessage doipMessage, DoipMessageCallback cb) throws IrpClientException, ScriptException { - JsonObject doInfo = ClusterClientUtil.getDOInfo(irsClient, doid); - BDWType doType = ClusterClientUtil.getDOType(doInfo); + JsonObject doInfo = ClientCommonUtil.getDOInfo(irsClient, doid); + BDWType doType = ClientCommonUtil.getDOType(doInfo); switch (doType) { case DDO: - ClusterClientUtil.verifyDDOInfo(doInfo); + ClientCommonUtil.verifyDDOInfo(doInfo); DDOEntity ddoEntity = ddoInfo2DDOEntity(doid, doInfo); ddoCall(ddoEntity, doipMessage, cb); break; @@ -105,28 +102,6 @@ public class TreeClusterClient extends DoipClientImpl { } } - public static AuditDoipClient getClientByUrl(String address, String version) { - if (address == null && version == null) return null; - AuditDoipClient doipClientImpl = addressToClient.get(address); - if (doipClientImpl == null) { - // Create the client only if it doesn't exist in the map - doipClientImpl = ConnectionUtil.createWithoutConnect(address, version); - // Use putIfAbsent() to ensure thread-safety - AuditDoipClient existingClient = addressToClient.putIfAbsent(address, doipClientImpl); - if (existingClient != null) { - // If another thread added a client while we were creating ours, use that instead - doipClientImpl = existingClient; - } - } - try { - // Always try to reconnect, regardless of whether the client was just created or already existed - ConnectionUtil.tryReconnectSync(doipClientImpl, 2000); - } catch (Exception e) { - e.printStackTrace(); - } - return doipClientImpl; - } - public DDOEntity ddoInfo2DDOEntity(String ddoID, JsonObject ddoInfo) throws ScriptException { DDOEntity ddoEntity = ddoCache.get(ddoID); if(ddoEntity != null) return ddoEntity; @@ -135,9 +110,9 @@ public class TreeClusterClient extends DoipClientImpl { JsonObject appendixes = ddoInfo.get("appendixes").getAsJsonObject(); // 将所有的functions通过engine注册到functionBindings中 JsonObject bcoAccessRules = appendixes.get(bcoId).getAsJsonObject().get("accessRules").getAsJsonObject(); - Map ddoRouteInfo = ClusterClientUtil.getDDORouteInfo(bcoAccessRules); - Map ddoJoinInfo = ClusterClientUtil.getDDOJoinInfo(bcoAccessRules); - NashornScriptEngineUtil engineUtil = ClusterClientUtil.instantiateNashornScriptEngineUtil(ddoInfo); + Map ddoRouteInfo = ClientCommonUtil.getDDORouteInfo(bcoAccessRules); + Map ddoJoinInfo = ClientCommonUtil.getDDOJoinInfo(bcoAccessRules); + NashornScriptEngineUtil engineUtil = ClientCommonUtil.instantiateNashornScriptEngineUtil(ddoInfo); ddoCache.putIfAbsent(ddoID, new DDOEntity(ddoRouteInfo, ddoJoinInfo, engineUtil, appendixes)); return ddoCache.get(ddoID); } @@ -155,7 +130,7 @@ public class TreeClusterClient extends DoipClientImpl { public void bdoCall(BDOEntity bdoEntity, DoipMessage doipMessage, DoipMessageCallback cb) { String address = bdoEntity.getAddress(); String version = bdoEntity.getVersion(); - AuditDoipClient doipClient = getClientByUrl(address, version); + AuditDoipClient doipClient = ClientCommonUtil.getClientByUrl(address, version); assert doipClient != null; doipMessageSigner.signMessage(doipMessage); diff --git a/src/main/java/org/bdware/doip/cluster/flowcontrol/RequestPack.java b/src/main/java/org/bdware/doip/cluster/flowcontrol/RequestPack.java new file mode 100644 index 0000000..7acf938 --- /dev/null +++ b/src/main/java/org/bdware/doip/cluster/flowcontrol/RequestPack.java @@ -0,0 +1,19 @@ +package org.bdware.doip.cluster.flowcontrol; + +import org.bdware.doip.codec.doipMessage.DoipMessage; +import org.bdware.doip.endpoint.client.DoipMessageCallback; +import org.bdware.sc.bean.JoinInfo; + +public class RequestPack { + public final DoipMessage message; + public final JoinInfo joinInfo; + public final String[] bdRepoIds; + public final DoipMessageCallback cb; + + public RequestPack(DoipMessage message, JoinInfo joinInfo, String[] bdRepoIDs, DoipMessageCallback cb) { + this.message = message; + this.joinInfo = joinInfo; + this.bdRepoIds = bdRepoIDs; + this.cb = cb; + } +} diff --git a/src/main/java/org/bdware/doip/cluster/flowcontrol/RateThresholdFlowControl.java b/src/main/java/org/bdware/doip/cluster/flowcontrol/core/RateThresholdFlowControl.java similarity index 91% rename from src/main/java/org/bdware/doip/cluster/flowcontrol/RateThresholdFlowControl.java rename to src/main/java/org/bdware/doip/cluster/flowcontrol/core/RateThresholdFlowControl.java index a9c7568..71e3466 100644 --- a/src/main/java/org/bdware/doip/cluster/flowcontrol/RateThresholdFlowControl.java +++ b/src/main/java/org/bdware/doip/cluster/flowcontrol/core/RateThresholdFlowControl.java @@ -1,4 +1,6 @@ -package org.bdware.doip.cluster.flowcontrol; +package org.bdware.doip.cluster.flowcontrol.core; + +import org.bdware.doip.cluster.flowcontrol.FlowControl; import java.util.concurrent.atomic.AtomicLong; diff --git a/src/main/java/org/bdware/doip/cluster/flowcontrol/SlidingWindowFlowControl.java b/src/main/java/org/bdware/doip/cluster/flowcontrol/core/SlidingWindowFlowControl.java similarity index 93% rename from src/main/java/org/bdware/doip/cluster/flowcontrol/SlidingWindowFlowControl.java rename to src/main/java/org/bdware/doip/cluster/flowcontrol/core/SlidingWindowFlowControl.java index 9e1befd..470d4f2 100644 --- a/src/main/java/org/bdware/doip/cluster/flowcontrol/SlidingWindowFlowControl.java +++ b/src/main/java/org/bdware/doip/cluster/flowcontrol/core/SlidingWindowFlowControl.java @@ -1,4 +1,6 @@ -package org.bdware.doip.cluster.flowcontrol; +package org.bdware.doip.cluster.flowcontrol.core; + +import org.bdware.doip.cluster.flowcontrol.FlowControl; import java.util.ArrayDeque; import java.util.Deque; diff --git a/src/main/java/org/bdware/doip/cluster/flowcontrol/TokenBucketFlowControl.java b/src/main/java/org/bdware/doip/cluster/flowcontrol/core/TokenBucketFlowControl.java similarity index 93% rename from src/main/java/org/bdware/doip/cluster/flowcontrol/TokenBucketFlowControl.java rename to src/main/java/org/bdware/doip/cluster/flowcontrol/core/TokenBucketFlowControl.java index 01e2e26..f51975b 100644 --- a/src/main/java/org/bdware/doip/cluster/flowcontrol/TokenBucketFlowControl.java +++ b/src/main/java/org/bdware/doip/cluster/flowcontrol/core/TokenBucketFlowControl.java @@ -1,4 +1,6 @@ -package org.bdware.doip.cluster.flowcontrol; +package org.bdware.doip.cluster.flowcontrol.core; + +import org.bdware.doip.cluster.flowcontrol.FlowControl; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledThreadPoolExecutor; diff --git a/src/main/java/org/bdware/doip/cluster/util/ClusterClientUtil.java b/src/main/java/org/bdware/doip/cluster/util/ClientCommonUtil.java similarity index 81% rename from src/main/java/org/bdware/doip/cluster/util/ClusterClientUtil.java rename to src/main/java/org/bdware/doip/cluster/util/ClientCommonUtil.java index de2c7cc..deb833d 100644 --- a/src/main/java/org/bdware/doip/cluster/util/ClusterClientUtil.java +++ b/src/main/java/org/bdware/doip/cluster/util/ClientCommonUtil.java @@ -3,6 +3,7 @@ package org.bdware.doip.cluster.util; import com.google.gson.*; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.bdware.doip.audit.client.AuditDoipClient; import org.bdware.doip.audit.client.AuditIrpClient; import org.bdware.doip.cluster.entity.BDWType; import org.bdware.irp.exception.IrpClientException; @@ -16,11 +17,14 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; -public class ClusterClientUtil { - private static final Logger Logger = LogManager.getLogger(ClusterClientUtil.class); +public class ClientCommonUtil { + private static final Logger Logger = LogManager.getLogger(ClientCommonUtil.class); private static final Gson gson = new Gson(); + public static final Map clientCache = new ConcurrentHashMap<>(); + public static String[] getDOIDClusterInfo(JsonObject ddoInfo) { JsonArray bdoList = ddoInfo.get("bdoList").getAsJsonArray(); JsonObject appendixes = ddoInfo.get("appendixes").getAsJsonObject(); @@ -155,4 +159,37 @@ public class ClusterClientUtil { String bdwType = doInfo.get("bdwType").getAsString(); return BDWType.valueOf(bdwType); } + + public static AuditDoipClient getClientByUrl(String address, String version) { + if (address == null && version == null) return null; + AuditDoipClient doipClientImpl = clientCache.get(address); + if (doipClientImpl == null) { + // Create the client only if it doesn't exist in the map + doipClientImpl = ConnectionUtil.createWithoutConnect(address); + // Use putIfAbsent() to ensure thread-safety + AuditDoipClient existingClient = clientCache.putIfAbsent(address, doipClientImpl); + if (existingClient != null) { + // If another thread added a client while we were creating ours, use that instead + doipClientImpl = existingClient; + } + } + try { + // Always try to reconnect, regardless of whether the client was just created or already existed + ConnectionUtil.tryReconnectSync(doipClientImpl, 2000); + } catch (Exception e) { + e.printStackTrace(); + } + return doipClientImpl; + } + + public static void closeAll() { + for (AuditDoipClient client : clientCache.values()) { + try { + if (client.isConnected()) client.close(); + } catch (Exception e) { + e.printStackTrace(); + } + } + clientCache.clear(); + } } diff --git a/src/main/java/org/bdware/doip/cluster/util/ConnectionUtil.java b/src/main/java/org/bdware/doip/cluster/util/ConnectionUtil.java index 32f1755..34e0d66 100644 --- a/src/main/java/org/bdware/doip/cluster/util/ConnectionUtil.java +++ b/src/main/java/org/bdware/doip/cluster/util/ConnectionUtil.java @@ -1,5 +1,6 @@ package org.bdware.doip.cluster.util; +import com.google.gson.JsonObject; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.bdware.doip.audit.client.AuditDoipClient; @@ -8,7 +9,9 @@ import org.bdware.doip.audit.writer.AuditType; import java.io.ByteArrayOutputStream; import java.io.PrintStream; +import java.util.Map; import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicInteger; /** * @author liuyihao @@ -16,16 +19,16 @@ import java.util.concurrent.*; public class ConnectionUtil { private static final Logger LOGGER = LogManager.getLogger(ConnectionUtil.class); - private static final ExecutorService connectionTaskPool = Executors.newFixedThreadPool(8, new ThreadFactory() { - @Override - public Thread newThread(Runnable r) { - Thread t = new Thread(r); - t.setDaemon(true); - return t; - } + private static final ExecutorService connectionTaskPool = Executors.newFixedThreadPool(8, r -> { + Thread t = new Thread(r); + t.setDaemon(true); + return t; }); - public static AuditDoipClient createWithoutConnect(String address, String version) { + private final static Map reconnectRequests = new ConcurrentHashMap<>(); + private final static Map lastReconnectDate = new ConcurrentHashMap<>(); + + public static AuditDoipClient createWithoutConnect(String address) { AuditDoipClient doipClientImpl = new AuditDoipClient(AuditConfig.newInstance(null, AuditType.None, null), null); LOGGER.info("connect to:" + address); doipClientImpl.setRepoUrl(address); @@ -57,4 +60,68 @@ public class ConnectionUtil { future.cancel(true); } } + + public static void popReconnectRequest(String[] targetServers) { + if (targetServers != null) { + for (String str : targetServers) { + if (str == null) continue; + synchronized (reconnectRequests) { + if (!reconnectRequests.containsKey(str)) { + reconnectRequests.put(str, new AtomicInteger(0)); + } + } + AtomicInteger i = reconnectRequests.get(str); + if (i.decrementAndGet() < 0) { + i.set(0); + } + } + } + } + + public static void pushReconnectRequest(String[] targetServers, JsonObject appendixes) { + if (targetServers != null) { + for (String str : targetServers) { + if (str == null) continue; + synchronized (reconnectRequests) { + if (!reconnectRequests.containsKey(str)) { + reconnectRequests.put(str, new AtomicInteger(0)); + } + } + AtomicInteger i = reconnectRequests.get(str); + LOGGER.info("===== pushReconnectRequest add:" + str + " --> " + i.get()); + if (i.incrementAndGet() > 10) { + connectionTaskPool.submit(() -> triggerReconnect(str, appendixes)); + } + } + } + } + + public static void triggerReconnect(String doid, JsonObject appendixes) { + try { + String address = appendixes.get(doid).getAsJsonObject().get("address").getAsString(); + String version = appendixes.get(doid).getAsJsonObject().get("version").getAsString(); + AuditDoipClient doipClient = ClientCommonUtil.getClientByUrl(address, null); + LOGGER.info("==========TRIGGER Reconnect:" + doid + " address:" + address); + Long date = lastReconnectDate.get(doid); + if (date == null || System.currentTimeMillis() - date > 10000) { + lastReconnectDate.put(doid, System.currentTimeMillis()); + reconnectRequests.get(doid).set(0); + } else { + LOGGER.info("==========Reject Reconnect:" + doid + " " + address + ", try too fast!"); + return; + } + if (doipClient != null) { + String oldAddress = doipClient.getRepoUrl(); + doipClient.disconnectAndSet(oldAddress); + ConnectionUtil.tryReconnectSync(doipClient, 10000L); + } else { + doipClient = ClientCommonUtil.getClientByUrl(address, version); + String oldAddress = doipClient.getRepoUrl(); + doipClient.disconnectAndSet(oldAddress); + ConnectionUtil.tryReconnectSync(doipClient, 10000L); + } + } catch (Exception e) { + e.printStackTrace(); + } + } } diff --git a/src/main/java/org/bdware/doip/cluster/util/ReconnectionUtil.java b/src/main/java/org/bdware/doip/cluster/util/ReconnectionUtil.java deleted file mode 100644 index 7bb23f6..0000000 --- a/src/main/java/org/bdware/doip/cluster/util/ReconnectionUtil.java +++ /dev/null @@ -1,104 +0,0 @@ -package org.bdware.doip.cluster.util; - -import com.google.gson.JsonObject; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.bdware.doip.audit.client.AuditDoipClient; -import org.bdware.doip.cluster.client.TreeClusterClient; - -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.atomic.AtomicInteger; - -public class ReconnectionUtil { - public static final Logger LOGGER = LogManager.getLogger(ReconnectionUtil.class); - private final Map reconnectRequests; - private final Map lastReconnectDate; - private final JsonObject appendixes; - private static final ExecutorService connectionTaskPool = Executors.newFixedThreadPool(8, new ThreadFactory() { - @Override - public Thread newThread(Runnable r) { - Thread t = new Thread(r); - t.setDaemon(true); - return t; - } - }); - - public ReconnectionUtil(JsonObject appendixes) { - reconnectRequests = new ConcurrentHashMap<>(); - lastReconnectDate = new ConcurrentHashMap<>(); - this.appendixes = appendixes; - } - - public void popReconnectRequest(String[] targetServers) { - if (targetServers != null) { - for (String str : targetServers) { - if (str == null) continue; - synchronized (reconnectRequests) { - if (!reconnectRequests.containsKey(str)) { - reconnectRequests.put(str, new AtomicInteger(0)); - } - } - AtomicInteger i = reconnectRequests.get(str); - if (i.decrementAndGet() < 0) { - i.set(0); - } - } - } - } - - public void pushReconnectRequest(String[] targetServers) { - if (targetServers != null) { - for (String str : targetServers) { - if (str == null) continue; - synchronized (reconnectRequests) { - if (!reconnectRequests.containsKey(str)) { - reconnectRequests.put(str, new AtomicInteger(0)); - } - } - AtomicInteger i = reconnectRequests.get(str); - LOGGER.info("===== pushReconnectRequest add:" + str + " --> " + i.get()); - if (i.incrementAndGet() > 10) { - connectionTaskPool.submit(new Runnable() { - @Override - public void run() { - triggerReconnect(str); - } - }); - } - } - } - } - - public void triggerReconnect(String doid) { - try { - String address = appendixes.get(doid).getAsJsonObject().get("address").getAsString(); - String version = appendixes.get(doid).getAsJsonObject().get("version").getAsString(); - AuditDoipClient doipClient = TreeClusterClient.getClientByUrl(address, null); - LOGGER.info("==========TRIGGER Reconnect:" + doid + " address:" + address); - Long date = lastReconnectDate.get(doid); - if (date == null || System.currentTimeMillis() - date > 10000) { - lastReconnectDate.put(doid, System.currentTimeMillis()); - reconnectRequests.get(doid).set(0); - } else { - LOGGER.info("==========Reject Reconnect:" + doid + " " + address + ", try too fast!"); - return; - } - if (doipClient != null) { - String oldAddress = doipClient.getRepoUrl(); - doipClient.disconnectAndSet(oldAddress); - ConnectionUtil.tryReconnectSync(doipClient, 10000L); - } else { - doipClient = TreeClusterClient.getClientByUrl(address, version); - String oldAddress = doipClient.getRepoUrl(); - doipClient.disconnectAndSet(oldAddress); - ConnectionUtil.tryReconnectSync(doipClient, 10000L); - } - } catch (Exception e) { - e.printStackTrace(); - } - } -} diff --git a/src/main/java/org/bdware/doip/cluster/util/ResultCollector.java b/src/main/java/org/bdware/doip/cluster/util/ResultCollector.java index 48a6381..830cec7 100644 --- a/src/main/java/org/bdware/doip/cluster/util/ResultCollector.java +++ b/src/main/java/org/bdware/doip/cluster/util/ResultCollector.java @@ -1,6 +1,5 @@ package org.bdware.doip.cluster.util; -import com.google.gson.Gson; import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonObject; @@ -40,9 +39,10 @@ public class ResultCollector implements DoipMessageCallback { JoinInfo joinInfo; String[] targetServers; DoipClusterClient doipClusterClient; + JsonObject appendixes; - public ResultCollector(DoipClusterClient doipClusterClient, String[] targetServers, DoipMessage msg, DoipMessageCallback cb, int count, NashornScriptEngineUtil engineUtil, JoinInfo joinInfo) { + public ResultCollector(DoipClusterClient doipClusterClient, String[] targetServers, DoipMessage msg, DoipMessageCallback cb, int count, NashornScriptEngineUtil engineUtil, JoinInfo joinInfo, JsonObject appendixes) { this.request = msg; this.originalCallback = cb; this.joinCount = count; @@ -50,6 +50,7 @@ public class ResultCollector implements DoipMessageCallback { this.joinInfo = joinInfo; this.targetServers = targetServers; this.doipClusterClient = doipClusterClient; + this.appendixes = appendixes; } @Override @@ -98,11 +99,11 @@ public class ResultCollector implements DoipMessageCallback { } private void popReconnectRequest() { - doipClusterClient.popReconnectRequest(targetServers); + ConnectionUtil.popReconnectRequest(targetServers); } private void pushReconnectRequest() { - doipClusterClient.pushReconnectRequest(targetServers); + ConnectionUtil.pushReconnectRequest(targetServers, appendixes); } public synchronized void addDoipResult(String nodeID, DoipMessage msg) { diff --git a/src/main/java/org/bdware/doip/cluster/util/RouteJoinUtil.java b/src/main/java/org/bdware/doip/cluster/util/RouteJoinUtil.java index 22e8a79..45d199d 100644 --- a/src/main/java/org/bdware/doip/cluster/util/RouteJoinUtil.java +++ b/src/main/java/org/bdware/doip/cluster/util/RouteJoinUtil.java @@ -70,7 +70,7 @@ public class RouteJoinUtil { return null; } - private static JsonElement tryLoadJsonProp(JsonObject routeInfoArg, String param) { + public static JsonElement tryLoadJsonProp(JsonObject routeInfoArg, String param) { try { if (routeInfoArg == null) return JsonNull.INSTANCE; @@ -117,7 +117,7 @@ public class RouteJoinUtil { } } - protected static int getJoinCount(EndpointConfig routerConfig, NashornScriptEngineUtil engineUtil, JoinInfo joinInfo, int serversNum, JsonObject joinParams) { + public static int getJoinCount(EndpointConfig routerConfig, NashornScriptEngineUtil engineUtil, JoinInfo joinInfo, int serversNum, JsonObject joinParams) { try { if (joinInfo == null) return serversNum; if (joinInfo.joinCountFuncName != null) { @@ -130,4 +130,18 @@ public class RouteJoinUtil { } return serversNum; } + + public static int getJoinCount(String publicKey, NashornScriptEngineUtil engineUtil, JoinInfo joinInfo, int serversNum, JsonObject joinParams) { + try { + if (joinInfo == null) return serversNum; + if (joinInfo.joinCountFuncName != null) { + JsonElement requester = publicKey == null ? JsonNull.INSTANCE : new JsonPrimitive(publicKey); + return engineUtil.invokeFunction(joinInfo.joinCountFuncName, Integer.class, joinParams, requester); + } + if (joinInfo.joinCount != 0) return joinInfo.joinCount; + } catch (Exception e) { + e.printStackTrace(); + } + return serversNum; + } } -- Gitee From f816e5c70b96ebaaef5e9004eadd1bc74f4295df Mon Sep 17 00:00:00 2001 From: "haoeliu@foxmail.com" Date: Tue, 11 Jul 2023 23:09:29 +0800 Subject: [PATCH 07/14] fix getJoinCount --- .../org/bdware/doip/cluster/ClusterDoaClient.java | 5 ++--- .../doip/cluster/client/DoipClusterClient.java | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/bdware/doip/cluster/ClusterDoaClient.java b/src/main/java/org/bdware/doip/cluster/ClusterDoaClient.java index 08638d1..f7eb582 100644 --- a/src/main/java/org/bdware/doip/cluster/ClusterDoaClient.java +++ b/src/main/java/org/bdware/doip/cluster/ClusterDoaClient.java @@ -106,7 +106,7 @@ public class ClusterDoaClient extends DoipClientImpl { String[] BDRepoIDs = route(message); // joinParams都来自于callback，joinParams就是所有节点callback到达之后，merge之后的结果 // todo optimize getJoinInfo - JoinInfo joinInfo = getJoinInfo(op); + JoinInfo joinInfo = (op); sendMsgAndJoin(doipMessage, joinInfo, BDRepoIDs, cb); } @@ -115,7 +115,7 @@ public class ClusterDoaClient extends DoipClientImpl { public void sendMsgAndJoin(DoipMessage msg, JoinInfo joinInfo, String[] BDRepoIDs, DoipMessageCallback cb) { int serversNum = BDRepoIDs.length; // todo optimize getJoinInfo - int count = RouteJoinUtil.getJoinCount(null, joinInfo, serversNum, new JsonObject()); + int count = getJoinCount(joinInfo, serversNum, new JsonObject()); // 如果joinCountFunc执行算出的joinCount，比所有的servers还要多，就更新为servers的数量 if (serversNum < count) { count = serversNum; @@ -125,7 +125,6 @@ public class ClusterDoaClient extends DoipClientImpl { builder.setBody("success".getBytes()); cb.onResult(builder.create()); } - }; bdoClients.put(bdoid, doipClusterClient); } diff --git a/src/main/java/org/bdware/doip/cluster/client/DoipClusterClient.java b/src/main/java/org/bdware/doip/cluster/client/DoipClusterClient.java index 0cb15cf..1037a86 100644 --- a/src/main/java/org/bdware/doip/cluster/client/DoipClusterClient.java +++ b/src/main/java/org/bdware/doip/cluster/client/DoipClusterClient.java @@ -335,6 +335,20 @@ public class DoipClusterClient extends DoipClientImpl { } + protected int getJoinCount(JoinInfo joinInfo, int serversNum, JsonObject joinParams) { + try { + if (joinInfo == null) return serversNum; + if (joinInfo.joinCountFuncName != null) { + JsonElement requester = routerConfig.publicKey == null ? JsonNull.INSTANCE : new JsonPrimitive(routerConfig.publicKey); + return engineUtil.invokeFunction(joinInfo.joinCountFuncName, Integer.class, joinParams, requester); + } + if (joinInfo.joinCount != 0) return joinInfo.joinCount; + } catch (Exception e) { + e.printStackTrace(); + } + return serversNum; + } + public void closeAll() { close(); ClientCommonUtil.closeAll(); -- Gitee From cd31af2bd1a48286f8468215a1f5e591409fba04 Mon Sep 17 00:00:00 2001 From: "haoeliu@foxmail.com" Date: Tue, 11 Jul 2023 23:14:39 +0800 Subject: [PATCH 08/14] fix ClusterDoaClient --- .../bdware/doip/cluster/ClusterDoaClient.java | 23 ++++++++++--- .../cluster/client/DoipClusterClient.java | 32 +++++-------------- 2 files changed, 26 insertions(+), 29 deletions(-) diff --git a/src/main/java/org/bdware/doip/cluster/ClusterDoaClient.java b/src/main/java/org/bdware/doip/cluster/ClusterDoaClient.java index f7eb582..5104225 100644 --- a/src/main/java/org/bdware/doip/cluster/ClusterDoaClient.java +++ b/src/main/java/org/bdware/doip/cluster/ClusterDoaClient.java @@ -1,10 +1,12 @@ package org.bdware.doip.cluster; +import com.google.gson.JsonElement; +import com.google.gson.JsonNull; import com.google.gson.JsonObject; +import com.google.gson.JsonPrimitive; import org.bdware.doip.audit.EndpointConfig; import org.bdware.doip.audit.client.AuditIrpClient; import org.bdware.doip.cluster.client.DoipClusterClient; -import org.bdware.doip.cluster.util.RouteJoinUtil; import org.bdware.doip.codec.JsonDoipMessage; import org.bdware.doip.codec.digitalObject.DigitalObject; import org.bdware.doip.codec.doipMessage.DoipMessage; @@ -105,16 +107,13 @@ public class ClusterDoaClient extends DoipClientImpl { JsonDoipMessage message = JsonDoipMessage.fromDoipMessage(doipMessage); String[] BDRepoIDs = route(message); // joinParams都来自于callback，joinParams就是所有节点callback到达之后，merge之后的结果 - // todo optimize getJoinInfo - JoinInfo joinInfo = (op); + JoinInfo joinInfo = this.doipOperationToJoinInfo.get(op.getName()); sendMsgAndJoin(doipMessage, joinInfo, BDRepoIDs, cb); - } @Override public void sendMsgAndJoin(DoipMessage msg, JoinInfo joinInfo, String[] BDRepoIDs, DoipMessageCallback cb) { int serversNum = BDRepoIDs.length; - // todo optimize getJoinInfo int count = getJoinCount(joinInfo, serversNum, new JsonObject()); // 如果joinCountFunc执行算出的joinCount，比所有的servers还要多，就更新为servers的数量 if (serversNum < count) { @@ -125,6 +124,20 @@ public class ClusterDoaClient extends DoipClientImpl { builder.setBody("success".getBytes()); cb.onResult(builder.create()); } + + int getJoinCount(JoinInfo joinInfo, int serversNum, JsonObject joinParams) { + try { + if (joinInfo == null) return serversNum; + if (joinInfo.joinCountFuncName != null) { + JsonElement requester = routerConfig.publicKey == null ? JsonNull.INSTANCE : new JsonPrimitive(routerConfig.publicKey); + return engineUtil.invokeFunction(joinInfo.joinCountFuncName, Integer.class, joinParams, requester); + } + if (joinInfo.joinCount != 0) return joinInfo.joinCount; + } catch (Exception e) { + e.printStackTrace(); + } + return serversNum; + } }; bdoClients.put(bdoid, doipClusterClient); } diff --git a/src/main/java/org/bdware/doip/cluster/client/DoipClusterClient.java b/src/main/java/org/bdware/doip/cluster/client/DoipClusterClient.java index 1037a86..3ef4d71 100644 --- a/src/main/java/org/bdware/doip/cluster/client/DoipClusterClient.java +++ b/src/main/java/org/bdware/doip/cluster/client/DoipClusterClient.java @@ -46,17 +46,23 @@ import java.util.concurrent.*; public class DoipClusterClient extends DoipClientImpl { private final DoipMessageSigner doipMessageSigner; private final AuditIrpClient irsClient; - private final EndpointConfig routerConfig; + protected final EndpointConfig routerConfig; // all the doipOperation and routeInfo binding relationships private final Map doipOperationToRouteInfo; // all the doipOperation and joinInfo binding relationships - private final Map doipOperationToJoinInfo; + protected final Map doipOperationToJoinInfo; // 维护了地址 与 doipPort之间的映射关系 // BDO和DoipClusterClient是一一对应的关系，一个BDO对应一组BDRepo，一个BDRepo对应了一个DoipServer端口，因此这里维护Repo -> 端口的映射 private final Map bdRepoToBDOClient; //Queue length based control. private final static Queue sendMessageTaskQueue = new ConcurrentLinkedDeque<>(); private final FlowControl flowControl; + Logger Logger = LogManager.getLogger(DoipClusterClient.class); + private final RouterTool routerTool; + String curDOID; + JsonObject appendixes; + + protected NashornScriptEngineUtil engineUtil; // todo optimize thread pool ExecutorService sendMessagePool = Executors.newFixedThreadPool(8, new ThreadFactory() { @@ -68,14 +74,6 @@ public class DoipClusterClient extends DoipClientImpl { } }); - - Logger Logger = LogManager.getLogger(DoipClusterClient.class); - private final RouterTool routerTool; - String curDOID; - JsonObject appendixes; - - private NashornScriptEngineUtil engineUtil; - public DoipClusterClient(EndpointConfig routerConfig) throws IrpClientException, ScriptException { this.routerConfig = routerConfig; irsClient = new AuditIrpClient(routerConfig); @@ -335,20 +333,6 @@ public class DoipClusterClient extends DoipClientImpl { } - protected int getJoinCount(JoinInfo joinInfo, int serversNum, JsonObject joinParams) { - try { - if (joinInfo == null) return serversNum; - if (joinInfo.joinCountFuncName != null) { - JsonElement requester = routerConfig.publicKey == null ? JsonNull.INSTANCE : new JsonPrimitive(routerConfig.publicKey); - return engineUtil.invokeFunction(joinInfo.joinCountFuncName, Integer.class, joinParams, requester); - } - if (joinInfo.joinCount != 0) return joinInfo.joinCount; - } catch (Exception e) { - e.printStackTrace(); - } - return serversNum; - } - public void closeAll() { close(); ClientCommonUtil.closeAll(); -- Gitee From 1fae00a1b3e4c6c9f3dffd67b4e8633011fd34f1 Mon Sep 17 00:00:00 2001 From: "haoeliu@foxmail.com" Date: Tue, 11 Jul 2023 23:19:36 +0800 Subject: [PATCH 09/14] fix ResultCollector --- .../ClusterClientResultCollector.java} | 11 ++++++----- .../DDODelegateDoipMessageCallback.java | 6 ++---- .../bdware/doip/cluster/client/DOAClient.java | 18 +++++++++--------- .../doip/cluster/client/DoipClusterClient.java | 6 +++--- .../doip/cluster/util/RouteJoinUtil.java | 4 ++-- 5 files changed, 22 insertions(+), 23 deletions(-) rename src/main/java/org/bdware/doip/cluster/{util/ResultCollector.java => callback/ClusterClientResultCollector.java} (93%) diff --git a/src/main/java/org/bdware/doip/cluster/util/ResultCollector.java b/src/main/java/org/bdware/doip/cluster/callback/ClusterClientResultCollector.java similarity index 93% rename from src/main/java/org/bdware/doip/cluster/util/ResultCollector.java rename to src/main/java/org/bdware/doip/cluster/callback/ClusterClientResultCollector.java index 830cec7..051953e 100644 --- a/src/main/java/org/bdware/doip/cluster/util/ResultCollector.java +++ b/src/main/java/org/bdware/doip/cluster/callback/ClusterClientResultCollector.java @@ -1,4 +1,4 @@ -package org.bdware.doip.cluster.util; +package org.bdware.doip.cluster.callback; import com.google.gson.JsonArray; import com.google.gson.JsonElement; @@ -6,6 +6,7 @@ import com.google.gson.JsonObject; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.bdware.doip.cluster.client.DoipClusterClient; +import org.bdware.doip.cluster.util.ConnectionUtil; import org.bdware.doip.codec.JsonDoipMessage; import org.bdware.doip.codec.doipMessage.DoipMessage; import org.bdware.doip.codec.doipMessage.DoipMessageFactory; @@ -23,14 +24,14 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; -public class ResultCollector implements DoipMessageCallback { - static Logger LOGGER = LogManager.getLogger(ResultCollector.class); +public class ClusterClientResultCollector implements DoipMessageCallback { + static Logger LOGGER = LogManager.getLogger(ClusterClientResultCollector.class); private final DoipMessage request; // joinCount记录开始Merge的上限 int joinCount; AtomicInteger count = new AtomicInteger(0); DoipMessageCallback originalCallback; - Logger Logger = LogManager.getLogger(ResultCollector.class); + Logger Logger = LogManager.getLogger(ClusterClientResultCollector.class); Map successResMap = new HashMap<>(); Set problemResult = new HashSet<>(); // 记录所有收到结果列表 @@ -42,7 +43,7 @@ public class ResultCollector implements DoipMessageCallback { JsonObject appendixes; - public ResultCollector(DoipClusterClient doipClusterClient, String[] targetServers, DoipMessage msg, DoipMessageCallback cb, int count, NashornScriptEngineUtil engineUtil, JoinInfo joinInfo, JsonObject appendixes) { + public ClusterClientResultCollector(DoipClusterClient doipClusterClient, String[] targetServers, DoipMessage msg, DoipMessageCallback cb, int count, NashornScriptEngineUtil engineUtil, JoinInfo joinInfo, JsonObject appendixes) { this.request = msg; this.originalCallback = cb; this.joinCount = count; diff --git a/src/main/java/org/bdware/doip/cluster/callback/DDODelegateDoipMessageCallback.java b/src/main/java/org/bdware/doip/cluster/callback/DDODelegateDoipMessageCallback.java index 9aa4b68..4ec42bd 100644 --- a/src/main/java/org/bdware/doip/cluster/callback/DDODelegateDoipMessageCallback.java +++ b/src/main/java/org/bdware/doip/cluster/callback/DDODelegateDoipMessageCallback.java @@ -8,7 +8,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.bdware.doip.cluster.util.ConnectionUtil; import org.bdware.doip.cluster.client.TreeClusterClient; -import org.bdware.doip.cluster.util.ResultCollector; import org.bdware.doip.codec.JsonDoipMessage; import org.bdware.doip.codec.doipMessage.DoipMessage; import org.bdware.doip.codec.doipMessage.DoipMessageFactory; @@ -27,13 +26,12 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; public class DDODelegateDoipMessageCallback implements DoipMessageCallback { - static org.apache.logging.log4j.Logger LOGGER = LogManager.getLogger(ResultCollector.class); + static Logger LOGGER = LogManager.getLogger(DDODelegateDoipMessageCallback.class); private final DoipMessage request; // joinCount记录开始Merge的上限 int joinCount; AtomicInteger count = new AtomicInteger(0); DoipMessageCallback originalCallback; - Logger Logger = LogManager.getLogger(ResultCollector.class); Map successResMap = new HashMap<>(); Set problemResult = new HashSet<>(); // 记录所有收到结果列表 @@ -89,7 +87,7 @@ public class DDODelegateDoipMessageCallback implements DoipMessageCallback { if (nodeID != null) { ConnectionUtil.popReconnectRequest(targetServers); } else { - Logger.error("收到错误的节点回复 : " + "Something wrong happens in" + new Gson().toJson(targetServers) + " msg:" + new Gson().toJson(msg.header.parameters)); + LOGGER.error("收到错误的节点回复 : " + "Something wrong happens in" + new Gson().toJson(targetServers) + " msg:" + new Gson().toJson(msg.header.parameters)); ConnectionUtil.pushReconnectRequest(targetServers, appendixes); } } catch (Exception e) { diff --git a/src/main/java/org/bdware/doip/cluster/client/DOAClient.java b/src/main/java/org/bdware/doip/cluster/client/DOAClient.java index c53b4b5..d498eea 100644 --- a/src/main/java/org/bdware/doip/cluster/client/DOAClient.java +++ b/src/main/java/org/bdware/doip/cluster/client/DOAClient.java @@ -5,7 +5,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.bdware.doip.audit.EndpointConfig; import org.bdware.doip.audit.client.AuditIrpClient; -import org.bdware.doip.cluster.util.ResultCollector; +import org.bdware.doip.cluster.callback.ClusterClientResultCollector; import org.bdware.doip.codec.digitalObject.DigitalObject; import org.bdware.doip.codec.doipMessage.DoipMessage; import org.bdware.doip.codec.doipMessage.DoipMessageSigner; @@ -369,30 +369,30 @@ public class DOAClient extends DoipClientImpl { if (count > 0) { //TODO//TODO//TODO//TODO//TODO//TODO//TODO//TODO//TODO - ResultCollector resultCollector = new ResultCollector(null, null, null, cb, count, engineUtil, joinInfo, null); + ClusterClientResultCollector clusterClientResultCollector = new ClusterClientResultCollector(null, null, null, cb, count, engineUtil, joinInfo, null); for (String targetDoipServer : targetDoipServers) { DoipClientImpl doipClientImpl = getDoipClientForServer(targetDoipServer); switch (doipParam.operation) { case Hello: - doipClientImpl.hello(doipParam.identifier, resultCollector); + doipClientImpl.hello(doipParam.identifier, clusterClientResultCollector); break; case ListOps: - doipClientImpl.listOperations(doipParam.identifier, resultCollector); + doipClientImpl.listOperations(doipParam.identifier, clusterClientResultCollector); break; case Retrieve: - doipClientImpl.retrieve(doipParam.identifier, doipParam.element, doipParam.includeElementData, resultCollector); + doipClientImpl.retrieve(doipParam.identifier, doipParam.element, doipParam.includeElementData, clusterClientResultCollector); break; case Create: - doipClientImpl.create(doipParam.identifier, doipParam.digitalObject, resultCollector); + doipClientImpl.create(doipParam.identifier, doipParam.digitalObject, clusterClientResultCollector); break; case Update: - doipClientImpl.update(doipParam.digitalObject, resultCollector); + doipClientImpl.update(doipParam.digitalObject, clusterClientResultCollector); break; case Delete: - doipClientImpl.delete(doipParam.identifier, resultCollector); + doipClientImpl.delete(doipParam.identifier, clusterClientResultCollector); break; case Search: - doipClientImpl.search(doipParam.identifier, doipParam.sp, resultCollector); + doipClientImpl.search(doipParam.identifier, doipParam.sp, clusterClientResultCollector); break; } } diff --git a/src/main/java/org/bdware/doip/cluster/client/DoipClusterClient.java b/src/main/java/org/bdware/doip/cluster/client/DoipClusterClient.java index 3ef4d71..1646a04 100644 --- a/src/main/java/org/bdware/doip/cluster/client/DoipClusterClient.java +++ b/src/main/java/org/bdware/doip/cluster/client/DoipClusterClient.java @@ -11,7 +11,7 @@ import org.bdware.doip.cluster.flowcontrol.FlowControl; import org.bdware.doip.cluster.flowcontrol.RequestPack; import org.bdware.doip.cluster.flowcontrol.core.RateThresholdFlowControl; import org.bdware.doip.cluster.util.ClientCommonUtil; -import org.bdware.doip.cluster.util.ResultCollector; +import org.bdware.doip.cluster.callback.ClusterClientResultCollector; import org.bdware.doip.cluster.util.RouteJoinUtil; import org.bdware.doip.cluster.util.RouterTool; import org.bdware.doip.codec.JsonDoipMessage; @@ -297,7 +297,7 @@ public class DoipClusterClient extends DoipClientImpl { } if (count > 0) { - ResultCollector resultCollector = new ResultCollector(this, BDRepoIDs, msg, cb, count, engineUtil, joinInfo, appendixes); + ClusterClientResultCollector clusterClientResultCollector = new ClusterClientResultCollector(this, BDRepoIDs, msg, cb, count, engineUtil, joinInfo, appendixes); for (String BDRepoId : BDRepoIDs) { // 如果已经建立过BDRepo -> BDO的映射，可以通过BDRepo的地址，直接找到BDO的Client，对其进行操作（本质BDRepo和BDO是一一对应的关系） String address = appendixes.get(BDRepoId).getAsJsonObject().get("address").getAsString(); @@ -312,7 +312,7 @@ public class DoipClusterClient extends DoipClientImpl { // DoipMessage toSend = msg.deepCopy(); // 要么response wait中的waitObj改为非static，并加入一点流控制机制。 // 这里的doipClient为BDRepoClient或者BDOClient，无所谓，如果是BDRepoClient，DelegateDoipMessageCallback会出手转为BDOClient - ClusterClientDoipMessageCallback clusterClientDoipMessageCallback = new ClusterClientDoipMessageCallback(doipClient, this, resultCollector, msg); + ClusterClientDoipMessageCallback clusterClientDoipMessageCallback = new ClusterClientDoipMessageCallback(doipClient, this, clusterClientResultCollector, msg); doipClient.sendMessage(msg, clusterClientDoipMessageCallback); } } else { diff --git a/src/main/java/org/bdware/doip/cluster/util/RouteJoinUtil.java b/src/main/java/org/bdware/doip/cluster/util/RouteJoinUtil.java index 45d199d..240e134 100644 --- a/src/main/java/org/bdware/doip/cluster/util/RouteJoinUtil.java +++ b/src/main/java/org/bdware/doip/cluster/util/RouteJoinUtil.java @@ -102,7 +102,7 @@ public class RouteJoinUtil { } if (count > 0) { - DDODelegateDoipMessageCallback resultCollector = new DDODelegateDoipMessageCallback(treeClusterClient, toCalls, msg, cb, count, engineUtil, joinInfo, appendixes); + DDODelegateDoipMessageCallback ddoDelegateDoipMessageCallback = new DDODelegateDoipMessageCallback(treeClusterClient, toCalls, msg, cb, count, engineUtil, joinInfo, appendixes); for (String toCall : toCalls) { // todo 对于DDO嵌套 -> appendix可以只放bdwtype // todo 消息嵌套？同一个消息发给多个Server？每个消息改一个requestId = 0 & DOID -> DDOID/BDOID，并且发给多个Server？ @@ -110,7 +110,7 @@ public class RouteJoinUtil { // DoipMessage toSend = msg.deepCopy(); // 要么response wait中的waitObj改为非static，并加入一点流控制机制。 // 这里的doipClient为BDRepoClient或者BDOClient，无所谓，如果是BDRepoClient，DelegateDoipMessageCallback会出手转为BDOClient - treeClusterClient.sendMessage(msg, toCall, resultCollector); + treeClusterClient.sendMessage(msg, toCall, ddoDelegateDoipMessageCallback); } } else { LOGGER.error("joinCount is not a positive num"); -- Gitee From 1a87d4e89ca0a5b6be689a3a81781f57921c4a04 Mon Sep 17 00:00:00 2001 From: "haoeliu@foxmail.com" Date: Wed, 12 Jul 2023 22:57:36 +0800 Subject: [PATCH 10/14] refactor TreeClusterClient and DoipClusterClient and add modify flow control to both clients --- build.gradle | 3 +- .../BDODelegateDoipMessageCallback.java | 5 +- .../cluster/client/DoipClusterClient.java | 129 +++++++----------- .../cluster/client/TreeClusterClient.java | 111 +++++++++++++-- .../entity/DoipClusterRequestPack.java | 55 ++++++++ .../entity/TreeClusterRequestPack.java | 65 +++++++++ .../doip/cluster/flowcontrol/RequestPack.java | 19 --- .../util/AuditDoipClientCacheUtil.java | 43 ++++++ .../doip/cluster/util/ConnectionUtil.java | 4 +- ...tCommonUtil.java => DOResolutionUtil.java} | 41 +----- 10 files changed, 319 insertions(+), 156 deletions(-) create mode 100644 src/main/java/org/bdware/doip/cluster/entity/DoipClusterRequestPack.java create mode 100644 src/main/java/org/bdware/doip/cluster/entity/TreeClusterRequestPack.java delete mode 100644 src/main/java/org/bdware/doip/cluster/flowcontrol/RequestPack.java create mode 100644 src/main/java/org/bdware/doip/cluster/util/AuditDoipClientCacheUtil.java rename src/main/java/org/bdware/doip/cluster/util/{ClientCommonUtil.java => DOResolutionUtil.java} (80%) diff --git a/build.gradle b/build.gradle index d1f9791..e9618bb 100644 --- a/build.gradle +++ b/build.gradle @@ -12,7 +12,7 @@ sourceCompatibility = JavaVersion.VERSION_1_8 targetCompatibility = JavaVersion.VERSION_1_8 repositories { mavenCentral() - mavenLocal() +// mavenLocal() } sourceSets { main { @@ -59,7 +59,6 @@ dependencies { implementation 'org.postgresql:postgresql:42.6.0' implementation 'org.bdware.sc:common:1.7.5' implementation fileTree(dir: 'lib', include: '*.jar') - //---- testImplementation 'io.netty:netty-all:4.1.77.Final' testImplementation 'junit:junit:4.12' } diff --git a/src/main/java/org/bdware/doip/cluster/callback/BDODelegateDoipMessageCallback.java b/src/main/java/org/bdware/doip/cluster/callback/BDODelegateDoipMessageCallback.java index 6ff44bc..7bab021 100644 --- a/src/main/java/org/bdware/doip/cluster/callback/BDODelegateDoipMessageCallback.java +++ b/src/main/java/org/bdware/doip/cluster/callback/BDODelegateDoipMessageCallback.java @@ -3,8 +3,7 @@ package org.bdware.doip.cluster.callback; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.bdware.doip.audit.client.AuditDoipClient; -import org.bdware.doip.cluster.client.TreeClusterClient; -import org.bdware.doip.cluster.util.ClientCommonUtil; +import org.bdware.doip.cluster.util.AuditDoipClientCacheUtil; import org.bdware.doip.codec.doipMessage.DoipMessage; import org.bdware.doip.codec.doipMessage.DoipResponseCode; import org.bdware.doip.endpoint.client.DoipMessageCallback; @@ -40,7 +39,7 @@ public class BDODelegateDoipMessageCallback implements DoipMessageCallback { version = msg.header.parameters.attributes.get("version").getAsString(); LOGGER.info("recv delegate:"+newAddress +" previousRepo:"+previousClient.getRepoUrl()); // BDRepo doipMsgCallback -> BDOClient，维护repoToBDO的映射关系 - AuditDoipClient client = ClientCommonUtil.getClientByUrl(newAddress, version); + AuditDoipClient client = AuditDoipClientCacheUtil.getClientByUrl(newAddress, version); previousClient = client; client.sendRawMessage(originalMessage, this); } diff --git a/src/main/java/org/bdware/doip/cluster/client/DoipClusterClient.java b/src/main/java/org/bdware/doip/cluster/client/DoipClusterClient.java index 1646a04..365e093 100644 --- a/src/main/java/org/bdware/doip/cluster/client/DoipClusterClient.java +++ b/src/main/java/org/bdware/doip/cluster/client/DoipClusterClient.java @@ -8,9 +8,9 @@ import org.bdware.doip.audit.client.AuditDoipClient; import org.bdware.doip.audit.client.AuditIrpClient; import org.bdware.doip.cluster.callback.ClusterClientDoipMessageCallback; import org.bdware.doip.cluster.flowcontrol.FlowControl; -import org.bdware.doip.cluster.flowcontrol.RequestPack; +import org.bdware.doip.cluster.entity.DoipClusterRequestPack; import org.bdware.doip.cluster.flowcontrol.core.RateThresholdFlowControl; -import org.bdware.doip.cluster.util.ClientCommonUtil; +import org.bdware.doip.cluster.util.AuditDoipClientCacheUtil; import org.bdware.doip.cluster.callback.ClusterClientResultCollector; import org.bdware.doip.cluster.util.RouteJoinUtil; import org.bdware.doip.cluster.util.RouterTool; @@ -45,7 +45,6 @@ import java.util.concurrent.*; //connections between different bdo is reused if possible. public class DoipClusterClient extends DoipClientImpl { private final DoipMessageSigner doipMessageSigner; - private final AuditIrpClient irsClient; protected final EndpointConfig routerConfig; // all the doipOperation and routeInfo binding relationships private final Map doipOperationToRouteInfo; @@ -54,8 +53,7 @@ public class DoipClusterClient extends DoipClientImpl { // 维护了地址 与 doipPort之间的映射关系 // BDO和DoipClusterClient是一一对应的关系，一个BDO对应一组BDRepo，一个BDRepo对应了一个DoipServer端口，因此这里维护Repo -> 端口的映射 private final Map bdRepoToBDOClient; - //Queue length based control. - private final static Queue sendMessageTaskQueue = new ConcurrentLinkedDeque<>(); + private final static Queue sendMessageTaskQueue = new ConcurrentLinkedDeque<>(); private final FlowControl flowControl; Logger Logger = LogManager.getLogger(DoipClusterClient.class); private final RouterTool routerTool; @@ -65,24 +63,16 @@ public class DoipClusterClient extends DoipClientImpl { protected NashornScriptEngineUtil engineUtil; // todo optimize thread pool - ExecutorService sendMessagePool = Executors.newFixedThreadPool(8, new ThreadFactory() { - @Override - public Thread newThread(Runnable r) { - Thread t = new Thread(r); - t.setDaemon(true); - return t; - } + ExecutorService sendMessagePool = Executors.newFixedThreadPool(8, r -> { + Thread t = new Thread(r); + t.setDaemon(true); + return t; }); public DoipClusterClient(EndpointConfig routerConfig) throws IrpClientException, ScriptException { this.routerConfig = routerConfig; - irsClient = new AuditIrpClient(routerConfig); - new Thread(new Runnable() { - @Override - public void run() { - executeSendMsgTask(); - } - }).start(); + AuditIrpClient irsClient = new AuditIrpClient(routerConfig); + new Thread(this::consumeSendMessageTask).start(); try { irsClient.reconnect(); } catch (IrpConnectException e) { @@ -149,14 +139,11 @@ public class DoipClusterClient extends DoipClientImpl { String[] BDRepoIDs = route(message); // joinParams都来自于callback，joinParams就是所有节点callback到达之后，merge之后的结果 JoinInfo joinInfo = doipOperationToJoinInfo.get(op.getName()); - RequestPack pack = new RequestPack(doipMessage, joinInfo, BDRepoIDs, new DoipMessageCallback() { - @Override - public void onResult(DoipMessage msg) { - cb.onResult(msg); - flowControl.maintainFlowControl(); - } + DoipClusterRequestPack pack = new DoipClusterRequestPack(doipMessage, joinInfo, BDRepoIDs, msg -> { + cb.onResult(msg); + flowControl.maintainFlowControl(); }); - addSendMsgTask(pack); + produceSendMessageTask(pack); } catch (Exception e) { e.printStackTrace(); DoipMessageFactory.DoipMessageBuilder builder = new DoipMessageFactory.DoipMessageBuilder(); @@ -166,57 +153,14 @@ public class DoipClusterClient extends DoipClientImpl { } - private void addSendMsgTask(RequestPack pack) { - for (int i = 0; i < 10 && !flowControl.enableRequestPass(); i++) { - try { - // todo uncomment rate info - LOGGER.info("rate is too fast"); - Thread.sleep(100); - } catch (Exception e) { - e.printStackTrace(); - } - } - sendMessageTaskQueue.add(pack); - // todo -> 如果有人在等，我就去synchronize sendMessageTaskQueue 并且 notify，不然性能可能会有瓶颈 - synchronized (sendMessageTaskQueue) { - sendMessageTaskQueue.notify(); - } - } - - //pool为单个线程操作。 - public void executeSendMsgTask() { - while (true) { - try { - RequestPack pack = sendMessageTaskQueue.poll(); - if (pack != null) - sendMessagePool.execute(new Runnable() { - @Override - public void run() { - sendMessageInternal(pack); - } - }); - else - synchronized (sendMessageTaskQueue) { - sendMessageTaskQueue.wait(10000); - } - } catch (InterruptedException e) { - } - } - - } - - public void sendMessageInternal(RequestPack pack) { + public void sendMessageInternal(DoipClusterRequestPack pack) { try { sendMsgAndJoin(pack.message, pack.joinInfo, pack.bdRepoIds, pack.cb); } catch (Exception e) { e.printStackTrace(); - if (pack != null) { - DoipMessageFactory.DoipMessageBuilder builder = new DoipMessageFactory.DoipMessageBuilder(); - builder.createResponse(DoipResponseCode.UnKnownError, pack.message); - pack.cb.onResult(builder.create()); - } - } finally { - // flowControl.maintainFlowControl(); + DoipMessageFactory.DoipMessageBuilder builder = new DoipMessageFactory.DoipMessageBuilder(); + builder.createResponse(DoipResponseCode.UnKnownError, pack.message); + pack.cb.onResult(builder.create()); } } @@ -225,7 +169,7 @@ public class DoipClusterClient extends DoipClientImpl { if (existingConn != null) { return existingConn; } - AuditDoipClient newGetClient = ClientCommonUtil.getClientByUrl(address, version); + AuditDoipClient newGetClient = AuditDoipClientCacheUtil.getClientByUrl(address, version); if (newGetClient == null) { return null; } @@ -304,7 +248,7 @@ public class DoipClusterClient extends DoipClientImpl { String version = appendixes.get(BDRepoId).getAsJsonObject().get("version").getAsString(); AuditDoipClient doipClient = getFastClientByUrl(address, null, null); if (doipClient == null) - doipClient = ClientCommonUtil.getClientByUrl(address, version); + doipClient = AuditDoipClientCacheUtil.getClientByUrl(address, version); if (doipMessageSigner != null) { doipMessageSigner.signMessage(msg); } @@ -320,6 +264,39 @@ public class DoipClusterClient extends DoipClientImpl { } } + private void produceSendMessageTask(DoipClusterRequestPack pack) { + for (int i = 0; i < 10 && !flowControl.enableRequestPass(); i++) { + try { + LOGGER.info("rate is too fast"); + Thread.sleep(100); + } catch (Exception e) { + e.printStackTrace(); + } + } + sendMessageTaskQueue.add(pack); + // todo -> 如果有人在等，我就去synchronize sendMessageTaskQueue 并且 notify，不然性能可能会有瓶颈 + synchronized (sendMessageTaskQueue) { + sendMessageTaskQueue.notify(); + } + } + + public void consumeSendMessageTask() { + while (true) { + try { + DoipClusterRequestPack pack = sendMessageTaskQueue.poll(); + if (pack != null) { + sendMessagePool.execute(() -> sendMessageInternal(pack)); + } else { + synchronized (sendMessageTaskQueue) { + sendMessageTaskQueue.wait(10000); + } + } + } catch (InterruptedException ignored) { + } + } + + } + @Override public void close() { for (AuditDoipClient client : bdRepoToBDOClient.values()) @@ -335,7 +312,7 @@ public class DoipClusterClient extends DoipClientImpl { public void closeAll() { close(); - ClientCommonUtil.closeAll(); + AuditDoipClientCacheUtil.closeAll(); } } diff --git a/src/main/java/org/bdware/doip/cluster/client/TreeClusterClient.java b/src/main/java/org/bdware/doip/cluster/client/TreeClusterClient.java index 00f85ee..8cca823 100644 --- a/src/main/java/org/bdware/doip/cluster/client/TreeClusterClient.java +++ b/src/main/java/org/bdware/doip/cluster/client/TreeClusterClient.java @@ -8,10 +8,11 @@ import org.bdware.doip.audit.EndpointConfig; import org.bdware.doip.audit.client.AuditDoipClient; import org.bdware.doip.audit.client.AuditIrpClient; import org.bdware.doip.cluster.callback.BDODelegateDoipMessageCallback; -import org.bdware.doip.cluster.entity.BDOEntity; -import org.bdware.doip.cluster.entity.BDWType; -import org.bdware.doip.cluster.entity.DDOEntity; -import org.bdware.doip.cluster.util.ClientCommonUtil; +import org.bdware.doip.cluster.entity.*; +import org.bdware.doip.cluster.flowcontrol.FlowControl; +import org.bdware.doip.cluster.flowcontrol.core.RateThresholdFlowControl; +import org.bdware.doip.cluster.util.AuditDoipClientCacheUtil; +import org.bdware.doip.cluster.util.DOResolutionUtil; import org.bdware.doip.cluster.util.RouteJoinUtil; import org.bdware.doip.codec.JsonDoipMessage; import org.bdware.doip.codec.doipMessage.DoipMessage; @@ -31,9 +32,12 @@ import wrp.jdk.nashorn.api.scripting.NashornScriptEngineUtil; import javax.script.ScriptException; import java.util.Map; +import java.util.Queue; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedDeque; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; -// todo add flow control public class TreeClusterClient extends DoipClientImpl { public final DoipMessageSigner doipMessageSigner; // router config -> provide to DoipClusterClient @@ -43,8 +47,15 @@ public class TreeClusterClient extends DoipClientImpl { // DDOID -> Contract Method -> RouteInfo private static final Map ddoCache = new ConcurrentHashMap<>(); private static final Map bdoCache = new ConcurrentHashMap<>(); - private static final Logger LOGGER = LogManager.getLogger(TreeClusterClient.class); + private final static Queue sendMessageTaskQueue = new ConcurrentLinkedDeque<>(); + private final FlowControl flowControl; + // todo optimize thread pool + ExecutorService sendMessagePool = Executors.newFixedThreadPool(8, r -> { + Thread t = new Thread(r); + t.setDaemon(true); + return t; + }); public TreeClusterClient(EndpointConfig config) { this.routerConfig = config; @@ -54,6 +65,12 @@ public class TreeClusterClient extends DoipClientImpl { } else doipMessageSigner = new SM2Signer(SM2Util.generateSM2KeyPair()); + int threshold = 1000; + if (routerConfig.extraConfig.has("rateThresholdFlowControl")) + threshold = routerConfig.extraConfig.get("rateThresholdFlowControl").getAsInt(); + flowControl = new RateThresholdFlowControl(threshold); + new Thread(this::consumeSendMessageTask).start(); + LOGGER.info("The SmartClusterClient has been initialized"); } @@ -82,24 +99,33 @@ public class TreeClusterClient extends DoipClientImpl { // 返回当前doid对应的所有的BDRepo public void parseDORecursive(String doid, DoipMessage doipMessage, DoipMessageCallback cb) throws IrpClientException, ScriptException { - JsonObject doInfo = ClientCommonUtil.getDOInfo(irsClient, doid); - BDWType doType = ClientCommonUtil.getDOType(doInfo); + JsonObject doInfo = DOResolutionUtil.getDOInfo(irsClient, doid); + BDWType doType = DOResolutionUtil.getDOType(doInfo); + DoipMessageCallback flowControlCallback = msg -> { + cb.onResult(msg); + flowControl.maintainFlowControl(); + }; + TreeClusterRequestPack requestPack = new TreeClusterRequestPack(); + requestPack.setDoipMessage(doipMessage); + requestPack.setDoipMessageCallback(flowControlCallback); switch (doType) { case DDO: - ClientCommonUtil.verifyDDOInfo(doInfo); + DOResolutionUtil.verifyDDOInfo(doInfo); DDOEntity ddoEntity = ddoInfo2DDOEntity(doid, doInfo); - ddoCall(ddoEntity, doipMessage, cb); + requestPack.setDdoEntity(ddoEntity); break; case BDO: case BDRepo: BDOEntity bdoEntity = bdoInfo2BDOEntity(doid, doInfo); - bdoCall(bdoEntity, doipMessage, cb); + requestPack.setBdoEntity(bdoEntity); case BCRepo: throw new IrpClientException("BCRepo should not appear in SmartClusterClient verification"); default: LOGGER.error("Wrong doType: " + doType); throw new IrpClientException("doType cannot be parsed"); } + + produceSendMessageTask(requestPack); } public DDOEntity ddoInfo2DDOEntity(String ddoID, JsonObject ddoInfo) throws ScriptException { @@ -110,9 +136,9 @@ public class TreeClusterClient extends DoipClientImpl { JsonObject appendixes = ddoInfo.get("appendixes").getAsJsonObject(); // 将所有的functions通过engine注册到functionBindings中 JsonObject bcoAccessRules = appendixes.get(bcoId).getAsJsonObject().get("accessRules").getAsJsonObject(); - Map ddoRouteInfo = ClientCommonUtil.getDDORouteInfo(bcoAccessRules); - Map ddoJoinInfo = ClientCommonUtil.getDDOJoinInfo(bcoAccessRules); - NashornScriptEngineUtil engineUtil = ClientCommonUtil.instantiateNashornScriptEngineUtil(ddoInfo); + Map ddoRouteInfo = DOResolutionUtil.getDDORouteInfo(bcoAccessRules); + Map ddoJoinInfo = DOResolutionUtil.getDDOJoinInfo(bcoAccessRules); + NashornScriptEngineUtil engineUtil = DOResolutionUtil.instantiateNashornScriptEngineUtil(ddoInfo); ddoCache.putIfAbsent(ddoID, new DDOEntity(ddoRouteInfo, ddoJoinInfo, engineUtil, appendixes)); return ddoCache.get(ddoID); } @@ -130,7 +156,7 @@ public class TreeClusterClient extends DoipClientImpl { public void bdoCall(BDOEntity bdoEntity, DoipMessage doipMessage, DoipMessageCallback cb) { String address = bdoEntity.getAddress(); String version = bdoEntity.getVersion(); - AuditDoipClient doipClient = ClientCommonUtil.getClientByUrl(address, version); + AuditDoipClient doipClient = AuditDoipClientCacheUtil.getClientByUrl(address, version); assert doipClient != null; doipMessageSigner.signMessage(doipMessage); @@ -144,4 +170,59 @@ public class TreeClusterClient extends DoipClientImpl { String[] toCalls = RouteJoinUtil.route(doipParam, routerConfig, ddoEntity.getDDORouteInfo().get(operation.getName()), ddoEntity.getEngineUtil()); RouteJoinUtil.sendMsgAndJoin(this, ddoEntity.getEngineUtil(), doipMessage, ddoEntity.getDDOJoinInfo().get(operation.getName()), toCalls, cb, ddoEntity.getAppendixes()); } + + public void sendMessageInternal(TreeClusterRequestPack pack) { + try { + switch (pack.getDoType()) { + case DDO: + ddoCall(pack.getDdoEntity(), pack.getDoipMessage(), pack.getDoipMessageCallback()); + break; + case BDO: + case BDRepo: + bdoCall(pack.getBdoEntity(), pack.getDoipMessage(), pack.getDoipMessageCallback()); + break; + case BCRepo: + throw new IrpClientException("BCRepo should not appear in SmartClusterClient verification"); + default: + LOGGER.error("Wrong doType: " + pack.getDoType()); + throw new IrpClientException("doType cannot be parsed"); + } + } catch (IrpClientException e) { + e.printStackTrace(); + DoipMessageFactory.DoipMessageBuilder builder = new DoipMessageFactory.DoipMessageBuilder(); + builder.createResponse(DoipResponseCode.UnKnownError, pack.getDoipMessage()); + pack.getDoipMessageCallback().onResult(builder.create()); + } + } + + private void produceSendMessageTask(TreeClusterRequestPack pack) { + for (int i = 0; i < 10 && !flowControl.enableRequestPass(); i++) { + try { + LOGGER.info("rate is too fast"); + Thread.sleep(100); + } catch (Exception e) { + e.printStackTrace(); + } + } + sendMessageTaskQueue.add(pack); + // todo -> 如果有人在等，我就去synchronize sendMessageTaskQueue 并且 notify，不然性能可能会有瓶颈 + synchronized (sendMessageTaskQueue) { + sendMessageTaskQueue.notify(); + } + } + + public void consumeSendMessageTask() { + while (true) { + try { + TreeClusterRequestPack pack = sendMessageTaskQueue.poll(); + if (pack != null) + sendMessagePool.execute(() -> sendMessageInternal(pack)); + else + synchronized (sendMessageTaskQueue) { + sendMessageTaskQueue.wait(10000); + } + } catch (InterruptedException e) { + } + } + } } diff --git a/src/main/java/org/bdware/doip/cluster/entity/DoipClusterRequestPack.java b/src/main/java/org/bdware/doip/cluster/entity/DoipClusterRequestPack.java new file mode 100644 index 0000000..0eb63cb --- /dev/null +++ b/src/main/java/org/bdware/doip/cluster/entity/DoipClusterRequestPack.java @@ -0,0 +1,55 @@ +package org.bdware.doip.cluster.entity; + +import org.bdware.doip.codec.doipMessage.DoipMessage; +import org.bdware.doip.endpoint.client.DoipMessageCallback; +import org.bdware.sc.bean.JoinInfo; + +public class DoipClusterRequestPack { + public DoipMessage getMessage() { + return message; + } + + public DoipClusterRequestPack() { + } + + public void setMessage(DoipMessage message) { + this.message = message; + } + + public JoinInfo getJoinInfo() { + return joinInfo; + } + + public void setJoinInfo(JoinInfo joinInfo) { + this.joinInfo = joinInfo; + } + + public String[] getBdRepoIds() { + return bdRepoIds; + } + + public void setBdRepoIds(String[] bdRepoIds) { + this.bdRepoIds = bdRepoIds; + } + + public DoipMessageCallback getCb() { + return cb; + } + + public void setCb(DoipMessageCallback cb) { + this.cb = cb; + } + + public DoipMessage message; + public JoinInfo joinInfo; + + public DoipClusterRequestPack(DoipMessage message, JoinInfo joinInfo, String[] bdRepoIds, DoipMessageCallback cb) { + this.message = message; + this.joinInfo = joinInfo; + this.bdRepoIds = bdRepoIds; + this.cb = cb; + } + + public String[] bdRepoIds; + public DoipMessageCallback cb; +} diff --git a/src/main/java/org/bdware/doip/cluster/entity/TreeClusterRequestPack.java b/src/main/java/org/bdware/doip/cluster/entity/TreeClusterRequestPack.java new file mode 100644 index 0000000..7ef8fdc --- /dev/null +++ b/src/main/java/org/bdware/doip/cluster/entity/TreeClusterRequestPack.java @@ -0,0 +1,65 @@ +package org.bdware.doip.cluster.entity; + +import org.bdware.doip.codec.doipMessage.DoipMessage; +import org.bdware.doip.endpoint.client.DoipMessageCallback; + +public class TreeClusterRequestPack { + BDOEntity bdoEntity; + DDOEntity ddoEntity; + BDWType doType; + + public TreeClusterRequestPack() { + } + + DoipMessage doipMessage; + + public BDOEntity getBdoEntity() { + return bdoEntity; + } + + public void setBdoEntity(BDOEntity bdoEntity) { + this.bdoEntity = bdoEntity; + } + + public DDOEntity getDdoEntity() { + return ddoEntity; + } + + public void setDdoEntity(DDOEntity ddoEntity) { + this.ddoEntity = ddoEntity; + } + + public BDWType getDoType() { + return doType; + } + + public void setDoType(BDWType doType) { + this.doType = doType; + } + + public DoipMessage getDoipMessage() { + return doipMessage; + } + + public void setDoipMessage(DoipMessage doipMessage) { + this.doipMessage = doipMessage; + } + + public DoipMessageCallback getDoipMessageCallback() { + return doipMessageCallback; + } + + public void setDoipMessageCallback(DoipMessageCallback doipMessageCallback) { + this.doipMessageCallback = doipMessageCallback; + } + + DoipMessageCallback doipMessageCallback; + + public TreeClusterRequestPack(BDOEntity bdoEntity, DDOEntity ddoEntity, BDWType doType, DoipMessage doipMessage, DoipMessageCallback doipMessageCallback) { + this.bdoEntity = bdoEntity; + this.ddoEntity = ddoEntity; + this.doType = doType; + this.doipMessage = doipMessage; + this.doipMessageCallback = doipMessageCallback; + } +} diff --git a/src/main/java/org/bdware/doip/cluster/flowcontrol/RequestPack.java b/src/main/java/org/bdware/doip/cluster/flowcontrol/RequestPack.java deleted file mode 100644 index 7acf938..0000000 --- a/src/main/java/org/bdware/doip/cluster/flowcontrol/RequestPack.java +++ /dev/null @@ -1,19 +0,0 @@ -package org.bdware.doip.cluster.flowcontrol; - -import org.bdware.doip.codec.doipMessage.DoipMessage; -import org.bdware.doip.endpoint.client.DoipMessageCallback; -import org.bdware.sc.bean.JoinInfo; - -public class RequestPack { - public final DoipMessage message; - public final JoinInfo joinInfo; - public final String[] bdRepoIds; - public final DoipMessageCallback cb; - - public RequestPack(DoipMessage message, JoinInfo joinInfo, String[] bdRepoIDs, DoipMessageCallback cb) { - this.message = message; - this.joinInfo = joinInfo; - this.bdRepoIds = bdRepoIDs; - this.cb = cb; - } -} diff --git a/src/main/java/org/bdware/doip/cluster/util/AuditDoipClientCacheUtil.java b/src/main/java/org/bdware/doip/cluster/util/AuditDoipClientCacheUtil.java new file mode 100644 index 0000000..23a4db0 --- /dev/null +++ b/src/main/java/org/bdware/doip/cluster/util/AuditDoipClientCacheUtil.java @@ -0,0 +1,43 @@ +package org.bdware.doip.cluster.util; + +import org.bdware.doip.audit.client.AuditDoipClient; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +public class AuditDoipClientCacheUtil { + public static final Map clientCache = new ConcurrentHashMap<>(); + + public static AuditDoipClient getClientByUrl(String address, String version) { + if (address == null && version == null) return null; + AuditDoipClient doipClientImpl = clientCache.get(address); + if (doipClientImpl == null) { + // Create the client only if it doesn't exist in the map + doipClientImpl = ConnectionUtil.createWithoutConnect(address); + // Use putIfAbsent() to ensure thread-safety + AuditDoipClient existingClient = clientCache.putIfAbsent(address, doipClientImpl); + if (existingClient != null) { + // If another thread added a client while we were creating ours, use that instead + doipClientImpl = existingClient; + } + } + try { + // Always try to reconnect, regardless of whether the client was just created or already existed + ConnectionUtil.tryReconnectSync(doipClientImpl, 2000); + } catch (Exception e) { + e.printStackTrace(); + } + return doipClientImpl; + } + + public static void closeAll() { + for (AuditDoipClient client : clientCache.values()) { + try { + if (client.isConnected()) client.close(); + } catch (Exception e) { + e.printStackTrace(); + } + } + clientCache.clear(); + } +} diff --git a/src/main/java/org/bdware/doip/cluster/util/ConnectionUtil.java b/src/main/java/org/bdware/doip/cluster/util/ConnectionUtil.java index 34e0d66..859c518 100644 --- a/src/main/java/org/bdware/doip/cluster/util/ConnectionUtil.java +++ b/src/main/java/org/bdware/doip/cluster/util/ConnectionUtil.java @@ -100,7 +100,7 @@ public class ConnectionUtil { try { String address = appendixes.get(doid).getAsJsonObject().get("address").getAsString(); String version = appendixes.get(doid).getAsJsonObject().get("version").getAsString(); - AuditDoipClient doipClient = ClientCommonUtil.getClientByUrl(address, null); + AuditDoipClient doipClient = AuditDoipClientCacheUtil.getClientByUrl(address, null); LOGGER.info("==========TRIGGER Reconnect:" + doid + " address:" + address); Long date = lastReconnectDate.get(doid); if (date == null || System.currentTimeMillis() - date > 10000) { @@ -115,7 +115,7 @@ public class ConnectionUtil { doipClient.disconnectAndSet(oldAddress); ConnectionUtil.tryReconnectSync(doipClient, 10000L); } else { - doipClient = ClientCommonUtil.getClientByUrl(address, version); + doipClient = AuditDoipClientCacheUtil.getClientByUrl(address, version); String oldAddress = doipClient.getRepoUrl(); doipClient.disconnectAndSet(oldAddress); ConnectionUtil.tryReconnectSync(doipClient, 10000L); diff --git a/src/main/java/org/bdware/doip/cluster/util/ClientCommonUtil.java b/src/main/java/org/bdware/doip/cluster/util/DOResolutionUtil.java similarity index 80% rename from src/main/java/org/bdware/doip/cluster/util/ClientCommonUtil.java rename to src/main/java/org/bdware/doip/cluster/util/DOResolutionUtil.java index deb833d..1bd48c2 100644 --- a/src/main/java/org/bdware/doip/cluster/util/ClientCommonUtil.java +++ b/src/main/java/org/bdware/doip/cluster/util/DOResolutionUtil.java @@ -3,7 +3,6 @@ package org.bdware.doip.cluster.util; import com.google.gson.*; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.bdware.doip.audit.client.AuditDoipClient; import org.bdware.doip.audit.client.AuditIrpClient; import org.bdware.doip.cluster.entity.BDWType; import org.bdware.irp.exception.IrpClientException; @@ -17,14 +16,11 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -public class ClientCommonUtil { - private static final Logger Logger = LogManager.getLogger(ClientCommonUtil.class); +public class DOResolutionUtil { + private static final Logger Logger = LogManager.getLogger(DOResolutionUtil.class); private static final Gson gson = new Gson(); - public static final Map clientCache = new ConcurrentHashMap<>(); - public static String[] getDOIDClusterInfo(JsonObject ddoInfo) { JsonArray bdoList = ddoInfo.get("bdoList").getAsJsonArray(); JsonObject appendixes = ddoInfo.get("appendixes").getAsJsonObject(); @@ -159,37 +155,4 @@ public class ClientCommonUtil { String bdwType = doInfo.get("bdwType").getAsString(); return BDWType.valueOf(bdwType); } - - public static AuditDoipClient getClientByUrl(String address, String version) { - if (address == null && version == null) return null; - AuditDoipClient doipClientImpl = clientCache.get(address); - if (doipClientImpl == null) { - // Create the client only if it doesn't exist in the map - doipClientImpl = ConnectionUtil.createWithoutConnect(address); - // Use putIfAbsent() to ensure thread-safety - AuditDoipClient existingClient = clientCache.putIfAbsent(address, doipClientImpl); - if (existingClient != null) { - // If another thread added a client while we were creating ours, use that instead - doipClientImpl = existingClient; - } - } - try { - // Always try to reconnect, regardless of whether the client was just created or already existed - ConnectionUtil.tryReconnectSync(doipClientImpl, 2000); - } catch (Exception e) { - e.printStackTrace(); - } - return doipClientImpl; - } - - public static void closeAll() { - for (AuditDoipClient client : clientCache.values()) { - try { - if (client.isConnected()) client.close(); - } catch (Exception e) { - e.printStackTrace(); - } - } - clientCache.clear(); - } } -- Gitee From c8f664c6b5545a1d33457e8204265944c67995ea Mon Sep 17 00:00:00 2001 From: "haoeliu@foxmail.com" Date: Thu, 13 Jul 2023 00:04:03 +0800 Subject: [PATCH 11/14] refactor RouteJoinUtil --- .../bdware/doip/cluster/ClusterDoaClient.java | 1 - .../cluster/client/DoipClusterClient.java | 55 ++++++------------- .../cluster/client/TreeClusterClient.java | 2 +- .../doip/cluster/util/RouteJoinUtil.java | 53 +++++++++++++++++- 4 files changed, 70 insertions(+), 41 deletions(-) diff --git a/src/main/java/org/bdware/doip/cluster/ClusterDoaClient.java b/src/main/java/org/bdware/doip/cluster/ClusterDoaClient.java index 5104225..0820d8e 100644 --- a/src/main/java/org/bdware/doip/cluster/ClusterDoaClient.java +++ b/src/main/java/org/bdware/doip/cluster/ClusterDoaClient.java @@ -111,7 +111,6 @@ public class ClusterDoaClient extends DoipClientImpl { sendMsgAndJoin(doipMessage, joinInfo, BDRepoIDs, cb); } - @Override public void sendMsgAndJoin(DoipMessage msg, JoinInfo joinInfo, String[] BDRepoIDs, DoipMessageCallback cb) { int serversNum = BDRepoIDs.length; int count = getJoinCount(joinInfo, serversNum, new JsonObject()); diff --git a/src/main/java/org/bdware/doip/cluster/client/DoipClusterClient.java b/src/main/java/org/bdware/doip/cluster/client/DoipClusterClient.java index 365e093..73c0a72 100644 --- a/src/main/java/org/bdware/doip/cluster/client/DoipClusterClient.java +++ b/src/main/java/org/bdware/doip/cluster/client/DoipClusterClient.java @@ -6,12 +6,10 @@ import org.apache.logging.log4j.Logger; import org.bdware.doip.audit.EndpointConfig; import org.bdware.doip.audit.client.AuditDoipClient; import org.bdware.doip.audit.client.AuditIrpClient; -import org.bdware.doip.cluster.callback.ClusterClientDoipMessageCallback; -import org.bdware.doip.cluster.flowcontrol.FlowControl; import org.bdware.doip.cluster.entity.DoipClusterRequestPack; +import org.bdware.doip.cluster.flowcontrol.FlowControl; import org.bdware.doip.cluster.flowcontrol.core.RateThresholdFlowControl; import org.bdware.doip.cluster.util.AuditDoipClientCacheUtil; -import org.bdware.doip.cluster.callback.ClusterClientResultCollector; import org.bdware.doip.cluster.util.RouteJoinUtil; import org.bdware.doip.cluster.util.RouterTool; import org.bdware.doip.codec.JsonDoipMessage; @@ -37,7 +35,10 @@ import java.math.BigInteger; import java.util.HashMap; import java.util.Map; import java.util.Queue; -import java.util.concurrent.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedDeque; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; //Client for single bdo running in cluster model. @@ -155,7 +156,18 @@ public class DoipClusterClient extends DoipClientImpl { public void sendMessageInternal(DoipClusterRequestPack pack) { try { - sendMsgAndJoin(pack.message, pack.joinInfo, pack.bdRepoIds, pack.cb); + RouteJoinUtil.bdosSendMsgAndJoin( + this, + pack.getMessage(), + pack.getJoinInfo(), + pack.getBdRepoIds(), + pack.getCb(), + doipMessageSigner, + bdRepoToBDOClient, + engineUtil, + routerConfig.publicKey, + appendixes + ); } catch (Exception e) { e.printStackTrace(); DoipMessageFactory.DoipMessageBuilder builder = new DoipMessageFactory.DoipMessageBuilder(); @@ -231,39 +243,6 @@ public class DoipClusterClient extends DoipClientImpl { return null; } - public void sendMsgAndJoin(DoipMessage msg, JoinInfo joinInfo, String[] BDRepoIDs, DoipMessageCallback cb) { - int serversNum = BDRepoIDs.length; - // todo optimize joinParams - int count = RouteJoinUtil.getJoinCount(routerConfig.publicKey, engineUtil, joinInfo, serversNum, new JsonObject()); - // 如果joinCountFunc执行算出的joinCount，比所有的servers还要多，就更新为servers的数量 - if (serversNum < count) { - count = serversNum; - } - - if (count > 0) { - ClusterClientResultCollector clusterClientResultCollector = new ClusterClientResultCollector(this, BDRepoIDs, msg, cb, count, engineUtil, joinInfo, appendixes); - for (String BDRepoId : BDRepoIDs) { - // 如果已经建立过BDRepo -> BDO的映射，可以通过BDRepo的地址，直接找到BDO的Client，对其进行操作（本质BDRepo和BDO是一一对应的关系） - String address = appendixes.get(BDRepoId).getAsJsonObject().get("address").getAsString(); - String version = appendixes.get(BDRepoId).getAsJsonObject().get("version").getAsString(); - AuditDoipClient doipClient = getFastClientByUrl(address, null, null); - if (doipClient == null) - doipClient = AuditDoipClientCacheUtil.getClientByUrl(address, version); - if (doipMessageSigner != null) { - doipMessageSigner.signMessage(msg); - } - // TODO TODO TODO - // DoipMessage toSend = msg.deepCopy(); - // 要么response wait中的waitObj改为非static，并加入一点流控制机制。 - // 这里的doipClient为BDRepoClient或者BDOClient，无所谓，如果是BDRepoClient，DelegateDoipMessageCallback会出手转为BDOClient - ClusterClientDoipMessageCallback clusterClientDoipMessageCallback = new ClusterClientDoipMessageCallback(doipClient, this, clusterClientResultCollector, msg); - doipClient.sendMessage(msg, clusterClientDoipMessageCallback); - } - } else { - Logger.error("joinCount is not a positive num"); - } - } - private void produceSendMessageTask(DoipClusterRequestPack pack) { for (int i = 0; i < 10 && !flowControl.enableRequestPass(); i++) { try { diff --git a/src/main/java/org/bdware/doip/cluster/client/TreeClusterClient.java b/src/main/java/org/bdware/doip/cluster/client/TreeClusterClient.java index 8cca823..45e793a 100644 --- a/src/main/java/org/bdware/doip/cluster/client/TreeClusterClient.java +++ b/src/main/java/org/bdware/doip/cluster/client/TreeClusterClient.java @@ -168,7 +168,7 @@ public class TreeClusterClient extends DoipClientImpl { JsonDoipMessage doipParam = JsonDoipMessage.fromDoipMessage(doipMessage); BasicOperations operation = BasicOperations.getDoOp(doipParam.header.operation); String[] toCalls = RouteJoinUtil.route(doipParam, routerConfig, ddoEntity.getDDORouteInfo().get(operation.getName()), ddoEntity.getEngineUtil()); - RouteJoinUtil.sendMsgAndJoin(this, ddoEntity.getEngineUtil(), doipMessage, ddoEntity.getDDOJoinInfo().get(operation.getName()), toCalls, cb, ddoEntity.getAppendixes()); + RouteJoinUtil.ddoSendMsgAndJoin(this, ddoEntity.getEngineUtil(), doipMessage, ddoEntity.getDDOJoinInfo().get(operation.getName()), toCalls, cb, ddoEntity.getAppendixes()); } public void sendMessageInternal(TreeClusterRequestPack pack) { diff --git a/src/main/java/org/bdware/doip/cluster/util/RouteJoinUtil.java b/src/main/java/org/bdware/doip/cluster/util/RouteJoinUtil.java index 240e134..245e6ac 100644 --- a/src/main/java/org/bdware/doip/cluster/util/RouteJoinUtil.java +++ b/src/main/java/org/bdware/doip/cluster/util/RouteJoinUtil.java @@ -7,11 +7,15 @@ import com.google.gson.JsonPrimitive; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.bdware.doip.audit.EndpointConfig; +import org.bdware.doip.audit.client.AuditDoipClient; +import org.bdware.doip.cluster.callback.ClusterClientDoipMessageCallback; +import org.bdware.doip.cluster.callback.ClusterClientResultCollector; import org.bdware.doip.cluster.callback.DDODelegateDoipMessageCallback; import org.bdware.doip.cluster.client.DoipClusterClient; import org.bdware.doip.cluster.client.TreeClusterClient; import org.bdware.doip.codec.JsonDoipMessage; import org.bdware.doip.codec.doipMessage.DoipMessage; +import org.bdware.doip.codec.doipMessage.DoipMessageSigner; import org.bdware.doip.endpoint.client.DoipMessageCallback; import org.bdware.sc.bean.JoinInfo; import org.bdware.sc.bean.RouteInfo; @@ -19,6 +23,7 @@ import org.bdware.sc.util.JsonUtil; import wrp.jdk.nashorn.api.scripting.NashornScriptEngineUtil; import java.math.BigInteger; +import java.util.Map; public class RouteJoinUtil { public static final Logger LOGGER = LogManager.getLogger(DoipClusterClient.class); @@ -93,7 +98,7 @@ public class RouteJoinUtil { } } - public static void sendMsgAndJoin(TreeClusterClient treeClusterClient, NashornScriptEngineUtil engineUtil, DoipMessage msg, JoinInfo joinInfo, String[] toCalls, DoipMessageCallback cb, JsonObject appendixes) { + public static void ddoSendMsgAndJoin(TreeClusterClient treeClusterClient, NashornScriptEngineUtil engineUtil, DoipMessage msg, JoinInfo joinInfo, String[] toCalls, DoipMessageCallback cb, JsonObject appendixes) { int serversNum = toCalls.length; int count = getJoinCount(treeClusterClient.routerConfig, engineUtil, joinInfo, serversNum, new JsonObject()); // 如果joinCountFunc执行算出的joinCount，比所有的servers还要多，就更新为servers的数量 @@ -117,6 +122,52 @@ public class RouteJoinUtil { } } + public static void bdosSendMsgAndJoin(DoipClusterClient doipClusterClient, DoipMessage msg, JoinInfo joinInfo, String[] BDRepoIDs, DoipMessageCallback cb, DoipMessageSigner doipMessageSigner, Map bdRepoToBDOClient, NashornScriptEngineUtil engineUtil, String publicKey, JsonObject appendixes) { + int serversNum = BDRepoIDs.length; + // todo optimize joinParams + int count = RouteJoinUtil.getJoinCount(publicKey, engineUtil, joinInfo, serversNum, new JsonObject()); + // 如果joinCountFunc执行算出的joinCount，比所有的servers还要多，就更新为servers的数量 + if (serversNum < count) { + count = serversNum; + } + + if (count > 0) { + ClusterClientResultCollector clusterClientResultCollector = new ClusterClientResultCollector(doipClusterClient, BDRepoIDs, msg, cb, count, engineUtil, joinInfo, appendixes); + for (String BDRepoId : BDRepoIDs) { + // 如果已经建立过BDRepo -> BDO的映射，可以通过BDRepo的地址，直接找到BDO的Client，对其进行操作（本质BDRepo和BDO是一一对应的关系） + String address = appendixes.get(BDRepoId).getAsJsonObject().get("address").getAsString(); + String version = appendixes.get(BDRepoId).getAsJsonObject().get("version").getAsString(); + AuditDoipClient doipClient = getFastClientByUrl(bdRepoToBDOClient, address, null, null); + if (doipClient == null) + doipClient = AuditDoipClientCacheUtil.getClientByUrl(address, version); + if (doipMessageSigner != null) { + doipMessageSigner.signMessage(msg); + } + // TODO TODO TODO + // DoipMessage toSend = msg.deepCopy(); + // 要么response wait中的waitObj改为非static，并加入一点流控制机制。 + // 这里的doipClient为BDRepoClient或者BDOClient，无所谓，如果是BDRepoClient，DelegateDoipMessageCallback会出手转为BDOClient + ClusterClientDoipMessageCallback clusterClientDoipMessageCallback = new ClusterClientDoipMessageCallback(doipClient, doipClusterClient, clusterClientResultCollector, msg); + doipClient.sendMessage(msg, clusterClientDoipMessageCallback); + } + } else { + LOGGER.error("joinCount is not a positive num"); + } + } + + public static AuditDoipClient getFastClientByUrl(Map bdRepoToBDOClient, String previousAddress, String address, String version) { + AuditDoipClient existingConn = bdRepoToBDOClient.get(previousAddress); + if (existingConn != null) { + return existingConn; + } + AuditDoipClient newGetClient = AuditDoipClientCacheUtil.getClientByUrl(address, version); + if (newGetClient == null) { + return null; + } + existingConn = bdRepoToBDOClient.putIfAbsent(previousAddress, newGetClient); + return existingConn == null ? newGetClient : existingConn; + } + public static int getJoinCount(EndpointConfig routerConfig, NashornScriptEngineUtil engineUtil, JoinInfo joinInfo, int serversNum, JsonObject joinParams) { try { if (joinInfo == null) return serversNum; -- Gitee From 5583f0aff4b780a70324f7485863d6835da6726b Mon Sep 17 00:00:00 2001 From: "haoeliu@foxmail.com" Date: Thu, 13 Jul 2023 23:37:24 +0800 Subject: [PATCH 12/14] add clone to DoipMessage and extract fastClientCache --- .../org/bdware/dogp/client/AliTestClient.java | 2 +- .../BDODelegateDoipMessageCallback.java | 4 +- .../ClusterClientDoipMessageCallback.java | 54 ----- .../ClusterClientResultCollector.java | 204 ------------------ .../DDODelegateDoipMessageCallback.java | 8 +- .../bdware/doip/cluster/client/DOAClient.java | 18 +- .../cluster/client/DoipClusterClient.java | 33 +-- .../cluster/client/TreeClusterClient.java | 10 +- .../util/AuditDoipClientCacheUtil.java | 24 +++ .../doip/cluster/util/RouteJoinUtil.java | 32 +-- 10 files changed, 57 insertions(+), 332 deletions(-) delete mode 100644 src/main/java/org/bdware/doip/cluster/callback/ClusterClientDoipMessageCallback.java delete mode 100644 src/main/java/org/bdware/doip/cluster/callback/ClusterClientResultCollector.java diff --git a/src/main/dogp/org/bdware/dogp/client/AliTestClient.java b/src/main/dogp/org/bdware/dogp/client/AliTestClient.java index 5fe37f2..b480c94 100644 --- a/src/main/dogp/org/bdware/dogp/client/AliTestClient.java +++ b/src/main/dogp/org/bdware/dogp/client/AliTestClient.java @@ -170,7 +170,7 @@ public class AliTestClient { FileOutputStream output = new FileOutputStream(testCase.getResultFile()); output.write(jo.toString().getBytes(StandardCharsets.UTF_8)); output.close(); - clusterClient.closeAll(); + clusterClient.close(); } public void createDOL() throws Exception { diff --git a/src/main/java/org/bdware/doip/cluster/callback/BDODelegateDoipMessageCallback.java b/src/main/java/org/bdware/doip/cluster/callback/BDODelegateDoipMessageCallback.java index 7bab021..829b75d 100644 --- a/src/main/java/org/bdware/doip/cluster/callback/BDODelegateDoipMessageCallback.java +++ b/src/main/java/org/bdware/doip/cluster/callback/BDODelegateDoipMessageCallback.java @@ -11,7 +11,7 @@ import org.bdware.doip.endpoint.client.DoipMessageCallback; import java.net.URI; public class BDODelegateDoipMessageCallback implements DoipMessageCallback { - static Logger LOGGER = LogManager.getLogger(ClusterClientDoipMessageCallback.class); + static Logger LOGGER = LogManager.getLogger(BDODelegateDoipMessageCallback.class); private final DoipMessageCallback originalCallback; DoipMessage originalMessage; AuditDoipClient previousClient; @@ -39,7 +39,7 @@ public class BDODelegateDoipMessageCallback implements DoipMessageCallback { version = msg.header.parameters.attributes.get("version").getAsString(); LOGGER.info("recv delegate:"+newAddress +" previousRepo:"+previousClient.getRepoUrl()); // BDRepo doipMsgCallback -> BDOClient，维护repoToBDO的映射关系 - AuditDoipClient client = AuditDoipClientCacheUtil.getClientByUrl(newAddress, version); + AuditDoipClient client = AuditDoipClientCacheUtil.getFastClientByUrl(previousClient.getRepoUrl(), newAddress, version); previousClient = client; client.sendRawMessage(originalMessage, this); } diff --git a/src/main/java/org/bdware/doip/cluster/callback/ClusterClientDoipMessageCallback.java b/src/main/java/org/bdware/doip/cluster/callback/ClusterClientDoipMessageCallback.java deleted file mode 100644 index 656972f..0000000 --- a/src/main/java/org/bdware/doip/cluster/callback/ClusterClientDoipMessageCallback.java +++ /dev/null @@ -1,54 +0,0 @@ -package org.bdware.doip.cluster.callback; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.bdware.doip.audit.client.AuditDoipClient; -import org.bdware.doip.cluster.client.DoipClusterClient; -import org.bdware.doip.codec.doipMessage.DoipMessage; -import org.bdware.doip.codec.doipMessage.DoipResponseCode; -import org.bdware.doip.endpoint.client.DoipMessageCallback; - -import java.net.URI; - -public class ClusterClientDoipMessageCallback implements DoipMessageCallback { - static Logger LOGGER = LogManager.getLogger(ClusterClientDoipMessageCallback.class); - private final DoipMessageCallback originalCallback; - DoipMessage originalMessage; - AuditDoipClient previousClient; - DoipClusterClient doipClusterClient; - - public ClusterClientDoipMessageCallback(AuditDoipClient previousClient, DoipClusterClient doipClusterClient, DoipMessageCallback doipMessageCallback, DoipMessage originalMessage) { - this.previousClient = previousClient; - this.originalCallback = doipMessageCallback; - this.originalMessage = originalMessage; - this.doipClusterClient = doipClusterClient; - } - - @Override - public void onResult(DoipMessage msg) { - if (msg.header.parameters.response == DoipResponseCode.DelegateRequired) { - - try { - if (msg.header.parameters.attributes.has("port")) { - int port = msg.header.parameters.attributes.get("port").getAsInt(); - URI uri = new URI(previousClient.getRepoUrl()); - String host = uri.getHost(); - if (msg.header.parameters.attributes.has("host")) - host = msg.header.parameters.attributes.get("host").getAsString(); - String newAddress = String.format("%s://%s:%d", uri.getScheme(), host, port); - String version = "2.1"; - if (msg.header.parameters.attributes.has("version")) - version = msg.header.parameters.attributes.get("version").getAsString(); - LOGGER.info("recv delegate:"+newAddress +" previousRepo:"+previousClient.getRepoUrl()); - // BDRepo doipMsgCallback -> BDOClient，维护repoToBDO的映射关系 - AuditDoipClient client = doipClusterClient.getFastClientByUrl(previousClient.getRepoUrl(),newAddress, version); - previousClient = client; - client.sendRawMessage(originalMessage, this); - } - } catch (Exception e) { - e.printStackTrace(); - originalCallback.onResult(msg); - } - } else originalCallback.onResult(msg); - } -} diff --git a/src/main/java/org/bdware/doip/cluster/callback/ClusterClientResultCollector.java b/src/main/java/org/bdware/doip/cluster/callback/ClusterClientResultCollector.java deleted file mode 100644 index 051953e..0000000 --- a/src/main/java/org/bdware/doip/cluster/callback/ClusterClientResultCollector.java +++ /dev/null @@ -1,204 +0,0 @@ -package org.bdware.doip.cluster.callback; - -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.bdware.doip.cluster.client.DoipClusterClient; -import org.bdware.doip.cluster.util.ConnectionUtil; -import org.bdware.doip.codec.JsonDoipMessage; -import org.bdware.doip.codec.doipMessage.DoipMessage; -import org.bdware.doip.codec.doipMessage.DoipMessageFactory; -import org.bdware.doip.codec.doipMessage.DoipResponseCode; -import org.bdware.doip.endpoint.client.DoipMessageCallback; -import org.bdware.sc.bean.JoinInfo; -import org.bdware.sc.util.JsonUtil; -import wrp.jdk.nashorn.api.scripting.NashornScriptEngineUtil; - -import java.io.ByteArrayOutputStream; -import java.io.PrintStream; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; - -public class ClusterClientResultCollector implements DoipMessageCallback { - static Logger LOGGER = LogManager.getLogger(ClusterClientResultCollector.class); - private final DoipMessage request; - // joinCount记录开始Merge的上限 - int joinCount; - AtomicInteger count = new AtomicInteger(0); - DoipMessageCallback originalCallback; - Logger Logger = LogManager.getLogger(ClusterClientResultCollector.class); - Map successResMap = new HashMap<>(); - Set problemResult = new HashSet<>(); - // 记录所有收到结果列表 - Set nodeIDs = new HashSet<>(); - NashornScriptEngineUtil engineUtil; - JoinInfo joinInfo; - String[] targetServers; - DoipClusterClient doipClusterClient; - JsonObject appendixes; - - - public ClusterClientResultCollector(DoipClusterClient doipClusterClient, String[] targetServers, DoipMessage msg, DoipMessageCallback cb, int count, NashornScriptEngineUtil engineUtil, JoinInfo joinInfo, JsonObject appendixes) { - this.request = msg; - this.originalCallback = cb; - this.joinCount = count; - this.engineUtil = engineUtil; - this.joinInfo = joinInfo; - this.targetServers = targetServers; - this.doipClusterClient = doipClusterClient; - this.appendixes = appendixes; - } - - @Override - public void onResult(DoipMessage msg) { - logDoipMsgInfo(msg); - // 如果nodeID不存在，或者该节点的返回值已经被处理过了，不需要再次调用onResult了 - String nodeID = getNodeID(msg); - if (nodeID == null) { - // 如果当前nodeID == null, 记录到problemResult中 - problemResult.add(msg); - } else if (nodeIDs.contains(nodeID)) { - LOGGER.info("Receive Duplicated Msg"); - } else { - nodeIDs.add(nodeID); - addDoipResult(nodeID, msg); - } - - int curCount = count.incrementAndGet(); - // 小于joinCount，意味着还没达到聚合的标准，需要继续处理 - if (curCount <= joinCount) { - // 达到聚合标准 - if (curCount == joinCount) { - // 完成聚合 & 结果统计 - DoipMessage finalResult = executeJoinFunc(msg); - originalCallback.onResult(finalResult); - } - } - } - - public DoipMessageCallback getOriginalCallback() { - return originalCallback; - } - - public void logDoipMsgInfo(DoipMessage msg) { - try { - String nodeID = getNodeID(msg); - if (nodeID != null) { - popReconnectRequest(); - } else { - //Logger.error("收到错误的节点回复 : " + "Something wrong happens in" + new Gson().toJson(targetServers) + " msg:" + new Gson().toJson(msg.header.parameters)); - pushReconnectRequest(); - } - } catch (Exception e) { - e.printStackTrace(); - } - } - - private void popReconnectRequest() { - ConnectionUtil.popReconnectRequest(targetServers); - } - - private void pushReconnectRequest() { - ConnectionUtil.pushReconnectRequest(targetServers, appendixes); - } - - public synchronized void addDoipResult(String nodeID, DoipMessage msg) { - successResMap.put(nodeID, msg); - } - - - public String getNodeID(DoipMessage msg) { - String nodeID = null; - if (msg != null && msg.header != null && msg.header.parameters != null && msg.header.parameters.attributes != null) { - if (msg.header.parameters.attributes.get("nodeID") != null) { - nodeID = msg.header.parameters.attributes.get("nodeID").getAsString(); - } - } - - return nodeID; - } - - public DoipMessage executeJoinFunc(DoipMessage request) { - DoipMessage resultMessage; - if (joinInfo != null && problemResult.size() == 0) { - resultMessage = handleJoinInfo(joinInfo); - return resultMessage; - } - if (successResMap.size() + problemResult.size() == 1) { - problemResult.addAll(successResMap.values()); - for (DoipMessage message : problemResult) - return message; - } - - DoipMessageFactory.DoipMessageBuilder builder = new DoipMessageFactory.DoipMessageBuilder(); - if (problemResult.size() > 0) - builder.createResponse(DoipResponseCode.UnKnownError, request); - else - builder.createResponse(DoipResponseCode.Success, request); - JsonObject successResponses = new JsonObject(); - for (String key : successResMap.keySet()) { - successResponses.add(key, JsonUtil.parseObject(JsonDoipMessage.fromDoipMessage(successResMap.get(key)))); - } - JsonArray problemResponses = new JsonArray(); - for (DoipMessage msg : problemResult) { - problemResponses.add(JsonUtil.parseObject(JsonDoipMessage.fromDoipMessage(msg))); - } - builder.addAttributes("successResponses", successResponses); - builder.addAttributes("problemResponses", problemResponses); - return builder.create(); - } - - private DoipMessage handleJoinInfo(JoinInfo joinInfo) { - try { - if (joinInfo.useDefault == null) { - if (joinInfo.joinFuncName != null) { - JsonObject jo = new JsonObject(); - for (String key : successResMap.keySet()) { - jo.add(key, JsonUtil.parseObject(JsonDoipMessage.fromDoipMessage(successResMap.get(key)))); - } - JsonArray array = new JsonArray(); - for (DoipMessage resp : problemResult) { - array.add(JsonUtil.parseObject(JsonDoipMessage.fromDoipMessage(resp))); - } - JsonElement returnVal = engineUtil.invokeFunction(joinInfo.joinFuncName, JsonElement.class, jo, array); - JsonDoipMessage result = JsonUtil.fromJson(returnVal, JsonDoipMessage.class); - return result.toResponseDoipMessage(request); - } else throw new IllegalArgumentException("missing joinFuncName"); - } - DoipMessage returnedMsg = null; - switch (joinInfo.useDefault) { - case add: - double val = 0; - for (DoipMessage msg : successResMap.values()) { - val += Double.parseDouble(msg.body.getDataAsJsonString()); - returnedMsg = msg; - } - assert returnedMsg != null; - returnedMsg.body.encodedData = (val + "").getBytes(); - return returnedMsg; - case multiply: - val = 1; - for (DoipMessage msg : successResMap.values()) { - val *= Double.parseDouble(msg.body.getDataAsJsonString()); - returnedMsg = msg; - } - return returnedMsg; - default: - throw new IllegalArgumentException("missing default merge rule"); - } - } catch (Exception e) { - DoipMessageFactory.DoipMessageBuilder builder = new DoipMessageFactory.DoipMessageBuilder(); - builder.createResponse(DoipResponseCode.Declined, request); - ByteArrayOutputStream bo = new ByteArrayOutputStream(); - e.printStackTrace(new PrintStream(bo)); - builder.setBody(bo.toByteArray()); - e.printStackTrace(); - return builder.create(); - } - } -} diff --git a/src/main/java/org/bdware/doip/cluster/callback/DDODelegateDoipMessageCallback.java b/src/main/java/org/bdware/doip/cluster/callback/DDODelegateDoipMessageCallback.java index 4ec42bd..f8a904b 100644 --- a/src/main/java/org/bdware/doip/cluster/callback/DDODelegateDoipMessageCallback.java +++ b/src/main/java/org/bdware/doip/cluster/callback/DDODelegateDoipMessageCallback.java @@ -7,11 +7,11 @@ import com.google.gson.JsonObject; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.bdware.doip.cluster.util.ConnectionUtil; -import org.bdware.doip.cluster.client.TreeClusterClient; import org.bdware.doip.codec.JsonDoipMessage; import org.bdware.doip.codec.doipMessage.DoipMessage; import org.bdware.doip.codec.doipMessage.DoipMessageFactory; import org.bdware.doip.codec.doipMessage.DoipResponseCode; +import org.bdware.doip.endpoint.client.DoipClientImpl; import org.bdware.doip.endpoint.client.DoipMessageCallback; import org.bdware.sc.bean.JoinInfo; import org.bdware.sc.util.JsonUtil; @@ -39,18 +39,18 @@ public class DDODelegateDoipMessageCallback implements DoipMessageCallback { NashornScriptEngineUtil engineUtil; JoinInfo joinInfo; String[] targetServers; - TreeClusterClient treeClusterClient; + DoipClientImpl doipClientImpl; JsonObject appendixes; - public DDODelegateDoipMessageCallback(TreeClusterClient treeClusterClient, String[] targetServers, DoipMessage msg, DoipMessageCallback cb, int count, NashornScriptEngineUtil engineUtil, JoinInfo joinInfo, JsonObject appendixes) { + public DDODelegateDoipMessageCallback(DoipClientImpl doipClientImpl, String[] targetServers, DoipMessage msg, DoipMessageCallback cb, int count, NashornScriptEngineUtil engineUtil, JoinInfo joinInfo, JsonObject appendixes) { this.request = msg; this.originalCallback = cb; this.joinCount = count; this.engineUtil = engineUtil; this.joinInfo = joinInfo; this.targetServers = targetServers; - this.treeClusterClient = treeClusterClient; + this.doipClientImpl = doipClientImpl; this.appendixes = appendixes; } diff --git a/src/main/java/org/bdware/doip/cluster/client/DOAClient.java b/src/main/java/org/bdware/doip/cluster/client/DOAClient.java index d498eea..0e9d27f 100644 --- a/src/main/java/org/bdware/doip/cluster/client/DOAClient.java +++ b/src/main/java/org/bdware/doip/cluster/client/DOAClient.java @@ -5,7 +5,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.bdware.doip.audit.EndpointConfig; import org.bdware.doip.audit.client.AuditIrpClient; -import org.bdware.doip.cluster.callback.ClusterClientResultCollector; +import org.bdware.doip.cluster.callback.DDODelegateDoipMessageCallback; import org.bdware.doip.codec.digitalObject.DigitalObject; import org.bdware.doip.codec.doipMessage.DoipMessage; import org.bdware.doip.codec.doipMessage.DoipMessageSigner; @@ -369,30 +369,30 @@ public class DOAClient extends DoipClientImpl { if (count > 0) { //TODO//TODO//TODO//TODO//TODO//TODO//TODO//TODO//TODO - ClusterClientResultCollector clusterClientResultCollector = new ClusterClientResultCollector(null, null, null, cb, count, engineUtil, joinInfo, null); + DDODelegateDoipMessageCallback ddoDelegateDoipMessageCallback = new DDODelegateDoipMessageCallback(null, null, null, cb, count, engineUtil, joinInfo, null); for (String targetDoipServer : targetDoipServers) { DoipClientImpl doipClientImpl = getDoipClientForServer(targetDoipServer); switch (doipParam.operation) { case Hello: - doipClientImpl.hello(doipParam.identifier, clusterClientResultCollector); + doipClientImpl.hello(doipParam.identifier, ddoDelegateDoipMessageCallback); break; case ListOps: - doipClientImpl.listOperations(doipParam.identifier, clusterClientResultCollector); + doipClientImpl.listOperations(doipParam.identifier, ddoDelegateDoipMessageCallback); break; case Retrieve: - doipClientImpl.retrieve(doipParam.identifier, doipParam.element, doipParam.includeElementData, clusterClientResultCollector); + doipClientImpl.retrieve(doipParam.identifier, doipParam.element, doipParam.includeElementData, ddoDelegateDoipMessageCallback); break; case Create: - doipClientImpl.create(doipParam.identifier, doipParam.digitalObject, clusterClientResultCollector); + doipClientImpl.create(doipParam.identifier, doipParam.digitalObject, ddoDelegateDoipMessageCallback); break; case Update: - doipClientImpl.update(doipParam.digitalObject, clusterClientResultCollector); + doipClientImpl.update(doipParam.digitalObject, ddoDelegateDoipMessageCallback); break; case Delete: - doipClientImpl.delete(doipParam.identifier, clusterClientResultCollector); + doipClientImpl.delete(doipParam.identifier, ddoDelegateDoipMessageCallback); break; case Search: - doipClientImpl.search(doipParam.identifier, doipParam.sp, clusterClientResultCollector); + doipClientImpl.search(doipParam.identifier, doipParam.sp, ddoDelegateDoipMessageCallback); break; } } diff --git a/src/main/java/org/bdware/doip/cluster/client/DoipClusterClient.java b/src/main/java/org/bdware/doip/cluster/client/DoipClusterClient.java index 73c0a72..25c425f 100644 --- a/src/main/java/org/bdware/doip/cluster/client/DoipClusterClient.java +++ b/src/main/java/org/bdware/doip/cluster/client/DoipClusterClient.java @@ -4,7 +4,6 @@ import com.google.gson.*; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.bdware.doip.audit.EndpointConfig; -import org.bdware.doip.audit.client.AuditDoipClient; import org.bdware.doip.audit.client.AuditIrpClient; import org.bdware.doip.cluster.entity.DoipClusterRequestPack; import org.bdware.doip.cluster.flowcontrol.FlowControl; @@ -35,7 +34,6 @@ import java.math.BigInteger; import java.util.HashMap; import java.util.Map; import java.util.Queue; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedDeque; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -53,7 +51,6 @@ public class DoipClusterClient extends DoipClientImpl { protected final Map doipOperationToJoinInfo; // 维护了地址 与 doipPort之间的映射关系 // BDO和DoipClusterClient是一一对应的关系，一个BDO对应一组BDRepo，一个BDRepo对应了一个DoipServer端口，因此这里维护Repo -> 端口的映射 - private final Map bdRepoToBDOClient; private final static Queue sendMessageTaskQueue = new ConcurrentLinkedDeque<>(); private final FlowControl flowControl; Logger Logger = LogManager.getLogger(DoipClusterClient.class); @@ -88,7 +85,6 @@ public class DoipClusterClient extends DoipClientImpl { engineUtil = new NashornScriptEngineUtil(); doipOperationToRouteInfo = new HashMap<>(); doipOperationToJoinInfo = new HashMap<>(); - bdRepoToBDOClient = new ConcurrentHashMap<>(); routerTool = new RouterTool(irsClient, engineUtil); // todo optimize flowControl // flowControl = new TokenBucket(10240, 2048); @@ -156,14 +152,13 @@ public class DoipClusterClient extends DoipClientImpl { public void sendMessageInternal(DoipClusterRequestPack pack) { try { - RouteJoinUtil.bdosSendMsgAndJoin( + RouteJoinUtil.bdoSendMsgAndJoin( this, pack.getMessage(), pack.getJoinInfo(), pack.getBdRepoIds(), pack.getCb(), doipMessageSigner, - bdRepoToBDOClient, engineUtil, routerConfig.publicKey, appendixes @@ -176,19 +171,6 @@ public class DoipClusterClient extends DoipClientImpl { } } - public AuditDoipClient getFastClientByUrl(String previousAddress, String address, String version) { - AuditDoipClient existingConn = bdRepoToBDOClient.get(previousAddress); - if (existingConn != null) { - return existingConn; - } - AuditDoipClient newGetClient = AuditDoipClientCacheUtil.getClientByUrl(address, version); - if (newGetClient == null) { - return null; - } - existingConn = bdRepoToBDOClient.putIfAbsent(previousAddress, newGetClient); - return existingConn == null ? newGetClient : existingConn; - } - static Logger LOGGER = LogManager.getLogger(DoipClusterClient.class); // 根据请求参数，执行RouteInfo，寻找DoipServers @@ -278,19 +260,6 @@ public class DoipClusterClient extends DoipClientImpl { @Override public void close() { - for (AuditDoipClient client : bdRepoToBDOClient.values()) - try { - client.close(); - } catch (Exception e) { - e.printStackTrace(); - } - - bdRepoToBDOClient.clear(); - - } - - public void closeAll() { - close(); AuditDoipClientCacheUtil.closeAll(); } } diff --git a/src/main/java/org/bdware/doip/cluster/client/TreeClusterClient.java b/src/main/java/org/bdware/doip/cluster/client/TreeClusterClient.java index 45e793a..420b4d4 100644 --- a/src/main/java/org/bdware/doip/cluster/client/TreeClusterClient.java +++ b/src/main/java/org/bdware/doip/cluster/client/TreeClusterClient.java @@ -8,7 +8,10 @@ import org.bdware.doip.audit.EndpointConfig; import org.bdware.doip.audit.client.AuditDoipClient; import org.bdware.doip.audit.client.AuditIrpClient; import org.bdware.doip.cluster.callback.BDODelegateDoipMessageCallback; -import org.bdware.doip.cluster.entity.*; +import org.bdware.doip.cluster.entity.BDOEntity; +import org.bdware.doip.cluster.entity.BDWType; +import org.bdware.doip.cluster.entity.DDOEntity; +import org.bdware.doip.cluster.entity.TreeClusterRequestPack; import org.bdware.doip.cluster.flowcontrol.FlowControl; import org.bdware.doip.cluster.flowcontrol.core.RateThresholdFlowControl; import org.bdware.doip.cluster.util.AuditDoipClientCacheUtil; @@ -158,8 +161,8 @@ public class TreeClusterClient extends DoipClientImpl { String version = bdoEntity.getVersion(); AuditDoipClient doipClient = AuditDoipClientCacheUtil.getClientByUrl(address, version); - assert doipClient != null; - doipMessageSigner.signMessage(doipMessage); + if(doipMessageSigner != null) + doipMessageSigner.signMessage(doipMessage); BDODelegateDoipMessageCallback BDODelegateDoipMessageCallback = new BDODelegateDoipMessageCallback(doipClient, cb, doipMessage); doipClient.sendMessage(doipMessage, BDODelegateDoipMessageCallback); } @@ -168,6 +171,7 @@ public class TreeClusterClient extends DoipClientImpl { JsonDoipMessage doipParam = JsonDoipMessage.fromDoipMessage(doipMessage); BasicOperations operation = BasicOperations.getDoOp(doipParam.header.operation); String[] toCalls = RouteJoinUtil.route(doipParam, routerConfig, ddoEntity.getDDORouteInfo().get(operation.getName()), ddoEntity.getEngineUtil()); + assert toCalls != null; RouteJoinUtil.ddoSendMsgAndJoin(this, ddoEntity.getEngineUtil(), doipMessage, ddoEntity.getDDOJoinInfo().get(operation.getName()), toCalls, cb, ddoEntity.getAppendixes()); } diff --git a/src/main/java/org/bdware/doip/cluster/util/AuditDoipClientCacheUtil.java b/src/main/java/org/bdware/doip/cluster/util/AuditDoipClientCacheUtil.java index 23a4db0..e9fc224 100644 --- a/src/main/java/org/bdware/doip/cluster/util/AuditDoipClientCacheUtil.java +++ b/src/main/java/org/bdware/doip/cluster/util/AuditDoipClientCacheUtil.java @@ -7,6 +7,7 @@ import java.util.concurrent.ConcurrentHashMap; public class AuditDoipClientCacheUtil { public static final Map clientCache = new ConcurrentHashMap<>(); + public static final Map fastClientCache = new ConcurrentHashMap<>(); public static AuditDoipClient getClientByUrl(String address, String version) { if (address == null && version == null) return null; @@ -30,6 +31,19 @@ public class AuditDoipClientCacheUtil { return doipClientImpl; } + public static AuditDoipClient getFastClientByUrl(String previousAddress, String address, String version) { + AuditDoipClient existingConn = fastClientCache.get(previousAddress); + if (existingConn != null) { + return existingConn; + } + AuditDoipClient newGetClient = getClientByUrl(address, version); + if (newGetClient == null) { + return null; + } + existingConn = fastClientCache.putIfAbsent(previousAddress, newGetClient); + return existingConn == null ? newGetClient : existingConn; + } + public static void closeAll() { for (AuditDoipClient client : clientCache.values()) { try { @@ -38,6 +52,16 @@ public class AuditDoipClientCacheUtil { e.printStackTrace(); } } + + for (AuditDoipClient client : fastClientCache.values()) { + try { + if (client.isConnected()) client.close(); + } catch (Exception e) { + e.printStackTrace(); + } + } + clientCache.clear(); + fastClientCache.clear(); } } diff --git a/src/main/java/org/bdware/doip/cluster/util/RouteJoinUtil.java b/src/main/java/org/bdware/doip/cluster/util/RouteJoinUtil.java index 245e6ac..ea30208 100644 --- a/src/main/java/org/bdware/doip/cluster/util/RouteJoinUtil.java +++ b/src/main/java/org/bdware/doip/cluster/util/RouteJoinUtil.java @@ -8,8 +8,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.bdware.doip.audit.EndpointConfig; import org.bdware.doip.audit.client.AuditDoipClient; -import org.bdware.doip.cluster.callback.ClusterClientDoipMessageCallback; -import org.bdware.doip.cluster.callback.ClusterClientResultCollector; +import org.bdware.doip.cluster.callback.BDODelegateDoipMessageCallback; import org.bdware.doip.cluster.callback.DDODelegateDoipMessageCallback; import org.bdware.doip.cluster.client.DoipClusterClient; import org.bdware.doip.cluster.client.TreeClusterClient; @@ -23,7 +22,6 @@ import org.bdware.sc.util.JsonUtil; import wrp.jdk.nashorn.api.scripting.NashornScriptEngineUtil; import java.math.BigInteger; -import java.util.Map; public class RouteJoinUtil { public static final Logger LOGGER = LogManager.getLogger(DoipClusterClient.class); @@ -48,14 +46,14 @@ public class RouteJoinUtil { case byArgHash: val = JsonUtil.toJson(doipParam).hashCode(); val = val % BDRepoIDs.length; - while (val < 0 && BDRepoIDs.length > 0) { + while (val < 0) { val += BDRepoIDs.length; } return new String[]{BDRepoIDs[val]}; case byJsonPropHash: JsonElement jo = tryLoadJsonProp(JsonUtil.parseObject(doipParam).getAsJsonObject(), routeInfo.param); val = jo.toString().hashCode() % BDRepoIDs.length; - while (val < 0 && BDRepoIDs.length > 0) { + while (val < 0) { val += BDRepoIDs.length; } return new String[]{BDRepoIDs[val]}; @@ -115,14 +113,15 @@ public class RouteJoinUtil { // DoipMessage toSend = msg.deepCopy(); // 要么response wait中的waitObj改为非static，并加入一点流控制机制。 // 这里的doipClient为BDRepoClient或者BDOClient，无所谓，如果是BDRepoClient，DelegateDoipMessageCallback会出手转为BDOClient - treeClusterClient.sendMessage(msg, toCall, ddoDelegateDoipMessageCallback); + DoipMessage toCallDoipMessage = msg.withNewDOID(msg, toCall); + treeClusterClient.sendMessage(toCallDoipMessage, ddoDelegateDoipMessageCallback); } } else { LOGGER.error("joinCount is not a positive num"); } } - public static void bdosSendMsgAndJoin(DoipClusterClient doipClusterClient, DoipMessage msg, JoinInfo joinInfo, String[] BDRepoIDs, DoipMessageCallback cb, DoipMessageSigner doipMessageSigner, Map bdRepoToBDOClient, NashornScriptEngineUtil engineUtil, String publicKey, JsonObject appendixes) { + public static void bdoSendMsgAndJoin(DoipClusterClient doipClusterClient, DoipMessage msg, JoinInfo joinInfo, String[] BDRepoIDs, DoipMessageCallback cb, DoipMessageSigner doipMessageSigner, NashornScriptEngineUtil engineUtil, String publicKey, JsonObject appendixes) { int serversNum = BDRepoIDs.length; // todo optimize joinParams int count = RouteJoinUtil.getJoinCount(publicKey, engineUtil, joinInfo, serversNum, new JsonObject()); @@ -132,12 +131,12 @@ public class RouteJoinUtil { } if (count > 0) { - ClusterClientResultCollector clusterClientResultCollector = new ClusterClientResultCollector(doipClusterClient, BDRepoIDs, msg, cb, count, engineUtil, joinInfo, appendixes); + DDODelegateDoipMessageCallback ddoDelegateDoipMessageCallback = new DDODelegateDoipMessageCallback(doipClusterClient, BDRepoIDs, msg, cb, count, engineUtil, joinInfo, appendixes); for (String BDRepoId : BDRepoIDs) { // 如果已经建立过BDRepo -> BDO的映射，可以通过BDRepo的地址，直接找到BDO的Client，对其进行操作（本质BDRepo和BDO是一一对应的关系） String address = appendixes.get(BDRepoId).getAsJsonObject().get("address").getAsString(); String version = appendixes.get(BDRepoId).getAsJsonObject().get("version").getAsString(); - AuditDoipClient doipClient = getFastClientByUrl(bdRepoToBDOClient, address, null, null); + AuditDoipClient doipClient = AuditDoipClientCacheUtil.getFastClientByUrl(address, null, null); if (doipClient == null) doipClient = AuditDoipClientCacheUtil.getClientByUrl(address, version); if (doipMessageSigner != null) { @@ -147,7 +146,7 @@ public class RouteJoinUtil { // DoipMessage toSend = msg.deepCopy(); // 要么response wait中的waitObj改为非static，并加入一点流控制机制。 // 这里的doipClient为BDRepoClient或者BDOClient，无所谓，如果是BDRepoClient，DelegateDoipMessageCallback会出手转为BDOClient - ClusterClientDoipMessageCallback clusterClientDoipMessageCallback = new ClusterClientDoipMessageCallback(doipClient, doipClusterClient, clusterClientResultCollector, msg); + BDODelegateDoipMessageCallback clusterClientDoipMessageCallback = new BDODelegateDoipMessageCallback(doipClient, ddoDelegateDoipMessageCallback, msg); doipClient.sendMessage(msg, clusterClientDoipMessageCallback); } } else { @@ -155,19 +154,6 @@ public class RouteJoinUtil { } } - public static AuditDoipClient getFastClientByUrl(Map bdRepoToBDOClient, String previousAddress, String address, String version) { - AuditDoipClient existingConn = bdRepoToBDOClient.get(previousAddress); - if (existingConn != null) { - return existingConn; - } - AuditDoipClient newGetClient = AuditDoipClientCacheUtil.getClientByUrl(address, version); - if (newGetClient == null) { - return null; - } - existingConn = bdRepoToBDOClient.putIfAbsent(previousAddress, newGetClient); - return existingConn == null ? newGetClient : existingConn; - } - public static int getJoinCount(EndpointConfig routerConfig, NashornScriptEngineUtil engineUtil, JoinInfo joinInfo, int serversNum, JsonObject joinParams) { try { if (joinInfo == null) return serversNum; -- Gitee From a81c1ac2798e83b14c9130b42d8d3e01cb75c193 Mon Sep 17 00:00:00 2001 From: "haoeliu@foxmail.com" Date: Sat, 15 Jul 2023 20:57:06 +0800 Subject: [PATCH 13/14] Import lombok and use annotationProcessor to simplify code --- build.gradle | 2 + .../bdware/doip/cluster/ClusterDoaClient.java | 100 ++++++++---------- .../cluster/client/DoipClusterClient.java | 4 +- .../cluster/client/TreeClusterClient.java | 14 +-- .../bdware/doip/cluster/entity/BDOEntity.java | 30 ++---- .../bdware/doip/cluster/entity/DDOEntity.java | 47 ++------ .../entity/DoipClusterRequestPack.java | 51 ++------- .../entity/TreeClusterRequestPack.java | 61 ++--------- 8 files changed, 87 insertions(+), 222 deletions(-) diff --git a/build.gradle b/build.gradle index e9618bb..b1f5afd 100644 --- a/build.gradle +++ b/build.gradle @@ -53,6 +53,8 @@ dependencies { implementation 'org.bouncycastle:bcpkix-jdk15on:1.69' implementation 'org.bouncycastle:bcprov-jdk15on:1.69' implementation 'com.nimbusds:nimbus-jose-jwt:9.10' + implementation 'org.projectlombok:lombok:1.18.28' + annotationProcessor 'org.projectlombok:lombok:1.18.28' //---- repo example implementation 'mysql:mysql-connector-java:8.0.33' implementation 'org.rocksdb:rocksdbjni:6.22.1' diff --git a/src/main/java/org/bdware/doip/cluster/ClusterDoaClient.java b/src/main/java/org/bdware/doip/cluster/ClusterDoaClient.java index 0820d8e..c3ecf35 100644 --- a/src/main/java/org/bdware/doip/cluster/ClusterDoaClient.java +++ b/src/main/java/org/bdware/doip/cluster/ClusterDoaClient.java @@ -16,10 +16,8 @@ import org.bdware.doip.codec.metadata.SearchParameter; import org.bdware.doip.codec.operations.BasicOperations; import org.bdware.doip.endpoint.client.DoipClientImpl; import org.bdware.doip.endpoint.client.DoipMessageCallback; -import org.bdware.irp.exception.IrpClientException; import org.bdware.sc.bean.JoinInfo; -import javax.script.ScriptException; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -83,68 +81,56 @@ public class ClusterDoaClient extends DoipClientImpl { } public synchronized DoipClusterClient createOrGetClusterClient(String bdoid) { - try { - if (!bdoClients.containsKey(bdoid)) { - DoipClusterClient doipClusterClient = new DoipClusterClient(routerConfig); - bdoClients.put(bdoid, doipClusterClient); - } - return bdoClients.get(bdoid); - } catch (IrpClientException | ScriptException e) { - e.printStackTrace(); + if (!bdoClients.containsKey(bdoid)) { + DoipClusterClient doipClusterClient = new DoipClusterClient(routerConfig); + bdoClients.put(bdoid, doipClusterClient); } - - return null; + return bdoClients.get(bdoid); } public DoipClusterClient createOrGetClusterClientLocalTest(String bdoid) { - try { - if (!bdoClients.containsKey(bdoid)) { - DoipClusterClient doipClusterClient = new DoipClusterClient(routerConfig) { - @Override - public void sendMessage(DoipMessage doipMessage, DoipMessageCallback cb) { - - BasicOperations op = BasicOperations.getDoOp(doipMessage.header.parameters.operation); - JsonDoipMessage message = JsonDoipMessage.fromDoipMessage(doipMessage); - String[] BDRepoIDs = route(message); - // joinParams都来自于callback，joinParams就是所有节点callback到达之后，merge之后的结果 - JoinInfo joinInfo = this.doipOperationToJoinInfo.get(op.getName()); - sendMsgAndJoin(doipMessage, joinInfo, BDRepoIDs, cb); - } - - public void sendMsgAndJoin(DoipMessage msg, JoinInfo joinInfo, String[] BDRepoIDs, DoipMessageCallback cb) { - int serversNum = BDRepoIDs.length; - int count = getJoinCount(joinInfo, serversNum, new JsonObject()); - // 如果joinCountFunc执行算出的joinCount，比所有的servers还要多，就更新为servers的数量 - if (serversNum < count) { - count = serversNum; - } - DoipMessageFactory.DoipMessageBuilder builder = new DoipMessageFactory.DoipMessageBuilder(); - builder.createResponse(DoipResponseCode.Success, msg); - builder.setBody("success".getBytes()); - cb.onResult(builder.create()); + if (!bdoClients.containsKey(bdoid)) { + DoipClusterClient doipClusterClient = new DoipClusterClient(routerConfig) { + @Override + public void sendMessage(DoipMessage doipMessage, DoipMessageCallback cb) { + + BasicOperations op = BasicOperations.getDoOp(doipMessage.header.parameters.operation); + JsonDoipMessage message = JsonDoipMessage.fromDoipMessage(doipMessage); + String[] BDRepoIDs = route(message); + // joinParams都来自于callback，joinParams就是所有节点callback到达之后，merge之后的结果 + JoinInfo joinInfo = this.doipOperationToJoinInfo.get(op.getName()); + sendMsgAndJoin(doipMessage, joinInfo, BDRepoIDs, cb); + } + + public void sendMsgAndJoin(DoipMessage msg, JoinInfo joinInfo, String[] BDRepoIDs, DoipMessageCallback cb) { + int serversNum = BDRepoIDs.length; + int count = getJoinCount(joinInfo, serversNum, new JsonObject()); + // 如果joinCountFunc执行算出的joinCount，比所有的servers还要多，就更新为servers的数量 + if (serversNum < count) { + count = serversNum; } - - int getJoinCount(JoinInfo joinInfo, int serversNum, JsonObject joinParams) { - try { - if (joinInfo == null) return serversNum; - if (joinInfo.joinCountFuncName != null) { - JsonElement requester = routerConfig.publicKey == null ? JsonNull.INSTANCE : new JsonPrimitive(routerConfig.publicKey); - return engineUtil.invokeFunction(joinInfo.joinCountFuncName, Integer.class, joinParams, requester); - } - if (joinInfo.joinCount != 0) return joinInfo.joinCount; - } catch (Exception e) { - e.printStackTrace(); + DoipMessageFactory.DoipMessageBuilder builder = new DoipMessageFactory.DoipMessageBuilder(); + builder.createResponse(DoipResponseCode.Success, msg); + builder.setBody("success".getBytes()); + cb.onResult(builder.create()); + } + + int getJoinCount(JoinInfo joinInfo, int serversNum, JsonObject joinParams) { + try { + if (joinInfo == null) return serversNum; + if (joinInfo.joinCountFuncName != null) { + JsonElement requester = routerConfig.publicKey == null ? JsonNull.INSTANCE : new JsonPrimitive(routerConfig.publicKey); + return engineUtil.invokeFunction(joinInfo.joinCountFuncName, Integer.class, joinParams, requester); } - return serversNum; + if (joinInfo.joinCount != 0) return joinInfo.joinCount; + } catch (Exception e) { + e.printStackTrace(); } - }; - bdoClients.put(bdoid, doipClusterClient); - } - return bdoClients.get(bdoid); - } catch (IrpClientException | ScriptException e) { - e.printStackTrace(); + return serversNum; + } + }; + bdoClients.put(bdoid, doipClusterClient); } - - return null; + return bdoClients.get(bdoid); } } diff --git a/src/main/java/org/bdware/doip/cluster/client/DoipClusterClient.java b/src/main/java/org/bdware/doip/cluster/client/DoipClusterClient.java index 25c425f..9662795 100644 --- a/src/main/java/org/bdware/doip/cluster/client/DoipClusterClient.java +++ b/src/main/java/org/bdware/doip/cluster/client/DoipClusterClient.java @@ -20,7 +20,6 @@ import org.bdware.doip.codec.operations.BasicOperations; import org.bdware.doip.encrypt.SM2Signer; import org.bdware.doip.endpoint.client.DoipClientImpl; import org.bdware.doip.endpoint.client.DoipMessageCallback; -import org.bdware.irp.exception.IrpClientException; import org.bdware.irp.irplib.exception.IrpConnectException; import org.bdware.sc.bean.JoinInfo; import org.bdware.sc.bean.RouteInfo; @@ -29,7 +28,6 @@ import org.zz.gmhelper.SM2KeyPair; import org.zz.gmhelper.SM2Util; import wrp.jdk.nashorn.api.scripting.NashornScriptEngineUtil; -import javax.script.ScriptException; import java.math.BigInteger; import java.util.HashMap; import java.util.Map; @@ -67,7 +65,7 @@ public class DoipClusterClient extends DoipClientImpl { return t; }); - public DoipClusterClient(EndpointConfig routerConfig) throws IrpClientException, ScriptException { + public DoipClusterClient(EndpointConfig routerConfig) { this.routerConfig = routerConfig; AuditIrpClient irsClient = new AuditIrpClient(routerConfig); new Thread(this::consumeSendMessageTask).start(); diff --git a/src/main/java/org/bdware/doip/cluster/client/TreeClusterClient.java b/src/main/java/org/bdware/doip/cluster/client/TreeClusterClient.java index 420b4d4..6b6b65b 100644 --- a/src/main/java/org/bdware/doip/cluster/client/TreeClusterClient.java +++ b/src/main/java/org/bdware/doip/cluster/client/TreeClusterClient.java @@ -2,6 +2,7 @@ package org.bdware.doip.cluster.client; import com.google.gson.Gson; import com.google.gson.JsonObject; +import lombok.SneakyThrows; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.bdware.doip.audit.EndpointConfig; @@ -33,7 +34,6 @@ import org.zz.gmhelper.SM2KeyPair; import org.zz.gmhelper.SM2Util; import wrp.jdk.nashorn.api.scripting.NashornScriptEngineUtil; -import javax.script.ScriptException; import java.util.Map; import java.util.Queue; import java.util.concurrent.ConcurrentHashMap; @@ -101,7 +101,8 @@ public class TreeClusterClient extends DoipClientImpl { } // 返回当前doid对应的所有的BDRepo - public void parseDORecursive(String doid, DoipMessage doipMessage, DoipMessageCallback cb) throws IrpClientException, ScriptException { + @SneakyThrows + public void parseDORecursive(String doid, DoipMessage doipMessage, DoipMessageCallback cb) { JsonObject doInfo = DOResolutionUtil.getDOInfo(irsClient, doid); BDWType doType = DOResolutionUtil.getDOType(doInfo); DoipMessageCallback flowControlCallback = msg -> { @@ -131,7 +132,8 @@ public class TreeClusterClient extends DoipClientImpl { produceSendMessageTask(requestPack); } - public DDOEntity ddoInfo2DDOEntity(String ddoID, JsonObject ddoInfo) throws ScriptException { + @SneakyThrows + public DDOEntity ddoInfo2DDOEntity(String ddoID, JsonObject ddoInfo) { DDOEntity ddoEntity = ddoCache.get(ddoID); if(ddoEntity != null) return ddoEntity; @@ -146,7 +148,7 @@ public class TreeClusterClient extends DoipClientImpl { return ddoCache.get(ddoID); } - public BDOEntity bdoInfo2BDOEntity(String bdoID, JsonObject bdoInfo) throws ScriptException { + public BDOEntity bdoInfo2BDOEntity(String bdoID, JsonObject bdoInfo) { BDOEntity bdoEntity = bdoCache.get(bdoID); if(bdoEntity != null) return bdoEntity; @@ -170,9 +172,9 @@ public class TreeClusterClient extends DoipClientImpl { public void ddoCall(DDOEntity ddoEntity, DoipMessage doipMessage, DoipMessageCallback cb) { JsonDoipMessage doipParam = JsonDoipMessage.fromDoipMessage(doipMessage); BasicOperations operation = BasicOperations.getDoOp(doipParam.header.operation); - String[] toCalls = RouteJoinUtil.route(doipParam, routerConfig, ddoEntity.getDDORouteInfo().get(operation.getName()), ddoEntity.getEngineUtil()); + String[] toCalls = RouteJoinUtil.route(doipParam, routerConfig, ddoEntity.getDdoRouteInfo().get(operation.getName()), ddoEntity.getEngineUtil()); assert toCalls != null; - RouteJoinUtil.ddoSendMsgAndJoin(this, ddoEntity.getEngineUtil(), doipMessage, ddoEntity.getDDOJoinInfo().get(operation.getName()), toCalls, cb, ddoEntity.getAppendixes()); + RouteJoinUtil.ddoSendMsgAndJoin(this, ddoEntity.getEngineUtil(), doipMessage, ddoEntity.getDdoJoinInfo().get(operation.getName()), toCalls, cb, ddoEntity.getAppendixes()); } public void sendMessageInternal(TreeClusterRequestPack pack) { diff --git a/src/main/java/org/bdware/doip/cluster/entity/BDOEntity.java b/src/main/java/org/bdware/doip/cluster/entity/BDOEntity.java index 227371a..fb081bd 100644 --- a/src/main/java/org/bdware/doip/cluster/entity/BDOEntity.java +++ b/src/main/java/org/bdware/doip/cluster/entity/BDOEntity.java @@ -1,27 +1,15 @@ package org.bdware.doip.cluster.entity; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor public class BDOEntity { private String address; private String version; - - public String getAddress() { - return address; - } - - public void setAddress(String address) { - this.address = address; - } - - public String getVersion() { - return version; - } - - public void setVersion(String version) { - this.version = version; - } - - public BDOEntity(String address, String version) { - this.address = address; - this.version = version; - } } diff --git a/src/main/java/org/bdware/doip/cluster/entity/DDOEntity.java b/src/main/java/org/bdware/doip/cluster/entity/DDOEntity.java index 547d6fc..fa52cb9 100644 --- a/src/main/java/org/bdware/doip/cluster/entity/DDOEntity.java +++ b/src/main/java/org/bdware/doip/cluster/entity/DDOEntity.java @@ -1,12 +1,20 @@ package org.bdware.doip.cluster.entity; import com.google.gson.JsonObject; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; import org.bdware.sc.bean.JoinInfo; import org.bdware.sc.bean.RouteInfo; import wrp.jdk.nashorn.api.scripting.NashornScriptEngineUtil; import java.util.Map; +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor public class DDOEntity { // hello -> routeInfo private Map ddoRouteInfo; @@ -15,43 +23,4 @@ public class DDOEntity { // script engine private NashornScriptEngineUtil engineUtil; private JsonObject appendixes; - - public DDOEntity(Map ddoRouteInfo, Map ddoJoinInfo, NashornScriptEngineUtil engineUtil, JsonObject appendixes) { - this.ddoRouteInfo = ddoRouteInfo; - this.ddoJoinInfo = ddoJoinInfo; - this.engineUtil = engineUtil; - this.appendixes = appendixes; - } - - public JsonObject getAppendixes() { - return appendixes; - } - - public void setAppendixes(JsonObject appendixes) { - this.appendixes = appendixes; - } - - public Map getDDORouteInfo() { - return ddoRouteInfo; - } - - public void setDDORouteInfo(Map ddoRouteInfo) { - this.ddoRouteInfo = ddoRouteInfo; - } - - public Map getDDOJoinInfo() { - return ddoJoinInfo; - } - - public void setDDOJoinInfo(Map ddoJoinInfo) { - this.ddoJoinInfo = ddoJoinInfo; - } - - public NashornScriptEngineUtil getEngineUtil() { - return engineUtil; - } - - public void setEngineUtil(NashornScriptEngineUtil engineUtil) { - this.engineUtil = engineUtil; - } } diff --git a/src/main/java/org/bdware/doip/cluster/entity/DoipClusterRequestPack.java b/src/main/java/org/bdware/doip/cluster/entity/DoipClusterRequestPack.java index 0eb63cb..ffaf40b 100644 --- a/src/main/java/org/bdware/doip/cluster/entity/DoipClusterRequestPack.java +++ b/src/main/java/org/bdware/doip/cluster/entity/DoipClusterRequestPack.java @@ -1,55 +1,20 @@ package org.bdware.doip.cluster.entity; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; import org.bdware.doip.codec.doipMessage.DoipMessage; import org.bdware.doip.endpoint.client.DoipMessageCallback; import org.bdware.sc.bean.JoinInfo; +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor public class DoipClusterRequestPack { - public DoipMessage getMessage() { - return message; - } - - public DoipClusterRequestPack() { - } - - public void setMessage(DoipMessage message) { - this.message = message; - } - - public JoinInfo getJoinInfo() { - return joinInfo; - } - - public void setJoinInfo(JoinInfo joinInfo) { - this.joinInfo = joinInfo; - } - - public String[] getBdRepoIds() { - return bdRepoIds; - } - - public void setBdRepoIds(String[] bdRepoIds) { - this.bdRepoIds = bdRepoIds; - } - - public DoipMessageCallback getCb() { - return cb; - } - - public void setCb(DoipMessageCallback cb) { - this.cb = cb; - } - public DoipMessage message; public JoinInfo joinInfo; - - public DoipClusterRequestPack(DoipMessage message, JoinInfo joinInfo, String[] bdRepoIds, DoipMessageCallback cb) { - this.message = message; - this.joinInfo = joinInfo; - this.bdRepoIds = bdRepoIds; - this.cb = cb; - } - public String[] bdRepoIds; public DoipMessageCallback cb; } diff --git a/src/main/java/org/bdware/doip/cluster/entity/TreeClusterRequestPack.java b/src/main/java/org/bdware/doip/cluster/entity/TreeClusterRequestPack.java index 7ef8fdc..12e0a77 100644 --- a/src/main/java/org/bdware/doip/cluster/entity/TreeClusterRequestPack.java +++ b/src/main/java/org/bdware/doip/cluster/entity/TreeClusterRequestPack.java @@ -1,65 +1,20 @@ package org.bdware.doip.cluster.entity; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; import org.bdware.doip.codec.doipMessage.DoipMessage; import org.bdware.doip.endpoint.client.DoipMessageCallback; +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor public class TreeClusterRequestPack { BDOEntity bdoEntity; DDOEntity ddoEntity; BDWType doType; - - public TreeClusterRequestPack() { - } - DoipMessage doipMessage; - - public BDOEntity getBdoEntity() { - return bdoEntity; - } - - public void setBdoEntity(BDOEntity bdoEntity) { - this.bdoEntity = bdoEntity; - } - - public DDOEntity getDdoEntity() { - return ddoEntity; - } - - public void setDdoEntity(DDOEntity ddoEntity) { - this.ddoEntity = ddoEntity; - } - - public BDWType getDoType() { - return doType; - } - - public void setDoType(BDWType doType) { - this.doType = doType; - } - - public DoipMessage getDoipMessage() { - return doipMessage; - } - - public void setDoipMessage(DoipMessage doipMessage) { - this.doipMessage = doipMessage; - } - - public DoipMessageCallback getDoipMessageCallback() { - return doipMessageCallback; - } - - public void setDoipMessageCallback(DoipMessageCallback doipMessageCallback) { - this.doipMessageCallback = doipMessageCallback; - } - DoipMessageCallback doipMessageCallback; - - public TreeClusterRequestPack(BDOEntity bdoEntity, DDOEntity ddoEntity, BDWType doType, DoipMessage doipMessage, DoipMessageCallback doipMessageCallback) { - this.bdoEntity = bdoEntity; - this.ddoEntity = ddoEntity; - this.doType = doType; - this.doipMessage = doipMessage; - this.doipMessageCallback = doipMessageCallback; - } } -- Gitee From 656a95c53bb61ff54e60ad91aefbe9b7ec556753 Mon Sep 17 00:00:00 2001 From: "haoeliu@foxmail.com" Date: Wed, 19 Jul 2023 00:37:33 +0800 Subject: [PATCH 14/14] extract IRSUtil to resolve Router too large problem && fix instantiateNashornScriptEngineUtil bug --- .../bdware/doip/cluster/ClusterDoaClient.java | 90 +++++++------- .../DDODelegateDoipMessageCallback.java | 15 ++- .../bdware/doip/cluster/client/DOAClient.java | 51 +------- .../cluster/client/DoipClusterClient.java | 33 ++--- .../cluster/client/TreeClusterClient.java | 11 +- .../entity/DoipClusterRequestPack.java | 2 +- .../doip/cluster/util/DOResolutionUtil.java | 74 ++++++----- .../org/bdware/doip/cluster/util/IRSUtil.java | 31 +++++ .../doip/cluster/util/RouteJoinUtil.java | 26 ++-- .../bdware/doip/cluster/util/RouterTool.java | 116 ------------------ 10 files changed, 172 insertions(+), 277 deletions(-) create mode 100644 src/main/java/org/bdware/doip/cluster/util/IRSUtil.java delete mode 100644 src/main/java/org/bdware/doip/cluster/util/RouterTool.java diff --git a/src/main/java/org/bdware/doip/cluster/ClusterDoaClient.java b/src/main/java/org/bdware/doip/cluster/ClusterDoaClient.java index c3ecf35..ca9857d 100644 --- a/src/main/java/org/bdware/doip/cluster/ClusterDoaClient.java +++ b/src/main/java/org/bdware/doip/cluster/ClusterDoaClient.java @@ -88,49 +88,49 @@ public class ClusterDoaClient extends DoipClientImpl { return bdoClients.get(bdoid); } - public DoipClusterClient createOrGetClusterClientLocalTest(String bdoid) { - if (!bdoClients.containsKey(bdoid)) { - DoipClusterClient doipClusterClient = new DoipClusterClient(routerConfig) { - @Override - public void sendMessage(DoipMessage doipMessage, DoipMessageCallback cb) { - - BasicOperations op = BasicOperations.getDoOp(doipMessage.header.parameters.operation); - JsonDoipMessage message = JsonDoipMessage.fromDoipMessage(doipMessage); - String[] BDRepoIDs = route(message); - // joinParams都来自于callback，joinParams就是所有节点callback到达之后，merge之后的结果 - JoinInfo joinInfo = this.doipOperationToJoinInfo.get(op.getName()); - sendMsgAndJoin(doipMessage, joinInfo, BDRepoIDs, cb); - } - - public void sendMsgAndJoin(DoipMessage msg, JoinInfo joinInfo, String[] BDRepoIDs, DoipMessageCallback cb) { - int serversNum = BDRepoIDs.length; - int count = getJoinCount(joinInfo, serversNum, new JsonObject()); - // 如果joinCountFunc执行算出的joinCount，比所有的servers还要多，就更新为servers的数量 - if (serversNum < count) { - count = serversNum; - } - DoipMessageFactory.DoipMessageBuilder builder = new DoipMessageFactory.DoipMessageBuilder(); - builder.createResponse(DoipResponseCode.Success, msg); - builder.setBody("success".getBytes()); - cb.onResult(builder.create()); - } - - int getJoinCount(JoinInfo joinInfo, int serversNum, JsonObject joinParams) { - try { - if (joinInfo == null) return serversNum; - if (joinInfo.joinCountFuncName != null) { - JsonElement requester = routerConfig.publicKey == null ? JsonNull.INSTANCE : new JsonPrimitive(routerConfig.publicKey); - return engineUtil.invokeFunction(joinInfo.joinCountFuncName, Integer.class, joinParams, requester); - } - if (joinInfo.joinCount != 0) return joinInfo.joinCount; - } catch (Exception e) { - e.printStackTrace(); - } - return serversNum; - } - }; - bdoClients.put(bdoid, doipClusterClient); - } - return bdoClients.get(bdoid); - } +// public DoipClusterClient createOrGetClusterClientLocalTest(String bdoid) { +// if (!bdoClients.containsKey(bdoid)) { +// DoipClusterClient doipClusterClient = new DoipClusterClient(routerConfig) { +// @Override +// public void sendMessage(DoipMessage doipMessage, DoipMessageCallback cb) { +// +// BasicOperations op = BasicOperations.getDoOp(doipMessage.header.parameters.operation); +// JsonDoipMessage message = JsonDoipMessage.fromDoipMessage(doipMessage); +// String[] BDRepoIDs = route(message); +// // joinParams都来自于callback，joinParams就是所有节点callback到达之后，merge之后的结果 +// JoinInfo joinInfo = this.doipOperationToJoinInfo.get(op.getName()); +// sendMsgAndJoin(doipMessage, joinInfo, BDRepoIDs, cb); +// } +// +// public void sendMsgAndJoin(DoipMessage msg, JoinInfo joinInfo, String[] BDRepoIDs, DoipMessageCallback cb) { +// int serversNum = BDRepoIDs.length; +// int count = getJoinCount(joinInfo, serversNum, new JsonObject()); +// // 如果joinCountFunc执行算出的joinCount，比所有的servers还要多，就更新为servers的数量 +// if (serversNum < count) { +// count = serversNum; +// } +// DoipMessageFactory.DoipMessageBuilder builder = new DoipMessageFactory.DoipMessageBuilder(); +// builder.createResponse(DoipResponseCode.Success, msg); +// builder.setBody("success".getBytes()); +// cb.onResult(builder.create()); +// } +// +// int getJoinCount(JoinInfo joinInfo, int serversNum, JsonObject joinParams) { +// try { +// if (joinInfo == null) return serversNum; +// if (joinInfo.joinCountFuncName != null) { +// JsonElement requester = routerConfig.publicKey == null ? JsonNull.INSTANCE : new JsonPrimitive(routerConfig.publicKey); +// return engineUtil.invokeFunction(joinInfo.joinCountFuncName, Integer.class, joinParams, requester); +// } +// if (joinInfo.joinCount != 0) return joinInfo.joinCount; +// } catch (Exception e) { +// e.printStackTrace(); +// } +// return serversNum; +// } +// }; +// bdoClients.put(bdoid, doipClusterClient); +// } +// return bdoClients.get(bdoid); +// } } diff --git a/src/main/java/org/bdware/doip/cluster/callback/DDODelegateDoipMessageCallback.java b/src/main/java/org/bdware/doip/cluster/callback/DDODelegateDoipMessageCallback.java index f8a904b..1eb5cde 100644 --- a/src/main/java/org/bdware/doip/cluster/callback/DDODelegateDoipMessageCallback.java +++ b/src/main/java/org/bdware/doip/cluster/callback/DDODelegateDoipMessageCallback.java @@ -38,22 +38,24 @@ public class DDODelegateDoipMessageCallback implements DoipMessageCallback { Set nodeIDs = new HashSet<>(); NashornScriptEngineUtil engineUtil; JoinInfo joinInfo; - String[] targetServers; + String[] targetDOIDs; DoipClientImpl doipClientImpl; JsonObject appendixes; - public DDODelegateDoipMessageCallback(DoipClientImpl doipClientImpl, String[] targetServers, DoipMessage msg, DoipMessageCallback cb, int count, NashornScriptEngineUtil engineUtil, JoinInfo joinInfo, JsonObject appendixes) { + public DDODelegateDoipMessageCallback(DoipClientImpl doipClientImpl, String[] targetDOIDs, DoipMessage msg, DoipMessageCallback cb, int count, NashornScriptEngineUtil engineUtil, JoinInfo joinInfo, JsonObject appendixes) { this.request = msg; this.originalCallback = cb; this.joinCount = count; this.engineUtil = engineUtil; this.joinInfo = joinInfo; - this.targetServers = targetServers; + this.targetDOIDs = targetDOIDs; this.doipClientImpl = doipClientImpl; this.appendixes = appendixes; } + // todo 并发有问题，多个msg同时回调，可能会多次触发addDoipResult的逻辑 + // todo 处理逻辑，nodeIDs.add(nodeID);的逻辑是否成功或者失败都要触发？ @Override public void onResult(DoipMessage msg) { logDoipMsgInfo(msg); @@ -85,10 +87,10 @@ public class DDODelegateDoipMessageCallback implements DoipMessageCallback { try { String nodeID = getNodeID(msg); if (nodeID != null) { - ConnectionUtil.popReconnectRequest(targetServers); + ConnectionUtil.popReconnectRequest(targetDOIDs); } else { - LOGGER.error("收到错误的节点回复 : " + "Something wrong happens in" + new Gson().toJson(targetServers) + " msg:" + new Gson().toJson(msg.header.parameters)); - ConnectionUtil.pushReconnectRequest(targetServers, appendixes); + LOGGER.error("收到错误的节点回复 : " + "Something wrong happens in" + new Gson().toJson(targetDOIDs) + " msg:" + new Gson().toJson(msg.header.parameters)); + ConnectionUtil.pushReconnectRequest(targetDOIDs, appendixes); } } catch (Exception e) { e.printStackTrace(); @@ -138,6 +140,7 @@ public class DDODelegateDoipMessageCallback implements DoipMessageCallback { } builder.addAttributes("successResponses", successResponses); builder.addAttributes("problemResponses", problemResponses); + builder.addAttributes("nodeID", String.valueOf(request.requestID)); return builder.create(); } diff --git a/src/main/java/org/bdware/doip/cluster/client/DOAClient.java b/src/main/java/org/bdware/doip/cluster/client/DOAClient.java index 0e9d27f..774106e 100644 --- a/src/main/java/org/bdware/doip/cluster/client/DOAClient.java +++ b/src/main/java/org/bdware/doip/cluster/client/DOAClient.java @@ -6,6 +6,7 @@ import org.apache.logging.log4j.Logger; import org.bdware.doip.audit.EndpointConfig; import org.bdware.doip.audit.client.AuditIrpClient; import org.bdware.doip.cluster.callback.DDODelegateDoipMessageCallback; +import org.bdware.doip.cluster.util.RouteJoinUtil; import org.bdware.doip.codec.digitalObject.DigitalObject; import org.bdware.doip.codec.doipMessage.DoipMessage; import org.bdware.doip.codec.doipMessage.DoipMessageSigner; @@ -258,14 +259,14 @@ public class DOAClient extends DoipClientImpl { case byArgHash: val = routeInfoArg.hashCode(); val = val % members.length; - while (val < 0 && members.length > 0) { + while (val < 0) { val += members.length; } return new String[]{members[val]}; case byJsonPropHash: - JsonElement jo = tryLoadJsonProp(routeInfoArg, routeInfo.param); + JsonElement jo = RouteJoinUtil.tryLoadJsonProp(routeInfoArg, routeInfo.param); val = jo.toString().hashCode() % members.length; - while (val < 0 && members.length > 0) { + while (val < 0) { val += members.length; } return new String[]{members[val]}; @@ -282,29 +283,6 @@ public class DOAClient extends DoipClientImpl { } } - private JsonElement tryLoadJsonProp(JsonObject routeInfoArg, String param) { - try { - if (routeInfoArg == null) - return JsonNull.INSTANCE; - JsonObject arg; - if (routeInfoArg.isJsonPrimitive()) { - arg = JsonUtil.parseString(routeInfoArg.getAsString()).getAsJsonObject(); - } else arg = routeInfoArg.getAsJsonObject(); - if (!param.contains(".")) { - return arg.get(param); - } else { - String[] props = param.split("\\."); - JsonElement result = arg; - for (String str : props) - result = result.getAsJsonObject().get(str); - return result; - } - } catch (Exception e) { - e.printStackTrace(); - return JsonNull.INSTANCE; - } - } - // 根据请求参数，组装RouteInfo的参数Arg public JsonObject composeRouteInfoArg(BasicOperations operation, String identifier, SearchParameter sp, String element, boolean includeElementData, DigitalObject digitalObject) { JsonObject composedRes = new JsonObject(); @@ -340,28 +318,11 @@ public class DOAClient extends DoipClientImpl { return composedRes; } - private int getJoinCount(JoinInfo joinInfo, int serversNum, JsonObject joinParams) { - try { - if (joinInfo == null) return serversNum; - if (joinInfo.joinCountFuncName != null) { - JsonElement requester = routerConfig.publicKey == null ? JsonNull.INSTANCE : new JsonPrimitive(routerConfig.publicKey); - return engineUtil.invokeFunction(joinInfo.joinCountFuncName, Integer.class, joinParams, requester); - } - if (joinInfo.joinCount != 0) return joinInfo.joinCount; - } catch (Exception e) { - e.printStackTrace(); - } - return serversNum; - } - - public JoinInfo getJoinInfo(BasicOperations operation) { - return doipOperationToJoinInfo.getOrDefault(operation.getName(), null); - } public void callServersAndJoin(DoipParam doipParam, String[] targetDoipServers, DoipMessageCallback cb) { - JoinInfo joinInfo = getJoinInfo(doipParam.operation); + JoinInfo joinInfo = doipOperationToJoinInfo.get(doipParam.operation.getName()); int serversNum = targetDoipServers.length; - int count = getJoinCount(joinInfo, serversNum, new JsonObject()); + int count = RouteJoinUtil.getJoinCount(routerConfig.publicKey, engineUtil, joinInfo, targetDoipServers.length, new JsonObject()); // 如果joinCountFunc执行算出的joinCount，比所有的servers还要多，就更新为servers的数量 if (serversNum < count) { count = serversNum; diff --git a/src/main/java/org/bdware/doip/cluster/client/DoipClusterClient.java b/src/main/java/org/bdware/doip/cluster/client/DoipClusterClient.java index 9662795..aa5dc16 100644 --- a/src/main/java/org/bdware/doip/cluster/client/DoipClusterClient.java +++ b/src/main/java/org/bdware/doip/cluster/client/DoipClusterClient.java @@ -9,8 +9,9 @@ import org.bdware.doip.cluster.entity.DoipClusterRequestPack; import org.bdware.doip.cluster.flowcontrol.FlowControl; import org.bdware.doip.cluster.flowcontrol.core.RateThresholdFlowControl; import org.bdware.doip.cluster.util.AuditDoipClientCacheUtil; +import org.bdware.doip.cluster.util.DOResolutionUtil; +import org.bdware.doip.cluster.util.IRSUtil; import org.bdware.doip.cluster.util.RouteJoinUtil; -import org.bdware.doip.cluster.util.RouterTool; import org.bdware.doip.codec.JsonDoipMessage; import org.bdware.doip.codec.doipMessage.DoipMessage; import org.bdware.doip.codec.doipMessage.DoipMessageFactory; @@ -44,15 +45,15 @@ public class DoipClusterClient extends DoipClientImpl { private final DoipMessageSigner doipMessageSigner; protected final EndpointConfig routerConfig; // all the doipOperation and routeInfo binding relationships - private final Map doipOperationToRouteInfo; + protected Map doipOperationToRouteInfo; // all the doipOperation and joinInfo binding relationships - protected final Map doipOperationToJoinInfo; + protected Map doipOperationToJoinInfo; // 维护了地址 与 doipPort之间的映射关系 // BDO和DoipClusterClient是一一对应的关系，一个BDO对应一组BDRepo，一个BDRepo对应了一个DoipServer端口，因此这里维护Repo -> 端口的映射 private final static Queue sendMessageTaskQueue = new ConcurrentLinkedDeque<>(); + public final AuditIrpClient irsClient; private final FlowControl flowControl; Logger Logger = LogManager.getLogger(DoipClusterClient.class); - private final RouterTool routerTool; String curDOID; JsonObject appendixes; @@ -67,7 +68,7 @@ public class DoipClusterClient extends DoipClientImpl { public DoipClusterClient(EndpointConfig routerConfig) { this.routerConfig = routerConfig; - AuditIrpClient irsClient = new AuditIrpClient(routerConfig); + irsClient = new AuditIrpClient(routerConfig); new Thread(this::consumeSendMessageTask).start(); try { irsClient.reconnect(); @@ -83,7 +84,6 @@ public class DoipClusterClient extends DoipClientImpl { engineUtil = new NashornScriptEngineUtil(); doipOperationToRouteInfo = new HashMap<>(); doipOperationToJoinInfo = new HashMap<>(); - routerTool = new RouterTool(irsClient, engineUtil); // todo optimize flowControl // flowControl = new TokenBucket(10240, 2048); int threshold = 1000; @@ -94,23 +94,26 @@ public class DoipClusterClient extends DoipClientImpl { } + // TODO 存在并发问题，parse之后就应该立刻进行调用，原子性，不然会出现问题 public void parseDOID(String doid) { for (int retry = 0; retry < 5; retry++) { try { // 如果已经解析过了，就不需要再次进行解析了 synchronized (this) { if (!doid.equals(curDOID)) { - JsonObject BDOInfo = routerTool.verifyCluster(doid); + JsonObject ddoInfo = DOResolutionUtil.getAndVerifyDDOInfo(irsClient, doid); // 读取appendixes并放入变量 - appendixes = BDOInfo.get("appendixes").getAsJsonObject(); + appendixes = IRSUtil.getAppendixesFromDDOInfo(ddoInfo); // 传递的是BDO.clusterInfo -> BDRepoIDs // BDRepoID -> BDRepo.address - routerTool.parseClusterInfo(BDOInfo, engineUtil); + engineUtil = DOResolutionUtil.instantiateNashornScriptEngineUtil(ddoInfo); + + String bcoId = ddoInfo.get("bcoId").getAsString(); + JsonObject bcoAccessRules = appendixes.get(bcoId).getAsJsonObject().get("accessRules").getAsJsonObject(); + this.doipOperationToRouteInfo= DOResolutionUtil.getDDORouteInfo(bcoAccessRules); + this.doipOperationToJoinInfo = DOResolutionUtil.getDDOJoinInfo(bcoAccessRules); - // 传递的是BDO.clusterInfo -> BCOID - // 传递的是BCOID -> BCO.accessRules - routerTool.parseAccessRules(doipOperationToRouteInfo, doipOperationToJoinInfo, BDOInfo); curDOID = doid; } } @@ -131,10 +134,10 @@ public class DoipClusterClient extends DoipClientImpl { try { BasicOperations op = BasicOperations.getDoOp(doipMessage.header.parameters.operation); JsonDoipMessage message = JsonDoipMessage.fromDoipMessage(doipMessage); - String[] BDRepoIDs = route(message); + String[] targetDOIDs = route(message); // joinParams都来自于callback，joinParams就是所有节点callback到达之后，merge之后的结果 JoinInfo joinInfo = doipOperationToJoinInfo.get(op.getName()); - DoipClusterRequestPack pack = new DoipClusterRequestPack(doipMessage, joinInfo, BDRepoIDs, msg -> { + DoipClusterRequestPack pack = new DoipClusterRequestPack(doipMessage, joinInfo, targetDOIDs, msg -> { cb.onResult(msg); flowControl.maintainFlowControl(); }); @@ -154,7 +157,7 @@ public class DoipClusterClient extends DoipClientImpl { this, pack.getMessage(), pack.getJoinInfo(), - pack.getBdRepoIds(), + pack.getTargetDOIDs(), pack.getCb(), doipMessageSigner, engineUtil, diff --git a/src/main/java/org/bdware/doip/cluster/client/TreeClusterClient.java b/src/main/java/org/bdware/doip/cluster/client/TreeClusterClient.java index 6b6b65b..f64151c 100644 --- a/src/main/java/org/bdware/doip/cluster/client/TreeClusterClient.java +++ b/src/main/java/org/bdware/doip/cluster/client/TreeClusterClient.java @@ -17,6 +17,7 @@ import org.bdware.doip.cluster.flowcontrol.FlowControl; import org.bdware.doip.cluster.flowcontrol.core.RateThresholdFlowControl; import org.bdware.doip.cluster.util.AuditDoipClientCacheUtil; import org.bdware.doip.cluster.util.DOResolutionUtil; +import org.bdware.doip.cluster.util.IRSUtil; import org.bdware.doip.cluster.util.RouteJoinUtil; import org.bdware.doip.codec.JsonDoipMessage; import org.bdware.doip.codec.doipMessage.DoipMessage; @@ -138,9 +139,9 @@ public class TreeClusterClient extends DoipClientImpl { if(ddoEntity != null) return ddoEntity; String bcoId = ddoInfo.get("bcoId").getAsString(); - JsonObject appendixes = ddoInfo.get("appendixes").getAsJsonObject(); + JsonObject appendixes = IRSUtil.getAppendixesFromDDOInfo(ddoInfo); // 将所有的functions通过engine注册到functionBindings中 - JsonObject bcoAccessRules = appendixes.get(bcoId).getAsJsonObject().get("accessRules").getAsJsonObject(); + JsonObject bcoAccessRules = IRSUtil.parseTargetDOID(appendixes, bcoId, irsClient).getAsJsonObject().get("accessRules").getAsJsonObject(); Map ddoRouteInfo = DOResolutionUtil.getDDORouteInfo(bcoAccessRules); Map ddoJoinInfo = DOResolutionUtil.getDDOJoinInfo(bcoAccessRules); NashornScriptEngineUtil engineUtil = DOResolutionUtil.instantiateNashornScriptEngineUtil(ddoInfo); @@ -172,9 +173,9 @@ public class TreeClusterClient extends DoipClientImpl { public void ddoCall(DDOEntity ddoEntity, DoipMessage doipMessage, DoipMessageCallback cb) { JsonDoipMessage doipParam = JsonDoipMessage.fromDoipMessage(doipMessage); BasicOperations operation = BasicOperations.getDoOp(doipParam.header.operation); - String[] toCalls = RouteJoinUtil.route(doipParam, routerConfig, ddoEntity.getDdoRouteInfo().get(operation.getName()), ddoEntity.getEngineUtil()); - assert toCalls != null; - RouteJoinUtil.ddoSendMsgAndJoin(this, ddoEntity.getEngineUtil(), doipMessage, ddoEntity.getDdoJoinInfo().get(operation.getName()), toCalls, cb, ddoEntity.getAppendixes()); + String[] targetDOIDs = RouteJoinUtil.route(doipParam, routerConfig, ddoEntity.getDdoRouteInfo().get(operation.getName()), ddoEntity.getEngineUtil()); + assert targetDOIDs != null; + RouteJoinUtil.ddoSendMsgAndJoin(this, ddoEntity.getEngineUtil(), doipMessage, ddoEntity.getDdoJoinInfo().get(operation.getName()), targetDOIDs, cb, ddoEntity.getAppendixes()); } public void sendMessageInternal(TreeClusterRequestPack pack) { diff --git a/src/main/java/org/bdware/doip/cluster/entity/DoipClusterRequestPack.java b/src/main/java/org/bdware/doip/cluster/entity/DoipClusterRequestPack.java index ffaf40b..4038115 100644 --- a/src/main/java/org/bdware/doip/cluster/entity/DoipClusterRequestPack.java +++ b/src/main/java/org/bdware/doip/cluster/entity/DoipClusterRequestPack.java @@ -15,6 +15,6 @@ import org.bdware.sc.bean.JoinInfo; public class DoipClusterRequestPack { public DoipMessage message; public JoinInfo joinInfo; - public String[] bdRepoIds; + public String[] targetDOIDs; public DoipMessageCallback cb; } diff --git a/src/main/java/org/bdware/doip/cluster/util/DOResolutionUtil.java b/src/main/java/org/bdware/doip/cluster/util/DOResolutionUtil.java index 1bd48c2..18a22d8 100644 --- a/src/main/java/org/bdware/doip/cluster/util/DOResolutionUtil.java +++ b/src/main/java/org/bdware/doip/cluster/util/DOResolutionUtil.java @@ -1,6 +1,7 @@ package org.bdware.doip.cluster.util; import com.google.gson.*; +import lombok.SneakyThrows; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.bdware.doip.audit.client.AuditIrpClient; @@ -11,7 +12,6 @@ import org.bdware.sc.bean.JoinInfo; import org.bdware.sc.bean.RouteInfo; import wrp.jdk.nashorn.api.scripting.NashornScriptEngineUtil; -import javax.script.ScriptException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -21,32 +21,33 @@ public class DOResolutionUtil { private static final Logger Logger = LogManager.getLogger(DOResolutionUtil.class); private static final Gson gson = new Gson(); - public static String[] getDOIDClusterInfo(JsonObject ddoInfo) { - JsonArray bdoList = ddoInfo.get("bdoList").getAsJsonArray(); - JsonObject appendixes = ddoInfo.get("appendixes").getAsJsonObject(); + // TODO rename bdoList in Router + public static String[] getDDOClusterInfo(JsonObject ddoInfo) { + JsonArray doList = ddoInfo.get("bdoList").getAsJsonArray(); List subDOIDsList = new ArrayList<>(); - for (JsonElement bdoId : bdoList) { - String bdo = bdoId.getAsString(); - if (appendixes.get(bdo) != null || !appendixes.get(bdo).isJsonNull()) { - subDOIDsList.add(bdo); - } else { - Logger.error(bdo + " cannot be parsed by Router"); - } + for (JsonElement doid : doList) { + subDOIDsList.add(doid.getAsString()); } - String[] subDOIDs = new String[subDOIDsList.size()]; - subDOIDsList.toArray(subDOIDs); - return subDOIDs; + return subDOIDsList.toArray(new String[0]); } - - public static NashornScriptEngineUtil instantiateNashornScriptEngineUtil(JsonObject ddoInfo) throws ScriptException { + public static NashornScriptEngineUtil instantiateNashornScriptEngineUtil(JsonObject ddoInfo) { NashornScriptEngineUtil engineUtil = new NashornScriptEngineUtil(); String bcoId = ddoInfo.get("bcoId").getAsString(); JsonObject appendixes = ddoInfo.get("appendixes").getAsJsonObject(); // 将所有的functions通过engine注册到functionBindings中 JsonObject bcoAccessRules = appendixes.get(bcoId).getAsJsonObject().get("accessRules").getAsJsonObject(); + + registerAccessClusterInfoToEngineUtil(engineUtil, ddoInfo); + registerAccessRuleFunctionsToEngineUtil(engineUtil, bcoAccessRules); + + return engineUtil; + } + + @SneakyThrows + public static void registerAccessRuleFunctionsToEngineUtil(NashornScriptEngineUtil engineUtil, JsonObject bcoAccessRules) { JsonElement functions = bcoAccessRules.get("functions"); if (functions != null && !functions.isJsonNull() && functions.isJsonObject()) { JsonObject functionsObj = functions.getAsJsonObject(); @@ -55,8 +56,11 @@ public class DOResolutionUtil { engineUtil.evalFunction(funcScript); } } + } - return engineUtil; + @SneakyThrows + public static void registerAccessClusterInfoToEngineUtil(NashornScriptEngineUtil engineUtil, JsonObject ddoInfo) { + engineUtil.doipClusterUtil.doipServers = getDDOClusterInfo(ddoInfo); } public static Map getDDORouteInfo(JsonObject bcoAccessRules) { @@ -94,16 +98,18 @@ public class DOResolutionUtil { return res; } - public static JsonObject getAndVerifyDDOInfo(AuditIrpClient irsClient, String doid) throws IrpClientException { - StateInfoBase content = irsClient.resolve(doid); - if (content == null || content.handleValues == null || content.handleValues.isJsonNull()) { + @SneakyThrows + public static JsonObject getAndVerifyDDOInfo(AuditIrpClient irsClient, String ddoID) { + JsonObject ddoInfo = getDOInfo(irsClient, ddoID); + if (ddoInfo.isJsonNull()) { throw new IrpClientException("DDO's info isn't in the router, id cannot be resolved"); } - return verifyDDOInfo(content.handleValues); + return verifyDDOInfo(ddoInfo); } - public static JsonObject getDOInfo(AuditIrpClient irsClient, String doid) throws IrpClientException { + @SneakyThrows + public static JsonObject getDOInfo(AuditIrpClient irsClient, String doid) { StateInfoBase content = irsClient.resolve(doid); if (content == null || content.handleValues == null || content.handleValues.isJsonNull()) { throw new IrpClientException("DO's info isn't in the router, id cannot be resolved"); @@ -112,27 +118,30 @@ public class DOResolutionUtil { return content.handleValues; } - public static JsonObject verifyDDOInfo(JsonObject doInfo) throws IrpClientException { + // todo refactor appendixes logic to cater to IRSUtil demand + @SneakyThrows + public static JsonObject verifyDDOInfo(JsonObject ddoInfo) { // 如果不含有 ClusterInfo 或者 BCOID(ClusterInfo & JoinInfo) - if (doInfo.get("bdoList") == null || doInfo.get("bdoList").isJsonNull() - || doInfo.get("bcoId") == null || doInfo.get("bcoId").isJsonNull() - || doInfo.get("appendixes") == null || doInfo.get("appendixes").isJsonNull()) { + if (ddoInfo.get("bdoList") == null || ddoInfo.get("bdoList").isJsonNull() + || ddoInfo.get("bcoId") == null || ddoInfo.get("bcoId").isJsonNull() + || ddoInfo.get("appendixes") == null || ddoInfo.get("appendixes").isJsonNull()) { throw new IrpClientException("DDO doesn't have enough info, bdoList or bcoId or appendixes cannot be verified"); } // 如果不含有 ClusterInfo & JoinInfo - String bcoId = doInfo.get("bcoId").getAsString(); - JsonObject appendixes = doInfo.get("appendixes").getAsJsonObject(); + String bcoId = ddoInfo.get("bcoId").getAsString(); + JsonObject appendixes = ddoInfo.get("appendixes").getAsJsonObject(); if (appendixes.get(bcoId) == null || appendixes.get(bcoId).isJsonNull() || appendixes.get(bcoId).getAsJsonObject().get("accessRules") == null || appendixes.get(bcoId).getAsJsonObject().get("accessRules").isJsonNull()) { throw new IrpClientException("DDO doesn't have enough info , accessRules cannot be verified"); } - Logger.info(new GsonBuilder().setPrettyPrinting().create().toJson(doInfo)); - return doInfo; + Logger.info(new GsonBuilder().setPrettyPrinting().create().toJson(ddoInfo)); + return ddoInfo; } - public static BDWType getDOType(AuditIrpClient irsClient, String doid) throws IrpClientException { + @SneakyThrows + public static BDWType getDOType(AuditIrpClient irsClient, String doid) { StateInfoBase content = irsClient.resolve(doid); if (content == null || content.handleValues == null || content.handleValues.isJsonNull()) { throw new IrpClientException("DO's info isn't in the router, id cannot be resolved"); @@ -147,7 +156,8 @@ public class DOResolutionUtil { return null; } - public static BDWType getDOType(JsonObject doInfo) throws IrpClientException { + @SneakyThrows + public static BDWType getDOType(JsonObject doInfo) { if(doInfo.get("bdwType") == null || doInfo.get("bdwType").isJsonNull()) { throw new IrpClientException("DO's info has not bdwType, please check the DO in the router"); } diff --git a/src/main/java/org/bdware/doip/cluster/util/IRSUtil.java b/src/main/java/org/bdware/doip/cluster/util/IRSUtil.java new file mode 100644 index 0000000..fb972f1 --- /dev/null +++ b/src/main/java/org/bdware/doip/cluster/util/IRSUtil.java @@ -0,0 +1,31 @@ +package org.bdware.doip.cluster.util; + +import com.google.gson.JsonObject; +import lombok.SneakyThrows; +import org.bdware.doip.audit.client.AuditIrpClient; + +// TODO Router should be updated to cater to IRSUtil demand, bcoid and bdolist can be appended to appendixes first +// TODO bdoList can be recursively resolved +public class IRSUtil { + @SneakyThrows + public static JsonObject parseTargetDOID(JsonObject currentAppendixes, String targetDOID, AuditIrpClient irsClient) { + if(targetDOID == null) { + throw new NullPointerException("targetDOID to resolve from appendix is null"); + } else { + if (!currentAppendixes.has(targetDOID) + || currentAppendixes.get(targetDOID).isJsonNull() + || !currentAppendixes.get(targetDOID).isJsonObject()) { + JsonObject targetHandleValues = DOResolutionUtil.getDOInfo(irsClient, targetDOID); + currentAppendixes.add(targetDOID, targetHandleValues); + return targetHandleValues; + } else { + return currentAppendixes.get(targetDOID).getAsJsonObject(); + } + } + } + + public static JsonObject getAppendixesFromDDOInfo(JsonObject ddoInfo) { + if(!ddoInfo.has("appendixes") || ddoInfo.get("appendixes").isJsonNull()) return null; + return ddoInfo.get("appendixes").getAsJsonObject(); + } +} diff --git a/src/main/java/org/bdware/doip/cluster/util/RouteJoinUtil.java b/src/main/java/org/bdware/doip/cluster/util/RouteJoinUtil.java index ea30208..d631236 100644 --- a/src/main/java/org/bdware/doip/cluster/util/RouteJoinUtil.java +++ b/src/main/java/org/bdware/doip/cluster/util/RouteJoinUtil.java @@ -96,8 +96,9 @@ public class RouteJoinUtil { } } - public static void ddoSendMsgAndJoin(TreeClusterClient treeClusterClient, NashornScriptEngineUtil engineUtil, DoipMessage msg, JoinInfo joinInfo, String[] toCalls, DoipMessageCallback cb, JsonObject appendixes) { - int serversNum = toCalls.length; + public static void ddoSendMsgAndJoin(TreeClusterClient treeClusterClient, NashornScriptEngineUtil engineUtil, DoipMessage msg, JoinInfo joinInfo, String[] doidList, DoipMessageCallback cb, JsonObject appendixes) { + int serversNum = doidList.length; + // TODO joinParams to expose int count = getJoinCount(treeClusterClient.routerConfig, engineUtil, joinInfo, serversNum, new JsonObject()); // 如果joinCountFunc执行算出的joinCount，比所有的servers还要多，就更新为servers的数量 if (serversNum < count) { @@ -105,15 +106,15 @@ public class RouteJoinUtil { } if (count > 0) { - DDODelegateDoipMessageCallback ddoDelegateDoipMessageCallback = new DDODelegateDoipMessageCallback(treeClusterClient, toCalls, msg, cb, count, engineUtil, joinInfo, appendixes); - for (String toCall : toCalls) { + DDODelegateDoipMessageCallback ddoDelegateDoipMessageCallback = new DDODelegateDoipMessageCallback(treeClusterClient, doidList, msg, cb, count, engineUtil, joinInfo, appendixes); + for (String doid : doidList) { // todo 对于DDO嵌套 -> appendix可以只放bdwtype // todo 消息嵌套？同一个消息发给多个Server？每个消息改一个requestId = 0 & DOID -> DDOID/BDOID，并且发给多个Server？ // TODO TODO TODO // DoipMessage toSend = msg.deepCopy(); // 要么response wait中的waitObj改为非static，并加入一点流控制机制。 // 这里的doipClient为BDRepoClient或者BDOClient，无所谓，如果是BDRepoClient，DelegateDoipMessageCallback会出手转为BDOClient - DoipMessage toCallDoipMessage = msg.withNewDOID(msg, toCall); + DoipMessage toCallDoipMessage = msg.withNewDOID(msg, doid); treeClusterClient.sendMessage(toCallDoipMessage, ddoDelegateDoipMessageCallback); } } else { @@ -121,9 +122,9 @@ public class RouteJoinUtil { } } - public static void bdoSendMsgAndJoin(DoipClusterClient doipClusterClient, DoipMessage msg, JoinInfo joinInfo, String[] BDRepoIDs, DoipMessageCallback cb, DoipMessageSigner doipMessageSigner, NashornScriptEngineUtil engineUtil, String publicKey, JsonObject appendixes) { - int serversNum = BDRepoIDs.length; - // todo optimize joinParams + public static void bdoSendMsgAndJoin(DoipClusterClient doipClusterClient, DoipMessage msg, JoinInfo joinInfo, String[] doidList, DoipMessageCallback cb, DoipMessageSigner doipMessageSigner, NashornScriptEngineUtil engineUtil, String publicKey, JsonObject appendixes) { + int serversNum = doidList.length; + // TODO joinParams to expose int count = RouteJoinUtil.getJoinCount(publicKey, engineUtil, joinInfo, serversNum, new JsonObject()); // 如果joinCountFunc执行算出的joinCount，比所有的servers还要多，就更新为servers的数量 if (serversNum < count) { @@ -131,11 +132,12 @@ public class RouteJoinUtil { } if (count > 0) { - DDODelegateDoipMessageCallback ddoDelegateDoipMessageCallback = new DDODelegateDoipMessageCallback(doipClusterClient, BDRepoIDs, msg, cb, count, engineUtil, joinInfo, appendixes); - for (String BDRepoId : BDRepoIDs) { + DDODelegateDoipMessageCallback ddoDelegateDoipMessageCallback = new DDODelegateDoipMessageCallback(doipClusterClient, doidList, msg, cb, count, engineUtil, joinInfo, appendixes); + for (String bdoID : doidList) { // 如果已经建立过BDRepo -> BDO的映射，可以通过BDRepo的地址，直接找到BDO的Client，对其进行操作（本质BDRepo和BDO是一一对应的关系） - String address = appendixes.get(BDRepoId).getAsJsonObject().get("address").getAsString(); - String version = appendixes.get(BDRepoId).getAsJsonObject().get("version").getAsString(); + JsonObject bdoInfo = IRSUtil.parseTargetDOID(appendixes, bdoID, doipClusterClient.irsClient); + String address = bdoInfo.get("address").getAsString(); + String version = bdoInfo.get("version").getAsString(); AuditDoipClient doipClient = AuditDoipClientCacheUtil.getFastClientByUrl(address, null, null); if (doipClient == null) doipClient = AuditDoipClientCacheUtil.getClientByUrl(address, version); diff --git a/src/main/java/org/bdware/doip/cluster/util/RouterTool.java b/src/main/java/org/bdware/doip/cluster/util/RouterTool.java deleted file mode 100644 index 4fc81ae..0000000 --- a/src/main/java/org/bdware/doip/cluster/util/RouterTool.java +++ /dev/null @@ -1,116 +0,0 @@ -package org.bdware.doip.cluster.util; - -import com.google.gson.*; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.bdware.doip.audit.client.AuditIrpClient; -import org.bdware.irp.exception.IrpClientException; -import org.bdware.irp.stateinfo.StateInfoBase; -import org.bdware.sc.bean.JoinInfo; -import org.bdware.sc.bean.RouteInfo; -import wrp.jdk.nashorn.api.scripting.NashornScriptEngineUtil; - -import javax.script.ScriptException; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; - -public class RouterTool { - private AuditIrpClient irsClient; - private NashornScriptEngineUtil engineUtil; - - Logger Logger = LogManager.getLogger(RouterTool.class); - ; - - public RouterTool(AuditIrpClient irsClient, NashornScriptEngineUtil engineUtil) { - this.irsClient = irsClient; - this.engineUtil = engineUtil; - } - - public JsonObject verifyCluster(String doid) throws IrpClientException { - StateInfoBase content = irsClient.resolve(doid); - if (content == null || content.handleValues == null || content.handleValues.isJsonNull()) { - throw new IrpClientException("BDO's info isn't in the router, id cannot be resolved"); - } - - // 如果不含有clusterInfo或者BCOID(ClusterInfo & JoinInfo) - JsonObject DOInfo = content.handleValues; - if (DOInfo.get("bdoList") == null || DOInfo.get("bdoList").isJsonNull() - || DOInfo.get("bcoId") == null || DOInfo.get("bcoId").isJsonNull() - || DOInfo.get("appendixes") == null || DOInfo.get("appendixes").isJsonNull()) { - throw new IrpClientException("BDO doesn't have enough cluster info"); - } - - // 如果不有ClusterInfo & JoinInfo - String bcoId = DOInfo.get("bcoId").getAsString(); - JsonObject appendixes = DOInfo.get("appendixes").getAsJsonObject(); - if (appendixes.get(bcoId) == null || appendixes.get(bcoId).isJsonNull() - || appendixes.get(bcoId).getAsJsonObject().get("accessRules") == null - || appendixes.get(bcoId).getAsJsonObject().get("accessRules").isJsonNull()) { - throw new IrpClientException("BDO doesn't have enough cluster info"); - } - Logger.info(new GsonBuilder().setPrettyPrinting().create().toJson(DOInfo)); - return DOInfo; - } - - public void parseClusterInfo(JsonObject BDOInfo, NashornScriptEngineUtil engineUtil) { - JsonArray bdoList = BDOInfo.get("bdoList").getAsJsonArray(); - JsonObject appendixes = BDOInfo.get("appendixes").getAsJsonObject(); - List BDRepoIDs = new LinkedList<>(); - for (JsonElement bdoId : bdoList) { - String bdo = bdoId.getAsString(); - if (appendixes.get(bdo) != null || !appendixes.get(bdo).isJsonNull()) { - BDRepoIDs.add(bdo); - } else { - Logger.error(bdo + " cannot be parsed by Router"); - } - } - - engineUtil.doipClusterUtil.doipServers = new String[BDRepoIDs.size()]; - for (int i = 0; i < BDRepoIDs.size(); i++) { - engineUtil.doipClusterUtil.doipServers[i] = BDRepoIDs.get(i); - } - } - - // accessRules中保存了所有的Func/Join和相关的所有函数，进行解析 - public void parseAccessRules(Map doipOperationToRouteInfo, Map doipOperationToJoinInfo, JsonObject BDOInfo) throws ScriptException { - String bcoId = BDOInfo.get("bcoId").getAsString(); - JsonObject appendixes = BDOInfo.get("appendixes").getAsJsonObject(); - - // 将所有的functions通过engine注册到functionBindings中 - JsonObject accessRulesJO = appendixes.get(bcoId).getAsJsonObject().get("accessRules").getAsJsonObject(); - JsonElement functions = accessRulesJO.get("functions"); - if (functions != null && !functions.isJsonNull() && functions.isJsonObject()) { - JsonObject functionsObj = functions.getAsJsonObject(); - for (String funcName : functionsObj.keySet()) { - String funcScript = functionsObj.get(funcName).getAsString(); - engineUtil.evalFunction(funcScript); - } - } - - // 解析RouteInfo - Gson gson = new Gson(); - JsonElement routeInfos = accessRulesJO.get("routeInfo"); - if (routeInfos != null && !routeInfos.isJsonNull() && routeInfos.isJsonObject()) { - JsonObject routeInfosMap = routeInfos.getAsJsonObject(); - - for (String doipOperationName : routeInfosMap.keySet()) { - JsonObject doipFunctionRouteInfoJson = routeInfosMap.get(doipOperationName).getAsJsonObject(); - RouteInfo doipFunctionRouteInfo = gson.fromJson(doipFunctionRouteInfoJson, RouteInfo.class); - doipOperationToRouteInfo.put(doipOperationName, doipFunctionRouteInfo); - } - } - - // 解析JoinInfo - JsonElement joinInfos = accessRulesJO.get("joinInfo"); - if (joinInfos != null && !joinInfos.isJsonNull() && joinInfos.isJsonObject()) { - JsonObject joinInfosMap = joinInfos.getAsJsonObject(); - - for (String doipOperationName : joinInfosMap.keySet()) { - JsonObject doipFunctionJoinInfoJson = joinInfosMap.get(doipOperationName).getAsJsonObject(); - JoinInfo doipFunctionJoinInfo = gson.fromJson(doipFunctionJoinInfoJson, JoinInfo.class); - doipOperationToJoinInfo.put(doipOperationName, doipFunctionJoinInfo); - } - } - } -} -- Gitee