From ec9705834cff909b60e7efe508ca9452c8ebc3c7 Mon Sep 17 00:00:00 2001 From: WangYihan Date: Wed, 6 Dec 2017 21:53:04 +0800 Subject: [PATCH 1/9] FirstVersion: Exception Manager(Spark) --- k2de-governance-exception-manager/pom.xml | 179 ++++++++------ .../manager/ExceptionManagerSingle.java | 150 ++++++++++-- .../manager/ExceptionManagerSpark.java | 94 +++++++ .../manager/analyzer/GWAnalyzeResult.java | 100 ++++++++ .../manager/analyzer/GWWindTurbineKey.java | 71 ++++++ .../manager/analyzer/GWhnAnalyzer.java | 97 +++++--- .../manager/analyzer/GWkmxAnalyzer.java | 71 ++++-- .../manager/analyzer/IAnalyzer.java | 4 + .../common/{ => constants}/Constants.java | 2 +- .../common/constants/SparkConstants.java | 14 ++ .../manager/common/type/ErrorType.java | 13 + .../manager/common/type/GWDataType.java | 8 +- .../manager/common/type/HNErrorType.java | 8 +- .../governance/manager/filter/GWFilter.java | 22 +- .../governance/manager/message/GWMessage.java | 2 +- .../metadata/AbstractGWMetaDatabase.java | 22 +- .../metadata/GWSqliteMetaDatabase.java | 61 +++-- .../manager/parser/GWParserSingle.java | 3 + .../governance/manager/parser/ParserMR.java | 4 +- .../manager/util/ConfigureUtil.java | 230 ++++++++++++++++++ .../governance/manager/util/InitUtil.java | 24 ++ .../src/main/resources/config.properties | 3 + .../src/main/resources/log4j.properties | 2 +- pom.xml | 9 + 24 files changed, 970 insertions(+), 223 deletions(-) create mode 100644 k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/ExceptionManagerSpark.java create mode 100644 k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/GWAnalyzeResult.java create mode 100644 k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/GWWindTurbineKey.java rename k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/common/{ => constants}/Constants.java (93%) create mode 100644 k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/common/constants/SparkConstants.java create mode 100644 k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/common/type/ErrorType.java create mode 100644 k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/util/ConfigureUtil.java create mode 100644 k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/util/InitUtil.java create mode 100644 k2de-governance-exception-manager/src/main/resources/config.properties diff --git a/k2de-governance-exception-manager/pom.xml b/k2de-governance-exception-manager/pom.xml index dc1cef1..3035a24 100644 --- a/k2de-governance-exception-manager/pom.xml +++ b/k2de-governance-exception-manager/pom.xml @@ -1,92 +1,115 @@ - 4.0.0 + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + 4.0.0 - - com.k2data - k2de-governance-parent - 1.0-SNAPSHOT - + + com.k2data + k2de-governance-parent + 1.0-SNAPSHOT + - k2de-governance-exception-manager - 0.0.1-SNAPSHOT - jar + k2de-governance-exception-manager + 0.0.1-SNAPSHOT + jar - ExceptionManager - http://maven.apache.org + ExceptionManager + http://maven.apache.org - - UTF-8 - 1.8 - 2.6.0 - + + UTF-8 + 1.8 + 2.6.0 + - - - org.apache.hadoop - hadoop-client - + + + org.apache.hadoop + hadoop-client + + + javax.servlet + * + + + - - org.apache.hadoop - hadoop-common - + + org.apache.hadoop + hadoop-common + + + javax.servlet + * + + + - - org.apache.hadoop - hadoop-hdfs - + + org.apache.hadoop + hadoop-hdfs + + + javax.servlet + * + + + - - org.xerial - sqlite-jdbc - + + org.xerial + sqlite-jdbc + - - com.sun - tools - 1.4.2 - system - ${java.home}/../lib/tools.jar - + + org.apache.spark + spark-core_${spark.comp.version} + - + + com.sun + tools + 1.4.2 + system + ${java.home}/../lib/tools.jar + + - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.6.1 - - ${compile.version} - ${compile.version} - - - - maven-assembly-plugin - 2.4 - - - - com.k2data.governance.manager.ExceptionManagerSingle - - - - jar-with-dependencies - - - - - make-assembly - package - - single - - - - - - + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.6.1 + + ${compile.version} + ${compile.version} + + + + maven-assembly-plugin + 2.4 + + + + com.k2data.governance.manager.ExceptionManagerSingle + + + + jar-with-dependencies + + + + + make-assembly + package + + single + + + + + + diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/ExceptionManagerSingle.java b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/ExceptionManagerSingle.java index 2d8d647..f6fa8e1 100644 --- a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/ExceptionManagerSingle.java +++ b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/ExceptionManagerSingle.java @@ -1,9 +1,12 @@ package com.k2data.governance.manager; -import com.k2data.governance.manager.analyzer.GWhnAnalyzer; -import com.k2data.governance.manager.analyzer.GWkmxAnalyzer; +import com.k2data.governance.manager.analyzer.GWHnAnalyzer; +import com.k2data.governance.manager.analyzer.GWKmxAnalyzer; +import com.k2data.governance.manager.analyzer.GWWindTurbineKey; import com.k2data.governance.manager.analyzer.IAnalyzer; -import com.k2data.governance.manager.common.Constants; +import com.k2data.governance.manager.common.constants.Constants; +import com.k2data.governance.manager.common.Pair; +import com.k2data.governance.manager.common.type.ErrorType; import com.k2data.governance.manager.common.type.GWDataType; import com.k2data.governance.manager.common.type.GWSourceType; import com.k2data.governance.manager.common.type.HNErrorType; @@ -12,12 +15,14 @@ import com.k2data.governance.manager.filter.IFilter; import com.k2data.governance.manager.message.IMessage; import com.k2data.governance.manager.parser.GWParserSingle; import com.k2data.governance.manager.util.FileHandler; +import org.apache.commons.lang.StringUtils; import java.io.File; import java.io.FileWriter; import java.io.IOException; import java.io.PrintWriter; -import java.util.List; +import java.nio.file.Paths; +import java.util.*; /** * Created by stoke on 2017/11/16. @@ -32,10 +37,16 @@ public class ExceptionManagerSingle { private int[] kmxTypeNums; private int[] hnTypeNums; + private Map> exceptionCount; + private Map>> uniqueIds; + private final String wtidKeyName = "wtid"; + private final String protocolKeyName = "protocolId"; public ExceptionManagerSingle() { kmxTypeNums = new int[GWDataType.values().length]; hnTypeNums = new int[HNErrorType.values().length]; + exceptionCount = new HashMap<>(); + uniqueIds = new HashMap<>(); } public int[] getKmxTypeNums() { @@ -46,6 +57,50 @@ public class ExceptionManagerSingle { return hnTypeNums; } + private void updateExceptionCount(Map> newCount) { + GWWindTurbineKey allKey = new GWWindTurbineKey("all", "all"); + newCount.forEach((errorType, typeCount) -> { + boolean updateIds = false; + if (errorType instanceof HNErrorType) { + if (!uniqueIds.containsKey(errorType)) { + Map> idsMap = new HashMap<>(); + idsMap.put(wtidKeyName, new HashSet<>()); + idsMap.put(protocolKeyName, new HashSet<>()); + uniqueIds.put(errorType, idsMap); + } + updateIds = true; + } + if (!exceptionCount.containsKey(errorType)) { + exceptionCount.put(errorType, new HashMap<>()); + } + Map originCount = exceptionCount.get(errorType); + if (!originCount.containsKey(allKey)) { + originCount.put(allKey, 0); + } + Map> idsMap = uniqueIds.get(errorType); + for (Map.Entry entry : typeCount.entrySet()) { + GWWindTurbineKey key = entry.getKey(); + int count = entry.getValue(); + if (!originCount.containsKey(key)) { + originCount.put(key, 0); + } + originCount.put(key, originCount.get(key) + count); + originCount.put(allKey, originCount.get(allKey) + count); + if (!updateIds) { + continue; + } + String wtid = key.getWtid(); + if (wtid != null) { + idsMap.get(wtidKeyName).add(wtid); + } + String protocolId = key.getProtocolId(); + if (protocolId != null) { + idsMap.get(protocolKeyName).add(protocolId); + } + } + }); + } + private void sumTypeNums(int[] singleTypeNums, int[] finalTypeNums) { if (singleTypeNums.length != finalTypeNums.length) { System.out.println("Length does not match"); @@ -81,6 +136,67 @@ public class ExceptionManagerSingle { } } + private StringBuilder appendIdsInfo(ErrorType errorType, StringBuilder sb) { + Map> idsMap = uniqueIds.get(errorType); + if (idsMap == null) { + return sb; + } + for (Map.Entry> entry : idsMap.entrySet()) { + String idType = entry.getKey(); + Set ids = entry.getValue(); + sb.append(String.format("%s,%s:%s,IDs:%s\r\n", errorType, idType, + ids.size(), StringUtils.join(ids, ";"))); + } + return sb; + } + + private StringBuilder appendCountInfo(ErrorType errorType, + Map typeCountMap, StringBuilder sb ) { + for (Map.Entry countEntry : typeCountMap.entrySet()) { + GWWindTurbineKey key = countEntry.getKey(); + if (key == null) { + continue; + } + int count = countEntry.getValue(); + sb.append(String.format("%s,%s,%s,%s\r\n", errorType, key.getWtid(), key.getProtocolId(), count)); + } + return sb; + } + + private Pair getCountString() { + StringBuilder kmxExceptionStringBuilder = new StringBuilder(); + StringBuilder hnExceptionStringBuilder = new StringBuilder(); + kmxExceptionStringBuilder.append("ExceptionType,Count\r\n"); + hnExceptionStringBuilder.append("ExceptionType,WTID,ProtocolID,Count\r\n"); + for (Map.Entry> entry : exceptionCount.entrySet()) { + ErrorType errorType = entry.getKey(); + StringBuilder activeBuilder; + if (errorType instanceof HNErrorType) { + activeBuilder = appendIdsInfo(errorType, hnExceptionStringBuilder); + } else { + activeBuilder = kmxExceptionStringBuilder; + } + activeBuilder = appendCountInfo(errorType, entry.getValue(), activeBuilder); + } + return new Pair<>(hnExceptionStringBuilder.toString(), kmxExceptionStringBuilder.toString()); + } + + private void writeExceptionCount(String outputPath) { + String hnCountFilename = Paths.get(outputPath, "hn_exceptions_count.out").toString(); + String kmxCountFilename = Paths.get(outputPath, "kmx_exceptions_count.out").toString(); + try { + Pair countStrings = getCountString(); + FileWriter fw = new FileWriter(hnCountFilename); + fw.write(countStrings.fst); + fw.close(); + fw = new FileWriter(kmxCountFilename); + fw.write(countStrings.snd); + fw.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + public static void main(String[] args) { System.out.println("ExceptionManagerSingle begin..."); @@ -94,12 +210,9 @@ public class ExceptionManagerSingle { return; } - // String parsePath = "result/anomaly-10k"; List messageList; int sel = Integer.parseInt(args[0]); String inputPath = args[1]; - // -1 for printing all - int exampleNum = Integer.parseInt(args[2]); String dbPath = args[3]; if (!dbPath.trim().equals("")) { Constants.PROTOCOL_MAP_DIR = dbPath; @@ -120,48 +233,43 @@ public class ExceptionManagerSingle { ExceptionManagerSingle ems = new ExceptionManagerSingle(); + assert fileNames != null; for (String fileName : fileNames) { if (!FileHandler.isGWFileValid(fileName)) { continue; } if (sel == 1) { - messageList = FileHandler.readAnomalyString(inputPath + fileName); + messageList = FileHandler.readAnomalyString(Paths.get(inputPath, fileName).toString()); } else { - GWParserSingle parser = new GWParserSingle(inputPath + fileName); + GWParserSingle parser = new GWParserSingle(Paths.get(inputPath, fileName).toString()); parser.parse(); messageList = parser.getMessageList(); -// FileHandler.writeAnomalyString(outputPath + fileName + ".out", messageList); } IFilter filter = new GWFilter(messageList); filter.filter(); // KMX error - IAnalyzer kmxAnalyzer = new GWkmxAnalyzer( + IAnalyzer kmxAnalyzer = new GWKmxAnalyzer( filter.getCertainSourceTypeList(GWSourceType.KMX)); kmxAnalyzer.printNumber(); for (IMessage message : kmxAnalyzer.getMessageList()) { kmxAnalyzer.analyze(message); } - int[] kmxNums = kmxAnalyzer.printType(); - ems.sumTypeNums(kmxNums, ems.getKmxTypeNums()); - // kmxAnalyzer.printCertainTypeList(GWDataType.DOUBLE.getValue(), exampleNum); + ems.updateExceptionCount(kmxAnalyzer.countExceptions()); // HN error - IAnalyzer hnAnalyzer = new GWhnAnalyzer(Constants.PROTOCOL_MAP_DIR, + IAnalyzer hnAnalyzer = new GWHnAnalyzer(Constants.PROTOCOL_MAP_DIR, filter.getCertainSourceTypeList(GWSourceType.HN)); hnAnalyzer.printNumber(); for (IMessage message : hnAnalyzer.getMessageList()) { hnAnalyzer.analyze(message); } - int[] hnNums = hnAnalyzer.printType(); - ems.sumTypeNums(hnNums, ems.getHnTypeNums()); -// for (HNErrorType hType : HNErrorType.values()) { -// hnAnalyzer.printCertainTypeList(hType.getValue(), exampleNum); -// } + ems.updateExceptionCount(hnAnalyzer.countExceptions()); } - ems.printResult(outputPath); + System.out.println("Building output file"); + ems.writeExceptionCount(outputPath); System.out.println("ExceptionManagerSingle end"); } } diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/ExceptionManagerSpark.java b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/ExceptionManagerSpark.java new file mode 100644 index 0000000..c0feaba --- /dev/null +++ b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/ExceptionManagerSpark.java @@ -0,0 +1,94 @@ +package com.k2data.governance.manager; + +import com.k2data.governance.manager.analyzer.*; +import com.k2data.governance.manager.common.type.GWSourceType; +import com.k2data.governance.manager.filter.GWFilter; +import com.k2data.governance.manager.util.InitUtil; +import org.apache.hadoop.io.Text; +import org.apache.log4j.Logger; +import org.apache.spark.api.java.JavaPairRDD; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.api.java.function.Function; +import org.apache.spark.broadcast.Broadcast; +import scala.Tuple2; + +import java.util.HashSet; +import java.util.Set; + +/** + * Created by wangyihan on 2017/12/6 下午3:29. + * E-mail address is yihanwang22@163.com. + * Copyright © 2017 wangyihan. All Rights Reserved. + * + * @author wangyihan + */ +public class ExceptionManagerSpark { + private Logger logger = Logger.getLogger(ExceptionManagerSpark.class); + + public Logger getLogger() { + return logger; + } + + public static void main(String[] args) { + ExceptionManagerSpark app = new ExceptionManagerSpark(); + Logger logger = app.getLogger(); + if (args.length != 3) { + logger.error("Invalid arguments"); + System.exit(-1); + } + String inputFilePath = args[0]; + String outputFilePath = args[1]; + String metadataFilePath = args[2]; + JavaSparkContext sc = InitUtil.initSparkContext(); + JavaPairRDD dataRdd = sc.sequenceFile(inputFilePath, Text.class, Text.class); + IAnalyzer hnAnalyzer = new GWHnAnalyzer(metadataFilePath); + IAnalyzer kmxAnalyzer = new GWKmxAnalyzer(); + Tuple2 analyzers = new Tuple2<>(hnAnalyzer, kmxAnalyzer); + final Broadcast> broadcastAnalyzer =sc.broadcast(analyzers); + + // analyze exceptions + JavaRDD exceptionRdd = dataRdd.map(new Function, GWAnalyzeResult>() { + IAnalyzer hnAnalyzer = broadcastAnalyzer.value()._1(); + IAnalyzer kmxAnalyzer = broadcastAnalyzer.value()._2(); + @Override + public GWAnalyzeResult call(Tuple2 pair) throws Exception { + String message = pair._2().toString(); + GWSourceType sourceType = GWFilter.getSourceType(message); + GWAnalyzeResult result = null; + if (sourceType.equals(GWSourceType.HN)) { + result = hnAnalyzer.analyzeRecord(message); + } else if (sourceType.equals(GWSourceType.KMX)) { + result = kmxAnalyzer.analyzeRecord(message); + } + return result; + } + }); + + exceptionRdd = exceptionRdd.cache(); + System.out.println(exceptionRdd.count()); + + // exception type count + JavaPairRDD typeRdd = exceptionRdd + .mapToPair(result -> new Tuple2<>(result, 1)) + .reduceByKey((value1, value2) -> value1 + value2); + System.out.println(typeRdd.count()); + + // protocol count + JavaPairRDD> protocolRdd = exceptionRdd + .groupBy(GWAnalyzeResult::getProtocolId) + .mapToPair(pair -> { + String protocolId = pair._1(); + Iterable results = pair._2(); + int recordCount = 0; + Set wtSet = new HashSet<>(); + for (GWAnalyzeResult result : results) { + recordCount += 1; + wtSet.add(result.getWtid()); + } + return new Tuple2<>(protocolId, new Tuple2<>(wtSet.size(), recordCount)); + }); + + System.out.println(protocolRdd.count()); + } +} diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/GWAnalyzeResult.java b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/GWAnalyzeResult.java new file mode 100644 index 0000000..52efb28 --- /dev/null +++ b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/GWAnalyzeResult.java @@ -0,0 +1,100 @@ +package com.k2data.governance.manager.analyzer; + +import com.k2data.governance.manager.common.type.ErrorType; +import scala.Serializable; + +/** + * Created by wangyihan on 2017/11/30 下午4:27. + * E-mail address is yihanwang22@163.com. + * Copyright © 2017 wangyihan. All Rights Reserved. + * + * @author wangyihan + */ +public class GWAnalyzeResult implements Serializable { + private ErrorType errorType; + private GWWindTurbineKey wt; + private String originalData; + + public GWAnalyzeResult() { + this.wt = new GWWindTurbineKey(null, null); + } + + public GWAnalyzeResult(ErrorType errorType) { + this.wt = new GWWindTurbineKey(null, null); + this.errorType = errorType; + } + + @Override + public String toString() { + return String.format("ExceptionType: %s, %s", errorType, wt); + } + + @Override + public int hashCode() { + return this.toString().hashCode(); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (!(obj instanceof GWAnalyzeResult)) { + return false; + } + GWAnalyzeResult resultObj = (GWAnalyzeResult)obj; + if (this.wt == null && resultObj.getWindTurbineKey() != null) { + return false; + } else if (this.wt != null && !this.wt.equals(resultObj.getWindTurbineKey())) { + return false; + } else if (this.errorType != resultObj.getErrorType()) { + return false; + } + return true; + } + + public void setErrorType(ErrorType errorType) { + this.errorType = errorType; + } + + public void setWtid(String wtid) { + this.wt.setWtid(wtid); + } + + public void setProtocolId(String protocolId) { + this.wt.setProtocolId(protocolId); + } + + public void setOriginalData(String originalData) { + this.originalData = originalData; + } + + public GWWindTurbineKey getWindTurbineKey() { + return wt; + } + + public ErrorType getErrorType() { + return errorType; + } + + public String getWtid() { + if (wt == null) { + return ""; + } + return wt.getWtid(); + } + + public String getProtocolId() { + if (wt == null) { + return ""; + } + return wt.getProtocolId(); + } + + public String getOriginalData() { + return originalData; + } +} diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/GWWindTurbineKey.java b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/GWWindTurbineKey.java new file mode 100644 index 0000000..432f737 --- /dev/null +++ b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/GWWindTurbineKey.java @@ -0,0 +1,71 @@ +package com.k2data.governance.manager.analyzer; + +import scala.Serializable; + +/** + * Created by wangyihan on 2017/12/1 上午11:02. + * E-mail address is yihanwang22@163.com. + * Copyright © 2017 wangyihan. All Rights Reserved. + * + * @author wangyihan + */ +public class GWWindTurbineKey implements Serializable { + private String wtid; + private String protocolId; + + public GWWindTurbineKey(String wtid, String protocolId) { + this.wtid = wtid; + this.protocolId = protocolId; + } + + public String getWtid() { + return wtid; + } + + public String getProtocolId() { + return protocolId; + } + + public void setWtid(String wtid) { + this.wtid = wtid; + } + + public void setProtocolId(String protocolId) { + this.protocolId = protocolId; + } + + @Override + public String toString() { + return String.format("WTID: %s, ProtocolID: %s", wtid, protocolId); + } + + @Override + public int hashCode() { + return this.toString().hashCode(); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (!(obj instanceof GWWindTurbineKey)) { + return false; + } + GWWindTurbineKey wtObj = (GWWindTurbineKey)obj; + if (this.wtid != null && !this.wtid.equals(wtObj.getWtid())) { + return false; + } else if (this.wtid == null && wtObj.getWtid() != null) { + return false; + } + if (this.protocolId != null && !this.protocolId.equals(wtObj.getProtocolId())) { + return false; + } else if (this.protocolId == null && wtObj.getProtocolId() != null) { + return false; + } + return true; + } +} diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/GWhnAnalyzer.java b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/GWhnAnalyzer.java index e0c5610..529944e 100644 --- a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/GWhnAnalyzer.java +++ b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/GWhnAnalyzer.java @@ -1,10 +1,12 @@ package com.k2data.governance.manager.analyzer; -import com.k2data.governance.manager.common.Constants; +import com.k2data.governance.manager.common.constants.Constants; +import com.k2data.governance.manager.common.type.ErrorType; import com.k2data.governance.manager.common.type.HNErrorType; import com.k2data.governance.manager.message.IMessage; import com.k2data.governance.manager.metadata.GWSqliteMetaDatabase; import com.k2data.governance.manager.metadata.IMetaDatabase; +import scala.Serializable; import java.text.ParseException; import java.text.SimpleDateFormat; @@ -20,21 +22,27 @@ import java.util.Map; * * @author stoke */ -public class GWhnAnalyzer implements IAnalyzer { +public class GWHnAnalyzer implements IAnalyzer, Serializable { private IMetaDatabase metaDatabase; private SimpleDateFormat format = new SimpleDateFormat(Constants.TIME_FORMAT); private List messageList; - private Map> hnTypeMessageMap; + private Map> hnTypeMessageMap; + private Map> errorTypeResultMap; + private int fieldDiffTolerance = 1; - public GWhnAnalyzer(String dbFilePath, List messageList) { + public GWHnAnalyzer(String dbFilePath) { + this(dbFilePath, null); + } + + public GWHnAnalyzer(String dbFilePath, List messageList) { metaDatabase = new GWSqliteMetaDatabase(dbFilePath); this.messageList = messageList; - hnTypeMessageMap = new HashMap<>(); + errorTypeResultMap = new HashMap<>(); for (HNErrorType type : HNErrorType.values()) { - hnTypeMessageMap.put(type, new ArrayList<>()); + errorTypeResultMap.put(type, new ArrayList<>()); } } @@ -51,62 +59,63 @@ public class GWhnAnalyzer implements IAnalyzer { */ @Override public void analyze(IMessage message) { - String key = message.getKey(); - if (!key.trim().equals("")) { - System.out.println(key); - } - String msg = message.getMsg(); + GWAnalyzeResult analyzeResult = analyzeRecord(msg); + ErrorType errorType = analyzeResult.getErrorType(); + errorTypeResultMap.get(errorType).add(analyzeResult); + } + public void setFieldDiffTolerance(int fieldDiffTolerance) { + this.fieldDiffTolerance = fieldDiffTolerance; + } + + public GWAnalyzeResult analyzeRecord(String msg) { + GWAnalyzeResult result = new GWAnalyzeResult(); int formatIndex = msg.indexOf(Constants.GW_HN_DATA_START_TAG); if (formatIndex < 0) { - hnTypeMessageMap.get(HNErrorType.FORMAT).add(message); - return; + result.setErrorType(HNErrorType.FORMAT); + return result; } - int startIndex = formatIndex + Constants.GW_HN_DATA_START_TAG.length(); String dataSubstr = msg.substring(startIndex); String[] splitData = dataSubstr.split("\\|"); String wtid = splitData[0].trim(); String[] data = splitData[1].split(","); - String ts = data[0]; try { format.parse(ts); } catch (ParseException e) { - hnTypeMessageMap.get(HNErrorType.TIME).add(message); - return; + result.setErrorType(HNErrorType.TIME); } - if (wtid.trim().isEmpty()) { - hnTypeMessageMap.get(HNErrorType.WTID_EMPTY).add(message); - return; + result.setErrorType(HNErrorType.WTID_EMPTY); + return result; } - - // String wfid = ((GWSqliteMetaDatabase)metaDatabase).getWfid(wtid); + result.setWtid(wtid); String fieldGroup = metaDatabase.getFieldGroup(wtid); if (fieldGroup == null) { - hnTypeMessageMap.get(HNErrorType.PROTO_NULL).add(message); - return; + result.setErrorType(HNErrorType.WTID_NULL); + return result; } - + result.setProtocolId(fieldGroup); List fields = metaDatabase.getFieldsByFieldGroup(fieldGroup); if (fields == null) { - hnTypeMessageMap.get(HNErrorType.FIELDS_NULL).add(message); - return; + result.setErrorType(HNErrorType.PROTO_NULL); + result.setWtid(null); // this type of exception dose not need wtid + return result; } - - int fSize = fields.size(); - if (fSize > data.length - 1) { - hnTypeMessageMap.get(HNErrorType.FIELDS_NUM_LESS).add(message); - return; - } else if (fSize < data.length - 1) { - hnTypeMessageMap.get(HNErrorType.FIELDS_NUM_MORE).add(message); - return; + if (fields.size() - (data.length - 1) > fieldDiffTolerance) { + result.setErrorType(HNErrorType.FIELDS_NUM_LESS); + return result; + } + if ((data.length - 1) - fields.size() > fieldDiffTolerance) { + result.setErrorType(HNErrorType.FIELDS_NUM_MORE); + return result; } // if not in all errors above, then put it in OTHER - hnTypeMessageMap.get(HNErrorType.OHTER).add(message); + result.setErrorType(HNErrorType.OTHER); + return result; } @Override @@ -156,4 +165,20 @@ public class GWhnAnalyzer implements IAnalyzer { System.out.println(msgList.get(i).toString()); } } + + public Map> countExceptions() { + Map> countResult = new HashMap<>(); + errorTypeResultMap.forEach((errorType, resultList) -> { + Map typeResultCount = new HashMap<>(); + for (GWAnalyzeResult result : resultList) { + GWWindTurbineKey key = result.getWindTurbineKey(); + if (!typeResultCount.containsKey(key)) { + typeResultCount.put(key, 0); + } + typeResultCount.put(key, typeResultCount.get(key) + 1); + } + countResult.put(errorType, typeResultCount); + }); + return countResult; + } } diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/GWkmxAnalyzer.java b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/GWkmxAnalyzer.java index 9ba1553..7b48e6f 100644 --- a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/GWkmxAnalyzer.java +++ b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/GWkmxAnalyzer.java @@ -1,12 +1,16 @@ package com.k2data.governance.manager.analyzer; +import com.k2data.governance.manager.common.type.ErrorType; import com.k2data.governance.manager.common.type.GWDataType; import com.k2data.governance.manager.message.IMessage; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; + +import org.apache.hadoop.io.serializer.Serialization; import org.apache.log4j.Logger; +import scala.Serializable; /** * Created by stoke on 2017/11/16. @@ -17,7 +21,7 @@ import org.apache.log4j.Logger; * * @author stoke */ -public class GWkmxAnalyzer implements IAnalyzer { +public class GWKmxAnalyzer implements IAnalyzer, Serializable { private static final String TYPE_PREFIX = ": "; private static final String TYPE_SUFFIX = "_EX"; @@ -28,14 +32,18 @@ public class GWkmxAnalyzer implements IAnalyzer { private static final String FIELD_PREFIX = "field "; private static final String FIELD_SUFFIX = " [GENERATED"; - private Logger logger = Logger.getLogger(GWkmxAnalyzer.class); - private List messageList; - private Map> kmxTypeMessageMap; + private Map> kmxTypeMessageMap; + private Map> errorTypeResultMap; + + public GWKmxAnalyzer() { + this(null); + } - public GWkmxAnalyzer(List messageList) { + public GWKmxAnalyzer(List messageList) { this.messageList = messageList; kmxTypeMessageMap = new HashMap<>(); + errorTypeResultMap = new HashMap<>(); } @Override @@ -52,23 +60,15 @@ public class GWkmxAnalyzer implements IAnalyzer { */ @Override public void analyze(IMessage message) { - String key = message.getKey(); - if (!key.trim().equals("")) { - System.out.println(key); - } - String msg = message.getMsg(); - int indexType = msg.indexOf(TYPE_SUFFIX); - if (indexType == -1) { - String err = String.format("The pattern is wrong since no _EX included in %s", msg); - logger.error(err); + GWAnalyzeResult analyzeResult = analyzeRecord(msg); + ErrorType errorType = analyzeResult.getErrorType(); + if (!kmxTypeMessageMap.containsKey(errorType)) { +// kmxTypeMessageMap.put(errorType, new ArrayList<>()); + errorTypeResultMap.put(errorType, new ArrayList<>()); } - String type = msg.substring(msg.indexOf(TYPE_PREFIX) + TYPE_PREFIX.length(), indexType); - GWDataType dataType = GWDataType.fromString(type); - if (!kmxTypeMessageMap.containsKey(dataType)) { - kmxTypeMessageMap.put(dataType, new ArrayList<>()); - } - kmxTypeMessageMap.get(dataType).add(message); +// kmxTypeMessageMap.get(errorType).add(message); + errorTypeResultMap.get(errorType).add(analyzeResult); // String type2 = msg.substring(msg.indexOf(TYPE_PREFIX_SMALL) + TYPE_PREFIX_SMALL.length(), // msg.indexOf(TYPE_SUFFIX_SMALL)); @@ -79,6 +79,17 @@ public class GWkmxAnalyzer implements IAnalyzer { // // TODO check the value type } + @Override + public GWAnalyzeResult analyzeRecord(String msg) { + int indexType = msg.indexOf(TYPE_SUFFIX); + if (indexType == -1) { + String err = String.format("The pattern is wrong since no _EX included in %s", msg); + getLogger().error(err); + } + String type = msg.substring(msg.indexOf(TYPE_PREFIX) + TYPE_PREFIX.length(), indexType); + return new GWAnalyzeResult(GWDataType.fromString(type)); + } + @Override public void printNumber() { System.out.println("KMX anomaly number : " + messageList.size()); @@ -118,4 +129,24 @@ public class GWkmxAnalyzer implements IAnalyzer { System.out.println(msgList.get(i).toString()); } } + + public Map> countExceptions() { + Map> countResult = new HashMap<>(); + errorTypeResultMap.forEach((errorType, resultList) -> { + Map typeResultCount = new HashMap<>(); + for (GWAnalyzeResult result : resultList) { + GWWindTurbineKey key = result.getWindTurbineKey(); + if (!typeResultCount.containsKey(key)) { + typeResultCount.put(key, 0); + } + typeResultCount.put(key, typeResultCount.get(key) + 1); + } + countResult.put(errorType, typeResultCount); + }); + return countResult; + } + + private Logger getLogger() { + return Logger.getLogger(GWKmxAnalyzer.class); + } } diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/IAnalyzer.java b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/IAnalyzer.java index 82ba7de..ce780ec 100644 --- a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/IAnalyzer.java +++ b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/IAnalyzer.java @@ -1,7 +1,9 @@ package com.k2data.governance.manager.analyzer; +import com.k2data.governance.manager.common.type.ErrorType; import com.k2data.governance.manager.message.IMessage; import java.util.List; +import java.util.Map; /** * Created by stoke on 2017/11/16. @@ -13,8 +15,10 @@ import java.util.List; public interface IAnalyzer { List getMessageList(); void analyze(IMessage message); + GWAnalyzeResult analyzeRecord(String msg); void printNumber(); void printAll(); int[] printType(); void printCertainTypeList(int id, int size); + Map> countExceptions(); } diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/common/Constants.java b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/common/constants/Constants.java similarity index 93% rename from k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/common/Constants.java rename to k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/common/constants/Constants.java index 64e08fb..9d90ab7 100644 --- a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/common/Constants.java +++ b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/common/constants/Constants.java @@ -1,4 +1,4 @@ -package com.k2data.governance.manager.common; +package com.k2data.governance.manager.common.constants; /** * Created by stoke on 2017/11/16. diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/common/constants/SparkConstants.java b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/common/constants/SparkConstants.java new file mode 100644 index 0000000..d1642b9 --- /dev/null +++ b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/common/constants/SparkConstants.java @@ -0,0 +1,14 @@ +package com.k2data.governance.manager.common.constants; + +/** + * Created by wangyihan on 2017/12/6 下午4:04. + * E-mail address is yihanwang22@163.com. + * Copyright © 2017 wangyihan. All Rights Reserved. + * + * @author wangyihan + */ +public class SparkConstants { + public final static String SPARK_LOCAL = "spark.local"; + public final static String SPARK_MASTER = "spark.master"; + public final static String SPARK_APP_NAME = "spark.app.name"; +} diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/common/type/ErrorType.java b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/common/type/ErrorType.java new file mode 100644 index 0000000..ea0ff87 --- /dev/null +++ b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/common/type/ErrorType.java @@ -0,0 +1,13 @@ +package com.k2data.governance.manager.common.type; + +/** + * Created by wangyihan on 2017/11/30 下午2:08. + * E-mail address is yihanwang22@163.com. + * Copyright © 2017 wangyihan. All Rights Reserved. + * + * @author wangyihan + */ +public interface ErrorType { + String toErrorString(); + int getValue(); +} diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/common/type/GWDataType.java b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/common/type/GWDataType.java index 39539c5..33ca415 100644 --- a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/common/type/GWDataType.java +++ b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/common/type/GWDataType.java @@ -7,7 +7,7 @@ package com.k2data.governance.manager.common.type; * * @author stoke */ -public enum GWDataType { +public enum GWDataType implements ErrorType { DOUBLE(0), BOOLEAN(1), LONG(2), STRING(3), INT(4), FLOAT(5), OTHER(6); private final int value; @@ -16,10 +16,16 @@ public enum GWDataType { this.value = value; } + @Override public int getValue() { return value; } + @Override + public String toErrorString() { + return String.format("KMX_ERROR_DATATYPE_%s", this); + } + public static GWDataType fromInteger(int id) { return GWDataType.values()[id]; } diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/common/type/HNErrorType.java b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/common/type/HNErrorType.java index d63e965..50db060 100644 --- a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/common/type/HNErrorType.java +++ b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/common/type/HNErrorType.java @@ -7,7 +7,7 @@ package com.k2data.governance.manager.common.type; * * @author stoke */ -public enum HNErrorType { +public enum HNErrorType implements ErrorType { FORMAT(0), // no realtime| in the msg WTID_NULL(1), // wtid is not in DB TIME(2), // time format wrong @@ -16,7 +16,7 @@ public enum HNErrorType { FIELDS_NUM_LESS(5), // number of data less than in the DB WTID_EMPTY(6), FIELDS_NULL(7), // fields null - OHTER(8); + OTHER(8); private final int value; @@ -24,6 +24,10 @@ public enum HNErrorType { this.value = value; } + public String toErrorString() { + return String.format("HN_ERROR_%s", this); + } + public int getValue() { return value; } diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/filter/GWFilter.java b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/filter/GWFilter.java index 2b62d40..9bc6717 100644 --- a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/filter/GWFilter.java +++ b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/filter/GWFilter.java @@ -1,6 +1,6 @@ package com.k2data.governance.manager.filter; -import com.k2data.governance.manager.common.Constants; +import com.k2data.governance.manager.common.constants.Constants; import com.k2data.governance.manager.common.type.GWSourceType; import com.k2data.governance.manager.message.IMessage; @@ -51,17 +51,21 @@ public class GWFilter implements IFilter { public void filter() { for (IMessage message : messageList) { String msg = message.getMsg(); - - if (msg.startsWith(Constants.GW_KMX_START_TAG)) { - gwSourceTypeMessageMap.get(GWSourceType.KMX).add(message); - } else if (msg.startsWith(Constants.GW_HN_START_TAG)) { - gwSourceTypeMessageMap.get(GWSourceType.HN).add(message); - } else { - gwSourceTypeMessageMap.get(GWSourceType.OTHER).add(message); - String err = String.format("Messages for other type, %s", message.toString()); + GWSourceType type = GWFilter.getSourceType(msg); + gwSourceTypeMessageMap.get(type).add(message); + if (type == GWSourceType.OTHER) { + String err = String.format("Messages for other type: %s", message.toString()); logger.error(err); } } } + public static GWSourceType getSourceType(String message) { + if (message.startsWith(Constants.GW_KMX_START_TAG)) { + return GWSourceType.KMX; + } else if (message.startsWith(Constants.GW_HN_START_TAG)) { + return GWSourceType.HN; + } + return GWSourceType.OTHER; + } } diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/message/GWMessage.java b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/message/GWMessage.java index cf991ec..c54135b 100644 --- a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/message/GWMessage.java +++ b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/message/GWMessage.java @@ -1,6 +1,6 @@ package com.k2data.governance.manager.message; -import com.k2data.governance.manager.common.Constants; +import com.k2data.governance.manager.common.constants.Constants; /** * Created by stoke on 2017/11/16. diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/metadata/AbstractGWMetaDatabase.java b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/metadata/AbstractGWMetaDatabase.java index 8daa730..5f9138c 100644 --- a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/metadata/AbstractGWMetaDatabase.java +++ b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/metadata/AbstractGWMetaDatabase.java @@ -3,6 +3,7 @@ package com.k2data.governance.manager.metadata; import java.util.List; import java.util.Map; import org.apache.log4j.Logger; +import scala.Serializable; /** * Created by wangyihan on 2017/11/17 上午10:20. @@ -11,8 +12,7 @@ import org.apache.log4j.Logger; * * @author wangyihan */ -abstract class AbstractGWMetaDatabase implements IMetaDatabase { - Logger logger = getLogger(); +abstract class AbstractGWMetaDatabase implements IMetaDatabase, Serializable { Map wtidFieldGroupMap; Map> fieldGroupFieldsMap; @@ -38,26 +38,14 @@ abstract class AbstractGWMetaDatabase implements IMetaDatabase { @Override public String getFieldGroup(String wtid) { if (wtid == null || wtid.isEmpty()) { - logger.error("Empty wtid: " + wtid); + getLogger().error("Empty wtid: " + wtid); return null; } - String fieldGroup = wtidFieldGroupMap.get(wtid); -// if (fieldGroup == null) { -// logger.error("Could not find wtid info from database: " + wtid); -// } - return fieldGroup; + return wtidFieldGroupMap.get(wtid); } @Override public List getFieldsByFieldGroup(String fieldGroup) { -// if (fieldGroup == null || fieldGroup.isEmpty()) { -// logger.error("Empty fieldGroup: " + fieldGroup); -// return null; -// } - List fields = fieldGroupFieldsMap.get(fieldGroup); -// if (fields == null) { -// logger.error("Could not find fields info from database: " + fieldGroup); -// } - return fields; + return fieldGroupFieldsMap.get(fieldGroup); } } diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/metadata/GWSqliteMetaDatabase.java b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/metadata/GWSqliteMetaDatabase.java index 41fde8b..33f5470 100644 --- a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/metadata/GWSqliteMetaDatabase.java +++ b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/metadata/GWSqliteMetaDatabase.java @@ -1,6 +1,6 @@ package com.k2data.governance.manager.metadata; -import com.k2data.governance.manager.common.Constants; +import com.k2data.governance.manager.common.constants.Constants; import com.k2data.governance.manager.common.Pair; import java.io.IOException; import java.sql.Connection; @@ -16,6 +16,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.log4j.Logger; +import scala.Serializable; /** @@ -25,12 +26,10 @@ import org.apache.log4j.Logger; * * @author wangyihan */ -public class GWSqliteMetaDatabase extends AbstractGWMetaDatabase implements IMetaDatabase { +public class GWSqliteMetaDatabase extends AbstractGWMetaDatabase { private static final String SQLITE_JDBC_DRIVER = "org.sqlite.JDBC"; private static final String SQLITE_BASE_URL = "jdbc:sqlite:"; - private String dbFilePath; - private Connection conn; public GWSqliteMetaDatabase(String dbFilePath) { this.dbFilePath = dbFilePath; @@ -53,7 +52,7 @@ public class GWSqliteMetaDatabase extends AbstractGWMetaDatabase implements IMet Pattern p = Pattern.compile("^([A-Za-z]+:/+[^/]+).*"); Matcher m = p.matcher(dbFilePath); if (!m.find()) { - logger.error("Cannot parse the hdfs path:" + dbFilePath); + getLogger().error("Cannot parse the hdfs path:" + dbFilePath); return SQLITE_BASE_URL + dbFilePath; } String hdfs = m.group(1); @@ -65,60 +64,56 @@ public class GWSqliteMetaDatabase extends AbstractGWMetaDatabase implements IMet FileSystem fs = FileSystem.get(conf); fs.copyToLocalFile(false, hdfsPath, localPath); } catch (IOException e) { - logger.error("Initial file system failed"); + getLogger().error("Initial file system failed"); e.printStackTrace(); } return SQLITE_BASE_URL + Constants.GW_META_SQLITE_FILE_DEFAULT; } - private void connect() { + private Connection connect() { String url = getSqliteUrl(); - if (conn != null) { - try { - conn.close(); - } catch (SQLException e) { - logger.error("Close connection failed: " + e.getMessage()); - } - } try { Class.forName(SQLITE_JDBC_DRIVER); } catch (ClassNotFoundException e) { - logger.error("Load SQLite driver error: "+e.getMessage()); + getLogger().error("Load SQLite driver error: "+e.getMessage()); System.exit(-1); } try { - conn = DriverManager.getConnection(url); + Connection conn = DriverManager.getConnection(url); Statement statement = conn.createStatement(); statement.execute("PRAGMA synchronous=OFF"); statement.close(); + return conn; } catch (SQLException e) { - logger.error("Build connection failed: " + e.getMessage()); + getLogger().error("Build connection failed: " + e.getMessage()); System.exit(-1); } + return null; } - private Pair executeSql(String sql) { + private Pair executeSql(Connection conn, String sql) { if (conn == null) { - connect(); + conn = connect(); } Statement stat = null; ResultSet rs = null; try { + assert conn != null; stat = conn.createStatement(); rs = stat.executeQuery(sql); } catch (SQLException e) { String err = String.format("Execute SQL (%s) failed: %s", sql, e.getMessage()); - logger.error(err); + getLogger().error(err); } return new Pair<>(rs, stat); } - private void buildWtidFieldGroupMap() { + private void buildWtidFieldGroupMap(Connection conn) { if (wtidFieldGroupMap == null) { wtidFieldGroupMap = new HashMap<>(); } String sql = "SELECT wtid, protocolid FROM wtinfo"; - Pair resultPair = executeSql(sql); + Pair resultPair = executeSql(conn, sql); ResultSet rs = resultPair.fst; try { while (rs.next()) { @@ -129,19 +124,19 @@ public class GWSqliteMetaDatabase extends AbstractGWMetaDatabase implements IMet rs.close(); resultPair.snd.close(); } catch (SQLException e) { - logger.error("Read field group result set error: " + e.getMessage()); + getLogger().error("Read field group result set error: " + e.getMessage()); e.printStackTrace(); } } - private void buildFieldGroupFieldsMap() { + private void buildFieldGroupFieldsMap(Connection conn) { if (fieldGroupFieldsMap == null) { fieldGroupFieldsMap = new HashMap<>(); } String sql = "SELECT protocolid, iecpath FROM propaths " + "WHERE iecpath <> 'WTUR.Tm.Rw.Dt' AND transtype='1' AND bsave=1 " + "ORDER BY protocolid, pathid"; - Pair resultPair = executeSql(sql); + Pair resultPair = executeSql(conn, sql); ResultSet rs = resultPair.fst; try { while (rs.next()) { @@ -155,17 +150,17 @@ public class GWSqliteMetaDatabase extends AbstractGWMetaDatabase implements IMet rs.close(); resultPair.snd.close(); } catch (SQLException e) { - logger.error("Read fields result set error: " + e.getMessage()); + getLogger().error("Read fields result set error: " + e.getMessage()); e.printStackTrace(); } } - private void buildWfidMap() { + private void buildWfidMap(Connection conn) { if (wtidWfidMap == null) { wtidWfidMap = new HashMap<>(); } String sql = "SELECT wtid,wfid FROM wtinfo"; - Pair resultPair = executeSql(sql); + Pair resultPair = executeSql(conn, sql); ResultSet rs = resultPair.fst; try { while (rs.next()) { @@ -176,16 +171,16 @@ public class GWSqliteMetaDatabase extends AbstractGWMetaDatabase implements IMet rs.close(); resultPair.snd.close(); } catch (SQLException e) { - logger.error("Read WFID result set error: " + e.getMessage()); + getLogger().error("Read WFID result set error: " + e.getMessage()); e.printStackTrace(); } } @Override public void init() { - connect(); - buildWtidFieldGroupMap(); - buildFieldGroupFieldsMap(); - buildWfidMap(); + Connection conn = connect(); + buildWtidFieldGroupMap(conn); + buildFieldGroupFieldsMap(conn); + buildWfidMap(conn); } } diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/parser/GWParserSingle.java b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/parser/GWParserSingle.java index 64c06e2..51252f8 100644 --- a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/parser/GWParserSingle.java +++ b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/parser/GWParserSingle.java @@ -48,6 +48,9 @@ public class GWParserSingle { } catch (IOException e) { e.printStackTrace(); + } catch (OutOfMemoryError e) { + System.out.println(messageList.size()); + System.exit(-1); } } diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/parser/ParserMR.java b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/parser/ParserMR.java index b638217..41386d5 100644 --- a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/parser/ParserMR.java +++ b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/parser/ParserMR.java @@ -69,9 +69,7 @@ public class ParserMR { while (reader.next(key, value)) { context.write(key, value); } - } catch (IOException e) { - e.printStackTrace(); - } catch (InterruptedException e) { + } catch (IOException | InterruptedException e) { e.printStackTrace(); } } diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/util/ConfigureUtil.java b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/util/ConfigureUtil.java new file mode 100644 index 0000000..3f5cf57 --- /dev/null +++ b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/util/ConfigureUtil.java @@ -0,0 +1,230 @@ +package com.k2data.governance.manager.util; + +import org.apache.log4j.Logger; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Properties; + +/** + * Created by wangyihan on 2017/12/6 下午3:58. + * E-mail address is yihanwang22@163.com. + * Copyright © 2017 wangyihan. All Rights Reserved. + * + * @author wangyihan + */ +public class ConfigureUtil { + private static Properties properties = new Properties(); + private static final Logger logger = Logger.getLogger(ConfigureUtil.class); + + static { + InputStream in = ConfigureUtil.class.getClassLoader().getResourceAsStream("config.properties"); + try { + properties.load(in); + } catch (IOException e) { + logger.warn("No User Custom Properties File"); + } + } + + /** + * get property value with property key + * + * @param key property key + * @return + */ + private static String getProperty(String key) { + try { + return properties.getProperty(key); + } catch (Exception e) { + e.printStackTrace(); + } + + return null; + } + + /** + * get property value with property key + * + * @param key property key + * @param defaultValue + * @return + */ + private static String getProperty(String key, String defaultValue) { + try { + return properties.getProperty(key, defaultValue); + } catch (Exception e) { + e.printStackTrace(); + } + + return null; + } + + + /** + * get string value with property key + * + * @param key + * @return + */ + public static String getString(String key) { + return getProperty(key); + } + + + /** + * get string value with property key + * + * @param key + * @param defaultValue + * @return + */ + public static String getString(String key, String defaultValue) { + return getProperty(key, defaultValue); + } + + /** + * get integer value with property key + * + * @param key + * @return + */ + public static Integer getInteger(String key) { + String value = getProperty(key); + try { + return Integer.valueOf(value); + } catch (Exception e) { + e.printStackTrace(); + } + + return 0; + } + + + /** + * get integer value with property key + * + * @param key + * @param defaultValue + * @return + */ + public static Integer getInteger(String key, Integer defaultValue) { + String value = getProperty(key); + try { + return (value == null) ? defaultValue : Integer.valueOf(value); + } catch (Exception e) { + e.printStackTrace(); + } + + return 0; + } + + + /** + * get boolean value with property key + * + * @param key + * @return + */ + public static Boolean getBoolean(String key) { + String value = getProperty(key); + try { + return Boolean.valueOf(value); + } catch (Exception e) { + e.printStackTrace(); + } + + return false; + } + + + /** + * get boolean value with property key + * + * @param key + * @param defaultValue + * @return + */ + public static Boolean getBoolean(String key, Boolean defaultValue) { + String value = getProperty(key); + try { + return (value == null) ? defaultValue : Boolean.valueOf(value); + } catch (Exception e) { + e.printStackTrace(); + } + + return false; + } + + /** + * get long value with property key + * + * @param key + * @return + */ + public static Long getLong(String key) { + String value = getProperty(key); + try { + return Long.valueOf(value); + } catch (Exception e) { + e.printStackTrace(); + } + + return 0L; + } + + + /** + * get long value with property key + * + * @param key + * @param defaultValue + * @return + */ + public static Long getLong(String key, Long defaultValue) { + String value = getProperty(key); + try { + return (value == null) ? defaultValue : Long.valueOf(value); + } catch (Exception e) { + e.printStackTrace(); + } + + return 0L; + } + + /** + * get double value with property key + * + * @param key + * @return + */ + public static Double getDouble(String key) { + String value = getProperty(key); + try { + return Double.valueOf(value); + } catch (Exception e) { + e.printStackTrace(); + } + + return 0.0D; + } + + + /** + * get double value with property key + * + * @param key + * @param defaultValue + * @return + */ + public static Double getDouble(String key, Double defaultValue) { + String value = getProperty(key); + try { + return (value == null) ? defaultValue : Double.valueOf(value); + } catch (Exception e) { + e.printStackTrace(); + } + + return 0.0D; + } + +} diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/util/InitUtil.java b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/util/InitUtil.java new file mode 100644 index 0000000..46a8f16 --- /dev/null +++ b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/util/InitUtil.java @@ -0,0 +1,24 @@ +package com.k2data.governance.manager.util; + +import com.k2data.governance.manager.common.constants.SparkConstants; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.JavaSparkContext; + +/** + * Created by wangyihan on 2017/12/6 下午3:42. + * E-mail address is yihanwang22@163.com. + * Copyright © 2017 wangyihan. All Rights Reserved. + * + * @author wangyihan + */ +public class InitUtil { + public static JavaSparkContext initSparkContext() { + SparkConf conf = new SparkConf(); + boolean local = ConfigureUtil.getBoolean(SparkConstants.SPARK_LOCAL); + conf.setAppName(ConfigureUtil.getString(SparkConstants.SPARK_APP_NAME)); + if (local) { + conf.setMaster(ConfigureUtil.getString(SparkConstants.SPARK_MASTER)); + } + return new JavaSparkContext(conf); + } +} diff --git a/k2de-governance-exception-manager/src/main/resources/config.properties b/k2de-governance-exception-manager/src/main/resources/config.properties new file mode 100644 index 0000000..ae1d63a --- /dev/null +++ b/k2de-governance-exception-manager/src/main/resources/config.properties @@ -0,0 +1,3 @@ +spark.local=true +spark.master=local[4] +spark.app.name=GWExceptionManager \ No newline at end of file diff --git a/k2de-governance-exception-manager/src/main/resources/log4j.properties b/k2de-governance-exception-manager/src/main/resources/log4j.properties index a1426d2..68429ad 100644 --- a/k2de-governance-exception-manager/src/main/resources/log4j.properties +++ b/k2de-governance-exception-manager/src/main/resources/log4j.properties @@ -1,6 +1,6 @@ # You should copy this file to resources, and rename it as log4j.properties -log4j.rootLogger=INFO,stdout +log4j.rootLogger=ERROR,stdout log4j.appender.stdout=org.apache.log4j.ConsoleAppender log4j.appender.stdout.layout=org.apache.log4j.PatternLayout log4j.appender.stdout.layout.ConversionPattern=%d [%t] %-5p %c - %m%n \ No newline at end of file diff --git a/pom.xml b/pom.xml index 680fb15..cba331a 100644 --- a/pom.xml +++ b/pom.xml @@ -13,6 +13,8 @@ UTF-8 1.8 2.6.0 + 1.6.3 + 2.11 @@ -45,6 +47,13 @@ sqlite-jdbc 3.20.1 + + + org.apache.spark + spark-core_${spark.comp.version} + ${spark.version} + provided + -- Gitee From e18219d4c757f1af8f7f857b1c3b02299f555460 Mon Sep 17 00:00:00 2001 From: WangYihan Date: Thu, 7 Dec 2017 20:10:00 +0800 Subject: [PATCH 2/9] ExceptionManager: Spark version --- k2de-governance-exception-manager/pom.xml | 2 +- .../manager/ExceptionManagerSpark.java | 35 ++++++--- .../manager/analyzer/GWAnalyzeResult.java | 68 ++++++++-------- .../manager/analyzer/GWWindTurbineKey.java | 6 +- .../manager/common/constants/Constants.java | 3 - .../governance/manager/parser/ParserMR.java | 77 ------------------- 6 files changed, 67 insertions(+), 124 deletions(-) delete mode 100644 k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/parser/ParserMR.java diff --git a/k2de-governance-exception-manager/pom.xml b/k2de-governance-exception-manager/pom.xml index 3035a24..35eeef8 100644 --- a/k2de-governance-exception-manager/pom.xml +++ b/k2de-governance-exception-manager/pom.xml @@ -93,7 +93,7 @@ - com.k2data.governance.manager.ExceptionManagerSingle + com.k2data.governance.manager.ExceptionManagerSpark diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/ExceptionManagerSpark.java b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/ExceptionManagerSpark.java index c0feaba..e45b02f 100644 --- a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/ExceptionManagerSpark.java +++ b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/ExceptionManagerSpark.java @@ -1,6 +1,7 @@ package com.k2data.governance.manager; import com.k2data.governance.manager.analyzer.*; +import com.k2data.governance.manager.common.type.ErrorType; import com.k2data.governance.manager.common.type.GWSourceType; import com.k2data.governance.manager.filter.GWFilter; import com.k2data.governance.manager.util.InitUtil; @@ -13,6 +14,7 @@ import org.apache.spark.api.java.function.Function; import org.apache.spark.broadcast.Broadcast; import scala.Tuple2; +import java.nio.file.Paths; import java.util.HashSet; import java.util.Set; @@ -25,6 +27,9 @@ import java.util.Set; */ public class ExceptionManagerSpark { private Logger logger = Logger.getLogger(ExceptionManagerSpark.class); + private final String typeDetailOutputPath = "TypeDetailResult"; + private final String typeCountOutputPath = "TypeCountResult"; + private final String protocolCountOutputPath = "ProtocolResult"; public Logger getLogger() { return logger; @@ -63,19 +68,27 @@ public class ExceptionManagerSpark { } return result; } - }); - - exceptionRdd = exceptionRdd.cache(); - System.out.println(exceptionRdd.count()); + }).cache(); // exception type count - JavaPairRDD typeRdd = exceptionRdd + JavaPairRDD typeDetailRdd = exceptionRdd .mapToPair(result -> new Tuple2<>(result, 1)) - .reduceByKey((value1, value2) -> value1 + value2); - System.out.println(typeRdd.count()); + .reduceByKey((value1, value2) -> value1 + value2) + .cache(); + String typeDetailOutputPath = Paths.get(outputFilePath, app.typeDetailOutputPath).toString(); + typeDetailRdd.saveAsTextFile(typeDetailOutputPath); + + // sum type count + JavaPairRDD typeTotalRdd = typeDetailRdd + .mapToPair(result -> new Tuple2<>(result._1().getErrorType(), result._2())) + .reduceByKey((count1, count2) -> count1 + count2); + String typeOutputPath = Paths.get(outputFilePath, app.typeCountOutputPath).toString(); + typeTotalRdd.saveAsTextFile(typeOutputPath); + + typeDetailRdd.unpersist(); // protocol count - JavaPairRDD> protocolRdd = exceptionRdd + JavaPairRDD protocolRdd = exceptionRdd .groupBy(GWAnalyzeResult::getProtocolId) .mapToPair(pair -> { String protocolId = pair._1(); @@ -86,9 +99,11 @@ public class ExceptionManagerSpark { recordCount += 1; wtSet.add(result.getWtid()); } - return new Tuple2<>(protocolId, new Tuple2<>(wtSet.size(), recordCount)); + return new Tuple2<>(protocolId, String.format("%s,%s", wtSet.size(), recordCount)); }); + String protocolOutputPath = Paths.get(outputFilePath, app.protocolCountOutputPath).toString(); + protocolRdd.saveAsTextFile(protocolOutputPath); - System.out.println(protocolRdd.count()); + exceptionRdd.unpersist(); } } diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/GWAnalyzeResult.java b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/GWAnalyzeResult.java index 52efb28..c0d1b3f 100644 --- a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/GWAnalyzeResult.java +++ b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/GWAnalyzeResult.java @@ -24,38 +24,6 @@ public class GWAnalyzeResult implements Serializable { this.errorType = errorType; } - @Override - public String toString() { - return String.format("ExceptionType: %s, %s", errorType, wt); - } - - @Override - public int hashCode() { - return this.toString().hashCode(); - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null) { - return false; - } - if (!(obj instanceof GWAnalyzeResult)) { - return false; - } - GWAnalyzeResult resultObj = (GWAnalyzeResult)obj; - if (this.wt == null && resultObj.getWindTurbineKey() != null) { - return false; - } else if (this.wt != null && !this.wt.equals(resultObj.getWindTurbineKey())) { - return false; - } else if (this.errorType != resultObj.getErrorType()) { - return false; - } - return true; - } - public void setErrorType(ErrorType errorType) { this.errorType = errorType; } @@ -97,4 +65,40 @@ public class GWAnalyzeResult implements Serializable { public String getOriginalData() { return originalData; } + + public String toReadableString() { + return String.format("ExceptionType: %s, %s", errorType, wt.toReadableString()); + } + + @Override + public String toString() { + return String.format("%s,%s", errorType, wt); + } + + @Override + public int hashCode() { + return this.toString().hashCode(); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (!(obj instanceof GWAnalyzeResult)) { + return false; + } + GWAnalyzeResult resultObj = (GWAnalyzeResult)obj; + if (this.wt == null && resultObj.getWindTurbineKey() != null) { + return false; + } else if (this.wt != null && !this.wt.equals(resultObj.getWindTurbineKey())) { + return false; + } else if (this.errorType != resultObj.getErrorType()) { + return false; + } + return true; + } } diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/GWWindTurbineKey.java b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/GWWindTurbineKey.java index 432f737..079f89d 100644 --- a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/GWWindTurbineKey.java +++ b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/GWWindTurbineKey.java @@ -34,9 +34,13 @@ public class GWWindTurbineKey implements Serializable { this.protocolId = protocolId; } + public String toReadableString() { + return String.format("ProtocolID: %s, WTID: %s", protocolId, wtid); + } + @Override public String toString() { - return String.format("WTID: %s, ProtocolID: %s", wtid, protocolId); + return String.format("%s,%s", protocolId, wtid); } @Override diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/common/constants/Constants.java b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/common/constants/Constants.java index 9d90ab7..74ed77e 100644 --- a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/common/constants/Constants.java +++ b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/common/constants/Constants.java @@ -10,7 +10,6 @@ package com.k2data.governance.manager.common.constants; public class Constants { public static String GW_MSG_SPILT_OP = ","; - public static String GW_EXCEPTION_END = ".complete"; public static String TIME_FORMAT = "yyyy-MM-dd HH:mm:ss.SSS"; @@ -18,8 +17,6 @@ public class Constants { public static String GW_KMX_START_TAG = "[EXCEPTION]"; public static String GW_HN_DATA_START_TAG = "realtimedata|"; -// public static String PROTOCOL_MAP_DIR = "/Users/stoke/code/github/GWDataEngineering/" -// + "ExceptionManager/src/main/resources/configdata.file"; public static String PROTOCOL_MAP_DIR = "src/main/resources/configdata.file"; public static String GW_META_SQLITE_FILE_DEFAULT = "metadata.db"; } diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/parser/ParserMR.java b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/parser/ParserMR.java deleted file mode 100644 index 41386d5..0000000 --- a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/parser/ParserMR.java +++ /dev/null @@ -1,77 +0,0 @@ -package com.k2data.governance.manager.parser; - -import com.k2data.governance.manager.util.FileHandler; -import java.io.IOException; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.SequenceFile.Reader; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.io.Writable; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.mapreduce.Mapper; -import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; -import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; -import org.apache.hadoop.util.ReflectionUtils; - -/** - * Created by stoke on 2017/11/15. - * E-mail address is zaqthss2009@gmail.com - * Copyright © stoke. All Rights Reserved. - * - * TODO: to refactor - * - * @author stoke - */ -public class ParserMR { - - private static Reader reader = null; - private static Configuration conf = new Configuration(); - - // ref: http://blog.csdn.net/inte_sleeper/article/details/7010966 - public static void main(String[] args) - throws IOException, ClassNotFoundException, InterruptedException { - String inputPath = "/Users/stoke/lab/NEL/k2data/thu/abnormal/hdfs-bolt-anomaly-9-2-1509685150773.log.complete"; - // String outputPath = "result"; - // String inputPath = args[0]; - String outputPath = args[1]; - - // remove the output path - FileHandler.removeOutputDir(outputPath); - - System.out.println(inputPath); - System.out.println(outputPath); - - Job job = new Job(conf, "read sequence file"); - job.setJarByClass(ParserMR.class); - job.setMapperClass(ReadFileMapper.class); - // This should be matched with the file - job.setMapOutputKeyClass(Text.class); - job.setMapOutputValueClass(Text.class); - - Path path = new Path(inputPath); - FileSystem fs = FileSystem.get(conf); - reader = new Reader(fs, path, conf); - FileInputFormat.addInputPath(job, path); - FileOutputFormat.setOutputPath(job, new Path(outputPath)); - System.exit(job.waitForCompletion(true) ? 0 : 1); - } - - public static class ReadFileMapper extends Mapper { - - @Override - public void map(Writable key, Text value, Context context) { - key = (Writable) ReflectionUtils.newInstance(reader.getKeyClass(), conf); - value = (Text) ReflectionUtils.newInstance(reader.getValueClass(), conf); - - try { - while (reader.next(key, value)) { - context.write(key, value); - } - } catch (IOException | InterruptedException e) { - e.printStackTrace(); - } - } - } -} \ No newline at end of file -- Gitee From 3a375c26c64a6888d3666679beaab443573a9825 Mon Sep 17 00:00:00 2001 From: WangYihan Date: Thu, 7 Dec 2017 20:11:29 +0800 Subject: [PATCH 3/9] ExceptionManager: readme --- k2de-governance-exception-manager/README.md | 25 +++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 k2de-governance-exception-manager/README.md diff --git a/k2de-governance-exception-manager/README.md b/k2de-governance-exception-manager/README.md new file mode 100644 index 0000000..f885311 --- /dev/null +++ b/k2de-governance-exception-manager/README.md @@ -0,0 +1,25 @@ +# 金风 KMX 异常统计工具 + +## 使用说明 +* spark 版本主类为 `com.k2data.governance.manager.ExceptionManagerSpark` +* 运行参数： + 1. 文件输入路径 `inputPath` + 2. 结果输出路径 `outputPath` + 3. 协议点表 SQLite 文件路径 + +## 功能概述 +* 输入：异常区异常文件（SequenceFile） +* 输出 + 1. 异常分类储存（暂未实现，输出格式待明确） + 2. 异常类型统计（异常类型，异常计数），输出路径为 `outputPath/TypeCountResult/` + 3. 异常类型细节统计（异常类型，协议号，风机号，异常计数），输出路径为 `outputPath/TypeDetailResult/` + 4. 异常协议统计（协议号，风机计数，异常计数），输出路径为 `outputPath/ProtocolResult/` + +## TODO +* 测试大数据量的效率表现 +* 明确输出格式 + * 分类储存文件格式（SequenceFile?Text?） + * 结果文件是否用标准 csv（现在前后有括号，即直接使用 Tuple2 输出） + * 是否需要合并结果文件 + + -- Gitee From 9e75619894083520d61a7fe074ff18821ba53aad Mon Sep 17 00:00:00 2001 From: WangYihan Date: Wed, 13 Dec 2017 20:31:14 +0800 Subject: [PATCH 4/9] store original data by exception types --- k2de-governance-exception-manager/pom.xml | 5 ++ .../manager/ExceptionManagerSpark.java | 73 +++++++++++++++---- .../manager/analyzer/GWAnalyzeResult.java | 9 --- .../manager/analyzer/GWkmxAnalyzer.java | 3 +- .../manager/common/constants/Constants.java | 2 +- .../metadata/GWSqliteMetaDatabase.java | 10 ++- .../governance/manager/util/InitUtil.java | 20 +++-- .../src/main/resources/config.properties | 2 +- pom.xml | 9 ++- 9 files changed, 99 insertions(+), 34 deletions(-) diff --git a/k2de-governance-exception-manager/pom.xml b/k2de-governance-exception-manager/pom.xml index 35eeef8..478f046 100644 --- a/k2de-governance-exception-manager/pom.xml +++ b/k2de-governance-exception-manager/pom.xml @@ -65,6 +65,11 @@ spark-core_${spark.comp.version} + + org.apache.spark + spark-sql_${spark.comp.version} + + com.sun tools diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/ExceptionManagerSpark.java b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/ExceptionManagerSpark.java index e45b02f..cd39526 100644 --- a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/ExceptionManagerSpark.java +++ b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/ExceptionManagerSpark.java @@ -7,11 +7,18 @@ import com.k2data.governance.manager.filter.GWFilter; import com.k2data.governance.manager.util.InitUtil; import org.apache.hadoop.io.Text; import org.apache.log4j.Logger; +import org.apache.spark.HashPartitioner; import org.apache.spark.api.java.JavaPairRDD; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.api.java.function.Function; +import org.apache.spark.api.java.function.PairFlatMapFunction; +import org.apache.spark.api.java.function.PairFunction; import org.apache.spark.broadcast.Broadcast; +import org.apache.spark.sql.*; +import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.sql.types.StructField; +import org.apache.spark.sql.types.StructType; import scala.Tuple2; import java.nio.file.Paths; @@ -27,6 +34,7 @@ import java.util.Set; */ public class ExceptionManagerSpark { private Logger logger = Logger.getLogger(ExceptionManagerSpark.class); + private final String originDataOutputPath = "OriginalData"; private final String typeDetailOutputPath = "TypeDetailResult"; private final String typeCountOutputPath = "TypeCountResult"; private final String protocolCountOutputPath = "ProtocolResult"; @@ -38,27 +46,48 @@ public class ExceptionManagerSpark { public static void main(String[] args) { ExceptionManagerSpark app = new ExceptionManagerSpark(); Logger logger = app.getLogger(); - if (args.length != 3) { + if (args.length < 3) { logger.error("Invalid arguments"); System.exit(-1); } String inputFilePath = args[0]; String outputFilePath = args[1]; String metadataFilePath = args[2]; - JavaSparkContext sc = InitUtil.initSparkContext(); - JavaPairRDD dataRdd = sc.sequenceFile(inputFilePath, Text.class, Text.class); + int sourceFileType; + try { + sourceFileType = Integer.parseInt(args[3]); // 0 for TextFile, 1 for SequenceFile + } catch (IndexOutOfBoundsException e) { + sourceFileType = 1; // SequenceFile by default + } + SparkSession session; + if (args.length >= 5) { + session = InitUtil.initSparkContext(args[5]); + } else { + session = InitUtil.initSparkContext(); + } + JavaSparkContext sc = new JavaSparkContext(session.sparkContext()); + JavaRDD dataRdd = null; + if (sourceFileType == 1) { // convert SequenceFile to JavaRDD + dataRdd = sc.sequenceFile(inputFilePath, Text.class, Text.class) + .map(textTextTuple2 -> textTextTuple2._2().toString()); + } else if (sourceFileType == 0) { + dataRdd = sc.textFile(inputFilePath); + } else { + logger.error("Invalid file type argument!"); + System.exit(-1); + } IAnalyzer hnAnalyzer = new GWHnAnalyzer(metadataFilePath); IAnalyzer kmxAnalyzer = new GWKmxAnalyzer(); Tuple2 analyzers = new Tuple2<>(hnAnalyzer, kmxAnalyzer); final Broadcast> broadcastAnalyzer =sc.broadcast(analyzers); // analyze exceptions - JavaRDD exceptionRdd = dataRdd.map(new Function, GWAnalyzeResult>() { + assert dataRdd != null; + JavaPairRDD exceptionRdd = dataRdd.mapToPair(new PairFunction() { IAnalyzer hnAnalyzer = broadcastAnalyzer.value()._1(); IAnalyzer kmxAnalyzer = broadcastAnalyzer.value()._2(); @Override - public GWAnalyzeResult call(Tuple2 pair) throws Exception { - String message = pair._2().toString(); + public Tuple2 call(String message) throws Exception { GWSourceType sourceType = GWFilter.getSourceType(message); GWAnalyzeResult result = null; if (sourceType.equals(GWSourceType.HN)) { @@ -66,21 +95,38 @@ public class ExceptionManagerSpark { } else if (sourceType.equals(GWSourceType.KMX)) { result = kmxAnalyzer.analyzeRecord(message); } - return result; + return new Tuple2<>(result, message); } }).cache(); + // Save by type + String originOutputPath = Paths.get(outputFilePath, app.originDataOutputPath).toString(); + JavaRDD rowRdd = exceptionRdd.map(new Function, Row>() { + @Override + public Row call(Tuple2 pair) throws Exception { + return RowFactory.create(pair._1().getErrorType().toString(), pair._2()); + } + }); + StructType schema = DataTypes.createStructType(new StructField[] { + DataTypes.createStructField("ErrorType", DataTypes.StringType, false), + DataTypes.createStructField("Data", DataTypes.StringType, false) + }); + Dataset df = session.createDataFrame(rowRdd, schema); + df.write().partitionBy("ErrorType").text(originOutputPath); + + JavaRDD exceptionRddWithoutData = exceptionRdd.map(Tuple2::_1).cache(); + exceptionRdd.unpersist(); + // exception type count - JavaPairRDD typeDetailRdd = exceptionRdd + JavaPairRDD typeDetailRdd = exceptionRddWithoutData .mapToPair(result -> new Tuple2<>(result, 1)) - .reduceByKey((value1, value2) -> value1 + value2) - .cache(); + .reduceByKey((value1, value2) -> value1 + value2); String typeDetailOutputPath = Paths.get(outputFilePath, app.typeDetailOutputPath).toString(); typeDetailRdd.saveAsTextFile(typeDetailOutputPath); // sum type count - JavaPairRDD typeTotalRdd = typeDetailRdd - .mapToPair(result -> new Tuple2<>(result._1().getErrorType(), result._2())) + JavaPairRDD typeTotalRdd = exceptionRddWithoutData + .mapToPair(result -> new Tuple2<>(result.getErrorType(), 1)) .reduceByKey((count1, count2) -> count1 + count2); String typeOutputPath = Paths.get(outputFilePath, app.typeCountOutputPath).toString(); typeTotalRdd.saveAsTextFile(typeOutputPath); @@ -88,7 +134,7 @@ public class ExceptionManagerSpark { typeDetailRdd.unpersist(); // protocol count - JavaPairRDD protocolRdd = exceptionRdd + JavaPairRDD protocolRdd = exceptionRddWithoutData .groupBy(GWAnalyzeResult::getProtocolId) .mapToPair(pair -> { String protocolId = pair._1(); @@ -105,5 +151,6 @@ public class ExceptionManagerSpark { protocolRdd.saveAsTextFile(protocolOutputPath); exceptionRdd.unpersist(); + session.stop(); } } diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/GWAnalyzeResult.java b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/GWAnalyzeResult.java index c0d1b3f..71dc751 100644 --- a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/GWAnalyzeResult.java +++ b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/GWAnalyzeResult.java @@ -13,7 +13,6 @@ import scala.Serializable; public class GWAnalyzeResult implements Serializable { private ErrorType errorType; private GWWindTurbineKey wt; - private String originalData; public GWAnalyzeResult() { this.wt = new GWWindTurbineKey(null, null); @@ -36,10 +35,6 @@ public class GWAnalyzeResult implements Serializable { this.wt.setProtocolId(protocolId); } - public void setOriginalData(String originalData) { - this.originalData = originalData; - } - public GWWindTurbineKey getWindTurbineKey() { return wt; } @@ -62,10 +57,6 @@ public class GWAnalyzeResult implements Serializable { return wt.getProtocolId(); } - public String getOriginalData() { - return originalData; - } - public String toReadableString() { return String.format("ExceptionType: %s, %s", errorType, wt.toReadableString()); } diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/GWkmxAnalyzer.java b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/GWkmxAnalyzer.java index 7b48e6f..97d40ae 100644 --- a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/GWkmxAnalyzer.java +++ b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/GWkmxAnalyzer.java @@ -87,7 +87,8 @@ public class GWKmxAnalyzer implements IAnalyzer, Serializable { getLogger().error(err); } String type = msg.substring(msg.indexOf(TYPE_PREFIX) + TYPE_PREFIX.length(), indexType); - return new GWAnalyzeResult(GWDataType.fromString(type)); + GWAnalyzeResult result = new GWAnalyzeResult(GWDataType.fromString(type)); + return result; } @Override diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/common/constants/Constants.java b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/common/constants/Constants.java index 74ed77e..eb60d3b 100644 --- a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/common/constants/Constants.java +++ b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/common/constants/Constants.java @@ -18,5 +18,5 @@ public class Constants { public static String GW_HN_DATA_START_TAG = "realtimedata|"; public static String PROTOCOL_MAP_DIR = "src/main/resources/configdata.file"; - public static String GW_META_SQLITE_FILE_DEFAULT = "metadata.db"; + public static String GW_META_SQLITE_FILE_DEFAULT = "/tmp/metadata.db"; } diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/metadata/GWSqliteMetaDatabase.java b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/metadata/GWSqliteMetaDatabase.java index 33f5470..de9cf66 100644 --- a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/metadata/GWSqliteMetaDatabase.java +++ b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/metadata/GWSqliteMetaDatabase.java @@ -45,7 +45,7 @@ public class GWSqliteMetaDatabase extends AbstractGWMetaDatabase { return wtidWfidMap.get(wtid); } - private String getSqliteUrl() { + private String getSqliteUrl() throws IOException { if (!dbFilePath.toLowerCase().startsWith("hdfs")) { return SQLITE_BASE_URL + dbFilePath; } @@ -65,15 +65,19 @@ public class GWSqliteMetaDatabase extends AbstractGWMetaDatabase { fs.copyToLocalFile(false, hdfsPath, localPath); } catch (IOException e) { getLogger().error("Initial file system failed"); - e.printStackTrace(); + throw e; } return SQLITE_BASE_URL + Constants.GW_META_SQLITE_FILE_DEFAULT; } private Connection connect() { - String url = getSqliteUrl(); + String url = null; try { + url = getSqliteUrl(); Class.forName(SQLITE_JDBC_DRIVER); + } catch (IOException e) { + e.printStackTrace(); + System.exit(-1); } catch (ClassNotFoundException e) { getLogger().error("Load SQLite driver error: "+e.getMessage()); System.exit(-1); diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/util/InitUtil.java b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/util/InitUtil.java index 46a8f16..6cf23ac 100644 --- a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/util/InitUtil.java +++ b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/util/InitUtil.java @@ -3,6 +3,7 @@ package com.k2data.governance.manager.util; import com.k2data.governance.manager.common.constants.SparkConstants; import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.sql.SparkSession; /** * Created by wangyihan on 2017/12/6 下午3:42. @@ -12,13 +13,22 @@ import org.apache.spark.api.java.JavaSparkContext; * @author wangyihan */ public class InitUtil { - public static JavaSparkContext initSparkContext() { - SparkConf conf = new SparkConf(); + public static SparkSession initSparkContext() { + SparkSession.Builder builder = SparkSession.builder(); boolean local = ConfigureUtil.getBoolean(SparkConstants.SPARK_LOCAL); - conf.setAppName(ConfigureUtil.getString(SparkConstants.SPARK_APP_NAME)); + builder.appName(ConfigureUtil.getString(SparkConstants.SPARK_APP_NAME)); if (local) { - conf.setMaster(ConfigureUtil.getString(SparkConstants.SPARK_MASTER)); + builder.master(ConfigureUtil.getString(SparkConstants.SPARK_MASTER)); } - return new JavaSparkContext(conf); + return builder.getOrCreate(); + } + + public static SparkSession initSparkContext(String local) { + SparkSession.Builder builder = SparkSession.builder(); + builder.appName(ConfigureUtil.getString(SparkConstants.SPARK_APP_NAME)); + if (Boolean.parseBoolean(local)) { + builder.master(ConfigureUtil.getString(SparkConstants.SPARK_MASTER)); + } + return builder.getOrCreate(); } } diff --git a/k2de-governance-exception-manager/src/main/resources/config.properties b/k2de-governance-exception-manager/src/main/resources/config.properties index ae1d63a..966d419 100644 --- a/k2de-governance-exception-manager/src/main/resources/config.properties +++ b/k2de-governance-exception-manager/src/main/resources/config.properties @@ -1,3 +1,3 @@ -spark.local=true +spark.local=false spark.master=local[4] spark.app.name=GWExceptionManager \ No newline at end of file diff --git a/pom.xml b/pom.xml index cba331a..b43b124 100644 --- a/pom.xml +++ b/pom.xml @@ -13,7 +13,7 @@ UTF-8 1.8 2.6.0 - 1.6.3 + 2.1.1 2.11 @@ -54,6 +54,13 @@ ${spark.version} provided + + + org.apache.spark + spark-sql_${spark.comp.version} + ${spark.version} + provided + -- Gitee From 382d251eaafceb259e488709dcf218a8a0b8c471 Mon Sep 17 00:00:00 2001 From: WangYihan Date: Wed, 13 Dec 2017 20:36:00 +0800 Subject: [PATCH 5/9] readme --- k2de-governance-exception-manager/README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/k2de-governance-exception-manager/README.md b/k2de-governance-exception-manager/README.md index f885311..bb91dc4 100644 --- a/k2de-governance-exception-manager/README.md +++ b/k2de-governance-exception-manager/README.md @@ -1,11 +1,16 @@ # 金风 KMX 异常统计工具 +## 环境说明 +需再 JDK1.8 环境下运行 + ## 使用说明 * spark 版本主类为 `com.k2data.governance.manager.ExceptionManagerSpark` * 运行参数： 1. 文件输入路径 `inputPath` 2. 结果输出路径 `outputPath` 3. 协议点表 SQLite 文件路径 + 4. （可选）输入文件格式，1: SequenceFile, 2: 文本文件 + 5. （可选）是否在 local 模式下运行 （true/false） ## 功能概述 * 输入：异常区异常文件（SequenceFile） -- Gitee From d108025dec9ba4c520ca1427f795afd2963a4eda Mon Sep 17 00:00:00 2001 From: WangYihan Date: Mon, 18 Dec 2017 15:29:45 +0800 Subject: [PATCH 6/9] convert JDK version from 1.8 to 1.7 --- k2de-governance-data-extractor/pom.xml | 4 +- k2de-governance-exception-manager/pom.xml | 6 +- .../manager/ExceptionManagerSingle.java | 29 ++++---- .../manager/ExceptionManagerSpark.java | 71 ++++++++++++++----- .../manager/analyzer/GWhnAnalyzer.java | 28 +++++--- .../manager/analyzer/GWkmxAnalyzer.java | 19 +++-- .../governance/manager/filter/GWFilter.java | 2 +- .../metadata/GWSqliteMetaDatabase.java | 2 +- pom.xml | 10 ++- 9 files changed, 115 insertions(+), 56 deletions(-) diff --git a/k2de-governance-data-extractor/pom.xml b/k2de-governance-data-extractor/pom.xml index d8055df..93f56d4 100644 --- a/k2de-governance-data-extractor/pom.xml +++ b/k2de-governance-data-extractor/pom.xml @@ -16,7 +16,7 @@ UTF-8 - 1.8 + 1.7 2.6.0 @@ -44,8 +44,6 @@ com.sun tools - 1.4.2 - system ${java.home}/../lib/tools.jar diff --git a/k2de-governance-exception-manager/pom.xml b/k2de-governance-exception-manager/pom.xml index 478f046..e3ac0e0 100644 --- a/k2de-governance-exception-manager/pom.xml +++ b/k2de-governance-exception-manager/pom.xml @@ -17,7 +17,7 @@ UTF-8 - 1.8 + 1.7 2.6.0 @@ -73,8 +73,6 @@ com.sun tools - 1.4.2 - system ${java.home}/../lib/tools.jar @@ -98,7 +96,7 @@ - com.k2data.governance.manager.ExceptionManagerSpark + com.k2data.governance.manager.ExceptionManagerSingle diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/ExceptionManagerSingle.java b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/ExceptionManagerSingle.java index f6fa8e1..5e35e7e 100644 --- a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/ExceptionManagerSingle.java +++ b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/ExceptionManagerSingle.java @@ -59,28 +59,31 @@ public class ExceptionManagerSingle { private void updateExceptionCount(Map> newCount) { GWWindTurbineKey allKey = new GWWindTurbineKey("all", "all"); - newCount.forEach((errorType, typeCount) -> { + + for (Map.Entry> entry : newCount.entrySet()) { boolean updateIds = false; + ErrorType errorType = entry.getKey(); + Map typeCount = entry.getValue(); if (errorType instanceof HNErrorType) { if (!uniqueIds.containsKey(errorType)) { Map> idsMap = new HashMap<>(); - idsMap.put(wtidKeyName, new HashSet<>()); - idsMap.put(protocolKeyName, new HashSet<>()); + idsMap.put(wtidKeyName, new HashSet()); + idsMap.put(protocolKeyName, new HashSet()); uniqueIds.put(errorType, idsMap); } updateIds = true; } if (!exceptionCount.containsKey(errorType)) { - exceptionCount.put(errorType, new HashMap<>()); + exceptionCount.put(errorType, new HashMap()); } Map originCount = exceptionCount.get(errorType); if (!originCount.containsKey(allKey)) { originCount.put(allKey, 0); } Map> idsMap = uniqueIds.get(errorType); - for (Map.Entry entry : typeCount.entrySet()) { - GWWindTurbineKey key = entry.getKey(); - int count = entry.getValue(); + for (Map.Entry countEntry : typeCount.entrySet()) { + GWWindTurbineKey key = countEntry.getKey(); + int count = countEntry.getValue(); if (!originCount.containsKey(key)) { originCount.put(key, 0); } @@ -98,16 +101,16 @@ public class ExceptionManagerSingle { idsMap.get(protocolKeyName).add(protocolId); } } - }); + } } - private void sumTypeNums(int[] singleTypeNums, int[] finalTypeNums) { - if (singleTypeNums.length != finalTypeNums.length) { + private void sumTypeNum(int[] singleTypeNum, int[] finalTypeNum) { + if (singleTypeNum.length != finalTypeNum.length) { System.out.println("Length does not match"); return; } - for (int i = 0; i < finalTypeNums.length; ++i) { - finalTypeNums[i] += singleTypeNums[i]; + for (int i = 0; i < finalTypeNum.length; ++i) { + finalTypeNum[i] += singleTypeNum[i]; } } @@ -176,7 +179,7 @@ public class ExceptionManagerSingle { } else { activeBuilder = kmxExceptionStringBuilder; } - activeBuilder = appendCountInfo(errorType, entry.getValue(), activeBuilder); + appendCountInfo(errorType, entry.getValue(), activeBuilder); } return new Pair<>(hnExceptionStringBuilder.toString(), kmxExceptionStringBuilder.toString()); } diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/ExceptionManagerSpark.java b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/ExceptionManagerSpark.java index cd39526..ac409d0 100644 --- a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/ExceptionManagerSpark.java +++ b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/ExceptionManagerSpark.java @@ -12,6 +12,7 @@ import org.apache.spark.api.java.JavaPairRDD; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.api.java.function.Function; +import org.apache.spark.api.java.function.Function2; import org.apache.spark.api.java.function.PairFlatMapFunction; import org.apache.spark.api.java.function.PairFunction; import org.apache.spark.broadcast.Broadcast; @@ -69,7 +70,12 @@ public class ExceptionManagerSpark { JavaRDD dataRdd = null; if (sourceFileType == 1) { // convert SequenceFile to JavaRDD dataRdd = sc.sequenceFile(inputFilePath, Text.class, Text.class) - .map(textTextTuple2 -> textTextTuple2._2().toString()); + .map(new Function, String>() { + @Override + public String call(Tuple2 textTextTuple2) throws Exception { + return textTextTuple2._2().toString(); + } + }); } else if (sourceFileType == 0) { dataRdd = sc.textFile(inputFilePath); } else { @@ -114,20 +120,45 @@ public class ExceptionManagerSpark { Dataset df = session.createDataFrame(rowRdd, schema); df.write().partitionBy("ErrorType").text(originOutputPath); - JavaRDD exceptionRddWithoutData = exceptionRdd.map(Tuple2::_1).cache(); + JavaRDD exceptionRddWithoutData = exceptionRdd.map(new Function, GWAnalyzeResult>() { + @Override + public GWAnalyzeResult call(Tuple2 tuple) throws Exception { + return tuple._1(); + } + }).cache(); exceptionRdd.unpersist(); // exception type count JavaPairRDD typeDetailRdd = exceptionRddWithoutData - .mapToPair(result -> new Tuple2<>(result, 1)) - .reduceByKey((value1, value2) -> value1 + value2); + .mapToPair(new PairFunction() { + @Override + public Tuple2 call(GWAnalyzeResult result) throws Exception { + return new Tuple2<>(result, 1); + } + }) + .reduceByKey(new Function2() { + @Override + public Integer call(Integer value1, Integer value2) throws Exception { + return value1 + value2; + } + }); String typeDetailOutputPath = Paths.get(outputFilePath, app.typeDetailOutputPath).toString(); typeDetailRdd.saveAsTextFile(typeDetailOutputPath); // sum type count JavaPairRDD typeTotalRdd = exceptionRddWithoutData - .mapToPair(result -> new Tuple2<>(result.getErrorType(), 1)) - .reduceByKey((count1, count2) -> count1 + count2); + .mapToPair(new PairFunction() { + @Override + public Tuple2 call(GWAnalyzeResult result) throws Exception { + return new Tuple2<>(result.getErrorType(), 1); + } + }) + .reduceByKey(new Function2() { + @Override + public Integer call(Integer count1, Integer count2) throws Exception { + return count1 + count2; + } + }); String typeOutputPath = Paths.get(outputFilePath, app.typeCountOutputPath).toString(); typeTotalRdd.saveAsTextFile(typeOutputPath); @@ -135,17 +166,25 @@ public class ExceptionManagerSpark { // protocol count JavaPairRDD protocolRdd = exceptionRddWithoutData - .groupBy(GWAnalyzeResult::getProtocolId) - .mapToPair(pair -> { - String protocolId = pair._1(); - Iterable results = pair._2(); - int recordCount = 0; - Set wtSet = new HashSet<>(); - for (GWAnalyzeResult result : results) { - recordCount += 1; - wtSet.add(result.getWtid()); + .groupBy(new Function() { + @Override + public String call(GWAnalyzeResult gwAnalyzeResult) throws Exception { + return gwAnalyzeResult.getProtocolId(); + } + }) + .mapToPair(new PairFunction>, String, String>() { + @Override + public Tuple2 call(Tuple2> pair) throws Exception { + String protocolId = pair._1(); + Iterable results = pair._2(); + int recordCount = 0; + Set wtSet = new HashSet<>(); + for (GWAnalyzeResult result : results) { + recordCount += 1; + wtSet.add(result.getWtid()); + } + return new Tuple2<>(protocolId, String.format("%s,%s", wtSet.size(), recordCount)); } - return new Tuple2<>(protocolId, String.format("%s,%s", wtSet.size(), recordCount)); }); String protocolOutputPath = Paths.get(outputFilePath, app.protocolCountOutputPath).toString(); protocolRdd.saveAsTextFile(protocolOutputPath); diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/GWhnAnalyzer.java b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/GWhnAnalyzer.java index 529944e..b46e5d9 100644 --- a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/GWhnAnalyzer.java +++ b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/GWhnAnalyzer.java @@ -42,7 +42,7 @@ public class GWHnAnalyzer implements IAnalyzer, Serializable { errorTypeResultMap = new HashMap<>(); for (HNErrorType type : HNErrorType.values()) { - errorTypeResultMap.put(type, new ArrayList<>()); + errorTypeResultMap.put(type, new ArrayList()); } } @@ -114,7 +114,9 @@ public class GWHnAnalyzer implements IAnalyzer, Serializable { } // if not in all errors above, then put it in OTHER - result.setErrorType(HNErrorType.OTHER); + if (result.getErrorType() != HNErrorType.TIME) { + result.setErrorType(HNErrorType.OTHER); + } return result; } @@ -134,20 +136,23 @@ public class GWHnAnalyzer implements IAnalyzer, Serializable { @Override public int[] printType() { - int[] typeNums = new int[HNErrorType.values().length]; + int[] typeNum = new int[HNErrorType.values().length]; - hnTypeMessageMap.forEach((type, certainList) -> { + for (Map.Entry> entry : hnTypeMessageMap.entrySet()) { + ErrorType type = entry.getKey(); + List certainList = entry.getValue(); int size = certainList.size(); - typeNums[type.getValue()] = size; + typeNum[type.getValue()] = size; System.out.println( String.format("Exception type %s has %d messages", type.toString(), size)); - }); + } + int sumNum = 0; - for (int num : typeNums) { + for (int num : typeNum) { sumNum += num; } System.out.println("There are " + sumNum + " hit exception messages in total"); - return typeNums; + return typeNum; } /** @@ -168,7 +173,10 @@ public class GWHnAnalyzer implements IAnalyzer, Serializable { public Map> countExceptions() { Map> countResult = new HashMap<>(); - errorTypeResultMap.forEach((errorType, resultList) -> { + + for (Map.Entry> entry : errorTypeResultMap.entrySet()) { + ErrorType errorType = entry.getKey(); + List resultList = entry.getValue(); Map typeResultCount = new HashMap<>(); for (GWAnalyzeResult result : resultList) { GWWindTurbineKey key = result.getWindTurbineKey(); @@ -178,7 +186,7 @@ public class GWHnAnalyzer implements IAnalyzer, Serializable { typeResultCount.put(key, typeResultCount.get(key) + 1); } countResult.put(errorType, typeResultCount); - }); + } return countResult; } } diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/GWkmxAnalyzer.java b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/GWkmxAnalyzer.java index 97d40ae..33afc7c 100644 --- a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/GWkmxAnalyzer.java +++ b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/GWkmxAnalyzer.java @@ -3,6 +3,7 @@ package com.k2data.governance.manager.analyzer; import com.k2data.governance.manager.common.type.ErrorType; import com.k2data.governance.manager.common.type.GWDataType; import com.k2data.governance.manager.message.IMessage; + import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -16,7 +17,7 @@ import scala.Serializable; * Created by stoke on 2017/11/16. * E-mail address is zaqthss2009@gmail.com * Copyright © stoke. All Rights Reserved. - * + *

* Analyze KMX type Exception * * @author stoke @@ -65,7 +66,7 @@ public class GWKmxAnalyzer implements IAnalyzer, Serializable { ErrorType errorType = analyzeResult.getErrorType(); if (!kmxTypeMessageMap.containsKey(errorType)) { // kmxTypeMessageMap.put(errorType, new ArrayList<>()); - errorTypeResultMap.put(errorType, new ArrayList<>()); + errorTypeResultMap.put(errorType, new ArrayList()); } // kmxTypeMessageMap.get(errorType).add(message); errorTypeResultMap.get(errorType).add(analyzeResult); @@ -109,13 +110,14 @@ public class GWKmxAnalyzer implements IAnalyzer, Serializable { public int[] printType() { int[] typeNums = new int[GWDataType.values().length]; - kmxTypeMessageMap.forEach((type, certainList) -> { + for (Map.Entry> entry : kmxTypeMessageMap.entrySet()) { + ErrorType type = entry.getKey(); + List certainList = entry.getValue(); int size = certainList.size(); typeNums[type.getValue()] = size; System.out.println( String.format("Exception type %s has %d messages", type.toString(), certainList.size())); - }); - + } return typeNums; } @@ -133,7 +135,10 @@ public class GWKmxAnalyzer implements IAnalyzer, Serializable { public Map> countExceptions() { Map> countResult = new HashMap<>(); - errorTypeResultMap.forEach((errorType, resultList) -> { + + for (Map.Entry> entry : errorTypeResultMap.entrySet()) { + ErrorType errorType = entry.getKey(); + List resultList = entry.getValue(); Map typeResultCount = new HashMap<>(); for (GWAnalyzeResult result : resultList) { GWWindTurbineKey key = result.getWindTurbineKey(); @@ -143,7 +148,7 @@ public class GWKmxAnalyzer implements IAnalyzer, Serializable { typeResultCount.put(key, typeResultCount.get(key) + 1); } countResult.put(errorType, typeResultCount); - }); + } return countResult; } diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/filter/GWFilter.java b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/filter/GWFilter.java index 9bc6717..7fb4218 100644 --- a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/filter/GWFilter.java +++ b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/filter/GWFilter.java @@ -29,7 +29,7 @@ public class GWFilter implements IFilter { gwSourceTypeMessageMap = new HashMap<>(); for (GWSourceType type : GWSourceType.values()) { - gwSourceTypeMessageMap.put(type, new ArrayList<>()); + gwSourceTypeMessageMap.put(type, new ArrayList()); } } diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/metadata/GWSqliteMetaDatabase.java b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/metadata/GWSqliteMetaDatabase.java index de9cf66..aac37dd 100644 --- a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/metadata/GWSqliteMetaDatabase.java +++ b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/metadata/GWSqliteMetaDatabase.java @@ -147,7 +147,7 @@ public class GWSqliteMetaDatabase extends AbstractGWMetaDatabase { String fieldGroup = rs.getString(1).replace(" ","").toLowerCase(); String fieldName = rs.getString(2).replace(" ", "").toLowerCase(); if (!fieldGroupFieldsMap.containsKey(fieldGroup)) { - fieldGroupFieldsMap.put(fieldGroup, new ArrayList<>()); + fieldGroupFieldsMap.put(fieldGroup, new ArrayList()); } fieldGroupFieldsMap.get(fieldGroup).add(fieldName); } diff --git a/pom.xml b/pom.xml index b43b124..119a43d 100644 --- a/pom.xml +++ b/pom.xml @@ -11,10 +11,11 @@ UTF-8 - 1.8 + 1.7 2.6.0 2.1.1 2.11 + 1.4.2 @@ -61,6 +62,13 @@ ${spark.version} provided + + + com.sun + tools + 1.4.2 + system + -- Gitee From 8501ecfdcb256dc48e4f9c30cee06aaeb5bc26c3 Mon Sep 17 00:00:00 2001 From: WangYihan Date: Mon, 18 Dec 2017 15:31:26 +0800 Subject: [PATCH 7/9] spark version 2.1.1 -> 2.1.0 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 119a43d..00d0835 100644 --- a/pom.xml +++ b/pom.xml @@ -13,7 +13,7 @@ UTF-8 1.7 2.6.0 - 2.1.1 + 2.1.0 2.11 1.4.2 -- Gitee From 6d8a101dbb223eac4925b8cf7fc3c91d46cc6119 Mon Sep 17 00:00:00 2001 From: WangYihan Date: Mon, 18 Dec 2017 15:33:57 +0800 Subject: [PATCH 8/9] README - exception manager --- k2de-governance-exception-manager/README.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/k2de-governance-exception-manager/README.md b/k2de-governance-exception-manager/README.md index bb91dc4..fb20944 100644 --- a/k2de-governance-exception-manager/README.md +++ b/k2de-governance-exception-manager/README.md @@ -1,7 +1,7 @@ # 金风 KMX 异常统计工具 ## 环境说明 -需再 JDK1.8 环境下运行 +需再 JDK1.7 及以上环境下运行 ## 使用说明 * spark 版本主类为 `com.k2data.governance.manager.ExceptionManagerSpark` @@ -22,9 +22,7 @@ ## TODO * 测试大数据量的效率表现 -* 明确输出格式 - * 分类储存文件格式（SequenceFile?Text?） - * 结果文件是否用标准 csv（现在前后有括号，即直接使用 Tuple2 输出） - * 是否需要合并结果文件 +* 数据倾斜优化 +* 单元测试 -- Gitee From f3b3f635d9ce4637b0a8fe3c759ddb79a4ac4451 Mon Sep 17 00:00:00 2001 From: WangYihan Date: Fri, 29 Dec 2017 19:25:48 +0800 Subject: [PATCH 9/9] Refactor Exception Manager --- .gitignore | 2 + k2de-governance-data-extractor/pom.xml | 4 +- .../metadata/GWSqliteMetaDatabase.java | 2 +- .../README.md | 18 +- .../pom.xml | 23 +- .../manager/ExceptionManagerSpark.java | 202 ++++++++----- .../governance/manager/common/Pair.java | 0 .../common/constants/ArgumentsConstants.java | 18 ++ .../manager/common/constants/Constants.java | 10 - .../common/constants/SparkConstants.java | 0 .../governance/manager/error/IErrorType.java | 12 +- .../governance/manager/filter/IFilter.java | 8 +- .../metadata/AbstractMetaDatabase.java | 16 +- .../manager/metadata/IMetaDatabase.java | 3 + .../manager/metadata/SqliteMetaDatabase.java | 69 ++--- .../manager/result/AnalyzeResultImpl.java | 86 +++--- .../governance/manager/result/AssetImpl.java | 53 ++-- .../manager/result/IAnalyzeResult.java | 28 ++ .../governance/manager/result/IAsset.java | 18 ++ .../manager/util/ConfigureUtil.java | 10 - .../governance/manager/util/DateUtil.java | 38 +++ .../governance/manager/util/InitUtil.java | 4 +- .../src/main/resources/config.properties | 0 .../src/main/resources/log4j.properties | 0 .../pom.xml | 71 +++++ .../manager/huineng/HnErrorType.java | 118 ++++++++ .../huineng/HnExceptionManagerMain.java | 34 +++ .../governance/manager/huineng/HnFilter.java | 31 ++ .../src/main/resources/log4j.properties | 6 + k2de-governance-exception-manager-kmx/pom.xml | 67 +++++ .../governance/manager/kmx/KmxErrorType.java | 91 ++++++ .../manager/kmx/KmxExceptionManagerMain.java | 20 ++ .../governance/manager/kmx/KmxFilter.java | 19 ++ .../src/main/resources/log4j.properties | 6 + .../manager/ExceptionManagerSingle.java | 278 ------------------ .../manager/analyzer/GWhnAnalyzer.java | 192 ------------ .../manager/analyzer/GWkmxAnalyzer.java | 158 ---------- .../manager/analyzer/IAnalyzer.java | 24 -- .../manager/common/type/GWDataType.java | 62 ---- .../manager/common/type/GWSourceType.java | 44 --- .../manager/common/type/HNErrorType.java | 39 --- .../governance/manager/filter/GWFilter.java | 71 ----- .../manager/message/AbstractMessage.java | 38 --- .../governance/manager/message/GWMessage.java | 23 -- .../governance/manager/message/IMessage.java | 15 - .../manager/parser/GWParserSingle.java | 57 ---- .../governance/manager/util/FileHandler.java | 99 ------- pom.xml | 11 +- 48 files changed, 858 insertions(+), 1340 deletions(-) rename {k2de-governance-exception-manager => k2de-governance-exception-manager-base}/README.md (53%) rename {k2de-governance-exception-manager => k2de-governance-exception-manager-base}/pom.xml (87%) rename {k2de-governance-exception-manager => k2de-governance-exception-manager-base}/src/main/java/com/k2data/governance/manager/ExceptionManagerSpark.java (41%) rename {k2de-governance-exception-manager => k2de-governance-exception-manager-base}/src/main/java/com/k2data/governance/manager/common/Pair.java (100%) create mode 100644 k2de-governance-exception-manager-base/src/main/java/com/k2data/governance/manager/common/constants/ArgumentsConstants.java rename {k2de-governance-exception-manager => k2de-governance-exception-manager-base}/src/main/java/com/k2data/governance/manager/common/constants/Constants.java (45%) rename {k2de-governance-exception-manager => k2de-governance-exception-manager-base}/src/main/java/com/k2data/governance/manager/common/constants/SparkConstants.java (100%) rename k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/common/type/ErrorType.java => k2de-governance-exception-manager-base/src/main/java/com/k2data/governance/manager/error/IErrorType.java (41%) rename {k2de-governance-exception-manager => k2de-governance-exception-manager-base}/src/main/java/com/k2data/governance/manager/filter/IFilter.java (47%) rename k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/metadata/AbstractGWMetaDatabase.java => k2de-governance-exception-manager-base/src/main/java/com/k2data/governance/manager/metadata/AbstractMetaDatabase.java (68%) rename {k2de-governance-exception-manager => k2de-governance-exception-manager-base}/src/main/java/com/k2data/governance/manager/metadata/IMetaDatabase.java (99%) rename k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/metadata/GWSqliteMetaDatabase.java => k2de-governance-exception-manager-base/src/main/java/com/k2data/governance/manager/metadata/SqliteMetaDatabase.java (73%) rename k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/GWAnalyzeResult.java => k2de-governance-exception-manager-base/src/main/java/com/k2data/governance/manager/result/AnalyzeResultImpl.java (31%) rename k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/GWWindTurbineKey.java => k2de-governance-exception-manager-base/src/main/java/com/k2data/governance/manager/result/AssetImpl.java (35%) create mode 100644 k2de-governance-exception-manager-base/src/main/java/com/k2data/governance/manager/result/IAnalyzeResult.java create mode 100644 k2de-governance-exception-manager-base/src/main/java/com/k2data/governance/manager/result/IAsset.java rename {k2de-governance-exception-manager => k2de-governance-exception-manager-base}/src/main/java/com/k2data/governance/manager/util/ConfigureUtil.java (99%) create mode 100644 k2de-governance-exception-manager-base/src/main/java/com/k2data/governance/manager/util/DateUtil.java rename {k2de-governance-exception-manager => k2de-governance-exception-manager-base}/src/main/java/com/k2data/governance/manager/util/InitUtil.java (91%) rename {k2de-governance-exception-manager => k2de-governance-exception-manager-base}/src/main/resources/config.properties (100%) rename {k2de-governance-exception-manager => k2de-governance-exception-manager-base}/src/main/resources/log4j.properties (100%) create mode 100644 k2de-governance-exception-manager-huineng/pom.xml create mode 100644 k2de-governance-exception-manager-huineng/src/main/java/com/k2data/governance/manager/huineng/HnErrorType.java create mode 100644 k2de-governance-exception-manager-huineng/src/main/java/com/k2data/governance/manager/huineng/HnExceptionManagerMain.java create mode 100644 k2de-governance-exception-manager-huineng/src/main/java/com/k2data/governance/manager/huineng/HnFilter.java create mode 100644 k2de-governance-exception-manager-huineng/src/main/resources/log4j.properties create mode 100644 k2de-governance-exception-manager-kmx/pom.xml create mode 100644 k2de-governance-exception-manager-kmx/src/main/java/com/k2data/governance/manager/kmx/KmxErrorType.java create mode 100644 k2de-governance-exception-manager-kmx/src/main/java/com/k2data/governance/manager/kmx/KmxExceptionManagerMain.java create mode 100644 k2de-governance-exception-manager-kmx/src/main/java/com/k2data/governance/manager/kmx/KmxFilter.java create mode 100644 k2de-governance-exception-manager-kmx/src/main/resources/log4j.properties delete mode 100644 k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/ExceptionManagerSingle.java delete mode 100644 k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/GWhnAnalyzer.java delete mode 100644 k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/GWkmxAnalyzer.java delete mode 100644 k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/IAnalyzer.java delete mode 100644 k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/common/type/GWDataType.java delete mode 100644 k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/common/type/GWSourceType.java delete mode 100644 k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/common/type/HNErrorType.java delete mode 100644 k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/filter/GWFilter.java delete mode 100644 k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/message/AbstractMessage.java delete mode 100644 k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/message/GWMessage.java delete mode 100644 k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/message/IMessage.java delete mode 100644 k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/parser/GWParserSingle.java delete mode 100644 k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/util/FileHandler.java diff --git a/.gitignore b/.gitignore index bf33abf..13042f9 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,5 @@ *.csv *META-INF/ + +**/LocalTest.java diff --git a/k2de-governance-data-extractor/pom.xml b/k2de-governance-data-extractor/pom.xml index 93f56d4..c262e4c 100644 --- a/k2de-governance-data-extractor/pom.xml +++ b/k2de-governance-data-extractor/pom.xml @@ -9,10 +9,10 @@ k2de-governance-data-extractor - 0.0.1-SNAPSHOT + 1.0-SNAPSHOT jar - DataExtractor + k2de-governance-data-extractor UTF-8 diff --git a/k2de-governance-data-extractor/src/main/java/com/k2data/governance/extractor/metadata/GWSqliteMetaDatabase.java b/k2de-governance-data-extractor/src/main/java/com/k2data/governance/extractor/metadata/GWSqliteMetaDatabase.java index 2b880d3..bb7f275 100644 --- a/k2de-governance-data-extractor/src/main/java/com/k2data/governance/extractor/metadata/GWSqliteMetaDatabase.java +++ b/k2de-governance-data-extractor/src/main/java/com/k2data/governance/extractor/metadata/GWSqliteMetaDatabase.java @@ -145,7 +145,7 @@ public class GWSqliteMetaDatabase extends AbstractGWMetaDatabase implements IMet String fieldGroup = rs.getString(1).replace(" ","").toLowerCase(); String fieldName = rs.getString(2).replace(" ", "").toLowerCase(); if (!fieldGroupFieldsMap.containsKey(fieldGroup)) { - fieldGroupFieldsMap.put(fieldGroup, new ArrayList<>()); + fieldGroupFieldsMap.put(fieldGroup, new ArrayList()); } fieldGroupFieldsMap.get(fieldGroup).add(fieldName); } diff --git a/k2de-governance-exception-manager/README.md b/k2de-governance-exception-manager-base/README.md similarity index 53% rename from k2de-governance-exception-manager/README.md rename to k2de-governance-exception-manager-base/README.md index fb20944..c64fee5 100644 --- a/k2de-governance-exception-manager/README.md +++ b/k2de-governance-exception-manager-base/README.md @@ -4,13 +4,16 @@ 需再 JDK1.7 及以上环境下运行 ## 使用说明 -* spark 版本主类为 `com.k2data.governance.manager.ExceptionManagerSpark` +* spark 版本基项目入口函数为 `com.k2data.governance.manager.ExceptionManagerSpark.run` * 运行参数： - 1. 文件输入路径 `inputPath` - 2. 结果输出路径 `outputPath` - 3. 协议点表 SQLite 文件路径 - 4. （可选）输入文件格式，1: SequenceFile, 2: 文本文件 - 5. （可选）是否在 local 模式下运行 （true/false） + usage: ExceptionManager [-h] [-m METADATAFILE] [-t {text,sequence}] [-l] + InputPath OutputPath + * InputPath：文件输入路径 + * OutputPath：结果输出路径 + * -m METADATAFILE：协议点表 SQLite 文件路径 + * -t：输入文件格式，文本文件格式，或 SequenceFile 格式，默认为 seqence + * -l：以 Spark Local 模式运行 + * -h：运行参数说明 ## 功能概述 * 输入：异常区异常文件（SequenceFile） @@ -24,5 +27,8 @@ * 测试大数据量的效率表现 * 数据倾斜优化 * 单元测试 +* 似乎重构之后效率慢了一些，需后续检查测试 +* 执行任务可配置 +* 文档及报告 diff --git a/k2de-governance-exception-manager/pom.xml b/k2de-governance-exception-manager-base/pom.xml similarity index 87% rename from k2de-governance-exception-manager/pom.xml rename to k2de-governance-exception-manager-base/pom.xml index e3ac0e0..92ed68e 100644 --- a/k2de-governance-exception-manager/pom.xml +++ b/k2de-governance-exception-manager-base/pom.xml @@ -8,18 +8,11 @@ 1.0-SNAPSHOT - k2de-governance-exception-manager - 0.0.1-SNAPSHOT + k2de-governance-exception-manager-base + 1.0-SNAPSHOT jar - ExceptionManager - http://maven.apache.org - - - UTF-8 - 1.7 - 2.6.0 - + k2de-governance-exception-manager-base @@ -65,6 +58,11 @@ spark-core_${spark.comp.version} + + net.sourceforge.argparse4j + argparse4j + + org.apache.spark spark-sql_${spark.comp.version} @@ -73,12 +71,9 @@ com.sun tools - ${java.home}/../lib/tools.jar - - @@ -96,7 +91,7 @@ - com.k2data.governance.manager.ExceptionManagerSingle + com.k2data.governance.manager.ExceptionManagerSpark diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/ExceptionManagerSpark.java b/k2de-governance-exception-manager-base/src/main/java/com/k2data/governance/manager/ExceptionManagerSpark.java similarity index 41% rename from k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/ExceptionManagerSpark.java rename to k2de-governance-exception-manager-base/src/main/java/com/k2data/governance/manager/ExceptionManagerSpark.java index ac409d0..34a303d 100644 --- a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/ExceptionManagerSpark.java +++ b/k2de-governance-exception-manager-base/src/main/java/com/k2data/governance/manager/ExceptionManagerSpark.java @@ -1,22 +1,29 @@ package com.k2data.governance.manager; -import com.k2data.governance.manager.analyzer.*; -import com.k2data.governance.manager.common.type.ErrorType; -import com.k2data.governance.manager.common.type.GWSourceType; -import com.k2data.governance.manager.filter.GWFilter; +import com.k2data.governance.manager.result.IAnalyzeResult; +import com.k2data.governance.manager.common.constants.ArgumentsConstants; +import com.k2data.governance.manager.error.IErrorType; +import com.k2data.governance.manager.filter.IFilter; +import com.k2data.governance.manager.metadata.IMetaDatabase; import com.k2data.governance.manager.util.InitUtil; +import net.sourceforge.argparse4j.ArgumentParsers; +import net.sourceforge.argparse4j.impl.Arguments; +import net.sourceforge.argparse4j.inf.ArgumentParser; +import net.sourceforge.argparse4j.inf.ArgumentParserException; +import net.sourceforge.argparse4j.inf.Namespace; import org.apache.hadoop.io.Text; import org.apache.log4j.Logger; -import org.apache.spark.HashPartitioner; import org.apache.spark.api.java.JavaPairRDD; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.api.java.function.Function; import org.apache.spark.api.java.function.Function2; -import org.apache.spark.api.java.function.PairFlatMapFunction; import org.apache.spark.api.java.function.PairFunction; import org.apache.spark.broadcast.Broadcast; -import org.apache.spark.sql.*; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.RowFactory; +import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.types.DataTypes; import org.apache.spark.sql.types.StructField; import org.apache.spark.sql.types.StructType; @@ -39,100 +46,141 @@ public class ExceptionManagerSpark { private final String typeDetailOutputPath = "TypeDetailResult"; private final String typeCountOutputPath = "TypeCountResult"; private final String protocolCountOutputPath = "ProtocolResult"; + private static IFilter errorTypeFilter; + private static IMetaDatabase metadataBase; public Logger getLogger() { return logger; } - public static void main(String[] args) { - ExceptionManagerSpark app = new ExceptionManagerSpark(); - Logger logger = app.getLogger(); - if (args.length < 3) { - logger.error("Invalid arguments"); - System.exit(-1); - } - String inputFilePath = args[0]; - String outputFilePath = args[1]; - String metadataFilePath = args[2]; - int sourceFileType; + public static void setFilter(IFilter filter) { + errorTypeFilter = filter; + } + + public static void setMetadataBase(IMetaDatabase database) { + metadataBase = database; + } + + public static Namespace parseArguments(String[] args) { + ArgumentParser argumentParser = ArgumentParsers.newArgumentParser("ExceptionManager") + .defaultHelp(true) + .description("k2data exception data statistics tool"); + argumentParser.addArgument(ArgumentsConstants.INPUT_PATH) + .type(String.class) + .metavar("InputPath") + .help("Input data path, HDFS or local path"); + argumentParser.addArgument(ArgumentsConstants.OUTPUT_PATH) + .type(String.class) + .metavar("OutputPath") + .help("Output data path, HDFS of local path"); + argumentParser.addArgument("-m", "--metadata-file") + .dest(ArgumentsConstants.METADATA_FILE) + .help("Metadata SQLite file"); + argumentParser.addArgument("-t", "--file-type") + .dest(ArgumentsConstants.FILE_TYPE) + .choices(ArgumentsConstants.FILE_TYPE_TEXT, ArgumentsConstants.FILE_TYPE_SEQUENCE) + .setDefault(ArgumentsConstants.FILE_TYPE_SEQUENCE) + .help("Input file format (text, sequence)"); + argumentParser.addArgument("-l", "--local") + .dest(ArgumentsConstants.LOCAL) + .action(Arguments.storeConst()) + .setConst(true) + .setDefault(false) + .help("Run spark with local mode"); + Namespace ns = null; try { - sourceFileType = Integer.parseInt(args[3]); // 0 for TextFile, 1 for SequenceFile - } catch (IndexOutOfBoundsException e) { - sourceFileType = 1; // SequenceFile by default - } - SparkSession session; - if (args.length >= 5) { - session = InitUtil.initSparkContext(args[5]); - } else { - session = InitUtil.initSparkContext(); + ns = argumentParser.parseArgs(args); + } catch (ArgumentParserException e) { + argumentParser.handleError(e); + System.exit(1); } + return ns; + } + + public static void run(String[] args) { + Namespace ns = parseArguments(args); + run(ns); + } + + public static void run(Namespace ns) { + ExceptionManagerSpark app = new ExceptionManagerSpark(); + Logger logger = app.getLogger(); + + String inputFilePath = ns.getString(ArgumentsConstants.INPUT_PATH); + String outputFilePath = ns.getString(ArgumentsConstants.OUTPUT_PATH); + String sourceFileType = ns.getString(ArgumentsConstants.FILE_TYPE); + boolean local = ns.getBoolean(ArgumentsConstants.LOCAL); + + // Initial SparkSession & SparkContext + SparkSession session = InitUtil.initSparkContext(local); JavaSparkContext sc = new JavaSparkContext(session.sparkContext()); JavaRDD dataRdd = null; - if (sourceFileType == 1) { // convert SequenceFile to JavaRDD - dataRdd = sc.sequenceFile(inputFilePath, Text.class, Text.class) - .map(new Function, String>() { - @Override - public String call(Tuple2 textTextTuple2) throws Exception { - return textTextTuple2._2().toString(); - } - }); - } else if (sourceFileType == 0) { - dataRdd = sc.textFile(inputFilePath); - } else { - logger.error("Invalid file type argument!"); - System.exit(-1); + // Read data + switch (sourceFileType) { + case ArgumentsConstants.FILE_TYPE_SEQUENCE: // convert SequenceFile to JavaRDD + dataRdd = sc.sequenceFile(inputFilePath, Text.class, Text.class) + .map(new Function, String>() { + @Override + public String call(Tuple2 textTextTuple2) throws Exception { + return textTextTuple2._2().toString(); + } + }); + break; + case ArgumentsConstants.FILE_TYPE_TEXT: + dataRdd = sc.textFile(inputFilePath); + break; + default: + logger.error("Invalid file type argument!"); + System.exit(-1); } - IAnalyzer hnAnalyzer = new GWHnAnalyzer(metadataFilePath); - IAnalyzer kmxAnalyzer = new GWKmxAnalyzer(); - Tuple2 analyzers = new Tuple2<>(hnAnalyzer, kmxAnalyzer); - final Broadcast> broadcastAnalyzer =sc.broadcast(analyzers); + final Broadcast broadcastFilter = sc.broadcast(errorTypeFilter); + final Broadcast broadcastMetadataBase = sc.broadcast(metadataBase); // analyze exceptions assert dataRdd != null; - JavaPairRDD exceptionRdd = dataRdd.mapToPair(new PairFunction() { - IAnalyzer hnAnalyzer = broadcastAnalyzer.value()._1(); - IAnalyzer kmxAnalyzer = broadcastAnalyzer.value()._2(); + JavaPairRDD exceptionRdd = dataRdd.mapToPair(new PairFunction() { + IFilter filter = broadcastFilter.value(); + IMetaDatabase metaDatabase = broadcastMetadataBase.value(); + @Override - public Tuple2 call(String message) throws Exception { - GWSourceType sourceType = GWFilter.getSourceType(message); - GWAnalyzeResult result = null; - if (sourceType.equals(GWSourceType.HN)) { - result = hnAnalyzer.analyzeRecord(message); - } else if (sourceType.equals(GWSourceType.KMX)) { - result = kmxAnalyzer.analyzeRecord(message); - } + public Tuple2 call(String message) throws Exception { + Class errorTypeClass = filter.getErrorClass(message); + // TODO: handle null pointer or invalid errorTypeClass + IErrorType errorType = errorTypeClass.getEnumConstants()[0]; + IAnalyzeResult result = errorType.analyzeRecord(message, metaDatabase); return new Tuple2<>(result, message); } }).cache(); // Save by type String originOutputPath = Paths.get(outputFilePath, app.originDataOutputPath).toString(); - JavaRDD rowRdd = exceptionRdd.map(new Function, Row>() { + JavaRDD rowRdd = exceptionRdd.map(new Function, Row>() { @Override - public Row call(Tuple2 pair) throws Exception { + public Row call(Tuple2 pair) throws Exception { return RowFactory.create(pair._1().getErrorType().toString(), pair._2()); } }); - StructType schema = DataTypes.createStructType(new StructField[] { + StructType schema = DataTypes.createStructType(new StructField[]{ DataTypes.createStructField("ErrorType", DataTypes.StringType, false), DataTypes.createStructField("Data", DataTypes.StringType, false) }); Dataset df = session.createDataFrame(rowRdd, schema); df.write().partitionBy("ErrorType").text(originOutputPath); - JavaRDD exceptionRddWithoutData = exceptionRdd.map(new Function, GWAnalyzeResult>() { + JavaRDD exceptionRddWithoutData = exceptionRdd.map(new Function, IAnalyzeResult>() { @Override - public GWAnalyzeResult call(Tuple2 tuple) throws Exception { + public IAnalyzeResult call(Tuple2 tuple) throws Exception { return tuple._1(); } }).cache(); + exceptionRdd.unpersist(); - // exception type count - JavaPairRDD typeDetailRdd = exceptionRddWithoutData - .mapToPair(new PairFunction() { + // Count exception type + JavaPairRDD typeDetailRdd = exceptionRddWithoutData + .mapToPair(new PairFunction() { @Override - public Tuple2 call(GWAnalyzeResult result) throws Exception { + public Tuple2 call(IAnalyzeResult result) throws Exception { return new Tuple2<>(result, 1); } }) @@ -145,11 +193,11 @@ public class ExceptionManagerSpark { String typeDetailOutputPath = Paths.get(outputFilePath, app.typeDetailOutputPath).toString(); typeDetailRdd.saveAsTextFile(typeDetailOutputPath); - // sum type count - JavaPairRDD typeTotalRdd = exceptionRddWithoutData - .mapToPair(new PairFunction() { + // Aggregate count by type + JavaPairRDD typeTotalRdd = exceptionRddWithoutData + .mapToPair(new PairFunction() { @Override - public Tuple2 call(GWAnalyzeResult result) throws Exception { + public Tuple2 call(IAnalyzeResult result) throws Exception { return new Tuple2<>(result.getErrorType(), 1); } }) @@ -164,24 +212,24 @@ public class ExceptionManagerSpark { typeDetailRdd.unpersist(); - // protocol count + // Count by protocol JavaPairRDD protocolRdd = exceptionRddWithoutData - .groupBy(new Function() { + .groupBy(new Function() { @Override - public String call(GWAnalyzeResult gwAnalyzeResult) throws Exception { - return gwAnalyzeResult.getProtocolId(); + public String call(IAnalyzeResult gwAnalyzeResult) throws Exception { + return gwAnalyzeResult.getFieldGroup(); } }) - .mapToPair(new PairFunction>, String, String>() { + .mapToPair(new PairFunction>, String, String>() { @Override - public Tuple2 call(Tuple2> pair) throws Exception { + public Tuple2 call(Tuple2> pair) throws Exception { String protocolId = pair._1(); - Iterable results = pair._2(); + Iterable results = pair._2(); int recordCount = 0; Set wtSet = new HashSet<>(); - for (GWAnalyzeResult result : results) { + for (IAnalyzeResult result : results) { recordCount += 1; - wtSet.add(result.getWtid()); + wtSet.add(result.getAssetId()); } return new Tuple2<>(protocolId, String.format("%s,%s", wtSet.size(), recordCount)); } diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/common/Pair.java b/k2de-governance-exception-manager-base/src/main/java/com/k2data/governance/manager/common/Pair.java similarity index 100% rename from k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/common/Pair.java rename to k2de-governance-exception-manager-base/src/main/java/com/k2data/governance/manager/common/Pair.java diff --git a/k2de-governance-exception-manager-base/src/main/java/com/k2data/governance/manager/common/constants/ArgumentsConstants.java b/k2de-governance-exception-manager-base/src/main/java/com/k2data/governance/manager/common/constants/ArgumentsConstants.java new file mode 100644 index 0000000..91f84ee --- /dev/null +++ b/k2de-governance-exception-manager-base/src/main/java/com/k2data/governance/manager/common/constants/ArgumentsConstants.java @@ -0,0 +1,18 @@ +package com.k2data.governance.manager.common.constants; + +/** + * Created by wangyihan on 2017/12/28 下午10:58. + * E-mail address is yihanwang22@163.com. + * Copyright © 2017 wangyihan. All Rights Reserved. + * + * @author wangyihan + */ +public class ArgumentsConstants { + public final static String INPUT_PATH = "inputPath"; + public final static String OUTPUT_PATH = "outputPath"; + public final static String METADATA_FILE = "metadataFile"; + public final static String FILE_TYPE = "fileType"; + public final static String LOCAL = "local"; + public final static String FILE_TYPE_TEXT = "text"; + public final static String FILE_TYPE_SEQUENCE = "sequence"; +} diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/common/constants/Constants.java b/k2de-governance-exception-manager-base/src/main/java/com/k2data/governance/manager/common/constants/Constants.java similarity index 45% rename from k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/common/constants/Constants.java rename to k2de-governance-exception-manager-base/src/main/java/com/k2data/governance/manager/common/constants/Constants.java index eb60d3b..8cbd88b 100644 --- a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/common/constants/Constants.java +++ b/k2de-governance-exception-manager-base/src/main/java/com/k2data/governance/manager/common/constants/Constants.java @@ -8,15 +8,5 @@ package com.k2data.governance.manager.common.constants; * @author stoke */ public class Constants { - - public static String GW_MSG_SPILT_OP = ","; - - public static String TIME_FORMAT = "yyyy-MM-dd HH:mm:ss.SSS"; - - public static String GW_HN_START_TAG = "sediment"; - public static String GW_KMX_START_TAG = "[EXCEPTION]"; - public static String GW_HN_DATA_START_TAG = "realtimedata|"; - - public static String PROTOCOL_MAP_DIR = "src/main/resources/configdata.file"; public static String GW_META_SQLITE_FILE_DEFAULT = "/tmp/metadata.db"; } diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/common/constants/SparkConstants.java b/k2de-governance-exception-manager-base/src/main/java/com/k2data/governance/manager/common/constants/SparkConstants.java similarity index 100% rename from k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/common/constants/SparkConstants.java rename to k2de-governance-exception-manager-base/src/main/java/com/k2data/governance/manager/common/constants/SparkConstants.java diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/common/type/ErrorType.java b/k2de-governance-exception-manager-base/src/main/java/com/k2data/governance/manager/error/IErrorType.java similarity index 41% rename from k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/common/type/ErrorType.java rename to k2de-governance-exception-manager-base/src/main/java/com/k2data/governance/manager/error/IErrorType.java index ea0ff87..c6a8464 100644 --- a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/common/type/ErrorType.java +++ b/k2de-governance-exception-manager-base/src/main/java/com/k2data/governance/manager/error/IErrorType.java @@ -1,4 +1,7 @@ -package com.k2data.governance.manager.common.type; +package com.k2data.governance.manager.error; + +import com.k2data.governance.manager.result.IAnalyzeResult; +import com.k2data.governance.manager.metadata.IMetaDatabase; /** * Created by wangyihan on 2017/11/30 下午2:08. @@ -7,7 +10,12 @@ package com.k2data.governance.manager.common.type; * * @author wangyihan */ -public interface ErrorType { +public interface IErrorType { String toErrorString(); + int getValue(); + + IAnalyzeResult analyzeRecord(String msg); + + IAnalyzeResult analyzeRecord(String msg, IMetaDatabase metaDatabase); } diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/filter/IFilter.java b/k2de-governance-exception-manager-base/src/main/java/com/k2data/governance/manager/filter/IFilter.java similarity index 47% rename from k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/filter/IFilter.java rename to k2de-governance-exception-manager-base/src/main/java/com/k2data/governance/manager/filter/IFilter.java index 0cd3d2d..e10fd08 100644 --- a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/filter/IFilter.java +++ b/k2de-governance-exception-manager-base/src/main/java/com/k2data/governance/manager/filter/IFilter.java @@ -1,8 +1,6 @@ package com.k2data.governance.manager.filter; -import com.k2data.governance.manager.common.type.GWSourceType; -import com.k2data.governance.manager.message.IMessage; -import java.util.List; +import com.k2data.governance.manager.error.IErrorType; /** * Created by stoke on 2017/11/16. @@ -12,7 +10,5 @@ import java.util.List; * @author stoke */ public interface IFilter { - void filter(); - List getMessageList(); - List getCertainSourceTypeList(GWSourceType type); + Class getErrorClass(String msg); } diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/metadata/AbstractGWMetaDatabase.java b/k2de-governance-exception-manager-base/src/main/java/com/k2data/governance/manager/metadata/AbstractMetaDatabase.java similarity index 68% rename from k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/metadata/AbstractGWMetaDatabase.java rename to k2de-governance-exception-manager-base/src/main/java/com/k2data/governance/manager/metadata/AbstractMetaDatabase.java index 5f9138c..e6c3a8a 100644 --- a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/metadata/AbstractGWMetaDatabase.java +++ b/k2de-governance-exception-manager-base/src/main/java/com/k2data/governance/manager/metadata/AbstractMetaDatabase.java @@ -2,6 +2,7 @@ package com.k2data.governance.manager.metadata; import java.util.List; import java.util.Map; + import org.apache.log4j.Logger; import scala.Serializable; @@ -12,14 +13,13 @@ import scala.Serializable; * * @author wangyihan */ -abstract class AbstractGWMetaDatabase implements IMetaDatabase, Serializable { +abstract class AbstractMetaDatabase implements IMetaDatabase, Serializable { - Map wtidFieldGroupMap; + Map assetFieldGroupMap; Map> fieldGroupFieldsMap; - Map wtidWfidMap; protected Logger getLogger() { - return Logger.getLogger(AbstractGWMetaDatabase.class); + return Logger.getLogger(AbstractMetaDatabase.class); } @Override @@ -36,12 +36,12 @@ abstract class AbstractGWMetaDatabase implements IMetaDatabase, Serializable { } @Override - public String getFieldGroup(String wtid) { - if (wtid == null || wtid.isEmpty()) { - getLogger().error("Empty wtid: " + wtid); + public String getFieldGroup(String assetId) { + if (assetId == null || assetId.isEmpty()) { + getLogger().error("Empty AssetID: " + assetId); return null; } - return wtidFieldGroupMap.get(wtid); + return assetFieldGroupMap.get(assetId); } @Override diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/metadata/IMetaDatabase.java b/k2de-governance-exception-manager-base/src/main/java/com/k2data/governance/manager/metadata/IMetaDatabase.java similarity index 99% rename from k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/metadata/IMetaDatabase.java rename to k2de-governance-exception-manager-base/src/main/java/com/k2data/governance/manager/metadata/IMetaDatabase.java index 67f2983..ec2ac0c 100644 --- a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/metadata/IMetaDatabase.java +++ b/k2de-governance-exception-manager-base/src/main/java/com/k2data/governance/manager/metadata/IMetaDatabase.java @@ -11,7 +11,10 @@ import java.util.List; */ public interface IMetaDatabase { void init(); + List getFields(String assetId); + String getFieldGroup(String assetId); + List getFieldsByFieldGroup(String fieldGroup); } diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/metadata/GWSqliteMetaDatabase.java b/k2de-governance-exception-manager-base/src/main/java/com/k2data/governance/manager/metadata/SqliteMetaDatabase.java similarity index 73% rename from k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/metadata/GWSqliteMetaDatabase.java rename to k2de-governance-exception-manager-base/src/main/java/com/k2data/governance/manager/metadata/SqliteMetaDatabase.java index aac37dd..0db53b9 100644 --- a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/metadata/GWSqliteMetaDatabase.java +++ b/k2de-governance-exception-manager-base/src/main/java/com/k2data/governance/manager/metadata/SqliteMetaDatabase.java @@ -2,6 +2,7 @@ package com.k2data.governance.manager.metadata; import com.k2data.governance.manager.common.constants.Constants; import com.k2data.governance.manager.common.Pair; + import java.io.IOException; import java.sql.Connection; import java.sql.DriverManager; @@ -12,11 +13,11 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.regex.Matcher; import java.util.regex.Pattern; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.log4j.Logger; -import scala.Serializable; /** @@ -26,25 +27,32 @@ import scala.Serializable; * * @author wangyihan */ -public class GWSqliteMetaDatabase extends AbstractGWMetaDatabase { +public class SqliteMetaDatabase extends AbstractMetaDatabase { private static final String SQLITE_JDBC_DRIVER = "org.sqlite.JDBC"; private static final String SQLITE_BASE_URL = "jdbc:sqlite:"; private String dbFilePath; + private String assetFieldGroupSql; + private String fieldGroupFieldsSql; - public GWSqliteMetaDatabase(String dbFilePath) { + public SqliteMetaDatabase(String dbFilePath, String assetFieldGroupSql, String fieldGroupFieldsSql) { this.dbFilePath = dbFilePath; + this.assetFieldGroupSql = assetFieldGroupSql; + this.fieldGroupFieldsSql = fieldGroupFieldsSql; init(); } @Override protected Logger getLogger() { - return Logger.getLogger(GWSqliteMetaDatabase.class); - } - - public String getWfid(String wtid) { - return wtidWfidMap.get(wtid); + return Logger.getLogger(SqliteMetaDatabase.class); } + /** + * Concat jdbc:sqlite with sqlite file path + * If file path is a hdfs address, download the file to local + * + * @return full url of sqlite database + * @throws IOException when initial file system failed + */ private String getSqliteUrl() throws IOException { if (!dbFilePath.toLowerCase().startsWith("hdfs")) { return SQLITE_BASE_URL + dbFilePath; @@ -79,7 +87,7 @@ public class GWSqliteMetaDatabase extends AbstractGWMetaDatabase { e.printStackTrace(); System.exit(-1); } catch (ClassNotFoundException e) { - getLogger().error("Load SQLite driver error: "+e.getMessage()); + getLogger().error("Load SQLite driver error: " + e.getMessage()); System.exit(-1); } try { @@ -112,18 +120,17 @@ public class GWSqliteMetaDatabase extends AbstractGWMetaDatabase { return new Pair<>(rs, stat); } - private void buildWtidFieldGroupMap(Connection conn) { - if (wtidFieldGroupMap == null) { - wtidFieldGroupMap = new HashMap<>(); + private void buildAssetFieldGroupMap(Connection conn) { + if (assetFieldGroupMap == null) { + assetFieldGroupMap = new HashMap<>(); } - String sql = "SELECT wtid, protocolid FROM wtinfo"; - Pair resultPair = executeSql(conn, sql); + Pair resultPair = executeSql(conn, assetFieldGroupSql); ResultSet rs = resultPair.fst; try { while (rs.next()) { - String wtid = rs.getString(1).replace(" ","").toLowerCase(); - String fieldGroup = rs.getString(2).replace(" ","").toLowerCase(); - wtidFieldGroupMap.put(wtid, fieldGroup); + String wtid = rs.getString(1).replace(" ", "").toLowerCase(); + String fieldGroup = rs.getString(2).replace(" ", "").toLowerCase(); + assetFieldGroupMap.put(wtid, fieldGroup); } rs.close(); resultPair.snd.close(); @@ -140,11 +147,11 @@ public class GWSqliteMetaDatabase extends AbstractGWMetaDatabase { String sql = "SELECT protocolid, iecpath FROM propaths " + "WHERE iecpath <> 'WTUR.Tm.Rw.Dt' AND transtype='1' AND bsave=1 " + "ORDER BY protocolid, pathid"; - Pair resultPair = executeSql(conn, sql); + Pair resultPair = executeSql(conn, fieldGroupFieldsSql); ResultSet rs = resultPair.fst; try { while (rs.next()) { - String fieldGroup = rs.getString(1).replace(" ","").toLowerCase(); + String fieldGroup = rs.getString(1).replace(" ", "").toLowerCase(); String fieldName = rs.getString(2).replace(" ", "").toLowerCase(); if (!fieldGroupFieldsMap.containsKey(fieldGroup)) { fieldGroupFieldsMap.put(fieldGroup, new ArrayList()); @@ -159,32 +166,10 @@ public class GWSqliteMetaDatabase extends AbstractGWMetaDatabase { } } - private void buildWfidMap(Connection conn) { - if (wtidWfidMap == null) { - wtidWfidMap = new HashMap<>(); - } - String sql = "SELECT wtid,wfid FROM wtinfo"; - Pair resultPair = executeSql(conn, sql); - ResultSet rs = resultPair.fst; - try { - while (rs.next()) { - String wtid = rs.getString(1).replace(" ", "").toLowerCase(); - String wfid = rs.getString(2).replace(" ", "").toLowerCase(); - wtidWfidMap.put(wtid, wfid); - } - rs.close(); - resultPair.snd.close(); - } catch (SQLException e) { - getLogger().error("Read WFID result set error: " + e.getMessage()); - e.printStackTrace(); - } - } - @Override public void init() { Connection conn = connect(); - buildWtidFieldGroupMap(conn); + buildAssetFieldGroupMap(conn); buildFieldGroupFieldsMap(conn); - buildWfidMap(conn); } } diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/GWAnalyzeResult.java b/k2de-governance-exception-manager-base/src/main/java/com/k2data/governance/manager/result/AnalyzeResultImpl.java similarity index 31% rename from k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/GWAnalyzeResult.java rename to k2de-governance-exception-manager-base/src/main/java/com/k2data/governance/manager/result/AnalyzeResultImpl.java index 71dc751..e6aabf2 100644 --- a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/GWAnalyzeResult.java +++ b/k2de-governance-exception-manager-base/src/main/java/com/k2data/governance/manager/result/AnalyzeResultImpl.java @@ -1,69 +1,87 @@ -package com.k2data.governance.manager.analyzer; +package com.k2data.governance.manager.result; -import com.k2data.governance.manager.common.type.ErrorType; +import com.k2data.governance.manager.error.IErrorType; import scala.Serializable; /** - * Created by wangyihan on 2017/11/30 下午4:27. + * Created by wangyihan on 2017/12/19 下午6:32. * E-mail address is yihanwang22@163.com. * Copyright © 2017 wangyihan. All Rights Reserved. * * @author wangyihan */ -public class GWAnalyzeResult implements Serializable { - private ErrorType errorType; - private GWWindTurbineKey wt; +public class AnalyzeResultImpl implements IAnalyzeResult, Serializable { - public GWAnalyzeResult() { - this.wt = new GWWindTurbineKey(null, null); + private IErrorType errorType; + private IAsset asset; + + public AnalyzeResultImpl() { + } + + public AnalyzeResultImpl(IErrorType errorType) { + this(errorType, null); } - public GWAnalyzeResult(ErrorType errorType) { - this.wt = new GWWindTurbineKey(null, null); + public AnalyzeResultImpl(IErrorType errorType, IAsset asset) { this.errorType = errorType; + this.asset = asset; } - public void setErrorType(ErrorType errorType) { + @Override + public void setErrorType(IErrorType errorType) { this.errorType = errorType; } - public void setWtid(String wtid) { - this.wt.setWtid(wtid); + @Override + public void setAsset(IAsset asset) { + this.asset = asset; } - public void setProtocolId(String protocolId) { - this.wt.setProtocolId(protocolId); + @Override + public void setFieldGroup(String fieldGroup) { + if (this.asset == null) { + this.asset = new AssetImpl(); + } + this.asset.setFieldGroup(fieldGroup); } - public GWWindTurbineKey getWindTurbineKey() { - return wt; + @Override + public void setAssetId(String assetId) { + if (this.asset == null) { + this.asset = new AssetImpl(); + } + this.asset.setAssetId(assetId); } - public ErrorType getErrorType() { + @Override + public IErrorType getErrorType() { return errorType; } - public String getWtid() { - if (wt == null) { - return ""; - } - return wt.getWtid(); + @Override + public IAsset getAsset() { + return asset; } - public String getProtocolId() { - if (wt == null) { - return ""; + @Override + public String getFieldGroup() { + if (this.asset == null) { + return null; } - return wt.getProtocolId(); + return this.asset.getFieldGroup(); } - public String toReadableString() { - return String.format("ExceptionType: %s, %s", errorType, wt.toReadableString()); + @Override + public String getAssetId() { + if (this.asset == null) { + return null; + } + return this.asset.getAssetId(); } @Override public String toString() { - return String.format("%s,%s", errorType, wt); + return String.format("%s,%s", errorType, asset); } @Override @@ -79,13 +97,13 @@ public class GWAnalyzeResult implements Serializable { if (obj == null) { return false; } - if (!(obj instanceof GWAnalyzeResult)) { + if (!(obj instanceof AnalyzeResultImpl)) { return false; } - GWAnalyzeResult resultObj = (GWAnalyzeResult)obj; - if (this.wt == null && resultObj.getWindTurbineKey() != null) { + AnalyzeResultImpl resultObj = (AnalyzeResultImpl) obj; + if (this.asset == null && resultObj.getAsset() != null) { return false; - } else if (this.wt != null && !this.wt.equals(resultObj.getWindTurbineKey())) { + } else if (this.asset != null && !this.asset.equals(resultObj.getAsset())) { return false; } else if (this.errorType != resultObj.getErrorType()) { return false; diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/GWWindTurbineKey.java b/k2de-governance-exception-manager-base/src/main/java/com/k2data/governance/manager/result/AssetImpl.java similarity index 35% rename from k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/GWWindTurbineKey.java rename to k2de-governance-exception-manager-base/src/main/java/com/k2data/governance/manager/result/AssetImpl.java index 079f89d..60976ed 100644 --- a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/GWWindTurbineKey.java +++ b/k2de-governance-exception-manager-base/src/main/java/com/k2data/governance/manager/result/AssetImpl.java @@ -1,46 +1,49 @@ -package com.k2data.governance.manager.analyzer; +package com.k2data.governance.manager.result; import scala.Serializable; /** - * Created by wangyihan on 2017/12/1 上午11:02. + * Created by wangyihan on 2017/12/19 下午4:53. * E-mail address is yihanwang22@163.com. * Copyright © 2017 wangyihan. All Rights Reserved. * * @author wangyihan */ -public class GWWindTurbineKey implements Serializable { - private String wtid; - private String protocolId; +public class AssetImpl implements IAsset, Serializable { + private String fieldGroup; + private String assetId; - public GWWindTurbineKey(String wtid, String protocolId) { - this.wtid = wtid; - this.protocolId = protocolId; + public AssetImpl() { } - public String getWtid() { - return wtid; + public AssetImpl(String fieldGroup, String assetId) { + this.fieldGroup = fieldGroup; + this.assetId = assetId; } - public String getProtocolId() { - return protocolId; + @Override + public String getFieldGroup() { + return fieldGroup; } - public void setWtid(String wtid) { - this.wtid = wtid; + @Override + public String getAssetId() { + return assetId; } - public void setProtocolId(String protocolId) { - this.protocolId = protocolId; + @Override + public void setFieldGroup(String fieldGroup) { + this.fieldGroup = fieldGroup; } - public String toReadableString() { - return String.format("ProtocolID: %s, WTID: %s", protocolId, wtid); + @Override + public void setAssetId(String assetId) { + this.assetId = assetId; } @Override public String toString() { - return String.format("%s,%s", protocolId, wtid); + return String.format("%s,%s", fieldGroup, assetId); } @Override @@ -56,18 +59,18 @@ public class GWWindTurbineKey implements Serializable { if (obj == null) { return false; } - if (!(obj instanceof GWWindTurbineKey)) { + if (!(obj instanceof AssetImpl)) { return false; } - GWWindTurbineKey wtObj = (GWWindTurbineKey)obj; - if (this.wtid != null && !this.wtid.equals(wtObj.getWtid())) { + AssetImpl assetObj = (AssetImpl) obj; + if (this.assetId != null && !this.assetId.equals(assetObj.getAssetId())) { return false; - } else if (this.wtid == null && wtObj.getWtid() != null) { + } else if (this.assetId == null && assetObj.getAssetId() != null) { return false; } - if (this.protocolId != null && !this.protocolId.equals(wtObj.getProtocolId())) { + if (this.fieldGroup != null && !this.fieldGroup.equals(assetObj.getFieldGroup())) { return false; - } else if (this.protocolId == null && wtObj.getProtocolId() != null) { + } else if (this.fieldGroup == null && assetObj.getFieldGroup() != null) { return false; } return true; diff --git a/k2de-governance-exception-manager-base/src/main/java/com/k2data/governance/manager/result/IAnalyzeResult.java b/k2de-governance-exception-manager-base/src/main/java/com/k2data/governance/manager/result/IAnalyzeResult.java new file mode 100644 index 0000000..d520e69 --- /dev/null +++ b/k2de-governance-exception-manager-base/src/main/java/com/k2data/governance/manager/result/IAnalyzeResult.java @@ -0,0 +1,28 @@ +package com.k2data.governance.manager.result; + +import com.k2data.governance.manager.error.IErrorType; + +/** + * Created by wangyihan on 2017/12/19 下午4:39. + * E-mail address is yihanwang22@163.com. + * Copyright © 2017 wangyihan. All Rights Reserved. + * + * @author wangyihan + */ +public interface IAnalyzeResult { + void setErrorType(IErrorType errorType); + + void setAsset(IAsset asset); + + void setFieldGroup(String fieldGroup); + + void setAssetId(String assetId); + + IErrorType getErrorType(); + + IAsset getAsset(); + + String getFieldGroup(); + + String getAssetId(); +} diff --git a/k2de-governance-exception-manager-base/src/main/java/com/k2data/governance/manager/result/IAsset.java b/k2de-governance-exception-manager-base/src/main/java/com/k2data/governance/manager/result/IAsset.java new file mode 100644 index 0000000..3473a8f --- /dev/null +++ b/k2de-governance-exception-manager-base/src/main/java/com/k2data/governance/manager/result/IAsset.java @@ -0,0 +1,18 @@ +package com.k2data.governance.manager.result; + +/** + * Created by wangyihan on 2017/12/19 下午4:44. + * E-mail address is yihanwang22@163.com. + * Copyright © 2017 wangyihan. All Rights Reserved. + * + * @author wangyihan + */ +public interface IAsset { + void setAssetId(String assetId); + + void setFieldGroup(String fieldGroup); + + String getAssetId(); + + String getFieldGroup(); +} diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/util/ConfigureUtil.java b/k2de-governance-exception-manager-base/src/main/java/com/k2data/governance/manager/util/ConfigureUtil.java similarity index 99% rename from k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/util/ConfigureUtil.java rename to k2de-governance-exception-manager-base/src/main/java/com/k2data/governance/manager/util/ConfigureUtil.java index 3f5cf57..929a03c 100644 --- a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/util/ConfigureUtil.java +++ b/k2de-governance-exception-manager-base/src/main/java/com/k2data/governance/manager/util/ConfigureUtil.java @@ -59,7 +59,6 @@ public class ConfigureUtil { return null; } - /** * get string value with property key * @@ -99,7 +98,6 @@ public class ConfigureUtil { return 0; } - /** * get integer value with property key * @@ -132,7 +130,6 @@ public class ConfigureUtil { } catch (Exception e) { e.printStackTrace(); } - return false; } @@ -151,7 +148,6 @@ public class ConfigureUtil { } catch (Exception e) { e.printStackTrace(); } - return false; } @@ -168,11 +164,9 @@ public class ConfigureUtil { } catch (Exception e) { e.printStackTrace(); } - return 0L; } - /** * get long value with property key * @@ -187,7 +181,6 @@ public class ConfigureUtil { } catch (Exception e) { e.printStackTrace(); } - return 0L; } @@ -204,11 +197,9 @@ public class ConfigureUtil { } catch (Exception e) { e.printStackTrace(); } - return 0.0D; } - /** * get double value with property key * @@ -223,7 +214,6 @@ public class ConfigureUtil { } catch (Exception e) { e.printStackTrace(); } - return 0.0D; } diff --git a/k2de-governance-exception-manager-base/src/main/java/com/k2data/governance/manager/util/DateUtil.java b/k2de-governance-exception-manager-base/src/main/java/com/k2data/governance/manager/util/DateUtil.java new file mode 100644 index 0000000..6ab24a2 --- /dev/null +++ b/k2de-governance-exception-manager-base/src/main/java/com/k2data/governance/manager/util/DateUtil.java @@ -0,0 +1,38 @@ +package com.k2data.governance.manager.util; + +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; + +/** + * Created by wangyihan on 2017/12/29 下午6:46. + * E-mail address is yihanwang22@163.com. + * Copyright © 2017 wangyihan. All Rights Reserved. + * + * @author wangyihan + */ +public class DateUtil { + private static ThreadLocal local = new ThreadLocal(); + private static String formatString = "yyyy-MM-dd HH:mm:ss"; + + public static void setDateFormat(String formatStr) { + formatString = formatStr; + } + + private static SimpleDateFormat getDateFormat() { + SimpleDateFormat dateFormat = local.get(); + if (dateFormat == null) { + dateFormat = new SimpleDateFormat(formatString); + local.set(dateFormat); + } + return dateFormat; + } + + public static String format(Date date) { + return getDateFormat().format(date); + } + + public static Date parse(String dateStr) throws ParseException { + return getDateFormat().parse(dateStr); + } +} diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/util/InitUtil.java b/k2de-governance-exception-manager-base/src/main/java/com/k2data/governance/manager/util/InitUtil.java similarity index 91% rename from k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/util/InitUtil.java rename to k2de-governance-exception-manager-base/src/main/java/com/k2data/governance/manager/util/InitUtil.java index 6cf23ac..5a96f82 100644 --- a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/util/InitUtil.java +++ b/k2de-governance-exception-manager-base/src/main/java/com/k2data/governance/manager/util/InitUtil.java @@ -23,10 +23,10 @@ public class InitUtil { return builder.getOrCreate(); } - public static SparkSession initSparkContext(String local) { + public static SparkSession initSparkContext(boolean local) { SparkSession.Builder builder = SparkSession.builder(); builder.appName(ConfigureUtil.getString(SparkConstants.SPARK_APP_NAME)); - if (Boolean.parseBoolean(local)) { + if (local) { builder.master(ConfigureUtil.getString(SparkConstants.SPARK_MASTER)); } return builder.getOrCreate(); diff --git a/k2de-governance-exception-manager/src/main/resources/config.properties b/k2de-governance-exception-manager-base/src/main/resources/config.properties similarity index 100% rename from k2de-governance-exception-manager/src/main/resources/config.properties rename to k2de-governance-exception-manager-base/src/main/resources/config.properties diff --git a/k2de-governance-exception-manager/src/main/resources/log4j.properties b/k2de-governance-exception-manager-base/src/main/resources/log4j.properties similarity index 100% rename from k2de-governance-exception-manager/src/main/resources/log4j.properties rename to k2de-governance-exception-manager-base/src/main/resources/log4j.properties diff --git a/k2de-governance-exception-manager-huineng/pom.xml b/k2de-governance-exception-manager-huineng/pom.xml new file mode 100644 index 0000000..7fcddcd --- /dev/null +++ b/k2de-governance-exception-manager-huineng/pom.xml @@ -0,0 +1,71 @@ + + + + k2de-governance-parent + com.k2data + 1.0-SNAPSHOT + + 4.0.0 + + k2de-governance-exception-manager-huineng + + + + com.k2data + k2de-governance-exception-manager-base + 1.0-SNAPSHOT + + + com.k2data + k2de-governance-exception-manager-kmx + 1.0-SNAPSHOT + + + org.apache.spark + spark-core_${spark.comp.version} + + + org.apache.spark + spark-sql_${spark.comp.version} + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.6.1 + + ${compile.version} + ${compile.version} + + + + maven-assembly-plugin + 2.4 + + + + com.k2data.governance.manager.ExceptionManagerSpark + + + + jar-with-dependencies + + + + + make-assembly + package + + single + + + + + + + \ No newline at end of file diff --git a/k2de-governance-exception-manager-huineng/src/main/java/com/k2data/governance/manager/huineng/HnErrorType.java b/k2de-governance-exception-manager-huineng/src/main/java/com/k2data/governance/manager/huineng/HnErrorType.java new file mode 100644 index 0000000..99255c3 --- /dev/null +++ b/k2de-governance-exception-manager-huineng/src/main/java/com/k2data/governance/manager/huineng/HnErrorType.java @@ -0,0 +1,118 @@ +package com.k2data.governance.manager.huineng; + +import com.k2data.governance.manager.error.IErrorType; +import com.k2data.governance.manager.metadata.IMetaDatabase; +import com.k2data.governance.manager.result.AnalyzeResultImpl; +import com.k2data.governance.manager.result.IAnalyzeResult; +import com.k2data.governance.manager.util.DateUtil; +import scala.Serializable; + +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.List; + +/** + * Created by wangyihan on 2017/12/29 下午12:01. + * E-mail address is yihanwang22@163.com. + * Copyright © 2017 wangyihan. All Rights Reserved. + * + * @author wangyihan + */ +public enum HnErrorType implements IErrorType, Serializable { + FORMAT(0), // no realtime| in the msg + WTID_NULL(1), // wtid is not in DB + TIME(2), // time format wrong + PROTO_NULL(3), // protocolId is not in DB + FIELDS_NUM_MORE(4), // number of data more than in the DB + FIELDS_NUM_LESS(5), // number of data less than in the DB + WTID_EMPTY(6), + OTHER(7); + + public static String TIME_FORMAT = "yyyy-MM-dd HH:mm:ss.SSS"; + private final int value; + private static String HN_DATA_START_TAG = "realtimedata|"; + private static int fieldDiffTolerance = 30; + private static SimpleDateFormat format = new SimpleDateFormat(TIME_FORMAT); + + HnErrorType(int value) { + this.value = value; + } + + public String toErrorString() { + return String.format("HN_ERROR_%s", this); + } + + public int getValue() { + return value; + } + + public static HnErrorType fromInteger(int id) { + return HnErrorType.values()[id]; + } + + + public void setFiledDiffTolerance(int tolerance) { + fieldDiffTolerance = tolerance; + } + + @Override + public IAnalyzeResult analyzeRecord(String msg) { + return this.analyzeRecord(msg, null); + } + + @Override + public IAnalyzeResult analyzeRecord(String msg, IMetaDatabase metaDatabase) { + IAnalyzeResult result = new AnalyzeResultImpl(); + int formatIndex = msg.indexOf(HN_DATA_START_TAG); + if (formatIndex < 0) { + result.setErrorType(HnErrorType.FORMAT); + return result; + } + int startIndex = formatIndex + HN_DATA_START_TAG.length(); + String dataSubstr = msg.substring(startIndex); + String[] splitData = dataSubstr.split("\\|"); + String wtid = splitData[0].trim(); + String[] data = splitData[1].split(","); + String ts = data[0]; + try { + DateUtil.parse(ts); + } catch (ParseException e) { + result.setErrorType(HnErrorType.TIME); + } + if (wtid.trim().isEmpty()) { + result.setErrorType(HnErrorType.WTID_EMPTY); + return result; + } + result.setAssetId(wtid); + if (metaDatabase == null) { + result.setErrorType(HnErrorType.OTHER); + return result; + } + String fieldGroup = metaDatabase.getFieldGroup(wtid); + if (fieldGroup == null) { + result.setErrorType(HnErrorType.WTID_NULL); + return result; + } + result.setFieldGroup(fieldGroup); + List fields = metaDatabase.getFieldsByFieldGroup(fieldGroup); + if (fields == null) { + result.setErrorType(HnErrorType.PROTO_NULL); + result.setAssetId(null); // this type of exception dose not need wtid + return result; + } + if (fields.size() - (data.length - 1) > fieldDiffTolerance) { + result.setErrorType(HnErrorType.FIELDS_NUM_LESS); + return result; + } + if ((data.length - 1) - fields.size() > fieldDiffTolerance) { + result.setErrorType(HnErrorType.FIELDS_NUM_MORE); + return result; + } + + // if not in all errors above, then put it in OTHER + if (result.getErrorType() != HnErrorType.TIME) { + result.setErrorType(HnErrorType.OTHER); + } + return result; + } +} diff --git a/k2de-governance-exception-manager-huineng/src/main/java/com/k2data/governance/manager/huineng/HnExceptionManagerMain.java b/k2de-governance-exception-manager-huineng/src/main/java/com/k2data/governance/manager/huineng/HnExceptionManagerMain.java new file mode 100644 index 0000000..fca9034 --- /dev/null +++ b/k2de-governance-exception-manager-huineng/src/main/java/com/k2data/governance/manager/huineng/HnExceptionManagerMain.java @@ -0,0 +1,34 @@ +package com.k2data.governance.manager.huineng; + +import com.k2data.governance.manager.ExceptionManagerSpark; +import com.k2data.governance.manager.common.constants.ArgumentsConstants; +import com.k2data.governance.manager.filter.IFilter; +import com.k2data.governance.manager.metadata.IMetaDatabase; +import com.k2data.governance.manager.metadata.SqliteMetaDatabase; +import com.k2data.governance.manager.util.DateUtil; +import net.sourceforge.argparse4j.inf.Namespace; + +/** + * Created by wangyihan on 2017/12/29 下午5:10. + * E-mail address is yihanwang22@163.com. + * Copyright © 2017 wangyihan. All Rights Reserved. + * + * @author wangyihan + */ +public class HnExceptionManagerMain { + + public static void main(String[] args) { + DateUtil.setDateFormat(HnErrorType.TIME_FORMAT); + Namespace ns = ExceptionManagerSpark.parseArguments(args); + String metadataFilePath = ns.getString(ArgumentsConstants.METADATA_FILE); + String wtidProtocolSql = "SELECT wtid, protocolid FROM wtinfo"; + String protocolFieldsSql = "SELECT protocolid, iecpath FROM propaths " + + "WHERE iecpath <> 'WTUR.Tm.Rw.Dt' AND transtype='1' AND bsave=1 " + + "ORDER BY protocolid, pathid"; + IMetaDatabase metadataBase = new SqliteMetaDatabase(metadataFilePath, wtidProtocolSql, protocolFieldsSql); + ExceptionManagerSpark.setMetadataBase(metadataBase); + IFilter filter = new HnFilter(); + ExceptionManagerSpark.setFilter(filter); + ExceptionManagerSpark.run(ns); + } +} diff --git a/k2de-governance-exception-manager-huineng/src/main/java/com/k2data/governance/manager/huineng/HnFilter.java b/k2de-governance-exception-manager-huineng/src/main/java/com/k2data/governance/manager/huineng/HnFilter.java new file mode 100644 index 0000000..1fed7e0 --- /dev/null +++ b/k2de-governance-exception-manager-huineng/src/main/java/com/k2data/governance/manager/huineng/HnFilter.java @@ -0,0 +1,31 @@ +package com.k2data.governance.manager.huineng; + +import com.k2data.governance.manager.filter.IFilter; +import com.k2data.governance.manager.kmx.KmxErrorType; +import org.apache.log4j.Logger; +import scala.Serializable; + +/** + * Created by wangyihan on 2017/12/29 下午2:43. + * E-mail address is yihanwang22@163.com. + * Copyright © 2017 wangyihan. All Rights Reserved. + * + * @author wangyihan + */ +public class HnFilter implements IFilter, Serializable { + private String KMX_START_TAG = "[EXCEPTION]"; + private String HN_START_TAG = "sediment"; + + public HnFilter() { + } + + @Override + public Class getErrorClass(String message) { + if (message.startsWith(KMX_START_TAG)) { + return KmxErrorType.class; + } else if (message.startsWith(HN_START_TAG)) { + return HnErrorType.class; + } + return null; + } +} diff --git a/k2de-governance-exception-manager-huineng/src/main/resources/log4j.properties b/k2de-governance-exception-manager-huineng/src/main/resources/log4j.properties new file mode 100644 index 0000000..68429ad --- /dev/null +++ b/k2de-governance-exception-manager-huineng/src/main/resources/log4j.properties @@ -0,0 +1,6 @@ +# You should copy this file to resources, and rename it as log4j.properties + +log4j.rootLogger=ERROR,stdout +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d [%t] %-5p %c - %m%n \ No newline at end of file diff --git a/k2de-governance-exception-manager-kmx/pom.xml b/k2de-governance-exception-manager-kmx/pom.xml new file mode 100644 index 0000000..3c545fb --- /dev/null +++ b/k2de-governance-exception-manager-kmx/pom.xml @@ -0,0 +1,67 @@ + + + + com.k2data + k2de-governance-parent + 1.0-SNAPSHOT + + 4.0.0 + + k2de-governance-exception-manager-kmx + + + + com.k2data + k2de-governance-exception-manager-base + 1.0-SNAPSHOT + + + org.apache.spark + spark-core_${spark.comp.version} + + + org.apache.spark + spark-sql_${spark.comp.version} + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.6.1 + + ${compile.version} + ${compile.version} + + + + maven-assembly-plugin + 2.4 + + + + com.k2data.governance.manager.ExceptionManagerSpark + + + + jar-with-dependencies + + + + + make-assembly + package + + single + + + + + + + + \ No newline at end of file diff --git a/k2de-governance-exception-manager-kmx/src/main/java/com/k2data/governance/manager/kmx/KmxErrorType.java b/k2de-governance-exception-manager-kmx/src/main/java/com/k2data/governance/manager/kmx/KmxErrorType.java new file mode 100644 index 0000000..71f8c6c --- /dev/null +++ b/k2de-governance-exception-manager-kmx/src/main/java/com/k2data/governance/manager/kmx/KmxErrorType.java @@ -0,0 +1,91 @@ +package com.k2data.governance.manager.kmx; + +import com.k2data.governance.manager.error.IErrorType; +import com.k2data.governance.manager.metadata.IMetaDatabase; +import com.k2data.governance.manager.result.AnalyzeResultImpl; +import com.k2data.governance.manager.result.IAnalyzeResult; +import org.apache.log4j.Logger; +import scala.Serializable; + +/** + * Created by wangyihan on 2017/12/29 上午11:32. + * E-mail address is yihanwang22@163.com. + * Copyright © 2017 wangyihan. All Rights Reserved. + * + * @author wangyihan + */ +public enum KmxErrorType implements IErrorType, Serializable { + DOUBLE(0), BOOLEAN(1), LONG(2), STRING(3), INT(4), FLOAT(5), OTHER(6); + + private final int value; + private static final String TYPE_PREFIX = ": "; + private static final String TYPE_SUFFIX = "_EX"; + private final Logger logger = Logger.getLogger(KmxErrorType.class); + + KmxErrorType() { + this.value = 6; + } + + KmxErrorType(int value) { + this.value = value; + } + + @Override + public int getValue() { + return value; + } + + @Override + public String toErrorString() { + return String.format("KMX_ERROR_DATATYPE_%s", this); + } + + public static KmxErrorType fromInteger(int id) { + return KmxErrorType.values()[id]; + } + + public static KmxErrorType fromString(String type) { + KmxErrorType dataType; + switch (type.toLowerCase()) { + case "double": + dataType = KmxErrorType.DOUBLE; + break; + case "boolean": + dataType = KmxErrorType.BOOLEAN; + break; + case "long": + dataType = KmxErrorType.LONG; + break; + case "string": + dataType = KmxErrorType.STRING; + break; + case "int": + dataType = KmxErrorType.INT; + break; + case "float": + dataType = KmxErrorType.FLOAT; + break; + default: + dataType = KmxErrorType.OTHER; + break; + } + + return dataType; + } + + @Override + public IAnalyzeResult analyzeRecord(String msg) { + int indexType = msg.indexOf(TYPE_SUFFIX); + if (indexType == -1) { + return new AnalyzeResultImpl(KmxErrorType.OTHER); + } + String type = msg.substring(msg.indexOf(TYPE_PREFIX) + TYPE_PREFIX.length(), indexType); + return new AnalyzeResultImpl(KmxErrorType.fromString(type)); + } + + @Override + public IAnalyzeResult analyzeRecord(String msg, IMetaDatabase database) { + return analyzeRecord(msg); + } + +} diff --git a/k2de-governance-exception-manager-kmx/src/main/java/com/k2data/governance/manager/kmx/KmxExceptionManagerMain.java b/k2de-governance-exception-manager-kmx/src/main/java/com/k2data/governance/manager/kmx/KmxExceptionManagerMain.java new file mode 100644 index 0000000..37a0502 --- /dev/null +++ b/k2de-governance-exception-manager-kmx/src/main/java/com/k2data/governance/manager/kmx/KmxExceptionManagerMain.java @@ -0,0 +1,20 @@ +package com.k2data.governance.manager.kmx; + +import com.k2data.governance.manager.ExceptionManagerSpark; +import com.k2data.governance.manager.filter.IFilter; + +/** + * Created by wangyihan on 2017/12/29 下午5:18. + * E-mail address is yihanwang22@163.com. + * Copyright © 2017 wangyihan. All Rights Reserved. + * + * @author wangyihan + */ +public class KmxExceptionManagerMain { + + public static void main(String[] args) { + IFilter filter = new KmxFilter(); + ExceptionManagerSpark.setFilter(filter); + ExceptionManagerSpark.run(args); + } +} diff --git a/k2de-governance-exception-manager-kmx/src/main/java/com/k2data/governance/manager/kmx/KmxFilter.java b/k2de-governance-exception-manager-kmx/src/main/java/com/k2data/governance/manager/kmx/KmxFilter.java new file mode 100644 index 0000000..22d3b84 --- /dev/null +++ b/k2de-governance-exception-manager-kmx/src/main/java/com/k2data/governance/manager/kmx/KmxFilter.java @@ -0,0 +1,19 @@ +package com.k2data.governance.manager.kmx; + +import com.k2data.governance.manager.filter.IFilter; +import scala.Serializable; + +/** + * Created by wangyihan on 2017/12/29 上午11:27. + * E-mail address is yihanwang22@163.com. + * Copyright © 2017 wangyihan. All Rights Reserved. + * + * @author wangyihan + */ +public class KmxFilter implements IFilter, Serializable { + + @Override + public Class getErrorClass(String message) { + return KmxErrorType.class; + } +} diff --git a/k2de-governance-exception-manager-kmx/src/main/resources/log4j.properties b/k2de-governance-exception-manager-kmx/src/main/resources/log4j.properties new file mode 100644 index 0000000..68429ad --- /dev/null +++ b/k2de-governance-exception-manager-kmx/src/main/resources/log4j.properties @@ -0,0 +1,6 @@ +# You should copy this file to resources, and rename it as log4j.properties + +log4j.rootLogger=ERROR,stdout +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d [%t] %-5p %c - %m%n \ No newline at end of file diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/ExceptionManagerSingle.java b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/ExceptionManagerSingle.java deleted file mode 100644 index 5e35e7e..0000000 --- a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/ExceptionManagerSingle.java +++ /dev/null @@ -1,278 +0,0 @@ -package com.k2data.governance.manager; - -import com.k2data.governance.manager.analyzer.GWHnAnalyzer; -import com.k2data.governance.manager.analyzer.GWKmxAnalyzer; -import com.k2data.governance.manager.analyzer.GWWindTurbineKey; -import com.k2data.governance.manager.analyzer.IAnalyzer; -import com.k2data.governance.manager.common.constants.Constants; -import com.k2data.governance.manager.common.Pair; -import com.k2data.governance.manager.common.type.ErrorType; -import com.k2data.governance.manager.common.type.GWDataType; -import com.k2data.governance.manager.common.type.GWSourceType; -import com.k2data.governance.manager.common.type.HNErrorType; -import com.k2data.governance.manager.filter.GWFilter; -import com.k2data.governance.manager.filter.IFilter; -import com.k2data.governance.manager.message.IMessage; -import com.k2data.governance.manager.parser.GWParserSingle; -import com.k2data.governance.manager.util.FileHandler; -import org.apache.commons.lang.StringUtils; - -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.io.PrintWriter; -import java.nio.file.Paths; -import java.util.*; - -/** - * Created by stoke on 2017/11/16. - * E-mail address is zaqthss2009@gmail.com - * Copyright © stoke. All Rights Reserved. - * - * Main process for exception manager - * - * @author stoke - */ -public class ExceptionManagerSingle { - - private int[] kmxTypeNums; - private int[] hnTypeNums; - private Map> exceptionCount; - private Map>> uniqueIds; - private final String wtidKeyName = "wtid"; - private final String protocolKeyName = "protocolId"; - - public ExceptionManagerSingle() { - kmxTypeNums = new int[GWDataType.values().length]; - hnTypeNums = new int[HNErrorType.values().length]; - exceptionCount = new HashMap<>(); - uniqueIds = new HashMap<>(); - } - - public int[] getKmxTypeNums() { - return kmxTypeNums; - } - - public int[] getHnTypeNums() { - return hnTypeNums; - } - - private void updateExceptionCount(Map> newCount) { - GWWindTurbineKey allKey = new GWWindTurbineKey("all", "all"); - - for (Map.Entry> entry : newCount.entrySet()) { - boolean updateIds = false; - ErrorType errorType = entry.getKey(); - Map typeCount = entry.getValue(); - if (errorType instanceof HNErrorType) { - if (!uniqueIds.containsKey(errorType)) { - Map> idsMap = new HashMap<>(); - idsMap.put(wtidKeyName, new HashSet()); - idsMap.put(protocolKeyName, new HashSet()); - uniqueIds.put(errorType, idsMap); - } - updateIds = true; - } - if (!exceptionCount.containsKey(errorType)) { - exceptionCount.put(errorType, new HashMap()); - } - Map originCount = exceptionCount.get(errorType); - if (!originCount.containsKey(allKey)) { - originCount.put(allKey, 0); - } - Map> idsMap = uniqueIds.get(errorType); - for (Map.Entry countEntry : typeCount.entrySet()) { - GWWindTurbineKey key = countEntry.getKey(); - int count = countEntry.getValue(); - if (!originCount.containsKey(key)) { - originCount.put(key, 0); - } - originCount.put(key, originCount.get(key) + count); - originCount.put(allKey, originCount.get(allKey) + count); - if (!updateIds) { - continue; - } - String wtid = key.getWtid(); - if (wtid != null) { - idsMap.get(wtidKeyName).add(wtid); - } - String protocolId = key.getProtocolId(); - if (protocolId != null) { - idsMap.get(protocolKeyName).add(protocolId); - } - } - } - } - - private void sumTypeNum(int[] singleTypeNum, int[] finalTypeNum) { - if (singleTypeNum.length != finalTypeNum.length) { - System.out.println("Length does not match"); - return; - } - for (int i = 0; i < finalTypeNum.length; ++i) { - finalTypeNum[i] += singleTypeNum[i]; - } - } - - private void printResult(String outputPath) { - String kmxFile = outputPath + "kmx.out"; - String hnFile = outputPath + "hn.out"; - - try { - FileWriter fw = new FileWriter(kmxFile); - PrintWriter pw = new PrintWriter(fw); - for (GWDataType type : GWDataType.values()) { - pw.println(type + ":" + kmxTypeNums[type.getValue()]); - } - pw.close(); - fw.close(); - - fw = new FileWriter(hnFile); - pw = new PrintWriter(fw); - for (HNErrorType type : HNErrorType.values()) { - pw.println(type + ":" + hnTypeNums[type.getValue()]); - } - pw.close(); - fw.close(); - } catch (IOException e) { - e.printStackTrace(); - } - } - - private StringBuilder appendIdsInfo(ErrorType errorType, StringBuilder sb) { - Map> idsMap = uniqueIds.get(errorType); - if (idsMap == null) { - return sb; - } - for (Map.Entry> entry : idsMap.entrySet()) { - String idType = entry.getKey(); - Set ids = entry.getValue(); - sb.append(String.format("%s,%s:%s,IDs:%s\r\n", errorType, idType, - ids.size(), StringUtils.join(ids, ";"))); - } - return sb; - } - - private StringBuilder appendCountInfo(ErrorType errorType, - Map typeCountMap, StringBuilder sb ) { - for (Map.Entry countEntry : typeCountMap.entrySet()) { - GWWindTurbineKey key = countEntry.getKey(); - if (key == null) { - continue; - } - int count = countEntry.getValue(); - sb.append(String.format("%s,%s,%s,%s\r\n", errorType, key.getWtid(), key.getProtocolId(), count)); - } - return sb; - } - - private Pair getCountString() { - StringBuilder kmxExceptionStringBuilder = new StringBuilder(); - StringBuilder hnExceptionStringBuilder = new StringBuilder(); - kmxExceptionStringBuilder.append("ExceptionType,Count\r\n"); - hnExceptionStringBuilder.append("ExceptionType,WTID,ProtocolID,Count\r\n"); - for (Map.Entry> entry : exceptionCount.entrySet()) { - ErrorType errorType = entry.getKey(); - StringBuilder activeBuilder; - if (errorType instanceof HNErrorType) { - activeBuilder = appendIdsInfo(errorType, hnExceptionStringBuilder); - } else { - activeBuilder = kmxExceptionStringBuilder; - } - appendCountInfo(errorType, entry.getValue(), activeBuilder); - } - return new Pair<>(hnExceptionStringBuilder.toString(), kmxExceptionStringBuilder.toString()); - } - - private void writeExceptionCount(String outputPath) { - String hnCountFilename = Paths.get(outputPath, "hn_exceptions_count.out").toString(); - String kmxCountFilename = Paths.get(outputPath, "kmx_exceptions_count.out").toString(); - try { - Pair countStrings = getCountString(); - FileWriter fw = new FileWriter(hnCountFilename); - fw.write(countStrings.fst); - fw.close(); - fw = new FileWriter(kmxCountFilename); - fw.write(countStrings.snd); - fw.close(); - } catch (IOException e) { - e.printStackTrace(); - } - } - - public static void main(String[] args) { - System.out.println("ExceptionManagerSingle begin..."); - - if (args.length != 5) { - System.out.println("Input arguments number error:"); - System.out.println("args[0]: 0:sequence file; 1:file after parse;"); - System.out.println("args[1]: the input directory;"); - System.out.println("args[2]: the number of example to be printed;"); - System.out.println("args[3]: the configdata.file directory;"); - System.out.println("args[4]: the output directory;"); - return; - } - - List messageList; - int sel = Integer.parseInt(args[0]); - String inputPath = args[1]; - String dbPath = args[3]; - if (!dbPath.trim().equals("")) { - Constants.PROTOCOL_MAP_DIR = dbPath; - } - String outputPath = args[4]; - - File dir = new File(inputPath); - String[] fileNames; - if (dir.exists() && dir.isDirectory()) { - fileNames = dir.list(); - if (fileNames == null) { - System.out.println("No files in " + inputPath); - } - } else { - System.out.println("Can not find the directory " + inputPath); - return; - } - - ExceptionManagerSingle ems = new ExceptionManagerSingle(); - - assert fileNames != null; - for (String fileName : fileNames) { - if (!FileHandler.isGWFileValid(fileName)) { - continue; - } - if (sel == 1) { - messageList = FileHandler.readAnomalyString(Paths.get(inputPath, fileName).toString()); - } else { - GWParserSingle parser = new GWParserSingle(Paths.get(inputPath, fileName).toString()); - parser.parse(); - messageList = parser.getMessageList(); - } - - IFilter filter = new GWFilter(messageList); - filter.filter(); - - // KMX error - IAnalyzer kmxAnalyzer = new GWKmxAnalyzer( - filter.getCertainSourceTypeList(GWSourceType.KMX)); - kmxAnalyzer.printNumber(); - for (IMessage message : kmxAnalyzer.getMessageList()) { - kmxAnalyzer.analyze(message); - } - ems.updateExceptionCount(kmxAnalyzer.countExceptions()); - - // HN error - IAnalyzer hnAnalyzer = new GWHnAnalyzer(Constants.PROTOCOL_MAP_DIR, - filter.getCertainSourceTypeList(GWSourceType.HN)); - hnAnalyzer.printNumber(); - for (IMessage message : hnAnalyzer.getMessageList()) { - hnAnalyzer.analyze(message); - } - ems.updateExceptionCount(hnAnalyzer.countExceptions()); - } - - System.out.println("Building output file"); - ems.writeExceptionCount(outputPath); - System.out.println("ExceptionManagerSingle end"); - } -} diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/GWhnAnalyzer.java b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/GWhnAnalyzer.java deleted file mode 100644 index b46e5d9..0000000 --- a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/GWhnAnalyzer.java +++ /dev/null @@ -1,192 +0,0 @@ -package com.k2data.governance.manager.analyzer; - -import com.k2data.governance.manager.common.constants.Constants; -import com.k2data.governance.manager.common.type.ErrorType; -import com.k2data.governance.manager.common.type.HNErrorType; -import com.k2data.governance.manager.message.IMessage; -import com.k2data.governance.manager.metadata.GWSqliteMetaDatabase; -import com.k2data.governance.manager.metadata.IMetaDatabase; -import scala.Serializable; - -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * Created by stoke on 2017/11/16. - * E-mail address is zaqthss2009@gmail.com - * Copyright © stoke. All Rights Reserved. - * - * @author stoke - */ -public class GWHnAnalyzer implements IAnalyzer, Serializable { - - private IMetaDatabase metaDatabase; - private SimpleDateFormat format = new SimpleDateFormat(Constants.TIME_FORMAT); - - private List messageList; - private Map> hnTypeMessageMap; - private Map> errorTypeResultMap; - private int fieldDiffTolerance = 1; - - public GWHnAnalyzer(String dbFilePath) { - this(dbFilePath, null); - } - - public GWHnAnalyzer(String dbFilePath, List messageList) { - metaDatabase = new GWSqliteMetaDatabase(dbFilePath); - this.messageList = messageList; - - errorTypeResultMap = new HashMap<>(); - for (HNErrorType type : HNErrorType.values()) { - errorTypeResultMap.put(type, new ArrayList()); - } - } - - @Override - public List getMessageList() { - return messageList; - } - - /** - * Message Pattern: - * time: yyyy-MM-dd hh:mm:ss.SSS - * sendiment|time|(realtimedata|wtid|time,data[0],data[1],…, - * @param message message - */ - @Override - public void analyze(IMessage message) { - String msg = message.getMsg(); - GWAnalyzeResult analyzeResult = analyzeRecord(msg); - ErrorType errorType = analyzeResult.getErrorType(); - errorTypeResultMap.get(errorType).add(analyzeResult); - } - - public void setFieldDiffTolerance(int fieldDiffTolerance) { - this.fieldDiffTolerance = fieldDiffTolerance; - } - - public GWAnalyzeResult analyzeRecord(String msg) { - GWAnalyzeResult result = new GWAnalyzeResult(); - int formatIndex = msg.indexOf(Constants.GW_HN_DATA_START_TAG); - if (formatIndex < 0) { - result.setErrorType(HNErrorType.FORMAT); - return result; - } - int startIndex = formatIndex + Constants.GW_HN_DATA_START_TAG.length(); - String dataSubstr = msg.substring(startIndex); - String[] splitData = dataSubstr.split("\\|"); - String wtid = splitData[0].trim(); - String[] data = splitData[1].split(","); - String ts = data[0]; - try { - format.parse(ts); - } catch (ParseException e) { - result.setErrorType(HNErrorType.TIME); - } - if (wtid.trim().isEmpty()) { - result.setErrorType(HNErrorType.WTID_EMPTY); - return result; - } - result.setWtid(wtid); - String fieldGroup = metaDatabase.getFieldGroup(wtid); - if (fieldGroup == null) { - result.setErrorType(HNErrorType.WTID_NULL); - return result; - } - result.setProtocolId(fieldGroup); - List fields = metaDatabase.getFieldsByFieldGroup(fieldGroup); - if (fields == null) { - result.setErrorType(HNErrorType.PROTO_NULL); - result.setWtid(null); // this type of exception dose not need wtid - return result; - } - if (fields.size() - (data.length - 1) > fieldDiffTolerance) { - result.setErrorType(HNErrorType.FIELDS_NUM_LESS); - return result; - } - if ((data.length - 1) - fields.size() > fieldDiffTolerance) { - result.setErrorType(HNErrorType.FIELDS_NUM_MORE); - return result; - } - - // if not in all errors above, then put it in OTHER - if (result.getErrorType() != HNErrorType.TIME) { - result.setErrorType(HNErrorType.OTHER); - } - return result; - } - - @Override - public void printNumber() { - System.out.println("HN anomaly number : " + messageList.size()); - } - - @Override - public void printAll() { - printNumber(); - - for (IMessage message : messageList) { - System.out.println(message.toString()); - } - } - - @Override - public int[] printType() { - int[] typeNum = new int[HNErrorType.values().length]; - - for (Map.Entry> entry : hnTypeMessageMap.entrySet()) { - ErrorType type = entry.getKey(); - List certainList = entry.getValue(); - int size = certainList.size(); - typeNum[type.getValue()] = size; - System.out.println( - String.format("Exception type %s has %d messages", type.toString(), size)); - } - - int sumNum = 0; - for (int num : typeNum) { - sumNum += num; - } - System.out.println("There are " + sumNum + " hit exception messages in total"); - return typeNum; - } - - /** - * - * @param id type.value() - * @param size default msgList.size() - */ - public void printCertainTypeList(int id, int size) { - HNErrorType type = HNErrorType.fromInteger(id); - System.out.println("Error type : " + type); - List msgList = hnTypeMessageMap.get(type); - int realSize = msgList.size(); - int num = size > realSize ? realSize : size; - for (int i = 0; i < num; ++i) { - System.out.println(msgList.get(i).toString()); - } - } - - public Map> countExceptions() { - Map> countResult = new HashMap<>(); - - for (Map.Entry> entry : errorTypeResultMap.entrySet()) { - ErrorType errorType = entry.getKey(); - List resultList = entry.getValue(); - Map typeResultCount = new HashMap<>(); - for (GWAnalyzeResult result : resultList) { - GWWindTurbineKey key = result.getWindTurbineKey(); - if (!typeResultCount.containsKey(key)) { - typeResultCount.put(key, 0); - } - typeResultCount.put(key, typeResultCount.get(key) + 1); - } - countResult.put(errorType, typeResultCount); - } - return countResult; - } -} diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/GWkmxAnalyzer.java b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/GWkmxAnalyzer.java deleted file mode 100644 index 33afc7c..0000000 --- a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/GWkmxAnalyzer.java +++ /dev/null @@ -1,158 +0,0 @@ -package com.k2data.governance.manager.analyzer; - -import com.k2data.governance.manager.common.type.ErrorType; -import com.k2data.governance.manager.common.type.GWDataType; -import com.k2data.governance.manager.message.IMessage; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.apache.hadoop.io.serializer.Serialization; -import org.apache.log4j.Logger; -import scala.Serializable; - -/** - * Created by stoke on 2017/11/16. - * E-mail address is zaqthss2009@gmail.com - * Copyright © stoke. All Rights Reserved. - *

- * Analyze KMX type Exception - * - * @author stoke - */ -public class GWKmxAnalyzer implements IAnalyzer, Serializable { - - private static final String TYPE_PREFIX = ": "; - private static final String TYPE_SUFFIX = "_EX"; - private static final String TYPE_PREFIX_SMALL = "Invalid "; - private static final String TYPE_SUFFIX_SMALL = " value"; - private static final String VALUE_PREFIX = "value "; - private static final String VALUE_SUFFIX = " for field"; - private static final String FIELD_PREFIX = "field "; - private static final String FIELD_SUFFIX = " [GENERATED"; - - private List messageList; - private Map> kmxTypeMessageMap; - private Map> errorTypeResultMap; - - public GWKmxAnalyzer() { - this(null); - } - - public GWKmxAnalyzer(List messageList) { - this.messageList = messageList; - kmxTypeMessageMap = new HashMap<>(); - errorTypeResultMap = new HashMap<>(); - } - - @Override - public List getMessageList() { - return messageList; - } - - /** - * Message Pattern: - * [EXCEPTION] [Increase-Counters: _EX, RECORD_DROP] Invalid value for field - * [GENERATED AVRO RECORDS] [] - * - * @param message the exception message - */ - @Override - public void analyze(IMessage message) { - String msg = message.getMsg(); - GWAnalyzeResult analyzeResult = analyzeRecord(msg); - ErrorType errorType = analyzeResult.getErrorType(); - if (!kmxTypeMessageMap.containsKey(errorType)) { -// kmxTypeMessageMap.put(errorType, new ArrayList<>()); - errorTypeResultMap.put(errorType, new ArrayList()); - } -// kmxTypeMessageMap.get(errorType).add(message); - errorTypeResultMap.get(errorType).add(analyzeResult); - -// String type2 = msg.substring(msg.indexOf(TYPE_PREFIX_SMALL) + TYPE_PREFIX_SMALL.length(), -// msg.indexOf(TYPE_SUFFIX_SMALL)); -// String field = msg -// .substring(msg.indexOf(FIELD_PREFIX) + FIELD_PREFIX.length(), msg.indexOf(FIELD_SUFFIX)); -// String value = msg -// .substring(msg.indexOf(VALUE_PREFIX) + VALUE_PREFIX.length(), msg.indexOf(VALUE_SUFFIX)); -// // TODO check the value type - } - - @Override - public GWAnalyzeResult analyzeRecord(String msg) { - int indexType = msg.indexOf(TYPE_SUFFIX); - if (indexType == -1) { - String err = String.format("The pattern is wrong since no _EX included in %s", msg); - getLogger().error(err); - } - String type = msg.substring(msg.indexOf(TYPE_PREFIX) + TYPE_PREFIX.length(), indexType); - GWAnalyzeResult result = new GWAnalyzeResult(GWDataType.fromString(type)); - return result; - } - - @Override - public void printNumber() { - System.out.println("KMX anomaly number : " + messageList.size()); - } - - @Override - public void printAll() { - printNumber(); - - for (IMessage message : messageList) { - System.out.println(message.toString()); - } - } - - @Override - public int[] printType() { - int[] typeNums = new int[GWDataType.values().length]; - - for (Map.Entry> entry : kmxTypeMessageMap.entrySet()) { - ErrorType type = entry.getKey(); - List certainList = entry.getValue(); - int size = certainList.size(); - typeNums[type.getValue()] = size; - System.out.println( - String.format("Exception type %s has %d messages", type.toString(), certainList.size())); - } - return typeNums; - } - - @Override - public void printCertainTypeList(int id, int size) { - GWDataType type = GWDataType.fromInteger(id); - System.out.println("Error type : " + type); - List msgList = kmxTypeMessageMap.get(type); - int realsize = msgList.size(); - int num = size > realsize ? realsize : size; - for (int i = 0; i < num; ++i) { - System.out.println(msgList.get(i).toString()); - } - } - - public Map> countExceptions() { - Map> countResult = new HashMap<>(); - - for (Map.Entry> entry : errorTypeResultMap.entrySet()) { - ErrorType errorType = entry.getKey(); - List resultList = entry.getValue(); - Map typeResultCount = new HashMap<>(); - for (GWAnalyzeResult result : resultList) { - GWWindTurbineKey key = result.getWindTurbineKey(); - if (!typeResultCount.containsKey(key)) { - typeResultCount.put(key, 0); - } - typeResultCount.put(key, typeResultCount.get(key) + 1); - } - countResult.put(errorType, typeResultCount); - } - return countResult; - } - - private Logger getLogger() { - return Logger.getLogger(GWKmxAnalyzer.class); - } -} diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/IAnalyzer.java b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/IAnalyzer.java deleted file mode 100644 index ce780ec..0000000 --- a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/analyzer/IAnalyzer.java +++ /dev/null @@ -1,24 +0,0 @@ -package com.k2data.governance.manager.analyzer; - -import com.k2data.governance.manager.common.type.ErrorType; -import com.k2data.governance.manager.message.IMessage; -import java.util.List; -import java.util.Map; - -/** - * Created by stoke on 2017/11/16. - * E-mail address is zaqthss2009@gmail.com - * Copyright © stoke. All Rights Reserved. - * - * @author stoke - */ -public interface IAnalyzer { - List getMessageList(); - void analyze(IMessage message); - GWAnalyzeResult analyzeRecord(String msg); - void printNumber(); - void printAll(); - int[] printType(); - void printCertainTypeList(int id, int size); - Map> countExceptions(); -} diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/common/type/GWDataType.java b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/common/type/GWDataType.java deleted file mode 100644 index 33ca415..0000000 --- a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/common/type/GWDataType.java +++ /dev/null @@ -1,62 +0,0 @@ -package com.k2data.governance.manager.common.type; - -/** - * Created by stoke on 2017/11/16. - * E-mail address is zaqthss2009@gmail.com - * Copyright © stoke. All Rights Reserved. - * - * @author stoke - */ -public enum GWDataType implements ErrorType { - DOUBLE(0), BOOLEAN(1), LONG(2), STRING(3), INT(4), FLOAT(5), OTHER(6); - - private final int value; - - GWDataType(int value) { - this.value = value; - } - - @Override - public int getValue() { - return value; - } - - @Override - public String toErrorString() { - return String.format("KMX_ERROR_DATATYPE_%s", this); - } - - public static GWDataType fromInteger(int id) { - return GWDataType.values()[id]; - } - - public static GWDataType fromString(String type) { - GWDataType dataType; - switch (type.toLowerCase()) { - case "double": - dataType = GWDataType.DOUBLE; - break; - case "boolean": - dataType = GWDataType.BOOLEAN; - break; - case "long": - dataType = GWDataType.LONG; - break; - case "string": - dataType = GWDataType.STRING; - break; - case "int": - dataType = GWDataType.INT; - break; - case "float": - dataType = GWDataType.FLOAT; - break; - default: - dataType = GWDataType.OTHER; - break; - } - - return dataType; - } - -} diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/common/type/GWSourceType.java b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/common/type/GWSourceType.java deleted file mode 100644 index e04a947..0000000 --- a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/common/type/GWSourceType.java +++ /dev/null @@ -1,44 +0,0 @@ -package com.k2data.governance.manager.common.type; - -/** - * Created by stoke on 2017/11/16. - * E-mail address is zaqthss2009@gmail.com - * Copyright © stoke. All Rights Reserved. - * - * @author stoke - */ -public enum GWSourceType { - KMX(0), HN(1), OTHER(2); - - private final int value; - - GWSourceType(int value) { - this.value = value; - } - - public int getValue() { - return value; - } - - public static GWSourceType fromInteger(int id) { - return GWSourceType.values()[id]; - } - - public static GWSourceType fromString(String type) { - GWSourceType sourceType; - switch (type.toLowerCase()) { - case "kmx": - sourceType = GWSourceType.KMX; - break; - case "hn": - sourceType = GWSourceType.HN; - break; - default: - sourceType = GWSourceType.OTHER; - break; - } - - return sourceType; - } - -} diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/common/type/HNErrorType.java b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/common/type/HNErrorType.java deleted file mode 100644 index 50db060..0000000 --- a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/common/type/HNErrorType.java +++ /dev/null @@ -1,39 +0,0 @@ -package com.k2data.governance.manager.common.type; - -/** - * Created by stoke on 2017/11/16. - * E-mail address is zaqthss2009@gmail.com - * Copyright © stoke. All Rights Reserved. - * - * @author stoke - */ -public enum HNErrorType implements ErrorType { - FORMAT(0), // no realtime| in the msg - WTID_NULL(1), // wtid is not in DB - TIME(2), // time format wrong - PROTO_NULL(3), // protocolid is not in DB - FIELDS_NUM_MORE(4), // number of data more than in the DB - FIELDS_NUM_LESS(5), // number of data less than in the DB - WTID_EMPTY(6), - FIELDS_NULL(7), // fields null - OTHER(8); - - private final int value; - - HNErrorType(int value) { - this.value = value; - } - - public String toErrorString() { - return String.format("HN_ERROR_%s", this); - } - - public int getValue() { - return value; - } - - public static HNErrorType fromInteger(int id) { - return HNErrorType.values()[id]; - } - -} diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/filter/GWFilter.java b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/filter/GWFilter.java deleted file mode 100644 index 7fb4218..0000000 --- a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/filter/GWFilter.java +++ /dev/null @@ -1,71 +0,0 @@ -package com.k2data.governance.manager.filter; - -import com.k2data.governance.manager.common.constants.Constants; -import com.k2data.governance.manager.common.type.GWSourceType; -import com.k2data.governance.manager.message.IMessage; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.apache.log4j.Logger; - -/** - * Created by stoke on 2017/11/16. - * E-mail address is zaqthss2009@gmail.com - * Copyright © stoke. All Rights Reserved. - * - * @author stoke - */ -public class GWFilter implements IFilter { - private Logger logger = Logger.getLogger(GWFilter.class); - - private List messageList; - private Map> gwSourceTypeMessageMap; - - public GWFilter(List messageList) { - this.messageList = messageList; - gwSourceTypeMessageMap = new HashMap<>(); - - for (GWSourceType type : GWSourceType.values()) { - gwSourceTypeMessageMap.put(type, new ArrayList()); - } - } - - @Override - public List getMessageList() { - return messageList; - } - - public Map> getGwSourceTypeMessageMap() { - return gwSourceTypeMessageMap; - } - - @Override - public List getCertainSourceTypeList(GWSourceType type) { - return gwSourceTypeMessageMap.get(type); - } - - @Override - public void filter() { - for (IMessage message : messageList) { - String msg = message.getMsg(); - GWSourceType type = GWFilter.getSourceType(msg); - gwSourceTypeMessageMap.get(type).add(message); - if (type == GWSourceType.OTHER) { - String err = String.format("Messages for other type: %s", message.toString()); - logger.error(err); - } - } - } - - public static GWSourceType getSourceType(String message) { - if (message.startsWith(Constants.GW_KMX_START_TAG)) { - return GWSourceType.KMX; - } else if (message.startsWith(Constants.GW_HN_START_TAG)) { - return GWSourceType.HN; - } - return GWSourceType.OTHER; - } -} diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/message/AbstractMessage.java b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/message/AbstractMessage.java deleted file mode 100644 index ae0b901..0000000 --- a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/message/AbstractMessage.java +++ /dev/null @@ -1,38 +0,0 @@ -package com.k2data.governance.manager.message; - -/** - * Created by stoke on 2017/11/16. - * E-mail address is zaqthss2009@gmail.com - * Copyright © stoke. All Rights Reserved. - * - * @author stoke - */ -public abstract class AbstractMessage implements IMessage { - private String key; - private String msg; - - @Override - public String getKey() { - return key; - } - - @Override - public void setKey(String key) { - this.key = key; - } - - @Override - public String getMsg() { - return msg; - } - - @Override - public void setMsg(String msg) { - this.msg = msg; - } - - @Override - public String toString() { - return String.format("%s,%s", getKey(), getMsg()); - } -} diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/message/GWMessage.java b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/message/GWMessage.java deleted file mode 100644 index c54135b..0000000 --- a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/message/GWMessage.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.k2data.governance.manager.message; - -import com.k2data.governance.manager.common.constants.Constants; - -/** - * Created by stoke on 2017/11/16. - * E-mail address is zaqthss2009@gmail.com - * Copyright © stoke. All Rights Reserved. - * - * @author stoke - */ -public class GWMessage extends AbstractMessage { - - public GWMessage(String key, String msg) { - setKey(key); - setMsg(msg); - } - - @Override - public String toString() { - return String.format("%s" + Constants.GW_MSG_SPILT_OP + "%s", getKey(), getMsg()); - } -} diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/message/IMessage.java b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/message/IMessage.java deleted file mode 100644 index e2fad55..0000000 --- a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/message/IMessage.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.k2data.governance.manager.message; - -/** - * Created by stoke on 2017/11/17. - * E-mail address is zaqthss2009@gmail.com - * Copyright © stoke. All Rights Reserved. - * - * @author stoke - */ -public interface IMessage { - void setKey(String key); - String getKey(); - void setMsg(String msg); - String getMsg(); -} diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/parser/GWParserSingle.java b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/parser/GWParserSingle.java deleted file mode 100644 index 51252f8..0000000 --- a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/parser/GWParserSingle.java +++ /dev/null @@ -1,57 +0,0 @@ -package com.k2data.governance.manager.parser; - -import com.k2data.governance.manager.message.GWMessage; -import com.k2data.governance.manager.message.IMessage; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.IOUtils; -import org.apache.hadoop.io.SequenceFile.Reader; -import org.apache.hadoop.io.Text; - -/** - * Created by stoke on 2017/11/20. - * E-mail address is zaqthss2009@gmail.com - * Copyright © stoke. All Rights Reserved. - * - * @author stoke - */ -public class GWParserSingle { - private static Configuration conf = new Configuration(); - private String path; - private List messageList; - - public List getMessageList() { - return messageList; - } - - public GWParserSingle(String path) { - messageList = new ArrayList<>(); - this.path = path; - } - - public void parse() { - Path filePath = new Path(path); - try { - FileSystem fs = FileSystem.get(conf); - Reader reader = new Reader(fs, filePath, conf); - Text key = new Text(); - Text value = new Text(); - while (reader.next(key, value)) { - messageList.add(new GWMessage(key.toString(), value.toString())); - } - IOUtils.closeStream(reader); - - } catch (IOException e) { - e.printStackTrace(); - } catch (OutOfMemoryError e) { - System.out.println(messageList.size()); - System.exit(-1); - } - - } -} diff --git a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/util/FileHandler.java b/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/util/FileHandler.java deleted file mode 100644 index 0bf30ae..0000000 --- a/k2de-governance-exception-manager/src/main/java/com/k2data/governance/manager/util/FileHandler.java +++ /dev/null @@ -1,99 +0,0 @@ -package com.k2data.governance.manager.util; - -import com.k2data.governance.manager.message.GWMessage; -import com.k2data.governance.manager.message.IMessage; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.io.FileWriter; -import java.io.IOException; -import java.io.PrintWriter; -import java.util.ArrayList; -import java.util.List; - -/** - * Created by stoke on 2017/11/15. - * E-mail address is zaqthss2009@gmail.com - * Copyright © stoke. All Rights Reserved. - * - * @author stoke - */ -public class FileHandler { - public static final String SPLIT_OP = "\t"; - - /** - * used for MR job, remove the outputDir in advance - * @param outputPath outputPath - */ - public static void removeOutputDir(String outputPath) { - File dir = new File(outputPath); - if (dir.exists()) { - File[] files = dir.listFiles(); - if (files != null) { - for (File file : files) { - file.delete(); - } - } - dir.delete(); - } - } - - /** - * @param path - */ - public static List readAnomalyString(String path) { - List messageList = new ArrayList<>(); - try { - FileReader fr = new FileReader(path); - BufferedReader br = new BufferedReader(fr); - - String line; - while ((line = br.readLine()) != null) { - String[] vals = line.split(SPLIT_OP); - if (vals.length <= 1) { - System.out.println(line); - } - String key = vals[0]; - String msg = vals[1]; - messageList.add(new GWMessage(key, msg)); - } - - br.close(); - fr.close(); - } catch (IOException e) { - e.printStackTrace(); - } - - return messageList; - } - - public static void writeAnomalyString(String outputPath, List messageList) { - try { - FileWriter fw = new FileWriter(outputPath); - PrintWriter pw = new PrintWriter(fw); - - for (IMessage message : messageList) { - pw.println(message.getKey() + SPLIT_OP + message.getMsg()); - } - - pw.close(); - fw.close(); - } catch (IOException e) { - e.printStackTrace(); - } - } - - public static boolean isGWFileValid(String fileName) { - return !fileName.startsWith("."); -// return fileName.endsWith(Constants.GW_EXCEPTION_END); - } - - public static void main(String[] args) { - String[] outputPaths = { "result/", "output/" }; - - for(String outputPath : outputPaths) { - removeOutputDir(outputPath); - } - } -} diff --git a/pom.xml b/pom.xml index 00d0835..1bc6dde 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,9 @@ k2de-governance-data-extractor - k2de-governance-exception-manager + k2de-governance-exception-manager-base + k2de-governance-exception-manager-kmx + k2de-governance-exception-manager-huineng @@ -49,6 +51,12 @@ 3.20.1 + + net.sourceforge.argparse4j + argparse4j + 0.7.0 + + org.apache.spark spark-core_${spark.comp.version} @@ -68,6 +76,7 @@ tools 1.4.2 system + ${java.home}/../lib/tools.jar -- Gitee