Merge branch 'master' of https://github.com/apache/incubator-griffin
Project: http://git-wip-us.apache.org/repos/asf/incubator-griffin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-griffin/commit/c16cbc26 Tree: http://git-wip-us.apache.org/repos/asf/incubator-griffin/tree/c16cbc26 Diff: http://git-wip-us.apache.org/repos/asf/incubator-griffin/diff/c16cbc26 Branch: refs/heads/master Commit: c16cbc26717d1f770b115b3f3a512c9d3dae9fdd Parents: 56b6785 6fd22ae Author: William Guo <guo...@icloud.com> Authored: Fri Aug 11 15:12:51 2017 +0800 Committer: William Guo <guo...@icloud.com> Committed: Fri Aug 11 15:12:51 2017 +0800 ---------------------------------------------------------------------- griffin-doc/measure-streaming-sample.md | 204 +++++++++++ measure/pom.xml | 8 + measure/src/main/resources/config-old.json | 45 --- .../src/main/resources/config-streaming.json | 61 ++++ measure/src/main/resources/config.json | 2 + measure/src/main/resources/env.json | 16 + .../apache/griffin/measure/Application.scala | 133 +++++++ .../griffin/measure/algo/AccuracyAlgo.scala | 24 ++ .../org/apache/griffin/measure/algo/Algo.scala | 34 ++ .../griffin/measure/algo/MeasureType.scala | 26 ++ .../griffin/measure/algo/ProcessType.scala | 26 ++ .../griffin/measure/algo/ProfileAlgo.scala | 23 ++ .../measure/algo/batch/BatchAccuracyAlgo.scala | 190 ++++++++++ .../measure/algo/batch/BatchProfileAlgo.scala | 162 +++++++++ .../measure/algo/core/AccuracyCore.scala | 103 ++++++ .../griffin/measure/algo/core/ProfileCore.scala | 73 ++++ .../algo/streaming/StreamingAccuracyAlgo.scala | 358 +++++++++++++++++++ .../streaming/StreamingAccuracyProcess.scala | 234 ++++++++++++ .../measure/algo/streaming/TimingProcess.scala | 46 +++ .../griffin/measure/batch/Application.scala | 113 ------ .../measure/batch/algo/AccuracyAlgo.scala | 24 -- .../griffin/measure/batch/algo/Algo.scala | 34 -- .../measure/batch/algo/BatchAccuracyAlgo.scala | 183 ---------- .../measure/batch/algo/BatchProfileAlgo.scala | 155 -------- .../measure/batch/algo/MeasureType.scala | 26 -- .../measure/batch/algo/ProfileAlgo.scala | 23 -- .../measure/batch/algo/core/AccuracyCore.scala | 101 ------ .../measure/batch/algo/core/ProfileCore.scala | 73 ---- .../measure/batch/config/params/AllParam.scala | 32 -- .../measure/batch/config/params/Param.scala | 25 -- .../batch/config/params/env/CleanerParam.scala | 28 -- .../batch/config/params/env/EnvParam.scala | 31 -- .../batch/config/params/env/PersistParam.scala | 30 -- .../batch/config/params/env/SparkParam.scala | 31 -- .../config/params/user/DataConnectorParam.scala | 31 -- .../config/params/user/EvaluateRuleParam.scala | 30 -- .../batch/config/params/user/UserParam.scala | 33 -- .../batch/config/reader/ParamFileReader.scala | 38 -- .../config/reader/ParamHdfsFileReader.scala | 38 -- .../config/reader/ParamRawStringReader.scala | 35 -- .../batch/config/reader/ParamReader.scala | 30 -- .../config/reader/ParamReaderFactory.scala | 40 --- .../config/validator/AllParamValidator.scala | 34 -- .../batch/config/validator/ParamValidator.scala | 30 -- .../batch/connector/AvroDataConnector.scala | 109 ------ .../measure/batch/connector/DataConnector.scala | 34 -- .../batch/connector/DataConnectorFactory.scala | 49 --- .../batch/connector/HiveDataConnector.scala | 131 ------- .../griffin/measure/batch/log/Loggable.scala | 43 --- .../measure/batch/persist/HdfsPersist.scala | 171 --------- .../measure/batch/persist/HttpPersist.scala | 78 ---- .../measure/batch/persist/LoggerPersist.scala | 89 ----- .../measure/batch/persist/MultiPersists.scala | 49 --- .../griffin/measure/batch/persist/Persist.scala | 44 --- .../measure/batch/persist/PersistFactory.scala | 51 --- .../measure/batch/result/AccuracyResult.scala | 44 --- .../measure/batch/result/ProfileResult.scala | 44 --- .../griffin/measure/batch/result/Result.scala | 32 -- .../measure/batch/result/ResultInfo.scala | 57 --- .../measure/batch/rule/CalculationUtil.scala | 315 ---------------- .../measure/batch/rule/ExprValueUtil.scala | 93 ----- .../measure/batch/rule/RuleAnalyzer.scala | 78 ---- .../measure/batch/rule/RuleFactory.scala | 52 --- .../griffin/measure/batch/rule/RuleParser.scala | 240 ------------- .../batch/rule/expr/AnalyzableExpr.scala | 25 -- .../measure/batch/rule/expr/Cacheable.scala | 33 -- .../measure/batch/rule/expr/Calculatable.scala | 25 -- .../batch/rule/expr/DataSourceable.scala | 28 -- .../measure/batch/rule/expr/Describable.scala | 33 -- .../griffin/measure/batch/rule/expr/Expr.scala | 51 --- .../measure/batch/rule/expr/ExprDescOnly.scala | 40 --- .../measure/batch/rule/expr/ExprIdCounter.scala | 60 ---- .../measure/batch/rule/expr/FieldDescOnly.scala | 58 --- .../measure/batch/rule/expr/LiteralExpr.scala | 96 ----- .../measure/batch/rule/expr/LogicalExpr.scala | 177 --------- .../measure/batch/rule/expr/MathExpr.scala | 97 ----- .../measure/batch/rule/expr/SelectExpr.scala | 71 ---- .../measure/batch/rule/expr/StatementExpr.scala | 70 ---- .../griffin/measure/batch/utils/HdfsUtil.scala | 80 ----- .../griffin/measure/batch/utils/HttpUtil.scala | 48 --- .../griffin/measure/batch/utils/JsonUtil.scala | 50 --- .../griffin/measure/cache/info/InfoCache.scala | 39 ++ .../measure/cache/info/InfoCacheFactory.scala | 41 +++ .../measure/cache/info/InfoCacheInstance.scala | 53 +++ .../measure/cache/info/TimeInfoCache.scala | 116 ++++++ .../measure/cache/info/ZKInfoCache.scala | 210 +++++++++++ .../griffin/measure/cache/lock/CacheLock.scala | 31 ++ .../measure/cache/lock/MultiCacheLock.scala | 39 ++ .../measure/cache/lock/ZKCacheLock.scala | 53 +++ .../measure/cache/result/CacheResult.scala | 29 ++ .../cache/result/CacheResultProcesser.scala | 71 ++++ .../measure/config/params/AllParam.scala | 32 ++ .../griffin/measure/config/params/Param.scala | 25 ++ .../config/params/env/CleanerParam.scala | 29 ++ .../measure/config/params/env/EnvParam.scala | 32 ++ .../config/params/env/InfoCacheParam.scala | 30 ++ .../config/params/env/PersistParam.scala | 30 ++ .../measure/config/params/env/SparkParam.scala | 33 ++ .../config/params/user/DataCacheParam.scala | 31 ++ .../config/params/user/DataConnectorParam.scala | 37 ++ .../config/params/user/EvaluateRuleParam.scala | 30 ++ .../measure/config/params/user/UserParam.scala | 34 ++ .../measure/config/reader/ParamFileReader.scala | 38 ++ .../config/reader/ParamHdfsFileReader.scala | 38 ++ .../config/reader/ParamRawStringReader.scala | 35 ++ .../measure/config/reader/ParamReader.scala | 30 ++ .../config/reader/ParamReaderFactory.scala | 40 +++ .../config/validator/AllParamValidator.scala | 34 ++ .../config/validator/ParamValidator.scala | 30 ++ .../measure/connector/DataConnector.scala | 32 ++ .../connector/DataConnectorFactory.scala | 139 +++++++ .../connector/cache/CacheDataConnector.scala | 33 ++ .../measure/connector/cache/DataCacheable.scala | 86 +++++ .../measure/connector/cache/DataUpdatable.scala | 30 ++ .../cache/HiveCacheDataConnector.scala | 351 ++++++++++++++++++ .../cache/TextCacheDataConnector.scala | 311 ++++++++++++++++ .../direct/AvroDirectDataConnector.scala | 132 +++++++ .../connector/direct/DirectDataConnector.scala | 34 ++ .../direct/HiveDirectDataConnector.scala | 158 ++++++++ .../direct/KafkaCacheDirectDataConnector.scala | 125 +++++++ .../StreamingCacheDirectDataConnector.scala | 60 ++++ .../streaming/KafkaStreamingDataConnector.scala | 58 +++ .../streaming/StreamingDataConnector.scala | 34 ++ .../apache/griffin/measure/log/Loggable.scala | 43 +++ .../griffin/measure/persist/HdfsPersist.scala | 206 +++++++++++ .../griffin/measure/persist/HttpPersist.scala | 88 +++++ .../griffin/measure/persist/LoggerPersist.scala | 118 ++++++ .../griffin/measure/persist/MultiPersists.scala | 52 +++ .../measure/persist/OldHttpPersist.scala | 87 +++++ .../griffin/measure/persist/Persist.scala | 52 +++ .../measure/persist/PersistFactory.scala | 53 +++ .../measure/persist/PersistThreadPool.scala | 62 ++++ .../griffin/measure/result/AccuracyResult.scala | 44 +++ .../griffin/measure/result/DataInfo.scala | 50 +++ .../griffin/measure/result/ProfileResult.scala | 44 +++ .../apache/griffin/measure/result/Result.scala | 32 ++ .../griffin/measure/rule/CalculationUtil.scala | 315 ++++++++++++++++ .../measure/rule/DataTypeCalculationUtil.scala | 159 ++++++++ .../griffin/measure/rule/ExprValueUtil.scala | 263 ++++++++++++++ .../griffin/measure/rule/RuleAnalyzer.scala | 72 ++++ .../griffin/measure/rule/RuleFactory.scala | 52 +++ .../griffin/measure/rule/RuleParser.scala | 244 +++++++++++++ .../measure/rule/SchemaValueCombineUtil.scala | 187 ++++++++++ .../measure/rule/expr/AnalyzableExpr.scala | 24 ++ .../griffin/measure/rule/expr/Cacheable.scala | 33 ++ .../measure/rule/expr/Calculatable.scala | 25 ++ .../griffin/measure/rule/expr/ClauseExpr.scala | 109 ++++++ .../measure/rule/expr/DataSourceable.scala | 28 ++ .../griffin/measure/rule/expr/Describable.scala | 33 ++ .../apache/griffin/measure/rule/expr/Expr.scala | 53 +++ .../measure/rule/expr/ExprDescOnly.scala | 40 +++ .../measure/rule/expr/ExprIdCounter.scala | 60 ++++ .../measure/rule/expr/FieldDescOnly.scala | 58 +++ .../griffin/measure/rule/expr/LiteralExpr.scala | 83 +++++ .../griffin/measure/rule/expr/LogicalExpr.scala | 178 +++++++++ .../griffin/measure/rule/expr/MathExpr.scala | 99 +++++ .../griffin/measure/rule/expr/SelectExpr.scala | 88 +++++ .../rule/func/DefaultFunctionDefine.scala | 36 ++ .../measure/rule/func/FunctionDefine.scala | 25 ++ .../measure/rule/func/FunctionUtil.scala | 75 ++++ .../measure/utils/HdfsFileDumpUtil.scala | 83 +++++ .../apache/griffin/measure/utils/HdfsUtil.scala | 120 +++++++ .../apache/griffin/measure/utils/HttpUtil.scala | 54 +++ .../apache/griffin/measure/utils/JsonUtil.scala | 54 +++ .../apache/griffin/measure/utils/TimeUtil.scala | 79 ++++ measure/src/test/resources/config-profile.json | 2 + .../src/test/resources/config-streaming.json | 69 ++++ .../src/test/resources/config-streaming1.json | 65 ++++ .../src/test/resources/config-streaming2.json | 65 ++++ .../src/test/resources/config-streaming3.json | 65 ++++ measure/src/test/resources/config.json | 4 +- measure/src/test/resources/config1.json | 2 + measure/src/test/resources/env-streaming.json | 45 +++ measure/src/test/resources/env.json | 24 +- .../algo/batch/BatchAccuracyAlgoTest.scala | 198 ++++++++++ .../algo/batch/BatchProfileAlgoTest.scala | 173 +++++++++ .../measure/algo/batch/DataFrameSaveTest.scala | 172 +++++++++ .../measure/algo/core/AccuracyCoreTest.scala | 89 +++++ .../measure/algo/core/ProfileCoreTest.scala | 79 ++++ .../streaming/StreamingAccuracyAlgoTest.scala | 267 ++++++++++++++ .../batch/algo/BatchAccuracyAlgoTest.scala | 192 ---------- .../batch/algo/BatchProfileAlgoTest.scala | 167 --------- .../batch/algo/core/AccuracyCoreTest.scala | 89 ----- .../batch/algo/core/ProfileCoreTest.scala | 79 ---- .../reader/ParamRawStringReaderTest.scala | 38 -- .../validator/AllParamValidatorTest.scala | 40 --- .../measure/batch/persist/HdfsPersistTest.scala | 48 --- .../measure/batch/persist/HttpPersistTest.scala | 42 --- .../batch/result/AccuracyResultTest.scala | 57 --- .../batch/result/ProfileResultTest.scala | 57 --- .../measure/batch/rule/RuleAnalyzerTest.scala | 64 ---- .../measure/batch/rule/RuleFactoryTest.scala | 44 --- .../measure/batch/rule/RuleParserTest.scala | 202 ----------- .../measure/batch/utils/JsonUtilTest.scala | 60 ---- .../measure/cache/InfoCacheInstanceTest.scala | 78 ++++ .../griffin/measure/cache/ZKCacheLockTest.scala | 84 +++++ .../griffin/measure/cache/ZKInfoCacheTest.scala | 90 +++++ .../reader/ParamRawStringReaderTest.scala | 38 ++ .../validator/AllParamValidatorTest.scala | 40 +++ .../measure/connector/ConnectorTest.scala | 70 ++++ .../measure/persist/HdfsPersistTest.scala | 48 +++ .../measure/persist/HttpPersistTest.scala | 42 +++ .../measure/result/AccuracyResultTest.scala | 57 +++ .../measure/result/ProfileResultTest.scala | 57 +++ .../measure/rule/ExprValueUtilTest.scala | 86 +++++ .../griffin/measure/rule/RuleAnalyzerTest.scala | 60 ++++ .../griffin/measure/rule/RuleFactoryTest.scala | 44 +++ .../griffin/measure/rule/RuleParserTest.scala | 213 +++++++++++ .../griffin/measure/utils/JsonUtilTest.scala | 60 ++++ service/src/main/resources/sparkJob.properties | 2 +- 210 files changed, 10619 insertions(+), 5322 deletions(-) ----------------------------------------------------------------------
