[kylin] branch master updated: KYLIN-3365 Add unit test for the coprocessor code, CubeVisitService
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/master by this push:
new 91efa0c KYLIN-3365 Add unit test for the coprocessor code,
CubeVisitService
91efa0c is described below
commit 91efa0cff4c23246dd4e9c8eee87b7aeee54837e
Author: kyotoYaho
AuthorDate: Wed Dec 19 14:49:32 2018 +0800
KYLIN-3365 Add unit test for the coprocessor code, CubeVisitService
---
storage-hbase/pom.xml | 13 +-
.../hbase/cube/v2/CubeHBaseEndpointRPC.java| 4 +-
.../coprocessor/endpoint/CubeVisitServiceTest.java | 568 +
3 files changed, 581 insertions(+), 4 deletions(-)
diff --git a/storage-hbase/pom.xml b/storage-hbase/pom.xml
index 190dd8c..167280d 100644
--- a/storage-hbase/pom.xml
+++ b/storage-hbase/pom.xml
@@ -17,7 +17,8 @@
limitations under the License.
-->
-http://maven.apache.org/POM/4.0.0;
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
+http://maven.apache.org/POM/4.0.0;
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
4.0.0
kylin-storage-hbase
@@ -65,6 +66,13 @@
test
+org.apache.hadoop
+hadoop-hdfs
+${hadoop2.version}
+test-jar
+test
+
+
org.apache.hbase
hbase-common
provided
@@ -135,7 +143,8 @@
true
coprocessor
-
+
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index c9be666..d1aee2f 100644
---
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -427,7 +427,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
}
}
-private ByteString serializeGTScanReq(GTScanRequest scanRequest) {
+public static ByteString serializeGTScanReq(GTScanRequest scanRequest) {
ByteString scanRequestByteString;
int scanRequestBufferSize = BytesSerializer.SERIALIZE_BUFFER_SIZE;
while (true) {
@@ -445,7 +445,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
return scanRequestByteString;
}
-private ByteString serializeRawScans(List rawScans) {
+public static ByteString serializeRawScans(List rawScans) {
ByteString rawScanByteString;
int rawScanBufferSize = BytesSerializer.SERIALIZE_BUFFER_SIZE;
while (true) {
diff --git
a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitServiceTest.java
b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitServiceTest.java
new file mode 100644
index 000..b3e5a93
--- /dev/null
+++
b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitServiceTest.java
@@ -0,0 +1,568 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint;
+
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.util.BitSet;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.UUID;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.CoprocessorEnvironment;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import
[kylin] branch master updated: KYLIN-3544 refine guava cache for the recommended cuboids of cube planner
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/master by this push:
new af77cfa KYLIN-3544 refine guava cache for the recommended cuboids of
cube planner
af77cfa is described below
commit af77cfa28b5f2b47b57dce71b00f69115275748c
Author: kyotoYaho
AuthorDate: Wed Dec 19 11:48:57 2018 +0800
KYLIN-3544 refine guava cache for the recommended cuboids of cube planner
---
.../kylin/cube/cuboid/algorithm/CuboidRecommender.java | 13 +++--
1 file changed, 7 insertions(+), 6 deletions(-)
diff --git
a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidRecommender.java
b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidRecommender.java
index 54c6764..baacb51 100644
---
a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidRecommender.java
+++
b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidRecommender.java
@@ -46,7 +46,7 @@ public class CuboidRecommender {
.removalListener(new RemovalListener>() {
@Override
public void onRemoval(RemovalNotification> notification) {
-logger.info("Dict with resource path " +
notification.getKey() + " is removed due to "
+logger.info("Recommended cuboids for cube " +
notification.getKey() + " is removed due to "
+ notification.getCause());
}
}).maximumSize(KylinConfig.getInstanceFromEnv().getCubePlannerRecommendCuboidCacheMaxSize())
@@ -66,8 +66,8 @@ public class CuboidRecommender {
}
public CuboidRecommender() {
-
Broadcaster.getInstance(KylinConfig.getInstanceFromEnv()).registerListener(new
CuboidRecommenderSyncListener(),
-"cube");
+Broadcaster.getInstance(KylinConfig.getInstanceFromEnv())
+.registerStaticListener(new CuboidRecommenderSyncListener(),
"cube", "cube_desc");
}
private static CuboidRecommender instance = new CuboidRecommender();
@@ -94,8 +94,7 @@ public class CuboidRecommender {
Map emptyMap = Maps.newHashMap();
cuboidRecommendCache.put(key, emptyMap);
try {
-Map recommendCuboid =
getRecommendCuboidList(cuboidStats, kylinConfig,
-true);
+Map recommendCuboid =
getRecommendCuboidList(cuboidStats, kylinConfig, true);
if (recommendCuboid != null) {
logger.info(String.format(Locale.ROOT, "Add
recommend cuboids for %s to cache", key));
@@ -105,7 +104,9 @@ public class CuboidRecommender {
return recommendCuboid;
} catch (Exception e) {
cuboidRecommendCache.invalidate(key);
-logger.error(String.format(Locale.ROOT, "Failed to
get recommend cuboids for %s in cache", key), e);
+logger.error(
+String.format(Locale.ROOT, "Failed to get
recommend cuboids for %s in cache", key),
+e);
throw e;
}
}
[kylin] branch master updated: KYLIN-3699 Spark cubing failed when build with empty data
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/master by this push:
new d4ad0fa KYLIN-3699 Spark cubing failed when build with empty data
d4ad0fa is described below
commit d4ad0fa0f11fa5f5b40e77a3631a6b03b3ac7a12
Author: chao long
AuthorDate: Tue Dec 18 16:57:14 2018 +0800
KYLIN-3699 Spark cubing failed when build with empty data
---
.../src/main/java/org/apache/kylin/common/util/HadoopUtil.java| 4 ++--
.../main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git
a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
index 5d09ea7..4afce58 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
@@ -256,8 +256,8 @@ public class HadoopUtil {
if (fileStatuses != null && fileStatuses.length > 0) {
return isSequenceFile(conf, fileStatuses[0].getPath());
+} else {
+return true;
}
-
-return false;
}
}
diff --git
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
index be5bdbf..58f0e66 100644
---
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
+++
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
@@ -299,7 +299,7 @@ public class CubeStatsReader {
Map cuboidSizeMap = getCuboidSizeMap();
double ret = 0;
for (Long cuboidId : layeredCuboids.get(level)) {
-ret += cuboidSizeMap.get(cuboidId);
+ret += cuboidSizeMap.get(cuboidId) == null ? 0.0 :
cuboidSizeMap.get(cuboidId);
}
logger.info("Estimating size for layer {}, all cuboids are {}, total
size is {}", level,
