kylin git commit: KYLIN-1579 IT preparation classes like BuildCubeWithEngine should exit with status code upon build exception

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/yang-m1 bb1b6c3f9 -> 3a8a83b2a


KYLIN-1579 IT preparation classes like BuildCubeWithEngine should exit with 
status code upon build exception


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3a8a83b2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3a8a83b2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3a8a83b2

Branch: refs/heads/yang-m1
Commit: 3a8a83b2a90a7b234ba75e9f227c83164294eca6
Parents: bb1b6c3
Author: Hongbin Ma 
Authored: Tue Apr 12 11:35:54 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Apr 12 11:40:33 2016 +0800

--
 .../kylin/provision/BuildCubeWithEngine.java| 23 +++
 .../kylin/provision/BuildCubeWithStream.java| 24 
 .../kylin/provision/BuildIIWithStream.java  | 21 ++---
 3 files changed, 42 insertions(+), 26 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3a8a83b2/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
index cfefef3..942692a 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
@@ -76,15 +76,20 @@ public class BuildCubeWithEngine {
 private static final Log logger = 
LogFactory.getLog(BuildCubeWithEngine.class);
 
 public static void main(String[] args) throws Exception {
-beforeClass();
-
-BuildCubeWithEngine buildCubeWithEngine = new BuildCubeWithEngine();
-buildCubeWithEngine.before();
-buildCubeWithEngine.build();
-logger.info("Build is done");
-afterClass();
-logger.info("Going to exit");
-System.exit(0);
+try {
+beforeClass();
+
+BuildCubeWithEngine buildCubeWithEngine = new 
BuildCubeWithEngine();
+buildCubeWithEngine.before();
+buildCubeWithEngine.build();
+logger.info("Build is done");
+afterClass();
+logger.info("Going to exit");
+System.exit(0);
+} catch (Exception e) {
+logger.error("error", e);
+System.exit(1);
+}
 }
 
 public static void beforeClass() throws Exception {

http://git-wip-us.apache.org/repos/asf/kylin/blob/3a8a83b2/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
index eeff999..b48fde7 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
@@ -53,15 +53,21 @@ public class BuildCubeWithStream {
 private KylinConfig kylinConfig;
 
 public static void main(String[] args) throws Exception {
-beforeClass();
-BuildCubeWithStream buildCubeWithStream = new BuildCubeWithStream();
-buildCubeWithStream.before();
-buildCubeWithStream.build();
-logger.info("Build is done");
-afterClass();
-logger.info("Going to exit");
-System.exit(0);
-
+try {
+beforeClass();
+
+BuildCubeWithStream buildCubeWithStream = new 
BuildCubeWithStream();
+buildCubeWithStream.before();
+buildCubeWithStream.build();
+logger.info("Build is done");
+afterClass();
+logger.info("Going to exit");
+System.exit(0);
+} catch (Exception e) {
+logger.error("error", e);
+System.exit(1);
+}
+
 }
 
 public static void beforeClass() throws Exception {

http://git-wip-us.apache.org/repos/asf/kylin/blob/3a8a83b2/kylin-it/src/test/java/org/apache/kylin/provision/BuildIIWithStream.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildIIWithStream.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildIIWithStream.java
index 9b7cd14..a77a1c4 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildIIWithStream.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildIIWithStream.java
@@ -93,14 +93,19 @@ public class BuildIIWithStream {
 private KylinConfig kylinConfig;
 
 public static void 

kylin git commit: KYLIN-1578 Coprocessor thread voluntarily stop itself when it reaches timeout

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/yang-m1 3eeabeb51 -> bb1b6c3f9


 KYLIN-1578 Coprocessor thread voluntarily stop itself when it reaches timeout


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bb1b6c3f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bb1b6c3f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bb1b6c3f

Branch: refs/heads/yang-m1
Commit: bb1b6c3f97e36e1b473c0ba33c6544c1debcacee
Parents: 3eeabeb
Author: Hongbin Ma 
Authored: Tue Apr 12 10:32:32 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Apr 12 10:32:32 2016 +0800

--
 .../hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/bb1b6c3f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index 0879661..9dfc607 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -250,10 +250,10 @@ public class CubeVisitService extends 
CubeVisitProtos.CubeVisitService implement
 byte[] compressedAllRows;
 if (normalComplete.booleanValue()) {
 allRows = outputStream.toByteArray();
-compressedAllRows = CompressionUtils.compress(allRows);
 } else {
-compressedAllRows = new byte[0];
+allRows = new byte[0];
 }
+compressedAllRows = CompressionUtils.compress(allRows);
 
 
 appendProfileInfo(sb, "compress done");

kylin git commit: KYLIN-1578 Coprocessor thread voluntarily stop itself when it reaches timeout

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/yang-m1 9513b8e47 -> 3eeabeb51


 KYLIN-1578 Coprocessor thread voluntarily stop itself when it reaches timeout


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3eeabeb5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3eeabeb5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3eeabeb5

Branch: refs/heads/yang-m1
Commit: 3eeabeb519d39fbfd5805b41faaf3d94013f1c29
Parents: 9513b8e
Author: Hongbin Ma 
Authored: Tue Apr 12 10:30:39 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Apr 12 10:30:39 2016 +0800

--
 .../cube/v2/coprocessor/endpoint/CubeVisitService.java| 10 --
 1 file changed, 8 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3eeabeb5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index 596814f..0879661 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -247,8 +247,14 @@ public class CubeVisitService extends 
CubeVisitProtos.CubeVisitService implement
 appendProfileInfo(sb, "agg done");
 
 //outputStream.close() is not necessary
-allRows = outputStream.toByteArray();
-byte[] compressedAllRows = CompressionUtils.compress(allRows);
+byte[] compressedAllRows;
+if (normalComplete.booleanValue()) {
+allRows = outputStream.toByteArray();
+compressedAllRows = CompressionUtils.compress(allRows);
+} else {
+compressedAllRows = new byte[0];
+}
+
 
 appendProfileInfo(sb, "compress done");
 

kylin git commit: revise hdfs filter cache

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/master 4d16e4d0e -> 737ba33b6


revise hdfs filter cache


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/737ba33b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/737ba33b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/737ba33b

Branch: refs/heads/master
Commit: 737ba33b6be4e69ab6533c03e1bb3ba5aeb776a4
Parents: 4d16e4d
Author: Hongbin Ma 
Authored: Mon Apr 11 18:07:20 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Apr 11 18:08:13 2016 +0800

--
 .../cube/v2/filter/MassInValueProviderImpl.java | 57 ++--
 1 file changed, 30 insertions(+), 27 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/737ba33b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/filter/MassInValueProviderImpl.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/filter/MassInValueProviderImpl.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/filter/MassInValueProviderImpl.java
index 16157ec..525645d 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/filter/MassInValueProviderImpl.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/filter/MassInValueProviderImpl.java
@@ -46,7 +46,7 @@ import com.google.common.collect.Sets;
 public class MassInValueProviderImpl implements MassInValueProvider {
 public static final Logger logger = 
LoggerFactory.getLogger(MassInValueProviderImpl.class);
 
-private final static Cache> hdfs_caches 
= CacheBuilder.newBuilder().maximumSize(10).weakValues().removalListener(new 
RemovalListener() {
+private final static Cache> hdfs_caches 
= CacheBuilder.newBuilder().maximumSize(3).removalListener(new 
RemovalListener() {
 @Override
 public void onRemoval(RemovalNotification 
notification) {
 logger.debug(String.valueOf(notification.getCause()));
@@ -64,39 +64,42 @@ public class MassInValueProviderImpl implements 
MassInValueProvider {
 
 FileSystem fileSystem = null;
 try {
-fileSystem = FileSystem.get(HBaseConfiguration.create());
-
-long modificationTime = fileSystem.getFileStatus(new 
Path(filterResourceIdentifier)).getModificationTime();
-Pair cached = 
hdfs_caches.getIfPresent(filterResourceIdentifier);
-if (cached != null && 
cached.getFirst().equals(modificationTime)) {
-ret = cached.getSecond();
-logger.info("Load HDFS from cache using " + 
stopwatch.elapsedMillis() + " millis");
-return;
-}
+synchronized (hdfs_caches) {
+
+fileSystem = FileSystem.get(HBaseConfiguration.create());
+
+long modificationTime = fileSystem.getFileStatus(new 
Path(filterResourceIdentifier)).getModificationTime();
+Pair cached = 
hdfs_caches.getIfPresent(filterResourceIdentifier);
+if (cached != null && 
cached.getFirst().equals(modificationTime)) {
+ret = cached.getSecond();
+logger.info("Load HDFS from cache using " + 
stopwatch.elapsedMillis() + " millis");
+return;
+}
 
-InputStream inputStream = fileSystem.open(new 
Path(filterResourceIdentifier));
-List lines = IOUtils.readLines(inputStream);
+InputStream inputStream = fileSystem.open(new 
Path(filterResourceIdentifier));
+List lines = IOUtils.readLines(inputStream);
 
-logger.info("Load HDFS finished after " + 
stopwatch.elapsedMillis() + " millis");
+logger.info("Load HDFS finished after " + 
stopwatch.elapsedMillis() + " millis");
 
-for (String line : lines) {
-if (StringUtils.isEmpty(line)) {
-continue;
-}
+for (String line : lines) {
+if (StringUtils.isEmpty(line)) {
+continue;
+}
 
-try {
-ByteArray byteArray = 
ByteArray.allocate(encoding.getLengthOfEncoding());
-encoding.encode(line.getBytes(), 
line.getBytes().length, byteArray.array(), 0);
-ret.add(byteArray);
-} catch (Exception e) {
-  

kylin git commit: add cache to hdfs filter

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/yang-m1 36c76427a -> 7036e4b8f


add cache to hdfs filter


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7036e4b8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7036e4b8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7036e4b8

Branch: refs/heads/yang-m1
Commit: 7036e4b8fb594af75ee7127ae8cbbe46ade32514
Parents: 36c7642
Author: Hongbin Ma 
Authored: Mon Apr 11 17:02:52 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Apr 11 17:02:52 2016 +0800

--
 .../kylin/common/util/CacheBuilderTest.java | 43 
 .../cube/v2/filter/MassInValueProviderImpl.java | 40 --
 2 files changed, 79 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7036e4b8/core-common/src/test/java/org/apache/kylin/common/util/CacheBuilderTest.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/util/CacheBuilderTest.java 
b/core-common/src/test/java/org/apache/kylin/common/util/CacheBuilderTest.java
new file mode 100644
index 000..e30f0b3
--- /dev/null
+++ 
b/core-common/src/test/java/org/apache/kylin/common/util/CacheBuilderTest.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.util;
+
+import org.junit.Test;
+
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.RemovalListener;
+import com.google.common.cache.RemovalNotification;
+
+public class CacheBuilderTest {
+@Test
+public void foo() {
+Cache build = 
CacheBuilder.newBuilder().maximumSize(1).weakValues().removalListener(new 
RemovalListener() {
+@Override
+public void onRemoval(RemovalNotification 
notification) {
+System.out.println(notification.getCause());
+}
+}).build();
+
+build.put(1, 1);
+build.put(1, 2);
+build.put(2, 2);
+
+}
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/7036e4b8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/filter/MassInValueProviderImpl.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/filter/MassInValueProviderImpl.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/filter/MassInValueProviderImpl.java
index 83a6671..16157ec 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/filter/MassInValueProviderImpl.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/filter/MassInValueProviderImpl.java
@@ -24,10 +24,12 @@ import java.util.List;
 import java.util.Set;
 
 import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.Pair;
 import org.apache.kylin.dimension.DimensionEncoding;
 import org.apache.kylin.metadata.filter.UDF.MassInValueProvider;
 import org.apache.kylin.metadata.filter.function.Functions;
@@ -35,11 +37,22 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.google.common.base.Stopwatch;
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.RemovalListener;
+import com.google.common.cache.RemovalNotification;
 import com.google.common.collect.Sets;
 
 public class MassInValueProviderImpl implements MassInValueProvider {
 public static final Logger logger = 
LoggerFactory.getLogger(MassInValueProviderImpl.class);
 
+private final static Cache> hdfs_caches 
=