kylin git commit: KYLIN-2377 Add kylin client query timeout

2017-02-07 Thread kangkaisen 
Repository: kylin
Updated Branches:
  refs/heads/master 0dcce15f7 -> 8263752a4


KYLIN-2377 Add kylin client query timeout


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8263752a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8263752a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8263752a

Branch: refs/heads/master
Commit: 8263752a499158342e0588fda851a4006e8b1669
Parents: 0dcce15
Author: kangkaisen 
Authored: Tue Jan 10 14:02:18 2017 +0800
Committer: kangkaisen 
Committed: Wed Feb 8 15:51:30 2017 +0800

--
 .../org/apache/kylin/common/KylinConfigBase.java|  4 
 .../java/org/apache/kylin/cube/CubeInstance.java|  1 +
 .../kylin/metadata/realization/IRealization.java|  5 -
 .../org/apache/kylin/storage/StorageContext.java| 16 +++-
 .../storage/gtrecord/GTCubeStorageQueryBase.java|  2 ++
 .../gtrecord/SequentialCubeTupleIterator.java   |  5 +
 .../apache/kylin/storage/hybrid/HybridInstance.java |  1 +
 7 files changed, 32 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8263752a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index fe15b1e..ebd9dfc 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -877,6 +877,10 @@ abstract public class KylinConfigBase implements 
Serializable {
 return udfMap;
 }
 
+public int getQueryTimeoutSeconds() {
+return 
Integer.parseInt(this.getOptional("kylin.query.timeout-seconds", "0"));
+}
+
 // 

 // SERVER
 // 


http://git-wip-us.apache.org/repos/asf/kylin/blob/8263752a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 1d60575..fb9a7a7 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -178,6 +178,7 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 return sizeRecordSize;
 }
 
+@Override
 public KylinConfig getConfig() {
 return config;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/8263752a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
index a0243f4..aafc0f0 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
@@ -21,6 +21,7 @@ package org.apache.kylin.metadata.realization;
 import java.util.List;
 import java.util.Set;
 
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.metadata.model.ColumnDesc;
 import org.apache.kylin.metadata.model.DataModelDesc;
 import org.apache.kylin.metadata.model.IStorageAware;
@@ -42,7 +43,7 @@ public interface IRealization extends IStorageAware {
 public DataModelDesc getModel();
 
 public Set getAllColumns();
-
+
 public Set getAllColumnDescs();
 
 public List getAllDimensions();
@@ -60,4 +61,6 @@ public interface IRealization extends IStorageAware {
 public long getDateRangeEnd();
 
 public boolean supportsLimitPushDown();
+
+public KylinConfig getConfig();
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/8263752a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java 
b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
index ec46f83..ab0ea73 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
@@ -41,6 +41,7 @@ public class StorageContext {
 private int

kylin git commit: minor, import correct Lists

2017-02-07 Thread liyang 
Repository: kylin
Updated Branches:
  refs/heads/master 39afa5197 -> 0dcce15f7


minor, import correct Lists


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0dcce15f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0dcce15f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0dcce15f

Branch: refs/heads/master
Commit: 0dcce15f7fc01f6d8c388661025e5dab99ef05ad
Parents: 39afa51
Author: Hongbin Ma 
Authored: Wed Feb 8 15:18:10 2017 +0800
Committer: Hongbin Ma 
Committed: Wed Feb 8 15:18:10 2017 +0800

--
 kylin-it/src/test/java/org/apache/kylin/query/H2Database.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/0dcce15f/kylin-it/src/test/java/org/apache/kylin/query/H2Database.java
--
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/H2Database.java 
b/kylin-it/src/test/java/org/apache/kylin/query/H2Database.java
index 8b60c49..78ed1b6 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/H2Database.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/H2Database.java
@@ -29,7 +29,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
-import com.clearspring.analytics.util.Lists;
+import com.google.common.collect.Lists;
 import org.apache.commons.io.IOUtils;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.metadata.MetadataManager;

kylin git commit: KYLIN-2428 add kylin-tools-init-log4j for get-properties usage

2017-02-07 Thread billyliu 
Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2428 6d5ed7ec1 -> f2f0c7eb0


KYLIN-2428 add kylin-tools-init-log4j for get-properties usage


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f2f0c7eb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f2f0c7eb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f2f0c7eb

Branch: refs/heads/KYLIN-2428
Commit: f2f0c7eb0e2c36198569bc2c0b9b016e2e8a50bb
Parents: 6d5ed7e
Author: Billy Liu 
Authored: Wed Feb 8 14:51:43 2017 +0800
Committer: Billy Liu 
Committed: Wed Feb 8 14:51:43 2017 +0800

--
 build/bin/get-properties.sh  |  2 +-
 build/bin/kylin.sh   |  2 +-
 build/conf/kylin-tools-init-log4j.properties | 32 +++
 3 files changed, 34 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f2f0c7eb/build/bin/get-properties.sh
--
diff --git a/build/bin/get-properties.sh b/build/bin/get-properties.sh
index 170442d..4be9740 100755
--- a/build/bin/get-properties.sh
+++ b/build/bin/get-properties.sh
@@ -27,5 +27,5 @@ fi
 
 job_jar=$(ls $KYLIN_HOME/lib/kylin-job-*.jar)
 tool_jar=$(ls $KYLIN_HOME/tool/kylin-tool-*.jar)
-result=`java -cp $job_jar:$tool_jar org.apache.kylin.tool.KylinConfigCLI $1 
2>/dev/null`
+result=`java -cp $job_jar:$tool_jar 
-Dlog4j.configuration=file:${KYLIN_HOME}/conf/kylin-tools-init-log4j.properties 
org.apache.kylin.tool.KylinConfigCLI $1 2>/dev/null`
 echo "$result"

http://git-wip-us.apache.org/repos/asf/kylin/blob/f2f0c7eb/build/bin/kylin.sh
--
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
index abaca62..d95d4ae 100644
--- a/build/bin/kylin.sh
+++ b/build/bin/kylin.sh
@@ -107,7 +107,7 @@ then
 # KYLIN_EXTRA_START_OPTS is for customized settings, checkout bin/setenv.sh
 hbase ${KYLIN_EXTRA_START_OPTS} \
 -Djava.util.logging.manager=org.apache.juli.ClassLoaderLogManager \
--Dlog4j.configuration=kylin-server-log4j.properties \
+
-Dlog4j.configuration=file:${KYLIN_HOME}/conf/kylin-server-log4j.properties \
 -Dorg.apache.tomcat.util.buf.UDecoder.ALLOW_ENCODED_SLASH=true \
 -Dorg.apache.catalina.connector.CoyoteAdapter.ALLOW_BACKSLASH=true \
 -Djava.endorsed.dirs=${tomcat_root}/endorsed  \

http://git-wip-us.apache.org/repos/asf/kylin/blob/f2f0c7eb/build/conf/kylin-tools-init-log4j.properties
--
diff --git a/build/conf/kylin-tools-init-log4j.properties 
b/build/conf/kylin-tools-init-log4j.properties
new file mode 100644
index 000..650745d
--- /dev/null
+++ b/build/conf/kylin-tools-init-log4j.properties
@@ -0,0 +1,32 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#  
+# http://www.apache.org/licenses/LICENSE-2.0
+#  
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+# the kylin-tools-init-log4j.properties is mainly for configuring log 
properties on kylin tools, including:
+#   1. tools launched by get-properties.sh script
+# 
+# It's called kylin-tools-log4j.properties so that it won't distract users 
from the other more important log4j config file: kylin-server-log4j.properties  
+# enable this by -Dlog4j.configuration=kylin-tools-log4j.properties
+
+log4j.rootLogger=INFO,stdout
+
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p [%t %c{1}:%L]: 
%m%n
+
+log4j.logger.org.apache.kylin=ERROR

[5/8] kylin git commit: KYLIN-2434 minor fix

2017-02-07 Thread billyliu 
KYLIN-2434 minor fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/eee9ecbc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/eee9ecbc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/eee9ecbc

Branch: refs/heads/KYLIN-2428
Commit: eee9ecbcb031862ed316f630933bb320cf93a9d1
Parents: f62465c
Author: lidongsjtu 
Authored: Tue Feb 7 17:37:21 2017 +0800
Committer: lidongsjtu 
Committed: Tue Feb 7 17:37:21 2017 +0800

--
 .../java/org/apache/kylin/engine/spark/SparkCubingByLayer.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/eee9ecbc/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
--
diff --git 
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
 
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
index 071806c..317d2bd 100644
--- 
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
+++ 
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
@@ -150,12 +150,12 @@ public class SparkCubingByLayer extends 
AbstractApplication implements Serializa
 setupClasspath(sc, confPath);
 HadoopUtil.deletePath(sc.hadoopConfiguration(), new Path(outputPath));
 
+System.setProperty(KylinConfig.KYLIN_CONF, confPath);
 final KylinConfig envConfig = KylinConfig.getInstanceFromEnv();
 
 HiveContext sqlContext = new HiveContext(sc.sc());
 final DataFrame intermediateTable = 
sqlContext.table(envConfig.getHiveDatabaseForIntermediateTable() + "." + 
hiveTable);
 
-System.setProperty(KylinConfig.KYLIN_CONF, confPath);
 final CubeInstance cubeInstance = 
CubeManager.getInstance(envConfig).getCube(cubeName);
 final CubeDesc cubeDesc = cubeInstance.getDescriptor();
 final CubeSegment cubeSegment = cubeInstance.getSegmentById(segmentId);

[8/8] kylin git commit: KYLIN-2428 cleanup duplicated shaded libraries

2017-02-07 Thread billyliu 
KYLIN-2428 cleanup duplicated shaded libraries


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6d5ed7ec
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6d5ed7ec
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6d5ed7ec

Branch: refs/heads/KYLIN-2428
Commit: 6d5ed7ec16308ec2a2c910710449d0b79e41da84
Parents: 39afa51
Author: Billy Liu 
Authored: Wed Feb 8 14:03:23 2017 +0800
Committer: Billy Liu 
Committed: Wed Feb 8 14:03:23 2017 +0800

--
 atopcalcite/pom.xml |   1 +
 build/bin/kylin.sh  |   4 +-
 core-common/pom.xml |  22 +-
 core-cube/pom.xml   |  18 +-
 .../apache/kylin/gridtable/GTInvertedIndex.java | 223 ---
 .../gridtable/GTInvertedIndexOfColumn.java  | 133 ---
 .../gridtable/SimpleInvertedIndexTest.java  | 196 
 core-job/pom.xml|   9 +
 core-metadata/pom.xml   |  23 ++
 core-storage/pom.xml|  20 ++
 engine-spark/pom.xml|   1 +
 jdbc/pom.xml|  14 +-
 pom.xml | 214 ++
 query/pom.xml   |   6 +
 server/pom.xml  |   4 +
 source-kafka/pom.xml|  16 +-
 storage-hbase/pom.xml   |   4 -
 tool/pom.xml|  33 +++
 18 files changed, 251 insertions(+), 690 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6d5ed7ec/atopcalcite/pom.xml
--
diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml
index 1b327fe..c65c0d1 100644
--- a/atopcalcite/pom.xml
+++ b/atopcalcite/pom.xml
@@ -43,6 +43,7 @@
 
 
 
+
 
 org.apache.calcite.avatica
 avatica

http://git-wip-us.apache.org/repos/asf/kylin/blob/6d5ed7ec/build/bin/kylin.sh
--
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
index 7813b79..abaca62 100644
--- a/build/bin/kylin.sh
+++ b/build/bin/kylin.sh
@@ -38,8 +38,8 @@ function retrieveDependency() {
 source ${dir}/setenv.sh
 fi
 
-export 
HBASE_CLASSPATH_PREFIX=${KYLIN_HOME}/conf:${KYLIN_HOME}/lib/*:${KYLIN_HOME}/tool/*:${KYLIN_HOME}/ext/*:${HBASE_CLASSPATH_PREFIX}
-export HBASE_CLASSPATH=${HBASE_CLASSPATH}:${hive_dependency}
+export 
HBASE_CLASSPATH_PREFIX=${KYLIN_HOME}/conf:${KYLIN_HOME}/lib/*:${KYLIN_HOME}/ext/*:${HBASE_CLASSPATH_PREFIX}
+export 
HBASE_CLASSPATH=${HBASE_CLASSPATH}:${hive_dependency}:${KYLIN_HOME}/tool/*
 if [ -n "$KAFKA_HOME" ]
 then
 source ${dir}/find-kafka-dependency.sh

http://git-wip-us.apache.org/repos/asf/kylin/blob/6d5ed7ec/core-common/pom.xml
--
diff --git a/core-common/pom.xml b/core-common/pom.xml
index 016d470..95d3c29 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -35,26 +35,10 @@
 
 
 
-com.fasterxml.jackson.core
-jackson-databind
-
-
-commons-lang
-commons-lang
-
-
 org.apache.commons
 commons-lang3
 
 
-commons-io
-commons-io
-
-
-commons-cli
-commons-cli
-
-
 org.apache.commons
 commons-email
 
@@ -63,9 +47,11 @@
 httpclient
 
 
-com.google.guava
-guava
+com.fasterxml.jackson.core
+jackson-databind
 
+
+
 
 com.google.code.findbugs
 jsr305

http://git-wip-us.apache.org/repos/asf/kylin/blob/6d5ed7ec/core-cube/pom.xml
--
diff --git a/core-cube/pom.xml b/core-cube/pom.xml
index 0a30432..fe80783 100644
--- a/core-cube/pom.xml
+++ b/core-cube/pom.xml
@@ -48,9 +48,23 @@
 commons-collections
 commons-collections
 
+
+
+
+commons-lang
+commons-lang
+
+
+commons-io
+commons-io
+
+
+com.google.guava
+guava
+
 
-com.n3twork.druid
-extendedset
+com.google.code.findbugs
+jsr305
 
 
 com.esotericsoftware

[4/8] kylin git commit: KYLIN-2434 use kylin.source.hive.database-for-flat-table in spark cubing

2017-02-07 Thread billyliu 
KYLIN-2434 use kylin.source.hive.database-for-flat-table in spark cubing


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f62465cc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f62465cc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f62465cc

Branch: refs/heads/KYLIN-2428
Commit: f62465cc5a5fe0f90142c0606c0c75e8c721bf0e
Parents: b32cc95
Author: lidongsjtu 
Authored: Tue Feb 7 17:31:07 2017 +0800
Committer: lidongsjtu 
Committed: Tue Feb 7 17:31:14 2017 +0800

--
 .../java/org/apache/kylin/engine/spark/SparkCubingByLayer.java  | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f62465cc/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
--
diff --git 
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
 
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
index 8892a73..071806c 100644
--- 
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
+++ 
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
@@ -150,11 +150,12 @@ public class SparkCubingByLayer extends 
AbstractApplication implements Serializa
 setupClasspath(sc, confPath);
 HadoopUtil.deletePath(sc.hadoopConfiguration(), new Path(outputPath));
 
+final KylinConfig envConfig = KylinConfig.getInstanceFromEnv();
+
 HiveContext sqlContext = new HiveContext(sc.sc());
-final DataFrame intermediateTable = sqlContext.table(hiveTable);
+final DataFrame intermediateTable = 
sqlContext.table(envConfig.getHiveDatabaseForIntermediateTable() + "." + 
hiveTable);
 
 System.setProperty(KylinConfig.KYLIN_CONF, confPath);
-final KylinConfig envConfig = KylinConfig.getInstanceFromEnv();
 final CubeInstance cubeInstance = 
CubeManager.getInstance(envConfig).getCube(cubeName);
 final CubeDesc cubeDesc = cubeInstance.getDescriptor();
 final CubeSegment cubeSegment = cubeInstance.getSegmentById(segmentId);

[2/8] kylin git commit: KYLIN-2432 Couldn't select partition column in some old browser

2017-02-07 Thread billyliu 
KYLIN-2432 Couldn't select partition column in some old browser

Signed-off-by: zhongjian 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6d11dd1d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6d11dd1d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6d11dd1d

Branch: refs/heads/KYLIN-2428
Commit: 6d11dd1d27a74fe50cd4fa58d2acf26f728f82e7
Parents: 8581f1d
Author: luguosheng <550175...@qq.com>
Authored: Tue Feb 7 16:54:05 2017 +0800
Committer: zhongjian 
Committed: Tue Feb 7 17:08:45 2017 +0800

--
 webapp/app/js/config.js | 12 +++-
 1 file changed, 11 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6d11dd1d/webapp/app/js/config.js
--
diff --git a/webapp/app/js/config.js b/webapp/app/js/config.js
index 5f72eb2..b9ae6a5 100644
--- a/webapp/app/js/config.js
+++ b/webapp/app/js/config.js
@@ -35,7 +35,17 @@ var Config = {
   },
   contact_mail: ''
 };
-
+//resolve startsWith and endsWidth not work in low version chrome
+if (typeof String.prototype.startsWith != 'function') {
+  String.prototype.startsWith = function (prefix){
+return this.slice(0, prefix.length) === prefix;
+  };
+}
+if (typeof String.prototype.endsWith != 'function') {
+  String.prototype.endsWith = function(suffix) {
+return this.indexOf(suffix, this.length - suffix.length) !== -1;
+  };
+}
 // Angular module to load routes.
 KylinApp.config(function ($routeProvider, $httpProvider, $locationProvider, 
$logProvider) {
 //resolve http always use cache data in IE11,IE10

[6/8] kylin git commit: KYLIN-2430 Get exitCode form FsShell.run in BulkLoadJob

2017-02-07 Thread billyliu 
KYLIN-2430 Get exitCode form FsShell.run in BulkLoadJob


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7b860adb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7b860adb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7b860adb

Branch: refs/heads/KYLIN-2428
Commit: 7b860adb0a23b53d6e2e40337f4317a3a2b067ba
Parents: eee9ecb
Author: kangkaisen 
Authored: Tue Feb 7 17:35:48 2017 +0800
Committer: kangkaisen 
Committed: Tue Feb 7 17:40:39 2017 +0800

--
 .../kylin/storage/hbase/steps/BulkLoadJob.java | 17 -
 1 file changed, 12 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7b860adb/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java
index 1c05767..30616c5 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java
@@ -55,11 +55,18 @@ public class BulkLoadJob extends AbstractHadoopJob {
 
 Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
 FsShell shell = new FsShell(conf);
-try {
-shell.run(new String[] { "-chmod", "-R", "777", input });
-} catch (Exception e) {
-logger.error("Couldn't change the file permissions ", e);
-throw new IOException(e);
+
+int exitCode = -1;
+int retryCount = 10;
+while (exitCode != 0 && retryCount >= 1) {
+exitCode = shell.run(new String[] { "-chmod", "-R", "777", input 
});
+retryCount--;
+Thread.sleep(5000);
+}
+
+if (exitCode != 0) {
+logger.error("Failed to change the file permissions: " + input);
+throw new IOException("Failed to change the file permissions: " + 
input);
 }
 
 String[] newArgs = new String[2];

kylin git commit: KYLIN-2388 Hot load kylin config from web

2017-02-07 Thread kangkaisen 
Repository: kylin
Updated Branches:
  refs/heads/master 7b860adb0 -> 39afa5197


KYLIN-2388 Hot load kylin config from web


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/39afa519
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/39afa519
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/39afa519

Branch: refs/heads/master
Commit: 39afa5197e197d5143843a3bb62c528749b03418
Parents: 7b860ad
Author: kangkaisen 
Authored: Thu Jan 12 14:35:59 2017 +0800
Committer: kangkaisen 
Committed: Tue Feb 7 19:57:29 2017 +0800

--
 .../org/apache/kylin/common/KylinConfig.java|  6 +-
 .../apache/kylin/common/KylinConfigTest.java| 28 -
 .../util/HotLoadKylinPropertiesTestCase.java| 64 
 .../kylin/cube/CubeSpecificConfigTest.java  | 37 ++-
 .../apache/kylin/job/JobEngineConfigTest.java   | 47 ++
 .../kylin/rest/controller/CacheController.java  |  9 ++-
 webapp/app/js/controllers/admin.js  | 26 
 webapp/app/js/services/cache.js |  3 +-
 webapp/app/partials/admin/admin.html|  3 +
 9 files changed, 191 insertions(+), 32 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/39afa519/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
--
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
index 0f40654..c7a18c6 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -201,7 +201,7 @@ public class KylinConfig extends KylinConfigBase {
 return kylinHome + File.separator + "conf";
 }
 
-static File getKylinPropertiesFile() {
+public static File getKylinPropertiesFile() {
 String kylinConfHome = System.getProperty(KYLIN_CONF);
 if (!StringUtils.isEmpty(kylinConfHome)) {
 logger.info("Use KYLIN_CONF=" + kylinConfHome);
@@ -385,4 +385,8 @@ public class KylinConfig extends KylinConfigBase {
 //}
 //logger.info(buf.toString());
 }
+
+public synchronized void hotLoadKylinProperties() {
+reloadKylinConfig(getKylinProperties());
+}
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/39afa519/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java 
b/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
index a426fc6..4d5f130 100644
--- a/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
@@ -26,24 +26,12 @@ import static org.junit.Assert.assertTrue;
 
 import java.util.Map;
 
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.junit.After;
-import org.junit.Before;
+import org.apache.kylin.common.util.HotLoadKylinPropertiesTestCase;
 import org.junit.Test;
 
 import com.google.common.collect.Maps;
 
-public class KylinConfigTest extends LocalFileMetadataTestCase {
-@Before
-public void setUp() throws Exception {
-this.createTestMetadata();
-}
-
-@After
-public void after() throws Exception {
-this.cleanupTestMetadata();
-}
-
+public class KylinConfigTest extends HotLoadKylinPropertiesTestCase{
 @Test
 public void testMRConfigOverride() {
 KylinConfig config = KylinConfig.getInstanceFromEnv();
@@ -78,9 +66,19 @@ public class KylinConfigTest extends 
LocalFileMetadataTestCase {
 KylinConfig config = KylinConfig.getInstanceFromEnv();
 Map override = Maps.newHashMap();
 KylinConfig configExt = KylinConfigExt.createInstance(config, 
override);
-
 assertTrue(config.properties == configExt.properties);
 config.setProperty("1234", "1234");
 assertEquals("1234", configExt.getOptional("1234"));
 }
+
+@Test
+public void testPropertiesHotLoad() {
+KylinConfig config = KylinConfig.getInstanceFromEnv();
+assertEquals("who...@kylin.apache.org", config.getKylinOwner());
+
+updateProperty("kylin.storage.hbase.owner-tag", 
"ky...@kylin.apache.org");
+KylinConfig.getInstanceFromEnv().hotLoadKylinProperties();
+
+assertEquals("ky...@kylin.apache.org", config.getKylinOwner());
+}
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/39afa519/core-common/src/test/java/org/apache/kylin/common/util/HotLoadKylinPropertiesTestCase.java

kylin git commit: KYLIN-2430 Get exitCode form FsShell.run in BulkLoadJob

2017-02-07 Thread kangkaisen 
Repository: kylin
Updated Branches:
  refs/heads/master eee9ecbcb -> 7b860adb0


KYLIN-2430 Get exitCode form FsShell.run in BulkLoadJob


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7b860adb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7b860adb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7b860adb

Branch: refs/heads/master
Commit: 7b860adb0a23b53d6e2e40337f4317a3a2b067ba
Parents: eee9ecb
Author: kangkaisen 
Authored: Tue Feb 7 17:35:48 2017 +0800
Committer: kangkaisen 
Committed: Tue Feb 7 17:40:39 2017 +0800

--
 .../kylin/storage/hbase/steps/BulkLoadJob.java | 17 -
 1 file changed, 12 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7b860adb/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java
index 1c05767..30616c5 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java
@@ -55,11 +55,18 @@ public class BulkLoadJob extends AbstractHadoopJob {
 
 Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
 FsShell shell = new FsShell(conf);
-try {
-shell.run(new String[] { "-chmod", "-R", "777", input });
-} catch (Exception e) {
-logger.error("Couldn't change the file permissions ", e);
-throw new IOException(e);
+
+int exitCode = -1;
+int retryCount = 10;
+while (exitCode != 0 && retryCount >= 1) {
+exitCode = shell.run(new String[] { "-chmod", "-R", "777", input 
});
+retryCount--;
+Thread.sleep(5000);
+}
+
+if (exitCode != 0) {
+logger.error("Failed to change the file permissions: " + input);
+throw new IOException("Failed to change the file permissions: " + 
input);
 }
 
 String[] newArgs = new String[2];

kylin git commit: KYLIN-2434 minor fix

2017-02-07 Thread lidong 
Repository: kylin
Updated Branches:
  refs/heads/master f62465cc5 -> eee9ecbcb


KYLIN-2434 minor fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/eee9ecbc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/eee9ecbc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/eee9ecbc

Branch: refs/heads/master
Commit: eee9ecbcb031862ed316f630933bb320cf93a9d1
Parents: f62465c
Author: lidongsjtu 
Authored: Tue Feb 7 17:37:21 2017 +0800
Committer: lidongsjtu 
Committed: Tue Feb 7 17:37:21 2017 +0800

--
 .../java/org/apache/kylin/engine/spark/SparkCubingByLayer.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/eee9ecbc/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
--
diff --git 
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
 
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
index 071806c..317d2bd 100644
--- 
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
+++ 
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
@@ -150,12 +150,12 @@ public class SparkCubingByLayer extends 
AbstractApplication implements Serializa
 setupClasspath(sc, confPath);
 HadoopUtil.deletePath(sc.hadoopConfiguration(), new Path(outputPath));
 
+System.setProperty(KylinConfig.KYLIN_CONF, confPath);
 final KylinConfig envConfig = KylinConfig.getInstanceFromEnv();
 
 HiveContext sqlContext = new HiveContext(sc.sc());
 final DataFrame intermediateTable = 
sqlContext.table(envConfig.getHiveDatabaseForIntermediateTable() + "." + 
hiveTable);
 
-System.setProperty(KylinConfig.KYLIN_CONF, confPath);
 final CubeInstance cubeInstance = 
CubeManager.getInstance(envConfig).getCube(cubeName);
 final CubeDesc cubeDesc = cubeInstance.getDescriptor();
 final CubeSegment cubeSegment = cubeInstance.getSegmentById(segmentId);

kylin git commit: KYLIN-2434 use kylin.source.hive.database-for-flat-table in spark cubing

2017-02-07 Thread lidong 
Repository: kylin
Updated Branches:
  refs/heads/master b32cc9545 -> f62465cc5


KYLIN-2434 use kylin.source.hive.database-for-flat-table in spark cubing


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f62465cc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f62465cc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f62465cc

Branch: refs/heads/master
Commit: f62465cc5a5fe0f90142c0606c0c75e8c721bf0e
Parents: b32cc95
Author: lidongsjtu 
Authored: Tue Feb 7 17:31:07 2017 +0800
Committer: lidongsjtu 
Committed: Tue Feb 7 17:31:14 2017 +0800

--
 .../java/org/apache/kylin/engine/spark/SparkCubingByLayer.java  | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f62465cc/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
--
diff --git 
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
 
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
index 8892a73..071806c 100644
--- 
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
+++ 
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
@@ -150,11 +150,12 @@ public class SparkCubingByLayer extends 
AbstractApplication implements Serializa
 setupClasspath(sc, confPath);
 HadoopUtil.deletePath(sc.hadoopConfiguration(), new Path(outputPath));
 
+final KylinConfig envConfig = KylinConfig.getInstanceFromEnv();
+
 HiveContext sqlContext = new HiveContext(sc.sc());
-final DataFrame intermediateTable = sqlContext.table(hiveTable);
+final DataFrame intermediateTable = 
sqlContext.table(envConfig.getHiveDatabaseForIntermediateTable() + "." + 
hiveTable);
 
 System.setProperty(KylinConfig.KYLIN_CONF, confPath);
-final KylinConfig envConfig = KylinConfig.getInstanceFromEnv();
 final CubeInstance cubeInstance = 
CubeManager.getInstance(envConfig).getCube(cubeName);
 final CubeDesc cubeDesc = cubeInstance.getDescriptor();
 final CubeSegment cubeSegment = cubeInstance.getSegmentById(segmentId);

kylin git commit: KYLIN-2433 Fix NPE in MergeCuboidMapper

2017-02-07 Thread kangkaisen 
Repository: kylin
Updated Branches:
  refs/heads/master 6d11dd1d2 -> b32cc9545


KYLIN-2433 Fix NPE in MergeCuboidMapper


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b32cc954
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b32cc954
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b32cc954

Branch: refs/heads/master
Commit: b32cc9545a76657570c4a3353469ded43892c772
Parents: 6d11dd1
Author: kangkaisen 
Authored: Fri Feb 3 14:00:50 2017 +0800
Committer: kangkaisen 
Committed: Tue Feb 7 17:20:30 2017 +0800

--
 .../apache/kylin/engine/mr/steps/MergeCuboidMapper.java   | 10 ++
 1 file changed, 10 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b32cc954/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
index 047e2b1..acf1403 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
@@ -129,6 +129,11 @@ public class MergeCuboidMapper extends KylinMapper {
 List columns = 
measureType.getColumnsNeedDictionary(measureDesc.getFunction());
 boolean needReEncode = false;
 for (TblColRef col : columns) {
+//handle the column that all records is null
+if (sourceCubeSegment.getDictionary(col) == null) {
+continue;
+}
+
 if 
(!sourceCubeSegment.getDictionary(col).equals(mergedCubeSegment.getDictionary(col)))
 {
 oldDicts.put(col, sourceCubeSegment.getDictionary(col));
 newDicts.put(col, mergedCubeSegment.getDictionary(col));
@@ -249,6 +254,11 @@ public class MergeCuboidMapper extends KylinMapper {
 }
 
 private Boolean checkNeedMerging(TblColRef col) throws IOException {
+//handle the column that all records is null
+if (sourceCubeSegment.getDictionary(col) == null) {
+return false;
+}
+
 Boolean ret = dimensionsNeedDict.get(col);
 if (ret != null)
 return ret;

[2/2] kylin git commit: KYLIN-2432 Couldn't select partition column in some old browser

2017-02-07 Thread zhongjian 
KYLIN-2432 Couldn't select partition column in some old browser

Signed-off-by: zhongjian 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6d11dd1d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6d11dd1d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6d11dd1d

Branch: refs/heads/master
Commit: 6d11dd1d27a74fe50cd4fa58d2acf26f728f82e7
Parents: 8581f1d
Author: luguosheng <550175...@qq.com>
Authored: Tue Feb 7 16:54:05 2017 +0800
Committer: zhongjian 
Committed: Tue Feb 7 17:08:45 2017 +0800

--
 webapp/app/js/config.js | 12 +++-
 1 file changed, 11 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6d11dd1d/webapp/app/js/config.js
--
diff --git a/webapp/app/js/config.js b/webapp/app/js/config.js
index 5f72eb2..b9ae6a5 100644
--- a/webapp/app/js/config.js
+++ b/webapp/app/js/config.js
@@ -35,7 +35,17 @@ var Config = {
   },
   contact_mail: ''
 };
-
+//resolve startsWith and endsWidth not work in low version chrome
+if (typeof String.prototype.startsWith != 'function') {
+  String.prototype.startsWith = function (prefix){
+return this.slice(0, prefix.length) === prefix;
+  };
+}
+if (typeof String.prototype.endsWith != 'function') {
+  String.prototype.endsWith = function(suffix) {
+return this.indexOf(suffix, this.length - suffix.length) !== -1;
+  };
+}
 // Angular module to load routes.
 KylinApp.config(function ($routeProvider, $httpProvider, $locationProvider, 
$logProvider) {
 //resolve http always use cache data in IE11,IE10

[1/2] kylin git commit: KYLIN-1875 Kylin support SnowFlake schema (alias name rule)

2017-02-07 Thread zhongjian 
Repository: kylin
Updated Branches:
  refs/heads/master 7a6dd1ca3 -> 6d11dd1d2


KYLIN-1875 Kylin support SnowFlake schema (alias name rule)

Signed-off-by: zhongjian 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8581f1d2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8581f1d2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8581f1d2

Branch: refs/heads/master
Commit: 8581f1d203cee9fe3d834d6beb9f6bf7717f921a
Parents: 7a6dd1c
Author: luguosheng <550175...@qq.com>
Authored: Tue Feb 7 16:24:23 2017 +0800
Committer: zhongjian 
Committed: Tue Feb 7 17:08:24 2017 +0800

--
 webapp/app/partials/modelDesigner/data_model.html | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8581f1d2/webapp/app/partials/modelDesigner/data_model.html
--
diff --git a/webapp/app/partials/modelDesigner/data_model.html 
b/webapp/app/partials/modelDesigner/data_model.html
index 4d72266..167c088 100644
--- a/webapp/app/partials/modelDesigner/data_model.html
+++ b/webapp/app/partials/modelDesigner/data_model.html
@@ -157,8 +157,8 @@
 Alias
 
   
-   Alias is invalid.
+ ng-model="newLookup.alias"  
ng-pattern="/^[A-Z_\d]+$/">
+   Table alias is invalid(A Combination 
of numbers, uppercase letters or underscores).