kylin git commit: KYLIN-2105 remove QueryIdGenerator and QueryIdGeneratorTest, simply use UUID as QueryId

[gaodayue]

Repository: kylin
Updated Branches:
  refs/heads/master 4b3047a76 -> 5c1c24a9f


KYLIN-2105 remove QueryIdGenerator and QueryIdGeneratorTest, simply use UUID as 
QueryId


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5c1c24a9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5c1c24a9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5c1c24a9

Branch: refs/heads/master
Commit: 5c1c24a9f7cc5635e0fde5eb89be595f6a07ea25
Parents: 4b3047a
Author: gaodayue 
Authored: Thu Oct 27 12:14:49 2016 +0800
Committer: gaodayue 
Committed: Thu Oct 27 12:16:36 2016 +0800

--
 .../apache/kylin/rest/service/QueryService.java |   8 +-
 .../kylin/rest/util/QueryIdGenerator.java   |  47 
 .../kylin/rest/util/QueryIdGeneratorTest.java   | 108 ---
 .../coprocessor/endpoint/CubeVisitService.java  |   2 +-
 4 files changed, 4 insertions(+), 161 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5c1c24a9/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 94f2dd5..81af044 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -39,6 +39,7 @@ import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.UUID;
 
 import javax.annotation.PostConstruct;
 import javax.sql.DataSource;
@@ -72,7 +73,6 @@ import org.apache.kylin.rest.model.TableMeta;
 import org.apache.kylin.rest.request.PrepareSqlRequest;
 import org.apache.kylin.rest.request.SQLRequest;
 import org.apache.kylin.rest.response.SQLResponse;
-import org.apache.kylin.rest.util.QueryIdGenerator;
 import org.apache.kylin.rest.util.QueryUtil;
 import org.apache.kylin.rest.util.Serializer;
 import org.apache.kylin.rest.util.TableauInterceptor;
@@ -117,8 +117,6 @@ public class QueryService extends BasicService {
 private final String hbaseUrl;
 private final String userTableName;
 
-private QueryIdGenerator queryIdGenerator = new QueryIdGenerator();
-
 @Autowired
 private CacheManager cacheManager;
 
@@ -325,7 +323,7 @@ public class QueryService extends BasicService {
 throw new InternalErrorException("Query is not allowed in " + 
serverMode + " mode.");
 }
 
-final String queryId = 
queryIdGenerator.nextId(sqlRequest.getProject());
+final String queryId = UUID.randomUUID().toString();
 
 Map toggles = new HashMap<>();
 toggles.put(BackdoorToggles.KEY_QUERY_ID, queryId);
@@ -334,7 +332,7 @@ public class QueryService extends BasicService {
 }
 BackdoorToggles.setToggles(toggles);
 
-try (SetThreadName ignored = new SetThreadName("Query-%s", queryId)) {
+try (SetThreadName ignored = new SetThreadName("Query %s", queryId)) {
 String sql = sqlRequest.getSql();
 String project = sqlRequest.getProject();
 logger.info("Using project: " + project);

http://git-wip-us.apache.org/repos/asf/kylin/blob/5c1c24a9/server-base/src/main/java/org/apache/kylin/rest/util/QueryIdGenerator.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/util/QueryIdGenerator.java 
b/server-base/src/main/java/org/apache/kylin/rest/util/QueryIdGenerator.java
deleted file mode 100644
index 2dd19c2..000
--- a/server-base/src/main/java/org/apache/kylin/rest/util/QueryIdGenerator.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.rest.util;
-
-import org.apache.commons.lang3.time.FastDateFormat;
-
-import javax.annotation.Nonnull;

kylin git commit: KYLIN-2116 Remove field delimiter in hive flatten table ddl

[lidong]

Repository: kylin
Updated Branches:
  refs/heads/master 71963cbeb -> 4b3047a76


KYLIN-2116 Remove field delimiter in hive flatten table ddl


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4b3047a7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4b3047a7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4b3047a7

Branch: refs/heads/master
Commit: 4b3047a767aa7a696bf663cab96c11894a1f901e
Parents: 71963cb
Author: lidongsjtu 
Authored: Wed Oct 26 22:49:19 2016 +0800
Committer: lidongsjtu 
Committed: Thu Oct 27 11:15:51 2016 +0800

--
 core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java | 2 --
 1 file changed, 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4b3047a7/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
--
diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java 
b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
index 3cc27ba..9c73a53 100644
--- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
@@ -92,8 +92,6 @@ public class JoinedFlatTable {
 ddl.append(colName(col.getCanonicalName()) + " " + 
getHiveDataType(col.getDatatype()) + "\n");
 }
 ddl.append(")" + "\n");
-
-ddl.append("ROW FORMAT DELIMITED FIELDS TERMINATED BY '\\177'" + "\n");
 ddl.append("STORED AS SEQUENCEFILE" + "\n");
 ddl.append("LOCATION '" + getTableDir(flatDesc, storageDfsDir) + 
"';").append("\n");
 // ddl.append("TBLPROPERTIES ('serialization.null.format'='N')" +

kylin git commit: Add step result check for BuildCubeWithEngine

[shaofengshi]

Repository: kylin
Updated Branches:
  refs/heads/master 3047b53d6 -> 71963cbeb


Add step result check for BuildCubeWithEngine

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/71963cbe
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/71963cbe
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/71963cbe

Branch: refs/heads/master
Commit: 71963cbeb332a4a035d83dc18a2b090b87c15d42
Parents: 3047b53
Author: shaofengshi 
Authored: Thu Oct 27 10:48:10 2016 +0800
Committer: shaofengshi 
Committed: Thu Oct 27 10:48:10 2016 +0800

--
 .../kylin/provision/BuildCubeWithEngine.java| 119 ++-
 1 file changed, 61 insertions(+), 58 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/71963cbe/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
index f6c8801..3ae9bac 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
@@ -149,8 +149,7 @@ public class BuildCubeWithEngine {
 
 }
 
-
-public void after(){
+public void after() {
 DefaultScheduler.destroyInstance();
 }
 
@@ -167,11 +166,11 @@ public class BuildCubeWithEngine {
 KylinConfig.getInstanceFromEnv().setHBaseHFileSizeGB(0.0f);
 }
 
-protected void waitForJob(String jobId) {
+protected ExecutableState waitForJob(String jobId) {
 while (true) {
 AbstractExecutable job = jobService.getJob(jobId);
 if (job.getStatus() == ExecutableState.SUCCEED || job.getStatus() 
== ExecutableState.ERROR) {
-break;
+return job.getStatus();
 } else {
 try {
 Thread.sleep(5000);
@@ -200,17 +199,17 @@ public class BuildCubeWithEngine {
 private void runTestAndAssertSucceed(String[] testCase) throws Exception {
 ExecutorService executorService = 
Executors.newFixedThreadPool(testCase.length);
 final CountDownLatch countDownLatch = new 
CountDownLatch(testCase.length);
-List> tasks = 
Lists.newArrayListWithExpectedSize(testCase.length);
+List tasks = 
Lists.newArrayListWithExpectedSize(testCase.length);
 for (int i = 0; i < testCase.length; i++) {
 tasks.add(executorService.submit(new TestCallable(testCase[i], 
countDownLatch)));
 }
 countDownLatch.await();
 try {
 for (int i = 0; i < tasks.size(); ++i) {
-Future task = tasks.get(i);
-final List jobIds = task.get();
-for (String jobId : jobIds) {
-assertJobSucceed(jobId);
+Future task = tasks.get(i);
+final Boolean result = task.get();
+if (result == false) {
+throw new RuntimeException("The test '" + testCase[i] + "' 
is failed.");
 }
 }
 } catch (Exception ex) {
@@ -219,13 +218,7 @@ public class BuildCubeWithEngine {
 }
 }
 
-private void assertJobSucceed(String jobId) {
-if (jobService.getOutput(jobId).getState() != ExecutableState.SUCCEED) 
{
-throw new RuntimeException("The job '" + jobId + "' is failed.");
-}
-}
-
-private class TestCallable implements Callable {
+private class TestCallable implements Callable {
 
 private final String methodName;
 private final CountDownLatch countDownLatch;
@@ -237,11 +230,11 @@ public class BuildCubeWithEngine {
 
 @SuppressWarnings("unchecked")
 @Override
-public List call() throws Exception {
+public Boolean call() throws Exception {
 try {
 final Method method = 
BuildCubeWithEngine.class.getDeclaredMethod(methodName);
 method.setAccessible(true);
-return (List) method.invoke(BuildCubeWithEngine.this);
+return (Boolean) method.invoke(BuildCubeWithEngine.this);
 } catch (Exception e) {
 logger.error(e.getMessage());
 throw e;
@@ -251,9 +244,13 @@ public class BuildCubeWithEngine {
 }
 }
 
+private void assertJobSuccess() {
+
+}
+
 @SuppressWarnings("unused")
 // called by reflection
-private List testInnerJoinCubeWithSlr() throws Exception {
+private boolean testInnerJoinCubeWithSlr() throws Exception {
 final String cubeName 

[1/3] kylin git commit: KYLIN-1726 update sampleProducer and kylin.sh

[shaofengshi]

Repository: kylin
Updated Branches:
  refs/heads/master c804dc8d7 -> 3047b53d6


KYLIN-1726 update sampleProducer and kylin.sh

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7a793e5c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7a793e5c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7a793e5c

Branch: refs/heads/master
Commit: 7a793e5c33f1f7bffd5a53d64ec92065abd5856a
Parents: c804dc8
Author: shaofengshi 
Authored: Tue Oct 25 18:19:40 2016 +0800
Committer: shaofengshi 
Committed: Thu Oct 27 09:10:16 2016 +0800

--
 build/bin/kylin.sh  |  7 +++--
 .../source/kafka/util/KafkaSampleProducer.java  | 29 
 2 files changed, 22 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7a793e5c/build/bin/kylin.sh
--
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
index 9286055e..ad3a952 100644
--- a/build/bin/kylin.sh
+++ b/build/bin/kylin.sh
@@ -32,7 +32,6 @@ function retrieveDependency() {
 #retrive $hive_dependency and $hbase_dependency
 source ${dir}/find-hive-dependency.sh
 source ${dir}/find-hbase-dependency.sh
-#source ${dir}/find-kafka-dependency.sh
 
 #retrive $KYLIN_EXTRA_START_OPTS
 if [ -f "${dir}/setenv.sh" ]
@@ -41,7 +40,11 @@ function retrieveDependency() {
 
 export 
HBASE_CLASSPATH_PREFIX=${KYLIN_HOME}/conf:${KYLIN_HOME}/lib/*:${KYLIN_HOME}/tool/*:${KYLIN_HOME}/ext/*:${HBASE_CLASSPATH_PREFIX}
 export HBASE_CLASSPATH=${HBASE_CLASSPATH}:${hive_dependency}
-#export HBASE_CLASSPATH=${HBASE_CLASSPATH}:${kafka_dependency}
+if [ -n "$KAFKA_HOME" ]
+then
+source ${dir}/find-kafka-dependency.sh
+export HBASE_CLASSPATH=${HBASE_CLASSPATH}:${kafka_dependency}
+fi
 }
 
 # start command

http://git-wip-us.apache.org/repos/asf/kylin/blob/7a793e5c/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/KafkaSampleProducer.java
--
diff --git 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/KafkaSampleProducer.java
 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/KafkaSampleProducer.java
index 3d26d3d..b8f98aa 100644
--- 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/KafkaSampleProducer.java
+++ 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/KafkaSampleProducer.java
@@ -26,6 +26,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 import java.util.Random;
+import java.util.UUID;
 
 import org.apache.commons.cli.Option;
 import org.apache.commons.cli.OptionBuilder;
@@ -48,7 +49,6 @@ public class KafkaSampleProducer {
 @SuppressWarnings("static-access")
 private static final Option OPTION_TOPIC = 
OptionBuilder.withArgName("topic").hasArg().isRequired(true).withDescription("Kafka
 topic").create("topic");
 private static final Option OPTION_BROKER = 
OptionBuilder.withArgName("broker").hasArg().isRequired(true).withDescription("Kafka
 broker").create("broker");
-private static final Option OPTION_DELAY = 
OptionBuilder.withArgName("delay").hasArg().isRequired(false).withDescription("Simulated
 message delay in mili-seconds, default 0").create("delay");
 private static final Option OPTION_INTERVAL = 
OptionBuilder.withArgName("interval").hasArg().isRequired(false).withDescription("Simulated
 message interval in mili-seconds, default 1000").create("interval");
 
 private static final ObjectMapper mapper = new ObjectMapper();
@@ -60,21 +60,14 @@ public class KafkaSampleProducer {
 String topic, broker;
 options.addOption(OPTION_TOPIC);
 options.addOption(OPTION_BROKER);
-options.addOption(OPTION_DELAY);
-options.addOption(OPTION_INTERVAL);
 optionsHelper.parseOptions(options, args);
 
 logger.info("options: '" + optionsHelper.getOptionsAsString() + "'");
 
 topic = optionsHelper.getOptionValue(OPTION_TOPIC);
 broker = optionsHelper.getOptionValue(OPTION_BROKER);
-long delay = 0;
-String delayString = optionsHelper.getOptionValue(OPTION_DELAY);
-if (delayString != null) {
-delay = Long.parseLong(delayString);
-}
 
-long interval = 1000;
+long interval = 10;
 String intervalString = optionsHelper.getOptionValue(OPTION_INTERVAL);
 if (intervalString != null) {
 interval = Long.parseLong(intervalString);
@@ -101,6 +94,10 @@ public class KafkaSampleProducer {
 devices.add("Andriod");
 devices.add("Other");
 
+List genders = new ArrayList();
+genders.add("Male");
+   

[2/3] kylin git commit: KYLIN-2070 UI changes for allowing concurrent build/refresh/merge

[shaofengshi]

KYLIN-2070 UI changes for allowing concurrent build/refresh/merge

Signed-off-by: shaofengshi 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/efa945e5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/efa945e5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/efa945e5

Branch: refs/heads/master
Commit: efa945e56b1d07648c3f0507831b769d0a018e7e
Parents: 7a793e5
Author: luguosheng <550175...@qq.com>
Authored: Fri Oct 21 11:40:19 2016 +0800
Committer: shaofengshi 
Committed: Thu Oct 27 09:10:23 2016 +0800

--
 webapp/app/js/controllers/cubes.js | 40 -
 webapp/app/js/services/cubes.js|  1 +
 2 files changed, 36 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/efa945e5/webapp/app/js/controllers/cubes.js
--
diff --git a/webapp/app/js/controllers/cubes.js 
b/webapp/app/js/controllers/cubes.js
index 499e93c..f735fb8 100644
--- a/webapp/app/js/controllers/cubes.js
+++ b/webapp/app/js/controllers/cubes.js
@@ -334,12 +334,42 @@ KylinApp.controller('CubesCtrl', function ($scope, $q, 
$routeParams, $location,
   $scope.loadDetail(cube);
   // for streaming cube build tip
   if(cube.streaming){
-$modal.open({
-  templateUrl: 'streamingBuild.html',
-  controller: streamingBuildCtrl,
-  resolve: {
+
+SweetAlert.swal({
+  title: '',
+  text: "Are you sure to start the build?",
+  type: '',
+  showCancelButton: true,
+  confirmButtonColor: '#DD6B55',
+  confirmButtonText: "Yes",
+  closeOnConfirm: true
+}, function(isConfirm) {
+  if(isConfirm){
+loadingRequest.show();
+CubeService.rebuildStreamingCube(
+  {
+cubeId: cube.name
+  },
+  {
+sourceOffsetStart:0,
+sourceOffsetEnd:'9223372036854775807',
+buildType:'BUILD'
+  }, function (job) {
+loadingRequest.hide();
+SweetAlert.swal('Success!', 'Rebuild job was submitted 
successfully', 'success');
+  },function(e){
+
+loadingRequest.hide();
+if(e.data&& e.data.exception){
+  var message =e.data.exception;
+  var msg = !!(message) ? message : 'Failed to take action.';
+  SweetAlert.swal('Oops...', msg, 'error');
+}else{
+  SweetAlert.swal('Oops...', "Failed to take action.", 
'error');
+}
+});
   }
-});
+})
 return;
   }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/efa945e5/webapp/app/js/services/cubes.js
--
diff --git a/webapp/app/js/services/cubes.js b/webapp/app/js/services/cubes.js
index 953f157..de60f55 100644
--- a/webapp/app/js/services/cubes.js
+++ b/webapp/app/js/services/cubes.js
@@ -26,6 +26,7 @@ KylinApp.factory('CubeService', ['$resource', function 
($resource, config) {
 cost: {method: 'PUT', params: {action: 'cost'}, isArray: false},
 rebuildLookUp: {method: 'PUT', params: {propName: 'segs', action: 
'refresh_lookup'}, isArray: false},
 rebuildCube: {method: 'PUT', params: {action: 'rebuild'}, isArray: false},
+rebuildStreamingCube: {method: 'PUT', params: {action: 'build2'}, isArray: 
false},
 disable: {method: 'PUT', params: {action: 'disable'}, isArray: false},
 enable: {method: 'PUT', params: {action: 'enable'}, isArray: false},
 purge: {method: 'PUT', params: {action: 'purge'}, isArray: false},

[3/3] kylin git commit: minor, remove unused entry in kylin.properties

[shaofengshi]

minor, remove unused entry in kylin.properties


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3047b53d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3047b53d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3047b53d

Branch: refs/heads/master
Commit: 3047b53d6fed4ae9a9f455223afc9dfefb758534
Parents: efa945e
Author: shaofengshi 
Authored: Wed Oct 26 11:59:44 2016 +0800
Committer: shaofengshi 
Committed: Thu Oct 27 09:10:31 2016 +0800

--
 build/conf/kylin.properties | 3 ---
 1 file changed, 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3047b53d/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index ed86bdb..33a4e76 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -48,9 +48,6 @@ kylin.metadata.url=kylin_metadata@hbase
 # The storage for final cube file in hbase
 kylin.storage.url=hbase
 
-# In seconds (2 days)
-kylin.storage.cleanup.time.threshold=17280
-
 # Working folder in HDFS, make sure user has the right access to the hdfs 
directory
 kylin.hdfs.working.dir=/kylin
 

[3/3] kylin git commit: port KYLIN-2012 to new interface introduced in KYLIN-2125

[mahongbin]

port KYLIN-2012 to new interface introduced in KYLIN-2125


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b502a174
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b502a174
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b502a174

Branch: refs/heads/yang21
Commit: b502a17414468652b75f84994fb371a767784a47
Parents: 6fc1c86
Author: Hongbin Ma 
Authored: Wed Oct 26 14:04:56 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Oct 26 18:49:34 2016 +0800

--
 .../java/org/apache/kylin/job/DeployUtil.java   |  5 ++-
 .../kylin/source/hive/BeelineHiveClient.java|  6 +--
 .../source/hive/HiveSourceTableLoader.java  | 32 +++
 .../apache/kylin/source/hive/SchemaChecker.java | 41 
 4 files changed, 38 insertions(+), 46 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b502a174/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
--
diff --git a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java 
b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
index 8c64f91..0734f4f 100644
--- a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
+++ b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
@@ -38,8 +38,9 @@ import org.apache.kylin.metadata.MetadataManager;
 import org.apache.kylin.metadata.model.ColumnDesc;
 import org.apache.kylin.metadata.model.TableDesc;
 import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.source.hive.HiveClient;
+import org.apache.kylin.source.hive.HiveClientFactory;
 import org.apache.kylin.source.hive.HiveCmdBuilder;
+import org.apache.kylin.source.hive.IHiveClient;
 import org.apache.kylin.source.kafka.TimedJsonStreamParser;
 import org.apache.maven.model.Model;
 import org.apache.maven.model.io.xpp3.MavenXpp3Reader;
@@ -205,7 +206,7 @@ public class DeployUtil {
 String tableFileDir = temp.getParent();
 temp.delete();
 
-HiveClient hiveClient = new HiveClient();
+IHiveClient hiveClient = HiveClientFactory.getHiveClient();
 // create hive tables
 hiveClient.executeHQL("CREATE DATABASE IF NOT EXISTS EDW");
 
hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_CAL_DT.toUpperCase(;

http://git-wip-us.apache.org/repos/asf/kylin/blob/b502a174/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
index 0fbc39b..b027f2e 100644
--- 
a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
+++ 
b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
@@ -205,9 +205,9 @@ public class BeelineHiveClient implements IHiveClient {
 
 public static void main(String[] args) throws SQLException {
 
-//BeelineHiveClient loader = new BeelineHiveClient("-n root --hiveconf 
hive.security.authorization.sqlstd.confwhitelist.append='mapreduce.job.*|dfs.*' 
-u 'jdbc:hive2://sandbox:1'");
-BeelineHiveClient loader = new 
BeelineHiveClient(StringUtils.join(args, " "));
-HiveTableMeta hiveTableMeta = loader.getHiveTableMeta("default", 
"events");
+BeelineHiveClient loader = new BeelineHiveClient("-n root --hiveconf 
hive.security.authorization.sqlstd.confwhitelist.append='mapreduce.job.*|dfs.*' 
-u 'jdbc:hive2://sandbox:1'");
+//BeelineHiveClient loader = new 
BeelineHiveClient(StringUtils.join(args, " "));
+HiveTableMeta hiveTableMeta = loader.getHiveTableMeta("default", 
"test_kylin_fact_part");
 System.out.println(hiveTableMeta);
 loader.close();
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/b502a174/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
 
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
index 346d278..1335ec3 100644
--- 
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
+++ 
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
@@ -26,6 +26,7 @@ import java.util.Set;
 import java.util.UUID;
 
 import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.cube.CubeManager;
 import org.apache.kylin.engine.mr.HadoopUtil;
 import org.apache.kylin.metadata.MetadataConstants;
 import 

[2/3] kylin git commit: KYLIN-2125 add BeelineHiveClient

[mahongbin]

KYLIN-2125 add BeelineHiveClient


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6fc1c865
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6fc1c865
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6fc1c865

Branch: refs/heads/yang21
Commit: 6fc1c865ae35a86839e226ae2ca7b3f88661c1dd
Parents: d5565f7
Author: Hongbin Ma 
Authored: Thu Oct 13 15:31:11 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Oct 26 18:43:34 2016 +0800

--
 .../kylin/rest/controller/TableController.java  |   9 +-
 source-hive/pom.xml |   5 +
 .../kylin/source/hive/BeelineHiveClient.java| 214 +++
 .../source/hive/BeelineOptionsProcessor.java|  47 
 .../apache/kylin/source/hive/CLIHiveClient.java | 169 +++
 .../apache/kylin/source/hive/HiveClient.java| 170 ---
 .../kylin/source/hive/HiveClientFactory.java|  33 +++
 .../source/hive/HiveSourceTableLoader.java  |  88 
 .../org/apache/kylin/source/hive/HiveTable.java |  16 +-
 .../apache/kylin/source/hive/HiveTableMeta.java |  71 ++
 .../kylin/source/hive/HiveTableMetaBuilder.java | 102 +
 .../apache/kylin/source/hive/HqlExecutable.java | 107 --
 .../apache/kylin/source/hive/IHiveClient.java   |  36 
 .../hive/BeelineOptionsProcessorTest.java   |  38 
 14 files changed, 769 insertions(+), 336 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6fc1c865/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
index eefeba8..b7dd728 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
@@ -47,7 +47,8 @@ import org.apache.kylin.rest.service.KafkaConfigService;
 import org.apache.kylin.rest.service.ModelService;
 import org.apache.kylin.rest.service.ProjectService;
 import org.apache.kylin.rest.service.StreamingService;
-import org.apache.kylin.source.hive.HiveClient;
+import org.apache.kylin.source.hive.HiveClientFactory;
+import org.apache.kylin.source.hive.IHiveClient;
 import org.apache.kylin.source.kafka.config.KafkaConfig;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -189,7 +190,7 @@ public class TableController extends BasicController {
 String[] dbTableName = HadoopUtil.parseHiveTableName(tableName);
 tableName = dbTableName[0] + "." + dbTableName[1];
 TableDesc desc = 
cubeMgmtService.getMetadataManager().getTableDesc(tableName);
-if(desc == null)
+if (desc == null)
 return false;
 tableType = desc.getSourceType();
 
@@ -312,7 +313,7 @@ public class TableController extends BasicController {
 @RequestMapping(value = "/hive", method = { RequestMethod.GET })
 @ResponseBody
 private static List showHiveDatabases() throws IOException {
-HiveClient hiveClient = new HiveClient();
+IHiveClient hiveClient = HiveClientFactory.getHiveClient();
 List results = null;
 
 try {
@@ -333,7 +334,7 @@ public class TableController extends BasicController {
 @RequestMapping(value = "/hive/{database}", method = { RequestMethod.GET })
 @ResponseBody
 private static List showHiveTables(@PathVariable String database) 
throws IOException {
-HiveClient hiveClient = new HiveClient();
+IHiveClient hiveClient = HiveClientFactory.getHiveClient();
 List results = null;
 
 try {

http://git-wip-us.apache.org/repos/asf/kylin/blob/6fc1c865/source-hive/pom.xml
--
diff --git a/source-hive/pom.xml b/source-hive/pom.xml
index 08019d0..16cb3b4 100644
--- a/source-hive/pom.xml
+++ b/source-hive/pom.xml
@@ -65,6 +65,11 @@
 provided
 
 
+org.apache.hive
+hive-jdbc
+provided
+
+
 org.apache.mrunit
 mrunit
 hadoop2

http://git-wip-us.apache.org/repos/asf/kylin/blob/6fc1c865/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
new file mode 100644
index 000..0fbc39b
--- /dev/null
+++ 

[3/3] kylin git commit: KYLIN-2125 add BeelineHiveClient

[mahongbin]

KYLIN-2125 add BeelineHiveClient


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9ed775c3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9ed775c3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9ed775c3

Branch: refs/heads/v1.6.0-rc1
Commit: 9ed775c35426fea47c9c63a37677d73cddb70e9c
Parents: e094b76
Author: Hongbin Ma 
Authored: Thu Oct 13 15:31:11 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Oct 27 08:34:11 2016 +0800

--
 .../kylin/rest/controller/TableController.java  |   9 +-
 source-hive/pom.xml |   5 +
 .../kylin/source/hive/BeelineHiveClient.java| 214 +++
 .../source/hive/BeelineOptionsProcessor.java|  47 
 .../apache/kylin/source/hive/CLIHiveClient.java | 169 +++
 .../apache/kylin/source/hive/HiveClient.java| 170 ---
 .../kylin/source/hive/HiveClientFactory.java|  33 +++
 .../source/hive/HiveSourceTableLoader.java  |  89 
 .../org/apache/kylin/source/hive/HiveTable.java |  16 +-
 .../apache/kylin/source/hive/HiveTableMeta.java |  71 ++
 .../kylin/source/hive/HiveTableMetaBuilder.java | 102 +
 .../apache/kylin/source/hive/HqlExecutable.java | 107 --
 .../apache/kylin/source/hive/IHiveClient.java   |  36 
 .../hive/BeelineOptionsProcessorTest.java   |  38 
 14 files changed, 769 insertions(+), 337 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9ed775c3/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
index d9050c1..47ff3fe 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
@@ -47,7 +47,8 @@ import org.apache.kylin.rest.service.KafkaConfigService;
 import org.apache.kylin.rest.service.ModelService;
 import org.apache.kylin.rest.service.ProjectService;
 import org.apache.kylin.rest.service.StreamingService;
-import org.apache.kylin.source.hive.HiveClient;
+import org.apache.kylin.source.hive.HiveClientFactory;
+import org.apache.kylin.source.hive.IHiveClient;
 import org.apache.kylin.source.kafka.config.KafkaConfig;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -189,7 +190,7 @@ public class TableController extends BasicController {
 String[] dbTableName = HadoopUtil.parseHiveTableName(tableName);
 tableName = dbTableName[0] + "." + dbTableName[1];
 TableDesc desc = 
cubeMgmtService.getMetadataManager().getTableDesc(tableName);
-if(desc == null)
+if (desc == null)
 return false;
 tableType = desc.getSourceType();
 
@@ -315,7 +316,7 @@ public class TableController extends BasicController {
 @RequestMapping(value = "/hive", method = { RequestMethod.GET })
 @ResponseBody
 private static List showHiveDatabases() throws IOException {
-HiveClient hiveClient = new HiveClient();
+IHiveClient hiveClient = HiveClientFactory.getHiveClient();
 List results = null;
 
 try {
@@ -336,7 +337,7 @@ public class TableController extends BasicController {
 @RequestMapping(value = "/hive/{database}", method = { RequestMethod.GET })
 @ResponseBody
 private static List showHiveTables(@PathVariable String database) 
throws IOException {
-HiveClient hiveClient = new HiveClient();
+IHiveClient hiveClient = HiveClientFactory.getHiveClient();
 List results = null;
 
 try {

http://git-wip-us.apache.org/repos/asf/kylin/blob/9ed775c3/source-hive/pom.xml
--
diff --git a/source-hive/pom.xml b/source-hive/pom.xml
index 08019d0..16cb3b4 100644
--- a/source-hive/pom.xml
+++ b/source-hive/pom.xml
@@ -65,6 +65,11 @@
 provided
 
 
+org.apache.hive
+hive-jdbc
+provided
+
+
 org.apache.mrunit
 mrunit
 hadoop2

http://git-wip-us.apache.org/repos/asf/kylin/blob/9ed775c3/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
new file mode 100644
index 000..0fbc39b
--- /dev/null
+++ 

[2/3] kylin git commit: port KYLIN-2068 to BeelineHiveClient(KYLIN-2125)

[mahongbin]

port KYLIN-2068 to BeelineHiveClient(KYLIN-2125)


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/553d7c56
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/553d7c56
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/553d7c56

Branch: refs/heads/v1.6.0-rc1
Commit: 553d7c56a0af49cb1c003f94543b4e600cd6f66e
Parents: 9ed775c
Author: Hongbin Ma 
Authored: Wed Oct 26 13:37:03 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Oct 27 08:34:11 2016 +0800

--
 .../org/apache/kylin/source/hive/BeelineHiveClient.java   | 10 +-
 .../java/org/apache/kylin/source/hive/CLIHiveClient.java  |  4 ++--
 .../apache/kylin/source/hive/HiveSourceTableLoader.java   |  1 +
 .../java/org/apache/kylin/source/hive/HiveTableMeta.java  |  6 --
 4 files changed, 12 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/553d7c56/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
index 0fbc39b..c8d56a5 100644
--- 
a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
+++ 
b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
@@ -115,7 +115,7 @@ public class BeelineHiveClient implements IHiveClient {
 
 List allColumns = 
Lists.newArrayList();
 while (columns.next()) {
-allColumns.add(new 
HiveTableMeta.HiveTableColumnMeta(columns.getString(4), columns.getString(6)));
+allColumns.add(new 
HiveTableMeta.HiveTableColumnMeta(columns.getString(4), columns.getString(6), 
columns.getString(12)));
 }
 builder.setAllColumns(allColumns);
 
@@ -139,7 +139,7 @@ public class BeelineHiveClient implements IHiveClient {
 if ("".equals(resultSet.getString(1).trim())) {
 break;
 }
-partitionColumns.add(new 
HiveTableMeta.HiveTableColumnMeta(resultSet.getString(1).trim(), 
resultSet.getString(2).trim()));
+partitionColumns.add(new 
HiveTableMeta.HiveTableColumnMeta(resultSet.getString(1).trim(), 
resultSet.getString(2).trim(), resultSet.getString(3).trim()));
 }
 builder.setPartitionColumns(partitionColumns);
 }
@@ -205,9 +205,9 @@ public class BeelineHiveClient implements IHiveClient {
 
 public static void main(String[] args) throws SQLException {
 
-//BeelineHiveClient loader = new BeelineHiveClient("-n root --hiveconf 
hive.security.authorization.sqlstd.confwhitelist.append='mapreduce.job.*|dfs.*' 
-u 'jdbc:hive2://sandbox:1'");
-BeelineHiveClient loader = new 
BeelineHiveClient(StringUtils.join(args, " "));
-HiveTableMeta hiveTableMeta = loader.getHiveTableMeta("default", 
"events");
+BeelineHiveClient loader = new BeelineHiveClient("-n root --hiveconf 
hive.security.authorization.sqlstd.confwhitelist.append='mapreduce.job.*|dfs.*' 
-u 'jdbc:hive2://sandbox:1'");
+//BeelineHiveClient loader = new 
BeelineHiveClient(StringUtils.join(args, " "));
+HiveTableMeta hiveTableMeta = loader.getHiveTableMeta("default", 
"test001");
 System.out.println(hiveTableMeta);
 loader.close();
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/553d7c56/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
index ea74470..60cf47a 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
@@ -89,11 +89,11 @@ public class CLIHiveClient implements IHiveClient {
 List allColumns = 
Lists.newArrayList();
 List partitionColumns = 
Lists.newArrayList();
 for (FieldSchema fieldSchema : allFields) {
-allColumns.add(new 
HiveTableMeta.HiveTableColumnMeta(fieldSchema.getName(), 
fieldSchema.getType()));
+allColumns.add(new 
HiveTableMeta.HiveTableColumnMeta(fieldSchema.getName(), fieldSchema.getType(), 
fieldSchema.getComment()));
 }
 if (partitionFields != null && partitionFields.size() > 0) {
 for (FieldSchema fieldSchema : partitionFields) {
-partitionColumns.add(new 
HiveTableMeta.HiveTableColumnMeta(fieldSchema.getName(), 

[1/3] kylin git commit: port KYLIN-2012 to new interface introduced in KYLIN-2125

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/v1.6.0-rc1 e094b7687 -> cd2a06a5d


port KYLIN-2012 to new interface introduced in KYLIN-2125


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cd2a06a5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cd2a06a5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cd2a06a5

Branch: refs/heads/v1.6.0-rc1
Commit: cd2a06a5d373bdd5cfa90e78649d42e891711c43
Parents: 553d7c5
Author: Hongbin Ma 
Authored: Wed Oct 26 14:04:56 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Oct 27 08:34:11 2016 +0800

--
 .../java/org/apache/kylin/job/DeployUtil.java   |  5 ++-
 .../kylin/source/hive/BeelineHiveClient.java|  2 +-
 .../source/hive/HiveSourceTableLoader.java  | 32 +++
 .../apache/kylin/source/hive/SchemaChecker.java | 41 
 4 files changed, 36 insertions(+), 44 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/cd2a06a5/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
--
diff --git a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java 
b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
index 4a24ad2..54feb24 100644
--- a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
+++ b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
@@ -45,8 +45,9 @@ import org.apache.kylin.metadata.model.ColumnDesc;
 import org.apache.kylin.metadata.model.TableDesc;
 import org.apache.kylin.metadata.model.TableRef;
 import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.source.hive.HiveClient;
+import org.apache.kylin.source.hive.HiveClientFactory;
 import org.apache.kylin.source.hive.HiveCmdBuilder;
+import org.apache.kylin.source.hive.IHiveClient;
 import org.apache.kylin.source.kafka.TimedJsonStreamParser;
 import org.apache.maven.model.Model;
 import org.apache.maven.model.io.xpp3.MavenXpp3Reader;
@@ -234,7 +235,7 @@ public class DeployUtil {
 String tableFileDir = temp.getParent();
 temp.delete();
 
-HiveClient hiveClient = new HiveClient();
+IHiveClient hiveClient = HiveClientFactory.getHiveClient();
 // create hive tables
 hiveClient.executeHQL("CREATE DATABASE IF NOT EXISTS EDW");
 
hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_CAL_DT.toUpperCase(;

http://git-wip-us.apache.org/repos/asf/kylin/blob/cd2a06a5/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
index c8d56a5..a84aeb1 100644
--- 
a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
+++ 
b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
@@ -207,7 +207,7 @@ public class BeelineHiveClient implements IHiveClient {
 
 BeelineHiveClient loader = new BeelineHiveClient("-n root --hiveconf 
hive.security.authorization.sqlstd.confwhitelist.append='mapreduce.job.*|dfs.*' 
-u 'jdbc:hive2://sandbox:1'");
 //BeelineHiveClient loader = new 
BeelineHiveClient(StringUtils.join(args, " "));
-HiveTableMeta hiveTableMeta = loader.getHiveTableMeta("default", 
"test001");
+HiveTableMeta hiveTableMeta = loader.getHiveTableMeta("default", 
"test_kylin_fact_part");
 System.out.println(hiveTableMeta);
 loader.close();
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/cd2a06a5/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
 
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
index 388e72b..401e720 100644
--- 
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
+++ 
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
@@ -26,6 +26,7 @@ import java.util.Set;
 import java.util.UUID;
 
 import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.cube.CubeManager;
 import org.apache.kylin.engine.mr.HadoopUtil;
 import org.apache.kylin.metadata.MetadataConstants;
 import org.apache.kylin.metadata.MetadataManager;
@@ -34,8 +35,10 @@ import org.apache.kylin.metadata.model.TableDesc;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.common.collect.LinkedHashMultimap;
 import com.google.common.collect.Lists;
 import 

[2/7] kylin git commit: KYLIN-2030 enhancement

[mahongbin]

KYLIN-2030 enhancement


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/46284bdc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/46284bdc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/46284bdc

Branch: refs/heads/master
Commit: 46284bdca211d59f4a3274f5579dc8fe8a48cc6e
Parents: 1429da5
Author: Hongbin Ma 
Authored: Sat Oct 8 16:14:26 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Oct 27 08:30:13 2016 +0800

--
 .../kylin/cube/CubeCapabilityChecker.java   | 43 +++-
 .../org/apache/kylin/cube/model/CubeDesc.java   |  6 +--
 .../kylin/metadata/model/FunctionDesc.java  |  1 +
 .../test/resources/query/sql_lookup/query06.sql |  1 +
 .../test/resources/query/sql_lookup/query07.sql |  1 +
 5 files changed, 39 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/46284bdc/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
index c8460f0..ee21b1c 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
@@ -25,18 +25,22 @@ import java.util.List;
 import java.util.Set;
 
 import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.measure.MeasureType;
 import org.apache.kylin.measure.basic.BasicMeasureType;
+import org.apache.kylin.metadata.MetadataManager;
 import org.apache.kylin.metadata.filter.UDF.MassInTupleFilter;
+import org.apache.kylin.metadata.model.ColumnDesc;
 import org.apache.kylin.metadata.model.FunctionDesc;
 import org.apache.kylin.metadata.model.IStorageAware;
 import org.apache.kylin.metadata.model.MeasureDesc;
 import org.apache.kylin.metadata.model.ParameterDesc;
+import org.apache.kylin.metadata.model.TableDesc;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.apache.kylin.metadata.realization.CapabilityResult;
-import 
org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence;
 import org.apache.kylin.metadata.realization.SQLDigest;
+import 
org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -66,20 +70,39 @@ public class CubeCapabilityChecker {
 Collection unmatchedAggregations = 
unmatchedAggregations(aggrFunctions, cube);
 
 // try custom measure types
+// in RAW query, unmatchedDimensions and unmatchedAggregations will 
null, so can't chose RAW cube well!
+//if (!unmatchedDimensions.isEmpty() || 
!unmatchedAggregations.isEmpty()) {
 tryCustomMeasureTypes(unmatchedDimensions, unmatchedAggregations, 
digest, cube, result);
+//}
 
-// try dimension-as-measure
-if (!unmatchedAggregations.isEmpty()) {
-if (cube.getDescriptor().getFactTable().equals(digest.factTable)) {
+//more tricks
+if (cube.getDescriptor().getFactTable().equals(digest.factTable)) {
+//for query-on-facttable
+//1. dimension as measure
+
+if (!unmatchedAggregations.isEmpty()) {
 tryDimensionAsMeasures(unmatchedAggregations, digest, cube, 
result, cube.getDescriptor().listDimensionColumnsIncludingDerived());
-} else {
-//deal with query on lookup table, like 
https://issues.apache.org/jira/browse/KYLIN-2030
-if 
(cube.getSegments().get(0).getSnapshots().containsKey(digest.factTable)) {
-Set dimCols = 
Sets.newHashSet(cube.getDataModelDesc().findFirstTable(digest.factTable).getColumns());
+}
+} else {
+//for non query-on-facttable 
+if 
(cube.getSegments().get(0).getSnapshots().containsKey(digest.factTable)) {
+TableDesc tableDesc = 
MetadataManager.getInstance(KylinConfig.getInstanceFromEnv()).getTableDesc(digest.factTable);
+Set dimCols = Sets.newHashSet();
+for (ColumnDesc columnDesc : tableDesc.getColumns()) {
+dimCols.add(columnDesc.getRef());
+}
+
+//1. dimension as measure, like max(cal_dt) or count( distinct 
col) from lookup
+if (!unmatchedAggregations.isEmpty()) {
 tryDimensionAsMeasures(unmatchedAggregations, digest, 
cube, result, dimCols);
-} else {
-logger.info("Skip tryDimensionAsMeasures 

[7/7] kylin git commit: improve integer type upgrade logic

[mahongbin]

improve integer type upgrade logic


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1429da5f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1429da5f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1429da5f

Branch: refs/heads/master
Commit: 1429da5f650852402d1494785368e9175f13c379
Parents: 615e21d
Author: Hongbin Ma 
Authored: Fri Sep 30 18:31:04 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Oct 27 08:30:13 2016 +0800

--
 .../filter/EvaluatableFunctionTupleFilter.java  | 151 ---
 .../metadata/filter/TupleFilterSerializer.java  |  20 ++-
 .../apache/kylin/metadata/model/ColumnDesc.java |  20 ++-
 .../apache/kylin/metadata/tuple/TupleInfo.java  |   4 +-
 .../apache/kylin/query/schema/OLAPTable.java|  31 +++-
 5 files changed, 63 insertions(+), 163 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1429da5f/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableFunctionTupleFilter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableFunctionTupleFilter.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableFunctionTupleFilter.java
deleted file mode 100644
index ff24172..000
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableFunctionTupleFilter.java
+++ /dev/null
@@ -1,151 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *  
- * http://www.apache.org/licenses/LICENSE-2.0
- *  
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.metadata.filter;
-
-import java.lang.reflect.InvocationTargetException;
-import java.nio.ByteBuffer;
-import java.util.Collection;
-import java.util.List;
-
-import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.metadata.datatype.DataType;
-import org.apache.kylin.metadata.datatype.StringSerializer;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
-
-import com.google.common.collect.Lists;
-
-public class EvaluatableFunctionTupleFilter extends BuiltInFunctionTupleFilter 
{
-
-private boolean constantsInitted = false;
-
-//about non-like
-private List values;
-private Object tupleValue;
-
-public EvaluatableFunctionTupleFilter(String name) {
-super(name, FilterOperatorEnum.EVAL_FUNC);
-values = Lists.newArrayListWithCapacity(1);
-values.add(null);
-}
-
-@Override
-public boolean evaluate(IEvaluatableTuple tuple, IFilterCodeSystem cs) {
-
-// extract tuple value
-Object tupleValue = null;
-for (TupleFilter filter : this.children) {
-if (!isConstant(filter)) {
-filter.evaluate(tuple, cs);
-tupleValue = filter.getValues().iterator().next();
-}
-}
-
-TblColRef tblColRef = this.getColumn();
-DataType strDataType = DataType.getType("string");
-if (tblColRef.getType() != strDataType) {
-throw new IllegalStateException("Only String type is allow in 
BuiltInFunction");
-}
-ByteArray valueByteArray = (ByteArray) tupleValue;
-StringSerializer serializer = new StringSerializer(strDataType);
-String value = 
serializer.deserialize(ByteBuffer.wrap(valueByteArray.array(), 
valueByteArray.offset(), valueByteArray.length()));
-
-try {
-if (isLikeFunction()) {
-return (Boolean) invokeFunction(value);
-} else {
-this.tupleValue = invokeFunction(value);
-//convert back to ByteArray format because the outer 
EvaluatableFunctionTupleFilter assumes input as ByteArray
-ByteBuffer buffer = 
ByteBuffer.allocate(valueByteArray.length() * 2);
-serializer.serialize((String) this.tupleValue, buffer);
-this.tupleValue = new ByteArray(buffer.array(), 0, 
buffer.position());
-
-

[5/7] kylin git commit: port KYLIN-2012 to new interface introduced in KYLIN-2125

[mahongbin]

port KYLIN-2012 to new interface introduced in KYLIN-2125


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/615e21d7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/615e21d7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/615e21d7

Branch: refs/heads/master
Commit: 615e21d7fb0dae651ab643949ef5078bef7b12ea
Parents: ca6837d
Author: Hongbin Ma 
Authored: Wed Oct 26 14:04:56 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Oct 27 08:30:13 2016 +0800

--
 .../java/org/apache/kylin/job/DeployUtil.java   |  5 ++-
 .../kylin/source/hive/BeelineHiveClient.java|  2 +-
 .../source/hive/HiveSourceTableLoader.java  | 32 +++
 .../apache/kylin/source/hive/SchemaChecker.java | 41 
 4 files changed, 36 insertions(+), 44 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/615e21d7/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
--
diff --git a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java 
b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
index 4a24ad2..54feb24 100644
--- a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
+++ b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
@@ -45,8 +45,9 @@ import org.apache.kylin.metadata.model.ColumnDesc;
 import org.apache.kylin.metadata.model.TableDesc;
 import org.apache.kylin.metadata.model.TableRef;
 import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.source.hive.HiveClient;
+import org.apache.kylin.source.hive.HiveClientFactory;
 import org.apache.kylin.source.hive.HiveCmdBuilder;
+import org.apache.kylin.source.hive.IHiveClient;
 import org.apache.kylin.source.kafka.TimedJsonStreamParser;
 import org.apache.maven.model.Model;
 import org.apache.maven.model.io.xpp3.MavenXpp3Reader;
@@ -234,7 +235,7 @@ public class DeployUtil {
 String tableFileDir = temp.getParent();
 temp.delete();
 
-HiveClient hiveClient = new HiveClient();
+IHiveClient hiveClient = HiveClientFactory.getHiveClient();
 // create hive tables
 hiveClient.executeHQL("CREATE DATABASE IF NOT EXISTS EDW");
 
hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_CAL_DT.toUpperCase(;

http://git-wip-us.apache.org/repos/asf/kylin/blob/615e21d7/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
index c8d56a5..a84aeb1 100644
--- 
a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
+++ 
b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
@@ -207,7 +207,7 @@ public class BeelineHiveClient implements IHiveClient {
 
 BeelineHiveClient loader = new BeelineHiveClient("-n root --hiveconf 
hive.security.authorization.sqlstd.confwhitelist.append='mapreduce.job.*|dfs.*' 
-u 'jdbc:hive2://sandbox:1'");
 //BeelineHiveClient loader = new 
BeelineHiveClient(StringUtils.join(args, " "));
-HiveTableMeta hiveTableMeta = loader.getHiveTableMeta("default", 
"test001");
+HiveTableMeta hiveTableMeta = loader.getHiveTableMeta("default", 
"test_kylin_fact_part");
 System.out.println(hiveTableMeta);
 loader.close();
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/615e21d7/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
 
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
index 388e72b..401e720 100644
--- 
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
+++ 
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
@@ -26,6 +26,7 @@ import java.util.Set;
 import java.util.UUID;
 
 import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.cube.CubeManager;
 import org.apache.kylin.engine.mr.HadoopUtil;
 import org.apache.kylin.metadata.MetadataConstants;
 import org.apache.kylin.metadata.MetadataManager;
@@ -34,8 +35,10 @@ import org.apache.kylin.metadata.model.TableDesc;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.common.collect.LinkedHashMultimap;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
+import com.google.common.collect.SetMultimap;
 import 

[1/7] kylin git commit: KYLIN-2030 fix with KYLIN-1971

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/master ea3d02ff1 -> c804dc8d7


KYLIN-2030 fix with KYLIN-1971


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c804dc8d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c804dc8d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c804dc8d

Branch: refs/heads/master
Commit: c804dc8d77bb0402d6cd391e582a47f323edbb21
Parents: 324e423
Author: Hongbin Ma 
Authored: Wed Oct 26 14:41:06 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Oct 27 08:30:13 2016 +0800

--
 .../org/apache/kylin/cube/CubeCapabilityChecker.java   | 13 +++--
 .../org/apache/kylin/metadata/model/ColumnDesc.java|  8 +---
 2 files changed, 4 insertions(+), 17 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c804dc8d/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
index e509d98..a3c89e5 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
@@ -25,22 +25,18 @@ import java.util.List;
 import java.util.Set;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.measure.MeasureType;
 import org.apache.kylin.measure.basic.BasicMeasureType;
-import org.apache.kylin.metadata.MetadataManager;
 import org.apache.kylin.metadata.filter.UDF.MassInTupleFilter;
-import org.apache.kylin.metadata.model.ColumnDesc;
 import org.apache.kylin.metadata.model.FunctionDesc;
 import org.apache.kylin.metadata.model.IStorageAware;
 import org.apache.kylin.metadata.model.MeasureDesc;
 import org.apache.kylin.metadata.model.ParameterDesc;
-import org.apache.kylin.metadata.model.TableDesc;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.apache.kylin.metadata.realization.CapabilityResult;
-import org.apache.kylin.metadata.realization.SQLDigest;
 import 
org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence;
+import org.apache.kylin.metadata.realization.SQLDigest;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -86,11 +82,8 @@ public class CubeCapabilityChecker {
 } else {
 //for non query-on-facttable 
 if 
(cube.getSegments().get(0).getSnapshots().containsKey(digest.factTable)) {
-TableDesc tableDesc = 
MetadataManager.getInstance(KylinConfig.getInstanceFromEnv()).getTableDesc(digest.factTable);
-Set dimCols = Sets.newHashSet();
-for (ColumnDesc columnDesc : tableDesc.getColumns()) {
-dimCols.add(columnDesc.getRef());
-}
+
+Set dimCols = 
Sets.newHashSet(cube.getDataModelDesc().findFirstTable(digest.factTable).getColumns());
 
 //1. all aggregations on lookup table can be done. For 
distinct count, mark them all DimensionAsMeasures
 // so that the measure has a chance to be upgraded to 
DimCountDistinctMeasureType in 
org.apache.kylin.metadata.model.FunctionDesc#reInitMeasureType

http://git-wip-us.apache.org/repos/asf/kylin/blob/c804dc8d/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
index a13bd37..e0184b4 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
@@ -26,7 +26,6 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect;
 import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
 
 /**
  * Column Metadata from Source. All name should be uppercase.
@@ -203,11 +202,6 @@ public class ColumnDesc implements Serializable {
 
 @Override
 public String toString() {
-return "ColumnDesc{" +
-"id='" + id + '\'' +
-", name='" + name + '\'' +
-", datatype='" + datatype + '\'' +
-", comment='" + comment + '\'' +
-'}';
+return "ColumnDesc{" + "id='" + id + '\'' + ", name='" + name + '\'' + 

[6/7] kylin git commit: KYLIN-2030 bug fix

[mahongbin]

KYLIN-2030 bug fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/324e4239
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/324e4239
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/324e4239

Branch: refs/heads/master
Commit: 324e4239c308ba5b14a46eb3187dad3186489f92
Parents: 46284bd
Author: Hongbin Ma 
Authored: Sun Oct 9 19:07:52 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Oct 27 08:30:13 2016 +0800

--
 .../kylin/cube/CubeCapabilityChecker.java   | 23 ++--
 .../kylin/query/relnode/OLAPAggregateRel.java   | 22 +++
 2 files changed, 25 insertions(+), 20 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/324e4239/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
index ee21b1c..e509d98 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
@@ -81,7 +81,7 @@ public class CubeCapabilityChecker {
 //1. dimension as measure
 
 if (!unmatchedAggregations.isEmpty()) {
-tryDimensionAsMeasures(unmatchedAggregations, digest, cube, 
result, cube.getDescriptor().listDimensionColumnsIncludingDerived());
+tryDimensionAsMeasures(unmatchedAggregations, result, 
cube.getDescriptor().listDimensionColumnsIncludingDerived());
 }
 } else {
 //for non query-on-facttable 
@@ -92,10 +92,18 @@ public class CubeCapabilityChecker {
 dimCols.add(columnDesc.getRef());
 }
 
-//1. dimension as measure, like max(cal_dt) or count( distinct 
col) from lookup
+//1. all aggregations on lookup table can be done. For 
distinct count, mark them all DimensionAsMeasures
+// so that the measure has a chance to be upgraded to 
DimCountDistinctMeasureType in 
org.apache.kylin.metadata.model.FunctionDesc#reInitMeasureType
 if (!unmatchedAggregations.isEmpty()) {
-tryDimensionAsMeasures(unmatchedAggregations, digest, 
cube, result, dimCols);
+Iterator itr = 
unmatchedAggregations.iterator();
+while (itr.hasNext()) {
+FunctionDesc functionDesc = itr.next();
+if 
(dimCols.containsAll(functionDesc.getParameter().getColRefs())) {
+itr.remove();
+}
+}
 }
+tryDimensionAsMeasures(Lists.newArrayList(aggrFunctions), 
result, dimCols);
 
 //2. more "dimensions" contributed by snapshot
 if (!unmatchedDimensions.isEmpty()) {
@@ -159,19 +167,12 @@ public class CubeCapabilityChecker {
 return result;
 }
 
-private static void tryDimensionAsMeasures(Collection 
unmatchedAggregations, SQLDigest digest, CubeInstance cube, CapabilityResult 
result, Set dimCols) {
-CubeDesc cubeDesc = cube.getDescriptor();
-Collection cubeFuncs = cubeDesc.listAllFunctions();
+private static void tryDimensionAsMeasures(Collection 
unmatchedAggregations, CapabilityResult result, Set dimCols) {
 
 Iterator it = unmatchedAggregations.iterator();
 while (it.hasNext()) {
 FunctionDesc functionDesc = it.next();
 
-if (cubeFuncs.contains(functionDesc)) {
-it.remove();
-continue;
-}
-
 // let calcite handle count
 if (functionDesc.isCount()) {
 it.remove();

http://git-wip-us.apache.org/repos/asf/kylin/blob/324e4239/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java 
b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
index d73206a..62351d3 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
@@ -296,16 +296,20 @@ public class OLAPAggregateRel extends Aggregate 
implements OLAPRel {
 }
 
 private void translateAggregation() {
-// now the realization is known, replace aggregations with what's 
defined on MeasureDesc
-List measures = this.context.realization.getMeasures();
-List newAggrs = 

[3/7] kylin git commit: port KYLIN-2068 to BeelineHiveClient(KYLIN-2125)

[mahongbin]

port KYLIN-2068 to BeelineHiveClient(KYLIN-2125)


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ca6837d0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ca6837d0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ca6837d0

Branch: refs/heads/master
Commit: ca6837d004c0da31c95027b48db589238179f1ed
Parents: e2a932e
Author: Hongbin Ma 
Authored: Wed Oct 26 13:37:03 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Oct 27 08:30:13 2016 +0800

--
 .../org/apache/kylin/source/hive/BeelineHiveClient.java   | 10 +-
 .../java/org/apache/kylin/source/hive/CLIHiveClient.java  |  4 ++--
 .../apache/kylin/source/hive/HiveSourceTableLoader.java   |  1 +
 .../java/org/apache/kylin/source/hive/HiveTableMeta.java  |  6 --
 4 files changed, 12 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ca6837d0/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
index 0fbc39b..c8d56a5 100644
--- 
a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
+++ 
b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
@@ -115,7 +115,7 @@ public class BeelineHiveClient implements IHiveClient {
 
 List allColumns = 
Lists.newArrayList();
 while (columns.next()) {
-allColumns.add(new 
HiveTableMeta.HiveTableColumnMeta(columns.getString(4), columns.getString(6)));
+allColumns.add(new 
HiveTableMeta.HiveTableColumnMeta(columns.getString(4), columns.getString(6), 
columns.getString(12)));
 }
 builder.setAllColumns(allColumns);
 
@@ -139,7 +139,7 @@ public class BeelineHiveClient implements IHiveClient {
 if ("".equals(resultSet.getString(1).trim())) {
 break;
 }
-partitionColumns.add(new 
HiveTableMeta.HiveTableColumnMeta(resultSet.getString(1).trim(), 
resultSet.getString(2).trim()));
+partitionColumns.add(new 
HiveTableMeta.HiveTableColumnMeta(resultSet.getString(1).trim(), 
resultSet.getString(2).trim(), resultSet.getString(3).trim()));
 }
 builder.setPartitionColumns(partitionColumns);
 }
@@ -205,9 +205,9 @@ public class BeelineHiveClient implements IHiveClient {
 
 public static void main(String[] args) throws SQLException {
 
-//BeelineHiveClient loader = new BeelineHiveClient("-n root --hiveconf 
hive.security.authorization.sqlstd.confwhitelist.append='mapreduce.job.*|dfs.*' 
-u 'jdbc:hive2://sandbox:1'");
-BeelineHiveClient loader = new 
BeelineHiveClient(StringUtils.join(args, " "));
-HiveTableMeta hiveTableMeta = loader.getHiveTableMeta("default", 
"events");
+BeelineHiveClient loader = new BeelineHiveClient("-n root --hiveconf 
hive.security.authorization.sqlstd.confwhitelist.append='mapreduce.job.*|dfs.*' 
-u 'jdbc:hive2://sandbox:1'");
+//BeelineHiveClient loader = new 
BeelineHiveClient(StringUtils.join(args, " "));
+HiveTableMeta hiveTableMeta = loader.getHiveTableMeta("default", 
"test001");
 System.out.println(hiveTableMeta);
 loader.close();
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/ca6837d0/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
index ea74470..60cf47a 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
@@ -89,11 +89,11 @@ public class CLIHiveClient implements IHiveClient {
 List allColumns = 
Lists.newArrayList();
 List partitionColumns = 
Lists.newArrayList();
 for (FieldSchema fieldSchema : allFields) {
-allColumns.add(new 
HiveTableMeta.HiveTableColumnMeta(fieldSchema.getName(), 
fieldSchema.getType()));
+allColumns.add(new 
HiveTableMeta.HiveTableColumnMeta(fieldSchema.getName(), fieldSchema.getType(), 
fieldSchema.getComment()));
 }
 if (partitionFields != null && partitionFields.size() > 0) {
 for (FieldSchema fieldSchema : partitionFields) {
-partitionColumns.add(new 
HiveTableMeta.HiveTableColumnMeta(fieldSchema.getName(), 

[4/7] kylin git commit: KYLIN-2125 add BeelineHiveClient

[mahongbin]

KYLIN-2125 add BeelineHiveClient


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e2a932e7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e2a932e7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e2a932e7

Branch: refs/heads/master
Commit: e2a932e7439d0034b26b99897d7c293cadb1b8e3
Parents: ea3d02f
Author: Hongbin Ma 
Authored: Thu Oct 13 15:31:11 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Oct 27 08:30:13 2016 +0800

--
 .../kylin/rest/controller/TableController.java  |   9 +-
 source-hive/pom.xml |   5 +
 .../kylin/source/hive/BeelineHiveClient.java| 214 +++
 .../source/hive/BeelineOptionsProcessor.java|  47 
 .../apache/kylin/source/hive/CLIHiveClient.java | 169 +++
 .../apache/kylin/source/hive/HiveClient.java| 170 ---
 .../kylin/source/hive/HiveClientFactory.java|  33 +++
 .../source/hive/HiveSourceTableLoader.java  |  89 
 .../org/apache/kylin/source/hive/HiveTable.java |  16 +-
 .../apache/kylin/source/hive/HiveTableMeta.java |  71 ++
 .../kylin/source/hive/HiveTableMetaBuilder.java | 102 +
 .../apache/kylin/source/hive/HqlExecutable.java | 107 --
 .../apache/kylin/source/hive/IHiveClient.java   |  36 
 .../hive/BeelineOptionsProcessorTest.java   |  38 
 14 files changed, 769 insertions(+), 337 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e2a932e7/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
index d9050c1..47ff3fe 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
@@ -47,7 +47,8 @@ import org.apache.kylin.rest.service.KafkaConfigService;
 import org.apache.kylin.rest.service.ModelService;
 import org.apache.kylin.rest.service.ProjectService;
 import org.apache.kylin.rest.service.StreamingService;
-import org.apache.kylin.source.hive.HiveClient;
+import org.apache.kylin.source.hive.HiveClientFactory;
+import org.apache.kylin.source.hive.IHiveClient;
 import org.apache.kylin.source.kafka.config.KafkaConfig;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -189,7 +190,7 @@ public class TableController extends BasicController {
 String[] dbTableName = HadoopUtil.parseHiveTableName(tableName);
 tableName = dbTableName[0] + "." + dbTableName[1];
 TableDesc desc = 
cubeMgmtService.getMetadataManager().getTableDesc(tableName);
-if(desc == null)
+if (desc == null)
 return false;
 tableType = desc.getSourceType();
 
@@ -315,7 +316,7 @@ public class TableController extends BasicController {
 @RequestMapping(value = "/hive", method = { RequestMethod.GET })
 @ResponseBody
 private static List showHiveDatabases() throws IOException {
-HiveClient hiveClient = new HiveClient();
+IHiveClient hiveClient = HiveClientFactory.getHiveClient();
 List results = null;
 
 try {
@@ -336,7 +337,7 @@ public class TableController extends BasicController {
 @RequestMapping(value = "/hive/{database}", method = { RequestMethod.GET })
 @ResponseBody
 private static List showHiveTables(@PathVariable String database) 
throws IOException {
-HiveClient hiveClient = new HiveClient();
+IHiveClient hiveClient = HiveClientFactory.getHiveClient();
 List results = null;
 
 try {

http://git-wip-us.apache.org/repos/asf/kylin/blob/e2a932e7/source-hive/pom.xml
--
diff --git a/source-hive/pom.xml b/source-hive/pom.xml
index 08019d0..16cb3b4 100644
--- a/source-hive/pom.xml
+++ b/source-hive/pom.xml
@@ -65,6 +65,11 @@
 provided
 
 
+org.apache.hive
+hive-jdbc
+provided
+
+
 org.apache.mrunit
 mrunit
 hadoop2

http://git-wip-us.apache.org/repos/asf/kylin/blob/e2a932e7/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
new file mode 100644
index 000..0fbc39b
--- /dev/null
+++ 

kylin git commit: KYLIN-1971 TblColRef != ColumnDesc, turns on

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/master 69213e999 -> ea3d02ff1


KYLIN-1971 TblColRef != ColumnDesc, turns on


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ea3d02ff
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ea3d02ff
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ea3d02ff

Branch: refs/heads/master
Commit: ea3d02ff16fe96b0865b81d60063adbab3e462d2
Parents: 69213e9
Author: Li Yang 
Authored: Wed Oct 26 18:24:15 2016 +0800
Committer: Yang Li 
Committed: Thu Oct 27 08:15:44 2016 +0800

--
 .../org/apache/kylin/cube/CubeInstance.java |   6 +
 .../org/apache/kylin/cube/model/CubeDesc.java   |  14 ++-
 .../apache/kylin/metadata/model/JoinDesc.java   |   2 +-
 .../apache/kylin/metadata/model/TblColRef.java  |   4 +-
 .../metadata/realization/IRealization.java  |   3 +
 .../kylin/storage/hybrid/HybridInstance.java|  17 +++
 .../apache/kylin/query/ITKylinQueryTest.java|   2 +-
 .../relnode/OLAPToEnumerableConverter.java  |  53 +
 .../kylin/query/routing/ModelChooser.java   | 111 ---
 .../common/coprocessor/CoprocessorRowType.java  |   3 +
 .../v1/coprocessor/observer/RowTypeTest.java|   5 +-
 11 files changed, 146 insertions(+), 74 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ea3d02ff/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 2ccdde7..6bd3826 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -28,6 +28,7 @@ import org.apache.kylin.common.KylinConfigExt;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.common.persistence.RootPersistentEntity;
 import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.metadata.model.ColumnDesc;
 import org.apache.kylin.metadata.model.DataModelDesc;
 import org.apache.kylin.metadata.model.IBuildable;
 import org.apache.kylin.metadata.model.LookupDesc;
@@ -395,6 +396,11 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 }
 
 @Override
+public Set getAllColumnDescs() {
+return getDescriptor().listAllColumnDescs();
+}
+
+@Override
 public long getDateRangeStart() {
 List readySegs = getSegments(SegmentStatusEnum.READY);
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/ea3d02ff/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 5c73f21..0c0b4f3 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -54,6 +54,7 @@ import org.apache.kylin.measure.MeasureType;
 import org.apache.kylin.measure.extendedcolumn.ExtendedColumnMeasureType;
 import org.apache.kylin.metadata.MetadataConstants;
 import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.ColumnDesc;
 import org.apache.kylin.metadata.model.DataModelDesc;
 import org.apache.kylin.metadata.model.FunctionDesc;
 import org.apache.kylin.metadata.model.IEngineAware;
@@ -159,8 +160,9 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 @JsonProperty("override_kylin_properties")
 private LinkedHashMap overrideKylinProps = new 
LinkedHashMap();
 
-private LinkedHashSet allColumns = new 
LinkedHashSet();
-private LinkedHashSet dimensionColumns = new 
LinkedHashSet();
+private LinkedHashSet allColumns = new LinkedHashSet<>();
+private LinkedHashSet allColumnDescs = new LinkedHashSet<>();
+private LinkedHashSet dimensionColumns = new LinkedHashSet<>();
 
 private Map derivedToHostMap = Maps.newHashMap();
 private Map hostToDerivedMap = 
Maps.newHashMap();
@@ -191,6 +193,10 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 public Set listAllColumns() {
 return allColumns;
 }
+
+public Set listAllColumnDescs() {
+return allColumnDescs;
+}
 
 /**
  * @return dimension columns including derived, BUT NOT measures
@@ -549,6 +555,10 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 checkState(rowkey.getRowKeyColumns().length == 

kylin git commit: tmp

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-1971 293eb85b6 -> ab23cc242


tmp


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ab23cc24
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ab23cc24
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ab23cc24

Branch: refs/heads/KYLIN-1971
Commit: ab23cc2424c488d860c4727abd2a187aefd7d6e5
Parents: 293eb85
Author: Yang Li 
Authored: Thu Oct 27 07:58:16 2016 +0800
Committer: Yang Li 
Committed: Thu Oct 27 07:58:16 2016 +0800

--
 .../storage/hbase/common/coprocessor/CoprocessorRowType.java| 3 +++
 .../storage/hbase/cube/v1/coprocessor/observer/RowTypeTest.java | 5 -
 2 files changed, 7 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ab23cc24/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorRowType.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorRowType.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorRowType.java
index f1e26f5..7314fc4 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorRowType.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorRowType.java
@@ -73,6 +73,7 @@ public class CoprocessorRowType {
 for (int i = 0; i < n; i++) {
 BytesUtil.writeAsciiString(o.columns[i].getTable(), out);
 BytesUtil.writeAsciiString(o.columns[i].getName(), out);
+BytesUtil.writeAsciiString(o.columns[i].getDatatype(), out);
 BytesUtil.writeVInt(o.columnSizes[i], out);
 }
 }
@@ -86,11 +87,13 @@ public class CoprocessorRowType {
 for (int i = 0; i < n; i++) {
 String tableName = BytesUtil.readAsciiString(in);
 String colName = BytesUtil.readAsciiString(in);
+String datatype = BytesUtil.readAsciiString(in);
 TableDesc table = new TableDesc();
 table.setName(tableName);
 ColumnDesc col = new ColumnDesc();
 col.setTable(table);
 col.setName(colName);
+col.setDatatype(datatype);
 col.init(table);
 cols[i] = col.getRef();
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/ab23cc24/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/RowTypeTest.java
--
diff --git 
a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/RowTypeTest.java
 
b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/RowTypeTest.java
index ba4b86a..364be9a 100644
--- 
a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/RowTypeTest.java
+++ 
b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/RowTypeTest.java
@@ -18,6 +18,7 @@
 
 package org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer;
 
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
 import java.util.Arrays;
@@ -60,7 +61,9 @@ public class RowTypeTest extends LocalFileMetadataTestCase {
 byte[] bytes = CoprocessorRowType.serialize(rowType);
 CoprocessorRowType copy = CoprocessorRowType.deserialize(bytes);
 
-assertTrue(Arrays.equals(rowType.columns, copy.columns));
 assertTrue(Arrays.equals(rowType.columnSizes, copy.columnSizes));
+for (int i = 0; i < rowType.columns.length; i++) {
+assertEquals(rowType.columns[i].getColumnDesc(), 
copy.columns[i].getColumnDesc());
+}
 }
 }

[2/3] kylin git commit: minor, fix concurrent issue with CubeSegment.cuboidBaseShards

[liyang]

minor, fix concurrent issue with CubeSegment.cuboidBaseShards


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/69213e99
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/69213e99
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/69213e99

Branch: refs/heads/KYLIN-1971
Commit: 69213e9991a1a69b26803852193a78628246b127
Parents: 577052b
Author: gaodayue 
Authored: Wed Oct 26 21:39:11 2016 +0800
Committer: gaodayue 
Committed: Wed Oct 26 21:39:11 2016 +0800

--
 .../src/main/java/org/apache/kylin/cube/CubeSegment.java  | 7 +--
 1 file changed, 5 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/69213e99/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
index fdf1fb0..b3d6d6b 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
@@ -49,6 +49,8 @@ import 
com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 
+import javax.annotation.concurrent.GuardedBy;
+
 @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = 
Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = 
Visibility.NONE)
 public class CubeSegment implements Comparable, IBuildable, 
ISegment {
 
@@ -115,7 +117,8 @@ public class CubeSegment implements 
Comparable, IBuildable, ISegmen
 @JsonInclude(JsonInclude.Include.NON_EMPTY)
 private Map additionalInfo = new LinkedHashMap();
 
-private volatile Map cuboidBaseShards = 
Maps.newHashMap();//cuboid id ==> base(starting) shard for this cuboid
+@GuardedBy("this")
+private Map cuboidBaseShards = Maps.newHashMap(); // cuboid 
id ==> base(starting) shard for this cuboid
 
 public CubeDesc getCubeDesc() {
 return getCubeInstance().getDescriptor();
@@ -516,7 +519,7 @@ public class CubeSegment implements 
Comparable, IBuildable, ISegmen
 this.totalShards = totalShards;
 }
 
-public short getCuboidBaseShard(Long cuboidId) {
+public synchronized short getCuboidBaseShard(Long cuboidId) {
 if (totalShards > 0) {
 //shard squashed case
 

[1/3] kylin git commit: KYLIN-2127 UI bug fix for Extend Column [Forced Update!]

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-1971 8974b99af -> 293eb85b6 (forced update)


KYLIN-2127 UI bug fix for Extend Column

Signed-off-by: Jason 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/577052bd
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/577052bd
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/577052bd

Branch: refs/heads/KYLIN-1971
Commit: 577052bd2246ec1feb02771f11ca2642a8076c4b
Parents: 5cdd225
Author: kangkaisen 
Authored: Tue Oct 25 22:27:54 2016 +0800
Committer: Jason 
Committed: Wed Oct 26 19:13:45 2016 +0800

--
 webapp/app/js/controllers/cubeMeasures.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/577052bd/webapp/app/js/controllers/cubeMeasures.js
--
diff --git a/webapp/app/js/controllers/cubeMeasures.js 
b/webapp/app/js/controllers/cubeMeasures.js
index dcf26b7..938e918 100644
--- a/webapp/app/js/controllers/cubeMeasures.js
+++ b/webapp/app/js/controllers/cubeMeasures.js
@@ -229,7 +229,7 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, 
$modal,MetaModel,cubes
 $scope.nextPara = {
   "type":"column",
   "value":"",
-  "next_parameter":{}
+  "next_parameter": null
 }
 if($scope.newMeasure){
   $scope.newMeasure.function.parameter.next_parameter = null;

[3/3] kylin git commit: TblColRef != ColumnDesc, turns on

[liyang]

TblColRef != ColumnDesc, turns on


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/293eb85b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/293eb85b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/293eb85b

Branch: refs/heads/KYLIN-1971
Commit: 293eb85b63869f25d356309b28a0b6a3a55aad06
Parents: 69213e9
Author: Li Yang 
Authored: Wed Oct 26 18:24:15 2016 +0800
Committer: Yang Li 
Committed: Thu Oct 27 07:15:53 2016 +0800

--
 .../org/apache/kylin/cube/CubeInstance.java |   6 +
 .../org/apache/kylin/cube/model/CubeDesc.java   |  14 ++-
 .../apache/kylin/metadata/model/JoinDesc.java   |   2 +-
 .../apache/kylin/metadata/model/TblColRef.java  |   4 +-
 .../metadata/realization/IRealization.java  |   3 +
 .../kylin/storage/hybrid/HybridInstance.java|  17 +++
 .../apache/kylin/query/ITKylinQueryTest.java|   2 +-
 .../relnode/OLAPToEnumerableConverter.java  |  53 +
 .../kylin/query/routing/ModelChooser.java   | 111 ---
 9 files changed, 139 insertions(+), 73 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/293eb85b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 2ccdde7..6bd3826 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -28,6 +28,7 @@ import org.apache.kylin.common.KylinConfigExt;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.common.persistence.RootPersistentEntity;
 import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.metadata.model.ColumnDesc;
 import org.apache.kylin.metadata.model.DataModelDesc;
 import org.apache.kylin.metadata.model.IBuildable;
 import org.apache.kylin.metadata.model.LookupDesc;
@@ -395,6 +396,11 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 }
 
 @Override
+public Set getAllColumnDescs() {
+return getDescriptor().listAllColumnDescs();
+}
+
+@Override
 public long getDateRangeStart() {
 List readySegs = getSegments(SegmentStatusEnum.READY);
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/293eb85b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 5c73f21..0c0b4f3 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -54,6 +54,7 @@ import org.apache.kylin.measure.MeasureType;
 import org.apache.kylin.measure.extendedcolumn.ExtendedColumnMeasureType;
 import org.apache.kylin.metadata.MetadataConstants;
 import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.ColumnDesc;
 import org.apache.kylin.metadata.model.DataModelDesc;
 import org.apache.kylin.metadata.model.FunctionDesc;
 import org.apache.kylin.metadata.model.IEngineAware;
@@ -159,8 +160,9 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 @JsonProperty("override_kylin_properties")
 private LinkedHashMap overrideKylinProps = new 
LinkedHashMap();
 
-private LinkedHashSet allColumns = new 
LinkedHashSet();
-private LinkedHashSet dimensionColumns = new 
LinkedHashSet();
+private LinkedHashSet allColumns = new LinkedHashSet<>();
+private LinkedHashSet allColumnDescs = new LinkedHashSet<>();
+private LinkedHashSet dimensionColumns = new LinkedHashSet<>();
 
 private Map derivedToHostMap = Maps.newHashMap();
 private Map hostToDerivedMap = 
Maps.newHashMap();
@@ -191,6 +193,10 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 public Set listAllColumns() {
 return allColumns;
 }
+
+public Set listAllColumnDescs() {
+return allColumnDescs;
+}
 
 /**
  * @return dimension columns including derived, BUT NOT measures
@@ -549,6 +555,10 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 checkState(rowkey.getRowKeyColumns().length == dimCols.size(), "RowKey 
columns count (%d) doesn't match dimensions columns count (%d)", 
rowkey.getRowKeyColumns().length, dimCols.size());
 
 initDictionaryDesc();
+
+for 

kylin git commit: KYLIN-2116 Remove field delimiter in hive flatten table ddl

[lidong]

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2116 [created] 56a84c27c


KYLIN-2116 Remove field delimiter in hive flatten table ddl


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/56a84c27
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/56a84c27
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/56a84c27

Branch: refs/heads/KYLIN-2116
Commit: 56a84c27cda3e0600d8676937ee7500f5317d1f2
Parents: 69213e9
Author: lidongsjtu 
Authored: Wed Oct 26 22:49:19 2016 +0800
Committer: lidongsjtu 
Committed: Wed Oct 26 22:49:19 2016 +0800

--
 core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java | 2 --
 1 file changed, 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/56a84c27/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
--
diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java 
b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
index 3cc27ba..9c73a53 100644
--- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
@@ -92,8 +92,6 @@ public class JoinedFlatTable {
 ddl.append(colName(col.getCanonicalName()) + " " + 
getHiveDataType(col.getDatatype()) + "\n");
 }
 ddl.append(")" + "\n");
-
-ddl.append("ROW FORMAT DELIMITED FIELDS TERMINATED BY '\\177'" + "\n");
 ddl.append("STORED AS SEQUENCEFILE" + "\n");
 ddl.append("LOCATION '" + getTableDir(flatDesc, storageDfsDir) + 
"';").append("\n");
 // ddl.append("TBLPROPERTIES ('serialization.null.format'='N')" +

kylin git commit: ITQuery pass

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-1971 c1e6ff680 -> 8974b99af


ITQuery pass


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8974b99a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8974b99a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8974b99a

Branch: refs/heads/KYLIN-1971
Commit: 8974b99afb4b1f18f33ea8b533925a9e73a736ca
Parents: c1e6ff6
Author: Yang Li 
Authored: Wed Oct 26 21:58:02 2016 +0800
Committer: Yang Li 
Committed: Wed Oct 26 21:58:02 2016 +0800

--
 .../relnode/OLAPToEnumerableConverter.java  | 41 
 1 file changed, 17 insertions(+), 24 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8974b99a/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
 
b/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
index 24fc430..1233422 100644
--- 
a/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
+++ 
b/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
@@ -48,7 +48,6 @@ import 
org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.apache.kylin.metadata.realization.IRealization;
 import org.apache.kylin.query.routing.ModelChooser;
-import org.apache.kylin.query.routing.NoRealizationFoundException;
 import org.apache.kylin.query.routing.QueryRouter;
 import org.apache.kylin.query.schema.OLAPTable;
 
@@ -79,30 +78,23 @@ public class OLAPToEnumerableConverter extends 
ConverterImpl implements Enumerab
 OLAPRel.OLAPImplementor olapImplementor = new 
OLAPRel.OLAPImplementor();
 olapImplementor.visitChild(getInput(), this);
 
-// find cube from olap context and apply cell level security
-try {
-for (OLAPContext context : OLAPContext.getThreadLocalContexts()) {
-// Context has no table scan is created by OLAPJoinRel which 
looks like
-// (sub-query) as A join (sub-query) as B
-// No realization needed for such context.
-if (context.firstTableScan == null) {
-continue;
-}
-
-Set candidates = 
ModelChooser.selectModel(context);
-IRealization realization = 
QueryRouter.selectRealization(context, candidates);
-context.realization = realization;
-
-doAccessControl(context);
-}
-} catch (NoRealizationFoundException e) {
-OLAPContext ctx0 = (OLAPContext) 
OLAPContext.getThreadLocalContexts().toArray()[0];
-if (ctx0 != null && ctx0.olapSchema.hasStarSchemaUrl()) {
-// generate hive result
-return buildHiveResult(enumImplementor, pref, ctx0);
-} else {
-throw e;
+// identify cube from olap context
+int contextCount = OLAPContext.getThreadLocalContexts().size();
+for (int i = contextCount - 1; i >= 0; i--) {
+OLAPContext context = OLAPContext.getThreadLocalContextById(i);
+
+// Context has no table scan is created by OLAPJoinRel which looks 
like
+// (sub-query) as A join (sub-query) as B
+// No realization needed for such context.
+if (context.firstTableScan == null) {
+continue;
 }
+
+Set candidates = ModelChooser.selectModel(context);
+IRealization realization = QueryRouter.selectRealization(context, 
candidates);
+context.realization = realization;
+
+doAccessControl(context);
 }
 
 // rewrite query if necessary
@@ -176,6 +168,7 @@ public class OLAPToEnumerableConverter extends 
ConverterImpl implements Enumerab
 }
 }
 
+@SuppressWarnings("unused")
 private Result buildHiveResult(EnumerableRelImplementor enumImplementor, 
Prefer pref, OLAPContext context) {
 RelDataType hiveRowType = getRowType();
 

kylin git commit: minor, fix concurrent issue with CubeSegment.cuboidBaseShards

[gaodayue]

Repository: kylin
Updated Branches:
  refs/heads/master 577052bd2 -> 69213e999


minor, fix concurrent issue with CubeSegment.cuboidBaseShards


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/69213e99
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/69213e99
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/69213e99

Branch: refs/heads/master
Commit: 69213e9991a1a69b26803852193a78628246b127
Parents: 577052b
Author: gaodayue 
Authored: Wed Oct 26 21:39:11 2016 +0800
Committer: gaodayue 
Committed: Wed Oct 26 21:39:11 2016 +0800

--
 .../src/main/java/org/apache/kylin/cube/CubeSegment.java  | 7 +--
 1 file changed, 5 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/69213e99/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
index fdf1fb0..b3d6d6b 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
@@ -49,6 +49,8 @@ import 
com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 
+import javax.annotation.concurrent.GuardedBy;
+
 @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = 
Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = 
Visibility.NONE)
 public class CubeSegment implements Comparable, IBuildable, 
ISegment {
 
@@ -115,7 +117,8 @@ public class CubeSegment implements 
Comparable, IBuildable, ISegmen
 @JsonInclude(JsonInclude.Include.NON_EMPTY)
 private Map additionalInfo = new LinkedHashMap();
 
-private volatile Map cuboidBaseShards = 
Maps.newHashMap();//cuboid id ==> base(starting) shard for this cuboid
+@GuardedBy("this")
+private Map cuboidBaseShards = Maps.newHashMap(); // cuboid 
id ==> base(starting) shard for this cuboid
 
 public CubeDesc getCubeDesc() {
 return getCubeInstance().getDescriptor();
@@ -516,7 +519,7 @@ public class CubeSegment implements 
Comparable, IBuildable, ISegmen
 this.totalShards = totalShards;
 }
 
-public short getCuboidBaseShard(Long cuboidId) {
+public synchronized short getCuboidBaseShard(Long cuboidId) {
 if (totalShards > 0) {
 //shard squashed case
 

kylin git commit: KYLIN-2127 UI bug fix for Extend Column

[zhongjian]

Repository: kylin
Updated Branches:
  refs/heads/master 5cdd22523 -> 577052bd2


KYLIN-2127 UI bug fix for Extend Column

Signed-off-by: Jason 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/577052bd
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/577052bd
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/577052bd

Branch: refs/heads/master
Commit: 577052bd2246ec1feb02771f11ca2642a8076c4b
Parents: 5cdd225
Author: kangkaisen 
Authored: Tue Oct 25 22:27:54 2016 +0800
Committer: Jason 
Committed: Wed Oct 26 19:13:45 2016 +0800

--
 webapp/app/js/controllers/cubeMeasures.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/577052bd/webapp/app/js/controllers/cubeMeasures.js
--
diff --git a/webapp/app/js/controllers/cubeMeasures.js 
b/webapp/app/js/controllers/cubeMeasures.js
index dcf26b7..938e918 100644
--- a/webapp/app/js/controllers/cubeMeasures.js
+++ b/webapp/app/js/controllers/cubeMeasures.js
@@ -229,7 +229,7 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, 
$modal,MetaModel,cubes
 $scope.nextPara = {
   "type":"column",
   "value":"",
-  "next_parameter":{}
+  "next_parameter": null
 }
 if($scope.newMeasure){
   $scope.newMeasure.function.parameter.next_parameter = null;

kylin git commit: KYLIN-2127 UI bug fix for Extend Column

[zhongjian]

Repository: kylin
Updated Branches:
  refs/heads/v1.6.0-rc1 772fa541a -> e094b7687


KYLIN-2127 UI bug fix for Extend Column

Signed-off-by: Jason 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e094b768
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e094b768
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e094b768

Branch: refs/heads/v1.6.0-rc1
Commit: e094b76870bbc12ecf0a29e5ff23e4dfce7a0243
Parents: 772fa54
Author: kangkaisen 
Authored: Tue Oct 25 22:27:54 2016 +0800
Committer: Jason 
Committed: Wed Oct 26 18:47:02 2016 +0800

--
 webapp/app/js/controllers/cubeMeasures.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e094b768/webapp/app/js/controllers/cubeMeasures.js
--
diff --git a/webapp/app/js/controllers/cubeMeasures.js 
b/webapp/app/js/controllers/cubeMeasures.js
index dcf26b7..938e918 100644
--- a/webapp/app/js/controllers/cubeMeasures.js
+++ b/webapp/app/js/controllers/cubeMeasures.js
@@ -229,7 +229,7 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, 
$modal,MetaModel,cubes
 $scope.nextPara = {
   "type":"column",
   "value":"",
-  "next_parameter":{}
+  "next_parameter": null
 }
 if($scope.newMeasure){
   $scope.newMeasure.function.parameter.next_parameter = null;

[3/6] kylin git commit: KYLIN-2126 use column name as default dimension name when auto generate dimension for lookup table

[liyang]

KYLIN-2126 use column name as default dimension name when auto generate 
dimension for lookup table


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6350e31f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6350e31f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6350e31f

Branch: refs/heads/KYLIN-1971
Commit: 6350e31f4267f456706f8964c4875af90c170f99
Parents: 92c823b
Author: Jason 
Authored: Tue Oct 25 21:00:01 2016 +0800
Committer: Jason 
Committed: Tue Oct 25 21:00:01 2016 +0800

--
 webapp/app/js/controllers/cubeDimensions.js  |  6 +++---
 webapp/app/partials/cubeDesigner/dimensions.html | 14 +-
 2 files changed, 4 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6350e31f/webapp/app/js/controllers/cubeDimensions.js
--
diff --git a/webapp/app/js/controllers/cubeDimensions.js 
b/webapp/app/js/controllers/cubeDimensions.js
index 5525fe4..ba71c96 100644
--- a/webapp/app/js/controllers/cubeDimensions.js
+++ b/webapp/app/js/controllers/cubeDimensions.js
@@ -96,7 +96,7 @@ KylinApp.controller('CubeDimensionsCtrl', function ($scope, 
$modal,MetaModel,cub
 cols2[k].isLookup = true;
 
 // Default not selected and not disabled.
-lookupSelectAvailable[cols2[k].name] = 
{name:cols2[k].table+"_derived",selected: false, disabled: false};
+lookupSelectAvailable[cols2[k].name] = 
{name:cols2[k].name,selected: false, disabled: false};
 }
 
 $scope.availableColumns[lookups[j].table] = cols2;
@@ -302,7 +302,7 @@ KylinApp.controller('CubeDimensionsCtrl', function ($scope, 
$modal,MetaModel,cub
 if(dim.table==$scope.metaModel.model.fact_table){
$scope.selectedColumns[dim.table][colName] = 
{name:colName,selected: false, disabled: false};
 }else{
-   $scope.selectedColumns[dim.table][colName] = 
{name:dim.table+"_derived",selected: false, disabled: false};
+   $scope.selectedColumns[dim.table][colName] = 
{name:colName,selected: false, disabled: false};
 }
 });
 };
@@ -400,7 +400,7 @@ KylinApp.controller('CubeDimensionsCtrl', function ($scope, 
$modal,MetaModel,cub
  if(table==$scope.metaModel.model.fact_table){
  $scope.selectedColumns[table][name].name=name;
  }else{
- $scope.selectedColumns[table][name].name=table+"_derived";
+ $scope.selectedColumns[table][name].name=name;
  }
 }else{
  if($scope.metaModel.model.fact_table!=table){

http://git-wip-us.apache.org/repos/asf/kylin/blob/6350e31f/webapp/app/partials/cubeDesigner/dimensions.html
--
diff --git a/webapp/app/partials/cubeDesigner/dimensions.html 
b/webapp/app/partials/cubeDesigner/dimensions.html
index 3a2efb6..0c4236e 100644
--- a/webapp/app/partials/cubeDesigner/dimensions.html
+++ b/webapp/app/partials/cubeDesigner/dimensions.html
@@ -276,7 +276,7 @@
   
 
 
-  
+  
 
 
   {{col.name}}
@@ -295,18 +295,6 @@
 
 
 
-
-
-
-
-
-
-
-
-
-
-
-
 
 
 

[2/6] kylin git commit: KYLIN-2082 straming config edit

[liyang]

KYLIN-2082 straming config edit

Signed-off-by: Jason 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/92c823bb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/92c823bb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/92c823bb

Branch: refs/heads/KYLIN-1971
Commit: 92c823bb68c1280ae1ae48d325fa1c678a4c7812
Parents: 85717fe
Author: luguosheng <550175...@qq.com>
Authored: Tue Oct 25 18:49:02 2016 +0800
Committer: Jason 
Committed: Tue Oct 25 20:20:25 2016 +0800

--
 .../partials/cubeDesigner/streamingConfig.html  | 153 ---
 1 file changed, 64 insertions(+), 89 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/92c823bb/webapp/app/partials/cubeDesigner/streamingConfig.html
--
diff --git a/webapp/app/partials/cubeDesigner/streamingConfig.html 
b/webapp/app/partials/cubeDesigner/streamingConfig.html
index 0148ac3..8bdcd25 100644
--- a/webapp/app/partials/cubeDesigner/streamingConfig.html
+++ b/webapp/app/partials/cubeDesigner/streamingConfig.html
@@ -124,176 +124,151 @@
 
   
 
-
   
-  
   
-
+
+
   
-Advanced Setting
+Parser Setting
 
+   ng-class="{'glyphicon-chevron-down': state.isParserHeaderOpen, 
'glyphicon-chevron-right': !state.isParserHeaderOpen}">
   
 
   
 
   
-Timeout
-
+Parser Name
+
   
 
   
-
+
 
-  Kafka timeout is invalid.
+   
ng-show="!form.cube_streaming_form.parserName.$error.required && 
form.cube_streaming_form.parserName.$invalid && 
(form.cube_streaming_form.parserName.$dirty||form.cube_streaming_form.$submitted)">
+  Kafka parser name is invalid.
 
 
-  Kafka timeout is required.
+   
ng-show="form.cube_streaming_form.parserName.$error.required && 
(form.cube_streaming_form.parserName.$dirty||form.cube_streaming_form.$submitted)">
+  Kafka parser name is required.
 
-{{kafkaMeta.timeout}}
+{{kafkaMeta.parserName}}
   
 
   
-
-  
+  
 
   
-Buffer Size
-
+Parser Timestamp Field
+
   
 
-  
-
-
-  Kafka bufferSize is invalid.
-
-
-  Kafka bufferSize is required.
-
-{{kafkaMeta.bufferSize}}
+  
+
+  
+
   
 
   
-
   
 
   
-Margin
-
+Parser Properties
+
   
 
   
-
+
 
-  Kafka margin is invalid.
+   
ng-show="!form.cube_streaming_form.parserProperties.$error.required && 
form.cube_streaming_form.parserProperties.$invalid && 
(form.cube_streaming_form.parserProperties.$dirty||form.cube_streaming_form.$submitted)">
+  Parser properties is invalid.
 
 
-  Kafka margin is required.
+   
ng-show="form.cube_streaming_form.parserProperties.$error.required && 
(form.cube_streaming_form.parserProperties.$dirty||form.cube_streaming_form.$submitted)">
+  Parser properties is required.
 
-{{kafkaMeta.margin}}
+{{kafkaMeta.parserProperties}}
   
 
   
 
   
-
   
+  
   
-
-
+
   
-Parser Setting
+Advanced Setting
 
+   ng-class="{'glyphicon-chevron-down': state.isStreamingAdOpen, 
'glyphicon-chevron-right': !state.isStreamingAdOpen}">
   
 
   
 
   
-Parser Name
-
+Timeout
+
   
 
   
-
+
 
-  Kafka parser name is invalid.
+   
ng-show="!form.cube_streaming_form.timeout.$error.required && 
form.cube_streaming_form.timeout.$invalid && 
(form.cube_streaming_form.timeout.$dirty||form.cube_streaming_form.$submitted)">
+  Kafka 

[4/6] kylin git commit: KYLIN-1321 tips update

[liyang]

KYLIN-1321 tips update


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/151ec3bd
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/151ec3bd
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/151ec3bd

Branch: refs/heads/KYLIN-1971
Commit: 151ec3bd13c290ef250df10cef36ee94f77abaaa
Parents: 6350e31
Author: Jason 
Authored: Tue Oct 25 21:53:07 2016 +0800
Committer: Jason 
Committed: Tue Oct 25 21:53:07 2016 +0800

--
 webapp/app/partials/cubeDesigner/dimensions.html | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/151ec3bd/webapp/app/partials/cubeDesigner/dimensions.html
--
diff --git a/webapp/app/partials/cubeDesigner/dimensions.html 
b/webapp/app/partials/cubeDesigner/dimensions.html
index 0c4236e..d826575 100644
--- a/webapp/app/partials/cubeDesigner/dimensions.html
+++ b/webapp/app/partials/cubeDesigner/dimensions.html
@@ -224,7 +224,7 @@
 
 

[5/6] kylin git commit: KYLIN-1971 ModelChooser, match Model against query joins

[liyang]

KYLIN-1971 ModelChooser, match Model against query joins


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5cdd2252
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5cdd2252
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5cdd2252

Branch: refs/heads/KYLIN-1971
Commit: 5cdd22523a68faa4879092d4b2510ea461609f6e
Parents: 151ec3b
Author: Yang Li 
Authored: Tue Oct 25 06:16:15 2016 +0800
Committer: Li Yang 
Committed: Wed Oct 26 11:06:42 2016 +0800

--
 .../org/apache/kylin/cube/CubeInstance.java |  13 +-
 .../apache/kylin/metadata/model/ColumnDesc.java |   8 +-
 .../kylin/metadata/model/DataModelDesc.java |  13 +-
 .../kylin/metadata/model/FunctionDesc.java  |  18 --
 .../apache/kylin/metadata/model/JoinDesc.java   |  44 -
 .../apache/kylin/metadata/model/TableRef.java   |   4 +-
 .../apache/kylin/metadata/model/TblColRef.java  |  21 ++-
 .../kylin/metadata/project/ProjectL2Cache.java  |   2 +-
 .../metadata/realization/IRealization.java  | 123 ++---
 .../kylin/storage/hybrid/HybridInstance.java|   7 +-
 .../apache/kylin/query/ITCombinationTest.java   |   9 +-
 .../kylin/query/relnode/OLAPTableScan.java  |  16 +-
 .../relnode/OLAPToEnumerableConverter.java  |  13 +-
 .../kylin/query/routing/ModelChooser.java   | 178 +++
 .../apache/kylin/query/routing/QueryRouter.java |   6 +-
 .../rules/RemoveBlackoutRealizationsRule.java   |  18 +-
 16 files changed, 351 insertions(+), 142 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5cdd2252/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 720690d..2ccdde7 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -21,6 +21,7 @@ package org.apache.kylin.cube;
 import java.util.ArrayList;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Set;
 
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.KylinConfigExt;
@@ -52,9 +53,9 @@ import com.google.common.collect.Lists;
 @SuppressWarnings("serial")
 @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = 
Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = 
Visibility.NONE)
 public class CubeInstance extends RootPersistentEntity implements 
IRealization, IBuildable {
-private static final int COST_WEIGHT_MEASURE = 1;
-private static final int COST_WEIGHT_DIMENSION = 10;
-private static final int COST_WEIGHT_INNER_JOIN = 100;
+public static final int COST_WEIGHT_MEASURE = 1;
+public static final int COST_WEIGHT_DIMENSION = 10;
+public static final int COST_WEIGHT_INNER_JOIN = 100;
 
 public static CubeInstance create(String cubeName, CubeDesc cubeDesc) {
 CubeInstance cubeInstance = new CubeInstance();
@@ -374,7 +375,7 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 
 for (LookupDesc lookupDesc : 
this.getDescriptor().getModel().getLookups()) {
 // more tables, more cost
-if ("inner".equals(lookupDesc.getJoin().getType())) {
+if (lookupDesc.getJoin().isInnerJoin()) {
 // inner join cost is bigger than left join, as it will filter 
some records
 calculatedCost += COST_WEIGHT_INNER_JOIN;
 }
@@ -389,8 +390,8 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 }
 
 @Override
-public List getAllColumns() {
-return Lists.newArrayList(getDescriptor().listAllColumns());
+public Set getAllColumns() {
+return getDescriptor().listAllColumns();
 }
 
 @Override

http://git-wip-us.apache.org/repos/asf/kylin/blob/5cdd2252/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
index 772297f..3bf0de9 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
@@ -20,12 +20,11 @@ package org.apache.kylin.metadata.model;
 
 import java.io.Serializable;
 
-import com.fasterxml.jackson.annotation.JsonInclude;
-import org.apache.commons.lang.StringUtils;
 import org.apache.kylin.metadata.datatype.DataType;
 
 

[6/6] kylin git commit: tmp

[liyang]

tmp


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c1e6ff68
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c1e6ff68
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c1e6ff68

Branch: refs/heads/KYLIN-1971
Commit: c1e6ff6807097324a8c303796217a34c37ad08a2
Parents: 5cdd225
Author: Li Yang 
Authored: Wed Oct 26 18:24:15 2016 +0800
Committer: Li Yang 
Committed: Wed Oct 26 18:24:15 2016 +0800

--
 .../java/org/apache/kylin/cube/CubeInstance.java   |  6 ++
 .../java/org/apache/kylin/cube/model/CubeDesc.java | 14 --
 .../org/apache/kylin/metadata/model/JoinDesc.java  |  2 +-
 .../org/apache/kylin/metadata/model/TblColRef.java |  4 ++--
 .../kylin/metadata/realization/IRealization.java   |  3 +++
 .../kylin/storage/hybrid/HybridInstance.java   | 17 +
 .../org/apache/kylin/query/ITKylinQueryTest.java   |  6 +++---
 .../apache/kylin/query/routing/ModelChooser.java   | 12 +++-
 8 files changed, 55 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c1e6ff68/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 2ccdde7..6bd3826 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -28,6 +28,7 @@ import org.apache.kylin.common.KylinConfigExt;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.common.persistence.RootPersistentEntity;
 import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.metadata.model.ColumnDesc;
 import org.apache.kylin.metadata.model.DataModelDesc;
 import org.apache.kylin.metadata.model.IBuildable;
 import org.apache.kylin.metadata.model.LookupDesc;
@@ -395,6 +396,11 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 }
 
 @Override
+public Set getAllColumnDescs() {
+return getDescriptor().listAllColumnDescs();
+}
+
+@Override
 public long getDateRangeStart() {
 List readySegs = getSegments(SegmentStatusEnum.READY);
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/c1e6ff68/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 5c73f21..0c0b4f3 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -54,6 +54,7 @@ import org.apache.kylin.measure.MeasureType;
 import org.apache.kylin.measure.extendedcolumn.ExtendedColumnMeasureType;
 import org.apache.kylin.metadata.MetadataConstants;
 import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.ColumnDesc;
 import org.apache.kylin.metadata.model.DataModelDesc;
 import org.apache.kylin.metadata.model.FunctionDesc;
 import org.apache.kylin.metadata.model.IEngineAware;
@@ -159,8 +160,9 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 @JsonProperty("override_kylin_properties")
 private LinkedHashMap overrideKylinProps = new 
LinkedHashMap();
 
-private LinkedHashSet allColumns = new 
LinkedHashSet();
-private LinkedHashSet dimensionColumns = new 
LinkedHashSet();
+private LinkedHashSet allColumns = new LinkedHashSet<>();
+private LinkedHashSet allColumnDescs = new LinkedHashSet<>();
+private LinkedHashSet dimensionColumns = new LinkedHashSet<>();
 
 private Map derivedToHostMap = Maps.newHashMap();
 private Map hostToDerivedMap = 
Maps.newHashMap();
@@ -191,6 +193,10 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 public Set listAllColumns() {
 return allColumns;
 }
+
+public Set listAllColumnDescs() {
+return allColumnDescs;
+}
 
 /**
  * @return dimension columns including derived, BUT NOT measures
@@ -549,6 +555,10 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 checkState(rowkey.getRowKeyColumns().length == dimCols.size(), "RowKey 
columns count (%d) doesn't match dimensions columns count (%d)", 
rowkey.getRowKeyColumns().length, dimCols.size());
 
 initDictionaryDesc();
+
+for (TblColRef col : allColumns) {
+

[1/6] kylin git commit: KYLIN-2106 integer encoding bug fix [Forced Update!]

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-1971 0f8b2393d -> c1e6ff680 (forced update)


KYLIN-2106 integer encoding bug fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/85717fe3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/85717fe3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/85717fe3

Branch: refs/heads/KYLIN-1971
Commit: 85717fe33712fe1b0f6d78e371aa9194ce5f1d4a
Parents: eb39437
Author: Jason 
Authored: Tue Oct 25 18:39:51 2016 +0800
Committer: Jason 
Committed: Tue Oct 25 20:19:55 2016 +0800

--
 webapp/app/js/controllers/cubeAdvanceSetting.js | 12 ++--
 webapp/app/js/controllers/cubeMeasures.js   | 16 
 2 files changed, 22 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/85717fe3/webapp/app/js/controllers/cubeAdvanceSetting.js
--
diff --git a/webapp/app/js/controllers/cubeAdvanceSetting.js 
b/webapp/app/js/controllers/cubeAdvanceSetting.js
index be90f65..0ba321c 100644
--- a/webapp/app/js/controllers/cubeAdvanceSetting.js
+++ b/webapp/app/js/controllers/cubeAdvanceSetting.js
@@ -27,18 +27,24 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function 
($scope, $modal,cubeConfi
   angular.forEach($scope.cubeMetaFrame.rowkey.rowkey_columns,function(item){
 //var _isDictionaries = item.encoding === "dict"?"true":"false";
 var _isFixedLength = item.encoding.substring(0,12) === 
"fixed_length"?"true":"false";//fixed_length:12
-var _isIntLength = item.encoding.substring(0,3) === 
"int"?"true":"false";//fixed_length:12
+var _isIntegerLength = item.encoding.substring(0,7) === 
"integer"?"true":"false";
+var _isIntLength = item.encoding.substring(0,3) === "int"?"true":"false";
 var _encoding = item.encoding;
 var _valueLength ;
 if(_isFixedLength !=="false"){
   _valueLength = item.encoding.substring(13,item.encoding.length);
   _encoding = "fixed_length";
 }
-if(_isIntLength!="false"){
+if(_isIntLength!="false" && _isIntegerLength=="false" ){
   _valueLength = item.encoding.substring(4,item.encoding.length);
   _encoding = "int";
 }
 
+if(_isIntegerLength!="false" ){
+  _valueLength = item.encoding.substring(8,item.encoding.length);
+  _encoding = "integer";
+}
+
 var rowkeyObj = {
   column:item.column,
   encoding:_encoding,
@@ -66,6 +72,8 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function 
($scope, $modal,cubeConfi
   }
   else if(item.encoding=="int" && item.valueLength){
 encoding = "int:"+item.valueLength;
+  }else if(item.encoding=="integer" && item.valueLength){
+encoding = "integer:"+item.valueLength;
   }else{
 encoding = item.encoding;
   }

http://git-wip-us.apache.org/repos/asf/kylin/blob/85717fe3/webapp/app/js/controllers/cubeMeasures.js
--
diff --git a/webapp/app/js/controllers/cubeMeasures.js 
b/webapp/app/js/controllers/cubeMeasures.js
index 006c6c2..dcf26b7 100644
--- a/webapp/app/js/controllers/cubeMeasures.js
+++ b/webapp/app/js/controllers/cubeMeasures.js
@@ -54,17 +54,24 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, 
$modal,MetaModel,cubes
 var _name=configuration.slice(14);
 var item=$scope.newMeasure.function.configuration[configuration];
 var _isFixedLength = item.substring(0,12) === 
"fixed_length"?"true":"false";//fixed_length:12
-var _isIntLength = item.substring(0,3) === 
"int"?"true":"false";//fixed_length:12
+var _isIntegerLength = item.substring(0,7) === 
"integer"?"true":"false";
+var _isIntLength = item.substring(0,3) === "int"?"true":"false";
 var _encoding = item;
 var _valueLength = 0 ;
 if(_isFixedLength !=="false"){
   _valueLength = item.substring(13,item.length);
   _encoding = "fixed_length";
 }
-if(_isIntLength!="false"){
+if(_isIntLength!="false" && _isIntegerLength=="false" ){
   _valueLength = item.substring(4,item.length);
   _encoding = "int";
 }
+
+if(_isIntegerLength!="false" ){
+  _valueLength = item.substring(8,item.length);
+  _encoding = "integer";
+}
+
 $scope.GroupBy = {
   name:_name,
   encoding:_encoding,
@@ -172,8 +179,9 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, 
$modal,MetaModel,cubes
   if(item.encoding!=="dict" && item.encoding!=="date"&& 
item.encoding!=="time"){
 if(item.encoding=="fixed_length" && item.valueLength){
   encoding = "fixed_length:"+item.valueLength;
-}
-

[4/7] kylin git commit: KYLIN-2030 enhancement

[mahongbin]

KYLIN-2030 enhancement


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/55584c6f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/55584c6f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/55584c6f

Branch: refs/heads/orderedbytes4
Commit: 55584c6fef5051b5ef59ab9d3e50a70c94664290
Parents: 2c29fbb
Author: Hongbin Ma 
Authored: Sat Oct 8 16:14:26 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Oct 26 14:42:48 2016 +0800

--
 .../kylin/cube/CubeCapabilityChecker.java   | 43 +++-
 .../org/apache/kylin/cube/model/CubeDesc.java   |  6 +--
 .../kylin/metadata/model/FunctionDesc.java  |  1 +
 .../test/resources/query/sql_lookup/query06.sql |  1 +
 .../test/resources/query/sql_lookup/query07.sql |  1 +
 5 files changed, 39 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/55584c6f/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
index c8460f0..ee21b1c 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
@@ -25,18 +25,22 @@ import java.util.List;
 import java.util.Set;
 
 import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.measure.MeasureType;
 import org.apache.kylin.measure.basic.BasicMeasureType;
+import org.apache.kylin.metadata.MetadataManager;
 import org.apache.kylin.metadata.filter.UDF.MassInTupleFilter;
+import org.apache.kylin.metadata.model.ColumnDesc;
 import org.apache.kylin.metadata.model.FunctionDesc;
 import org.apache.kylin.metadata.model.IStorageAware;
 import org.apache.kylin.metadata.model.MeasureDesc;
 import org.apache.kylin.metadata.model.ParameterDesc;
+import org.apache.kylin.metadata.model.TableDesc;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.apache.kylin.metadata.realization.CapabilityResult;
-import 
org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence;
 import org.apache.kylin.metadata.realization.SQLDigest;
+import 
org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -66,20 +70,39 @@ public class CubeCapabilityChecker {
 Collection unmatchedAggregations = 
unmatchedAggregations(aggrFunctions, cube);
 
 // try custom measure types
+// in RAW query, unmatchedDimensions and unmatchedAggregations will 
null, so can't chose RAW cube well!
+//if (!unmatchedDimensions.isEmpty() || 
!unmatchedAggregations.isEmpty()) {
 tryCustomMeasureTypes(unmatchedDimensions, unmatchedAggregations, 
digest, cube, result);
+//}
 
-// try dimension-as-measure
-if (!unmatchedAggregations.isEmpty()) {
-if (cube.getDescriptor().getFactTable().equals(digest.factTable)) {
+//more tricks
+if (cube.getDescriptor().getFactTable().equals(digest.factTable)) {
+//for query-on-facttable
+//1. dimension as measure
+
+if (!unmatchedAggregations.isEmpty()) {
 tryDimensionAsMeasures(unmatchedAggregations, digest, cube, 
result, cube.getDescriptor().listDimensionColumnsIncludingDerived());
-} else {
-//deal with query on lookup table, like 
https://issues.apache.org/jira/browse/KYLIN-2030
-if 
(cube.getSegments().get(0).getSnapshots().containsKey(digest.factTable)) {
-Set dimCols = 
Sets.newHashSet(cube.getDataModelDesc().findFirstTable(digest.factTable).getColumns());
+}
+} else {
+//for non query-on-facttable 
+if 
(cube.getSegments().get(0).getSnapshots().containsKey(digest.factTable)) {
+TableDesc tableDesc = 
MetadataManager.getInstance(KylinConfig.getInstanceFromEnv()).getTableDesc(digest.factTable);
+Set dimCols = Sets.newHashSet();
+for (ColumnDesc columnDesc : tableDesc.getColumns()) {
+dimCols.add(columnDesc.getRef());
+}
+
+//1. dimension as measure, like max(cal_dt) or count( distinct 
col) from lookup
+if (!unmatchedAggregations.isEmpty()) {
 tryDimensionAsMeasures(unmatchedAggregations, digest, 
cube, result, dimCols);
-} else {
-logger.info("Skip 

[5/7] kylin git commit: improve integer type upgrade logic

[mahongbin]

improve integer type upgrade logic


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2c29fbb9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2c29fbb9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2c29fbb9

Branch: refs/heads/orderedbytes4
Commit: 2c29fbb953e1b4f458381f17b6fa368b9493aceb
Parents: 2f7ab44
Author: Hongbin Ma 
Authored: Fri Sep 30 18:31:04 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Oct 26 14:42:48 2016 +0800

--
 .../filter/EvaluatableFunctionTupleFilter.java  | 151 ---
 .../metadata/filter/TupleFilterSerializer.java  |  20 ++-
 .../apache/kylin/metadata/model/ColumnDesc.java |  20 ++-
 .../apache/kylin/metadata/tuple/TupleInfo.java  |   4 +-
 .../apache/kylin/query/schema/OLAPTable.java|  31 +++-
 5 files changed, 63 insertions(+), 163 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2c29fbb9/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableFunctionTupleFilter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableFunctionTupleFilter.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableFunctionTupleFilter.java
deleted file mode 100644
index ff24172..000
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableFunctionTupleFilter.java
+++ /dev/null
@@ -1,151 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *  
- * http://www.apache.org/licenses/LICENSE-2.0
- *  
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.metadata.filter;
-
-import java.lang.reflect.InvocationTargetException;
-import java.nio.ByteBuffer;
-import java.util.Collection;
-import java.util.List;
-
-import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.metadata.datatype.DataType;
-import org.apache.kylin.metadata.datatype.StringSerializer;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
-
-import com.google.common.collect.Lists;
-
-public class EvaluatableFunctionTupleFilter extends BuiltInFunctionTupleFilter 
{
-
-private boolean constantsInitted = false;
-
-//about non-like
-private List values;
-private Object tupleValue;
-
-public EvaluatableFunctionTupleFilter(String name) {
-super(name, FilterOperatorEnum.EVAL_FUNC);
-values = Lists.newArrayListWithCapacity(1);
-values.add(null);
-}
-
-@Override
-public boolean evaluate(IEvaluatableTuple tuple, IFilterCodeSystem cs) {
-
-// extract tuple value
-Object tupleValue = null;
-for (TupleFilter filter : this.children) {
-if (!isConstant(filter)) {
-filter.evaluate(tuple, cs);
-tupleValue = filter.getValues().iterator().next();
-}
-}
-
-TblColRef tblColRef = this.getColumn();
-DataType strDataType = DataType.getType("string");
-if (tblColRef.getType() != strDataType) {
-throw new IllegalStateException("Only String type is allow in 
BuiltInFunction");
-}
-ByteArray valueByteArray = (ByteArray) tupleValue;
-StringSerializer serializer = new StringSerializer(strDataType);
-String value = 
serializer.deserialize(ByteBuffer.wrap(valueByteArray.array(), 
valueByteArray.offset(), valueByteArray.length()));
-
-try {
-if (isLikeFunction()) {
-return (Boolean) invokeFunction(value);
-} else {
-this.tupleValue = invokeFunction(value);
-//convert back to ByteArray format because the outer 
EvaluatableFunctionTupleFilter assumes input as ByteArray
-ByteBuffer buffer = 
ByteBuffer.allocate(valueByteArray.length() * 2);
-serializer.serialize((String) this.tupleValue, buffer);
-this.tupleValue = new ByteArray(buffer.array(), 0, 
buffer.position());
-
- 

[7/7] kylin git commit: KYLIN-2030 fix with KYLIN-1971

[mahongbin]

KYLIN-2030 fix with KYLIN-1971


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/601d40a2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/601d40a2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/601d40a2

Branch: refs/heads/orderedbytes4
Commit: 601d40a26d20378153f768c803ed33f367e3a93f
Parents: 662bfe9
Author: Hongbin Ma 
Authored: Wed Oct 26 14:41:06 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Oct 26 15:02:49 2016 +0800

--
 .../org/apache/kylin/cube/CubeCapabilityChecker.java   | 13 +++--
 .../org/apache/kylin/metadata/model/ColumnDesc.java|  8 +---
 2 files changed, 4 insertions(+), 17 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/601d40a2/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
index e509d98..a3c89e5 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
@@ -25,22 +25,18 @@ import java.util.List;
 import java.util.Set;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.measure.MeasureType;
 import org.apache.kylin.measure.basic.BasicMeasureType;
-import org.apache.kylin.metadata.MetadataManager;
 import org.apache.kylin.metadata.filter.UDF.MassInTupleFilter;
-import org.apache.kylin.metadata.model.ColumnDesc;
 import org.apache.kylin.metadata.model.FunctionDesc;
 import org.apache.kylin.metadata.model.IStorageAware;
 import org.apache.kylin.metadata.model.MeasureDesc;
 import org.apache.kylin.metadata.model.ParameterDesc;
-import org.apache.kylin.metadata.model.TableDesc;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.apache.kylin.metadata.realization.CapabilityResult;
-import org.apache.kylin.metadata.realization.SQLDigest;
 import 
org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence;
+import org.apache.kylin.metadata.realization.SQLDigest;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -86,11 +82,8 @@ public class CubeCapabilityChecker {
 } else {
 //for non query-on-facttable 
 if 
(cube.getSegments().get(0).getSnapshots().containsKey(digest.factTable)) {
-TableDesc tableDesc = 
MetadataManager.getInstance(KylinConfig.getInstanceFromEnv()).getTableDesc(digest.factTable);
-Set dimCols = Sets.newHashSet();
-for (ColumnDesc columnDesc : tableDesc.getColumns()) {
-dimCols.add(columnDesc.getRef());
-}
+
+Set dimCols = 
Sets.newHashSet(cube.getDataModelDesc().findFirstTable(digest.factTable).getColumns());
 
 //1. all aggregations on lookup table can be done. For 
distinct count, mark them all DimensionAsMeasures
 // so that the measure has a chance to be upgraded to 
DimCountDistinctMeasureType in 
org.apache.kylin.metadata.model.FunctionDesc#reInitMeasureType

http://git-wip-us.apache.org/repos/asf/kylin/blob/601d40a2/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
index a13bd37..e0184b4 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
@@ -26,7 +26,6 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect;
 import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
 
 /**
  * Column Metadata from Source. All name should be uppercase.
@@ -203,11 +202,6 @@ public class ColumnDesc implements Serializable {
 
 @Override
 public String toString() {
-return "ColumnDesc{" +
-"id='" + id + '\'' +
-", name='" + name + '\'' +
-", datatype='" + datatype + '\'' +
-", comment='" + comment + '\'' +
-'}';
+return "ColumnDesc{" + "id='" + id + '\'' + ", name='" + name + '\'' + 
", datatype='" + datatype + '\'' + ", comment='" + comment + '\'' + '}';

[2/7] kylin git commit: port KYLIN-2068 to BeelineHiveClient(KYLIN-2125)

[mahongbin]

port KYLIN-2068 to BeelineHiveClient(KYLIN-2125)


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/313c47bf
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/313c47bf
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/313c47bf

Branch: refs/heads/orderedbytes4
Commit: 313c47bfba676dad1caca746ba6f34dbb07d8862
Parents: 92a2f1d
Author: Hongbin Ma 
Authored: Wed Oct 26 13:37:03 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Oct 26 14:05:34 2016 +0800

--
 .../org/apache/kylin/source/hive/BeelineHiveClient.java   | 10 +-
 .../java/org/apache/kylin/source/hive/CLIHiveClient.java  |  4 ++--
 .../apache/kylin/source/hive/HiveSourceTableLoader.java   |  1 +
 .../java/org/apache/kylin/source/hive/HiveTableMeta.java  |  6 --
 4 files changed, 12 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/313c47bf/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
index 0fbc39b..c8d56a5 100644
--- 
a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
+++ 
b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
@@ -115,7 +115,7 @@ public class BeelineHiveClient implements IHiveClient {
 
 List allColumns = 
Lists.newArrayList();
 while (columns.next()) {
-allColumns.add(new 
HiveTableMeta.HiveTableColumnMeta(columns.getString(4), columns.getString(6)));
+allColumns.add(new 
HiveTableMeta.HiveTableColumnMeta(columns.getString(4), columns.getString(6), 
columns.getString(12)));
 }
 builder.setAllColumns(allColumns);
 
@@ -139,7 +139,7 @@ public class BeelineHiveClient implements IHiveClient {
 if ("".equals(resultSet.getString(1).trim())) {
 break;
 }
-partitionColumns.add(new 
HiveTableMeta.HiveTableColumnMeta(resultSet.getString(1).trim(), 
resultSet.getString(2).trim()));
+partitionColumns.add(new 
HiveTableMeta.HiveTableColumnMeta(resultSet.getString(1).trim(), 
resultSet.getString(2).trim(), resultSet.getString(3).trim()));
 }
 builder.setPartitionColumns(partitionColumns);
 }
@@ -205,9 +205,9 @@ public class BeelineHiveClient implements IHiveClient {
 
 public static void main(String[] args) throws SQLException {
 
-//BeelineHiveClient loader = new BeelineHiveClient("-n root --hiveconf 
hive.security.authorization.sqlstd.confwhitelist.append='mapreduce.job.*|dfs.*' 
-u 'jdbc:hive2://sandbox:1'");
-BeelineHiveClient loader = new 
BeelineHiveClient(StringUtils.join(args, " "));
-HiveTableMeta hiveTableMeta = loader.getHiveTableMeta("default", 
"events");
+BeelineHiveClient loader = new BeelineHiveClient("-n root --hiveconf 
hive.security.authorization.sqlstd.confwhitelist.append='mapreduce.job.*|dfs.*' 
-u 'jdbc:hive2://sandbox:1'");
+//BeelineHiveClient loader = new 
BeelineHiveClient(StringUtils.join(args, " "));
+HiveTableMeta hiveTableMeta = loader.getHiveTableMeta("default", 
"test001");
 System.out.println(hiveTableMeta);
 loader.close();
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/313c47bf/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
index ea74470..60cf47a 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
@@ -89,11 +89,11 @@ public class CLIHiveClient implements IHiveClient {
 List allColumns = 
Lists.newArrayList();
 List partitionColumns = 
Lists.newArrayList();
 for (FieldSchema fieldSchema : allFields) {
-allColumns.add(new 
HiveTableMeta.HiveTableColumnMeta(fieldSchema.getName(), 
fieldSchema.getType()));
+allColumns.add(new 
HiveTableMeta.HiveTableColumnMeta(fieldSchema.getName(), fieldSchema.getType(), 
fieldSchema.getComment()));
 }
 if (partitionFields != null && partitionFields.size() > 0) {
 for (FieldSchema fieldSchema : partitionFields) {
-partitionColumns.add(new 
HiveTableMeta.HiveTableColumnMeta(fieldSchema.getName(), 

[3/7] kylin git commit: KYLIN-2125 add BeelineHiveClient

[mahongbin]

KYLIN-2125 add BeelineHiveClient


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/92a2f1dd
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/92a2f1dd
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/92a2f1dd

Branch: refs/heads/orderedbytes4
Commit: 92a2f1dd66a10f7408c1159f256d236c45ba40ca
Parents: 5cdd225
Author: Hongbin Ma 
Authored: Thu Oct 13 15:31:11 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Oct 26 14:05:34 2016 +0800

--
 .../kylin/rest/controller/TableController.java  |   9 +-
 source-hive/pom.xml |   5 +
 .../kylin/source/hive/BeelineHiveClient.java| 214 +++
 .../source/hive/BeelineOptionsProcessor.java|  47 
 .../apache/kylin/source/hive/CLIHiveClient.java | 169 +++
 .../apache/kylin/source/hive/HiveClient.java| 170 ---
 .../kylin/source/hive/HiveClientFactory.java|  33 +++
 .../source/hive/HiveSourceTableLoader.java  |  89 
 .../org/apache/kylin/source/hive/HiveTable.java |  16 +-
 .../apache/kylin/source/hive/HiveTableMeta.java |  71 ++
 .../kylin/source/hive/HiveTableMetaBuilder.java | 102 +
 .../apache/kylin/source/hive/HqlExecutable.java | 107 --
 .../apache/kylin/source/hive/IHiveClient.java   |  36 
 .../hive/BeelineOptionsProcessorTest.java   |  38 
 14 files changed, 769 insertions(+), 337 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/92a2f1dd/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
index d9050c1..47ff3fe 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
@@ -47,7 +47,8 @@ import org.apache.kylin.rest.service.KafkaConfigService;
 import org.apache.kylin.rest.service.ModelService;
 import org.apache.kylin.rest.service.ProjectService;
 import org.apache.kylin.rest.service.StreamingService;
-import org.apache.kylin.source.hive.HiveClient;
+import org.apache.kylin.source.hive.HiveClientFactory;
+import org.apache.kylin.source.hive.IHiveClient;
 import org.apache.kylin.source.kafka.config.KafkaConfig;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -189,7 +190,7 @@ public class TableController extends BasicController {
 String[] dbTableName = HadoopUtil.parseHiveTableName(tableName);
 tableName = dbTableName[0] + "." + dbTableName[1];
 TableDesc desc = 
cubeMgmtService.getMetadataManager().getTableDesc(tableName);
-if(desc == null)
+if (desc == null)
 return false;
 tableType = desc.getSourceType();
 
@@ -315,7 +316,7 @@ public class TableController extends BasicController {
 @RequestMapping(value = "/hive", method = { RequestMethod.GET })
 @ResponseBody
 private static List showHiveDatabases() throws IOException {
-HiveClient hiveClient = new HiveClient();
+IHiveClient hiveClient = HiveClientFactory.getHiveClient();
 List results = null;
 
 try {
@@ -336,7 +337,7 @@ public class TableController extends BasicController {
 @RequestMapping(value = "/hive/{database}", method = { RequestMethod.GET })
 @ResponseBody
 private static List showHiveTables(@PathVariable String database) 
throws IOException {
-HiveClient hiveClient = new HiveClient();
+IHiveClient hiveClient = HiveClientFactory.getHiveClient();
 List results = null;
 
 try {

http://git-wip-us.apache.org/repos/asf/kylin/blob/92a2f1dd/source-hive/pom.xml
--
diff --git a/source-hive/pom.xml b/source-hive/pom.xml
index 08019d0..16cb3b4 100644
--- a/source-hive/pom.xml
+++ b/source-hive/pom.xml
@@ -65,6 +65,11 @@
 provided
 
 
+org.apache.hive
+hive-jdbc
+provided
+
+
 org.apache.mrunit
 mrunit
 hadoop2

http://git-wip-us.apache.org/repos/asf/kylin/blob/92a2f1dd/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
new file mode 100644
index 000..0fbc39b
--- /dev/null
+++ 

[6/7] kylin git commit: KYLIN-2030 bug fix

[mahongbin]

KYLIN-2030 bug fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/662bfe91
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/662bfe91
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/662bfe91

Branch: refs/heads/orderedbytes4
Commit: 662bfe918e0fa2f3f195acfda815330dbcb32f21
Parents: 55584c6
Author: Hongbin Ma 
Authored: Sun Oct 9 19:07:52 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Oct 26 14:42:48 2016 +0800

--
 .../kylin/cube/CubeCapabilityChecker.java   | 23 ++--
 .../kylin/query/relnode/OLAPAggregateRel.java   | 22 +++
 2 files changed, 25 insertions(+), 20 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/662bfe91/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
index ee21b1c..e509d98 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
@@ -81,7 +81,7 @@ public class CubeCapabilityChecker {
 //1. dimension as measure
 
 if (!unmatchedAggregations.isEmpty()) {
-tryDimensionAsMeasures(unmatchedAggregations, digest, cube, 
result, cube.getDescriptor().listDimensionColumnsIncludingDerived());
+tryDimensionAsMeasures(unmatchedAggregations, result, 
cube.getDescriptor().listDimensionColumnsIncludingDerived());
 }
 } else {
 //for non query-on-facttable 
@@ -92,10 +92,18 @@ public class CubeCapabilityChecker {
 dimCols.add(columnDesc.getRef());
 }
 
-//1. dimension as measure, like max(cal_dt) or count( distinct 
col) from lookup
+//1. all aggregations on lookup table can be done. For 
distinct count, mark them all DimensionAsMeasures
+// so that the measure has a chance to be upgraded to 
DimCountDistinctMeasureType in 
org.apache.kylin.metadata.model.FunctionDesc#reInitMeasureType
 if (!unmatchedAggregations.isEmpty()) {
-tryDimensionAsMeasures(unmatchedAggregations, digest, 
cube, result, dimCols);
+Iterator itr = 
unmatchedAggregations.iterator();
+while (itr.hasNext()) {
+FunctionDesc functionDesc = itr.next();
+if 
(dimCols.containsAll(functionDesc.getParameter().getColRefs())) {
+itr.remove();
+}
+}
 }
+tryDimensionAsMeasures(Lists.newArrayList(aggrFunctions), 
result, dimCols);
 
 //2. more "dimensions" contributed by snapshot
 if (!unmatchedDimensions.isEmpty()) {
@@ -159,19 +167,12 @@ public class CubeCapabilityChecker {
 return result;
 }
 
-private static void tryDimensionAsMeasures(Collection 
unmatchedAggregations, SQLDigest digest, CubeInstance cube, CapabilityResult 
result, Set dimCols) {
-CubeDesc cubeDesc = cube.getDescriptor();
-Collection cubeFuncs = cubeDesc.listAllFunctions();
+private static void tryDimensionAsMeasures(Collection 
unmatchedAggregations, CapabilityResult result, Set dimCols) {
 
 Iterator it = unmatchedAggregations.iterator();
 while (it.hasNext()) {
 FunctionDesc functionDesc = it.next();
 
-if (cubeFuncs.contains(functionDesc)) {
-it.remove();
-continue;
-}
-
 // let calcite handle count
 if (functionDesc.isCount()) {
 it.remove();

http://git-wip-us.apache.org/repos/asf/kylin/blob/662bfe91/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java 
b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
index d73206a..62351d3 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
@@ -296,16 +296,20 @@ public class OLAPAggregateRel extends Aggregate 
implements OLAPRel {
 }
 
 private void translateAggregation() {
-// now the realization is known, replace aggregations with what's 
defined on MeasureDesc
-List measures = this.context.realization.getMeasures();
-List newAggrs = 

[1/7] kylin git commit: port KYLIN-2012 to new interface introduced in KYLIN-2125

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/orderedbytes4 [created] 601d40a26


port KYLIN-2012 to new interface introduced in KYLIN-2125


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2f7ab44d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2f7ab44d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2f7ab44d

Branch: refs/heads/orderedbytes4
Commit: 2f7ab44d641d73b333c7e05bc27e1baa88b911c4
Parents: 313c47b
Author: Hongbin Ma 
Authored: Wed Oct 26 14:04:56 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Oct 26 14:05:34 2016 +0800

--
 .../java/org/apache/kylin/job/DeployUtil.java   |  5 ++-
 .../kylin/source/hive/BeelineHiveClient.java|  2 +-
 .../source/hive/HiveSourceTableLoader.java  | 32 +++
 .../apache/kylin/source/hive/SchemaChecker.java | 41 
 4 files changed, 36 insertions(+), 44 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2f7ab44d/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
--
diff --git a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java 
b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
index 4a24ad2..54feb24 100644
--- a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
+++ b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
@@ -45,8 +45,9 @@ import org.apache.kylin.metadata.model.ColumnDesc;
 import org.apache.kylin.metadata.model.TableDesc;
 import org.apache.kylin.metadata.model.TableRef;
 import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.source.hive.HiveClient;
+import org.apache.kylin.source.hive.HiveClientFactory;
 import org.apache.kylin.source.hive.HiveCmdBuilder;
+import org.apache.kylin.source.hive.IHiveClient;
 import org.apache.kylin.source.kafka.TimedJsonStreamParser;
 import org.apache.maven.model.Model;
 import org.apache.maven.model.io.xpp3.MavenXpp3Reader;
@@ -234,7 +235,7 @@ public class DeployUtil {
 String tableFileDir = temp.getParent();
 temp.delete();
 
-HiveClient hiveClient = new HiveClient();
+IHiveClient hiveClient = HiveClientFactory.getHiveClient();
 // create hive tables
 hiveClient.executeHQL("CREATE DATABASE IF NOT EXISTS EDW");
 
hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_CAL_DT.toUpperCase(;

http://git-wip-us.apache.org/repos/asf/kylin/blob/2f7ab44d/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
index c8d56a5..a84aeb1 100644
--- 
a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
+++ 
b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
@@ -207,7 +207,7 @@ public class BeelineHiveClient implements IHiveClient {
 
 BeelineHiveClient loader = new BeelineHiveClient("-n root --hiveconf 
hive.security.authorization.sqlstd.confwhitelist.append='mapreduce.job.*|dfs.*' 
-u 'jdbc:hive2://sandbox:1'");
 //BeelineHiveClient loader = new 
BeelineHiveClient(StringUtils.join(args, " "));
-HiveTableMeta hiveTableMeta = loader.getHiveTableMeta("default", 
"test001");
+HiveTableMeta hiveTableMeta = loader.getHiveTableMeta("default", 
"test_kylin_fact_part");
 System.out.println(hiveTableMeta);
 loader.close();
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/2f7ab44d/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
 
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
index 388e72b..401e720 100644
--- 
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
+++ 
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
@@ -26,6 +26,7 @@ import java.util.Set;
 import java.util.UUID;
 
 import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.cube.CubeManager;
 import org.apache.kylin.engine.mr.HadoopUtil;
 import org.apache.kylin.metadata.MetadataConstants;
 import org.apache.kylin.metadata.MetadataManager;
@@ -34,8 +35,10 @@ import org.apache.kylin.metadata.model.TableDesc;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.common.collect.LinkedHashMultimap;
 import com.google.common.collect.Lists;
 

kylin git commit: update download page and tutorial for streaming

[shaofengshi]

Repository: kylin
Updated Branches:
  refs/heads/document 59913167a -> 1f3a45faa


update download page and tutorial for streaming


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1f3a45fa
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1f3a45fa
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1f3a45fa

Branch: refs/heads/document
Commit: 1f3a45faa0e859ad76f456b3a4d68faf0f1cecac
Parents: 5991316
Author: shaofengshi 
Authored: Wed Oct 26 15:04:34 2016 +0800
Committer: shaofengshi 
Committed: Wed Oct 26 15:04:34 2016 +0800

--
 website/KEYS| 304 +++
 website/_docs15/index.md|   1 +
 website/_docs16/tutorial/cube_streaming.md  |   6 -
 .../_posts/blog/2016-10-18-new-nrt-streaming.md |   6 +-
 website/download/index.cn.md|  65 ++--
 website/download/index.md   |  50 ++-
 6 files changed, 355 insertions(+), 77 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1f3a45fa/website/KEYS
--
diff --git a/website/KEYS b/website/KEYS
new file mode 100644
index 000..a9a123d
--- /dev/null
+++ b/website/KEYS
@@ -0,0 +1,304 @@
+This file contains the PGP keys of various developers.
+Please don't use them for email unless you have to. Their main
+purpose is code signing.
+
+Examples of importing this file in your keystore:
+ gpg --import KEYS.txt
+ (need pgp and other examples here)
+
+Examples of adding your key to this file:
+ pgp -kxa  and append it to this file.
+ (pgpk -ll  && pgpk -xa ) >> this file.
+ (gpg --list-sigs 
+ && gpg --armor --export ) >> this file.
+
+---
+
+pub   4096R/027DC364 2015-04-10
+uid  Luke Han 
+sig 3027DC364 2015-04-10  Luke Han 
+sub   4096R/19BCFD87 2015-04-10
+sig  027DC364 2015-04-10  Luke Han 
+
+-BEGIN PGP PUBLIC KEY BLOCK-
+Version: GnuPG v1
+
+mQINBFUno+8BEACxRnKkHFkA/qbYKyidAsMgvgsZZIwONlAoQE+01MKsGPbJU/DR
+zVyzD5nFskaoZ3dLTeQfKP/BJ9qfXpm0zlS4PPHkrbKOeV+SawiiOvSLYCijz6vU
+T1lbgW3yIpMkFp0zpIaZo7BpA2Q9s0qvPeZrDenu1X4IeNrUHs2QInYFDrUpyghw
+BOrt4Yrx7KmtWDPia/Hvbd5nqy9coHZO+HYzBJhXPXK5arbrCDF+NEJFcUI3VlP1
+mPF0LG/SfeVtsjPVo+Ub9HIMlwd6NnzIveSXc5FWIaweQ5ko38TGgKLJ9UXkH5C+
+c/sPOaKVAYIcX5i94BLigjxAQAg15kDr1dX7gPnjQNZdLjMRWiTf4eSN5+rWK+j+
+IeA8QIcVGz7fmhyZ8J/9Gr9TdERvMuhoV/ll8aCGAQU5ApLfgVMIK8kW1vr8dFDh
+x0tnrFZIPcNRg4K9ISR/DEFxYvSkXlxaBudIJiM5itgJ2dgltIIUX5uD8h2gi94A
+4wJfI9m1F2v84lDjv6ve+O3O/P3OPErAKaKjn8w/PHFmGfVAGLkRIV/GQx39gDOf
+QFMoGhqOdLeSIWLodg73HtpZAlVEVZ0+50cHtKqWfQ357Lw3aRJdO/WWrcDZQxut
+lTiULde+vxgwWExpZf98HbpcwFNckA6+m0bwCCc9GEXkDUbPbuvpPvjJOQARAQAB
+tB1MdWtlIEhhbiA8bHVrZWhhbkBhcGFjaGUub3JnPokCOAQTAQIAIgUCVSej7wIb
+AwYLCQgHAwIGFQgCCQoLBBYCAwECHgECF4AACgkQoRMvEgJ9w2RoDhAArSo9kHlQ
+e84hP//+4VSccubHudPTX/+GdDcrAa0+QVS+lhnqe47sLsrvY2Hslm77AIZnYDOk
+U76FpKkK4AcMjP+ayLT6KI/ds4k9ca3OvIgj9TYtFczOg2w1dq0sTKx5MXPXJpDL
+o/Ee7Vuk10rKK9796B8keUHfFvexgfIFW39LaIOG0US5sx7IYNMIbpzEknl9dPi/
+/Mxa0+TTbeh+Jj+SkeGOV9SlcT3HWX+oXK3+saV5ChQHR4Db+wh0vhfgixi0YMbJ
+iik0aPeATiLWvccpS3t5mBE5jP+Ss1xeptXaAmXJl9QC3J5lnIMZPhI7ie83agO8
+ZRcffUMfWw/4HJbB6kj9Cs82zIT8Qfz+w+3PyoLvU65IHmY8FxknQX2jg9XFMZ+C
+c9NgJQyAOl2watglEJytjZ9UMfg36Zg0NYUmK+TMJzuMjtPbXgS7EyboMju9cdbS
+1U4AStGFHUUcnw9r7jP9O9w7DtIjo9fdIDsfHkuNf/RMbZ4xAXWF/tbk++NpIjSY
+qBXD2ZRUg+E6QfDvXOabpJ/14cVoYUlcFYeN9MpgDAKh94xmsyyxpnnyCWI3WTBj
+aMT/p7xCCvYJ5XmyyV4zRHpQ/tHuZ/XnNfkeJavCxMXuslf5AyBP64uRB00EksDf
+eMShtxE026rc9XyGf/pb7Ofg1zpbGl+Fhca5Ag0EVSej7wEQAKykheo7Hpq+m5Pl
+NFGPp3Y8kN5hV5XoJLvKgbF4UnSB8/4IXRyLfTO+5F1ePNRiRBGtWAseWKtsN/qj
+dgkc103Ckhl3NX/bH7o7mXsyOUThxAFCz/hzR1ca20H3lv63fB1L6wUD24aZ+m+9
+xf3RqiPNC/OYEvIN0NKeFNzpWWGNTaYcHReKiwwT8VpL0HzI8iu7oJI3Z/8Cmugk
+N39FiR8lJTQspmRzUJerQHqB2E6M0B252DTD7syZZYHOgP56A9V9KsMApQ4mC/qz
+ntdc0wTwLdAejrNs+qaSuIxSXhW9kPPJbazsDnvqEI3oAIU/4Zo+xt/QZpENgZTO
+xEnzIoxby75Qq54vCZSxnVsjJfJv4ZB3DIq0bwEeOK75gcyGAShnxDCzdD0rkvwj
+oyqiGu15bRKW+AiJAll2qs7PCV7sEUFx9pvz9QOx2JYyNKY246koe0j5nUDPY5uU
+NnFdq2x+LpmuT3hZ/ptEhXzgQUEyw0LFGz8sMtkZ8Vk0gbJnv1nVqYVDjfcv9Ggy
+DePBFkIpp/jWEVPWhlgQwYouNHF+W5Rs/r7eKpIwAQMw+aOcmn7BCxLkznZCvodG
+373kIjy5to+hg0tIWiVcirGdCh1ktPxeRrvTkoFd001x8GU+z+E6JGYrIIuTi4JE
+/cEqaV+wuKKqr2cFxxql9RXGYUWhABEBAAGJAh8EGAECAAkFAlUno+8CGwwACgkQ
+oRMvEgJ9w2R2VQ/9FGp/O1F8TYITJWAyOUQONdk8pZriAk+9Oon9DkxhsYdDD1oB
+td43wvFLr5dC7j9wvbnmYDWTlAwbeZYfZSS38YlrAH8/jwsuEtz6ez2faqb4qAz6
+OSGuJv3sp1T4fFUpd/O5KyGej90fmoFygSD7PdY9OEPKCHZpJuVqYMA+IaU2QXyL
+giX4jspR5SJVaJJ+kyB7+IVIVwMqxhDQ0PU4v+7hx+kVyoL7f26P+SvMkPYzM2/w
+HQrWMvuVT1NhPclk9wxdOJL1WJLNoR5U/wtZ/DBIV2gu457AyzuopHB8ye55J3q3

svn commit: r16698 - /dev/kylin/apache-kylin-1.6.0-rc1/

[shaofengshi]

Author: shaofengshi
Date: Wed Oct 26 06:30:14 2016
New Revision: 16698

Log:
add asc and md5 files

Added:
dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-SNAPSHOT-bin.tar.gz.asc
dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-SNAPSHOT-bin.tar.gz.md5

dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-cdh5.7-SNAPSHOT-bin.tar.gz.asc

dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-cdh5.7-SNAPSHOT-bin.tar.gz.md5

dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-hbase1.x-SNAPSHOT-bin.tar.gz.asc

dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-hbase1.x-SNAPSHOT-bin.tar.gz.md5

Added: 
dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-SNAPSHOT-bin.tar.gz.asc
==
--- dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-SNAPSHOT-bin.tar.gz.asc 
(added)
+++ dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-SNAPSHOT-bin.tar.gz.asc 
Wed Oct 26 06:30:14 2016
@@ -0,0 +1,17 @@
+-BEGIN PGP SIGNATURE-
+Version: GnuPG v2.0.14 (GNU/Linux)
+
+iQIcBAABAgAGBQJYEEjZAAoJEKvJ+IVvz0cg3I4P/A2wNtgmcEW7kyP3sWMAb6zE
+T5L95rC+v89yRd7V1n1OLUKPIPPB5PjGSQiahtZKwoJ3nNpeX8B8LWY5rmY6XkwC
+oN3t6KshrWJeX7ADdh7n3CMelpeG8lho35VHCo61y13VA9mxd+AGUQQ5KLgkSUP7
+tl9c46Ub9TEDVhH4rDXDfY5NlH9Im8Xww3iR6BJH43PCzD6/ODL6QnC86N6aLlox
+0dobwA0Zz1/lD7ajv689V+3JpPMJSsKhiZ34lFcehqGhp0c8naLoBY9dE2WX+YWR
+vZHECiuS6i9B0bhwLAQx9e1wO+IZ0QG7jrxuPf/MW684Epv6TVtQlgEpqZUIPSOv
+Mu7aqIgG6smKWT8UqLbNsC0rJT9d5MtdU9ey5gtWYDAWizuqAJN8ryVAN7Sv2kCc
+EAHxgdPB0Wtle/DvpDYSLvtTo+zH2/uLx79kK0utYELt4BiRyHQH2puTeHTA2oSq
+XZ0/p2yppYKCgQ8KKS6dQn00PiO2b/uxoyB8Gi5H7NNfe3YXkjEwFANzhywfMxsT
+bPHXEkYmfXEa2o5hyu1kgv/ZuCTmQiA7cFF+Xs4CdWVWgxFiDi9LsJsI71onvsBJ
+OBQwVJwdeINpuIIp2caGSOP6+a0Be3BjsUQE5o/XJh02mR61gFP5o9ALLRAcmejN
+5/f476VqWJqJjIOag7GA
+=uVIx
+-END PGP SIGNATURE-

Added: 
dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-SNAPSHOT-bin.tar.gz.md5
==
--- dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-SNAPSHOT-bin.tar.gz.md5 
(added)
+++ dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-SNAPSHOT-bin.tar.gz.md5 
Wed Oct 26 06:30:14 2016
@@ -0,0 +1 @@
+88e69c6a7be3658da63b92d17bdcb74c  -

Added: 
dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-cdh5.7-SNAPSHOT-bin.tar.gz.asc
==
--- 
dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-cdh5.7-SNAPSHOT-bin.tar.gz.asc
 (added)
+++ 
dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-cdh5.7-SNAPSHOT-bin.tar.gz.asc
 Wed Oct 26 06:30:14 2016
@@ -0,0 +1,17 @@
+-BEGIN PGP SIGNATURE-
+Version: GnuPG v2.0.14 (GNU/Linux)
+
+iQIcBAABAgAGBQJYEElDAAoJEKvJ+IVvz0cgNuwP/0BaSTB2Ta0CIThT5ZW4fuk/
+81u9YdM6WF2kO98nTUcb9K9iUgzb2Cxj9ecc4ox0AYc48pEb6GVESzZv0TVAWPzB
+6Vato8rBgTQzzmCRNL5+WE3+DnJl2l07n19ISCTDoD3jU/OTfzLvre0yEA3CXTZB
+Sr9vnHpVdDWkfo0t6uE2iohServkGhGkjr0fyqpw5mshou/u3wvvcK2K2LQsjXtm
+apM1EykyPE0EgIdRprpoWdfVGyd9h169/xN/YEnH45BzaNNRmP3eWEmOlAg2Z+3O
+9WdXS1K4acymmWgV9iXeILkGJh6E2AI1dbmM0qa4PKP4rlRxMOMSXPrk6rJBtXPM
+smULzzbWXgO+7V8azsBl6fUA19MGkHBNIPL91DcQwhyRzTNjFMoAbCIg7xVFG/j1
+PFgv8WF41mtQepl70SllQ5OBmq3DhljEkah0J3KLCPnK9uxoFDODFiDGxvQPPrEF
+oJULakDIFahLB4LSELMYLiEG3UXXyrfvY3//zWKCxbvm0WG31s82sSTS/j8M2pJA
+LAxd7CQQddVhSSXk9HVQTkVxeFb9fVNuIhhrU/Dz6vAahmIXxfUhthV3zUue2Ix7
+pwR4cqQovMnf4ZBKEOtA98/bnkUVwKiSYdIq4oFT8SjJsdpf3rY4JdnVNEOQewho
+eC56aumW1dY7Z/1zIsA3
+=6BS2
+-END PGP SIGNATURE-

Added: 
dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-cdh5.7-SNAPSHOT-bin.tar.gz.md5
==
--- 
dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-cdh5.7-SNAPSHOT-bin.tar.gz.md5
 (added)
+++ 
dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-cdh5.7-SNAPSHOT-bin.tar.gz.md5
 Wed Oct 26 06:30:14 2016
@@ -0,0 +1 @@
+060e8d515efbc19902c73e0373d1930a  -

Added: 
dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-hbase1.x-SNAPSHOT-bin.tar.gz.asc
==
--- 
dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-hbase1.x-SNAPSHOT-bin.tar.gz.asc
 (added)
+++ 
dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-hbase1.x-SNAPSHOT-bin.tar.gz.asc
 Wed Oct 26 06:30:14 2016
@@ -0,0 +1,17 @@
+-BEGIN PGP SIGNATURE-
+Version: GnuPG v2.0.14 (GNU/Linux)
+
+iQIcBAABAgAGBQJYEEkgAAoJEKvJ+IVvz0cg7kgP/AyHcOWqiBM9fGZOZIC9DDaQ
+NaTiXfZmIr2hDWoDblyBHWNYNQpN6Zsijr4LpyFqAZFbv504+H9rhtbK9kh22XNX
+ZJebj0mvDk6fjYVKulyz/04S6N56Cy8MAEpuVuqGKC8wS3qH5g0nOIu5ASm2b0l+
+1fSVujf4km6qIyk0HmaWhmOz1H1++Gc0LFr7mhOM9k2JRWJvfiawdNebfkyEXgs/
+cbqoxPTrmGvg9oGu7kBYNGXQ8C+jleIfhneQISKiLQm0aspseloVxAAB1FWCaL+2
+wFJVmYQdpC6cOsLfll1TCC5XSSgXNCsdc/KeywjCl2lrTwmdS9QSXj1c27Ad+UOU
+nSEJBlYZwOMSYJvtweUMTtd/YUoqyiZjnnsptClfmYiVCalRsqG0BrL/I3a2cguZ
+/zg/2l+6PW3yQJZBB+7v/rfQsKSvCKA99a05MiLR3R62ecveUofjop42AFQbXWix